L3 cross connect

- all packets input on interface X are load-balanced over the set of
paths provided.

Change-Id: Ic27cb88c4cd5d6d3462570632daff7a43d5a652d
Signed-off-by: Neale Ranns <nranns@cisco.com>

10 files changed, 1260 insertions, 4 deletions

diff --git a/src/plugins/l3xc/CMakeLists.txt b/src/plugins/l3xc/CMakeLists.txt
new file mode 100644
index 00000000000..82ea08404d9
--- /dev/null
+++ b/src/plugins/l3xc/CMakeLists.txt
@@ -0,0 +1,26 @@
+# Copyright (c) 2018 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_vpp_plugin(l3xc
+  SOURCES
+  l3xc.c
+  l3xc_api.c
+  l3xc_node.c
+
+  API_FILES
+  l3xc.api
+
+  INSTALL_HEADERS
+  l3xc_all_api_h.h
+  l3xc_msg_enum.h
+)
diff --git a/src/plugins/l3xc/l3xc.api b/src/plugins/l3xc/l3xc.api
new file mode 100644
index 00000000000..512bedab778
--- /dev/null
+++ b/src/plugins/l3xc/l3xc.api
@@ -0,0 +1,98 @@
+/* Hey Emacs use -*- mode: C -*- */
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** \file
+    This file defines the vpp control-plane API messages
+    used to control the L3XC plugin
+*/
+
+option version = "1.0.0";
+
+import "vnet/fib/fib_types.api";
+
+/** \brief Get the plugin version
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define l3xc_plugin_get_version
+{
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief Reply to get the plugin version
+    @param context - returned sender context, to match reply w/ request
+    @param major - Incremented every time a known breaking behavior change is introduced
+    @param minor - Incremented with small changes, may be used to avoid buggy versions
+*/
+define l3xc_plugin_get_version_reply
+{
+  u32 context;
+  u32 major;
+  u32 minor;
+};
+
+/** \brief A description of an L3XC policy
+    @param input interface of the x-connect
+    @param n_paths Number of paths
+    @param paths The set of forwarding paths.
+ */
+typeonly define l3xc
+{
+  u32 sw_if_index;
+  u8 is_ip6;
+  u8 n_paths;
+  vl_api_fib_path_t paths[n_paths];
+};
+
+define l3xc_update
+{
+  u32 client_index;
+  u32 context;
+  vl_api_l3xc_t l3xc;
+};
+define l3xc_update_reply
+{
+  u32 context;
+  i32 retval;
+  u32 stats_index;
+};
+
+
+autoreply define l3xc_del
+{
+  u32 client_index;
+  u32 context;
+  u32 sw_if_index;
+  u8 is_ip6;
+};
+
+/** \brief Dump all L3XC policies
+ */
+define l3xc_dump
+{
+  u32 client_index;
+  u32 context;
+  u32 sw_if_index;
+};
+
+/** \brief description returned in the dump
+ */
+define l3xc_details
+{
+  u32 context;
+  vl_api_l3xc_t l3xc;
+};
diff --git a/src/plugins/l3xc/l3xc.c b/src/plugins/l3xc/l3xc.c
new file mode 100644
index 00000000000..77c062f39d6
--- /dev/null
+++ b/src/plugins/l3xc/l3xc.c
@@ -0,0 +1,395 @@
+/*
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <plugins/l3xc/l3xc.h>
+
+#include <vlib/vlib.h>
+#include <vnet/plugin/plugin.h>
+#include <vnet/fib/fib_path_list.h>
+
+/**
+ * FIB node type the attachment is registered
+ */
+fib_node_type_t l3xc_fib_node_type;
+
+/**
+ * Pool of L3XC objects
+ */
+l3xc_t *l3xc_pool;
+
+/**
+ * DB of L3XC objects
+ */
+static u32 *l3xc_db[FIB_PROTOCOL_IP_MAX];
+
+index_t
+l3xc_find (u32 sw_if_index, fib_protocol_t fproto)
+{
+  if (vec_len (l3xc_db[fproto]) <= sw_if_index)
+    return ~0;
+
+  return (l3xc_db[fproto][sw_if_index]);
+}
+
+static void
+l3xc_db_add (u32 sw_if_index, fib_protocol_t fproto, index_t l3xci)
+{
+  vec_validate_init_empty (l3xc_db[fproto], sw_if_index, ~0);
+
+  l3xc_db[fproto][sw_if_index] = l3xci;
+}
+
+static void
+l3xc_db_remove (u32 sw_if_index, fib_protocol_t fproto)
+{
+  vec_validate_init_empty (l3xc_db[fproto], sw_if_index, ~0);
+
+  l3xc_db[fproto][sw_if_index] = ~0;
+}
+
+static void
+l3xc_stack (l3xc_t * l3xc)
+{
+  /*
+   * stack the DPO on the forwarding contributed by the path-list
+   */
+  dpo_id_t via_dpo = DPO_INVALID;
+
+  fib_path_list_contribute_forwarding (l3xc->l3xc_pl,
+				       (FIB_PROTOCOL_IP4 == l3xc->l3xc_proto ?
+					FIB_FORW_CHAIN_TYPE_UNICAST_IP4 :
+					FIB_FORW_CHAIN_TYPE_UNICAST_IP6),
+				       FIB_PATH_LIST_FWD_FLAG_NONE, &via_dpo);
+
+  dpo_stack_from_node ((FIB_PROTOCOL_IP4 == l3xc->l3xc_proto ?
+			l3xc_ip4_node.index :
+			l3xc_ip6_node.index), &l3xc->l3xc_dpo, &via_dpo);
+  dpo_reset (&via_dpo);
+}
+
+int
+l3xc_update (u32 sw_if_index, u8 is_ip6, const fib_route_path_t * rpaths)
+{
+  fib_protocol_t fproto;
+  l3xc_t *l3xc;
+  u32 l3xci;
+
+  fproto = (is_ip6 ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4);
+
+  l3xci = l3xc_find (sw_if_index, fproto);
+
+  if (INDEX_INVALID == l3xci)
+    {
+      /*
+       * create a new x-connect
+       */
+      pool_get_aligned_zero (l3xc_pool, l3xc, CLIB_CACHE_LINE_BYTES);
+
+      l3xci = l3xc - l3xc_pool;
+      fib_node_init (&l3xc->l3xc_node, l3xc_fib_node_type);
+      l3xc->l3xc_sw_if_index = sw_if_index;
+      l3xc->l3xc_proto = fproto;
+
+      /*
+       * create and become a child of a path list so we get poked when
+       * the forwarding changes and stack on the DPO the path-list provides
+       */
+      l3xc->l3xc_pl = fib_path_list_create ((FIB_PATH_LIST_FLAG_SHARED |
+					     FIB_PATH_LIST_FLAG_NO_URPF),
+					    rpaths);
+      l3xc->l3xc_sibling = fib_path_list_child_add (l3xc->l3xc_pl,
+						    l3xc_fib_node_type,
+						    l3xci);
+      l3xc_stack (l3xc);
+
+      /*
+       * add this new policy to the DB and enable the feature on input interface
+       */
+      l3xc_db_add (sw_if_index, fproto, l3xci);
+
+      vnet_feature_enable_disable ((FIB_PROTOCOL_IP4 == fproto ?
+				    "ip4-unicast" :
+				    "ip6-unicast"),
+				   (FIB_PROTOCOL_IP4 == fproto ?
+				    "l3xc-input-ip4" :
+				    "l3xc-input-ip6"),
+				   l3xc->l3xc_sw_if_index,
+				   1, &l3xci, sizeof (l3xci));
+    }
+  else
+    {
+      /*
+       * update an existing x-connect.
+       * - add the path to the path-list and swap our ancestry
+       */
+      l3xc = l3xc_get (l3xci);
+
+      if (FIB_NODE_INDEX_INVALID != l3xc->l3xc_pl)
+	{
+	  fib_path_list_child_remove (l3xc->l3xc_pl, l3xc->l3xc_sibling);
+	}
+
+      l3xc->l3xc_pl = fib_path_list_create ((FIB_PATH_LIST_FLAG_SHARED |
+					     FIB_PATH_LIST_FLAG_NO_URPF),
+					    rpaths);
+
+      l3xc->l3xc_sibling = fib_path_list_child_add (l3xc->l3xc_pl,
+						    l3xc_fib_node_type,
+						    l3xci);
+    }
+  return (0);
+}
+
+int
+l3xc_delete (u32 sw_if_index, u8 is_ip6)
+{
+  fib_protocol_t fproto;
+  l3xc_t *l3xc;
+  u32 l3xci;
+
+  fproto = (is_ip6 ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4);
+
+  l3xci = l3xc_find (sw_if_index, fproto);
+
+  if (INDEX_INVALID == l3xci)
+    {
+      /*
+       * no such policy
+       */
+      return (VNET_API_ERROR_INVALID_VALUE);
+    }
+  else
+    {
+      l3xc = l3xc_get (l3xci);
+
+      vnet_feature_enable_disable ((FIB_PROTOCOL_IP4 == fproto ?
+				    "ip4-unicast" :
+				    "ip6-unicast"),
+				   (FIB_PROTOCOL_IP4 == fproto ?
+				    "l3xc-input-ip4" :
+				    "l3xc-input-ip6"),
+				   l3xc->l3xc_sw_if_index,
+				   0, &l3xci, sizeof (l3xci));
+
+      fib_path_list_child_remove (l3xc->l3xc_pl, l3xc->l3xc_sibling);
+
+      l3xc_db_remove (l3xc->l3xc_sw_if_index, fproto);
+      pool_put (l3xc_pool, l3xc);
+    }
+
+  return (0);
+}
+
+static clib_error_t *
+l3xc_cmd (vlib_main_t * vm,
+	  unformat_input_t * main_input, vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  fib_route_path_t *rpaths = NULL, rpath;
+  u32 sw_if_index, is_del, is_ip6;
+  vnet_main_t *vnm;
+  int rv = 0;
+
+  is_ip6 = is_del = 0;
+  sw_if_index = ~0;
+  vnm = vnet_get_main ();
+
+  /* Get a line of input. */
+  if (!unformat_user (main_input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat
+	  (line_input, "%U", unformat_vnet_sw_interface, vnm, &sw_if_index))
+	;
+      else if (unformat (line_input, "ip6"))
+	is_ip6 = 1;
+      else if (unformat (line_input, "ip4"))
+	is_ip6 = 0;
+      else if (unformat (line_input, "del"))
+	is_del = 1;
+      else if (unformat (line_input, "add"))
+	is_del = 0;
+      else if (unformat (line_input, "via %U",
+			 unformat_fib_route_path, &rpath))
+	vec_add1 (rpaths, rpath);
+      else
+	return (clib_error_return (0, "unknown input '%U'",
+				   format_unformat_error, line_input));
+    }
+
+  if (~0 == sw_if_index)
+    {
+      vlib_cli_output (vm, "Specify an input interface");
+      goto out;
+    }
+  if (vec_len (rpaths) == 0)
+    {
+      vlib_cli_output (vm, "Specify some paths");
+      goto out;
+    }
+
+  if (!is_del)
+    {
+      rv = l3xc_update (sw_if_index, is_ip6, rpaths);
+
+      if (rv)
+	{
+	  vlib_cli_output (vm, "Failed: %d", rv);
+	  goto out;
+	}
+    }
+  else
+    {
+      l3xc_delete (sw_if_index, is_ip6);
+    }
+
+out:
+  unformat_free (line_input);
+  return (NULL);
+}
+
+/* *INDENT-OFF* */
+/**
+ * Create an L3XC policy.
+ */
+VLIB_CLI_COMMAND (l3xc_cmd_node, static) = {
+  .path = "l3xc",
+  .function = l3xc_cmd,
+  .short_help = "l3xc [add|del] <INTERFACE> via ...",
+  .is_mp_safe = 1,
+};
+/* *INDENT-ON* */
+
+static u8 *
+format_l3xc (u8 * s, va_list * args)
+{
+  l3xc_t *l3xc = va_arg (*args, l3xc_t *);
+  vnet_main_t *vnm = vnet_get_main ();
+
+  s = format (s, "l3xc:[%d]: %U",
+	      l3xc - l3xc_pool, format_vnet_sw_if_index_name,
+	      vnm, l3xc->l3xc_sw_if_index);
+  s = format (s, "\n");
+  if (FIB_NODE_INDEX_INVALID == l3xc->l3xc_pl)
+    {
+      s = format (s, "no forwarding");
+    }
+  else
+    {
+      s = fib_path_list_format (l3xc->l3xc_pl, s);
+
+      s = format (s, "\n  %U", format_dpo_id, &l3xc->l3xc_dpo, 4);
+    }
+
+  return (s);
+}
+
+void
+l3xc_walk (l3xc_walk_cb_t cb, void *ctx)
+{
+  u32 l3xci;
+
+  /* *INDENT-OFF* */
+  pool_foreach_index(l3xci, l3xc_pool,
+  ({
+    if (!cb(l3xci, ctx))
+      break;
+  }));
+  /* *INDENT-ON* */
+}
+
+static clib_error_t *
+l3xc_show_cmd (vlib_main_t * vm,
+	       unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  l3xc_t *l3xc;
+
+  /* *INDENT-OFF* */
+  pool_foreach(l3xc, l3xc_pool,
+  ({
+    vlib_cli_output(vm, "%U", format_l3xc, l3xc);
+  }));
+  /* *INDENT-ON* */
+
+  return (NULL);
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (l3xc_show_cmd_node, static) = {
+  .path = "show l3xc",
+  .function = l3xc_show_cmd,
+  .short_help = "show l3xc",
+  .is_mp_safe = 1,
+};
+/* *INDENT-ON* */
+
+static fib_node_t *
+l3xc_get_node (fib_node_index_t index)
+{
+  l3xc_t *l3xc = l3xc_get (index);
+  return (&(l3xc->l3xc_node));
+}
+
+static l3xc_t *
+l3xc_get_from_node (fib_node_t * node)
+{
+  return ((l3xc_t *) (((char *) node) -
+		      STRUCT_OFFSET_OF (l3xc_t, l3xc_node)));
+}
+
+static void
+l3xc_last_lock_gone (fib_node_t * node)
+{
+}
+
+/*
+ * A back walk has reached this L3XC policy
+ */
+static fib_node_back_walk_rc_t
+l3xc_back_walk_notify (fib_node_t * node, fib_node_back_walk_ctx_t * ctx)
+{
+  l3xc_stack (l3xc_get_from_node (node));
+
+  return (FIB_NODE_BACK_WALK_CONTINUE);
+}
+
+/*
+ * The BIER fmask's graph node virtual function table
+ */
+static const fib_node_vft_t l3xc_vft = {
+  .fnv_get = l3xc_get_node,
+  .fnv_last_lock = l3xc_last_lock_gone,
+  .fnv_back_walk = l3xc_back_walk_notify,
+};
+
+static clib_error_t *
+l3xc_init (vlib_main_t * vm)
+{
+  l3xc_fib_node_type = fib_node_register_new_type (&l3xc_vft);
+
+  return (NULL);
+}
+
+VLIB_INIT_FUNCTION (l3xc_init);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/l3xc/l3xc.h b/src/plugins/l3xc/l3xc.h
new file mode 100644
index 00000000000..d5e1d372a86
--- /dev/null
+++ b/src/plugins/l3xc/l3xc.h
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2019 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A L3 cross connect will send all traffic that is received on the input
+ * interface to the [set of] paths requested.
+ * It is a much more memory efficient solution than using a separate IP table
+ * for each input interface and much faster than an ABF match all rule.
+ */
+
+#ifndef __L3XC_H__
+#define __L3XC_H__
+
+#include <vnet/fib/fib_node.h>
+
+#define L3XC_PLUGIN_VERSION_MAJOR 1
+#define L3XC_PLUGIN_VERSION_MINOR 0
+
+/**
+ */
+typedef struct l3xc_t_
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  /**
+   * Linkage into the FIB graph
+   */
+  fib_node_t l3xc_node;
+
+  /**
+   * The path-list describing how to forward in case of a match
+   */
+  fib_node_index_t l3xc_pl;
+
+  fib_protocol_t l3xc_proto;
+
+  /**
+   * Sibling index on the path-list
+   */
+  u32 l3xc_sibling;
+
+  /**
+   * The input interface
+   */
+  u32 l3xc_sw_if_index;
+
+  /**
+   * DPO for forwarding
+   */
+  dpo_id_t l3xc_dpo;
+} l3xc_t;
+
+/**
+ * Create or update an L3XC Policy
+ *
+ * @param sw_if_index_index the input interface
+ * @param rpaths The set of paths to add to the forwarding set
+ * @return error code
+ */
+extern int l3xc_update (u32 sw_if_index,
+			u8 is_ip6, const fib_route_path_t * rpaths);
+
+/**
+ * Delete an L3XC.
+ *
+ * @param sw_if_index_index the input interface
+ */
+extern int l3xc_delete (u32 sw_if_index, u8 is_ip6);
+
+/**
+ * Callback function invoked during a walk of all policies
+ */
+typedef int (*l3xc_walk_cb_t) (index_t l3xci, void *ctx);
+
+/**
+ * Walk/visit each of the L3XC policies
+ */
+extern void l3xc_walk (l3xc_walk_cb_t cb, void *ctx);
+
+/**
+ * Find a L3 XC object from an interfce and FIB protocol
+ */
+extern index_t l3xc_find (u32 sw_if_index, fib_protocol_t fproto);
+
+/**
+ * Data-plane functions
+ */
+extern l3xc_t *l3xc_pool;
+
+static_always_inline l3xc_t *
+l3xc_get (u32 index)
+{
+  return (pool_elt_at_index (l3xc_pool, index));
+}
+
+extern vlib_node_registration_t l3xc_ip4_node;
+extern vlib_node_registration_t l3xc_ip6_node;
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
+
+#endif
diff --git a/src/plugins/l3xc/l3xc_all_api_h.h b/src/plugins/l3xc/l3xc_all_api_h.h
new file mode 100644
index 00000000000..ca53dbcde45
--- /dev/null
+++ b/src/plugins/l3xc/l3xc_all_api_h.h
@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* Include the generated file, see BUILT_SOURCES in Makefile.am */
+#include <l3xc/l3xc.api.h>
diff --git a/src/plugins/l3xc/l3xc_api.c b/src/plugins/l3xc/l3xc_api.c
new file mode 100644
index 00000000000..45c01fa4e7a
--- /dev/null
+++ b/src/plugins/l3xc/l3xc_api.c
@@ -0,0 +1,296 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stddef.h>
+
+#include <vnet/vnet.h>
+#include <vnet/plugin/plugin.h>
+#include <l3xc/l3xc.h>
+#include <vnet/mpls/mpls_types.h>
+#include <vnet/fib/fib_path_list.h>
+#include <vnet/fib/fib_api.h>
+
+#include <vpp/app/version.h>
+
+#include <vlibapi/api.h>
+#include <vlibmemory/api.h>
+
+/* define message IDs */
+#include <l3xc/l3xc_msg_enum.h>
+
+/* define message structures */
+#define vl_typedefs
+#include <l3xc/l3xc_all_api_h.h>
+#undef vl_typedefs
+
+/* define generated endian-swappers */
+#define vl_endianfun
+#include <l3xc/l3xc_all_api_h.h>
+#undef vl_endianfun
+
+/* instantiate all the print functions we know about */
+#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
+#define vl_printfun
+#include <l3xc/l3xc_all_api_h.h>
+#undef vl_printfun
+
+/* Get the API version number */
+#define vl_api_version(n,v) static u32 api_version=(v);
+#include <l3xc/l3xc_all_api_h.h>
+#undef vl_api_version
+
+/**
+ * Base message ID fot the plugin
+ */
+static u32 l3xc_base_msg_id;
+
+#include <vlibapi/api_helper_macros.h>
+
+/* List of message types that this plugin understands */
+
+#define foreach_l3xc_plugin_api_msg                     \
+  _(L3XC_PLUGIN_GET_VERSION, l3xc_plugin_get_version)   \
+  _(L3XC_UPDATE, l3xc_update)                           \
+  _(L3XC_DEL, l3xc_del)                                 \
+  _(L3XC_DUMP, l3xc_dump)
+
+static void
+vl_api_l3xc_plugin_get_version_t_handler (vl_api_l3xc_plugin_get_version_t *
+					  mp)
+{
+  vl_api_l3xc_plugin_get_version_reply_t *rmp;
+  vl_api_registration_t *rp;
+
+  rp = vl_api_client_index_to_registration (mp->client_index);
+  if (rp == 0)
+    return;
+
+  rmp = vl_msg_api_alloc (sizeof (*rmp));
+  rmp->_vl_msg_id =
+    ntohs (VL_API_L3XC_PLUGIN_GET_VERSION_REPLY + l3xc_base_msg_id);
+  rmp->context = mp->context;
+  rmp->major = htonl (L3XC_PLUGIN_VERSION_MAJOR);
+  rmp->minor = htonl (L3XC_PLUGIN_VERSION_MINOR);
+
+  vl_api_send_msg (rp, (u8 *) rmp);
+}
+
+static void
+vl_api_l3xc_update_t_handler (vl_api_l3xc_update_t * mp)
+{
+  vl_api_l3xc_update_reply_t *rmp;
+  fib_route_path_t *paths = NULL, *path;
+  int rv = 0;
+  u8 pi;
+
+  VALIDATE_SW_IF_INDEX (&mp->l3xc);
+
+  if (0 == mp->l3xc.n_paths)
+    {
+      rv = VNET_API_ERROR_INVALID_VALUE;
+      goto done;
+    }
+
+  vec_validate (paths, mp->l3xc.n_paths - 1);
+
+  for (pi = 0; pi < mp->l3xc.n_paths; pi++)
+    {
+      path = &paths[pi];
+      rv = fib_path_api_parse (&mp->l3xc.paths[pi], path);
+
+      if (0 != rv)
+	{
+	  goto done;
+	}
+    }
+
+  rv = l3xc_update (ntohl (mp->l3xc.sw_if_index), mp->l3xc.is_ip6, paths);
+
+done:
+  vec_free (paths);
+
+  BAD_SW_IF_INDEX_LABEL;
+
+  /* *INDENT-OFF* */
+  REPLY_MACRO2 (VL_API_L3XC_UPDATE_REPLY + l3xc_base_msg_id,
+  ({
+    rmp->stats_index = 0;
+  }))
+  /* *INDENT-ON* */
+}
+
+static void
+vl_api_l3xc_del_t_handler (vl_api_l3xc_del_t * mp)
+{
+  vl_api_l3xc_del_reply_t *rmp;
+  int rv = 0;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  rv = l3xc_delete (ntohl (mp->sw_if_index), mp->is_ip6);
+
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_L3XC_DEL_REPLY + l3xc_base_msg_id);
+}
+
+typedef struct l3xc_dump_walk_ctx_t_
+{
+  vl_api_registration_t *rp;
+  u32 context;
+} l3xc_dump_walk_ctx_t;
+
+static int
+l3xc_send_details (u32 l3xci, void *args)
+{
+  fib_route_path_encode_t *api_rpaths = NULL, *api_rpath;
+  vl_api_l3xc_details_t *mp;
+  l3xc_dump_walk_ctx_t *ctx;
+  vl_api_fib_path_t *fp;
+  size_t msg_size;
+  l3xc_t *l3xc;
+  u8 n_paths;
+
+  ctx = args;
+  l3xc = l3xc_get (l3xci);
+  n_paths = fib_path_list_get_n_paths (l3xc->l3xc_pl);
+  msg_size = sizeof (*mp) + sizeof (mp->l3xc.paths[0]) * n_paths;
+
+  mp = vl_msg_api_alloc (msg_size);
+  clib_memset (mp, 0, msg_size);
+  mp->_vl_msg_id = ntohs (VL_API_L3XC_DETAILS + l3xc_base_msg_id);
+
+  /* fill in the message */
+  mp->context = ctx->context;
+  mp->l3xc.n_paths = n_paths;
+  mp->l3xc.sw_if_index = htonl (l3xc->l3xc_sw_if_index);
+
+  fib_path_list_walk_w_ext (l3xc->l3xc_pl, NULL, fib_path_encode,
+			    &api_rpaths);
+
+  fp = mp->l3xc.paths;
+  vec_foreach (api_rpath, api_rpaths)
+  {
+    fib_api_path_encode (api_rpath, fp);
+    fp++;
+  }
+
+  vl_api_send_msg (ctx->rp, (u8 *) mp);
+
+  return (1);
+}
+
+static void
+vl_api_l3xc_dump_t_handler (vl_api_l3xc_dump_t * mp)
+{
+  vl_api_registration_t *rp;
+  u32 sw_if_index;
+
+  rp = vl_api_client_index_to_registration (mp->client_index);
+  if (rp == 0)
+    return;
+
+  l3xc_dump_walk_ctx_t ctx = {
+    .rp = rp,
+    .context = mp->context,
+  };
+
+  sw_if_index = ntohl (mp->sw_if_index);
+
+  if (~0 == sw_if_index)
+    l3xc_walk (l3xc_send_details, &ctx);
+  else
+    {
+      fib_protocol_t fproto;
+      index_t l3xci;
+
+      FOR_EACH_FIB_IP_PROTOCOL (fproto)
+      {
+	l3xci = l3xc_find (sw_if_index, fproto);
+
+	if (INDEX_INVALID != l3xci)
+	  l3xc_send_details (l3xci, &ctx);
+      }
+    }
+}
+
+#define vl_msg_name_crc_list
+#include <l3xc/l3xc_all_api_h.h>
+#undef vl_msg_name_crc_list
+
+/* Set up the API message handling tables */
+static clib_error_t *
+l3xc_plugin_api_hookup (vlib_main_t * vm)
+{
+#define _(N,n)                                                  \
+    vl_msg_api_set_handlers((VL_API_##N + l3xc_base_msg_id),     \
+                            #n,					\
+                            vl_api_##n##_t_handler,             \
+                            vl_noop_handler,                    \
+                            vl_api_##n##_t_endian,              \
+                            vl_api_##n##_t_print,               \
+                            sizeof(vl_api_##n##_t), 1);
+  foreach_l3xc_plugin_api_msg;
+#undef _
+
+  return 0;
+}
+
+static void
+setup_message_id_table (api_main_t * apim)
+{
+#define _(id,n,crc) \
+  vl_msg_api_add_msg_name_crc (apim, #n "_" #crc, id + l3xc_base_msg_id);
+  foreach_vl_msg_name_crc_l3xc;
+#undef _
+}
+
+static clib_error_t *
+l3xc_api_init (vlib_main_t * vm)
+{
+  clib_error_t *error = 0;
+
+  u8 *name = format (0, "l3xc_%08x%c", api_version, 0);
+
+  /* Ask for a correctly-sized block of API message decode slots */
+  l3xc_base_msg_id = vl_msg_api_get_msg_ids ((char *) name,
+					     VL_MSG_FIRST_AVAILABLE);
+
+  error = l3xc_plugin_api_hookup (vm);
+
+  /* Add our API messages to the global name_crc hash table */
+  setup_message_id_table (&api_main);
+
+  vec_free (name);
+
+  return error;
+}
+
+VLIB_INIT_FUNCTION (l3xc_api_init);
+
+/* *INDENT-OFF* */
+VLIB_PLUGIN_REGISTER () = {
+    .version = VPP_BUILD_VER,
+    .description = "L3 Cross-Connect (L3XC)",
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/l3xc/l3xc_error.def b/src/plugins/l3xc/l3xc_error.def
new file mode 100644
index 00000000000..f00275b75cf
--- /dev/null
+++ b/src/plugins/l3xc/l3xc_error.def
@@ -0,0 +1,19 @@
+/*
+ * l3xc_error.def: L3XC errors
+ *
+ * Copyright (c) 2012 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+l3xc_error (NONE, "no match")
+l3xc_error (MATCHED, "matched")
diff --git a/src/plugins/l3xc/l3xc_msg_enum.h b/src/plugins/l3xc/l3xc_msg_enum.h
new file mode 100644
index 00000000000..51baddcad80
--- /dev/null
+++ b/src/plugins/l3xc/l3xc_msg_enum.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef included_l3xc_msg_enum_h
+#define included_l3xc_msg_enum_h
+
+#include <vppinfra/byte_order.h>
+
+#define vl_msg_id(n,h) n,
+typedef enum {
+#include <l3xc/l3xc_all_api_h.h>
+    /* We'll want to know how many messages IDs we need... */
+    VL_MSG_FIRST_AVAILABLE,
+} vl_msg_id_t;
+#undef vl_msg_id
+
+#endif
diff --git a/src/plugins/l3xc/l3xc_node.c b/src/plugins/l3xc/l3xc_node.c
new file mode 100644
index 00000000000..62db8c328b0
--- /dev/null
+++ b/src/plugins/l3xc/l3xc_node.c
@@ -0,0 +1,256 @@
+/*
+ * Copyright (c) 2019 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <plugins/l3xc/l3xc.h>
+#include <vnet/feature/feature.h>
+
+typedef enum l3xc_next_t_
+{
+  L3XC_NEXT_DROP,
+  L3XC_N_NEXT,
+} l3xc_next_t;
+
+typedef struct l3xc_input_trace_t_
+{
+  index_t l3xci;
+  index_t lbi;
+} l3xc_input_trace_t;
+
+typedef enum
+{
+#define l3xc_error(n,s) L3XC_ERROR_##n,
+#include "l3xc_error.def"
+#undef l3xc_error
+  L3XC_N_ERROR,
+} l3xc_error_t;
+
+always_inline uword
+l3xc_input_inline (vlib_main_t * vm,
+		   vlib_node_runtime_t * node,
+		   vlib_frame_t * frame, fib_protocol_t fproto)
+{
+  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
+  u16 nexts[VLIB_FRAME_SIZE], *next;
+  u32 n_left, *from;
+
+  from = vlib_frame_vector_args (frame);
+  n_left = frame->n_vectors;
+  b = bufs;
+  next = nexts;
+
+  vlib_get_buffers (vm, from, bufs, n_left);
+
+  while (n_left >= 8)
+    {
+      const l3xc_t *l3xc0, *l3xc1, *l3xc2, *l3xc3;
+      u32 l3xci0, l3xci1, l3xci2, l3xci3;
+      u32 next_u32;
+
+      /* Prefetch next iteration. */
+      {
+	vlib_prefetch_buffer_header (b[4], LOAD);
+	vlib_prefetch_buffer_header (b[5], LOAD);
+	vlib_prefetch_buffer_header (b[6], LOAD);
+	vlib_prefetch_buffer_header (b[7], LOAD);
+      }
+
+      l3xci0 =
+	*(u32 *) vnet_feature_next_with_data (&next_u32, b[0],
+					      sizeof (l3xci0));
+      l3xci1 =
+	*(u32 *) vnet_feature_next_with_data (&next_u32, b[1],
+					      sizeof (l3xci1));
+      l3xci2 =
+	*(u32 *) vnet_feature_next_with_data (&next_u32, b[2],
+					      sizeof (l3xci2));
+      l3xci3 =
+	*(u32 *) vnet_feature_next_with_data (&next_u32, b[3],
+					      sizeof (l3xci3));
+
+      l3xc0 = l3xc_get (l3xci0);
+      l3xc1 = l3xc_get (l3xci1);
+      l3xc2 = l3xc_get (l3xci2);
+      l3xc3 = l3xc_get (l3xci3);
+
+      next[0] = l3xc0->l3xc_dpo.dpoi_next_node;
+      next[1] = l3xc1->l3xc_dpo.dpoi_next_node;
+      next[2] = l3xc2->l3xc_dpo.dpoi_next_node;
+      next[3] = l3xc3->l3xc_dpo.dpoi_next_node;
+
+      vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = l3xc0->l3xc_dpo.dpoi_index;
+      vnet_buffer (b[1])->ip.adj_index[VLIB_TX] = l3xc1->l3xc_dpo.dpoi_index;
+      vnet_buffer (b[2])->ip.adj_index[VLIB_TX] = l3xc2->l3xc_dpo.dpoi_index;
+      vnet_buffer (b[3])->ip.adj_index[VLIB_TX] = l3xc3->l3xc_dpo.dpoi_index;
+
+      if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	{
+	  if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      l3xc_input_trace_t *tr;
+
+	      tr = vlib_add_trace (vm, node, b[0], sizeof (*tr));
+	      tr->l3xci = l3xci0;
+	      tr->lbi = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
+	    }
+	  if (PREDICT_FALSE (b[1]->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      l3xc_input_trace_t *tr;
+
+	      tr = vlib_add_trace (vm, node, b[1], sizeof (*tr));
+	      tr->l3xci = l3xci1;
+	      tr->lbi = vnet_buffer (b[1])->ip.adj_index[VLIB_TX];
+	    }
+	  if (PREDICT_FALSE (b[2]->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      l3xc_input_trace_t *tr;
+
+	      tr = vlib_add_trace (vm, node, b[2], sizeof (*tr));
+	      tr->l3xci = l3xci2;
+	      tr->lbi = vnet_buffer (b[2])->ip.adj_index[VLIB_TX];
+	    }
+	  if (PREDICT_FALSE (b[3]->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+	      l3xc_input_trace_t *tr;
+
+	      tr = vlib_add_trace (vm, node, b[3], sizeof (*tr));
+	      tr->l3xci = l3xci3;
+	      tr->lbi = vnet_buffer (b[3])->ip.adj_index[VLIB_TX];
+	    }
+	}
+
+      b += 4;
+      next += 4;
+      n_left -= 4;
+    }
+
+  while (n_left > 0)
+    {
+      u32 l3xci0, next_u32;
+      const l3xc_t *l3xc0;
+
+      l3xci0 =
+	*(u32 *) vnet_feature_next_with_data (&next_u32, b[0],
+					      sizeof (l3xci0));
+
+      l3xc0 = l3xc_get (l3xci0);
+
+      next[0] = l3xc0->l3xc_dpo.dpoi_next_node;
+
+      vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = l3xc0->l3xc_dpo.dpoi_index;
+
+      if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
+	{
+	  l3xc_input_trace_t *tr;
+
+	  tr = vlib_add_trace (vm, node, b[0], sizeof (*tr));
+	  tr->l3xci = l3xci0;
+	  tr->lbi = vnet_buffer (b[0])->ip.adj_index[VLIB_TX];
+	}
+
+      b += 1;
+      next += 1;
+      n_left -= 1;
+    }
+
+  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
+  return frame->n_vectors;
+}
+
+static uword
+l3xc_input_ip4 (vlib_main_t * vm,
+		vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  return l3xc_input_inline (vm, node, frame, FIB_PROTOCOL_IP4);
+}
+
+static uword
+l3xc_input_ip6 (vlib_main_t * vm,
+		vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  return l3xc_input_inline (vm, node, frame, FIB_PROTOCOL_IP6);
+}
+
+static u8 *
+format_l3xc_input_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  l3xc_input_trace_t *t = va_arg (*args, l3xc_input_trace_t *);
+
+  s = format (s, "l3xc-index:%d lb-index:%d", t->l3xci, t->lbi);
+  return s;
+}
+
+static char *l3xc_error_strings[] = {
+#define l3xc_error(n,s) s,
+#include "l3xc_error.def"
+#undef l3xc_error
+};
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (l3xc_ip4_node) =
+{
+  .function = l3xc_input_ip4,
+  .name = "l3xc-input-ip4",
+  .vector_size = sizeof (u32),
+  .format_trace = format_l3xc_input_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = L3XC_N_ERROR,
+  .error_strings = l3xc_error_strings,
+  .n_next_nodes = L3XC_N_NEXT,
+  .next_nodes =
+  {
+    [L3XC_NEXT_DROP] = "error-drop",
+  }
+};
+
+VLIB_REGISTER_NODE (l3xc_ip6_node) =
+{
+  .function = l3xc_input_ip6,
+  .name = "l3xc-input-ip6",
+  .vector_size = sizeof (u32),
+  .format_trace = format_l3xc_input_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = 0,
+  .n_next_nodes = L3XC_N_NEXT,
+
+  .next_nodes =
+  {
+    [L3XC_NEXT_DROP] = "error-drop",
+  }
+};
+
+VNET_FEATURE_INIT (l3xc_ip4_feat, static) =
+{
+  .arc_name = "ip4-unicast",
+  .node_name = "l3xc-input-ip4",
+  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa"),
+};
+
+VNET_FEATURE_INIT (l3xc_ip6_feat, static) =
+{
+  .arc_name = "ip6-unicast",
+  .node_name = "l3xc-input-ip6",
+  .runs_after = VNET_FEATURES ("acl-plugin-in-ip6-fa"),
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/fib/fib_path_list.c b/src/vnet/fib/fib_path_list.c
index f1d54430d66..47170adf864 100644
--- a/src/vnet/fib/fib_path_list.c
+++ b/src/vnet/fib/fib_path_list.c
@@ -25,6 +25,7 @@
 #include <vnet/fib/fib_walk.h>
 #include <vnet/fib/fib_urpf_list.h>
 #include <vnet/fib/fib_path_ext.h>
+#include <vnet/fib/fib_table.h>
 
 /**
  * The magic number of child entries that make a path-list popular.
@@ -364,10 +365,12 @@ fib_path_list_mk_lb (fib_path_list_t *path_list,
 		     dpo_id_t *dpo,
                      fib_path_list_fwd_flags_t flags)
 {
-    load_balance_path_t *nhs;
     fib_node_index_t *path_index;
+    load_balance_path_t *nhs;
+    dpo_proto_t dproto;
 
     nhs = NULL;
+    dproto = fib_forw_chain_type_to_dpo_proto(fct);
 
     /*
      * We gather the DPOs from resolved paths.
@@ -388,10 +391,11 @@ fib_path_list_mk_lb (fib_path_list_t *path_list,
      */
     dpo_set(dpo,
             DPO_LOAD_BALANCE,
-            fib_forw_chain_type_to_dpo_proto(fct),
+            dproto,
             load_balance_create(vec_len(nhs),
-                                fib_forw_chain_type_to_dpo_proto(fct),
-                                0 /* FIXME FLOW HASH */));
+                                dproto,
+                                fib_table_get_default_flow_hash_config(
+                                    dpo_proto_to_fib(dproto))));
     load_balance_multipath_update(dpo, nhs,
                                   fib_path_list_fwd_flags_2_load_balance(flags));