From 7cd468a3d7dee7d6c92f69a0bb7061ae208ec727 Mon Sep 17 00:00:00 2001
From: Damjan Marion <damarion@cisco.com>
Date: Mon, 19 Dec 2016 23:05:39 +0100
Subject: Reorganize source tree to use single autotools instance

Change-Id: I7b51f88292e057c6443b12224486f2d0c9f8ae23
Signed-off-by: Damjan Marion <damarion@cisco.com>
---
 src/vnet/ip/lookup.c | 967 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 967 insertions(+)
 create mode 100644 src/vnet/ip/lookup.c

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
new file mode 100644
index 00000000..734a4cd7
--- /dev/null
+++ b/src/vnet/ip/lookup.c
@@ -0,0 +1,967 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * ip/ip_lookup.c: ip4/6 adjacency and lookup table managment
+ *
+ * Copyright (c) 2008 Eliot Dresselhaus
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <vnet/ip/ip.h>
+#include <vnet/adj/adj.h>
+#include <vnet/fib/fib_table.h>
+#include <vnet/fib/ip4_fib.h>
+#include <vnet/fib/ip6_fib.h>
+#include <vnet/mpls/mpls.h>
+#include <vnet/dpo/drop_dpo.h>
+#include <vnet/dpo/classify_dpo.h>
+#include <vnet/dpo/punt_dpo.h>
+#include <vnet/dpo/receive_dpo.h>
+#include <vnet/dpo/ip_null_dpo.h>
+
+/**
+ * @file
+ * @brief IPv4 and IPv6 adjacency and lookup table managment.
+ *
+ */
+
+clib_error_t *
+ip_interface_address_add_del (ip_lookup_main_t * lm,
+			      u32 sw_if_index,
+			      void *addr_fib,
+			      u32 address_length,
+			      u32 is_del, u32 * result_if_address_index)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  ip_interface_address_t *a, *prev, *next;
+  uword *p = mhash_get (&lm->address_to_if_address_index, addr_fib);
+
+  vec_validate_init_empty (lm->if_address_pool_index_by_sw_if_index,
+			   sw_if_index, ~0);
+  a = p ? pool_elt_at_index (lm->if_address_pool, p[0]) : 0;
+
+  /* Verify given length. */
+  if ((a && (address_length != a->address_length)) || (address_length == 0))
+    {
+      vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH;
+      return clib_error_create
+	("%U wrong length (expected %d) for interface %U",
+	 lm->format_address_and_length, addr_fib,
+	 address_length, a ? a->address_length : -1,
+	 format_vnet_sw_if_index_name, vnm, sw_if_index);
+    }
+
+  if (is_del)
+    {
+      if (!a)
+	{
+	  vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index);
+	  vnm->api_errno = VNET_API_ERROR_ADDRESS_NOT_FOUND_FOR_INTERFACE;
+	  return clib_error_create ("%U not found for interface %U",
+				    lm->format_address_and_length,
+				    addr_fib, address_length,
+				    format_vnet_sw_interface_name, vnm, si);
+	}
+
+      if (a->prev_this_sw_interface != ~0)
+	{
+	  prev =
+	    pool_elt_at_index (lm->if_address_pool,
+			       a->prev_this_sw_interface);
+	  prev->next_this_sw_interface = a->next_this_sw_interface;
+	}
+      if (a->next_this_sw_interface != ~0)
+	{
+	  next =
+	    pool_elt_at_index (lm->if_address_pool,
+			       a->next_this_sw_interface);
+	  next->prev_this_sw_interface = a->prev_this_sw_interface;
+
+	  if (a->prev_this_sw_interface == ~0)
+	    lm->if_address_pool_index_by_sw_if_index[sw_if_index] =
+	      a->next_this_sw_interface;
+	}
+
+      if ((a->next_this_sw_interface == ~0)
+	  && (a->prev_this_sw_interface == ~0))
+	lm->if_address_pool_index_by_sw_if_index[sw_if_index] = ~0;
+
+      mhash_unset (&lm->address_to_if_address_index, addr_fib,
+		   /* old_value */ 0);
+      pool_put (lm->if_address_pool, a);
+
+      if (result_if_address_index)
+	*result_if_address_index = ~0;
+    }
+
+  else if (!a)
+    {
+      u32 pi;			/* previous index */
+      u32 ai;
+      u32 hi;			/* head index */
+
+      pool_get (lm->if_address_pool, a);
+      memset (a, ~0, sizeof (a[0]));
+      ai = a - lm->if_address_pool;
+
+      hi = pi = lm->if_address_pool_index_by_sw_if_index[sw_if_index];
+      prev = 0;
+      while (pi != (u32) ~ 0)
+	{
+	  prev = pool_elt_at_index (lm->if_address_pool, pi);
+	  pi = prev->next_this_sw_interface;
+	}
+      pi = prev ? prev - lm->if_address_pool : (u32) ~ 0;
+
+      a->address_key = mhash_set (&lm->address_to_if_address_index,
+				  addr_fib, ai, /* old_value */ 0);
+      a->address_length = address_length;
+      a->sw_if_index = sw_if_index;
+      a->flags = 0;
+      a->prev_this_sw_interface = pi;
+      a->next_this_sw_interface = ~0;
+      if (prev)
+	prev->next_this_sw_interface = ai;
+
+      lm->if_address_pool_index_by_sw_if_index[sw_if_index] =
+	(hi != ~0) ? hi : ai;
+      if (result_if_address_index)
+	*result_if_address_index = ai;
+    }
+  else
+    {
+      if (result_if_address_index)
+	*result_if_address_index = a - lm->if_address_pool;
+    }
+
+
+  return /* no error */ 0;
+}
+
+void
+ip_lookup_init (ip_lookup_main_t * lm, u32 is_ip6)
+{
+  /* ensure that adjacency is cacheline aligned and sized */
+  STATIC_ASSERT (STRUCT_OFFSET_OF (ip_adjacency_t, cacheline0) == 0,
+		 "Cache line marker must be 1st element in struct");
+  STATIC_ASSERT (STRUCT_OFFSET_OF (ip_adjacency_t, cacheline1) ==
+		 CLIB_CACHE_LINE_BYTES,
+		 "Data in cache line 0 is bigger than cache line size");
+
+  /* Preallocate three "special" adjacencies */
+  lm->adjacency_heap = adj_pool;
+
+  if (!lm->fib_result_n_bytes)
+    lm->fib_result_n_bytes = sizeof (uword);
+
+  lm->is_ip6 = is_ip6;
+  if (is_ip6)
+    {
+      lm->format_address_and_length = format_ip6_address_and_length;
+      mhash_init (&lm->address_to_if_address_index, sizeof (uword),
+		  sizeof (ip6_address_fib_t));
+    }
+  else
+    {
+      lm->format_address_and_length = format_ip4_address_and_length;
+      mhash_init (&lm->address_to_if_address_index, sizeof (uword),
+		  sizeof (ip4_address_fib_t));
+    }
+
+  {
+    int i;
+
+    /* Setup all IP protocols to be punted and builtin-unknown. */
+    for (i = 0; i < 256; i++)
+      {
+	lm->local_next_by_ip_protocol[i] = IP_LOCAL_NEXT_PUNT;
+	lm->builtin_protocol_by_ip_protocol[i] = IP_BUILTIN_PROTOCOL_UNKNOWN;
+      }
+
+    lm->local_next_by_ip_protocol[IP_PROTOCOL_UDP] = IP_LOCAL_NEXT_UDP_LOOKUP;
+    lm->local_next_by_ip_protocol[is_ip6 ? IP_PROTOCOL_ICMP6 :
+				  IP_PROTOCOL_ICMP] = IP_LOCAL_NEXT_ICMP;
+    lm->builtin_protocol_by_ip_protocol[IP_PROTOCOL_UDP] =
+      IP_BUILTIN_PROTOCOL_UDP;
+    lm->builtin_protocol_by_ip_protocol[is_ip6 ? IP_PROTOCOL_ICMP6 :
+					IP_PROTOCOL_ICMP] =
+      IP_BUILTIN_PROTOCOL_ICMP;
+  }
+}
+
+u8 *
+format_ip_flow_hash_config (u8 * s, va_list * args)
+{
+  flow_hash_config_t flow_hash_config = va_arg (*args, u32);
+
+#define _(n,v) if (flow_hash_config & v) s = format (s, "%s ", #n);
+  foreach_flow_hash_bit;
+#undef _
+
+  return s;
+}
+
+u8 *
+format_ip_lookup_next (u8 * s, va_list * args)
+{
+  ip_lookup_next_t n = va_arg (*args, ip_lookup_next_t);
+  char *t = 0;
+
+  switch (n)
+    {
+    default:
+      s = format (s, "unknown %d", n);
+      return s;
+
+    case IP_LOOKUP_NEXT_DROP:
+      t = "drop";
+      break;
+    case IP_LOOKUP_NEXT_PUNT:
+      t = "punt";
+      break;
+    case IP_LOOKUP_NEXT_ARP:
+      t = "arp";
+      break;
+    case IP_LOOKUP_NEXT_MIDCHAIN:
+      t = "midchain";
+      break;
+    case IP_LOOKUP_NEXT_GLEAN:
+      t = "glean";
+      break;
+    case IP_LOOKUP_NEXT_REWRITE:
+      break;
+    }
+
+  if (t)
+    vec_add (s, t, strlen (t));
+
+  return s;
+}
+
+u8 *
+format_ip_adjacency_packet_data (u8 * s, va_list * args)
+{
+  vnet_main_t *vnm = va_arg (*args, vnet_main_t *);
+  u32 adj_index = va_arg (*args, u32);
+  u8 *packet_data = va_arg (*args, u8 *);
+  u32 n_packet_data_bytes = va_arg (*args, u32);
+  ip_adjacency_t *adj = adj_get (adj_index);
+
+  switch (adj->lookup_next_index)
+    {
+    case IP_LOOKUP_NEXT_REWRITE:
+      s = format (s, "%U",
+		  format_vnet_rewrite_header,
+		  vnm->vlib_main, &adj->rewrite_header, packet_data,
+		  n_packet_data_bytes);
+      break;
+
+    default:
+      break;
+    }
+
+  return s;
+}
+
+static uword
+unformat_dpo (unformat_input_t * input, va_list * args)
+{
+  dpo_id_t *dpo = va_arg (*args, dpo_id_t *);
+  fib_protocol_t fp = va_arg (*args, int);
+  dpo_proto_t proto;
+
+  proto = fib_proto_to_dpo (fp);
+
+  if (unformat (input, "drop"))
+    dpo_copy (dpo, drop_dpo_get (proto));
+  else if (unformat (input, "punt"))
+    dpo_copy (dpo, punt_dpo_get (proto));
+  else if (unformat (input, "local"))
+    receive_dpo_add_or_lock (proto, ~0, NULL, dpo);
+  else if (unformat (input, "null-send-unreach"))
+    ip_null_dpo_add_and_lock (proto, IP_NULL_ACTION_SEND_ICMP_UNREACH, dpo);
+  else if (unformat (input, "null-send-prohibit"))
+    ip_null_dpo_add_and_lock (proto, IP_NULL_ACTION_SEND_ICMP_PROHIBIT, dpo);
+  else if (unformat (input, "null"))
+    ip_null_dpo_add_and_lock (proto, IP_NULL_ACTION_NONE, dpo);
+  else if (unformat (input, "classify"))
+    {
+      u32 classify_table_index;
+
+      if (!unformat (input, "%d", &classify_table_index))
+	{
+	  clib_warning ("classify adj must specify table index");
+	  return 0;
+	}
+
+      dpo_set (dpo, DPO_CLASSIFY, proto,
+	       classify_dpo_create (proto, classify_table_index));
+    }
+  else
+    return 0;
+
+  return 1;
+}
+
+const ip46_address_t zero_addr = {
+  .as_u64 = {
+	     0, 0},
+};
+
+u32
+fib_table_id_find_fib_index (fib_protocol_t proto, u32 table_id)
+{
+  ip4_main_t *im4 = &ip4_main;
+  ip6_main_t *im6 = &ip6_main;
+  uword *p;
+
+  switch (proto)
+    {
+    case FIB_PROTOCOL_IP4:
+      p = hash_get (im4->fib_index_by_table_id, table_id);
+      break;
+    case FIB_PROTOCOL_IP6:
+      p = hash_get (im6->fib_index_by_table_id, table_id);
+      break;
+    default:
+      p = NULL;
+      break;
+    }
+  if (NULL != p)
+    {
+      return (p[0]);
+    }
+  return (~0);
+}
+
+clib_error_t *
+vnet_ip_route_cmd (vlib_main_t * vm,
+		   unformat_input_t * main_input, vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  fib_route_path_t *rpaths = NULL, rpath;
+  dpo_id_t dpo = DPO_INVALID, *dpos = NULL;
+  fib_prefix_t *prefixs = NULL, pfx;
+  mpls_label_t out_label, via_label;
+  clib_error_t *error = NULL;
+  u32 table_id, is_del;
+  vnet_main_t *vnm;
+  u32 fib_index;
+  f64 count;
+  int i;
+
+  vnm = vnet_get_main ();
+  is_del = 0;
+  table_id = 0;
+  count = 1;
+  memset (&pfx, 0, sizeof (pfx));
+  out_label = via_label = MPLS_LABEL_INVALID;
+
+  /* Get a line of input. */
+  if (!unformat_user (main_input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      memset (&rpath, 0, sizeof (rpath));
+
+      if (unformat (line_input, "table %d", &table_id))
+	;
+      else if (unformat (line_input, "del"))
+	is_del = 1;
+      else if (unformat (line_input, "add"))
+	is_del = 0;
+      else if (unformat (line_input, "resolve-via-host"))
+	{
+	  if (vec_len (rpaths) == 0)
+	    {
+	      error = clib_error_return (0, "Paths then flags");
+	      goto done;
+	    }
+	  rpaths[vec_len (rpaths) - 1].frp_flags |=
+	    FIB_ROUTE_PATH_RESOLVE_VIA_HOST;
+	}
+      else if (unformat (line_input, "resolve-via-attached"))
+	{
+	  if (vec_len (rpaths) == 0)
+	    {
+	      error = clib_error_return (0, "Paths then flags");
+	      goto done;
+	    }
+	  rpaths[vec_len (rpaths) - 1].frp_flags |=
+	    FIB_ROUTE_PATH_RESOLVE_VIA_ATTACHED;
+	}
+      else if (unformat (line_input, "out-label %U",
+			 unformat_mpls_unicast_label, &out_label))
+	{
+	  if (vec_len (rpaths) == 0)
+	    {
+	      error = clib_error_return (0, "Paths then labels");
+	      goto done;
+	    }
+	  vec_add1 (rpaths[vec_len (rpaths) - 1].frp_label_stack, out_label);
+	}
+      else if (unformat (line_input, "via-label %U",
+			 unformat_mpls_unicast_label, &rpath.frp_local_label))
+	{
+	  rpath.frp_weight = 1;
+	  rpath.frp_proto = FIB_PROTOCOL_MPLS;
+	  rpath.frp_sw_if_index = ~0;
+	  vec_add1 (rpaths, rpath);
+	}
+      else if (unformat (line_input, "count %f", &count))
+	;
+
+      else if (unformat (line_input, "%U/%d",
+			 unformat_ip4_address, &pfx.fp_addr.ip4, &pfx.fp_len))
+	{
+	  pfx.fp_proto = FIB_PROTOCOL_IP4;
+	  vec_add1 (prefixs, pfx);
+	}
+      else if (unformat (line_input, "%U/%d",
+			 unformat_ip6_address, &pfx.fp_addr.ip6, &pfx.fp_len))
+	{
+	  pfx.fp_proto = FIB_PROTOCOL_IP6;
+	  vec_add1 (prefixs, pfx);
+	}
+      else if (unformat (line_input, "via %U %U weight %u",
+			 unformat_ip4_address,
+			 &rpath.frp_addr.ip4,
+			 unformat_vnet_sw_interface, vnm,
+			 &rpath.frp_sw_if_index, &rpath.frp_weight))
+	{
+	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  vec_add1 (rpaths, rpath);
+	}
+
+      else if (unformat (line_input, "via %U %U weight %u",
+			 unformat_ip6_address,
+			 &rpath.frp_addr.ip6,
+			 unformat_vnet_sw_interface, vnm,
+			 &rpath.frp_sw_if_index, &rpath.frp_weight))
+	{
+	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  vec_add1 (rpaths, rpath);
+	}
+
+      else if (unformat (line_input, "via %U %U",
+			 unformat_ip4_address,
+			 &rpath.frp_addr.ip4,
+			 unformat_vnet_sw_interface, vnm,
+			 &rpath.frp_sw_if_index))
+	{
+	  rpath.frp_weight = 1;
+	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  vec_add1 (rpaths, rpath);
+	}
+
+      else if (unformat (line_input, "via %U %U",
+			 unformat_ip6_address,
+			 &rpath.frp_addr.ip6,
+			 unformat_vnet_sw_interface, vnm,
+			 &rpath.frp_sw_if_index))
+	{
+	  rpath.frp_weight = 1;
+	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  vec_add1 (rpaths, rpath);
+	}
+      else if (unformat (line_input, "via %U next-hop-table %d",
+			 unformat_ip4_address,
+			 &rpath.frp_addr.ip4, &rpath.frp_fib_index))
+	{
+	  rpath.frp_weight = 1;
+	  rpath.frp_sw_if_index = ~0;
+	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  vec_add1 (rpaths, rpath);
+	}
+      else if (unformat (line_input, "via %U next-hop-table %d",
+			 unformat_ip6_address,
+			 &rpath.frp_addr.ip6, &rpath.frp_fib_index))
+	{
+	  rpath.frp_weight = 1;
+	  rpath.frp_sw_if_index = ~0;
+	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  vec_add1 (rpaths, rpath);
+	}
+      else if (unformat (line_input, "via %U",
+			 unformat_ip4_address, &rpath.frp_addr.ip4))
+	{
+	  /*
+	   * the recursive next-hops are by default in the same table
+	   * as the prefix
+	   */
+	  rpath.frp_fib_index = table_id;
+	  rpath.frp_weight = 1;
+	  rpath.frp_sw_if_index = ~0;
+	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  vec_add1 (rpaths, rpath);
+	}
+      else if (unformat (line_input, "via %U",
+			 unformat_ip6_address, &rpath.frp_addr.ip6))
+	{
+	  rpath.frp_fib_index = table_id;
+	  rpath.frp_weight = 1;
+	  rpath.frp_sw_if_index = ~0;
+	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  vec_add1 (rpaths, rpath);
+	}
+      else if (unformat (line_input,
+			 "lookup in table %d", &rpath.frp_fib_index))
+	{
+	  rpath.frp_proto = pfx.fp_proto;
+	  rpath.frp_sw_if_index = ~0;
+	  vec_add1 (rpaths, rpath);
+	}
+      else if (vec_len (prefixs) > 0 &&
+	       unformat (line_input, "via %U",
+			 unformat_vnet_sw_interface, vnm,
+			 &rpath.frp_sw_if_index))
+	{
+	  rpath.frp_weight = 1;
+	  rpath.frp_proto = prefixs[0].fp_proto;
+	  vec_add1 (rpaths, rpath);
+	}
+      else if (vec_len (prefixs) > 0 &&
+	       unformat (line_input, "via %U",
+			 unformat_dpo, &dpo, prefixs[0].fp_proto))
+	{
+	  vec_add1 (dpos, dpo);
+	}
+      else
+	{
+	  error = unformat_parse_error (line_input);
+	  goto done;
+	}
+    }
+
+  unformat_free (line_input);
+
+  if (vec_len (prefixs) == 0)
+    {
+      error =
+	clib_error_return (0, "expected ip4/ip6 destination address/length.");
+      goto done;
+    }
+
+  if (!is_del && vec_len (rpaths) + vec_len (dpos) == 0)
+    {
+      error = clib_error_return (0, "expected paths.");
+      goto done;
+    }
+
+  if (~0 == table_id)
+    {
+      /*
+       * if no table_id is passed we will manipulate the default
+       */
+      fib_index = 0;
+    }
+  else
+    {
+      fib_index = fib_table_id_find_fib_index (prefixs[0].fp_proto, table_id);
+
+      if (~0 == fib_index)
+	{
+	  error = clib_error_return (0, "Nonexistent table id %d", table_id);
+	  goto done;
+	}
+    }
+
+  for (i = 0; i < vec_len (prefixs); i++)
+    {
+      if (is_del && 0 == vec_len (rpaths))
+	{
+	  fib_table_entry_delete (fib_index, &prefixs[i], FIB_SOURCE_CLI);
+	}
+      else if (!is_del && 1 == vec_len (dpos))
+	{
+	  fib_table_entry_special_dpo_add (fib_index,
+					   &prefixs[i],
+					   FIB_SOURCE_CLI,
+					   FIB_ENTRY_FLAG_EXCLUSIVE,
+					   &dpos[0]);
+	  dpo_reset (&dpos[0]);
+	}
+      else if (vec_len (dpos) > 0)
+	{
+	  error =
+	    clib_error_return (0,
+			       "Load-balancing over multiple special adjacencies is unsupported");
+	  goto done;
+	}
+      else if (0 < vec_len (rpaths))
+	{
+	  u32 k, j, n, incr;
+	  ip46_address_t dst = prefixs[i].fp_addr;
+	  f64 t[2];
+	  n = count;
+	  t[0] = vlib_time_now (vm);
+	  incr = 1 << ((FIB_PROTOCOL_IP4 == prefixs[0].fp_proto ? 32 : 128) -
+		       prefixs[i].fp_len);
+
+	  for (k = 0; k < n; k++)
+	    {
+	      for (j = 0; j < vec_len (rpaths); j++)
+		{
+		  u32 fi;
+		  /*
+		   * the CLI parsing stored table Ids, swap to FIB indicies
+		   */
+		  fi = fib_table_id_find_fib_index (prefixs[i].fp_proto,
+						    rpaths[i].frp_fib_index);
+
+		  if (~0 == fi)
+		    {
+		      error =
+			clib_error_return (0, "Via table %d does not exist",
+					   rpaths[i].frp_fib_index);
+		      goto done;
+		    }
+		  rpaths[i].frp_fib_index = fi;
+
+		  fib_prefix_t rpfx = {
+		    .fp_len = prefixs[i].fp_len,
+		    .fp_proto = prefixs[i].fp_proto,
+		    .fp_addr = dst,
+		  };
+
+		  if (is_del)
+		    fib_table_entry_path_remove2 (fib_index,
+						  &rpfx,
+						  FIB_SOURCE_CLI, &rpaths[j]);
+		  else
+		    fib_table_entry_path_add2 (fib_index,
+					       &rpfx,
+					       FIB_SOURCE_CLI,
+					       FIB_ENTRY_FLAG_NONE,
+					       &rpaths[j]);
+		}
+
+	      if (FIB_PROTOCOL_IP4 == prefixs[0].fp_proto)
+		{
+		  dst.ip4.as_u32 =
+		    clib_host_to_net_u32 (incr +
+					  clib_net_to_host_u32 (dst.
+								ip4.as_u32));
+		}
+	      else
+		{
+		  int bucket = (incr < 64 ? 0 : 1);
+		  dst.ip6.as_u64[bucket] =
+		    clib_host_to_net_u64 (incr +
+					  clib_net_to_host_u64 (dst.ip6.as_u64
+								[bucket]));
+
+		}
+	    }
+	  t[1] = vlib_time_now (vm);
+	  if (count > 1)
+	    vlib_cli_output (vm, "%.6e routes/sec", count / (t[1] - t[0]));
+	}
+      else
+	{
+	  error = clib_error_return (0, "Don't understand what you want...");
+	  goto done;
+	}
+    }
+
+
+done:
+  vec_free (dpos);
+  vec_free (prefixs);
+  vec_free (rpaths);
+  return error;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (vlib_cli_ip_command, static) = {
+  .path = "ip",
+  .short_help = "Internet protocol (IP) commands",
+};
+/* *INDENT-ON* */
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (vlib_cli_ip6_command, static) = {
+  .path = "ip6",
+  .short_help = "Internet protocol version 6 (IPv6) commands",
+};
+/* *INDENT-ON* */
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (vlib_cli_show_ip_command, static) = {
+  .path = "show ip",
+  .short_help = "Internet protocol (IP) show commands",
+};
+/* *INDENT-ON* */
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (vlib_cli_show_ip6_command, static) = {
+  .path = "show ip6",
+  .short_help = "Internet protocol version 6 (IPv6) show commands",
+};
+/* *INDENT-ON* */
+
+/*?
+ * This command is used to add or delete IPv4 or IPv6 routes. All
+ * IP Addresses ('<em><dst-ip-addr>/<width></em>',
+ * '<em><next-hop-ip-addr></em>' and '<em><adj-hop-ip-addr></em>')
+ * can be IPv4 or IPv6, but all must be of the same form in a single
+ * command. To display the current set of routes, use the commands
+ * '<em>show ip fib</em>' and '<em>show ip6 fib</em>'.
+ *
+ * @cliexpar
+ * Example of how to add a straight forward static route:
+ * @cliexcmd{ip route add 6.0.1.2/32 via 6.0.0.1 GigabitEthernet2/0/0}
+ * Example of how to delete a straight forward static route:
+ * @cliexcmd{ip route del 6.0.1.2/32 via 6.0.0.1 GigabitEthernet2/0/0}
+ * Mainly for route add/del performance testing, one can add or delete
+ * multiple routes by adding 'count N' to the previous item:
+ * @cliexcmd{ip route add count 10 7.0.0.0/24 via 6.0.0.1 GigabitEthernet2/0/0}
+ * Add multiple routes for the same destination to create equal-cost multipath:
+ * @cliexcmd{ip route add 7.0.0.1/32 via 6.0.0.1 GigabitEthernet2/0/0}
+ * @cliexcmd{ip route add 7.0.0.1/32 via 6.0.0.2 GigabitEthernet2/0/0}
+ * For unequal-cost multipath, specify the desired weights. This
+ * combination of weights results in 3/4 of the traffic following the
+ * second path, 1/4 following the first path:
+ * @cliexcmd{ip route add 7.0.0.1/32 via 6.0.0.1 GigabitEthernet2/0/0 weight 1}
+ * @cliexcmd{ip route add 7.0.0.1/32 via 6.0.0.2 GigabitEthernet2/0/0 weight 3}
+ * To add a route to a particular FIB table (VRF), use:
+ * @cliexcmd{ip route add 172.16.24.0/24 table 7 via GigabitEthernet2/0/0}
+ ?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip_route_command, static) = {
+  .path = "ip route",
+  .short_help = "ip route [add|del] [count <n>] <dst-ip-addr>/<width> [table <table-id>] [via <next-hop-ip-addr> [<interface>] [weight <weight>]] | [via arp <interface> <adj-hop-ip-addr>] | [via drop|punt|local<id>|arp|classify <classify-idx>] [lookup in table <out-table-id>]",
+  .function = vnet_ip_route_cmd,
+  .is_mp_safe = 1,
+};
+/* *INDENT-ON* */
+
+/*
+ * The next two routines address a longstanding script hemorrhoid.
+ * Probing a v4 or v6 neighbor needs to appear to be synchronous,
+ * or dependent route-adds will simply fail.
+ */
+static clib_error_t *
+ip6_probe_neighbor_wait (vlib_main_t * vm, ip6_address_t * a, u32 sw_if_index,
+			 int retry_count)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  clib_error_t *e;
+  int i;
+  int resolved = 0;
+  uword event_type;
+  uword *event_data = 0;
+
+  ASSERT (vlib_in_process_context (vm));
+
+  if (retry_count > 0)
+    vnet_register_ip6_neighbor_resolution_event
+      (vnm, a, vlib_get_current_process (vm)->node_runtime.node_index,
+       1 /* event */ , 0 /* data */ );
+
+  for (i = 0; i < retry_count; i++)
+    {
+      /* The interface may be down, etc. */
+      e = ip6_probe_neighbor (vm, a, sw_if_index);
+
+      if (e)
+	return e;
+
+      vlib_process_wait_for_event_or_clock (vm, 1.0);
+      event_type = vlib_process_get_events (vm, &event_data);
+      switch (event_type)
+	{
+	case 1:		/* resolved... */
+	  vlib_cli_output (vm, "Resolved %U", format_ip6_address, a);
+	  resolved = 1;
+	  goto done;
+
+	case ~0:		/* timeout */
+	  break;
+
+	default:
+	  clib_warning ("unknown event_type %d", event_type);
+	}
+      vec_reset_length (event_data);
+    }
+
+done:
+
+  if (!resolved)
+    return clib_error_return (0, "Resolution failed for %U",
+			      format_ip6_address, a);
+  return 0;
+}
+
+static clib_error_t *
+ip4_probe_neighbor_wait (vlib_main_t * vm, ip4_address_t * a, u32 sw_if_index,
+			 int retry_count)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  clib_error_t *e;
+  int i;
+  int resolved = 0;
+  uword event_type;
+  uword *event_data = 0;
+
+  ASSERT (vlib_in_process_context (vm));
+
+  if (retry_count > 0)
+    vnet_register_ip4_arp_resolution_event
+      (vnm, a, vlib_get_current_process (vm)->node_runtime.node_index,
+       1 /* event */ , 0 /* data */ );
+
+  for (i = 0; i < retry_count; i++)
+    {
+      /* The interface may be down, etc. */
+      e = ip4_probe_neighbor (vm, a, sw_if_index);
+
+      if (e)
+	return e;
+
+      vlib_process_wait_for_event_or_clock (vm, 1.0);
+      event_type = vlib_process_get_events (vm, &event_data);
+      switch (event_type)
+	{
+	case 1:		/* resolved... */
+	  vlib_cli_output (vm, "Resolved %U", format_ip4_address, a);
+	  resolved = 1;
+	  goto done;
+
+	case ~0:		/* timeout */
+	  break;
+
+	default:
+	  clib_warning ("unknown event_type %d", event_type);
+	}
+      vec_reset_length (event_data);
+    }
+
+done:
+
+  vec_reset_length (event_data);
+
+  if (!resolved)
+    return clib_error_return (0, "Resolution failed for %U",
+			      format_ip4_address, a);
+  return 0;
+}
+
+static clib_error_t *
+probe_neighbor_address (vlib_main_t * vm,
+			unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  unformat_input_t _line_input, *line_input = &_line_input;
+  ip4_address_t a4;
+  ip6_address_t a6;
+  clib_error_t *error = 0;
+  u32 sw_if_index = ~0;
+  int retry_count = 3;
+  int is_ip4 = 1;
+  int address_set = 0;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat_user (line_input, unformat_vnet_sw_interface, vnm,
+			 &sw_if_index))
+	;
+      else if (unformat (line_input, "retry %d", &retry_count))
+	;
+
+      else if (unformat (line_input, "%U", unformat_ip4_address, &a4))
+	address_set++;
+      else if (unformat (line_input, "%U", unformat_ip6_address, &a6))
+	{
+	  address_set++;
+	  is_ip4 = 0;
+	}
+      else
+	return clib_error_return (0, "unknown input '%U'",
+				  format_unformat_error, line_input);
+    }
+
+  unformat_free (line_input);
+
+  if (sw_if_index == ~0)
+    return clib_error_return (0, "Interface required, not set.");
+  if (address_set == 0)
+    return clib_error_return (0, "ip address required, not set.");
+  if (address_set > 1)
+    return clib_error_return (0, "Multiple ip addresses not supported.");
+
+  if (is_ip4)
+    error = ip4_probe_neighbor_wait (vm, &a4, sw_if_index, retry_count);
+  else
+    error = ip6_probe_neighbor_wait (vm, &a6, sw_if_index, retry_count);
+
+  return error;
+}
+
+/*?
+ * The '<em>ip probe-neighbor</em>' command ARPs for IPv4 addresses or
+ * attempts IPv6 neighbor discovery depending on the supplied IP address
+ * format.
+ *
+ * @note This command will not immediately affect the indicated FIB; it
+ * is not suitable for use in establishing a FIB entry prior to adding
+ * recursive FIB entries. As in: don't use it in a script to probe a
+ * gateway prior to adding a default route. It won't work. Instead,
+ * configure a static ARP cache entry [see '<em>set ip arp</em>'], or
+ * a static IPv6 neighbor [see '<em>set ip6 neighbor</em>'].
+ *
+ * @cliexpar
+ * Example of probe for an IPv4 address:
+ * @cliexcmd{ip probe-neighbor GigabitEthernet2/0/0 172.16.1.2}
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip_probe_neighbor_command, static) = {
+  .path = "ip probe-neighbor",
+  .function = probe_neighbor_address,
+  .short_help = "ip probe-neighbor <interface> <ip4-addr> | <ip6-addr> [retry nn]",
+  .is_mp_safe = 1,
+};
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
-- 
cgit 1.2.3-korg


From 32e1c010b0c34fd0984f7fc45fae648a182025c5 Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Tue, 22 Nov 2016 17:07:28 +0000
Subject: IP Multicast FIB (mfib)

 - IPv[46] mfib tables with support for (*,G/m), (*,G) and (S,G) exact and longest prefix match
 - Replication represented via a new replicate DPO.
 - RPF configuration and data-plane checking
 - data-plane signals sent to listening control planes.

The functions of multicast forwarding entries differ from their unicast conterparts, so we introduce a new mfib_table_t and mfib_entry_t objects. However, we re-use the fib_path_list to resolve and build the entry's output list. the fib_path_list provides the service to construct a replicate DPO for multicast.

'make tests' is added to with two new suites; TEST=mfib, this is invocation of the CLI command 'test mfib' which deals with many path add/remove, flag set/unset scenarios, TEST=ip-mcast, data-plane forwarding tests.

Updated applications to use the new MIFB functions;
  - IPv6 NS/RA.
  - DHCPv6
 unit tests for these are undated accordingly.

Change-Id: I49ec37b01f1b170335a5697541c8fd30e6d3a961
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/scripts/vnet/mcast/ip4         |   22 +
 src/vat/api_format.c               |  173 +++++
 src/vnet.am                        |   35 +-
 src/vnet/adj/adj.c                 |   11 +-
 src/vnet/adj/adj.h                 |    1 -
 src/vnet/adj/adj_internal.h        |    2 +
 src/vnet/adj/adj_mcast.c           |  346 ++++++++++
 src/vnet/adj/adj_mcast.h           |   78 +++
 src/vnet/adj/adj_nbr.c             |    2 +-
 src/vnet/adj/adj_rewrite.c         |   53 --
 src/vnet/adj/adj_rewrite.h         |   49 --
 src/vnet/dhcpv6/proxy_node.c       |   46 +-
 src/vnet/dpo/dpo.c                 |    2 +
 src/vnet/dpo/dpo.h                 |    8 +-
 src/vnet/dpo/load_balance.c        |   13 +-
 src/vnet/dpo/load_balance.h        |    8 +
 src/vnet/dpo/replicate_dpo.c       |  759 ++++++++++++++++++++++
 src/vnet/dpo/replicate_dpo.h       |  143 +++++
 src/vnet/ethernet/arp.c            |   84 ++-
 src/vnet/ethernet/ethernet.h       |    2 +
 src/vnet/ethernet/interface.c      |   20 +
 src/vnet/fib/fib_attached_export.c |    4 +-
 src/vnet/fib/fib_entry.h           |    2 +-
 src/vnet/fib/fib_entry_delegate.c  |    3 +
 src/vnet/fib/fib_entry_src.c       |    4 +
 src/vnet/fib/fib_node.h            |    2 +
 src/vnet/fib/fib_path.c            |   91 ++-
 src/vnet/fib/fib_path_list.c       |    9 +-
 src/vnet/fib/fib_path_list.h       |    6 +
 src/vnet/fib/fib_table.c           |   69 +-
 src/vnet/fib/fib_table.h           |   16 +
 src/vnet/fib/fib_test.c            |  207 +++---
 src/vnet/fib/fib_types.c           |    4 +
 src/vnet/fib/fib_types.h           |   14 +
 src/vnet/fib/fib_urpf_list.c       |   20 +-
 src/vnet/fib/ip4_fib.c             |   50 +-
 src/vnet/fib/ip4_fib.h             |    9 +
 src/vnet/fib/ip6_fib.c             |  117 ++--
 src/vnet/fib/ip6_fib.h             |   11 +-
 src/vnet/fib/mpls_fib.c            |   17 +-
 src/vnet/fib/mpls_fib.h            |    9 +
 src/vnet/ip/ip.api                 |   53 ++
 src/vnet/ip/ip4.h                  |   24 +
 src/vnet/ip/ip4_forward.c          |  498 +++++----------
 src/vnet/ip/ip4_input.c            |    4 +-
 src/vnet/ip/ip6.h                  |   27 +
 src/vnet/ip/ip6_forward.c          |   83 ++-
 src/vnet/ip/ip6_input.c            |   43 +-
 src/vnet/ip/ip6_neighbor.c         |  134 ++--
 src/vnet/ip/ip_api.c               |  210 +++++++
 src/vnet/ip/lookup.c               |  171 +++++
 src/vnet/ip/lookup.h               |   82 +--
 src/vnet/mcast/mcast.c             |  565 -----------------
 src/vnet/mcast/mcast.h             |   50 --
 src/vnet/mcast/mcast_test.c        |  149 -----
 src/vnet/mfib/ip4_mfib.c           |  465 ++++++++++++++
 src/vnet/mfib/ip4_mfib.h           |   95 +++
 src/vnet/mfib/ip6_mfib.c           |  663 +++++++++++++++++++
 src/vnet/mfib/ip6_mfib.h           |  109 ++++
 src/vnet/mfib/mfib_entry.c         | 1096 ++++++++++++++++++++++++++++++++
 src/vnet/mfib/mfib_entry.h         |  172 +++++
 src/vnet/mfib/mfib_forward.c       |  512 +++++++++++++++
 src/vnet/mfib/mfib_itf.c           |  119 ++++
 src/vnet/mfib/mfib_itf.h           |   63 ++
 src/vnet/mfib/mfib_signal.c        |  201 ++++++
 src/vnet/mfib/mfib_signal.h        |   59 ++
 src/vnet/mfib/mfib_table.c         |  489 ++++++++++++++
 src/vnet/mfib/mfib_table.h         |  331 ++++++++++
 src/vnet/mfib/mfib_test.c          | 1225 ++++++++++++++++++++++++++++++++++++
 src/vnet/mfib/mfib_types.c         |  213 +++++++
 src/vnet/mfib/mfib_types.h         |  185 ++++++
 src/vnet/misc.c                    |    3 +
 src/vnet/rewrite.h                 |   31 +
 src/vnet/sr/sr.c                   |    4 +-
 src/vnet/util/radix.c              | 1104 ++++++++++++++++++++++++++++++++
 src/vnet/util/radix.h              |  147 +++++
 src/vnet/vxlan/vxlan.c             |  112 +++-
 src/vpp/api/api.c                  |   14 +-
 src/vppinfra.am                    |    2 +-
 src/vppinfra/dlist.h               |    2 +-
 src/vppinfra/format.c              |    8 +-
 src/vppinfra/format.h              |    4 +-
 src/vppinfra/unformat.c            |   16 +-
 test/test_dhcp.py                  |   16 -
 test/test_ip6.py                   |  131 ++--
 test/test_ip_mcast.py              |  612 ++++++++++++++++++
 test/test_mfib.py                  |   23 +
 test/vpp_interface.py              |    3 +-
 test/vpp_ip_route.py               |  101 ++-
 test/vpp_papi_provider.py          |   34 +-
 90 files changed, 11211 insertions(+), 1767 deletions(-)
 create mode 100644 src/scripts/vnet/mcast/ip4
 create mode 100644 src/vnet/adj/adj_mcast.c
 create mode 100644 src/vnet/adj/adj_mcast.h
 delete mode 100644 src/vnet/adj/adj_rewrite.c
 delete mode 100644 src/vnet/adj/adj_rewrite.h
 create mode 100644 src/vnet/dpo/replicate_dpo.c
 create mode 100644 src/vnet/dpo/replicate_dpo.h
 delete mode 100644 src/vnet/mcast/mcast.c
 delete mode 100644 src/vnet/mcast/mcast.h
 delete mode 100644 src/vnet/mcast/mcast_test.c
 create mode 100644 src/vnet/mfib/ip4_mfib.c
 create mode 100644 src/vnet/mfib/ip4_mfib.h
 create mode 100644 src/vnet/mfib/ip6_mfib.c
 create mode 100644 src/vnet/mfib/ip6_mfib.h
 create mode 100644 src/vnet/mfib/mfib_entry.c
 create mode 100644 src/vnet/mfib/mfib_entry.h
 create mode 100644 src/vnet/mfib/mfib_forward.c
 create mode 100644 src/vnet/mfib/mfib_itf.c
 create mode 100644 src/vnet/mfib/mfib_itf.h
 create mode 100644 src/vnet/mfib/mfib_signal.c
 create mode 100644 src/vnet/mfib/mfib_signal.h
 create mode 100644 src/vnet/mfib/mfib_table.c
 create mode 100644 src/vnet/mfib/mfib_table.h
 create mode 100644 src/vnet/mfib/mfib_test.c
 create mode 100644 src/vnet/mfib/mfib_types.c
 create mode 100644 src/vnet/mfib/mfib_types.h
 create mode 100644 src/vnet/util/radix.c
 create mode 100644 src/vnet/util/radix.h
 create mode 100644 test/test_ip_mcast.py
 create mode 100644 test/test_mfib.py

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/scripts/vnet/mcast/ip4 b/src/scripts/vnet/mcast/ip4
new file mode 100644
index 00000000..69f1ee00
--- /dev/null
+++ b/src/scripts/vnet/mcast/ip4
@@ -0,0 +1,22 @@
+packet-generator new {
+  name x
+  limit 1
+  node ip4-input
+  size 64-64
+  no-recycle
+  data {
+    ICMP: 1.0.0.2 -> 232.1.1.1
+    ICMP echo_request
+    incrementing 100
+  }
+}
+
+trace add pg-input 100
+loop create
+loop create
+set int state loop0 up
+set int state loop1 up
+
+ip mroute add 232.1.1.1 via pg0 Accept
+ip mroute add 232.1.1.1 via loop0 Forward
+ip mroute add 232.1.1.1 via loop1 Forward
diff --git a/src/vat/api_format.c b/src/vat/api_format.c
index b83313de..4cfe4a58 100644
--- a/src/vat/api_format.c
+++ b/src/vat/api_format.c
@@ -48,6 +48,7 @@
 #include <vnet/span/span.h>
 #include <vnet/policer/policer.h>
 #include <vnet/policer/police.h>
+#include <vnet/mfib/mfib_types.h>
 
 #include "vat/json_format.h"
 
@@ -505,6 +506,53 @@ unformat_flow_classify_table_type (unformat_input_t * input, va_list * va)
   return 1;
 }
 
+static const char *mfib_flag_names[] = MFIB_ENTRY_NAMES_SHORT;
+static const char *mfib_flag_long_names[] = MFIB_ENTRY_NAMES_LONG;
+static const char *mfib_itf_flag_long_names[] = MFIB_ITF_NAMES_LONG;
+static const char *mfib_itf_flag_names[] = MFIB_ITF_NAMES_SHORT;
+
+uword
+unformat_mfib_itf_flags (unformat_input_t * input, va_list * args)
+{
+  mfib_itf_flags_t old, *iflags = va_arg (*args, mfib_itf_flags_t *);
+  mfib_itf_attribute_t attr;
+
+  old = *iflags;
+  FOR_EACH_MFIB_ITF_ATTRIBUTE (attr)
+  {
+    if (unformat (input, mfib_itf_flag_long_names[attr]))
+      *iflags |= (1 << attr);
+  }
+  FOR_EACH_MFIB_ITF_ATTRIBUTE (attr)
+  {
+    if (unformat (input, mfib_itf_flag_names[attr]))
+      *iflags |= (1 << attr);
+  }
+
+  return (old == *iflags ? 0 : 1);
+}
+
+uword
+unformat_mfib_entry_flags (unformat_input_t * input, va_list * args)
+{
+  mfib_entry_flags_t old, *eflags = va_arg (*args, mfib_entry_flags_t *);
+  mfib_entry_attribute_t attr;
+
+  old = *eflags;
+  FOR_EACH_MFIB_ATTRIBUTE (attr)
+  {
+    if (unformat (input, mfib_flag_long_names[attr]))
+      *eflags |= (1 << attr);
+  }
+  FOR_EACH_MFIB_ATTRIBUTE (attr)
+  {
+    if (unformat (input, mfib_flag_names[attr]))
+      *eflags |= (1 << attr);
+  }
+
+  return (old == *eflags ? 0 : 1);
+}
+
 #if (VPP_API_TEST_BUILTIN==0)
 u8 *
 format_ip4_address (u8 * s, va_list * args)
@@ -3592,6 +3640,7 @@ _(bridge_domain_add_del_reply)                          \
 _(sw_interface_set_l2_xconnect_reply)                   \
 _(l2fib_add_del_reply)                                  \
 _(ip_add_del_route_reply)                               \
+_(ip_mroute_add_del_reply)                              \
 _(mpls_route_add_del_reply)                             \
 _(mpls_ip_bind_unbind_reply)                            \
 _(proxy_arp_add_del_reply)                              \
@@ -3792,6 +3841,7 @@ _(TAP_MODIFY_REPLY, tap_modify_reply)					\
 _(TAP_DELETE_REPLY, tap_delete_reply)					\
 _(SW_INTERFACE_TAP_DETAILS, sw_interface_tap_details)                   \
 _(IP_ADD_DEL_ROUTE_REPLY, ip_add_del_route_reply)			\
+_(IP_MROUTE_ADD_DEL_REPLY, ip_mroute_add_del_reply)			\
 _(MPLS_ROUTE_ADD_DEL_REPLY, mpls_route_add_del_reply)			\
 _(MPLS_IP_BIND_UNBIND_REPLY, mpls_ip_bind_unbind_reply)			\
 _(PROXY_ARP_ADD_DEL_REPLY, proxy_arp_add_del_reply)                     \
@@ -6383,6 +6433,126 @@ api_ip_add_del_route (vat_main_t * vam)
   return (vam->retval);
 }
 
+static int
+api_ip_mroute_add_del (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  vl_api_ip_mroute_add_del_t *mp;
+  f64 timeout;
+  u32 sw_if_index = ~0, vrf_id = 0;
+  u8 is_ipv6 = 0;
+  u8 is_local = 0;
+  u8 create_vrf_if_needed = 0;
+  u8 is_add = 1;
+  u8 address_set = 0;
+  u32 grp_address_length = 0;
+  ip4_address_t v4_grp_address, v4_src_address;
+  ip6_address_t v6_grp_address, v6_src_address;
+  mfib_itf_flags_t iflags = 0;
+  mfib_entry_flags_t eflags = 0;
+
+  /* Parse args required to build the message */
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "sw_if_index %d", &sw_if_index))
+	;
+      else if (unformat (i, "%U %U",
+			 unformat_ip4_address, &v4_src_address,
+			 unformat_ip4_address, &v4_grp_address))
+	{
+	  grp_address_length = 64;
+	  address_set = 1;
+	  is_ipv6 = 0;
+	}
+      else if (unformat (i, "%U %U",
+			 unformat_ip6_address, &v6_src_address,
+			 unformat_ip6_address, &v6_grp_address))
+	{
+	  grp_address_length = 256;
+	  address_set = 1;
+	  is_ipv6 = 1;
+	}
+      else if (unformat (i, "%U", unformat_ip4_address, &v4_grp_address))
+	{
+	  memset (&v4_src_address, 0, sizeof (v4_src_address));
+	  grp_address_length = 32;
+	  address_set = 1;
+	  is_ipv6 = 0;
+	}
+      else if (unformat (i, "%U", unformat_ip6_address, &v6_grp_address))
+	{
+	  memset (&v6_src_address, 0, sizeof (v6_src_address));
+	  grp_address_length = 128;
+	  address_set = 1;
+	  is_ipv6 = 1;
+	}
+      else if (unformat (i, "/%d", &grp_address_length))
+	;
+      else if (unformat (i, "local"))
+	{
+	  is_local = 1;
+	}
+      else if (unformat (i, "del"))
+	is_add = 0;
+      else if (unformat (i, "add"))
+	is_add = 1;
+      else if (unformat (i, "vrf %d", &vrf_id))
+	;
+      else if (unformat (i, "create-vrf"))
+	create_vrf_if_needed = 1;
+      else if (unformat (i, "%U", unformat_mfib_itf_flags, &iflags))
+	;
+      else if (unformat (i, "%U", unformat_mfib_entry_flags, &eflags))
+	;
+      else
+	{
+	  clib_warning ("parse error '%U'", format_unformat_error, i);
+	  return -99;
+	}
+    }
+
+  if (address_set == 0)
+    {
+      errmsg ("missing addresses\n");
+      return -99;
+    }
+
+  /* Construct the API message */
+  M (IP_MROUTE_ADD_DEL, ip_mroute_add_del);
+
+  mp->next_hop_sw_if_index = ntohl (sw_if_index);
+  mp->table_id = ntohl (vrf_id);
+  mp->create_vrf_if_needed = create_vrf_if_needed;
+
+  mp->is_add = is_add;
+  mp->is_ipv6 = is_ipv6;
+  mp->is_local = is_local;
+  mp->itf_flags = ntohl (iflags);
+  mp->entry_flags = ntohl (eflags);
+  mp->grp_address_length = grp_address_length;
+  mp->grp_address_length = ntohs (mp->grp_address_length);
+
+  if (is_ipv6)
+    {
+      clib_memcpy (mp->grp_address, &v6_grp_address, sizeof (v6_grp_address));
+      clib_memcpy (mp->src_address, &v6_src_address, sizeof (v6_src_address));
+    }
+  else
+    {
+      clib_memcpy (mp->grp_address, &v4_grp_address, sizeof (v4_grp_address));
+      clib_memcpy (mp->src_address, &v4_src_address, sizeof (v4_src_address));
+
+    }
+
+  /* send it... */
+  S;
+  /* Wait for a reply... */
+  W;
+
+  /* Return the good/bad news */
+  return (vam->retval);
+}
+
 static int
 api_mpls_route_add_del (vat_main_t * vam)
 {
@@ -17512,6 +17682,9 @@ _(ip_add_del_route,                                                     \
   "[<intfc> | sw_if_index <id>] [resolve-attempts <n>]\n"               \
   "[weight <n>] [drop] [local] [classify <n>] [del]\n"                  \
   "[multipath] [count <n>]")                                            \
+_(ip_mroute_add_del,                                                    \
+  "<src> <grp>/<mask> [table-id <n>]\n"                                 \
+  "[<intfc> | sw_if_index <id>] [local] [del]")                         \
 _(mpls_route_add_del,                                                   \
   "<label> <eos> via <addr> [table-id <n>]\n"                           \
   "[<intfc> | sw_if_index <id>] [resolve-attempts <n>]\n"               \
diff --git a/src/vnet.am b/src/vnet.am
index c6922493..6d043e32 100644
--- a/src/vnet.am
+++ b/src/vnet.am
@@ -946,16 +946,15 @@ nobase_include_HEADERS +=			\
 
 libvnet_la_SOURCES +=				\
   vnet/adj/adj_nbr.c				\
-  vnet/adj/adj_rewrite.c			\
   vnet/adj/adj_glean.c   			\
   vnet/adj/adj_midchain.c   			\
+  vnet/adj/adj_mcast.c   			\
   vnet/adj/adj_l2.c      			\
   vnet/adj/adj.c
 
 nobase_include_HEADERS +=			\
   vnet/adj/adj.h				\
   vnet/adj/adj_types.h				\
-  vnet/adj/adj_rewrite.h			\
   vnet/adj/adj_glean.h  			\
   vnet/adj/adj_nbr.h
 
@@ -971,8 +970,9 @@ libvnet_la_SOURCES +=				\
   vnet/dpo/receive_dpo.c			\
   vnet/dpo/load_balance.c			\
   vnet/dpo/load_balance_map.c			\
-  vnet/dpo/lookup_dpo.c				\
-  vnet/dpo/classify_dpo.c			\
+  vnet/dpo/lookup_dpo.c   			\
+  vnet/dpo/classify_dpo.c   			\
+  vnet/dpo/replicate_dpo.c   			\
   vnet/dpo/mpls_label_dpo.c
 
 nobase_include_HEADERS +=			\
@@ -985,6 +985,33 @@ nobase_include_HEADERS +=			\
   vnet/dpo/ip_null_dpo.h			\
   vnet/dpo/dpo.h
 
+########################################
+# Multicast FIB
+########################################
+
+libvnet_la_SOURCES +=				\
+  vnet/mfib/mfib_test.c                         \
+  vnet/mfib/mfib_forward.c                      \
+  vnet/mfib/ip4_mfib.c                          \
+  vnet/mfib/ip6_mfib.c                          \
+  vnet/mfib/mfib_types.c                        \
+  vnet/mfib/mfib_signal.c                       \
+  vnet/mfib/mfib_itf.c                          \
+  vnet/mfib/mfib_entry.c                        \
+  vnet/mfib/mfib_table.c
+
+nobase_include_HEADERS +=			\
+  vnet/mfib/ip4_mfib.h                          \
+  vnet/mfib/mfib_types.h                        \
+  vnet/mfib/mfib_table.h
+
+########################################
+# Utilities
+########################################
+
+libvnet_la_SOURCES +=                    \
+  vnet/util/radix.c
+
 ########################################
 # Plugin client library
 ########################################
diff --git a/src/vnet/adj/adj.c b/src/vnet/adj/adj.c
index d0be0f0e..a99f173f 100644
--- a/src/vnet/adj/adj.c
+++ b/src/vnet/adj/adj.c
@@ -17,6 +17,7 @@
 #include <vnet/adj/adj_internal.h>
 #include <vnet/adj/adj_glean.h>
 #include <vnet/adj/adj_midchain.h>
+#include <vnet/adj/adj_mcast.h>
 #include <vnet/fib/fib_node_list.h>
 
 /*
@@ -58,8 +59,6 @@ adj_alloc (fib_protocol_t proto)
                                    adj_get_index(adj));
 
     adj->rewrite_header.sw_if_index = ~0;
-    adj->mcast_group_index = ~0;
-    adj->saved_lookup_next_index = 0;
     adj->n_adj = 1;
     adj->lookup_next_index = 0;
 
@@ -116,6 +115,9 @@ format_ip_adjacency (u8 * s, va_list * args)
     case IP_LOOKUP_NEXT_MIDCHAIN:
 	s = format (s, "%U", format_adj_midchain, adj_index, 2);
 	break;
+    case IP_LOOKUP_NEXT_MCAST:
+	s = format (s, "%U", format_adj_mcast, adj_index, 0);
+	break;
     default:
 	break;
     }
@@ -179,6 +181,10 @@ adj_last_lock_gone (ip_adjacency_t *adj)
 	adj_glean_remove(adj->ia_nh_proto,
 			 adj->rewrite_header.sw_if_index);
 	break;
+    case IP_LOOKUP_NEXT_MCAST:
+	adj_mcast_remove(adj->ia_nh_proto,
+			 adj->rewrite_header.sw_if_index);
+	break;
     default:
 	/*
 	 * type not stored in any DB from which we need to remove it
@@ -350,6 +356,7 @@ adj_module_init (vlib_main_t * vm)
     adj_nbr_module_init();
     adj_glean_module_init();
     adj_midchain_module_init();
+    adj_mcast_module_init();
 
     /*
      * one special adj to reserve index 0
diff --git a/src/vnet/adj/adj.h b/src/vnet/adj/adj.h
index e85625db..29bae673 100644
--- a/src/vnet/adj/adj.h
+++ b/src/vnet/adj/adj.h
@@ -45,7 +45,6 @@
 #include <vnet/ip/lookup.h>
 #include <vnet/adj/adj_types.h>
 #include <vnet/adj/adj_nbr.h>
-#include <vnet/adj/adj_rewrite.h>
 #include <vnet/adj/adj_glean.h>
 
 /**
diff --git a/src/vnet/adj/adj_internal.h b/src/vnet/adj/adj_internal.h
index 833bc7c9..ece59121 100644
--- a/src/vnet/adj/adj_internal.h
+++ b/src/vnet/adj/adj_internal.h
@@ -100,5 +100,7 @@ extern void adj_nbr_remove(adj_index_t ai,
 			   u32 sw_if_index);
 extern void adj_glean_remove(fib_protocol_t proto,
 			     u32 sw_if_index);
+extern void adj_mcast_remove(fib_protocol_t proto,
+			     u32 sw_if_index);
 
 #endif
diff --git a/src/vnet/adj/adj_mcast.c b/src/vnet/adj/adj_mcast.c
new file mode 100644
index 00000000..1345aedb
--- /dev/null
+++ b/src/vnet/adj/adj_mcast.c
@@ -0,0 +1,346 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/adj/adj.h>
+#include <vnet/adj/adj_internal.h>
+#include <vnet/fib/fib_walk.h>
+#include <vnet/ip/ip.h>
+
+/*
+ * The 'DB' of all mcast adjs.
+ * There is only one mcast per-interface per-protocol, so this is a per-interface
+ * vector
+ */
+static adj_index_t *adj_mcasts[FIB_PROTOCOL_MAX];
+
+static u32
+adj_get_mcast_node (fib_protocol_t proto)
+{
+    switch (proto) {
+    case FIB_PROTOCOL_IP4:
+	return (ip4_rewrite_mcast_node.index);
+    case FIB_PROTOCOL_IP6:
+	return (ip6_rewrite_mcast_node.index);
+    case FIB_PROTOCOL_MPLS:
+	break;
+    }
+    ASSERT(0);
+    return (0);
+}
+
+/*
+ * adj_mcast_add_or_lock
+ *
+ * The next_hop address here is used for source address selection in the DP.
+ * The mcast adj is added to an interface's connected prefix, the next-hop
+ * passed here is the local prefix on the same interface.
+ */
+adj_index_t
+adj_mcast_add_or_lock (fib_protocol_t proto,
+                       vnet_link_t link_type,
+		       u32 sw_if_index)
+{
+    ip_adjacency_t * adj;
+
+    vec_validate_init_empty(adj_mcasts[proto], sw_if_index, ADJ_INDEX_INVALID);
+
+    if (ADJ_INDEX_INVALID == adj_mcasts[proto][sw_if_index])
+    {
+        vnet_main_t *vnm;
+
+        vnm = vnet_get_main();
+	adj = adj_alloc(proto);
+
+	adj->lookup_next_index = IP_LOOKUP_NEXT_MCAST;
+	adj->ia_nh_proto = proto;
+	adj->ia_link = link_type;
+	adj_mcasts[proto][sw_if_index] = adj_get_index(adj);
+        adj_lock(adj_get_index(adj));
+
+	vnet_rewrite_init(vnm, sw_if_index,
+			  adj_get_mcast_node(proto),
+			  vnet_tx_node_index_for_sw_interface(vnm, sw_if_index),
+			  &adj->rewrite_header);
+
+	/*
+	 * we need a rewrite where the destination IP address is converted
+	 * to the appropriate link-layer address. This is interface specific.
+	 * So ask the interface to do it.
+	 */
+	vnet_update_adjacency_for_sw_interface(vnm, sw_if_index,
+                                               adj_get_index(adj));
+    }
+    else
+    {
+	adj = adj_get(adj_mcasts[proto][sw_if_index]);
+        adj_lock(adj_get_index(adj));
+    }
+
+    return (adj_get_index(adj));
+}
+
+/**
+ * adj_mcast_update_rewrite
+ *
+ * Update the adjacency's rewrite string. A NULL string implies the
+ * rewirte is reset (i.e. when ARP/ND etnry is gone).
+ * NB: the adj being updated may be handling traffic in the DP.
+ */
+void
+adj_mcast_update_rewrite (adj_index_t adj_index,
+                          u8 *rewrite)
+{
+    ip_adjacency_t *adj;
+
+    ASSERT(ADJ_INDEX_INVALID != adj_index);
+
+    adj = adj_get(adj_index);
+
+    /*
+     * update the adj's rewrite string and build the arc
+     * from the rewrite node to the interface's TX node
+     */
+    adj_nbr_update_rewrite_internal(adj, IP_LOOKUP_NEXT_MCAST,
+                                    adj_get_mcast_node(adj->ia_nh_proto),
+                                    vnet_tx_node_index_for_sw_interface(
+                                        vnet_get_main(),
+                                        adj->rewrite_header.sw_if_index),
+                                    rewrite);
+}
+
+void
+adj_mcast_remove (fib_protocol_t proto,
+		  u32 sw_if_index)
+{
+    ASSERT(sw_if_index < vec_len(adj_mcasts[proto]));
+
+    adj_mcasts[proto][sw_if_index] = ADJ_INDEX_INVALID;
+}
+
+static clib_error_t *
+adj_mcast_interface_state_change (vnet_main_t * vnm,
+				  u32 sw_if_index,
+				  u32 flags)
+{
+    /*
+     * for each mcast on the interface trigger a walk back to the children
+     */
+    fib_protocol_t proto;
+    ip_adjacency_t *adj;
+
+
+    for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
+    {
+	if (sw_if_index >= vec_len(adj_mcasts[proto]) ||
+	    ADJ_INDEX_INVALID == adj_mcasts[proto][sw_if_index])
+	    continue;
+
+	adj = adj_get(adj_mcasts[proto][sw_if_index]);
+
+	fib_node_back_walk_ctx_t bw_ctx = {
+	    .fnbw_reason = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP ?
+			    FIB_NODE_BW_REASON_FLAG_INTERFACE_UP :
+			    FIB_NODE_BW_REASON_FLAG_INTERFACE_DOWN),
+	};
+
+	fib_walk_sync(FIB_NODE_TYPE_ADJ, adj_get_index(adj), &bw_ctx);
+    }
+
+    return (NULL);
+}
+
+VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION(adj_mcast_interface_state_change);
+
+/**
+ * @brief Invoked on each SW interface of a HW interface when the
+ * HW interface state changes
+ */
+static void
+adj_nbr_hw_sw_interface_state_change (vnet_main_t * vnm,
+                                      u32 sw_if_index,
+                                      void *arg)
+{
+    adj_mcast_interface_state_change(vnm, sw_if_index, (uword) arg);
+}
+
+/**
+ * @brief Registered callback for HW interface state changes
+ */
+static clib_error_t *
+adj_mcast_hw_interface_state_change (vnet_main_t * vnm,
+                                     u32 hw_if_index,
+                                     u32 flags)
+{
+    /*
+     * walk SW interfaces on the HW
+     */
+    uword sw_flags;
+
+    sw_flags = ((flags & VNET_HW_INTERFACE_FLAG_LINK_UP) ?
+                VNET_SW_INTERFACE_FLAG_ADMIN_UP :
+                0);
+
+    vnet_hw_interface_walk_sw(vnm, hw_if_index,
+                              adj_nbr_hw_sw_interface_state_change,
+                              (void*) sw_flags);
+
+    return (NULL);
+}
+
+VNET_HW_INTERFACE_LINK_UP_DOWN_FUNCTION(
+    adj_mcast_hw_interface_state_change);
+
+static clib_error_t *
+adj_mcast_interface_delete (vnet_main_t * vnm,
+			    u32 sw_if_index,
+			    u32 is_add)
+{
+    /*
+     * for each mcast on the interface trigger a walk back to the children
+     */
+    fib_protocol_t proto;
+    ip_adjacency_t *adj;
+
+    if (is_add)
+    {
+	/*
+	 * not interested in interface additions. we will not back walk
+	 * to resolve paths through newly added interfaces. Why? The control
+	 * plane should have the brains to add interfaces first, then routes.
+	 * So the case where there are paths with a interface that matches
+	 * one just created is the case where the path resolved through an
+	 * interface that was deleted, and still has not been removed. The
+	 * new interface added, is NO GUARANTEE that the interface being
+	 * added now, even though it may have the same sw_if_index, is the
+	 * same interface that the path needs. So tough!
+	 * If the control plane wants these routes to resolve it needs to
+	 * remove and add them again.
+	 */
+	return (NULL);
+    }
+
+    for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
+    {
+	if (sw_if_index >= vec_len(adj_mcasts[proto]) ||
+	    ADJ_INDEX_INVALID == adj_mcasts[proto][sw_if_index])
+	    continue;
+
+	adj = adj_get(adj_mcasts[proto][sw_if_index]);
+
+	fib_node_back_walk_ctx_t bw_ctx = {
+	    .fnbw_reason =  FIB_NODE_BW_REASON_FLAG_INTERFACE_DELETE,
+	};
+
+	fib_walk_sync(FIB_NODE_TYPE_ADJ, adj_get_index(adj), &bw_ctx);
+    }
+
+    return (NULL);
+}
+
+VNET_SW_INTERFACE_ADD_DEL_FUNCTION(adj_mcast_interface_delete);
+
+u8*
+format_adj_mcast (u8* s, va_list *ap)
+{
+    index_t index = va_arg(*ap, index_t);
+    CLIB_UNUSED(u32 indent) = va_arg(*ap, u32);
+    vnet_main_t * vnm = vnet_get_main();
+    ip_adjacency_t * adj = adj_get(index);
+
+    s = format(s, "%U-mcast: ",
+               format_fib_protocol, adj->ia_nh_proto);
+    s = format (s, "%U",
+		format_vnet_rewrite,
+		vnm->vlib_main, &adj->rewrite_header,
+                sizeof (adj->rewrite_data), 0);
+
+    return (s);
+}
+
+
+static void
+adj_dpo_lock (dpo_id_t *dpo)
+{
+    adj_lock(dpo->dpoi_index);
+}
+static void
+adj_dpo_unlock (dpo_id_t *dpo)
+{
+    adj_unlock(dpo->dpoi_index);
+}
+
+const static dpo_vft_t adj_mcast_dpo_vft = {
+    .dv_lock = adj_dpo_lock,
+    .dv_unlock = adj_dpo_unlock,
+    .dv_format = format_adj_mcast,
+};
+
+/**
+ * @brief The per-protocol VLIB graph nodes that are assigned to a mcast
+ *        object.
+ *
+ * this means that these graph nodes are ones from which a mcast is the
+ * parent object in the DPO-graph.
+ */
+const static char* const adj_mcast_ip4_nodes[] =
+{
+    "ip4-rewrite-mcast",
+    NULL,
+};
+const static char* const adj_mcast_ip6_nodes[] =
+{
+    "ip6-rewrite-mcast",
+    NULL,
+};
+
+const static char* const * const adj_mcast_nodes[DPO_PROTO_NUM] =
+{
+    [DPO_PROTO_IP4]  = adj_mcast_ip4_nodes,
+    [DPO_PROTO_IP6]  = adj_mcast_ip6_nodes,
+    [DPO_PROTO_MPLS] = NULL,
+};
+
+/**
+ * @brief Return the size of the adj DB.
+ * This is only for testing purposes so an efficient implementation is not needed
+ */
+u32
+adj_mcast_db_size (void)
+{
+    u32 n_adjs, sw_if_index;
+    fib_protocol_t proto;
+
+    n_adjs = 0;
+    for (proto = FIB_PROTOCOL_IP4; proto <= FIB_PROTOCOL_IP6; proto++)
+    {
+        for (sw_if_index = 0;
+             sw_if_index < vec_len(adj_mcasts[proto]);
+             sw_if_index++)
+        {
+            if (ADJ_INDEX_INVALID != adj_mcasts[proto][sw_if_index])
+            {
+                n_adjs++;
+            }
+        }
+    }
+    
+    return (n_adjs);
+}
+
+void
+adj_mcast_module_init (void)
+{
+    dpo_register(DPO_ADJACENCY_MCAST, &adj_mcast_dpo_vft, adj_mcast_nodes);
+}
diff --git a/src/vnet/adj/adj_mcast.h b/src/vnet/adj/adj_mcast.h
new file mode 100644
index 00000000..21c5a141
--- /dev/null
+++ b/src/vnet/adj/adj_mcast.h
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @brief Mcast Adjacency
+ *
+ * The multicast adjacency forwards IP traffic on an interface toward a multicast
+ * group address. This is a different type of adjacency to a unicast adjacency
+ * since the application of the MAC header is different, and so the VLIB node
+ * visited is also different. DPO types have different VLIB nodes.
+ */
+
+#ifndef __ADJ_MCAST_H__
+#define __ADJ_MCAST_H__
+
+#include <vnet/adj/adj_types.h>
+
+/**
+ * @brief
+ *  Add (and lock) a new or lock an existing mcast adjacency
+ *
+ * @param proto
+ *  The protocol for the neighbours that we wish to mcast
+ *
+ * @param link_type
+ *  A description of the protocol of the packets that will forward
+ *  through this adj. On an ethernet interface this is the MAC header's
+ *  ether-type
+ *
+ * @param sw_if_index
+ *  The interface on which to mcast
+ */
+extern adj_index_t adj_mcast_add_or_lock(fib_protocol_t proto,
+                                         vnet_link_t link_type,
+					 u32 sw_if_index);
+
+/**
+ * @brief
+ *  Update the rewrite string for an existing adjacecny.
+ *
+ * @param
+ *  The index of the adj to update
+ *
+ * @param
+ *  The new rewrite
+ */
+extern void adj_mcast_update_rewrite(adj_index_t adj_index,
+                                     u8 *rewrite);
+
+/**
+ * @brief Format/display a mcast adjacency.
+ */
+extern u8* format_adj_mcast(u8* s, va_list *ap);
+
+/**
+ * @brief Get the sze of the mcast adj DB. Test purposes only.
+ */
+extern u32 adj_mcast_db_size(void);
+
+/**
+ * @brief
+ *  Module initialisation
+ */
+extern void adj_mcast_module_init(void);
+
+#endif
diff --git a/src/vnet/adj/adj_nbr.c b/src/vnet/adj/adj_nbr.c
index 1344bb67..9e8073d3 100644
--- a/src/vnet/adj/adj_nbr.c
+++ b/src/vnet/adj/adj_nbr.c
@@ -162,7 +162,7 @@ adj_nbr_alloc (fib_protocol_t nh_proto,
 }
 
 /*
- * adj_add_for_nbr
+ * adj_nbr_add_or_lock
  *
  * Add an adjacency for the neighbour requested.
  *
diff --git a/src/vnet/adj/adj_rewrite.c b/src/vnet/adj/adj_rewrite.c
deleted file mode 100644
index 7d792557..00000000
--- a/src/vnet/adj/adj_rewrite.c
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <vnet/adj/adj.h>
-#include <vnet/adj/adj_internal.h>
-
-/**
- * adj_rewrite_add_and_lock
- *
- * A rewrite sub-type has the rewrite string provided, but no key
- */
-adj_index_t
-adj_rewrite_add_and_lock (fib_protocol_t nh_proto,
-			  vnet_link_t link_type,
-			  u32 sw_if_index,
-			  u8 *rewrite)
-{
-    ip_adjacency_t *adj;
-
-    adj = adj_alloc(nh_proto);
-
-    adj->lookup_next_index = IP_LOOKUP_NEXT_REWRITE;
-    memset(&adj->sub_type.nbr.next_hop, 0, sizeof(adj->sub_type.nbr.next_hop));
-    adj->ia_link = link_type;
-    adj->ia_nh_proto = nh_proto;
-    adj->rewrite_header.sw_if_index = sw_if_index;
-
-    ASSERT(NULL != rewrite);
-
-    vnet_rewrite_for_sw_interface(vnet_get_main(),
-				  link_type,
-				  adj->rewrite_header.sw_if_index,
-				  adj_get_rewrite_node(link_type),
-				  rewrite,
-				  &adj->rewrite_header,
-				  sizeof (adj->rewrite_data));
-
-    adj_lock(adj_get_index(adj));
-
-    return (adj_get_index(adj));
-}
diff --git a/src/vnet/adj/adj_rewrite.h b/src/vnet/adj/adj_rewrite.h
deleted file mode 100644
index 25e6bba8..00000000
--- a/src/vnet/adj/adj_rewrite.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright (c) 2016 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @brief
- * A rewrite adjacency has no key, and thus cannot be 'found' from the
- * FIB resolution code. the client therefore needs to maange these adjacencies
- */
-
-#ifndef __ADJ_REWRITE_H__
-#define __ADJ_REWRITE_H__
-
-#include <vnet/adj/adj_types.h>
-
-/**
- * @brief
- *  Add (and lock) a new or lock an existing neighbour adjacency
- *
- * @param nh_proto
- *  The protocol for the next-hop address (v4 or v6)
- *
- * @param link_type
- *  A description of the protocol of the packets that will forward
- *  through this adj. On an ethernet interface this is the MAC header's
- *  ether-type
- *
- * @param sw_if_index
- *  The interface on which the peer resides
- *
- * @param rewrite
- *  The rewrite to prepend to packets
- */
-extern adj_index_t adj_rewrite_add_and_lock(fib_protocol_t nh_proto,
-					    vnet_link_t link_type,
-					    u32 sw_if_index,
-					    u8 *rewrite);
-
-#endif
diff --git a/src/vnet/dhcpv6/proxy_node.c b/src/vnet/dhcpv6/proxy_node.c
index 4dd2239f..77afef2a 100644
--- a/src/vnet/dhcpv6/proxy_node.c
+++ b/src/vnet/dhcpv6/proxy_node.c
@@ -19,6 +19,8 @@
 #include <vnet/pg/pg.h>
 #include <vnet/dhcpv6/proxy.h>
 #include <vnet/fib/ip6_fib.h>
+#include <vnet/mfib/mfib_table.h>
+#include <vnet/mfib/ip6_mfib.h>
 
 static char * dhcpv6_proxy_error_strings[] = {
 #define dhcpv6_proxy_error(n,s) s,
@@ -819,7 +821,7 @@ int dhcpv6_proxy_set_server_2 (ip6_address_t *addr, ip6_address_t *src_address,
   u32 server_fib_index = 0;
   u32 rx_fib_index = 0;
 
-  rx_fib_index = ip6_fib_table_find_or_create_and_lock(rx_fib_id);
+  rx_fib_index = ip6_mfib_table_find_or_create_and_lock(rx_fib_id);
   server_fib_index = ip6_fib_table_find_or_create_and_lock(server_fib_id);
 
   if (is_del)
@@ -848,8 +850,10 @@ int dhcpv6_proxy_set_server_2 (ip6_address_t *addr, ip6_address_t *src_address,
   if (rx_fib_id == 0)
     {
       server = pool_elt_at_index (dm->dhcp6_servers, 0);
-
-      goto initialize_it;
+      if (server->valid)
+          goto reconfigure_it;
+      else
+          goto initialize_it;
     }
 
   if (rx_fib_index < vec_len(dm->dhcp6_server_index_by_rx_fib_index))
@@ -866,6 +870,42 @@ int dhcpv6_proxy_set_server_2 (ip6_address_t *addr, ip6_address_t *src_address,
   pool_get (dm->dhcp6_servers, server);
 
   initialize_it:
+  {
+      const mfib_prefix_t all_dhcp_servers = {
+          .fp_len = 128,
+          .fp_proto = FIB_PROTOCOL_IP6,
+          .fp_grp_addr = {
+              .ip6 = dm->all_dhcpv6_server_relay_agent_address,
+          }
+      };
+      const fib_route_path_t path_for_us = {
+          .frp_proto = FIB_PROTOCOL_IP6,
+          .frp_addr = zero_addr,
+          .frp_sw_if_index = 0xffffffff,
+          .frp_fib_index = ~0,
+          .frp_weight = 0,
+          .frp_flags = FIB_ROUTE_PATH_LOCAL,
+      };
+      mfib_table_entry_path_update(rx_fib_index,
+                                   &all_dhcp_servers,
+                                   MFIB_SOURCE_DHCP,
+                                   &path_for_us,
+                                   MFIB_ITF_FLAG_FORWARD);
+      /*
+       * Each interface that is enabled in this table, needs to be added
+       * as an accepting interface, but this is not easily doable in VPP.
+       * So we cheat. Add a flag to the entry that indicates accept form
+       * any interface.
+       * We will still only accept on v6 enabled interfaces, since the input
+       * feature ensures this.
+       */
+      mfib_table_entry_update(rx_fib_index,
+                              &all_dhcp_servers,
+                              MFIB_SOURCE_DHCP,
+                              MFIB_ENTRY_FLAG_ACCEPT_ALL_ITF);
+  }
+
+reconfigure_it:
 
   copy_ip6_address(&server->dhcp6_server, addr);
   copy_ip6_address(&server->dhcp6_src_address, src_address);
diff --git a/src/vnet/dpo/dpo.c b/src/vnet/dpo/dpo.c
index 688d2892..cc2fa0eb 100644
--- a/src/vnet/dpo/dpo.c
+++ b/src/vnet/dpo/dpo.c
@@ -36,6 +36,7 @@
 #include <vnet/dpo/punt_dpo.h>
 #include <vnet/dpo/classify_dpo.h>
 #include <vnet/dpo/ip_null_dpo.h>
+#include <vnet/dpo/replicate_dpo.h>
 
 /**
  * Array of char* names for the DPO types and protos
@@ -449,6 +450,7 @@ dpo_module_init (vlib_main_t * vm)
     classify_dpo_module_init();
     lookup_dpo_module_init();
     ip_null_dpo_module_init();
+    replicate_module_init();
 
     return (NULL);
 }
diff --git a/src/vnet/dpo/dpo.h b/src/vnet/dpo/dpo.h
index 1efcbc88..aff4e1b8 100644
--- a/src/vnet/dpo/dpo.h
+++ b/src/vnet/dpo/dpo.h
@@ -100,15 +100,18 @@ typedef enum dpo_type_t_ {
      * @brief load-balancing over a choice of [un]equal cost paths
      */
     DPO_LOAD_BALANCE,
+    DPO_REPLICATE,
     DPO_ADJACENCY,
     DPO_ADJACENCY_INCOMPLETE,
     DPO_ADJACENCY_MIDCHAIN,
     DPO_ADJACENCY_GLEAN,
+    DPO_ADJACENCY_MCAST,
     DPO_RECEIVE,
     DPO_LOOKUP,
     DPO_LISP_CP,
     DPO_CLASSIFY,
     DPO_MPLS_LABEL,
+    DPO_MFIB_ENTRY,
     DPO_LAST,
 } __attribute__((packed)) dpo_type_t;
 
@@ -123,12 +126,15 @@ typedef enum dpo_type_t_ {
     [DPO_ADJACENCY_INCOMPLETE] = "dpo-adjacency-incomplete",	\
     [DPO_ADJACENCY_MIDCHAIN] = "dpo-adjacency-midcahin",	\
     [DPO_ADJACENCY_GLEAN] = "dpo-glean",	\
+    [DPO_ADJACENCY_MCAST] = "dpo-adj-mcast",	\
     [DPO_RECEIVE] = "dpo-receive",	\
     [DPO_LOOKUP] = "dpo-lookup",	\
     [DPO_LOAD_BALANCE] = "dpo-load-balance",	\
+    [DPO_REPLICATE] = "dpo-replicate",	\
     [DPO_LISP_CP] = "dpo-lisp-cp",	\
     [DPO_CLASSIFY] = "dpo-classify",	\
-    [DPO_MPLS_LABEL] = "dpo-mpls-label"	\
+    [DPO_MPLS_LABEL] = "dpo-mpls-label", \
+    [DPO_MFIB_ENTRY] = "dpo-mfib_entry"	\
 }
 
 /**
diff --git a/src/vnet/dpo/load_balance.c b/src/vnet/dpo/load_balance.c
index e70a7a30..f11b4e4d 100644
--- a/src/vnet/dpo/load_balance.c
+++ b/src/vnet/dpo/load_balance.c
@@ -238,6 +238,17 @@ load_balance_is_drop (const dpo_id_t *dpo)
     return (0);
 }
 
+void
+load_balance_set_fib_entry_flags (index_t lbi,
+                                  fib_entry_flag_t flags)
+{
+    load_balance_t *lb;
+
+    lb = load_balance_get(lbi);
+    lb->lb_fib_entry_flags = flags;
+}
+
+
 void
 load_balance_set_urpf (index_t lbi,
 		       index_t urpf)
@@ -683,7 +694,7 @@ load_balance_multipath_update (const dpo_id_t *dpo,
                                           buckets,
                                           n_buckets);
 
-                for (ii = old_n_buckets-n_buckets; ii < old_n_buckets; ii++)
+                for (ii = n_buckets; ii < old_n_buckets; ii++)
                 {
                     dpo_reset(&buckets[ii]);
                 }
diff --git a/src/vnet/dpo/load_balance.h b/src/vnet/dpo/load_balance.h
index 17996536..b901c5be 100644
--- a/src/vnet/dpo/load_balance.h
+++ b/src/vnet/dpo/load_balance.h
@@ -36,6 +36,7 @@
 #include <vnet/ip/lookup.h>
 #include <vnet/dpo/dpo.h>
 #include <vnet/fib/fib_types.h>
+#include <vnet/fib/fib_entry.h>
 
 /**
  * Load-balance main
@@ -98,6 +99,11 @@ typedef struct load_balance_t_ {
      */
     dpo_proto_t lb_proto;
 
+    /**
+     * Flags from the load-balance's associated fib_entry_t
+     */
+    fib_entry_flag_t lb_fib_entry_flags;
+
     /**
      * The number of locks, which is approximately the number of users,
      * of this load-balance.
@@ -167,6 +173,8 @@ extern void load_balance_set_bucket(index_t lbi,
 				    const dpo_id_t *next);
 extern void load_balance_set_urpf(index_t lbi,
 				  index_t urpf);
+extern void load_balance_set_fib_entry_flags(index_t lbi,
+                                             fib_entry_flag_t flags);
 extern index_t load_balance_get_urpf(index_t lbi);
 
 extern u8* format_load_balance(u8 * s, va_list * args);
diff --git a/src/vnet/dpo/replicate_dpo.c b/src/vnet/dpo/replicate_dpo.c
new file mode 100644
index 00000000..a2d5fdb6
--- /dev/null
+++ b/src/vnet/dpo/replicate_dpo.c
@@ -0,0 +1,759 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/ip/lookup.h>
+#include <vnet/dpo/replicate_dpo.h>
+#include <vnet/dpo/drop_dpo.h>
+#include <vnet/adj/adj.h>
+
+#undef REP_DEBUG
+
+#ifdef REP_DEBUG
+#define REP_DBG(_rep, _fmt, _args...)                                   \
+{                                                                       \
+    u8* _tmp =NULL;                                                     \
+    clib_warning("rep:[%s]:" _fmt,                                      \
+                 replicate_format(replicate_get_index((_rep)),          \
+                                  0, _tmp),                             \
+                 ##_args);                                              \
+    vec_free(_tmp);                                                     \
+}
+#else
+#define REP_DBG(_p, _fmt, _args...)
+#endif
+
+
+/**
+ * Pool of all DPOs. It's not static so the DP can have fast access
+ */
+replicate_t *replicate_pool;
+
+/**
+ * The one instance of replicate main
+ */
+replicate_main_t replicate_main;
+
+static inline index_t
+replicate_get_index (const replicate_t *rep)
+{
+    return (rep - replicate_pool);
+}
+
+static inline dpo_id_t*
+replicate_get_buckets (replicate_t *rep)
+{
+    if (REP_HAS_INLINE_BUCKETS(rep))
+    {
+        return (rep->rep_buckets_inline);
+    }
+    else
+    {
+        return (rep->rep_buckets);
+    }
+}
+
+static replicate_t *
+replicate_alloc_i (void)
+{
+    replicate_t *rep;
+
+    pool_get_aligned(replicate_pool, rep, CLIB_CACHE_LINE_BYTES);
+    memset(rep, 0, sizeof(*rep));
+
+    vlib_validate_combined_counter(&(replicate_main.repm_counters),
+                                   replicate_get_index(rep));
+    vlib_zero_combined_counter(&(replicate_main.repm_counters),
+                               replicate_get_index(rep));
+
+    return (rep);
+}
+
+static u8*
+replicate_format (index_t repi,
+                  replicate_format_flags_t flags,
+                  u32 indent,
+                  u8 *s)
+{
+    vlib_counter_t to;
+    replicate_t *rep;
+    dpo_id_t *buckets;
+    u32 i;
+
+    rep = replicate_get(repi);
+    vlib_get_combined_counter(&(replicate_main.repm_counters), repi, &to);
+    buckets = replicate_get_buckets(rep);
+
+    s = format(s, "%U: ", format_dpo_type, DPO_REPLICATE);
+    s = format(s, "[index:%d buckets:%d ", repi, rep->rep_n_buckets);
+    s = format(s, "to:[%Ld:%Ld]]", to.packets, to.bytes);
+
+    for (i = 0; i < rep->rep_n_buckets; i++)
+    {
+        s = format(s, "\n%U", format_white_space, indent+2);
+        s = format(s, "[%d]", i);
+        s = format(s, " %U", format_dpo_id, &buckets[i], indent+6);
+    }
+    return (s);
+}
+
+u8*
+format_replicate (u8 * s, va_list * args)
+{
+    index_t repi = va_arg(*args, index_t);
+    replicate_format_flags_t flags = va_arg(*args, replicate_format_flags_t);
+
+    return (replicate_format(repi, flags, 0, s));
+}
+static u8*
+format_replicate_dpo (u8 * s, va_list * args)
+{
+    index_t repi = va_arg(*args, index_t);
+    u32 indent = va_arg(*args, u32);
+
+    return (replicate_format(repi, REPLICATE_FORMAT_DETAIL, indent, s));
+}
+
+
+static replicate_t *
+replicate_create_i (u32 num_buckets,
+                    dpo_proto_t rep_proto)
+{
+    replicate_t *rep;
+
+    rep = replicate_alloc_i();
+    rep->rep_n_buckets = num_buckets;
+    rep->rep_proto = rep_proto;
+
+    if (!REP_HAS_INLINE_BUCKETS(rep))
+    {
+        vec_validate_aligned(rep->rep_buckets,
+                             rep->rep_n_buckets - 1,
+                             CLIB_CACHE_LINE_BYTES);
+    }
+
+    REP_DBG(rep, "create");
+
+    return (rep);
+}
+
+index_t
+replicate_create (u32 n_buckets,
+                  dpo_proto_t rep_proto)
+{
+    return (replicate_get_index(replicate_create_i(n_buckets, rep_proto)));
+}
+
+static inline void
+replicate_set_bucket_i (replicate_t *rep,
+                        u32 bucket,
+                        dpo_id_t *buckets,
+                        const dpo_id_t *next)
+{
+    dpo_stack(DPO_REPLICATE, rep->rep_proto, &buckets[bucket], next);
+}
+
+void
+replicate_set_bucket (index_t repi,
+                      u32 bucket,
+                      const dpo_id_t *next)
+{
+    replicate_t *rep;
+    dpo_id_t *buckets;
+
+    rep = replicate_get(repi);
+    buckets = replicate_get_buckets(rep);
+
+    ASSERT(bucket < rep->rep_n_buckets);
+
+    replicate_set_bucket_i(rep, bucket, buckets, next);
+}
+
+int
+replicate_is_drop (const dpo_id_t *dpo)
+{
+    replicate_t *rep;
+
+    if (DPO_REPLICATE != dpo->dpoi_type)
+        return (0);
+
+    rep = replicate_get(dpo->dpoi_index);
+
+    if (1 == rep->rep_n_buckets)
+    {
+        return (dpo_is_drop(replicate_get_bucket_i(rep, 0)));
+    }
+    return (0);
+}
+
+const dpo_id_t *
+replicate_get_bucket (index_t repi,
+                      u32 bucket)
+{
+    replicate_t *rep;
+
+    rep = replicate_get(repi);
+
+    return (replicate_get_bucket_i(rep, bucket));
+}
+
+
+static load_balance_path_t *
+replicate_multipath_next_hop_fixup (load_balance_path_t *nhs,
+                                    dpo_proto_t drop_proto)
+{
+    if (0 == vec_len(nhs))
+    {
+        load_balance_path_t *nh;
+
+        /*
+         * we need something for the replicate. so use the drop
+         */
+        vec_add2(nhs, nh, 1);
+
+        nh->path_weight = 1;
+        dpo_copy(&nh->path_dpo, drop_dpo_get(drop_proto));
+    }
+
+    return (nhs);
+}
+
+/*
+ * Fill in adjacencies in block based on corresponding
+ * next hop adjacencies.
+ */
+static void
+replicate_fill_buckets (replicate_t *rep,
+                        load_balance_path_t *nhs,
+                        dpo_id_t *buckets,
+                        u32 n_buckets)
+{
+    load_balance_path_t * nh;
+    u16 ii, bucket;
+
+    bucket = 0;
+
+    /*
+     * the next-hops have normalised weights. that means their sum is the number
+     * of buckets we need to fill.
+     */
+    vec_foreach (nh, nhs)
+    {
+        for (ii = 0; ii < nh->path_weight; ii++)
+        {
+            ASSERT(bucket < n_buckets);
+            replicate_set_bucket_i(rep, bucket++, buckets, &nh->path_dpo);
+        }
+    }
+}
+
+static inline void
+replicate_set_n_buckets (replicate_t *rep,
+                         u32 n_buckets)
+{
+    rep->rep_n_buckets = n_buckets;
+}
+
+void
+replicate_multipath_update (const dpo_id_t *dpo,
+                            load_balance_path_t * next_hops)
+{
+    load_balance_path_t * nh, * nhs;
+    dpo_id_t *tmp_dpo;
+    u32 ii, n_buckets;
+    replicate_t *rep;
+
+    ASSERT(DPO_REPLICATE == dpo->dpoi_type);
+    rep = replicate_get(dpo->dpoi_index);
+    nhs = replicate_multipath_next_hop_fixup(next_hops,
+                                             rep->rep_proto);
+    n_buckets = vec_len(nhs);
+
+    if (0 == rep->rep_n_buckets)
+    {
+        /*
+         * first time initialisation. no packets inflight, so we can write
+         * at leisure.
+         */
+        replicate_set_n_buckets(rep, n_buckets);
+
+        if (!REP_HAS_INLINE_BUCKETS(rep))
+            vec_validate_aligned(rep->rep_buckets,
+                                 rep->rep_n_buckets - 1,
+                                 CLIB_CACHE_LINE_BYTES);
+
+        replicate_fill_buckets(rep, nhs,
+                               replicate_get_buckets(rep),
+                               n_buckets);
+    }
+    else
+    {
+        /*
+         * This is a modification of an existing replicate.
+         * We need to ensure that packets in flight see a consistent state, that
+         * is the number of reported buckets the REP has
+         * is not more than it actually has. So if the
+         * number of buckets is increasing, we must update the bucket array first,
+         * then the reported number. vice-versa if the number of buckets goes down.
+         */
+        if (n_buckets == rep->rep_n_buckets)
+        {
+            /*
+             * no change in the number of buckets. we can simply fill what
+             * is new over what is old.
+             */
+            replicate_fill_buckets(rep, nhs,
+                                   replicate_get_buckets(rep),
+                                   n_buckets);
+        }
+        else if (n_buckets > rep->rep_n_buckets)
+        {
+            /*
+             * we have more buckets. the old replicate map (if there is one)
+             * will remain valid, i.e. mapping to indices within range, so we
+             * update it last.
+             */
+            if (n_buckets > REP_NUM_INLINE_BUCKETS &&
+                rep->rep_n_buckets <= REP_NUM_INLINE_BUCKETS)
+            {
+                /*
+                 * the new increased number of buckets is crossing the threshold
+                 * from the inline storage to out-line. Alloc the outline buckets
+                 * first, then fixup the number. then reset the inlines.
+                 */
+                ASSERT(NULL == rep->rep_buckets);
+                vec_validate_aligned(rep->rep_buckets,
+                                     n_buckets - 1,
+                                     CLIB_CACHE_LINE_BYTES);
+
+                replicate_fill_buckets(rep, nhs,
+                                       rep->rep_buckets,
+                                       n_buckets);
+                CLIB_MEMORY_BARRIER();
+                replicate_set_n_buckets(rep, n_buckets);
+
+                CLIB_MEMORY_BARRIER();
+
+                for (ii = 0; ii < REP_NUM_INLINE_BUCKETS; ii++)
+                {
+                    dpo_reset(&rep->rep_buckets_inline[ii]);
+                }
+            }
+            else
+            {
+                if (n_buckets <= REP_NUM_INLINE_BUCKETS)
+                {
+                    /*
+                     * we are not crossing the threshold and it's still inline buckets.
+                     * we can write the new on the old..
+                     */
+                    replicate_fill_buckets(rep, nhs,
+                                           replicate_get_buckets(rep),
+                                           n_buckets);
+                    CLIB_MEMORY_BARRIER();
+                    replicate_set_n_buckets(rep, n_buckets);
+                }
+                else
+                {
+                    /*
+                     * we are not crossing the threshold. We need a new bucket array to
+                     * hold the increased number of choices.
+                     */
+                    dpo_id_t *new_buckets, *old_buckets, *tmp_dpo;
+
+                    new_buckets = NULL;
+                    old_buckets = replicate_get_buckets(rep);
+
+                    vec_validate_aligned(new_buckets,
+                                         n_buckets - 1,
+                                         CLIB_CACHE_LINE_BYTES);
+
+                    replicate_fill_buckets(rep, nhs, new_buckets, n_buckets);
+                    CLIB_MEMORY_BARRIER();
+                    rep->rep_buckets = new_buckets;
+                    CLIB_MEMORY_BARRIER();
+                    replicate_set_n_buckets(rep, n_buckets);
+
+                    vec_foreach(tmp_dpo, old_buckets)
+                    {
+                        dpo_reset(tmp_dpo);
+                    }
+                    vec_free(old_buckets);
+                }
+            }
+        }
+        else
+        {
+            /*
+             * bucket size shrinkage.
+             */
+            if (n_buckets <= REP_NUM_INLINE_BUCKETS &&
+                rep->rep_n_buckets > REP_NUM_INLINE_BUCKETS)
+            {
+                /*
+                 * the new decreased number of buckets is crossing the threshold
+                 * from out-line storage to inline:
+                 *   1 - Fill the inline buckets,
+                 *   2 - fixup the number (and this point the inline buckets are
+                 *       used).
+                 *   3 - free the outline buckets
+                 */
+                replicate_fill_buckets(rep, nhs,
+                                       rep->rep_buckets_inline,
+                                       n_buckets);
+                CLIB_MEMORY_BARRIER();
+                replicate_set_n_buckets(rep, n_buckets);
+                CLIB_MEMORY_BARRIER();
+
+                vec_foreach(tmp_dpo, rep->rep_buckets)
+                {
+                    dpo_reset(tmp_dpo);
+                }
+                vec_free(rep->rep_buckets);
+            }
+            else
+            {
+                /*
+                 * not crossing the threshold.
+                 *  1 - update the number to the smaller size
+                 *  2 - write the new buckets
+                 *  3 - reset those no longer used.
+                 */
+                dpo_id_t *buckets;
+                u32 old_n_buckets;
+
+                old_n_buckets = rep->rep_n_buckets;
+                buckets = replicate_get_buckets(rep);
+
+                replicate_set_n_buckets(rep, n_buckets);
+                CLIB_MEMORY_BARRIER();
+
+                replicate_fill_buckets(rep, nhs,
+                                       buckets,
+                                       n_buckets);
+
+                for (ii = n_buckets; ii < old_n_buckets; ii++)
+                {
+                    dpo_reset(&buckets[ii]);
+                }
+            }
+        }
+    }
+
+    vec_foreach (nh, nhs)
+    {
+        dpo_reset(&nh->path_dpo);
+    }
+    vec_free(nhs);
+}
+
+static void
+replicate_lock (dpo_id_t *dpo)
+{
+    replicate_t *rep;
+
+    rep = replicate_get(dpo->dpoi_index);
+
+    rep->rep_locks++;
+}
+
+static void
+replicate_destroy (replicate_t *rep)
+{
+    dpo_id_t *buckets;
+    int i;
+
+    buckets = replicate_get_buckets(rep);
+
+    for (i = 0; i < rep->rep_n_buckets; i++)
+    {
+        dpo_reset(&buckets[i]);
+    }
+
+    REP_DBG(rep, "destroy");
+    if (!REP_HAS_INLINE_BUCKETS(rep))
+    {
+        vec_free(rep->rep_buckets);
+    }
+
+    pool_put(replicate_pool, rep);
+}
+
+static void
+replicate_unlock (dpo_id_t *dpo)
+{
+    replicate_t *rep;
+
+    rep = replicate_get(dpo->dpoi_index);
+
+    rep->rep_locks--;
+
+    if (0 == rep->rep_locks)
+    {
+        replicate_destroy(rep);
+    }
+}
+
+static void
+replicate_mem_show (void)
+{
+    fib_show_memory_usage("replicate",
+			  pool_elts(replicate_pool),
+			  pool_len(replicate_pool),
+			  sizeof(replicate_t));
+}
+
+const static dpo_vft_t rep_vft = {
+    .dv_lock = replicate_lock,
+    .dv_unlock = replicate_unlock,
+    .dv_format = format_replicate_dpo,
+    .dv_mem_show = replicate_mem_show,
+};
+
+/**
+ * @brief The per-protocol VLIB graph nodes that are assigned to a replicate
+ *        object.
+ *
+ * this means that these graph nodes are ones from which a replicate is the
+ * parent object in the DPO-graph.
+ */
+const static char* const replicate_ip4_nodes[] =
+{
+    "ip4-replicate",
+    NULL,
+};
+const static char* const replicate_ip6_nodes[] =
+{
+    "ip6-replicate",
+    NULL,
+};
+const static char* const replicate_mpls_nodes[] =
+{
+    "mpls-replicate",
+    NULL,
+};
+
+const static char* const * const replicate_nodes[DPO_PROTO_NUM] =
+{
+    [DPO_PROTO_IP4]  = replicate_ip4_nodes,
+    [DPO_PROTO_IP6]  = replicate_ip6_nodes,
+    [DPO_PROTO_MPLS] = replicate_mpls_nodes,
+};
+
+void
+replicate_module_init (void)
+{
+    dpo_register(DPO_REPLICATE, &rep_vft, replicate_nodes);
+}
+
+static clib_error_t *
+replicate_show (vlib_main_t * vm,
+                unformat_input_t * input,
+                vlib_cli_command_t * cmd)
+{
+    index_t repi = INDEX_INVALID;
+
+    while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+        if (unformat (input, "%d", &repi))
+            ;
+        else
+            break;
+    }
+
+    if (INDEX_INVALID != repi)
+    {
+        vlib_cli_output (vm, "%U", format_replicate, repi,
+                         REPLICATE_FORMAT_DETAIL);
+    }
+    else
+    {
+        replicate_t *rep;
+
+        pool_foreach(rep, replicate_pool,
+        ({
+            vlib_cli_output (vm, "%U", format_replicate,
+                             replicate_get_index(rep),
+                             REPLICATE_FORMAT_NONE);
+        }));
+    }
+
+    return 0;
+}
+
+VLIB_CLI_COMMAND (replicate_show_command, static) = {
+    .path = "show replicate",
+    .short_help = "show replicate [<index>]",
+    .function = replicate_show,
+};
+
+typedef struct replicate_trace_t_
+{
+    index_t rep_index;
+    index_t dpo_index;
+    dpo_type_t dpo_type;
+} replicate_trace_t;
+
+static uword
+replicate_inline (vlib_main_t * vm,
+                  vlib_node_runtime_t * node,
+                  vlib_frame_t * frame)
+{
+    vlib_combined_counter_main_t * cm = &replicate_main.repm_counters;
+    u32 n_left_from, * from, * to_next, next_index;
+    u32 cpu_index = os_get_cpu_number();
+
+    from = vlib_frame_vector_args (frame);
+    n_left_from = frame->n_vectors;
+    next_index = node->cached_next_index;
+  
+    while (n_left_from > 0)
+    {
+        u32 n_left_to_next;
+
+        vlib_get_next_frame (vm, node, next_index,
+                             to_next, n_left_to_next);
+
+        while (n_left_from > 0 && n_left_to_next > 0)
+	{
+            u32 next0, ci0, bi0, bucket, repi0;
+            const replicate_t *rep0;
+            vlib_buffer_t * b0, *c0;
+            const dpo_id_t *dpo0;
+
+            bi0 = from[0];
+            to_next[0] = bi0;
+            from += 1;
+            to_next += 1;
+            n_left_from -= 1;
+            n_left_to_next -= 1;
+
+            b0 = vlib_get_buffer (vm, bi0);
+            repi0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
+            rep0 = replicate_get(repi0);
+
+            vlib_increment_combined_counter(
+                cm, cpu_index, repi0, 1,
+                vlib_buffer_length_in_chain(vm, b0));
+
+            /* ship the original to the first bucket */
+            dpo0 = replicate_get_bucket_i(rep0, 0);
+            next0 = dpo0->dpoi_next_node;
+            vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
+
+            if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+                replicate_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
+                t->rep_index = repi0;
+                t->dpo_index = dpo0->dpoi_index;
+                t->dpo_type = dpo0->dpoi_type;
+            }
+            vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+                                             to_next, n_left_to_next,
+                                             bi0, next0);
+
+            /* ship copies to the rest of the buckets */
+            for (bucket = 1; bucket < rep0->rep_n_buckets; bucket++)
+            {
+                /* Make a copy */
+                c0 = vlib_buffer_copy(vm, b0);
+                ci0 = vlib_get_buffer_index(vm, c0);
+
+                to_next[0] = ci0;
+                to_next += 1;
+                n_left_to_next -= 1;
+
+                dpo0 = replicate_get_bucket_i(rep0, bucket);
+                next0 = dpo0->dpoi_next_node;
+                vnet_buffer (c0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
+
+                if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+                {
+                    replicate_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
+                    t->rep_index = repi0;
+                    t->dpo_index = dpo0->dpoi_index;
+                    t->dpo_type = dpo0->dpoi_type;
+                }
+
+                vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+                                                 to_next, n_left_to_next,
+                                                 ci0, next0);
+            }
+        }
+
+        vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+
+    return frame->n_vectors;
+}
+
+static u8 *
+format_replicate_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  replicate_trace_t *t = va_arg (*args, replicate_trace_t *);
+
+  s = format (s, "replicate: %d via %U:%d",
+              t->rep_index,
+              format_dpo_type, t->dpo_type,
+              t->dpo_index);
+  return s;
+}
+
+static uword
+ip4_replicate (vlib_main_t * vm,
+               vlib_node_runtime_t * node,
+               vlib_frame_t * frame)
+{
+    return (replicate_inline (vm, node, frame));
+}
+
+/**
+ * @brief
+ */
+VLIB_REGISTER_NODE (ip4_replicate_node) = {
+  .function = ip4_replicate,
+  .name = "ip4-replicate",
+  .vector_size = sizeof (u32),
+
+  .format_trace = format_replicate_trace,
+  .n_next_nodes = 1,
+  .next_nodes = {
+      [0] = "error-drop",
+  },
+};
+
+static uword
+ip6_replicate (vlib_main_t * vm,
+               vlib_node_runtime_t * node,
+               vlib_frame_t * frame)
+{
+    return (replicate_inline (vm, node, frame));
+}
+
+/**
+ * @brief
+ */
+VLIB_REGISTER_NODE (ip6_replicate_node) = {
+  .function = ip6_replicate,
+  .name = "ip6-replicate",
+  .vector_size = sizeof (u32),
+
+  .format_trace = format_replicate_trace,
+  .n_next_nodes = 1,
+  .next_nodes = {
+      [0] = "error-drop",
+  },
+};
diff --git a/src/vnet/dpo/replicate_dpo.h b/src/vnet/dpo/replicate_dpo.h
new file mode 100644
index 00000000..a564739c
--- /dev/null
+++ b/src/vnet/dpo/replicate_dpo.h
@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @brief
+ *
+ */
+
+#ifndef __REPLICATE_DPO_H__
+#define __REPLICATE_DPO_H__
+
+#include <vlib/vlib.h>
+#include <vnet/ip/lookup.h>
+#include <vnet/dpo/dpo.h>
+#include <vnet/dpo/load_balance.h>
+#include <vnet/fib/fib_types.h>
+
+/**
+ * replicate main
+ */
+typedef struct replicate_main_t_
+{
+    vlib_combined_counter_main_t repm_counters;
+} replicate_main_t;
+
+extern replicate_main_t replicate_main;
+
+/**
+ * The number of buckets that a load-balance object can have and still
+ * fit in one cache-line
+ */
+#define REP_NUM_INLINE_BUCKETS 4
+
+/**
+ * The FIB DPO provieds;
+ *  - load-balancing over the next DPOs in the chain/graph
+ *  - per-route counters
+ */
+typedef struct replicate_t_ {
+    /**
+     * number of buckets in the load-balance. always a power of 2.
+     */
+    u16 rep_n_buckets;
+
+   /**
+     * The protocol of packets that traverse this REP.
+     * need in combination with the flow hash config to determine how to hash.
+     * u8.
+     */
+    dpo_proto_t rep_proto;
+
+    /**
+     * The number of locks, which is approximately the number of users,
+     * of this load-balance.
+     * Load-balance objects of via-entries are heavily shared by recursives,
+     * so the lock count is a u32.
+     */
+    u32 rep_locks;
+
+    /**
+     * Vector of buckets containing the next DPOs, sized as repo_num
+     */
+    dpo_id_t *rep_buckets;
+
+    /**
+     * The rest of the cache line is used for buckets. In the common case
+     * where there there are less than 4 buckets, then the buckets are
+     * on the same cachlie and we save ourselves a pointer dereferance in 
+     * the data-path.
+     */
+    dpo_id_t rep_buckets_inline[REP_NUM_INLINE_BUCKETS];
+} replicate_t;
+
+STATIC_ASSERT(sizeof(replicate_t) <= CLIB_CACHE_LINE_BYTES,
+	      "A replicate object size exceeds one cachline");
+
+/**
+ * Flags controlling load-balance formatting/display
+ */
+typedef enum replicate_format_flags_t_ {
+    REPLICATE_FORMAT_NONE,
+    REPLICATE_FORMAT_DETAIL = (1 << 0),
+} replicate_format_flags_t;
+
+extern index_t replicate_create(u32 num_buckets,
+                                dpo_proto_t rep_proto);
+extern void replicate_multipath_update(
+    const dpo_id_t *dpo,
+    load_balance_path_t *next_hops);
+
+extern void replicate_set_bucket(index_t repi,
+				    u32 bucket,
+				    const dpo_id_t *next);
+
+extern u8* format_replicate(u8 * s, va_list * args);
+
+extern const dpo_id_t *replicate_get_bucket(index_t repi,
+					       u32 bucket);
+extern int replicate_is_drop(const dpo_id_t *dpo);
+
+/**
+ * The encapsulation breakages are for fast DP access
+ */
+extern replicate_t *replicate_pool;
+static inline replicate_t*
+replicate_get (index_t repi)
+{
+    return (pool_elt_at_index(replicate_pool, repi));
+}
+
+#define REP_HAS_INLINE_BUCKETS(_rep)		\
+    ((_rep)->rep_n_buckets <= REP_NUM_INLINE_BUCKETS)
+
+static inline const dpo_id_t *
+replicate_get_bucket_i (const replicate_t *rep,
+			   u32 bucket)
+{
+    ASSERT(bucket < rep->rep_n_buckets);
+
+    if (PREDICT_TRUE(REP_HAS_INLINE_BUCKETS(rep)))
+    {
+	return (&rep->rep_buckets_inline[bucket]);
+    }
+    else
+    {
+	return (&rep->rep_buckets[bucket]);
+    }
+}
+
+extern void replicate_module_init(void);
+
+#endif
diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c
index 0298541b..8f07fa55 100644
--- a/src/vnet/ethernet/arp.c
+++ b/src/vnet/ethernet/arp.c
@@ -23,6 +23,7 @@
 #include <vppinfra/mhash.h>
 #include <vnet/fib/ip4_fib.h>
 #include <vnet/adj/adj_nbr.h>
+#include <vnet/adj/adj_mcast.h>
 #include <vnet/mpls/mpls.h>
 
 /**
@@ -438,33 +439,74 @@ arp_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai)
   arp_int = &am->ethernet_arp_by_sw_if_index[sw_if_index];
   e = arp_entry_find (arp_int, &adj->sub_type.nbr.next_hop.ip4);
 
-  if (NULL != e)
-    {
-      adj_nbr_walk_nh4 (sw_if_index,
-			&e->ip4_address, arp_mk_complete_walk, e);
-    }
-  else
+  switch (adj->lookup_next_index)
     {
+    case IP_LOOKUP_NEXT_ARP:
+    case IP_LOOKUP_NEXT_GLEAN:
+      if (NULL != e)
+	{
+	  adj_nbr_walk_nh4 (sw_if_index,
+			    &e->ip4_address, arp_mk_complete_walk, e);
+	}
+      else
+	{
+	  /*
+	   * no matching ARP entry.
+	   * construct the rewrite required to for an ARP packet, and stick
+	   * that in the adj's pipe to smoke.
+	   */
+	  adj_nbr_update_rewrite
+	    (ai,
+	     ADJ_NBR_REWRITE_FLAG_INCOMPLETE,
+	     ethernet_build_rewrite
+	     (vnm,
+	      sw_if_index,
+	      VNET_LINK_ARP,
+	      VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST));
+
+	  /*
+	   * since the FIB has added this adj for a route, it makes sense it
+	   * may want to forward traffic sometime soon. Let's send a
+	   * speculative ARP. just one. If we were to do periodically that
+	   * wouldn't be bad either, but that's more code than i'm prepared to
+	   * write at this time for relatively little reward.
+	   */
+	  arp_nbr_probe (adj);
+	}
+      break;
+    case IP_LOOKUP_NEXT_MCAST:
       /*
-       * no matching ARP entry.
-       * construct the rewire required to for an ARP packet, and stick
-       * that in the adj's pipe to smoke.
+       * Construct a partial rewrite from the known ethernet mcast dest MAC
        */
-      adj_nbr_update_rewrite (ai,
-			      ADJ_NBR_REWRITE_FLAG_INCOMPLETE,
-			      ethernet_build_rewrite (vnm,
-						      sw_if_index,
-						      VNET_LINK_ARP,
-						      VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST));
+      adj_mcast_update_rewrite
+	(ai,
+	 ethernet_build_rewrite (vnm,
+				 sw_if_index,
+				 adj->ia_link,
+				 ethernet_ip4_mcast_dst_addr ()));
 
       /*
-       * since the FIB has added this adj for a route, it makes sense it may
-       * want to forward traffic sometime soon. Let's send a speculative ARP.
-       * just one. If we were to do periodically that wouldn't be bad either,
-       * but that's more code than i'm prepared to write at this time for
-       * relatively little reward.
+       * Complete the remaining fields of the adj's rewrite to direct the
+       * complete of the rewrite at switch time by copying in the IP
+       * dst address's bytes.
+       * Ofset is 11 bytes from the end of the MAC header - which is three
+       * bytes into the desintation address. And we write 3 bytes.
        */
-      arp_nbr_probe (adj);
+      adj->rewrite_header.dst_mcast_offset = 11;
+      adj->rewrite_header.dst_mcast_n_bytes = 3;
+
+      break;
+
+    case IP_LOOKUP_NEXT_DROP:
+    case IP_LOOKUP_NEXT_PUNT:
+    case IP_LOOKUP_NEXT_LOCAL:
+    case IP_LOOKUP_NEXT_REWRITE:
+    case IP_LOOKUP_NEXT_LOAD_BALANCE:
+    case IP_LOOKUP_NEXT_MIDCHAIN:
+    case IP_LOOKUP_NEXT_ICMP_ERROR:
+    case IP_LOOKUP_N_NEXT:
+      ASSERT (0);
+      break;
     }
 }
 
diff --git a/src/vnet/ethernet/ethernet.h b/src/vnet/ethernet/ethernet.h
index f88b0cf3..3acde421 100644
--- a/src/vnet/ethernet/ethernet.h
+++ b/src/vnet/ethernet/ethernet.h
@@ -547,6 +547,8 @@ void ethernet_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai);
 u8 *ethernet_build_rewrite (vnet_main_t * vnm,
 			    u32 sw_if_index,
 			    vnet_link_t link_type, const void *dst_address);
+const u8 *ethernet_ip4_mcast_dst_addr (void);
+const u8 *ethernet_ip6_mcast_dst_addr (void);
 
 extern vlib_node_registration_t ethernet_input_node;
 
diff --git a/src/vnet/ethernet/interface.c b/src/vnet/ethernet/interface.c
index 1c1f4353..95700309 100644
--- a/src/vnet/ethernet/interface.c
+++ b/src/vnet/ethernet/interface.c
@@ -51,6 +51,26 @@
  * This file contains code to manage loopback interfaces.
  */
 
+const u8 *
+ethernet_ip4_mcast_dst_addr (void)
+{
+  const static u8 ethernet_mcast_dst_mac[] = {
+    0x1, 0x0, 0x5e, 0x0, 0x0, 0x0,
+  };
+
+  return (ethernet_mcast_dst_mac);
+}
+
+const u8 *
+ethernet_ip6_mcast_dst_addr (void)
+{
+  const static u8 ethernet_mcast_dst_mac[] = {
+    0x33, 0x33, 0x00, 0x0, 0x0, 0x0,
+  };
+
+  return (ethernet_mcast_dst_mac);
+}
+
 /**
  * @brief build a rewrite string to use for sending packets of type 'link_type'
  * to 'dst_address'
diff --git a/src/vnet/fib/fib_attached_export.c b/src/vnet/fib/fib_attached_export.c
index fd597e3e..574131de 100644
--- a/src/vnet/fib/fib_attached_export.c
+++ b/src/vnet/fib/fib_attached_export.c
@@ -303,8 +303,8 @@ fib_attached_export_import (fib_entry_t *fib_entry,
      * may have realloc'd.
      */
     fib_entry = fib_entry_get(fei);
-        import->faei_export_sibling =
-	  fib_entry_cover_track(fib_entry_get(import->faei_export_entry), fei);
+    import->faei_export_sibling =
+	fib_entry_cover_track(fib_entry_get(import->faei_export_entry), fei);
 
     fed = fib_entry_delegate_find_or_add(fib_entry,
                                          FIB_ENTRY_DELEGATE_ATTACHED_IMPORT);
diff --git a/src/vnet/fib/fib_entry.h b/src/vnet/fib/fib_entry.h
index 44a5f2e6..f258b755 100644
--- a/src/vnet/fib/fib_entry.h
+++ b/src/vnet/fib/fib_entry.h
@@ -220,7 +220,7 @@ typedef enum fib_entry_flag_t_ {
     FIB_ENTRY_FLAG_EXCLUSIVE = (1 << FIB_ENTRY_ATTRIBUTE_EXCLUSIVE),
     FIB_ENTRY_FLAG_LOCAL     = (1 << FIB_ENTRY_ATTRIBUTE_LOCAL),
     FIB_ENTRY_FLAG_IMPORT    = (1 << FIB_ENTRY_ATTRIBUTE_IMPORT),
-} fib_entry_flag_t;
+} __attribute__((packed)) fib_entry_flag_t;
 
 /**
  * Flags for the source data
diff --git a/src/vnet/fib/fib_entry_delegate.c b/src/vnet/fib/fib_entry_delegate.c
index a0d45f97..efe402d1 100644
--- a/src/vnet/fib/fib_entry_delegate.c
+++ b/src/vnet/fib/fib_entry_delegate.c
@@ -119,6 +119,9 @@ fib_entry_chain_type_to_delegate_type (fib_forward_chain_type_t fct)
         return (FIB_ENTRY_DELEGATE_CHAIN_MPLS_NON_EOS);
     case FIB_FORW_CHAIN_TYPE_ETHERNET:
         return (FIB_ENTRY_DELEGATE_CHAIN_ETHERNET);
+    case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
+    case FIB_FORW_CHAIN_TYPE_MCAST_IP6:
+        break;
     }
     ASSERT(0);
     return (FIB_ENTRY_DELEGATE_CHAIN_UNICAST_IP4);
diff --git a/src/vnet/fib/fib_entry_src.c b/src/vnet/fib/fib_entry_src.c
index 1fb04060..d54787cd 100644
--- a/src/vnet/fib/fib_entry_src.c
+++ b/src/vnet/fib/fib_entry_src.c
@@ -313,6 +313,8 @@ fib_entry_src_collect_forwarding (fib_node_index_t pl_index,
         {
         case FIB_FORW_CHAIN_TYPE_UNICAST_IP4:
         case FIB_FORW_CHAIN_TYPE_UNICAST_IP6:
+        case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
+        case FIB_FORW_CHAIN_TYPE_MCAST_IP6:
             /*
              * EOS traffic with no label to stack, we need the IP Adj
              */
@@ -458,6 +460,8 @@ fib_entry_src_mk_lb (fib_entry_t *fib_entry,
     {
 	load_balance_set_urpf(dpo_lb->dpoi_index, ui);
     }
+    load_balance_set_fib_entry_flags(dpo_lb->dpoi_index,
+                                     fib_entry_get_flags_i(fib_entry));
 }
 
 void
diff --git a/src/vnet/fib/fib_node.h b/src/vnet/fib/fib_node.h
index 3ad8ee95..457dfb7a 100644
--- a/src/vnet/fib/fib_node.h
+++ b/src/vnet/fib/fib_node.h
@@ -31,6 +31,7 @@ typedef enum fib_node_type_t_ {
      */
     FIB_NODE_TYPE_WALK,
     FIB_NODE_TYPE_ENTRY,
+    FIB_NODE_TYPE_MFIB_ENTRY,
     FIB_NODE_TYPE_PATH_LIST,
     FIB_NODE_TYPE_PATH,
     FIB_NODE_TYPE_ADJ,
@@ -51,6 +52,7 @@ typedef enum fib_node_type_t_ {
 
 #define FIB_NODE_TYPES {                          \
     [FIB_NODE_TYPE_ENTRY]     = "entry",          \
+    [FIB_NODE_TYPE_MFIB_ENTRY] = "mfib-entry",    \
     [FIB_NODE_TYPE_WALK]      = "walk",           \
     [FIB_NODE_TYPE_PATH_LIST] = "path-list",      \
     [FIB_NODE_TYPE_PATH]      = "path",           \
diff --git a/src/vnet/fib/fib_path.c b/src/vnet/fib/fib_path.c
index 809e3e16..080057f3 100644
--- a/src/vnet/fib/fib_path.c
+++ b/src/vnet/fib/fib_path.c
@@ -23,6 +23,7 @@
 #include <vnet/dpo/lookup_dpo.h>
 
 #include <vnet/adj/adj.h>
+#include <vnet/adj/adj_mcast.h>
 
 #include <vnet/fib/fib_path.h>
 #include <vnet/fib/fib_node.h>
@@ -960,6 +961,8 @@ fib_path_route_flags_to_cfg_flags (const fib_route_path_t *rpath)
 	cfg_flags |= FIB_PATH_CFG_FLAG_RESOLVE_HOST;
     if (rpath->frp_flags & FIB_ROUTE_PATH_RESOLVE_VIA_ATTACHED)
 	cfg_flags |= FIB_PATH_CFG_FLAG_RESOLVE_ATTACHED;
+    if (rpath->frp_flags & FIB_ROUTE_PATH_LOCAL)
+	cfg_flags |= FIB_PATH_CFG_FLAG_LOCAL;
 
     return (cfg_flags);
 }
@@ -1003,28 +1006,25 @@ fib_path_create (fib_node_index_t pl_index,
     /*
      * deduce the path's tpye from the parementers and save what is needed.
      */
-    if (~0 != rpath->frp_sw_if_index)
+    if (path->fp_cfg_flags & FIB_PATH_CFG_FLAG_LOCAL)
     {
-	if (flags & FIB_PATH_CFG_FLAG_LOCAL)
-	{
-	    path->fp_type = FIB_PATH_TYPE_RECEIVE;
-	    path->receive.fp_interface = rpath->frp_sw_if_index;
-            path->receive.fp_addr = rpath->frp_addr;
-	}
-	else
-	{
-	    if (ip46_address_is_zero(&rpath->frp_addr))
-	    {
-		path->fp_type = FIB_PATH_TYPE_ATTACHED;
-		path->attached.fp_interface = rpath->frp_sw_if_index;
-	    }
-	    else
-	    {
-		path->fp_type = FIB_PATH_TYPE_ATTACHED_NEXT_HOP;
-		path->attached_next_hop.fp_interface = rpath->frp_sw_if_index;
-		path->attached_next_hop.fp_nh = rpath->frp_addr;
-	    }
-	}
+        path->fp_type = FIB_PATH_TYPE_RECEIVE;
+        path->receive.fp_interface = rpath->frp_sw_if_index;
+        path->receive.fp_addr = rpath->frp_addr;
+    }
+    else if (~0 != rpath->frp_sw_if_index)
+    {
+        if (ip46_address_is_zero(&rpath->frp_addr))
+        {
+            path->fp_type = FIB_PATH_TYPE_ATTACHED;
+            path->attached.fp_interface = rpath->frp_sw_if_index;
+        }
+        else
+        {
+            path->fp_type = FIB_PATH_TYPE_ATTACHED_NEXT_HOP;
+            path->attached_next_hop.fp_interface = rpath->frp_sw_if_index;
+            path->attached_next_hop.fp_nh = rpath->frp_addr;
+        }
     }
     else
     {
@@ -1199,7 +1199,7 @@ fib_path_cmp_i (const fib_path_t *path1,
     {
 	res = (path1->fp_type - path2->fp_type);
     }
-    if (path1->fp_nh_proto != path2->fp_nh_proto)
+    else if (path1->fp_nh_proto != path2->fp_nh_proto)
     {
 	res = (path1->fp_nh_proto - path2->fp_nh_proto);
     }
@@ -1770,8 +1770,11 @@ fib_path_contribute_forwarding (fib_node_index_t path_index,
 
 		break;
 	    }
-	    }
+	    case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
+	    case FIB_FORW_CHAIN_TYPE_MCAST_IP6:
 	    break;
+            }
+            break;
 	case FIB_PATH_TYPE_RECURSIVE:
 	    switch (fct)
 	    {
@@ -1781,13 +1784,15 @@ fib_path_contribute_forwarding (fib_node_index_t path_index,
 	    case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS:
 		fib_path_recursive_adj_update(path, fct, dpo);
 		break;
+	    case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
+	    case FIB_FORW_CHAIN_TYPE_MCAST_IP6:
 	    case FIB_FORW_CHAIN_TYPE_ETHERNET:
 		ASSERT(0);
 		break;
 	    }
 	    break;
 	case FIB_PATH_TYPE_DEAG:
-	    switch (fct)
+            switch (fct)
 	    {
 	    case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS:
                 lookup_dpo_add_or_lock_w_table_id(MPLS_FIB_DEFAULT_TABLE_ID,
@@ -1800,7 +1805,9 @@ fib_path_contribute_forwarding (fib_node_index_t path_index,
 	    case FIB_FORW_CHAIN_TYPE_UNICAST_IP6:
 	    case FIB_FORW_CHAIN_TYPE_MPLS_EOS:
 		dpo_copy(dpo, &path->fp_dpo);
-		break;		
+		break;
+	    case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
+	    case FIB_FORW_CHAIN_TYPE_MCAST_IP6:
 	    case FIB_FORW_CHAIN_TYPE_ETHERNET:
 		ASSERT(0);
 		break;
@@ -1810,12 +1817,38 @@ fib_path_contribute_forwarding (fib_node_index_t path_index,
 	    dpo_copy(dpo, &path->exclusive.fp_ex_dpo);
 	    break;
         case FIB_PATH_TYPE_ATTACHED:
-	case FIB_PATH_TYPE_RECEIVE:
-	case FIB_PATH_TYPE_SPECIAL:
-	    ASSERT(0);
+	    switch (fct)
+	    {
+	    case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS:
+	    case FIB_FORW_CHAIN_TYPE_UNICAST_IP4:
+	    case FIB_FORW_CHAIN_TYPE_UNICAST_IP6:
+	    case FIB_FORW_CHAIN_TYPE_MPLS_EOS:
+	    case FIB_FORW_CHAIN_TYPE_ETHERNET:
+                break;
+	    case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
+	    case FIB_FORW_CHAIN_TYPE_MCAST_IP6:
+                {
+                    adj_index_t ai;
+
+                    /*
+                     * Create the adj needed for sending IP multicast traffic
+                     */
+                    ai = adj_mcast_add_or_lock(path->fp_nh_proto,
+                                               fib_forw_chain_type_to_link_type(fct),
+                                               path->attached.fp_interface);
+                    dpo_set(dpo, DPO_ADJACENCY_MCAST,
+                            fib_forw_chain_type_to_dpo_proto(fct),
+                            ai);
+                    adj_unlock(ai);
+                }
+                break;
+            }
+            break;
+        case FIB_PATH_TYPE_RECEIVE:
+        case FIB_PATH_TYPE_SPECIAL:
+            dpo_copy(dpo, &path->fp_dpo);
             break;
 	}
-
     }
 }
 
diff --git a/src/vnet/fib/fib_path_list.c b/src/vnet/fib/fib_path_list.c
index db9d1af9..ce11cf45 100644
--- a/src/vnet/fib/fib_path_list.c
+++ b/src/vnet/fib/fib_path_list.c
@@ -585,8 +585,11 @@ fib_path_list_resolve (fib_path_list_t *path_list)
     path_list = fib_path_list_get(path_list_index);
 
     FIB_PATH_LIST_DBG(path_list, "resovled");
-    fib_path_list_mk_urpf(path_list);
 
+    if (!(path_list->fpl_flags & FIB_PATH_LIST_FLAG_NO_URPF))
+    {
+        fib_path_list_mk_urpf(path_list);
+    }
     return (path_list);
 }
 
@@ -1025,14 +1028,14 @@ fib_path_list_copy_and_path_remove (fib_node_index_t orig_path_list_index,
  */
 void
 fib_path_list_contribute_forwarding (fib_node_index_t path_list_index,
-				     fib_forward_chain_type_t type,
+				     fib_forward_chain_type_t fct,
 				     dpo_id_t *dpo)
 {
     fib_path_list_t *path_list;
 
     path_list = fib_path_list_get(path_list_index);
 
-    fib_path_list_mk_lb(path_list, type, dpo);
+    fib_path_list_mk_lb(path_list, fct, dpo);
 }
 
 /*
diff --git a/src/vnet/fib/fib_path_list.h b/src/vnet/fib/fib_path_list.h
index f4f94a1b..b4971add 100644
--- a/src/vnet/fib/fib_path_list.h
+++ b/src/vnet/fib/fib_path_list.h
@@ -60,6 +60,10 @@ typedef enum fib_path_list_attribute_t_ {
      * looped path-list. one path looped implies the whole list is
      */
     FIB_PATH_LIST_ATTRIBUTE_LOOPED,
+    /**
+     * no uRPF - do not generate unicast RPF list for this path-list
+     */
+    FIB_PATH_LIST_ATTRIBUTE_NO_URPF,
     /**
      * Marher. Add new flags before this one, and then update it.
      */
@@ -74,6 +78,7 @@ typedef enum fib_path_list_flags_t_ {
     FIB_PATH_LIST_FLAG_EXCLUSIVE = (1 << FIB_PATH_LIST_ATTRIBUTE_EXCLUSIVE),
     FIB_PATH_LIST_FLAG_RESOLVED  = (1 << FIB_PATH_LIST_ATTRIBUTE_RESOLVED),
     FIB_PATH_LIST_FLAG_LOOPED    = (1 << FIB_PATH_LIST_ATTRIBUTE_LOOPED),
+    FIB_PATH_LIST_FLAG_NO_URPF   = (1 << FIB_PATH_LIST_ATTRIBUTE_NO_URPF),
 } fib_path_list_flags_t;
 
 #define FIB_PATH_LIST_ATTRIBUTES {       		 \
@@ -83,6 +88,7 @@ typedef enum fib_path_list_flags_t_ {
     [FIB_PATH_LIST_ATTRIBUTE_EXCLUSIVE] = "exclusive",   \
     [FIB_PATH_LIST_ATTRIBUTE_LOCAL]     = "local",	 \
     [FIB_PATH_LIST_ATTRIBUTE_LOOPED]     = "looped",	 \
+    [FIB_PATH_LIST_ATTRIBUTE_NO_URPF]     = "no-uRPF",	 \
 }
 
 #define FOR_EACH_PATH_LIST_ATTRIBUTE(_item)		\
diff --git a/src/vnet/fib/fib_table.c b/src/vnet/fib/fib_table.c
index 76db42d0..57eb0ae8 100644
--- a/src/vnet/fib/fib_table.c
+++ b/src/vnet/fib/fib_table.c
@@ -1043,6 +1043,26 @@ fib_table_destroy (fib_table_t *fib_table)
 	break;
     }
 }
+void
+fib_table_walk (u32 fib_index,
+                fib_protocol_t proto,
+                fib_table_walk_fn_t fn,
+                void *ctx)
+{
+    switch (proto)
+    {
+    case FIB_PROTOCOL_IP4:
+	ip4_fib_table_walk(ip4_fib_get(fib_index), fn, ctx);
+	break;
+    case FIB_PROTOCOL_IP6:
+	ip6_fib_table_walk(fib_index, fn, ctx);
+	break;
+    case FIB_PROTOCOL_MPLS:
+	mpls_fib_table_walk(mpls_fib_get(fib_index), fn, ctx);
+	break;
+    }
+}
+
 
 void
 fib_table_unlock (u32 fib_index,
@@ -1094,11 +1114,56 @@ format_fib_table_name (u8* s, va_list ap)
     return (s);
 }
 
+/**
+ * @brief Table flush context. Store the indicies of matching FIB entries
+ * that need to be removed.
+ */
+typedef struct fib_table_flush_ctx_t_
+{
+    /**
+     * The list of entries to flush
+     */
+    fib_node_index_t *ftf_entries;
+
+    /**
+     * The source we are flushing
+     */
+    fib_source_t ftf_source;
+} fib_table_flush_ctx_t;
+
+static int
+fib_table_flush_cb (fib_node_index_t fib_entry_index,
+                    void *arg)
+{
+    fib_table_flush_ctx_t *ctx = arg;
+
+    if (fib_entry_is_sourced(fib_entry_index, ctx->ftf_source))
+    {
+        vec_add1(ctx->ftf_entries, fib_entry_index);
+    }
+    return (1);
+}
+
+
 void
 fib_table_flush (u32 fib_index,
 		 fib_protocol_t proto,
 		 fib_source_t source)
 {
-    // FIXME
-    ASSERT(0);
+    fib_node_index_t *fib_entry_index;
+    fib_table_flush_ctx_t ctx = {
+        .ftf_entries = NULL,
+        .ftf_source = source,
+    };
+
+    fib_table_walk(fib_index, proto,
+                   fib_table_flush_cb,
+                   &ctx);
+
+    vec_foreach(fib_entry_index, ctx.ftf_entries)
+    {
+        fib_entry_delete(*fib_entry_index, source);
+    }
+
+    vec_free(ctx.ftf_entries);
 }
diff --git a/src/vnet/fib/fib_table.h b/src/vnet/fib/fib_table.h
index cfec516d..e7e66acb 100644
--- a/src/vnet/fib/fib_table.h
+++ b/src/vnet/fib/fib_table.h
@@ -729,4 +729,20 @@ extern u32 fib_table_get_num_entries(u32 fib_index,
 extern fib_table_t *fib_table_get(fib_node_index_t index,
 				  fib_protocol_t proto);
 
+/**
+ * @brief Call back function when walking entries in a FIB table
+ */
+typedef int (*fib_table_walk_fn_t)(fib_node_index_t fei,
+                                   void *ctx);
+
+/**
+ * @brief Walk all entries in a FIB table
+ * N.B: This is NOT safe to deletes. If you need to delete walk the whole
+ * table and store elements in a vector, then delete the elements
+ */
+extern void fib_table_walk(u32 fib_index,
+                           fib_protocol_t proto,
+                           fib_table_walk_fn_t fn,
+                           void *ctx);
+
 #endif
diff --git a/src/vnet/fib/fib_test.c b/src/vnet/fib/fib_test.c
index 5083db26..1c4a63a2 100644
--- a/src/vnet/fib/fib_test.c
+++ b/src/vnet/fib/fib_test.c
@@ -663,14 +663,15 @@ fib_test_v4 (void)
     /*
      * at this stage there are 5 entries in the test FIB (plus 5 in the default),
      * all of which are special sourced and so none of which share path-lists.
-     * There are also 6 entries, and 6 non-shared path-lists, in the v6 default
-     * table
+     * There are also 2 entries, and 2 non-shared path-lists, in the v6 default
+     * table, and 4 path-lists in the v6 MFIB table
      */
-#define NBR (5+5+6)
+#define ENBR (5+5+2)
+#define PNBR (5+5+6)
     FIB_TEST((0 == fib_path_list_db_size()),   "path list DB is empty");
-    FIB_TEST((NBR == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -754,9 +755,9 @@ fib_test_v4 (void)
      * +2 interface routes +2 non-shared path-lists
      */
     FIB_TEST((0 == fib_path_list_db_size()),   "path list DB is empty");
-    FIB_TEST((NBR+2 == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNBR+2 == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+2 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+2 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -808,9 +809,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((1 == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+3 == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNBR+3 == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+2 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+2 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -838,9 +839,9 @@ fib_test_v4 (void)
      * -1 shared-path-list
      */
     FIB_TEST((0 == fib_path_list_db_size()),   "path list DB is empty");
-    FIB_TEST((NBR+2 == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNBR+2 == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+2 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+2 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -1018,9 +1019,9 @@ fib_test_v4 (void)
      * +2 adj-fibs, and their non-shared path-lists
      */
     FIB_TEST((0 == fib_path_list_db_size()),   "path list DB is empty");
-    FIB_TEST((NBR+4 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+4 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+4 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+4 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -1054,9 +1055,9 @@ fib_test_v4 (void)
      * +1 entry and a shared path-list
      */
     FIB_TEST((1 == fib_path_list_db_size()),   "path list DB is empty");
-    FIB_TEST((NBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+5 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+5 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /* 1.1.2.0/24 */
@@ -1087,9 +1088,9 @@ fib_test_v4 (void)
      * +1 entry only
      */
     FIB_TEST((1 == fib_path_list_db_size()),   "path list DB is empty");
-    FIB_TEST((NBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+6 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+6 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -1127,9 +1128,9 @@ fib_test_v4 (void)
      * +1 shared-pathlist
      */
     FIB_TEST((2 == fib_path_list_db_size()),   "path list DB is empty");
-    FIB_TEST((NBR+6 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+6 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+6 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+6 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -1158,9 +1159,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((1 == fib_path_list_db_size()),   "path list DB is %d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+6 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+6 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -1203,9 +1204,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((2  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+6 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+6 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+7 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+7 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     fib_prefix_t bgp_101_pfx = {
@@ -1239,9 +1240,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((2  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+6 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+6 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+8 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+8 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -1368,9 +1369,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((3  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+7 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+7 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+10 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+10 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -1983,9 +1984,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((4  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+12 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+12 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -2030,9 +2031,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((4  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+13 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+13 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -2080,9 +2081,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((5  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+9 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+9 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+14 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+14 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -2118,9 +2119,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((4  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+13 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+13 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -2154,9 +2155,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((4  == fib_path_list_db_size()),   "path list DB population:%d",
 	fib_path_list_db_size());
-    FIB_TEST((NBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
 	fib_path_list_pool_size());
-    FIB_TEST((NBR+12 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+12 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -2185,9 +2186,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((4  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+12 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+12 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -2215,9 +2216,9 @@ fib_test_v4 (void)
 
     FIB_TEST((3  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+7 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+7 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+10 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+10 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -2245,9 +2246,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((2  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+6 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+6 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+9 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+9 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -2355,9 +2356,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((1  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+7 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+7 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -2380,9 +2381,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((2  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+6 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+6 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+8 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+8 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     fei = fib_table_lookup_exact_match(fib_index, &bgp_200_pfx);
@@ -2428,9 +2429,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((3  == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NBR+10 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+10 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     ai_03 = adj_nbr_add_or_lock(FIB_PROTOCOL_IP4,
@@ -2492,9 +2493,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((1  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR+7 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+7 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
 
@@ -2562,9 +2563,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((4  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+8 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR+10 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+10 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
     /*
@@ -2753,9 +2754,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((1  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR+7 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+7 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
     /*
@@ -2830,9 +2831,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((1  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR+7 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+7 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
     /*
@@ -3180,9 +3181,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((1  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR+7 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+7 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
     /*
@@ -3247,9 +3248,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((1  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR+7 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+7 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
     /*
@@ -3297,9 +3298,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((1  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR+7 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+7 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
     /*
@@ -3342,9 +3343,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((1  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR+7 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+7 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
     /*
@@ -3438,9 +3439,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((0  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR+4 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+4 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR+4 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+4 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
     /*
@@ -3482,9 +3483,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((1  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+5 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR+5 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+5 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
     fib_table_entry_delete(fib_index,
@@ -3493,9 +3494,9 @@ fib_test_v4 (void)
 
     FIB_TEST((0  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR+4 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+4 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR+4 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+4 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
     /*
@@ -3577,9 +3578,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((0  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR+2 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR+2 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR+2 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR+2 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
     /*
@@ -3619,9 +3620,9 @@ fib_test_v4 (void)
      */
     FIB_TEST((0  == fib_path_list_db_size()),   "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
 
     /*
@@ -3644,11 +3645,11 @@ fib_test_v4 (void)
 
     FIB_TEST((0  == fib_path_list_db_size()), "path list DB population:%d",
     	     fib_path_list_db_size());
-    FIB_TEST((NBR-5 == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNBR-5 == fib_path_list_pool_size()), "path list pool size is %d",
     	     fib_path_list_pool_size());
-    FIB_TEST((NBR-5 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENBR-5 == fib_entry_pool_size()), "entry pool size is %d",
     	     fib_entry_pool_size());
-    FIB_TEST((NBR-5 == pool_elts(fib_urpf_list_pool)), "uRPF pool size is %d",
+    FIB_TEST((ENBR-5 == pool_elts(fib_urpf_list_pool)), "uRPF pool size is %d",
     	     pool_elts(fib_urpf_list_pool));
 
     return 0;
@@ -3720,13 +3721,15 @@ fib_test_v6 (void)
 
     /*
      * At this stage there is one v4 FIB with 5 routes and two v6 FIBs
-     * each with 6 entries. All entries are special so no path-list sharing.
+     * each with 2 entries and a v6 mfib with 4 path-lists.
+     * All entries are special so no path-list sharing.
      */
-#define NPS (5+6+6)
+#define ENPS (5+4)
+#define PNPS (5+4+4)
     FIB_TEST((0 == fib_path_list_db_size()),   "path list DB is empty");
-    FIB_TEST((NPS == fib_path_list_pool_size()), "path list pool size is %d",
+    FIB_TEST((PNPS == fib_path_list_pool_size()), "path list pool size is %d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NPS == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENPS == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -3816,9 +3819,9 @@ fib_test_v6 (void)
      * +2 entries. +2 unshared path-lists
      */
     FIB_TEST((0 == fib_path_list_db_size()),   "path list DB is empty");
-    FIB_TEST((NPS+2 == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNPS+2 == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NPS+2 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENPS+2 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -3862,9 +3865,9 @@ fib_test_v6 (void)
      */
     FIB_TEST((1 == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NPS+3 == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNPS+3 == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NPS+2 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENPS+2 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -3890,9 +3893,9 @@ fib_test_v6 (void)
      */
     FIB_TEST((0 == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NPS+2 == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNPS+2 == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NPS+2 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENPS+2 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -4006,9 +4009,9 @@ fib_test_v6 (void)
      */
     FIB_TEST((0 == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NPS+4 == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNPS+4 == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NPS+4 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENPS+4 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -4073,9 +4076,9 @@ fib_test_v6 (void)
      */
     FIB_TEST((1 == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NPS+5 == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNPS+5 == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NPS+6 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENPS+6 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -4201,9 +4204,9 @@ fib_test_v6 (void)
      */
     FIB_TEST((1 == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NPS+5 == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNPS+5 == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NPS+6 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENPS+6 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -4275,9 +4278,9 @@ fib_test_v6 (void)
      */
     FIB_TEST((1 == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NPS+7 == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNPS+7 == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NPS+8 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENPS+8 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
 
@@ -4401,9 +4404,9 @@ fib_test_v6 (void)
      */
     FIB_TEST((1 == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NPS+7 == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNPS+7 == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NPS+8 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENPS+8 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -4501,9 +4504,9 @@ fib_test_v6 (void)
      */
     FIB_TEST((0 == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NPS == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNPS == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NPS == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENPS == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     /*
@@ -4513,9 +4516,9 @@ fib_test_v6 (void)
 
     FIB_TEST((0 == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
-    FIB_TEST((NPS-6 == fib_path_list_pool_size()), "path list pool size is%d",
+    FIB_TEST((PNPS-2 == fib_path_list_pool_size()), "path list pool size is%d",
 	     fib_path_list_pool_size());
-    FIB_TEST((NPS-6 == fib_entry_pool_size()), "entry pool size is %d",
+    FIB_TEST((ENPS-2 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
     adj_unlock(ai_02);
diff --git a/src/vnet/fib/fib_types.c b/src/vnet/fib/fib_types.c
index b66e7194..3ecb38e8 100644
--- a/src/vnet/fib/fib_types.c
+++ b/src/vnet/fib/fib_types.c
@@ -290,8 +290,10 @@ fib_forw_chain_type_to_link_type (fib_forward_chain_type_t fct)
     switch (fct)
     {
     case FIB_FORW_CHAIN_TYPE_UNICAST_IP4:
+    case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
 	return (VNET_LINK_IP4);
     case FIB_FORW_CHAIN_TYPE_UNICAST_IP6:
+    case FIB_FORW_CHAIN_TYPE_MCAST_IP6:
 	return (VNET_LINK_IP6);
     case FIB_FORW_CHAIN_TYPE_ETHERNET:
 	return (VNET_LINK_ETHERNET);
@@ -313,8 +315,10 @@ fib_forw_chain_type_to_dpo_proto (fib_forward_chain_type_t fct)
     switch (fct)
     {
     case FIB_FORW_CHAIN_TYPE_UNICAST_IP4:
+    case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
 	return (DPO_PROTO_IP4);
     case FIB_FORW_CHAIN_TYPE_UNICAST_IP6:
+    case FIB_FORW_CHAIN_TYPE_MCAST_IP6:
 	return (DPO_PROTO_IP6);
     case FIB_FORW_CHAIN_TYPE_ETHERNET:
 	return (DPO_PROTO_ETHERNET);
diff --git a/src/vnet/fib/fib_types.h b/src/vnet/fib/fib_types.h
index 0a15fef1..c51bc9c0 100644
--- a/src/vnet/fib/fib_types.h
+++ b/src/vnet/fib/fib_types.h
@@ -95,6 +95,14 @@ typedef enum fib_forward_chain_type_t_ {
      * option is converted into one of the other three internally.
      */
     FIB_FORW_CHAIN_TYPE_MPLS_EOS,
+    /**
+     * Contribute an object that is to be used to forward IP4 packets
+     */
+    FIB_FORW_CHAIN_TYPE_MCAST_IP4,
+    /**
+     * Contribute an object that is to be used to forward IP6 packets
+     */
+    FIB_FORW_CHAIN_TYPE_MCAST_IP6,
     /**
      * Contribute an object that is to be used to forward Ethernet packets.
      * This is last in the list since it is not valid for many FIB objects,
@@ -107,6 +115,8 @@ typedef enum fib_forward_chain_type_t_ {
     [FIB_FORW_CHAIN_TYPE_ETHERNET]      = "ethernet",     	\
     [FIB_FORW_CHAIN_TYPE_UNICAST_IP4]   = "unicast-ip4",	\
     [FIB_FORW_CHAIN_TYPE_UNICAST_IP6]   = "unicast-ip6",	\
+    [FIB_FORW_CHAIN_TYPE_MCAST_IP4]     = "multicast-ip4",	\
+    [FIB_FORW_CHAIN_TYPE_MCAST_IP6]     = "multicast-ip6",	\
     [FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS]  = "mpls-neos",	        \
     [FIB_FORW_CHAIN_TYPE_MPLS_EOS]      = "mpls-eos",	        \
 }
@@ -263,6 +273,10 @@ typedef enum fib_route_path_flags_t_
      * Recursion constraint of via an attahced prefix
      */
     FIB_ROUTE_PATH_RESOLVE_VIA_ATTACHED = (1 << 1),
+    /**
+     * A for-us/local path
+     */
+    FIB_ROUTE_PATH_LOCAL = (1 << 2),
 } fib_route_path_flags_t;
 
 /**
diff --git a/src/vnet/fib/fib_urpf_list.c b/src/vnet/fib/fib_urpf_list.c
index 263812ad..b4844420 100644
--- a/src/vnet/fib/fib_urpf_list.c
+++ b/src/vnet/fib/fib_urpf_list.c
@@ -29,16 +29,24 @@ format_fib_urpf_list (u8 *s, va_list args)
     u32 *swi;
 
     ui = va_arg(args, index_t);
-    urpf = fib_urpf_list_get(ui);
 
-    s = format(s, "uPRF-list:%d len:%d itfs:[",
-	       ui, vec_len(urpf->furpf_itfs));
+    if (INDEX_INVALID != ui)
+    {
+        urpf = fib_urpf_list_get(ui);
+
+        s = format(s, "uPRF-list:%d len:%d itfs:[",
+                   ui, vec_len(urpf->furpf_itfs));
 
-    vec_foreach(swi, urpf->furpf_itfs)
+        vec_foreach(swi, urpf->furpf_itfs)
+        {
+            s = format(s, "%d, ", *swi);
+        }
+        s = format(s, "]");
+    }
+    else
     {
-	s = format(s, "%d, ", *swi);
+        s = format(s, "uRPF-list: None");
     }
-    s = format(s, "]");
 
     return (s);
 }
diff --git a/src/vnet/fib/ip4_fib.c b/src/vnet/fib/ip4_fib.c
index f6ebce00..e8211c80 100644
--- a/src/vnet/fib/ip4_fib.c
+++ b/src/vnet/fib/ip4_fib.c
@@ -378,16 +378,13 @@ ip4_fib_table_fwding_dpo_remove (ip4_fib_t *fib,
     ip4_fib_mtrie_add_del_route(fib, *addr, len, dpo->dpoi_index, 1); // DELETE
 }
 
-static void
-ip4_fib_table_show_all (ip4_fib_t *fib,
-			vlib_main_t * vm)
+void
+ip4_fib_table_walk (ip4_fib_t *fib,
+                    fib_table_walk_fn_t fn,
+                    void *ctx)
 {
-    fib_node_index_t *fib_entry_indicies;
-    fib_node_index_t *fib_entry_index;
     int i;
 
-    fib_entry_indicies = NULL;
-
     for (i = 0; i < ARRAY_LEN (fib->fib_entry_by_dst_address); i++)
     {
 	uword * hash = fib->fib_entry_by_dst_address[i];
@@ -398,14 +395,45 @@ ip4_fib_table_show_all (ip4_fib_t *fib,
 
 	    hash_foreach_pair (p, hash,
 	    ({
-		vec_add1(fib_entry_indicies, p->value[0]);
+		fn(p->value[0], ctx);
 	    }));
 	}
     }
+}
+
+/**
+ * Walk show context
+ */
+typedef struct ip4_fib_show_walk_ctx_t_
+{
+    fib_node_index_t *ifsw_indicies;
+} ip4_fib_show_walk_ctx_t;
+
+static int
+ip4_fib_show_walk_cb (fib_node_index_t fib_entry_index,
+                      void *arg)
+{
+    ip4_fib_show_walk_ctx_t *ctx = arg;
+
+    vec_add1(ctx->ifsw_indicies, fib_entry_index);
+
+    return (1);
+}
+
+static void
+ip4_fib_table_show_all (ip4_fib_t *fib,
+			vlib_main_t * vm)
+{
+    ip4_fib_show_walk_ctx_t ctx = {
+        .ifsw_indicies = NULL,
+    };
+    fib_node_index_t *fib_entry_index;
 
-    vec_sort_with_function(fib_entry_indicies, fib_entry_cmp_for_sort);
+    ip4_fib_table_walk(fib, ip4_fib_show_walk_cb, &ctx);
+    vec_sort_with_function(ctx.ifsw_indicies,
+                           fib_entry_cmp_for_sort);
 
-    vec_foreach(fib_entry_index, fib_entry_indicies)
+    vec_foreach(fib_entry_index, ctx.ifsw_indicies)
     {
 	vlib_cli_output(vm, "%U",
                         format_fib_entry,
@@ -413,7 +441,7 @@ ip4_fib_table_show_all (ip4_fib_t *fib,
                         FIB_ENTRY_FORMAT_BRIEF);
     }
 
-    vec_free(fib_entry_indicies);
+    vec_free(ctx.ifsw_indicies);
 }
 
 static void
diff --git a/src/vnet/fib/ip4_fib.h b/src/vnet/fib/ip4_fib.h
index cf312cdc..a8dc68b5 100644
--- a/src/vnet/fib/ip4_fib.h
+++ b/src/vnet/fib/ip4_fib.h
@@ -64,6 +64,15 @@ extern void ip4_fib_table_fwding_dpo_remove(ip4_fib_t *fib,
 extern u32 ip4_fib_table_lookup_lb (ip4_fib_t *fib,
 				    const ip4_address_t * dst);
 
+/**
+ * @brief Walk all entries in a FIB table
+ * N.B: This is NOT safe to deletes. If you need to delete walk the whole
+ * table and store elements in a vector, then delete the elements
+ */
+extern void ip4_fib_table_walk(ip4_fib_t *fib,
+                               fib_table_walk_fn_t fn,
+                               void *ctx);
+
 /**
  * @brief Get the FIB at the given index
  */
diff --git a/src/vnet/fib/ip6_fib.c b/src/vnet/fib/ip6_fib.c
index d5b9bdcb..343ff55e 100644
--- a/src/vnet/fib/ip6_fib.c
+++ b/src/vnet/fib/ip6_fib.c
@@ -38,57 +38,6 @@ vnet_ip6_fib_init (u32 fib_index)
 				FIB_ENTRY_FLAG_DROP,
 				ADJ_INDEX_INVALID);
 
-    /*
-     * Add ff02::1:ff00:0/104 via local route for all tables.
-     *  This is required for neighbor discovery to work.
-     */
-    ip6_set_solicited_node_multicast_address(&pfx.fp_addr.ip6, 0);
-    pfx.fp_len = 104;
-    fib_table_entry_special_add(fib_index,
-				&pfx,
-				FIB_SOURCE_SPECIAL,
-				FIB_ENTRY_FLAG_LOCAL,
-				ADJ_INDEX_INVALID);
-
-    /*
-     * Add all-routers multicast address via local route for all tables
-     */
-    ip6_set_reserved_multicast_address (&pfx.fp_addr.ip6,
-					IP6_MULTICAST_SCOPE_link_local,
-					IP6_MULTICAST_GROUP_ID_all_routers);
-    pfx.fp_len = 128;
-    fib_table_entry_special_add(fib_index,
-				&pfx,
-				FIB_SOURCE_SPECIAL,
-				FIB_ENTRY_FLAG_LOCAL,
-				ADJ_INDEX_INVALID);
-
-    /*
-     * Add all-nodes multicast address via local route for all tables
-     */
-    ip6_set_reserved_multicast_address (&pfx.fp_addr.ip6,
-					IP6_MULTICAST_SCOPE_link_local,
-					IP6_MULTICAST_GROUP_ID_all_hosts);
-    pfx.fp_len = 128;
-    fib_table_entry_special_add(fib_index,
-				&pfx,
-				FIB_SOURCE_SPECIAL,
-				FIB_ENTRY_FLAG_LOCAL,
-				ADJ_INDEX_INVALID);
-
-    /*
-     *  Add all-mldv2  multicast address via local route for all tables
-     */
-    ip6_set_reserved_multicast_address (&pfx.fp_addr.ip6,
-					IP6_MULTICAST_SCOPE_link_local,
-					IP6_MULTICAST_GROUP_ID_mldv2_routers);
-    pfx.fp_len = 128;
-    fib_table_entry_special_add(fib_index,
-				&pfx,
-				FIB_SOURCE_SPECIAL,
-				FIB_ENTRY_FLAG_LOCAL,
-				ADJ_INDEX_INVALID);
-
     /*
      * all link local for us
      */
@@ -512,27 +461,68 @@ ip6_fib_table_fwding_dpo_remove (u32 fib_index,
     if (--table->dst_address_length_refcounts[len] == 0)
     {
 	table->non_empty_dst_address_length_bitmap =
-            clib_bitmap_set (table->non_empty_dst_address_length_bitmap, 
+            clib_bitmap_set (table->non_empty_dst_address_length_bitmap,
                              128 - len, 0);
 	compute_prefix_lengths_in_search_order (table);
     }
 }
 
+/**
+ * @brief Context when walking the IPv6 table. Since all VRFs are in the
+ * same hash table, we need to filter only those we need as we walk
+ */
+typedef struct ip6_fib_walk_ctx_t_
+{
+    u32 i6w_fib_index;
+    fib_table_walk_fn_t i6w_fn;
+    void *i6w_ctx;
+} ip6_fib_walk_ctx_t;
+
+static int
+ip6_fib_walk_cb (clib_bihash_kv_24_8_t * kvp,
+                 void *arg)
+{
+    ip6_fib_walk_ctx_t *ctx = arg;
+
+    if ((kvp->key[2] >> 32) == ctx->i6w_fib_index)
+    {
+        ctx->i6w_fn(kvp->value, ctx->i6w_ctx);
+    }
+
+    return (1);
+}
+
+void
+ip6_fib_table_walk (u32 fib_index,
+                    fib_table_walk_fn_t fn,
+                    void *arg)
+{
+    ip6_fib_walk_ctx_t ctx = {
+        .i6w_fib_index = fib_index,
+        .i6w_fn = fn,
+        .i6w_ctx = arg,
+    };
+    ip6_main_t *im = &ip6_main;
+
+    BV(clib_bihash_foreach_key_value_pair)(&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash,
+					   ip6_fib_walk_cb,
+					   &ctx);
+
+}
+
 typedef struct ip6_fib_show_ctx_t_ {
-    u32 fib_index;
     fib_node_index_t *entries;
 } ip6_fib_show_ctx_t;
 
-static void
-ip6_fib_table_collect_entries (clib_bihash_kv_24_8_t * kvp,
-			       void *arg)
+static int
+ip6_fib_table_show_walk (fib_node_index_t fib_entry_index,
+                         void *arg)
 {
     ip6_fib_show_ctx_t *ctx = arg;
 
-    if ((kvp->key[2] >> 32) == ctx->fib_index)
-    {
-	vec_add1(ctx->entries, kvp->value);
-    }
+    vec_add1(ctx->entries, fib_entry_index);
+
+    return (1);
 }
 
 static void
@@ -541,15 +531,10 @@ ip6_fib_table_show_all (ip6_fib_t *fib,
 {
     fib_node_index_t *fib_entry_index;
     ip6_fib_show_ctx_t ctx = {
-	.fib_index = fib->index,
 	.entries = NULL,
     };
-    ip6_main_t *im = &ip6_main;
-
-    BV(clib_bihash_foreach_key_value_pair)(&im->ip6_table[IP6_FIB_TABLE_NON_FWDING].ip6_hash,
-					   ip6_fib_table_collect_entries,
-					   &ctx);
 
+    ip6_fib_table_walk(fib->index, ip6_fib_table_show_walk, &ctx);
     vec_sort_with_function(ctx.entries, fib_entry_cmp_for_sort);
 
     vec_foreach(fib_entry_index, ctx.entries)
diff --git a/src/vnet/fib/ip6_fib.h b/src/vnet/fib/ip6_fib.h
index f6af993a..78da3746 100644
--- a/src/vnet/fib/ip6_fib.h
+++ b/src/vnet/fib/ip6_fib.h
@@ -54,9 +54,18 @@ u32 ip6_fib_table_fwding_lookup_with_if_index(ip6_main_t * im,
 					      u32 sw_if_index,
 					      const ip6_address_t * dst);
 u32 ip6_fib_table_fwding_lookup(ip6_main_t * im,
-				u32 fib_index, 
+				u32 fib_index,
 				const ip6_address_t * dst);
 
+/**
+ * @brief Walk all entries in a FIB table
+ * N.B: This is NOT safe to deletes. If you need to delete walk the whole
+ * table and store elements in a vector, then delete the elements
+ */
+extern void ip6_fib_table_walk(u32 fib_index,
+                               fib_table_walk_fn_t fn,
+                               void *ctx);
+
 /**
  * @biref return the DPO that the LB stacks on.
  */
diff --git a/src/vnet/fib/mpls_fib.c b/src/vnet/fib/mpls_fib.c
index 6a9b1ac2..cc657975 100644
--- a/src/vnet/fib/mpls_fib.c
+++ b/src/vnet/fib/mpls_fib.c
@@ -44,10 +44,11 @@
  * Switching between schemes based on observed/measured action similarity is not
  * considered on the grounds of complexity and flip-flopping.
  *
- * VPP mantra - favour performance over memory. We choose a 21 bit key.  
+ * VPP mantra - favour performance over memory. We choose a 21 bit key.
  */
 
 #include <vnet/fib/fib_table.h>
+#include <vnet/fib/mpls_fib.h>
 #include <vnet/dpo/load_balance.h>
 #include <vnet/dpo/drop_dpo.h>
 #include <vnet/dpo/punt_dpo.h>
@@ -342,6 +343,20 @@ mpls_fib_table_get_flow_hash_config (u32 fib_index)
     return (0);
 }
 
+void
+mpls_fib_table_walk (mpls_fib_t *mpls_fib,
+                     fib_table_walk_fn_t fn,
+                     void *ctx)
+{
+    fib_node_index_t lfei;
+    mpls_label_t key;
+
+    hash_foreach(key, lfei, mpls_fib->mf_entries,
+    ({
+	fn(lfei, ctx);
+    }));
+}
+
 static void
 mpls_fib_table_show_all (const mpls_fib_t *mpls_fib,
 			 vlib_main_t * vm)
diff --git a/src/vnet/fib/mpls_fib.h b/src/vnet/fib/mpls_fib.h
index 93ae4623..e2ef9253 100644
--- a/src/vnet/fib/mpls_fib.h
+++ b/src/vnet/fib/mpls_fib.h
@@ -70,6 +70,15 @@ extern void mpls_fib_forwarding_table_reset(mpls_fib_t *mf,
 					    mpls_label_t label,
 					    mpls_eos_bit_t eos);
 
+/**
+ * @brief Walk all entries in a FIB table
+ * N.B: This is NOT safe to deletes. If you need to delete walk the whole
+ * table and store elements in a vector, then delete the elements
+ */
+extern void mpls_fib_table_walk(mpls_fib_t *fib,
+                                fib_table_walk_fn_t fn,
+                                void *ctx);
+
 /**
  * @brief
  *  Lookup a label and EOS bit in the MPLS_FIB table to retrieve the
diff --git a/src/vnet/ip/ip.api b/src/vnet/ip/ip.api
index 5371696c..65f6e7a7 100644
--- a/src/vnet/ip/ip.api
+++ b/src/vnet/ip/ip.api
@@ -395,6 +395,41 @@ define ip_add_del_route_reply
   i32 retval;
 };
 
+/** \brief Add / del route request
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - software index of the new vlan's parent interface
+    @param vrf_id - fib table /vrf associated with the route
+
+    FIXME
+*/
+define ip_mroute_add_del
+{
+  u32 client_index;
+  u32 context;
+  u32 next_hop_sw_if_index;
+  u32 table_id;
+  u32 entry_flags;
+  u32 itf_flags;
+  u16 grp_address_length;
+  u8 create_vrf_if_needed;
+  u8 is_add;
+  u8 is_ipv6;
+  u8 is_local;
+  u8 grp_address[16];
+  u8 src_address[16];
+};
+
+/** \brief Reply for add / del mroute request
+    @param context - returned sender context, to match reply w/ request
+    @param retval - return code
+*/
+define ip_mroute_add_del_reply
+{
+  u32 context;
+  i32 retval;
+};
+
 define ip_address_details
 {
   u32 client_index;
@@ -424,6 +459,24 @@ define ip_dump
   u8 is_ipv6;
 };
 
+define mfib_signal_dump
+{
+  u32 client_index;
+  u32 context;
+};
+
+define mfib_signal_details
+{
+  u32 client_index;
+  u32 context;
+  u32 sw_if_index;
+  u32 table_id;
+  u16 grp_address_len;
+  u8 grp_address[16];
+  u8 src_address[16];
+  u16 ip_packet_len;
+  u8 ip_packet_data[256];
+};
 
 /*
  * Local Variables:
diff --git a/src/vnet/ip/ip4.h b/src/vnet/ip/ip4.h
index cc33dff4..b184fbae 100644
--- a/src/vnet/ip/ip4.h
+++ b/src/vnet/ip/ip4.h
@@ -68,6 +68,18 @@ typedef struct ip4_fib_t
 
 } ip4_fib_t;
 
+typedef struct ip4_mfib_t
+{
+  /* Hash table for each prefix length mapping. */
+  uword *fib_entry_by_dst_address[65];
+
+  /* Table ID (hash key) for this FIB. */
+  u32 table_id;
+
+  /* Index into FIB vector. */
+  u32 index;
+} ip4_mfib_t;
+
 struct ip4_main_t;
 
 typedef void (ip4_add_del_interface_address_function_t)
@@ -99,11 +111,17 @@ typedef struct ip4_main_t
   /** Vector of FIBs. */
   struct fib_table_t_ *fibs;
 
+  /** Vector of MFIBs. */
+  struct mfib_table_t_ *mfibs;
+
   u32 fib_masks[33];
 
   /** Table index indexed by software interface. */
   u32 *fib_index_by_sw_if_index;
 
+  /** Table index indexed by software interface. */
+  u32 *mfib_index_by_sw_if_index;
+
   /* IP4 enabled count by software interface */
   u8 *ip_enabled_by_sw_if_index;
 
@@ -111,6 +129,10 @@ typedef struct ip4_main_t
      ID space is not necessarily dense; index space is dense. */
   uword *fib_index_by_table_id;
 
+  /** Hash table mapping table id to multicast fib index.
+     ID space is not necessarily dense; index space is dense. */
+  uword *mfib_index_by_table_id;
+
   /** Functions to call when interface address changes. */
     ip4_add_del_interface_address_callback_t
     * add_del_interface_address_callbacks;
@@ -140,7 +162,9 @@ extern ip4_main_t ip4_main;
 /** Global ip4 input node.  Errors get attached to ip4 input node. */
 extern vlib_node_registration_t ip4_input_node;
 extern vlib_node_registration_t ip4_lookup_node;
+extern vlib_node_registration_t ip4_local_node;
 extern vlib_node_registration_t ip4_rewrite_node;
+extern vlib_node_registration_t ip4_rewrite_mcast_node;
 extern vlib_node_registration_t ip4_rewrite_local_node;
 extern vlib_node_registration_t ip4_arp_node;
 extern vlib_node_registration_t ip4_glean_node;
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index 87b345bd..8081b34b 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -50,6 +50,7 @@
 #include <vnet/fib/ip4_fib.h>
 #include <vnet/dpo/load_balance.h>
 #include <vnet/dpo/classify_dpo.h>
+#include <vnet/mfib/mfib_table.h>	/* for mFIB table and entry creation */
 
 /**
  * @file
@@ -714,16 +715,17 @@ ip4_interface_first_address (ip4_main_t * im, u32 sw_if_index,
   ip_interface_address_t *ia = 0;
   ip4_address_t *result = 0;
 
-  foreach_ip_interface_address (lm, ia, sw_if_index,
-				1 /* honor unnumbered */ ,
-				(
-				  {
-				  ip4_address_t * a =
-				  ip_interface_address_get_address (lm, ia);
-				  result = a;
-				  break;
-				  }
-				));
+  /* *INDENT-OFF* */
+  foreach_ip_interface_address
+    (lm, ia, sw_if_index,
+     1 /* honor unnumbered */ ,
+     ({
+       ip4_address_t * a =
+         ip_interface_address_get_address (lm, ia);
+       result = a;
+       break;
+     }));
+  /* *INDENT-OFF* */
   if (result_ia)
     *result_ia = result ? ia : 0;
   return result;
@@ -748,9 +750,19 @@ ip4_add_interface_routes (u32 sw_if_index,
     {
       fib_node_index_t fei;
 
-      fei = fib_table_entry_update_one_path (fib_index, &pfx, FIB_SOURCE_INTERFACE, (FIB_ENTRY_FLAG_CONNECTED | FIB_ENTRY_FLAG_ATTACHED), FIB_PROTOCOL_IP4, NULL,	/* No next-hop address */
-					     sw_if_index, ~0,	// invalid FIB index
-					     1, NULL,	// no out-label stack
+      fei = fib_table_entry_update_one_path (fib_index, &pfx,
+                                             FIB_SOURCE_INTERFACE,
+                                             (FIB_ENTRY_FLAG_CONNECTED |
+                                              FIB_ENTRY_FLAG_ATTACHED),
+                                             FIB_PROTOCOL_IP4,
+                                             /* No next-hop address */
+                                             NULL,
+					     sw_if_index,
+                                             // invalid FIB index
+                                             ~0,
+					     1,
+                                             // no out-label stack
+                                             NULL,
 					     FIB_ROUTE_PATH_FLAG_NONE);
       a->neighbor_probe_adj_index = fib_entry_get_adj (fei);
     }
@@ -778,8 +790,16 @@ ip4_add_interface_routes (u32 sw_if_index,
 	}
     }
 
-  fib_table_entry_update_one_path (fib_index, &pfx, FIB_SOURCE_INTERFACE, (FIB_ENTRY_FLAG_CONNECTED | FIB_ENTRY_FLAG_LOCAL), FIB_PROTOCOL_IP4, &pfx.fp_addr, sw_if_index, ~0,	// invalid FIB index
-				   1, NULL,	// no out-label stack
+  fib_table_entry_update_one_path (fib_index, &pfx,
+                                   FIB_SOURCE_INTERFACE,
+                                   (FIB_ENTRY_FLAG_CONNECTED |
+                                    FIB_ENTRY_FLAG_LOCAL),
+                                   FIB_PROTOCOL_IP4,
+                                   &pfx.fp_addr,
+                                   sw_if_index,
+                                   // invalid FIB index
+                                   ~0,
+				   1, NULL,
 				   FIB_ROUTE_PATH_FLAG_NONE);
 }
 
@@ -827,9 +847,10 @@ ip4_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
 			       !is_enable, 0, 0);
 
-  vnet_feature_enable_disable ("ip4-multicast", "ip4-drop", sw_if_index,
-			       !is_enable, 0, 0);
 
+  vnet_feature_enable_disable ("ip4-multicast",
+			       "ip4-mfib-forward-lookup",
+			       sw_if_index, is_enable, 0, 0);
 }
 
 static clib_error_t *
@@ -855,36 +876,37 @@ ip4_add_del_interface_address_internal (vlib_main_t * vm,
    * subnets on interfaces. Easy fix - disallow overlapping subnets, like
    * most routers do.
    */
+  /* *INDENT-OFF* */
   if (!is_del)
     {
       /* When adding an address check that it does not conflict
          with an existing address. */
       ip_interface_address_t *ia;
-      foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
-				    0 /* honor unnumbered */ ,
-				    (
-				      {
-				      ip4_address_t * x =
-				      ip_interface_address_get_address
-				      (&im->lookup_main, ia);
-				      if (ip4_destination_matches_route
-					  (im, address, x, ia->address_length)
-					  ||
-					  ip4_destination_matches_route (im,
-									 x,
-									 address,
-									 address_length))
-				      return
-				      clib_error_create
-				      ("failed to add %U which conflicts with %U for interface %U",
-				       format_ip4_address_and_length, address,
-				       address_length,
-				       format_ip4_address_and_length, x,
-				       ia->address_length,
-				       format_vnet_sw_if_index_name, vnm,
-				       sw_if_index);}
-				    ));
+      foreach_ip_interface_address
+        (&im->lookup_main, ia, sw_if_index,
+         0 /* honor unnumbered */ ,
+         ({
+           ip4_address_t * x =
+             ip_interface_address_get_address
+             (&im->lookup_main, ia);
+           if (ip4_destination_matches_route
+               (im, address, x, ia->address_length) ||
+               ip4_destination_matches_route (im,
+                                              x,
+                                              address,
+                                              address_length))
+             return
+               clib_error_create
+               ("failed to add %U which conflicts with %U for interface %U",
+                format_ip4_address_and_length, address,
+                address_length,
+                format_ip4_address_and_length, x,
+                ia->address_length,
+                format_vnet_sw_if_index_name, vnm,
+                sw_if_index);
+         }));
     }
+  /* *INDENT-ON* */
 
   elts_before = pool_elts (lm->if_address_pool);
 
@@ -918,9 +940,10 @@ done:
 }
 
 clib_error_t *
-ip4_add_del_interface_address (vlib_main_t * vm, u32 sw_if_index,
-			       ip4_address_t * address, u32 address_length,
-			       u32 is_del)
+ip4_add_del_interface_address (vlib_main_t * vm,
+			       u32 sw_if_index,
+			       ip4_address_t * address,
+			       u32 address_length, u32 is_del)
 {
   return ip4_add_del_interface_address_internal
     (vm, sw_if_index, address, address_length, is_del);
@@ -1027,13 +1050,13 @@ VNET_FEATURE_INIT (ip4_vpath_mc, static) =
 {
   .arc_name = "ip4-multicast",
   .node_name = "vpath-input-ip4",
-  .runs_before = VNET_FEATURES ("ip4-lookup-multicast"),
+  .runs_before = VNET_FEATURES ("ip4-mfib-forward-lookup"),
 };
 
 VNET_FEATURE_INIT (ip4_lookup_mc, static) =
 {
   .arc_name = "ip4-multicast",
-  .node_name = "ip4-lookup-multicast",
+  .node_name = "ip4-mfib-forward-lookup",
   .runs_before = VNET_FEATURES ("ip4-drop"),
 };
 
@@ -1083,6 +1106,7 @@ ip4_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 
   /* Fill in lookup tables with default table (0). */
   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
+  vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
 
   vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
 			       is_add, 0, 0);
@@ -1123,6 +1147,7 @@ ip4_lookup_init (vlib_main_t * vm)
 
   /* Create FIB with index 0 and table id of 0. */
   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0);
+  mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0);
 
   {
     pg_node_t *pn;
@@ -1341,27 +1366,35 @@ ip4_punt (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
   return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_PUNT);
 }
 
+/* *INDENT-OFF* */
 VLIB_REGISTER_NODE (ip4_drop_node, static) =
 {
-  .function = ip4_drop,.name = "ip4-drop",.vector_size =
-    sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_next_nodes =
-    1,.next_nodes =
-  {
-  [0] = "error-drop",}
-,};
+  .function = ip4_drop,.
+  name = "ip4-drop",
+  .vector_size = sizeof (u32),
+  .format_trace = format_ip4_forward_next_trace,
+  .n_next_nodes = 1,
+  .next_nodes = {
+    [0] = "error-drop",
+  },
+};
 
 VLIB_NODE_FUNCTION_MULTIARCH (ip4_drop_node, ip4_drop);
 
 VLIB_REGISTER_NODE (ip4_punt_node, static) =
 {
-  .function = ip4_punt,.name = "ip4-punt",.vector_size =
-    sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_next_nodes =
-    1,.next_nodes =
-  {
-  [0] = "error-punt",}
-,};
+  .function = ip4_punt,
+  .name = "ip4-punt",
+  .vector_size = sizeof (u32),
+  .format_trace = format_ip4_forward_next_trace,
+  .n_next_nodes = 1,
+  .next_nodes = {
+    [0] = "error-punt",
+  },
+};
 
 VLIB_NODE_FUNCTION_MULTIARCH (ip4_punt_node, ip4_punt);
+/* *INDENT-ON */
 
 /* Compute TCP/UDP/ICMP4 checksum in software. */
 u16
@@ -1500,13 +1533,15 @@ ip4_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
 
 	  fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
 				vnet_buffer (p0)->sw_if_index[VLIB_RX]);
-	  fib_index0 = (vnet_buffer (p0)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
-	    fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
+	  fib_index0 =
+	    (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
+	     (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 
 	  fib_index1 = vec_elt (im->fib_index_by_sw_if_index,
 				vnet_buffer (p1)->sw_if_index[VLIB_RX]);
-	  fib_index1 = (vnet_buffer (p1)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
-	    fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
+	  fib_index1 =
+	    (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
+	     (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
 
 	  mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 	  mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
@@ -1726,8 +1761,9 @@ ip4_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
 
 	  fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
 				vnet_buffer (p0)->sw_if_index[VLIB_RX]);
-	  fib_index0 = (vnet_buffer (p0)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
-	    fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
+	  fib_index0 =
+	    (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
+	     (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
 
 	  mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
 
@@ -1838,10 +1874,11 @@ ip4_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
   return frame->n_vectors;
 }
 
-VLIB_REGISTER_NODE (ip4_local_node, static) =
+VLIB_REGISTER_NODE (ip4_local_node) =
 {
   .function = ip4_local,.name = "ip4-local",.vector_size =
-    sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_next_nodes =
+    sizeof (u32),.format_trace =
+    format_ip4_forward_next_trace,.n_next_nodes =
     IP_LOCAL_N_NEXT,.next_nodes =
   {
   [IP_LOCAL_NEXT_DROP] = "error-drop",
@@ -2022,8 +2059,8 @@ ip4_arp_inline (vlib_main_t * vm,
 	   * Can happen if the control-plane is programming tables
 	   * with traffic flowing; at least that's today's lame excuse.
 	   */
-	  if ((is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_GLEAN) ||
-	      (!is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP))
+	  if ((is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_GLEAN)
+	      || (!is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP))
 	    {
 	      p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
 	    }
@@ -2196,15 +2233,17 @@ ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
     {
       vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
       return clib_error_return
-	(0, "no matching interface address for destination %U (interface %U)",
-	 format_ip4_address, dst,
-	 format_vnet_sw_if_index_name, vnm, sw_if_index);
+	(0,
+	 "no matching interface address for destination %U (interface %U)",
+	 format_ip4_address, dst, format_vnet_sw_if_index_name, vnm,
+	 sw_if_index);
     }
 
   adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
 
   h =
-    vlib_packet_template_get_packet (vm, &im->ip4_arp_request_packet_template,
+    vlib_packet_template_get_packet (vm,
+				     &im->ip4_arp_request_packet_template,
 				     &bi);
 
   hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
@@ -2243,7 +2282,7 @@ typedef enum
 always_inline uword
 ip4_rewrite_inline (vlib_main_t * vm,
 		    vlib_node_runtime_t * node,
-		    vlib_frame_t * frame, int is_midchain)
+		    vlib_frame_t * frame, int is_midchain, int is_mcast)
 {
   ip_lookup_main_t *lm = &ip4_main.lookup_main;
   u32 *from = vlib_frame_vector_args (frame);
@@ -2457,6 +2496,14 @@ ip4_rewrite_inline (vlib_main_t * vm,
 	      adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
 	      adj1->sub_type.midchain.fixup_func (vm, adj1, p1);
 	    }
+	  if (is_mcast)
+	    {
+	      /*
+	       * copy bytes from the IP address into the MAC rewrite
+	       */
+	      vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0, 1);
+	      vnet_fixup_one_header (adj1[0], &ip1->dst_address, ip1, 1);
+	    }
 
 	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 					   to_next, n_left_to_next,
@@ -2530,6 +2577,13 @@ ip4_rewrite_inline (vlib_main_t * vm,
 
 	  /* Guess we are only writing on simple Ethernet header. */
 	  vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
+	  if (is_mcast)
+	    {
+	      /*
+	       * copy bytes from the IP address into the MAC rewrite
+	       */
+	      vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0, 1);
+	    }
 
 	  /* Update packet buffer attributes/set output interface. */
 	  rw_len0 = adj0[0].rewrite_header.data_bytes;
@@ -2624,36 +2678,58 @@ static uword
 ip4_rewrite (vlib_main_t * vm,
 	     vlib_node_runtime_t * node, vlib_frame_t * frame)
 {
-  return ip4_rewrite_inline (vm, node, frame, 0);
+  return ip4_rewrite_inline (vm, node, frame, 0, 0);
 }
 
 static uword
 ip4_midchain (vlib_main_t * vm,
 	      vlib_node_runtime_t * node, vlib_frame_t * frame)
 {
-  return ip4_rewrite_inline (vm, node, frame, 1);
+  return ip4_rewrite_inline (vm, node, frame, 1, 0);
 }
 
-
-VLIB_REGISTER_NODE (ip4_rewrite_node) =
+static uword
+ip4_rewrite_mcast (vlib_main_t * vm,
+		   vlib_node_runtime_t * node, vlib_frame_t * frame)
 {
-  .function = ip4_rewrite,.name = "ip4-rewrite",.vector_size =
-    sizeof (u32),.format_trace = format_ip4_rewrite_trace,.n_next_nodes =
-    2,.next_nodes =
-  {
-  [IP4_REWRITE_NEXT_DROP] = "error-drop",
-      [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",}
-,};
+  return ip4_rewrite_inline (vm, node, frame, 0, 1);
+}
 
-VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite);
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (ip4_rewrite_node) = {
+  .function = ip4_rewrite,
+  .name = "ip4-rewrite",
+  .vector_size = sizeof (u32),
+
+  .format_trace = format_ip4_rewrite_trace,
+
+  .n_next_nodes = 2,
+  .next_nodes = {
+    [IP4_REWRITE_NEXT_DROP] = "error-drop",
+    [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",
+  },
+};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite)
 
-VLIB_REGISTER_NODE (ip4_midchain_node) =
-{
-.function = ip4_midchain,.name = "ip4-midchain",.vector_size =
-    sizeof (u32),.format_trace = format_ip4_forward_next_trace,.sibling_of =
-    "ip4-rewrite",};
+VLIB_REGISTER_NODE (ip4_rewrite_mcast_node) = {
+  .function = ip4_rewrite_mcast,
+  .name = "ip4-rewrite-mcast",
+  .vector_size = sizeof (u32),
 
+  .format_trace = format_ip4_rewrite_trace,
+  .sibling_of = "ip4-rewrite",
+};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_mcast_node, ip4_rewrite_mcast)
+
+VLIB_REGISTER_NODE (ip4_midchain_node) = {
+  .function = ip4_midchain,
+  .name = "ip4-midchain",
+  .vector_size = sizeof (u32),
+  .format_trace = format_ip4_forward_next_trace,
+  .sibling_of =  "ip4-rewrite",
+};
 VLIB_NODE_FUNCTION_MULTIARCH (ip4_midchain_node, ip4_midchain);
+/* *INDENT-ON */
 
 static clib_error_t *
 add_del_interface_table (vlib_main_t * vm,
@@ -2695,6 +2771,11 @@ add_del_interface_table (vlib_main_t * vm,
     //
     vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
     im->fib_index_by_sw_if_index[sw_if_index] = fib_index;
+
+    fib_index = mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
+                                                   table_id);
+    vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
+    im->mfib_index_by_sw_if_index[sw_if_index] = fib_index;
   }
 
 done:
@@ -2730,243 +2811,6 @@ VLIB_CLI_COMMAND (set_interface_ip_table_command, static) =
 };
 /* *INDENT-ON* */
 
-
-static uword
-ip4_lookup_multicast (vlib_main_t * vm,
-		      vlib_node_runtime_t * node, vlib_frame_t * frame)
-{
-  ip4_main_t *im = &ip4_main;
-  vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters;
-  u32 n_left_from, n_left_to_next, *from, *to_next;
-  ip_lookup_next_t next;
-  u32 cpu_index = os_get_cpu_number ();
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
-
-      while (n_left_from >= 4 && n_left_to_next >= 2)
-	{
-	  vlib_buffer_t *p0, *p1;
-	  u32 pi0, pi1, lb_index0, lb_index1, wrong_next;
-	  ip_lookup_next_t next0, next1;
-	  ip4_header_t *ip0, *ip1;
-	  u32 fib_index0, fib_index1;
-	  const dpo_id_t *dpo0, *dpo1;
-	  const load_balance_t *lb0, *lb1;
-
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t *p2, *p3;
-
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-
-	    CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
-	    CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
-	  }
-
-	  pi0 = to_next[0] = from[0];
-	  pi1 = to_next[1] = from[1];
-
-	  p0 = vlib_get_buffer (vm, pi0);
-	  p1 = vlib_get_buffer (vm, pi1);
-
-	  ip0 = vlib_buffer_get_current (p0);
-	  ip1 = vlib_buffer_get_current (p1);
-
-	  fib_index0 =
-	    vec_elt (im->fib_index_by_sw_if_index,
-		     vnet_buffer (p0)->sw_if_index[VLIB_RX]);
-	  fib_index1 =
-	    vec_elt (im->fib_index_by_sw_if_index,
-		     vnet_buffer (p1)->sw_if_index[VLIB_RX]);
-	  fib_index0 =
-	    (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
-	     (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
-	  fib_index1 =
-	    (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
-	     (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
-
-	  lb_index0 = ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0),
-					       &ip0->dst_address);
-	  lb_index1 = ip4_fib_table_lookup_lb (ip4_fib_get (fib_index1),
-					       &ip1->dst_address);
-
-	  lb0 = load_balance_get (lb_index0);
-	  lb1 = load_balance_get (lb_index1);
-
-	  ASSERT (lb0->lb_n_buckets > 0);
-	  ASSERT (is_pow2 (lb0->lb_n_buckets));
-	  ASSERT (lb1->lb_n_buckets > 0);
-	  ASSERT (is_pow2 (lb1->lb_n_buckets));
-
-	  vnet_buffer (p0)->ip.flow_hash = ip4_compute_flow_hash
-	    (ip0, lb0->lb_hash_config);
-
-	  vnet_buffer (p1)->ip.flow_hash = ip4_compute_flow_hash
-	    (ip1, lb1->lb_hash_config);
-
-	  dpo0 = load_balance_get_bucket_i (lb0,
-					    (vnet_buffer (p0)->ip.flow_hash &
-					     (lb0->lb_n_buckets_minus_1)));
-	  dpo1 = load_balance_get_bucket_i (lb1,
-					    (vnet_buffer (p1)->ip.flow_hash &
-					     (lb1->lb_n_buckets_minus_1)));
-
-	  next0 = dpo0->dpoi_next_node;
-	  vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
-	  next1 = dpo1->dpoi_next_node;
-	  vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
-
-	  if (1)		/* $$$$$$ HACK FIXME */
-	    vlib_increment_combined_counter
-	      (cm, cpu_index, lb_index0, 1,
-	       vlib_buffer_length_in_chain (vm, p0));
-	  if (1)		/* $$$$$$ HACK FIXME */
-	    vlib_increment_combined_counter
-	      (cm, cpu_index, lb_index1, 1,
-	       vlib_buffer_length_in_chain (vm, p1));
-
-	  from += 2;
-	  to_next += 2;
-	  n_left_to_next -= 2;
-	  n_left_from -= 2;
-
-	  wrong_next = (next0 != next) + 2 * (next1 != next);
-	  if (PREDICT_FALSE (wrong_next != 0))
-	    {
-	      switch (wrong_next)
-		{
-		case 1:
-		  /* A B A */
-		  to_next[-2] = pi1;
-		  to_next -= 1;
-		  n_left_to_next += 1;
-		  vlib_set_next_frame_buffer (vm, node, next0, pi0);
-		  break;
-
-		case 2:
-		  /* A A B */
-		  to_next -= 1;
-		  n_left_to_next += 1;
-		  vlib_set_next_frame_buffer (vm, node, next1, pi1);
-		  break;
-
-		case 3:
-		  /* A B C */
-		  to_next -= 2;
-		  n_left_to_next += 2;
-		  vlib_set_next_frame_buffer (vm, node, next0, pi0);
-		  vlib_set_next_frame_buffer (vm, node, next1, pi1);
-		  if (next0 == next1)
-		    {
-		      /* A B B */
-		      vlib_put_next_frame (vm, node, next, n_left_to_next);
-		      next = next1;
-		      vlib_get_next_frame (vm, node, next, to_next,
-					   n_left_to_next);
-		    }
-		}
-	    }
-	}
-
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  vlib_buffer_t *p0;
-	  ip4_header_t *ip0;
-	  u32 pi0, lb_index0;
-	  ip_lookup_next_t next0;
-	  u32 fib_index0;
-	  const dpo_id_t *dpo0;
-	  const load_balance_t *lb0;
-
-	  pi0 = from[0];
-	  to_next[0] = pi0;
-
-	  p0 = vlib_get_buffer (vm, pi0);
-
-	  ip0 = vlib_buffer_get_current (p0);
-
-	  fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
-				vnet_buffer (p0)->sw_if_index[VLIB_RX]);
-	  fib_index0 = (vnet_buffer (p0)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
-	    fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
-
-	  lb_index0 = ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0),
-					       &ip0->dst_address);
-
-	  lb0 = load_balance_get (lb_index0);
-
-	  ASSERT (lb0->lb_n_buckets > 0);
-	  ASSERT (is_pow2 (lb0->lb_n_buckets));
-
-	  vnet_buffer (p0)->ip.flow_hash = ip4_compute_flow_hash
-	    (ip0, lb0->lb_hash_config);
-
-	  dpo0 = load_balance_get_bucket_i (lb0,
-					    (vnet_buffer (p0)->ip.flow_hash &
-					     (lb0->lb_n_buckets_minus_1)));
-
-	  next0 = dpo0->dpoi_next_node;
-	  vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
-
-	  if (1)		/* $$$$$$ HACK FIXME */
-	    vlib_increment_combined_counter
-	      (cm, cpu_index, lb_index0, 1,
-	       vlib_buffer_length_in_chain (vm, p0));
-
-	  from += 1;
-	  to_next += 1;
-	  n_left_to_next -= 1;
-	  n_left_from -= 1;
-
-	  if (PREDICT_FALSE (next0 != next))
-	    {
-	      n_left_to_next += 1;
-	      vlib_put_next_frame (vm, node, next, n_left_to_next);
-	      next = next0;
-	      vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
-	      to_next[0] = pi0;
-	      to_next += 1;
-	      n_left_to_next -= 1;
-	    }
-	}
-
-      vlib_put_next_frame (vm, node, next, n_left_to_next);
-    }
-
-  if (node->flags & VLIB_NODE_FLAG_TRACE)
-    ip4_forward_next_trace (vm, node, frame, VLIB_TX);
-
-  return frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (ip4_lookup_multicast_node, static) =
-{
-.function = ip4_lookup_multicast,.name =
-    "ip4-lookup-multicast",.vector_size = sizeof (u32),.sibling_of =
-    "ip4-lookup",.format_trace = format_ip4_lookup_trace,.n_next_nodes = 0,};
-
-VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_multicast_node,
-			      ip4_lookup_multicast);
-
-VLIB_REGISTER_NODE (ip4_multicast_node, static) =
-{
-  .function = ip4_drop,.name = "ip4-multicast",.vector_size =
-    sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_next_nodes =
-    1,.next_nodes =
-  {
-  [0] = "error-drop",}
-,};
-
 int
 ip4_lookup_validate (ip4_address_t * a, u32 fib_index0)
 {
diff --git a/src/vnet/ip/ip4_input.c b/src/vnet/ip/ip4_input.c
index 1cf5e0b8..ba200a9f 100644
--- a/src/vnet/ip/ip4_input.c
+++ b/src/vnet/ip/ip4_input.c
@@ -426,7 +426,7 @@ VLIB_REGISTER_NODE (ip4_input_node) = {
     [IP4_INPUT_NEXT_DROP] = "error-drop",
     [IP4_INPUT_NEXT_PUNT] = "error-punt",
     [IP4_INPUT_NEXT_LOOKUP] = "ip4-lookup",
-    [IP4_INPUT_NEXT_LOOKUP_MULTICAST] = "ip4-lookup-multicast",
+    [IP4_INPUT_NEXT_LOOKUP_MULTICAST] = "ip4-mfib-forward-lookup",
     [IP4_INPUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
   },
 
@@ -448,7 +448,7 @@ VLIB_REGISTER_NODE (ip4_input_no_checksum_node,static) = {
     [IP4_INPUT_NEXT_DROP] = "error-drop",
     [IP4_INPUT_NEXT_PUNT] = "error-punt",
     [IP4_INPUT_NEXT_LOOKUP] = "ip4-lookup",
-    [IP4_INPUT_NEXT_LOOKUP_MULTICAST] = "ip4-lookup-multicast",
+    [IP4_INPUT_NEXT_LOOKUP_MULTICAST] = "ip4-mfib-forward-lookup",
     [IP4_INPUT_NEXT_ICMP_ERROR] = "ip4-icmp-error",
   },
 
diff --git a/src/vnet/ip/ip6.h b/src/vnet/ip/ip6.h
index f493db01..6fecd42d 100644
--- a/src/vnet/ip/ip6.h
+++ b/src/vnet/ip/ip6.h
@@ -49,6 +49,7 @@
 #include <stdbool.h>
 #include <vppinfra/bihash_24_8.h>
 #include <vppinfra/bihash_template.h>
+#include <vnet/util/radix.h>
 
 /*
  * Default size of the ip6 fib hash table
@@ -75,6 +76,21 @@ typedef struct
   flow_hash_config_t flow_hash_config;
 } ip6_fib_t;
 
+typedef struct ip6_mfib_t
+{
+  /* Table ID (hash key) for this FIB. */
+  u32 table_id;
+
+  /* Index into FIB vector. */
+  u32 index;
+
+  /*
+   *  Pointer to the top of a radix tree.
+   * This cannot be realloc'd, hence it cannot be inlined with this table
+   */
+  struct radix_node_head *rhead;
+} ip6_mfib_t;
+
 struct ip6_main_t;
 
 typedef void (ip6_add_del_interface_address_function_t)
@@ -137,12 +153,18 @@ typedef struct ip6_main_t
   /* Pool of FIBs. */
   struct fib_table_t_ *fibs;
 
+  /** Vector of MFIBs. */
+  struct mfib_table_t_ *mfibs;
+
   /* Network byte orders subnet mask for each prefix length */
   ip6_address_t fib_masks[129];
 
   /* Table index indexed by software interface. */
   u32 *fib_index_by_sw_if_index;
 
+  /** Table index indexed by software interface. */
+  u32 *mfib_index_by_sw_if_index;
+
   /* IP6 enabled count by software interface */
   u8 *ip_enabled_by_sw_if_index;
 
@@ -150,6 +172,10 @@ typedef struct ip6_main_t
      ID space is not necessarily dense; index space is dense. */
   uword *fib_index_by_table_id;
 
+  /** Hash table mapping table id to multicast fib index.
+     ID space is not necessarily dense; index space is dense. */
+  uword *mfib_index_by_table_id;
+
   /* Hash table mapping interface rewrite adjacency index by sw if index. */
   uword *interface_route_adj_index_by_sw_if_index;
 
@@ -185,6 +211,7 @@ extern ip6_main_t ip6_main;
 /* Global ip6 input node.  Errors get attached to ip6 input node. */
 extern vlib_node_registration_t ip6_input_node;
 extern vlib_node_registration_t ip6_rewrite_node;
+extern vlib_node_registration_t ip6_rewrite_mcast_node;
 extern vlib_node_registration_t ip6_rewrite_local_node;
 extern vlib_node_registration_t ip6_discover_neighbor_node;
 extern vlib_node_registration_t ip6_glean_node;
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index 232f7283..ac47b3ad 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -42,8 +42,8 @@
 #include <vnet/ethernet/ethernet.h>	/* for ethernet_header_t */
 #include <vnet/srp/srp.h>	/* for srp_hw_interface_class */
 #include <vppinfra/cache.h>
-#include <vnet/fib/fib_table.h>
 #include <vnet/fib/ip6_fib.h>
+#include <vnet/mfib/ip6_mfib.h>
 #include <vnet/dpo/load_balance.h>
 #include <vnet/dpo/classify_dpo.h>
 
@@ -411,11 +411,14 @@ ip6_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
 	return;
     }
 
+  if (sw_if_index != 0)
+    ip6_mfib_interface_enable_disable (sw_if_index, is_enable);
+
   vnet_feature_enable_disable ("ip6-unicast", "ip6-lookup", sw_if_index,
 			       is_enable, 0, 0);
 
-  vnet_feature_enable_disable ("ip6-multicast", "ip6-lookup", sw_if_index,
-			       is_enable, 0, 0);
+  vnet_feature_enable_disable ("ip6-multicast", "ip6-mfib-forward-lookup",
+			       sw_if_index, is_enable, 0, 0);
 
 }
 
@@ -457,6 +460,8 @@ ip6_add_del_interface_address (vlib_main_t * vm,
   ip6_address_fib_t ip6_af, *addr_fib = 0;
 
   vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
+  vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
+
   ip6_addr_fib_init (&ip6_af, address,
 		     vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
   vec_add1 (addr_fib, ip6_af);
@@ -611,12 +616,12 @@ VNET_FEATURE_ARC_INIT (ip6_multicast, static) =
 VNET_FEATURE_INIT (ip6_vpath_mc, static) = {
   .arc_name = "ip6-multicast",
   .node_name = "vpath-input-ip6",
-  .runs_before = VNET_FEATURES ("ip6-lookup"),
+  .runs_before = VNET_FEATURES ("ip6-mfib-forward-lookup"),
 };
 
 VNET_FEATURE_INIT (ip6_mc_lookup, static) = {
   .arc_name = "ip6-multicast",
-  .node_name = "ip6-lookup",
+  .node_name = "ip6-mfib-forward-lookup",
   .runs_before = VNET_FEATURES ("ip6-drop"),
 };
 
@@ -1122,22 +1127,6 @@ VLIB_REGISTER_NODE (ip6_punt_node, static) =
 
 VLIB_NODE_FUNCTION_MULTIARCH (ip6_punt_node, ip6_punt);
 
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (ip6_multicast_node, static) =
-{
-  .function = ip6_drop,
-  .name = "ip6-multicast",
-  .vector_size = sizeof (u32),
-  .format_trace = format_ip6_forward_next_trace,
-  .n_next_nodes = 1,
-  .next_nodes =
-  {
-    [0] = "error-drop",
-  },
-};
-
-/* *INDENT-ON* */
-
 /* Compute TCP/UDP/ICMP6 checksum in software. */
 u16
 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0,
@@ -1977,7 +1966,7 @@ typedef enum
 always_inline uword
 ip6_rewrite_inline (vlib_main_t * vm,
 		    vlib_node_runtime_t * node,
-		    vlib_frame_t * frame, int is_midchain)
+		    vlib_frame_t * frame, int is_midchain, int is_mcast)
 {
   ip_lookup_main_t *lm = &ip6_main.lookup_main;
   u32 *from = vlib_frame_vector_args (frame);
@@ -2165,6 +2154,14 @@ ip6_rewrite_inline (vlib_main_t * vm,
 	      adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
 	      adj1->sub_type.midchain.fixup_func (vm, adj1, p1);
 	    }
+	  if (is_mcast)
+	    {
+	      /*
+	       * copy bytes from the IP address into the MAC rewrite
+	       */
+	      vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0, 0);
+	      vnet_fixup_one_header (adj1[0], &ip1->dst_address, ip1, 0);
+	    }
 
 	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 					   to_next, n_left_to_next,
@@ -2265,6 +2262,10 @@ ip6_rewrite_inline (vlib_main_t * vm,
 	    {
 	      adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
 	    }
+	  if (is_mcast)
+	    {
+	      vnet_fixup_one_header (adj0[0], &ip0->dst_address, ip0, 0);
+	    }
 
 	  p0->error = error_node->errors[error0];
 
@@ -2292,16 +2293,21 @@ static uword
 ip6_rewrite (vlib_main_t * vm,
 	     vlib_node_runtime_t * node, vlib_frame_t * frame)
 {
-  return ip6_rewrite_inline (vm, node, frame,
-			     /* midchain */ 0);
+  return ip6_rewrite_inline (vm, node, frame, 0, 0);
+}
+
+static uword
+ip6_rewrite_mcast (vlib_main_t * vm,
+		   vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  return ip6_rewrite_inline (vm, node, frame, 0, 1);
 }
 
 static uword
 ip6_midchain (vlib_main_t * vm,
 	      vlib_node_runtime_t * node, vlib_frame_t * frame)
 {
-  return ip6_rewrite_inline (vm, node, frame,
-			     /* midchain */ 1);
+  return ip6_rewrite_inline (vm, node, frame, 1, 0);
 }
 
 /* *INDENT-OFF* */
@@ -2335,10 +2341,22 @@ VLIB_REGISTER_NODE (ip6_rewrite_node) =
 
 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite);
 
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) =
+{
+  .function = ip6_rewrite_mcast,
+  .name = "ip6-rewrite-mcast",
+  .vector_size = sizeof (u32),
+  .format_trace = format_ip6_rewrite_trace,
+  .sibling_of = "ip6-rewrite",
+};
+/* *INDENT-ON* */
+
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_mcast_node, ip6_rewrite_mcast);
+
 /*
  * Hop-by-Hop handling
  */
-
 ip6_hop_by_hop_main_t ip6_hop_by_hop_main;
 
 #define foreach_ip6_hop_by_hop_error \
@@ -2346,13 +2364,15 @@ _(PROCESSED, "pkts with ip6 hop-by-hop options") \
 _(FORMAT, "incorrectly formatted hop-by-hop options") \
 _(UNKNOWN_OPTION, "unknown ip6 hop-by-hop options")
 
+/* *INDENT-OFF* */
 typedef enum
 {
 #define _(sym,str) IP6_HOP_BY_HOP_ERROR_##sym,
   foreach_ip6_hop_by_hop_error
 #undef _
-    IP6_HOP_BY_HOP_N_ERROR,
+  IP6_HOP_BY_HOP_N_ERROR,
 } ip6_hop_by_hop_error_t;
+/* *INDENT-ON* */
 
 /*
  * Primary h-b-h handler trace support
@@ -2878,6 +2898,7 @@ ip6_lookup_init (vlib_main_t * vm)
 
   /* Create FIB with index 0 and table id of 0. */
   fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0);
+  mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0);
 
   {
     pg_node_t *pn;
@@ -2955,6 +2976,12 @@ add_del_ip6_interface_table (vlib_main_t * vm,
 
     vec_validate (ip6_main.fib_index_by_sw_if_index, sw_if_index);
     ip6_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
+
+    fib_index = mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
+						    table_id);
+
+    vec_validate (ip6_main.mfib_index_by_sw_if_index, sw_if_index);
+    ip6_main.mfib_index_by_sw_if_index[sw_if_index] = fib_index;
   }
 
 
diff --git a/src/vnet/ip/ip6_input.c b/src/vnet/ip/ip6_input.c
index bbc2ceba..20306088 100644
--- a/src/vnet/ip/ip6_input.c
+++ b/src/vnet/ip/ip6_input.c
@@ -64,6 +64,7 @@ typedef enum
 {
   IP6_INPUT_NEXT_DROP,
   IP6_INPUT_NEXT_LOOKUP,
+  IP6_INPUT_NEXT_LOOKUP_MULTICAST,
   IP6_INPUT_NEXT_ICMP_ERROR,
   IP6_INPUT_N_NEXT,
 } ip6_input_next_t;
@@ -142,12 +143,27 @@ ip6_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
 	  sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
 	  sw_if_index1 = vnet_buffer (p1)->sw_if_index[VLIB_RX];
 
-	  arc0 =
-	    ip6_address_is_multicast (&ip0->dst_address) ?
-	    lm->mcast_feature_arc_index : lm->ucast_feature_arc_index;
-	  arc1 =
-	    ip6_address_is_multicast (&ip1->dst_address) ?
-	    lm->mcast_feature_arc_index : lm->ucast_feature_arc_index;
+	  if (PREDICT_FALSE (ip6_address_is_multicast (&ip0->dst_address)))
+	    {
+	      arc0 = lm->mcast_feature_arc_index;
+	      next0 = IP6_INPUT_NEXT_LOOKUP_MULTICAST;
+	    }
+	  else
+	    {
+	      arc0 = lm->ucast_feature_arc_index;
+	      next0 = IP6_INPUT_NEXT_LOOKUP;
+	    }
+
+	  if (PREDICT_FALSE (ip6_address_is_multicast (&ip1->dst_address)))
+	    {
+	      arc1 = lm->mcast_feature_arc_index;
+	      next1 = IP6_INPUT_NEXT_LOOKUP_MULTICAST;
+	    }
+	  else
+	    {
+	      arc1 = lm->ucast_feature_arc_index;
+	      next1 = IP6_INPUT_NEXT_LOOKUP;
+	    }
 
 	  vnet_buffer (p0)->ip.adj_index[VLIB_RX] = ~0;
 	  vnet_buffer (p1)->ip.adj_index[VLIB_RX] = ~0;
@@ -240,9 +256,17 @@ ip6_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
 	  ip0 = vlib_buffer_get_current (p0);
 
 	  sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
-	  arc0 =
-	    ip6_address_is_multicast (&ip0->dst_address) ?
-	    lm->mcast_feature_arc_index : lm->ucast_feature_arc_index;
+	  if (PREDICT_FALSE (ip6_address_is_multicast (&ip0->dst_address)))
+	    {
+	      arc0 = lm->mcast_feature_arc_index;
+	      next0 = IP6_INPUT_NEXT_LOOKUP_MULTICAST;
+	    }
+	  else
+	    {
+	      arc0 = lm->ucast_feature_arc_index;
+	      next0 = IP6_INPUT_NEXT_LOOKUP;
+	    }
+
 	  vnet_buffer (p0)->ip.adj_index[VLIB_RX] = ~0;
 	  vnet_feature_arc_start (arc0, sw_if_index0, &next0, p0);
 
@@ -313,6 +337,7 @@ VLIB_REGISTER_NODE (ip6_input_node) = {
     [IP6_INPUT_NEXT_DROP] = "error-drop",
     [IP6_INPUT_NEXT_LOOKUP] = "ip6-lookup",
     [IP6_INPUT_NEXT_ICMP_ERROR] = "ip6-icmp-error",
+    [IP6_INPUT_NEXT_LOOKUP_MULTICAST] = "ip6-mfib-forward-lookup",
   },
 
   .format_buffer = format_ip6_header,
diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c
index 46c0e316..46d04769 100644
--- a/src/vnet/ip/ip6_neighbor.c
+++ b/src/vnet/ip/ip6_neighbor.c
@@ -21,6 +21,7 @@
 #include <vppinfra/mhash.h>
 #include <vppinfra/md5.h>
 #include <vnet/adj/adj.h>
+#include <vnet/adj/adj_mcast.h>
 #include <vnet/fib/fib_table.h>
 #include <vnet/fib/ip6_fib.h>
 
@@ -116,9 +117,7 @@ typedef struct
   u32 seed;
   u64 randomizer;
   int ref_count;
-  adj_index_t all_nodes_adj_index;
-  adj_index_t all_routers_adj_index;
-  adj_index_t all_mldv2_routers_adj_index;
+  adj_index_t mcast_adj_index;
 
   /* timing information */
 #define DEF_MAX_RADV_INTERVAL 200
@@ -474,33 +473,72 @@ ip6_ethernet_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai)
 
   nbr = ip6_nd_find (sw_if_index, &adj->sub_type.nbr.next_hop.ip6);
 
-  if (NULL != nbr)
-    {
-      adj_nbr_walk_nh6 (sw_if_index, &nbr->key.ip6_address,
-			ip6_nd_mk_complete_walk, nbr);
-    }
-  else
+  switch (adj->lookup_next_index)
     {
+    case IP_LOOKUP_NEXT_ARP:
+    case IP_LOOKUP_NEXT_GLEAN:
+      if (NULL != nbr)
+	{
+	  adj_nbr_walk_nh6 (sw_if_index, &nbr->key.ip6_address,
+			    ip6_nd_mk_complete_walk, nbr);
+	}
+      else
+	{
+	  /*
+	   * no matching ND entry.
+	   * construct the rewrite required to for an ND packet, and stick
+	   * that in the adj's pipe to smoke.
+	   */
+	  adj_nbr_update_rewrite (ai,
+				  ADJ_NBR_REWRITE_FLAG_INCOMPLETE,
+				  ethernet_build_rewrite (vnm,
+							  sw_if_index,
+							  VNET_LINK_IP6,
+							  VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST));
+
+	  /*
+	   * since the FIB has added this adj for a route, it makes sense it may
+	   * want to forward traffic sometime soon. Let's send a speculative ND.
+	   * just one. If we were to do periodically that wouldn't be bad either,
+	   * but that's more code than i'm prepared to write at this time for
+	   * relatively little reward.
+	   */
+	  ip6_nbr_probe (adj);
+	}
+      break;
+    case IP_LOOKUP_NEXT_MCAST:
       /*
-       * no matching ND entry.
-       * construct the rewrite required to for an ND packet, and stick
-       * that in the adj's pipe to smoke.
+       * Construct a partial rewrite from the known ethernet mcast dest MAC
        */
-      adj_nbr_update_rewrite (ai,
-			      ADJ_NBR_REWRITE_FLAG_INCOMPLETE,
-			      ethernet_build_rewrite (vnm,
-						      sw_if_index,
-						      VNET_LINK_IP6,
-						      VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST));
+      adj_mcast_update_rewrite
+	(ai,
+	 ethernet_build_rewrite (vnm,
+				 sw_if_index,
+				 adj->ia_link,
+				 ethernet_ip6_mcast_dst_addr ()));
 
       /*
-       * since the FIB has added this adj for a route, it makes sense it may
-       * want to forward traffic sometime soon. Let's send a speculative ND.
-       * just one. If we were to do periodically that wouldn't be bad either,
-       * but that's more code than i'm prepared to write at this time for
-       * relatively little reward.
+       * Complete the remaining fields of the adj's rewrite to direct the
+       * complete of the rewrite at switch time by copying in the IP
+       * dst address's bytes.
+       * Ofset is 12 bytes from the end of the MAC header - which is 2
+       * bytes into the desintation address. And we write 4 bytes.
        */
-      ip6_nbr_probe (adj);
+      adj->rewrite_header.dst_mcast_offset = 12;
+      adj->rewrite_header.dst_mcast_n_bytes = 4;
+
+      break;
+
+    case IP_LOOKUP_NEXT_DROP:
+    case IP_LOOKUP_NEXT_PUNT:
+    case IP_LOOKUP_NEXT_LOCAL:
+    case IP_LOOKUP_NEXT_REWRITE:
+    case IP_LOOKUP_NEXT_LOAD_BALANCE:
+    case IP_LOOKUP_NEXT_MIDCHAIN:
+    case IP_LOOKUP_NEXT_ICMP_ERROR:
+    case IP_LOOKUP_N_NEXT:
+      ASSERT (0);
+      break;
     }
 }
 
@@ -1517,7 +1555,7 @@ icmp6_router_solicitation (vlib_main_t * vm,
 			}
 		      else
 			{
-			  adj_index0 = radv_info->all_nodes_adj_index;
+			  adj_index0 = radv_info->mcast_adj_index;
 			  if (adj_index0 == 0)
 			    error0 = ICMP6_ERROR_DST_LOOKUP_MISS;
 			  else
@@ -1918,10 +1956,8 @@ ip6_neighbor_sw_interface_add_del (vnet_main_t * vnm,
 	  ip6_radv_prefix_t *p;
 	  ip6_mldp_group_t *m;
 
-	  /* remove adjacencies */
-	  adj_unlock (a->all_nodes_adj_index);
-	  adj_unlock (a->all_routers_adj_index);
-	  adj_unlock (a->all_mldv2_routers_adj_index);
+	  /* release the lock on the interface's mcast adj */
+	  adj_unlock (a->mcast_adj_index);
 
 	  /* clean up prefix_pool */
 	  /* *INDENT-OFF* */
@@ -2017,36 +2053,9 @@ ip6_neighbor_sw_interface_add_del (vnet_main_t * vnm,
 	  mhash_init (&a->address_to_mldp_index, sizeof (uword),
 		      sizeof (ip6_address_t));
 
-	  {
-	    u8 link_layer_address[6] = { 0x33, 0x33, 0x00, 0x00, 0x00,
-	      IP6_MULTICAST_GROUP_ID_all_hosts
-	    };
-
-	    a->all_nodes_adj_index =
-	      adj_rewrite_add_and_lock (FIB_PROTOCOL_IP6, VNET_LINK_IP6,
-					sw_if_index, link_layer_address);
-	  }
-
-	  {
-	    u8 link_layer_address[6] = { 0x33, 0x33, 0x00, 0x00, 0x00,
-	      IP6_MULTICAST_GROUP_ID_all_routers
-	    };
-
-	    a->all_routers_adj_index =
-	      adj_rewrite_add_and_lock (FIB_PROTOCOL_IP6, VNET_LINK_IP6,
-					sw_if_index, link_layer_address);
-	  }
-
-	  {
-	    u8 link_layer_address[6] = { 0x33, 0x33, 0x00, 0x00, 0x00,
-	      IP6_MULTICAST_GROUP_ID_mldv2_routers
-	    };
-
-	    a->all_mldv2_routers_adj_index =
-	      adj_rewrite_add_and_lock (FIB_PROTOCOL_IP6,
-					VNET_LINK_IP6,
-					sw_if_index, link_layer_address);
-	  }
+	  a->mcast_adj_index = adj_mcast_add_or_lock (FIB_PROTOCOL_IP6,
+						      VNET_LINK_IP6,
+						      sw_if_index);
 
 	  /* add multicast groups we will always be reporting  */
 	  ip6_address_t addr;
@@ -2273,11 +2282,10 @@ ip6_neighbor_send_mldpv2_report (u32 sw_if_index)
   vnet_buffer (b0)->sw_if_index[VLIB_RX] =
     vnet_main.local_interface_sw_if_index;
 
-  vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
-    radv_info->all_mldv2_routers_adj_index;
+  vnet_buffer (b0)->ip.adj_index[VLIB_TX] = radv_info->mcast_adj_index;
   b0->flags |= VNET_BUFFER_LOCALLY_ORIGINATED;
 
-  vlib_node_t *node = vlib_get_node_by_name (vm, (u8 *) "ip6-rewrite");
+  vlib_node_t *node = vlib_get_node_by_name (vm, (u8 *) "ip6-rewrite-mcast");
 
   f = vlib_get_frame_to_node (vm, node->index);
   to_next = vlib_frame_vector_args (f);
@@ -2301,7 +2309,7 @@ VLIB_REGISTER_NODE (ip6_icmp_router_solicitation_node,static) =
   .n_next_nodes = ICMP6_ROUTER_SOLICITATION_N_NEXT,
   .next_nodes = {
     [ICMP6_ROUTER_SOLICITATION_NEXT_DROP] = "error-drop",
-    [ICMP6_ROUTER_SOLICITATION_NEXT_REPLY_RW] = "ip6-rewrite",
+    [ICMP6_ROUTER_SOLICITATION_NEXT_REPLY_RW] = "ip6-rewrite-mcast",
     [ICMP6_ROUTER_SOLICITATION_NEXT_REPLY_TX] = "interface-output",
   },
 };
diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c
index aafde464..437d2674 100644
--- a/src/vnet/ip/ip_api.c
+++ b/src/vnet/ip/ip_api.c
@@ -33,6 +33,9 @@
 #include <vnet/dpo/classify_dpo.h>
 #include <vnet/dpo/ip_null_dpo.h>
 #include <vnet/ethernet/arp_packet.h>
+//#include <vnet/mfib/ip6_mfib.h>
+#include <vnet/mfib/ip4_mfib.h>
+#include <vnet/mfib/mfib_signal.h>
 
 #include <vnet/vnet_msg_enum.h>
 
@@ -58,6 +61,8 @@ _(IP_FIB_DETAILS, ip_fib_details)                                       \
 _(IP6_FIB_DUMP, ip6_fib_dump)                                           \
 _(IP6_FIB_DETAILS, ip6_fib_details)                                     \
 _(IP_NEIGHBOR_DUMP, ip_neighbor_dump)                                   \
+_(IP_MROUTE_ADD_DEL, ip_mroute_add_del)                                 \
+_(MFIB_SIGNAL_DUMP, mfib_signal_dump)                                    \
 _(IP_NEIGHBOR_DETAILS, ip_neighbor_details)                             \
 _(IP_ADDRESS_DUMP, ip_address_dump)                                     \
 _(IP_DUMP, ip_dump)                                                     \
@@ -845,6 +850,144 @@ vl_api_ip_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
   REPLY_MACRO (VL_API_IP_ADD_DEL_ROUTE_REPLY);
 }
 
+static int
+add_del_mroute_check (fib_protocol_t table_proto,
+		      u32 table_id,
+		      u32 next_hop_sw_if_index,
+		      u8 is_local, u8 create_missing_tables, u32 * fib_index)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+
+  *fib_index = mfib_table_find (table_proto, ntohl (table_id));
+  if (~0 == *fib_index)
+    {
+      if (create_missing_tables)
+	{
+	  *fib_index = mfib_table_find_or_create_and_lock (table_proto,
+							   ntohl (table_id));
+	}
+      else
+	{
+	  /* No such VRF, and we weren't asked to create one */
+	  return VNET_API_ERROR_NO_SUCH_FIB;
+	}
+    }
+
+  if (~0 != ntohl (next_hop_sw_if_index))
+    {
+      if (pool_is_free_index (vnm->interface_main.sw_interfaces,
+			      ntohl (next_hop_sw_if_index)))
+	{
+	  return VNET_API_ERROR_NO_MATCHING_INTERFACE;
+	}
+    }
+
+  return (0);
+}
+
+static int
+mroute_add_del_handler (u8 is_add,
+			u8 is_local,
+			u32 fib_index,
+			const mfib_prefix_t * prefix,
+			u32 entry_flags,
+			u32 next_hop_sw_if_index, u32 itf_flags)
+{
+  stats_dslock_with_hint (1 /* release hint */ , 2 /* tag */ );
+
+  fib_route_path_t path = {
+    .frp_sw_if_index = next_hop_sw_if_index,
+    .frp_proto = prefix->fp_proto,
+  };
+
+  if (is_local)
+    path.frp_flags |= FIB_ROUTE_PATH_LOCAL;
+
+
+  if (!is_local && ~0 == next_hop_sw_if_index)
+    {
+      mfib_table_entry_update (fib_index, prefix,
+			       MFIB_SOURCE_API, entry_flags);
+    }
+  else
+    {
+      if (is_add)
+	{
+	  mfib_table_entry_path_update (fib_index, prefix,
+					MFIB_SOURCE_API, &path, itf_flags);
+	}
+      else
+	{
+	  mfib_table_entry_path_remove (fib_index, prefix,
+					MFIB_SOURCE_API, &path);
+	}
+    }
+
+  stats_dsunlock ();
+  return (0);
+}
+
+static int
+api_mroute_add_del_t_handler (vl_api_ip_mroute_add_del_t * mp)
+{
+  fib_protocol_t fproto;
+  u32 fib_index;
+  int rv;
+
+  fproto = (mp->is_ipv6 ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4);
+  rv = add_del_mroute_check (fproto,
+			     mp->table_id,
+			     mp->next_hop_sw_if_index,
+			     mp->is_local,
+			     mp->create_vrf_if_needed, &fib_index);
+
+  if (0 != rv)
+    return (rv);
+
+  mfib_prefix_t pfx = {
+    .fp_len = ntohs (mp->grp_address_length),
+    .fp_proto = fproto,
+  };
+
+  if (FIB_PROTOCOL_IP4 == fproto)
+    {
+      clib_memcpy (&pfx.fp_grp_addr.ip4, mp->grp_address,
+		   sizeof (pfx.fp_grp_addr.ip4));
+      clib_memcpy (&pfx.fp_src_addr.ip4, mp->src_address,
+		   sizeof (pfx.fp_src_addr.ip4));
+    }
+  else
+    {
+      clib_memcpy (&pfx.fp_grp_addr.ip6, mp->grp_address,
+		   sizeof (pfx.fp_grp_addr.ip6));
+      clib_memcpy (&pfx.fp_src_addr.ip6, mp->src_address,
+		   sizeof (pfx.fp_src_addr.ip6));
+    }
+
+  return (mroute_add_del_handler (mp->is_add,
+				  mp->is_local,
+				  fib_index, &pfx,
+				  ntohl (mp->entry_flags),
+				  ntohl (mp->next_hop_sw_if_index),
+				  ntohl (mp->itf_flags)));
+}
+
+void
+vl_api_ip_mroute_add_del_t_handler (vl_api_ip_mroute_add_del_t * mp)
+{
+  vl_api_ip_mroute_add_del_reply_t *rmp;
+  int rv;
+  vnet_main_t *vnm = vnet_get_main ();
+
+  vnm->api_errno = 0;
+
+  rv = api_mroute_add_del_t_handler (mp);
+
+  rv = (rv == 0) ? vnm->api_errno : rv;
+
+  REPLY_MACRO (VL_API_IP_MROUTE_ADD_DEL_REPLY);
+}
+
 static void
 send_ip_details (vpe_api_main_t * am,
 		 unix_shared_memory_queue_t * q, u32 sw_if_index, u32 context)
@@ -1148,6 +1291,73 @@ static void
   REPLY_MACRO (VL_API_SW_INTERFACE_IP6_SET_LINK_LOCAL_ADDRESS_REPLY);
 }
 
+void
+vl_mfib_signal_send_one (unix_shared_memory_queue_t * q,
+			 u32 context, const mfib_signal_t * mfs)
+{
+  vl_api_mfib_signal_details_t *mp;
+  mfib_prefix_t prefix;
+  mfib_table_t *mfib;
+  mfib_itf_t *mfi;
+
+  mp = vl_msg_api_alloc (sizeof (*mp));
+
+  memset (mp, 0, sizeof (*mp));
+  mp->_vl_msg_id = ntohs (VL_API_MFIB_SIGNAL_DETAILS);
+  mp->context = context;
+
+  mfi = mfib_itf_get (mfs->mfs_itf);
+  mfib_entry_get_prefix (mfs->mfs_entry, &prefix);
+  mfib = mfib_table_get (mfib_entry_get_fib_index (mfs->mfs_entry),
+			 prefix.fp_proto);
+  mp->table_id = ntohl (mfib->mft_table_id);
+  mp->sw_if_index = ntohl (mfi->mfi_sw_if_index);
+
+  if (FIB_PROTOCOL_IP4 == prefix.fp_proto)
+    {
+      mp->grp_address_len = ntohs (prefix.fp_len);
+
+      memcpy (mp->grp_address, &prefix.fp_grp_addr.ip4, 4);
+      if (prefix.fp_len > 32)
+	{
+	  memcpy (mp->src_address, &prefix.fp_src_addr.ip4, 4);
+	}
+    }
+  else
+    {
+      mp->grp_address_len = ntohs (prefix.fp_len);
+
+      ASSERT (0);
+    }
+
+  if (0 != mfs->mfs_buffer_len)
+    {
+      mp->ip_packet_len = ntohs (mfs->mfs_buffer_len);
+
+      memcpy (mp->ip_packet_data, mfs->mfs_buffer, mfs->mfs_buffer_len);
+    }
+  else
+    {
+      mp->ip_packet_len = 0;
+    }
+
+  vl_msg_api_send_shmem (q, (u8 *) & mp);
+}
+
+static void
+vl_api_mfib_signal_dump_t_handler (vl_api_mfib_signal_dump_t * mp)
+{
+  unix_shared_memory_queue_t *q;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    {
+      return;
+    }
+
+  while (q->cursize < q->maxsize && mfib_signal_send_one (q, mp->context))
+    ;
+}
 
 #define vl_msg_name_crc_list
 #include <vnet/ip/ip.api.h>
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 734a4cd7..6c5611d3 100644
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -43,6 +43,7 @@
 #include <vnet/fib/ip4_fib.h>
 #include <vnet/fib/ip6_fib.h>
 #include <vnet/mpls/mpls.h>
+#include <vnet/mfib/mfib_table.h>
 #include <vnet/dpo/drop_dpo.h>
 #include <vnet/dpo/classify_dpo.h>
 #include <vnet/dpo/punt_dpo.h>
@@ -258,6 +259,9 @@ format_ip_lookup_next (u8 * s, va_list * args)
     case IP_LOOKUP_NEXT_GLEAN:
       t = "glean";
       break;
+    case IP_LOOKUP_NEXT_MCAST:
+      t = "mcast";
+      break;
     case IP_LOOKUP_NEXT_REWRITE:
       break;
     }
@@ -767,6 +771,173 @@ VLIB_CLI_COMMAND (ip_route_command, static) = {
 };
 /* *INDENT-ON* */
 
+clib_error_t *
+vnet_ip_mroute_cmd (vlib_main_t * vm,
+		    unformat_input_t * main_input, vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = NULL;
+  fib_route_path_t rpath;
+  u32 table_id, is_del;
+  vnet_main_t *vnm;
+  mfib_prefix_t pfx;
+  u32 fib_index;
+  mfib_itf_flags_t iflags = 0;
+  mfib_entry_flags_t eflags = 0;
+
+  vnm = vnet_get_main ();
+  is_del = 0;
+  table_id = 0;
+  memset (&pfx, 0, sizeof (pfx));
+  memset (&rpath, 0, sizeof (rpath));
+  rpath.frp_sw_if_index = ~0;
+
+  /* Get a line of input. */
+  if (!unformat_user (main_input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "table %d", &table_id))
+	;
+      else if (unformat (line_input, "del"))
+	is_del = 1;
+      else if (unformat (line_input, "add"))
+	is_del = 0;
+      else if (unformat (line_input, "%U %U",
+			 unformat_ip4_address,
+			 &pfx.fp_src_addr.ip4,
+			 unformat_ip4_address, &pfx.fp_grp_addr.ip4))
+	{
+	  pfx.fp_proto = FIB_PROTOCOL_IP4;
+	  pfx.fp_len = 64;
+	}
+      else if (unformat (line_input, "%U %U",
+			 unformat_ip6_address,
+			 &pfx.fp_src_addr.ip6,
+			 unformat_ip6_address, &pfx.fp_grp_addr.ip6))
+	{
+	  pfx.fp_proto = FIB_PROTOCOL_IP6;
+	  pfx.fp_len = 256;
+	}
+      else if (unformat (line_input, "%U/%d",
+			 unformat_ip4_address,
+			 &pfx.fp_grp_addr.ip4, &pfx.fp_len))
+	{
+	  pfx.fp_proto = FIB_PROTOCOL_IP4;
+	}
+      else if (unformat (line_input, "%U/%d",
+			 unformat_ip6_address,
+			 &pfx.fp_grp_addr.ip6, &pfx.fp_len))
+	{
+	  pfx.fp_proto = FIB_PROTOCOL_IP6;
+	}
+      else if (unformat (line_input, "%U",
+			 unformat_ip4_address, &pfx.fp_grp_addr.ip4))
+	{
+	  memset (&pfx.fp_src_addr.ip4, 0, sizeof (pfx.fp_src_addr.ip4));
+	  pfx.fp_proto = FIB_PROTOCOL_IP4;
+	  pfx.fp_len = 32;
+	}
+      else if (unformat (line_input, "%U",
+			 unformat_ip6_address, &pfx.fp_grp_addr.ip6))
+	{
+	  memset (&pfx.fp_src_addr.ip6, 0, sizeof (pfx.fp_src_addr.ip6));
+	  pfx.fp_proto = FIB_PROTOCOL_IP6;
+	  pfx.fp_len = 128;
+	}
+      else if (unformat (line_input, "via %U",
+			 unformat_vnet_sw_interface, vnm,
+			 &rpath.frp_sw_if_index))
+	{
+	  rpath.frp_weight = 1;
+	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	}
+      else if (unformat (line_input, "%U", unformat_mfib_itf_flags, &iflags))
+	;
+      else if (unformat (line_input, "%U",
+			 unformat_mfib_entry_flags, &eflags))
+	;
+      else
+	{
+	  error = unformat_parse_error (line_input);
+	  goto done;
+	}
+    }
+
+  unformat_free (line_input);
+
+  if (~0 == table_id)
+    {
+      /*
+       * if no table_id is passed we will manipulate the default
+       */
+      fib_index = 0;
+    }
+  else
+    {
+      fib_index = mfib_table_find (pfx.fp_proto, table_id);
+
+      if (~0 == fib_index)
+	{
+	  error = clib_error_return (0, "Nonexistent table id %d", table_id);
+	  goto done;
+	}
+    }
+
+  if (is_del && 0 == rpath.frp_weight)
+    {
+      mfib_table_entry_delete (fib_index, &pfx, MFIB_SOURCE_CLI);
+    }
+  else if (eflags)
+    {
+      mfib_table_entry_update (fib_index, &pfx, MFIB_SOURCE_CLI, eflags);
+    }
+  else
+    {
+      if (is_del)
+	mfib_table_entry_path_remove (fib_index,
+				      &pfx, MFIB_SOURCE_CLI, &rpath);
+      else
+	mfib_table_entry_path_update (fib_index,
+				      &pfx, MFIB_SOURCE_CLI, &rpath, iflags);
+    }
+
+done:
+  return error;
+}
+
+/*?
+ * This command is used to add or delete IPv4 or IPv6  multicastroutes. All
+ * IP Addresses ('<em><dst-ip-addr>/<width></em>',
+ * '<em><next-hop-ip-addr></em>' and '<em><adj-hop-ip-addr></em>')
+ * can be IPv4 or IPv6, but all must be of the same form in a single
+ * command. To display the current set of routes, use the commands
+ * '<em>show ip mfib</em>' and '<em>show ip6 mfib</em>'.
+ * The full set of support flags for interfaces and route is shown via;
+ * '<em>show mfib route flags</em>' and '<em>show mfib itf flags</em>'
+ * respectively.
+ * @cliexpar
+ * Example of how to add a forwarding interface to a route (and create the
+ * route if it does not exist)
+ * @cliexcmd{ip mroute add 232.1.1.1 via GigabitEthernet2/0/0 Forward}
+ * Example of how to add an accepting interface to a route (and create the
+ * route if it does not exist)
+ * @cliexcmd{ip mroute add 232.1.1.1 via GigabitEthernet2/0/1 Accept}
+ * Example of changing the route's flags to send signals via the API
+ * @cliexcmd{ip mroute add 232.1.1.1 Signal}
+
+ ?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip_mroute_command, static) =
+{
+  .path = "ip mroute",
+  .short_help = "ip mroute [add|del] <dst-ip-addr>/<width> [table <table-id>] [via <next-hop-ip-addr> [<interface>],",
+  .function = vnet_ip_mroute_cmd,
+  .is_mp_safe = 1,
+};
+/* *INDENT-ON* */
+
 /*
  * The next two routines address a longstanding script hemorrhoid.
  * Probing a v4 or v6 neighbor needs to appear to be synchronous,
diff --git a/src/vnet/ip/lookup.h b/src/vnet/ip/lookup.h
index 3dbd7b3b..27c70943 100644
--- a/src/vnet/ip/lookup.h
+++ b/src/vnet/ip/lookup.h
@@ -91,6 +91,9 @@ typedef enum
   /** This packets needs to go to ICMP error */
   IP_LOOKUP_NEXT_ICMP_ERROR,
 
+  /** Multicast Adjacency. */
+  IP_LOOKUP_NEXT_MCAST,
+
   IP_LOOKUP_N_NEXT,
 } ip_lookup_next_t;
 
@@ -115,6 +118,7 @@ typedef enum
     [IP_LOOKUP_NEXT_ARP] = "ip4-arp",				\
     [IP_LOOKUP_NEXT_GLEAN] = "ip4-glean",			\
     [IP_LOOKUP_NEXT_REWRITE] = "ip4-rewrite",    		\
+    [IP_LOOKUP_NEXT_MCAST] = "ip4-rewrite-mcast",	        \
     [IP_LOOKUP_NEXT_MIDCHAIN] = "ip4-midchain",		        \
     [IP_LOOKUP_NEXT_LOAD_BALANCE] = "ip4-load-balance",		\
     [IP_LOOKUP_NEXT_ICMP_ERROR] = "ip4-icmp-error",		\
@@ -127,6 +131,7 @@ typedef enum
     [IP_LOOKUP_NEXT_ARP] = "ip6-discover-neighbor",		\
     [IP_LOOKUP_NEXT_GLEAN] = "ip6-glean",			\
     [IP_LOOKUP_NEXT_REWRITE] = "ip6-rewrite",			\
+    [IP_LOOKUP_NEXT_MCAST] = "ip6-rewrite-mcast",		\
     [IP_LOOKUP_NEXT_MIDCHAIN] = "ip6-midchain",			\
     [IP_LOOKUP_NEXT_LOAD_BALANCE] = "ip6-load-balance",		\
     [IP_LOOKUP_NEXT_ICMP_ERROR] = "ip6-icmp-error",		\
@@ -203,12 +208,6 @@ typedef struct ip_adjacency_t_
   /** Interface address index for this local/arp adjacency. */
   u32 if_address_index;
 
-  /** Force re-lookup in a different FIB. ~0 => normal behavior */
-  u16 mcast_group_index;
-
-  /** Highest possible perf subgraph arc interposition, e.g. for ip6 ioam */
-  u16 saved_lookup_next_index;
-
   /*
    * link/ether-type
    */
@@ -236,28 +235,28 @@ typedef struct ip_adjacency_t_
        */
     struct
     {
-	  /**
-	   * The recursive next-hop
-	   */
+      /**
+       * The recursive next-hop
+       */
       ip46_address_t next_hop;
-	  /**
-	   * The node index of the tunnel's post rewrite/TX function.
-	   */
+      /**
+       * The node index of the tunnel's post rewrite/TX function.
+       */
       u32 tx_function_node;
-	  /**
-	   * The next DPO to use
-	   */
+      /**
+       * The next DPO to use
+       */
       dpo_id_t next_dpo;
-	  /**
-	   * A function to perform the post-rewrite fixup
-	   */
-      adj_midchain_fixup_t fixup_func;
-    } midchain;
       /**
-       * IP_LOOKUP_NEXT_GLEAN
-       *
-       * Glean the address to ARP for from the packet's destination
+       * A function to perform the post-rewrite fixup
        */
+      adj_midchain_fixup_t fixup_func;
+    } midchain;
+    /**
+     * IP_LOOKUP_NEXT_GLEAN
+     *
+     * Glean the address to ARP for from the packet's destination
+     */
     struct
     {
       ip46_address_t receive_addr;
@@ -291,43 +290,6 @@ STATIC_ASSERT ((STRUCT_OFFSET_OF (ip_adjacency_t, cacheline1) ==
 /* An all zeros address */
 extern const ip46_address_t zero_addr;
 
-/* IP multicast adjacency. */
-typedef struct
-{
-  /* Handle for this adjacency in adjacency heap. */
-  u32 heap_handle;
-
-  /* Number of adjecencies in block. */
-  u32 n_adj;
-
-  /* Rewrite string. */
-    vnet_declare_rewrite (64 - 2 * sizeof (u32));
-}
-ip_multicast_rewrite_t;
-
-typedef struct
-{
-  /* ip4-multicast-rewrite next index. */
-  u32 next_index;
-
-  u8 n_rewrite_bytes;
-
-  u8 rewrite_string[64 - 1 * sizeof (u32) - 1 * sizeof (u8)];
-}
-ip_multicast_rewrite_string_t;
-
-typedef struct
-{
-  ip_multicast_rewrite_t *rewrite_heap;
-
-  ip_multicast_rewrite_string_t *rewrite_strings;
-
-  /* Negative rewrite string index; >= 0 sw_if_index.
-     Sorted.  Used to hash. */
-  i32 **adjacency_id_vector;
-
-  uword *adjacency_by_id_vector;
-} ip_multicast_lookup_main_t;
 
 typedef struct
 {
diff --git a/src/vnet/mcast/mcast.c b/src/vnet/mcast/mcast.c
deleted file mode 100644
index 55be89ae..00000000
--- a/src/vnet/mcast/mcast.c
+++ /dev/null
@@ -1,565 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include <vnet/mcast/mcast.h>
-
-#include <vlib/vlib.h>
-#include <vnet/vnet.h>
-#include <vnet/pg/pg.h>
-#include <vppinfra/error.h>
-#include <vnet/ip/ip4_packet.h>
-#include <vnet/ip/icmp46_packet.h>
-#include <vnet/ip/ip4.h>
-
-typedef struct {
-  u32 sw_if_index;
-  u32 next_index;
-  u32 group_index;
-} mcast_prep_trace_t;
-
-/* packet trace format function */
-static u8 * format_mcast_prep_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  mcast_prep_trace_t * t = va_arg (*args, mcast_prep_trace_t *);
-  
-  s = format (s, "MCAST_PREP: group %d, next index %d, tx_sw_if_index %d",
-              t->group_index, t->next_index, t->sw_if_index);
-  return s;
-}
-
-mcast_main_t mcast_main;
-vlib_node_registration_t mcast_prep_node;
-vlib_node_registration_t mcast_recycle_node;
-
-#define foreach_mcast_prep_error \
-_(MCASTS, "Multicast Packets")
-
-typedef enum {
-#define _(sym,str) MCAST_PREP_ERROR_##sym,
-  foreach_mcast_prep_error
-#undef _
-  MCAST_PREP_N_ERROR,
-} mcast_prep_error_t;
-
-static char * mcast_prep_error_strings[] = {
-#define _(sym,string) string,
-  foreach_mcast_prep_error
-#undef _
-};
-
-typedef enum {
-  MCAST_PREP_NEXT_DROP,
-  MCAST_PREP_N_NEXT,
-} mcast_prep_next_t;
-
-static uword
-mcast_prep_node_fn (vlib_main_t * vm,
-		  vlib_node_runtime_t * node,
-		  vlib_frame_t * frame)
-{
-  u32 n_left_from, * from, * to_next;
-  mcast_prep_next_t next_index;
-  mcast_main_t * mcm = &mcast_main;
-  vlib_node_t *n = vlib_get_node (vm, mcast_prep_node.index);
-  u32 node_counter_base_index = n->error_heap_index;
-  vlib_error_main_t * em = &vm->error_main;
-  ip4_main_t * im = &ip4_main;
-  ip_lookup_main_t * lm = &im->lookup_main;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
-
-      while (0 && n_left_from >= 4 && n_left_to_next >= 2)
-	{
-          u32 bi0, bi1;
-	  vlib_buffer_t * b0, * b1;
-          u32 next0, next1;
-          u32 sw_if_index0, sw_if_index1;
-          
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t * p2, * p3;
-            
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-            
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-
-	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
-	  }
-
-          /* speculatively enqueue b0 and b1 to the current next frame */
-	  to_next[0] = bi0 = from[0];
-	  to_next[1] = bi1 = from[1];
-	  from += 2;
-	  to_next += 2;
-	  n_left_from -= 2;
-	  n_left_to_next -= 2;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  b1 = vlib_get_buffer (vm, bi1);
-
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-          next0 = 0;
-          sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
-          next1 = 0;
-
-          /* $$$$ your message in this space. Process 2 x pkts */
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)))
-            {
-              if (b0->flags & VLIB_BUFFER_IS_TRACED) 
-                {
-                    mcast_prep_trace_t *t = 
-                      vlib_add_trace (vm, node, b0, sizeof (*t));
-                    t->sw_if_index = sw_if_index0;
-                    t->next_index = next0;
-                  }
-                if (b1->flags & VLIB_BUFFER_IS_TRACED) 
-                  {
-                    mcast_prep_trace_t *t = 
-                      vlib_add_trace (vm, node, b1, sizeof (*t));
-                    t->sw_if_index = sw_if_index1;
-                    t->next_index = next1;
-                  }
-              }
-            
-            /* verify speculative enqueues, maybe switch current next frame */
-            vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-                                             to_next, n_left_to_next,
-                                             bi0, bi1, next0, next1);
-        }
-      
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0, adj_index0;
-          mcast_group_t * g0;
-          ip_adjacency_t * adj0;
-          
-          /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-          adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
-          adj0 = ip_get_adjacency (lm, adj_index0);
-          vnet_buffer(b0)->mcast.mcast_group_index = adj0->mcast_group_index;
-          g0 = pool_elt_at_index (mcm->groups, adj0->mcast_group_index);
-
-          /* 
-           * Handle the degenerate single-copy case 
-           * If we don't change the freelist, the packet will never
-           * make it to the recycle node...
-           */
-          if (PREDICT_TRUE(vec_len (g0->members) > 1))
-            {
-              /* Save the original free list index */
-              vnet_buffer(b0)->mcast.original_free_list_index =
-                b0->free_list_index;
-              
-              /* Swap in the multicast recycle list */
-              b0->free_list_index = mcm->mcast_recycle_list_index;
-              
-              /* 
-               * Make sure that intermediate "frees" don't screw up 
-               */
-              b0->recycle_count = vec_len (g0->members);
-              b0->flags |= VLIB_BUFFER_RECYCLE;
-
-              /* Set up for the recycle node */
-              vnet_buffer(b0)->mcast.mcast_current_index = 1;
-            }
-
-          /* Transmit the pkt on the first interface */
-          next0 = g0->members[0].prep_and_recycle_node_next_index;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = 
-            g0->members[0].tx_sw_if_index;
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) 
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED))) {
-            mcast_prep_trace_t *t = 
-               vlib_add_trace (vm, node, b0, sizeof (*t));
-            t->next_index = next0;
-            t->sw_if_index = vnet_buffer(b0)->sw_if_index[VLIB_TX];
-            t->group_index = vnet_buffer(b0)->mcast.mcast_group_index;
-            }
-            
-          /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  em->counters[node_counter_base_index + MCAST_PREP_ERROR_MCASTS] += 
-      frame->n_vectors;
-
-  return frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (mcast_prep_node) = {
-  .function = mcast_prep_node_fn,
-  .name = "mcast_prep",
-  .vector_size = sizeof (u32),
-  .format_trace = format_mcast_prep_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  
-  .n_errors = ARRAY_LEN(mcast_prep_error_strings),
-  .error_strings = mcast_prep_error_strings,
-
-  .n_next_nodes = MCAST_PREP_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-        [MCAST_PREP_NEXT_DROP] = "error-drop",
-  },
-};
-
-typedef struct {
-  u32 sw_if_index;
-  u32 next_index;
-  u32 current_member;
-  u32 group_index;
-} mcast_recycle_trace_t;
-
-static u8 * format_mcast_recycle_trace (u8 * s, va_list * args)
-{
-  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
-  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
-  mcast_recycle_trace_t * t = va_arg (*args, mcast_recycle_trace_t *);
-  
-  s = format (s, 
-"MCAST_R: group %d, current member %d next (node) index %d, tx_sw_if_index %d",
-              t->group_index, t->current_member, t->next_index, t->sw_if_index);
-  return s;
-}
-
-#define foreach_mcast_recycle_error \
-_(RECYCLES, "Multicast Recycles")
-
-typedef enum {
-#define _(sym,str) MCAST_RECYCLE_ERROR_##sym,
-  foreach_mcast_recycle_error
-#undef _
-  MCAST_RECYCLE_N_ERROR,
-} mcast_recycle_error_t;
-
-static char * mcast_recycle_error_strings[] = {
-#define _(sym,string) string,
-  foreach_mcast_recycle_error
-#undef _
-};
-
-typedef enum {
-  MCAST_RECYCLE_NEXT_DROP,
-  MCAST_RECYCLE_N_NEXT,
-} mcast_recycle_next_t;
-
-static uword
-mcast_recycle_node_fn (vlib_main_t * vm,
-		  vlib_node_runtime_t * node,
-		  vlib_frame_t * frame)
-{
-  u32 n_left_from, * from, * to_next;
-  mcast_recycle_next_t next_index;
-  mcast_main_t * mcm = &mcast_main;
-  vlib_node_t *n = vlib_get_node (vm, mcast_recycle_node.index);
-  u32 node_counter_base_index = n->error_heap_index;
-  vlib_error_main_t * em = &vm->error_main;
-
-  from = vlib_frame_vector_args (frame);
-  n_left_from = frame->n_vectors;
-  next_index = node->cached_next_index;
-
-  while (n_left_from > 0)
-    {
-      u32 n_left_to_next;
-
-      vlib_get_next_frame (vm, node, next_index,
-			   to_next, n_left_to_next);
-
-      while (0 && n_left_from >= 4 && n_left_to_next >= 2)
-	{
-          u32 bi0, bi1;
-	  vlib_buffer_t * b0, * b1;
-          u32 next0, next1;
-          u32 sw_if_index0, sw_if_index1;
-          
-	  /* Prefetch next iteration. */
-	  {
-	    vlib_buffer_t * p2, * p3;
-            
-	    p2 = vlib_get_buffer (vm, from[2]);
-	    p3 = vlib_get_buffer (vm, from[3]);
-            
-	    vlib_prefetch_buffer_header (p2, LOAD);
-	    vlib_prefetch_buffer_header (p3, LOAD);
-
-	    CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
-	    CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE);
-	  }
-
-          /* speculatively enqueue b0 and b1 to the current next frame */
-	  to_next[0] = bi0 = from[0];
-	  to_next[1] = bi1 = from[1];
-	  from += 2;
-	  to_next += 2;
-	  n_left_from -= 2;
-	  n_left_to_next -= 2;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-	  b1 = vlib_get_buffer (vm, bi1);
-
-          sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
-          next0 = 0;
-          sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
-          next1 = 0;
-
-          /* $$$$ your message in this space. Process 2 x pkts */
-
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)))
-            {
-              if (b0->flags & VLIB_BUFFER_IS_TRACED) 
-                {
-                    mcast_recycle_trace_t *t = 
-                      vlib_add_trace (vm, node, b0, sizeof (*t));
-                    t->sw_if_index = sw_if_index0;
-                    t->next_index = next0;
-                  }
-                if (b1->flags & VLIB_BUFFER_IS_TRACED) 
-                  {
-                    mcast_recycle_trace_t *t = 
-                      vlib_add_trace (vm, node, b1, sizeof (*t));
-                    t->sw_if_index = sw_if_index1;
-                    t->next_index = next1;
-                  }
-              }
-            
-            /* verify speculative enqueues, maybe switch current next frame */
-            vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
-                                             to_next, n_left_to_next,
-                                             bi0, bi1, next0, next1);
-        }
-      
-      while (n_left_from > 0 && n_left_to_next > 0)
-	{
-          u32 bi0;
-	  vlib_buffer_t * b0;
-          u32 next0;
-          u32 current_member0;
-          mcast_group_t * g0;
-          
-          /* speculatively enqueue b0 to the current next frame */
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
-
-	  b0 = vlib_get_buffer (vm, bi0);
-
-          g0 = pool_elt_at_index (mcm->groups, 
-                                  vnet_buffer(b0)->mcast.mcast_group_index);
-
-          /* No more replicas? */
-          if (b0->recycle_count == 1)
-            {
-              /* Restore the original free list index */
-              b0->free_list_index = 
-                vnet_buffer(b0)->mcast.original_free_list_index;
-              b0->flags &= ~(VLIB_BUFFER_RECYCLE);
-            }
-          current_member0 = vnet_buffer(b0)->mcast.mcast_current_index;
-          
-          next0 = 
-            g0->members[current_member0].prep_and_recycle_node_next_index;
-          vnet_buffer(b0)->sw_if_index[VLIB_TX] = 
-            g0->members[current_member0].tx_sw_if_index;
-          
-          vnet_buffer(b0)->mcast.mcast_current_index = 
-            current_member0 + 1;
-          
-          if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE) 
-                            && (b0->flags & VLIB_BUFFER_IS_TRACED))) {
-            mcast_recycle_trace_t *t = 
-               vlib_add_trace (vm, node, b0, sizeof (*t));
-            t->next_index = next0;
-            t->sw_if_index = vnet_buffer(b0)->sw_if_index[VLIB_TX];
-            t->group_index = vnet_buffer(b0)->mcast.mcast_group_index;
-            t->current_member = current_member0;
-            }
-            
-          /* verify speculative enqueue, maybe switch current next frame */
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, next0);
-	}
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
-
-  em->counters[node_counter_base_index + MCAST_RECYCLE_ERROR_RECYCLES] += 
-      frame->n_vectors;
-
-  return frame->n_vectors;
-}
-
-VLIB_REGISTER_NODE (mcast_recycle_node) = {
-  .function = mcast_recycle_node_fn,
-  .name = "mcast-recycle",
-  .vector_size = sizeof (u32),
-  .format_trace = format_mcast_recycle_trace,
-  .type = VLIB_NODE_TYPE_INTERNAL,
-  
-  .n_errors = ARRAY_LEN(mcast_recycle_error_strings),
-  .error_strings = mcast_recycle_error_strings,
-
-  .n_next_nodes = MCAST_RECYCLE_N_NEXT,
-
-  /* edit / add dispositions here */
-  .next_nodes = {
-        [MCAST_RECYCLE_NEXT_DROP] = "error-drop",
-  },
-};
-
-/*
- * fish pkts back from the recycle queue/freelist
- * un-flatten the context chains
- */
-static void mcast_recycle_callback (vlib_main_t *vm, 
-                                    vlib_buffer_free_list_t * fl)
-{
-  vlib_frame_t * f = 0;
-  u32 n_left_from;
-  u32 n_left_to_next = 0;
-  u32 n_this_frame = 0;
-  u32 * from;
-  u32 * to_next;
-  u32 bi0, pi0;
-  vlib_buffer_t *b0;
-  vlib_buffer_t *bnext0;
-  int i;
-
-  /* aligned, unaligned buffers */
-  for (i = 0; i < 2; i++) 
-    {
-      if (i == 0)
-        {
-          from = fl->aligned_buffers;
-          n_left_from = vec_len (from);
-        }
-      else
-        {
-          from = fl->unaligned_buffers;
-          n_left_from = vec_len (from);
-        }
-    
-      while (n_left_from > 0)
-        {
-          if (PREDICT_FALSE(n_left_to_next == 0)) 
-            {
-              if (f)
-                {
-                  f->n_vectors = n_this_frame;
-                  vlib_put_frame_to_node (vm, mcast_recycle_node.index, f);
-                }
-              
-              f = vlib_get_frame_to_node (vm, mcast_recycle_node.index);
-              to_next = vlib_frame_vector_args (f);
-              n_left_to_next = VLIB_FRAME_SIZE;
-              n_this_frame = 0;
-            }
-          
-          bi0 = from[0];
-          if (PREDICT_TRUE(n_left_from > 1))
-            {
-              pi0 = from[1];
-              vlib_prefetch_buffer_with_index(vm,pi0,LOAD);
-            }
-        
-          bnext0 = b0 = vlib_get_buffer (vm, bi0);
-          
-          while (bnext0->flags & VLIB_BUFFER_NEXT_PRESENT)
-            {
-              from += 1;
-              n_left_from -= 1;
-              bnext0 = vlib_get_buffer (vm, bnext0->next_buffer);
-            }
-          to_next[0] = bi0;
-
-          if (CLIB_DEBUG > 0)
-            vlib_buffer_set_known_state (vm, bi0, VLIB_BUFFER_KNOWN_ALLOCATED);
-
-          from++;
-          to_next++;
-          n_this_frame++;
-          n_left_to_next--;
-          n_left_from--;
-        }
-    }
-  
-  vec_reset_length (fl->aligned_buffers);
-  vec_reset_length (fl->unaligned_buffers);
-
-  if (f)
-    {
-      ASSERT(n_this_frame);
-      f->n_vectors = n_this_frame;
-      vlib_put_frame_to_node (vm, mcast_recycle_node.index, f);
-    }
-}
-
-clib_error_t *mcast_init (vlib_main_t *vm)
-{
-  mcast_main_t * mcm = &mcast_main;
-  vlib_buffer_main_t * bm = vm->buffer_main;
-  vlib_buffer_free_list_t * fl;
-    
-  mcm->vlib_main = vm;
-  mcm->vnet_main = vnet_get_main();
-  mcm->mcast_recycle_list_index = 
-    vlib_buffer_create_free_list (vm, 1024 /* fictional */, "mcast-recycle");
-
-  fl = pool_elt_at_index (bm->buffer_free_list_pool, 
-                          mcm->mcast_recycle_list_index);
-
-  fl->buffers_added_to_freelist_function = mcast_recycle_callback;
-
-  return 0;
-}
-
-VLIB_INIT_FUNCTION (mcast_init);
-
-
diff --git a/src/vnet/mcast/mcast.h b/src/vnet/mcast/mcast.h
deleted file mode 100644
index 96e51442..00000000
--- a/src/vnet/mcast/mcast.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef __included_vnet_mcast_h__
-#define __included_vnet_mcast_h__
-
-#include <vnet/vnet.h>
-#include <vlib/buffer.h>
-#include <vlib/buffer_funcs.h>
-
-typedef struct {
-  /* Arrange for both prep and recycle nodes to have identical
-     next indices for a given output interface */
-  u32 prep_and_recycle_node_next_index;
-
-  /* Show command, etc. */
-  u32 tx_sw_if_index;
-} mcast_group_member_t;
-
-typedef struct {
-  /* vector of group members */
-  mcast_group_member_t * members;
-} mcast_group_t;
-
-typedef struct {
-  /* pool of multicast (interface) groups */
-  mcast_group_t * groups;
-
-  /* multicast "free" list, aka recycle list */
-  u32 mcast_recycle_list_index;
-
-  /* convenience */
-  vlib_main_t * vlib_main;
-  vnet_main_t * vnet_main;
-} mcast_main_t;
-
-mcast_main_t mcast_main;
-
-#endif /* __included_vnet_mcast_h__ */
diff --git a/src/vnet/mcast/mcast_test.c b/src/vnet/mcast/mcast_test.c
deleted file mode 100644
index be80c9fc..00000000
--- a/src/vnet/mcast/mcast_test.c
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include <vnet/mcast/mcast.h>
-
-#include <vlib/vlib.h>
-#include <vnet/vnet.h>
-#include <vnet/pg/pg.h>
-#include <vppinfra/error.h>
-#include <vnet/ip/lookup.h>
-#include <vnet/ip/ip4_packet.h>
-#include <vnet/ip/icmp46_packet.h>
-#include <vnet/ip/ip4.h>
-#include <vnet/mcast/mcast.h>
-
-typedef struct {
-  /* convenience */
-  vlib_main_t * vlib_main;
-  vnet_main_t * vnet_main;
-  mcast_main_t * mcast_main;
-} mcast_test_main_t;
-
-mcast_test_main_t mcast_test_main;
-vlib_node_registration_t mcast_prep_node;
-vlib_node_registration_t mcast_recycle_node;
-
-static clib_error_t *
-mcast_test_command_fn (vlib_main_t * vm,
-		 unformat_input_t * input,
-		 vlib_cli_command_t * cmd)
-{
-  /* u8 *rewrite_data; */
-  /* mcast_test_main_t * mtm = &mcast_test_main; */
-  /* mcast_main_t * mcm = mtm->mcast_main; */
-  /* ip_adjacency_t adj; */
-  /* u32 adj_index; */
-  /* mcast_group_t * g; */
-  /* mcast_group_member_t * member; */
-  /* unformat_input_t _line_input, * line_input = &_line_input; */
-  /* ip4_address_t dst_addr, zero; */
-  /* ip4_main_t * im = &ip4_main; */
-  /* ip_lookup_main_t * lm = &im->lookup_main; */
-
-  /* /\* Get a line of input. *\/ */
-  /* if (! unformat_user (input, unformat_line_input, line_input)) */
-  /*   return 0; */
-
-  /* pool_get (mcm->groups, g); */
-  /* memset (g, 0, sizeof (*g)); */
-
-  /* while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) */
-  /*   { */
-  /*     vnet_hw_interface_t *hw; */
-  /*     u32 next, sw_if_index; */
-
-  /*     if (unformat (line_input, "%U", unformat_vnet_sw_interface,  */
-  /*                   mtm->vnet_main, &sw_if_index))  */
-  /*       { */
-  /*         vec_add2 (g->members, member, 1); */
-  /*         member->tx_sw_if_index = sw_if_index; */
-          
-  /*         hw = vnet_get_sup_hw_interface (mtm->vnet_main,  */
-  /*                                         sw_if_index); */
-          
-  /*         next = vlib_node_add_next (mtm->vlib_main,  */
-  /*                                    mcast_prep_node.index, */
-  /*                                    hw->output_node_index); */
-          
-  /*         /\* Required to be the same next index... *\/ */
-  /*         vlib_node_add_next_with_slot (mtm->vlib_main, */
-  /*                                       mcast_recycle_node.index, */
-  /*                                       hw->output_node_index, next); */
-  /*         member->prep_and_recycle_node_next_index = next; */
-  /*       } */
-  /*     else */
-  /*       { */
-  /*         return unformat_parse_error (line_input); */
-  /*       } */
-  /*   } */
-
-  /* if (vec_len (g->members) == 0) */
-  /*   { */
-  /*     pool_put (mcm->groups, g); */
-  /*     vlib_cli_output (vm, "no group members specified"); */
-  /*     return 0; */
-  /*   } */
-
-
-  /* adj.lookup_next_index = IP_LOOKUP_NEXT_REWRITE; */
-  /* adj.mcast_group_index = g - mcm->groups; */
-  /* rewrite_data = format (0, "abcdefg"); */
-
-  /* vnet_rewrite_for_tunnel */
-  /*   (mtm->vnet_main, */
-  /*    (u32)~0, /\* tx_sw_if_index, we dont know yet *\/ */
-  /*    ip4_rewrite_node.index, */
-  /*    mcast_prep_node.index, */
-  /*    &adj.rewrite_header, */
-  /*    rewrite_data, vec_len(rewrite_data)); */
-
-  /* ip_add_adjacency (lm, &adj, 1 /\* one adj *\/, */
-  /*                   &adj_index); */
-  
-  /* dst_addr.as_u32 = clib_host_to_net_u32 (0x0a000002); */
-  /* zero.as_u32 = 0; */
-
-  /* ip4_add_del_route_next_hop (im, */
-  /*                             IP4_ROUTE_FLAG_ADD, */
-  /*                             &dst_addr, */
-  /*                             24 /\* mask width *\/, */
-  /*                             &zero /\* no next hop *\/, */
-                          
-  /*                             0, // next hop sw if index */
-  /*                             1, // weight */
-  /*                             adj_index, */
-  /*                             0 /\* explicit fib 0 *\/); */
-
-  return 0;
-}
-
-static VLIB_CLI_COMMAND (mcast_test_command) = {
-  .path = "test mc",
-  .short_help = "test mc",
-  .function = mcast_test_command_fn,
-};
-
-clib_error_t *mcast_test_init (vlib_main_t *vm)
-{
-  mcast_test_main_t * mtm = &mcast_test_main;
-    
-  mtm->vlib_main = vm;
-  mtm->vnet_main = vnet_get_main();
-  mtm->mcast_main = &mcast_main;
-
-  return 0;
-}
-
-VLIB_INIT_FUNCTION (mcast_test_init);
diff --git a/src/vnet/mfib/ip4_mfib.c b/src/vnet/mfib/ip4_mfib.c
new file mode 100644
index 00000000..08001c3f
--- /dev/null
+++ b/src/vnet/mfib/ip4_mfib.c
@@ -0,0 +1,465 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/mfib/ip4_mfib.h>
+
+#include <vnet/mfib/mfib_table.h>
+#include <vnet/mfib/mfib_entry.h>
+
+static const mfib_prefix_t ip4_specials[] = {
+    {
+        /* (*,*)/0 */
+        .fp_src_addr = {
+            .ip4.data_u32 = 0,
+        },
+        .fp_grp_addr = {
+            .ip4.data_u32 = 0,
+        },
+        .fp_len  = 0,
+        .fp_proto = FIB_PROTOCOL_IP4,
+    },
+};
+
+static u32
+ip4_create_mfib_with_table_id (u32 table_id)
+{
+    mfib_table_t *mfib_table;
+
+    pool_get_aligned(ip4_main.mfibs, mfib_table, CLIB_CACHE_LINE_BYTES);
+    memset(mfib_table, 0, sizeof(*mfib_table));
+
+    mfib_table->mft_proto = FIB_PROTOCOL_IP4;
+    mfib_table->mft_index =
+        mfib_table->v4.index =
+            (mfib_table - ip4_main.mfibs);
+
+    hash_set (ip4_main.mfib_index_by_table_id,
+              table_id,
+              mfib_table->mft_index);
+
+    mfib_table->mft_table_id =
+        mfib_table->v4.table_id =
+            table_id;
+
+    mfib_table_lock(mfib_table->mft_index, FIB_PROTOCOL_IP4);
+
+    /*
+     * add the special entries into the new FIB
+     */
+    int ii;
+
+    for (ii = 0; ii < ARRAY_LEN(ip4_specials); ii++)
+    {
+        mfib_prefix_t prefix = ip4_specials[ii];
+
+        prefix.fp_src_addr.ip4.data_u32 =
+            clib_host_to_net_u32(prefix.fp_src_addr.ip4.data_u32);
+        prefix.fp_grp_addr.ip4.data_u32 =
+            clib_host_to_net_u32(prefix.fp_grp_addr.ip4.data_u32);
+
+        mfib_table_entry_update(mfib_table->mft_index,
+                                &prefix,
+                                MFIB_SOURCE_DEFAULT_ROUTE,
+                                MFIB_ENTRY_FLAG_DROP);
+    }
+
+    return (mfib_table->mft_index);
+}
+
+void
+ip4_mfib_table_destroy (ip4_mfib_t *mfib)
+{
+    mfib_table_t *mfib_table = (mfib_table_t*)mfib;
+    int ii;
+
+    /*
+     * remove all the specials we added when the table was created.
+     */
+    for (ii = 0; ii < ARRAY_LEN(ip4_specials); ii++)
+    {
+        fib_node_index_t mfei;
+        mfib_prefix_t prefix = ip4_specials[ii];
+
+        prefix.fp_src_addr.ip4.data_u32 =
+            clib_host_to_net_u32(prefix.fp_src_addr.ip4.data_u32);
+        prefix.fp_grp_addr.ip4.data_u32 =
+            clib_host_to_net_u32(prefix.fp_grp_addr.ip4.data_u32);
+
+        mfei = mfib_table_lookup(mfib_table->mft_index, &prefix);
+        mfib_table_entry_delete_index(mfei, MFIB_SOURCE_DEFAULT_ROUTE);
+    }
+
+    /*
+     * validate no more routes.
+     */
+    ASSERT(0 == mfib_table->mft_total_route_counts);
+    ASSERT(~0 != mfib_table->mft_table_id);
+
+    hash_unset (ip4_main.mfib_index_by_table_id, mfib_table->mft_table_id);
+    pool_put(ip4_main.mfibs, mfib_table);
+}
+
+u32
+ip4_mfib_table_find_or_create_and_lock (u32 table_id)
+{
+    u32 index;
+
+    index = ip4_mfib_index_from_table_id(table_id);
+    if (~0 == index)
+        return ip4_create_mfib_with_table_id(table_id);
+    mfib_table_lock(index, FIB_PROTOCOL_IP4);
+
+    return (index);
+}
+
+u32
+ip4_mfib_table_get_index_for_sw_if_index (u32 sw_if_index)
+{
+    if (sw_if_index >= vec_len(ip4_main.mfib_index_by_sw_if_index))
+    {
+        /*
+         * This is the case for interfaces that are not yet mapped to
+         * a IP table
+         */
+        return (~0);
+    }
+    return (ip4_main.mfib_index_by_sw_if_index[sw_if_index]);
+}
+
+#define IPV4_MFIB_GRP_LEN(_len)\
+    (_len > 32 ? 32 : _len)
+
+#define IP4_MFIB_MK_KEY(_grp, _src, _len, _key)                         \
+{                                                                       \
+    _key  = ((u64)(_grp->data_u32 &                                     \
+                   ip4_main.fib_masks[IPV4_MFIB_GRP_LEN(_len)])) << 32; \
+    _key |= _src->data_u32;                                             \
+}
+#define IP4_MFIB_MK_GRP_KEY(_grp, _len, _key)                           \
+{                                                                       \
+    _key  = ((u64)(_grp->data_u32 &                                     \
+                   ip4_main.fib_masks[IPV4_MFIB_GRP_LEN(_len)])) << 32; \
+}
+
+/*
+ * ip4_fib_table_lookup_exact_match
+ *
+ * Exact match prefix lookup
+ */
+fib_node_index_t
+ip4_mfib_table_lookup_exact_match (const ip4_mfib_t *mfib,
+                                   const ip4_address_t *grp,
+                                   const ip4_address_t *src,
+                                   u32 len)
+{
+    uword * hash, * result;
+    u64 key;
+
+    hash = mfib->fib_entry_by_dst_address[len];
+    IP4_MFIB_MK_KEY(grp, src, len, key);
+
+    result = hash_get(hash, key);
+
+    if (NULL != result) {
+        return (result[0]);
+    }
+    return (FIB_NODE_INDEX_INVALID);
+}
+
+/*
+ * ip4_fib_table_lookup
+ *
+ * Longest prefix match
+ */
+fib_node_index_t
+ip4_mfib_table_lookup (const ip4_mfib_t *mfib,
+                       const ip4_address_t *src,
+                       const ip4_address_t *grp,
+                       u32 len)
+{
+    uword * hash, * result;
+    i32 mask_len;
+    u64 key;
+
+    mask_len = len;
+
+    if (PREDICT_TRUE(64 == mask_len))
+    {
+        hash = mfib->fib_entry_by_dst_address[mask_len];
+        IP4_MFIB_MK_KEY(grp, src, mask_len, key);
+
+        result = hash_get (hash, key);
+
+        if (NULL != result) {
+            return (result[0]);
+        }
+    }
+
+    for (mask_len = 32; mask_len >= 0; mask_len--)
+    {
+        hash = mfib->fib_entry_by_dst_address[mask_len];
+        IP4_MFIB_MK_GRP_KEY(grp, mask_len, key);
+
+        result = hash_get (hash, key);
+
+        if (NULL != result) {
+            return (result[0]);
+        }
+    }
+    return (FIB_NODE_INDEX_INVALID);
+}
+
+void
+ip4_mfib_table_entry_insert (ip4_mfib_t *mfib,
+                             const ip4_address_t *grp,
+                             const ip4_address_t *src,
+                             u32 len,
+                             fib_node_index_t fib_entry_index)
+{
+    uword * hash, * result;
+    u64 key;
+
+    IP4_MFIB_MK_KEY(grp, src, len, key);
+    hash = mfib->fib_entry_by_dst_address[len];
+    result = hash_get (hash, key);
+
+    if (NULL == result) {
+        /*
+         * adding a new entry
+         */
+        if (NULL == hash) {
+            hash = hash_create (32 /* elts */, sizeof (uword));
+            hash_set_flags (hash, HASH_FLAG_NO_AUTO_SHRINK);
+        }
+        hash = hash_set(hash, key, fib_entry_index);
+        mfib->fib_entry_by_dst_address[len] = hash;
+    }
+    else
+    {
+        ASSERT(0);
+    }
+}
+
+void
+ip4_mfib_table_entry_remove (ip4_mfib_t *mfib,
+                             const ip4_address_t *grp,
+                             const ip4_address_t *src,
+                             u32 len)
+{
+    uword * hash, * result;
+    u64 key;
+
+    IP4_MFIB_MK_KEY(grp, src, len, key);
+    hash = mfib->fib_entry_by_dst_address[len];
+    result = hash_get (hash, key);
+
+    if (NULL == result)
+    {
+        /*
+         * removing a non-existant entry. i'll allow it.
+         */
+    }
+    else
+    {
+        hash_unset(hash, key);
+    }
+
+    mfib->fib_entry_by_dst_address[len] = hash;
+}
+
+static void
+ip4_mfib_table_show_all (ip4_mfib_t *mfib,
+                         vlib_main_t * vm)
+{
+    fib_node_index_t *mfib_entry_indicies;
+    fib_node_index_t *mfib_entry_index;
+    int i;
+
+    mfib_entry_indicies = NULL;
+
+    for (i = 0; i < ARRAY_LEN (mfib->fib_entry_by_dst_address); i++)
+    {
+        uword * hash = mfib->fib_entry_by_dst_address[i];
+
+        if (NULL != hash)
+        {
+            hash_pair_t * p;
+
+            hash_foreach_pair (p, hash,
+            ({
+                vec_add1(mfib_entry_indicies, p->value[0]);
+            }));
+        }
+    }
+
+    vec_sort_with_function(mfib_entry_indicies, mfib_entry_cmp_for_sort);
+
+    vec_foreach(mfib_entry_index, mfib_entry_indicies)
+    {
+        vlib_cli_output(vm, "%U",
+                        format_mfib_entry,
+                        *mfib_entry_index,
+                        MFIB_ENTRY_FORMAT_BRIEF);
+    }
+
+    vec_free(mfib_entry_indicies);
+}
+
+static void
+ip4_mfib_table_show_one (ip4_mfib_t *mfib,
+                         vlib_main_t * vm,
+                         ip4_address_t *src,
+                         ip4_address_t *grp,
+                         u32 mask_len)
+{
+    vlib_cli_output(vm, "%U",
+                    format_mfib_entry,
+                    ip4_mfib_table_lookup(mfib, src, grp, mask_len),
+                    MFIB_ENTRY_FORMAT_DETAIL);
+}
+
+static clib_error_t *
+ip4_show_mfib (vlib_main_t * vm,
+               unformat_input_t * input,
+               vlib_cli_command_t * cmd)
+{
+    ip4_main_t * im4 = &ip4_main;
+    mfib_table_t *mfib_table;
+    int verbose, matching;
+    ip4_address_t grp, src = {{0}};
+    u32 mask = 32;
+    int i, table_id = -1, fib_index = ~0;
+
+    verbose = 1;
+    matching = 0;
+
+    while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+        if (unformat (input, "brief") || unformat (input, "summary")
+            || unformat (input, "sum"))
+            verbose = 0;
+
+        else if (unformat (input, "%U %U",
+                           unformat_ip4_address, &src,
+                           unformat_ip4_address, &grp))
+        {
+            matching = 1;
+            mask = 64;
+        }
+        else if (unformat (input, "%U", unformat_ip4_address, &grp))
+        {
+            matching = 1;
+            mask = 32;
+        }
+        else if (unformat (input, "%U/%d",
+                           unformat_ip4_address, &grp, &mask))
+            matching = 1;
+        else if (unformat (input, "table %d", &table_id))
+            ;
+        else if (unformat (input, "index %d", &fib_index))
+            ;
+        else
+            break;
+    }
+
+    pool_foreach (mfib_table, im4->mfibs,
+    ({
+        ip4_mfib_t *mfib = &mfib_table->v4;
+
+        if (table_id >= 0 && table_id != (int)mfib->table_id)
+            continue;
+        if (fib_index != ~0 && fib_index != (int)mfib->index)
+            continue;
+
+        vlib_cli_output (vm, "%U, fib_index %d",
+                         format_mfib_table_name, mfib->index, FIB_PROTOCOL_IP4,
+                         mfib->index);
+
+        /* Show summary? */
+        if (! verbose)
+        {
+            vlib_cli_output (vm, "%=20s%=16s", "Prefix length", "Count");
+            for (i = 0; i < ARRAY_LEN (mfib->fib_entry_by_dst_address); i++)
+            {
+                uword * hash = mfib->fib_entry_by_dst_address[i];
+                uword n_elts = hash_elts (hash);
+                if (n_elts > 0)
+                    vlib_cli_output (vm, "%20d%16d", i, n_elts);
+            }
+            continue;
+        }
+
+        if (!matching)
+        {
+            ip4_mfib_table_show_all(mfib, vm);
+        }
+        else
+        {
+            ip4_mfib_table_show_one(mfib, vm, &src, &grp, mask);
+        }
+    }));
+
+    return 0;
+}
+
+/*?
+ * This command displays the IPv4 MulticasrFIB Tables (VRF Tables) and
+ * the route entries for each table.
+ *
+ * @note This command will run for a long time when the FIB tables are
+ * comprised of millions of entries. For those senarios, consider displaying
+ * a single table or summary mode.
+ *
+ * @cliexpar
+ * Example of how to display all the IPv4 Multicast FIB tables:
+ * @cliexstart{show ip fib}
+ * ipv4-VRF:0, fib_index 0
+ * (*, 0.0.0.0/0):  flags:D,
+ *  Interfaces:
+ *  multicast-ip4-chain
+ *   [@1]: dpo-drop ip4
+ * (*, 232.1.1.1/32):
+ * Interfaces:
+ *  test-eth1: Forward,
+ *  test-eth2: Forward,
+ *  test-eth0: Accept,
+ * multicast-ip4-chain
+ * [@2]: dpo-replicate: [index:1 buckets:2 to:[0:0]]
+ *   [0] [@1]: ipv4-mcast: test-eth1: IP4: d0:d1:d2:d3:d4:01 -> 01:00:05:00:00:00
+ *   [1] [@1]: ipv4-mcast: test-eth2: IP4: d0:d1:d2:d3:d4:02 -> 01:00:05:00:00:00
+ *
+ * @cliexend
+ * Example of how to display a summary of all IPv4 FIB tables:
+ * @cliexstart{show ip fib summary}
+ * ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
+ *     Prefix length         Count
+ *                    0               1
+ *                    8               2
+ *                   32               4
+ * ipv4-VRF:7, fib_index 1, flow hash: src dst sport dport proto
+ *     Prefix length         Count
+ *                    0               1
+ *                    8               2
+ *                   24               2
+ *                   32               4
+ * @cliexend
+ ?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip4_show_fib_command, static) = {
+    .path = "show ip mfib",
+    .short_help = "show ip mfib [summary] [table <table-id>] [index <fib-id>] [<gre-addr>[/<mask>]] [<gre-addr>] [<src-addr> <gre-addr>]",
+    .function = ip4_show_mfib,
+};
+/* *INDENT-ON* */
diff --git a/src/vnet/mfib/ip4_mfib.h b/src/vnet/mfib/ip4_mfib.h
new file mode 100644
index 00000000..6fc74a36
--- /dev/null
+++ b/src/vnet/mfib/ip4_mfib.h
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @brief The IPv4 Multicast-FIB
+ *
+ * FIXME
+ *
+ * This IPv4 FIB is used by the protocol independent FIB. So directly using
+ * this APIs in client code is not encouraged. However, this IPv4 FIB can be
+ * used if all the client wants is an IPv4 prefix data-base
+ */
+
+#ifndef __IP4_MFIB_H__
+#define __IP4_MFIB_H__
+
+#include <vlib/vlib.h>
+#include <vnet/ip/ip.h>
+
+#include <vnet/mfib/mfib_table.h>
+
+extern fib_node_index_t ip4_mfib_table_lookup(const ip4_mfib_t *fib,
+                                              const ip4_address_t *src,
+                                              const ip4_address_t *grp,
+                                              u32 len);
+extern fib_node_index_t ip4_mfib_table_lookup_exact_match(const ip4_mfib_t *fib,
+                                                          const ip4_address_t *grp,
+                                                          const ip4_address_t *src,
+                                                          u32 len);
+
+extern void ip4_mfib_table_entry_remove(ip4_mfib_t *fib,
+                                        const ip4_address_t *grp,
+                                        const ip4_address_t *src,
+                                        u32 len);
+
+extern void ip4_mfib_table_entry_insert(ip4_mfib_t *fib,
+                                        const ip4_address_t *grp,
+                                        const ip4_address_t *src,
+                                        u32 len,
+                                        fib_node_index_t fib_entry_index);
+extern void ip4_mfib_table_destroy(ip4_mfib_t *fib);
+
+/**
+ * @brief Get the FIB at the given index
+ */
+static inline ip4_mfib_t *
+ip4_mfib_get (u32 index)
+{
+    return (&(pool_elt_at_index(ip4_main.mfibs, index)->v4));
+}
+
+/**
+ * @brief Get or create an IPv4 fib.
+ *
+ * Get or create an IPv4 fib with the provided table ID.
+ *
+ * @param table_id
+ *      When set to \c ~0, an arbitrary and unused fib ID is picked
+ *      and can be retrieved with \c ret->table_id.
+ *      Otherwise, the fib ID to be used to retrieve or create the desired fib.
+ * @returns A pointer to the retrieved or created fib.
+ *
+ */
+extern u32 ip4_mfib_table_find_or_create_and_lock(u32 table_id);
+extern u32 ip4_mfib_table_create_and_lock(void);
+
+static inline
+u32 ip4_mfib_index_from_table_id (u32 table_id)
+{
+  ip4_main_t * im = &ip4_main;
+  uword * p;
+
+  p = hash_get (im->mfib_index_by_table_id, table_id);
+  if (!p)
+    return ~0;
+
+  return p[0];
+}
+
+extern u32 ip4_mfib_table_get_index_for_sw_if_index(u32 sw_if_index);
+
+
+#endif
+
diff --git a/src/vnet/mfib/ip6_mfib.c b/src/vnet/mfib/ip6_mfib.c
new file mode 100644
index 00000000..0c2e4c7b
--- /dev/null
+++ b/src/vnet/mfib/ip6_mfib.c
@@ -0,0 +1,663 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/mfib/ip6_mfib.h>
+
+#include <vnet/mfib/mfib_table.h>
+#include <vnet/mfib/mfib_entry.h>
+#include <vnet/fib/ip6_fib.h>
+
+/**
+ * The number of bytes in an address/ask key in the radix tree
+ * First byte is the length in bytes.
+ */
+#define IP6_MFIB_KEY_LEN 33
+
+/**
+ * Key and mask for radix
+ */
+typedef struct ip6_mfib_key_t_
+{
+    u8 key[IP6_MFIB_KEY_LEN];
+    u8 mask[IP6_MFIB_KEY_LEN];
+} ip6_mfib_key_t;
+
+/**
+ * An object that is inserted into the radix tree.
+ * Since it's in the tree and has pointers, it cannot realloc and so cannot
+ * come from a vlib pool.
+ */
+typedef struct ip6_mfib_node_t_
+{
+    struct radix_node i6mn_nodes[2];
+    ip6_mfib_key_t i6mn_key;
+    index_t i6mn_entry;
+} ip6_mfib_node_t;
+
+static const mfib_prefix_t all_zeros = {
+    /* (*,*) */
+    .fp_src_addr = {
+        .ip6.as_u64 = {0, 0},
+    },
+    .fp_grp_addr = {
+        .ip6.as_u64 = {0, 0},
+    },
+    .fp_len  = 0,
+    .fp_proto = FIB_PROTOCOL_IP6,
+};
+
+typedef enum ip6_mfib_special_type_t_ {
+    IP6_MFIB_SPECIAL_TYPE_NONE,
+    IP6_MFIB_SPECIAL_TYPE_SOLICITED,
+} ip6_mfib_special_type_t;
+
+typedef struct ip6_mfib_special_t_ {
+    /**
+     * @brief solicited or not
+     */
+    ip6_mfib_special_type_t ims_type;
+
+    /**
+     * @brief the Prefix length
+     */
+    u8 ims_len;
+
+    /**
+     * @brief The last byte of the mcast address
+     */
+    u8 ims_byte;
+    /**
+     * @brief The scope of the address
+     */
+    u8 ims_scope;
+} ip6_mfib_special_t;
+
+static const ip6_mfib_special_t ip6_mfib_specials[] =
+{
+    {
+        /*
+         * Add ff02::1:ff00:0/104 via local route for all tables.
+         *  This is required for neighbor discovery to work.
+         */
+        .ims_type = IP6_MFIB_SPECIAL_TYPE_SOLICITED,
+        .ims_len = 104,
+    },
+    {
+        /*
+         * all-routers multicast address
+         */
+        .ims_type = IP6_MFIB_SPECIAL_TYPE_NONE,
+        .ims_scope = IP6_MULTICAST_SCOPE_link_local,
+        .ims_byte = IP6_MULTICAST_GROUP_ID_all_routers,
+        .ims_len = 128,
+    },
+    {
+        /*
+         * all-nodes multicast address
+         */
+        .ims_type = IP6_MFIB_SPECIAL_TYPE_NONE,
+        .ims_scope = IP6_MULTICAST_SCOPE_link_local,
+        .ims_byte = IP6_MULTICAST_GROUP_ID_all_hosts,
+        .ims_len = 128,
+    },
+    {
+        /*
+         *  Add all-mldv2  multicast address via local route for all tables
+         */
+        .ims_type = IP6_MFIB_SPECIAL_TYPE_NONE,
+        .ims_len = 128,
+        .ims_scope = IP6_MULTICAST_SCOPE_link_local,
+        .ims_byte = IP6_MULTICAST_GROUP_ID_mldv2_routers,
+    }
+};
+
+#define FOR_EACH_IP6_SPECIAL(_pfx, _body)                               \
+{                                                                       \
+    const ip6_mfib_special_t *_spec;                                    \
+    u8 _ii;                                                             \
+    for (_ii = 0;                                                       \
+         _ii < ARRAY_LEN(ip6_mfib_specials);                            \
+         _ii++)                                                         \
+    {                                                                   \
+        _spec = &ip6_mfib_specials[_ii];                                \
+        if (IP6_MFIB_SPECIAL_TYPE_SOLICITED == _spec->ims_type)         \
+        {                                                               \
+            ip6_set_solicited_node_multicast_address(                   \
+                &(_pfx)->fp_grp_addr.ip6, 0);                           \
+        }                                                               \
+        else                                                            \
+        {                                                               \
+            ip6_set_reserved_multicast_address (                        \
+                &(_pfx)->fp_grp_addr.ip6,                               \
+                _spec->ims_scope,                                       \
+                _spec->ims_byte);                                       \
+        }                                                               \
+        (_pfx)->fp_len = _spec->ims_len;                                \
+        do { _body; } while (0);                                        \
+    }                                                                   \
+}
+
+
+static u32
+ip6_create_mfib_with_table_id (u32 table_id)
+{
+    mfib_table_t *mfib_table;
+    mfib_prefix_t pfx = {
+        .fp_proto = FIB_PROTOCOL_IP6,
+    };
+    const fib_route_path_t path_for_us = {
+        .frp_proto = FIB_PROTOCOL_IP6,
+        .frp_addr = zero_addr,
+        .frp_sw_if_index = 0xffffffff,
+        .frp_fib_index = ~0,
+        .frp_weight = 0,
+        .frp_flags = FIB_ROUTE_PATH_LOCAL,
+    };
+
+    pool_get_aligned(ip6_main.mfibs, mfib_table, CLIB_CACHE_LINE_BYTES);
+    memset(mfib_table, 0, sizeof(*mfib_table));
+
+    mfib_table->mft_proto = FIB_PROTOCOL_IP6;
+    mfib_table->mft_index =
+        mfib_table->v6.index =
+            (mfib_table - ip6_main.mfibs);
+
+    hash_set (ip6_main.mfib_index_by_table_id,
+              table_id,
+              mfib_table->mft_index);
+
+    mfib_table->mft_table_id =
+        mfib_table->v6.table_id =
+            table_id;
+
+    mfib_table_lock(mfib_table->mft_index, FIB_PROTOCOL_IP6);
+
+    mfib_table->v6.rhead =
+        clib_mem_alloc_aligned (sizeof(*mfib_table->v6.rhead),
+                                CLIB_CACHE_LINE_BYTES);
+    rn_inithead0(mfib_table->v6.rhead, 8);
+
+    /*
+     * add the special entries into the new FIB
+     */
+    mfib_table_entry_update(mfib_table->mft_index,
+                            &all_zeros,
+                            MFIB_SOURCE_DEFAULT_ROUTE,
+                            MFIB_ENTRY_FLAG_DROP);
+
+    /*
+     * Add each of the specials
+     */
+    FOR_EACH_IP6_SPECIAL(&pfx,
+    ({
+        mfib_table_entry_path_update(mfib_table->mft_index,
+                                     &pfx,
+                                     MFIB_SOURCE_SPECIAL,
+                                     &path_for_us,
+                                     MFIB_ITF_FLAG_FORWARD);
+    }));
+
+    return (mfib_table->mft_index);
+}
+
+void
+ip6_mfib_table_destroy (ip6_mfib_t *mfib)
+{
+    mfib_table_t *mfib_table = (mfib_table_t*)mfib;
+    fib_node_index_t mfei;
+    mfib_prefix_t pfx = {
+        .fp_proto = FIB_PROTOCOL_IP6,
+    };
+    const fib_route_path_t path_for_us = {
+        .frp_proto = FIB_PROTOCOL_IP6,
+        .frp_addr = zero_addr,
+        .frp_sw_if_index = 0xffffffff,
+        .frp_fib_index = ~0,
+        .frp_weight = 0,
+        .frp_flags = FIB_ROUTE_PATH_LOCAL,
+    };
+
+    /*
+     * remove all the specials we added when the table was created.
+     */
+    FOR_EACH_IP6_SPECIAL(&pfx,
+    {
+        mfib_table_entry_path_remove(mfib_table->mft_index,
+                                     &pfx,
+                                     MFIB_SOURCE_SPECIAL,
+                                     &path_for_us);
+    });
+
+    mfei = mfib_table_lookup_exact_match(mfib_table->mft_index, &all_zeros);
+    mfib_table_entry_delete_index(mfei, MFIB_SOURCE_DEFAULT_ROUTE);
+
+    /*
+     * validate no more routes.
+     */
+    ASSERT(0 == mfib_table->mft_total_route_counts);
+    ASSERT(~0 != mfib_table->mft_table_id);
+
+    hash_unset (ip6_main.mfib_index_by_table_id, mfib_table->mft_table_id);
+    clib_mem_free(mfib_table->v6.rhead);
+    pool_put(ip6_main.mfibs, mfib_table);
+}
+
+void
+ip6_mfib_interface_enable_disable (u32 sw_if_index, int is_enable)
+{
+    const fib_route_path_t path = {
+        .frp_proto = FIB_PROTOCOL_IP6,
+        .frp_addr = zero_addr,
+        .frp_sw_if_index = sw_if_index,
+        .frp_fib_index = ~0,
+        .frp_weight = 0,
+    };
+    mfib_prefix_t pfx = {
+        .fp_proto = FIB_PROTOCOL_IP6,
+    };
+    u32 mfib_index;
+
+    vec_validate (ip6_main.mfib_index_by_sw_if_index, sw_if_index);
+    mfib_index = ip6_mfib_table_get_index_for_sw_if_index(sw_if_index);
+
+    if (is_enable)
+    {
+        FOR_EACH_IP6_SPECIAL(&pfx,
+        {
+            mfib_table_entry_path_update(mfib_index,
+                                         &pfx,
+                                         MFIB_SOURCE_SPECIAL,
+                                         &path,
+                                         MFIB_ITF_FLAG_ACCEPT);
+        });
+    }
+    else
+    {
+        FOR_EACH_IP6_SPECIAL(&pfx,
+        {
+            mfib_table_entry_path_remove(mfib_index,
+                                         &pfx,
+                                         MFIB_SOURCE_SPECIAL,
+                                         &path);
+        });
+    }
+}
+
+u32
+ip6_mfib_table_find_or_create_and_lock (u32 table_id)
+{
+    u32 index;
+
+    index = ip6_mfib_index_from_table_id(table_id);
+    if (~0 == index)
+        return ip6_create_mfib_with_table_id(table_id);
+    mfib_table_lock(index, FIB_PROTOCOL_IP6);
+
+    return (index);
+}
+
+u32
+ip6_mfib_table_get_index_for_sw_if_index (u32 sw_if_index)
+{
+    if (sw_if_index >= vec_len(ip6_main.mfib_index_by_sw_if_index))
+    {
+        /*
+         * This is the case for interfaces that are not yet mapped to
+         * a IP table
+         */
+        return (~0);
+    }
+    return (ip6_main.mfib_index_by_sw_if_index[sw_if_index]);
+}
+
+#define IP6_MFIB_MK_KEY(_grp, _src, _key)                           \
+{                                                                   \
+    (_key)->key[0] = 33;                                            \
+    memcpy((_key)->key+1, _grp, 16);                                \
+    memcpy((_key)->key+17, _src, 16);                               \
+}
+
+#define IP6_MFIB_MK_KEY_MASK(_grp, _src, _len, _key)                \
+{                                                                   \
+    IP6_MFIB_MK_KEY(_grp, _src, _key);                              \
+                                                                    \
+    (_key)->mask[0] = 33;                                           \
+    if (_len <= 128)                                                \
+    {                                                               \
+        memcpy((_key)->mask+1, &ip6_main.fib_masks[_len], 16);      \
+        memset((_key)->mask+17, 0, 16);                             \
+    }                                                               \
+    else                                                            \
+    {                                                               \
+        ASSERT(_len == 256);                                        \
+        memcpy((_key)->mask+1, &ip6_main.fib_masks[128], 16);       \
+        memcpy((_key)->mask+17, &ip6_main.fib_masks[128], 16);      \
+    }                                                               \
+}
+
+/*
+ * ip6_fib_table_lookup_exact_match
+ *
+ * Exact match prefix lookup
+ */
+fib_node_index_t
+ip6_mfib_table_lookup_exact_match (const ip6_mfib_t *mfib,
+                                   const ip6_address_t *grp,
+                                   const ip6_address_t *src,
+                                   u32 len)
+{
+    ip6_mfib_node_t *i6mn;
+    ip6_mfib_key_t key;
+
+    IP6_MFIB_MK_KEY_MASK(grp, src, len, &key);
+
+    i6mn = (ip6_mfib_node_t*) rn_lookup(key.key, key.mask,
+                                        (struct radix_node_head *)mfib->rhead);
+
+    if (NULL == i6mn)
+    {
+        return (INDEX_INVALID);
+    }
+
+    return (i6mn->i6mn_entry);
+}
+
+/*
+ * ip6_fib_table_lookup
+ *
+ * Longest prefix match
+ */
+fib_node_index_t
+ip6_mfib_table_lookup (const ip6_mfib_t *mfib,
+                       const ip6_address_t *src,
+                       const ip6_address_t *grp,
+                       u32 len)
+{
+    ip6_mfib_node_t *i6mn;
+    ip6_mfib_key_t key;
+
+    IP6_MFIB_MK_KEY_MASK(grp, src, len, &key);
+
+    i6mn = (ip6_mfib_node_t*) rn_search_m(key.key,
+                                          mfib->rhead->rnh_treetop,
+                                          key.mask);
+
+    ASSERT(NULL != i6mn);
+
+    return (i6mn->i6mn_entry);
+}
+
+/*
+ * ip6_fib_table_lookup
+ *
+ * Longest prefix match no mask
+ */
+fib_node_index_t
+ip6_mfib_table_lookup2 (const ip6_mfib_t *mfib,
+                        const ip6_address_t *src,
+                        const ip6_address_t *grp)
+{
+    ip6_mfib_node_t *i6mn;
+    ip6_mfib_key_t key;
+
+    IP6_MFIB_MK_KEY(grp, src, &key);
+
+    i6mn = (ip6_mfib_node_t*) rn_match(key.key,
+                                       (struct radix_node_head *)mfib->rhead); // const cast
+
+    ASSERT(NULL != i6mn);
+
+    return (i6mn->i6mn_entry);
+}
+
+void
+ip6_mfib_table_entry_insert (ip6_mfib_t *mfib,
+                             const ip6_address_t *grp,
+                             const ip6_address_t *src,
+                             u32 len,
+                             fib_node_index_t mfib_entry_index)
+{
+    ip6_mfib_node_t *i6mn = clib_mem_alloc(sizeof(*i6mn));
+
+    memset(i6mn, 0, sizeof(*i6mn));
+
+    IP6_MFIB_MK_KEY_MASK(grp, src, len, &i6mn->i6mn_key);
+    i6mn->i6mn_entry = mfib_entry_index;
+
+    if (NULL == rn_addroute(i6mn->i6mn_key.key,
+                            i6mn->i6mn_key.mask,
+                            mfib->rhead,
+                            i6mn->i6mn_nodes))
+    {
+        ASSERT(0);
+    }
+}
+
+void
+ip6_mfib_table_entry_remove (ip6_mfib_t *mfib,
+                             const ip6_address_t *grp,
+                             const ip6_address_t *src,
+                             u32 len)
+{
+    ip6_mfib_node_t *i6mn;
+    ip6_mfib_key_t key;
+
+    IP6_MFIB_MK_KEY_MASK(grp, src, len, &key);
+
+    i6mn = (ip6_mfib_node_t*) rn_delete(key.key, key.mask, mfib->rhead);
+
+    clib_mem_free(i6mn);
+}
+
+static clib_error_t *
+ip6_mfib_module_init (vlib_main_t * vm)
+{
+    return (NULL);
+}
+
+VLIB_INIT_FUNCTION(ip6_mfib_module_init);
+
+static void
+ip6_mfib_table_show_one (ip6_mfib_t *mfib,
+                         vlib_main_t * vm,
+                         ip6_address_t *src,
+                         ip6_address_t *grp,
+                         u32 mask_len)
+{
+    vlib_cli_output(vm, "%U",
+                    format_mfib_entry,
+                    ip6_mfib_table_lookup(mfib, src, grp, mask_len),
+                    MFIB_ENTRY_FORMAT_DETAIL);
+}
+
+typedef struct ip6_mfib_show_ctx_t_ {
+    u32 fib_index;
+    fib_node_index_t *entries;
+} ip6_mfib_show_ctx_t;
+
+
+static int
+ip6_mfib_table_collect_entries (struct radix_node *rn, void *arg)
+{
+    ip6_mfib_show_ctx_t *ctx = arg;
+    ip6_mfib_node_t *i6mn;
+
+    i6mn = (ip6_mfib_node_t*) rn;
+
+    vec_add1(ctx->entries, i6mn->i6mn_entry);
+
+    return (0);
+}
+
+static void
+ip6_mfib_table_show_all (ip6_mfib_t *mfib,
+                         vlib_main_t * vm)
+{
+    fib_node_index_t *mfib_entry_index;
+    ip6_mfib_show_ctx_t ctx = {
+        .fib_index = mfib->index,
+        .entries = NULL,
+    };
+
+    rn_walktree(mfib->rhead,
+                ip6_mfib_table_collect_entries,
+                &ctx);
+
+    vec_sort_with_function(ctx.entries, mfib_entry_cmp_for_sort);
+
+    vec_foreach(mfib_entry_index, ctx.entries)
+    {
+        vlib_cli_output(vm, "%U",
+                        format_mfib_entry,
+                        *mfib_entry_index,
+                        MFIB_ENTRY_FORMAT_BRIEF);
+    }
+
+    vec_free(ctx.entries);
+}
+
+static clib_error_t *
+ip6_show_mfib (vlib_main_t * vm,
+               unformat_input_t * input,
+               vlib_cli_command_t * cmd)
+{
+    ip6_main_t * im4 = &ip6_main;
+    mfib_table_t *mfib_table;
+    int verbose, matching;
+    ip6_address_t grp, src = {{0}};
+    u32 mask = 32;
+    int table_id = -1, fib_index = ~0;
+
+    verbose = 1;
+    matching = 0;
+
+    while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+        if (unformat (input, "brief") || unformat (input, "summary")
+            || unformat (input, "sum"))
+            verbose = 0;
+
+        else if (unformat (input, "%U %U",
+                           unformat_ip6_address, &src,
+                           unformat_ip6_address, &grp))
+        {
+            matching = 1;
+            mask = 64;
+        }
+        else if (unformat (input, "%U", unformat_ip6_address, &grp))
+        {
+            matching = 1;
+            mask = 32;
+        }
+        else if (unformat (input, "%U/%d",
+                           unformat_ip6_address, &grp, &mask))
+            matching = 1;
+        else if (unformat (input, "table %d", &table_id))
+            ;
+        else if (unformat (input, "index %d", &fib_index))
+            ;
+        else
+            break;
+    }
+
+    pool_foreach (mfib_table, im4->mfibs,
+    ({
+        ip6_mfib_t *mfib = &mfib_table->v6;
+
+        if (table_id >= 0 && table_id != (int)mfib->table_id)
+            continue;
+        if (fib_index != ~0 && fib_index != (int)mfib->index)
+            continue;
+
+        vlib_cli_output (vm, "%U, fib_index %d",
+                         format_mfib_table_name, mfib->index, FIB_PROTOCOL_IP6,
+                         mfib->index);
+
+        /* Show summary? */
+        if (! verbose)
+        {
+            /* vlib_cli_output (vm, "%=20s%=16s", "Prefix length", "Count"); */
+            /* for (i = 0; i < ARRAY_LEN (mfib->fib_entry_by_dst_address); i++) */
+            /* { */
+            /*     uword * hash = mfib->fib_entry_by_dst_address[i]; */
+            /*     uword n_elts = hash_elts (hash); */
+            /*     if (n_elts > 0) */
+            /*         vlib_cli_output (vm, "%20d%16d", i, n_elts); */
+            /* } */
+            continue;
+        }
+
+        if (!matching)
+        {
+            ip6_mfib_table_show_all(mfib, vm);
+        }
+        else
+        {
+            ip6_mfib_table_show_one(mfib, vm, &src, &grp, mask);
+        }
+    }));
+
+    return 0;
+}
+
+/*
+ * This command displays the IPv4 MulticasrFIB Tables (VRF Tables) and
+ * the route entries for each table.
+ *
+ * @note This command will run for a long time when the FIB tables are
+ * comprised of millions of entries. For those senarios, consider displaying
+ * a single table or summary mode.
+ *
+ * @cliexpar
+ * Example of how to display all the IPv4 Multicast FIB tables:
+ * @cliexstart{show ip fib}
+ * ipv4-VRF:0, fib_index 0
+ * (*, 0.0.0.0/0):  flags:D,
+ *  Interfaces:
+ *  multicast-ip6-chain
+ *   [@1]: dpo-drop ip6
+ * (*, 232.1.1.1/32):
+ * Interfaces:
+ *  test-eth1: Forward,
+ *  test-eth2: Forward,
+ *  test-eth0: Accept,
+ * multicast-ip6-chain
+ * [@2]: dpo-replicate: [index:1 buckets:2 to:[0:0]]
+ *   [0] [@1]: ipv4-mcast: test-eth1: IP6: d0:d1:d2:d3:d4:01 -> 01:00:05:00:00:00
+ *   [1] [@1]: ipv4-mcast: test-eth2: IP6: d0:d1:d2:d3:d4:02 -> 01:00:05:00:00:00
+ *
+ * @cliexend
+ * Example of how to display a summary of all IPv4 FIB tables:
+ * @cliexstart{show ip fib summary}
+ * ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
+ *     Prefix length         Count
+ *                    0               1
+ *                    8               2
+ *                   32               4
+ * ipv4-VRF:7, fib_index 1, flow hash: src dst sport dport proto
+ *     Prefix length         Count
+ *                    0               1
+ *                    8               2
+ *                   24               2
+ *                   32               4
+ * @cliexend
+ */
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip6_show_fib_command, static) = {
+    .path = "show ip6 mfib",
+    .short_help = "show ip mfib [summary] [table <table-id>] [index <fib-id>] [<grp-addr>[/<mask>]] [<grp-addr>] [<src-addr> <grp-addr>]",
+    .function = ip6_show_mfib,
+};
+/* *INDENT-ON* */
diff --git a/src/vnet/mfib/ip6_mfib.h b/src/vnet/mfib/ip6_mfib.h
new file mode 100644
index 00000000..d91af46d
--- /dev/null
+++ b/src/vnet/mfib/ip6_mfib.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @brief The IPv4 Multicast-FIB
+ *
+ * FIXME
+ *
+ * This IPv4 FIB is used by the protocol independent FIB. So directly using
+ * this APIs in client code is not encouraged. However, this IPv4 FIB can be
+ * used if all the client wants is an IPv4 prefix data-base
+ */
+
+#ifndef __IP6_MFIB_H__
+#define __IP6_MFIB_H__
+
+#include <vlib/vlib.h>
+#include <vnet/ip/ip.h>
+
+#include <vnet/mfib/mfib_table.h>
+
+extern fib_node_index_t ip6_mfib_table_lookup(const ip6_mfib_t *fib,
+                                              const ip6_address_t *src,
+                                              const ip6_address_t *grp,
+                                              u32 len);
+extern fib_node_index_t ip6_mfib_table_lookup_exact_match(const ip6_mfib_t *fib,
+                                                          const ip6_address_t *grp,
+                                                          const ip6_address_t *src,
+                                                          u32 len);
+
+extern void ip6_mfib_table_entry_remove(ip6_mfib_t *fib,
+                                        const ip6_address_t *grp,
+                                        const ip6_address_t *src,
+                                        u32 len);
+
+extern void ip6_mfib_table_entry_insert(ip6_mfib_t *fib,
+                                        const ip6_address_t *grp,
+                                        const ip6_address_t *src,
+                                        u32 len,
+                                        fib_node_index_t fib_entry_index);
+extern void ip6_mfib_table_destroy(ip6_mfib_t *fib);
+
+/**
+ * @brief
+ *  Add/remove the interface from the accepting list of the special MFIB entries
+ */
+extern void ip6_mfib_interface_enable_disable(u32 sw_if_index,
+                                              int is_enable);
+
+/**
+ * @brief Get the FIB at the given index
+ */
+static inline ip6_mfib_t *
+ip6_mfib_get (u32 index)
+{
+    return (&(pool_elt_at_index(ip6_main.mfibs, index)->v6));
+}
+
+/**
+ * @brief Get or create an IPv4 fib.
+ *
+ * Get or create an IPv4 fib with the provided table ID.
+ *
+ * @param table_id
+ *      When set to \c ~0, an arbitrary and unused fib ID is picked
+ *      and can be retrieved with \c ret->table_id.
+ *      Otherwise, the fib ID to be used to retrieve or create the desired fib.
+ * @returns A pointer to the retrieved or created fib.
+ *
+ */
+extern u32 ip6_mfib_table_find_or_create_and_lock(u32 table_id);
+extern u32 ip6_mfib_table_create_and_lock(void);
+
+
+static inline
+u32 ip6_mfib_index_from_table_id (u32 table_id)
+{
+  ip6_main_t * im = &ip6_main;
+  uword * p;
+
+  p = hash_get (im->mfib_index_by_table_id, table_id);
+  if (!p)
+    return ~0;
+
+  return p[0];
+}
+
+extern u32 ip6_mfib_table_get_index_for_sw_if_index(u32 sw_if_index);
+
+/**
+ * @brief Data-plane lookup function
+ */
+extern fib_node_index_t ip6_mfib_table_lookup2(const ip6_mfib_t *mfib,
+                                               const ip6_address_t *src,
+                                               const ip6_address_t *grp);
+
+#endif
+
diff --git a/src/vnet/mfib/mfib_entry.c b/src/vnet/mfib/mfib_entry.c
new file mode 100644
index 00000000..479ce5f1
--- /dev/null
+++ b/src/vnet/mfib/mfib_entry.c
@@ -0,0 +1,1096 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vlib/vlib.h>
+
+#include <vnet/mfib/mfib_entry.h>
+#include <vnet/fib/fib_path_list.h>
+
+#include <vnet/dpo/drop_dpo.h>
+#include <vnet/dpo/replicate_dpo.h>
+
+/**
+ * Debug macro
+ */
+#ifdef MFIB_DEBUG
+#DEFIne MFIB_ENTRY_DBG(_e, _fmt, _args...)		\
+{                                                       \
+    u8*__tmp = NULL;					\
+    __tmp = format(__tmp, "e:[%d:%U",                   \
+                   mfib_entry_get_index(_e),		\
+                   format_ip46_address,			\
+                   &_e->mfe_prefix.fp_grp_addr,		\
+                   IP46_TYPE_ANY);			\
+    __tmp = format(__tmp, "/%d,",			\
+                   _e->mfe_prefix.fp_len);		\
+    __tmp = format(__tmp, "%U]",                        \
+                   mfib_entry_get_index(_e),		\
+                   format_ip46_address,			\
+                   &_e->mfe_prefix.fp_src_addr,		\
+                   IP46_TYPE_ANY);			\
+    __tmp = format(__tmp, _fmt, ##_args);		\
+    clib_warning("%s", __tmp);				\
+    vec_free(__tmp);					\
+}
+#else
+#define MFIB_ENTRY_DBG(_e, _fmt, _args...)
+#endif
+
+/**
+ * The source of an MFIB entry
+ */
+typedef struct mfib_entry_src_t_
+{
+    /**
+     * Which source this is
+     */
+    mfib_source_t mfes_src;
+
+    /**
+     * The path-list of forwarding interfaces
+     */
+    fib_node_index_t mfes_pl;
+
+    /**
+     * Route flags
+     */
+    mfib_entry_flags_t mfes_flags;
+
+    /**
+     * The hash table of all interfaces
+     */
+    mfib_itf_t *mfes_itfs;
+} mfib_entry_src_t;
+
+/**
+ * String names for each source
+ */
+static const char *mfib_source_names[] = MFIB_SOURCE_NAMES;
+
+/*
+ * Pool for all fib_entries
+ */
+mfib_entry_t *mfib_entry_pool;
+
+static fib_node_t *
+mfib_entry_get_node (fib_node_index_t index)
+{
+    return ((fib_node_t*)mfib_entry_get(index));
+}
+
+static fib_protocol_t
+mfib_entry_get_proto (const mfib_entry_t * mfib_entry)
+{
+    return (mfib_entry->mfe_prefix.fp_proto);
+}
+
+fib_forward_chain_type_t
+mfib_entry_get_default_chain_type (const mfib_entry_t *mfib_entry)
+{
+    switch (mfib_entry->mfe_prefix.fp_proto)
+    {
+    case FIB_PROTOCOL_IP4:
+        return (FIB_FORW_CHAIN_TYPE_MCAST_IP4);
+    case FIB_PROTOCOL_IP6:
+        return (FIB_FORW_CHAIN_TYPE_MCAST_IP6);
+    case FIB_PROTOCOL_MPLS:
+        ASSERT(0);
+        break;
+    }
+    return (FIB_FORW_CHAIN_TYPE_MCAST_IP4);
+}
+
+static u8 *
+format_mfib_entry_dpo (u8 * s, va_list * args)
+{
+    index_t fei = va_arg(*args, index_t);
+    CLIB_UNUSED(u32 indent) = va_arg(*args, u32);
+
+    return (format(s, "%U",
+                   format_mfib_entry, fei,
+                   MFIB_ENTRY_FORMAT_BRIEF));
+}
+
+u8 *
+format_mfib_entry (u8 * s, va_list * args)
+{
+    fib_node_index_t fei, mfi;
+    mfib_entry_t *mfib_entry;
+    mfib_entry_src_t *msrc;
+    u32 sw_if_index;
+    int level;
+
+    fei = va_arg (*args, fib_node_index_t);
+    level = va_arg (*args, int);
+    mfib_entry = mfib_entry_get(fei);
+
+    s = format (s, "%U", format_mfib_prefix, &mfib_entry->mfe_prefix);
+    s = format (s, ": %U", format_mfib_entry_flags, mfib_entry->mfe_flags);
+
+    if (level >= MFIB_ENTRY_FORMAT_DETAIL)
+    {
+        s = format (s, "\n");
+        s = format (s, " fib:%d", mfib_entry->mfe_fib_index);
+        s = format (s, " index:%d", mfib_entry_get_index(mfib_entry));
+        s = format (s, " locks:%d\n", mfib_entry->mfe_node.fn_locks);
+        vec_foreach(msrc, mfib_entry->mfe_srcs)
+        {
+            s = format (s, "  src:%s", mfib_source_names[msrc->mfes_src]);
+            s = format (s, ": %U\n", format_mfib_entry_flags, msrc->mfes_flags);
+            if (FIB_NODE_INDEX_INVALID != msrc->mfes_pl)
+            {
+                s = fib_path_list_format(msrc->mfes_pl, s);
+            }
+            hash_foreach(sw_if_index, mfi, msrc->mfes_itfs,
+            ({
+                s = format(s, "    %U\n", format_mfib_itf, mfi);
+            }));
+        }
+    }
+
+    s = format(s, "\n  Interfaces:");
+    hash_foreach(sw_if_index, mfi, mfib_entry->mfe_itfs,
+    ({
+        s = format(s, "\n  %U", format_mfib_itf, mfi);
+    }));
+
+    s = format(s, "\n  %U-chain\n  %U",
+               format_fib_forw_chain_type,
+               mfib_entry_get_default_chain_type(mfib_entry),
+               format_dpo_id,
+               &mfib_entry->mfe_rep,
+               2);
+    s = format(s, "\n");
+
+    if (level >= MFIB_ENTRY_FORMAT_DETAIL2)
+    {
+        s = format(s, "\nchildren:");
+        s = fib_node_children_format(mfib_entry->mfe_node.fn_children, s);
+    }
+
+    return (s);
+}
+
+static mfib_entry_t*
+mfib_entry_from_fib_node (fib_node_t *node)
+{
+#if CLIB_DEBUG > 0
+    ASSERT(FIB_NODE_TYPE_MFIB_ENTRY == node->fn_type);
+#endif
+    return ((mfib_entry_t*)node);
+}
+
+static int
+mfib_entry_src_cmp_for_sort (void * v1,
+                             void * v2)
+{
+    mfib_entry_src_t *esrc1 = v1, *esrc2 = v2;
+
+    return (esrc1->mfes_src - esrc2->mfes_src);
+}
+
+static void
+mfib_entry_src_init (mfib_entry_t *mfib_entry,
+                     mfib_source_t source)
+
+{
+    mfib_entry_src_t esrc = {
+        .mfes_pl = FIB_NODE_INDEX_INVALID,
+        .mfes_flags = MFIB_ENTRY_FLAG_NONE,
+        .mfes_src = source,
+    };
+
+    vec_add1(mfib_entry->mfe_srcs, esrc);
+    vec_sort_with_function(mfib_entry->mfe_srcs,
+                           mfib_entry_src_cmp_for_sort);
+}
+
+static mfib_entry_src_t *
+mfib_entry_src_find (const mfib_entry_t *mfib_entry,
+                    mfib_source_t source,
+                    u32 *index)
+
+{
+    mfib_entry_src_t *esrc;
+    int ii;
+
+    ii = 0;
+    vec_foreach(esrc, mfib_entry->mfe_srcs)
+    {
+        if (esrc->mfes_src == source)
+        {
+            if (NULL != index)
+            {
+                *index = ii;
+            }
+            return (esrc);
+        }
+        else
+        {
+            ii++;
+        }
+    }
+
+    return (NULL);
+}
+
+static mfib_entry_src_t *
+mfib_entry_src_find_or_create (mfib_entry_t *mfib_entry,
+                              mfib_source_t source)
+{
+    mfib_entry_src_t *esrc;
+
+    esrc = mfib_entry_src_find(mfib_entry, source, NULL);
+
+    if (NULL == esrc)
+    {
+        mfib_entry_src_init(mfib_entry, source);
+    }
+
+    return (mfib_entry_src_find(mfib_entry, source, NULL));
+}
+
+static mfib_entry_src_t*
+mfib_entry_get_best_src (const mfib_entry_t *mfib_entry)
+{
+    mfib_entry_src_t *bsrc;
+
+    /*
+     * the enum of sources is deliberately arranged in priority order
+     */
+    if (0 == vec_len(mfib_entry->mfe_srcs))
+    {
+        bsrc = NULL;
+    }
+    else
+    {
+        bsrc = vec_elt_at_index(mfib_entry->mfe_srcs, 0);
+    }
+
+    return (bsrc);
+}
+
+static void
+mfib_entry_src_flush (mfib_entry_src_t *msrc)
+{
+    u32 sw_if_index;
+    index_t mfii;
+
+    hash_foreach(sw_if_index, mfii, msrc->mfes_itfs,
+    ({
+        mfib_itf_delete(mfib_itf_get(mfii));
+    }));
+}
+
+static void
+mfib_entry_src_remove (mfib_entry_t *mfib_entry,
+                       mfib_source_t source)
+
+{
+    mfib_entry_src_t *msrc;
+    u32 index = ~0;
+
+    msrc = mfib_entry_src_find(mfib_entry, source, &index);
+
+    if (NULL != msrc)
+    {
+        mfib_entry_src_flush(msrc);
+        vec_del1(mfib_entry->mfe_srcs, index);
+    }
+}
+
+static int
+mfib_entry_src_n_itfs (const mfib_entry_src_t *msrc)
+{
+    return (hash_elts(msrc->mfes_itfs));
+}
+
+
+static void
+mfib_entry_last_lock_gone (fib_node_t *node)
+{
+    mfib_entry_t *mfib_entry;
+    mfib_entry_src_t *msrc;
+
+    mfib_entry = mfib_entry_from_fib_node(node);
+
+    dpo_reset(&mfib_entry->mfe_rep);
+
+    MFIB_ENTRY_DBG(mfib_entry, "last-lock");
+
+    vec_foreach(msrc, mfib_entry->mfe_srcs)
+    {
+        mfib_entry_src_flush(msrc);
+    }
+
+    fib_path_list_unlock(mfib_entry->mfe_parent);
+    vec_free(mfib_entry->mfe_srcs);
+
+    fib_node_deinit(&mfib_entry->mfe_node);
+    pool_put(mfib_entry_pool, mfib_entry);
+}
+
+/*
+ * mfib_entry_back_walk_notify
+ *
+ * A back walk has reach this entry.
+ */
+static fib_node_back_walk_rc_t
+mfib_entry_back_walk_notify (fib_node_t *node,
+                            fib_node_back_walk_ctx_t *ctx)
+{
+    // FIXME - re-evalute
+
+    return (FIB_NODE_BACK_WALK_CONTINUE);
+}
+
+static void
+mfib_entry_show_memory (void)
+{
+    fib_show_memory_usage("multicast-Entry",
+                          pool_elts(mfib_entry_pool),
+                          pool_len(mfib_entry_pool),
+                          sizeof(mfib_entry_t));
+}
+
+/*
+ * The MFIB entry's graph node virtual function table
+ */
+static const fib_node_vft_t mfib_entry_vft = {
+    .fnv_get = mfib_entry_get_node,
+    .fnv_last_lock = mfib_entry_last_lock_gone,
+    .fnv_back_walk = mfib_entry_back_walk_notify,
+    .fnv_mem_show = mfib_entry_show_memory,
+};
+
+u32
+mfib_entry_child_add (fib_node_index_t mfib_entry_index,
+                      fib_node_type_t child_type,
+                      fib_node_index_t child_index)
+{
+    return (fib_node_child_add(FIB_NODE_TYPE_MFIB_ENTRY,
+                               mfib_entry_index,
+                               child_type,
+                               child_index));
+};
+
+void
+mfib_entry_child_remove (fib_node_index_t mfib_entry_index,
+                         u32 sibling_index)
+{
+    fib_node_child_remove(FIB_NODE_TYPE_MFIB_ENTRY,
+                          mfib_entry_index,
+                          sibling_index);
+}
+
+static mfib_entry_t *
+mfib_entry_alloc (u32 fib_index,
+                  const mfib_prefix_t *prefix,
+                  fib_node_index_t *mfib_entry_index)
+{
+    mfib_entry_t *mfib_entry;
+
+    pool_get(mfib_entry_pool, mfib_entry);
+    memset(mfib_entry, 0, sizeof(*mfib_entry));
+
+    fib_node_init(&mfib_entry->mfe_node,
+                  FIB_NODE_TYPE_MFIB_ENTRY);
+
+    mfib_entry->mfe_fib_index = fib_index;
+    mfib_entry->mfe_prefix = *prefix;
+    mfib_entry->mfe_parent = FIB_NODE_INDEX_INVALID;
+
+    dpo_reset(&mfib_entry->mfe_rep);
+
+    *mfib_entry_index = mfib_entry_get_index(mfib_entry);
+
+    MFIB_ENTRY_DBG(mfib_entry, "alloc");
+
+    return (mfib_entry);
+}
+
+typedef struct mfib_entry_collect_forwarding_ctx_t_
+{
+    load_balance_path_t * next_hops;
+    fib_forward_chain_type_t fct;
+} mfib_entry_collect_forwarding_ctx_t;
+
+static int
+mfib_entry_src_collect_forwarding (fib_node_index_t pl_index,
+                                   fib_node_index_t path_index,
+                                   void *arg)
+{
+    mfib_entry_collect_forwarding_ctx_t *ctx;
+    load_balance_path_t *nh;
+
+    ctx = arg;
+
+    /*
+     * if the path is not resolved, don't include it.
+     */
+    if (!fib_path_is_resolved(path_index))
+    {
+        return (!0);
+    }
+
+    switch (ctx->fct)
+    {
+    case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
+    case FIB_FORW_CHAIN_TYPE_MCAST_IP6:
+        /*
+         * EOS traffic with no label to stack, we need the IP Adj
+         */
+        vec_add2(ctx->next_hops, nh, 1);
+
+        nh->path_index = path_index;
+        nh->path_weight = fib_path_get_weight(path_index);
+        fib_path_contribute_forwarding(path_index, ctx->fct, &nh->path_dpo);
+        break;
+
+    case FIB_FORW_CHAIN_TYPE_UNICAST_IP4:
+    case FIB_FORW_CHAIN_TYPE_UNICAST_IP6:
+    case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS:
+    case FIB_FORW_CHAIN_TYPE_MPLS_EOS:
+    case FIB_FORW_CHAIN_TYPE_ETHERNET:
+        ASSERT(0);
+        break;
+    }
+
+    return (!0);
+}
+
+static void
+mfib_entry_stack (mfib_entry_t *mfib_entry)
+{
+    mfib_entry_collect_forwarding_ctx_t ctx = {
+        .next_hops = NULL,
+        .fct = mfib_entry_get_default_chain_type(mfib_entry),
+    };
+    dpo_proto_t dp;
+
+    dp = fib_proto_to_dpo(mfib_entry_get_proto(mfib_entry));
+
+    if (FIB_NODE_INDEX_INVALID != mfib_entry->mfe_parent)
+    {
+        fib_path_list_walk(mfib_entry->mfe_parent,
+                           mfib_entry_src_collect_forwarding,
+                           &ctx);
+
+        if (!dpo_id_is_valid(&mfib_entry->mfe_rep) ||
+            dpo_is_drop(&mfib_entry->mfe_rep))
+        {
+            dpo_id_t tmp_dpo = DPO_INVALID;
+
+            dpo_set(&tmp_dpo,
+                    DPO_REPLICATE, dp,
+                    replicate_create(0, dp));
+
+            dpo_stack(DPO_MFIB_ENTRY, dp,
+                      &mfib_entry->mfe_rep,
+                      &tmp_dpo);
+
+            dpo_reset(&tmp_dpo);
+        }
+        replicate_multipath_update(&mfib_entry->mfe_rep,
+                                   ctx.next_hops);
+    }
+    else
+    {
+        dpo_stack(DPO_MFIB_ENTRY, dp,
+                  &mfib_entry->mfe_rep,
+                  drop_dpo_get(dp));
+    }
+}
+
+static void
+mfib_entry_forwarding_path_add (mfib_entry_src_t *msrc,
+                                const fib_route_path_t *rpath)
+{
+    fib_node_index_t old_pl_index;
+    fib_route_path_t *rpaths;
+
+    /*
+     * path-lists require a vector of paths
+     */
+    rpaths = NULL;
+    vec_add1(rpaths, rpath[0]);
+
+    old_pl_index = msrc->mfes_pl;
+
+    if (FIB_NODE_INDEX_INVALID == msrc->mfes_pl)
+    {
+        msrc->mfes_pl =
+            fib_path_list_create(FIB_PATH_LIST_FLAG_NO_URPF,
+                                 rpaths);
+    }
+    else
+    {
+        msrc->mfes_pl =
+            fib_path_list_copy_and_path_add(msrc->mfes_pl,
+                                            FIB_PATH_LIST_FLAG_NO_URPF,
+                                            rpaths);
+    }
+    fib_path_list_lock(msrc->mfes_pl);
+    fib_path_list_unlock(old_pl_index);
+
+    vec_free(rpaths);
+}
+
+static int
+mfib_entry_forwarding_path_remove (mfib_entry_src_t *msrc,
+                                   const fib_route_path_t *rpath)
+{
+    fib_node_index_t old_pl_index;
+    fib_route_path_t *rpaths;
+
+    /*
+     * path-lists require a vector of paths
+     */
+    rpaths = NULL;
+    vec_add1(rpaths, rpath[0]);
+
+    old_pl_index = msrc->mfes_pl;
+
+    msrc->mfes_pl =
+        fib_path_list_copy_and_path_remove(msrc->mfes_pl,
+                                           FIB_PATH_LIST_FLAG_NONE,
+                                           rpaths);
+
+    fib_path_list_lock(msrc->mfes_pl);
+    fib_path_list_unlock(old_pl_index);
+
+    vec_free(rpaths);
+
+    return (FIB_NODE_INDEX_INVALID != msrc->mfes_pl);
+}
+
+static void
+mfib_entry_recalculate_forwarding (mfib_entry_t *mfib_entry)
+{
+    fib_node_index_t old_pl_index;
+    mfib_entry_src_t *bsrc;
+
+    old_pl_index = mfib_entry->mfe_parent;
+
+    /*
+     * copy the forwarding data from the bast source
+     */
+    bsrc = mfib_entry_get_best_src(mfib_entry);
+
+    if (NULL == bsrc)
+    {
+        mfib_entry->mfe_parent = FIB_NODE_INDEX_INVALID;
+    }
+    else
+    {
+        mfib_entry->mfe_parent = bsrc->mfes_pl;
+        mfib_entry->mfe_flags = bsrc->mfes_flags;
+        mfib_entry->mfe_itfs = bsrc->mfes_itfs;
+    }
+
+    /*
+     * re-stack the entry on the best forwarding info.
+     */
+    if (old_pl_index != mfib_entry->mfe_parent ||
+        FIB_NODE_INDEX_INVALID == old_pl_index)
+    {
+        mfib_entry_stack(mfib_entry);
+
+        fib_path_list_lock(mfib_entry->mfe_parent);
+        fib_path_list_unlock(old_pl_index);
+    }
+}
+
+
+fib_node_index_t
+mfib_entry_create (u32 fib_index,
+                   mfib_source_t source,
+                   const mfib_prefix_t *prefix,
+                   mfib_entry_flags_t entry_flags)
+{
+    fib_node_index_t mfib_entry_index;
+    mfib_entry_t *mfib_entry;
+    mfib_entry_src_t *msrc;
+
+    mfib_entry = mfib_entry_alloc(fib_index, prefix,
+                                  &mfib_entry_index);
+    msrc = mfib_entry_src_find_or_create(mfib_entry, source);
+    msrc->mfes_flags = entry_flags;
+
+    mfib_entry_recalculate_forwarding(mfib_entry);
+
+    return (mfib_entry_index);
+}
+
+static int
+mfib_entry_ok_for_delete (mfib_entry_t *mfib_entry)
+{
+    return (0 == vec_len(mfib_entry->mfe_srcs));
+}
+
+static int
+mfib_entry_src_ok_for_delete (const mfib_entry_src_t *msrc)
+{
+    return ((MFIB_ENTRY_FLAG_NONE == msrc->mfes_flags &&
+             0 == mfib_entry_src_n_itfs(msrc)));
+}
+
+int
+mfib_entry_update (fib_node_index_t mfib_entry_index,
+                   mfib_source_t source,
+                   mfib_entry_flags_t entry_flags)
+{
+    mfib_entry_t *mfib_entry;
+    mfib_entry_src_t *msrc;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+    msrc = mfib_entry_src_find_or_create(mfib_entry, source);
+    msrc->mfes_flags = entry_flags;
+
+    if (mfib_entry_src_ok_for_delete(msrc))
+    {
+        /*
+         * this source has no interfaces and no flags.
+         * it has nothing left to give - remove it
+         */
+        mfib_entry_src_remove(mfib_entry, source);
+    }
+
+    mfib_entry_recalculate_forwarding(mfib_entry);
+
+    return (mfib_entry_ok_for_delete(mfib_entry));
+}
+
+static void
+mfib_entry_itf_add (mfib_entry_src_t *msrc,
+                    u32 sw_if_index,
+                    index_t mi)
+{
+    hash_set(msrc->mfes_itfs, sw_if_index, mi);
+}
+
+static void
+mfib_entry_itf_remove (mfib_entry_src_t *msrc,
+                       u32 sw_if_index)
+{
+    mfib_itf_t *mfi;
+
+    mfi = mfib_entry_itf_find(msrc->mfes_itfs, sw_if_index);
+
+    mfib_itf_delete(mfi);
+
+    hash_unset(msrc->mfes_itfs, sw_if_index);
+}
+
+void
+mfib_entry_path_update (fib_node_index_t mfib_entry_index,
+                        mfib_source_t source,
+                        const fib_route_path_t *rpath,
+                        mfib_itf_flags_t itf_flags)
+{
+    mfib_entry_t *mfib_entry;
+    mfib_entry_src_t *msrc;
+    mfib_itf_t *mfib_itf;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+    ASSERT(NULL != mfib_entry);
+    msrc = mfib_entry_src_find_or_create(mfib_entry, source);
+
+    /*
+     * search for the interface in the current set
+     */
+    mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs,
+                                   rpath[0].frp_sw_if_index);
+
+    if (NULL == mfib_itf)
+    {
+        /*
+         * this is a path we do not yet have. If it is forwarding then we
+         * add it to the replication set
+         */
+        if (itf_flags & MFIB_ITF_FLAG_FORWARD)
+        {
+            mfib_entry_forwarding_path_add(msrc, rpath);
+        }
+        /*
+         * construct a new ITF for this entry's list
+         */
+        mfib_entry_itf_add(msrc,
+                           rpath[0].frp_sw_if_index,
+                           mfib_itf_create(rpath[0].frp_sw_if_index,
+                                           itf_flags));
+    }
+    else
+    {
+        int was_forwarding = !!(mfib_itf->mfi_flags & MFIB_ITF_FLAG_FORWARD);
+        int is_forwarding  = !!(itf_flags & MFIB_ITF_FLAG_FORWARD);
+
+        if (!was_forwarding && is_forwarding)
+        {
+            mfib_entry_forwarding_path_add(msrc, rpath);
+        }
+        else if (was_forwarding && !is_forwarding)
+        {
+            mfib_entry_forwarding_path_remove(msrc, rpath);
+        }
+        /*
+         * packets in flight see these updates.
+         */
+        mfib_itf->mfi_flags = itf_flags;
+    }
+
+    mfib_entry_recalculate_forwarding(mfib_entry);
+}
+
+/*
+ * mfib_entry_path_remove
+ *
+ * remove a path from the entry.
+ * return the mfib_entry's index if it is still present, INVALID otherwise.
+ */
+int
+mfib_entry_path_remove (fib_node_index_t mfib_entry_index,
+                        mfib_source_t source,
+                        const fib_route_path_t *rpath)
+{
+    mfib_entry_t *mfib_entry;
+    mfib_entry_src_t *msrc;
+    mfib_itf_t *mfib_itf;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+    ASSERT(NULL != mfib_entry);
+    msrc = mfib_entry_src_find(mfib_entry, source, NULL);
+
+    if (NULL == msrc)
+    {
+        /*
+         * there are no paths left for this source
+         */
+        return (mfib_entry_ok_for_delete(mfib_entry));
+    }
+
+    /*
+     * search for the interface in the current set
+     */
+    mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs,
+                                   rpath[0].frp_sw_if_index);
+
+    if (NULL == mfib_itf)
+    {
+        /*
+         * removing a path that does not exist
+         */
+        return (mfib_entry_ok_for_delete(mfib_entry));
+    }
+
+    /*
+     * we have this path. If it is forwarding then we
+     * remove it to the replication set
+     */
+    if (mfib_itf->mfi_flags & MFIB_ITF_FLAG_FORWARD)
+    {
+        mfib_entry_forwarding_path_remove(msrc, rpath);
+    }
+
+    /*
+     * remove the interface/path from this entry's list
+     */
+    mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index);
+
+    if (mfib_entry_src_ok_for_delete(msrc))
+    {
+        /*
+         * this source has no interfaces and no flags.
+         * it has nothing left to give - remove it
+         */
+        mfib_entry_src_remove(mfib_entry, source);
+    }
+
+    mfib_entry_recalculate_forwarding(mfib_entry);
+
+    return (mfib_entry_ok_for_delete(mfib_entry));
+}
+
+/**
+ * mfib_entry_delete
+ *
+ * The source is withdrawing all the paths it provided
+ */
+int
+mfib_entry_delete (fib_node_index_t mfib_entry_index,
+                   mfib_source_t source)
+{
+    mfib_entry_t *mfib_entry;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+    mfib_entry_src_remove(mfib_entry, source);
+
+    mfib_entry_recalculate_forwarding(mfib_entry);
+
+    return (mfib_entry_ok_for_delete(mfib_entry));
+}
+
+static int
+fib_ip4_address_compare (ip4_address_t * a1,
+                         ip4_address_t * a2)
+{
+    /*
+     * IP addresses are unsiged ints. the return value here needs to be signed
+     * a simple subtraction won't cut it.
+     * If the addresses are the same, the sort order is undefiend, so phoey.
+     */
+    return ((clib_net_to_host_u32(a1->data_u32) >
+             clib_net_to_host_u32(a2->data_u32) ) ?
+            1 : -1);
+}
+
+static int
+fib_ip6_address_compare (ip6_address_t * a1,
+                         ip6_address_t * a2)
+{
+  int i;
+  for (i = 0; i < ARRAY_LEN (a1->as_u16); i++)
+  {
+      int cmp = (clib_net_to_host_u16 (a1->as_u16[i]) -
+                 clib_net_to_host_u16 (a2->as_u16[i]));
+      if (cmp != 0)
+          return cmp;
+  }
+  return 0;
+}
+
+static int
+mfib_entry_cmp (fib_node_index_t mfib_entry_index1,
+                fib_node_index_t mfib_entry_index2)
+{
+    mfib_entry_t *mfib_entry1, *mfib_entry2;
+    int cmp = 0;
+
+    mfib_entry1 = mfib_entry_get(mfib_entry_index1);
+    mfib_entry2 = mfib_entry_get(mfib_entry_index2);
+
+    switch (mfib_entry1->mfe_prefix.fp_proto)
+    {
+    case FIB_PROTOCOL_IP4:
+        cmp = fib_ip4_address_compare(&mfib_entry1->mfe_prefix.fp_grp_addr.ip4,
+                                      &mfib_entry2->mfe_prefix.fp_grp_addr.ip4);
+
+        if (0 == cmp)
+        {
+            cmp = fib_ip4_address_compare(&mfib_entry1->mfe_prefix.fp_src_addr.ip4,
+                                          &mfib_entry2->mfe_prefix.fp_src_addr.ip4);
+        }
+        break;
+    case FIB_PROTOCOL_IP6:
+        cmp = fib_ip6_address_compare(&mfib_entry1->mfe_prefix.fp_grp_addr.ip6,
+                                      &mfib_entry2->mfe_prefix.fp_grp_addr.ip6);
+
+        if (0 == cmp)
+        {
+            cmp = fib_ip6_address_compare(&mfib_entry1->mfe_prefix.fp_src_addr.ip6,
+                                          &mfib_entry2->mfe_prefix.fp_src_addr.ip6);
+        }
+        break;
+    case FIB_PROTOCOL_MPLS:
+        ASSERT(0);
+        cmp = 0;
+        break;
+    }
+
+    if (0 == cmp) {
+        cmp = (mfib_entry1->mfe_prefix.fp_len - mfib_entry2->mfe_prefix.fp_len);
+    }
+    return (cmp);
+}
+
+int
+mfib_entry_cmp_for_sort (void *i1, void *i2)
+{
+    fib_node_index_t *mfib_entry_index1 = i1, *mfib_entry_index2 = i2;
+
+    return (mfib_entry_cmp(*mfib_entry_index1,
+                           *mfib_entry_index2));
+}
+
+void
+mfib_entry_lock (fib_node_index_t mfib_entry_index)
+{
+    mfib_entry_t *mfib_entry;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+
+    fib_node_lock(&mfib_entry->mfe_node);
+}
+
+void
+mfib_entry_unlock (fib_node_index_t mfib_entry_index)
+{
+    mfib_entry_t *mfib_entry;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+
+    fib_node_unlock(&mfib_entry->mfe_node);
+}
+
+static void
+mfib_entry_dpo_lock (dpo_id_t *dpo)
+{
+}
+static void
+mfib_entry_dpo_unlock (dpo_id_t *dpo)
+{
+}
+
+const static dpo_vft_t mfib_entry_dpo_vft = {
+    .dv_lock = mfib_entry_dpo_lock,
+    .dv_unlock = mfib_entry_dpo_unlock,
+    .dv_format = format_mfib_entry_dpo,
+    .dv_mem_show = mfib_entry_show_memory,
+};
+
+const static char* const mfib_entry_ip4_nodes[] =
+{
+    "ip4-mfib-forward-rpf",
+    NULL,
+};
+const static char* const mfib_entry_ip6_nodes[] =
+{
+    "ip6-mfib-forward-rpf",
+    NULL,
+};
+
+const static char* const * const mfib_entry_nodes[DPO_PROTO_NUM] =
+{
+    [DPO_PROTO_IP4]  = mfib_entry_ip4_nodes,
+    [DPO_PROTO_IP6]  = mfib_entry_ip6_nodes,
+};
+
+void
+mfib_entry_module_init (void)
+{
+    fib_node_register_type (FIB_NODE_TYPE_MFIB_ENTRY, &mfib_entry_vft);
+    dpo_register(DPO_MFIB_ENTRY, &mfib_entry_dpo_vft, mfib_entry_nodes);
+}
+
+void
+mfib_entry_encode (fib_node_index_t mfib_entry_index,
+                  fib_route_path_encode_t **api_rpaths)
+{
+    mfib_entry_t *mfib_entry;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+    fib_path_list_walk(mfib_entry->mfe_parent, fib_path_encode, api_rpaths);
+}
+
+void
+mfib_entry_get_prefix (fib_node_index_t mfib_entry_index,
+                      mfib_prefix_t *pfx)
+{
+    mfib_entry_t *mfib_entry;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+    *pfx = mfib_entry->mfe_prefix;
+}
+
+u32
+mfib_entry_get_fib_index (fib_node_index_t mfib_entry_index)
+{
+    mfib_entry_t *mfib_entry;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+
+    return (mfib_entry->mfe_fib_index);
+}
+
+void
+mfib_entry_contribute_forwarding (fib_node_index_t mfib_entry_index,
+                                  fib_forward_chain_type_t type,
+                                  dpo_id_t *dpo)
+{
+    /*
+     * An IP mFIB entry can only provide a forwarding chain that
+     * is the same IP proto as the prefix.
+     * No use-cases (i know of) for other combinations.
+     */
+    mfib_entry_t *mfib_entry;
+    dpo_proto_t dp;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+
+    dp = fib_proto_to_dpo(mfib_entry->mfe_prefix.fp_proto);
+
+    if (type == fib_forw_chain_type_from_dpo_proto(dp))
+    {
+        dpo_copy(dpo, &mfib_entry->mfe_rep);
+    }
+    else
+    {
+        dpo_copy(dpo, drop_dpo_get(dp));
+    }
+}
+
+u32
+mfib_entry_pool_size (void)
+{
+    return (pool_elts(mfib_entry_pool));
+}
+
+static clib_error_t *
+show_mfib_entry_command (vlib_main_t * vm,
+                        unformat_input_t * input,
+                        vlib_cli_command_t * cmd)
+{
+    fib_node_index_t fei;
+
+    if (unformat (input, "%d", &fei))
+    {
+        /*
+         * show one in detail
+         */
+        if (!pool_is_free_index(mfib_entry_pool, fei))
+        {
+            vlib_cli_output (vm, "%d@%U",
+                             fei,
+                             format_mfib_entry, fei,
+                             MFIB_ENTRY_FORMAT_DETAIL2);
+        }
+        else
+        {
+            vlib_cli_output (vm, "entry %d invalid", fei);
+        }
+    }
+    else
+    {
+        /*
+         * show all
+         */
+        vlib_cli_output (vm, "FIB Entries:");
+        pool_foreach_index(fei, mfib_entry_pool,
+        ({
+            vlib_cli_output (vm, "%d@%U",
+                             fei,
+                             format_mfib_entry, fei,
+                             MFIB_ENTRY_FORMAT_BRIEF);
+        }));
+    }
+
+    return (NULL);
+}
+
+VLIB_CLI_COMMAND (show_mfib_entry, static) = {
+  .path = "show mfib entry",
+  .function = show_mfib_entry_command,
+  .short_help = "show mfib entry",
+};
diff --git a/src/vnet/mfib/mfib_entry.h b/src/vnet/mfib/mfib_entry.h
new file mode 100644
index 00000000..cc5d5326
--- /dev/null
+++ b/src/vnet/mfib/mfib_entry.h
@@ -0,0 +1,172 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MFIB_ENTRY_H__
+#define __MFIB_ENTRY_H__
+
+#include <vnet/fib/fib_node.h>
+#include <vnet/mfib/mfib_types.h>
+#include <vnet/mfib/mfib_itf.h>
+#include <vnet/ip/ip.h>
+#include <vnet/dpo/dpo.h>
+
+/**
+ * An entry in a FIB table.
+ *
+ * This entry represents a route added to the FIB that is stored
+ * in one of the FIB tables.
+ */
+typedef struct mfib_entry_t_ {
+    CLIB_CACHE_LINE_ALIGN_MARK(cacheline0);
+    /**
+     * Base class. The entry's node representation in the graph.
+     */
+    fib_node_t mfe_node;
+    /**
+     * The prefix of the route
+     */
+    mfib_prefix_t mfe_prefix;
+    /**
+     * The index of the FIB table this entry is in
+     */
+    u32 mfe_fib_index;
+    /**
+     * the path-list for which this entry is a child. This is also the path-list
+     * that is contributing forwarding for this entry.
+     */
+    fib_node_index_t mfe_parent;
+    /**
+     * index of this entry in the parent's child list.
+     * This is set when this entry is added as a child, but can also
+     * be changed by the parent as it manages its list.
+     */
+    u32 mfe_sibling;
+
+    /**
+     * A vector of sources contributing forwarding
+     */
+    struct mfib_entry_src_t_ *mfe_srcs;
+
+    /**
+     * 2nd cache line has the members used in the data plane
+     */
+    CLIB_CACHE_LINE_ALIGN_MARK(cacheline1);
+
+    /**
+     * The Replicate used for forwarding.
+     */
+    dpo_id_t mfe_rep;
+
+    /**
+     * Route flags
+     */
+    mfib_entry_flags_t mfe_flags;
+
+    /**
+     * A hash table of interfaces
+     */
+    mfib_itf_t *mfe_itfs;
+} mfib_entry_t;
+
+#define MFIB_ENTRY_FORMAT_BRIEF   (0x0)
+#define MFIB_ENTRY_FORMAT_DETAIL  (0x1)
+#define MFIB_ENTRY_FORMAT_DETAIL2 (0x2)
+
+extern u8 *format_mfib_entry(u8 * s, va_list * args);
+
+
+extern fib_node_index_t mfib_entry_create(u32 fib_index,
+                                          mfib_source_t source,
+                                          const mfib_prefix_t *prefix,
+                                          mfib_entry_flags_t entry_flags);
+
+extern int mfib_entry_update(fib_node_index_t fib_entry_index,
+                             mfib_source_t source,
+                             mfib_entry_flags_t entry_flags);
+
+extern void mfib_entry_path_update(fib_node_index_t fib_entry_index,
+                                   mfib_source_t source,
+                                   const fib_route_path_t *rpath,
+                                   mfib_itf_flags_t itf_flags);
+
+
+extern int mfib_entry_path_remove(fib_node_index_t fib_entry_index,
+                                  mfib_source_t source,
+                                  const fib_route_path_t *rpath);
+
+extern int mfib_entry_delete(fib_node_index_t mfib_entry_index,
+                             mfib_source_t source);
+
+extern int mfib_entry_cmp_for_sort(void *i1, void *i2);
+
+extern u32 mfib_entry_child_add(fib_node_index_t mfib_entry_index,
+                                fib_node_type_t type,
+                                fib_node_index_t child_index);
+extern void mfib_entry_child_remove(fib_node_index_t mfib_entry_index,
+                                    u32 sibling_index);
+
+extern void mfib_entry_lock(fib_node_index_t fib_entry_index);
+extern void mfib_entry_unlock(fib_node_index_t fib_entry_index);
+
+extern void mfib_entry_get_prefix(fib_node_index_t fib_entry_index,
+                                  mfib_prefix_t *pfx);
+extern u32 mfib_entry_get_fib_index(fib_node_index_t fib_entry_index);
+
+extern void mfib_entry_contribute_forwarding(
+    fib_node_index_t mfib_entry_index,
+    fib_forward_chain_type_t type,
+    dpo_id_t *dpo);
+
+extern void mfib_entry_module_init(void);
+
+
+extern mfib_entry_t *mfib_entry_pool;
+
+static inline mfib_entry_t *
+mfib_entry_get (fib_node_index_t index)
+{
+    return (pool_elt_at_index(mfib_entry_pool, index));
+}
+static inline fib_node_index_t
+mfib_entry_get_index (const mfib_entry_t *mfe)
+{
+    return (mfe - mfib_entry_pool);
+}
+
+
+static inline mfib_itf_t *
+mfib_entry_itf_find (mfib_itf_t *itfs,
+                     u32 sw_if_index)
+{
+    uword *p;
+
+    p = hash_get(itfs, sw_if_index);
+
+    if (NULL != p)
+    {
+        return (mfib_itf_get(p[0]));
+    }
+
+    return (NULL);
+}
+
+static inline mfib_itf_t *
+mfib_entry_get_itf (const mfib_entry_t *mfe,
+                    u32 sw_if_index)
+{
+    return (mfib_entry_itf_find(mfe->mfe_itfs, sw_if_index));
+}
+
+#endif
diff --git a/src/vnet/mfib/mfib_forward.c b/src/vnet/mfib/mfib_forward.c
new file mode 100644
index 00000000..5fe0a57c
--- /dev/null
+++ b/src/vnet/mfib/mfib_forward.c
@@ -0,0 +1,512 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/mfib/mfib_itf.h>
+#include <vnet/mfib/mfib_entry.h>
+#include <vnet/dpo/replicate_dpo.h>
+#include <vnet/mfib/ip4_mfib.h>
+#include <vnet/mfib/ip6_mfib.h>
+#include <vnet/mfib/mfib_signal.h>
+#include <vnet/fib/ip4_fib.h>
+#include <vnet/fib/ip6_fib.h>
+
+#include <vnet/ip/ip4.h>
+#include <vnet/vnet.h>
+
+typedef struct mfib_forward_lookup_trace_t_ {
+    u32 entry_index;
+    u32 fib_index;
+} mfib_forward_lookup_trace_t;
+
+static u8 *
+format_mfib_forward_lookup_trace (u8 * s, va_list * args)
+{
+    CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+    CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+    mfib_forward_lookup_trace_t * t = va_arg (*args, mfib_forward_lookup_trace_t *);
+
+    s = format (s, "fib %d entry %d", t->fib_index, t->entry_index);
+    return s;
+}
+
+/* Common trace function for all ip4-forward next nodes. */
+void
+mfib_forward_lookup_trace (vlib_main_t * vm,
+                           vlib_node_runtime_t * node,
+                           vlib_frame_t * frame)
+{
+    u32 * from, n_left;
+    ip4_main_t * im = &ip4_main;
+
+    n_left = frame->n_vectors;
+    from = vlib_frame_vector_args (frame);
+
+    while (n_left >= 4)
+    {
+        mfib_forward_lookup_trace_t * t0, * t1;
+        vlib_buffer_t * b0, * b1;
+        u32 bi0, bi1;
+
+        /* Prefetch next iteration. */
+        vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
+        vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
+
+        bi0 = from[0];
+        bi1 = from[1];
+
+        b0 = vlib_get_buffer (vm, bi0);
+        b1 = vlib_get_buffer (vm, bi1);
+
+        if (b0->flags & VLIB_BUFFER_IS_TRACED)
+        {
+            t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
+            t0->entry_index = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
+            t0->fib_index = vec_elt (im->mfib_index_by_sw_if_index,
+                                     vnet_buffer(b1)->sw_if_index[VLIB_RX]);
+        }
+        if (b1->flags & VLIB_BUFFER_IS_TRACED)
+        {
+            t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
+            t1->entry_index = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
+            t1->fib_index = vec_elt (im->mfib_index_by_sw_if_index,
+                                     vnet_buffer(b1)->sw_if_index[VLIB_RX]);
+        }
+        from += 2;
+        n_left -= 2;
+    }
+
+    while (n_left >= 1)
+    {
+        mfib_forward_lookup_trace_t * t0;
+        vlib_buffer_t * b0;
+        u32 bi0;
+
+        bi0 = from[0];
+
+        b0 = vlib_get_buffer (vm, bi0);
+
+        if (b0->flags & VLIB_BUFFER_IS_TRACED)
+        {
+            t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
+            t0->entry_index = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
+            t0->fib_index = vec_elt (im->mfib_index_by_sw_if_index,
+                                     vnet_buffer(b0)->sw_if_index[VLIB_RX]);
+        }
+        from += 1;
+        n_left -= 1;
+    }
+}
+
+typedef enum mfib_forward_lookup_next_t_ {
+    MFIB_FORWARD_LOOKUP_NEXT_RPF,
+    MFIB_FORWARD_LOOKUP_N_NEXT,
+} mfib_forward_lookup_next_t;
+
+static uword
+mfib_forward_lookup (vlib_main_t * vm,
+                     vlib_node_runtime_t * node,
+                     vlib_frame_t * frame,
+                     int is_v4)
+{
+    u32 n_left_from, n_left_to_next, * from, * to_next;
+
+    from = vlib_frame_vector_args (frame);
+    n_left_from = frame->n_vectors;
+
+    while (n_left_from > 0)
+    {
+        vlib_get_next_frame (vm, node, MFIB_FORWARD_LOOKUP_NEXT_RPF,
+                             to_next, n_left_to_next);
+
+        while (n_left_from > 0 && n_left_to_next > 0)
+        {
+            fib_node_index_t mfei0;
+            vlib_buffer_t * p0;
+            u32 fib_index0;
+            u32 pi0;
+
+            pi0 = from[0];
+            to_next[0] = pi0;
+            from += 1;
+            to_next += 1;
+            n_left_to_next -= 1;
+            n_left_from -= 1;
+
+            p0 = vlib_get_buffer (vm, pi0);
+
+            if (is_v4)
+            {
+                ip4_header_t * ip0;
+
+                fib_index0 = vec_elt (ip4_main.mfib_index_by_sw_if_index,
+                                      vnet_buffer(p0)->sw_if_index[VLIB_RX]);
+                ip0 = vlib_buffer_get_current (p0);
+                mfei0 = ip4_mfib_table_lookup(ip4_mfib_get(fib_index0),
+                                              &ip0->src_address,
+                                              &ip0->dst_address,
+                                              64);
+            }
+            else
+            {
+                ip6_header_t * ip0;
+
+                fib_index0 = vec_elt (ip6_main.mfib_index_by_sw_if_index,
+                                      vnet_buffer(p0)->sw_if_index[VLIB_RX]);
+                ip0 = vlib_buffer_get_current (p0);
+                mfei0 = ip6_mfib_table_lookup2(ip6_mfib_get(fib_index0),
+                                               &ip0->src_address,
+                                               &ip0->dst_address);
+            }
+
+            vnet_buffer (p0)->ip.adj_index[VLIB_TX] = mfei0;
+        }
+
+        vlib_put_next_frame(vm, node,
+                            MFIB_FORWARD_LOOKUP_NEXT_RPF,
+                            n_left_to_next);
+    }
+
+    if (node->flags & VLIB_NODE_FLAG_TRACE)
+        mfib_forward_lookup_trace(vm, node, frame);
+
+    return frame->n_vectors;
+}
+
+static uword
+ip4_mfib_forward_lookup (vlib_main_t * vm,
+                         vlib_node_runtime_t * node,
+                         vlib_frame_t * frame)
+{
+    return (mfib_forward_lookup (vm, node, frame, 1));
+}
+
+VLIB_REGISTER_NODE (ip4_mfib_forward_lookup_node, static) = {
+    .function = ip4_mfib_forward_lookup,
+    .name = "ip4-mfib-forward-lookup",
+    .vector_size = sizeof (u32),
+
+    .format_trace = format_mfib_forward_lookup_trace,
+
+    .n_next_nodes = MFIB_FORWARD_LOOKUP_N_NEXT,
+    .next_nodes = {
+        [MFIB_FORWARD_LOOKUP_NEXT_RPF] = "ip4-mfib-forward-rpf",
+    },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_mfib_forward_lookup_node,
+                              ip4_mfib_forward_lookup)
+
+static uword
+ip6_mfib_forward_lookup (vlib_main_t * vm,
+                         vlib_node_runtime_t * node,
+                         vlib_frame_t * frame)
+{
+    return (mfib_forward_lookup (vm, node, frame, 0));
+}
+
+VLIB_REGISTER_NODE (ip6_mfib_forward_lookup_node, static) = {
+    .function = ip6_mfib_forward_lookup,
+    .name = "ip6-mfib-forward-lookup",
+    .vector_size = sizeof (u32),
+
+    .format_trace = format_mfib_forward_lookup_trace,
+
+    .n_next_nodes = MFIB_FORWARD_LOOKUP_N_NEXT,
+    .next_nodes = {
+        [MFIB_FORWARD_LOOKUP_NEXT_RPF] = "ip6-mfib-forward-rpf",
+    },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_mfib_forward_lookup_node,
+                              ip6_mfib_forward_lookup)
+
+
+typedef struct mfib_forward_rpf_trace_t_ {
+    u32 entry_index;
+    u32 sw_if_index;
+    mfib_itf_flags_t itf_flags;
+} mfib_forward_rpf_trace_t;
+
+typedef enum mfib_forward_rpf_next_t_ {
+    MFIB_FORWARD_RPF_NEXT_DROP,
+    MFIB_FORWARD_RPF_N_NEXT,
+} mfib_forward_rpf_next_t;
+
+static u8 *
+format_mfib_forward_rpf_trace (u8 * s, va_list * args)
+{
+    CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+    CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+    mfib_forward_rpf_trace_t * t = va_arg (*args, mfib_forward_rpf_trace_t *);
+
+    s = format (s, "entry %d", t->entry_index);
+    s = format (s, " %d", t->sw_if_index);
+    s = format (s, " %U", format_mfib_itf_flags, t->itf_flags);
+
+    return s;
+}
+
+static int
+mfib_forward_connected_check (vlib_buffer_t * b0,
+                              u32 sw_if_index,
+                              int is_v4)
+{
+    /*
+     * Lookup the source of the IP packet in the
+     * FIB. return true if the entry is attached.
+     */
+    index_t lbi0;
+
+    if (is_v4)
+    {
+        load_balance_t *lb0;
+        ip4_header_t *ip0;
+
+        ip0 = vlib_buffer_get_current(b0);
+
+        lbi0 = ip4_fib_forwarding_lookup(
+                   ip4_fib_table_get_index_for_sw_if_index(
+                       sw_if_index),
+                   &ip0->src_address);
+        lb0 = load_balance_get(lbi0);
+
+        return (FIB_ENTRY_FLAG_ATTACHED &
+                lb0->lb_fib_entry_flags);
+    }
+    else
+    {
+        ASSERT(0);
+    }
+    return (0);
+}
+
+static void
+mfib_forward_itf_signal (vlib_main_t *vm,
+                         const mfib_entry_t *mfe,
+                         mfib_itf_t *mfi,
+                         vlib_buffer_t *b0)
+{
+    mfib_itf_flags_t old_flags;
+
+    old_flags = __sync_fetch_and_or(&mfi->mfi_flags,
+                                    MFIB_ITF_FLAG_SIGNAL_PRESENT);
+
+    if (!(old_flags & MFIB_ITF_FLAG_SIGNAL_PRESENT))
+    {
+        /*
+         * we were the lucky ones to set the signal present flag
+         */
+        if (!(old_flags & MFIB_ITF_FLAG_DONT_PRESERVE))
+        {
+            /*
+             * preserve a copy of the packet for the control
+             * plane to examine.
+             * Only allow one preserved packet at at time, since
+             * when the signal present flag is cleared so is the
+             * preserved packet.
+             */
+            mfib_signal_push(mfe, mfi, b0);
+        }
+        else
+        {
+            /*
+             *  The control plane just wants the signal, not the packet as well
+             */
+            mfib_signal_push(mfe, mfi, NULL);
+        }
+    }
+    /*
+     * else
+     *   there is already a signal present on this interface that the
+     *   control plane has not yet acknowledged
+     */
+}
+
+always_inline uword
+mfib_forward_rpf (vlib_main_t * vm,
+                  vlib_node_runtime_t * node,
+                  vlib_frame_t * frame,
+                  int is_v4)
+{
+    u32 n_left_from, n_left_to_next, * from, * to_next;
+    mfib_forward_rpf_next_t next;
+
+    from = vlib_frame_vector_args (frame);
+    n_left_from = frame->n_vectors;
+    next = MFIB_FORWARD_RPF_NEXT_DROP;
+
+    while (n_left_from > 0)
+    {
+        vlib_get_next_frame (vm, node, next,
+                             to_next, n_left_to_next);
+
+        while (n_left_from > 0 && n_left_to_next > 0)
+        {
+            fib_node_index_t mfei0;
+            const mfib_entry_t *mfe0;
+            mfib_itf_t *mfi0;
+            vlib_buffer_t * b0;
+            u32 pi0, next0;
+            mfib_itf_flags_t iflags0;
+            mfib_entry_flags_t eflags0;
+
+            pi0 = from[0];
+            to_next[0] = pi0;
+            from += 1;
+            to_next += 1;
+            n_left_to_next -= 1;
+            n_left_from -= 1;
+
+            b0 = vlib_get_buffer (vm, pi0);
+            mfei0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
+            mfe0 = mfib_entry_get(mfei0);
+            mfi0 = mfib_entry_get_itf(mfe0,
+                                      vnet_buffer(b0)->sw_if_index[VLIB_RX]);
+
+            /*
+             * throughout this function we are 'PREDICT' optimising
+             * for the case of throughput traffic that is not replicated
+             * to the host stack nor sets local flags
+             */
+            if (PREDICT_TRUE(NULL != mfi0))
+            {
+                iflags0 = mfi0->mfi_flags;
+            }
+            else
+            {
+                iflags0 = MFIB_ITF_FLAG_NONE;
+            }
+            eflags0 = mfe0->mfe_flags;
+
+            if (PREDICT_FALSE(eflags0 & MFIB_ENTRY_FLAG_CONNECTED))
+            {
+                /*
+                 * lookup the source in the unicast FIB - check it
+                 * matches a connected.
+                 */
+                if (mfib_forward_connected_check(
+                        b0,
+                        vnet_buffer(b0)->sw_if_index[VLIB_RX],
+                        is_v4))
+                {
+                    mfib_forward_itf_signal(vm, mfe0, mfi0, b0);
+                }
+            }
+            if (PREDICT_FALSE((eflags0 & MFIB_ENTRY_FLAG_SIGNAL) ^
+                              (iflags0 & MFIB_ITF_FLAG_NEGATE_SIGNAL)))
+            {
+                /*
+                 * Entry signal XOR interface negate-signal
+                 */
+                if (NULL != mfi0)
+                {
+                    mfib_forward_itf_signal(vm, mfe0, mfi0, b0);
+                }
+            }
+
+            if (PREDICT_TRUE((iflags0 & MFIB_ITF_FLAG_ACCEPT) ||
+                             (eflags0 & MFIB_ENTRY_FLAG_ACCEPT_ALL_ITF)))
+            {
+                /*
+                 * This interface is accepting packets for the matching entry
+                 */
+                next0 = mfe0->mfe_rep.dpoi_next_node;
+
+                vnet_buffer(b0)->ip.adj_index[VLIB_TX] =
+                    mfe0->mfe_rep.dpoi_index;
+            }
+            else
+            {
+                next0 = MFIB_FORWARD_RPF_NEXT_DROP;
+            }
+
+            if (b0->flags & VLIB_BUFFER_IS_TRACED)
+            {
+                mfib_forward_rpf_trace_t *t0;
+
+                t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
+                t0->entry_index = mfei0;
+                if (NULL == mfi0)
+                {
+                    t0->sw_if_index = ~0;
+                    t0->itf_flags = MFIB_ITF_FLAG_NONE;
+                }
+                else
+                {
+                    t0->sw_if_index = mfi0->mfi_sw_if_index;
+                    t0->itf_flags = mfi0->mfi_flags;
+                }
+            }
+            vlib_validate_buffer_enqueue_x1 (vm, node, next,
+                                             to_next, n_left_to_next,
+                                             pi0, next0);
+        }
+
+        vlib_put_next_frame(vm, node, next, n_left_to_next);
+    }
+
+    return frame->n_vectors;
+}
+
+static uword
+ip4_mfib_forward_rpf (vlib_main_t * vm,
+                      vlib_node_runtime_t * node,
+                      vlib_frame_t * frame)
+{
+    return (mfib_forward_rpf(vm, node, frame, 1));
+}
+
+
+VLIB_REGISTER_NODE (ip4_mfib_forward_rpf_node, static) = {
+    .function = ip4_mfib_forward_rpf,
+    .name = "ip4-mfib-forward-rpf",
+    .vector_size = sizeof (u32),
+
+    .format_trace = format_mfib_forward_rpf_trace,
+
+    .n_next_nodes = MFIB_FORWARD_RPF_N_NEXT,
+    .next_nodes = {
+        [MFIB_FORWARD_RPF_NEXT_DROP] = "error-drop",
+    },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_mfib_forward_rpf_node,
+                              ip4_mfib_forward_rpf)
+
+static uword
+ip6_mfib_forward_rpf (vlib_main_t * vm,
+                      vlib_node_runtime_t * node,
+                      vlib_frame_t * frame)
+{
+    return (mfib_forward_rpf(vm, node, frame, 1));
+}
+
+
+VLIB_REGISTER_NODE (ip6_mfib_forward_rpf_node, static) = {
+    .function = ip6_mfib_forward_rpf,
+    .name = "ip6-mfib-forward-rpf",
+    .vector_size = sizeof (u32),
+
+    .format_trace = format_mfib_forward_rpf_trace,
+
+    .n_next_nodes = MFIB_FORWARD_RPF_N_NEXT,
+    .next_nodes = {
+        [MFIB_FORWARD_RPF_NEXT_DROP] = "error-drop",
+    },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_mfib_forward_rpf_node,
+                              ip6_mfib_forward_rpf)
+
diff --git a/src/vnet/mfib/mfib_itf.c b/src/vnet/mfib/mfib_itf.c
new file mode 100644
index 00000000..b9fa1ec6
--- /dev/null
+++ b/src/vnet/mfib/mfib_itf.c
@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/vnet.h>
+
+#include <vnet/mfib/mfib_itf.h>
+#include <vnet/mfib/mfib_signal.h>
+
+mfib_itf_t *mfib_itf_pool;
+
+index_t
+mfib_itf_create (u32 sw_if_index,
+                 mfib_itf_flags_t mfi_flags)
+{
+    mfib_itf_t *mfib_itf;
+
+    pool_get_aligned(mfib_itf_pool, mfib_itf,
+                     CLIB_CACHE_LINE_BYTES);
+
+    mfib_itf->mfi_sw_if_index = sw_if_index;
+    mfib_itf->mfi_flags = mfi_flags;
+    mfib_itf->mfi_si = INDEX_INVALID;
+
+    return (mfib_itf - mfib_itf_pool);
+}
+
+void
+mfib_itf_delete (mfib_itf_t *mfi)
+{
+    mfib_signal_remove_itf(mfi);
+    pool_put(mfib_itf_pool, mfi);
+}
+
+u8 *
+format_mfib_itf (u8 * s, va_list * args)
+{
+    mfib_itf_t *mfib_itf;
+    vnet_main_t *vnm;
+    index_t mfi;
+
+    mfi = va_arg (*args, index_t);
+
+    vnm = vnet_get_main();
+    mfib_itf = mfib_itf_get(mfi);
+
+    if (~0 != mfib_itf->mfi_sw_if_index)
+    {
+        return (format(s, " %U: %U",
+                       format_vnet_sw_interface_name,
+                       vnm,
+                       vnet_get_sw_interface(vnm,
+                                             mfib_itf->mfi_sw_if_index),
+                       format_mfib_itf_flags, mfib_itf->mfi_flags));
+    }
+    else
+    {
+        return (format(s, " local: %U",
+                       format_mfib_itf_flags, mfib_itf->mfi_flags));
+    }
+    return (s);
+}
+
+static clib_error_t *
+show_mfib_itf_command (vlib_main_t * vm,
+                        unformat_input_t * input,
+                        vlib_cli_command_t * cmd)
+{
+    index_t mfii;
+
+    if (unformat (input, "%d", &mfii))
+    {
+        /*
+         * show one in detail
+         */
+        if (!pool_is_free_index(mfib_itf_pool, mfii))
+        {
+            vlib_cli_output (vm, "%d@%U",
+                             mfii,
+                             format_mfib_itf, mfii);
+        }
+        else
+        {
+            vlib_cli_output (vm, "itf %d invalid", mfii);
+        }
+    }
+    else
+    {
+        /*
+         * show all
+         */
+        vlib_cli_output (vm, "mFIB interfaces::");
+        pool_foreach_index(mfii, mfib_itf_pool,
+        ({
+            vlib_cli_output (vm, "%d@%U",
+                             mfii,
+                             format_mfib_itf, mfii);
+        }));
+    }
+
+    return (NULL);
+}
+
+VLIB_CLI_COMMAND (show_mfib_itf, static) = {
+  .path = "show mfib interface",
+  .function = show_mfib_itf_command,
+  .short_help = "show mfib interface",
+};
diff --git a/src/vnet/mfib/mfib_itf.h b/src/vnet/mfib/mfib_itf.h
new file mode 100644
index 00000000..5f26a476
--- /dev/null
+++ b/src/vnet/mfib/mfib_itf.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MFIB_ITF_H__
+#define __MFIB_ITF_H__
+
+#include <vlib/vlib.h>
+#include <vnet/mfib/mfib_types.h>
+
+/**
+ * @brief An interface associated with a particular MFIB entry
+ */
+typedef struct mfib_itf_t_
+{
+    /**
+     * @brief Falags on the entry
+     */
+    mfib_itf_flags_t mfi_flags;
+
+    /**
+     * The SW IF index that this MFIB interface represents
+     */
+    u32 mfi_sw_if_index;
+
+    /**
+     * The index of the signal in the pending list
+     */
+    u32 mfi_si;
+} mfib_itf_t;
+
+
+extern index_t mfib_itf_create(u32 sw_if_index,
+                               mfib_itf_flags_t mfi_flags);
+extern void mfib_itf_delete(mfib_itf_t *mfi);
+
+extern u8 *format_mfib_itf(u8 * s, va_list * args);
+
+extern mfib_itf_t *mfib_itf_pool;
+
+static inline mfib_itf_t *
+mfib_itf_get (index_t mi)
+{
+    return (pool_elt_at_index(mfib_itf_pool, mi));
+}
+static inline index_t
+mfib_itf_get_index (const mfib_itf_t *mfi)
+{
+    return (mfi - mfib_itf_pool);
+}
+
+#endif
diff --git a/src/vnet/mfib/mfib_signal.c b/src/vnet/mfib/mfib_signal.c
new file mode 100644
index 00000000..9f6205de
--- /dev/null
+++ b/src/vnet/mfib/mfib_signal.c
@@ -0,0 +1,201 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include <vnet/vnet.h>
+#include <vnet/mfib/mfib_signal.h>
+#include <vppinfra/dlist.h>
+
+/**
+ * @brief Pool of signals
+ */
+static mfib_signal_t *mfib_signal_pool;
+
+/**
+ * @brief pool of dlist elements
+ */
+static dlist_elt_t *mfib_signal_dlist_pool;
+
+/**
+ * the list/set of interfaces with signals pending
+ */
+typedef struct mfib_signal_q_t_
+{
+    /**
+     * the dlist indext that is the head of the list
+     */
+    u32 mip_head;
+
+    /**
+     * Spin lock to protect the list
+     */
+    int mip_lock;
+} mfib_signal_q_t;
+
+/**
+ * @brief The pending queue of signals to deliver to the control plane
+ */
+static mfib_signal_q_t mfib_signal_pending ;
+
+static void
+mfib_signal_list_init (void)
+{
+    dlist_elt_t *head;
+    u32 hi;
+
+    pool_get(mfib_signal_dlist_pool, head);
+    hi = head - mfib_signal_dlist_pool;
+
+    mfib_signal_pending.mip_head = hi;
+    clib_dlist_init(mfib_signal_dlist_pool, hi);
+}
+
+void
+mfib_signal_module_init (void)
+{
+    mfib_signal_list_init();
+}
+
+int
+mfib_signal_send_one (struct _unix_shared_memory_queue *q,
+                      u32 context)
+{
+    u32 li, si;
+
+    /*
+     * with the lock held, pop a signal from the q.
+     */
+    while (__sync_lock_test_and_set (&mfib_signal_pending.mip_lock, 1))
+        ;
+    {
+        li = clib_dlist_remove_head(mfib_signal_dlist_pool,
+                                    mfib_signal_pending.mip_head);
+    }
+    mfib_signal_pending.mip_lock = 0;
+
+    if (~0 != li)
+    {
+        mfib_signal_t *mfs;
+        mfib_itf_t *mfi;
+        dlist_elt_t *elt;
+
+        elt = pool_elt_at_index(mfib_signal_dlist_pool, li);
+        si = elt->value;
+
+        mfs = pool_elt_at_index(mfib_signal_pool, si);
+        mfi = mfib_itf_get(mfs->mfs_itf);
+        mfi->mfi_si = INDEX_INVALID;
+        __sync_fetch_and_and(&mfi->mfi_flags,
+                             ~MFIB_ITF_FLAG_SIGNAL_PRESENT);
+
+
+        vl_mfib_signal_send_one(q, context, mfs);
+
+        /*
+         * with the lock held, return the resoruces of the signals posted
+         */
+        while (__sync_lock_test_and_set(&mfib_signal_pending.mip_lock, 1))
+            ;
+        {
+            pool_put_index(mfib_signal_pool, si);
+            pool_put_index(mfib_signal_dlist_pool, li);
+        }
+        mfib_signal_pending.mip_lock = 0;
+
+        return (1);
+    }
+    return (0);
+}
+
+void
+mfib_signal_push (const mfib_entry_t *mfe,
+                  mfib_itf_t *mfi,
+                  vlib_buffer_t *b0)
+{
+    mfib_signal_t *mfs;
+    dlist_elt_t *elt;
+    u32 si, li;
+
+    while (__sync_lock_test_and_set (&mfib_signal_pending.mip_lock, 1))
+        ;
+    {
+        pool_get(mfib_signal_pool, mfs);
+        pool_get(mfib_signal_dlist_pool, elt);
+
+        si = mfs - mfib_signal_pool;
+        li = elt - mfib_signal_dlist_pool;
+
+        elt->value = si;
+        mfi->mfi_si = li;
+
+        clib_dlist_addhead(mfib_signal_dlist_pool,
+                           mfib_signal_pending.mip_head,
+                           li);
+    }
+    mfib_signal_pending.mip_lock = 0;
+
+    mfs->mfs_entry = mfib_entry_get_index(mfe);
+    mfs->mfs_itf = mfib_itf_get_index(mfi);
+
+    if (NULL != b0)
+    {
+        mfs->mfs_buffer_len = b0->current_length;
+        memcpy(mfs->mfs_buffer,
+               vlib_buffer_get_current(b0),
+               (mfs->mfs_buffer_len > MFIB_SIGNAL_BUFFER_SIZE ?
+                MFIB_SIGNAL_BUFFER_SIZE :
+                mfs->mfs_buffer_len));
+    }
+    else
+    {
+        mfs->mfs_buffer_len = 0;
+    }
+}
+
+void
+mfib_signal_remove_itf (const mfib_itf_t *mfi)
+{
+    u32 li;
+
+    /*
+     * lock the queue to prevent further additions while we fiddle.
+     */
+    li = mfi->mfi_si;
+
+    if (INDEX_INVALID != li)
+    {
+        /*
+         * it's in the pending q
+         */
+        while (__sync_lock_test_and_set (&mfib_signal_pending.mip_lock, 1))
+            ;
+        {
+            dlist_elt_t *elt;
+
+            /*
+             * with the lock held;
+             *  - remove the signal from the pending list
+             *  - free up the signal and list entry obejcts
+             */
+            clib_dlist_remove(mfib_signal_dlist_pool, li);
+
+            elt = pool_elt_at_index(mfib_signal_dlist_pool, li);
+            pool_put_index(mfib_signal_pool, elt->value);
+            pool_put(mfib_signal_dlist_pool, elt);
+        }
+
+        mfib_signal_pending.mip_lock = 0;
+    }
+}
diff --git a/src/vnet/mfib/mfib_signal.h b/src/vnet/mfib/mfib_signal.h
new file mode 100644
index 00000000..732d8aff
--- /dev/null
+++ b/src/vnet/mfib/mfib_signal.h
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MFIB_SIGNAL_H__
+#define __MFIB_SIGNAL_H__
+
+#include <vlib/vlib.h>
+#include <vnet/mfib/mfib_types.h>
+#include <vnet/mfib/mfib_itf.h>
+#include <vnet/mfib/mfib_entry.h>
+
+#define MFIB_SIGNAL_BUFFER_SIZE 255
+
+/**
+ * A pair of indicies, for the entry and interface resp.
+ */
+typedef struct mfib_signal_t_
+{
+    fib_node_index_t mfs_entry;
+    index_t mfs_itf;
+
+    /**
+     * @brief A buffer copied from the DP plane that triggered the signal
+     */
+    u8 mfs_buffer[MFIB_SIGNAL_BUFFER_SIZE];
+
+    u8 mfs_buffer_len;
+} mfib_signal_t;
+
+
+extern void mfib_signal_push(const mfib_entry_t *mfe,
+                             mfib_itf_t *mfi,
+                             vlib_buffer_t *b0);
+extern void mfib_signal_remove_itf(const mfib_itf_t *mfi);
+
+extern void mfib_signal_module_init(void);
+
+struct _unix_shared_memory_queue;
+
+extern void vl_mfib_signal_send_one(struct _unix_shared_memory_queue *q,
+                                    u32 context,
+                                    const mfib_signal_t *mfs);
+extern int mfib_signal_send_one(struct _unix_shared_memory_queue *q,
+                                u32 context);
+
+#endif
+
diff --git a/src/vnet/mfib/mfib_table.c b/src/vnet/mfib/mfib_table.c
new file mode 100644
index 00000000..e4c0936d
--- /dev/null
+++ b/src/vnet/mfib/mfib_table.c
@@ -0,0 +1,489 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vlib/vlib.h>
+#include <vnet/dpo/drop_dpo.h>
+
+#include <vnet/mfib/mfib_table.h>
+#include <vnet/mfib/ip4_mfib.h>
+#include <vnet/mfib/ip6_mfib.h>
+#include <vnet/mfib/mfib_entry.h>
+#include <vnet/mfib/mfib_signal.h>
+
+mfib_table_t *
+mfib_table_get (fib_node_index_t index,
+                fib_protocol_t proto)
+{
+    switch (proto)
+    {
+    case FIB_PROTOCOL_IP4:
+        return (pool_elt_at_index(ip4_main.mfibs, index));
+    case FIB_PROTOCOL_IP6:
+        return (pool_elt_at_index(ip6_main.mfibs, index));
+    case FIB_PROTOCOL_MPLS:
+        break;
+    }
+    ASSERT(0);
+    return (NULL);
+}
+
+static inline fib_node_index_t
+mfib_table_lookup_i (const mfib_table_t *mfib_table,
+                     const mfib_prefix_t *prefix)
+{
+    switch (prefix->fp_proto)
+    {
+    case FIB_PROTOCOL_IP4:
+        return (ip4_mfib_table_lookup(&mfib_table->v4,
+                                      &prefix->fp_src_addr.ip4,
+                                      &prefix->fp_grp_addr.ip4,
+                                      prefix->fp_len));
+    case FIB_PROTOCOL_IP6:
+        return (ip6_mfib_table_lookup(&mfib_table->v6,
+                                      &prefix->fp_src_addr.ip6,
+                                      &prefix->fp_grp_addr.ip6,
+                                      prefix->fp_len));
+    case FIB_PROTOCOL_MPLS:
+        break;
+    }
+    return (FIB_NODE_INDEX_INVALID);
+}
+
+fib_node_index_t
+mfib_table_lookup (u32 fib_index,
+                   const mfib_prefix_t *prefix)
+{
+    return (mfib_table_lookup_i(mfib_table_get(fib_index, prefix->fp_proto), prefix));
+}
+
+static inline fib_node_index_t
+mfib_table_lookup_exact_match_i (const mfib_table_t *mfib_table,
+                                 const mfib_prefix_t *prefix)
+{
+    switch (prefix->fp_proto)
+    {
+    case FIB_PROTOCOL_IP4:
+        return (ip4_mfib_table_lookup_exact_match(&mfib_table->v4,
+                                                  &prefix->fp_grp_addr.ip4,
+                                                  &prefix->fp_src_addr.ip4,
+                                                  prefix->fp_len));
+    case FIB_PROTOCOL_IP6:
+        return (ip6_mfib_table_lookup_exact_match(&mfib_table->v6,
+                                                  &prefix->fp_grp_addr.ip6,
+                                                  &prefix->fp_src_addr.ip6,
+                                                  prefix->fp_len));
+    case FIB_PROTOCOL_MPLS:
+        break;
+    }
+    return (FIB_NODE_INDEX_INVALID);
+}
+
+fib_node_index_t
+mfib_table_lookup_exact_match (u32 fib_index,
+                              const mfib_prefix_t *prefix)
+{
+    return (mfib_table_lookup_exact_match_i(mfib_table_get(fib_index,
+                                                          prefix->fp_proto),
+                                            prefix));
+}
+
+static void
+mfib_table_entry_remove (mfib_table_t *mfib_table,
+                         const mfib_prefix_t *prefix,
+                         fib_node_index_t fib_entry_index)
+{
+    vlib_smp_unsafe_warning();
+
+    mfib_table->mft_total_route_counts--;
+
+    switch (prefix->fp_proto)
+    {
+    case FIB_PROTOCOL_IP4:
+        ip4_mfib_table_entry_remove(&mfib_table->v4,
+                                    &prefix->fp_grp_addr.ip4,
+                                    &prefix->fp_src_addr.ip4,
+                                    prefix->fp_len);
+        break;
+    case FIB_PROTOCOL_IP6:
+        ip6_mfib_table_entry_remove(&mfib_table->v6,
+                                    &prefix->fp_grp_addr.ip6,
+                                    &prefix->fp_src_addr.ip6,
+                                    prefix->fp_len);
+        break;
+    case FIB_PROTOCOL_MPLS:
+        ASSERT(0);
+        break;
+    }
+
+    mfib_entry_unlock(fib_entry_index);
+}
+
+static void
+mfib_table_entry_insert (mfib_table_t *mfib_table,
+                         const mfib_prefix_t *prefix,
+                         fib_node_index_t mfib_entry_index)
+{
+    vlib_smp_unsafe_warning();
+
+    mfib_entry_lock(mfib_entry_index);
+    mfib_table->mft_total_route_counts++;
+
+    switch (prefix->fp_proto)
+    {
+    case FIB_PROTOCOL_IP4:
+        ip4_mfib_table_entry_insert(&mfib_table->v4,
+                                    &prefix->fp_grp_addr.ip4,
+                                    &prefix->fp_src_addr.ip4,
+                                    prefix->fp_len,
+                                    mfib_entry_index);
+        break;
+    case FIB_PROTOCOL_IP6:
+        ip6_mfib_table_entry_insert(&mfib_table->v6,
+                                    &prefix->fp_grp_addr.ip6,
+                                    &prefix->fp_src_addr.ip6,
+                                    prefix->fp_len,
+                                    mfib_entry_index);
+        break;
+    case FIB_PROTOCOL_MPLS:
+        break;
+    }
+}
+
+fib_node_index_t
+mfib_table_entry_update (u32 fib_index,
+                         const mfib_prefix_t *prefix,
+                         mfib_source_t source,
+                         mfib_entry_flags_t entry_flags)
+{
+    fib_node_index_t mfib_entry_index;
+    mfib_table_t *mfib_table;
+
+    mfib_table = mfib_table_get(fib_index, prefix->fp_proto);
+    mfib_entry_index = mfib_table_lookup_exact_match_i(mfib_table, prefix);
+
+    if (FIB_NODE_INDEX_INVALID == mfib_entry_index)
+    {
+        if (MFIB_ENTRY_FLAG_NONE != entry_flags)
+        {
+            /*
+             * update to a non-existing entry with non-zero flags
+             */
+            mfib_entry_index = mfib_entry_create(fib_index, source,
+                                                 prefix, entry_flags);
+
+            mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index);
+        }
+        /*
+         * else
+         *   the entry doesn't exist and the request is to set no flags
+         *   the result would be an entry that doesn't exist - so do nothing
+         */
+    }
+    else
+    {
+        mfib_entry_lock(mfib_entry_index);
+
+        if (mfib_entry_update(mfib_entry_index, source, entry_flags))
+        {
+            /*
+             * this update means we can now remove the entry.
+             */
+            mfib_table_entry_remove(mfib_table, prefix, mfib_entry_index);
+        }
+
+        mfib_entry_unlock(mfib_entry_index);
+    }
+
+    return (mfib_entry_index);
+}
+
+fib_node_index_t
+mfib_table_entry_path_update (u32 fib_index,
+                              const mfib_prefix_t *prefix,
+                              mfib_source_t source,
+                              const fib_route_path_t *rpath,
+                              mfib_itf_flags_t itf_flags)
+{
+    fib_node_index_t mfib_entry_index;
+    mfib_table_t *mfib_table;
+
+    mfib_table = mfib_table_get(fib_index, prefix->fp_proto);
+    mfib_entry_index = mfib_table_lookup_exact_match_i(mfib_table, prefix);
+
+    if (FIB_NODE_INDEX_INVALID == mfib_entry_index)
+    {
+        mfib_entry_index = mfib_entry_create(fib_index,
+                                             source,
+                                             prefix,
+                                             MFIB_ENTRY_FLAG_NONE);
+
+        mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index);
+    }
+
+    mfib_entry_path_update(mfib_entry_index,
+                           source,
+                           rpath,
+                           itf_flags);
+
+    return (mfib_entry_index);
+}
+
+void
+mfib_table_entry_path_remove (u32 fib_index,
+                              const mfib_prefix_t *prefix,
+                              mfib_source_t source,
+                              const fib_route_path_t *rpath)
+{
+    fib_node_index_t mfib_entry_index;
+    mfib_table_t *mfib_table;
+
+    mfib_table = mfib_table_get(fib_index, prefix->fp_proto);
+    mfib_entry_index = mfib_table_lookup_exact_match_i(mfib_table, prefix);
+
+    if (FIB_NODE_INDEX_INVALID == mfib_entry_index)
+    {
+        /*
+         * removing an etry that does not exist. i'll allow it.
+         */
+    }
+    else
+    {
+        int no_more_sources;
+
+        /*
+         * don't nobody go nowhere
+         */
+        mfib_entry_lock(mfib_entry_index);
+
+        no_more_sources = mfib_entry_path_remove(mfib_entry_index,
+                                                 source,
+                                                 rpath);
+
+        if (no_more_sources)
+        {
+            /*
+             * last source gone. remove from the table
+             */
+            mfib_table_entry_remove(mfib_table, prefix, mfib_entry_index);
+        }
+
+        mfib_entry_unlock(mfib_entry_index);
+    }
+}
+
+static void
+mfib_table_entry_delete_i (u32 fib_index,
+                           fib_node_index_t mfib_entry_index,
+                           const mfib_prefix_t *prefix,
+                           mfib_source_t source)
+{
+    mfib_table_t *mfib_table;
+
+    mfib_table = mfib_table_get(fib_index, prefix->fp_proto);
+
+    /*
+     * don't nobody go nowhere
+     */
+    mfib_entry_lock(mfib_entry_index);
+
+    if (mfib_entry_delete(mfib_entry_index, source))
+    {
+        /*
+         * last source gone. remove from the table
+         */
+        mfib_table_entry_remove(mfib_table, prefix, mfib_entry_index);
+    }
+    /*
+     * else
+     *   still has sources, leave it be.
+     */
+
+    mfib_entry_unlock(mfib_entry_index);
+}
+
+void
+mfib_table_entry_delete (u32 fib_index,
+                         const mfib_prefix_t *prefix,
+                         mfib_source_t source)
+{
+    fib_node_index_t mfib_entry_index;
+
+    mfib_entry_index = mfib_table_lookup_exact_match(fib_index, prefix);
+
+    if (FIB_NODE_INDEX_INVALID == mfib_entry_index)
+    {
+        /*
+         * removing an etry that does not exist.
+         * i'll allow it, but i won't like it.
+         */
+        clib_warning("%U not in FIB", format_mfib_prefix, prefix);
+    }
+    else
+    {
+        mfib_table_entry_delete_i(fib_index, mfib_entry_index,
+                                  prefix, source);
+    }
+}
+
+void
+mfib_table_entry_delete_index (fib_node_index_t mfib_entry_index,
+                               mfib_source_t source)
+{
+    mfib_prefix_t prefix;
+
+    mfib_entry_get_prefix(mfib_entry_index, &prefix);
+
+    mfib_table_entry_delete_i(mfib_entry_get_fib_index(mfib_entry_index),
+                              mfib_entry_index, &prefix, source);
+}
+
+u32
+mfib_table_get_index_for_sw_if_index (fib_protocol_t proto,
+                                      u32 sw_if_index)
+{
+    switch (proto)
+    {
+    case FIB_PROTOCOL_IP4:
+        return (ip4_mfib_table_get_index_for_sw_if_index(sw_if_index));
+    case FIB_PROTOCOL_IP6:
+        return (ip6_mfib_table_get_index_for_sw_if_index(sw_if_index));
+    case FIB_PROTOCOL_MPLS:
+        ASSERT(0);
+        break;
+    }
+    return (~0);
+}
+
+u32
+mfib_table_find (fib_protocol_t proto,
+                 u32 table_id)
+{
+    switch (proto)
+    {
+    case FIB_PROTOCOL_IP4:
+        return (ip4_mfib_index_from_table_id(table_id));
+    case FIB_PROTOCOL_IP6:
+        return (ip6_mfib_index_from_table_id(table_id));
+    case FIB_PROTOCOL_MPLS:
+        ASSERT(0);
+        break;
+    }
+    return (~0);
+}
+
+u32
+mfib_table_find_or_create_and_lock (fib_protocol_t proto,
+                                    u32 table_id)
+{
+    mfib_table_t *mfib_table;
+    fib_node_index_t fi;
+
+    switch (proto)
+    {
+    case FIB_PROTOCOL_IP4:
+        fi = ip4_mfib_table_find_or_create_and_lock(table_id);
+        break;
+    case FIB_PROTOCOL_IP6:
+        fi = ip6_mfib_table_find_or_create_and_lock(table_id);
+        break;
+    case FIB_PROTOCOL_MPLS:
+    default:
+        return (~0);
+    }
+
+    mfib_table = mfib_table_get(fi, proto);
+
+    mfib_table->mft_desc = format(NULL, "%U-VRF:%d",
+                                  format_fib_protocol, proto,
+                                  table_id);
+
+    return (fi);
+}
+
+static void
+mfib_table_destroy (mfib_table_t *mfib_table)
+{
+    vec_free(mfib_table->mft_desc);
+
+    switch (mfib_table->mft_proto)
+    {
+    case FIB_PROTOCOL_IP4:
+        ip4_mfib_table_destroy(&mfib_table->v4);
+        break;
+    case FIB_PROTOCOL_IP6:
+        ip6_mfib_table_destroy(&mfib_table->v6);
+        break;
+    case FIB_PROTOCOL_MPLS:
+        ASSERT(0);
+        break;
+    }
+}
+
+void
+mfib_table_unlock (u32 fib_index,
+                   fib_protocol_t proto)
+{
+    mfib_table_t *mfib_table;
+
+    mfib_table = mfib_table_get(fib_index, proto);
+    mfib_table->mft_locks--;
+
+    if (0 == mfib_table->mft_locks)
+    {
+        mfib_table_destroy(mfib_table);
+    }
+}
+
+void
+mfib_table_lock (u32 fib_index,
+                 fib_protocol_t proto)
+{
+    mfib_table_t *mfib_table;
+
+    mfib_table = mfib_table_get(fib_index, proto);
+    mfib_table->mft_locks++;
+}
+
+u8*
+format_mfib_table_name (u8* s, va_list ap)
+{
+    fib_node_index_t fib_index = va_arg(ap, fib_node_index_t);
+    fib_protocol_t proto = va_arg(ap, int); // int promotion
+    mfib_table_t *mfib_table;
+
+    mfib_table = mfib_table_get(fib_index, proto);
+
+    s = format(s, "%v", mfib_table->mft_desc);
+
+    return (s);
+}
+
+static clib_error_t *
+mfib_module_init (vlib_main_t * vm)
+{
+    clib_error_t * error;
+
+    if ((error = vlib_call_init_function (vm, fib_module_init)))
+        return (error);
+    if ((error = vlib_call_init_function (vm, rn_module_init)))
+        return (error);
+
+    mfib_entry_module_init();
+    mfib_signal_module_init();
+
+    return (error);
+}
+
+VLIB_INIT_FUNCTION(mfib_module_init);
diff --git a/src/vnet/mfib/mfib_table.h b/src/vnet/mfib/mfib_table.h
new file mode 100644
index 00000000..4faa69ee
--- /dev/null
+++ b/src/vnet/mfib/mfib_table.h
@@ -0,0 +1,331 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MFIB_TABLE_H__
+#define __MFIB_TABLE_H__
+
+#include <vnet/ip/ip.h>
+#include <vnet/adj/adj.h>
+
+#include <vnet/mfib/mfib_types.h>
+
+/**
+ * @brief
+ *   A protocol Independent IP multicast FIB table
+ */
+typedef struct mfib_table_t_
+{
+    /**
+     * A union of the protocol specific FIBs that provide the
+     * underlying LPM mechanism.
+     * This element is first in the struct so that it is in the
+     * first cache line.
+     */
+    union {
+        ip4_mfib_t v4;
+        ip6_mfib_t v6;
+    };
+
+    /**
+     * Which protocol this table serves. Used to switch on the union above.
+     */
+    fib_protocol_t mft_proto;
+
+    /**
+     * number of locks on the table
+     */
+    u16 mft_locks;
+
+    /**
+     * Table ID (hash key) for this FIB.
+     */
+    u32 mft_table_id;
+
+    /**
+     * Index into FIB vector.
+     */
+    fib_node_index_t mft_index;
+
+    /**
+     * Total route counters
+     */
+    u32 mft_total_route_counts;
+
+    /**
+     * Table description
+     */
+    u8* mft_desc;
+} mfib_table_t;
+
+/**
+ * @brief
+ *  Format the description/name of the table
+ */
+extern u8* format_mfib_table_name(u8* s, va_list ap);
+
+/**
+ * @brief
+ *  Perfom a longest prefix match in the non-forwarding table
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @param prefix
+ *  The prefix to lookup
+ *
+ * @return
+ *  The index of the fib_entry_t for the best match, which may be the default route
+ */
+extern fib_node_index_t mfib_table_lookup(u32 fib_index,
+                                         const mfib_prefix_t *prefix);
+
+/**
+ * @brief
+ *  Perfom an exact match in the non-forwarding table
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @param prefix
+ *  The prefix to lookup
+ *
+ * @return
+ *  The index of the fib_entry_t for the exact match, or INVALID
+ *  is there is no match.
+ */
+extern fib_node_index_t mfib_table_lookup_exact_match(u32 fib_index,
+                                                      const mfib_prefix_t *prefix);
+
+/**
+ * @brief
+ * Add a new (with no replication) or lock an existing entry
+ *
+ * @param prefix
+ *  The prefix for the entry to add
+ *
+ * @return
+ *  the index of the fib_entry_t that is created (or existed already).
+ */
+extern fib_node_index_t mfib_table_entry_update(u32 fib_index,
+                                                const mfib_prefix_t *prefix,
+                                                mfib_source_t source,
+                                                mfib_entry_flags_t flags);
+
+/**
+ * @brief
+ *  Add n paths to an entry (aka route) in the FIB. If the entry does not
+ *  exist, it will be created.
+ * See the documentation for fib_route_path_t for more descirptions of
+ * the path parameters.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @param prefix
+ *  The prefix for the entry to add
+ *
+ * @param source
+ *  The ID of the client/source adding the entry.
+ *
+ * @param flags
+ *  Flags for the entry.
+ *
+ * @param rpaths
+ *  A vector of paths.
+ *
+ * @return
+ *  the index of the fib_entry_t that is created (or existed already).
+ */
+extern fib_node_index_t mfib_table_entry_path_update(u32 fib_index,
+                                                     const mfib_prefix_t *prefix,
+                                                     mfib_source_t source,
+                                                     const fib_route_path_t *rpath,
+                                                     mfib_itf_flags_t flags);
+
+/**
+ * @brief
+ * Remove n paths to an entry (aka route) in the FIB. If this is the entry's
+ * last path, then the entry will be removed, unless it has other sources.
+ * See the documentation for fib_route_path_t for more descirptions of
+ * the path parameters.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @param prefix
+ *  The prefix for the entry to add
+ *
+ * @param source
+ *  The ID of the client/source adding the entry.
+ *
+ * @param rpaths
+ *  A vector of paths.
+ */
+extern void mfib_table_entry_path_remove(u32 fib_index,
+                                         const mfib_prefix_t *prefix,
+                                         mfib_source_t source,
+                                         const fib_route_path_t *paths);
+
+
+
+/**
+ * @brief
+ *  Delete a FIB entry. If the entry has no more sources, then it is
+ * removed from the table.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @param prefix
+ *  The prefix for the entry to remove
+ *
+ * @param source
+ *  The ID of the client/source adding the entry.
+ */
+extern void mfib_table_entry_delete(u32 fib_index,
+                                    const mfib_prefix_t *prefix,
+                                    mfib_source_t source);
+
+/**
+ * @brief
+ *  Delete a FIB entry. If the entry has no more sources, then it is
+ * removed from the table.
+ *
+ * @param entry_index
+ *  The index of the FIB entry
+ *
+ * @param source
+ *  The ID of the client/source adding the entry.
+ */
+extern void mfib_table_entry_delete_index(fib_node_index_t entry_index,
+                                          mfib_source_t source);
+
+/**
+ * @brief
+ *  Flush all entries from a table for the source
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @paran proto
+ *  The protocol of the entries in the table
+ *
+ * @param source
+ *  the source to flush
+ */
+extern void mfib_table_flush(u32 fib_index,
+                             fib_protocol_t proto);
+
+/**
+ * @brief
+ *  Get the index of the FIB bound to the interface
+ *
+ * @paran proto
+ *  The protocol of the FIB (and thus the entries therein)
+ *
+ * @param sw_if_index
+ *  The interface index
+ *
+ * @return fib_index
+ *  The index of the FIB
+ */
+extern u32 mfib_table_get_index_for_sw_if_index(fib_protocol_t proto,
+                                                u32 sw_if_index);
+
+/**
+ * @brief
+ *  Get the index of the FIB for a Table-ID. This DOES NOT create the
+ * FIB if it does not exist.
+ *
+ * @paran proto
+ *  The protocol of the FIB (and thus the entries therein)
+ *
+ * @param table-id
+ *  The Table-ID
+ *
+ * @return fib_index
+ *  The index of the FIB, which may be INVALID.
+ */
+extern u32 mfib_table_find(fib_protocol_t proto, u32 table_id);
+
+
+/**
+ * @brief
+ *  Get the index of the FIB for a Table-ID. This DOES create the
+ * FIB if it does not exist.
+ *
+ * @paran proto
+ *  The protocol of the FIB (and thus the entries therein)
+ *
+ * @param table-id
+ *  The Table-ID
+ *
+ * @return fib_index
+ *  The index of the FIB
+ */
+extern u32 mfib_table_find_or_create_and_lock(fib_protocol_t proto,
+                                              u32 table_id);
+
+
+/**
+ * @brief
+ * Take a reference counting lock on the table
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @paran proto
+ *  The protocol of the FIB (and thus the entries therein)
+ */
+extern void mfib_table_unlock(u32 fib_index,
+                              fib_protocol_t proto);
+
+/**
+ * @brief
+ * Release a reference counting lock on the table. When the last lock
+ * has gone. the FIB is deleted.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @paran proto
+ *  The protocol of the FIB (and thus the entries therein)
+ */
+extern void mfib_table_lock(u32 fib_index,
+                            fib_protocol_t proto);
+
+/**
+ * @brief
+ * Return the number of entries in the FIB added by a given source.
+ *
+ * @param fib_index
+ *  The index of the FIB
+ *
+ * @paran proto
+ *  The protocol of the FIB (and thus the entries therein)
+ *
+ * @return number of sourced entries.
+ */
+extern u32 mfib_table_get_num_entries(u32 fib_index,
+                                      fib_protocol_t proto);
+
+/**
+ * @brief
+ * Get a pointer to a FIB table
+ */
+extern mfib_table_t *mfib_table_get(fib_node_index_t index,
+                                    fib_protocol_t proto);
+
+#endif
diff --git a/src/vnet/mfib/mfib_test.c b/src/vnet/mfib/mfib_test.c
new file mode 100644
index 00000000..8735bfa7
--- /dev/null
+++ b/src/vnet/mfib/mfib_test.c
@@ -0,0 +1,1225 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/mpls/mpls_types.h>
+
+#include <vnet/mfib/mfib_table.h>
+#include <vnet/mfib/mfib_entry.h>
+#include <vnet/mfib/mfib_signal.h>
+#include <vnet/mfib/ip6_mfib.h>
+
+#include <vnet/dpo/replicate_dpo.h>
+#include <vnet/adj/adj_mcast.h>
+
+#define MFIB_TEST_I(_cond, _comment, _args...)			\
+({								\
+    int _evald = (_cond);					\
+    if (!(_evald)) {						\
+        fformat(stderr, "FAIL:%d: " _comment "\n",		\
+                __LINE__, ##_args);				\
+    } else {							\
+        fformat(stderr, "PASS:%d: " _comment "\n",		\
+                __LINE__, ##_args);				\
+    }								\
+    _evald;							\
+})
+#define MFIB_TEST(_cond, _comment, _args...)			\
+{								\
+    if (!MFIB_TEST_I(_cond, _comment, ##_args)) {		\
+        return 1;\
+        ASSERT(!("FAIL: " _comment));				\
+    }								\
+}
+#define MFIB_TEST_NS(_cond)                                     \
+{								\
+    if (!MFIB_TEST_I(_cond, "")) {                              \
+        return 1;\
+        ASSERT(!("FAIL: "));                                    \
+    }								\
+}
+
+/**
+ * A 'i'm not fussed is this is not efficient' store of test data
+ */
+typedef struct test_main_t_ {
+    /**
+     * HW if indicies
+     */
+    u32 hw_if_indicies[4];
+    /**
+     * HW interfaces
+     */
+    vnet_hw_interface_t * hw[4];
+
+} test_main_t;
+static test_main_t test_main;
+
+/* fake ethernet device class, distinct from "fake-ethX" */
+static u8 * format_test_interface_name (u8 * s, va_list * args)
+{
+  u32 dev_instance = va_arg (*args, u32);
+  return format (s, "test-eth%d", dev_instance);
+}
+
+static uword dummy_interface_tx (vlib_main_t * vm,
+                                 vlib_node_runtime_t * node,
+                                 vlib_frame_t * frame)
+{
+  clib_warning ("you shouldn't be here, leaking buffers...");
+  return frame->n_vectors;
+}
+
+static clib_error_t *
+test_interface_admin_up_down (vnet_main_t * vnm,
+                              u32 hw_if_index,
+                              u32 flags)
+{
+  u32 hw_flags = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ?
+    VNET_HW_INTERFACE_FLAG_LINK_UP : 0;
+  vnet_hw_interface_set_flags (vnm, hw_if_index, hw_flags);
+  return 0;
+}
+
+VNET_DEVICE_CLASS (test_interface_device_class,static) = {
+  .name = "Test interface",
+  .format_device_name = format_test_interface_name,
+  .tx_function = dummy_interface_tx,
+  .admin_up_down_function = test_interface_admin_up_down,
+};
+
+static u8 *hw_address;
+
+static int
+mfib_test_mk_intf (u32 ninterfaces)
+{
+    clib_error_t * error = NULL;
+    test_main_t *tm = &test_main;
+    u8 byte;
+    u32 i;
+
+    ASSERT(ninterfaces <= ARRAY_LEN(tm->hw_if_indicies));
+
+    for (i=0; i<6; i++)
+    {
+        byte = 0xd0+i;
+        vec_add1(hw_address, byte);
+    }
+
+    for (i = 0; i < ninterfaces; i++)
+    {
+        hw_address[5] = i;
+
+        error = ethernet_register_interface(vnet_get_main(),
+                                            test_interface_device_class.index,
+                                            i /* instance */,
+                                            hw_address,
+                                            &tm->hw_if_indicies[i],
+                                            /* flag change */ 0);
+
+        MFIB_TEST((NULL == error), "ADD interface %d", i);
+
+        error = vnet_hw_interface_set_flags(vnet_get_main(),
+                                            tm->hw_if_indicies[i],
+                                            VNET_HW_INTERFACE_FLAG_LINK_UP);
+        tm->hw[i] = vnet_get_hw_interface(vnet_get_main(),
+                                          tm->hw_if_indicies[i]);
+        vec_validate (ip4_main.fib_index_by_sw_if_index,
+                      tm->hw[i]->sw_if_index);
+        vec_validate (ip6_main.fib_index_by_sw_if_index,
+                      tm->hw[i]->sw_if_index);
+        ip4_main.fib_index_by_sw_if_index[tm->hw[i]->sw_if_index] = 0;
+        ip6_main.fib_index_by_sw_if_index[tm->hw[i]->sw_if_index] = 0;
+
+        vec_validate (ip4_main.mfib_index_by_sw_if_index,
+                      tm->hw[i]->sw_if_index);
+        vec_validate (ip6_main.mfib_index_by_sw_if_index,
+                      tm->hw[i]->sw_if_index);
+        ip4_main.mfib_index_by_sw_if_index[tm->hw[i]->sw_if_index] = 0;
+        ip6_main.mfib_index_by_sw_if_index[tm->hw[i]->sw_if_index] = 0;
+
+        error = vnet_sw_interface_set_flags(vnet_get_main(),
+                                            tm->hw[i]->sw_if_index,
+                                            VNET_SW_INTERFACE_FLAG_ADMIN_UP);
+        MFIB_TEST((NULL == error), "UP interface %d", i);
+    }
+    /*
+     * re-eval after the inevitable realloc
+     */
+    for (i = 0; i < ninterfaces; i++)
+    {
+        tm->hw[i] = vnet_get_hw_interface(vnet_get_main(),
+                                          tm->hw_if_indicies[i]);
+    }
+
+    return (0);
+}
+
+#define MFIB_TEST_REP(_cond, _comment, _args...)		\
+{								\
+    if (!MFIB_TEST_I(_cond, _comment, ##_args)) {		\
+        return (0);						\
+    }								\
+}
+
+static int
+mfib_test_validate_rep_v (const replicate_t *rep,
+                          u16 n_buckets,
+                          va_list ap)
+{
+    const dpo_id_t *dpo;
+    adj_index_t ai;
+    dpo_type_t dt;
+    int bucket;
+
+    MFIB_TEST_REP((n_buckets == rep->rep_n_buckets),
+                  "n_buckets = %d", rep->rep_n_buckets);
+
+    for (bucket = 0; bucket < n_buckets; bucket++)
+    {
+        dt = va_arg(ap, int);  // type promotion
+        ai = va_arg(ap, adj_index_t);
+        dpo = replicate_get_bucket_i(rep, bucket);
+
+        MFIB_TEST_REP((dt == dpo->dpoi_type),
+                      "bucket %d stacks on %U",
+                      bucket,
+                      format_dpo_type, dpo->dpoi_type);
+
+        if (DPO_RECEIVE != dt)
+        {
+            MFIB_TEST_REP((ai == dpo->dpoi_index),
+                          "bucket %d stacks on %U",
+                          bucket,
+                          format_dpo_id, dpo, 0);
+        }
+    }
+    return (!0);
+}
+
+static fib_forward_chain_type_t
+fib_forw_chain_type_from_fib_proto (fib_protocol_t proto)
+{
+    switch (proto)
+    {
+    case FIB_PROTOCOL_IP4:
+        return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4);
+    case FIB_PROTOCOL_IP6:
+        return (FIB_FORW_CHAIN_TYPE_UNICAST_IP6);
+    default:
+        break;
+    }
+    ASSERT(0);
+    return (0);
+}
+
+
+static int
+mfib_test_entry (fib_node_index_t fei,
+                 mfib_entry_flags_t eflags,
+                 u16 n_buckets,
+                 ...)
+{
+    const mfib_entry_t *mfe;
+    const replicate_t *rep;
+    mfib_prefix_t pfx;
+    va_list ap;
+
+    va_start(ap, n_buckets);
+
+    mfe = mfib_entry_get(fei);
+    mfib_entry_get_prefix(fei, &pfx);
+
+    MFIB_TEST_REP((eflags == mfe->mfe_flags),
+                  "%U has %U expect %U",
+                  format_mfib_prefix, &pfx,
+                  format_mfib_entry_flags, mfe->mfe_flags,
+                  format_mfib_entry_flags, eflags);
+
+    if (0 == n_buckets)
+    {
+        MFIB_TEST_REP((DPO_DROP == mfe->mfe_rep.dpoi_type),
+                      "%U links to %U",
+                      format_mfib_prefix, &pfx,
+                      format_dpo_id, &mfe->mfe_rep, 0);
+        return (!0);
+    }
+    else
+    {
+        dpo_id_t tmp = DPO_INVALID;
+        int res;
+
+        mfib_entry_contribute_forwarding(
+            fei,
+            fib_forw_chain_type_from_fib_proto(pfx.fp_proto),
+            &tmp);
+        rep = replicate_get(tmp.dpoi_index);
+
+        MFIB_TEST_REP((DPO_REPLICATE == tmp.dpoi_type),
+                      "%U links to %U",
+                      format_mfib_prefix, &pfx,
+                      format_dpo_type, tmp.dpoi_type);
+
+        res = mfib_test_validate_rep_v(rep, n_buckets, ap);
+
+        dpo_reset(&tmp);
+
+        return (res);
+    }
+}
+
+static int
+mfib_test_entry_itf (fib_node_index_t fei,
+                     u32 sw_if_index,
+                     mfib_itf_flags_t flags)
+{
+    const mfib_entry_t *mfe;
+    const mfib_itf_t *mfi;
+    mfib_prefix_t pfx;
+
+    mfe = mfib_entry_get(fei);
+    mfi = mfib_entry_get_itf(mfe, sw_if_index);
+    mfib_entry_get_prefix(fei, &pfx);
+
+    MFIB_TEST_REP((NULL != mfi),
+                  "%U has interface %d",
+                  format_mfib_prefix, &pfx, sw_if_index);
+
+    MFIB_TEST_REP((flags == mfi->mfi_flags),
+                  "%U interface %d has flags %U expect %U",
+                  format_mfib_prefix, &pfx, sw_if_index,
+                  format_mfib_itf_flags, flags,
+                  format_mfib_itf_flags, mfi->mfi_flags);
+
+    return (!0);
+}
+
+static int
+mfib_test_entry_no_itf (fib_node_index_t fei,
+                        u32 sw_if_index)
+{
+    const mfib_entry_t *mfe;
+    const mfib_itf_t *mfi;
+    mfib_prefix_t pfx;
+
+    mfe = mfib_entry_get(fei);
+    mfi = mfib_entry_get_itf(mfe, sw_if_index);
+    mfib_entry_get_prefix(fei, &pfx);
+
+    MFIB_TEST_REP((NULL == mfi),
+                  "%U has no interface %d",
+                  format_mfib_prefix, &pfx, sw_if_index);
+
+    return (!0);
+}
+
+static int
+mfib_test_i (fib_protocol_t PROTO,
+             vnet_link_t LINKT,
+             const mfib_prefix_t *pfx_no_forward,
+             const mfib_prefix_t *pfx_s_g,
+             const mfib_prefix_t *pfx_star_g_1,
+             const mfib_prefix_t *pfx_star_g_2,
+             const mfib_prefix_t *pfx_star_g_3,
+             const mfib_prefix_t *pfx_star_g_slash_m)
+{
+    fib_node_index_t mfei, mfei_dflt, mfei_no_f, mfei_s_g, mfei_g_1, mfei_g_2, mfei_g_3, mfei_g_m;
+    u32 fib_index, n_entries, n_itfs, n_reps;
+    fib_node_index_t ai_1, ai_2, ai_3;
+    test_main_t *tm;
+
+    mfib_prefix_t all_1s;
+    memset(&all_1s, 0xfd, sizeof(all_1s));
+
+    n_entries = pool_elts(mfib_entry_pool);
+    n_itfs = pool_elts(mfib_itf_pool);
+    n_reps = pool_elts(replicate_pool);
+    tm = &test_main;
+
+    ai_1 = adj_mcast_add_or_lock(PROTO,
+                                 LINKT,
+                                 tm->hw[1]->sw_if_index);
+    ai_2 = adj_mcast_add_or_lock(PROTO,
+                                 LINKT,
+                                 tm->hw[2]->sw_if_index);
+    ai_3 = adj_mcast_add_or_lock(PROTO,
+                                 LINKT,
+                                 tm->hw[3]->sw_if_index);
+
+    MFIB_TEST(3 == adj_mcast_db_size(), "3 MCAST adjs");
+
+    /* Find or create FIB table 11 */
+    fib_index = mfib_table_find_or_create_and_lock(PROTO, 11);
+
+    mfib_prefix_t pfx_dft = {
+        .fp_len = 0,
+        .fp_proto = PROTO,
+    };
+    mfei_dflt = mfib_table_lookup_exact_match(fib_index, &pfx_dft);
+    MFIB_TEST(FIB_NODE_INDEX_INVALID != mfei_dflt, "(*,*) presnet");
+    MFIB_TEST(mfib_test_entry(mfei_dflt,
+                              MFIB_ENTRY_FLAG_DROP,
+                              0),
+              "(*,*) no replcaitions");
+
+    MFIB_TEST(FIB_NODE_INDEX_INVALID != mfei_dflt, "(*,*) presnet");
+    MFIB_TEST(mfib_test_entry(mfei_dflt,
+                              MFIB_ENTRY_FLAG_DROP,
+                              0),
+              "(*,*) no replcaitions");
+
+
+    fib_route_path_t path_via_if0 = {
+        .frp_proto = PROTO,
+        .frp_addr = zero_addr,
+        .frp_sw_if_index = tm->hw[0]->sw_if_index,
+        .frp_fib_index = ~0,
+        .frp_weight = 0,
+        .frp_flags = 0,
+    };
+
+    mfib_table_entry_path_update(fib_index,
+                                 pfx_no_forward,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if0,
+                                 MFIB_ITF_FLAG_ACCEPT);
+
+    mfei_no_f = mfib_table_lookup_exact_match(fib_index, pfx_no_forward);
+    MFIB_TEST(mfib_test_entry(mfei_no_f,
+                              MFIB_ENTRY_FLAG_NONE,
+                              0),
+              "%U no replcaitions",
+              format_mfib_prefix, pfx_no_forward);
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei_no_f, tm->hw[0]->sw_if_index,
+                                     MFIB_ITF_FLAG_ACCEPT));
+
+    fib_route_path_t path_via_if1 = {
+        .frp_proto = PROTO,
+        .frp_addr = zero_addr,
+        .frp_sw_if_index = tm->hw[1]->sw_if_index,
+        .frp_fib_index = ~0,
+        .frp_weight = 0,
+        .frp_flags = 0,
+    };
+    fib_route_path_t path_via_if2 = {
+        .frp_proto = PROTO,
+        .frp_addr = zero_addr,
+        .frp_sw_if_index = tm->hw[2]->sw_if_index,
+        .frp_fib_index = ~0,
+        .frp_weight = 0,
+        .frp_flags = 0,
+    };
+    fib_route_path_t path_via_if3 = {
+        .frp_proto = PROTO,
+        .frp_addr = zero_addr,
+        .frp_sw_if_index = tm->hw[3]->sw_if_index,
+        .frp_fib_index = ~0,
+        .frp_weight = 0,
+        .frp_flags = 0,
+    };
+    fib_route_path_t path_for_us = {
+        .frp_proto = PROTO,
+        .frp_addr = zero_addr,
+        .frp_sw_if_index = 0xffffffff,
+        .frp_fib_index = ~0,
+        .frp_weight = 0,
+        .frp_flags = FIB_ROUTE_PATH_LOCAL,
+    };
+
+    /*
+     * An (S,G) with 1 accepting and 3 forwarding paths
+     */
+    mfib_table_entry_path_update(fib_index,
+                                 pfx_s_g,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if0,
+                                 MFIB_ITF_FLAG_ACCEPT);
+    mfib_table_entry_path_update(fib_index,
+                                 pfx_s_g,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if1,
+                                 MFIB_ITF_FLAG_FORWARD);
+    mfib_table_entry_path_update(fib_index,
+                                 pfx_s_g,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if2,
+                                 MFIB_ITF_FLAG_FORWARD);
+    mfib_table_entry_path_update(fib_index,
+                                 pfx_s_g,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if3,
+                                 (MFIB_ITF_FLAG_FORWARD |
+                                  MFIB_ITF_FLAG_NEGATE_SIGNAL));
+
+    mfei_s_g = mfib_table_lookup_exact_match(fib_index, pfx_s_g);
+
+    MFIB_TEST(FIB_NODE_INDEX_INVALID != mfei_s_g,
+              "%U present",
+              format_mfib_prefix, pfx_s_g);
+    MFIB_TEST(mfib_test_entry(mfei_s_g,
+                              MFIB_ENTRY_FLAG_NONE,
+                              3,
+                              DPO_ADJACENCY_MCAST, ai_1,
+                              DPO_ADJACENCY_MCAST, ai_2,
+                              DPO_ADJACENCY_MCAST, ai_3),
+              "%U replicate ok",
+              format_mfib_prefix, pfx_s_g);
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei_s_g, tm->hw[0]->sw_if_index,
+                                     MFIB_ITF_FLAG_ACCEPT));
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei_s_g, tm->hw[1]->sw_if_index,
+                                     MFIB_ITF_FLAG_FORWARD));
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei_s_g, tm->hw[2]->sw_if_index,
+                                     MFIB_ITF_FLAG_FORWARD));
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei_s_g, tm->hw[3]->sw_if_index,
+                                     (MFIB_ITF_FLAG_FORWARD |
+                                      MFIB_ITF_FLAG_NEGATE_SIGNAL)));
+
+    /*
+     * A (*,G), which the same G as the (S,G).
+     * different paths. test our LPM.
+     */
+    mfei_g_1 = mfib_table_entry_path_update(fib_index,
+                                            pfx_star_g_1,
+                                            MFIB_SOURCE_API,
+                                            &path_via_if0,
+                                            MFIB_ITF_FLAG_ACCEPT);
+    mfib_table_entry_path_update(fib_index,
+                                 pfx_star_g_1,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if1,
+                                 MFIB_ITF_FLAG_FORWARD);
+
+    /*
+     * test we find the *,G and S,G via LPM and exact matches
+     */
+    mfei = mfib_table_lookup_exact_match(fib_index,
+                                         pfx_star_g_1);
+    MFIB_TEST(mfei == mfei_g_1,
+              "%U found via exact match",
+              format_mfib_prefix, pfx_star_g_1);
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              1,
+                              DPO_ADJACENCY_MCAST, ai_1),
+              "%U replicate ok",
+              format_mfib_prefix, pfx_star_g_1);
+
+    mfei = mfib_table_lookup(fib_index,
+                             pfx_star_g_1);
+    MFIB_TEST(mfei == mfei_g_1,
+              "%U found via LP match",
+              format_mfib_prefix, pfx_star_g_1);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              1,
+                              DPO_ADJACENCY_MCAST, ai_1),
+              "%U replicate ok",
+              format_mfib_prefix, pfx_star_g_1);
+
+    mfei = mfib_table_lookup_exact_match(fib_index, pfx_s_g);
+    MFIB_TEST(mfei == mfei_s_g,
+              "%U found via exact match",
+              format_mfib_prefix, pfx_s_g);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              3,
+                              DPO_ADJACENCY_MCAST, ai_1,
+                              DPO_ADJACENCY_MCAST, ai_2,
+                              DPO_ADJACENCY_MCAST, ai_3),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_s_g);
+    mfei = mfib_table_lookup(fib_index, pfx_s_g);
+    MFIB_TEST(mfei == mfei_s_g,
+              "%U found via LP match",
+              format_mfib_prefix, pfx_s_g);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              3,
+                              DPO_ADJACENCY_MCAST, ai_1,
+                              DPO_ADJACENCY_MCAST, ai_2,
+                              DPO_ADJACENCY_MCAST, ai_3),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_s_g);
+
+    /*
+     * A (*,G/m), which the same root G as the (*,G).
+     * different paths. test our LPM.
+     */
+    mfei_g_m = mfib_table_entry_path_update(fib_index,
+                                            pfx_star_g_slash_m,
+                                            MFIB_SOURCE_API,
+                                            &path_via_if2,
+                                            MFIB_ITF_FLAG_ACCEPT);
+    mfib_table_entry_path_update(fib_index,
+                                 pfx_star_g_slash_m,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if3,
+                                 MFIB_ITF_FLAG_FORWARD);
+
+    /*
+     * test we find the (*,G/m), (*,G) and (S,G) via LPM and exact matches
+     */
+    mfei = mfib_table_lookup_exact_match(fib_index, pfx_star_g_1);
+    MFIB_TEST((mfei_g_1 == mfei),
+              "%U found via DP LPM: %d",
+              format_mfib_prefix, pfx_star_g_1, mfei);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              1,
+                              DPO_ADJACENCY_MCAST, ai_1),
+              "%U replicate ok",
+              format_mfib_prefix, pfx_star_g_1);
+
+    mfei = mfib_table_lookup(fib_index, pfx_star_g_1);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              1,
+                              DPO_ADJACENCY_MCAST, ai_1),
+              "%U replicate ok",
+              format_mfib_prefix, pfx_star_g_1);
+
+    mfei = mfib_table_lookup_exact_match(fib_index, pfx_s_g);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              3,
+                              DPO_ADJACENCY_MCAST, ai_1,
+                              DPO_ADJACENCY_MCAST, ai_2,
+                              DPO_ADJACENCY_MCAST, ai_3),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_s_g);
+    mfei = mfib_table_lookup(fib_index, pfx_s_g);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              3,
+                              DPO_ADJACENCY_MCAST, ai_1,
+                              DPO_ADJACENCY_MCAST, ai_2,
+                              DPO_ADJACENCY_MCAST, ai_3),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_s_g);
+
+    mfei = mfib_table_lookup_exact_match(fib_index, pfx_star_g_slash_m);
+    MFIB_TEST(mfei = mfei_g_m,
+              "%U Found via exact match",
+              format_mfib_prefix, pfx_star_g_slash_m);
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              1,
+                              DPO_ADJACENCY_MCAST, ai_3),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_star_g_slash_m);
+    MFIB_TEST(mfei_g_m == mfib_table_lookup(fib_index, pfx_star_g_slash_m),
+              "%U found via LPM",
+              format_mfib_prefix, pfx_star_g_slash_m);
+
+    /*
+     * Add a for-us path
+     */
+    mfei = mfib_table_entry_path_update(fib_index,
+                                        pfx_s_g,
+                                        MFIB_SOURCE_API,
+                                        &path_for_us,
+                                        MFIB_ITF_FLAG_FORWARD);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              4,
+                              DPO_ADJACENCY_MCAST, ai_1,
+                              DPO_ADJACENCY_MCAST, ai_2,
+                              DPO_ADJACENCY_MCAST, ai_3,
+                              DPO_RECEIVE, 0),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_s_g);
+
+    /*
+     * remove a for-us path
+     */
+    mfib_table_entry_path_remove(fib_index,
+                                 pfx_s_g,
+                                 MFIB_SOURCE_API,
+                                 &path_for_us);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              3,
+                              DPO_ADJACENCY_MCAST, ai_1,
+                              DPO_ADJACENCY_MCAST, ai_2,
+                              DPO_ADJACENCY_MCAST, ai_3),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_s_g);
+
+    /*
+     * update an existing forwarding path to be only accepting
+     *   - expect it to be removed from the replication set.
+     */
+    mfib_table_entry_path_update(fib_index,
+                                 pfx_s_g,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if3,
+                                 MFIB_ITF_FLAG_ACCEPT);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              2,
+                              DPO_ADJACENCY_MCAST, ai_1,
+                              DPO_ADJACENCY_MCAST, ai_2),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_s_g);
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[0]->sw_if_index,
+                                     MFIB_ITF_FLAG_ACCEPT));
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[1]->sw_if_index,
+                                     MFIB_ITF_FLAG_FORWARD));
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[2]->sw_if_index,
+                                     MFIB_ITF_FLAG_FORWARD));
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[3]->sw_if_index,
+                                     MFIB_ITF_FLAG_ACCEPT));
+    /*
+     * Make the path forwarding again
+     *  - expect it to be added back to the replication set
+     */
+    mfib_table_entry_path_update(fib_index,
+                                 pfx_s_g,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if3,
+                                 (MFIB_ITF_FLAG_FORWARD |
+                                  MFIB_ITF_FLAG_ACCEPT |
+                                  MFIB_ITF_FLAG_NEGATE_SIGNAL));
+
+    mfei = mfib_table_lookup_exact_match(fib_index,
+                                         pfx_s_g);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              3,
+                              DPO_ADJACENCY_MCAST, ai_1,
+                              DPO_ADJACENCY_MCAST, ai_2,
+                              DPO_ADJACENCY_MCAST, ai_3),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_s_g);
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[0]->sw_if_index,
+                                     MFIB_ITF_FLAG_ACCEPT));
+     MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[1]->sw_if_index,
+                                     MFIB_ITF_FLAG_FORWARD));
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[2]->sw_if_index,
+                                     MFIB_ITF_FLAG_FORWARD));
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[3]->sw_if_index,
+                                     (MFIB_ITF_FLAG_FORWARD |
+                                      MFIB_ITF_FLAG_ACCEPT |
+                                      MFIB_ITF_FLAG_NEGATE_SIGNAL)));
+
+    /*
+     * update flags on the entry
+     */
+    mfib_table_entry_update(fib_index,
+                            pfx_s_g,
+                            MFIB_SOURCE_API,
+                            MFIB_ENTRY_FLAG_SIGNAL);
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_SIGNAL,
+                              3,
+                              DPO_ADJACENCY_MCAST, ai_1,
+                              DPO_ADJACENCY_MCAST, ai_2,
+                              DPO_ADJACENCY_MCAST, ai_3),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_s_g);
+
+    /*
+     * remove paths
+     */
+    mfib_table_entry_path_remove(fib_index,
+                                 pfx_s_g,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if3);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_SIGNAL,
+                              2,
+                              DPO_ADJACENCY_MCAST, ai_1,
+                              DPO_ADJACENCY_MCAST, ai_2),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_s_g);
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[0]->sw_if_index,
+                                     MFIB_ITF_FLAG_ACCEPT));
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[1]->sw_if_index,
+                                     MFIB_ITF_FLAG_FORWARD));
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[2]->sw_if_index,
+                                     MFIB_ITF_FLAG_FORWARD));
+    MFIB_TEST_NS(mfib_test_entry_no_itf(mfei, tm->hw[3]->sw_if_index));
+
+    mfib_table_entry_path_remove(fib_index,
+                                 pfx_s_g,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if1);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_SIGNAL,
+                              1,
+                              DPO_ADJACENCY_MCAST, ai_2),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_s_g);
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[0]->sw_if_index,
+                                     MFIB_ITF_FLAG_ACCEPT));
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[2]->sw_if_index,
+                                     MFIB_ITF_FLAG_FORWARD));
+    MFIB_TEST_NS(mfib_test_entry_no_itf(mfei, tm->hw[3]->sw_if_index));
+
+    /*
+     * remove the accpeting only interface
+     */
+    mfib_table_entry_path_remove(fib_index,
+                                 pfx_s_g,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if0);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_SIGNAL,
+                              1,
+                              DPO_ADJACENCY_MCAST, ai_2),
+              "%U replicate OK",
+              format_mfib_prefix, pfx_s_g);
+    MFIB_TEST_NS(mfib_test_entry_itf(mfei, tm->hw[2]->sw_if_index,
+                                     MFIB_ITF_FLAG_FORWARD));
+    MFIB_TEST_NS(mfib_test_entry_no_itf(mfei, tm->hw[0]->sw_if_index));
+    MFIB_TEST_NS(mfib_test_entry_no_itf(mfei, tm->hw[1]->sw_if_index));
+    MFIB_TEST_NS(mfib_test_entry_no_itf(mfei, tm->hw[3]->sw_if_index));
+
+    /*
+     * remove the last path, the entry still has flags so it remains
+     */
+    mfib_table_entry_path_remove(fib_index,
+                                 pfx_s_g,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if2);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_SIGNAL,
+                              0),
+              "%U no replications",
+              format_mfib_prefix, pfx_s_g);
+
+    /*
+     * update flags on the entry
+     */
+    mfib_table_entry_update(fib_index,
+                            pfx_s_g,
+                            MFIB_SOURCE_API,
+                            (MFIB_ENTRY_FLAG_SIGNAL |
+                             MFIB_ENTRY_FLAG_CONNECTED));
+    MFIB_TEST(mfib_test_entry(mfei,
+                              (MFIB_ENTRY_FLAG_SIGNAL |
+                               MFIB_ENTRY_FLAG_CONNECTED),
+                              0),
+              "%U no replications",
+              format_mfib_prefix, pfx_s_g);
+
+    /*
+     * An entry with a NS interface
+     */
+    mfei_g_2 = mfib_table_entry_path_update(fib_index,
+                                            pfx_star_g_2,
+                                            MFIB_SOURCE_API,
+                                            &path_via_if0,
+                                            (MFIB_ITF_FLAG_ACCEPT |
+                                             MFIB_ITF_FLAG_NEGATE_SIGNAL));
+    MFIB_TEST(mfib_test_entry(mfei_g_2,
+                              MFIB_ENTRY_FLAG_NONE,
+                              0),
+              "%U No replications",
+              format_mfib_prefix, pfx_star_g_2);
+
+    /*
+     * Simulate a signal from the data-plane
+     */
+    {
+        mfib_entry_t *mfe;
+        mfib_itf_t *mfi;
+
+        mfe = mfib_entry_get(mfei_g_2);
+        mfi = mfib_entry_get_itf(mfe, path_via_if0.frp_sw_if_index);
+
+        mfib_signal_push(mfe, mfi, NULL);
+    }
+
+    /*
+     * An entry with a NS interface
+     */
+    mfei_g_3 = mfib_table_entry_path_update(fib_index,
+                                            pfx_star_g_3,
+                                            MFIB_SOURCE_API,
+                                            &path_via_if0,
+                                            (MFIB_ITF_FLAG_ACCEPT |
+                                             MFIB_ITF_NEGATE_SIGNAL));
+    MFIB_TEST(mfib_test_entry(mfei_g_3,
+                              MFIB_ENTRY_FLAG_NONE,
+                              0),
+              "%U No replications",
+              format_mfib_prefix, pfx_star_g_3);
+
+    /*
+     * Simulate a signal from the data-plane
+     */
+    {
+        mfib_entry_t *mfe;
+        mfib_itf_t *mfi;
+
+        mfe = mfib_entry_get(mfei_g_3);
+        mfi = mfib_entry_get_itf(mfe, path_via_if0.frp_sw_if_index);
+
+        mfib_signal_push(mfe, mfi, NULL);
+    }
+
+    if (FIB_PROTOCOL_IP6 == PROTO)
+    {
+        /*
+         * All the entries are present. let's ensure we can find them all
+         * via exact and longest prefix matches.
+         */
+        /*
+         * A source address we will never match
+         */
+        ip6_address_t src = {
+            .as_u64[0] = clib_host_to_net_u64(0x3001000000000000),
+            .as_u64[1] = clib_host_to_net_u64(0xffffffffffffffff),
+        };
+
+        /*
+         * Find the (*,G/m)
+         */
+        MFIB_TEST((mfei_g_m == ip6_mfib_table_lookup2(
+                                   ip6_mfib_get(fib_index),
+                                   &src,
+                                   &pfx_star_g_slash_m->fp_grp_addr.ip6)),
+                  "%U found via DP LPM grp=%U",
+                  format_mfib_prefix, pfx_star_g_slash_m,
+                  format_ip6_address, &pfx_star_g_slash_m->fp_grp_addr.ip6);
+
+        ip6_address_t tmp = pfx_star_g_slash_m->fp_grp_addr.ip6;
+        tmp.as_u8[15] = 0xff;
+
+        MFIB_TEST((mfei_g_m == ip6_mfib_table_lookup2(
+                                   ip6_mfib_get(fib_index),
+                                   &pfx_s_g->fp_src_addr.ip6,
+                                   &tmp)),
+                  "%U found via DP LPM grp=%U",
+                  format_mfib_prefix, pfx_star_g_slash_m,
+                  format_ip6_address, &tmp);
+
+        /*
+         * Find the (S,G).
+         */
+        mfei = ip6_mfib_table_lookup2(ip6_mfib_get(fib_index),
+                                      &pfx_s_g->fp_src_addr.ip6,
+                                      &pfx_s_g->fp_grp_addr.ip6);
+        MFIB_TEST((mfei_s_g == mfei),
+                  "%U found via DP LPM: %d",
+                  format_mfib_prefix, pfx_s_g, mfei);
+
+        /*
+         * Find the 3 (*,G) s
+         */
+        mfei = ip6_mfib_table_lookup2(ip6_mfib_get(fib_index),
+                                      &src,
+                                      &pfx_star_g_1->fp_grp_addr.ip6);
+        MFIB_TEST((mfei_g_1 == mfei),
+                  "%U found via DP LPM: %d",
+                  format_mfib_prefix, pfx_star_g_1, mfei);
+        mfei = ip6_mfib_table_lookup2(ip6_mfib_get(fib_index),
+                                      &src,
+                                      &pfx_star_g_2->fp_grp_addr.ip6);
+        MFIB_TEST((mfei_g_2 == mfei),
+                  "%U found via DP LPM: %d",
+                  format_mfib_prefix, pfx_star_g_2, mfei);
+        mfei = ip6_mfib_table_lookup2(ip6_mfib_get(fib_index),
+                                      &src,
+                                      &pfx_star_g_3->fp_grp_addr.ip6);
+        MFIB_TEST((mfei_g_3 == mfei),
+                  "%U found via DP LPM: %d",
+                  format_mfib_prefix, pfx_star_g_3, mfei);
+    }
+
+    /*
+     * remove flags on the entry. This is the last of the
+     * state associated with the entry, so now it goes.
+     */
+    mfib_table_entry_update(fib_index,
+                            pfx_s_g,
+                            MFIB_SOURCE_API,
+                            MFIB_ENTRY_FLAG_NONE);
+    mfei = mfib_table_lookup_exact_match(fib_index,
+                                         pfx_s_g);
+    MFIB_TEST(FIB_NODE_INDEX_INVALID == mfei,
+              "%U gone",
+              format_mfib_prefix, pfx_s_g);
+
+    /*
+     * remove the last path on the no forward entry - the last entry
+     */
+    mfib_table_entry_path_remove(fib_index,
+                                 pfx_no_forward,
+                                 MFIB_SOURCE_API,
+                                 &path_via_if0);
+
+    mfei = mfib_table_lookup_exact_match(fib_index, pfx_no_forward);
+    MFIB_TEST(FIB_NODE_INDEX_INVALID == mfei,
+              "%U gone",
+              format_mfib_prefix, pfx_no_forward);
+
+    /*
+     * hard delete the (*,232.1.1.1)
+     */
+    mfib_table_entry_delete(fib_index,
+                            pfx_star_g_1,
+                            MFIB_SOURCE_API);
+
+    mfei = mfib_table_lookup_exact_match(fib_index, pfx_star_g_1);
+    MFIB_TEST(FIB_NODE_INDEX_INVALID == mfei,
+              "%U gone",
+              format_mfib_prefix, pfx_star_g_1);
+    /*
+     * remove the entry whilst the signal is pending
+     */
+    mfib_table_entry_delete(fib_index,
+                            pfx_star_g_2,
+                            MFIB_SOURCE_API);
+
+    mfei = mfib_table_lookup_exact_match(fib_index, pfx_star_g_2);
+    MFIB_TEST(FIB_NODE_INDEX_INVALID == mfei,
+              "%U Gone",
+              format_mfib_prefix, pfx_star_g_2);
+    mfib_table_entry_delete(fib_index,
+                            pfx_star_g_3,
+                            MFIB_SOURCE_API);
+
+    mfei = mfib_table_lookup_exact_match(fib_index, pfx_star_g_3);
+    MFIB_TEST(FIB_NODE_INDEX_INVALID == mfei,
+              "%U Gone",
+              format_mfib_prefix, pfx_star_g_3);
+
+    mfib_table_entry_delete(fib_index,
+                            pfx_star_g_slash_m,
+                            MFIB_SOURCE_API);
+
+    mfei = mfib_table_lookup_exact_match(fib_index, pfx_star_g_slash_m);
+    MFIB_TEST(FIB_NODE_INDEX_INVALID == mfei,
+              "%U Gone",
+              format_mfib_prefix, pfx_star_g_slash_m);
+
+    /*
+     * Unlock the table - it's the last lock so should be gone thereafter
+     */
+    mfib_table_unlock(fib_index, PROTO);
+
+    MFIB_TEST((FIB_NODE_INDEX_INVALID ==
+               mfib_table_find(PROTO, fib_index)),
+              "MFIB table %d gone", fib_index);
+
+    adj_unlock(ai_1);
+    adj_unlock(ai_2);
+    adj_unlock(ai_3);
+
+    /*
+     * test we've leaked no resources
+     */
+    MFIB_TEST(0 == adj_mcast_db_size(), "%d MCAST adjs", adj_mcast_db_size());
+    MFIB_TEST(n_reps == pool_elts(replicate_pool), "%d=%d replicates",
+              n_reps, pool_elts(replicate_pool));
+    MFIB_TEST(n_entries == pool_elts(mfib_entry_pool),
+              " No more entries %d!=%d",
+              n_entries, pool_elts(mfib_entry_pool));
+    MFIB_TEST(n_itfs == pool_elts(mfib_itf_pool),
+              " No more Interfaces %d!=%d",
+              n_itfs, pool_elts(mfib_itf_pool));
+
+    return (0);
+}
+
+static int
+mfib_test_v4 (void)
+{
+    const mfib_prefix_t pfx_224_s_8 = {
+        .fp_len = 8,
+        .fp_proto = FIB_PROTOCOL_IP4,
+        .fp_grp_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0xe0000000),
+        }
+    };
+    const mfib_prefix_t pfx_1_1_1_1_c_239_1_1_1 = {
+        .fp_len = 64,
+        .fp_proto = FIB_PROTOCOL_IP4,
+        .fp_grp_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0xef010101),
+        },
+        .fp_src_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0x01010101),
+        },
+    };
+    const mfib_prefix_t pfx_239_1_1_1 = {
+        .fp_len = 32,
+        .fp_proto = FIB_PROTOCOL_IP4,
+        .fp_grp_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0xef010101),
+        },
+        .fp_src_addr = {
+            .ip4.as_u32 = 0,
+        },
+    };
+    const mfib_prefix_t pfx_239_1_1_2 = {
+        .fp_len = 32,
+        .fp_proto = FIB_PROTOCOL_IP4,
+        .fp_grp_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0xef010102),
+        },
+        .fp_src_addr = {
+            .ip4.as_u32 = 0,
+        },
+    };
+    const mfib_prefix_t pfx_239_1_1_3 = {
+        .fp_len = 32,
+        .fp_proto = FIB_PROTOCOL_IP4,
+        .fp_grp_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0xef010103),
+        },
+        .fp_src_addr = {
+            .ip4.as_u32 = 0,
+        },
+    };
+    const mfib_prefix_t pfx_239 = {
+        .fp_len = 8,
+        .fp_proto = FIB_PROTOCOL_IP4,
+        .fp_grp_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0xef000000),
+        },
+        .fp_src_addr = {
+            .ip4.as_u32 = 0,
+        },
+    };
+
+    return (mfib_test_i(FIB_PROTOCOL_IP4,
+                        VNET_LINK_IP4,
+                        &pfx_224_s_8,
+                        &pfx_1_1_1_1_c_239_1_1_1,
+                        &pfx_239_1_1_1,
+                        &pfx_239_1_1_2,
+                        &pfx_239_1_1_3,
+                        &pfx_239));
+}
+
+static int
+mfib_test_v6 (void)
+{
+    const mfib_prefix_t pfx_ffd_s_12 = {
+        .fp_len = 12,
+        .fp_proto = FIB_PROTOCOL_IP6,
+        .fp_grp_addr = {
+            .ip6.as_u64[0] = clib_host_to_net_u64(0xffd0000000000000),
+        }
+    };
+    const mfib_prefix_t pfx_2001_1_c_ff_1 = {
+        .fp_len = 256,
+        .fp_proto = FIB_PROTOCOL_IP6,
+        .fp_grp_addr = {
+            .ip6.as_u64[0] = clib_host_to_net_u64(0xff01000000000000),
+            .ip6.as_u64[1] = clib_host_to_net_u64(0x0000000000000001),
+        },
+        .fp_src_addr = {
+            .ip6.as_u64[0] = clib_host_to_net_u64(0x2001000000000000),
+            .ip6.as_u64[1] = clib_host_to_net_u64(0x0000000000000001),
+        },
+    };
+    const mfib_prefix_t pfx_ff_1 = {
+        .fp_len = 128,
+        .fp_proto = FIB_PROTOCOL_IP6,
+        .fp_grp_addr = {
+            .ip6.as_u64[0] = clib_host_to_net_u64(0xff01000000000000),
+            .ip6.as_u64[1] = clib_host_to_net_u64(0x0000000000000001),
+        },
+    };
+    const mfib_prefix_t pfx_ff_2 = {
+        .fp_len = 128,
+        .fp_proto = FIB_PROTOCOL_IP6,
+        .fp_grp_addr = {
+            .ip6.as_u64[0] = clib_host_to_net_u64(0xff01000000000000),
+            .ip6.as_u64[1] = clib_host_to_net_u64(0x0000000000000002),
+        },
+    };
+    const mfib_prefix_t pfx_ff_3 = {
+        /*
+         * this is the ALL DHCP routers address
+         */
+        .fp_len = 128,
+        .fp_proto = FIB_PROTOCOL_IP6,
+        .fp_grp_addr = {
+            .ip6.as_u64[0] = clib_host_to_net_u64(0xff02000100000000),
+            .ip6.as_u64[1] = clib_host_to_net_u64(0x0000000000000002),
+        },
+    };
+    const mfib_prefix_t pfx_ff = {
+        .fp_len = 16,
+        .fp_proto = FIB_PROTOCOL_IP6,
+        .fp_grp_addr = {
+            .ip6.as_u64[0] = clib_host_to_net_u64(0xff01000000000000),
+            .ip6.as_u64[1] = clib_host_to_net_u64(0x0000000000000000),
+        },
+    };
+
+    return (mfib_test_i(FIB_PROTOCOL_IP6,
+                        VNET_LINK_IP6,
+                        &pfx_ffd_s_12,
+                        &pfx_2001_1_c_ff_1,
+                        &pfx_ff_1,
+                        &pfx_ff_2,
+                        &pfx_ff_3,
+                        &pfx_ff));
+}
+
+static clib_error_t *
+mfib_test (vlib_main_t * vm,
+           unformat_input_t * input,
+           vlib_cli_command_t * cmd_arg)
+{
+    int res = 0;
+
+    res += mfib_test_mk_intf(4);
+    res += mfib_test_v4();
+    res += mfib_test_v6();
+
+    if (res)
+    {
+        return clib_error_return(0, "MFIB Unit Test Failed");
+    }
+    else
+    {
+        return (NULL);
+    }
+}
+
+VLIB_CLI_COMMAND (test_fib_command, static) = {
+    .path = "test mfib",
+    .short_help = "fib unit tests - DO NOT RUN ON A LIVE SYSTEM",
+    .function = mfib_test,
+};
+
+clib_error_t *
+mfib_test_init (vlib_main_t *vm)
+{
+    return 0;
+}
+
+VLIB_INIT_FUNCTION (mfib_test_init);
diff --git a/src/vnet/mfib/mfib_types.c b/src/vnet/mfib/mfib_types.c
new file mode 100644
index 00000000..6d77c3d8
--- /dev/null
+++ b/src/vnet/mfib/mfib_types.c
@@ -0,0 +1,213 @@
+ /*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/mfib/mfib_types.h>
+
+#include <vnet/ip/ip.h>
+
+/**
+ * String names for each flag
+ */
+static const char *mfib_flag_names[] = MFIB_ENTRY_NAMES_SHORT;
+static const char *mfib_flag_names_long[] = MFIB_ENTRY_NAMES_LONG;
+
+static const char *mfib_itf_flag_long_names[] = MFIB_ITF_NAMES_LONG;
+static const char *mfib_itf_flag_names[] = MFIB_ITF_NAMES_SHORT;
+
+u8 *
+format_mfib_prefix (u8 * s, va_list * args)
+{
+    mfib_prefix_t *fp = va_arg (*args, mfib_prefix_t *);
+
+    /*
+     * protocol specific so it prints ::/0 correctly.
+     */
+    switch (fp->fp_proto)
+    {
+    case FIB_PROTOCOL_IP6:
+    {
+        ip6_address_t p6 = fp->fp_grp_addr.ip6;
+        u32 len = (fp->fp_len > 128 ? 128 : fp->fp_len);
+
+        ip6_address_mask(&p6, &(ip6_main.fib_masks[len]));
+
+        if (ip6_address_is_zero(&fp->fp_src_addr.ip6))
+        {
+            s = format(s, "(*, ");
+        }
+        else
+        {
+            s = format (s, "(%U, ", format_ip6_address, &fp->fp_src_addr.ip6);
+        }
+        s = format (s, "%U", format_ip6_address, &p6);
+        s = format (s, "/%d)", len);
+        break;
+    }
+    case FIB_PROTOCOL_IP4:
+    {
+        ip4_address_t p4 = fp->fp_grp_addr.ip4;
+        u32 len = (fp->fp_len > 32 ? 32 : fp->fp_len);
+
+        p4.as_u32 &= ip4_main.fib_masks[len];
+
+        if (0 == fp->fp_src_addr.ip4.as_u32)
+        {
+            s = format(s, "(*, ");
+        }
+        else
+        {
+            s = format (s, "(%U, ", format_ip4_address, &fp->fp_src_addr.ip4);
+        }
+        s = format (s, "%U", format_ip4_address, &p4);
+        s = format (s, "/%d)", len);
+        break;
+    }
+    case FIB_PROTOCOL_MPLS:
+        break;
+    }
+
+    return (s);
+}
+
+u8 *
+format_mfib_entry_flags (u8 * s, va_list * args)
+{
+    mfib_entry_attribute_t attr;
+    mfib_entry_flags_t flags;
+
+    flags = va_arg (*args, mfib_entry_flags_t);
+
+    if (MFIB_ENTRY_FLAG_NONE != flags) {
+        s = format(s, " flags:");
+        FOR_EACH_MFIB_ATTRIBUTE(attr) {
+            if ((1<<attr) & flags) {
+                s = format (s, "%s,", mfib_flag_names[attr]);
+            }
+        }
+    }
+
+    return (s);
+}
+
+u8 *
+format_mfib_itf_flags (u8 * s, va_list * args)
+{
+    mfib_itf_attribute_t attr;
+    mfib_itf_flags_t flags;
+
+    flags = va_arg (*args, mfib_itf_flags_t);
+
+    FOR_EACH_MFIB_ITF_ATTRIBUTE(attr) {
+        if ((1<<attr) & flags) {
+            s = format (s, "%s,", mfib_itf_flag_long_names[attr]);
+        }
+    }
+
+    return (s);
+}
+
+uword
+unformat_mfib_itf_flags (unformat_input_t * input,
+                         va_list * args)
+{
+    mfib_itf_flags_t old, *iflags = va_arg (*args, mfib_itf_flags_t*);
+    mfib_itf_attribute_t attr;
+
+    old = *iflags;
+    FOR_EACH_MFIB_ITF_ATTRIBUTE(attr) {
+        if (unformat (input, mfib_itf_flag_long_names[attr]))
+            *iflags |= (1 << attr);
+    }
+    FOR_EACH_MFIB_ITF_ATTRIBUTE(attr) {
+        if (unformat (input, mfib_itf_flag_names[attr]))
+            *iflags |= (1 << attr);
+    }
+
+    return (old == *iflags ? 0 : 1);
+}
+
+uword
+unformat_mfib_entry_flags (unformat_input_t * input,
+                           va_list * args)
+{
+    mfib_entry_flags_t old, *eflags = va_arg (*args, mfib_entry_flags_t*);
+    mfib_entry_attribute_t attr;
+
+    old = *eflags;
+    FOR_EACH_MFIB_ATTRIBUTE(attr) {
+        if (unformat (input, mfib_flag_names[attr]))
+            *eflags |= (1 << attr);
+    }
+
+    return (old == *eflags ? 0 : 1);
+}
+
+clib_error_t *
+mfib_show_route_flags (vlib_main_t * vm,
+                       unformat_input_t * main_input,
+                       vlib_cli_command_t * cmd)
+{
+    mfib_entry_attribute_t attr;
+
+    FOR_EACH_MFIB_ATTRIBUTE(attr) {
+        vlib_cli_output(vm, "%s = %s",
+                        mfib_flag_names[attr],
+                        mfib_flag_names_long[attr]);
+    }
+
+    return (NULL);
+}
+
+/*?
+ * This command display the set of support flags applicable to the MFIB route
+ */
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (mfib_route_flags_command, static) =
+{
+  .path = "sh mfib route flags",
+  .short_help = "Flags applicable to an MFIB route",
+  .function = mfib_show_route_flags,
+  .is_mp_safe = 1,
+};
+/* *INDENT-ON* */
+
+clib_error_t *
+mfib_show_itf_flags (vlib_main_t * vm,
+                     unformat_input_t * main_input,
+                     vlib_cli_command_t * cmd)
+{
+    mfib_itf_attribute_t attr;
+
+    FOR_EACH_MFIB_ITF_ATTRIBUTE(attr) {
+        vlib_cli_output(vm, "%s = %s",
+                        mfib_itf_flag_names[attr],
+                        mfib_itf_flag_long_names[attr]);
+    }
+
+    return (NULL);
+}
+
+/*?
+ * This command display the set of support flags applicable to the MFIB route
+ */
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (mfib_itf_flags_command, static) =
+{
+  .path = "sh mfib itf flags",
+  .short_help = "Flags applicable to an MFIB interfaces",
+  .function = mfib_show_itf_flags,
+  .is_mp_safe = 1,
+};
+/* *INDENT-ON* */
diff --git a/src/vnet/mfib/mfib_types.h b/src/vnet/mfib/mfib_types.h
new file mode 100644
index 00000000..37898a07
--- /dev/null
+++ b/src/vnet/mfib/mfib_types.h
@@ -0,0 +1,185 @@
+ /*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MFIB_TYPES_H__
+#define __MFIB_TYPES_H__
+
+#include <vnet/fib/fib_types.h>
+
+/**
+ * Aggregrate type for a prefix
+ */
+typedef struct mfib_prefix_t_ {
+    /**
+     * The mask length
+     */
+    u16 fp_len;
+
+    /**
+     * protocol type
+     */
+    fib_protocol_t fp_proto;
+
+    /**
+     * Pad to keep the address 4 byte aligned
+     */
+    u8 ___fp___pad;
+
+    /**
+     * The address type is not deriveable from the fp_addr member.
+     * If it's v4, then the first 3 u32s of the address will be 0.
+     * v6 addresses (even v4 mapped ones) have at least 2 u32s assigned
+     * to non-zero values. true. but when it's all zero, one cannot decide.
+     */
+    ip46_address_t fp_grp_addr;
+    ip46_address_t fp_src_addr;
+} mfib_prefix_t;
+
+typedef enum mfib_entry_attribute_t_
+{
+    MFIB_ENTRY_ATTRIBUTE_FIRST = 0,
+    /**
+     * The control planes needs packets mathing this entry to generate
+     * a signal.
+     */
+    MFIB_ENTRY_SIGNAL =  MFIB_ENTRY_ATTRIBUTE_FIRST,
+    /**
+     * Drop all traffic to this route
+     */
+    MFIB_ENTRY_DROP,
+    /**
+     * The control plane needs to be informed of coneected sources
+     */
+    MFIB_ENTRY_CONNECTED,
+    /**
+     * Accept packets from any incpoming interface
+     *        Use with extreme caution
+     */
+    MFIB_ENTRY_ACCEPT_ALL_ITF,
+    MFIB_ENTRY_INHERIT_ACCEPT,
+    MFIB_ENTRY_ATTRIBUTE_LAST = MFIB_ENTRY_INHERIT_ACCEPT,
+} mfib_entry_attribute_t;
+
+#define FOR_EACH_MFIB_ATTRIBUTE(_item)			\
+    for (_item = MFIB_ENTRY_ATTRIBUTE_FIRST;		\
+	 _item <= MFIB_ENTRY_ATTRIBUTE_LAST;		\
+	 _item++)
+
+#define MFIB_ENTRY_NAMES_SHORT  {          \
+    [MFIB_ENTRY_SIGNAL]         = "S",     \
+    [MFIB_ENTRY_CONNECTED]      = "C",     \
+    [MFIB_ENTRY_DROP]           = "D",     \
+    [MFIB_ENTRY_ACCEPT_ALL_ITF] = "AA",    \
+    [MFIB_ENTRY_INHERIT_ACCEPT] = "IA",    \
+}
+
+#define MFIB_ENTRY_NAMES_LONG  {                    \
+    [MFIB_ENTRY_SIGNAL]         = "Signal",         \
+    [MFIB_ENTRY_CONNECTED]      = "Connected",      \
+    [MFIB_ENTRY_DROP]           = "Drop",           \
+    [MFIB_ENTRY_ACCEPT_ALL_ITF] = "Accept-all-itf", \
+    [MFIB_ENTRY_INHERIT_ACCEPT] = "Inherit-Accept", \
+}
+
+typedef enum mfib_entry_flags_t_
+{
+    MFIB_ENTRY_FLAG_NONE,
+    MFIB_ENTRY_FLAG_SIGNAL = (1 << MFIB_ENTRY_SIGNAL),
+    MFIB_ENTRY_FLAG_DROP = (1 << MFIB_ENTRY_DROP),
+    MFIB_ENTRY_FLAG_CONNECTED = (1 << MFIB_ENTRY_CONNECTED),
+    MFIB_ENTRY_FLAG_INHERIT_ACCEPT = (1 << MFIB_ENTRY_INHERIT_ACCEPT),
+    MFIB_ENTRY_FLAG_ACCEPT_ALL_ITF = (1 << MFIB_ENTRY_ACCEPT_ALL_ITF),
+} mfib_entry_flags_t;
+
+typedef enum mfib_itf_attribute_t_
+{
+    MFIB_ITF_ATTRIBUTE_FIRST,
+    MFIB_ITF_NEGATE_SIGNAL = MFIB_ITF_ATTRIBUTE_FIRST,
+    MFIB_ITF_ACCEPT,
+    MFIB_ITF_FORWARD,
+    MFIB_ITF_SIGNAL_PRESENT,
+    MFIB_ITF_DONT_PRESERVE,
+    MFIB_ITF_ATTRIBUTE_LAST = MFIB_ITF_DONT_PRESERVE,
+} mfib_itf_attribute_t;
+
+#define FOR_EACH_MFIB_ITF_ATTRIBUTE(_item)	       	\
+    for (_item = MFIB_ITF_ATTRIBUTE_FIRST;       	\
+	 _item <= MFIB_ITF_ATTRIBUTE_LAST;		\
+	 _item++)
+
+#define MFIB_ITF_NAMES_SHORT  {             \
+    [MFIB_ITF_NEGATE_SIGNAL] = "NS",        \
+    [MFIB_ITF_ACCEPT] = "A",                \
+    [MFIB_ITF_FORWARD] = "F",               \
+    [MFIB_ITF_SIGNAL_PRESENT] = "SP",       \
+    [MFIB_ITF_DONT_PRESERVE] = "DP",        \
+}
+
+#define MFIB_ITF_NAMES_LONG  {                    \
+    [MFIB_ITF_NEGATE_SIGNAL] = "Negate-Signal",   \
+    [MFIB_ITF_ACCEPT] = "Accept",                 \
+    [MFIB_ITF_FORWARD] = "Forward",               \
+    [MFIB_ITF_SIGNAL_PRESENT] = "Signal-Present", \
+    [MFIB_ITF_DONT_PRESERVE] = "Don't-Preserve", \
+}
+
+typedef enum mfib_itf_flags_t_
+{
+    MFIB_ITF_FLAG_NONE,
+    MFIB_ITF_FLAG_NEGATE_SIGNAL = (1 << MFIB_ITF_NEGATE_SIGNAL),
+    MFIB_ITF_FLAG_ACCEPT = (1 << MFIB_ITF_ACCEPT),
+    MFIB_ITF_FLAG_FORWARD = (1 << MFIB_ITF_FORWARD),
+    MFIB_ITF_FLAG_SIGNAL_PRESENT = (1 << MFIB_ITF_SIGNAL_PRESENT),
+    MFIB_ITF_FLAG_DONT_PRESERVE = (1 << MFIB_ITF_DONT_PRESERVE),
+} mfib_itf_flags_t;
+
+/**
+ * Possible [control plane] sources of MFIB entries
+ */
+typedef enum mfib_source_t_
+{
+    MFIB_SOURCE_SPECIAL,
+    MFIB_SOURCE_API,
+    MFIB_SOURCE_CLI,
+    MFIB_SOURCE_VXLAN,
+    MFIB_SOURCE_DHCP,
+    MFIB_SOURCE_DEFAULT_ROUTE,
+} mfib_source_t;
+
+#define MFIB_SOURCE_NAMES {                        \
+    [MFIB_SOURCE_SPECIAL] = "Special",             \
+    [MFIB_SOURCE_API] = "API",                     \
+    [MFIB_SOURCE_CLI] = "CLI",                     \
+    [MFIB_SOURCE_DHCP] = "DHCP",                   \
+    [MFIB_SOURCE_VXLAN] = "VXLAN",                 \
+    [MFIB_SOURCE_DEFAULT_ROUTE] = "Default Route", \
+}
+
+/**
+ * \brief Compare two prefixes for equality
+ */
+extern int mfib_prefix_cmp(const mfib_prefix_t *p1,
+                           const mfib_prefix_t *p2);
+
+extern u8 * format_mfib_prefix(u8 * s, va_list * args);
+
+extern u8 *format_mfib_entry_flags(u8 * s, va_list * args);
+extern u8 *format_mfib_itf_flags(u8 * s, va_list * args);
+extern uword unformat_mfib_itf_flags(unformat_input_t * input,
+                                     va_list * args);
+extern uword unformat_mfib_entry_flags(unformat_input_t * input,
+                                       va_list * args);
+
+#endif
diff --git a/src/vnet/misc.c b/src/vnet/misc.c
index 4c8c4cad..9cfe8394 100644
--- a/src/vnet/misc.c
+++ b/src/vnet/misc.c
@@ -83,6 +83,9 @@ vnet_main_init (vlib_main_t * vm)
   if ((error = vlib_call_init_function (vm, fib_module_init)))
     return error;
 
+  if ((error = vlib_call_init_function (vm, mfib_module_init)))
+    return error;
+
   if ((error = vlib_call_init_function (vm, ip_main_init)))
     return error;
 
diff --git a/src/vnet/rewrite.h b/src/vnet/rewrite.h
index 00c1efbd..ce2bce3a 100644
--- a/src/vnet/rewrite.h
+++ b/src/vnet/rewrite.h
@@ -64,6 +64,16 @@ typedef CLIB_PACKED (struct {
      Used for MTU check after packet rewrite. */
   u16 max_l3_packet_bytes;
 
+  /* When dynamically writing a multicast destination L2 addresss
+   * this is the offset within the address to start writing n
+   * bytes of the IP mcast address */
+  u8 dst_mcast_offset;
+
+  /* When dynamically writing a multicast destination L2 addresss
+   * this is the number of bytes of the dest IP address to write into
+   * the MAC rewrite */
+  u8 dst_mcast_n_bytes;
+
   /* Rewrite string starting at end and going backwards. */
   u8 data[0];
 }) vnet_rewrite_header_t;
@@ -261,6 +271,27 @@ _vnet_rewrite_two_headers (vnet_rewrite_header_t * h0,
 			     sizeof ((rw0).rewrite_data),		\
 			     (most_likely_size))
 
+always_inline void
+_vnet_fixup_one_header (vnet_rewrite_header_t * h0,
+			u8 * addr, u32 addr_len,
+			u8 * packet0, int clear_first_bit)
+{
+  /* location to write to in the packet */
+  u8 *p0 = packet0 - h0->dst_mcast_offset;
+  u8 *p1 = p0;
+  /* location to write from in the L3 dest address */
+  u8 *a0 = addr + addr_len - h0->dst_mcast_n_bytes;
+
+  clib_memcpy (p0, a0, h0->dst_mcast_n_bytes);
+  if (clear_first_bit)
+    *p1 &= 0x7f;
+}
+
+#define vnet_fixup_one_header(rw0,addr,p0,clear_first_bit)              \
+  _vnet_fixup_one_header (&((rw0).rewrite_header),                      \
+                          (u8*)(addr), sizeof((*addr)),                 \
+                          (u8*)(p0), (clear_first_bit))
+
 #define VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST ((void *) 0)
 /** Deprecated */
 void vnet_rewrite_for_sw_interface (struct vnet_main_t *vnm,
diff --git a/src/vnet/sr/sr.c b/src/vnet/sr/sr.c
index 5d0275d9..f30c0da9 100644
--- a/src/vnet/sr/sr.c
+++ b/src/vnet/sr/sr.c
@@ -2161,7 +2161,7 @@ sr_fix_dst_addr (vlib_main_t * vm,
 
 	  adj0 =
 	    ip_get_adjacency (lm, vnet_buffer (b0)->ip.adj_index[VLIB_TX]);
-	  next0 = adj0->mcast_group_index;
+	  next0 = adj0->if_address_index;
 
 	  /* We should be pointing at an Ethernet header... */
 	  eh0 = vlib_buffer_get_current (b0);
@@ -3088,7 +3088,7 @@ set_ip6_sr_rewrite_fn (vlib_main_t * vm,
   adj->rewrite_header.node_index = sr_fix_dst_addr_node.index;
 
   /* $$$$$ hack... steal the mcast group index */
-  adj->mcast_group_index =
+  adj->if_address_index =
     vlib_node_add_next (vm, sr_fix_dst_addr_node.index,
 			hi->output_node_index);
 
diff --git a/src/vnet/util/radix.c b/src/vnet/util/radix.c
new file mode 100644
index 00000000..ff0b0f7b
--- /dev/null
+++ b/src/vnet/util/radix.c
@@ -0,0 +1,1104 @@
+/*	$NetBSD: radix.c,v 1.47 2016/12/12 03:55:57 ozaki-r Exp $	*/
+
+/*
+ * Copyright (c) 1988, 1989, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)radix.c	8.6 (Berkeley) 10/17/95
+ */
+
+/*
+ * Routines to build and maintain radix trees for routing lookups.
+ */
+
+#include <vnet/util/radix.h>
+
+typedef void (*rn_printer_t)(void *, const char *fmt, ...);
+
+static int max_keylen = 33; // me
+struct radix_mask *rn_mkfreelist;
+struct radix_node_head *mask_rnhead;
+static char *addmask_key;
+static const char normal_chars[] =
+    {0, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe, -1};
+static char *rn_zeros, *rn_ones;
+
+#define rn_masktop (mask_rnhead->rnh_treetop)
+
+static int rn_satisfies_leaf(const char *, struct radix_node *, int);
+static int rn_lexobetter(const void *, const void *);
+static struct radix_mask *rn_new_radix_mask(struct radix_node *,
+    struct radix_mask *);
+static struct radix_node *rn_walknext(struct radix_node *, rn_printer_t,
+    void *);
+static struct radix_node *rn_walkfirst(struct radix_node *, rn_printer_t,
+    void *);
+static void rn_nodeprint(struct radix_node *, rn_printer_t, void *,
+    const char *);
+
+#define	SUBTREE_OPEN	"[ "
+#define	SUBTREE_CLOSE	" ]"
+
+#ifdef RN_DEBUG
+static void rn_treeprint(struct radix_node_head *, rn_printer_t, void *);
+#endif /* RN_DEBUG */
+
+#define MIN(x,y) (((x)<(y))?(x):(y))
+
+static struct radix_mask*
+rm_alloc (void)
+{
+    struct radix_mask *rm = clib_mem_alloc(sizeof(struct radix_mask));
+
+    memset(rm, 0, sizeof(*rm));
+
+    return (rm);
+}
+
+static void
+rm_free (struct radix_mask *rm)
+{
+    clib_mem_free(rm);
+}
+
+#define R_Malloc(p, t, n)                               \
+{                                                       \
+    p = (t) clib_mem_alloc((unsigned int)(n));          \
+    memset(p, 0, n);                                    \
+}
+#define Free(p) clib_mem_free((p))
+#define log(a,b, c...)
+#define bool i32
+
+/*
+ * The data structure for the keys is a radix tree with one way
+ * branching removed.  The index rn_b at an internal node n represents a bit
+ * position to be tested.  The tree is arranged so that all descendants
+ * of a node n have keys whose bits all agree up to position rn_b - 1.
+ * (We say the index of n is rn_b.)
+ *
+ * There is at least one descendant which has a one bit at position rn_b,
+ * and at least one with a zero there.
+ *
+ * A route is determined by a pair of key and mask.  We require that the
+ * bit-wise logical and of the key and mask to be the key.
+ * We define the index of a route to associated with the mask to be
+ * the first bit number in the mask where 0 occurs (with bit number 0
+ * representing the highest order bit).
+ *
+ * We say a mask is normal if every bit is 0, past the index of the mask.
+ * If a node n has a descendant (k, m) with index(m) == index(n) == rn_b,
+ * and m is a normal mask, then the route applies to every descendant of n.
+ * If the index(m) < rn_b, this implies the trailing last few bits of k
+ * before bit b are all 0, (and hence consequently true of every descendant
+ * of n), so the route applies to all descendants of the node as well.
+ *
+ * Similar logic shows that a non-normal mask m such that
+ * index(m) <= index(n) could potentially apply to many children of n.
+ * Thus, for each non-host route, we attach its mask to a list at an internal
+ * node as high in the tree as we can go.
+ *
+ * The present version of the code makes use of normal routes in short-
+ * circuiting an explicit mask and compare operation when testing whether
+ * a key satisfies a normal route, and also in remembering the unique leaf
+ * that governs a subtree.
+ */
+
+struct radix_node *
+rn_search(
+	const void *v_arg,
+	struct radix_node *head)
+{
+	const u8 * const v = v_arg;
+	struct radix_node *x;
+
+	for (x = head; x->rn_b >= 0;) {
+		if (x->rn_bmask & v[x->rn_off])
+			x = x->rn_r;
+		else
+			x = x->rn_l;
+	}
+	return x;
+}
+
+struct radix_node *
+rn_search_m(
+	const void *v_arg,
+	struct radix_node *head,
+	const void *m_arg)
+{
+	struct radix_node *x;
+	const u8 * const v = v_arg;
+	const u8 * const m = m_arg;
+
+	for (x = head; x->rn_b >= 0;) {
+		if ((x->rn_bmask & m[x->rn_off]) &&
+		    (x->rn_bmask & v[x->rn_off]))
+			x = x->rn_r;
+		else
+			x = x->rn_l;
+	}
+	return x;
+}
+
+int
+rn_refines(
+	const void *m_arg,
+	const void *n_arg)
+{
+	const char *m = m_arg;
+	const char *n = n_arg;
+	const char *lim = n + *(const u8 *)n;
+	const char *lim2 = lim;
+	int longer = (*(const u8 *)n++) - (int)(*(const u8 *)m++);
+	int masks_are_equal = 1;
+
+	if (longer > 0)
+		lim -= longer;
+	while (n < lim) {
+		if (*n & ~(*m))
+			return 0;
+		if (*n++ != *m++)
+			masks_are_equal = 0;
+	}
+	while (n < lim2)
+		if (*n++)
+			return 0;
+	if (masks_are_equal && (longer < 0))
+		for (lim2 = m - longer; m < lim2; )
+			if (*m++)
+				return 1;
+	return !masks_are_equal;
+}
+
+struct radix_node *
+rn_lookup(
+	const void *v_arg,
+	const void *m_arg,
+	struct radix_node_head *head)
+{
+	struct radix_node *x;
+	const char *netmask = NULL;
+
+	if (m_arg) {
+		if ((x = rn_addmask(m_arg, 1, head->rnh_treetop->rn_off)) == 0)
+			return NULL;
+		netmask = x->rn_key;
+	}
+	x = rn_match(v_arg, head);
+	if (x != NULL && netmask != NULL) {
+		while (x != NULL && x->rn_mask != netmask)
+			x = x->rn_dupedkey;
+	}
+	return x;
+}
+
+static int
+rn_satisfies_leaf(
+	const char *trial,
+	struct radix_node *leaf,
+	int skip)
+{
+	const char *cp = trial;
+	const char *cp2 = leaf->rn_key;
+	const char *cp3 = leaf->rn_mask;
+	const char *cplim;
+	int length = MIN(*(const u8 *)cp, *(const u8 *)cp2);
+
+	if (cp3 == 0)
+		cp3 = rn_ones;
+	else
+		length = MIN(length, *(const u8 *)cp3);
+	cplim = cp + length; cp3 += skip; cp2 += skip;
+	for (cp += skip; cp < cplim; cp++, cp2++, cp3++)
+		if ((*cp ^ *cp2) & *cp3)
+			return 0;
+	return 1;
+}
+
+struct radix_node *
+rn_match(
+	const void *v_arg,
+	struct radix_node_head *head)
+{
+	const char * const v = v_arg;
+	struct radix_node *t = head->rnh_treetop;
+	struct radix_node *top = t;
+	struct radix_node *x;
+	struct radix_node *saved_t;
+	const char *cp = v;
+	const char *cp2;
+	const char *cplim;
+	int off = t->rn_off;
+	int vlen = *(const u8 *)cp;
+	int matched_off;
+	int test, b, rn_b;
+
+	/*
+	 * Open code rn_search(v, top) to avoid overhead of extra
+	 * subroutine call.
+	 */
+	for (; t->rn_b >= 0; ) {
+		if (t->rn_bmask & cp[t->rn_off])
+			t = t->rn_r;
+		else
+			t = t->rn_l;
+	}
+	/*
+	 * See if we match exactly as a host destination
+	 * or at least learn how many bits match, for normal mask finesse.
+	 *
+	 * It doesn't hurt us to limit how many bytes to check
+	 * to the length of the mask, since if it matches we had a genuine
+	 * match and the leaf we have is the most specific one anyway;
+	 * if it didn't match with a shorter length it would fail
+	 * with a long one.  This wins big for class B&C netmasks which
+	 * are probably the most common case...
+	 */
+	if (t->rn_mask)
+		vlen = *(const u8 *)t->rn_mask;
+	cp += off; cp2 = t->rn_key + off; cplim = v + vlen;
+	for (; cp < cplim; cp++, cp2++)
+		if (*cp != *cp2)
+			goto on1;
+	/*
+	 * This extra grot is in case we are explicitly asked
+	 * to look up the default.  Ugh!
+	 */
+	if ((t->rn_flags & RNF_ROOT) && t->rn_dupedkey)
+		t = t->rn_dupedkey;
+	return t;
+on1:
+	test = (*cp ^ *cp2) & 0xff; /* find first bit that differs */
+	for (b = 7; (test >>= 1) > 0;)
+		b--;
+	matched_off = cp - v;
+	b += matched_off << 3;
+	rn_b = -1 - b;
+	/*
+	 * If there is a host route in a duped-key chain, it will be first.
+	 */
+	if ((saved_t = t)->rn_mask == 0)
+		t = t->rn_dupedkey;
+	for (; t; t = t->rn_dupedkey)
+		/*
+		 * Even if we don't match exactly as a host,
+		 * we may match if the leaf we wound up at is
+		 * a route to a net.
+		 */
+		if (t->rn_flags & RNF_NORMAL) {
+			if (rn_b <= t->rn_b)
+				return t;
+		} else if (rn_satisfies_leaf(v, t, matched_off))
+				return t;
+	t = saved_t;
+	/* start searching up the tree */
+	do {
+		struct radix_mask *m;
+		t = t->rn_p;
+		m = t->rn_mklist;
+		if (m) {
+			/*
+			 * If non-contiguous masks ever become important
+			 * we can restore the masking and open coding of
+			 * the search and satisfaction test and put the
+			 * calculation of "off" back before the "do".
+			 */
+			do {
+				if (m->rm_flags & RNF_NORMAL) {
+					if (rn_b <= m->rm_b)
+						return m->rm_leaf;
+				} else {
+					off = MIN(t->rn_off, matched_off);
+					x = rn_search_m(v, t, m->rm_mask);
+					while (x && x->rn_mask != m->rm_mask)
+						x = x->rn_dupedkey;
+					if (x && rn_satisfies_leaf(v, x, off))
+						return x;
+				}
+				m = m->rm_mklist;
+			} while (m);
+		}
+	} while (t != top);
+	return NULL;
+}
+
+static void
+rn_nodeprint(struct radix_node *rn, rn_printer_t printer, void *arg,
+    const char *delim)
+{
+	(*printer)(arg, "%s(%s%p: p<%p> l<%p> r<%p>)",
+	    delim, ((void *)rn == arg) ? "*" : "", rn, rn->rn_p,
+	    rn->rn_l, rn->rn_r);
+}
+
+#ifdef RN_DEBUG
+int	rn_debug =  1;
+
+static void
+rn_dbg_print(void *arg, const char *fmt, ...)
+{
+	va_list ap;
+
+	va_start(ap, fmt);
+	vlog(LOG_DEBUG, fmt, ap);
+	va_end(ap);
+}
+
+static void
+rn_treeprint(struct radix_node_head *h, rn_printer_t printer, void *arg)
+{
+	struct radix_node *dup, *rn;
+	const char *delim;
+
+	if (printer == NULL)
+		return;
+
+	rn = rn_walkfirst(h->rnh_treetop, printer, arg);
+	for (;;) {
+		/* Process leaves */
+		delim = "";
+		for (dup = rn; dup != NULL; dup = dup->rn_dupedkey) {
+			if ((dup->rn_flags & RNF_ROOT) != 0)
+				continue;
+			rn_nodeprint(dup, printer, arg, delim);
+			delim = ", ";
+		}
+		rn = rn_walknext(rn, printer, arg);
+		if (rn->rn_flags & RNF_ROOT)
+			return;
+	}
+	/* NOTREACHED */
+}
+
+#define	traverse(__head, __rn)	rn_treeprint((__head), rn_dbg_print, (__rn))
+#endif /* RN_DEBUG */
+
+struct radix_node *
+rn_newpair(
+	const void *v,
+	int b,
+	struct radix_node nodes[2])
+{
+	struct radix_node *tt = nodes;
+	struct radix_node *t = tt + 1;
+	t->rn_b = b; t->rn_bmask = 0x80 >> (b & 7);
+	t->rn_l = tt; t->rn_off = b >> 3;
+	tt->rn_b = -1; tt->rn_key = v; tt->rn_p = t;
+	tt->rn_flags = t->rn_flags = RNF_ACTIVE;
+	return t;
+}
+
+struct radix_node *
+rn_insert(
+	const void *v_arg,
+	struct radix_node_head *head,
+	int *dupentry,
+	struct radix_node nodes[2])
+{
+	struct radix_node *top = head->rnh_treetop;
+	struct radix_node *t = rn_search(v_arg, top);
+	struct radix_node *tt;
+	const char *v = v_arg;
+	int head_off = top->rn_off;
+	int vlen = *((const u8 *)v);
+	const char *cp = v + head_off;
+	int b;
+    	/*
+	 * Find first bit at which v and t->rn_key differ
+	 */
+    {
+	const char *cp2 = t->rn_key + head_off;
+	const char *cplim = v + vlen;
+	int cmp_res;
+
+	while (cp < cplim)
+		if (*cp2++ != *cp++)
+			goto on1;
+	*dupentry = 1;
+	return t;
+on1:
+	*dupentry = 0;
+	cmp_res = (cp[-1] ^ cp2[-1]) & 0xff;
+	for (b = (cp - v) << 3; cmp_res; b--)
+		cmp_res >>= 1;
+    }
+    {
+	struct radix_node *p, *x = top;
+	cp = v;
+	do {
+		p = x;
+		if (cp[x->rn_off] & x->rn_bmask)
+			x = x->rn_r;
+		else x = x->rn_l;
+	} while (b > (unsigned) x->rn_b); /* x->rn_b < b && x->rn_b >= 0 */
+#ifdef RN_DEBUG
+	if (rn_debug)
+		log(LOG_DEBUG, "%s: Going In:\n", __func__), traverse(head, p);
+#endif
+	t = rn_newpair(v_arg, b, nodes); tt = t->rn_l;
+	if ((cp[p->rn_off] & p->rn_bmask) == 0)
+		p->rn_l = t;
+	else
+		p->rn_r = t;
+	x->rn_p = t; t->rn_p = p; /* frees x, p as temp vars below */
+	if ((cp[t->rn_off] & t->rn_bmask) == 0) {
+		t->rn_r = x;
+	} else {
+		t->rn_r = tt; t->rn_l = x;
+	}
+#ifdef RN_DEBUG
+	if (rn_debug) {
+		log(LOG_DEBUG, "%s: Coming Out:\n", __func__),
+		    traverse(head, p);
+	}
+#endif /* RN_DEBUG */
+    }
+	return tt;
+}
+
+struct radix_node *
+rn_addmask(
+	const void *n_arg,
+	int search,
+	int skip)
+{
+	const char *netmask = n_arg;
+	const char *cp;
+	const char *cplim;
+	struct radix_node *x;
+	struct radix_node *saved_x;
+	int b = 0, mlen, j;
+	int maskduplicated, m0, isnormal;
+	static int last_zeroed = 0;
+
+	if ((mlen = *(const u8 *)netmask) > max_keylen)
+		mlen = max_keylen;
+	if (skip == 0)
+		skip = 1;
+	if (mlen <= skip)
+		return mask_rnhead->rnh_nodes;
+	if (skip > 1)
+		memmove(addmask_key + 1, rn_ones + 1, skip - 1);
+	if ((m0 = mlen) > skip)
+		memmove(addmask_key + skip, netmask + skip, mlen - skip);
+	/*
+	 * Trim trailing zeroes.
+	 */
+	for (cp = addmask_key + mlen; (cp > addmask_key) && cp[-1] == 0;)
+		cp--;
+	mlen = cp - addmask_key;
+	if (mlen <= skip) {
+		if (m0 >= last_zeroed)
+			last_zeroed = mlen;
+		return mask_rnhead->rnh_nodes;
+	}
+	if (m0 < last_zeroed)
+		memset(addmask_key + m0, 0, last_zeroed - m0);
+	*addmask_key = last_zeroed = mlen;
+	x = rn_search(addmask_key, rn_masktop);
+	if (memcmp(addmask_key, x->rn_key, mlen) != 0)
+		x = 0;
+	if (x || search)
+		return x;
+	R_Malloc(x, struct radix_node *, max_keylen + 2 * sizeof (*x));
+	if ((saved_x = x) == NULL)
+		return NULL;
+	memset(x, 0, max_keylen + 2 * sizeof (*x));
+	cp = netmask = (void *)(x + 2);
+	memmove(x + 2, addmask_key, mlen);
+	x = rn_insert(cp, mask_rnhead, &maskduplicated, x);
+	if (maskduplicated) {
+                log(LOG_ERR, "rn_addmask: mask impossibly already in tree\n");
+		Free(saved_x);
+		return x;
+	}
+	/*
+	 * Calculate index of mask, and check for normalcy.
+	 */
+	cplim = netmask + mlen; isnormal = 1;
+	for (cp = netmask + skip; (cp < cplim) && *(const u8 *)cp == 0xff;)
+		cp++;
+	if (cp != cplim) {
+		for (j = 0x80; (j & *cp) != 0; j >>= 1)
+			b++;
+		if (*cp != normal_chars[b] || cp != (cplim - 1))
+			isnormal = 0;
+	}
+	b += (cp - netmask) << 3;
+	x->rn_b = -1 - b;
+	if (isnormal)
+		x->rn_flags |= RNF_NORMAL;
+	return x;
+}
+
+static int	/* XXX: arbitrary ordering for non-contiguous masks */
+rn_lexobetter(
+	const void *m_arg,
+	const void *n_arg)
+{
+	const u8 *mp = m_arg;
+	const u8 *np = n_arg;
+	const u8 *lim;
+
+	if (*mp > *np)
+		return 1;  /* not really, but need to check longer one first */
+	if (*mp == *np)
+		for (lim = mp + *mp; mp < lim;)
+			if (*mp++ > *np++)
+				return 1;
+	return 0;
+}
+
+static struct radix_mask *
+rn_new_radix_mask(
+	struct radix_node *tt,
+	struct radix_mask *next)
+{
+	struct radix_mask *m;
+
+	m = rm_alloc();
+	if (m == NULL) {
+		log(LOG_ERR, "Mask for route not entered\n");
+		return NULL;
+	}
+	memset(m, 0, sizeof(*m));
+	m->rm_b = tt->rn_b;
+	m->rm_flags = tt->rn_flags;
+	if (tt->rn_flags & RNF_NORMAL)
+		m->rm_leaf = tt;
+	else
+		m->rm_mask = tt->rn_mask;
+	m->rm_mklist = next;
+	tt->rn_mklist = m;
+	return m;
+}
+
+struct radix_node *
+rn_addroute(
+	const void *v_arg,
+	const void *n_arg,
+	struct radix_node_head *head,
+	struct radix_node treenodes[2])
+{
+	const char *v = v_arg, *netmask = n_arg;
+	struct radix_node *t, *x = NULL, *tt;
+	struct radix_node *saved_tt, *top = head->rnh_treetop;
+	short b = 0, b_leaf = 0;
+	int keyduplicated;
+	const char *mmask;
+	struct radix_mask *m, **mp;
+
+	/*
+	 * In dealing with non-contiguous masks, there may be
+	 * many different routes which have the same mask.
+	 * We will find it useful to have a unique pointer to
+	 * the mask to speed avoiding duplicate references at
+	 * nodes and possibly save time in calculating indices.
+	 */
+	if (netmask != NULL) {
+		if ((x = rn_addmask(netmask, 0, top->rn_off)) == NULL)
+			return NULL;
+		b_leaf = x->rn_b;
+		b = -1 - x->rn_b;
+		netmask = x->rn_key;
+	}
+	/*
+	 * Deal with duplicated keys: attach node to previous instance
+	 */
+	saved_tt = tt = rn_insert(v, head, &keyduplicated, treenodes);
+	if (keyduplicated) {
+		for (t = tt; tt != NULL; t = tt, tt = tt->rn_dupedkey) {
+			if (tt->rn_mask == netmask)
+				return NULL;
+			if (netmask == NULL ||
+			    (tt->rn_mask != NULL &&
+			     (b_leaf < tt->rn_b || /* index(netmask) > node */
+			       rn_refines(netmask, tt->rn_mask) ||
+			       rn_lexobetter(netmask, tt->rn_mask))))
+				break;
+		}
+		/*
+		 * If the mask is not duplicated, we wouldn't
+		 * find it among possible duplicate key entries
+		 * anyway, so the above test doesn't hurt.
+		 *
+		 * We sort the masks for a duplicated key the same way as
+		 * in a masklist -- most specific to least specific.
+		 * This may require the unfortunate nuisance of relocating
+		 * the head of the list.
+		 *
+		 * We also reverse, or doubly link the list through the
+		 * parent pointer.
+		 */
+		if (tt == saved_tt) {
+			struct	radix_node *xx = x;
+			/* link in at head of list */
+			(tt = treenodes)->rn_dupedkey = t;
+			tt->rn_flags = t->rn_flags;
+			tt->rn_p = x = t->rn_p;
+			t->rn_p = tt;
+			if (x->rn_l == t)
+				x->rn_l = tt;
+			else
+				x->rn_r = tt;
+			saved_tt = tt;
+			x = xx;
+		} else {
+			(tt = treenodes)->rn_dupedkey = t->rn_dupedkey;
+			t->rn_dupedkey = tt;
+			tt->rn_p = t;
+			if (tt->rn_dupedkey)
+				tt->rn_dupedkey->rn_p = tt;
+		}
+		tt->rn_key = v;
+		tt->rn_b = -1;
+		tt->rn_flags = RNF_ACTIVE;
+	}
+	/*
+	 * Put mask in tree.
+	 */
+	if (netmask != NULL) {
+		tt->rn_mask = netmask;
+		tt->rn_b = x->rn_b;
+		tt->rn_flags |= x->rn_flags & RNF_NORMAL;
+	}
+	t = saved_tt->rn_p;
+	if (keyduplicated)
+		goto on2;
+	b_leaf = -1 - t->rn_b;
+	if (t->rn_r == saved_tt)
+		x = t->rn_l;
+	else
+		x = t->rn_r;
+	/* Promote general routes from below */
+	if (x->rn_b < 0) {
+		for (mp = &t->rn_mklist; x != NULL; x = x->rn_dupedkey) {
+			if (x->rn_mask != NULL && x->rn_b >= b_leaf &&
+			    x->rn_mklist == NULL) {
+				*mp = m = rn_new_radix_mask(x, NULL);
+				if (m != NULL)
+					mp = &m->rm_mklist;
+			}
+		}
+	} else if (x->rn_mklist != NULL) {
+		/*
+		 * Skip over masks whose index is > that of new node
+		 */
+		for (mp = &x->rn_mklist; (m = *mp) != NULL; mp = &m->rm_mklist)
+			if (m->rm_b >= b_leaf)
+				break;
+		t->rn_mklist = m;
+		*mp = NULL;
+	}
+on2:
+	/* Add new route to highest possible ancestor's list */
+	if (netmask == NULL || b > t->rn_b)
+		return tt; /* can't lift at all */
+	b_leaf = tt->rn_b;
+	do {
+		x = t;
+		t = t->rn_p;
+	} while (b <= t->rn_b && x != top);
+	/*
+	 * Search through routes associated with node to
+	 * insert new route according to index.
+	 * Need same criteria as when sorting dupedkeys to avoid
+	 * double loop on deletion.
+	 */
+	for (mp = &x->rn_mklist; (m = *mp) != NULL; mp = &m->rm_mklist) {
+		if (m->rm_b < b_leaf)
+			continue;
+		if (m->rm_b > b_leaf)
+			break;
+		if (m->rm_flags & RNF_NORMAL) {
+			mmask = m->rm_leaf->rn_mask;
+			if (tt->rn_flags & RNF_NORMAL) {
+				log(LOG_ERR, "Non-unique normal route,"
+				    " mask not entered\n");
+				return tt;
+			}
+		} else
+			mmask = m->rm_mask;
+		if (mmask == netmask) {
+			m->rm_refs++;
+			tt->rn_mklist = m;
+			return tt;
+		}
+		if (rn_refines(netmask, mmask) || rn_lexobetter(netmask, mmask))
+			break;
+	}
+	*mp = rn_new_radix_mask(tt, *mp);
+	return tt;
+}
+
+struct radix_node *
+rn_delete1(
+	const void *v_arg,
+	const void *netmask_arg,
+	struct radix_node_head *head,
+	struct radix_node *rn)
+{
+	struct radix_node *t, *p, *x, *tt;
+	struct radix_mask *m, *saved_m, **mp;
+	struct radix_node *dupedkey, *saved_tt, *top;
+	const char *v, *netmask;
+	int b, head_off, vlen;
+
+	v = v_arg;
+	netmask = netmask_arg;
+	x = head->rnh_treetop;
+	tt = rn_search(v, x);
+	head_off = x->rn_off;
+	vlen =  *(const u8 *)v;
+	saved_tt = tt;
+	top = x;
+	if (tt == NULL ||
+	    memcmp(v + head_off, tt->rn_key + head_off, vlen - head_off) != 0)
+		return NULL;
+	/*
+	 * Delete our route from mask lists.
+	 */
+	if (netmask != NULL) {
+		if ((x = rn_addmask(netmask, 1, head_off)) == NULL)
+			return NULL;
+		netmask = x->rn_key;
+		while (tt->rn_mask != netmask)
+			if ((tt = tt->rn_dupedkey) == NULL)
+				return NULL;
+	}
+	if (tt->rn_mask == NULL || (saved_m = m = tt->rn_mklist) == NULL)
+		goto on1;
+	if (tt->rn_flags & RNF_NORMAL) {
+		if (m->rm_leaf != tt || m->rm_refs > 0) {
+			log(LOG_ERR, "rn_delete: inconsistent annotation\n");
+			return NULL;  /* dangling ref could cause disaster */
+		}
+	} else {
+		if (m->rm_mask != tt->rn_mask) {
+			log(LOG_ERR, "rn_delete: inconsistent annotation\n");
+			goto on1;
+		}
+		if (--m->rm_refs >= 0)
+			goto on1;
+	}
+	b = -1 - tt->rn_b;
+	t = saved_tt->rn_p;
+	if (b > t->rn_b)
+		goto on1; /* Wasn't lifted at all */
+	do {
+		x = t;
+		t = t->rn_p;
+	} while (b <= t->rn_b && x != top);
+	for (mp = &x->rn_mklist; (m = *mp) != NULL; mp = &m->rm_mklist) {
+		if (m == saved_m) {
+			*mp = m->rm_mklist;
+			rm_free(m);
+			break;
+		}
+	}
+	if (m == NULL) {
+		log(LOG_ERR, "rn_delete: couldn't find our annotation\n");
+		if (tt->rn_flags & RNF_NORMAL)
+			return NULL; /* Dangling ref to us */
+	}
+on1:
+	/*
+	 * Eliminate us from tree
+	 */
+	if (tt->rn_flags & RNF_ROOT)
+		return NULL;
+#ifdef RN_DEBUG
+	if (rn_debug)
+		log(LOG_DEBUG, "%s: Going In:\n", __func__), traverse(head, tt);
+#endif
+	t = tt->rn_p;
+	dupedkey = saved_tt->rn_dupedkey;
+	if (dupedkey != NULL) {
+		/*
+		 * Here, tt is the deletion target, and
+		 * saved_tt is the head of the dupedkey chain.
+		 */
+		if (tt == saved_tt) {
+			x = dupedkey;
+			x->rn_p = t;
+			if (t->rn_l == tt)
+				t->rn_l = x;
+			else
+				t->rn_r = x;
+		} else {
+			/* find node in front of tt on the chain */
+			for (x = p = saved_tt;
+			     p != NULL && p->rn_dupedkey != tt;)
+				p = p->rn_dupedkey;
+			if (p != NULL) {
+				p->rn_dupedkey = tt->rn_dupedkey;
+				if (tt->rn_dupedkey != NULL)
+					tt->rn_dupedkey->rn_p = p;
+			} else
+				log(LOG_ERR, "rn_delete: couldn't find us\n");
+		}
+		t = tt + 1;
+		if  (t->rn_flags & RNF_ACTIVE) {
+			*++x = *t;
+			p = t->rn_p;
+			if (p->rn_l == t)
+				p->rn_l = x;
+			else
+				p->rn_r = x;
+			x->rn_l->rn_p = x;
+			x->rn_r->rn_p = x;
+		}
+		goto out;
+	}
+	if (t->rn_l == tt)
+		x = t->rn_r;
+	else
+		x = t->rn_l;
+	p = t->rn_p;
+	if (p->rn_r == t)
+		p->rn_r = x;
+	else
+		p->rn_l = x;
+	x->rn_p = p;
+	/*
+	 * Demote routes attached to us.
+	 */
+	if (t->rn_mklist == NULL)
+		;
+	else if (x->rn_b >= 0) {
+		for (mp = &x->rn_mklist; (m = *mp) != NULL; mp = &m->rm_mklist)
+			;
+		*mp = t->rn_mklist;
+	} else {
+		/* If there are any key,mask pairs in a sibling
+		   duped-key chain, some subset will appear sorted
+		   in the same order attached to our mklist */
+		for (m = t->rn_mklist;
+		     m != NULL && x != NULL;
+		     x = x->rn_dupedkey) {
+			if (m == x->rn_mklist) {
+				struct radix_mask *mm = m->rm_mklist;
+				x->rn_mklist = NULL;
+				if (--(m->rm_refs) < 0)
+					rm_free(m);
+				m = mm;
+			}
+		}
+		if (m != NULL) {
+			log(LOG_ERR, "rn_delete: Orphaned Mask %p at %p\n",
+			    m, x);
+		}
+	}
+	/*
+	 * We may be holding an active internal node in the tree.
+	 */
+	x = tt + 1;
+	if (t != x) {
+		*t = *x;
+		t->rn_l->rn_p = t;
+		t->rn_r->rn_p = t;
+		p = x->rn_p;
+		if (p->rn_l == x)
+			p->rn_l = t;
+		else
+			p->rn_r = t;
+	}
+out:
+#ifdef RN_DEBUG
+	if (rn_debug) {
+		log(LOG_DEBUG, "%s: Coming Out:\n", __func__),
+		    traverse(head, tt);
+	}
+#endif /* RN_DEBUG */
+	tt->rn_flags &= ~RNF_ACTIVE;
+	tt[1].rn_flags &= ~RNF_ACTIVE;
+	return tt;
+}
+
+struct radix_node *
+rn_delete(
+	const void *v_arg,
+	const void *netmask_arg,
+	struct radix_node_head *head)
+{
+	return rn_delete1(v_arg, netmask_arg, head, NULL);
+}
+
+static struct radix_node *
+rn_walknext(struct radix_node *rn, rn_printer_t printer, void *arg)
+{
+	/* If at right child go back up, otherwise, go right */
+	while (rn->rn_p->rn_r == rn && (rn->rn_flags & RNF_ROOT) == 0) {
+		if (printer != NULL)
+			(*printer)(arg, SUBTREE_CLOSE);
+		rn = rn->rn_p;
+	}
+	if (printer)
+		rn_nodeprint(rn->rn_p, printer, arg, "");
+	/* Find the next *leaf* since next node might vanish, too */
+	for (rn = rn->rn_p->rn_r; rn->rn_b >= 0;) {
+		if (printer != NULL)
+			(*printer)(arg, SUBTREE_OPEN);
+		rn = rn->rn_l;
+	}
+	return rn;
+}
+
+static struct radix_node *
+rn_walkfirst(struct radix_node *rn, rn_printer_t printer, void *arg)
+{
+	/* First time through node, go left */
+	while (rn->rn_b >= 0) {
+		if (printer != NULL)
+			(*printer)(arg, SUBTREE_OPEN);
+		rn = rn->rn_l;
+	}
+	return rn;
+}
+
+int
+rn_walktree(
+	struct radix_node_head *h,
+	int (*f)(struct radix_node *, void *),
+	void *w)
+{
+	int error;
+	struct radix_node *base, *next, *rn;
+	/*
+	 * This gets complicated because we may delete the node
+	 * while applying the function f to it, so we need to calculate
+	 * the successor node in advance.
+	 */
+	rn = rn_walkfirst(h->rnh_treetop, NULL, NULL);
+	for (;;) {
+		base = rn;
+		next = rn_walknext(rn, NULL, NULL);
+		/* Process leaves */
+		while ((rn = base) != NULL) {
+			base = rn->rn_dupedkey;
+			if (!(rn->rn_flags & RNF_ROOT) && (error = (*f)(rn, w)))
+				return error;
+		}
+		rn = next;
+		if (rn->rn_flags & RNF_ROOT)
+			return 0;
+	}
+	/* NOTREACHED */
+}
+
+struct radix_node *
+rn_search_matched(struct radix_node_head *h,
+    int (*matcher)(struct radix_node *, void *), void *w)
+{
+	bool matched;
+	struct radix_node *base, *next, *rn;
+	/*
+	 * This gets complicated because we may delete the node
+	 * while applying the function f to it, so we need to calculate
+	 * the successor node in advance.
+	 */
+	rn = rn_walkfirst(h->rnh_treetop, NULL, NULL);
+	for (;;) {
+		base = rn;
+		next = rn_walknext(rn, NULL, NULL);
+		/* Process leaves */
+		while ((rn = base) != NULL) {
+			base = rn->rn_dupedkey;
+			if (!(rn->rn_flags & RNF_ROOT)) {
+				matched = (*matcher)(rn, w);
+				if (matched)
+					return rn;
+			}
+		}
+		rn = next;
+		if (rn->rn_flags & RNF_ROOT)
+			return NULL;
+	}
+	/* NOTREACHED */
+}
+
+int
+rn_inithead(void **head, int off)
+{
+	struct radix_node_head *rnh;
+
+	if (*head != NULL)
+		return 1;
+	R_Malloc(rnh, struct radix_node_head *, sizeof (*rnh));
+	if (rnh == NULL)
+		return 0;
+	*head = rnh;
+	return rn_inithead0(rnh, off);
+}
+
+int
+rn_inithead0(struct radix_node_head *rnh, int off)
+{
+	struct radix_node *t;
+	struct radix_node *tt;
+	struct radix_node *ttt;
+
+	memset(rnh, 0, sizeof(*rnh));
+	t = rn_newpair(rn_zeros, off, rnh->rnh_nodes);
+	ttt = rnh->rnh_nodes + 2;
+	t->rn_r = ttt;
+	t->rn_p = t;
+	tt = t->rn_l;
+	tt->rn_flags = t->rn_flags = RNF_ROOT | RNF_ACTIVE;
+	tt->rn_b = -1 - off;
+	*ttt = *tt;
+	ttt->rn_key = rn_ones;
+	rnh->rnh_addaddr = rn_addroute;
+	rnh->rnh_deladdr = rn_delete;
+	rnh->rnh_matchaddr = rn_match;
+	rnh->rnh_lookup = rn_lookup;
+	rnh->rnh_treetop = t;
+	return 1;
+}
+
+static clib_error_t *
+rn_module_init (vlib_main_t * vm)
+{
+	char *cp, *cplim;
+
+	R_Malloc(rn_zeros, char *, 3 * max_keylen);
+	if (rn_zeros == NULL)
+            return (clib_error_return (0, "RN Zeros..."));
+
+	memset(rn_zeros, 0, 3 * max_keylen);
+	rn_ones = cp = rn_zeros + max_keylen;
+	addmask_key = cplim = rn_ones + max_keylen;
+	while (cp < cplim)
+		*cp++ = -1;
+	if (rn_inithead((void *)&mask_rnhead, 0) == 0)
+            return (clib_error_return (0, "RN Init 2"));
+
+        return (NULL);
+}
+
+VLIB_INIT_FUNCTION(rn_module_init);
diff --git a/src/vnet/util/radix.h b/src/vnet/util/radix.h
new file mode 100644
index 00000000..d9ba6659
--- /dev/null
+++ b/src/vnet/util/radix.h
@@ -0,0 +1,147 @@
+/*	$NetBSD: radix.h,v 1.23 2016/11/15 01:50:06 ozaki-r Exp $	*/
+
+/*
+ * Copyright (c) 1988, 1989, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)radix.h	8.2 (Berkeley) 10/31/94
+ */
+
+#ifndef _NET_RADIX_H_
+#define	_NET_RADIX_H_
+
+#include <vlib/vlib.h>
+
+/*
+ * Radix search tree node layout.
+ */
+
+struct radix_node {
+	struct	radix_mask *rn_mklist;	/* list of masks contained in subtree */
+	struct	radix_node *rn_p;	/* parent */
+	i16	rn_b;			/* bit offset; -1-index(netmask) */
+	u8	rn_bmask;		/* node: mask for bit test*/
+	u8	rn_flags;		/* enumerated next */
+#define RNF_NORMAL	1		/* leaf contains normal route */
+#define RNF_ROOT	2		/* leaf is root leaf for tree */
+#define RNF_ACTIVE	4		/* This node is alive (for rtfree) */
+	union {
+		struct {			/* leaf only data: */
+			const char *rn_Key;	/* object of search */
+			const char *rn_Mask;	/* netmask, if present */
+			struct	radix_node *rn_Dupedkey;
+		} rn_leaf;
+		struct {			/* node only data: */
+			int	rn_Off;		/* where to start compare */
+			struct	radix_node *rn_L;/* progeny */
+			struct	radix_node *rn_R;/* progeny */
+		} rn_node;
+	} rn_u;
+#ifdef RN_DEBUG
+	i32 rn_info;
+	struct radix_node *rn_twin;
+	struct radix_node *rn_ybro;
+#endif
+};
+
+#define rn_dupedkey rn_u.rn_leaf.rn_Dupedkey
+#define rn_key rn_u.rn_leaf.rn_Key
+#define rn_mask rn_u.rn_leaf.rn_Mask
+#define rn_off rn_u.rn_node.rn_Off
+#define rn_l rn_u.rn_node.rn_L
+#define rn_r rn_u.rn_node.rn_R
+
+/*
+ * Annotations to tree concerning potential routes applying to subtrees.
+ */
+
+struct radix_mask {
+	i16	rm_b;			/* bit offset; -1-index(netmask) */
+	i8	rm_unused;		/* cf. rn_bmask */
+	u8	rm_flags;		/* cf. rn_flags */
+	struct	radix_mask *rm_mklist;	/* more masks to try */
+	union	{
+		const char *rmu_mask;		/* the mask */
+		struct	radix_node *rmu_leaf;	/* for normal routes */
+	}	rm_rmu;
+	i32	rm_refs;		/* # of references to this struct */
+};
+
+#define rm_mask rm_rmu.rmu_mask
+#define rm_leaf rm_rmu.rmu_leaf		/* extra field would make 32 bytes */
+
+struct radix_node_head {
+	struct	radix_node *rnh_treetop;
+	i32	rnh_addrsize;		/* permit, but not require fixed keys */
+	i32	rnh_pktsize;		/* permit, but not require fixed keys */
+	struct	radix_node *(*rnh_addaddr)	/* add based on sockaddr */
+		(const void *v, const void *mask,
+		     struct radix_node_head *head, struct radix_node nodes[]);
+	struct	radix_node *(*rnh_addpkt)	/* add based on packet hdr */
+		(const void *v, const void *mask,
+		     struct radix_node_head *head, struct radix_node nodes[]);
+	struct	radix_node *(*rnh_deladdr)	/* remove based on sockaddr */
+		(const void *v, const void *mask, struct radix_node_head *head);
+	struct	radix_node *(*rnh_delpkt)	/* remove based on packet hdr */
+		(const void *v, const void *mask, struct radix_node_head *head);
+	struct	radix_node *(*rnh_matchaddr)	/* locate based on sockaddr */
+		(const void *v, struct radix_node_head *head);
+	struct	radix_node *(*rnh_lookup)	/* locate based on sockaddr */
+		(const void *v, const void *mask, struct radix_node_head *head);
+	struct	radix_node *(*rnh_matchpkt)	/* locate based on packet hdr */
+		(const void *v, struct radix_node_head *head);
+	struct	radix_node rnh_nodes[3];	/* empty tree for common case */
+};
+
+void	rn_init(void);
+int	rn_inithead(void **, int);
+void	rn_delayedinit(void **, int);
+int	rn_inithead0(struct radix_node_head *, int);
+int	rn_refines(const void *, const void *);
+int	rn_walktree(struct radix_node_head *,
+	            int (*)(struct radix_node *, void *),
+		    void *);
+struct radix_node *
+	rn_search_matched(struct radix_node_head *,
+	                  int (*)(struct radix_node *, void *),
+		          void *);
+struct radix_node
+	 *rn_addmask(const void *, int, int),
+	 *rn_addroute(const void *, const void *, struct radix_node_head *,
+			struct radix_node [2]),
+	 *rn_delete1(const void *, const void *, struct radix_node_head *,
+			struct radix_node *),
+	 *rn_delete(const void *, const void *, struct radix_node_head *),
+	 *rn_insert(const void *, struct radix_node_head *, int *,
+			struct radix_node [2]),
+	 *rn_lookup(const void *, const void *, struct radix_node_head *),
+	 *rn_match(const void *, struct radix_node_head *),
+	 *rn_newpair(const void *, int, struct radix_node[2]),
+	 *rn_search(const void *, struct radix_node *),
+	 *rn_search_m(const void *, struct radix_node *, const void *);
+
+#endif /* !_NET_RADIX_H_ */
diff --git a/src/vnet/vxlan/vxlan.c b/src/vnet/vxlan/vxlan.c
index f749429f..9ed42875 100644
--- a/src/vnet/vxlan/vxlan.c
+++ b/src/vnet/vxlan/vxlan.c
@@ -16,7 +16,8 @@
 #include <vnet/ip/format.h>
 #include <vnet/fib/fib_entry.h>
 #include <vnet/fib/fib_table.h>
-#include <vnet/dpo/receive_dpo.h>
+#include <vnet/mfib/mfib_table.h>
+#include <vnet/adj/adj_mcast.h>
 #include <vlib/vlib.h>
 
 /**
@@ -337,7 +338,7 @@ vtep_addr_unref(ip46_address_t *ip)
 
 typedef CLIB_PACKED(union {
   struct {
-    fib_node_index_t fib_entry_index;
+    fib_node_index_t mfib_entry_index;
     adj_index_t mcast_adj_index;
   };
   u64 as_u64;
@@ -353,11 +354,28 @@ mcast_shared_get(ip46_address_t * ip)
 }
 
 static inline void
-ip46_multicast_ethernet_address(u8 * ethernet_address, ip46_address_t * ip) {
-          if (ip46_address_is_ip4(ip))
-              ip4_multicast_ethernet_address(ethernet_address, &ip->ip4);
-          else
-              ip6_multicast_ethernet_address(ethernet_address, ip->ip6.as_u32[0]);
+mcast_shared_add(ip46_address_t *dst,
+                 fib_node_index_t mfei,
+                 adj_index_t ai)
+{
+    mcast_shared_t new_ep = {
+        .mcast_adj_index = ai,
+        .mfib_entry_index = mfei,
+    };
+
+    hash_set_key_copy (&vxlan_main.mcast_shared, dst, new_ep.as_u64);
+}
+
+static inline void
+mcast_shared_remove(ip46_address_t *dst)
+{
+    mcast_shared_t ep = mcast_shared_get(dst);
+
+    adj_unlock(ep.mcast_adj_index);
+    mfib_table_entry_delete_index(ep.mfib_entry_index,
+                                  MFIB_SOURCE_VXLAN);
+
+    hash_unset_key_free (&vxlan_main.mcast_shared, dst);
 }
 
 int vnet_vxlan_add_del_tunnel 
@@ -503,28 +521,65 @@ int vnet_vxlan_add_del_tunnel
 	   */
           fib_protocol_t fp = (is_ip6) ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4;
           dpo_id_t dpo = DPO_INVALID;
-	  dpo_proto_t dproto = fib_proto_to_dpo(fp);
+          mcast_shared_t ep;
 
 	  if (vtep_addr_ref(&t->dst) == 1)
-	    {
-               u8 mcast_mac[6];
-
-               ip46_multicast_ethernet_address(mcast_mac, &t->dst);
-               receive_dpo_add_or_lock(dproto, ~0, NULL, &dpo);
-	       mcast_shared_t new_ep = {
-		 .mcast_adj_index = adj_rewrite_add_and_lock
-                   (fp, fib_proto_to_link(fp), a->mcast_sw_if_index, mcast_mac),
-                 /* Add VRF local mcast adj. */
-                 .fib_entry_index = fib_table_entry_special_dpo_add
-                   (t->encap_fib_index, &tun_dst_pfx,
-                    FIB_SOURCE_SPECIAL, FIB_ENTRY_FLAG_NONE, &dpo)
-		   };
-	       hash_set_key_copy (&vxm->mcast_shared, &t->dst, new_ep.as_u64);
-	       dpo_reset(&dpo);
-	    }
+          {
+              fib_node_index_t mfei;
+              adj_index_t ai;
+              fib_route_path_t path = {
+                  .frp_proto = fp,
+                  .frp_addr = zero_addr,
+                  .frp_sw_if_index = 0xffffffff,
+                  .frp_fib_index = ~0,
+                  .frp_weight = 0,
+                  .frp_flags = FIB_ROUTE_PATH_LOCAL,
+              };
+              const mfib_prefix_t mpfx = {
+                  .fp_proto = fp,
+                  .fp_len = (is_ip6 ? 128 : 32),
+                  .fp_grp_addr = tun_dst_pfx.fp_addr,
+              };
+
+              /*
+               * Setup the (*,G) to receive traffic on the mcast group
+               *  - the forwarding interface is for-us
+               *  - the accepting interface is that from the API
+               */
+              mfib_table_entry_path_update(t->encap_fib_index,
+                                           &mpfx,
+                                           MFIB_SOURCE_VXLAN,
+                                           &path,
+                                           MFIB_ITF_FLAG_FORWARD);
+
+              path.frp_sw_if_index = a->mcast_sw_if_index;
+              path.frp_flags = FIB_ROUTE_PATH_FLAG_NONE;
+              mfei = mfib_table_entry_path_update(t->encap_fib_index,
+                                                  &mpfx,
+                                                  MFIB_SOURCE_VXLAN,
+                                                  &path,
+                                                  MFIB_ITF_FLAG_ACCEPT);
+
+              /*
+               * Create the mcast adjacency to send traffic to the group
+               */
+              ai = adj_mcast_add_or_lock(fp,
+                                         fib_proto_to_link(fp),
+                                         a->mcast_sw_if_index);
+
+              /*
+               * create a new end-point
+               */
+              mcast_shared_add(&t->dst, mfei, ai);
+          }
+
+          ep = mcast_shared_get(&t->dst);
+
           /* Stack shared mcast dst mac addr rewrite on encap */
-	  mcast_shared_t ep = mcast_shared_get(&t->dst);
-          dpo_set (&dpo, DPO_ADJACENCY, dproto, ep.mcast_adj_index);
+          dpo_set (&dpo, DPO_ADJACENCY,
+                   fib_proto_to_dpo(fp),
+                   ep.mcast_adj_index);
+
           dpo_stack_from_node (encap_index, &t->next_dpo, &dpo);
           dpo_reset (&dpo);
 	  flood_class = VNET_FLOOD_CLASS_TUNNEL_MASTER;
@@ -563,10 +618,7 @@ int vnet_vxlan_add_del_tunnel
         }
       else if (vtep_addr_unref(&t->dst) == 0)
         {
-	  mcast_shared_t ep = mcast_shared_get(&t->dst);
-	  adj_unlock(ep.mcast_adj_index);
-	  fib_table_entry_delete_index(ep.fib_entry_index, FIB_SOURCE_SPECIAL);
-          hash_unset_key_free (&vxm->mcast_shared, &t->dst);
+	  mcast_shared_remove(&t->dst);
         }
 
       fib_node_deinit(&t->node);
diff --git a/src/vpp/api/api.c b/src/vpp/api/api.c
index e6227a68..6c8800ec 100644
--- a/src/vpp/api/api.c
+++ b/src/vpp/api/api.c
@@ -964,7 +964,6 @@ ip4_reset_fib_t_handler (vl_api_reset_fib_t * mp)
 
     for (i = 0; i < vec_len (sw_if_indices_to_shut); i++) {
       sw_if_index = sw_if_indices_to_shut[i];
-      // vec_foreach (sw_if_index, sw_if_indices_to_shut) {
 
       u32 flags = vnet_sw_interface_get_flags (vnm, sw_if_index);
       flags &= ~(VNET_SW_INTERFACE_FLAG_ADMIN_UP);
@@ -972,7 +971,6 @@ ip4_reset_fib_t_handler (vl_api_reset_fib_t * mp)
     }
 
     fib_table_flush(fib->index, FIB_PROTOCOL_IP4, FIB_SOURCE_API);
-    fib_table_flush(fib->index, FIB_PROTOCOL_IP4, FIB_SOURCE_INTERFACE);
 
     rv = 0;
     break;
@@ -1013,15 +1011,14 @@ ip6_reset_fib_t_handler (vl_api_reset_fib_t * mp)
 
     /* Shut down interfaces in this FIB / clean out intfc routes */
     pool_foreach (si, im->sw_interfaces,
-                  ({
-                    if (im6->fib_index_by_sw_if_index[si->sw_if_index] ==
-                        fib->index)
-                      vec_add1 (sw_if_indices_to_shut, si->sw_if_index);
-                  }));
+    ({
+      if (im6->fib_index_by_sw_if_index[si->sw_if_index] ==
+          fib->index)
+        vec_add1 (sw_if_indices_to_shut, si->sw_if_index);
+    }));
 
     for (i = 0; i < vec_len (sw_if_indices_to_shut); i++) {
       sw_if_index = sw_if_indices_to_shut[i];
-      // vec_foreach (sw_if_index, sw_if_indices_to_shut) {
 
       u32 flags = vnet_sw_interface_get_flags (vnm, sw_if_index);
       flags &= ~(VNET_SW_INTERFACE_FLAG_ADMIN_UP);
@@ -1029,7 +1026,6 @@ ip6_reset_fib_t_handler (vl_api_reset_fib_t * mp)
     }
 
     fib_table_flush(fib->index, FIB_PROTOCOL_IP6, FIB_SOURCE_API);
-    fib_table_flush(fib->index, FIB_PROTOCOL_IP6, FIB_SOURCE_INTERFACE);
 
     rv = 0;
     break;
diff --git a/src/vppinfra.am b/src/vppinfra.am
index 19485d2d..724f21c9 100644
--- a/src/vppinfra.am
+++ b/src/vppinfra.am
@@ -38,7 +38,7 @@ TESTS  +=  test_bihash_template \
 	   test_time \
 	   test_timing_wheel \
 	   test_vec \
-	   test_zvec 
+	   test_zvec
 endif
 
 noinst_PROGRAMS = $(TESTS)
diff --git a/src/vppinfra/dlist.h b/src/vppinfra/dlist.h
index 7d09b2bb..e445b39f 100644
--- a/src/vppinfra/dlist.h
+++ b/src/vppinfra/dlist.h
@@ -121,7 +121,7 @@ clib_dlist_remove_head (dlist_elt_t * pool, u32 head_index)
 
   ASSERT (head->value == ~0);
 
-  if (head->next == ~0)
+  if (head->next == ~0 || (head->next == head_index))
     return ~0;
 
   rv = head->next;
diff --git a/src/vppinfra/format.c b/src/vppinfra/format.c
index 78e52e9a..999b093c 100644
--- a/src/vppinfra/format.c
+++ b/src/vppinfra/format.c
@@ -150,13 +150,13 @@ justify (u8 * s, format_info_t * fi, uword s_len_orig)
   return s;
 }
 
-static u8 *
-do_percent (u8 ** _s, u8 * fmt, va_list * va)
+static const u8 *
+do_percent (u8 ** _s, const u8 * fmt, va_list * va)
 {
   u8 *s = *_s;
   uword c;
 
-  u8 *f = fmt;
+  const u8 *f = fmt;
 
   format_info_t fi = {
     .justify = '+',
@@ -385,7 +385,7 @@ done:
 u8 *
 va_format (u8 * s, const char *fmt, va_list * va)
 {
-  u8 *f = (u8 *) fmt, *g;
+  const u8 *f = (u8 *) fmt, *g;
   u8 c;
 
   g = f;
diff --git a/src/vppinfra/format.h b/src/vppinfra/format.h
index bc0d6d15..bec1b6b4 100644
--- a/src/vppinfra/format.h
+++ b/src/vppinfra/format.h
@@ -234,7 +234,7 @@ typedef uword (unformat_function_t) (unformat_input_t * input,
 /* External functions. */
 
 /* General unformatting function with programmable input stream. */
-uword unformat (unformat_input_t * i, char *fmt, ...);
+uword unformat (unformat_input_t * i, const char *fmt, ...);
 
 /* Call user defined parse function.
    unformat_user (i, f, ...) is equivalent to unformat (i, "%U", f, ...) */
@@ -242,7 +242,7 @@ uword unformat_user (unformat_input_t * input, unformat_function_t * func,
 		     ...);
 
 /* Alternate version which allows for extensions. */
-uword va_unformat (unformat_input_t * i, char *fmt, va_list * args);
+uword va_unformat (unformat_input_t * i, const char *fmt, va_list * args);
 
 /* Setup for unformat of Unix style command line. */
 void unformat_init_command_line (unformat_input_t * input, char *argv[]);
diff --git a/src/vppinfra/unformat.c b/src/vppinfra/unformat.c
index ac8b7ddc..7c636ccc 100644
--- a/src/vppinfra/unformat.c
+++ b/src/vppinfra/unformat.c
@@ -681,8 +681,8 @@ error:
   return 0;
 }
 
-static char *
-match_input_with_format (unformat_input_t * input, char *f)
+static const char *
+match_input_with_format (unformat_input_t * input, const char *f)
 {
   uword cf, ci;
 
@@ -703,8 +703,8 @@ match_input_with_format (unformat_input_t * input, char *f)
   return f;
 }
 
-static char *
-do_percent (unformat_input_t * input, va_list * va, char *f)
+static const char *
+do_percent (unformat_input_t * input, va_list * va, const char *f)
 {
   uword cf, n, data_bytes = ~0;
 
@@ -824,9 +824,9 @@ unformat_skip_white_space (unformat_input_t * input)
 }
 
 uword
-va_unformat (unformat_input_t * input, char *fmt, va_list * va)
+va_unformat (unformat_input_t * input, const char *fmt, va_list * va)
 {
-  char *f;
+  const char *f;
   uword input_matches_format;
   uword default_skip_input_white_space;
   uword n_input_white_space_skipped;
@@ -937,7 +937,7 @@ va_unformat (unformat_input_t * input, char *fmt, va_list * va)
 
       else
 	{
-	  char *g = match_input_with_format (input, f);
+	  const char *g = match_input_with_format (input, f);
 	  if (!g)
 	    goto parse_fail;
 	  last_non_white_space_match_format = g > f;
@@ -963,7 +963,7 @@ parse_fail:
 }
 
 uword
-unformat (unformat_input_t * input, char *fmt, ...)
+unformat (unformat_input_t * input, const char *fmt, ...)
 {
   va_list va;
   uword result;
diff --git a/test/test_dhcp.py b/test/test_dhcp.py
index bdff679c..04ab2e11 100644
--- a/test/test_dhcp.py
+++ b/test/test_dhcp.py
@@ -4,8 +4,6 @@ import unittest
 import socket
 
 from framework import VppTestCase, VppTestRunner
-from vpp_ip_route import IpRoute, RoutePath
-from vpp_lo_interface import VppLoInterface
 
 from scapy.layers.l2 import Ether, getmacbyip
 from scapy.layers.inet import IP, UDP, ICMP
@@ -482,17 +480,6 @@ class TestDHCP(VppTestCase):
         server_addr_vrf1 = self.pg1.remote_ip6n
         src_addr_vrf1 = self.pg1.local_ip6n
 
-        #
-        # Add the Route to receive the DHCP packets
-        #
-        route_dhcp_vrf0 = IpRoute(self, dhcp_solicit_dst, 128,
-                                  [], is_local=1, is_ip6=1)
-        route_dhcp_vrf0.add_vpp_config()
-        route_dhcp_vrf1 = IpRoute(self, dhcp_solicit_dst, 128,
-                                  [], is_local=1, is_ip6=1,
-                                  table_id=1)
-        route_dhcp_vrf1.add_vpp_config()
-
         dmac = in6_getnsmac(inet_pton(socket.AF_INET6, dhcp_solicit_dst))
         p_solicit_vrf0 = (Ether(dst=dmac, src=self.pg2.remote_mac) /
                           IPv6(src=dhcp_solicit_src_vrf0,
@@ -732,8 +719,5 @@ class TestDHCP(VppTestCase):
                                     is_ipv6=1,
                                     is_add=0)
 
-        route_dhcp_vrf0.remove_vpp_config()
-        route_dhcp_vrf1.remove_vpp_config()
-
 if __name__ == '__main__':
     unittest.main(testRunner=VppTestRunner)
diff --git a/test/test_ip6.py b/test/test_ip6.py
index ea669b70..e188970a 100644
--- a/test/test_ip6.py
+++ b/test/test_ip6.py
@@ -5,6 +5,7 @@ import socket
 
 from framework import VppTestCase, VppTestRunner
 from vpp_sub_interface import VppSubInterface, VppDot1QSubint
+from vpp_pg_interface import is_ipv6_misc
 
 from scapy.packet import Raw
 from scapy.layers.l2 import Ether, Dot1Q
@@ -12,10 +13,9 @@ from scapy.layers.inet6 import IPv6, UDP, ICMPv6ND_NS, ICMPv6ND_RS, \
     ICMPv6ND_RA, ICMPv6NDOptSrcLLAddr, getmacbyip6, ICMPv6MRD_Solicitation
 from util import ppp
 from scapy.utils6 import in6_getnsma, in6_getnsmac, in6_ptop, in6_islladdr, \
-    in6_mactoifaceid
+    in6_mactoifaceid, in6_ismaddr
 from scapy.utils import inet_pton, inet_ntop
 
-
 def mk_ll_addr(mac):
     euid = in6_mactoifaceid(mac)
     addr = "fe80::" + euid
@@ -287,28 +287,39 @@ class TestIPv6(VppTestCase):
         self.send_and_assert_no_replies(self.pg0, pkts,
                                         "No response to NS for unknown target")
 
-    def send_and_expect_ra(self, intf, pkts, remark, src_ip=None):
-        if not src_ip:
-            src_ip = intf.remote_ip6
-        intf.add_stream(pkts)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        rx = intf.get_capture(1)
+    def validate_ra(self, intf, rx, dst_ip=None):
+        if not dst_ip:
+            dst_ip = intf.remote_ip6
 
-        self.assertEqual(len(rx), 1)
-        rx = rx[0]
+        # unicasted packets must come to the unicast mac 
+        self.assertEqual(rx[Ether].dst, intf.remote_mac)
+
+        # and from the router's MAC
+        self.assertEqual(rx[Ether].src, intf.local_mac)
 
         # the rx'd RA should be addressed to the sender's source
         self.assertTrue(rx.haslayer(ICMPv6ND_RA))
         self.assertEqual(in6_ptop(rx[IPv6].dst),
-                         in6_ptop(src_ip))
+                         in6_ptop(dst_ip))
 
         # and come from the router's link local
         self.assertTrue(in6_islladdr(rx[IPv6].src))
         self.assertEqual(in6_ptop(rx[IPv6].src),
                          in6_ptop(mk_ll_addr(intf.local_mac)))
 
+
+    def send_and_expect_ra(self, intf, pkts, remark, dst_ip=None,
+                           filter_out_fn=is_ipv6_misc):
+        intf.add_stream(pkts)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        rx = intf.get_capture(1, filter_out_fn=filter_out_fn)
+
+        self.assertEqual(len(rx), 1)
+        rx = rx[0]
+        self.validate_ra(intf, rx, dst_ip)
+
     def test_rs(self):
         """ IPv6 Router Solicitation Exceptions
 
@@ -319,6 +330,9 @@ class TestIPv6(VppTestCase):
         # Before we begin change the IPv6 RA responses to use the unicast
         # address - that way we will not confuse them with the periodic
         # RAs which go to the mcast address
+        # Sit and wait for the first periodic RA.
+        #
+        # TODO
         #
         self.pg0.ip6_ra_config(send_unicast=1)
 
@@ -365,8 +379,23 @@ class TestIPv6(VppTestCase):
              IPv6(dst=self.pg0.local_ip6, src=ll) /
              ICMPv6ND_RS())
         pkts = [p]
-        self.send_and_expect_ra(
-            self.pg0, pkts, "RS sourced from link-local", src_ip=ll)
+        self.send_and_expect_ra(self.pg0, pkts,
+                                "RS sourced from link-local",
+                                dst_ip=ll)
+
+        #
+        # Send the RS multicast
+        #
+        self.pg0.ip6_ra_config(send_unicast=1)
+        dmac = in6_getnsmac(inet_pton(socket.AF_INET6, "ff02::2"))
+        ll = mk_ll_addr(self.pg0.remote_mac)
+        p = (Ether(dst=dmac, src=self.pg0.remote_mac) /
+             IPv6(dst="ff02::2", src=ll) /
+             ICMPv6ND_RS())
+        pkts = [p]
+        self.send_and_expect_ra(self.pg0, pkts,
+                                "RS sourced from link-local",
+                                dst_ip=ll)
 
         #
         # Source from the unspecified address ::. This happens when the RS
@@ -376,74 +405,20 @@ class TestIPv6(VppTestCase):
         # If we happen to pick up the periodic RA at this point then so be it,
         # it's not an error.
         #
-        self.pg0.ip6_ra_config(send_unicast=1)
-        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
-             IPv6(dst=self.pg0.local_ip6, src="::") /
+        self.pg0.ip6_ra_config(send_unicast=1, suppress=1)
+        p = (Ether(dst=dmac, src=self.pg0.remote_mac) /
+             IPv6(dst="ff02::2", src="::") /
              ICMPv6ND_RS())
         pkts = [p]
-
-        self.pg0.add_stream(pkts)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg0.get_capture(1, filter_out_fn=None)
-        found = 0
-        for rx in capture:
-            if (rx.haslayer(ICMPv6ND_RA)):
-                # and come from the router's link local
-                self.assertTrue(in6_islladdr(rx[IPv6].src))
-                self.assertEqual(in6_ptop(rx[IPv6].src),
-                                 in6_ptop(mk_ll_addr(self.pg0.local_mac)))
-                # sent to the all hosts mcast
-                self.assertEqual(in6_ptop(rx[IPv6].dst), "ff02::1")
-
-                found = 1
-        self.assertTrue(found)
-
-    @unittest.skip("Unsupported")
-    def test_mrs(self):
-        """ IPv6 Multicast Router Solicitation Exceptions
-
-        Test scenario:
-        """
-
-        #
-        # An RS from a link source address
-        #  - expect an RA in return
-        #
-        nsma = in6_getnsma(inet_pton(socket.AF_INET6, self.pg0.local_ip6))
-        d = inet_ntop(socket.AF_INET6, nsma)
-
-        p = (Ether(dst=getmacbyip6("ff02::2")) /
-             IPv6(dst=d, src=self.pg0.remote_ip6) /
-             ICMPv6MRD_Solicitation())
-        pkts = [p]
-
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        self.pg0.assert_nothing_captured(
-            remark="No response to NS source by address not on sub-net")
+        self.send_and_expect_ra(self.pg0, pkts,
+                                "RS sourced from unspecified",
+                                dst_ip="ff02::1",
+                                filter_out_fn=None)
 
         #
-        # An RS from a non link source address
+        # Reset the periodic advertisements back to default values 
         #
-        nsma = in6_getnsma(inet_pton(socket.AF_INET6, self.pg0.local_ip6))
-        d = inet_ntop(socket.AF_INET6, nsma)
-
-        p = (Ether(dst=getmacbyip6("ff02::2")) /
-             IPv6(dst=d, src="2002::2") /
-             ICMPv6MRD_Solicitation())
-        pkts = [p]
-
-        self.send_and_assert_no_replies(self.pg0, pkts,
-                                        "RA rate limited")
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        self.pg0.assert_nothing_captured(
-            remark="No response to NS source by address not on sub-net")
-
+        self.pg0.ip6_ra_config(no=1, suppress=1, send_unicast=0)
 
 if __name__ == '__main__':
     unittest.main(testRunner=VppTestRunner)
diff --git a/test/test_ip_mcast.py b/test/test_ip_mcast.py
new file mode 100644
index 00000000..028853d2
--- /dev/null
+++ b/test/test_ip_mcast.py
@@ -0,0 +1,612 @@
+#!/usr/bin/env python
+
+import unittest
+
+from framework import VppTestCase, VppTestRunner
+from vpp_sub_interface import VppSubInterface, VppDot1QSubint, VppDot1ADSubint
+from vpp_ip_route import IpMRoute, MRoutePath, MFibSignal
+
+from scapy.packet import Raw
+from scapy.layers.l2 import Ether
+from scapy.layers.inet import IP, UDP, getmacbyip
+from scapy.layers.inet6 import IPv6, getmacbyip6
+from util import ppp
+
+
+class MRouteItfFlags:
+    MFIB_ITF_FLAG_NONE = 0
+    MFIB_ITF_FLAG_NEGATE_SIGNAL = 1
+    MFIB_ITF_FLAG_ACCEPT = 2
+    MFIB_ITF_FLAG_FORWARD = 4
+    MFIB_ITF_FLAG_SIGNAL_PRESENT = 8
+    MFIB_ITF_FLAG_INTERNAL_COPY = 16
+
+
+class MRouteEntryFlags:
+    MFIB_ENTRY_FLAG_NONE = 0
+    MFIB_ENTRY_FLAG_SIGNAL = 1
+    MFIB_ENTRY_FLAG_DROP = 2
+    MFIB_ENTRY_FLAG_CONNECTED = 4
+    MFIB_ENTRY_FLAG_INHERIT_ACCEPT = 8
+
+
+class TestIPMcast(VppTestCase):
+    """ IP Multicast Test Case """
+
+    def setUp(self):
+        super(TestIPMcast, self).setUp()
+
+        # create 4 pg interfaces
+        self.create_pg_interfaces(range(4))
+
+        # setup interfaces
+        for i in self.pg_interfaces:
+            i.admin_up()
+            i.config_ip4()
+            i.config_ip6()
+            i.resolve_arp()
+            i.resolve_ndp()
+
+    def create_stream_ip4(self, src_if, src_ip, dst_ip):
+        pkts = []
+        for i in range(0, 65):
+            info = self.create_packet_info(src_if, src_if)
+            payload = self.info_to_payload(info)
+            p = (Ether(dst=src_if.local_mac, src=src_if.remote_mac) /
+                 IP(src=src_ip, dst=dst_ip) /
+                 UDP(sport=1234, dport=1234) /
+                 Raw(payload))
+            info.data = p.copy()
+            pkts.append(p)
+        return pkts
+
+    def create_stream_ip6(self, src_if, src_ip, dst_ip):
+        pkts = []
+        for i in range(0, 65):
+            info = self.create_packet_info(src_if, src_if)
+            payload = self.info_to_payload(info)
+            p = (Ether(dst=src_if.local_mac, src=src_if.remote_mac) /
+                 IPv6(src=src_ip, dst=dst_ip) /
+                 UDP(sport=1234, dport=1234) /
+                 Raw(payload))
+            info.data = p.copy()
+            pkts.append(p)
+        return pkts
+
+    def verify_filter(self, capture, sent):
+        if not len(capture) == len(sent):
+            # filter out any IPv6 RAs from the captur
+            for p in capture:
+                if (p.haslayer(IPv6)):
+                    capture.remove(p)
+        return capture
+
+    def verify_capture_ip4(self, src_if, sent):
+        rxd = self.pg1.get_capture(65)
+
+        try:
+            capture = self.verify_filter(rxd, sent)
+
+            self.assertEqual(len(capture), len(sent))
+
+            for i in range(len(capture)):
+                tx = sent[i]
+                rx = capture[i]
+
+                # the rx'd packet has the MPLS label popped
+                eth = rx[Ether]
+                self.assertEqual(eth.type, 0x800)
+
+                tx_ip = tx[IP]
+                rx_ip = rx[IP]
+
+                # check the MAC address on the RX'd packet is correctly formed
+                self.assertEqual(eth.dst, getmacbyip(rx_ip.dst))
+
+                self.assertEqual(rx_ip.src, tx_ip.src)
+                self.assertEqual(rx_ip.dst, tx_ip.dst)
+                # IP processing post pop has decremented the TTL
+                self.assertEqual(rx_ip.ttl + 1, tx_ip.ttl)
+
+        except:
+            raise
+
+    def verify_capture_ip6(self, src_if, sent):
+        capture = self.pg1.get_capture(65)
+
+        self.assertEqual(len(capture), len(sent))
+
+        for i in range(len(capture)):
+            tx = sent[i]
+            rx = capture[i]
+
+            # the rx'd packet has the MPLS label popped
+            eth = rx[Ether]
+            self.assertEqual(eth.type, 0x86DD)
+
+            tx_ip = tx[IPv6]
+            rx_ip = rx[IPv6]
+
+            # check the MAC address on the RX'd packet is correctly formed
+            self.assertEqual(eth.dst, getmacbyip6(rx_ip.dst))
+
+            self.assertEqual(rx_ip.src, tx_ip.src)
+            self.assertEqual(rx_ip.dst, tx_ip.dst)
+            # IP processing post pop has decremented the TTL
+            self.assertEqual(rx_ip.hlim + 1, tx_ip.hlim)
+
+    def test_ip_mcast(self):
+        """ IP Multicast Replication """
+
+        #
+        # a stream that matches the default route. gets dropped.
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip4(self.pg0, "1.1.1.1", "232.1.1.1")
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        self.pg0.assert_nothing_captured(
+            remark="IP multicast packets forwarded on default route")
+
+        #
+        # A (*,G).
+        # one accepting interface, pg0, 3 forwarding interfaces
+        #
+        route_232_1_1_1 = IpMRoute(
+            self,
+            "0.0.0.0",
+            "232.1.1.1", 32,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            [MRoutePath(self.pg0.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT),
+             MRoutePath(self.pg1.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD),
+             MRoutePath(self.pg2.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD),
+             MRoutePath(self.pg3.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)])
+        route_232_1_1_1.add_vpp_config()
+
+        #
+        # An (S,G).
+        # one accepting interface, pg0, 2 forwarding interfaces
+        #
+        route_1_1_1_1_232_1_1_1 = IpMRoute(
+            self,
+            "1.1.1.1",
+            "232.1.1.1", 64,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            [MRoutePath(self.pg0.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT),
+             MRoutePath(self.pg1.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD),
+             MRoutePath(self.pg2.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)])
+        route_1_1_1_1_232_1_1_1.add_vpp_config()
+
+        #
+        # An (*,G/m).
+        # one accepting interface, pg0, 1 forwarding interfaces
+        #
+        route_232 = IpMRoute(
+            self,
+            "0.0.0.0",
+            "232.0.0.0", 8,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            [MRoutePath(self.pg0.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT),
+             MRoutePath(self.pg1.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)])
+        route_232.add_vpp_config()
+
+        #
+        # a stream that matches the route for (1.1.1.1,232.1.1.1)
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip4(self.pg0, "1.1.1.1", "232.1.1.1")
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # We expect replications on Pg1, 2,
+        self.verify_capture_ip4(self.pg1, tx)
+        self.verify_capture_ip4(self.pg2, tx)
+
+        # no replications on Pg0
+        self.pg0.assert_nothing_captured(
+            remark="IP multicast packets forwarded on PG0")
+        self.pg3.assert_nothing_captured(
+            remark="IP multicast packets forwarded on PG3")
+
+        #
+        # a stream that matches the route for (*,232.0.0.0/8)
+        # Send packets with the 9th bit set so we test the correct clearing
+        # of that bit in the mac rewrite
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip4(self.pg0, "1.1.1.1", "232.255.255.255")
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # We expect replications on Pg1 only
+        self.verify_capture_ip4(self.pg1, tx)
+
+        # no replications on Pg0, Pg2 not Pg3
+        self.pg0.assert_nothing_captured(
+            remark="IP multicast packets forwarded on PG0")
+        self.pg2.assert_nothing_captured(
+            remark="IP multicast packets forwarded on PG2")
+        self.pg3.assert_nothing_captured(
+            remark="IP multicast packets forwarded on PG3")
+
+        #
+        # a stream that matches the route for (*,232.1.1.1)
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip4(self.pg0, "1.1.1.2", "232.1.1.1")
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # We expect replications on Pg1, 2, 3.
+        self.verify_capture_ip4(self.pg1, tx)
+        self.verify_capture_ip4(self.pg2, tx)
+        self.verify_capture_ip4(self.pg3, tx)
+
+        # no replications on Pg0
+        self.pg0.assert_nothing_captured(
+            remark="IP multicast packets forwarded on PG0")
+
+        route_232_1_1_1.remove_vpp_config()
+        route_1_1_1_1_232_1_1_1.remove_vpp_config()
+        route_232.remove_vpp_config()
+
+    def test_ip6_mcast(self):
+        """ IPv6 Multicast Replication """
+
+        #
+        # a stream that matches the default route. gets dropped.
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip6(self.pg0, "2001::1", "ff01::1")
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        self.pg0.assert_nothing_captured(
+            remark="IPv6 multicast packets forwarded on default route")
+
+        #
+        # A (*,G).
+        # one accepting interface, pg0, 3 forwarding interfaces
+        #
+        route_ff01_1 = IpMRoute(
+            self,
+            "::",
+            "ff01::1", 128,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            [MRoutePath(self.pg0.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT),
+             MRoutePath(self.pg1.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD),
+             MRoutePath(self.pg2.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD),
+             MRoutePath(self.pg3.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)],
+            is_ip6=1)
+        route_ff01_1.add_vpp_config()
+
+        #
+        # An (S,G).
+        # one accepting interface, pg0, 2 forwarding interfaces
+        #
+        route_2001_ff01_1 = IpMRoute(
+            self,
+            "2001::1",
+            "ff01::1", 256,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            [MRoutePath(self.pg0.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT),
+             MRoutePath(self.pg1.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD),
+             MRoutePath(self.pg2.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)],
+            is_ip6=1)
+        route_2001_ff01_1.add_vpp_config()
+
+        #
+        # An (*,G/m).
+        # one accepting interface, pg0, 1 forwarding interface
+        #
+        route_ff01 = IpMRoute(
+            self,
+            "::",
+            "ff01::", 16,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            [MRoutePath(self.pg0.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT),
+             MRoutePath(self.pg1.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)],
+            is_ip6=1)
+        route_ff01.add_vpp_config()
+
+        #
+        # a stream that matches the route for (*, ff01::/16)
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip6(self.pg0, "2002::1", "ff01:2::255")
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # We expect replications on Pg1
+        self.verify_capture_ip6(self.pg1, tx)
+
+        # no replications on Pg0, Pg3
+        self.pg0.assert_nothing_captured(
+            remark="IP multicast packets forwarded on PG0")
+        self.pg2.assert_nothing_captured(
+            remark="IP multicast packets forwarded on PG2")
+        self.pg3.assert_nothing_captured(
+            remark="IP multicast packets forwarded on PG3")
+
+        #
+        # a stream that matches the route for (*,ff01::1)
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip6(self.pg0, "2002::2", "ff01::1")
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # We expect replications on Pg1, 2, 3.
+        self.verify_capture_ip6(self.pg1, tx)
+        self.verify_capture_ip6(self.pg2, tx)
+        self.verify_capture_ip6(self.pg3, tx)
+
+        # no replications on Pg0
+        self.pg0.assert_nothing_captured(
+            remark="IPv6 multicast packets forwarded on PG0")
+
+        #
+        # a stream that matches the route for (2001::1, ff00::1)
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip6(self.pg0, "2001::1", "ff01::1")
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # We expect replications on Pg1, 2,
+        self.verify_capture_ip6(self.pg1, tx)
+        self.verify_capture_ip6(self.pg2, tx)
+
+        # no replications on Pg0, Pg3
+        self.pg0.assert_nothing_captured(
+            remark="IP multicast packets forwarded on PG0")
+        self.pg3.assert_nothing_captured(
+            remark="IP multicast packets forwarded on PG3")
+
+        route_ff01.remove_vpp_config()
+        route_ff01_1.remove_vpp_config()
+        route_2001_ff01_1.remove_vpp_config()
+
+    def _mcast_connected_send_stream(self, dst_ip):
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip4(self.pg0,
+                                    self.pg0.remote_ip4,
+                                    dst_ip)
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # We expect replications on Pg1.
+        self.verify_capture_ip4(self.pg1, tx)
+
+        return tx
+
+    def test_ip_mcast_connected(self):
+        """ IP Multicast Connected Source check """
+
+        #
+        # A (*,G).
+        # one accepting interface, pg0, 1 forwarding interfaces
+        #
+        route_232_1_1_1 = IpMRoute(
+            self,
+            "0.0.0.0",
+            "232.1.1.1", 32,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            [MRoutePath(self.pg0.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT),
+             MRoutePath(self.pg1.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)])
+
+        route_232_1_1_1.add_vpp_config()
+        route_232_1_1_1.update_entry_flags(
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_CONNECTED)
+
+        #
+        # Now the (*,G) is present, send from connected source
+        #
+        tx = self._mcast_connected_send_stream("232.1.1.1")
+
+        #
+        # Constrct a representation of the signal we expect on pg0
+        #
+        signal_232_1_1_1_itf_0 = MFibSignal(self,
+                                            route_232_1_1_1,
+                                            self.pg0.sw_if_index,
+                                            tx[0])
+
+        #
+        # read the only expected signal
+        #
+        signals = self.vapi.mfib_signal_dump()
+
+        self.assertEqual(1, len(signals))
+
+        signal_232_1_1_1_itf_0.compare(signals[0])
+
+        #
+        # reading the signal allows for the generation of another
+        # so send more packets and expect the next signal
+        #
+        tx = self._mcast_connected_send_stream("232.1.1.1")
+
+        signals = self.vapi.mfib_signal_dump()
+        self.assertEqual(1, len(signals))
+        signal_232_1_1_1_itf_0.compare(signals[0])
+
+        #
+        # A Second entry with connected check
+        # one accepting interface, pg0, 1 forwarding interfaces
+        #
+        route_232_1_1_2 = IpMRoute(
+            self,
+            "0.0.0.0",
+            "232.1.1.2", 32,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            [MRoutePath(self.pg0.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT),
+             MRoutePath(self.pg1.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)])
+
+        route_232_1_1_2.add_vpp_config()
+        route_232_1_1_2.update_entry_flags(
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_CONNECTED)
+
+        #
+        # Send traffic to both entries. One read should net us two signals
+        #
+        signal_232_1_1_2_itf_0 = MFibSignal(self,
+                                            route_232_1_1_2,
+                                            self.pg0.sw_if_index,
+                                            tx[0])
+        tx = self._mcast_connected_send_stream("232.1.1.1")
+        tx2 = self._mcast_connected_send_stream("232.1.1.2")
+
+        #
+        # read the only expected signal
+        #
+        signals = self.vapi.mfib_signal_dump()
+
+        self.assertEqual(2, len(signals))
+
+        signal_232_1_1_1_itf_0.compare(signals[1])
+        signal_232_1_1_2_itf_0.compare(signals[0])
+
+        route_232_1_1_1.remove_vpp_config()
+        route_232_1_1_2.remove_vpp_config()
+
+    def test_ip_mcast_signal(self):
+        """ IP Multicast Signal """
+
+        #
+        # A (*,G).
+        # one accepting interface, pg0, 1 forwarding interfaces
+        #
+        route_232_1_1_1 = IpMRoute(
+            self,
+            "0.0.0.0",
+            "232.1.1.1", 32,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            [MRoutePath(self.pg0.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT),
+             MRoutePath(self.pg1.sw_if_index,
+                        MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)])
+
+        route_232_1_1_1.add_vpp_config()
+        route_232_1_1_1.update_entry_flags(
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_SIGNAL)
+
+        #
+        # Now the (*,G) is present, send from connected source
+        #
+        tx = self._mcast_connected_send_stream("232.1.1.1")
+
+        #
+        # Constrct a representation of the signal we expect on pg0
+        #
+        signal_232_1_1_1_itf_0 = MFibSignal(self,
+                                            route_232_1_1_1,
+                                            self.pg0.sw_if_index,
+                                            tx[0])
+
+        #
+        # read the only expected signal
+        #
+        signals = self.vapi.mfib_signal_dump()
+
+        self.assertEqual(1, len(signals))
+
+        signal_232_1_1_1_itf_0.compare(signals[0])
+
+        #
+        # reading the signal allows for the generation of another
+        # so send more packets and expect the next signal
+        #
+        tx = self._mcast_connected_send_stream("232.1.1.1")
+
+        signals = self.vapi.mfib_signal_dump()
+        self.assertEqual(1, len(signals))
+        signal_232_1_1_1_itf_0.compare(signals[0])
+
+        #
+        # Set the negate-signal on the accepting interval - the signals
+        # should stop
+        #
+        route_232_1_1_1.update_path_flags(
+            self.pg0.sw_if_index,
+            (MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT |
+             MRouteItfFlags.MFIB_ITF_FLAG_NEGATE_SIGNAL))
+
+        tx = self._mcast_connected_send_stream("232.1.1.1")
+
+        signals = self.vapi.mfib_signal_dump()
+        self.assertEqual(0, len(signals))
+
+        #
+        # Clear the SIGNAL flag on the entry and the signals should
+        # come back since the interface is still NEGATE-SIGNAL
+        #
+        route_232_1_1_1.update_entry_flags(
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE)
+
+        tx = self._mcast_connected_send_stream("232.1.1.1")
+
+        signals = self.vapi.mfib_signal_dump()
+        self.assertEqual(1, len(signals))
+        signal_232_1_1_1_itf_0.compare(signals[0])
+
+        #
+        # Lastly remove the NEGATE-SIGNAL from the interface and the
+        # signals should stop
+        #
+        route_232_1_1_1.update_path_flags(self.pg0.sw_if_index,
+                                          MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT)
+
+        tx = self._mcast_connected_send_stream("232.1.1.1")
+        signals = self.vapi.mfib_signal_dump()
+        self.assertEqual(0, len(signals))
+
+        #
+        # Cleanup
+        #
+        route_232_1_1_1.remove_vpp_config()
+
+
+if __name__ == '__main__':
+    unittest.main(testRunner=VppTestRunner)
diff --git a/test/test_mfib.py b/test/test_mfib.py
new file mode 100644
index 00000000..4d0d2a3f
--- /dev/null
+++ b/test/test_mfib.py
@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+import unittest
+
+from framework import VppTestCase, VppTestRunner
+
+
+class TestMFIB(VppTestCase):
+    """ MFIB Test Case """
+
+    def setUp(self):
+        super(TestMFIB, self).setUp()
+
+    def test_mfib(self):
+        """ MFIB Unit Tests """
+        error = self.vapi.cli("test mfib")
+
+        if error:
+            self.logger.critical(error)
+        self.assertEqual(error.find("Failed"), -1)
+
+if __name__ == '__main__':
+    unittest.main(testRunner=VppTestRunner)
diff --git a/test/vpp_interface.py b/test/vpp_interface.py
index ee4a9ef6..29edb70e 100644
--- a/test/vpp_interface.py
+++ b/test/vpp_interface.py
@@ -260,9 +260,10 @@ class VppInterface(object):
         """Configure IPv6 RA suppress on the VPP interface."""
         self.test.vapi.sw_interface_ra_suppress(self.sw_if_index)
 
-    def ip6_ra_config(self, suppress=0, send_unicast=0):
+    def ip6_ra_config(self, no=0, suppress=0, send_unicast=0):
         """Configure IPv6 RA suppress on the VPP interface."""
         self.test.vapi.ip6_sw_interface_ra_config(self.sw_if_index,
+                                                  no,
                                                   suppress,
                                                   send_unicast)
 
diff --git a/test/vpp_ip_route.py b/test/vpp_ip_route.py
index fc9133fb..1804fbbd 100644
--- a/test/vpp_ip_route.py
+++ b/test/vpp_ip_route.py
@@ -11,7 +11,7 @@ MPLS_IETF_MAX_LABEL = 0xfffff
 MPLS_LABEL_INVALID = MPLS_IETF_MAX_LABEL + 1
 
 
-class RoutePath:
+class RoutePath(object):
 
     def __init__(
             self,
@@ -31,6 +31,14 @@ class RoutePath:
             self.nh_addr = socket.inet_pton(socket.AF_INET, nh_addr)
 
 
+class MRoutePath(RoutePath):
+
+    def __init__(self, nh_sw_if_index, flags):
+        super(MRoutePath, self).__init__("0.0.0.0",
+                                         nh_sw_if_index)
+        self.nh_i_flags = flags
+
+
 class IpRoute:
     """
     IP Route
@@ -94,6 +102,97 @@ class IpRoute:
                                                  is_add=0)
 
 
+class IpMRoute:
+    """
+    IP Multicast Route
+    """
+
+    def __init__(self, test, src_addr, grp_addr,
+                 grp_addr_len, e_flags, paths, table_id=0, is_ip6=0):
+        self._test = test
+        self.paths = paths
+        self.grp_addr_len = grp_addr_len
+        self.table_id = table_id
+        self.e_flags = e_flags
+        self.is_ip6 = is_ip6
+
+        if is_ip6:
+            self.grp_addr = socket.inet_pton(socket.AF_INET6, grp_addr)
+            self.src_addr = socket.inet_pton(socket.AF_INET6, src_addr)
+        else:
+            self.grp_addr = socket.inet_pton(socket.AF_INET, grp_addr)
+            self.src_addr = socket.inet_pton(socket.AF_INET, src_addr)
+
+    def add_vpp_config(self):
+        for path in self.paths:
+            self._test.vapi.ip_mroute_add_del(self.src_addr,
+                                              self.grp_addr,
+                                              self.grp_addr_len,
+                                              self.e_flags,
+                                              path.nh_itf,
+                                              path.nh_i_flags,
+                                              table_id=self.table_id,
+                                              is_ipv6=self.is_ip6)
+
+    def remove_vpp_config(self):
+        for path in self.paths:
+            self._test.vapi.ip_mroute_add_del(self.src_addr,
+                                              self.grp_addr,
+                                              self.grp_addr_len,
+                                              self.e_flags,
+                                              path.nh_itf,
+                                              path.nh_i_flags,
+                                              table_id=self.table_id,
+                                              is_add=0,
+                                              is_ipv6=self.is_ip6)
+
+    def update_entry_flags(self, flags):
+        self.e_flags = flags
+        self._test.vapi.ip_mroute_add_del(self.src_addr,
+                                          self.grp_addr,
+                                          self.grp_addr_len,
+                                          self.e_flags,
+                                          0xffffffff,
+                                          0,
+                                          table_id=self.table_id,
+                                          is_ipv6=self.is_ip6)
+
+    def update_path_flags(self, itf, flags):
+        for path in self.paths:
+            if path.nh_itf == itf:
+                path.nh_i_flags = flags
+                break
+        self._test.vapi.ip_mroute_add_del(self.src_addr,
+                                          self.grp_addr,
+                                          self.grp_addr_len,
+                                          self.e_flags,
+                                          path.nh_itf,
+                                          path.nh_i_flags,
+                                          table_id=self.table_id,
+                                          is_ipv6=self.is_ip6)
+
+
+class MFibSignal:
+    def __init__(self, test, route, interface, packet):
+        self.route = route
+        self.interface = interface
+        self.packet = packet
+        self.test = test
+
+    def compare(self, signal):
+        self.test.assertEqual(self.interface, signal.sw_if_index)
+        self.test.assertEqual(self.route.table_id, signal.table_id)
+        self.test.assertEqual(self.route.grp_addr_len,
+                              signal.grp_address_len)
+        for i in range(self.route.grp_addr_len / 8):
+            self.test.assertEqual(self.route.grp_addr[i],
+                                  signal.grp_address[i])
+        if (self.route.grp_addr_len > 32):
+            for i in range(4):
+                self.test.assertEqual(self.route.src_addr[i],
+                                      signal.src_address[i])
+
+
 class MplsIpBind:
     """
     MPLS to IP Binding
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index 1b2895e9..90c954dc 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -252,10 +252,12 @@ class VppPapiProvider(object):
                         {'sw_if_index': sw_if_index})
 
     def ip6_sw_interface_ra_config(self, sw_if_index,
+                                   no,
                                    suppress,
-                                   send_unicast,):
+                                   send_unicast):
         return self.api(self.papi.sw_interface_ip6nd_ra_config,
                         {'sw_if_index': sw_if_index,
+                         'is_no': no,
                          'suppress': suppress,
                          'send_unicast': send_unicast})
 
@@ -1178,3 +1180,33 @@ class VppPapiProvider(object):
                 'is_add': is_add,
                 'oui': oui,
             })
+
+    def ip_mroute_add_del(self,
+                          src_address,
+                          grp_address,
+                          grp_address_length,
+                          e_flags,
+                          next_hop_sw_if_index,
+                          i_flags,
+                          table_id=0,
+                          create_vrf_if_needed=0,
+                          is_add=1,
+                          is_ipv6=0,
+                          is_local=0):
+        """
+        """
+        return self.api(
+            self.papi.ip_mroute_add_del,
+            {'next_hop_sw_if_index': next_hop_sw_if_index,
+             'entry_flags': e_flags,
+             'itf_flags': i_flags,
+             'create_vrf_if_needed': create_vrf_if_needed,
+             'is_add': is_add,
+             'is_ipv6': is_ipv6,
+             'is_local': is_local,
+             'grp_address_length': grp_address_length,
+             'grp_address': grp_address,
+             'src_address': src_address})
+
+    def mfib_signal_dump(self):
+        return self.api(self.papi.mfib_signal_dump, {})
-- 
cgit 1.2.3-korg


From 52456fca3d922d71098d4c1aef8dcee6c5459f8c Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Wed, 15 Feb 2017 00:38:27 -0800
Subject: CLI extension to add multiple (S,G)s at once and time it

Change-Id: Id17060fd0e8ac80c8cf1999b0b82d0241b3b969a
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/vnet/ip/lookup.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 76 insertions(+), 10 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 6c5611d3..0ef0e7a6 100644
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -784,7 +784,10 @@ vnet_ip_mroute_cmd (vlib_main_t * vm,
   u32 fib_index;
   mfib_itf_flags_t iflags = 0;
   mfib_entry_flags_t eflags = 0;
+  u32 gcount, scount, ss, gg, incr;
+  f64 timet[2];
 
+  gcount = scount = 1;
   vnm = vnet_get_main ();
   is_del = 0;
   table_id = 0;
@@ -804,6 +807,10 @@ vnet_ip_mroute_cmd (vlib_main_t * vm,
 	is_del = 1;
       else if (unformat (line_input, "add"))
 	is_del = 0;
+      else if (unformat (line_input, "scount %d", &scount))
+	;
+      else if (unformat (line_input, "gcount %d", &gcount))
+	;
       else if (unformat (line_input, "%U %U",
 			 unformat_ip4_address,
 			 &pfx.fp_src_addr.ip4,
@@ -885,24 +892,83 @@ vnet_ip_mroute_cmd (vlib_main_t * vm,
 	}
     }
 
-  if (is_del && 0 == rpath.frp_weight)
+  timet[0] = vlib_time_now (vm);
+
+  if (FIB_PROTOCOL_IP4 == pfx.fp_proto)
     {
-      mfib_table_entry_delete (fib_index, &pfx, MFIB_SOURCE_CLI);
+      incr = 1 << (32 - (pfx.fp_len % 32));
     }
-  else if (eflags)
+  else
     {
-      mfib_table_entry_update (fib_index, &pfx, MFIB_SOURCE_CLI, eflags);
+      incr = 1 << (128 - (pfx.fp_len % 128));
     }
-  else
+
+  for (ss = 0; ss < scount; ss++)
     {
-      if (is_del)
-	mfib_table_entry_path_remove (fib_index,
-				      &pfx, MFIB_SOURCE_CLI, &rpath);
+      for (gg = 0; gg < gcount; gg++)
+	{
+	  if (is_del && 0 == rpath.frp_weight)
+	    {
+	      /* no path provided => route delete */
+	      mfib_table_entry_delete (fib_index, &pfx, MFIB_SOURCE_CLI);
+	    }
+	  else if (eflags)
+	    {
+	      mfib_table_entry_update (fib_index, &pfx, MFIB_SOURCE_CLI,
+				       eflags);
+	    }
+	  else
+	    {
+	      if (is_del)
+		mfib_table_entry_path_remove (fib_index,
+					      &pfx, MFIB_SOURCE_CLI, &rpath);
+	      else
+		mfib_table_entry_path_update (fib_index,
+					      &pfx, MFIB_SOURCE_CLI, &rpath,
+					      iflags);
+	    }
+
+	  if (FIB_PROTOCOL_IP4 == pfx.fp_proto)
+	    {
+	      pfx.fp_grp_addr.ip4.as_u32 =
+		clib_host_to_net_u32 (incr +
+				      clib_net_to_host_u32 (pfx.
+							    fp_grp_addr.ip4.
+							    as_u32));
+	    }
+	  else
+	    {
+	      int bucket = (incr < 64 ? 0 : 1);
+	      pfx.fp_grp_addr.ip6.as_u64[bucket] =
+		clib_host_to_net_u64 (incr +
+				      clib_net_to_host_u64 (pfx.
+							    fp_grp_addr.ip6.as_u64
+							    [bucket]));
+
+	    }
+	}
+      if (FIB_PROTOCOL_IP4 == pfx.fp_proto)
+	{
+	  pfx.fp_src_addr.ip4.as_u32 =
+	    clib_host_to_net_u32 (1 +
+				  clib_net_to_host_u32 (pfx.fp_src_addr.
+							ip4.as_u32));
+	}
       else
-	mfib_table_entry_path_update (fib_index,
-				      &pfx, MFIB_SOURCE_CLI, &rpath, iflags);
+	{
+	  pfx.fp_src_addr.ip6.as_u64[1] =
+	    clib_host_to_net_u64 (1 +
+				  clib_net_to_host_u64 (pfx.fp_src_addr.
+							ip6.as_u64[1]));
+	}
     }
 
+  timet[1] = vlib_time_now (vm);
+
+  if (scount > 1 || gcount > 1)
+    vlib_cli_output (vm, "%.6e routes/sec",
+		     (scount * gcount) / (timet[1] - timet[0]));
+
 done:
   return error;
 }
-- 
cgit 1.2.3-korg


From a9a20e7f69f4a91a4d5267ab5ce14125bdc7d6c6 Mon Sep 17 00:00:00 2001
From: Billy McFall <bmcfall@redhat.com>
Date: Wed, 15 Feb 2017 11:39:12 -0500
Subject: VPP-635: CLI Memory leak with invalid parameter

In the CLI parsing, below is a common pattern:
  /* Get a line of input. */
  if (!unformat_user (input, unformat_line_input, line_input))
    return 0;

  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
    {
      if (unformat (line_input, "x"))
	x = 1;
      :
      else
	return clib_error_return (0, "unknown input `%U'",
				  format_unformat_error, line_input);
    }
  unformat_free (line_input);

The 'else' returns if an unknown string is encountered. There a memory
leak because the 'unformat_free(line_input)' is not called. There is a
large number of instances of this pattern.

Replaced the previous pattern with:
  /* Get a line of input. */
  if (!unformat_user (input, unformat_line_input, line_input))
    return 0;

  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
    {
      if (unformat (line_input, "x"))
	x = 1;
      :
      else
        {
	  error = clib_error_return (0, "unknown input `%U'",
				     format_unformat_error, line_input);
	  goto done:
        }
    }

  /* ...Remaining code... */

done:
  unformat_free (line_input);
  return error;
}

In multiple files, 'unformat_free (line_input);' was never called, so
there was a memory leak whether an invalid string was entered or not.

Also, there were multiple instance where:
	  error = clib_error_return (0, "unknown input `%U'",
				     format_unformat_error, line_input);
used 'input' as the last parameter instead of 'line_input'. The result
is that output did not contain the substring in error, instead just an
empty string. Fixed all of those as well.

There are a lot of file, and very mind numbing work, so tried to keep
it to a pattern to avoid mistakes.

Change-Id: I8902f0c32a47dd7fb3bb3471a89818571702f1d2
Signed-off-by: Billy McFall <bmcfall@redhat.com>
Signed-off-by: Dave Barach <dave@barachs.net>
---
 build-root/emacs-lisp/tunnel-c-skel.el |  19 ++-
 src/plugins/ila/ila.c                  |  25 ++-
 src/plugins/lb/cli.c                   |  99 ++++++-----
 src/plugins/sixrd/sixrd.c              |  42 +++--
 src/plugins/snat/snat.c                | 139 +++++++++++-----
 src/vlib/threads_cli.c                 |  79 ++++++---
 src/vlib/trace.c                       |  13 +-
 src/vlib/unix/cli.c                    |  22 ++-
 src/vnet/devices/af_packet/cli.c       |  56 +++++--
 src/vnet/devices/dpdk/cli.c            | 290 +++++++++++++++++++++++----------
 src/vnet/devices/dpdk/ipsec/cli.c      |  15 +-
 src/vnet/devices/netmap/cli.c          |  54 ++++--
 src/vnet/devices/virtio/vhost-user.c   |  62 +++++--
 src/vnet/gre/interface.c               |  35 ++--
 src/vnet/ip/ip4_source_check.c         |   6 +-
 src/vnet/ip/ip4_test.c                 |  15 +-
 src/vnet/ip/ip6_neighbor.c             |  27 ++-
 src/vnet/ip/lookup.c                   |  34 ++--
 src/vnet/ipsec-gre/interface.c         |  34 ++--
 src/vnet/ipsec/ipsec_cli.c             | 177 +++++++++++++-------
 src/vnet/l2/l2_patch.c                 |  26 ++-
 src/vnet/l2/l2_xcrw.c                  |  34 +++-
 src/vnet/l2tp/l2tp.c                   |  39 +++--
 src/vnet/lisp-cp/lisp_cli.c            | 139 ++++++++++++----
 src/vnet/lisp-gpe/interface.c          |  58 +++++--
 src/vnet/lisp-gpe/lisp_gpe.c           |  13 +-
 src/vnet/map/map.c                     | 186 +++++++++++++++------
 src/vnet/mpls/mpls.c                   |   2 +
 src/vnet/mpls/mpls_tunnel.c            |  19 ++-
 src/vnet/pg/cli.c                      |  39 +++--
 src/vnet/policer/node_funcs.c          |  19 ++-
 src/vnet/policer/policer.c             |  13 +-
 src/vnet/unix/tapcli.c                 |  57 +++++--
 src/vnet/vxlan-gpe/vxlan_gpe.c         |  62 +++++--
 src/vnet/vxlan/vxlan.c                 |  81 ++++++---
 src/vpp/app/l2t.c                      |   9 +-
 src/vpp/app/vpe_cli.c                  |  24 ++-
 37 files changed, 1487 insertions(+), 576 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/build-root/emacs-lisp/tunnel-c-skel.el b/build-root/emacs-lisp/tunnel-c-skel.el
index aa260e53..a1b1757d 100644
--- a/build-root/emacs-lisp/tunnel-c-skel.el
+++ b/build-root/emacs-lisp/tunnel-c-skel.el
@@ -288,6 +288,7 @@ static clib_error_t *
                                    vlib_cli_command_t * cmd)
 {
   unformat_input_t _line_input, * line_input = &_line_input;
+  clib_error_t *error = 0;
   ip4_address_t src, dst;
   u8 is_add = 1;
   u8 src_set = 0;
@@ -322,13 +323,19 @@ static clib_error_t *
       {
         encap_fib_index = fib_index_from_fib_id (tmp);
         if (encap_fib_index == ~0)
-          return clib_error_return (0, \"nonexistent encap fib id %d\", tmp);
+          {
+            unformat_free (line_input);
+            return clib_error_return (0, \"nonexistent encap fib id %d\", tmp);
+          }
       }
     else if (unformat (line_input, \"decap-vrf-id %d\", &tmp))
       {
         decap_fib_index = fib_index_from_fib_id (tmp);
         if (decap_fib_index == ~0)
-          return clib_error_return (0, \"nonexistent decap fib id %d\", tmp);
+          {
+            unformat_free (line_input);
+            return clib_error_return (0, \"nonexistent decap fib id %d\", tmp);
+          }
       }
     else if (unformat (line_input, \"decap-next %U\", unformat_decap_next, 
                        &decap_next_index))
@@ -346,8 +353,12 @@ static clib_error_t *
      * in the " ENCAP_STACK " header
      */
     else 
-      return clib_error_return (0, \"parse error: '%U'\", 
-                                format_unformat_error, line_input);
+      {
+        error = clib_error_return (0, \"parse error: '%U'\",
+                                   format_unformat_error, line_input);
+        unformat_free (line_input);
+        return error;
+      }
   }
 
   unformat_free (line_input);
diff --git a/src/plugins/ila/ila.c b/src/plugins/ila/ila.c
index e0f3907f..52c7ea55 100644
--- a/src/plugins/ila/ila.c
+++ b/src/plugins/ila/ila.c
@@ -949,6 +949,7 @@ ila_entry_command_fn (vlib_main_t * vm,
   ila_add_del_entry_args_t args = { 0 };
   u8 next_hop_set = 0;
   int ret;
+  clib_error_t *error = 0;
 
   args.type = ILA_TYPE_IID;
   args.csum_mode = ILA_CSUM_MODE_NO_ACTION;
@@ -986,19 +987,29 @@ ila_entry_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "del"))
 	args.is_del = 1;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (!next_hop_set)
-      return clib_error_return (0, "Specified a next hop");
+    {
+      error = clib_error_return (0, "Specified a next hop");
+      goto done;
+    }
 
   if ((ret = ila_add_del_entry (&args)))
-    return clib_error_return (0, "ila_add_del_entry returned error %d", ret);
+    {
+      error = clib_error_return (0, "ila_add_del_entry returned error %d", ret);
+      goto done;
+    }
 
-  return NULL;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 VLIB_CLI_COMMAND (ila_entry_command, static) =
diff --git a/src/plugins/lb/cli.c b/src/plugins/lb/cli.c
index b59c6426..6452a875 100644
--- a/src/plugins/lb/cli.c
+++ b/src/plugins/lb/cli.c
@@ -28,13 +28,16 @@ lb_vip_command_fn (vlib_main_t * vm,
   int ret;
   u32 gre4 = 0;
   lb_vip_type_t type;
+  clib_error_t *error = 0;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
 
-  if (!unformat(line_input, "%U", unformat_ip46_prefix, &prefix, &plen, IP46_TYPE_ANY, &plen))
-    return clib_error_return (0, "invalid vip prefix: '%U'",
-                                  format_unformat_error, line_input);
+  if (!unformat(line_input, "%U", unformat_ip46_prefix, &prefix, &plen, IP46_TYPE_ANY, &plen)) {
+    error = clib_error_return (0, "invalid vip prefix: '%U'",
+                               format_unformat_error, line_input);
+    goto done;
+  }
 
   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
   {
@@ -46,13 +49,13 @@ lb_vip_command_fn (vlib_main_t * vm,
       gre4 = 1;
     else if (unformat(line_input, "encap gre6"))
       gre4 = 0;
-    else
-      return clib_error_return (0, "parse error: '%U'",
-                              format_unformat_error, line_input);
+    else {
+      error = clib_error_return (0, "parse error: '%U'",
+                                format_unformat_error, line_input);
+      goto done;
+    }
   }
 
-  unformat_free (line_input);
-
 
   if (ip46_prefix_is_ip4(&prefix, plen)) {
     type = (gre4)?LB_VIP_TYPE_IP4_GRE4:LB_VIP_TYPE_IP4_GRE6;
@@ -65,17 +68,25 @@ lb_vip_command_fn (vlib_main_t * vm,
   u32 index;
   if (!del) {
     if ((ret = lb_vip_add(&prefix, plen, type, new_len, &index))) {
-      return clib_error_return (0, "lb_vip_add error %d", ret);
+      error = clib_error_return (0, "lb_vip_add error %d", ret);
+      goto done;
     } else {
       vlib_cli_output(vm, "lb_vip_add ok %d", index);
     }
   } else {
-    if ((ret = lb_vip_find_index(&prefix, plen, &index)))
-      return clib_error_return (0, "lb_vip_find_index error %d", ret);
-    else if ((ret = lb_vip_del(index)))
-      return clib_error_return (0, "lb_vip_del error %d", ret);
+    if ((ret = lb_vip_find_index(&prefix, plen, &index))) {
+      error = clib_error_return (0, "lb_vip_find_index error %d", ret);
+      goto done;
+    } else if ((ret = lb_vip_del(index))) {
+      error = clib_error_return (0, "lb_vip_del error %d", ret);
+      goto done;
+    }
   }
-  return NULL;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 VLIB_CLI_COMMAND (lb_vip_command, static) =
@@ -96,16 +107,21 @@ lb_as_command_fn (vlib_main_t * vm,
   u32 vip_index;
   u8 del = 0;
   int ret;
+  clib_error_t *error = 0;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
 
-  if (!unformat(line_input, "%U", unformat_ip46_prefix, &vip_prefix, &vip_plen, IP46_TYPE_ANY))
-    return clib_error_return (0, "invalid as address: '%U'",
-                              format_unformat_error, line_input);
+  if (!unformat(line_input, "%U", unformat_ip46_prefix, &vip_prefix, &vip_plen, IP46_TYPE_ANY)) {
+    error = clib_error_return (0, "invalid as address: '%U'",
+                               format_unformat_error, line_input);
+    goto done;
+  }
 
-  if ((ret = lb_vip_find_index(&vip_prefix, vip_plen, &vip_index)))
-    return clib_error_return (0, "lb_vip_find_index error %d", ret);
+  if ((ret = lb_vip_find_index(&vip_prefix, vip_plen, &vip_index))) {
+    error = clib_error_return (0, "lb_vip_find_index error %d", ret);
+    goto done;
+  }
 
   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
   {
@@ -114,15 +130,15 @@ lb_as_command_fn (vlib_main_t * vm,
     } else if (unformat(line_input, "del")) {
       del = 1;
     } else {
-      vec_free(as_array);
-      return clib_error_return (0, "parse error: '%U'",
-                                format_unformat_error, line_input);
+      error = clib_error_return (0, "parse error: '%U'",
+                                 format_unformat_error, line_input);
+      goto done;
     }
   }
 
   if (!vec_len(as_array)) {
-    vec_free(as_array);
-    return clib_error_return (0, "No AS address provided");
+    error = clib_error_return (0, "No AS address provided");
+    goto done;
   }
 
   lb_garbage_collection();
@@ -130,18 +146,21 @@ lb_as_command_fn (vlib_main_t * vm,
 
   if (del) {
     if ((ret = lb_vip_del_ass(vip_index, as_array, vec_len(as_array)))) {
-      vec_free(as_array);
-      return clib_error_return (0, "lb_vip_del_ass error %d", ret);
+      error = clib_error_return (0, "lb_vip_del_ass error %d", ret);
+      goto done;
     }
   } else {
     if ((ret = lb_vip_add_ass(vip_index, as_array, vec_len(as_array)))) {
-      vec_free(as_array);
-      return clib_error_return (0, "lb_vip_add_ass error %d", ret);
+      error = clib_error_return (0, "lb_vip_add_ass error %d", ret);
+      goto done;
     }
   }
 
+done:
+  unformat_free (line_input);
   vec_free(as_array);
-  return 0;
+
+  return error;
 }
 
 VLIB_CLI_COMMAND (lb_as_command, static) =
@@ -163,6 +182,7 @@ lb_conf_command_fn (vlib_main_t * vm,
   u32 per_cpu_sticky_buckets_log2 = 0;
   u32 flow_timeout = lbm->flow_timeout;
   int ret;
+  clib_error_t *error = 0;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -181,19 +201,24 @@ lb_conf_command_fn (vlib_main_t * vm,
       per_cpu_sticky_buckets = 1 << per_cpu_sticky_buckets_log2;
     } else if (unformat(line_input, "timeout %d", &flow_timeout))
       ;
-    else
-      return clib_error_return (0, "parse error: '%U'",
-                                format_unformat_error, line_input);
+    else {
+      error = clib_error_return (0, "parse error: '%U'",
+                                 format_unformat_error, line_input);
+      goto done;
+    }
   }
 
-  unformat_free (line_input);
-
   lb_garbage_collection();
 
-  if ((ret = lb_conf(&ip4, &ip6, per_cpu_sticky_buckets, flow_timeout)))
-    return clib_error_return (0, "lb_conf error %d", ret);
+  if ((ret = lb_conf(&ip4, &ip6, per_cpu_sticky_buckets, flow_timeout))) {
+    error = clib_error_return (0, "lb_conf error %d", ret);
+    goto done;
+  }
 
-  return NULL;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 VLIB_CLI_COMMAND (lb_conf_command, static) =
diff --git a/src/plugins/sixrd/sixrd.c b/src/plugins/sixrd/sixrd.c
index 71fc181f..67a9a3ad 100644
--- a/src/plugins/sixrd/sixrd.c
+++ b/src/plugins/sixrd/sixrd.c
@@ -192,6 +192,7 @@ sixrd_add_domain_command_fn (vlib_main_t *vm,
   u32 num_m_args = 0;
   /* Optional arguments */
   u32 mtu = 0;
+  clib_error_t *error = 0;
 
   /* Get a line of input. */
   if (!unformat_user(input, unformat_line_input, line_input))
@@ -205,19 +206,25 @@ sixrd_add_domain_command_fn (vlib_main_t *vm,
       num_m_args++;
     else if (unformat(line_input, "mtu %d", &mtu))
       num_m_args++;
-    else
-      return clib_error_return(0, "unknown input `%U'",
-                               format_unformat_error, input);
+    else {
+      error = clib_error_return(0, "unknown input `%U'",
+                                format_unformat_error, line_input);
+      goto done;
+    }
   }
-  unformat_free(line_input);
 
-  if (num_m_args < 3)
-    return clib_error_return(0, "mandatory argument(s) missing");
+  if (num_m_args < 3) {
+    error = clib_error_return(0, "mandatory argument(s) missing");
+    goto done;
+  }
 
   sixrd_create_domain(&ip6_prefix, ip6_prefix_len, &ip4_prefix, ip4_prefix_len,
 		      &ip4_src, &sixrd_domain_index, mtu);
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 static clib_error_t *
@@ -228,6 +235,7 @@ sixrd_del_domain_command_fn (vlib_main_t *vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   u32 num_m_args = 0;
   u32 sixrd_domain_index;
+  clib_error_t *error = 0;
 
   /* Get a line of input. */
   if (! unformat_user(input, unformat_line_input, line_input))
@@ -236,18 +244,24 @@ sixrd_del_domain_command_fn (vlib_main_t *vm,
   while (unformat_check_input(line_input) != UNFORMAT_END_OF_INPUT) {
     if (unformat(line_input, "index %d", &sixrd_domain_index))
       num_m_args++;
-    else
-      return clib_error_return(0, "unknown input `%U'",
-				format_unformat_error, input);
+    else {
+      error = clib_error_return(0, "unknown input `%U'",
+				format_unformat_error, line_input);
+      goto done;
+    }
   }
-  unformat_free(line_input);
 
-  if (num_m_args != 1)
-    return clib_error_return(0, "mandatory argument(s) missing");
+  if (num_m_args != 1) {
+    error = clib_error_return(0, "mandatory argument(s) missing");
+    goto done;
+  }
 
   sixrd_delete_domain(sixrd_domain_index);
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 static u8 *
diff --git a/src/plugins/snat/snat.c b/src/plugins/snat/snat.c
index 73854a7a..8c2bacdb 100644
--- a/src/plugins/snat/snat.c
+++ b/src/plugins/snat/snat.c
@@ -1705,6 +1705,7 @@ add_address_command_fn (vlib_main_t * vm,
   int i, count;
   int is_add = 1;
   int rv = 0;
+  clib_error_t *error = 0;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -1721,19 +1722,27 @@ add_address_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "del"))
         is_add = 0;
       else
-        return clib_error_return (0, "unknown input '%U'",
-          format_unformat_error, input);
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+            format_unformat_error, line_input);
+          goto done;
+        }
      }
-  unformat_free (line_input);
 
   if (sm->static_mapping_only)
-    return clib_error_return (0, "static mapping only mode");
+    {
+      error = clib_error_return (0, "static mapping only mode");
+      goto done;
+    }
 
   start_host_order = clib_host_to_net_u32 (start_addr.as_u32);
   end_host_order = clib_host_to_net_u32 (end_addr.as_u32);
   
   if (end_host_order < start_host_order)
-    return clib_error_return (0, "end address less than start address");
+    {
+      error = clib_error_return (0, "end address less than start address");
+      goto done;
+    }
 
   count = (end_host_order - start_host_order) + 1;
 
@@ -1755,11 +1764,11 @@ add_address_command_fn (vlib_main_t * vm,
       switch (rv)
         {
         case VNET_API_ERROR_NO_SUCH_ENTRY:
-          return clib_error_return (0, "S-NAT address not exist.");
-          break;
+          error = clib_error_return (0, "S-NAT address not exist.");
+          goto done;
         case VNET_API_ERROR_UNSPECIFIED:
-          return clib_error_return (0, "S-NAT address used in static mapping.");
-          break;
+          error = clib_error_return (0, "S-NAT address used in static mapping.");
+          goto done;
         default:
           break;
         }
@@ -1767,7 +1776,10 @@ add_address_command_fn (vlib_main_t * vm,
       increment_v4_address (&this_addr);
     }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 VLIB_CLI_COMMAND (add_address_command, static) = {
@@ -1807,10 +1819,12 @@ snat_feature_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "del"))
         is_del = 1;
       else
-        return clib_error_return (0, "unknown input '%U'",
-          format_unformat_error, input);
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+            format_unformat_error, line_input);
+          goto done;
+        }
     }
-  unformat_free (line_input);
 
   if (vec_len (inside_sw_if_indices))
     {
@@ -1830,6 +1844,8 @@ snat_feature_command_fn (vlib_main_t * vm,
         }
     }
 
+done:
+  unformat_free (line_input);
   vec_free (inside_sw_if_indices);
   vec_free (outside_sw_if_indices);
 
@@ -1923,13 +1939,18 @@ add_static_mapping_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "del"))
         is_add = 0;
       else
-        return clib_error_return (0, "unknown input: '%U'",
-          format_unformat_error, line_input);
+        {
+          error = clib_error_return (0, "unknown input: '%U'",
+            format_unformat_error, line_input);
+          goto done;
+        }
     }
-  unformat_free (line_input);
 
   if (!addr_only && !proto_set)
-    return clib_error_return (0, "missing protocol");
+    {
+      error = clib_error_return (0, "missing protocol");
+      goto done;
+    }
 
   rv = snat_add_static_mapping(l_addr, e_addr, (u16) l_port, (u16) e_port,
                                vrf_id, addr_only, sw_if_index, proto, is_add);
@@ -1937,22 +1958,27 @@ add_static_mapping_command_fn (vlib_main_t * vm,
   switch (rv)
     {
     case VNET_API_ERROR_INVALID_VALUE:
-      return clib_error_return (0, "External port already in use.");
-      break;
+      error = clib_error_return (0, "External port already in use.");
+      goto done;
     case VNET_API_ERROR_NO_SUCH_ENTRY:
       if (is_add)
-        return clib_error_return (0, "External addres must be allocated.");
+        error = clib_error_return (0, "External addres must be allocated.");
       else
-        return clib_error_return (0, "Mapping not exist.");
-      break;
+        error = clib_error_return (0, "Mapping not exist.");
+      goto done;
     case VNET_API_ERROR_NO_SUCH_FIB:
-      return clib_error_return (0, "No such VRF id.");
+      error = clib_error_return (0, "No such VRF id.");
+      goto done;
     case VNET_API_ERROR_VALUE_EXIST:
-      return clib_error_return (0, "Mapping already exist.");
+      error = clib_error_return (0, "Mapping already exist.");
+      goto done;
     default:
       break;
     }
 
+done:
+  unformat_free (line_input);
+
   return error;
 }
 
@@ -1985,6 +2011,7 @@ set_workers_command_fn (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   uword *bitmap = 0;
   int rv = 0;
+  clib_error_t *error = 0;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -1995,13 +2022,18 @@ set_workers_command_fn (vlib_main_t * vm,
       if (unformat (line_input, "%U", unformat_bitmap_list, &bitmap))
         ;
       else
-        return clib_error_return (0, "unknown input '%U'",
-          format_unformat_error, input);
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+            format_unformat_error, line_input);
+          goto done;
+        }
      }
-  unformat_free (line_input);
 
   if (bitmap == 0)
-    return clib_error_return (0, "List of workers must be specified.");
+    {
+      error = clib_error_return (0, "List of workers must be specified.");
+      goto done;
+    }
 
   rv = snat_set_workers(bitmap);
 
@@ -2010,17 +2042,20 @@ set_workers_command_fn (vlib_main_t * vm,
   switch (rv)
     {
     case VNET_API_ERROR_INVALID_WORKER:
-      return clib_error_return (0, "Invalid worker(s).");
-      break;
+      error = clib_error_return (0, "Invalid worker(s).");
+      goto done;
     case VNET_API_ERROR_FEATURE_DISABLED:
-      return clib_error_return (0,
+      error = clib_error_return (0,
         "Supported only if 2 or more workes available.");
-      break;
+      goto done;
     default:
       break;
     }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /*?
@@ -2047,6 +2082,7 @@ snat_ipfix_logging_enable_disable_command_fn (vlib_main_t * vm,
   u32 src_port = 0;
   u8 enable = 1;
   int rv = 0;
+  clib_error_t *error = 0;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -2061,17 +2097,25 @@ snat_ipfix_logging_enable_disable_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "disable"))
         enable = 0;
       else
-        return clib_error_return (0, "unknown input '%U'",
-          format_unformat_error, input);
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+            format_unformat_error, line_input);
+          goto done;
+        }
      }
-  unformat_free (line_input);
 
   rv = snat_ipfix_logging_enable_disable (enable, domain_id, (u16) src_port);
 
   if (rv)
-    return clib_error_return (0, "ipfix logging enable failed");
+    {
+      error = clib_error_return (0, "ipfix logging enable failed");
+      goto done;
+    }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /*?
@@ -2604,6 +2648,7 @@ snat_add_interface_address_command_fn (vlib_main_t * vm,
   u32 sw_if_index;
   int rv;
   int is_del = 0;
+  clib_error_t *error = 0;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -2617,8 +2662,11 @@ snat_add_interface_address_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "del"))
         is_del = 1;
       else
-        return clib_error_return (0, "unknown input '%U'",
-				  format_unformat_error, line_input);
+        {
+          error = clib_error_return (0, "unknown input '%U'",
+				     format_unformat_error, line_input);
+          goto done;
+        }
     }
 
   rv = snat_add_interface_address (sm, sw_if_index, is_del);
@@ -2629,10 +2677,15 @@ snat_add_interface_address_command_fn (vlib_main_t * vm,
       break;
 
     default:
-      return clib_error_return (0, "snat_add_interface_address returned %d",
-                                rv);
+      error = clib_error_return (0, "snat_add_interface_address returned %d",
+                                 rv);
+      goto done;
     }
-  return 0;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 VLIB_CLI_COMMAND (snat_add_interface_address_command, static) = {
diff --git a/src/vlib/threads_cli.c b/src/vlib/threads_cli.c
index 54cc1aed..36f8109e 100644
--- a/src/vlib/threads_cli.c
+++ b/src/vlib/threads_cli.c
@@ -163,21 +163,31 @@ trace_frame_queue (vlib_main_t * vm, unformat_input_t * input,
       else if (unformat (line_input, "index %u", &index))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (enable > 1)
-    return clib_error_return (0, "expecting on or off");
+    {
+      error = clib_error_return (0, "expecting on or off");
+      goto done;
+    }
 
   if (vec_len (tm->frame_queue_mains) == 0)
-    return clib_error_return (0, "no worker handoffs exist");
+    {
+      error = clib_error_return (0, "no worker handoffs exist");
+      goto done;
+    }
 
   if (index > vec_len (tm->frame_queue_mains) - 1)
-    return clib_error_return (0,
-			      "expecting valid worker handoff queue index");
+    {
+      error = clib_error_return (0,
+				 "expecting valid worker handoff queue index");
+      goto done;
+    }
 
   fqm = vec_elt_at_index (tm->frame_queue_mains, index);
 
@@ -185,7 +195,7 @@ trace_frame_queue (vlib_main_t * vm, unformat_input_t * input,
   if (num_fq == 0)
     {
       vlib_cli_output (vm, "No frame queues exist\n");
-      return error;
+      goto done;
     }
 
   // Allocate storage for trace if necessary
@@ -204,6 +214,10 @@ trace_frame_queue (vlib_main_t * vm, unformat_input_t * input,
       memset (fqh, 0, sizeof (*fqh));
       fqm->vlib_frame_queues[fqix]->trace = enable;
     }
+
+done:
+  unformat_free (line_input);
+
   return error;
 }
 
@@ -432,28 +446,33 @@ test_frame_queue_nelts (vlib_main_t * vm, unformat_input_t * input,
       else if (unformat (line_input, "index %u", &index))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (index > vec_len (tm->frame_queue_mains) - 1)
-    return clib_error_return (0,
-			      "expecting valid worker handoff queue index");
+    {
+      error = clib_error_return (0,
+				 "expecting valid worker handoff queue index");
+      goto done;
+    }
 
   fqm = vec_elt_at_index (tm->frame_queue_mains, index);
 
   if ((nelts != 4) && (nelts != 8) && (nelts != 16) && (nelts != 32))
     {
-      return clib_error_return (0, "expecting 4,8,16,32");
+      error = clib_error_return (0, "expecting 4,8,16,32");
+      goto done;
     }
 
   num_fq = vec_len (fqm->vlib_frame_queues);
   if (num_fq == 0)
     {
       vlib_cli_output (vm, "No frame queues exist\n");
-      return error;
+      goto done;
     }
 
   for (fqix = 0; fqix < num_fq; fqix++)
@@ -461,6 +480,9 @@ test_frame_queue_nelts (vlib_main_t * vm, unformat_input_t * input,
       fqm->vlib_frame_queues[fqix]->nelts = nelts;
     }
 
+done:
+  unformat_free (line_input);
+
   return error;
 }
 
@@ -499,15 +521,19 @@ test_frame_queue_threshold (vlib_main_t * vm, unformat_input_t * input,
       else if (unformat (line_input, "index %u", &index))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (index > vec_len (tm->frame_queue_mains) - 1)
-    return clib_error_return (0,
-			      "expecting valid worker handoff queue index");
+    {
+      error = clib_error_return (0,
+				 "expecting valid worker handoff queue index");
+      goto done;
+    }
 
   fqm = vec_elt_at_index (tm->frame_queue_mains, index);
 
@@ -515,7 +541,7 @@ test_frame_queue_threshold (vlib_main_t * vm, unformat_input_t * input,
   if (threshold == ~(u32) 0)
     {
       vlib_cli_output (vm, "expecting threshold value\n");
-      return error;
+      goto done;
     }
 
   if (threshold == 0)
@@ -525,7 +551,7 @@ test_frame_queue_threshold (vlib_main_t * vm, unformat_input_t * input,
   if (num_fq == 0)
     {
       vlib_cli_output (vm, "No frame queues exist\n");
-      return error;
+      goto done;
     }
 
   for (fqix = 0; fqix < num_fq; fqix++)
@@ -533,6 +559,9 @@ test_frame_queue_threshold (vlib_main_t * vm, unformat_input_t * input,
       fqm->vlib_frame_queues[fqix]->vector_threshold = threshold;
     }
 
+done:
+  unformat_free (line_input);
+
   return error;
 }
 
diff --git a/src/vlib/trace.c b/src/vlib/trace.c
index dcdb837f..6d487ae1 100644
--- a/src/vlib/trace.c
+++ b/src/vlib/trace.c
@@ -372,6 +372,7 @@ cli_add_trace_buffer (vlib_main_t * vm,
   vlib_trace_node_t *tn;
   u32 node_index, add;
   u8 verbose = 0;
+  clib_error_t *error = 0;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -384,8 +385,11 @@ cli_add_trace_buffer (vlib_main_t * vm,
       else if (unformat (line_input, "verbose"))
 	verbose = 1;
       else
-	return clib_error_create ("expected NODE COUNT, got `%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_create ("expected NODE COUNT, got `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
   /* *INDENT-OFF* */
@@ -403,7 +407,10 @@ cli_add_trace_buffer (vlib_main_t * vm,
     }));
   /* *INDENT-ON* */
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
diff --git a/src/vlib/unix/cli.c b/src/vlib/unix/cli.c
index 69fca6ec..88e2453c 100644
--- a/src/vlib/unix/cli.c
+++ b/src/vlib/unix/cli.c
@@ -2835,6 +2835,7 @@ unix_cli_set_terminal_pager (vlib_main_t * vm,
   unix_cli_main_t *cm = &unix_cli_main;
   unix_cli_file_t *cf;
   unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = 0;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -2852,13 +2853,17 @@ unix_cli_set_terminal_pager (vlib_main_t * vm,
 			 "Pager limit set to %u lines; note, this is global.\n",
 			 um->cli_pager_buffer_limit);
       else
-	return clib_error_return (0, "unknown parameter: `%U`",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "unknown parameter: `%U`",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
+done:
   unformat_free (line_input);
 
-  return 0;
+  return error;
 }
 
 /*?
@@ -2886,6 +2891,7 @@ unix_cli_set_terminal_history (vlib_main_t * vm,
   unix_cli_file_t *cf;
   unformat_input_t _line_input, *line_input = &_line_input;
   u32 limit;
+  clib_error_t *error = 0;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -2901,8 +2907,11 @@ unix_cli_set_terminal_history (vlib_main_t * vm,
       else if (unformat (line_input, "limit %u", &cf->history_limit))
 	;
       else
-	return clib_error_return (0, "unknown parameter: `%U`",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "unknown parameter: `%U`",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
 
       /* If we reduced history size, or turned it off, purge the history */
       limit = cf->has_history ? cf->history_limit : 0;
@@ -2914,9 +2923,10 @@ unix_cli_set_terminal_history (vlib_main_t * vm,
 	}
     }
 
+done:
   unformat_free (line_input);
 
-  return 0;
+  return error;
 }
 
 /*?
diff --git a/src/vnet/devices/af_packet/cli.c b/src/vnet/devices/af_packet/cli.c
index 6baa26e1..d4aa7016 100644
--- a/src/vnet/devices/af_packet/cli.c
+++ b/src/vnet/devices/af_packet/cli.c
@@ -49,6 +49,7 @@ af_packet_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
   u8 *hw_addr_ptr = 0;
   u32 sw_if_index;
   int r;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -63,29 +64,47 @@ af_packet_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
 	    (line_input, "hw-addr %U", unformat_ethernet_address, hwaddr))
 	hw_addr_ptr = hwaddr;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
   if (host_if_name == NULL)
-    return clib_error_return (0, "missing host interface name");
+    {
+      error = clib_error_return (0, "missing host interface name");
+      goto done;
+    }
 
   r = af_packet_create_if (vm, host_if_name, hw_addr_ptr, &sw_if_index);
-  vec_free (host_if_name);
 
   if (r == VNET_API_ERROR_SYSCALL_ERROR_1)
-    return clib_error_return (0, "%s (errno %d)", strerror (errno), errno);
+    {
+      error = clib_error_return (0, "%s (errno %d)", strerror (errno), errno);
+      goto done;
+    }
 
   if (r == VNET_API_ERROR_INVALID_INTERFACE)
-    return clib_error_return (0, "Invalid interface name");
+    {
+      error = clib_error_return (0, "Invalid interface name");
+      goto done;
+    }
 
   if (r == VNET_API_ERROR_SUBIF_ALREADY_EXISTS)
-    return clib_error_return (0, "Interface elready exists");
+    {
+      error = clib_error_return (0, "Interface elready exists");
+      goto done;
+    }
 
   vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main (),
 		   sw_if_index);
-  return 0;
+
+done:
+  vec_free (host_if_name);
+  unformat_free (line_input);
+
+  return error;
 }
 
 /*?
@@ -124,6 +143,7 @@ af_packet_delete_command_fn (vlib_main_t * vm, unformat_input_t * input,
 {
   unformat_input_t _line_input, *line_input = &_line_input;
   u8 *host_if_name = NULL;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -134,18 +154,26 @@ af_packet_delete_command_fn (vlib_main_t * vm, unformat_input_t * input,
       if (unformat (line_input, "name %s", &host_if_name))
 	;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
   if (host_if_name == NULL)
-    return clib_error_return (0, "missing host interface name");
+    {
+      error = clib_error_return (0, "missing host interface name");
+      goto done;
+    }
 
   af_packet_delete_if (vm, host_if_name);
+
+done:
   vec_free (host_if_name);
+  unformat_free (line_input);
 
-  return 0;
+  return error;
 }
 
 /*?
diff --git a/src/vnet/devices/dpdk/cli.c b/src/vnet/devices/dpdk/cli.c
index d133cfd9..1fc665ac 100644
--- a/src/vnet/devices/dpdk/cli.c
+++ b/src/vnet/devices/dpdk/cli.c
@@ -398,7 +398,7 @@ set_dpdk_if_desc (vlib_main_t * vm, unformat_input_t * input,
   u32 hw_if_index = (u32) ~ 0;
   u32 nb_rx_desc = (u32) ~ 0;
   u32 nb_tx_desc = (u32) ~ 0;
-  clib_error_t *rv;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -414,25 +414,37 @@ set_dpdk_if_desc (vlib_main_t * vm, unformat_input_t * input,
       else if (unformat (line_input, "rx %d", &nb_rx_desc))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (hw_if_index == (u32) ~ 0)
-    return clib_error_return (0, "please specify valid interface name");
+    {
+      error = clib_error_return (0, "please specify valid interface name");
+      goto done;
+    }
 
   hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
   xd = vec_elt_at_index (dm->devices, hw->dev_instance);
 
   if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
-    return clib_error_return (0, "number of descriptors can be set only for "
-			      "physical devices");
+    {
+      error =
+	clib_error_return (0,
+			   "number of descriptors can be set only for "
+			   "physical devices");
+      goto done;
+    }
 
   if ((nb_rx_desc == (u32) ~ 0 || nb_rx_desc == xd->nb_rx_desc) &&
       (nb_tx_desc == (u32) ~ 0 || nb_tx_desc == xd->nb_tx_desc))
-    return clib_error_return (0, "nothing changed");
+    {
+      error = clib_error_return (0, "nothing changed");
+      goto done;
+    }
 
   if (nb_rx_desc != (u32) ~ 0)
     xd->nb_rx_desc = nb_rx_desc;
@@ -440,9 +452,12 @@ set_dpdk_if_desc (vlib_main_t * vm, unformat_input_t * input,
   if (nb_tx_desc != (u32) ~ 0)
     xd->nb_tx_desc = nb_tx_desc;
 
-  rv = dpdk_port_setup (dm, xd);
+  error = dpdk_port_setup (dm, xd);
+
+done:
+  unformat_free (line_input);
 
-  return rv;
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -523,6 +538,7 @@ set_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input,
   u32 queue = (u32) 0;
   u32 cpu = (u32) ~ 0;
   int i;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -538,18 +554,25 @@ set_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input,
       else if (unformat (line_input, "thread %d", &cpu))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (hw_if_index == (u32) ~ 0)
-    return clib_error_return (0, "please specify valid interface name");
+    {
+      error = clib_error_return (0, "please specify valid interface name");
+      goto done;
+    }
 
   if (cpu < dm->input_cpu_first_index ||
       cpu >= (dm->input_cpu_first_index + dm->input_cpu_count))
-    return clib_error_return (0, "please specify valid thread id");
+    {
+      error = clib_error_return (0, "please specify valid thread id");
+      goto done;
+    }
 
   hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
   xd = vec_elt_at_index (dm->devices, hw->dev_instance);
@@ -563,7 +586,7 @@ set_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input,
               queue == dq->queue_id)
             {
               if (cpu == i) /* nothing to do */
-                return 0;
+                goto done;
 
               vec_del1(dm->devices_by_cpu[i], dq - dm->devices_by_cpu[i]);
               vec_add2(dm->devices_by_cpu[cpu], dq, 1);
@@ -586,13 +609,18 @@ set_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input,
                 vlib_node_set_state (vlib_mains[cpu], dpdk_input_node.index,
                                      VLIB_NODE_STATE_POLLING);
 
-              return 0;
+              goto done;
             }
         }
       /* *INDENT-ON* */
     }
 
-  return clib_error_return (0, "not found");
+  error = clib_error_return (0, "not found");
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -653,6 +681,7 @@ set_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input,
   u32 hw_if_index = (u32) ~ 0;
   u32 cpu = (u32) ~ 0;
   int i;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -666,18 +695,22 @@ set_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input,
       else if (unformat (line_input, "thread %d", &cpu))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (hw_if_index == (u32) ~ 0)
     return clib_error_return (0, "please specify valid interface name");
 
   if (cpu < dm->hqos_cpu_first_index ||
       cpu >= (dm->hqos_cpu_first_index + dm->hqos_cpu_count))
-    return clib_error_return (0, "please specify valid thread id");
+    {
+      error = clib_error_return (0, "please specify valid thread id");
+      goto done;
+    }
 
   hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
   xd = vec_elt_at_index (dm->devices, hw->dev_instance);
@@ -689,7 +722,7 @@ set_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input,
 	if (hw_if_index == dm->devices[dq->device].vlib_hw_if_index)
 	  {
 	    if (cpu == i)	/* nothing to do */
-	      return 0;
+	      goto done;
 
 	    vec_del1 (dm->devices_by_hqos_cpu[i],
 		      dq - dm->devices_by_hqos_cpu[i]);
@@ -703,12 +736,17 @@ set_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input,
 	    vec_sort_with_function (dm->devices_by_hqos_cpu[cpu],
 				    dpdk_device_queue_sort);
 
-	    return 0;
+	    goto done;
 	  }
       }
     }
 
-  return clib_error_return (0, "not found");
+  error = clib_error_return (0, "not found");
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -732,6 +770,7 @@ set_dpdk_if_hqos_pipe (vlib_main_t * vm, unformat_input_t * input,
   u32 pipe_id = (u32) ~ 0;
   u32 profile_id = (u32) ~ 0;
   int rv;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -749,14 +788,18 @@ set_dpdk_if_hqos_pipe (vlib_main_t * vm, unformat_input_t * input,
       else if (unformat (line_input, "profile %d", &profile_id))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (hw_if_index == (u32) ~ 0)
-    return clib_error_return (0, "please specify valid interface name");
+    {
+      error = clib_error_return (0, "please specify valid interface name");
+      goto done;
+    }
 
   hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
   xd = vec_elt_at_index (dm->devices, hw->dev_instance);
@@ -765,9 +808,15 @@ set_dpdk_if_hqos_pipe (vlib_main_t * vm, unformat_input_t * input,
     rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id,
 			   profile_id);
   if (rv)
-    return clib_error_return (0, "pipe configuration failed");
+    {
+      error = clib_error_return (0, "pipe configuration failed");
+      goto done;
+    }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -797,6 +846,7 @@ set_dpdk_if_hqos_subport (vlib_main_t * vm, unformat_input_t * input,
     .tc_period = 10,
   };
   int rv;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -829,23 +879,33 @@ set_dpdk_if_hqos_subport (vlib_main_t * vm, unformat_input_t * input,
       else if (unformat (line_input, "period %d", &p.tc_period))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (hw_if_index == (u32) ~ 0)
-    return clib_error_return (0, "please specify valid interface name");
+    {
+      error = clib_error_return (0, "please specify valid interface name");
+      goto done;
+    }
 
   hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
   xd = vec_elt_at_index (dm->devices, hw->dev_instance);
 
   rv = rte_sched_subport_config (xd->hqos_ht->hqos, subport_id, &p);
   if (rv)
-    return clib_error_return (0, "subport configuration failed");
+    {
+      error = clib_error_return (0, "subport configuration failed");
+      goto done;
+    }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -872,6 +932,7 @@ set_dpdk_if_hqos_tctbl (vlib_main_t * vm, unformat_input_t * input,
   u32 queue = (u32) ~ 0;
   u32 entry = (u32) ~ 0;
   u32 val, i;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -889,20 +950,33 @@ set_dpdk_if_hqos_tctbl (vlib_main_t * vm, unformat_input_t * input,
       else if (unformat (line_input, "queue %d", &queue))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (hw_if_index == (u32) ~ 0)
-    return clib_error_return (0, "please specify valid interface name");
+    {
+      error = clib_error_return (0, "please specify valid interface name");
+      goto done;
+    }
   if (entry >= 64)
-    return clib_error_return (0, "invalid entry");
+    {
+      error = clib_error_return (0, "invalid entry");
+      goto done;
+    }
   if (tc >= RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE)
-    return clib_error_return (0, "invalid traffic class");
+    {
+      error = clib_error_return (0, "invalid traffic class");
+      goto done;
+    }
   if (queue >= RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS)
-    return clib_error_return (0, "invalid traffic class");
+    {
+      error = clib_error_return (0, "invalid traffic class");
+      goto done;
+    }
 
   hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
   xd = vec_elt_at_index (dm->devices, hw->dev_instance);
@@ -911,7 +985,10 @@ set_dpdk_if_hqos_tctbl (vlib_main_t * vm, unformat_input_t * input,
   uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers");
   /* Should never happen, shut up Coverity warning */
   if (p == 0)
-    return clib_error_return (0, "no worker registrations?");
+    {
+      error = clib_error_return (0, "no worker registrations?");
+      goto done;
+    }
 
   vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0];
   int worker_thread_first = tr->first_index;
@@ -921,7 +998,10 @@ set_dpdk_if_hqos_tctbl (vlib_main_t * vm, unformat_input_t * input,
   for (i = 0; i < worker_thread_count; i++)
     xd->hqos_wt[worker_thread_first + i].hqos_tc_table[entry] = val;
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -939,6 +1019,7 @@ set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input,
   unformat_input_t _line_input, *line_input = &_line_input;
   vlib_thread_main_t *tm = vlib_get_thread_main ();
   dpdk_main_t *dm = &dpdk_main;
+  clib_error_t *error = NULL;
 
   /* Device specific data */
   struct rte_eth_dev_info dev_info;
@@ -984,15 +1065,19 @@ set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input,
       else if (unformat (line_input, "mask %llx", &mask))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   /* Get interface */
   if (hw_if_index == (u32) ~ 0)
-    return clib_error_return (0, "please specify valid interface name");
+    {
+      error = clib_error_return (0, "please specify valid interface name");
+      goto done;
+    }
 
   hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
   xd = vec_elt_at_index (dm->devices, hw->dev_instance);
@@ -1019,7 +1104,7 @@ set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input,
   if (devconf->hqos_enabled == 0)
     {
       vlib_cli_output (vm, "HQoS disabled for this interface");
-      return 0;
+      goto done;
     }
 
   n_subports_per_port = devconf->hqos.port.n_subports_per_port;
@@ -1028,27 +1113,39 @@ set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input,
 
   /* Validate packet field configuration: id, offset and mask */
   if (id >= 3)
-    return clib_error_return (0, "invalid packet field id");
+    {
+      error = clib_error_return (0, "invalid packet field id");
+      goto done;
+    }
 
   switch (id)
     {
     case 0:
       if (dpdk_hqos_validate_mask (mask, n_subports_per_port) != 0)
-	return clib_error_return (0, "invalid subport ID mask "
-				  "(n_subports_per_port = %u)",
-				  n_subports_per_port);
+	{
+	  error = clib_error_return (0, "invalid subport ID mask "
+				     "(n_subports_per_port = %u)",
+				     n_subports_per_port);
+	  goto done;
+	}
       break;
     case 1:
       if (dpdk_hqos_validate_mask (mask, n_pipes_per_subport) != 0)
-	return clib_error_return (0, "invalid pipe ID mask "
-				  "(n_pipes_per_subport = %u)",
-				  n_pipes_per_subport);
+	{
+	  error = clib_error_return (0, "invalid pipe ID mask "
+				     "(n_pipes_per_subport = %u)",
+				     n_pipes_per_subport);
+	  goto done;
+	}
       break;
     case 2:
     default:
       if (dpdk_hqos_validate_mask (mask, tctbl_size) != 0)
-	return clib_error_return (0, "invalid TC table index mask "
-				  "(TC table size = %u)", tctbl_size);
+	{
+	  error = clib_error_return (0, "invalid TC table index mask "
+				     "(TC table size = %u)", tctbl_size);
+	  goto done;
+	}
     }
 
   /* Propagate packet field configuration to all workers */
@@ -1075,7 +1172,10 @@ set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input,
 	  __builtin_ctzll (mask);
       }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -1106,6 +1206,7 @@ show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input,
   dpdk_device_config_t *devconf = 0;
   vlib_thread_registration_t *tr;
   uword *p = 0;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -1117,14 +1218,18 @@ show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input,
 	   &hw_if_index))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (hw_if_index == (u32) ~ 0)
-    return clib_error_return (0, "please specify interface name!!");
+    {
+      error = clib_error_return (0, "please specify interface name!!");
+      goto done;
+    }
 
   hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
   xd = vec_elt_at_index (dm->devices, hw->dev_instance);
@@ -1151,7 +1256,7 @@ show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input,
   if (devconf->hqos_enabled == 0)
     {
       vlib_cli_output (vm, "HQoS disabled for this interface");
-      return 0;
+      goto done;
     }
 
   /* Detect the set of worker threads */
@@ -1159,7 +1264,10 @@ show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input,
 
   /* Should never happen, shut up Coverity warning */
   if (p == 0)
-    return clib_error_return (0, "no worker registrations?");
+    {
+      error = clib_error_return (0, "no worker registrations?");
+      goto done;
+    }
 
   tr = (vlib_thread_registration_t *) p[0];
 
@@ -1284,7 +1392,10 @@ show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input,
     }
 #endif
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -1315,6 +1426,7 @@ show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input,
   u32 qindex;
   struct rte_sched_queue_stats stats;
   u16 qlen;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -1339,14 +1451,18 @@ show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input,
 	;
 
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (hw_if_index == (u32) ~ 0)
-    return clib_error_return (0, "please specify interface name!!");
+    {
+      error = clib_error_return (0, "please specify interface name!!");
+      goto done;
+    }
 
   hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index);
   xd = vec_elt_at_index (dm->devices, hw->dev_instance);
@@ -1373,7 +1489,7 @@ show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input,
   if (devconf->hqos_enabled == 0)
     {
       vlib_cli_output (vm, "HQoS disabled for this interface");
-      return 0;
+      goto done;
     }
 
   /*
@@ -1386,7 +1502,10 @@ show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input,
 
   if (rte_sched_queue_read_stats (xd->hqos_ht->hqos, qindex, &stats, &qlen) !=
       0)
-    return clib_error_return (0, "failed to read stats");
+    {
+      error = clib_error_return (0, "failed to read stats");
+      goto done;
+    }
 
   vlib_cli_output (vm, "%=24s%=16s", "Stats Parameter", "Value");
   vlib_cli_output (vm, "%=24s%=16d", "Packets", stats.n_pkts);
@@ -1399,7 +1518,10 @@ show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input,
   vlib_cli_output (vm, "%=24s%=16d", "Bytes dropped", stats.n_bytes_dropped);
 
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
diff --git a/src/vnet/devices/dpdk/ipsec/cli.c b/src/vnet/devices/dpdk/ipsec/cli.c
index 93df4a64..f9d3a5d0 100644
--- a/src/vnet/devices/dpdk/ipsec/cli.c
+++ b/src/vnet/devices/dpdk/ipsec/cli.c
@@ -111,6 +111,7 @@ lcore_cryptodev_map_fn (vlib_main_t * vm, unformat_input_t * input,
 {
   unformat_input_t _line_input, *line_input = &_line_input;
   u16 detail = 0;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -120,15 +121,19 @@ lcore_cryptodev_map_fn (vlib_main_t * vm, unformat_input_t * input,
       if (unformat (line_input, "verbose"))
 	detail = 1;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   dpdk_ipsec_show_mapping (vm, detail);
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
diff --git a/src/vnet/devices/netmap/cli.c b/src/vnet/devices/netmap/cli.c
index 6157f27c..71363294 100644
--- a/src/vnet/devices/netmap/cli.c
+++ b/src/vnet/devices/netmap/cli.c
@@ -37,6 +37,7 @@ netmap_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
   u8 is_pipe = 0;
   u8 is_master = 0;
   u32 sw_if_index = ~0;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -57,30 +58,48 @@ netmap_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
       else if (unformat (line_input, "slave"))
 	is_master = 0;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
   if (host_if_name == NULL)
-    return clib_error_return (0, "missing host interface name");
+    {
+      error = clib_error_return (0, "missing host interface name");
+      goto done;
+    }
 
   r =
     netmap_create_if (vm, host_if_name, hw_addr_ptr, is_pipe, is_master,
 		      &sw_if_index);
 
   if (r == VNET_API_ERROR_SYSCALL_ERROR_1)
-    return clib_error_return (0, "%s (errno %d)", strerror (errno), errno);
+    {
+      error = clib_error_return (0, "%s (errno %d)", strerror (errno), errno);
+      goto done;
+    }
 
   if (r == VNET_API_ERROR_INVALID_INTERFACE)
-    return clib_error_return (0, "Invalid interface name");
+    {
+      error = clib_error_return (0, "Invalid interface name");
+      goto done;
+    }
 
   if (r == VNET_API_ERROR_SUBIF_ALREADY_EXISTS)
-    return clib_error_return (0, "Interface already exists");
+    {
+      error = clib_error_return (0, "Interface already exists");
+      goto done;
+    }
 
   vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main (),
 		   sw_if_index);
-  return 0;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /*?
@@ -144,6 +163,7 @@ netmap_delete_command_fn (vlib_main_t * vm, unformat_input_t * input,
 {
   unformat_input_t _line_input, *line_input = &_line_input;
   u8 *host_if_name = NULL;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -154,17 +174,25 @@ netmap_delete_command_fn (vlib_main_t * vm, unformat_input_t * input,
       if (unformat (line_input, "name %s", &host_if_name))
 	;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
   if (host_if_name == NULL)
-    return clib_error_return (0, "missing host interface name");
+    {
+      error = clib_error_return (0, "missing host interface name");
+      goto done;
+    }
 
   netmap_delete_if (vm, host_if_name);
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /*?
diff --git a/src/vnet/devices/virtio/vhost-user.c b/src/vnet/devices/virtio/vhost-user.c
index 315daa77..c43f6e67 100644
--- a/src/vnet/devices/virtio/vhost-user.c
+++ b/src/vnet/devices/virtio/vhost-user.c
@@ -2682,6 +2682,7 @@ vhost_user_connect_command_fn (vlib_main_t * vm,
   u32 custom_dev_instance = ~0;
   u8 hwaddr[6];
   u8 *hw = NULL;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -2704,10 +2705,12 @@ vhost_user_connect_command_fn (vlib_main_t * vm,
 	  renumber = 1;
 	}
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
   vnet_main_t *vnm = vnet_get_main ();
 
@@ -2716,14 +2719,18 @@ vhost_user_connect_command_fn (vlib_main_t * vm,
 				  is_server, &sw_if_index, feature_mask,
 				  renumber, custom_dev_instance, hw)))
     {
-      vec_free (sock_filename);
-      return clib_error_return (0, "vhost_user_create_if returned %d", rv);
+      error = clib_error_return (0, "vhost_user_create_if returned %d", rv);
+      goto done;
     }
 
-  vec_free (sock_filename);
   vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main (),
 		   sw_if_index);
-  return 0;
+
+done:
+  vec_free (sock_filename);
+  unformat_free (line_input);
+
+  return error;
 }
 
 clib_error_t *
@@ -2734,6 +2741,7 @@ vhost_user_delete_command_fn (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   u32 sw_if_index = ~0;
   vnet_main_t *vnm = vnet_get_main ();
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -2751,15 +2759,25 @@ vhost_user_delete_command_fn (vlib_main_t * vm,
 	    vnet_get_sup_hw_interface (vnm, sw_if_index);
 	  if (hwif == NULL ||
 	      vhost_user_dev_class.index != hwif->dev_class_index)
-	    return clib_error_return (0, "Not a vhost interface");
+	    {
+	      error = clib_error_return (0, "Not a vhost interface");
+	      goto done;
+	    }
 	}
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
+
   vhost_user_delete_if (vnm, vm, sw_if_index);
-  return 0;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 int
@@ -3286,6 +3304,7 @@ vhost_thread_command_fn (vlib_main_t * vm,
   u32 sw_if_index;
   u8 del = 0;
   int rv;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -3295,9 +3314,9 @@ vhost_thread_command_fn (vlib_main_t * vm,
       (line_input, "%U %d", unformat_vnet_sw_interface, vnet_get_main (),
        &sw_if_index, &worker_thread_index))
     {
-      unformat_free (line_input);
-      return clib_error_return (0, "unknown input `%U'",
-				format_unformat_error, input);
+      error = clib_error_return (0, "unknown input `%U'",
+				 format_unformat_error, line_input);
+      goto done;
     }
 
   if (unformat (line_input, "del"))
@@ -3305,9 +3324,16 @@ vhost_thread_command_fn (vlib_main_t * vm,
 
   if ((rv =
        vhost_user_thread_placement (sw_if_index, worker_thread_index, del)))
-    return clib_error_return (0, "vhost_user_thread_placement returned %d",
-			      rv);
-  return 0;
+    {
+      error = clib_error_return (0, "vhost_user_thread_placement returned %d",
+				 rv);
+      goto done;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 
diff --git a/src/vnet/gre/interface.c b/src/vnet/gre/interface.c
index d624587d..d4476ac4 100644
--- a/src/vnet/gre/interface.c
+++ b/src/vnet/gre/interface.c
@@ -491,6 +491,7 @@ create_gre_tunnel_command_fn (vlib_main_t * vm,
   u32 num_m_args = 0;
   u8 is_add = 1;
   u32 sw_if_index;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (! unformat_user (input, unformat_line_input, line_input))
@@ -508,16 +509,24 @@ create_gre_tunnel_command_fn (vlib_main_t * vm,
     else if (unformat (line_input, "teb"))
       teb = 1;
     else
-      return clib_error_return (0, "unknown input `%U'",
-                                format_unformat_error, input);
+      {
+        error = clib_error_return (0, "unknown input `%U'",
+                                   format_unformat_error, line_input);
+        goto done;
+      }
   }
-  unformat_free (line_input);
 
   if (num_m_args < 2)
-      return clib_error_return (0, "mandatory argument(s) missing");
+    {
+      error = clib_error_return (0, "mandatory argument(s) missing");
+      goto done;
+    }
 
   if (memcmp (&src, &dst, sizeof(src)) == 0)
-      return clib_error_return (0, "src and dst are identical");
+    {
+      error = clib_error_return (0, "src and dst are identical");
+      goto done;
+    }
 
   memset (a, 0, sizeof (*a));
   a->outer_fib_id = outer_fib_id;
@@ -536,15 +545,21 @@ create_gre_tunnel_command_fn (vlib_main_t * vm,
       vlib_cli_output(vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main(), sw_if_index);
       break;
     case VNET_API_ERROR_INVALID_VALUE:
-      return clib_error_return (0, "GRE tunnel already exists...");
+      error = clib_error_return (0, "GRE tunnel already exists...");
+      goto done;
     case VNET_API_ERROR_NO_SUCH_FIB:
-      return clib_error_return (0, "outer fib ID %d doesn't exist\n",
-                                outer_fib_id);
+      error = clib_error_return (0, "outer fib ID %d doesn't exist\n",
+                                 outer_fib_id);
+      goto done;
     default:
-      return clib_error_return (0, "vnet_gre_add_del_tunnel returned %d", rv);
+      error = clib_error_return (0, "vnet_gre_add_del_tunnel returned %d", rv);
+      goto done;
     }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 VLIB_CLI_COMMAND (create_gre_tunnel_command, static) = {
diff --git a/src/vnet/ip/ip4_source_check.c b/src/vnet/ip/ip4_source_check.c
index d461cc88..3af32f2e 100644
--- a/src/vnet/ip/ip4_source_check.c
+++ b/src/vnet/ip/ip4_source_check.c
@@ -399,6 +399,8 @@ set_ip_source_check (vlib_main_t * vm,
   vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index,
 			       is_del == 0, &config, sizeof (config));
 done:
+  unformat_free (line_input);
+
   return error;
 }
 
@@ -531,7 +533,9 @@ ip_source_check_accept (vlib_main_t * vm,
     }
 
 done:
-  return (error);
+  unformat_free (line_input);
+
+  return error;
 }
 
 /*?
diff --git a/src/vnet/ip/ip4_test.c b/src/vnet/ip/ip4_test.c
index 45d17113..73dabfdc 100644
--- a/src/vnet/ip/ip4_test.c
+++ b/src/vnet/ip/ip4_test.c
@@ -143,8 +143,11 @@ thrash (vlib_main_t * vm,
 	  else if (unformat (line_input, "verbose"))
 	    verbose = 1;
 	  else
-	    return clib_error_return (0, "unknown input `%U'",
-				      format_unformat_error, line_input);
+	    {
+	      error = clib_error_return (0, "unknown input `%U'",
+					 format_unformat_error, line_input);
+	      goto done;
+	    }
 	}
     }
 
@@ -178,7 +181,7 @@ thrash (vlib_main_t * vm,
   if (p == 0)
     {
       vlib_cli_output (vm, "Couldn't map fib id %d to fib index\n", table_id);
-      return 0;
+      goto done;
     }
   table_index = p[0];
 
@@ -294,7 +297,11 @@ thrash (vlib_main_t * vm,
 
       pool_free (tm->route_pool);
     }
-  return 0;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /*?
diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c
index 7229591e..6b53137f 100644
--- a/src/vnet/ip/ip6_neighbor.c
+++ b/src/vnet/ip/ip6_neighbor.c
@@ -2923,7 +2923,10 @@ ip6_neighbor_cmd (vlib_main_t * vm, unformat_input_t * main_input,
       else if (unformat (line_input, "ra-lifetime"))
 	{
 	  if (!unformat (line_input, "%d", &ra_lifetime))
-	    return (error = unformat_parse_error (line_input));
+	    {
+	      error = unformat_parse_error (line_input);
+	      goto done;
+	    }
 	  use_lifetime = 1;
 	  break;
 	}
@@ -2931,13 +2934,19 @@ ip6_neighbor_cmd (vlib_main_t * vm, unformat_input_t * main_input,
 	{
 	  if (!unformat
 	      (line_input, "%d %d", &ra_initial_count, &ra_initial_interval))
-	    return (error = unformat_parse_error (line_input));
+	    {
+	      error = unformat_parse_error (line_input);
+	      goto done;
+	    }
 	  break;
 	}
       else if (unformat (line_input, "ra-interval"))
 	{
 	  if (!unformat (line_input, "%d", &ra_max_interval))
-	    return (error = unformat_parse_error (line_input));
+	    {
+	      error = unformat_parse_error (line_input);
+	      goto done;
+	    }
 
 	  if (!unformat (line_input, "%d", &ra_min_interval))
 	    ra_min_interval = 0;
@@ -2949,7 +2958,10 @@ ip6_neighbor_cmd (vlib_main_t * vm, unformat_input_t * main_input,
 	  break;
 	}
       else
-	return (unformat_parse_error (line_input));
+	{
+	  error = unformat_parse_error (line_input);
+	  goto done;
+	}
     }
 
   if (add_radv_info)
@@ -3006,7 +3018,10 @@ ip6_neighbor_cmd (vlib_main_t * vm, unformat_input_t * main_input,
 	  else if (unformat (line_input, "no-onlink"))
 	    no_onlink = 1;
 	  else
-	    return (unformat_parse_error (line_input));
+	    {
+	      error = unformat_parse_error (line_input);
+	      goto done;
+	    }
 	}
 
       ip6_neighbor_ra_prefix (vm, sw_if_index,
@@ -3018,9 +3033,9 @@ ip6_neighbor_cmd (vlib_main_t * vm, unformat_input_t * main_input,
 			      off_link, no_autoconfig, no_onlink, is_no);
     }
 
+done:
   unformat_free (line_input);
 
-done:
   return error;
 }
 
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 0ef0e7a6..807b87b6 100644
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -568,8 +568,6 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 	}
     }
 
-  unformat_free (line_input);
-
   if (vec_len (prefixs) == 0)
     {
       error =
@@ -704,6 +702,7 @@ done:
   vec_free (dpos);
   vec_free (prefixs);
   vec_free (rpaths);
+  unformat_free (line_input);
   return error;
 }
 
@@ -872,8 +871,6 @@ vnet_ip_mroute_cmd (vlib_main_t * vm,
 	}
     }
 
-  unformat_free (line_input);
-
   if (~0 == table_id)
     {
       /*
@@ -970,6 +967,8 @@ vnet_ip_mroute_cmd (vlib_main_t * vm,
 		     (scount * gcount) / (timet[1] - timet[0]));
 
 done:
+  unformat_free (line_input);
+
   return error;
 }
 
@@ -1149,24 +1148,37 @@ probe_neighbor_address (vlib_main_t * vm,
 	  is_ip4 = 0;
 	}
       else
-	return clib_error_return (0, "unknown input '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "unknown input '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (sw_if_index == ~0)
-    return clib_error_return (0, "Interface required, not set.");
+    {
+      error = clib_error_return (0, "Interface required, not set.");
+      goto done;
+    }
   if (address_set == 0)
-    return clib_error_return (0, "ip address required, not set.");
+    {
+      error = clib_error_return (0, "ip address required, not set.");
+      goto done;
+    }
   if (address_set > 1)
-    return clib_error_return (0, "Multiple ip addresses not supported.");
+    {
+      error = clib_error_return (0, "Multiple ip addresses not supported.");
+      goto done;
+    }
 
   if (is_ip4)
     error = ip4_probe_neighbor_wait (vm, &a4, sw_if_index, retry_count);
   else
     error = ip6_probe_neighbor_wait (vm, &a6, sw_if_index, retry_count);
 
+done:
+  unformat_free (line_input);
+
   return error;
 }
 
diff --git a/src/vnet/ipsec-gre/interface.c b/src/vnet/ipsec-gre/interface.c
index 3b6e4ac2..0772ce73 100644
--- a/src/vnet/ipsec-gre/interface.c
+++ b/src/vnet/ipsec-gre/interface.c
@@ -232,6 +232,7 @@ create_ipsec_gre_tunnel_command_fn (vlib_main_t * vm,
   vnet_ipsec_gre_add_del_tunnel_args_t _a, *a = &_a;
   int rv;
   u32 sw_if_index;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -250,16 +251,24 @@ create_ipsec_gre_tunnel_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "remote-sa %d", &rsa))
 	num_m_args++;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
   if (num_m_args < 4)
-    return clib_error_return (0, "mandatory argument(s) missing");
+    {
+      error = clib_error_return (0, "mandatory argument(s) missing");
+      goto done;
+    }
 
   if (memcmp (&src, &dst, sizeof (src)) == 0)
-    return clib_error_return (0, "src and dst are identical");
+    {
+      error = clib_error_return (0, "src and dst are identical");
+      goto done;
+    }
 
   memset (a, 0, sizeof (*a));
   a->is_add = is_add;
@@ -277,14 +286,19 @@ create_ipsec_gre_tunnel_command_fn (vlib_main_t * vm,
 		       vnet_get_main (), sw_if_index);
       break;
     case VNET_API_ERROR_INVALID_VALUE:
-      return clib_error_return (0, "GRE tunnel already exists...");
+      error = clib_error_return (0, "GRE tunnel already exists...");
+      goto done;
     default:
-      return clib_error_return (0,
-				"vnet_ipsec_gre_add_del_tunnel returned %d",
-				rv);
+      error = clib_error_return (0,
+				 "vnet_ipsec_gre_add_del_tunnel returned %d",
+				 rv);
+      goto done;
     }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
diff --git a/src/vnet/ipsec/ipsec_cli.c b/src/vnet/ipsec/ipsec_cli.c
index 3c1e26f2..0e034402 100644
--- a/src/vnet/ipsec/ipsec_cli.c
+++ b/src/vnet/ipsec/ipsec_cli.c
@@ -32,6 +32,7 @@ set_interface_spd_command_fn (vlib_main_t * vm,
   u32 sw_if_index = (u32) ~ 0;
   u32 spd_id;
   int is_add = 1;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -43,14 +44,18 @@ set_interface_spd_command_fn (vlib_main_t * vm,
   else if (unformat (line_input, "del"))
     is_add = 0;
   else
-    return clib_error_return (0, "parse error: '%U'",
-			      format_unformat_error, line_input);
-
-  unformat_free (line_input);
+    {
+      error = clib_error_return (0, "parse error: '%U'",
+				 format_unformat_error, line_input);
+      goto done;
+    }
 
   ipsec_set_interface_spd (vm, sw_if_index, spd_id, is_add);
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -72,7 +77,7 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm,
   ipsec_sa_t sa;
   int is_add = ~0;
   u8 *ck = 0, *ik = 0;
-  clib_error_t *err = 0;
+  clib_error_t *error = NULL;
 
   memset (&sa, 0, sizeof (sa));
 
@@ -90,8 +95,11 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "esp"))
 	sa.protocol = IPSEC_PROTOCOL_ESP;
       else if (unformat (line_input, "ah"))
-	//sa.protocol = IPSEC_PROTOCOL_AH;
-	return clib_error_return (0, "unsupported security protocol 'AH'");
+	{
+	  //sa.protocol = IPSEC_PROTOCOL_AH;
+	  error = clib_error_return (0, "unsupported security protocol 'AH'");
+	  goto done;
+	}
       else
 	if (unformat (line_input, "crypto-key %U", unformat_hex_string, &ck))
 	sa.crypto_key_len = vec_len (ck);
@@ -102,8 +110,12 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm,
 	{
 	  if (sa.crypto_alg < IPSEC_CRYPTO_ALG_AES_CBC_128 ||
 	      sa.crypto_alg >= IPSEC_CRYPTO_N_ALG)
-	    return clib_error_return (0, "unsupported crypto-alg: '%U'",
-				      format_ipsec_crypto_alg, sa.crypto_alg);
+	    {
+	      error = clib_error_return (0, "unsupported crypto-alg: '%U'",
+					 format_ipsec_crypto_alg,
+					 sa.crypto_alg);
+	      goto done;
+	    }
 	}
       else
 	if (unformat (line_input, "integ-key %U", unformat_hex_string, &ik))
@@ -113,8 +125,12 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm,
 	{
 	  if (sa.integ_alg < IPSEC_INTEG_ALG_SHA1_96 ||
 	      sa.integ_alg >= IPSEC_INTEG_N_ALG)
-	    return clib_error_return (0, "unsupported integ-alg: '%U'",
-				      format_ipsec_integ_alg, sa.integ_alg);
+	    {
+	      error = clib_error_return (0, "unsupported integ-alg: '%U'",
+					 format_ipsec_integ_alg,
+					 sa.integ_alg);
+	      goto done;
+	    }
 	}
       else if (unformat (line_input, "tunnel-src %U",
 			 unformat_ip4_address, &sa.tunnel_src_addr.ip4))
@@ -135,12 +151,13 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm,
 	  sa.is_tunnel_ip6 = 1;
 	}
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (sa.crypto_key_len > sizeof (sa.crypto_key))
     sa.crypto_key_len = sizeof (sa.crypto_key);
 
@@ -156,14 +173,17 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm,
   if (is_add)
     {
       ASSERT (im->cb.check_support_cb);
-      err = im->cb.check_support_cb (&sa);
-      if (err)
-	return err;
+      error = im->cb.check_support_cb (&sa);
+      if (error)
+	goto done;
     }
 
   ipsec_add_del_sa (vm, &sa, is_add);
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -183,6 +203,7 @@ ipsec_spd_add_del_command_fn (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   u32 spd_id = ~0;
   int is_add = ~0;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -196,18 +217,25 @@ ipsec_spd_add_del_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "%u", &spd_id))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (spd_id == ~0)
-    return clib_error_return (0, "please specify SPD ID");
+    {
+      error = clib_error_return (0, "please specify SPD ID");
+      goto done;
+    }
 
   ipsec_add_del_spd (vm, spd_id, is_add);
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -230,6 +258,7 @@ ipsec_policy_add_del_command_fn (vlib_main_t * vm,
   int is_add = 0;
   int is_ip_any = 1;
   u32 tmp, tmp2;
+  clib_error_t *error = NULL;
 
   memset (&p, 0, sizeof (p));
   p.lport.stop = p.rport.stop = ~0;
@@ -262,7 +291,10 @@ ipsec_policy_add_del_command_fn (vlib_main_t * vm,
 	     &p.policy))
 	{
 	  if (p.policy == IPSEC_POLICY_ACTION_RESOLVE)
-	    return clib_error_return (0, "unsupported action: 'resolve'");
+	    {
+	      error = clib_error_return (0, "unsupported action: 'resolve'");
+	      goto done;
+	    }
 	}
       else if (unformat (line_input, "sa %u", &p.sa_id))
 	;
@@ -300,19 +332,24 @@ ipsec_policy_add_del_command_fn (vlib_main_t * vm,
 	  p.rport.stop = tmp2;
 	}
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   ipsec_add_del_policy (vm, &p, is_add);
   if (is_ip_any)
     {
       p.is_ipv6 = 1;
       ipsec_add_del_policy (vm, &p, is_add);
     }
-  return 0;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -332,6 +369,7 @@ set_ipsec_sa_key_command_fn (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   ipsec_sa_t sa;
   u8 *ck = 0, *ik = 0;
+  clib_error_t *error = NULL;
 
   memset (&sa, 0, sizeof (sa));
 
@@ -349,12 +387,13 @@ set_ipsec_sa_key_command_fn (vlib_main_t * vm,
 	if (unformat (line_input, "integ-key %U", unformat_hex_string, &ik))
 	sa.integ_key_len = vec_len (ik);
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (sa.crypto_key_len > sizeof (sa.crypto_key))
     sa.crypto_key_len = sizeof (sa.crypto_key);
 
@@ -369,7 +408,10 @@ set_ipsec_sa_key_command_fn (vlib_main_t * vm,
 
   ipsec_set_sa_key (vm, &sa);
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -649,6 +691,7 @@ create_ipsec_tunnel_command_fn (vlib_main_t * vm,
   ipsec_add_del_tunnel_args_t a;
   int rv;
   u32 num_m_args = 0;
+  clib_error_t *error = NULL;
 
   memset (&a, 0, sizeof (a));
   a.is_add = 1;
@@ -673,13 +716,18 @@ create_ipsec_tunnel_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "del"))
 	a.is_add = 0;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
   if (num_m_args < 4)
-    return clib_error_return (0, "mandatory argument(s) missing");
+    {
+      error = clib_error_return (0, "mandatory argument(s) missing");
+      goto done;
+    }
 
   rv = ipsec_add_del_tunnel_if (&a);
 
@@ -689,16 +737,21 @@ create_ipsec_tunnel_command_fn (vlib_main_t * vm,
       break;
     case VNET_API_ERROR_INVALID_VALUE:
       if (a.is_add)
-	return clib_error_return (0,
-				  "IPSec tunnel interface already exists...");
+	error = clib_error_return (0,
+				   "IPSec tunnel interface already exists...");
       else
-	return clib_error_return (0, "IPSec tunnel interface not exists...");
+	error = clib_error_return (0, "IPSec tunnel interface not exists...");
+      goto done;
     default:
-      return clib_error_return (0, "ipsec_register_interface returned %d",
-				rv);
+      error = clib_error_return (0, "ipsec_register_interface returned %d",
+				 rv);
+      goto done;
     }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -720,6 +773,7 @@ set_interface_key_command_fn (vlib_main_t * vm,
   u32 hw_if_index = (u32) ~ 0;
   u32 alg;
   u8 *key = 0;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -748,25 +802,38 @@ set_interface_key_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "%U", unformat_hex_string, &key))
 	;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (type == IPSEC_IF_SET_KEY_TYPE_NONE)
-    return clib_error_return (0, "unknown key type");
+    {
+      error = clib_error_return (0, "unknown key type");
+      goto done;
+    }
 
   if (alg > 0 && vec_len (key) == 0)
-    return clib_error_return (0, "key is not specified");
+    {
+      error = clib_error_return (0, "key is not specified");
+      goto done;
+    }
 
   if (hw_if_index == (u32) ~ 0)
-    return clib_error_return (0, "interface not specified");
+    {
+      error = clib_error_return (0, "interface not specified");
+      goto done;
+    }
 
   ipsec_set_interface_key (im->vnet_main, hw_if_index, type, alg, key);
+
+done:
   vec_free (key);
+  unformat_free (line_input);
 
-  return 0;
+  return error;
 }
 
 /* *INDENT-OFF* */
diff --git a/src/vnet/l2/l2_patch.c b/src/vnet/l2/l2_patch.c
index 5e4691f4..ff3d2f3a 100644
--- a/src/vnet/l2/l2_patch.c
+++ b/src/vnet/l2/l2_patch.c
@@ -315,6 +315,7 @@ test_patch_command_fn (vlib_main_t * vm,
   int rx_set = 0;
   int tx_set = 0;
   int is_add = 1;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -335,10 +336,16 @@ test_patch_command_fn (vlib_main_t * vm,
     }
 
   if (rx_set == 0)
-    return clib_error_return (0, "rx interface not set");
+    {
+      error = clib_error_return (0, "rx interface not set");
+      goto done;
+    }
 
   if (tx_set == 0)
-    return clib_error_return (0, "tx interface not set");
+    {
+      error = clib_error_return (0, "tx interface not set");
+      goto done;
+    }
 
   rv = vnet_l2_patch_add_del (rx_sw_if_index, tx_sw_if_index, is_add);
 
@@ -348,17 +355,24 @@ test_patch_command_fn (vlib_main_t * vm,
       break;
 
     case VNET_API_ERROR_INVALID_SW_IF_INDEX:
-      return clib_error_return (0, "rx interface not a physical port");
+      error = clib_error_return (0, "rx interface not a physical port");
+      goto done;
 
     case VNET_API_ERROR_INVALID_SW_IF_INDEX_2:
-      return clib_error_return (0, "tx interface not a physical port");
+      error = clib_error_return (0, "tx interface not a physical port");
+      goto done;
 
     default:
-      return clib_error_return
+      error = clib_error_return
 	(0, "WARNING: vnet_l2_patch_add_del returned %d", rv);
+      goto done;
     }
 
-  return 0;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /*?
diff --git a/src/vnet/l2/l2_xcrw.c b/src/vnet/l2/l2_xcrw.c
index 70610a85..d08a5d8f 100644
--- a/src/vnet/l2/l2_xcrw.c
+++ b/src/vnet/l2/l2_xcrw.c
@@ -409,6 +409,7 @@ set_l2_xcrw_command_fn (vlib_main_t * vm,
   u8 *rw = 0;
   vnet_main_t *vnm = vnet_get_main ();
   int rv;
+  clib_error_t *error = NULL;
 
 
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -416,8 +417,11 @@ set_l2_xcrw_command_fn (vlib_main_t * vm,
 
   if (!unformat (line_input, "%U",
 		 unformat_vnet_sw_interface, vnm, &l2_sw_if_index))
-    return clib_error_return (0, "unknown input '%U'",
-			      format_unformat_error, line_input);
+    {
+      error = clib_error_return (0, "unknown input '%U'",
+				 format_unformat_error, line_input);
+      goto done;
+    }
 
   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
     {
@@ -436,7 +440,10 @@ set_l2_xcrw_command_fn (vlib_main_t * vm,
     }
 
   if (next_node_index == ~0)
-    return clib_error_return (0, "next node not specified");
+    {
+      error = clib_error_return (0, "next node not specified");
+      goto done;
+    }
 
   if (tx_fib_id != ~0)
     {
@@ -448,7 +455,11 @@ set_l2_xcrw_command_fn (vlib_main_t * vm,
 	p = hash_get (ip4_main.fib_index_by_table_id, tx_fib_id);
 
       if (p == 0)
-	return clib_error_return (0, "nonexistent tx_fib_id %d", tx_fib_id);
+	{
+	  error =
+	    clib_error_return (0, "nonexistent tx_fib_id %d", tx_fib_id);
+	  goto done;
+	}
 
       tx_fib_index = p[0];
     }
@@ -463,16 +474,21 @@ set_l2_xcrw_command_fn (vlib_main_t * vm,
       break;
 
     case VNET_API_ERROR_INVALID_SW_IF_INDEX:
-      return clib_error_return (0, "%U not cross-connected",
-				format_vnet_sw_if_index_name,
-				vnm, l2_sw_if_index);
+      error = clib_error_return (0, "%U not cross-connected",
+				 format_vnet_sw_if_index_name,
+				 vnm, l2_sw_if_index);
+      goto done;
+
     default:
-      return clib_error_return (0, "vnet_configure_l2_xcrw returned %d", rv);
+      error = clib_error_return (0, "vnet_configure_l2_xcrw returned %d", rv);
+      goto done;
     }
 
+done:
   vec_free (rw);
+  unformat_free (line_input);
 
-  return 0;
+  return error;
 }
 
 /*?
diff --git a/src/vnet/l2tp/l2tp.c b/src/vnet/l2tp/l2tp.c
index a4531dab..2d323397 100644
--- a/src/vnet/l2tp/l2tp.c
+++ b/src/vnet/l2tp/l2tp.c
@@ -427,6 +427,7 @@ create_l2tpv3_tunnel_command_fn (vlib_main_t * vm,
   u32 sw_if_index;
   u32 encap_fib_id = ~0;
   u32 encap_fib_index = ~0;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -455,18 +456,22 @@ create_l2tpv3_tunnel_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "l2-sublayer-present"))
 	l2_sublayer_present = 1;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
-  unformat_free (line_input);
-
   if (encap_fib_id != ~0)
     {
       uword *p;
       ip6_main_t *im = &ip6_main;
       if (!(p = hash_get (im->fib_index_by_table_id, encap_fib_id)))
-	return clib_error_return (0, "No fib with id %d", encap_fib_id);
+	{
+	  error = clib_error_return (0, "No fib with id %d", encap_fib_id);
+	  goto done;
+	}
       encap_fib_index = p[0];
     }
   else
@@ -475,9 +480,15 @@ create_l2tpv3_tunnel_command_fn (vlib_main_t * vm,
     }
 
   if (our_address_set == 0)
-    return clib_error_return (0, "our address not specified");
+    {
+      error = clib_error_return (0, "our address not specified");
+      goto done;
+    }
   if (client_address_set == 0)
-    return clib_error_return (0, "client address not specified");
+    {
+      error = clib_error_return (0, "client address not specified");
+      goto done;
+    }
 
   rv = create_l2tpv3_ipv6_tunnel (lm, &client_address, &our_address,
 				  local_session_id, remote_session_id,
@@ -491,16 +502,22 @@ create_l2tpv3_tunnel_command_fn (vlib_main_t * vm,
 		       vnet_get_main (), sw_if_index);
       break;
     case VNET_API_ERROR_INVALID_VALUE:
-      return clib_error_return (0, "session already exists...");
+      error = clib_error_return (0, "session already exists...");
+      goto done;
 
     case VNET_API_ERROR_NO_SUCH_ENTRY:
-      return clib_error_return (0, "session does not exist...");
+      error = clib_error_return (0, "session does not exist...");
+      goto done;
 
     default:
-      return clib_error_return (0, "l2tp_session_add_del returned %d", rv);
+      error = clib_error_return (0, "l2tp_session_add_del returned %d", rv);
+      goto done;
     }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
diff --git a/src/vnet/lisp-cp/lisp_cli.c b/src/vnet/lisp-cp/lisp_cli.c
index 25d11c61..05df9fb6 100644
--- a/src/vnet/lisp-cp/lisp_cli.c
+++ b/src/vnet/lisp-cp/lisp_cli.c
@@ -25,6 +25,7 @@ lisp_show_adjacencies_command_fn (vlib_main_t * vm,
   vlib_cli_output (vm, "%s %40s\n", "leid", "reid");
   unformat_input_t _line_input, *line_input = &_line_input;
   u32 vni = ~0;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -38,14 +39,14 @@ lisp_show_adjacencies_command_fn (vlib_main_t * vm,
 	{
 	  vlib_cli_output (vm, "parse error: '%U'",
 			   format_unformat_error, line_input);
-	  return 0;
+	  goto done;
 	}
     }
 
   if (~0 == vni)
     {
       vlib_cli_output (vm, "error: no vni specified!");
-      return 0;
+      goto done;
     }
 
   adjs = vnet_lisp_adjacencies_get_by_vni (vni);
@@ -57,7 +58,10 @@ lisp_show_adjacencies_command_fn (vlib_main_t * vm,
   }
   vec_free (adjs);
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -77,6 +81,7 @@ lisp_add_del_map_server_command_fn (vlib_main_t * vm,
   u8 is_add = 1, ip_set = 0;
   ip_address_t ip;
   unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -94,14 +99,14 @@ lisp_add_del_map_server_command_fn (vlib_main_t * vm,
 	{
 	  vlib_cli_output (vm, "parse error: '%U'",
 			   format_unformat_error, line_input);
-	  return 0;
+	  goto done;
 	}
     }
 
   if (!ip_set)
     {
       vlib_cli_output (vm, "map-server ip address not set!");
-      return 0;
+      goto done;
     }
 
   rv = vnet_lisp_add_del_map_server (&ip, is_add);
@@ -109,7 +114,10 @@ lisp_add_del_map_server_command_fn (vlib_main_t * vm,
     vlib_cli_output (vm, "failed to %s map-server!",
 		     is_add ? "add" : "delete");
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -191,7 +199,7 @@ lisp_add_del_local_eid_command_fn (vlib_main_t * vm, unformat_input_t * input,
   if (key && (0 == key_id))
     {
       vlib_cli_output (vm, "invalid key_id!");
-      return 0;
+      goto done;;
     }
 
   gid_address_copy (&a->eid, &eid);
@@ -213,6 +221,8 @@ done:
     vec_free (locator_set_name);
   gid_address_free (&a->eid);
   vec_free (a->key);
+  unformat_free (line_input);
+
   return error;
 }
 
@@ -233,6 +243,7 @@ lisp_eid_table_map_command_fn (vlib_main_t * vm,
   u8 is_add = 1, is_l2 = 0;
   u32 vni = 0, dp_id = 0;
   unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -250,11 +261,16 @@ lisp_eid_table_map_command_fn (vlib_main_t * vm,
 	is_l2 = 1;
       else
 	{
-	  return unformat_parse_error (line_input);
+	  error = unformat_parse_error (line_input);
+	  goto done;
 	}
     }
   vnet_lisp_eid_table_map (vni, dp_id, is_l2, is_add);
-  return 0;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -479,7 +495,7 @@ lisp_add_del_adjacency_command_fn (vlib_main_t * vm, unformat_input_t * input,
 	  != ip_prefix_version (leid_ippref)))
     {
       clib_warning ("remote and local EIDs are of different types!");
-      return error;
+      goto done;
     }
 
   memset (a, 0, sizeof (a[0]));
@@ -512,6 +528,7 @@ lisp_map_request_mode_command_fn (vlib_main_t * vm,
 {
   unformat_input_t _i, *i = &_i;
   map_request_mode_t mr_mode = _MR_MODE_MAX;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, i))
@@ -533,12 +550,15 @@ lisp_map_request_mode_command_fn (vlib_main_t * vm,
   if (_MR_MODE_MAX == mr_mode)
     {
       clib_warning ("No LISP map request mode entered!");
-      return 0;
+      goto done;
     }
 
   vnet_lisp_set_map_request_mode (mr_mode);
+
 done:
-  return 0;
+  unformat_free (i);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -630,7 +650,10 @@ lisp_pitr_set_locator_set_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "disable"))
 	is_add = 0;
       else
-	return clib_error_return (0, "parse error");
+	{
+	  error = clib_error_return (0, "parse error");
+	  goto done;
+	}
     }
 
   if (!locator_name_set)
@@ -648,6 +671,8 @@ lisp_pitr_set_locator_set_command_fn (vlib_main_t * vm,
 done:
   if (locator_set_name)
     vec_free (locator_set_name);
+  unformat_free (line_input);
+
   return error;
 }
 
@@ -771,6 +796,7 @@ lisp_show_eid_table_command_fn (vlib_main_t * vm,
   gid_address_t eid;
   u8 print_all = 1;
   u8 filter = 0;
+  clib_error_t *error = NULL;
 
   memset (&eid, 0, sizeof (eid));
 
@@ -787,8 +813,11 @@ lisp_show_eid_table_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "remote"))
 	filter = 2;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
   vlib_cli_output (vm, "%-35s%-20s%-30s%-20s%-s",
@@ -818,7 +847,7 @@ lisp_show_eid_table_command_fn (vlib_main_t * vm,
     {
       mi = gid_dictionary_lookup (&lcm->mapping_index_by_gid, &eid);
       if ((u32) ~ 0 == mi)
-	return 0;
+	goto done;
 
       mapit = pool_elt_at_index (lcm->mapping_pool, mi);
       locator_set_t *ls = pool_elt_at_index (lcm->locator_set_pool,
@@ -827,14 +856,17 @@ lisp_show_eid_table_command_fn (vlib_main_t * vm,
       if (filter && !((1 == filter && ls->local) ||
 		      (2 == filter && !ls->local)))
 	{
-	  return 0;
+	  goto done;
 	}
 
       vlib_cli_output (vm, "%U,", format_eid_entry, lcm->vnet_main,
 		       lcm, mapit, ls);
     }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -853,6 +885,7 @@ lisp_enable_disable_command_fn (vlib_main_t * vm, unformat_input_t * input,
   unformat_input_t _line_input, *line_input = &_line_input;
   u8 is_enabled = 0;
   u8 is_set = 0;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -869,16 +902,24 @@ lisp_enable_disable_command_fn (vlib_main_t * vm, unformat_input_t * input,
 	is_set = 1;
       else
 	{
-	  return clib_error_return (0, "parse error: '%U'",
-				    format_unformat_error, line_input);
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
 	}
     }
 
   if (!is_set)
-    return clib_error_return (0, "state not set");
+    {
+      error = clib_error_return (0, "state not set");
+      goto done;
+    }
 
   vnet_lisp_enable_disable (is_enabled);
-  return 0;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -897,6 +938,7 @@ lisp_map_register_enable_disable_command_fn (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   u8 is_enabled = 0;
   u8 is_set = 0;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -915,18 +957,22 @@ lisp_map_register_enable_disable_command_fn (vlib_main_t * vm,
 	{
 	  vlib_cli_output (vm, "parse error: '%U'", format_unformat_error,
 			   line_input);
-	  return 0;
+	  goto done;
 	}
     }
 
   if (!is_set)
     {
       vlib_cli_output (vm, "state not set!");
-      return 0;
+      goto done;
     }
 
   vnet_lisp_map_register_enable_disable (is_enabled);
-  return 0;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -945,6 +991,7 @@ lisp_rloc_probe_enable_disable_command_fn (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   u8 is_enabled = 0;
   u8 is_set = 0;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -963,18 +1010,22 @@ lisp_rloc_probe_enable_disable_command_fn (vlib_main_t * vm,
 	{
 	  vlib_cli_output (vm, "parse error: '%U'", format_unformat_error,
 			   line_input);
-	  return 0;
+	  goto done;
 	}
     }
 
   if (!is_set)
     {
       vlib_cli_output (vm, "state not set!");
-      return 0;
+      goto done;
     }
 
   vnet_lisp_rloc_probe_enable_disable (is_enabled);
-  return 0;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -1022,6 +1073,7 @@ lisp_show_eid_table_map_command_fn (vlib_main_t * vm,
   lisp_cp_main_t *lcm = vnet_lisp_cp_get_main ();
   uword *vni_table = 0;
   u8 is_l2 = 0;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -1040,14 +1092,17 @@ lisp_show_eid_table_map_command_fn (vlib_main_t * vm,
 	  is_l2 = 0;
 	}
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
   if (!vni_table)
     {
       vlib_cli_output (vm, "Error: expected l2|l3 param!\n");
-      return 0;
+      goto done;
     }
 
   vlib_cli_output (vm, "%=10s%=10s", "VNI", is_l2 ? "BD" : "VRF");
@@ -1059,7 +1114,10 @@ lisp_show_eid_table_map_command_fn (vlib_main_t * vm,
   }));
   /* *INDENT-ON* */
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
@@ -1131,6 +1189,8 @@ done:
   vec_free (locators);
   if (locator_set_name)
     vec_free (locator_set_name);
+  unformat_free (line_input);
+
   return error;
 }
 
@@ -1205,6 +1265,8 @@ lisp_add_del_locator_in_set_command_fn (vlib_main_t * vm,
 done:
   vec_free (locators);
   vec_free (locator_set_name);
+  unformat_free (line_input);
+
   return error;
 }
 
@@ -1322,6 +1384,8 @@ lisp_add_del_map_resolver_command_fn (vlib_main_t * vm,
     }
 
 done:
+  unformat_free (line_input);
+
   return error;
 }
 
@@ -1372,11 +1436,11 @@ lisp_add_del_mreq_itr_rlocs_command_fn (vlib_main_t * vm,
 				 is_add ? "add" : "delete");
     }
 
+done:
   vec_free (locator_set_name);
+  unformat_free (line_input);
 
-done:
   return error;
-
 }
 
 /* *INDENT-OFF* */
@@ -1438,7 +1502,10 @@ lisp_use_petr_set_locator_set_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "disable"))
 	is_add = 0;
       else
-	return clib_error_return (0, "parse error");
+	{
+	  error = clib_error_return (0, "parse error");
+	  goto done;
+	}
     }
 
   if (!ip_set)
@@ -1454,6 +1521,8 @@ lisp_use_petr_set_locator_set_command_fn (vlib_main_t * vm,
     }
 
 done:
+  unformat_free (line_input);
+
   return error;
 }
 
diff --git a/src/vnet/lisp-gpe/interface.c b/src/vnet/lisp-gpe/interface.c
index 2142e095..19ac22e7 100644
--- a/src/vnet/lisp-gpe/interface.c
+++ b/src/vnet/lisp-gpe/interface.c
@@ -794,6 +794,7 @@ lisp_gpe_add_del_iface_command_fn (vlib_main_t * vm, unformat_input_t * input,
   u32 table_id, vni, bd_id;
   u8 vni_is_set = 0, vrf_is_set = 0, bd_index_is_set = 0;
   u8 nsh_iface = 0;
+  clib_error_t *error = NULL;
 
   if (vnet_lisp_gpe_enable_disable_status () == 0)
     {
@@ -828,8 +829,9 @@ lisp_gpe_add_del_iface_command_fn (vlib_main_t * vm, unformat_input_t * input,
 	}
       else
 	{
-	  return clib_error_return (0, "parse error: '%U'",
-				    format_unformat_error, line_input);
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
 	}
     }
 
@@ -839,7 +841,8 @@ lisp_gpe_add_del_iface_command_fn (vlib_main_t * vm, unformat_input_t * input,
 	{
 	  if (~0 == lisp_gpe_add_nsh_iface (&lisp_gpe_main))
 	    {
-	      return clib_error_return (0, "NSH interface not created");
+	      error = clib_error_return (0, "NSH interface not created");
+	      goto done;
 	    }
 	}
       else
@@ -850,21 +853,34 @@ lisp_gpe_add_del_iface_command_fn (vlib_main_t * vm, unformat_input_t * input,
     }
 
   if (vrf_is_set && bd_index_is_set)
-    return clib_error_return (0,
-			      "Cannot set both vrf and brdige domain index!");
+    {
+      error = clib_error_return
+	(0, "Cannot set both vrf and brdige domain index!");
+      goto done;
+    }
 
   if (!vni_is_set)
-    return clib_error_return (0, "vni must be set!");
+    {
+      error = clib_error_return (0, "vni must be set!");
+      goto done;
+    }
 
   if (!vrf_is_set && !bd_index_is_set)
-    return clib_error_return (0, "vrf or bridge domain index must be set!");
+    {
+      error =
+	clib_error_return (0, "vrf or bridge domain index must be set!");
+      goto done;
+    }
 
   if (bd_index_is_set)
     {
       if (is_add)
 	{
 	  if (~0 == lisp_gpe_tenant_l2_iface_add_or_lock (vni, bd_id))
-	    return clib_error_return (0, "L2 interface not created");
+	    {
+	      error = clib_error_return (0, "L2 interface not created");
+	      goto done;
+	    }
 	}
       else
 	lisp_gpe_tenant_l2_iface_unlock (vni);
@@ -874,13 +890,35 @@ lisp_gpe_add_del_iface_command_fn (vlib_main_t * vm, unformat_input_t * input,
       if (is_add)
 	{
 	  if (~0 == lisp_gpe_tenant_l3_iface_add_or_lock (vni, table_id))
-	    return clib_error_return (0, "L3 interface not created");
+	    {
+	      error = clib_error_return (0, "L3 interface not created");
+	      goto done;
+	    }
 	}
       else
 	lisp_gpe_tenant_l3_iface_unlock (vni);
     }
 
-  return (NULL);
+  if (nsh_iface)
+    {
+      if (is_add)
+	{
+	  if (~0 == lisp_gpe_add_nsh_iface (&lisp_gpe_main))
+	    {
+	      error = clib_error_return (0, "NSH interface not created");
+	      goto done;
+	    }
+	  else
+	    {
+	      lisp_gpe_del_nsh_iface (&lisp_gpe_main);
+	    }
+	}
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
diff --git a/src/vnet/lisp-gpe/lisp_gpe.c b/src/vnet/lisp-gpe/lisp_gpe.c
index 1f8afdae..f2fbcbd5 100644
--- a/src/vnet/lisp-gpe/lisp_gpe.c
+++ b/src/vnet/lisp-gpe/lisp_gpe.c
@@ -218,6 +218,7 @@ lisp_gpe_enable_disable_command_fn (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   u8 is_en = 1;
   vnet_lisp_gpe_enable_disable_args_t _a, *a = &_a;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -231,12 +232,18 @@ lisp_gpe_enable_disable_command_fn (vlib_main_t * vm,
 	is_en = 0;
       else
 	{
-	  return clib_error_return (0, "parse error: '%U'",
-				    format_unformat_error, line_input);
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
 	}
     }
   a->is_en = is_en;
-  return vnet_lisp_gpe_enable_disable (a);
+  error = vnet_lisp_gpe_enable_disable (a);
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
diff --git a/src/vnet/map/map.c b/src/vnet/map/map.c
index aeec6a94..a2d28118 100644
--- a/src/vnet/map/map.c
+++ b/src/vnet/map/map.c
@@ -465,6 +465,8 @@ map_security_check_command_fn (vlib_main_t * vm,
 {
   unformat_input_t _line_input, *line_input = &_line_input;
   map_main_t *mm = &map_main;
+  clib_error_t *error = NULL;
+
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -476,11 +478,17 @@ map_security_check_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "on"))
 	mm->sec_check = true;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
+
+done:
   unformat_free (line_input);
-  return 0;
+
+  return error;
 }
 
 static clib_error_t *
@@ -490,6 +498,8 @@ map_security_check_frag_command_fn (vlib_main_t * vm,
 {
   unformat_input_t _line_input, *line_input = &_line_input;
   map_main_t *mm = &map_main;
+  clib_error_t *error = NULL;
+
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -501,11 +511,17 @@ map_security_check_frag_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "on"))
 	mm->sec_check_frag = true;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
+
+done:
   unformat_free (line_input);
-  return 0;
+
+  return error;
 }
 
 static clib_error_t *
@@ -523,6 +539,7 @@ map_add_domain_command_fn (vlib_main_t * vm,
   u32 mtu = 0;
   u8 flags = 0;
   ip6_src_len = 128;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -559,20 +576,28 @@ map_add_domain_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "map-t"))
 	flags |= MAP_DOMAIN_TRANSLATION;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
   if (num_m_args < 3)
-    return clib_error_return (0, "mandatory argument(s) missing");
+    {
+      error = clib_error_return (0, "mandatory argument(s) missing");
+      goto done;
+    }
 
   map_create_domain (&ip4_prefix, ip4_prefix_len,
 		     &ip6_prefix, ip6_prefix_len, &ip6_src, ip6_src_len,
 		     ea_bits_len, psid_offset, psid_length, &map_domain_index,
 		     mtu, flags);
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 static clib_error_t *
@@ -582,6 +607,7 @@ map_del_domain_command_fn (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   u32 num_m_args = 0;
   u32 map_domain_index;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -592,17 +618,25 @@ map_del_domain_command_fn (vlib_main_t * vm,
       if (unformat (line_input, "index %d", &map_domain_index))
 	num_m_args++;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
   if (num_m_args != 1)
-    return clib_error_return (0, "mandatory argument(s) missing");
+    {
+      error = clib_error_return (0, "mandatory argument(s) missing");
+      goto done;
+    }
 
   map_delete_domain (map_domain_index);
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 static clib_error_t *
@@ -613,6 +647,7 @@ map_add_rule_command_fn (vlib_main_t * vm,
   ip6_address_t tep;
   u32 num_m_args = 0;
   u32 psid = 0, map_domain_index;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -628,19 +663,29 @@ map_add_rule_command_fn (vlib_main_t * vm,
 	if (unformat (line_input, "ip6-dst %U", unformat_ip6_address, &tep))
 	num_m_args++;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
   if (num_m_args != 3)
-    return clib_error_return (0, "mandatory argument(s) missing");
+    {
+      error = clib_error_return (0, "mandatory argument(s) missing");
+      goto done;
+    }
 
   if (map_add_del_psid (map_domain_index, psid, &tep, 1) != 0)
     {
-      return clib_error_return (0, "Failing to add Mapping Rule");
+      error = clib_error_return (0, "Failing to add Mapping Rule");
+      goto done;
     }
-  return 0;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 #if MAP_SKIP_IP6_LOOKUP
@@ -653,6 +698,7 @@ map_pre_resolve_command_fn (vlib_main_t * vm,
   ip4_address_t ip4nh;
   ip6_address_t ip6nh;
   map_main_t *mm = &map_main;
+  clib_error_t *error = NULL;
 
   memset (&ip4nh, 0, sizeof (ip4nh));
   memset (&ip6nh, 0, sizeof (ip6nh));
@@ -669,14 +715,19 @@ map_pre_resolve_command_fn (vlib_main_t * vm,
 	if (unformat (line_input, "ip6-nh %U", unformat_ip6_address, &ip6nh))
 	mm->preresolve_ip6 = ip6nh;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
   map_pre_resolve (&ip4nh, &ip6nh);
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 #endif
 
@@ -688,6 +739,7 @@ map_icmp_relay_source_address_command_fn (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   ip4_address_t icmp_src_address;
   map_main_t *mm = &map_main;
+  clib_error_t *error = NULL;
 
   mm->icmp4_src_address.as_u32 = 0;
 
@@ -701,12 +753,17 @@ map_icmp_relay_source_address_command_fn (vlib_main_t * vm,
 	  (line_input, "%U", unformat_ip4_address, &icmp_src_address))
 	mm->icmp4_src_address = icmp_src_address;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
+
+done:
   unformat_free (line_input);
 
-  return 0;
+  return error;
 }
 
 static clib_error_t *
@@ -717,6 +774,7 @@ map_icmp_unreachables_command_fn (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   map_main_t *mm = &map_main;
   int num_m_args = 0;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -730,16 +788,21 @@ map_icmp_unreachables_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "off"))
 	mm->icmp6_enabled = false;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
 
   if (num_m_args != 1)
-    return clib_error_return (0, "mandatory argument(s) missing");
+    error = clib_error_return (0, "mandatory argument(s) missing");
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 static clib_error_t *
@@ -748,6 +811,7 @@ map_fragment_command_fn (vlib_main_t * vm,
 {
   unformat_input_t _line_input, *line_input = &_line_input;
   map_main_t *mm = &map_main;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -760,12 +824,17 @@ map_fragment_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "outer"))
 	mm->frag_inner = false;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
+
+done:
   unformat_free (line_input);
 
-  return 0;
+  return error;
 }
 
 static clib_error_t *
@@ -775,6 +844,7 @@ map_fragment_df_command_fn (vlib_main_t * vm,
 {
   unformat_input_t _line_input, *line_input = &_line_input;
   map_main_t *mm = &map_main;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -787,12 +857,17 @@ map_fragment_df_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "off"))
 	mm->frag_ignore_df = false;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
+
+done:
   unformat_free (line_input);
 
-  return 0;
+  return error;
 }
 
 static clib_error_t *
@@ -803,6 +878,7 @@ map_traffic_class_command_fn (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   map_main_t *mm = &map_main;
   u32 tc = 0;
+  clib_error_t *error = NULL;
 
   mm->tc_copy = false;
 
@@ -817,12 +893,17 @@ map_traffic_class_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "%x", &tc))
 	mm->tc = tc & 0xff;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
+
+done:
   unformat_free (line_input);
 
-  return 0;
+  return error;
 }
 
 static u8 *
@@ -922,6 +1003,7 @@ show_map_domain_command_fn (vlib_main_t * vm, unformat_input_t * input,
   map_domain_t *d;
   bool counters = false;
   u32 map_domain_index = ~0;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -934,10 +1016,12 @@ show_map_domain_command_fn (vlib_main_t * vm, unformat_input_t * input,
       else if (unformat (line_input, "index %d", &map_domain_index))
 	;
       else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
-  unformat_free (line_input);
 
   if (pool_elts (mm->domains) == 0)
     vlib_cli_output (vm, "No MAP domains are configured...");
@@ -952,15 +1036,19 @@ show_map_domain_command_fn (vlib_main_t * vm, unformat_input_t * input,
     {
       if (pool_is_free_index (mm->domains, map_domain_index))
 	{
-	  return clib_error_return (0, "MAP domain does not exists %d",
-				    map_domain_index);
+	  error = clib_error_return (0, "MAP domain does not exists %d",
+				     map_domain_index);
+	  goto done;
 	}
 
       d = pool_elt_at_index (mm->domains, map_domain_index);
       vlib_cli_output (vm, "%U", format_map_domain, d, counters);
     }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 static clib_error_t *
diff --git a/src/vnet/mpls/mpls.c b/src/vnet/mpls/mpls.c
index 0e610e17..7ae4aa00 100644
--- a/src/vnet/mpls/mpls.c
+++ b/src/vnet/mpls/mpls.c
@@ -470,6 +470,8 @@ vnet_mpls_local_label (vlib_main_t * vm,
   }
 
 done:
+  unformat_free (line_input);
+
   return error;
 }
 
diff --git a/src/vnet/mpls/mpls_tunnel.c b/src/vnet/mpls/mpls_tunnel.c
index 8d1e30a3..e488271d 100644
--- a/src/vnet/mpls/mpls_tunnel.c
+++ b/src/vnet/mpls/mpls_tunnel.c
@@ -535,6 +535,7 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm,
     fib_route_path_t rpath, *rpaths = NULL;
     mpls_label_t out_label = MPLS_LABEL_INVALID, *labels = NULL;
     u32 sw_if_index;
+    clib_error_t *error = NULL;
 
     memset(&rpath, 0, sizeof(rpath));
 
@@ -595,8 +596,11 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm,
 	else if (unformat (line_input, "l2-only"))
 	    l2_only = 1;
 	else
-	    return clib_error_return (0, "unknown input '%U'",
-				      format_unformat_error, line_input);
+	{
+	    error = clib_error_return (0, "unknown input '%U'",
+				       format_unformat_error, line_input);
+	    goto done;
+	}
     }
 
     if (is_del)
@@ -606,17 +610,22 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm,
     else
     {
 	if (0 == vec_len(labels))
-	    return clib_error_return (0, "No Output Labels '%U'",
-				      format_unformat_error, line_input);
+	{
+	    error = clib_error_return (0, "No Output Labels '%U'",
+				       format_unformat_error, line_input);
+	    goto done;
+	}
 
 	vec_add1(rpaths, rpath);
 	vnet_mpls_tunnel_add(rpaths, labels, l2_only, &sw_if_index);
     }
 
+done:
     vec_free(labels);
     vec_free(rpaths);
+    unformat_free (line_input);
 
-    return (NULL);
+    return error;
 }
 
 /*?
diff --git a/src/vnet/pg/cli.c b/src/vnet/pg/cli.c
index f5896b43..3c249a7b 100644
--- a/src/vnet/pg/cli.c
+++ b/src/vnet/pg/cli.c
@@ -547,21 +547,30 @@ pg_capture_cmd_fn (vlib_main_t * vm,
       else
 	{
 	  error = clib_error_create ("unknown input `%U'",
-				     format_unformat_error, input);
-	  return error;
+				     format_unformat_error, line_input);
+	  goto done;
 	}
     }
 
   if (!hi)
-    return clib_error_return (0, "Please specify interface name");
+    {
+      error = clib_error_return (0, "Please specify interface name");
+      goto done;
+    }
 
   if (hi->dev_class_index != pg_dev_class.index)
-    return clib_error_return (0, "Please specify packet-generator interface");
+    {
+      error =
+	clib_error_return (0, "Please specify packet-generator interface");
+      goto done;
+    }
 
   if (!pcap_file_name && is_disable == 0)
-    return clib_error_return (0, "Please specify pcap file name");
+    {
+      error = clib_error_return (0, "Please specify pcap file name");
+      goto done;
+    }
 
-  unformat_free (line_input);
 
   pg_capture_args_t _a, *a = &_a;
 
@@ -572,6 +581,10 @@ pg_capture_cmd_fn (vlib_main_t * vm,
   a->count = count;
 
   error = pg_capture (a);
+
+done:
+  unformat_free (line_input);
+
   return error;
 }
 
@@ -590,6 +603,7 @@ create_pg_if_cmd_fn (vlib_main_t * vm,
   pg_main_t *pg = &pg_main;
   unformat_input_t _line_input, *line_input = &_line_input;
   u32 if_id;
+  clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
@@ -600,14 +614,19 @@ create_pg_if_cmd_fn (vlib_main_t * vm,
 	;
 
       else
-	return clib_error_create ("unknown input `%U'",
-				  format_unformat_error, input);
+	{
+	  error = clib_error_create ("unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
+  pg_interface_add_or_get (pg, if_id);
+
+done:
   unformat_free (line_input);
 
-  pg_interface_add_or_get (pg, if_id);
-  return 0;
+  return error;
 }
 
 /* *INDENT-OFF* */
diff --git a/src/vnet/policer/node_funcs.c b/src/vnet/policer/node_funcs.c
index 1f4997ff..457dd09f 100644
--- a/src/vnet/policer/node_funcs.c
+++ b/src/vnet/policer/node_funcs.c
@@ -447,6 +447,7 @@ test_policer_command_fn (vlib_main_t * vm,
   int rx_set = 0;
   int is_add = 1;
   int is_show = 0;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -468,7 +469,10 @@ test_policer_command_fn (vlib_main_t * vm,
     }
 
   if (rx_set == 0)
-    return clib_error_return (0, "interface not set");
+    {
+      error = clib_error_return (0, "interface not set");
+      goto done;
+    }
 
   if (is_show)
     {
@@ -477,12 +481,13 @@ test_policer_command_fn (vlib_main_t * vm,
       policer = pool_elt_at_index (pm->policers, pi);
 
       vlib_cli_output (vm, "%U", format_policer_instance, policer);
-      return 0;
+      goto done;
     }
 
   if (is_add && config_name == 0)
     {
-      return clib_error_return (0, "policer config name required");
+      error = clib_error_return (0, "policer config name required");
+      goto done;
     }
 
   rv = test_policer_add_del (rx_sw_if_index, config_name, is_add);
@@ -493,11 +498,15 @@ test_policer_command_fn (vlib_main_t * vm,
       break;
 
     default:
-      return clib_error_return
+      error = clib_error_return
 	(0, "WARNING: vnet_vnet_policer_add_del returned %d", rv);
+      goto done;
     }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /* *INDENT-OFF* */
diff --git a/src/vnet/policer/policer.c b/src/vnet/policer/policer.c
index 290a6af5..cd754e29 100644
--- a/src/vnet/policer/policer.c
+++ b/src/vnet/policer/policer.c
@@ -413,6 +413,7 @@ configure_policer_command_fn (vlib_main_t * vm,
   u8 is_add = 1;
   u8 *name = 0;
   u32 pi;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -433,13 +434,19 @@ configure_policer_command_fn (vlib_main_t * vm,
       foreach_config_param
 #undef _
 	else
-	return clib_error_return (0, "unknown input `%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
     }
 
+  error = policer_add_del (vm, name, &c, &pi, is_add);
+
+done:
   unformat_free (line_input);
 
-  return policer_add_del (vm, name, &c, &pi, is_add);
+  return error;
 }
 
 /* *INDENT-OFF* */
diff --git a/src/vnet/unix/tapcli.c b/src/vnet/unix/tapcli.c
index 48e81b50..25c930c6 100644
--- a/src/vnet/unix/tapcli.c
+++ b/src/vnet/unix/tapcli.c
@@ -1308,6 +1308,7 @@ tap_connect_command_fn (vlib_main_t * vm,
   int ip6_address_set = 0;
   u32 ip4_mask_width = 0;
   u32 ip6_mask_width = 0;
+  clib_error_t *error = NULL;
 
   if (tm->is_disabled)
     return clib_error_return (0, "device disabled...");
@@ -1336,12 +1337,18 @@ tap_connect_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "%s", &intfc_name))
         ;
       else
-        return clib_error_return (0, "unknown input `%U'",
-                                  format_unformat_error, line_input);
+        {
+          error = clib_error_return (0, "unknown input `%U'",
+                                     format_unformat_error, line_input);
+          goto done;
+        }
     }
   
   if (intfc_name == 0)
-      return clib_error_return (0, "interface name must be specified");
+    {
+      error = clib_error_return (0, "interface name must be specified");
+      goto done;
+    }
 
   memset (ap, 0, sizeof (*ap));
 
@@ -1367,48 +1374,64 @@ tap_connect_command_fn (vlib_main_t * vm,
   switch (rv) 
     {
     case VNET_API_ERROR_SYSCALL_ERROR_1:
-      return clib_error_return (0, "Couldn't open /dev/net/tun");
+      error = clib_error_return (0, "Couldn't open /dev/net/tun");
+      goto done;
       
     case VNET_API_ERROR_SYSCALL_ERROR_2:
-      return clib_error_return (0, "Error setting flags on '%s'", intfc_name);
-  
+      error = clib_error_return (0, "Error setting flags on '%s'", intfc_name);
+      goto done;
+
     case VNET_API_ERROR_SYSCALL_ERROR_3:
-      return clib_error_return (0,  "Couldn't open provisioning socket");
+      error = clib_error_return (0,  "Couldn't open provisioning socket");
+      goto done;
 
     case VNET_API_ERROR_SYSCALL_ERROR_4:
-      return clib_error_return (0,  "Couldn't get if_index");
+      error = clib_error_return (0,  "Couldn't get if_index");
+      goto done;
     
     case VNET_API_ERROR_SYSCALL_ERROR_5:
-      return clib_error_return (0,  "Couldn't bind provisioning socket");
+      error = clib_error_return (0,  "Couldn't bind provisioning socket");
+      goto done;
 
     case VNET_API_ERROR_SYSCALL_ERROR_6:
-      return clib_error_return (0,  "Couldn't set device non-blocking flag");
+      error = clib_error_return (0,  "Couldn't set device non-blocking flag");
+      goto done;
 
     case VNET_API_ERROR_SYSCALL_ERROR_7:
-      return clib_error_return (0,  "Couldn't set device MTU");
+      error = clib_error_return (0,  "Couldn't set device MTU");
+      goto done;
 
     case VNET_API_ERROR_SYSCALL_ERROR_8:
-      return clib_error_return (0,  "Couldn't get interface flags");
+      error = clib_error_return (0,  "Couldn't get interface flags");
+      goto done;
 
     case VNET_API_ERROR_SYSCALL_ERROR_9:
-      return clib_error_return (0,  "Couldn't set intfc admin state up");
+      error = clib_error_return (0,  "Couldn't set intfc admin state up");
+      goto done;
 
     case VNET_API_ERROR_SYSCALL_ERROR_10:
-      return clib_error_return (0,  "Couldn't set intfc address/mask");
+      error = clib_error_return (0,  "Couldn't set intfc address/mask");
+      goto done;
 
     case VNET_API_ERROR_INVALID_REGISTRATION:
-      return clib_error_return (0,  "Invalid registration");
+      error = clib_error_return (0,  "Invalid registration");
+      goto done;
 
     case 0:
       break;
 
     default:
-      return clib_error_return (0,  "Unknown error: %d", rv);
+      error = clib_error_return (0,  "Unknown error: %d", rv);
+      goto done;
     }
 
   vlib_cli_output(vm, "%U\n", format_vnet_sw_if_index_name, 
                   vnet_get_main(), sw_if_index);
-  return 0;
+
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 VLIB_CLI_COMMAND (tap_connect_command, static) = {
diff --git a/src/vnet/vxlan-gpe/vxlan_gpe.c b/src/vnet/vxlan-gpe/vxlan_gpe.c
index b97510c4..2cba596f 100644
--- a/src/vnet/vxlan-gpe/vxlan_gpe.c
+++ b/src/vnet/vxlan-gpe/vxlan_gpe.c
@@ -454,6 +454,7 @@ vxlan_gpe_add_del_tunnel_command_fn (vlib_main_t * vm,
   u32 tmp;
   vnet_vxlan_gpe_add_del_tunnel_args_t _a, * a = &_a;
   u32 sw_if_index;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (! unformat_user (input, unformat_line_input, line_input))
@@ -494,7 +495,10 @@ vxlan_gpe_add_del_tunnel_command_fn (vlib_main_t * vm,
           encap_fib_index =  ip4_fib_index_from_table_id (tmp);
 
         if (encap_fib_index == ~0)
-          return clib_error_return (0, "nonexistent encap fib id %d", tmp);
+          {
+            error = clib_error_return (0, "nonexistent encap fib id %d", tmp);
+            goto done;
+          }
       }
     else if (unformat (line_input, "decap-vrf-id %d", &tmp))
       {
@@ -504,7 +508,10 @@ vxlan_gpe_add_del_tunnel_command_fn (vlib_main_t * vm,
           decap_fib_index = ip4_fib_index_from_table_id (tmp);
 
         if (decap_fib_index == ~0)
-          return clib_error_return (0, "nonexistent decap fib id %d", tmp);
+          {
+            error = clib_error_return (0, "nonexistent decap fib id %d", tmp);
+            goto done;
+          }
       }
     else if (unformat (line_input, "vni %d", &vni))
       vni_set = 1;
@@ -517,27 +524,43 @@ vxlan_gpe_add_del_tunnel_command_fn (vlib_main_t * vm,
     else if (unformat(line_input, "next-nsh"))
       protocol = VXLAN_GPE_PROTOCOL_NSH;
     else
-      return clib_error_return (0, "parse error: '%U'",
-                                format_unformat_error, line_input);
+      {
+        error = clib_error_return (0, "parse error: '%U'",
+                                   format_unformat_error, line_input);
+        goto done;
+      }
   }
 
-  unformat_free (line_input);
-
   if (local_set == 0)
-    return clib_error_return (0, "tunnel local address not specified");
+    {
+      error = clib_error_return (0, "tunnel local address not specified");
+      goto done;
+    }
 
   if (remote_set == 0)
-    return clib_error_return (0, "tunnel remote address not specified");
+    {
+      error = clib_error_return (0, "tunnel remote address not specified");
+      goto done;
+    }
 
   if (ipv4_set && ipv6_set)
-    return clib_error_return (0, "both IPv4 and IPv6 addresses specified");
+    {
+      error = clib_error_return (0, "both IPv4 and IPv6 addresses specified");
+      goto done;
+    }
 
   if ((ipv4_set && memcmp(&local.ip4, &remote.ip4, sizeof(local.ip4)) == 0) ||
       (ipv6_set && memcmp(&local.ip6, &remote.ip6, sizeof(local.ip6)) == 0))
-    return clib_error_return (0, "src and dst addresses are identical");
+    {
+      error = clib_error_return (0, "src and dst addresses are identical");
+      goto done;
+    }
 
   if (vni_set == 0)
-    return clib_error_return (0, "vni not specified");
+    {
+      error = clib_error_return (0, "vni not specified");
+      goto done;
+    }
 
   memset (a, 0, sizeof (*a));
 
@@ -558,20 +581,27 @@ vxlan_gpe_add_del_tunnel_command_fn (vlib_main_t * vm,
       vlib_cli_output(vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main(), sw_if_index);
       break;
     case VNET_API_ERROR_INVALID_DECAP_NEXT:
-      return clib_error_return (0, "invalid decap-next...");
+      error = clib_error_return (0, "invalid decap-next...");
+      goto done;
 
     case VNET_API_ERROR_TUNNEL_EXIST:
-      return clib_error_return (0, "tunnel already exists...");
+      error = clib_error_return (0, "tunnel already exists...");
+      goto done;
 
     case VNET_API_ERROR_NO_SUCH_ENTRY:
-      return clib_error_return (0, "tunnel does not exist...");
+      error = clib_error_return (0, "tunnel does not exist...");
+      goto done;
 
     default:
-      return clib_error_return
+      error = clib_error_return
         (0, "vnet_vxlan_gpe_add_del_tunnel returned %d", rv);
+      goto done;
     }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 VLIB_CLI_COMMAND (create_vxlan_gpe_tunnel_command, static) = {
diff --git a/src/vnet/vxlan/vxlan.c b/src/vnet/vxlan/vxlan.c
index 849fc25d..eedc16f8 100644
--- a/src/vnet/vxlan/vxlan.c
+++ b/src/vnet/vxlan/vxlan.c
@@ -657,6 +657,7 @@ vxlan_add_del_tunnel_command_fn (vlib_main_t * vm,
   int rv;
   vnet_vxlan_add_del_tunnel_args_t _a, * a = &_a;
   u32 tunnel_sw_if_index;
+  clib_error_t *error = NULL;
 
   /* Cant "universally zero init" (={0}) due to GCC bug 53119 */
   memset(&src, 0, sizeof src);
@@ -715,7 +716,10 @@ vxlan_add_del_tunnel_command_fn (vlib_main_t * vm,
       {
         encap_fib_index = fib_table_find (fib_ip_proto (ipv6_set), tmp);
         if (encap_fib_index == ~0)
-          return clib_error_return (0, "nonexistent encap-vrf-id %d", tmp);
+          {
+            error = clib_error_return (0, "nonexistent encap-vrf-id %d", tmp);
+            goto done;
+          }
       }
     else if (unformat (line_input, "decap-next %U", unformat_decap_next, 
                        &decap_next_index, ipv4_set))
@@ -723,41 +727,72 @@ vxlan_add_del_tunnel_command_fn (vlib_main_t * vm,
     else if (unformat (line_input, "vni %d", &vni))
       {
         if (vni >> 24)  
-          return clib_error_return (0, "vni %d out of range", vni);
+          {
+            error = clib_error_return (0, "vni %d out of range", vni);
+            goto done;
+          }
       }
     else 
-      return clib_error_return (0, "parse error: '%U'", 
-                                format_unformat_error, line_input);
+      {
+        error = clib_error_return (0, "parse error: '%U'",
+                                   format_unformat_error, line_input);
+        goto done;
+      }
   }
 
-  unformat_free (line_input);
-
   if (src_set == 0)
-    return clib_error_return (0, "tunnel src address not specified");
+    {
+      error = clib_error_return (0, "tunnel src address not specified");
+      goto done;
+    }
 
   if (dst_set == 0)
-    return clib_error_return (0, "tunnel dst address not specified");
+    {
+      error = clib_error_return (0, "tunnel dst address not specified");
+      goto done;
+    }
 
   if (grp_set && !ip46_address_is_multicast(&dst))
-    return clib_error_return (0, "tunnel group address not multicast");
+    {
+      error = clib_error_return (0, "tunnel group address not multicast");
+      goto done;
+    }
 
   if (grp_set == 0 && ip46_address_is_multicast(&dst))
-    return clib_error_return (0, "dst address must be unicast");
+    {
+      error = clib_error_return (0, "dst address must be unicast");
+      goto done;
+    }
 
   if (grp_set && mcast_sw_if_index == ~0)
-    return clib_error_return (0, "tunnel nonexistent multicast device");
+    {
+      error = clib_error_return (0, "tunnel nonexistent multicast device");
+      goto done;
+    }
 
   if (ipv4_set && ipv6_set)
-    return clib_error_return (0, "both IPv4 and IPv6 addresses specified");
+    {
+      error = clib_error_return (0, "both IPv4 and IPv6 addresses specified");
+      goto done;
+    }
 
   if (ip46_address_cmp(&src, &dst) == 0)
-    return clib_error_return (0, "src and dst addresses are identical");
+    {
+      error = clib_error_return (0, "src and dst addresses are identical");
+      goto done;
+    }
 
   if (decap_next_index == ~0)
-    return clib_error_return (0, "next node not found");
+    {
+      error = clib_error_return (0, "next node not found");
+      goto done;
+    }
 
   if (vni == 0)
-    return clib_error_return (0, "vni not specified");
+    {
+      error = clib_error_return (0, "vni not specified");
+      goto done;
+    }
 
   memset (a, 0, sizeof (*a));
 
@@ -779,17 +814,23 @@ vxlan_add_del_tunnel_command_fn (vlib_main_t * vm,
       break;
 
     case VNET_API_ERROR_TUNNEL_EXIST:
-      return clib_error_return (0, "tunnel already exists...");
+      error = clib_error_return (0, "tunnel already exists...");
+      goto done;
 
     case VNET_API_ERROR_NO_SUCH_ENTRY:
-      return clib_error_return (0, "tunnel does not exist...");
+      error = clib_error_return (0, "tunnel does not exist...");
+      goto done;
 
     default:
-      return clib_error_return 
+      error = clib_error_return
         (0, "vnet_vxlan_add_del_tunnel returned %d", rv);
+      goto done;
     }
 
-  return 0;
+done:
+  unformat_free (line_input);
+
+  return error;
 }
 
 /*?
@@ -912,6 +953,8 @@ set_ip_vxlan_bypass (u32 is_ip6,
   vnet_int_vxlan_bypass_mode (sw_if_index, is_ip6, is_enable);
 
  done:
+  unformat_free (line_input);
+
   return error;
 }
 
diff --git a/src/vpp/app/l2t.c b/src/vpp/app/l2t.c
index 45dd2807..e1eda155 100644
--- a/src/vpp/app/l2t.c
+++ b/src/vpp/app/l2t.c
@@ -254,6 +254,7 @@ l2tp_session_add_command_fn (vlib_main_t * vm,
   u32 local_session_id = 1, remote_session_id = 1;
   int our_address_set = 0, client_address_set = 0;
   int l2_sublayer_present = 0;
+  clib_error_t *error = NULL;
 
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -290,8 +291,12 @@ l2tp_session_add_command_fn (vlib_main_t * vm,
       else if (unformat (line_input, "l2-sublayer-present"))
 	l2_sublayer_present = 1;
       else
-	return clib_error_return (0, "parse error: '%U'",
-				  format_unformat_error, line_input);
+	{
+	  error = clib_error_return (0, "parse error: '%U'",
+				     format_unformat_error, line_input);
+	  unformat_free (line_input);
+	  return error;
+	}
     }
 
   unformat_free (line_input);
diff --git a/src/vpp/app/vpe_cli.c b/src/vpp/app/vpe_cli.c
index a26bf71f..94bdc84c 100644
--- a/src/vpp/app/vpe_cli.c
+++ b/src/vpp/app/vpe_cli.c
@@ -36,6 +36,7 @@ virtual_ip_cmd_fn_command_fn (vlib_main_t * vm,
   mac_addr_t *mac_addrs = 0;
   u32 sw_if_index;
   u32 i;
+  clib_error_t *error = NULL;
 
   next_hops = NULL;
   rpaths = NULL;
@@ -49,7 +50,11 @@ virtual_ip_cmd_fn_command_fn (vlib_main_t * vm,
   if (!unformat (line_input, "%U %U",
 		 unformat_ip4_address, &prefix.fp_addr.ip4,
 		 unformat_vnet_sw_interface, vnm, &sw_if_index))
-    goto barf;
+    {
+      error = clib_error_return (0, "unknown input `%U'",
+				 format_unformat_error, line_input);
+      goto done;
+    }
 
   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
     {
@@ -67,13 +72,18 @@ virtual_ip_cmd_fn_command_fn (vlib_main_t * vm,
 	}
       else
 	{
-	barf:
-	  return clib_error_return (0, "unknown input `%U'",
-				    format_unformat_error, input);
+	  error = clib_error_return (0, "unknown input `%U'",
+				     format_unformat_error, line_input);
+	  goto done;
 	}
     }
+
   if (vec_len (mac_addrs) == 0 || vec_len (mac_addrs) != vec_len (next_hops))
-    goto barf;
+    {
+      error = clib_error_return (0, "unknown input `%U'",
+				 format_unformat_error, line_input);
+      goto done;
+    }
 
   /* Create / delete special interface route /32's */
 
@@ -100,10 +110,12 @@ virtual_ip_cmd_fn_command_fn (vlib_main_t * vm,
 			     &prefix,
 			     FIB_SOURCE_CLI, FIB_ENTRY_FLAG_NONE, rpaths);
 
+done:
   vec_free (mac_addrs);
   vec_free (next_hops);
+  unformat_free (line_input);
 
-  return 0;
+  return error;
 }
 
 /* *INDENT-OFF* */
-- 
cgit 1.2.3-korg


From 3f844d0bc900e5db40ba74724e2b61e7943682d3 Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Sat, 18 Feb 2017 00:03:54 -0800
Subject: Proxy ND (RFC4389 - or a sub-set thereof). This allows the
 'emulation' of bridging. That is hosts in one sub-net reachable via
 differenet interfaces.

Introducate a new API command:
  ip6 nd proxy <host-address> <interface>

this indicates 2 things;
 1) that host <host-address> is reachable out of interface <interface>. VPP will thus install that route.
 2) NS requests sent to <host-address> will be responeded to (i.e. proxied).

Change-Id: I863f967fdb5097ab3b574769c70afdbfc8d5478a
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/vat/api_format.c             | 109 +++++++++++++++
 src/vnet/fib/fib_entry.h         |   5 +
 src/vnet/fib/fib_entry_src_api.c |   1 +
 src/vnet/ip/ip.api               |  50 +++++++
 src/vnet/ip/ip6.h                |  55 --------
 src/vnet/ip/ip6_neighbor.c       | 136 +++++++++++++++++-
 src/vnet/ip/ip6_neighbor.h       |  42 +++++-
 src/vnet/ip/ip_api.c             | 102 ++++++++++++++
 src/vnet/ip/lookup.c             |   1 +
 test/test_ip6.py                 | 293 ++++++++++++++++++++++++++++++++++++---
 test/vpp_papi_provider.py        |   6 +
 11 files changed, 721 insertions(+), 79 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vat/api_format.c b/src/vat/api_format.c
index fb596fe6..08a2a774 100644
--- a/src/vat/api_format.c
+++ b/src/vat/api_format.c
@@ -3913,6 +3913,7 @@ _(dhcp_client_config_reply)                             \
 _(set_ip_flow_hash_reply)                               \
 _(sw_interface_ip6_enable_disable_reply)                \
 _(sw_interface_ip6_set_link_local_address_reply)        \
+_(ip6nd_proxy_add_del_reply)                            \
 _(sw_interface_ip6nd_ra_prefix_reply)                   \
 _(sw_interface_ip6nd_ra_config_reply)                   \
 _(set_arp_neighbor_limit_reply)                         \
@@ -4092,6 +4093,8 @@ _(SW_INTERFACE_IP6_ENABLE_DISABLE_REPLY,                                \
   sw_interface_ip6_enable_disable_reply)                                \
 _(SW_INTERFACE_IP6_SET_LINK_LOCAL_ADDRESS_REPLY,                        \
   sw_interface_ip6_set_link_local_address_reply)                        \
+_(IP6ND_PROXY_ADD_DEL_REPLY, ip6nd_proxy_add_del_reply)                 \
+_(IP6ND_PROXY_DETAILS, ip6nd_proxy_details)                             \
 _(SW_INTERFACE_IP6ND_RA_PREFIX_REPLY,                                   \
   sw_interface_ip6nd_ra_prefix_reply)                                   \
 _(SW_INTERFACE_IP6ND_RA_CONFIG_REPLY,                                   \
@@ -7968,6 +7971,109 @@ api_sw_interface_ip6_set_link_local_address (vat_main_t * vam)
   return ret;
 }
 
+static int
+api_ip6nd_proxy_add_del (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  vl_api_ip6nd_proxy_add_del_t *mp;
+  u32 sw_if_index = ~0;
+  u8 v6_address_set = 0;
+  ip6_address_t v6address;
+  u8 is_del = 0;
+  int ret;
+
+  /* Parse args required to build the message */
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "%U", api_unformat_sw_if_index, vam, &sw_if_index))
+	;
+      else if (unformat (i, "sw_if_index %d", &sw_if_index))
+	;
+      else if (unformat (i, "%U", unformat_ip6_address, &v6address))
+	v6_address_set = 1;
+      if (unformat (i, "del"))
+	is_del = 1;
+      else
+	{
+	  clib_warning ("parse error '%U'", format_unformat_error, i);
+	  return -99;
+	}
+    }
+
+  if (sw_if_index == ~0)
+    {
+      errmsg ("missing interface name or sw_if_index");
+      return -99;
+    }
+  if (!v6_address_set)
+    {
+      errmsg ("no address set");
+      return -99;
+    }
+
+  /* Construct the API message */
+  M (IP6ND_PROXY_ADD_DEL, mp);
+
+  mp->is_del = is_del;
+  mp->sw_if_index = ntohl (sw_if_index);
+  clib_memcpy (mp->address, &v6address, sizeof (v6address));
+
+  /* send it... */
+  S (mp);
+
+  /* Wait for a reply, return good/bad news  */
+  W (ret);
+  return ret;
+}
+
+static int
+api_ip6nd_proxy_dump (vat_main_t * vam)
+{
+  vl_api_ip6nd_proxy_dump_t *mp;
+  vl_api_control_ping_t *mp_ping;
+  int ret;
+
+  M (IP6ND_PROXY_DUMP, mp);
+
+  S (mp);
+
+  /* Use a control ping for synchronization */
+  M (CONTROL_PING, mp_ping);
+  S (mp_ping);
+
+  W (ret);
+  return ret;
+}
+
+static void vl_api_ip6nd_proxy_details_t_handler
+  (vl_api_ip6nd_proxy_details_t * mp)
+{
+  vat_main_t *vam = &vat_main;
+
+  print (vam->ofp, "host %U sw_if_index %d",
+	 format_ip6_address, mp->address, ntohl (mp->sw_if_index));
+}
+
+static void vl_api_ip6nd_proxy_details_t_handler_json
+  (vl_api_ip6nd_proxy_details_t * mp)
+{
+  vat_main_t *vam = &vat_main;
+  struct in6_addr ip6;
+  vat_json_node_t *node = NULL;
+
+  if (VAT_JSON_ARRAY != vam->json_tree.type)
+    {
+      ASSERT (VAT_JSON_NONE == vam->json_tree.type);
+      vat_json_init_array (&vam->json_tree);
+    }
+  node = vat_json_array_add (&vam->json_tree);
+
+  vat_json_init_object (node);
+  vat_json_object_add_uint (node, "sw_if_index", ntohl (mp->sw_if_index));
+
+  clib_memcpy (&ip6, mp->address, sizeof (ip6));
+  vat_json_object_add_ip6 (node, "host", ip6);
+}
 
 static int
 api_sw_interface_ip6nd_ra_prefix (vat_main_t * vam)
@@ -17974,6 +18080,9 @@ _(sw_interface_ip6_enable_disable,                                      \
   "<intfc> | sw_if_index <id> enable | disable")                        \
 _(sw_interface_ip6_set_link_local_address,                              \
   "<intfc> | sw_if_index <id> <ip6-address>/<mask-width>")              \
+_(ip6nd_proxy_add_del,                                                  \
+  "<intfc> | sw_if_index <id> <ip6-address>")                           \
+_(ip6nd_proxy_dump, "")                                                 \
 _(sw_interface_ip6nd_ra_prefix,                                         \
   "<intfc> | sw_if_index <id> <ip6-address>/<mask-width>\n"             \
   "val_life <n> pref_life <n> [def] [noadv] [offl] [noauto]\n"          \
diff --git a/src/vnet/fib/fib_entry.h b/src/vnet/fib/fib_entry.h
index 9dfb8127..12fa9eb4 100644
--- a/src/vnet/fib/fib_entry.h
+++ b/src/vnet/fib/fib_entry.h
@@ -80,6 +80,10 @@ typedef enum fib_source_t_ {
      * DHCP
      */
     FIB_SOURCE_DHCP,
+    /**
+     * IPv6 Proxy ND
+     */
+    FIB_SOURCE_IP6_ND_PROXY,
     /**
      * Adjacency source.
      * routes created as a result of ARP/ND entries. This is lower priority
@@ -141,6 +145,7 @@ STATIC_ASSERT (sizeof(fib_source_t) == 1,
     [FIB_SOURCE_LISP] = "LISP", 			\
     [FIB_SOURCE_CLASSIFY] = "classify",			\
     [FIB_SOURCE_DHCP] = "DHCP",   			\
+    [FIB_SOURCE_IP6_ND_PROXY] = "IPv6-proxy-nd",        \
     [FIB_SOURCE_RR] = "recursive-resolution",	        \
     [FIB_SOURCE_AE] = "attached_export",	        \
     [FIB_SOURCE_MPLS] = "mpls",           	        \
diff --git a/src/vnet/fib/fib_entry_src_api.c b/src/vnet/fib/fib_entry_src_api.c
index edc8a47b..1277bd65 100644
--- a/src/vnet/fib/fib_entry_src_api.c
+++ b/src/vnet/fib/fib_entry_src_api.c
@@ -116,4 +116,5 @@ fib_entry_src_api_register (void)
     fib_entry_src_register(FIB_SOURCE_API, &api_src_vft);
     fib_entry_src_register(FIB_SOURCE_CLI, &api_src_vft);
     fib_entry_src_register(FIB_SOURCE_DHCP, &api_src_vft);
+    fib_entry_src_register(FIB_SOURCE_IP6_ND_PROXY, &api_src_vft);
 }
diff --git a/src/vnet/ip/ip.api b/src/vnet/ip/ip.api
index ff16e781..b5e730fc 100644
--- a/src/vnet/ip/ip.api
+++ b/src/vnet/ip/ip.api
@@ -294,6 +294,56 @@ define sw_interface_ip6nd_ra_prefix_reply
   i32 retval;
 };
 
+/** \brief IPv6 ND proxy config
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param sw_if_index - The interface the host is on
+    @param address - The address of the host for which to proxy for
+    @param is_add - Adding or deleting
+*/
+define ip6nd_proxy_add_del
+{
+  u32 client_index;
+  u32 context;
+  u32 sw_if_index;
+  u8 is_del;
+  u8 address[16];
+};
+
+/** \brief IPv6 ND proxy response
+    @param context - sender context, to match reply w/ request
+    @param retval - return code for the request
+*/
+define ip6nd_proxy_add_del_reply
+{
+  u32 context;
+  i32 retval;
+};
+
+/** \brief IPv6 ND proxy details returned after request
+    @param context - sender context, to match reply w/ request
+    @param retval - return code for the request
+*/
+define ip6nd_proxy_details
+{
+  u32 client_index;
+  u32 context;
+  u32 sw_if_index;
+  u8 address[16];
+};
+
+/** \brief IPv6 ND proxy dump request
+    @param context - sender context, to match reply w/ request
+    @param retval - return code for the request
+    @param sw_if_index - The interface the host is on
+    @param address - The address of the host for which to proxy for
+*/
+define ip6nd_proxy_dump
+{
+  u32 client_index;
+  u32 context;
+};
+
 /** \brief IPv6 interface enable / disable request
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
diff --git a/src/vnet/ip/ip6.h b/src/vnet/ip/ip6.h
index 535f24c0..f402b011 100644
--- a/src/vnet/ip/ip6.h
+++ b/src/vnet/ip/ip6.h
@@ -217,24 +217,6 @@ extern vlib_node_registration_t ip6_discover_neighbor_node;
 extern vlib_node_registration_t ip6_glean_node;
 extern vlib_node_registration_t ip6_midchain_node;
 
-extern vlib_node_registration_t ip6_icmp_neighbor_discovery_event_node;
-
-/* ipv6 neighbor discovery - timer/event types */
-typedef enum
-{
-  ICMP6_ND_EVENT_INIT,
-} ip6_icmp_neighbor_discovery_event_type_t;
-
-typedef union
-{
-  u32 add_del_swindex;
-  struct
-  {
-    u32 up_down_swindex;
-    u32 fib_index;
-  } up_down_event;
-} ip6_icmp_neighbor_discovery_event_data_t;
-
 always_inline uword
 ip6_destination_matches_route (const ip6_main_t * im,
 			       const ip6_address_t * key,
@@ -342,8 +324,6 @@ int ip6_address_compare (ip6_address_t * a1, ip6_address_t * a2);
 clib_error_t *ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst,
 				  u32 sw_if_index);
 
-clib_error_t *ip6_set_neighbor_limit (u32 neighbor_limit);
-
 uword
 ip6_udp_register_listener (vlib_main_t * vm,
 			   u16 dst_port, u32 next_node_index);
@@ -359,19 +339,6 @@ serialize_function_t serialize_vnet_ip6_main, unserialize_vnet_ip6_main;
 void ip6_ethernet_update_adjacency (vnet_main_t * vnm,
 				    u32 sw_if_index, u32 ai);
 
-int
-vnet_set_ip6_ethernet_neighbor (vlib_main_t * vm,
-				u32 sw_if_index,
-				ip6_address_t * a,
-				u8 * link_layer_address,
-				uword n_bytes_link_layer_address,
-				int is_static);
-int
-vnet_unset_ip6_ethernet_neighbor (vlib_main_t * vm,
-				  u32 sw_if_index,
-				  ip6_address_t * a,
-				  u8 * link_layer_address,
-				  uword n_bytes_link_layer_address);
 
 void
 ip6_link_local_address_from_ethernet_mac_address (ip6_address_t * ip,
@@ -384,22 +351,6 @@ ip6_ethernet_mac_address_from_link_local_address (u8 * mac,
 int vnet_set_ip6_flow_hash (u32 table_id,
 			    flow_hash_config_t flow_hash_config);
 
-int
-ip6_neighbor_ra_config (vlib_main_t * vm, u32 sw_if_index,
-			u8 suppress, u8 managed, u8 other,
-			u8 ll_option, u8 send_unicast, u8 cease,
-			u8 use_lifetime, u32 lifetime,
-			u32 initial_count, u32 initial_interval,
-			u32 max_interval, u32 min_interval, u8 is_no);
-
-int
-ip6_neighbor_ra_prefix (vlib_main_t * vm, u32 sw_if_index,
-			ip6_address_t * prefix_addr, u8 prefix_len,
-			u8 use_default, u32 val_lifetime, u32 pref_lifetime,
-			u8 no_advertise, u8 off_link, u8 no_autoconfig,
-			u8 no_onlink, u8 is_no);
-
-
 clib_error_t *enable_ip6_interface (vlib_main_t * vm, u32 sw_if_index);
 
 clib_error_t *disable_ip6_interface (vlib_main_t * vm, u32 sw_if_index);
@@ -410,12 +361,6 @@ clib_error_t *set_ip6_link_local_address (vlib_main_t * vm,
 					  u32 sw_if_index,
 					  ip6_address_t * address);
 
-void vnet_register_ip6_neighbor_resolution_event (vnet_main_t * vnm,
-						  void *address_arg,
-						  uword node_index,
-						  uword type_opaque,
-						  uword data);
-
 int vnet_add_del_ip6_nd_change_event (vnet_main_t * vnm,
 				      void *data_callback,
 				      u32 pid,
diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c
index 43d68cd1..715891b8 100644
--- a/src/vnet/ip/ip6_neighbor.c
+++ b/src/vnet/ip/ip6_neighbor.c
@@ -197,6 +197,22 @@ typedef struct
 
 } ip6_neighbor_main_t;
 
+/* ipv6 neighbor discovery - timer/event types */
+typedef enum
+{
+  ICMP6_ND_EVENT_INIT,
+} ip6_icmp_neighbor_discovery_event_type_t;
+
+typedef union
+{
+  u32 add_del_swindex;
+  struct
+  {
+    u32 up_down_swindex;
+    u32 fib_index;
+  } up_down_event;
+} ip6_icmp_neighbor_discovery_event_data_t;
+
 static ip6_neighbor_main_t ip6_neighbor_main;
 static ip6_address_t ip6a_zero;	/* ip6 address 0 */
 
@@ -312,7 +328,7 @@ static void ip6_neighbor_set_unset_rpc_callback
 static void set_unset_ip6_neighbor_rpc
   (vlib_main_t * vm,
    u32 sw_if_index,
-   ip6_address_t * a, u8 * link_layer_addreess, int is_add, int is_static)
+   ip6_address_t * a, u8 * link_layer_address, int is_add, int is_static)
 {
   ip6_neighbor_set_unset_rpc_args_t args;
   void vl_api_rpc_call_main_thread (void *fp, u8 * data, u32 data_length);
@@ -321,7 +337,8 @@ static void set_unset_ip6_neighbor_rpc
   args.is_add = is_add;
   args.is_static = is_static;
   clib_memcpy (&args.addr, a, sizeof (*a));
-  clib_memcpy (args.link_layer_address, link_layer_addreess, 6);
+  if (NULL != link_layer_address)
+    clib_memcpy (args.link_layer_address, link_layer_address, 6);
 
   vl_api_rpc_call_main_thread (ip6_neighbor_set_unset_rpc_callback,
 			       (u8 *) & args, sizeof (args));
@@ -1028,14 +1045,35 @@ icmp6_neighbor_solicitation_or_advertisement (vlib_main_t * vm,
 							  &h0->target_address,
 							  128);
 
-		  if (FIB_NODE_INDEX_INVALID == fei ||
-		      !(FIB_ENTRY_FLAG_LOCAL &
-			fib_entry_get_flags_for_source (fei,
-							FIB_SOURCE_INTERFACE)))
+		  if (FIB_NODE_INDEX_INVALID == fei)
 		    {
+		      /* The target address is not in the FIB */
 		      error0 =
 			ICMP6_ERROR_NEIGHBOR_SOLICITATION_SOURCE_UNKNOWN;
 		    }
+		  else
+		    {
+		      if (FIB_ENTRY_FLAG_LOCAL &
+			  fib_entry_get_flags_for_source (fei,
+							  FIB_SOURCE_INTERFACE))
+			{
+			  /* It's an address that belongs to one of our interfaces
+			   * that's good. */
+			}
+		      else
+			if (fib_entry_is_sourced
+			    (fei, FIB_SOURCE_IP6_ND_PROXY))
+			{
+			  /* The address was added by IPv6 Proxy ND config.
+			   * We should only respond to these if the NS arrived on
+			   * the link that has a matching covering prefix */
+			}
+		      else
+			{
+			  error0 =
+			    ICMP6_ERROR_NEIGHBOR_SOLICITATION_SOURCE_UNKNOWN;
+			}
+		    }
 		}
 	    }
 
@@ -4021,6 +4059,92 @@ vnet_ip6_nd_term (vlib_main_t * vm,
 
 }
 
+int
+ip6_neighbor_proxy_add_del (u32 sw_if_index, ip6_address_t * addr, u8 is_del)
+{
+  u32 fib_index;
+
+  fib_prefix_t pfx = {
+    .fp_len = 128,
+    .fp_proto = FIB_PROTOCOL_IP6,
+    .fp_addr = {
+		.ip6 = *addr,
+		},
+  };
+  ip46_address_t nh = {
+    .ip6 = *addr,
+  };
+
+  fib_index = ip6_fib_table_get_index_for_sw_if_index (sw_if_index);
+
+  if (~0 == fib_index)
+    return VNET_API_ERROR_NO_SUCH_FIB;
+
+  if (is_del)
+    {
+      fib_table_entry_path_remove (fib_index,
+				   &pfx,
+				   FIB_SOURCE_IP6_ND_PROXY,
+				   FIB_PROTOCOL_IP6,
+				   &nh,
+				   sw_if_index,
+				   ~0, 1, FIB_ROUTE_PATH_FLAG_NONE);
+      /* flush the ND cache of this address if it's there */
+      vnet_unset_ip6_ethernet_neighbor (vlib_get_main (),
+					sw_if_index, addr, NULL, 0);
+    }
+  else
+    {
+      fib_table_entry_path_add (fib_index,
+				&pfx,
+				FIB_SOURCE_IP6_ND_PROXY,
+				FIB_ENTRY_FLAG_NONE,
+				FIB_PROTOCOL_IP6,
+				&nh,
+				sw_if_index,
+				~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
+    }
+  return (0);
+}
+
+static clib_error_t *
+set_ip6_nd_proxy_cmd (vlib_main_t * vm,
+		      unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  clib_error_t *error = 0;
+  ip6_address_t addr;
+  u32 sw_if_index;
+  u8 is_del = 0;
+
+  if (unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
+    {
+      /* get the rest of the command */
+      while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+	{
+	  if (unformat (input, "%U", unformat_ip6_address, &addr))
+	    break;
+	  else if (unformat (input, "delete") || unformat (input, "del"))
+	    is_del = 1;
+	  else
+	    return (unformat_parse_error (input));
+	}
+    }
+
+  ip6_neighbor_proxy_add_del (sw_if_index, &addr, is_del);
+
+  return error;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (set_ip6_nd_proxy_command, static) =
+{
+  .path = "set ip6 nd proxy",
+  .short_help = "set ip6 nd proxy <HOST> <INTERFACE>",
+  .function = set_ip6_nd_proxy_cmd,
+};
+/* *INDENT-ON* */
+
 void
 ethernet_ndp_change_mac (u32 sw_if_index)
 {
diff --git a/src/vnet/ip/ip6_neighbor.h b/src/vnet/ip/ip6_neighbor.h
index b2c9f48a..3d256316 100644
--- a/src/vnet/ip/ip6_neighbor.h
+++ b/src/vnet/ip/ip6_neighbor.h
@@ -39,7 +39,47 @@ typedef struct
   fib_node_index_t fib_entry_index;
 } ip6_neighbor_t;
 
-ip6_neighbor_t *ip6_neighbors_entries (u32 sw_if_index);
+extern ip6_neighbor_t *ip6_neighbors_entries (u32 sw_if_index);
+
+extern int ip6_neighbor_ra_config (vlib_main_t * vm, u32 sw_if_index,
+				   u8 suppress, u8 managed, u8 other,
+				   u8 ll_option, u8 send_unicast, u8 cease,
+				   u8 use_lifetime, u32 lifetime,
+				   u32 initial_count, u32 initial_interval,
+				   u32 max_interval, u32 min_interval,
+				   u8 is_no);
+
+extern int ip6_neighbor_ra_prefix (vlib_main_t * vm, u32 sw_if_index,
+				   ip6_address_t * prefix_addr, u8 prefix_len,
+				   u8 use_default, u32 val_lifetime,
+				   u32 pref_lifetime, u8 no_advertise,
+				   u8 off_link, u8 no_autoconfig,
+				   u8 no_onlink, u8 is_no);
+
+extern clib_error_t *ip6_set_neighbor_limit (u32 neighbor_limit);
+
+extern void vnet_register_ip6_neighbor_resolution_event (vnet_main_t * vnm,
+							 void *address_arg,
+							 uword node_index,
+							 uword type_opaque,
+							 uword data);
+
+extern int vnet_set_ip6_ethernet_neighbor (vlib_main_t * vm,
+					   u32 sw_if_index,
+					   ip6_address_t * a,
+					   u8 * link_layer_address,
+					   uword n_bytes_link_layer_address,
+					   int is_static);
+
+extern int vnet_unset_ip6_ethernet_neighbor (vlib_main_t * vm,
+					     u32 sw_if_index,
+					     ip6_address_t * a,
+					     u8 * link_layer_address,
+					     uword
+					     n_bytes_link_layer_address);
+
+extern int ip6_neighbor_proxy_add_del (u32 sw_if_index,
+				       ip6_address_t * addr, u8 is_add);
 
 #endif /* included_ip6_neighbor_h */
 
diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c
index ab164a5f..df9ca939 100644
--- a/src/vnet/ip/ip_api.c
+++ b/src/vnet/ip/ip_api.c
@@ -72,6 +72,9 @@ _(IP_ADD_DEL_ROUTE, ip_add_del_route)                                   \
 _(SET_IP_FLOW_HASH,set_ip_flow_hash)                                    \
 _(SW_INTERFACE_IP6ND_RA_CONFIG, sw_interface_ip6nd_ra_config)           \
 _(SW_INTERFACE_IP6ND_RA_PREFIX, sw_interface_ip6nd_ra_prefix)           \
+_(IP6ND_PROXY_ADD_DEL, ip6nd_proxy_add_del)                             \
+_(IP6ND_PROXY_DUMP, ip6nd_proxy_dump)                                   \
+_(IP6ND_PROXY_DETAILS, ip6nd_proxy_details)                             \
 _(SW_INTERFACE_IP6_ENABLE_DISABLE, sw_interface_ip6_enable_disable )    \
 _(SW_INTERFACE_IP6_SET_LINK_LOCAL_ADDRESS, 				\
   sw_interface_ip6_set_link_local_address)
@@ -1399,6 +1402,105 @@ static void
   REPLY_MACRO (VL_API_SW_INTERFACE_IP6ND_RA_PREFIX_REPLY);
 }
 
+static void
+send_ip6nd_proxy_details (unix_shared_memory_queue_t * q,
+			  u32 context,
+			  const ip46_address_t * addr, u32 sw_if_index)
+{
+  vl_api_ip6nd_proxy_details_t *mp;
+
+  mp = vl_msg_api_alloc (sizeof (*mp));
+  memset (mp, 0, sizeof (*mp));
+  mp->_vl_msg_id = ntohs (VL_API_IP6ND_PROXY_DETAILS);
+  mp->context = context;
+  mp->sw_if_index = htonl (sw_if_index);
+  memcpy (mp->address, addr, 16);
+
+  vl_msg_api_send_shmem (q, (u8 *) & mp);
+}
+
+static void
+vl_api_ip6nd_proxy_details_t_handler (vl_api_ip_neighbor_details_t * mp)
+{
+  clib_warning ("BUG");
+}
+
+typedef struct api_ip6nd_proxy_fib_table_walk_ctx_t_
+{
+  u32 *indices;
+} api_ip6nd_proxy_fib_table_walk_ctx_t;
+
+static int
+api_ip6nd_proxy_fib_table_walk (fib_node_index_t fei, void *arg)
+{
+  api_ip6nd_proxy_fib_table_walk_ctx_t *ctx = arg;
+
+  if (fib_entry_is_sourced (fei, FIB_SOURCE_IP6_ND_PROXY))
+    {
+      vec_add1 (ctx->indices, fei);
+    }
+
+  return (1);
+}
+
+static void
+vl_api_ip6nd_proxy_dump_t_handler (vl_api_ip6nd_proxy_dump_t * mp)
+{
+  ip6_main_t *im6 = &ip6_main;
+  fib_table_t *fib_table;
+  api_ip6nd_proxy_fib_table_walk_ctx_t ctx = {
+    .indices = NULL,
+  };
+  fib_node_index_t *feip;
+  fib_prefix_t pfx;
+  unix_shared_memory_queue_t *q;
+
+  q = vl_api_client_index_to_input_queue (mp->client_index);
+  if (q == 0)
+    {
+      return;
+    }
+
+  /* *INDENT-OFF* */
+  pool_foreach (fib_table, im6->fibs,
+  ({
+    fib_table_walk(fib_table->ft_index,
+                   FIB_PROTOCOL_IP6,
+                   api_ip6nd_proxy_fib_table_walk,
+                   &ctx);
+  }));
+  /* *INDENT-ON* */
+
+  vec_sort_with_function (ctx.indices, fib_entry_cmp_for_sort);
+
+  vec_foreach (feip, ctx.indices)
+  {
+    fib_entry_get_prefix (*feip, &pfx);
+
+    send_ip6nd_proxy_details (q,
+			      mp->context,
+			      &pfx.fp_addr,
+			      fib_entry_get_resolving_interface (*feip));
+  }
+
+  vec_free (ctx.indices);
+}
+
+static void
+vl_api_ip6nd_proxy_add_del_t_handler (vl_api_ip6nd_proxy_add_del_t * mp)
+{
+  vl_api_ip6nd_proxy_add_del_reply_t *rmp;
+  int rv = 0;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  rv = ip6_neighbor_proxy_add_del (ntohl (mp->sw_if_index),
+				   (ip6_address_t *) mp->address, mp->is_del);
+
+  BAD_SW_IF_INDEX_LABEL;
+  REPLY_MACRO (VL_API_IP6ND_PROXY_ADD_DEL_REPLY);
+}
+
 static void
   vl_api_sw_interface_ip6_enable_disable_t_handler
   (vl_api_sw_interface_ip6_enable_disable_t * mp)
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 807b87b6..8607fa5f 100644
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -49,6 +49,7 @@
 #include <vnet/dpo/punt_dpo.h>
 #include <vnet/dpo/receive_dpo.h>
 #include <vnet/dpo/ip_null_dpo.h>
+#include <vnet/ip/ip6_neighbor.h>
 
 /**
  * @file
diff --git a/test/test_ip6.py b/test/test_ip6.py
index b8278329..070e2d7d 100644
--- a/test/test_ip6.py
+++ b/test/test_ip6.py
@@ -1,17 +1,20 @@
 #!/usr/bin/env python
 
 import unittest
-import socket
+from socket import AF_INET6
 
 from framework import VppTestCase, VppTestRunner
 from vpp_sub_interface import VppSubInterface, VppDot1QSubint
 from vpp_pg_interface import is_ipv6_misc
+from vpp_neighbor import find_nbr
 
 from scapy.packet import Raw
 from scapy.layers.l2 import Ether, Dot1Q
 from scapy.layers.inet6 import IPv6, UDP, ICMPv6ND_NS, ICMPv6ND_RS, \
     ICMPv6ND_RA, ICMPv6NDOptSrcLLAddr, getmacbyip6, ICMPv6MRD_Solicitation, \
-    ICMPv6NDOptMTU, ICMPv6NDOptSrcLLAddr, ICMPv6NDOptPrefixInfo
+    ICMPv6NDOptMTU, ICMPv6NDOptSrcLLAddr, ICMPv6NDOptPrefixInfo, \
+    ICMPv6ND_NA, ICMPv6NDOptDstLLAddr
+
 from util import ppp
 from scapy.utils6 import in6_getnsma, in6_getnsmac, in6_ptop, in6_islladdr, \
     in6_mactoifaceid, in6_ismaddr
@@ -24,7 +27,71 @@ def mk_ll_addr(mac):
     return addr
 
 
-class TestIPv6(VppTestCase):
+class TestIPv6ND(VppTestCase):
+    def validate_ra(self, intf, rx, dst_ip=None):
+        if not dst_ip:
+            dst_ip = intf.remote_ip6
+
+        # unicasted packets must come to the unicast mac
+        self.assertEqual(rx[Ether].dst, intf.remote_mac)
+
+        # and from the router's MAC
+        self.assertEqual(rx[Ether].src, intf.local_mac)
+
+        # the rx'd RA should be addressed to the sender's source
+        self.assertTrue(rx.haslayer(ICMPv6ND_RA))
+        self.assertEqual(in6_ptop(rx[IPv6].dst),
+                         in6_ptop(dst_ip))
+
+        # and come from the router's link local
+        self.assertTrue(in6_islladdr(rx[IPv6].src))
+        self.assertEqual(in6_ptop(rx[IPv6].src),
+                         in6_ptop(mk_ll_addr(intf.local_mac)))
+
+    def validate_na(self, intf, rx, dst_ip=None, tgt_ip=None):
+        if not dst_ip:
+            dst_ip = intf.remote_ip6
+        if not tgt_ip:
+            dst_ip = intf.local_ip6
+
+        # unicasted packets must come to the unicast mac
+        self.assertEqual(rx[Ether].dst, intf.remote_mac)
+
+        # and from the router's MAC
+        self.assertEqual(rx[Ether].src, intf.local_mac)
+
+        # the rx'd NA should be addressed to the sender's source
+        self.assertTrue(rx.haslayer(ICMPv6ND_NA))
+        self.assertEqual(in6_ptop(rx[IPv6].dst),
+                         in6_ptop(dst_ip))
+
+        # and come from the target address
+        self.assertEqual(in6_ptop(rx[IPv6].src), in6_ptop(tgt_ip))
+
+        # Dest link-layer options should have the router's MAC
+        dll = rx[ICMPv6NDOptDstLLAddr]
+        self.assertEqual(dll.lladdr, intf.local_mac)
+
+    def send_and_expect_ra(self, intf, pkts, remark, dst_ip=None,
+                           filter_out_fn=is_ipv6_misc):
+        intf.add_stream(pkts)
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        rx = intf.get_capture(1, filter_out_fn=filter_out_fn)
+
+        self.assertEqual(len(rx), 1)
+        rx = rx[0]
+        self.validate_ra(intf, rx, dst_ip)
+
+    def send_and_assert_no_replies(self, intf, pkts, remark):
+        intf.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        intf.assert_nothing_captured(remark=remark)
+
+
+class TestIPv6(TestIPv6ND):
     """ IPv6 Test Case """
 
     @classmethod
@@ -112,7 +179,7 @@ class TestIPv6(VppTestCase):
         n_int = len(self.interfaces)
         percent = 0
         counter = 0.0
-        dest_addr = socket.inet_pton(socket.AF_INET6, "fd02::1")
+        dest_addr = inet_pton(AF_INET6, "fd02::1")
         dest_addr_len = 128
         for i in self.interfaces:
             next_hop_address = i.local_ip6n
@@ -225,12 +292,6 @@ class TestIPv6(VppTestCase):
             pkts = i.parent.get_capture()
             self.verify_capture(i, pkts)
 
-    def send_and_assert_no_replies(self, intf, pkts, remark):
-        intf.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        intf.assert_nothing_captured(remark=remark)
-
     def test_ns(self):
         """ IPv6 Neighbour Solicitation Exceptions
 
@@ -243,8 +304,8 @@ class TestIPv6(VppTestCase):
         #
         # An NS from a non link source address
         #
-        nsma = in6_getnsma(inet_pton(socket.AF_INET6, self.pg0.local_ip6))
-        d = inet_ntop(socket.AF_INET6, nsma)
+        nsma = in6_getnsma(inet_pton(AF_INET6, self.pg0.local_ip6))
+        d = inet_ntop(AF_INET6, nsma)
 
         p = (Ether(dst=in6_getnsmac(nsma)) /
              IPv6(dst=d, src="2002::2") /
@@ -261,8 +322,8 @@ class TestIPv6(VppTestCase):
         # not a member of FAILS
         #
         if 0:
-            nsma = in6_getnsma(inet_pton(socket.AF_INET6, "fd::ffff"))
-            d = inet_ntop(socket.AF_INET6, nsma)
+            nsma = in6_getnsma(inet_pton(AF_INET6, "fd::ffff"))
+            d = inet_ntop(AF_INET6, nsma)
 
             p = (Ether(dst=in6_getnsmac(nsma)) /
                  IPv6(dst=d, src=self.pg0.remote_ip6) /
@@ -277,8 +338,8 @@ class TestIPv6(VppTestCase):
         #
         # An NS whose target address is one the router does not own
         #
-        nsma = in6_getnsma(inet_pton(socket.AF_INET6, self.pg0.local_ip6))
-        d = inet_ntop(socket.AF_INET6, nsma)
+        nsma = in6_getnsma(inet_pton(AF_INET6, self.pg0.local_ip6))
+        d = inet_ntop(AF_INET6, nsma)
 
         p = (Ether(dst=in6_getnsmac(nsma)) /
              IPv6(dst=d, src=self.pg0.remote_ip6) /
@@ -418,7 +479,7 @@ class TestIPv6(VppTestCase):
         # Send the RS multicast
         #
         self.pg0.ip6_ra_config(send_unicast=1)
-        dmac = in6_getnsmac(inet_pton(socket.AF_INET6, "ff02::2"))
+        dmac = in6_getnsmac(inet_pton(AF_INET6, "ff02::2"))
         ll = mk_ll_addr(self.pg0.remote_mac)
         p = (Ether(dst=dmac, src=self.pg0.remote_mac) /
              IPv6(dst="ff02::2", src=ll) /
@@ -629,5 +690,203 @@ class TestIPv6(VppTestCase):
         #
         self.pg0.ip6_ra_config(no=1, suppress=1, send_unicast=0)
 
+
+class IPv6NDProxyTest(TestIPv6ND):
+    """ IPv6 ND ProxyTest Case """
+
+    def setUp(self):
+        super(IPv6NDProxyTest, self).setUp()
+
+        # create 3 pg interfaces
+        self.create_pg_interfaces(range(3))
+
+        # pg0 is the master interface, with the configured subnet
+        self.pg0.admin_up()
+        self.pg0.config_ip6()
+        self.pg0.resolve_ndp()
+
+        self.pg1.ip6_enable()
+        self.pg2.ip6_enable()
+
+    def tearDown(self):
+        super(IPv6NDProxyTest, self).tearDown()
+
+    def test_nd_proxy(self):
+        """ IPv6 Proxy ND """
+
+        #
+        # Generate some hosts in the subnet that we are proxying
+        #
+        self.pg0.generate_remote_hosts(8)
+
+        nsma = in6_getnsma(inet_pton(AF_INET6, self.pg0.local_ip6))
+        d = inet_ntop(AF_INET6, nsma)
+
+        #
+        # Send an NS for one of those remote hosts on one of the proxy links
+        # expect no response since it's from an address that is not
+        # on the link that has the prefix configured
+        #
+        ns_pg1 = (Ether(dst=in6_getnsmac(nsma), src=self.pg1.remote_mac) /
+                  IPv6(dst=d, src=self.pg0._remote_hosts[2].ip6) /
+                  ICMPv6ND_NS(tgt=self.pg0.local_ip6) /
+                  ICMPv6NDOptSrcLLAddr(lladdr=self.pg0._remote_hosts[2].mac))
+
+        self.send_and_assert_no_replies(self.pg1, ns_pg1, "Off link NS")
+
+        #
+        # Add proxy support for the host
+        #
+        self.vapi.ip6_nd_proxy(
+            inet_pton(AF_INET6, self.pg0._remote_hosts[2].ip6),
+            self.pg1.sw_if_index)
+
+        #
+        # try that NS again. this time we expect an NA back
+        #
+        self.pg1.add_stream(ns_pg1)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        rx = self.pg1.get_capture(1)
+
+        self.validate_na(self.pg1, rx[0],
+                         dst_ip=self.pg0._remote_hosts[2].ip6,
+                         tgt_ip=self.pg0.local_ip6)
+
+        #
+        # ... and that we have an entry in the ND cache
+        #
+        self.assertTrue(find_nbr(self,
+                                 self.pg1.sw_if_index,
+                                 self.pg0._remote_hosts[2].ip6,
+                                 inet=AF_INET6))
+
+        #
+        # ... and we can route traffic to it
+        #
+        t = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(dst=self.pg0._remote_hosts[2].ip6,
+                  src=self.pg0.remote_ip6) /
+             UDP(sport=10000, dport=20000) /
+             Raw('\xa5' * 100))
+
+        self.pg0.add_stream(t)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        rx = self.pg1.get_capture(1)
+        rx = rx[0]
+
+        self.assertEqual(rx[Ether].dst, self.pg0._remote_hosts[2].mac)
+        self.assertEqual(rx[Ether].src, self.pg1.local_mac)
+
+        self.assertEqual(rx[IPv6].src, t[IPv6].src)
+        self.assertEqual(rx[IPv6].dst, t[IPv6].dst)
+
+        #
+        # Test we proxy for the host on the main interface
+        #
+        ns_pg0 = (Ether(dst=in6_getnsmac(nsma), src=self.pg0.remote_mac) /
+                  IPv6(dst=d, src=self.pg0.remote_ip6) /
+                  ICMPv6ND_NS(tgt=self.pg0._remote_hosts[2].ip6) /
+                  ICMPv6NDOptSrcLLAddr(lladdr=self.pg0.remote_mac))
+
+        self.pg0.add_stream(ns_pg0)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        rx = self.pg0.get_capture(1)
+
+        self.validate_na(self.pg0, rx[0],
+                         tgt_ip=self.pg0._remote_hosts[2].ip6,
+                         dst_ip=self.pg0.remote_ip6)
+
+        #
+        # Setup and resolve proxy for another host on another interface
+        #
+        ns_pg2 = (Ether(dst=in6_getnsmac(nsma), src=self.pg2.remote_mac) /
+                  IPv6(dst=d, src=self.pg0._remote_hosts[3].ip6) /
+                  ICMPv6ND_NS(tgt=self.pg0.local_ip6) /
+                  ICMPv6NDOptSrcLLAddr(lladdr=self.pg0._remote_hosts[2].mac))
+
+        self.vapi.ip6_nd_proxy(
+            inet_pton(AF_INET6, self.pg0._remote_hosts[3].ip6),
+            self.pg2.sw_if_index)
+
+        self.pg2.add_stream(ns_pg2)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        rx = self.pg2.get_capture(1)
+
+        self.validate_na(self.pg2, rx[0],
+                         dst_ip=self.pg0._remote_hosts[3].ip6,
+                         tgt_ip=self.pg0.local_ip6)
+
+        self.assertTrue(find_nbr(self,
+                                 self.pg2.sw_if_index,
+                                 self.pg0._remote_hosts[3].ip6,
+                                 inet=AF_INET6))
+
+        #
+        # hosts can communicate. pg2->pg1
+        #
+        t2 = (Ether(dst=self.pg2.local_mac,
+                    src=self.pg0.remote_hosts[3].mac) /
+              IPv6(dst=self.pg0._remote_hosts[2].ip6,
+                   src=self.pg0._remote_hosts[3].ip6) /
+              UDP(sport=10000, dport=20000) /
+              Raw('\xa5' * 100))
+
+        self.pg2.add_stream(t2)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        rx = self.pg1.get_capture(1)
+        rx = rx[0]
+
+        self.assertEqual(rx[Ether].dst, self.pg0._remote_hosts[2].mac)
+        self.assertEqual(rx[Ether].src, self.pg1.local_mac)
+
+        self.assertEqual(rx[IPv6].src, t2[IPv6].src)
+        self.assertEqual(rx[IPv6].dst, t2[IPv6].dst)
+
+        #
+        # remove the proxy configs
+        #
+        self.vapi.ip6_nd_proxy(
+            inet_pton(AF_INET6, self.pg0._remote_hosts[2].ip6),
+            self.pg1.sw_if_index,
+            is_del=1)
+        self.vapi.ip6_nd_proxy(
+            inet_pton(AF_INET6, self.pg0._remote_hosts[3].ip6),
+            self.pg2.sw_if_index,
+            is_del=1)
+
+        self.assertFalse(find_nbr(self,
+                                  self.pg2.sw_if_index,
+                                  self.pg0._remote_hosts[3].ip6,
+                                  inet=AF_INET6))
+        self.assertFalse(find_nbr(self,
+                                  self.pg1.sw_if_index,
+                                  self.pg0._remote_hosts[2].ip6,
+                                  inet=AF_INET6))
+
+        #
+        # no longer proxy-ing...
+        #
+        self.send_and_assert_no_replies(self.pg0, ns_pg0, "Proxy unconfigured")
+        self.send_and_assert_no_replies(self.pg1, ns_pg1, "Proxy unconfigured")
+        self.send_and_assert_no_replies(self.pg2, ns_pg2, "Proxy unconfigured")
+
+        #
+        # no longer forwarding. traffic generates NS out of the glean/main
+        # interface
+        #
+        self.pg2.add_stream(t2)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx = self.pg0.get_capture(1)
+
+        self.assertTrue(rx[0].haslayer(ICMPv6ND_NS))
+
+
 if __name__ == '__main__':
     unittest.main(testRunner=VppTestRunner)
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index b9c4d0c1..f04537bf 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -276,6 +276,12 @@ class VppPapiProvider(object):
                         {'sw_if_index': sw_if_index,
                          'suppress': suppress})
 
+    def ip6_nd_proxy(self, address, sw_if_index, is_del=0):
+        return self.api(self.papi.ip6nd_proxy_add_del,
+                        {'address': address,
+                         'sw_if_index': sw_if_index,
+                         'is_del': is_del})
+
     def ip6_sw_interface_ra_config(self, sw_if_index,
                                    no,
                                    suppress,
-- 
cgit 1.2.3-korg


From d96bad8ceb2ca0b798434619c5c5d1a199ec6382 Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Wed, 8 Mar 2017 01:12:54 -0800
Subject: ARP resilience in the absence of IP config on input and output
 interfaces

Change-Id: Ia9c6790436f8f6337b5351e3017ef5e441a8a4e9
Signed-off-by: Neale Ranns <nranns@cisco.com>
(cherry picked from commit 5e323329ed6ded6f6944c56069329207ad9069c0)
---
 src/vnet/ethernet/arp.c |  6 ++++++
 src/vnet/ip/lookup.c    | 15 +++++++++++++++
 2 files changed, 21 insertions(+)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c
index 8898d33b..222415b0 100644
--- a/src/vnet/ethernet/arp.c
+++ b/src/vnet/ethernet/arp.c
@@ -792,6 +792,7 @@ typedef enum
   _ (missing_interface_address, "ARP missing interface address") \
   _ (gratuitous_arp, "ARP probe or announcement dropped") \
   _ (interface_no_table, "Interface is not mapped to an IP table") \
+  _ (interface_not_ip_enabled, "Interface is not IP enabled") \
 
 typedef enum
 {
@@ -1048,6 +1049,11 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
 
 	  sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX];
 
+	  /* not playing the ARP game if the interface is not IPv4 enabled */
+	  error0 =
+	    (im4->ip_enabled_by_sw_if_index[sw_if_index0] == 0 ?
+	     ETHERNET_ARP_ERROR_interface_not_ip_enabled : error0);
+
 	  if (error0)
 	    goto drop2;
 
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 8607fa5f..9ae269cc 100644
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -170,6 +170,21 @@ ip_interface_address_add_del (ip_lookup_main_t * lm,
   return /* no error */ 0;
 }
 
+static clib_error_t *
+ip_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
+{
+  vec_validate_init_empty (ip4_main.
+			   lookup_main.if_address_pool_index_by_sw_if_index,
+			   sw_if_index, ~0);
+  vec_validate_init_empty (ip6_main.
+			   lookup_main.if_address_pool_index_by_sw_if_index,
+			   sw_if_index, ~0);
+
+  return (NULL);
+}
+
+VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip_sw_interface_add_del);
+
 void
 ip_lookup_init (ip_lookup_main_t * lm, u32 is_ip6)
 {
-- 
cgit 1.2.3-korg


From b069a6910aa2b95316ccdb5d9e5b95143b9dc7c0 Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@wasa-ucs-11.cisco.com>
Date: Wed, 15 Mar 2017 12:34:25 -0400
Subject: Cache a 'has-features' flag on the adjacency for faster access.
 Reclaim the node_index memeber from the rewrite for space - this is only used
 for formtting

before:
  ip4-rewrite * * * *  2.66e1  256.00
after:
  ip4-rewrite * * * *  2.40e1  256.00

Change-Id: Ic397150727cad38811564777419ad6bd26b8a3a6
Signed-off-by: Neale Ranns <nranns@wasa-ucs-11.cisco.com>
---
 src/vnet/adj/adj.c          |  82 +++++++++++++++++++++++++++++++-----
 src/vnet/adj/adj.h          |   6 +++
 src/vnet/adj/adj_mcast.c    |   4 +-
 src/vnet/adj/adj_midchain.c |   4 +-
 src/vnet/adj/adj_nbr.c      |   4 +-
 src/vnet/feature/feature.c  |   2 +
 src/vnet/ip/ip4_forward.c   |  21 ++++++----
 src/vnet/ip/ip6_forward.c   |  21 ++++++----
 src/vnet/ip/lookup.c        |   8 ++--
 src/vnet/mpls/mpls_output.c |  42 ++++++++++++-------
 src/vnet/rewrite.c          | 100 +-------------------------------------------
 src/vnet/rewrite.h          |  30 +++++++++----
 12 files changed, 162 insertions(+), 162 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet/adj/adj.c b/src/vnet/adj/adj.c
index 4ed4999f..f3d483ac 100644
--- a/src/vnet/adj/adj.c
+++ b/src/vnet/adj/adj.c
@@ -135,16 +135,6 @@ format_ip_adjacency (u8 * s, va_list * args)
         vlib_get_combined_counter(&adjacency_counters, adj_index, &counts);
         s = format (s, "\n counts:[%Ld:%Ld]", counts.packets, counts.bytes);
 	s = format (s, "\n locks:%d", adj->ia_node.fn_locks);
-	s = format (s, " node:[%d]:%U",
-		    adj->rewrite_header.node_index,
-		    format_vlib_node_name, vlib_get_main(),
-		    adj->rewrite_header.node_index);
-	s = format (s, " next:[%d]:%U",
-		    adj->rewrite_header.next_index,
-		    format_vlib_next_node_name,
-		    vlib_get_main(),
-		    adj->rewrite_header.node_index,
-		    adj->rewrite_header.next_index);
 	s = format(s, "\n children:\n  ");
 	s = fib_node_children_format(adj->ia_node.fn_children, s);
     }
@@ -271,6 +261,78 @@ adj_child_remove (adj_index_t adj_index,
                           sibling_index);
 }
 
+/*
+ * Context for the walk to update the cached feture flags.
+ */
+typedef struct adj_feature_update_t_
+{
+    u8 arc;
+    u8 enable;
+} adj_feature_update_ctx_t;
+
+static adj_walk_rc_t
+adj_feature_update_walk_cb (adj_index_t ai,
+                            void *arg)
+{
+    adj_feature_update_ctx_t *ctx = arg;
+    ip_adjacency_t *adj;
+
+    adj = adj_get(ai);
+
+    /*
+     * this ugly mess matches the feature arc that is changing with affected
+     * adjacencies
+     */
+    if (((ctx->arc == ip6_main.lookup_main.output_feature_arc_index) &&
+         (VNET_LINK_IP6 == adj->ia_link)) ||
+        ((ctx->arc == ip4_main.lookup_main.output_feature_arc_index) &&
+         (VNET_LINK_IP4 == adj->ia_link)) ||
+        ((ctx->arc == mpls_main.output_feature_arc_index) &&
+         (VNET_LINK_MPLS == adj->ia_link)))
+    {
+        if (ctx->enable)
+            adj->rewrite_header.flags |= VNET_REWRITE_HAS_FEATURES;
+        else
+            adj->rewrite_header.flags &= ~VNET_REWRITE_HAS_FEATURES;
+    }
+    return (ADJ_WALK_RC_CONTINUE);
+}
+
+void
+adj_feature_update (u32 sw_if_index,
+                    u8 arc_index,
+                    u8 is_enable)
+{
+    /*
+     * Walk all the adjacencies on the interface to update the cached
+     * 'has-features' flag
+     */
+    adj_feature_update_ctx_t ctx = {
+        .arc = arc_index,
+        .enable = is_enable,
+    };
+    adj_walk (sw_if_index, adj_feature_update_walk_cb, &ctx);
+}
+
+/**
+ * @brief Walk the Adjacencies on a given interface
+ */
+void
+adj_walk (u32 sw_if_index,
+          adj_walk_cb_t cb,
+          void *ctx)
+{
+    /*
+     * walk all the neighbor adjacencies
+     */
+    fib_protocol_t proto;
+
+    FOR_EACH_FIB_IP_PROTOCOL(proto)
+    {
+        adj_nbr_walk(sw_if_index, proto, cb, ctx);
+    }
+}
+
 /**
  * @brief Return the link type of the adjacency
  */
diff --git a/src/vnet/adj/adj.h b/src/vnet/adj/adj.h
index fcc5890c..271fdbc6 100644
--- a/src/vnet/adj/adj.h
+++ b/src/vnet/adj/adj.h
@@ -96,6 +96,12 @@ extern u32 adj_get_sw_if_index (adj_index_t ai);
  */
 extern const u8* adj_get_rewrite (adj_index_t ai);
 
+/**
+ * @brief Notify the adjacency subsystem that the features settings for
+ * an interface have changed
+ */
+extern void adj_feature_update (u32 sw_if_index, u8 arc_index, u8 is_enable);
+
 /**
  * @brief
  * The global adjacnecy pool. Exposed for fast/inline data-plane access
diff --git a/src/vnet/adj/adj_mcast.c b/src/vnet/adj/adj_mcast.c
index 1345aedb..a3ba4d6a 100644
--- a/src/vnet/adj/adj_mcast.c
+++ b/src/vnet/adj/adj_mcast.c
@@ -256,15 +256,13 @@ format_adj_mcast (u8* s, va_list *ap)
 {
     index_t index = va_arg(*ap, index_t);
     CLIB_UNUSED(u32 indent) = va_arg(*ap, u32);
-    vnet_main_t * vnm = vnet_get_main();
     ip_adjacency_t * adj = adj_get(index);
 
     s = format(s, "%U-mcast: ",
                format_fib_protocol, adj->ia_nh_proto);
     s = format (s, "%U",
 		format_vnet_rewrite,
-		vnm->vlib_main, &adj->rewrite_header,
-                sizeof (adj->rewrite_data), 0);
+                &adj->rewrite_header, sizeof (adj->rewrite_data), 0);
 
     return (s);
 }
diff --git a/src/vnet/adj/adj_midchain.c b/src/vnet/adj/adj_midchain.c
index 7d315651..55b5e44b 100644
--- a/src/vnet/adj/adj_midchain.c
+++ b/src/vnet/adj/adj_midchain.c
@@ -501,7 +501,6 @@ format_adj_midchain (u8* s, va_list *ap)
 {
     index_t index = va_arg(*ap, index_t);
     u32 indent = va_arg(*ap, u32);
-    vnet_main_t * vnm = vnet_get_main();
     ip_adjacency_t * adj = adj_get(index);
 
     s = format (s, "%U", format_vnet_link, adj->ia_link);
@@ -509,8 +508,7 @@ format_adj_midchain (u8* s, va_list *ap)
 		format_ip46_address, &adj->sub_type.nbr.next_hop);
     s = format (s, " %U",
 		format_vnet_rewrite,
-		vnm->vlib_main, &adj->rewrite_header,
-		sizeof (adj->rewrite_data), indent);
+		&adj->rewrite_header, sizeof (adj->rewrite_data), indent);
     s = format (s, "\n%Ustacked-on:\n%U%U",
 		format_white_space, indent,
 		format_white_space, indent+2,
diff --git a/src/vnet/adj/adj_nbr.c b/src/vnet/adj/adj_nbr.c
index 9e8073d3..9ef3651d 100644
--- a/src/vnet/adj/adj_nbr.c
+++ b/src/vnet/adj/adj_nbr.c
@@ -433,7 +433,6 @@ adj_nbr_update_rewrite_internal (ip_adjacency_t *adj,
 	vnet_rewrite_clear_data_internal(&adj->rewrite_header,
 					 sizeof(adj->rewrite_data));
     }
-    adj->rewrite_header.node_index = this_node;
     adj->rewrite_header.next_index = vlib_node_add_next(vlib_get_main(),
                                                         this_node,
                                                         next_node);
@@ -971,7 +970,6 @@ format_adj_nbr (u8* s, va_list *ap)
 {
     index_t index = va_arg(*ap, index_t);
     CLIB_UNUSED(u32 indent) = va_arg(*ap, u32);
-    vnet_main_t * vnm = vnet_get_main();
     ip_adjacency_t * adj = adj_get(index);
 
     s = format (s, "%U", format_vnet_link, adj->ia_link);
@@ -980,7 +978,7 @@ format_adj_nbr (u8* s, va_list *ap)
 		adj_proto_to_46(adj->ia_nh_proto));
     s = format (s, "%U",
 		format_vnet_rewrite,
-		vnm->vlib_main, &adj->rewrite_header, sizeof (adj->rewrite_data), 0);
+		&adj->rewrite_header, sizeof (adj->rewrite_data), 0);
 
     return (s);
 }
diff --git a/src/vnet/feature/feature.c b/src/vnet/feature/feature.c
index 80ef08d0..5a4be029 100644
--- a/src/vnet/feature/feature.c
+++ b/src/vnet/feature/feature.c
@@ -14,6 +14,7 @@
  */
 
 #include <vnet/feature/feature.h>
+#include <vnet/adj/adj.h>
 
 vnet_feature_main_t feature_main;
 
@@ -229,6 +230,7 @@ vnet_feature_enable_disable_with_index (u8 arc_index, u32 feature_index,
   fm->sw_if_index_has_features[arc_index] =
     clib_bitmap_set (fm->sw_if_index_has_features[arc_index], sw_if_index,
 		     (feature_count > 0));
+  adj_feature_update (sw_if_index, arc_index, (feature_count > 0));
 
   fm->feature_count_by_sw_if_index[arc_index][sw_if_index] = feature_count;
   return 0;
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index 5472428f..34bc6c5d 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -1232,7 +1232,6 @@ format_ip4_rewrite_trace (u8 * s, va_list * args)
   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
   ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
-  vnet_main_t *vnm = vnet_get_main ();
   uword indent = format_get_indent (s);
 
   s = format (s, "tx_sw_if_index %d dpo-idx %d : %U flow hash: 0x%08x",
@@ -1241,7 +1240,7 @@ format_ip4_rewrite_trace (u8 * s, va_list * args)
   s = format (s, "\n%U%U",
 	      format_white_space, indent,
 	      format_ip_adjacency_packet_data,
-	      vnm, t->dpo_index, t->packet_data, sizeof (t->packet_data));
+	      t->dpo_index, t->packet_data, sizeof (t->packet_data));
   return s;
 }
 
@@ -2507,8 +2506,10 @@ ip4_rewrite_inline (vlib_main_t * vm,
 	      tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
 	      vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
 
-	      vnet_feature_arc_start (lm->output_feature_arc_index,
-				      tx_sw_if_index0, &next0, p0);
+	      if (PREDICT_FALSE
+		  (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+		vnet_feature_arc_start (lm->output_feature_arc_index,
+					tx_sw_if_index0, &next0, p0);
 	    }
 	  if (PREDICT_TRUE (error1 == IP4_ERROR_NONE))
 	    {
@@ -2519,8 +2520,10 @@ ip4_rewrite_inline (vlib_main_t * vm,
 	      tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
 	      vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
 
-	      vnet_feature_arc_start (lm->output_feature_arc_index,
-				      tx_sw_if_index1, &next1, p1);
+	      if (PREDICT_FALSE
+		  (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+		vnet_feature_arc_start (lm->output_feature_arc_index,
+					tx_sw_if_index1, &next1, p1);
 	    }
 
 	  /* Guess we are only writing on simple Ethernet header. */
@@ -2671,8 +2674,10 @@ ip4_rewrite_inline (vlib_main_t * vm,
 		  adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
 		}
 
-	      vnet_feature_arc_start (lm->output_feature_arc_index,
-				      tx_sw_if_index0, &next0, p0);
+	      if (PREDICT_FALSE
+		  (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+		vnet_feature_arc_start (lm->output_feature_arc_index,
+					tx_sw_if_index0, &next0, p0);
 
 	    }
 
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index b80c757a..0b8691bf 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -961,7 +961,6 @@ format_ip6_rewrite_trace (u8 * s, va_list * args)
   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
   ip6_forward_next_trace_t *t = va_arg (*args, ip6_forward_next_trace_t *);
-  vnet_main_t *vnm = vnet_get_main ();
   uword indent = format_get_indent (s);
 
   s = format (s, "tx_sw_if_index %d adj-idx %d : %U flow hash: 0x%08x",
@@ -970,7 +969,7 @@ format_ip6_rewrite_trace (u8 * s, va_list * args)
   s = format (s, "\n%U%U",
 	      format_white_space, indent,
 	      format_ip_adjacency_packet_data,
-	      vnm, t->adj_index, t->packet_data, sizeof (t->packet_data));
+	      t->adj_index, t->packet_data, sizeof (t->packet_data));
   return s;
 }
 
@@ -2054,8 +2053,10 @@ ip6_rewrite_inline (vlib_main_t * vm,
 	      vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
 	      next0 = adj0[0].rewrite_header.next_index;
 
-	      vnet_feature_arc_start (lm->output_feature_arc_index,
-				      tx_sw_if_index0, &next0, p0);
+	      if (PREDICT_FALSE
+		  (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+		vnet_feature_arc_start (lm->output_feature_arc_index,
+					tx_sw_if_index0, &next0, p0);
 	    }
 	  if (PREDICT_TRUE (error1 == IP6_ERROR_NONE))
 	    {
@@ -2066,8 +2067,10 @@ ip6_rewrite_inline (vlib_main_t * vm,
 	      vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
 	      next1 = adj1[0].rewrite_header.next_index;
 
-	      vnet_feature_arc_start (lm->output_feature_arc_index,
-				      tx_sw_if_index1, &next1, p1);
+	      if (PREDICT_FALSE
+		  (adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+		vnet_feature_arc_start (lm->output_feature_arc_index,
+					tx_sw_if_index1, &next1, p1);
 	    }
 
 	  /* Guess we are only writing on simple Ethernet header. */
@@ -2182,8 +2185,10 @@ ip6_rewrite_inline (vlib_main_t * vm,
 	      vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
 	      next0 = adj0[0].rewrite_header.next_index;
 
-	      vnet_feature_arc_start (lm->output_feature_arc_index,
-				      tx_sw_if_index0, &next0, p0);
+	      if (PREDICT_FALSE
+		  (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+		vnet_feature_arc_start (lm->output_feature_arc_index,
+					tx_sw_if_index0, &next0, p0);
 	    }
 
 	  if (is_midchain)
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 9ae269cc..dabfa012 100644
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -291,7 +291,6 @@ format_ip_lookup_next (u8 * s, va_list * args)
 u8 *
 format_ip_adjacency_packet_data (u8 * s, va_list * args)
 {
-  vnet_main_t *vnm = va_arg (*args, vnet_main_t *);
   u32 adj_index = va_arg (*args, u32);
   u8 *packet_data = va_arg (*args, u8 *);
   u32 n_packet_data_bytes = va_arg (*args, u32);
@@ -300,10 +299,9 @@ format_ip_adjacency_packet_data (u8 * s, va_list * args)
   switch (adj->lookup_next_index)
     {
     case IP_LOOKUP_NEXT_REWRITE:
-      s = format (s, "%U",
-		  format_vnet_rewrite_header,
-		  vnm->vlib_main, &adj->rewrite_header, packet_data,
-		  n_packet_data_bytes);
+    case IP_LOOKUP_NEXT_MCAST:
+      s =
+	format (s, "%U", format_hex_bytes, packet_data, n_packet_data_bytes);
       break;
 
     default:
diff --git a/src/vnet/mpls/mpls_output.c b/src/vnet/mpls/mpls_output.c
index c06cf917..cf354006 100644
--- a/src/vnet/mpls/mpls_output.c
+++ b/src/vnet/mpls/mpls_output.c
@@ -35,7 +35,6 @@ format_mpls_output_trace (u8 * s, va_list * args)
   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
   mpls_output_trace_t * t = va_arg (*args, mpls_output_trace_t *);
-  vnet_main_t * vnm = vnet_get_main();
   uword indent = format_get_indent (s);
 
   s = format (s, "adj-idx %d : %U flow hash: 0x%08x",
@@ -45,8 +44,7 @@ format_mpls_output_trace (u8 * s, va_list * args)
   s = format (s, "\n%U%U",
               format_white_space, indent,
               format_ip_adjacency_packet_data,
-              vnm, t->adj_index,
-              t->packet_data, sizeof (t->packet_data));
+              t->adj_index, t->packet_data, sizeof (t->packet_data));
   return s;
 }
 
@@ -59,12 +57,14 @@ mpls_output_inline (vlib_main_t * vm,
   u32 n_left_from, next_index, * from, * to_next, cpu_index;
   vlib_node_runtime_t * error_node;
   u32 n_left_to_next;
+  mpls_main_t *mm;
 
   cpu_index = os_get_cpu_number();
   error_node = vlib_node_get_runtime (vm, mpls_output_node.index);
   from = vlib_frame_vector_args (from_frame);
   n_left_from = from_frame->n_vectors;
   next_index = node->cached_next_index;
+  mm = &mpls_main;
 
   while (n_left_from > 0)
     {
@@ -154,10 +154,10 @@ mpls_output_inline (vlib_main_t * vm,
               next0 = adj0[0].rewrite_header.next_index;
               error0 = IP4_ERROR_NONE;
 
-              if (is_midchain)
-                {
-                  adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
-                }
+              if (PREDICT_FALSE(adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+                vnet_feature_arc_start (mm->output_feature_arc_index,
+                                        adj0[0].rewrite_header.sw_if_index,
+                                        &next0, p0);
             }
           else
             {
@@ -175,16 +175,21 @@ mpls_output_inline (vlib_main_t * vm,
               next1 = adj1[0].rewrite_header.next_index;
               error1 = IP4_ERROR_NONE;
 
-              if (is_midchain)
-                {
-                  adj1->sub_type.midchain.fixup_func(vm, adj1, p1);
-                }
+              if (PREDICT_FALSE(adj1[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+                vnet_feature_arc_start (mm->output_feature_arc_index,
+                                        adj1[0].rewrite_header.sw_if_index,
+                                        &next1, p1);
             }
           else
             {
               error1 = IP4_ERROR_MTU_EXCEEDED;
               next1 = MPLS_OUTPUT_NEXT_DROP;
             }
+          if (is_midchain)
+          {
+              adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
+              adj1->sub_type.midchain.fixup_func(vm, adj1, p1);
+          }
 
           p0->error = error_node->errors[error0];
           p1->error = error_node->errors[error1];
@@ -254,17 +259,22 @@ mpls_output_inline (vlib_main_t * vm,
               next0 = adj0[0].rewrite_header.next_index;
               error0 = IP4_ERROR_NONE;
 
-	      if (is_midchain)
-	        {
-		  adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
-		}
+              if (PREDICT_FALSE(adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
+                vnet_feature_arc_start (mm->output_feature_arc_index,
+                                        adj0[0].rewrite_header.sw_if_index,
+                                        &next0, p0);
             }
           else
             {
               error0 = IP4_ERROR_MTU_EXCEEDED;
               next0 = MPLS_OUTPUT_NEXT_DROP;
             }
-	  p0->error = error_node->errors[error0];
+          if (is_midchain)
+          {
+              adj0->sub_type.midchain.fixup_func(vm, adj0, p0);
+          }
+
+          p0->error = error_node->errors[error0];
 
 	  from += 1;
 	  n_left_from -= 1;
diff --git a/src/vnet/rewrite.c b/src/vnet/rewrite.c
index 8925ad61..c4a171c1 100644
--- a/src/vnet/rewrite.c
+++ b/src/vnet/rewrite.c
@@ -67,14 +67,10 @@ vnet_rewrite_copy_slow_path (vnet_rewrite_data_t * p0,
 u8 *
 format_vnet_rewrite (u8 * s, va_list * args)
 {
-  vlib_main_t *vm = va_arg (*args, vlib_main_t *);
   vnet_rewrite_header_t *rw = va_arg (*args, vnet_rewrite_header_t *);
   u32 max_data_bytes = va_arg (*args, u32);
   CLIB_UNUSED (uword indent) = va_arg (*args, u32);
   vnet_main_t *vnm = vnet_get_main ();
-  vlib_node_t *next;
-
-  next = vlib_get_next_node (vm, rw->node_index, rw->next_index);
 
   if (rw->sw_if_index != ~0)
     {
@@ -85,108 +81,17 @@ format_vnet_rewrite (u8 * s, va_list * args)
       else
 	s = format (s, "DELETED");
     }
-  else
-    s = format (s, "%v: ", next->name);
 
   /* Format rewrite string. */
   if (rw->data_bytes > 0)
 
     s = format (s, "%U",
-		next->format_buffer ? next->format_buffer : format_hex_bytes,
+		format_hex_bytes,
 		rw->data + max_data_bytes - rw->data_bytes, rw->data_bytes);
 
   return s;
 }
 
-u8 *
-format_vnet_rewrite_header (u8 * s, va_list * args)
-{
-  vlib_main_t *vm = va_arg (*args, vlib_main_t *);
-  vnet_rewrite_header_t *rw = va_arg (*args, vnet_rewrite_header_t *);
-  u8 *packet_data = va_arg (*args, u8 *);
-  u32 packet_data_bytes = va_arg (*args, u32);
-  vlib_node_t *next;
-
-  next = vlib_get_next_node (vm, rw->node_index, rw->next_index);
-
-  /* Format rewrite string. */
-  s = format (s, "%U",
-	      next->format_buffer ? next->format_buffer : format_hex_bytes,
-	      packet_data, packet_data_bytes);
-
-  return s;
-}
-
-uword
-unformat_vnet_rewrite (unformat_input_t * input, va_list * args)
-{
-  vlib_main_t *vm = va_arg (*args, vlib_main_t *);
-  vnet_rewrite_header_t *rw = va_arg (*args, vnet_rewrite_header_t *);
-  u32 max_data_bytes = va_arg (*args, u32);
-  vnet_main_t *vnm = vnet_get_main ();
-  vlib_node_t *next;
-  u32 next_index, sw_if_index, max_packet_bytes, error;
-  u8 *rw_data;
-
-  rw_data = 0;
-  sw_if_index = ~0;
-  max_packet_bytes = ~0;
-  error = 1;
-
-  /* Parse sw interface. */
-  if (unformat (input, "%U", unformat_vnet_sw_interface, vnm, &sw_if_index))
-    {
-      vnet_hw_interface_t *hi;
-
-      hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
-
-      next_index = hi->output_node_index;
-      max_packet_bytes = hi->max_l3_packet_bytes[VLIB_RX];
-    }
-
-  else if (unformat (input, "%U", unformat_vlib_node, vm, &next_index))
-    ;
-
-  else
-    goto done;
-
-  next = vlib_get_node (vm, next_index);
-
-  if (next->unformat_buffer
-      && unformat_user (input, next->unformat_buffer, &rw_data))
-    ;
-
-  else if (unformat_user (input, unformat_hex_string, &rw_data)
-	   || unformat (input, "0x%U", unformat_hex_string, &rw_data))
-    ;
-
-  else
-    goto done;
-
-  /* Re-write does not fit. */
-  if (vec_len (rw_data) >= max_data_bytes)
-    goto done;
-
-  {
-    u32 tmp;
-
-    if (unformat (input, "mtu %d", &tmp)
-	&& tmp < (1 << BITS (rw->max_l3_packet_bytes)))
-      max_packet_bytes = tmp;
-  }
-
-  error = 0;
-  rw->sw_if_index = sw_if_index;
-  rw->max_l3_packet_bytes = max_packet_bytes;
-  rw->next_index = vlib_node_add_next (vm, rw->node_index, next_index);
-  vnet_rewrite_set_data_internal (rw, max_data_bytes, rw_data,
-				  vec_len (rw_data));
-
-done:
-  vec_free (rw_data);
-  return error == 0;
-}
-
 u32
 vnet_tx_node_index_for_sw_interface (vnet_main_t * vnm, u32 sw_if_index)
 {
@@ -200,7 +105,6 @@ vnet_rewrite_init (vnet_main_t * vnm,
 		   u32 this_node, u32 next_node, vnet_rewrite_header_t * rw)
 {
   rw->sw_if_index = sw_if_index;
-  rw->node_index = this_node;
   rw->next_index = vlib_node_add_next (vnm->vlib_main, this_node, next_node);
   rw->max_l3_packet_bytes =
     vnet_sw_interface_get_mtu (vnm, sw_if_index, VLIB_TX);
@@ -249,7 +153,6 @@ vnet_rewrite_for_tunnel (vnet_main_t * vnm,
    * ipX-forward, this will be interpreted as a FIB number.
    */
   rw->sw_if_index = tx_sw_if_index;
-  rw->node_index = rewrite_node_index;
   rw->next_index = vlib_node_add_next (vnm->vlib_main, rewrite_node_index,
 				       post_rewrite_node_index);
   rw->max_l3_packet_bytes = (u16) ~ 0;	/* we can't know at this point */
@@ -284,7 +187,6 @@ unserialize_vnet_rewrite (serialize_main_t * m, va_list * va)
   u8 *p;
 
   /* It is up to user to fill these in. */
-  rw->node_index = ~0;
   rw->next_index = ~0;
 
   unserialize_integer (m, &rw->sw_if_index, sizeof (rw->sw_if_index));
diff --git a/src/vnet/rewrite.h b/src/vnet/rewrite.h
index ce2bce3a..8435a726 100644
--- a/src/vnet/rewrite.h
+++ b/src/vnet/rewrite.h
@@ -46,14 +46,22 @@
 /* Consider using vector types for speed? */
 typedef uword vnet_rewrite_data_t;
 
+/**
+ * Flags associated with the rewrite/adjacency
+ */
+typedef enum vnet_rewrite_flags_t_
+{
+  /**
+   * This adjacency/interface has output features configured
+   */
+  VNET_REWRITE_HAS_FEATURES = (1 << 0),
+} __attribute__ ((packed)) vnet_rewrite_flags_t;
+
 /* *INDENT-OFF* */
 typedef CLIB_PACKED (struct {
   /* Interface to mark re-written packets with. */
   u32 sw_if_index;
 
-  /* Packet processing node where rewrite happens. */
-  u32 node_index;
-
   /* Next node to feed after packet rewrite is done. */
   u16 next_index;
 
@@ -64,6 +72,11 @@ typedef CLIB_PACKED (struct {
      Used for MTU check after packet rewrite. */
   u16 max_l3_packet_bytes;
 
+  u16 unused1;
+  u8  unused2;
+
+  vnet_rewrite_flags_t flags;
+
   /* When dynamically writing a multicast destination L2 addresss
    * this is the offset within the address to start writing n
    * bytes of the IP mcast address */
@@ -79,6 +92,13 @@ typedef CLIB_PACKED (struct {
 }) vnet_rewrite_header_t;
 /* *INDENT-ON* */
 
+/**
+ * At 16 bytes of rewrite herader we have enought space left for a IPv6
+ * (40 bytes) + LISP-GPE (8 bytes) in the cache line
+ */
+STATIC_ASSERT (sizeof (vnet_rewrite_header_t) <= 16,
+	       "Rewrite header too big");
+
 /*
   Helper macro for declaring rewrite string w/ given max-size.
 
@@ -317,11 +337,7 @@ u8 *vnet_build_rewrite_for_sw_interface (struct vnet_main_t *vnm,
 void vnet_update_adjacency_for_sw_interface (struct vnet_main_t *vnm,
 					     u32 sw_if_index, u32 ai);
 
-/* Parser for unformat header & rewrite string. */
-unformat_function_t unformat_vnet_rewrite;
-
 format_function_t format_vnet_rewrite;
-format_function_t format_vnet_rewrite_header;
 
 serialize_function_t serialize_vnet_rewrite, unserialize_vnet_rewrite;
 
-- 
cgit 1.2.3-korg


From fa5d19829759cef45cc34efe844d9471f5a1fc61 Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Mon, 20 Feb 2017 14:19:51 -0800
Subject: Adjacency layout change and move to vnet/adj

Change-Id: I03195a86c69f84a301051c6b3ab64456bbf28645
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/vnet.am                            |   9 +-
 src/vnet/adj/adj.c                     |   3 +-
 src/vnet/adj/adj.h                     | 220 ++++++++++++++++++++-
 src/vnet/adj/adj_midchain.c            |  25 ++-
 src/vnet/adj/adj_midchain.h            |  22 +--
 src/vnet/adj/adj_nbr.c                 |   8 +-
 src/vnet/adj/rewrite.c                 | 234 ++++++++++++++++++++++
 src/vnet/adj/rewrite.h                 | 350 +++++++++++++++++++++++++++++++++
 src/vnet/ethernet/arp.c                |   1 -
 src/vnet/gre/gre.c                     |   4 +-
 src/vnet/ip/ip6_neighbor.c             |   1 -
 src/vnet/ip/lookup.c                   |  10 +-
 src/vnet/ip/lookup.h                   | 209 +-------------------
 src/vnet/lisp-gpe/lisp_gpe_adjacency.c |   4 +-
 src/vnet/mpls/mpls_tunnel.c            |   2 +-
 src/vnet/rewrite.c                     | 234 ----------------------
 src/vnet/rewrite.h                     | 350 ---------------------------------
 17 files changed, 837 insertions(+), 849 deletions(-)
 create mode 100644 src/vnet/adj/rewrite.c
 create mode 100644 src/vnet/adj/rewrite.h
 delete mode 100644 src/vnet/rewrite.c
 delete mode 100644 src/vnet/rewrite.h

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet.am b/src/vnet.am
index 823dd0d6..f95ec2fb 100644
--- a/src/vnet.am
+++ b/src/vnet.am
@@ -40,8 +40,7 @@ libvnet_la_SOURCES +=				\
   vnet/interface_format.c			\
   vnet/interface_output.c			\
   vnet/misc.c					\
-  vnet/replication.c                            \
-  vnet/rewrite.c
+  vnet/replication.c
 
 nobase_include_HEADERS +=			\
   vnet/api_errno.h				\
@@ -963,14 +962,16 @@ libvnet_la_SOURCES +=				\
   vnet/adj/adj_mcast.c   			\
   vnet/adj/adj_l2.c      			\
   vnet/adj/adj_nsh.c      			\
-  vnet/adj/adj.c
+  vnet/adj/adj.c		                \
+  vnet/adj/rewrite.c
 
 nobase_include_HEADERS +=			\
   vnet/adj/adj.h				\
   vnet/adj/adj_types.h				\
   vnet/adj/adj_glean.h  			\
   vnet/adj/adj_nsh.h  				\
-  vnet/adj/adj_nbr.h
+  vnet/adj/adj_nbr.h 				\
+  vnet/adj/rewrite.h
 
 ########################################
 # Data-Plane Objects
diff --git a/src/vnet/adj/adj.c b/src/vnet/adj/adj.c
index c1d036a0..7cf9e9d0 100644
--- a/src/vnet/adj/adj.c
+++ b/src/vnet/adj/adj.c
@@ -48,7 +48,7 @@ adj_alloc (fib_protocol_t proto)
 {
     ip_adjacency_t *adj;
 
-    pool_get(adj_pool, adj);
+    pool_get_aligned(adj_pool, adj, CLIB_CACHE_LINE_BYTES);
 
     adj_poison(adj);
 
@@ -58,7 +58,6 @@ adj_alloc (fib_protocol_t proto)
                                    adj_get_index(adj));
 
     adj->rewrite_header.sw_if_index = ~0;
-    adj->n_adj = 1;
     adj->lookup_next_index = 0;
 
     fib_node_init(&adj->ia_node,
diff --git a/src/vnet/adj/adj.h b/src/vnet/adj/adj.h
index 271fdbc6..af7730f7 100644
--- a/src/vnet/adj/adj.h
+++ b/src/vnet/adj/adj.h
@@ -24,7 +24,6 @@
  *            address in the ARP packet.
  *          UNSHARED. Only one per-interface.
  *   - midchain: a nighbour adj on a virtual/tunnel interface.
- *   - rewrite: an adj with no key, but with a rewrite string.
  *
  * The API to create and update the adjacency is very sub-type specific. This
  * is intentional as it encourages the user to carefully consider which adjacency
@@ -42,10 +41,227 @@
 #ifndef __ADJ_H__
 #define __ADJ_H__
 
-#include <vnet/ip/lookup.h>
 #include <vnet/adj/adj_types.h>
 #include <vnet/adj/adj_nbr.h>
 #include <vnet/adj/adj_glean.h>
+#include <vnet/adj/rewrite.h>
+
+/** @brief Common (IP4/IP6) next index stored in adjacency. */
+typedef enum
+{
+  /** Adjacency to drop this packet. */
+  IP_LOOKUP_NEXT_DROP,
+  /** Adjacency to punt this packet. */
+  IP_LOOKUP_NEXT_PUNT,
+
+  /** This packet is for one of our own IP addresses. */
+  IP_LOOKUP_NEXT_LOCAL,
+
+  /** This packet matches an "incomplete adjacency" and packets
+     need to be passed to ARP to find rewrite string for
+     this destination. */
+  IP_LOOKUP_NEXT_ARP,
+
+  /** This packet matches an "interface route" and packets
+     need to be passed to ARP to find rewrite string for
+     this destination. */
+  IP_LOOKUP_NEXT_GLEAN,
+
+  /** This packet is to be rewritten and forwarded to the next
+     processing node.  This is typically the output interface but
+     might be another node for further output processing. */
+  IP_LOOKUP_NEXT_REWRITE,
+
+  /** This packets follow a mid-chain adjacency */
+  IP_LOOKUP_NEXT_MIDCHAIN,
+
+  /** This packets needs to go to ICMP error */
+  IP_LOOKUP_NEXT_ICMP_ERROR,
+
+  /** Multicast Adjacency. */
+  IP_LOOKUP_NEXT_MCAST,
+
+  IP_LOOKUP_N_NEXT,
+} __attribute__ ((packed)) ip_lookup_next_t;
+
+typedef enum
+{
+  IP4_LOOKUP_N_NEXT = IP_LOOKUP_N_NEXT,
+} ip4_lookup_next_t;
+
+typedef enum
+{
+  /* Hop-by-hop header handling */
+  IP6_LOOKUP_NEXT_HOP_BY_HOP = IP_LOOKUP_N_NEXT,
+  IP6_LOOKUP_NEXT_ADD_HOP_BY_HOP,
+  IP6_LOOKUP_NEXT_POP_HOP_BY_HOP,
+  IP6_LOOKUP_N_NEXT,
+} ip6_lookup_next_t;
+
+#define IP4_LOOKUP_NEXT_NODES {					\
+    [IP_LOOKUP_NEXT_DROP] = "ip4-drop",				\
+    [IP_LOOKUP_NEXT_PUNT] = "ip4-punt",				\
+    [IP_LOOKUP_NEXT_LOCAL] = "ip4-local",			\
+    [IP_LOOKUP_NEXT_ARP] = "ip4-arp",				\
+    [IP_LOOKUP_NEXT_GLEAN] = "ip4-glean",			\
+    [IP_LOOKUP_NEXT_REWRITE] = "ip4-rewrite",    		\
+    [IP_LOOKUP_NEXT_MCAST] = "ip4-rewrite-mcast",	        \
+    [IP_LOOKUP_NEXT_MIDCHAIN] = "ip4-midchain",		        \
+    [IP_LOOKUP_NEXT_ICMP_ERROR] = "ip4-icmp-error",		\
+}
+
+#define IP6_LOOKUP_NEXT_NODES {					\
+    [IP_LOOKUP_NEXT_DROP] = "ip6-drop",				\
+    [IP_LOOKUP_NEXT_PUNT] = "ip6-punt",				\
+    [IP_LOOKUP_NEXT_LOCAL] = "ip6-local",			\
+    [IP_LOOKUP_NEXT_ARP] = "ip6-discover-neighbor",		\
+    [IP_LOOKUP_NEXT_GLEAN] = "ip6-glean",			\
+    [IP_LOOKUP_NEXT_REWRITE] = "ip6-rewrite",			\
+    [IP_LOOKUP_NEXT_MCAST] = "ip6-rewrite-mcast",		\
+    [IP_LOOKUP_NEXT_MIDCHAIN] = "ip6-midchain",			\
+    [IP_LOOKUP_NEXT_ICMP_ERROR] = "ip6-icmp-error",		\
+    [IP6_LOOKUP_NEXT_HOP_BY_HOP] = "ip6-hop-by-hop",		\
+    [IP6_LOOKUP_NEXT_ADD_HOP_BY_HOP] = "ip6-add-hop-by-hop",	\
+    [IP6_LOOKUP_NEXT_POP_HOP_BY_HOP] = "ip6-pop-hop-by-hop",	\
+}
+
+/**
+ * Forward delcartion
+ */
+struct ip_adjacency_t_;
+
+/**
+ * @brief A function type for post-rewrite fixups on midchain adjacency
+ */
+typedef void (*adj_midchain_fixup_t) (vlib_main_t * vm,
+				      struct ip_adjacency_t_ * adj,
+				      vlib_buffer_t * b0);
+
+/**
+ * @brief Flags on an IP adjacency
+ */
+typedef enum ip_adjacency_flags_t_
+{
+    ADJ_FLAG_NONE = 0,
+
+    /**
+     * Currently a sync walk is active. Used to prevent re-entrant walking
+     */
+    ADJ_FLAG_SYNC_WALK_ACTIVE = (1 << 0),
+
+    /**
+     * Packets TX through the midchain do not increment the interface
+     * counters. This should be used when the adj is associated with an L2
+     * interface and that L2 interface is in a bridege domain. In that case
+     * the packet will have traversed the interface's TX node, and hence have
+     * been counted, before it traverses ths midchain
+     */
+    ADJ_FLAG_MIDCHAIN_NO_COUNT = (1 << 1),
+}  __attribute__ ((packed)) adj_flags_t;
+
+/**
+ * @brief IP unicast adjacency.
+ *  @note cache aligned.
+ *
+ * An adjacency is a represenation of a peer on a particular link.
+ */
+typedef struct ip_adjacency_t_
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+
+  /**
+   * Linkage into the FIB node grpah. First member since this type
+   * has 8 byte alignment requirements.
+   */
+  fib_node_t ia_node;
+
+  /**
+   * Next hop after ip4-lookup.
+   *  This is not accessed in the rewrite nodes.
+   * 1-bytes
+   */
+  ip_lookup_next_t lookup_next_index;
+
+  /**
+   * link/ether-type
+   * 1 bytes
+   */
+  vnet_link_t ia_link;
+
+  /**
+   * The protocol of the neighbor/peer. i.e. the protocol with
+   * which to interpret the 'next-hop' attirbutes of the sub-types.
+   * 1-btyes
+   */
+  fib_protocol_t ia_nh_proto;
+
+  /**
+   * Flags on the adjacency
+   * 1-bytes
+   */
+  adj_flags_t ia_flags;
+
+  union
+  {
+    /**
+     * IP_LOOKUP_NEXT_ARP/IP_LOOKUP_NEXT_REWRITE
+     *
+     * neighbour adjacency sub-type;
+     */
+    struct
+    {
+      ip46_address_t next_hop;
+    } nbr;
+      /**
+       * IP_LOOKUP_NEXT_MIDCHAIN
+       *
+       * A nbr adj that is also recursive. Think tunnels.
+       * A nbr adj can transition to be of type MDICHAIN
+       * so be sure to leave the two structs with the next_hop
+       * fields aligned.
+       */
+    struct
+    {
+      /**
+       * The recursive next-hop.
+       *  This field MUST be at the same memory location as
+       *   sub_type.nbr.next_hop
+       */
+      ip46_address_t next_hop;
+      /**
+       * The next DPO to use
+       */
+      dpo_id_t next_dpo;
+      /**
+       * A function to perform the post-rewrite fixup
+       */
+      adj_midchain_fixup_t fixup_func;
+    } midchain;
+    /**
+     * IP_LOOKUP_NEXT_GLEAN
+     *
+     * Glean the address to ARP for from the packet's destination.
+     * Technically these aren't adjacencies, i.e. they are not a
+     * representation of a peer. One day we might untangle this coupling
+     * and use a new Glean DPO.
+     */
+    struct
+    {
+      ip46_address_t receive_addr;
+    } glean;
+  } sub_type;
+
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
+
+  /* Rewrite in second/third cache lines */
+  vnet_declare_rewrite (VLIB_BUFFER_PRE_DATA_SIZE);
+} ip_adjacency_t;
+
+STATIC_ASSERT ((STRUCT_OFFSET_OF (ip_adjacency_t, cacheline0) == 0),
+	       "IP adjacency cachline 0 is not offset");
+STATIC_ASSERT ((STRUCT_OFFSET_OF (ip_adjacency_t, cacheline1) ==
+		CLIB_CACHE_LINE_BYTES),
+	       "IP adjacency cachline 1 is more than one cachline size offset");
 
 /**
  * @brief
diff --git a/src/vnet/adj/adj_midchain.c b/src/vnet/adj/adj_midchain.c
index 55b5e44b..e8087f08 100644
--- a/src/vnet/adj/adj_midchain.c
+++ b/src/vnet/adj/adj_midchain.c
@@ -384,6 +384,14 @@ adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
     return (arc);
 }
 
+static u32
+adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj)
+{
+    return ((adj->ia_flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
+            adj_midchain_tx_no_count_node.index :
+            adj_midchain_tx_node.index);
+}
+
 /**
  * adj_nbr_midchain_update_rewrite
  *
@@ -394,12 +402,12 @@ adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
 void
 adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
 				 adj_midchain_fixup_t fixup,
-				 adj_midchain_flag_t flags,
+				 adj_flags_t flags,
 				 u8 *rewrite)
 {
+    u32 feature_index, tx_node;
     ip_adjacency_t *adj;
     u8 arc_index;
-    u32 feature_index;
 
     ASSERT(ADJ_INDEX_INVALID != adj_index);
 
@@ -416,15 +424,14 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
     ASSERT(NULL != rewrite);
 
     adj->sub_type.midchain.fixup_func = fixup;
+    adj->ia_flags |= flags;
 
     arc_index = adj_midchain_get_feature_arc_index_for_link_type (adj);
-    feature_index = (flags & ADJ_MIDCHAIN_FLAG_NO_COUNT) ?
+    feature_index = (flags & ADJ_FLAG_MIDCHAIN_NO_COUNT) ?
                     adj_midchain_tx_no_count_feature_node[adj->ia_link] :
                     adj_midchain_tx_feature_node[adj->ia_link];
 
-    adj->sub_type.midchain.tx_function_node = (flags & ADJ_MIDCHAIN_FLAG_NO_COUNT) ?
-                                               adj_midchain_tx_no_count_node.index :
-                                               adj_midchain_tx_node.index;
+    tx_node = adj_nbr_midchain_get_tx_node(adj);
 
     vnet_feature_enable_disable_with_index (arc_index, feature_index,
 					    adj->rewrite_header.sw_if_index,
@@ -437,7 +444,7 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
      * node are any output features, then the midchain-tx.  from there we
      * need to get to the stacked child's node.
      */
-    dpo_stack_from_node(adj->sub_type.midchain.tx_function_node,
+    dpo_stack_from_node(tx_node,
 			&adj->sub_type.midchain.next_dpo,
 			drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
 
@@ -447,7 +454,7 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
     adj_nbr_update_rewrite_internal(adj,
 				    IP_LOOKUP_NEXT_MIDCHAIN,
 				    adj_get_midchain_node(adj->ia_link),
-				    adj->sub_type.midchain.tx_function_node,
+				    tx_node,
 				    rewrite);
 }
 
@@ -491,7 +498,7 @@ adj_nbr_midchain_stack (adj_index_t adj_index,
 
     ASSERT(IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index);
 
-    dpo_stack_from_node(adj->sub_type.midchain.tx_function_node,
+    dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj),
 			&adj->sub_type.midchain.next_dpo,
 			next);
 }
diff --git a/src/vnet/adj/adj_midchain.h b/src/vnet/adj/adj_midchain.h
index ae414aea..27ca1d33 100644
--- a/src/vnet/adj/adj_midchain.h
+++ b/src/vnet/adj/adj_midchain.h
@@ -24,26 +24,6 @@
 
 #include <vnet/adj/adj.h>
 
-/**
- * @brief Flags controlling the midchain adjacency
- */
-typedef enum adj_midchain_flag_t_
-{
-    /**
-     * No flags
-     */
-    ADJ_MIDCHAIN_FLAG_NONE = 0,
-
-    /**
-     * Packets TX through the midchain do not increment the interface
-     * counters. This should be used when the adj is associated with an L2
-     * interface and that L2 interface is in a bridege domain. In that case
-     * the packet will have traversed the interface's TX node, and hence have
-     * been counted, before it traverses ths midchain
-     */
-    ADJ_MIDCHAIN_FLAG_NO_COUNT = (1 << 0),
-} adj_midchain_flag_t;
-
 /**
  * @brief
  *  Convert an existing neighbour adjacency into a midchain
@@ -60,7 +40,7 @@ typedef enum adj_midchain_flag_t_
  */
 extern void adj_nbr_midchain_update_rewrite(adj_index_t adj_index,
 					    adj_midchain_fixup_t fixup,
-					    adj_midchain_flag_t flags,
+					    adj_flags_t flags,
 					    u8 *rewrite);
 
 /**
diff --git a/src/vnet/adj/adj_nbr.c b/src/vnet/adj/adj_nbr.c
index 072abd03..ddacb030 100644
--- a/src/vnet/adj/adj_nbr.c
+++ b/src/vnet/adj/adj_nbr.c
@@ -333,7 +333,7 @@ adj_nbr_update_rewrite (adj_index_t adj_index,
  */
 void
 adj_nbr_update_rewrite_internal (ip_adjacency_t *adj,
-				 u32 adj_next_index,
+				 ip_lookup_next_t adj_next_index,
 				 u32 this_node,
 				 u32 next_node,
 				 u8 *rewrite)
@@ -367,7 +367,7 @@ adj_nbr_update_rewrite_internal (ip_adjacency_t *adj,
     if (ADJ_INDEX_INVALID != walk_ai)
     {
         walk_adj = adj_get(walk_ai);
-        if (IP_ADJ_SYNC_WALK_ACTIVE & walk_adj->ia_flags)
+        if (ADJ_FLAG_SYNC_WALK_ACTIVE & walk_adj->ia_flags)
         {
             do_walk = 0;
         }
@@ -376,7 +376,7 @@ adj_nbr_update_rewrite_internal (ip_adjacency_t *adj,
             /*
              * Prevent re-entrant walk of the same adj
              */
-            walk_adj->ia_flags |= IP_ADJ_SYNC_WALK_ACTIVE;
+            walk_adj->ia_flags |= ADJ_FLAG_SYNC_WALK_ACTIVE;
             do_walk = 1;
         }
     }
@@ -502,7 +502,7 @@ adj_nbr_update_rewrite_internal (ip_adjacency_t *adj,
      */
     if (do_walk)
     {
-        walk_adj->ia_flags &= ~IP_ADJ_SYNC_WALK_ACTIVE;
+        walk_adj->ia_flags &= ~ADJ_FLAG_SYNC_WALK_ACTIVE;
     }
 
     adj_unlock(adj_get_index(adj));
diff --git a/src/vnet/adj/rewrite.c b/src/vnet/adj/rewrite.c
new file mode 100644
index 00000000..47fb74df
--- /dev/null
+++ b/src/vnet/adj/rewrite.c
@@ -0,0 +1,234 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * rewrite.c: packet rewrite
+ *
+ * Copyright (c) 2008 Eliot Dresselhaus
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <vnet/vnet.h>
+#include <vnet/ip/lookup.h>
+
+void
+vnet_rewrite_copy_slow_path (vnet_rewrite_data_t * p0,
+			     vnet_rewrite_data_t * rw0,
+			     word n_left, uword most_likely_size)
+{
+  uword n_done =
+    round_pow2 (most_likely_size, sizeof (rw0[0])) / sizeof (rw0[0]);
+
+  p0 -= n_done;
+  rw0 -= n_done;
+
+  /* As we enter the cleanup loop, p0 and rw0 point to the last chunk written
+     by the fast path. Hence, the constant 1, which the
+     vnet_rewrite_copy_one macro renders as p0[-1] = rw0[-1]. */
+
+  while (n_left > 0)
+    {
+      vnet_rewrite_copy_one (p0, rw0, 1);
+      p0--;
+      rw0--;
+      n_left--;
+    }
+}
+
+u8 *
+format_vnet_rewrite (u8 * s, va_list * args)
+{
+  vnet_rewrite_header_t *rw = va_arg (*args, vnet_rewrite_header_t *);
+  u32 max_data_bytes = va_arg (*args, u32);
+  CLIB_UNUSED (uword indent) = va_arg (*args, u32);
+  vnet_main_t *vnm = vnet_get_main ();
+
+  if (rw->sw_if_index != ~0)
+    {
+      vnet_sw_interface_t *si;
+      si = vnet_get_sw_interface_safe (vnm, rw->sw_if_index);
+      if (NULL != si)
+	s = format (s, "%U: ", format_vnet_sw_interface_name, vnm, si);
+      else
+	s = format (s, "DELETED:%d", rw->sw_if_index);
+    }
+
+  /* Format rewrite string. */
+  if (rw->data_bytes > 0)
+
+    s = format (s, "%U",
+		format_hex_bytes,
+		rw->data + max_data_bytes - rw->data_bytes, rw->data_bytes);
+
+  return s;
+}
+
+u32
+vnet_tx_node_index_for_sw_interface (vnet_main_t * vnm, u32 sw_if_index)
+{
+  vnet_hw_interface_t *hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
+  return (hw->output_node_index);
+}
+
+void
+vnet_rewrite_init (vnet_main_t * vnm,
+		   u32 sw_if_index,
+		   u32 this_node, u32 next_node, vnet_rewrite_header_t * rw)
+{
+  rw->sw_if_index = sw_if_index;
+  rw->next_index = vlib_node_add_next (vnm->vlib_main, this_node, next_node);
+  rw->max_l3_packet_bytes =
+    vnet_sw_interface_get_mtu (vnm, sw_if_index, VLIB_TX);
+}
+
+void
+vnet_rewrite_for_sw_interface (vnet_main_t * vnm,
+			       vnet_link_t link_type,
+			       u32 sw_if_index,
+			       u32 node_index,
+			       void *dst_address,
+			       vnet_rewrite_header_t * rw,
+			       u32 max_rewrite_bytes)
+{
+
+  vnet_hw_interface_t *hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
+  vnet_hw_interface_class_t *hc =
+    vnet_get_hw_interface_class (vnm, hw->hw_class_index);
+  u8 *rewrite = NULL;
+
+  vnet_rewrite_init (vnm, sw_if_index, node_index,
+		     vnet_tx_node_index_for_sw_interface (vnm, sw_if_index),
+		     rw);
+
+  ASSERT (hc->build_rewrite);
+  rewrite = hc->build_rewrite (vnm, sw_if_index, link_type, dst_address);
+
+  ASSERT (vec_len (rewrite) < max_rewrite_bytes);
+  vnet_rewrite_set_data_internal (rw, max_rewrite_bytes, rewrite,
+				  vec_len (rewrite));
+  vec_free (rewrite);
+}
+
+void
+vnet_rewrite_for_tunnel (vnet_main_t * vnm,
+			 u32 tx_sw_if_index,
+			 u32 rewrite_node_index,
+			 u32 post_rewrite_node_index,
+			 vnet_rewrite_header_t * rw,
+			 u8 * rewrite_data, u32 rewrite_length)
+{
+  ip_adjacency_t *adj = 0;
+  /*
+   * Installed into vnet_buffer(b)->sw_if_index[VLIB_TX] e.g.
+   * by ip4_rewrite_inline. If the post-rewrite node injects into
+   * ipX-forward, this will be interpreted as a FIB number.
+   */
+  rw->sw_if_index = tx_sw_if_index;
+  rw->next_index = vlib_node_add_next (vnm->vlib_main, rewrite_node_index,
+				       post_rewrite_node_index);
+  rw->max_l3_packet_bytes = (u16) ~ 0;	/* we can't know at this point */
+
+  ASSERT (rewrite_length < sizeof (adj->rewrite_data));
+  /* Leave room for ethernet + VLAN tag */
+  vnet_rewrite_set_data_internal (rw, sizeof (adj->rewrite_data),
+				  rewrite_data, rewrite_length);
+}
+
+void
+serialize_vnet_rewrite (serialize_main_t * m, va_list * va)
+{
+  vnet_rewrite_header_t *rw = va_arg (*va, vnet_rewrite_header_t *);
+  u32 max_data_bytes = va_arg (*va, u32);
+  u8 *p;
+
+  serialize_integer (m, rw->sw_if_index, sizeof (rw->sw_if_index));
+  serialize_integer (m, rw->data_bytes, sizeof (rw->data_bytes));
+  serialize_integer (m, rw->max_l3_packet_bytes,
+		     sizeof (rw->max_l3_packet_bytes));
+  p = serialize_get (m, rw->data_bytes);
+  clib_memcpy (p, vnet_rewrite_get_data_internal (rw, max_data_bytes),
+	       rw->data_bytes);
+}
+
+void
+unserialize_vnet_rewrite (serialize_main_t * m, va_list * va)
+{
+  vnet_rewrite_header_t *rw = va_arg (*va, vnet_rewrite_header_t *);
+  u32 max_data_bytes = va_arg (*va, u32);
+  u8 *p;
+
+  /* It is up to user to fill these in. */
+  rw->next_index = ~0;
+
+  unserialize_integer (m, &rw->sw_if_index, sizeof (rw->sw_if_index));
+  unserialize_integer (m, &rw->data_bytes, sizeof (rw->data_bytes));
+  unserialize_integer (m, &rw->max_l3_packet_bytes,
+		       sizeof (rw->max_l3_packet_bytes));
+  p = unserialize_get (m, rw->data_bytes);
+  clib_memcpy (vnet_rewrite_get_data_internal (rw, max_data_bytes), p,
+	       rw->data_bytes);
+}
+
+u8 *
+vnet_build_rewrite_for_sw_interface (vnet_main_t * vnm,
+				     u32 sw_if_index,
+				     vnet_link_t link_type,
+				     const void *dst_address)
+{
+  vnet_hw_interface_t *hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
+  vnet_hw_interface_class_t *hc =
+    vnet_get_hw_interface_class (vnm, hw->hw_class_index);
+
+  ASSERT (hc->build_rewrite);
+  return (hc->build_rewrite (vnm, sw_if_index, link_type, dst_address));
+}
+
+
+void
+vnet_update_adjacency_for_sw_interface (vnet_main_t * vnm,
+					u32 sw_if_index, u32 ai)
+{
+  vnet_hw_interface_t *hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
+  vnet_hw_interface_class_t *hc =
+    vnet_get_hw_interface_class (vnm, hw->hw_class_index);
+
+  ASSERT (hc->update_adjacency);
+  hc->update_adjacency (vnm, sw_if_index, ai);
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/adj/rewrite.h b/src/vnet/adj/rewrite.h
new file mode 100644
index 00000000..1dea72f5
--- /dev/null
+++ b/src/vnet/adj/rewrite.h
@@ -0,0 +1,350 @@
+/*
+ * Copyright (c) 2015 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * rewrite.h: packet rewrite
+ *
+ * Copyright (c) 2008 Eliot Dresselhaus
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef included_vnet_rewrite_h
+#define included_vnet_rewrite_h
+
+#include <vlib/vlib.h>
+#include <vnet/l3_types.h>
+
+/* Consider using vector types for speed? */
+typedef uword vnet_rewrite_data_t;
+
+/**
+ * Flags associated with the rewrite/adjacency
+ */
+typedef enum vnet_rewrite_flags_t_
+{
+  /**
+   * This adjacency/interface has output features configured
+   */
+  VNET_REWRITE_HAS_FEATURES = (1 << 0),
+} __attribute__ ((packed)) vnet_rewrite_flags_t;
+
+/* *INDENT-OFF* */
+typedef CLIB_PACKED (struct {
+  /* Interface to mark re-written packets with. */
+  u32 sw_if_index;
+
+  /* Next node to feed after packet rewrite is done. */
+  u16 next_index;
+
+  /* Number of bytes in rewrite data. */
+  u16 data_bytes;
+
+  /* Max packet size layer 3 (MTU) for output interface.
+     Used for MTU check after packet rewrite. */
+  u16 max_l3_packet_bytes;
+
+  /* Data-plane flags on the adjacency/rewrite */
+  vnet_rewrite_flags_t flags;
+
+  /* When dynamically writing a multicast destination L2 addresss
+   * this is the offset from the IP address at which to write in the
+   * IP->MAC address translation.
+   */
+  u8 dst_mcast_offset;
+
+  /* The mask to apply to the lower 4 bytes of the IP address before ORing
+   * into the destinaiton MAC address */
+  u32 dst_mcast_mask;
+
+  /* Rewrite string starting at end and going backwards. */
+  u8 data[0];
+}) vnet_rewrite_header_t;
+/* *INDENT-ON* */
+
+/**
+ * At 16 bytes of rewrite herader we have enought space left for a IPv6
+ * (40 bytes) + LISP-GPE (8 bytes) in the cache line
+ */
+STATIC_ASSERT (sizeof (vnet_rewrite_header_t) <= 16,
+	       "Rewrite header too big");
+
+/*
+  Helper macro for declaring rewrite string w/ given max-size.
+
+  Typical usage:
+    typedef struct {
+      //
+      int a, b;
+
+      // Total adjacency is 64 bytes.
+      vnet_rewrite_declare(64 - 2*sizeof(int)) rw;
+    } my_adjacency_t;
+*/
+#define vnet_declare_rewrite(total_bytes)				\
+struct {								\
+  vnet_rewrite_header_t rewrite_header;  			        \
+									\
+  u8 rewrite_data[(total_bytes) - sizeof (vnet_rewrite_header_t)];	\
+}
+
+always_inline void
+vnet_rewrite_clear_data_internal (vnet_rewrite_header_t * rw, int max_size)
+{
+  /* Sanity check values carefully for this memset operation */
+  ASSERT ((max_size > 0) && (max_size < VLIB_BUFFER_PRE_DATA_SIZE));
+
+  rw->data_bytes = 0;
+  memset (rw->data, 0xfe, max_size);
+}
+
+always_inline void
+vnet_rewrite_set_data_internal (vnet_rewrite_header_t * rw,
+				int max_size, void *data, int data_bytes)
+{
+  /* Sanity check values carefully for this memset operation */
+  ASSERT ((max_size > 0) && (max_size < VLIB_BUFFER_PRE_DATA_SIZE));
+  ASSERT ((data_bytes >= 0) && (data_bytes < max_size));
+
+  rw->data_bytes = data_bytes;
+  clib_memcpy (rw->data + max_size - data_bytes, data, data_bytes);
+  memset (rw->data, 0xfe, max_size - data_bytes);
+}
+
+#define vnet_rewrite_set_data(rw,data,data_bytes)		\
+  vnet_rewrite_set_data_internal (&((rw).rewrite_header),	\
+				  sizeof ((rw).rewrite_data),	\
+				  (data),			\
+				  (data_bytes))
+
+always_inline void *
+vnet_rewrite_get_data_internal (vnet_rewrite_header_t * rw, int max_size)
+{
+  ASSERT (rw->data_bytes <= max_size);
+  return rw->data + max_size - rw->data_bytes;
+}
+
+#define vnet_rewrite_get_data(rw) \
+  vnet_rewrite_get_data_internal (&((rw).rewrite_header), sizeof ((rw).rewrite_data))
+
+always_inline void
+vnet_rewrite_copy_one (vnet_rewrite_data_t * p0, vnet_rewrite_data_t * rw0,
+		       int i)
+{
+  p0[-i] = rw0[-i];
+}
+
+void vnet_rewrite_copy_slow_path (vnet_rewrite_data_t * p0,
+				  vnet_rewrite_data_t * rw0,
+				  word n_left, uword most_likely_size);
+
+/* *INDENT-OFF* */
+typedef CLIB_PACKED (struct {
+  u64 a;
+  u32 b;
+  u16 c;
+}) eh_copy_t;
+/* *INDENT-ON* */
+
+always_inline void
+_vnet_rewrite_one_header (vnet_rewrite_header_t * h0,
+			  void *packet0, int max_size, int most_likely_size)
+{
+  vnet_rewrite_data_t *p0 = packet0;
+  vnet_rewrite_data_t *rw0 = (vnet_rewrite_data_t *) (h0->data + max_size);
+  word n_left0;
+
+  /* 0xfefe => poisoned adjacency => crash */
+  ASSERT (h0->data_bytes != 0xfefe);
+
+  if (PREDICT_TRUE (h0->data_bytes == sizeof (eh_copy_t)))
+    {
+      eh_copy_t *s, *d;
+      s = (eh_copy_t *) (h0->data + max_size - sizeof (eh_copy_t));
+      d = (eh_copy_t *) (((u8 *) packet0) - sizeof (eh_copy_t));
+      clib_memcpy (d, s, sizeof (eh_copy_t));
+      return;
+    }
+
+
+#define _(i)								\
+  do {									\
+    if (most_likely_size > ((i)-1)*sizeof (vnet_rewrite_data_t))	\
+      vnet_rewrite_copy_one (p0, rw0, (i));				\
+  } while (0)
+
+  _(4);
+  _(3);
+  _(2);
+  _(1);
+
+#undef _
+
+  n_left0 = (int)
+    (((int) h0->data_bytes - most_likely_size) + (sizeof (rw0[0]) - 1))
+    / (int) sizeof (rw0[0]);
+  if (PREDICT_FALSE (n_left0 > 0))
+    vnet_rewrite_copy_slow_path (p0, rw0, n_left0, most_likely_size);
+}
+
+always_inline void
+_vnet_rewrite_two_headers (vnet_rewrite_header_t * h0,
+			   vnet_rewrite_header_t * h1,
+			   void *packet0,
+			   void *packet1, int max_size, int most_likely_size)
+{
+  vnet_rewrite_data_t *p0 = packet0;
+  vnet_rewrite_data_t *p1 = packet1;
+  vnet_rewrite_data_t *rw0 = (vnet_rewrite_data_t *) (h0->data + max_size);
+  vnet_rewrite_data_t *rw1 = (vnet_rewrite_data_t *) (h1->data + max_size);
+  word n_left0, n_left1;
+  int slow_path;
+
+  /* 0xfefe => poisoned adjacency => crash */
+  ASSERT (h0->data_bytes != 0xfefe);
+  ASSERT (h1->data_bytes != 0xfefe);
+
+  /* Arithmetic calculation: bytes0 == bytes1 == 14 */
+  slow_path = h0->data_bytes ^ h1->data_bytes;
+  slow_path += h0->data_bytes ^ sizeof (eh_copy_t);
+
+  if (PREDICT_TRUE (slow_path == 0))
+    {
+      eh_copy_t *s0, *d0, *s1, *d1;
+      s0 = (eh_copy_t *) (h0->data + max_size - sizeof (eh_copy_t));
+      d0 = (eh_copy_t *) (((u8 *) packet0) - sizeof (eh_copy_t));
+      clib_memcpy (d0, s0, sizeof (eh_copy_t));
+      s1 = (eh_copy_t *) (h1->data + max_size - sizeof (eh_copy_t));
+      d1 = (eh_copy_t *) (((u8 *) packet1) - sizeof (eh_copy_t));
+      clib_memcpy (d1, s1, sizeof (eh_copy_t));
+      return;
+    }
+
+#define _(i)								\
+  do {									\
+    if (most_likely_size > ((i)-1)*sizeof (vnet_rewrite_data_t))	\
+      {									\
+	vnet_rewrite_copy_one (p0, rw0, (i));				\
+	vnet_rewrite_copy_one (p1, rw1, (i));				\
+      }									\
+  } while (0)
+
+  _(4);
+  _(3);
+  _(2);
+  _(1);
+
+#undef _
+
+  n_left0 = (int)
+    (((int) h0->data_bytes - most_likely_size) + (sizeof (rw0[0]) - 1))
+    / (int) sizeof (rw0[0]);
+  n_left1 = (int)
+    (((int) h1->data_bytes - most_likely_size) + (sizeof (rw1[0]) - 1))
+    / (int) sizeof (rw1[0]);
+
+  if (PREDICT_FALSE (n_left0 > 0 || n_left1 > 0))
+    {
+      vnet_rewrite_copy_slow_path (p0, rw0, n_left0, most_likely_size);
+      vnet_rewrite_copy_slow_path (p1, rw1, n_left1, most_likely_size);
+    }
+}
+
+#define vnet_rewrite_one_header(rw0,p0,most_likely_size)	\
+  _vnet_rewrite_one_header (&((rw0).rewrite_header), (p0),	\
+			    sizeof ((rw0).rewrite_data),	\
+			    (most_likely_size))
+
+#define vnet_rewrite_two_headers(rw0,rw1,p0,p1,most_likely_size)	\
+  _vnet_rewrite_two_headers (&((rw0).rewrite_header), &((rw1).rewrite_header), \
+			     (p0), (p1),				\
+			     sizeof ((rw0).rewrite_data),		\
+			     (most_likely_size))
+
+always_inline void
+_vnet_fixup_one_header (vnet_rewrite_header_t * h0,
+			u8 * addr, u32 addr_len, u8 * packet0)
+{
+  if (PREDICT_TRUE (h0->dst_mcast_mask))
+    {
+      /* location to write to in the packet */
+      u8 *p0 = packet0 - h0->dst_mcast_offset;
+      u32 *p1 = (u32 *) p0;
+      /* location to copy from in the L3 dest address */
+      u32 *a0 = (u32 *) (addr + addr_len - sizeof (h0->dst_mcast_mask));
+
+      *p1 |= (*a0 & h0->dst_mcast_mask);
+    }
+}
+
+#define vnet_fixup_one_header(rw0,addr,p0)              \
+  _vnet_fixup_one_header (&((rw0).rewrite_header),                      \
+                          (u8*)(addr), sizeof((*addr)),                 \
+                          (u8*)(p0))
+
+#define VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST ((void *) 0)
+/** Deprecated */
+void vnet_rewrite_for_sw_interface (struct vnet_main_t *vnm,
+				    vnet_link_t packet_type,
+				    u32 sw_if_index,
+				    u32 node_index,
+				    void *dst_address,
+				    vnet_rewrite_header_t * rw,
+				    u32 max_rewrite_bytes);
+
+u32 vnet_tx_node_index_for_sw_interface (struct vnet_main_t *vnm,
+					 u32 sw_if_index);
+
+void vnet_rewrite_init (struct vnet_main_t *vnm,
+			u32 sw_if_index,
+			u32 this_node,
+			u32 next_node, vnet_rewrite_header_t * rw);
+
+u8 *vnet_build_rewrite_for_sw_interface (struct vnet_main_t *vnm,
+					 u32 sw_if_index,
+					 vnet_link_t packet_type,
+					 const void *dst_address);
+void vnet_update_adjacency_for_sw_interface (struct vnet_main_t *vnm,
+					     u32 sw_if_index, u32 ai);
+
+format_function_t format_vnet_rewrite;
+
+serialize_function_t serialize_vnet_rewrite, unserialize_vnet_rewrite;
+
+#endif /* included_vnet_rewrite_h */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c
index 09ecd0cc..ee757505 100644
--- a/src/vnet/ethernet/arp.c
+++ b/src/vnet/ethernet/arp.c
@@ -507,7 +507,6 @@ arp_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai)
     case IP_LOOKUP_NEXT_PUNT:
     case IP_LOOKUP_NEXT_LOCAL:
     case IP_LOOKUP_NEXT_REWRITE:
-    case IP_LOOKUP_NEXT_LOAD_BALANCE:
     case IP_LOOKUP_NEXT_MIDCHAIN:
     case IP_LOOKUP_NEXT_ICMP_ERROR:
     case IP_LOOKUP_N_NEXT:
diff --git a/src/vnet/gre/gre.c b/src/vnet/gre/gre.c
index 29d0f0fe..3d381388 100644
--- a/src/vnet/gre/gre.c
+++ b/src/vnet/gre/gre.c
@@ -242,8 +242,8 @@ gre_update_adj (vnet_main_t * vnm,
 {
     adj_nbr_midchain_update_rewrite (ai, gre_fixup, 
                                      (VNET_LINK_ETHERNET == adj_get_link_type (ai) ?
-                                      ADJ_MIDCHAIN_FLAG_NO_COUNT :
-                                      ADJ_MIDCHAIN_FLAG_NONE),
+                                      ADJ_FLAG_MIDCHAIN_NO_COUNT :
+                                      ADJ_FLAG_NONE),
 				     gre_build_rewrite(vnm, sw_if_index,
 						       adj_get_link_type(ai),
 						       NULL));
diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c
index b1a03546..5d1fb6f8 100644
--- a/src/vnet/ip/ip6_neighbor.c
+++ b/src/vnet/ip/ip6_neighbor.c
@@ -557,7 +557,6 @@ ip6_ethernet_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai)
     case IP_LOOKUP_NEXT_PUNT:
     case IP_LOOKUP_NEXT_LOCAL:
     case IP_LOOKUP_NEXT_REWRITE:
-    case IP_LOOKUP_NEXT_LOAD_BALANCE:
     case IP_LOOKUP_NEXT_MIDCHAIN:
     case IP_LOOKUP_NEXT_ICMP_ERROR:
     case IP_LOOKUP_N_NEXT:
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index dabfa012..95f36d41 100644
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -188,13 +188,6 @@ VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip_sw_interface_add_del);
 void
 ip_lookup_init (ip_lookup_main_t * lm, u32 is_ip6)
 {
-  /* ensure that adjacency is cacheline aligned and sized */
-  STATIC_ASSERT (STRUCT_OFFSET_OF (ip_adjacency_t, cacheline0) == 0,
-		 "Cache line marker must be 1st element in struct");
-  STATIC_ASSERT (STRUCT_OFFSET_OF (ip_adjacency_t, cacheline1) ==
-		 CLIB_CACHE_LINE_BYTES,
-		 "Data in cache line 0 is bigger than cache line size");
-
   /* Preallocate three "special" adjacencies */
   lm->adjacency_heap = adj_pool;
 
@@ -251,7 +244,8 @@ format_ip_flow_hash_config (u8 * s, va_list * args)
 u8 *
 format_ip_lookup_next (u8 * s, va_list * args)
 {
-  ip_lookup_next_t n = va_arg (*args, ip_lookup_next_t);
+  /* int promotion of ip_lookup_next_t */
+  ip_lookup_next_t n = va_arg (*args, int);
   char *t = 0;
 
   switch (n)
diff --git a/src/vnet/ip/lookup.h b/src/vnet/ip/lookup.h
index f76ddb37..662a1cc0 100644
--- a/src/vnet/ip/lookup.h
+++ b/src/vnet/ip/lookup.h
@@ -53,93 +53,9 @@
 #include <vnet/ip/ip4_packet.h>
 #include <vnet/ip/ip6_packet.h>
 #include <vnet/fib/fib_node.h>
+#include <vnet/adj/adj.h>
 #include <vnet/dpo/dpo.h>
 #include <vnet/feature/feature.h>
-#include <vnet/rewrite.h>
-
-/** @brief Common (IP4/IP6) next index stored in adjacency. */
-typedef enum
-{
-  /** Adjacency to drop this packet. */
-  IP_LOOKUP_NEXT_DROP,
-  /** Adjacency to punt this packet. */
-  IP_LOOKUP_NEXT_PUNT,
-
-  /** This packet is for one of our own IP addresses. */
-  IP_LOOKUP_NEXT_LOCAL,
-
-  /** This packet matches an "incomplete adjacency" and packets
-     need to be passed to ARP to find rewrite string for
-     this destination. */
-  IP_LOOKUP_NEXT_ARP,
-
-  /** This packet matches an "interface route" and packets
-     need to be passed to ARP to find rewrite string for
-     this destination. */
-  IP_LOOKUP_NEXT_GLEAN,
-
-  /** This packet is to be rewritten and forwarded to the next
-     processing node.  This is typically the output interface but
-     might be another node for further output processing. */
-  IP_LOOKUP_NEXT_REWRITE,
-
-  /** This packets follow a load-balance */
-  IP_LOOKUP_NEXT_LOAD_BALANCE,
-
-  /** This packets follow a mid-chain adjacency */
-  IP_LOOKUP_NEXT_MIDCHAIN,
-
-  /** This packets needs to go to ICMP error */
-  IP_LOOKUP_NEXT_ICMP_ERROR,
-
-  /** Multicast Adjacency. */
-  IP_LOOKUP_NEXT_MCAST,
-
-  IP_LOOKUP_N_NEXT,
-} ip_lookup_next_t;
-
-typedef enum
-{
-  IP4_LOOKUP_N_NEXT = IP_LOOKUP_N_NEXT,
-} ip4_lookup_next_t;
-
-typedef enum
-{
-  /* Hop-by-hop header handling */
-  IP6_LOOKUP_NEXT_HOP_BY_HOP = IP_LOOKUP_N_NEXT,
-  IP6_LOOKUP_NEXT_ADD_HOP_BY_HOP,
-  IP6_LOOKUP_NEXT_POP_HOP_BY_HOP,
-  IP6_LOOKUP_N_NEXT,
-} ip6_lookup_next_t;
-
-#define IP4_LOOKUP_NEXT_NODES {					\
-    [IP_LOOKUP_NEXT_DROP] = "ip4-drop",				\
-    [IP_LOOKUP_NEXT_PUNT] = "ip4-punt",				\
-    [IP_LOOKUP_NEXT_LOCAL] = "ip4-local",			\
-    [IP_LOOKUP_NEXT_ARP] = "ip4-arp",				\
-    [IP_LOOKUP_NEXT_GLEAN] = "ip4-glean",			\
-    [IP_LOOKUP_NEXT_REWRITE] = "ip4-rewrite",    		\
-    [IP_LOOKUP_NEXT_MCAST] = "ip4-rewrite-mcast",	        \
-    [IP_LOOKUP_NEXT_MIDCHAIN] = "ip4-midchain",		        \
-    [IP_LOOKUP_NEXT_LOAD_BALANCE] = "ip4-load-balance",		\
-    [IP_LOOKUP_NEXT_ICMP_ERROR] = "ip4-icmp-error",		\
-}
-
-#define IP6_LOOKUP_NEXT_NODES {					\
-    [IP_LOOKUP_NEXT_DROP] = "ip6-drop",				\
-    [IP_LOOKUP_NEXT_PUNT] = "ip6-punt",				\
-    [IP_LOOKUP_NEXT_LOCAL] = "ip6-local",			\
-    [IP_LOOKUP_NEXT_ARP] = "ip6-discover-neighbor",		\
-    [IP_LOOKUP_NEXT_GLEAN] = "ip6-glean",			\
-    [IP_LOOKUP_NEXT_REWRITE] = "ip6-rewrite",			\
-    [IP_LOOKUP_NEXT_MCAST] = "ip6-rewrite-mcast",		\
-    [IP_LOOKUP_NEXT_MIDCHAIN] = "ip6-midchain",			\
-    [IP_LOOKUP_NEXT_LOAD_BALANCE] = "ip6-load-balance",		\
-    [IP_LOOKUP_NEXT_ICMP_ERROR] = "ip6-icmp-error",		\
-    [IP6_LOOKUP_NEXT_HOP_BY_HOP] = "ip6-hop-by-hop",		\
-    [IP6_LOOKUP_NEXT_ADD_HOP_BY_HOP] = "ip6-add-hop-by-hop",	\
-    [IP6_LOOKUP_NEXT_POP_HOP_BY_HOP] = "ip6-pop-hop-by-hop",	\
-}
 
 /** Flow hash configuration */
 #define IP_FLOW_HASH_SRC_ADDR (1<<0)
@@ -165,129 +81,6 @@ _(reverse, IP_FLOW_HASH_REVERSE_SRC_DST)
  */
 typedef u32 flow_hash_config_t;
 
-/**
- * Forward delcartion
- */
-struct ip_adjacency_t_;
-
-/**
- * @brief A function type for post-rewrite fixups on midchain adjacency
- */
-typedef void (*adj_midchain_fixup_t) (vlib_main_t * vm,
-				      struct ip_adjacency_t_ * adj,
-				      vlib_buffer_t * b0);
-
-/**
- * @brief Flags on an IP adjacency
- */
-typedef enum ip_adjacency_flags_t_
-{
-    /**
-     * Currently a sync walk is active. Used to prevent re-entrant walking
-     */
-  IP_ADJ_SYNC_WALK_ACTIVE = (1 << 0),
-} ip_adjacency_flags_t;
-
-/** @brief IP unicast adjacency.
-    @note cache aligned.
-*/
-typedef struct ip_adjacency_t_
-{
-  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
-
-  /** Number of adjecencies in block.  Greater than 1 means multipath;
-     otherwise equal to 1. */
-  u16 n_adj;
-
-  /** Next hop after ip4-lookup. */
-  union
-  {
-    ip_lookup_next_t lookup_next_index:16;
-    u16 lookup_next_index_as_int;
-  };
-
-  /** Interface address index for this local/arp adjacency. */
-  u32 if_address_index;
-
-  /*
-   * link/ether-type
-   */
-  vnet_link_t ia_link;
-  u8 ia_nh_proto;
-
-  union
-  {
-    /**
-     * IP_LOOKUP_NEXT_ARP/IP_LOOKUP_NEXT_REWRITE
-     *
-     * neighbour adjacency sub-type;
-     */
-    struct
-    {
-      ip46_address_t next_hop;
-    } nbr;
-      /**
-       * IP_LOOKUP_NEXT_MIDCHAIN
-       *
-       * A nbr adj that is also recursive. Think tunnels.
-       * A nbr adj can transition to be of type MDICHAIN
-       * so be sure to leave the two structs with the next_hop
-       * fields aligned.
-       */
-    struct
-    {
-      /**
-       * The recursive next-hop
-       */
-      ip46_address_t next_hop;
-      /**
-       * The node index of the tunnel's post rewrite/TX function.
-       */
-      u32 tx_function_node;
-      /**
-       * The next DPO to use
-       */
-      dpo_id_t next_dpo;
-      /**
-       * A function to perform the post-rewrite fixup
-       */
-      adj_midchain_fixup_t fixup_func;
-    } midchain;
-    /**
-     * IP_LOOKUP_NEXT_GLEAN
-     *
-     * Glean the address to ARP for from the packet's destination
-     */
-    struct
-    {
-      ip46_address_t receive_addr;
-    } glean;
-  } sub_type;
-
-    CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
-
-  /* Rewrite in second/third cache lines */
-    vnet_declare_rewrite (VLIB_BUFFER_PRE_DATA_SIZE);
-
-  /*
-   * member not accessed in the data plane are relgated to the
-   * remaining cachelines
-   */
-  fib_node_t ia_node;
-
-  /**
-   * Flags on the adjacency
-   */
-  ip_adjacency_flags_t ia_flags;
-
-} ip_adjacency_t;
-
-STATIC_ASSERT ((STRUCT_OFFSET_OF (ip_adjacency_t, cacheline0) == 0),
-	       "IP adjacency cachline 0 is not offset");
-STATIC_ASSERT ((STRUCT_OFFSET_OF (ip_adjacency_t, cacheline1) ==
-		CLIB_CACHE_LINE_BYTES),
-	       "IP adjacency cachline 1 is more than one cachline size offset");
-
 /* An all zeros address */
 extern const ip46_address_t zero_addr;
 
diff --git a/src/vnet/lisp-gpe/lisp_gpe_adjacency.c b/src/vnet/lisp-gpe/lisp_gpe_adjacency.c
index 50662dd6..79b2a07f 100644
--- a/src/vnet/lisp-gpe/lisp_gpe_adjacency.c
+++ b/src/vnet/lisp-gpe/lisp_gpe_adjacency.c
@@ -361,8 +361,8 @@ lisp_gpe_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, adj_index_t ai)
   adj_nbr_midchain_update_rewrite
     (ai, lisp_gpe_fixup,
      (VNET_LINK_ETHERNET == linkt ?
-      ADJ_MIDCHAIN_FLAG_NO_COUNT :
-      ADJ_MIDCHAIN_FLAG_NONE),
+      ADJ_FLAG_MIDCHAIN_NO_COUNT :
+      ADJ_FLAG_NONE),
      lisp_gpe_tunnel_build_rewrite (lgt, ladj,
 				    lisp_gpe_adj_proto_from_vnet_link_type
 				    (linkt)));
diff --git a/src/vnet/mpls/mpls_tunnel.c b/src/vnet/mpls/mpls_tunnel.c
index e488271d..ac6fdcdf 100644
--- a/src/vnet/mpls/mpls_tunnel.c
+++ b/src/vnet/mpls/mpls_tunnel.c
@@ -275,7 +275,7 @@ mpls_tunnel_update_adj (vnet_main_t * vnm,
 {
     adj_nbr_midchain_update_rewrite(
 	ai, mpls_tunnel_fixup, 
-	ADJ_MIDCHAIN_FLAG_NONE,
+	ADJ_FLAG_NONE,
 	mpls_tunnel_build_rewrite(vnm, sw_if_index,
 				  adj_get_link_type(ai),
 				  NULL));
diff --git a/src/vnet/rewrite.c b/src/vnet/rewrite.c
deleted file mode 100644
index 47fb74df..00000000
--- a/src/vnet/rewrite.c
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/*
- * rewrite.c: packet rewrite
- *
- * Copyright (c) 2008 Eliot Dresselhaus
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
- *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include <vnet/vnet.h>
-#include <vnet/ip/lookup.h>
-
-void
-vnet_rewrite_copy_slow_path (vnet_rewrite_data_t * p0,
-			     vnet_rewrite_data_t * rw0,
-			     word n_left, uword most_likely_size)
-{
-  uword n_done =
-    round_pow2 (most_likely_size, sizeof (rw0[0])) / sizeof (rw0[0]);
-
-  p0 -= n_done;
-  rw0 -= n_done;
-
-  /* As we enter the cleanup loop, p0 and rw0 point to the last chunk written
-     by the fast path. Hence, the constant 1, which the
-     vnet_rewrite_copy_one macro renders as p0[-1] = rw0[-1]. */
-
-  while (n_left > 0)
-    {
-      vnet_rewrite_copy_one (p0, rw0, 1);
-      p0--;
-      rw0--;
-      n_left--;
-    }
-}
-
-u8 *
-format_vnet_rewrite (u8 * s, va_list * args)
-{
-  vnet_rewrite_header_t *rw = va_arg (*args, vnet_rewrite_header_t *);
-  u32 max_data_bytes = va_arg (*args, u32);
-  CLIB_UNUSED (uword indent) = va_arg (*args, u32);
-  vnet_main_t *vnm = vnet_get_main ();
-
-  if (rw->sw_if_index != ~0)
-    {
-      vnet_sw_interface_t *si;
-      si = vnet_get_sw_interface_safe (vnm, rw->sw_if_index);
-      if (NULL != si)
-	s = format (s, "%U: ", format_vnet_sw_interface_name, vnm, si);
-      else
-	s = format (s, "DELETED:%d", rw->sw_if_index);
-    }
-
-  /* Format rewrite string. */
-  if (rw->data_bytes > 0)
-
-    s = format (s, "%U",
-		format_hex_bytes,
-		rw->data + max_data_bytes - rw->data_bytes, rw->data_bytes);
-
-  return s;
-}
-
-u32
-vnet_tx_node_index_for_sw_interface (vnet_main_t * vnm, u32 sw_if_index)
-{
-  vnet_hw_interface_t *hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
-  return (hw->output_node_index);
-}
-
-void
-vnet_rewrite_init (vnet_main_t * vnm,
-		   u32 sw_if_index,
-		   u32 this_node, u32 next_node, vnet_rewrite_header_t * rw)
-{
-  rw->sw_if_index = sw_if_index;
-  rw->next_index = vlib_node_add_next (vnm->vlib_main, this_node, next_node);
-  rw->max_l3_packet_bytes =
-    vnet_sw_interface_get_mtu (vnm, sw_if_index, VLIB_TX);
-}
-
-void
-vnet_rewrite_for_sw_interface (vnet_main_t * vnm,
-			       vnet_link_t link_type,
-			       u32 sw_if_index,
-			       u32 node_index,
-			       void *dst_address,
-			       vnet_rewrite_header_t * rw,
-			       u32 max_rewrite_bytes)
-{
-
-  vnet_hw_interface_t *hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
-  vnet_hw_interface_class_t *hc =
-    vnet_get_hw_interface_class (vnm, hw->hw_class_index);
-  u8 *rewrite = NULL;
-
-  vnet_rewrite_init (vnm, sw_if_index, node_index,
-		     vnet_tx_node_index_for_sw_interface (vnm, sw_if_index),
-		     rw);
-
-  ASSERT (hc->build_rewrite);
-  rewrite = hc->build_rewrite (vnm, sw_if_index, link_type, dst_address);
-
-  ASSERT (vec_len (rewrite) < max_rewrite_bytes);
-  vnet_rewrite_set_data_internal (rw, max_rewrite_bytes, rewrite,
-				  vec_len (rewrite));
-  vec_free (rewrite);
-}
-
-void
-vnet_rewrite_for_tunnel (vnet_main_t * vnm,
-			 u32 tx_sw_if_index,
-			 u32 rewrite_node_index,
-			 u32 post_rewrite_node_index,
-			 vnet_rewrite_header_t * rw,
-			 u8 * rewrite_data, u32 rewrite_length)
-{
-  ip_adjacency_t *adj = 0;
-  /*
-   * Installed into vnet_buffer(b)->sw_if_index[VLIB_TX] e.g.
-   * by ip4_rewrite_inline. If the post-rewrite node injects into
-   * ipX-forward, this will be interpreted as a FIB number.
-   */
-  rw->sw_if_index = tx_sw_if_index;
-  rw->next_index = vlib_node_add_next (vnm->vlib_main, rewrite_node_index,
-				       post_rewrite_node_index);
-  rw->max_l3_packet_bytes = (u16) ~ 0;	/* we can't know at this point */
-
-  ASSERT (rewrite_length < sizeof (adj->rewrite_data));
-  /* Leave room for ethernet + VLAN tag */
-  vnet_rewrite_set_data_internal (rw, sizeof (adj->rewrite_data),
-				  rewrite_data, rewrite_length);
-}
-
-void
-serialize_vnet_rewrite (serialize_main_t * m, va_list * va)
-{
-  vnet_rewrite_header_t *rw = va_arg (*va, vnet_rewrite_header_t *);
-  u32 max_data_bytes = va_arg (*va, u32);
-  u8 *p;
-
-  serialize_integer (m, rw->sw_if_index, sizeof (rw->sw_if_index));
-  serialize_integer (m, rw->data_bytes, sizeof (rw->data_bytes));
-  serialize_integer (m, rw->max_l3_packet_bytes,
-		     sizeof (rw->max_l3_packet_bytes));
-  p = serialize_get (m, rw->data_bytes);
-  clib_memcpy (p, vnet_rewrite_get_data_internal (rw, max_data_bytes),
-	       rw->data_bytes);
-}
-
-void
-unserialize_vnet_rewrite (serialize_main_t * m, va_list * va)
-{
-  vnet_rewrite_header_t *rw = va_arg (*va, vnet_rewrite_header_t *);
-  u32 max_data_bytes = va_arg (*va, u32);
-  u8 *p;
-
-  /* It is up to user to fill these in. */
-  rw->next_index = ~0;
-
-  unserialize_integer (m, &rw->sw_if_index, sizeof (rw->sw_if_index));
-  unserialize_integer (m, &rw->data_bytes, sizeof (rw->data_bytes));
-  unserialize_integer (m, &rw->max_l3_packet_bytes,
-		       sizeof (rw->max_l3_packet_bytes));
-  p = unserialize_get (m, rw->data_bytes);
-  clib_memcpy (vnet_rewrite_get_data_internal (rw, max_data_bytes), p,
-	       rw->data_bytes);
-}
-
-u8 *
-vnet_build_rewrite_for_sw_interface (vnet_main_t * vnm,
-				     u32 sw_if_index,
-				     vnet_link_t link_type,
-				     const void *dst_address)
-{
-  vnet_hw_interface_t *hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
-  vnet_hw_interface_class_t *hc =
-    vnet_get_hw_interface_class (vnm, hw->hw_class_index);
-
-  ASSERT (hc->build_rewrite);
-  return (hc->build_rewrite (vnm, sw_if_index, link_type, dst_address));
-}
-
-
-void
-vnet_update_adjacency_for_sw_interface (vnet_main_t * vnm,
-					u32 sw_if_index, u32 ai)
-{
-  vnet_hw_interface_t *hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
-  vnet_hw_interface_class_t *hc =
-    vnet_get_hw_interface_class (vnm, hw->hw_class_index);
-
-  ASSERT (hc->update_adjacency);
-  hc->update_adjacency (vnm, sw_if_index, ai);
-}
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
diff --git a/src/vnet/rewrite.h b/src/vnet/rewrite.h
deleted file mode 100644
index 1dea72f5..00000000
--- a/src/vnet/rewrite.h
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Copyright (c) 2015 Cisco and/or its affiliates.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/*
- * rewrite.h: packet rewrite
- *
- * Copyright (c) 2008 Eliot Dresselhaus
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
- *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#ifndef included_vnet_rewrite_h
-#define included_vnet_rewrite_h
-
-#include <vlib/vlib.h>
-#include <vnet/l3_types.h>
-
-/* Consider using vector types for speed? */
-typedef uword vnet_rewrite_data_t;
-
-/**
- * Flags associated with the rewrite/adjacency
- */
-typedef enum vnet_rewrite_flags_t_
-{
-  /**
-   * This adjacency/interface has output features configured
-   */
-  VNET_REWRITE_HAS_FEATURES = (1 << 0),
-} __attribute__ ((packed)) vnet_rewrite_flags_t;
-
-/* *INDENT-OFF* */
-typedef CLIB_PACKED (struct {
-  /* Interface to mark re-written packets with. */
-  u32 sw_if_index;
-
-  /* Next node to feed after packet rewrite is done. */
-  u16 next_index;
-
-  /* Number of bytes in rewrite data. */
-  u16 data_bytes;
-
-  /* Max packet size layer 3 (MTU) for output interface.
-     Used for MTU check after packet rewrite. */
-  u16 max_l3_packet_bytes;
-
-  /* Data-plane flags on the adjacency/rewrite */
-  vnet_rewrite_flags_t flags;
-
-  /* When dynamically writing a multicast destination L2 addresss
-   * this is the offset from the IP address at which to write in the
-   * IP->MAC address translation.
-   */
-  u8 dst_mcast_offset;
-
-  /* The mask to apply to the lower 4 bytes of the IP address before ORing
-   * into the destinaiton MAC address */
-  u32 dst_mcast_mask;
-
-  /* Rewrite string starting at end and going backwards. */
-  u8 data[0];
-}) vnet_rewrite_header_t;
-/* *INDENT-ON* */
-
-/**
- * At 16 bytes of rewrite herader we have enought space left for a IPv6
- * (40 bytes) + LISP-GPE (8 bytes) in the cache line
- */
-STATIC_ASSERT (sizeof (vnet_rewrite_header_t) <= 16,
-	       "Rewrite header too big");
-
-/*
-  Helper macro for declaring rewrite string w/ given max-size.
-
-  Typical usage:
-    typedef struct {
-      //
-      int a, b;
-
-      // Total adjacency is 64 bytes.
-      vnet_rewrite_declare(64 - 2*sizeof(int)) rw;
-    } my_adjacency_t;
-*/
-#define vnet_declare_rewrite(total_bytes)				\
-struct {								\
-  vnet_rewrite_header_t rewrite_header;  			        \
-									\
-  u8 rewrite_data[(total_bytes) - sizeof (vnet_rewrite_header_t)];	\
-}
-
-always_inline void
-vnet_rewrite_clear_data_internal (vnet_rewrite_header_t * rw, int max_size)
-{
-  /* Sanity check values carefully for this memset operation */
-  ASSERT ((max_size > 0) && (max_size < VLIB_BUFFER_PRE_DATA_SIZE));
-
-  rw->data_bytes = 0;
-  memset (rw->data, 0xfe, max_size);
-}
-
-always_inline void
-vnet_rewrite_set_data_internal (vnet_rewrite_header_t * rw,
-				int max_size, void *data, int data_bytes)
-{
-  /* Sanity check values carefully for this memset operation */
-  ASSERT ((max_size > 0) && (max_size < VLIB_BUFFER_PRE_DATA_SIZE));
-  ASSERT ((data_bytes >= 0) && (data_bytes < max_size));
-
-  rw->data_bytes = data_bytes;
-  clib_memcpy (rw->data + max_size - data_bytes, data, data_bytes);
-  memset (rw->data, 0xfe, max_size - data_bytes);
-}
-
-#define vnet_rewrite_set_data(rw,data,data_bytes)		\
-  vnet_rewrite_set_data_internal (&((rw).rewrite_header),	\
-				  sizeof ((rw).rewrite_data),	\
-				  (data),			\
-				  (data_bytes))
-
-always_inline void *
-vnet_rewrite_get_data_internal (vnet_rewrite_header_t * rw, int max_size)
-{
-  ASSERT (rw->data_bytes <= max_size);
-  return rw->data + max_size - rw->data_bytes;
-}
-
-#define vnet_rewrite_get_data(rw) \
-  vnet_rewrite_get_data_internal (&((rw).rewrite_header), sizeof ((rw).rewrite_data))
-
-always_inline void
-vnet_rewrite_copy_one (vnet_rewrite_data_t * p0, vnet_rewrite_data_t * rw0,
-		       int i)
-{
-  p0[-i] = rw0[-i];
-}
-
-void vnet_rewrite_copy_slow_path (vnet_rewrite_data_t * p0,
-				  vnet_rewrite_data_t * rw0,
-				  word n_left, uword most_likely_size);
-
-/* *INDENT-OFF* */
-typedef CLIB_PACKED (struct {
-  u64 a;
-  u32 b;
-  u16 c;
-}) eh_copy_t;
-/* *INDENT-ON* */
-
-always_inline void
-_vnet_rewrite_one_header (vnet_rewrite_header_t * h0,
-			  void *packet0, int max_size, int most_likely_size)
-{
-  vnet_rewrite_data_t *p0 = packet0;
-  vnet_rewrite_data_t *rw0 = (vnet_rewrite_data_t *) (h0->data + max_size);
-  word n_left0;
-
-  /* 0xfefe => poisoned adjacency => crash */
-  ASSERT (h0->data_bytes != 0xfefe);
-
-  if (PREDICT_TRUE (h0->data_bytes == sizeof (eh_copy_t)))
-    {
-      eh_copy_t *s, *d;
-      s = (eh_copy_t *) (h0->data + max_size - sizeof (eh_copy_t));
-      d = (eh_copy_t *) (((u8 *) packet0) - sizeof (eh_copy_t));
-      clib_memcpy (d, s, sizeof (eh_copy_t));
-      return;
-    }
-
-
-#define _(i)								\
-  do {									\
-    if (most_likely_size > ((i)-1)*sizeof (vnet_rewrite_data_t))	\
-      vnet_rewrite_copy_one (p0, rw0, (i));				\
-  } while (0)
-
-  _(4);
-  _(3);
-  _(2);
-  _(1);
-
-#undef _
-
-  n_left0 = (int)
-    (((int) h0->data_bytes - most_likely_size) + (sizeof (rw0[0]) - 1))
-    / (int) sizeof (rw0[0]);
-  if (PREDICT_FALSE (n_left0 > 0))
-    vnet_rewrite_copy_slow_path (p0, rw0, n_left0, most_likely_size);
-}
-
-always_inline void
-_vnet_rewrite_two_headers (vnet_rewrite_header_t * h0,
-			   vnet_rewrite_header_t * h1,
-			   void *packet0,
-			   void *packet1, int max_size, int most_likely_size)
-{
-  vnet_rewrite_data_t *p0 = packet0;
-  vnet_rewrite_data_t *p1 = packet1;
-  vnet_rewrite_data_t *rw0 = (vnet_rewrite_data_t *) (h0->data + max_size);
-  vnet_rewrite_data_t *rw1 = (vnet_rewrite_data_t *) (h1->data + max_size);
-  word n_left0, n_left1;
-  int slow_path;
-
-  /* 0xfefe => poisoned adjacency => crash */
-  ASSERT (h0->data_bytes != 0xfefe);
-  ASSERT (h1->data_bytes != 0xfefe);
-
-  /* Arithmetic calculation: bytes0 == bytes1 == 14 */
-  slow_path = h0->data_bytes ^ h1->data_bytes;
-  slow_path += h0->data_bytes ^ sizeof (eh_copy_t);
-
-  if (PREDICT_TRUE (slow_path == 0))
-    {
-      eh_copy_t *s0, *d0, *s1, *d1;
-      s0 = (eh_copy_t *) (h0->data + max_size - sizeof (eh_copy_t));
-      d0 = (eh_copy_t *) (((u8 *) packet0) - sizeof (eh_copy_t));
-      clib_memcpy (d0, s0, sizeof (eh_copy_t));
-      s1 = (eh_copy_t *) (h1->data + max_size - sizeof (eh_copy_t));
-      d1 = (eh_copy_t *) (((u8 *) packet1) - sizeof (eh_copy_t));
-      clib_memcpy (d1, s1, sizeof (eh_copy_t));
-      return;
-    }
-
-#define _(i)								\
-  do {									\
-    if (most_likely_size > ((i)-1)*sizeof (vnet_rewrite_data_t))	\
-      {									\
-	vnet_rewrite_copy_one (p0, rw0, (i));				\
-	vnet_rewrite_copy_one (p1, rw1, (i));				\
-      }									\
-  } while (0)
-
-  _(4);
-  _(3);
-  _(2);
-  _(1);
-
-#undef _
-
-  n_left0 = (int)
-    (((int) h0->data_bytes - most_likely_size) + (sizeof (rw0[0]) - 1))
-    / (int) sizeof (rw0[0]);
-  n_left1 = (int)
-    (((int) h1->data_bytes - most_likely_size) + (sizeof (rw1[0]) - 1))
-    / (int) sizeof (rw1[0]);
-
-  if (PREDICT_FALSE (n_left0 > 0 || n_left1 > 0))
-    {
-      vnet_rewrite_copy_slow_path (p0, rw0, n_left0, most_likely_size);
-      vnet_rewrite_copy_slow_path (p1, rw1, n_left1, most_likely_size);
-    }
-}
-
-#define vnet_rewrite_one_header(rw0,p0,most_likely_size)	\
-  _vnet_rewrite_one_header (&((rw0).rewrite_header), (p0),	\
-			    sizeof ((rw0).rewrite_data),	\
-			    (most_likely_size))
-
-#define vnet_rewrite_two_headers(rw0,rw1,p0,p1,most_likely_size)	\
-  _vnet_rewrite_two_headers (&((rw0).rewrite_header), &((rw1).rewrite_header), \
-			     (p0), (p1),				\
-			     sizeof ((rw0).rewrite_data),		\
-			     (most_likely_size))
-
-always_inline void
-_vnet_fixup_one_header (vnet_rewrite_header_t * h0,
-			u8 * addr, u32 addr_len, u8 * packet0)
-{
-  if (PREDICT_TRUE (h0->dst_mcast_mask))
-    {
-      /* location to write to in the packet */
-      u8 *p0 = packet0 - h0->dst_mcast_offset;
-      u32 *p1 = (u32 *) p0;
-      /* location to copy from in the L3 dest address */
-      u32 *a0 = (u32 *) (addr + addr_len - sizeof (h0->dst_mcast_mask));
-
-      *p1 |= (*a0 & h0->dst_mcast_mask);
-    }
-}
-
-#define vnet_fixup_one_header(rw0,addr,p0)              \
-  _vnet_fixup_one_header (&((rw0).rewrite_header),                      \
-                          (u8*)(addr), sizeof((*addr)),                 \
-                          (u8*)(p0))
-
-#define VNET_REWRITE_FOR_SW_INTERFACE_ADDRESS_BROADCAST ((void *) 0)
-/** Deprecated */
-void vnet_rewrite_for_sw_interface (struct vnet_main_t *vnm,
-				    vnet_link_t packet_type,
-				    u32 sw_if_index,
-				    u32 node_index,
-				    void *dst_address,
-				    vnet_rewrite_header_t * rw,
-				    u32 max_rewrite_bytes);
-
-u32 vnet_tx_node_index_for_sw_interface (struct vnet_main_t *vnm,
-					 u32 sw_if_index);
-
-void vnet_rewrite_init (struct vnet_main_t *vnm,
-			u32 sw_if_index,
-			u32 this_node,
-			u32 next_node, vnet_rewrite_header_t * rw);
-
-u8 *vnet_build_rewrite_for_sw_interface (struct vnet_main_t *vnm,
-					 u32 sw_if_index,
-					 vnet_link_t packet_type,
-					 const void *dst_address);
-void vnet_update_adjacency_for_sw_interface (struct vnet_main_t *vnm,
-					     u32 sw_if_index, u32 ai);
-
-format_function_t format_vnet_rewrite;
-
-serialize_function_t serialize_vnet_rewrite, unserialize_vnet_rewrite;
-
-#endif /* included_vnet_rewrite_h */
-
-/*
- * fd.io coding-style-patch-verification: ON
- *
- * Local Variables:
- * eval: (c-set-style "gnu")
- * End:
- */
-- 
cgit 1.2.3-korg


From d1ed4867b9bc3a9169838ae544d217bfaf23780b Mon Sep 17 00:00:00 2001
From: flyingeagle23 <wang.hui56@zte.com.cn>
Date: Thu, 6 Apr 2017 16:47:46 +0800
Subject: VPP-684.Add ip which mask length exceeding upper limit,ping
 segmentfault

Change-Id: I2d1c5e4dbdcab2a81d6c88801183ca0bbc83e1d9
Signed-off-by: flyingeagle23 <wang.hui56@zte.com.cn>
---
 src/vnet/ip/lookup.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 src/vnet/ip/lookup.c

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
old mode 100644
new mode 100755
index 95f36d41..ec9a1f97
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -73,7 +73,10 @@ ip_interface_address_add_del (ip_lookup_main_t * lm,
   a = p ? pool_elt_at_index (lm->if_address_pool, p[0]) : 0;
 
   /* Verify given length. */
-  if ((a && (address_length != a->address_length)) || (address_length == 0))
+  if ((a && (address_length != a->address_length)) ||
+      (address_length == 0) ||
+      (lm->is_ip6 && address_length > 128) ||
+      (!lm->is_ip6 && address_length > 32))
     {
       vnm->api_errno = VNET_API_ERROR_ADDRESS_LENGTH_MISMATCH;
       return clib_error_create
-- 
cgit 1.2.3-korg


From 0f26c5a0138ac86d7ebd197c31a09d8d624c35fe Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Wed, 1 Mar 2017 15:12:11 -0800
Subject: MPLS Mcast

 1 - interface-DPO
        Used in the Data-plane to change a packet's input interface
 2 - MPLS multicast FIB entry
        Same as a unicast entry but it links to a replicate not a load-balance DPO
 3 - Multicast MPLS tunnel
        Update MPLS tunnels to use a FIB path-list to describe the endpoint[s]. Use the path-list to generate the forwarding chain (DPOs) to link to .
 4 - Resolve a path via a local label (of an mLDP LSP)
        For IP multicast entries to use an LSP in the replication list, we need to decribe the 'resolve-via-label' where the label is that of a multicast LSP.
 5 - MPLS disposition path sets RPF-ID
        For a interface-less LSP (i.e. mLDP not RSVP-TE) at the tail of the LSP we still need to perform an RPF check. An MPLS disposition DPO performs the MPLS pop validation checks and sets the RPF-ID in the packet.
 6 - RPF check with per-entry RPF-ID
       An RPF-ID is used instead of a real interface SW if index in the case the IP traffic arrives from an LSP that does not have an associated interface.

Change-Id: Ib92e177be919147bafeb599729abf3d1abc2f4b3
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/plugins/dpdk/device/node.c         |   2 +-
 src/vat/api_format.c                   | 133 ++---
 src/vnet.am                            |   2 +
 src/vnet/adj/adj.c                     |  13 +-
 src/vnet/adj/adj.h                     |   6 +
 src/vnet/adj/adj_internal.h            |  14 +-
 src/vnet/adj/adj_mcast.c               | 134 ++++-
 src/vnet/adj/adj_mcast.h               |  27 +
 src/vnet/adj/adj_midchain.c            |  62 ++-
 src/vnet/adj/adj_nbr.c                 |   2 -
 src/vnet/buffer.h                      |   3 +
 src/vnet/devices/ssvm/node.c           |   2 +-
 src/vnet/dhcp/dhcp6_proxy_node.c       |   1 +
 src/vnet/dpo/dpo.c                     |  10 +
 src/vnet/dpo/dpo.h                     |   8 +-
 src/vnet/dpo/interface_dpo.c           | 416 ++++++++++++++++
 src/vnet/dpo/interface_dpo.h           |  67 +++
 src/vnet/dpo/lookup_dpo.c              | 211 +++++++-
 src/vnet/dpo/lookup_dpo.h              |  20 +
 src/vnet/dpo/mpls_disposition.c        | 364 ++++++++++++++
 src/vnet/dpo/mpls_disposition.h        |  85 ++++
 src/vnet/dpo/mpls_label_dpo.c          |   6 +-
 src/vnet/dpo/replicate_dpo.c           |  48 +-
 src/vnet/dpo/replicate_dpo.h           |   2 +
 src/vnet/ethernet/arp.c                |   1 +
 src/vnet/ethernet/interface.c          |   2 +-
 src/vnet/ethernet/node.c               |   4 +-
 src/vnet/ethernet/types.def            |   4 +-
 src/vnet/fib/fib_api.h                 |   4 +
 src/vnet/fib/fib_entry.c               |  47 +-
 src/vnet/fib/fib_entry.h               |  13 +-
 src/vnet/fib/fib_entry_src.c           | 154 +++---
 src/vnet/fib/fib_internal.h            |   1 +
 src/vnet/fib/fib_path.c                | 222 +++++++--
 src/vnet/fib/fib_path.h                |  17 +-
 src/vnet/fib/fib_path_ext.c            |   4 +-
 src/vnet/fib/fib_path_ext.h            |   3 +-
 src/vnet/fib/fib_path_list.c           | 270 ++++++----
 src/vnet/fib/fib_path_list.h           |  22 +-
 src/vnet/fib/fib_table.c               |  47 +-
 src/vnet/fib/fib_test.c                | 345 +++++++++++--
 src/vnet/fib/fib_test.h                | 111 +++++
 src/vnet/fib/fib_types.c               |  15 +-
 src/vnet/fib/fib_types.h               |  60 ++-
 src/vnet/fib/mpls_fib.c                |  15 +-
 src/vnet/handoff.h                     |  10 +-
 src/vnet/interface.c                   |   2 +-
 src/vnet/ip/ip.api                     |   3 +
 src/vnet/ip/ip4_forward.c              |  20 +
 src/vnet/ip/ip6_forward.c              |  23 +
 src/vnet/ip/ip6_neighbor.c             |   1 +
 src/vnet/ip/ip_api.c                   |  98 ++--
 src/vnet/ip/lookup.c                   |   3 +-
 src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c |   1 +
 src/vnet/mfib/ip4_mfib.c               |   1 +
 src/vnet/mfib/ip6_mfib.c               |   1 +
 src/vnet/mfib/mfib_entry.c             | 395 +++++++++------
 src/vnet/mfib/mfib_entry.h             |  20 +-
 src/vnet/mfib/mfib_forward.c           |  29 +-
 src/vnet/mfib/mfib_table.c             |   8 +-
 src/vnet/mfib/mfib_table.h             |   1 +
 src/vnet/mfib/mfib_test.c              | 127 ++++-
 src/vnet/mpls/mpls.api                 |  87 ++--
 src/vnet/mpls/mpls.c                   |  17 +-
 src/vnet/mpls/mpls_api.c               |  97 ++--
 src/vnet/mpls/mpls_input.c             |   2 +-
 src/vnet/mpls/mpls_lookup.c            | 236 ++++++---
 src/vnet/mpls/mpls_tunnel.c            | 883 ++++++++++++++++++++++-----------
 src/vnet/mpls/mpls_tunnel.h            |  57 ++-
 src/vnet/mpls/mpls_types.h             |  20 +
 src/vnet/srp/interface.c               |   2 +-
 test/test_ip_mcast.py                  |   1 +
 test/test_mpls.py                      | 277 ++++++++++-
 test/vpp_ip_route.py                   |  38 +-
 test/vpp_mpls_tunnel_interface.py      |  46 ++
 test/vpp_papi_provider.py              |  16 +-
 76 files changed, 4393 insertions(+), 1128 deletions(-)
 create mode 100644 src/vnet/dpo/interface_dpo.c
 create mode 100644 src/vnet/dpo/interface_dpo.h
 create mode 100644 src/vnet/dpo/mpls_disposition.c
 create mode 100644 src/vnet/dpo/mpls_disposition.h
 create mode 100644 src/vnet/fib/fib_test.h
 create mode 100644 test/vpp_mpls_tunnel_interface.py

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c
index b10e0fad..0549ba5d 100644
--- a/src/plugins/dpdk/device/node.c
+++ b/src/plugins/dpdk/device/node.c
@@ -52,7 +52,7 @@ always_inline int
 vlib_buffer_is_mpls (vlib_buffer_t * b)
 {
   ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b);
-  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST));
+  return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS));
 }
 
 always_inline u32
diff --git a/src/vat/api_format.c b/src/vat/api_format.c
index 61b8e1d8..107aa012 100644
--- a/src/vat/api_format.c
+++ b/src/vat/api_format.c
@@ -16369,32 +16369,82 @@ api_netmap_delete (vat_main_t * vam)
   return ret;
 }
 
-static void vl_api_mpls_tunnel_details_t_handler
-  (vl_api_mpls_tunnel_details_t * mp)
+static void
+vl_api_mpls_fib_path_print (vat_main_t * vam, vl_api_fib_path2_t * fp)
+{
+  if (fp->afi == IP46_TYPE_IP6)
+    print (vam->ofp,
+	   "  weight %d, sw_if_index %d, is_local %d, is_drop %d, "
+	   "is_unreach %d, is_prohitbit %d, afi %d, next_hop %U",
+	   ntohl (fp->weight), ntohl (fp->sw_if_index), fp->is_local,
+	   fp->is_drop, fp->is_unreach, fp->is_prohibit, fp->afi,
+	   format_ip6_address, fp->next_hop);
+  else if (fp->afi == IP46_TYPE_IP4)
+    print (vam->ofp,
+	   "  weight %d, sw_if_index %d, is_local %d, is_drop %d, "
+	   "is_unreach %d, is_prohitbit %d, afi %d, next_hop %U",
+	   ntohl (fp->weight), ntohl (fp->sw_if_index), fp->is_local,
+	   fp->is_drop, fp->is_unreach, fp->is_prohibit, fp->afi,
+	   format_ip4_address, fp->next_hop);
+}
+
+static void
+vl_api_mpls_fib_path_json_print (vat_json_node_t * node,
+				 vl_api_fib_path2_t * fp)
+{
+  struct in_addr ip4;
+  struct in6_addr ip6;
+
+  vat_json_object_add_uint (node, "weight", ntohl (fp->weight));
+  vat_json_object_add_uint (node, "sw_if_index", ntohl (fp->sw_if_index));
+  vat_json_object_add_uint (node, "is_local", fp->is_local);
+  vat_json_object_add_uint (node, "is_drop", fp->is_drop);
+  vat_json_object_add_uint (node, "is_unreach", fp->is_unreach);
+  vat_json_object_add_uint (node, "is_prohibit", fp->is_prohibit);
+  vat_json_object_add_uint (node, "next_hop_afi", fp->afi);
+  if (fp->afi == IP46_TYPE_IP4)
+    {
+      clib_memcpy (&ip4, &fp->next_hop, sizeof (ip4));
+      vat_json_object_add_ip4 (node, "next_hop", ip4);
+    }
+  else if (fp->afi == IP46_TYPE_IP6)
+    {
+      clib_memcpy (&ip6, &fp->next_hop, sizeof (ip6));
+      vat_json_object_add_ip6 (node, "next_hop", ip6);
+    }
+}
+
+static void
+vl_api_mpls_tunnel_details_t_handler (vl_api_mpls_tunnel_details_t * mp)
 {
   vat_main_t *vam = &vat_main;
-  i32 len = mp->mt_next_hop_n_labels;
+  int count = ntohl (mp->mt_count);
+  vl_api_fib_path2_t *fp;
   i32 i;
 
-  print (vam->ofp, "[%d]: via %U %d labels ",
-	 mp->tunnel_index,
-	 format_ip4_address, mp->mt_next_hop,
-	 ntohl (mp->mt_next_hop_sw_if_index));
-  for (i = 0; i < len; i++)
+  print (vam->ofp, "[%d]: sw_if_index %d via:",
+	 ntohl (mp->mt_tunnel_index), ntohl (mp->mt_sw_if_index));
+  fp = mp->mt_paths;
+  for (i = 0; i < count; i++)
     {
-      print (vam->ofp, "%u ", ntohl (mp->mt_next_hop_out_labels[i]));
+      vl_api_mpls_fib_path_print (vam, fp);
+      fp++;
     }
+
   print (vam->ofp, "");
 }
 
-static void vl_api_mpls_tunnel_details_t_handler_json
-  (vl_api_mpls_tunnel_details_t * mp)
+#define vl_api_mpls_tunnel_details_t_endian vl_noop_handler
+#define vl_api_mpls_tunnel_details_t_print vl_noop_handler
+
+static void
+vl_api_mpls_tunnel_details_t_handler_json (vl_api_mpls_tunnel_details_t * mp)
 {
   vat_main_t *vam = &vat_main;
   vat_json_node_t *node = NULL;
-  struct in_addr ip4;
+  int count = ntohl (mp->mt_count);
+  vl_api_fib_path2_t *fp;
   i32 i;
-  i32 len = mp->mt_next_hop_n_labels;
 
   if (VAT_JSON_ARRAY != vam->json_tree.type)
     {
@@ -16404,17 +16454,17 @@ static void vl_api_mpls_tunnel_details_t_handler_json
   node = vat_json_array_add (&vam->json_tree);
 
   vat_json_init_object (node);
-  vat_json_object_add_uint (node, "tunnel_index", ntohl (mp->tunnel_index));
-  clib_memcpy (&ip4, &(mp->mt_next_hop), sizeof (ip4));
-  vat_json_object_add_ip4 (node, "next_hop", ip4);
-  vat_json_object_add_uint (node, "next_hop_sw_if_index",
-			    ntohl (mp->mt_next_hop_sw_if_index));
-  vat_json_object_add_uint (node, "l2_only", ntohl (mp->mt_l2_only));
-  vat_json_object_add_uint (node, "label_count", len);
-  for (i = 0; i < len; i++)
+  vat_json_object_add_uint (node, "tunnel_index",
+			    ntohl (mp->mt_tunnel_index));
+  vat_json_object_add_uint (node, "sw_if_index", ntohl (mp->mt_sw_if_index));
+
+  vat_json_object_add_uint (node, "l2_only", mp->mt_l2_only);
+
+  fp = mp->mt_paths;
+  for (i = 0; i < count; i++)
     {
-      vat_json_object_add_uint (node, "label",
-				ntohl (mp->mt_next_hop_out_labels[i]));
+      vl_api_mpls_fib_path_json_print (node, fp);
+      fp++;
     }
 }
 
@@ -16453,6 +16503,7 @@ api_mpls_tunnel_dump (vat_main_t * vam)
 #define vl_api_mpls_fib_details_t_endian vl_noop_handler
 #define vl_api_mpls_fib_details_t_print vl_noop_handler
 
+
 static void
 vl_api_mpls_fib_details_t_handler (vl_api_mpls_fib_details_t * mp)
 {
@@ -16467,20 +16518,7 @@ vl_api_mpls_fib_details_t_handler (vl_api_mpls_fib_details_t * mp)
   fp = mp->path;
   for (i = 0; i < count; i++)
     {
-      if (fp->afi == IP46_TYPE_IP6)
-	print (vam->ofp,
-	       "  weight %d, sw_if_index %d, is_local %d, is_drop %d, "
-	       "is_unreach %d, is_prohitbit %d, afi %d, next_hop %U",
-	       ntohl (fp->weight), ntohl (fp->sw_if_index), fp->is_local,
-	       fp->is_drop, fp->is_unreach, fp->is_prohibit, fp->afi,
-	       format_ip6_address, fp->next_hop);
-      else if (fp->afi == IP46_TYPE_IP4)
-	print (vam->ofp,
-	       "  weight %d, sw_if_index %d, is_local %d, is_drop %d, "
-	       "is_unreach %d, is_prohitbit %d, afi %d, next_hop %U",
-	       ntohl (fp->weight), ntohl (fp->sw_if_index), fp->is_local,
-	       fp->is_drop, fp->is_unreach, fp->is_prohibit, fp->afi,
-	       format_ip4_address, fp->next_hop);
+      vl_api_mpls_fib_path_print (vam, fp);
       fp++;
     }
 }
@@ -16491,8 +16529,6 @@ static void vl_api_mpls_fib_details_t_handler_json
   vat_main_t *vam = &vat_main;
   int count = ntohl (mp->count);
   vat_json_node_t *node = NULL;
-  struct in_addr ip4;
-  struct in6_addr ip6;
   vl_api_fib_path2_t *fp;
   int i;
 
@@ -16511,23 +16547,8 @@ static void vl_api_mpls_fib_details_t_handler_json
   fp = mp->path;
   for (i = 0; i < count; i++)
     {
-      vat_json_object_add_uint (node, "weight", ntohl (fp->weight));
-      vat_json_object_add_uint (node, "sw_if_index", ntohl (fp->sw_if_index));
-      vat_json_object_add_uint (node, "is_local", fp->is_local);
-      vat_json_object_add_uint (node, "is_drop", fp->is_drop);
-      vat_json_object_add_uint (node, "is_unreach", fp->is_unreach);
-      vat_json_object_add_uint (node, "is_prohibit", fp->is_prohibit);
-      vat_json_object_add_uint (node, "next_hop_afi", fp->afi);
-      if (fp->afi == IP46_TYPE_IP4)
-	{
-	  clib_memcpy (&ip4, &fp->next_hop, sizeof (ip4));
-	  vat_json_object_add_ip4 (node, "next_hop", ip4);
-	}
-      else if (fp->afi == IP46_TYPE_IP6)
-	{
-	  clib_memcpy (&ip6, &fp->next_hop, sizeof (ip6));
-	  vat_json_object_add_ip6 (node, "next_hop", ip6);
-	}
+      vl_api_mpls_fib_path_json_print (node, fp);
+      fp++;
     }
 }
 
diff --git a/src/vnet.am b/src/vnet.am
index 643ae92e..bed4902b 100644
--- a/src/vnet.am
+++ b/src/vnet.am
@@ -990,6 +990,8 @@ libvnet_la_SOURCES +=				\
   vnet/dpo/lookup_dpo.c   			\
   vnet/dpo/classify_dpo.c   			\
   vnet/dpo/replicate_dpo.c   			\
+  vnet/dpo/interface_dpo.c   			\
+  vnet/dpo/mpls_disposition.c   		\
   vnet/dpo/mpls_label_dpo.c
 
 nobase_include_HEADERS +=			\
diff --git a/src/vnet/adj/adj.c b/src/vnet/adj/adj.c
index 90182006..36dfe500 100644
--- a/src/vnet/adj/adj.c
+++ b/src/vnet/adj/adj.c
@@ -67,6 +67,10 @@ adj_alloc (fib_protocol_t proto)
     adj->lookup_next_index = 0;
     adj->ia_delegates = NULL;
 
+    /* lest it become a midchain in the future */
+    memset(&adj->sub_type.midchain.next_dpo, 0,
+           sizeof(adj->sub_type.midchain.next_dpo));
+
     ip4_main.lookup_main.adjacency_heap = adj_pool;
     ip6_main.lookup_main.adjacency_heap = adj_pool;
 
@@ -118,6 +122,9 @@ format_ip_adjacency (u8 * s, va_list * args)
     case IP_LOOKUP_NEXT_MCAST:
 	s = format (s, "%U", format_adj_mcast, adj_index, 0);
 	break;
+    case IP_LOOKUP_NEXT_MCAST_MIDCHAIN:
+	s = format (s, "%U", format_adj_mcast_midchain, adj_index, 0);
+	break;
     default:
 	break;
     }
@@ -180,6 +187,7 @@ adj_last_lock_gone (ip_adjacency_t *adj)
 			 adj->rewrite_header.sw_if_index);
 	break;
     case IP_LOOKUP_NEXT_MCAST:
+    case IP_LOOKUP_NEXT_MCAST_MIDCHAIN:
 	adj_mcast_remove(adj->ia_nh_proto,
 			 adj->rewrite_header.sw_if_index);
 	break;
@@ -338,6 +346,7 @@ adj_walk (u32 sw_if_index,
     FOR_EACH_FIB_IP_PROTOCOL(proto)
     {
         adj_nbr_walk(sw_if_index, proto, cb, ctx);
+        adj_mcast_walk(sw_if_index, proto, cb, ctx);
     }
 }
 
@@ -544,9 +553,9 @@ adj_show (vlib_main_t * vm,
  * [@0]
  * [@1]  glean: loop0
  * [@2] ipv4 via 1.0.0.2 loop0: IP4: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
- * [@3] mpls via 1.0.0.2 loop0: MPLS_UNICAST: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
+ * [@3] mpls via 1.0.0.2 loop0: MPLS: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
  * [@4] ipv4 via 1.0.0.3 loop0: IP4: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
- * [@5] mpls via 1.0.0.3 loop0: MPLS_UNICAST: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
+ * [@5] mpls via 1.0.0.3 loop0: MPLS: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc
  * @cliexend
  ?*/
 VLIB_CLI_COMMAND (adj_show_command, static) = {
diff --git a/src/vnet/adj/adj.h b/src/vnet/adj/adj.h
index 32997c91..ed5eb1f1 100644
--- a/src/vnet/adj/adj.h
+++ b/src/vnet/adj/adj.h
@@ -81,6 +81,10 @@ typedef enum
   /** Multicast Adjacency. */
   IP_LOOKUP_NEXT_MCAST,
 
+  /** Multicast Midchain Adjacency. An Adjacency for sending macst packets
+   *  on a tunnel/virtual interface */
+  IP_LOOKUP_NEXT_MCAST_MIDCHAIN,
+
   IP_LOOKUP_N_NEXT,
 } __attribute__ ((packed)) ip_lookup_next_t;
 
@@ -107,6 +111,7 @@ typedef enum
     [IP_LOOKUP_NEXT_REWRITE] = "ip4-rewrite",    		\
     [IP_LOOKUP_NEXT_MCAST] = "ip4-rewrite-mcast",	        \
     [IP_LOOKUP_NEXT_MIDCHAIN] = "ip4-midchain",		        \
+    [IP_LOOKUP_NEXT_MCAST_MIDCHAIN] = "ip4-mcast-midchain",     \
     [IP_LOOKUP_NEXT_ICMP_ERROR] = "ip4-icmp-error",		\
 }
 
@@ -119,6 +124,7 @@ typedef enum
     [IP_LOOKUP_NEXT_REWRITE] = "ip6-rewrite",			\
     [IP_LOOKUP_NEXT_MCAST] = "ip6-rewrite-mcast",		\
     [IP_LOOKUP_NEXT_MIDCHAIN] = "ip6-midchain",			\
+    [IP_LOOKUP_NEXT_MCAST_MIDCHAIN] = "ip6-mcast-midchain",     \
     [IP_LOOKUP_NEXT_ICMP_ERROR] = "ip6-icmp-error",		\
     [IP6_LOOKUP_NEXT_HOP_BY_HOP] = "ip6-hop-by-hop",		\
     [IP6_LOOKUP_NEXT_ADD_HOP_BY_HOP] = "ip6-add-hop-by-hop",	\
diff --git a/src/vnet/adj/adj_internal.h b/src/vnet/adj/adj_internal.h
index 30668625..2c123c54 100644
--- a/src/vnet/adj/adj_internal.h
+++ b/src/vnet/adj/adj_internal.h
@@ -17,6 +17,7 @@
 #define __ADJ_INTERNAL_H__
 
 #include <vnet/adj/adj.h>
+#include <vnet/adj/adj_mcast.h>
 #include <vnet/ip/ip.h>
 #include <vnet/mpls/mpls.h>
 #include <vnet/adj/adj_l2.h>
@@ -87,11 +88,14 @@ adj_get_index (ip_adjacency_t *adj)
     return (adj - adj_pool);
 }
 
-extern void adj_nbr_update_rewrite_internal (ip_adjacency_t *adj,
-					     ip_lookup_next_t adj_next_index,
-					     u32 complete_next_index,
-					     u32 next_index,
-					     u8 *rewrite);
+extern void adj_nbr_update_rewrite_internal(ip_adjacency_t *adj,
+                                            ip_lookup_next_t adj_next_index,
+                                            u32 complete_next_index,
+                                            u32 next_index,
+                                            u8 *rewrite);
+extern void adj_midchain_setup(adj_index_t adj_index,
+                               adj_midchain_fixup_t fixup,
+                               adj_flags_t flags);
 
 extern ip_adjacency_t * adj_alloc(fib_protocol_t proto);
 
diff --git a/src/vnet/adj/adj_mcast.c b/src/vnet/adj/adj_mcast.c
index 4f678e43..755abfd4 100644
--- a/src/vnet/adj/adj_mcast.c
+++ b/src/vnet/adj/adj_mcast.c
@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 
-#include <vnet/adj/adj.h>
+#include <vnet/adj/adj_mcast.h>
 #include <vnet/adj/adj_internal.h>
 #include <vnet/fib/fib_walk.h>
 #include <vnet/ip/ip.h>
@@ -129,6 +129,59 @@ adj_mcast_update_rewrite (adj_index_t adj_index,
     adj->rewrite_header.dst_mcast_mask = clib_host_to_net_u32(mask);
 }
 
+/**
+ * adj_mcast_midchain_update_rewrite
+ *
+ * Update the adjacency's rewrite string. A NULL string implies the
+ * rewirte is reset (i.e. when ARP/ND etnry is gone).
+ * NB: the adj being updated may be handling traffic in the DP.
+ */
+void
+adj_mcast_midchain_update_rewrite (adj_index_t adj_index,
+                                   adj_midchain_fixup_t fixup,
+                                   adj_flags_t flags,
+                                   u8 *rewrite,
+                                   u8 offset,
+                                   u32 mask)
+{
+    ip_adjacency_t *adj;
+
+    ASSERT(ADJ_INDEX_INVALID != adj_index);
+
+    adj = adj_get(adj_index);
+
+    /*
+     * one time only update. since we don't support chainging the tunnel
+     * src,dst, this is all we need.
+     */
+    ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_MCAST);
+    /*
+     * tunnels can always provide a rewrite.
+     */
+    ASSERT(NULL != rewrite);
+
+    adj_midchain_setup(adj_index, fixup, flags);
+
+    /*
+     * update the adj's rewrite string and build the arc
+     * from the rewrite node to the interface's TX node
+     */
+    adj_nbr_update_rewrite_internal(adj, IP_LOOKUP_NEXT_MCAST_MIDCHAIN,
+                                    adj_get_mcast_node(adj->ia_nh_proto),
+                                    vnet_tx_node_index_for_sw_interface(
+                                        vnet_get_main(),
+                                        adj->rewrite_header.sw_if_index),
+                                    rewrite);
+
+    /*
+     * set the fields corresponding to the mcast IP address rewrite
+     * The mask must be stored in network byte order, since the packet's
+     * IP address will also be in network order.
+     */
+    adj->rewrite_header.dst_mcast_offset = offset;
+    adj->rewrite_header.dst_mcast_mask = clib_host_to_net_u32(mask);
+}
+
 void
 adj_mcast_remove (fib_protocol_t proto,
 		  u32 sw_if_index)
@@ -260,6 +313,24 @@ adj_mcast_interface_delete (vnet_main_t * vnm,
 
 VNET_SW_INTERFACE_ADD_DEL_FUNCTION(adj_mcast_interface_delete);
 
+/**
+ * @brief Walk the multicast Adjacencies on a given interface
+ */
+void
+adj_mcast_walk (u32 sw_if_index,
+                fib_protocol_t proto,
+                adj_walk_cb_t cb,
+                void *ctx)
+{
+    if (vec_len(adj_mcasts[proto]) > sw_if_index)
+    {
+        if (ADJ_INDEX_INVALID != adj_mcasts[proto][sw_if_index])
+        {
+            cb(adj_mcasts[proto][sw_if_index], ctx);
+        }
+    }
+}
+
 u8*
 format_adj_mcast (u8* s, va_list *ap)
 {
@@ -269,6 +340,8 @@ format_adj_mcast (u8* s, va_list *ap)
 
     s = format(s, "%U-mcast: ",
                format_fib_protocol, adj->ia_nh_proto);
+    if (adj->rewrite_header.flags & VNET_REWRITE_HAS_FEATURES)
+        s = format(s, "[features] ");
     s = format (s, "%U",
 		format_vnet_rewrite,
                 &adj->rewrite_header, sizeof (adj->rewrite_data), 0);
@@ -276,6 +349,28 @@ format_adj_mcast (u8* s, va_list *ap)
     return (s);
 }
 
+u8*
+format_adj_mcast_midchain (u8* s, va_list *ap)
+{
+    index_t index = va_arg(*ap, index_t);
+    CLIB_UNUSED(u32 indent) = va_arg(*ap, u32);
+    vnet_main_t * vnm = vnet_get_main();
+    ip_adjacency_t * adj = adj_get(index);
+
+    s = format(s, "%U-mcast-midchain: ",
+               format_fib_protocol, adj->ia_nh_proto);
+    s = format (s, "%U",
+		format_vnet_rewrite,
+		vnm->vlib_main, &adj->rewrite_header,
+                sizeof (adj->rewrite_data), 0);
+    s = format (s, "\n%Ustacked-on:\n%U%U",
+		format_white_space, indent,
+		format_white_space, indent+2,
+		format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2);
+
+    return (s);
+}
+
 
 static void
 adj_dpo_lock (dpo_id_t *dpo)
@@ -293,6 +388,11 @@ const static dpo_vft_t adj_mcast_dpo_vft = {
     .dv_unlock = adj_dpo_unlock,
     .dv_format = format_adj_mcast,
 };
+const static dpo_vft_t adj_mcast_midchain_dpo_vft = {
+    .dv_lock = adj_dpo_lock,
+    .dv_unlock = adj_dpo_unlock,
+    .dv_format = format_adj_mcast_midchain,
+};
 
 /**
  * @brief The per-protocol VLIB graph nodes that are assigned to a mcast
@@ -319,6 +419,31 @@ const static char* const * const adj_mcast_nodes[DPO_PROTO_NUM] =
     [DPO_PROTO_MPLS] = NULL,
 };
 
+/**
+ * @brief The per-protocol VLIB graph nodes that are assigned to a mcast
+ *        object.
+ *
+ * this means that these graph nodes are ones from which a mcast is the
+ * parent object in the DPO-graph.
+ */
+const static char* const adj_mcast_midchain_ip4_nodes[] =
+{
+    "ip4-mcast-midchain",
+    NULL,
+};
+const static char* const adj_mcast_midchain_ip6_nodes[] =
+{
+    "ip6-mcast-midchain",
+    NULL,
+};
+
+const static char* const * const adj_mcast_midchain_nodes[DPO_PROTO_NUM] =
+{
+    [DPO_PROTO_IP4]  = adj_mcast_midchain_ip4_nodes,
+    [DPO_PROTO_IP6]  = adj_mcast_midchain_ip6_nodes,
+    [DPO_PROTO_MPLS] = NULL,
+};
+
 /**
  * @brief Return the size of the adj DB.
  * This is only for testing purposes so an efficient implementation is not needed
@@ -349,5 +474,10 @@ adj_mcast_db_size (void)
 void
 adj_mcast_module_init (void)
 {
-    dpo_register(DPO_ADJACENCY_MCAST, &adj_mcast_dpo_vft, adj_mcast_nodes);
+    dpo_register(DPO_ADJACENCY_MCAST,
+                 &adj_mcast_dpo_vft,
+                 adj_mcast_nodes);
+    dpo_register(DPO_ADJACENCY_MCAST_MIDCHAIN,
+                 &adj_mcast_midchain_dpo_vft,
+                 adj_mcast_midchain_nodes);
 }
diff --git a/src/vnet/adj/adj_mcast.h b/src/vnet/adj/adj_mcast.h
index 40d44313..bfb0d6f6 100644
--- a/src/vnet/adj/adj_mcast.h
+++ b/src/vnet/adj/adj_mcast.h
@@ -26,6 +26,7 @@
 #define __ADJ_MCAST_H__
 
 #include <vnet/adj/adj_types.h>
+#include <vnet/adj/adj_midchain.h>
 
 /**
  * @brief
@@ -68,10 +69,36 @@ extern void adj_mcast_update_rewrite(adj_index_t adj_index,
                                      u8 offset,
                                      u32 mask);
 
+/**
+ * @brief
+ *  Update the rewrite string for an existing adjacecny and
+ *  Convert the adjacency into a midchain
+ *
+ * @param
+ *  The index of the adj to update
+ *
+ * @param
+ *  The new rewrite
+ */
+extern void adj_mcast_midchain_update_rewrite(adj_index_t adj_index,
+                                              adj_midchain_fixup_t fixup,
+                                              adj_flags_t flags,
+                                              u8 *rewrite,
+                                              u8 offset,
+                                              u32 mask);
+/**
+ * @brief Walk the multicast Adjacencies on a given interface
+ */
+extern void adj_mcast_walk (u32 sw_if_index,
+                            fib_protocol_t adj_nh_proto,
+                            adj_walk_cb_t cb,
+                            void *ctx);
+
 /**
  * @brief Format/display a mcast adjacency.
  */
 extern u8* format_adj_mcast(u8* s, va_list *ap);
+extern u8* format_adj_mcast_midchain(u8* s, va_list *ap);
 
 /**
  * @brief Get the sze of the mcast adj DB. Test purposes only.
diff --git a/src/vnet/adj/adj_midchain.c b/src/vnet/adj/adj_midchain.c
index 5756de43..a93a1c3e 100644
--- a/src/vnet/adj/adj_midchain.c
+++ b/src/vnet/adj/adj_midchain.c
@@ -346,7 +346,7 @@ adj_get_midchain_node (vnet_link_t link)
 static u8
 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj)
 {
-  u8 arc = (u8) ~0;
+    u8 arc = (u8) ~0;
     switch (adj->ia_link)
     {
     case VNET_LINK_IP4:
@@ -393,17 +393,14 @@ adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj)
 }
 
 /**
- * adj_nbr_midchain_update_rewrite
+ * adj_midchain_setup
  *
- * Update the adjacency's rewrite string. A NULL string implies the
- * rewrite is reset (i.e. when ARP/ND etnry is gone).
- * NB: the adj being updated may be handling traffic in the DP.
+ * Setup the adj as a mid-chain
  */
 void
-adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
-				 adj_midchain_fixup_t fixup,
-				 adj_flags_t flags,
-				 u8 *rewrite)
+adj_midchain_setup (adj_index_t adj_index,
+                    adj_midchain_fixup_t fixup,
+                    adj_flags_t flags)
 {
     u32 feature_index, tx_node;
     ip_adjacency_t *adj;
@@ -413,16 +410,6 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
 
     adj = adj_get(adj_index);
 
-    /*
-     * one time only update. since we don't support chainging the tunnel
-     * src,dst, this is all we need.
-     */
-    ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP);
-    /*
-     * tunnels can always provide a rewrite.
-     */
-    ASSERT(NULL != rewrite);
-
     adj->sub_type.midchain.fixup_func = fixup;
     adj->ia_flags |= flags;
 
@@ -447,6 +434,38 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
     dpo_stack_from_node(tx_node,
 			&adj->sub_type.midchain.next_dpo,
 			drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link)));
+}
+
+/**
+ * adj_nbr_midchain_update_rewrite
+ *
+ * Update the adjacency's rewrite string. A NULL string implies the
+ * rewrite is reset (i.e. when ARP/ND etnry is gone).
+ * NB: the adj being updated may be handling traffic in the DP.
+ */
+void
+adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
+				 adj_midchain_fixup_t fixup,
+				 adj_flags_t flags,
+				 u8 *rewrite)
+{
+    ip_adjacency_t *adj;
+
+    ASSERT(ADJ_INDEX_INVALID != adj_index);
+
+    adj = adj_get(adj_index);
+
+    /*
+     * one time only update. since we don't support chainging the tunnel
+     * src,dst, this is all we need.
+     */
+    ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP);
+    /*
+     * tunnels can always provide a rewrite.
+     */
+    ASSERT(NULL != rewrite);
+
+    adj_midchain_setup(adj_index, fixup, flags);
 
     /*
      * update the rewirte with the workers paused.
@@ -454,7 +473,7 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index,
     adj_nbr_update_rewrite_internal(adj,
 				    IP_LOOKUP_NEXT_MIDCHAIN,
 				    adj_get_midchain_node(adj->ia_link),
-				    tx_node,
+				    adj_nbr_midchain_get_tx_node(adj),
 				    rewrite);
 }
 
@@ -496,7 +515,8 @@ adj_nbr_midchain_stack (adj_index_t adj_index,
 
     adj = adj_get(adj_index);
 
-    ASSERT(IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index);
+    ASSERT((IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index) ||
+           (IP_LOOKUP_NEXT_MCAST_MIDCHAIN == adj->lookup_next_index));
 
     dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj),
 			&adj->sub_type.midchain.next_dpo,
diff --git a/src/vnet/adj/adj_nbr.c b/src/vnet/adj/adj_nbr.c
index ddacb030..3d450d1f 100644
--- a/src/vnet/adj/adj_nbr.c
+++ b/src/vnet/adj/adj_nbr.c
@@ -195,8 +195,6 @@ adj_nbr_alloc (fib_protocol_t nh_proto,
     adj->ia_link = link_type;
     adj->ia_nh_proto = nh_proto;
     adj->rewrite_header.sw_if_index = sw_if_index;
-    memset(&adj->sub_type.midchain.next_dpo, 0,
-           sizeof(adj->sub_type.midchain.next_dpo));
 
     adj_nbr_evaluate_feature (adj_get_index(adj));
     return (adj);
diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h
index ea3ce093..ed869d1f 100644
--- a/src/vnet/buffer.h
+++ b/src/vnet/buffer.h
@@ -130,6 +130,9 @@ typedef struct
 
 	  /* Rewrite length */
 	  u32 save_rewrite_length;
+
+	  /* MFIB RPF ID */
+	  u32 rpf_id;
 	};
 
 	/* ICMP */
diff --git a/src/vnet/devices/ssvm/node.c b/src/vnet/devices/ssvm/node.c
index 539b4161..b7a8db05 100644
--- a/src/vnet/devices/ssvm/node.c
+++ b/src/vnet/devices/ssvm/node.c
@@ -210,7 +210,7 @@ ssvm_eth_device_input (ssvm_eth_main_t * em,
 	    next0 = SSVM_ETH_INPUT_NEXT_IP4_INPUT;
 	  else if (type0 == ETHERNET_TYPE_IP6)
 	    next0 = SSVM_ETH_INPUT_NEXT_IP6_INPUT;
-	  else if (type0 == ETHERNET_TYPE_MPLS_UNICAST)
+	  else if (type0 == ETHERNET_TYPE_MPLS)
 	    next0 = SSVM_ETH_INPUT_NEXT_MPLS_INPUT;
 
 	  l3_offset0 = ((next0 == SSVM_ETH_INPUT_NEXT_IP4_INPUT ||
diff --git a/src/vnet/dhcp/dhcp6_proxy_node.c b/src/vnet/dhcp/dhcp6_proxy_node.c
index 524cb095..de73154d 100644
--- a/src/vnet/dhcp/dhcp6_proxy_node.c
+++ b/src/vnet/dhcp/dhcp6_proxy_node.c
@@ -883,6 +883,7 @@ dhcp6_proxy_set_server (ip46_address_t *addr,
          mfib_table_entry_update(rx_fib_index,
                                  &all_dhcp_servers,
                                  MFIB_SOURCE_DHCP,
+                                 MFIB_RPF_ID_NONE,
                                  MFIB_ENTRY_FLAG_ACCEPT_ALL_ITF);
          mfib_table_lock(rx_fib_index, FIB_PROTOCOL_IP6);
      }
diff --git a/src/vnet/dpo/dpo.c b/src/vnet/dpo/dpo.c
index d8e075a7..dfc2bd92 100644
--- a/src/vnet/dpo/dpo.c
+++ b/src/vnet/dpo/dpo.c
@@ -37,6 +37,8 @@
 #include <vnet/dpo/classify_dpo.h>
 #include <vnet/dpo/ip_null_dpo.h>
 #include <vnet/dpo/replicate_dpo.h>
+#include <vnet/dpo/interface_dpo.h>
+#include <vnet/dpo/mpls_disposition.h>
 
 /**
  * Array of char* names for the DPO types and protos
@@ -182,6 +184,12 @@ dpo_set (dpo_id_t *dpo,
 	case IP_LOOKUP_NEXT_MIDCHAIN:
 	    dpo->dpoi_type = DPO_ADJACENCY_MIDCHAIN;
 	    break;
+	case IP_LOOKUP_NEXT_MCAST_MIDCHAIN:
+	    dpo->dpoi_type = DPO_ADJACENCY_MCAST_MIDCHAIN;
+	    break;
+	case IP_LOOKUP_NEXT_MCAST:
+	    dpo->dpoi_type = DPO_ADJACENCY_MCAST;
+	    break;
 	default:
 	    break;
 	}
@@ -453,6 +461,8 @@ dpo_module_init (vlib_main_t * vm)
     lookup_dpo_module_init();
     ip_null_dpo_module_init();
     replicate_module_init();
+    interface_dpo_module_init();
+    mpls_disp_dpo_module_init();
 
     return (NULL);
 }
diff --git a/src/vnet/dpo/dpo.h b/src/vnet/dpo/dpo.h
index 48b92d3d..5aa4e2d2 100644
--- a/src/vnet/dpo/dpo.h
+++ b/src/vnet/dpo/dpo.h
@@ -108,12 +108,15 @@ typedef enum dpo_type_t_ {
     DPO_ADJACENCY_MIDCHAIN,
     DPO_ADJACENCY_GLEAN,
     DPO_ADJACENCY_MCAST,
+    DPO_ADJACENCY_MCAST_MIDCHAIN,
     DPO_RECEIVE,
     DPO_LOOKUP,
     DPO_LISP_CP,
     DPO_CLASSIFY,
     DPO_MPLS_LABEL,
+    DPO_MPLS_DISPOSITION,
     DPO_MFIB_ENTRY,
+    DPO_INTERFACE,
     DPO_LAST,
 } __attribute__((packed)) dpo_type_t;
 
@@ -129,6 +132,7 @@ typedef enum dpo_type_t_ {
     [DPO_ADJACENCY_MIDCHAIN] = "dpo-adjacency-midcahin",	\
     [DPO_ADJACENCY_GLEAN] = "dpo-glean",	\
     [DPO_ADJACENCY_MCAST] = "dpo-adj-mcast",	\
+    [DPO_ADJACENCY_MCAST_MIDCHAIN] = "dpo-adj-mcast-midchain",	\
     [DPO_RECEIVE] = "dpo-receive",	\
     [DPO_LOOKUP] = "dpo-lookup",	\
     [DPO_LOAD_BALANCE] = "dpo-load-balance",	\
@@ -136,7 +140,9 @@ typedef enum dpo_type_t_ {
     [DPO_LISP_CP] = "dpo-lisp-cp",	\
     [DPO_CLASSIFY] = "dpo-classify",	\
     [DPO_MPLS_LABEL] = "dpo-mpls-label", \
-    [DPO_MFIB_ENTRY] = "dpo-mfib_entry"	\
+    [DPO_MPLS_DISPOSITION] = "dpo-mpls-diposition", \
+    [DPO_MFIB_ENTRY] = "dpo-mfib_entry", \
+    [DPO_INTERFACE] = "dpo-interface"	\
 }
 
 /**
diff --git a/src/vnet/dpo/interface_dpo.c b/src/vnet/dpo/interface_dpo.c
new file mode 100644
index 00000000..50ca756f
--- /dev/null
+++ b/src/vnet/dpo/interface_dpo.c
@@ -0,0 +1,416 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/dpo/interface_dpo.h>
+#include <vnet/fib/fib_node.h>
+
+/*
+ * The 'DB' of interface DPOs.
+ * There is only one  per-interface per-protocol, so this is a per-interface
+ * vector
+ */
+static index_t *interface_dpo_db[DPO_PROTO_NUM];
+
+static interface_dpo_t *
+interface_dpo_alloc (void)
+{
+    interface_dpo_t *ido;
+
+    pool_get(interface_dpo_pool, ido);
+
+    return (ido);
+}
+
+static inline interface_dpo_t *
+interface_dpo_get_from_dpo (const dpo_id_t *dpo)
+{
+    ASSERT(DPO_INTERFACE == dpo->dpoi_type);
+
+    return (interface_dpo_get(dpo->dpoi_index));
+}
+
+static inline index_t
+interface_dpo_get_index (interface_dpo_t *ido)
+{
+    return (ido - interface_dpo_pool);
+}
+
+static void
+interface_dpo_lock (dpo_id_t *dpo)
+{
+    interface_dpo_t *ido;
+
+    ido = interface_dpo_get_from_dpo(dpo);
+    ido->ido_locks++;
+}
+
+static void
+interface_dpo_unlock (dpo_id_t *dpo)
+{
+    interface_dpo_t *ido;
+
+    ido = interface_dpo_get_from_dpo(dpo);
+    ido->ido_locks--;
+
+    if (0 == ido->ido_locks)
+    {
+	interface_dpo_db[ido->ido_proto][ido->ido_sw_if_index] =
+            INDEX_INVALID;
+        pool_put(interface_dpo_pool, ido);
+    }
+}
+
+/*
+ * interface_dpo_add_or_lock
+ *
+ * Add/create and lock a new or lock an existing for the interface DPO
+ * on the interface and protocol given
+ */
+void
+interface_dpo_add_or_lock (dpo_proto_t proto,
+                           u32 sw_if_index,
+                           dpo_id_t *dpo)
+{
+    interface_dpo_t *ido;
+
+    vec_validate_init_empty(interface_dpo_db[proto],
+                            sw_if_index,
+                            INDEX_INVALID);
+
+    if (INDEX_INVALID == interface_dpo_db[proto][sw_if_index])
+    {
+	ido = interface_dpo_alloc();
+
+        ido->ido_sw_if_index = sw_if_index;
+        ido->ido_proto = proto;
+
+	interface_dpo_db[proto][sw_if_index] =
+            interface_dpo_get_index(ido);
+    }
+    else
+    {
+	ido = interface_dpo_get(interface_dpo_db[proto][sw_if_index]);
+    }
+
+    dpo_set(dpo, DPO_INTERFACE, proto, interface_dpo_get_index(ido));
+}
+
+
+static clib_error_t *
+interface_dpo_interface_state_change (vnet_main_t * vnm,
+                                      u32 sw_if_index,
+                                      u32 flags)
+{
+    /*
+     */
+    return (NULL);
+}
+
+VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION(
+    interface_dpo_interface_state_change);
+
+/**
+ * @brief Registered callback for HW interface state changes
+ */
+static clib_error_t *
+interface_dpo_hw_interface_state_change (vnet_main_t * vnm,
+                                         u32 hw_if_index,
+                                         u32 flags)
+{
+    return (NULL);
+}
+
+VNET_HW_INTERFACE_LINK_UP_DOWN_FUNCTION(
+    interface_dpo_hw_interface_state_change);
+
+static clib_error_t *
+interface_dpo_interface_delete (vnet_main_t * vnm,
+                                u32 sw_if_index,
+                                u32 is_add)
+{
+    return (NULL);
+}
+
+VNET_SW_INTERFACE_ADD_DEL_FUNCTION(
+    interface_dpo_interface_delete);
+
+u8*
+format_interface_dpo (u8* s, va_list *ap)
+{
+    index_t index = va_arg(*ap, index_t);
+    CLIB_UNUSED(u32 indent) = va_arg(*ap, u32);
+    vnet_main_t * vnm = vnet_get_main();
+    interface_dpo_t *ido = interface_dpo_get(index);
+
+    return (format(s, "%U-dpo: %U",
+                   format_vnet_sw_interface_name,
+                   vnm,
+                   vnet_get_sw_interface(vnm, ido->ido_sw_if_index),
+                   format_dpo_proto, ido->ido_proto));
+}
+
+static void
+interface_dpo_mem_show (void)
+{
+    fib_show_memory_usage("Interface",
+			  pool_elts(interface_dpo_pool),
+			  pool_len(interface_dpo_pool),
+			  sizeof(interface_dpo_t));
+}
+
+
+const static dpo_vft_t interface_dpo_vft = {
+    .dv_lock = interface_dpo_lock,
+    .dv_unlock = interface_dpo_unlock,
+    .dv_format = format_interface_dpo,
+    .dv_mem_show = interface_dpo_mem_show,
+};
+
+/**
+ * @brief The per-protocol VLIB graph nodes that are assigned to a glean
+ *        object.
+ *
+ * this means that these graph nodes are ones from which a glean is the
+ * parent object in the DPO-graph.
+ */
+const static char* const interface_dpo_ip4_nodes[] =
+{
+    "interface-dpo-ip4",
+    NULL,
+};
+const static char* const interface_dpo_ip6_nodes[] =
+{
+    "interface-dpo-ip4",
+    NULL,
+};
+
+const static char* const * const interface_dpo_nodes[DPO_PROTO_NUM] =
+{
+    [DPO_PROTO_IP4]  = interface_dpo_ip4_nodes,
+    [DPO_PROTO_IP6]  = interface_dpo_ip6_nodes,
+    [DPO_PROTO_MPLS] = NULL,
+};
+
+void
+interface_dpo_module_init (void)
+{
+    dpo_register(DPO_INTERFACE,
+                 &interface_dpo_vft,
+                 interface_dpo_nodes);
+}
+
+/**
+ * @brief Interface DPO trace data
+ */
+typedef struct interface_dpo_trace_t_
+{
+    u32 sw_if_index;
+} interface_dpo_trace_t;
+
+typedef enum interface_dpo_next_t_
+{
+    INTERFACE_DPO_DROP = 0,
+    INTERFACE_DPO_INPUT = 1,
+} interface_dpo_next_t;
+
+always_inline uword
+interface_dpo_inline (vlib_main_t * vm,
+                      vlib_node_runtime_t * node,
+                      vlib_frame_t * from_frame)
+{
+    u32 n_left_from, next_index, * from, * to_next;
+    u32 cpu_index = os_get_cpu_number();
+    vnet_interface_main_t *im;
+
+    im = &vnet_get_main ()->interface_main;
+    from = vlib_frame_vector_args (from_frame);
+    n_left_from = from_frame->n_vectors;
+
+    next_index = node->cached_next_index;
+
+    while (n_left_from > 0)
+    {
+        u32 n_left_to_next;
+
+        vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
+
+	while (n_left_from >= 4 && n_left_to_next > 2)
+	{
+	    const interface_dpo_t *ido0, *ido1;
+	    u32 bi0, idoi0, bi1, idoi1;
+	    vlib_buffer_t *b0, *b1;
+
+	    bi0 = from[0];
+	    to_next[0] = bi0;
+	    bi1 = from[1];
+	    to_next[1] = bi1;
+	    from += 2;
+	    to_next += 2;
+	    n_left_from -= 2;
+	    n_left_to_next -= 2;
+
+	    b0 = vlib_get_buffer (vm, bi0);
+	    b1 = vlib_get_buffer (vm, bi1);
+
+	    idoi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
+	    idoi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
+	    ido0 = interface_dpo_get(idoi0);
+	    ido1 = interface_dpo_get(idoi1);
+
+	    vnet_buffer(b0)->sw_if_index[VLIB_RX] = ido0->ido_sw_if_index;
+	    vnet_buffer(b1)->sw_if_index[VLIB_RX] = ido1->ido_sw_if_index;
+
+            vlib_increment_combined_counter (im->combined_sw_if_counters
+                                             + VNET_INTERFACE_COUNTER_RX,
+                                             cpu_index,
+                                             ido0->ido_sw_if_index,
+                                             1,
+                                             vlib_buffer_length_in_chain (vm, b0));
+            vlib_increment_combined_counter (im->combined_sw_if_counters
+                                             + VNET_INTERFACE_COUNTER_RX,
+                                             cpu_index,
+                                             ido1->ido_sw_if_index,
+                                             1,
+                                             vlib_buffer_length_in_chain (vm, b1));
+
+	    if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+		interface_dpo_trace_t *tr0;
+
+                tr0 = vlib_add_trace (vm, node, b0, sizeof (*tr0));
+		tr0->sw_if_index = ido0->ido_sw_if_index;
+	    }
+	    if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+		interface_dpo_trace_t *tr1;
+
+                tr1 = vlib_add_trace (vm, node, b1, sizeof (*tr1));
+		tr1->sw_if_index = ido1->ido_sw_if_index;
+	    }
+
+	    vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
+					    n_left_to_next, bi0, bi1,
+                                            INTERFACE_DPO_INPUT,
+                                            INTERFACE_DPO_INPUT);
+	}
+
+	while (n_left_from > 0 && n_left_to_next > 0)
+	{
+	    const interface_dpo_t * ido0;
+	    vlib_buffer_t * b0;
+	    u32 bi0, idoi0;
+
+	    bi0 = from[0];
+	    to_next[0] = bi0;
+	    from += 1;
+	    to_next += 1;
+	    n_left_from -= 1;
+	    n_left_to_next -= 1;
+
+	    b0 = vlib_get_buffer (vm, bi0);
+
+	    idoi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
+	    ido0 = interface_dpo_get(idoi0);
+
+            /* Swap the RX interface of the packet to the one the
+             * interface DPR represents */
+	    vnet_buffer(b0)->sw_if_index[VLIB_RX] = ido0->ido_sw_if_index;
+
+            /* Bump the interface's RX coutners */
+            vlib_increment_combined_counter (im->combined_sw_if_counters
+                                             + VNET_INTERFACE_COUNTER_RX,
+                                             cpu_index,
+                                             ido0->ido_sw_if_index,
+                                             1,
+                                             vlib_buffer_length_in_chain (vm, b0));
+
+	    if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+	    {
+		interface_dpo_trace_t *tr;
+
+                tr = vlib_add_trace (vm, node, b0, sizeof (*tr));
+		tr->sw_if_index = ido0->ido_sw_if_index;
+	    }
+
+	    vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
+					    n_left_to_next, bi0,
+                                            INTERFACE_DPO_INPUT);
+	}
+        vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+    return from_frame->n_vectors;
+}
+
+static u8 *
+format_interface_dpo_trace (u8 * s, va_list * args)
+{
+    CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+    CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+    interface_dpo_trace_t * t = va_arg (*args, interface_dpo_trace_t *);
+    uword indent = format_get_indent (s);
+    s = format (s, "%U sw_if_index:%d",
+                format_white_space, indent,
+                t->sw_if_index);
+    return s;
+}
+
+static uword
+interface_dpo_ip4 (vlib_main_t * vm,
+                   vlib_node_runtime_t * node,
+                   vlib_frame_t * from_frame)
+{
+    return (interface_dpo_inline(vm, node, from_frame));
+}
+
+static uword
+interface_dpo_ip6 (vlib_main_t * vm,
+                   vlib_node_runtime_t * node,
+                   vlib_frame_t * from_frame)
+{
+    return (interface_dpo_inline(vm, node, from_frame));
+}
+
+VLIB_REGISTER_NODE (interface_dpo_ip4_node) = {
+    .function = interface_dpo_ip4,
+    .name = "interface-dpo-ip4",
+    .vector_size = sizeof (u32),
+    .format_trace = format_interface_dpo_trace,
+
+    .n_next_nodes = 2,
+    .next_nodes = {
+        [INTERFACE_DPO_DROP] = "ip4-drop",
+        [INTERFACE_DPO_INPUT] = "ip4-input",
+    },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (interface_dpo_ip4_node,
+                              interface_dpo_ip4)
+
+VLIB_REGISTER_NODE (interface_dpo_ip6_node) = {
+    .function = interface_dpo_ip6,
+    .name = "interface-dpo-ip6",
+    .vector_size = sizeof (u32),
+    .format_trace = format_interface_dpo_trace,
+
+    .n_next_nodes = 2,
+    .next_nodes = {
+        [INTERFACE_DPO_DROP] = "ip6-drop",
+        [INTERFACE_DPO_INPUT] = "ip6-input",
+    },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (interface_dpo_ip6_node,
+                              interface_dpo_ip6)
+
diff --git a/src/vnet/dpo/interface_dpo.h b/src/vnet/dpo/interface_dpo.h
new file mode 100644
index 00000000..1538dfbb
--- /dev/null
+++ b/src/vnet/dpo/interface_dpo.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @brief
+ * The data-path object representing interfaceing the packet, i.e. it's for-us
+ */
+
+#ifndef __INTERFACE_DPO_H__
+#define __INTERFACE_DPO_H__
+
+#include <vnet/dpo/dpo.h>
+
+typedef struct interface_dpo_t_
+{
+    /**
+     * The Software interface index that the packets will be given
+     * as the ingress/rx interface
+     */
+    u32 ido_sw_if_index;
+
+    /**
+     * next VLIB node. A '<proto>-input' node.
+     */
+    u32 ido_next_node;
+
+    /**
+     * DPO protocol that the packets will have as they 'ingress'
+     * on this interface
+     */
+    dpo_proto_t ido_proto;
+
+    /**
+     * number of locks.
+     */
+    u16 ido_locks;
+} interface_dpo_t;
+
+extern void interface_dpo_add_or_lock (dpo_proto_t proto,
+                                       u32 sw_if_index,
+                                       dpo_id_t *dpo);
+
+extern void interface_dpo_module_init(void);
+
+/**
+ * @brief pool of all interface DPOs
+ */
+interface_dpo_t *interface_dpo_pool;
+
+static inline interface_dpo_t *
+interface_dpo_get (index_t index)
+{
+    return (pool_elt_at_index(interface_dpo_pool, index));
+}
+
+#endif
diff --git a/src/vnet/dpo/lookup_dpo.c b/src/vnet/dpo/lookup_dpo.c
index 97ad0a44..e5b00a79 100644
--- a/src/vnet/dpo/lookup_dpo.c
+++ b/src/vnet/dpo/lookup_dpo.c
@@ -21,8 +21,12 @@
 #include <vnet/fib/ip4_fib.h>
 #include <vnet/fib/ip6_fib.h>
 #include <vnet/fib/mpls_fib.h>
+#include <vnet/mfib/mfib_table.h>
+#include <vnet/mfib/ip4_mfib.h>
+#include <vnet/mfib/ip6_mfib.h>
 
 static const char *const lookup_input_names[] = LOOKUP_INPUTS;
+static const char *const lookup_cast_names[] = LOOKUP_CASTS;
 
 /**
  * @brief Enumeration of the lookup subtypes
@@ -31,6 +35,7 @@ typedef enum lookup_sub_type_t_
 {
     LOOKUP_SUB_TYPE_SRC,
     LOOKUP_SUB_TYPE_DST,
+    LOOKUP_SUB_TYPE_DST_MCAST,
     LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE,
 } lookup_sub_type_t;
 #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1)
@@ -67,6 +72,7 @@ lookup_dpo_get_index (lookup_dpo_t *lkd)
 static void
 lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
                           dpo_proto_t proto,
+                          lookup_cast_t cast,
                           lookup_input_t input,
                           lookup_table_t table_config,
                           dpo_id_t *dpo)
@@ -79,6 +85,7 @@ lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
     lkd->lkd_proto = proto;
     lkd->lkd_input = input;
     lkd->lkd_table = table_config;
+    lkd->lkd_cast  = cast;
 
     /*
      * use the input type to select the lookup sub-type
@@ -100,6 +107,10 @@ lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
             type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST];
             break;
         }
+        if (LOOKUP_MULTICAST == cast)
+        {
+            type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST];
+        }
     }
 
     if (0 == type)
@@ -115,20 +126,29 @@ lookup_dpo_add_or_lock_i (fib_node_index_t fib_index,
 void
 lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
                                     dpo_proto_t proto,
+                                    lookup_cast_t cast,
                                     lookup_input_t input,
                                     lookup_table_t table_config,
                                     dpo_id_t *dpo)
 {
     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
     {
-	fib_table_lock(fib_index, dpo_proto_to_fib(proto));
+        if (LOOKUP_UNICAST == cast)
+        {
+            fib_table_lock(fib_index, dpo_proto_to_fib(proto));
+        }
+        else
+        {
+            mfib_table_lock(fib_index, dpo_proto_to_fib(proto));
+        }
     }
-    lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo);
+    lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
 }
 
 void
 lookup_dpo_add_or_lock_w_table_id (u32 table_id,
                                    dpo_proto_t proto,
+                                   lookup_cast_t cast,
                                    lookup_input_t input,
                                    lookup_table_t table_config,
                                    dpo_id_t *dpo)
@@ -137,13 +157,22 @@ lookup_dpo_add_or_lock_w_table_id (u32 table_id,
 
     if (LOOKUP_TABLE_FROM_CONFIG == table_config)
     {
-	fib_index =
-	    fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
-					      table_id);
+        if (LOOKUP_UNICAST == cast)
+        {
+            fib_index =
+                fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
+                                                  table_id);
+        }
+        else
+        {
+            fib_index =
+                mfib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
+                                                   table_id);
+        }
     }
 
     ASSERT(FIB_NODE_INDEX_INVALID != fib_index);
-    lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo);    
+    lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
 }
 
 u8*
@@ -156,16 +185,29 @@ format_lookup_dpo (u8 *s, va_list *args)
 
     if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table)
     {
-        s = format(s, "%s lookup in interface's %U table",
+        s = format(s, "%s,%s lookup in interface's %U table",
                    lookup_input_names[lkd->lkd_input],
+                   lookup_cast_names[lkd->lkd_cast],
                    format_dpo_proto, lkd->lkd_proto);
     }
     else
     {
-	s = format(s, "%s lookup in %U",
-		   lookup_input_names[lkd->lkd_input],
-		   format_fib_table_name, lkd->lkd_fib_index,
-		   dpo_proto_to_fib(lkd->lkd_proto));
+        if (LOOKUP_UNICAST == lkd->lkd_cast)
+        {
+            s = format(s, "%s,%s lookup in %U",
+                       lookup_input_names[lkd->lkd_input],
+                       lookup_cast_names[lkd->lkd_cast],
+                       format_fib_table_name, lkd->lkd_fib_index,
+                       dpo_proto_to_fib(lkd->lkd_proto));
+        }
+        else
+        {
+            s = format(s, "%s,%s lookup in %U",
+                       lookup_input_names[lkd->lkd_input],
+                       lookup_cast_names[lkd->lkd_cast],
+                       format_mfib_table_name, lkd->lkd_fib_index,
+                       dpo_proto_to_fib(lkd->lkd_proto));
+        }
     }
     return (s);
 }
@@ -193,8 +235,16 @@ lookup_dpo_unlock (dpo_id_t *dpo)
     {
         if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table)
         {
-	    fib_table_unlock(lkd->lkd_fib_index,
-			     dpo_proto_to_fib(lkd->lkd_proto));
+            if (LOOKUP_UNICAST == lkd->lkd_cast)
+            {
+                fib_table_unlock(lkd->lkd_fib_index,
+                                 dpo_proto_to_fib(lkd->lkd_proto));
+            }
+            else
+            {
+                mfib_table_unlock(lkd->lkd_fib_index,
+                                  dpo_proto_to_fib(lkd->lkd_proto));
+            }
         }
         pool_put(lookup_dpo_pool, lkd);
     }
@@ -1069,6 +1119,123 @@ VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = {
 };
 VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_itf_node, lookup_mpls_dst_itf)
 
+typedef enum lookup_ip_dst_mcast_next_t_ {
+    LOOKUP_IP_DST_MCAST_NEXT_RPF,
+    LOOKUP_IP_DST_MCAST_N_NEXT,
+} mfib_forward_lookup_next_t;
+
+always_inline uword
+lookup_dpo_ip_dst_mcast_inline (vlib_main_t * vm,
+                                vlib_node_runtime_t * node,
+                                vlib_frame_t * from_frame,
+                                int is_v4)
+{
+    u32 n_left_from, next_index, * from, * to_next;
+
+    from = vlib_frame_vector_args (from_frame);
+    n_left_from = from_frame->n_vectors;
+
+    next_index = LOOKUP_IP_DST_MCAST_NEXT_RPF;
+
+    while (n_left_from > 0)
+    {
+        u32 n_left_to_next;
+
+        vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
+
+        /* while (n_left_from >= 4 && n_left_to_next >= 2) */
+        /*   } */
+
+        while (n_left_from > 0 && n_left_to_next > 0)
+        {
+            u32 bi0, lkdi0, fib_index0,  next0;
+            const lookup_dpo_t * lkd0;
+            fib_node_index_t mfei0;
+            vlib_buffer_t * b0;
+
+            bi0 = from[0];
+            to_next[0] = bi0;
+            from += 1;
+            to_next += 1;
+            n_left_from -= 1;
+            n_left_to_next -= 1;
+
+            b0 = vlib_get_buffer (vm, bi0);
+
+            /* dst lookup was done by mpls lookup */
+            lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
+            lkd0 = lookup_dpo_get(lkdi0);
+            fib_index0 = lkd0->lkd_fib_index;
+            next0 = LOOKUP_IP_DST_MCAST_NEXT_RPF;
+
+            if (is_v4)
+            {
+                ip4_header_t * ip0;
+
+                ip0 = vlib_buffer_get_current (b0);
+                mfei0 = ip4_mfib_table_lookup(ip4_mfib_get(fib_index0),
+                                              &ip0->src_address,
+                                              &ip0->dst_address,
+                                              64);
+                if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+                {
+                    lookup_trace_t *tr = vlib_add_trace (vm, node,
+                                                         b0, sizeof (*tr));
+                    tr->fib_index = fib_index0;
+                    tr->lbi = mfei0;
+                    tr->addr.ip4 = ip0->dst_address;
+                }
+            }
+            else
+            {
+                ip6_header_t * ip0;
+
+                ip0 = vlib_buffer_get_current (b0);
+                mfei0 = ip6_mfib_table_lookup2(ip6_mfib_get(fib_index0),
+                                               &ip0->src_address,
+                                               &ip0->dst_address);
+                if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+                {
+                    lookup_trace_t *tr = vlib_add_trace (vm, node,
+                                                         b0, sizeof (*tr));
+                    tr->fib_index = fib_index0;
+                    tr->lbi = mfei0;
+                    tr->addr.ip6 = ip0->dst_address;
+                }
+            }
+
+            vnet_buffer (b0)->ip.adj_index[VLIB_TX] = mfei0;
+
+           vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
+                                            n_left_to_next, bi0, next0);
+        }
+        vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+    return from_frame->n_vectors;
+}
+
+always_inline uword
+lookup_ip4_dst_mcast (vlib_main_t * vm,
+                      vlib_node_runtime_t * node,
+                      vlib_frame_t * from_frame)
+{
+    return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 1));
+}
+
+VLIB_REGISTER_NODE (lookup_ip4_dst_mcast_node) = {
+    .function = lookup_ip4_dst_mcast,
+    .name = "lookup-ip4-dst-mcast",
+    .vector_size = sizeof (u32),
+
+    .format_trace = format_lookup_trace,
+    .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT,
+    .next_nodes = {
+        [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip4-mfib-forward-rpf",
+    },
+};
+VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_mcast_node,
+                              lookup_ip4_dst_mcast)
+
 static void
 lookup_dpo_mem_show (void)
 {
@@ -1129,6 +1296,22 @@ const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] =
     [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes,
 };
 
+const static char* const lookup_dst_mcast_ip4_nodes[] =
+{
+    "lookup-ip4-dst-mcast",
+    NULL,
+};
+const static char* const lookup_dst_mcast_ip6_nodes[] =
+{
+    "lookup-ip6-dst-mcast",
+    NULL,
+};
+const static char* const * const lookup_dst_mcast_nodes[DPO_PROTO_NUM] =
+{
+    [DPO_PROTO_IP4]  = lookup_dst_mcast_ip4_nodes,
+    [DPO_PROTO_IP6]  = lookup_dst_mcast_ip6_nodes,
+};
+
 const static char* const lookup_dst_from_interface_ip4_nodes[] =
 {
     "lookup-ip4-dst-itf",
@@ -1168,6 +1351,8 @@ lookup_dpo_module_init (void)
         dpo_register_new_type(&lkd_vft, lookup_src_nodes);
     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] =
         dpo_register_new_type(&lkd_vft, lookup_dst_nodes);
+    lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST] =
+        dpo_register_new_type(&lkd_vft, lookup_dst_mcast_nodes);
     lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] =
         dpo_register_new_type(&lkd_vft, lookup_dst_from_interface_nodes);
 }
diff --git a/src/vnet/dpo/lookup_dpo.h b/src/vnet/dpo/lookup_dpo.h
index ff283388..7dfd0385 100644
--- a/src/vnet/dpo/lookup_dpo.h
+++ b/src/vnet/dpo/lookup_dpo.h
@@ -46,6 +46,19 @@ typedef enum lookup_table_t_ {
     [LOOKUP_INPUT_DST_ADDR] = "table-configured",         \
 }
 
+/**
+ * Switch to use the packet's source or destination address for lookup
+ */
+typedef enum lookup_cast_t_ {
+    LOOKUP_UNICAST,
+    LOOKUP_MULTICAST,
+} __attribute__ ((packed)) lookup_cast_t;
+
+#define LOOKUP_CASTS {                 \
+    [LOOKUP_UNICAST]   = "unicast",    \
+    [LOOKUP_MULTICAST] = "multicast",  \
+}
+
 /**
  * A representation of an MPLS label for imposition in the data-path
  */
@@ -73,6 +86,11 @@ typedef struct lookup_dpo_t
      */
     lookup_table_t lkd_table;
 
+    /**
+     * Unicast of rmulticast FIB lookup
+     */
+    lookup_cast_t lkd_cast;
+
     /**
      * Number of locks
      */
@@ -81,11 +99,13 @@ typedef struct lookup_dpo_t
 
 extern void lookup_dpo_add_or_lock_w_fib_index(fib_node_index_t fib_index,
                                                dpo_proto_t proto,
+                                               lookup_cast_t cast,
                                                lookup_input_t input,
                                                lookup_table_t table,
                                                dpo_id_t *dpo);
 extern void lookup_dpo_add_or_lock_w_table_id(u32 table_id,
                                               dpo_proto_t proto,
+                                              lookup_cast_t cast,
                                               lookup_input_t input,
                                               lookup_table_t table,
                                               dpo_id_t *dpo);
diff --git a/src/vnet/dpo/mpls_disposition.c b/src/vnet/dpo/mpls_disposition.c
new file mode 100644
index 00000000..5dc33fcf
--- /dev/null
+++ b/src/vnet/dpo/mpls_disposition.c
@@ -0,0 +1,364 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vnet/ip/ip.h>
+#include <vnet/dpo/mpls_disposition.h>
+#include <vnet/mpls/mpls.h>
+
+/*
+ * pool of all MPLS Label DPOs
+ */
+mpls_disp_dpo_t *mpls_disp_dpo_pool;
+
+static mpls_disp_dpo_t *
+mpls_disp_dpo_alloc (void)
+{
+    mpls_disp_dpo_t *mdd;
+
+    pool_get_aligned(mpls_disp_dpo_pool, mdd, CLIB_CACHE_LINE_BYTES);
+    memset(mdd, 0, sizeof(*mdd));
+
+    dpo_reset(&mdd->mdd_dpo);
+
+    return (mdd);
+}
+
+static index_t
+mpls_disp_dpo_get_index (mpls_disp_dpo_t *mdd)
+{
+    return (mdd - mpls_disp_dpo_pool);
+}
+
+index_t
+mpls_disp_dpo_create (dpo_proto_t payload_proto,
+                      fib_rpf_id_t rpf_id,
+                      const dpo_id_t *dpo)
+{
+    mpls_disp_dpo_t *mdd;
+
+    mdd = mpls_disp_dpo_alloc();
+
+    mdd->mdd_payload_proto = payload_proto;
+    mdd->mdd_rpf_id = rpf_id;
+
+    dpo_stack(DPO_MPLS_DISPOSITION,
+              mdd->mdd_payload_proto,
+              &mdd->mdd_dpo,
+              dpo);
+
+    return (mpls_disp_dpo_get_index(mdd));
+}
+
+u8*
+format_mpls_disp_dpo (u8 *s, va_list *args)
+{
+    index_t index = va_arg (*args, index_t);
+    u32 indent = va_arg (*args, u32);
+    mpls_disp_dpo_t *mdd;
+
+    mdd = mpls_disp_dpo_get(index);
+
+    s = format(s, "mpls-disposition:[%d]:[%U]",
+               index,
+               format_dpo_proto, mdd->mdd_payload_proto);
+
+    s = format(s, "\n%U", format_white_space, indent);
+    s = format(s, "%U", format_dpo_id, &mdd->mdd_dpo, indent+2);
+
+    return (s);
+}
+
+static void
+mpls_disp_dpo_lock (dpo_id_t *dpo)
+{
+    mpls_disp_dpo_t *mdd;
+
+    mdd = mpls_disp_dpo_get(dpo->dpoi_index);
+
+    mdd->mdd_locks++;
+}
+
+static void
+mpls_disp_dpo_unlock (dpo_id_t *dpo)
+{
+    mpls_disp_dpo_t *mdd;
+
+    mdd = mpls_disp_dpo_get(dpo->dpoi_index);
+
+    mdd->mdd_locks--;
+
+    if (0 == mdd->mdd_locks)
+    {
+	dpo_reset(&mdd->mdd_dpo);
+	pool_put(mpls_disp_dpo_pool, mdd);
+    }
+}
+
+/**
+ * @brief A struct to hold tracing information for the MPLS label disposition
+ * node.
+ */
+typedef struct mpls_label_disposition_trace_t_
+{
+    index_t mdd;
+} mpls_label_disposition_trace_t;
+
+always_inline uword
+mpls_label_disposition_inline (vlib_main_t * vm,
+                              vlib_node_runtime_t * node,
+                              vlib_frame_t * from_frame,
+                              u8 payload_is_ip4,
+                              u8 payload_is_ip6)
+{
+    u32 n_left_from, next_index, * from, * to_next;
+
+    from = vlib_frame_vector_args (from_frame);
+    n_left_from = from_frame->n_vectors;
+
+    next_index = node->cached_next_index;
+
+    while (n_left_from > 0)
+    {
+        u32 n_left_to_next;
+
+        vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next);
+
+        while (n_left_from >= 4 && n_left_to_next >= 2)
+        {
+            mpls_disp_dpo_t *mdd0, *mdd1;
+            u32 bi0, mddi0, bi1, mddi1;
+            vlib_buffer_t * b0, *b1;
+            u32 next0, next1;
+
+            bi0 = to_next[0] = from[0];
+            bi1 = to_next[1] = from[1];
+
+            /* Prefetch next iteration. */
+            {
+                vlib_buffer_t * p2, * p3;
+
+                p2 = vlib_get_buffer (vm, from[2]);
+                p3 = vlib_get_buffer (vm, from[3]);
+
+                vlib_prefetch_buffer_header (p2, STORE);
+                vlib_prefetch_buffer_header (p3, STORE);
+
+                CLIB_PREFETCH (p2->data, sizeof (ip6_header_t), STORE);
+                CLIB_PREFETCH (p3->data, sizeof (ip6_header_t), STORE);
+            }
+
+            from += 2;
+            to_next += 2;
+            n_left_from -= 2;
+            n_left_to_next -= 2;
+
+            b0 = vlib_get_buffer (vm, bi0);
+            b1 = vlib_get_buffer (vm, bi1);
+
+            /* dst lookup was done by ip4 lookup */
+            mddi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
+            mddi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX];
+            mdd0 = mpls_disp_dpo_get(mddi0);
+            mdd1 = mpls_disp_dpo_get(mddi1);
+
+            if (payload_is_ip4)
+            {
+                /*
+                 * decrement the TTL on ingress to the LSP
+                 */
+            }
+            else if (payload_is_ip6)
+            {
+                /*
+                 * decrement the TTL on ingress to the LSP
+                 */
+            }
+ 
+            next0 = mdd0->mdd_dpo.dpoi_next_node;
+            next1 = mdd1->mdd_dpo.dpoi_next_node;
+            vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mdd0->mdd_dpo.dpoi_index;
+            vnet_buffer(b1)->ip.adj_index[VLIB_TX] = mdd1->mdd_dpo.dpoi_index;
+            vnet_buffer(b0)->ip.rpf_id = mdd0->mdd_rpf_id;
+            vnet_buffer(b1)->ip.rpf_id = mdd1->mdd_rpf_id;
+
+            if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+                mpls_label_disposition_trace_t *tr =
+                    vlib_add_trace (vm, node, b0, sizeof (*tr));
+
+                tr->mdd = mddi0;
+            }
+            if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
+            {
+                mpls_label_disposition_trace_t *tr =
+                    vlib_add_trace (vm, node, b1, sizeof (*tr));
+                tr->mdd = mddi1;
+            }
+
+            vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next,
+                                            n_left_to_next,
+                                            bi0, bi1, next0, next1);
+        }
+
+        while (n_left_from > 0 && n_left_to_next > 0)
+        {
+            mpls_disp_dpo_t *mdd0;
+            vlib_buffer_t * b0;
+            u32 bi0, mddi0;
+            u32 next0;
+
+            bi0 = from[0];
+            to_next[0] = bi0;
+            from += 1;
+            to_next += 1;
+            n_left_from -= 1;
+            n_left_to_next -= 1;
+
+            b0 = vlib_get_buffer (vm, bi0);
+
+            /* dst lookup was done by ip4 lookup */
+            mddi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX];
+            mdd0 = mpls_disp_dpo_get(mddi0);
+
+            if (payload_is_ip4)
+            {
+                /*
+                 * decrement the TTL on ingress to the LSP
+                 */
+            }
+            else if (payload_is_ip6)
+            {
+                /*
+                 * decrement the TTL on ingress to the LSP
+                 */
+            }
+            else
+            {
+            }
+
+            next0 = mdd0->mdd_dpo.dpoi_next_node;
+            vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mdd0->mdd_dpo.dpoi_index;
+            vnet_buffer(b0)->ip.rpf_id = mdd0->mdd_rpf_id;
+
+            if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+                mpls_label_disposition_trace_t *tr =
+                    vlib_add_trace (vm, node, b0, sizeof (*tr));
+                tr->mdd = mddi0;
+            }
+
+            vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next,
+                                            n_left_to_next, bi0, next0);
+        }
+        vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+    }
+    return from_frame->n_vectors;
+}
+
+static u8 *
+format_mpls_label_disposition_trace (u8 * s, va_list * args)
+{
+    CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+    CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+    CLIB_UNUSED (mpls_label_disposition_trace_t * t);
+
+    t = va_arg (*args, mpls_label_disposition_trace_t *);
+
+    s = format(s, "disp:%d", t->mdd);
+    return (s);
+}
+
+static uword
+ip4_mpls_label_disposition (vlib_main_t * vm,
+                           vlib_node_runtime_t * node,
+                           vlib_frame_t * frame)
+{
+    return (mpls_label_disposition_inline(vm, node, frame, 1, 0));
+}
+
+VLIB_REGISTER_NODE (ip4_mpls_label_disposition_node) = {
+    .function = ip4_mpls_label_disposition,
+    .name = "ip4-mpls-label-disposition",
+    .vector_size = sizeof (u32),
+
+    .format_trace = format_mpls_label_disposition_trace,
+    .n_next_nodes = 1,
+    .next_nodes = {
+        [0] = "ip4-drop",
+    }
+};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_mpls_label_disposition_node,
+                              ip4_mpls_label_disposition)
+
+static uword
+ip6_mpls_label_disposition (vlib_main_t * vm,
+                           vlib_node_runtime_t * node,
+                           vlib_frame_t * frame)
+{
+    return (mpls_label_disposition_inline(vm, node, frame, 0, 1));
+}
+
+VLIB_REGISTER_NODE (ip6_mpls_label_disposition_node) = {
+    .function = ip6_mpls_label_disposition,
+    .name = "ip6-mpls-label-disposition",
+    .vector_size = sizeof (u32),
+
+    .format_trace = format_mpls_label_disposition_trace,
+    .n_next_nodes = 1,
+    .next_nodes = {
+        [0] = "ip6-drop",
+    }
+};
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_mpls_label_disposition_node,
+                              ip6_mpls_label_disposition)
+
+static void
+mpls_disp_dpo_mem_show (void)
+{
+    fib_show_memory_usage("MPLS label",
+			  pool_elts(mpls_disp_dpo_pool),
+			  pool_len(mpls_disp_dpo_pool),
+			  sizeof(mpls_disp_dpo_t));
+}
+
+const static dpo_vft_t mdd_vft = {
+    .dv_lock = mpls_disp_dpo_lock,
+    .dv_unlock = mpls_disp_dpo_unlock,
+    .dv_format = format_mpls_disp_dpo,
+    .dv_mem_show = mpls_disp_dpo_mem_show,
+};
+
+const static char* const mpls_label_disp_ip4_nodes[] =
+{
+    "ip4-mpls-label-disposition",
+    NULL,
+};
+const static char* const mpls_label_disp_ip6_nodes[] =
+{
+    "ip6-mpls-label-disposition",
+    NULL,
+};
+const static char* const * const mpls_label_disp_nodes[DPO_PROTO_NUM] =
+{
+    [DPO_PROTO_IP4]  = mpls_label_disp_ip4_nodes,
+    [DPO_PROTO_IP6]  = mpls_label_disp_ip6_nodes,
+};
+
+
+void
+mpls_disp_dpo_module_init (void)
+{
+    dpo_register(DPO_MPLS_DISPOSITION, &mdd_vft, mpls_label_disp_nodes);
+}
diff --git a/src/vnet/dpo/mpls_disposition.h b/src/vnet/dpo/mpls_disposition.h
new file mode 100644
index 00000000..9c015083
--- /dev/null
+++ b/src/vnet/dpo/mpls_disposition.h
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MPLS_DISP_DPO_H__
+#define __MPLS_DISP_DPO_H__
+
+#include <vnet/vnet.h>
+#include <vnet/mpls/packet.h>
+#include <vnet/dpo/dpo.h>
+#include <vnet/mfib/mfib_types.h>
+
+/**
+ * A representation of an MPLS label for imposition in the data-path
+ */
+typedef struct mpls_disp_dpo_t
+{
+    /**
+     * Next DPO in the graph
+     */
+    dpo_id_t mdd_dpo;
+
+    /**
+     * The protocol of the payload/packets that are being encapped
+     */
+    dpo_proto_t mdd_payload_proto;
+
+    /**
+     * RPF-ID (if this is an mcast disposition)
+     */
+    fib_rpf_id_t mdd_rpf_id;
+
+    /**
+     * Number of locks/users of the label
+     */
+    u16 mdd_locks;
+} mpls_disp_dpo_t;
+
+/**
+ * @brief Assert that the MPLS label object is less than a cache line in size.
+ * Should this get any bigger then we will need to reconsider how many labels
+ * can be pushed in one object.
+ */
+_Static_assert((sizeof(mpls_disp_dpo_t) <= CLIB_CACHE_LINE_BYTES),
+	       "MPLS Disposition DPO is larger than one cache line.");
+
+/**
+ * @brief Create an MPLS label object
+ *
+ * @param payload_proto The ptocool of the payload packets that will
+ *                      be imposed with this label header.
+ * @param dpo The parent of the created MPLS label object
+ */
+extern index_t mpls_disp_dpo_create(dpo_proto_t payload_proto,
+                                    fib_rpf_id_t rpf_id,
+                                    const dpo_id_t *dpo);
+
+extern u8* format_mpls_disp_dpo(u8 *s, va_list *args);
+
+
+/*
+ * Encapsulation violation for fast data-path access
+ */
+extern mpls_disp_dpo_t *mpls_disp_dpo_pool;
+
+static inline mpls_disp_dpo_t *
+mpls_disp_dpo_get (index_t index)
+{
+    return (pool_elt_at_index(mpls_disp_dpo_pool, index));
+}
+
+extern void mpls_disp_dpo_module_init(void);
+
+#endif
diff --git a/src/vnet/dpo/mpls_label_dpo.c b/src/vnet/dpo/mpls_label_dpo.c
index be9b2850..4d84b900 100644
--- a/src/vnet/dpo/mpls_label_dpo.c
+++ b/src/vnet/dpo/mpls_label_dpo.c
@@ -562,7 +562,7 @@ VLIB_REGISTER_NODE (mpls_label_imposition_node) = {
     .format_trace = format_mpls_label_imposition_trace,
     .n_next_nodes = 1,
     .next_nodes = {
-        [0] = "error-drop",
+        [0] = "mpls-drop",
     }
 };
 VLIB_NODE_FUNCTION_MULTIARCH (mpls_label_imposition_node,
@@ -584,7 +584,7 @@ VLIB_REGISTER_NODE (ip4_mpls_label_imposition_node) = {
     .format_trace = format_mpls_label_imposition_trace,
     .n_next_nodes = 1,
     .next_nodes = {
-        [0] = "error-drop",
+        [0] = "ip4-drop",
     }
 };
 VLIB_NODE_FUNCTION_MULTIARCH (ip4_mpls_label_imposition_node,
@@ -606,7 +606,7 @@ VLIB_REGISTER_NODE (ip6_mpls_label_imposition_node) = {
     .format_trace = format_mpls_label_imposition_trace,
     .n_next_nodes = 1,
     .next_nodes = {
-        [0] = "error-drop",
+        [0] = "ip6-drop",
     }
 };
 VLIB_NODE_FUNCTION_MULTIARCH (ip6_mpls_label_imposition_node,
diff --git a/src/vnet/dpo/replicate_dpo.c b/src/vnet/dpo/replicate_dpo.c
index e25ceae9..9fdb9a05 100644
--- a/src/vnet/dpo/replicate_dpo.c
+++ b/src/vnet/dpo/replicate_dpo.c
@@ -17,6 +17,7 @@
 #include <vnet/dpo/replicate_dpo.h>
 #include <vnet/dpo/drop_dpo.h>
 #include <vnet/adj/adj.h>
+#include <vnet/mpls/mpls_types.h>
 
 #undef REP_DEBUG
 
@@ -106,6 +107,7 @@ replicate_format (index_t repi,
     dpo_id_t *buckets;
     u32 i;
 
+    repi &= ~MPLS_IS_REPLICATE;
     rep = replicate_get(repi);
     vlib_get_combined_counter(&(replicate_main.repm_counters), repi, &to);
     buckets = replicate_get_buckets(rep);
@@ -187,6 +189,7 @@ replicate_set_bucket (index_t repi,
     replicate_t *rep;
     dpo_id_t *buckets;
 
+    repi &= ~MPLS_IS_REPLICATE;
     rep = replicate_get(repi);
     buckets = replicate_get_buckets(rep);
 
@@ -199,11 +202,13 @@ int
 replicate_is_drop (const dpo_id_t *dpo)
 {
     replicate_t *rep;
+    index_t repi;
 
     if (DPO_REPLICATE != dpo->dpoi_type)
         return (0);
 
-    rep = replicate_get(dpo->dpoi_index);
+    repi = dpo->dpoi_index & ~MPLS_IS_REPLICATE;
+    rep = replicate_get(repi);
 
     if (1 == rep->rep_n_buckets)
     {
@@ -218,6 +223,7 @@ replicate_get_bucket (index_t repi,
 {
     replicate_t *rep;
 
+    repi &= ~MPLS_IS_REPLICATE;
     rep = replicate_get(repi);
 
     return (replicate_get_bucket_i(rep, bucket));
@@ -288,9 +294,11 @@ replicate_multipath_update (const dpo_id_t *dpo,
     dpo_id_t *tmp_dpo;
     u32 ii, n_buckets;
     replicate_t *rep;
+    index_t repi;
 
     ASSERT(DPO_REPLICATE == dpo->dpoi_type);
-    rep = replicate_get(dpo->dpoi_index);
+    repi = dpo->dpoi_index & ~MPLS_IS_REPLICATE;
+    rep = replicate_get(repi);
     nhs = replicate_multipath_next_hop_fixup(next_hops,
                                              rep->rep_proto);
     n_buckets = vec_len(nhs);
@@ -718,7 +726,7 @@ format_replicate_trace (u8 * s, va_list * args)
 
   s = format (s, "replicate: %d via %U",
               t->rep_index,
-              format_dpo_id, &t->dpo);
+              format_dpo_id, &t->dpo, 0);
   return s;
 }
 
@@ -731,7 +739,7 @@ ip4_replicate (vlib_main_t * vm,
 }
 
 /**
- * @brief
+ * @brief IP4 replication node
  */
 VLIB_REGISTER_NODE (ip4_replicate_node) = {
   .function = ip4_replicate,
@@ -744,7 +752,7 @@ VLIB_REGISTER_NODE (ip4_replicate_node) = {
   .format_trace = format_replicate_trace,
   .n_next_nodes = 1,
   .next_nodes = {
-      [0] = "error-drop",
+      [0] = "ip4-drop",
   },
 };
 
@@ -757,7 +765,7 @@ ip6_replicate (vlib_main_t * vm,
 }
 
 /**
- * @brief
+ * @brief IPv6 replication node
  */
 VLIB_REGISTER_NODE (ip6_replicate_node) = {
   .function = ip6_replicate,
@@ -770,7 +778,33 @@ VLIB_REGISTER_NODE (ip6_replicate_node) = {
   .format_trace = format_replicate_trace,
   .n_next_nodes = 1,
   .next_nodes = {
-      [0] = "error-drop",
+      [0] = "ip6-drop",
+  },
+};
+
+static uword
+mpls_replicate (vlib_main_t * vm,
+                vlib_node_runtime_t * node,
+                vlib_frame_t * frame)
+{
+    return (replicate_inline (vm, node, frame));
+}
+
+/**
+ * @brief MPLS replication node
+ */
+VLIB_REGISTER_NODE (mpls_replicate_node) = {
+  .function = mpls_replicate,
+  .name = "mpls-replicate",
+  .vector_size = sizeof (u32),
+
+  .n_errors = ARRAY_LEN(replicate_dpo_error_strings),
+  .error_strings = replicate_dpo_error_strings,
+
+  .format_trace = format_replicate_trace,
+  .n_next_nodes = 1,
+  .next_nodes = {
+      [0] = "mpls-drop",
   },
 };
 
diff --git a/src/vnet/dpo/replicate_dpo.h b/src/vnet/dpo/replicate_dpo.h
index 77273015..7383184a 100644
--- a/src/vnet/dpo/replicate_dpo.h
+++ b/src/vnet/dpo/replicate_dpo.h
@@ -25,6 +25,7 @@
 #include <vnet/dpo/dpo.h>
 #include <vnet/dpo/load_balance.h>
 #include <vnet/fib/fib_types.h>
+#include <vnet/mpls/mpls_types.h>
 
 /**
  * replicate main
@@ -119,6 +120,7 @@ extern replicate_t *replicate_pool;
 static inline replicate_t*
 replicate_get (index_t repi)
 {
+    repi &= ~MPLS_IS_REPLICATE;
     return (pool_elt_at_index(replicate_pool, repi));
 }
 
diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c
index c74a097e..dd509193 100644
--- a/src/vnet/ethernet/arp.c
+++ b/src/vnet/ethernet/arp.c
@@ -507,6 +507,7 @@ arp_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai)
     case IP_LOOKUP_NEXT_PUNT:
     case IP_LOOKUP_NEXT_LOCAL:
     case IP_LOOKUP_NEXT_REWRITE:
+    case IP_LOOKUP_NEXT_MCAST_MIDCHAIN:
     case IP_LOOKUP_NEXT_MIDCHAIN:
     case IP_LOOKUP_NEXT_ICMP_ERROR:
     case IP_LOOKUP_N_NEXT:
diff --git a/src/vnet/ethernet/interface.c b/src/vnet/ethernet/interface.c
index 335e3f9f..9ac30bc6 100644
--- a/src/vnet/ethernet/interface.c
+++ b/src/vnet/ethernet/interface.c
@@ -115,7 +115,7 @@ ethernet_build_rewrite (vnet_main_t * vnm,
 #define _(a,b) case VNET_LINK_##a: type = ETHERNET_TYPE_##b; break
       _(IP4, IP4);
       _(IP6, IP6);
-      _(MPLS, MPLS_UNICAST);
+      _(MPLS, MPLS);
       _(ARP, ARP);
 #undef _
     default:
diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c
index f7787ed2..5305012f 100755
--- a/src/vnet/ethernet/node.c
+++ b/src/vnet/ethernet/node.c
@@ -249,7 +249,7 @@ determine_next_node (ethernet_main_t * em,
     {
       *next0 = em->l3_next.input_next_ip6;
     }
-  else if (type0 == ETHERNET_TYPE_MPLS_UNICAST)
+  else if (type0 == ETHERNET_TYPE_MPLS)
     {
       *next0 = em->l3_next.input_next_mpls;
 
@@ -1252,7 +1252,7 @@ next_by_ethertype_register (next_by_ethertype_t * l3_next,
 	{
 	  l3_next->input_next_ip6 = next_index;
 	}
-      else if (ethertype == ETHERNET_TYPE_MPLS_UNICAST)
+      else if (ethertype == ETHERNET_TYPE_MPLS)
 	{
 	  l3_next->input_next_mpls = next_index;
 	}
diff --git a/src/vnet/ethernet/types.def b/src/vnet/ethernet/types.def
index 643f3152..7dab8ee1 100644
--- a/src/vnet/ethernet/types.def
+++ b/src/vnet/ethernet/types.def
@@ -85,8 +85,8 @@ ethernet_type (0x876D, SECURE_DATA)
 ethernet_type (0x8808, MAC_CONTROL)
 ethernet_type (0x8809, SLOW_PROTOCOLS)
 ethernet_type (0x880B, PPP)
-ethernet_type (0x8847, MPLS_UNICAST)
-ethernet_type (0x8848, MPLS_MULTICAST)
+ethernet_type (0x8847, MPLS)
+ethernet_type (0x8848, MPLS_UPSTREAM_ASSIGNED)
 ethernet_type (0x8863, PPPOE_DISCOVERY)
 ethernet_type (0x8864, PPPOE_SESSION)
 ethernet_type (0x886D, INTEL_ANS)
diff --git a/src/vnet/fib/fib_api.h b/src/vnet/fib/fib_api.h
index f8275317..10d0cb58 100644
--- a/src/vnet/fib/fib_api.h
+++ b/src/vnet/fib/fib_api.h
@@ -24,6 +24,7 @@ add_del_route_check (fib_protocol_t table_proto,
 		     fib_protocol_t next_hop_table_proto,
 		     u32 next_hop_table_id,
 		     u8 create_missing_tables,
+                     u8 is_rpf_id,
 		     u32 * fib_index, u32 * next_hop_fib_index);
 
 int
@@ -33,10 +34,13 @@ add_del_route_t_handler (u8 is_multipath,
 			 u8 is_unreach,
 			 u8 is_prohibit,
 			 u8 is_local,
+			 u8 is_multicast,
 			 u8 is_classify,
 			 u32 classify_table_index,
 			 u8 is_resolve_host,
 			 u8 is_resolve_attached,
+			 u8 is_interface_rx,
+                         u8 is_rpf_id,
 			 u32 fib_index,
 			 const fib_prefix_t * prefix,
 			 u8 next_hop_proto_is_ip4,
diff --git a/src/vnet/fib/fib_entry.c b/src/vnet/fib/fib_entry.c
index dac1fce9..6f811aa1 100644
--- a/src/vnet/fib/fib_entry.c
+++ b/src/vnet/fib/fib_entry.c
@@ -75,13 +75,7 @@ fib_entry_get_default_chain_type (const fib_entry_t *fib_entry)
 	return (FIB_FORW_CHAIN_TYPE_UNICAST_IP6);
     case FIB_PROTOCOL_MPLS:
 	if (MPLS_EOS == fib_entry->fe_prefix.fp_eos)
-	    /*
-	     * If the entry being asked is a eos-MPLS label entry,
-	     * then use the payload-protocol field, that we stashed there
-	     * for just this purpose
-	     */
-	    return (fib_forw_chain_type_from_dpo_proto(
-			fib_entry->fe_prefix.fp_payload_proto));
+	    return (FIB_FORW_CHAIN_TYPE_MPLS_EOS);
 	else
 	    return (FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS);
     }
@@ -370,6 +364,35 @@ fib_entry_contribute_urpf (fib_node_index_t entry_index,
     return (fib_path_list_contribute_urpf(fib_entry->fe_parent, urpf));
 }
 
+/*
+ * If the client is request a chain for multicast forwarding then swap
+ * the chain type to one that can provide such transport.
+ */
+static fib_forward_chain_type_t
+fib_entry_chain_type_mcast_to_ucast (fib_forward_chain_type_t fct)
+{
+    switch (fct)
+    {
+    case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
+    case FIB_FORW_CHAIN_TYPE_MCAST_IP6:
+        /*
+         * we can only transport IP multicast packets if there is an
+         * LSP.
+         */
+        fct = FIB_FORW_CHAIN_TYPE_MPLS_EOS;
+        break;
+    case FIB_FORW_CHAIN_TYPE_MPLS_EOS:
+    case FIB_FORW_CHAIN_TYPE_UNICAST_IP4:
+    case FIB_FORW_CHAIN_TYPE_UNICAST_IP6:
+    case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS:
+    case FIB_FORW_CHAIN_TYPE_ETHERNET:
+    case FIB_FORW_CHAIN_TYPE_NSH:
+        break;
+    }
+
+    return (fct);
+}
+
 /*
  * fib_entry_contribute_forwarding
  *
@@ -385,6 +408,11 @@ fib_entry_contribute_forwarding (fib_node_index_t fib_entry_index,
 
     fib_entry = fib_entry_get(fib_entry_index);
 
+    /*
+     * mfib children ask for mcast chains. fix these to the appropriate ucast types.
+     */
+    fct = fib_entry_chain_type_mcast_to_ucast(fct);
+
     if (fct == fib_entry_get_default_chain_type(fib_entry))
     {
         dpo_copy(dpo, &fib_entry->fe_lb);
@@ -414,6 +442,11 @@ fib_entry_contribute_forwarding (fib_node_index_t fib_entry_index,
 
         dpo_copy(dpo, &fed->fd_dpo);
     }
+    /*
+     * don't allow the special index indicating replicate.vs.load-balance
+     * to escape to the clients
+     */
+    dpo->dpoi_index &= ~MPLS_IS_REPLICATE;
 }
 
 const dpo_id_t *
diff --git a/src/vnet/fib/fib_entry.h b/src/vnet/fib/fib_entry.h
index a3f75e60..b17a0b64 100644
--- a/src/vnet/fib/fib_entry.h
+++ b/src/vnet/fib/fib_entry.h
@@ -192,6 +192,11 @@ typedef enum fib_entry_attribute_t_ {
      * The prefix/address is local to this device
      */
     FIB_ENTRY_ATTRIBUTE_LOCAL,
+    /**
+     * The prefix/address is a multicast prefix.
+     *  this aplies only to MPLS. IP multicast is handled by mfib
+     */
+    FIB_ENTRY_ATTRIBUTE_MULTICAST,
     /**
      * The prefix/address exempted from loose uRPF check
      * To be used with caution
@@ -200,7 +205,7 @@ typedef enum fib_entry_attribute_t_ {
     /**
      * Marker. add new entries before this one.
      */
-    FIB_ENTRY_ATTRIBUTE_LAST = FIB_ENTRY_ATTRIBUTE_URPF_EXEMPT,
+    FIB_ENTRY_ATTRIBUTE_LAST = FIB_ENTRY_ATTRIBUTE_MULTICAST,
 } fib_entry_attribute_t;
 
 /**
@@ -215,7 +220,8 @@ typedef enum fib_entry_attribute_t_ {
     [FIB_ENTRY_ATTRIBUTE_DROP]      = "drop",		\
     [FIB_ENTRY_ATTRIBUTE_EXCLUSIVE] = "exclusive",      \
     [FIB_ENTRY_ATTRIBUTE_LOCAL]     = "local",		\
-    [FIB_ENTRY_ATTRIBUTE_URPF_EXEMPT] = "uRPF-exempt"   \
+    [FIB_ENTRY_ATTRIBUTE_URPF_EXEMPT] = "uRPF-exempt",  \
+    [FIB_ENTRY_ATTRIBUTE_MULTICAST] = "multicast",	\
 }
 
 #define FOR_EACH_FIB_ATTRIBUTE(_item)			\
@@ -232,6 +238,7 @@ typedef enum fib_entry_flag_t_ {
     FIB_ENTRY_FLAG_LOCAL     = (1 << FIB_ENTRY_ATTRIBUTE_LOCAL),
     FIB_ENTRY_FLAG_IMPORT    = (1 << FIB_ENTRY_ATTRIBUTE_IMPORT),
     FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT = (1 << FIB_ENTRY_ATTRIBUTE_URPF_EXEMPT),
+    FIB_ENTRY_FLAG_MULTICAST = (1 << FIB_ENTRY_ATTRIBUTE_MULTICAST),
 } __attribute__((packed)) fib_entry_flag_t;
 
 /**
@@ -396,7 +403,7 @@ typedef struct fib_entry_t_ {
      *     paint the header straight on without the need to check the packet
      *     type to derive the EOS bit value.
      */
-    dpo_id_t fe_lb; // [FIB_FORW_CHAIN_MPLS_NUM];
+    dpo_id_t fe_lb;
     /**
      * Vector of source infos.
      * Most entries will only have 1 source. So we optimise for memory usage,
diff --git a/src/vnet/fib/fib_entry_src.c b/src/vnet/fib/fib_entry_src.c
index aa1d5a24..a700282e 100644
--- a/src/vnet/fib/fib_entry_src.c
+++ b/src/vnet/fib/fib_entry_src.c
@@ -17,6 +17,7 @@
 #include <vnet/dpo/load_balance.h>
 #include <vnet/dpo/mpls_label_dpo.h>
 #include <vnet/dpo/drop_dpo.h>
+#include <vnet/dpo/replicate_dpo.h>
 
 #include <vnet/fib/fib_entry_src.h>
 #include <vnet/fib/fib_table.h>
@@ -229,8 +230,6 @@ fib_forward_chain_type_t
 fib_entry_chain_type_fixup (const fib_entry_t *entry,
 			    fib_forward_chain_type_t fct)
 {
-    ASSERT(FIB_FORW_CHAIN_TYPE_MPLS_EOS == fct);
-
     /*
      * The EOS chain is a tricky since one cannot know the adjacency
      * to link to without knowing what the packets payload protocol
@@ -238,6 +237,11 @@ fib_entry_chain_type_fixup (const fib_entry_t *entry,
      */
     fib_forward_chain_type_t dfct;
 
+    if (FIB_FORW_CHAIN_TYPE_MPLS_EOS != fct)
+    {
+        return (fct);
+    }
+
     dfct = fib_entry_get_default_chain_type(entry);
 
     if (FIB_FORW_CHAIN_TYPE_MPLS_EOS == dfct)
@@ -303,7 +307,12 @@ fib_entry_src_collect_forwarding (fib_node_index_t pl_index,
          * found a matching extension. stack it to obtain the forwarding
          * info for this path.
          */
-        ctx->next_hops = fib_path_ext_stack(path_ext, ctx->fib_entry, ctx->fct, ctx->next_hops);
+        ctx->next_hops =
+            fib_path_ext_stack(path_ext,
+                               ctx->fct,
+                               fib_entry_chain_type_fixup(ctx->fib_entry,
+                                                          ctx->fct),
+                               ctx->next_hops);
     }
     else
     {
@@ -355,6 +364,9 @@ fib_entry_src_collect_forwarding (fib_node_index_t pl_index,
                                            fib_entry_chain_type_fixup(ctx->fib_entry,
                                                                       ctx->fct),
                                            &nh->path_dpo);
+            fib_path_stack_mpls_disp(path_index,
+                                     ctx->fib_entry->fe_prefix.fp_payload_proto,
+                                     &nh->path_dpo);
 
             break;
         }
@@ -424,50 +436,70 @@ fib_entry_src_mk_lb (fib_entry_t *fib_entry,
         /*
          * first time create
          */
-        flow_hash_config_t fhc;
-
-        fhc = fib_table_get_flow_hash_config(fib_entry->fe_fib_index,
-                                             dpo_proto_to_fib(lb_proto));
-        dpo_set(dpo_lb,
-                DPO_LOAD_BALANCE,
-                lb_proto,
-                load_balance_create(0, lb_proto, fhc));
+        if (esrc->fes_entry_flags & FIB_ENTRY_FLAG_MULTICAST)
+        {
+            dpo_set(dpo_lb,
+                    DPO_REPLICATE,
+                    lb_proto,
+                    MPLS_IS_REPLICATE | replicate_create(0, lb_proto));
+        }
+        else
+        {
+            flow_hash_config_t fhc;
+
+            fhc = fib_table_get_flow_hash_config(fib_entry->fe_fib_index,
+                                                 dpo_proto_to_fib(lb_proto));
+            dpo_set(dpo_lb,
+                    DPO_LOAD_BALANCE,
+                    lb_proto,
+                    load_balance_create(0, lb_proto, fhc));
+        }
     }
 
-    load_balance_multipath_update(dpo_lb,
-                                  ctx.next_hops,
-                                  fib_entry_calc_lb_flags(&ctx));
-    vec_free(ctx.next_hops);
-
-    /*
-     * if this entry is sourced by the uRPF-exempt source then we
-     * append the always present local0 interface (index 0) to the
-     * uRPF list so it is not empty. that way packets pass the loose check.
-     */
-    index_t ui = fib_path_list_get_urpf(esrc->fes_pl);
-
-    if ((fib_entry_is_sourced(fib_entry_get_index(fib_entry),
-			      FIB_SOURCE_URPF_EXEMPT) ||
-	 (esrc->fes_entry_flags & FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT))&&
-	(0 == fib_urpf_check_size(ui)))
+    if (esrc->fes_entry_flags & FIB_ENTRY_FLAG_MULTICAST)
     {
-	/*
-	 * The uRPF list we get from the path-list is shared by all
-	 * other users of the list, but the uRPF exemption applies
-	 * only to this prefix. So we need our own list.
-	 */
-	ui = fib_urpf_list_alloc_and_lock();
-	fib_urpf_list_append(ui, 0);
-	fib_urpf_list_bake(ui);
-	load_balance_set_urpf(dpo_lb->dpoi_index, ui);
-	fib_urpf_list_unlock(ui);
+        /*
+         * MPLS multicast
+         */
+        replicate_multipath_update(dpo_lb, ctx.next_hops);
     }
     else
     {
-	load_balance_set_urpf(dpo_lb->dpoi_index, ui);
+        load_balance_multipath_update(dpo_lb,
+                                      ctx.next_hops,
+                                      fib_entry_calc_lb_flags(&ctx));
+        vec_free(ctx.next_hops);
+
+        /*
+         * if this entry is sourced by the uRPF-exempt source then we
+         * append the always present local0 interface (index 0) to the
+         * uRPF list so it is not empty. that way packets pass the loose check.
+         */
+        index_t ui = fib_path_list_get_urpf(esrc->fes_pl);
+
+        if ((fib_entry_is_sourced(fib_entry_get_index(fib_entry),
+                                  FIB_SOURCE_URPF_EXEMPT) ||
+             (esrc->fes_entry_flags & FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT))&&
+            (0 == fib_urpf_check_size(ui)))
+        {
+            /*
+             * The uRPF list we get from the path-list is shared by all
+             * other users of the list, but the uRPF exemption applies
+             * only to this prefix. So we need our own list.
+             */
+            ui = fib_urpf_list_alloc_and_lock();
+            fib_urpf_list_append(ui, 0);
+            fib_urpf_list_bake(ui);
+            load_balance_set_urpf(dpo_lb->dpoi_index, ui);
+            fib_urpf_list_unlock(ui);
+        }
+        else
+        {
+            load_balance_set_urpf(dpo_lb->dpoi_index, ui);
+        }
+        load_balance_set_fib_entry_flags(dpo_lb->dpoi_index,
+                                         fib_entry_get_flags_i(fib_entry));
     }
-    load_balance_set_fib_entry_flags(dpo_lb->dpoi_index,
-                                     fib_entry_get_flags_i(fib_entry));
 }
 
 void
@@ -887,21 +919,6 @@ fib_entry_src_action_remove (fib_entry_t *fib_entry,
     return (sflags);
 }
 
-static inline int
-fib_route_recurses_via_self (const fib_prefix_t *prefix,
-			     const fib_route_path_t *rpath)
-{
-    /*
-     * not all zeros next hop &&
-     * is recursive path &&
-     * nexthop is same as the route's address
-     */
-    return ((!ip46_address_is_zero(&rpath->frp_addr)) &&
-	    (~0 == rpath->frp_sw_if_index) &&
-	    (0 == ip46_address_cmp(&rpath->frp_addr, &prefix->fp_addr)));
-
-}
-
 /*
  * fib_route_attached_cross_table
  *
@@ -962,14 +979,14 @@ fib_entry_src_flags_2_path_list_flags (fib_entry_flag_t eflags)
     {
 	plf |= FIB_PATH_LIST_FLAG_DROP;
     }
-    if (eflags & FIB_ENTRY_FLAG_LOCAL)
-    {
-	plf |= FIB_PATH_LIST_FLAG_LOCAL;
-    }
     if (eflags & FIB_ENTRY_FLAG_EXCLUSIVE)
     {
 	plf |= FIB_PATH_LIST_FLAG_EXCLUSIVE;
     }
+    if (eflags & FIB_ENTRY_FLAG_LOCAL)
+    {
+	plf |= FIB_PATH_LIST_FLAG_LOCAL;
+    }
 
     return (plf);
 }
@@ -980,25 +997,6 @@ fib_entry_flags_update (const fib_entry_t *fib_entry,
 			fib_path_list_flags_t *pl_flags,
 			fib_entry_src_t *esrc)
 {
-    /*
-     * don't allow the addition of a recursive looped path for prefix
-     * via itself.
-     */
-    if (fib_route_recurses_via_self(&fib_entry->fe_prefix, rpath))	
-    {
-	/*
-	 * force the install of a drop path-list.
-	 * we want the entry to have some path-list, mainly so
-	 * the dodgy path can be rmeoved when the source stops playing
-	 * silly buggers.
-	 */
-	*pl_flags |= FIB_PATH_LIST_FLAG_DROP;
-    }
-    else
-    {
-	*pl_flags &= ~FIB_PATH_LIST_FLAG_DROP;
-    }
-
     if ((esrc->fes_src == FIB_SOURCE_API) ||
 	(esrc->fes_src == FIB_SOURCE_CLI))
     {
diff --git a/src/vnet/fib/fib_internal.h b/src/vnet/fib/fib_internal.h
index 2d980bcc..8abc0e07 100644
--- a/src/vnet/fib/fib_internal.h
+++ b/src/vnet/fib/fib_internal.h
@@ -25,6 +25,7 @@
 #undef FIB_DEBUG
 
 extern void fib_prefix_from_mpls_label(mpls_label_t label,
+                                       mpls_eos_bit_t eos,
 				       fib_prefix_t *prf);
 
 extern int fib_route_path_cmp(const fib_route_path_t *rpath1,
diff --git a/src/vnet/fib/fib_path.c b/src/vnet/fib/fib_path.c
index 6b202a97..f81f4170 100644
--- a/src/vnet/fib/fib_path.c
+++ b/src/vnet/fib/fib_path.c
@@ -21,6 +21,8 @@
 #include <vnet/dpo/receive_dpo.h>
 #include <vnet/dpo/load_balance_map.h>
 #include <vnet/dpo/lookup_dpo.h>
+#include <vnet/dpo/interface_dpo.h>
+#include <vnet/dpo/mpls_disposition.h>
 
 #include <vnet/adj/adj.h>
 #include <vnet/adj/adj_mcast.h>
@@ -66,6 +68,10 @@ typedef enum fib_path_type_t_ {
      * deag. Link to a lookup adj in the next table
      */
     FIB_PATH_TYPE_DEAG,
+    /**
+     * interface receive.
+     */
+    FIB_PATH_TYPE_INTF_RX,
     /**
      * receive. it's for-us.
      */
@@ -88,6 +94,7 @@ typedef enum fib_path_type_t_ {
     [FIB_PATH_TYPE_SPECIAL]           = "special",	        \
     [FIB_PATH_TYPE_EXCLUSIVE]         = "exclusive",	        \
     [FIB_PATH_TYPE_DEAG]              = "deag",	                \
+    [FIB_PATH_TYPE_INTF_RX]           = "intf-rx",	        \
     [FIB_PATH_TYPE_RECEIVE]           = "receive",	        \
 }
 
@@ -220,10 +227,16 @@ typedef struct fib_path_t_ {
 		 * The next-hop
 		 */
 		ip46_address_t fp_ip;
-		/**
-		 * The local label to resolve through.
-		 */
-		mpls_label_t fp_local_label;
+		struct {
+                    /**
+                     * The local label to resolve through.
+                     */
+                    mpls_label_t fp_local_label;
+                    /**
+                     * The EOS bit of the resolving label
+                     */
+                    mpls_eos_bit_t fp_eos;
+                };
 	    } fp_nh;
 	    /**
 	     * The FIB table index in which to find the next-hop.
@@ -254,6 +267,10 @@ typedef struct fib_path_t_ {
 	     * The FIB index in which to perfom the next lookup
 	     */
 	    fib_node_index_t fp_tbl_id;
+            /**
+             * The RPF-ID to tag the packets with
+             */
+            fib_rpf_id_t fp_rpf_id;
 	} deag;
 	struct {
 	} special;
@@ -273,6 +290,12 @@ typedef struct fib_path_t_ {
 	     */
 	    ip46_address_t fp_addr;
 	} receive;
+	struct {
+	    /**
+	     * The interface on which the packets will be input.
+	     */
+	    u32 fp_interface;
+	} intf_rx;
     };
     STRUCT_MARK(path_hash_end);
 
@@ -444,9 +467,11 @@ format_fib_path (u8 * s, va_list * args)
     case FIB_PATH_TYPE_RECURSIVE:
 	if (FIB_PROTOCOL_MPLS == path->fp_nh_proto)
 	{
-	    s = format (s, "via %U",
+	    s = format (s, "via %U %U",
 			format_mpls_unicast_label,
-			path->recursive.fp_nh.fp_local_label);
+			path->recursive.fp_nh.fp_local_label,
+			format_mpls_eos_bit,
+			path->recursive.fp_nh.fp_eos);
 	}
 	else
 	{
@@ -465,6 +490,7 @@ format_fib_path (u8 * s, va_list * args)
 
 	break;
     case FIB_PATH_TYPE_RECEIVE:
+    case FIB_PATH_TYPE_INTF_RX:
     case FIB_PATH_TYPE_SPECIAL:
     case FIB_PATH_TYPE_DEAG:
     case FIB_PATH_TYPE_EXCLUSIVE:
@@ -736,6 +762,7 @@ fib_path_unresolve (fib_path_t *path)
         break;
     case FIB_PATH_TYPE_SPECIAL:
     case FIB_PATH_TYPE_RECEIVE:
+    case FIB_PATH_TYPE_INTF_RX:
     case FIB_PATH_TYPE_DEAG:
         /*
          * these hold only the path's DPO, which is reset below.
@@ -754,16 +781,24 @@ fib_path_unresolve (fib_path_t *path)
 }
 
 static fib_forward_chain_type_t
-fib_path_proto_to_chain_type (fib_protocol_t proto)
+fib_path_to_chain_type (const fib_path_t *path)
 {
-    switch (proto)
+    switch (path->fp_nh_proto)
     {
     case FIB_PROTOCOL_IP4:
 	return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4);
     case FIB_PROTOCOL_IP6:
 	return (FIB_FORW_CHAIN_TYPE_UNICAST_IP6);
     case FIB_PROTOCOL_MPLS:
-	return (FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS);
+        if (FIB_PATH_TYPE_RECURSIVE == path->fp_type &&
+            MPLS_EOS == path->recursive.fp_nh.fp_eos)
+        {
+            return (FIB_FORW_CHAIN_TYPE_MPLS_EOS);
+        }
+        else
+        {
+            return (FIB_FORW_CHAIN_TYPE_MPLS_EOS);
+        }
     }
     return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4);
 }
@@ -793,7 +828,7 @@ fib_path_back_walk_notify (fib_node_t *node,
 	     */
 	    fib_path_recursive_adj_update(
 		path,
-		fib_path_proto_to_chain_type(path->fp_nh_proto),
+		fib_path_to_chain_type(path),
 		&path->fp_dpo);
 	}
 	if ((FIB_NODE_BW_REASON_FLAG_ADJ_UPDATE & ctx->fnbw_reason) ||
@@ -931,6 +966,8 @@ FIXME comment
 	    path->fp_oper_flags |= FIB_PATH_OPER_FLAG_DROP;
 	}
 	break;
+    case FIB_PATH_TYPE_INTF_RX:
+        ASSERT(0);
     case FIB_PATH_TYPE_DEAG:
 	/*
 	 * FIXME When VRF delete is allowed this will need a poke.
@@ -986,6 +1023,14 @@ fib_path_route_flags_to_cfg_flags (const fib_route_path_t *rpath)
 	cfg_flags |= FIB_PATH_CFG_FLAG_LOCAL;
     if (rpath->frp_flags & FIB_ROUTE_PATH_ATTACHED)
 	cfg_flags |= FIB_PATH_CFG_FLAG_ATTACHED;
+    if (rpath->frp_flags & FIB_ROUTE_PATH_INTF_RX)
+	cfg_flags |= FIB_PATH_CFG_FLAG_INTF_RX;
+    if (rpath->frp_flags & FIB_ROUTE_PATH_RPF_ID)
+	cfg_flags |= FIB_PATH_CFG_FLAG_RPF_ID;
+    if (rpath->frp_flags & FIB_ROUTE_PATH_EXCLUSIVE)
+	cfg_flags |= FIB_PATH_CFG_FLAG_EXCLUSIVE;
+    if (rpath->frp_flags & FIB_ROUTE_PATH_DROP)
+	cfg_flags |= FIB_PATH_CFG_FLAG_DROP;
 
     return (cfg_flags);
 }
@@ -998,8 +1043,6 @@ fib_path_route_flags_to_cfg_flags (const fib_route_path_t *rpath)
  */
 fib_node_index_t
 fib_path_create (fib_node_index_t pl_index,
-		 fib_protocol_t nh_proto,
-		 fib_path_cfg_flags_t flags,
 		 const fib_route_path_t *rpath)
 {
     fib_path_t *path;
@@ -1012,7 +1055,7 @@ fib_path_create (fib_node_index_t pl_index,
 
     dpo_reset(&path->fp_dpo);
     path->fp_pl_index = pl_index;
-    path->fp_nh_proto = nh_proto;
+    path->fp_nh_proto = rpath->frp_proto;
     path->fp_via_fib = FIB_NODE_INDEX_INVALID;
     path->fp_weight = rpath->frp_weight;
     if (0 == path->fp_weight)
@@ -1023,8 +1066,7 @@ fib_path_create (fib_node_index_t pl_index,
          */
         path->fp_weight = 1;
     }
-    path->fp_cfg_flags = flags;
-    path->fp_cfg_flags |= fib_path_route_flags_to_cfg_flags(rpath);
+    path->fp_cfg_flags = fib_path_route_flags_to_cfg_flags(rpath);
 
     /*
      * deduce the path's tpye from the parementers and save what is needed.
@@ -1035,6 +1077,17 @@ fib_path_create (fib_node_index_t pl_index,
         path->receive.fp_interface = rpath->frp_sw_if_index;
         path->receive.fp_addr = rpath->frp_addr;
     }
+    else if (path->fp_cfg_flags & FIB_PATH_CFG_FLAG_INTF_RX)
+    {
+        path->fp_type = FIB_PATH_TYPE_INTF_RX;
+        path->intf_rx.fp_interface = rpath->frp_sw_if_index;
+    }
+    else if (path->fp_cfg_flags & FIB_PATH_CFG_FLAG_RPF_ID)
+    {
+        path->fp_type = FIB_PATH_TYPE_DEAG;
+        path->deag.fp_tbl_id = rpath->frp_fib_index;
+        path->deag.fp_rpf_id = rpath->frp_rpf_id;
+    }
     else if (~0 != rpath->frp_sw_if_index)
     {
         if (ip46_address_is_zero(&rpath->frp_addr))
@@ -1069,6 +1122,7 @@ fib_path_create (fib_node_index_t pl_index,
 	    if (FIB_PROTOCOL_MPLS == path->fp_nh_proto)
 	    {
 		path->recursive.fp_nh.fp_local_label = rpath->frp_local_label;
+                path->recursive.fp_nh.fp_eos = rpath->frp_eos;
 	    }
 	    else
 	    {
@@ -1238,17 +1292,13 @@ fib_path_cmp_i (const fib_path_t *path1,
 	    res = ip46_address_cmp(&path1->attached_next_hop.fp_nh,
 				   &path2->attached_next_hop.fp_nh);
 	    if (0 == res) {
-		res = vnet_sw_interface_compare(
-			  vnet_get_main(),
-			  path1->attached_next_hop.fp_interface,
-			  path2->attached_next_hop.fp_interface);
+		res = (path1->attached_next_hop.fp_interface -
+                       path2->attached_next_hop.fp_interface);
 	    }
 	    break;
 	case FIB_PATH_TYPE_ATTACHED:
-	    res = vnet_sw_interface_compare(
-		      vnet_get_main(),
-		      path1->attached.fp_interface,
-		      path2->attached.fp_interface);
+	    res = (path1->attached.fp_interface -
+                   path2->attached.fp_interface);
 	    break;
 	case FIB_PATH_TYPE_RECURSIVE:
 	    res = ip46_address_cmp(&path1->recursive.fp_nh,
@@ -1261,6 +1311,13 @@ fib_path_cmp_i (const fib_path_t *path1,
 	    break;
 	case FIB_PATH_TYPE_DEAG:
 	    res = (path1->deag.fp_tbl_id - path2->deag.fp_tbl_id);
+	    if (0 == res)
+	    {
+                res = (path1->deag.fp_rpf_id - path2->deag.fp_rpf_id);
+            }
+	    break;
+	case FIB_PATH_TYPE_INTF_RX:
+	    res = (path1->intf_rx.fp_interface - path2->intf_rx.fp_interface);
 	    break;
 	case FIB_PATH_TYPE_SPECIAL:
 	case FIB_PATH_TYPE_RECEIVE:
@@ -1336,22 +1393,22 @@ fib_path_cmp_w_route_path (fib_node_index_t path_index,
 				   &rpath->frp_addr);
 	    if (0 == res)
 	    {
-		res = vnet_sw_interface_compare(
-			  vnet_get_main(),
-			  path->attached_next_hop.fp_interface,
-			  rpath->frp_sw_if_index);
+		res = (path->attached_next_hop.fp_interface -
+                       rpath->frp_sw_if_index);
 	    }
 	    break;
 	case FIB_PATH_TYPE_ATTACHED:
-	    res = vnet_sw_interface_compare(
-		      vnet_get_main(),
-		      path->attached.fp_interface,
-		      rpath->frp_sw_if_index);
+	    res = (path->attached.fp_interface - rpath->frp_sw_if_index);
 	    break;
 	case FIB_PATH_TYPE_RECURSIVE:
             if (FIB_PROTOCOL_MPLS == path->fp_nh_proto)
             {
                 res = path->recursive.fp_nh.fp_local_label - rpath->frp_local_label;
+
+                if (res == 0)
+                {
+                    res = path->recursive.fp_nh.fp_eos - rpath->frp_eos;
+                }
             }
             else
             {
@@ -1364,9 +1421,16 @@ fib_path_cmp_w_route_path (fib_node_index_t path_index,
                 res = (path->recursive.fp_tbl_id - rpath->frp_fib_index);
             }
 	    break;
+	case FIB_PATH_TYPE_INTF_RX:
+	    res = (path->intf_rx.fp_interface - rpath->frp_sw_if_index);
+            break;
 	case FIB_PATH_TYPE_DEAG:
 	    res = (path->deag.fp_tbl_id - rpath->frp_fib_index);
-	    break;
+	    if (0 == res)
+            {
+                res = (path->deag.fp_rpf_id - rpath->frp_rpf_id);
+            }
+            break;
 	case FIB_PATH_TYPE_SPECIAL:
 	case FIB_PATH_TYPE_RECEIVE:
 	case FIB_PATH_TYPE_EXCLUSIVE:
@@ -1465,6 +1529,7 @@ fib_path_recursive_loop_detect (fib_node_index_t path_index,
     case FIB_PATH_TYPE_SPECIAL:
     case FIB_PATH_TYPE_DEAG:
     case FIB_PATH_TYPE_RECEIVE:
+    case FIB_PATH_TYPE_INTF_RX:
     case FIB_PATH_TYPE_EXCLUSIVE:
 	/*
 	 * these path types cannot be part of a loop, since they are the leaves
@@ -1563,7 +1628,9 @@ fib_path_resolve (fib_node_index_t path_index)
 
 	if (FIB_PROTOCOL_MPLS == path->fp_nh_proto)
 	{
-	    fib_prefix_from_mpls_label(path->recursive.fp_nh.fp_local_label, &pfx);
+	    fib_prefix_from_mpls_label(path->recursive.fp_nh.fp_local_label,
+                                       path->recursive.fp_nh.fp_eos,
+                                       &pfx);
 	}
 	else
 	{
@@ -1592,7 +1659,7 @@ fib_path_resolve (fib_node_index_t path_index)
 	 */
 	fib_path_recursive_adj_update(
 	    path,
-	    fib_path_proto_to_chain_type(path->fp_nh_proto),
+	    fib_path_to_chain_type(path),
 	    &path->fp_dpo);
 
 	break;
@@ -1605,16 +1672,25 @@ fib_path_resolve (fib_node_index_t path_index)
                  drop_dpo_get(fib_proto_to_dpo(path->fp_nh_proto)));
 	break;
     case FIB_PATH_TYPE_DEAG:
+    {
 	/*
 	 * Resolve via a lookup DPO.
          * FIXME. control plane should add routes with a table ID
 	 */
-	lookup_dpo_add_or_lock_w_fib_index(path->deag.fp_tbl_id,
-                                          fib_proto_to_dpo(path->fp_nh_proto),
-                                          LOOKUP_INPUT_DST_ADDR,
-                                          LOOKUP_TABLE_FROM_CONFIG,
-                                          &path->fp_dpo);
+        lookup_cast_t cast;
+        
+        cast = (path->fp_cfg_flags & FIB_PATH_CFG_FLAG_RPF_ID ?
+                LOOKUP_MULTICAST :
+                LOOKUP_UNICAST);
+
+        lookup_dpo_add_or_lock_w_fib_index(path->deag.fp_tbl_id,
+                                           fib_proto_to_dpo(path->fp_nh_proto),
+                                           cast,
+                                           LOOKUP_INPUT_DST_ADDR,
+                                           LOOKUP_TABLE_FROM_CONFIG,
+                                           &path->fp_dpo);
 	break;
+    }
     case FIB_PATH_TYPE_RECEIVE:
 	/*
 	 * Resolve via a receive DPO.
@@ -1624,6 +1700,15 @@ fib_path_resolve (fib_node_index_t path_index)
                                 &path->receive.fp_addr,
                                 &path->fp_dpo);
 	break;
+    case FIB_PATH_TYPE_INTF_RX: {
+	/*
+	 * Resolve via a receive DPO.
+	 */
+	interface_dpo_add_or_lock(fib_proto_to_dpo(path->fp_nh_proto),
+                                  path->intf_rx.fp_interface,
+                                  &path->fp_dpo);
+	break;
+    }
     case FIB_PATH_TYPE_EXCLUSIVE:
 	/*
 	 * Resolve via the user provided DPO
@@ -1652,6 +1737,7 @@ fib_path_get_resolving_interface (fib_node_index_t path_index)
 	return (path->receive.fp_interface);
     case FIB_PATH_TYPE_RECURSIVE:
 	return (fib_entry_get_resolving_interface(path->fp_via_fib));    
+    case FIB_PATH_TYPE_INTF_RX:
     case FIB_PATH_TYPE_SPECIAL:
     case FIB_PATH_TYPE_DEAG:
     case FIB_PATH_TYPE_EXCLUSIVE:
@@ -1743,6 +1829,7 @@ fib_path_contribute_urpf (fib_node_index_t path_index,
 
     case FIB_PATH_TYPE_DEAG:
     case FIB_PATH_TYPE_RECEIVE:
+    case FIB_PATH_TYPE_INTF_RX:
 	/*
 	 * these path types don't link to an adj
 	 */
@@ -1750,6 +1837,44 @@ fib_path_contribute_urpf (fib_node_index_t path_index,
     }
 }
 
+void
+fib_path_stack_mpls_disp (fib_node_index_t path_index,
+                          dpo_proto_t payload_proto,
+                          dpo_id_t *dpo)
+{
+    fib_path_t *path;
+
+    path = fib_path_get(path_index);
+
+    ASSERT(path);
+
+    switch (path->fp_type)
+    {
+    case FIB_PATH_TYPE_DEAG:
+    {
+        dpo_id_t tmp = DPO_INVALID;
+
+        dpo_copy(&tmp, dpo);
+        dpo_set(dpo,
+                DPO_MPLS_DISPOSITION,
+                payload_proto,
+                mpls_disp_dpo_create(payload_proto,
+                                     path->deag.fp_rpf_id,
+                                     &tmp));
+        dpo_reset(&tmp);
+        break;
+    }                
+    case FIB_PATH_TYPE_RECEIVE:
+    case FIB_PATH_TYPE_ATTACHED:
+    case FIB_PATH_TYPE_ATTACHED_NEXT_HOP:
+    case FIB_PATH_TYPE_RECURSIVE:
+    case FIB_PATH_TYPE_INTF_RX:
+    case FIB_PATH_TYPE_EXCLUSIVE:
+    case FIB_PATH_TYPE_SPECIAL:
+        break;
+    }
+}
+
 void
 fib_path_contribute_forwarding (fib_node_index_t path_index,
 				fib_forward_chain_type_t fct,
@@ -1769,7 +1894,7 @@ fib_path_contribute_forwarding (fib_node_index_t path_index,
      * This then represents the path's 'native' protocol; IP.
      * For all others will need to go find something else.
      */
-    if (fib_path_proto_to_chain_type(path->fp_nh_proto) == fct)
+    if (fib_path_to_chain_type(path) == fct)
     {
 	dpo_copy(dpo, &path->fp_dpo);
     }
@@ -1813,10 +1938,10 @@ fib_path_contribute_forwarding (fib_node_index_t path_index,
 	    case FIB_FORW_CHAIN_TYPE_UNICAST_IP4:
 	    case FIB_FORW_CHAIN_TYPE_UNICAST_IP6:
 	    case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS:
-		fib_path_recursive_adj_update(path, fct, dpo);
-		break;
 	    case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
 	    case FIB_FORW_CHAIN_TYPE_MCAST_IP6:
+		fib_path_recursive_adj_update(path, fct, dpo);
+		break;
 	    case FIB_FORW_CHAIN_TYPE_ETHERNET:
 	    case FIB_FORW_CHAIN_TYPE_NSH:
 		ASSERT(0);
@@ -1829,13 +1954,14 @@ fib_path_contribute_forwarding (fib_node_index_t path_index,
 	    case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS:
                 lookup_dpo_add_or_lock_w_table_id(MPLS_FIB_DEFAULT_TABLE_ID,
                                                   DPO_PROTO_MPLS,
+                                                  LOOKUP_UNICAST,
                                                   LOOKUP_INPUT_DST_ADDR,
                                                   LOOKUP_TABLE_FROM_CONFIG,
                                                   dpo);
                 break;
+	    case FIB_FORW_CHAIN_TYPE_MPLS_EOS:
 	    case FIB_FORW_CHAIN_TYPE_UNICAST_IP4:
 	    case FIB_FORW_CHAIN_TYPE_UNICAST_IP6:
-	    case FIB_FORW_CHAIN_TYPE_MPLS_EOS:
 		dpo_copy(dpo, &path->fp_dpo);
 		break;
 	    case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
@@ -1870,7 +1996,7 @@ fib_path_contribute_forwarding (fib_node_index_t path_index,
                     ai = adj_mcast_add_or_lock(path->fp_nh_proto,
                                                fib_forw_chain_type_to_link_type(fct),
                                                path->attached.fp_interface);
-                    dpo_set(dpo, DPO_ADJACENCY_MCAST,
+                    dpo_set(dpo, DPO_ADJACENCY,
                             fib_forw_chain_type_to_dpo_proto(fct),
                             ai);
                     adj_unlock(ai);
@@ -1878,6 +2004,14 @@ fib_path_contribute_forwarding (fib_node_index_t path_index,
                 break;
             }
             break;
+        case FIB_PATH_TYPE_INTF_RX:
+            /*
+             * Create the adj needed for sending IP multicast traffic
+             */
+            interface_dpo_add_or_lock(fib_forw_chain_type_to_dpo_proto(fct),
+                                      path->attached.fp_interface,
+                                      dpo);
+            break;
         case FIB_PATH_TYPE_RECEIVE:
         case FIB_PATH_TYPE_SPECIAL:
             dpo_copy(dpo, &path->fp_dpo);
diff --git a/src/vnet/fib/fib_path.h b/src/vnet/fib/fib_path.h
index 14efc1ab..334be6f5 100644
--- a/src/vnet/fib/fib_path.h
+++ b/src/vnet/fib/fib_path.h
@@ -69,6 +69,14 @@ typedef enum fib_path_cfg_attribute_t_ {
     /**
      * The path is a for-us path
      */
+    FIB_PATH_CFG_ATTRIBUTE_INTF_RX,
+    /**
+     * The path is a deag with rpf-id
+     */
+    FIB_PATH_CFG_ATTRIBUTE_RPF_ID,
+    /**
+     * The path is an interface recieve
+     */
     FIB_PATH_CFG_ATTRIBUTE_LOCAL,
     /**
      * Marker. Add new types before this one, then update it.
@@ -88,6 +96,8 @@ typedef enum fib_path_cfg_attribute_t_ {
     [FIB_PATH_CFG_ATTRIBUTE_RESOLVE_ATTACHED] = "resolve-attached", \
     [FIB_PATH_CFG_ATTRIBUTE_LOCAL] = "local",	        \
     [FIB_PATH_CFG_ATTRIBUTE_ATTACHED] = "attached",	\
+    [FIB_PATH_CFG_ATTRIBUTE_INTF_RX] = "interface-rx",	\
+    [FIB_PATH_CFG_ATTRIBUTE_RPF_ID] = "rpf-id",         \
 }
 
 #define FOR_EACH_FIB_PATH_CFG_ATTRIBUTE(_item) \
@@ -106,6 +116,8 @@ typedef enum fib_path_cfg_flags_t_ {
     FIB_PATH_CFG_FLAG_RESOLVE_ATTACHED = (1 << FIB_PATH_CFG_ATTRIBUTE_RESOLVE_ATTACHED),
     FIB_PATH_CFG_FLAG_LOCAL = (1 << FIB_PATH_CFG_ATTRIBUTE_LOCAL),
     FIB_PATH_CFG_FLAG_ATTACHED = (1 << FIB_PATH_CFG_ATTRIBUTE_ATTACHED),
+    FIB_PATH_CFG_FLAG_INTF_RX = (1 << FIB_PATH_CFG_ATTRIBUTE_INTF_RX),
+    FIB_PATH_CFG_FLAG_RPF_ID = (1 << FIB_PATH_CFG_ATTRIBUTE_RPF_ID),
 } __attribute__ ((packed)) fib_path_cfg_flags_t;
 
 
@@ -117,8 +129,6 @@ extern u8 *fib_path_adj_format(fib_node_index_t pi,
 extern u8 * format_fib_path(u8 * s, va_list * args);
 
 extern fib_node_index_t fib_path_create(fib_node_index_t pl_index,
-					fib_protocol_t nh_proto,
-					fib_path_cfg_flags_t flags,
 					const fib_route_path_t *path);
 extern fib_node_index_t fib_path_create_special(fib_node_index_t pl_index,
 						fib_protocol_t nh_proto,
@@ -145,6 +155,9 @@ extern load_balance_path_t * fib_path_append_nh_for_multipath_hash(
     fib_node_index_t path_index,
     fib_forward_chain_type_t fct,
     load_balance_path_t *hash_key);
+extern void fib_path_stack_mpls_disp(fib_node_index_t path_index,
+                                     dpo_proto_t payload_proto,
+                                     dpo_id_t *dpo);
 extern void fib_path_contribute_forwarding(fib_node_index_t path_index,
 					   fib_forward_chain_type_t type,
 					   dpo_id_t *dpo);
diff --git a/src/vnet/fib/fib_path_ext.c b/src/vnet/fib/fib_path_ext.c
index f75b5626..08293bcf 100644
--- a/src/vnet/fib/fib_path_ext.c
+++ b/src/vnet/fib/fib_path_ext.c
@@ -103,8 +103,8 @@ fib_path_ext_is_imp_null (fib_path_ext_t *path_ext)
 
 load_balance_path_t *
 fib_path_ext_stack (fib_path_ext_t *path_ext,
-		    const fib_entry_t *entry,
                     fib_forward_chain_type_t child_fct,
+                    fib_forward_chain_type_t imp_null_fct,
 		    load_balance_path_t *nhs)
 {
     fib_forward_chain_type_t parent_fct;
@@ -129,7 +129,7 @@ fib_path_ext_stack (fib_path_ext_t *path_ext,
 	 */
 	if (fib_path_ext_is_imp_null(path_ext))
 	{
-            parent_fct = fib_entry_chain_type_fixup(entry, child_fct);
+            parent_fct = imp_null_fct;
         }
         else
         {
diff --git a/src/vnet/fib/fib_path_ext.h b/src/vnet/fib/fib_path_ext.h
index cf8f8df0..d617700d 100644
--- a/src/vnet/fib/fib_path_ext.h
+++ b/src/vnet/fib/fib_path_ext.h
@@ -18,6 +18,7 @@
 
 #include <vnet/mpls/mpls.h>
 #include <vnet/fib/fib_types.h>
+#include <vnet/dpo/load_balance.h>
 
 /**
  * A path extension is a per-entry addition to the forwarding information
@@ -61,8 +62,8 @@ extern void fib_path_ext_resolve(fib_path_ext_t *path_ext,
 				 fib_node_index_t path_list_index);
 
 extern load_balance_path_t *fib_path_ext_stack(fib_path_ext_t *path_ext,
-                                               const struct fib_entry_t_ *entry,
                                                fib_forward_chain_type_t fct,
+                                               fib_forward_chain_type_t imp_null_fct,
                                                load_balance_path_t *nhs);
 
 #endif
diff --git a/src/vnet/fib/fib_path_list.c b/src/vnet/fib/fib_path_list.c
index b9a391b3..ea6565dd 100644
--- a/src/vnet/fib/fib_path_list.c
+++ b/src/vnet/fib/fib_path_list.c
@@ -40,13 +40,6 @@ typedef struct fib_path_list_t_ {
      */
     fib_path_list_flags_t fpl_flags;
 
-    /**
-     * The next-hop protocol for the paths in this path list.
-     * Note that fixing the proto here means we don't support a mix of
-     * v4 and v6 paths. ho hum.
-     */
-    fib_protocol_t fpl_nh_proto;
-
     /**
      * Vector of paths indicies for all configured paths.
      * For shareable path-lists this list MUST not change.
@@ -57,6 +50,11 @@ typedef struct fib_path_list_t_ {
      * the RPF list calculated for this path list
      */
     fib_node_index_t fpl_urpf;
+
+    /**
+     * Hash table of paths. valid only with INDEXED flag
+     */
+    uword *fpl_db;
 } fib_path_list_t;
 
 /*
@@ -131,7 +129,6 @@ format_fib_path_list (u8 * s, va_list * args)
     
     s = format (s, "    index:%u", fib_path_list_get_index(path_list));
     s = format (s, " locks:%u", path_list->fpl_node.fn_locks);
-    s = format (s, " proto:%U", format_fib_protocol, path_list->fpl_nh_proto);
 
     if (FIB_PATH_LIST_FLAG_NONE != path_list->fpl_flags)
     {
@@ -155,26 +152,6 @@ format_fib_path_list (u8 * s, va_list * args)
     return (s);
 }
 
-u8 *
-fib_path_list_adjs_format (fib_node_index_t path_list_index,
-			   u32 indent,
-			   u8 * s)
-{
-    fib_path_list_t *path_list;
-    u32 i;
-
-    path_list = fib_path_list_get(path_list_index);
-
-    vec_foreach_index (i, path_list->fpl_paths)
-    {
-	s = fib_path_adj_format(path_list->fpl_paths[i],
-				indent, s);
-    }
-
-    return (s);
-}
-
-
 u8 *
 fib_path_list_format (fib_node_index_t path_list_index,
 		      u8 * s)
@@ -648,27 +625,6 @@ fib_path_list_is_looped (fib_node_index_t path_list_index)
     return (path_list->fpl_flags & FIB_PATH_LIST_FLAG_LOOPED);
 }
 
-static fib_path_cfg_flags_t 
-fib_path_list_flags_2_path_flags (fib_path_list_flags_t plf)
-{
-    fib_path_cfg_flags_t pf = FIB_PATH_CFG_FLAG_NONE;
-
-    if (plf & FIB_PATH_LIST_FLAG_LOCAL)
-    {
-	pf |= FIB_PATH_CFG_FLAG_LOCAL;
-    }
-    if (plf & FIB_PATH_LIST_FLAG_DROP)
-    {
-	pf |= FIB_PATH_CFG_FLAG_DROP;
-    }
-    if (plf & FIB_PATH_LIST_FLAG_EXCLUSIVE)
-    {
-	pf |= FIB_PATH_CFG_FLAG_EXCLUSIVE;
-    }
-
-    return (pf);
-}
-
 static fib_path_list_flags_t
 fib_path_list_flags_fixup (fib_path_list_flags_t flags)
 {
@@ -695,18 +651,15 @@ fib_path_list_create (fib_path_list_flags_t flags,
     flags = fib_path_list_flags_fixup(flags);
     path_list = fib_path_list_alloc(&path_list_index);
     path_list->fpl_flags = flags;
-    /*
-     * we'll assume for now all paths are the same next-hop protocol
-     */
-    path_list->fpl_nh_proto = rpaths[0].frp_proto;
 
-    vec_foreach_index(i, rpaths)
+    if (NULL != rpaths)
     {
-	vec_add1(path_list->fpl_paths,
-		 fib_path_create(path_list_index,
-				 path_list->fpl_nh_proto,
-				 fib_path_list_flags_2_path_flags(flags),
-				 &rpaths[i]));
+        vec_foreach_index(i, rpaths)
+        {
+            vec_add1(path_list->fpl_paths,
+                     fib_path_create(path_list_index,
+                                     &rpaths[i]));
+        }
     }
 
     /*
@@ -748,6 +701,27 @@ fib_path_list_create (fib_path_list_flags_t flags,
     return (path_list_index);
 }
 
+static fib_path_cfg_flags_t 
+fib_path_list_flags_2_path_flags (fib_path_list_flags_t plf)
+{
+    fib_path_cfg_flags_t pf = FIB_PATH_CFG_FLAG_NONE;
+
+    if (plf & FIB_PATH_LIST_FLAG_DROP)
+    {
+	pf |= FIB_PATH_CFG_FLAG_DROP;
+    }
+    if (plf & FIB_PATH_LIST_FLAG_EXCLUSIVE)
+    {
+	pf |= FIB_PATH_CFG_FLAG_EXCLUSIVE;
+    }
+    if (plf & FIB_PATH_LIST_FLAG_LOCAL)
+    {
+        pf |= FIB_PATH_CFG_FLAG_LOCAL;
+    }
+
+    return (pf);
+}
+
 fib_node_index_t
 fib_path_list_create_special (fib_protocol_t nh_proto,
 			      fib_path_list_flags_t flags,
@@ -758,11 +732,10 @@ fib_path_list_create_special (fib_protocol_t nh_proto,
 
     path_list = fib_path_list_alloc(&path_list_index);
     path_list->fpl_flags = flags;
-    path_list->fpl_nh_proto = nh_proto;
 
     path_index =
 	fib_path_create_special(path_list_index,
-				path_list->fpl_nh_proto,
+                                nh_proto,
 				fib_path_list_flags_2_path_flags(flags),
 				dpo);
     vec_add1(path_list->fpl_paths, path_index);
@@ -775,6 +748,30 @@ fib_path_list_create_special (fib_protocol_t nh_proto,
     return (path_list_index);
 }
 
+/*
+ * return the index info the path-lists's vector of paths, of the matching path.
+ * ~0 if not found
+ */
+u32
+fib_path_list_find_rpath (fib_node_index_t path_list_index,
+                          const fib_route_path_t *rpath)
+{
+    fib_path_list_t *path_list;
+    u32 ii;
+
+    path_list = fib_path_list_get(path_list_index);
+
+    vec_foreach_index (ii, path_list->fpl_paths)
+    {
+        if (!fib_path_cmp_w_route_path(path_list->fpl_paths[ii], rpath))
+        {
+            return (ii);
+        }
+    }
+    return (~0);
+}
+
+
 /*
  * fib_path_list_copy_and_path_add
  *
@@ -782,13 +779,62 @@ fib_path_list_create_special (fib_protocol_t nh_proto,
  * The path-list returned could either have been newly created, or
  * can be a shared path-list from the data-base.
  */
+fib_node_index_t
+fib_path_list_path_add (fib_node_index_t path_list_index,
+                        const fib_route_path_t *rpaths)
+{
+    fib_node_index_t new_path_index, *orig_path_index;
+    fib_path_list_t *path_list;
+
+    /*
+     * alloc the new list before we retrieve the old one, lest
+     * the alloc result in a realloc
+     */
+    path_list = fib_path_list_get(path_list_index);
+
+    ASSERT(1 == vec_len(rpaths));
+    ASSERT(!(path_list->fpl_flags & FIB_PATH_LIST_FLAG_SHARED));
+
+    FIB_PATH_LIST_DBG(orig_path_list, "path-add");
+
+    new_path_index = fib_path_create(path_list_index,
+                                     rpaths);
+
+    vec_foreach (orig_path_index, path_list->fpl_paths)
+    {
+        /*
+         * don't add duplicate paths
+         */
+	if (0 == fib_path_cmp(new_path_index, *orig_path_index))
+        {
+            return (*orig_path_index);
+        }
+    }
+
+    /*
+     * Add the new path - no sort, no sharing, no key..
+     */
+    vec_add1(path_list->fpl_paths, new_path_index);
+
+    FIB_PATH_LIST_DBG(path_list, "path-added");
+
+    /*
+     * no shared path list requested. resolve and use the one
+     * just created.
+     */
+    fib_path_resolve(new_path_index);
+
+    return (new_path_index);
+}
+
 fib_node_index_t
 fib_path_list_copy_and_path_add (fib_node_index_t orig_path_list_index,
-				 fib_path_list_flags_t flags,
-				 const fib_route_path_t *rpaths)
+                                 fib_path_list_flags_t flags,
+                                 const fib_route_path_t *rpaths)
 {
     fib_node_index_t path_index, new_path_index, *orig_path_index;
     fib_path_list_t *path_list, *orig_path_list;
+    fib_node_index_t exist_path_list_index;
     fib_node_index_t path_list_index;
     fib_node_index_t pi;
 
@@ -806,13 +852,11 @@ fib_path_list_copy_and_path_add (fib_node_index_t orig_path_list_index,
 
     flags = fib_path_list_flags_fixup(flags);
     path_list->fpl_flags = flags;
-    path_list->fpl_nh_proto = orig_path_list->fpl_nh_proto;
+
     vec_validate(path_list->fpl_paths, vec_len(orig_path_list->fpl_paths));
     pi = 0;
 
     new_path_index = fib_path_create(path_list_index,
-                                     path_list->fpl_nh_proto,
-                                     fib_path_list_flags_2_path_flags(flags),
                                      rpaths);
 
     vec_foreach (orig_path_index, orig_path_list->fpl_paths)
@@ -845,46 +889,79 @@ fib_path_list_copy_and_path_add (fib_node_index_t orig_path_list_index,
     FIB_PATH_LIST_DBG(path_list, "path-added");
 
     /*
-     * If a shared path list is requested, consult the DB for a match
+     * check for a matching path-list in the DB.
+     * If we find one then we can return the existing one and destroy the
+     * new one just created.
      */
-    if (path_list->fpl_flags & FIB_PATH_LIST_FLAG_SHARED)
+    exist_path_list_index = fib_path_list_db_find(path_list);
+    if (FIB_NODE_INDEX_INVALID != exist_path_list_index)
     {
-	fib_node_index_t exist_path_list_index;
-	/*
-	 * check for a matching path-list in the DB.
-	 * If we find one then we can return the existing one and destroy the
-	 * new one just created.
-	 */
-	exist_path_list_index = fib_path_list_db_find(path_list);
-	if (FIB_NODE_INDEX_INVALID != exist_path_list_index)
-	{
-	    fib_path_list_destroy(path_list);
+        fib_path_list_destroy(path_list);
 	
-	    path_list_index = exist_path_list_index;
-	}
-	else
-	{
-	    /*
-	     * if there was not a matching path-list, then this
-	     * new one will need inserting into the DB and resolving.
-	     */
-	    fib_path_list_db_insert(path_list_index);
-
-	    path_list = fib_path_list_resolve(path_list);
-	}
+        path_list_index = exist_path_list_index;
     }
     else
     {
-	/*
-	 * no shared path list requested. resolve and use the one
-	 * just created.
-	 */
-	path_list = fib_path_list_resolve(path_list);
+        /*
+         * if there was not a matching path-list, then this
+         * new one will need inserting into the DB and resolving.
+         */
+        fib_path_list_db_insert(path_list_index);
+
+        path_list = fib_path_list_resolve(path_list);
     }
 
     return (path_list_index);
 }
 
+/*
+ * fib_path_list_path_remove
+ */
+fib_node_index_t
+fib_path_list_path_remove (fib_node_index_t path_list_index,
+                           const fib_route_path_t *rpaths)
+{
+    fib_node_index_t match_path_index, tmp_path_index;
+    fib_path_list_t *path_list;
+    fib_node_index_t pi;
+
+    path_list = fib_path_list_get(path_list_index);
+
+    ASSERT(1 == vec_len(rpaths));
+    ASSERT(!(path_list->fpl_flags & FIB_PATH_LIST_FLAG_SHARED));
+
+    FIB_PATH_LIST_DBG(orig_path_list, "path-remove");
+
+    /*
+     * create a representation of the path to be removed, so it
+     * can be used as a comparison object during the copy.
+     */
+    tmp_path_index = fib_path_create(path_list_index,
+				     rpaths);
+    match_path_index = FIB_NODE_INDEX_INVALID;
+
+    vec_foreach_index (pi, path_list->fpl_paths)
+    {
+	if (0 == fib_path_cmp(tmp_path_index,
+                              path_list->fpl_paths[pi]))
+        {
+            /*
+             * match - remove it
+             */
+            match_path_index = path_list->fpl_paths[pi];
+            fib_path_destroy(match_path_index);
+            vec_del1(path_list->fpl_paths, pi);
+	}
+    }
+
+    /*
+     * done with the temporary now
+     */
+    fib_path_destroy(tmp_path_index);
+
+    return (match_path_index);
+}
+
 /*
  * fib_path_list_copy_and_path_remove
  *
@@ -911,7 +988,6 @@ fib_path_list_copy_and_path_remove (fib_node_index_t orig_path_list_index,
     FIB_PATH_LIST_DBG(orig_path_list, "copy-remove");
 
     path_list->fpl_flags = flags;
-    path_list->fpl_nh_proto = orig_path_list->fpl_nh_proto;
     /*
      * allocate as many paths as we might need in one go, rather than
      * using vec_add to do a few at a time.
@@ -927,8 +1003,6 @@ fib_path_list_copy_and_path_remove (fib_node_index_t orig_path_list_index,
      * can be used as a comparison object during the copy.
      */
     tmp_path_index = fib_path_create(path_list_index,
-				     path_list->fpl_nh_proto,
-				     fib_path_list_flags_2_path_flags(flags),
 				     rpaths);
 
     vec_foreach (orig_path_index, orig_path_list->fpl_paths)
diff --git a/src/vnet/fib/fib_path_list.h b/src/vnet/fib/fib_path_list.h
index b4971add..9d246211 100644
--- a/src/vnet/fib/fib_path_list.h
+++ b/src/vnet/fib/fib_path_list.h
@@ -38,6 +38,11 @@ typedef enum fib_path_list_attribute_t_ {
      * be searched for each route update.
      */
     FIB_PATH_LIST_ATTRIBUTE_SHARED = FIB_PATH_LIST_ATTRIBUTE_FIRST,
+    /**
+     * Indexed means the path-list keeps a hash table of all paths for
+     * fast lookup. The lookup result is the fib_node_index of the path.
+     */
+    FIB_PATH_LIST_ATTRIBUTE_INDEXED,
     /**
      * explicit drop path-list. Used when the entry source needs to 
      * force a drop, despite the fact the path info is present.
@@ -73,6 +78,7 @@ typedef enum fib_path_list_attribute_t_ {
 typedef enum fib_path_list_flags_t_ {
     FIB_PATH_LIST_FLAG_NONE      = 0,
     FIB_PATH_LIST_FLAG_SHARED    = (1 << FIB_PATH_LIST_ATTRIBUTE_SHARED),
+    FIB_PATH_LIST_FLAG_INDEXED    = (1 << FIB_PATH_LIST_ATTRIBUTE_INDEXED),
     FIB_PATH_LIST_FLAG_DROP      = (1 << FIB_PATH_LIST_ATTRIBUTE_DROP),
     FIB_PATH_LIST_FLAG_LOCAL     = (1 << FIB_PATH_LIST_ATTRIBUTE_LOCAL),
     FIB_PATH_LIST_FLAG_EXCLUSIVE = (1 << FIB_PATH_LIST_ATTRIBUTE_EXCLUSIVE),
@@ -83,10 +89,11 @@ typedef enum fib_path_list_flags_t_ {
 
 #define FIB_PATH_LIST_ATTRIBUTES {       		 \
     [FIB_PATH_LIST_ATTRIBUTE_SHARED]    = "shared",	 \
+    [FIB_PATH_LIST_ATTRIBUTE_INDEXED]    = "indexed",	 \
     [FIB_PATH_LIST_ATTRIBUTE_RESOLVED]  = "resolved",	 \
     [FIB_PATH_LIST_ATTRIBUTE_DROP]      = "drop",	 \
     [FIB_PATH_LIST_ATTRIBUTE_EXCLUSIVE] = "exclusive",   \
-    [FIB_PATH_LIST_ATTRIBUTE_LOCAL]     = "local",	 \
+    [FIB_PATH_LIST_ATTRIBUTE_LOCAL]     = "local",      \
     [FIB_PATH_LIST_ATTRIBUTE_LOOPED]     = "looped",	 \
     [FIB_PATH_LIST_ATTRIBUTE_NO_URPF]     = "no-uRPF",	 \
 }
@@ -110,6 +117,13 @@ extern fib_node_index_t fib_path_list_copy_and_path_remove(
     fib_node_index_t pl_index,
     fib_path_list_flags_t flags,
     const fib_route_path_t *path);
+extern fib_node_index_t fib_path_list_path_add (
+    fib_node_index_t path_list_index,
+    const fib_route_path_t *rpaths);
+extern fib_node_index_t fib_path_list_path_remove (
+    fib_node_index_t path_list_index,
+    const fib_route_path_t *rpaths);
+
 extern u32 fib_path_list_get_n_paths(fib_node_index_t pl_index);
 
 extern void fib_path_list_contribute_forwarding(fib_node_index_t path_list_index,
@@ -137,11 +151,11 @@ extern int fib_path_list_is_looped(fib_node_index_t path_list_index);
 extern fib_protocol_t fib_path_list_get_proto(fib_node_index_t path_list_index);
 extern u8 * fib_path_list_format(fib_node_index_t pl_index,
 				 u8 * s);
-extern u8 * fib_path_list_adjs_format(fib_node_index_t pl_index,
-				      u32 indent,
-				      u8 * s);
 extern index_t fib_path_list_lb_map_add_or_lock(fib_node_index_t pl_index,
                                                 const fib_node_index_t *pis);
+extern u32 fib_path_list_find_rpath (fib_node_index_t path_list_index,
+                                     const fib_route_path_t *rpath);
+
 /**
  * A callback function type for walking a path-list's paths
  */
diff --git a/src/vnet/fib/fib_table.c b/src/vnet/fib/fib_table.c
index 6c3162e7..b31f35e3 100644
--- a/src/vnet/fib/fib_table.c
+++ b/src/vnet/fib/fib_table.c
@@ -475,8 +475,21 @@ fib_table_entry_special_remove (u32 fib_index,
  */
 static void
 fib_table_route_path_fixup (const fib_prefix_t *prefix,
+                            fib_entry_flag_t eflags,
 			    fib_route_path_t *path)
 {
+    /*
+     * not all zeros next hop &&
+     * is recursive path &&
+     * nexthop is same as the route's address
+     */
+    if ((!ip46_address_is_zero(&path->frp_addr)) &&
+        (~0 == path->frp_sw_if_index) &&
+        (0 == ip46_address_cmp(&path->frp_addr, &prefix->fp_addr)))
+    {
+        /* Prefix recurses via itse;f */
+	path->frp_flags |= FIB_ROUTE_PATH_DROP;
+    }
     if (fib_prefix_is_host(prefix) &&
 	ip46_address_is_zero(&path->frp_addr) &&
 	path->frp_sw_if_index != ~0)
@@ -484,7 +497,19 @@ fib_table_route_path_fixup (const fib_prefix_t *prefix,
 	path->frp_addr = prefix->fp_addr;
         path->frp_flags |= FIB_ROUTE_PATH_ATTACHED;
     }
-}		  
+    if (eflags & FIB_ENTRY_FLAG_DROP)
+    {
+	path->frp_flags |= FIB_ROUTE_PATH_DROP;
+    }
+    if (eflags & FIB_ENTRY_FLAG_LOCAL)
+    {
+	path->frp_flags |= FIB_ROUTE_PATH_LOCAL;
+    }
+    if (eflags & FIB_ENTRY_FLAG_EXCLUSIVE)
+    {
+	path->frp_flags |= FIB_ROUTE_PATH_EXCLUSIVE;
+    }
+}
 
 fib_node_index_t
 fib_table_entry_path_add (u32 fib_index,
@@ -536,7 +561,7 @@ fib_table_entry_path_add2 (u32 fib_index,
 
     for (ii = 0; ii < vec_len(rpath); ii++)
     {
-	fib_table_route_path_fixup(prefix, &rpath[ii]);
+	fib_table_route_path_fixup(prefix, flags, &rpath[ii]);
     }
 
     if (FIB_NODE_INDEX_INVALID == fib_entry_index)
@@ -583,11 +608,6 @@ fib_table_entry_path_remove2 (u32 fib_index,
     fib_table = fib_table_get(fib_index, prefix->fp_proto);
     fib_entry_index = fib_table_lookup_exact_match_i(fib_table, prefix);
 
-    for (ii = 0; ii < vec_len(rpath); ii++)
-    {
-	fib_table_route_path_fixup(prefix, &rpath[ii]);
-    }
-
     if (FIB_NODE_INDEX_INVALID == fib_entry_index)
     {
 	/*
@@ -605,6 +625,15 @@ fib_table_entry_path_remove2 (u32 fib_index,
 	fib_entry_lock(fib_entry_index);
         was_sourced = fib_entry_is_sourced(fib_entry_index, source);
 
+        for (ii = 0; ii < vec_len(rpath); ii++)
+        {
+            fib_table_route_path_fixup(
+                prefix,
+                fib_entry_get_flags_for_source(fib_entry_index,
+                                               source),
+                &rpath[ii]);
+        }
+
 	src_flag = fib_entry_path_remove(fib_entry_index, source, rpath);
 
 	if (!(FIB_ENTRY_SRC_FLAG_ADDED & src_flag))
@@ -661,7 +690,6 @@ fib_table_entry_path_remove (u32 fib_index,
     };
     fib_route_path_t *paths = NULL;
 
-    fib_table_route_path_fixup(prefix, &path);
     vec_add1(paths, path);
 
     fib_table_entry_path_remove2(fib_index, prefix, source, paths);
@@ -692,7 +720,7 @@ fib_table_entry_update (u32 fib_index,
 
     for (ii = 0; ii < vec_len(paths); ii++)
     {
-	fib_table_route_path_fixup(prefix, &paths[ii]);
+	fib_table_route_path_fixup(prefix, flags, &paths[ii]);
     }
     /*
      * sort the paths provided by the control plane. this means
@@ -750,7 +778,6 @@ fib_table_entry_update_one_path (u32 fib_index,
     };
     fib_route_path_t *paths = NULL;
 
-    fib_table_route_path_fixup(prefix, &path);
     vec_add1(paths, path);
 
     fib_entry_index = 
diff --git a/src/vnet/fib/fib_test.c b/src/vnet/fib/fib_test.c
index 3c9b8a38..e4a8a70e 100644
--- a/src/vnet/fib/fib_test.c
+++ b/src/vnet/fib/fib_test.c
@@ -25,6 +25,8 @@
 #include <vnet/dpo/receive_dpo.h>
 #include <vnet/dpo/ip_null_dpo.h>
 #include <vnet/bfd/bfd_main.h>
+#include <vnet/dpo/interface_dpo.h>
+#include <vnet/dpo/replicate_dpo.h>
 
 #include <vnet/mpls/mpls.h>
 
@@ -271,6 +273,7 @@ typedef enum fib_test_lb_bucket_type_t_ {
     FT_LB_O_LB,
     FT_LB_SPECIAL,
     FT_LB_ADJ,
+    FT_LB_INTF,
 } fib_test_lb_bucket_type_t;
 
 typedef struct fib_test_lb_bucket_t_ {
@@ -315,6 +318,31 @@ typedef struct fib_test_lb_bucket_t_ {
     };
 } fib_test_lb_bucket_t;
 
+typedef enum fib_test_rep_bucket_type_t_ {
+    FT_REP_LABEL_O_ADJ,
+    FT_REP_DISP_MFIB_LOOKUP,
+    FT_REP_INTF,
+} fib_test_rep_bucket_type_t;
+
+typedef struct fib_test_rep_bucket_t_ {
+    fib_test_rep_bucket_type_t type;
+
+    union
+    {
+	struct
+	{
+	    mpls_eos_bit_t eos;
+	    mpls_label_t label;
+	    u8 ttl;
+	    adj_index_t adj;
+	} label_o_adj;
+ 	struct
+	{
+	    adj_index_t adj;
+	} adj;
+   };
+} fib_test_rep_bucket_t;
+
 #define FIB_TEST_LB(_cond, _comment, _args...)			\
 {								\
     if (!FIB_TEST_I(_cond, _comment, ##_args)) {		\
@@ -322,7 +350,83 @@ typedef struct fib_test_lb_bucket_t_ {
     }								\
 }
 
-static int
+int
+fib_test_validate_rep_v (const replicate_t *rep,
+                         u16 n_buckets,
+                         va_list ap)
+{
+    const fib_test_rep_bucket_t *exp;
+    const dpo_id_t *dpo;
+    int bucket;
+
+    FIB_TEST_LB((n_buckets == rep->rep_n_buckets),
+                "n_buckets = %d", rep->rep_n_buckets);
+
+    for (bucket = 0; bucket < n_buckets; bucket++)
+    {
+	exp = va_arg(ap, fib_test_rep_bucket_t*);
+
+        dpo = replicate_get_bucket_i(rep, bucket);
+
+	switch (exp->type)
+	{
+	case FT_REP_LABEL_O_ADJ:
+	    {
+		const mpls_label_dpo_t *mld;
+                mpls_label_t hdr;
+		FIB_TEST_LB((DPO_MPLS_LABEL == dpo->dpoi_type),
+                            "bucket %d stacks on %U",
+                            bucket,
+                            format_dpo_type, dpo->dpoi_type);
+	    
+		mld = mpls_label_dpo_get(dpo->dpoi_index);
+                hdr = clib_net_to_host_u32(mld->mld_hdr[0].label_exp_s_ttl);
+
+		FIB_TEST_LB((vnet_mpls_uc_get_label(hdr) ==
+			     exp->label_o_adj.label),
+			    "bucket %d stacks on label %d",
+			    bucket,
+			    exp->label_o_adj.label);
+
+		FIB_TEST_LB((vnet_mpls_uc_get_s(hdr) ==
+			     exp->label_o_adj.eos),
+			    "bucket %d stacks on label %d %U",
+			    bucket,
+			    exp->label_o_adj.label,
+			    format_mpls_eos_bit, exp->label_o_adj.eos);
+
+		FIB_TEST_LB((DPO_ADJACENCY_INCOMPLETE == mld->mld_dpo.dpoi_type),
+			    "bucket %d label stacks on %U",
+			    bucket,
+			    format_dpo_type, mld->mld_dpo.dpoi_type);
+
+		FIB_TEST_LB((exp->label_o_adj.adj == mld->mld_dpo.dpoi_index),
+			    "bucket %d label stacks on adj %d",
+			    bucket,
+			    exp->label_o_adj.adj);
+	    }
+	    break;
+	case FT_REP_INTF:
+            FIB_TEST_LB((DPO_INTERFACE == dpo->dpoi_type),
+                        "bucket %d stacks on %U",
+                        bucket,
+                        format_dpo_type, dpo->dpoi_type);
+
+            FIB_TEST_LB((exp->adj.adj == dpo->dpoi_index),
+                        "bucket %d stacks on adj %d",
+                        bucket,
+                        exp->adj.adj);
+	    break;
+        case FT_REP_DISP_MFIB_LOOKUP:
+//            ASSERT(0);
+            break;
+        }
+    }
+
+    return (!0);
+}
+
+int
 fib_test_validate_lb_v (const load_balance_t *lb,
 			u16 n_buckets,
 			va_list ap)
@@ -484,6 +588,16 @@ fib_test_validate_lb_v (const load_balance_t *lb,
 			bucket,
 			exp->adj.adj);
 	    break;
+	case FT_LB_INTF:
+	    FIB_TEST_I((DPO_INTERFACE == dpo->dpoi_type),
+		       "bucket %d stacks on %U",
+		       bucket,
+		       format_dpo_type, dpo->dpoi_type);
+	    FIB_TEST_LB((exp->adj.adj == dpo->dpoi_index),
+			"bucket %d stacks on adj %d",
+			bucket,
+			exp->adj.adj);
+	    break;
 	case FT_LB_O_LB:
 	    FIB_TEST_I((DPO_LOAD_BALANCE == dpo->dpoi_type),
                        "bucket %d stacks on %U",
@@ -509,14 +623,13 @@ fib_test_validate_lb_v (const load_balance_t *lb,
     return (!0);
 }
 
-static int
+int
 fib_test_validate_entry (fib_node_index_t fei,
 			 fib_forward_chain_type_t fct,
 			 u16 n_buckets,
 			 ...)
 {
     dpo_id_t dpo = DPO_INVALID;
-    const load_balance_t *lb;
     fib_prefix_t pfx;
     index_t fw_lbi;
     u32 fib_index;
@@ -529,47 +642,59 @@ fib_test_validate_entry (fib_node_index_t fei,
     fib_index = fib_entry_get_fib_index(fei);
     fib_entry_contribute_forwarding(fei, fct, &dpo);
 
-    FIB_TEST_LB((DPO_LOAD_BALANCE == dpo.dpoi_type),
-		"Entry links to %U",
-		format_dpo_type, dpo.dpoi_type);
-    lb = load_balance_get(dpo.dpoi_index);
-
-    res = fib_test_validate_lb_v(lb, n_buckets, ap);
+    if (DPO_REPLICATE == dpo.dpoi_type)
+    {
+        const replicate_t *rep;
 
-    /*
-     * ensure that the LB contributed by the entry is the
-     * same as the LB in the forwarding tables
-     */
-    if (fct == fib_entry_get_default_chain_type(fib_entry_get(fei)))
+        rep = replicate_get(dpo.dpoi_index);
+        res = fib_test_validate_rep_v(rep, n_buckets, ap);
+    }
+    else
     {
-        switch (pfx.fp_proto)
-        {
-        case FIB_PROTOCOL_IP4:
-            fw_lbi = ip4_fib_forwarding_lookup(fib_index, &pfx.fp_addr.ip4);
-            break;
-        case FIB_PROTOCOL_IP6:
-            fw_lbi = ip6_fib_table_fwding_lookup(&ip6_main, fib_index, &pfx.fp_addr.ip6);
-            break;
-        case FIB_PROTOCOL_MPLS:
-            {
-                mpls_unicast_header_t hdr = {
-                    .label_exp_s_ttl = 0,
-                };
+        const load_balance_t *lb;
+
+        FIB_TEST_LB((DPO_LOAD_BALANCE == dpo.dpoi_type),
+                    "Entry links to %U",
+                    format_dpo_type, dpo.dpoi_type);
 
-                vnet_mpls_uc_set_label(&hdr.label_exp_s_ttl, pfx.fp_label);
-                vnet_mpls_uc_set_s(&hdr.label_exp_s_ttl, pfx.fp_eos);
-                hdr.label_exp_s_ttl = clib_host_to_net_u32(hdr.label_exp_s_ttl);
+        lb = load_balance_get(dpo.dpoi_index);
+        res = fib_test_validate_lb_v(lb, n_buckets, ap);
 
-                fw_lbi = mpls_fib_table_forwarding_lookup(fib_index, &hdr);
+        /*
+         * ensure that the LB contributed by the entry is the
+         * same as the LB in the forwarding tables
+         */
+        if (fct == fib_entry_get_default_chain_type(fib_entry_get(fei)))
+        {
+            switch (pfx.fp_proto)
+            {
+            case FIB_PROTOCOL_IP4:
+                fw_lbi = ip4_fib_forwarding_lookup(fib_index, &pfx.fp_addr.ip4);
+                break;
+            case FIB_PROTOCOL_IP6:
+                fw_lbi = ip6_fib_table_fwding_lookup(&ip6_main, fib_index, &pfx.fp_addr.ip6);
                 break;
+            case FIB_PROTOCOL_MPLS:
+                {
+                    mpls_unicast_header_t hdr = {
+                        .label_exp_s_ttl = 0,
+                    };
+
+                    vnet_mpls_uc_set_label(&hdr.label_exp_s_ttl, pfx.fp_label);
+                    vnet_mpls_uc_set_s(&hdr.label_exp_s_ttl, pfx.fp_eos);
+                    hdr.label_exp_s_ttl = clib_host_to_net_u32(hdr.label_exp_s_ttl);
+
+                    fw_lbi = mpls_fib_table_forwarding_lookup(fib_index, &hdr);
+                    break;
+                }
+            default:
+                fw_lbi = 0;
             }
-        default:
-            fw_lbi = 0;
+            FIB_TEST_LB((fw_lbi == dpo.dpoi_index),
+                        "Contributed LB = FW LB: %U\n %U",
+                        format_load_balance, fw_lbi, 0,
+                        format_load_balance, dpo.dpoi_index, 0);
         }
-        FIB_TEST_LB((fw_lbi == dpo.dpoi_index),
-                    "Contributed LB = FW LB: %U\n %U",
-                    format_load_balance, fw_lbi, 0,
-                    format_load_balance, dpo.dpoi_index, 0);
     }
 
     dpo_reset(&dpo);
@@ -1289,6 +1414,7 @@ fib_test_v4 (void)
 
     lookup_dpo_add_or_lock_w_fib_index(fib_index,
                                        DPO_PROTO_IP4,
+                                       LOOKUP_UNICAST,
                                        LOOKUP_INPUT_DST_ADDR,
                                        LOOKUP_TABLE_FROM_CONFIG,
                                        &ex_dpo);
@@ -2605,7 +2731,6 @@ fib_test_v4 (void)
 			     NULL,
 			     FIB_ROUTE_PATH_FLAG_NONE);
 
-
     fei = fib_table_lookup(fib_index, &pfx_5_5_5_6_s_32);
     dpo1 = fib_entry_contribute_ip_forwarding(fei);
 
@@ -7493,6 +7618,7 @@ lfib_test (void)
     fib_route_path_t *rpaths = NULL, rpath = {
     	.frp_proto = FIB_PROTOCOL_MPLS,
     	.frp_local_label = 1200,
+        .frp_eos = MPLS_NON_EOS,
     	.frp_sw_if_index = ~0, // recurive
     	.frp_fib_index = 0, // Default MPLS fib
     	.frp_weight = 1,
@@ -7607,6 +7733,146 @@ lfib_test (void)
 
     dpo_reset(&ip_1200);
 
+    /*
+     * An rx-interface route.
+     *  like the tail of an mcast LSP
+     */
+    dpo_id_t idpo = DPO_INVALID;
+
+    interface_dpo_add_or_lock(DPO_PROTO_IP4,
+                              tm->hw[0]->sw_if_index,
+                              &idpo);
+
+    fib_prefix_t pfx_2500 = {
+	.fp_len = 21,
+	.fp_proto = FIB_PROTOCOL_MPLS,
+	.fp_label = 2500,
+	.fp_eos = MPLS_EOS,
+	.fp_payload_proto = DPO_PROTO_IP4,
+    };
+    fib_test_lb_bucket_t rx_intf_0 = {
+        .type = FT_LB_INTF,
+        .adj = {
+            .adj = idpo.dpoi_index,
+        },
+    };
+
+    lfe = fib_table_entry_update_one_path(fib_index,
+					  &pfx_2500,
+					  FIB_SOURCE_API,
+					  FIB_ENTRY_FLAG_NONE,
+					  FIB_PROTOCOL_IP4,
+					  NULL,
+					  tm->hw[0]->sw_if_index,
+					  ~0, // invalid fib index
+					  0,
+					  NULL,
+					  FIB_ROUTE_PATH_INTF_RX);
+    FIB_TEST(fib_test_validate_entry(lfe,
+    				     FIB_FORW_CHAIN_TYPE_MPLS_EOS,
+    				     1,
+    				     &rx_intf_0),
+    	     "2500 rx-interface 0");
+    fib_table_entry_delete(fib_index, &pfx_2500, FIB_SOURCE_API);
+
+    /*
+     * An MPLS mulicast entry
+     */
+    fib_prefix_t pfx_3500 = {
+	.fp_len = 21,
+	.fp_proto = FIB_PROTOCOL_MPLS,
+	.fp_label = 3500,
+	.fp_eos = MPLS_EOS,
+	.fp_payload_proto = DPO_PROTO_IP4,
+    };
+    fib_test_rep_bucket_t mc_0 = {
+        .type = FT_REP_LABEL_O_ADJ,
+	.label_o_adj = {
+	    .adj = ai_mpls_10_10_10_1,
+	    .label = 3300,
+	    .eos = MPLS_EOS,
+	},
+    };
+    fib_test_rep_bucket_t mc_intf_0 = {
+        .type = FT_REP_INTF,
+        .adj = {
+            .adj = idpo.dpoi_index,
+        },
+    };
+    mpls_label_t *l3300 = NULL;
+    vec_add1(l3300, 3300);
+
+    lfe = fib_table_entry_update_one_path(lfib_index,
+					  &pfx_3500,
+					  FIB_SOURCE_API,
+					  FIB_ENTRY_FLAG_MULTICAST,
+					  FIB_PROTOCOL_IP4,
+					  &nh_10_10_10_1,
+					  tm->hw[0]->sw_if_index,
+					  ~0, // invalid fib index
+					  1,
+					  l3300,
+					  FIB_ROUTE_PATH_FLAG_NONE);
+    FIB_TEST(fib_test_validate_entry(lfe,
+    				     FIB_FORW_CHAIN_TYPE_MPLS_EOS,
+    				     1,
+    				     &mc_0),
+    	     "3500 via replicate over 10.10.10.1");
+
+    /*
+     * MPLS Bud-node. Add a replication via an interface-receieve path
+     */
+    lfe = fib_table_entry_path_add(lfib_index,
+				   &pfx_3500,
+				   FIB_SOURCE_API,
+				   FIB_ENTRY_FLAG_MULTICAST,
+				   FIB_PROTOCOL_IP4,
+                                   NULL,
+                                   tm->hw[0]->sw_if_index,
+                                   ~0, // invalid fib index
+                                   0,
+                                   NULL,
+                                   FIB_ROUTE_PATH_INTF_RX);
+    FIB_TEST(fib_test_validate_entry(lfe,
+                                     FIB_FORW_CHAIN_TYPE_MPLS_EOS,
+                                     2,
+                                     &mc_0,
+                                     &mc_intf_0),
+    	     "3500 via replicate over 10.10.10.1 and interface-rx");
+
+    /*
+     * Add a replication via an interface-free for-us path
+     */
+    fib_test_rep_bucket_t mc_disp = {
+        .type = FT_REP_DISP_MFIB_LOOKUP,
+        .adj = {
+            .adj = idpo.dpoi_index,
+        },
+    };
+    lfe = fib_table_entry_path_add(lfib_index,
+				   &pfx_3500,
+				   FIB_SOURCE_API,
+				   FIB_ENTRY_FLAG_MULTICAST,
+				   FIB_PROTOCOL_IP4,
+                                   NULL,
+                                   5, // rpf-id
+                                   0, // default table
+                                   0,
+                                   NULL,
+                                   FIB_ROUTE_PATH_RPF_ID);
+    FIB_TEST(fib_test_validate_entry(lfe,
+                                     FIB_FORW_CHAIN_TYPE_MPLS_EOS,
+                                     3,
+                                     &mc_0,
+                                     &mc_disp,
+                                     &mc_intf_0),
+    	     "3500 via replicate over 10.10.10.1 and interface-rx");
+
+
+    
+    fib_table_entry_delete(fib_index, &pfx_3500, FIB_SOURCE_API);
+    dpo_reset(&idpo);
+
     /*
      * cleanup
      */
@@ -7617,6 +7883,9 @@ lfib_test (void)
     FIB_TEST(lb_count == pool_elts(load_balance_pool),
 	     "Load-balance resources freed %d of %d",
              lb_count, pool_elts(load_balance_pool));
+    FIB_TEST(0 == pool_elts(interface_dpo_pool),
+	     "interface_dpo resources freed %d of %d",
+             0, pool_elts(interface_dpo_pool));
 
     return (0);
 }
diff --git a/src/vnet/fib/fib_test.h b/src/vnet/fib/fib_test.h
new file mode 100644
index 00000000..b98680bf
--- /dev/null
+++ b/src/vnet/fib/fib_test.h
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __FIB_TEST_H__
+#define __FIB_TEST_H__
+
+#include <vnet/fib/fib_types.h>
+
+typedef enum fib_test_lb_bucket_type_t_ {
+    FT_LB_LABEL_O_ADJ,
+    FT_LB_LABEL_STACK_O_ADJ,
+    FT_LB_LABEL_O_LB,
+    FT_LB_O_LB,
+    FT_LB_SPECIAL,
+    FT_LB_ADJ,
+    FT_LB_INTF,
+} fib_test_lb_bucket_type_t;
+
+typedef struct fib_test_lb_bucket_t_ {
+    fib_test_lb_bucket_type_t type;
+
+    union
+    {
+	struct
+	{
+	    mpls_eos_bit_t eos;
+	    mpls_label_t label;
+	    u8 ttl;
+	    adj_index_t adj;
+	} label_o_adj;
+	struct
+	{
+	    mpls_eos_bit_t eos;
+	    mpls_label_t label_stack[8];
+	    u8 label_stack_size;
+	    u8 ttl;
+	    adj_index_t adj;
+	} label_stack_o_adj;
+	struct
+	{
+	    mpls_eos_bit_t eos;
+	    mpls_label_t label;
+	    u8 ttl;
+	    index_t lb;
+	} label_o_lb;
+	struct
+	{
+	    index_t adj;
+	} adj;
+	struct
+	{
+	    index_t lb;
+	} lb;
+	struct
+	{
+	    index_t adj;
+	} special;
+    };
+} fib_test_lb_bucket_t;
+
+typedef enum fib_test_rep_bucket_type_t_ {
+    FT_REP_LABEL_O_ADJ,
+    FT_REP_INTF,
+} fib_test_rep_bucket_type_t;
+
+typedef struct fib_test_rep_bucket_t_ {
+    fib_test_rep_bucket_type_t type;
+
+    union
+    {
+	struct
+	{
+	    mpls_eos_bit_t eos;
+	    mpls_label_t label;
+	    u8 ttl;
+	    adj_index_t adj;
+	} label_o_adj;
+ 	struct
+	{
+	    adj_index_t adj;
+	} adj;
+   };
+} fib_test_rep_bucket_t;
+
+
+extern int fib_test_validate_rep_v(const replicate_t *rep,
+                                   u16 n_buckets,
+                                   va_list ap);
+
+extern int fib_test_validate_lb_v(const load_balance_t *lb,
+                                  u16 n_buckets,
+                                  va_list ap);
+
+extern int fib_test_validate_entry(fib_node_index_t fei,
+                                   fib_forward_chain_type_t fct,
+                                   u16 n_buckets,
+                                   ...);
+
+#endif
diff --git a/src/vnet/fib/fib_types.c b/src/vnet/fib/fib_types.c
index 2837a59d..8165f3eb 100644
--- a/src/vnet/fib/fib_types.c
+++ b/src/vnet/fib/fib_types.c
@@ -66,12 +66,13 @@ fib_prefix_from_ip46_addr (const ip46_address_t *addr,
 
 void
 fib_prefix_from_mpls_label (mpls_label_t label,
+                            mpls_eos_bit_t eos,
 			    fib_prefix_t *pfx)
 {
     pfx->fp_proto = FIB_PROTOCOL_MPLS;
     pfx->fp_len = 21;
     pfx->fp_label = label;
-    pfx->fp_eos = MPLS_NON_EOS;
+    pfx->fp_eos = eos;
 }
 
 int
@@ -194,17 +195,7 @@ fib_route_path_cmp (const fib_route_path_t *rpath1,
 
     if (0 != res) return (res);
 
-    if (~0 != rpath1->frp_sw_if_index &&
-        ~0 != rpath2->frp_sw_if_index)
-    {
-        res = vnet_sw_interface_compare(vnet_get_main(),
-                                        rpath1->frp_sw_if_index,
-                                        rpath2->frp_sw_if_index);
-    }
-    else
-    {
-        res = rpath1->frp_sw_if_index - rpath2->frp_sw_if_index;
-    }
+    res = (rpath1->frp_sw_if_index - rpath2->frp_sw_if_index);
 
     if (0 != res) return (res);
 
diff --git a/src/vnet/fib/fib_types.h b/src/vnet/fib/fib_types.h
index 1c5299a9..4cb73e8a 100644
--- a/src/vnet/fib/fib_types.h
+++ b/src/vnet/fib/fib_types.h
@@ -286,8 +286,36 @@ typedef enum fib_route_path_flags_t_
      * Attached path
      */
     FIB_ROUTE_PATH_ATTACHED = (1 << 3),
+    /**
+     * A Drop path - resolve the path on the drop DPO
+     */
+    FIB_ROUTE_PATH_DROP = (1 << 4),
+    /**
+     * Don't resolve the path, use the DPO the client provides
+     */
+    FIB_ROUTE_PATH_EXCLUSIVE = (1 << 5),
+    /**
+     * A path that result in received traffic being recieved/recirculated
+     * so that it appears to have arrived on the new interface
+     */
+    FIB_ROUTE_PATH_INTF_RX = (1 << 6),
+    /**
+     * A local path with a RPF-ID => multicast traffic
+     */
+    FIB_ROUTE_PATH_RPF_ID = (1 << 7),
 } fib_route_path_flags_t;
 
+/**
+ * An RPF-ID is numerical value that is used RPF validate. An entry
+ * has-a RPF-ID, when a packet egress from (e.g. an LSP) it gains an
+ * RPF-ID, these two are compared for the RPF check.
+ * This replaces the interfce based chack (since the LSP has no associated
+ * interface.
+ */
+typedef u32 fib_rpf_id_t;
+
+#define MFIB_RPF_ID_NONE (0)
+
 /**
  * @brief 
  * A representation of a path as described by a route producer.
@@ -321,17 +349,29 @@ typedef struct fib_route_path_t_ {
 	 */
 	ip46_address_t frp_addr;
 
-	/**
-	 * The MPLS local Label to reursively resolve through.
-	 * This is valid when the path type is MPLS.
-	 */
-	mpls_label_t frp_local_label;
+        struct {
+            /**
+             * The MPLS local Label to reursively resolve through.
+             * This is valid when the path type is MPLS.
+             */
+            mpls_label_t frp_local_label;
+            /**
+             * EOS bit for the resolving label
+             */
+            mpls_eos_bit_t frp_eos;
+        };
+    };
+    union {
+        /**
+         * The interface.
+         * Will be invalid for recursive paths.
+         */
+        u32 frp_sw_if_index;
+        /**
+         * The RPF-ID
+         */
+        fib_rpf_id_t frp_rpf_id;
     };
-    /**
-     * The interface.
-     * Will be invalid for recursive paths.
-     */
-    u32 frp_sw_if_index;
     /**
      * The FIB index to lookup the nexthop
      * Only valid for recursive paths.
diff --git a/src/vnet/fib/mpls_fib.c b/src/vnet/fib/mpls_fib.c
index 4b2b76ea..19f9f3c1 100644
--- a/src/vnet/fib/mpls_fib.c
+++ b/src/vnet/fib/mpls_fib.c
@@ -165,6 +165,7 @@ mpls_fib_create_with_table_id (u32 table_id)
 
     lookup_dpo_add_or_lock_w_fib_index(0, // unused
                                        DPO_PROTO_IP4,
+                                       LOOKUP_UNICAST,
                                        LOOKUP_INPUT_DST_ADDR,
                                        LOOKUP_TABLE_FROM_INPUT_INTERFACE,
                                        &dpo);
@@ -179,6 +180,7 @@ mpls_fib_create_with_table_id (u32 table_id)
 
     lookup_dpo_add_or_lock_w_fib_index(0, //unsued
                                        DPO_PROTO_MPLS,
+                                       LOOKUP_UNICAST,
                                        LOOKUP_INPUT_DST_ADDR,
                                        LOOKUP_TABLE_FROM_INPUT_INTERFACE,
                                        &dpo);
@@ -197,6 +199,7 @@ mpls_fib_create_with_table_id (u32 table_id)
 
     lookup_dpo_add_or_lock_w_fib_index(0, //unused
                                        DPO_PROTO_IP6,
+                                       LOOKUP_UNICAST,
                                        LOOKUP_INPUT_DST_ADDR,
                                        LOOKUP_TABLE_FROM_INPUT_INTERFACE,
                                        &dpo);
@@ -210,6 +213,7 @@ mpls_fib_create_with_table_id (u32 table_id)
     prefix.fp_eos = MPLS_NON_EOS;
     lookup_dpo_add_or_lock_w_fib_index(0, // unsued
                                        DPO_PROTO_MPLS,
+                                       LOOKUP_UNICAST,
                                        LOOKUP_INPUT_DST_ADDR,
                                        LOOKUP_TABLE_FROM_INPUT_INTERFACE,
                                        &dpo);
@@ -320,8 +324,15 @@ mpls_fib_forwarding_table_update (mpls_fib_t *mf,
 {
     mpls_label_t key;
 
-    ASSERT(DPO_LOAD_BALANCE == dpo->dpoi_type);
-
+    ASSERT((DPO_LOAD_BALANCE == dpo->dpoi_type) ||
+           (DPO_REPLICATE == dpo->dpoi_type));
+    if (CLIB_DEBUG > 0)
+    {
+        if (DPO_REPLICATE == dpo->dpoi_type)
+            ASSERT(dpo->dpoi_index & MPLS_IS_REPLICATE);
+        if (DPO_LOAD_BALANCE == dpo->dpoi_type)
+            ASSERT(!(dpo->dpoi_index & MPLS_IS_REPLICATE));
+    }
     key = mpls_fib_entry_mk_key(label, eos);
 
     mf->mf_lbs[key] = dpo->dpoi_index;
diff --git a/src/vnet/handoff.h b/src/vnet/handoff.h
index 815206a9..04ba8bfb 100644
--- a/src/vnet/handoff.h
+++ b/src/vnet/handoff.h
@@ -150,7 +150,7 @@ eth_get_sym_key (ethernet_header_t * h0)
 			ip->dst_address.as_u64[0] ^
 			ip->dst_address.as_u64[1] ^ ip->protocol);
     }
-  else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST))
+  else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
     {
       hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1));
     }
@@ -179,8 +179,7 @@ eth_get_sym_key (ethernet_header_t * h0)
 		   ip->dst_address.as_u64[0] ^
 		   ip->dst_address.as_u64[1] ^ ip->protocol);
 	}
-      else if (outer->type ==
-	       clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST))
+      else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
 	{
 	  hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1));
 	}
@@ -210,7 +209,7 @@ eth_get_key (ethernet_header_t * h0)
     {
       hash_key = ipv6_get_key ((ip6_header_t *) (h0 + 1));
     }
-  else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST))
+  else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
     {
       hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1));
     }
@@ -230,8 +229,7 @@ eth_get_key (ethernet_header_t * h0)
 	{
 	  hash_key = ipv6_get_key ((ip6_header_t *) (outer + 1));
 	}
-      else if (outer->type ==
-	       clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST))
+      else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
 	{
 	  hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1));
 	}
diff --git a/src/vnet/interface.c b/src/vnet/interface.c
index 2a1e70e8..45417b2f 100644
--- a/src/vnet/interface.c
+++ b/src/vnet/interface.c
@@ -1360,7 +1360,7 @@ vnet_link_to_l3_proto (vnet_link_t link)
     case VNET_LINK_IP6:
       return (VNET_L3_PACKET_TYPE_IP6);
     case VNET_LINK_MPLS:
-      return (VNET_L3_PACKET_TYPE_MPLS_UNICAST);
+      return (VNET_L3_PACKET_TYPE_MPLS);
     case VNET_LINK_ARP:
       return (VNET_L3_PACKET_TYPE_ARP);
     case VNET_LINK_ETHERNET:
diff --git a/src/vnet/ip/ip.api b/src/vnet/ip/ip.api
index 5c2df32c..6af1714f 100644
--- a/src/vnet/ip/ip.api
+++ b/src/vnet/ip/ip.api
@@ -478,6 +478,7 @@ define ip_mroute_add_del
   u32 table_id;
   u32 entry_flags;
   u32 itf_flags;
+  u32 rpf_id;
   u16 grp_address_length;
   u8 create_vrf_if_needed;
   u8 is_add;
@@ -518,6 +519,8 @@ manual_endian manual_print define ip_mfib_details
 {
   u32 context;
   u32 table_id;
+  u32 entry_flags;
+  u32 rpf_id;
   u8  address_length;
   u8  grp_address[4];
   u8  src_address[4];
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index fdfe7f63..9fdf9b3c 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -2752,6 +2752,16 @@ ip4_rewrite_mcast (vlib_main_t * vm,
     return ip4_rewrite_inline (vm, node, frame, 0, 0, 1);
 }
 
+static uword
+ip4_mcast_midchain (vlib_main_t * vm,
+		    vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  if (adj_are_counters_enabled ())
+    return ip4_rewrite_inline (vm, node, frame, 1, 1, 1);
+  else
+    return ip4_rewrite_inline (vm, node, frame, 0, 1, 1);
+}
+
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (ip4_rewrite_node) = {
   .function = ip4_rewrite,
@@ -2778,6 +2788,16 @@ VLIB_REGISTER_NODE (ip4_rewrite_mcast_node) = {
 };
 VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_mcast_node, ip4_rewrite_mcast)
 
+VLIB_REGISTER_NODE (ip4_mcast_midchain_node, static) = {
+  .function = ip4_mcast_midchain,
+  .name = "ip4-mcast-midchain",
+  .vector_size = sizeof (u32),
+
+  .format_trace = format_ip4_rewrite_trace,
+  .sibling_of = "ip4-rewrite",
+};
+VLIB_NODE_FUNCTION_MULTIARCH (ip4_mcast_midchain_node, ip4_mcast_midchain)
+
 VLIB_REGISTER_NODE (ip4_midchain_node) = {
   .function = ip4_midchain,
   .name = "ip4-midchain",
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index c2fc4f87..a369f79f 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -2246,6 +2246,16 @@ ip6_midchain (vlib_main_t * vm,
     return ip6_rewrite_inline (vm, node, frame, 0, 1, 0);
 }
 
+static uword
+ip6_mcast_midchain (vlib_main_t * vm,
+		    vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  if (adj_are_counters_enabled ())
+    return ip6_rewrite_inline (vm, node, frame, 1, 1, 1);
+  else
+    return ip6_rewrite_inline (vm, node, frame, 1, 1, 1);
+}
+
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (ip6_midchain_node) =
 {
@@ -2290,6 +2300,19 @@ VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) =
 
 VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_mcast_node, ip6_rewrite_mcast);
 
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (ip6_mcast_midchain_node, static) =
+{
+  .function = ip6_mcast_midchain,
+  .name = "ip6-mcast-midchain",
+  .vector_size = sizeof (u32),
+  .format_trace = format_ip6_rewrite_trace,
+  .sibling_of = "ip6-rewrite",
+};
+/* *INDENT-ON* */
+
+VLIB_NODE_FUNCTION_MULTIARCH (ip6_mcast_midchain_node, ip6_mcast_midchain);
+
 /*
  * Hop-by-Hop handling
  */
diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c
index 2af546df..58b997aa 100644
--- a/src/vnet/ip/ip6_neighbor.c
+++ b/src/vnet/ip/ip6_neighbor.c
@@ -557,6 +557,7 @@ ip6_ethernet_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai)
     case IP_LOOKUP_NEXT_PUNT:
     case IP_LOOKUP_NEXT_LOCAL:
     case IP_LOOKUP_NEXT_REWRITE:
+    case IP_LOOKUP_NEXT_MCAST_MIDCHAIN:
     case IP_LOOKUP_NEXT_MIDCHAIN:
     case IP_LOOKUP_NEXT_ICMP_ERROR:
     case IP_LOOKUP_N_NEXT:
diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c
index b9f1782b..9c9cb4a4 100644
--- a/src/vnet/ip/ip_api.c
+++ b/src/vnet/ip/ip_api.c
@@ -438,17 +438,20 @@ vl_api_ip6_fib_dump_t_handler (vl_api_ip6_fib_dump_t * mp)
 }
 
 static void
-send_ip_mfib_details (vpe_api_main_t * am,
-		      unix_shared_memory_queue_t * q,
-		      u32 table_id,
-		      mfib_prefix_t * pfx,
-		      fib_route_path_encode_t * api_rpaths, u32 context)
+send_ip_mfib_details (unix_shared_memory_queue_t * q,
+		      u32 context, u32 table_id, fib_node_index_t mfei)
 {
+  fib_route_path_encode_t *api_rpath, *api_rpaths = NULL;
   vl_api_ip_mfib_details_t *mp;
-  fib_route_path_encode_t *api_rpath;
+  mfib_entry_t *mfib_entry;
   vl_api_fib_path_t *fp;
+  mfib_prefix_t pfx;
   int path_count;
 
+  mfib_entry = mfib_entry_get (mfei);
+  mfib_entry_get_prefix (mfei, &pfx);
+  mfib_entry_encode (mfei, &api_rpaths);
+
   path_count = vec_len (api_rpaths);
   mp = vl_msg_api_alloc (sizeof (*mp) + path_count * sizeof (*fp));
   if (!mp)
@@ -457,12 +460,14 @@ send_ip_mfib_details (vpe_api_main_t * am,
   mp->_vl_msg_id = ntohs (VL_API_IP_FIB_DETAILS);
   mp->context = context;
 
+  mp->rpf_id = mfib_entry->mfe_rpf_id;
+  mp->entry_flags = mfib_entry->mfe_flags;
   mp->table_id = htonl (table_id);
-  mp->address_length = pfx->fp_len;
-  memcpy (mp->grp_address, &pfx->fp_grp_addr.ip4,
-	  sizeof (pfx->fp_grp_addr.ip4));
-  memcpy (mp->src_address, &pfx->fp_src_addr.ip4,
-	  sizeof (pfx->fp_src_addr.ip4));
+  mp->address_length = pfx.fp_len;
+  memcpy (mp->grp_address, &pfx.fp_grp_addr.ip4,
+	  sizeof (pfx.fp_grp_addr.ip4));
+  memcpy (mp->src_address, &pfx.fp_src_addr.ip4,
+	  sizeof (pfx.fp_src_addr.ip4));
 
   mp->count = htonl (path_count);
   fp = mp->path;
@@ -475,6 +480,7 @@ send_ip_mfib_details (vpe_api_main_t * am,
     copy_fib_next_hop (api_rpath, fp);
     fp++;
   }
+  vec_free (api_rpaths);
 
   vl_msg_api_send_shmem (q, (u8 *) & mp);
 }
@@ -497,13 +503,10 @@ vl_api_ip_mfib_table_dump_walk (fib_node_index_t fei, void *arg)
 static void
 vl_api_ip_mfib_dump_t_handler (vl_api_ip_mfib_dump_t * mp)
 {
-  vpe_api_main_t *am = &vpe_api_main;
   unix_shared_memory_queue_t *q;
   ip4_main_t *im = &ip4_main;
   mfib_table_t *mfib_table;
   fib_node_index_t *mfeip;
-  mfib_prefix_t pfx;
-  fib_route_path_encode_t *api_rpaths = NULL;
   vl_api_ip_mfib_dump_ctc_t ctx = {
     .entries = NULL,
   };
@@ -524,21 +527,16 @@ vl_api_ip_mfib_dump_t_handler (vl_api_ip_mfib_dump_t * mp)
 
     vec_foreach (mfeip, ctx.entries)
     {
-      mfib_entry_get_prefix (*mfeip, &pfx);
-      mfib_entry_encode (*mfeip, &api_rpaths);
-      send_ip_mfib_details (am, q,
+      send_ip_mfib_details (q, mp->context,
                             mfib_table->mft_table_id,
-                            &pfx, api_rpaths,
-                            mp->context);
+                            *mfeip);
     }
-    vec_reset_length (api_rpaths);
     vec_reset_length (ctx.entries);
 
   }));
   /* *INDENT-ON* */
 
   vec_free (ctx.entries);
-  vec_free (api_rpaths);
 }
 
 static void
@@ -705,10 +703,13 @@ add_del_route_t_handler (u8 is_multipath,
 			 u8 is_unreach,
 			 u8 is_prohibit,
 			 u8 is_local,
+			 u8 is_multicast,
 			 u8 is_classify,
 			 u32 classify_table_index,
 			 u8 is_resolve_host,
 			 u8 is_resolve_attached,
+			 u8 is_interface_rx,
+			 u8 is_rpf_id,
 			 u32 fib_index,
 			 const fib_prefix_t * prefix,
 			 u8 next_hop_proto_is_ip4,
@@ -731,16 +732,24 @@ add_del_route_t_handler (u8 is_multipath,
     .frp_label_stack = next_hop_out_label_stack,
   };
   fib_route_path_t *paths = NULL;
+  fib_entry_flag_t entry_flags = FIB_ENTRY_FLAG_NONE;
 
   if (MPLS_LABEL_INVALID != next_hop_via_label)
     {
       path.frp_proto = FIB_PROTOCOL_MPLS;
       path.frp_local_label = next_hop_via_label;
+      path.frp_eos = MPLS_NON_EOS;
     }
   if (is_resolve_host)
     path_flags |= FIB_ROUTE_PATH_RESOLVE_VIA_HOST;
   if (is_resolve_attached)
     path_flags |= FIB_ROUTE_PATH_RESOLVE_VIA_ATTACHED;
+  if (is_interface_rx)
+    path_flags |= FIB_ROUTE_PATH_INTF_RX;
+  if (is_rpf_id)
+    path_flags |= FIB_ROUTE_PATH_RPF_ID;
+  if (is_multicast)
+    entry_flags |= FIB_ENTRY_FLAG_MULTICAST;
 
   path.frp_flags = path_flags;
 
@@ -754,8 +763,7 @@ add_del_route_t_handler (u8 is_multipath,
       if (is_add)
 	fib_table_entry_path_add2 (fib_index,
 				   prefix,
-				   FIB_SOURCE_API,
-				   FIB_ENTRY_FLAG_NONE, paths);
+				   FIB_SOURCE_API, entry_flags, paths);
       else
 	fib_table_entry_path_remove2 (fib_index,
 				      prefix, FIB_SOURCE_API, paths);
@@ -826,8 +834,7 @@ add_del_route_t_handler (u8 is_multipath,
 	{
 	  vec_add1 (paths, path);
 	  fib_table_entry_update (fib_index,
-				  prefix,
-				  FIB_SOURCE_API, FIB_ENTRY_FLAG_NONE, paths);
+				  prefix, FIB_SOURCE_API, entry_flags, paths);
 	  vec_free (paths);
 	}
       else
@@ -847,7 +854,7 @@ add_del_route_check (fib_protocol_t table_proto,
 		     fib_protocol_t next_hop_table_proto,
 		     u32 next_hop_table_id,
 		     u8 create_missing_tables,
-		     u32 * fib_index, u32 * next_hop_fib_index)
+		     u8 is_rpf_id, u32 * fib_index, u32 * next_hop_fib_index)
 {
   vnet_main_t *vnm = vnet_get_main ();
 
@@ -866,7 +873,7 @@ add_del_route_check (fib_protocol_t table_proto,
 	}
     }
 
-  if (~0 != ntohl (next_hop_sw_if_index))
+  if (!is_rpf_id && ~0 != ntohl (next_hop_sw_if_index))
     {
       if (pool_is_free_index (vnm->interface_main.sw_interfaces,
 			      ntohl (next_hop_sw_if_index)))
@@ -876,16 +883,27 @@ add_del_route_check (fib_protocol_t table_proto,
     }
   else
     {
-      *next_hop_fib_index = fib_table_find (next_hop_table_proto,
-					    ntohl (next_hop_table_id));
+      if (is_rpf_id)
+	*next_hop_fib_index = mfib_table_find (next_hop_table_proto,
+					       ntohl (next_hop_table_id));
+      else
+	*next_hop_fib_index = fib_table_find (next_hop_table_proto,
+					      ntohl (next_hop_table_id));
 
       if (~0 == *next_hop_fib_index)
 	{
 	  if (create_missing_tables)
 	    {
-	      *next_hop_fib_index =
-		fib_table_find_or_create_and_lock (next_hop_table_proto,
-						   ntohl (next_hop_table_id));
+	      if (is_rpf_id)
+		*next_hop_fib_index =
+		  mfib_table_find_or_create_and_lock (next_hop_table_proto,
+						      ntohl
+						      (next_hop_table_id));
+	      else
+		*next_hop_fib_index =
+		  fib_table_find_or_create_and_lock (next_hop_table_proto,
+						     ntohl
+						     (next_hop_table_id));
 	    }
 	  else
 	    {
@@ -910,7 +928,7 @@ ip4_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
 			    mp->next_hop_sw_if_index,
 			    FIB_PROTOCOL_IP4,
 			    mp->next_hop_table_id,
-			    mp->create_vrf_if_needed,
+			    mp->create_vrf_if_needed, 0,
 			    &fib_index, &next_hop_fib_index);
 
   if (0 != rv)
@@ -943,11 +961,11 @@ ip4_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
 				   mp->is_drop,
 				   mp->is_unreach,
 				   mp->is_prohibit,
-				   mp->is_local,
+				   mp->is_local, 0,
 				   mp->is_classify,
 				   mp->classify_table_index,
 				   mp->is_resolve_host,
-				   mp->is_resolve_attached,
+				   mp->is_resolve_attached, 0, 0,
 				   fib_index, &pfx, 1,
 				   &nh,
 				   ntohl (mp->next_hop_sw_if_index),
@@ -969,7 +987,7 @@ ip6_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
 			    mp->next_hop_sw_if_index,
 			    FIB_PROTOCOL_IP6,
 			    mp->next_hop_table_id,
-			    mp->create_vrf_if_needed,
+			    mp->create_vrf_if_needed, 0,
 			    &fib_index, &next_hop_fib_index);
 
   if (0 != rv)
@@ -1002,11 +1020,11 @@ ip6_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
 				   mp->is_drop,
 				   mp->is_unreach,
 				   mp->is_prohibit,
-				   mp->is_local,
+				   mp->is_local, 0,
 				   mp->is_classify,
 				   mp->classify_table_index,
 				   mp->is_resolve_host,
-				   mp->is_resolve_attached,
+				   mp->is_resolve_attached, 0, 0,
 				   fib_index, &pfx, 0,
 				   &nh, ntohl (mp->next_hop_sw_if_index),
 				   next_hop_fib_index,
@@ -1075,6 +1093,7 @@ mroute_add_del_handler (u8 is_add,
 			u32 fib_index,
 			const mfib_prefix_t * prefix,
 			u32 entry_flags,
+			fib_rpf_id_t rpf_id,
 			u32 next_hop_sw_if_index, u32 itf_flags)
 {
   stats_dslock_with_hint (1 /* release hint */ , 2 /* tag */ );
@@ -1091,7 +1110,7 @@ mroute_add_del_handler (u8 is_add,
   if (!is_local && ~0 == next_hop_sw_if_index)
     {
       mfib_table_entry_update (fib_index, prefix,
-			       MFIB_SOURCE_API, entry_flags);
+			       MFIB_SOURCE_API, rpf_id, entry_flags);
     }
   else
     {
@@ -1152,6 +1171,7 @@ api_mroute_add_del_t_handler (vl_api_ip_mroute_add_del_t * mp)
 				  mp->is_local,
 				  fib_index, &pfx,
 				  ntohl (mp->entry_flags),
+				  ntohl (mp->rpf_id),
 				  ntohl (mp->next_hop_sw_if_index),
 				  ntohl (mp->itf_flags)));
 }
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index ec9a1f97..597de06b 100755
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -450,6 +450,7 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 			 unformat_mpls_unicast_label, &rpath.frp_local_label))
 	{
 	  rpath.frp_weight = 1;
+	  rpath.frp_eos = MPLS_NON_EOS;
 	  rpath.frp_proto = FIB_PROTOCOL_MPLS;
 	  rpath.frp_sw_if_index = ~0;
 	  vec_add1 (rpaths, rpath);
@@ -923,7 +924,7 @@ vnet_ip_mroute_cmd (vlib_main_t * vm,
 	  else if (eflags)
 	    {
 	      mfib_table_entry_update (fib_index, &pfx, MFIB_SOURCE_CLI,
-				       eflags);
+				       MFIB_RPF_ID_NONE, eflags);
 	    }
 	  else
 	    {
diff --git a/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c b/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c
index efa724e0..d2954e96 100644
--- a/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c
+++ b/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c
@@ -88,6 +88,7 @@ ip_dst_fib_add_route (u32 dst_fib_index, const ip_prefix_t * dst_prefix)
 					  (ip_prefix_version (dst_prefix) ==
 					   IP6 ? DPO_PROTO_IP6 :
 					   DPO_PROTO_IP4),
+					  LOOKUP_UNICAST,
 					  LOOKUP_INPUT_SRC_ADDR,
 					  LOOKUP_TABLE_FROM_CONFIG,
 					  &src_lkup_dpo);
diff --git a/src/vnet/mfib/ip4_mfib.c b/src/vnet/mfib/ip4_mfib.c
index 164cafa1..3ed7cba7 100644
--- a/src/vnet/mfib/ip4_mfib.c
+++ b/src/vnet/mfib/ip4_mfib.c
@@ -72,6 +72,7 @@ ip4_create_mfib_with_table_id (u32 table_id)
         mfib_table_entry_update(mfib_table->mft_index,
                                 &prefix,
                                 MFIB_SOURCE_DEFAULT_ROUTE,
+                                MFIB_RPF_ID_NONE,
                                 MFIB_ENTRY_FLAG_DROP);
     }
 
diff --git a/src/vnet/mfib/ip6_mfib.c b/src/vnet/mfib/ip6_mfib.c
index 991b91c6..116fee22 100644
--- a/src/vnet/mfib/ip6_mfib.c
+++ b/src/vnet/mfib/ip6_mfib.c
@@ -195,6 +195,7 @@ ip6_create_mfib_with_table_id (u32 table_id)
     mfib_table_entry_update(mfib_table->mft_index,
                             &all_zeros,
                             MFIB_SOURCE_DEFAULT_ROUTE,
+                            MFIB_RPF_ID_NONE,
                             MFIB_ENTRY_FLAG_DROP);
 
     /*
diff --git a/src/vnet/mfib/mfib_entry.c b/src/vnet/mfib/mfib_entry.c
index 1aa8e086..847f25e7 100644
--- a/src/vnet/mfib/mfib_entry.c
+++ b/src/vnet/mfib/mfib_entry.c
@@ -48,6 +48,15 @@
 #define MFIB_ENTRY_DBG(_e, _fmt, _args...)
 #endif
 
+/**
+ * MFIB extensions to each path
+ */
+typedef struct mfib_path_ext_t_
+{
+    mfib_itf_flags_t mfpe_flags;
+    fib_node_index_t mfpe_path;
+} mfib_path_ext_t;
+
 /**
  * The source of an MFIB entry
  */
@@ -58,22 +67,39 @@ typedef struct mfib_entry_src_t_
      */
     mfib_source_t mfes_src;
 
+    /**
+     * Route flags
+     */
+    mfib_entry_flags_t mfes_flags;
+
     /**
      * The path-list of forwarding interfaces
      */
     fib_node_index_t mfes_pl;
 
     /**
-     * Route flags
+     * RPF-ID
      */
-    mfib_entry_flags_t mfes_flags;
+    fib_rpf_id_t mfes_rpf_id;
+
+    /**
+     * Hash table of path extensions
+     */
+    mfib_path_ext_t *mfes_exts;
 
     /**
-     * The hash table of all interfaces
+     * The hash table of all interfaces.
+     *  This is forwarding time information derived from the paths
+     *  and their extensions.
      */
     mfib_itf_t *mfes_itfs;
 } mfib_entry_src_t;
 
+/**
+ * Pool of path extensions
+ */
+static mfib_path_ext_t *mfib_path_ext_pool;
+
 /**
  * String names for each source
  */
@@ -123,6 +149,24 @@ format_mfib_entry_dpo (u8 * s, va_list * args)
                    MFIB_ENTRY_FORMAT_BRIEF));
 }
 
+static inline mfib_path_ext_t *
+mfib_entry_path_ext_get (index_t mi)
+{
+    return (pool_elt_at_index(mfib_path_ext_pool, mi));
+}
+
+static u8 *
+format_mfib_entry_path_ext (u8 * s, va_list * args)
+{
+    mfib_path_ext_t *path_ext;
+    index_t mpi = va_arg(*args, index_t);
+
+    path_ext = mfib_entry_path_ext_get(mpi);
+    return (format(s, "path:%d flags:%U",
+                   path_ext->mfpe_path,
+                   format_mfib_itf_flags, path_ext->mfpe_flags));
+}
+
 u8 *
 format_mfib_entry (u8 * s, va_list * args)
 {
@@ -141,6 +185,8 @@ format_mfib_entry (u8 * s, va_list * args)
 
     if (level >= MFIB_ENTRY_FORMAT_DETAIL)
     {
+        fib_node_index_t path_index, mpi;
+
         s = format (s, "\n");
         s = format (s, " fib:%d", mfib_entry->mfe_fib_index);
         s = format (s, " index:%d", mfib_entry_get_index(mfib_entry));
@@ -153,6 +199,14 @@ format_mfib_entry (u8 * s, va_list * args)
             {
                 s = fib_path_list_format(msrc->mfes_pl, s);
             }
+            s = format (s, "    Extensions:\n",
+                        mfib_source_names[msrc->mfes_src]);
+            hash_foreach(path_index, mpi, msrc->mfes_exts,
+            ({
+                s = format(s, "     %U\n", format_mfib_entry_path_ext, mpi);
+            }));
+            s = format (s, "    Interface-Forwarding:\n",
+                        mfib_source_names[msrc->mfes_src]);
             hash_foreach(sw_if_index, mfi, msrc->mfes_itfs,
             ({
                 s = format(s, "    %U\n", format_mfib_itf, mfi);
@@ -165,7 +219,7 @@ format_mfib_entry (u8 * s, va_list * args)
     ({
         s = format(s, "\n  %U", format_mfib_itf, mfi);
     }));
-
+    s = format(s, "\n  RPF-ID:%d", mfib_entry->mfe_rpf_id);
     s = format(s, "\n  %U-chain\n  %U",
                format_fib_forw_chain_type,
                mfib_entry_get_default_chain_type(mfib_entry),
@@ -314,13 +368,6 @@ mfib_entry_src_remove (mfib_entry_t *mfib_entry,
     }
 }
 
-static int
-mfib_entry_src_n_itfs (const mfib_entry_src_t *msrc)
-{
-    return (hash_elts(msrc->mfes_itfs));
-}
-
-
 static void
 mfib_entry_last_lock_gone (fib_node_t *node)
 {
@@ -338,7 +385,6 @@ mfib_entry_last_lock_gone (fib_node_t *node)
         mfib_entry_src_flush(msrc);
     }
 
-    fib_path_list_unlock(mfib_entry->mfe_parent);
     vec_free(mfib_entry->mfe_srcs);
 
     fib_node_deinit(&mfib_entry->mfe_node);
@@ -417,10 +463,9 @@ mfib_entry_alloc (u32 fib_index,
     mfib_entry->mfe_flags = 0;
     mfib_entry->mfe_fib_index = fib_index;
     mfib_entry->mfe_prefix = *prefix;
-    mfib_entry->mfe_parent = FIB_NODE_INDEX_INVALID;
-    mfib_entry->mfe_sibling = FIB_NODE_INDEX_INVALID;
     mfib_entry->mfe_srcs = NULL;
     mfib_entry->mfe_itfs = NULL;
+    mfib_entry->mfe_rpf_id = MFIB_RPF_ID_NONE;
 
     dpo_reset(&mfib_entry->mfe_rep);
 
@@ -431,10 +476,57 @@ mfib_entry_alloc (u32 fib_index,
     return (mfib_entry);
 }
 
+static inline mfib_path_ext_t *
+mfib_entry_path_ext_find (mfib_path_ext_t *exts,
+                          fib_node_index_t path_index)
+{
+    uword *p;
+
+    p = hash_get(exts, path_index);
+
+    if (NULL != p)
+    {
+        return (mfib_entry_path_ext_get(p[0]));
+    }
+
+    return (NULL);
+}
+
+static mfib_path_ext_t*
+mfib_path_ext_add (mfib_entry_src_t *msrc,
+                   fib_node_index_t path_index,
+                   mfib_itf_flags_t mfi_flags)
+{
+    mfib_path_ext_t *path_ext;
+
+    pool_get(mfib_path_ext_pool, path_ext);
+
+    path_ext->mfpe_flags = mfi_flags;
+    path_ext->mfpe_path = path_index;
+
+    hash_set(msrc->mfes_exts, path_index,
+             path_ext - mfib_path_ext_pool);
+
+    return (path_ext);
+}
+
+static void
+mfib_path_ext_remove (mfib_entry_src_t *msrc,
+                      fib_node_index_t path_index)
+{
+    mfib_path_ext_t *path_ext;
+
+    path_ext = mfib_entry_path_ext_find(msrc->mfes_exts, path_index);
+
+    hash_unset(msrc->mfes_exts, path_index);
+    pool_put(mfib_path_ext_pool, path_ext);
+}
+
 typedef struct mfib_entry_collect_forwarding_ctx_t_
 {
     load_balance_path_t * next_hops;
     fib_forward_chain_type_t fct;
+    mfib_entry_src_t *msrc;
 } mfib_entry_collect_forwarding_ctx_t;
 
 static int
@@ -455,6 +547,20 @@ mfib_entry_src_collect_forwarding (fib_node_index_t pl_index,
         return (!0);
     }
 
+    /*
+     * If the path is not forwarding to use it
+     */
+    mfib_path_ext_t *path_ext;
+    
+    path_ext = mfib_entry_path_ext_find(ctx->msrc->mfes_exts,
+                                        path_index);
+
+    if (NULL != path_ext &&
+        !(path_ext->mfpe_flags & MFIB_ITF_FLAG_FORWARD))
+    {
+        return (!0);
+    }
+    
     switch (ctx->fct)
     {
     case FIB_FORW_CHAIN_TYPE_MCAST_IP4:
@@ -483,46 +589,61 @@ mfib_entry_src_collect_forwarding (fib_node_index_t pl_index,
 }
 
 static void
-mfib_entry_stack (mfib_entry_t *mfib_entry)
+mfib_entry_stack (mfib_entry_t *mfib_entry,
+                  mfib_entry_src_t *msrc)
 {
     dpo_proto_t dp;
 
     dp = fib_proto_to_dpo(mfib_entry_get_proto(mfib_entry));
 
-    if (FIB_NODE_INDEX_INVALID != mfib_entry->mfe_parent)
+    if (NULL != msrc &&
+        FIB_NODE_INDEX_INVALID != msrc->mfes_pl)
     {
         mfib_entry_collect_forwarding_ctx_t ctx = {
             .next_hops = NULL,
             .fct = mfib_entry_get_default_chain_type(mfib_entry),
+            .msrc = msrc,
         };
 
-        fib_path_list_walk(mfib_entry->mfe_parent,
+        fib_path_list_walk(msrc->mfes_pl,
                            mfib_entry_src_collect_forwarding,
                            &ctx);
 
         if (!(MFIB_ENTRY_FLAG_EXCLUSIVE & mfib_entry->mfe_flags))
         {
-            /*
-             * each path contirbutes a next-hop. form a replicate
-             * from those choices.
-             */
-            if (!dpo_id_is_valid(&mfib_entry->mfe_rep) ||
-                dpo_is_drop(&mfib_entry->mfe_rep))
+            if (NULL == ctx.next_hops)
             {
-                dpo_id_t tmp_dpo = DPO_INVALID;
-
-                dpo_set(&tmp_dpo,
-                        DPO_REPLICATE, dp,
-                        replicate_create(0, dp));
-
+                /*
+                 * no next-hops, stack directly on the drop
+                 */
                 dpo_stack(DPO_MFIB_ENTRY, dp,
                           &mfib_entry->mfe_rep,
-                          &tmp_dpo);
-
-                dpo_reset(&tmp_dpo);
+                          drop_dpo_get(dp));
+            }
+            else
+            {
+                /*
+                 * each path contirbutes a next-hop. form a replicate
+                 * from those choices.
+                 */
+                if (!dpo_id_is_valid(&mfib_entry->mfe_rep) ||
+                    dpo_is_drop(&mfib_entry->mfe_rep))
+                {
+                    dpo_id_t tmp_dpo = DPO_INVALID;
+
+                    dpo_set(&tmp_dpo,
+                            DPO_REPLICATE, dp,
+                            replicate_create(0, dp));
+
+                    dpo_stack(DPO_MFIB_ENTRY, dp,
+                              &mfib_entry->mfe_rep,
+                              &tmp_dpo);
+
+                    dpo_reset(&tmp_dpo);
+                }
+                replicate_multipath_update(&mfib_entry->mfe_rep,
+                                           ctx.next_hops);
             }
-            replicate_multipath_update(&mfib_entry->mfe_rep,
-                                       ctx.next_hops);
         }
         else
         {
@@ -548,11 +669,11 @@ mfib_entry_stack (mfib_entry_t *mfib_entry)
     }
 }
 
-static void
-mfib_entry_forwarding_path_add (mfib_entry_src_t *msrc,
-                                const fib_route_path_t *rpath)
+static fib_node_index_t
+mfib_entry_src_path_add (mfib_entry_src_t *msrc,
+                         const fib_route_path_t *rpath)
 {
-    fib_node_index_t old_pl_index;
+    fib_node_index_t path_index;
     fib_route_path_t *rpaths;
 
     ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_flags));
@@ -563,32 +684,26 @@ mfib_entry_forwarding_path_add (mfib_entry_src_t *msrc,
     rpaths = NULL;
     vec_add1(rpaths, rpath[0]);
 
-    old_pl_index = msrc->mfes_pl;
-
     if (FIB_NODE_INDEX_INVALID == msrc->mfes_pl)
     {
-        msrc->mfes_pl =
-            fib_path_list_create(FIB_PATH_LIST_FLAG_NO_URPF,
-                                 rpaths);
-    }
-    else
-    {
-        msrc->mfes_pl =
-            fib_path_list_copy_and_path_add(msrc->mfes_pl,
-                                            FIB_PATH_LIST_FLAG_NO_URPF,
-                                            rpaths);
+        /* A non-shared path-list */
+        msrc->mfes_pl = fib_path_list_create(FIB_PATH_LIST_FLAG_NO_URPF,
+                                             NULL);
+        fib_path_list_lock(msrc->mfes_pl);
     }
-    fib_path_list_lock(msrc->mfes_pl);
-    fib_path_list_unlock(old_pl_index);
+
+    path_index = fib_path_list_path_add(msrc->mfes_pl, rpaths);
 
     vec_free(rpaths);
+
+    return (path_index);
 }
 
-static int
-mfib_entry_forwarding_path_remove (mfib_entry_src_t *msrc,
-                                   const fib_route_path_t *rpath)
+static fib_node_index_t
+mfib_entry_src_path_remove (mfib_entry_src_t *msrc,
+                            const fib_route_path_t *rpath)
 {
-    fib_node_index_t old_pl_index;
+    fib_node_index_t path_index;
     fib_route_path_t *rpaths;
 
     ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_flags));
@@ -599,56 +714,31 @@ mfib_entry_forwarding_path_remove (mfib_entry_src_t *msrc,
     rpaths = NULL;
     vec_add1(rpaths, rpath[0]);
 
-    old_pl_index = msrc->mfes_pl;
-
-    msrc->mfes_pl =
-        fib_path_list_copy_and_path_remove(msrc->mfes_pl,
-                                           FIB_PATH_LIST_FLAG_NONE,
-                                           rpaths);
-
-    fib_path_list_lock(msrc->mfes_pl);
-    fib_path_list_unlock(old_pl_index);
+    path_index = fib_path_list_path_remove(msrc->mfes_pl, rpaths);
 
     vec_free(rpaths);
 
-    return (FIB_NODE_INDEX_INVALID != msrc->mfes_pl);
+    return (path_index);
 }
 
 static void
 mfib_entry_recalculate_forwarding (mfib_entry_t *mfib_entry)
 {
-    fib_node_index_t old_pl_index;
     mfib_entry_src_t *bsrc;
 
-    old_pl_index = mfib_entry->mfe_parent;
-
     /*
      * copy the forwarding data from the bast source
      */
     bsrc = mfib_entry_get_best_src(mfib_entry);
 
-    if (NULL == bsrc)
-    {
-        mfib_entry->mfe_parent = FIB_NODE_INDEX_INVALID;
-    }
-    else
+    if (NULL != bsrc)
     {
-        mfib_entry->mfe_parent = bsrc->mfes_pl;
         mfib_entry->mfe_flags = bsrc->mfes_flags;
         mfib_entry->mfe_itfs = bsrc->mfes_itfs;
+        mfib_entry->mfe_rpf_id = bsrc->mfes_rpf_id;
     }
 
-    /*
-     * re-stack the entry on the best forwarding info.
-     */
-    if (old_pl_index != mfib_entry->mfe_parent ||
-        FIB_NODE_INDEX_INVALID == old_pl_index)
-    {
-        mfib_entry_stack(mfib_entry);
-
-        fib_path_list_lock(mfib_entry->mfe_parent);
-        fib_path_list_unlock(old_pl_index);
-    }
+    mfib_entry_stack(mfib_entry, bsrc);
 }
 
 
@@ -656,6 +746,7 @@ fib_node_index_t
 mfib_entry_create (u32 fib_index,
                    mfib_source_t source,
                    const mfib_prefix_t *prefix,
+                   fib_rpf_id_t rpf_id,
                    mfib_entry_flags_t entry_flags)
 {
     fib_node_index_t mfib_entry_index;
@@ -666,6 +757,7 @@ mfib_entry_create (u32 fib_index,
                                   &mfib_entry_index);
     msrc = mfib_entry_src_find_or_create(mfib_entry, source);
     msrc->mfes_flags = entry_flags;
+    msrc->mfes_rpf_id = rpf_id;
 
     mfib_entry_recalculate_forwarding(mfib_entry);
 
@@ -682,13 +774,14 @@ static int
 mfib_entry_src_ok_for_delete (const mfib_entry_src_t *msrc)
 {
     return ((MFIB_ENTRY_FLAG_NONE == msrc->mfes_flags &&
-             0 == mfib_entry_src_n_itfs(msrc)));
+             0 == fib_path_list_get_n_paths(msrc->mfes_pl)));
 }
 
 int
 mfib_entry_update (fib_node_index_t mfib_entry_index,
                    mfib_source_t source,
                    mfib_entry_flags_t entry_flags,
+                   fib_rpf_id_t rpf_id,
                    index_t repi)
 {
     mfib_entry_t *mfib_entry;
@@ -697,6 +790,7 @@ mfib_entry_update (fib_node_index_t mfib_entry_index,
     mfib_entry = mfib_entry_get(mfib_entry_index);
     msrc = mfib_entry_src_find_or_create(mfib_entry, source);
     msrc->mfes_flags = entry_flags;
+    msrc->mfes_rpf_id = rpf_id;
 
     if (INDEX_INVALID != repi)
     {
@@ -768,55 +862,79 @@ mfib_entry_path_update (fib_node_index_t mfib_entry_index,
                         const fib_route_path_t *rpath,
                         mfib_itf_flags_t itf_flags)
 {
+    fib_node_index_t path_index;
+    mfib_path_ext_t *path_ext;
+    mfib_itf_flags_t old, new;
     mfib_entry_t *mfib_entry;
     mfib_entry_src_t *msrc;
-    mfib_itf_t *mfib_itf;
 
     mfib_entry = mfib_entry_get(mfib_entry_index);
     ASSERT(NULL != mfib_entry);
     msrc = mfib_entry_src_find_or_create(mfib_entry, source);
 
     /*
-     * search for the interface in the current set
+     * add the path to the path-list. If it's a duplicate we'll get
+     * back the original path.
+     */
+    path_index = mfib_entry_src_path_add(msrc, rpath);
+
+    /*
+     * find the path extension for that path
      */
-    mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs,
-                                   rpath[0].frp_sw_if_index);
+    path_ext = mfib_entry_path_ext_find(msrc->mfes_exts, path_index);
 
-    if (NULL == mfib_itf)
+    if (NULL == path_ext)
     {
-        /*
-         * this is a path we do not yet have. If it is forwarding then we
-         * add it to the replication set
-         */
-        if (itf_flags & MFIB_ITF_FLAG_FORWARD)
-        {
-            mfib_entry_forwarding_path_add(msrc, rpath);
-        }
-        /*
-         * construct a new ITF for this entry's list
-         */
-        mfib_entry_itf_add(msrc,
-                           rpath[0].frp_sw_if_index,
-                           mfib_itf_create(rpath[0].frp_sw_if_index,
-                                           itf_flags));
+        old = MFIB_ITF_FLAG_NONE;
+        path_ext = mfib_path_ext_add(msrc, path_index, itf_flags);
     }
     else
     {
-        int was_forwarding = !!(mfib_itf->mfi_flags & MFIB_ITF_FLAG_FORWARD);
-        int is_forwarding  = !!(itf_flags & MFIB_ITF_FLAG_FORWARD);
+        old = path_ext->mfpe_flags;
+        path_ext->mfpe_flags = itf_flags;
+    }
 
-        if (!was_forwarding && is_forwarding)
-        {
-            mfib_entry_forwarding_path_add(msrc, rpath);
-        }
-        else if (was_forwarding && !is_forwarding)
+    /*
+     * Has the path changed its contribution to the input interface set.
+     * Which only paths with interfaces can do...
+     */
+    if (~0 != rpath[0].frp_sw_if_index)
+    {
+        mfib_itf_t *mfib_itf;
+
+        new = itf_flags;
+
+        if (old != new)
         {
-            mfib_entry_forwarding_path_remove(msrc, rpath);
+            if (MFIB_ITF_FLAG_NONE == new)
+            {
+                /*
+                 * no more interface flags on this path, remove
+                 * from the data-plane set
+                 */
+                mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index);
+            }
+            else if (MFIB_ITF_FLAG_NONE == old)
+            {
+                /*
+                 * This interface is now contributing
+                 */
+                mfib_entry_itf_add(msrc,
+                                   rpath[0].frp_sw_if_index,
+                                   mfib_itf_create(rpath[0].frp_sw_if_index,
+                                                   itf_flags));
+            }
+            else
+            {
+                /*
+                 * change of flag contributions
+                 */
+                mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs,
+                                               rpath[0].frp_sw_if_index);
+                /* Seen by packets inflight */
+                mfib_itf->mfi_flags = new;
+            }
         }
-        /*
-         * packets in flight see these updates.
-         */
-        mfib_itf->mfi_flags = itf_flags;
     }
 
     mfib_entry_recalculate_forwarding(mfib_entry);
@@ -833,9 +951,9 @@ mfib_entry_path_remove (fib_node_index_t mfib_entry_index,
                         mfib_source_t source,
                         const fib_route_path_t *rpath)
 {
+    fib_node_index_t path_index;
     mfib_entry_t *mfib_entry;
     mfib_entry_src_t *msrc;
-    mfib_itf_t *mfib_itf;
 
     mfib_entry = mfib_entry_get(mfib_entry_index);
     ASSERT(NULL != mfib_entry);
@@ -850,33 +968,23 @@ mfib_entry_path_remove (fib_node_index_t mfib_entry_index,
     }
 
     /*
-     * search for the interface in the current set
+     * remove the path from the path-list. If it's not there we'll get
+     * back invalid
      */
-    mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs,
-                                   rpath[0].frp_sw_if_index);
+    path_index = mfib_entry_src_path_remove(msrc, rpath);
 
-    if (NULL == mfib_itf)
+    if (FIB_NODE_INDEX_INVALID != path_index)
     {
         /*
-         * removing a path that does not exist
+         * don't need the extension, nor the interface anymore
          */
-        return (mfib_entry_ok_for_delete(mfib_entry));
-    }
-
-    /*
-     * we have this path. If it is forwarding then we
-     * remove it to the replication set
-     */
-    if (mfib_itf->mfi_flags & MFIB_ITF_FLAG_FORWARD)
-    {
-        mfib_entry_forwarding_path_remove(msrc, rpath);
+        mfib_path_ext_remove(msrc, path_index);
+        if (~0 != rpath[0].frp_sw_if_index)
+        {
+            mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index);
+        }
     }
 
-    /*
-     * remove the interface/path from this entry's list
-     */
-    mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index);
-
     if (mfib_entry_src_ok_for_delete(msrc))
     {
         /*
@@ -1057,11 +1165,14 @@ mfib_entry_encode (fib_node_index_t mfib_entry_index,
                   fib_route_path_encode_t **api_rpaths)
 {
     mfib_entry_t *mfib_entry;
+    mfib_entry_src_t *bsrc;
 
     mfib_entry = mfib_entry_get(mfib_entry_index);
-    if (FIB_NODE_INDEX_INVALID != mfib_entry->mfe_parent)
+    bsrc = mfib_entry_get_best_src(mfib_entry);
+
+    if (FIB_NODE_INDEX_INVALID != bsrc->mfes_pl)
     {
-        fib_path_list_walk(mfib_entry->mfe_parent,
+        fib_path_list_walk(bsrc->mfes_pl,
                            fib_path_encode,
                            api_rpaths);
     }
diff --git a/src/vnet/mfib/mfib_entry.h b/src/vnet/mfib/mfib_entry.h
index dc8f49aa..4f62b18e 100644
--- a/src/vnet/mfib/mfib_entry.h
+++ b/src/vnet/mfib/mfib_entry.h
@@ -42,17 +42,6 @@ typedef struct mfib_entry_t_ {
      * The index of the FIB table this entry is in
      */
     u32 mfe_fib_index;
-    /**
-     * the path-list for which this entry is a child. This is also the path-list
-     * that is contributing forwarding for this entry.
-     */
-    fib_node_index_t mfe_parent;
-    /**
-     * index of this entry in the parent's child list.
-     * This is set when this entry is added as a child, but can also
-     * be changed by the parent as it manages its list.
-     */
-    u32 mfe_sibling;
 
     /**
      * A vector of sources contributing forwarding
@@ -65,7 +54,7 @@ typedef struct mfib_entry_t_ {
     CLIB_CACHE_LINE_ALIGN_MARK(cacheline1);
 
     /**
-     * The Replicate DPO used for forwarding.
+     * The DPO used for forwarding; replicate, drop, etc..
      */
     dpo_id_t mfe_rep;
 
@@ -74,6 +63,11 @@ typedef struct mfib_entry_t_ {
      */
     mfib_entry_flags_t mfe_flags;
 
+    /**
+     * RPF-ID used when the packets ingress not from an interface
+     */
+    fib_rpf_id_t mfe_rpf_id;
+
     /**
      * A hash table of interfaces
      */
@@ -90,11 +84,13 @@ extern u8 *format_mfib_entry(u8 * s, va_list * args);
 extern fib_node_index_t mfib_entry_create(u32 fib_index,
                                           mfib_source_t source,
                                           const mfib_prefix_t *prefix,
+                                          fib_rpf_id_t rpf_id,
                                           mfib_entry_flags_t entry_flags);
 
 extern int mfib_entry_update(fib_node_index_t fib_entry_index,
                              mfib_source_t source,
                              mfib_entry_flags_t entry_flags,
+                             fib_rpf_id_t rpf_id,
                              index_t rep_dpo);
 
 extern void mfib_entry_path_update(fib_node_index_t fib_entry_index,
diff --git a/src/vnet/mfib/mfib_forward.c b/src/vnet/mfib/mfib_forward.c
index 5fe0a57c..3d8f4f98 100644
--- a/src/vnet/mfib/mfib_forward.c
+++ b/src/vnet/mfib/mfib_forward.c
@@ -380,13 +380,27 @@ mfib_forward_rpf (vlib_main_t * vm,
              * for the case of throughput traffic that is not replicated
              * to the host stack nor sets local flags
              */
-            if (PREDICT_TRUE(NULL != mfi0))
+
+            /*
+             * If the mfib entry has a configured RPF-ID check that
+             * in preference to an interface based RPF
+             */
+            if (MFIB_RPF_ID_NONE != mfe0->mfe_rpf_id)
             {
-                iflags0 = mfi0->mfi_flags;
+                iflags0 = (mfe0->mfe_rpf_id == vnet_buffer(b0)->ip.rpf_id ?
+                           MFIB_ITF_FLAG_ACCEPT :
+                           MFIB_ITF_FLAG_NONE);
             }
             else
             {
-                iflags0 = MFIB_ITF_FLAG_NONE;
+                if (PREDICT_TRUE(NULL != mfi0))
+                {
+                    iflags0 = mfi0->mfi_flags;
+                }
+                else
+                {
+                    iflags0 = MFIB_ITF_FLAG_NONE;
+                }
             }
             eflags0 = mfe0->mfe_flags;
 
@@ -436,17 +450,16 @@ mfib_forward_rpf (vlib_main_t * vm,
             {
                 mfib_forward_rpf_trace_t *t0;
 
-                t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
+                t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
                 t0->entry_index = mfei0;
+                t0->itf_flags = iflags0;
                 if (NULL == mfi0)
                 {
                     t0->sw_if_index = ~0;
-                    t0->itf_flags = MFIB_ITF_FLAG_NONE;
                 }
                 else
                 {
                     t0->sw_if_index = mfi0->mfi_sw_if_index;
-                    t0->itf_flags = mfi0->mfi_flags;
                 }
             }
             vlib_validate_buffer_enqueue_x1 (vm, node, next,
@@ -478,7 +491,7 @@ VLIB_REGISTER_NODE (ip4_mfib_forward_rpf_node, static) = {
 
     .n_next_nodes = MFIB_FORWARD_RPF_N_NEXT,
     .next_nodes = {
-        [MFIB_FORWARD_RPF_NEXT_DROP] = "error-drop",
+        [MFIB_FORWARD_RPF_NEXT_DROP] = "ip4-drop",
     },
 };
 
@@ -503,7 +516,7 @@ VLIB_REGISTER_NODE (ip6_mfib_forward_rpf_node, static) = {
 
     .n_next_nodes = MFIB_FORWARD_RPF_N_NEXT,
     .next_nodes = {
-        [MFIB_FORWARD_RPF_NEXT_DROP] = "error-drop",
+        [MFIB_FORWARD_RPF_NEXT_DROP] = "ip6-drop",
     },
 };
 
diff --git a/src/vnet/mfib/mfib_table.c b/src/vnet/mfib/mfib_table.c
index 3b4bd985..7ffe8941 100644
--- a/src/vnet/mfib/mfib_table.c
+++ b/src/vnet/mfib/mfib_table.c
@@ -165,6 +165,7 @@ fib_node_index_t
 mfib_table_entry_update (u32 fib_index,
                          const mfib_prefix_t *prefix,
                          mfib_source_t source,
+                         fib_rpf_id_t rpf_id,
                          mfib_entry_flags_t entry_flags)
 {
     fib_node_index_t mfib_entry_index;
@@ -181,7 +182,8 @@ mfib_table_entry_update (u32 fib_index,
              * update to a non-existing entry with non-zero flags
              */
             mfib_entry_index = mfib_entry_create(fib_index, source,
-                                                 prefix, entry_flags);
+                                                 prefix, rpf_id,
+                                                 entry_flags);
 
             mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index);
         }
@@ -198,6 +200,7 @@ mfib_table_entry_update (u32 fib_index,
         if (mfib_entry_update(mfib_entry_index,
                               source,
                               entry_flags,
+                              rpf_id,
                               INDEX_INVALID))
         {
             /*
@@ -230,6 +233,7 @@ mfib_table_entry_path_update (u32 fib_index,
         mfib_entry_index = mfib_entry_create(fib_index,
                                              source,
                                              prefix,
+                                             MFIB_RPF_ID_NONE,
                                              MFIB_ENTRY_FLAG_NONE);
 
         mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index);
@@ -304,6 +308,7 @@ mfib_table_entry_special_add (u32 fib_index,
         mfib_entry_index = mfib_entry_create(fib_index,
                                              source,
                                              prefix,
+                                             MFIB_RPF_ID_NONE,
                                              MFIB_ENTRY_FLAG_NONE);
 
         mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index);
@@ -311,6 +316,7 @@ mfib_table_entry_special_add (u32 fib_index,
 
     mfib_entry_update(mfib_entry_index, source,
                       (MFIB_ENTRY_FLAG_EXCLUSIVE | entry_flags),
+                      MFIB_RPF_ID_NONE,
                       rep_dpo);
 
     return (mfib_entry_index);
diff --git a/src/vnet/mfib/mfib_table.h b/src/vnet/mfib/mfib_table.h
index 95239f7c..83aa04ef 100644
--- a/src/vnet/mfib/mfib_table.h
+++ b/src/vnet/mfib/mfib_table.h
@@ -122,6 +122,7 @@ extern fib_node_index_t mfib_table_lookup_exact_match(u32 fib_index,
 extern fib_node_index_t mfib_table_entry_update(u32 fib_index,
                                                 const mfib_prefix_t *prefix,
                                                 mfib_source_t source,
+                                                fib_rpf_id_t rpf_id,
                                                 mfib_entry_flags_t flags);
 
 /**
diff --git a/src/vnet/mfib/mfib_test.c b/src/vnet/mfib/mfib_test.c
index 36a303e8..7c92ae99 100644
--- a/src/vnet/mfib/mfib_test.c
+++ b/src/vnet/mfib/mfib_test.c
@@ -20,6 +20,8 @@
 #include <vnet/mfib/mfib_signal.h>
 #include <vnet/mfib/ip6_mfib.h>
 #include <vnet/fib/fib_path_list.h>
+#include <vnet/fib/fib_test.h>
+#include <vnet/fib/fib_table.h>
 
 #include <vnet/dpo/replicate_dpo.h>
 #include <vnet/adj/adj_mcast.h>
@@ -201,8 +203,8 @@ mfib_test_validate_rep_v (const replicate_t *rep,
         if (DPO_RECEIVE != dt)
         {
             MFIB_TEST_REP((ai == dpo->dpoi_index),
-                          "bucket %d stacks on %U",
-                          bucket,
+                          "bucket %d [exp:%d] stacks on %U",
+                          bucket, ai,
                           format_dpo_id, dpo, 0);
         }
     }
@@ -734,6 +736,7 @@ mfib_test_i (fib_protocol_t PROTO,
     mfib_table_entry_update(fib_index,
                             pfx_s_g,
                             MFIB_SOURCE_API,
+                            MFIB_RPF_ID_NONE,
                             MFIB_ENTRY_FLAG_SIGNAL);
     MFIB_TEST(mfib_test_entry(mfei,
                               MFIB_ENTRY_FLAG_SIGNAL,
@@ -824,6 +827,7 @@ mfib_test_i (fib_protocol_t PROTO,
     mfib_table_entry_update(fib_index,
                             pfx_s_g,
                             MFIB_SOURCE_API,
+                            MFIB_RPF_ID_NONE,
                             (MFIB_ENTRY_FLAG_SIGNAL |
                              MFIB_ENTRY_FLAG_CONNECTED));
     MFIB_TEST(mfib_test_entry(mfei,
@@ -965,6 +969,7 @@ mfib_test_i (fib_protocol_t PROTO,
     mfib_table_entry_update(fib_index,
                             pfx_s_g,
                             MFIB_SOURCE_API,
+                            MFIB_RPF_ID_NONE,
                             MFIB_ENTRY_FLAG_NONE);
     mfei = mfib_table_lookup_exact_match(fib_index,
                                          pfx_s_g);
@@ -1073,6 +1078,117 @@ mfib_test_i (fib_protocol_t PROTO,
                             MFIB_SOURCE_SRv6);
     dpo_reset(&td);
 
+    /*
+     * A Multicast LSP. This a mLDP head-end
+     */
+    fib_node_index_t ai_mpls_10_10_10_1, lfei;
+    ip46_address_t nh_10_10_10_1 = {
+	.ip4 = {
+	    .as_u32 = clib_host_to_net_u32(0x0a0a0a01),
+	},
+    };
+    ai_mpls_10_10_10_1 = adj_nbr_add_or_lock(FIB_PROTOCOL_IP4,
+                                             VNET_LINK_MPLS,
+                                             &nh_10_10_10_1,
+                                             tm->hw[0]->sw_if_index);
+
+    fib_prefix_t pfx_3500 = {
+	.fp_len = 21,
+	.fp_proto = FIB_PROTOCOL_MPLS,
+	.fp_label = 3500,
+	.fp_eos = MPLS_EOS,
+	.fp_payload_proto = DPO_PROTO_IP4,
+    };
+    fib_test_rep_bucket_t mc_0 = {
+        .type = FT_REP_LABEL_O_ADJ,
+	.label_o_adj = {
+	    .adj = ai_mpls_10_10_10_1,
+	    .label = 3300,
+	    .eos = MPLS_EOS,
+	},
+    };
+    mpls_label_t *l3300 = NULL;
+    vec_add1(l3300, 3300);
+
+    /*
+     * MPLS enable an interface so we get the MPLS table created
+     */
+    mpls_sw_interface_enable_disable(&mpls_main,
+                                     tm->hw[0]->sw_if_index,
+                                     1);
+
+    lfei = fib_table_entry_update_one_path(0, // default MPLS Table
+                                           &pfx_3500,
+                                           FIB_SOURCE_API,
+                                           FIB_ENTRY_FLAG_MULTICAST,
+                                           FIB_PROTOCOL_IP4,
+                                           &nh_10_10_10_1,
+                                           tm->hw[0]->sw_if_index,
+                                           ~0, // invalid fib index
+                                           1,
+                                           l3300,
+                                           FIB_ROUTE_PATH_FLAG_NONE);
+    MFIB_TEST(fib_test_validate_entry(lfei,
+                                      FIB_FORW_CHAIN_TYPE_MPLS_EOS,
+                                      1,
+                                      &mc_0),
+              "3500 via replicate over 10.10.10.1");
+
+    /*
+     * An (S,G) that resolves via the mLDP head-end
+     */
+    fib_route_path_t path_via_mldp = {
+        .frp_proto = FIB_PROTOCOL_MPLS,
+        .frp_local_label = pfx_3500.fp_label,
+        .frp_eos = MPLS_EOS,
+        .frp_sw_if_index = 0xffffffff,
+        .frp_fib_index = 0,
+        .frp_weight = 1,
+        .frp_flags = FIB_ROUTE_PATH_FLAG_NONE,
+    };
+    dpo_id_t mldp_dpo = DPO_INVALID;
+
+    fib_entry_contribute_forwarding(lfei,
+                                    FIB_FORW_CHAIN_TYPE_MPLS_EOS,
+                                    &mldp_dpo);
+
+    mfei = mfib_table_entry_path_update(fib_index,
+                                        pfx_s_g,
+                                        MFIB_SOURCE_API,
+                                        &path_via_mldp,
+                                        MFIB_ITF_FLAG_FORWARD);
+
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              1,
+                              DPO_REPLICATE, mldp_dpo.dpoi_index),
+              "%U over-mLDP replicate OK",
+              format_mfib_prefix, pfx_s_g);
+
+    /*
+     * add a for-us path. this tests two types of non-attached paths on one entry
+     */
+    mfei = mfib_table_entry_path_update(fib_index,
+                                        pfx_s_g,
+                                        MFIB_SOURCE_API,
+                                        &path_for_us,
+                                        MFIB_ITF_FLAG_FORWARD);
+    MFIB_TEST(mfib_test_entry(mfei,
+                              MFIB_ENTRY_FLAG_NONE,
+                              2,
+                              DPO_REPLICATE, mldp_dpo.dpoi_index,
+                              DPO_RECEIVE, 0),
+              "%U mLDP+for-us replicate OK",
+              format_mfib_prefix, pfx_s_g);
+
+    mfib_table_entry_delete(fib_index,
+                            pfx_s_g,
+                            MFIB_SOURCE_API);
+    fib_table_entry_delete(0,
+                           &pfx_3500,
+                           FIB_SOURCE_API);
+    dpo_reset(&mldp_dpo);
+
     /*
      * Unlock the table - it's the last lock so should be gone thereafter
      */
@@ -1086,6 +1202,13 @@ mfib_test_i (fib_protocol_t PROTO,
     adj_unlock(ai_2);
     adj_unlock(ai_3);
 
+    /*
+     * MPLS disable the interface
+     */
+    mpls_sw_interface_enable_disable(&mpls_main,
+                                     tm->hw[0]->sw_if_index,
+                                     0);
+
     /*
      * test we've leaked no resources
      */
diff --git a/src/vnet/mpls/mpls.api b/src/vnet/mpls/mpls.api
index 2e3bfaf5..a1e1270a 100644
--- a/src/vnet/mpls/mpls.api
+++ b/src/vnet/mpls/mpls.api
@@ -55,6 +55,7 @@ define mpls_ip_bind_unbind_reply
     @param context - sender context, to match reply w/ request
     @param mt_is_add - Is this a route add or delete
     @param mt_sw_if_index - The SW interface index of the tunnel to delete
+    @param mt_is_multicast - Is the tunnel's underlying LSP multicast
     @param mt_next_hop_proto_is_ip4 - The next-hop is IPV4
     @param mt_next_hop_weight - The weight, for UCMP
     @param mt_next_hop[16] - the nextop address
@@ -70,6 +71,7 @@ define mpls_tunnel_add_del
   u32 mt_sw_if_index;
   u8 mt_is_add;
   u8 mt_l2_only;
+  u8 mt_is_multicast;
   u8 mt_next_hop_proto_is_ip4;
   u8 mt_next_hop_weight;
   u8 mt_next_hop[16];
@@ -102,30 +104,43 @@ define mpls_tunnel_dump
   i32 tunnel_index;
 };
 
-/** \brief mpls eth tunnel operational state response
-    @param tunnel_index - eth tunnel identifier
-    @param intfc_address - interface ipv4 addr
-    @param mask_width - interface ipv4 addr mask
-    @param hw_if_index - interface id
-    @param l2_only -
-    @param tunnel_dst_mac -
-    @param tx_sw_if_index -
-    @param encap_index - reference to mpls label table
-    @param nlabels - number of resolved labels
-    @param labels - resolved labels
+/** \brief FIB path
+    @param sw_if_index - index of the interface
+    @param weight - The weight, for UCMP
+    @param is_local - local if non-zero, else remote
+    @param is_drop - Drop the packet
+    @param is_unreach - Drop the packet and rate limit send ICMP unreachable
+    @param is_prohibit - Drop the packet and rate limit send ICMP prohibited
+    @param afi - the afi of the next hop, IP46_TYPE_IP4=1, IP46_TYPE_IP6=2
+    @param next_hop[16] - the next hop address
+
+    WARNING: this type is replicated, pending cleanup completion
+
+*/
+typeonly manual_print manual_endian define fib_path2
+{
+  u32 sw_if_index;
+  u32 weight;
+  u8 is_local;
+  u8 is_drop;
+  u8 is_unreach;
+  u8 is_prohibit;
+  u8 afi;
+  u8 next_hop[16];
+  u32 labels[16];
+};
+
+/** \brief mpls tunnel details
 */
-define mpls_tunnel_details
+manual_endian manual_print define mpls_tunnel_details
 {
   u32 context;
-  u32 tunnel_index;
-  u8 mt_l2_only;
   u8 mt_sw_if_index;
-  u8 mt_next_hop_proto_is_ip4;
-  u8 mt_next_hop[16];
-  u32 mt_next_hop_sw_if_index;
-  u32 mt_next_hop_table_id;
-  u32 mt_next_hop_n_labels;
-  u32 mt_next_hop_out_labels[mt_next_hop_n_labels];
+  u8 mt_tunnel_index;
+  u8 mt_l2_only;
+  u8 mt_is_multicast;
+  u32 mt_count;
+  vl_api_fib_path2_t mt_paths[mt_count];
 };
 
 /** \brief MPLS Route Add / del route
@@ -140,10 +155,14 @@ define mpls_tunnel_details
                                         create them
     @param mr_is_add - Is this a route add or delete
     @param mr_is_classify - Is this route result a classify
+    @param mr_is_multicast - Is this a multicast route
     @param mr_is_multipath - Is this route update a multipath - i.e. is this
                              a path addition to an existing route
     @param mr_is_resolve_host - Recurse resolution constraint via a host prefix
     @param mr_is_resolve_attached - Recurse resolution constraint via attached prefix
+    @param mr_is_interface_rx - Interface Receive path
+    @param mr_is_interface_rx - RPF-ID Receive path. The next-hop interface
+                                is used as the RPF-ID
     @param mr_next_hop_proto_is_ip4 - The next-hop is IPV4
     @param mr_next_hop_weight - The weight, for UCMP
     @param mr_next_hop[16] - the nextop address
@@ -164,9 +183,12 @@ define mpls_route_add_del
   u8 mr_create_table_if_needed;
   u8 mr_is_add;
   u8 mr_is_classify;
+  u8 mr_is_multicast;
   u8 mr_is_multipath;
   u8 mr_is_resolve_host;
   u8 mr_is_resolve_attached;
+  u8 mr_is_interface_rx;
+  u8 mr_is_rpf_id;
   u8 mr_next_hop_proto_is_ip4;
   u8 mr_next_hop_weight;
   u8 mr_next_hop[16];
@@ -187,31 +209,6 @@ define mpls_route_add_del_reply
   i32 retval;
 };
 
-/** \brief FIB path
-    @param sw_if_index - index of the interface
-    @param weight - The weight, for UCMP
-    @param is_local - local if non-zero, else remote
-    @param is_drop - Drop the packet
-    @param is_unreach - Drop the packet and rate limit send ICMP unreachable
-    @param is_prohibit - Drop the packet and rate limit send ICMP prohibited
-    @param afi - the afi of the next hop, IP46_TYPE_IP4=1, IP46_TYPE_IP6=2
-    @param next_hop[16] - the next hop address
-
-    WARNING: this type is replicated, pending cleanup completion
-
-*/
-typeonly manual_print manual_endian define fib_path2
-{
-  u32 sw_if_index;
-  u32 weight;
-  u8 is_local;
-  u8 is_drop;
-  u8 is_unreach;
-  u8 is_prohibit;
-  u8 afi;
-  u8 next_hop[16];
-};
-
 /** \brief Dump MPLS fib table
     @param client_index - opaque cookie to identify the sender
 */
diff --git a/src/vnet/mpls/mpls.c b/src/vnet/mpls/mpls.c
index 482577b1..451b15cf 100644
--- a/src/vnet/mpls/mpls.c
+++ b/src/vnet/mpls/mpls.c
@@ -286,7 +286,15 @@ vnet_mpls_local_label (vlib_main_t * vm,
 	  rpath.frp_proto = FIB_PROTOCOL_IP4;
 	  vec_add1(rpaths, rpath);
       }
-			 
+      else if (unformat (line_input, "rx-ip4 %U",
+			 unformat_vnet_sw_interface, vnm,
+			 &rpath.frp_sw_if_index))
+      {
+	  rpath.frp_weight = 1;
+	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+          rpath.frp_flags = FIB_ROUTE_PATH_INTF_RX;
+	  vec_add1(rpaths, rpath);
+      }
       else if (unformat (line_input, "via %U %U",
 			 unformat_ip6_address,
  			 &rpath.frp_addr.ip6,
@@ -512,10 +520,3 @@ mpls_init (vlib_main_t * vm)
 }
 
 VLIB_INIT_FUNCTION (mpls_init);
-
-mpls_main_t * mpls_get_main (vlib_main_t * vm)
-{
-  vlib_call_init_function (vm, mpls_init);
-  return &mpls_main;
-}
-
diff --git a/src/vnet/mpls/mpls_api.c b/src/vnet/mpls/mpls_api.c
index f1aef6c9..6bfc491d 100644
--- a/src/vnet/mpls/mpls_api.c
+++ b/src/vnet/mpls/mpls_api.c
@@ -27,6 +27,7 @@
 #include <vnet/fib/fib_table.h>
 #include <vnet/fib/fib_api.h>
 #include <vnet/fib/mpls_fib.h>
+#include <vnet/fib/fib_path_list.h>
 
 #include <vnet/vnet_msg_enum.h>
 
@@ -163,6 +164,7 @@ mpls_route_add_del_t_handler (vnet_main_t * vnm,
 			    dpo_proto_to_fib (pfx.fp_payload_proto),
 			    mp->mr_next_hop_table_id,
 			    mp->mr_create_table_if_needed,
+			    mp->mr_is_rpf_id,
 			    &fib_index, &next_hop_fib_index);
 
   if (0 != rv)
@@ -192,10 +194,13 @@ mpls_route_add_del_t_handler (vnet_main_t * vnm,
 				   0,	// mp->is_unreach,
 				   0,	// mp->is_prohibit,
 				   0,	// mp->is_local,
+				   mp->mr_is_multicast,
 				   mp->mr_is_classify,
 				   mp->mr_classify_table_index,
 				   mp->mr_is_resolve_host,
 				   mp->mr_is_resolve_attached,
+				   mp->mr_is_interface_rx,
+				   mp->mr_is_rpf_id,
 				   fib_index, &pfx,
 				   mp->mr_next_hop_proto_is_ip4,
 				   &nh, ntohl (mp->mr_next_hop_sw_if_index),
@@ -229,46 +234,54 @@ vl_api_mpls_tunnel_add_del_t_handler (vl_api_mpls_tunnel_add_del_t * mp)
   int rv = 0;
   u32 tunnel_sw_if_index;
   int ii;
+  fib_route_path_t rpath, *rpaths = NULL;
+
+  memset (&rpath, 0, sizeof (rpath));
 
   stats_dslock_with_hint (1 /* release hint */ , 5 /* tag */ );
 
-  if (mp->mt_is_add)
+  if (mp->mt_next_hop_proto_is_ip4)
     {
-      fib_route_path_t rpath, *rpaths = NULL;
-      mpls_label_t *label_stack = NULL;
-
-      memset (&rpath, 0, sizeof (rpath));
-
-      if (mp->mt_next_hop_proto_is_ip4)
-	{
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
-	  clib_memcpy (&rpath.frp_addr.ip4,
-		       mp->mt_next_hop, sizeof (rpath.frp_addr.ip4));
-	}
-      else
-	{
-	  rpath.frp_proto = FIB_PROTOCOL_IP6;
-	  clib_memcpy (&rpath.frp_addr.ip6,
-		       mp->mt_next_hop, sizeof (rpath.frp_addr.ip6));
-	}
-      rpath.frp_sw_if_index = ntohl (mp->mt_next_hop_sw_if_index);
+      rpath.frp_proto = FIB_PROTOCOL_IP4;
+      clib_memcpy (&rpath.frp_addr.ip4,
+		   mp->mt_next_hop, sizeof (rpath.frp_addr.ip4));
+    }
+  else
+    {
+      rpath.frp_proto = FIB_PROTOCOL_IP6;
+      clib_memcpy (&rpath.frp_addr.ip6,
+		   mp->mt_next_hop, sizeof (rpath.frp_addr.ip6));
+    }
+  rpath.frp_sw_if_index = ntohl (mp->mt_next_hop_sw_if_index);
+  rpath.frp_weight = 1;
 
+  if (mp->mt_is_add)
+    {
       for (ii = 0; ii < mp->mt_next_hop_n_out_labels; ii++)
-	vec_add1 (label_stack, ntohl (mp->mt_next_hop_out_label_stack[ii]));
+	vec_add1 (rpath.frp_label_stack,
+		  ntohl (mp->mt_next_hop_out_label_stack[ii]));
+    }
 
-      vec_add1 (rpaths, rpath);
+  vec_add1 (rpaths, rpath);
 
-      vnet_mpls_tunnel_add (rpaths, label_stack,
-			    mp->mt_l2_only, &tunnel_sw_if_index);
-      vec_free (rpaths);
-      vec_free (label_stack);
+  tunnel_sw_if_index = ntohl (mp->mt_sw_if_index);
+
+  if (mp->mt_is_add)
+    {
+      if (~0 == tunnel_sw_if_index)
+	tunnel_sw_if_index = vnet_mpls_tunnel_create (mp->mt_l2_only,
+						      mp->mt_is_multicast);
+      vnet_mpls_tunnel_path_add (tunnel_sw_if_index, rpaths);
     }
   else
     {
       tunnel_sw_if_index = ntohl (mp->mt_sw_if_index);
-      vnet_mpls_tunnel_del (tunnel_sw_if_index);
+      if (!vnet_mpls_tunnel_path_remove (tunnel_sw_if_index, rpaths))
+	vnet_mpls_tunnel_del (tunnel_sw_if_index);
     }
 
+  vec_free (rpaths);
+
   stats_dsunlock ();
 
   /* *INDENT-OFF* */
@@ -289,10 +302,12 @@ typedef struct mpls_tunnel_send_walk_ctx_t_
 static void
 send_mpls_tunnel_entry (u32 mti, void *arg)
 {
+  fib_route_path_encode_t *api_rpaths, *api_rpath;
   mpls_tunnel_send_walk_ctx_t *ctx;
   vl_api_mpls_tunnel_details_t *mp;
   const mpls_tunnel_t *mt;
-  u32 nlabels;
+  vl_api_fib_path2_t *fp;
+  u32 n;
 
   ctx = arg;
 
@@ -300,18 +315,34 @@ send_mpls_tunnel_entry (u32 mti, void *arg)
     return;
 
   mt = mpls_tunnel_get (mti);
-  nlabels = vec_len (mt->mt_label_stack);
+  n = fib_path_list_get_n_paths (mt->mt_path_list);
+
+  mp = vl_msg_api_alloc (sizeof (*mp) + n * sizeof (vl_api_fib_path2_t));
+  memset (mp, 0, sizeof (*mp) + n * sizeof (vl_api_fib_path2_t));
 
-  mp = vl_msg_api_alloc (sizeof (*mp) + nlabels * sizeof (u32));
-  memset (mp, 0, sizeof (*mp));
   mp->_vl_msg_id = ntohs (VL_API_MPLS_TUNNEL_DETAILS);
   mp->context = ctx->context;
 
-  mp->tunnel_index = ntohl (mti);
-  memcpy (mp->mt_next_hop_out_labels,
-	  mt->mt_label_stack, nlabels * sizeof (u32));
+  mp->mt_tunnel_index = ntohl (mti);
+  mp->mt_count = ntohl (n);
+
+  fib_path_list_walk (mt->mt_path_list, fib_path_encode, &api_rpaths);
+
+  fp = mp->mt_paths;
+  vec_foreach (api_rpath, api_rpaths)
+  {
+    memset (fp, 0, sizeof (*fp));
+
+    fp->weight = htonl (api_rpath->rpath.frp_weight);
+    fp->sw_if_index = htonl (api_rpath->rpath.frp_sw_if_index);
+    copy_fib_next_hop (api_rpath, fp);
+    fp++;
+  }
 
   // FIXME
+  // memcpy (mp->mt_next_hop_out_labels,
+  //   mt->mt_label_stack, nlabels * sizeof (u32));
+
 
   vl_msg_api_send_shmem (ctx->q, (u8 *) & mp);
 }
diff --git a/src/vnet/mpls/mpls_input.c b/src/vnet/mpls/mpls_input.c
index 1b9bdd05..86ad8bba 100644
--- a/src/vnet/mpls/mpls_input.c
+++ b/src/vnet/mpls/mpls_input.c
@@ -291,7 +291,7 @@ mpls_setup_nodes (vlib_main_t * vm)
   rt->last_outer_fib_index = 0;
   rt->mpls_main = &mpls_main;
 
-  ethernet_register_input_type (vm, ETHERNET_TYPE_MPLS_UNICAST,
+  ethernet_register_input_type (vm, ETHERNET_TYPE_MPLS,
                                 mpls_input_node.index);
 }
 
diff --git a/src/vnet/mpls/mpls_lookup.c b/src/vnet/mpls/mpls_lookup.c
index ace6a70f..3c6be7e8 100644
--- a/src/vnet/mpls/mpls_lookup.c
+++ b/src/vnet/mpls/mpls_lookup.c
@@ -20,8 +20,17 @@
 #include <vnet/mpls/mpls.h>
 #include <vnet/fib/mpls_fib.h>
 #include <vnet/dpo/load_balance.h>
+#include <vnet/dpo/replicate_dpo.h>
 
-vlib_node_registration_t mpls_lookup_node;
+/**
+ * Static MPLS VLIB forwarding node
+ */
+static vlib_node_registration_t mpls_lookup_node;
+
+/**
+ * The arc/edge from the MPLS lookup node to the MPLS replicate node
+ */
+static u32 mpls_lookup_to_replicate_edge;
 
 typedef struct {
   u32 next_index;
@@ -156,81 +165,123 @@ mpls_lookup (vlib_main_t * vm,
           lbi2 = mpls_fib_table_forwarding_lookup (lfib_index2, h2);
           lbi3 = mpls_fib_table_forwarding_lookup (lfib_index3, h3);
 
-          lb0 = load_balance_get(lbi0);
-          lb1 = load_balance_get(lbi1);
-          lb2 = load_balance_get(lbi2);
-          lb3 = load_balance_get(lbi3);
-
           hash_c0 = vnet_buffer(b0)->ip.flow_hash = 0;
           hash_c1 = vnet_buffer(b1)->ip.flow_hash = 0;
           hash_c2 = vnet_buffer(b2)->ip.flow_hash = 0;
           hash_c3 = vnet_buffer(b3)->ip.flow_hash = 0;
 
-          if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
+          if (MPLS_IS_REPLICATE & lbi0)
           {
-              hash_c0 = vnet_buffer (b0)->ip.flow_hash =
-                  mpls_compute_flow_hash(h0, lb0->lb_hash_config);
+              next0 = mpls_lookup_to_replicate_edge;
+              vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
+                  (lbi0 & ~MPLS_IS_REPLICATE);
           }
-          if (PREDICT_FALSE(lb1->lb_n_buckets > 1))
+          else
           {
-              hash_c1 = vnet_buffer (b1)->ip.flow_hash =
-                  mpls_compute_flow_hash(h1, lb1->lb_hash_config);
+              lb0 = load_balance_get(lbi0);
+
+              if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
+              {
+                  hash_c0 = vnet_buffer (b0)->ip.flow_hash =
+                      mpls_compute_flow_hash(h0, lb0->lb_hash_config);
+              }
+              ASSERT (lb0->lb_n_buckets > 0);
+              ASSERT (is_pow2 (lb0->lb_n_buckets));
+              dpo0 = load_balance_get_bucket_i(lb0,
+                                               (hash_c0 &
+                                                (lb0->lb_n_buckets_minus_1)));
+              next0 = dpo0->dpoi_next_node;
+
+              vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
+
+              vlib_increment_combined_counter
+                  (cm, thread_index, lbi0, 1,
+                   vlib_buffer_length_in_chain (vm, b0));
           }
-          if (PREDICT_FALSE(lb2->lb_n_buckets > 1))
+          if (MPLS_IS_REPLICATE & lbi1)
           {
-              hash_c2 = vnet_buffer (b2)->ip.flow_hash =
-                  mpls_compute_flow_hash(h2, lb2->lb_hash_config);
+              next1 = mpls_lookup_to_replicate_edge;
+              vnet_buffer (b1)->ip.adj_index[VLIB_TX] =
+                  (lbi1 & ~MPLS_IS_REPLICATE);
           }
-          if (PREDICT_FALSE(lb3->lb_n_buckets > 1))
+          else
           {
-              hash_c3 = vnet_buffer (b3)->ip.flow_hash =
-                  mpls_compute_flow_hash(h3, lb3->lb_hash_config);
-          }
-
-          ASSERT (lb0->lb_n_buckets > 0);
-          ASSERT (is_pow2 (lb0->lb_n_buckets));
-          ASSERT (lb1->lb_n_buckets > 0);
-          ASSERT (is_pow2 (lb1->lb_n_buckets));
-          ASSERT (lb2->lb_n_buckets > 0);
-          ASSERT (is_pow2 (lb2->lb_n_buckets));
-          ASSERT (lb3->lb_n_buckets > 0);
-          ASSERT (is_pow2 (lb3->lb_n_buckets));
-
-          dpo0 = load_balance_get_bucket_i(lb0,
-                                           (hash_c0 &
-                                            (lb0->lb_n_buckets_minus_1)));
-          dpo1 = load_balance_get_bucket_i(lb1,
-                                           (hash_c1 &
-                                            (lb1->lb_n_buckets_minus_1)));
-          dpo2 = load_balance_get_bucket_i(lb2,
-                                           (hash_c2 &
-                                            (lb2->lb_n_buckets_minus_1)));
-          dpo3 = load_balance_get_bucket_i(lb3,
-                                           (hash_c3 &
-                                            (lb3->lb_n_buckets_minus_1)));
+              lb1 = load_balance_get(lbi1);
 
-          next0 = dpo0->dpoi_next_node;
-          next1 = dpo1->dpoi_next_node;
-          next2 = dpo2->dpoi_next_node;
-          next3 = dpo3->dpoi_next_node;
+              if (PREDICT_FALSE(lb1->lb_n_buckets > 1))
+              {
+                  hash_c1 = vnet_buffer (b1)->ip.flow_hash =
+                      mpls_compute_flow_hash(h1, lb1->lb_hash_config);
+              }
+              ASSERT (lb1->lb_n_buckets > 0);
+              ASSERT (is_pow2 (lb1->lb_n_buckets));
+              dpo1 = load_balance_get_bucket_i(lb1,
+                                               (hash_c1 &
+                                                (lb1->lb_n_buckets_minus_1)));
+              next1 = dpo1->dpoi_next_node;
+
+              vnet_buffer (b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
+
+              vlib_increment_combined_counter
+                  (cm, thread_index, lbi1, 1,
+                   vlib_buffer_length_in_chain (vm, b1));
+          }
+          if (MPLS_IS_REPLICATE & lbi2)
+          {
+              next2 = mpls_lookup_to_replicate_edge;
+              vnet_buffer (b2)->ip.adj_index[VLIB_TX] =
+                  (lbi2 & ~MPLS_IS_REPLICATE);
+          }
+          else
+          {
+              lb2 = load_balance_get(lbi2);
 
-          vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
-          vnet_buffer (b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
-          vnet_buffer (b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
-          vnet_buffer (b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
+              if (PREDICT_FALSE(lb2->lb_n_buckets > 1))
+              {
+                  hash_c2 = vnet_buffer (b2)->ip.flow_hash =
+                      mpls_compute_flow_hash(h2, lb2->lb_hash_config);
+              }
+              ASSERT (lb2->lb_n_buckets > 0);
+              ASSERT (is_pow2 (lb2->lb_n_buckets));
+              dpo2 = load_balance_get_bucket_i(lb2,
+                                               (hash_c2 &
+                                                (lb2->lb_n_buckets_minus_1)));
+              next2 = dpo2->dpoi_next_node;
+
+              vnet_buffer (b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
+
+              vlib_increment_combined_counter
+                  (cm, thread_index, lbi2, 1,
+                   vlib_buffer_length_in_chain (vm, b2));
+          }
+          if (MPLS_IS_REPLICATE & lbi3)
+          {
+              next3 = mpls_lookup_to_replicate_edge;
+              vnet_buffer (b3)->ip.adj_index[VLIB_TX] =
+                  (lbi3 & ~MPLS_IS_REPLICATE);
+          }
+          else
+          {
+              lb3 = load_balance_get(lbi3);
 
-          vlib_increment_combined_counter
-              (cm, thread_index, lbi0, 1,
-               vlib_buffer_length_in_chain (vm, b0));
-          vlib_increment_combined_counter
-              (cm, thread_index, lbi1, 1,
-               vlib_buffer_length_in_chain (vm, b1));
-          vlib_increment_combined_counter
-              (cm, thread_index, lbi2, 1,
-               vlib_buffer_length_in_chain (vm, b2));
-          vlib_increment_combined_counter
-              (cm, thread_index, lbi3, 1,
-               vlib_buffer_length_in_chain (vm, b3));
+              if (PREDICT_FALSE(lb3->lb_n_buckets > 1))
+              {
+                  hash_c3 = vnet_buffer (b3)->ip.flow_hash =
+                      mpls_compute_flow_hash(h3, lb3->lb_hash_config);
+              }
+              ASSERT (lb3->lb_n_buckets > 0);
+              ASSERT (is_pow2 (lb3->lb_n_buckets));
+              dpo3 = load_balance_get_bucket_i(lb3,
+                                               (hash_c3 &
+                                                (lb3->lb_n_buckets_minus_1)));
+              next3 = dpo3->dpoi_next_node;
+
+              vnet_buffer (b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
+
+              vlib_increment_combined_counter
+                  (cm, thread_index, lbi3, 1,
+                   vlib_buffer_length_in_chain (vm, b3));
+          }
 
           /*
            * before we pop the label copy th values we need to maintain.
@@ -331,31 +382,41 @@ mpls_lookup (vlib_main_t * vm,
                                 vnet_buffer(b0)->sw_if_index[VLIB_RX]);
 
           lbi0 = mpls_fib_table_forwarding_lookup(lfib_index0, h0);
-	  lb0 = load_balance_get(lbi0);
-
           hash_c0 = vnet_buffer(b0)->ip.flow_hash = 0;
-          if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
+
+          if (MPLS_IS_REPLICATE & lbi0)
           {
-              hash_c0 = vnet_buffer (b0)->ip.flow_hash =
-                  mpls_compute_flow_hash(h0, lb0->lb_hash_config);
+              next0 = mpls_lookup_to_replicate_edge;
+              vnet_buffer (b0)->ip.adj_index[VLIB_TX] =
+                  (lbi0 & ~MPLS_IS_REPLICATE);
           }
+          else
+          {
+              lb0 = load_balance_get(lbi0);
 
-          ASSERT (lb0->lb_n_buckets > 0);
-          ASSERT (is_pow2 (lb0->lb_n_buckets));
+              if (PREDICT_FALSE(lb0->lb_n_buckets > 1))
+              {
+                  hash_c0 = vnet_buffer (b0)->ip.flow_hash =
+                      mpls_compute_flow_hash(h0, lb0->lb_hash_config);
+              }
 
-          dpo0 = load_balance_get_bucket_i(lb0,
-                                           (hash_c0 &
-                                            (lb0->lb_n_buckets_minus_1)));
+              ASSERT (lb0->lb_n_buckets > 0);
+              ASSERT (is_pow2 (lb0->lb_n_buckets));
 
-          next0 = dpo0->dpoi_next_node;
-          vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
+              dpo0 = load_balance_get_bucket_i(lb0,
+                                               (hash_c0 &
+                                                (lb0->lb_n_buckets_minus_1)));
 
-          vlib_increment_combined_counter
-              (cm, thread_index, lbi0, 1,
-               vlib_buffer_length_in_chain (vm, b0));
+              next0 = dpo0->dpoi_next_node;
+              vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
+
+              vlib_increment_combined_counter
+                  (cm, thread_index, lbi0, 1,
+                   vlib_buffer_length_in_chain (vm, b0));
+          }
 
           /*
-           * before we pop the label copy th values we need to maintain.
+           * before we pop the label copy, values we need to maintain.
            * The label header is in network byte order.
            *  last byte is the TTL.
            *  bits 2 to 4 inclusive are the EXP bits
@@ -398,7 +459,7 @@ static char * mpls_error_strings[] = {
 #undef mpls_error
 };
 
-VLIB_REGISTER_NODE (mpls_lookup_node) = {
+VLIB_REGISTER_NODE (mpls_lookup_node, static) = {
   .function = mpls_lookup,
   .name = "mpls-lookup",
   /* Takes a vector of packets. */
@@ -621,3 +682,22 @@ VLIB_REGISTER_NODE (mpls_load_balance_node) = {
 };
 
 VLIB_NODE_FUNCTION_MULTIARCH (mpls_load_balance_node, mpls_load_balance)
+
+
+static clib_error_t *
+mpls_lookup_init (vlib_main_t * vm)
+{
+  clib_error_t * error;
+
+  if ((error = vlib_call_init_function (vm, mpls_init)))
+    return error;
+
+  mpls_lookup_to_replicate_edge =
+      vlib_node_add_named_next(vm,
+                               mpls_lookup_node.index,
+                               "mpls-replicate");
+
+  return (NULL);
+}
+
+VLIB_INIT_FUNCTION (mpls_lookup_init);
diff --git a/src/vnet/mpls/mpls_tunnel.c b/src/vnet/mpls/mpls_tunnel.c
index ac6fdcdf..1254dd9d 100644
--- a/src/vnet/mpls/mpls_tunnel.c
+++ b/src/vnet/mpls/mpls_tunnel.c
@@ -18,9 +18,12 @@
 #include <vnet/vnet.h>
 #include <vnet/pg/pg.h>
 #include <vnet/mpls/mpls_tunnel.h>
+#include <vnet/mpls/mpls_types.h>
 #include <vnet/ip/ip.h>
 #include <vnet/fib/fib_path_list.h>
 #include <vnet/adj/adj_midchain.h>
+#include <vnet/adj/adj_mcast.h>
+#include <vnet/dpo/replicate_dpo.h>
 
 /**
  * @brief pool of tunnel instances
@@ -37,6 +40,11 @@ static u32 * mpls_tunnel_free_hw_if_indices;
  */
 static u32 *mpls_tunnel_db;
 
+/**
+ * @brief MPLS tunnel flags strings
+ */
+static const char *mpls_tunnel_attribute_names[] = MPLS_TUNNEL_ATTRIBUTES;
+
 /**
  * @brief Get a tunnel object from a SW interface index
  */
@@ -44,103 +52,178 @@ static mpls_tunnel_t*
 mpls_tunnel_get_from_sw_if_index (u32 sw_if_index)
 {
     if ((vec_len(mpls_tunnel_db) < sw_if_index) ||
-	(~0 == mpls_tunnel_db[sw_if_index]))
-	return (NULL);
+        (~0 == mpls_tunnel_db[sw_if_index]))
+        return (NULL);
 
     return (pool_elt_at_index(mpls_tunnel_pool,
-			      mpls_tunnel_db[sw_if_index]));
+                              mpls_tunnel_db[sw_if_index]));
 }
 
 /**
- * @brief Return true if the label stack is imp-null only
+ * @brief Build a rewrite string for the MPLS tunnel.
  */
-static fib_forward_chain_type_t
-mpls_tunnel_get_fwd_chain_type (const mpls_tunnel_t *mt)
+static u8*
+mpls_tunnel_build_rewrite_i (void)
 {
-    if ((1 == vec_len(mt->mt_label_stack)) &&
-	(mt->mt_label_stack[0] == MPLS_IETF_IMPLICIT_NULL_LABEL))
-    {
-	/*
-	 * the only label in the label stack is implicit null
-	 * we need to build an IP chain.
-	 */
-	if (FIB_PROTOCOL_IP4 == fib_path_list_get_proto(mt->mt_path_list))
-	{
-	    return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4);
-	}
-	else
-	{
-	    return (FIB_FORW_CHAIN_TYPE_UNICAST_IP6);
-	}
-    }
-    else
-    {
-	return (FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS);
-    }
+    /*
+     * passing the adj code a NULL rewirte means 'i don't have one cos
+     * t'other end is unresolved'. That's not the case here. For the mpls
+     * tunnel there are just no bytes of encap to apply in the adj. We'll impose
+     * the label stack once we choose a path. So return a zero length rewrite.
+     */
+    u8 *rewrite = NULL;
+
+    vec_validate(rewrite, 0);
+    vec_reset_length(rewrite);
+
+    return (rewrite);
 }
 
 /**
  * @brief Build a rewrite string for the MPLS tunnel.
- *
- * We have choices here;
- *  1 - have an Adjacency with a zero length string and stack it on
- *       MPLS label objects
- *  2 - put the label header rewrites in the adjacency string.
- *
- * We choose 2 since it results in fewer graph nodes in the egress path
  */
 static u8*
 mpls_tunnel_build_rewrite (vnet_main_t * vnm,
-			   u32 sw_if_index,
-			   vnet_link_t link_type,
-			   const void *dst_address)
+                           u32 sw_if_index,
+                           vnet_link_t link_type,
+                           const void *dst_address)
 {
-    mpls_unicast_header_t *muh;
-    mpls_tunnel_t *mt;
-    u8 *rewrite;
-    u32 mti, ii;
+    return (mpls_tunnel_build_rewrite_i());
+}
 
-    rewrite = NULL;
-    mti = mpls_tunnel_db[sw_if_index];
-    mt = pool_elt_at_index(mpls_tunnel_pool, mti);
+typedef struct mpls_tunnel_collect_forwarding_ctx_t_
+{
+    load_balance_path_t * next_hops;
+    const mpls_tunnel_t *mt;
+    fib_forward_chain_type_t fct;
+} mpls_tunnel_collect_forwarding_ctx_t;
+
+static int
+mpls_tunnel_collect_forwarding (fib_node_index_t pl_index,
+                                fib_node_index_t path_index,
+                                void *arg)
+{
+    mpls_tunnel_collect_forwarding_ctx_t *ctx;
+    fib_path_ext_t *path_ext;
+    int have_path_ext;
+
+    ctx = arg;
 
     /*
-     * The vector must be allocated as u8 so the length is correct
+     * if the path is not resolved, don't include it.
      */
-    ASSERT(0 < vec_len(mt->mt_label_stack));
-    vec_validate(rewrite, (sizeof(*muh) * vec_len(mt->mt_label_stack)) - 1);
-    ASSERT(rewrite);
-    muh = (mpls_unicast_header_t *)rewrite;
+    if (!fib_path_is_resolved(path_index))
+    {
+        return (!0);
+    }
 
     /*
-     * The last (inner most) label in the stack may be EOS, all the rest Non-EOS
+     * get the matching path-extension for the path being visited.
      */
-    for (ii = 0; ii < vec_len(mt->mt_label_stack)-1; ii++)
+    have_path_ext = 0;
+    vec_foreach(path_ext, ctx->mt->mt_path_exts)
     {
-	vnet_mpls_uc_set_label(&muh[ii].label_exp_s_ttl, mt->mt_label_stack[ii]);
-	vnet_mpls_uc_set_ttl(&muh[ii].label_exp_s_ttl, 255);
-	vnet_mpls_uc_set_exp(&muh[ii].label_exp_s_ttl, 0);
-	vnet_mpls_uc_set_s(&muh[ii].label_exp_s_ttl, MPLS_NON_EOS);
-	muh[ii].label_exp_s_ttl = clib_host_to_net_u32(muh[ii].label_exp_s_ttl);
+        if (path_ext->fpe_path_index == path_index)
+        {
+            have_path_ext = 1;
+            break;
+        }
     }
 
-    vnet_mpls_uc_set_label(&muh[ii].label_exp_s_ttl, mt->mt_label_stack[ii]);
-    vnet_mpls_uc_set_ttl(&muh[ii].label_exp_s_ttl, 255);
-    vnet_mpls_uc_set_exp(&muh[ii].label_exp_s_ttl, 0);
-
-    if ((VNET_LINK_MPLS == link_type) &&
-	(mt->mt_label_stack[ii] != MPLS_IETF_IMPLICIT_NULL_LABEL))
+    if (have_path_ext)
     {
-	vnet_mpls_uc_set_s(&muh[ii].label_exp_s_ttl, MPLS_NON_EOS);
+        /*
+         * found a matching extension. stack it to obtain the forwarding
+         * info for this path.
+         */
+        ctx->next_hops = fib_path_ext_stack(path_ext,
+                                            ctx->fct,
+                                            ctx->fct,
+                                            ctx->next_hops);
     }
     else
+        ASSERT(0);
+    /*
+     * else
+     *   There should be a path-extenios associated with each path
+     */
+
+    return (!0);
+}
+
+static void
+mpls_tunnel_mk_lb (mpls_tunnel_t *mt,
+                   vnet_link_t linkt,
+                   fib_forward_chain_type_t fct,
+                   dpo_id_t *dpo_lb)
+{
+    dpo_proto_t lb_proto;
+
+    /*
+     * If the entry has path extensions then we construct a load-balance
+     * by stacking the extensions on the forwarding chains of the paths.
+     * Otherwise we use the load-balance of the path-list
+     */
+    mpls_tunnel_collect_forwarding_ctx_t ctx = {
+        .mt = mt,
+        .next_hops = NULL,
+        .fct = fct,
+    };
+
+    /*
+     * As an optimisation we allocate the vector of next-hops to be sized
+     * equal to the maximum nuber of paths we will need, which is also the
+     * most likely number we will need, since in most cases the paths are 'up'.
+     */
+    vec_validate(ctx.next_hops, fib_path_list_get_n_paths(mt->mt_path_list));
+    vec_reset_length(ctx.next_hops);
+
+    lb_proto = vnet_link_to_dpo_proto(linkt);
+
+    fib_path_list_walk(mt->mt_path_list,
+                       mpls_tunnel_collect_forwarding,
+                       &ctx);
+
+    if (!dpo_id_is_valid(dpo_lb))
     {
-	vnet_mpls_uc_set_s(&muh[ii].label_exp_s_ttl, MPLS_EOS);
+        /*
+         * first time create
+         */
+        if (mt->mt_flags & MPLS_TUNNEL_FLAG_MCAST)
+        {
+            dpo_set(dpo_lb,
+                    DPO_REPLICATE,
+                    lb_proto,
+                    replicate_create(0, lb_proto));
+        }
+        else
+        {
+            flow_hash_config_t fhc;
+
+            fhc = 0; // FIXME
+            /* fhc = fib_table_get_flow_hash_config(fib_entry->fe_fib_index, */
+            /*                                      dpo_proto_to_fib(lb_proto)); */
+            dpo_set(dpo_lb,
+                    DPO_LOAD_BALANCE,
+                    lb_proto,
+                    load_balance_create(0, lb_proto, fhc));
+        }
     }
 
-    muh[ii].label_exp_s_ttl = clib_host_to_net_u32(muh[ii].label_exp_s_ttl);
-
-    return (rewrite);
+    if (mt->mt_flags & MPLS_TUNNEL_FLAG_MCAST)
+    {
+        /*
+         * MPLS multicast
+         */
+        replicate_multipath_update(dpo_lb, ctx.next_hops);
+    }
+    else
+    {
+        load_balance_multipath_update(dpo_lb,
+                                      ctx.next_hops,
+                                      LOAD_BALANCE_FLAG_NONE);
+        vec_free(ctx.next_hops);
+    }
 }
 
 /**
@@ -161,45 +244,47 @@ mpls_tunnel_stack (adj_index_t ai)
     mt = mpls_tunnel_get_from_sw_if_index(sw_if_index);
 
     if (NULL == mt)
-	return;
+        return;
 
     /*
-     * find the adjacency that is contributed by the FIB path-list
-     * that this tunnel resovles via, and use it as the next adj
-     * in the midchain
+     * while we're stacking the adj, remove the tunnel from the child list
+     * of the path list. this breaks a circular dependency of walk updates
+     * where the create of adjacencies in the children can lead to walks
+     * that get back here.
      */
-    if (vnet_hw_interface_get_flags(vnet_get_main(),
-				    mt->mt_hw_if_index) &
-	VNET_HW_INTERFACE_FLAG_LINK_UP)
-    {
-	dpo_id_t dpo = DPO_INVALID;
+    fib_path_list_lock(mt->mt_path_list);
 
-	fib_path_list_contribute_forwarding(mt->mt_path_list,
-					    mpls_tunnel_get_fwd_chain_type(mt),
-					    &dpo);
-
-	if (DPO_LOAD_BALANCE == dpo.dpoi_type)
-	{
-	    /*
-	     * we don't support multiple paths, so no need to load-balance.
-	     * pull the first and only choice and stack directly on that.
-	     */
-	    load_balance_t *lb;
-
-	    lb = load_balance_get (dpo.dpoi_index);
+    fib_path_list_child_remove(mt->mt_path_list,
+                               mt->mt_sibling_index);
 
-	    ASSERT(1 == lb->lb_n_buckets);
+    /*
+     * Construct the DPO (load-balance or replicate) that we can stack
+     * the tunnel's midchain on
+     */
+    if (vnet_hw_interface_get_flags(vnet_get_main(),
+                                    mt->mt_hw_if_index) &
+        VNET_HW_INTERFACE_FLAG_LINK_UP)
+    {
+        dpo_id_t dpo = DPO_INVALID;
 
-	    dpo_copy(&dpo, load_balance_get_bucket_i (lb, 0));
-	}
+        mpls_tunnel_mk_lb(mt,
+                          adj->ia_link,
+                          FIB_FORW_CHAIN_TYPE_MPLS_EOS,
+                          &dpo);
 
-	adj_nbr_midchain_stack(ai, &dpo);
-	dpo_reset(&dpo);
+        adj_nbr_midchain_stack(ai, &dpo);
+        dpo_reset(&dpo);
     }
     else
     {
-	adj_nbr_midchain_unstack(ai);
+        adj_nbr_midchain_unstack(ai);
     }
+
+    mt->mt_sibling_index = fib_path_list_child_add(mt->mt_path_list,
+                                                   FIB_NODE_TYPE_MPLS_TUNNEL,
+                                                   mt - mpls_tunnel_pool);
+
+    fib_path_list_lock(mt->mt_path_list);
 }
 
 /**
@@ -207,7 +292,7 @@ mpls_tunnel_stack (adj_index_t ai)
  */
 static adj_walk_rc_t
 mpls_adj_walk_cb (adj_index_t ai,
-		 void *ctx)
+                 void *ctx)
 {
     mpls_tunnel_stack(ai);
 
@@ -224,17 +309,17 @@ mpls_tunnel_restack (mpls_tunnel_t *mt)
      */
     FOR_EACH_FIB_PROTOCOL(proto)
     {
-	adj_nbr_walk(mt->mt_sw_if_index,
-		     proto,
-		     mpls_adj_walk_cb,
-		     NULL);
+        adj_nbr_walk(mt->mt_sw_if_index,
+                     proto,
+                     mpls_adj_walk_cb,
+                     NULL);
     }
 }
 
 static clib_error_t *
 mpls_tunnel_admin_up_down (vnet_main_t * vnm,
-			   u32 hw_if_index,
-			   u32 flags)
+                           u32 hw_if_index,
+                           u32 flags)
 {
     vnet_hw_interface_t * hi;
     mpls_tunnel_t *mt;
@@ -244,13 +329,13 @@ mpls_tunnel_admin_up_down (vnet_main_t * vnm,
     mt = mpls_tunnel_get_from_sw_if_index(hi->sw_if_index);
 
     if (NULL == mt)
-	return (NULL);
+        return (NULL);
 
     if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
-	vnet_hw_interface_set_flags (vnm, hw_if_index,
-				     VNET_HW_INTERFACE_FLAG_LINK_UP);
+        vnet_hw_interface_set_flags (vnm, hw_if_index,
+                                     VNET_HW_INTERFACE_FLAG_LINK_UP);
     else
-	vnet_hw_interface_set_flags (vnm, hw_if_index, 0 /* down */);
+        vnet_hw_interface_set_flags (vnm, hw_if_index, 0 /* down */);
 
     mpls_tunnel_restack(mt);
 
@@ -263,22 +348,58 @@ mpls_tunnel_admin_up_down (vnet_main_t * vnm,
  */
 static void
 mpls_tunnel_fixup (vlib_main_t *vm,
-		   ip_adjacency_t *adj,
-		   vlib_buffer_t *b0)
+                   ip_adjacency_t *adj,
+                   vlib_buffer_t *b0)
 {
+    /*
+     * A no-op w.r.t. the header. but reset the 'have we pushed any
+     * MPLS labels onto the packet' flag. That way when we enter the
+     * tunnel we'll get a TTL set to 255
+     */
+    vnet_buffer(b0)->mpls.first = 0;
 }
 
 static void
 mpls_tunnel_update_adj (vnet_main_t * vnm,
-			u32 sw_if_index,
-			adj_index_t ai)
+                        u32 sw_if_index,
+                        adj_index_t ai)
 {
-    adj_nbr_midchain_update_rewrite(
-	ai, mpls_tunnel_fixup, 
-	ADJ_FLAG_NONE,
-	mpls_tunnel_build_rewrite(vnm, sw_if_index,
-				  adj_get_link_type(ai),
-				  NULL));
+    ip_adjacency_t *adj;
+
+    ASSERT(ADJ_INDEX_INVALID != ai);
+
+    adj = adj_get(ai);
+
+    switch (adj->lookup_next_index)
+    {
+    case IP_LOOKUP_NEXT_ARP:
+    case IP_LOOKUP_NEXT_GLEAN:
+        adj_nbr_midchain_update_rewrite(ai, mpls_tunnel_fixup,
+                                        ADJ_FLAG_NONE,
+                                        mpls_tunnel_build_rewrite_i());
+        break;
+    case IP_LOOKUP_NEXT_MCAST:
+        /*
+         * Construct a partial rewrite from the known ethernet mcast dest MAC
+         * There's no MAC fixup, so the last 2 parameters are 0
+         */
+        adj_mcast_midchain_update_rewrite(ai, mpls_tunnel_fixup,
+                                          ADJ_FLAG_NONE,
+                                          mpls_tunnel_build_rewrite_i(),
+                                          0, 0);
+        break;
+
+    case IP_LOOKUP_NEXT_DROP:
+    case IP_LOOKUP_NEXT_PUNT:
+    case IP_LOOKUP_NEXT_LOCAL:
+    case IP_LOOKUP_NEXT_REWRITE:
+    case IP_LOOKUP_NEXT_MIDCHAIN:
+    case IP_LOOKUP_NEXT_MCAST_MIDCHAIN:
+    case IP_LOOKUP_NEXT_ICMP_ERROR:
+    case IP_LOOKUP_N_NEXT:
+      ASSERT (0);
+      break;
+    }
 
     mpls_tunnel_stack(ai);
 }
@@ -312,7 +433,7 @@ typedef struct mpls_tunnel_trace_t_
 
 static u8 *
 format_mpls_tunnel_tx_trace (u8 * s,
-			     va_list * args)
+                             va_list * args)
 {
   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
@@ -327,8 +448,8 @@ format_mpls_tunnel_tx_trace (u8 * s,
  */
 static uword
 mpls_tunnel_tx (vlib_main_t * vm,
-		vlib_node_runtime_t * node,
-		vlib_frame_t * frame)
+                vlib_node_runtime_t * node,
+                vlib_frame_t * frame)
 {
   u32 next_index;
   u32 * from, * to_next, n_left_from, n_left_to_next;
@@ -355,32 +476,32 @@ mpls_tunnel_tx (vlib_main_t * vm,
        * FIXME DUAL LOOP
        */
       while (n_left_from > 0 && n_left_to_next > 0)
-	{
-	  vlib_buffer_t * b0;
-	  u32 bi0;
+        {
+          vlib_buffer_t * b0;
+          u32 bi0;
 
-	  bi0 = from[0];
-	  to_next[0] = bi0;
-	  from += 1;
-	  to_next += 1;
-	  n_left_from -= 1;
-	  n_left_to_next -= 1;
+          bi0 = from[0];
+          to_next[0] = bi0;
+          from += 1;
+          to_next += 1;
+          n_left_from -= 1;
+          n_left_to_next -= 1;
 
-	  b0 = vlib_get_buffer(vm, bi0);
+          b0 = vlib_get_buffer(vm, bi0);
 
-	  vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mt->mt_l2_adj;
+          vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mt->mt_l2_adj;
 
-	  if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
-	    {
-	      mpls_tunnel_trace_t *tr = vlib_add_trace (vm, node,
-						   b0, sizeof (*tr));
-	      tr->tunnel_id = rd->dev_instance;
-	    }
+          if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
+            {
+              mpls_tunnel_trace_t *tr = vlib_add_trace (vm, node,
+                                                   b0, sizeof (*tr));
+              tr->tunnel_id = rd->dev_instance;
+            }
 
-	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
-					   to_next, n_left_to_next,
-					   bi0, mt->mt_l2_tx_arc);
-	}
+          vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+                                           to_next, n_left_to_next,
+                                           bi0, mt->mt_l2_tx_arc);
+        }
 
       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
     }
@@ -417,13 +538,13 @@ mpls_tunnel_get (u32 mti)
  */
 void
 mpls_tunnel_walk (mpls_tunnel_walk_cb_t cb,
-		  void *ctx)
+                  void *ctx)
 {
     u32 mti;
 
     pool_foreach_index(mti, mpls_tunnel_pool,
     ({
-	cb(mti, ctx);
+        cb(mti, ctx);
     }));
 }
 
@@ -435,25 +556,22 @@ vnet_mpls_tunnel_del (u32 sw_if_index)
     mt = mpls_tunnel_get_from_sw_if_index(sw_if_index);
 
     if (NULL == mt)
-	return;
-    
-    fib_path_list_child_remove(mt->mt_path_list,
-			       mt->mt_sibling_index);
-    if (ADJ_INDEX_INVALID != mt->mt_l2_adj)
-	adj_unlock(mt->mt_l2_adj);
+        return;
 
-    vec_free(mt->mt_label_stack);
+    if (FIB_NODE_INDEX_INVALID != mt->mt_path_list)
+        fib_path_list_child_remove(mt->mt_path_list,
+                                   mt->mt_sibling_index);
+    if (ADJ_INDEX_INVALID != mt->mt_l2_adj)
+        adj_unlock(mt->mt_l2_adj);
 
     vec_add1 (mpls_tunnel_free_hw_if_indices, mt->mt_hw_if_index);
     pool_put(mpls_tunnel_pool, mt);
     mpls_tunnel_db[sw_if_index] = ~0;
 }
 
-void
-vnet_mpls_tunnel_add (fib_route_path_t *rpaths,
-		      mpls_label_t *label_stack,
-		      u8 l2_only,
-		      u32 *sw_if_index)
+u32
+vnet_mpls_tunnel_create (u8 l2_only,
+                         u8 is_multicast)
 {
     vnet_hw_interface_t * hi;
     mpls_tunnel_t *mt;
@@ -466,28 +584,33 @@ vnet_mpls_tunnel_add (fib_route_path_t *rpaths,
     mti = mt - mpls_tunnel_pool;
     fib_node_init(&mt->mt_node, FIB_NODE_TYPE_MPLS_TUNNEL);
     mt->mt_l2_adj = ADJ_INDEX_INVALID;
+    mt->mt_path_list = FIB_NODE_INDEX_INVALID;
+    mt->mt_sibling_index = FIB_NODE_INDEX_INVALID;
+
+    if (is_multicast)
+        mt->mt_flags |= MPLS_TUNNEL_FLAG_MCAST;
 
     /*
      * Create a new, or re=use and old, tunnel HW interface
      */
     if (vec_len (mpls_tunnel_free_hw_if_indices) > 0)
     {
-	mt->mt_hw_if_index = 
-	    mpls_tunnel_free_hw_if_indices[vec_len(mpls_tunnel_free_hw_if_indices)-1];
-	_vec_len (mpls_tunnel_free_hw_if_indices) -= 1;
-	hi = vnet_get_hw_interface (vnm, mt->mt_hw_if_index);
-	hi->hw_instance = mti;
-	hi->dev_instance = mti;
+        mt->mt_hw_if_index =
+            mpls_tunnel_free_hw_if_indices[vec_len(mpls_tunnel_free_hw_if_indices)-1];
+        _vec_len (mpls_tunnel_free_hw_if_indices) -= 1;
+        hi = vnet_get_hw_interface (vnm, mt->mt_hw_if_index);
+        hi->hw_instance = mti;
+        hi->dev_instance = mti;
     }
-    else 
+    else
     {
-	mt->mt_hw_if_index = vnet_register_interface(
-	                         vnm,
-				 mpls_tunnel_class.index,
-				 mti,
-				 mpls_tunnel_hw_interface_class.index,
-				 mti);
-	hi = vnet_get_hw_interface(vnm, mt->mt_hw_if_index);
+        mt->mt_hw_if_index = vnet_register_interface(
+                                 vnm,
+                                 mpls_tunnel_class.index,
+                                 mti,
+                                 mpls_tunnel_hw_interface_class.index,
+                                 mti);
+        hi = vnet_get_hw_interface(vnm, mt->mt_hw_if_index);
     }
 
     /*
@@ -497,43 +620,218 @@ vnet_mpls_tunnel_add (fib_route_path_t *rpaths,
     vec_validate_init_empty(mpls_tunnel_db, mt->mt_sw_if_index, ~0);
     mpls_tunnel_db[mt->mt_sw_if_index] = mti;
 
+    if (l2_only)
+    {
+        mt->mt_l2_adj =
+            adj_nbr_add_or_lock(fib_path_list_get_proto(mt->mt_path_list),
+                                VNET_LINK_ETHERNET,
+                                &zero_addr,
+                                mt->mt_sw_if_index);
+
+        mt->mt_l2_tx_arc = vlib_node_add_named_next(vlib_get_main(),
+                                                    hi->tx_node_index,
+                                                    "adj-l2-midchain");
+    }
+
+    return (mt->mt_sw_if_index);
+}
+
+/*
+ * mpls_tunnel_path_ext_add
+ *
+ * append a path extension to the entry's list
+ */
+static void
+mpls_tunnel_path_ext_append (mpls_tunnel_t *mt,
+                             const fib_route_path_t *rpath)
+{
+    if (NULL != rpath->frp_label_stack)
+    {
+        fib_path_ext_t *path_ext;
+
+        vec_add2(mt->mt_path_exts, path_ext, 1);
+
+        fib_path_ext_init(path_ext, mt->mt_path_list, rpath);
+    }
+}
+
+/*
+ * mpls_tunnel_path_ext_insert
+ *
+ * insert, sorted, a path extension to the entry's list.
+ * It's not strictly necessary in sort the path extensions, since each
+ * extension has the path index to which it resolves. However, by being
+ * sorted the load-balance produced has a deterministic order, not an order
+ * based on the sequence of extension additions. this is a considerable benefit.
+ */
+static void
+mpls_tunnel_path_ext_insert (mpls_tunnel_t *mt,
+                             const fib_route_path_t *rpath)
+{
+    if (0 == vec_len(mt->mt_path_exts))
+        return (mpls_tunnel_path_ext_append(mt, rpath));
+
+    if (NULL != rpath->frp_label_stack)
+    {
+        fib_path_ext_t path_ext;
+        int i = 0;
+
+        fib_path_ext_init(&path_ext, mt->mt_path_list, rpath);
+
+        while (i < vec_len(mt->mt_path_exts) &&
+               (fib_path_ext_cmp(&mt->mt_path_exts[i], rpath) < 0))
+        {
+            i++;
+        }
+
+        vec_insert_elts(mt->mt_path_exts, &path_ext, 1, i);
+    }
+}
+
+void
+vnet_mpls_tunnel_path_add (u32 sw_if_index,
+                           fib_route_path_t *rpaths)
+{
+    mpls_tunnel_t *mt;
+    u32 mti;
+
+    mt = mpls_tunnel_get_from_sw_if_index(sw_if_index);
+
+    if (NULL == mt)
+        return;
+
+    mti = mt - mpls_tunnel_pool;
+
     /*
      * construct a path-list from the path provided
      */
-    mt->mt_path_list = fib_path_list_create(FIB_PATH_LIST_FLAG_SHARED, rpaths);
-    mt->mt_sibling_index = fib_path_list_child_add(mt->mt_path_list,
-						   FIB_NODE_TYPE_MPLS_TUNNEL,
-						   mti);
+    if (FIB_NODE_INDEX_INVALID == mt->mt_path_list)
+    {
+        mt->mt_path_list = fib_path_list_create(FIB_PATH_LIST_FLAG_SHARED, rpaths);
+        mt->mt_sibling_index = fib_path_list_child_add(mt->mt_path_list,
+                                                       FIB_NODE_TYPE_MPLS_TUNNEL,
+                                                       mti);
+    }
+    else
+    {
+        fib_node_index_t old_pl_index;
+        fib_path_ext_t *path_ext;
+
+        old_pl_index = mt->mt_path_list;
+
+        mt->mt_path_list =
+            fib_path_list_copy_and_path_add(old_pl_index,
+                                            FIB_PATH_LIST_FLAG_SHARED,
+                                            rpaths);
+
+        fib_path_list_child_remove(old_pl_index,
+                                   mt->mt_sibling_index);
+        mt->mt_sibling_index = fib_path_list_child_add(mt->mt_path_list,
+                                                       FIB_NODE_TYPE_MPLS_TUNNEL,
+                                                       mti);
+        /*
+         * re-resolve all the path-extensions with the new path-list
+         */
+        vec_foreach(path_ext, mt->mt_path_exts)
+        {
+            fib_path_ext_resolve(path_ext, mt->mt_path_list);
+        }
+    }
+    mpls_tunnel_path_ext_insert(mt, rpaths);
+    mpls_tunnel_restack(mt);
+}
+
+int
+vnet_mpls_tunnel_path_remove (u32 sw_if_index,
+                              fib_route_path_t *rpaths)
+{
+    mpls_tunnel_t *mt;
+    u32 mti;
 
-    mt->mt_label_stack = vec_dup(label_stack);
+    mt = mpls_tunnel_get_from_sw_if_index(sw_if_index);
 
-    if (l2_only)
+    if (NULL == mt)
+        return (0);
+
+    mti = mt - mpls_tunnel_pool;
+
+    /*
+     * construct a path-list from the path provided
+     */
+    if (FIB_NODE_INDEX_INVALID == mt->mt_path_list)
     {
-	mt->mt_l2_adj =
-	    adj_nbr_add_or_lock(fib_path_list_get_proto(mt->mt_path_list),
-				VNET_LINK_ETHERNET,
-				&zero_addr,
-				mt->mt_sw_if_index);
-
-	mt->mt_l2_tx_arc = vlib_node_add_named_next(vlib_get_main(),
-						    hi->tx_node_index,
-						    "adj-l2-midchain");
+        /* can't remove a path if we have onoe */
+        return (0);
     }
-
-    *sw_if_index = mt->mt_sw_if_index;
+    else
+    {
+        fib_node_index_t old_pl_index;
+        fib_path_ext_t *path_ext;
+
+        old_pl_index = mt->mt_path_list;
+
+        mt->mt_path_list =
+            fib_path_list_copy_and_path_remove(old_pl_index,
+                                               FIB_PATH_LIST_FLAG_SHARED,
+                                               rpaths);
+
+        fib_path_list_child_remove(old_pl_index,
+                                   mt->mt_sibling_index);
+
+        if (FIB_NODE_INDEX_INVALID == mt->mt_path_list)
+        {
+            /* no paths left */
+            return (0);
+        }
+        else
+        {
+            mt->mt_sibling_index =
+                fib_path_list_child_add(mt->mt_path_list,
+                                        FIB_NODE_TYPE_MPLS_TUNNEL,
+                                        mti);
+        }
+        /*
+         * find the matching path extension and remove it
+         */
+        vec_foreach(path_ext, mt->mt_path_exts)
+        {
+            if (!fib_path_ext_cmp(path_ext, rpaths))
+            {
+                /*
+                 * delete the element moving the remaining elements down 1 position.
+                 * this preserves the sorted order.
+                 */
+                vec_free(path_ext->fpe_label_stack);
+                vec_delete(mt->mt_path_exts, 1,
+                           (path_ext - mt->mt_path_exts));
+                break;
+            }
+        }
+       /*
+         * re-resolve all the path-extensions with the new path-list
+         */
+        vec_foreach(path_ext, mt->mt_path_exts)
+        {
+            fib_path_ext_resolve(path_ext, mt->mt_path_list);
+        }
+
+        mpls_tunnel_restack(mt);
+   }
+
+    return (fib_path_list_get_n_paths(mt->mt_path_list));
 }
 
+
 static clib_error_t *
 vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm,
-				    unformat_input_t * input,
-				    vlib_cli_command_t * cmd)
+                                    unformat_input_t * input,
+                                    vlib_cli_command_t * cmd)
 {
     unformat_input_t _line_input, * line_input = &_line_input;
     vnet_main_t * vnm = vnet_get_main();
-    u8 is_del = 0;
-    u8 l2_only = 0;
+    u8 is_del = 0, l2_only = 0, is_multicast =0;
     fib_route_path_t rpath, *rpaths = NULL;
-    mpls_label_t out_label = MPLS_LABEL_INVALID, *labels = NULL;
+    mpls_label_t out_label = MPLS_LABEL_INVALID;
     u32 sw_if_index;
     clib_error_t *error = NULL;
 
@@ -541,87 +839,89 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm,
 
     /* Get a line of input. */
     if (! unformat_user (input, unformat_line_input, line_input))
-	return 0;
+        return 0;
 
     while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
     {
-	if (unformat (line_input, "del %U",
-		      unformat_vnet_sw_interface, vnm,
-		      &sw_if_index))
-	    is_del = 1;
-	else if (unformat (line_input, "add"))
-	    is_del = 0;
-	else if (unformat (line_input, "out-label %U",
-			   unformat_mpls_unicast_label, &out_label))
-	{
-	    vec_add1(labels, out_label);
-	}
-	else if (unformat (line_input, "via %U %U",
-			   unformat_ip4_address,
-			   &rpath.frp_addr.ip4,
-			   unformat_vnet_sw_interface, vnm,
-			   &rpath.frp_sw_if_index))
-	{
-	    rpath.frp_weight = 1;
-	    rpath.frp_proto = FIB_PROTOCOL_IP4;
-	}
-			 
-	else if (unformat (line_input, "via %U %U",
-			   unformat_ip6_address,
-			   &rpath.frp_addr.ip6,
-			   unformat_vnet_sw_interface, vnm,
-			   &rpath.frp_sw_if_index))
-	{
-	    rpath.frp_weight = 1;
-	    rpath.frp_proto = FIB_PROTOCOL_IP6;
-	}
-	else if (unformat (line_input, "via %U",
-			   unformat_ip6_address,
-			   &rpath.frp_addr.ip6))
-	{
-	    rpath.frp_fib_index = 0;
-	    rpath.frp_weight = 1;
-	    rpath.frp_sw_if_index = ~0;
-	    rpath.frp_proto = FIB_PROTOCOL_IP6;
-	}
-	else if (unformat (line_input, "via %U",
-			   unformat_ip4_address,
-			   &rpath.frp_addr.ip4))
-	{
-	    rpath.frp_fib_index = 0;
-	    rpath.frp_weight = 1;
-	    rpath.frp_sw_if_index = ~0;
-	    rpath.frp_proto = FIB_PROTOCOL_IP4;
-	}
-	else if (unformat (line_input, "l2-only"))
-	    l2_only = 1;
-	else
-	{
-	    error = clib_error_return (0, "unknown input '%U'",
-				       format_unformat_error, line_input);
-	    goto done;
-	}
+        if (unformat (line_input, "del %U",
+                      unformat_vnet_sw_interface, vnm,
+                      &sw_if_index))
+            is_del = 1;
+        else if (unformat (line_input, "add"))
+            is_del = 0;
+        else if (unformat (line_input, "out-label %U",
+                           unformat_mpls_unicast_label, &out_label))
+        {
+            vec_add1(rpath.frp_label_stack, out_label);
+        }
+        else if (unformat (line_input, "via %U %U",
+                           unformat_ip4_address,
+                           &rpath.frp_addr.ip4,
+                           unformat_vnet_sw_interface, vnm,
+                           &rpath.frp_sw_if_index))
+        {
+            rpath.frp_weight = 1;
+            rpath.frp_proto = FIB_PROTOCOL_IP4;
+        }
+
+        else if (unformat (line_input, "via %U %U",
+                           unformat_ip6_address,
+                           &rpath.frp_addr.ip6,
+                           unformat_vnet_sw_interface, vnm,
+                           &rpath.frp_sw_if_index))
+        {
+            rpath.frp_weight = 1;
+            rpath.frp_proto = FIB_PROTOCOL_IP6;
+        }
+        else if (unformat (line_input, "via %U",
+                           unformat_ip6_address,
+                           &rpath.frp_addr.ip6))
+        {
+            rpath.frp_fib_index = 0;
+            rpath.frp_weight = 1;
+            rpath.frp_sw_if_index = ~0;
+            rpath.frp_proto = FIB_PROTOCOL_IP6;
+        }
+        else if (unformat (line_input, "via %U",
+                           unformat_ip4_address,
+                           &rpath.frp_addr.ip4))
+        {
+            rpath.frp_fib_index = 0;
+            rpath.frp_weight = 1;
+            rpath.frp_sw_if_index = ~0;
+            rpath.frp_proto = FIB_PROTOCOL_IP4;
+        }
+        else if (unformat (line_input, "l2-only"))
+            l2_only = 1;
+        else if (unformat (line_input, "multicast"))
+            is_multicast = 1;
+        else
+        {
+            error = clib_error_return (0, "unknown input '%U'",
+                                       format_unformat_error, line_input);
+            goto done;
+        }
     }
 
     if (is_del)
     {
-	vnet_mpls_tunnel_del(sw_if_index);
+        vnet_mpls_tunnel_del(sw_if_index);
     }
     else
     {
-	if (0 == vec_len(labels))
-	{
-	    error = clib_error_return (0, "No Output Labels '%U'",
-				       format_unformat_error, line_input);
-	    goto done;
-	}
-
-	vec_add1(rpaths, rpath);
-	vnet_mpls_tunnel_add(rpaths, labels, l2_only, &sw_if_index);
+        if (0 == vec_len(rpath.frp_label_stack))
+        {
+            error = clib_error_return (0, "No Output Labels '%U'",
+                                       format_unformat_error, line_input);
+            goto done;
+        }
+
+        vec_add1(rpaths, rpath);
+        sw_if_index = vnet_mpls_tunnel_create(l2_only, is_multicast);
+        vnet_mpls_tunnel_path_add(sw_if_index, rpaths);
     }
 
 done:
-    vec_free(labels);
     vec_free(rpaths);
     unformat_free (line_input);
 
@@ -638,7 +938,7 @@ done:
  ?*/
 VLIB_CLI_COMMAND (create_mpls_tunnel_command, static) = {
   .path = "mpls tunnel",
-  .short_help = 
+  .short_help =
   "mpls tunnel via [addr] [interface] [out-labels]",
   .function = vnet_create_mpls_tunnel_command_fn,
 };
@@ -647,19 +947,28 @@ static u8 *
 format_mpls_tunnel (u8 * s, va_list * args)
 {
     mpls_tunnel_t *mt = va_arg (*args, mpls_tunnel_t *);
-    int ii;
+    mpls_tunnel_attribute_t attr;
+    fib_path_ext_t *path_ext;
 
     s = format(s, "mpls_tunnel%d: sw_if_index:%d hw_if_index:%d",
-	       mt - mpls_tunnel_pool,
-	       mt->mt_sw_if_index,
-	       mt->mt_hw_if_index);
-    s = format(s, "\n label-stack:\n  ");
-    for (ii = 0; ii < vec_len(mt->mt_label_stack); ii++)
-    {
-	s = format(s, "%d, ", mt->mt_label_stack[ii]);
+               mt - mpls_tunnel_pool,
+               mt->mt_sw_if_index,
+               mt->mt_hw_if_index);
+    if (MPLS_TUNNEL_FLAG_NONE != mt->mt_flags) {
+        s = format(s, " \n flags:");
+        FOR_EACH_MPLS_TUNNEL_ATTRIBUTE(attr) {
+            if ((1<<attr) & mt->mt_flags) {
+                s = format (s, "%s,", mpls_tunnel_attribute_names[attr]);
+            }
+        }
     }
     s = format(s, "\n via:\n");
     s = fib_path_list_format(mt->mt_path_list, s);
+    s = format(s, "    Extensions:");
+    vec_foreach(path_ext, mt->mt_path_exts)
+    {
+        s = format(s, "\n     %U", format_fib_path_ext, path_ext);
+    }
     s = format(s, "\n");
 
     return (s);
@@ -667,42 +976,42 @@ format_mpls_tunnel (u8 * s, va_list * args)
 
 static clib_error_t *
 show_mpls_tunnel_command_fn (vlib_main_t * vm,
-			     unformat_input_t * input,
-			     vlib_cli_command_t * cmd)
+                             unformat_input_t * input,
+                             vlib_cli_command_t * cmd)
 {
     mpls_tunnel_t * mt;
     u32 mti = ~0;
 
     if (pool_elts (mpls_tunnel_pool) == 0)
-	vlib_cli_output (vm, "No MPLS tunnels configured...");
+        vlib_cli_output (vm, "No MPLS tunnels configured...");
 
     while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
     {
-	if (unformat (input, "%d", &mti))
-	    ;
-	else
-	    break;
+        if (unformat (input, "%d", &mti))
+            ;
+        else
+            break;
     }
 
     if (~0 == mti)
     {
-	pool_foreach (mt, mpls_tunnel_pool,
-	({
-	    vlib_cli_output (vm, "[@%d] %U",
-			     mt - mpls_tunnel_pool,
-			     format_mpls_tunnel, mt);
-	}));
+        pool_foreach (mt, mpls_tunnel_pool,
+        ({
+            vlib_cli_output (vm, "[@%d] %U",
+                             mt - mpls_tunnel_pool,
+                             format_mpls_tunnel, mt);
+        }));
     }
     else
     {
-	if (pool_is_free_index(mpls_tunnel_pool, mti))
-	    return clib_error_return (0, "Not atunnel index %d", mti);
+        if (pool_is_free_index(mpls_tunnel_pool, mti))
+            return clib_error_return (0, "Not atunnel index %d", mti);
 
-	mt = pool_elt_at_index(mpls_tunnel_pool, mti);
+        mt = pool_elt_at_index(mpls_tunnel_pool, mti);
 
-	vlib_cli_output (vm, "[@%d] %U",
-			 mt - mpls_tunnel_pool,
-			 format_mpls_tunnel, mt);
+        vlib_cli_output (vm, "[@%d] %U",
+                         mt - mpls_tunnel_pool,
+                         format_mpls_tunnel, mt);
     }
 
     return 0;
@@ -715,7 +1024,7 @@ show_mpls_tunnel_command_fn (vlib_main_t * vm,
  * @cliexstart{sh mpls tunnel 2}
  * [@2] mpls_tunnel2: sw_if_index:5 hw_if_index:5
  *  label-stack:
- *    3, 
+ *    3,
  *  via:
  *   index:26 locks:1 proto:ipv4 uPRF-list:26 len:1 itfs:[2, ]
  *     index:26 pl-index:26 ipv4 weight=1 attached-nexthop:  oper-flags:resolved,
@@ -743,7 +1052,7 @@ mpls_tunnel_from_fib_node (fib_node_t *node)
  */
 static fib_node_back_walk_rc_t
 mpls_tunnel_back_walk (fib_node_t *node,
-		      fib_node_back_walk_ctx_t *ctx)
+                      fib_node_back_walk_ctx_t *ctx)
 {
     mpls_tunnel_restack(mpls_tunnel_from_fib_node(node));
 
diff --git a/src/vnet/mpls/mpls_tunnel.h b/src/vnet/mpls/mpls_tunnel.h
index ee56c0fc..0b55d0db 100644
--- a/src/vnet/mpls/mpls_tunnel.h
+++ b/src/vnet/mpls/mpls_tunnel.h
@@ -17,6 +17,31 @@
 #define __MPLS_TUNNEL_H__
 
 #include <vnet/mpls/mpls.h>
+#include <vnet/fib/fib_path_ext.h>
+
+typedef enum mpls_tunnel_attribute_t_
+{
+    MPLS_TUNNEL_ATTRIBUTE_FIRST = 0,
+    /**
+     * @brief The tunnel has an underlying multicast LSP
+     */
+    MPLS_TUNNEL_ATTRIBUTE_MCAST = MPLS_TUNNEL_ATTRIBUTE_FIRST,
+    MPLS_TUNNEL_ATTRIBUTE_LAST = MPLS_TUNNEL_ATTRIBUTE_MCAST,
+} mpls_tunnel_attribute_t;
+
+#define MPLS_TUNNEL_ATTRIBUTES {		  \
+    [MPLS_TUNNEL_ATTRIBUTE_MCAST]  = "multicast", \
+}
+#define FOR_EACH_MPLS_TUNNEL_ATTRIBUTE(_item)		\
+    for (_item = MPLS_TUNNEL_ATTRIBUTE_FIRST;		\
+	 _item < MPLS_TUNNEL_ATTRIBUTE_LAST;		\
+	 _item++)
+
+typedef enum mpls_tunnel_flag_t_ {
+    MPLS_TUNNEL_FLAG_NONE   = 0,
+    MPLS_TUNNEL_FLAG_MCAST  = (1 << MPLS_TUNNEL_ATTRIBUTE_MCAST),
+} __attribute__ ((packed)) mpls_tunnel_flags_t;
+
 
 /**
  * @brief A uni-directional MPLS tunnel
@@ -28,6 +53,11 @@ typedef struct mpls_tunnel_t_
      */
     fib_node_t mt_node;
 
+    /**
+     * @brief Tunnel flags
+     */
+    mpls_tunnel_flags_t mt_flags;
+
     /**
      * @brief If the tunnel is an L2 tunnel, this is the link type ETHERNET
      * adjacency
@@ -50,9 +80,9 @@ typedef struct mpls_tunnel_t_
     u32 mt_sibling_index;
 
     /**
-     * @brief The Label stack to apply to egress packets
+     * A vector of path extensions o hold the label stack for each path
      */
-    mpls_label_t *mt_label_stack;
+    fib_path_ext_t *mt_path_exts;
 
     /**
      * @brief Flag to indicate the tunnel is only for L2 traffic, that is
@@ -74,12 +104,27 @@ typedef struct mpls_tunnel_t_
 
 /**
  * @brief Create a new MPLS tunnel
+ * @return the SW Interface index of the newly created tuneel
  */
-extern void vnet_mpls_tunnel_add (fib_route_path_t *rpath,
-				  mpls_label_t *label_stack,
-				  u8 l2_only,
-				  u32 *sw_if_index);
+extern u32 vnet_mpls_tunnel_create (u8 l2_only,
+                                    u8 is_multicast);
 
+/**
+ * @brief Add a path to an MPLS tunnel
+ */
+extern void vnet_mpls_tunnel_path_add (u32 sw_if_index,
+                                       fib_route_path_t *rpath);
+
+/**
+ * @brief remove a path from a tunnel.
+ * @return the number of remaining paths. 0 implies the tunnel can be deleted
+ */
+extern int vnet_mpls_tunnel_path_remove (u32 sw_if_index,
+                                         fib_route_path_t *rpath);
+
+/**
+ * @brief Delete an MPLS tunnel
+ */
 extern void vnet_mpls_tunnel_del (u32 sw_if_index);
 
 extern const mpls_tunnel_t *mpls_tunnel_get(u32 index);
diff --git a/src/vnet/mpls/mpls_types.h b/src/vnet/mpls/mpls_types.h
index d7c629df..b1075cdd 100644
--- a/src/vnet/mpls/mpls_types.h
+++ b/src/vnet/mpls/mpls_types.h
@@ -1,3 +1,17 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 #ifndef __MPLS_TYPES_H__
 #define __MPLS_TYPES_H__
 
@@ -36,4 +50,10 @@
     (((_lbl) > MPLS_IETF_MIN_UNRES_LABEL) &&	\
      ((_lbl) <= MPLS_IETF_MAX_UNRES_LABEL))
 
+/**
+ * The top bit of the index, which is the result of the MPLS lookup
+ * is used to determine if the DPO is a load-balance or a replicate
+ */
+#define MPLS_IS_REPLICATE 0x80000000
+
 #endif
diff --git a/src/vnet/srp/interface.c b/src/vnet/srp/interface.c
index d427cc3c..44e2b0d6 100644
--- a/src/vnet/srp/interface.c
+++ b/src/vnet/srp/interface.c
@@ -58,7 +58,7 @@ srp_build_rewrite (vnet_main_t * vnm,
 #define _(a,b) case VNET_LINK_##a: type = ETHERNET_TYPE_##b; break
     _ (IP4, IP4);
     _ (IP6, IP6);
-    _ (MPLS, MPLS_UNICAST);
+    _ (MPLS, MPLS);
     _ (ARP, ARP);
 #undef _
   default:
diff --git a/test/test_ip_mcast.py b/test/test_ip_mcast.py
index 36d597a7..c1397d70 100644
--- a/test/test_ip_mcast.py
+++ b/test/test_ip_mcast.py
@@ -622,6 +622,7 @@ class TestIPMcast(VppTestCase):
             (MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT |
              MRouteItfFlags.MFIB_ITF_FLAG_NEGATE_SIGNAL))
 
+        self.vapi.cli("clear trace")
         tx = self._mcast_connected_send_stream("232.1.1.1")
 
         signals = self.vapi.mfib_signal_dump()
diff --git a/test/test_mpls.py b/test/test_mpls.py
index fc832644..700b7091 100644
--- a/test/test_mpls.py
+++ b/test/test_mpls.py
@@ -5,7 +5,9 @@ import socket
 
 from framework import VppTestCase, VppTestRunner
 from vpp_ip_route import VppIpRoute, VppRoutePath, VppMplsRoute, \
-    VppMplsIpBind
+    VppMplsIpBind, VppIpMRoute, VppMRoutePath, \
+    MRouteItfFlags, MRouteEntryFlags
+from vpp_mpls_tunnel_interface import VppMPLSTunnelInterface
 
 from scapy.packet import Raw
 from scapy.layers.l2 import Ether
@@ -21,7 +23,7 @@ class TestMPLS(VppTestCase):
         super(TestMPLS, self).setUp()
 
         # create 2 pg interfaces
-        self.create_pg_interfaces(range(2))
+        self.create_pg_interfaces(range(4))
 
         # setup both interfaces
         # assign them different tables.
@@ -53,10 +55,12 @@ class TestMPLS(VppTestCase):
             mpls_labels,
             mpls_ttl=255,
             ping=0,
-            ip_itf=None):
+            ip_itf=None,
+            dst_ip=None,
+            n=257):
         self.reset_packet_infos()
         pkts = []
-        for i in range(0, 257):
+        for i in range(0, n):
             info = self.create_packet_info(src_if, src_if)
             payload = self.info_to_payload(info)
             p = Ether(dst=src_if.local_mac, src=src_if.remote_mac)
@@ -67,9 +71,14 @@ class TestMPLS(VppTestCase):
                 else:
                     p = p / MPLS(label=mpls_labels[ii], ttl=mpls_ttl, s=0)
             if not ping:
-                p = (p / IP(src=src_if.local_ip4, dst=src_if.remote_ip4) /
-                     UDP(sport=1234, dport=1234) /
-                     Raw(payload))
+                if not dst_ip:
+                    p = (p / IP(src=src_if.local_ip4, dst=src_if.remote_ip4) /
+                         UDP(sport=1234, dport=1234) /
+                         Raw(payload))
+                else:
+                    p = (p / IP(src=src_if.local_ip4, dst=dst_ip) /
+                         UDP(sport=1234, dport=1234) /
+                         Raw(payload))
             else:
                 p = (p / IP(src=ip_itf.remote_ip4,
                             dst=ip_itf.local_ip4) /
@@ -254,6 +263,13 @@ class TestMPLS(VppTestCase):
         except:
             raise
 
+    def send_and_assert_no_replies(self, intf, pkts, remark):
+        intf.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        for i in self.pg_interfaces:
+            i.assert_nothing_captured(remark=remark)
+
     def test_swap(self):
         """ MPLS label swap tests """
 
@@ -278,7 +294,7 @@ class TestMPLS(VppTestCase):
         self.pg_start()
 
         rx = self.pg0.get_capture()
-        self.verify_capture_labelled_ip4(self.pg0, rx, tx, [33])
+        self.verify_capture_labelled(self.pg0, rx, tx, [33])
 
         #
         # A simple MPLS xconnect - non-eos label in label out
@@ -358,7 +374,7 @@ class TestMPLS(VppTestCase):
         self.pg_start()
 
         rx = self.pg0.get_capture()
-        self.verify_capture_labelled_ip4(self.pg0, rx, tx, [33, 44, 45])
+        self.verify_capture_labelled(self.pg0, rx, tx, [33, 44, 45], num=2)
 
         #
         # A recursive non-EOS x-connect, which resolves through another
@@ -576,25 +592,19 @@ class TestMPLS(VppTestCase):
         #
         # Create a tunnel with a single out label
         #
-        nh_addr = socket.inet_pton(socket.AF_INET, self.pg0.remote_ip4)
-
-        reply = self.vapi.mpls_tunnel_add_del(
-            0xffffffff,  # don't know the if index yet
-            1,  # IPv4 next-hop
-            nh_addr,
-            self.pg0.sw_if_index,
-            0,  # next-hop-table-id
-            1,  # next-hop-weight
-            2,  # num-out-labels,
-            [44, 46])
-        self.vapi.sw_interface_set_flags(reply.sw_if_index, admin_up_down=1)
+        mpls_tun = VppMPLSTunnelInterface(self,
+                                          [VppRoutePath(self.pg0.remote_ip4,
+                                                        self.pg0.sw_if_index,
+                                                        labels=[44, 46])])
+        mpls_tun.add_vpp_config()
+        mpls_tun.admin_up()
 
         #
         # add an unlabelled route through the new tunnel
         #
         route_10_0_0_3 = VppIpRoute(self, "10.0.0.3", 32,
                                     [VppRoutePath("0.0.0.0",
-                                                  reply.sw_if_index)])
+                                                  mpls_tun._sw_if_index)])
         route_10_0_0_3.add_vpp_config()
 
         self.vapi.cli("clear trace")
@@ -738,6 +748,229 @@ class TestMPLS(VppTestCase):
         route_35_eos.remove_vpp_config()
         route_34_eos.remove_vpp_config()
 
+    def test_interface_rx(self):
+        """ MPLS Interface Receive """
+
+        #
+        # Add a non-recursive route that will forward the traffic
+        # post-interface-rx
+        #
+        route_10_0_0_1 = VppIpRoute(self, "10.0.0.1", 32,
+                                    table_id=1,
+                                    paths=[VppRoutePath(self.pg1.remote_ip4,
+                                                        self.pg1.sw_if_index)])
+        route_10_0_0_1.add_vpp_config()
+
+        #
+        # An interface receive label that maps traffic to RX on interface
+        # pg1
+        # by injecting the packet in on pg0, which is in table 0
+        # doing an interface-rx on pg1 and matching a route in table 1
+        # if the packet egresses, then we must have swapped to pg1
+        # so as to have matched the route in table 1
+        #
+        route_34_eos = VppMplsRoute(self, 34, 1,
+                                    [VppRoutePath("0.0.0.0",
+                                                  self.pg1.sw_if_index,
+                                                  is_interface_rx=1)])
+        route_34_eos.add_vpp_config()
+
+        #
+        # ping an interface in the default table
+        # PG0 is in the default table
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_labelled_ip4(self.pg0, [34], n=257,
+                                             dst_ip="10.0.0.1")
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx = self.pg1.get_capture(257)
+        self.verify_capture_ip4(self.pg1, rx, tx)
+
+    def test_mcast_mid_point(self):
+        """ MPLS Multicast Mid Point """
+
+        #
+        # Add a non-recursive route that will forward the traffic
+        # post-interface-rx
+        #
+        route_10_0_0_1 = VppIpRoute(self, "10.0.0.1", 32,
+                                    table_id=1,
+                                    paths=[VppRoutePath(self.pg1.remote_ip4,
+                                                        self.pg1.sw_if_index)])
+        route_10_0_0_1.add_vpp_config()
+
+        #
+        # Add a mcast entry that replicate to pg2 and pg3
+        # and replicate to a interface-rx (like a bud node would)
+        #
+        route_3400_eos = VppMplsRoute(self, 3400, 1,
+                                      [VppRoutePath(self.pg2.remote_ip4,
+                                                    self.pg2.sw_if_index,
+                                                    labels=[3401]),
+                                       VppRoutePath(self.pg3.remote_ip4,
+                                                    self.pg3.sw_if_index,
+                                                    labels=[3402]),
+                                       VppRoutePath("0.0.0.0",
+                                                    self.pg1.sw_if_index,
+                                                    is_interface_rx=1)],
+                                      is_multicast=1)
+        route_3400_eos.add_vpp_config()
+
+        #
+        # ping an interface in the default table
+        # PG0 is in the default table
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_labelled_ip4(self.pg0, [3400], n=257,
+                                             dst_ip="10.0.0.1")
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx = self.pg1.get_capture(257)
+        self.verify_capture_ip4(self.pg1, rx, tx)
+
+        rx = self.pg2.get_capture(257)
+        self.verify_capture_labelled(self.pg2, rx, tx, [3401])
+        rx = self.pg3.get_capture(257)
+        self.verify_capture_labelled(self.pg3, rx, tx, [3402])
+
+    def test_mcast_head(self):
+        """ MPLS Multicast Head-end """
+
+        #
+        # Create a multicast tunnel with two replications
+        #
+        mpls_tun = VppMPLSTunnelInterface(self,
+                                          [VppRoutePath(self.pg2.remote_ip4,
+                                                        self.pg2.sw_if_index,
+                                                        labels=[42]),
+                                           VppRoutePath(self.pg3.remote_ip4,
+                                                        self.pg3.sw_if_index,
+                                                        labels=[43])],
+                                          is_multicast=1)
+        mpls_tun.add_vpp_config()
+        mpls_tun.admin_up()
+
+        #
+        # add an unlabelled route through the new tunnel
+        #
+        route_10_0_0_3 = VppIpRoute(self, "10.0.0.3", 32,
+                                    [VppRoutePath("0.0.0.0",
+                                                  mpls_tun._sw_if_index)])
+        route_10_0_0_3.add_vpp_config()
+
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip4(self.pg0, "10.0.0.3")
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx = self.pg2.get_capture(257)
+        self.verify_capture_tunneled_ip4(self.pg0, rx, tx, [42])
+        rx = self.pg3.get_capture(257)
+        self.verify_capture_tunneled_ip4(self.pg0, rx, tx, [43])
+
+        #
+        # An an IP multicast route via the tunnel
+        # A (*,G).
+        # one accepting interface, pg0, 1 forwarding interface via the tunnel
+        #
+        route_232_1_1_1 = VppIpMRoute(
+            self,
+            "0.0.0.0",
+            "232.1.1.1", 32,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            [VppMRoutePath(self.pg0.sw_if_index,
+                           MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT),
+             VppMRoutePath(mpls_tun._sw_if_index,
+                           MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)])
+        route_232_1_1_1.add_vpp_config()
+
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip4(self.pg0, "232.1.1.1")
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx = self.pg2.get_capture(257)
+        self.verify_capture_tunneled_ip4(self.pg0, rx, tx, [42])
+        rx = self.pg3.get_capture(257)
+        self.verify_capture_tunneled_ip4(self.pg0, rx, tx, [43])
+
+    def test_mcast_tail(self):
+        """ MPLS Multicast Tail """
+
+        #
+        # Add a multicast route that will forward the traffic
+        # post-disposition
+        #
+        route_232_1_1_1 = VppIpMRoute(
+            self,
+            "0.0.0.0",
+            "232.1.1.1", 32,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            table_id=1,
+            paths=[VppMRoutePath(self.pg1.sw_if_index,
+                                 MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)])
+        route_232_1_1_1.add_vpp_config()
+
+        #
+        # An interface receive label that maps traffic to RX on interface
+        # pg1
+        # by injecting the packet in on pg0, which is in table 0
+        # doing an rpf-id  and matching a route in table 1
+        # if the packet egresses, then we must have matched the route in
+        # table 1
+        #
+        route_34_eos = VppMplsRoute(self, 34, 1,
+                                    [VppRoutePath("0.0.0.0",
+                                                  self.pg1.sw_if_index,
+                                                  nh_table_id=1,
+                                                  rpf_id=55)],
+                                    is_multicast=1)
+
+        route_34_eos.add_vpp_config()
+
+        #
+        # Drop due to interface lookup miss
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_labelled_ip4(self.pg0, [34],
+                                             dst_ip="232.1.1.1", n=1)
+        self.send_and_assert_no_replies(self.pg0, tx, "RPF-ID drop none")
+
+        #
+        # set the RPF-ID of the enrtry to match the input packet's
+        #
+        route_232_1_1_1.update_rpf_id(55)
+
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_labelled_ip4(self.pg0, [34],
+                                             dst_ip="232.1.1.1", n=257)
+        self.pg0.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx = self.pg1.get_capture(257)
+        self.verify_capture_ip4(self.pg1, rx, tx)
+
+        #
+        # set the RPF-ID of the enrtry to not match the input packet's
+        #
+        route_232_1_1_1.update_rpf_id(56)
+        tx = self.create_stream_labelled_ip4(self.pg0, [34],
+                                             dst_ip="232.1.1.1")
+        self.send_and_assert_no_replies(self.pg0, tx, "RPF-ID drop 56")
+
 
 class TestMPLSDisabled(VppTestCase):
     """ MPLS disabled """
diff --git a/test/vpp_ip_route.py b/test/vpp_ip_route.py
index faf5f801..d6146f28 100644
--- a/test/vpp_ip_route.py
+++ b/test/vpp_ip_route.py
@@ -55,15 +55,24 @@ class VppRoutePath(object):
             nh_table_id=0,
             labels=[],
             nh_via_label=MPLS_LABEL_INVALID,
-            is_ip6=0):
+            is_ip6=0,
+            rpf_id=0,
+            is_interface_rx=0):
         self.nh_itf = nh_sw_if_index
         self.nh_table_id = nh_table_id
         self.nh_via_label = nh_via_label
         self.nh_labels = labels
+        self.weight = 1
+        self.rpf_id = rpf_id
         if is_ip6:
             self.nh_addr = inet_pton(AF_INET6, nh_addr)
         else:
             self.nh_addr = inet_pton(AF_INET, nh_addr)
+        self.is_interface_rx = is_interface_rx
+        self.is_rpf_id = 0
+        if rpf_id != 0:
+            self.is_rpf_id = 1
+            self.nh_itf = rpf_id
 
 
 class VppMRoutePath(VppRoutePath):
@@ -176,13 +185,15 @@ class VppIpMRoute(VppObject):
     """
 
     def __init__(self, test, src_addr, grp_addr,
-                 grp_addr_len, e_flags, paths, table_id=0, is_ip6=0):
+                 grp_addr_len, e_flags, paths, table_id=0,
+                 rpf_id=0, is_ip6=0):
         self._test = test
         self.paths = paths
         self.grp_addr_len = grp_addr_len
         self.table_id = table_id
         self.e_flags = e_flags
         self.is_ip6 = is_ip6
+        self.rpf_id = rpf_id
 
         if is_ip6:
             self.grp_addr = inet_pton(AF_INET6, grp_addr)
@@ -199,6 +210,7 @@ class VppIpMRoute(VppObject):
                                               self.e_flags,
                                               path.nh_itf,
                                               path.nh_i_flags,
+                                              rpf_id=self.rpf_id,
                                               table_id=self.table_id,
                                               is_ipv6=self.is_ip6)
         self._test.registry.register(self, self._test.logger)
@@ -226,6 +238,18 @@ class VppIpMRoute(VppObject):
                                           table_id=self.table_id,
                                           is_ipv6=self.is_ip6)
 
+    def update_rpf_id(self, rpf_id):
+        self.rpf_id = rpf_id
+        self._test.vapi.ip_mroute_add_del(self.src_addr,
+                                          self.grp_addr,
+                                          self.grp_addr_len,
+                                          self.e_flags,
+                                          0xffffffff,
+                                          0,
+                                          rpf_id=self.rpf_id,
+                                          table_id=self.table_id,
+                                          is_ipv6=self.is_ip6)
+
     def update_path_flags(self, itf, flags):
         for path in self.paths:
             if path.nh_itf == itf:
@@ -342,14 +366,17 @@ class VppMplsRoute(VppObject):
     MPLS Route/LSP
     """
 
-    def __init__(self, test, local_label, eos_bit, paths, table_id=0):
+    def __init__(self, test, local_label, eos_bit, paths, table_id=0,
+                 is_multicast=0):
         self._test = test
         self.paths = paths
         self.local_label = local_label
         self.eos_bit = eos_bit
         self.table_id = table_id
+        self.is_multicast = is_multicast
 
     def add_vpp_config(self):
+        is_multipath = len(self.paths) > 1
         for path in self.paths:
             self._test.vapi.mpls_route_add_del(
                 self.local_label,
@@ -357,7 +384,11 @@ class VppMplsRoute(VppObject):
                 1,
                 path.nh_addr,
                 path.nh_itf,
+                is_multicast=self.is_multicast,
+                is_multipath=is_multipath,
                 table_id=self.table_id,
+                is_interface_rx=path.is_interface_rx,
+                is_rpf_id=path.is_rpf_id,
                 next_hop_out_label_stack=path.nh_labels,
                 next_hop_n_out_labels=len(
                     path.nh_labels),
@@ -372,6 +403,7 @@ class VppMplsRoute(VppObject):
                                                1,
                                                path.nh_addr,
                                                path.nh_itf,
+                                               is_rpf_id=path.is_rpf_id,
                                                table_id=self.table_id,
                                                is_add=0)
 
diff --git a/test/vpp_mpls_tunnel_interface.py b/test/vpp_mpls_tunnel_interface.py
new file mode 100644
index 00000000..f2001574
--- /dev/null
+++ b/test/vpp_mpls_tunnel_interface.py
@@ -0,0 +1,46 @@
+
+from vpp_interface import VppInterface
+from vpp_ip_route import VppRoutePath
+import socket
+
+
+class VppMPLSTunnelInterface(VppInterface):
+    """
+    VPP MPLS Tunnel interface
+    """
+
+    def __init__(self, test, paths, is_multicast=0):
+        """ Create MPLS Tunnel interface """
+        self._sw_if_index = 0
+        super(VppMPLSTunnelInterface, self).__init__(test)
+        self._test = test
+        self.t_paths = paths
+        self.is_multicast = is_multicast
+
+    def add_vpp_config(self):
+        self._sw_if_index = 0xffffffff
+        for path in self.t_paths:
+            reply = self.test.vapi.mpls_tunnel_add_del(
+                self._sw_if_index,
+                1,  # IPv4 next-hop
+                path.nh_addr,
+                path.nh_itf,
+                path.nh_table_id,
+                path.weight,
+                next_hop_out_label_stack=path.nh_labels,
+                next_hop_n_out_labels=len(path.nh_labels),
+                is_multicast=self.is_multicast)
+            self._sw_if_index = reply.sw_if_index
+
+    def remove_vpp_config(self):
+        for path in self.t_paths:
+            reply = self.test.vapi.mpls_tunnel_add_del(
+                self.sw_if_index,
+                1,  # IPv4 next-hop
+                path.nh_addr,
+                path.nh_itf,
+                path.nh_table_id,
+                path.weight,
+                next_hop_out_label_stack=path.nh_labels,
+                next_hop_n_out_labels=len(path.nh_labels),
+                is_add=0)
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index e8025dff..ceb684b7 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -849,6 +849,9 @@ class VppPapiProvider(object):
             create_vrf_if_needed=0,
             is_resolve_host=0,
             is_resolve_attached=0,
+            is_interface_rx=0,
+            is_rpf_id=0,
+            is_multicast=0,
             is_add=1,
             is_drop=0,
             is_multipath=0,
@@ -872,6 +875,7 @@ class VppPapiProvider(object):
         :param is_local:  (Default value = 0)
         :param is_classify:  (Default value = 0)
         :param is_multipath:  (Default value = 0)
+        :param is_multicast:  (Default value = 0)
         :param is_resolve_host:  (Default value = 0)
         :param is_resolve_attached:  (Default value = 0)
         :param not_last:  (Default value = 0)
@@ -889,8 +893,11 @@ class VppPapiProvider(object):
              'mr_is_add': is_add,
              'mr_is_classify': is_classify,
              'mr_is_multipath': is_multipath,
+             'mr_is_multicast': is_multicast,
              'mr_is_resolve_host': is_resolve_host,
              'mr_is_resolve_attached': is_resolve_attached,
+             'mr_is_interface_rx': is_interface_rx,
+             'mr_is_rpf_id': is_rpf_id,
              'mr_next_hop_proto_is_ip4': next_hop_proto_is_ip4,
              'mr_next_hop_weight': next_hop_weight,
              'mr_next_hop': next_hop_address,
@@ -936,7 +943,8 @@ class VppPapiProvider(object):
             next_hop_via_label=MPLS_LABEL_INVALID,
             create_vrf_if_needed=0,
             is_add=1,
-            l2_only=0):
+            l2_only=0,
+            is_multicast=0):
         """
 
         :param dst_address_length:
@@ -956,8 +964,8 @@ class VppPapiProvider(object):
         :param is_multipath:  (Default value = 0)
         :param is_resolve_host:  (Default value = 0)
         :param is_resolve_attached:  (Default value = 0)
-        :param not_last:  (Default value = 0)
         :param next_hop_weight:  (Default value = 1)
+        :param is_multicast:  (Default value = 0)
 
         """
         return self.api(
@@ -965,6 +973,7 @@ class VppPapiProvider(object):
             {'mt_sw_if_index': tun_sw_if_index,
              'mt_is_add': is_add,
              'mt_l2_only': l2_only,
+             'mt_is_multicast': is_multicast,
              'mt_next_hop_proto_is_ip4': next_hop_proto_is_ip4,
              'mt_next_hop_weight': next_hop_weight,
              'mt_next_hop': next_hop_address,
@@ -1469,6 +1478,7 @@ class VppPapiProvider(object):
                           e_flags,
                           next_hop_sw_if_index,
                           i_flags,
+                          rpf_id=0,
                           table_id=0,
                           create_vrf_if_needed=0,
                           is_add=1,
@@ -1481,6 +1491,8 @@ class VppPapiProvider(object):
             {'next_hop_sw_if_index': next_hop_sw_if_index,
              'entry_flags': e_flags,
              'itf_flags': i_flags,
+             'table_id': table_id,
+             'rpf_id': rpf_id,
              'create_vrf_if_needed': create_vrf_if_needed,
              'is_add': is_add,
              'is_ipv6': is_ipv6,
-- 
cgit 1.2.3-korg


From 107e7d4b5375295e94e01653e3cf064ea6647064 Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Tue, 11 Apr 2017 09:55:19 -0700
Subject: Remove usued, redundant and deprecated code from lookup.h

Change-Id: Ic16bc10d0b2877b2afdf052615f9334f31b9519f
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/plugins/ioam/encap/ip6_ioam_trace.c            |  4 +--
 src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c    |  3 +-
 .../ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c      | 10 +++---
 src/vnet/adj/adj.c                                 |  3 --
 src/vnet/ethernet/arp.c                            | 14 +++------
 src/vnet/fib/fib_path.c                            | 19 ------------
 src/vnet/ip/ip4_forward.c                          | 19 ++++++------
 src/vnet/ip/ip4_source_check.c                     |  2 +-
 src/vnet/ip/ip6_forward.c                          | 29 +++++++++--------
 src/vnet/ip/ip6_hop_by_hop.c                       |  8 ++---
 src/vnet/ip/ip6_neighbor.c                         |  6 ++--
 src/vnet/ip/lookup.c                               | 35 ++-------------------
 src/vnet/ip/lookup.h                               | 29 -----------------
 src/vnet/mpls/mpls.c                               |  4 +--
 src/vnet/sr/sr_localsid.c                          |  8 ++---
 src/vnet/sr/sr_policy_rewrite.c                    | 23 ++++++--------
 src/vnet/sr/sr_steering.c                          | 36 +++++++++++-----------
 17 files changed, 78 insertions(+), 174 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/plugins/ioam/encap/ip6_ioam_trace.c b/src/plugins/ioam/encap/ip6_ioam_trace.c
index 299ee88f..3ec3ea82 100644
--- a/src/plugins/ioam/encap/ip6_ioam_trace.c
+++ b/src/plugins/ioam/encap/ip6_ioam_trace.c
@@ -254,13 +254,11 @@ int
 ip6_hbh_ioam_trace_data_list_handler (vlib_buffer_t * b, ip6_header_t * ip,
 				      ip6_hop_by_hop_option_t * opt)
 {
-  ip6_main_t *im = &ip6_main;
-  ip_lookup_main_t *lm = &im->lookup_main;
   ip6_hop_by_hop_ioam_main_t *hm = &ip6_hop_by_hop_ioam_main;
   u8 elt_index = 0;
   ioam_trace_option_t *trace = (ioam_trace_option_t *) opt;
   u32 adj_index = vnet_buffer (b)->ip.adj_index[VLIB_TX];
-  ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index);
+  ip_adjacency_t *adj = adj_get (adj_index);
   time_u64_t time_u64;
   u32 *elt;
   int rv = 0;
diff --git a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c
index 6c04d9af..88d7d205 100644
--- a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c
+++ b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam.c
@@ -265,8 +265,7 @@ vxlan_gpe_enable_disable_ioam_for_dest (vlib_main_t * vm,
 		      continue;
 		    }
 
-		  adj0 =
-		    ip_get_adjacency (&(ip4_main.lookup_main), adj_index0);
+		  adj0 = adj_get (adj_index0);
 		  sw_if_index0 = adj0->rewrite_header.sw_if_index;
 
 		  if (~0 == sw_if_index0)
diff --git a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c
index 48edb4b0..f3d03b67 100644
--- a/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c
+++ b/src/plugins/ioam/lib-vxlan-gpe/vxlan_gpe_ioam_trace.c
@@ -302,11 +302,10 @@ vxlan_gpe_ioam_trace_data_list_handler (vlib_buffer_t * b,
 	    {
 	      u16 tx_if = 0;
 	      u32 adj_index = vnet_buffer (b)->ip.adj_index[VLIB_TX];
-	      ip4_main_t *im4 = &ip4_main;
-	      ip_lookup_main_t *lm = &im4->lookup_main;
+
 	      if (use_adj)
 		{
-		  ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index);
+		  ip_adjacency_t *adj = adj_get (adj_index);
 		  tx_if = adj->rewrite_header.sw_if_index & 0xFFFF;
 		}
 
@@ -330,11 +329,10 @@ vxlan_gpe_ioam_trace_data_list_handler (vlib_buffer_t * b,
 	    {
 	      u16 tx_if = 0;
 	      u32 adj_index = vnet_buffer (b)->ip.adj_index[VLIB_TX];
-	      ip6_main_t *im6 = &ip6_main;
-	      ip_lookup_main_t *lm = &im6->lookup_main;
+
 	      if (use_adj)
 		{
-		  ip_adjacency_t *adj = ip_get_adjacency (lm, adj_index);
+		  ip_adjacency_t *adj = adj_get (adj_index);
 		  tx_if = adj->rewrite_header.sw_if_index & 0xFFFF;
 		}
 
diff --git a/src/vnet/adj/adj.c b/src/vnet/adj/adj.c
index 36dfe500..bf44383f 100644
--- a/src/vnet/adj/adj.c
+++ b/src/vnet/adj/adj.c
@@ -71,9 +71,6 @@ adj_alloc (fib_protocol_t proto)
     memset(&adj->sub_type.midchain.next_dpo, 0,
            sizeof(adj->sub_type.midchain.next_dpo));
 
-    ip4_main.lookup_main.adjacency_heap = adj_pool;
-    ip6_main.lookup_main.adjacency_heap = adj_pool;
-
     return (adj);
 }
 
diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c
index dd509193..3e292e4d 100644
--- a/src/vnet/ethernet/arp.c
+++ b/src/vnet/ethernet/arp.c
@@ -1915,18 +1915,15 @@ vnet_proxy_arp_add_del (ip4_address_t * lo_addr,
 int
 vnet_proxy_arp_fib_reset (u32 fib_id)
 {
-  ip4_main_t *im = &ip4_main;
   ethernet_arp_main_t *am = &ethernet_arp_main;
   ethernet_proxy_arp_t *pa;
   u32 *entries_to_delete = 0;
   u32 fib_index;
-  uword *p;
   int i;
 
-  p = hash_get (im->fib_index_by_table_id, fib_id);
-  if (!p)
+  fib_index = fib_table_find (FIB_PROTOCOL_IP4, fib_id);
+  if (~0 == fib_index)
     return VNET_API_ERROR_NO_SUCH_ENTRY;
-  fib_index = p[0];
 
   vec_foreach (pa, am->proxy_arps)
   {
@@ -1985,11 +1982,10 @@ ip_arp_add_del_command_fn (vlib_main_t * vm,
 
       else if (unformat (input, "fib-id %d", &fib_id))
 	{
-	  ip4_main_t *im = &ip4_main;
-	  uword *p = hash_get (im->fib_index_by_table_id, fib_id);
-	  if (!p)
+	  fib_index = fib_table_find (FIB_PROTOCOL_IP4, fib_id);
+
+	  if (~0 == fib_index)
 	    return clib_error_return (0, "fib ID %d doesn't exist\n", fib_id);
-	  fib_index = p[0];
 	}
 
       else if (unformat (input, "proxy %U - %U",
diff --git a/src/vnet/fib/fib_path.c b/src/vnet/fib/fib_path.c
index f81f4170..6c1bd6bd 100644
--- a/src/vnet/fib/fib_path.c
+++ b/src/vnet/fib/fib_path.c
@@ -240,25 +240,6 @@ typedef struct fib_path_t_ {
 	    } fp_nh;
 	    /**
 	     * The FIB table index in which to find the next-hop.
-	     * This needs to be fixed. We should lookup the adjacencies in
-	     * a separate table of adjacencies, rather than from the FIB.
-	     * Two reasons I can think of:
-	     *   - consider:
-	     *       int ip addr Gig0 10.0.0.1/24
-	     *       ip route 10.0.0.2/32 via Gig1 192.168.1.2
-	     *       ip route 1.1.1.1/32 via Gig0 10.0.0.2
-	     *     this is perfectly valid.
-	     *     Packets addressed to 10.0.0.2 should be sent via Gig1.
-	     *     Packets address to 1.1.1.1 should be sent via Gig0.
-	     *    when we perform the adj resolution from the FIB for the path
-	     *    "via Gig0 10.0.0.2" the lookup will result in the route via Gig1
-	     *    and so we will pick up the adj via Gig1 - which was not what the
-	     *    operator wanted.
-	     *  - we can only return link-type IPv4 and so not the link-type MPLS.
-	     *    more on this in a later commit.
-	     *
-	     * The table ID should only belong to a recursive path and indicate
-	     * which FIB should be used to resolve the next-hop.
 	     */
 	    fib_node_index_t fp_tbl_id;
 	} recursive;
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index 9fdf9b3c..e42b3637 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -2036,7 +2036,7 @@ ip4_arp_inline (vlib_main_t * vm,
 	  p0 = vlib_get_buffer (vm, pi0);
 
 	  adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
-	  adj0 = ip_get_adjacency (lm, adj_index0);
+	  adj0 = adj_get (adj_index0);
 	  ip0 = vlib_buffer_get_current (p0);
 
 	  a0 = hash_seeds[0];
@@ -2276,7 +2276,7 @@ ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index)
 	 sw_if_index);
     }
 
-  adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
+  adj = adj_get (ia->neighbor_probe_adj_index);
 
   h =
     vlib_packet_template_get_packet (vm,
@@ -2465,8 +2465,8 @@ ip4_rewrite_inline (vlib_main_t * vm,
 	    }
 
 	  /* Rewrite packet header and updates lengths. */
-	  adj0 = ip_get_adjacency (lm, adj_index0);
-	  adj1 = ip_get_adjacency (lm, adj_index1);
+	  adj0 = adj_get (adj_index0);
+	  adj1 = adj_get (adj_index1);
 
 	  /* Worth pipelining. No guarantee that adj0,1 are hot... */
 	  rw_len0 = adj0[0].rewrite_header.data_bytes;
@@ -2571,7 +2571,7 @@ ip4_rewrite_inline (vlib_main_t * vm,
 
 	  adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 
-	  adj0 = ip_get_adjacency (lm, adj_index0);
+	  adj0 = adj_get (adj_index0);
 
 	  ip0 = vlib_buffer_get_current (p0);
 
@@ -3003,14 +3003,15 @@ VLIB_CLI_COMMAND (lookup_test_command, static) =
 int
 vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
 {
-  ip4_main_t *im4 = &ip4_main;
   ip4_fib_t *fib;
-  uword *p = hash_get (im4->fib_index_by_table_id, table_id);
+  u32 fib_index;
+
+  fib_index = fib_table_find (FIB_PROTOCOL_IP4, table_id);
 
-  if (p == 0)
+  if (~0 == fib_index)
     return VNET_API_ERROR_NO_SUCH_FIB;
 
-  fib = ip4_fib_get (p[0]);
+  fib = ip4_fib_get (fib_index);
 
   fib->flow_hash_config = flow_hash_config;
   return 0;
diff --git a/src/vnet/ip/ip4_source_check.c b/src/vnet/ip/ip4_source_check.c
index 6831066e..63b7594d 100644
--- a/src/vnet/ip/ip4_source_check.c
+++ b/src/vnet/ip/ip4_source_check.c
@@ -492,7 +492,7 @@ ip_source_check_accept (vlib_main_t * vm,
 
   if (~0 != table_id)
     {
-      fib_index = fib_table_id_find_fib_index (pfx.fp_proto, table_id);
+      fib_index = fib_table_find (pfx.fp_proto, table_id);
       if (~0 == fib_index)
 	{
 	  error = clib_error_return (0, "Nonexistent table id %d", table_id);
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index a369f79f..90a88b1c 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -1611,7 +1611,7 @@ ip6_discover_neighbor_inline (vlib_main_t * vm,
 
 	  ip0 = vlib_buffer_get_current (p0);
 
-	  adj0 = ip_get_adjacency (lm, adj_index0);
+	  adj0 = adj_get (adj_index0);
 
 	  if (!is_glean)
 	    {
@@ -1862,7 +1862,7 @@ ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index)
     vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
 
   /* Add encapsulation string for software interface (e.g. ethernet header). */
-  adj = ip_get_adjacency (&im->lookup_main, ia->neighbor_probe_adj_index);
+  adj = adj_get (ia->neighbor_probe_adj_index);
   vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
   vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
 
@@ -2007,8 +2007,8 @@ ip6_rewrite_inline (vlib_main_t * vm,
 	    {
 	      p1->flags &= ~VNET_BUFFER_LOCALLY_ORIGINATED;
 	    }
-	  adj0 = ip_get_adjacency (lm, adj_index0);
-	  adj1 = ip_get_adjacency (lm, adj_index1);
+	  adj0 = adj_get (adj_index0);
+	  adj1 = adj_get (adj_index1);
 
 	  rw_len0 = adj0[0].rewrite_header.data_bytes;
 	  rw_len1 = adj1[0].rewrite_header.data_bytes;
@@ -2108,7 +2108,7 @@ ip6_rewrite_inline (vlib_main_t * vm,
 
 	  adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
 
-	  adj0 = ip_get_adjacency (lm, adj_index0);
+	  adj0 = adj_get (adj_index0);
 
 	  ip0 = vlib_buffer_get_current (p0);
 
@@ -2531,8 +2531,6 @@ ip6_hop_by_hop (vlib_main_t * vm,
   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
   u32 n_left_from, *from, *to_next;
   ip_lookup_next_t next_index;
-  ip6_main_t *im = &ip6_main;
-  ip_lookup_main_t *lm = &im->lookup_main;
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;
@@ -2581,9 +2579,9 @@ ip6_hop_by_hop (vlib_main_t * vm,
 
 	  /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
 	  u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
-	  ip_adjacency_t *adj0 = ip_get_adjacency (lm, adj_index0);
+	  ip_adjacency_t *adj0 = adj_get (adj_index0);
 	  u32 adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
-	  ip_adjacency_t *adj1 = ip_get_adjacency (lm, adj_index1);
+	  ip_adjacency_t *adj1 = adj_get (adj_index1);
 
 	  /* Default use the next_index from the adjacency. A HBH option rarely redirects to a different node */
 	  next0 = adj0->lookup_next_index;
@@ -2704,7 +2702,7 @@ ip6_hop_by_hop (vlib_main_t * vm,
 	   * A HBH option rarely redirects to a different node
 	   */
 	  u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
-	  ip_adjacency_t *adj0 = ip_get_adjacency (lm, adj_index0);
+	  ip_adjacency_t *adj0 = adj_get (adj_index0);
 	  next0 = adj0->lookup_next_index;
 
 	  ip0 = vlib_buffer_get_current (b0);
@@ -3114,14 +3112,15 @@ VLIB_CLI_COMMAND (test_link_command, static) =
 int
 vnet_set_ip6_flow_hash (u32 table_id, u32 flow_hash_config)
 {
-  ip6_main_t *im6 = &ip6_main;
   ip6_fib_t *fib;
-  uword *p = hash_get (im6->fib_index_by_table_id, table_id);
+  u32 fib_index;
+
+  fib_index = fib_table_find (FIB_PROTOCOL_IP6, table_id);
 
-  if (p == 0)
-    return -1;
+  if (~0 == fib_index)
+    return VNET_API_ERROR_NO_SUCH_FIB;
 
-  fib = ip6_fib_get (p[0]);
+  fib = ip6_fib_get (fib_index);
 
   fib->flow_hash_config = flow_hash_config;
   return 1;
diff --git a/src/vnet/ip/ip6_hop_by_hop.c b/src/vnet/ip/ip6_hop_by_hop.c
index df2a932e..14fbb392 100644
--- a/src/vnet/ip/ip6_hop_by_hop.c
+++ b/src/vnet/ip/ip6_hop_by_hop.c
@@ -581,8 +581,6 @@ static uword
 ip6_pop_hop_by_hop_node_fn (vlib_main_t * vm,
 			    vlib_node_runtime_t * node, vlib_frame_t * frame)
 {
-  ip6_main_t *im = &ip6_main;
-  ip_lookup_main_t *lm = &im->lookup_main;
   u32 n_left_from, *from, *to_next;
   ip_lookup_next_t next_index;
   u32 processed = 0;
@@ -640,8 +638,8 @@ ip6_pop_hop_by_hop_node_fn (vlib_main_t * vm,
 	  ip1 = vlib_buffer_get_current (b1);
 	  adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
 	  adj_index1 = vnet_buffer (b1)->ip.adj_index[VLIB_TX];
-	  adj0 = ip_get_adjacency (lm, adj_index0);
-	  adj1 = ip_get_adjacency (lm, adj_index1);
+	  adj0 = adj_get (adj_index0);
+	  adj1 = adj_get (adj_index1);
 
 	  next0 = adj0->lookup_next_index;
 	  next1 = adj1->lookup_next_index;
@@ -729,7 +727,7 @@ ip6_pop_hop_by_hop_node_fn (vlib_main_t * vm,
 
 	  ip0 = vlib_buffer_get_current (b0);
 	  adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
-	  adj0 = ip_get_adjacency (lm, adj_index0);
+	  adj0 = adj_get (adj_index0);
 
 	  /* Default use the next_index from the adjacency. */
 	  next0 = adj0->lookup_next_index;
diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c
index 58b997aa..42edb79a 100644
--- a/src/vnet/ip/ip6_neighbor.c
+++ b/src/vnet/ip/ip6_neighbor.c
@@ -1001,8 +1001,7 @@ icmp6_neighbor_solicitation_or_advertisement (vlib_main_t * vm,
 
 	      if (ADJ_INDEX_INVALID != src_adj_index0)
 		{
-		  ip_adjacency_t *adj0 =
-		    ip_get_adjacency (&im->lookup_main, src_adj_index0);
+		  ip_adjacency_t *adj0 = adj_get (src_adj_index0);
 
 		  /* Allow all realistic-looking rewrite adjacencies to pass */
 		  ni0 = adj0->lookup_next_index;
@@ -1322,8 +1321,7 @@ icmp6_router_solicitation (vlib_main_t * vm,
 
 	      if (ADJ_INDEX_INVALID != src_adj_index0)
 		{
-		  ip_adjacency_t *adj0 = ip_get_adjacency (&im->lookup_main,
-							   src_adj_index0);
+		  ip_adjacency_t *adj0 = adj_get (src_adj_index0);
 
 		  error0 = (adj0->rewrite_header.sw_if_index != sw_if_index0
 			    ?
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 597de06b..2321c072 100755
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -191,9 +191,6 @@ VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ip_sw_interface_add_del);
 void
 ip_lookup_init (ip_lookup_main_t * lm, u32 is_ip6)
 {
-  /* Preallocate three "special" adjacencies */
-  lm->adjacency_heap = adj_pool;
-
   if (!lm->fib_result_n_bytes)
     lm->fib_result_n_bytes = sizeof (uword);
 
@@ -353,32 +350,6 @@ const ip46_address_t zero_addr = {
 	     0, 0},
 };
 
-u32
-fib_table_id_find_fib_index (fib_protocol_t proto, u32 table_id)
-{
-  ip4_main_t *im4 = &ip4_main;
-  ip6_main_t *im6 = &ip6_main;
-  uword *p;
-
-  switch (proto)
-    {
-    case FIB_PROTOCOL_IP4:
-      p = hash_get (im4->fib_index_by_table_id, table_id);
-      break;
-    case FIB_PROTOCOL_IP6:
-      p = hash_get (im6->fib_index_by_table_id, table_id);
-      break;
-    default:
-      p = NULL;
-      break;
-    }
-  if (NULL != p)
-    {
-      return (p[0]);
-    }
-  return (~0);
-}
-
 clib_error_t *
 vnet_ip_route_cmd (vlib_main_t * vm,
 		   unformat_input_t * main_input, vlib_cli_command_t * cmd)
@@ -602,7 +573,7 @@ vnet_ip_route_cmd (vlib_main_t * vm,
     }
   else
     {
-      fib_index = fib_table_id_find_fib_index (prefixs[0].fp_proto, table_id);
+      fib_index = fib_table_find (prefixs[0].fp_proto, table_id);
 
       if (~0 == fib_index)
 	{
@@ -651,8 +622,8 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 		  /*
 		   * the CLI parsing stored table Ids, swap to FIB indicies
 		   */
-		  fi = fib_table_id_find_fib_index (prefixs[i].fp_proto,
-						    rpaths[i].frp_fib_index);
+		  fi = fib_table_find (prefixs[i].fp_proto,
+				       rpaths[i].frp_fib_index);
 
 		  if (~0 == fi)
 		    {
diff --git a/src/vnet/ip/lookup.h b/src/vnet/ip/lookup.h
index 662a1cc0..941a6df9 100644
--- a/src/vnet/ip/lookup.h
+++ b/src/vnet/ip/lookup.h
@@ -121,12 +121,6 @@ struct ip_lookup_main_t;
 
 typedef struct ip_lookup_main_t
 {
-  /* Adjacency heap. */
-  ip_adjacency_t *adjacency_heap;
-
-  /** load-balance  packet/byte counters indexed by LB index. */
-  vlib_combined_counter_main_t load_balance_counters;
-
   /** Pool of addresses that are assigned to interfaces. */
   ip_interface_address_t *if_address_pool;
 
@@ -149,17 +143,12 @@ typedef struct ip_lookup_main_t
      sizeof (uword).  First word is always adjacency index. */
   u32 fib_result_n_bytes, fib_result_n_words;
 
-  format_function_t *format_fib_result;
-
   /** 1 for ip6; 0 for ip4. */
   u32 is_ip6;
 
   /** Either format_ip4_address_and_length or format_ip6_address_and_length. */
   format_function_t *format_address_and_length;
 
-  /** Special adjacency format functions */
-  format_function_t **special_adjacency_format_functions;
-
   /** Table mapping ip protocol to ip[46]-local node next index. */
   u8 local_next_by_ip_protocol[256];
 
@@ -167,22 +156,6 @@ typedef struct ip_lookup_main_t
   u8 builtin_protocol_by_ip_protocol[256];
 } ip_lookup_main_t;
 
-always_inline ip_adjacency_t *
-ip_get_adjacency (ip_lookup_main_t * lm, u32 adj_index)
-{
-  ip_adjacency_t *adj;
-
-  adj = vec_elt_at_index (lm->adjacency_heap, adj_index);
-
-  return adj;
-}
-
-#define ip_prefetch_adjacency(lm,adj_index,type)		\
-do {								\
-  ip_adjacency_t * _adj = (lm)->adjacency_heap + (adj_index);	\
-  CLIB_PREFETCH (_adj, sizeof (_adj[0]), type);			\
-} while (0)
-
 clib_error_t *ip_interface_address_add_del (ip_lookup_main_t * lm,
 					    u32 sw_if_index,
 					    void *address,
@@ -198,8 +171,6 @@ ip_get_interface_address (ip_lookup_main_t * lm, void *addr_fib)
   return p ? pool_elt_at_index (lm->if_address_pool, p[0]) : 0;
 }
 
-u32 fib_table_id_find_fib_index (fib_protocol_t proto, u32 table_id);
-
 always_inline void *
 ip_interface_address_get_address (ip_lookup_main_t * lm,
 				  ip_interface_address_t * a)
diff --git a/src/vnet/mpls/mpls.c b/src/vnet/mpls/mpls.c
index 451b15cf..a67f47a2 100644
--- a/src/vnet/mpls/mpls.c
+++ b/src/vnet/mpls/mpls.c
@@ -447,8 +447,8 @@ vnet_mpls_local_label (vlib_main_t * vm,
        */
       if (FIB_NODE_INDEX_INVALID == rpath.frp_sw_if_index)
       {
-	  fi = fib_table_id_find_fib_index(dpo_proto_to_fib(pfx.fp_payload_proto),
-					   rpaths[0].frp_fib_index);
+	  fi = fib_table_find(dpo_proto_to_fib(pfx.fp_payload_proto),
+                              rpaths[0].frp_fib_index);
 
 	  if (~0 == fi)
 	  {
diff --git a/src/vnet/sr/sr_localsid.c b/src/vnet/sr/sr_localsid.c
index 6d72a506..32fc5f82 100755
--- a/src/vnet/sr/sr_localsid.c
+++ b/src/vnet/sr/sr_localsid.c
@@ -93,9 +93,9 @@ sr_cli_localsid (char is_del, ip6_address_t * localsid_addr,
 			}
 	  };
 
-	  fib_table_entry_delete (fib_table_id_find_fib_index
-				  (FIB_PROTOCOL_IP6, fib_table), &pfx,
-				  FIB_SOURCE_SR);
+	  fib_table_entry_delete (fib_table_find (FIB_PROTOCOL_IP6,
+						  fib_table),
+				  &pfx, FIB_SOURCE_SR);
 
 	  /* In case it is a Xconnect iface remove the (OIF, NHOP) adj */
 	  if (ls->behavior == SR_BEHAVIOR_X || ls->behavior == SR_BEHAVIOR_DX6
@@ -135,7 +135,7 @@ sr_cli_localsid (char is_del, ip6_address_t * localsid_addr,
   pfx.fp_addr.as_u64[1] = localsid_addr->as_u64[1];
 
   /* Lookup the FIB index associated to the table id provided */
-  u32 fib_index = fib_table_id_find_fib_index (FIB_PROTOCOL_IP6, fib_table);
+  u32 fib_index = fib_table_find (FIB_PROTOCOL_IP6, fib_table);
   if (fib_index == ~0)
     return -3;
 
diff --git a/src/vnet/sr/sr_policy_rewrite.c b/src/vnet/sr/sr_policy_rewrite.c
index 61cf4437..c4024070 100755
--- a/src/vnet/sr/sr_policy_rewrite.c
+++ b/src/vnet/sr/sr_policy_rewrite.c
@@ -376,10 +376,9 @@ update_lb (ip6_sr_policy_t * sr_policy)
 	       load_balance_create (0, DPO_PROTO_IP6, fhc));
 
       /* Update FIB entry's to point to the LB DPO in the main FIB and hidden one */
-      fib_table_entry_special_dpo_update (fib_table_id_find_fib_index
-					  (FIB_PROTOCOL_IP6,
-					   sr_policy->fib_table), &pfx,
-					  FIB_SOURCE_SR,
+      fib_table_entry_special_dpo_update (fib_table_find (FIB_PROTOCOL_IP6,
+							  sr_policy->fib_table),
+					  &pfx, FIB_SOURCE_SR,
 					  FIB_ENTRY_FLAG_EXCLUSIVE,
 					  &sr_policy->bsid_dpo);
 
@@ -470,10 +469,9 @@ update_replicate (ip6_sr_policy_t * sr_policy)
 		    .ip6 = sr_policy->bsid,
 		    }
       };
-      fib_table_entry_special_dpo_update (fib_table_id_find_fib_index
-					  (FIB_PROTOCOL_IP6,
-					   sr_policy->fib_table), &pfx,
-					  FIB_SOURCE_SR,
+      fib_table_entry_special_dpo_update (fib_table_find (FIB_PROTOCOL_IP6,
+							  sr_policy->fib_table),
+					  &pfx, FIB_SOURCE_SR,
 					  FIB_ENTRY_FLAG_EXCLUSIVE,
 					  &sr_policy->bsid_dpo);
 
@@ -568,9 +566,8 @@ sr_policy_add (ip6_address_t * bsid, ip6_address_t * segments,
   };
 
   /* Lookup the FIB index associated to the table selected */
-  u32 fib_index = fib_table_id_find_fib_index (FIB_PROTOCOL_IP6,
-					       (fib_table !=
-						(u32) ~ 0 ? fib_table : 0));
+  u32 fib_index = fib_table_find (FIB_PROTOCOL_IP6,
+				  (fib_table != (u32) ~ 0 ? fib_table : 0));
   if (fib_index == ~0)
     return -13;
 
@@ -653,8 +650,8 @@ sr_policy_del (ip6_address_t * bsid, u32 index)
     ,
   };
 
-  fib_table_entry_special_remove (fib_table_id_find_fib_index
-				  (FIB_PROTOCOL_IP6, sr_policy->fib_table),
+  fib_table_entry_special_remove (fib_table_find (FIB_PROTOCOL_IP6,
+						  sr_policy->fib_table),
 				  &pfx, FIB_SOURCE_SR);
 
   fib_table_entry_special_remove (sm->fib_table_ip6, &pfx, FIB_SOURCE_SR);
diff --git a/src/vnet/sr/sr_steering.c b/src/vnet/sr/sr_steering.c
index 5156b204..04646198 100755
--- a/src/vnet/sr/sr_steering.c
+++ b/src/vnet/sr/sr_steering.c
@@ -115,10 +115,10 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
 	      pfx.fp_len = steer_pl->classify.l3.mask_width;
 	      pfx.fp_addr.ip6 = steer_pl->classify.l3.prefix.ip6;
 
-	      fib_table_entry_delete (fib_table_id_find_fib_index
+	      fib_table_entry_delete (fib_table_find
 				      (FIB_PROTOCOL_IP6,
-				       steer_pl->classify.l3.fib_table), &pfx,
-				      FIB_SOURCE_SR);
+				       steer_pl->classify.l3.fib_table),
+				      &pfx, FIB_SOURCE_SR);
 	    }
 	  else if (steer_pl->classify.traffic_type == SR_STEER_IPV4)
 	    {
@@ -127,7 +127,7 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
 	      pfx.fp_len = steer_pl->classify.l3.mask_width;
 	      pfx.fp_addr.ip4 = steer_pl->classify.l3.prefix.ip4;
 
-	      fib_table_entry_delete (fib_table_id_find_fib_index
+	      fib_table_entry_delete (fib_table_find
 				      (FIB_PROTOCOL_IP4,
 				       steer_pl->classify.l3.fib_table), &pfx,
 				      FIB_SOURCE_SR);
@@ -194,10 +194,10 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
 	      pfx.fp_len = steer_pl->classify.l3.mask_width;
 	      pfx.fp_addr.ip6 = steer_pl->classify.l3.prefix.ip6;
 
-	      fib_table_entry_delete (fib_table_id_find_fib_index
+	      fib_table_entry_delete (fib_table_find
 				      (FIB_PROTOCOL_IP6,
-				       steer_pl->classify.l3.fib_table), &pfx,
-				      FIB_SOURCE_SR);
+				       steer_pl->classify.l3.fib_table),
+				      &pfx, FIB_SOURCE_SR);
 
 	      /* Create a new one */
 	      goto update_fib;
@@ -209,10 +209,10 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
 	      pfx.fp_len = steer_pl->classify.l3.mask_width;
 	      pfx.fp_addr.ip4 = steer_pl->classify.l3.prefix.ip4;
 
-	      fib_table_entry_delete (fib_table_id_find_fib_index
+	      fib_table_entry_delete (fib_table_find
 				      (FIB_PROTOCOL_IP4,
-				       steer_pl->classify.l3.fib_table), &pfx,
-				      FIB_SOURCE_SR);
+				       steer_pl->classify.l3.fib_table),
+				      &pfx, FIB_SOURCE_SR);
 
 	      /* Create a new one */
 	      goto update_fib;
@@ -304,10 +304,10 @@ update_fib:
       pfx.fp_len = steer_pl->classify.l3.mask_width;
       pfx.fp_addr.ip6 = steer_pl->classify.l3.prefix.ip6;
 
-      fib_table_entry_path_add (fib_table_id_find_fib_index (FIB_PROTOCOL_IP6,
-							     (table_id !=
-							      (u32) ~ 0 ?
-							      table_id : 0)),
+      fib_table_entry_path_add (fib_table_find (FIB_PROTOCOL_IP6,
+						(table_id !=
+						 (u32) ~ 0 ?
+						 table_id : 0)),
 				&pfx, FIB_SOURCE_SR,
 				FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT,
 				FIB_PROTOCOL_IP6,
@@ -321,10 +321,10 @@ update_fib:
       pfx.fp_len = steer_pl->classify.l3.mask_width;
       pfx.fp_addr.ip4 = steer_pl->classify.l3.prefix.ip4;
 
-      fib_table_entry_path_add (fib_table_id_find_fib_index (FIB_PROTOCOL_IP4,
-							     (table_id !=
-							      (u32) ~ 0 ?
-							      table_id : 0)),
+      fib_table_entry_path_add (fib_table_find (FIB_PROTOCOL_IP4,
+						(table_id !=
+						 (u32) ~ 0 ?
+						 table_id : 0)),
 				&pfx, FIB_SOURCE_SR,
 				FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT,
 				FIB_PROTOCOL_IP6,
-- 
cgit 1.2.3-korg


From 5d85f2dea9cc77753c68947b6b2ae56101fab9a6 Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Tue, 2 May 2017 10:17:17 -0700
Subject: Allow local/for-us replications for IP multicast routes on the CLI

Change-Id: I046d9402b79fdbc6a6663199ffff8e40a278fd51
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/vnet/ip/lookup.c       | 7 ++++++-
 src/vnet/mfib/mfib_entry.c | 6 ++----
 2 files changed, 8 insertions(+), 5 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 2321c072..b9b5caa5 100755
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -840,7 +840,12 @@ vnet_ip_mroute_cmd (vlib_main_t * vm,
 			 &rpath.frp_sw_if_index))
 	{
 	  rpath.frp_weight = 1;
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	}
+      else if (unformat (line_input, "via local"))
+	{
+	  rpath.frp_sw_if_index = ~0;
+	  rpath.frp_weight = 1;
+	  rpath.frp_flags |= FIB_ROUTE_PATH_LOCAL;
 	}
       else if (unformat (line_input, "%U", unformat_mfib_itf_flags, &iflags))
 	;
diff --git a/src/vnet/mfib/mfib_entry.c b/src/vnet/mfib/mfib_entry.c
index 847f25e7..a9378425 100644
--- a/src/vnet/mfib/mfib_entry.c
+++ b/src/vnet/mfib/mfib_entry.c
@@ -199,14 +199,12 @@ format_mfib_entry (u8 * s, va_list * args)
             {
                 s = fib_path_list_format(msrc->mfes_pl, s);
             }
-            s = format (s, "    Extensions:\n",
-                        mfib_source_names[msrc->mfes_src]);
+            s = format (s, "    Extensions:\n");
             hash_foreach(path_index, mpi, msrc->mfes_exts,
             ({
                 s = format(s, "     %U\n", format_mfib_entry_path_ext, mpi);
             }));
-            s = format (s, "    Interface-Forwarding:\n",
-                        mfib_source_names[msrc->mfes_src]);
+            s = format (s, "    Interface-Forwarding:\n");
             hash_foreach(sw_if_index, mfi, msrc->mfes_itfs,
             ({
                 s = format(s, "    %U\n", format_mfib_itf, mfi);
-- 
cgit 1.2.3-korg


From c7409dc906ebff22d9803940e1ad73a0b79887c4 Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Fri, 19 May 2017 02:54:32 -0700
Subject: mfib CLI bugs (VPP-852)

Change-Id: I1b71010e20742eee02693d672e24c85897e37759
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/vnet/ip/lookup.c     |  2 ++
 src/vnet/mfib/ip4_mfib.c |  9 ++++++---
 src/vnet/mfib/ip6_mfib.c | 13 ++++++++-----
 3 files changed, 16 insertions(+), 8 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index b9b5caa5..6547cad5 100755
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -813,12 +813,14 @@ vnet_ip_mroute_cmd (vlib_main_t * vm,
 			 unformat_ip4_address,
 			 &pfx.fp_grp_addr.ip4, &pfx.fp_len))
 	{
+	  memset (&pfx.fp_src_addr.ip4, 0, sizeof (pfx.fp_src_addr.ip4));
 	  pfx.fp_proto = FIB_PROTOCOL_IP4;
 	}
       else if (unformat (line_input, "%U/%d",
 			 unformat_ip6_address,
 			 &pfx.fp_grp_addr.ip6, &pfx.fp_len))
 	{
+	  memset (&pfx.fp_src_addr.ip6, 0, sizeof (pfx.fp_src_addr.ip6));
 	  pfx.fp_proto = FIB_PROTOCOL_IP6;
 	}
       else if (unformat (line_input, "%U",
diff --git a/src/vnet/mfib/ip4_mfib.c b/src/vnet/mfib/ip4_mfib.c
index 3ed7cba7..1849a3a4 100644
--- a/src/vnet/mfib/ip4_mfib.c
+++ b/src/vnet/mfib/ip4_mfib.c
@@ -382,14 +382,17 @@ ip4_show_mfib (vlib_main_t * vm,
             matching = 1;
             mask = 64;
         }
+        else if (unformat (input, "%U/%d", unformat_ip4_address, &grp, &mask))
+        {
+            memset(&src, 0, sizeof(src));
+            matching = 1;
+        }
         else if (unformat (input, "%U", unformat_ip4_address, &grp))
         {
+            memset(&src, 0, sizeof(src));
             matching = 1;
             mask = 32;
         }
-        else if (unformat (input, "%U/%d",
-                           unformat_ip4_address, &grp, &mask))
-            matching = 1;
         else if (unformat (input, "table %d", &table_id))
             ;
         else if (unformat (input, "index %d", &fib_index))
diff --git a/src/vnet/mfib/ip6_mfib.c b/src/vnet/mfib/ip6_mfib.c
index 116fee22..5c6f8126 100644
--- a/src/vnet/mfib/ip6_mfib.c
+++ b/src/vnet/mfib/ip6_mfib.c
@@ -585,16 +585,19 @@ ip6_show_mfib (vlib_main_t * vm,
                            unformat_ip6_address, &grp))
         {
             matching = 1;
-            mask = 64;
+            mask = 256;
         }
-        else if (unformat (input, "%U", unformat_ip6_address, &grp))
+        else if (unformat (input, "%U/%d", unformat_ip6_address, &grp, &mask))
         {
+            memset(&src, 0, sizeof(src));
             matching = 1;
-            mask = 32;
         }
-        else if (unformat (input, "%U/%d",
-                           unformat_ip6_address, &grp, &mask))
+        else if (unformat (input, "%U", unformat_ip6_address, &grp))
+        {
+            memset(&src, 0, sizeof(src));
             matching = 1;
+            mask = 128;
+        }
         else if (unformat (input, "table %d", &table_id))
             ;
         else if (unformat (input, "index %d", &fib_index))
-- 
cgit 1.2.3-korg


From 57b5860f013953ce161d05302e05370db9cd6ee2 Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Sat, 15 Jul 2017 07:37:25 -0700
Subject: FIB path preference

Paths are given a preference, lowest value is 'best'. Only paths that are up are up contribute to fprwarding - that's unchanged. What's new is that of the path's that re up only those that have the best preference contribute. A poor man's primary and backup. It's not true primary/backup function because the FIB must converge before the lower preference paths are used.

Change-Id: Ie4453c4a7b1094c6c2b51fe1594b8302103bb68e
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/vnet/fib/fib_api.h       |   3 +-
 src/vnet/fib/fib_entry.c     |   2 +-
 src/vnet/fib/fib_entry_src.c |  18 ++
 src/vnet/fib/fib_path.c      |  47 +++-
 src/vnet/fib/fib_path.h      |   3 +-
 src/vnet/fib/fib_path_list.c |  10 +
 src/vnet/fib/fib_test.c      | 572 ++++++++++++++++++++++++++++++++++++++++---
 src/vnet/fib/fib_types.h     |   8 +-
 src/vnet/ip/ip.api           |   5 +-
 src/vnet/ip/ip_api.c         |   6 +-
 src/vnet/ip/lookup.c         |  31 +--
 src/vnet/mpls/mpls.api       |   3 +
 src/vnet/mpls/mpls_api.c     |   1 +
 13 files changed, 647 insertions(+), 62 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet/fib/fib_api.h b/src/vnet/fib/fib_api.h
index 10d0cb58..73d76a42 100644
--- a/src/vnet/fib/fib_api.h
+++ b/src/vnet/fib/fib_api.h
@@ -47,7 +47,8 @@ add_del_route_t_handler (u8 is_multipath,
 			 const ip46_address_t * next_hop,
 			 u32 next_hop_sw_if_index,
 			 u8 next_hop_fib_index,
-			 u32 next_hop_weight,
+			 u16 next_hop_weight,
+			 u16 next_hop_preference,
 			 mpls_label_t next_hop_via_label,
 			 mpls_label_t * next_hop_out_label_stack);
 
diff --git a/src/vnet/fib/fib_entry.c b/src/vnet/fib/fib_entry.c
index 1143f05a..d7ff1c8c 100644
--- a/src/vnet/fib/fib_entry.c
+++ b/src/vnet/fib/fib_entry.c
@@ -1373,7 +1373,7 @@ fib_entry_is_resolved (fib_node_index_t fib_entry_index)
     if (NULL == fed)
     {
         /*
-         * no BFD tracking - resolved
+         * no BFD tracking - consider it resolved.
          */
         return (!0);
     }
diff --git a/src/vnet/fib/fib_entry_src.c b/src/vnet/fib/fib_entry_src.c
index 8c508284..ff73cbf9 100644
--- a/src/vnet/fib/fib_entry_src.c
+++ b/src/vnet/fib/fib_entry_src.c
@@ -193,6 +193,7 @@ typedef struct fib_entry_src_collect_forwarding_ctx_t_
     const fib_entry_src_t *esrc;
     fib_forward_chain_type_t fct;
     int n_recursive_constrained;
+    u16 preference;
 } fib_entry_src_collect_forwarding_ctx_t;
 
 /**
@@ -351,6 +352,22 @@ fib_entry_src_collect_forwarding (fib_node_index_t pl_index,
     {
         ctx->n_recursive_constrained += 1;
     }
+    if (0xffff == ctx->preference)
+    {
+        /*
+         * not set a preference yet, so the first path we encounter
+         * sets the preference we are collecting.
+         */
+        ctx->preference = fib_path_get_preference(path_index);
+    }
+    else if (ctx->preference != fib_path_get_preference(path_index))
+    {
+        /*
+         * this path does not belong to the same preference as the
+         * previous paths encountered. we are done now.
+         */
+        return (FIB_PATH_LIST_WALK_STOP);
+    }
 
     /*
      * get the matching path-extension for the path being visited.
@@ -422,6 +439,7 @@ fib_entry_src_mk_lb (fib_entry_t *fib_entry,
         .next_hops = NULL,
         .n_recursive_constrained = 0,
         .fct = fct,
+        .preference = 0xffff,
     };
 
     /*
diff --git a/src/vnet/fib/fib_path.c b/src/vnet/fib/fib_path.c
index 274b0ef4..a32baf21 100644
--- a/src/vnet/fib/fib_path.c
+++ b/src/vnet/fib/fib_path.c
@@ -196,9 +196,15 @@ typedef struct fib_path_t_ {
     fib_protocol_t fp_nh_proto;
 
     /**
-     * UCMP [unnormalised] weigt
+     * UCMP [unnormalised] weigth
      */
-    u32 fp_weight;
+    u16 fp_weight;
+    /**
+     * A path preference. 0 is the best.
+     * Only paths of the best preference, that are 'up', are considered
+     * for forwarding.
+     */
+    u16 fp_preference;
 
     /**
      * per-type union of the data required to resolve the path
@@ -376,6 +382,7 @@ format_fib_path (u8 * s, va_list * args)
     s = format (s, "pl-index:%d ", path->fp_pl_index);
     s = format (s, "%U ", format_fib_protocol, path->fp_nh_proto);
     s = format (s, "weight=%d ", path->fp_weight);
+    s = format (s, "pref=%d ", path->fp_preference);
     s = format (s, "%s: ", fib_path_type_names[path->fp_type]);
     if (FIB_PATH_OPER_FLAG_NONE != path->fp_oper_flags) {
 	s = format(s, " oper-flags:");
@@ -699,6 +706,14 @@ fib_path_recursive_adj_update (fib_path_t *path,
         load_balance_map_path_state_change(fib_path_get_index(path));
     }
 
+    /*
+     * If this path is contributing a drop, then it's not resolved
+     */
+    if (dpo_is_drop(&via_dpo) || load_balance_is_drop(&via_dpo))
+    {
+        path->fp_oper_flags &= ~FIB_PATH_OPER_FLAG_RESOLVED;
+    }
+
     /*
      * update the path's contributed DPO
      */
@@ -1071,6 +1086,7 @@ fib_path_create (fib_node_index_t pl_index,
          */
         path->fp_weight = 1;
     }
+    path->fp_preference = rpath->frp_preference;
     path->fp_cfg_flags = fib_path_route_flags_to_cfg_flags(rpath);
 
     /*
@@ -1165,6 +1181,7 @@ fib_path_create_special (fib_node_index_t pl_index,
 
     path->fp_pl_index = pl_index;
     path->fp_weight = 1;
+    path->fp_preference = 0;
     path->fp_nh_proto = nh_proto;
     path->fp_via_fib = FIB_NODE_INDEX_INVALID;
     path->fp_cfg_flags = flags;
@@ -1275,7 +1292,7 @@ fib_path_cmp_i (const fib_path_t *path1,
 
     /*
      * paths of different types and protocol are not equal.
-     * different weights only are the same path.
+     * different weights and/or preference only are the same path.
      */
     if (path1->fp_type != path2->fp_type)
     {
@@ -1350,6 +1367,15 @@ fib_path_cmp_for_sort (void * v1,
     path1 = fib_path_get(*pi1);
     path2 = fib_path_get(*pi2);
 
+    /*
+     * when sorting paths we want the highest preference paths
+     * first, so that the choices set built is in prefernce order
+     */
+    if (path1->fp_preference != path2->fp_preference)
+    {
+	return (path1->fp_preference - path2->fp_preference);
+    }
+
     return (fib_path_cmp_i(path1, path2));
 }
 
@@ -1750,7 +1776,7 @@ fib_path_get_adj (fib_node_index_t path_index)
     return (ADJ_INDEX_INVALID);
 }
 
-int
+u16
 fib_path_get_weight (fib_node_index_t path_index)
 {
     fib_path_t *path;
@@ -1762,6 +1788,18 @@ fib_path_get_weight (fib_node_index_t path_index)
     return (path->fp_weight);
 }
 
+u16
+fib_path_get_preference (fib_node_index_t path_index)
+{
+    fib_path_t *path;
+
+    path = fib_path_get(path_index);
+
+    ASSERT(path);
+
+    return (path->fp_preference);
+}
+
 /**
  * @brief Contribute the path's adjacency to the list passed.
  * By calling this function over all paths, recursively, a child
@@ -2116,6 +2154,7 @@ fib_path_encode (fib_node_index_t path_list_index,
       return (FIB_PATH_LIST_WALK_CONTINUE);
     vec_add2(*api_rpaths, api_rpath, 1);
     api_rpath->rpath.frp_weight = path->fp_weight;
+    api_rpath->rpath.frp_preference = path->fp_preference;
     api_rpath->rpath.frp_proto = path->fp_nh_proto;
     api_rpath->rpath.frp_sw_if_index = ~0;
     api_rpath->dpo = path->exclusive.fp_ex_dpo;
diff --git a/src/vnet/fib/fib_path.h b/src/vnet/fib/fib_path.h
index 0b5e6072..a34cb43f 100644
--- a/src/vnet/fib/fib_path.h
+++ b/src/vnet/fib/fib_path.h
@@ -167,7 +167,8 @@ extern adj_index_t fib_path_get_adj(fib_node_index_t path_index);
 extern int fib_path_recursive_loop_detect(fib_node_index_t path_index,
 					  fib_node_index_t **entry_indicies);
 extern u32 fib_path_get_resolving_interface(fib_node_index_t fib_entry_index);
-extern int fib_path_get_weight(fib_node_index_t path_index);
+extern u16 fib_path_get_weight(fib_node_index_t path_index);
+extern u16 fib_path_get_preference(fib_node_index_t path_index);
 
 extern void fib_path_module_init(void);
 extern fib_path_list_walk_rc_t fib_path_encode(fib_node_index_t path_list_index,
diff --git a/src/vnet/fib/fib_path_list.c b/src/vnet/fib/fib_path_list.c
index 3e4c3333..7a9c328c 100644
--- a/src/vnet/fib/fib_path_list.c
+++ b/src/vnet/fib/fib_path_list.c
@@ -680,6 +680,16 @@ fib_path_list_create (fib_path_list_flags_t flags,
                      fib_path_create(path_list_index,
                                      &rpaths[i]));
         }
+        /*
+         * we sort the paths since the key for the path-list is
+         * the description of the paths it contains. The paths need to
+         * be sorted else this description will differ.
+         */
+        if (vec_len(path_list->fpl_paths) > 1)
+        {
+            vec_sort_with_function(path_list->fpl_paths,
+                                   fib_path_cmp_for_sort);
+        }
     }
 
     /*
diff --git a/src/vnet/fib/fib_test.c b/src/vnet/fib/fib_test.c
index 06aeda62..720f54e7 100644
--- a/src/vnet/fib/fib_test.c
+++ b/src/vnet/fib/fib_test.c
@@ -604,9 +604,10 @@ fib_test_validate_lb_v (const load_balance_t *lb,
                        bucket,
                        format_dpo_type, dpo->dpoi_type);
 	    FIB_TEST_LB((exp->lb.lb == dpo->dpoi_index),
-			"bucket %d stacks on lb %d",
+			"bucket %d stacks on lb %d not %d",
 			bucket,
-			exp->lb.lb);
+			dpo->dpoi_index,
+                        exp->lb.lb);
 	    break;
 	case FT_LB_SPECIAL:
 	    FIB_TEST_I((DPO_DROP == dpo->dpoi_type),
@@ -1482,19 +1483,20 @@ fib_test_v4 (void)
 	},
     };
 
-    fib_table_entry_path_add(fib_index,
-			     &bgp_200_pfx,
-			     FIB_SOURCE_API,
-			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
-			     &pfx_1_1_1_2_s_32.fp_addr,
-			     ~0, // no index provided.
-			     fib_index, // nexthop in same fib as route
-			     1,
-			     NULL,
-			     FIB_ROUTE_PATH_FLAG_NONE);
+    fei = fib_table_entry_path_add(fib_index,
+                                   &bgp_200_pfx,
+                                   FIB_SOURCE_API,
+                                   FIB_ENTRY_FLAG_NONE,
+                                   FIB_PROTOCOL_IP4,
+                                   &pfx_1_1_1_2_s_32.fp_addr,
+                                   ~0, // no index provided.
+                                   fib_index, // nexthop in same fib as route
+                                   1,
+                                   NULL,
+                                   FIB_ROUTE_PATH_FLAG_NONE);
 
-    FIB_TEST_REC_FORW(&bgp_200_pfx, &pfx_1_1_1_2_s_32, 0);
+    FIB_TEST(load_balance_is_drop(fib_entry_contribute_ip_forwarding(fei)),
+             "Recursive via unresolved is drop");
 
     /*
      * the adj should be recursive via drop, since the route resolves via
@@ -2101,19 +2103,20 @@ fib_test_v4 (void)
 	},
     };
 
-    fib_table_entry_path_add(fib_index,
-			     &bgp_201_pfx,
-			     FIB_SOURCE_API,
-			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
-			     &pfx_1_1_1_200_s_32.fp_addr,
-			     ~0, // no index provided.
-			     fib_index, // nexthop in same fib as route
-			     1,
-			     NULL,
-			     FIB_ROUTE_PATH_FLAG_NONE);
+    fei = fib_table_entry_path_add(fib_index,
+                                   &bgp_201_pfx,
+                                   FIB_SOURCE_API,
+                                   FIB_ENTRY_FLAG_NONE,
+                                   FIB_PROTOCOL_IP4,
+                                   &pfx_1_1_1_200_s_32.fp_addr,
+                                   ~0, // no index provided.
+                                   fib_index, // nexthop in same fib as route
+                                   1,
+                                   NULL,
+                                   FIB_ROUTE_PATH_FLAG_NONE);
 
-    FIB_TEST_REC_FORW(&bgp_201_pfx, &pfx_1_1_1_200_s_32, 0);
+    FIB_TEST(load_balance_is_drop(fib_entry_contribute_ip_forwarding(fei)),
+             "Recursive via unresolved is drop");
 
     fei = fib_table_lookup_exact_match(fib_index, &pfx_1_1_1_200_s_32);
     FIB_TEST((FIB_ENTRY_FLAG_NONE == fib_entry_get_flags(fei)),
@@ -2285,12 +2288,16 @@ fib_test_v4 (void)
     fei = fib_table_lookup(fib_index, &pfx_1_1_1_2_s_32);
     FIB_TEST(load_balance_is_drop(fib_entry_contribute_ip_forwarding(fei)),
 	     "1.1.1.2/32 route is DROP");
-    fei = fib_table_lookup(fib_index, &pfx_1_1_1_200_s_32);
+    fei = fib_table_lookup_exact_match(fib_index, &pfx_1_1_1_200_s_32);
     FIB_TEST(load_balance_is_drop(fib_entry_contribute_ip_forwarding(fei)),
 	     "1.1.1.200/32 route is DROP");
 
-    FIB_TEST_REC_FORW(&bgp_201_pfx, &pfx_1_1_1_200_s_32, 0);
-    FIB_TEST_REC_FORW(&bgp_200_pfx, &pfx_1_1_1_2_s_32, 0);
+    fei = fib_table_lookup_exact_match(fib_index, &bgp_201_pfx);
+    FIB_TEST(load_balance_is_drop(fib_entry_contribute_ip_forwarding(fei)),
+             "201 is drop");
+    fei = fib_table_lookup_exact_match(fib_index, &bgp_200_pfx);
+    FIB_TEST(load_balance_is_drop(fib_entry_contribute_ip_forwarding(fei)),
+             "200 is drop");
 
     /*
      * -1 entry
@@ -2320,7 +2327,9 @@ fib_test_v4 (void)
     ai = fib_entry_get_adj(fei);
     FIB_TEST((ai = ai_01), "1.1.1.2/32 resolves via 10.10.10.1");
 
-    FIB_TEST_REC_FORW(&bgp_201_pfx, &pfx_1_1_1_200_s_32, 0);
+    fei = fib_table_lookup_exact_match(fib_index, &bgp_201_pfx);
+    FIB_TEST(load_balance_is_drop(fib_entry_contribute_ip_forwarding(fei)),
+             "201 is drop");
     FIB_TEST_REC_FORW(&bgp_200_pfx, &pfx_1_1_1_2_s_32, 0);
 
     /*
@@ -2333,6 +2342,33 @@ fib_test_v4 (void)
     FIB_TEST((ENBR+12 == fib_entry_pool_size()), "entry pool size is %d",
 	     fib_entry_pool_size());
 
+    /*
+     * give 201 a resolved path.
+     *  it now has the unresolved 1.1.1.200 and the resolved 1.1.1.2,
+     *  only the latter contributes forwarding.
+     */
+    fei = fib_table_entry_path_add(fib_index,
+                                   &bgp_201_pfx,
+                                   FIB_SOURCE_API,
+                                   FIB_ENTRY_FLAG_NONE,
+                                   FIB_PROTOCOL_IP4,
+                                   &pfx_1_1_1_2_s_32.fp_addr,
+                                   ~0,
+                                   fib_index,
+                                   1,
+                                   NULL,
+                                   FIB_ROUTE_PATH_FLAG_NONE);
+    FIB_TEST_REC_FORW(&bgp_201_pfx, &pfx_1_1_1_2_s_32, 0);
+    fib_table_entry_path_remove(fib_index,
+                                &bgp_201_pfx,
+                                FIB_SOURCE_API,
+                                FIB_PROTOCOL_IP4,
+                                &pfx_1_1_1_2_s_32.fp_addr,
+                                ~0,
+                                fib_index,
+                                1,
+                                FIB_ROUTE_PATH_FLAG_NONE);
+
     /*
      * remove 200.200.200.201/32 which does not have a valid via FIB
      */
@@ -3608,6 +3644,17 @@ fib_test_v4 (void)
 	    .ip4.as_u32 = clib_host_to_net_u32(0x22220101),
 	},
     };
+    fei = fib_table_entry_path_add(fib_index,
+                                   &pfx_34_34_1_1_s_32,
+                                   FIB_SOURCE_API,
+                                   FIB_ENTRY_FLAG_NONE,
+                                   FIB_PROTOCOL_IP4,
+                                   &nh_10_10_10_1,
+                                   tm->hw[0]->sw_if_index,
+                                   0,
+                                   1,
+                                   NULL,
+                                   FIB_ROUTE_PATH_FLAG_NONE);
     fei = fib_table_entry_path_add(fib_index,
                                    &pfx_34_1_1_1_s_32,
                                    FIB_SOURCE_API,
@@ -3632,6 +3679,9 @@ fib_test_v4 (void)
                                    FIB_ROUTE_PATH_FLAG_NONE);
     FIB_TEST_REC_FORW(&pfx_34_1_1_1_s_32, &pfx_34_34_1_1_s_32, 0);
     fib_table_entry_delete_index(fei, FIB_SOURCE_API);
+    fib_table_entry_delete(fib_index,
+                           &pfx_34_34_1_1_s_32,
+                           FIB_SOURCE_API);
 
     /*
      * CLEANUP
@@ -5554,6 +5604,459 @@ fib_test_ae (void)
     return (0);
 }
 
+/*
+ * Test Path Preference
+ */
+static int
+fib_test_pref (void)
+{
+    test_main_t *tm = &test_main;
+
+    const fib_prefix_t pfx_1_1_1_1_s_32 = {
+        .fp_len = 32,
+        .fp_proto = FIB_PROTOCOL_IP4,
+        .fp_addr = {
+            .ip4 = {
+                .as_u32 = clib_host_to_net_u32(0x01010101),
+            },
+        },
+    };
+
+    /*
+     * 2 high, 2 medium and 2 low preference non-recursive paths
+     */
+    fib_route_path_t nr_path_hi_1 = {
+        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_sw_if_index = tm->hw[0]->sw_if_index,
+        .frp_fib_index = ~0,
+        .frp_weight = 1,
+        .frp_preference = 0,
+        .frp_flags = FIB_ROUTE_PATH_FLAG_NONE,
+        .frp_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0x0a0a0a01),
+        },
+    };
+    fib_route_path_t nr_path_hi_2 = {
+        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_sw_if_index = tm->hw[0]->sw_if_index,
+        .frp_fib_index = ~0,
+        .frp_weight = 1,
+        .frp_preference = 0,
+        .frp_flags = FIB_ROUTE_PATH_FLAG_NONE,
+        .frp_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0x0a0a0a02),
+        },
+    };
+    fib_route_path_t nr_path_med_1 = {
+        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_sw_if_index = tm->hw[1]->sw_if_index,
+        .frp_fib_index = ~0,
+        .frp_weight = 1,
+        .frp_preference = 1,
+        .frp_flags = FIB_ROUTE_PATH_FLAG_NONE,
+        .frp_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0x0a0a0c01),
+        },
+    };
+    fib_route_path_t nr_path_med_2 = {
+        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_sw_if_index = tm->hw[1]->sw_if_index,
+        .frp_fib_index = ~0,
+        .frp_weight = 1,
+        .frp_preference = 1,
+        .frp_flags = FIB_ROUTE_PATH_FLAG_NONE,
+        .frp_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0x0a0a0c01),
+        },
+    };
+    fib_route_path_t nr_path_low_1 = {
+        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_sw_if_index = tm->hw[2]->sw_if_index,
+        .frp_fib_index = ~0,
+        .frp_weight = 1,
+        .frp_preference = 2,
+        .frp_flags = FIB_ROUTE_PATH_FLAG_NONE,
+        .frp_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0x0a0a0b01),
+        },
+    };
+    fib_route_path_t nr_path_low_2 = {
+        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_sw_if_index = tm->hw[2]->sw_if_index,
+        .frp_fib_index = ~0,
+        .frp_weight = 1,
+        .frp_preference = 2,
+        .frp_flags = FIB_ROUTE_PATH_FLAG_NONE,
+        .frp_addr = {
+            .ip4.as_u32 = clib_host_to_net_u32(0x0a0a0b02),
+        },
+    };
+    fib_route_path_t *nr_paths = NULL;
+
+    vec_add1(nr_paths, nr_path_hi_1);
+    vec_add1(nr_paths, nr_path_hi_2);
+    vec_add1(nr_paths, nr_path_med_1);
+    vec_add1(nr_paths, nr_path_med_2);
+    vec_add1(nr_paths, nr_path_low_1);
+    vec_add1(nr_paths, nr_path_low_2);
+
+    adj_index_t ai_hi_1 = adj_nbr_add_or_lock(FIB_PROTOCOL_IP4,
+                                              VNET_LINK_IP4,
+                                              &nr_path_hi_1.frp_addr,
+                                              nr_path_hi_1.frp_sw_if_index);
+    adj_index_t ai_hi_2 = adj_nbr_add_or_lock(FIB_PROTOCOL_IP4,
+                                              VNET_LINK_IP4,
+                                              &nr_path_hi_2.frp_addr,
+                                              nr_path_hi_2.frp_sw_if_index);
+    adj_index_t ai_med_1 = adj_nbr_add_or_lock(FIB_PROTOCOL_IP4,
+                                               VNET_LINK_IP4,
+                                               &nr_path_med_1.frp_addr,
+                                               nr_path_med_1.frp_sw_if_index);
+    adj_index_t ai_med_2 = adj_nbr_add_or_lock(FIB_PROTOCOL_IP4,
+                                               VNET_LINK_IP4,
+                                               &nr_path_med_2.frp_addr,
+                                               nr_path_med_2.frp_sw_if_index);
+    adj_index_t ai_low_1 = adj_nbr_add_or_lock(FIB_PROTOCOL_IP4,
+                                               VNET_LINK_IP4,
+                                               &nr_path_low_1.frp_addr,
+                                               nr_path_low_1.frp_sw_if_index);
+    adj_index_t ai_low_2 = adj_nbr_add_or_lock(FIB_PROTOCOL_IP4,
+                                               VNET_LINK_IP4,
+                                               &nr_path_low_2.frp_addr,
+                                               nr_path_low_2.frp_sw_if_index);
+
+    fib_test_lb_bucket_t ip_hi_1 = {
+        .type = FT_LB_ADJ,
+        .adj = {
+            .adj = ai_hi_1,
+        },
+    };
+    fib_test_lb_bucket_t ip_hi_2 = {
+        .type = FT_LB_ADJ,
+        .adj = {
+            .adj = ai_hi_2,
+        },
+    };
+    fib_test_lb_bucket_t ip_med_1 = {
+        .type = FT_LB_ADJ,
+        .adj = {
+            .adj = ai_med_1,
+        },
+    };
+    fib_test_lb_bucket_t ip_med_2 = {
+        .type = FT_LB_ADJ,
+        .adj = {
+            .adj = ai_med_2,
+        },
+    };
+    fib_test_lb_bucket_t ip_low_1 = {
+        .type = FT_LB_ADJ,
+        .adj = {
+            .adj = ai_low_1,
+        },
+    };
+    fib_test_lb_bucket_t ip_low_2 = {
+        .type = FT_LB_ADJ,
+        .adj = {
+            .adj = ai_low_2,
+        },
+    };
+
+    fib_node_index_t fei;
+
+    fei = fib_table_entry_path_add2(0,
+                                    &pfx_1_1_1_1_s_32,
+                                    FIB_SOURCE_API,
+                                    FIB_ENTRY_FLAG_NONE,
+                                    nr_paths);
+
+    FIB_TEST(fib_test_validate_entry(fei,
+                                     FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                     2,
+                                     &ip_hi_1,
+                                     &ip_hi_2),
+             "1.1.1.1/32 via high preference paths");
+
+    /*
+     * bring down the interface on which the high preference path lie
+     */
+    vnet_sw_interface_set_flags(vnet_get_main(),
+                                tm->hw[0]->sw_if_index,
+                                0);
+
+    FIB_TEST(fib_test_validate_entry(fei,
+                                     FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                     2,
+                                     &ip_med_1,
+                                     &ip_med_2),
+             "1.1.1.1/32 via medium preference paths");
+
+    /*
+     * bring down the interface on which the medium preference path lie
+     */
+    vnet_sw_interface_set_flags(vnet_get_main(),
+                                tm->hw[1]->sw_if_index,
+                                0);
+
+    FIB_TEST(fib_test_validate_entry(fei,
+                                     FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                     2,
+                                     &ip_low_1,
+                                     &ip_low_2),
+             "1.1.1.1/32 via low preference paths");
+
+    /*
+     * bring up the interface on which the high preference path lie
+     */
+    vnet_sw_interface_set_flags(vnet_get_main(),
+                                tm->hw[0]->sw_if_index,
+                                VNET_SW_INTERFACE_FLAG_ADMIN_UP);
+
+    FIB_TEST(fib_test_validate_entry(fei,
+                                     FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                     2,
+                                     &ip_hi_1,
+                                     &ip_hi_2),
+             "1.1.1.1/32 via high preference paths");
+
+    /*
+     * bring up the interface on which the medium preference path lie
+     */
+    vnet_sw_interface_set_flags(vnet_get_main(),
+                                tm->hw[1]->sw_if_index,
+                                VNET_SW_INTERFACE_FLAG_ADMIN_UP);
+
+    FIB_TEST(fib_test_validate_entry(fei,
+                                     FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                     2,
+                                     &ip_hi_1,
+                                     &ip_hi_2),
+             "1.1.1.1/32 via high preference paths");
+
+    dpo_id_t ip_1_1_1_1 = DPO_INVALID;
+    fib_entry_contribute_forwarding(fei,
+                                    FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                    &ip_1_1_1_1);
+
+    /*
+     * 3 recursive paths of different preference
+     */
+    const fib_prefix_t pfx_1_1_1_2_s_32 = {
+        .fp_len = 32,
+        .fp_proto = FIB_PROTOCOL_IP4,
+        .fp_addr = {
+            .ip4 = {
+                .as_u32 = clib_host_to_net_u32(0x01010102),
+            },
+        },
+    };
+    const fib_prefix_t pfx_1_1_1_3_s_32 = {
+        .fp_len = 32,
+        .fp_proto = FIB_PROTOCOL_IP4,
+        .fp_addr = {
+            .ip4 = {
+                .as_u32 = clib_host_to_net_u32(0x01010103),
+            },
+        },
+    };
+    fei = fib_table_entry_path_add2(0,
+                                    &pfx_1_1_1_2_s_32,
+                                    FIB_SOURCE_API,
+                                    FIB_ENTRY_FLAG_NONE,
+                                    nr_paths);
+    dpo_id_t ip_1_1_1_2 = DPO_INVALID;
+    fib_entry_contribute_forwarding(fei,
+                                    FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                    &ip_1_1_1_2);
+    fei = fib_table_entry_path_add2(0,
+                                    &pfx_1_1_1_3_s_32,
+                                    FIB_SOURCE_API,
+                                    FIB_ENTRY_FLAG_NONE,
+                                    nr_paths);
+    dpo_id_t ip_1_1_1_3 = DPO_INVALID;
+    fib_entry_contribute_forwarding(fei,
+                                    FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                    &ip_1_1_1_3);
+
+    fib_test_lb_bucket_t ip_o_1_1_1_1 = {
+        .type = FT_LB_O_LB,
+        .lb = {
+            .lb = ip_1_1_1_1.dpoi_index,
+        },
+    };
+    fib_test_lb_bucket_t ip_o_1_1_1_2 = {
+        .type = FT_LB_O_LB,
+        .lb = {
+            .lb = ip_1_1_1_2.dpoi_index,
+        },
+    };
+    fib_test_lb_bucket_t ip_o_1_1_1_3 = {
+        .type = FT_LB_O_LB,
+        .lb = {
+            .lb = ip_1_1_1_3.dpoi_index,
+        },
+    };
+    fib_route_path_t r_path_hi = {
+        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_sw_if_index = ~0,
+        .frp_fib_index = 0,
+        .frp_weight = 1,
+        .frp_preference = 0,
+        .frp_flags = FIB_ROUTE_PATH_RESOLVE_VIA_HOST,
+        .frp_addr = pfx_1_1_1_1_s_32.fp_addr,
+    };
+    fib_route_path_t r_path_med = {
+        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_sw_if_index = ~0,
+        .frp_fib_index = 0,
+        .frp_weight = 1,
+        .frp_preference = 10,
+        .frp_flags = FIB_ROUTE_PATH_FLAG_NONE,
+        .frp_addr = pfx_1_1_1_2_s_32.fp_addr,
+    };
+    fib_route_path_t r_path_low = {
+        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_sw_if_index = ~0,
+        .frp_fib_index = 0,
+        .frp_weight = 1,
+        .frp_preference = 1000,
+        .frp_flags = FIB_ROUTE_PATH_RESOLVE_VIA_HOST,
+        .frp_addr = pfx_1_1_1_3_s_32.fp_addr,
+    };
+    fib_route_path_t *r_paths = NULL;
+
+    vec_add1(r_paths, r_path_hi);
+    vec_add1(r_paths, r_path_low);
+    vec_add1(r_paths, r_path_med);
+
+    /*
+     * add many recursive so we get the LB MAp created
+     */
+    #define N_PFXS 64
+    fib_prefix_t pfx_r[N_PFXS];
+    uint32_t n_pfxs;
+    for (n_pfxs = 0; n_pfxs < N_PFXS; n_pfxs++)
+    {
+        pfx_r[n_pfxs].fp_len = 32;
+        pfx_r[n_pfxs].fp_proto = FIB_PROTOCOL_IP4;
+        pfx_r[n_pfxs].fp_addr.ip4.as_u32 =
+            clib_host_to_net_u32(0x02000000 + n_pfxs);
+
+        fei = fib_table_entry_path_add2(0,
+                                        &pfx_r[n_pfxs],
+                                        FIB_SOURCE_API,
+                                        FIB_ENTRY_FLAG_NONE,
+                                        r_paths);
+
+        FIB_TEST(fib_test_validate_entry(fei,
+                                         FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                         1,
+                                         &ip_o_1_1_1_1),
+                 "recursive via high preference paths");
+
+        /*
+         * withdraw hig pref resolving entry
+         */
+        fib_table_entry_delete(0,
+                               &pfx_1_1_1_1_s_32,
+                               FIB_SOURCE_API);
+
+        /* suspend so the update walk kicks int */
+        vlib_process_suspend(vlib_get_main(), 1e-5);
+
+        FIB_TEST(fib_test_validate_entry(fei,
+                                         FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                         1,
+                                         &ip_o_1_1_1_2),
+                 "recursive via medium preference paths");
+
+        /*
+         * withdraw medium pref resolving entry
+         */
+        fib_table_entry_delete(0,
+                               &pfx_1_1_1_2_s_32,
+                               FIB_SOURCE_API);
+
+        /* suspend so the update walk kicks int */
+        vlib_process_suspend(vlib_get_main(), 1e-5);
+
+        FIB_TEST(fib_test_validate_entry(fei,
+                                         FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                         1,
+                                         &ip_o_1_1_1_3),
+                 "recursive via low preference paths");
+
+        /*
+         * add back paths for next iteration
+         */
+        fei = fib_table_entry_update(0,
+                                     &pfx_1_1_1_2_s_32,
+                                     FIB_SOURCE_API,
+                                     FIB_ENTRY_FLAG_NONE,
+                                     nr_paths);
+        fei = fib_table_entry_update(0,
+                                     &pfx_1_1_1_1_s_32,
+                                     FIB_SOURCE_API,
+                                     FIB_ENTRY_FLAG_NONE,
+                                     nr_paths);
+
+        /* suspend so the update walk kicks int */
+        vlib_process_suspend(vlib_get_main(), 1e-5);
+
+        fei = fib_table_lookup_exact_match(0, &pfx_r[n_pfxs]);
+        FIB_TEST(fib_test_validate_entry(fei,
+                                         FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                         1,
+                                         &ip_o_1_1_1_1),
+                 "recursive via high preference paths");
+    }
+
+
+    fib_table_entry_delete(0,
+                           &pfx_1_1_1_1_s_32,
+                           FIB_SOURCE_API);
+
+    /* suspend so the update walk kicks int */
+    vlib_process_suspend(vlib_get_main(), 1e-5);
+
+    for (n_pfxs = 0; n_pfxs < N_PFXS; n_pfxs++)
+    {
+        fei = fib_table_lookup_exact_match(0, &pfx_r[n_pfxs]);
+
+        FIB_TEST(fib_test_validate_entry(fei,
+                                         FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
+                                         1,
+                                         &ip_o_1_1_1_2),
+                 "recursive via medium preference paths");
+    }
+    for (n_pfxs = 0; n_pfxs < N_PFXS; n_pfxs++)
+    {
+        fib_table_entry_delete(0,
+                               &pfx_r[n_pfxs],
+                               FIB_SOURCE_API);
+    }
+
+    /*
+     * Cleanup
+     */
+    fib_table_entry_delete(0,
+                           &pfx_1_1_1_2_s_32,
+                           FIB_SOURCE_API);
+    fib_table_entry_delete(0,
+                           &pfx_1_1_1_3_s_32,
+                           FIB_SOURCE_API);
+
+    dpo_reset(&ip_1_1_1_1);
+    dpo_reset(&ip_1_1_1_2);
+    dpo_reset(&ip_1_1_1_3);
+    adj_unlock(ai_low_2);
+    adj_unlock(ai_low_1);
+    adj_unlock(ai_med_2);
+    adj_unlock(ai_med_1);
+    adj_unlock(ai_hi_2);
+    adj_unlock(ai_hi_1);
+    return (0);
+}
 
 /*
  * Test the recursive route route handling for GRE tunnels
@@ -7996,8 +8499,8 @@ lfib_test (void)
     FIB_TEST(fib_test_validate_entry(lfe,
     				     FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
     				     1,
-    				     &ip_o_1200),
-    	     "2.2.2.2.4/32 LB 1 buckets via: label 1200 EOS");
+    				     &bucket_drop),
+    	     "2.2.2.2.4/32 LB 1 buckets via: drop");
     lfe = fib_table_lookup(fib_index, &pfx_1200);
     FIB_TEST(fib_test_validate_entry(lfe,
 				     FIB_FORW_CHAIN_TYPE_UNICAST_IP4,
@@ -8199,6 +8702,10 @@ fib_test (vlib_main_t * vm,
     {
 	res += fib_test_ae();
     }
+    else if (unformat (input, "pref"))
+    {
+	res += fib_test_pref();
+    }
     else if (unformat (input, "lfib"))
     {
 	res += lfib_test();
@@ -8217,6 +8724,7 @@ fib_test (vlib_main_t * vm,
 	res += fib_test_v6();
 	res += fib_test_ae();
 	res += fib_test_bfd();
+	res += fib_test_pref();
 	res += fib_test_label();
 	res += lfib_test();
 
diff --git a/src/vnet/fib/fib_types.h b/src/vnet/fib/fib_types.h
index 60873c4a..a7a23d74 100644
--- a/src/vnet/fib/fib_types.h
+++ b/src/vnet/fib/fib_types.h
@@ -380,7 +380,13 @@ typedef struct fib_route_path_t_ {
     /**
      * [un]equal cost path weight
      */
-    u32 frp_weight;
+    u16 frp_weight;
+    /**
+     * A path preference. 0 is the best.
+     * Only paths of the best preference, that are 'up', are considered
+     * for forwarding.
+     */
+    u16 frp_preference;
     /**
      * flags on the path
      */
diff --git a/src/vnet/ip/ip.api b/src/vnet/ip/ip.api
index 7097a130..fa363370 100644
--- a/src/vnet/ip/ip.api
+++ b/src/vnet/ip/ip.api
@@ -31,6 +31,7 @@ define ip_fib_dump
 /** \brief FIB path
     @param sw_if_index - index of the interface
     @param weight - The weight, for UCMP
+    @param preference - The preference of the path. lowest preference is prefered
     @param is_local - local if non-zero, else remote
     @param is_drop - Drop the packet
     @param is_unreach - Drop the packet and rate limit send ICMP unreachable
@@ -43,7 +44,8 @@ define ip_fib_dump
 typeonly manual_print manual_endian define fib_path
 {
   u32 sw_if_index;
-  u32 weight;
+  u8 weight;
+  u8 preference;
   u8 is_local;
   u8 is_drop;
   u8 is_unreach;
@@ -374,6 +376,7 @@ autoreply define ip_add_del_route
   /* Is last/not-last message in group of multiple add/del messages. */
   u8 not_last;
   u8 next_hop_weight;
+  u8 next_hop_preference;
   u8 dst_address_length;
   u8 dst_address[16];
   u8 next_hop_address[16];
diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c
index 2680d601..d00ae76c 100644
--- a/src/vnet/ip/ip_api.c
+++ b/src/vnet/ip/ip_api.c
@@ -716,7 +716,8 @@ add_del_route_t_handler (u8 is_multipath,
 			 const ip46_address_t * next_hop,
 			 u32 next_hop_sw_if_index,
 			 u8 next_hop_fib_index,
-			 u32 next_hop_weight,
+			 u16 next_hop_weight,
+			 u16 next_hop_preference,
 			 mpls_label_t next_hop_via_label,
 			 mpls_label_t * next_hop_out_label_stack)
 {
@@ -729,6 +730,7 @@ add_del_route_t_handler (u8 is_multipath,
     .frp_sw_if_index = next_hop_sw_if_index,
     .frp_fib_index = next_hop_fib_index,
     .frp_weight = next_hop_weight,
+    .frp_preference = next_hop_preference,
     .frp_label_stack = next_hop_out_label_stack,
   };
   fib_route_path_t *paths = NULL;
@@ -971,6 +973,7 @@ ip4_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
 				   ntohl (mp->next_hop_sw_if_index),
 				   next_hop_fib_index,
 				   mp->next_hop_weight,
+				   mp->next_hop_preference,
 				   ntohl (mp->next_hop_via_label),
 				   label_stack));
 }
@@ -1029,6 +1032,7 @@ ip6_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
 				   &nh, ntohl (mp->next_hop_sw_if_index),
 				   next_hop_fib_index,
 				   mp->next_hop_weight,
+				   mp->next_hop_preference,
 				   ntohl (mp->next_hop_via_label),
 				   label_stack));
 }
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 6547cad5..533d010a 100755
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -360,6 +360,7 @@ vnet_ip_route_cmd (vlib_main_t * vm,
   fib_prefix_t *prefixs = NULL, pfx;
   mpls_label_t out_label, via_label;
   clib_error_t *error = NULL;
+  u32 weight, preference;
   u32 table_id, is_del;
   vnet_main_t *vnm;
   u32 fib_index;
@@ -441,26 +442,6 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 	  pfx.fp_proto = FIB_PROTOCOL_IP6;
 	  vec_add1 (prefixs, pfx);
 	}
-      else if (unformat (line_input, "via %U %U weight %u",
-			 unformat_ip4_address,
-			 &rpath.frp_addr.ip4,
-			 unformat_vnet_sw_interface, vnm,
-			 &rpath.frp_sw_if_index, &rpath.frp_weight))
-	{
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
-	  vec_add1 (rpaths, rpath);
-	}
-
-      else if (unformat (line_input, "via %U %U weight %u",
-			 unformat_ip6_address,
-			 &rpath.frp_addr.ip6,
-			 unformat_vnet_sw_interface, vnm,
-			 &rpath.frp_sw_if_index, &rpath.frp_weight))
-	{
-	  rpath.frp_proto = FIB_PROTOCOL_IP6;
-	  vec_add1 (rpaths, rpath);
-	}
-
       else if (unformat (line_input, "via %U %U",
 			 unformat_ip4_address,
 			 &rpath.frp_addr.ip4,
@@ -482,6 +463,16 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 	  rpath.frp_proto = FIB_PROTOCOL_IP6;
 	  vec_add1 (rpaths, rpath);
 	}
+      else if (unformat (line_input, "weight %u", &weight))
+	{
+	  ASSERT (vec_len (rpaths));
+	  rpaths[vec_len (rpaths) - 1].frp_weight = weight;
+	}
+      else if (unformat (line_input, "preference %u", &preference))
+	{
+	  ASSERT (vec_len (rpaths));
+	  rpaths[vec_len (rpaths) - 1].frp_preference = preference;
+	}
       else if (unformat (line_input, "via %U next-hop-table %d",
 			 unformat_ip4_address,
 			 &rpath.frp_addr.ip4, &rpath.frp_fib_index))
diff --git a/src/vnet/mpls/mpls.api b/src/vnet/mpls/mpls.api
index c8a3ffb7..61d7fe6c 100644
--- a/src/vnet/mpls/mpls.api
+++ b/src/vnet/mpls/mpls.api
@@ -48,6 +48,7 @@ autoreply define mpls_ip_bind_unbind
     @param mt_is_multicast - Is the tunnel's underlying LSP multicast
     @param mt_next_hop_proto_is_ip4 - The next-hop is IPV4
     @param mt_next_hop_weight - The weight, for UCMP
+    @param mt_next_hop_preference - The preference
     @param mt_next_hop[16] - the nextop address
     @param mt_next_hop_sw_if_index - the next-hop SW interface
     @param mt_next_hop_table_id - the next-hop table-id (if appropriate)
@@ -64,6 +65,7 @@ define mpls_tunnel_add_del
   u8 mt_is_multicast;
   u8 mt_next_hop_proto_is_ip4;
   u8 mt_next_hop_weight;
+  u8 mt_next_hop_preference;
   u8 mt_next_hop[16];
   u8 mt_next_hop_n_out_labels;
   u32 mt_next_hop_sw_if_index;
@@ -181,6 +183,7 @@ autoreply define mpls_route_add_del
   u8 mr_is_rpf_id;
   u8 mr_next_hop_proto_is_ip4;
   u8 mr_next_hop_weight;
+  u8 mr_next_hop_preference;
   u8 mr_next_hop[16];
   u8 mr_next_hop_n_out_labels;
   u32 mr_next_hop_sw_if_index;
diff --git a/src/vnet/mpls/mpls_api.c b/src/vnet/mpls/mpls_api.c
index 6bfc491d..22fb7d43 100644
--- a/src/vnet/mpls/mpls_api.c
+++ b/src/vnet/mpls/mpls_api.c
@@ -206,6 +206,7 @@ mpls_route_add_del_t_handler (vnet_main_t * vnm,
 				   &nh, ntohl (mp->mr_next_hop_sw_if_index),
 				   next_hop_fib_index,
 				   mp->mr_next_hop_weight,
+				   mp->mr_next_hop_preference,
 				   ntohl (mp->mr_next_hop_via_label),
 				   label_stack));
 }
-- 
cgit 1.2.3-korg


From da78f957e46c686434149d332a477d7ea055d76a Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Wed, 24 May 2017 09:15:43 -0700
Subject: L2 over MPLS

[support for VPWS/VPLS]
- switch to using dpo_proto_t rather than fib_protocol_t in fib_paths so that we can describe L2 paths
- VLIB nodes to handle pop/push of MPLS labels to L2

Change-Id: Id050d06a11fd2c9c1c81ce5a0654e6c5ae6afa6e
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/plugins/gtpu/gtpu.c                    |   2 +-
 src/plugins/snat/snat.c                    |   2 +-
 src/vat/api_format.c                       |  17 +-
 src/vnet/dhcp/client.c                     |   6 +-
 src/vnet/dhcp/dhcp6_proxy_node.c           |   2 +-
 src/vnet/dpo/dpo.c                         |  19 ++
 src/vnet/dpo/dpo.h                         |  11 +-
 src/vnet/dpo/interface_dpo.c               |  30 +++
 src/vnet/dpo/mpls_label_dpo.c              |  45 +++-
 src/vnet/ethernet/arp.c                    |   4 +-
 src/vnet/fib/fib_api.h                     |   4 +-
 src/vnet/fib/fib_entry.c                   |   8 +-
 src/vnet/fib/fib_entry_src.c               |  16 +-
 src/vnet/fib/fib_entry_src.h               |   4 +-
 src/vnet/fib/fib_entry_src_api.c           |   2 +-
 src/vnet/fib/fib_entry_src_default_route.c |   2 +-
 src/vnet/fib/fib_entry_src_interface.c     |   2 +-
 src/vnet/fib/fib_entry_src_lisp.c          |   8 +-
 src/vnet/fib/fib_entry_src_mpls.c          |   4 +-
 src/vnet/fib/fib_entry_src_rr.c            |  15 +-
 src/vnet/fib/fib_entry_src_special.c       |   2 +-
 src/vnet/fib/fib_path.c                    |  79 +++----
 src/vnet/fib/fib_path.h                    |  11 +-
 src/vnet/fib/fib_path_ext.c                |   3 +
 src/vnet/fib/fib_path_list.c               |   4 +-
 src/vnet/fib/fib_path_list.h               |   4 +-
 src/vnet/fib/fib_table.c                   |   6 +-
 src/vnet/fib/fib_table.h                   |   6 +-
 src/vnet/fib/fib_test.c                    | 338 ++++++++++++++---------------
 src/vnet/fib/fib_types.h                   |   8 +-
 src/vnet/interface_format.c                |  12 +-
 src/vnet/ip/ip4_forward.c                  |   6 +-
 src/vnet/ip/ip6_forward.c                  |   4 +-
 src/vnet/ip/ip6_neighbor.c                 |  10 +-
 src/vnet/ip/ip_api.c                       |  38 ++--
 src/vnet/ip/lookup.c                       |  18 +-
 src/vnet/lisp-gpe/lisp_gpe.c               |  13 +-
 src/vnet/lisp-gpe/lisp_gpe_api.c           |  10 +-
 src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c     |   5 +-
 src/vnet/mfib/ip6_mfib.c                   |   6 +-
 src/vnet/mfib/mfib_entry.c                 |  10 +-
 src/vnet/mfib/mfib_test.c                  |  14 +-
 src/vnet/mpls/mpls.api                     |   4 +-
 src/vnet/mpls/mpls.c                       |  36 +--
 src/vnet/mpls/mpls_api.c                   |  21 +-
 src/vnet/mpls/mpls_tunnel.c                |  75 ++++---
 src/vnet/mpls/mpls_tunnel.h                |  38 ++--
 src/vnet/srmpls/sr_mpls_policy.c           |   6 +-
 src/vnet/srmpls/sr_mpls_steering.c         |   2 +-
 src/vnet/srv6/sr_steering.c                |   4 +-
 src/vnet/vxlan-gpe/vxlan_gpe.c             |   2 +-
 src/vnet/vxlan/vxlan.c                     |   2 +-
 src/vpp/app/vpe_cli.c                      |   2 +-
 test/test_bfd.py                           |   6 +-
 test/test_gre.py                           |  24 +-
 test/test_ip6.py                           |  26 +--
 test/test_map.py                           |  19 +-
 test/test_mpls.py                          | 318 ++++++++++++++++++++++-----
 test/test_p2p_ethernet.py                  |  16 +-
 test/vpp_ip_route.py                       |  24 +-
 test/vpp_mpls_tunnel_interface.py          |   6 +-
 test/vpp_papi_provider.py                  |   4 +-
 62 files changed, 889 insertions(+), 556 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/plugins/gtpu/gtpu.c b/src/plugins/gtpu/gtpu.c
index 84745bd8..3dfb4210 100755
--- a/src/plugins/gtpu/gtpu.c
+++ b/src/plugins/gtpu/gtpu.c
@@ -534,7 +534,7 @@ int vnet_gtpu_add_del_tunnel
 	      fib_node_index_t mfei;
 	      adj_index_t ai;
 	      fib_route_path_t path = {
-		.frp_proto = fp,
+		.frp_proto = fib_proto_to_dpo (fp),
 		.frp_addr = zero_addr,
 		.frp_sw_if_index = 0xffffffff,
 		.frp_fib_index = ~0,
diff --git a/src/plugins/snat/snat.c b/src/plugins/snat/snat.c
index 9fbc1e54..f196b5c2 100644
--- a/src/plugins/snat/snat.c
+++ b/src/plugins/snat/snat.c
@@ -135,7 +135,7 @@ snat_add_del_addr_to_fib (ip4_address_t * addr, u8 p_len, u32 sw_if_index,
                                     (FIB_ENTRY_FLAG_CONNECTED |
                                      FIB_ENTRY_FLAG_LOCAL |
                                      FIB_ENTRY_FLAG_EXCLUSIVE),
-                                    FIB_PROTOCOL_IP4,
+                                    DPO_PROTO_IP4,
                                     NULL,
                                     sw_if_index,
                                     ~0,
diff --git a/src/vat/api_format.c b/src/vat/api_format.c
index f97cdeef..009cf173 100644
--- a/src/vat/api_format.c
+++ b/src/vat/api_format.c
@@ -7498,7 +7498,7 @@ api_mpls_route_add_del (vat_main_t * vam)
   mpls_label_t *next_hop_out_label_stack = NULL;
   mpls_label_t local_label = MPLS_LABEL_INVALID;
   u8 is_eos = 0;
-  u8 next_hop_proto_is_ip4 = 1;
+  dpo_proto_t next_hop_proto = DPO_PROTO_IP4;
 
   /* Parse args required to build the message */
   while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
@@ -7517,13 +7517,13 @@ api_mpls_route_add_del (vat_main_t * vam)
 			 &v4_next_hop_address))
 	{
 	  next_hop_set = 1;
-	  next_hop_proto_is_ip4 = 1;
+	  next_hop_proto = DPO_PROTO_IP4;
 	}
       else if (unformat (i, "via %U", unformat_ip6_address,
 			 &v6_next_hop_address))
 	{
 	  next_hop_set = 1;
-	  next_hop_proto_is_ip4 = 0;
+	  next_hop_proto = DPO_PROTO_IP6;
 	}
       else if (unformat (i, "weight %d", &next_hop_weight))
 	;
@@ -7548,12 +7548,12 @@ api_mpls_route_add_del (vat_main_t * vam)
       else if (unformat (i, "lookup-in-ip4-table %d", &next_hop_table_id))
 	{
 	  next_hop_set = 1;
-	  next_hop_proto_is_ip4 = 1;
+	  next_hop_proto = DPO_PROTO_IP4;
 	}
       else if (unformat (i, "lookup-in-ip6-table %d", &next_hop_table_id))
 	{
 	  next_hop_set = 1;
-	  next_hop_proto_is_ip4 = 0;
+	  next_hop_proto = DPO_PROTO_IP6;
 	}
       else if (unformat (i, "next-hop-table %d", &next_hop_table_id))
 	;
@@ -7599,7 +7599,7 @@ api_mpls_route_add_del (vat_main_t * vam)
       mp->mr_create_table_if_needed = create_table_if_needed;
 
       mp->mr_is_add = is_add;
-      mp->mr_next_hop_proto_is_ip4 = next_hop_proto_is_ip4;
+      mp->mr_next_hop_proto = next_hop_proto;
       mp->mr_is_classify = is_classify;
       mp->mr_is_multipath = is_multipath;
       mp->mr_is_resolve_host = resolve_host;
@@ -7622,13 +7622,14 @@ api_mpls_route_add_del (vat_main_t * vam)
 
       if (next_hop_set)
 	{
-	  if (next_hop_proto_is_ip4)
+	  if (DPO_PROTO_IP4 == next_hop_proto)
 	    {
 	      clib_memcpy (mp->mr_next_hop,
 			   &v4_next_hop_address,
 			   sizeof (v4_next_hop_address));
 	    }
-	  else
+	  else if (DPO_PROTO_IP6 == next_hop_proto)
+
 	    {
 	      clib_memcpy (mp->mr_next_hop,
 			   &v6_next_hop_address,
diff --git a/src/vnet/dhcp/client.c b/src/vnet/dhcp/client.c
index cfe62a6f..dd5e99f2 100644
--- a/src/vnet/dhcp/client.c
+++ b/src/vnet/dhcp/client.c
@@ -296,7 +296,7 @@ int dhcp_client_for_us (u32 bi, vlib_buffer_t * b,
 					&all_0s,
 					FIB_SOURCE_DHCP,
 					FIB_ENTRY_FLAG_NONE,
-					FIB_PROTOCOL_IP4,
+					DPO_PROTO_IP4,
 					&nh,
 					c->sw_if_index,
 					~0,
@@ -605,7 +605,7 @@ dhcp_bound_state (dhcp_client_main_t * dcm, dhcp_client_t * c, f64 now)
 					  c->sw_if_index),
 				      &all_0s,
 				      FIB_SOURCE_DHCP,
-				      FIB_PROTOCOL_IP4,
+				      DPO_PROTO_IP4,
 				      &nh,
 				      c->sw_if_index,
 				      ~0,
@@ -900,7 +900,7 @@ int dhcp_client_add_del (dhcp_client_add_del_args_t * a)
 					  c->sw_if_index),
 				      &all_0s,
 				      FIB_SOURCE_DHCP,
-				      FIB_PROTOCOL_IP4,
+				      DPO_PROTO_IP4,
 				      &nh,
 				      c->sw_if_index,
 				      ~0,
diff --git a/src/vnet/dhcp/dhcp6_proxy_node.c b/src/vnet/dhcp/dhcp6_proxy_node.c
index e109cc4c..9c2f5220 100644
--- a/src/vnet/dhcp/dhcp6_proxy_node.c
+++ b/src/vnet/dhcp/dhcp6_proxy_node.c
@@ -857,7 +857,7 @@ dhcp6_proxy_set_server (ip46_address_t *addr,
   else
     {
      const fib_route_path_t path_for_us = {
-          .frp_proto = FIB_PROTOCOL_IP6,
+          .frp_proto = DPO_PROTO_IP6,
           .frp_addr = zero_addr,
           .frp_sw_if_index = 0xffffffff,
           .frp_fib_index = ~0,
diff --git a/src/vnet/dpo/dpo.c b/src/vnet/dpo/dpo.c
index 389f995b..aa770838 100644
--- a/src/vnet/dpo/dpo.c
+++ b/src/vnet/dpo/dpo.c
@@ -109,6 +109,25 @@ vnet_link_to_dpo_proto (vnet_link_t linkt)
     return (0);
 }
 
+vnet_link_t
+dpo_proto_to_link (dpo_proto_t dp)
+{
+    switch (dp)
+    {
+    case DPO_PROTO_IP6:
+        return (VNET_LINK_IP6);
+    case DPO_PROTO_IP4:
+        return (VNET_LINK_IP4);
+    case DPO_PROTO_MPLS:
+        return (VNET_LINK_MPLS);
+    case DPO_PROTO_ETHERNET:
+        return (VNET_LINK_ETHERNET);
+    case DPO_PROTO_NSH:
+        return (VNET_LINK_NSH);
+    }
+    return (~0);
+}
+
 u8 *
 format_dpo_type (u8 * s, va_list * args)
 {
diff --git a/src/vnet/dpo/dpo.h b/src/vnet/dpo/dpo.h
index 5aa4e2d2..42fc51d4 100644
--- a/src/vnet/dpo/dpo.h
+++ b/src/vnet/dpo/dpo.h
@@ -59,14 +59,10 @@ typedef u32 index_t;
  */
 typedef enum dpo_proto_t_
 {
-#if CLIB_DEBUG > 0
-    DPO_PROTO_IP4 = 1,
-#else
     DPO_PROTO_IP4 = 0,
-#endif
     DPO_PROTO_IP6,
-    DPO_PROTO_ETHERNET,
     DPO_PROTO_MPLS,
+    DPO_PROTO_ETHERNET,
     DPO_PROTO_NSH,
 } __attribute__((packed)) dpo_proto_t;
 
@@ -272,6 +268,11 @@ extern u8 *format_dpo_type(u8 * s, va_list * args);
  */
 extern u8 *format_dpo_proto(u8 * s, va_list * args);
 
+/**
+ * @brief format a DPO protocol
+ */
+extern vnet_link_t dpo_proto_to_link(dpo_proto_t dp);
+
 /**
  * @brief
  *  Set and stack a DPO.
diff --git a/src/vnet/dpo/interface_dpo.c b/src/vnet/dpo/interface_dpo.c
index 8d700c23..780bfa2a 100644
--- a/src/vnet/dpo/interface_dpo.c
+++ b/src/vnet/dpo/interface_dpo.c
@@ -195,11 +195,17 @@ const static char* const interface_dpo_ip6_nodes[] =
     "interface-dpo-ip4",
     NULL,
 };
+const static char* const interface_dpo_l2_nodes[] =
+{
+    "interface-dpo-l2",
+    NULL,
+};
 
 const static char* const * const interface_dpo_nodes[DPO_PROTO_NUM] =
 {
     [DPO_PROTO_IP4]  = interface_dpo_ip4_nodes,
     [DPO_PROTO_IP6]  = interface_dpo_ip6_nodes,
+    [DPO_PROTO_ETHERNET]  = interface_dpo_l2_nodes,
     [DPO_PROTO_MPLS] = NULL,
 };
 
@@ -382,6 +388,14 @@ interface_dpo_ip6 (vlib_main_t * vm,
     return (interface_dpo_inline(vm, node, from_frame));
 }
 
+static uword
+interface_dpo_l2 (vlib_main_t * vm,
+                   vlib_node_runtime_t * node,
+                   vlib_frame_t * from_frame)
+{
+    return (interface_dpo_inline(vm, node, from_frame));
+}
+
 VLIB_REGISTER_NODE (interface_dpo_ip4_node) = {
     .function = interface_dpo_ip4,
     .name = "interface-dpo-ip4",
@@ -414,3 +428,19 @@ VLIB_REGISTER_NODE (interface_dpo_ip6_node) = {
 VLIB_NODE_FUNCTION_MULTIARCH (interface_dpo_ip6_node,
                               interface_dpo_ip6)
 
+VLIB_REGISTER_NODE (interface_dpo_l2_node) = {
+    .function = interface_dpo_l2,
+    .name = "interface-dpo-l2",
+    .vector_size = sizeof (u32),
+    .format_trace = format_interface_dpo_trace,
+
+    .n_next_nodes = 2,
+    .next_nodes = {
+        [INTERFACE_DPO_DROP] = "error-drop",
+        [INTERFACE_DPO_INPUT] = "l2-input",
+    },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (interface_dpo_l2_node,
+                              interface_dpo_l2)
+
diff --git a/src/vnet/dpo/mpls_label_dpo.c b/src/vnet/dpo/mpls_label_dpo.c
index 1c451a51..b178a902 100644
--- a/src/vnet/dpo/mpls_label_dpo.c
+++ b/src/vnet/dpo/mpls_label_dpo.c
@@ -192,7 +192,8 @@ mpls_label_imposition_inline (vlib_main_t * vm,
                               vlib_node_runtime_t * node,
                               vlib_frame_t * from_frame,
                               u8 payload_is_ip4,
-                              u8 payload_is_ip6)
+                              u8 payload_is_ip6,
+                              u8 payload_is_ethernet)
 {
     u32 n_left_from, next_index, * from, * to_next;
 
@@ -320,6 +321,13 @@ mpls_label_imposition_inline (vlib_main_t * vm,
                 ttl2 = ip2->hop_limit;
                 ttl3 = ip3->hop_limit;
             }
+            else if (payload_is_ethernet)
+            {
+                /*
+                 * nothing to chang ein the ethernet header
+                 */
+                ttl0 = ttl1 = ttl2 = ttl3 = 255;
+            }
             else
             {
                 /*
@@ -551,7 +559,7 @@ mpls_label_imposition (vlib_main_t * vm,
                        vlib_node_runtime_t * node,
                        vlib_frame_t * frame)
 {
-    return (mpls_label_imposition_inline(vm, node, frame, 0, 0));
+    return (mpls_label_imposition_inline(vm, node, frame, 0, 0, 0));
 }
 
 VLIB_REGISTER_NODE (mpls_label_imposition_node) = {
@@ -573,7 +581,7 @@ ip4_mpls_label_imposition (vlib_main_t * vm,
                            vlib_node_runtime_t * node,
                            vlib_frame_t * frame)
 {
-    return (mpls_label_imposition_inline(vm, node, frame, 1, 0));
+    return (mpls_label_imposition_inline(vm, node, frame, 1, 0, 0));
 }
 
 VLIB_REGISTER_NODE (ip4_mpls_label_imposition_node) = {
@@ -595,7 +603,7 @@ ip6_mpls_label_imposition (vlib_main_t * vm,
                            vlib_node_runtime_t * node,
                            vlib_frame_t * frame)
 {
-    return (mpls_label_imposition_inline(vm, node, frame, 0, 1));
+    return (mpls_label_imposition_inline(vm, node, frame, 0, 1, 0));
 }
 
 VLIB_REGISTER_NODE (ip6_mpls_label_imposition_node) = {
@@ -612,6 +620,28 @@ VLIB_REGISTER_NODE (ip6_mpls_label_imposition_node) = {
 VLIB_NODE_FUNCTION_MULTIARCH (ip6_mpls_label_imposition_node,
                               ip6_mpls_label_imposition)
 
+static uword
+ethernet_mpls_label_imposition (vlib_main_t * vm,
+                                vlib_node_runtime_t * node,
+                                vlib_frame_t * frame)
+{
+    return (mpls_label_imposition_inline(vm, node, frame, 0, 0, 1));
+}
+
+VLIB_REGISTER_NODE (ethernet_mpls_label_imposition_node) = {
+    .function = ethernet_mpls_label_imposition,
+    .name = "ethernet-mpls-label-imposition",
+    .vector_size = sizeof (u32),
+
+    .format_trace = format_mpls_label_imposition_trace,
+    .n_next_nodes = 1,
+    .next_nodes = {
+        [0] = "error-drop",
+    }
+};
+VLIB_NODE_FUNCTION_MULTIARCH (ethernet_mpls_label_imposition_node,
+                              ethernet_mpls_label_imposition)
+
 static void
 mpls_label_dpo_mem_show (void)
 {
@@ -643,11 +673,18 @@ const static char* const mpls_label_imp_mpls_nodes[] =
     "mpls-label-imposition",
     NULL,
 };
+const static char* const mpls_label_imp_ethernet_nodes[] =
+{
+    "ethernet-mpls-label-imposition",
+    NULL,
+};
+
 const static char* const * const mpls_label_imp_nodes[DPO_PROTO_NUM] =
 {
     [DPO_PROTO_IP4]  = mpls_label_imp_ip4_nodes,
     [DPO_PROTO_IP6]  = mpls_label_imp_ip6_nodes,
     [DPO_PROTO_MPLS] = mpls_label_imp_mpls_nodes,
+    [DPO_PROTO_ETHERNET] = mpls_label_imp_ethernet_nodes,
 };
 
 
diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c
index 4d9edaf5..8a394006 100644
--- a/src/vnet/ethernet/arp.c
+++ b/src/vnet/ethernet/arp.c
@@ -588,7 +588,7 @@ vnet_arp_set_ip4_over_ethernet_internal (vnet_main_t * vnm,
 	  e->fib_entry_index =
 	    fib_table_entry_path_add (fib_index, &pfx, FIB_SOURCE_ADJ,
 				      FIB_ENTRY_FLAG_ATTACHED,
-				      FIB_PROTOCOL_IP4, &pfx.fp_addr,
+				      DPO_PROTO_IP4, &pfx.fp_addr,
 				      e->sw_if_index, ~0, 1, NULL,
 				      FIB_ROUTE_PATH_FLAG_NONE);
 	}
@@ -1621,7 +1621,7 @@ arp_entry_free (ethernet_arp_interface_t * eai, ethernet_arp_ip4_entry_t * e)
 
       fib_table_entry_path_remove (fib_index, &pfx,
 				   FIB_SOURCE_ADJ,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &pfx.fp_addr,
 				   e->sw_if_index, ~0, 1,
 				   FIB_ROUTE_PATH_FLAG_NONE);
diff --git a/src/vnet/fib/fib_api.h b/src/vnet/fib/fib_api.h
index 73d76a42..d07d6cae 100644
--- a/src/vnet/fib/fib_api.h
+++ b/src/vnet/fib/fib_api.h
@@ -21,7 +21,7 @@ int
 add_del_route_check (fib_protocol_t table_proto,
 		     u32 table_id,
 		     u32 next_hop_sw_if_index,
-		     fib_protocol_t next_hop_table_proto,
+		     dpo_proto_t next_hop_table_proto,
 		     u32 next_hop_table_id,
 		     u8 create_missing_tables,
                      u8 is_rpf_id,
@@ -43,7 +43,7 @@ add_del_route_t_handler (u8 is_multipath,
                          u8 is_rpf_id,
 			 u32 fib_index,
 			 const fib_prefix_t * prefix,
-			 u8 next_hop_proto_is_ip4,
+			 dpo_proto_t next_hop_proto,
 			 const ip46_address_t * next_hop,
 			 u32 next_hop_sw_if_index,
 			 u8 next_hop_fib_index,
diff --git a/src/vnet/fib/fib_entry.c b/src/vnet/fib/fib_entry.c
index d7ff1c8c..2027f2be 100644
--- a/src/vnet/fib/fib_entry.c
+++ b/src/vnet/fib/fib_entry.c
@@ -58,12 +58,18 @@ fib_entry_get_index (const fib_entry_t * fib_entry)
     return (fib_entry - fib_entry_pool);
 }
 
-static fib_protocol_t
+fib_protocol_t
 fib_entry_get_proto (const fib_entry_t * fib_entry)
 {
     return (fib_entry->fe_prefix.fp_proto);
 }
 
+dpo_proto_t
+fib_entry_get_dpo_proto (const fib_entry_t * fib_entry)
+{
+    return (fib_proto_to_dpo(fib_entry->fe_prefix.fp_proto));
+}
+
 fib_forward_chain_type_t
 fib_entry_get_default_chain_type (const fib_entry_t *fib_entry)
 {
diff --git a/src/vnet/fib/fib_entry_src.c b/src/vnet/fib/fib_entry_src.c
index ff73cbf9..173df74f 100644
--- a/src/vnet/fib/fib_entry_src.c
+++ b/src/vnet/fib/fib_entry_src.c
@@ -29,12 +29,6 @@
  */
 static fib_entry_src_vft_t fib_entry_src_vft[FIB_SOURCE_MAX];
 
-static fib_protocol_t
-fib_entry_get_proto (const fib_entry_t * fib_entry)
-{
-    return (fib_entry->fe_prefix.fp_proto);
-}
-
 void
 fib_entry_src_register (fib_source_t source,
 			const fib_entry_src_vft_t *vft)
@@ -861,7 +855,7 @@ fib_entry_src_action_add (fib_entry_t *fib_entry,
 	fib_entry_src_vft[source].fesv_add(esrc,
 					   fib_entry,
 					   flags,
-					   fib_entry_get_proto(fib_entry),
+                                           fib_entry_get_dpo_proto(fib_entry),
 					   dpo);
     }
 
@@ -914,7 +908,7 @@ fib_entry_src_action_update (fib_entry_t *fib_entry,
 	fib_entry_src_vft[source].fesv_add(esrc,
 					   fib_entry,
 					   flags,
-					   fib_entry_get_proto(fib_entry),
+					   fib_entry_get_dpo_proto(fib_entry),
 					   dpo);
     }
 
@@ -1106,8 +1100,7 @@ fib_entry_src_action_path_add (fib_entry_t *fib_entry,
                                      source,
                                      flags,
                                      drop_dpo_get(
-                                         fib_proto_to_dpo(
-                                             fib_entry_get_proto(fib_entry))));
+                                         fib_entry_get_dpo_proto(fib_entry)));
 	esrc = fib_entry_src_find(fib_entry, source, NULL);
     }
 
@@ -1166,8 +1159,7 @@ fib_entry_src_action_path_swap (fib_entry_t *fib_entry,
 					     source,
 					     flags,
                                              drop_dpo_get(
-                                                 fib_proto_to_dpo(
-                                                     fib_entry_get_proto(fib_entry))));
+                                                 fib_entry_get_dpo_proto(fib_entry)));
 	esrc = fib_entry_src_find(fib_entry, source, NULL);
     }
 
diff --git a/src/vnet/fib/fib_entry_src.h b/src/vnet/fib/fib_entry_src.h
index 640c174d..35c43936 100644
--- a/src/vnet/fib/fib_entry_src.h
+++ b/src/vnet/fib/fib_entry_src.h
@@ -73,7 +73,7 @@ typedef void (*fib_entry_src_deactivate_t)(fib_entry_src_t *src,
 typedef void (*fib_entry_src_add_t)(fib_entry_src_t *src,
 				    const fib_entry_t *entry,
 				    fib_entry_flag_t flags,
-				    fib_protocol_t proto,
+				    dpo_proto_t proto,
 				    const dpo_id_t *dpo);
 
 /**
@@ -277,6 +277,8 @@ extern void fib_entry_src_mk_lb (fib_entry_t *fib_entry,
 				 fib_forward_chain_type_t fct,
 				 dpo_id_t *dpo_lb);
 
+extern fib_protocol_t fib_entry_get_proto(const fib_entry_t * fib_entry);
+extern dpo_proto_t fib_entry_get_dpo_proto(const fib_entry_t * fib_entry);
 
 /*
  * Per-source registration. declared here so we save a separate .h file for each
diff --git a/src/vnet/fib/fib_entry_src_api.c b/src/vnet/fib/fib_entry_src_api.c
index f895886b..1cdcfbde 100644
--- a/src/vnet/fib/fib_entry_src_api.c
+++ b/src/vnet/fib/fib_entry_src_api.c
@@ -131,7 +131,7 @@ static void
 fib_entry_src_api_add (fib_entry_src_t *src,
 		       const fib_entry_t *entry,
 		       fib_entry_flag_t flags,
-		       fib_protocol_t proto,
+		       dpo_proto_t proto,
 		       const dpo_id_t *dpo)
 {
     if (FIB_ENTRY_FLAG_NONE != flags)
diff --git a/src/vnet/fib/fib_entry_src_default_route.c b/src/vnet/fib/fib_entry_src_default_route.c
index 9f4e7c36..431abb66 100644
--- a/src/vnet/fib/fib_entry_src_default_route.c
+++ b/src/vnet/fib/fib_entry_src_default_route.c
@@ -35,7 +35,7 @@ static void
 fib_entry_src_default_route_add (fib_entry_src_t *src,
 				 const fib_entry_t *entry,
 				 fib_entry_flag_t flags,
-				 fib_protocol_t proto,
+				 dpo_proto_t proto,
 				 const dpo_id_t *dpo)
 {
     src->fes_pl = fib_path_list_create_special(proto,
diff --git a/src/vnet/fib/fib_entry_src_interface.c b/src/vnet/fib/fib_entry_src_interface.c
index bb87818f..6c087f34 100644
--- a/src/vnet/fib/fib_entry_src_interface.c
+++ b/src/vnet/fib/fib_entry_src_interface.c
@@ -35,7 +35,7 @@ static void
 fib_entry_src_interface_add (fib_entry_src_t *src,
                              const fib_entry_t *entry,
                              fib_entry_flag_t flags,
-                             fib_protocol_t proto,
+                             dpo_proto_t proto,
                              const dpo_id_t *dpo)
 {
     src->fes_pl = fib_path_list_create_special(
diff --git a/src/vnet/fib/fib_entry_src_lisp.c b/src/vnet/fib/fib_entry_src_lisp.c
index 7f8b91bb..e72dce63 100644
--- a/src/vnet/fib/fib_entry_src_lisp.c
+++ b/src/vnet/fib/fib_entry_src_lisp.c
@@ -79,10 +79,10 @@ fib_entry_src_lisp_path_remove (fib_entry_src_t *src,
 
 static void
 fib_entry_src_lisp_add (fib_entry_src_t *src,
-		       const fib_entry_t *entry,
-		       fib_entry_flag_t flags,
-		       fib_protocol_t proto,
-		       const dpo_id_t *dpo)
+                        const fib_entry_t *entry,
+                        fib_entry_flag_t flags,
+                        dpo_proto_t proto,
+                        const dpo_id_t *dpo)
 {
     if (FIB_ENTRY_FLAG_NONE != flags)
     {
diff --git a/src/vnet/fib/fib_entry_src_mpls.c b/src/vnet/fib/fib_entry_src_mpls.c
index 14c7310f..a616458f 100644
--- a/src/vnet/fib/fib_entry_src_mpls.c
+++ b/src/vnet/fib/fib_entry_src_mpls.c
@@ -57,13 +57,13 @@ static void
 fib_entry_src_mpls_add (fib_entry_src_t *src,
                         const fib_entry_t *entry,
                         fib_entry_flag_t flags,
-                        fib_protocol_t proto,
+                        dpo_proto_t proto,
                         const dpo_id_t *dpo)
 {
     src->fes_pl =
 	fib_path_list_create_special(proto,
 				     FIB_PATH_LIST_FLAG_DROP,
-				     drop_dpo_get(fib_proto_to_dpo(proto)));
+				     drop_dpo_get(proto));
 }
 
 static void
diff --git a/src/vnet/fib/fib_entry_src_rr.c b/src/vnet/fib/fib_entry_src_rr.c
index d66ef7b1..1153f3f1 100644
--- a/src/vnet/fib/fib_entry_src_rr.c
+++ b/src/vnet/fib/fib_entry_src_rr.c
@@ -35,7 +35,7 @@ fib_entry_src_rr_resolve_via_connected (fib_entry_src_t *src,
 					const fib_entry_t *cover)
 {
     const fib_route_path_t path = {
-	.frp_proto = fib_entry->fe_prefix.fp_proto,
+	.frp_proto = fib_proto_to_dpo(fib_entry->fe_prefix.fp_proto),
 	.frp_addr = fib_entry->fe_prefix.fp_addr,
 	.frp_sw_if_index = fib_entry_get_resolving_interface(
 	                       fib_entry_get_index(cover)),
@@ -90,18 +90,17 @@ fib_entry_src_rr_use_covers_pl (fib_entry_src_t *src,
                                 const fib_entry_t *cover)
 {
     fib_node_index_t *entries = NULL;
-    fib_protocol_t proto;
+    dpo_proto_t proto;
 
-    proto = fib_entry->fe_prefix.fp_proto;
+    proto = fib_proto_to_dpo(fib_entry->fe_prefix.fp_proto);
     vec_add1(entries, fib_entry_get_index(fib_entry));
 
     if (fib_path_list_recursive_loop_detect(cover->fe_parent,
                                             &entries))
     {
-        src->fes_pl = fib_path_list_create_special(
-            proto,
-            FIB_PATH_LIST_FLAG_DROP,
-            drop_dpo_get(fib_proto_to_dpo(proto)));
+        src->fes_pl = fib_path_list_create_special(proto,
+                                                   FIB_PATH_LIST_FLAG_DROP,
+                                                   drop_dpo_get(proto));
     }
     else
     {
@@ -126,7 +125,7 @@ fib_entry_src_rr_activate (fib_entry_src_t *src,
      */
     if (FIB_PROTOCOL_MPLS == fib_entry->fe_prefix.fp_proto)
     {
-	src->fes_pl = fib_path_list_create_special(FIB_PROTOCOL_MPLS,
+	src->fes_pl = fib_path_list_create_special(DPO_PROTO_MPLS,
 						   FIB_PATH_LIST_FLAG_DROP,
 						   NULL);
 	fib_path_list_lock(src->fes_pl);
diff --git a/src/vnet/fib/fib_entry_src_special.c b/src/vnet/fib/fib_entry_src_special.c
index 75605d7f..e979e18f 100644
--- a/src/vnet/fib/fib_entry_src_special.c
+++ b/src/vnet/fib/fib_entry_src_special.c
@@ -43,7 +43,7 @@ static void
 fib_entry_src_special_add (fib_entry_src_t *src,
 			   const fib_entry_t *entry,
 			   fib_entry_flag_t flags,
-			   fib_protocol_t proto,
+			   dpo_proto_t proto,
 			   const dpo_id_t *dpo)
 {
     src->fes_pl =
diff --git a/src/vnet/fib/fib_path.c b/src/vnet/fib/fib_path.c
index 3a67a544..58050ccb 100644
--- a/src/vnet/fib/fib_path.c
+++ b/src/vnet/fib/fib_path.c
@@ -193,7 +193,7 @@ typedef struct fib_path_t_ {
      * next-hop's address. We can't derive this from the address itself
      * since the address can be all zeros
      */
-    fib_protocol_t fp_nh_proto;
+    dpo_proto_t fp_nh_proto;
 
     /**
      * UCMP [unnormalised] weigth
@@ -381,7 +381,7 @@ format_fib_path (u8 * s, va_list * args)
 
     s = format (s, "      index:%d ", fib_path_get_index(path));
     s = format (s, "pl-index:%d ", path->fp_pl_index);
-    s = format (s, "%U ", format_fib_protocol, path->fp_nh_proto);
+    s = format (s, "%U ", format_dpo_proto, path->fp_nh_proto);
     s = format (s, "weight=%d ", path->fp_weight);
     s = format (s, "pref=%d ", path->fp_preference);
     s = format (s, "%s: ", fib_path_type_names[path->fp_type]);
@@ -454,7 +454,7 @@ format_fib_path (u8 * s, va_list * args)
 	}
 	break;
     case FIB_PATH_TYPE_RECURSIVE:
-	if (FIB_PROTOCOL_MPLS == path->fp_nh_proto)
+	if (DPO_PROTO_MPLS == path->fp_nh_proto)
 	{
 	    s = format (s, "via %U %U",
 			format_mpls_unicast_label,
@@ -552,14 +552,14 @@ fib_path_attached_next_hop_get_adj (fib_path_t *path,
 	 * the subnet address (the attached route) links to the
 	 * auto-adj (see below), we want that adj here too.
 	 */
-	return (adj_nbr_add_or_lock(path->fp_nh_proto,
+	return (adj_nbr_add_or_lock(dpo_proto_to_fib(path->fp_nh_proto),
 				    link,
 				    &zero_addr,
 				    path->attached_next_hop.fp_interface));
     }
     else
     {
-	return (adj_nbr_add_or_lock(path->fp_nh_proto,
+	return (adj_nbr_add_or_lock(dpo_proto_to_fib(path->fp_nh_proto),
 				    link,
 				    &path->attached_next_hop.fp_nh,
 				    path->attached_next_hop.fp_interface));
@@ -575,10 +575,10 @@ fib_path_attached_next_hop_set (fib_path_t *path)
      */
     dpo_set(&path->fp_dpo,
 	    DPO_ADJACENCY,
-	    fib_proto_to_dpo(path->fp_nh_proto),
+	    path->fp_nh_proto,
 	    fib_path_attached_next_hop_get_adj(
 		 path,
-		 fib_proto_to_link(path->fp_nh_proto)));
+		 dpo_proto_to_link(path->fp_nh_proto)));
 
     /*
      * become a child of the adjacency so we receive updates
@@ -607,14 +607,14 @@ fib_path_attached_get_adj (fib_path_t *path,
          * point-2-point interfaces do not require a glean, since
          * there is nothing to ARP. Install a rewrite/nbr adj instead
          */
-        return (adj_nbr_add_or_lock(path->fp_nh_proto,
+        return (adj_nbr_add_or_lock(dpo_proto_to_fib(path->fp_nh_proto),
                                     link,
                                     &zero_addr,
                                     path->attached.fp_interface));
     }
     else
     {
-        return (adj_glean_add_or_lock(path->fp_nh_proto,
+        return (adj_glean_add_or_lock(dpo_proto_to_fib(path->fp_nh_proto),
                                       path->attached.fp_interface,
                                       NULL));
     }
@@ -650,7 +650,7 @@ fib_path_recursive_adj_update (fib_path_t *path,
     if (path->fp_oper_flags & FIB_PATH_OPER_FLAG_RECURSIVE_LOOP)
     {
 	path->fp_oper_flags &= ~FIB_PATH_OPER_FLAG_RESOLVED;
-	dpo_copy(&via_dpo, drop_dpo_get(fib_proto_to_dpo(path->fp_nh_proto)));
+	dpo_copy(&via_dpo, drop_dpo_get(path->fp_nh_proto));
     }
     else if (path->fp_cfg_flags & FIB_PATH_CFG_FLAG_RESOLVE_HOST)
     {
@@ -668,7 +668,7 @@ fib_path_recursive_adj_update (fib_path_t *path,
 	if (fib_entry_get_best_source(path->fp_via_fib) >= FIB_SOURCE_RR)
 	{
 	    path->fp_oper_flags &= ~FIB_PATH_OPER_FLAG_RESOLVED;
-            dpo_copy(&via_dpo, drop_dpo_get(fib_proto_to_dpo(path->fp_nh_proto)));
+            dpo_copy(&via_dpo, drop_dpo_get(path->fp_nh_proto));
 
             /*
              * PIC edge trigger. let the load-balance maps know
@@ -685,7 +685,7 @@ fib_path_recursive_adj_update (fib_path_t *path,
 	if (!(FIB_ENTRY_FLAG_ATTACHED & fib_entry_get_flags(path->fp_via_fib)))
 	{
 	    path->fp_oper_flags &= ~FIB_PATH_OPER_FLAG_RESOLVED;
-            dpo_copy(&via_dpo, drop_dpo_get(fib_proto_to_dpo(path->fp_nh_proto)));
+            dpo_copy(&via_dpo, drop_dpo_get(path->fp_nh_proto));
 
             /*
              * PIC edge trigger. let the load-balance maps know
@@ -699,7 +699,7 @@ fib_path_recursive_adj_update (fib_path_t *path,
     if (!fib_entry_is_resolved(path->fp_via_fib))
     {
         path->fp_oper_flags &= ~FIB_PATH_OPER_FLAG_RESOLVED;
-        dpo_copy(&via_dpo, drop_dpo_get(fib_proto_to_dpo(path->fp_nh_proto)));
+        dpo_copy(&via_dpo, drop_dpo_get(path->fp_nh_proto));
 
         /*
          * PIC edge trigger. let the load-balance maps know
@@ -720,9 +720,7 @@ fib_path_recursive_adj_update (fib_path_t *path,
      */
     dpo_copy(dpo, &via_dpo);
 
-    FIB_PATH_DBG(path, "recursive update: %U",
-		 fib_get_lookup_main(path->fp_nh_proto),
-		 &path->fp_dpo, 2);
+    FIB_PATH_DBG(path, "recursive update:");
 
     dpo_reset(&via_dpo);
 }
@@ -804,13 +802,8 @@ fib_path_unresolve (fib_path_t *path)
 static fib_forward_chain_type_t
 fib_path_to_chain_type (const fib_path_t *path)
 {
-    switch (path->fp_nh_proto)
+    if (DPO_PROTO_MPLS == path->fp_nh_proto)
     {
-    case FIB_PROTOCOL_IP4:
-	return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4);
-    case FIB_PROTOCOL_IP6:
-	return (FIB_FORW_CHAIN_TYPE_UNICAST_IP6);
-    case FIB_PROTOCOL_MPLS:
         if (FIB_PATH_TYPE_RECURSIVE == path->fp_type &&
             MPLS_EOS == path->recursive.fp_nh.fp_eos)
         {
@@ -821,7 +814,10 @@ fib_path_to_chain_type (const fib_path_t *path)
             return (FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS);
         }
     }
-    return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4);
+    else
+    {
+        return (fib_forw_chain_type_from_dpo_proto(path->fp_nh_proto));
+    }
 }
 
 /*
@@ -927,7 +923,7 @@ FIXME comment
 
             ai = fib_path_attached_next_hop_get_adj(
                      path,
-                     fib_proto_to_link(path->fp_nh_proto));
+                     dpo_proto_to_link(path->fp_nh_proto));
 
             path->fp_oper_flags &= ~FIB_PATH_OPER_FLAG_RESOLVED;
             if (if_is_up && adj_is_up(ai))
@@ -935,9 +931,7 @@ FIXME comment
                 path->fp_oper_flags |= FIB_PATH_OPER_FLAG_RESOLVED;
             }
 
-            dpo_set(&path->fp_dpo, DPO_ADJACENCY,
-                    fib_proto_to_dpo(path->fp_nh_proto),
-                    ai);
+            dpo_set(&path->fp_dpo, DPO_ADJACENCY, path->fp_nh_proto, ai);
             adj_unlock(ai);
 
             if (!if_is_up)
@@ -1141,7 +1135,7 @@ fib_path_create (fib_node_index_t pl_index,
 	else
 	{
 	    path->fp_type = FIB_PATH_TYPE_RECURSIVE;
-	    if (FIB_PROTOCOL_MPLS == path->fp_nh_proto)
+	    if (DPO_PROTO_MPLS == path->fp_nh_proto)
 	    {
 		path->recursive.fp_nh.fp_local_label = rpath->frp_local_label;
                 path->recursive.fp_nh.fp_eos = rpath->frp_eos;
@@ -1167,7 +1161,7 @@ fib_path_create (fib_node_index_t pl_index,
  */
 fib_node_index_t
 fib_path_create_special (fib_node_index_t pl_index,
-			 fib_protocol_t nh_proto,
+			 dpo_proto_t nh_proto,
 			 fib_path_cfg_flags_t flags,
 			 const dpo_id_t *dpo)
 {
@@ -1433,7 +1427,7 @@ fib_path_cmp_w_route_path (fib_node_index_t path_index,
 	    res = (path->attached.fp_interface - rpath->frp_sw_if_index);
 	    break;
 	case FIB_PATH_TYPE_RECURSIVE:
-            if (FIB_PROTOCOL_MPLS == path->fp_nh_proto)
+            if (DPO_PROTO_MPLS == path->fp_nh_proto)
             {
                 res = path->recursive.fp_nh.fp_local_label - rpath->frp_local_label;
 
@@ -1535,8 +1529,7 @@ fib_path_recursive_loop_detect (fib_node_index_t path_index,
 	    FIB_PATH_DBG(path, "recursive loop formed");
 	    path->fp_oper_flags |= FIB_PATH_OPER_FLAG_RECURSIVE_LOOP;
 
-	    dpo_copy(&path->fp_dpo,
-                    drop_dpo_get(fib_proto_to_dpo(path->fp_nh_proto)));
+	    dpo_copy(&path->fp_dpo, drop_dpo_get(path->fp_nh_proto));
 	}
 	else
 	{
@@ -1590,8 +1583,7 @@ fib_path_resolve (fib_node_index_t path_index)
      */
     if (fib_path_is_permanent_drop(path))
     {
-	dpo_copy(&path->fp_dpo,
-                 drop_dpo_get(fib_proto_to_dpo(path->fp_nh_proto)));
+	dpo_copy(&path->fp_dpo, drop_dpo_get(path->fp_nh_proto));
 	path->fp_oper_flags &= ~FIB_PATH_OPER_FLAG_RESOLVED;
 	return (fib_path_is_resolved(path_index));
     }
@@ -1612,9 +1604,9 @@ fib_path_resolve (fib_node_index_t path_index)
 	}
         dpo_set(&path->fp_dpo,
                 DPO_ADJACENCY,
-                fib_proto_to_dpo(path->fp_nh_proto),
+                path->fp_nh_proto,
                 fib_path_attached_get_adj(path,
-                                          fib_proto_to_link(path->fp_nh_proto)));
+                                          dpo_proto_to_link(path->fp_nh_proto)));
 
 	/*
 	 * become a child of the adjacency so we receive updates
@@ -1639,7 +1631,7 @@ fib_path_resolve (fib_node_index_t path_index)
 
 	ASSERT(FIB_NODE_INDEX_INVALID == path->fp_via_fib);
 
-	if (FIB_PROTOCOL_MPLS == path->fp_nh_proto)
+	if (DPO_PROTO_MPLS == path->fp_nh_proto)
 	{
 	    fib_prefix_from_mpls_label(path->recursive.fp_nh.fp_local_label,
                                        path->recursive.fp_nh.fp_eos,
@@ -1680,8 +1672,7 @@ fib_path_resolve (fib_node_index_t path_index)
 	/*
 	 * Resolve via the drop
 	 */
-	dpo_copy(&path->fp_dpo,
-                 drop_dpo_get(fib_proto_to_dpo(path->fp_nh_proto)));
+	dpo_copy(&path->fp_dpo, drop_dpo_get(path->fp_nh_proto));
 	break;
     case FIB_PATH_TYPE_DEAG:
     {
@@ -1696,7 +1687,7 @@ fib_path_resolve (fib_node_index_t path_index)
                 LOOKUP_UNICAST);
 
         lookup_dpo_add_or_lock_w_fib_index(path->deag.fp_tbl_id,
-                                           fib_proto_to_dpo(path->fp_nh_proto),
+                                           path->fp_nh_proto,
                                            cast,
                                            LOOKUP_INPUT_DST_ADDR,
                                            LOOKUP_TABLE_FROM_CONFIG,
@@ -1707,7 +1698,7 @@ fib_path_resolve (fib_node_index_t path_index)
 	/*
 	 * Resolve via a receive DPO.
 	 */
-	receive_dpo_add_or_lock(fib_proto_to_dpo(path->fp_nh_proto),
+	receive_dpo_add_or_lock(path->fp_nh_proto,
                                 path->receive.fp_interface,
                                 &path->receive.fp_addr,
                                 &path->fp_dpo);
@@ -1716,7 +1707,7 @@ fib_path_resolve (fib_node_index_t path_index)
 	/*
 	 * Resolve via a receive DPO.
 	 */
-	interface_dpo_add_or_lock(fib_proto_to_dpo(path->fp_nh_proto),
+	interface_dpo_add_or_lock(path->fp_nh_proto,
                                   path->intf_rx.fp_interface,
                                   &path->fp_dpo);
 	break;
@@ -2035,7 +2026,7 @@ fib_path_contribute_forwarding (fib_node_index_t path_index,
                     /*
                      * Create the adj needed for sending IP multicast traffic
                      */
-                    ai = adj_mcast_add_or_lock(path->fp_nh_proto,
+                    ai = adj_mcast_add_or_lock(dpo_proto_to_fib(path->fp_nh_proto),
                                                fib_forw_chain_type_to_link_type(fct),
                                                path->attached.fp_interface);
                     dpo_set(dpo, DPO_ADJACENCY,
@@ -2187,7 +2178,7 @@ fib_path_encode (fib_node_index_t path_list_index,
     return (FIB_PATH_LIST_WALK_CONTINUE);
 }
 
-fib_protocol_t
+dpo_proto_t
 fib_path_get_proto (fib_node_index_t path_index)
 {
     fib_path_t *path;
diff --git a/src/vnet/fib/fib_path.h b/src/vnet/fib/fib_path.h
index a34cb43f..f986e437 100644
--- a/src/vnet/fib/fib_path.h
+++ b/src/vnet/fib/fib_path.h
@@ -78,6 +78,11 @@ typedef enum fib_path_cfg_attribute_t_ {
      * The path is an interface recieve
      */
     FIB_PATH_CFG_ATTRIBUTE_LOCAL,
+    /**
+     * The path is L2. i.e. the parameters therein are to be interpreted as
+     * pertaining to L2 config.
+     */
+    FIB_PATH_CFG_ATTRIBUTE_L2,
     /**
      * Marker. Add new types before this one, then update it.
      */
@@ -98,6 +103,7 @@ typedef enum fib_path_cfg_attribute_t_ {
     [FIB_PATH_CFG_ATTRIBUTE_ATTACHED] = "attached",	\
     [FIB_PATH_CFG_ATTRIBUTE_INTF_RX] = "interface-rx",	\
     [FIB_PATH_CFG_ATTRIBUTE_RPF_ID] = "rpf-id",         \
+    [FIB_PATH_CFG_ATTRIBUTE_L2] = "l2",         \
 }
 
 #define FOR_EACH_FIB_PATH_CFG_ATTRIBUTE(_item) \
@@ -118,6 +124,7 @@ typedef enum fib_path_cfg_flags_t_ {
     FIB_PATH_CFG_FLAG_ATTACHED = (1 << FIB_PATH_CFG_ATTRIBUTE_ATTACHED),
     FIB_PATH_CFG_FLAG_INTF_RX = (1 << FIB_PATH_CFG_ATTRIBUTE_INTF_RX),
     FIB_PATH_CFG_FLAG_RPF_ID = (1 << FIB_PATH_CFG_ATTRIBUTE_RPF_ID),
+    FIB_PATH_CFG_FLAG_L2 = (1 << FIB_PATH_CFG_ATTRIBUTE_L2),
 } __attribute__ ((packed)) fib_path_cfg_flags_t;
 
 
@@ -131,7 +138,7 @@ extern u8 * format_fib_path(u8 * s, va_list * args);
 extern fib_node_index_t fib_path_create(fib_node_index_t pl_index,
 					const fib_route_path_t *path);
 extern fib_node_index_t fib_path_create_special(fib_node_index_t pl_index,
-						fib_protocol_t nh_proto,
+						dpo_proto_t nh_proto,
 						fib_path_cfg_flags_t flags,
 						const dpo_id_t *dpo);
 
@@ -148,7 +155,7 @@ extern int fib_path_is_recursive_constrained(fib_node_index_t path_index);
 extern int fib_path_is_exclusive(fib_node_index_t path_index);
 extern int fib_path_is_deag(fib_node_index_t path_index);
 extern int fib_path_is_looped(fib_node_index_t path_index);
-extern fib_protocol_t fib_path_get_proto(fib_node_index_t path_index);
+extern dpo_proto_t fib_path_get_proto(fib_node_index_t path_index);
 extern void fib_path_destroy(fib_node_index_t path_index);
 extern uword fib_path_hash(fib_node_index_t path_index);
 extern load_balance_path_t * fib_path_append_nh_for_multipath_hash(
diff --git a/src/vnet/fib/fib_path_ext.c b/src/vnet/fib/fib_path_ext.c
index 26f2b9b6..4438671b 100644
--- a/src/vnet/fib/fib_path_ext.c
+++ b/src/vnet/fib/fib_path_ext.c
@@ -191,6 +191,9 @@ fib_path_ext_stack (fib_path_ext_t *path_ext,
     case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS:
         parent_fct = child_fct;
 	break;
+    case FIB_FORW_CHAIN_TYPE_ETHERNET:
+        parent_fct = FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS;
+	break;
     default:
         return (nhs);
 	break;
diff --git a/src/vnet/fib/fib_path_list.c b/src/vnet/fib/fib_path_list.c
index 7a9c328c..f30fd7ea 100644
--- a/src/vnet/fib/fib_path_list.c
+++ b/src/vnet/fib/fib_path_list.c
@@ -611,7 +611,7 @@ fib_path_list_get_resolving_interface (fib_node_index_t path_list_index)
     return (sw_if_index);
 }
 
-fib_protocol_t
+dpo_proto_t
 fib_path_list_get_proto (fib_node_index_t path_list_index)
 {
     fib_path_list_t *path_list;
@@ -753,7 +753,7 @@ fib_path_list_flags_2_path_flags (fib_path_list_flags_t plf)
 }
 
 fib_node_index_t
-fib_path_list_create_special (fib_protocol_t nh_proto,
+fib_path_list_create_special (dpo_proto_t nh_proto,
 			      fib_path_list_flags_t flags,
 			      const dpo_id_t *dpo)
 {
diff --git a/src/vnet/fib/fib_path_list.h b/src/vnet/fib/fib_path_list.h
index b4b6985b..a54b79e2 100644
--- a/src/vnet/fib/fib_path_list.h
+++ b/src/vnet/fib/fib_path_list.h
@@ -106,7 +106,7 @@ typedef enum fib_path_list_flags_t_ {
 
 extern fib_node_index_t fib_path_list_create(fib_path_list_flags_t flags,
 					     const fib_route_path_t *paths);
-extern fib_node_index_t fib_path_list_create_special(fib_protocol_t nh_proto,
+extern fib_node_index_t fib_path_list_create_special(dpo_proto_t nh_proto,
 						     fib_path_list_flags_t flags,
 						     const dpo_id_t *dpo);
 
@@ -150,7 +150,7 @@ extern int fib_path_list_recursive_loop_detect(fib_node_index_t path_list_index,
 extern u32 fib_path_list_get_resolving_interface(fib_node_index_t path_list_index);
 extern int fib_path_list_is_looped(fib_node_index_t path_list_index);
 extern int fib_path_list_is_popular(fib_node_index_t path_list_index);
-extern fib_protocol_t fib_path_list_get_proto(fib_node_index_t path_list_index);
+extern dpo_proto_t fib_path_list_get_proto(fib_node_index_t path_list_index);
 extern u8 * fib_path_list_format(fib_node_index_t pl_index,
 				 u8 * s);
 extern index_t fib_path_list_lb_map_add_or_lock(fib_node_index_t pl_index,
diff --git a/src/vnet/fib/fib_table.c b/src/vnet/fib/fib_table.c
index 5aa02dd0..6b6cc5cb 100644
--- a/src/vnet/fib/fib_table.c
+++ b/src/vnet/fib/fib_table.c
@@ -505,7 +505,7 @@ fib_table_entry_path_add (u32 fib_index,
 			  const fib_prefix_t *prefix,
 			  fib_source_t source,
 			  fib_entry_flag_t flags,
-			  fib_protocol_t next_hop_proto,
+			  dpo_proto_t next_hop_proto,
 			  const ip46_address_t *next_hop,
 			  u32 next_hop_sw_if_index,
 			  u32 next_hop_fib_index,
@@ -664,7 +664,7 @@ void
 fib_table_entry_path_remove (u32 fib_index,
 			     const fib_prefix_t *prefix,
 			     fib_source_t source,
-			     fib_protocol_t next_hop_proto,
+			     dpo_proto_t next_hop_proto,
 			     const ip46_address_t *next_hop,
 			     u32 next_hop_sw_if_index,
 			     u32 next_hop_fib_index,
@@ -755,7 +755,7 @@ fib_table_entry_update_one_path (u32 fib_index,
 				 const fib_prefix_t *prefix,
 				 fib_source_t source,
 				 fib_entry_flag_t flags,
-				 fib_protocol_t next_hop_proto,
+				 dpo_proto_t next_hop_proto,
 				 const ip46_address_t *next_hop,
 				 u32 next_hop_sw_if_index,
 				 u32 next_hop_fib_index,
diff --git a/src/vnet/fib/fib_table.h b/src/vnet/fib/fib_table.h
index a65fea74..579740e9 100644
--- a/src/vnet/fib/fib_table.h
+++ b/src/vnet/fib/fib_table.h
@@ -288,7 +288,7 @@ extern fib_node_index_t fib_table_entry_path_add(u32 fib_index,
 						 const fib_prefix_t *prefix,
 						 fib_source_t source,
 						 fib_entry_flag_t flags,
-						 fib_protocol_t next_hop_proto,
+						 dpo_proto_t next_hop_proto,
 						 const ip46_address_t *next_hop,
 						 u32 next_hop_sw_if_index,
 						 u32 next_hop_fib_index,
@@ -364,7 +364,7 @@ extern fib_node_index_t fib_table_entry_path_add2(u32 fib_index,
 extern void fib_table_entry_path_remove(u32 fib_index,
 					const fib_prefix_t *prefix,
 					fib_source_t source,
-					fib_protocol_t next_hop_proto,
+					dpo_proto_t next_hop_proto,
 					const ip46_address_t *next_hop,
 					u32 next_hop_sw_if_index,
 					u32 next_hop_fib_index,
@@ -471,7 +471,7 @@ extern fib_node_index_t fib_table_entry_update_one_path(u32 fib_index,
 							const fib_prefix_t *prefix,
 							fib_source_t source,
 							fib_entry_flag_t flags,
-							fib_protocol_t next_hop_proto,
+							dpo_proto_t next_hop_proto,
 							const ip46_address_t *next_hop,
 							u32 next_hop_sw_if_index,
 							u32 next_hop_fib_index,
diff --git a/src/vnet/fib/fib_test.c b/src/vnet/fib/fib_test.c
index 4c891667..59d5da2a 100644
--- a/src/vnet/fib/fib_test.c
+++ b/src/vnet/fib/fib_test.c
@@ -833,7 +833,7 @@ fib_test_v4 (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_ATTACHED),
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -861,7 +861,7 @@ fib_test_v4 (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_LOCAL),
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -911,7 +911,7 @@ fib_test_v4 (void)
     fib_table_entry_path_add(fib_index, &pfx,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_1,
 			     tm->hw[0]->sw_if_index,
 			     ~0, // invalid fib index
@@ -963,7 +963,7 @@ fib_test_v4 (void)
     pfx.fp_len = 0;
     fib_table_entry_path_remove(fib_index, &pfx,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_1,
 				tm->hw[0]->sw_if_index,
 				~0, // non-recursive path, so no FIB index
@@ -1029,7 +1029,7 @@ fib_test_v4 (void)
                                           &pfx_11_11_11_11_s_32,
                                           FIB_SOURCE_API,
                                           FIB_ENTRY_FLAG_ATTACHED,
-					  FIB_PROTOCOL_IP4,
+					  DPO_PROTO_IP4,
                                           &pfx_10_10_10_1_s_32.fp_addr,
                                           tm->hw[0]->sw_if_index,
                                           ~0, // invalid fib index
@@ -1095,7 +1095,7 @@ fib_test_v4 (void)
                                    &pfx_10_10_10_1_s_32,
                                    FIB_SOURCE_ADJ,
                                    FIB_ENTRY_FLAG_ATTACHED,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &pfx_10_10_10_1_s_32.fp_addr,
                                    tm->hw[0]->sw_if_index,
                                    ~0, // invalid fib index
@@ -1110,7 +1110,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
                                 &pfx_11_11_11_11_s_32,
                                 FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
                                 &pfx_10_10_10_1_s_32.fp_addr,
                                 tm->hw[0]->sw_if_index,
                                 ~0, // invalid fib index
@@ -1144,7 +1144,7 @@ fib_test_v4 (void)
                              &pfx_10_10_10_2_s_32,
                              FIB_SOURCE_ADJ,
                              FIB_ENTRY_FLAG_ATTACHED,
-                             FIB_PROTOCOL_IP4,
+                             DPO_PROTO_IP4,
                              &pfx_10_10_10_2_s_32.fp_addr,
                              tm->hw[0]->sw_if_index,
                              ~0, // invalid fib index
@@ -1181,7 +1181,7 @@ fib_test_v4 (void)
 			     &pfx_1_1_1_1_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_1,
 			     tm->hw[0]->sw_if_index,
 			     ~0, // invalid fib index
@@ -1214,7 +1214,7 @@ fib_test_v4 (void)
 			     &pfx_1_1_2_0_s_24,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_1,
 			     tm->hw[0]->sw_if_index,
 			     ~0, // invalid fib index
@@ -1241,7 +1241,7 @@ fib_test_v4 (void)
 			     &pfx_1_1_2_0_s_24,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_2,
 			     tm->hw[0]->sw_if_index,
 			     ~0, // invalid fib index
@@ -1280,7 +1280,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_1_1_2_0_s_24,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_2,
 				tm->hw[0]->sw_if_index,
 				~0,
@@ -1327,7 +1327,7 @@ fib_test_v4 (void)
 				   &bgp_100_pfx,
 				   FIB_SOURCE_API,
 				   FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &nh_1_1_1_1,
 				   ~0, // no index provided.
 				   fib_index, // nexthop in same fib as route
@@ -1363,7 +1363,7 @@ fib_test_v4 (void)
 			     &bgp_101_pfx,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_1_1_1_1,
 			     ~0, // no index provided.
 			     fib_index, // nexthop in same fib as route
@@ -1487,7 +1487,7 @@ fib_test_v4 (void)
                                    &bgp_200_pfx,
                                    FIB_SOURCE_API,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &pfx_1_1_1_2_s_32.fp_addr,
                                    ~0, // no index provided.
                                    fib_index, // nexthop in same fib as route
@@ -1534,7 +1534,7 @@ fib_test_v4 (void)
 			     &pfx_1_2_3_4_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
                              &nh_10_10_10_1,
                              tm->hw[0]->sw_if_index,
                              ~0,
@@ -1545,7 +1545,7 @@ fib_test_v4 (void)
                                    &pfx_1_2_3_4_s_32,
                                    FIB_SOURCE_API,
                                    FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
                                    &nh_12_12_12_12,
                                    tm->hw[1]->sw_if_index,
                                    ~0,
@@ -1586,7 +1586,7 @@ fib_test_v4 (void)
 			     &pfx_1_2_3_5_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
                              &nh_12_12_12_12,
                              tm->hw[1]->sw_if_index,
                              ~0,
@@ -1597,7 +1597,7 @@ fib_test_v4 (void)
                                    &pfx_1_2_3_5_s_32,
                                    FIB_SOURCE_API,
                                    FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
                                    &nh_10_10_10_1,
                                    tm->hw[0]->sw_if_index,
                                    ~0,
@@ -1669,7 +1669,7 @@ fib_test_v4 (void)
 				    &pfx_6_6_6_6_s_32,
 				    FIB_SOURCE_API,
 				    FIB_ENTRY_FLAG_NONE,
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    &nh_10_10_10_1,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -1688,7 +1688,7 @@ fib_test_v4 (void)
                              &pfx_6_6_6_6_s_32,
                              FIB_SOURCE_API,
                              FIB_ENTRY_FLAG_NONE,
-                             FIB_PROTOCOL_IP4,
+                             DPO_PROTO_IP4,
                              &nh_10_10_10_2,
                              tm->hw[0]->sw_if_index,
                              ~0, // invalid fib index
@@ -1770,7 +1770,7 @@ fib_test_v4 (void)
                              &pfx_6_6_6_6_s_32,
                              FIB_SOURCE_API,
                              FIB_ENTRY_FLAG_NONE,
-                             FIB_PROTOCOL_IP4,
+                             DPO_PROTO_IP4,
                              &nh_12_12_12_12,
                              tm->hw[1]->sw_if_index,
                              ~0, // invalid fib index
@@ -1915,7 +1915,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
                                 &pfx_6_6_6_6_s_32,
                                 FIB_SOURCE_API,
-                                FIB_PROTOCOL_IP4,
+                                DPO_PROTO_IP4,
                                 &nh_12_12_12_12,
                                 tm->hw[1]->sw_if_index,
                                 ~0, // invalid fib index
@@ -1995,7 +1995,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
                                 &pfx_6_6_6_6_s_32,
                                 FIB_SOURCE_API,
-                                FIB_PROTOCOL_IP4,
+                                DPO_PROTO_IP4,
                                 &nh_10_10_10_2,
                                 tm->hw[0]->sw_if_index,
                                 ~0, // invalid fib index
@@ -2026,7 +2026,7 @@ fib_test_v4 (void)
                                    &bgp_44_s_32,
                                    FIB_SOURCE_API,
                                    FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &pfx_1_2_3_4_s_32.fp_addr,
                                    ~0,
                                    fib_index,
@@ -2037,7 +2037,7 @@ fib_test_v4 (void)
                                    &bgp_44_s_32,
                                    FIB_SOURCE_API,
                                    FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &pfx_1_2_3_5_s_32.fp_addr,
                                    ~0,
                                    fib_index,
@@ -2107,7 +2107,7 @@ fib_test_v4 (void)
                                    &bgp_201_pfx,
                                    FIB_SOURCE_API,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &pfx_1_1_1_200_s_32.fp_addr,
                                    ~0, // no index provided.
                                    fib_index, // nexthop in same fib as route
@@ -2151,7 +2151,7 @@ fib_test_v4 (void)
 			     &pfx_1_1_1_0_s_24,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_1,
 			     tm->hw[0]->sw_if_index,
 			     ~0, // invalid fib index
@@ -2209,7 +2209,7 @@ fib_test_v4 (void)
 			     &pfx_1_1_1_0_s_28,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_2,
 			     tm->hw[0]->sw_if_index,
 			     ~0, // invalid fib index
@@ -2244,7 +2244,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_1_1_1_0_s_28,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_2,
 				tm->hw[0]->sw_if_index,
 				~0,
@@ -2275,7 +2275,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_1_1_1_0_s_24,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_1,
 				tm->hw[0]->sw_if_index,
 				~0,
@@ -2316,7 +2316,7 @@ fib_test_v4 (void)
 	                           &pfx_1_1_1_2_s_32,
 				   FIB_SOURCE_API,
 				   FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &nh_10_10_10_1,
 				   tm->hw[0]->sw_if_index,
 				   ~0, // invalid fib index
@@ -2351,7 +2351,7 @@ fib_test_v4 (void)
                                    &bgp_201_pfx,
                                    FIB_SOURCE_API,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &pfx_1_1_1_2_s_32.fp_addr,
                                    ~0,
                                    fib_index,
@@ -2362,7 +2362,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
                                 &bgp_201_pfx,
                                 FIB_SOURCE_API,
-                                FIB_PROTOCOL_IP4,
+                                DPO_PROTO_IP4,
                                 &pfx_1_1_1_2_s_32.fp_addr,
                                 ~0,
                                 fib_index,
@@ -2375,7 +2375,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&bgp_201_pfx,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&pfx_1_1_1_200_s_32.fp_addr,
 				~0, // no index provided.
 				fib_index,
@@ -2405,7 +2405,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&bgp_200_pfx,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&pfx_1_1_1_2_s_32.fp_addr,
 				~0, // no index provided.
 				fib_index,
@@ -2446,7 +2446,7 @@ fib_test_v4 (void)
 			     &bgp_102,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &pfx_1_1_1_1_s_32.fp_addr,
 			     ~0, // no index provided.
 			     fib_index, // same as route
@@ -2457,7 +2457,7 @@ fib_test_v4 (void)
 			     &bgp_102,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &pfx_1_1_1_2_s_32.fp_addr,
 			     ~0, // no index provided.
 			     fib_index, // same as route's FIB
@@ -2483,7 +2483,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&bgp_102,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&pfx_1_1_1_1_s_32.fp_addr,
 				~0, // no index provided.
 				fib_index, // same as route's FIB
@@ -2492,7 +2492,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&bgp_102,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&pfx_1_1_1_2_s_32.fp_addr,
 				~0, // no index provided.
 				fib_index, // same as route's FIB
@@ -2507,7 +2507,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&bgp_100_pfx,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&pfx_1_1_1_1_s_32.fp_addr,
 				~0, // no index provided.
 				fib_index, // same as route's FIB
@@ -2516,7 +2516,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&bgp_101_pfx,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&pfx_1_1_1_1_s_32.fp_addr,
 				~0, // no index provided.
 				fib_index, // same as route's FIB
@@ -2546,7 +2546,7 @@ fib_test_v4 (void)
 			     &bgp_200_pfx,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_1,
 			     ~0, // no index provided.
 			     fib_index, // Same as route's FIB
@@ -2593,7 +2593,7 @@ fib_test_v4 (void)
 			     &bgp_201_pfx,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_3,
 			     ~0, // no index provided.
 			     fib_index,
@@ -2639,7 +2639,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
     				&bgp_200_pfx,
     				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
     				&nh_10_10_10_1,
     				~0, // no index provided.
 				fib_index, // same as route's FIB
@@ -2648,7 +2648,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
     				&bgp_201_pfx,
     				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
     				&nh_10_10_10_3,
     				~0, // no index provided.
 				fib_index, // same as route's FIB
@@ -2707,7 +2707,7 @@ fib_test_v4 (void)
 			     &pfx_5_5_5_5_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &pfx_5_5_5_6_s_32.fp_addr,
 			     ~0, // no index provided.
 			     fib_index,
@@ -2718,7 +2718,7 @@ fib_test_v4 (void)
 			     &pfx_5_5_5_6_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &pfx_5_5_5_7_s_32.fp_addr,
 			     ~0, // no index provided.
 			     fib_index,
@@ -2729,7 +2729,7 @@ fib_test_v4 (void)
 			     &pfx_5_5_5_7_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &pfx_5_5_5_5_s_32.fp_addr,
 			     ~0, // no index provided.
 			     fib_index,
@@ -2768,7 +2768,7 @@ fib_test_v4 (void)
 			     &pfx_5_5_5_6_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_1,
 			     tm->hw[0]->sw_if_index,
 			     ~0,
@@ -2801,7 +2801,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_5_5_5_6_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_1,
 				tm->hw[0]->sw_if_index,
 				~0,
@@ -2826,7 +2826,7 @@ fib_test_v4 (void)
 				    &pfx_5_5_5_5_s_32,
 				    FIB_SOURCE_API,
 				    FIB_ENTRY_FLAG_NONE,
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    &nh_10_10_10_1,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -2868,7 +2868,7 @@ fib_test_v4 (void)
 				    &pfx_5_5_5_5_s_32,
 				    FIB_SOURCE_API,
 				    FIB_ENTRY_FLAG_NONE,
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    &pfx_5_5_5_6_s_32.fp_addr,
 				    ~0, // no index provided.
 				    fib_index,
@@ -2892,7 +2892,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_5_5_5_5_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&pfx_5_5_5_6_s_32.fp_addr,
 				~0, // no index provided.
 				fib_index, // same as route's FIB
@@ -2901,7 +2901,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_5_5_5_6_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&pfx_5_5_5_7_s_32.fp_addr,
 				~0, // no index provided.
 				fib_index, // same as route's FIB
@@ -2910,7 +2910,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_5_5_5_7_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&pfx_5_5_5_5_s_32.fp_addr,
 				~0, // no index provided.
 				fib_index, // same as route's FIB
@@ -2919,7 +2919,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_5_5_5_6_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_2,
 				~0, // no index provided.
 				fib_index, // same as route's FIB
@@ -2943,7 +2943,7 @@ fib_test_v4 (void)
 			     &pfx_5_5_5_6_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &pfx_5_5_5_6_s_32.fp_addr,
 			     ~0, // no index provided.
 			     fib_index,
@@ -2957,7 +2957,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_5_5_5_6_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&pfx_5_5_5_6_s_32.fp_addr,
 				~0, // no index provided.
 				fib_index, // same as route's FIB
@@ -2991,7 +2991,7 @@ fib_test_v4 (void)
 				   &pfx_23_23_23_0_s_24,
 				   FIB_SOURCE_API,
 				   FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &pfx_23_23_23_23_s_32.fp_addr,
 				   ~0, // recursive
 				   fib_index,
@@ -3021,7 +3021,7 @@ fib_test_v4 (void)
 				   &pfx_0_0_0_0_s_0,
 				   FIB_SOURCE_API,
 				   FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &pfx_23_23_23_23_s_32.fp_addr,
 				   ~0, // recursive
 				   fib_index,
@@ -3051,7 +3051,7 @@ fib_test_v4 (void)
 			     &bgp_200_pfx,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_1_1_1_1,
 			     ~0,
 			     fib_index,
@@ -3081,7 +3081,7 @@ fib_test_v4 (void)
 			     &pfx_1_1_1_0_s_28,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_1,
 			     tm->hw[0]->sw_if_index,
 			     ~0, // invalid fib index
@@ -3099,7 +3099,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_1_1_1_1_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_1,
 				tm->hw[0]->sw_if_index,
 				~0, // invalid fib index
@@ -3116,7 +3116,7 @@ fib_test_v4 (void)
 			     &pfx_1_1_1_1_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_1,
 			     tm->hw[0]->sw_if_index,
 			     ~0, // invalid fib index
@@ -3140,7 +3140,7 @@ fib_test_v4 (void)
 			     &pfx_1_1_1_3_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_2,
 			     tm->hw[0]->sw_if_index,
 			     ~0, // invalid fib index
@@ -3152,7 +3152,7 @@ fib_test_v4 (void)
 			     &bgp_200_pfx,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &pfx_1_1_1_3_s_32.fp_addr,
 			     ~0,
 			     fib_index,
@@ -3177,7 +3177,7 @@ fib_test_v4 (void)
                                  &bgp_78s[ii],
                                  FIB_SOURCE_API,
                                  FIB_ENTRY_FLAG_NONE,
-                                 FIB_PROTOCOL_IP4,
+                                 DPO_PROTO_IP4,
                                  &pfx_1_1_1_3_s_32.fp_addr,
                                  ~0,
                                  fib_index,
@@ -3188,7 +3188,7 @@ fib_test_v4 (void)
                                  &bgp_78s[ii],
                                  FIB_SOURCE_API,
                                  FIB_ENTRY_FLAG_NONE,
-                                 FIB_PROTOCOL_IP4,
+                                 DPO_PROTO_IP4,
                                  &nh_1_1_1_1,
                                  ~0,
                                  fib_index,
@@ -3238,7 +3238,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_1_1_1_1_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_1,
 				tm->hw[0]->sw_if_index,
 				~0, // invalid fib index
@@ -3277,7 +3277,7 @@ fib_test_v4 (void)
 			     &pfx_1_1_1_1_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_1,
 			     tm->hw[0]->sw_if_index,
 			     ~0, // invalid fib index
@@ -3307,7 +3307,7 @@ fib_test_v4 (void)
 			     &bgp_200_pfx,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &pfx_1_1_1_2_s_32.fp_addr,
 			     ~0,
 			     fib_index,
@@ -3320,7 +3320,7 @@ fib_test_v4 (void)
                                  &bgp_78s[ii],
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &pfx_1_1_1_2_s_32.fp_addr,
 			     ~0,
 			     fib_index,
@@ -3354,7 +3354,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_1_1_1_1_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_1,
 				tm->hw[0]->sw_if_index,
 				~0,
@@ -3391,7 +3391,7 @@ fib_test_v4 (void)
                              &pfx_1_1_1_1_s_32,
                              FIB_SOURCE_API,
                              FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
                              &nh_10_10_10_1,
                              tm->hw[0]->sw_if_index,
                              ~0,
@@ -3412,7 +3412,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
                                 &bgp_200_pfx,
                                 FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
                                 &pfx_1_1_1_2_s_32.fp_addr,
                                 ~0,
                                 fib_index,
@@ -3421,7 +3421,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&bgp_200_pfx,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_1_1_1_1,
 				~0,
 				fib_index,
@@ -3430,7 +3430,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&bgp_200_pfx,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&pfx_1_1_1_3_s_32.fp_addr,
 				~0,
 				fib_index,
@@ -3481,7 +3481,7 @@ fib_test_v4 (void)
 			     &pfx_4_4_4_4_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_1,
 			     tm->hw[0]->sw_if_index,
 			     ~0,
@@ -3492,7 +3492,7 @@ fib_test_v4 (void)
 			     &pfx_4_4_4_4_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_2,
 			     tm->hw[0]->sw_if_index,
 			     ~0,
@@ -3503,7 +3503,7 @@ fib_test_v4 (void)
 			     &pfx_4_4_4_4_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_3,
 			     tm->hw[0]->sw_if_index,
 			     ~0,
@@ -3539,7 +3539,7 @@ fib_test_v4 (void)
     for (ii = 0; ii < 4; ii++)
     {
 	fib_route_path_t r_path = {
-	    .frp_proto = FIB_PROTOCOL_IP4,
+	    .frp_proto = DPO_PROTO_IP4,
 	    .frp_addr = {
 		.ip4.as_u32 = clib_host_to_net_u32(0x0a0a0a02 + ii),
 	    },
@@ -3588,7 +3588,7 @@ fib_test_v4 (void)
 			     &pfx_4_4_4_4_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &zero_addr,
 			     ~0,
 			     fib_index,
@@ -3648,7 +3648,7 @@ fib_test_v4 (void)
                                    &pfx_34_34_1_1_s_32,
                                    FIB_SOURCE_API,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &nh_10_10_10_1,
                                    tm->hw[0]->sw_if_index,
                                    0,
@@ -3659,7 +3659,7 @@ fib_test_v4 (void)
                                    &pfx_34_1_1_1_s_32,
                                    FIB_SOURCE_API,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &pfx_34_34_1_1_s_32.fp_addr,
                                    ~0,
                                    fib_index,
@@ -3670,7 +3670,7 @@ fib_test_v4 (void)
                                    &pfx_34_1_1_1_s_32,
                                    FIB_SOURCE_API,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &pfx_34_34_1_1_s_32.fp_addr,
                                    ~0,
                                    fib_index,
@@ -3691,7 +3691,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_1_1_1_2_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_1,
 				tm->hw[0]->sw_if_index,
 				~0,
@@ -3700,7 +3700,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_1_1_1_1_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_1,
 				tm->hw[0]->sw_if_index,
 				~0,
@@ -3709,7 +3709,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_1_1_2_0_s_24,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_1,
 				tm->hw[0]->sw_if_index,
 				~0,
@@ -3751,7 +3751,7 @@ fib_test_v4 (void)
 			     &pfx_4_1_1_1_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &zero_addr,
 			     tm->hw[0]->sw_if_index,
 			     fib_index,
@@ -3805,7 +3805,7 @@ fib_test_v4 (void)
 				   &pfx_2001_s_64,
 				   FIB_SOURCE_API,
 				   FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &nh_10_10_10_1,
 				   tm->hw[0]->sw_if_index,
 				   fib_index,
@@ -3863,7 +3863,7 @@ fib_test_v4 (void)
                              &pfx_12_10_10_2_s_32,
                              FIB_SOURCE_ADJ,
                              FIB_ENTRY_FLAG_ATTACHED,
-                             FIB_PROTOCOL_IP4,
+                             DPO_PROTO_IP4,
                              &pfx_12_10_10_2_s_32.fp_addr,
                              tm->hw[0]->sw_if_index,
                              ~0, // invalid fib index
@@ -3897,7 +3897,7 @@ fib_test_v4 (void)
                              &pfx_10_10_10_127_s_32,
                              FIB_SOURCE_ADJ,
                              FIB_ENTRY_FLAG_ATTACHED,
-                             FIB_PROTOCOL_IP4,
+                             DPO_PROTO_IP4,
                              &pfx_10_10_10_127_s_32.fp_addr,
                              tm->hw[1]->sw_if_index,
                              ~0, // invalid fib index
@@ -3945,7 +3945,7 @@ fib_test_v4 (void)
                                    &pfx_10_10_10_3_s_32,
                                    FIB_SOURCE_ADJ,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &nh_10_10_10_3,
                                    tm->hw[0]->sw_if_index,
                                    fib_index,
@@ -3956,7 +3956,7 @@ fib_test_v4 (void)
                                    &pfx_10_10_10_3_s_32,
                                    FIB_SOURCE_ADJ,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &nh_12_12_12_12,
                                    tm->hw[1]->sw_if_index,
                                    fib_index,
@@ -3975,7 +3975,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
                                 &pfx_10_10_10_3_s_32,
                                 FIB_SOURCE_ADJ,
-                                FIB_PROTOCOL_IP4,
+                                DPO_PROTO_IP4,
                                 &nh_10_10_10_3,
                                 tm->hw[0]->sw_if_index,
                                 fib_index,
@@ -3992,7 +3992,7 @@ fib_test_v4 (void)
                                    &pfx_10_10_10_3_s_32,
                                    FIB_SOURCE_ADJ,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &nh_10_10_10_3,
                                    tm->hw[0]->sw_if_index,
                                    fib_index,
@@ -4011,7 +4011,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
                                 &pfx_10_10_10_3_s_32,
                                 FIB_SOURCE_ADJ,
-                                FIB_PROTOCOL_IP4,
+                                DPO_PROTO_IP4,
                                 &nh_12_12_12_12,
                                 tm->hw[1]->sw_if_index,
                                 fib_index,
@@ -4030,7 +4030,7 @@ fib_test_v4 (void)
     fib_table_entry_path_remove(fib_index,
                                 &pfx_10_10_10_3_s_32,
                                 FIB_SOURCE_ADJ,
-                                FIB_PROTOCOL_IP4,
+                                DPO_PROTO_IP4,
                                 &nh_10_10_10_3,
                                 tm->hw[0]->sw_if_index,
                                 fib_index,
@@ -4269,7 +4269,7 @@ fib_test_v6 (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_ATTACHED),
-				    FIB_PROTOCOL_IP6,
+				    DPO_PROTO_IP6,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0,
@@ -4300,7 +4300,7 @@ fib_test_v6 (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_LOCAL),
-				    FIB_PROTOCOL_IP6,
+				    DPO_PROTO_IP6,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -4345,7 +4345,7 @@ fib_test_v6 (void)
     fib_table_entry_path_add(fib_index, &pfx_0_0,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP6,
+			     DPO_PROTO_IP6,
 			     &nh_2001_2,
 			     tm->hw[0]->sw_if_index,
 			     ~0,
@@ -4389,7 +4389,7 @@ fib_test_v6 (void)
      */
     fib_table_entry_path_remove(fib_index, &pfx_0_0,
 				FIB_SOURCE_API,	
-				FIB_PROTOCOL_IP6,
+				DPO_PROTO_IP6,
 				&nh_2001_2,
 				tm->hw[0]->sw_if_index,
 				~0,
@@ -4466,7 +4466,7 @@ fib_test_v6 (void)
                              &pfx_2001_1_2_s_128,
                              FIB_SOURCE_ADJ,
                              FIB_ENTRY_FLAG_ATTACHED,
-                             FIB_PROTOCOL_IP6,
+                             DPO_PROTO_IP6,
                              &pfx_2001_1_2_s_128.fp_addr,
                              tm->hw[0]->sw_if_index,
                              ~0,
@@ -4505,7 +4505,7 @@ fib_test_v6 (void)
                              &pfx_2001_1_3_s_128,
                              FIB_SOURCE_ADJ,
                              FIB_ENTRY_FLAG_ATTACHED,
-                             FIB_PROTOCOL_IP6,
+                             DPO_PROTO_IP6,
                              &pfx_2001_1_3_s_128.fp_addr,
                              tm->hw[0]->sw_if_index,
                              ~0,
@@ -4559,7 +4559,7 @@ fib_test_v6 (void)
 			     &pfx_2001_a_s_64,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP6,
+			     DPO_PROTO_IP6,
 			     &nh_2001_2,
 			     tm->hw[0]->sw_if_index,
 			     ~0,
@@ -4573,7 +4573,7 @@ fib_test_v6 (void)
 			     &pfx_2001_b_s_64,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP6,
+			     DPO_PROTO_IP6,
 			     &nh_2001_2,
 			     tm->hw[0]->sw_if_index,
 			     ~0,
@@ -4608,7 +4608,7 @@ fib_test_v6 (void)
 				   &pfx_1_1_1_1_s_32,
 				   FIB_SOURCE_API,
 				   FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP6,
+				   DPO_PROTO_IP6,
 				   &nh_2001_2,
 				   tm->hw[0]->sw_if_index,
 				   ~0,
@@ -4646,7 +4646,7 @@ fib_test_v6 (void)
 			     &pfx_2001_c_s_64,
 			     FIB_SOURCE_CLI,
 			     FIB_ENTRY_FLAG_ATTACHED,
-			     FIB_PROTOCOL_IP6,
+			     DPO_PROTO_IP6,
 			     NULL,
 			     tm->hw[0]->sw_if_index,
 			     ~0,
@@ -4663,7 +4663,7 @@ fib_test_v6 (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_2001_c_s_64,
 				FIB_SOURCE_CLI,
-				FIB_PROTOCOL_IP6,
+				DPO_PROTO_IP6,
 				NULL,
 				tm->hw[0]->sw_if_index,
 				~0,
@@ -4748,7 +4748,7 @@ fib_test_v6 (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_ATTACHED),
-				    FIB_PROTOCOL_IP6,
+				    DPO_PROTO_IP6,
 				    NULL,
 				    tm->hw[1]->sw_if_index,
 				    ~0,
@@ -4767,7 +4767,7 @@ fib_test_v6 (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_LOCAL),
-				    FIB_PROTOCOL_IP6,
+				    DPO_PROTO_IP6,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -5095,7 +5095,7 @@ fib_test_ae (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_ATTACHED),
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0,
@@ -5111,7 +5111,7 @@ fib_test_ae (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_LOCAL),
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -5140,7 +5140,7 @@ fib_test_ae (void)
                              &pfx_10_10_10_1_s_32,
                              FIB_SOURCE_ADJ,
                              FIB_ENTRY_FLAG_ATTACHED,
-                             FIB_PROTOCOL_IP4,
+                             DPO_PROTO_IP4,
                              &pfx_10_10_10_1_s_32.fp_addr,
                              tm->hw[0]->sw_if_index,
                              ~0, // invalid fib index
@@ -5167,7 +5167,7 @@ fib_test_ae (void)
 				    &local_pfx,
 				    FIB_SOURCE_API,
 				    FIB_ENTRY_FLAG_NONE,
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -5209,7 +5209,7 @@ fib_test_ae (void)
                              &pfx_10_10_10_2_s_32,
                              FIB_SOURCE_ADJ,
                              FIB_ENTRY_FLAG_ATTACHED,
-                             FIB_PROTOCOL_IP4,
+                             DPO_PROTO_IP4,
                              &pfx_10_10_10_2_s_32.fp_addr,
                              tm->hw[0]->sw_if_index,
                              ~0, // invalid fib index
@@ -5243,7 +5243,7 @@ fib_test_ae (void)
 				    &local_pfx,
 				    FIB_SOURCE_API,
 				    FIB_ENTRY_FLAG_NONE,
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -5280,7 +5280,7 @@ fib_test_ae (void)
                              &pfx_10_10_10_3_s_32,
                              FIB_SOURCE_ADJ,
                              FIB_ENTRY_FLAG_ATTACHED,
-                             FIB_PROTOCOL_IP4,
+                             DPO_PROTO_IP4,
                              &pfx_10_10_10_3_s_32.fp_addr,
                              tm->hw[0]->sw_if_index,
                              ~0, // invalid fib index
@@ -5352,7 +5352,7 @@ fib_test_ae (void)
 				    &local_pfx,
 				    FIB_SOURCE_API,
 				    FIB_ENTRY_FLAG_NONE,
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    &pfx_10_10_10_2_s_32.fp_addr,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -5375,7 +5375,7 @@ fib_test_ae (void)
 				    &local_pfx,
 				    FIB_SOURCE_API,
 				    FIB_ENTRY_FLAG_NONE,
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -5407,7 +5407,7 @@ fib_test_ae (void)
                                           &pfx_10_0_0_0_s_8,
                                           FIB_SOURCE_API,
                                           FIB_ENTRY_FLAG_NONE,
-					  FIB_PROTOCOL_IP4,
+					  DPO_PROTO_IP4,
                                           &pfx_10_10_10_3_s_32.fp_addr,
                                           tm->hw[0]->sw_if_index,
                                           ~0, // invalid fib index
@@ -5463,7 +5463,7 @@ fib_test_ae (void)
                                           &local_pfx,
                                           FIB_SOURCE_API,
                                           FIB_ENTRY_FLAG_NONE,
-					  FIB_PROTOCOL_IP4,
+					  DPO_PROTO_IP4,
                                           &pfx_10_10_10_1_s_32.fp_addr,
                                           tm->hw[0]->sw_if_index,
                                           ~0, // invalid fib index
@@ -5499,7 +5499,7 @@ fib_test_ae (void)
 				    &local_pfx,
 				    FIB_SOURCE_API,
 				    FIB_ENTRY_FLAG_NONE,
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -5538,7 +5538,7 @@ fib_test_ae (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_ATTACHED),
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0,
@@ -5626,7 +5626,7 @@ fib_test_pref (void)
      * 2 high, 2 medium and 2 low preference non-recursive paths
      */
     fib_route_path_t nr_path_hi_1 = {
-        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_proto = DPO_PROTO_IP4,
         .frp_sw_if_index = tm->hw[0]->sw_if_index,
         .frp_fib_index = ~0,
         .frp_weight = 1,
@@ -5637,7 +5637,7 @@ fib_test_pref (void)
         },
     };
     fib_route_path_t nr_path_hi_2 = {
-        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_proto = DPO_PROTO_IP4,
         .frp_sw_if_index = tm->hw[0]->sw_if_index,
         .frp_fib_index = ~0,
         .frp_weight = 1,
@@ -5648,7 +5648,7 @@ fib_test_pref (void)
         },
     };
     fib_route_path_t nr_path_med_1 = {
-        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_proto = DPO_PROTO_IP4,
         .frp_sw_if_index = tm->hw[1]->sw_if_index,
         .frp_fib_index = ~0,
         .frp_weight = 1,
@@ -5659,7 +5659,7 @@ fib_test_pref (void)
         },
     };
     fib_route_path_t nr_path_med_2 = {
-        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_proto = DPO_PROTO_IP4,
         .frp_sw_if_index = tm->hw[1]->sw_if_index,
         .frp_fib_index = ~0,
         .frp_weight = 1,
@@ -5670,7 +5670,7 @@ fib_test_pref (void)
         },
     };
     fib_route_path_t nr_path_low_1 = {
-        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_proto = DPO_PROTO_IP4,
         .frp_sw_if_index = tm->hw[2]->sw_if_index,
         .frp_fib_index = ~0,
         .frp_weight = 1,
@@ -5681,7 +5681,7 @@ fib_test_pref (void)
         },
     };
     fib_route_path_t nr_path_low_2 = {
-        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_proto = DPO_PROTO_IP4,
         .frp_sw_if_index = tm->hw[2]->sw_if_index,
         .frp_fib_index = ~0,
         .frp_weight = 1,
@@ -5897,7 +5897,7 @@ fib_test_pref (void)
         },
     };
     fib_route_path_t r_path_hi = {
-        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_proto = DPO_PROTO_IP4,
         .frp_sw_if_index = ~0,
         .frp_fib_index = 0,
         .frp_weight = 1,
@@ -5906,7 +5906,7 @@ fib_test_pref (void)
         .frp_addr = pfx_1_1_1_1_s_32.fp_addr,
     };
     fib_route_path_t r_path_med = {
-        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_proto = DPO_PROTO_IP4,
         .frp_sw_if_index = ~0,
         .frp_fib_index = 0,
         .frp_weight = 1,
@@ -5915,7 +5915,7 @@ fib_test_pref (void)
         .frp_addr = pfx_1_1_1_2_s_32.fp_addr,
     };
     fib_route_path_t r_path_low = {
-        .frp_proto = FIB_PROTOCOL_IP4,
+        .frp_proto = DPO_PROTO_IP4,
         .frp_sw_if_index = ~0,
         .frp_fib_index = 0,
         .frp_weight = 1,
@@ -6099,7 +6099,7 @@ fib_test_label (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_ATTACHED),
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0,
@@ -6115,7 +6115,7 @@ fib_test_label (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_LOCAL),
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -6145,7 +6145,7 @@ fib_test_label (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_ATTACHED),
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[1]->sw_if_index,
 				    ~0,
@@ -6161,7 +6161,7 @@ fib_test_label (void)
 				    FIB_SOURCE_INTERFACE,
 				    (FIB_ENTRY_FLAG_CONNECTED |
 				     FIB_ENTRY_FLAG_LOCAL),
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    NULL,
 				    tm->hw[1]->sw_if_index,
 				    ~0, // invalid fib index
@@ -6243,7 +6243,7 @@ fib_test_label (void)
 				    &pfx_1_1_1_1_s_32,
 				    FIB_SOURCE_API,
 				    FIB_ENTRY_FLAG_NONE,
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    &nh_10_10_10_1,
 				    tm->hw[0]->sw_if_index,
 				    ~0, // invalid fib index
@@ -6282,7 +6282,7 @@ fib_test_label (void)
 				   &pfx_1_1_1_1_s_32,
 				   FIB_SOURCE_API,
 				   FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &nh_10_10_11_1,
 				   tm->hw[1]->sw_if_index,
 				   ~0, // invalid fib index
@@ -6360,7 +6360,7 @@ fib_test_label (void)
 				   &pfx_1_1_1_1_s_32,
 				   FIB_SOURCE_API,
 				   FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &nh_10_10_11_2,
 				   tm->hw[1]->sw_if_index,
 				   ~0, // invalid fib index
@@ -6440,7 +6440,7 @@ fib_test_label (void)
 				    &pfx_2_2_2_2_s_32,
 				    FIB_SOURCE_API,
 				    FIB_ENTRY_FLAG_NONE,
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    &pfx_1_1_1_1_s_32.fp_addr,
 				    ~0,
 				    fib_index,
@@ -6612,7 +6612,7 @@ fib_test_label (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_1_1_1_1_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_10_1,
 				tm->hw[0]->sw_if_index,
 				~0, // invalid fib index
@@ -6669,7 +6669,7 @@ fib_test_label (void)
     fib_table_entry_path_remove(fib_index,
 				&pfx_1_1_1_1_s_32,
 				FIB_SOURCE_API,
-				FIB_PROTOCOL_IP4,
+				DPO_PROTO_IP4,
 				&nh_10_10_11_1,
 				tm->hw[1]->sw_if_index,
 				~0, // invalid fib index
@@ -6711,7 +6711,7 @@ fib_test_label (void)
 			     &pfx_1_1_1_1_s_32,
 			     FIB_SOURCE_API,
 			     FIB_ENTRY_FLAG_NONE,
-			     FIB_PROTOCOL_IP4,
+			     DPO_PROTO_IP4,
 			     &nh_10_10_10_1,
 			     tm->hw[0]->sw_if_index,
 			     ~0, // invalid fib index
@@ -6842,7 +6842,7 @@ fib_test_label (void)
 					  &pfx_1_1_1_2_s_32,
 					  FIB_SOURCE_API,
 					  FIB_ENTRY_FLAG_NONE,
-					  FIB_PROTOCOL_IP4,
+					  DPO_PROTO_IP4,
 					  &nh_10_10_10_1,
 					  tm->hw[0]->sw_if_index,
 					  ~0, // invalid fib index
@@ -6884,7 +6884,7 @@ fib_test_label (void)
 				   &pfx_2_2_2_2_s_32,
 				   FIB_SOURCE_API,
 				   FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &pfx_1_1_1_2_s_32.fp_addr,
 				   ~0,
 				   fib_index,
@@ -6912,7 +6912,7 @@ fib_test_label (void)
 					  &pfx_1_1_1_2_s_32,
 					  FIB_SOURCE_API,
 					  FIB_ENTRY_FLAG_NONE,
-					  FIB_PROTOCOL_IP4,
+					  DPO_PROTO_IP4,
 					  &nh_10_10_11_1,
 					  tm->hw[1]->sw_if_index,
 					  ~0, // invalid fib index
@@ -6945,7 +6945,7 @@ fib_test_label (void)
 					  &pfx_1_1_1_2_s_32,
 					  FIB_SOURCE_API,
 					  FIB_ENTRY_FLAG_NONE,
-					  FIB_PROTOCOL_IP4,
+					  DPO_PROTO_IP4,
 					  &nh_10_10_11_1,
 					  tm->hw[1]->sw_if_index,
 					  ~0, // invalid fib index
@@ -6987,7 +6987,7 @@ fib_test_label (void)
 				    &pfx_2_2_2_3_s_32,
 				    FIB_SOURCE_API,
 				    FIB_ENTRY_FLAG_NONE,
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    &pfx_1_1_1_1_s_32.fp_addr,
 				    ~0,
 				    fib_index,
@@ -7031,7 +7031,7 @@ fib_test_label (void)
 				    &pfx_2_2_2_4_s_32,
 				    FIB_SOURCE_API,
 				    FIB_ENTRY_FLAG_NONE,
-				    FIB_PROTOCOL_IP4,
+				    DPO_PROTO_IP4,
 				    &pfx_1_1_1_1_s_32.fp_addr,
 				    ~0,
 				    fib_index,
@@ -7081,7 +7081,7 @@ fib_test_label (void)
 					  &pfx_2_2_5_5_s_32,
 					  FIB_SOURCE_API,
 					  FIB_ENTRY_FLAG_NONE,
-					  FIB_PROTOCOL_IP4,
+					  DPO_PROTO_IP4,
 					  &nh_10_10_11_1,
 					  tm->hw[1]->sw_if_index,
 					  ~0, // invalid fib index
@@ -7689,7 +7689,7 @@ fib_test_bfd (void)
                                     FIB_SOURCE_INTERFACE,
                                     (FIB_ENTRY_FLAG_CONNECTED |
                                      FIB_ENTRY_FLAG_ATTACHED),
-                                    FIB_PROTOCOL_IP4,
+                                    DPO_PROTO_IP4,
                                     NULL,
                                     tm->hw[0]->sw_if_index,
                                     ~0, // invalid fib index
@@ -7706,7 +7706,7 @@ fib_test_bfd (void)
                                     FIB_SOURCE_INTERFACE,
                                     (FIB_ENTRY_FLAG_CONNECTED |
                                      FIB_ENTRY_FLAG_LOCAL),
-                                    FIB_PROTOCOL_IP4,
+                                    DPO_PROTO_IP4,
                                     NULL,
                                     tm->hw[0]->sw_if_index,
                                     ~0, // invalid fib index
@@ -7780,7 +7780,7 @@ fib_test_bfd (void)
                                    &pfx_10_10_10_1_s_32,
                                    FIB_SOURCE_ADJ,
                                    FIB_ENTRY_FLAG_ATTACHED,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &nh_10_10_10_1,
                                    tm->hw[0]->sw_if_index,
                                    ~0, // invalid fib index
@@ -7819,7 +7819,7 @@ fib_test_bfd (void)
                              &pfx_10_10_10_2_s_32,
                              FIB_SOURCE_ADJ,
                              FIB_ENTRY_FLAG_ATTACHED,
-                             FIB_PROTOCOL_IP4,
+                             DPO_PROTO_IP4,
                              &nh_10_10_10_2,
                              tm->hw[0]->sw_if_index,
                              ~0, // invalid fib index
@@ -7851,7 +7851,7 @@ fib_test_bfd (void)
                              &pfx_10_10_10_2_s_32,
                              FIB_SOURCE_ADJ,
                              FIB_ENTRY_FLAG_ATTACHED,
-                             FIB_PROTOCOL_IP4,
+                             DPO_PROTO_IP4,
                              &nh_10_10_10_2,
                              tm->hw[0]->sw_if_index,
                              ~0, // invalid fib index
@@ -7907,7 +7907,7 @@ fib_test_bfd (void)
                                    &pfx_200_0_0_0_s_24,
                                    FIB_SOURCE_API,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &nh_10_10_10_2,
                                    ~0, // recursive
                                    0, // default fib index
@@ -7926,7 +7926,7 @@ fib_test_bfd (void)
                                    &pfx_200_0_0_0_s_24,
                                    FIB_SOURCE_API,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &nh_10_10_10_1,
                                    ~0, // recursive
                                    0, // default fib index
@@ -8065,7 +8065,7 @@ fib_test_bfd (void)
                                    &pfx_5_5_5_5_s_32,
                                    FIB_SOURCE_CLI,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &nh_10_10_10_1,
                                    tm->hw[0]->sw_if_index,
                                    ~0, // invalid fib index
@@ -8096,7 +8096,7 @@ fib_test_bfd (void)
                                    &pfx_5_5_5_5_s_32,
                                    FIB_SOURCE_CLI,
                                    FIB_ENTRY_FLAG_NONE,
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &nh_10_10_10_2,
                                    tm->hw[0]->sw_if_index,
                                    ~0, // invalid fib index
@@ -8234,7 +8234,7 @@ lfib_test (void)
 				   &pfx,
 				   FIB_SOURCE_CLI,
 				   FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &zero_addr,
 				   ~0,
 				   fib_index,
@@ -8285,7 +8285,7 @@ lfib_test (void)
 				   &pfx,
 				   FIB_SOURCE_CLI,
 				   FIB_ENTRY_FLAG_NONE,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
 				   &zero_addr,
 				   ~0,
 				   lfib_index,
@@ -8363,7 +8363,7 @@ lfib_test (void)
 					  &pfx_1200,
 					  FIB_SOURCE_API,
 					  FIB_ENTRY_FLAG_NONE,
-					  FIB_PROTOCOL_IP4,
+					  DPO_PROTO_IP4,
 					  &nh_10_10_10_1,
 					  tm->hw[0]->sw_if_index,
 					  ~0, // invalid fib index
@@ -8389,7 +8389,7 @@ lfib_test (void)
     	},
     };
     fib_route_path_t *rpaths = NULL, rpath = {
-    	.frp_proto = FIB_PROTOCOL_MPLS,
+        .frp_proto = DPO_PROTO_MPLS,
     	.frp_local_label = 1200,
         .frp_eos = MPLS_NON_EOS,
     	.frp_sw_if_index = ~0, // recurive
@@ -8545,7 +8545,7 @@ lfib_test (void)
 					  &pfx_2500,
 					  FIB_SOURCE_API,
 					  FIB_ENTRY_FLAG_NONE,
-					  FIB_PROTOCOL_IP4,
+					  DPO_PROTO_IP4,
 					  NULL,
 					  tm->hw[0]->sw_if_index,
 					  ~0, // invalid fib index
@@ -8590,7 +8590,7 @@ lfib_test (void)
 					  &pfx_3500,
 					  FIB_SOURCE_API,
 					  FIB_ENTRY_FLAG_MULTICAST,
-					  FIB_PROTOCOL_IP4,
+					  DPO_PROTO_IP4,
 					  &nh_10_10_10_1,
 					  tm->hw[0]->sw_if_index,
 					  ~0, // invalid fib index
@@ -8610,7 +8610,7 @@ lfib_test (void)
 				   &pfx_3500,
 				   FIB_SOURCE_API,
 				   FIB_ENTRY_FLAG_MULTICAST,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
                                    NULL,
                                    tm->hw[0]->sw_if_index,
                                    ~0, // invalid fib index
@@ -8637,7 +8637,7 @@ lfib_test (void)
 				   &pfx_3500,
 				   FIB_SOURCE_API,
 				   FIB_ENTRY_FLAG_MULTICAST,
-				   FIB_PROTOCOL_IP4,
+				   DPO_PROTO_IP4,
                                    NULL,
                                    5, // rpf-id
                                    0, // default table
diff --git a/src/vnet/fib/fib_types.h b/src/vnet/fib/fib_types.h
index a209ff3c..f11a55da 100644
--- a/src/vnet/fib/fib_types.h
+++ b/src/vnet/fib/fib_types.h
@@ -32,9 +32,9 @@ typedef u32 fib_node_index_t;
  * Protocol Type. packed so it consumes a u8 only
  */
 typedef enum fib_protocol_t_ {
-    FIB_PROTOCOL_IP4 = 0,
-    FIB_PROTOCOL_IP6,
-    FIB_PROTOCOL_MPLS,
+    FIB_PROTOCOL_IP4 = DPO_PROTO_IP4,
+    FIB_PROTOCOL_IP6 = DPO_PROTO_IP6,
+    FIB_PROTOCOL_MPLS = DPO_PROTO_MPLS,
 }  __attribute__ ((packed)) fib_protocol_t;
 
 #define FIB_PROTOCOLS {			\
@@ -338,7 +338,7 @@ typedef struct fib_route_path_t_ {
      * The protocol of the address below. We need this since the all
      * zeros address is ambiguous.
      */
-    fib_protocol_t frp_proto;
+    dpo_proto_t frp_proto;
 
     union {
 	/**
diff --git a/src/vnet/interface_format.c b/src/vnet/interface_format.c
index df7e9388..5694bb2f 100644
--- a/src/vnet/interface_format.c
+++ b/src/vnet/interface_format.c
@@ -165,9 +165,15 @@ format_vnet_sw_if_index_name (u8 * s, va_list * args)
 {
   vnet_main_t *vnm = va_arg (*args, vnet_main_t *);
   u32 sw_if_index = va_arg (*args, u32);
-  return format (s, "%U",
-		 format_vnet_sw_interface_name, vnm,
-		 vnet_get_sw_interface (vnm, sw_if_index));
+  vnet_sw_interface_t *si;
+
+  si = vnet_get_sw_interface_safe (vnm, sw_if_index);
+
+  if (NULL == si)
+    {
+      return format (s, "DELETED");
+    }
+  return format (s, "%U", format_vnet_sw_interface_name, vnm, si);
 }
 
 u8 *
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index ee17ea88..7a8d7a0c 100755
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -724,7 +724,7 @@ ip4_add_interface_routes (u32 sw_if_index,
                                        FIB_SOURCE_INTERFACE,
                                        (FIB_ENTRY_FLAG_CONNECTED |
                                         FIB_ENTRY_FLAG_ATTACHED),
-                                       FIB_PROTOCOL_IP4,
+                                       DPO_PROTO_IP4,
                                        /* No next-hop address */
                                        NULL,
                                        sw_if_index,
@@ -767,7 +767,7 @@ ip4_add_interface_routes (u32 sw_if_index,
       fib_table_entry_update_one_path (fib_index, &net_pfx,
                                        FIB_SOURCE_INTERFACE,
                                        (FIB_ENTRY_FLAG_ATTACHED),
-                                       FIB_PROTOCOL_IP4,
+                                       DPO_PROTO_IP4,
                                        &net_pfx.fp_addr,
                                        sw_if_index,
                                        // invalid FIB index
@@ -803,7 +803,7 @@ ip4_add_interface_routes (u32 sw_if_index,
                                    FIB_SOURCE_INTERFACE,
                                    (FIB_ENTRY_FLAG_CONNECTED |
                                     FIB_ENTRY_FLAG_LOCAL),
-                                   FIB_PROTOCOL_IP4,
+                                   DPO_PROTO_IP4,
                                    &pfx.fp_addr,
                                    sw_if_index,
                                    // invalid FIB index
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index bc66416e..8ae08a01 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -355,7 +355,7 @@ ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
 				       FIB_SOURCE_INTERFACE,
 				       (FIB_ENTRY_FLAG_CONNECTED |
 					FIB_ENTRY_FLAG_ATTACHED),
-				       FIB_PROTOCOL_IP6,
+				       DPO_PROTO_IP6,
 				       /* No next-hop address */
 				       NULL, sw_if_index,
 				       /* invalid FIB index */
@@ -390,7 +390,7 @@ ip6_add_interface_routes (vnet_main_t * vnm, u32 sw_if_index,
 				   FIB_SOURCE_INTERFACE,
 				   (FIB_ENTRY_FLAG_CONNECTED |
 				    FIB_ENTRY_FLAG_LOCAL),
-				   FIB_PROTOCOL_IP6,
+				   DPO_PROTO_IP6,
 				   &pfx.fp_addr,
 				   sw_if_index, ~0,
 				   1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c
index e8eebd4e..6a9139ab 100644
--- a/src/vnet/ip/ip6_neighbor.c
+++ b/src/vnet/ip/ip6_neighbor.c
@@ -284,7 +284,7 @@ ip6_neighbor_sw_interface_up_down (vnet_main_t * vnm,
 		(ip6_fib_table_get_index_for_sw_if_index (n->key.sw_if_index),
 		 &pfx,
 		 FIB_SOURCE_ADJ,
-		 FIB_PROTOCOL_IP6,
+		 DPO_PROTO_IP6,
 		 &pfx.fp_addr,
 		 n->key.sw_if_index, ~0, 1, FIB_ROUTE_PATH_FLAG_NONE);
 	      pool_put (nm->neighbor_pool, n);
@@ -645,7 +645,7 @@ vnet_set_ip6_ethernet_neighbor (vlib_main_t * vm,
 	  n->fib_entry_index =
 	    fib_table_entry_path_add (fib_index, &pfx, FIB_SOURCE_ADJ,
 				      FIB_ENTRY_FLAG_ATTACHED,
-				      FIB_PROTOCOL_IP6, &pfx.fp_addr,
+				      DPO_PROTO_IP6, &pfx.fp_addr,
 				      n->key.sw_if_index, ~0, 1, NULL,
 				      FIB_ROUTE_PATH_FLAG_NONE);
 	}
@@ -776,7 +776,7 @@ vnet_unset_ip6_ethernet_neighbor (vlib_main_t * vm,
 	(ip6_fib_table_get_index_for_sw_if_index (n->key.sw_if_index),
 	 &pfx,
 	 FIB_SOURCE_ADJ,
-	 FIB_PROTOCOL_IP6,
+	 DPO_PROTO_IP6,
 	 &pfx.fp_addr, n->key.sw_if_index, ~0, 1, FIB_ROUTE_PATH_FLAG_NONE);
     }
   pool_put (nm->neighbor_pool, n);
@@ -4110,7 +4110,7 @@ ip6_neighbor_proxy_add_del (u32 sw_if_index, ip6_address_t * addr, u8 is_del)
       fib_table_entry_path_remove (fib_index,
 				   &pfx,
 				   FIB_SOURCE_IP6_ND_PROXY,
-				   FIB_PROTOCOL_IP6,
+				   DPO_PROTO_IP6,
 				   &nh,
 				   sw_if_index,
 				   ~0, 1, FIB_ROUTE_PATH_FLAG_NONE);
@@ -4124,7 +4124,7 @@ ip6_neighbor_proxy_add_del (u32 sw_if_index, ip6_address_t * addr, u8 is_del)
 				&pfx,
 				FIB_SOURCE_IP6_ND_PROXY,
 				FIB_ENTRY_FLAG_NONE,
-				FIB_PROTOCOL_IP6,
+				DPO_PROTO_IP6,
 				&nh,
 				sw_if_index,
 				~0, 1, NULL, FIB_ROUTE_PATH_FLAG_NONE);
diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c
index 4cbf75a3..0676a387 100644
--- a/src/vnet/ip/ip_api.c
+++ b/src/vnet/ip/ip_api.c
@@ -156,9 +156,9 @@ copy_fib_next_hop (fib_route_path_encode_t * api_rpath, void *fp_arg)
   int is_ip4;
   vl_api_fib_path_t *fp = (vl_api_fib_path_t *) fp_arg;
 
-  if (api_rpath->rpath.frp_proto == FIB_PROTOCOL_IP4)
+  if (api_rpath->rpath.frp_proto == DPO_PROTO_IP4)
     fp->afi = IP46_TYPE_IP4;
-  else if (api_rpath->rpath.frp_proto == FIB_PROTOCOL_IP6)
+  else if (api_rpath->rpath.frp_proto == DPO_PROTO_IP6)
     fp->afi = IP46_TYPE_IP6;
   else
     {
@@ -714,7 +714,7 @@ add_del_route_t_handler (u8 is_multipath,
 			 u8 is_rpf_id,
 			 u32 fib_index,
 			 const fib_prefix_t * prefix,
-			 u8 next_hop_proto_is_ip4,
+			 dpo_proto_t next_hop_proto,
 			 const ip46_address_t * next_hop,
 			 u32 next_hop_sw_if_index,
 			 u8 next_hop_fib_index,
@@ -726,8 +726,7 @@ add_del_route_t_handler (u8 is_multipath,
   vnet_classify_main_t *cm = &vnet_classify_main;
   fib_route_path_flags_t path_flags = FIB_ROUTE_PATH_FLAG_NONE;
   fib_route_path_t path = {
-    .frp_proto = (next_hop_proto_is_ip4 ?
-		  FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6),
+    .frp_proto = next_hop_proto,
     .frp_addr = (NULL == next_hop ? zero_addr : *next_hop),
     .frp_sw_if_index = next_hop_sw_if_index,
     .frp_fib_index = next_hop_fib_index,
@@ -740,7 +739,7 @@ add_del_route_t_handler (u8 is_multipath,
 
   if (MPLS_LABEL_INVALID != next_hop_via_label)
     {
-      path.frp_proto = FIB_PROTOCOL_MPLS;
+      path.frp_proto = DPO_PROTO_MPLS;
       path.frp_local_label = next_hop_via_label;
       path.frp_eos = MPLS_NON_EOS;
     }
@@ -855,7 +854,7 @@ int
 add_del_route_check (fib_protocol_t table_proto,
 		     u32 table_id,
 		     u32 next_hop_sw_if_index,
-		     fib_protocol_t next_hop_table_proto,
+		     dpo_proto_t next_hop_table_proto,
 		     u32 next_hop_table_id,
 		     u8 create_missing_tables,
 		     u8 is_rpf_id, u32 * fib_index, u32 * next_hop_fib_index)
@@ -887,11 +886,18 @@ add_del_route_check (fib_protocol_t table_proto,
     }
   else
     {
+      fib_protocol_t fib_nh_proto;
+
+      if (next_hop_table_proto > DPO_PROTO_MPLS)
+	return (0);
+
+      fib_nh_proto = dpo_proto_to_fib (next_hop_table_proto);
+
       if (is_rpf_id)
-	*next_hop_fib_index = mfib_table_find (next_hop_table_proto,
+	*next_hop_fib_index = mfib_table_find (fib_nh_proto,
 					       ntohl (next_hop_table_id));
       else
-	*next_hop_fib_index = fib_table_find (next_hop_table_proto,
+	*next_hop_fib_index = fib_table_find (fib_nh_proto,
 					      ntohl (next_hop_table_id));
 
       if (~0 == *next_hop_fib_index)
@@ -900,12 +906,12 @@ add_del_route_check (fib_protocol_t table_proto,
 	    {
 	      if (is_rpf_id)
 		*next_hop_fib_index =
-		  mfib_table_find_or_create_and_lock (next_hop_table_proto,
+		  mfib_table_find_or_create_and_lock (fib_nh_proto,
 						      ntohl
 						      (next_hop_table_id));
 	      else
 		*next_hop_fib_index =
-		  fib_table_find_or_create_and_lock (next_hop_table_proto,
+		  fib_table_find_or_create_and_lock (fib_nh_proto,
 						     ntohl
 						     (next_hop_table_id));
 	    }
@@ -930,7 +936,7 @@ ip4_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
   rv = add_del_route_check (FIB_PROTOCOL_IP4,
 			    mp->table_id,
 			    mp->next_hop_sw_if_index,
-			    FIB_PROTOCOL_IP4,
+			    DPO_PROTO_IP4,
 			    mp->next_hop_table_id,
 			    mp->create_vrf_if_needed, 0,
 			    &fib_index, &next_hop_fib_index);
@@ -970,7 +976,7 @@ ip4_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
 				   mp->classify_table_index,
 				   mp->is_resolve_host,
 				   mp->is_resolve_attached, 0, 0,
-				   fib_index, &pfx, 1,
+				   fib_index, &pfx, DPO_PROTO_IP4,
 				   &nh,
 				   ntohl (mp->next_hop_sw_if_index),
 				   next_hop_fib_index,
@@ -990,7 +996,7 @@ ip6_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
   rv = add_del_route_check (FIB_PROTOCOL_IP6,
 			    mp->table_id,
 			    mp->next_hop_sw_if_index,
-			    FIB_PROTOCOL_IP6,
+			    DPO_PROTO_IP6,
 			    mp->next_hop_table_id,
 			    mp->create_vrf_if_needed, 0,
 			    &fib_index, &next_hop_fib_index);
@@ -1030,7 +1036,7 @@ ip6_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
 				   mp->classify_table_index,
 				   mp->is_resolve_host,
 				   mp->is_resolve_attached, 0, 0,
-				   fib_index, &pfx, 0,
+				   fib_index, &pfx, DPO_PROTO_IP6,
 				   &nh, ntohl (mp->next_hop_sw_if_index),
 				   next_hop_fib_index,
 				   mp->next_hop_weight,
@@ -1106,7 +1112,7 @@ mroute_add_del_handler (u8 is_add,
 
   fib_route_path_t path = {
     .frp_sw_if_index = next_hop_sw_if_index,
-    .frp_proto = prefix->fp_proto,
+    .frp_proto = fib_proto_to_dpo (prefix->fp_proto),
   };
 
   if (is_local)
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 533d010a..41e46070 100755
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -423,7 +423,7 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 	{
 	  rpath.frp_weight = 1;
 	  rpath.frp_eos = MPLS_NON_EOS;
-	  rpath.frp_proto = FIB_PROTOCOL_MPLS;
+	  rpath.frp_proto = DPO_PROTO_MPLS;
 	  rpath.frp_sw_if_index = ~0;
 	  vec_add1 (rpaths, rpath);
 	}
@@ -449,7 +449,7 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 			 &rpath.frp_sw_if_index))
 	{
 	  rpath.frp_weight = 1;
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  rpath.frp_proto = DPO_PROTO_IP4;
 	  vec_add1 (rpaths, rpath);
 	}
 
@@ -460,7 +460,7 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 			 &rpath.frp_sw_if_index))
 	{
 	  rpath.frp_weight = 1;
-	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  rpath.frp_proto = DPO_PROTO_IP6;
 	  vec_add1 (rpaths, rpath);
 	}
       else if (unformat (line_input, "weight %u", &weight))
@@ -479,7 +479,7 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 	{
 	  rpath.frp_weight = 1;
 	  rpath.frp_sw_if_index = ~0;
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  rpath.frp_proto = DPO_PROTO_IP4;
 	  vec_add1 (rpaths, rpath);
 	}
       else if (unformat (line_input, "via %U next-hop-table %d",
@@ -488,7 +488,7 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 	{
 	  rpath.frp_weight = 1;
 	  rpath.frp_sw_if_index = ~0;
-	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  rpath.frp_proto = DPO_PROTO_IP6;
 	  vec_add1 (rpaths, rpath);
 	}
       else if (unformat (line_input, "via %U",
@@ -501,7 +501,7 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 	  rpath.frp_fib_index = table_id;
 	  rpath.frp_weight = 1;
 	  rpath.frp_sw_if_index = ~0;
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  rpath.frp_proto = DPO_PROTO_IP4;
 	  vec_add1 (rpaths, rpath);
 	}
       else if (unformat (line_input, "via %U",
@@ -510,13 +510,13 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 	  rpath.frp_fib_index = table_id;
 	  rpath.frp_weight = 1;
 	  rpath.frp_sw_if_index = ~0;
-	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  rpath.frp_proto = DPO_PROTO_IP6;
 	  vec_add1 (rpaths, rpath);
 	}
       else if (unformat (line_input,
 			 "lookup in table %d", &rpath.frp_fib_index))
 	{
-	  rpath.frp_proto = pfx.fp_proto;
+	  rpath.frp_proto = fib_proto_to_dpo (pfx.fp_proto);
 	  rpath.frp_sw_if_index = ~0;
 	  vec_add1 (rpaths, rpath);
 	}
@@ -526,7 +526,7 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 			 &rpath.frp_sw_if_index))
 	{
 	  rpath.frp_weight = 1;
-	  rpath.frp_proto = prefixs[0].fp_proto;
+	  rpath.frp_proto = fib_proto_to_dpo (prefixs[0].fp_proto);
 	  vec_add1 (rpaths, rpath);
 	}
       else if (vec_len (prefixs) > 0 &&
diff --git a/src/vnet/lisp-gpe/lisp_gpe.c b/src/vnet/lisp-gpe/lisp_gpe.c
index 0acc7349..018895ad 100644
--- a/src/vnet/lisp-gpe/lisp_gpe.c
+++ b/src/vnet/lisp-gpe/lisp_gpe.c
@@ -454,7 +454,7 @@ vnet_gpe_add_del_native_fwd_rpath (vnet_gpe_native_fwd_rpath_args_t * a)
   fib_route_path_t *rpath;
   u8 ip_version;
 
-  ip_version = a->rpath.frp_proto == FIB_PROTOCOL_IP4 ? IP4 : IP6;
+  ip_version = a->rpath.frp_proto == DPO_PROTO_IP4 ? IP4 : IP6;
 
   if (a->is_add)
     {
@@ -511,7 +511,7 @@ gpe_native_forward_command_fn (vlib_main_t * vm, unformat_input_t * input,
 			 &rpath.frp_sw_if_index))
 	{
 	  rpath.frp_weight = 1;
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  rpath.frp_proto = DPO_PROTO_IP4;
 	}
       else if (unformat (line_input, "via %U %U",
 			 unformat_ip6_address,
@@ -520,21 +520,21 @@ gpe_native_forward_command_fn (vlib_main_t * vm, unformat_input_t * input,
 			 &rpath.frp_sw_if_index))
 	{
 	  rpath.frp_weight = 1;
-	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  rpath.frp_proto = DPO_PROTO_IP6;
 	}
       else if (unformat (line_input, "via %U",
 			 unformat_ip4_address, &rpath.frp_addr.ip4))
 	{
 	  rpath.frp_weight = 1;
 	  rpath.frp_sw_if_index = ~0;
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  rpath.frp_proto = DPO_PROTO_IP4;
 	}
       else if (unformat (line_input, "via %U",
 			 unformat_ip6_address, &rpath.frp_addr.ip6))
 	{
 	  rpath.frp_weight = 1;
 	  rpath.frp_sw_if_index = ~0;
-	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  rpath.frp_proto = DPO_PROTO_IP6;
 	}
       else
 	{
@@ -549,7 +549,8 @@ gpe_native_forward_command_fn (vlib_main_t * vm, unformat_input_t * input,
     }
   else
     {
-      rpath.frp_fib_index = fib_table_find (rpath.frp_proto, table_id);
+      rpath.frp_fib_index =
+	fib_table_find (dpo_proto_to_fib (rpath.frp_proto), table_id);
       if ((u32) ~ 0 == rpath.frp_fib_index)
 	{
 	  error = clib_error_return (0, "Nonexistent table id %d", table_id);
diff --git a/src/vnet/lisp-gpe/lisp_gpe_api.c b/src/vnet/lisp-gpe/lisp_gpe_api.c
index f1663699..4367a719 100644
--- a/src/vnet/lisp-gpe/lisp_gpe_api.c
+++ b/src/vnet/lisp-gpe/lisp_gpe_api.c
@@ -455,10 +455,10 @@ static void
     clib_memcpy (&a->rpath.frp_addr.ip6, mp->nh_addr, sizeof (ip6_address_t));
 
   a->is_add = mp->is_add;
-  a->rpath.frp_proto = mp->is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6;
-  a->rpath.frp_fib_index = fib_table_find (a->rpath.frp_proto,
-					   clib_net_to_host_u32
-					   (mp->table_id));
+  a->rpath.frp_proto = mp->is_ip4 ? DPO_PROTO_IP4 : DPO_PROTO_IP6;
+  a->rpath.frp_fib_index =
+    fib_table_find (dpo_proto_to_fib (a->rpath.frp_proto),
+		    clib_net_to_host_u32 (mp->table_id));
   if (~0 == a->rpath.frp_fib_index)
     {
       rv = VNET_API_ERROR_INVALID_VALUE;
@@ -484,7 +484,7 @@ gpe_native_fwd_rpaths_copy (vl_api_gpe_native_fwd_rpath_t * dst,
   vec_foreach (e, src)
   {
     memset (&dst[i], 0, sizeof (*dst));
-    table = fib_table_get (e->frp_fib_index, e->frp_proto);
+    table = fib_table_get (e->frp_fib_index, dpo_proto_to_fib (e->frp_proto));
     dst[i].fib_index = table->ft_table_id;
     dst[i].nh_sw_if_index = e->frp_sw_if_index;
     dst[i].is_ip4 = is_ip4;
diff --git a/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c b/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c
index 395b493a..ac048149 100644
--- a/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c
+++ b/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c
@@ -225,6 +225,7 @@ lisp_gpe_mk_fib_paths (const lisp_fwd_path_t * paths)
 {
   const lisp_gpe_adjacency_t *ladj;
   fib_route_path_t *rpaths = NULL;
+  fib_protocol_t fp;
   u8 best_priority;
   u32 ii;
 
@@ -239,9 +240,9 @@ lisp_gpe_mk_fib_paths (const lisp_fwd_path_t * paths)
 
     ladj = lisp_gpe_adjacency_get (paths[ii].lisp_adj);
 
-    ip_address_to_46 (&ladj->remote_rloc,
-		      &rpaths[ii].frp_addr, &rpaths[ii].frp_proto);
+    ip_address_to_46 (&ladj->remote_rloc, &rpaths[ii].frp_addr, &fp);
 
+    rpaths[ii].frp_proto = fib_proto_to_dpo (fp);
     rpaths[ii].frp_sw_if_index = ladj->sw_if_index;
     rpaths[ii].frp_weight = (paths[ii].weight ? paths[ii].weight : 1);
   }
diff --git a/src/vnet/mfib/ip6_mfib.c b/src/vnet/mfib/ip6_mfib.c
index 5c6f8126..5e48e919 100644
--- a/src/vnet/mfib/ip6_mfib.c
+++ b/src/vnet/mfib/ip6_mfib.c
@@ -158,7 +158,7 @@ ip6_create_mfib_with_table_id (u32 table_id)
         .fp_proto = FIB_PROTOCOL_IP6,
     };
     const fib_route_path_t path_for_us = {
-        .frp_proto = FIB_PROTOCOL_IP6,
+        .frp_proto = DPO_PROTO_IP6,
         .frp_addr = zero_addr,
         .frp_sw_if_index = 0xffffffff,
         .frp_fib_index = ~0,
@@ -222,7 +222,7 @@ ip6_mfib_table_destroy (ip6_mfib_t *mfib)
         .fp_proto = FIB_PROTOCOL_IP6,
     };
     const fib_route_path_t path_for_us = {
-        .frp_proto = FIB_PROTOCOL_IP6,
+        .frp_proto = DPO_PROTO_IP6,
         .frp_addr = zero_addr,
         .frp_sw_if_index = 0xffffffff,
         .frp_fib_index = ~0,
@@ -259,7 +259,7 @@ void
 ip6_mfib_interface_enable_disable (u32 sw_if_index, int is_enable)
 {
     const fib_route_path_t path = {
-        .frp_proto = FIB_PROTOCOL_IP6,
+        .frp_proto = DPO_PROTO_IP6,
         .frp_addr = zero_addr,
         .frp_sw_if_index = sw_if_index,
         .frp_fib_index = ~0,
diff --git a/src/vnet/mfib/mfib_entry.c b/src/vnet/mfib/mfib_entry.c
index cf25b67a..b37f8825 100644
--- a/src/vnet/mfib/mfib_entry.c
+++ b/src/vnet/mfib/mfib_entry.c
@@ -764,18 +764,16 @@ mfib_entry_update (fib_node_index_t mfib_entry_index,
          * entry
          */
         fib_node_index_t old_pl_index;
-        fib_protocol_t fp;
+        dpo_proto_t dp;
         dpo_id_t dpo = DPO_INVALID;
 
-        fp = mfib_entry_get_proto(mfib_entry);
+        dp = fib_proto_to_dpo(mfib_entry_get_proto(mfib_entry));
         old_pl_index = msrc->mfes_pl;
 
-        dpo_set(&dpo, DPO_REPLICATE,
-                fib_proto_to_dpo(fp),
-                repi);
+        dpo_set(&dpo, DPO_REPLICATE, dp, repi);
 
         msrc->mfes_pl =
-            fib_path_list_create_special(fp,
+            fib_path_list_create_special(dp,
                                          FIB_PATH_LIST_FLAG_EXCLUSIVE,
                                          &dpo);
 
diff --git a/src/vnet/mfib/mfib_test.c b/src/vnet/mfib/mfib_test.c
index 7c92ae99..57787eca 100644
--- a/src/vnet/mfib/mfib_test.c
+++ b/src/vnet/mfib/mfib_test.c
@@ -387,7 +387,7 @@ mfib_test_i (fib_protocol_t PROTO,
 
 
     fib_route_path_t path_via_if0 = {
-        .frp_proto = PROTO,
+        .frp_proto = fib_proto_to_dpo(PROTO),
         .frp_addr = zero_addr,
         .frp_sw_if_index = tm->hw[0]->sw_if_index,
         .frp_fib_index = ~0,
@@ -411,7 +411,7 @@ mfib_test_i (fib_protocol_t PROTO,
                                      MFIB_ITF_FLAG_ACCEPT));
 
     fib_route_path_t path_via_if1 = {
-        .frp_proto = PROTO,
+        .frp_proto = fib_proto_to_dpo(PROTO),
         .frp_addr = zero_addr,
         .frp_sw_if_index = tm->hw[1]->sw_if_index,
         .frp_fib_index = ~0,
@@ -419,7 +419,7 @@ mfib_test_i (fib_protocol_t PROTO,
         .frp_flags = 0,
     };
     fib_route_path_t path_via_if2 = {
-        .frp_proto = PROTO,
+        .frp_proto = fib_proto_to_dpo(PROTO),
         .frp_addr = zero_addr,
         .frp_sw_if_index = tm->hw[2]->sw_if_index,
         .frp_fib_index = ~0,
@@ -427,7 +427,7 @@ mfib_test_i (fib_protocol_t PROTO,
         .frp_flags = 0,
     };
     fib_route_path_t path_via_if3 = {
-        .frp_proto = PROTO,
+        .frp_proto = fib_proto_to_dpo(PROTO),
         .frp_addr = zero_addr,
         .frp_sw_if_index = tm->hw[3]->sw_if_index,
         .frp_fib_index = ~0,
@@ -435,7 +435,7 @@ mfib_test_i (fib_protocol_t PROTO,
         .frp_flags = 0,
     };
     fib_route_path_t path_for_us = {
-        .frp_proto = PROTO,
+        .frp_proto = fib_proto_to_dpo(PROTO),
         .frp_addr = zero_addr,
         .frp_sw_if_index = 0xffffffff,
         .frp_fib_index = ~0,
@@ -1121,7 +1121,7 @@ mfib_test_i (fib_protocol_t PROTO,
                                            &pfx_3500,
                                            FIB_SOURCE_API,
                                            FIB_ENTRY_FLAG_MULTICAST,
-                                           FIB_PROTOCOL_IP4,
+                                           DPO_PROTO_IP4,
                                            &nh_10_10_10_1,
                                            tm->hw[0]->sw_if_index,
                                            ~0, // invalid fib index
@@ -1138,7 +1138,7 @@ mfib_test_i (fib_protocol_t PROTO,
      * An (S,G) that resolves via the mLDP head-end
      */
     fib_route_path_t path_via_mldp = {
-        .frp_proto = FIB_PROTOCOL_MPLS,
+        .frp_proto = DPO_PROTO_MPLS,
         .frp_local_label = pfx_3500.fp_label,
         .frp_eos = MPLS_EOS,
         .frp_sw_if_index = 0xffffffff,
diff --git a/src/vnet/mpls/mpls.api b/src/vnet/mpls/mpls.api
index 67f1045d..5973a0a6 100644
--- a/src/vnet/mpls/mpls.api
+++ b/src/vnet/mpls/mpls.api
@@ -156,7 +156,7 @@ manual_endian manual_print define mpls_tunnel_details
     @param mr_is_interface_rx - Interface Receive path
     @param mr_is_interface_rx - RPF-ID Receive path. The next-hop interface
                                 is used as the RPF-ID
-    @param mr_next_hop_proto_is_ip4 - The next-hop is IPV4
+    @param mr_next_hop_proto - The next-hop protocol, of type dpo_proto_t
     @param mr_next_hop_weight - The weight, for UCMP
     @param mr_next_hop[16] - the nextop address
     @param mr_next_hop_sw_if_index - the next-hop SW interface
@@ -182,7 +182,7 @@ autoreply define mpls_route_add_del
   u8 mr_is_resolve_attached;
   u8 mr_is_interface_rx;
   u8 mr_is_rpf_id;
-  u8 mr_next_hop_proto_is_ip4;
+  u8 mr_next_hop_proto;
   u8 mr_next_hop_weight;
   u8 mr_next_hop_preference;
   u8 mr_next_hop[16];
diff --git a/src/vnet/mpls/mpls.c b/src/vnet/mpls/mpls.c
index 068d31f4..266ba42c 100644
--- a/src/vnet/mpls/mpls.c
+++ b/src/vnet/mpls/mpls.c
@@ -261,7 +261,7 @@ vnet_mpls_local_label (vlib_main_t * vm,
 			 &rpath.frp_sw_if_index,
 			 &rpath.frp_weight))
       {
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  rpath.frp_proto = DPO_PROTO_IP4;
 	  vec_add1(rpaths, rpath);
       }
 
@@ -272,7 +272,7 @@ vnet_mpls_local_label (vlib_main_t * vm,
 			 &rpath.frp_sw_if_index,
 			 &rpath.frp_weight))
       {
-	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  rpath.frp_proto = DPO_PROTO_IP6;
 	  vec_add1(rpaths, rpath);
       }
 
@@ -283,7 +283,7 @@ vnet_mpls_local_label (vlib_main_t * vm,
 			 &rpath.frp_sw_if_index))
       {
 	  rpath.frp_weight = 1;
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  rpath.frp_proto = DPO_PROTO_IP4;
 	  vec_add1(rpaths, rpath);
       }
       else if (unformat (line_input, "rx-ip4 %U",
@@ -291,7 +291,7 @@ vnet_mpls_local_label (vlib_main_t * vm,
 			 &rpath.frp_sw_if_index))
       {
 	  rpath.frp_weight = 1;
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  rpath.frp_proto = DPO_PROTO_IP4;
           rpath.frp_flags = FIB_ROUTE_PATH_INTF_RX;
 	  vec_add1(rpaths, rpath);
       }
@@ -302,7 +302,7 @@ vnet_mpls_local_label (vlib_main_t * vm,
 			 &rpath.frp_sw_if_index))
       {
 	  rpath.frp_weight = 1;
-	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  rpath.frp_proto = DPO_PROTO_IP6;
 	  vec_add1(rpaths, rpath);
       }
       else if (unformat (line_input, "via %U next-hop-table %d",
@@ -312,7 +312,7 @@ vnet_mpls_local_label (vlib_main_t * vm,
       {
 	  rpath.frp_weight = 1;
 	  rpath.frp_sw_if_index = ~0;
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  rpath.frp_proto = DPO_PROTO_IP4;
 	  vec_add1(rpaths, rpath);
       }
       else if (unformat (line_input, "via %U next-hop-table %d",
@@ -322,7 +322,7 @@ vnet_mpls_local_label (vlib_main_t * vm,
       {
 	  rpath.frp_weight = 1;
 	  rpath.frp_sw_if_index = ~0;
-	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  rpath.frp_proto = DPO_PROTO_IP6;
 	  vec_add1(rpaths, rpath);
       }
       else if (unformat (line_input, "via %U",
@@ -336,7 +336,7 @@ vnet_mpls_local_label (vlib_main_t * vm,
 	  rpath.frp_fib_index = table_id;
 	  rpath.frp_weight = 1;
 	  rpath.frp_sw_if_index = ~0;
-	  rpath.frp_proto = FIB_PROTOCOL_IP4;
+	  rpath.frp_proto = DPO_PROTO_IP4;
 	  vec_add1(rpaths, rpath);
       }
       else if (unformat (line_input, "via %U",
@@ -346,7 +346,7 @@ vnet_mpls_local_label (vlib_main_t * vm,
 	  rpath.frp_fib_index = table_id;
 	  rpath.frp_weight = 1;
 	  rpath.frp_sw_if_index = ~0;
-	  rpath.frp_proto = FIB_PROTOCOL_IP6;
+	  rpath.frp_proto = DPO_PROTO_IP6;
 	  vec_add1(rpaths, rpath);
       }
       else if (unformat (line_input, "%d", &local_label))
@@ -355,7 +355,7 @@ vnet_mpls_local_label (vlib_main_t * vm,
 			 "ip4-lookup-in-table %d",
 			 &rpath.frp_fib_index))
       {
-          rpath.frp_proto = FIB_PROTOCOL_IP4;
+          rpath.frp_proto = DPO_PROTO_IP4;
           rpath.frp_sw_if_index = FIB_NODE_INDEX_INVALID;
 	  pfx.fp_payload_proto = DPO_PROTO_IP4;
 	  vec_add1(rpaths, rpath);
@@ -364,7 +364,7 @@ vnet_mpls_local_label (vlib_main_t * vm,
 			 "ip6-lookup-in-table %d",
 			 &rpath.frp_fib_index))
       {
-          rpath.frp_proto = FIB_PROTOCOL_IP6;
+          rpath.frp_proto = DPO_PROTO_IP6;
           rpath.frp_sw_if_index = FIB_NODE_INDEX_INVALID;
 	  vec_add1(rpaths, rpath);
 	  pfx.fp_payload_proto = DPO_PROTO_IP6;
@@ -373,11 +373,21 @@ vnet_mpls_local_label (vlib_main_t * vm,
 			 "mpls-lookup-in-table %d",
 			 &rpath.frp_fib_index))
       {
-          rpath.frp_proto = FIB_PROTOCOL_MPLS;
+          rpath.frp_proto = DPO_PROTO_MPLS;
           rpath.frp_sw_if_index = FIB_NODE_INDEX_INVALID;
 	  pfx.fp_payload_proto = DPO_PROTO_MPLS;
 	  vec_add1(rpaths, rpath);
       }
+      else if (unformat (line_input,
+			 "l2-input-on %U",
+			 unformat_vnet_sw_interface, vnm,
+			 &rpath.frp_sw_if_index))
+      {
+          rpath.frp_proto = DPO_PROTO_ETHERNET;
+	  pfx.fp_payload_proto = DPO_PROTO_ETHERNET;
+          rpath.frp_flags = FIB_ROUTE_PATH_INTF_RX;
+	  vec_add1(rpaths, rpath);
+      }
       else if (unformat (line_input, "out-label %U",
                          unformat_mpls_unicast_label,
 			 &out_label))
@@ -440,7 +450,7 @@ vnet_mpls_local_label (vlib_main_t * vm,
       pfx.fp_proto = FIB_PROTOCOL_MPLS;
       pfx.fp_len = 21;
       pfx.fp_label = local_label;
-      pfx.fp_payload_proto = fib_proto_to_dpo(rpaths[0].frp_proto);
+      pfx.fp_payload_proto = rpaths[0].frp_proto;
 
       /*
        * the CLI parsing stored table Ids, swap to FIB indicies
diff --git a/src/vnet/mpls/mpls_api.c b/src/vnet/mpls/mpls_api.c
index 92fb24a6..737299e6 100644
--- a/src/vnet/mpls/mpls_api.c
+++ b/src/vnet/mpls/mpls_api.c
@@ -144,14 +144,7 @@ mpls_route_add_del_t_handler (vnet_main_t * vnm,
   };
   if (pfx.fp_eos)
     {
-      if (mp->mr_next_hop_proto_is_ip4)
-	{
-	  pfx.fp_payload_proto = DPO_PROTO_IP4;
-	}
-      else
-	{
-	  pfx.fp_payload_proto = DPO_PROTO_IP6;
-	}
+      pfx.fp_payload_proto = mp->mr_next_hop_proto;
     }
   else
     {
@@ -161,7 +154,7 @@ mpls_route_add_del_t_handler (vnet_main_t * vnm,
   rv = add_del_route_check (FIB_PROTOCOL_MPLS,
 			    mp->mr_table_id,
 			    mp->mr_next_hop_sw_if_index,
-			    dpo_proto_to_fib (pfx.fp_payload_proto),
+			    pfx.fp_payload_proto,
 			    mp->mr_next_hop_table_id,
 			    mp->mr_create_table_if_needed,
 			    mp->mr_is_rpf_id,
@@ -173,9 +166,9 @@ mpls_route_add_del_t_handler (vnet_main_t * vnm,
   ip46_address_t nh;
   memset (&nh, 0, sizeof (nh));
 
-  if (mp->mr_next_hop_proto_is_ip4)
+  if (DPO_PROTO_IP4 == mp->mr_next_hop_proto)
     memcpy (&nh.ip4, mp->mr_next_hop, sizeof (nh.ip4));
-  else
+  else if (DPO_PROTO_IP6 == mp->mr_next_hop_proto)
     memcpy (&nh.ip6, mp->mr_next_hop, sizeof (nh.ip6));
 
   n_labels = mp->mr_next_hop_n_out_labels;
@@ -202,7 +195,7 @@ mpls_route_add_del_t_handler (vnet_main_t * vnm,
 				   mp->mr_is_interface_rx,
 				   mp->mr_is_rpf_id,
 				   fib_index, &pfx,
-				   mp->mr_next_hop_proto_is_ip4,
+				   mp->mr_next_hop_proto,
 				   &nh, ntohl (mp->mr_next_hop_sw_if_index),
 				   next_hop_fib_index,
 				   mp->mr_next_hop_weight,
@@ -243,13 +236,13 @@ vl_api_mpls_tunnel_add_del_t_handler (vl_api_mpls_tunnel_add_del_t * mp)
 
   if (mp->mt_next_hop_proto_is_ip4)
     {
-      rpath.frp_proto = FIB_PROTOCOL_IP4;
+      rpath.frp_proto = DPO_PROTO_IP4;
       clib_memcpy (&rpath.frp_addr.ip4,
 		   mp->mt_next_hop, sizeof (rpath.frp_addr.ip4));
     }
   else
     {
-      rpath.frp_proto = FIB_PROTOCOL_IP6;
+      rpath.frp_proto = DPO_PROTO_IP6;
       clib_memcpy (&rpath.frp_addr.ip6,
 		   mp->mt_next_hop, sizeof (rpath.frp_addr.ip6));
     }
diff --git a/src/vnet/mpls/mpls_tunnel.c b/src/vnet/mpls/mpls_tunnel.c
index c025cc58..6452a60b 100644
--- a/src/vnet/mpls/mpls_tunnel.c
+++ b/src/vnet/mpls/mpls_tunnel.c
@@ -171,7 +171,7 @@ mpls_tunnel_mk_lb (mpls_tunnel_t *mt,
     vec_validate(ctx.next_hops, fib_path_list_get_n_paths(mt->mt_path_list));
     vec_reset_length(ctx.next_hops);
 
-    lb_proto = vnet_link_to_dpo_proto(linkt);
+    lb_proto = fib_forw_chain_type_to_dpo_proto(fct);
 
     fib_path_list_walk(mt->mt_path_list,
                        mpls_tunnel_collect_forwarding,
@@ -313,12 +313,34 @@ mpls_tunnel_restack (mpls_tunnel_t *mt)
     /*
      * walk all the adjacencies on the MPLS interface and restack them
      */
-    FOR_EACH_FIB_PROTOCOL(proto)
+    if (mt->mt_flags & MPLS_TUNNEL_FLAG_L2)
     {
-        adj_nbr_walk(mt->mt_sw_if_index,
-                     proto,
-                     mpls_adj_walk_cb,
-                     NULL);
+        /*
+         * Stack a load-balance that drops, whilst we have no paths
+         */
+        vnet_hw_interface_t * hi;
+        dpo_id_t dpo = DPO_INVALID;
+
+        mpls_tunnel_mk_lb(mt,
+                          VNET_LINK_MPLS,
+                          FIB_FORW_CHAIN_TYPE_ETHERNET,
+                          &dpo);
+
+        hi = vnet_get_hw_interface(vnet_get_main(), mt->mt_hw_if_index);
+        dpo_stack_from_node(hi->tx_node_index,
+                            &mt->mt_l2_lb,
+                            &dpo);
+        dpo_reset(&dpo);
+    }
+    else
+    {
+        FOR_EACH_FIB_PROTOCOL(proto)
+        {
+            adj_nbr_walk(mt->mt_sw_if_index,
+                         proto,
+                         mpls_adj_walk_cb,
+                         NULL);
+        }
     }
 }
 
@@ -495,7 +517,7 @@ mpls_tunnel_tx (vlib_main_t * vm,
 
           b0 = vlib_get_buffer(vm, bi0);
 
-          vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mt->mt_l2_adj;
+          vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mt->mt_l2_lb.dpoi_index;
 
           if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
             {
@@ -506,7 +528,7 @@ mpls_tunnel_tx (vlib_main_t * vm,
 
           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
                                            to_next, n_left_to_next,
-                                           bi0, mt->mt_l2_tx_arc);
+                                           bi0, mt->mt_l2_lb.dpoi_next_node);
         }
 
       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
@@ -565,8 +587,7 @@ vnet_mpls_tunnel_del (u32 sw_if_index)
     if (FIB_NODE_INDEX_INVALID != mt->mt_path_list)
         fib_path_list_child_remove(mt->mt_path_list,
                                    mt->mt_sibling_index);
-    if (ADJ_INDEX_INVALID != mt->mt_l2_adj)
-        adj_unlock(mt->mt_l2_adj);
+    dpo_reset(&mt->mt_l2_lb);
 
     vec_add1 (mpls_tunnel_free_hw_if_indices, mt->mt_hw_if_index);
     pool_put(mpls_tunnel_pool, mt);
@@ -587,12 +608,13 @@ vnet_mpls_tunnel_create (u8 l2_only,
     memset (mt, 0, sizeof (*mt));
     mti = mt - mpls_tunnel_pool;
     fib_node_init(&mt->mt_node, FIB_NODE_TYPE_MPLS_TUNNEL);
-    mt->mt_l2_adj = ADJ_INDEX_INVALID;
     mt->mt_path_list = FIB_NODE_INDEX_INVALID;
     mt->mt_sibling_index = FIB_NODE_INDEX_INVALID;
 
     if (is_multicast)
         mt->mt_flags |= MPLS_TUNNEL_FLAG_MCAST;
+    if (l2_only)
+        mt->mt_flags |= MPLS_TUNNEL_FLAG_L2;
 
     /*
      * Create a new, or re=use and old, tunnel HW interface
@@ -614,7 +636,7 @@ vnet_mpls_tunnel_create (u8 l2_only,
                                  mti,
                                  mpls_tunnel_hw_interface_class.index,
                                  mti);
-        hi = vnet_get_hw_interface(vnm, mt->mt_hw_if_index);
+        hi = vnet_get_hw_interface (vnm, mt->mt_hw_if_index);
     }
 
     /*
@@ -624,19 +646,6 @@ vnet_mpls_tunnel_create (u8 l2_only,
     vec_validate_init_empty(mpls_tunnel_db, mt->mt_sw_if_index, ~0);
     mpls_tunnel_db[mt->mt_sw_if_index] = mti;
 
-    if (l2_only)
-    {
-        mt->mt_l2_adj =
-            adj_nbr_add_or_lock(fib_path_list_get_proto(mt->mt_path_list),
-                                VNET_LINK_ETHERNET,
-                                &zero_addr,
-                                mt->mt_sw_if_index);
-
-        mt->mt_l2_tx_arc = vlib_node_add_named_next(vlib_get_main(),
-                                                    hi->tx_node_index,
-                                                    "adj-l2-midchain");
-    }
-
     return (mt->mt_sw_if_index);
 }
 
@@ -803,7 +812,7 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm,
                            &rpath.frp_sw_if_index))
         {
             rpath.frp_weight = 1;
-            rpath.frp_proto = FIB_PROTOCOL_IP4;
+            rpath.frp_proto = DPO_PROTO_IP4;
         }
 
         else if (unformat (line_input, "via %U %U",
@@ -813,7 +822,7 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm,
                            &rpath.frp_sw_if_index))
         {
             rpath.frp_weight = 1;
-            rpath.frp_proto = FIB_PROTOCOL_IP6;
+            rpath.frp_proto = DPO_PROTO_IP6;
         }
         else if (unformat (line_input, "via %U",
                            unformat_ip6_address,
@@ -822,7 +831,7 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm,
             rpath.frp_fib_index = 0;
             rpath.frp_weight = 1;
             rpath.frp_sw_if_index = ~0;
-            rpath.frp_proto = FIB_PROTOCOL_IP6;
+            rpath.frp_proto = DPO_PROTO_IP6;
         }
         else if (unformat (line_input, "via %U",
                            unformat_ip4_address,
@@ -831,7 +840,7 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm,
             rpath.frp_fib_index = 0;
             rpath.frp_weight = 1;
             rpath.frp_sw_if_index = ~0;
-            rpath.frp_proto = FIB_PROTOCOL_IP4;
+            rpath.frp_proto = DPO_PROTO_IP4;
         }
         else if (unformat (line_input, "l2-only"))
             l2_only = 1;
@@ -915,6 +924,14 @@ format_mpls_tunnel (u8 * s, va_list * args)
     s = format(s, "%U", format_fib_path_ext_list, &mt->mt_path_exts);
     s = format(s, "\n");
 
+    if (mt->mt_flags & MPLS_TUNNEL_FLAG_L2)
+    {
+        s = format(s, " forwarding: %U\n",
+                   format_fib_forw_chain_type,
+                   FIB_FORW_CHAIN_TYPE_ETHERNET);
+        s = format(s, " %U\n", format_dpo_id, &mt->mt_l2_lb, 2);
+    }
+
     return (s);
 }
 
diff --git a/src/vnet/mpls/mpls_tunnel.h b/src/vnet/mpls/mpls_tunnel.h
index 4cb0a860..285817c3 100644
--- a/src/vnet/mpls/mpls_tunnel.h
+++ b/src/vnet/mpls/mpls_tunnel.h
@@ -22,15 +22,20 @@
 typedef enum mpls_tunnel_attribute_t_
 {
     MPLS_TUNNEL_ATTRIBUTE_FIRST = 0,
+    /**
+     * @brief The tunnel is L2 only
+     */
+    MPLS_TUNNEL_ATTRIBUTE_L2 = MPLS_TUNNEL_ATTRIBUTE_FIRST,
     /**
      * @brief The tunnel has an underlying multicast LSP
      */
-    MPLS_TUNNEL_ATTRIBUTE_MCAST = MPLS_TUNNEL_ATTRIBUTE_FIRST,
+    MPLS_TUNNEL_ATTRIBUTE_MCAST,
     MPLS_TUNNEL_ATTRIBUTE_LAST = MPLS_TUNNEL_ATTRIBUTE_MCAST,
 } mpls_tunnel_attribute_t;
 
 #define MPLS_TUNNEL_ATTRIBUTES {		  \
     [MPLS_TUNNEL_ATTRIBUTE_MCAST]  = "multicast", \
+    [MPLS_TUNNEL_ATTRIBUTE_L2]     = "L2",   \
 }
 #define FOR_EACH_MPLS_TUNNEL_ATTRIBUTE(_item)		\
     for (_item = MPLS_TUNNEL_ATTRIBUTE_FIRST;		\
@@ -39,6 +44,7 @@ typedef enum mpls_tunnel_attribute_t_
 
 typedef enum mpls_tunnel_flag_t_ {
     MPLS_TUNNEL_FLAG_NONE   = 0,
+    MPLS_TUNNEL_FLAG_L2     = (1 << MPLS_TUNNEL_ATTRIBUTE_L2),
     MPLS_TUNNEL_FLAG_MCAST  = (1 << MPLS_TUNNEL_ATTRIBUTE_MCAST),
 } __attribute__ ((packed)) mpls_tunnel_flags_t;
 
@@ -60,14 +66,19 @@ typedef struct mpls_tunnel_t_
 
     /**
      * @brief If the tunnel is an L2 tunnel, this is the link type ETHERNET
-     * adjacency
+     * load-balance
+     */
+    dpo_id_t mt_l2_lb;
+
+    /**
+     * @brief The HW interface index of the tunnel interfaces
      */
-    adj_index_t mt_l2_adj;
+    u32 mt_hw_if_index;
 
     /**
-     * @brief on a L2 tunnel this is the VLIB arc from the L2-tx to the l2-midchain
+     * @brief The SW interface index of the tunnel interfaces
      */
-    u32 mt_l2_tx_arc;
+    u32 mt_sw_if_index;
 
     /**
      * @brief The path-list over which the tunnel's destination is reachable
@@ -83,23 +94,6 @@ typedef struct mpls_tunnel_t_
      * A vector of path extensions o hold the label stack for each path
      */
     fib_path_ext_list_t mt_path_exts;
-
-    /**
-     * @brief Flag to indicate the tunnel is only for L2 traffic, that is
-     * this tunnel belongs in a bridge domain.
-     */
-    u8 mt_l2_only;
-
-    /**
-     * @brief The HW interface index of the tunnel interfaces
-     */
-    u32 mt_hw_if_index;
-
-    /**
-     * @brief The SW interface index of the tunnel interfaces
-     */
-    u32 mt_sw_if_index;
-
 } mpls_tunnel_t;
 
 /**
diff --git a/src/vnet/srmpls/sr_mpls_policy.c b/src/vnet/srmpls/sr_mpls_policy.c
index 5ebbc60d..db4ad2a7 100755
--- a/src/vnet/srmpls/sr_mpls_policy.c
+++ b/src/vnet/srmpls/sr_mpls_policy.c
@@ -75,7 +75,7 @@ create_sl (mpls_sr_policy_t * sr_policy, mpls_label_t * sl, u32 weight)
   segment_list->segments = vec_dup (sl);
 
   fib_route_path_t path = {
-    .frp_proto = FIB_PROTOCOL_MPLS,
+    .frp_proto = DPO_PROTO_MPLS,
     .frp_sw_if_index = ~0,
     .frp_fib_index = 0,
     .frp_weight = segment_list->weight,
@@ -203,7 +203,7 @@ sr_mpls_policy_del (mpls_label_t bsid, u32 index)
     segment_list = pool_elt_at_index (sm->sid_lists, *sl_index);
 
     fib_route_path_t path = {
-      .frp_proto = FIB_PROTOCOL_MPLS,
+      .frp_proto = DPO_PROTO_MPLS,
       .frp_sw_if_index = ~0,
       .frp_fib_index = 0,
       .frp_weight = segment_list->weight,
@@ -308,7 +308,7 @@ sr_mpls_policy_mod (mpls_label_t bsid, u32 index, u8 operation,
 
       mpls_eos_bit_t eos;
       fib_route_path_t path = {
-	.frp_proto = FIB_PROTOCOL_MPLS,
+	.frp_proto = DPO_PROTO_MPLS,
 	.frp_sw_if_index = ~0,
 	.frp_fib_index = 0,
 	.frp_weight = segment_list->weight,
diff --git a/src/vnet/srmpls/sr_mpls_steering.c b/src/vnet/srmpls/sr_mpls_steering.c
index 37707049..3a9aea2d 100755
--- a/src/vnet/srmpls/sr_mpls_steering.c
+++ b/src/vnet/srmpls/sr_mpls_steering.c
@@ -218,7 +218,7 @@ sr_mpls_steering_policy (int is_del, mpls_label_t bsid, u32 sr_policy_index,
 update_fib:;
 
   fib_route_path_t path = {
-    .frp_proto = FIB_PROTOCOL_MPLS,
+    .frp_proto = DPO_PROTO_MPLS,
     .frp_local_label = sr_policy->bsid,
     .frp_eos = MPLS_EOS,
     .frp_sw_if_index = ~0,
diff --git a/src/vnet/srv6/sr_steering.c b/src/vnet/srv6/sr_steering.c
index a7903751..704adaa7 100755
--- a/src/vnet/srv6/sr_steering.c
+++ b/src/vnet/srv6/sr_steering.c
@@ -310,7 +310,7 @@ update_fib:
 						 table_id : 0)),
 				&pfx, FIB_SOURCE_SR,
 				FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT,
-				FIB_PROTOCOL_IP6,
+				DPO_PROTO_IP6,
 				(ip46_address_t *) & sr_policy->bsid, ~0,
 				sm->fib_table_ip6, 1, NULL,
 				FIB_ROUTE_PATH_FLAG_NONE);
@@ -327,7 +327,7 @@ update_fib:
 						 table_id : 0)),
 				&pfx, FIB_SOURCE_SR,
 				FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT,
-				FIB_PROTOCOL_IP6,
+				DPO_PROTO_IP6,
 				(ip46_address_t *) & sr_policy->bsid, ~0,
 				sm->fib_table_ip4, 1, NULL,
 				FIB_ROUTE_PATH_FLAG_NONE);
diff --git a/src/vnet/vxlan-gpe/vxlan_gpe.c b/src/vnet/vxlan-gpe/vxlan_gpe.c
index 97bb1b15..462c79a0 100644
--- a/src/vnet/vxlan-gpe/vxlan_gpe.c
+++ b/src/vnet/vxlan-gpe/vxlan_gpe.c
@@ -638,7 +638,7 @@ int vnet_vxlan_gpe_add_del_tunnel
               fib_node_index_t mfei;
               adj_index_t ai;
               fib_route_path_t path = {
-                  .frp_proto = fp,
+                  .frp_proto = fib_proto_to_dpo(fp),
                   .frp_addr = zero_addr,
                   .frp_sw_if_index = 0xffffffff,
                   .frp_fib_index = ~0,
diff --git a/src/vnet/vxlan/vxlan.c b/src/vnet/vxlan/vxlan.c
index 1b3df2a8..dc973372 100644
--- a/src/vnet/vxlan/vxlan.c
+++ b/src/vnet/vxlan/vxlan.c
@@ -505,7 +505,7 @@ int vnet_vxlan_add_del_tunnel
               fib_node_index_t mfei;
               adj_index_t ai;
               fib_route_path_t path = {
-                  .frp_proto = fp,
+                  .frp_proto = fib_proto_to_dpo(fp),
                   .frp_addr = zero_addr,
                   .frp_sw_if_index = 0xffffffff,
                   .frp_fib_index = ~0,
diff --git a/src/vpp/app/vpe_cli.c b/src/vpp/app/vpe_cli.c
index 94bdc84c..fcc496ad 100644
--- a/src/vpp/app/vpe_cli.c
+++ b/src/vpp/app/vpe_cli.c
@@ -98,7 +98,7 @@ virtual_ip_cmd_fn_command_fn (vlib_main_t * vm,
 
       vec_add2 (rpaths, rpath, 1);
 
-      rpath->frp_proto = FIB_PROTOCOL_IP4;
+      rpath->frp_proto = DPO_PROTO_IP4;
       rpath->frp_addr = next_hops[i];
       rpath->frp_sw_if_index = sw_if_index;
       rpath->frp_fib_index = ~0;
diff --git a/test/test_bfd.py b/test/test_bfd.py
index be42cdad..4cb6d379 100644
--- a/test/test_bfd.py
+++ b/test/test_bfd.py
@@ -20,7 +20,7 @@ from vpp_pg_interface import CaptureTimeoutError, is_ipv6_misc
 from vpp_lo_interface import VppLoInterface
 from util import ppp
 from vpp_papi_provider import UnexpectedApiReturnValueError
-from vpp_ip_route import VppIpRoute, VppRoutePath
+from vpp_ip_route import VppIpRoute, VppRoutePath, DpoProto
 
 USEC_IN_SEC = 1000000
 
@@ -1678,12 +1678,12 @@ class BFDFIBTestCase(VppTestCase):
         ip_2001_s_64 = VppIpRoute(self, "2001::", 64,
                                   [VppRoutePath(self.pg0.remote_ip6,
                                                 self.pg0.sw_if_index,
-                                                is_ip6=1)],
+                                                proto=DPO_PROTO_IP6)],
                                   is_ip6=1)
         ip_2002_s_64 = VppIpRoute(self, "2002::", 64,
                                   [VppRoutePath(self.pg0.remote_ip6,
                                                 0xffffffff,
-                                                is_ip6=1)],
+                                                proto=DPO_PROTO_IP6)],
                                   is_ip6=1)
         ip_2001_s_64.add_vpp_config()
         ip_2002_s_64.add_vpp_config()
diff --git a/test/test_gre.py b/test/test_gre.py
index 18b67dbd..1afc44fb 100644
--- a/test/test_gre.py
+++ b/test/test_gre.py
@@ -6,7 +6,7 @@ from logging import *
 from framework import VppTestCase, VppTestRunner
 from vpp_sub_interface import VppDot1QSubint
 from vpp_gre_interface import VppGreInterface, VppGre6Interface
-from vpp_ip_route import VppIpRoute, VppRoutePath
+from vpp_ip_route import VppIpRoute, VppRoutePath, DpoProto
 from vpp_papi_provider import L2_VTR_OP
 
 from scapy.packet import Raw
@@ -516,11 +516,12 @@ class TestGRE(VppTestCase):
         gre_if.admin_up()
         gre_if.config_ip6()
 
-        route_via_tun = VppIpRoute(self, "4004::1", 128,
-                                   [VppRoutePath("0::0",
-                                                 gre_if.sw_if_index,
-                                                 is_ip6=1)],
-                                   is_ip6=1)
+        route_via_tun = VppIpRoute(
+            self, "4004::1", 128,
+            [VppRoutePath("0::0",
+                          gre_if.sw_if_index,
+                          proto=DpoProto.DPO_PROTO_IP6)],
+            is_ip6=1)
 
         route_via_tun.add_vpp_config()
 
@@ -542,11 +543,12 @@ class TestGRE(VppTestCase):
         #
         # Add a route that resolves the tunnel's destination
         #
-        route_tun_dst = VppIpRoute(self, "1002::1", 128,
-                                   [VppRoutePath(self.pg2.remote_ip6,
-                                                 self.pg2.sw_if_index,
-                                                 is_ip6=1)],
-                                   is_ip6=1)
+        route_tun_dst = VppIpRoute(
+            self, "1002::1", 128,
+            [VppRoutePath(self.pg2.remote_ip6,
+                          self.pg2.sw_if_index,
+                          proto=DpoProto.DPO_PROTO_IP6)],
+            is_ip6=1)
         route_tun_dst.add_vpp_config()
 
         #
diff --git a/test/test_ip6.py b/test/test_ip6.py
index 593f6868..285ce181 100644
--- a/test/test_ip6.py
+++ b/test/test_ip6.py
@@ -8,7 +8,7 @@ from vpp_sub_interface import VppSubInterface, VppDot1QSubint
 from vpp_pg_interface import is_ipv6_misc
 from vpp_ip_route import VppIpRoute, VppRoutePath, find_route, VppIpMRoute, \
     VppMRoutePath, MRouteItfFlags, MRouteEntryFlags, VppMplsIpBind, \
-    VppMplsRoute
+    VppMplsRoute, DpoProto
 from vpp_neighbor import find_nbr, VppNeighbor
 
 from scapy.packet import Raw
@@ -490,7 +490,7 @@ class TestIPv6(TestIPv6ND):
                                     inet=AF_INET6))
 
     def test_ns_duplicates(self):
-        """ ARP Duplicates"""
+        """ ND Duplicates"""
 
         #
         # Generate some hosts on the LAN
@@ -537,7 +537,7 @@ class TestIPv6(TestIPv6ND):
 
         #
         # remove the duplicate on pg1
-        # packet stream shoud generate ARPs out of pg1
+        # packet stream shoud generate NSs out of pg1
         #
         ns_pg1.remove_vpp_config()
 
@@ -1347,10 +1347,10 @@ class TestIP6LoadBalance(VppTestCase):
         route_3000_1 = VppIpRoute(self, "3000::1", 128,
                                   [VppRoutePath(self.pg1.remote_ip6,
                                                 self.pg1.sw_if_index,
-                                                is_ip6=1),
+                                                proto=DpoProto.DPO_PROTO_IP6),
                                    VppRoutePath(self.pg2.remote_ip6,
                                                 self.pg2.sw_if_index,
-                                                is_ip6=1)],
+                                                proto=DpoProto.DPO_PROTO_IP6)],
                                   is_ip6=1)
         route_3000_1.add_vpp_config()
 
@@ -1367,11 +1367,11 @@ class TestIP6LoadBalance(VppTestCase):
                                 [VppRoutePath(self.pg1.remote_ip6,
                                               self.pg1.sw_if_index,
                                               labels=[67],
-                                              is_ip6=1),
+                                              proto=DpoProto.DPO_PROTO_IP6),
                                  VppRoutePath(self.pg2.remote_ip6,
                                               self.pg2.sw_if_index,
                                               labels=[67],
-                                              is_ip6=1)])
+                                              proto=DpoProto.DPO_PROTO_IP6)])
         route_67.add_vpp_config()
 
         #
@@ -1441,20 +1441,20 @@ class TestIP6LoadBalance(VppTestCase):
         route_3000_2 = VppIpRoute(self, "3000::2", 128,
                                   [VppRoutePath(self.pg3.remote_ip6,
                                                 self.pg3.sw_if_index,
-                                                is_ip6=1),
+                                                proto=DpoProto.DPO_PROTO_IP6),
                                    VppRoutePath(self.pg4.remote_ip6,
                                                 self.pg4.sw_if_index,
-                                                is_ip6=1)],
+                                                proto=DpoProto.DPO_PROTO_IP6)],
                                   is_ip6=1)
         route_3000_2.add_vpp_config()
 
         route_4000_1 = VppIpRoute(self, "4000::1", 128,
                                   [VppRoutePath("3000::1",
                                                 0xffffffff,
-                                                is_ip6=1),
+                                                proto=DpoProto.DPO_PROTO_IP6),
                                    VppRoutePath("3000::2",
                                                 0xffffffff,
-                                                is_ip6=1)],
+                                                proto=DpoProto.DPO_PROTO_IP6)],
                                   is_ip6=1)
         route_4000_1.add_vpp_config()
 
@@ -1485,14 +1485,14 @@ class TestIP6LoadBalance(VppTestCase):
         route_5000_2 = VppIpRoute(self, "5000::2", 128,
                                   [VppRoutePath(self.pg3.remote_ip6,
                                                 self.pg3.sw_if_index,
-                                                is_ip6=1)],
+                                                proto=DpoProto.DPO_PROTO_IP6)],
                                   is_ip6=1)
         route_5000_2.add_vpp_config()
 
         route_6000_1 = VppIpRoute(self, "6000::1", 128,
                                   [VppRoutePath("5000::2",
                                                 0xffffffff,
-                                                is_ip6=1)],
+                                                proto=DpoProto.DPO_PROTO_IP6)],
                                   is_ip6=1)
         route_6000_1.add_vpp_config()
 
diff --git a/test/test_map.py b/test/test_map.py
index 9ac3948a..bbf4aec2 100644
--- a/test/test_map.py
+++ b/test/test_map.py
@@ -4,7 +4,7 @@ import unittest
 import socket
 
 from framework import VppTestCase, VppTestRunner
-from vpp_ip_route import VppIpRoute, VppRoutePath
+from vpp_ip_route import VppIpRoute, VppRoutePath, DpoProto
 
 from scapy.layers.l2 import Ether, Raw
 from scapy.layers.inet import IP, UDP, ICMP
@@ -75,7 +75,7 @@ class TestMAP(VppTestCase):
                                map_br_pfx_len,
                                [VppRoutePath(self.pg1.remote_ip6,
                                              self.pg1.sw_if_index,
-                                             is_ip6=1)],
+                                             proto=DpoProto.DPO_PROTO_IP6)],
                                is_ip6=1)
         map_route.add_vpp_config()
 
@@ -138,13 +138,12 @@ class TestMAP(VppTestCase):
         # Add a route to 4001::1. Expect the encapped traffic to be
         # sent via that routes next-hop
         #
-        pre_res_route = VppIpRoute(self,
-                                   "4001::1",
-                                   128,
-                                   [VppRoutePath(self.pg1.remote_hosts[2].ip6,
-                                                 self.pg1.sw_if_index,
-                                                 is_ip6=1)],
-                                   is_ip6=1)
+        pre_res_route = VppIpRoute(
+            self, "4001::1", 128,
+            [VppRoutePath(self.pg1.remote_hosts[2].ip6,
+                          self.pg1.sw_if_index,
+                          proto=DpoProto.DPO_PROTO_IP6)],
+            is_ip6=1)
         pre_res_route.add_vpp_config()
 
         self.send_and_assert_encapped(v4, map_src,
@@ -156,7 +155,7 @@ class TestMAP(VppTestCase):
         #
         pre_res_route.modify([VppRoutePath(self.pg1.remote_hosts[3].ip6,
                                            self.pg1.sw_if_index,
-                                           is_ip6=1)])
+                                           proto=DpoProto.DPO_PROTO_IP6)])
         pre_res_route.add_vpp_config()
 
         self.send_and_assert_encapped(v4, map_src,
diff --git a/test/test_mpls.py b/test/test_mpls.py
index e3d013af..b2226a74 100644
--- a/test/test_mpls.py
+++ b/test/test_mpls.py
@@ -6,7 +6,7 @@ import socket
 from framework import VppTestCase, VppTestRunner
 from vpp_ip_route import VppIpRoute, VppRoutePath, VppMplsRoute, \
     VppMplsIpBind, VppIpMRoute, VppMRoutePath, \
-    MRouteItfFlags, MRouteEntryFlags
+    MRouteItfFlags, MRouteEntryFlags, DpoProto
 from vpp_mpls_tunnel_interface import VppMPLSTunnelInterface
 
 from scapy.packet import Raw
@@ -16,6 +16,38 @@ from scapy.layers.inet6 import IPv6
 from scapy.contrib.mpls import MPLS
 
 
+def verify_filter(capture, sent):
+    if not len(capture) == len(sent):
+        # filter out any IPv6 RAs from the capture
+        for p in capture:
+            if p.haslayer(IPv6):
+                capture.remove(p)
+    return capture
+
+
+def verify_mpls_stack(tst, rx, mpls_labels, ttl=255, num=0):
+    # the rx'd packet has the MPLS label popped
+    eth = rx[Ether]
+    tst.assertEqual(eth.type, 0x8847)
+
+    rx_mpls = rx[MPLS]
+
+    for ii in range(len(mpls_labels)):
+        tst.assertEqual(rx_mpls.label, mpls_labels[ii])
+        tst.assertEqual(rx_mpls.cos, 0)
+        if ii == num:
+            tst.assertEqual(rx_mpls.ttl, ttl)
+        else:
+            tst.assertEqual(rx_mpls.ttl, 255)
+        if ii == len(mpls_labels) - 1:
+            tst.assertEqual(rx_mpls.s, 1)
+        else:
+            # not end of stack
+            tst.assertEqual(rx_mpls.s, 0)
+            # pop the label to expose the next
+            rx_mpls = rx_mpls[MPLS].payload
+
+
 class TestMPLS(VppTestCase):
     """ MPLS Test Case """
 
@@ -120,18 +152,9 @@ class TestMPLS(VppTestCase):
             pkts.append(p)
         return pkts
 
-    @staticmethod
-    def verify_filter(capture, sent):
-        if not len(capture) == len(sent):
-            # filter out any IPv6 RAs from the capture
-            for p in capture:
-                if p.haslayer(IPv6):
-                    capture.remove(p)
-        return capture
-
     def verify_capture_ip4(self, src_if, capture, sent, ping_resp=0):
         try:
-            capture = self.verify_filter(capture, sent)
+            capture = verify_filter(capture, sent)
 
             self.assertEqual(len(capture), len(sent))
 
@@ -158,33 +181,10 @@ class TestMPLS(VppTestCase):
         except:
             raise
 
-    def verify_mpls_stack(self, rx, mpls_labels, ttl=255, num=0):
-        # the rx'd packet has the MPLS label popped
-        eth = rx[Ether]
-        self.assertEqual(eth.type, 0x8847)
-
-        rx_mpls = rx[MPLS]
-
-        for ii in range(len(mpls_labels)):
-            self.assertEqual(rx_mpls.label, mpls_labels[ii])
-            self.assertEqual(rx_mpls.cos, 0)
-            if ii == num:
-                self.assertEqual(rx_mpls.ttl, ttl)
-            else:
-                self.assertEqual(rx_mpls.ttl, 255)
-
-            if ii == len(mpls_labels) - 1:
-                self.assertEqual(rx_mpls.s, 1)
-            else:
-                # not end of stack
-                self.assertEqual(rx_mpls.s, 0)
-                # pop the label to expose the next
-                rx_mpls = rx_mpls[MPLS].payload
-
     def verify_capture_labelled_ip4(self, src_if, capture, sent,
                                     mpls_labels):
         try:
-            capture = self.verify_filter(capture, sent)
+            capture = verify_filter(capture, sent)
 
             self.assertEqual(len(capture), len(sent))
 
@@ -195,8 +195,8 @@ class TestMPLS(VppTestCase):
                 rx_ip = rx[IP]
 
                 # the MPLS TTL is copied from the IP
-                self.verify_mpls_stack(
-                    rx, mpls_labels, rx_ip.ttl, len(mpls_labels) - 1)
+                verify_mpls_stack(self, rx, mpls_labels, rx_ip.ttl,
+                                  len(mpls_labels) - 1)
 
                 self.assertEqual(rx_ip.src, tx_ip.src)
                 self.assertEqual(rx_ip.dst, tx_ip.dst)
@@ -211,7 +211,7 @@ class TestMPLS(VppTestCase):
         if top is None:
             top = len(mpls_labels) - 1
         try:
-            capture = self.verify_filter(capture, sent)
+            capture = verify_filter(capture, sent)
 
             self.assertEqual(len(capture), len(sent))
 
@@ -222,8 +222,7 @@ class TestMPLS(VppTestCase):
                 rx_ip = rx[IP]
 
                 # the MPLS TTL is 255 since it enters a new tunnel
-                self.verify_mpls_stack(
-                    rx, mpls_labels, ttl, top)
+                verify_mpls_stack(self, rx, mpls_labels, ttl, top)
 
                 self.assertEqual(rx_ip.src, tx_ip.src)
                 self.assertEqual(rx_ip.dst, tx_ip.dst)
@@ -236,13 +235,13 @@ class TestMPLS(VppTestCase):
     def verify_capture_labelled(self, src_if, capture, sent,
                                 mpls_labels, ttl=254, num=0):
         try:
-            capture = self.verify_filter(capture, sent)
+            capture = verify_filter(capture, sent)
 
             self.assertEqual(len(capture), len(sent))
 
             for i in range(len(capture)):
                 rx = capture[i]
-                self.verify_mpls_stack(rx, mpls_labels, ttl, num)
+                verify_mpls_stack(self, rx, mpls_labels, ttl, num)
         except:
             raise
 
@@ -1049,7 +1048,7 @@ class TestMPLS(VppTestCase):
                           self.pg1.sw_if_index,
                           nh_table_id=1,
                           rpf_id=55,
-                          is_ip6=1)],
+                          proto=DpoProto.DPO_PROTO_IP6)],
             is_multicast=1)
 
         route_34_eos.add_vpp_config()
@@ -1440,19 +1439,20 @@ class TestMPLSPIC(VppTestCase):
         for ii in range(64):
             dst = "3000::%d" % ii
             local_label = 1600 + ii
-            vpn_routes.append(VppIpRoute(self, dst, 128,
-                                         [VppRoutePath(self.pg2.remote_ip6,
-                                                       0xffffffff,
-                                                       nh_table_id=1,
-                                                       is_resolve_attached=1,
-                                                       is_ip6=1),
-                                          VppRoutePath(self.pg3.remote_ip6,
-                                                       0xffffffff,
-                                                       nh_table_id=1,
-                                                       is_ip6=1,
-                                                       is_resolve_attached=1)],
-                                         table_id=1,
-                                         is_ip6=1))
+            vpn_routes.append(VppIpRoute(
+                self, dst, 128,
+                [VppRoutePath(self.pg2.remote_ip6,
+                              0xffffffff,
+                              nh_table_id=1,
+                              is_resolve_attached=1,
+                              proto=DpoProto.DPO_PROTO_IP6),
+                 VppRoutePath(self.pg3.remote_ip6,
+                              0xffffffff,
+                              nh_table_id=1,
+                              proto=DpoProto.DPO_PROTO_IP6,
+                              is_resolve_attached=1)],
+                table_id=1,
+                is_ip6=1))
             vpn_routes[ii].add_vpp_config()
 
             vpn_bindings.append(VppMplsIpBind(self, local_label, dst, 128,
@@ -1525,5 +1525,211 @@ class TestMPLSPIC(VppTestCase):
         self.assertNotEqual(0, len(rx1))
 
 
+class TestMPLSL2(VppTestCase):
+    """ MPLS-L2 """
+
+    def setUp(self):
+        super(TestMPLSL2, self).setUp()
+
+        # create 2 pg interfaces
+        self.create_pg_interfaces(range(2))
+
+        # use pg0 as the core facing interface
+        self.pg0.admin_up()
+        self.pg0.config_ip4()
+        self.pg0.resolve_arp()
+        self.pg0.enable_mpls()
+
+        # use the other 2 for customer facg L2 links
+        for i in self.pg_interfaces[1:]:
+            i.admin_up()
+
+    def tearDown(self):
+        super(TestMPLSL2, self).tearDown()
+        for i in self.pg_interfaces[1:]:
+            i.admin_down()
+
+        self.pg0.disable_mpls()
+        self.pg0.unconfig_ip4()
+        self.pg0.admin_down()
+
+    def verify_capture_tunneled_ethernet(self, capture, sent, mpls_labels,
+                                         ttl=255, top=None):
+        if top is None:
+            top = len(mpls_labels) - 1
+
+        capture = verify_filter(capture, sent)
+
+        self.assertEqual(len(capture), len(sent))
+
+        for i in range(len(capture)):
+            tx = sent[i]
+            rx = capture[i]
+
+            # the MPLS TTL is 255 since it enters a new tunnel
+            verify_mpls_stack(self, rx, mpls_labels, ttl, top)
+
+            tx_eth = tx[Ether]
+            rx_eth = Ether(str(rx[MPLS].payload))
+
+            self.assertEqual(rx_eth.src, tx_eth.src)
+            self.assertEqual(rx_eth.dst, tx_eth.dst)
+
+    def test_vpws(self):
+        """ Virtual Private Wire Service """
+
+        #
+        # Create an MPLS tunnel that pushes 1 label
+        #
+        mpls_tun_1 = VppMPLSTunnelInterface(self,
+                                            [VppRoutePath(self.pg0.remote_ip4,
+                                                          self.pg0.sw_if_index,
+                                                          labels=[42])],
+                                            is_l2=1)
+        mpls_tun_1.add_vpp_config()
+        mpls_tun_1.admin_up()
+
+        #
+        # Create a label entry to for 55 that does L2 input to the tunnel
+        #
+        route_55_eos = VppMplsRoute(
+            self, 55, 1,
+            [VppRoutePath("0.0.0.0",
+                          mpls_tun_1.sw_if_index,
+                          is_interface_rx=1,
+                          proto=DpoProto.DPO_PROTO_ETHERNET)])
+        route_55_eos.add_vpp_config()
+
+        #
+        # Cross-connect the tunnel with one of the customers L2 interfaces
+        #
+        self.vapi.sw_interface_set_l2_xconnect(self.pg1.sw_if_index,
+                                               mpls_tun_1.sw_if_index,
+                                               enable=1)
+        self.vapi.sw_interface_set_l2_xconnect(mpls_tun_1.sw_if_index,
+                                               self.pg1.sw_if_index,
+                                               enable=1)
+
+        #
+        # inject a packet from the core
+        #
+        pcore = (Ether(dst=self.pg0.local_mac,
+                       src=self.pg0.remote_mac) /
+                 MPLS(label=55, ttl=64) /
+                 Ether(dst="00:00:de:ad:ba:be",
+                       src="00:00:de:ad:be:ef") /
+                 IP(src="10.10.10.10", dst="11.11.11.11") /
+                 UDP(sport=1234, dport=1234) /
+                 Raw('\xa5' * 100))
+
+        self.pg0.add_stream(pcore * 65)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx0 = self.pg1.get_capture(65)
+        tx = pcore[MPLS].payload
+
+        self.assertEqual(rx0[0][Ether].dst, tx[Ether].dst)
+        self.assertEqual(rx0[0][Ether].src, tx[Ether].src)
+
+        #
+        # Inject a packet from the custoer/L2 side
+        #
+        self.pg1.add_stream(tx * 65)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx0 = self.pg0.get_capture(65)
+
+        self.verify_capture_tunneled_ethernet(rx0, tx*65, [42])
+
+    def test_vpls(self):
+        """ Virtual Private LAN Service """
+        #
+        # Create an L2 MPLS tunnel
+        #
+        mpls_tun = VppMPLSTunnelInterface(self,
+                                          [VppRoutePath(self.pg0.remote_ip4,
+                                                        self.pg0.sw_if_index,
+                                                        labels=[42])],
+                                          is_l2=1)
+        mpls_tun.add_vpp_config()
+        mpls_tun.admin_up()
+
+        #
+        # Create a label entry to for 55 that does L2 input to the tunnel
+        #
+        route_55_eos = VppMplsRoute(
+            self, 55, 1,
+            [VppRoutePath("0.0.0.0",
+                          mpls_tun.sw_if_index,
+                          is_interface_rx=1,
+                          proto=DpoProto.DPO_PROTO_ETHERNET)])
+        route_55_eos.add_vpp_config()
+
+        #
+        # add to tunnel to the customers bridge-domain
+        #
+        self.vapi.sw_interface_set_l2_bridge(mpls_tun.sw_if_index,
+                                             bd_id=1)
+        self.vapi.sw_interface_set_l2_bridge(self.pg1.sw_if_index,
+                                             bd_id=1)
+
+        #
+        # Packet from the customer interface and from the core
+        #
+        p_cust = (Ether(dst="00:00:de:ad:ba:be",
+                        src="00:00:de:ad:be:ef") /
+                  IP(src="10.10.10.10", dst="11.11.11.11") /
+                  UDP(sport=1234, dport=1234) /
+                  Raw('\xa5' * 100))
+        p_core = (Ether(src="00:00:de:ad:ba:be",
+                        dst="00:00:de:ad:be:ef") /
+                  IP(dst="10.10.10.10", src="11.11.11.11") /
+                  UDP(sport=1234, dport=1234) /
+                  Raw('\xa5' * 100))
+
+        #
+        # The BD is learning, so send in one of each packet to learn
+        #
+        p_core_encap = (Ether(dst=self.pg0.local_mac,
+                              src=self.pg0.remote_mac) /
+                        MPLS(label=55, ttl=64) /
+                        p_core)
+
+        self.pg1.add_stream(p_cust)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        self.pg0.add_stream(p_core_encap)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # we've learnt this so expect it be be forwarded
+        rx0 = self.pg1.get_capture(1)
+
+        self.assertEqual(rx0[0][Ether].dst, p_core[Ether].dst)
+        self.assertEqual(rx0[0][Ether].src, p_core[Ether].src)
+
+        #
+        # now a stream in each direction
+        #
+        self.pg1.add_stream(p_cust * 65)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        rx0 = self.pg0.get_capture(65)
+
+        self.verify_capture_tunneled_ethernet(rx0, p_cust*65, [42])
+
+        #
+        # remove interfaces from customers bridge-domain
+        #
+        self.vapi.sw_interface_set_l2_bridge(mpls_tun.sw_if_index,
+                                             bd_id=1,
+                                             enable=0)
+        self.vapi.sw_interface_set_l2_bridge(self.pg1.sw_if_index,
+                                             bd_id=1,
+                                             enable=0)
+
 if __name__ == '__main__':
     unittest.main(testRunner=VppTestRunner)
diff --git a/test/test_p2p_ethernet.py b/test/test_p2p_ethernet.py
index 37a1d18b..8688f7e6 100644
--- a/test/test_p2p_ethernet.py
+++ b/test/test_p2p_ethernet.py
@@ -11,7 +11,7 @@ from scapy.layers.inet6 import IPv6
 
 from framework import VppTestCase, VppTestRunner, running_extended_tests
 from vpp_sub_interface import VppP2PSubint
-from vpp_ip_route import VppIpRoute, VppRoutePath
+from vpp_ip_route import VppIpRoute, VppRoutePath, DpoProto
 from util import mactobinary
 
 
@@ -219,7 +219,7 @@ class P2PEthernetIPV6(VppTestCase):
         route_8000 = VppIpRoute(self, "8000::", 64,
                                 [VppRoutePath(self.pg0.remote_ip6,
                                               self.pg0.sw_if_index,
-                                              is_ip6=1)],
+                                              proto=DpoProto.DPO_PROTO_IP6)],
                                 is_ip6=1)
         route_8000.add_vpp_config()
 
@@ -239,7 +239,7 @@ class P2PEthernetIPV6(VppTestCase):
         route_9001 = VppIpRoute(self, "9001::", 64,
                                 [VppRoutePath(self.pg1.remote_ip6,
                                               self.pg1.sw_if_index,
-                                              is_ip6=1)],
+                                              proto=DpoProto.DPO_PROTO_IP6)],
                                 is_ip6=1)
         route_9001.add_vpp_config()
 
@@ -264,7 +264,7 @@ class P2PEthernetIPV6(VppTestCase):
         route_3 = VppIpRoute(self, "9000::", 64,
                              [VppRoutePath(self.pg1._remote_hosts[0].ip6,
                                            self.pg1.sw_if_index,
-                                           is_ip6=1)],
+                                           proto=DpoProto.DPO_PROTO_IP6)],
                              is_ip6=1)
         route_3.add_vpp_config()
 
@@ -289,7 +289,7 @@ class P2PEthernetIPV6(VppTestCase):
         route_9001 = VppIpRoute(self, "9000::", 64,
                                 [VppRoutePath(self.pg1._remote_hosts[0].ip6,
                                               self.pg1.sw_if_index,
-                                              is_ip6=1)],
+                                              proto=DpoProto.DPO_PROTO_IP6)],
                                 is_ip6=1)
         route_9001.add_vpp_config()
 
@@ -310,19 +310,19 @@ class P2PEthernetIPV6(VppTestCase):
         route_8000 = VppIpRoute(self, "8000::", 64,
                                 [VppRoutePath(self.pg0.remote_ip6,
                                               self.pg0.sw_if_index,
-                                              is_ip6=1)],
+                                              proto=DpoProto.DPO_PROTO_IP6)],
                                 is_ip6=1)
         route_8000.add_vpp_config()
         route_8001 = VppIpRoute(self, "8001::", 64,
                                 [VppRoutePath(self.p2p_sub_ifs[0].remote_ip6,
                                               self.p2p_sub_ifs[0].sw_if_index,
-                                              is_ip6=1)],
+                                              proto=DpoProto.DPO_PROTO_IP6)],
                                 is_ip6=1)
         route_8001.add_vpp_config()
         route_8002 = VppIpRoute(self, "8002::", 64,
                                 [VppRoutePath(self.p2p_sub_ifs[1].remote_ip6,
                                               self.p2p_sub_ifs[1].sw_if_index,
-                                              is_ip6=1)],
+                                              proto=DpoProto.DPO_PROTO_IP6)],
                                 is_ip6=1)
         route_8002.add_vpp_config()
 
diff --git a/test/vpp_ip_route.py b/test/vpp_ip_route.py
index badb3102..2c489e3c 100644
--- a/test/vpp_ip_route.py
+++ b/test/vpp_ip_route.py
@@ -29,6 +29,14 @@ class MRouteEntryFlags:
     MFIB_ENTRY_FLAG_INHERIT_ACCEPT = 8
 
 
+class DpoProto:
+    DPO_PROTO_IP4 = 0
+    DPO_PROTO_IP6 = 1
+    DPO_PROTO_MPLS = 2
+    DPO_PROTO_ETHERNET = 3
+    DPO_PROTO_NSH = 4
+
+
 def find_route(test, ip_addr, len, table_id=0, inet=AF_INET):
     if inet == AF_INET:
         s = 4
@@ -55,22 +63,24 @@ class VppRoutePath(object):
             nh_table_id=0,
             labels=[],
             nh_via_label=MPLS_LABEL_INVALID,
-            is_ip6=0,
             rpf_id=0,
             is_interface_rx=0,
             is_resolve_host=0,
-            is_resolve_attached=0):
+            is_resolve_attached=0,
+            proto=DpoProto.DPO_PROTO_IP4):
         self.nh_itf = nh_sw_if_index
         self.nh_table_id = nh_table_id
         self.nh_via_label = nh_via_label
         self.nh_labels = labels
         self.weight = 1
         self.rpf_id = rpf_id
-        self.is_ip4 = 1 if is_ip6 == 0 else 0
-        if self.is_ip4:
+        self.proto = proto
+        if self.proto is DpoProto.DPO_PROTO_IP6:
+            self.nh_addr = inet_pton(AF_INET6, nh_addr)
+        elif self.proto is DpoProto.DPO_PROTO_IP4:
             self.nh_addr = inet_pton(AF_INET, nh_addr)
         else:
-            self.nh_addr = inet_pton(AF_INET6, nh_addr)
+            self.nh_addr = inet_pton(AF_INET6, "::")
         self.is_resolve_host = is_resolve_host
         self.is_resolve_attached = is_resolve_attached
         self.is_interface_rx = is_interface_rx
@@ -401,7 +411,7 @@ class VppMplsRoute(VppObject):
             self._test.vapi.mpls_route_add_del(
                 self.local_label,
                 self.eos_bit,
-                path.is_ip4,
+                path.proto,
                 path.nh_addr,
                 path.nh_itf,
                 is_multicast=self.is_multicast,
@@ -420,7 +430,7 @@ class VppMplsRoute(VppObject):
         for path in self.paths:
             self._test.vapi.mpls_route_add_del(self.local_label,
                                                self.eos_bit,
-                                               1,
+                                               path.proto,
                                                path.nh_addr,
                                                path.nh_itf,
                                                is_rpf_id=path.is_rpf_id,
diff --git a/test/vpp_mpls_tunnel_interface.py b/test/vpp_mpls_tunnel_interface.py
index f2001574..0542b05c 100644
--- a/test/vpp_mpls_tunnel_interface.py
+++ b/test/vpp_mpls_tunnel_interface.py
@@ -9,13 +9,14 @@ class VppMPLSTunnelInterface(VppInterface):
     VPP MPLS Tunnel interface
     """
 
-    def __init__(self, test, paths, is_multicast=0):
+    def __init__(self, test, paths, is_multicast=0, is_l2=0):
         """ Create MPLS Tunnel interface """
         self._sw_if_index = 0
         super(VppMPLSTunnelInterface, self).__init__(test)
         self._test = test
         self.t_paths = paths
         self.is_multicast = is_multicast
+        self.is_l2 = is_l2
 
     def add_vpp_config(self):
         self._sw_if_index = 0xffffffff
@@ -29,7 +30,8 @@ class VppMPLSTunnelInterface(VppInterface):
                 path.weight,
                 next_hop_out_label_stack=path.nh_labels,
                 next_hop_n_out_labels=len(path.nh_labels),
-                is_multicast=self.is_multicast)
+                is_multicast=self.is_multicast,
+                l2_only=self.is_l2)
             self._sw_if_index = reply.sw_if_index
 
     def remove_vpp_config(self):
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index 801a6c2d..3ba2ad4a 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -921,7 +921,7 @@ class VppPapiProvider(object):
             self,
             label,
             eos,
-            next_hop_proto_is_ip4,
+            next_hop_proto,
             next_hop_address,
             next_hop_sw_if_index=0xFFFFFFFF,
             table_id=0,
@@ -982,7 +982,7 @@ class VppPapiProvider(object):
              'mr_is_resolve_attached': is_resolve_attached,
              'mr_is_interface_rx': is_interface_rx,
              'mr_is_rpf_id': is_rpf_id,
-             'mr_next_hop_proto_is_ip4': next_hop_proto_is_ip4,
+             'mr_next_hop_proto': next_hop_proto,
              'mr_next_hop_weight': next_hop_weight,
              'mr_next_hop': next_hop_address,
              'mr_next_hop_n_out_labels': next_hop_n_out_labels,
-- 
cgit 1.2.3-korg


From f73d0e2ea6bf4e0dc9c69ec4f1d0c7b9b41d2fa3 Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Tue, 8 Aug 2017 13:07:16 -0700
Subject: Allow multiple MPLS output labels to be passed on the CLI

Change-Id: Ib5af105e32b6b0df86923e189ab6bf6ee59de5b9
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/vnet/ip/lookup.c        | 13 ++++++++++---
 src/vnet/mpls/mpls.c        | 33 +++++++++++++++++++++++++--------
 src/vnet/mpls/mpls_tunnel.c | 23 ++++++++++++++++++-----
 3 files changed, 53 insertions(+), 16 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 41e46070..4506ae14 100755
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -408,15 +408,22 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 	  rpaths[vec_len (rpaths) - 1].frp_flags |=
 	    FIB_ROUTE_PATH_RESOLVE_VIA_ATTACHED;
 	}
-      else if (unformat (line_input, "out-label %U",
-			 unformat_mpls_unicast_label, &out_label))
+      else if (unformat (line_input, "out-labels"))
 	{
 	  if (vec_len (rpaths) == 0)
 	    {
 	      error = clib_error_return (0, "Paths then labels");
 	      goto done;
 	    }
-	  vec_add1 (rpaths[vec_len (rpaths) - 1].frp_label_stack, out_label);
+	  else
+	    {
+	      while (unformat (line_input, "%U",
+			       unformat_mpls_unicast_label, &out_label))
+		{
+		  vec_add1 (rpaths[vec_len (rpaths) - 1].frp_label_stack,
+			    out_label);
+		}
+	    }
 	}
       else if (unformat (line_input, "via-label %U",
 			 unformat_mpls_unicast_label, &rpath.frp_local_label))
diff --git a/src/vnet/mpls/mpls.c b/src/vnet/mpls/mpls.c
index 266ba42c..5021ac23 100644
--- a/src/vnet/mpls/mpls.c
+++ b/src/vnet/mpls/mpls.c
@@ -66,8 +66,18 @@ uword unformat_mpls_unicast_label (unformat_input_t * input, va_list * args)
       *label = MPLS_IETF_ROUTER_ALERT_LABEL;
   else if (unformat (input, MPLS_IETF_IMPLICIT_NULL_STRING))
       *label = MPLS_IETF_IMPLICIT_NULL_LABEL;
+  else if (unformat (input, MPLS_IETF_IPV4_EXPLICIT_NULL_BRIEF_STRING))
+      *label = MPLS_IETF_IPV4_EXPLICIT_NULL_LABEL;
+  else if (unformat (input, MPLS_IETF_IPV6_EXPLICIT_NULL_BRIEF_STRING))
+      *label = MPLS_IETF_IPV6_EXPLICIT_NULL_LABEL;
+  else if (unformat (input, MPLS_IETF_ROUTER_ALERT_BRIEF_STRING))
+      *label = MPLS_IETF_ROUTER_ALERT_LABEL;
+  else if (unformat (input, MPLS_IETF_IMPLICIT_NULL_BRIEF_STRING))
+      *label = MPLS_IETF_IMPLICIT_NULL_LABEL;
   else if (unformat (input, "%d", label))
       ;
+  else
+    return (0);
 
   return (1);
 }
@@ -388,16 +398,23 @@ vnet_mpls_local_label (vlib_main_t * vm,
           rpath.frp_flags = FIB_ROUTE_PATH_INTF_RX;
 	  vec_add1(rpaths, rpath);
       }
-      else if (unformat (line_input, "out-label %U",
-                         unformat_mpls_unicast_label,
-			 &out_label))
+      else if (unformat (line_input, "out-labels"))
       {
-	  if (vec_len(rpaths) == 0)
-	  {
-	      error = clib_error_return(0 , "Paths then labels");
+	  if (vec_len (rpaths) == 0)
+          {
+	      error = clib_error_return (0, "Paths then labels");
 	      goto done;
-	  }
-	  vec_add1(rpaths[vec_len(rpaths)-1].frp_label_stack, out_label);
+          }
+          else
+          {
+              while (unformat (line_input, "%U",
+                               unformat_mpls_unicast_label,
+                               &out_label))
+              {
+                  vec_add1 (rpaths[vec_len (rpaths) - 1].frp_label_stack,
+                            out_label);
+              }
+          }
       }
       else
       {
diff --git a/src/vnet/mpls/mpls_tunnel.c b/src/vnet/mpls/mpls_tunnel.c
index 6452a60b..5211c926 100644
--- a/src/vnet/mpls/mpls_tunnel.c
+++ b/src/vnet/mpls/mpls_tunnel.c
@@ -800,11 +800,24 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm,
             is_del = 0;
         else if (unformat (line_input, "add"))
             is_del = 0;
-        else if (unformat (line_input, "out-label %U",
-                           unformat_mpls_unicast_label, &out_label))
-        {
-            vec_add1(rpath.frp_label_stack, out_label);
-        }
+        else if (unformat (line_input, "out-labels"))
+	{
+            if (vec_len (rpaths) == 0)
+	    {
+                error = clib_error_return (0, "Paths then labels");
+                goto done;
+            }
+            else
+            {
+                while (unformat (line_input, "%U",
+                                 unformat_mpls_unicast_label,
+                                 &out_label))
+                {
+                    vec_add1 (rpaths[vec_len (rpaths) - 1].frp_label_stack,
+                              out_label);
+                }
+            }
+	}
         else if (unformat (line_input, "via %U %U",
                            unformat_ip4_address,
                            &rpath.frp_addr.ip4,
-- 
cgit 1.2.3-korg


From 352829f538783afe8609f09850f45536a4779b70 Mon Sep 17 00:00:00 2001
From: pragash <pxv3620@rit.edu>
Date: Thu, 17 Aug 2017 22:53:24 -0400
Subject: Fixed ip add bug for ip6 with review comments

Change-Id: Ie1fba1a32020775444c7b1ed800e8559b54691ff
Signed-off-by: pragash <pxv3620@rit.edu>
---
 src/vnet/ip/lookup.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 4506ae14..5537bb04 100755
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -384,10 +384,6 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 
       if (unformat (line_input, "table %d", &table_id))
 	;
-      else if (unformat (line_input, "del"))
-	is_del = 1;
-      else if (unformat (line_input, "add"))
-	is_del = 0;
       else if (unformat (line_input, "resolve-via-host"))
 	{
 	  if (vec_len (rpaths) == 0)
@@ -542,6 +538,10 @@ vnet_ip_route_cmd (vlib_main_t * vm,
 	{
 	  vec_add1 (dpos, dpo);
 	}
+      else if (unformat (line_input, "del"))
+	is_del = 1;
+      else if (unformat (line_input, "add"))
+	is_del = 0;
       else
 	{
 	  error = unformat_parse_error (line_input);
-- 
cgit 1.2.3-korg


From 1500254bee11355bbd69cc1dd9705be4f002f2bd Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Sun, 10 Sep 2017 04:39:11 -0700
Subject: FIB table add/delete API

part 2;
  - this adds the code to create an IP and MPLS table via the API.
  - but the enforcement that the table must be created before it is used is still missing, this is so that CSIT can pass.

Change-Id: Id124d884ade6cb7da947225200e3bb193454c555
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/plugins/nat/nat.c                         |  17 +-
 src/plugins/nat/nat64.c                       |  13 +-
 src/vnet/classify/vnet_classify.c             |  16 +-
 src/vnet/dhcp/dhcp4_proxy_node.c              |   9 +-
 src/vnet/dhcp/dhcp6_proxy_node.c              |   9 +-
 src/vnet/dhcp/dhcp_proxy.c                    |  19 ++-
 src/vnet/dpo/lookup_dpo.c                     |  20 ++-
 src/vnet/dpo/mpls_label_dpo.c                 |  12 +-
 src/vnet/ethernet/arp.c                       | 127 +++++++++++----
 src/vnet/fib/fib_api.h                        |   1 -
 src/vnet/fib/fib_entry.c                      |  15 +-
 src/vnet/fib/fib_entry.h                      |   1 +
 src/vnet/fib/fib_entry_src_mpls.c             |   7 +-
 src/vnet/fib/fib_table.c                      |  43 +++--
 src/vnet/fib/fib_table.h                      |  32 +++-
 src/vnet/fib/fib_test.c                       |  27 ++--
 src/vnet/fib/ip4_fib.c                        |  41 +++--
 src/vnet/fib/ip4_fib.h                        |   5 +-
 src/vnet/fib/ip6_fib.c                        |  41 +++--
 src/vnet/fib/ip6_fib.h                        |   5 +-
 src/vnet/fib/mpls_fib.c                       |  16 +-
 src/vnet/fib/mpls_fib.h                       |   5 +-
 src/vnet/interface_api.c                      | 177 ++++++++++++++++----
 src/vnet/ip/ip.h                              |   7 +
 src/vnet/ip/ip4.h                             |  13 ++
 src/vnet/ip/ip4_forward.c                     | 101 +-----------
 src/vnet/ip/ip4_source_and_port_range_check.c |  11 +-
 src/vnet/ip/ip6.h                             |  13 ++
 src/vnet/ip/ip6_forward.c                     | 103 +-----------
 src/vnet/ip/ip6_neighbor.c                    | 108 +++++++++----
 src/vnet/ip/ip_api.c                          | 122 +++++++++++---
 src/vnet/ip/lookup.c                          | 225 ++++++++++++++++++++++++++
 src/vnet/lisp-gpe/interface.c                 |  11 +-
 src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c        |   9 +-
 src/vnet/lisp-gpe/lisp_gpe_sub_interface.c    |  11 +-
 src/vnet/mfib/ip4_mfib.c                      |  12 +-
 src/vnet/mfib/ip4_mfib.h                      |   5 +-
 src/vnet/mfib/ip6_mfib.c                      |  12 +-
 src/vnet/mfib/ip6_mfib.h                      |   5 +-
 src/vnet/mfib/mfib_entry.c                    |  11 ++
 src/vnet/mfib/mfib_entry.h                    |   2 +
 src/vnet/mfib/mfib_table.c                    |  88 ++++++++--
 src/vnet/mfib/mfib_table.h                    |  29 +++-
 src/vnet/mfib/mfib_test.c                     |  11 +-
 src/vnet/mfib/mfib_types.h                    |   8 +-
 src/vnet/mpls/interface.c                     |  26 ++-
 src/vnet/mpls/mpls.c                          |  76 ++++++++-
 src/vnet/mpls/mpls.h                          |  16 +-
 src/vnet/mpls/mpls_api.c                      |  66 ++++++--
 src/vnet/srv6/sr_policy_rewrite.c             |   6 +-
 src/vnet/srv6/sr_steering.c                   |   6 +-
 src/vpp/api/api.c                             |   5 +-
 src/vpp/api/custom_dump.c                     |   3 -
 test/test_dhcp.py                             |  24 ++-
 test/test_gre.py                              |   8 +-
 test/test_ip4.py                              |  11 +-
 test/test_ip4_vrf_multi_instance.py           |   4 +-
 test/test_ip6.py                              |   7 +-
 test/test_ip6_vrf_multi_instance.py           |   4 +-
 test/test_ip_mcast.py                         |  98 ++++++++++-
 test/test_mpls.py                             |  48 +++++-
 test/test_nat.py                              |  13 ++
 test/test_neighbor.py                         |  66 +++++++-
 test/vpp_ip_route.py                          |  73 +++++++++
 test/vpp_papi_provider.py                     |  46 ++++--
 65 files changed, 1643 insertions(+), 538 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
index aa7ef10a..8aecac6d 100644
--- a/src/plugins/nat/nat.c
+++ b/src/plugins/nat/nat.c
@@ -167,7 +167,8 @@ void snat_add_address (snat_main_t *sm, ip4_address_t *addr, u32 vrf_id)
   ap->addr = *addr;
   if (vrf_id != ~0)
     ap->fib_index =
-      fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id);
+      fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
+                                         FIB_SOURCE_PLUGIN_HI);
   else
     ap->fib_index = ~0;
 #define _(N, i, n, s) \
@@ -625,7 +626,8 @@ int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
         return VNET_API_ERROR_INVALID_VALUE;
 
       fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
-                                                     vrf_id);
+                                                     vrf_id,
+                                                     FIB_SOURCE_PLUGIN_HI);
 
       /* Find external address in allocated addresses and reserve port for
          address and port pair mapping when dynamic translations enabled */
@@ -754,7 +756,7 @@ int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
       if (!m)
         return VNET_API_ERROR_NO_SUCH_ENTRY;
 
-      fib_table_unlock (m->fib_index, FIB_PROTOCOL_IP4);
+      fib_table_unlock (m->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_HI);
 
       /* Free external address port */
       if (!sm->static_mapping_only)
@@ -874,7 +876,8 @@ int snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm)
     }
 
   if (a->fib_index != ~0)
-    fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4);
+    fib_table_unlock(a->fib_index, FIB_PROTOCOL_IP4,
+                     FIB_SOURCE_PLUGIN_HI);
 
   /* Delete sessions using address */
   if (a->busy_tcp_ports || a->busy_udp_ports || a->busy_icmp_ports)
@@ -2151,10 +2154,12 @@ snat_config (vlib_main_t * vm, unformat_input_t * input)
   sm->max_translations_per_user = max_translations_per_user;
   sm->outside_vrf_id = outside_vrf_id;
   sm->outside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
-                                                             outside_vrf_id);
+                                                             outside_vrf_id,
+                                                             FIB_SOURCE_PLUGIN_HI);
   sm->inside_vrf_id = inside_vrf_id;
   sm->inside_fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
-                                                            inside_vrf_id);
+                                                            inside_vrf_id,
+                                                            FIB_SOURCE_PLUGIN_HI);
   sm->static_mapping_only = static_mapping_only;
   sm->static_mapping_connection_tracking = static_mapping_connection_tracking;
 
diff --git a/src/plugins/nat/nat64.c b/src/plugins/nat/nat64.c
index b04901fa..bfcfa9b3 100644
--- a/src/plugins/nat/nat64.c
+++ b/src/plugins/nat/nat64.c
@@ -107,7 +107,8 @@ nat64_add_del_pool_addr (ip4_address_t * addr, u32 vrf_id, u8 is_add)
       a->fib_index = 0;
       if (vrf_id != ~0)
 	a->fib_index =
-	  fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, vrf_id);
+	  fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, vrf_id,
+					     FIB_SOURCE_PLUGIN_HI);
 #define _(N, i, n, s) \
       clib_bitmap_alloc (a->busy_##n##_port_bitmap, 65535);
       foreach_snat_protocol
@@ -119,7 +120,8 @@ nat64_add_del_pool_addr (ip4_address_t * addr, u32 vrf_id, u8 is_add)
 	return VNET_API_ERROR_NO_SUCH_ENTRY;
 
       if (a->fib_index)
-	fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP6);
+	fib_table_unlock (a->fib_index, FIB_PROTOCOL_IP6,
+			  FIB_SOURCE_PLUGIN_HI);
 
 #define _(N, id, n, s) \
       clib_bitmap_free (a->busy_##n##_port_bitmap);
@@ -353,8 +355,8 @@ nat64_add_del_static_bib_entry (ip6_address_t * in_addr,
 {
   nat64_main_t *nm = &nat64_main;
   nat64_db_bib_entry_t *bibe;
-  u32 fib_index =
-    fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, vrf_id);
+  u32 fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, vrf_id,
+						     FIB_SOURCE_PLUGIN_HI);
   snat_protocol_t p = ip_proto_to_snat_proto (proto);
   ip46_address_t addr;
   int i;
@@ -644,7 +646,8 @@ nat64_add_del_prefix (ip6_address_t * prefix, u8 plen, u32 vrf_id, u8 is_add)
 	{
 	  vec_add2 (nm->pref64, p, 1);
 	  p->fib_index =
-	    fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, vrf_id);
+	    fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, vrf_id,
+					       FIB_SOURCE_PLUGIN_HI);
 	  p->vrf_id = vrf_id;
 	}
 
diff --git a/src/vnet/classify/vnet_classify.c b/src/vnet/classify/vnet_classify.c
index 879fba3c..57d86748 100644
--- a/src/vnet/classify/vnet_classify.c
+++ b/src/vnet/classify/vnet_classify.c
@@ -368,10 +368,10 @@ vnet_classify_entry_claim_resource (vnet_classify_entry_t *e)
     switch (e->action)
     {
     case CLASSIFY_ACTION_SET_IP4_FIB_INDEX:
-        fib_table_lock (e->metadata, FIB_PROTOCOL_IP4);
+        fib_table_lock (e->metadata, FIB_PROTOCOL_IP4, FIB_SOURCE_CLASSIFY);
         break;
     case CLASSIFY_ACTION_SET_IP6_FIB_INDEX:
-        fib_table_lock (e->metadata, FIB_PROTOCOL_IP6);
+        fib_table_lock (e->metadata, FIB_PROTOCOL_IP6, FIB_SOURCE_CLASSIFY);
         break;
     }
 }
@@ -382,10 +382,10 @@ vnet_classify_entry_release_resource (vnet_classify_entry_t *e)
     switch (e->action)
     {
     case CLASSIFY_ACTION_SET_IP4_FIB_INDEX:
-        fib_table_unlock (e->metadata, FIB_PROTOCOL_IP4);
+        fib_table_unlock (e->metadata, FIB_PROTOCOL_IP4, FIB_SOURCE_CLASSIFY);
         break;
     case CLASSIFY_ACTION_SET_IP6_FIB_INDEX:
-        fib_table_unlock (e->metadata, FIB_PROTOCOL_IP6);
+        fib_table_unlock (e->metadata, FIB_PROTOCOL_IP6, FIB_SOURCE_CLASSIFY);
         break;
     }
 }
@@ -2096,9 +2096,13 @@ int vnet_classify_add_del_session (vnet_classify_main_t * cm,
   e->flags = 0;
   e->action = action;
   if (e->action == CLASSIFY_ACTION_SET_IP4_FIB_INDEX)
-    e->metadata = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, metadata);
+    e->metadata = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
+                                                     metadata,
+                                                     FIB_SOURCE_CLASSIFY);
   else if (e->action == CLASSIFY_ACTION_SET_IP6_FIB_INDEX)
-    e->metadata = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, metadata);
+    e->metadata = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
+                                                     metadata,
+                                                     FIB_SOURCE_CLASSIFY);
   else
     e->metadata = 0;
 
diff --git a/src/vnet/dhcp/dhcp4_proxy_node.c b/src/vnet/dhcp/dhcp4_proxy_node.c
index 1b59cdea..339a7885 100644
--- a/src/vnet/dhcp/dhcp4_proxy_node.c
+++ b/src/vnet/dhcp/dhcp4_proxy_node.c
@@ -785,7 +785,8 @@ dhcp4_proxy_set_server (ip46_address_t *addr,
     return VNET_API_ERROR_INVALID_SRC_ADDRESS;
 
   rx_fib_index = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP4,
-                                                   rx_table_id);
+                                                   rx_table_id,
+                                                   FIB_SOURCE_DHCP);
 
   if (is_del)
     {
@@ -795,7 +796,7 @@ dhcp4_proxy_set_server (ip46_address_t *addr,
           fib_table_entry_special_remove(rx_fib_index,
                                          &all_1s,
                                          FIB_SOURCE_DHCP);
-          fib_table_unlock (rx_fib_index, FIB_PROTOCOL_IP4);
+          fib_table_unlock (rx_fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_DHCP);
       }
     }
   else
@@ -808,10 +809,10 @@ dhcp4_proxy_set_server (ip46_address_t *addr,
                                       &all_1s,
                                       FIB_SOURCE_DHCP,
                                       FIB_ENTRY_FLAG_LOCAL);
-          fib_table_lock (rx_fib_index, FIB_PROTOCOL_IP4);
+          fib_table_lock (rx_fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_DHCP);
       }
   }
-  fib_table_unlock (rx_fib_index, FIB_PROTOCOL_IP4);
+  fib_table_unlock (rx_fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_DHCP);
 
   return (rc);
 }
diff --git a/src/vnet/dhcp/dhcp6_proxy_node.c b/src/vnet/dhcp/dhcp6_proxy_node.c
index 9c2f5220..ce7a8fca 100644
--- a/src/vnet/dhcp/dhcp6_proxy_node.c
+++ b/src/vnet/dhcp/dhcp6_proxy_node.c
@@ -841,7 +841,8 @@ dhcp6_proxy_set_server (ip46_address_t *addr,
     return VNET_API_ERROR_INVALID_SRC_ADDRESS;
 
   rx_fib_index = mfib_table_find_or_create_and_lock(FIB_PROTOCOL_IP6,
-                                                    rx_table_id);
+                                                    rx_table_id,
+                                                    MFIB_SOURCE_DHCP);
 
   if (is_del)
     {
@@ -851,7 +852,7 @@ dhcp6_proxy_set_server (ip46_address_t *addr,
           mfib_table_entry_delete(rx_fib_index,
                                   &all_dhcp_servers,
                                   MFIB_SOURCE_DHCP);
-          mfib_table_unlock(rx_fib_index, FIB_PROTOCOL_IP6);
+          mfib_table_unlock(rx_fib_index, FIB_PROTOCOL_IP6, MFIB_SOURCE_DHCP);
       }
     }
   else
@@ -885,11 +886,11 @@ dhcp6_proxy_set_server (ip46_address_t *addr,
                                  MFIB_SOURCE_DHCP,
                                  MFIB_RPF_ID_NONE,
                                  MFIB_ENTRY_FLAG_ACCEPT_ALL_ITF);
-         mfib_table_lock(rx_fib_index, FIB_PROTOCOL_IP6);
+         mfib_table_lock(rx_fib_index, FIB_PROTOCOL_IP6, MFIB_SOURCE_DHCP);
      }
     }
 
-  mfib_table_unlock(rx_fib_index, FIB_PROTOCOL_IP6);
+  mfib_table_unlock(rx_fib_index, FIB_PROTOCOL_IP6, MFIB_SOURCE_DHCP);
 
   return (rc);
 }
diff --git a/src/vnet/dhcp/dhcp_proxy.c b/src/vnet/dhcp/dhcp_proxy.c
index ba7f354e..1784906b 100644
--- a/src/vnet/dhcp/dhcp_proxy.c
+++ b/src/vnet/dhcp/dhcp_proxy.c
@@ -29,9 +29,9 @@ dhcp_proxy_rx_table_lock (fib_protocol_t proto,
                           u32 fib_index)
 {
     if (FIB_PROTOCOL_IP4 == proto)
-        fib_table_lock(fib_index, proto);
+        fib_table_lock(fib_index, proto, FIB_SOURCE_DHCP);
     else
-        mfib_table_lock(fib_index, proto);
+        mfib_table_lock(fib_index, proto, MFIB_SOURCE_DHCP);
 }
 
 static void
@@ -39,9 +39,9 @@ dhcp_proxy_rx_table_unlock (fib_protocol_t proto,
                             u32 fib_index)
 {
     if (FIB_PROTOCOL_IP4 == proto)
-        fib_table_unlock(fib_index, proto);
+        fib_table_unlock(fib_index, proto, FIB_SOURCE_DHCP);
     else
-        mfib_table_unlock(fib_index, proto);
+        mfib_table_unlock(fib_index, proto, MFIB_SOURCE_DHCP);
 }
 
  u32
@@ -169,7 +169,7 @@ dhcp_proxy_server_del (fib_protocol_t proto,
       if (~0 != index)
       {
           server = &proxy->dhcp_servers[index];
-          fib_table_unlock (server->server_fib_index, proto);
+          fib_table_unlock (server->server_fib_index, proto, FIB_SOURCE_DHCP);
 
           vec_del1(proxy->dhcp_servers, index);
 
@@ -228,7 +228,8 @@ dhcp_proxy_server_add (fib_protocol_t proto,
   dhcp_server_t server = {
       .dhcp_server = *addr,
       .server_fib_index = fib_table_find_or_create_and_lock(proto,
-                                                            server_table_id),
+                                                            server_table_id,
+                                                            FIB_SOURCE_DHCP),
   };
 
   vec_add1(proxy->dhcp_servers, server);
@@ -297,9 +298,11 @@ int dhcp_proxy_set_vss (fib_protocol_t proto,
   int rc = 0;
   
   if (proto == FIB_PROTOCOL_IP4)
-      rx_fib_index = fib_table_find_or_create_and_lock(proto, tbl_id);
+      rx_fib_index = fib_table_find_or_create_and_lock(proto, tbl_id,
+                                                       FIB_SOURCE_DHCP);
   else
-      rx_fib_index = mfib_table_find_or_create_and_lock(proto, tbl_id);
+      rx_fib_index = mfib_table_find_or_create_and_lock(proto, tbl_id,
+                                                        MFIB_SOURCE_DHCP);
   v = dhcp_get_vss_info(dm, rx_fib_index, proto);
 
   if (NULL != v)
diff --git a/src/vnet/dpo/lookup_dpo.c b/src/vnet/dpo/lookup_dpo.c
index 26363a2f..af189eda 100644
--- a/src/vnet/dpo/lookup_dpo.c
+++ b/src/vnet/dpo/lookup_dpo.c
@@ -135,11 +135,15 @@ lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index,
     {
         if (LOOKUP_UNICAST == cast)
         {
-            fib_table_lock(fib_index, dpo_proto_to_fib(proto));
+            fib_table_lock(fib_index,
+                           dpo_proto_to_fib(proto),
+                           FIB_SOURCE_RR);
         }
         else
         {
-            mfib_table_lock(fib_index, dpo_proto_to_fib(proto));
+            mfib_table_lock(fib_index,
+                            dpo_proto_to_fib(proto),
+                            MFIB_SOURCE_RR);
         }
     }
     lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo);
@@ -161,13 +165,15 @@ lookup_dpo_add_or_lock_w_table_id (u32 table_id,
         {
             fib_index =
                 fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
-                                                  table_id);
+                                                  table_id,
+                                                  FIB_SOURCE_RR);
         }
         else
         {
             fib_index =
                 mfib_table_find_or_create_and_lock(dpo_proto_to_fib(proto),
-                                                   table_id);
+                                                   table_id,
+                                                   MFIB_SOURCE_RR);
         }
     }
 
@@ -238,12 +244,14 @@ lookup_dpo_unlock (dpo_id_t *dpo)
             if (LOOKUP_UNICAST == lkd->lkd_cast)
             {
                 fib_table_unlock(lkd->lkd_fib_index,
-                                 dpo_proto_to_fib(lkd->lkd_proto));
+                                 dpo_proto_to_fib(lkd->lkd_proto),
+                                 FIB_SOURCE_RR);
             }
             else
             {
                 mfib_table_unlock(lkd->lkd_fib_index,
-                                  dpo_proto_to_fib(lkd->lkd_proto));
+                                  dpo_proto_to_fib(lkd->lkd_proto),
+                                  MFIB_SOURCE_RR);
             }
         }
         pool_put(lookup_dpo_pool, lkd);
diff --git a/src/vnet/dpo/mpls_label_dpo.c b/src/vnet/dpo/mpls_label_dpo.c
index b178a902..2a6e7dd5 100644
--- a/src/vnet/dpo/mpls_label_dpo.c
+++ b/src/vnet/dpo/mpls_label_dpo.c
@@ -105,10 +105,18 @@ format_mpls_label_dpo (u8 *s, va_list *args)
     mpls_label_dpo_t *mld;
     u32 ii;
 
-    mld = mpls_label_dpo_get(index);
-
     s = format(s, "mpls-label:[%d]:", index);
 
+    if (pool_is_free_index(mpls_label_dpo_pool, index))
+    {
+        /*
+         * the packet trace can be printed after the DPO has been deleted
+         */
+        return (s);
+    }
+
+    mld = mpls_label_dpo_get(index);
+
     for (ii = 0; ii < mld->mld_n_labels; ii++)
     {
 	hdr.label_exp_s_ttl =
diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c
index c84ff47b..08e91373 100644
--- a/src/vnet/ethernet/arp.c
+++ b/src/vnet/ethernet/arp.c
@@ -522,6 +522,24 @@ arp_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai)
     }
 }
 
+static void
+arp_adj_fib_add (ethernet_arp_ip4_entry_t * e, uint32_t fib_index)
+{
+  fib_prefix_t pfx = {
+    .fp_len = 32,
+    .fp_proto = FIB_PROTOCOL_IP4,
+    .fp_addr.ip4 = e->ip4_address,
+  };
+
+  e->fib_entry_index =
+    fib_table_entry_path_add (fib_index, &pfx, FIB_SOURCE_ADJ,
+			      FIB_ENTRY_FLAG_ATTACHED,
+			      DPO_PROTO_IP4, &pfx.fp_addr,
+			      e->sw_if_index, ~0, 1, NULL,
+			      FIB_ROUTE_PATH_FLAG_NONE);
+  fib_table_lock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_ADJ);
+}
+
 int
 vnet_arp_set_ip4_over_ethernet_internal (vnet_main_t * vnm,
 					 vnet_arp_set_ip4_over_ethernet_rpc_args_t
@@ -576,21 +594,9 @@ vnet_arp_set_ip4_over_ethernet_internal (vnet_main_t * vnm,
 
       if (!is_no_fib_entry)
 	{
-	  fib_prefix_t pfx = {
-	    .fp_len = 32,
-	    .fp_proto = FIB_PROTOCOL_IP4,
-	    .fp_addr.ip4 = a->ip4,
-	  };
-	  u32 fib_index;
-
-	  fib_index =
-	    ip4_fib_table_get_index_for_sw_if_index (e->sw_if_index);
-	  e->fib_entry_index =
-	    fib_table_entry_path_add (fib_index, &pfx, FIB_SOURCE_ADJ,
-				      FIB_ENTRY_FLAG_ATTACHED,
-				      DPO_PROTO_IP4, &pfx.fp_addr,
-				      e->sw_if_index, ~0, 1, NULL,
-				      FIB_ROUTE_PATH_FLAG_NONE);
+	  arp_adj_fib_add (e,
+			   ip4_fib_table_get_index_for_sw_if_index
+			   (e->sw_if_index));
 	}
       else
 	{
@@ -1561,6 +1567,65 @@ arp_add_del_interface_address (ip4_main_t * im,
     }
 }
 
+void
+arp_adj_fib_remove (ethernet_arp_ip4_entry_t * e, uint32_t fib_index)
+{
+  if (FIB_NODE_INDEX_INVALID != e->fib_entry_index)
+    {
+      fib_prefix_t pfx = {
+	.fp_len = 32,
+	.fp_proto = FIB_PROTOCOL_IP4,
+	.fp_addr.ip4 = e->ip4_address,
+      };
+      u32 fib_index;
+
+      fib_index = ip4_fib_table_get_index_for_sw_if_index (e->sw_if_index);
+
+      fib_table_entry_path_remove (fib_index, &pfx,
+				   FIB_SOURCE_ADJ,
+				   DPO_PROTO_IP4,
+				   &pfx.fp_addr,
+				   e->sw_if_index, ~0, 1,
+				   FIB_ROUTE_PATH_FLAG_NONE);
+      fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_ADJ);
+    }
+}
+
+static void
+arp_table_bind (ip4_main_t * im,
+		uword opaque,
+		u32 sw_if_index, u32 new_fib_index, u32 old_fib_index)
+{
+  ethernet_arp_main_t *am = &ethernet_arp_main;
+  ethernet_arp_interface_t *eai;
+  ethernet_arp_ip4_entry_t *e;
+  hash_pair_t *pair;
+
+  /*
+   * the IP table that the interface is bound to has changed.
+   * reinstall all the adj fibs.
+   */
+
+  if (vec_len (am->ethernet_arp_by_sw_if_index) <= sw_if_index)
+    return;
+
+  eai = &am->ethernet_arp_by_sw_if_index[sw_if_index];
+
+  /* *INDENT-OFF* */
+  hash_foreach_pair (pair, eai->arp_entries,
+  ({
+    e = pool_elt_at_index(am->ip4_entry_pool,
+                          pair->value[0]);
+    /*
+     * remove the adj-fib from the old table and add to the new
+     */
+    arp_adj_fib_remove(e, old_fib_index);
+    arp_adj_fib_add(e, new_fib_index);
+  }));
+  /* *INDENT-ON* */
+
+}
+
 static clib_error_t *
 ethernet_arp_init (vlib_main_t * vm)
 {
@@ -1606,6 +1671,11 @@ ethernet_arp_init (vlib_main_t * vm)
   cb.function_opaque = 0;
   vec_add1 (im->add_del_interface_address_callbacks, cb);
 
+  ip4_table_bind_callback_t cbt;
+  cbt.function = arp_table_bind;
+  cbt.function_opaque = 0;
+  vec_add1 (im->table_bind_callbacks, cbt);
+
   return 0;
 }
 
@@ -1616,24 +1686,9 @@ arp_entry_free (ethernet_arp_interface_t * eai, ethernet_arp_ip4_entry_t * e)
 {
   ethernet_arp_main_t *am = &ethernet_arp_main;
 
-  if (FIB_NODE_INDEX_INVALID != e->fib_entry_index)
-    {
-      fib_prefix_t pfx = {
-	.fp_len = 32,
-	.fp_proto = FIB_PROTOCOL_IP4,
-	.fp_addr.ip4 = e->ip4_address,
-      };
-      u32 fib_index;
-
-      fib_index = ip4_fib_table_get_index_for_sw_if_index (e->sw_if_index);
-
-      fib_table_entry_path_remove (fib_index, &pfx,
-				   FIB_SOURCE_ADJ,
-				   DPO_PROTO_IP4,
-				   &pfx.fp_addr,
-				   e->sw_if_index, ~0, 1,
-				   FIB_ROUTE_PATH_FLAG_NONE);
-    }
+  arp_adj_fib_remove (e,
+		      ip4_fib_table_get_index_for_sw_if_index
+		      (e->sw_if_index));
   hash_unset (eai->arp_entries, e->ip4_address.as_u32);
   pool_put (am->ip4_entry_pool, e);
 }
@@ -1693,7 +1748,11 @@ vnet_arp_flush_ip4_over_ethernet_internal (vnet_main_t * vnm,
        * does in response to interface events. unset is only done
        * by the control plane.
        */
-      if (e->flags & ETHERNET_ARP_IP4_ENTRY_FLAG_DYNAMIC)
+      if (e->flags & ETHERNET_ARP_IP4_ENTRY_FLAG_STATIC)
+	{
+	  e->flags &= ETHERNET_ARP_IP4_ENTRY_FLAG_DYNAMIC;
+	}
+      else if (e->flags & ETHERNET_ARP_IP4_ENTRY_FLAG_DYNAMIC)
 	{
 	  arp_entry_free (eai, e);
 	}
diff --git a/src/vnet/fib/fib_api.h b/src/vnet/fib/fib_api.h
index d07d6cae..f5a107ca 100644
--- a/src/vnet/fib/fib_api.h
+++ b/src/vnet/fib/fib_api.h
@@ -23,7 +23,6 @@ add_del_route_check (fib_protocol_t table_proto,
 		     u32 next_hop_sw_if_index,
 		     dpo_proto_t next_hop_table_proto,
 		     u32 next_hop_table_id,
-		     u8 create_missing_tables,
                      u8 is_rpf_id,
 		     u32 * fib_index, u32 * next_hop_fib_index);
 
diff --git a/src/vnet/fib/fib_entry.c b/src/vnet/fib/fib_entry.c
index 2027f2be..4cb6cf60 100644
--- a/src/vnet/fib/fib_entry.c
+++ b/src/vnet/fib/fib_entry.c
@@ -89,6 +89,17 @@ fib_entry_get_default_chain_type (const fib_entry_t *fib_entry)
     return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4);
 }
 
+u8 *
+format_fib_source (u8 * s, va_list * args)
+{
+    fib_source_t source = va_arg (*args, int);
+
+    s = format (s, "\n  src:%s ",
+                fib_source_names[source]);
+
+    return (s);
+}
+
 u8 *
 format_fib_entry (u8 * s, va_list * args)
 {
@@ -114,8 +125,8 @@ format_fib_entry (u8 * s, va_list * args)
 
 	FOR_EACH_SRC_ADDED(fib_entry, src, source,
         ({
-	    s = format (s, "\n  src:%s ",
-			fib_source_names[source]);
+	    s = format (s, "\n  src:%U ",
+			format_fib_source, source);
 	    s = fib_entry_src_format(fib_entry, source, s);
 	    s = format (s, " refs:%d ", src->fes_ref_count);
 	    if (FIB_ENTRY_FLAG_NONE != src->fes_entry_flags) {
diff --git a/src/vnet/fib/fib_entry.h b/src/vnet/fib/fib_entry.h
index 93b8016d..2f6e37fe 100644
--- a/src/vnet/fib/fib_entry.h
+++ b/src/vnet/fib/fib_entry.h
@@ -431,6 +431,7 @@ typedef struct fib_entry_t_ {
 #define FIB_ENTRY_FORMAT_DETAIL2 (0x2)
 
 extern u8 *format_fib_entry (u8 * s, va_list * args);
+extern u8 *format_fib_source (u8 * s, va_list * args);
 
 extern fib_node_index_t fib_entry_create_special(u32 fib_index,
 						 const fib_prefix_t *prefix,
diff --git a/src/vnet/fib/fib_entry_src_mpls.c b/src/vnet/fib/fib_entry_src_mpls.c
index a616458f..6fdd5c0a 100644
--- a/src/vnet/fib/fib_entry_src_mpls.c
+++ b/src/vnet/fib/fib_entry_src_mpls.c
@@ -94,7 +94,9 @@ fib_entry_src_mpls_set_data (fib_entry_src_t *src,
 	    fib_table_entry_delete_index(src->mpls.fesm_lfes[eos],
 					 FIB_SOURCE_SPECIAL);
         }
-        fib_table_unlock(MPLS_FIB_DEFAULT_TABLE_ID, FIB_PROTOCOL_MPLS);
+        fib_table_unlock(MPLS_FIB_DEFAULT_TABLE_ID,
+                         FIB_PROTOCOL_MPLS,
+                         FIB_SOURCE_MPLS);
         src->mpls.fesm_label = label;
     }
     else
@@ -113,7 +115,8 @@ fib_entry_src_mpls_set_data (fib_entry_src_t *src,
         {
             fib_index =
 		fib_table_find_or_create_and_lock(FIB_PROTOCOL_MPLS,
-						  MPLS_FIB_DEFAULT_TABLE_ID);
+						  MPLS_FIB_DEFAULT_TABLE_ID,
+                                                  FIB_SOURCE_MPLS);
         }
 	else
 	{
diff --git a/src/vnet/fib/fib_table.c b/src/vnet/fib/fib_table.c
index 6b6cc5cb..75d15628 100644
--- a/src/vnet/fib/fib_table.c
+++ b/src/vnet/fib/fib_table.c
@@ -1039,7 +1039,8 @@ fib_table_find (fib_protocol_t proto,
 
 u32
 fib_table_find_or_create_and_lock (fib_protocol_t proto,
-				   u32 table_id)
+				   u32 table_id,
+                                   fib_source_t src)
 {
     fib_table_t *fib_table;
     fib_node_index_t fi;
@@ -1047,13 +1048,13 @@ fib_table_find_or_create_and_lock (fib_protocol_t proto,
     switch (proto)
     {
     case FIB_PROTOCOL_IP4:
-	fi = ip4_fib_table_find_or_create_and_lock(table_id);
+	fi = ip4_fib_table_find_or_create_and_lock(table_id, src);
         break;
     case FIB_PROTOCOL_IP6:
-	fi = ip6_fib_table_find_or_create_and_lock(table_id);
+	fi = ip6_fib_table_find_or_create_and_lock(table_id, src);
         break;
     case FIB_PROTOCOL_MPLS:
-	fi = mpls_fib_table_find_or_create_and_lock(table_id);
+	fi = mpls_fib_table_find_or_create_and_lock(table_id, src);
         break;
     default:
         return (~0);        
@@ -1070,6 +1071,7 @@ fib_table_find_or_create_and_lock (fib_protocol_t proto,
 
 u32
 fib_table_create_and_lock (fib_protocol_t proto,
+                           fib_source_t src,
                            const char *const fmt,
                            ...)
 {
@@ -1082,13 +1084,13 @@ fib_table_create_and_lock (fib_protocol_t proto,
     switch (proto)
     {
     case FIB_PROTOCOL_IP4:
-	fi = ip4_fib_table_create_and_lock();
+	fi = ip4_fib_table_create_and_lock(src);
         break;
     case FIB_PROTOCOL_IP6:
-	fi = ip6_fib_table_create_and_lock();
+	fi = ip6_fib_table_create_and_lock(src);
         break;
      case FIB_PROTOCOL_MPLS:
-	fi = mpls_fib_table_create_and_lock();
+	fi = mpls_fib_table_create_and_lock(src);
         break;
    default:
         return (~0);        
@@ -1143,26 +1145,43 @@ fib_table_walk (u32 fib_index,
 
 void
 fib_table_unlock (u32 fib_index,
-		  fib_protocol_t proto)
+		  fib_protocol_t proto,
+                  fib_source_t source)
 {
     fib_table_t *fib_table;
 
     fib_table = fib_table_get(fib_index, proto);
-    fib_table->ft_locks--;
+    fib_table->ft_locks[source]--;
+    fib_table->ft_locks[FIB_TABLE_TOTAL_LOCKS]--;
 
-    if (0 == fib_table->ft_locks)
+    if (0 == fib_table->ft_locks[source])
     {
+        /*
+         * The source no longer needs the table. flush any routes
+         * from it just in case
+         */
+        fib_table_flush(fib_index, proto, source);
+    }
+
+    if (0 == fib_table->ft_locks[FIB_TABLE_TOTAL_LOCKS])
+    {
+        /*
+         * no more locak from any source - kill it
+         */
 	fib_table_destroy(fib_table);
     }
 }
+
 void
 fib_table_lock (u32 fib_index,
-		fib_protocol_t proto)
+		fib_protocol_t proto,
+                fib_source_t source)
 {
     fib_table_t *fib_table;
 
     fib_table = fib_table_get(fib_index, proto);
-    fib_table->ft_locks++;
+    fib_table->ft_locks[source]++;
+    fib_table->ft_locks[FIB_TABLE_TOTAL_LOCKS]++;
 }
 
 u32
diff --git a/src/vnet/fib/fib_table.h b/src/vnet/fib/fib_table.h
index 579740e9..6b7011b3 100644
--- a/src/vnet/fib/fib_table.h
+++ b/src/vnet/fib/fib_table.h
@@ -22,6 +22,12 @@
 #include <vnet/mpls/mpls.h>
 #include <vnet/mpls/packet.h>
 
+/**
+ * Keep a lock per-source and a total
+ */
+#define FIB_TABLE_N_LOCKS (FIB_SOURCE_MAX+1)
+#define FIB_TABLE_TOTAL_LOCKS FIB_SOURCE_MAX
+
 /**
  * @brief 
  *   A protocol Independent FIB table
@@ -34,9 +40,9 @@ typedef struct fib_table_t_
     fib_protocol_t ft_proto;
 
     /**
-     * number of locks on the table
+     * per-source number of locks on the table
      */
-    u16 ft_locks;
+    u16 ft_locks[FIB_TABLE_N_LOCKS];
 
     /**
      * Table ID (hash key) for this FIB.
@@ -628,9 +634,13 @@ extern u32 fib_table_find(fib_protocol_t proto, u32 table_id);
  *
  * @return fib_index
  *  The index of the FIB
+ *
+ * @param source
+ *  The ID of the client/source.
  */
 extern u32 fib_table_find_or_create_and_lock(fib_protocol_t proto,
-					     u32 table_id);
+					     u32 table_id,
+                                             fib_source_t source);
 
 /**
  * @brief
@@ -643,10 +653,14 @@ extern u32 fib_table_find_or_create_and_lock(fib_protocol_t proto,
  * @param fmt
  *  A string to describe the table
  *
+ * @param source
+ *  The ID of the client/source.
+ *
  * @return fib_index
  *  The index of the FIB
  */
 extern u32 fib_table_create_and_lock(fib_protocol_t proto,
+                                     fib_source_t source,
                                      const char *const fmt,
                                      ...);
 
@@ -704,9 +718,13 @@ extern void fib_table_set_flow_hash_config(u32 fib_index,
  *
  * @paran proto
  *  The protocol of the FIB (and thus the entries therein)
+ *
+ * @param source
+ *  The ID of the client/source.
  */ 
 extern void fib_table_unlock(u32 fib_index,
-			     fib_protocol_t proto);
+			     fib_protocol_t proto,
+                             fib_source_t source);
 
 /**
  * @brief
@@ -718,9 +736,13 @@ extern void fib_table_unlock(u32 fib_index,
  *
  * @paran proto
  *  The protocol of the FIB (and thus the entries therein)
+ *
+ * @param source
+ *  The ID of the client/source.
  */ 
 extern void fib_table_lock(u32 fib_index,
-			   fib_protocol_t proto);
+			   fib_protocol_t proto,
+                           fib_source_t source);
 
 /**
  * @brief
diff --git a/src/vnet/fib/fib_test.c b/src/vnet/fib/fib_test.c
index 6867cca8..572d7f0d 100644
--- a/src/vnet/fib/fib_test.c
+++ b/src/vnet/fib/fib_test.c
@@ -739,7 +739,8 @@ fib_test_v4 (void)
     lb_count = pool_elts(load_balance_pool);
 
     /* Find or create FIB table 11 */
-    fib_index = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP4, 11);
+    fib_index = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP4, 11,
+                                                  FIB_SOURCE_API);
 
     for (ii = 0; ii < 4; ii++)
     {
@@ -4150,7 +4151,7 @@ fib_test_v4 (void)
                                              FIB_SOURCE_INTERFACE)),
              "NO INterface Source'd prefixes");
 
-    fib_table_unlock(fib_index, FIB_PROTOCOL_IP4);
+    fib_table_unlock(fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_API);
 
     FIB_TEST((0  == fib_path_list_db_size()), "path list DB population:%d",
     	     fib_path_list_db_size());
@@ -4201,7 +4202,8 @@ fib_test_v6 (void)
     dpo_drop = drop_dpo_get(DPO_PROTO_IP6);
 
     /* Find or create FIB table 11 */
-    fib_index = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP6, 11);
+    fib_index = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP6, 11,
+                                                  FIB_SOURCE_API);
 
     for (ii = 0; ii < 4; ii++)
     {
@@ -5025,7 +5027,7 @@ fib_test_v6 (void)
     /*
      * now remove the VRF
      */
-    fib_table_unlock(fib_index, FIB_PROTOCOL_IP6);
+    fib_table_unlock(fib_index, FIB_PROTOCOL_IP6, FIB_SOURCE_API);
 
     FIB_TEST((0 == fib_path_list_db_size()),   "path list DB population:%d",
 	     fib_path_list_db_size());
@@ -5157,7 +5159,9 @@ fib_test_ae (void)
      */
     u32 import_fib_index1;
 
-    import_fib_index1 = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP4, 11);
+    import_fib_index1 = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP4,
+                                                          11,
+                                                          FIB_SOURCE_CLI);
 
     /*
      * Add an attached route in the import FIB
@@ -5233,7 +5237,8 @@ fib_test_ae (void)
      */
     u32 import_fib_index2;
 
-    import_fib_index2 = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP4, 12);
+    import_fib_index2 = fib_table_find_or_create_and_lock(FIB_PROTOCOL_IP4, 12,
+                                                          FIB_SOURCE_CLI);
 
     /*
      * Add an attached route in the import FIB
@@ -5595,8 +5600,8 @@ fib_test_ae (void)
 			   &local_pfx,
 			   FIB_SOURCE_API);
 
-    fib_table_unlock(import_fib_index1, FIB_PROTOCOL_IP4);
-    fib_table_unlock(import_fib_index2, FIB_PROTOCOL_IP4);
+    fib_table_unlock(import_fib_index1, FIB_PROTOCOL_IP4, FIB_SOURCE_CLI);
+    fib_table_unlock(import_fib_index2, FIB_PROTOCOL_IP4, FIB_SOURCE_CLI);
 
     FIB_TEST((0 == adj_nbr_db_size()), "ADJ DB size is %d",
 	     adj_nbr_db_size());
@@ -8168,9 +8173,10 @@ lfib_test (void)
     /*
      * MPLS enable an interface so we get the MPLS table created
      */
+    mpls_table_create(MPLS_FIB_DEFAULT_TABLE_ID, FIB_SOURCE_API);
     mpls_sw_interface_enable_disable(&mpls_main,
                                      tm->hw[0]->sw_if_index,
-                                     1);
+                                     1, 1);
 
     ip46_address_t nh_10_10_10_1 = {
 	.ip4.as_u32 = clib_host_to_net_u32(0x0a0a0a01),
@@ -8662,7 +8668,8 @@ lfib_test (void)
      */
     mpls_sw_interface_enable_disable(&mpls_main,
                                      tm->hw[0]->sw_if_index,
-                                     0);
+                                     0, 1);
+    mpls_table_delete(MPLS_FIB_DEFAULT_TABLE_ID, FIB_SOURCE_API);
 
     FIB_TEST(lb_count == pool_elts(load_balance_pool),
 	     "Load-balance resources freed %d of %d",
diff --git a/src/vnet/fib/ip4_fib.c b/src/vnet/fib/ip4_fib.c
index d563bafd..865e2dd5 100644
--- a/src/vnet/fib/ip4_fib.c
+++ b/src/vnet/fib/ip4_fib.c
@@ -101,7 +101,8 @@ static const ip4_fib_table_special_prefix_t ip4_specials[] = {
 
 
 static u32
-ip4_create_fib_with_table_id (u32 table_id)
+ip4_create_fib_with_table_id (u32 table_id,
+                              fib_source_t src)
 {
     fib_table_t *fib_table;
     ip4_fib_t *v4_fib;
@@ -128,7 +129,7 @@ ip4_create_fib_with_table_id (u32 table_id)
     v4_fib->fwd_classify_table_index = ~0;
     v4_fib->rev_classify_table_index = ~0;
     
-    fib_table_lock(fib_table->ft_index, FIB_PROTOCOL_IP4);
+    fib_table_lock(fib_table->ft_index, FIB_PROTOCOL_IP4, src);
 
     ip4_mtrie_init(&v4_fib->mtrie);
 
@@ -198,23 +199,24 @@ ip4_fib_table_destroy (u32 fib_index)
 
 
 u32
-ip4_fib_table_find_or_create_and_lock (u32 table_id)
+ip4_fib_table_find_or_create_and_lock (u32 table_id,
+                                       fib_source_t src)
 {
     u32 index;
 
     index = ip4_fib_index_from_table_id(table_id);
     if (~0 == index)
-	return ip4_create_fib_with_table_id(table_id);
+	return ip4_create_fib_with_table_id(table_id, src);
 
-    fib_table_lock(index, FIB_PROTOCOL_IP4);
+    fib_table_lock(index, FIB_PROTOCOL_IP4, src);
 
     return (index);
 }
 
 u32
-ip4_fib_table_create_and_lock (void)
+ip4_fib_table_create_and_lock (fib_source_t src)
 {
-    return (ip4_create_fib_with_table_id(~0));
+    return (ip4_create_fib_with_table_id(~0, src));
 }
 
 u32
@@ -525,17 +527,32 @@ ip4_show_fib (vlib_main_t * vm,
     pool_foreach (fib_table, im4->fibs,
     ({
 	ip4_fib_t *fib = pool_elt_at_index(im4->v4_fibs, fib_table->ft_index);
+        fib_source_t source;
+        u8 *s = NULL;
 
 	if (table_id >= 0 && table_id != (int)fib->table_id)
 	    continue;
 	if (fib_index != ~0 && fib_index != (int)fib->index)
 	    continue;
 
-	vlib_cli_output (vm, "%U, fib_index:%d, flow hash:[%U] locks:%d", 
-			 format_fib_table_name, fib->index, FIB_PROTOCOL_IP4,
-			 fib->index,
-			 format_ip_flow_hash_config, fib_table->ft_flow_hash_config,
-                         fib_table->ft_locks);
+	s = format(s, "%U, fib_index:%d, flow hash:[%U] locks:[",
+                   format_fib_table_name, fib->index,
+                   FIB_PROTOCOL_IP4,
+                   fib->index,
+                   format_ip_flow_hash_config,
+                   fib_table->ft_flow_hash_config);
+	FOR_EACH_FIB_SOURCE(source)
+        {
+            if (0 != fib_table->ft_locks[source])
+            {
+                s = format(s, "%U:%d, ",
+                           format_fib_source, source,
+                           fib_table->ft_locks[source]);
+            }
+        }
+        s = format (s, "]");
+        vlib_cli_output (vm, "%V", s);
+        vec_free(s);
 
 	/* Show summary? */
 	if (! verbose)
diff --git a/src/vnet/fib/ip4_fib.h b/src/vnet/fib/ip4_fib.h
index 006163b4..495b45cc 100644
--- a/src/vnet/fib/ip4_fib.h
+++ b/src/vnet/fib/ip4_fib.h
@@ -127,8 +127,9 @@ ip4_fib_lookup (ip4_main_t * im, u32 sw_if_index, ip4_address_t * dst)
  * @returns A pointer to the retrieved or created fib.
  *
  */
-extern u32 ip4_fib_table_find_or_create_and_lock(u32 table_id);
-extern u32 ip4_fib_table_create_and_lock(void);
+extern u32 ip4_fib_table_find_or_create_and_lock(u32 table_id,
+                                                 fib_source_t src);
+extern u32 ip4_fib_table_create_and_lock(fib_source_t src);
 
 
 static inline 
diff --git a/src/vnet/fib/ip6_fib.c b/src/vnet/fib/ip6_fib.c
index 8fde6f9f..3ddb8453 100644
--- a/src/vnet/fib/ip6_fib.c
+++ b/src/vnet/fib/ip6_fib.c
@@ -50,7 +50,8 @@ vnet_ip6_fib_init (u32 fib_index)
 }
 
 static u32
-create_fib_with_table_id (u32 table_id)
+create_fib_with_table_id (u32 table_id,
+                          fib_source_t src)
 {
     fib_table_t *fib_table;
     ip6_fib_t *v6_fib;
@@ -77,29 +78,30 @@ create_fib_with_table_id (u32 table_id)
     fib_table->ft_flow_hash_config = IP_FLOW_HASH_DEFAULT;
 
     vnet_ip6_fib_init(fib_table->ft_index);
-    fib_table_lock(fib_table->ft_index, FIB_PROTOCOL_IP6);
+    fib_table_lock(fib_table->ft_index, FIB_PROTOCOL_IP6, src);
 
     return (fib_table->ft_index);
 }
 
 u32
-ip6_fib_table_find_or_create_and_lock (u32 table_id)
+ip6_fib_table_find_or_create_and_lock (u32 table_id,
+                                       fib_source_t src)
 {
     uword * p;
 
     p = hash_get (ip6_main.fib_index_by_table_id, table_id);
     if (NULL == p)
-	return create_fib_with_table_id(table_id);
+	return create_fib_with_table_id(table_id, src);
     
-    fib_table_lock(p[0], FIB_PROTOCOL_IP6);
+    fib_table_lock(p[0], FIB_PROTOCOL_IP6, src);
 
     return (p[0]);
 }
 
 u32
-ip6_fib_table_create_and_lock (void)
+ip6_fib_table_create_and_lock (fib_source_t src)
 {
-    return (create_fib_with_table_id(~0));
+    return (create_fib_with_table_id(~0, src));
 }
 
 void
@@ -588,16 +590,33 @@ ip6_show_fib (vlib_main_t * vm,
 
     pool_foreach (fib_table, im6->fibs,
     ({
+        fib_source_t source;
+        u8 *s = NULL;
+
 	fib = pool_elt_at_index(im6->v6_fibs, fib_table->ft_index);
 	if (table_id >= 0 && table_id != (int)fib->table_id)
 	    continue;
 	if (fib_index != ~0 && fib_index != (int)fib->index)
 	    continue;
 
-	vlib_cli_output (vm, "%s, fib_index:%d, flow hash:[%U] locks:%d", 
-			 fib_table->ft_desc, fib->index,
-			 format_ip_flow_hash_config, fib_table->ft_flow_hash_config,
-                         fib_table->ft_locks);
+	s = format(s, "%U, fib_index:%d, flow hash:[%U] locks:[",
+                   format_fib_table_name, fib->index,
+                   FIB_PROTOCOL_IP6,
+                   fib->index,
+                   format_ip_flow_hash_config,
+                   fib_table->ft_flow_hash_config);
+	FOR_EACH_FIB_SOURCE(source)
+        {
+            if (0 != fib_table->ft_locks[source])
+            {
+                s = format(s, "%U:%d, ",
+                           format_fib_source, source,
+                           fib_table->ft_locks[source]);
+            }
+        }
+        s = format (s, "]");
+        vlib_cli_output (vm, "%V", s);
+        vec_free(s);
 
 	/* Show summary? */
 	if (! verbose)
diff --git a/src/vnet/fib/ip6_fib.h b/src/vnet/fib/ip6_fib.h
index aad8305c..9728eecc 100644
--- a/src/vnet/fib/ip6_fib.h
+++ b/src/vnet/fib/ip6_fib.h
@@ -144,8 +144,9 @@ ip6_src_lookup_for_packet (ip6_main_t * im,
  * \returns A pointer to the retrieved or created fib.
  *
  */
-extern u32 ip6_fib_table_find_or_create_and_lock(u32 table_id);
-extern u32 ip6_fib_table_create_and_lock(void);
+extern u32 ip6_fib_table_find_or_create_and_lock(u32 table_id,
+                                                 fib_source_t src);
+extern u32 ip6_fib_table_create_and_lock(fib_source_t src);
 
 static inline ip6_fib_t *
 ip6_fib_get (fib_node_index_t index)
diff --git a/src/vnet/fib/mpls_fib.c b/src/vnet/fib/mpls_fib.c
index ca6271fe..4eeef7ab 100644
--- a/src/vnet/fib/mpls_fib.c
+++ b/src/vnet/fib/mpls_fib.c
@@ -83,7 +83,8 @@ mpls_fib_index_from_table_id (u32 table_id)
 }
 
 static u32
-mpls_fib_create_with_table_id (u32 table_id)
+mpls_fib_create_with_table_id (u32 table_id,
+                               fib_source_t src)
 {
     dpo_id_t dpo = DPO_INVALID;
     fib_table_t *fib_table;
@@ -107,7 +108,7 @@ mpls_fib_create_with_table_id (u32 table_id)
     fib_table->ft_table_id = table_id;
     fib_table->ft_flow_hash_config = MPLS_FLOW_HASH_DEFAULT;
     
-    fib_table_lock(fib_table->ft_index, FIB_PROTOCOL_MPLS);
+    fib_table_lock(fib_table->ft_index, FIB_PROTOCOL_MPLS, src);
 
     if (INDEX_INVALID == mpls_fib_drop_dpo_index)
     {
@@ -220,22 +221,23 @@ mpls_fib_create_with_table_id (u32 table_id)
 }
 
 u32
-mpls_fib_table_find_or_create_and_lock (u32 table_id)
+mpls_fib_table_find_or_create_and_lock (u32 table_id,
+                                        fib_source_t src)
 {
     u32 index;
 
     index = mpls_fib_index_from_table_id(table_id);
     if (~0 == index)
-	return mpls_fib_create_with_table_id(table_id);
+	return mpls_fib_create_with_table_id(table_id, src);
 
-    fib_table_lock(index, FIB_PROTOCOL_MPLS);
+    fib_table_lock(index, FIB_PROTOCOL_MPLS, src);
 
     return (index);
 }
 u32
-mpls_fib_table_create_and_lock (void)
+mpls_fib_table_create_and_lock (fib_source_t src)
 {
-    return (mpls_fib_create_with_table_id(~0));
+    return (mpls_fib_create_with_table_id(~0, src));
 }
 
 void
diff --git a/src/vnet/fib/mpls_fib.h b/src/vnet/fib/mpls_fib.h
index dfb8b7fc..29cd1d20 100644
--- a/src/vnet/fib/mpls_fib.h
+++ b/src/vnet/fib/mpls_fib.h
@@ -59,8 +59,9 @@ mpls_fib_get (fib_node_index_t index)
     return (pool_elt_at_index(mpls_main.mpls_fibs, index));
 }
 
-extern u32 mpls_fib_table_find_or_create_and_lock(u32 table_id);
-extern u32 mpls_fib_table_create_and_lock(void);
+extern u32 mpls_fib_table_find_or_create_and_lock(u32 table_id,
+                                                  fib_source_t src);
+extern u32 mpls_fib_table_create_and_lock(fib_source_t src);
 // extern mpls_fib_t * mpls_fib_find(u32 table_id);
 extern u32 mpls_fib_index_from_table_id(u32 table_id);
 
diff --git a/src/vnet/interface_api.c b/src/vnet/interface_api.c
index 113728cd..419fef94 100644
--- a/src/vnet/interface_api.c
+++ b/src/vnet/interface_api.c
@@ -320,68 +320,189 @@ stats_dsunlock (void)
 static void
 vl_api_sw_interface_set_table_t_handler (vl_api_sw_interface_set_table_t * mp)
 {
-  int rv = 0;
-  u32 table_id = ntohl (mp->vrf_id);
-  u32 sw_if_index = ntohl (mp->sw_if_index);
   vl_api_sw_interface_set_table_reply_t *rmp;
-  CLIB_UNUSED (ip_interface_address_t * ia);
-  u32 fib_index;
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+  u32 table_id = ntohl (mp->vrf_id);
+  int rv = 0;
 
   VALIDATE_SW_IF_INDEX (mp);
 
   stats_dslock_with_hint (1 /* release hint */ , 4 /* tag */ );
 
   if (mp->is_ipv6)
+    rv = ip_table_bind (FIB_PROTOCOL_IP6, sw_if_index, table_id, 1);
+  else
+    rv = ip_table_bind (FIB_PROTOCOL_IP4, sw_if_index, table_id, 1);
+
+  stats_dsunlock ();
+
+  BAD_SW_IF_INDEX_LABEL;
+
+  REPLY_MACRO (VL_API_SW_INTERFACE_SET_TABLE_REPLY);
+}
+
+int
+ip_table_bind (fib_protocol_t fproto,
+	       uint32_t sw_if_index, uint32_t table_id, u8 is_api)
+{
+  CLIB_UNUSED (ip_interface_address_t * ia);
+  u32 fib_index, mfib_index;
+  fib_source_t src;
+  mfib_source_t msrc;
+
+  if (is_api)
+    {
+      src = FIB_SOURCE_API;
+      msrc = MFIB_SOURCE_API;
+    }
+  else
+    {
+      src = FIB_SOURCE_CLI;
+      msrc = MFIB_SOURCE_CLI;
+    }
+
+  /*
+   * This is temporary whilst I do the song and dance with the CSIT version
+   */
+  if (0 != table_id)
     {
+      fib_index = fib_table_find_or_create_and_lock (fproto, table_id, src);
+      mfib_index =
+	mfib_table_find_or_create_and_lock (fproto, table_id, msrc);
+    }
+  else
+    {
+      fib_index = 0;
+      mfib_index = 0;
+    }
+
+  /*
+   * This if table does not exist = error is what we want in the end.
+   */
+  /* fib_index = fib_table_find (fproto, table_id); */
+  /* mfib_index = mfib_table_find (fproto, table_id); */
+
+  /* if (~0 == fib_index || ~0 == mfib_index) */
+  /*   { */
+  /*     return (VNET_API_ERROR_NO_SUCH_FIB); */
+  /*   } */
+
+  if (FIB_PROTOCOL_IP6 == fproto)
+    {
+      /*
+       * If the interface already has in IP address, then a change int
+       * VRF is not allowed. The IP address applied must first be removed.
+       * We do not do that automatically here, since VPP has no knowledge
+       * of whether thoses subnets are valid in the destination VRF.
+       */
       /* *INDENT-OFF* */
       foreach_ip_interface_address (&ip6_main.lookup_main,
 				    ia, sw_if_index,
 				    1 /* honor unnumbered */ ,
       ({
-        rv = VNET_API_ERROR_ADDRESS_FOUND_FOR_INTERFACE;
-        goto done;
+        return (VNET_API_ERROR_ADDRESS_FOUND_FOR_INTERFACE);
       }));
       /* *INDENT-ON* */
 
-      fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
-						     table_id);
       vec_validate (ip6_main.fib_index_by_sw_if_index, sw_if_index);
-      ip6_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
-
-      fib_index = mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
-						      table_id);
       vec_validate (ip6_main.mfib_index_by_sw_if_index, sw_if_index);
-      ip6_main.mfib_index_by_sw_if_index[sw_if_index] = fib_index;
+
+      /*
+       * tell those that are interested that the binding is changing.
+       */
+      ip6_table_bind_callback_t *cb;
+      vec_foreach (cb, ip6_main.table_bind_callbacks)
+	cb->function (&ip6_main, cb->function_opaque,
+		      sw_if_index,
+		      fib_index,
+		      ip6_main.fib_index_by_sw_if_index[sw_if_index]);
+
+      if (0 == table_id)
+	{
+	  /* reset back to default */
+	  if (0 != ip6_main.fib_index_by_sw_if_index[sw_if_index])
+	    fib_table_unlock (ip6_main.fib_index_by_sw_if_index[sw_if_index],
+			      FIB_PROTOCOL_IP6, src);
+	  if (0 != ip6_main.mfib_index_by_sw_if_index[sw_if_index])
+	    mfib_table_unlock (ip6_main.mfib_index_by_sw_if_index
+			       [sw_if_index], FIB_PROTOCOL_IP6, msrc);
+
+	}
+      else
+	{
+	  /* we need to lock the table now it's inuse */
+	  fib_table_lock (fib_index, FIB_PROTOCOL_IP6, src);
+	  mfib_table_lock (mfib_index, FIB_PROTOCOL_IP6, msrc);
+	}
+
+      ip6_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
+      ip6_main.mfib_index_by_sw_if_index[sw_if_index] = mfib_index;
     }
   else
     {
+      /*
+       * If the interface already has in IP address, then a change int
+       * VRF is not allowed. The IP address applied must first be removed.
+       * We do not do that automatically here, since VPP has no knowledge
+       * of whether thoses subnets are valid in the destination VRF.
+       */
       /* *INDENT-OFF* */
       foreach_ip_interface_address (&ip4_main.lookup_main,
 				    ia, sw_if_index,
 				    1 /* honor unnumbered */ ,
       ({
-        rv = VNET_API_ERROR_ADDRESS_FOUND_FOR_INTERFACE;
-        goto done;
+        return (VNET_API_ERROR_ADDRESS_FOUND_FOR_INTERFACE);
       }));
       /* *INDENT-ON* */
 
-      fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
-						     table_id);
       vec_validate (ip4_main.fib_index_by_sw_if_index, sw_if_index);
-      ip4_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
-
-      fib_index = mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
-						      table_id);
       vec_validate (ip4_main.mfib_index_by_sw_if_index, sw_if_index);
-      ip4_main.mfib_index_by_sw_if_index[sw_if_index] = fib_index;
-    }
 
-done:
-  stats_dsunlock ();
+      /*
+       * tell those that are interested that the binding is changing.
+       */
+      ip4_table_bind_callback_t *cb;
+      vec_foreach (cb, ip4_main.table_bind_callbacks)
+	cb->function (&ip4_main, cb->function_opaque,
+		      sw_if_index,
+		      fib_index,
+		      ip4_main.fib_index_by_sw_if_index[sw_if_index]);
+
+      if (0 == table_id)
+	{
+	  /* reset back to default */
+	  if (0 != ip4_main.fib_index_by_sw_if_index[sw_if_index])
+	    fib_table_unlock (ip4_main.fib_index_by_sw_if_index[sw_if_index],
+			      FIB_PROTOCOL_IP4, src);
+	  if (0 != ip4_main.mfib_index_by_sw_if_index[sw_if_index])
+	    mfib_table_unlock (ip4_main.mfib_index_by_sw_if_index
+			       [sw_if_index], FIB_PROTOCOL_IP4, msrc);
 
-  BAD_SW_IF_INDEX_LABEL;
+	}
+      else
+	{
+	  /* we need to lock the table now it's inuse */
+	  fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
+							 table_id, src);
 
-  REPLY_MACRO (VL_API_SW_INTERFACE_SET_TABLE_REPLY);
+	  mfib_index = mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
+							   table_id, msrc);
+	}
+
+      ip4_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
+      ip4_main.mfib_index_by_sw_if_index[sw_if_index] = mfib_index;
+    }
+
+  /*
+   * Temporary. undo the locks from the find and create at the staart
+   */
+  if (0 != table_id)
+    {
+      fib_table_unlock (fib_index, fproto, src);
+      mfib_table_unlock (mfib_index, fproto, msrc);
+    }
+
+  return (0);
 }
 
 static void
diff --git a/src/vnet/ip/ip.h b/src/vnet/ip/ip.h
index 70b4ccd8..7aae73ff 100644
--- a/src/vnet/ip/ip.h
+++ b/src/vnet/ip/ip.h
@@ -184,6 +184,13 @@ void ip_del_all_interface_addresses (vlib_main_t * vm, u32 sw_if_index);
 extern vlib_node_registration_t ip4_inacl_node;
 extern vlib_node_registration_t ip6_inacl_node;
 
+void ip_table_create (fib_protocol_t fproto, uint32_t table_id, u8 is_api);
+
+void ip_table_delete (fib_protocol_t fproto, uint32_t table_id, u8 is_api);
+
+int ip_table_bind (fib_protocol_t fproto,
+		   uint32_t sw_if_index, uint32_t table_id, u8 is_api);
+
 #endif /* included_ip_main_h */
 
 /*
diff --git a/src/vnet/ip/ip4.h b/src/vnet/ip/ip4.h
index 8f9a8e27..decb840b 100644
--- a/src/vnet/ip/ip4.h
+++ b/src/vnet/ip/ip4.h
@@ -72,6 +72,16 @@ typedef struct
   uword function_opaque;
 } ip4_add_del_interface_address_callback_t;
 
+typedef void (ip4_table_bind_function_t)
+  (struct ip4_main_t * im,
+   uword opaque, u32 sw_if_index, u32 new_fib_index, u32 old_fib_index);
+
+typedef struct
+{
+  ip4_table_bind_function_t *function;
+  uword function_opaque;
+} ip4_table_bind_callback_t;
+
 /**
  * @brief IPv4 main type.
  *
@@ -117,6 +127,9 @@ typedef struct ip4_main_t
     ip4_add_del_interface_address_callback_t
     * add_del_interface_address_callbacks;
 
+  /** Functions to call when interface to table biding changes. */
+  ip4_table_bind_callback_t *table_bind_callbacks;
+
   /** Template used to generate IP4 ARP packets. */
   vlib_packet_template_t ip4_arp_request_packet_template;
 
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index 2d48e8a9..ec4287bb 100755
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -1198,8 +1198,10 @@ ip4_lookup_init (vlib_main_t * vm)
   ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
 
   /* Create FIB with index 0 and table id of 0. */
-  fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0);
-  mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0);
+  fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0,
+				     FIB_SOURCE_DEFAULT_ROUTE);
+  mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, 0,
+				      MFIB_SOURCE_DEFAULT_ROUTE);
 
   {
     pg_node_t *pn;
@@ -2794,101 +2796,6 @@ VLIB_REGISTER_NODE (ip4_midchain_node) = {
 VLIB_NODE_FUNCTION_MULTIARCH (ip4_midchain_node, ip4_midchain);
 /* *INDENT-ON */
 
-static clib_error_t *
-add_del_interface_table (vlib_main_t * vm,
-			 unformat_input_t * input, vlib_cli_command_t * cmd)
-{
-  vnet_main_t *vnm = vnet_get_main ();
-  ip_interface_address_t *ia;
-  clib_error_t *error = 0;
-  u32 sw_if_index, table_id;
-
-  sw_if_index = ~0;
-
-  if (!unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
-    {
-      error = clib_error_return (0, "unknown interface `%U'",
-				 format_unformat_error, input);
-      goto done;
-    }
-
-  if (unformat (input, "%d", &table_id))
-    ;
-  else
-    {
-      error = clib_error_return (0, "expected table id `%U'",
-				 format_unformat_error, input);
-      goto done;
-    }
-
-  /*
-   * If the interface already has in IP address, then a change int
-   * VRF is not allowed. The IP address applied must first be removed.
-   * We do not do that automatically here, since VPP has no knowledge
-   * of whether thoses subnets are valid in the destination VRF.
-   */
-  /* *INDENT-OFF* */
-  foreach_ip_interface_address (&ip4_main.lookup_main,
-                                ia, sw_if_index,
-                                1 /* honor unnumbered */,
-  ({
-      ip4_address_t * a;
-
-      a = ip_interface_address_get_address (&ip4_main.lookup_main, ia);
-      error = clib_error_return (0, "interface %U has address %U",
-                                 format_vnet_sw_if_index_name, vnm,
-                                 sw_if_index,
-                                 format_ip4_address, a);
-      goto done;
-   }));
-   /* *INDENT-ON* */
-
-{
-  ip4_main_t *im = &ip4_main;
-  u32 fib_index;
-
-  fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, table_id);
-
-  vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
-  im->fib_index_by_sw_if_index[sw_if_index] = fib_index;
-
-  fib_index = mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, table_id);
-  vec_validate (im->mfib_index_by_sw_if_index, sw_if_index);
-  im->mfib_index_by_sw_if_index[sw_if_index] = fib_index;
-}
-
-done:
-return error;
-}
-
-/*?
- * Place the indicated interface into the supplied IPv4 FIB table (also known
- * as a VRF). If the FIB table does not exist, this command creates it. To
- * display the current IPv4 FIB table, use the command '<em>show ip fib</em>'.
- * FIB table will only be displayed if a route has been added to the table, or
- * an IP Address is assigned to an interface in the table (which adds a route
- * automatically).
- *
- * @note IP addresses added after setting the interface IP table are added to
- * the indicated FIB table. If an IP address is added prior to changing the
- * table then this is an error. The control plane must remove these addresses
- * first and then change the table. VPP will not automatically move the
- * addresses from the old to the new table as it does not know the validity
- * of such a change.
- *
- * @cliexpar
- * Example of how to add an interface to an IPv4 FIB table (where 2 is the table-id):
- * @cliexcmd{set interface ip table GigabitEthernet2/0/0 2}
- ?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (set_interface_ip_table_command, static) =
-{
-  .path = "set interface ip table",
-  .function = add_del_interface_table,
-  .short_help = "set interface ip table <interface> <table-id>",
-};
-/* *INDENT-ON* */
-
 int
 ip4_lookup_validate (ip4_address_t * a, u32 fib_index0)
 {
diff --git a/src/vnet/ip/ip4_source_and_port_range_check.c b/src/vnet/ip/ip4_source_and_port_range_check.c
index ae836a11..9aa880ae 100644
--- a/src/vnet/ip/ip4_source_and_port_range_check.c
+++ b/src/vnet/ip/ip4_source_and_port_range_check.c
@@ -1126,6 +1126,14 @@ ip6_source_and_port_range_check_add_del (ip6_address_t * address,
 					 u16 * low_ports,
 					 u16 * high_ports, int is_add)
 {
+  uint32_t fib_index;
+
+  fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
+
+  ASSERT (~0 != fib_index);
+
+  fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_CLASSIFY);
+
   return 0;
 }
 
@@ -1138,7 +1146,8 @@ ip4_source_and_port_range_check_add_del (ip4_address_t * address,
 {
   u32 fib_index;
 
-  fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id);
+  fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
+						 FIB_SOURCE_CLASSIFY);
 
   if (is_add == 0)
     {
diff --git a/src/vnet/ip/ip6.h b/src/vnet/ip/ip6.h
index fa922725..8aef53a9 100644
--- a/src/vnet/ip/ip6.h
+++ b/src/vnet/ip/ip6.h
@@ -103,6 +103,16 @@ typedef struct
   uword function_opaque;
 } ip6_add_del_interface_address_callback_t;
 
+typedef void (ip6_table_bind_function_t)
+  (struct ip6_main_t * im,
+   uword opaque, u32 sw_if_index, u32 new_fib_index, u32 old_fib_index);
+
+typedef struct
+{
+  ip6_table_bind_function_t *function;
+  uword function_opaque;
+} ip6_table_bind_callback_t;
+
 /**
  * Enumeration of the FIB table instance types
  */
@@ -183,6 +193,9 @@ typedef struct ip6_main_t
     ip6_add_del_interface_address_callback_t
     * add_del_interface_address_callbacks;
 
+  /** Functions to call when interface to table biding changes. */
+  ip6_table_bind_callback_t *table_bind_callbacks;
+
   /* Template used to generate IP6 neighbor solicitation packets. */
   vlib_packet_template_t discover_neighbor_packet_template;
 
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index 5832bd0b..1002f6b6 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -2999,8 +2999,10 @@ ip6_lookup_init (vlib_main_t * vm)
 			 im->lookup_table_nbuckets, im->lookup_table_size);
 
   /* Create FIB with index 0 and table id of 0. */
-  fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0);
-  mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0);
+  fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
+				     FIB_SOURCE_DEFAULT_ROUTE);
+  mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, 0,
+				      MFIB_SOURCE_DEFAULT_ROUTE);
 
   {
     pg_node_t *pn;
@@ -3045,103 +3047,6 @@ ip6_lookup_init (vlib_main_t * vm)
 
 VLIB_INIT_FUNCTION (ip6_lookup_init);
 
-static clib_error_t *
-add_del_ip6_interface_table (vlib_main_t * vm,
-			     unformat_input_t * input,
-			     vlib_cli_command_t * cmd)
-{
-  vnet_main_t *vnm = vnet_get_main ();
-  ip_interface_address_t *ia;
-  clib_error_t *error = 0;
-  u32 sw_if_index, table_id;
-
-  sw_if_index = ~0;
-
-  if (!unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
-    {
-      error = clib_error_return (0, "unknown interface `%U'",
-				 format_unformat_error, input);
-      goto done;
-    }
-
-  if (unformat (input, "%d", &table_id))
-    ;
-  else
-    {
-      error = clib_error_return (0, "expected table id `%U'",
-				 format_unformat_error, input);
-      goto done;
-    }
-
-  /*
-   * If the interface already has in IP address, then a change int
-   * VRF is not allowed. The IP address applied must first be removed.
-   * We do not do that automatically here, since VPP has no knowledge
-   * of whether thoses subnets are valid in the destination VRF.
-   */
-  /* *INDENT-OFF* */
-  foreach_ip_interface_address (&ip6_main.lookup_main,
-                                ia, sw_if_index,
-                                1 /* honor unnumbered */,
-  ({
-      ip4_address_t * a;
-
-      a = ip_interface_address_get_address (&ip6_main.lookup_main, ia);
-      error = clib_error_return (0, "interface %U has address %U",
-                                 format_vnet_sw_if_index_name, vnm,
-                                 sw_if_index,
-                                 format_ip6_address, a);
-      goto done;
-  }));
-  /* *INDENT-ON* */
-
-  {
-    u32 fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
-						       table_id);
-
-    vec_validate (ip6_main.fib_index_by_sw_if_index, sw_if_index);
-    ip6_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
-
-    fib_index = mfib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6,
-						    table_id);
-
-    vec_validate (ip6_main.mfib_index_by_sw_if_index, sw_if_index);
-    ip6_main.mfib_index_by_sw_if_index[sw_if_index] = fib_index;
-  }
-
-
-done:
-  return error;
-}
-
-/*?
- * Place the indicated interface into the supplied IPv6 FIB table (also known
- * as a VRF). If the FIB table does not exist, this command creates it. To
- * display the current IPv6 FIB table, use the command '<em>show ip6 fib</em>'.
- * FIB table will only be displayed if a route has been added to the table, or
- * an IP Address is assigned to an interface in the table (which adds a route
- * automatically).
- *
- * @note IP addresses added after setting the interface IP table are added to
- * the indicated FIB table. If an IP address is added prior to changing the
- * table then this is an error. The control plane must remove these addresses
- * first and then change the table. VPP will not automatically move the
- * addresses from the old to the new table as it does not know the validity
- * of such a change.
- *
- * @cliexpar
- * Example of how to add an interface to an IPv6 FIB table (where 2 is the table-id):
- * @cliexcmd{set interface ip6 table GigabitEthernet2/0/0 2}
- ?*/
-/* *INDENT-OFF* */
-VLIB_CLI_COMMAND (set_interface_ip6_table_command, static) =
-{
-  .path = "set interface ip6 table",
-  .function = add_del_ip6_interface_table,
-  .short_help = "set interface ip6 table <interface> <table-id>"
-};
-/* *INDENT-ON* */
-
 void
 ip6_link_local_address_from_ethernet_mac_address (ip6_address_t * ip,
 						  u8 * mac)
diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c
index 62cf23ac..56f33ac8 100644
--- a/src/vnet/ip/ip6_neighbor.c
+++ b/src/vnet/ip/ip6_neighbor.c
@@ -250,6 +250,26 @@ format_ip6_neighbor_ip6_entry (u8 * s, va_list * va)
   return s;
 }
 
+static void
+ip6_neighbor_adj_fib_remove (ip6_neighbor_t * n, uint32_t fib_index)
+{
+  if (FIB_NODE_INDEX_INVALID != n->fib_entry_index)
+    {
+      fib_prefix_t pfx = {
+	.fp_len = 128,
+	.fp_proto = FIB_PROTOCOL_IP6,
+	.fp_addr.ip6 = n->key.ip6_address,
+      };
+      fib_table_entry_path_remove (fib_index,
+				   &pfx,
+				   FIB_SOURCE_ADJ,
+				   DPO_PROTO_IP6,
+				   &pfx.fp_addr,
+				   n->key.sw_if_index, ~0,
+				   1, FIB_ROUTE_PATH_FLAG_NONE);
+    }
+}
+
 static clib_error_t *
 ip6_neighbor_sw_interface_up_down (vnet_main_t * vnm,
 				   u32 sw_if_index, u32 flags)
@@ -273,22 +293,10 @@ ip6_neighbor_sw_interface_up_down (vnet_main_t * vnm,
 	{
 	  n = pool_elt_at_index (nm->neighbor_pool, to_delete[i]);
 	  mhash_unset (&nm->neighbor_index_by_key, &n->key, 0);
-	  if (FIB_NODE_INDEX_INVALID != n->fib_entry_index)
-	    {
-	      fib_prefix_t pfx = {
-		.fp_len = 128,
-		.fp_proto = FIB_PROTOCOL_IP6,
-		.fp_addr.ip6 = n->key.ip6_address,
-	      };
-	      fib_table_entry_path_remove
-		(ip6_fib_table_get_index_for_sw_if_index (n->key.sw_if_index),
-		 &pfx,
-		 FIB_SOURCE_ADJ,
-		 DPO_PROTO_IP6,
-		 &pfx.fp_addr,
-		 n->key.sw_if_index, ~0, 1, FIB_ROUTE_PATH_FLAG_NONE);
-	      pool_put (nm->neighbor_pool, n);
-	    }
+	  ip6_neighbor_adj_fib_remove (n,
+				       ip6_fib_table_get_index_for_sw_if_index
+				       (n->key.sw_if_index));
+	  pool_put (nm->neighbor_pool, n);
 	}
       vec_free (to_delete);
     }
@@ -579,6 +587,24 @@ ip6_ethernet_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai)
     }
 }
 
+
+static void
+ip6_neighbor_adj_fib_add (ip6_neighbor_t * n, uint32_t fib_index)
+{
+  fib_prefix_t pfx = {
+    .fp_len = 128,
+    .fp_proto = FIB_PROTOCOL_IP6,
+    .fp_addr.ip6 = n->key.ip6_address,
+  };
+
+  n->fib_entry_index =
+    fib_table_entry_path_add (fib_index, &pfx, FIB_SOURCE_ADJ,
+			      FIB_ENTRY_FLAG_ATTACHED,
+			      DPO_PROTO_IP6, &pfx.fp_addr,
+			      n->key.sw_if_index, ~0, 1, NULL,
+			      FIB_ROUTE_PATH_FLAG_NONE);
+}
+
 int
 vnet_set_ip6_ethernet_neighbor (vlib_main_t * vm,
 				u32 sw_if_index,
@@ -633,21 +659,9 @@ vnet_set_ip6_ethernet_neighbor (vlib_main_t * vm,
        */
       if (!is_no_fib_entry)
 	{
-	  fib_prefix_t pfx = {
-	    .fp_len = 128,
-	    .fp_proto = FIB_PROTOCOL_IP6,
-	    .fp_addr.ip6 = k.ip6_address,
-	  };
-	  u32 fib_index;
-
-	  fib_index =
-	    ip6_fib_table_get_index_for_sw_if_index (n->key.sw_if_index);
-	  n->fib_entry_index =
-	    fib_table_entry_path_add (fib_index, &pfx, FIB_SOURCE_ADJ,
-				      FIB_ENTRY_FLAG_ATTACHED,
-				      DPO_PROTO_IP6, &pfx.fp_addr,
-				      n->key.sw_if_index, ~0, 1, NULL,
-				      FIB_ROUTE_PATH_FLAG_NONE);
+	  ip6_neighbor_adj_fib_add (n,
+				    ip6_fib_table_get_index_for_sw_if_index
+				    (n->key.sw_if_index));
 	}
       else
 	{
@@ -3843,6 +3857,33 @@ ip6_set_neighbor_limit (u32 neighbor_limit)
   return 0;
 }
 
+static void
+ip6_neighbor_table_bind (ip6_main_t * im,
+			 uword opaque,
+			 u32 sw_if_index,
+			 u32 new_fib_index, u32 old_fib_index)
+{
+  ip6_neighbor_main_t *nm = &ip6_neighbor_main;
+  ip6_neighbor_t *n = NULL;
+  u32 i, *to_re_add = 0;
+
+  /* *INDENT-OFF* */
+  pool_foreach (n, nm->neighbor_pool,
+  ({
+    if (n->key.sw_if_index == sw_if_index)
+      vec_add1 (to_re_add, n - nm->neighbor_pool);
+  }));
+  /* *INDENT-ON* */
+
+  for (i = 0; i < vec_len (to_re_add); i++)
+    {
+      n = pool_elt_at_index (nm->neighbor_pool, to_re_add[i]);
+      ip6_neighbor_adj_fib_remove (n, old_fib_index);
+      ip6_neighbor_adj_fib_add (n, new_fib_index);
+    }
+  vec_free (to_re_add);
+}
+
 static clib_error_t *
 ip6_neighbor_init (vlib_main_t * vm)
 {
@@ -3874,6 +3915,11 @@ ip6_neighbor_init (vlib_main_t * vm)
   cb.function_opaque = 0;
   vec_add1 (im->add_del_interface_address_callbacks, cb);
 
+  ip6_table_bind_callback_t cbt;
+  cbt.function = ip6_neighbor_table_bind;
+  cbt.function_opaque = 0;
+  vec_add1 (im->table_bind_callbacks, cbt);
+
   mhash_init (&nm->pending_resolutions_by_address,
 	      /* value size */ sizeof (uword),
 	      /* key size */ sizeof (ip6_address_t));
diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c
index bba65ab4..384ec3e0 100644
--- a/src/vnet/ip/ip_api.c
+++ b/src/vnet/ip/ip_api.c
@@ -699,12 +699,58 @@ vl_api_ip_neighbor_add_del_t_handler (vl_api_ip_neighbor_add_del_t * mp,
   REPLY_MACRO (VL_API_IP_NEIGHBOR_ADD_DEL_REPLY);
 }
 
+void
+ip_table_delete (fib_protocol_t fproto, u32 table_id, u8 is_api)
+{
+  u32 fib_index, mfib_index;
+
+  /*
+   * ignore action on the default table - this is always present
+   * and cannot be added nor deleted from the API
+   */
+  if (0 != table_id)
+    {
+      /*
+       * The API holds only one lock on the table.
+       * i.e. it can be added many times via the API but needs to be
+       * deleted only once.
+       * The FIB index for unicast and multicast is not necessarily the
+       * same, since internal VPP systesm (like LISP and SR) create
+       * their own unicast tables.
+       */
+      fib_index = fib_table_find (fproto, table_id);
+      mfib_index = mfib_table_find (fproto, table_id);
+
+      if (~0 != fib_index)
+	{
+	  fib_table_unlock (fib_index, fproto,
+			    (is_api ? FIB_SOURCE_API : FIB_SOURCE_CLI));
+	}
+      if (~0 != mfib_index)
+	{
+	  mfib_table_unlock (mfib_index, fproto,
+			     (is_api ? MFIB_SOURCE_API : MFIB_SOURCE_CLI));
+	}
+    }
+}
+
 void
 vl_api_ip_table_add_del_t_handler (vl_api_ip_table_add_del_t * mp)
 {
   vl_api_ip_table_add_del_reply_t *rmp;
+  fib_protocol_t fproto = (mp->is_ipv6 ? FIB_PROTOCOL_IP6 : FIB_PROTOCOL_IP4);
+  u32 table_id = ntohl (mp->table_id);
   int rv = 0;
 
+  if (mp->is_add)
+    {
+      ip_table_create (fproto, table_id, 1);
+    }
+  else
+    {
+      ip_table_delete (fproto, table_id, 1);
+    }
+
   REPLY_MACRO (VL_API_IP_TABLE_ADD_DEL_REPLY);
 }
 
@@ -866,18 +912,21 @@ add_del_route_check (fib_protocol_t table_proto,
 		     u32 next_hop_sw_if_index,
 		     dpo_proto_t next_hop_table_proto,
 		     u32 next_hop_table_id,
-		     u8 create_missing_tables,
 		     u8 is_rpf_id, u32 * fib_index, u32 * next_hop_fib_index)
 {
   vnet_main_t *vnm = vnet_get_main ();
 
+  /* Temporaray whilst I do the CSIT dance */
+  u8 create_missing_tables = 1;
+
   *fib_index = fib_table_find (table_proto, ntohl (table_id));
   if (~0 == *fib_index)
     {
       if (create_missing_tables)
 	{
 	  *fib_index = fib_table_find_or_create_and_lock (table_proto,
-							  ntohl (table_id));
+							  ntohl (table_id),
+							  FIB_SOURCE_API);
 	}
       else
 	{
@@ -918,12 +967,14 @@ add_del_route_check (fib_protocol_t table_proto,
 		*next_hop_fib_index =
 		  mfib_table_find_or_create_and_lock (fib_nh_proto,
 						      ntohl
-						      (next_hop_table_id));
+						      (next_hop_table_id),
+						      MFIB_SOURCE_API);
 	      else
 		*next_hop_fib_index =
 		  fib_table_find_or_create_and_lock (fib_nh_proto,
 						     ntohl
-						     (next_hop_table_id));
+						     (next_hop_table_id),
+						     FIB_SOURCE_API);
 	    }
 	  else
 	    {
@@ -948,8 +999,7 @@ ip4_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
 			    mp->next_hop_sw_if_index,
 			    DPO_PROTO_IP4,
 			    mp->next_hop_table_id,
-			    mp->create_vrf_if_needed, 0,
-			    &fib_index, &next_hop_fib_index);
+			    0, &fib_index, &next_hop_fib_index);
 
   if (0 != rv)
     return (rv);
@@ -1008,8 +1058,7 @@ ip6_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
 			    mp->next_hop_sw_if_index,
 			    DPO_PROTO_IP6,
 			    mp->next_hop_table_id,
-			    mp->create_vrf_if_needed, 0,
-			    &fib_index, &next_hop_fib_index);
+			    0, &fib_index, &next_hop_fib_index);
 
   if (0 != rv)
     return (rv);
@@ -1074,27 +1123,57 @@ vl_api_ip_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
   REPLY_MACRO (VL_API_IP_ADD_DEL_ROUTE_REPLY);
 }
 
+void
+ip_table_create (fib_protocol_t fproto, u32 table_id, u8 is_api)
+{
+  u32 fib_index, mfib_index;
+
+  /*
+   * ignore action on the default table - this is always present
+   * and cannot be added nor deleted from the API
+   */
+  if (0 != table_id)
+    {
+      /*
+       * The API holds only one lock on the table.
+       * i.e. it can be added many times via the API but needs to be
+       * deleted only once.
+       * The FIB index for unicast and multicast is not necessarily the
+       * same, since internal VPP systesm (like LISP and SR) create
+       * their own unicast tables.
+       */
+      fib_index = fib_table_find (fproto, table_id);
+      mfib_index = mfib_table_find (fproto, table_id);
+
+      if (~0 == fib_index)
+	{
+	  fib_table_find_or_create_and_lock (fproto, table_id,
+					     (is_api ?
+					      FIB_SOURCE_API :
+					      FIB_SOURCE_CLI));
+	}
+      if (~0 == mfib_index)
+	{
+	  mfib_table_find_or_create_and_lock (fproto, table_id,
+					      (is_api ?
+					       MFIB_SOURCE_API :
+					       MFIB_SOURCE_CLI));
+	}
+    }
+}
+
 static int
 add_del_mroute_check (fib_protocol_t table_proto,
 		      u32 table_id,
-		      u32 next_hop_sw_if_index,
-		      u8 is_local, u8 create_missing_tables, u32 * fib_index)
+		      u32 next_hop_sw_if_index, u8 is_local, u32 * fib_index)
 {
   vnet_main_t *vnm = vnet_get_main ();
 
   *fib_index = mfib_table_find (table_proto, ntohl (table_id));
   if (~0 == *fib_index)
     {
-      if (create_missing_tables)
-	{
-	  *fib_index = mfib_table_find_or_create_and_lock (table_proto,
-							   ntohl (table_id));
-	}
-      else
-	{
-	  /* No such VRF, and we weren't asked to create one */
-	  return VNET_API_ERROR_NO_SUCH_FIB;
-	}
+      /* No such table */
+      return VNET_API_ERROR_NO_SUCH_FIB;
     }
 
   if (~0 != ntohl (next_hop_sw_if_index))
@@ -1163,8 +1242,7 @@ api_mroute_add_del_t_handler (vl_api_ip_mroute_add_del_t * mp)
   rv = add_del_mroute_check (fproto,
 			     mp->table_id,
 			     mp->next_hop_sw_if_index,
-			     mp->is_local,
-			     mp->create_vrf_if_needed, &fib_index);
+			     mp->is_local, &fib_index);
 
   if (0 != rv)
     return (rv);
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 5537bb04..667c6791 100755
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -687,6 +687,78 @@ done:
   return error;
 }
 
+clib_error_t *
+vnet_ip_table_cmd (vlib_main_t * vm,
+		   unformat_input_t * main_input,
+		   vlib_cli_command_t * cmd, fib_protocol_t fproto)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = NULL;
+  u32 table_id, is_add;
+
+  is_add = 1;
+  table_id = ~0;
+
+  /* Get a line of input. */
+  if (!unformat_user (main_input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%d", &table_id))
+	;
+      else if (unformat (line_input, "del"))
+	is_add = 0;
+      else if (unformat (line_input, "add"))
+	is_add = 1;
+      else
+	{
+	  error = unformat_parse_error (line_input);
+	  goto done;
+	}
+    }
+
+  if (~0 == table_id)
+    {
+      error = clib_error_return (0, "No table id");
+      goto done;
+    }
+  else if (0 == table_id)
+    {
+      error = clib_error_return (0, "Can't change the default table");
+      goto done;
+    }
+  else
+    {
+      if (is_add)
+	{
+	  ip_table_create (fproto, table_id, 0);
+	}
+      else
+	{
+	  ip_table_delete (fproto, table_id, 0);
+	}
+    }
+
+done:
+  unformat_free (line_input);
+  return error;
+}
+
+clib_error_t *
+vnet_ip4_table_cmd (vlib_main_t * vm,
+		    unformat_input_t * main_input, vlib_cli_command_t * cmd)
+{
+  return (vnet_ip_table_cmd (vm, main_input, cmd, FIB_PROTOCOL_IP4));
+}
+
+clib_error_t *
+vnet_ip6_table_cmd (vlib_main_t * vm,
+		    unformat_input_t * main_input, vlib_cli_command_t * cmd)
+{
+  return (vnet_ip_table_cmd (vm, main_input, cmd, FIB_PROTOCOL_IP6));
+}
+
 /* *INDENT-OFF* */
 VLIB_CLI_COMMAND (vlib_cli_ip_command, static) = {
   .path = "ip",
@@ -749,6 +821,159 @@ VLIB_CLI_COMMAND (ip_route_command, static) = {
   .function = vnet_ip_route_cmd,
   .is_mp_safe = 1,
 };
+
+/* *INDENT-ON* */
+/*?
+ * This command is used to add or delete IPv4  Tables. All
+ * Tables must be explicitly added before that can be used. Creating a
+ * table will add both unicast and multicast FIBs
+ *
+ ?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip4_table_command, static) = {
+  .path = "ip table",
+  .short_help = "ip table [add|del] <table-id>",
+  .function = vnet_ip4_table_cmd,
+  .is_mp_safe = 1,
+};
+/* *INDENT-ON* */
+
+/* *INDENT-ON* */
+/*?
+ * This command is used to add or delete IPv4  Tables. All
+ * Tables must be explicitly added before that can be used. Creating a
+ * table will add both unicast and multicast FIBs
+ *
+ ?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip6_table_command, static) = {
+  .path = "ip6 table",
+  .short_help = "ip6 table [add|del] <table-id>",
+  .function = vnet_ip6_table_cmd,
+  .is_mp_safe = 1,
+};
+
+static clib_error_t *
+ip_table_bind_cmd (vlib_main_t * vm,
+                   unformat_input_t * input,
+                   vlib_cli_command_t * cmd,
+                   fib_protocol_t fproto)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  clib_error_t *error = 0;
+  u32 sw_if_index, table_id;
+  int rv;
+
+  sw_if_index = ~0;
+
+  if (!unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
+    {
+      error = clib_error_return (0, "unknown interface `%U'",
+				 format_unformat_error, input);
+      goto done;
+    }
+
+  if (unformat (input, "%d", &table_id))
+    ;
+  else
+    {
+      error = clib_error_return (0, "expected table id `%U'",
+				 format_unformat_error, input);
+      goto done;
+    }
+
+  rv = ip_table_bind (fproto, sw_if_index, table_id, 0);
+
+  if (VNET_API_ERROR_ADDRESS_FOUND_FOR_INTERFACE == rv)
+    {
+      error = clib_error_return (0, "IP addresses are still present on %U",
+                                 format_vnet_sw_if_index_name,
+                                 vnet_get_main(),
+                                 sw_if_index);
+    }
+  else if (VNET_API_ERROR_NO_SUCH_FIB == rv)
+    {
+      error = clib_error_return (0, "no such table %d", table_id);
+    }
+  else if (0 != rv)
+    {
+      error = clib_error_return (0, "unknown error");
+    }
+
+ done:
+  return error;
+}
+
+static clib_error_t *
+ip4_table_bind_cmd (vlib_main_t * vm,
+                    unformat_input_t * input,
+                    vlib_cli_command_t * cmd)
+{
+  return (ip_table_bind_cmd (vm , input, cmd, FIB_PROTOCOL_IP4));
+}
+
+static clib_error_t *
+ip6_table_bind_cmd (vlib_main_t * vm,
+                    unformat_input_t * input,
+                    vlib_cli_command_t * cmd)
+{
+  return (ip_table_bind_cmd (vm , input, cmd, FIB_PROTOCOL_IP6));
+}
+
+/*?
+ * Place the indicated interface into the supplied IPv4 FIB table (also known
+ * as a VRF). If the FIB table does not exist, this command creates it. To
+ * display the current IPv4 FIB table, use the command '<em>show ip fib</em>'.
+ * FIB table will only be displayed if a route has been added to the table, or
+ * an IP Address is assigned to an interface in the table (which adds a route
+ * automatically).
+ *
+ * @note IP addresses added after setting the interface IP table are added to
+ * the indicated FIB table. If an IP address is added prior to changing the
+ * table then this is an error. The control plane must remove these addresses
+ * first and then change the table. VPP will not automatically move the
+ * addresses from the old to the new table as it does not know the validity
+ * of such a change.
+ *
+ * @cliexpar
+ * Example of how to add an interface to an IPv4 FIB table (where 2 is the table-id):
+ * @cliexcmd{set interface ip table GigabitEthernet2/0/0 2}
+ ?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (set_interface_ip_table_command, static) =
+{
+  .path = "set interface ip table",
+  .function = ip4_table_bind_cmd,
+  .short_help = "set interface ip table <interface> <table-id>",
+};
+/* *INDENT-ON* */
+
+/*?
+ * Place the indicated interface into the supplied IPv6 FIB table (also known
+ * as a VRF). If the FIB table does not exist, this command creates it. To
+ * display the current IPv6 FIB table, use the command '<em>show ip6 fib</em>'.
+ * FIB table will only be displayed if a route has been added to the table, or
+ * an IP Address is assigned to an interface in the table (which adds a route
+ * automatically).
+ *
+ * @note IP addresses added after setting the interface IP table are added to
+ * the indicated FIB table. If an IP address is added prior to changing the
+ * table then this is an error. The control plane must remove these addresses
+ * first and then change the table. VPP will not automatically move the
+ * addresses from the old to the new table as it does not know the validity
+ * of such a change.
+ *
+ * @cliexpar
+ * Example of how to add an interface to an IPv6 FIB table (where 2 is the table-id):
+ * @cliexcmd{set interface ip6 table GigabitEthernet2/0/0 2}
+ ?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (set_interface_ip6_table_command, static) =
+{
+  .path = "set interface ip6 table",
+  .function = ip6_table_bind_cmd,
+  .short_help = "set interface ip6 table <interface> <table-id>"
+};
 /* *INDENT-ON* */
 
 clib_error_t *
diff --git a/src/vnet/lisp-gpe/interface.c b/src/vnet/lisp-gpe/interface.c
index e832c23f..a0c05e85 100644
--- a/src/vnet/lisp-gpe/interface.c
+++ b/src/vnet/lisp-gpe/interface.c
@@ -505,12 +505,14 @@ lisp_gpe_iface_set_table (u32 sw_if_index, u32 table_id)
 {
   fib_node_index_t fib_index;
 
-  fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, table_id);
+  fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, table_id,
+						 FIB_SOURCE_LISP);
   vec_validate (ip4_main.fib_index_by_sw_if_index, sw_if_index);
   ip4_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
   ip4_sw_interface_enable_disable (sw_if_index, 1);
 
-  fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, table_id);
+  fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, table_id,
+						 FIB_SOURCE_LISP);
   vec_validate (ip6_main.fib_index_by_sw_if_index, sw_if_index);
   ip6_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
   ip6_sw_interface_enable_disable (sw_if_index, 1);
@@ -530,7 +532,7 @@ lisp_gpe_tenant_del_default_routes (u32 table_id)
 
     fib_index = fib_table_find (prefix.fp_proto, table_id);
     fib_table_entry_special_remove (fib_index, &prefix, FIB_SOURCE_LISP);
-    fib_table_unlock (fib_index, prefix.fp_proto);
+    fib_table_unlock (fib_index, prefix.fp_proto, FIB_SOURCE_LISP);
   }
 }
 
@@ -549,7 +551,8 @@ lisp_gpe_tenant_add_default_routes (u32 table_id)
     /*
      * Add a deafult route that results in a control plane punt DPO
      */
-    fib_index = fib_table_find_or_create_and_lock (prefix.fp_proto, table_id);
+    fib_index = fib_table_find_or_create_and_lock (prefix.fp_proto, table_id,
+						   FIB_SOURCE_LISP);
     fib_table_entry_special_dpo_add (fib_index, &prefix, FIB_SOURCE_LISP,
 				     FIB_ENTRY_FLAG_EXCLUSIVE,
 				     lisp_cp_dpo_get (fib_proto_to_dpo
diff --git a/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c b/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c
index d7d3cb86..0a8dc039 100644
--- a/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c
+++ b/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c
@@ -66,6 +66,7 @@ ip_dst_fib_add_route (u32 dst_fib_index, const ip_prefix_t * dst_prefix)
       /* create a new src FIB.  */
       src_fib_index =
 	fib_table_create_and_lock (dst_fib_prefix.fp_proto,
+				   FIB_SOURCE_LISP,
 				   "LISP-src for [%d,%U]",
 				   dst_fib_index,
 				   format_fib_prefix, &dst_fib_prefix);
@@ -180,7 +181,8 @@ ip_src_dst_fib_del_route (u32 src_fib_index,
        */
       fib_table_entry_special_remove (dst_fib_index,
 				      &dst_fib_prefix, FIB_SOURCE_LISP);
-      fib_table_unlock (src_fib_index, src_fib_prefix.fp_proto);
+      fib_table_unlock (src_fib_index, src_fib_prefix.fp_proto,
+			FIB_SOURCE_LISP);
     }
 }
 
@@ -544,7 +546,8 @@ add_ip_fwd_entry (lisp_gpe_main_t * lgm,
   lfe->tenant = lisp_gpe_tenant_find_or_create (lfe->key->vni);
   lfe->eid_table_id = a->table_id;
   lfe->eid_fib_index = fib_table_find_or_create_and_lock (fproto,
-							  lfe->eid_table_id);
+							  lfe->eid_table_id,
+							  FIB_SOURCE_LISP);
   lfe->is_src_dst = a->is_src_dst;
 
   if (LISP_GPE_FWD_ENTRY_TYPE_NEGATIVE != lfe->type)
@@ -578,7 +581,7 @@ del_ip_fwd_entry_i (lisp_gpe_main_t * lgm, lisp_gpe_fwd_entry_t * lfe)
 
   fproto = (IP4 == ip_prefix_version (&fid_addr_ippref (&lfe->key->rmt)) ?
 	    FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6);
-  fib_table_unlock (lfe->eid_fib_index, fproto);
+  fib_table_unlock (lfe->eid_fib_index, fproto, FIB_SOURCE_LISP);
 
   hash_unset_mem (lgm->lisp_gpe_fwd_entries, lfe->key);
   clib_mem_free (lfe->key);
diff --git a/src/vnet/lisp-gpe/lisp_gpe_sub_interface.c b/src/vnet/lisp-gpe/lisp_gpe_sub_interface.c
index b234d9dc..26664f53 100644
--- a/src/vnet/lisp-gpe/lisp_gpe_sub_interface.c
+++ b/src/vnet/lisp-gpe/lisp_gpe_sub_interface.c
@@ -89,13 +89,15 @@ lisp_gpe_sub_interface_set_table (u32 sw_if_index, u32 table_id)
 {
   fib_node_index_t fib_index;
 
-  fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, table_id);
+  fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, table_id,
+						 FIB_SOURCE_LISP);
   ASSERT (FIB_NODE_INDEX_INVALID != fib_index);
 
   vec_validate (ip4_main.fib_index_by_sw_if_index, sw_if_index);
   ip4_main.fib_index_by_sw_if_index[sw_if_index] = fib_index;
 
-  fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, table_id);
+  fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP6, table_id,
+						 FIB_SOURCE_LISP);
   ASSERT (FIB_NODE_INDEX_INVALID != fib_index);
 
   vec_validate (ip6_main.fib_index_by_sw_if_index, sw_if_index);
@@ -105,9 +107,13 @@ lisp_gpe_sub_interface_set_table (u32 sw_if_index, u32 table_id)
 static void
 lisp_gpe_sub_interface_unset_table (u32 sw_if_index, u32 table_id)
 {
+  fib_table_unlock (ip4_main.fib_index_by_sw_if_index[sw_if_index],
+		    FIB_PROTOCOL_IP4, FIB_SOURCE_LISP);
   ip4_main.fib_index_by_sw_if_index[sw_if_index] = 0;
   ip4_sw_interface_enable_disable (sw_if_index, 0);
 
+  fib_table_unlock (ip6_main.fib_index_by_sw_if_index[sw_if_index],
+		    FIB_PROTOCOL_IP6, FIB_SOURCE_LISP);
   ip6_main.fib_index_by_sw_if_index[sw_if_index] = 0;
   ip6_sw_interface_enable_disable (sw_if_index, 0);
 }
@@ -185,6 +191,7 @@ lisp_gpe_sub_interface_unlock (index_t l3si)
 
   l3s = lisp_gpe_sub_interface_get_i (l3si);
 
+  ASSERT (0 != l3s->locks);
   l3s->locks--;
 
   if (0 == l3s->locks)
diff --git a/src/vnet/mfib/ip4_mfib.c b/src/vnet/mfib/ip4_mfib.c
index 1849a3a4..b2482580 100644
--- a/src/vnet/mfib/ip4_mfib.c
+++ b/src/vnet/mfib/ip4_mfib.c
@@ -33,7 +33,8 @@ static const mfib_prefix_t ip4_specials[] = {
 };
 
 static u32
-ip4_create_mfib_with_table_id (u32 table_id)
+ip4_create_mfib_with_table_id (u32 table_id,
+                               mfib_source_t src)
 {
     mfib_table_t *mfib_table;
 
@@ -53,7 +54,7 @@ ip4_create_mfib_with_table_id (u32 table_id)
         mfib_table->v4.table_id =
             table_id;
 
-    mfib_table_lock(mfib_table->mft_index, FIB_PROTOCOL_IP4);
+    mfib_table_lock(mfib_table->mft_index, FIB_PROTOCOL_IP4, src);
 
     /*
      * add the special entries into the new FIB
@@ -113,14 +114,15 @@ ip4_mfib_table_destroy (ip4_mfib_t *mfib)
 }
 
 u32
-ip4_mfib_table_find_or_create_and_lock (u32 table_id)
+ip4_mfib_table_find_or_create_and_lock (u32 table_id,
+                                        mfib_source_t src)
 {
     u32 index;
 
     index = ip4_mfib_index_from_table_id(table_id);
     if (~0 == index)
-        return ip4_create_mfib_with_table_id(table_id);
-    mfib_table_lock(index, FIB_PROTOCOL_IP4);
+        return ip4_create_mfib_with_table_id(table_id, src);
+    mfib_table_lock(index, FIB_PROTOCOL_IP4, src);
 
     return (index);
 }
diff --git a/src/vnet/mfib/ip4_mfib.h b/src/vnet/mfib/ip4_mfib.h
index ea682651..e31fb744 100644
--- a/src/vnet/mfib/ip4_mfib.h
+++ b/src/vnet/mfib/ip4_mfib.h
@@ -72,8 +72,9 @@ ip4_mfib_get (u32 index)
  * @returns A pointer to the retrieved or created fib.
  *
  */
-extern u32 ip4_mfib_table_find_or_create_and_lock(u32 table_id);
-extern u32 ip4_mfib_table_create_and_lock(void);
+extern u32 ip4_mfib_table_find_or_create_and_lock(u32 table_id,
+                                                  mfib_source_t src);
+extern u32 ip4_mfib_table_create_and_lock(mfib_source_t src);
 
 static inline
 u32 ip4_mfib_index_from_table_id (u32 table_id)
diff --git a/src/vnet/mfib/ip6_mfib.c b/src/vnet/mfib/ip6_mfib.c
index 5e48e919..e4861330 100644
--- a/src/vnet/mfib/ip6_mfib.c
+++ b/src/vnet/mfib/ip6_mfib.c
@@ -151,7 +151,8 @@ static const ip6_mfib_special_t ip6_mfib_specials[] =
 
 
 static u32
-ip6_create_mfib_with_table_id (u32 table_id)
+ip6_create_mfib_with_table_id (u32 table_id,
+                               mfib_source_t src)
 {
     mfib_table_t *mfib_table;
     mfib_prefix_t pfx = {
@@ -182,7 +183,7 @@ ip6_create_mfib_with_table_id (u32 table_id)
         mfib_table->v6.table_id =
             table_id;
 
-    mfib_table_lock(mfib_table->mft_index, FIB_PROTOCOL_IP6);
+    mfib_table_lock(mfib_table->mft_index, FIB_PROTOCOL_IP6, src);
 
     mfib_table->v6.rhead =
         clib_mem_alloc_aligned (sizeof(*mfib_table->v6.rhead),
@@ -297,14 +298,15 @@ ip6_mfib_interface_enable_disable (u32 sw_if_index, int is_enable)
 }
 
 u32
-ip6_mfib_table_find_or_create_and_lock (u32 table_id)
+ip6_mfib_table_find_or_create_and_lock (u32 table_id,
+                                        mfib_source_t src)
 {
     u32 index;
 
     index = ip6_mfib_index_from_table_id(table_id);
     if (~0 == index)
-        return ip6_create_mfib_with_table_id(table_id);
-    mfib_table_lock(index, FIB_PROTOCOL_IP6);
+        return ip6_create_mfib_with_table_id(table_id, src);
+    mfib_table_lock(index, FIB_PROTOCOL_IP6, src);
 
     return (index);
 }
diff --git a/src/vnet/mfib/ip6_mfib.h b/src/vnet/mfib/ip6_mfib.h
index adaa7ec2..ea81b553 100644
--- a/src/vnet/mfib/ip6_mfib.h
+++ b/src/vnet/mfib/ip6_mfib.h
@@ -79,8 +79,9 @@ ip6_mfib_get (u32 index)
  * @returns A pointer to the retrieved or created fib.
  *
  */
-extern u32 ip6_mfib_table_find_or_create_and_lock(u32 table_id);
-extern u32 ip6_mfib_table_create_and_lock(void);
+extern u32 ip6_mfib_table_find_or_create_and_lock(u32 table_id,
+                                                  mfib_source_t src);
+extern u32 ip6_mfib_table_create_and_lock(mfib_source_t src);
 
 
 static inline
diff --git a/src/vnet/mfib/mfib_entry.c b/src/vnet/mfib/mfib_entry.c
index 804e10ab..2302b9a1 100644
--- a/src/vnet/mfib/mfib_entry.c
+++ b/src/vnet/mfib/mfib_entry.c
@@ -334,6 +334,17 @@ mfib_entry_get_best_src (const mfib_entry_t *mfib_entry)
     return (bsrc);
 }
 
+int
+mfib_entry_is_sourced (fib_node_index_t mfib_entry_index,
+                       mfib_source_t source)
+{
+    mfib_entry_t *mfib_entry;
+
+    mfib_entry = mfib_entry_get(mfib_entry_index);
+
+    return (NULL != mfib_entry_src_find(mfib_entry, source, NULL));
+}
+
 static void
 mfib_entry_src_flush (mfib_entry_src_t *msrc)
 {
diff --git a/src/vnet/mfib/mfib_entry.h b/src/vnet/mfib/mfib_entry.h
index d4377878..96ee49f7 100644
--- a/src/vnet/mfib/mfib_entry.h
+++ b/src/vnet/mfib/mfib_entry.h
@@ -130,6 +130,8 @@ extern void mfib_entry_unlock(fib_node_index_t fib_entry_index);
 extern void mfib_entry_get_prefix(fib_node_index_t fib_entry_index,
                                   mfib_prefix_t *pfx);
 extern u32 mfib_entry_get_fib_index(fib_node_index_t fib_entry_index);
+extern int mfib_entry_is_sourced(fib_node_index_t fib_entry_index,
+                                 mfib_source_t source);
 
 extern void mfib_entry_contribute_forwarding(
     fib_node_index_t mfib_entry_index,
diff --git a/src/vnet/mfib/mfib_table.c b/src/vnet/mfib/mfib_table.c
index 7ffe8941..e5550adc 100644
--- a/src/vnet/mfib/mfib_table.c
+++ b/src/vnet/mfib/mfib_table.c
@@ -424,7 +424,8 @@ mfib_table_find (fib_protocol_t proto,
 
 u32
 mfib_table_find_or_create_and_lock (fib_protocol_t proto,
-                                    u32 table_id)
+                                    u32 table_id,
+                                    mfib_source_t src)
 {
     mfib_table_t *mfib_table;
     fib_node_index_t fi;
@@ -432,10 +433,10 @@ mfib_table_find_or_create_and_lock (fib_protocol_t proto,
     switch (proto)
     {
     case FIB_PROTOCOL_IP4:
-        fi = ip4_mfib_table_find_or_create_and_lock(table_id);
+        fi = ip4_mfib_table_find_or_create_and_lock(table_id, src);
         break;
     case FIB_PROTOCOL_IP6:
-        fi = ip6_mfib_table_find_or_create_and_lock(table_id);
+        fi = ip6_mfib_table_find_or_create_and_lock(table_id, src);
         break;
     case FIB_PROTOCOL_MPLS:
     default:
@@ -451,6 +452,59 @@ mfib_table_find_or_create_and_lock (fib_protocol_t proto,
     return (fi);
 }
 
+/**
+ * @brief Table flush context. Store the indicies of matching FIB entries
+ * that need to be removed.
+ */
+typedef struct mfib_table_flush_ctx_t_
+{
+    /**
+     * The list of entries to flush
+     */
+    fib_node_index_t *mftf_entries;
+
+    /**
+     * The source we are flushing
+     */
+    mfib_source_t mftf_source;
+} mfib_table_flush_ctx_t;
+
+static int
+mfib_table_flush_cb (fib_node_index_t mfib_entry_index,
+                     void *arg)
+{
+    mfib_table_flush_ctx_t *ctx = arg;
+
+    if (mfib_entry_is_sourced(mfib_entry_index, ctx->mftf_source))
+    {
+        vec_add1(ctx->mftf_entries, mfib_entry_index);
+    }
+    return (1);
+}
+
+void
+mfib_table_flush (u32 mfib_index,
+                  fib_protocol_t proto,
+                  mfib_source_t source)
+{
+    fib_node_index_t *mfib_entry_index;
+    mfib_table_flush_ctx_t ctx = {
+        .mftf_entries = NULL,
+        .mftf_source = source,
+    };
+
+    mfib_table_walk(mfib_index, proto,
+                    mfib_table_flush_cb,
+                    &ctx);
+
+    vec_foreach(mfib_entry_index, ctx.mftf_entries)
+    {
+        mfib_table_entry_delete_index(*mfib_entry_index, source);
+    }
+
+    vec_free(ctx.mftf_entries);
+}
+
 static void
 mfib_table_destroy (mfib_table_t *mfib_table)
 {
@@ -472,27 +526,43 @@ mfib_table_destroy (mfib_table_t *mfib_table)
 
 void
 mfib_table_unlock (u32 fib_index,
-                   fib_protocol_t proto)
+                   fib_protocol_t proto,
+                   mfib_source_t source)
 {
     mfib_table_t *mfib_table;
 
     mfib_table = mfib_table_get(fib_index, proto);
-    mfib_table->mft_locks--;
+    mfib_table->mft_locks[source]--;
+    mfib_table->mft_locks[MFIB_TABLE_TOTAL_LOCKS]--;
+
+    if (0 == mfib_table->mft_locks[source])
+    {
+        /*
+         * The source no longer needs the table. flush any routes
+         * from it just in case
+         */
+        mfib_table_flush(fib_index, proto, source);
+    }
 
-    if (0 == mfib_table->mft_locks)
+    if (0 == mfib_table->mft_locks[MFIB_TABLE_TOTAL_LOCKS])
     {
-        mfib_table_destroy(mfib_table);
+        /*
+         * no more locak from any source - kill it
+         */
+	mfib_table_destroy(mfib_table);
     }
 }
 
 void
 mfib_table_lock (u32 fib_index,
-                 fib_protocol_t proto)
+                 fib_protocol_t proto,
+                 mfib_source_t source)
 {
     mfib_table_t *mfib_table;
 
     mfib_table = mfib_table_get(fib_index, proto);
-    mfib_table->mft_locks++;
+    mfib_table->mft_locks[source]++;
+    mfib_table->mft_locks[MFIB_TABLE_TOTAL_LOCKS]++;
 }
 
 void
diff --git a/src/vnet/mfib/mfib_table.h b/src/vnet/mfib/mfib_table.h
index 83aa04ef..c6b0b097 100644
--- a/src/vnet/mfib/mfib_table.h
+++ b/src/vnet/mfib/mfib_table.h
@@ -22,6 +22,12 @@
 
 #include <vnet/mfib/mfib_types.h>
 
+/**
+ * Keep a lock per-source and a total
+ */
+#define MFIB_TABLE_N_LOCKS (MFIB_N_SOURCES+1)
+#define MFIB_TABLE_TOTAL_LOCKS MFIB_N_SOURCES
+
 /**
  * @brief
  *   A protocol Independent IP multicast FIB table
@@ -47,7 +53,7 @@ typedef struct mfib_table_t_
     /**
      * number of locks on the table
      */
-    u16 mft_locks;
+    u16 mft_locks[MFIB_TABLE_N_LOCKS];
 
     /**
      * Table ID (hash key) for this FIB.
@@ -259,7 +265,8 @@ extern fib_node_index_t mfib_table_entry_special_add(u32 fib_index,
  *  the source to flush
  */
 extern void mfib_table_flush(u32 fib_index,
-                             fib_protocol_t proto);
+                             fib_protocol_t proto,
+                             mfib_source_t source);
 
 /**
  * @brief
@@ -307,9 +314,13 @@ extern u32 mfib_table_find(fib_protocol_t proto, u32 table_id);
  *
  * @return fib_index
  *  The index of the FIB
+ *
+ * @param source
+ *  The ID of the client/source.
  */
 extern u32 mfib_table_find_or_create_and_lock(fib_protocol_t proto,
-                                              u32 table_id);
+                                              u32 table_id,
+                                              mfib_source_t source);
 
 
 /**
@@ -321,9 +332,13 @@ extern u32 mfib_table_find_or_create_and_lock(fib_protocol_t proto,
  *
  * @paran proto
  *  The protocol of the FIB (and thus the entries therein)
+ *
+ * @param source
+ *  The ID of the client/source.
  */
 extern void mfib_table_unlock(u32 fib_index,
-                              fib_protocol_t proto);
+                              fib_protocol_t proto,
+                              mfib_source_t source);
 
 /**
  * @brief
@@ -335,9 +350,13 @@ extern void mfib_table_unlock(u32 fib_index,
  *
  * @paran proto
  *  The protocol of the FIB (and thus the entries therein)
+ *
+ * @param source
+ *  The ID of the client/source.
  */
 extern void mfib_table_lock(u32 fib_index,
-                            fib_protocol_t proto);
+                            fib_protocol_t proto,
+                            mfib_source_t source);
 
 /**
  * @brief
diff --git a/src/vnet/mfib/mfib_test.c b/src/vnet/mfib/mfib_test.c
index 57787eca..3055844d 100644
--- a/src/vnet/mfib/mfib_test.c
+++ b/src/vnet/mfib/mfib_test.c
@@ -22,6 +22,7 @@
 #include <vnet/fib/fib_path_list.h>
 #include <vnet/fib/fib_test.h>
 #include <vnet/fib/fib_table.h>
+#include <vnet/fib/mpls_fib.h>
 
 #include <vnet/dpo/replicate_dpo.h>
 #include <vnet/adj/adj_mcast.h>
@@ -366,7 +367,7 @@ mfib_test_i (fib_protocol_t PROTO,
     MFIB_TEST(3 == adj_mcast_db_size(), "3 MCAST adjs");
 
     /* Find or create FIB table 11 */
-    fib_index = mfib_table_find_or_create_and_lock(PROTO, 11);
+    fib_index = mfib_table_find_or_create_and_lock(PROTO, 11, MFIB_SOURCE_API);
 
     mfib_prefix_t pfx_dft = {
         .fp_len = 0,
@@ -1113,9 +1114,10 @@ mfib_test_i (fib_protocol_t PROTO,
     /*
      * MPLS enable an interface so we get the MPLS table created
      */
+    mpls_table_create(MPLS_FIB_DEFAULT_TABLE_ID, FIB_SOURCE_API);
     mpls_sw_interface_enable_disable(&mpls_main,
                                      tm->hw[0]->sw_if_index,
-                                     1);
+                                     1, 0);
 
     lfei = fib_table_entry_update_one_path(0, // default MPLS Table
                                            &pfx_3500,
@@ -1192,7 +1194,7 @@ mfib_test_i (fib_protocol_t PROTO,
     /*
      * Unlock the table - it's the last lock so should be gone thereafter
      */
-    mfib_table_unlock(fib_index, PROTO);
+    mfib_table_unlock(fib_index, PROTO, MFIB_SOURCE_API);
 
     MFIB_TEST((FIB_NODE_INDEX_INVALID ==
                mfib_table_find(PROTO, fib_index)),
@@ -1207,7 +1209,8 @@ mfib_test_i (fib_protocol_t PROTO,
      */
     mpls_sw_interface_enable_disable(&mpls_main,
                                      tm->hw[0]->sw_if_index,
-                                     0);
+                                     0, 0);
+    mpls_table_delete(MPLS_FIB_DEFAULT_TABLE_ID, FIB_SOURCE_API);
 
     /*
      * test we've leaked no resources
diff --git a/src/vnet/mfib/mfib_types.h b/src/vnet/mfib/mfib_types.h
index 863fad16..50aede04 100644
--- a/src/vnet/mfib/mfib_types.h
+++ b/src/vnet/mfib/mfib_types.h
@@ -166,9 +166,10 @@ typedef enum mfib_source_t_
     MFIB_SOURCE_VXLAN,
     MFIB_SOURCE_DHCP,
     MFIB_SOURCE_SRv6,
-    MFIB_SOURCE_DEFAULT_ROUTE,
     MFIB_SOURCE_GTPU,
     MFIB_SOURCE_VXLAN_GPE,
+    MFIB_SOURCE_RR,
+    MFIB_SOURCE_DEFAULT_ROUTE,
 } mfib_source_t;
 
 #define MFIB_SOURCE_NAMES {                        \
@@ -178,11 +179,14 @@ typedef enum mfib_source_t_
     [MFIB_SOURCE_DHCP] = "DHCP",                   \
     [MFIB_SOURCE_VXLAN] = "VXLAN",                 \
     [MFIB_SOURCE_SRv6] = "SRv6",                   \
-    [MFIB_SOURCE_DEFAULT_ROUTE] = "Default Route", \
     [MFIB_SOURCE_GTPU] = "GTPU",                   \
     [MFIB_SOURCE_VXLAN_GPE] = "VXLAN-GPE",         \
+    [MFIB_SOURCE_RR] = "Recursive-resolution",     \
+    [MFIB_SOURCE_DEFAULT_ROUTE] = "Default Route", \
 }
 
+#define MFIB_N_SOURCES (MFIB_SOURCE_DEFAULT_ROUTE)
+
 /**
  * \brief Compare two prefixes for equality
  */
diff --git a/src/vnet/mpls/interface.c b/src/vnet/mpls/interface.c
index a085aaa2..d7c8e7d3 100644
--- a/src/vnet/mpls/interface.c
+++ b/src/vnet/mpls/interface.c
@@ -35,25 +35,33 @@ mpls_sw_interface_is_enabled (u32 sw_if_index)
     return (mm->mpls_enabled_by_sw_if_index[sw_if_index]);
 }
 
-void
+int
 mpls_sw_interface_enable_disable (mpls_main_t * mm,
                                   u32 sw_if_index,
-                                  u8 is_enable)
+                                  u8 is_enable,
+                                  u8 is_api)
 {
   fib_node_index_t lfib_index;
 
   vec_validate_init_empty (mm->mpls_enabled_by_sw_if_index, sw_if_index, 0);
 
+  lfib_index = fib_table_find(FIB_PROTOCOL_MPLS,
+                              MPLS_FIB_DEFAULT_TABLE_ID);
+
+  if (~0 == lfib_index)
+       return VNET_API_ERROR_NO_SUCH_FIB;
+
   /*
    * enable/disable only on the 1<->0 transition
    */
   if (is_enable)
     {
       if (1 != ++mm->mpls_enabled_by_sw_if_index[sw_if_index])
-        return;
+          return (0);
+
+      fib_table_lock(lfib_index, FIB_PROTOCOL_MPLS,
+                     (is_api? FIB_SOURCE_API: FIB_SOURCE_CLI));
 
-      lfib_index = fib_table_find_or_create_and_lock(FIB_PROTOCOL_MPLS,
-						     MPLS_FIB_DEFAULT_TABLE_ID);
       vec_validate(mm->fib_index_by_sw_if_index, 0);
       mm->fib_index_by_sw_if_index[sw_if_index] = lfib_index;
     }
@@ -61,15 +69,17 @@ mpls_sw_interface_enable_disable (mpls_main_t * mm,
     {
       ASSERT(mm->mpls_enabled_by_sw_if_index[sw_if_index] > 0);
       if (0 != --mm->mpls_enabled_by_sw_if_index[sw_if_index])
-        return;
+          return (0);
 
       fib_table_unlock(mm->fib_index_by_sw_if_index[sw_if_index],
-		       FIB_PROTOCOL_MPLS);
+		       FIB_PROTOCOL_MPLS,
+                       (is_api? FIB_SOURCE_API: FIB_SOURCE_CLI));
     }
 
   vnet_feature_enable_disable ("mpls-input", "mpls-not-enabled",
                                sw_if_index, !is_enable, 0, 0);
 
+  return (0);
 }
 
 static clib_error_t *
@@ -101,7 +111,7 @@ mpls_interface_enable_disable (vlib_main_t * vm,
       goto done;
     }
 
-  mpls_sw_interface_enable_disable(&mpls_main, sw_if_index, enable);
+  mpls_sw_interface_enable_disable(&mpls_main, sw_if_index, enable, 0);
 
  done:
   return error;
diff --git a/src/vnet/mpls/mpls.c b/src/vnet/mpls/mpls.c
index 5021ac23..7bdfd8c7 100644
--- a/src/vnet/mpls/mpls.c
+++ b/src/vnet/mpls/mpls.c
@@ -536,6 +536,78 @@ VLIB_CLI_COMMAND (mpls_local_label_command, static) = {
   .short_help = "Create/Delete MPL local labels",
 };
 
+clib_error_t *
+vnet_mpls_table_cmd (vlib_main_t * vm,
+                     unformat_input_t * main_input,
+                     vlib_cli_command_t * cmdo)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = NULL;
+  u32 table_id, is_add;
+
+  is_add = 1;
+  table_id = ~0;
+
+  /* Get a line of input. */
+  if (!unformat_user (main_input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%d", &table_id))
+	;
+      else if (unformat (line_input, "del"))
+	is_add = 0;
+      else if (unformat (line_input, "add"))
+	is_add = 1;
+      else
+	{
+	  error = unformat_parse_error (line_input);
+	  goto done;
+	}
+    }
+
+  if (~0 == table_id)
+    {
+      error = clib_error_return (0, "No table id");
+      goto done;
+    }
+  else if (0 == table_id)
+    {
+      error = clib_error_return (0, "Can't change the default table");
+      goto done;
+    }
+  else
+    {
+      if (is_add)
+        {
+          mpls_table_create (table_id, 0);
+        }
+      else
+        {
+          mpls_table_delete (table_id, 0);
+        }
+    }
+
+ done:
+  unformat_free (line_input);
+  return error;
+}
+
+/* *INDENT-ON* */
+/*?
+ * This command is used to add or delete MPLS Tables. All
+ * Tables must be explicitly added before that can be used,
+ * Including the default table.
+ ?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (ip6_table_command, static) = {
+  .path = "mpla table",
+  .short_help = "mpls table [add|del] <table-id>",
+  .function = vnet_mpls_table_cmd,
+  .is_mp_safe = 1,
+};
+
 int
 mpls_fib_reset_labels (u32 fib_id)
 {
@@ -546,12 +618,8 @@ mpls_fib_reset_labels (u32 fib_id)
 static clib_error_t *
 mpls_init (vlib_main_t * vm)
 {
-  mpls_main_t * mm = &mpls_main;
   clib_error_t * error;
 
-  mm->vlib_main = vm;
-  mm->vnet_main = vnet_get_main();
-
   if ((error = vlib_call_init_function (vm, ip_main_init)))
     return error;
 
diff --git a/src/vnet/mpls/mpls.h b/src/vnet/mpls/mpls.h
index b0125e60..31cb1746 100644
--- a/src/vnet/mpls/mpls.h
+++ b/src/vnet/mpls/mpls.h
@@ -56,10 +56,6 @@ typedef struct {
 
   /* IP4 enabled count by software interface */
   u8 * mpls_enabled_by_sw_if_index;
-
-  /* convenience */
-  vlib_main_t * vlib_main;
-  vnet_main_t * vnet_main;
 } mpls_main_t;
 
 extern mpls_main_t mpls_main;
@@ -77,8 +73,6 @@ extern vlib_node_registration_t mpls_midchain_node;
 
 /* Parse mpls protocol as 0xXXXX or protocol name.
    In either host or network byte order. */
-unformat_function_t unformat_mpls_protocol_host_byte_order;
-unformat_function_t unformat_mpls_protocol_net_byte_order;
 unformat_function_t unformat_mpls_label_net_byte_order;
 unformat_function_t unformat_mpls_unicast_label;
 
@@ -86,9 +80,10 @@ unformat_function_t unformat_mpls_unicast_label;
 unformat_function_t unformat_mpls_header;
 unformat_function_t unformat_pg_mpls_header;
 
-void mpls_sw_interface_enable_disable (mpls_main_t * mm,
-				       u32 sw_if_index,
-				       u8 is_enable);
+int mpls_sw_interface_enable_disable (mpls_main_t * mm,
+                                      u32 sw_if_index,
+                                      u8 is_enable,
+                                      u8 is_api);
 
 u8 mpls_sw_interface_is_enabled (u32 sw_if_index);
 
@@ -103,4 +98,7 @@ mpls_fib_index_cmp(void * a1, void * a2);
 int
 mpls_label_cmp(void * a1, void * a2);
 
+void mpls_table_create(uint32_t table_id, u8 is_api);
+void mpls_table_delete(uint32_t table_id, u8 is_api);
+
 #endif /* included_vnet_mpls_h */
diff --git a/src/vnet/mpls/mpls_api.c b/src/vnet/mpls/mpls_api.c
index a44b1a25..38f5b014 100644
--- a/src/vnet/mpls/mpls_api.c
+++ b/src/vnet/mpls/mpls_api.c
@@ -58,6 +58,29 @@ _(MPLS_FIB_DUMP, mpls_fib_dump)
 extern void stats_dslock_with_hint (int hint, int tag);
 extern void stats_dsunlock (void);
 
+void
+mpls_table_delete (u32 table_id, u8 is_api)
+{
+  u32 fib_index;
+
+  /*
+   * The MPLS defult table must also be explicitly created via the API.
+   * So in contrast to IP, it gets no special treatment here.
+   *
+   * The API holds only one lock on the table.
+   * i.e. it can be added many times via the API but needs to be
+   * deleted only once.
+   */
+  fib_index = fib_table_find (FIB_PROTOCOL_MPLS, table_id);
+
+  if (~0 != fib_index)
+    {
+      fib_table_unlock (fib_index,
+			FIB_PROTOCOL_MPLS,
+			(is_api ? FIB_SOURCE_API : FIB_SOURCE_CLI));
+    }
+}
+
 void
 vl_api_mpls_table_add_del_t_handler (vl_api_mpls_table_add_del_t * mp)
 {
@@ -68,6 +91,13 @@ vl_api_mpls_table_add_del_t_handler (vl_api_mpls_table_add_del_t * mp)
   vnm = vnet_get_main ();
   vnm->api_errno = 0;
 
+  if (mp->mt_is_add)
+    mpls_table_create (ntohl (mp->mt_table_id), 1);
+  else
+    mpls_table_delete (ntohl (mp->mt_table_id), 1);
+
+  rv = (rv == 0) ? vnm->api_errno : rv;
+
   REPLY_MACRO (VL_API_MPLS_TABLE_ADD_DEL_REPLY);
 }
 
@@ -82,14 +112,7 @@ mpls_ip_bind_unbind_handler (vnet_main_t * vnm,
 
   if (~0 == mpls_fib_index)
     {
-      if (mp->mb_create_table_if_needed)
-	{
-	  mpls_fib_index =
-	    fib_table_find_or_create_and_lock (FIB_PROTOCOL_MPLS,
-					       ntohl (mp->mb_mpls_table_id));
-	}
-      else
-	return VNET_API_ERROR_NO_SUCH_FIB;
+      return VNET_API_ERROR_NO_SUCH_FIB;
     }
 
   ip_fib_index = fib_table_find ((mp->mb_is_ip4 ?
@@ -170,7 +193,6 @@ mpls_route_add_del_t_handler (vnet_main_t * vnm,
 			    mp->mr_next_hop_sw_if_index,
 			    pfx.fp_payload_proto,
 			    mp->mr_next_hop_table_id,
-			    mp->mr_create_table_if_needed,
 			    mp->mr_is_rpf_id,
 			    &fib_index, &next_hop_fib_index);
 
@@ -235,6 +257,32 @@ vl_api_mpls_route_add_del_t_handler (vl_api_mpls_route_add_del_t * mp)
   REPLY_MACRO (VL_API_MPLS_ROUTE_ADD_DEL_REPLY);
 }
 
+void
+mpls_table_create (u32 table_id, u8 is_api)
+{
+  u32 fib_index;
+
+  /*
+   * The MPLS defult table must also be explicitly created via the API.
+   * So in contrast to IP, it gets no special treatment here.
+   */
+
+  /*
+   * The API holds only one lock on the table.
+   * i.e. it can be added many times via the API but needs to be
+   * deleted only once.
+   */
+  fib_index = fib_table_find (FIB_PROTOCOL_MPLS, table_id);
+
+  if (~0 == fib_index)
+    {
+      fib_table_find_or_create_and_lock (FIB_PROTOCOL_MPLS,
+					 table_id,
+					 (is_api ?
+					  FIB_SOURCE_API : FIB_SOURCE_CLI));
+    }
+}
+
 static void
 vl_api_mpls_tunnel_add_del_t_handler (vl_api_mpls_tunnel_add_del_t * mp)
 {
diff --git a/src/vnet/srv6/sr_policy_rewrite.c b/src/vnet/srv6/sr_policy_rewrite.c
index f427bbf3..2f90993a 100755
--- a/src/vnet/srv6/sr_policy_rewrite.c
+++ b/src/vnet/srv6/sr_policy_rewrite.c
@@ -595,8 +595,10 @@ sr_policy_add (ip6_address_t * bsid, ip6_address_t * segments,
   if (sm->fib_table_ip6 == (u32) ~ 0)
     {
       sm->fib_table_ip6 = fib_table_create_and_lock (FIB_PROTOCOL_IP6,
+						     FIB_SOURCE_SR,
 						     "SRv6 steering of IP6 prefixes through BSIDs");
       sm->fib_table_ip4 = fib_table_create_and_lock (FIB_PROTOCOL_IP6,
+						     FIB_SOURCE_SR,
 						     "SRv6 steering of IP4 prefixes through BSIDs");
     }
 
@@ -684,8 +686,8 @@ sr_policy_del (ip6_address_t * bsid, u32 index)
   /* If FIB empty unlock it */
   if (!pool_elts (sm->sr_policies) && !pool_elts (sm->steer_policies))
     {
-      fib_table_unlock (sm->fib_table_ip6, FIB_PROTOCOL_IP6);
-      fib_table_unlock (sm->fib_table_ip4, FIB_PROTOCOL_IP6);
+      fib_table_unlock (sm->fib_table_ip6, FIB_PROTOCOL_IP6, FIB_SOURCE_SR);
+      fib_table_unlock (sm->fib_table_ip4, FIB_PROTOCOL_IP6, FIB_SOURCE_SR);
       sm->fib_table_ip6 = (u32) ~ 0;
       sm->fib_table_ip4 = (u32) ~ 0;
     }
diff --git a/src/vnet/srv6/sr_steering.c b/src/vnet/srv6/sr_steering.c
index 57fe21f6..cf4e81ab 100755
--- a/src/vnet/srv6/sr_steering.c
+++ b/src/vnet/srv6/sr_steering.c
@@ -159,8 +159,10 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index,
 	  /* If no more SR policies or steering policies */
 	  if (!pool_elts (sm->sr_policies) && !pool_elts (sm->steer_policies))
 	    {
-	      fib_table_unlock (sm->fib_table_ip6, FIB_PROTOCOL_IP6);
-	      fib_table_unlock (sm->fib_table_ip4, FIB_PROTOCOL_IP6);
+	      fib_table_unlock (sm->fib_table_ip6,
+				FIB_PROTOCOL_IP6, FIB_SOURCE_SR);
+	      fib_table_unlock (sm->fib_table_ip4,
+				FIB_PROTOCOL_IP6, FIB_SOURCE_SR);
 	      sm->fib_table_ip6 = (u32) ~ 0;
 	      sm->fib_table_ip4 = (u32) ~ 0;
 	    }
diff --git a/src/vpp/api/api.c b/src/vpp/api/api.c
index f9c3129c..044ddb5b 100644
--- a/src/vpp/api/api.c
+++ b/src/vpp/api/api.c
@@ -699,8 +699,9 @@ static void
 
   VALIDATE_SW_IF_INDEX (mp);
 
-  mpls_sw_interface_enable_disable (&mpls_main,
-				    ntohl (mp->sw_if_index), mp->enable);
+  rv = mpls_sw_interface_enable_disable (&mpls_main,
+					 ntohl (mp->sw_if_index),
+					 mp->enable, 1);
 
   BAD_SW_IF_INDEX_LABEL;
   REPLY_MACRO (VL_API_SW_INTERFACE_SET_MPLS_ENABLE_REPLY);
diff --git a/src/vpp/api/custom_dump.c b/src/vpp/api/custom_dump.c
index 1353fe28..be74b83a 100644
--- a/src/vpp/api/custom_dump.c
+++ b/src/vpp/api/custom_dump.c
@@ -560,9 +560,6 @@ static void *vl_api_ip_add_del_route_t_print
   if (mp->table_id != 0)
     s = format (s, "vrf %d ", ntohl (mp->table_id));
 
-  if (mp->create_vrf_if_needed)
-    s = format (s, "create-vrf ");
-
   if (mp->next_hop_weight != 1)
     s = format (s, "weight %d ", mp->next_hop_weight);
 
diff --git a/test/test_dhcp.py b/test/test_dhcp.py
index 6fc29182..fe97f6c9 100644
--- a/test/test_dhcp.py
+++ b/test/test_dhcp.py
@@ -6,7 +6,7 @@ import struct
 
 from framework import VppTestCase, VppTestRunner
 from vpp_neighbor import VppNeighbor
-from vpp_ip_route import find_route
+from vpp_ip_route import find_route, VppIpTable
 from util import mk_ll_addr
 
 from scapy.layers.l2 import Ether, getmacbyip, ARP
@@ -34,9 +34,19 @@ class TestDHCP(VppTestCase):
 
         # create 3 pg interfaces
         self.create_pg_interfaces(range(4))
+        self.tables = []
 
         # pg0 and 1 are IP configured in VRF 0 and 1.
         # pg2 and 3 are non IP-configured in VRF 0 and 1
+        table_id = 0
+        for table_id in range(1, 4):
+            tbl4 = VppIpTable(self, table_id)
+            tbl4.add_vpp_config()
+            self.tables.append(tbl4)
+            tbl6 = VppIpTable(self, table_id, is_ip6=1)
+            tbl6.add_vpp_config()
+            self.tables.append(tbl6)
+
         table_id = 0
         for i in self.pg_interfaces[:2]:
             i.admin_up()
@@ -56,11 +66,15 @@ class TestDHCP(VppTestCase):
             table_id += 1
 
     def tearDown(self):
-        super(TestDHCP, self).tearDown()
-        for i in self.pg_interfaces:
+        for i in self.pg_interfaces[:2]:
             i.unconfig_ip4()
             i.unconfig_ip6()
+
+        for i in self.pg_interfaces:
+            i.set_table_ip4(0)
+            i.set_table_ip6(0)
             i.admin_down()
+        super(TestDHCP, self).tearDown()
 
     def send_and_assert_no_replies(self, intf, pkts, remark):
         intf.add_stream(pkts)
@@ -667,6 +681,8 @@ class TestDHCP(VppTestCase):
                                         "DHCP cleanup VRF 0")
         self.send_and_assert_no_replies(self.pg3, pkts_disc_vrf1,
                                         "DHCP cleanup VRF 1")
+        self.pg2.unconfig_ip4()
+        self.pg3.unconfig_ip4()
 
     def test_dhcp6_proxy(self):
         """ DHCPv6 Proxy"""
@@ -1045,6 +1061,8 @@ class TestDHCP(VppTestCase):
                                     server_table_id=0,
                                     is_ipv6=1,
                                     is_add=0)
+        self.pg2.unconfig_ip6()
+        self.pg3.unconfig_ip6()
 
     def test_dhcp_client(self):
         """ DHCP Client"""
diff --git a/test/test_gre.py b/test/test_gre.py
index 1afc44fb..9046b05f 100644
--- a/test/test_gre.py
+++ b/test/test_gre.py
@@ -6,7 +6,7 @@ from logging import *
 from framework import VppTestCase, VppTestRunner
 from vpp_sub_interface import VppDot1QSubint
 from vpp_gre_interface import VppGreInterface, VppGre6Interface
-from vpp_ip_route import VppIpRoute, VppRoutePath, DpoProto
+from vpp_ip_route import VppIpRoute, VppRoutePath, DpoProto, VppIpTable
 from vpp_papi_provider import L2_VTR_OP
 
 from scapy.packet import Raw
@@ -30,6 +30,9 @@ class TestGRE(VppTestCase):
 
         # create 3 pg interfaces - set one in a non-default table.
         self.create_pg_interfaces(range(3))
+
+        self.tbl = VppIpTable(self, 1)
+        self.tbl.add_vpp_config()
         self.pg1.set_table_ip4(1)
 
         for i in self.pg_interfaces:
@@ -43,11 +46,12 @@ class TestGRE(VppTestCase):
         self.pg2.resolve_ndp()
 
     def tearDown(self):
-        super(TestGRE, self).tearDown()
         for i in self.pg_interfaces:
             i.unconfig_ip4()
             i.unconfig_ip6()
             i.admin_down()
+        self.pg1.set_table_ip4(0)
+        super(TestGRE, self).tearDown()
 
     def create_stream_ip4(self, src_if, src_ip, dst_ip):
         pkts = []
diff --git a/test/test_ip4.py b/test/test_ip4.py
index 7a7098c3..55d16735 100644
--- a/test/test_ip4.py
+++ b/test/test_ip4.py
@@ -6,7 +6,8 @@ import unittest
 from framework import VppTestCase, VppTestRunner
 from vpp_sub_interface import VppSubInterface, VppDot1QSubint, VppDot1ADSubint
 from vpp_ip_route import VppIpRoute, VppRoutePath, VppIpMRoute, \
-    VppMRoutePath, MRouteItfFlags, MRouteEntryFlags, VppMplsIpBind
+    VppMRoutePath, MRouteItfFlags, MRouteEntryFlags, VppMplsIpBind, \
+    VppMplsTable
 
 from scapy.packet import Raw
 from scapy.layers.l2 import Ether, Dot1Q, ARP
@@ -774,6 +775,8 @@ class TestIPLoadBalance(VppTestCase):
         super(TestIPLoadBalance, self).setUp()
 
         self.create_pg_interfaces(range(5))
+        mpls_tbl = VppMplsTable(self, 0)
+        mpls_tbl.add_vpp_config()
 
         for i in self.pg_interfaces:
             i.admin_up()
@@ -782,11 +785,11 @@ class TestIPLoadBalance(VppTestCase):
             i.enable_mpls()
 
     def tearDown(self):
-        super(TestIPLoadBalance, self).tearDown()
         for i in self.pg_interfaces:
             i.disable_mpls()
             i.unconfig_ip4()
             i.admin_down()
+        super(TestIPLoadBalance, self).tearDown()
 
     def send_and_expect_load_balancing(self, input, pkts, outputs):
         input.add_stream(pkts)
@@ -966,6 +969,8 @@ class TestIPVlan0(VppTestCase):
         super(TestIPVlan0, self).setUp()
 
         self.create_pg_interfaces(range(2))
+        mpls_tbl = VppMplsTable(self, 0)
+        mpls_tbl.add_vpp_config()
 
         for i in self.pg_interfaces:
             i.admin_up()
@@ -974,11 +979,11 @@ class TestIPVlan0(VppTestCase):
             i.enable_mpls()
 
     def tearDown(self):
-        super(TestIPVlan0, self).tearDown()
         for i in self.pg_interfaces:
             i.disable_mpls()
             i.unconfig_ip4()
             i.admin_down()
+        super(TestIPVlan0, self).tearDown()
 
     def send_and_expect(self, input, pkts, output):
         input.add_stream(pkts)
diff --git a/test/test_ip4_vrf_multi_instance.py b/test/test_ip4_vrf_multi_instance.py
index b73ac948..5a8d6760 100644
--- a/test/test_ip4_vrf_multi_instance.py
+++ b/test/test_ip4_vrf_multi_instance.py
@@ -172,9 +172,10 @@ class TestIp4VrfMultiInst(VppTestCase):
             pg_if = self.pg_if_by_vrf_id[vrf_id][0]
             dest_addr = pg_if.remote_hosts[0].ip4n
             dest_addr_len = 24
+            self.vapi.ip_table_add_del(vrf_id, is_add=1)
             self.vapi.ip_add_del_route(
                 dest_addr, dest_addr_len, pg_if.local_ip4n,
-                table_id=vrf_id, create_vrf_if_needed=1, is_multipath=1)
+                table_id=vrf_id, is_multipath=1)
             self.logger.info("IPv4 VRF ID %d created" % vrf_id)
             if vrf_id not in self.vrf_list:
                 self.vrf_list.append(vrf_id)
@@ -216,6 +217,7 @@ class TestIp4VrfMultiInst(VppTestCase):
         self.logger.info("IPv4 VRF ID %d reset" % vrf_id)
         self.logger.debug(self.vapi.ppcli("show ip fib"))
         self.logger.debug(self.vapi.ppcli("show ip arp"))
+        self.vapi.ip_table_add_del(vrf_id, is_add=0)
 
     def create_stream(self, src_if, packet_sizes):
         """
diff --git a/test/test_ip6.py b/test/test_ip6.py
index 285ce181..aad3713c 100644
--- a/test/test_ip6.py
+++ b/test/test_ip6.py
@@ -8,7 +8,7 @@ from vpp_sub_interface import VppSubInterface, VppDot1QSubint
 from vpp_pg_interface import is_ipv6_misc
 from vpp_ip_route import VppIpRoute, VppRoutePath, find_route, VppIpMRoute, \
     VppMRoutePath, MRouteItfFlags, MRouteEntryFlags, VppMplsIpBind, \
-    VppMplsRoute, DpoProto
+    VppMplsRoute, DpoProto, VppMplsTable
 from vpp_neighbor import find_nbr, VppNeighbor
 
 from scapy.packet import Raw
@@ -1260,6 +1260,9 @@ class TestIP6LoadBalance(VppTestCase):
 
         self.create_pg_interfaces(range(5))
 
+        mpls_tbl = VppMplsTable(self, 0)
+        mpls_tbl.add_vpp_config()
+
         for i in self.pg_interfaces:
             i.admin_up()
             i.config_ip6()
@@ -1267,11 +1270,11 @@ class TestIP6LoadBalance(VppTestCase):
             i.enable_mpls()
 
     def tearDown(self):
-        super(TestIP6LoadBalance, self).tearDown()
         for i in self.pg_interfaces:
             i.unconfig_ip6()
             i.admin_down()
             i.disable_mpls()
+        super(TestIP6LoadBalance, self).tearDown()
 
     def send_and_expect_load_balancing(self, input, pkts, outputs):
         input.add_stream(pkts)
diff --git a/test/test_ip6_vrf_multi_instance.py b/test/test_ip6_vrf_multi_instance.py
index af80b5ba..769cb2e5 100644
--- a/test/test_ip6_vrf_multi_instance.py
+++ b/test/test_ip6_vrf_multi_instance.py
@@ -187,9 +187,10 @@ class TestIP6VrfMultiInst(VppTestCase):
             pg_if = self.pg_if_by_vrf_id[vrf_id][0]
             dest_addr = pg_if.remote_hosts[0].ip6n
             dest_addr_len = 64
+            self.vapi.ip_table_add_del(vrf_id, is_add=1, is_ipv6=1)
             self.vapi.ip_add_del_route(
                 dest_addr, dest_addr_len, pg_if.local_ip6n, is_ipv6=1,
-                table_id=vrf_id, create_vrf_if_needed=1, is_multipath=1)
+                table_id=vrf_id, is_multipath=1)
             self.logger.info("IPv6 VRF ID %d created" % vrf_id)
             if vrf_id not in self.vrf_list:
                 self.vrf_list.append(vrf_id)
@@ -232,6 +233,7 @@ class TestIP6VrfMultiInst(VppTestCase):
         self.logger.info("IPv6 VRF ID %d reset" % vrf_id)
         self.logger.debug(self.vapi.ppcli("show ip6 fib"))
         self.logger.debug(self.vapi.ppcli("show ip6 neighbors"))
+        self.vapi.ip_table_add_del(vrf_id, is_add=0, is_ipv6=1)
 
     def create_stream(self, src_if, packet_sizes):
         """
diff --git a/test/test_ip_mcast.py b/test/test_ip_mcast.py
index 276555d6..7cad683c 100644
--- a/test/test_ip_mcast.py
+++ b/test/test_ip_mcast.py
@@ -5,7 +5,7 @@ import unittest
 from framework import VppTestCase, VppTestRunner
 from vpp_sub_interface import VppSubInterface, VppDot1QSubint, VppDot1ADSubint
 from vpp_ip_route import VppIpMRoute, VppMRoutePath, VppMFibSignal, \
-    MRouteItfFlags, MRouteEntryFlags
+    MRouteItfFlags, MRouteEntryFlags, VppIpTable
 
 from scapy.packet import Raw
 from scapy.layers.l2 import Ether
@@ -44,16 +44,37 @@ class TestIPMcast(VppTestCase):
         super(TestIPMcast, self).setUp()
 
         # create 8 pg interfaces
-        self.create_pg_interfaces(range(8))
+        self.create_pg_interfaces(range(9))
 
         # setup interfaces
-        for i in self.pg_interfaces:
+        for i in self.pg_interfaces[:8]:
             i.admin_up()
             i.config_ip4()
             i.config_ip6()
             i.resolve_arp()
             i.resolve_ndp()
 
+        # one more in a vrf
+        tbl4 = VppIpTable(self, 10)
+        tbl4.add_vpp_config()
+        self.pg8.set_table_ip4(10)
+        self.pg8.config_ip4()
+
+        tbl6 = VppIpTable(self, 10, is_ip6=1)
+        tbl6.add_vpp_config()
+        self.pg8.set_table_ip6(10)
+        self.pg8.config_ip6()
+
+    def tearDown(self):
+        for i in self.pg_interfaces:
+            i.unconfig_ip4()
+            i.unconfig_ip6()
+            i.admin_down()
+
+        self.pg8.set_table_ip4(0)
+        self.pg8.set_table_ip6(0)
+        super(TestIPMcast, self).tearDown()
+
     def create_stream_ip4(self, src_if, src_ip, dst_ip, payload_size=0):
         pkts = []
         # default to small packet sizes
@@ -663,6 +684,77 @@ class TestIPMcast(VppTestCase):
         #
         route_232_1_1_1.remove_vpp_config()
 
+    def test_ip_mcast_vrf(self):
+        """ IP Multicast Replication in non-default table"""
+
+        #
+        # An (S,G).
+        # one accepting interface, pg0, 2 forwarding interfaces
+        #
+        route_1_1_1_1_232_1_1_1 = VppIpMRoute(
+            self,
+            "1.1.1.1",
+            "232.1.1.1", 64,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            [VppMRoutePath(self.pg8.sw_if_index,
+                           MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT),
+             VppMRoutePath(self.pg1.sw_if_index,
+                           MRouteItfFlags.MFIB_ITF_FLAG_FORWARD),
+             VppMRoutePath(self.pg2.sw_if_index,
+                           MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)],
+            table_id=10)
+        route_1_1_1_1_232_1_1_1.add_vpp_config()
+
+        #
+        # a stream that matches the route for (1.1.1.1,232.1.1.1)
+        #  small packets
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip4(self.pg8, "1.1.1.1", "232.1.1.1")
+        self.pg8.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # We expect replications on Pg1 & 2
+        self.verify_capture_ip4(self.pg1, tx)
+        self.verify_capture_ip4(self.pg2, tx)
+
+    def test_ip6_mcast_vrf(self):
+        """ IPv6 Multicast Replication in non-default table"""
+
+        #
+        # An (S,G).
+        # one accepting interface, pg0, 2 forwarding interfaces
+        #
+        route_2001_ff01_1 = VppIpMRoute(
+            self,
+            "2001::1",
+            "ff01::1", 256,
+            MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE,
+            [VppMRoutePath(self.pg8.sw_if_index,
+                           MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT),
+             VppMRoutePath(self.pg1.sw_if_index,
+                           MRouteItfFlags.MFIB_ITF_FLAG_FORWARD),
+             VppMRoutePath(self.pg2.sw_if_index,
+                           MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)],
+            table_id=10,
+            is_ip6=1)
+        route_2001_ff01_1.add_vpp_config()
+
+        #
+        # a stream that matches the route for (2001::1, ff00::1)
+        #
+        self.vapi.cli("clear trace")
+        tx = self.create_stream_ip6(self.pg8, "2001::1", "ff01::1")
+        self.pg8.add_stream(tx)
+
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+
+        # We expect replications on Pg1, 2,
+        self.verify_capture_ip6(self.pg1, tx)
+        self.verify_capture_ip6(self.pg2, tx)
 
 if __name__ == '__main__':
     unittest.main(testRunner=VppTestRunner)
diff --git a/test/test_mpls.py b/test/test_mpls.py
index b2226a74..460a32d1 100644
--- a/test/test_mpls.py
+++ b/test/test_mpls.py
@@ -6,7 +6,7 @@ import socket
 from framework import VppTestCase, VppTestRunner
 from vpp_ip_route import VppIpRoute, VppRoutePath, VppMplsRoute, \
     VppMplsIpBind, VppIpMRoute, VppMRoutePath, \
-    MRouteItfFlags, MRouteEntryFlags, DpoProto
+    MRouteItfFlags, MRouteEntryFlags, DpoProto, VppIpTable, VppMplsTable
 from vpp_mpls_tunnel_interface import VppMPLSTunnelInterface
 
 from scapy.packet import Raw
@@ -60,9 +60,23 @@ class TestMPLS(VppTestCase):
         # setup both interfaces
         # assign them different tables.
         table_id = 0
+        self.tables = []
+
+        tbl = VppMplsTable(self, 0)
+        tbl.add_vpp_config()
+        self.tables.append(tbl)
 
         for i in self.pg_interfaces:
             i.admin_up()
+
+            if table_id != 0:
+                tbl = VppIpTable(self, table_id)
+                tbl.add_vpp_config()
+                self.tables.append(tbl)
+                tbl = VppIpTable(self, table_id, is_ip6=1)
+                tbl.add_vpp_config()
+                self.tables.append(tbl)
+
             i.set_table_ip4(table_id)
             i.set_table_ip6(table_id)
             i.config_ip4()
@@ -73,12 +87,15 @@ class TestMPLS(VppTestCase):
             table_id += 1
 
     def tearDown(self):
-        super(TestMPLS, self).tearDown()
         for i in self.pg_interfaces:
             i.unconfig_ip4()
             i.unconfig_ip6()
             i.ip6_disable()
+            i.set_table_ip4(0)
+            i.set_table_ip6(0)
+            i.disable_mpls()
             i.admin_down()
+        super(TestMPLS, self).tearDown()
 
     # the default of 64 matches the IP packet TTL default
     def create_stream_labelled_ip4(
@@ -1092,6 +1109,9 @@ class TestMPLSDisabled(VppTestCase):
         # create 2 pg interfaces
         self.create_pg_interfaces(range(2))
 
+        self.tbl = VppMplsTable(self, 0)
+        self.tbl.add_vpp_config()
+
         # PG0 is MPLS enalbed
         self.pg0.admin_up()
         self.pg0.config_ip4()
@@ -1102,11 +1122,13 @@ class TestMPLSDisabled(VppTestCase):
         self.pg1.admin_up()
 
     def tearDown(self):
-        super(TestMPLSDisabled, self).tearDown()
         for i in self.pg_interfaces:
             i.unconfig_ip4()
             i.admin_down()
 
+        self.pg0.disable_mpls()
+        super(TestMPLSDisabled, self).tearDown()
+
     def send_and_assert_no_replies(self, intf, pkts, remark):
         intf.add_stream(pkts)
         self.pg_enable_capture(self.pg_interfaces)
@@ -1174,6 +1196,13 @@ class TestMPLSPIC(VppTestCase):
         # create 2 pg interfaces
         self.create_pg_interfaces(range(4))
 
+        mpls_tbl = VppMplsTable(self, 0)
+        mpls_tbl.add_vpp_config()
+        tbl4 = VppIpTable(self, 1)
+        tbl4.add_vpp_config()
+        tbl6 = VppIpTable(self, 1, is_ip6=1)
+        tbl6.add_vpp_config()
+
         # core links
         self.pg0.admin_up()
         self.pg0.config_ip4()
@@ -1201,14 +1230,15 @@ class TestMPLSPIC(VppTestCase):
         self.pg3.resolve_ndp()
 
     def tearDown(self):
-        super(TestMPLSPIC, self).tearDown()
         self.pg0.disable_mpls()
+        self.pg1.disable_mpls()
         for i in self.pg_interfaces:
             i.unconfig_ip4()
             i.unconfig_ip6()
             i.set_table_ip4(0)
             i.set_table_ip6(0)
             i.admin_down()
+        super(TestMPLSPIC, self).tearDown()
 
     def test_mpls_ibgp_pic(self):
         """ MPLS iBGP PIC edge convergence
@@ -1534,24 +1564,30 @@ class TestMPLSL2(VppTestCase):
         # create 2 pg interfaces
         self.create_pg_interfaces(range(2))
 
+        # create the default MPLS table
+        self.tables = []
+        tbl = VppMplsTable(self, 0)
+        tbl.add_vpp_config()
+        self.tables.append(tbl)
+
         # use pg0 as the core facing interface
         self.pg0.admin_up()
         self.pg0.config_ip4()
         self.pg0.resolve_arp()
         self.pg0.enable_mpls()
 
-        # use the other 2 for customer facg L2 links
+        # use the other 2 for customer facing L2 links
         for i in self.pg_interfaces[1:]:
             i.admin_up()
 
     def tearDown(self):
-        super(TestMPLSL2, self).tearDown()
         for i in self.pg_interfaces[1:]:
             i.admin_down()
 
         self.pg0.disable_mpls()
         self.pg0.unconfig_ip4()
         self.pg0.admin_down()
+        super(TestMPLSL2, self).tearDown()
 
     def verify_capture_tunneled_ethernet(self, capture, sent, mpls_labels,
                                          ttl=255, top=None):
diff --git a/test/test_nat.py b/test/test_nat.py
index 1f2d17ab..73e9e217 100644
--- a/test/test_nat.py
+++ b/test/test_nat.py
@@ -549,6 +549,8 @@ class TestNAT44(MethodHolder):
             cls.pg0.configure_ipv4_neighbors()
 
             cls.overlapping_interfaces = list(list(cls.pg_interfaces[4:7]))
+            cls.vapi.ip_table_add_del(10, is_add=1)
+            cls.vapi.ip_table_add_del(20, is_add=1)
 
             cls.pg4._local_ip4 = "172.16.255.1"
             cls.pg4._local_ip4n = socket.inet_pton(socket.AF_INET, i.local_ip4)
@@ -1797,6 +1799,8 @@ class TestNAT44(MethodHolder):
 
         self.pg0.unconfig_ip4()
         self.pg1.unconfig_ip4()
+        self.vapi.ip_table_add_del(vrf_id1, is_add=1)
+        self.vapi.ip_table_add_del(vrf_id2, is_add=1)
         self.pg0.set_table_ip4(vrf_id1)
         self.pg1.set_table_ip4(vrf_id2)
         self.pg0.config_ip4()
@@ -1825,6 +1829,13 @@ class TestNAT44(MethodHolder):
         capture = self.pg2.get_capture(len(pkts))
         self.verify_capture_out(capture, nat_ip2)
 
+        self.pg0.unconfig_ip4()
+        self.pg1.unconfig_ip4()
+        self.pg0.set_table_ip4(0)
+        self.pg1.set_table_ip4(0)
+        self.vapi.ip_table_add_del(vrf_id1, is_add=0)
+        self.vapi.ip_table_add_del(vrf_id2, is_add=0)
+
     def test_vrf_feature_independent(self):
         """ NAT44 tenant VRF independent address pool mode """
 
@@ -3042,6 +3053,8 @@ class TestNAT64(MethodHolder):
             cls.ip6_interfaces.append(cls.pg_interfaces[2])
             cls.ip4_interfaces = list(cls.pg_interfaces[1:2])
 
+            cls.vapi.ip_table_add_del(cls.vrf1_id, is_add=1, is_ipv6=1)
+
             cls.pg_interfaces[2].set_table_ip6(cls.vrf1_id)
 
             cls.pg0.generate_remote_hosts(2)
diff --git a/test/test_neighbor.py b/test/test_neighbor.py
index 1c7cc267..68dde2fb 100644
--- a/test/test_neighbor.py
+++ b/test/test_neighbor.py
@@ -5,7 +5,8 @@ from socket import AF_INET, AF_INET6, inet_pton
 
 from framework import VppTestCase, VppTestRunner
 from vpp_neighbor import VppNeighbor, find_nbr
-from vpp_ip_route import VppIpRoute, VppRoutePath, find_route
+from vpp_ip_route import VppIpRoute, VppRoutePath, find_route, \
+    VppIpTable
 
 from scapy.packet import Raw
 from scapy.layers.l2 import Ether, ARP, Dot1Q
@@ -39,11 +40,13 @@ class ARPTestCase(VppTestCase):
         self.pg1.config_ip6()
 
         # pg3 in a different VRF
+        self.tbl = VppIpTable(self, 1)
+        self.tbl.add_vpp_config()
+
         self.pg3.set_table_ip4(1)
         self.pg3.config_ip4()
 
     def tearDown(self):
-        super(ARPTestCase, self).tearDown()
         self.pg0.unconfig_ip4()
         self.pg0.unconfig_ip6()
 
@@ -51,10 +54,13 @@ class ARPTestCase(VppTestCase):
         self.pg1.unconfig_ip6()
 
         self.pg3.unconfig_ip4()
+        self.pg3.set_table_ip4(0)
 
         for i in self.pg_interfaces:
             i.admin_down()
 
+        super(ARPTestCase, self).tearDown()
+
     def verify_arp_req(self, rx, smac, sip, dip):
         ether = rx[Ether]
         self.assertEqual(ether.dst, "ff:ff:ff:ff:ff:ff")
@@ -1080,6 +1086,62 @@ class ARPTestCase(VppTestCase):
                        self.pg0.remote_ip4,
                        self.pg1.remote_hosts[1].ip4)
 
+    def test_arp_static(self):
+        """ ARP Static"""
+        self.pg2.generate_remote_hosts(3)
+
+        #
+        # Add a static ARP entry
+        #
+        static_arp = VppNeighbor(self,
+                                 self.pg2.sw_if_index,
+                                 self.pg2.remote_hosts[1].mac,
+                                 self.pg2.remote_hosts[1].ip4,
+                                 is_static=1)
+        static_arp.add_vpp_config()
+
+        #
+        # Add the connected prefix to the interface
+        #
+        self.pg2.config_ip4()
+
+        #
+        # We should now find the adj-fib
+        #
+        self.assertTrue(find_nbr(self,
+                                 self.pg2.sw_if_index,
+                                 self.pg2.remote_hosts[1].ip4,
+                                 is_static=1))
+        self.assertTrue(find_route(self,
+                                   self.pg2.remote_hosts[1].ip4,
+                                   32))
+
+        #
+        # remove the connected
+        #
+        self.pg2.unconfig_ip4()
+
+        #
+        # put the interface into table 1
+        #
+        self.pg2.set_table_ip4(1)
+
+        #
+        # configure the same connected and expect to find the
+        # adj fib in the new table
+        #
+        self.pg2.config_ip4()
+        self.assertTrue(find_route(self,
+                                   self.pg2.remote_hosts[1].ip4,
+                                   32,
+                                   table_id=1))
+
+        #
+        # clean-up
+        #
+        self.pg2.unconfig_ip4()
+        self.pg2.set_table_ip4(0)
+
 
 if __name__ == '__main__':
     unittest.main(testRunner=VppTestRunner)
diff --git a/test/vpp_ip_route.py b/test/vpp_ip_route.py
index 2c489e3c..b7993793 100644
--- a/test/vpp_ip_route.py
+++ b/test/vpp_ip_route.py
@@ -54,6 +54,46 @@ def find_route(test, ip_addr, len, table_id=0, inet=AF_INET):
     return False
 
 
+class VppIpTable(VppObject):
+
+    def __init__(self,
+                 test,
+                 table_id,
+                 is_ip6=0):
+        self._test = test
+        self.table_id = table_id
+        self.is_ip6 = is_ip6
+
+    def add_vpp_config(self):
+        self._test.vapi.ip_table_add_del(
+            self.table_id,
+            is_ipv6=self.is_ip6,
+            is_add=1)
+        self._test.registry.register(self, self._test.logger)
+
+    def remove_vpp_config(self):
+        self._test.vapi.ip_table_add_del(
+            self.table_id,
+            is_ipv6=self.is_ip6,
+            is_add=0)
+
+    def query_vpp_config(self):
+        # find the default route
+        return find_route(self._test,
+                          "::" if self.is_ip6 else "0.0.0.0",
+                          0,
+                          self.table_id,
+                          inet=AF_INET6 if self.is_ip6 == 1 else AF_INET)
+
+    def __str__(self):
+        return self.object_id()
+
+    def object_id(self):
+        return ("table-%s-%d" %
+                ("v6" if self.is_ip6 == 1 else "v4",
+                 self.table_id))
+
+
 class VppRoutePath(object):
 
     def __init__(
@@ -391,6 +431,39 @@ class VppMplsIpBind(VppObject):
                    self.dest_addr_len))
 
 
+class VppMplsTable(VppObject):
+
+    def __init__(self,
+                 test,
+                 table_id):
+        self._test = test
+        self.table_id = table_id
+
+    def add_vpp_config(self):
+        self._test.vapi.mpls_table_add_del(
+            self.table_id,
+            is_add=1)
+        self._test.registry.register(self, self._test.logger)
+
+    def remove_vpp_config(self):
+        self._test.vapi.mpls_table_add_del(
+            self.table_id,
+            is_add=0)
+
+    def query_vpp_config(self):
+        # find the default route
+        dump = self._test.vapi.mpls_fib_dump()
+        if len(dump):
+            return True
+        return False
+
+    def __str__(self):
+        return self.object_id()
+
+    def object_id(self):
+        return ("table-mpls-%d" % (self.table_id))
+
+
 class VppMplsRoute(VppObject):
     """
     MPLS Route/LSP
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index b70da026..519aff80 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -652,6 +652,24 @@ class VppPapiProvider(object):
         return self.api(self.papi.delete_loopback,
                         {'sw_if_index': sw_if_index, })
 
+    def ip_table_add_del(self,
+                         table_id,
+                         is_add=1,
+                         is_ipv6=0):
+        """
+
+        :param table_id
+        :param is_add:  (Default value = 1)
+        :param is_ipv6:  (Default value = 0)
+
+        """
+
+        return self.api(
+            self.papi.ip_table_add_del,
+            {'table_id': table_id,
+             'is_add': is_add,
+             'is_ipv6': is_ipv6})
+
     def ip_add_del_route(
             self,
             dst_address,
@@ -664,7 +682,6 @@ class VppPapiProvider(object):
             next_hop_n_out_labels=0,
             next_hop_out_label_stack=[],
             next_hop_via_label=MPLS_LABEL_INVALID,
-            create_vrf_if_needed=0,
             is_resolve_host=0,
             is_resolve_attached=0,
             classify_table_index=0xFFFFFFFF,
@@ -687,7 +704,6 @@ class VppPapiProvider(object):
         :param vrf_id:  (Default value = 0)
         :param lookup_in_vrf:  (Default value = 0)
         :param classify_table_index:  (Default value = 0xFFFFFFFF)
-        :param create_vrf_if_needed:  (Default value = 0)
         :param is_add:  (Default value = 1)
         :param is_drop:  (Default value = 0)
         :param is_ipv6:  (Default value = 0)
@@ -707,7 +723,6 @@ class VppPapiProvider(object):
              'table_id': table_id,
              'classify_table_index': classify_table_index,
              'next_hop_table_id': next_hop_table_id,
-             'create_vrf_if_needed': create_vrf_if_needed,
              'is_add': is_add,
              'is_drop': is_drop,
              'is_unreach': is_unreach,
@@ -912,6 +927,22 @@ class VppPapiProvider(object):
     def mpls_fib_dump(self):
         return self.api(self.papi.mpls_fib_dump, {})
 
+    def mpls_table_add_del(
+            self,
+            table_id,
+            is_add=1):
+        """
+
+        :param table_id
+        :param is_add:  (Default value = 1)
+
+        """
+
+        return self.api(
+            self.papi.mpls_table_add_del,
+            {'mt_table_id': table_id,
+             'mt_is_add': is_add})
+
     def mpls_route_add_del(
             self,
             label,
@@ -925,7 +956,6 @@ class VppPapiProvider(object):
             next_hop_n_out_labels=0,
             next_hop_out_label_stack=[],
             next_hop_via_label=MPLS_LABEL_INVALID,
-            create_vrf_if_needed=0,
             is_resolve_host=0,
             is_resolve_attached=0,
             is_interface_rx=0,
@@ -947,7 +977,6 @@ class VppPapiProvider(object):
         :param vrf_id:  (Default value = 0)
         :param lookup_in_vrf:  (Default value = 0)
         :param classify_table_index:  (Default value = 0xFFFFFFFF)
-        :param create_vrf_if_needed:  (Default value = 0)
         :param is_add:  (Default value = 1)
         :param is_drop:  (Default value = 0)
         :param is_ipv6:  (Default value = 0)
@@ -968,7 +997,6 @@ class VppPapiProvider(object):
              'mr_eos': eos,
              'mr_table_id': table_id,
              'mr_classify_table_index': classify_table_index,
-             'mr_create_table_if_needed': create_vrf_if_needed,
              'mr_is_add': is_add,
              'mr_is_classify': is_classify,
              'mr_is_multipath': is_multipath,
@@ -994,7 +1022,6 @@ class VppPapiProvider(object):
             table_id=0,
             ip_table_id=0,
             is_ip4=1,
-            create_vrf_if_needed=0,
             is_bind=1):
         """
         """
@@ -1003,7 +1030,6 @@ class VppPapiProvider(object):
             {'mb_mpls_table_id': table_id,
              'mb_label': label,
              'mb_ip_table_id': ip_table_id,
-             'mb_create_table_if_needed': create_vrf_if_needed,
              'mb_is_bind': is_bind,
              'mb_is_ip4': is_ip4,
              'mb_address_length': dst_address_length,
@@ -1020,7 +1046,6 @@ class VppPapiProvider(object):
             next_hop_n_out_labels=0,
             next_hop_out_label_stack=[],
             next_hop_via_label=MPLS_LABEL_INVALID,
-            create_vrf_if_needed=0,
             is_add=1,
             l2_only=0,
             is_multicast=0):
@@ -1034,7 +1059,6 @@ class VppPapiProvider(object):
         :param vrf_id:  (Default value = 0)
         :param lookup_in_vrf:  (Default value = 0)
         :param classify_table_index:  (Default value = 0xFFFFFFFF)
-        :param create_vrf_if_needed:  (Default value = 0)
         :param is_add:  (Default value = 1)
         :param is_drop:  (Default value = 0)
         :param is_ipv6:  (Default value = 0)
@@ -1844,7 +1868,6 @@ class VppPapiProvider(object):
                           i_flags,
                           rpf_id=0,
                           table_id=0,
-                          create_vrf_if_needed=0,
                           is_add=1,
                           is_ipv6=0,
                           is_local=0):
@@ -1857,7 +1880,6 @@ class VppPapiProvider(object):
              'itf_flags': i_flags,
              'table_id': table_id,
              'rpf_id': rpf_id,
-             'create_vrf_if_needed': create_vrf_if_needed,
              'is_add': is_add,
              'is_ipv6': is_ipv6,
              'is_local': is_local,
-- 
cgit 1.2.3-korg


From 2297af016d4c1ecdd0c695dc736e8f5a988e89bd Mon Sep 17 00:00:00 2001
From: Neale Ranns <nranns@cisco.com>
Date: Tue, 12 Sep 2017 09:45:04 -0700
Subject: Add a name to the creation of an IP and MPLS table

Change-Id: I4b4648831551519b2ffb6f93255d28a4b8726c22
Signed-off-by: Neale Ranns <nranns@cisco.com>
---
 src/vnet/fib/fib_entry.c            |  6 ++---
 src/vnet/fib/fib_table.c            | 44 +++++++++++++++++++++++++++++++------
 src/vnet/fib/fib_table.h            | 25 +++++++++++++++++++++
 src/vnet/fib/fib_test.c             |  2 +-
 src/vnet/fib/ip4_fib.c              |  2 +-
 src/vnet/fib/ip6_fib.c              |  2 +-
 src/vnet/ip/ip.api                  | 12 +++++++---
 src/vnet/ip/ip.h                    |  3 ++-
 src/vnet/ip/ip_api.c                | 32 ++++++++++++++-------------
 src/vnet/ip/lookup.c                |  5 ++++-
 src/vnet/mfib/mfib_table.c          | 44 +++++++++++++++++++++++++++++++------
 src/vnet/mfib/mfib_table.h          | 25 +++++++++++++++++++++
 src/vnet/mfib/mfib_test.c           |  2 +-
 src/vnet/mpls/mpls.api              |  5 +++++
 src/vnet/mpls/mpls.c                |  5 ++++-
 src/vnet/mpls/mpls.h                | 40 +++++++++++++++++++--------------
 src/vnet/mpls/mpls_api.c            | 24 +++++++++++---------
 test/test_ip6_vrf_multi_instance.py |  4 ++--
 18 files changed, 210 insertions(+), 72 deletions(-)

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet/fib/fib_entry.c b/src/vnet/fib/fib_entry.c
index 4cb6cf60..4c9b1abd 100644
--- a/src/vnet/fib/fib_entry.c
+++ b/src/vnet/fib/fib_entry.c
@@ -94,8 +94,7 @@ format_fib_source (u8 * s, va_list * args)
 {
     fib_source_t source = va_arg (*args, int);
 
-    s = format (s, "\n  src:%s ",
-                fib_source_names[source]);
+    s = format (s, "src:%s", fib_source_names[source]);
 
     return (s);
 }
@@ -125,8 +124,7 @@ format_fib_entry (u8 * s, va_list * args)
 
 	FOR_EACH_SRC_ADDED(fib_entry, src, source,
         ({
-	    s = format (s, "\n  src:%U ",
-			format_fib_source, source);
+	    s = format (s, "\n  %U", format_fib_source, source);
 	    s = fib_entry_src_format(fib_entry, source, s);
 	    s = format (s, " refs:%d ", src->fes_ref_count);
 	    if (FIB_ENTRY_FLAG_NONE != src->fes_entry_flags) {
diff --git a/src/vnet/fib/fib_table.c b/src/vnet/fib/fib_table.c
index 75d15628..ba1e2720 100644
--- a/src/vnet/fib/fib_table.c
+++ b/src/vnet/fib/fib_table.c
@@ -1037,10 +1037,11 @@ fib_table_find (fib_protocol_t proto,
     return (~0);
 }
 
-u32
-fib_table_find_or_create_and_lock (fib_protocol_t proto,
-				   u32 table_id,
-                                   fib_source_t src)
+static u32
+fib_table_find_or_create_and_lock_i (fib_protocol_t proto,
+                                     u32 table_id,
+                                     fib_source_t src,
+                                     const u8 *name)
 {
     fib_table_t *fib_table;
     fib_node_index_t fi;
@@ -1062,13 +1063,42 @@ fib_table_find_or_create_and_lock (fib_protocol_t proto,
 
     fib_table = fib_table_get(fi, proto);
 
-    fib_table->ft_desc = format(NULL, "%U-VRF:%d",
-                                format_fib_protocol, proto,
-                                table_id);
+    if (NULL == fib_table->ft_desc)
+    {
+        if (name && name[0])
+        {
+            fib_table->ft_desc = format(NULL, "%s", name);
+        }
+        else
+        {
+            fib_table->ft_desc = format(NULL, "%U-VRF:%d",
+                                        format_fib_protocol, proto,
+                                        table_id);
+        }
+    }
 
     return (fi);
 }
 
+u32
+fib_table_find_or_create_and_lock (fib_protocol_t proto,
+				   u32 table_id,
+                                   fib_source_t src)
+{
+    return (fib_table_find_or_create_and_lock_i(proto, table_id,
+                                                src, NULL));
+}
+
+u32
+fib_table_find_or_create_and_lock_w_name (fib_protocol_t proto,
+                                          u32 table_id,
+                                          fib_source_t src,
+                                          const u8 *name)
+{
+    return (fib_table_find_or_create_and_lock_i(proto, table_id,
+                                                src, name));
+}
+
 u32
 fib_table_create_and_lock (fib_protocol_t proto,
                            fib_source_t src,
diff --git a/src/vnet/fib/fib_table.h b/src/vnet/fib/fib_table.h
index 6b7011b3..923d7aff 100644
--- a/src/vnet/fib/fib_table.h
+++ b/src/vnet/fib/fib_table.h
@@ -642,6 +642,31 @@ extern u32 fib_table_find_or_create_and_lock(fib_protocol_t proto,
 					     u32 table_id,
                                              fib_source_t source);
 
+/**
+ * @brief
+ *  Get the index of the FIB for a Table-ID. This DOES create the
+ * FIB if it does not exist.
+ *
+ * @paran proto
+ *  The protocol of the FIB (and thus the entries therein)
+ *
+ * @param table-id
+ *  The Table-ID
+ *
+ * @return fib_index
+ *  The index of the FIB
+ *
+ * @param source
+ *  The ID of the client/source.
+ *
+ * @param name
+ *  The client is choosing the name they want the table to have
+ */
+extern u32 fib_table_find_or_create_and_lock_w_name(fib_protocol_t proto,
+                                                    u32 table_id,
+                                                    fib_source_t source,
+                                                    const u8 *name);
+
 /**
  * @brief
  *  Create a new table with no table ID. This means it does not get
diff --git a/src/vnet/fib/fib_test.c b/src/vnet/fib/fib_test.c
index 572d7f0d..540289ce 100644
--- a/src/vnet/fib/fib_test.c
+++ b/src/vnet/fib/fib_test.c
@@ -8173,7 +8173,7 @@ lfib_test (void)
     /*
      * MPLS enable an interface so we get the MPLS table created
      */
-    mpls_table_create(MPLS_FIB_DEFAULT_TABLE_ID, FIB_SOURCE_API);
+    mpls_table_create(MPLS_FIB_DEFAULT_TABLE_ID, FIB_SOURCE_API, NULL);
     mpls_sw_interface_enable_disable(&mpls_main,
                                      tm->hw[0]->sw_if_index,
                                      1, 1);
diff --git a/src/vnet/fib/ip4_fib.c b/src/vnet/fib/ip4_fib.c
index 865e2dd5..48dc2c6c 100644
--- a/src/vnet/fib/ip4_fib.c
+++ b/src/vnet/fib/ip4_fib.c
@@ -551,7 +551,7 @@ ip4_show_fib (vlib_main_t * vm,
             }
         }
         s = format (s, "]");
-        vlib_cli_output (vm, "%V", s);
+        vlib_cli_output (vm, "%v", s);
         vec_free(s);
 
 	/* Show summary? */
diff --git a/src/vnet/fib/ip6_fib.c b/src/vnet/fib/ip6_fib.c
index 3ddb8453..f37ae0d2 100644
--- a/src/vnet/fib/ip6_fib.c
+++ b/src/vnet/fib/ip6_fib.c
@@ -615,7 +615,7 @@ ip6_show_fib (vlib_main_t * vm,
             }
         }
         s = format (s, "]");
-        vlib_cli_output (vm, "%V", s);
+        vlib_cli_output (vm, "%v", s);
         vec_free(s);
 
 	/* Show summary? */
diff --git a/src/vnet/ip/ip.api b/src/vnet/ip/ip.api
index e57c2fe1..f26d7943 100644
--- a/src/vnet/ip/ip.api
+++ b/src/vnet/ip/ip.api
@@ -27,6 +27,9 @@
     @param table_id - table ID associated with the route
                       This table ID will apply to both the unicats
 		      and mlticast FIBs
+    @param name - A client provided name/tag for the table. If this is
+                  not set by the client, then VPP will generate something
+		  meaningfull.
 */
 autoreply define ip_table_add_del
 {
@@ -35,6 +38,7 @@ autoreply define ip_table_add_del
   u32 table_id;
   u8 is_ipv6;
   u8 is_add;
+  u8 name[64];
 };
 
 /** \brief Dump IP fib table
@@ -83,6 +87,7 @@ manual_endian manual_print define ip_fib_details
 {
   u32 context;
   u32 table_id;
+  u8  table_name[64];
   u8  address_length;
   u8  address[4];
   u32 count;
@@ -98,10 +103,10 @@ define ip6_fib_dump
   u32 context;
 };
 
-/** \brief IP6 FIB table response
+/** \brief IP6 FIB table entry response
     @param table_id - IP6 fib table id
-    @address_length - mask length
-    @address - ip6 prefix
+    @param address_length - mask length
+    @param address - ip6 prefix
     @param count - the number of fib_path in path
     @param path  - array of of fib_path structures
 */
@@ -109,6 +114,7 @@ manual_endian manual_print define ip6_fib_details
 {
   u32 context;
   u32 table_id;
+  u8  table_name[64];
   u8  address_length;
   u8  address[16];
   u32 count;
diff --git a/src/vnet/ip/ip.h b/src/vnet/ip/ip.h
index 78807866..7e26bc6c 100644
--- a/src/vnet/ip/ip.h
+++ b/src/vnet/ip/ip.h
@@ -184,7 +184,8 @@ void ip_del_all_interface_addresses (vlib_main_t * vm, u32 sw_if_index);
 extern vlib_node_registration_t ip4_inacl_node;
 extern vlib_node_registration_t ip6_inacl_node;
 
-void ip_table_create (fib_protocol_t fproto, u32 table_id, u8 is_api);
+void ip_table_create (fib_protocol_t fproto, u32 table_id, u8 is_api,
+		      const u8 * name);
 
 void ip_table_delete (fib_protocol_t fproto, u32 table_id, u8 is_api);
 
diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c
index 384ec3e0..bb29e0bc 100644
--- a/src/vnet/ip/ip_api.c
+++ b/src/vnet/ip/ip_api.c
@@ -180,7 +180,8 @@ copy_fib_next_hop (fib_route_path_encode_t * api_rpath, void *fp_arg)
 static void
 send_ip_fib_details (vpe_api_main_t * am,
 		     unix_shared_memory_queue_t * q,
-		     u32 table_id, fib_prefix_t * pfx,
+		     const fib_table_t * table,
+		     const fib_prefix_t * pfx,
 		     fib_route_path_encode_t * api_rpaths, u32 context)
 {
   vl_api_ip_fib_details_t *mp;
@@ -196,7 +197,9 @@ send_ip_fib_details (vpe_api_main_t * am,
   mp->_vl_msg_id = ntohs (VL_API_IP_FIB_DETAILS);
   mp->context = context;
 
-  mp->table_id = htonl (table_id);
+  mp->table_id = htonl (table->ft_table_id);
+  memcpy (mp->table_name, table->ft_desc,
+	  clib_min (vec_len (table->ft_desc), sizeof (mp->table_name)));
   mp->address_length = pfx->fp_len;
   memcpy (mp->address, &pfx->fp_addr.ip4, sizeof (pfx->fp_addr.ip4));
 
@@ -295,9 +298,7 @@ vl_api_ip_fib_dump_t_handler (vl_api_ip_fib_dump_t * mp)
     fib_table = fib_table_get (fib_index, pfx.fp_proto);
     api_rpaths = NULL;
     fib_entry_encode (*lfeip, &api_rpaths);
-    send_ip_fib_details (am, q,
-			 fib_table->ft_table_id, &pfx, api_rpaths,
-			 mp->context);
+    send_ip_fib_details (am, q, fib_table, &pfx, api_rpaths, mp->context);
     vec_free (api_rpaths);
   }
 
@@ -744,7 +745,7 @@ vl_api_ip_table_add_del_t_handler (vl_api_ip_table_add_del_t * mp)
 
   if (mp->is_add)
     {
-      ip_table_create (fproto, table_id, 1);
+      ip_table_create (fproto, table_id, 1, mp->name);
     }
   else
     {
@@ -1124,7 +1125,8 @@ vl_api_ip_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp)
 }
 
 void
-ip_table_create (fib_protocol_t fproto, u32 table_id, u8 is_api)
+ip_table_create (fib_protocol_t fproto,
+		 u32 table_id, u8 is_api, const u8 * name)
 {
   u32 fib_index, mfib_index;
 
@@ -1147,17 +1149,17 @@ ip_table_create (fib_protocol_t fproto, u32 table_id, u8 is_api)
 
       if (~0 == fib_index)
 	{
-	  fib_table_find_or_create_and_lock (fproto, table_id,
-					     (is_api ?
-					      FIB_SOURCE_API :
-					      FIB_SOURCE_CLI));
+	  fib_table_find_or_create_and_lock_w_name (fproto, table_id,
+						    (is_api ?
+						     FIB_SOURCE_API :
+						     FIB_SOURCE_CLI), name);
 	}
       if (~0 == mfib_index)
 	{
-	  mfib_table_find_or_create_and_lock (fproto, table_id,
-					      (is_api ?
-					       MFIB_SOURCE_API :
-					       MFIB_SOURCE_CLI));
+	  mfib_table_find_or_create_and_lock_w_name (fproto, table_id,
+						     (is_api ?
+						      MFIB_SOURCE_API :
+						      MFIB_SOURCE_CLI), name);
 	}
     }
 }
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
index 667c6791..d9922f46 100755
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -695,6 +695,7 @@ vnet_ip_table_cmd (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   clib_error_t *error = NULL;
   u32 table_id, is_add;
+  u8 *name = NULL;
 
   is_add = 1;
   table_id = ~0;
@@ -711,6 +712,8 @@ vnet_ip_table_cmd (vlib_main_t * vm,
 	is_add = 0;
       else if (unformat (line_input, "add"))
 	is_add = 1;
+      else if (unformat (line_input, "name %s", &name))
+	;
       else
 	{
 	  error = unformat_parse_error (line_input);
@@ -732,7 +735,7 @@ vnet_ip_table_cmd (vlib_main_t * vm,
     {
       if (is_add)
 	{
-	  ip_table_create (fproto, table_id, 0);
+	  ip_table_create (fproto, table_id, 0, name);
 	}
       else
 	{
diff --git a/src/vnet/mfib/mfib_table.c b/src/vnet/mfib/mfib_table.c
index e5550adc..838864ff 100644
--- a/src/vnet/mfib/mfib_table.c
+++ b/src/vnet/mfib/mfib_table.c
@@ -422,10 +422,11 @@ mfib_table_find (fib_protocol_t proto,
     return (~0);
 }
 
-u32
-mfib_table_find_or_create_and_lock (fib_protocol_t proto,
-                                    u32 table_id,
-                                    mfib_source_t src)
+static u32
+mfib_table_find_or_create_and_lock_i (fib_protocol_t proto,
+                                      u32 table_id,
+                                      mfib_source_t src,
+                                      const u8 *name)
 {
     mfib_table_t *mfib_table;
     fib_node_index_t fi;
@@ -445,13 +446,42 @@ mfib_table_find_or_create_and_lock (fib_protocol_t proto,
 
     mfib_table = mfib_table_get(fi, proto);
 
-    mfib_table->mft_desc = format(NULL, "%U-VRF:%d",
-                                  format_fib_protocol, proto,
-                                  table_id);
+    if (NULL == mfib_table->mft_desc)
+    {
+        if (name && name[0])
+        {
+            mfib_table->mft_desc = format(NULL, "%s", name);
+        }
+        else
+        {
+            mfib_table->mft_desc = format(NULL, "%U-VRF:%d",
+                                          format_fib_protocol, proto,
+                                          table_id);
+        }
+    }
 
     return (fi);
 }
 
+u32
+mfib_table_find_or_create_and_lock (fib_protocol_t proto,
+                                    u32 table_id,
+                                    mfib_source_t src)
+{
+    return (mfib_table_find_or_create_and_lock_i(proto, table_id,
+                                                 src, NULL));
+}
+
+u32
+mfib_table_find_or_create_and_lock_w_name (fib_protocol_t proto,
+                                           u32 table_id,
+                                           mfib_source_t src,
+                                           const u8 *name)
+{
+    return (mfib_table_find_or_create_and_lock_i(proto, table_id,
+                                                 src, name));
+}
+
 /**
  * @brief Table flush context. Store the indicies of matching FIB entries
  * that need to be removed.
diff --git a/src/vnet/mfib/mfib_table.h b/src/vnet/mfib/mfib_table.h
index c6b0b097..93f90dd5 100644
--- a/src/vnet/mfib/mfib_table.h
+++ b/src/vnet/mfib/mfib_table.h
@@ -322,6 +322,31 @@ extern u32 mfib_table_find_or_create_and_lock(fib_protocol_t proto,
                                               u32 table_id,
                                               mfib_source_t source);
 
+/**
+ * @brief
+ *  Get the index of the FIB for a Table-ID. This DOES create the
+ * FIB if it does not exist.
+ *
+ * @paran proto
+ *  The protocol of the FIB (and thus the entries therein)
+ *
+ * @param table-id
+ *  The Table-ID
+ *
+ * @return fib_index
+ *  The index of the FIB
+ *
+ * @param source
+ *  The ID of the client/source.
+ *
+ * @param name
+ *  The client is choosing the name they want the table to have
+ */
+extern u32 mfib_table_find_or_create_and_lock_w_name(fib_protocol_t proto,
+                                                     u32 table_id,
+                                                     mfib_source_t source,
+                                                     const u8 *name);
+
 
 /**
  * @brief
diff --git a/src/vnet/mfib/mfib_test.c b/src/vnet/mfib/mfib_test.c
index 3055844d..2562bc14 100644
--- a/src/vnet/mfib/mfib_test.c
+++ b/src/vnet/mfib/mfib_test.c
@@ -1114,7 +1114,7 @@ mfib_test_i (fib_protocol_t PROTO,
     /*
      * MPLS enable an interface so we get the MPLS table created
      */
-    mpls_table_create(MPLS_FIB_DEFAULT_TABLE_ID, FIB_SOURCE_API);
+    mpls_table_create(MPLS_FIB_DEFAULT_TABLE_ID, FIB_SOURCE_API, NULL);
     mpls_sw_interface_enable_disable(&mpls_main,
                                      tm->hw[0]->sw_if_index,
                                      1, 0);
diff --git a/src/vnet/mpls/mpls.api b/src/vnet/mpls/mpls.api
index bb849998..36488d0c 100644
--- a/src/vnet/mpls/mpls.api
+++ b/src/vnet/mpls/mpls.api
@@ -141,6 +141,9 @@ manual_endian manual_print define mpls_tunnel_details
     @param context - sender context, to match reply w/ request
     @param mt_table_id - The MPLS table-id the route is added in
     @param mt_is_add - Is this a route add or delete
+    @param mt_name - A client provided name/tag for the table. If this
+                     is not set by the client, then VPP will generate
+		     something meaningfull.
 */
 autoreply define mpls_table_add_del
 {
@@ -148,6 +151,7 @@ autoreply define mpls_table_add_del
   u32 context;
   u32 mt_table_id;
   u8  mt_is_add;
+  u8  mt_name[64];
 };
 
 /** \brief MPLS Route Add / del route
@@ -227,6 +231,7 @@ manual_endian manual_print define mpls_fib_details
 {
   u32 context;
   u32 table_id;
+  u8  table_name[64];
   u8  eos_bit;
   u32 label;
   u32 count;
diff --git a/src/vnet/mpls/mpls.c b/src/vnet/mpls/mpls.c
index 7bdfd8c7..f8bbb8a9 100644
--- a/src/vnet/mpls/mpls.c
+++ b/src/vnet/mpls/mpls.c
@@ -544,6 +544,7 @@ vnet_mpls_table_cmd (vlib_main_t * vm,
   unformat_input_t _line_input, *line_input = &_line_input;
   clib_error_t *error = NULL;
   u32 table_id, is_add;
+  u8 *name = NULL;
 
   is_add = 1;
   table_id = ~0;
@@ -560,6 +561,8 @@ vnet_mpls_table_cmd (vlib_main_t * vm,
 	is_add = 0;
       else if (unformat (line_input, "add"))
 	is_add = 1;
+      else if (unformat (line_input, "name %s", &name))
+	;
       else
 	{
 	  error = unformat_parse_error (line_input);
@@ -581,7 +584,7 @@ vnet_mpls_table_cmd (vlib_main_t * vm,
     {
       if (is_add)
         {
-          mpls_table_create (table_id, 0);
+            mpls_table_create (table_id, 0, name);
         }
       else
         {
diff --git a/src/vnet/mpls/mpls.h b/src/vnet/mpls/mpls.h
index 8dd92887..cc3eeed0 100644
--- a/src/vnet/mpls/mpls.h
+++ b/src/vnet/mpls/mpls.h
@@ -23,7 +23,8 @@
 #include <vnet/fib/fib_node.h>
 #include <vnet/adj/adj.h>
 
-typedef enum {
+typedef enum
+{
 #define mpls_error(n,s) MPLS_ERROR_##n,
 #include <vnet/mpls/error.def>
 #undef mpls_error
@@ -34,10 +35,11 @@ typedef enum {
  * @brief Definition of a callback for receiving MPLS interface state change
  * notifications
  */
-typedef void (*mpls_interface_state_change_callback_t)(u32 sw_if_index,
-                                                       u32 is_enable);
+typedef void (*mpls_interface_state_change_callback_t) (u32 sw_if_index,
+							u32 is_enable);
 
-typedef struct {
+typedef struct
+{
   /* MPLS FIB index for each software interface */
   u32 *fib_index_by_sw_if_index;
 
@@ -55,12 +57,12 @@ typedef struct {
   u8 output_feature_arc_index;
 
   /* IP4 enabled count by software interface */
-  u8 * mpls_enabled_by_sw_if_index;
+  u8 *mpls_enabled_by_sw_if_index;
 } mpls_main_t;
 
 extern mpls_main_t mpls_main;
 
-extern clib_error_t * mpls_feature_init(vlib_main_t * vm);
+extern clib_error_t *mpls_feature_init (vlib_main_t * vm);
 
 format_function_t format_mpls_eos_bit;
 format_function_t format_mpls_unicast_header_net_byte_order;
@@ -81,24 +83,28 @@ unformat_function_t unformat_mpls_header;
 unformat_function_t unformat_pg_mpls_header;
 
 int mpls_sw_interface_enable_disable (mpls_main_t * mm,
-                                      u32 sw_if_index,
-                                      u8 is_enable,
-                                      u8 is_api);
+				      u32 sw_if_index,
+				      u8 is_enable, u8 is_api);
 
 u8 mpls_sw_interface_is_enabled (u32 sw_if_index);
 
 int mpls_fib_reset_labels (u32 fib_id);
 
-int
-mpls_dest_cmp(void * a1, void * a2);
+int mpls_dest_cmp (void *a1, void *a2);
 
-int
-mpls_fib_index_cmp(void * a1, void * a2);
+int mpls_fib_index_cmp (void *a1, void *a2);
 
-int
-mpls_label_cmp(void * a1, void * a2);
+int mpls_label_cmp (void *a1, void *a2);
 
-void mpls_table_create(u32 table_id, u8 is_api);
-void mpls_table_delete(u32 table_id, u8 is_api);
+void mpls_table_create (u32 table_id, u8 is_api, const u8 * name);
+void mpls_table_delete (u32 table_id, u8 is_api);
 
 #endif /* included_vnet_mpls_h */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vnet/mpls/mpls_api.c b/src/vnet/mpls/mpls_api.c
index 38f5b014..988c2c98 100644
--- a/src/vnet/mpls/mpls_api.c
+++ b/src/vnet/mpls/mpls_api.c
@@ -92,7 +92,7 @@ vl_api_mpls_table_add_del_t_handler (vl_api_mpls_table_add_del_t * mp)
   vnm->api_errno = 0;
 
   if (mp->mt_is_add)
-    mpls_table_create (ntohl (mp->mt_table_id), 1);
+    mpls_table_create (ntohl (mp->mt_table_id), 1, mp->mt_name);
   else
     mpls_table_delete (ntohl (mp->mt_table_id), 1);
 
@@ -258,7 +258,7 @@ vl_api_mpls_route_add_del_t_handler (vl_api_mpls_route_add_del_t * mp)
 }
 
 void
-mpls_table_create (u32 table_id, u8 is_api)
+mpls_table_create (u32 table_id, u8 is_api, const u8 * name)
 {
   u32 fib_index;
 
@@ -276,10 +276,11 @@ mpls_table_create (u32 table_id, u8 is_api)
 
   if (~0 == fib_index)
     {
-      fib_table_find_or_create_and_lock (FIB_PROTOCOL_MPLS,
-					 table_id,
-					 (is_api ?
-					  FIB_SOURCE_API : FIB_SOURCE_CLI));
+      fib_table_find_or_create_and_lock_w_name (FIB_PROTOCOL_MPLS,
+						table_id,
+						(is_api ?
+						 FIB_SOURCE_API :
+						 FIB_SOURCE_CLI), name);
     }
 }
 
@@ -424,7 +425,8 @@ vl_api_mpls_tunnel_dump_t_handler (vl_api_mpls_tunnel_dump_t * mp)
 static void
 send_mpls_fib_details (vpe_api_main_t * am,
 		       unix_shared_memory_queue_t * q,
-		       u32 table_id, u32 label, u32 eos,
+		       const fib_table_t * table,
+		       u32 label, u32 eos,
 		       fib_route_path_encode_t * api_rpaths, u32 context)
 {
   vl_api_mpls_fib_details_t *mp;
@@ -440,7 +442,9 @@ send_mpls_fib_details (vpe_api_main_t * am,
   mp->_vl_msg_id = ntohs (VL_API_MPLS_FIB_DETAILS);
   mp->context = context;
 
-  mp->table_id = htonl (table_id);
+  mp->table_id = htonl (table->ft_table_id);
+  memcpy (mp->table_name, table->ft_desc,
+	  clib_min (vec_len (table->ft_desc), sizeof (mp->table_name)));
   mp->eos_bit = eos;
   mp->label = htonl (label);
 
@@ -512,8 +516,8 @@ vl_api_mpls_fib_dump_t_handler (vl_api_mpls_fib_dump_t * mp)
     api_rpaths = NULL;
     fib_entry_encode (*lfeip, &api_rpaths);
     send_mpls_fib_details (am, q,
-			   fib_table->ft_table_id,
-			   pfx.fp_label, pfx.fp_eos, api_rpaths, mp->context);
+			   fib_table, pfx.fp_label,
+			   pfx.fp_eos, api_rpaths, mp->context);
     vec_free (api_rpaths);
   }
 
diff --git a/test/test_ip6_vrf_multi_instance.py b/test/test_ip6_vrf_multi_instance.py
index 769cb2e5..8dd228ae 100644
--- a/test/test_ip6_vrf_multi_instance.py
+++ b/test/test_ip6_vrf_multi_instance.py
@@ -316,10 +316,10 @@ class TestIP6VrfMultiInst(VppTestCase):
         vrf_exist = False
         vrf_count = 0
         for ip6_fib_details in ip6_fib_dump:
-            if ip6_fib_details[2] == vrf_id:
+            if ip6_fib_details.table_id == vrf_id:
                 if not vrf_exist:
                     vrf_exist = True
-                addr = inet_ntop(socket.AF_INET6, ip6_fib_details[4])
+                addr = inet_ntop(socket.AF_INET6, ip6_fib_details.address)
                 addrtype = in6_getAddrType(addr)
                 vrf_count += 1 if addrtype == IPV6_ADDR_UNICAST else 0
         if not vrf_exist and vrf_count == 0:
-- 
cgit 1.2.3-korg


From de6c03c0f104114d43a53776839600ce792731de Mon Sep 17 00:00:00 2001
From: Jon Loeliger <jdl@netgate.com>
Date: Thu, 28 Sep 2017 13:54:16 -0500
Subject: Propagate duplicate IF addr add/del error up to API.

Identify and complain when the same IP prefix is assigned
to two different SW interfaces:

    vpp# set int ip address TenGigabitEthernet6/0/0 1.2.3.4/32
    vpp# set int ip address TenGigabitEthernet6/0/1 1.2.3.4/32
    set interface ip address: Prefix 1.2.3.4/32 already found on
	interface TenGigabitEthernet6/0/0

Change-Id: I1aee1b6a7ddd00d3109a53d8e1b6ce97bf45e372
Signed-off-by: Jon Loeliger <jdl@netgate.com>
(cherry picked from commit 35ffa3e8f6b032f6e324234d495f769049d8feea)
---
 src/vnet/api_errno.h     |  3 ++-
 src/vnet/interface_api.c | 23 +++++++++++++++++------
 src/vnet/ip/lookup.c     | 12 +++++++++++-
 3 files changed, 30 insertions(+), 8 deletions(-)
 mode change 100755 => 100644 src/vnet/ip/lookup.c

(limited to 'src/vnet/ip/lookup.c')

diff --git a/src/vnet/api_errno.h b/src/vnet/api_errno.h
index c0deb1d0..22cfaee4 100644
--- a/src/vnet/api_errno.h
+++ b/src/vnet/api_errno.h
@@ -115,7 +115,8 @@ _(BD_ID_EXCEED_MAX, -122, "Bridge domain ID exceed 16M limit")		\
 _(SUBIF_DOESNT_EXIST, -123, "Subinterface doesn't exist")               \
 _(L2_MACS_EVENT_CLINET_PRESENT, -124, "Client already exist for L2 MACs events") \
 _(INVALID_QUEUE, -125, "Invalid queue")                 		\
-_(UNSUPPORTED, -126, "Unsupported")
+_(UNSUPPORTED, -126, "Unsupported")					\
+_(DUPLICATE_IF_ADDRESS, -127, "Address already present on another interface")
 
 typedef enum
 {
diff --git a/src/vnet/interface_api.c b/src/vnet/interface_api.c
index 05d9f211..6374a880 100644
--- a/src/vnet/interface_api.c
+++ b/src/vnet/interface_api.c
@@ -281,27 +281,38 @@ static void
   (vl_api_sw_interface_add_del_address_t * mp)
 {
   vlib_main_t *vm = vlib_get_main ();
+  vnet_main_t *vnm = vnet_get_main ();
   vl_api_sw_interface_add_del_address_reply_t *rmp;
   int rv = 0;
   u32 is_del;
+  clib_error_t *error = 0;
 
   VALIDATE_SW_IF_INDEX (mp);
 
   is_del = mp->is_add == 0;
+  vnm->api_errno = 0;
 
   if (mp->del_all)
     ip_del_all_interface_addresses (vm, ntohl (mp->sw_if_index));
   else if (mp->is_ipv6)
-    ip6_add_del_interface_address (vm, ntohl (mp->sw_if_index),
-				   (void *) mp->address,
-				   mp->address_length, is_del);
+    error = ip6_add_del_interface_address (vm, ntohl (mp->sw_if_index),
+					   (void *) mp->address,
+					   mp->address_length, is_del);
   else
-    ip4_add_del_interface_address (vm, ntohl (mp->sw_if_index),
-				   (void *) mp->address,
-				   mp->address_length, is_del);
+    error = ip4_add_del_interface_address (vm, ntohl (mp->sw_if_index),
+					   (void *) mp->address,
+					   mp->address_length, is_del);
+
+  if (error)
+    {
+      rv = vnm->api_errno;
+      clib_error_report (error);
+      goto done;
+    }
 
   BAD_SW_IF_INDEX_LABEL;
 
+done:
   REPLY_MACRO (VL_API_SW_INTERFACE_ADD_DEL_ADDRESS_REPLY);
 }
 
diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c
old mode 100755
new mode 100644
index d9922f46..856c4942
--- a/src/vnet/ip/lookup.c
+++ b/src/vnet/ip/lookup.c
@@ -165,11 +165,21 @@ ip_interface_address_add_del (ip_lookup_main_t * lm,
     }
   else
     {
+      if (sw_if_index != a->sw_if_index)
+	{
+	  if (result_if_address_index)
+	    *result_if_address_index = ~0;
+	  vnm->api_errno = VNET_API_ERROR_DUPLICATE_IF_ADDRESS;
+	  return clib_error_create
+	    ("Prefix %U already found on interface %U",
+	     lm->format_address_and_length, addr_fib, address_length,
+	     format_vnet_sw_if_index_name, vnm, a->sw_if_index);
+	}
+
       if (result_if_address_index)
 	*result_if_address_index = a - lm->if_address_pool;
     }
 
-
   return /* no error */ 0;
 }
 
-- 
cgit 1.2.3-korg