Add support for installing ipv4 routes via unresolved next hop

Change-Id: I71f3ba0c8192fe0ac3b5b81fb1275b64ec02876a
Signed-off-by: Damjan Marion <damarion@cisco.com>

6 files changed, 393 insertions, 159 deletions

diff --git a/vnet/vnet/ethernet/arp.c b/vnet/vnet/ethernet/arp.c
index 3eb6a11391e..205023a6ffb 100644
--- a/vnet/vnet/ethernet/arp.c
+++ b/vnet/vnet/ethernet/arp.c
@@ -35,8 +35,11 @@ typedef struct {
 
   u16 flags;
 #define ETHERNET_ARP_IP4_ENTRY_FLAG_STATIC (1 << 0)
+#define ETHERNET_ARP_IP4_ENTRY_FLAG_GLEAN  (2 << 0)
 
   u64 cpu_time_last_updated;
+
+  u32 * adjacencies;
 } ethernet_arp_ip4_entry_t;
 
 typedef struct {
@@ -210,22 +213,31 @@ static u8 * format_ethernet_arp_ip4_entry (u8 * s, va_list * va)
   ethernet_arp_ip4_entry_t * e = va_arg (*va, ethernet_arp_ip4_entry_t *);
   vnet_sw_interface_t * si;
   ip4_fib_t * fib;
+  u8 * flags = 0;
 
   if (! e)
-    return format (s, "%=12s%=6s%=16s%=4s%=20s%=24s", "Time", "FIB", "IP4", 
-                   "Static", "Ethernet", "Interface");
+    return format (s, "%=12s%=6s%=16s%=6s%=20s%=24s", "Time", "FIB", "IP4",
+                   "Flags", "Ethernet", "Interface");
 
   fib = find_ip4_fib_by_table_index_or_id (&ip4_main, e->key.fib_index,
                                            IP4_ROUTE_FLAG_FIB_INDEX);
   si = vnet_get_sw_interface (vnm, e->key.sw_if_index);
-  s = format (s, "%=12U%=6u%=16U%=4s%=20U%=25U",
+
+  if (e->flags & ETHERNET_ARP_IP4_ENTRY_FLAG_GLEAN)
+    flags = format(flags, "G");
+
+  if (e->flags & ETHERNET_ARP_IP4_ENTRY_FLAG_STATIC)
+    flags = format(flags, "S");
+
+  s = format (s, "%=12U%=6u%=16U%=6s%=20U%=24U",
 	      format_vlib_cpu_time, vnm->vlib_main, e->cpu_time_last_updated,
               fib->table_id,
 	      format_ip4_address, &e->key.ip4_address,
-              (e->flags & ETHERNET_ARP_IP4_ENTRY_FLAG_STATIC) ? "S" : "",
+	      flags ? (char *) flags : "",
 	      format_ethernet_address, e->ethernet_address,
 	      format_vnet_sw_interface_name, vnm, si);
 
+  vec_free(flags);
   return s;
 }
 
@@ -357,13 +369,15 @@ vnet_arp_set_ip4_over_ethernet_internal (vnet_main_t * vnm,
   ethernet_arp_ip4_over_ethernet_address_t * a = a_arg;
   vlib_main_t * vm = vlib_get_main();
   ip4_main_t * im = &ip4_main;
+  ip_lookup_main_t * lm = &im->lookup_main;
   int make_new_arp_cache_entry=1;
   uword * p;
   ip4_add_del_route_args_t args;
-  ip_adjacency_t adj;
+  ip_adjacency_t adj, * existing_adj;
   pending_resolution_t * pr, * mc;
   
   u32 next_index;
+  u32 adj_index;
 
   fib_index = (fib_index != (u32)~0) 
     ? fib_index : im->fib_index_by_sw_if_index[sw_if_index];
@@ -396,15 +410,40 @@ vnet_arp_set_ip4_over_ethernet_internal (vnet_main_t * vnm,
      &adj.rewrite_header,
      sizeof (adj.rewrite_data));
 
-  args.table_index_or_table_id = fib_index;
-  args.flags = IP4_ROUTE_FLAG_FIB_INDEX | IP4_ROUTE_FLAG_ADD | IP4_ROUTE_FLAG_NEIGHBOR;
-  args.dst_address = a->ip4;
-  args.dst_address_length = 32;
-  args.adj_index = ~0;
-  args.add_adj = &adj;
-  args.n_add_adj = 1;
+  /* result of this lookup should be next-hop adjacency */
+  adj_index = ip4_fib_lookup_with_table (im, fib_index, &a->ip4, 0);
+  existing_adj = ip_get_adjacency(lm, adj_index);
+
+  if (existing_adj->lookup_next_index == IP_LOOKUP_NEXT_ARP &&
+      existing_adj->arp.next_hop.ip4.as_u32 == a->ip4.as_u32)
+    {
+      u32 * ai;
+      u32 * adjs = vec_dup(e->adjacencies);
+      /* Update all adj assigned to this arp entry */
+      vec_foreach(ai, adjs)
+	{
+	  int i;
+	  ip_adjacency_t * uadj = ip_get_adjacency(lm, *ai);
+	  for (i = 0; i < uadj->n_adj; i++)
+	    if (uadj[i].lookup_next_index == IP_LOOKUP_NEXT_ARP &&
+		uadj[i].arp.next_hop.ip4.as_u32 == a->ip4.as_u32)
+	      ip_update_adjacency (lm, *ai + i, &adj);
+	}
+      vec_free(adjs);
+    }
+  else
+    {
+      /* create new adj */
+      args.table_index_or_table_id = fib_index;
+      args.flags = IP4_ROUTE_FLAG_FIB_INDEX | IP4_ROUTE_FLAG_ADD | IP4_ROUTE_FLAG_NEIGHBOR;
+      args.dst_address = a->ip4;
+      args.dst_address_length = 32;
+      args.adj_index = ~0;
+      args.add_adj = &adj;
+      args.n_add_adj = 1;
+      ip4_add_del_route (im, &args);
+    }
 
-  ip4_add_del_route (im, &args);
   if (make_new_arp_cache_entry)
     {
       pool_get (am->ip4_entry_pool, e);
@@ -1242,11 +1281,88 @@ clib_error_t *ip4_set_arp_limit (u32 arp_limit)
   return 0;
 }
 
+static void
+arp_ip4_entry_del_adj(ethernet_arp_ip4_entry_t *e, u32 adj_index)
+{
+  int done = 0;
+  int i;
+
+  while (!done)
+    {
+      vec_foreach_index(i, e->adjacencies)
+	if (vec_elt(e->adjacencies, i) == adj_index)
+	  {
+	    vec_del1(e->adjacencies, i);
+	    continue;
+	  }
+      done = 1;
+    }
+}
+
+static void
+arp_ip4_entry_add_adj(ethernet_arp_ip4_entry_t *e, u32 adj_index)
+{
+  int i;
+  vec_foreach_index(i, e->adjacencies)
+    if (vec_elt(e->adjacencies, i) == adj_index)
+      return;
+  vec_add1(e->adjacencies, adj_index);
+}
+
+static void
+arp_add_del_adj_cb (struct ip_lookup_main_t * lm,
+		    u32 adj_index,
+		    ip_adjacency_t * adj,
+		    u32 is_del)
+{
+  ethernet_arp_main_t * am = &ethernet_arp_main;
+  ip4_main_t * im = &ip4_main;
+  ethernet_arp_ip4_key_t k;
+  ethernet_arp_ip4_entry_t * e = 0;
+  uword * p;
+  u32 ai;
+
+  for(ai = adj->heap_handle; ai < adj->heap_handle + adj->n_adj ; ai++)
+    {
+      adj = ip_get_adjacency (lm, ai);
+      if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP && adj->arp.next_hop.ip4.as_u32)
+	{
+	  k.sw_if_index = adj->rewrite_header.sw_if_index;
+	  k.ip4_address.as_u32 = adj->arp.next_hop.ip4.as_u32;
+	  k.fib_index = im->fib_index_by_sw_if_index[adj->rewrite_header.sw_if_index];
+	  p = mhash_get (&am->ip4_entry_by_key, &k);
+	  if (p)
+	    e = pool_elt_at_index (am->ip4_entry_pool, p[0]);
+	}
+      else
+	continue;
+
+      if (is_del)
+	{
+	  if (!e)
+	    clib_warning("Adjacency contains unknown ARP next hop %U (del)",
+			 format_ip4_address, &adj->arp.next_hop);
+	  else
+	    arp_ip4_entry_del_adj(e, adj->heap_handle);
+	}
+      else /* add */
+	{
+	  if (!e)
+	    clib_warning("Adjacency contains unknown ARP next hop %U (add)",
+			 format_ip4_address, &adj->arp.next_hop);
+	  else
+	    arp_ip4_entry_add_adj(e, adj->heap_handle);
+	}
+    }
+}
+
 static clib_error_t * ethernet_arp_init (vlib_main_t * vm)
 {
   ethernet_arp_main_t * am = &ethernet_arp_main;
   pg_node_t * pn;
   clib_error_t * error;
+  ip4_main_t * im = &ip4_main;
+  ip_lookup_main_t * lm = &im->lookup_main;
 
   if ((error = vlib_call_init_function (vm, ethernet_init)))
     return error;
@@ -1283,7 +1399,9 @@ static clib_error_t * ethernet_arp_init (vlib_main_t * vm)
     foreach_ethernet_arp_error
 #undef _
   }
-  
+
+  ip_register_add_del_adjacency_callback(lm, arp_add_del_adj_cb);
+
   return 0;
 }
 
@@ -1474,6 +1592,55 @@ int vnet_proxy_arp_fib_reset (u32 fib_id)
    return 0;
 }
 
+u32
+vnet_arp_glean_add(u32 fib_index, void * next_hop_arg)
+{
+  ethernet_arp_main_t * am = &ethernet_arp_main;
+  ip4_main_t * im = &ip4_main;
+  ip_lookup_main_t * lm = &im->lookup_main;
+  ip4_address_t * next_hop = next_hop_arg;
+  ip_adjacency_t add_adj, *adj;
+  ip4_add_del_route_args_t args;
+  ethernet_arp_ip4_entry_t * e;
+  ethernet_arp_ip4_key_t k;
+  u32 adj_index;
+
+  adj_index = ip4_fib_lookup_with_table(im, fib_index, next_hop, 0);
+  adj = ip_get_adjacency(lm, adj_index);
+
+  if (!adj || adj->lookup_next_index != IP_LOOKUP_NEXT_ARP)
+    return ~0;
+
+  if (adj->arp.next_hop.ip4.as_u32 != 0)
+    return adj_index;
+
+  k.sw_if_index = adj->rewrite_header.sw_if_index;
+  k.fib_index = fib_index;
+  k.ip4_address.as_u32 = next_hop->as_u32;
+
+  if (mhash_get (&am->ip4_entry_by_key, &k))
+    return adj_index;
+
+  pool_get (am->ip4_entry_pool, e);
+  mhash_set (&am->ip4_entry_by_key, &k, e - am->ip4_entry_pool, /* old value */ 0);
+  e->key = k;
+  e->cpu_time_last_updated = clib_cpu_time_now ();
+  e->flags = ETHERNET_ARP_IP4_ENTRY_FLAG_GLEAN;
+
+  memset(&args, 0, sizeof(args));
+  memcpy(&add_adj, adj, sizeof(add_adj));
+  add_adj.arp.next_hop.ip4.as_u32 = next_hop->as_u32; /* install neighbor /32 route */
+  args.table_index_or_table_id = fib_index;
+  args.flags = IP4_ROUTE_FLAG_FIB_INDEX | IP4_ROUTE_FLAG_ADD| IP4_ROUTE_FLAG_NEIGHBOR;
+  args.dst_address.as_u32 = next_hop->as_u32;
+  args.dst_address_length = 32;
+  args.adj_index = ~0;
+  args.add_adj = &add_adj;
+  args.n_add_adj = 1;
+  ip4_add_del_route (im, &args);
+  return ip4_fib_lookup_with_table (im, fib_index, next_hop, 0);
+}
+
 static clib_error_t *
 ip_arp_add_del_command_fn (vlib_main_t * vm,
 		 unformat_input_t * input,
diff --git a/vnet/vnet/ethernet/ethernet.h b/vnet/vnet/ethernet/ethernet.h
index ea01463c072..492aa759d00 100644
--- a/vnet/vnet/ethernet/ethernet.h
+++ b/vnet/vnet/ethernet/ethernet.h
@@ -448,6 +448,8 @@ int vnet_add_del_ip4_arp_change_event (vnet_main_t * vnm,
                                        uword type_opaque,
                                        uword data, int is_add);
 
+u32 vnet_arp_glean_add(u32 fib_index, void * next_hop_arg);
+
 extern vlib_node_registration_t ethernet_input_node;
 
 #endif /* included_ethernet_h */
diff --git a/vnet/vnet/ip/ip4_forward.c b/vnet/vnet/ip/ip4_forward.c
index 6bfe1cff2f2..e099cd9d047 100644
--- a/vnet/vnet/ip/ip4_forward.c
+++ b/vnet/vnet/ip/ip4_forward.c
@@ -242,7 +242,7 @@ void ip4_add_del_route (ip4_main_t * im, ip4_add_del_route_args_t * a)
   old_adj_index = fib->old_hash_values[0];
 
   /* Avoid spurious reference count increments */
-  if (old_adj_index == adj_index)
+  if (old_adj_index == adj_index && !(a->flags & IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY))
     {
       ip_adjacency_t * adj = ip_get_adjacency (lm, adj_index);
       if (adj->share_count > 0)
@@ -318,13 +318,29 @@ ip4_add_del_route_next_hop (ip4_main_t * im,
           /* Next hop must be known. */
           if (! nh_result)
             {
-              vnm->api_errno = VNET_API_ERROR_NEXT_HOP_NOT_IN_FIB;
-              error = clib_error_return (0, "next-hop %U/32 not in FIB",
-                                         format_ip4_address, next_hop);
-              goto done;
-            }
-          nh_adj_index = *nh_result;
-        }
+	      ip_adjacency_t * adj;
+
+	      nh_adj_index = ip4_fib_lookup_with_table (im, fib_index,
+							next_hop, 0);
+	      adj = ip_get_adjacency (lm, nh_adj_index);
+	      /* if ARP interface adjacencty is present, we need to
+		 install ARP adjaceny for specific next hop */
+	      if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP &&
+		  adj->arp.next_hop.ip4.as_u32 == 0)
+		{
+		  nh_adj_index = vnet_arp_glean_add(fib_index, next_hop);
+		}
+	      else
+		{
+		  vnm->api_errno = VNET_API_ERROR_NEXT_HOP_NOT_IN_FIB;
+		  error = clib_error_return (0, "next-hop %U/32 not in FIB",
+					     format_ip4_address, next_hop);
+		  goto done;
+		}
+	    }
+	  else
+	    nh_adj_index = *nh_result;
+	}
     }
   else
     {
@@ -369,6 +385,29 @@ ip4_add_del_route_next_hop (ip4_main_t * im,
       goto done;
     }
 
+  /* Destination is not known and default weight is set so add route
+     to existing non-multipath adjacency */
+  if (dst_adj_index == ~0 && next_hop_weight == 1 && next_hop_sw_if_index == ~0)
+    {
+      /* create new adjacency */
+      ip4_add_del_route_args_t a;
+      a.table_index_or_table_id = fib_index;
+      a.flags = ((is_del ? IP4_ROUTE_FLAG_DEL : IP4_ROUTE_FLAG_ADD)
+		 | IP4_ROUTE_FLAG_FIB_INDEX
+		 | IP4_ROUTE_FLAG_KEEP_OLD_ADJACENCY
+		 | (flags & (IP4_ROUTE_FLAG_NO_REDISTRIBUTE
+			     | IP4_ROUTE_FLAG_NOT_LAST_IN_GROUP)));
+      a.dst_address = dst_address[0];
+      a.dst_address_length = dst_address_length;
+      a.adj_index = nh_adj_index;
+      a.add_adj = 0;
+      a.n_add_adj = 0;
+
+      ip4_add_del_route (im, &a);
+
+      goto done;
+    }
+
   old_mp_adj_index = dst_adj ? dst_adj->heap_handle : ~0;
 
   if (! ip_multipath_adjacency_add_del_next_hop
@@ -934,6 +973,7 @@ void ip4_adjacency_set_interface_route (vnet_main_t * vnm,
       n = IP_LOOKUP_NEXT_ARP;
       node_index = ip4_arp_node.index;
       adj->if_address_index = if_address_index;
+      adj->arp.next_hop.ip4.as_u32 = 0;
       packet_type = VNET_L3_PACKET_TYPE_ARP;
     }
   else
@@ -2084,6 +2124,10 @@ ip4_arp (vlib_main_t * vm,
 	  adj0 = ip_get_adjacency (lm, adj_index0);
 	  ip0 = vlib_buffer_get_current (p0);
 
+	  /* If packet destination is not local, send ARP to next hop */
+	  if (adj0->arp.next_hop.ip4.as_u32)
+	    ip0->dst_address.data_u32 = adj0->arp.next_hop.ip4.as_u32;
+
 	  /* 
 	   * if ip4_rewrite_local applied the IP_LOOKUP_NEXT_ARP
 	   * rewrite to this packet, we need to skip it here.
diff --git a/vnet/vnet/ip/ip6_forward.c b/vnet/vnet/ip/ip6_forward.c
index 3e8261a05a5..27c776e3d44 100644
--- a/vnet/vnet/ip/ip6_forward.c
+++ b/vnet/vnet/ip/ip6_forward.c
@@ -266,7 +266,7 @@ void ip6_add_del_route (ip6_main_t * im, ip6_add_del_route_args_t * a)
     }
 
   /* Avoid spurious reference count increments */
-  if (old_adj_index == adj_index)
+  if (old_adj_index == adj_index && !(a->flags & IP6_ROUTE_FLAG_KEEP_OLD_ADJACENCY))
     {
       ip_adjacency_t * adj = ip_get_adjacency (lm, adj_index);
       if (adj->share_count > 0)
diff --git a/vnet/vnet/ip/lookup.c b/vnet/vnet/ip/lookup.c
index 629c900a6b6..010e9e93cd3 100644
--- a/vnet/vnet/ip/lookup.c
+++ b/vnet/vnet/ip/lookup.c
@@ -59,6 +59,81 @@ ip_poison_adjacencies (ip_adjacency_t * adj, uword n_adj)
     }
 }
 
+static void
+ip_share_adjacency(ip_lookup_main_t * lm, u32 adj_index)
+{
+  ip_adjacency_t * adj = ip_get_adjacency(lm, adj_index);
+  uword * p;
+  u32 old_ai;
+  uword signature = vnet_ip_adjacency_signature (adj);
+
+  p = hash_get (lm->adj_index_by_signature, signature);
+  /* Hash collision? */
+  if (p)
+    {
+      /* Save the adj index, p[0] will be toast after the unset! */
+      old_ai = p[0];
+      hash_unset (lm->adj_index_by_signature, signature);
+      hash_set (lm->adj_index_by_signature, signature, adj_index);
+      adj->next_adj_with_signature = old_ai;
+    }
+  else
+    {
+      adj->next_adj_with_signature = 0;
+      hash_set (lm->adj_index_by_signature, signature, adj_index);
+    }
+}
+
+static void
+ip_unshare_adjacency(ip_lookup_main_t * lm, u32 adj_index)
+{
+  ip_adjacency_t * adj = ip_get_adjacency(lm, adj_index);
+  uword signature;
+  uword * p;
+  u32 this_ai;
+  ip_adjacency_t * this_adj, * prev_adj = 0;
+
+  signature = vnet_ip_adjacency_signature (adj);
+  p = hash_get (lm->adj_index_by_signature, signature);
+  if (p == 0)
+      return;
+
+  this_ai = p[0];
+  /* At the top of the signature chain (likely)? */
+  if (this_ai == adj_index)
+    {
+      if (adj->next_adj_with_signature == 0)
+	{
+	  hash_unset (lm->adj_index_by_signature, signature);
+	  return;
+	}
+      else
+	{
+	  this_adj = ip_get_adjacency (lm, adj->next_adj_with_signature);
+	  hash_unset (lm->adj_index_by_signature, signature);
+	  hash_set (lm->adj_index_by_signature, signature,
+		    this_adj->heap_handle);
+	}
+    }
+  else                      /* walk signature chain */
+    {
+      this_adj = ip_get_adjacency (lm, this_ai);
+      while (this_adj != adj)
+	{
+	  prev_adj = this_adj;
+	  this_adj = ip_get_adjacency
+	    (lm, this_adj->next_adj_with_signature);
+	  /*
+	   * This can happen when creating the first multipath adj of a set
+	   * We end up looking at the miss adjacency (handle==0).
+	   */
+	  if (this_adj->heap_handle == 0)
+            return;
+        }
+      prev_adj->next_adj_with_signature = this_adj->next_adj_with_signature;
+    }
+}
+
 /* Create new block of given number of contiguous adjacencies. */
 ip_adjacency_t *
 ip_add_adjacency (ip_lookup_main_t * lm,
@@ -149,32 +224,33 @@ ip_add_adjacency (ip_lookup_main_t * lm,
 
   /* Set up to share the adj later */
   if (copy_adj && n_adj == 1)
-    {
-      uword * p;
-      u32 old_ai;
-      uword signature = vnet_ip_adjacency_signature (adj);
-
-      p = hash_get (lm->adj_index_by_signature, signature);
-      /* Hash collision? */
-      if (p)
-        {
-          /* Save the adj index, p[0] will be toast after the unset! */
-          old_ai = p[0];
-          hash_unset (lm->adj_index_by_signature, signature);
-          hash_set (lm->adj_index_by_signature, signature, ai);
-          adj->next_adj_with_signature = old_ai;
-        }
-      else
-        {
-          adj->next_adj_with_signature = 0;
-          hash_set (lm->adj_index_by_signature, signature, ai);
-        }
-    }
+    ip_share_adjacency(lm, ai);
 
   *adj_index_return = ai;
   return adj;
 }
 
+void
+ip_update_adjacency (ip_lookup_main_t * lm,
+		     u32 adj_index,
+		     ip_adjacency_t * copy_adj)
+{
+  ip_adjacency_t * adj = ip_get_adjacency(lm, adj_index);
+
+  ip_call_add_del_adjacency_callbacks (lm, adj_index, /* is_del */ 1);
+  ip_unshare_adjacency(lm, adj_index);
+
+  /* temporary redirect to drop while updating rewrite data */
+  adj->lookup_next_index = IP_LOOKUP_NEXT_ARP;
+  CLIB_MEMORY_BARRIER();
+
+  memcpy (&adj->rewrite_header, &copy_adj->rewrite_header,
+	  VLIB_BUFFER_PRE_DATA_SIZE);
+  adj->lookup_next_index = copy_adj->lookup_next_index;
+  ip_share_adjacency(lm, adj_index);
+  ip_call_add_del_adjacency_callbacks (lm, adj_index, /* is_del */ 0);
+}
+
 static void ip_del_adjacency2 (ip_lookup_main_t * lm, u32 adj_index, u32 delete_multipath_adjacency)
 {
   ip_adjacency_t * adj;
@@ -189,58 +265,15 @@ static void ip_del_adjacency2 (ip_lookup_main_t * lm, u32 adj_index, u32 delete_
 
   if (adj->n_adj == 1)
     {
-      uword signature;
-      uword * p;
-      u32 this_ai;
-      ip_adjacency_t * this_adj, * prev_adj = 0;
       if (adj->share_count > 0)
         {
           adj->share_count --;
           return;
         }
 
-      signature = vnet_ip_adjacency_signature (adj);
-      p = hash_get (lm->adj_index_by_signature, signature);
-      if (p == 0)
-          goto bag_it;
-
-      this_ai = p[0];
-      /* At the top of the signature chain (likely)? */
-      if (this_ai == adj_index)
-        {
-          if (adj->next_adj_with_signature == 0)
-            {
-              hash_unset (lm->adj_index_by_signature, signature);
-              goto bag_it;
-            }
-          else
-            {
-              this_adj = ip_get_adjacency (lm, adj->next_adj_with_signature);
-              hash_unset (lm->adj_index_by_signature, signature);
-              hash_set (lm->adj_index_by_signature, signature,
-                        this_adj->heap_handle);
-            }
-        }
-      else                      /* walk signature chain */
-        {
-          this_adj = ip_get_adjacency (lm, this_ai);
-          while (this_adj != adj)
-            {
-              prev_adj = this_adj;
-              this_adj = ip_get_adjacency
-                (lm, this_adj->next_adj_with_signature);
-              /* 
-               * This can happen when creating the first multipath adj of a set
-               * We end up looking at the miss adjacency (handle==0).
-               */
-              if (this_adj->heap_handle == 0)
-                  goto bag_it;
-            }
-          prev_adj->next_adj_with_signature = this_adj->next_adj_with_signature;
-        }
+      ip_unshare_adjacency(lm, adj_index);
     }
 
- bag_it:
   if (delete_multipath_adjacency)
     ip_multipath_del_adjacency (lm, adj_index);
 
@@ -475,6 +508,17 @@ ip_multipath_adjacency_add_del_next_hop (ip_lookup_main_t * lm,
   i_nh = 0;
   nhs = 0;
 
+  /* If old adj is not multipath, we need to "convert" it by calling this
+   * function recursively */
+  if (old_mp_adj_index != ~0 && !ip_adjacency_is_multipath(lm, old_mp_adj_index))
+    {
+      ip_multipath_adjacency_add_del_next_hop(lm, /* is_del */ 0,
+					      /* old_mp_adj_index */ ~0,
+					      /* nh_adj_index */ old_mp_adj_index,
+					      /* weight * */ 1,
+					      &old_mp_adj_index);
+    }
+
   /* If old multipath adjacency is valid, find requested next hop. */
   if (old_mp_adj_index < vec_len (lm->multipath_adjacencies)
       && lm->multipath_adjacencies[old_mp_adj_index].normalized_next_hops.count > 0)
@@ -952,6 +996,11 @@ u8 * format_ip_adjacency (u8 * s, va_list * args)
       switch (adj->lookup_next_index)
 	{
 	case IP_LOOKUP_NEXT_ARP:
+	  if (adj->if_address_index != ~0)
+	    s = format (s, " %U", format_ip_interface_address, lm, adj->if_address_index);
+	  if (adj->arp.next_hop.ip4.as_u32)
+	    s = format (s, " via %U", format_ip4_address, &adj->arp.next_hop.ip4.as_u32);
+	  break;
 	case IP_LOOKUP_NEXT_LOCAL:
 	  if (adj->if_address_index != ~0)
 	    s = format (s, " %U", format_ip_interface_address, lm, adj->if_address_index);
@@ -1275,73 +1324,6 @@ vnet_ip_route_cmd (vlib_main_t * vm, unformat_input_t * main_input, vlib_cli_com
       goto done;
     }
 
-  if (vec_len(ip4_via_next_hops))
-    {
-      if (sw_if_indices[0] == (u32)~0)
-        {
-          u32 ai;
-          uword * p;
-          u32 fib_index;
-          ip_adjacency_t *nh_adj;
-
-          p = hash_get (ip4_main.fib_index_by_table_id, table_ids[0]);
-          if (p == 0)
-            {
-              error = clib_error_return (0, "Nonexistent FIB id %d",
-                                         table_ids[0]);
-              goto done;
-            }
-
-          fib_index = p[0];
-
-          ai = ip4_fib_lookup_with_table (&ip4_main,
-                                          fib_index,
-                                          ip4_via_next_hops,
-                                          1 /* disable default route */);
-          if (ai == 0)
-            {
-              error = clib_error_return (0, "next hop %U not in FIB",
-                                         format_ip4_address,
-                                         ip4_via_next_hops);
-              goto done;
-            }
-          nh_adj = ip_get_adjacency (&ip4_main.lookup_main, ai);
-          vec_add1 (add_adj, nh_adj[0]);
-        }
-    }
-  if (vec_len(ip6_via_next_hops))
-    {
-      if (sw_if_indices[0] == (u32)~0)
-        {
-          u32 ai;
-          uword * p;
-          u32 fib_index;
-          ip_adjacency_t *nh_adj;
-
-          p = hash_get (ip6_main.fib_index_by_table_id, table_ids[0]);
-          if (p == 0)
-            {
-              error = clib_error_return (0, "Nonexistent FIB id %d",
-                                         table_ids[0]);
-              goto done;
-            }
-
-          fib_index = p[0];
-          ai = ip6_fib_lookup_with_table (&ip6_main,
-                                          fib_index,
-                                          ip6_via_next_hops);
-          if (ai == 0)
-            {
-              error = clib_error_return (0, "next hop %U not in FIB",
-                                         format_ip6_address,
-                                         ip6_via_next_hops);
-              goto done;
-            }
-          nh_adj = ip_get_adjacency (&ip6_main.lookup_main, ai);
-          vec_add1 (add_adj, nh_adj[0]);
-        }
-    }
-
   {
     int i;
     ip4_main_t * im4 = &ip4_main;
@@ -2017,8 +1999,13 @@ ip4_show_fib (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * c
 		  msg = format (msg, "%16Ld%16Ld ", sum.packets, sum.bytes);
 
 		  indent = vec_len (msg);
-		  msg = format (msg, "weight %d, index %d\n%U%U",
-				nhs[j].weight, adj_index + i,
+		  msg = format (msg, "weight %d, index %d",
+				nhs[j].weight, adj_index + i);
+
+		  if (ip_adjacency_is_multipath(lm, adj_index))
+		      msg = format (msg, ", multipath");
+
+		  msg = format (msg, "\n%U%U",
 				format_white_space, indent,
 				format_ip_adjacency,
 				vnm, lm, adj_index + i);
@@ -2240,8 +2227,13 @@ ip6_show_fib (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * c
 		  msg = format (msg, "%16Ld%16Ld ", sum.packets, sum.bytes);
 
 		  indent = vec_len (msg);
-		  msg = format (msg, "weight %d, index %d\n%U%U",
-				nhs[j].weight, adj_index + i,
+		  msg = format (msg, "weight %d, index %d",
+				nhs[j].weight, adj_index + i);
+
+		  if (ip_adjacency_is_multipath(lm, adj_index + i))
+		      msg = format (msg, ", multipath");
+
+		  msg = format (msg, "\n%U%U",
 				format_white_space, indent,
 				format_ip_adjacency,
 				vnm, lm, adj_index + i);
diff --git a/vnet/vnet/ip/lookup.h b/vnet/vnet/ip/lookup.h
index ba242ef5de4..fcac675d899 100644
--- a/vnet/vnet/ip/lookup.h
+++ b/vnet/vnet/ip/lookup.h
@@ -42,6 +42,7 @@
 
 #include <vnet/vnet.h>
 #include <vlib/buffer.h>
+#include <vnet/ip/ip4_packet.h>
 
 /* Next index stored in adjacency. */
 typedef enum {
@@ -133,6 +134,13 @@ typedef struct {
   u16 saved_lookup_next_index;
 
   union {
+    /* IP_LOOKUP_NEXT_ARP only */
+    struct {
+      union {
+        ip4_address_t ip4;
+      } next_hop;
+      u32 next_adj_index_with_same_next_hop;
+    } arp;
     /* IP_LOOKUP_NEXT_CLASSIFY only */
     struct {
       u16 table_index;
@@ -391,6 +399,13 @@ do {								\
   CLIB_PREFETCH (_adj, sizeof (_adj[0]), type);			\
 } while (0)
 
+static inline void
+ip_register_add_del_adjacency_callback(ip_lookup_main_t * lm,
+				       ip_add_del_adjacency_callback_t cb)
+{
+  vec_add1(lm->add_del_adjacency_callbacks, cb);
+}
+
 always_inline void
 ip_call_add_del_adjacency_callbacks (ip_lookup_main_t * lm, u32 adj_index, u32 is_del)
 {
@@ -409,6 +424,20 @@ ip_add_adjacency (ip_lookup_main_t * lm,
 		  u32 * adj_index_result);
 
 void ip_del_adjacency (ip_lookup_main_t * lm, u32 adj_index);
+void
+ip_update_adjacency (ip_lookup_main_t * lm,
+		     u32 adj_index,
+		     ip_adjacency_t * copy_adj);
+
+static inline int
+ip_adjacency_is_multipath(ip_lookup_main_t * lm, u32 adj_index)
+{
+  if (vec_len(lm->multipath_adjacencies) < adj_index - 1)
+    return 0;
+
+  return (lm->multipath_adjacencies[adj_index].adj_index == adj_index &&
+	  lm->multipath_adjacencies[adj_index].n_adj_in_block > 0);
+}
 
 void
 ip_multipath_adjacency_free (ip_lookup_main_t * lm,