Fix IP neighbor/arp pool full and static entry handling

Move handling of IP neighbor pool full into main thread on entry
creation and make sure static entriesare not deleted for reuse.
Fix IPv6 neighbor handling on interface down and up so that static
entries are not deleted.

Change-Id: I073794949a41a5b86201e519ebe479febfc506c8
Signed-off-by: John Lo <loj@cisco.com>

2 files changed, 169 insertions, 166 deletions

diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c
index 49a16f764f3..249e3b6e8cf 100644
--- a/src/vnet/ethernet/arp.c
+++ b/src/vnet/ethernet/arp.c
@@ -545,6 +545,62 @@ arp_adj_fib_add (ethernet_arp_ip4_entry_t * e, u32 fib_index)
   fib_table_lock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_ADJ);
 }
 
+void
+arp_adj_fib_remove (ethernet_arp_ip4_entry_t * e, u32 fib_index)
+{
+  if (FIB_NODE_INDEX_INVALID != e->fib_entry_index)
+    {
+      fib_prefix_t pfx = {
+	.fp_len = 32,
+	.fp_proto = FIB_PROTOCOL_IP4,
+	.fp_addr.ip4 = e->ip4_address,
+      };
+      u32 fib_index;
+
+      fib_index = ip4_fib_table_get_index_for_sw_if_index (e->sw_if_index);
+
+      fib_table_entry_path_remove (fib_index, &pfx,
+				   FIB_SOURCE_ADJ,
+				   DPO_PROTO_IP4,
+				   &pfx.fp_addr,
+				   e->sw_if_index, ~0, 1,
+				   FIB_ROUTE_PATH_FLAG_NONE);
+      fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_ADJ);
+    }
+}
+
+static ethernet_arp_ip4_entry_t *
+force_reuse_arp_entry (void)
+{
+  ethernet_arp_ip4_entry_t *e;
+  ethernet_arp_main_t *am = &ethernet_arp_main;
+  u32 count = 0;
+  u32 index = pool_next_index (am->ip4_entry_pool, am->arp_delete_rotor);
+  if (index == ~0)		/* Try again from elt 0 */
+    index = pool_next_index (am->ip4_entry_pool, index);
+
+  /* Find a non-static random entry to free up for reuse */
+  do
+    {
+      if ((count++ == 100) || (index == ~0))
+	return NULL;		/* give up after 100 entries */
+      e = pool_elt_at_index (am->ip4_entry_pool, index);
+      am->arp_delete_rotor = index;
+      index = pool_next_index (am->ip4_entry_pool, index);
+    }
+  while (e->flags & ETHERNET_ARP_IP4_ENTRY_FLAG_STATIC);
+
+  /* Remove ARP entry from its interface and update fib */
+  hash_unset
+    (am->ethernet_arp_by_sw_if_index[e->sw_if_index].arp_entries,
+     e->ip4_address.as_u32);
+  arp_adj_fib_remove
+    (e, ip4_fib_table_get_index_for_sw_if_index (e->sw_if_index));
+  adj_nbr_walk_nh4 (e->sw_if_index,
+		    &e->ip4_address, arp_mk_incomplete_walk, NULL);
+  return e;
+}
+
 static int
 vnet_arp_set_ip4_over_ethernet_internal (vnet_main_t * vnm,
 					 vnet_arp_set_ip4_over_ethernet_rpc_args_t
@@ -582,12 +638,18 @@ vnet_arp_set_ip4_over_ethernet_internal (vnet_main_t * vnm,
 
   if (make_new_arp_cache_entry)
     {
-      pool_get (am->ip4_entry_pool, e);
-
-      if (NULL == arp_int->arp_entries)
+      if (am->limit_arp_cache_size &&
+	  pool_elts (am->ip4_entry_pool) >= am->limit_arp_cache_size)
 	{
-	  arp_int->arp_entries = hash_create (0, sizeof (u32));
+	  e = force_reuse_arp_entry ();
+	  if (NULL == e)
+	    return -2;
 	}
+      else
+	pool_get (am->ip4_entry_pool, e);
+
+      if (NULL == arp_int->arp_entries)
+	arp_int->arp_entries = hash_create (0, sizeof (u32));
 
       hash_set (arp_int->arp_entries, a->ip4.as_u32, e - am->ip4_entry_pool);
 
@@ -820,38 +882,6 @@ typedef enum
     ETHERNET_ARP_N_ERROR,
 } ethernet_arp_input_error_t;
 
-
-static void
-unset_random_arp_entry (void)
-{
-  ethernet_arp_main_t *am = &ethernet_arp_main;
-  ethernet_arp_ip4_entry_t *e;
-  vnet_main_t *vnm = vnet_get_main ();
-  ethernet_arp_ip4_over_ethernet_address_t delme;
-  u32 index;
-
-  index = pool_next_index (am->ip4_entry_pool, am->arp_delete_rotor);
-  am->arp_delete_rotor = index;
-
-  /* Try again from elt 0, could happen if an intfc goes down */
-  if (index == ~0)
-    {
-      index = pool_next_index (am->ip4_entry_pool, am->arp_delete_rotor);
-      am->arp_delete_rotor = index;
-    }
-
-  /* Nothing left in the pool */
-  if (index == ~0)
-    return;
-
-  e = pool_elt_at_index (am->ip4_entry_pool, index);
-
-  clib_memcpy (&delme.ethernet, e->ethernet_address, 6);
-  delme.ip4.as_u32 = e->ip4_address.as_u32;
-
-  vnet_arp_unset_ip4_over_ethernet (vnm, e->sw_if_index, &delme);
-}
-
 static int
 arp_unnumbered (vlib_buffer_t * p0,
 		u32 input_sw_if_index, u32 conn_sw_if_index)
@@ -881,10 +911,6 @@ static u32
 arp_learn (vnet_main_t * vnm,
 	   ethernet_arp_main_t * am, u32 sw_if_index, void *addr)
 {
-  if (am->limit_arp_cache_size &&
-      pool_elts (am->ip4_entry_pool) >= am->limit_arp_cache_size)
-    unset_random_arp_entry ();
-
   vnet_arp_set_ip4_over_ethernet (vnm, sw_if_index, addr, 0, 0);
   return (ETHERNET_ARP_ERROR_l3_src_address_learned);
 }
@@ -1641,30 +1667,6 @@ arp_add_del_interface_address (ip4_main_t * im,
     }
 }
 
-void
-arp_adj_fib_remove (ethernet_arp_ip4_entry_t * e, u32 fib_index)
-{
-  if (FIB_NODE_INDEX_INVALID != e->fib_entry_index)
-    {
-      fib_prefix_t pfx = {
-	.fp_len = 32,
-	.fp_proto = FIB_PROTOCOL_IP4,
-	.fp_addr.ip4 = e->ip4_address,
-      };
-      u32 fib_index;
-
-      fib_index = ip4_fib_table_get_index_for_sw_if_index (e->sw_if_index);
-
-      fib_table_entry_path_remove (fib_index, &pfx,
-				   FIB_SOURCE_ADJ,
-				   DPO_PROTO_IP4,
-				   &pfx.fp_addr,
-				   e->sw_if_index, ~0, 1,
-				   FIB_ROUTE_PATH_FLAG_NONE);
-      fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_ADJ);
-    }
-}
-
 static void
 arp_table_bind (ip4_main_t * im,
 		uword opaque,
@@ -1761,9 +1763,8 @@ arp_entry_free (ethernet_arp_interface_t * eai, ethernet_arp_ip4_entry_t * e)
 {
   ethernet_arp_main_t *am = &ethernet_arp_main;
 
-  arp_adj_fib_remove (e,
-		      ip4_fib_table_get_index_for_sw_if_index
-		      (e->sw_if_index));
+  arp_adj_fib_remove
+    (e, ip4_fib_table_get_index_for_sw_if_index (e->sw_if_index));
   hash_unset (eai->arp_entries, e->ip4_address.as_u32);
   pool_put (am->ip4_entry_pool, e);
 }
@@ -1786,10 +1787,9 @@ vnet_arp_unset_ip4_over_ethernet_internal (vnet_main_t * vnm,
 
   if (NULL != e)
     {
-      arp_entry_free (eai, e);
-
       adj_nbr_walk_nh4 (e->sw_if_index,
 			&e->ip4_address, arp_mk_incomplete_walk, NULL);
+      arp_entry_free (eai, e);
     }
 
   return 0;
diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c
index 5e98475f94d..1b37e549e83 100644
--- a/src/vnet/ip/ip6_neighbor.c
+++ b/src/vnet/ip/ip6_neighbor.c
@@ -374,73 +374,6 @@ ip6_neighbor_adj_fib_remove (ip6_neighbor_t * n, u32 fib_index)
     }
 }
 
-static clib_error_t *
-ip6_neighbor_sw_interface_up_down (vnet_main_t * vnm,
-				   u32 sw_if_index, u32 flags)
-{
-  ip6_neighbor_main_t *nm = &ip6_neighbor_main;
-  ip6_neighbor_t *n;
-
-  if (!(flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP))
-    {
-      u32 i, *to_delete = 0;
-
-      /* *INDENT-OFF* */
-      pool_foreach (n, nm->neighbor_pool,
-      ({
-	if (n->key.sw_if_index == sw_if_index)
-	  vec_add1 (to_delete, n - nm->neighbor_pool);
-      }));
-      /* *INDENT-ON* */
-
-      for (i = 0; i < vec_len (to_delete); i++)
-	{
-	  n = pool_elt_at_index (nm->neighbor_pool, to_delete[i]);
-	  mhash_unset (&nm->neighbor_index_by_key, &n->key, 0);
-	  ip6_neighbor_adj_fib_remove (n,
-				       ip6_fib_table_get_index_for_sw_if_index
-				       (n->key.sw_if_index));
-	  pool_put (nm->neighbor_pool, n);
-	}
-      vec_free (to_delete);
-    }
-
-  return 0;
-}
-
-VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_neighbor_sw_interface_up_down);
-
-static void
-unset_random_neighbor_entry (void)
-{
-  ip6_neighbor_main_t *nm = &ip6_neighbor_main;
-  vnet_main_t *vnm = vnet_get_main ();
-  vlib_main_t *vm = vnm->vlib_main;
-  ip6_neighbor_t *e;
-  u32 index;
-
-  index = pool_next_index (nm->neighbor_pool, nm->neighbor_delete_rotor);
-  nm->neighbor_delete_rotor = index;
-
-  /* Try again from elt 0, could happen if an intfc goes down */
-  if (index == ~0)
-    {
-      index = pool_next_index (nm->neighbor_pool, nm->neighbor_delete_rotor);
-      nm->neighbor_delete_rotor = index;
-    }
-
-  /* Nothing left in the pool */
-  if (index == ~0)
-    return;
-
-  e = pool_elt_at_index (nm->neighbor_pool, index);
-
-  vnet_unset_ip6_ethernet_neighbor (vm, e->key.sw_if_index,
-				    &e->key.ip6_address,
-				    e->link_layer_address,
-				    ETHER_MAC_ADDR_LEN);
-}
-
 typedef struct
 {
   u8 is_add;
@@ -611,6 +544,54 @@ ip6_nd_mk_incomplete_walk (adj_index_t ai, void *ctx)
   return (ADJ_WALK_RC_CONTINUE);
 }
 
+static clib_error_t *
+ip6_neighbor_sw_interface_up_down (vnet_main_t * vnm,
+				   u32 sw_if_index, u32 flags)
+{
+  ip6_neighbor_main_t *nm = &ip6_neighbor_main;
+  ip6_neighbor_t *n;
+  u32 i, *to_delete = 0;
+
+  /* *INDENT-OFF* */
+  pool_foreach (n, nm->neighbor_pool,
+  ({
+    if (n->key.sw_if_index == sw_if_index)
+      vec_add1 (to_delete, n - nm->neighbor_pool);
+  }));
+  /* *INDENT-ON* */
+
+  if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
+    {
+      for (i = 0; i < vec_len (to_delete); i++)
+	{
+	  n = pool_elt_at_index (nm->neighbor_pool, to_delete[i]);
+	  adj_nbr_walk_nh6 (n->key.sw_if_index, &n->key.ip6_address,
+			    ip6_nd_mk_complete_walk, n);
+	}
+    }
+  else
+    {
+      for (i = 0; i < vec_len (to_delete); i++)
+	{
+	  n = pool_elt_at_index (nm->neighbor_pool, to_delete[i]);
+	  adj_nbr_walk_nh6 (n->key.sw_if_index, &n->key.ip6_address,
+			    ip6_nd_mk_incomplete_walk, NULL);
+	  if (n->flags & IP6_NEIGHBOR_FLAG_STATIC)
+	    continue;
+	  ip6_neighbor_adj_fib_remove (n,
+				       ip6_fib_table_get_index_for_sw_if_index
+				       (n->key.sw_if_index));
+	  mhash_unset (&nm->neighbor_index_by_key, &n->key, 0);
+	  pool_put (nm->neighbor_pool, n);
+	}
+    }
+
+  vec_free (to_delete);
+  return 0;
+}
+
+VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ip6_neighbor_sw_interface_up_down);
+
 void
 ip6_ethernet_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai)
 {
@@ -723,6 +704,37 @@ ip6_neighbor_adj_fib_add (ip6_neighbor_t * n, u32 fib_index)
     }
 }
 
+static ip6_neighbor_t *
+force_reuse_neighbor_entry (void)
+{
+  ip6_neighbor_t *n;
+  ip6_neighbor_main_t *nm = &ip6_neighbor_main;
+  u32 count = 0;
+  u32 index = pool_next_index (nm->neighbor_pool, nm->neighbor_delete_rotor);
+  if (index == ~0)		/* Try again from elt 0 */
+    index = pool_next_index (nm->neighbor_pool, index);
+
+  /* Find a non-static random entry to free up for reuse */
+  do
+    {
+      if ((count++ == 100) || (index == ~0))
+	return NULL;		/* give up after 100 entries */
+      n = pool_elt_at_index (nm->neighbor_pool, index);
+      nm->neighbor_delete_rotor = index;
+      index = pool_next_index (nm->neighbor_pool, index);
+    }
+  while (n->flags & IP6_NEIGHBOR_FLAG_STATIC);
+
+  /* Remove ARP entry from its interface and update fib */
+  adj_nbr_walk_nh6 (n->key.sw_if_index,
+		    &n->key.ip6_address, ip6_nd_mk_incomplete_walk, NULL);
+  ip6_neighbor_adj_fib_remove
+    (n, ip6_fib_table_get_index_for_sw_if_index (n->key.sw_if_index));
+  mhash_unset (&nm->neighbor_index_by_key, &n->key, 0);
+
+  return n;
+}
+
 int
 vnet_set_ip6_ethernet_neighbor (vlib_main_t * vm,
 				u32 sw_if_index,
@@ -763,7 +775,16 @@ vnet_set_ip6_ethernet_neighbor (vlib_main_t * vm,
 
   if (make_new_nd_cache_entry)
     {
-      pool_get (nm->neighbor_pool, n);
+      if (nm->limit_neighbor_cache_size &&
+	  pool_elts (nm->neighbor_pool) >= nm->limit_neighbor_cache_size)
+	{
+	  n = force_reuse_neighbor_entry ();
+	  if (NULL == n)
+	    return -2;
+	}
+      else
+	pool_get (nm->neighbor_pool, n);
+
       mhash_set (&nm->neighbor_index_by_key, &k, n - nm->neighbor_pool,
 		 /* old value */ 0);
       n->key = k;
@@ -805,9 +826,15 @@ vnet_set_ip6_ethernet_neighbor (vlib_main_t * vm,
   /* Update time stamp and flags. */
   n->time_last_updated = vlib_time_now (vm);
   if (is_static)
-    n->flags |= IP6_NEIGHBOR_FLAG_STATIC;
+    {
+      n->flags |= IP6_NEIGHBOR_FLAG_STATIC;
+      n->flags &= ~IP6_NEIGHBOR_FLAG_DYNAMIC;
+    }
   else
-    n->flags |= IP6_NEIGHBOR_FLAG_DYNAMIC;
+    {
+      n->flags |= IP6_NEIGHBOR_FLAG_DYNAMIC;
+      n->flags &= ~IP6_NEIGHBOR_FLAG_STATIC;
+    }
 
   adj_nbr_walk_nh6 (sw_if_index,
 		    &n->key.ip6_address, ip6_nd_mk_complete_walk, n);
@@ -894,26 +921,13 @@ vnet_unset_ip6_ethernet_neighbor (vlib_main_t * vm,
     }
 
   n = pool_elt_at_index (nm->neighbor_pool, p[0]);
-  mhash_unset (&nm->neighbor_index_by_key, &n->key, 0);
 
   adj_nbr_walk_nh6 (sw_if_index,
 		    &n->key.ip6_address, ip6_nd_mk_incomplete_walk, NULL);
+  ip6_neighbor_adj_fib_remove
+    (n, ip6_fib_table_get_index_for_sw_if_index (sw_if_index));
 
-
-  if (FIB_NODE_INDEX_INVALID != n->fib_entry_index)
-    {
-      fib_prefix_t pfx = {
-	.fp_len = 128,
-	.fp_proto = FIB_PROTOCOL_IP6,
-	.fp_addr.ip6 = n->key.ip6_address,
-      };
-      fib_table_entry_path_remove
-	(ip6_fib_table_get_index_for_sw_if_index (n->key.sw_if_index),
-	 &pfx,
-	 FIB_SOURCE_ADJ,
-	 DPO_PROTO_IP6,
-	 &pfx.fp_addr, n->key.sw_if_index, ~0, 1, FIB_ROUTE_PATH_FLAG_NONE);
-    }
+  mhash_unset (&nm->neighbor_index_by_key, &n->key, 0);
   pool_put (nm->neighbor_pool, n);
 
 out:
@@ -1202,11 +1216,6 @@ icmp6_neighbor_solicitation_or_advertisement (vlib_main_t * vm,
 	  if (PREDICT_TRUE (error0 == ICMP6_ERROR_NONE && o0 != 0 &&
 			    !ip6_sadd_unspecified))
 	    {
-	      ip6_neighbor_main_t *nm = &ip6_neighbor_main;
-	      if (nm->limit_neighbor_cache_size &&
-		  pool_elts (nm->neighbor_pool) >=
-		  nm->limit_neighbor_cache_size)
-		unset_random_neighbor_entry ();
 	      vnet_set_ip6_ethernet_neighbor (vm, sw_if_index0,
 					      is_solicitation ?
 					      &ip0->src_address :
@@ -1544,12 +1553,6 @@ icmp6_router_solicitation (vlib_main_t * vm,
 	  if (PREDICT_TRUE (error0 == ICMP6_ERROR_NONE && o0 != 0 &&
 			    !is_unspecified && !is_link_local))
 	    {
-	      ip6_neighbor_main_t *nm = &ip6_neighbor_main;
-	      if (nm->limit_neighbor_cache_size &&
-		  pool_elts (nm->neighbor_pool) >=
-		  nm->limit_neighbor_cache_size)
-		unset_random_neighbor_entry ();
-
 	      vnet_set_ip6_ethernet_neighbor (vm, sw_if_index0,
 					      &ip0->src_address,
 					      o0->ethernet_address,