ip-neighbor: Send API event when neighbor is removed

Type: fix

Signed-off-by: Neale Ranns <neale.ranns@cisco.com>
Change-Id: I9952497a108bac26445af95c28d4eed46099c2fc

9 files changed, 209 insertions, 51 deletions

diff --git a/src/vnet/ip-neighbor/ip_neighbor.api b/src/vnet/ip-neighbor/ip_neighbor.api
index fe344af5c82..62730e7c1e3 100644
--- a/src/vnet/ip-neighbor/ip_neighbor.api
+++ b/src/vnet/ip-neighbor/ip_neighbor.api
@@ -179,8 +179,7 @@ autoreply define ip_neighbor_flush
   vl_api_interface_index_t sw_if_index [default=0xffffffff];
 };
 
-/** \brief Register for IP4 ARP resolution event on receiving ARP reply or
-           MAC/IP info from ARP requests in L2 BDs
+/** \brief Register for IP neighbour events creation
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
     @param enable - 1 => register for events, 0 => cancel registration
@@ -190,6 +189,7 @@ autoreply define ip_neighbor_flush
 */
 autoreply define want_ip_neighbor_events
 {
+  option deprecated;
   u32 client_index;
   u32 context;
   bool enable;
@@ -206,6 +206,7 @@ autoreply define want_ip_neighbor_events
 */
 define ip_neighbor_event
 {
+  option deprecated;
   u32 client_index;
   u32 pid;
   vl_api_ip_neighbor_t neighbor;
@@ -216,6 +217,53 @@ service {
     events ip_neighbor_event;
 };
 
+
+/** \brief Register for IP neighbour events (creation or deletion)
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param enable - 1 => register for events, 0 => cancel registration
+    @param pid - sender's pid
+    @param ip - exact IP address of interested neighbor resolution event
+    @param sw_if_index - interface on which the IP address is present.
+*/
+autoreply define want_ip_neighbor_events_v2
+{
+  u32 client_index;
+  u32 context;
+  bool enable;
+  u32 pid;
+  vl_api_address_t ip;
+  vl_api_interface_index_t sw_if_index [default=0xffffffff];
+};
+
+enum ip_neighbor_event_flags
+{
+ /* The neighbor has been added/learned */
+ IP_NEIGHBOR_API_EVENT_FLAG_ADDED = 0x1,
+ /* The neighbor has been removed/expired */
+ IP_NEIGHBOR_API_EVENT_FLAG_REMOVED = 0x2,
+};
+
+/** \brief Tell client about an IP4 ARP resolution event or
+           MAC/IP info from ARP requests in L2 BDs
+    @param client_index - opaque cookie to identify the sender
+    @param pid - client pid registered to receive notification
+    @param flags - Flags
+    @param neighbor -  neighbor
+*/
+define ip_neighbor_event_v2
+{
+  u32 client_index;
+  u32 pid;
+  vl_api_ip_neighbor_event_flags_t flags;
+  vl_api_ip_neighbor_t neighbor;
+};
+
+service {
+  rpc want_ip_neighbor_events_v2 returns want_ip_neighbor_events_v2_reply
+    events ip_neighbor_event_v2;
+};
+
 /*
  * Local Variables:
  * eval: (c-set-style "gnu")
diff --git a/src/vnet/ip-neighbor/ip_neighbor.c b/src/vnet/ip-neighbor/ip_neighbor.c
index 12ce0d6a271..5786775dc98 100644
--- a/src/vnet/ip-neighbor/ip_neighbor.c
+++ b/src/vnet/ip-neighbor/ip_neighbor.c
@@ -386,7 +386,7 @@ ip_neighbor_mk_incomplete_walk (adj_index_t ai, void *ctx)
 }
 
 static void
-ip_neighbor_free (ip_neighbor_t * ipn)
+ip_neighbor_destroy (ip_neighbor_t * ipn)
 {
   ip_address_family_t af;
 
@@ -395,6 +395,9 @@ ip_neighbor_free (ip_neighbor_t * ipn)
   IP_NEIGHBOR_DBG ("free: %U", format_ip_neighbor,
 		   ip_neighbor_get_index (ipn));
 
+  ip_neighbor_publish (ip_neighbor_get_index (ipn),
+		       IP_NEIGHBOR_EVENT_REMOVED);
+
   adj_nbr_walk_nh (ipn->ipn_key->ipnk_sw_if_index,
 		   ip_address_family_to_fib_proto (af),
 		   &ip_addr_46 (&ipn->ipn_key->ipnk_ip),
@@ -426,7 +429,7 @@ ip_neighbor_force_reuse (ip_address_family_t af)
     return (false);
 
   elt = clib_llist_prev (ip_neighbor_elt_pool, ipne_anchor, head);
-  ip_neighbor_free (ip_neighbor_get (elt->ipne_index));
+  ip_neighbor_destroy (ip_neighbor_get (elt->ipne_index));
 
   return (true);
 }
@@ -556,7 +559,7 @@ ip_neighbor_add (const ip_address_t * ip,
 
 check_customers:
   /* Customer(s) requesting event for this address? */
-  ip_neighbor_publish (ip_neighbor_get_index (ipn));
+  ip_neighbor_publish (ip_neighbor_get_index (ipn), IP_NEIGHBOR_EVENT_ADDED);
 
   if (stats_index)
     *stats_index = adj_nbr_find (fproto,
@@ -588,7 +591,7 @@ ip_neighbor_del (const ip_address_t * ip, u32 sw_if_index)
   if (NULL == ipn)
     return (VNET_API_ERROR_NO_SUCH_ENTRY);
 
-  ip_neighbor_free (ipn);
+  ip_neighbor_destroy (ipn);
 
   return (0);
 }
@@ -623,7 +626,8 @@ ip_neighbor_del_all (ip_address_family_t af, u32 sw_if_index)
 
   ip_neighbor_walk (af, sw_if_index, ip_neighbor_del_all_walk_cb, &ctx);
 
-  vec_foreach (ipni, ctx.ipn_del) ip_neighbor_free (ip_neighbor_get (*ipni));
+  vec_foreach (ipni,
+	       ctx.ipn_del) ip_neighbor_destroy (ip_neighbor_get (*ipni));
   vec_free (ctx.ipn_del);
 }
 
@@ -1220,7 +1224,7 @@ ip_neighbor_flush (ip_address_family_t af, u32 sw_if_index)
   }));
   /* *INDENT-ON* */
 
-  vec_foreach (ipni, ipnis) ip_neighbor_free (ip_neighbor_get (*ipni));
+  vec_foreach (ipni, ipnis) ip_neighbor_destroy (ip_neighbor_get (*ipni));
   vec_free (ipnis);
 }
 
@@ -1273,7 +1277,7 @@ ip_neighbor_sweep (ip_address_family_t af)
 
   vec_foreach (ipni, ctx.ipnsc_stale)
   {
-    ip_neighbor_free (ip_neighbor_get (*ipni));
+    ip_neighbor_destroy (ip_neighbor_get (*ipni));
   }
   vec_free (ctx.ipnsc_stale);
 }
@@ -1407,7 +1411,7 @@ ip_neighbor_add_del_interface_address_v4 (ip4_main_t * im,
       ip_neighbor_walk (AF_IP4, sw_if_index, ip_neighbor_walk_covered, &ctx);
 
       vec_foreach (ipni, ctx.ipnis)
-	ip_neighbor_free (ip_neighbor_get (*ipni));
+	ip_neighbor_destroy (ip_neighbor_get (*ipni));
 
       vec_free (ctx.ipnis);
     }
@@ -1449,7 +1453,7 @@ ip_neighbor_add_del_interface_address_v6 (ip6_main_t * im,
       ip_neighbor_walk (AF_IP6, sw_if_index, ip_neighbor_walk_covered, &ctx);
 
       vec_foreach (ipni, ctx.ipnis)
-	ip_neighbor_free (ip_neighbor_get (*ipni));
+	ip_neighbor_destroy (ip_neighbor_get (*ipni));
 
       vec_free (ctx.ipnis);
     }
@@ -1623,7 +1627,7 @@ ip_neighbor_age_loop (vlib_main_t * vm,
             else if (IP_NEIGHBOR_AGE_DEAD == res) {
               /* the oldest neighbor is dead, pop it, then restart the walk
                * again from the back */
-              ip_neighbor_free (ip_neighbor_get(elt->ipne_index));
+              ip_neighbor_destroy (ip_neighbor_get(elt->ipne_index));
               goto restart;
             }
 
diff --git a/src/vnet/ip-neighbor/ip_neighbor.h b/src/vnet/ip-neighbor/ip_neighbor.h
index 46194796c31..419c49491a3 100644
--- a/src/vnet/ip-neighbor/ip_neighbor.h
+++ b/src/vnet/ip-neighbor/ip_neighbor.h
@@ -22,8 +22,6 @@
 
 #include <vnet/adj/adj.h>
 
-void ip_neighbor_scan_enable_disable (ip_neighbor_scan_arg_t * arg);
-
 
 /*****
  * APIs external modules can invoke on the neighbor subsystem
@@ -65,7 +63,7 @@ extern void ip_neighbor_sweep (ip_address_family_t af);
 /**
  * From the watcher to the API to publish a new neighbor
  */
-extern void ip_neighbor_handle_event (const ip_neighbor_event_t * ipne);
+extern void ip_neighbor_handle_event (ip_neighbor_event_t * ipne);
 
 /**
  * The set of function that vnet requires from the IP neighbour module.
diff --git a/src/vnet/ip-neighbor/ip_neighbor_api.c b/src/vnet/ip-neighbor/ip_neighbor_api.c
index 39d77855e9f..81af86211de 100644
--- a/src/vnet/ip-neighbor/ip_neighbor_api.c
+++ b/src/vnet/ip-neighbor/ip_neighbor_api.c
@@ -61,13 +61,12 @@ ip_neighbor_encode (vl_api_ip_neighbor_t * api, const ip_neighbor_t * ipn)
 }
 
 void
-ip_neighbor_handle_event (const ip_neighbor_event_t * ipne)
+ip_neighbor_handle_event (ip_neighbor_event_t * ipne)
 {
-  vl_api_ip_neighbor_event_t *mp;
   vl_api_registration_t *reg;
-  const ip_neighbor_t *ipn;
+  ip_neighbor_t *ipn;
 
-  ipn = ip_neighbor_get (ipne->ipne_index);
+  ipn = &ipne->ipne_nbr;
 
   if (NULL == ipn)
     /* Client can cancel, die, etc. */
@@ -80,15 +79,37 @@ ip_neighbor_handle_event (const ip_neighbor_event_t * ipne)
 
   if (vl_api_can_send_msg (reg))
     {
-      mp = vl_msg_api_alloc (sizeof (*mp));
-      clib_memset (mp, 0, sizeof (*mp));
-      mp->_vl_msg_id = ntohs (VL_API_IP_NEIGHBOR_EVENT + REPLY_MSG_ID_BASE);
-      mp->client_index = ipne->ipne_watch.ipw_client;
-      mp->pid = ipne->ipne_watch.ipw_pid;
+      if (1 == ipne->ipne_watch.ipw_api_version)
+	{
+	  vl_api_ip_neighbor_event_t *mp;
+
+	  mp = vl_msg_api_alloc (sizeof (*mp));
+	  clib_memset (mp, 0, sizeof (*mp));
+	  mp->_vl_msg_id =
+	    ntohs (VL_API_IP_NEIGHBOR_EVENT + REPLY_MSG_ID_BASE);
+	  mp->client_index = ipne->ipne_watch.ipw_client;
+	  mp->pid = ipne->ipne_watch.ipw_pid;
 
-      ip_neighbor_encode (&mp->neighbor, ipn);
+	  ip_neighbor_encode (&mp->neighbor, ipn);
 
-      vl_api_send_msg (reg, (u8 *) mp);
+	  vl_api_send_msg (reg, (u8 *) mp);
+	}
+      else if (2 == ipne->ipne_watch.ipw_api_version)
+	{
+	  vl_api_ip_neighbor_event_v2_t *mp;
+
+	  mp = vl_msg_api_alloc (sizeof (*mp));
+	  clib_memset (mp, 0, sizeof (*mp));
+	  mp->_vl_msg_id =
+	    ntohs (VL_API_IP_NEIGHBOR_EVENT_V2 + REPLY_MSG_ID_BASE);
+	  mp->client_index = ipne->ipne_watch.ipw_client;
+	  mp->pid = ipne->ipne_watch.ipw_pid;
+	  mp->flags = clib_host_to_net_u32 (ipne->ipne_flags);
+
+	  ip_neighbor_encode (&mp->neighbor, ipn);
+
+	  vl_api_send_msg (reg, (u8 *) mp);
+	}
     }
   else
     {
@@ -99,12 +120,14 @@ ip_neighbor_handle_event (const ip_neighbor_event_t * ipne)
        */
       if (vlib_time_now (vlib_get_main ()) > last_time + 10.0)
 	{
-	  clib_warning ("ip6 nd event for %U to pid %d: queue stuffed!",
+	  clib_warning ("neighbor event for %U to pid %d: queue stuffed!",
 			format_ip_address, &ipn->ipn_key->ipnk_ip,
 			ipne->ipne_watch.ipw_pid);
 	  last_time = vlib_time_now (vlib_get_main ());
 	}
     }
+
+  ip_neighbor_free (ipn);
 }
 
 typedef struct ip_neighbor_dump_ctx_t_
@@ -234,6 +257,7 @@ vl_api_want_ip_neighbor_events_t_handler (vl_api_want_ip_neighbor_events_t *
   ip_neighbor_watcher_t watch = {
     .ipw_client = mp->client_index,
     .ipw_pid = mp->pid,
+    .ipw_api_version = 1,
   };
 
   if (mp->enable)
@@ -246,6 +270,33 @@ vl_api_want_ip_neighbor_events_t_handler (vl_api_want_ip_neighbor_events_t *
 }
 
 static void
+  vl_api_want_ip_neighbor_events_v2_t_handler
+  (vl_api_want_ip_neighbor_events_v2_t * mp)
+{
+  vl_api_want_ip_neighbor_events_reply_t *rmp;
+  ip_address_t ip;
+  int rv = 0;
+
+  if (mp->sw_if_index != ~0)
+    VALIDATE_SW_IF_INDEX (mp);
+  ip_address_decode2 (&mp->ip, &ip);
+
+  ip_neighbor_watcher_t watch = {
+    .ipw_client = mp->client_index,
+    .ipw_pid = mp->pid,
+    .ipw_api_version = 2,
+  };
+
+  if (mp->enable)
+    ip_neighbor_watch (&ip, ntohl (mp->sw_if_index), &watch);
+  else
+    ip_neighbor_unwatch (&ip, ntohl (mp->sw_if_index), &watch);
+
+  BAD_SW_IF_INDEX_LABEL;
+  REPLY_MACRO (VL_API_WANT_IP_NEIGHBOR_EVENTS_V2_REPLY);
+}
+
+static void
 vl_api_ip_neighbor_config_t_handler (vl_api_ip_neighbor_config_t * mp)
 {
   vl_api_ip_neighbor_config_reply_t *rmp;
diff --git a/src/vnet/ip-neighbor/ip_neighbor_types.c b/src/vnet/ip-neighbor/ip_neighbor_types.c
index c18e4948c43..76fbc5ac8a9 100644
--- a/src/vnet/ip-neighbor/ip_neighbor_types.c
+++ b/src/vnet/ip-neighbor/ip_neighbor_types.c
@@ -17,6 +17,21 @@
 
 #include <vnet/ip-neighbor/ip_neighbor_types.h>
 
+void
+ip_neighbor_clone (const ip_neighbor_t * ipn, ip_neighbor_t * clone)
+{
+  clib_memcpy (clone, ipn, sizeof (*ipn));
+
+  clone->ipn_key = clib_mem_alloc (sizeof (ip_neighbor_key_t));
+  clib_memcpy (clone->ipn_key, ipn->ipn_key, sizeof (ip_neighbor_key_t));
+}
+
+void
+ip_neighbor_free (ip_neighbor_t * ipn)
+{
+  clib_mem_free (ipn->ipn_key);
+}
+
 u8 *
 format_ip_neighbor_flags (u8 * s, va_list * args)
 {
diff --git a/src/vnet/ip-neighbor/ip_neighbor_types.h b/src/vnet/ip-neighbor/ip_neighbor_types.h
index 8dfc438e7ad..2eb8fd0841f 100644
--- a/src/vnet/ip-neighbor/ip_neighbor_types.h
+++ b/src/vnet/ip-neighbor/ip_neighbor_types.h
@@ -22,21 +22,6 @@
 #include <vnet/ethernet/mac_address.h>
 #include <vnet/fib/fib_types.h>
 
-#define IP_SCAN_DISABLED	0
-#define IP_SCAN_V4_NEIGHBORS	(1 << 0)
-#define IP_SCAN_V6_NEIGHBORS	(1 << 1)
-#define IP_SCAN_V46_NEIGHBORS	(IP_SCAN_V4_NEIGHBORS | IP_SCAN_V6_NEIGHBORS)
-
-typedef struct
-{
-  u8 mode;			/* 0: disable, 1: ip4, 2: ip6, 3: both */
-  u8 scan_interval;		/* neighbor scan interval in minutes */
-  u8 max_proc_time;		/* max processing time per run, in usecs */
-  u8 max_update;		/* max probe/delete operations per run */
-  u8 scan_int_delay;		/* delay in msecs, to resume scan on max */
-  u8 stale_threshold;		/* Threshold in minutes to delete nei entry */
-} ip_neighbor_scan_arg_t;
-
 #define foreach_ip_neighbor_flag                 \
   _(STATIC, 1 << 0, "static", "S")               \
   _(DYNAMIC, 1 << 1, "dynamic", "D")             \
@@ -50,12 +35,13 @@ typedef enum ip_neighbor_flags_t_
 #define _(a,b,c,d) IP_NEIGHBOR_FLAG_##a = b,
   foreach_ip_neighbor_flag
 #undef _
-} __attribute__ ((packed)) ip_neighbor_flags_t;
+} __clib_packed ip_neighbor_flags_t;
 
 typedef struct ip_neighbor_watcher_t_
 {
   u32 ipw_pid;
   u32 ipw_client;
+  int ipw_api_version;
 } ip_neighbor_watcher_t;
 
 extern u8 *format_ip_neighbor_watcher (u8 * s, va_list * args);
@@ -115,12 +101,27 @@ typedef struct ip_neighbor_learn_t_
   u32 sw_if_index;
 } ip_neighbor_learn_t;
 
+
+typedef enum ip_neighbor_event_flags_t_
+{
+  IP_NEIGHBOR_EVENT_ADDED = (1 << 0),
+  IP_NEIGHBOR_EVENT_REMOVED = (1 << 1),
+} ip_neighbor_event_flags_t;
+
 typedef struct ip_neighbor_event_t_
 {
   ip_neighbor_watcher_t ipne_watch;
-  index_t ipne_index;
+  ip_neighbor_event_flags_t ipne_flags;
+  ip_neighbor_t ipne_nbr;
 } ip_neighbor_event_t;
 
+extern void ip_neighbor_clone (const ip_neighbor_t * ipn,
+			       ip_neighbor_t * clone);
+
+extern void ip_neighbor_free (ip_neighbor_t * ipn);
+
+
+
 #endif /* __INCLUDE_IP_NEIGHBOR_H__ */
 
 /*
diff --git a/src/vnet/ip-neighbor/ip_neighbor_watch.c b/src/vnet/ip-neighbor/ip_neighbor_watch.c
index 71d6440641d..72908f4e613 100644
--- a/src/vnet/ip-neighbor/ip_neighbor_watch.c
+++ b/src/vnet/ip-neighbor/ip_neighbor_watch.c
@@ -16,6 +16,7 @@
  */
 
 #include <vnet/ip-neighbor/ip_neighbor.h>
+#include <vnet/ip-neighbor/ip_neighbor_watch.h>
 #include <vnet/ip/ip_types_api.h>
 #include <vnet/ethernet/ethernet_types_api.h>
 
@@ -174,7 +175,9 @@ ip_neighbor_unwatch (const ip_address_t * ip,
 }
 
 static void
-ip_neighbor_signal (ip_neighbor_watcher_t *watchers, index_t ipni)
+ip_neighbor_signal (ip_neighbor_watcher_t *watchers,
+                    index_t ipni,
+                    ip_neighbor_event_flags_t flags)
 {
   ip_neighbor_watcher_t *watcher;
 
@@ -185,12 +188,14 @@ ip_neighbor_signal (ip_neighbor_watcher_t *watchers, index_t ipni)
                                            ip_neighbor_event_process_node.index,
                                            0, 1, sizeof(*ipne));
     ipne->ipne_watch = *watcher;
-    ipne->ipne_index = ipni;
+    ipne->ipne_flags = flags;
+    ip_neighbor_clone(ip_neighbor_get(ipni), &ipne->ipne_nbr);
   }
 }
 
 void
-ip_neighbor_publish (index_t ipni)
+ip_neighbor_publish (index_t ipni,
+                     ip_neighbor_event_flags_t flags)
 {
   const ip_neighbor_t *ipn;
   ip_neighbor_key_t key;
@@ -204,21 +209,21 @@ ip_neighbor_publish (index_t ipni)
   p = mhash_get (&ipnw_db.ipnwdb_hash, &key);
 
   if (p) {
-    ip_neighbor_signal ((ip_neighbor_watcher_t*) p[0], ipni);
+    ip_neighbor_signal ((ip_neighbor_watcher_t*) p[0], ipni, flags);
   }
 
   ip_address_reset (&key.ipnk_ip);
   p = mhash_get (&ipnw_db.ipnwdb_hash, &key);
 
   if (p) {
-    ip_neighbor_signal ((ip_neighbor_watcher_t*) p[0], ipni);
+    ip_neighbor_signal ((ip_neighbor_watcher_t*) p[0], ipni, flags);
   }
 
   key.ipnk_sw_if_index = ~0;
   p = mhash_get (&ipnw_db.ipnwdb_hash, &key);
 
   if (p) {
-    ip_neighbor_signal ((ip_neighbor_watcher_t*) p[0], ipni);
+    ip_neighbor_signal ((ip_neighbor_watcher_t*) p[0], ipni, flags);
   }
 }
 
diff --git a/src/vnet/ip-neighbor/ip_neighbor_watch.h b/src/vnet/ip-neighbor/ip_neighbor_watch.h
index fc190b5423c..3ab30321ab4 100644
--- a/src/vnet/ip-neighbor/ip_neighbor_watch.h
+++ b/src/vnet/ip-neighbor/ip_neighbor_watch.h
@@ -27,7 +27,8 @@ extern void ip_neighbor_unwatch (const ip_address_t * ip,
 				 u32 sw_if_index,
 				 const ip_neighbor_watcher_t * watch);
 
-extern void ip_neighbor_publish (index_t ipni);
+extern void ip_neighbor_publish (index_t ipni,
+				 ip_neighbor_event_flags_t flags);
 
 #endif
 
diff --git a/test/test_neighbor.py b/test/test_neighbor.py
index d81fe1b3f80..2dc27a83b0a 100644
--- a/test/test_neighbor.py
+++ b/test/test_neighbor.py
@@ -1991,16 +1991,51 @@ class NeighborAgeTestCase(VppTestCase):
         #
         # load up some neighbours again with 2s aging enabled
         # they should be removed after 10s (2s age + 4s for probes + gap)
+        # check for the add and remove events
         #
+        enum = VppEnum.vl_api_ip_neighbor_event_flags_t
+
+        self.vapi.want_ip_neighbor_events_v2(enable=1)
         for ii in range(10):
             VppNeighbor(self,
                         self.pg0.sw_if_index,
                         self.pg0.remote_hosts[ii].mac,
                         self.pg0.remote_hosts[ii].ip4).add_vpp_config()
+
+            e = self.vapi.wait_for_event(1, "ip_neighbor_event_v2")
+            self.assertEqual(e.flags,
+                             enum.IP_NEIGHBOR_API_EVENT_FLAG_ADDED)
+            self.assertEqual(str(e.neighbor.ip_address),
+                             self.pg0.remote_hosts[ii].ip4)
+            self.assertEqual(e.neighbor.mac_address,
+                             self.pg0.remote_hosts[ii].mac)
+
         self.sleep(10)
         self.assertFalse(self.vapi.ip_neighbor_dump(sw_if_index=0xffffffff,
                                                     af=vaf.ADDRESS_IP4))
 
+        evs = []
+        for ii in range(10):
+            e = self.vapi.wait_for_event(1, "ip_neighbor_event_v2")
+            self.assertEqual(e.flags,
+                             enum.IP_NEIGHBOR_API_EVENT_FLAG_REMOVED)
+            evs.append(e)
+
+        # check we got the correct mac/ip pairs - done separately
+        # because we don't care about the order the remove notifications
+        # arrive
+        for ii in range(10):
+            found = False
+            mac = self.pg0.remote_hosts[ii].mac
+            ip = self.pg0.remote_hosts[ii].ip4
+
+            for e in evs:
+                if (e.neighbor.mac_address == mac and
+                   str(e.neighbor.ip_address) == ip):
+                    found = True
+                    break
+            self.assertTrue(found)
+
         #
         # check if we can set age and recycle with empty neighbor list
         #