Add support for API client to receive L2 MAC events

Added APIs want_l2_macs_events and l2_macs_event to allow an API
client to receive notification events from VPP for MAC learned
or aged in L2FIB. Only one API client is allowed for L2 MAC events.

The want_l2_macs_events API allow caller to specify MAC learn
limit, event scan delay and max number of MACs that can be included
in a event message. These parameters should be choosen properly as
to not have too many MAC events sent by VPP and overwhelm the API
share memory. They can all be left as 0's so VPP will setup reasonable
defaults which are: 1000 learn limit, 100 msec scan delay and 100
MACs per event message.

If want_l2_macs_events is never called, VPP learning and aging
should behave as before except that MAC entries provisioned by API
or CLI will not be aged, even if it is not set as static_mac. These
non static MACs, however, can be overwritten by MAC learning on a
MAC move as a leared MAC. Only learned MACs are subject to aging.

Change-Id: Ia3757a80cf8adb2811a089d2eafbd6439461285c
Signed-off-by: John Lo <loj@cisco.com>

1 files changed, 294 insertions, 100 deletions

diff --git a/src/vnet/l2/l2_fib.c b/src/vnet/l2/l2_fib.c
index 7e59b09848a..8aa0ac295dd 100644
--- a/src/vnet/l2/l2_fib.c
+++ b/src/vnet/l2/l2_fib.c
@@ -31,6 +31,17 @@
 
 #include <vppinfra/bihash_template.c>
 
+#include <vlibmemory/api.h>
+#include <vnet/vnet_msg_enum.h>
+
+#define vl_typedefs		/* define message structures */
+#include <vnet/vnet_all_api_h.h>
+#undef vl_typedefs
+
+#define vl_endianfun		/* define message structures */
+#include <vnet/vnet_all_api_h.h>
+#undef vl_endianfun
+
 /**
  * @file
  * @brief Ethernet MAC Address FIB Table Management.
@@ -117,6 +128,7 @@ show_l2fib (vlib_main_t * vm,
   int i, j, k;
   u8 verbose = 0;
   u8 raw = 0;
+  u8 learn = 0;
   u32 bd_id, bd_index = ~0;
   u8 now = (u8) (vlib_time_now (vm) / 60);
   u8 *s = 0;
@@ -127,12 +139,18 @@ show_l2fib (vlib_main_t * vm,
     verbose = 1;
   else if (unformat (input, "bd_index %d", &bd_index))
     verbose = 1;
+  else if (unformat (input, "learn"))
+    {
+      learn = 1;
+      verbose = 0;
+    }
   else if (unformat (input, "bd_id %d", &bd_id))
     {
       uword *p = hash_get (bdm->bd_index_by_bd_id, bd_id);
       if (p)
 	{
-	  verbose = 1;
+	  if (learn == 0)
+	    verbose = 1;
 	  bd_index = p[0];
 	}
       else
@@ -155,7 +173,7 @@ show_l2fib (vlib_main_t * vm,
 	      if (v->kvp[k].key == ~0ULL && v->kvp[k].value == ~0ULL)
 		continue;
 
-	      if (verbose && first_entry)
+	      if ((verbose || learn) && first_entry)
 		{
 		  first_entry = 0;
 		  vlib_cli_output (vm,
@@ -168,13 +186,19 @@ show_l2fib (vlib_main_t * vm,
 	      key.raw = v->kvp[k].key;
 	      result.raw = v->kvp[k].value;
 
-	      if (verbose
+	      if ((verbose || learn)
 		  & ((bd_index >> 31) || (bd_index == key.fields.bd_index)))
 		{
+		  if (learn && result.fields.age_not)
+		    {
+		      total_entries++;
+		      continue;	/* skip provisioned macs */
+		    }
+
 		  bd_config = vec_elt_at_index (l2input_main.bd_configs,
 						key.fields.bd_index);
 
-		  if (bd_config->mac_age && !result.fields.static_mac)
+		  if (bd_config->mac_age && !result.fields.age_not)
 		    {
 		      i16 delta = now - result.fields.timestamp;
 		      delta += delta < 0 ? 256 : 0;
@@ -206,9 +230,19 @@ show_l2fib (vlib_main_t * vm,
   if (total_entries == 0)
     vlib_cli_output (vm, "no l2fib entries");
   else
-    vlib_cli_output (vm,
-		     "%lld l2fib entries with %d learned (or non-static) entries",
-		     total_entries, l2learn_main.global_learn_count);
+    {
+      l2learn_main_t *lm = &l2learn_main;
+      vlib_cli_output (vm, "L2FIB total/learned entries: %d/%d  "
+		       "Last scan time: %.4esec  Learn limit: %d ",
+		       total_entries, lm->global_learn_count,
+		       msm->age_scan_duration, lm->global_learn_limit);
+      if (lm->client_pid)
+	vlib_cli_output (vm, "L2MAC events client PID: %d  "
+			 "Last e-scan time: %.4esec  Delay: %.2esec  "
+			 "Max macs in event: %d",
+			 lm->client_pid, msm->evt_scan_duration,
+			 msm->event_scan_delay, msm->max_macs_in_event);
+    }
 
   if (raw)
     vlib_cli_output (vm, "Raw Hash Table:\n%U\n",
@@ -242,7 +276,7 @@ show_l2fib (vlib_main_t * vm,
 /* *INDENT-OFF* */
 VLIB_CLI_COMMAND (show_l2fib_cli, static) = {
   .path = "show l2fib",
-  .short_help = "show l2fib [verbose | bd_id <nn> | bd_index <nn> | raw]",
+  .short_help = "show l2fib [verbose | learn | bd_id <nn> | bd_index <nn> | raw",
   .function = show_l2fib,
 };
 /* *INDENT-ON* */
@@ -309,36 +343,39 @@ l2fib_cur_seq_num (u32 bd_index, u32 sw_if_index)
  */
 void
 l2fib_add_entry (u64 mac, u32 bd_index,
-		 u32 sw_if_index, u32 static_mac, u32 filter_mac, u32 bvi_mac)
+		 u32 sw_if_index, u8 static_mac, u8 filter_mac, u8 bvi_mac)
 {
   l2fib_entry_key_t key;
   l2fib_entry_result_t result;
   __attribute__ ((unused)) u32 bucket_contents;
-  l2fib_main_t *mp = &l2fib_main;
+  l2fib_main_t *fm = &l2fib_main;
+  l2learn_main_t *lm = &l2learn_main;
   BVT (clib_bihash_kv) kv;
 
   /* set up key */
   key.raw = l2fib_make_key ((u8 *) & mac, bd_index);
 
+  /* check if entry alread exist */
+  if (BV (clib_bihash_search) (&fm->mac_table, &kv, &kv))
+    {
+      /* decrement counter if overwriting a learned mac  */
+      result.raw = kv.value;
+      if ((result.fields.age_not == 0) && (lm->global_learn_count))
+	lm->global_learn_count--;
+    }
+
   /* set up result */
   result.raw = 0;		/* clear all fields */
   result.fields.sw_if_index = sw_if_index;
   result.fields.static_mac = static_mac;
   result.fields.filter = filter_mac;
   result.fields.bvi = bvi_mac;
-  if (!static_mac)
-    result.fields.sn = l2fib_cur_seq_num (bd_index, sw_if_index);
+  result.fields.age_not = 1;	/* no aging for provisioned entry */
 
   kv.key = key.raw;
   kv.value = result.raw;
 
-  BV (clib_bihash_add_del) (&mp->mac_table, &kv, 1 /* is_add */ );
-
-  /* increment counter if dynamically learned mac */
-  if (result.fields.static_mac == 0)
-    {
-      l2learn_main.global_learn_count++;
-    }
+  BV (clib_bihash_add_del) (&fm->mac_table, &kv, 1 /* is_add */ );
 }
 
 /**
@@ -630,13 +667,8 @@ l2fib_del_entry_by_key (u64 raw_key)
   result.raw = kv.value;
 
   /* decrement counter if dynamically learned mac */
-  if (result.fields.static_mac == 0)
-    {
-      if (l2learn_main.global_learn_count > 0)
-	{
-	  l2learn_main.global_learn_count--;
-	}
-    }
+  if ((result.fields.age_not == 0) && (l2learn_main.global_learn_count))
+    l2learn_main.global_learn_count--;
 
   /* Remove entry from hash table */
   BV (clib_bihash_add_del) (&mp->mac_table, &kv, 0 /* is_add */ );
@@ -910,111 +942,273 @@ BVT (clib_bihash) * get_mac_table (void)
   return &mp->mac_table;
 }
 
+static_always_inline void *
+allocate_mac_evt_buf (u32 client, u32 client_index)
+{
+  l2fib_main_t *fm = &l2fib_main;
+  vl_api_l2_macs_event_t *mp = vl_msg_api_alloc
+    (sizeof (*mp) + (fm->max_macs_in_event * sizeof (vl_api_mac_entry_t)));
+  mp->_vl_msg_id = htons (VL_API_L2_MACS_EVENT);
+  mp->pid = htonl (client);
+  mp->client_index = client_index;
+  return mp;
+}
+
+static_always_inline f64
+l2fib_scan (vlib_main_t * vm, f64 start_time, u8 event_only)
+{
+  l2fib_main_t *fm = &l2fib_main;
+  l2learn_main_t *lm = &l2learn_main;
+
+  BVT (clib_bihash) * h = &fm->mac_table;
+  int i, j, k;
+  f64 last_start = start_time;
+  f64 accum_t = 0;
+  f64 delta_t = 0;
+  u32 evt_idx = 0;
+  u32 learn_count = 0;
+  u32 client = lm->client_pid;
+  u32 cl_idx = lm->client_index;
+  vl_api_l2_macs_event_t *mp = 0;
+  unix_shared_memory_queue_t *q = 0;
+
+  if (client)
+    {
+      mp = allocate_mac_evt_buf (client, cl_idx);
+      q = vl_api_client_index_to_input_queue (lm->client_index);
+    }
+
+  for (i = 0; i < h->nbuckets; i++)
+    {
+      /* allow no more than 20us without a pause */
+      delta_t = vlib_time_now (vm) - last_start;
+      if (delta_t > 20e-6)
+	{
+	  vlib_process_suspend (vm, 100e-6);	/* suspend for 100 us */
+	  last_start = vlib_time_now (vm);
+	  accum_t += delta_t;
+	}
+
+      if (i < (h->nbuckets - 3))
+	{
+	  BVT (clib_bihash_bucket) * b = &h->buckets[i + 3];
+	  CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
+	  b = &h->buckets[i + 1];
+	  if (b->offset)
+	    {
+	      BVT (clib_bihash_value) * v =
+		BV (clib_bihash_get_value) (h, b->offset);
+	      CLIB_PREFETCH (v, CLIB_CACHE_LINE_BYTES, LOAD);
+	    }
+	}
+
+      BVT (clib_bihash_bucket) * b = &h->buckets[i];
+      if (b->offset == 0)
+	continue;
+      BVT (clib_bihash_value) * v = BV (clib_bihash_get_value) (h, b->offset);
+      for (j = 0; j < (1 << b->log2_pages); j++)
+	{
+	  for (k = 0; k < BIHASH_KVP_PER_PAGE; k++)
+	    {
+	      if (v->kvp[k].key == ~0ULL && v->kvp[k].value == ~0ULL)
+		continue;
+
+	      l2fib_entry_key_t key = {.raw = v->kvp[k].key };
+	      l2fib_entry_result_t result = {.raw = v->kvp[k].value };
+
+	      if (result.fields.age_not == 0)
+		learn_count++;
+
+	      if (PREDICT_FALSE (evt_idx >= fm->max_macs_in_event))
+		{
+		  /* evet message full, sent it and start a new one */
+		  if (q && (q->cursize < q->maxsize))
+		    {
+		      mp->n_macs = htonl (evt_idx);
+		      vl_msg_api_send_shmem (q, (u8 *) & mp);
+		      mp = allocate_mac_evt_buf (client, cl_idx);
+		    }
+		  else
+		    {
+		      clib_warning ("MAC event to pid %d queue stuffed!"
+				    " %d MAC entries lost", client, evt_idx);
+		    }
+		  evt_idx = 0;
+		}
+
+	      if (client)
+		{
+		  if (result.fields.lrn_evt)
+		    {
+		      /* copy mac entry to event msg */
+		      clib_memcpy (mp->mac[evt_idx].mac_addr, key.fields.mac,
+				   6);
+		      mp->mac[evt_idx].is_del = 0;
+		      mp->mac[evt_idx].sw_if_index =
+			htonl (result.fields.sw_if_index);
+		      /* clear event bit and update mac entry */
+		      result.fields.lrn_evt = 0;
+		      BVT (clib_bihash_kv) kv;
+		      kv.key = key.raw;
+		      kv.value = result.raw;
+		      BV (clib_bihash_add_del) (&fm->mac_table, &kv, 1);
+		      evt_idx++;
+		      continue;	/* skip aging */
+		    }
+		}
+
+	      if (event_only || result.fields.age_not)
+		continue;	/* skip aging - static_mac alsways age_not */
+
+	      /* start aging processing */
+	      u32 bd_index = key.fields.bd_index;
+	      u32 sw_if_index = result.fields.sw_if_index;
+	      u16 sn = l2fib_cur_seq_num (bd_index, sw_if_index).as_u16;
+	      if (result.fields.sn.as_u16 != sn)
+		goto age_out;	/* stale mac */
+
+	      l2_bridge_domain_t *bd_config =
+		vec_elt_at_index (l2input_main.bd_configs, bd_index);
+
+	      if (bd_config->mac_age == 0)
+		continue;	/* skip aging */
+
+	      i16 delta = (u8) (start_time / 60) - result.fields.timestamp;
+	      delta += delta < 0 ? 256 : 0;
+
+	      if (delta < bd_config->mac_age)
+		continue;	/* still valid */
+
+	    age_out:
+	      if (client)
+		{
+		  /* copy mac entry to event msg */
+		  clib_memcpy (mp->mac[evt_idx].mac_addr, key.fields.mac, 6);
+		  mp->mac[evt_idx].is_del = 1;
+		  mp->mac[evt_idx].sw_if_index =
+		    htonl (result.fields.sw_if_index);
+		  evt_idx++;
+		}
+	      /* delete mac entry */
+	      BVT (clib_bihash_kv) kv;
+	      kv.key = key.raw;
+	      BV (clib_bihash_add_del) (&fm->mac_table, &kv, 0);
+	      learn_count--;
+	    }
+	  v++;
+	}
+    }
+
+  /* keep learn count consistent */
+  l2learn_main.global_learn_count = learn_count;
+
+  if (mp)
+    {
+      /*  send any outstanding mac event message else free message buffer */
+      if (evt_idx)
+	{
+	  if (q && (q->cursize < q->maxsize))
+	    {
+	      mp->n_macs = htonl (evt_idx);
+	      vl_msg_api_send_shmem (q, (u8 *) & mp);
+	    }
+	  else
+	    {
+	      clib_warning ("MAC event to pid %d queue stuffed!"
+			    " %d MAC entries lost", client, evt_idx);
+	      vl_msg_api_free (mp);
+	    }
+	}
+      else
+	vl_msg_api_free (mp);
+    }
+  return delta_t + accum_t;
+}
+
+/* Type of scan */
+#define SCAN_MAC_AGE   0
+#define SCAN_MAC_EVENT 1
+
+/* Maximum f64 value */
+#define TIME_MAX (1.7976931348623157e+308)
+
 static uword
 l2fib_mac_age_scanner_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
 			       vlib_frame_t * f)
 {
   uword event_type, *event_data = 0;
-  l2fib_main_t *msm = &l2fib_main;
+  l2fib_main_t *fm = &l2fib_main;
+  l2learn_main_t *lm = &l2learn_main;
   bool enabled = 0;
-  f64 start_time, last_run_duration = 0, t;
+  bool scan = SCAN_MAC_AGE;	/* SCAN_FOR_AGE or SCAN_FOR_EVENT */
+  f64 start_time, next_age_scan_time = TIME_MAX;
 
   while (1)
     {
       if (enabled)
-	vlib_process_wait_for_event_or_clock (vm, 60 - last_run_duration);
+	{
+	  if (lm->client_pid)	/* mac event client waiting */
+	    vlib_process_wait_for_event_or_clock (vm, fm->event_scan_delay);
+	  else			/* agin only */
+	    {
+	      f64 t = next_age_scan_time - vlib_time_now (vm);
+	      if (t < fm->event_scan_delay)
+		t = fm->event_scan_delay;
+	      vlib_process_wait_for_event_or_clock (vm, t);
+	    }
+	}
       else
 	vlib_process_wait_for_event (vm);
 
       event_type = vlib_process_get_events (vm, &event_data);
       vec_reset_length (event_data);
 
+      start_time = vlib_time_now (vm);
+
       switch (event_type)
 	{
-	case ~0:
+	case ~0:		/* timer expired */
+	  if ((lm->client_pid == 0) || (start_time >= next_age_scan_time))
+	    {
+	      scan = SCAN_MAC_AGE;
+	      if (enabled)
+		next_age_scan_time = start_time + L2FIB_AGE_SCAN_INTERVAL;
+	      else
+		next_age_scan_time = TIME_MAX;
+	    }
+	  else
+	    scan = SCAN_MAC_EVENT;
 	  break;
+
 	case L2_MAC_AGE_PROCESS_EVENT_START:
+	  scan = SCAN_MAC_AGE;
+	  next_age_scan_time = start_time + L2FIB_AGE_SCAN_INTERVAL;
 	  enabled = 1;
 	  break;
+
 	case L2_MAC_AGE_PROCESS_EVENT_STOP:
 	  enabled = 0;
+	  next_age_scan_time = TIME_MAX;
+	  l2fib_main.age_scan_duration = 0;
+	  l2fib_main.evt_scan_duration = 0;
 	  continue;
+
 	case L2_MAC_AGE_PROCESS_EVENT_ONE_PASS:
-	  enabled = 0;
+	  scan = SCAN_MAC_AGE;
+	  if (enabled)
+	    next_age_scan_time = start_time + L2FIB_AGE_SCAN_INTERVAL;
+	  else
+	    next_age_scan_time = TIME_MAX;
 	  break;
+
 	default:
 	  ASSERT (0);
 	}
-      last_run_duration = start_time = vlib_time_now (vm);
 
-      BVT (clib_bihash) * h = &msm->mac_table;
-      int i, j, k;
-      for (i = 0; i < h->nbuckets; i++)
-	{
-	  /* Allow no more than 10us without a pause */
-	  t = vlib_time_now (vm);
-	  if (t > start_time + 10e-6)
-	    {
-	      vlib_process_suspend (vm, 100e-6);	/* suspend for 100 us */
-	      start_time = vlib_time_now (vm);
-	    }
-
-	  if (i < (h->nbuckets - 3))
-	    {
-	      BVT (clib_bihash_bucket) * b = &h->buckets[i + 3];
-	      CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
-	      b = &h->buckets[i + 1];
-	      if (b->offset)
-		{
-		  BVT (clib_bihash_value) * v =
-		    BV (clib_bihash_get_value) (h, b->offset);
-		  CLIB_PREFETCH (v, CLIB_CACHE_LINE_BYTES, LOAD);
-		}
-	    }
-
-	  BVT (clib_bihash_bucket) * b = &h->buckets[i];
-	  if (b->offset == 0)
-	    continue;
-	  BVT (clib_bihash_value) * v =
-	    BV (clib_bihash_get_value) (h, b->offset);
-	  for (j = 0; j < (1 << b->log2_pages); j++)
-	    {
-	      for (k = 0; k < BIHASH_KVP_PER_PAGE; k++)
-		{
-		  if (v->kvp[k].key == ~0ULL && v->kvp[k].value == ~0ULL)
-		    continue;
-
-		  l2fib_entry_key_t key = {.raw = v->kvp[k].key };
-		  l2fib_entry_result_t result = {.raw = v->kvp[k].value };
-
-		  if (result.fields.static_mac)
-		    continue;
-
-		  u32 bd_index = key.fields.bd_index;
-		  u32 sw_if_index = result.fields.sw_if_index;
-		  u16 sn = l2fib_cur_seq_num (bd_index, sw_if_index).as_u16;
-		  if (result.fields.sn.as_u16 != sn)
-		    {
-		      l2fib_del_entry_by_key (key.raw);
-		      continue;
-		    }
-		  l2_bridge_domain_t *bd_config =
-		    vec_elt_at_index (l2input_main.bd_configs, bd_index);
-
-		  if (bd_config->mac_age == 0)
-		    continue;
-
-		  i16 delta =
-		    (u8) (start_time / 60) - result.fields.timestamp;
-		  delta += delta < 0 ? 256 : 0;
-
-		  if (delta > bd_config->mac_age)
-		    l2fib_del_entry_by_key (key.raw);
-		}
-	      v++;
-	    }
-	}
-      last_run_duration = vlib_time_now (vm) - last_run_duration;
+      if (scan == SCAN_MAC_EVENT)
+	l2fib_main.evt_scan_duration = l2fib_scan (vm, start_time, 1);
+      else
+	l2fib_main.age_scan_duration = l2fib_scan (vm, start_time, 0);
     }
   return 0;
 }