Add support for API client to receive L2 MAC events

Added APIs want_l2_macs_events and l2_macs_event to allow an API
client to receive notification events from VPP for MAC learned
or aged in L2FIB. Only one API client is allowed for L2 MAC events.

The want_l2_macs_events API allow caller to specify MAC learn
limit, event scan delay and max number of MACs that can be included
in a event message. These parameters should be choosen properly as
to not have too many MAC events sent by VPP and overwhelm the API
share memory. They can all be left as 0's so VPP will setup reasonable
defaults which are: 1000 learn limit, 100 msec scan delay and 100
MACs per event message.

If want_l2_macs_events is never called, VPP learning and aging
should behave as before except that MAC entries provisioned by API
or CLI will not be aged, even if it is not set as static_mac. These
non static MACs, however, can be overwritten by MAC learning on a
MAC move as a leared MAC. Only learned MACs are subject to aging.

Change-Id: Ia3757a80cf8adb2811a089d2eafbd6439461285c
Signed-off-by: John Lo <loj@cisco.com>

8 files changed, 494 insertions, 160 deletions

diff --git a/src/vnet/l2/l2.api b/src/vnet/l2/l2.api
index bb3990c6fa4..e508bfb529c 100644
--- a/src/vnet/l2/l2.api
+++ b/src/vnet/l2/l2.api
@@ -133,12 +133,64 @@ autoreply define l2fib_add_del
   u8 bvi_mac;
 };
 
-/** \brief Set L2 flags request !!! TODO - need more info, feature bits in l2_input.h
+/** \brief Register to recive L2 MAC events for leanred and aged MAC
+    Will also change MAC learn limit to L2LEARN_INFORM_LIMIT
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param learn_limit - MAC learn limit, 0 => default to 1000
+    @param scan_delay - event scan delay in 10 msec unit, 0 => default to 100 msec
+    @param max_macs_in_event - in units of 10 mac entries, 0 => default to 100 entries
+    @param enable_disable - 1 => register for MAC events, 0 => cancel registration
+    @param pid - sender's pid
+*/
+autoreply define want_l2_macs_events
+{
+  u32 client_index;
+  u32 context;
+  u32 learn_limit;
+  u8  scan_delay;
+  u8  max_macs_in_event;
+  u8  enable_disable;
+  u32 pid;
+};
+
+/** \brief Entry for learned or aged MAC in L2 MAC Events
+    @param sw_if_index - sw_if_index in the domain
+    @param mac_addr - mac_address
+    @is_del - 0 => newly learned MAC, 1 => aged out MAC
+*/
+typeonly define mac_entry
+{
+  u32 sw_if_index;
+  u8  mac_addr[6];
+  u8  is_del;
+  u8  spare;
+};
+
+/** \brief L2 MAC event for a list of learned or aged MACs
+    @param client_index - opaque cookie to identify the sender
+    @param pid - client pid registered to receive notification
+    @param n_macs - number of learned/aged MAC enntries
+    @param mac - array of learned/aged MAC entries
+*/
+define l2_macs_event
+{
+  u32 client_index;
+  u32 pid;
+  u32 n_macs;
+  vl_api_mac_entry_t mac[n_macs];
+};
+
+/** \brief Set interface L2 flags (such as L2_LEARN, L2_FWD, 
+    L2_FLOOD, L2_UU_FLOOD, or L2_ARP_TERM bits). This can be used
+    to disable one or more of the features represented by the
+    flag bits on an interface to override what is set as default
+    for all interfaces in the bridge domain
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
     @param sw_if_index - interface 
     @param is_set - if non-zero, set the bits, else clear them
-    @param feature_bitmap - non-zero bits to set or clear
+    @param feature_bitmap - non-zero bits (as above) to set or clear
 */
 define l2_flags
 {
@@ -149,9 +201,10 @@ define l2_flags
   u32 feature_bitmap;
 };
 
-/** \brief Set L2 bits response
+/** \brief Set interface L2 flags response
     @param context - sender context, to match reply w/ request
     @param retval - return code for the set l2 bits request
+    @param resulting_feature_bitmap - the internal l2 feature bitmap after the request is implemented
 */
 define l2_flags_reply
 {
@@ -250,12 +303,12 @@ manual_print manual_endian define bridge_domain_details
 };
 
 /** \brief Set bridge flags (such as L2_LEARN, L2_FWD, L2_FLOOD, 
-    L2_UU_FLOOD, or L2_ARP_TERM) request
+    L2_UU_FLOOD, or L2_ARP_TERM bits) request
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
     @param bd_id - the bridge domain to set the flags for
     @param is_set - if non-zero, set the flags, else clear them
-    @param feature_bitmap - bits that are non-zero to set or clear
+    @param feature_bitmap - bits (as above) that are non-zero to set or clear
 */
 define bridge_flags
 {
@@ -269,7 +322,7 @@ define bridge_flags
 /** \brief Set bridge flags response
     @param context - sender context, to match reply w/ request
     @param retval - return code for the set bridge flags request
-    @param resulting_feature_bitmap - the feature bitmap value after the request is implemented
+    @param resulting_feature_bitmap - the internal L2 feature bitmap after the request is implemented
 */
 define bridge_flags_reply
 {
diff --git a/src/vnet/l2/l2_api.c b/src/vnet/l2/l2_api.c
index a0b40d6d840..c81cbad77d7 100644
--- a/src/vnet/l2/l2_api.c
+++ b/src/vnet/l2/l2_api.c
@@ -25,6 +25,7 @@
 #include <vnet/l2/l2_input.h>
 #include <vnet/l2/l2_fib.h>
 #include <vnet/l2/l2_vtr.h>
+#include <vnet/l2/l2_learn.h>
 
 #include <vnet/vnet_msg_enum.h>
 
@@ -55,6 +56,7 @@ _(L2FIB_FLUSH_ALL, l2fib_flush_all)                         \
 _(L2FIB_FLUSH_INT, l2fib_flush_int)                         \
 _(L2FIB_FLUSH_BD, l2fib_flush_bd)                           \
 _(L2FIB_ADD_DEL, l2fib_add_del)                             \
+_(WANT_L2_MACS_EVENTS, want_l2_macs_events)		    \
 _(L2_FLAGS, l2_flags)                                       \
 _(BRIDGE_DOMAIN_ADD_DEL, bridge_domain_add_del)             \
 _(BRIDGE_DOMAIN_DUMP, bridge_domain_dump)                   \
@@ -221,8 +223,8 @@ vl_api_l2fib_add_del_t_handler (vl_api_l2fib_add_del_t * mp)
 		  goto bad_sw_if_index;
 		}
 	    }
-	  u32 static_mac = mp->static_mac ? 1 : 0;
-	  u32 bvi_mac = mp->bvi_mac ? 1 : 0;
+	  u8 static_mac = mp->static_mac ? 1 : 0;
+	  u8 bvi_mac = mp->bvi_mac ? 1 : 0;
 	  l2fib_add_fwd_entry (mac, bd_index, sw_if_index, static_mac,
 			       bvi_mac);
 	}
@@ -238,6 +240,58 @@ vl_api_l2fib_add_del_t_handler (vl_api_l2fib_add_del_t * mp)
 }
 
 static void
+vl_api_want_l2_macs_events_t_handler (vl_api_want_l2_macs_events_t * mp)
+{
+  int rv = 0;
+  vl_api_want_l2_macs_events_reply_t *rmp;
+  l2learn_main_t *lm = &l2learn_main;
+  l2fib_main_t *fm = &l2fib_main;
+  u32 pid = ntohl (mp->pid);
+  u32 learn_limit = ntohl (mp->learn_limit);
+
+  if (mp->enable_disable)
+    {
+      if (lm->client_pid == 0)
+	{
+	  lm->client_pid = pid;
+	  lm->client_index = mp->client_index;
+
+	  if (mp->max_macs_in_event)
+	    fm->max_macs_in_event = mp->max_macs_in_event * 10;
+	  else
+	    fm->max_macs_in_event = L2FIB_EVENT_MAX_MACS_DEFAULT;
+
+	  if (mp->scan_delay)
+	    fm->event_scan_delay = (f64) (mp->scan_delay) * 10e-3;
+	  else
+	    fm->event_scan_delay = L2FIB_EVENT_SCAN_DELAY_DEFAULT;
+
+	  /* change learn limit and flush all learned MACs */
+	  if (learn_limit && (learn_limit < L2LEARN_DEFAULT_LIMIT))
+	    lm->global_learn_limit = learn_limit;
+	  else
+	    lm->global_learn_limit = L2FIB_EVENT_LEARN_LIMIT_DEFAULT;
+
+	  l2fib_flush_all_mac (vlib_get_main ());
+	}
+      else if (lm->client_pid != pid)
+	{
+	  rv = VNET_API_ERROR_L2_MACS_EVENT_CLINET_PRESENT;
+	  goto exit;
+	}
+    }
+  else if (lm->client_pid)
+    {
+      lm->client_pid = 0;
+      lm->client_index = 0;
+      lm->global_learn_limit = L2LEARN_DEFAULT_LIMIT;
+    }
+
+exit:
+  REPLY_MACRO (VL_API_WANT_L2_MACS_EVENTS_REPLY);
+}
+
+static void
 vl_api_l2fib_flush_int_t_handler (vl_api_l2fib_flush_int_t * mp)
 {
   int rv = 0;
@@ -293,8 +347,25 @@ vl_api_l2_flags_t_handler (vl_api_l2_flags_t * mp)
   VALIDATE_SW_IF_INDEX (mp);
 
   u32 sw_if_index = ntohl (mp->sw_if_index);
-  u32 flags = ntohl (mp->feature_bitmap) & L2INPUT_VALID_MASK;
-  rbm = l2input_intf_bitmap_enable (sw_if_index, flags, mp->is_set);
+  u32 flags = ntohl (mp->feature_bitmap);
+  u32 bitmap = 0;
+
+  if (flags & L2_LEARN)
+    bitmap |= L2INPUT_FEAT_LEARN;
+
+  if (flags & L2_FWD)
+    bitmap |= L2INPUT_FEAT_FWD;
+
+  if (flags & L2_FLOOD)
+    bitmap |= L2INPUT_FEAT_FLOOD;
+
+  if (flags & L2_UU_FLOOD)
+    bitmap |= L2INPUT_FEAT_UU_FLOOD;
+
+  if (flags & L2_ARP_TERM)
+    bitmap |= L2INPUT_FEAT_ARP_TERM;
+
+  rbm = l2input_intf_bitmap_enable (sw_if_index, bitmap, mp->is_set);
 
   BAD_SW_IF_INDEX_LABEL;
 
@@ -455,13 +526,13 @@ vl_api_bridge_flags_t_handler (vl_api_bridge_flags_t * mp)
       goto out;
     }
 
-  bd_set_flags (vm, bd_index, flags, mp->is_set);
+  u32 bitmap = bd_set_flags (vm, bd_index, flags, mp->is_set);
 
 out:
   /* *INDENT-OFF* */
   REPLY_MACRO2(VL_API_BRIDGE_FLAGS_REPLY,
   ({
-    rmp->resulting_feature_bitmap = ntohl(flags);
+    rmp->resulting_feature_bitmap = ntohl(bitmap);
   }));
   /* *INDENT-ON* */
 }
diff --git a/src/vnet/l2/l2_bd.c b/src/vnet/l2/l2_bd.c
index a87d02f2ffd..6e0db058e31 100644
--- a/src/vnet/l2/l2_bd.c
+++ b/src/vnet/l2/l2_bd.c
@@ -263,7 +263,7 @@ bd_set_flags (vlib_main_t * vm, u32 bd_index, u32 flags, u32 enable)
       bd_config->feature_bitmap &= ~feature_bitmap;
     }
 
-  return 0;
+  return bd_config->feature_bitmap;
 }
 
 /**
@@ -328,12 +328,7 @@ bd_learn (vlib_main_t * vm,
     }
 
   /* set the bridge domain flag */
-  if (bd_set_flags (vm, bd_index, L2_LEARN, enable))
-    {
-      error =
-	clib_error_return (0, "bridge-domain id %d out of range", bd_index);
-      goto done;
-    }
+  bd_set_flags (vm, bd_index, L2_LEARN, enable);
 
 done:
   return error;
@@ -397,12 +392,7 @@ bd_fwd (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd)
     }
 
   /* set the bridge domain flag */
-  if (bd_set_flags (vm, bd_index, L2_FWD, enable))
-    {
-      error =
-	clib_error_return (0, "bridge-domain id %d out of range", bd_index);
-      goto done;
-    }
+  bd_set_flags (vm, bd_index, L2_FWD, enable);
 
 done:
   return error;
@@ -468,12 +458,7 @@ bd_flood (vlib_main_t * vm,
     }
 
   /* set the bridge domain flag */
-  if (bd_set_flags (vm, bd_index, L2_FLOOD, enable))
-    {
-      error =
-	clib_error_return (0, "bridge-domain id %d out of range", bd_index);
-      goto done;
-    }
+  bd_set_flags (vm, bd_index, L2_FLOOD, enable);
 
 done:
   return error;
@@ -538,12 +523,7 @@ bd_uu_flood (vlib_main_t * vm,
     }
 
   /* set the bridge domain flag */
-  if (bd_set_flags (vm, bd_index, L2_UU_FLOOD, enable))
-    {
-      error =
-	clib_error_return (0, "bridge-domain id %d out of range", bd_index);
-      goto done;
-    }
+  bd_set_flags (vm, bd_index, L2_UU_FLOOD, enable);
 
 done:
   return error;
@@ -605,12 +585,7 @@ bd_arp_term (vlib_main_t * vm,
     enable = 0;
 
   /* set the bridge domain flag */
-  if (bd_set_flags (vm, bd_index, L2_ARP_TERM, enable))
-    {
-      error =
-	clib_error_return (0, "bridge-domain id %d out of range", bd_index);
-      goto done;
-    }
+  bd_set_flags (vm, bd_index, L2_ARP_TERM, enable);
 
 done:
   return error;
diff --git a/src/vnet/l2/l2_fib.c b/src/vnet/l2/l2_fib.c
index 7e59b09848a..8aa0ac295dd 100644
--- a/src/vnet/l2/l2_fib.c
+++ b/src/vnet/l2/l2_fib.c
@@ -31,6 +31,17 @@
 
 #include <vppinfra/bihash_template.c>
 
+#include <vlibmemory/api.h>
+#include <vnet/vnet_msg_enum.h>
+
+#define vl_typedefs		/* define message structures */
+#include <vnet/vnet_all_api_h.h>
+#undef vl_typedefs
+
+#define vl_endianfun		/* define message structures */
+#include <vnet/vnet_all_api_h.h>
+#undef vl_endianfun
+
 /**
  * @file
  * @brief Ethernet MAC Address FIB Table Management.
@@ -117,6 +128,7 @@ show_l2fib (vlib_main_t * vm,
   int i, j, k;
   u8 verbose = 0;
   u8 raw = 0;
+  u8 learn = 0;
   u32 bd_id, bd_index = ~0;
   u8 now = (u8) (vlib_time_now (vm) / 60);
   u8 *s = 0;
@@ -127,12 +139,18 @@ show_l2fib (vlib_main_t * vm,
     verbose = 1;
   else if (unformat (input, "bd_index %d", &bd_index))
     verbose = 1;
+  else if (unformat (input, "learn"))
+    {
+      learn = 1;
+      verbose = 0;
+    }
   else if (unformat (input, "bd_id %d", &bd_id))
     {
       uword *p = hash_get (bdm->bd_index_by_bd_id, bd_id);
       if (p)
 	{
-	  verbose = 1;
+	  if (learn == 0)
+	    verbose = 1;
 	  bd_index = p[0];
 	}
       else
@@ -155,7 +173,7 @@ show_l2fib (vlib_main_t * vm,
 	      if (v->kvp[k].key == ~0ULL && v->kvp[k].value == ~0ULL)
 		continue;
 
-	      if (verbose && first_entry)
+	      if ((verbose || learn) && first_entry)
 		{
 		  first_entry = 0;
 		  vlib_cli_output (vm,
@@ -168,13 +186,19 @@ show_l2fib (vlib_main_t * vm,
 	      key.raw = v->kvp[k].key;
 	      result.raw = v->kvp[k].value;
 
-	      if (verbose
+	      if ((verbose || learn)
 		  & ((bd_index >> 31) || (bd_index == key.fields.bd_index)))
 		{
+		  if (learn && result.fields.age_not)
+		    {
+		      total_entries++;
+		      continue;	/* skip provisioned macs */
+		    }
+
 		  bd_config = vec_elt_at_index (l2input_main.bd_configs,
 						key.fields.bd_index);
 
-		  if (bd_config->mac_age && !result.fields.static_mac)
+		  if (bd_config->mac_age && !result.fields.age_not)
 		    {
 		      i16 delta = now - result.fields.timestamp;
 		      delta += delta < 0 ? 256 : 0;
@@ -206,9 +230,19 @@ show_l2fib (vlib_main_t * vm,
   if (total_entries == 0)
     vlib_cli_output (vm, "no l2fib entries");
   else
-    vlib_cli_output (vm,
-		     "%lld l2fib entries with %d learned (or non-static) entries",
-		     total_entries, l2learn_main.global_learn_count);
+    {
+      l2learn_main_t *lm = &l2learn_main;
+      vlib_cli_output (vm, "L2FIB total/learned entries: %d/%d  "
+		       "Last scan time: %.4esec  Learn limit: %d ",
+		       total_entries, lm->global_learn_count,
+		       msm->age_scan_duration, lm->global_learn_limit);
+      if (lm->client_pid)
+	vlib_cli_output (vm, "L2MAC events client PID: %d  "
+			 "Last e-scan time: %.4esec  Delay: %.2esec  "
+			 "Max macs in event: %d",
+			 lm->client_pid, msm->evt_scan_duration,
+			 msm->event_scan_delay, msm->max_macs_in_event);
+    }
 
   if (raw)
     vlib_cli_output (vm, "Raw Hash Table:\n%U\n",
@@ -242,7 +276,7 @@ show_l2fib (vlib_main_t * vm,
 /* *INDENT-OFF* */
 VLIB_CLI_COMMAND (show_l2fib_cli, static) = {
   .path = "show l2fib",
-  .short_help = "show l2fib [verbose | bd_id <nn> | bd_index <nn> | raw]",
+  .short_help = "show l2fib [verbose | learn | bd_id <nn> | bd_index <nn> | raw",
   .function = show_l2fib,
 };
 /* *INDENT-ON* */
@@ -309,36 +343,39 @@ l2fib_cur_seq_num (u32 bd_index, u32 sw_if_index)
  */
 void
 l2fib_add_entry (u64 mac, u32 bd_index,
-		 u32 sw_if_index, u32 static_mac, u32 filter_mac, u32 bvi_mac)
+		 u32 sw_if_index, u8 static_mac, u8 filter_mac, u8 bvi_mac)
 {
   l2fib_entry_key_t key;
   l2fib_entry_result_t result;
   __attribute__ ((unused)) u32 bucket_contents;
-  l2fib_main_t *mp = &l2fib_main;
+  l2fib_main_t *fm = &l2fib_main;
+  l2learn_main_t *lm = &l2learn_main;
   BVT (clib_bihash_kv) kv;
 
   /* set up key */
   key.raw = l2fib_make_key ((u8 *) & mac, bd_index);
 
+  /* check if entry alread exist */
+  if (BV (clib_bihash_search) (&fm->mac_table, &kv, &kv))
+    {
+      /* decrement counter if overwriting a learned mac  */
+      result.raw = kv.value;
+      if ((result.fields.age_not == 0) && (lm->global_learn_count))
+	lm->global_learn_count--;
+    }
+
   /* set up result */
   result.raw = 0;		/* clear all fields */
   result.fields.sw_if_index = sw_if_index;
   result.fields.static_mac = static_mac;
   result.fields.filter = filter_mac;
   result.fields.bvi = bvi_mac;
-  if (!static_mac)
-    result.fields.sn = l2fib_cur_seq_num (bd_index, sw_if_index);
+  result.fields.age_not = 1;	/* no aging for provisioned entry */
 
   kv.key = key.raw;
   kv.value = result.raw;
 
-  BV (clib_bihash_add_del) (&mp->mac_table, &kv, 1 /* is_add */ );
-
-  /* increment counter if dynamically learned mac */
-  if (result.fields.static_mac == 0)
-    {
-      l2learn_main.global_learn_count++;
-    }
+  BV (clib_bihash_add_del) (&fm->mac_table, &kv, 1 /* is_add */ );
 }
 
 /**
@@ -630,13 +667,8 @@ l2fib_del_entry_by_key (u64 raw_key)
   result.raw = kv.value;
 
   /* decrement counter if dynamically learned mac */
-  if (result.fields.static_mac == 0)
-    {
-      if (l2learn_main.global_learn_count > 0)
-	{
-	  l2learn_main.global_learn_count--;
-	}
-    }
+  if ((result.fields.age_not == 0) && (l2learn_main.global_learn_count))
+    l2learn_main.global_learn_count--;
 
   /* Remove entry from hash table */
   BV (clib_bihash_add_del) (&mp->mac_table, &kv, 0 /* is_add */ );
@@ -910,111 +942,273 @@ BVT (clib_bihash) * get_mac_table (void)
   return &mp->mac_table;
 }
 
+static_always_inline void *
+allocate_mac_evt_buf (u32 client, u32 client_index)
+{
+  l2fib_main_t *fm = &l2fib_main;
+  vl_api_l2_macs_event_t *mp = vl_msg_api_alloc
+    (sizeof (*mp) + (fm->max_macs_in_event * sizeof (vl_api_mac_entry_t)));
+  mp->_vl_msg_id = htons (VL_API_L2_MACS_EVENT);
+  mp->pid = htonl (client);
+  mp->client_index = client_index;
+  return mp;
+}
+
+static_always_inline f64
+l2fib_scan (vlib_main_t * vm, f64 start_time, u8 event_only)
+{
+  l2fib_main_t *fm = &l2fib_main;
+  l2learn_main_t *lm = &l2learn_main;
+
+  BVT (clib_bihash) * h = &fm->mac_table;
+  int i, j, k;
+  f64 last_start = start_time;
+  f64 accum_t = 0;
+  f64 delta_t = 0;
+  u32 evt_idx = 0;
+  u32 learn_count = 0;
+  u32 client = lm->client_pid;
+  u32 cl_idx = lm->client_index;
+  vl_api_l2_macs_event_t *mp = 0;
+  unix_shared_memory_queue_t *q = 0;
+
+  if (client)
+    {
+      mp = allocate_mac_evt_buf (client, cl_idx);
+      q = vl_api_client_index_to_input_queue (lm->client_index);
+    }
+
+  for (i = 0; i < h->nbuckets; i++)
+    {
+      /* allow no more than 20us without a pause */
+      delta_t = vlib_time_now (vm) - last_start;
+      if (delta_t > 20e-6)
+	{
+	  vlib_process_suspend (vm, 100e-6);	/* suspend for 100 us */
+	  last_start = vlib_time_now (vm);
+	  accum_t += delta_t;
+	}
+
+      if (i < (h->nbuckets - 3))
+	{
+	  BVT (clib_bihash_bucket) * b = &h->buckets[i + 3];
+	  CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
+	  b = &h->buckets[i + 1];
+	  if (b->offset)
+	    {
+	      BVT (clib_bihash_value) * v =
+		BV (clib_bihash_get_value) (h, b->offset);
+	      CLIB_PREFETCH (v, CLIB_CACHE_LINE_BYTES, LOAD);
+	    }
+	}
+
+      BVT (clib_bihash_bucket) * b = &h->buckets[i];
+      if (b->offset == 0)
+	continue;
+      BVT (clib_bihash_value) * v = BV (clib_bihash_get_value) (h, b->offset);
+      for (j = 0; j < (1 << b->log2_pages); j++)
+	{
+	  for (k = 0; k < BIHASH_KVP_PER_PAGE; k++)
+	    {
+	      if (v->kvp[k].key == ~0ULL && v->kvp[k].value == ~0ULL)
+		continue;
+
+	      l2fib_entry_key_t key = {.raw = v->kvp[k].key };
+	      l2fib_entry_result_t result = {.raw = v->kvp[k].value };
+
+	      if (result.fields.age_not == 0)
+		learn_count++;
+
+	      if (PREDICT_FALSE (evt_idx >= fm->max_macs_in_event))
+		{
+		  /* evet message full, sent it and start a new one */
+		  if (q && (q->cursize < q->maxsize))
+		    {
+		      mp->n_macs = htonl (evt_idx);
+		      vl_msg_api_send_shmem (q, (u8 *) & mp);
+		      mp = allocate_mac_evt_buf (client, cl_idx);
+		    }
+		  else
+		    {
+		      clib_warning ("MAC event to pid %d queue stuffed!"
+				    " %d MAC entries lost", client, evt_idx);
+		    }
+		  evt_idx = 0;
+		}
+
+	      if (client)
+		{
+		  if (result.fields.lrn_evt)
+		    {
+		      /* copy mac entry to event msg */
+		      clib_memcpy (mp->mac[evt_idx].mac_addr, key.fields.mac,
+				   6);
+		      mp->mac[evt_idx].is_del = 0;
+		      mp->mac[evt_idx].sw_if_index =
+			htonl (result.fields.sw_if_index);
+		      /* clear event bit and update mac entry */
+		      result.fields.lrn_evt = 0;
+		      BVT (clib_bihash_kv) kv;
+		      kv.key = key.raw;
+		      kv.value = result.raw;
+		      BV (clib_bihash_add_del) (&fm->mac_table, &kv, 1);
+		      evt_idx++;
+		      continue;	/* skip aging */
+		    }
+		}
+
+	      if (event_only || result.fields.age_not)
+		continue;	/* skip aging - static_mac alsways age_not */
+
+	      /* start aging processing */
+	      u32 bd_index = key.fields.bd_index;
+	      u32 sw_if_index = result.fields.sw_if_index;
+	      u16 sn = l2fib_cur_seq_num (bd_index, sw_if_index).as_u16;
+	      if (result.fields.sn.as_u16 != sn)
+		goto age_out;	/* stale mac */
+
+	      l2_bridge_domain_t *bd_config =
+		vec_elt_at_index (l2input_main.bd_configs, bd_index);
+
+	      if (bd_config->mac_age == 0)
+		continue;	/* skip aging */
+
+	      i16 delta = (u8) (start_time / 60) - result.fields.timestamp;
+	      delta += delta < 0 ? 256 : 0;
+
+	      if (delta < bd_config->mac_age)
+		continue;	/* still valid */
+
+	    age_out:
+	      if (client)
+		{
+		  /* copy mac entry to event msg */
+		  clib_memcpy (mp->mac[evt_idx].mac_addr, key.fields.mac, 6);
+		  mp->mac[evt_idx].is_del = 1;
+		  mp->mac[evt_idx].sw_if_index =
+		    htonl (result.fields.sw_if_index);
+		  evt_idx++;
+		}
+	      /* delete mac entry */
+	      BVT (clib_bihash_kv) kv;
+	      kv.key = key.raw;
+	      BV (clib_bihash_add_del) (&fm->mac_table, &kv, 0);
+	      learn_count--;
+	    }
+	  v++;
+	}
+    }
+
+  /* keep learn count consistent */
+  l2learn_main.global_learn_count = learn_count;
+
+  if (mp)
+    {
+      /*  send any outstanding mac event message else free message buffer */
+      if (evt_idx)
+	{
+	  if (q && (q->cursize < q->maxsize))
+	    {
+	      mp->n_macs = htonl (evt_idx);
+	      vl_msg_api_send_shmem (q, (u8 *) & mp);
+	    }
+	  else
+	    {
+	      clib_warning ("MAC event to pid %d queue stuffed!"
+			    " %d MAC entries lost", client, evt_idx);
+	      vl_msg_api_free (mp);
+	    }
+	}
+      else
+	vl_msg_api_free (mp);
+    }
+  return delta_t + accum_t;
+}
+
+/* Type of scan */
+#define SCAN_MAC_AGE   0
+#define SCAN_MAC_EVENT 1
+
+/* Maximum f64 value */
+#define TIME_MAX (1.7976931348623157e+308)
+
 static uword
 l2fib_mac_age_scanner_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
 			       vlib_frame_t * f)
 {
   uword event_type, *event_data = 0;
-  l2fib_main_t *msm = &l2fib_main;
+  l2fib_main_t *fm = &l2fib_main;
+  l2learn_main_t *lm = &l2learn_main;
   bool enabled = 0;
-  f64 start_time, last_run_duration = 0, t;
+  bool scan = SCAN_MAC_AGE;	/* SCAN_FOR_AGE or SCAN_FOR_EVENT */
+  f64 start_time, next_age_scan_time = TIME_MAX;
 
   while (1)
     {
       if (enabled)
-	vlib_process_wait_for_event_or_clock (vm, 60 - last_run_duration);
+	{
+	  if (lm->client_pid)	/* mac event client waiting */
+	    vlib_process_wait_for_event_or_clock (vm, fm->event_scan_delay);
+	  else			/* agin only */
+	    {
+	      f64 t = next_age_scan_time - vlib_time_now (vm);
+	      if (t < fm->event_scan_delay)
+		t = fm->event_scan_delay;
+	      vlib_process_wait_for_event_or_clock (vm, t);
+	    }
+	}
       else
 	vlib_process_wait_for_event (vm);
 
       event_type = vlib_process_get_events (vm, &event_data);
       vec_reset_length (event_data);
 
+      start_time = vlib_time_now (vm);
+
       switch (event_type)
 	{
-	case ~0:
+	case ~0:		/* timer expired */
+	  if ((lm->client_pid == 0) || (start_time >= next_age_scan_time))
+	    {
+	      scan = SCAN_MAC_AGE;
+	      if (enabled)
+		next_age_scan_time = start_time + L2FIB_AGE_SCAN_INTERVAL;
+	      else
+		next_age_scan_time = TIME_MAX;
+	    }
+	  else
+	    scan = SCAN_MAC_EVENT;
 	  break;
+
 	case L2_MAC_AGE_PROCESS_EVENT_START:
+	  scan = SCAN_MAC_AGE;
+	  next_age_scan_time = start_time + L2FIB_AGE_SCAN_INTERVAL;
 	  enabled = 1;
 	  break;
+
 	case L2_MAC_AGE_PROCESS_EVENT_STOP:
 	  enabled = 0;
+	  next_age_scan_time = TIME_MAX;
+	  l2fib_main.age_scan_duration = 0;
+	  l2fib_main.evt_scan_duration = 0;
 	  continue;
+
 	case L2_MAC_AGE_PROCESS_EVENT_ONE_PASS:
-	  enabled = 0;
+	  scan = SCAN_MAC_AGE;
+	  if (enabled)
+	    next_age_scan_time = start_time + L2FIB_AGE_SCAN_INTERVAL;
+	  else
+	    next_age_scan_time = TIME_MAX;
 	  break;
+
 	default:
 	  ASSERT (0);
 	}
-      last_run_duration = start_time = vlib_time_now (vm);
 
-      BVT (clib_bihash) * h = &msm->mac_table;
-      int i, j, k;
-      for (i = 0; i < h->nbuckets; i++)
-	{
-	  /* Allow no more than 10us without a pause */
-	  t = vlib_time_now (vm);
-	  if (t > start_time + 10e-6)
-	    {
-	      vlib_process_suspend (vm, 100e-6);	/* suspend for 100 us */
-	      start_time = vlib_time_now (vm);
-	    }
-
-	  if (i < (h->nbuckets - 3))
-	    {
-	      BVT (clib_bihash_bucket) * b = &h->buckets[i + 3];
-	      CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD);
-	      b = &h->buckets[i + 1];
-	      if (b->offset)
-		{
-		  BVT (clib_bihash_value) * v =
-		    BV (clib_bihash_get_value) (h, b->offset);
-		  CLIB_PREFETCH (v, CLIB_CACHE_LINE_BYTES, LOAD);
-		}
-	    }
-
-	  BVT (clib_bihash_bucket) * b = &h->buckets[i];
-	  if (b->offset == 0)
-	    continue;
-	  BVT (clib_bihash_value) * v =
-	    BV (clib_bihash_get_value) (h, b->offset);
-	  for (j = 0; j < (1 << b->log2_pages); j++)
-	    {
-	      for (k = 0; k < BIHASH_KVP_PER_PAGE; k++)
-		{
-		  if (v->kvp[k].key == ~0ULL && v->kvp[k].value == ~0ULL)
-		    continue;
-
-		  l2fib_entry_key_t key = {.raw = v->kvp[k].key };
-		  l2fib_entry_result_t result = {.raw = v->kvp[k].value };
-
-		  if (result.fields.static_mac)
-		    continue;
-
-		  u32 bd_index = key.fields.bd_index;
-		  u32 sw_if_index = result.fields.sw_if_index;
-		  u16 sn = l2fib_cur_seq_num (bd_index, sw_if_index).as_u16;
-		  if (result.fields.sn.as_u16 != sn)
-		    {
-		      l2fib_del_entry_by_key (key.raw);
-		      continue;
-		    }
-		  l2_bridge_domain_t *bd_config =
-		    vec_elt_at_index (l2input_main.bd_configs, bd_index);
-
-		  if (bd_config->mac_age == 0)
-		    continue;
-
-		  i16 delta =
-		    (u8) (start_time / 60) - result.fields.timestamp;
-		  delta += delta < 0 ? 256 : 0;
-
-		  if (delta > bd_config->mac_age)
-		    l2fib_del_entry_by_key (key.raw);
-		}
-	      v++;
-	    }
-	}
-      last_run_duration = vlib_time_now (vm) - last_run_duration;
+      if (scan == SCAN_MAC_EVENT)
+	l2fib_main.evt_scan_duration = l2fib_scan (vm, start_time, 1);
+      else
+	l2fib_main.age_scan_duration = l2fib_scan (vm, start_time, 0);
     }
   return 0;
 }
diff --git a/src/vnet/l2/l2_fib.h b/src/vnet/l2/l2_fib.h
index ee6f0dc56a4..49a8b5b628c 100644
--- a/src/vnet/l2/l2_fib.h
+++ b/src/vnet/l2/l2_fib.h
@@ -27,6 +27,18 @@
 #define L2FIB_NUM_BUCKETS (64 * 1024)
 #define L2FIB_MEMORY_SIZE (256<<20)
 
+/* Ager scan interval is 1 minute for aging */
+#define L2FIB_AGE_SCAN_INTERVAL		(60.0)
+
+/* MAC event scan delay is 100 msec unless specified by MAC event client */
+#define L2FIB_EVENT_SCAN_DELAY_DEFAULT	(0.1)
+
+/* Max MACs in a event message is 100 unless specified by MAC event client */
+#define L2FIB_EVENT_MAX_MACS_DEFAULT	(100)
+
+/* MAC event learn limit is 1000 unless specified by MAC event client */
+#define L2FIB_EVENT_LEARN_LIMIT_DEFAULT	(1000)
+
 typedef struct
 {
 
@@ -36,6 +48,16 @@ typedef struct
   /* per swif vector of sequence number for interface based flush of MACs */
   u8 *swif_seq_num;
 
+  /* last event or ager scan duration */
+  f64 evt_scan_duration;
+  f64 age_scan_duration;
+
+  /* delay between event scans, default to 100 msec */
+  f64 event_scan_delay;
+
+  /* max macs in evet message, default to 100 entries */
+  u32 max_macs_in_event;
+
   /* convenience variables */
   vlib_main_t *vlib_main;
   vnet_main_t *vnet_main;
@@ -89,12 +111,15 @@ typedef struct
   {
     struct
     {
-      u32 sw_if_index;		/* output sw_if_index (L3 interface if bvi==1) */
+      u32 sw_if_index;		/* output sw_if_index (L3 intf if bvi==1) */
 
-      u8 static_mac:1;		/* static mac, no dataplane learning */
+      u8 static_mac:1;		/* static mac, no MAC move */
+      u8 age_not:1;		/* not subject to age */
       u8 bvi:1;			/* mac is for a bridged virtual interface */
       u8 filter:1;		/* drop packets to/from this mac */
-      u8 unused1:5;
+      u8 lrn_evt:1;		/* MAC learned to be sent in L2 MAC event */
+      u8 unused:3;
+
       u8 timestamp;		/* timestamp for aging */
       l2fib_seq_num_t sn;	/* bd/int seq num */
     } fields;
@@ -348,11 +373,11 @@ void l2fib_clear_table (void);
 void
 l2fib_add_entry (u64 mac,
 		 u32 bd_index,
-		 u32 sw_if_index, u32 static_mac, u32 drop_mac, u32 bvi_mac);
+		 u32 sw_if_index, u8 static_mac, u8 drop_mac, u8 bvi_mac);
 
 static inline void
-l2fib_add_fwd_entry (u64 mac, u32 bd_index, u32 sw_if_index, u32 static_mac,
-		     u32 bvi_mac)
+l2fib_add_fwd_entry (u64 mac, u32 bd_index, u32 sw_if_index, u8 static_mac,
+		     u8 bvi_mac)
 {
   l2fib_add_entry (mac, bd_index, sw_if_index, static_mac, 0, bvi_mac);
 }
diff --git a/src/vnet/l2/l2_fwd.c b/src/vnet/l2/l2_fwd.c
index 8140728b5e1..2bb7307c68c 100644
--- a/src/vnet/l2/l2_fwd.c
+++ b/src/vnet/l2/l2_fwd.c
@@ -141,8 +141,9 @@ l2fwd_process (vlib_main_t * vm,
       vnet_buffer (b0)->sw_if_index[VLIB_TX] = result0->fields.sw_if_index;
       *next0 = L2FWD_NEXT_L2_OUTPUT;
       int l2fib_seq_num_valid = 1;
+
       /* check l2fib seq num for stale entries */
-      if (!result0->fields.static_mac)
+      if (!result0->fields.age_not)
 	{
 	  l2fib_seq_num_t in_sn = {.as_u16 = vnet_buffer (b0)->l2.l2fib_sn };
 	  l2fib_seq_num_t expected_sn = {
diff --git a/src/vnet/l2/l2_learn.c b/src/vnet/l2/l2_learn.c
index 65406292cc7..47c036b0bf1 100644
--- a/src/vnet/l2/l2_learn.c
+++ b/src/vnet/l2/l2_learn.c
@@ -123,11 +123,9 @@ l2learn_process (vlib_node_runtime_t * node,
   /* Check mac table lookup result */
   if (PREDICT_TRUE (result0->fields.sw_if_index == sw_if_index0))
     {
-      /*
-       * The entry was in the table, and the sw_if_index matched, the normal case
-       */
+      /* Entry in L2FIB with matching sw_if_index matched - normal fast path */
       counter_base[L2LEARN_ERROR_HIT] += 1;
-      int update = !result0->fields.static_mac &&
+      int update = !result0->fields.age_not &&	/* static_mac always age_not */
 	(result0->fields.timestamp != timestamp ||
 	 result0->fields.sn.as_u16 != vnet_buffer (b0)->l2.l2fib_sn);
 
@@ -136,10 +134,10 @@ l2learn_process (vlib_node_runtime_t * node,
     }
   else if (result0->raw == ~0)
     {
-      /* The entry was not in table, so add it  */
+      /* Entry not in L2FIB - add it  */
       counter_base[L2LEARN_ERROR_MISS] += 1;
 
-      if (msm->global_learn_count == msm->global_learn_limit)
+      if (msm->global_learn_count >= msm->global_learn_limit)
 	{
 	  /*
 	   * Global limit reached. Do not learn the mac but forward the packet.
@@ -149,15 +147,22 @@ l2learn_process (vlib_node_runtime_t * node,
 	  return;
 	}
 
+      /* Do not learn if mac is 0 */
+      l2fib_entry_key_t key = *key0;
+      key.fields.bd_index = 0;
+      if (key.raw == 0)
+	return;
+
       /* It is ok to learn */
       msm->global_learn_count++;
       result0->raw = 0;		/* clear all fields */
       result0->fields.sw_if_index = sw_if_index0;
+      result0->fields.lrn_evt = (msm->client_pid != 0);
       cached_key->raw = ~0;	/* invalidate the cache */
     }
   else
     {
-      /* The entry was in the table, but with the wrong sw_if_index mapping (mac move) */
+      /* Entry in L2FIB with different sw_if_index - mac move or filter */
       if (result0->fields.filter)
 	{
 	  ASSERT (result0->fields.sw_if_index == ~0);
@@ -167,8 +172,6 @@ l2learn_process (vlib_node_runtime_t * node,
 	  return;
 	}
 
-      counter_base[L2LEARN_ERROR_MAC_MOVE] += 1;
-
       if (result0->fields.static_mac)
 	{
 	  /*
@@ -185,6 +188,13 @@ l2learn_process (vlib_node_runtime_t * node,
        * TODO: check global/bridge domain/interface learn limits
        */
       result0->fields.sw_if_index = sw_if_index0;
+      if (result0->fields.age_not)	/* The mac was provisioned */
+	{
+	  msm->global_learn_count++;
+	  result0->fields.age_not = 0;
+	}
+      result0->fields.lrn_evt = (msm->client_pid != 0);
+      counter_base[L2LEARN_ERROR_MAC_MOVE] += 1;
     }
 
   /* Update the entry */
@@ -479,7 +489,7 @@ VLIB_NODE_FUNCTION_MULTIARCH (l2learn_node, l2learn_node_fn)
    * Set the default number of dynamically learned macs to the number
    * of buckets.
    */
-  mp->global_learn_limit = L2FIB_NUM_BUCKETS * 16;
+  mp->global_learn_limit = L2LEARN_DEFAULT_LIMIT;
 
   return 0;
 }
diff --git a/src/vnet/l2/l2_learn.h b/src/vnet/l2/l2_learn.h
index 0d95de04b66..000ab59ee7f 100644
--- a/src/vnet/l2/l2_learn.h
+++ b/src/vnet/l2/l2_learn.h
@@ -34,6 +34,10 @@ typedef struct
   /* maximum number of dynamically learned mac entries */
   u32 global_learn_limit;
 
+  /* client waiting for L2 MAC events for learned and aged MACs */
+  u32 client_pid;
+  u32 client_index;
+
   /* Next nodes for each feature */
   u32 feat_next_node_index[32];
 
@@ -42,6 +46,7 @@ typedef struct
   vnet_main_t *vnet_main;
 } l2learn_main_t;
 
+#define L2LEARN_DEFAULT_LIMIT (L2FIB_NUM_BUCKETS * 16)
 
 l2learn_main_t l2learn_main;