Improve L2FIB PDR/NDR performance (VPP-963)

1. Limit MAC entry update per l2-learn call to reduce update burst
   when wall clock advance to the the next minute so all MAC time
   stamps are behind current time needing update.
2. Optimize l2-learn node fast path code sequence.
3. Invalidate cache_key when update MAC entry.
4. Change L2 learn hit counter to L2 learn hit-update counter.
5. Increase L2FIB table memory size to 512MB to fit 4M entries
6. Set MAC learn limit at 4M entries

Change-Id: I3075ee8fb59645a56850126bac2e3e6d341cef4d
Signed-off-by: John Lo <loj@cisco.com>

4 files changed, 30 insertions, 18 deletions

diff --git a/src/vnet/l2/l2_api.c b/src/vnet/l2/l2_api.c
index 7e79d6fa4bf..989081fb4e2 100644
--- a/src/vnet/l2/l2_api.c
+++ b/src/vnet/l2/l2_api.c
@@ -284,7 +284,7 @@ vl_api_want_l2_macs_events_t_handler (vl_api_want_l2_macs_events_t * mp)
     {
       lm->client_pid = 0;
       lm->client_index = 0;
-      if (learn_limit)
+      if (learn_limit && (learn_limit < L2LEARN_DEFAULT_LIMIT))
 	lm->global_learn_limit = learn_limit;
       else
 	lm->global_learn_limit = L2LEARN_DEFAULT_LIMIT;
diff --git a/src/vnet/l2/l2_fib.h b/src/vnet/l2/l2_fib.h
index 49a8b5b628c..7cc2dc5e15e 100644
--- a/src/vnet/l2/l2_fib.h
+++ b/src/vnet/l2/l2_fib.h
@@ -25,7 +25,7 @@
  * The size of the hash table
  */
 #define L2FIB_NUM_BUCKETS (64 * 1024)
-#define L2FIB_MEMORY_SIZE (256<<20)
+#define L2FIB_MEMORY_SIZE (512<<20)
 
 /* Ager scan interval is 1 minute for aging */
 #define L2FIB_AGE_SCAN_INTERVAL		(60.0)
diff --git a/src/vnet/l2/l2_learn.c b/src/vnet/l2/l2_learn.c
index 623c2de202c..066bb54fe2e 100644
--- a/src/vnet/l2/l2_learn.c
+++ b/src/vnet/l2/l2_learn.c
@@ -78,7 +78,7 @@ _(MISS,              "L2 learn misses")			\
 _(MAC_MOVE,          "L2 mac moves")			\
 _(MAC_MOVE_VIOLATE,  "L2 mac move violations")		\
 _(LIMIT,             "L2 not learned due to limit")	\
-_(HIT,               "L2 learn hits")			\
+_(HIT_UPDATE,        "L2 learn hit updates")		\
 _(FILTER_DROP,       "L2 filter mac drops")
 
 typedef enum
@@ -113,7 +113,7 @@ l2learn_process (vlib_node_runtime_t * node,
 		 u32 sw_if_index0,
 		 l2fib_entry_key_t * key0,
 		 l2fib_entry_key_t * cached_key,
-		 u32 * bucket0,
+		 u32 * count,
 		 l2fib_entry_result_t * result0, u32 * next0, u8 timestamp)
 {
   /* Set up the default next node (typically L2FWD) */
@@ -124,15 +124,24 @@ l2learn_process (vlib_node_runtime_t * node,
   if (PREDICT_TRUE (result0->fields.sw_if_index == sw_if_index0))
     {
       /* Entry in L2FIB with matching sw_if_index matched - normal fast path */
-      counter_base[L2LEARN_ERROR_HIT] += 1;
-      int update = !result0->fields.age_not &&	/* static_mac always age_not */
-	(result0->fields.timestamp != timestamp ||
-	 result0->fields.sn.as_u16 != vnet_buffer (b0)->l2.l2fib_sn);
+      u32 dtime = timestamp - result0->fields.timestamp;
+      u32 dsn = result0->fields.sn.as_u16 - vnet_buffer (b0)->l2.l2fib_sn;
+      u32 check = dtime | dsn;
+
+      if (PREDICT_TRUE (check == 0))
+	return;			/* MAC entry up to date */
+      if (result0->fields.age_not)
+	return;			/* Static MAC always age_not */
+      if (msm->global_learn_count > msm->global_learn_limit)
+	return;			/* Above learn limit - do not update */
 
-      if (PREDICT_TRUE (!update))
+      /* Limit updates per l2-learn node call to avoid prolonged update burst
+       * as dtime advance over 1 minute mark, unless more than 1 min behind */
+      if ((*count > 2) && (dtime == 1))
 	return;
-      else if (msm->global_learn_count > msm->global_learn_limit)
-	return;			/* Above learn limit - do not update */
+
+      counter_base[L2LEARN_ERROR_HIT_UPDATE] += 1;
+      *count += 1;
     }
   else if (result0->raw == ~0)
     {
@@ -160,7 +169,6 @@ l2learn_process (vlib_node_runtime_t * node,
       result0->raw = 0;		/* clear all fields */
       result0->fields.sw_if_index = sw_if_index0;
       result0->fields.lrn_evt = (msm->client_pid != 0);
-      cached_key->raw = ~0;	/* invalidate the cache */
     }
   else
     {
@@ -207,6 +215,9 @@ l2learn_process (vlib_node_runtime_t * node,
   kv.key = key0->raw;
   kv.value = result0->raw;
   BV (clib_bihash_add_del) (msm->mac_table, &kv, 1 /* is_add */ );
+
+  /* Invalidate the cache */
+  cached_key->raw = ~0;
 }
 
 
@@ -223,6 +234,7 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   l2fib_entry_key_t cached_key;
   l2fib_entry_result_t cached_result;
   u8 timestamp = (u8) (vlib_time_now (vm) / 60);
+  u32 count = 0;
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;	/* number of packets to process */
@@ -358,19 +370,19 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
 	  l2learn_process (node, msm, &em->counters[node_counter_base_index],
 			   b0, sw_if_index0, &key0, &cached_key,
-			   &bucket0, &result0, &next0, timestamp);
+			   &count, &result0, &next0, timestamp);
 
 	  l2learn_process (node, msm, &em->counters[node_counter_base_index],
 			   b1, sw_if_index1, &key1, &cached_key,
-			   &bucket1, &result1, &next1, timestamp);
+			   &count, &result1, &next1, timestamp);
 
 	  l2learn_process (node, msm, &em->counters[node_counter_base_index],
 			   b2, sw_if_index2, &key2, &cached_key,
-			   &bucket2, &result2, &next2, timestamp);
+			   &count, &result2, &next2, timestamp);
 
 	  l2learn_process (node, msm, &em->counters[node_counter_base_index],
 			   b3, sw_if_index3, &key3, &cached_key,
-			   &bucket3, &result3, &next3, timestamp);
+			   &count, &result3, &next3, timestamp);
 
 	  /* verify speculative enqueues, maybe switch current next frame */
 	  /* if next0==next1==next_index then nothing special needs to be done */
@@ -425,7 +437,7 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
 	  l2learn_process (node, msm, &em->counters[node_counter_base_index],
 			   b0, sw_if_index0, &key0, &cached_key,
-			   &bucket0, &result0, &next0, timestamp);
+			   &count, &result0, &next0, timestamp);
 
 	  /* verify speculative enqueue, maybe switch current next frame */
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
diff --git a/src/vnet/l2/l2_learn.h b/src/vnet/l2/l2_learn.h
index d6f41d40d49..3aaf48e2544 100644
--- a/src/vnet/l2/l2_learn.h
+++ b/src/vnet/l2/l2_learn.h
@@ -46,7 +46,7 @@ typedef struct
   vnet_main_t *vnet_main;
 } l2learn_main_t;
 
-#define L2LEARN_DEFAULT_LIMIT (L2FIB_NUM_BUCKETS * 128)
+#define L2LEARN_DEFAULT_LIMIT (L2FIB_NUM_BUCKETS * 64)
 
 l2learn_main_t l2learn_main;