Improve L2 Input/Output Feature Infrastructure and Usage

Simplify L2 output feature infra to unify with L2 input feature
infra using the newly improved feature bitmap mechanism.
Updated all L2 features to use the more efficient infra functions.

Change-Id: If8f463826b0af0717129befe92a27ea8cfc40449
Signed-off-by: John Lo <loj@cisco.com>

16 files changed, 160 insertions, 339 deletions

diff --git a/src/plugins/acl/fa_node.c b/src/plugins/acl/fa_node.c
index 0bbc7423a39..c483044dfd8 100644
--- a/src/plugins/acl/fa_node.c
+++ b/src/plugins/acl/fa_node.c
@@ -931,7 +931,6 @@ acl_fa_node_fn (vlib_main_t * vm,
   u32 pkts_acl_permit = 0;
   u32 pkts_restart_session_timer = 0;
   u32 trace_bitmap = 0;
-  u32 feature_bitmap0;
   acl_main_t *am = &acl_main;
   fa_5tuple_t fa_5tuple, kv_sess;
   clib_bihash_kv_40_8_t value_sess;
@@ -977,8 +976,6 @@ acl_fa_node_fn (vlib_main_t * vm,
 	    sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 	  else
 	    sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
-	  if (is_l2_path)
-	    feature_bitmap0 = vnet_buffer (b0)->l2.feature_bitmap;
 
 	  /*
 	   * Extract the L3/L4 matching info into a 5-tuple structure,
@@ -1089,9 +1086,7 @@ acl_fa_node_fn (vlib_main_t * vm,
 	  if (action > 0)
 	    {
 	      if (is_l2_path)
-		next0 =
-		  feat_bitmap_get_next_node_index (l2_feat_next_node_index,
-						   feature_bitmap0);
+		next0 = vnet_l2_feature_next (b0, l2_feat_next_node_index, 0);
 	      else
 		vnet_feature_next (sw_if_index0, &next0, b0);
 	    }
diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c
index df68175055e..1bce3328402 100644
--- a/src/vnet/ethernet/arp.c
+++ b/src/vnet/ethernet/arp.c
@@ -2291,12 +2291,8 @@ arp_term_l2bd (vlib_main_t * vm,
 
 	next_l2_feature:
 	  {
-	    u32 feature_bitmap0 =
-	      vnet_buffer (p0)->l2.feature_bitmap & ~L2INPUT_FEAT_ARP_TERM;
-	    vnet_buffer (p0)->l2.feature_bitmap = feature_bitmap0;
-	    next0 =
-	      feat_bitmap_get_next_node_index (arp_term_next_node_index,
-					       feature_bitmap0);
+	    next0 = vnet_l2_feature_next (p0, arp_term_next_node_index,
+					  L2INPUT_FEAT_ARP_TERM);
 	    vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 					     to_next, n_left_to_next,
 					     pi0, next0);
diff --git a/src/vnet/l2/feat_bitmap.h b/src/vnet/l2/feat_bitmap.h
index c6e02ecc7c9..5940ff7ea9b 100644
--- a/src/vnet/l2/feat_bitmap.h
+++ b/src/vnet/l2/feat_bitmap.h
@@ -75,8 +75,8 @@ feat_bitmap_init_next_nodes (vlib_main_t * vm, u32 node_index,	/* the current gr
  Return the graph node index for the feature corresponding to the
  first set bit in the bitmap.
 */
-always_inline
-  u32 feat_bitmap_get_next_node_index (u32 * next_nodes, u32 bitmap)
+always_inline u32
+feat_bitmap_get_next_node_index (u32 * next_nodes, u32 bitmap)
 {
   u32 first_bit;
 
@@ -85,6 +85,20 @@ always_inline
   return next_nodes[first_bit];
 }
 
+/**
+ Return the graph node index for the feature corresponding to the next
+ set bit after clearing the current feature bit in the feature_bitmap
+ of the current packet.
+*/
+always_inline u32
+vnet_l2_feature_next (vlib_buffer_t * b, u32 * next_nodes, u32 feat_bit)
+{
+  vnet_buffer (b)->l2.feature_bitmap &= ~feat_bit;
+  u32 fb = vnet_buffer (b)->l2.feature_bitmap;
+  ASSERT (fb != 0);
+  return feat_bitmap_get_next_node_index (next_nodes, fb);
+}
+
 #endif /* included_vnet_l2_feat_bitmap_h */
 
 /*
diff --git a/src/vnet/l2/l2_classify.h b/src/vnet/l2/l2_classify.h
index 184187ff879..100c584a556 100644
--- a/src/vnet/l2/l2_classify.h
+++ b/src/vnet/l2/l2_classify.h
@@ -68,15 +68,13 @@ typedef enum
 
 typedef struct _l2_classify_main
 {
-  /* Next nodes for each feature */
-  u32 feat_next_node_index[32];
+  /* Next nodes for L2 input and output features */
+  u32 l2_inp_feat_next[32];
+  u32 l2_out_feat_next[32];
 
   /* Per-address-family classifier table vectors */
   u32 *classify_table_index_by_sw_if_index[L2_INPUT_CLASSIFY_N_TABLES];
 
-  /* Next nodes for features and output interfaces */
-  l2_output_next_nodes_st next_nodes;
-
   /* convenience variables */
   vlib_main_t *vlib_main;
   vnet_main_t *vnet_main;
diff --git a/src/vnet/l2/l2_efp_filter.c b/src/vnet/l2/l2_efp_filter.c
index f9ba8f2f487..faf78153448 100644
--- a/src/vnet/l2/l2_efp_filter.c
+++ b/src/vnet/l2/l2_efp_filter.c
@@ -43,9 +43,8 @@
  */
 typedef struct
 {
-
-  /* Next nodes for features and output interfaces */
-  l2_output_next_nodes_st next_nodes;
+  /* Next nodes for L2 output features */
+  u32 l2_out_feat_next[32];
 
   /* convenience variables */
   vlib_main_t *vlib_main;
@@ -180,11 +179,6 @@ l2_efp_filter_node_fn (vlib_main_t * vm,
   vlib_node_t *n = vlib_get_node (vm, l2_efp_filter_node.index);
   u32 node_counter_base_index = n->error_heap_index;
   vlib_error_main_t *em = &vm->error_main;
-  u32 cached_sw_if_index = ~0;
-  u32 cached_next_index = ~0;
-
-  /* invalidate cache to begin with */
-  cached_sw_if_index = ~0;
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;	/* number of packets to process */
@@ -203,7 +197,6 @@ l2_efp_filter_node_fn (vlib_main_t * vm,
 	  vlib_buffer_t *b0, *b1;
 	  u32 next0, next1;
 	  u32 sw_if_index0, sw_if_index1;
-	  u32 feature_bitmap0, feature_bitmap1;
 	  u16 first_ethertype0, first_ethertype1;
 	  u16 outer_id0, inner_id0, outer_id1, inner_id1;
 	  u32 match_flags0, match_flags1;
@@ -267,29 +260,11 @@ l2_efp_filter_node_fn (vlib_main_t * vm,
 	  em->counters[node_counter_base_index +
 		       L2_EFP_FILTER_ERROR_L2_EFP_FILTER] += 2;
 
-	  /* Remove ourself from the feature bitmap */
-	  feature_bitmap0 =
-	    vnet_buffer (b0)->l2.feature_bitmap & ~L2OUTPUT_FEAT_EFP_FILTER;
-	  feature_bitmap1 =
-	    vnet_buffer (b1)->l2.feature_bitmap & ~L2OUTPUT_FEAT_EFP_FILTER;
-
 	  /* Determine next node */
-	  l2_output_dispatch (msm->vlib_main,
-			      msm->vnet_main,
-			      node,
-			      l2_efp_filter_node.index,
-			      &cached_sw_if_index,
-			      &cached_next_index,
-			      &msm->next_nodes,
-			      b0, sw_if_index0, feature_bitmap0, &next0);
-	  l2_output_dispatch (msm->vlib_main,
-			      msm->vnet_main,
-			      node,
-			      l2_efp_filter_node.index,
-			      &cached_sw_if_index,
-			      &cached_next_index,
-			      &msm->next_nodes,
-			      b1, sw_if_index1, feature_bitmap1, &next1);
+	  next0 = vnet_l2_feature_next (b0, msm->l2_out_feat_next,
+					L2OUTPUT_FEAT_EFP_FILTER);
+	  next1 = vnet_l2_feature_next (b1, msm->l2_out_feat_next,
+					L2OUTPUT_FEAT_EFP_FILTER);
 
 	  /* perform the efp filter check on two packets */
 
@@ -394,7 +369,6 @@ l2_efp_filter_node_fn (vlib_main_t * vm,
 	  vlib_buffer_t *b0;
 	  u32 next0;
 	  u32 sw_if_index0;
-	  u32 feature_bitmap0;
 	  u16 first_ethertype0;
 	  u16 outer_id0, inner_id0;
 	  u32 match_flags0;
@@ -422,19 +396,9 @@ l2_efp_filter_node_fn (vlib_main_t * vm,
 	  em->counters[node_counter_base_index +
 		       L2_EFP_FILTER_ERROR_L2_EFP_FILTER] += 1;
 
-	  /* Remove ourself from the feature bitmap */
-	  feature_bitmap0 =
-	    vnet_buffer (b0)->l2.feature_bitmap & ~L2OUTPUT_FEAT_EFP_FILTER;
-
 	  /* Determine next node */
-	  l2_output_dispatch (msm->vlib_main,
-			      msm->vnet_main,
-			      node,
-			      l2_efp_filter_node.index,
-			      &cached_sw_if_index,
-			      &cached_next_index,
-			      &msm->next_nodes,
-			      b0, sw_if_index0, feature_bitmap0, &next0);
+	  next0 = vnet_l2_feature_next (b0, msm->l2_out_feat_next,
+					L2OUTPUT_FEAT_EFP_FILTER);
 
 	  /* perform the efp filter check on one packet */
 
@@ -528,7 +492,7 @@ VLIB_NODE_FUNCTION_MULTIARCH (l2_efp_filter_node, l2_efp_filter_node_fn)
 			       l2_efp_filter_node.index,
 			       L2OUTPUT_N_FEAT,
 			       l2output_get_feat_names (),
-			       mp->next_nodes.feat_next_node_index);
+			       mp->l2_out_feat_next);
 
   return 0;
 }
diff --git a/src/vnet/l2/l2_input.c b/src/vnet/l2/l2_input.c
index aa15621320e..26c832adfb1 100644
--- a/src/vnet/l2/l2_input.c
+++ b/src/vnet/l2/l2_input.c
@@ -115,10 +115,7 @@ typedef enum
 
 
 static_always_inline void
-classify_and_dispatch (vlib_main_t * vm,
-		       vlib_node_runtime_t * node,
-		       u32 thread_index,
-		       l2input_main_t * msm, vlib_buffer_t * b0, u32 * next0)
+classify_and_dispatch (l2input_main_t * msm, vlib_buffer_t * b0, u32 * next0)
 {
   /*
    * Load L2 input feature struct
@@ -187,12 +184,7 @@ classify_and_dispatch (vlib_main_t * vm,
     }
 
 
-  if (config->xconnect)
-    {
-      /* Set the output interface */
-      vnet_buffer (b0)->sw_if_index[VLIB_TX] = config->output_sw_if_index;
-    }
-  else
+  if (config->bridge)
     {
       /* Do bridge-domain processing */
       bd_index0 = config->bd_index;
@@ -220,6 +212,13 @@ classify_and_dispatch (vlib_main_t * vm,
        */
       feat_mask = feat_mask & bd_config->feature_bitmap;
     }
+  else if (config->xconnect)
+    {
+      /* Set the output interface */
+      vnet_buffer (b0)->sw_if_index[VLIB_TX] = config->output_sw_if_index;
+    }
+  else
+    feat_mask = L2INPUT_FEAT_DROP;
 
   /* mask out features from bitmap using packet type and bd config */
   feature_bitmap = config->feature_bitmap & feat_mask;
@@ -240,7 +239,6 @@ l2input_node_inline (vlib_main_t * vm,
   u32 n_left_from, *from, *to_next;
   l2input_next_t next_index;
   l2input_main_t *msm = &l2input_main;
-  u32 thread_index = vlib_get_thread_index ();
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;	/* number of packets to process */
@@ -353,10 +351,10 @@ l2input_node_inline (vlib_main_t * vm,
 	  vlib_node_increment_counter (vm, l2input_node.index,
 				       L2INPUT_ERROR_L2INPUT, 4);
 
-	  classify_and_dispatch (vm, node, thread_index, msm, b0, &next0);
-	  classify_and_dispatch (vm, node, thread_index, msm, b1, &next1);
-	  classify_and_dispatch (vm, node, thread_index, msm, b2, &next2);
-	  classify_and_dispatch (vm, node, thread_index, msm, b3, &next3);
+	  classify_and_dispatch (msm, b0, &next0);
+	  classify_and_dispatch (msm, b1, &next1);
+	  classify_and_dispatch (msm, b2, &next2);
+	  classify_and_dispatch (msm, b3, &next3);
 
 	  /* verify speculative enqueues, maybe switch current next frame */
 	  /* if next0==next1==next_index then nothing special needs to be done */
@@ -396,7 +394,7 @@ l2input_node_inline (vlib_main_t * vm,
 	  vlib_node_increment_counter (vm, l2input_node.index,
 				       L2INPUT_ERROR_L2INPUT, 1);
 
-	  classify_and_dispatch (vm, node, thread_index, msm, b0, &next0);
+	  classify_and_dispatch (msm, b0, &next0);
 
 	  /* verify speculative enqueue, maybe switch current next frame */
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
@@ -574,8 +572,8 @@ set_int_l2_mode (vlib_main_t * vm, vnet_main_t * vnet_main,	/*           */
     }
 
   /* Make sure vector is big enough */
-  vec_validate_init_empty (l2om->next_nodes.output_node_index_vec,
-			   sw_if_index, L2OUTPUT_NEXT_DROP);
+  vec_validate_init_empty (l2om->output_node_index_vec, sw_if_index,
+			   L2OUTPUT_NEXT_DROP);
 
   /* Initialize the l2-input configuration for the interface */
   if (mode == MODE_L3)
@@ -594,8 +592,7 @@ set_int_l2_mode (vlib_main_t * vm, vnet_main_t * vnet_main,	/*           */
 
       /* Make sure any L2-output packet to this interface now in L3 mode is
        * dropped. This may happen if L2 FIB MAC entry is stale */
-      l2om->next_nodes.output_node_index_vec[sw_if_index] =
-	L2OUTPUT_NEXT_BAD_INTF;
+      l2om->output_node_index_vec[sw_if_index] = L2OUTPUT_NEXT_BAD_INTF;
     }
   else
     {
diff --git a/src/vnet/l2/l2_input_acl.c b/src/vnet/l2/l2_input_acl.c
index 104fcd15b85..84030888c04 100644
--- a/src/vnet/l2/l2_input_acl.c
+++ b/src/vnet/l2/l2_input_acl.c
@@ -269,14 +269,11 @@ l2_inacl_node_fn (vlib_main_t * vm,
 	  e0 = 0;
 	  t0 = 0;
 
-	  /* Feature bitmap update */
-	  vnet_buffer (b0)->l2.feature_bitmap &= ~L2INPUT_FEAT_ACL;
-
 	  vnet_buffer (b0)->l2_classify.opaque_index = ~0;
+
 	  /* Determine the next node */
-	  next0 = feat_bitmap_get_next_node_index (msm->feat_next_node_index,
-						   vnet_buffer (b0)->
-						   l2.feature_bitmap);
+	  next0 = vnet_l2_feature_next (b0, msm->feat_next_node_index,
+					L2INPUT_FEAT_ACL);
 
 	  if (PREDICT_TRUE (table_index0 != ~0))
 	    {
diff --git a/src/vnet/l2/l2_input_classify.c b/src/vnet/l2/l2_input_classify.c
index 485b9abd47e..ee8042a064e 100644
--- a/src/vnet/l2/l2_input_classify.c
+++ b/src/vnet/l2/l2_input_classify.c
@@ -152,7 +152,6 @@ l2_input_classify_node_fn (vlib_main_t * vm,
   vnet_classify_main_t *vcm = cm->vnet_classify_main;
   l2_input_classify_runtime_t *rt =
     (l2_input_classify_runtime_t *) node->runtime_data;
-  u32 feature_bitmap;
   u32 hits = 0;
   u32 misses = 0;
   u32 chain_hits = 0;
@@ -354,13 +353,6 @@ l2_input_classify_node_fn (vlib_main_t * vm,
 	  e0 = 0;
 	  vnet_buffer (b0)->l2_classify.opaque_index = ~0;
 
-	  /* Remove ourself from the feature bitmap */
-	  feature_bitmap = vnet_buffer (b0)->l2.feature_bitmap
-	    & ~L2INPUT_FEAT_INPUT_CLASSIFY;
-
-	  /* save for next feature graph nodes */
-	  vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap;
-
 	  if (PREDICT_TRUE (table_index0 != ~0))
 	    {
 	      hash0 = vnet_buffer (b0)->l2_classify.hash;
@@ -412,13 +404,13 @@ l2_input_classify_node_fn (vlib_main_t * vm,
 	  if (PREDICT_FALSE (next0 == 0))
 	    b0->error = node->errors[L2_INPUT_CLASSIFY_ERROR_DROP];
 
+	  /* Determine the next node and remove ourself from bitmap */
 	  if (PREDICT_TRUE (next0 == ~0))
-	    {
-	      // Determine the next node
-	      next0 =
-		feat_bitmap_get_next_node_index (cm->feat_next_node_index,
-						 feature_bitmap);
-	    }
+	    next0 = vnet_l2_feature_next (b0, cm->l2_inp_feat_next,
+					  L2INPUT_FEAT_INPUT_CLASSIFY);
+	  else
+	    vnet_buffer (b0)->l2.feature_bitmap &=
+	      ~L2INPUT_FEAT_INPUT_CLASSIFY;
 
 	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
@@ -496,7 +488,7 @@ l2_input_classify_init (vlib_main_t * vm)
 			       l2_input_classify_node.index,
 			       L2INPUT_N_FEAT,
 			       l2input_get_feat_names (),
-			       cm->feat_next_node_index);
+			       cm->l2_inp_feat_next);
   rt->l2cm = cm;
   rt->vcm = cm->vnet_classify_main;
 
diff --git a/src/vnet/l2/l2_input_vtr.c b/src/vnet/l2/l2_input_vtr.c
index ded23095849..9470752f0e0 100644
--- a/src/vnet/l2/l2_input_vtr.c
+++ b/src/vnet/l2/l2_input_vtr.c
@@ -111,7 +111,6 @@ l2_invtr_node_fn (vlib_main_t * vm,
 	  vlib_buffer_t *b0, *b1;
 	  u32 next0, next1;
 	  u32 sw_if_index0, sw_if_index1;
-	  u32 feature_bitmap0, feature_bitmap1;
 
 	  /* Prefetch next iteration. */
 	  {
@@ -160,23 +159,11 @@ l2_invtr_node_fn (vlib_main_t * vm,
 	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 	  sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 
-	  /* process 2 packets */
-
-	  /* Remove ourself from the feature bitmap */
-	  feature_bitmap0 =
-	    vnet_buffer (b0)->l2.feature_bitmap & ~L2INPUT_FEAT_VTR;
-	  feature_bitmap1 =
-	    vnet_buffer (b1)->l2.feature_bitmap & ~L2INPUT_FEAT_VTR;
-
-	  /* save for next feature graph nodes */
-	  vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap0;
-	  vnet_buffer (b1)->l2.feature_bitmap = feature_bitmap1;
-
 	  /* Determine the next node */
-	  next0 = feat_bitmap_get_next_node_index (msm->feat_next_node_index,
-						   feature_bitmap0);
-	  next1 = feat_bitmap_get_next_node_index (msm->feat_next_node_index,
-						   feature_bitmap1);
+	  next0 = vnet_l2_feature_next (b0, msm->feat_next_node_index,
+					L2INPUT_FEAT_VTR);
+	  next1 = vnet_l2_feature_next (b1, msm->feat_next_node_index,
+					L2INPUT_FEAT_VTR);
 
 	  l2_output_config_t *config0;
 	  l2_output_config_t *config1;
@@ -264,7 +251,6 @@ l2_invtr_node_fn (vlib_main_t * vm,
 	  vlib_buffer_t *b0;
 	  u32 next0;
 	  u32 sw_if_index0;
-	  u32 feature_bitmap0;
 
 	  /* speculatively enqueue b0 to the current next frame */
 	  bi0 = from[0];
@@ -278,18 +264,9 @@ l2_invtr_node_fn (vlib_main_t * vm,
 
 	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 
-	  /* process 1 packet */
-
-	  /* Remove ourself from the feature bitmap */
-	  feature_bitmap0 =
-	    vnet_buffer (b0)->l2.feature_bitmap & ~L2INPUT_FEAT_VTR;
-
-	  /* save for next feature graph nodes */
-	  vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap0;
-
 	  /* Determine the next node */
-	  next0 = feat_bitmap_get_next_node_index (msm->feat_next_node_index,
-						   feature_bitmap0);
+	  next0 = vnet_l2_feature_next (b0, msm->feat_next_node_index,
+					L2INPUT_FEAT_VTR);
 
 	  l2_output_config_t *config0;
 	  config0 = vec_elt_at_index (l2output_main.configs, sw_if_index0);
diff --git a/src/vnet/l2/l2_learn.c b/src/vnet/l2/l2_learn.c
index b9904d3edf5..65406292cc7 100644
--- a/src/vnet/l2/l2_learn.c
+++ b/src/vnet/l2/l2_learn.c
@@ -116,19 +116,9 @@ l2learn_process (vlib_node_runtime_t * node,
 		 u32 * bucket0,
 		 l2fib_entry_result_t * result0, u32 * next0, u8 timestamp)
 {
-  u32 feature_bitmap;
-
   /* Set up the default next node (typically L2FWD) */
-
-  /* Remove ourself from the feature bitmap */
-  feature_bitmap = vnet_buffer (b0)->l2.feature_bitmap & ~L2INPUT_FEAT_LEARN;
-
-  /* Save for next feature graph nodes */
-  vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap;
-
-  /* Determine the next node */
-  *next0 = feat_bitmap_get_next_node_index (msm->feat_next_node_index,
-					    feature_bitmap);
+  *next0 = vnet_l2_feature_next (b0, msm->feat_next_node_index,
+				 L2INPUT_FEAT_LEARN);
 
   /* Check mac table lookup result */
   if (PREDICT_TRUE (result0->fields.sw_if_index == sw_if_index0))
diff --git a/src/vnet/l2/l2_output.c b/src/vnet/l2/l2_output.c
index 51d5e1453fb..b3537a355cf 100644
--- a/src/vnet/l2/l2_output.c
+++ b/src/vnet/l2/l2_output.c
@@ -98,6 +98,57 @@ split_horizon_violation (u8 shg1, u8 shg2)
     }
 }
 
+/** Determine the next L2 node based on the output feature bitmap */
+static_always_inline void
+l2_output_dispatch (vlib_buffer_t * b0, vlib_node_runtime_t * node,
+		    u32 * cached_sw_if_index, u32 * cached_next_index,
+		    u32 sw_if_index, u32 feature_bitmap, u32 * next0)
+{
+  /*
+   * The output feature bitmap always have at least the L2 output bit set
+   * for a normal L2 interface (or 0 if the interface is changed from L2
+   * to L3 mode). So if the feature bitmap is 0 or just have L2 output bits set,
+   * we know there is no more feature and will just output packets on interface.
+   * Otherwise, get the index of the next feature node.
+   */
+  if (PREDICT_FALSE ((feature_bitmap & ~L2OUTPUT_FEAT_OUTPUT) != 0))
+    {
+      /* Save bitmap for the next feature graph nodes */
+      vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap;
+
+      /* Determine the next node */
+      *next0 =
+	feat_bitmap_get_next_node_index (l2output_main.l2_out_feat_next,
+					 feature_bitmap);
+    }
+  else
+    {
+      /*
+       * There are no features. Send packet to TX node for sw_if_index0
+       * This is a little tricky in that the output interface next node indexes
+       * are not precomputed at init time.
+       */
+
+      if (sw_if_index == *cached_sw_if_index)
+	{
+	  /* We hit in the one-entry cache. Use it. */
+	  *next0 = *cached_next_index;
+	}
+      else
+	{
+	  /* Look up the output TX node for the sw_if_index */
+	  *next0 = vec_elt (l2output_main.output_node_index_vec, sw_if_index);
+
+	  if (PREDICT_FALSE (*next0 == L2OUTPUT_NEXT_DROP))
+	    b0->error = node->errors[L2OUTPUT_ERROR_MAPPING_DROP];
+
+	  /* Update the one-entry cache */
+	  *cached_sw_if_index = sw_if_index;
+	  *cached_next_index = *next0;
+	}
+    }
+}
+
 static_always_inline void
 l2output_vtr (vlib_node_runtime_t * node, l2_output_config_t * config,
 	      u32 feature_bitmap, vlib_buffer_t * b, u32 * next)
@@ -239,41 +290,18 @@ l2output_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  feature_bitmap3 = config3->feature_bitmap;
 
 	  /* Determine next node */
-	  l2_output_dispatch (msm->vlib_main,
-			      msm->vnet_main,
-			      node,
-			      l2output_node.index,
-			      &cached_sw_if_index,
-			      &cached_next_index,
-			      &msm->next_nodes,
-			      b0, sw_if_index0, feature_bitmap0, &next0);
-
-	  l2_output_dispatch (msm->vlib_main,
-			      msm->vnet_main,
-			      node,
-			      l2output_node.index,
-			      &cached_sw_if_index,
-			      &cached_next_index,
-			      &msm->next_nodes,
-			      b1, sw_if_index1, feature_bitmap1, &next1);
-
-	  l2_output_dispatch (msm->vlib_main,
-			      msm->vnet_main,
-			      node,
-			      l2output_node.index,
-			      &cached_sw_if_index,
-			      &cached_next_index,
-			      &msm->next_nodes,
-			      b2, sw_if_index2, feature_bitmap2, &next2);
-
-	  l2_output_dispatch (msm->vlib_main,
-			      msm->vnet_main,
-			      node,
-			      l2output_node.index,
-			      &cached_sw_if_index,
-			      &cached_next_index,
-			      &msm->next_nodes,
-			      b3, sw_if_index3, feature_bitmap3, &next3);
+	  l2_output_dispatch (b0, node, &cached_sw_if_index,
+			      &cached_next_index, sw_if_index0,
+			      feature_bitmap0, &next0);
+	  l2_output_dispatch (b1, node, &cached_sw_if_index,
+			      &cached_next_index, sw_if_index1,
+			      feature_bitmap1, &next1);
+	  l2_output_dispatch (b2, node, &cached_sw_if_index,
+			      &cached_next_index, sw_if_index2,
+			      feature_bitmap2, &next2);
+	  l2_output_dispatch (b3, node, &cached_sw_if_index,
+			      &cached_next_index, sw_if_index3,
+			      feature_bitmap3, &next3);
 
 	  l2output_vtr (node, config0, feature_bitmap0, b0, &next0);
 	  l2output_vtr (node, config1, feature_bitmap1, b1, &next1);
@@ -401,14 +429,9 @@ l2output_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  feature_bitmap0 = config0->feature_bitmap;
 
 	  /* Determine next node */
-	  l2_output_dispatch (msm->vlib_main,
-			      msm->vnet_main,
-			      node,
-			      l2output_node.index,
-			      &cached_sw_if_index,
-			      &cached_next_index,
-			      &msm->next_nodes,
-			      b0, sw_if_index0, feature_bitmap0, &next0);
+	  l2_output_dispatch (b0, node, &cached_sw_if_index,
+			      &cached_next_index, sw_if_index0,
+			      feature_bitmap0, &next0);
 
 	  l2output_vtr (node, config0, feature_bitmap0, b0, &next0);
 
@@ -598,10 +621,10 @@ VLIB_NODE_FUNCTION_MULTIARCH (l2output_node, l2output_node_fn)
 			       l2output_node.index,
 			       L2OUTPUT_N_FEAT,
 			       l2output_get_feat_names (),
-			       mp->next_nodes.feat_next_node_index);
+			       mp->l2_out_feat_next);
 
   /* Initialize the output node mapping table */
-  vec_validate_init_empty (mp->next_nodes.output_node_index_vec, 100,
+  vec_validate_init_empty (mp->output_node_index_vec, 100,
 			   L2OUTPUT_NEXT_DROP);
 
   return 0;
@@ -621,7 +644,7 @@ l2output_create_output_node_mapping (vlib_main_t * vlib_main,
   /* dynamically create graph node arc  */
   u32 next = vlib_node_add_next (vlib_main, l2output_node.index,
 				 hw0->output_node_index);
-  l2output_main.next_nodes.output_node_index_vec[sw_if_index] = next;
+  l2output_main.output_node_index_vec[sw_if_index] = next;
 }
 
 /* Get a pointer to the config for the given interface */
diff --git a/src/vnet/l2/l2_output.h b/src/vnet/l2/l2_output.h
index 82cefd2cbaa..6da3e303c3b 100644
--- a/src/vnet/l2/l2_output.h
+++ b/src/vnet/l2/l2_output.h
@@ -52,11 +52,6 @@ typedef struct
 
 } l2_output_config_t;
 
-
-/*
- * The set of next nodes for features and interface output.
- * Each output feature node should include this.
- */
 typedef struct
 {
   /*
@@ -70,15 +65,7 @@ typedef struct
    * array of next node index for each output feature, indexed
    * by l2output_feat_t. Used to determine next feature node.
    */
-  u32 feat_next_node_index[32];
-
-} l2_output_next_nodes_st;
-
-
-typedef struct
-{
-  /* Next nodes for features and output interfaces */
-  l2_output_next_nodes_st next_nodes;
+  u32 l2_out_feat_next[32];
 
   /* config vector indexed by sw_if_index */
   l2_output_config_t *configs;
@@ -163,71 +150,6 @@ void l2output_create_output_node_mapping (vlib_main_t * vlib_main,
 					  vnet_main_t * vnet_main,
 					  u32 sw_if_index);
 
-/** Determine the next L2 node based on the output feature bitmap */
-always_inline void
-l2_output_dispatch (vlib_main_t * vlib_main,
-		    vnet_main_t * vnet_main,
-		    vlib_node_runtime_t * node,
-		    u32 node_index,
-		    u32 * cached_sw_if_index,
-		    u32 * cached_next_index,
-		    l2_output_next_nodes_st * next_nodes,
-		    vlib_buffer_t * b0,
-		    u32 sw_if_index, u32 feature_bitmap, u32 * next0)
-{
-  /*
-   * The output feature bitmap always have at least the output feature bit set
-   * for a normal L2 interface (or all 0's if the interface is changed from L2
-   * to L3 mode). So if next_nodes specified is that from the l2-output node and
-   * the bitmap is all clear except output feature bit, we know there is no more
-   * feature and will fall through to output packet. If next_nodes is from a L2
-   * output feature node (and not l2-output), we always want to get the node for
-   * the next L2 output feature, including the last feature being interface-
-   * output node to output packet.
-   */
-  if ((next_nodes != &l2output_main.next_nodes)
-      || ((feature_bitmap & ~L2OUTPUT_FEAT_OUTPUT) != 0))
-    {
-      /* There are some features to execute */
-      ASSERT (feature_bitmap != 0);
-
-      /* Save bitmap for the next feature graph nodes */
-      vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap;
-
-      /* Determine the next node */
-      *next0 =
-	feat_bitmap_get_next_node_index (next_nodes->feat_next_node_index,
-					 feature_bitmap);
-    }
-  else
-    {
-      /*
-       * There are no features. Send packet to TX node for sw_if_index0
-       * This is a little tricky in that the output interface next node indexes
-       * are not precomputed at init time.
-       */
-
-      if (sw_if_index == *cached_sw_if_index)
-	{
-	  /* We hit in the one-entry cache. Use it. */
-	  *next0 = *cached_next_index;
-	}
-      else
-	{
-	  /* Look up the output TX node for the sw_if_index */
-	  *next0 = vec_elt (l2output_main.next_nodes.output_node_index_vec,
-			    sw_if_index);
-
-	  if (*next0 == L2OUTPUT_NEXT_DROP)
-	    b0->error = node->errors[L2OUTPUT_ERROR_MAPPING_DROP];
-
-	  /* Update the one-entry cache */
-	  *cached_sw_if_index = sw_if_index;
-	  *cached_next_index = *next0;
-	}
-    }
-}
-
 /** Get a pointer to the config for the given interface */
 l2_output_config_t *l2output_intf_config (u32 sw_if_index);
 
diff --git a/src/vnet/l2/l2_output_acl.c b/src/vnet/l2/l2_output_acl.c
index 1d1971a5516..7d051326ee0 100644
--- a/src/vnet/l2/l2_output_acl.c
+++ b/src/vnet/l2/l2_output_acl.c
@@ -34,8 +34,8 @@
 
 typedef struct
 {
-  /* Next nodes for features and output interfaces */
-  l2_output_next_nodes_st next_nodes;
+  /* Next nodes for L2 output features */
+  u32 l2_out_feat_next[32];
 
   /* convenience variables */
   vlib_main_t *vlib_main;
@@ -108,8 +108,6 @@ l2_outacl_node_fn (vlib_main_t * vm,
   vlib_node_t *n = vlib_get_node (vm, l2_outacl_node.index);
   u32 node_counter_base_index = n->error_heap_index;
   vlib_error_main_t *em = &vm->error_main;
-  u32 cached_sw_if_index = (u32) ~ 0;
-  u32 cached_next_index = (u32) ~ 0;
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;	/* number of packets to process */
@@ -201,7 +199,6 @@ l2_outacl_node_fn (vlib_main_t * vm,
 	  u32 next0;
 	  u32 sw_if_index0;
 	  ethernet_header_t *h0;
-	  u32 feature_bitmap0;
 
 	  /* speculatively enqueue b0 to the current next frame */
 	  bi0 = from[0];
@@ -234,20 +231,9 @@ l2_outacl_node_fn (vlib_main_t * vm,
 	   * Dummy for now, just go to next feature node
 	   */
 
-
-	  /* Remove ourself from the feature bitmap */
-	  feature_bitmap0 =
-	    vnet_buffer (b0)->l2.feature_bitmap & ~L2OUTPUT_FEAT_ACL;
-
 	  /* Determine next node */
-	  l2_output_dispatch (msm->vlib_main,
-			      msm->vnet_main,
-			      node,
-			      l2_outacl_node.index,
-			      &cached_sw_if_index,
-			      &cached_next_index,
-			      &msm->next_nodes,
-			      b0, sw_if_index0, feature_bitmap0, &next0);
+	  next0 = vnet_l2_feature_next (b0, msm->l2_out_feat_next,
+					L2OUTPUT_FEAT_ACL);
 
 	  /* verify speculative enqueue, maybe switch current next frame */
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
@@ -295,7 +281,7 @@ VLIB_NODE_FUNCTION_MULTIARCH (l2_outacl_node, l2_outacl_node_fn)
 			       l2_outacl_node.index,
 			       L2OUTPUT_N_FEAT,
 			       l2output_get_feat_names (),
-			       mp->next_nodes.feat_next_node_index);
+			       mp->l2_out_feat_next);
 
   return 0;
 }
diff --git a/src/vnet/l2/l2_output_classify.c b/src/vnet/l2/l2_output_classify.c
index 869b0656355..a49abec2781 100644
--- a/src/vnet/l2/l2_output_classify.c
+++ b/src/vnet/l2/l2_output_classify.c
@@ -144,14 +144,11 @@ l2_output_classify_node_fn (vlib_main_t * vm,
   vnet_classify_main_t *vcm = cm->vnet_classify_main;
   l2_output_classify_runtime_t *rt =
     (l2_output_classify_runtime_t *) node->runtime_data;
-  u32 feature_bitmap0;
   u32 hits = 0;
   u32 misses = 0;
   u32 chain_hits = 0;
   f64 now;
   u32 n_next_nodes;
-  u32 cached_sw_if_index = (u32) ~ 0;
-  u32 cached_next_index = (u32) ~ 0;
   u32 sw_if_index0;
 
   n_next_nodes = node->n_next_nodes;
@@ -347,12 +344,6 @@ l2_output_classify_node_fn (vlib_main_t * vm,
 	  table_index0 = vnet_buffer (b0)->l2_classify.table_index;
 	  e0 = 0;
 	  vnet_buffer (b0)->l2_classify.opaque_index = ~0;
-	  /* Remove ourself from the feature bitmap */
-	  feature_bitmap0 = vnet_buffer (b0)->l2.feature_bitmap
-	    & ~L2OUTPUT_FEAT_OUTPUT_CLASSIFY;
-
-	  /* save for next feature graph nodes */
-	  vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap0;
 
 	  if (PREDICT_TRUE (table_index0 != ~0))
 	    {
@@ -405,20 +396,13 @@ l2_output_classify_node_fn (vlib_main_t * vm,
 	  if (PREDICT_FALSE (next0 == 0))
 	    b0->error = node->errors[L2_OUTPUT_CLASSIFY_ERROR_DROP];
 
+	  /* Determine the next node and remove ourself from bitmap */
 	  if (PREDICT_FALSE (next0 == ~0))
-	    {
-	      sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
-
-	      /* Determine next node */
-	      l2_output_dispatch (cm->vlib_main,
-				  cm->vnet_main,
-				  node,
-				  l2_output_classify_node.index,
-				  &cached_sw_if_index,
-				  &cached_next_index,
-				  &cm->next_nodes,
-				  b0, sw_if_index0, feature_bitmap0, &next0);
-	    }
+	    next0 = vnet_l2_feature_next (b0, cm->l2_out_feat_next,
+					  L2OUTPUT_FEAT_OUTPUT_CLASSIFY);
+	  else
+	    vnet_buffer (b0)->l2.feature_bitmap &=
+	      ~L2OUTPUT_FEAT_OUTPUT_CLASSIFY;
 
 	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
@@ -493,7 +477,7 @@ l2_output_classify_init (vlib_main_t * vm)
 			       l2_output_classify_node.index,
 			       L2OUTPUT_N_FEAT,
 			       l2output_get_feat_names (),
-			       cm->next_nodes.feat_next_node_index);
+			       cm->l2_out_feat_next);
   rt->l2cm = cm;
   rt->vcm = cm->vnet_classify_main;
 
diff --git a/src/vnet/l2/l2_rw.c b/src/vnet/l2/l2_rw.c
index c54509d048b..fec04774056 100644
--- a/src/vnet/l2/l2_rw.c
+++ b/src/vnet/l2/l2_rw.c
@@ -179,8 +179,8 @@ l2_rw_node_fn (vlib_main_t * vm,
 
       while (n_left_from >= 4 && n_left_to_next >= 2)
 	{
-	  u32 bi0, next0, sw_if_index0, feature_bitmap0, rwe_index0;
-	  u32 bi1, next1, sw_if_index1, feature_bitmap1, rwe_index1;
+	  u32 bi0, next0, sw_if_index0, rwe_index0;
+	  u32 bi1, next1, sw_if_index1, rwe_index1;
 	  vlib_buffer_t *b0, *b1;
 	  ethernet_header_t *h0, *h1;
 	  l2_rw_config_t *config0, *config1;
@@ -273,16 +273,10 @@ l2_rw_node_fn (vlib_main_t * vm,
 	    }
 
 	  /* Update feature bitmap and get next feature index */
-	  feature_bitmap0 =
-	    vnet_buffer (b0)->l2.feature_bitmap & ~L2INPUT_FEAT_RW;
-	  feature_bitmap1 =
-	    vnet_buffer (b1)->l2.feature_bitmap & ~L2INPUT_FEAT_RW;
-	  vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap0;
-	  vnet_buffer (b1)->l2.feature_bitmap = feature_bitmap1;
-	  next0 = feat_bitmap_get_next_node_index (rw->feat_next_node_index,
-						   feature_bitmap0);
-	  next1 = feat_bitmap_get_next_node_index (rw->feat_next_node_index,
-						   feature_bitmap1);
+	  next0 = vnet_l2_feature_next (b0, rw->feat_next_node_index,
+					L2INPUT_FEAT_RW);
+	  next1 = vnet_l2_feature_next (b1, rw->feat_next_node_index,
+					L2INPUT_FEAT_RW);
 
 	  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 					   to_next, n_left_to_next,
@@ -291,7 +285,7 @@ l2_rw_node_fn (vlib_main_t * vm,
 
       while (n_left_from > 0 && n_left_to_next > 0)
 	{
-	  u32 bi0, next0, sw_if_index0, feature_bitmap0, rwe_index0;
+	  u32 bi0, next0, sw_if_index0, rwe_index0;
 	  vlib_buffer_t *b0;
 	  ethernet_header_t *h0;
 	  l2_rw_config_t *config0;
@@ -341,11 +335,8 @@ l2_rw_node_fn (vlib_main_t * vm,
 	    }
 
 	  /* Update feature bitmap and get next feature index */
-	  feature_bitmap0 =
-	    vnet_buffer (b0)->l2.feature_bitmap & ~L2INPUT_FEAT_RW;
-	  vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap0;
-	  next0 = feat_bitmap_get_next_node_index (rw->feat_next_node_index,
-						   feature_bitmap0);
+	  next0 = vnet_l2_feature_next (b0, rw->feat_next_node_index,
+					L2INPUT_FEAT_RW);
 
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 					   to_next, n_left_to_next,
diff --git a/src/vnet/policer/node_funcs.c b/src/vnet/policer/node_funcs.c
index 457dd09fd12..fd031d02eea 100644
--- a/src/vnet/policer/node_funcs.c
+++ b/src/vnet/policer/node_funcs.c
@@ -736,14 +736,9 @@ policer_classify_inline (vlib_main_t * vm,
 
 	  if (tid == POLICER_CLASSIFY_TABLE_L2)
 	    {
-	      /* Feature bitmap update */
-	      vnet_buffer (b0)->l2.feature_bitmap &=
-		~L2INPUT_FEAT_POLICER_CLAS;
-	      /* Determine the next node */
-	      next0 =
-		feat_bitmap_get_next_node_index (pcm->feat_next_node_index,
-						 vnet_buffer (b0)->
-						 l2.feature_bitmap);
+	      /* Feature bitmap update and determine the next node */
+	      next0 = vnet_l2_feature_next (b0, pcm->feat_next_node_index,
+					    L2INPUT_FEAT_POLICER_CLAS);
 	    }
 	  else
 	    vnet_get_config_data (pcm->vnet_config_main[tid],