Fix crash with worker threads on 4K VXLAN/BD setup (VPP-907)

Cleanup mapping of interface output node for the l2-output node
when interface is configured to L2 or L3 modes. The mapping is
now always done in the main thread as part of API/CLI processing,
instead of initiate mapping in the forwarding path which can be
in the worker threads.

Change-Id: Ia789493e7d9f5c76d68edfaf34db43f3e3f53506
Signed-off-by: John Lo <loj@cisco.com>

8 files changed, 51 insertions, 173 deletions

diff --git a/src/vnet/interface.h b/src/vnet/interface.h
index d684e356ca4..402acd1715f 100644
--- a/src/vnet/interface.h
+++ b/src/vnet/interface.h
@@ -419,10 +419,6 @@ typedef struct vnet_hw_interface_t
    VNET_HW_INTERFACE_FLAG_SPEED_40G |		\
    VNET_HW_INTERFACE_FLAG_SPEED_100G)
 
-  /* l2output node flags */
-#define VNET_HW_INTERFACE_FLAG_L2OUTPUT_SHIFT	9
-#define VNET_HW_INTERFACE_FLAG_L2OUTPUT_MAPPED	(1 << 9)
-
   /* rx mode flags */
 #define VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE (1 << 10)
 
@@ -572,10 +568,10 @@ typedef struct
 
 #define VNET_SW_INTERFACE_FLAG_BOND_SLAVE (1 << 4)
 
-/* Interface does not appear in CLI/API */
+  /* Interface does not appear in CLI/API */
 #define VNET_SW_INTERFACE_FLAG_HIDDEN (1 << 5)
 
-/* Interface in ERROR state */
+  /* Interface in ERROR state */
 #define VNET_SW_INTERFACE_FLAG_ERROR (1 << 6)
 
   /* Index for this interface. */
diff --git a/src/vnet/l2/l2_efp_filter.c b/src/vnet/l2/l2_efp_filter.c
index 2db4dc69c9c..f9ba8f2f487 100644
--- a/src/vnet/l2/l2_efp_filter.c
+++ b/src/vnet/l2/l2_efp_filter.c
@@ -530,9 +530,6 @@ VLIB_NODE_FUNCTION_MULTIARCH (l2_efp_filter_node, l2_efp_filter_node_fn)
 			       l2output_get_feat_names (),
 			       mp->next_nodes.feat_next_node_index);
 
-  /* Initialize the output node mapping table */
-  l2output_init_output_node_vec (&mp->next_nodes.output_node_index_vec);
-
   return 0;
 }
 
diff --git a/src/vnet/l2/l2_fib.c b/src/vnet/l2/l2_fib.c
index 6f8f6e06330..4ed16987404 100644
--- a/src/vnet/l2/l2_fib.c
+++ b/src/vnet/l2/l2_fib.c
@@ -413,6 +413,13 @@ l2fib_add (vlib_main_t * vm,
 	}
     }
 
+  if (vec_len (l2input_main.configs) <= sw_if_index)
+    {
+      error = clib_error_return (0, "Interface sw_if_index %d not in L2 mode",
+				 sw_if_index);
+      goto done;
+    }
+
   if (filter_mac)
     l2fib_add_filter_entry (mac, bd_index);
   else
diff --git a/src/vnet/l2/l2_input.c b/src/vnet/l2/l2_input.c
index d536d15b8de..9a3148c533a 100644
--- a/src/vnet/l2/l2_input.c
+++ b/src/vnet/l2/l2_input.c
@@ -573,13 +573,9 @@ set_int_l2_mode (vlib_main_t * vm, vnet_main_t * vnet_main,	/*           */
       l2_if_adjust--;
     }
 
-  /*
-   * Directs the l2 output path to work out the interface
-   * output next-arc itself. Needed when recycling a sw_if_index.
-   */
+  /* Make sure vector is big enough */
   vec_validate_init_empty (l2om->next_nodes.output_node_index_vec,
-			   sw_if_index, ~0);
-  l2om->next_nodes.output_node_index_vec[sw_if_index] = ~0;
+			   sw_if_index, L2OUTPUT_NEXT_DROP);
 
   /* Initialize the l2-input configuration for the interface */
   if (mode == MODE_L3)
@@ -601,26 +597,11 @@ set_int_l2_mode (vlib_main_t * vm, vnet_main_t * vnet_main,	/*           */
       l2om->next_nodes.output_node_index_vec[sw_if_index] =
 	L2OUTPUT_NEXT_BAD_INTF;
     }
-  else if (mode == MODE_L2_CLASSIFY)
-    {
-      config->xconnect = 1;
-      config->bridge = 0;
-      config->output_sw_if_index = xc_sw_if_index;
-
-      /* Make sure last-chance drop is configured */
-      config->feature_bitmap |=
-	L2INPUT_FEAT_DROP | L2INPUT_FEAT_INPUT_CLASSIFY;
-
-      /* Make sure bridging features are disabled */
-      config->feature_bitmap &=
-	~(L2INPUT_FEAT_LEARN | L2INPUT_FEAT_FWD | L2INPUT_FEAT_FLOOD);
-      shg = 0;			/* not used in xconnect */
-
-      /* Insure all packets go to ethernet-input */
-      ethernet_set_rx_redirect (vnet_main, hi, 1);
-    }
   else
     {
+      /* Add or update l2-output node next-arc and output_node_index_vec table
+       * for the interface */
+      l2output_create_output_node_mapping (vm, vnet_main, sw_if_index);
 
       if (mode == MODE_L2_BRIDGE)
 	{
@@ -693,7 +674,7 @@ set_int_l2_mode (vlib_main_t * vm, vnet_main_t * vnet_main,	/*           */
 	  bd_add_member (bd_config, &member);
 
 	}
-      else
+      else if (mode == MODE_L2_XC)
 	{
 	  config->xconnect = 1;
 	  config->bridge = 0;
@@ -709,6 +690,24 @@ set_int_l2_mode (vlib_main_t * vm, vnet_main_t * vnet_main,	/*           */
 	  config->feature_bitmap |= L2INPUT_FEAT_XCONNECT;
 	  shg = 0;		/* not used in xconnect */
 	}
+      else if (mode == MODE_L2_CLASSIFY)
+	{
+	  config->xconnect = 1;
+	  config->bridge = 0;
+	  config->output_sw_if_index = xc_sw_if_index;
+
+	  /* Make sure last-chance drop is configured */
+	  config->feature_bitmap |=
+	    L2INPUT_FEAT_DROP | L2INPUT_FEAT_INPUT_CLASSIFY;
+
+	  /* Make sure bridging features are disabled */
+	  config->feature_bitmap &=
+	    ~(L2INPUT_FEAT_LEARN | L2INPUT_FEAT_FWD | L2INPUT_FEAT_FLOOD);
+	  shg = 0;		/* not used in xconnect */
+
+	  /* Insure all packets go to ethernet-input */
+	  ethernet_set_rx_redirect (vnet_main, hi, 1);
+	}
 
       /* set up split-horizon group and set output feature bit */
       config->shg = shg;
diff --git a/src/vnet/l2/l2_output.c b/src/vnet/l2/l2_output.c
index e17b2a16675..51d5e1453fb 100644
--- a/src/vnet/l2/l2_output.c
+++ b/src/vnet/l2/l2_output.c
@@ -601,90 +601,27 @@ VLIB_NODE_FUNCTION_MULTIARCH (l2output_node, l2output_node_fn)
 			       mp->next_nodes.feat_next_node_index);
 
   /* Initialize the output node mapping table */
-  l2output_init_output_node_vec (&mp->next_nodes.output_node_index_vec);
+  vec_validate_init_empty (mp->next_nodes.output_node_index_vec, 100,
+			   L2OUTPUT_NEXT_DROP);
 
   return 0;
 }
 
 VLIB_INIT_FUNCTION (l2output_init);
 
-typedef struct
-{
-  u32 node_index;
-  u32 sw_if_index;
-} output_node_mapping_rpc_args_t;
-
-static void output_node_rpc_callback (output_node_mapping_rpc_args_t * a);
-
-static void
-output_node_mapping_send_rpc (u32 node_index, u32 sw_if_index)
-{
-  output_node_mapping_rpc_args_t args;
-  void vl_api_rpc_call_main_thread (void *fp, u8 * data, u32 data_length);
-
-  args.node_index = node_index;
-  args.sw_if_index = sw_if_index;
-
-  vl_api_rpc_call_main_thread (output_node_rpc_callback,
-			       (u8 *) & args, sizeof (args));
-}
-
 
 /** Create a mapping in the next node mapping table for the given sw_if_index. */
-u32
-l2output_create_output_node_mapping (vlib_main_t * vlib_main, vnet_main_t * vnet_main, u32 node_index,	/* index of current node */
-				     u32 * output_node_index_vec,
-				     u32 sw_if_index)
-{
-
-  u32 next;			/* index of next graph node */
-  vnet_hw_interface_t *hw0;
-  u32 *node;
-
-  hw0 = vnet_get_sup_hw_interface (vnet_main, sw_if_index);
-
-  uword thread_index;
-
-  thread_index = vlib_get_thread_index ();
-
-  if (thread_index)
-    {
-      u32 oldflags;
-
-      oldflags = __sync_fetch_and_or (&hw0->flags,
-				      VNET_HW_INTERFACE_FLAG_L2OUTPUT_MAPPED);
-
-      if ((oldflags & VNET_HW_INTERFACE_FLAG_L2OUTPUT_MAPPED))
-	return L2OUTPUT_NEXT_DROP;
-
-      output_node_mapping_send_rpc (node_index, sw_if_index);
-      return L2OUTPUT_NEXT_DROP;
-    }
-
-  /* dynamically create graph node arc  */
-  next = vlib_node_add_next (vlib_main, node_index, hw0->output_node_index);
-
-  /* Initialize vector with the mapping */
-
-  node = vec_elt_at_index (output_node_index_vec, sw_if_index);
-  *node = next;
-
-  /* reset mapping bit, includes memory barrier */
-  __sync_fetch_and_and (&hw0->flags, ~VNET_HW_INTERFACE_FLAG_L2OUTPUT_MAPPED);
-
-  return next;
-}
-
 void
-output_node_rpc_callback (output_node_mapping_rpc_args_t * a)
+l2output_create_output_node_mapping (vlib_main_t * vlib_main,
+				     vnet_main_t * vnet_main, u32 sw_if_index)
 {
-  vlib_main_t *vm = vlib_get_main ();
-  vnet_main_t *vnm = vnet_get_main ();
-  l2output_main_t *mp = &l2output_main;
+  vnet_hw_interface_t *hw0 =
+    vnet_get_sup_hw_interface (vnet_main, sw_if_index);
 
-  (void) l2output_create_output_node_mapping
-    (vm, vnm, a->node_index, mp->next_nodes.output_node_index_vec,
-     a->sw_if_index);
+  /* dynamically create graph node arc  */
+  u32 next = vlib_node_add_next (vlib_main, l2output_node.index,
+				 hw0->output_node_index);
+  l2output_main.next_nodes.output_node_index_vec[sw_if_index] = next;
 }
 
 /* Get a pointer to the config for the given interface */
diff --git a/src/vnet/l2/l2_output.h b/src/vnet/l2/l2_output.h
index 9597205caed..82cefd2cbaa 100644
--- a/src/vnet/l2/l2_output.h
+++ b/src/vnet/l2/l2_output.h
@@ -130,7 +130,7 @@ _(EFP_DROP,     "L2 EFP filter pre-rewrite drops")	\
 _(VTR_DROP,     "L2 output tag rewrite drops")		\
 _(SHG_DROP,     "L2 split horizon drops")		\
 _(DROP,         "L2 output drops")			\
-_(MAPPING_DROP, "L2 Output interface mapping in progress")
+_(MAPPING_DROP, "L2 Output interface not valid")
 
 typedef enum
 {
@@ -159,52 +159,9 @@ char **l2output_get_feat_names (void);
  */
 
 /* Create a mapping to the output graph node for the given sw_if_index */
-u32 l2output_create_output_node_mapping (vlib_main_t * vlib_main, vnet_main_t * vnet_main, u32 node_index,	/* index of current node */
-					 u32 * output_node_index_vec,
-					 u32 sw_if_index);
-
-/* Initialize the next node mapping table */
-always_inline void
-l2output_init_output_node_vec (u32 ** output_node_index_vec)
-{
-
-  /*
-   * Size it at 100 sw_if_indexes initially
-   * Uninitialized mappings are set to ~0
-   */
-  vec_validate_init_empty (*output_node_index_vec, 100, ~0);
-}
-
-
-/**
- *  Get a mapping from the output node mapping table,
- * creating the entry if necessary.
- */
-always_inline u32
-l2output_get_output_node (vlib_main_t * vlib_main, vnet_main_t * vnet_main, u32 node_index,	/* index of current node */
-			  u32 sw_if_index, u32 ** output_node_index_vec)	/* may be updated */
-{
-  u32 next;			/* index of next graph node */
-
-  /* Insure the vector is big enough */
-  vec_validate_init_empty (*output_node_index_vec, sw_if_index, ~0);
-
-  /* Get the mapping for the sw_if_index */
-  next = vec_elt (*output_node_index_vec, sw_if_index);
-
-  if (next == ~0)
-    {
-      /* Mapping doesn't exist so create it */
-      next = l2output_create_output_node_mapping (vlib_main,
-						  vnet_main,
-						  node_index,
-						  *output_node_index_vec,
-						  sw_if_index);
-    }
-
-  return next;
-}
-
+void l2output_create_output_node_mapping (vlib_main_t * vlib_main,
+					  vnet_main_t * vnet_main,
+					  u32 sw_if_index);
 
 /** Determine the next L2 node based on the output feature bitmap */
 always_inline void
@@ -257,21 +214,12 @@ l2_output_dispatch (vlib_main_t * vlib_main,
 	}
       else
 	{
-	  /* Look up the output TX node */
-	  *next0 = l2output_get_output_node (vlib_main,
-					     vnet_main,
-					     node_index,
-					     sw_if_index,
-					     &next_nodes->output_node_index_vec);
+	  /* Look up the output TX node for the sw_if_index */
+	  *next0 = vec_elt (l2output_main.next_nodes.output_node_index_vec,
+			    sw_if_index);
 
 	  if (*next0 == L2OUTPUT_NEXT_DROP)
-	    {
-	      vnet_hw_interface_t *hw0;
-	      hw0 = vnet_get_sup_hw_interface (vnet_main, sw_if_index);
-
-	      if (hw0->flags & VNET_HW_INTERFACE_FLAG_L2OUTPUT_MAPPED)
-		b0->error = node->errors[L2OUTPUT_ERROR_MAPPING_DROP];
-	    }
+	    b0->error = node->errors[L2OUTPUT_ERROR_MAPPING_DROP];
 
 	  /* Update the one-entry cache */
 	  *cached_sw_if_index = sw_if_index;
diff --git a/src/vnet/l2/l2_output_acl.c b/src/vnet/l2/l2_output_acl.c
index 94a4d66b48f..1d1971a5516 100644
--- a/src/vnet/l2/l2_output_acl.c
+++ b/src/vnet/l2/l2_output_acl.c
@@ -297,9 +297,6 @@ VLIB_NODE_FUNCTION_MULTIARCH (l2_outacl_node, l2_outacl_node_fn)
 			       l2output_get_feat_names (),
 			       mp->next_nodes.feat_next_node_index);
 
-  /* Initialize the output node mapping table */
-  l2output_init_output_node_vec (&mp->next_nodes.output_node_index_vec);
-
   return 0;
 }
 
diff --git a/src/vnet/l2/l2_output_classify.c b/src/vnet/l2/l2_output_classify.c
index c1bdaddc06e..869b0656355 100644
--- a/src/vnet/l2/l2_output_classify.c
+++ b/src/vnet/l2/l2_output_classify.c
@@ -497,9 +497,6 @@ l2_output_classify_init (vlib_main_t * vm)
   rt->l2cm = cm;
   rt->vcm = cm->vnet_classify_main;
 
-  /* Initialize the output node mapping table */
-  l2output_init_output_node_vec (&cm->next_nodes.output_node_index_vec);
-
   return 0;
 }