interface: interface-output end node rework

 - rename node to more meaningful name
 - introduce lookup tables
 - enable multiarch
 - quad-loop node
 - enqqueue to next instead of enqueueing to node

Type: improvement
Change-Id: Ibb208047ae04bb6cfe56db558d3b8938bc14b4fe
Signed-off-by: Damjan Marion <damarion@cisco.com>

9 files changed, 87 insertions, 75 deletions

diff --git a/src/plugins/ct6/ct6.c b/src/plugins/ct6/ct6.c
index d569a7c3f5b..205cd3f50ef 100644
--- a/src/plugins/ct6/ct6.c
+++ b/src/plugins/ct6/ct6.c
@@ -226,11 +226,10 @@ VNET_FEATURE_INIT (ct6out2in, static) =
 /* *INDENT-ON */
 
 /* *INDENT-OFF* */
-VNET_FEATURE_INIT (ct6in2out, static) =
-{
+VNET_FEATURE_INIT (ct6in2out, static) = {
   .arc_name = "interface-output",
   .node_name = "ct6-in2out",
-  .runs_before = VNET_FEATURES ("interface-tx"),
+  .runs_before = VNET_FEATURES ("interface-output-arc-end"),
 };
 /* *INDENT-ON */
 
diff --git a/src/plugins/flowprobe/flowprobe.c b/src/plugins/flowprobe/flowprobe.c
index 2a32fbf027f..ffc43bcd440 100644
--- a/src/plugins/flowprobe/flowprobe.c
+++ b/src/plugins/flowprobe/flowprobe.c
@@ -60,11 +60,10 @@ VNET_FEATURE_INIT (flow_perpacket_ip6, static) =
   .runs_before = VNET_FEATURES ("interface-output"),
 };
 
-VNET_FEATURE_INIT (flow_perpacket_l2, static) =
-{
+VNET_FEATURE_INIT (flow_perpacket_l2, static) = {
   .arc_name = "interface-output",
   .node_name = "flowprobe-l2",
-  .runs_before = VNET_FEATURES ("interface-tx"),
+  .runs_before = VNET_FEATURES ("interface-output-arc-end"),
 };
 /* *INDENT-ON* */
 
diff --git a/src/plugins/lisp/lisp-gpe/lisp_gpe.c b/src/plugins/lisp/lisp-gpe/lisp_gpe.c
index 1f087fd0625..1ae3131323c 100644
--- a/src/plugins/lisp/lisp-gpe/lisp_gpe.c
+++ b/src/plugins/lisp/lisp-gpe/lisp_gpe.c
@@ -672,8 +672,8 @@ lisp_gpe_test_send_nsh_packet (u8 * file_name)
   clib_memcpy_fast (p, pm.packets_read[0], vec_len (pm.packets_read[0]));
   vlib_buffer_pull (b, sizeof (ethernet_header_t));
 
-  vlib_node_t *n = vlib_get_node_by_name (lgm->vlib_main,
-					  (u8 *) "interface-tx");
+  vlib_node_t *n =
+    vlib_get_node_by_name (lgm->vlib_main, (u8 *) "interface-output-arc-end");
   f = vlib_get_frame_to_node (lgm->vlib_main, n->index);
   u32 *to_next = vlib_frame_vector_args (f);
   to_next[0] = bi;
diff --git a/src/plugins/mactime/mactime.c b/src/plugins/mactime/mactime.c
index a2219bebe9a..b8b1884119d 100644
--- a/src/plugins/mactime/mactime.c
+++ b/src/plugins/mactime/mactime.c
@@ -472,11 +472,10 @@ VNET_FEATURE_INIT (mactime, static) =
 /* *INDENT-ON */
 
 /* *INDENT-OFF* */
-VNET_FEATURE_INIT (mactime_tx, static) =
-{
+VNET_FEATURE_INIT (mactime_tx, static) = {
   .arc_name = "interface-output",
   .node_name = "mactime-tx",
-  .runs_before = VNET_FEATURES ("interface-tx"),
+  .runs_before = VNET_FEATURES ("interface-output-arc-end"),
 };
 /* *INDENT-ON */
 
diff --git a/src/plugins/nsim/nsim.c b/src/plugins/nsim/nsim.c
index 272067382d8..fc8ffebd4a2 100644
--- a/src/plugins/nsim/nsim.c
+++ b/src/plugins/nsim/nsim.c
@@ -559,11 +559,10 @@ VNET_FEATURE_INIT (nsim, static) =
 /* *INDENT-ON */
 
 /* *INDENT-OFF* */
-VNET_FEATURE_INIT (nsim_feature, static) =
-{
+VNET_FEATURE_INIT (nsim_feature, static) = {
   .arc_name = "interface-output",
   .node_name = "nsim-output-feature",
-  .runs_before = VNET_FEATURES ("interface-tx"),
+  .runs_before = VNET_FEATURES ("interface-output-arc-end"),
 };
 /* *INDENT-ON */
 
diff --git a/src/vnet/interface.c b/src/vnet/interface.c
index f648440fbf8..cd734cedd5c 100644
--- a/src/vnet/interface.c
+++ b/src/vnet/interface.c
@@ -554,6 +554,21 @@ vnet_sw_interface_admin_down (vnet_main_t * vnm, u32 sw_if_index)
     }
 }
 
+static void
+vnet_if_update_lookup_tables (vnet_main_t *vnm, u32 sw_if_index)
+{
+  vnet_interface_main_t *im = &vnm->interface_main;
+  vnet_hw_interface_t *hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
+
+  vec_validate_init_empty (im->hw_if_index_by_sw_if_index, sw_if_index, ~0);
+  vec_validate_init_empty (im->if_out_arc_end_next_index_by_sw_if_index,
+			   sw_if_index, ~0);
+
+  im->hw_if_index_by_sw_if_index[sw_if_index] = hi->hw_if_index;
+  im->if_out_arc_end_next_index_by_sw_if_index[sw_if_index] =
+    hi->if_out_arc_end_node_next_index;
+}
+
 static u32
 vnet_create_sw_interface_no_callbacks (vnet_main_t * vnm,
 				       vnet_sw_interface_t * template)
@@ -595,6 +610,7 @@ vnet_create_sw_interface_no_callbacks (vnet_main_t * vnm,
     vnet_interface_counter_unlock (im);
   }
 
+  vnet_if_update_lookup_tables (vnm, sw_if_index);
   return sw_if_index;
 }
 
@@ -991,7 +1007,6 @@ vnet_register_interface (vnet_main_t * vnm,
       vlib_node_add_next_with_slot (vm, hw->output_node_index,
 				    hw->tx_node_index,
 				    VNET_INTERFACE_OUTPUT_NEXT_TX);
-
       /* add interface to the list of "output-interface" feature arc start nodes
          and clone nexts from 1st interface if it exists */
       fcm = vnet_feature_get_config_main (im->output_feature_arc_index);
@@ -1015,6 +1030,9 @@ vnet_register_interface (vnet_main_t * vnm,
 	}
     }
 
+  hw->if_out_arc_end_node_next_index = vlib_node_add_next (
+    vm, vnet_interface_output_arc_end_node.index, hw->tx_node_index);
+  vnet_if_update_lookup_tables (vnm, hw->sw_if_index);
   setup_output_node (vm, hw->output_node_index, hw_class);
   setup_tx_node (vm, hw->tx_node_index, dev_class);
 
diff --git a/src/vnet/interface.h b/src/vnet/interface.h
index 8424b71c545..b36d02c32bf 100644
--- a/src/vnet/interface.h
+++ b/src/vnet/interface.h
@@ -603,18 +603,12 @@ typedef struct
 typedef struct vnet_hw_interface_t
 {
   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
-  /* Interface name. */
-  u8 *name;
-
   /* flags */
   vnet_hw_interface_flags_t flags;
 
   /* capabilities flags */
   vnet_hw_interface_capabilities_t caps;
 
-  /* link speed in kbps */
-  u32 link_speed;
-
   /* Hardware address as vector.  Zero (e.g. zero-length vector) if no
      address for this class (e.g. PPP). */
   u8 *hw_address;
@@ -623,6 +617,9 @@ typedef struct vnet_hw_interface_t
   /* NAME.{output,tx} nodes for this interface. */
   u32 output_node_index, tx_node_index;
 
+  /* interface-output-arc-end node next index for tx node */
+  u32 if_out_arc_end_node_next_index;
+
   /* (dev_class, dev_instance) uniquely identifies hw interface. */
   u32 dev_class_index;
   u32 dev_instance;
@@ -639,6 +636,12 @@ typedef struct vnet_hw_interface_t
 
   CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
 
+  /* Interface name. */
+  u8 *name;
+
+  /* link speed in kbps */
+  u32 link_speed;
+
   /* Next index in interface-output node for this interface
      used by node function vnet_per_buffer_interface_output() */
   u32 output_node_next_index;
@@ -992,6 +995,10 @@ typedef struct
 
   /* feature_arc_index */
   u8 output_feature_arc_index;
+
+  /* fast lookup tables */
+  u32 *hw_if_index_by_sw_if_index;
+  u16 *if_out_arc_end_next_index_by_sw_if_index;
 } vnet_interface_main_t;
 
 static inline void
@@ -1037,6 +1044,7 @@ typedef struct
 int vnet_pcap_dispatch_trace_configure (vnet_pcap_dispatch_trace_args_t *);
 
 extern vlib_node_registration_t vnet_interface_output_node;
+extern vlib_node_registration_t vnet_interface_output_arc_end_node;
 
 #endif /* included_vnet_interface_h */
 
diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c
index 9ae07714485..2d220b7257b 100644
--- a/src/vnet/interface_output.c
+++ b/src/vnet/interface_output.c
@@ -984,63 +984,55 @@ VLIB_REGISTER_NODE (interface_punt) = {
 };
 /* *INDENT-ON* */
 
-/* *INDENT-OFF* */
 VLIB_REGISTER_NODE (vnet_per_buffer_interface_output_node) = {
   .name = "interface-output",
   .vector_size = sizeof (u32),
 };
-/* *INDENT-ON* */
 
-static uword
-interface_tx_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
-		      vlib_frame_t * from_frame)
+VLIB_NODE_FN (vnet_interface_output_arc_end_node)
+(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
 {
   vnet_main_t *vnm = vnet_get_main ();
-  u32 last_sw_if_index = ~0;
-  vlib_frame_t *to_frame = 0;
-  vnet_hw_interface_t *hw = 0;
-  u32 *from, *to_next = 0;
-  u32 n_left_from;
-
-  from = vlib_frame_vector_args (from_frame);
-  n_left_from = from_frame->n_vectors;
-  while (n_left_from > 0)
-    {
-      u32 bi0;
-      vlib_buffer_t *b0;
-      u32 sw_if_index0;
+  vnet_interface_main_t *im = &vnm->interface_main;
+  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
+  u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
+  u32 *from, n_left;
+  u16 *lt = im->if_out_arc_end_next_index_by_sw_if_index;
 
-      bi0 = from[0];
-      from++;
-      n_left_from--;
-      b0 = vlib_get_buffer (vm, bi0);
-      sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
+  from = vlib_frame_vector_args (frame);
+  n_left = frame->n_vectors;
+  vlib_get_buffers (vm, from, bufs, n_left);
 
-      if (PREDICT_FALSE ((last_sw_if_index != sw_if_index0) || to_frame == 0))
-	{
-	  if (to_frame)
-	    {
-	      hw = vnet_get_sup_hw_interface (vnm, last_sw_if_index);
-	      vlib_put_frame_to_node (vm, hw->tx_node_index, to_frame);
-	    }
-	  last_sw_if_index = sw_if_index0;
-	  hw = vnet_get_sup_hw_interface (vnm, sw_if_index0);
-	  to_frame = vlib_get_frame_to_node (vm, hw->tx_node_index);
-	  to_next = vlib_frame_vector_args (to_frame);
-	}
+  while (n_left >= 8)
+    {
+      vlib_prefetch_buffer_header (b[4], LOAD);
+      vlib_prefetch_buffer_header (b[5], LOAD);
+      vlib_prefetch_buffer_header (b[6], LOAD);
+      vlib_prefetch_buffer_header (b[7], LOAD);
+      next[0] = vec_elt (lt, vnet_buffer (b[0])->sw_if_index[VLIB_TX]);
+      next[1] = vec_elt (lt, vnet_buffer (b[1])->sw_if_index[VLIB_TX]);
+      next[2] = vec_elt (lt, vnet_buffer (b[2])->sw_if_index[VLIB_TX]);
+      next[3] = vec_elt (lt, vnet_buffer (b[3])->sw_if_index[VLIB_TX]);
 
-      to_next[0] = bi0;
-      to_next++;
-      to_frame->n_vectors++;
+      b += 4;
+      next += 4;
+      n_left -= 4;
     }
-  vlib_put_frame_to_node (vm, hw->tx_node_index, to_frame);
-  return from_frame->n_vectors;
+
+  while (n_left)
+    {
+      next[0] = vec_elt (lt, vnet_buffer (b[0])->sw_if_index[VLIB_TX]);
+      b++;
+      next++;
+      n_left--;
+    }
+
+  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
+  return frame->n_vectors;
 }
 
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (interface_tx) = {
-  .function = interface_tx_node_fn,
-  .name = "interface-tx",
+VLIB_REGISTER_NODE (vnet_interface_output_arc_end_node) = {
+  .name = "interface-output-arc-end",
   .vector_size = sizeof (u32),
   .n_next_nodes = 1,
   .next_nodes = {
@@ -1048,32 +1040,30 @@ VLIB_REGISTER_NODE (interface_tx) = {
   },
 };
 
-VNET_FEATURE_ARC_INIT (interface_output, static) =
-{
-  .arc_name  = "interface-output",
+VNET_FEATURE_ARC_INIT (interface_output, static) = {
+  .arc_name = "interface-output",
   .start_nodes = VNET_FEATURES (0),
-  .last_in_arc = "interface-tx",
+  .last_in_arc = "interface-output-arc-end",
   .arc_index_ptr = &vnet_main.interface_main.output_feature_arc_index,
 };
 
 VNET_FEATURE_INIT (span_tx, static) = {
   .arc_name = "interface-output",
   .node_name = "span-output",
-  .runs_before = VNET_FEATURES ("interface-tx"),
+  .runs_before = VNET_FEATURES ("interface-output-arc-end"),
 };
 
 VNET_FEATURE_INIT (ipsec_if_tx, static) = {
   .arc_name = "interface-output",
   .node_name = "ipsec-if-output",
-  .runs_before = VNET_FEATURES ("interface-tx"),
+  .runs_before = VNET_FEATURES ("interface-output-arc-end"),
 };
 
-VNET_FEATURE_INIT (interface_tx, static) = {
+VNET_FEATURE_INIT (interface_output_arc_end, static) = {
   .arc_name = "interface-output",
-  .node_name = "interface-tx",
+  .node_name = "interface-output-arc-end",
   .runs_before = 0,
 };
-/* *INDENT-ON* */
 
 #ifndef CLIB_MARCH_VARIANT
 clib_error_t *
diff --git a/src/vnet/interface_stats.c b/src/vnet/interface_stats.c
index 40a22b48e8f..3afde0ea54f 100644
--- a/src/vnet/interface_stats.c
+++ b/src/vnet/interface_stats.c
@@ -198,7 +198,7 @@ VNET_FEATURE_INIT (stats_collect_rx_node, static) = {
 VNET_FEATURE_INIT (stats_collect_tx_node, static) = {
   .arc_name = "interface-output",
   .node_name = "stats-collect-tx",
-  .runs_before = VNET_FEATURES ("interface-tx"),
+  .runs_before = VNET_FEATURES ("interface-output-arc-end"),
 };
 
 /* *INDENT-ON* */