l2 input: avoid per-packet trace checks in the fast path

Change-Id: Ib0c8572773499d8dd4d81b3a565c24412ccc3510
Signed-off-by: Dave Barach <dave@barachs.net>

4 files changed, 54 insertions, 25 deletions

diff --git a/src/vnet/l2/l2_fwd.c b/src/vnet/l2/l2_fwd.c
index 710a9d9e8c3..f7e2ccb65b7 100644
--- a/src/vnet/l2/l2_fwd.c
+++ b/src/vnet/l2/l2_fwd.c
@@ -195,9 +195,9 @@ l2fwd_process (vlib_main_t * vm,
 }
 
 
-static uword
-l2fwd_node_fn (vlib_main_t * vm,
-	       vlib_node_runtime_t * node, vlib_frame_t * frame)
+static_always_inline uword
+l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+		   vlib_frame_t * frame, int do_trace)
 {
   u32 n_left_from, *from, *to_next;
   l2fwd_next_t next_index;
@@ -281,7 +281,7 @@ l2fwd_node_fn (vlib_main_t * vm,
 	  h2 = vlib_buffer_get_current (b2);
 	  h3 = vlib_buffer_get_current (b3);
 
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	  if (do_trace)
 	    {
 	      if (b0->flags & VLIB_BUFFER_IS_TRACED)
 		{
@@ -388,8 +388,7 @@ l2fwd_node_fn (vlib_main_t * vm,
 
 	  h0 = vlib_buffer_get_current (b0);
 
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
-			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+	  if (do_trace && PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
 	    {
 	      l2fwd_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
 	      t->sw_if_index = sw_if_index0;
@@ -420,6 +419,15 @@ l2fwd_node_fn (vlib_main_t * vm,
   return frame->n_vectors;
 }
 
+static uword
+l2fwd_node_fn (vlib_main_t * vm,
+	       vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+    return l2fwd_node_inline (vm, node, frame, 1 /* do_trace */ );
+  return l2fwd_node_inline (vm, node, frame, 0 /* do_trace */ );
+}
+
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (l2fwd_node,static) = {
   .function = l2fwd_node_fn,
diff --git a/src/vnet/l2/l2_input.c b/src/vnet/l2/l2_input.c
index a104ec9eebb..ead9ca85494 100644
--- a/src/vnet/l2/l2_input.c
+++ b/src/vnet/l2/l2_input.c
@@ -222,10 +222,10 @@ classify_and_dispatch (vlib_main_t * vm,
 					    feature_bitmap);
 }
 
-
-static uword
-l2input_node_fn (vlib_main_t * vm,
-		 vlib_node_runtime_t * node, vlib_frame_t * frame)
+static_always_inline uword
+l2input_node_inline (vlib_main_t * vm,
+		     vlib_node_runtime_t * node, vlib_frame_t * frame,
+		     int do_trace)
 {
   u32 n_left_from, *from, *to_next;
   l2input_next_t next_index;
@@ -294,7 +294,7 @@ l2input_node_fn (vlib_main_t * vm,
 	  b2 = vlib_get_buffer (vm, bi2);
 	  b3 = vlib_get_buffer (vm, bi3);
 
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	  if (do_trace)
 	    {
 	      /* RX interface handles */
 	      sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
@@ -373,8 +373,7 @@ l2input_node_fn (vlib_main_t * vm,
 
 	  b0 = vlib_get_buffer (vm, bi0);
 
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
-			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+	  if (do_trace && PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
 	    {
 	      ethernet_header_t *h0 = vlib_buffer_get_current (b0);
 	      l2input_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
@@ -401,6 +400,14 @@ l2input_node_fn (vlib_main_t * vm,
   return frame->n_vectors;
 }
 
+static uword
+l2input_node_fn (vlib_main_t * vm,
+		 vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+    return l2input_node_inline (vm, node, frame, 1 /* do_trace */ );
+  return l2input_node_inline (vm, node, frame, 0 /* do_trace */ );
+}
 
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (l2input_node) = {
diff --git a/src/vnet/l2/l2_learn.c b/src/vnet/l2/l2_learn.c
index 7f19f936d70..afe7f478f8c 100644
--- a/src/vnet/l2/l2_learn.c
+++ b/src/vnet/l2/l2_learn.c
@@ -231,9 +231,9 @@ done:
 }
 
 
-static uword
-l2learn_node_fn (vlib_main_t * vm,
-		 vlib_node_runtime_t * node, vlib_frame_t * frame)
+static_always_inline uword
+l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+		     vlib_frame_t * frame, int do_trace)
 {
   u32 n_left_from, *from, *to_next;
   l2learn_next_t next_index;
@@ -320,7 +320,7 @@ l2learn_node_fn (vlib_main_t * vm,
 	  h2 = vlib_buffer_get_current (b2);
 	  h3 = vlib_buffer_get_current (b3);
 
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	  if (do_trace)
 	    {
 	      if (b0->flags & VLIB_BUFFER_IS_TRACED)
 		{
@@ -426,8 +426,7 @@ l2learn_node_fn (vlib_main_t * vm,
 
 	  h0 = vlib_buffer_get_current (b0);
 
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
-			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+	  if (do_trace && PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
 	    {
 	      l2learn_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t));
 	      t->sw_if_index = sw_if_index0;
@@ -461,6 +460,14 @@ l2learn_node_fn (vlib_main_t * vm,
   return frame->n_vectors;
 }
 
+static uword
+l2learn_node_fn (vlib_main_t * vm,
+		 vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+    return l2learn_node_inline (vm, node, frame, 1 /* do_trace */ );
+  return l2learn_node_inline (vm, node, frame, 0 /* do_trace */ );
+}
 
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (l2learn_node,static) = {
diff --git a/src/vnet/l2/l2_output.c b/src/vnet/l2/l2_output.c
index 953fcb0222a..acfe3abad90 100644
--- a/src/vnet/l2/l2_output.c
+++ b/src/vnet/l2/l2_output.c
@@ -140,9 +140,9 @@ l2output_vtr (vlib_node_runtime_t * node, l2_output_config_t * config,
 
 static vlib_node_registration_t l2output_node;
 
-static uword
-l2output_node_fn (vlib_main_t * vm,
-		  vlib_node_runtime_t * node, vlib_frame_t * frame)
+static_always_inline uword
+l2output_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+		      vlib_frame_t * frame, int do_trace)
 {
   u32 n_left_from, *from, *to_next;
   l2output_next_t next_index;
@@ -214,7 +214,7 @@ l2output_node_fn (vlib_main_t * vm,
 	  sw_if_index2 = vnet_buffer (b2)->sw_if_index[VLIB_TX];
 	  sw_if_index3 = vnet_buffer (b3)->sw_if_index[VLIB_TX];
 
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+	  if (do_trace)
 	    {
 	      h0 = vlib_buffer_get_current (b0);
 	      h1 = vlib_buffer_get_current (b1);
@@ -378,8 +378,7 @@ l2output_node_fn (vlib_main_t * vm,
 
 	  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX];
 
-	  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
-			     && (b0->flags & VLIB_BUFFER_IS_TRACED)))
+	  if (do_trace && PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
 	    {
 	      l2output_trace_t *t =
 		vlib_add_trace (vm, node, b0, sizeof (*t));
@@ -434,6 +433,14 @@ l2output_node_fn (vlib_main_t * vm,
   return frame->n_vectors;
 }
 
+static uword
+l2output_node_fn (vlib_main_t * vm,
+		  vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+    return l2output_node_inline (vm, node, frame, 1 /* do_trace */ );
+  return l2output_node_inline (vm, node, frame, 0 /* do_trace */ );
+}
 
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (l2output_node,static) = {