1 files changed, 321 insertions, 22 deletions

diff --git a/src/plugins/nsim/node.c b/src/plugins/nsim/node.c
index 25112abe299..559147b1280 100644
--- a/src/plugins/nsim/node.c
+++ b/src/plugins/nsim/node.c
@@ -80,20 +80,24 @@ typedef enum
 
 always_inline uword
 nsim_inline (vlib_main_t * vm,
-	     vlib_node_runtime_t * node, vlib_frame_t * frame, int is_trace)
+	     vlib_node_runtime_t * node, vlib_frame_t * frame, int is_trace,
+	     int is_cross_connect)
 {
   nsim_main_t *nsm = &nsim_main;
   u32 n_left_from, *from;
+  u32 *to_next, n_left_to_next;
+  u32 drops[VLIB_FRAME_SIZE], *drop;
   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
+  u8 is_drop[4];
   u16 nexts[VLIB_FRAME_SIZE], *next;
   u32 my_thread_index = vm->thread_index;
   nsim_wheel_t *wp = nsm->wheel_by_thread[my_thread_index];
   f64 now = vlib_time_now (vm);
   f64 expires = now + nsm->delay;
-  int is_drop0;
-  u32 no_error = node->errors[NSIM_ERROR_BUFFERED];
+  f64 rnd[4];
   u32 no_buffer_error = node->errors[NSIM_ERROR_DROPPED];
   u32 loss_error = node->errors[NSIM_ERROR_LOSS];
+  u32 buffered = 0;
   nsim_wheel_entry_t *ep = 0;
 
   ASSERT (wp);
@@ -104,24 +108,246 @@ nsim_inline (vlib_main_t * vm,
   vlib_get_buffers (vm, from, bufs, n_left_from);
   b = bufs;
   next = nexts;
+  drop = drops;
+
+  while (n_left_from >= 8)
+    {
+      vlib_prefetch_buffer_header (b[4], STORE);
+      vlib_prefetch_buffer_header (b[5], STORE);
+      vlib_prefetch_buffer_header (b[6], STORE);
+      vlib_prefetch_buffer_header (b[7], STORE);
+
+      memset (&is_drop, 0, sizeof (is_drop));
+      next[0] = next[1] = next[2] = next[3] = NSIM_NEXT_DROP;
+      if (PREDICT_FALSE (wp->cursize + 4 >= wp->wheel_size))
+	goto slow_path;
+      if (PREDICT_FALSE (nsm->drop_fraction != 0.0))
+	{
+	  rnd[0] = random_f64 (&nsm->seed);
+	  rnd[1] = random_f64 (&nsm->seed);
+	  rnd[2] = random_f64 (&nsm->seed);
+	  rnd[3] = random_f64 (&nsm->seed);
+
+	  if (rnd[0] <= nsm->drop_fraction)
+	    {
+	      b[0]->error = loss_error;
+	      is_drop[0] = 1;
+	    }
+	  if (rnd[1] <= nsm->drop_fraction)
+	    {
+	      b[1]->error = loss_error;
+	      is_drop[1] = 1;
+	    }
+	  if (rnd[2] <= nsm->drop_fraction)
+	    {
+	      b[2]->error = loss_error;
+	      is_drop[2] = 1;
+	    }
+	  if (rnd[3] <= nsm->drop_fraction)
+	    {
+	      b[3]->error = loss_error;
+	      is_drop[3] = 1;
+	    }
+	}
+
+      if (PREDICT_TRUE (is_drop[0] == 0))
+	{
+	  ep = wp->entries + wp->tail;
+	  wp->tail++;
+	  if (wp->tail == wp->wheel_size)
+	    wp->tail = 0;
+	  wp->cursize++;
+
+	  ep->tx_time = expires;
+	  ep->rx_sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
+	  if (is_cross_connect)
+	    {
+	      ep->tx_sw_if_index =
+		(vnet_buffer (b[0])->sw_if_index[VLIB_RX] ==
+		 nsm->sw_if_index0) ? nsm->sw_if_index1 : nsm->sw_if_index0;
+	      ep->output_next_index =
+		(ep->tx_sw_if_index ==
+		 nsm->sw_if_index0) ? nsm->
+		output_next_index0 : nsm->output_next_index1;
+	    }
+	  else			/* output feature, even easier... */
+	    {
+	      ep->tx_sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX];
+	      ep->output_next_index =
+		nsm->output_next_index_by_sw_if_index[ep->tx_sw_if_index];
+	    }
+	  ep->buffer_index = from[0];
+	  buffered++;
+	}
+
+      if (is_trace)
+	{
+	  if (b[1]->flags & VLIB_BUFFER_IS_TRACED)
+	    {
+	      nsim_trace_t *t = vlib_add_trace (vm, node, b[1], sizeof (*t));
+	      t->expires = expires;
+	      t->is_drop = is_drop[1];
+	      t->is_lost = b[1]->error == loss_error;
+	      t->tx_sw_if_index = (is_drop[1] == 0) ? ep->tx_sw_if_index : 0;
+	    }
+	}
+
+      if (PREDICT_TRUE (is_drop[1] == 0))
+	{
+	  ep = wp->entries + wp->tail;
+	  wp->tail++;
+	  if (wp->tail == wp->wheel_size)
+	    wp->tail = 0;
+	  wp->cursize++;
+
+	  ep->tx_time = expires;
+	  ep->rx_sw_if_index = vnet_buffer (b[1])->sw_if_index[VLIB_RX];
+	  if (is_cross_connect)
+	    {
+	      ep->tx_sw_if_index =
+		(vnet_buffer (b[1])->sw_if_index[VLIB_RX] ==
+		 nsm->sw_if_index0) ? nsm->sw_if_index1 : nsm->sw_if_index0;
+	      ep->output_next_index =
+		(ep->tx_sw_if_index ==
+		 nsm->sw_if_index0) ? nsm->
+		output_next_index0 : nsm->output_next_index1;
+	    }
+	  else			/* output feature, even easier... */
+	    {
+	      ep->tx_sw_if_index = vnet_buffer (b[1])->sw_if_index[VLIB_TX];
+	      ep->output_next_index =
+		nsm->output_next_index_by_sw_if_index[ep->tx_sw_if_index];
+	    }
+	  ep->buffer_index = from[1];
+	  buffered++;
+	}
+
+      if (is_trace)
+	{
+	  if (b[2]->flags & VLIB_BUFFER_IS_TRACED)
+	    {
+	      nsim_trace_t *t = vlib_add_trace (vm, node, b[2], sizeof (*t));
+	      t->expires = expires;
+	      t->is_drop = is_drop[2];
+	      t->is_lost = b[2]->error == loss_error;
+	      t->tx_sw_if_index = (is_drop[2] == 0) ? ep->tx_sw_if_index : 0;
+	    }
+	}
+      if (PREDICT_TRUE (is_drop[2] == 0))
+	{
+	  ep = wp->entries + wp->tail;
+	  wp->tail++;
+	  if (wp->tail == wp->wheel_size)
+	    wp->tail = 0;
+	  wp->cursize++;
+
+	  ep->tx_time = expires;
+	  ep->rx_sw_if_index = vnet_buffer (b[2])->sw_if_index[VLIB_RX];
+	  if (is_cross_connect)
+	    {
+	      ep->tx_sw_if_index =
+		(vnet_buffer (b[2])->sw_if_index[VLIB_RX] ==
+		 nsm->sw_if_index0) ? nsm->sw_if_index1 : nsm->sw_if_index0;
+	      ep->output_next_index =
+		(ep->tx_sw_if_index ==
+		 nsm->sw_if_index0) ? nsm->
+		output_next_index0 : nsm->output_next_index1;
+	    }
+	  else			/* output feature, even easier... */
+	    {
+	      ep->tx_sw_if_index = vnet_buffer (b[2])->sw_if_index[VLIB_TX];
+	      ep->output_next_index =
+		nsm->output_next_index_by_sw_if_index[ep->tx_sw_if_index];
+	    }
+	  ep->buffer_index = from[2];
+	  buffered++;
+	}
+
+      if (is_trace)
+	{
+	  if (b[2]->flags & VLIB_BUFFER_IS_TRACED)
+	    {
+	      nsim_trace_t *t = vlib_add_trace (vm, node, b[2], sizeof (*t));
+	      t->expires = expires;
+	      t->is_drop = is_drop[2];
+	      t->is_lost = b[2]->error == loss_error;
+	      t->tx_sw_if_index = (is_drop[2] == 0) ? ep->tx_sw_if_index : 0;
+	    }
+	}
+      if (PREDICT_TRUE (is_drop[3] == 0))
+	{
+	  ep = wp->entries + wp->tail;
+	  wp->tail++;
+	  if (wp->tail == wp->wheel_size)
+	    wp->tail = 0;
+	  wp->cursize++;
+
+	  ep->tx_time = expires;
+	  ep->rx_sw_if_index = vnet_buffer (b[3])->sw_if_index[VLIB_RX];
+	  if (is_cross_connect)
+	    {
+	      ep->tx_sw_if_index =
+		(vnet_buffer (b[3])->sw_if_index[VLIB_RX] ==
+		 nsm->sw_if_index0) ? nsm->sw_if_index1 : nsm->sw_if_index0;
+	      ep->output_next_index =
+		(ep->tx_sw_if_index ==
+		 nsm->sw_if_index0) ? nsm->
+		output_next_index0 : nsm->output_next_index1;
+	    }
+	  else			/* output feature, even easier... */
+	    {
+	      ep->tx_sw_if_index = vnet_buffer (b[3])->sw_if_index[VLIB_TX];
+	      ep->output_next_index =
+		nsm->output_next_index_by_sw_if_index[ep->tx_sw_if_index];
+	    }
+	  ep->buffer_index = from[3];
+	  buffered++;
+	}
+
+      if (is_trace)
+	{
+	  if (b[3]->flags & VLIB_BUFFER_IS_TRACED)
+	    {
+	      nsim_trace_t *t = vlib_add_trace (vm, node, b[3], sizeof (*t));
+	      t->expires = expires;
+	      t->is_drop = is_drop[3];
+	      t->is_lost = b[3]->error == loss_error;
+	      t->tx_sw_if_index = (is_drop[3] == 0) ? ep->tx_sw_if_index : 0;
+	    }
+	}
+
+      if (PREDICT_FALSE (is_drop[0]))
+	*drop++ = from[0];
+      if (PREDICT_FALSE (is_drop[1]))
+	*drop++ = from[1];
+      if (PREDICT_FALSE (is_drop[2]))
+	*drop++ = from[2];
+      if (PREDICT_FALSE (is_drop[3]))
+	*drop++ = from[3];
+
+      b += 4;
+      next += 4;
+      from += 4;
+      n_left_from -= 4;
+    }
+
+slow_path:
 
-  /* There is no point in trying to do more than 1 pkt here */
   while (n_left_from > 0)
     {
-      b[0]->error = no_error;
       next[0] = NSIM_NEXT_DROP;
-      is_drop0 = 0;
+      is_drop[0] = 0;
       if (PREDICT_TRUE (wp->cursize < wp->wheel_size))
 	{
 	  if (PREDICT_FALSE (nsm->drop_fraction != 0.0))
 	    {
 	      /* Get a random number on the closed interval [0,1] */
-	      f64 rnd = random_f64 (&nsm->seed);
+	      rnd[0] = random_f64 (&nsm->seed);
 	      /* Drop the pkt? */
-	      if (rnd <= nsm->drop_fraction)
+	      if (rnd[0] <= nsm->drop_fraction)
 		{
 		  b[0]->error = loss_error;
-		  is_drop0 = 1;
+		  is_drop[0] = 1;
 		  goto do_trace;
 		}
 	    }
@@ -133,18 +359,30 @@ nsim_inline (vlib_main_t * vm,
 	  wp->cursize++;
 
 	  ep->tx_time = expires;
-	  ep->tx_sw_if_index =
-	    (vnet_buffer (b[0])->sw_if_index[VLIB_RX] == nsm->sw_if_index0)
-	    ? nsm->sw_if_index1 : nsm->sw_if_index0;
-	  ep->current_length = vlib_buffer_length_in_chain (vm, b[0]);
-	  ASSERT (ep->current_length <= WHEEL_ENTRY_DATA_SIZE);
-	  clib_memcpy_fast (ep->data, vlib_buffer_get_current (b[0]),
-			    ep->current_length);
+	  ep->rx_sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
+	  if (is_cross_connect)
+	    {
+	      ep->tx_sw_if_index =
+		(vnet_buffer (b[0])->sw_if_index[VLIB_RX] ==
+		 nsm->sw_if_index0) ? nsm->sw_if_index1 : nsm->sw_if_index0;
+	      ep->output_next_index =
+		(ep->tx_sw_if_index ==
+		 nsm->sw_if_index0) ? nsm->
+		output_next_index0 : nsm->output_next_index1;
+	    }
+	  else			/* output feature, even easier... */
+	    {
+	      ep->tx_sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX];
+	      ep->output_next_index =
+		nsm->output_next_index_by_sw_if_index[ep->tx_sw_if_index];
+	    }
+	  ep->buffer_index = from[0];
+	  buffered++;
 	}
       else			/* out of wheel space, drop pkt */
 	{
 	  b[0]->error = no_buffer_error;
-	  is_drop0 = 1;
+	  is_drop[0] = 1;
 	}
 
     do_trace:
@@ -154,17 +392,42 @@ nsim_inline (vlib_main_t * vm,
 	    {
 	      nsim_trace_t *t = vlib_add_trace (vm, node, b[0], sizeof (*t));
 	      t->expires = expires;
-	      t->is_drop = is_drop0;
+	      t->is_drop = is_drop[0];
 	      t->is_lost = b[0]->error == loss_error;
-	      t->tx_sw_if_index = (is_drop0 == 0) ? ep->tx_sw_if_index : 0;
+	      t->tx_sw_if_index = (is_drop[0] == 0) ? ep->tx_sw_if_index : 0;
 	    }
 	}
 
       b += 1;
       next += 1;
+      if (PREDICT_FALSE (is_drop[0]))
+	{
+	  drop[0] = from[0];
+	  drop++;
+	}
+      from++;
       n_left_from -= 1;
     }
-  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
+  if (PREDICT_FALSE (drop > drops))
+    {
+      u32 n_left_to_drop = drop - drops;
+      drop = drops;
+
+      while (n_left_to_drop > 0)
+	{
+	  u32 this_copy_size;
+	  vlib_get_next_frame (vm, node, NSIM_NEXT_DROP, to_next,
+			       n_left_to_next);
+	  this_copy_size = clib_min (n_left_to_drop, n_left_to_next);
+	  clib_memcpy_fast (to_next, drop, this_copy_size * sizeof (u32));
+	  n_left_to_next -= this_copy_size;
+	  vlib_put_next_frame (vm, node, NSIM_NEXT_DROP, n_left_to_next);
+	  drop += this_copy_size;
+	  n_left_to_drop -= this_copy_size;
+	}
+    }
+  vlib_node_increment_counter (vm, node->node_index,
+			       NSIM_ERROR_BUFFERED, buffered);
   return frame->n_vectors;
 }
 
@@ -172,9 +435,11 @@ VLIB_NODE_FN (nsim_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
 			  vlib_frame_t * frame)
 {
   if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
-    return nsim_inline (vm, node, frame, 1 /* is_trace */ );
+    return nsim_inline (vm, node, frame,
+			1 /* is_trace */ , 1 /* is_cross_connect */ );
   else
-    return nsim_inline (vm, node, frame, 0 /* is_trace */ );
+    return nsim_inline (vm, node, frame,
+			0 /* is_trace */ , 1 /* is_cross_connect */ );
 }
 
 /* *INDENT-OFF* */
@@ -199,6 +464,40 @@ VLIB_REGISTER_NODE (nsim_node) =
 #endif /* CLIB_MARCH_VARIANT */
 /* *INDENT-ON* */
 
+VLIB_NODE_FN (nsim_feature_node) (vlib_main_t * vm,
+				  vlib_node_runtime_t * node,
+				  vlib_frame_t * frame)
+{
+  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
+    return nsim_inline (vm, node, frame,
+			1 /* is_trace */ , 0 /* is_cross_connect */ );
+  else
+    return nsim_inline (vm, node, frame,
+			0 /* is_trace */ , 0 /* is_cross_connect */ );
+}
+
+/* *INDENT-OFF* */
+#ifndef CLIB_MARCH_VARIANT
+VLIB_REGISTER_NODE (nsim_feature_node) =
+{
+  .name = "nsim-output-feature",
+  .vector_size = sizeof (u32),
+  .format_trace = format_nsim_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(nsim_error_strings),
+  .error_strings = nsim_error_strings,
+
+  .n_next_nodes = NSIM_N_NEXT,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+        [NSIM_NEXT_DROP] = "error-drop",
+  },
+};
+#endif /* CLIB_MARCH_VARIANT */
+/* *INDENT-ON* */
+
 /*
  * fd.io coding-style-patch-verification: ON
  *