Make the loss / delay sim available as an output feature

Add binary api and debug cli support.

Rewrite for speed: enqueue vlib_buffer_t's to the wheel, instead of
memcpy'ing data. Quad-loop the output feature / x-connect (interior)
node. Prefetch wheel entries in the input node.

Save packet-generator-based unit-test setup in extras/nsim.

Simple config example:

set nsim delay 20 ms bandwidth 1 gbit packet-size 1024
nsim output-feature enable-disable GigabitEthernet3/0/0

Change-Id: I852a32d4eb596e7e2aa1d9b30bf3b53525e39fd1
Signed-off-by: Dave Barach <dave@barachs.net>c

6 files changed, 629 insertions, 149 deletions

diff --git a/src/plugins/nsim/node.c b/src/plugins/nsim/node.c
index 25112abe299..559147b1280 100644
--- a/src/plugins/nsim/node.c
+++ b/src/plugins/nsim/node.c
@@ -80,20 +80,24 @@ typedef enum
 
 always_inline uword
 nsim_inline (vlib_main_t * vm,
-	     vlib_node_runtime_t * node, vlib_frame_t * frame, int is_trace)
+	     vlib_node_runtime_t * node, vlib_frame_t * frame, int is_trace,
+	     int is_cross_connect)
 {
   nsim_main_t *nsm = &nsim_main;
   u32 n_left_from, *from;
+  u32 *to_next, n_left_to_next;
+  u32 drops[VLIB_FRAME_SIZE], *drop;
   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
+  u8 is_drop[4];
   u16 nexts[VLIB_FRAME_SIZE], *next;
   u32 my_thread_index = vm->thread_index;
   nsim_wheel_t *wp = nsm->wheel_by_thread[my_thread_index];
   f64 now = vlib_time_now (vm);
   f64 expires = now + nsm->delay;
-  int is_drop0;
-  u32 no_error = node->errors[NSIM_ERROR_BUFFERED];
+  f64 rnd[4];
   u32 no_buffer_error = node->errors[NSIM_ERROR_DROPPED];
   u32 loss_error = node->errors[NSIM_ERROR_LOSS];
+  u32 buffered = 0;
   nsim_wheel_entry_t *ep = 0;
 
   ASSERT (wp);
@@ -104,24 +108,246 @@ nsim_inline (vlib_main_t * vm,
   vlib_get_buffers (vm, from, bufs, n_left_from);
   b = bufs;
   next = nexts;
+  drop = drops;
+
+  while (n_left_from >= 8)
+    {
+      vlib_prefetch_buffer_header (b[4], STORE);
+      vlib_prefetch_buffer_header (b[5], STORE);
+      vlib_prefetch_buffer_header (b[6], STORE);
+      vlib_prefetch_buffer_header (b[7], STORE);
+
+      memset (&is_drop, 0, sizeof (is_drop));
+      next[0] = next[1] = next[2] = next[3] = NSIM_NEXT_DROP;
+      if (PREDICT_FALSE (wp->cursize + 4 >= wp->wheel_size))
+	goto slow_path;
+      if (PREDICT_FALSE (nsm->drop_fraction != 0.0))
+	{
+	  rnd[0] = random_f64 (&nsm->seed);
+	  rnd[1] = random_f64 (&nsm->seed);
+	  rnd[2] = random_f64 (&nsm->seed);
+	  rnd[3] = random_f64 (&nsm->seed);
+
+	  if (rnd[0] <= nsm->drop_fraction)
+	    {
+	      b[0]->error = loss_error;
+	      is_drop[0] = 1;
+	    }
+	  if (rnd[1] <= nsm->drop_fraction)
+	    {
+	      b[1]->error = loss_error;
+	      is_drop[1] = 1;
+	    }
+	  if (rnd[2] <= nsm->drop_fraction)
+	    {
+	      b[2]->error = loss_error;
+	      is_drop[2] = 1;
+	    }
+	  if (rnd[3] <= nsm->drop_fraction)
+	    {
+	      b[3]->error = loss_error;
+	      is_drop[3] = 1;
+	    }
+	}
+
+      if (PREDICT_TRUE (is_drop[0] == 0))
+	{
+	  ep = wp->entries + wp->tail;
+	  wp->tail++;
+	  if (wp->tail == wp->wheel_size)
+	    wp->tail = 0;
+	  wp->cursize++;
+
+	  ep->tx_time = expires;
+	  ep->rx_sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
+	  if (is_cross_connect)
+	    {
+	      ep->tx_sw_if_index =
+		(vnet_buffer (b[0])->sw_if_index[VLIB_RX] ==
+		 nsm->sw_if_index0) ? nsm->sw_if_index1 : nsm->sw_if_index0;
+	      ep->output_next_index =
+		(ep->tx_sw_if_index ==
+		 nsm->sw_if_index0) ? nsm->
+		output_next_index0 : nsm->output_next_index1;
+	    }
+	  else			/* output feature, even easier... */
+	    {
+	      ep->tx_sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX];
+	      ep->output_next_index =
+		nsm->output_next_index_by_sw_if_index[ep->tx_sw_if_index];
+	    }
+	  ep->buffer_index = from[0];
+	  buffered++;
+	}
+
+      if (is_trace)
+	{
+	  if (b[1]->flags & VLIB_BUFFER_IS_TRACED)
+	    {
+	      nsim_trace_t *t = vlib_add_trace (vm, node, b[1], sizeof (*t));
+	      t->expires = expires;
+	      t->is_drop = is_drop[1];
+	      t->is_lost = b[1]->error == loss_error;
+	      t->tx_sw_if_index = (is_drop[1] == 0) ? ep->tx_sw_if_index : 0;
+	    }
+	}
+
+      if (PREDICT_TRUE (is_drop[1] == 0))
+	{
+	  ep = wp->entries + wp->tail;
+	  wp->tail++;
+	  if (wp->tail == wp->wheel_size)
+	    wp->tail = 0;
+	  wp->cursize++;
+
+	  ep->tx_time = expires;
+	  ep->rx_sw_if_index = vnet_buffer (b[1])->sw_if_index[VLIB_RX];
+	  if (is_cross_connect)
+	    {
+	      ep->tx_sw_if_index =
+		(vnet_buffer (b[1])->sw_if_index[VLIB_RX] ==
+		 nsm->sw_if_index0) ? nsm->sw_if_index1 : nsm->sw_if_index0;
+	      ep->output_next_index =
+		(ep->tx_sw_if_index ==
+		 nsm->sw_if_index0) ? nsm->
+		output_next_index0 : nsm->output_next_index1;
+	    }
+	  else			/* output feature, even easier... */
+	    {
+	      ep->tx_sw_if_index = vnet_buffer (b[1])->sw_if_index[VLIB_TX];
+	      ep->output_next_index =
+		nsm->output_next_index_by_sw_if_index[ep->tx_sw_if_index];
+	    }
+	  ep->buffer_index = from[1];
+	  buffered++;
+	}
+
+      if (is_trace)
+	{
+	  if (b[2]->flags & VLIB_BUFFER_IS_TRACED)
+	    {
+	      nsim_trace_t *t = vlib_add_trace (vm, node, b[2], sizeof (*t));
+	      t->expires = expires;
+	      t->is_drop = is_drop[2];
+	      t->is_lost = b[2]->error == loss_error;
+	      t->tx_sw_if_index = (is_drop[2] == 0) ? ep->tx_sw_if_index : 0;
+	    }
+	}
+      if (PREDICT_TRUE (is_drop[2] == 0))
+	{
+	  ep = wp->entries + wp->tail;
+	  wp->tail++;
+	  if (wp->tail == wp->wheel_size)
+	    wp->tail = 0;
+	  wp->cursize++;
+
+	  ep->tx_time = expires;
+	  ep->rx_sw_if_index = vnet_buffer (b[2])->sw_if_index[VLIB_RX];
+	  if (is_cross_connect)
+	    {
+	      ep->tx_sw_if_index =
+		(vnet_buffer (b[2])->sw_if_index[VLIB_RX] ==
+		 nsm->sw_if_index0) ? nsm->sw_if_index1 : nsm->sw_if_index0;
+	      ep->output_next_index =
+		(ep->tx_sw_if_index ==
+		 nsm->sw_if_index0) ? nsm->
+		output_next_index0 : nsm->output_next_index1;
+	    }
+	  else			/* output feature, even easier... */
+	    {
+	      ep->tx_sw_if_index = vnet_buffer (b[2])->sw_if_index[VLIB_TX];
+	      ep->output_next_index =
+		nsm->output_next_index_by_sw_if_index[ep->tx_sw_if_index];
+	    }
+	  ep->buffer_index = from[2];
+	  buffered++;
+	}
+
+      if (is_trace)
+	{
+	  if (b[2]->flags & VLIB_BUFFER_IS_TRACED)
+	    {
+	      nsim_trace_t *t = vlib_add_trace (vm, node, b[2], sizeof (*t));
+	      t->expires = expires;
+	      t->is_drop = is_drop[2];
+	      t->is_lost = b[2]->error == loss_error;
+	      t->tx_sw_if_index = (is_drop[2] == 0) ? ep->tx_sw_if_index : 0;
+	    }
+	}
+      if (PREDICT_TRUE (is_drop[3] == 0))
+	{
+	  ep = wp->entries + wp->tail;
+	  wp->tail++;
+	  if (wp->tail == wp->wheel_size)
+	    wp->tail = 0;
+	  wp->cursize++;
+
+	  ep->tx_time = expires;
+	  ep->rx_sw_if_index = vnet_buffer (b[3])->sw_if_index[VLIB_RX];
+	  if (is_cross_connect)
+	    {
+	      ep->tx_sw_if_index =
+		(vnet_buffer (b[3])->sw_if_index[VLIB_RX] ==
+		 nsm->sw_if_index0) ? nsm->sw_if_index1 : nsm->sw_if_index0;
+	      ep->output_next_index =
+		(ep->tx_sw_if_index ==
+		 nsm->sw_if_index0) ? nsm->
+		output_next_index0 : nsm->output_next_index1;
+	    }
+	  else			/* output feature, even easier... */
+	    {
+	      ep->tx_sw_if_index = vnet_buffer (b[3])->sw_if_index[VLIB_TX];
+	      ep->output_next_index =
+		nsm->output_next_index_by_sw_if_index[ep->tx_sw_if_index];
+	    }
+	  ep->buffer_index = from[3];
+	  buffered++;
+	}
+
+      if (is_trace)
+	{
+	  if (b[3]->flags & VLIB_BUFFER_IS_TRACED)
+	    {
+	      nsim_trace_t *t = vlib_add_trace (vm, node, b[3], sizeof (*t));
+	      t->expires = expires;
+	      t->is_drop = is_drop[3];
+	      t->is_lost = b[3]->error == loss_error;
+	      t->tx_sw_if_index = (is_drop[3] == 0) ? ep->tx_sw_if_index : 0;
+	    }
+	}
+
+      if (PREDICT_FALSE (is_drop[0]))
+	*drop++ = from[0];
+      if (PREDICT_FALSE (is_drop[1]))
+	*drop++ = from[1];
+      if (PREDICT_FALSE (is_drop[2]))
+	*drop++ = from[2];
+      if (PREDICT_FALSE (is_drop[3]))
+	*drop++ = from[3];
+
+      b += 4;
+      next += 4;
+      from += 4;
+      n_left_from -= 4;
+    }
+
+slow_path:
 
-  /* There is no point in trying to do more than 1 pkt here */
   while (n_left_from > 0)
     {
-      b[0]->error = no_error;
       next[0] = NSIM_NEXT_DROP;
-      is_drop0 = 0;
+      is_drop[0] = 0;
       if (PREDICT_TRUE (wp->cursize < wp->wheel_size))
 	{
 	  if (PREDICT_FALSE (nsm->drop_fraction != 0.0))
 	    {
 	      /* Get a random number on the closed interval [0,1] */
-	      f64 rnd = random_f64 (&nsm->seed);
+	      rnd[0] = random_f64 (&nsm->seed);
 	      /* Drop the pkt? */
-	      if (rnd <= nsm->drop_fraction)
+	      if (rnd[0] <= nsm->drop_fraction)
 		{
 		  b[0]->error = loss_error;
-		  is_drop0 = 1;
+		  is_drop[0] = 1;
 		  goto do_trace;
 		}
 	    }
@@ -133,18 +359,30 @@ nsim_inline (vlib_main_t * vm,
 	  wp->cursize++;
 
 	  ep->tx_time = expires;
-	  ep->tx_sw_if_index =
-	    (vnet_buffer (b[0])->sw_if_index[VLIB_RX] == nsm->sw_if_index0)
-	    ? nsm->sw_if_index1 : nsm->sw_if_index0;
-	  ep->current_length = vlib_buffer_length_in_chain (vm, b[0]);
-	  ASSERT (ep->current_length <= WHEEL_ENTRY_DATA_SIZE);
-	  clib_memcpy_fast (ep->data, vlib_buffer_get_current (b[0]),
-			    ep->current_length);
+	  ep->rx_sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
+	  if (is_cross_connect)
+	    {
+	      ep->tx_sw_if_index =
+		(vnet_buffer (b[0])->sw_if_index[VLIB_RX] ==
+		 nsm->sw_if_index0) ? nsm->sw_if_index1 : nsm->sw_if_index0;
+	      ep->output_next_index =
+		(ep->tx_sw_if_index ==
+		 nsm->sw_if_index0) ? nsm->
+		output_next_index0 : nsm->output_next_index1;
+	    }
+	  else			/* output feature, even easier... */
+	    {
+	      ep->tx_sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX];
+	      ep->output_next_index =
+		nsm->output_next_index_by_sw_if_index[ep->tx_sw_if_index];
+	    }
+	  ep->buffer_index = from[0];
+	  buffered++;
 	}
       else			/* out of wheel space, drop pkt */
 	{
 	  b[0]->error = no_buffer_error;
-	  is_drop0 = 1;
+	  is_drop[0] = 1;
 	}
 
     do_trace:
@@ -154,17 +392,42 @@ nsim_inline (vlib_main_t * vm,
 	    {
 	      nsim_trace_t *t = vlib_add_trace (vm, node, b[0], sizeof (*t));
 	      t->expires = expires;
-	      t->is_drop = is_drop0;
+	      t->is_drop = is_drop[0];
 	      t->is_lost = b[0]->error == loss_error;
-	      t->tx_sw_if_index = (is_drop0 == 0) ? ep->tx_sw_if_index : 0;
+	      t->tx_sw_if_index = (is_drop[0] == 0) ? ep->tx_sw_if_index : 0;
 	    }
 	}
 
       b += 1;
       next += 1;
+      if (PREDICT_FALSE (is_drop[0]))
+	{
+	  drop[0] = from[0];
+	  drop++;
+	}
+      from++;
       n_left_from -= 1;
     }
-  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
+  if (PREDICT_FALSE (drop > drops))
+    {
+      u32 n_left_to_drop = drop - drops;
+      drop = drops;
+
+      while (n_left_to_drop > 0)
+	{
+	  u32 this_copy_size;
+	  vlib_get_next_frame (vm, node, NSIM_NEXT_DROP, to_next,
+			       n_left_to_next);
+	  this_copy_size = clib_min (n_left_to_drop, n_left_to_next);
+	  clib_memcpy_fast (to_next, drop, this_copy_size * sizeof (u32));
+	  n_left_to_next -= this_copy_size;
+	  vlib_put_next_frame (vm, node, NSIM_NEXT_DROP, n_left_to_next);
+	  drop += this_copy_size;
+	  n_left_to_drop -= this_copy_size;
+	}
+    }
+  vlib_node_increment_counter (vm, node->node_index,
+			       NSIM_ERROR_BUFFERED, buffered);
   return frame->n_vectors;
 }
 
@@ -172,9 +435,11 @@ VLIB_NODE_FN (nsim_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
 			  vlib_frame_t * frame)
 {
   if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
-    return nsim_inline (vm, node, frame, 1 /* is_trace */ );
+    return nsim_inline (vm, node, frame,
+			1 /* is_trace */ , 1 /* is_cross_connect */ );
   else
-    return nsim_inline (vm, node, frame, 0 /* is_trace */ );
+    return nsim_inline (vm, node, frame,
+			0 /* is_trace */ , 1 /* is_cross_connect */ );
 }
 
 /* *INDENT-OFF* */
@@ -199,6 +464,40 @@ VLIB_REGISTER_NODE (nsim_node) =
 #endif /* CLIB_MARCH_VARIANT */
 /* *INDENT-ON* */
 
+VLIB_NODE_FN (nsim_feature_node) (vlib_main_t * vm,
+				  vlib_node_runtime_t * node,
+				  vlib_frame_t * frame)
+{
+  if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
+    return nsim_inline (vm, node, frame,
+			1 /* is_trace */ , 0 /* is_cross_connect */ );
+  else
+    return nsim_inline (vm, node, frame,
+			0 /* is_trace */ , 0 /* is_cross_connect */ );
+}
+
+/* *INDENT-OFF* */
+#ifndef CLIB_MARCH_VARIANT
+VLIB_REGISTER_NODE (nsim_feature_node) =
+{
+  .name = "nsim-output-feature",
+  .vector_size = sizeof (u32),
+  .format_trace = format_nsim_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+
+  .n_errors = ARRAY_LEN(nsim_error_strings),
+  .error_strings = nsim_error_strings,
+
+  .n_next_nodes = NSIM_N_NEXT,
+
+  /* edit / add dispositions here */
+  .next_nodes = {
+        [NSIM_NEXT_DROP] = "error-drop",
+  },
+};
+#endif /* CLIB_MARCH_VARIANT */
+/* *INDENT-ON* */
+
 /*
  * fd.io coding-style-patch-verification: ON
  *
diff --git a/src/plugins/nsim/nsim.api b/src/plugins/nsim/nsim.api
index 7bb84bafacd..42531cd2e0f 100644
--- a/src/plugins/nsim/nsim.api
+++ b/src/plugins/nsim/nsim.api
@@ -3,7 +3,7 @@
  * @brief VPP control-plane API messages for the network delay simulator
  */
 
-option version = "1.1.0";
+option version = "2.1.0";
 
 /** \brief enable / disable the network delay simulation cross-connect
     @param client_index - opaque cookie to identify the sender
@@ -12,7 +12,7 @@ option version = "1.1.0";
     @param sw_if_index0 - one interface to cross-connect
     @param sw_if_index1 - the other interface to cross-connect
 */
-autoreply define nsim_enable_disable 
+autoreply define nsim_cross_connect_enable_disable 
 {
   /* Client identifier, set from api_main.my_client_index */
   u32 client_index;
@@ -28,6 +28,27 @@ autoreply define nsim_enable_disable
   u32 sw_if_index1;
 };
 
+/** \brief enable / disable the network delay simulation output feature
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param enable_disable - enable or disable the feature
+    @param sw_if_index0 - interface 
+*/
+autoreply define nsim_output_feature_enable_disable 
+{
+  /* Client identifier, set from api_main.my_client_index */
+  u32 client_index;
+  
+  /* Arbitrary context, so client can match reply to request */
+  u32 context;
+  
+  /* Enable / disable the feature on the interfaces */
+  u8 enable_disable;
+  
+  /* Interface handles */
+  u32 sw_if_index;
+};
+
 /** \brief configure the network delay simulation cross-connect
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
@@ -49,3 +70,5 @@ autoreply define nsim_configure
   u64 bandwidth_in_bits_per_second;
   u32 packets_per_drop;
 };
+
+
diff --git a/src/plugins/nsim/nsim.c b/src/plugins/nsim/nsim.c
index ce3396c5845..4120585c282 100644
--- a/src/plugins/nsim/nsim.c
+++ b/src/plugins/nsim/nsim.c
@@ -60,15 +60,16 @@ nsim_main_t nsim_main;
 
 /* List of message types that this plugin understands */
 
-#define foreach_nsim_plugin_api_msg             \
-_(NSIM_ENABLE_DISABLE, nsim_enable_disable)     \
+#define foreach_nsim_plugin_api_msg                                     \
+_(NSIM_CROSS_CONNECT_ENABLE_DISABLE, nsim_cross_connect_enable_disable) \
+_(NSIM_OUTPUT_FEATURE_ENABLE_DISABLE, nsim_output_feature_enable_disable) \
 _(NSIM_CONFIGURE, nsim_configure)
 
-/* Action function shared between message handler and debug CLI */
+/* Action functions shared between message handlers and debug CLI */
 
 int
-nsim_enable_disable (nsim_main_t * nsm, u32 sw_if_index0,
-		     u32 sw_if_index1, int enable_disable)
+nsim_cross_connect_enable_disable (nsim_main_t * nsm, u32 sw_if_index0,
+				   u32 sw_if_index1, int enable_disable)
 {
   vnet_sw_interface_t *sw;
   vnet_hw_interface_t *hw;
@@ -117,6 +118,41 @@ nsim_enable_disable (nsim_main_t * nsm, u32 sw_if_index0,
   return rv;
 }
 
+int
+nsim_output_feature_enable_disable (nsim_main_t * nsm, u32 sw_if_index,
+				    int enable_disable)
+{
+  vnet_sw_interface_t *sw;
+  vnet_hw_interface_t *hw;
+  int rv = 0;
+
+  if (nsm->is_configured == 0)
+    return VNET_API_ERROR_CANNOT_ENABLE_DISABLE_FEATURE;
+
+  /* Utterly wrong? */
+  if (pool_is_free_index (nsm->vnet_main->interface_main.sw_interfaces,
+			  sw_if_index))
+    return VNET_API_ERROR_INVALID_SW_IF_INDEX;
+
+  /* Not a physical port? */
+  sw = vnet_get_sw_interface (nsm->vnet_main, sw_if_index);
+  if (sw->type != VNET_SW_INTERFACE_TYPE_HARDWARE)
+    return VNET_API_ERROR_INVALID_SW_IF_INDEX;
+
+  /* Add a graph arc for the input / wheel scraper node */
+  hw = vnet_get_hw_interface (nsm->vnet_main, sw_if_index);
+  vec_validate_init_empty (nsm->output_next_index_by_sw_if_index, sw_if_index,
+			   ~0);
+  /* Note: use the tx node, this pkt has already visited the output node... */
+  nsm->output_next_index_by_sw_if_index[sw_if_index] =
+    vlib_node_add_next (nsm->vlib_main, nsim_input_node.index,
+			hw->tx_node_index);
+
+  vnet_feature_enable_disable ("interface-output", "nsim-output-feature",
+			       sw_if_index, enable_disable, 0, 0);
+  return rv;
+}
+
 static int
 nsim_configure (nsim_main_t * nsm, f64 bandwidth, f64 delay, f64 packet_size,
 		f64 drop_fraction)
@@ -134,7 +170,7 @@ nsim_configure (nsim_main_t * nsm, f64 bandwidth, f64 delay, f64 packet_size,
   if (delay == 0.0)
     return VNET_API_ERROR_INVALID_VALUE_2;
 
-  if (packet_size < 64.0 || packet_size > (f64) WHEEL_ENTRY_DATA_SIZE)
+  if (packet_size < 64.0 || packet_size > 9000.0)
     return VNET_API_ERROR_INVALID_VALUE_3;
 
   /* Toss the old wheel(s)... */
@@ -171,7 +207,6 @@ nsim_configure (nsim_main_t * nsm, f64 bandwidth, f64 delay, f64 packet_size,
   nsm->packet_size = packet_size;
 
   vec_validate (nsm->wheel_by_thread, num_workers);
-  vec_validate (nsm->buffer_indices_by_thread, num_workers);
 
   /* Initialize the output scheduler wheels */
   for (i = num_workers ? 1 : 0; i < num_workers + 1; i++)
@@ -192,8 +227,6 @@ nsim_configure (nsim_main_t * nsm, f64 bandwidth, f64 delay, f64 packet_size,
       wp->tail = 0;
       wp->entries = (void *) (wp + 1);
       nsm->wheel_by_thread[i] = wp;
-      vec_validate (nsm->buffer_indices_by_thread[i], VLIB_FRAME_SIZE - 1);
-      _vec_len (nsm->buffer_indices_by_thread[i]) = 0;
     }
 
   vlib_worker_thread_barrier_sync (vm);
@@ -217,22 +250,27 @@ nsim_configure (nsim_main_t * nsm, f64 bandwidth, f64 delay, f64 packet_size,
  * enable or disable the cross-connect
  */
 static clib_error_t *
-nsim_enable_disable_command_fn (vlib_main_t * vm,
-				unformat_input_t * input,
-				vlib_cli_command_t * cmd)
+nsim_cross_connect_enable_disable_command_fn (vlib_main_t * vm,
+					      unformat_input_t * input,
+					      vlib_cli_command_t * cmd)
 {
   nsim_main_t *nsm = &nsim_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
   u32 sw_if_index0 = ~0;
   u32 sw_if_index1 = ~0;
   int enable_disable = 1;
   u32 tmp;
   int rv;
 
-  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
     {
-      if (unformat (input, "disable"))
+      if (unformat (line_input, "disable"))
 	enable_disable = 0;
-      else if (unformat (input, "%U", unformat_vnet_sw_interface,
+      else if (unformat (line_input, "%U", unformat_vnet_sw_interface,
 			 nsm->vnet_main, &tmp))
 	{
 	  if (sw_if_index0 == ~0)
@@ -244,10 +282,13 @@ nsim_enable_disable_command_fn (vlib_main_t * vm,
 	break;
     }
 
+  unformat_free (line_input);
+
   if (sw_if_index0 == ~0 || sw_if_index1 == ~0)
     return clib_error_return (0, "Please specify two interfaces...");
 
-  rv = nsim_enable_disable (nsm, sw_if_index0, sw_if_index1, enable_disable);
+  rv = nsim_cross_connect_enable_disable (nsm, sw_if_index0,
+					  sw_if_index1, enable_disable);
 
   switch (rv)
     {
@@ -284,34 +325,49 @@ nsim_enable_disable_command_fn (vlib_main_t * vm,
  * @cliexpar
  * To enable or disable network simulation cross-connect
  * @clistart
- * nsim enable-disable TenGigabitEthernet2/0/0 TenGigabitEthernet2/0
- * nsim enable-disable TenGigabitEthernet2/0/0 TenGigabitEthernet2/0 disable
+ * nsim cross-connect enable-disable TenGigabitEthernet2/0/0 TenGigabitEthernet2/0
+ * nsim cross-connect enable-disable TenGigabitEthernet2/0/0 TenGigabitEthernet2/0 disable
  * @cliend
  * @cliexcmd{nsim enable-disable <intfc> <intfc> [disable]}
 ?*/
 /* *INDENT-OFF* */
 VLIB_CLI_COMMAND (nsim_enable_disable_command, static) =
 {
-  .path = "nsim enable-disable",
+  .path = "nsim cross-connect enable-disable",
   .short_help =
-  "nsim enable-disable <interface-name-1> <interface-name-2> [disable]",
-  .function = nsim_enable_disable_command_fn,
+  "nsim cross-connect enable-disable <interface-name-1> "
+  "<interface-name-2> [disable]",
+  .function = nsim_cross_connect_enable_disable_command_fn,
 };
 /* *INDENT-ON* */
 
 /* API message handler */
-static void vl_api_nsim_enable_disable_t_handler
-  (vl_api_nsim_enable_disable_t * mp)
+static void vl_api_nsim_cross_connect_enable_disable_t_handler
+  (vl_api_nsim_cross_connect_enable_disable_t * mp)
 {
-  vl_api_nsim_enable_disable_reply_t *rmp;
+  vl_api_nsim_cross_connect_enable_disable_reply_t *rmp;
   nsim_main_t *nsm = &nsim_main;
   int rv;
 
-  rv = nsim_enable_disable (nsm, ntohl (mp->sw_if_index0),
-			    ntohl (mp->sw_if_index1),
-			    (int) (mp->enable_disable));
+  rv = nsim_cross_connect_enable_disable (nsm, ntohl (mp->sw_if_index0),
+					  ntohl (mp->sw_if_index1),
+					  (int) (mp->enable_disable));
 
-  REPLY_MACRO (VL_API_NSIM_ENABLE_DISABLE_REPLY);
+  REPLY_MACRO (VL_API_NSIM_CROSS_CONNECT_ENABLE_DISABLE_REPLY);
+}
+
+/* API message handler */
+static void vl_api_nsim_output_feature_enable_disable_t_handler
+  (vl_api_nsim_output_feature_enable_disable_t * mp)
+{
+  vl_api_nsim_output_feature_enable_disable_reply_t *rmp;
+  nsim_main_t *nsm = &nsim_main;
+  int rv;
+
+  rv = nsim_output_feature_enable_disable (nsm, ntohl (mp->sw_if_index),
+					   (int) (mp->enable_disable));
+
+  REPLY_MACRO (VL_API_NSIM_OUTPUT_FEATURE_ENABLE_DISABLE_REPLY);
 }
 
 /* API message handler */
@@ -340,6 +396,96 @@ vl_api_nsim_configure_t_handler (vl_api_nsim_configure_t * mp)
 }
 
 
+/*
+ * enable or disable the output_feature
+ */
+static clib_error_t *
+nsim_output_feature_enable_disable_command_fn (vlib_main_t * vm,
+					       unformat_input_t * input,
+					       vlib_cli_command_t * cmd)
+{
+  nsim_main_t *nsm = &nsim_main;
+  unformat_input_t _line_input, *line_input = &_line_input;
+  u32 sw_if_index = ~0;
+  int enable_disable = 1;
+  int rv;
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "disable"))
+	enable_disable = 0;
+      else if (unformat (line_input, "%U", unformat_vnet_sw_interface,
+			 nsm->vnet_main, &sw_if_index))
+	;
+      else
+	{
+	  clib_error_t *error = clib_error_return (0, "unknown input `%U'",
+						   format_unformat_error,
+						   line_input);
+	  unformat_free (line_input);
+	  return error;
+	}
+    }
+
+  unformat_free (line_input);
+
+  if (sw_if_index == ~0)
+    return clib_error_return (0, "Please specify one interface...");
+
+  rv = nsim_output_feature_enable_disable (nsm, sw_if_index, enable_disable);
+
+  switch (rv)
+    {
+    case 0:
+      break;
+
+    case VNET_API_ERROR_CANNOT_ENABLE_DISABLE_FEATURE:
+      return clib_error_return (0, "Not configured, please 'set nsim' first");
+
+    case VNET_API_ERROR_INVALID_SW_IF_INDEX:
+      return clib_error_return
+	(0, "Invalid interface, only works on physical ports");
+      break;
+
+    case VNET_API_ERROR_UNIMPLEMENTED:
+      return clib_error_return (0,
+				"Device driver doesn't support redirection");
+      break;
+
+    default:
+      return clib_error_return
+	(0, "nsim_output_feature_enable_disable returned %d", rv);
+    }
+  return 0;
+}
+
+/*?
+ * Enable or disable network simulation output feature on an interface
+ * The network simulator must have already been configured, see
+ * the "nsim_configure" command.
+ *
+ * @cliexpar
+ * To enable or disable network simulation output feature
+ * @clistart
+ * nsim output-feature enable-disable TenGigabitEthernet2/0/0
+ * nsim output-feature enable-disable TenGigabitEthernet2/0/0 disable
+ * @cliend
+ * @cliexcmd{nsim output-feature enable-disable <intfc> [disable]}
+?*/
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (nsim_output_feature_enable_disable_command, static) =
+{
+  .path = "nsim output-feature enable-disable",
+  .short_help =
+  "nsim output-feature enable-disable <interface-name> [disable]",
+  .function = nsim_output_feature_enable_disable_command_fn,
+};
+/* *INDENT-ON* */
+
 /* Set up the API message handling tables */
 static clib_error_t *
 nsim_plugin_api_hookup (vlib_main_t * vm)
@@ -409,6 +555,15 @@ VNET_FEATURE_INIT (nsim, static) =
 /* *INDENT-ON */
 
 /* *INDENT-OFF* */
+VNET_FEATURE_INIT (nsim_feature, static) =
+{
+  .arc_name = "interface-output",
+  .node_name = "nsim-output-feature",
+  .runs_before = VNET_FEATURES ("interface-tx"),
+};
+/* *INDENT-ON */
+
+/* *INDENT-OFF* */
 VLIB_PLUGIN_REGISTER () =
 {
   .version = VPP_BUILD_VER,
@@ -541,7 +696,8 @@ set_nsim_command_fn (vlib_main_t * vm,
 VLIB_CLI_COMMAND (set_nsim_command, static) =
 {
   .path = "set nsim",
-  .short_help = "set nsim delay <time> bandwidth <bps> packet-size <nbytes>",
+  .short_help = "set nsim delay <time> bandwidth <bps> packet-size <nbytes>\n"
+  "    [packets-per-drop <nn>][drop-fraction <f64: 0.0 - 1.0>]",
   .function = set_nsim_command_fn,
 };
 /* *INDENT-ON*/
diff --git a/src/plugins/nsim/nsim.h b/src/plugins/nsim/nsim.h
index c5264ecb244..6afe32dc7f7 100644
--- a/src/plugins/nsim/nsim.h
+++ b/src/plugins/nsim/nsim.h
@@ -25,15 +25,14 @@
 #include <vppinfra/hash.h>
 #include <vppinfra/error.h>
 
-#define WHEEL_ENTRY_DATA_SIZE 1536	/* an even multiple of 64, pls */
-
 typedef struct
 {
   f64 tx_time;
+  u32 rx_sw_if_index;
   u32 tx_sw_if_index;
-  u32 current_length;
-    CLIB_CACHE_LINE_ALIGN_MARK (pad);
-  u8 data[WHEEL_ENTRY_DATA_SIZE];
+  u32 output_next_index;
+  u32 buffer_index;
+  u32 pad;			/* pad to 32-bytes */
 } nsim_wheel_entry_t;
 
 typedef struct
@@ -54,12 +53,15 @@ typedef struct
   /* Two interfaces, cross-connected with delay */
   u32 sw_if_index0, sw_if_index1;
   u32 output_next_index0, output_next_index1;
+
+  /* N interfaces, using the output feature */
+  u32 *output_next_index_by_sw_if_index;
+
   /* Random seed for loss-rate simulation */
   u32 seed;
 
-  /* Per-thread buffer / scheduler wheels */
+  /* Per-thread scheduler wheels */
   nsim_wheel_t **wheel_by_thread;
-  u32 **buffer_indices_by_thread;
 
   /* Config parameters */
   f64 delay;
diff --git a/src/plugins/nsim/nsim_input.c b/src/plugins/nsim/nsim_input.c
index 44c9f535f92..def6fcd9f26 100644
--- a/src/plugins/nsim/nsim_input.c
+++ b/src/plugins/nsim/nsim_input.c
@@ -24,7 +24,7 @@
 typedef struct
 {
   f64 expired;
-  u32 tx_sw_if_index;
+  u32 next_index;
 } nsim_tx_trace_t;
 
 #ifndef CLIB_MARCH_VARIANT
@@ -36,22 +36,20 @@ format_nsim_tx_trace (u8 * s, va_list * args)
   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
   nsim_tx_trace_t *t = va_arg (*args, nsim_tx_trace_t *);
 
-  s = format (s, "NSIM: tx at %.6f sw_if_index %d",
-	      t->expired, t->tx_sw_if_index);
+  s = format (s, "NSIM: tx at %.6f next_index %d", t->expired, t->next_index);
   return s;
 }
 #endif /* CLIB_MARCH_VARIANT */
 
-#define foreach_nsim_tx_error                      \
-_(TX, "Packets transmitted")                    \
-_(DROPPED, "No buffer drops")
+#define foreach_nsim_tx_error                   \
+_(TRANSMITTED, "Packets transmitted")
 
 typedef enum
 {
 #define _(sym,str) NSIM_TX_ERROR_##sym,
   foreach_nsim_tx_error
 #undef _
-    NSIM_N_ERROR,
+    NSIM_TX_N_ERROR,
 } nsim_tx_error_t;
 
 #ifndef CLIB_MARCH_VARIANT
@@ -74,12 +72,9 @@ nsim_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 {
   nsim_main_t *nsm = &nsim_main;
   u32 my_thread_index = vm->thread_index;
-  u32 *my_buffer_cache = nsm->buffer_indices_by_thread[my_thread_index];
   nsim_wheel_t *wp = nsm->wheel_by_thread[my_thread_index];
-  u32 n_trace = vlib_get_trace_count (vm, node);
   f64 now = vlib_time_now (vm);
-  uword n_rx_packets = 0;
-  vlib_buffer_t *b0;
+  uword n_tx_packets = 0;
   u32 bi0, next0;
   u32 *to_next;
   u32 next_index;
@@ -93,85 +88,30 @@ nsim_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   /* First entry on the wheel isn't expired? */
   ep = wp->entries + wp->head;
   if (ep->tx_time > now)
-    return n_rx_packets;
+    return n_tx_packets;
 
-  /*
-   * We use per-thread buffer caches, so we need the freelist to
-   * initialize them...
-   */
   next_index = node->cached_next_index;
 
-  while (wp->cursize)
+  /* Be aware: this is not the usual coding pattern */
+  while (wp->cursize && ep->tx_time <= now)
     {
-      /* Be aware: this is not the usual coding pattern */
+      vlib_buffer_t *b0;
+
       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 
-      while (n_left_to_next > 0 && ep->tx_time <= now)
+      while (n_left_to_next > 0)
 	{
-	  /* Out of local buffer cache? */
-	  if (PREDICT_FALSE (_vec_len (my_buffer_cache) == 0))
-	    {
-	      u32 n =
-		vlib_buffer_alloc (vm, my_buffer_cache, VLIB_FRAME_SIZE);
-	      _vec_len (my_buffer_cache) = n;
-
-	      /* Ugh, drop the rest of the expired entries */
-	      if (n == 0)
-		{
-		  u32 drops = 0;
-		  while (ep->tx_time <= now && wp->cursize)
-		    {
-		      wp->head++;
-		      if (wp->head == wp->wheel_size)
-			wp->head = 0;
-		      ep = wp->entries + wp->head;
-		      wp->cursize--;
-		      drops++;
-		    }
-		  /* Count the drops */
-		  vlib_node_increment_counter (vm, node->node_index,
-					       NSIM_TX_ERROR_DROPPED, drops);
-		  /* Ship any pkts we already processed */
-		  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-		  return n_rx_packets + drops;
-		}
-	    }
-
-	  /* Allocate a buffer */
-	  bi0 = my_buffer_cache[_vec_len (my_buffer_cache) - 1];
-	  _vec_len (my_buffer_cache) -= 1;
+	  /* prefetch one line / 2 entries ahead */
+	  if ((((uword) ep) & (CLIB_CACHE_LINE_BYTES - 1)) == 0)
+	    CLIB_PREFETCH ((ep + 2), CLIB_CACHE_LINE_BYTES, LOAD);
+	  /* Pick up buffer from the wheel */
+	  bi0 = ep->buffer_index;
 
 	  to_next[0] = bi0;
 	  to_next += 1;
 	  n_left_to_next -= 1;
 
-	  b0 = vlib_get_buffer (vm, bi0);
-	  /* Initialize the buffer */
-
-	  b0->current_data = 0;
-	  b0->current_length = ep->current_length;
-
-	  VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0);
-
-	  if (PREDICT_FALSE (n_trace))
-	    {
-	      nsim_tx_trace_t *t0;
-	      vlib_trace_buffer (vm, node, next_index, b0,
-				 0 /* follow_chain */ );
-	      t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
-	      t0->expired = ep->tx_time;
-	      t0->tx_sw_if_index = ep->tx_sw_if_index;
-	    }
-
-	  /* Copy data from the ring */
-	  clib_memcpy_fast (b0->data, ep->data, ep->current_length);
-	  b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
-	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = ep->tx_sw_if_index;
-	  vnet_buffer (b0)->sw_if_index[VLIB_RX] =
-	    (ep->tx_sw_if_index == nsm->sw_if_index0) ? nsm->sw_if_index1 :
-	    nsm->sw_if_index0;
-	  next0 = (ep->tx_sw_if_index == nsm->sw_if_index0) ?
-	    nsm->output_next_index0 : nsm->output_next_index1;
+	  next0 = ep->output_next_index;
 
 	  /* verify speculative enqueue, maybe switch current next frame */
 	  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
@@ -183,7 +123,19 @@ nsim_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	    wp->head = 0;
 	  wp->cursize--;
 	  ep = wp->entries + wp->head;
-	  n_rx_packets++;
+	  n_tx_packets++;
+
+	  if (is_trace)
+	    {
+	      b0 = vlib_get_buffer (vm, ep->buffer_index);
+	      if (b0->flags & VLIB_BUFFER_IS_TRACED)
+		{
+		  nsim_tx_trace_t *t =
+		    vlib_add_trace (vm, node, b0, sizeof (*t));
+		  t->expired = now;
+		  t->next_index = next0;
+		}
+	    }
 
 	  /* Out of ring entries? */
 	  if (PREDICT_FALSE (wp->cursize == 0))
@@ -196,7 +148,9 @@ nsim_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       if (ep->tx_time > now)
 	break;
     }
-  return n_rx_packets;
+  vlib_node_increment_counter (vm, node->node_index,
+			       NSIM_TX_ERROR_TRANSMITTED, n_tx_packets);
+  return n_tx_packets;
 }
 
 VLIB_NODE_FN (nsim_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
@@ -221,7 +175,7 @@ VLIB_REGISTER_NODE (nsim_input_node) =
 
   .format_trace = format_nsim_tx_trace,
 
-  .n_errors = NSIM_N_ERROR,
+  .n_errors = NSIM_TX_N_ERROR,
   .error_strings = nsim_tx_error_strings,
 };
 #endif /* CLIB_MARCH_VARIANT */
diff --git a/src/plugins/nsim/nsim_test.c b/src/plugins/nsim/nsim_test.c
index 7123703fd42..35f222eadf0 100644
--- a/src/plugins/nsim/nsim_test.c
+++ b/src/plugins/nsim/nsim_test.c
@@ -60,7 +60,8 @@ nsim_test_main_t nsim_test_main;
 #include <vlibapi/vat_helper_macros.h>
 
 #define foreach_standard_reply_retval_handler   \
-_(nsim_enable_disable_reply)                    \
+_(nsim_cross_connect_enable_disable_reply)      \
+_(nsim_output_feature_enable_disable_reply)     \
 _(nsim_configure_reply)
 
 #define _(n)                                            \
@@ -83,19 +84,22 @@ foreach_standard_reply_retval_handler;
  * Table of message reply handlers, must include boilerplate handlers
  * we just generated
  */
-#define foreach_vpe_api_reply_msg                       \
-_(NSIM_ENABLE_DISABLE_REPLY, nsim_enable_disable_reply) \
+#define foreach_vpe_api_reply_msg               \
+_(NSIM_CROSS_CONNECT_ENABLE_DISABLE_REPLY,      \
+nsim_cross_connect_enable_disable_reply)        \
+_(NSIM_OUTPUT_FEATURE_ENABLE_DISABLE_REPLY,     \
+nsim_output_feature_enable_disable_reply)       \
 _(NSIM_CONFIGURE_REPLY, nsim_configure_reply)
 
 static int
-api_nsim_enable_disable (vat_main_t * vam)
+api_nsim_cross_connect_enable_disable (vat_main_t * vam)
 {
   unformat_input_t *i = vam->input;
   int enable_disable = 1;
   u32 sw_if_index0 = ~0;
   u32 sw_if_index1 = ~0;
   u32 tmp;
-  vl_api_nsim_enable_disable_t *mp;
+  vl_api_nsim_cross_connect_enable_disable_t *mp;
   int ret;
 
   /* Parse args required to build the message */
@@ -128,7 +132,7 @@ api_nsim_enable_disable (vat_main_t * vam)
     }
 
   /* Construct the API message */
-  M (NSIM_ENABLE_DISABLE, mp);
+  M (NSIM_CROSS_CONNECT_ENABLE_DISABLE, mp);
   mp->sw_if_index0 = ntohl (sw_if_index0);
   mp->sw_if_index1 = ntohl (sw_if_index1);
   mp->enable_disable = enable_disable;
@@ -141,6 +145,47 @@ api_nsim_enable_disable (vat_main_t * vam)
   return ret;
 }
 
+static int
+api_nsim_output_feature_enable_disable (vat_main_t * vam)
+{
+  unformat_input_t *i = vam->input;
+  int enable_disable = 1;
+  u32 sw_if_index = ~0;
+  vl_api_nsim_output_feature_enable_disable_t *mp;
+  int ret;
+
+  /* Parse args required to build the message */
+  while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index))
+	;
+      else if (unformat (i, "sw_if_index %d", &sw_if_index))
+	;
+      else if (unformat (i, "disable"))
+	enable_disable = 0;
+      else
+	break;
+    }
+
+  if (sw_if_index == ~0)
+    {
+      errmsg ("missing interface name / explicit sw_if_index number \n");
+      return -99;
+    }
+
+  /* Construct the API message */
+  M (NSIM_OUTPUT_FEATURE_ENABLE_DISABLE, mp);
+  mp->sw_if_index = ntohl (sw_if_index);
+  mp->enable_disable = enable_disable;
+
+  /* send it... */
+  S (mp);
+
+  /* Wait for a reply... */
+  W (ret);
+  return ret;
+}
+
 static uword
 unformat_delay (unformat_input_t * input, va_list * args)
 {
@@ -229,9 +274,10 @@ api_nsim_configure (vat_main_t * vam)
  * and that the data plane plugin processes
  */
 #define foreach_vpe_api_msg                                             \
-_(nsim_enable_disable,                                                  \
+_(nsim_cross_connect_enable_disable,                                    \
 "[<intfc0> | sw_if_index <swif0>] [<intfc1> | sw_if_index <swif1>] [disable]") \
-_(nsim_configure, "delay <time> bandwidth <bw> [packet-size <nn>]" \
+_(nsim_output_feature_enable_disable,"[<intfc> | sw_if_index <nnn> [disable]") \
+_(nsim_configure, "delay <time> bandwidth <bw> [packet-size <nn>]"      \
 "[packets-per-drop <nnnn>]")
 
 static void