pg: support for packet generator on different worker threads

New Debug CLI:
packet-generator new {
  worker X
}

Change-Id: I71499733a29b49cd36d753d54846d2553af6d024
Signed-off-by: Damjan Marion <damarion@cisco.com>

13 files changed, 98 insertions, 38 deletions

diff --git a/vlib/vlib/threads.h b/vlib/vlib/threads.h
index 2772fa86e0f..e30b9e35118 100644
--- a/vlib/vlib/threads.h
+++ b/vlib/vlib/threads.h
@@ -353,6 +353,29 @@ static void __vlib_add_thread_registration_##x (void)   \
 }                                                       \
 __VA_ARGS__ vlib_thread_registration_t x
 
+always_inline u32
+vlib_num_workers ()
+{
+  return vlib_thread_main.n_vlib_mains - 1;
+}
+
+always_inline u32
+vlib_get_worker_cpu_index (u32 worker_index)
+{
+  return worker_index + 1;
+}
+
+always_inline vlib_main_t *
+vlib_get_worker_vlib_main (u32 worker_index)
+{
+  vlib_main_t *vm;
+  vlib_thread_main_t *tm = &vlib_thread_main;
+  ASSERT (worker_index < tm->n_vlib_mains - 1);
+  vm = vlib_mains[worker_index + 1];
+  ASSERT (vm);
+  return vm;
+}
+
 #endif /* included_vlib_threads_h */
 
 /*
diff --git a/vnet/vnet/ip/icmp4.c b/vnet/vnet/ip/icmp4.c
index a02b52950a3..c160f88a600 100644
--- a/vnet/vnet/ip/icmp4.c
+++ b/vnet/vnet/ip/icmp4.c
@@ -601,7 +601,7 @@ icmp4_pg_edit_function (pg_main_t * pg,
 			u32 * packets,
 			u32 n_packets)
 {
-  vlib_main_t * vm = pg->vlib_main;
+  vlib_main_t * vm = vlib_get_main();
   u32 ip_offset, icmp_offset;
 
   icmp_offset = g->start_byte_offset;
diff --git a/vnet/vnet/ip/icmp6.c b/vnet/vnet/ip/icmp6.c
index 313ab98d169..a04373a3a7e 100644
--- a/vnet/vnet/ip/icmp6.c
+++ b/vnet/vnet/ip/icmp6.c
@@ -678,7 +678,7 @@ icmp6_pg_edit_function (pg_main_t * pg,
 			u32 * packets,
 			u32 n_packets)
 {
-  vlib_main_t * vm = pg->vlib_main;
+  vlib_main_t * vm = vlib_get_main();
   u32 ip_offset, icmp_offset;
   int bogus_length;
 
diff --git a/vnet/vnet/ip/ip4_pg.c b/vnet/vnet/ip/ip4_pg.c
index 9710d8d4c5a..42dad85272a 100644
--- a/vnet/vnet/ip/ip4_pg.c
+++ b/vnet/vnet/ip/ip4_pg.c
@@ -130,7 +130,7 @@ ip4_pg_edit_function (pg_main_t * pg,
 		      u32 * packets,
 		      u32 n_packets)
 {
-  vlib_main_t * vm = pg->vlib_main;
+  vlib_main_t * vm = vlib_get_main();
   u32 ip_offset;
 
   ip_offset = g->start_byte_offset;
diff --git a/vnet/vnet/ip/ip6_pg.c b/vnet/vnet/ip/ip6_pg.c
index 2c3852765d4..872f197076a 100644
--- a/vnet/vnet/ip/ip6_pg.c
+++ b/vnet/vnet/ip/ip6_pg.c
@@ -47,7 +47,7 @@ ip6_pg_edit_function (pg_main_t * pg,
 		      u32 * packets,
 		      u32 n_packets)
 {
-  vlib_main_t * vm = pg->vlib_main;
+  vlib_main_t * vm = vlib_get_main();
   u32 ip_header_offset = g->start_byte_offset;
 
   while (n_packets >= 2)
diff --git a/vnet/vnet/ip/udp_pg.c b/vnet/vnet/ip/udp_pg.c
index a33a56294fb..76383a9a9c2 100644
--- a/vnet/vnet/ip/udp_pg.c
+++ b/vnet/vnet/ip/udp_pg.c
@@ -51,7 +51,7 @@ udp_pg_edit_function_inline (pg_main_t * pg,
 			     u32 n_packets,
 			     u32 flags)
 {
-  vlib_main_t * vm = pg->vlib_main;
+  vlib_main_t * vm = vlib_get_main();
   u32 ip_offset, udp_offset;
 
   udp_offset = g->start_byte_offset;
diff --git a/vnet/vnet/pg/cli.c b/vnet/vnet/pg/cli.c
index e6bd11e09ce..f5896b4326e 100644
--- a/vnet/vnet/pg/cli.c
+++ b/vnet/vnet/pg/cli.c
@@ -173,6 +173,8 @@ format_pg_stream (u8 * s, va_list * va)
 
   v = format (v, "buffer-size %d, ", t->buffer_bytes);
 
+  v = format (v, "worker %d, ", t->worker_index);
+
   if (v)
     {
       s = format (s, "  %v", v);
@@ -330,6 +332,9 @@ new_stream (vlib_main_t * vm,
 			 unformat_vlib_node, vm, &s.node_index))
 	;
 
+      else if (unformat (input, "worker %u", &s.worker_index))
+	;
+
       else if (unformat (input, "interface %U",
 			 unformat_vnet_sw_interface, vnm,
 			 &s.sw_if_index[VLIB_RX]))
@@ -389,6 +394,9 @@ new_stream (vlib_main_t * vm,
     else
       n = 0;
 
+    if (s.worker_index >= vlib_num_workers ())
+      s.worker_index = 0;
+
     if (pcap_file_name != 0)
       {
 	error = pg_pcap_read (&s, pcap_file_name);
diff --git a/vnet/vnet/pg/edit.c b/vnet/vnet/pg/edit.c
index c6fc3fd124e..cb4d070fb19 100644
--- a/vnet/vnet/pg/edit.c
+++ b/vnet/vnet/pg/edit.c
@@ -127,8 +127,7 @@ uword
 unformat_pg_payload (unformat_input_t * input, va_list * args)
 {
   pg_stream_t *s = va_arg (*args, pg_stream_t *);
-  pg_main_t *pg = &pg_main;
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
   pg_edit_t *e;
   u32 i, node_index, len, max_len;
   u8 *v;
diff --git a/vnet/vnet/pg/init.c b/vnet/vnet/pg/init.c
index d939a31e1d0..631be25ea3c 100644
--- a/vnet/vnet/pg/init.c
+++ b/vnet/vnet/pg/init.c
@@ -49,7 +49,6 @@ pg_init (vlib_main_t * vm)
   clib_error_t *error;
   pg_main_t *pg = &pg_main;
 
-  pg->vlib_main = vm;
   pg->if_index_by_if_id = hash_create (0, sizeof (uword));
 
   if ((error = vlib_call_init_function (vm, vnet_main_init)))
diff --git a/vnet/vnet/pg/input.c b/vnet/vnet/pg/input.c
index af81d5386cb..46c8f4fa2f7 100644
--- a/vnet/vnet/pg/input.c
+++ b/vnet/vnet/pg/input.c
@@ -194,7 +194,7 @@ do_set_fixed (pg_main_t * pg,
 	      u32 n_bits,
 	      u32 byte_offset, u32 is_net_byte_order, u64 v_min, u64 v_max)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
 
   while (n_buffers >= 4)
     {
@@ -247,7 +247,7 @@ do_set_increment (pg_main_t * pg,
 		  u32 is_net_byte_order,
 		  u32 want_sum, u64 * sum_result, u64 v_min, u64 v_max, u64 v)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
   u64 sum = 0;
 
   ASSERT (v >= v_min && v <= v_max);
@@ -342,7 +342,7 @@ do_set_random (pg_main_t * pg,
 	       u32 is_net_byte_order,
 	       u32 want_sum, u64 * sum_result, u64 v_min, u64 v_max)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
   u64 v_diff = v_max - v_min + 1;
   u64 r_mask = max_pow2 (v_diff) - 1;
   u64 v0, v1;
@@ -533,7 +533,7 @@ do_setbits_fixed (pg_main_t * pg,
 		  u32 n_bits,
 		  u32 byte_offset, u64 v_min, u64 v_max, u64 mask, u32 shift)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
 
   while (n_buffers >= 4)
     {
@@ -586,7 +586,7 @@ do_setbits_increment (pg_main_t * pg,
 		      u32 byte_offset,
 		      u64 v_min, u64 v_max, u64 v, u64 mask, u32 shift)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
 
   ASSERT (v >= v_min && v <= v_max);
 
@@ -664,7 +664,7 @@ do_setbits_random (pg_main_t * pg,
 		   u32 n_bits,
 		   u32 byte_offset, u64 v_min, u64 v_max, u64 mask, u32 shift)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
   u64 v_diff = v_max - v_min + 1;
   u64 r_mask = max_pow2 (v_diff) - 1;
   u64 v0, v1;
@@ -938,7 +938,7 @@ pg_generate_fix_multi_buffer_lengths (pg_main_t * pg,
 				      pg_stream_t * s,
 				      u32 * buffers, u32 n_buffers)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
   pg_buffer_index_t *pbi;
   uword n_bytes_left;
   static u32 *unused_buffers = 0;
@@ -1046,7 +1046,7 @@ pg_set_next_buffer_pointers (pg_main_t * pg,
 			     pg_stream_t * s,
 			     u32 * buffers, u32 * next_buffers, u32 n_buffers)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
 
   while (n_buffers >= 4)
     {
@@ -1237,7 +1237,7 @@ pg_stream_fill_helper (pg_main_t * pg,
 		       pg_buffer_index_t * bi,
 		       u32 * buffers, u32 * next_buffers, u32 n_alloc)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
   vlib_buffer_free_list_t *f;
   uword is_start_of_packet = bi == s->buffer_indices;
   u32 n_allocated;
@@ -1477,7 +1477,7 @@ pg_input_trace (pg_main_t * pg,
 		vlib_node_runtime_t * node,
 		pg_stream_t * s, u32 * buffers, u32 n_buffers)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
   u32 *b, n_left, stream_index, next_index;
 
   n_left = n_buffers;
@@ -1548,7 +1548,7 @@ pg_generate_packets (vlib_node_runtime_t * node,
 		     pg_main_t * pg,
 		     pg_stream_t * s, uword n_packets_to_generate)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
   u32 *to_next, n_this_frame, n_left, n_trace, n_packets_in_fifo;
   uword n_packets_generated;
   pg_buffer_index_t *bi, *bi0;
@@ -1632,7 +1632,7 @@ pg_generate_packets (vlib_node_runtime_t * node,
 static uword
 pg_input_stream (vlib_node_runtime_t * node, pg_main_t * pg, pg_stream_t * s)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
   uword n_packets;
   f64 time_now, dt;
 
@@ -1683,10 +1683,15 @@ pg_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
   uword i;
   pg_main_t *pg = &pg_main;
   uword n_packets = 0;
+  u32 num_workers = vlib_num_workers ();
+  u32 cpu_index = os_get_cpu_number ();
 
   /* *INDENT-OFF* */
   clib_bitmap_foreach (i, pg->enabled_streams, ({
-    n_packets += pg_input_stream (node, pg, vec_elt_at_index (pg->streams, i));
+    pg_stream_t *s = vec_elt_at_index (pg->streams, i);
+    if (num_workers == 0 ||
+	vlib_get_worker_cpu_index (s->worker_index) == cpu_index)
+      n_packets += pg_input_stream (node, pg, s);
   }));
   /* *INDENT-ON* */
 
diff --git a/vnet/vnet/pg/output.c b/vnet/vnet/pg/output.c
index 2de06acaea5..64a0b7b9d92 100644
--- a/vnet/vnet/pg/output.c
+++ b/vnet/vnet/pg/output.c
@@ -52,6 +52,10 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
   vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
   pg_interface_t *pif = pool_elt_at_index (pg->interfaces, rd->dev_instance);
 
+  if (PREDICT_FALSE (pif->lockp != 0))
+    while (__sync_lock_test_and_set (pif->lockp, 1))
+      ;
+
   if (pif->pcap_file_name != 0)
     {
       while (n_left > 0)
@@ -67,6 +71,8 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
     }
 
   vlib_buffer_free_no_next (vm, vlib_frame_args (frame), n_buffers);
+  if (PREDICT_FALSE (pif->lockp != 0))
+    *pif->lockp = 0;
   return n_buffers;
 }
 
diff --git a/vnet/vnet/pg/pg.h b/vnet/vnet/pg/pg.h
index 067172da174..1cd42229882 100644
--- a/vnet/vnet/pg/pg.h
+++ b/vnet/vnet/pg/pg.h
@@ -143,6 +143,9 @@ typedef struct pg_stream_t
   /* Node where stream's buffers get put. */
   u32 node_index;
 
+  /* Worker thread index */
+  u32 worker_index;
+
   /* Output next index to reach output node from stream input node. */
   u32 next_index;
 
@@ -284,6 +287,9 @@ pg_free_edit_group (pg_stream_t * s)
 
 typedef struct
 {
+  /* TX lock */
+  volatile u32 *lockp;
+
   /* VLIB interface indices. */
   u32 hw_if_index, sw_if_index;
 
@@ -303,9 +309,6 @@ typedef struct
 
 typedef struct pg_main_t
 {
-  /* Back pointer to main structure. */
-  vlib_main_t *vlib_main;
-
   /* Pool of streams. */
   pg_stream_t *streams;
 
diff --git a/vnet/vnet/pg/stream.c b/vnet/vnet/pg/stream.c
index 20f54a74b8c..eba99419b5a 100644
--- a/vnet/vnet/pg/stream.c
+++ b/vnet/vnet/pg/stream.c
@@ -47,6 +47,7 @@
 void
 pg_stream_enable_disable (pg_main_t * pg, pg_stream_t * s, int want_enabled)
 {
+  vlib_main_t *vm;
   vnet_main_t *vnm = vnet_get_main ();
   pg_interface_t *pi = pool_elt_at_index (pg->interfaces, s->pg_if_index);
 
@@ -76,11 +77,14 @@ pg_stream_enable_disable (pg_main_t * pg, pg_stream_t * s, int want_enabled)
 				   VNET_SW_INTERFACE_FLAG_ADMIN_UP);
     }
 
-  vlib_node_set_state (pg->vlib_main,
-		       pg_input_node.index,
-		       (clib_bitmap_is_zero (pg->enabled_streams)
-			? VLIB_NODE_STATE_DISABLED
-			: VLIB_NODE_STATE_POLLING));
+  if (vlib_num_workers ())
+    vm = vlib_get_worker_vlib_main (s->worker_index);
+  else
+    vm = vlib_get_main ();
+
+  vlib_node_set_state (vm, pg_input_node.index,
+		       (clib_bitmap_is_zero (pg->enabled_streams) ?
+			VLIB_NODE_STATE_DISABLED : VLIB_NODE_STATE_POLLING));
 
   s->packet_accumulator = 0;
   s->time_last_generate = 0;
@@ -189,6 +193,13 @@ pg_interface_add_or_get (pg_main_t * pg, uword if_id)
 
       hash_set (pg->if_index_by_if_id, if_id, i);
 
+      if (vlib_num_workers ())
+	{
+	  pi->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
+					      CLIB_CACHE_LINE_BYTES);
+	  *pi->lockp = 0;
+	}
+
       ip4_sw_interface_enable_disable (pi->hw_if_index, 1);
       ip6_sw_interface_enable_disable (pi->hw_if_index, 1);
       mpls_sw_interface_enable_disable (&mpls_main, pi->hw_if_index, 1);
@@ -353,7 +364,7 @@ perform_fixed_edits (pg_stream_t * s)
 void
 pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
   pg_stream_t *s;
   uword *p;
 
@@ -418,12 +429,16 @@ pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
     vec_resize (s->buffer_indices, n);
 
     vec_foreach (bi, s->buffer_indices)
-      bi->free_list_index = vlib_buffer_create_free_list (vm, s->buffer_bytes,
-							  "pg stream %d buffer #%d",
-							  s - pg->streams,
-							  1 + (bi -
-							       s->
-							       buffer_indices));
+    {
+      vlib_main_t *vmt;
+      vmt =
+	vlib_num_workers ()? vlib_get_worker_vlib_main (s->worker_index) : vm;
+      bi->free_list_index =
+	vlib_buffer_create_free_list (vmt, s->buffer_bytes,
+				      "pg stream %d buffer #%d",
+				      s - pg->streams,
+				      1 + (bi - s->buffer_indices));
+    }
   }
 
   /* Find an interface to use. */
@@ -447,7 +462,7 @@ pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
 void
 pg_stream_del (pg_main_t * pg, uword index)
 {
-  vlib_main_t *vm = pg->vlib_main;
+  vlib_main_t *vm = vlib_get_main ();
   pg_stream_t *s;
   pg_buffer_index_t *bi;
 
@@ -458,7 +473,9 @@ pg_stream_del (pg_main_t * pg, uword index)
 
   vec_foreach (bi, s->buffer_indices)
   {
-    vlib_buffer_delete_free_list (vm, bi->free_list_index);
+    vlib_buffer_delete_free_list (vlib_num_workers ()?
+				  vlib_get_worker_vlib_main (s->worker_index)
+				  : vm, bi->free_list_index);
     clib_fifo_free (bi->buffer_fifo);
   }