netmap: multithreading support

This patch adds multithreading support for netmap interfaces.

Change-Id: Iba94386fe309a4aac71646fe567f8dabbebd0459
Signed-off-by: Mohsin KAZMI <sykazmi@cisco.com>

4 files changed, 70 insertions, 16 deletions

diff --git a/vnet/vnet/devices/netmap/device.c b/vnet/vnet/devices/netmap/device.c
index a966ffef668..a13326035de 100644
--- a/vnet/vnet/devices/netmap/device.c
+++ b/vnet/vnet/devices/netmap/device.c
@@ -103,6 +103,12 @@ netmap_interface_tx (vlib_main_t * vm,
   netmap_if_t * nif = pool_elt_at_index (nm->interfaces, rd->dev_instance);
   int cur_ring;
 
+  if (PREDICT_FALSE(nif->lockp != 0))
+    {
+      while (__sync_lock_test_and_set (nif->lockp, 1))
+        ;
+    }
+
   cur_ring = nif->first_tx_ring;
 
   while(n_left && cur_ring <= nif->last_tx_ring)
@@ -156,6 +162,9 @@ netmap_interface_tx (vlib_main_t * vm,
   if (n_left < frame->n_vectors)
       ioctl(nif->fd, NIOCTXSYNC, NULL);
 
+  if (PREDICT_FALSE(nif->lockp != 0))
+      *nif->lockp = 0;
+
   if (n_left)
     vlib_error_count (vm, node->node_index,
     (n_left == frame->n_vectors ? NETMAP_TX_ERROR_PENDING_MSGS : NETMAP_TX_ERROR_NO_FREE_SLOTS), n_left);
diff --git a/vnet/vnet/devices/netmap/netmap.c b/vnet/vnet/devices/netmap/netmap.c
index df3b2be499e..ed90862fdf0 100644
--- a/vnet/vnet/devices/netmap/netmap.c
+++ b/vnet/vnet/devices/netmap/netmap.c
@@ -92,6 +92,7 @@ netmap_create_if(vlib_main_t * vm, u8 * if_name, u8 * hw_addr_set,
   uword * p;
   struct nmreq * req = 0;
   netmap_mem_region_t * reg;
+  vlib_thread_main_t * tm = vlib_get_thread_main();
   int fd;
 
   p = mhash_get (&nm->if_index_by_host_if_name, if_name);
@@ -152,6 +153,13 @@ netmap_create_if(vlib_main_t * vm, u8 * if_name, u8 * hw_addr_set,
   nif->host_if_name = if_name;
   nif->per_interface_next_index = ~0;
 
+  if (tm->n_vlib_mains > 1)
+  {
+    nif->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
+                                         CLIB_CACHE_LINE_BYTES);
+    memset ((void *) nif->lockp, 0, CLIB_CACHE_LINE_BYTES);
+  }
+
   {
     unix_file_t template = {0};
     template.read_function = netmap_fd_read_ready;
@@ -232,11 +240,37 @@ static clib_error_t *
 netmap_init (vlib_main_t * vm)
 {
   netmap_main_t * nm = &netmap_main;
+  vlib_thread_main_t * tm = vlib_get_thread_main();
+  vlib_thread_registration_t * tr;
+  uword * p;
 
   memset (nm, 0, sizeof (netmap_main_t));
 
+  nm->input_cpu_first_index = 0;
+  nm->input_cpu_count = 1;
+
+  /* find out which cpus will be used for input */
+  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
+  tr = p ? (vlib_thread_registration_t *) p[0] : 0;
+
+  if (tr && tr->count > 0)
+    {
+      nm->input_cpu_first_index = tr->first_index;
+      nm->input_cpu_count = tr->count;
+    }
+
+  /* if worker threads are enabled, switch to polling mode */
+  if (tm->n_vlib_mains > 1)
+    foreach_vlib_main (
+    ({
+        vlib_node_set_state(this_vlib_main, netmap_input_node.index, VLIB_NODE_STATE_POLLING);
+    }));
+
   mhash_init_vec_string (&nm->if_index_by_host_if_name, sizeof (uword));
 
+  vec_validate_aligned (nm->rx_buffers, tm->n_vlib_mains - 1,
+                        CLIB_CACHE_LINE_BYTES);
+
   return 0;
 }
 
diff --git a/vnet/vnet/devices/netmap/netmap.h b/vnet/vnet/devices/netmap/netmap.h
index 6f7791c3732..d29057febcf 100644
--- a/vnet/vnet/devices/netmap/netmap.h
+++ b/vnet/vnet/devices/netmap/netmap.h
@@ -42,6 +42,7 @@
 
 typedef struct {
   CLIB_CACHE_LINE_ALIGN_MARK(cacheline0);
+  volatile u32 * lockp;
   u8 * host_if_name;
   uword if_index;
   u32 hw_if_index;
@@ -77,13 +78,19 @@ typedef struct {
   uword * pending_input_bitmap;
 
   /* rx buffer cache */
-  u32 * rx_buffers;
+  u32 ** rx_buffers;
 
   /* hash of host interface names */
   mhash_t if_index_by_host_if_name;
 
   /* vector of memory regions */
   netmap_mem_region_t * mem_regions;
+
+  /* first cpu index */
+  u32 input_cpu_first_index;
+
+  /* total cpu count */
+  u32 input_cpu_count;
 } netmap_main_t;
 
 netmap_main_t netmap_main;
diff --git a/vnet/vnet/devices/netmap/node.c b/vnet/vnet/devices/netmap/node.c
index eae189c0e64..9378dfd95bb 100644
--- a/vnet/vnet/devices/netmap/node.c
+++ b/vnet/vnet/devices/netmap/node.c
@@ -99,30 +99,30 @@ buffer_add_to_chain(vlib_main_t *vm, u32 bi, u32 first_bi, u32 prev_bi)
 
 always_inline uword
 netmap_device_input_fn  (vlib_main_t * vm, vlib_node_runtime_t * node,
-			    vlib_frame_t * frame, u32 device_idx)
+			    vlib_frame_t * frame, netmap_if_t * nif)
 {
   u32 next_index = NETMAP_INPUT_NEXT_ETHERNET_INPUT;
   uword n_trace = vlib_get_trace_count (vm, node);
   netmap_main_t * nm = &netmap_main;
-  netmap_if_t * nif = pool_elt_at_index(nm->interfaces, device_idx);
   u32 n_rx_packets = 0;
   u32 n_rx_bytes = 0;
   u32 * to_next = 0;
   u32 n_free_bufs;
   struct netmap_ring * ring;
   int cur_ring;
+  u32 cpu_index = os_get_cpu_number();
   u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm,
     VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
 
   if (nif->per_interface_next_index != ~0)
       next_index = nif->per_interface_next_index;
 
-  n_free_bufs = vec_len (nm->rx_buffers);
+  n_free_bufs = vec_len (nm->rx_buffers[cpu_index]);
   if (PREDICT_FALSE(n_free_bufs < VLIB_FRAME_SIZE))
     {
-      vec_validate(nm->rx_buffers, VLIB_FRAME_SIZE + n_free_bufs - 1);
-      n_free_bufs += vlib_buffer_alloc(vm, &nm->rx_buffers[n_free_bufs], VLIB_FRAME_SIZE);
-      _vec_len (nm->rx_buffers) = n_free_bufs;
+      vec_validate(nm->rx_buffers[cpu_index], VLIB_FRAME_SIZE + n_free_bufs - 1);
+      n_free_bufs += vlib_buffer_alloc(vm, &nm->rx_buffers[cpu_index][n_free_bufs], VLIB_FRAME_SIZE);
+      _vec_len (nm->rx_buffers[cpu_index]) = n_free_bufs;
     }
 
   cur_ring = nif->first_rx_ring;
@@ -168,11 +168,11 @@ netmap_device_input_fn  (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      while (data_len && n_free_bufs)
 		{
 		  /* grab free buffer */
-		  u32 last_empty_buffer = vec_len (nm->rx_buffers) - 1;
+		  u32 last_empty_buffer = vec_len (nm->rx_buffers[cpu_index]) - 1;
 		  prev_bi0 = bi0;
-		  bi0 = nm->rx_buffers[last_empty_buffer];
+		  bi0 = nm->rx_buffers[cpu_index][last_empty_buffer];
 		  b0 = vlib_get_buffer (vm, bi0);
-		  _vec_len (nm->rx_buffers) = last_empty_buffer;
+		  _vec_len (nm->rx_buffers[cpu_index]) = last_empty_buffer;
 		  n_free_bufs--;
 
 		  /* copy data */
@@ -257,14 +257,17 @@ netmap_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 {
   int i;
   u32 n_rx_packets = 0;
-
+  u32 cpu_index = os_get_cpu_number();
   netmap_main_t * nm = &netmap_main;
+  netmap_if_t * nmi;
 
-  clib_bitmap_foreach (i, nm->pending_input_bitmap,
-    ({
-      clib_bitmap_set (nm->pending_input_bitmap, i, 0);
-      n_rx_packets += netmap_device_input_fn(vm, node, frame, i);
-    }));
+  for(i = 0; i < vec_len(nm->interfaces); i++ )
+    {
+      nmi = vec_elt_at_index(nm->interfaces, i);
+      if (nmi->is_admin_up &&
+         (i % nm->input_cpu_count) == (cpu_index - nm->input_cpu_first_index))
+        n_rx_packets += netmap_device_input_fn(vm, node, frame, nmi);
+    }
 
   return n_rx_packets;
 }
@@ -274,6 +277,7 @@ VLIB_REGISTER_NODE (netmap_input_node) = {
   .name = "netmap-input",
   .format_trace = format_netmap_input_trace,
   .type = VLIB_NODE_TYPE_INPUT,
+  /* default state is INTERRUPT mode, switch to POLLING if worker threads are enabled */
   .state = VLIB_NODE_STATE_INTERRUPT,
   .n_errors = NETMAP_INPUT_N_ERROR,
   .error_strings = netmap_input_error_strings,