vnet: add device-input threadplacement infra

This change adds two new debug CLI command:

- "show interface placmenet" to display which
thread (main or worker) is responsible for processing
interface rx queue

vpp# show interface placement
Thread 0 (vpp_main):
  node af-packet-input:
    host-vpp1 queue 0
Thread 1 (vpp_wk_0):
  node af-packet-input:
    host-virbr0 queue 0
Thread 2 (vpp_wk_1):
  node af-packet-input:
    host-vpp2 queue 0
    host-lxcbr0 queue 0

- "set interface placmenet" to assign thread (main or worker)
which process specific interface rx queue

vpp# set interface placement host-vpp1 queue 0 main

Change-Id: Id4dd00cf2b05e10fae2125ac7cb4411b446c5e9c
Signed-off-by: Damjan Marion <damarion@cisco.com>

3 files changed, 15 insertions, 68 deletions

diff --git a/src/vnet/devices/af_packet/af_packet.c b/src/vnet/devices/af_packet/af_packet.c
index e491ba473d5..5fdc59f2a20 100644
--- a/src/vnet/devices/af_packet/af_packet.c
+++ b/src/vnet/devices/af_packet/af_packet.c
@@ -67,15 +67,16 @@ af_packet_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi,
 static clib_error_t *
 af_packet_fd_read_ready (unix_file_t * uf)
 {
-  vlib_main_t *vm = vlib_get_main ();
   af_packet_main_t *apm = &af_packet_main;
+  vnet_main_t *vnm = vnet_get_main ();
   u32 idx = uf->private_data;
+  af_packet_if_t *apif = pool_elt_at_index (apm->interfaces, idx);
 
   apm->pending_input_bitmap =
     clib_bitmap_set (apm->pending_input_bitmap, idx, 1);
 
   /* Schedule the rx node */
-  vlib_node_set_interrupt_pending (vm, af_packet_input_node.index);
+  vnet_device_input_set_interrupt_pending (vnm, apif->hw_if_index, 0);
 
   return 0;
 }
@@ -171,31 +172,6 @@ error:
   return ret;
 }
 
-static void
-af_packet_worker_thread_enable ()
-{
-  /* If worker threads are enabled, switch to polling mode */
-  foreach_vlib_main ((
-		       {
-		       vlib_node_set_state (this_vlib_main,
-					    af_packet_input_node.index,
-					    VLIB_NODE_STATE_POLLING);
-		       }));
-
-}
-
-static void
-af_packet_worker_thread_disable ()
-{
-  foreach_vlib_main ((
-		       {
-		       vlib_node_set_state (this_vlib_main,
-					    af_packet_input_node.index,
-					    VLIB_NODE_STATE_INTERRUPT);
-		       }));
-
-}
-
 int
 af_packet_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set,
 		     u32 * sw_if_index)
@@ -298,6 +274,9 @@ af_packet_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set,
 
   sw = vnet_get_hw_sw_interface (vnm, apif->hw_if_index);
   apif->sw_if_index = sw->sw_if_index;
+  vnet_set_device_input_node (apif->hw_if_index, af_packet_input_node.index);
+  vnet_device_input_assign_thread (apif->hw_if_index, 0,	/* queue */
+				   ~0 /* any cpu */ );
 
   vnet_hw_interface_set_flags (vnm, apif->hw_if_index,
 			       VNET_HW_INTERFACE_FLAG_LINK_UP);
@@ -307,9 +286,6 @@ af_packet_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set,
   if (sw_if_index)
     *sw_if_index = apif->sw_if_index;
 
-  if (tm->n_vlib_mains > 1 && pool_elts (apm->interfaces) == 1)
-    af_packet_worker_thread_enable ();
-
   return 0;
 
 error:
@@ -323,7 +299,6 @@ int
 af_packet_delete_if (vlib_main_t * vm, u8 * host_if_name)
 {
   vnet_main_t *vnm = vnet_get_main ();
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
   af_packet_main_t *apm = &af_packet_main;
   af_packet_if_t *apif;
   uword *p;
@@ -373,8 +348,6 @@ af_packet_delete_if (vlib_main_t * vm, u8 * host_if_name)
   ethernet_delete_interface (vnm, apif->hw_if_index);
 
   pool_put (apm->interfaces, apif);
-  if (tm->n_vlib_mains > 1 && pool_elts (apm->interfaces) == 0)
-    af_packet_worker_thread_disable ();
 
   return 0;
 }
@@ -384,24 +357,9 @@ af_packet_init (vlib_main_t * vm)
 {
   af_packet_main_t *apm = &af_packet_main;
   vlib_thread_main_t *tm = vlib_get_thread_main ();
-  vlib_thread_registration_t *tr;
-  uword *p;
 
   memset (apm, 0, sizeof (af_packet_main_t));
 
-  apm->input_cpu_first_index = 0;
-  apm->input_cpu_count = 1;
-
-  /* find out which cpus will be used for input */
-  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
-  tr = p ? (vlib_thread_registration_t *) p[0] : 0;
-
-  if (tr && tr->count > 0)
-    {
-      apm->input_cpu_first_index = tr->first_index;
-      apm->input_cpu_count = tr->count;
-    }
-
   mhash_init_vec_string (&apm->if_index_by_host_if_name, sizeof (uword));
 
   vec_validate_aligned (apm->rx_buffers, tm->n_vlib_mains - 1,
diff --git a/src/vnet/devices/af_packet/af_packet.h b/src/vnet/devices/af_packet/af_packet.h
index e00e5cb415f..50ec23785ae 100644
--- a/src/vnet/devices/af_packet/af_packet.h
+++ b/src/vnet/devices/af_packet/af_packet.h
@@ -51,12 +51,6 @@ typedef struct
 
   /* hash of host interface names */
   mhash_t if_index_by_host_if_name;
-
-  /* first cpu index */
-  u32 input_cpu_first_index;
-
-  /* total cpu count */
-  u32 input_cpu_count;
 } af_packet_main_t;
 
 af_packet_main_t af_packet_main;
diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c
index ab7fd80005f..ba337f3f70b 100644
--- a/src/vnet/devices/af_packet/node.c
+++ b/src/vnet/devices/af_packet/node.c
@@ -246,20 +246,18 @@ static uword
 af_packet_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 		    vlib_frame_t * frame)
 {
-  int i;
   u32 n_rx_packets = 0;
-  u32 cpu_index = os_get_cpu_number ();
   af_packet_main_t *apm = &af_packet_main;
-  af_packet_if_t *apif;
+  vnet_device_input_runtime_t *rt = (void *) node->runtime_data;
+  vnet_device_and_queue_t *dq;
 
-  for (i = 0; i < vec_len (apm->interfaces); i++)
-    {
-      apif = vec_elt_at_index (apm->interfaces, i);
-      if (apif->is_admin_up &&
-	  (i % apm->input_cpu_count) ==
-	  (cpu_index - apm->input_cpu_first_index))
-	n_rx_packets += af_packet_device_input_fn (vm, node, frame, apif);
-    }
+  vec_foreach (dq, rt->devices_and_queues)
+  {
+    af_packet_if_t *apif;
+    apif = vec_elt_at_index (apm->interfaces, dq->dev_instance);
+    if (apif->is_admin_up)
+      n_rx_packets += af_packet_device_input_fn (vm, node, frame, apif);
+  }
 
   return n_rx_packets;
 }
@@ -271,9 +269,6 @@ VLIB_REGISTER_NODE (af_packet_input_node) = {
   .sibling_of = "device-input",
   .format_trace = format_af_packet_input_trace,
   .type = VLIB_NODE_TYPE_INPUT,
-  /**
-   * default state is INTERRUPT mode, switch to POLLING if worker threads are enabled
-   */
   .state = VLIB_NODE_STATE_INTERRUPT,
   .n_errors = AF_PACKET_INPUT_N_ERROR,
   .error_strings = af_packet_input_error_strings,