Common device-input interrupt infra

Change-Id: I23b588eb56a3f5690158449a1f9bc8053cd3d251
Signed-off-by: Damjan Marion <damarion@cisco.com>

7 files changed, 194 insertions, 29 deletions

diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h
index 54e3687443a..4d7cc1928f3 100644
--- a/src/vlib/node_funcs.h
+++ b/src/vlib/node_funcs.h
@@ -177,6 +177,20 @@ vlib_node_set_state (vlib_main_t * vm, u32 node_index,
   r->state = new_state;
 }
 
+/** \brief Get node dispatch state.
+ @param vm vlib_main_t pointer, varies by thread
+ @param node_index index of the node
+ @return state for node, see vlib_node_state_t
+*/
+always_inline vlib_node_state_t
+vlib_node_get_state (vlib_main_t * vm, u32 node_index)
+{
+  vlib_node_main_t *nm = &vm->node_main;
+  vlib_node_t *n;
+  n = vec_elt (nm->nodes, node_index);
+  return n->state;
+}
+
 always_inline void
 vlib_node_set_interrupt_pending (vlib_main_t * vm, u32 node_index)
 {
diff --git a/src/vnet/devices/af_packet/af_packet.c b/src/vnet/devices/af_packet/af_packet.c
index 2028510716e..7464d4e629d 100644
--- a/src/vnet/devices/af_packet/af_packet.c
+++ b/src/vnet/devices/af_packet/af_packet.c
@@ -270,9 +270,12 @@ af_packet_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set,
 
   sw = vnet_get_hw_sw_interface (vnm, apif->hw_if_index);
   apif->sw_if_index = sw->sw_if_index;
-  vnet_set_device_input_node (apif->hw_if_index, af_packet_input_node.index);
-  vnet_device_input_assign_thread (apif->hw_if_index, 0,	/* queue */
+  vnet_set_device_input_node (vnm, apif->hw_if_index,
+			      af_packet_input_node.index);
+  vnet_device_input_assign_thread (vnm, apif->hw_if_index, 0,	/* queue */
 				   ~0 /* any cpu */ );
+  vnet_device_input_set_mode (vnm, apif->hw_if_index, 0,
+			      VNET_DEVICE_INPUT_MODE_INTERRUPT);
 
   vnet_hw_interface_set_flags (vnm, apif->hw_if_index,
 			       VNET_HW_INTERFACE_FLAG_LINK_UP);
diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c
index 76980102f71..d3af41b5d90 100644
--- a/src/vnet/devices/af_packet/node.c
+++ b/src/vnet/devices/af_packet/node.c
@@ -251,7 +251,7 @@ af_packet_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
   vnet_device_input_runtime_t *rt = (void *) node->runtime_data;
   vnet_device_and_queue_t *dq;
 
-  vec_foreach (dq, rt->devices_and_queues)
+  foreach_device_and_queue (dq, rt->devices_and_queues)
   {
     af_packet_if_t *apif;
     apif = vec_elt_at_index (apm->interfaces, dq->dev_instance);
diff --git a/src/vnet/devices/devices.c b/src/vnet/devices/devices.c
index 5e5e812c48b..c8a95087ec6 100644
--- a/src/vnet/devices/devices.c
+++ b/src/vnet/devices/devices.c
@@ -102,11 +102,26 @@ vnet_device_queue_sort (void *a1, void *a2)
     return 0;
 }
 
+static void
+vnet_device_queue_update (vnet_main_t * vnm, vnet_device_input_runtime_t * rt)
+{
+  vnet_device_and_queue_t *dq;
+  vnet_hw_interface_t *hw;
+
+  vec_sort_with_function (rt->devices_and_queues, vnet_device_queue_sort);
+
+  vec_foreach (dq, rt->devices_and_queues)
+  {
+    hw = vnet_get_hw_interface (vnm, dq->hw_if_index);
+    vec_validate (hw->dq_runtime_index_by_queue, dq->queue_id);
+    hw->dq_runtime_index_by_queue[dq->queue_id] = dq - rt->devices_and_queues;
+  }
+}
+
 void
-vnet_device_input_assign_thread (u32 hw_if_index,
+vnet_device_input_assign_thread (vnet_main_t * vnm, u32 hw_if_index,
 				 u16 queue_id, uword thread_index)
 {
-  vnet_main_t *vnm = vnet_get_main ();
   vnet_device_main_t *vdm = &vnet_device_main;
   vlib_main_t *vm;
   vnet_device_input_runtime_t *rt;
@@ -135,16 +150,17 @@ vnet_device_input_assign_thread (u32 hw_if_index,
   dq->dev_instance = hw->dev_instance;
   dq->queue_id = queue_id;
 
-  vec_sort_with_function (rt->devices_and_queues, vnet_device_queue_sort);
+  vnet_device_queue_update (vnm, rt);
   vec_validate (hw->input_node_thread_index_by_queue, queue_id);
   hw->input_node_thread_index_by_queue[queue_id] = thread_index;
+  vlib_node_set_state (vm, hw->input_node_index, rt->enabled_node_state);
 }
 
-static int
-vnet_device_input_unassign_thread (u32 hw_if_index, u16 queue_id,
-				   uword thread_index)
+int
+vnet_device_input_unassign_thread (vnet_main_t * vnm, u32 hw_if_index,
+				   u16 queue_id, uword thread_index)
 {
-  vnet_main_t *vnm = vnet_get_main ();
+  vlib_main_t *vm;
   vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
   vnet_device_input_runtime_t *rt;
   vnet_device_and_queue_t *dq;
@@ -161,9 +177,9 @@ vnet_device_input_unassign_thread (u32 hw_if_index, u16 queue_id,
   if (old_thread_index == thread_index)
     return 0;
 
-  rt =
-    vlib_node_get_runtime_data (vlib_mains[old_thread_index],
-				hw->input_node_index);
+  vm = vlib_mains[old_thread_index];
+
+  rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
 
   vec_foreach (dq, rt->devices_and_queues)
     if (dq->hw_if_index == hw_if_index && dq->queue_id == queue_id)
@@ -175,11 +191,89 @@ vnet_device_input_unassign_thread (u32 hw_if_index, u16 queue_id,
   return VNET_API_ERROR_INVALID_INTERFACE;
 
 deleted:
-  vec_sort_with_function (rt->devices_and_queues, vnet_device_queue_sort);
+
+  vnet_device_queue_update (vnm, rt);
+
+  if (vec_len (rt->devices_and_queues) == 0)
+    vlib_node_set_state (vm, hw->input_node_index, VLIB_NODE_STATE_DISABLED);
+
+  return 0;
+}
+
+
+int
+vnet_device_input_set_mode (vnet_main_t * vnm, u32 hw_if_index, u16 queue_id,
+			    vnet_device_input_mode_t mode)
+{
+  vlib_main_t *vm;
+  uword thread_index;
+  vnet_device_and_queue_t *dq;
+  vlib_node_state_t enabled_node_state;
+  ASSERT (mode < VNET_DEVICE_INPUT_N_MODES);
+  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
+  vnet_device_input_runtime_t *rt;
+  int is_polling = 0;
+
+  if (hw->input_node_thread_index_by_queue == 0)
+    return VNET_API_ERROR_INVALID_INTERFACE;
+
+  thread_index = hw->input_node_thread_index_by_queue[queue_id];
+  vm = vlib_mains[thread_index];
+
+  rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
+
+  vec_foreach (dq, rt->devices_and_queues)
+  {
+    if (dq->hw_if_index == hw_if_index && dq->queue_id == queue_id)
+      dq->mode = mode;
+    if (dq->mode == VNET_DEVICE_INPUT_MODE_POLLING)
+      is_polling = 1;
+  }
+
+  if (is_polling)
+    enabled_node_state = VLIB_NODE_STATE_POLLING;
+  else
+    enabled_node_state = VLIB_NODE_STATE_INTERRUPT;
+
+  if (rt->enabled_node_state != enabled_node_state)
+    {
+      rt->enabled_node_state = enabled_node_state;
+      if (vlib_node_get_state (vm, hw->input_node_index) !=
+	  VLIB_NODE_STATE_DISABLED)
+	vlib_node_set_state (vm, hw->input_node_index, enabled_node_state);
+    }
 
   return 0;
 }
 
+int
+vnet_device_input_get_mode (vnet_main_t * vnm, u32 hw_if_index, u16 queue_id,
+			    vnet_device_input_mode_t * mode)
+{
+  vlib_main_t *vm;
+  uword thread_index;
+  vnet_device_and_queue_t *dq;
+  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
+  vnet_device_input_runtime_t *rt;
+
+  if (hw->input_node_thread_index_by_queue == 0)
+    return VNET_API_ERROR_INVALID_INTERFACE;
+
+  thread_index = hw->input_node_thread_index_by_queue[queue_id];
+  vm = vlib_mains[thread_index];
+
+  rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
+
+  vec_foreach (dq, rt->devices_and_queues)
+    if (dq->hw_if_index == hw_if_index && dq->queue_id == queue_id)
+    {
+      *mode = dq->mode;
+      return 0;
+    }
+
+  return VNET_API_ERROR_INVALID_INTERFACE;
+}
+
 static clib_error_t *
 show_device_placement_fn (vlib_main_t * vm, unformat_input_t * input,
 			  vlib_cli_command_t * cmd)
@@ -203,9 +297,11 @@ show_device_placement_fn (vlib_main_t * vm, unformat_input_t * input,
 
         vec_foreach (dq, rt->devices_and_queues)
 	  {
-	    s = format (s, "    %U queue %u\n",
+	    s = format (s, "    %U queue %u (%s)\n",
 			format_vnet_sw_if_index_name, vnm, dq->hw_if_index,
-			dq->queue_id);
+			dq->queue_id,
+			dq->mode == VNET_DEVICE_INPUT_MODE_POLLING ?
+			"polling" : "interrupt");
 	  }
       }));
     if (vec_len (s) > 0)
@@ -238,6 +334,7 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input,
   unformat_input_t _line_input, *line_input = &_line_input;
   vnet_main_t *vnm = vnet_get_main ();
   vnet_device_main_t *vdm = &vnet_device_main;
+  vnet_device_input_mode_t mode;
   u32 hw_if_index = (u32) ~ 0;
   u32 queue_id = (u32) 0;
   u32 thread_index = (u32) ~ 0;
@@ -275,13 +372,19 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input,
     return clib_error_return (0,
 			      "please specify valid worker thread or main");
 
-  rv =
-    vnet_device_input_unassign_thread (hw_if_index, queue_id, thread_index);
+  rv = vnet_device_input_get_mode (vnm, hw_if_index, queue_id, &mode);
+
+  if (rv)
+    return clib_error_return (0, "not found");
+
+  rv = vnet_device_input_unassign_thread (vnm, hw_if_index, queue_id,
+					  thread_index);
 
   if (rv)
     return clib_error_return (0, "not found");
 
-  vnet_device_input_assign_thread (hw_if_index, queue_id, thread_index);
+  vnet_device_input_assign_thread (vnm, hw_if_index, queue_id, thread_index);
+  vnet_device_input_set_mode (vnm, hw_if_index, queue_id, mode);
 
   return 0;
 }
diff --git a/src/vnet/devices/devices.h b/src/vnet/devices/devices.h
index 966f8302230..baf03b7cd2a 100644
--- a/src/vnet/devices/devices.h
+++ b/src/vnet/devices/devices.h
@@ -55,16 +55,26 @@ typedef struct
   uword next_worker_thread_index;
 } vnet_device_main_t;
 
+typedef enum
+{
+  VNET_DEVICE_INPUT_MODE_POLLING = 0,
+  VNET_DEVICE_INPUT_MODE_INTERRUPT,
+  VNET_DEVICE_INPUT_N_MODES,
+} vnet_device_input_mode_t;
+
 typedef struct
 {
   u32 hw_if_index;
   u32 dev_instance;
   u16 queue_id;
+  vnet_device_input_mode_t mode;
+  uword interrupt_pending;
 } vnet_device_and_queue_t;
 
 typedef struct
 {
   vnet_device_and_queue_t *devices_and_queues;
+  vlib_node_state_t enabled_node_state;
 } vnet_device_input_runtime_t;
 
 extern vnet_device_main_t vnet_device_main;
@@ -72,15 +82,22 @@ extern vlib_node_registration_t device_input_node;
 extern const u32 device_input_next_node_advance[];
 
 static inline void
-vnet_set_device_input_node (u32 hw_if_index, u32 node_index)
+vnet_set_device_input_node (vnet_main_t * vnm, u32 hw_if_index,
+			    u32 node_index)
 {
-  vnet_main_t *vnm = vnet_get_main ();
   vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
   hw->input_node_index = node_index;
 }
 
-void vnet_device_input_assign_thread (u32 hw_if_index, u16 queue_id,
-				      uword thread_index);
+void vnet_device_input_assign_thread (vnet_main_t * vnm, u32 hw_if_index,
+				      u16 queue_id, uword thread_index);
+int vnet_device_input_unassign_thread (vnet_main_t * vnm, u32 hw_if_index,
+				       u16 queue_id, uword thread_index);
+int vnet_device_input_set_mode (vnet_main_t * vnm, u32 hw_if_index,
+				u16 queue_id, vnet_device_input_mode_t mode);
+int vnet_device_input_get_mode (vnet_main_t * vnm, u32 hw_if_index,
+				u16 queue_id,
+				vnet_device_input_mode_t * mode);
 
 static inline u64
 vnet_get_aggregate_rx_packets (void)
@@ -111,18 +128,41 @@ vnet_get_device_and_queue (vlib_main_t * vm, vlib_node_runtime_t * node)
   return rt->devices_and_queues;
 }
 
+static_always_inline uword
+vnet_get_device_input_thread_index (vnet_main_t * vnm, u32 hw_if_index,
+				    u16 queue_id)
+{
+  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
+  ASSERT (queue_id < vec_len (hw->input_node_thread_index_by_queue));
+  return hw->input_node_thread_index_by_queue[queue_id];
+}
+
 static_always_inline void
 vnet_device_input_set_interrupt_pending (vnet_main_t * vnm, u32 hw_if_index,
 					 u16 queue_id)
 {
-  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
-
-  ASSERT (queue_id < vec_len (hw->input_node_thread_index_by_queue));
-  u32 thread_index = hw->input_node_thread_index_by_queue[queue_id];
-  vlib_node_set_interrupt_pending (vlib_mains[thread_index],
-				   hw->input_node_index);
+  vlib_main_t *vm;
+  vnet_hw_interface_t *hw;
+  vnet_device_input_runtime_t *rt;
+  vnet_device_and_queue_t *dq;
+  uword idx;
+
+  hw = vnet_get_hw_interface (vnm, hw_if_index);
+  idx = vnet_get_device_input_thread_index (vnm, hw_if_index, queue_id);
+  vm = vlib_mains[idx];
+  rt = vlib_node_get_runtime_data (vm, hw->input_node_index);
+  idx = hw->dq_runtime_index_by_queue[queue_id];
+  dq = vec_elt_at_index (rt->devices_and_queues, idx);
+  dq->interrupt_pending = 1;
+
+  vlib_node_set_interrupt_pending (vm, hw->input_node_index);
 }
 
+#define foreach_device_and_queue(var,vec) \
+  for (var = (vec); var < vec_end (vec); var++)			\
+    if (clib_smp_swap (&((var)->interrupt_pending), 0) ||	\
+	var->mode == VNET_DEVICE_INPUT_MODE_POLLING)
+
 #endif /* included_vnet_vnet_device_h */
 
 /*
diff --git a/src/vnet/interface.c b/src/vnet/interface.c
index 45417b2f5c0..24f216f63b9 100644
--- a/src/vnet/interface.c
+++ b/src/vnet/interface.c
@@ -919,6 +919,8 @@ vnet_delete_hw_interface (vnet_main_t * vnm, u32 hw_if_index)
 
   hash_unset_mem (im->hw_interface_by_name, hw->name);
   vec_free (hw->name);
+  vec_free (hw->input_node_thread_index_by_queue);
+  vec_free (hw->dq_runtime_index_by_queue);
 
   pool_put (im->hw_interfaces, hw);
 }
diff --git a/src/vnet/interface.h b/src/vnet/interface.h
index 08f08b105fc..9c2230400c6 100644
--- a/src/vnet/interface.h
+++ b/src/vnet/interface.h
@@ -470,6 +470,9 @@ typedef struct vnet_hw_interface_t
   /* input node cpu index by queue */
   u32 *input_node_thread_index_by_queue;
 
+  /* device input device_and_queue runtime index */
+  uword *dq_runtime_index_by_queue;
+
 } vnet_hw_interface_t;
 
 extern vnet_device_class_t vnet_local_interface_device_class;