vlib: interrupt mode support for pre-input nodes

Type: improvement
Change-Id: Ic6e60597d2be63e3a0ae4399a81dbbd72392f30d
Signed-off-by: Damjan Marion <damarion@cisco.com>

6 files changed, 53 insertions, 17 deletions

diff --git a/src/vlib/main.c b/src/vlib/main.c
index 0f67bb86554..9d34aa1d948 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -1471,8 +1471,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
   else
     cpu_time_now = clib_cpu_time_now ();
 
-  /* Pre-allocate interupt runtime indices and lock. */
-  vec_validate_aligned (nm->pending_interrupts, 0, CLIB_CACHE_LINE_BYTES);
+  nm->pending_interrupts = 0;
 
   /* Pre-allocate expired nodes. */
   if (!nm->polling_threshold_vector_length)
@@ -1505,6 +1504,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
   while (1)
     {
       vlib_node_runtime_t *n;
+      u8 pending_interrupts;
 
       if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0))
 	{
@@ -1552,6 +1552,27 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 				      /* frame */ 0,
 				      cpu_time_now);
 
+      pending_interrupts =
+	__atomic_load_n (&nm->pending_interrupts, __ATOMIC_ACQUIRE);
+
+      if (pending_interrupts)
+	{
+	  int int_num = -1;
+	  nm->pending_interrupts = 0;
+
+	  while ((int_num = clib_interrupt_get_next (
+		    nm->pre_input_node_interrupts, int_num)) != -1)
+	    {
+	      vlib_node_runtime_t *n;
+	      clib_interrupt_clear (nm->pre_input_node_interrupts, int_num);
+	      n = vec_elt_at_index (
+		nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT], int_num);
+	      cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_PRE_INPUT,
+					    VLIB_NODE_STATE_INTERRUPT,
+					    /* frame */ 0, cpu_time_now);
+	    }
+	}
+
       /* Next process input nodes. */
       vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT])
 	cpu_time_now = dispatch_node (vm, n,
@@ -1563,16 +1584,15 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
       if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
 	vm->queue_signal_callback (vm);
 
-      if (__atomic_load_n (nm->pending_interrupts, __ATOMIC_ACQUIRE))
+      if (pending_interrupts)
 	{
 	  int int_num = -1;
-	  *nm->pending_interrupts = 0;
 
-	  while ((int_num =
-		    clib_interrupt_get_next (nm->interrupts, int_num)) != -1)
+	  while ((int_num = clib_interrupt_get_next (nm->input_node_interrupts,
+						     int_num)) != -1)
 	    {
 	      vlib_node_runtime_t *n;
-	      clib_interrupt_clear (nm->interrupts, int_num);
+	      clib_interrupt_clear (nm->input_node_interrupts, int_num);
 	      n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
 				    int_num);
 	      cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT,
diff --git a/src/vlib/node.c b/src/vlib/node.c
index 4d1ef989e7d..8b726ff9aad 100644
--- a/src/vlib/node.c
+++ b/src/vlib/node.c
@@ -530,7 +530,10 @@ vlib_register_node (vlib_main_t *vm, vlib_node_registration_t *r, char *fmt,
 	vec_add2_aligned (nm->nodes_by_type[n->type], rt, 1,
 			  /* align */ CLIB_CACHE_LINE_BYTES);
 	if (n->type == VLIB_NODE_TYPE_INPUT)
-	  clib_interrupt_resize (&nm->interrupts,
+	  clib_interrupt_resize (&nm->input_node_interrupts,
+				 vec_len (nm->nodes_by_type[n->type]));
+	else if (n->type == VLIB_NODE_TYPE_PRE_INPUT)
+	  clib_interrupt_resize (&nm->pre_input_node_interrupts,
 				 vec_len (nm->nodes_by_type[n->type]));
 	n->runtime_index = rt - nm->nodes_by_type[n->type];
       }
diff --git a/src/vlib/node.h b/src/vlib/node.h
index e40208b4bed..955ffb98600 100644
--- a/src/vlib/node.h
+++ b/src/vlib/node.h
@@ -690,8 +690,9 @@ typedef struct
   vlib_node_runtime_t *nodes_by_type[VLIB_N_NODE_TYPE];
 
   /* Node runtime indices for input nodes with pending interrupts. */
-  void *interrupts;
-  volatile u32 *pending_interrupts;
+  void *input_node_interrupts;
+  void *pre_input_node_interrupts;
+  volatile u8 pending_interrupts;
 
   /* Input nodes are switched from/to interrupt to/from polling mode
      when average vector length goes above/below polling/interrupt
diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h
index a1fca14faad..66f079c0547 100644
--- a/src/vlib/node_funcs.h
+++ b/src/vlib/node_funcs.h
@@ -252,15 +252,21 @@ vlib_node_set_interrupt_pending (vlib_main_t *vm, u32 node_index)
 {
   vlib_node_main_t *nm = &vm->node_main;
   vlib_node_t *n = vec_elt (nm->nodes, node_index);
+  void *interrupts;
 
-  ASSERT (n->type == VLIB_NODE_TYPE_INPUT);
+  if (n->type == VLIB_NODE_TYPE_INPUT)
+    interrupts = nm->input_node_interrupts;
+  else if (n->type == VLIB_NODE_TYPE_PRE_INPUT)
+    interrupts = nm->pre_input_node_interrupts;
+  else
+    ASSERT (0);
 
   if (vm != vlib_get_main ())
-    clib_interrupt_set_atomic (nm->interrupts, n->runtime_index);
+    clib_interrupt_set_atomic (interrupts, n->runtime_index);
   else
-    clib_interrupt_set (nm->interrupts, n->runtime_index);
+    clib_interrupt_set (interrupts, n->runtime_index);
 
-  __atomic_store_n (nm->pending_interrupts, 1, __ATOMIC_RELEASE);
+  __atomic_store_n (&nm->pending_interrupts, 1, __ATOMIC_RELEASE);
 }
 
 always_inline vlib_process_t *
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index adf225ba87f..4b70642d3cb 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -699,8 +699,11 @@ start_workers (vlib_main_t * vm)
 		vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
 				 CLIB_CACHE_LINE_BYTES);
 	      clib_interrupt_init (
-		&nm_clone->interrupts,
+		&nm_clone->input_node_interrupts,
 		vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
+	      clib_interrupt_init (
+		&nm_clone->pre_input_node_interrupts,
+		vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]));
 	      vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
 	      {
 		vlib_node_t *n = vlib_get_node (vm, rt->node_index);
@@ -1022,8 +1025,11 @@ vlib_worker_thread_node_refork (void)
     vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
 		     CLIB_CACHE_LINE_BYTES);
   clib_interrupt_resize (
-    &nm_clone->interrupts,
+    &nm_clone->input_node_interrupts,
     vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
+  clib_interrupt_resize (
+    &nm_clone->pre_input_node_interrupts,
+    vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]));
 
   vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
   {
diff --git a/src/vlib/unix/input.c b/src/vlib/unix/input.c
index 9c7c54f6b1b..39c124cf8df 100644
--- a/src/vlib/unix/input.c
+++ b/src/vlib/unix/input.c
@@ -250,7 +250,7 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 		while (nanosleep (&ts, &tsrem) < 0)
 		  ts = tsrem;
 		if (*vlib_worker_threads->wait_at_barrier ||
-		    *nm->pending_interrupts)
+		    nm->pending_interrupts)
 		  goto done;
 	      }
 	  }