summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/vlib/main.c34
-rw-r--r--src/vlib/node.c5
-rw-r--r--src/vlib/node.h5
-rw-r--r--src/vlib/node_funcs.h14
-rw-r--r--src/vlib/threads.c10
-rw-r--r--src/vlib/unix/input.c2
6 files changed, 53 insertions, 17 deletions
diff --git a/src/vlib/main.c b/src/vlib/main.c
index 0f67bb86554..9d34aa1d948 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -1471,8 +1471,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
else
cpu_time_now = clib_cpu_time_now ();
- /* Pre-allocate interupt runtime indices and lock. */
- vec_validate_aligned (nm->pending_interrupts, 0, CLIB_CACHE_LINE_BYTES);
+ nm->pending_interrupts = 0;
/* Pre-allocate expired nodes. */
if (!nm->polling_threshold_vector_length)
@@ -1505,6 +1504,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
while (1)
{
vlib_node_runtime_t *n;
+ u8 pending_interrupts;
if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0))
{
@@ -1552,6 +1552,27 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
/* frame */ 0,
cpu_time_now);
+ pending_interrupts =
+ __atomic_load_n (&nm->pending_interrupts, __ATOMIC_ACQUIRE);
+
+ if (pending_interrupts)
+ {
+ int int_num = -1;
+ nm->pending_interrupts = 0;
+
+ while ((int_num = clib_interrupt_get_next (
+ nm->pre_input_node_interrupts, int_num)) != -1)
+ {
+ vlib_node_runtime_t *n;
+ clib_interrupt_clear (nm->pre_input_node_interrupts, int_num);
+ n = vec_elt_at_index (
+ nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT], int_num);
+ cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_PRE_INPUT,
+ VLIB_NODE_STATE_INTERRUPT,
+ /* frame */ 0, cpu_time_now);
+ }
+ }
+
/* Next process input nodes. */
vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT])
cpu_time_now = dispatch_node (vm, n,
@@ -1563,16 +1584,15 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
vm->queue_signal_callback (vm);
- if (__atomic_load_n (nm->pending_interrupts, __ATOMIC_ACQUIRE))
+ if (pending_interrupts)
{
int int_num = -1;
- *nm->pending_interrupts = 0;
- while ((int_num =
- clib_interrupt_get_next (nm->interrupts, int_num)) != -1)
+ while ((int_num = clib_interrupt_get_next (nm->input_node_interrupts,
+ int_num)) != -1)
{
vlib_node_runtime_t *n;
- clib_interrupt_clear (nm->interrupts, int_num);
+ clib_interrupt_clear (nm->input_node_interrupts, int_num);
n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
int_num);
cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT,
diff --git a/src/vlib/node.c b/src/vlib/node.c
index 4d1ef989e7d..8b726ff9aad 100644
--- a/src/vlib/node.c
+++ b/src/vlib/node.c
@@ -530,7 +530,10 @@ vlib_register_node (vlib_main_t *vm, vlib_node_registration_t *r, char *fmt,
vec_add2_aligned (nm->nodes_by_type[n->type], rt, 1,
/* align */ CLIB_CACHE_LINE_BYTES);
if (n->type == VLIB_NODE_TYPE_INPUT)
- clib_interrupt_resize (&nm->interrupts,
+ clib_interrupt_resize (&nm->input_node_interrupts,
+ vec_len (nm->nodes_by_type[n->type]));
+ else if (n->type == VLIB_NODE_TYPE_PRE_INPUT)
+ clib_interrupt_resize (&nm->pre_input_node_interrupts,
vec_len (nm->nodes_by_type[n->type]));
n->runtime_index = rt - nm->nodes_by_type[n->type];
}
diff --git a/src/vlib/node.h b/src/vlib/node.h
index e40208b4bed..955ffb98600 100644
--- a/src/vlib/node.h
+++ b/src/vlib/node.h
@@ -690,8 +690,9 @@ typedef struct
vlib_node_runtime_t *nodes_by_type[VLIB_N_NODE_TYPE];
/* Node runtime indices for input nodes with pending interrupts. */
- void *interrupts;
- volatile u32 *pending_interrupts;
+ void *input_node_interrupts;
+ void *pre_input_node_interrupts;
+ volatile u8 pending_interrupts;
/* Input nodes are switched from/to interrupt to/from polling mode
when average vector length goes above/below polling/interrupt
diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h
index a1fca14faad..66f079c0547 100644
--- a/src/vlib/node_funcs.h
+++ b/src/vlib/node_funcs.h
@@ -252,15 +252,21 @@ vlib_node_set_interrupt_pending (vlib_main_t *vm, u32 node_index)
{
vlib_node_main_t *nm = &vm->node_main;
vlib_node_t *n = vec_elt (nm->nodes, node_index);
+ void *interrupts;
- ASSERT (n->type == VLIB_NODE_TYPE_INPUT);
+ if (n->type == VLIB_NODE_TYPE_INPUT)
+ interrupts = nm->input_node_interrupts;
+ else if (n->type == VLIB_NODE_TYPE_PRE_INPUT)
+ interrupts = nm->pre_input_node_interrupts;
+ else
+ ASSERT (0);
if (vm != vlib_get_main ())
- clib_interrupt_set_atomic (nm->interrupts, n->runtime_index);
+ clib_interrupt_set_atomic (interrupts, n->runtime_index);
else
- clib_interrupt_set (nm->interrupts, n->runtime_index);
+ clib_interrupt_set (interrupts, n->runtime_index);
- __atomic_store_n (nm->pending_interrupts, 1, __ATOMIC_RELEASE);
+ __atomic_store_n (&nm->pending_interrupts, 1, __ATOMIC_RELEASE);
}
always_inline vlib_process_t *
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index adf225ba87f..4b70642d3cb 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -699,8 +699,11 @@ start_workers (vlib_main_t * vm)
vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
CLIB_CACHE_LINE_BYTES);
clib_interrupt_init (
- &nm_clone->interrupts,
+ &nm_clone->input_node_interrupts,
vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
+ clib_interrupt_init (
+ &nm_clone->pre_input_node_interrupts,
+ vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]));
vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
{
vlib_node_t *n = vlib_get_node (vm, rt->node_index);
@@ -1022,8 +1025,11 @@ vlib_worker_thread_node_refork (void)
vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
CLIB_CACHE_LINE_BYTES);
clib_interrupt_resize (
- &nm_clone->interrupts,
+ &nm_clone->input_node_interrupts,
vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
+ clib_interrupt_resize (
+ &nm_clone->pre_input_node_interrupts,
+ vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]));
vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
{
diff --git a/src/vlib/unix/input.c b/src/vlib/unix/input.c
index 9c7c54f6b1b..39c124cf8df 100644
--- a/src/vlib/unix/input.c
+++ b/src/vlib/unix/input.c
@@ -250,7 +250,7 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
while (nanosleep (&ts, &tsrem) < 0)
ts = tsrem;
if (*vlib_worker_threads->wait_at_barrier ||
- *nm->pending_interrupts)
+ nm->pending_interrupts)
goto done;
}
}