diff options
Diffstat (limited to 'src/vlib')
| -rw-r--r-- | src/vlib/main.c | 315 | ||||
| -rw-r--r-- | src/vlib/main.h | 5 | ||||
| -rw-r--r-- | src/vlib/node.c | 13 | ||||
| -rw-r--r-- | src/vlib/node.h | 98 | ||||
| -rw-r--r-- | src/vlib/node_cli.c | 3 | ||||
| -rw-r--r-- | src/vlib/node_funcs.h | 77 | ||||
| -rw-r--r-- | src/vlib/threads.c | 222 | ||||
| -rw-r--r-- | src/vlib/time.h | 3 | ||||
| -rw-r--r-- | src/vlib/unix/input.c | 20 |
9 files changed, 378 insertions, 378 deletions
diff --git a/src/vlib/main.c b/src/vlib/main.c index 731e788d30d..c4b1de41069 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -871,16 +871,16 @@ add_trajectory_trace (vlib_buffer_t * b, u32 node_index) } static_always_inline u64 -dispatch_node (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_node_type_t type, - vlib_node_state_t dispatch_state, - vlib_frame_t * frame, u64 last_time_stamp) +dispatch_node (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_node_type_t type, vlib_frame_t *frame, + vlib_node_dispatch_reason_t dispatch_reason, + u64 last_time_stamp) { uword n, v; u64 t; vlib_node_main_t *nm = &vm->node_main; vlib_next_frame_t *nf; + vlib_node_type_atts_t attr = node_type_attrs[type]; if (CLIB_DEBUG > 0) { @@ -888,15 +888,14 @@ dispatch_node (vlib_main_t * vm, ASSERT (n->type == type); } - /* Only non-internal nodes may be disabled. */ - if (type != VLIB_NODE_TYPE_INTERNAL && node->state != dispatch_state) + if (attr.can_be_disabled == 0 && node->state == VLIB_NODE_STATE_DISABLED) { - ASSERT (type != VLIB_NODE_TYPE_INTERNAL); + ASSERT (0); /* disabled node should not be dispatched */ return last_time_stamp; } - if ((type == VLIB_NODE_TYPE_PRE_INPUT || type == VLIB_NODE_TYPE_INPUT) - && dispatch_state != VLIB_NODE_STATE_INTERRUPT) + if (attr.decrement_main_loop_per_calls_if_polling && + node->state == VLIB_NODE_STATE_POLLING) { u32 c = node->input_main_loops_per_call; /* Only call node when count reaches zero. */ @@ -938,18 +937,13 @@ dispatch_node (vlib_main_t * vm, vlib_buffer_t *b = vlib_get_buffer (vm, from[i]); add_trajectory_trace (b, node->node_index); } - if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0)) - n = node->function (vm, node, frame); - else - n = vm->dispatch_wrapper_fn (vm, node, frame); } + + node->dispatch_reason = dispatch_reason; + if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0)) + n = node->function (vm, node, frame); else - { - if (PREDICT_TRUE (vm->dispatch_wrapper_fn == 0)) - n = node->function (vm, node, frame); - else - n = vm->dispatch_wrapper_fn (vm, node, frame); - } + n = vm->dispatch_wrapper_fn (vm, node, frame); t = clib_cpu_time_now (); @@ -968,7 +962,8 @@ dispatch_node (vlib_main_t * vm, /* When in adaptive mode and vector rate crosses threshold switch to polling mode and vice versa. */ - if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_ADAPTIVE_MODE)) + if (PREDICT_FALSE (attr.supports_adaptive_mode && + node->flags & VLIB_NODE_FLAG_ADAPTIVE_MODE)) { ELOG_TYPE_DECLARE (e) = { @@ -985,8 +980,8 @@ dispatch_node (vlib_main_t * vm, u32 node_name, vector_length, is_polling; } *ed; - if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT - && v >= nm->polling_threshold_vector_length) && + if ((node->state == VLIB_NODE_STATE_INTERRUPT && + v >= nm->polling_threshold_vector_length) && !(node->flags & VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE)) { @@ -1012,8 +1007,8 @@ dispatch_node (vlib_main_t * vm, ed->is_polling = 1; } } - else if (dispatch_state == VLIB_NODE_STATE_POLLING - && v <= nm->interrupt_threshold_vector_length) + else if (node->state == VLIB_NODE_STATE_POLLING && + v <= nm->interrupt_threshold_vector_length) { vlib_node_t *n = vlib_get_node (vm, node->node_index); if (node->flags & @@ -1103,10 +1098,9 @@ dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index, n->flags |= (nf->flags & VLIB_FRAME_TRACE) ? VLIB_NODE_FLAG_TRACE : 0; nf->flags &= ~VLIB_FRAME_TRACE; - last_time_stamp = dispatch_node (vm, n, - VLIB_NODE_TYPE_INTERNAL, - VLIB_NODE_STATE_POLLING, - f, last_time_stamp); + last_time_stamp = + dispatch_node (vm, n, VLIB_NODE_TYPE_INTERNAL, f, + VLIB_NODE_DISPATCH_REASON_PENDING_FRAME, last_time_stamp); /* Internal node vector-rate accounting, for summary stats */ vm->internal_node_vectors += f->n_vectors; vm->internal_node_calls++; @@ -1260,6 +1254,20 @@ vlib_process_resume (vlib_main_t * vm, vlib_process_t * p) return r; } +static void +process_timer_start (vlib_main_t *vm, vlib_process_t *p, u32 runtime_index) +{ + TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; + vlib_tw_event_t e = { .type = VLIB_TW_EVENT_T_PROCESS_NODE, + .index = runtime_index }; + + if (p->resume_clock_interval == 0) + return; + + p->stop_timer_handle = TW (tw_timer_start) (tw, e.as_u32, 0 /* timer_id */, + p->resume_clock_interval); +} + static u64 dispatch_process (vlib_main_t * vm, vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp) @@ -1309,15 +1317,7 @@ dispatch_process (vlib_main_t * vm, p->n_suspends += 1; p->suspended_process_frame_index = pf - nm->suspended_process_frames; - if (p->resume_clock_interval) - { - TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) nm->timing_wheel; - p->stop_timer_handle = - TW (tw_timer_start) (tw, - vlib_timing_wheel_data_set_suspended_process ( - node->runtime_index) /* [sic] pool idex */, - 0 /* timer_id */, p->resume_clock_interval); - } + process_timer_start (vm, p, node->runtime_index); } else p->state = VLIB_PROCESS_STATE_NOT_STARTED; @@ -1419,15 +1419,7 @@ dispatch_suspended_process (vlib_main_t *vm, vlib_process_restore_t *r, /* Suspend it again. */ n_vectors = 0; p->n_suspends += 1; - if (p->resume_clock_interval) - { - p->stop_timer_handle = - TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel, - vlib_timing_wheel_data_set_suspended_process - (node->runtime_index) /* [sic] pool idex */ , - 0 /* timer_id */ , - p->resume_clock_interval); - } + process_timer_start (vm, p, node->runtime_index); } else { @@ -1452,6 +1444,61 @@ dispatch_suspended_process (vlib_main_t *vm, vlib_process_restore_t *r, return t; } +static void +process_expired_timer_cb (u32 *expired_timer_handles) +{ + vlib_main_t *vm = vlib_get_main (); + vlib_node_main_t *nm = &vm->node_main; + u32 *handle; + + vec_foreach (handle, expired_timer_handles) + { + vlib_tw_event_t e = { .as_u32 = *handle }; + vlib_process_restore_t restore = {}; + + if (e.type == VLIB_TW_EVENT_T_TIMED_EVENT) + { + restore.reason = VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT; + restore.timed_event_data_pool_index = e.index; + vec_add1 (nm->process_restore_current, restore); + } + else if (e.type == VLIB_TW_EVENT_T_PROCESS_NODE) + { + vlib_process_t *p = vec_elt (nm->processes, e.index); + p->stop_timer_handle = ~0; + restore.reason = VLIB_PROCESS_RESTORE_REASON_CLOCK; + restore.runtime_index = e.index; + vec_add1 (nm->process_restore_current, restore); + } + else if (e.type == VLIB_TW_EVENT_T_SCHED_NODE) + { + vec_add1 (nm->sched_node_pending, e.index); + } + else + ASSERT (0); + } +} + +static void +vlib_tw_init (vlib_main_t *vm) +{ + TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; + tw = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)), + CLIB_CACHE_LINE_BYTES); + /* Create the process timing wheel */ + TW (tw_timer_wheel_init) + (tw, process_expired_timer_cb /* callback */, 1 / VLIB_TW_TICKS_PER_SECOND, + ~0 /* max expirations per call */); + vm->timing_wheel = tw; +} + +static void +vlib_tw_expire_timers (vlib_main_t *vm) +{ + TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; + TW (tw_timer_expire_timers) (tw, vlib_time_now (vm)); +} + static_always_inline void vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) { @@ -1548,58 +1595,59 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) clib_call_callbacks (vm->worker_thread_main_loop_callbacks, vm, cpu_time_now); - /* Process pre-input nodes. */ cpu_time_now = clib_cpu_time_now (); - vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]) - cpu_time_now = dispatch_node (vm, n, - VLIB_NODE_TYPE_PRE_INPUT, - VLIB_NODE_STATE_POLLING, - /* frame */ 0, - cpu_time_now); - - if (clib_interrupt_is_any_pending (nm->pre_input_node_interrupts)) - { - int int_num = -1; - while ((int_num = clib_interrupt_get_next_and_clear ( - nm->pre_input_node_interrupts, int_num)) != -1) + for (vlib_node_type_t nt = 0; nt < VLIB_N_NODE_TYPE; nt++) + { + if (node_type_attrs[nt].can_be_polled) + vec_foreach (n, nm->nodes_by_type[nt]) + if (n->state == VLIB_NODE_STATE_POLLING) + cpu_time_now = dispatch_node ( + vm, n, nt, + /* frame */ 0, VLIB_NODE_DISPATCH_REASON_POLL, cpu_time_now); + + if (node_type_attrs[nt].may_receive_interrupts && + nm->node_interrupts[nt] && + clib_interrupt_is_any_pending (nm->node_interrupts[nt])) { - vlib_node_runtime_t *n; - n = vec_elt_at_index ( - nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT], int_num); - cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_PRE_INPUT, - VLIB_NODE_STATE_INTERRUPT, - /* frame */ 0, cpu_time_now); + int int_num = -1; + + while ((int_num = clib_interrupt_get_next_and_clear ( + nm->node_interrupts[nt], int_num)) != -1) + { + vlib_node_runtime_t *n; + n = vec_elt_at_index (nm->nodes_by_type[nt], int_num); + cpu_time_now = dispatch_node ( + vm, n, nt, + /* frame */ 0, VLIB_NODE_DISPATCH_REASON_INTERRUPT, + cpu_time_now); + } } } - /* Next process input nodes. */ - vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]) - cpu_time_now = dispatch_node (vm, n, - VLIB_NODE_TYPE_INPUT, - VLIB_NODE_STATE_POLLING, - /* frame */ 0, - cpu_time_now); - - if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0)) - vm->queue_signal_callback (vm); - - if (clib_interrupt_is_any_pending (nm->input_node_interrupts)) + /* Process sched nodes. */ + if (vec_len (nm->sched_node_pending)) { - int int_num = -1; - - while ((int_num = clib_interrupt_get_next_and_clear ( - nm->input_node_interrupts, int_num)) != -1) + vec_foreach_index (i, nm->sched_node_pending) { - vlib_node_runtime_t *n; - n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], - int_num); - cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT, - VLIB_NODE_STATE_INTERRUPT, - /* frame */ 0, cpu_time_now); + vlib_node_t *n = vlib_get_node (vm, nm->sched_node_pending[i]); + if (n->type == VLIB_NODE_TYPE_SCHED) + { + vlib_node_runtime_t *nr = + vlib_node_get_runtime (vm, n->index); + nr->stop_timer_handle_plus_1 = 0; + cpu_time_now = dispatch_node ( + vm, nr, VLIB_NODE_TYPE_SCHED, + /* frame */ 0, VLIB_NODE_DISPATCH_REASON_SCHED, + cpu_time_now); + } } + vec_reset_length (nm->sched_node_pending); } + if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0)) + vm->queue_signal_callback (vm); + /* Input nodes may have added work to the pending vector. Process pending vector until there is nothing left. All pending vectors will be processed from input -> output. */ @@ -1632,8 +1680,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) if (PREDICT_FALSE (vm->elog_trace_graph_dispatch)) ed = ELOG_DATA (&vlib_global_main.elog_main, es); - TW (tw_timer_expire_timers) - ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm)); + vlib_tw_expire_timers (vm); ASSERT (nm->process_restore_current != 0); @@ -1664,7 +1711,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) p->stop_timer_handle = ~0; void *data; data = vlib_process_signal_event_helper ( - nm, n, p, te->event_type_index, te->n_data_elts, + vm, nm, n, p, te->event_type_index, te->n_data_elts, te->n_data_elt_bytes); if (te->n_data_bytes < sizeof (te->inline_event_data)) clib_memcpy_fast (data, te->inline_event_data, @@ -1691,6 +1738,9 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) nm->process_restore_next); } } + else + vlib_tw_expire_timers (vm); + vlib_increment_main_loop_counter (vm); /* Record time stamp in case there are no enabled nodes and above calls do not update time stamp. */ @@ -1725,18 +1775,6 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) } } -static void -vlib_main_loop (vlib_main_t * vm) -{ - vlib_main_or_worker_loop (vm, /* is_main */ 1); -} - -void -vlib_worker_loop (vlib_main_t * vm) -{ - vlib_main_or_worker_loop (vm, /* is_main */ 0); -} - vlib_global_main_t vlib_global_main; void @@ -1862,34 +1900,6 @@ vl_api_get_elog_trace_api_messages (void) return 0; } -static void -process_expired_timer_cb (u32 *expired_timer_handles) -{ - vlib_main_t *vm = vlib_get_main (); - vlib_node_main_t *nm = &vm->node_main; - u32 *handle; - - vec_foreach (handle, expired_timer_handles) - { - u32 index = vlib_timing_wheel_data_get_index (*handle); - vlib_process_restore_t restore = {}; - - if (vlib_timing_wheel_data_is_timed_event (*handle)) - { - restore.reason = VLIB_PROCESS_RESTORE_REASON_TIMED_EVENT; - restore.timed_event_data_pool_index = index; - } - else - { - vlib_process_t *p = vec_elt (nm->processes, index); - p->stop_timer_handle = ~0; - restore.reason = VLIB_PROCESS_RESTORE_REASON_CLOCK; - restore.runtime_index = index; - } - vec_add1 (nm->process_restore_current, restore); - } -} - /* Main function. */ int vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) @@ -1988,20 +1998,13 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) if ((error = vlib_call_all_init_functions (vm))) goto done; - nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)), - CLIB_CACHE_LINE_BYTES); + vlib_tw_init (vm); vec_validate (nm->process_restore_current, 10); vec_validate (nm->process_restore_next, 10); vec_set_len (nm->process_restore_current, 0); vec_set_len (nm->process_restore_next, 0); - /* Create the process timing wheel */ - TW (tw_timer_wheel_init) - ((TWT (tw_timer_wheel) *) nm->timing_wheel, - process_expired_timer_cb /* callback */, 10e-6 /* timer period 10us */, - ~0 /* max expirations per call */); - vec_validate (vm->pending_rpc_requests, 0); vec_set_len (vm->pending_rpc_requests, 0); vec_validate (vm->processing_rpc_requests, 0); @@ -2052,7 +2055,7 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) goto done; } - vlib_main_loop (vm); + vlib_main_or_worker_loop (vm, /* is_main */ 1); done: /* Stop worker threads, barrier will not be released */ @@ -2072,6 +2075,38 @@ done: return vm->main_loop_exit_status; } +static void +vlib_worker_thread_fn (void *arg) +{ + vlib_global_main_t *vgm = vlib_get_global_main (); + vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; + vlib_main_t *vm = vlib_get_main (); + clib_error_t *e; + + ASSERT (vm->thread_index == vlib_get_thread_index ()); + + vlib_worker_thread_init (w); + clib_time_init (&vm->clib_time); + clib_mem_set_heap (w->thread_mheap); + vlib_tw_init (vm); + + vm->worker_init_functions_called = hash_create (0, 0); + + e = vlib_call_init_exit_functions_no_sort ( + vm, &vgm->worker_init_function_registrations, 1 /* call_once */, + 0 /* is_global */); + if (e) + clib_error_report (e); + + vlib_main_or_worker_loop (vm, /* is_main */ 0); +} + +VLIB_REGISTER_THREAD (worker_thread_reg, static) = { + .name = "workers", + .short_name = "wk", + .function = vlib_worker_thread_fn, +}; + vlib_main_t * vlib_get_main_not_inline (void) { diff --git a/src/vlib/main.h b/src/vlib/main.h index 94b8c4fa954..907346beda2 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -258,6 +258,9 @@ typedef struct vlib_main_t u32 buffer_alloc_success_seed; f64 buffer_alloc_success_rate; + /* Timing wheel for scheduling time-based node dispatch. */ + void *timing_wheel; + #ifdef CLIB_SANITIZE_ADDR /* address sanitizer stack save */ void *asan_stack_save; @@ -319,8 +322,6 @@ typedef struct vlib_global_main_t /* Global main structure. */ extern vlib_global_main_t vlib_global_main; -void vlib_worker_loop (vlib_main_t * vm); - always_inline f64 vlib_time_now (vlib_main_t * vm) { diff --git a/src/vlib/node.c b/src/vlib/node.c index c0572f3cf83..edeb6dc70a7 100644 --- a/src/vlib/node.c +++ b/src/vlib/node.c @@ -518,7 +518,7 @@ vlib_register_node (vlib_main_t *vm, vlib_node_registration_t *r, char *fmt, vlib_node_runtime_t *rt; u32 i; - if (n->type == VLIB_NODE_TYPE_PROCESS) + if (node_type_attrs[n->type].is_process) { vlib_process_t *p; uword log2_n_stack_bytes; @@ -559,15 +559,14 @@ vlib_register_node (vlib_main_t *vm, vlib_node_registration_t *r, char *fmt, { vec_add2_aligned (nm->nodes_by_type[n->type], rt, 1, /* align */ CLIB_CACHE_LINE_BYTES); - if (n->type == VLIB_NODE_TYPE_INPUT) - clib_interrupt_resize (&nm->input_node_interrupts, - vec_len (nm->nodes_by_type[n->type])); - else if (n->type == VLIB_NODE_TYPE_PRE_INPUT) - clib_interrupt_resize (&nm->pre_input_node_interrupts, - vec_len (nm->nodes_by_type[n->type])); + n->runtime_index = rt - nm->nodes_by_type[n->type]; } + if (node_type_attrs[n->type].may_receive_interrupts) + clib_interrupt_resize (&nm->node_interrupts[n->type], + vec_len (nm->nodes_by_type[n->type])); + if (n->type == VLIB_NODE_TYPE_INPUT) nm->input_node_counts_by_state[n->state] += 1; diff --git a/src/vlib/node.h b/src/vlib/node.h index bb6d8f818a8..22d5a4843d7 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -83,9 +83,46 @@ typedef enum /* "Process" nodes which can be suspended and later resumed. */ VLIB_NODE_TYPE_PROCESS, + /* Nodes to by called by per-thread timing wheel. */ + VLIB_NODE_TYPE_SCHED, + VLIB_N_NODE_TYPE, } vlib_node_type_t; +typedef struct +{ + u8 can_be_disabled : 1; + u8 may_receive_interrupts : 1; + u8 decrement_main_loop_per_calls_if_polling : 1; + u8 supports_adaptive_mode : 1; + u8 can_be_polled : 1; + u8 is_process : 1; +} vlib_node_type_atts_t; + +static const vlib_node_type_atts_t node_type_attrs[VLIB_N_NODE_TYPE] ={ + [VLIB_NODE_TYPE_PRE_INPUT] = { + .can_be_disabled = 1, + .may_receive_interrupts = 1, + .decrement_main_loop_per_calls_if_polling = 1, + .can_be_polled = 1, + }, + [VLIB_NODE_TYPE_INPUT] = { + .can_be_disabled = 1, + .may_receive_interrupts = 1, + .decrement_main_loop_per_calls_if_polling = 1, + .supports_adaptive_mode = 1, + .can_be_polled = 1, + }, + [VLIB_NODE_TYPE_PROCESS] = { + .can_be_disabled = 1, + .is_process = 1, + }, + [VLIB_NODE_TYPE_SCHED] = { + .can_be_disabled = 1, + .may_receive_interrupts = 1, + }, +}; + typedef struct _vlib_node_fn_registration { vlib_node_function_t *function; @@ -245,7 +282,16 @@ typedef enum foreach_vlib_node_state #undef _ VLIB_N_NODE_STATE, -} vlib_node_state_t; +} __clib_packed vlib_node_state_t; + +typedef enum +{ + VLIB_NODE_DISPATCH_REASON_UNKNOWN = 0, + VLIB_NODE_DISPATCH_REASON_PENDING_FRAME, + VLIB_NODE_DISPATCH_REASON_POLL, + VLIB_NODE_DISPATCH_REASON_INTERRUPT, + VLIB_NODE_DISPATCH_REASON_SCHED, +} __clib_packed vlib_node_dispatch_reason_t; typedef struct vlib_node_t { @@ -498,7 +544,10 @@ typedef struct vlib_node_runtime_t u16 flags; /**< Copy of main node flags. */ - u16 state; /**< Input node state. */ + vlib_node_state_t state; /**< Input node state. */ + + vlib_node_dispatch_reason_t + dispatch_reason; /**< Reason for running this node. */ u16 n_next_nodes; @@ -507,6 +556,9 @@ typedef struct vlib_node_runtime_t last time this node ran. Set to zero before first run of this node. */ + u32 stop_timer_handle_plus_1; /**< Timing wheel stop handle for + SCHED node incremented by 1, + 0 = no timer running. */ CLIB_ALIGN_MARK (runtime_data_pad, 8); @@ -679,29 +731,22 @@ typedef struct } vlib_signal_timed_event_data_t; -always_inline uword -vlib_timing_wheel_data_is_timed_event (u32 d) -{ - return d & 1; -} - -always_inline u32 -vlib_timing_wheel_data_set_suspended_process (u32 i) -{ - return 0 + 2 * i; -} - -always_inline u32 -vlib_timing_wheel_data_set_timed_event (u32 i) +typedef enum { - return 1 + 2 * i; -} + VLIB_TW_EVENT_T_PROCESS_NODE = 1, + VLIB_TW_EVENT_T_TIMED_EVENT = 2, + VLIB_TW_EVENT_T_SCHED_NODE = 3, +} vlib_tw_event_type_t; -always_inline uword -vlib_timing_wheel_data_get_index (u32 d) +typedef union { - return d / 2; -} + struct + { + u32 type : 2; /* vlib_tw_event_type_t */ + u32 index : 30; + }; + u32 as_u32; +} vlib_tw_event_t; typedef struct { @@ -727,8 +772,7 @@ typedef struct vlib_node_runtime_t *nodes_by_type[VLIB_N_NODE_TYPE]; /* Node runtime indices for input nodes with pending interrupts. */ - void *input_node_interrupts; - void *pre_input_node_interrupts; + void *node_interrupts[VLIB_N_NODE_TYPE]; /* Input nodes are switched from/to interrupt to/from polling mode when average vector length goes above/below polling/interrupt @@ -742,14 +786,14 @@ typedef struct /* Vector of internal node's frames waiting to be called. */ vlib_pending_frame_t *pending_frames; - /* Timing wheel for scheduling time-based node dispatch. */ - void *timing_wheel; - vlib_signal_timed_event_data_t *signal_timed_event_data_pool; /* Vector of process nodes waiting for restore */ vlib_process_restore_t *process_restore_current; + /* Vector of sched nodes waiting to be calleed */ + u32 *sched_node_pending; + /* Vector of process nodes waiting for restore in next greaph scheduler run */ vlib_process_restore_t *process_restore_next; diff --git a/src/vlib/node_cli.c b/src/vlib/node_cli.c index 16e904e8433..375b17bd7ae 100644 --- a/src/vlib/node_cli.c +++ b/src/vlib/node_cli.c @@ -734,6 +734,9 @@ show_node (vlib_main_t * vm, unformat_input_t * input, case VLIB_NODE_TYPE_PROCESS: type_str = "process"; break; + case VLIB_NODE_TYPE_SCHED: + type_str = "sched"; + break; default: type_str = "unknown"; } diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h index ffa17ba7bb1..91fedaa4c9c 100644 --- a/src/vlib/node_funcs.h +++ b/src/vlib/node_funcs.h @@ -50,6 +50,8 @@ #include <vppinfra/tw_timer_1t_3w_1024sl_ov.h> #include <vppinfra/interrupt.h> +#define VLIB_TW_TICKS_PER_SECOND 1e5 /* 10 us */ + #ifdef CLIB_SANITIZE_ADDR #include <sanitizer/asan_interface.h> #endif @@ -249,17 +251,9 @@ vlib_node_set_interrupt_pending (vlib_main_t *vm, u32 node_index) { vlib_node_main_t *nm = &vm->node_main; vlib_node_t *n = vec_elt (nm->nodes, node_index); - void *interrupts = 0; + void *interrupts = nm->node_interrupts[n->type]; - if (n->type == VLIB_NODE_TYPE_INPUT) - interrupts = nm->input_node_interrupts; - else if (n->type == VLIB_NODE_TYPE_PRE_INPUT) - interrupts = nm->pre_input_node_interrupts; - else - { - ASSERT (0); - return; - } + ASSERT (interrupts); if (vm != vlib_get_main ()) clib_interrupt_set_atomic (interrupts, n->runtime_index); @@ -267,6 +261,25 @@ vlib_node_set_interrupt_pending (vlib_main_t *vm, u32 node_index) clib_interrupt_set (interrupts, n->runtime_index); } +always_inline void +vlib_node_schedule (vlib_main_t *vm, u32 node_index, f64 dt) +{ + TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; + u64 ticks; + + vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, node_index); + vlib_tw_event_t e = { + .type = VLIB_TW_EVENT_T_SCHED_NODE, + .index = node_index, + }; + + dt = flt_round_nearest (dt * VLIB_TW_TICKS_PER_SECOND); + ticks = clib_max ((u64) dt, 1); + + rt->stop_timer_handle_plus_1 = + 1 + TW (tw_timer_start) (tw, e.as_u32, 0 /* timer_id */, ticks); +} + always_inline vlib_process_t * vlib_get_process_from_node (vlib_main_t * vm, vlib_node_t * node) { @@ -570,14 +583,14 @@ vlib_get_current_process_node_index (vlib_main_t * vm) return process->node_runtime.node_index; } -/** Returns TRUE if a process suspend time is less than 10us +/** Returns TRUE if a process suspend time is less than vlib timer wheel tick @param dt - remaining poll time in seconds - @returns 1 if dt < 10e-6, 0 otherwise + @returns 1 if dt < 1/VLIB_TW_TICKS_PER_SECOND, 0 otherwise */ always_inline uword vlib_process_suspend_time_is_zero (f64 dt) { - return dt < 10e-6; + return dt < (1 / VLIB_TW_TICKS_PER_SECOND); } /** Suspend a vlib cooperative multi-tasking thread for a period of time @@ -601,7 +614,7 @@ vlib_process_suspend (vlib_main_t * vm, f64 dt) if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) { /* expiration time in 10us ticks */ - p->resume_clock_interval = dt * 1e5; + p->resume_clock_interval = dt * VLIB_TW_TICKS_PER_SECOND; vlib_process_start_switch_stack (vm, 0); clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); } @@ -912,7 +925,7 @@ vlib_process_wait_for_event_or_clock (vlib_main_t * vm, f64 dt) r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) { - p->resume_clock_interval = dt * 1e5; + p->resume_clock_interval = dt * VLIB_TW_TICKS_PER_SECOND; vlib_process_start_switch_stack (vm, 0); clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); } @@ -963,12 +976,11 @@ vlib_process_delete_one_time_event (vlib_main_t * vm, uword node_index, } always_inline void * -vlib_process_signal_event_helper (vlib_node_main_t * nm, - vlib_node_t * n, - vlib_process_t * p, - uword t, +vlib_process_signal_event_helper (vlib_main_t *vm, vlib_node_main_t *nm, + vlib_node_t *n, vlib_process_t *p, uword t, uword n_data_elts, uword n_data_elt_bytes) { + TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; uword add_to_pending = 0, delete_from_wheel = 0; u8 *data_to_be_written_by_caller; vec_attr_t va = { .elt_sz = n_data_elt_bytes }; @@ -1016,8 +1028,7 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm, break; } - if (TW (tw_timer_handle_is_free) ((TWT (tw_timer_wheel) *) nm->timing_wheel, - p->stop_timer_handle)) + if (TW (tw_timer_handle_is_free) (tw, p->stop_timer_handle)) delete_from_wheel = 0; /* Never add current process to pending vector since current process is @@ -1036,8 +1047,7 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm, if (delete_from_wheel) { - TW (tw_timer_stop) - ((TWT (tw_timer_wheel) *) nm->timing_wheel, p->stop_timer_handle); + TW (tw_timer_stop) (tw, p->stop_timer_handle); p->stop_timer_handle = ~0; } @@ -1069,7 +1079,7 @@ vlib_process_signal_event_data (vlib_main_t * vm, else t = h[0]; - return vlib_process_signal_event_helper (nm, n, p, t, n_data_elts, + return vlib_process_signal_event_helper (vm, nm, n, p, t, n_data_elts, n_data_elt_bytes); } @@ -1080,6 +1090,7 @@ vlib_process_signal_event_at_time (vlib_main_t * vm, uword type_opaque, uword n_data_elts, uword n_data_elt_bytes) { + TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; vlib_node_main_t *nm = &vm->node_main; vlib_node_t *n = vlib_get_node (vm, node_index); vlib_process_t *p = vec_elt (nm->processes, n->runtime_index); @@ -1097,7 +1108,7 @@ vlib_process_signal_event_at_time (vlib_main_t * vm, t = h[0]; if (vlib_process_suspend_time_is_zero (dt)) - return vlib_process_signal_event_helper (nm, n, p, t, n_data_elts, + return vlib_process_signal_event_helper (vm, nm, n, p, t, n_data_elts, n_data_elt_bytes); else { @@ -1118,11 +1129,13 @@ vlib_process_signal_event_at_time (vlib_main_t * vm, te->event_type_index = t; p->stop_timer_handle = - TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel, - vlib_timing_wheel_data_set_timed_event - (te - nm->signal_timed_event_data_pool), - 0 /* timer_id */ , - (vlib_time_now (vm) + dt) * 1e5); + TW (tw_timer_start) (tw, + (vlib_tw_event_t){ + .type = VLIB_TW_EVENT_T_TIMED_EVENT, + .index = te - nm->signal_timed_event_data_pool, + } + .as_u32, + 0 /* timer_id */, dt * VLIB_TW_TICKS_PER_SECOND); /* Inline data big enough to hold event? */ if (te->n_data_bytes < sizeof (te->inline_event_data)) @@ -1146,8 +1159,8 @@ vlib_process_signal_one_time_event_data (vlib_main_t * vm, vlib_node_main_t *nm = &vm->node_main; vlib_node_t *n = vlib_get_node (vm, node_index); vlib_process_t *p = vec_elt (nm->processes, n->runtime_index); - return vlib_process_signal_event_helper (nm, n, p, type_index, n_data_elts, - n_data_elt_bytes); + return vlib_process_signal_event_helper (vm, nm, n, p, type_index, + n_data_elts, n_data_elt_bytes); } always_inline void diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 9dce1b8c8bd..0a5779c2d2e 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -705,52 +705,29 @@ start_workers (vlib_main_t * vm) vec_add1 (nm_clone->nodes, n); n++; } - nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] = - vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL], - CLIB_CACHE_LINE_BYTES); - vec_foreach (rt, - nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]) - { - vlib_node_t *n = vlib_get_node (vm, rt->node_index); - /* copy initial runtime_data from node */ - if (n->runtime_data && n->runtime_data_bytes > 0) - clib_memcpy (rt->runtime_data, n->runtime_data, - clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, - n->runtime_data_bytes)); - } - - nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] = - vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], - CLIB_CACHE_LINE_BYTES); - clib_interrupt_init ( - &nm_clone->input_node_interrupts, - vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])); - clib_interrupt_init ( - &nm_clone->pre_input_node_interrupts, - vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])); - vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) - { - vlib_node_t *n = vlib_get_node (vm, rt->node_index); - /* copy initial runtime_data from node */ - if (n->runtime_data && n->runtime_data_bytes > 0) - clib_memcpy (rt->runtime_data, n->runtime_data, - clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, - n->runtime_data_bytes)); - } - - nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT] = - vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT], - CLIB_CACHE_LINE_BYTES); - vec_foreach (rt, - nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]) - { - vlib_node_t *n = vlib_get_node (vm, rt->node_index); - /* copy initial runtime_data from node */ - if (n->runtime_data && n->runtime_data_bytes > 0) - clib_memcpy (rt->runtime_data, n->runtime_data, - clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, - n->runtime_data_bytes)); - } + + foreach_int (nt, VLIB_NODE_TYPE_INTERNAL, + VLIB_NODE_TYPE_PRE_INPUT, VLIB_NODE_TYPE_INPUT, + VLIB_NODE_TYPE_SCHED) + { + u32 n_nodes = vec_len (nm_clone->nodes_by_type[nt]); + nm_clone->nodes_by_type[nt] = vec_dup_aligned ( + nm->nodes_by_type[nt], CLIB_CACHE_LINE_BYTES); + + if (node_type_attrs[nt].may_receive_interrupts) + clib_interrupt_init (&nm_clone->node_interrupts[nt], + n_nodes); + + vec_foreach (rt, nm_clone->nodes_by_type[nt]) + { + vlib_node_t *n = vlib_get_node (vm, rt->node_index); + /* copy initial runtime_data from node */ + if (n->runtime_data && n->runtime_data_bytes > 0) + clib_memcpy (rt->runtime_data, n->runtime_data, + clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, + n->runtime_data_bytes)); + } + } nm_clone->processes = vec_dup_aligned (nm->processes, CLIB_CACHE_LINE_BYTES); @@ -1016,101 +993,53 @@ vlib_worker_thread_node_refork (void) vec_free (old_nodes_clone); + /* re-clone nodes */ - /* re-clone internal nodes */ - old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]; - nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] = - vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL], - CLIB_CACHE_LINE_BYTES); - - vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]) - { - vlib_node_t *n = vlib_get_node (vm, rt->node_index); - /* copy runtime_data, will be overwritten later for existing rt */ - if (n->runtime_data && n->runtime_data_bytes > 0) - clib_memcpy_fast (rt->runtime_data, n->runtime_data, - clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, - n->runtime_data_bytes)); - } - - for (j = 0; j < vec_len (old_rt); j++) + foreach_int (nt, VLIB_NODE_TYPE_INTERNAL, VLIB_NODE_TYPE_PRE_INPUT, + VLIB_NODE_TYPE_INPUT, VLIB_NODE_TYPE_SCHED) { - rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); - rt->state = old_rt[j].state; - rt->flags = old_rt[j].flags; - clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data, - VLIB_NODE_RUNTIME_DATA_SIZE); - } + old_rt = nm_clone->nodes_by_type[nt]; + u32 n_nodes = vec_len (nm->nodes_by_type[nt]); - vec_free (old_rt); - - /* re-clone input nodes */ - old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]; - nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] = - vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], - CLIB_CACHE_LINE_BYTES); - clib_interrupt_resize ( - &nm_clone->input_node_interrupts, - vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])); - clib_interrupt_resize ( - &nm_clone->pre_input_node_interrupts, - vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])); - - vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) - { - vlib_node_t *n = vlib_get_node (vm, rt->node_index); - /* copy runtime_data, will be overwritten later for existing rt */ - if (n->runtime_data && n->runtime_data_bytes > 0) - clib_memcpy_fast (rt->runtime_data, n->runtime_data, - clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, - n->runtime_data_bytes)); - } + nm_clone->nodes_by_type[nt] = + vec_dup_aligned (nm->nodes_by_type[nt], CLIB_CACHE_LINE_BYTES); - for (j = 0; j < vec_len (old_rt); j++) - { - rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); - rt->state = old_rt[j].state; - rt->flags = old_rt[j].flags; - clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data, - VLIB_NODE_RUNTIME_DATA_SIZE); - } + if (nm_clone->node_interrupts[nt]) + clib_interrupt_resize (&nm_clone->node_interrupts[nt], n_nodes); - for (j = vec_len (old_rt); - j < vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]); j++) - { - rt = &nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT][j]; - nm_clone->input_node_counts_by_state[rt->state] += 1; - } - - vec_free (old_rt); + vec_foreach (rt, nm_clone->nodes_by_type[nt]) + { + vlib_node_t *n = vlib_get_node (vm, rt->node_index); + /* copy runtime_data, will be overwritten later for existing rt */ + if (n->runtime_data && n->runtime_data_bytes > 0) + clib_memcpy_fast ( + rt->runtime_data, n->runtime_data, + clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, n->runtime_data_bytes)); + } - /* re-clone pre-input nodes */ - old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]; - nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT] = - vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT], - CLIB_CACHE_LINE_BYTES); + for (j = 0; j < vec_len (old_rt); j++) + { + rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); + rt->state = old_rt[j].state; + rt->flags = old_rt[j].flags; + clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); + } - vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]) - { - vlib_node_t *n = vlib_get_node (vm, rt->node_index); - /* copy runtime_data, will be overwritten later for existing rt */ - if (n->runtime_data && n->runtime_data_bytes > 0) - clib_memcpy_fast (rt->runtime_data, n->runtime_data, - clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, - n->runtime_data_bytes)); - } + if (nt == VLIB_NODE_TYPE_INPUT) + { + for (j = vec_len (old_rt); + j < vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]); + j++) + { + rt = &nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT][j]; + nm_clone->input_node_counts_by_state[rt->state] += 1; + } + } - for (j = 0; j < vec_len (old_rt); j++) - { - rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); - rt->state = old_rt[j].state; - rt->flags = old_rt[j].flags; - clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data, - VLIB_NODE_RUNTIME_DATA_SIZE); + vec_free (old_rt); } - vec_free (old_rt); - vec_free (nm_clone->processes); nm_clone->processes = vec_dup_aligned (nm->processes, CLIB_CACHE_LINE_BYTES); @@ -1630,37 +1559,6 @@ vlib_worker_flush_pending_rpc_requests (vlib_main_t *vm) clib_spinlock_unlock_if_init (&vm_global->pending_rpc_lock); } -void -vlib_worker_thread_fn (void *arg) -{ - vlib_global_main_t *vgm = vlib_get_global_main (); - vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; - vlib_main_t *vm = vlib_get_main (); - clib_error_t *e; - - ASSERT (vm->thread_index == vlib_get_thread_index ()); - - vlib_worker_thread_init (w); - clib_time_init (&vm->clib_time); - clib_mem_set_heap (w->thread_mheap); - - vm->worker_init_functions_called = hash_create (0, 0); - - e = vlib_call_init_exit_functions_no_sort ( - vm, &vgm->worker_init_function_registrations, 1 /* call_once */, - 0 /* is_global */); - if (e) - clib_error_report (e); - - vlib_worker_loop (vm); -} - -VLIB_REGISTER_THREAD (worker_thread_reg, static) = { - .name = "workers", - .short_name = "wk", - .function = vlib_worker_thread_fn, -}; - extern clib_march_fn_registration *vlib_frame_queue_dequeue_with_aux_fn_march_fn_registrations; extern clib_march_fn_registration diff --git a/src/vlib/time.h b/src/vlib/time.h index 61873bb2ef3..a9dc5395661 100644 --- a/src/vlib/time.h +++ b/src/vlib/time.h @@ -12,8 +12,7 @@ static inline f64 vlib_time_get_next_timer (vlib_main_t *vm) { - vlib_node_main_t *nm = &vm->node_main; - TWT (tw_timer_wheel) *wheel = nm->timing_wheel; + TWT (tw_timer_wheel) *wheel = vm->timing_wheel; return TW (tw_timer_first_expires_in_ticks) (wheel) * wheel->timer_interval; } diff --git a/src/vlib/unix/input.c b/src/vlib/unix/input.c index e96cd902466..302d1eb1194 100644 --- a/src/vlib/unix/input.c +++ b/src/vlib/unix/input.c @@ -130,6 +130,17 @@ linux_epoll_file_update (clib_file_t * f, clib_file_update_type_t update_type) } } +static int +is_int_pending (vlib_node_main_t *nm) +{ + + for (int nt = 0; nt < VLIB_N_NODE_TYPE; nt++) + if (nm->node_interrupts[nt] && + clib_interrupt_is_any_pending (nm->node_interrupts[nt])) + return 1; + return 0; +} + static_always_inline uword linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame, u32 thread_index) @@ -174,8 +185,8 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, else if (is_main && vector_rate < 2 && vm->api_queue_nonempty == 0 && nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] == 0) { - ticks_until_expiration = TW (tw_timer_first_expires_in_ticks) - ((TWT (tw_timer_wheel) *) nm->timing_wheel); + ticks_until_expiration = TW (tw_timer_first_expires_in_ticks) ( + (TWT (tw_timer_wheel) *) vm->timing_wheel); /* Nothing on the fast wheel, sleep 10ms */ if (ticks_until_expiration == TW_SLOTS_PER_RING) @@ -250,10 +261,7 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, while (nanosleep (&ts, &tsrem) < 0) ts = tsrem; if (*vlib_worker_threads->wait_at_barrier || - clib_interrupt_is_any_pending ( - nm->input_node_interrupts) || - clib_interrupt_is_any_pending ( - nm->pre_input_node_interrupts)) + is_int_pending (nm)) goto done; } } |