diff options
| -rw-r--r-- | src/vlib/main.c | 176 | ||||
| -rw-r--r-- | src/vlib/main.h | 2 | ||||
| -rw-r--r-- | src/vlib/threads.c | 76 | ||||
| -rw-r--r-- | src/vlib/threads.h | 4 |
4 files changed, 116 insertions, 142 deletions
diff --git a/src/vlib/main.c b/src/vlib/main.c index 09f34bbd04a..917607067eb 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -1398,51 +1398,75 @@ dispatch_suspended_process (vlib_main_t * vm, return t; } -static void -vlib_main_loop (vlib_main_t * vm) +static_always_inline void +vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) { vlib_node_main_t *nm = &vm->node_main; + vlib_thread_main_t *tm = vlib_get_thread_main (); uword i; u64 cpu_time_now; + vlib_frame_queue_main_t *fqm; /* Initialize pending node vector. */ - vec_resize (nm->pending_frames, 32); - _vec_len (nm->pending_frames) = 0; + if (is_main) + { + vec_resize (nm->pending_frames, 32); + _vec_len (nm->pending_frames) = 0; + } /* Mark time of main loop start. */ - cpu_time_now = vm->clib_time.last_cpu_time; - vm->cpu_time_main_loop_start = cpu_time_now; + if (is_main) + { + cpu_time_now = vm->clib_time.last_cpu_time; + vm->cpu_time_main_loop_start = cpu_time_now; + } + else + cpu_time_now = clib_cpu_time_now (); /* Arrange for first level of timing wheel to cover times we care most about. */ - nm->timing_wheel.min_sched_time = 10e-6; - nm->timing_wheel.max_sched_time = 10e-3; - timing_wheel_init (&nm->timing_wheel, - cpu_time_now, vm->clib_time.clocks_per_second); + if (is_main) + { + nm->timing_wheel.min_sched_time = 10e-6; + nm->timing_wheel.max_sched_time = 10e-3; + timing_wheel_init (&nm->timing_wheel, + cpu_time_now, vm->clib_time.clocks_per_second); + vec_alloc (nm->data_from_advancing_timing_wheel, 32); + } /* Pre-allocate expired nodes. */ - vec_alloc (nm->data_from_advancing_timing_wheel, 32); vec_alloc (nm->pending_interrupt_node_runtime_indices, 32); - if (!nm->polling_threshold_vector_length) - nm->polling_threshold_vector_length = 10; - if (!nm->interrupt_threshold_vector_length) - nm->interrupt_threshold_vector_length = 5; + if (is_main) + { + if (!nm->polling_threshold_vector_length) + nm->polling_threshold_vector_length = 10; + if (!nm->interrupt_threshold_vector_length) + nm->interrupt_threshold_vector_length = 5; - nm->current_process_index = ~0; + nm->current_process_index = ~0; + } /* Start all processes. */ - { - uword i; - for (i = 0; i < vec_len (nm->processes); i++) - cpu_time_now = - dispatch_process (vm, nm->processes[i], /* frame */ 0, cpu_time_now); - } + if (is_main) + { + uword i; + for (i = 0; i < vec_len (nm->processes); i++) + cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0, + cpu_time_now); + } while (1) { vlib_node_runtime_t *n; + if (!is_main) + { + vlib_worker_thread_barrier_check (); + vec_foreach (fqm, tm->frame_queue_mains) + vlib_frame_queue_dequeue (vm, fqm); + } + /* Process pre-input nodes. */ vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]) cpu_time_now = dispatch_node (vm, n, @@ -1459,7 +1483,7 @@ vlib_main_loop (vlib_main_t * vm) /* frame */ 0, cpu_time_now); - if (PREDICT_TRUE (vm->queue_signal_pending == 0)) + if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0)) vm->queue_signal_callback (vm); /* Next handle interrupts. */ @@ -1484,58 +1508,64 @@ vlib_main_loop (vlib_main_t * vm) } } - /* Check if process nodes have expired from timing wheel. */ - nm->data_from_advancing_timing_wheel - = timing_wheel_advance (&nm->timing_wheel, cpu_time_now, - nm->data_from_advancing_timing_wheel, - &nm->cpu_time_next_process_ready); - - ASSERT (nm->data_from_advancing_timing_wheel != 0); - if (PREDICT_FALSE (_vec_len (nm->data_from_advancing_timing_wheel) > 0)) + if (is_main) { - uword i; - - processes_timing_wheel_data: - for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel); - i++) + /* Check if process nodes have expired from timing wheel. */ + nm->data_from_advancing_timing_wheel + = timing_wheel_advance (&nm->timing_wheel, cpu_time_now, + nm->data_from_advancing_timing_wheel, + &nm->cpu_time_next_process_ready); + + ASSERT (nm->data_from_advancing_timing_wheel != 0); + if (PREDICT_FALSE + (_vec_len (nm->data_from_advancing_timing_wheel) > 0)) { - u32 d = nm->data_from_advancing_timing_wheel[i]; - u32 di = vlib_timing_wheel_data_get_index (d); + uword i; - if (vlib_timing_wheel_data_is_timed_event (d)) + processes_timing_wheel_data: + for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel); + i++) { - vlib_signal_timed_event_data_t *te = - pool_elt_at_index (nm->signal_timed_event_data_pool, di); - vlib_node_t *n = vlib_get_node (vm, te->process_node_index); - vlib_process_t *p = - vec_elt (nm->processes, n->runtime_index); - void *data; - data = - vlib_process_signal_event_helper (nm, n, p, - te->event_type_index, - te->n_data_elts, - te->n_data_elt_bytes); - if (te->n_data_bytes < sizeof (te->inline_event_data)) - clib_memcpy (data, te->inline_event_data, - te->n_data_bytes); + u32 d = nm->data_from_advancing_timing_wheel[i]; + u32 di = vlib_timing_wheel_data_get_index (d); + + if (vlib_timing_wheel_data_is_timed_event (d)) + { + vlib_signal_timed_event_data_t *te = + pool_elt_at_index (nm->signal_timed_event_data_pool, + di); + vlib_node_t *n = + vlib_get_node (vm, te->process_node_index); + vlib_process_t *p = + vec_elt (nm->processes, n->runtime_index); + void *data; + data = + vlib_process_signal_event_helper (nm, n, p, + te->event_type_index, + te->n_data_elts, + te->n_data_elt_bytes); + if (te->n_data_bytes < sizeof (te->inline_event_data)) + clib_memcpy (data, te->inline_event_data, + te->n_data_bytes); + else + { + clib_memcpy (data, te->event_data_as_vector, + te->n_data_bytes); + vec_free (te->event_data_as_vector); + } + pool_put (nm->signal_timed_event_data_pool, te); + } else { - clib_memcpy (data, te->event_data_as_vector, - te->n_data_bytes); - vec_free (te->event_data_as_vector); + cpu_time_now = clib_cpu_time_now (); + cpu_time_now = + dispatch_suspended_process (vm, di, cpu_time_now); } - pool_put (nm->signal_timed_event_data_pool, te); } - else - { - cpu_time_now = clib_cpu_time_now (); - cpu_time_now = - dispatch_suspended_process (vm, di, cpu_time_now); - } - } - /* Reset vector. */ - _vec_len (nm->data_from_advancing_timing_wheel) = 0; + /* Reset vector. */ + _vec_len (nm->data_from_advancing_timing_wheel) = 0; + } } /* Input nodes may have added work to the pending vector. @@ -1548,7 +1578,7 @@ vlib_main_loop (vlib_main_t * vm) _vec_len (nm->pending_frames) = 0; /* Pending internal nodes may resume processes. */ - if (_vec_len (nm->data_from_advancing_timing_wheel) > 0) + if (is_main && _vec_len (nm->data_from_advancing_timing_wheel) > 0) goto processes_timing_wheel_data; vlib_increment_main_loop_counter (vm); @@ -1559,6 +1589,18 @@ vlib_main_loop (vlib_main_t * vm) } } +static void +vlib_main_loop (vlib_main_t * vm) +{ + vlib_main_or_worker_loop (vm, /* is_main */ 1); +} + +void +vlib_worker_loop (vlib_main_t * vm) +{ + vlib_main_or_worker_loop (vm, /* is_main */ 0); +} + vlib_main_t vlib_global_main; static clib_error_t * diff --git a/src/vlib/main.h b/src/vlib/main.h index d9ac1445ddd..a6d50b39dac 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -178,6 +178,8 @@ typedef struct vlib_main_t /* Global main structure. */ extern vlib_main_t vlib_global_main; +void vlib_worker_loop (vlib_main_t * vm); + always_inline f64 vlib_time_now (vlib_main_t * vm) { diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 4676be97412..07dbff33dc2 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -1208,9 +1208,8 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm) * If so, pull the packets off the frames and put them to * the handoff node. */ -static inline int -vlib_frame_queue_dequeue_internal (vlib_main_t * vm, - vlib_frame_queue_main_t * fqm) +int +vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm) { u32 thread_id = vm->cpu_index; vlib_frame_queue_t *fq = fqm->vlib_frame_queues[thread_id]; @@ -1337,75 +1336,6 @@ vlib_frame_queue_dequeue_internal (vlib_main_t * vm, return processed; } -static_always_inline void -vlib_worker_thread_internal (vlib_main_t * vm) -{ - vlib_node_main_t *nm = &vm->node_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - u64 cpu_time_now = clib_cpu_time_now (); - vlib_frame_queue_main_t *fqm; - - vec_alloc (nm->pending_interrupt_node_runtime_indices, 32); - - while (1) - { - vlib_worker_thread_barrier_check (); - - vec_foreach (fqm, tm->frame_queue_mains) - vlib_frame_queue_dequeue_internal (vm, fqm); - - vlib_node_runtime_t *n; - vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]) - { - cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT, - VLIB_NODE_STATE_POLLING, /* frame */ 0, - cpu_time_now); - } - - /* Next handle interrupts. */ - { - uword l = _vec_len (nm->pending_interrupt_node_runtime_indices); - uword i; - if (l > 0) - { - _vec_len (nm->pending_interrupt_node_runtime_indices) = 0; - for (i = 0; i < l; i++) - { - n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], - nm-> - pending_interrupt_node_runtime_indices - [i]); - cpu_time_now = - dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT, - VLIB_NODE_STATE_INTERRUPT, - /* frame */ 0, - cpu_time_now); - } - } - } - - if (_vec_len (nm->pending_frames)) - { - int i; - cpu_time_now = clib_cpu_time_now (); - for (i = 0; i < _vec_len (nm->pending_frames); i++) - { - vlib_pending_frame_t *p; - - p = nm->pending_frames + i; - - cpu_time_now = dispatch_pending_node (vm, p, cpu_time_now); - } - _vec_len (nm->pending_frames) = 0; - } - vlib_increment_main_loop_counter (vm); - - /* Record time stamp in case there are no enabled nodes and above - calls do not update time stamp. */ - cpu_time_now = clib_cpu_time_now (); - } -} - void vlib_worker_thread_fn (void *arg) { @@ -1423,7 +1353,7 @@ vlib_worker_thread_fn (void *arg) while (tm->extern_thread_mgmt && tm->worker_thread_release == 0) vlib_worker_thread_barrier_check (); - vlib_worker_thread_internal (vm); + vlib_worker_loop (vm); } /* *INDENT-OFF* */ diff --git a/src/vlib/threads.h b/src/vlib/threads.h index a032311c689..fc1633f610e 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -159,8 +159,8 @@ int vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t * frame, vlib_frame_queue_msg_type_t type); -int vlib_frame_queue_dequeue (int thread_id, - vlib_main_t * vm, vlib_node_main_t * nm); +int +vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm); u64 dispatch_node (vlib_main_t * vm, vlib_node_runtime_t * node, |