diff options
| -rw-r--r-- | src/vlib/threads.c | 50 | ||||
| -rw-r--r-- | src/vlib/threads.h | 14 | ||||
| -rw-r--r-- | src/vnet/session/session.h | 40 |
3 files changed, 66 insertions, 38 deletions
diff --git a/src/vlib/threads.c b/src/vlib/threads.c index b470976d3d3..e34ef7c30cb 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -1480,6 +1480,56 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm) vm->clib_time.last_cpu_time, 1 /* leave */ ); } +static void +vlib_worker_sync_rpc (void *args) +{ + ASSERT (vlib_thread_is_main_w_barrier ()); + vlib_worker_threads->wait_before_barrier = 0; +} + +void +vlib_workers_sync (void) +{ + if (PREDICT_FALSE (!vlib_num_workers ())) + return; + + if (!(*vlib_worker_threads->wait_at_barrier) && + !clib_atomic_swap_rel_n (&vlib_worker_threads->wait_before_barrier, 1)) + { + u32 thread_index = vlib_get_thread_index (); + vlib_rpc_call_main_thread (vlib_worker_sync_rpc, (u8 *) &thread_index, + sizeof (thread_index)); + } + + /* Wait until main thread asks for barrier */ + while (!(*vlib_worker_threads->wait_at_barrier)) + ; + + /* Stop before barrier and make sure all threads are either + * at worker barrier or the barrier before it */ + clib_atomic_fetch_add (&vlib_worker_threads->workers_before_barrier, 1); + while (vlib_num_workers () > (*vlib_worker_threads->workers_at_barrier + + vlib_worker_threads->workers_before_barrier)) + ; +} + +void +vlib_workers_continue (void) +{ + if (PREDICT_FALSE (!vlib_num_workers ())) + return; + + clib_atomic_fetch_add (&vlib_worker_threads->done_work_before_barrier, 1); + + /* Wait until all workers are done with work before barrier */ + while (vlib_worker_threads->done_work_before_barrier < + vlib_worker_threads->workers_before_barrier) + ; + + clib_atomic_fetch_add (&vlib_worker_threads->done_work_before_barrier, -1); + clib_atomic_fetch_add (&vlib_worker_threads->workers_before_barrier, -1); +} + /** * Wait until each of the workers has been once around the track */ diff --git a/src/vlib/threads.h b/src/vlib/threads.h index e406dde5b07..b25d4764168 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -101,6 +101,9 @@ typedef struct const char *barrier_caller; const char *barrier_context; volatile u32 *node_reforks_required; + volatile u32 wait_before_barrier; + volatile u32 workers_before_barrier; + volatile u32 done_work_before_barrier; long lwp; int cpu_id; @@ -484,6 +487,17 @@ void vlib_rpc_call_main_thread (void *function, u8 * args, u32 size); void vlib_get_thread_core_numa (vlib_worker_thread_t * w, unsigned cpu_id); vlib_thread_main_t *vlib_get_thread_main_not_inline (void); +/** + * Force workers sync from within worker + * + * Must be paired with @ref vlib_workers_continue + */ +void vlib_workers_sync (void); +/** + * Release barrier after workers sync + */ +void vlib_workers_continue (void); + #endif /* included_vlib_threads_h */ /* diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h index 56e30e6533b..0cf6cd084fe 100644 --- a/src/vnet/session/session.h +++ b/src/vnet/session/session.h @@ -803,41 +803,6 @@ pool_program_safe_realloc (void *p, u32 thread_index, uword_to_pointer (thread_index, void *)); } -#define pool_realloc_all_at_barrier(_not) \ - (*vlib_worker_threads->workers_at_barrier >= (vlib_num_workers () - _not)) - -always_inline void -pool_realloc_wait_at_barrier (void) -{ - session_main_t *sm = &session_main; - - /* Wait until main thread asks for barrier */ - while (!(*vlib_worker_threads->wait_at_barrier)) - ; - - /* Stop at realloc barrier and make sure all threads are either - * at worker barrier or at pool realloc barrier */ - clib_atomic_fetch_add (&sm->pool_realloc_at_barrier, 1); - while (!pool_realloc_all_at_barrier (sm->pool_realloc_at_barrier)) - ; - - /* Track all workers that are doing work */ - clib_atomic_fetch_add (&sm->pool_realloc_doing_work, 1); -} - -always_inline void -pool_realloc_done_wait_at_barrier (void) -{ - session_main_t *sm = &session_main; - - /* Wait until all workers at pool realloc barrier have started reallocs */ - while (sm->pool_realloc_doing_work < sm->pool_realloc_at_barrier) - ; - - clib_atomic_fetch_add (&sm->pool_realloc_doing_work, -1); - clib_atomic_fetch_add (&sm->pool_realloc_at_barrier, -1); -} - #define pool_needs_realloc(P) \ ((!P) || \ (vec_len (pool_header (P)->free_indices) < POOL_REALLOC_SAFE_ELT_THRESH && \ @@ -857,10 +822,9 @@ pool_realloc_done_wait_at_barrier (void) } \ else if (PREDICT_FALSE (!pool_free_elts (P))) \ { \ - pool_program_safe_realloc (P, thread_index, rpc_fn); \ - pool_realloc_wait_at_barrier (); \ + vlib_workers_sync (); \ pool_alloc_aligned (P, pool_max_len (P), align); \ - pool_realloc_done_wait_at_barrier (); \ + vlib_workers_continue (); \ ALWAYS_ASSERT (pool_free_elts (P) > 0); \ } \ else \ |