diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/vnet/session/session.h | 104 |
1 files changed, 42 insertions, 62 deletions
diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h index 215588edae1..56e30e6533b 100644 --- a/src/vnet/session/session.h +++ b/src/vnet/session/session.h @@ -201,7 +201,10 @@ typedef struct session_main_ transport_proto_t last_transport_proto_type; /** Number of workers at pool realloc barrier */ - u32 pool_realloc_at_barrier; + volatile u32 pool_realloc_at_barrier; + + /** Number of workers doing reallocs */ + volatile u32 pool_realloc_doing_work; /** Lock to synchronize parallel forced reallocs */ clib_spinlock_t pool_realloc_lock; @@ -774,11 +777,14 @@ STATIC_ASSERT_SIZEOF (pool_safe_realloc_header_t, sizeof (pool_header_t)); ASSERT (vlib_get_thread_index () == 0); \ vlib_worker_thread_barrier_sync (vm); \ free_elts = pool_free_elts (P); \ - max_elts = pool_max_len (P); \ - n_alloc = clib_max (2 * max_elts, POOL_REALLOC_SAFE_ELT_THRESH); \ - pool_alloc_aligned (P, free_elts + n_alloc, align); \ - clib_bitmap_validate (pool_header (P)->free_bitmap, \ - max_elts + n_alloc); \ + if (free_elts < POOL_REALLOC_SAFE_ELT_THRESH) \ + { \ + max_elts = pool_max_len (P); \ + n_alloc = clib_max (2 * max_elts, POOL_REALLOC_SAFE_ELT_THRESH); \ + pool_alloc_aligned (P, free_elts + n_alloc, align); \ + clib_bitmap_validate (pool_header (P)->free_bitmap, \ + max_elts + n_alloc); \ + } \ pool_realloc_flag (P) = 0; \ vlib_worker_thread_barrier_release (vm); \ } \ @@ -797,49 +803,40 @@ pool_program_safe_realloc (void *p, u32 thread_index, uword_to_pointer (thread_index, void *)); } +#define pool_realloc_all_at_barrier(_not) \ + (*vlib_worker_threads->workers_at_barrier >= (vlib_num_workers () - _not)) + always_inline void -pool_realloc_maybe_wait_at_barrier (void) +pool_realloc_wait_at_barrier (void) { - if (!(*vlib_worker_threads->wait_at_barrier)) - return; + session_main_t *sm = &session_main; - /* Node refork required. Don't stop at the barrier from within a node */ - if (*vlib_worker_threads->node_reforks_required) - return; - - clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1); + /* Wait until main thread asks for barrier */ + while (!(*vlib_worker_threads->wait_at_barrier)) + ; - while (*vlib_worker_threads->wait_at_barrier) + /* Stop at realloc barrier and make sure all threads are either + * at worker barrier or at pool realloc barrier */ + clib_atomic_fetch_add (&sm->pool_realloc_at_barrier, 1); + while (!pool_realloc_all_at_barrier (sm->pool_realloc_at_barrier)) ; - clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1); + /* Track all workers that are doing work */ + clib_atomic_fetch_add (&sm->pool_realloc_doing_work, 1); } -#define pool_realloc_all_at_barrier(_not) \ - (*vlib_worker_threads->workers_at_barrier >= (vlib_num_workers () - _not)) +always_inline void +pool_realloc_done_wait_at_barrier (void) +{ + session_main_t *sm = &session_main; -#define pool_realloc_safe_force(P) \ - do \ - { \ - ALWAYS_ASSERT (*vlib_worker_threads->node_reforks_required); \ - if (pool_realloc_all_at_barrier (1)) \ - { \ - pool_alloc (P, pool_max_len (P)); \ - } \ - else \ - { \ - session_main_t *sm = &session_main; \ - clib_warning ("forced pool realloc"); \ - clib_atomic_fetch_add (&sm->pool_realloc_at_barrier, 1); \ - while (!pool_realloc_all_at_barrier (sm->pool_realloc_at_barrier)) \ - ; \ - clib_spinlock_lock (&sm->pool_realloc_lock); \ - pool_alloc (P, pool_max_len (P)); \ - clib_spinlock_unlock (&sm->pool_realloc_lock); \ - clib_atomic_fetch_add (&sm->pool_realloc_at_barrier, -1); \ - } \ - } \ - while (0) + /* Wait until all workers at pool realloc barrier have started reallocs */ + while (sm->pool_realloc_doing_work < sm->pool_realloc_at_barrier) + ; + + clib_atomic_fetch_add (&sm->pool_realloc_doing_work, -1); + clib_atomic_fetch_add (&sm->pool_realloc_at_barrier, -1); +} #define pool_needs_realloc(P) \ ((!P) || \ @@ -858,30 +855,13 @@ pool_realloc_maybe_wait_at_barrier (void) pool_alloc_aligned (P, 2 * POOL_REALLOC_SAFE_ELT_THRESH, \ align); \ } \ - else if (PREDICT_FALSE (pool_free_elts (P) < \ - POOL_REALLOC_SAFE_ELT_THRESH / 2)) \ + else if (PREDICT_FALSE (!pool_free_elts (P))) \ { \ - volatile typeof (P) *PP = &(P); \ pool_program_safe_realloc (P, thread_index, rpc_fn); \ - if (thread_index) \ - { \ - while (pool_realloc_flag (P)) \ - { \ - /* If refork required abort and consume existing elt */ \ - if (*vlib_worker_threads->node_reforks_required) \ - { \ - /* All workers at barrier realloc now */ \ - if (pool_realloc_all_at_barrier (1)) \ - pool_alloc_aligned (P, pool_max_len (P), align); \ - break; \ - } \ - pool_realloc_maybe_wait_at_barrier (); \ - } \ - if (pool_free_elts (P) == 0) \ - pool_realloc_safe_force (P); \ - ALWAYS_ASSERT (pool_free_elts (P) > 0); \ - } \ - (P) = *PP; \ + pool_realloc_wait_at_barrier (); \ + pool_alloc_aligned (P, pool_max_len (P), align); \ + pool_realloc_done_wait_at_barrier (); \ + ALWAYS_ASSERT (pool_free_elts (P) > 0); \ } \ else \ { \ |