diff options
| -rw-r--r-- | src/plugins/unittest/session_test.c | 2 | ||||
| -rw-r--r-- | src/plugins/unittest/svm_fifo_test.c | 27 | ||||
| -rw-r--r-- | src/svm/fifo_segment.c | 485 | ||||
| -rw-r--r-- | src/svm/fifo_segment.h | 39 | ||||
| -rw-r--r-- | src/svm/svm_fifo.h | 5 | ||||
| -rw-r--r-- | src/vnet/session/application_local.c | 3 | ||||
| -rw-r--r-- | src/vnet/session/application_worker.c | 3 | ||||
| -rw-r--r-- | src/vnet/session/segment_manager.c | 114 | ||||
| -rw-r--r-- | src/vnet/session/segment_manager.h | 3 |
9 files changed, 411 insertions, 270 deletions
diff --git a/src/plugins/unittest/session_test.c b/src/plugins/unittest/session_test.c index 819c42661ae..8d5566feb46 100644 --- a/src/plugins/unittest/session_test.c +++ b/src/plugins/unittest/session_test.c @@ -1814,7 +1814,7 @@ session_test_mq_speed (vlib_main_t * vm, unformat_input_t * input) } sm = app_worker_get_or_alloc_connect_segment_manager (app_wrk); - segment_manager_alloc_session_fifos (sm, &rx_fifo, &tx_fifo); + segment_manager_alloc_session_fifos (sm, 0, &rx_fifo, &tx_fifo); s.rx_fifo = rx_fifo; s.tx_fifo = tx_fifo; s.session_state = SESSION_STATE_READY; diff --git a/src/plugins/unittest/svm_fifo_test.c b/src/plugins/unittest/svm_fifo_test.c index a715fa2a660..9b3ad06fc5a 100644 --- a/src/plugins/unittest/svm_fifo_test.c +++ b/src/plugins/unittest/svm_fifo_test.c @@ -2012,6 +2012,13 @@ sfifo_test_fifo_replay (vlib_main_t * vm, unformat_input_t * input) static fifo_segment_main_t segment_main; +static svm_fifo_t * +fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes, + fifo_segment_ftype_t ftype) +{ + return fifo_segment_alloc_fifo_w_slice (fs, 0, data_bytes, ftype); +} + static int sfifo_test_fifo_segment_hello_world (int verbose) { @@ -2340,9 +2347,7 @@ sfifo_test_fifo_segment_slave (int verbose) fifo_segment_create_args_t _a, *a = &_a; fifo_segment_main_t *sm = &segment_main; u8 *test_data, *retrieved_data = 0; - fifo_segment_t *sp; - fifo_segment_header_t *fsh; - ssvm_shared_header_t *sh; + fifo_segment_t *fs; svm_fifo_t *f; u32 *result; int rv, i; @@ -2357,13 +2362,11 @@ sfifo_test_fifo_segment_slave (int verbose) SFIFO_TEST (!rv, "svm_fifo_segment_attach returned %d", rv); - sp = fifo_segment_get_segment (sm, a->new_segment_indices[0]); + fs = fifo_segment_get_segment (sm, a->new_segment_indices[0]); vec_free (a->new_segment_indices); - sh = sp->ssvm.sh; - fsh = (fifo_segment_header_t *) sh->opaque[0]; /* might wanna wait.. */ - f = fsh->fifos; + f = fifo_segment_get_slice_fifo_list (fs, 0); /* Lazy bastards united */ test_data = format (0, "Hello world%c", 0); @@ -2530,7 +2533,7 @@ sfifo_test_fifo_segment_prealloc (int verbose) free_space = fifo_segment_free_bytes (fs); SFIFO_TEST (free_space <= 256 << 10, "free space expected %u is %u", 256 << 10, free_space); - rv = fifo_segment_prealloc_fifo_chunks (fs, 4096, 50); + rv = fifo_segment_prealloc_fifo_chunks (fs, 0, 4096, 50); SFIFO_TEST (rv == 0, "chunk prealloc should work"); rv = fifo_segment_num_free_chunks (fs, 4096); SFIFO_TEST (rv == 50, "prealloc chunks expected %u is %u", 50, rv); @@ -2542,7 +2545,7 @@ sfifo_test_fifo_segment_prealloc (int verbose) SFIFO_TEST (rv == 4096 * 50, "chunk free space expected %u is %u", 4096 * 50, rv); - rv = fifo_segment_prealloc_fifo_hdrs (fs, 50); + rv = fifo_segment_prealloc_fifo_hdrs (fs, 0, 50); SFIFO_TEST (rv == 0, "fifo hdr prealloc should work"); rv = fifo_segment_num_free_fifos (fs); SFIFO_TEST (rv == 50, "prealloc fifo hdrs expected %u is %u", 50, rv); @@ -2585,7 +2588,7 @@ sfifo_test_fifo_segment_prealloc (int verbose) /* Preallocate as many more chunks as possible. Heap is almost full * so we may not use all the free space*/ alloc = 0; - while (!fifo_segment_prealloc_fifo_chunks (fs, 4096, 1)) + while (!fifo_segment_prealloc_fifo_chunks (fs, 0, 4096, 1)) alloc++; SFIFO_TEST (alloc, "chunk prealloc should work %u", alloc); rv = fifo_segment_num_free_chunks (fs, 4096); @@ -2607,10 +2610,10 @@ sfifo_test_fifo_segment_prealloc (int verbose) f = fifo_segment_alloc_fifo (fs, 200 << 10, FIFO_SEGMENT_RX_FIFO); SFIFO_TEST (f == 0, "fifo alloc should fail"); - rv = fifo_segment_prealloc_fifo_chunks (fs, 4096, 50); + rv = fifo_segment_prealloc_fifo_chunks (fs, 0, 4096, 50); SFIFO_TEST (rv == -1, "chunk prealloc should fail"); - rv = fifo_segment_prealloc_fifo_hdrs (fs, 50); + rv = fifo_segment_prealloc_fifo_hdrs (fs, 0, 50); SFIFO_TEST (rv == -1, "fifo hdr prealloc should fail"); /* diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c index 698a422127d..95b88836ddb 100644 --- a/src/svm/fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -34,6 +34,30 @@ fs_free_space (fifo_segment_t * fs) return dlminfo.fordblks; } +static inline void +fsh_free_bytes_sub (fifo_segment_header_t * fsh, int size) +{ + clib_atomic_fetch_sub_rel (&fsh->n_free_bytes, size); +} + +static inline uword +fsh_n_free_bytes (fifo_segment_header_t * fsh) +{ + return clib_atomic_load_relax_n (&fsh->n_free_bytes); +} + +static inline fifo_segment_slice_t * +fsh_slice_get (fifo_segment_header_t * fsh, u32 slice_index) +{ + return &fsh->slices[slice_index]; +} + +static inline void +fsh_active_fifos_update (fifo_segment_header_t * fsh, int inc) +{ + clib_atomic_fetch_add_rel (&fsh->n_active_fifos, inc); +} + /** * Initialize fifo segment shared header */ @@ -41,15 +65,45 @@ int fifo_segment_init (fifo_segment_t * fs) { fifo_segment_header_t *fsh; + fifo_segment_slice_t *fss; ssvm_shared_header_t *sh; + u32 max_chunk_sz; + uword max_fifo; void *oldheap; + int i; sh = fs->ssvm.sh; oldheap = ssvm_push_heap (sh); - fsh = clib_mem_alloc (sizeof (*fsh)); + /* + * Manually align the fifo segment header to sizeof(uword) = 8 bytes. + * Long story made short: the "process-private" fifo segment + * is allocated from the main heap, not mmapped. dlmalloc + * only guarantees 4-byte alignment, and on aarch64 + * the fsh can end up 4-byte but not 8-byte aligned. + * That eventually causes the atomic op in fifo_segment_update_free_bytes + * to backfire. + */ + fsh = clib_mem_alloc_aligned (sizeof (*fsh), sizeof (uword)); clib_memset (fsh, 0, sizeof (*fsh)); fs->h = sh->opaque[0] = fsh; + fs->n_slices = clib_max (fs->n_slices, 1); + + fsh->ssvm_sh = fs->ssvm.sh; + fsh->n_slices = fs->n_slices; + max_fifo = clib_min ((fs_free_space (fs) - 4096) / 2, + FIFO_SEGMENT_MAX_FIFO_SIZE); + fsh->max_log2_chunk_size = max_log2 (max_fifo); + + fsh->slices = clib_mem_alloc (sizeof (*fss) * fs->n_slices); + clib_memset (fsh->slices, 0, sizeof (*fss) * fs->n_slices); + max_chunk_sz = fsh->max_log2_chunk_size - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE; + + for (i = 0; i < fs->n_slices; i++) + { + fss = fsh_slice_get (fsh, i); + vec_validate_init_empty (fss->free_chunks, max_chunk_sz, 0); + } ssvm_pop_heap (oldheap); @@ -98,30 +152,30 @@ fifo_segment_create (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) int fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) { - fifo_segment_t *s; + fifo_segment_t *fs; int rv; - pool_get_zero (sm->segments, s); + pool_get_zero (sm->segments, fs); - s->ssvm.ssvm_size = a->segment_size; - s->ssvm.my_pid = getpid (); - s->ssvm.name = format (0, "%s%c", a->segment_name, 0); - s->ssvm.requested_va = sm->next_baseva; + fs->ssvm.ssvm_size = a->segment_size; + fs->ssvm.my_pid = getpid (); + fs->ssvm.name = format (0, "%s%c", a->segment_name, 0); + fs->ssvm.requested_va = sm->next_baseva; if (a->segment_type == SSVM_SEGMENT_MEMFD) - s->ssvm.fd = a->memfd_fd; + fs->ssvm.fd = a->memfd_fd; else - s->ssvm.attach_timeout = sm->timeout_in_seconds; + fs->ssvm.attach_timeout = sm->timeout_in_seconds; - if ((rv = ssvm_slave_init (&s->ssvm, a->segment_type))) + if ((rv = ssvm_slave_init (&fs->ssvm, a->segment_type))) { - _vec_len (s) = vec_len (s) - 1; + _vec_len (fs) = vec_len (fs) - 1; return (rv); } /* Fish the segment header */ - s->h = s->ssvm.sh->opaque[0]; + fs->h = fs->ssvm.sh->opaque[0]; - vec_add1 (a->new_segment_indices, s - sm->segments); + vec_add1 (a->new_segment_indices, fs - sm->segments); return (0); } @@ -139,12 +193,6 @@ fifo_segment_index (fifo_segment_main_t * sm, fifo_segment_t * s) return s - sm->segments; } -void * -svm_fifo_segment_heap (fifo_segment_t * seg) -{ - return seg->ssvm.sh->heap; -} - fifo_segment_t * fifo_segment_get_segment (fifo_segment_main_t * sm, u32 segment_index) { @@ -169,41 +217,41 @@ fifo_segment_main_init (fifo_segment_main_t * sm, u64 baseva, static inline u32 fs_freelist_for_size (u32 size) { - return max_log2 (size) - max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE); + return max_log2 (size) - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE; } static inline u32 fs_freelist_index_to_size (u32 fl_index) { - return 1 << (fl_index + max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE)); + return 1 << (fl_index + FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE); } static inline int -fs_chunk_size_is_valid (u32 size) +fs_chunk_size_is_valid (fifo_segment_header_t * fsh, u32 size) { /* * 4K minimum. It's not likely that anything good will happen * with a smaller FIFO. */ return size >= FIFO_SEGMENT_MIN_FIFO_SIZE - && size <= FIFO_SEGMENT_MAX_FIFO_SIZE; + && size <= (1 << fsh->max_log2_chunk_size); } static svm_fifo_t * -fs_try_alloc_fifo_freelist (fifo_segment_t * fs, u32 fl_index, u32 data_bytes) +fs_try_alloc_fifo_freelist (fifo_segment_slice_t * fss, + u32 fl_index, u32 data_bytes) { - fifo_segment_header_t *fsh = fs->h; svm_fifo_chunk_t *c; svm_fifo_t *f; - f = fsh->free_fifos; - c = fsh->free_chunks[fl_index]; + f = fss->free_fifos; + c = fss->free_chunks[fl_index]; if (!f || !c) return 0; - fsh->free_fifos = f->next; - fsh->free_chunks[fl_index] = c->next; + fss->free_fifos = f->next; + fss->free_chunks[fl_index] = c->next; c->next = c; c->start_byte = 0; c->length = data_bytes; @@ -211,44 +259,44 @@ fs_try_alloc_fifo_freelist (fifo_segment_t * fs, u32 fl_index, u32 data_bytes) f->start_chunk = c; f->end_chunk = c; - fsh->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index); + fss->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index); return f; } static svm_fifo_t * -fs_try_alloc_fifo_freelist_multi_chunk (fifo_segment_t * fs, u32 data_bytes) +fs_try_alloc_fifo_freelist_multi_chunk (fifo_segment_header_t * fsh, + fifo_segment_slice_t * fss, + u32 data_bytes) { svm_fifo_chunk_t *c, *first = 0, *last = 0; - fifo_segment_header_t *fsh = fs->h; u32 fl_index, fl_size, n_alloc = 0; svm_fifo_t *f; - f = fsh->free_fifos; + f = fss->free_fifos; if (!f) { - void *oldheap = ssvm_push_heap (fs->ssvm.sh); + void *oldheap = ssvm_push_heap (fsh->ssvm_sh); f = clib_mem_alloc_aligned (sizeof (*f), CLIB_CACHE_LINE_BYTES); ssvm_pop_heap (oldheap); if (!f) return 0; memset (f, 0, sizeof (*f)); - fsh->n_free_bytes -= sizeof (*f); + fsh_free_bytes_sub (fsh, sizeof (*f)); } else { - fsh->free_fifos = f->next; + fss->free_fifos = f->next; } fl_index = fs_freelist_for_size (data_bytes) - 1; - vec_validate_init_empty (fsh->free_chunks, fl_index, 0); fl_size = fs_freelist_index_to_size (fl_index); while (data_bytes) { - c = fsh->free_chunks[fl_index]; + c = fss->free_chunks[fl_index]; if (c) { - fsh->free_chunks[fl_index] = c->next; + fss->free_chunks[fl_index] = c->next; if (!last) last = c; c->next = first; @@ -267,14 +315,15 @@ fs_try_alloc_fifo_freelist_multi_chunk (fifo_segment_t * fs, u32 data_bytes) f->start_chunk = first; f->end_chunk = last; last->next = first; - fsh->n_fl_chunk_bytes -= n_alloc; + fss->n_fl_chunk_bytes -= n_alloc; return f; } static int -fs_try_alloc_fifo_batch (fifo_segment_t * fs, u32 fl_index, u32 batch_size) +fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh, + fifo_segment_slice_t * fss, + u32 fl_index, u32 batch_size) { - fifo_segment_header_t *fsh = fs->h; u32 size, hdrs, rounded_data_size; svm_fifo_chunk_t *c; svm_fifo_t *f; @@ -282,12 +331,11 @@ fs_try_alloc_fifo_batch (fifo_segment_t * fs, u32 fl_index, u32 batch_size) u8 *fmem; int i; - vec_validate_init_empty (fsh->free_chunks, fl_index, 0); rounded_data_size = fs_freelist_index_to_size (fl_index); hdrs = sizeof (*f) + sizeof (*c); size = (hdrs + rounded_data_size) * batch_size; - oldheap = ssvm_push_heap (fs->ssvm.sh); + oldheap = ssvm_push_heap (fsh->ssvm_sh); fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ , 0 /* os_out_of_memory */ ); @@ -302,18 +350,18 @@ fs_try_alloc_fifo_batch (fifo_segment_t * fs, u32 fl_index, u32 batch_size) { f = (svm_fifo_t *) fmem; memset (f, 0, sizeof (*f)); - f->next = fsh->free_fifos; - fsh->free_fifos = f; + f->next = fss->free_fifos; + fss->free_fifos = f; c = (svm_fifo_chunk_t *) (fmem + sizeof (*f)); c->start_byte = 0; c->length = rounded_data_size; - c->next = fsh->free_chunks[fl_index]; - fsh->free_chunks[fl_index] = c; + c->next = fss->free_chunks[fl_index]; + fss->free_chunks[fl_index] = c; fmem += hdrs + rounded_data_size; } - fsh->n_fl_chunk_bytes += batch_size * rounded_data_size; - fsh->n_free_bytes -= size; + fss->n_fl_chunk_bytes += batch_size * rounded_data_size; + fsh_free_bytes_sub (fsh, size); return 0; } @@ -328,45 +376,46 @@ fs_try_alloc_fifo_batch (fifo_segment_t * fs, u32 fl_index, u32 batch_size) * - grab multiple fifo chunks from freelists */ static svm_fifo_t * -fs_try_alloc_fifo (fifo_segment_t * fs, u32 data_bytes) +fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, + u32 data_bytes) { - fifo_segment_header_t *fsh = fs->h; u32 fifo_sz, fl_index; svm_fifo_t *f = 0; + uword n_free_bytes; fl_index = fs_freelist_for_size (data_bytes); - vec_validate_init_empty (fsh->free_chunks, fl_index, 0); fifo_sz = sizeof (svm_fifo_t) + sizeof (svm_fifo_chunk_t); fifo_sz += 1 << max_log2 (data_bytes); - if (fsh->free_fifos && fsh->free_chunks[fl_index]) + if (fss->free_fifos && fss->free_chunks[fl_index]) { - f = fs_try_alloc_fifo_freelist (fs, fl_index, data_bytes); + f = fs_try_alloc_fifo_freelist (fss, fl_index, data_bytes); if (f) goto done; } - if (fifo_sz * FIFO_SEGMENT_ALLOC_BATCH_SIZE < fsh->n_free_bytes) + n_free_bytes = fsh_n_free_bytes (fsh); + if (fifo_sz * FIFO_SEGMENT_ALLOC_BATCH_SIZE < n_free_bytes) { - if (fs_try_alloc_fifo_batch (fs, fl_index, + if (fs_try_alloc_fifo_batch (fsh, fss, fl_index, FIFO_SEGMENT_ALLOC_BATCH_SIZE)) goto done; - f = fs_try_alloc_fifo_freelist (fs, fl_index, data_bytes); + f = fs_try_alloc_fifo_freelist (fss, fl_index, data_bytes); goto done; } - if (fifo_sz <= fsh->n_free_bytes) + if (fifo_sz <= n_free_bytes) { - void *oldheap = ssvm_push_heap (fs->ssvm.sh); + void *oldheap = ssvm_push_heap (fsh->ssvm_sh); f = svm_fifo_create (data_bytes); ssvm_pop_heap (oldheap); if (f) { - fsh->n_free_bytes -= fifo_sz; + fsh_free_bytes_sub (fsh, fifo_sz); goto done; } } - if (data_bytes <= fsh->n_fl_chunk_bytes) - f = fs_try_alloc_fifo_freelist_multi_chunk (fs, data_bytes); + if (data_bytes <= fss->n_fl_chunk_bytes) + f = fs_try_alloc_fifo_freelist_multi_chunk (fsh, fss, data_bytes); done: @@ -377,32 +426,29 @@ done: * Allocate fifo in fifo segment */ svm_fifo_t * -fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes, - fifo_segment_ftype_t ftype) +fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index, + u32 data_bytes, fifo_segment_ftype_t ftype) { - fifo_segment_header_t *fsh; + fifo_segment_header_t *fsh = fs->h; + fifo_segment_slice_t *fss; svm_fifo_t *f = 0; - if (!fs_chunk_size_is_valid (data_bytes)) - { - clib_warning ("fifo size out of range %d", data_bytes); - return 0; - } - - fsh = fs->h; - ssvm_lock_non_recursive (fs->ssvm.sh, 1); + ASSERT (slice_index < fs->n_slices); - f = fs_try_alloc_fifo (fs, data_bytes); + fss = fsh_slice_get (fsh, slice_index); + f = fs_try_alloc_fifo (fsh, fss, data_bytes); if (!f) goto done; + f->slice_index = slice_index; + /* (re)initialize the fifo, as in svm_fifo_create */ svm_fifo_init (f, data_bytes); /* Initialize chunks and rbtree for multi-chunk fifos */ if (f->start_chunk->next != f->start_chunk) { - void *oldheap = ssvm_push_heap (fs->ssvm.sh); + void *oldheap = ssvm_push_heap (fsh->ssvm_sh); svm_fifo_init_chunks (f); ssvm_pop_heap (oldheap); } @@ -413,18 +459,17 @@ fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes, * only one. */ if (ftype == FIFO_SEGMENT_RX_FIFO) { - if (fsh->fifos) + if (fss->fifos) { - fsh->fifos->prev = f; - f->next = fsh->fifos; + fss->fifos->prev = f; + f->next = fss->fifos; } - fsh->fifos = f; + fss->fifos = f; f->flags |= SVM_FIFO_F_LL_TRACKED; } - fsh->n_active_fifos++; + fsh_active_fifos_update (fsh, 1); done: - ssvm_unlock_non_recursive (fs->ssvm.sh); return (f); } @@ -434,9 +479,9 @@ done: void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) { + fifo_segment_header_t *fsh = fs->h; svm_fifo_chunk_t *cur, *next; - fifo_segment_header_t *fsh; - ssvm_shared_header_t *sh; + fifo_segment_slice_t *fss; void *oldheap; int fl_index; @@ -445,10 +490,7 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) if (--f->refcnt > 0) return; - sh = fs->ssvm.sh; - fsh = fs->h; - - ssvm_lock_non_recursive (sh, 2); + fss = fsh_slice_get (fsh, f->slice_index); /* Remove from active list. Only rx fifos are tracked */ if (f->flags & SVM_FIFO_F_LL_TRACKED) @@ -456,16 +498,16 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) if (f->prev) f->prev->next = f->next; else - fsh->fifos = f->next; + fss->fifos = f->next; if (f->next) f->next->prev = f->prev; f->flags &= ~SVM_FIFO_F_LL_TRACKED; } /* Add to free list */ - f->next = fsh->free_fifos; + f->next = fss->free_fifos; f->prev = 0; - fsh->free_fifos = f; + fss->free_fifos = f; /* Free fifo chunks */ cur = f->start_chunk; @@ -473,10 +515,10 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) { next = cur->next; fl_index = fs_freelist_for_size (cur->length); - ASSERT (fl_index < vec_len (fsh->free_chunks)); - cur->next = fsh->free_chunks[fl_index]; - fsh->free_chunks[fl_index] = cur; - fsh->n_fl_chunk_bytes += fs_freelist_index_to_size (fl_index); + ASSERT (fl_index < vec_len (fss->free_chunks)); + cur->next = fss->free_chunks[fl_index]; + fss->free_chunks[fl_index] = cur; + fss->n_fl_chunk_bytes += fs_freelist_index_to_size (fl_index); cur = next; } while (cur != f->start_chunk); @@ -484,7 +526,7 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) f->start_chunk = f->end_chunk = f->new_chunks = 0; f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0; - oldheap = ssvm_push_heap (sh); + oldheap = ssvm_push_heap (fsh->ssvm_sh); svm_fifo_free_chunk_lookup (f); ssvm_pop_heap (oldheap); @@ -497,23 +539,25 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) f->master_thread_index = ~0; } - fsh->n_active_fifos--; - ssvm_unlock_non_recursive (sh); + fsh_active_fifos_update (fsh, -1); } int -fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 batch_size) +fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 slice_index, + u32 batch_size) { fifo_segment_header_t *fsh = fs->h; + fifo_segment_slice_t *fss; svm_fifo_t *f; void *oldheap; u32 size; u8 *fmem; int i; + fss = fsh_slice_get (fsh, slice_index); size = (sizeof (*f)) * batch_size; - oldheap = ssvm_push_heap (fs->ssvm.sh); + oldheap = ssvm_push_heap (fsh->ssvm_sh); fmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ , 0 /* os_out_of_memory */ ); @@ -528,39 +572,39 @@ fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 batch_size) { f = (svm_fifo_t *) fmem; memset (f, 0, sizeof (*f)); - f->next = fsh->free_fifos; - fsh->free_fifos = f; + f->next = fss->free_fifos; + fss->free_fifos = f; fmem += sizeof (*f); } - fsh->n_free_bytes -= size; + fsh_free_bytes_sub (fsh, size); return 0; } int -fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 chunk_size, - u32 batch_size) +fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 slice_index, + u32 chunk_size, u32 batch_size) { - fifo_segment_header_t *fsh = fs->h; u32 size, rounded_data_size, fl_index; + fifo_segment_header_t *fsh = fs->h; + fifo_segment_slice_t *fss; svm_fifo_chunk_t *c; void *oldheap; u8 *cmem; int i; - if (!fs_chunk_size_is_valid (chunk_size)) + if (!fs_chunk_size_is_valid (fsh, chunk_size)) { clib_warning ("chunk size out of range %d", chunk_size); return -1; } fl_index = fs_freelist_for_size (chunk_size); - vec_validate_init_empty (fsh->free_chunks, fl_index, 0); rounded_data_size = fs_freelist_index_to_size (fl_index); size = (sizeof (*c) + rounded_data_size) * batch_size; - oldheap = ssvm_push_heap (fs->ssvm.sh); + oldheap = ssvm_push_heap (fsh->ssvm_sh); cmem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ , 0 /* os_out_of_memory */ ); @@ -570,19 +614,21 @@ fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 chunk_size, if (cmem == 0) return -1; + fss = fsh_slice_get (fsh, slice_index); + /* Carve fifo + chunk space */ for (i = 0; i < batch_size; i++) { c = (svm_fifo_chunk_t *) cmem; c->start_byte = 0; c->length = rounded_data_size; - c->next = fsh->free_chunks[fl_index]; - fsh->free_chunks[fl_index] = c; + c->next = fss->free_chunks[fl_index]; + fss->free_chunks[fl_index] = c; cmem += sizeof (*c) + rounded_data_size; } - fsh->n_fl_chunk_bytes += batch_size * rounded_data_size; - fsh->n_free_bytes -= size; + fss->n_fl_chunk_bytes += batch_size * rounded_data_size; + fsh_free_bytes_sub (fsh, size); return 0; } @@ -596,21 +642,23 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, u32 * n_fifo_pairs) { u32 rx_rounded_data_size, tx_rounded_data_size, pair_size, pairs_to_alloc; - int rx_fl_index, tx_fl_index; + fifo_segment_header_t *fsh = fs->h; + int rx_fl_index, tx_fl_index, i; + fifo_segment_slice_t *fss; + u32 hdrs, pairs_per_slice; uword space_available; - u32 hdrs; /* Parameter check */ if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0) return; - if (!fs_chunk_size_is_valid (rx_fifo_size)) + if (!fs_chunk_size_is_valid (fsh, rx_fifo_size)) { clib_warning ("rx fifo_size out of range %d", rx_fifo_size); return; } - if (!fs_chunk_size_is_valid (tx_fifo_size)) + if (!fs_chunk_size_is_valid (fsh, tx_fifo_size)) { clib_warning ("tx fifo_size out of range %d", tx_fifo_size); return; @@ -628,42 +676,39 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, space_available = fs_free_space (fs); pairs_to_alloc = space_available / pair_size; pairs_to_alloc = clib_min (pairs_to_alloc, *n_fifo_pairs); + pairs_per_slice = pairs_to_alloc / fs->n_slices; - if (!pairs_to_alloc) + if (!pairs_per_slice) return; - if (fs_try_alloc_fifo_batch (fs, rx_fl_index, pairs_to_alloc)) - clib_warning ("rx prealloc failed: pairs %u", pairs_to_alloc); - if (fs_try_alloc_fifo_batch (fs, tx_fl_index, pairs_to_alloc)) - clib_warning ("tx prealloc failed: pairs %u", pairs_to_alloc); + for (i = 0; i < fs->n_slices; i++) + { + fss = fsh_slice_get (fsh, i); + if (fs_try_alloc_fifo_batch (fsh, fss, rx_fl_index, pairs_to_alloc)) + clib_warning ("rx prealloc failed: pairs %u", pairs_to_alloc); + if (fs_try_alloc_fifo_batch (fsh, fss, tx_fl_index, pairs_to_alloc)) + clib_warning ("tx prealloc failed: pairs %u", pairs_to_alloc); + } /* Account for the pairs allocated */ - *n_fifo_pairs -= pairs_to_alloc; + *n_fifo_pairs -= pairs_per_slice * fs->n_slices; } int fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, u32 chunk_size) { - ssvm_shared_header_t *sh; + fifo_segment_header_t *fsh = fs->h; + fifo_segment_slice_t *fss; svm_fifo_chunk_t *c; void *oldheap; int fl_index; - if (!fs_chunk_size_is_valid (chunk_size)) - { - clib_warning ("chunk size out of range %d", chunk_size); - return -1; - } - fl_index = fs_freelist_for_size (chunk_size); + fss = fsh_slice_get (fsh, f->slice_index); - sh = fs->ssvm.sh; - ssvm_lock_non_recursive (sh, 1); - - vec_validate_init_empty (fs->h->free_chunks, fl_index, 0); - c = fs->h->free_chunks[fl_index]; + oldheap = ssvm_push_heap (fsh->ssvm_sh); - oldheap = ssvm_push_heap (sh); + c = fss->free_chunks[fl_index]; if (!c) { @@ -671,50 +716,47 @@ fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, u32 chunk_size) if (!c) { ssvm_pop_heap (oldheap); - ssvm_unlock_non_recursive (sh); return -1; } - fs->h->n_free_bytes -= chunk_size + sizeof (*c); + fsh_free_bytes_sub (fsh, chunk_size + sizeof (*c)); } else { - fs->h->free_chunks[fl_index] = c->next; + fss->free_chunks[fl_index] = c->next; c->next = 0; - fs->h->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index); + fss->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index); } svm_fifo_add_chunk (f, c); ssvm_pop_heap (oldheap); - ssvm_unlock_non_recursive (sh); return 0; } int fifo_segment_collect_fifo_chunks (fifo_segment_t * fs, svm_fifo_t * f) { + fifo_segment_header_t *fsh = fs->h; svm_fifo_chunk_t *cur, *next; - ssvm_shared_header_t *sh; + fifo_segment_slice_t *fss; void *oldheap; int fl_index; - sh = fs->ssvm.sh; - ssvm_lock_non_recursive (sh, 1); - - oldheap = ssvm_push_heap (sh); + oldheap = ssvm_push_heap (fsh->ssvm_sh); cur = svm_fifo_collect_chunks (f); + fss = fsh_slice_get (fsh, f->slice_index); + while (cur) { next = cur->next; fl_index = fs_freelist_for_size (cur->length); - cur->next = fs->h->free_chunks[fl_index]; - fs->h->free_chunks[fl_index] = cur; + cur->next = fss->free_chunks[fl_index]; + fss->free_chunks[fl_index] = cur; cur = next; } ssvm_pop_heap (oldheap); - ssvm_unlock_non_recursive (sh); return 0; } @@ -725,17 +767,16 @@ fifo_segment_collect_fifo_chunks (fifo_segment_t * fs, svm_fifo_t * f) u32 fifo_segment_num_fifos (fifo_segment_t * fs) { - return fs->h->n_active_fifos; + return clib_atomic_load_relax_n (&fs->h->n_active_fifos); } -u32 -fifo_segment_num_free_fifos (fifo_segment_t * fs) +static u32 +fs_slice_num_free_fifos (fifo_segment_slice_t * fss) { - fifo_segment_header_t *fsh = fs->h; svm_fifo_t *f; u32 count = 0; - f = fsh->free_fifos; + f = fss->free_fifos; if (f == 0) return 0; @@ -748,21 +789,34 @@ fifo_segment_num_free_fifos (fifo_segment_t * fs) } u32 -fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size) +fifo_segment_num_free_fifos (fifo_segment_t * fs) +{ + fifo_segment_header_t *fsh = fs->h; + fifo_segment_slice_t *fss; + int slice_index; + u32 count = 0; + + for (slice_index = 0; slice_index < fs->n_slices; slice_index++) + { + fss = fsh_slice_get (fsh, slice_index); + count += fs_slice_num_free_fifos (fss); + } + return count; +} + +static u32 +fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size) { u32 count = 0, rounded_size, fl_index; - fifo_segment_header_t *fsh; svm_fifo_chunk_t *c; int i; - fsh = fs->h; - /* Count all free chunks? */ if (size == ~0) { - for (i = 0; i < vec_len (fsh->free_chunks); i++) + for (i = 0; i < vec_len (fss->free_chunks); i++) { - c = fsh->free_chunks[i]; + c = fss->free_chunks[i]; if (c == 0) continue; @@ -778,10 +832,10 @@ fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size) rounded_size = (1 << (max_log2 (size))); fl_index = fs_freelist_for_size (rounded_size); - if (fl_index >= vec_len (fsh->free_chunks)) + if (fl_index >= vec_len (fss->free_chunks)) return 0; - c = fsh->free_chunks[fl_index]; + c = fss->free_chunks[fl_index]; if (c == 0) return 0; @@ -793,34 +847,76 @@ fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size) return count; } +u32 +fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size) +{ + fifo_segment_header_t *fsh = fs->h; + fifo_segment_slice_t *fss; + int slice_index; + u32 count = 0; + + for (slice_index = 0; slice_index < fs->n_slices; slice_index++) + { + fss = fsh_slice_get (fsh, slice_index); + count += fs_slice_num_free_chunks (fss, size); + } + return count; +} + void fifo_segment_update_free_bytes (fifo_segment_t * fs) { - fs->h->n_free_bytes = fs_free_space (fs); + fifo_segment_header_t *fsh = fs->h; + clib_atomic_store_rel_n (&fsh->n_free_bytes, fs_free_space (fs)); } u32 fifo_segment_free_bytes (fifo_segment_t * fs) { - return fs->h->n_free_bytes; + return fsh_n_free_bytes (fs->h); } -u32 +uword fifo_segment_fl_chunk_bytes (fifo_segment_t * fs) { - return fs->h->n_fl_chunk_bytes; + fifo_segment_header_t *fsh = fs->h; + fifo_segment_slice_t *fss; + uword n_bytes = 0; + int slice_index; + + for (slice_index = 0; slice_index < fs->n_slices; slice_index++) + { + fss = fsh_slice_get (fsh, slice_index); + n_bytes += fss->n_fl_chunk_bytes; + } + + return n_bytes; } u8 fifo_segment_has_fifos (fifo_segment_t * fs) { - return fs->h->fifos != 0; + fifo_segment_header_t *fsh = fs->h; + fifo_segment_slice_t *fss; + int slice_index; + + for (slice_index = 0; slice_index < fs->n_slices; slice_index++) + { + fss = fsh_slice_get (fsh, slice_index); + if (fss->fifos) + return 1; + } + return 0; } svm_fifo_t * -fifo_segment_get_fifo_list (fifo_segment_t * fs) +fifo_segment_get_slice_fifo_list (fifo_segment_t * fs, u32 slice_index) { - return fs->h->fifos; + fifo_segment_header_t *fsh = fs->h; + fifo_segment_slice_t *fss; + + fss = fsh_slice_get (fsh, slice_index); + return fss->fifos; } u8 * @@ -850,10 +946,12 @@ format_fifo_segment (u8 * s, va_list * args) u32 count, indent, active_fifos, free_fifos, fifo_hdr = 0, chunk_size; fifo_segment_t *fs = va_arg (*args, fifo_segment_t *); int verbose __attribute__ ((unused)) = va_arg (*args, int); - u32 est_chunk_bytes, est_free_seg_bytes; + u32 est_chunk_bytes, est_free_seg_bytes, free_chunks; + uword chunk_bytes = 0, free_seg_bytes; fifo_segment_header_t *fsh; + fifo_segment_slice_t *fss; svm_fifo_chunk_t *c; - u64 chunk_bytes = 0; + u32 slice_index; char *address; size_t size; int i; @@ -861,9 +959,9 @@ format_fifo_segment (u8 * s, va_list * args) indent = format_get_indent (s) + 2; #if USE_DLMALLOC == 0 s = format (s, "%U segment heap: %U\n", format_white_space, indent, - format_mheap, svm_fifo_segment_heap (fs), verbose); + format_mheap, fsh->ssvm_sh->heap, verbose); s = format (s, "%U segment has %u active fifos\n", - format_white_space, indent, fifo_segment_num_fifos (fs)); + format_white_space, indent, fifo_segment_num_fifos (fsh)); #endif if (fs == 0) @@ -873,7 +971,6 @@ format_fifo_segment (u8 * s, va_list * args) return s; } - fsh = fs->h; fifo_segment_info (fs, &address, &size); active_fifos = fifo_segment_num_fifos (fs); free_fifos = fifo_segment_num_free_fifos (fs); @@ -885,39 +982,47 @@ format_fifo_segment (u8 * s, va_list * args) if (!verbose) return s; - if (fsh->free_chunks) + fsh = fs->h; + + free_chunks = fifo_segment_num_free_chunks (fs, ~0); + if (free_chunks) s = format (s, "\n\n%UFree chunks by size:\n", format_white_space, indent + 2); else s = format (s, "\n"); - for (i = 0; i < vec_len (fsh->free_chunks); i++) + for (slice_index = 0; slice_index < fs->n_slices; slice_index++) { - c = fsh->free_chunks[i]; - if (c == 0) - continue; - count = 0; - while (c) + fss = fsh_slice_get (fsh, slice_index); + for (i = 0; i < vec_len (fss->free_chunks); i++) { - c = c->next; - count++; - } + c = fss->free_chunks[i]; + if (c == 0) + continue; + count = 0; + while (c) + { + c = c->next; + count++; + } - chunk_size = fs_freelist_index_to_size (i); - s = format (s, "%U%-5u kB: %u\n", format_white_space, indent + 2, - chunk_size >> 10, count); + chunk_size = fs_freelist_index_to_size (i); + s = format (s, "%U%-5u kB: %u\n", format_white_space, indent + 2, + chunk_size >> 10, count); - chunk_bytes += count * chunk_size; + chunk_bytes += count * chunk_size; + } } fifo_hdr = free_fifos * sizeof (svm_fifo_t); est_chunk_bytes = fifo_segment_fl_chunk_bytes (fs); - est_free_seg_bytes = fsh->n_free_bytes; + est_free_seg_bytes = fifo_segment_free_bytes (fs); fifo_segment_update_free_bytes (fs); + free_seg_bytes = fifo_segment_free_bytes (fs); s = format (s, "\n%Useg free bytes: %U (%u) estimated: %U (%u)\n", format_white_space, indent + 2, format_memory_size, - fsh->n_free_bytes, fsh->n_free_bytes, format_memory_size, + free_seg_bytes, free_seg_bytes, format_memory_size, est_free_seg_bytes, est_free_seg_bytes); s = format (s, "%Uchunk free bytes: %U (%lu) estimated: %U (%u)\n", format_white_space, indent + 2, format_memory_size, chunk_bytes, diff --git a/src/svm/fifo_segment.h b/src/svm/fifo_segment.h index 7922274cecf..ff36f45c4f2 100644 --- a/src/svm/fifo_segment.h +++ b/src/svm/fifo_segment.h @@ -26,8 +26,9 @@ typedef enum FIFO_SEGMENT_N_FTYPES } fifo_segment_ftype_t; -#define FIFO_SEGMENT_MIN_FIFO_SIZE 4096 /* 4kB min fifo size */ -#define FIFO_SEGMENT_MAX_FIFO_SIZE (2 << 30) /* 2GB max fifo size */ +#define FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE 12 /**< 4kB min fifo size */ +#define FIFO_SEGMENT_MIN_FIFO_SIZE 4096 /**< 4kB min fifo size */ +#define FIFO_SEGMENT_MAX_FIFO_SIZE (2 << 30) /**< 2GB max fifo size */ #define FIFO_SEGMENT_ALLOC_BATCH_SIZE 32 /* Allocation quantum */ typedef enum fifo_segment_flags_ @@ -36,21 +37,30 @@ typedef enum fifo_segment_flags_ FIFO_SEGMENT_F_WILL_DELETE = 1 << 1, } fifo_segment_flags_t; -typedef struct +typedef struct fifo_segment_slice_ { svm_fifo_t *fifos; /**< Linked list of active RX fifos */ svm_fifo_t *free_fifos; /**< Freelists by fifo size */ svm_fifo_chunk_t **free_chunks; /**< Freelists by chunk size */ + u32 n_fl_chunk_bytes; /**< Chunk bytes on freelist */ +} fifo_segment_slice_t; + +typedef struct +{ + fifo_segment_slice_t *slices; /** Fixed array of slices */ + ssvm_shared_header_t *ssvm_sh; /**< Pointer to fs ssvm shared hdr */ + uword n_free_bytes; /**< Segment free bytes */ u32 n_active_fifos; /**< Number of active fifos */ + u32 max_log2_chunk_size; /**< Max log2(chunk size) for fs */ u8 flags; /**< Segment flags */ - u32 n_free_bytes; /**< Bytes usable for new allocs */ - u32 n_fl_chunk_bytes; /**< Chunk bytes on freelist */ + u8 n_slices; /**< Number of slices */ } fifo_segment_header_t; typedef struct { ssvm_private_t ssvm; /**< ssvm segment data */ fifo_segment_header_t *h; /**< fifo segment data */ + u8 n_slices; /**< number of fifo segment slices */ } fifo_segment_t; typedef struct @@ -90,9 +100,10 @@ void fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size); * @param ftype fifo type @ref fifo_segment_ftype_t * @return new fifo or 0 if alloc failed */ -svm_fifo_t *fifo_segment_alloc_fifo (fifo_segment_t * fs, - u32 data_bytes, - fifo_segment_ftype_t ftype); +svm_fifo_t *fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, + u32 slice_index, + u32 data_bytes, + fifo_segment_ftype_t ftype); /** * Free fifo allocated in fifo segment @@ -111,7 +122,8 @@ void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f); * @param batch_size number of chunks to be allocated * @return 0 on success, negative number otherwise */ -int fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 batch_size); +int fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 slice_index, + u32 batch_size); /** * Try to preallocate fifo chunks on segment @@ -124,8 +136,8 @@ int fifo_segment_prealloc_fifo_hdrs (fifo_segment_t * fs, u32 batch_size); * @param batch_size number of chunks to be allocated * @return 0 on success, negative number otherwise */ -int fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 chunk_size, - u32 batch_size); +int fifo_segment_prealloc_fifo_chunks (fifo_segment_t * fs, u32 slice_index, + u32 chunk_size, u32 batch_size); /** * Pre-allocates fifo pairs in fifo segment * @@ -192,9 +204,10 @@ void fifo_segment_update_free_bytes (fifo_segment_t * fs); * @param fs fifo segment * @return free bytes on chunk free lists */ -u32 fifo_segment_fl_chunk_bytes (fifo_segment_t * fs); +uword fifo_segment_fl_chunk_bytes (fifo_segment_t * fs); u8 fifo_segment_has_fifos (fifo_segment_t * fs); -svm_fifo_t *fifo_segment_get_fifo_list (fifo_segment_t * fs); +svm_fifo_t *fifo_segment_get_slice_fifo_list (fifo_segment_t * fs, + u32 slice_index); u32 fifo_segment_num_fifos (fifo_segment_t * fs); u32 fifo_segment_num_free_fifos (fifo_segment_t * fs); /** diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index ce4c53d9abe..0b7c70790c0 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -78,11 +78,12 @@ typedef struct _svm_fifo CLIB_CACHE_LINE_ALIGN_MARK (shared_first); u32 size; /**< size of the fifo in bytes */ u32 nitems; /**< usable size (size-1) */ - u8 flags; /**< fifo flags */ svm_fifo_chunk_t *start_chunk;/**< first chunk in fifo chunk list */ svm_fifo_chunk_t *end_chunk; /**< end chunk in fifo chunk list */ svm_fifo_chunk_t *new_chunks; /**< chunks yet to be added to list */ - rb_tree_t chunk_lookup; + rb_tree_t chunk_lookup; /**< rbtree for chunk lookup */ + u8 flags; /**< fifo flags */ + u8 slice_index; /**< segment slice for fifo */ CLIB_CACHE_LINE_ALIGN_MARK (shared_second); volatile u32 has_event; /**< non-zero if deq event exists */ diff --git a/src/vnet/session/application_local.c b/src/vnet/session/application_local.c index 7d8fb468f5a..97cda12b7da 100644 --- a/src/vnet/session/application_local.c +++ b/src/vnet/session/application_local.c @@ -191,7 +191,8 @@ ct_init_local_session (app_worker_t * client_wrk, app_worker_t * server_wrk, } seg = segment_manager_get_segment_w_lock (sm, seg_index); - rv = segment_manager_try_alloc_fifos (seg, props->rx_fifo_size, + rv = segment_manager_try_alloc_fifos (seg, ls->thread_index, + props->rx_fifo_size, props->tx_fifo_size, &ls->rx_fifo, &ls->tx_fifo); if (rv) diff --git a/src/vnet/session/application_worker.c b/src/vnet/session/application_worker.c index c45679735b9..bb51cb71f03 100644 --- a/src/vnet/session/application_worker.c +++ b/src/vnet/session/application_worker.c @@ -150,7 +150,8 @@ app_worker_alloc_session_fifos (segment_manager_t * sm, session_t * s) svm_fifo_t *rx_fifo = 0, *tx_fifo = 0; int rv; - if ((rv = segment_manager_alloc_session_fifos (sm, &rx_fifo, &tx_fifo))) + if ((rv = segment_manager_alloc_session_fifos (sm, s->thread_index, + &rx_fifo, &tx_fifo))) return rv; rx_fifo->master_session_index = s->session_index; diff --git a/src/vnet/session/segment_manager.c b/src/vnet/session/segment_manager.c index 9c33e35e8f0..5247b0e7461 100644 --- a/src/vnet/session/segment_manager.c +++ b/src/vnet/session/segment_manager.c @@ -54,6 +54,7 @@ segment_manager_props_init (segment_manager_props_t * props) props->rx_fifo_size = sm_main.default_fifo_size; props->tx_fifo_size = sm_main.default_fifo_size; props->evt_q_size = sm_main.default_app_mq_size; + props->n_slices = vlib_num_workers () + 1; return props; } @@ -151,6 +152,7 @@ segment_manager_add_segment (segment_manager_t * sm, u32 segment_size) /* * Initialize fifo segment */ + fs->n_slices = props->n_slices; fifo_segment_init (fs); /* @@ -470,30 +472,34 @@ segment_manager_has_fifos (segment_manager_t * sm) void segment_manager_del_sessions (segment_manager_t * sm) { - fifo_segment_t *fifo_segment; session_handle_t *handles = 0, *handle; + fifo_segment_t *fs; session_t *session; - svm_fifo_t *fifo; + int slice_index; + svm_fifo_t *f; ASSERT (pool_elts (sm->segments) != 0); /* Across all fifo segments used by the server */ /* *INDENT-OFF* */ - segment_manager_foreach_segment_w_lock (fifo_segment, sm, ({ - fifo = fifo_segment_get_fifo_list (fifo_segment); - - /* - * Remove any residual sessions from the session lookup table - * Don't bother deleting the individual fifos, we're going to - * throw away the fifo segment in a minute. - */ - while (fifo) + segment_manager_foreach_segment_w_lock (fs, sm, ({ + for (slice_index = 0; slice_index < fs->n_slices; slice_index++) { - session = session_get_if_valid (fifo->master_session_index, - fifo->master_thread_index); - if (session) - vec_add1 (handles, session_handle (session)); - fifo = fifo->next; + f = fifo_segment_get_slice_fifo_list (fs, slice_index); + + /* + * Remove any residual sessions from the session lookup table + * Don't bother deleting the individual fifos, we're going to + * throw away the fifo segment in a minute. + */ + while (f) + { + session = session_get_if_valid (f->master_session_index, + f->master_thread_index); + if (session) + vec_add1 (handles, session_handle (session)); + f = f->next; + } } /* Instead of removing the segment, test when cleaning up disconnected @@ -508,16 +514,19 @@ segment_manager_del_sessions (segment_manager_t * sm) int segment_manager_try_alloc_fifos (fifo_segment_t * fifo_segment, + u32 thread_index, u32 rx_fifo_size, u32 tx_fifo_size, svm_fifo_t ** rx_fifo, svm_fifo_t ** tx_fifo) { rx_fifo_size = clib_max (rx_fifo_size, sm_main.default_fifo_size); - *rx_fifo = fifo_segment_alloc_fifo (fifo_segment, rx_fifo_size, - FIFO_SEGMENT_RX_FIFO); + *rx_fifo = fifo_segment_alloc_fifo_w_slice (fifo_segment, thread_index, + rx_fifo_size, + FIFO_SEGMENT_RX_FIFO); tx_fifo_size = clib_max (tx_fifo_size, sm_main.default_fifo_size); - *tx_fifo = fifo_segment_alloc_fifo (fifo_segment, tx_fifo_size, - FIFO_SEGMENT_TX_FIFO); + *tx_fifo = fifo_segment_alloc_fifo_w_slice (fifo_segment, thread_index, + tx_fifo_size, + FIFO_SEGMENT_TX_FIFO); if (*rx_fifo == 0) { @@ -544,6 +553,7 @@ segment_manager_try_alloc_fifos (fifo_segment_t * fifo_segment, int segment_manager_alloc_session_fifos (segment_manager_t * sm, + u32 thread_index, svm_fifo_t ** rx_fifo, svm_fifo_t ** tx_fifo) { @@ -563,6 +573,7 @@ segment_manager_alloc_session_fifos (segment_manager_t * sm, /* *INDENT-OFF* */ segment_manager_foreach_segment_w_lock (fs, sm, ({ alloc_fail = segment_manager_try_alloc_fifos (fs, + thread_index, props->rx_fifo_size, props->tx_fifo_size, rx_fifo, tx_fifo); @@ -616,7 +627,8 @@ alloc_check: return SESSION_ERROR_SEG_CREATE; } fs = segment_manager_get_segment_w_lock (sm, new_fs_index); - alloc_fail = segment_manager_try_alloc_fifos (fs, props->rx_fifo_size, + alloc_fail = segment_manager_try_alloc_fifos (fs, thread_index, + props->rx_fifo_size, props->tx_fifo_size, rx_fifo, tx_fifo); added_a_segment = 1; @@ -883,11 +895,13 @@ VLIB_CLI_COMMAND (segment_manager_show_command, static) = void segment_manager_format_sessions (segment_manager_t * sm, int verbose) { - fifo_segment_t *fifo_segment; vlib_main_t *vm = vlib_get_main (); app_worker_t *app_wrk; + fifo_segment_t *fs; const u8 *app_name; - u8 *s = 0; + int slice_index; + u8 *s = 0, *str; + svm_fifo_t *f; if (!sm) { @@ -905,35 +919,35 @@ segment_manager_format_sessions (segment_manager_t * sm, int verbose) clib_rwlock_reader_lock (&sm->segments_rwlock); /* *INDENT-OFF* */ - pool_foreach (fifo_segment, sm->segments, ({ - svm_fifo_t *fifo; - u8 *str; - - fifo = fifo_segment_get_fifo_list (fifo_segment); - while (fifo) + pool_foreach (fs, sm->segments, ({ + for (slice_index = 0; slice_index < fs->n_slices; slice_index++) { - u32 session_index, thread_index; - session_t *session; - - session_index = fifo->master_session_index; - thread_index = fifo->master_thread_index; - - session = session_get (session_index, thread_index); - str = format (0, "%U", format_session, session, verbose); - - if (verbose) - s = format (s, "%-40s%-20s%-15u%-10u", str, app_name, - app_wrk->api_client_index, app_wrk->connects_seg_manager); - else - s = format (s, "%-40s%-20s", str, app_name); - - vlib_cli_output (vm, "%v", s); - vec_reset_length (s); - vec_free (str); - - fifo = fifo->next; + f = fifo_segment_get_slice_fifo_list (fs, slice_index); + while (f) + { + u32 session_index, thread_index; + session_t *session; + + session_index = f->master_session_index; + thread_index = f->master_thread_index; + + session = session_get (session_index, thread_index); + str = format (0, "%U", format_session, session, verbose); + + if (verbose) + s = format (s, "%-40s%-20s%-15u%-10u", str, app_name, + app_wrk->api_client_index, app_wrk->connects_seg_manager); + else + s = format (s, "%-40s%-20s", str, app_name); + + vlib_cli_output (vm, "%v", s); + vec_reset_length (s); + vec_free (str); + + f = f->next; + } + vec_free (s); } - vec_free (s); })); /* *INDENT-ON* */ diff --git a/src/vnet/session/segment_manager.h b/src/vnet/session/segment_manager.h index 8358c50ba06..f2a90c8f935 100644 --- a/src/vnet/session/segment_manager.h +++ b/src/vnet/session/segment_manager.h @@ -31,6 +31,7 @@ typedef struct _segment_manager_props u8 add_segment:1; /**< can add new segments flag */ u8 use_mq_eventfd:1; /**< use eventfds for mqs flag */ u8 reserved:6; /**< reserved flags */ + u8 n_slices; /**< number of fs slices/threads */ ssvm_segment_type_t segment_type; /**< seg type: if set to SSVM_N_TYPES, private segments are used */ } segment_manager_props_t; @@ -106,9 +107,11 @@ void segment_manager_segment_reader_unlock (segment_manager_t * sm); void segment_manager_segment_writer_unlock (segment_manager_t * sm); int segment_manager_alloc_session_fifos (segment_manager_t * sm, + u32 thread_index, svm_fifo_t ** rx_fifo, svm_fifo_t ** tx_fifo); int segment_manager_try_alloc_fifos (fifo_segment_t * fs, + u32 thread_index, u32 rx_fifo_size, u32 tx_fifo_size, svm_fifo_t ** rx_fifo, svm_fifo_t ** tx_fifo); |