diff options
Diffstat (limited to 'src/svm')
| -rw-r--r-- | src/svm/CMakeLists.txt | 12 | ||||
| -rw-r--r-- | src/svm/fifo_segment.c (renamed from src/svm/svm_fifo_segment.c) | 679 | ||||
| -rw-r--r-- | src/svm/fifo_segment.h | 139 | ||||
| -rw-r--r-- | src/svm/svm_fifo.c | 4 | ||||
| -rw-r--r-- | src/svm/svm_fifo.h | 9 | ||||
| -rw-r--r-- | src/svm/svm_fifo_segment.h | 136 |
6 files changed, 477 insertions, 502 deletions
diff --git a/src/svm/CMakeLists.txt b/src/svm/CMakeLists.txt index ab6f7f71954..7664014153a 100644 --- a/src/svm/CMakeLists.txt +++ b/src/svm/CMakeLists.txt @@ -16,23 +16,23 @@ ############################################################################## add_vpp_library(svm SOURCES + fifo_segment.c + message_queue.c + queue.c svm.c ssvm.c svm_fifo.c - svm_fifo_segment.c - queue.c - message_queue.c MULTIARCH_SOURCES svm_fifo.c INSTALL_HEADERS + fifo_segment.h + message_queue.h + queue.h ssvm.h svm_common.h svm_fifo.h - svm_fifo_segment.h - queue.h - message_queue.h svm.h svmdb.h diff --git a/src/svm/svm_fifo_segment.c b/src/svm/fifo_segment.c index 8a8bd4aa7ff..6bf2634a290 100644 --- a/src/svm/svm_fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -13,190 +13,24 @@ * limitations under the License. */ -#include <svm/svm_fifo_segment.h> - -static void -fifo_init_for_segment (svm_fifo_segment_header_t * fsh, u8 * fifo_space, - u32 size, u32 freelist_index) -{ - svm_fifo_t *f; - - f = (svm_fifo_t *) fifo_space; - f->freelist_index = freelist_index; - f->default_chunk.start_byte = 0; - f->default_chunk.length = size; - f->default_chunk.next = f->start_chunk = f->end_chunk = &f->default_chunk; - f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = f->start_chunk; - f->next = fsh->free_fifos[freelist_index]; - fsh->free_fifos[freelist_index] = f; -} - -static void -allocate_new_fifo_chunk (svm_fifo_segment_header_t * fsh, - u32 data_size_in_bytes, int chunk_size) -{ - int freelist_index; - u32 size; - u8 *fifo_space; - u32 rounded_data_size; - svm_fifo_t *f; - int i; - - rounded_data_size = (1 << (max_log2 (data_size_in_bytes))); - freelist_index = max_log2 (rounded_data_size) - - max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE); - - /* Calculate space requirement $$$ round-up data_size_in_bytes */ - size = (sizeof (*f) + rounded_data_size) * chunk_size; - - /* Allocate fifo space. May fail. */ - fifo_space = clib_mem_alloc_aligned_at_offset - (size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ , - 0 /* os_out_of_memory */ ); - - /* Out of space.. */ - if (fifo_space == 0) - return; - - /* Carve fifo space */ - for (i = 0; i < chunk_size; i++) - { - fifo_init_for_segment (fsh, fifo_space, rounded_data_size, - freelist_index); - fifo_space += sizeof (*f) + rounded_data_size; - } -} - -/** - * Pre-allocates fifo pairs in fifo segment - * - * The number of fifos pre-allocated is the minimum of the requested number - * of pairs and the maximum number that fit within the segment. If the maximum - * is hit, the number of fifo pairs requested is updated by subtracting the - * number of fifos that have been successfully allocated. - */ -void -svm_fifo_segment_preallocate_fifo_pairs (svm_fifo_segment_private_t * s, - u32 rx_fifo_size, u32 tx_fifo_size, - u32 * n_fifo_pairs) -{ - u32 rx_rounded_data_size, tx_rounded_data_size, pair_size; - u32 rx_fifos_size, tx_fifos_size, pairs_to_allocate; - int rx_freelist_index, tx_freelist_index; - ssvm_shared_header_t *sh = s->ssvm.sh; - svm_fifo_segment_header_t *fsh = s->h; - u8 *rx_fifo_space, *tx_fifo_space; - uword space_available; - void *oldheap; - svm_fifo_t *f; - int i; - - /* Parameter check */ - if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0) - return; - - if (rx_fifo_size < FIFO_SEGMENT_MIN_FIFO_SIZE || - rx_fifo_size > FIFO_SEGMENT_MAX_FIFO_SIZE) - { - clib_warning ("rx fifo_size out of range %d", rx_fifo_size); - return; - } - - if (tx_fifo_size < FIFO_SEGMENT_MIN_FIFO_SIZE || - tx_fifo_size > FIFO_SEGMENT_MAX_FIFO_SIZE) - { - clib_warning ("tx fifo_size out of range %d", rx_fifo_size); - return; - } - - rx_rounded_data_size = (1 << (max_log2 (rx_fifo_size))); - rx_freelist_index = max_log2 (rx_fifo_size) - - max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE); - tx_rounded_data_size = (1 << (max_log2 (tx_fifo_size))); - tx_freelist_index = max_log2 (tx_fifo_size) - - max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE); - - /* Calculate space requirements */ - pair_size = 2 * sizeof (*f) + rx_rounded_data_size + tx_rounded_data_size; -#if USE_DLMALLOC == 0 - space_available = s->ssvm.ssvm_size - mheap_bytes (sh->heap); -#else - space_available = s->ssvm.ssvm_size - mspace_usable_size (sh->heap); -#endif - - pairs_to_allocate = clib_min (space_available / pair_size, *n_fifo_pairs); - rx_fifos_size = (sizeof (*f) + rx_rounded_data_size) * pairs_to_allocate; - tx_fifos_size = (sizeof (*f) + tx_rounded_data_size) * pairs_to_allocate; - - vec_validate_init_empty (fsh->free_fifos, - clib_max (rx_freelist_index, tx_freelist_index), - 0); - - oldheap = ssvm_push_heap (sh); - /* Allocate rx fifo space. May fail. */ - rx_fifo_space = clib_mem_alloc_aligned_at_offset - (rx_fifos_size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ , - 0 /* os_out_of_memory */ ); - - /* Same for TX */ - tx_fifo_space = clib_mem_alloc_aligned_at_offset - (tx_fifos_size, CLIB_CACHE_LINE_BYTES, 0 /* align_offset */ , - 0 /* os_out_of_memory */ ); - - /* Make sure it worked. Clean up if it didn't... */ - if (rx_fifo_space == 0 || tx_fifo_space == 0) - { - if (rx_fifo_space) - clib_mem_free (rx_fifo_space); - else - clib_warning ("rx fifo preallocation failure: size %d npairs %d", - rx_fifo_size, *n_fifo_pairs); - - if (tx_fifo_space) - clib_mem_free (tx_fifo_space); - else - clib_warning ("tx fifo preallocation failure: size %d nfifos %d", - tx_fifo_size, *n_fifo_pairs); - ssvm_pop_heap (oldheap); - return; - } - - /* Carve rx fifo space */ - for (i = 0; i < pairs_to_allocate; i++) - { - fifo_init_for_segment (fsh, rx_fifo_space, rx_rounded_data_size, - rx_freelist_index); - rx_fifo_space += sizeof (*f) + rx_rounded_data_size; - } - /* Carve tx fifo space */ - for (i = 0; i < pairs_to_allocate; i++) - { - fifo_init_for_segment (fsh, tx_fifo_space, tx_rounded_data_size, - tx_freelist_index); - tx_fifo_space += sizeof (*f) + tx_rounded_data_size; - } - - /* Account for the pairs allocated */ - *n_fifo_pairs -= pairs_to_allocate; - ssvm_pop_heap (oldheap); -} +#include <svm/fifo_segment.h> /** - * Initialize svm fifo segment shared header + * Initialize fifo segment shared header */ int -svm_fifo_segment_init (svm_fifo_segment_private_t * s) +fifo_segment_init (fifo_segment_t * fs) { - svm_fifo_segment_header_t *fsh; + fifo_segment_header_t *fsh; ssvm_shared_header_t *sh; void *oldheap; - sh = s->ssvm.sh; + sh = fs->ssvm.sh; oldheap = ssvm_push_heap (sh); fsh = clib_mem_alloc (sizeof (*fsh)); clib_memset (fsh, 0, sizeof (*fsh)); - s->h = sh->opaque[0] = fsh; + fs->h = sh->opaque[0] = fsh; ssvm_pop_heap (oldheap); @@ -205,51 +39,17 @@ svm_fifo_segment_init (svm_fifo_segment_private_t * s) } /** - * Create an svm fifo segment and initialize as master + * Create a fifo segment in process-private memory */ -int -svm_fifo_segment_create (svm_fifo_segment_main_t * sm, - svm_fifo_segment_create_args_t * a) +static int +fifo_segment_create_process_private (fifo_segment_main_t * sm, + fifo_segment_create_args_t * a) { - svm_fifo_segment_private_t *s; - int rv; - - /* Allocate a fresh segment */ - pool_get (sm->segments, s); - clib_memset (s, 0, sizeof (*s)); - - s->ssvm.ssvm_size = a->segment_size; - s->ssvm.i_am_master = 1; - s->ssvm.my_pid = getpid (); - s->ssvm.name = format (0, "%s%c", a->segment_name, 0); - s->ssvm.requested_va = sm->next_baseva; - - if ((rv = ssvm_master_init (&s->ssvm, a->segment_type))) - { - pool_put (sm->segments, s); - return (rv); - } - - /* Note: requested_va updated due to seg base addr randomization */ - sm->next_baseva = s->ssvm.sh->ssvm_va + a->segment_size; - - svm_fifo_segment_init (s); - vec_add1 (a->new_segment_indices, s - sm->segments); - return (0); -} - -/** - * Create an svm fifo segment in process-private memory - */ -int -svm_fifo_segment_create_process_private (svm_fifo_segment_main_t * sm, - svm_fifo_segment_create_args_t * a) -{ - svm_fifo_segment_private_t *s; + u32 pagesize = clib_mem_get_page_size (); ssvm_shared_header_t *sh; + fifo_segment_t *s; u32 rnd_size = 0; u8 *heap; - u32 pagesize = clib_mem_get_page_size (); pool_get (sm->segments, s); clib_memset (s, 0, sizeof (*s)); @@ -286,27 +86,60 @@ svm_fifo_segment_create_process_private (svm_fifo_segment_main_t * sm, clib_memset (sh, 0, sizeof (*sh)); sh->heap = heap; - svm_fifo_segment_init (s); + fifo_segment_init (s); vec_add1 (a->new_segment_indices, s - sm->segments); return (0); } /** - * Attach as slave to an svm fifo segment + * Create a fifo segment and initialize as master */ int -svm_fifo_segment_attach (svm_fifo_segment_main_t * sm, - svm_fifo_segment_create_args_t * a) +fifo_segment_create (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) { - svm_fifo_segment_private_t *s; + fifo_segment_t *s; int rv; + if (a->segment_type == SSVM_SEGMENT_PRIVATE) + return fifo_segment_create_process_private (sm, a); + /* Allocate a fresh segment */ pool_get (sm->segments, s); clib_memset (s, 0, sizeof (*s)); s->ssvm.ssvm_size = a->segment_size; + s->ssvm.i_am_master = 1; + s->ssvm.my_pid = getpid (); + s->ssvm.name = format (0, "%s%c", a->segment_name, 0); + s->ssvm.requested_va = sm->next_baseva; + + if ((rv = ssvm_master_init (&s->ssvm, a->segment_type))) + { + pool_put (sm->segments, s); + return (rv); + } + + /* Note: requested_va updated due to seg base addr randomization */ + sm->next_baseva = s->ssvm.sh->ssvm_va + a->segment_size; + + fifo_segment_init (s); + vec_add1 (a->new_segment_indices, s - sm->segments); + return (0); +} + +/** + * Attach as slave to a fifo segment + */ +int +fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) +{ + fifo_segment_t *s; + int rv; + + pool_get_zero (sm->segments, s); + + s->ssvm.ssvm_size = a->segment_size; s->ssvm.my_pid = getpid (); s->ssvm.name = format (0, "%s%c", a->segment_name, 0); s->ssvm.requested_va = sm->next_baseva; @@ -329,92 +162,187 @@ svm_fifo_segment_attach (svm_fifo_segment_main_t * sm, } void -svm_fifo_segment_delete (svm_fifo_segment_main_t * sm, - svm_fifo_segment_private_t * s) +fifo_segment_delete (fifo_segment_main_t * sm, fifo_segment_t * s) { ssvm_delete (&s->ssvm); clib_memset (s, 0xfe, sizeof (*s)); pool_put (sm->segments, s); } +u32 +fifo_segment_index (fifo_segment_main_t * sm, fifo_segment_t * s) +{ + return s - sm->segments; +} + +void * +svm_fifo_segment_heap (fifo_segment_t * seg) +{ + return seg->ssvm.sh->heap; +} + +fifo_segment_t * +fifo_segment_get_segment (fifo_segment_main_t * sm, u32 segment_index) +{ + return pool_elt_at_index (sm->segments, segment_index); +} + +void +fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size) +{ + if (ssvm_type (&seg->ssvm) == SSVM_SEGMENT_PRIVATE) + { +#if USE_DLMALLOC == 0 + mheap_t *heap_header; + + *address = pointer_to_uword (seg->ssvm.sh->heap); + heap_header = mheap_header (seg->ssvm.sh->heap); + *size = heap_header->max_size; +#else + mspace_get_address_and_size (seg->ssvm.sh->heap, address, size); +#endif + } + else + { + *address = (char *) seg->ssvm.sh->ssvm_va; + *size = seg->ssvm.ssvm_size; + } +} + +void +fifo_segment_main_init (fifo_segment_main_t * sm, u64 baseva, + u32 timeout_in_seconds) +{ + sm->next_baseva = baseva; + sm->timeout_in_seconds = timeout_in_seconds; +} + +static void +fifo_init_for_segment (fifo_segment_header_t * fsh, svm_fifo_t * f, + u32 size, u32 fl_index) +{ + f->freelist_index = fl_index; + f->default_chunk.start_byte = 0; + f->default_chunk.length = size; + f->default_chunk.next = f->start_chunk = f->end_chunk = &f->default_chunk; + f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = f->start_chunk; + f->next = fsh->free_fifos[fl_index]; + fsh->free_fifos[fl_index] = f; +} + +static inline int +fs_free_list_for_size (u32 size) +{ + return max_log2 (size) - max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE); +} + +static inline int +fs_fifo_size_is_valid (u32 size) +{ + /* + * 4K minimum. It's not likely that anything good will happen + * with a smaller FIFO. + */ + return size >= FIFO_SEGMENT_MIN_FIFO_SIZE + && size <= FIFO_SEGMENT_MAX_FIFO_SIZE; +} + +static void +allocate_new_fifo_batch (fifo_segment_header_t * fsh, + u32 data_size_in_bytes, int chunk_size) +{ + u32 size, rounded_data_size; + int i, fl_index; + u8 *fifos_mem; + svm_fifo_t *f; + + rounded_data_size = (1 << (max_log2 (data_size_in_bytes))); + fl_index = fs_free_list_for_size (data_size_in_bytes); + + /* Calculate space requirement $$$ round-up data_size_in_bytes */ + size = (sizeof (*f) + rounded_data_size) * chunk_size; + + /* Allocate fifo space. May fail. */ + fifos_mem = clib_mem_alloc_aligned_at_offset (size, CLIB_CACHE_LINE_BYTES, + 0 /* align_offset */ , + 0 /* os_out_of_memory */ ); + + /* Out of space.. */ + if (fifos_mem == 0) + return; + + /* Carve fifo space */ + for (i = 0; i < chunk_size; i++) + { + f = (svm_fifo_t *) fifos_mem; + fifo_init_for_segment (fsh, f, rounded_data_size, fl_index); + fifos_mem += sizeof (*f) + rounded_data_size; + } +} + /** - * Allocate fifo in svm segment + * Allocate fifo in fifo segment */ svm_fifo_t * -svm_fifo_segment_alloc_fifo (svm_fifo_segment_private_t * fs, - u32 data_size_in_bytes, - svm_fifo_segment_freelist_t list_index) +fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes, + fifo_segment_ftype_t ftype) { + fifo_segment_header_t *fsh; ssvm_shared_header_t *sh; - svm_fifo_segment_header_t *fsh; svm_fifo_t *f = 0; void *oldheap; - int freelist_index; + int fl_index; - /* - * 4K minimum. It's not likely that anything good will happen - * with a smaller FIFO. - */ - if (data_size_in_bytes < FIFO_SEGMENT_MIN_FIFO_SIZE || - data_size_in_bytes > FIFO_SEGMENT_MAX_FIFO_SIZE) + if (!fs_fifo_size_is_valid (data_bytes)) { - clib_warning ("fifo size out of range %d", data_size_in_bytes); + clib_warning ("fifo size out of range %d", data_bytes); return 0; } - freelist_index = max_log2 (data_size_in_bytes) - - max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE); + fl_index = fs_free_list_for_size (data_bytes); sh = fs->ssvm.sh; ssvm_lock_non_recursive (sh, 1); - fsh = (svm_fifo_segment_header_t *) sh->opaque[0]; - switch (list_index) - { - case FIFO_SEGMENT_RX_FREELIST: - case FIFO_SEGMENT_TX_FREELIST: - vec_validate_init_empty (fsh->free_fifos, freelist_index, 0); - f = fsh->free_fifos[freelist_index]; - if (PREDICT_FALSE (!f)) - { - oldheap = ssvm_push_heap (sh); - allocate_new_fifo_chunk (fsh, data_size_in_bytes, - FIFO_SEGMENT_ALLOC_CHUNK_SIZE); - ssvm_pop_heap (oldheap); - f = fsh->free_fifos[freelist_index]; - } - if (PREDICT_TRUE (f != 0)) - { - fsh->free_fifos[freelist_index] = f->next; - /* (re)initialize the fifo, as in svm_fifo_create */ - clib_memset (f, 0, sizeof (*f)); - svm_fifo_init (f, data_size_in_bytes); - goto found; - } - break; - case FIFO_SEGMENT_FREELIST_NONE: - break; + fsh = (fifo_segment_header_t *) sh->opaque[0]; + vec_validate_init_empty (fsh->free_fifos, fl_index, 0); + f = fsh->free_fifos[fl_index]; - default: - clib_warning ("ignore bogus freelist %d", list_index); - break; + /* Try to allocate batch of fifos */ + if (PREDICT_FALSE (!f)) + { + oldheap = ssvm_push_heap (sh); + allocate_new_fifo_batch (fsh, data_bytes, + FIFO_SEGMENT_ALLOC_BATCH_SIZE); + ssvm_pop_heap (oldheap); + f = fsh->free_fifos[fl_index]; + } + if (PREDICT_TRUE (f != 0)) + { + fsh->free_fifos[fl_index] = f->next; + /* (re)initialize the fifo, as in svm_fifo_create */ + memset (f, 0, sizeof (*f)); + svm_fifo_init (f, data_bytes); + goto found; } - /* Catch all that allocates just one fifo. Note: this can fail, - * in which case: create another segment */ + /* Failed to allocate batch, try just one fifo. This can also fail, + * in which case, create another segment */ oldheap = ssvm_push_heap (sh); - f = svm_fifo_create (data_size_in_bytes); + f = svm_fifo_create (data_bytes); ssvm_pop_heap (oldheap); + if (PREDICT_FALSE (f == 0)) goto done; - f->freelist_index = freelist_index; + + f->freelist_index = fl_index; found: - /* If rx_freelist add to active fifos list. When cleaning up segment, + /* If rx fifo type add to active fifos list. When cleaning up segment, * we need a list of active sessions that should be disconnected. Since * both rx and tx fifos keep pointers to the session, it's enough to track * only one. */ - if (list_index == FIFO_SEGMENT_RX_FREELIST) + if (ftype == FIFO_SEGMENT_RX_FIFO) { if (fsh->fifos) { @@ -422,6 +350,7 @@ found: f->next = fsh->fifos; } fsh->fifos = f; + f->flags |= SVM_FIFO_F_LL_TRACKED; } fsh->n_active_fifos++; @@ -430,13 +359,15 @@ done: return (f); } +/** + * Free fifo allocated in fifo segment + */ void -svm_fifo_segment_free_fifo (svm_fifo_segment_private_t * s, svm_fifo_t * f, - svm_fifo_segment_freelist_t list_index) +fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f) { + fifo_segment_header_t *fsh; ssvm_shared_header_t *sh; - svm_fifo_segment_header_t *fsh; - int freelist_index; + int fl_index; ASSERT (f->refcnt > 0); @@ -444,39 +375,30 @@ svm_fifo_segment_free_fifo (svm_fifo_segment_private_t * s, svm_fifo_t * f, return; sh = s->ssvm.sh; - fsh = (svm_fifo_segment_header_t *) sh->opaque[0]; - - freelist_index = f->freelist_index; + fsh = (fifo_segment_header_t *) sh->opaque[0]; + fl_index = f->freelist_index; - ASSERT (freelist_index < vec_len (fsh->free_fifos)); + ASSERT (fl_index < vec_len (fsh->free_fifos)); ssvm_lock_non_recursive (sh, 2); - switch (list_index) + /* Remove from active list. Only rx fifos are tracked */ + if (f->flags & SVM_FIFO_F_LL_TRACKED) { - case FIFO_SEGMENT_RX_FREELIST: - /* Remove from active list */ if (f->prev) f->prev->next = f->next; else fsh->fifos = f->next; if (f->next) f->next->prev = f->prev; - /* Fall through: we add only rx fifos to active pool */ - case FIFO_SEGMENT_TX_FREELIST: - /* Add to free list */ - f->next = fsh->free_fifos[freelist_index]; - f->prev = 0; - fsh->free_fifos[freelist_index] = f; - break; - case FIFO_SEGMENT_FREELIST_NONE: - break; - - default: - clib_warning ("ignore bogus freelist %d", list_index); - break; + f->flags &= ~SVM_FIFO_F_LL_TRACKED; } + /* Add to free list */ + f->next = fsh->free_fifos[fl_index]; + f->prev = 0; + fsh->free_fifos[fl_index] = f; + if (CLIB_DEBUG) { f->master_session_index = ~0; @@ -487,51 +409,119 @@ svm_fifo_segment_free_fifo (svm_fifo_segment_private_t * s, svm_fifo_t * f, ssvm_unlock_non_recursive (sh); } +/** + * Pre-allocates fifo pairs in fifo segment + */ void -svm_fifo_segment_main_init (svm_fifo_segment_main_t * sm, u64 baseva, - u32 timeout_in_seconds) +fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, + u32 rx_fifo_size, u32 tx_fifo_size, + u32 * n_fifo_pairs) { - sm->next_baseva = baseva; - sm->timeout_in_seconds = timeout_in_seconds; -} + u32 rx_rounded_data_size, tx_rounded_data_size, pair_size; + u32 rx_fifos_size, tx_fifos_size, pairs_to_allocate; + ssvm_shared_header_t *sh = fs->ssvm.sh; + fifo_segment_header_t *fsh = fs->h; + int i, rx_fl_index, tx_fl_index; + u8 *rx_fifo_mem, *tx_fifo_mem; + uword space_available; + void *oldheap; + svm_fifo_t *f; -u32 -svm_fifo_segment_index (svm_fifo_segment_main_t * sm, - svm_fifo_segment_private_t * s) -{ - return s - sm->segments; -} + /* Parameter check */ + if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0) + return; -/** - * Retrieve svm segments pool. Used only for debug purposes. - */ -svm_fifo_segment_private_t * -svm_fifo_segment_segments_pool (svm_fifo_segment_main_t * sm) -{ - return sm->segments; + if (!fs_fifo_size_is_valid (rx_fifo_size)) + { + clib_warning ("rx fifo_size out of range %d", rx_fifo_size); + return; + } + + if (!fs_fifo_size_is_valid (tx_fifo_size)) + { + clib_warning ("tx fifo_size out of range %d", tx_fifo_size); + return; + } + + rx_rounded_data_size = (1 << (max_log2 (rx_fifo_size))); + rx_fl_index = fs_free_list_for_size (rx_fifo_size); + tx_rounded_data_size = (1 << (max_log2 (tx_fifo_size))); + tx_fl_index = fs_free_list_for_size (tx_fifo_size); + + /* Calculate space requirements */ + pair_size = 2 * sizeof (*f) + rx_rounded_data_size + tx_rounded_data_size; +#if USE_DLMALLOC == 0 + space_available = fs->ssvm.ssvm_size - mheap_bytes (sh->heap); +#else + space_available = fs->ssvm.ssvm_size - mspace_usable_size (sh->heap); +#endif + + pairs_to_allocate = clib_min (space_available / pair_size, *n_fifo_pairs); + rx_fifos_size = (sizeof (*f) + rx_rounded_data_size) * pairs_to_allocate; + tx_fifos_size = (sizeof (*f) + tx_rounded_data_size) * pairs_to_allocate; + + vec_validate_init_empty (fsh->free_fifos, + clib_max (rx_fl_index, tx_fl_index), 0); + + oldheap = ssvm_push_heap (sh); + + /* Allocate rx and tx fifo memory. May fail. */ + rx_fifo_mem = clib_mem_alloc_aligned_at_offset (rx_fifos_size, + CLIB_CACHE_LINE_BYTES, + 0 /* align_offset */ , + 0 /* os_out_of_memory */ ); + tx_fifo_mem = clib_mem_alloc_aligned_at_offset (tx_fifos_size, + CLIB_CACHE_LINE_BYTES, + 0 /* align_offset */ , + 0 /* os_out_of_memory */ ); + + /* Make sure it worked. Clean up if it didn't... */ + if (rx_fifo_mem == 0 || tx_fifo_mem == 0) + { + rx_fifo_mem ? clib_mem_free (rx_fifo_mem) : clib_mem_free (tx_fifo_mem); + clib_warning ("fifo preallocation failure: rx size %d tx size %u " + "npairs %d", rx_fifo_size, tx_fifo_size, *n_fifo_pairs); + ssvm_pop_heap (oldheap); + return; + } + + /* Carve rx and tx fifo memory */ + for (i = 0; i < pairs_to_allocate; i++) + { + f = (svm_fifo_t *) rx_fifo_mem; + fifo_init_for_segment (fsh, f, rx_rounded_data_size, rx_fl_index); + rx_fifo_mem += sizeof (*f) + rx_rounded_data_size; + + f = (svm_fifo_t *) tx_fifo_mem; + fifo_init_for_segment (fsh, f, tx_rounded_data_size, tx_fl_index); + tx_fifo_mem += sizeof (*f) + tx_rounded_data_size; + } + + /* Account for the pairs allocated */ + *n_fifo_pairs -= pairs_to_allocate; + ssvm_pop_heap (oldheap); } /** * Get number of active fifos */ u32 -svm_fifo_segment_num_fifos (svm_fifo_segment_private_t * fifo_segment) +fifo_segment_num_fifos (fifo_segment_t * fs) { - return fifo_segment->h->n_active_fifos; + return fs->h->n_active_fifos; } u32 -svm_fifo_segment_num_free_fifos (svm_fifo_segment_private_t * fifo_segment, - u32 fifo_size_in_bytes) +fifo_segment_num_free_fifos (fifo_segment_t * fs, u32 fifo_size_in_bytes) { + u32 count = 0, rounded_data_size, fl_index; + fifo_segment_header_t *fsh; ssvm_shared_header_t *sh; - svm_fifo_segment_header_t *fsh; svm_fifo_t *f; int i; - u32 count = 0, rounded_data_size, freelist_index; - sh = fifo_segment->ssvm.sh; - fsh = (svm_fifo_segment_header_t *) sh->opaque[0]; + sh = fs->ssvm.sh; + fsh = (fifo_segment_header_t *) sh->opaque[0]; /* Count all free fifos? */ if (fifo_size_in_bytes == ~0) @@ -552,13 +542,12 @@ svm_fifo_segment_num_free_fifos (svm_fifo_segment_private_t * fifo_segment, } rounded_data_size = (1 << (max_log2 (fifo_size_in_bytes))); - freelist_index = max_log2 (rounded_data_size) - - max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE); + fl_index = fs_free_list_for_size (rounded_data_size); - if (freelist_index >= vec_len (fsh->free_fifos)) + if (fl_index >= vec_len (fsh->free_fifos)) return 0; - f = fsh->free_fifos[freelist_index]; + f = fsh->free_fifos[fl_index]; if (f == 0) return 0; @@ -570,40 +559,23 @@ svm_fifo_segment_num_free_fifos (svm_fifo_segment_private_t * fifo_segment, return count; } -void -svm_fifo_segment_info (svm_fifo_segment_private_t * seg, char **address, - size_t * size) +u8 +fifo_segment_has_fifos (fifo_segment_t * fs) { - if (ssvm_type (&seg->ssvm) == SSVM_SEGMENT_PRIVATE) - { -#if USE_DLMALLOC == 0 - mheap_t *heap_header; - - *address = pointer_to_uword (seg->ssvm.sh->heap); - heap_header = mheap_header (seg->ssvm.sh->heap); - *size = heap_header->max_size; -#else - mspace_get_address_and_size (seg->ssvm.sh->heap, address, size); -#endif - } - else - { - *address = (char *) seg->ssvm.sh->ssvm_va; - *size = seg->ssvm.ssvm_size; - } + return fs->h->fifos != 0; } -void * -svm_fifo_segment_heap (svm_fifo_segment_private_t * seg) +svm_fifo_t * +fifo_segment_get_fifo_list (fifo_segment_t * fs) { - return seg->ssvm.sh->heap; + return fs->h->fifos; } u8 * -format_svm_fifo_segment_type (u8 * s, va_list * args) +format_fifo_segment_type (u8 * s, va_list * args) { - svm_fifo_segment_private_t *sp; - sp = va_arg (*args, svm_fifo_segment_private_t *); + fifo_segment_t *sp; + sp = va_arg (*args, fifo_segment_t *); ssvm_segment_type_t st = ssvm_type (&sp->ssvm); if (st == SSVM_SEGMENT_PRIVATE) @@ -621,12 +593,11 @@ format_svm_fifo_segment_type (u8 * s, va_list * args) * Segment format function */ u8 * -format_svm_fifo_segment (u8 * s, va_list * args) +format_fifo_segment (u8 * s, va_list * args) { - svm_fifo_segment_private_t *sp - = va_arg (*args, svm_fifo_segment_private_t *); + fifo_segment_t *sp = va_arg (*args, fifo_segment_t *); int verbose __attribute__ ((unused)) = va_arg (*args, int); - svm_fifo_segment_header_t *fsh = sp->h; + fifo_segment_header_t *fsh = sp->h; u32 count, indent; svm_fifo_t *f; int i; @@ -636,7 +607,7 @@ format_svm_fifo_segment (u8 * s, va_list * args) s = format (s, "%U segment heap: %U\n", format_white_space, indent, format_mheap, svm_fifo_segment_heap (sp), verbose); s = format (s, "%U segment has %u active fifos\n", - format_white_space, indent, svm_fifo_segment_num_fifos (sp)); + format_white_space, indent, fifo_segment_num_fifos (sp)); #endif for (i = 0; i < vec_len (fsh->free_fifos); i++) diff --git a/src/svm/fifo_segment.h b/src/svm/fifo_segment.h new file mode 100644 index 00000000000..cc0ff3adf3a --- /dev/null +++ b/src/svm/fifo_segment.h @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2016-2019 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_fifo_segment_h__ +#define __included_fifo_segment_h__ + +#include <svm/ssvm.h> +#include <svm/svm_fifo.h> + +typedef enum +{ + FIFO_SEGMENT_FTYPE_NONE = -1, + FIFO_SEGMENT_RX_FIFO = 0, + FIFO_SEGMENT_TX_FIFO, + FIFO_SEGMENT_N_FTYPES +} fifo_segment_ftype_t; + +#define FIFO_SEGMENT_MIN_FIFO_SIZE 4096 /* 4kB min fifo size */ +#define FIFO_SEGMENT_MAX_FIFO_SIZE (2 << 30) /* 2GB max fifo size */ +#define FIFO_SEGMENT_ALLOC_BATCH_SIZE 32 /* Allocation quantum */ + +typedef enum fifo_segment_flags_ +{ + FIFO_SEGMENT_F_IS_PREALLOCATED = 1 << 0, + FIFO_SEGMENT_F_WILL_DELETE = 1 << 1, +} fifo_segment_flags_t; + +typedef struct +{ + svm_fifo_t *fifos; /**< Linked list of active RX fifos */ + svm_fifo_t **free_fifos; /**< Freelists, by fifo size */ + u32 n_active_fifos; /**< Number of active fifos */ + u8 flags; /**< Segment flags */ +} fifo_segment_header_t; + +typedef struct +{ + ssvm_private_t ssvm; /**< ssvm segment data */ + fifo_segment_header_t *h; /**< fifo segment data */ +} fifo_segment_t; + +typedef struct +{ + fifo_segment_t *segments; /**< pool of fifo segments */ + u64 next_baseva; /**< Where to put the next one */ + u32 timeout_in_seconds; /**< Time to wait during attach */ +} fifo_segment_main_t; + +typedef struct +{ + ssvm_segment_type_t segment_type; /**< type of segment requested */ + u32 segment_size; /**< size of the segment */ + int memfd_fd; /**< fd for memfd segments */ + char *segment_name; /**< segment name */ + u32 *new_segment_indices; /**< return vec of new seg indices */ +} fifo_segment_create_args_t; + +#define fifo_segment_flags(_fs) _fs->h->flags + +int fifo_segment_init (fifo_segment_t * fs); +int fifo_segment_create (fifo_segment_main_t * sm, + fifo_segment_create_args_t * a); +int fifo_segment_attach (fifo_segment_main_t * sm, + fifo_segment_create_args_t * a); +void fifo_segment_delete (fifo_segment_main_t * sm, fifo_segment_t * fs); +fifo_segment_t *fifo_segment_get_segment (fifo_segment_main_t * sm, + u32 fs_index); +u32 fifo_segment_index (fifo_segment_main_t * sm, fifo_segment_t * fs); +void fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size); + +/** + * Allocate fifo in fifo segment + * + * @param fs fifo segment + * @param data_bytes size of default fifo chunk in bytes + * @param ftype fifo type @ref fifo_segment_ftype_t + * @return new fifo or 0 if alloc failed + */ +svm_fifo_t *fifo_segment_alloc_fifo (fifo_segment_t * fs, + u32 data_bytes, + fifo_segment_ftype_t ftype); + +/** + * Free fifo allocated in fifo segment + * + * @param fs fifo segment + * @param f fifo to be freed + */ +void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f); + +/** + * Pre-allocates fifo pairs in fifo segment + * + * The number of fifos pre-allocated is the minimum of the requested number + * of pairs and the maximum number that fit within the segment. If the maximum + * is hit, the number of fifo pairs requested is updated by subtracting the + * number of fifos that have been successfully allocated. + * + * @param fs fifo segment + * @param rx_fifo_size data size of rx fifos + * @param tx_fifo_size data size of tx fifos + * @param n_fifo_pairs number of pairs requested. Prior to returning, this + * is decremented by the the number of pairs allocated. + */ +void fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, + u32 rx_fifo_size, + u32 tx_fifo_size, + u32 * n_fifo_pairs); +u8 fifo_segment_has_fifos (fifo_segment_t * fs); +svm_fifo_t *fifo_segment_get_fifo_list (fifo_segment_t * fs); +u32 fifo_segment_num_fifos (fifo_segment_t * fs); +u32 fifo_segment_num_free_fifos (fifo_segment_t * fs, u32 fifo_size_in_bytes); + +void fifo_segment_main_init (fifo_segment_main_t * sm, u64 baseva, + u32 timeout_in_seconds); + +format_function_t format_fifo_segment; +format_function_t format_fifo_segment_type; + +#endif /* __included_fifo_segment_h__ */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index a335519d1e4..eafe497eec1 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -882,7 +882,7 @@ svm_fifo_dequeue_drop_all (svm_fifo_t * f) } int -svm_fifo_segments (svm_fifo_t * f, svm_fifo_segment_t * fs) +svm_fifo_segments (svm_fifo_t * f, svm_fifo_seg_t * fs) { u32 cursize, head, tail, head_idx; @@ -914,7 +914,7 @@ svm_fifo_segments (svm_fifo_t * f, svm_fifo_segment_t * fs) } void -svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs) +svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_seg_t * fs) { u32 head, head_idx; diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index 619d6093429..4f2f61939b7 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -67,6 +67,7 @@ typedef enum svm_fifo_flag_ { SVM_FIFO_F_SIZE_UPDATE = 1 << 0, SVM_FIFO_F_MULTI_CHUNK = 1 << 1, + SVM_FIFO_F_LL_TRACKED = 1 << 2, } svm_fifo_flag_t; typedef struct _svm_fifo @@ -124,11 +125,11 @@ typedef enum SVM_FIFO_FULL = -2, } svm_fifo_err_t; -typedef struct svm_fifo_segment_ +typedef struct svm_fifo_seg_ { u8 *data; u32 len; -} svm_fifo_segment_t; +} svm_fifo_seg_t; #if SVM_FIFO_TRACE #define svm_fifo_trace_add(_f, _s, _l, _t) \ @@ -352,8 +353,8 @@ int svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here); int svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 max_bytes, u8 * copy_here); int svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes); void svm_fifo_dequeue_drop_all (svm_fifo_t * f); -int svm_fifo_segments (svm_fifo_t * f, svm_fifo_segment_t * fs); -void svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs); +int svm_fifo_segments (svm_fifo_t * f, svm_fifo_seg_t * fs); +void svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_seg_t * fs); void svm_fifo_init_pointers (svm_fifo_t * f, u32 head, u32 tail); void svm_fifo_clone (svm_fifo_t * df, svm_fifo_t * sf); void svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len); diff --git a/src/svm/svm_fifo_segment.h b/src/svm/svm_fifo_segment.h deleted file mode 100644 index d4d48093ae6..00000000000 --- a/src/svm/svm_fifo_segment.h +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2016-2019 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_ssvm_fifo_segment_h__ -#define __included_ssvm_fifo_segment_h__ - -#include <svm/ssvm.h> -#include <svm/svm_fifo.h> -#include <vppinfra/lock.h> - -typedef enum -{ - FIFO_SEGMENT_FREELIST_NONE = -1, - FIFO_SEGMENT_RX_FREELIST = 0, - FIFO_SEGMENT_TX_FREELIST, - FIFO_SEGMENT_N_FREELISTS -} svm_fifo_segment_freelist_t; - -#define FIFO_SEGMENT_MIN_FIFO_SIZE 4096 -#define FIFO_SEGMENT_MAX_FIFO_SIZE (2 << 30) /* 2GB max fifo size */ -#define FIFO_SEGMENT_ALLOC_CHUNK_SIZE 32 /* Allocation quantum */ - -#define FIFO_SEGMENT_F_IS_PREALLOCATED (1 << 0) -#define FIFO_SEGMENT_F_WILL_DELETE (1 << 1) - -typedef struct -{ - svm_fifo_t *fifos; /**< Linked list of active RX fifos */ - svm_fifo_t **free_fifos; /**< Freelists, by fifo size */ - u32 n_active_fifos; /**< Number of active fifos */ - u8 flags; /**< Segment flags */ -} svm_fifo_segment_header_t; - -typedef struct -{ - ssvm_private_t ssvm; - svm_fifo_segment_header_t *h; -} svm_fifo_segment_private_t; - -typedef struct -{ - volatile u32 lock; - - /** pool of segments */ - svm_fifo_segment_private_t *segments; - /* Where to put the next one */ - u64 next_baseva; - u32 timeout_in_seconds; -} svm_fifo_segment_main_t; - -typedef struct -{ - ssvm_segment_type_t segment_type; - char *segment_name; - u32 segment_size; - u32 *new_segment_indices; - int memfd_fd; -} svm_fifo_segment_create_args_t; - -#define svm_fifo_segment_flags(_seg) _seg->h->flags - -static inline svm_fifo_segment_private_t * -svm_fifo_segment_get_segment (svm_fifo_segment_main_t * sm, u32 segment_index) -{ - return pool_elt_at_index (sm->segments, segment_index); -} - -static inline u8 -svm_fifo_segment_has_fifos (svm_fifo_segment_private_t * fifo_segment) -{ - return fifo_segment->h->fifos != 0; -} - -static inline svm_fifo_t * -svm_fifo_segment_get_fifo_list (svm_fifo_segment_private_t * fifo_segment) -{ - return fifo_segment->h->fifos; -} - -int svm_fifo_segment_init (svm_fifo_segment_private_t * s); -int svm_fifo_segment_create (svm_fifo_segment_main_t * sm, - svm_fifo_segment_create_args_t * a); -int svm_fifo_segment_create_process_private (svm_fifo_segment_main_t * sm, - svm_fifo_segment_create_args_t - *); -void svm_fifo_segment_preallocate_fifo_pairs (svm_fifo_segment_private_t * s, - u32 rx_fifo_size, - u32 tx_fifo_size, - u32 * n_fifo_pairs); -int svm_fifo_segment_attach (svm_fifo_segment_main_t * sm, - svm_fifo_segment_create_args_t * a); -void svm_fifo_segment_delete (svm_fifo_segment_main_t * sm, - svm_fifo_segment_private_t * s); - -svm_fifo_t *svm_fifo_segment_alloc_fifo (svm_fifo_segment_private_t * s, - u32 data_size_in_bytes, - svm_fifo_segment_freelist_t index); -void svm_fifo_segment_free_fifo (svm_fifo_segment_private_t * s, - svm_fifo_t * f, - svm_fifo_segment_freelist_t index); -void svm_fifo_segment_main_init (svm_fifo_segment_main_t * sm, u64 baseva, - u32 timeout_in_seconds); -u32 svm_fifo_segment_index (svm_fifo_segment_main_t * sm, - svm_fifo_segment_private_t * s); -u32 svm_fifo_segment_num_fifos (svm_fifo_segment_private_t * fifo_segment); -u32 svm_fifo_segment_num_free_fifos (svm_fifo_segment_private_t * - fifo_segment, u32 fifo_size_in_bytes); -void svm_fifo_segment_info (svm_fifo_segment_private_t * seg, char **address, - size_t * size); - -svm_fifo_segment_private_t - * svm_fifo_segment_segments_pool (svm_fifo_segment_main_t * sm); - -format_function_t format_svm_fifo_segment; -format_function_t format_svm_fifo_segment_type; - -#endif /* __included_ssvm_fifo_segment_h__ */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ |