diff options
| -rw-r--r-- | src/plugins/unittest/svm_fifo_test.c | 98 | ||||
| -rw-r--r-- | src/svm/fifo_segment.c | 130 | ||||
| -rw-r--r-- | src/svm/fifo_segment.h | 33 | ||||
| -rw-r--r-- | src/svm/svm_fifo.c | 40 | ||||
| -rw-r--r-- | src/svm/svm_fifo.h | 31 | ||||
| -rw-r--r-- | src/vnet/session/segment_manager.c | 15 | ||||
| -rw-r--r-- | src/vnet/session/segment_manager.h | 13 | ||||
| -rw-r--r-- | src/vppinfra/rbtree.c | 4 |
8 files changed, 338 insertions, 26 deletions
diff --git a/src/plugins/unittest/svm_fifo_test.c b/src/plugins/unittest/svm_fifo_test.c index 163a8e42c68..4e44441ebdc 100644 --- a/src/plugins/unittest/svm_fifo_test.c +++ b/src/plugins/unittest/svm_fifo_test.c @@ -1244,7 +1244,7 @@ sfifo_test_fifo_segment_hello_world (int verbose) fifo_segment_create_args_t _a, *a = &_a; fifo_segment_main_t *sm = &segment_main; u8 *test_data, *retrieved_data = 0; - fifo_segment_t *sp; + fifo_segment_t *fs; svm_fifo_t *f; int rv; @@ -1256,8 +1256,8 @@ sfifo_test_fifo_segment_hello_world (int verbose) SFIFO_TEST (!rv, "svm_fifo_segment_create returned %d", rv); - sp = fifo_segment_get_segment (sm, a->new_segment_indices[0]); - f = fifo_segment_alloc_fifo (sp, 4096, FIFO_SEGMENT_RX_FIFO); + fs = fifo_segment_get_segment (sm, a->new_segment_indices[0]); + f = fifo_segment_alloc_fifo (fs, 4096, FIFO_SEGMENT_RX_FIFO); SFIFO_TEST (f != 0, "svm_fifo_segment_alloc_fifo"); @@ -1281,8 +1281,87 @@ sfifo_test_fifo_segment_hello_world (int verbose) vec_free (test_data); vec_free (retrieved_data); - fifo_segment_free_fifo (sp, f); - fifo_segment_delete (sm, sp); + vec_free (a->new_segment_indices); + fifo_segment_free_fifo (fs, f); + fifo_segment_delete (sm, fs); + return 0; +} + +static int +sfifo_test_fifo_segment_fifo_grow (int verbose) +{ + fifo_segment_main_t *sm = &segment_main; + fifo_segment_create_args_t _a, *a = &_a; + int rv, fifo_size = 4096, n_chunks; + fifo_segment_t *fs; + svm_fifo_t *f; + + clib_memset (a, 0, sizeof (*a)); + a->segment_name = "fifo-test1"; + a->segment_size = 256 << 10; + + rv = fifo_segment_create (sm, a); + + SFIFO_TEST (!rv, "svm_fifo_segment_create returned %d", rv); + + /* + * Alloc and grow fifo + */ + fs = fifo_segment_get_segment (sm, a->new_segment_indices[0]); + f = fifo_segment_alloc_fifo (fs, fifo_size, FIFO_SEGMENT_RX_FIFO); + + SFIFO_TEST (f != 0, "svm_fifo_segment_alloc_fifo"); + + fifo_segment_grow_fifo (fs, f, fifo_size); + SFIFO_TEST (f->size == 2 * fifo_size, "fifo size should be %u is %u", + 2 * fifo_size, f->size); + + fifo_segment_grow_fifo (fs, f, 16 * fifo_size); + SFIFO_TEST (f->size == 18 * fifo_size, "fifo size should be %u is %u", + 18 * fifo_size, f->size); + + /* + * Free and test free list size + */ + fifo_segment_free_fifo (fs, f); + + n_chunks = fifo_segment_num_free_chunks (fs, fifo_size); + SFIFO_TEST (n_chunks == 1, "free 2^10B chunks should be %u is %u", 1, + n_chunks); + n_chunks = fifo_segment_num_free_chunks (fs, 16 * fifo_size); + SFIFO_TEST (n_chunks == 1, "free 2^14B chunks should be %u is %u", 1, + n_chunks); + n_chunks = fifo_segment_num_free_chunks (fs, ~0); + SFIFO_TEST (n_chunks == 2, "free chunks should be %u is %u", 2, n_chunks); + + /* + * Realloc fifo + */ + f = fifo_segment_alloc_fifo (fs, fifo_size, FIFO_SEGMENT_RX_FIFO); + + fifo_segment_grow_fifo (fs, f, fifo_size); + n_chunks = fifo_segment_num_free_chunks (fs, fifo_size); + SFIFO_TEST (n_chunks == 0, "free 2^10B chunks should be %u is %u", 0, + n_chunks); + + fifo_segment_grow_fifo (fs, f, 16 * fifo_size); + SFIFO_TEST (n_chunks == 0, "free 2^14B chunks should be %u is %u", 0, + n_chunks); + n_chunks = fifo_segment_num_free_chunks (fs, ~0); + SFIFO_TEST (n_chunks == 0, "free chunks should be %u is %u", 0, n_chunks); + + /* + * Free again + */ + fifo_segment_free_fifo (fs, f); + n_chunks = fifo_segment_num_free_chunks (fs, ~0); + SFIFO_TEST (n_chunks == 2, "free chunks should be %u is %u", 2, n_chunks); + + /* + * Cleanup + */ + fifo_segment_delete (sm, fs); + vec_free (a->new_segment_indices); return 0; } @@ -1310,6 +1389,7 @@ sfifo_test_fifo_segment_slave (int verbose) SFIFO_TEST (!rv, "svm_fifo_segment_attach returned %d", rv); sp = fifo_segment_get_segment (sm, a->new_segment_indices[0]); + vec_free (a->new_segment_indices); sh = sp->ssvm.sh; fsh = (fifo_segment_header_t *) sh->opaque[0]; @@ -1388,6 +1468,7 @@ sfifo_test_fifo_segment_master_slave (int verbose) result = (u32 *) f->head_chunk->data; SFIFO_TEST (*result == 0, "slave reported no error"); + vec_free (a->new_segment_indices); vec_free (test_data); fifo_segment_free_fifo (sp, f); fifo_segment_delete (sm, sp); @@ -1478,12 +1559,19 @@ sfifo_test_fifo_segment (vlib_main_t * vm, unformat_input_t * input) if ((rv = sfifo_test_fifo_segment_mempig (verbose))) return -1; } + else if (unformat (input, "grow fifo")) + { + if ((rv = sfifo_test_fifo_segment_fifo_grow (verbose))) + return -1; + } else if (unformat (input, "all")) { if ((rv = sfifo_test_fifo_segment_hello_world (verbose))) return -1; if ((rv = sfifo_test_fifo_segment_mempig (verbose))) return -1; + if ((rv = sfifo_test_fifo_segment_fifo_grow (verbose))) + return -1; /* Pretty slow so avoid running it always if ((rv = sfifo_test_fifo_segment_master_slave (verbose))) return -1; diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c index 6bf2634a290..9c332d6f1fa 100644 --- a/src/svm/fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -237,7 +237,7 @@ fs_free_list_for_size (u32 size) } static inline int -fs_fifo_size_is_valid (u32 size) +fs_chunk_size_is_valid (u32 size) { /* * 4K minimum. It's not likely that anything good will happen @@ -293,7 +293,7 @@ fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes, void *oldheap; int fl_index; - if (!fs_fifo_size_is_valid (data_bytes)) + if (!fs_chunk_size_is_valid (data_bytes)) { clib_warning ("fifo size out of range %d", data_bytes); return 0; @@ -304,7 +304,7 @@ fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes, sh = fs->ssvm.sh; ssvm_lock_non_recursive (sh, 1); - fsh = (fifo_segment_header_t *) sh->opaque[0]; + fsh = fs->h; vec_validate_init_empty (fsh->free_fifos, fl_index, 0); f = fsh->free_fifos[fl_index]; @@ -363,7 +363,7 @@ done: * Free fifo allocated in fifo segment */ void -fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f) +fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) { fifo_segment_header_t *fsh; ssvm_shared_header_t *sh; @@ -374,8 +374,8 @@ fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f) if (--f->refcnt > 0) return; - sh = s->ssvm.sh; - fsh = (fifo_segment_header_t *) sh->opaque[0]; + sh = fs->ssvm.sh; + fsh = fs->h; fl_index = f->freelist_index; ASSERT (fl_index < vec_len (fsh->free_fifos)); @@ -399,6 +399,29 @@ fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f) f->prev = 0; fsh->free_fifos[fl_index] = f; + /* If fifo has more chunks, free them */ + if (f->flags & SVM_FIFO_F_MULTI_CHUNK) + { + svm_fifo_chunk_t *cur, *next; + void *oldheap; + + next = f->start_chunk->next; + while (next != f->start_chunk) + { + cur = next; + next = next->next; + fl_index = fs_free_list_for_size (cur->length); + cur->next = fsh->free_chunks[fl_index]; + fsh->free_chunks[fl_index] = cur; + } + oldheap = ssvm_push_heap (sh); + svm_fifo_free_chunk_lookup (f); + ssvm_pop_heap (oldheap); + } + + /* not allocated on segment heap */ + svm_fifo_free_ooo_data (f); + if (CLIB_DEBUG) { f->master_session_index = ~0; @@ -431,13 +454,13 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0) return; - if (!fs_fifo_size_is_valid (rx_fifo_size)) + if (!fs_chunk_size_is_valid (rx_fifo_size)) { clib_warning ("rx fifo_size out of range %d", rx_fifo_size); return; } - if (!fs_fifo_size_is_valid (tx_fifo_size)) + if (!fs_chunk_size_is_valid (tx_fifo_size)) { clib_warning ("tx fifo_size out of range %d", tx_fifo_size); return; @@ -502,6 +525,51 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, ssvm_pop_heap (oldheap); } +int +fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, u32 chunk_size) +{ + ssvm_shared_header_t *sh; + svm_fifo_chunk_t *c; + void *oldheap; + int fl_index; + + if (!fs_chunk_size_is_valid (chunk_size)) + { + clib_warning ("chunk size out of range %d", chunk_size); + return 0; + } + + fl_index = fs_free_list_for_size (chunk_size); + + sh = fs->ssvm.sh; + ssvm_lock_non_recursive (sh, 1); + + vec_validate_init_empty (fs->h->free_chunks, fl_index, 0); + c = fs->h->free_chunks[fl_index]; + + oldheap = ssvm_push_heap (sh); + + if (!c) + { + c = svm_fifo_chunk_alloc (chunk_size); + if (!c) + { + ssvm_pop_heap (oldheap); + return -1; + } + } + else + { + fs->h->free_chunks[fl_index] = c->next; + } + + svm_fifo_add_chunk (f, c); + + ssvm_pop_heap (oldheap); + ssvm_unlock_non_recursive (sh); + return 0; +} + /** * Get number of active fifos */ @@ -559,6 +627,52 @@ fifo_segment_num_free_fifos (fifo_segment_t * fs, u32 fifo_size_in_bytes) return count; } +u32 +fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size) +{ + u32 count = 0, rounded_size, fl_index; + fifo_segment_header_t *fsh; + svm_fifo_chunk_t *c; + int i; + + fsh = fs->h; + + /* Count all free chunks? */ + if (size == ~0) + { + for (i = 0; i < vec_len (fsh->free_chunks); i++) + { + c = fsh->free_chunks[i]; + if (c == 0) + continue; + + while (c) + { + c = c->next; + count++; + } + } + return count; + } + + rounded_size = (1 << (max_log2 (size))); + fl_index = fs_free_list_for_size (rounded_size); + + if (fl_index >= vec_len (fsh->free_chunks)) + return 0; + + c = fsh->free_chunks[fl_index]; + if (c == 0) + return 0; + + while (c) + { + c = c->next; + count++; + } + return count; +} + u8 fifo_segment_has_fifos (fifo_segment_t * fs) { diff --git a/src/svm/fifo_segment.h b/src/svm/fifo_segment.h index cc0ff3adf3a..6ff538fcc07 100644 --- a/src/svm/fifo_segment.h +++ b/src/svm/fifo_segment.h @@ -38,10 +38,11 @@ typedef enum fifo_segment_flags_ typedef struct { - svm_fifo_t *fifos; /**< Linked list of active RX fifos */ - svm_fifo_t **free_fifos; /**< Freelists, by fifo size */ - u32 n_active_fifos; /**< Number of active fifos */ - u8 flags; /**< Segment flags */ + svm_fifo_t *fifos; /**< Linked list of active RX fifos */ + svm_fifo_t **free_fifos; /**< Freelists by fifo size */ + svm_fifo_chunk_t **free_chunks; /**< Freelists by chunk size */ + u32 n_active_fifos; /**< Number of active fifos */ + u8 flags; /**< Segment flags */ } fifo_segment_header_t; typedef struct @@ -82,7 +83,7 @@ void fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size); /** * Allocate fifo in fifo segment * - * @param fs fifo segment + * @param fs fifo segment for fifo * @param data_bytes size of default fifo chunk in bytes * @param ftype fifo type @ref fifo_segment_ftype_t * @return new fifo or 0 if alloc failed @@ -94,7 +95,7 @@ svm_fifo_t *fifo_segment_alloc_fifo (fifo_segment_t * fs, /** * Free fifo allocated in fifo segment * - * @param fs fifo segment + * @param fs fifo segment for fifo * @param f fifo to be freed */ void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f); @@ -107,7 +108,7 @@ void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f); * is hit, the number of fifo pairs requested is updated by subtracting the * number of fifos that have been successfully allocated. * - * @param fs fifo segment + * @param fs fifo segment for fifo * @param rx_fifo_size data size of rx fifos * @param tx_fifo_size data size of tx fifos * @param n_fifo_pairs number of pairs requested. Prior to returning, this @@ -117,10 +118,28 @@ void fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, u32 rx_fifo_size, u32 tx_fifo_size, u32 * n_fifo_pairs); +/** + * Grow fifo size by adding an additional chunk of memory + * + * @param fs fifo segment for fifo + * @param f fifo to be grown + * @param chunk_size number of bytes to be added to fifo + * @return 0 on success or a negative number otherwise + */ +int fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, + u32 chunk_size); u8 fifo_segment_has_fifos (fifo_segment_t * fs); svm_fifo_t *fifo_segment_get_fifo_list (fifo_segment_t * fs); u32 fifo_segment_num_fifos (fifo_segment_t * fs); u32 fifo_segment_num_free_fifos (fifo_segment_t * fs, u32 fifo_size_in_bytes); +/** + * Find number of free chunks of given size + * + * @param fs fifo segment + * @param size chunk size of interest or ~0 if all should be counted + * @return number of chunks of given size + */ +u32 fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size); void fifo_segment_main_init (fifo_segment_main_t * sm, u64 baseva, u32 timeout_in_seconds); diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index eafe497eec1..c8fd263c094 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -298,7 +298,9 @@ svm_fifo_init (svm_fifo_t * f, u32 size) f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = f->start_chunk; } -/** create an svm fifo, in the current heap. Fails vs blow up the process */ +/** + * Creates a fifo in the current heap. Fails vs blow up the process + */ svm_fifo_t * svm_fifo_create (u32 data_size_in_bytes) { @@ -317,6 +319,27 @@ svm_fifo_create (u32 data_size_in_bytes) return f; } +/** + * Creates a fifo chunk in the current heap + */ +svm_fifo_chunk_t * +svm_fifo_chunk_alloc (u32 size) +{ + svm_fifo_chunk_t *c; + u32 rounded_size; + + /* round chunk size to the next highest power-of-two */ + rounded_size = (1 << (max_log2 (size))); + c = clib_mem_alloc_aligned_or_null (sizeof (*c) + rounded_size, + CLIB_CACHE_LINE_BYTES); + if (c == 0) + return 0; + + clib_memset (c, 0, sizeof (*c)); + c->length = rounded_size; + return c; +} + static inline void svm_fifo_size_update (svm_fifo_t * f, svm_fifo_chunk_t * c) { @@ -462,13 +485,26 @@ svm_fifo_find_chunk (svm_fifo_t * f, u32 pos) } void +svm_fifo_free_chunk_lookup (svm_fifo_t * f) +{ + rb_tree_free_nodes (&f->chunk_lookup); +} + +void +svm_fifo_free_ooo_data (svm_fifo_t * f) +{ + pool_free (f->ooo_segments); +} + +void svm_fifo_free (svm_fifo_t * f) { ASSERT (f->refcnt > 0); if (--f->refcnt == 0) { - pool_free (f->ooo_segments); + /* ooo data is not allocated on segment heap */ + svm_fifo_free_chunk_lookup (f); clib_mem_free (f); } } diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index 4f2f61939b7..7c84fc82198 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -332,6 +332,19 @@ svm_fifo_unset_event (svm_fifo_t * f) svm_fifo_t *svm_fifo_create (u32 data_size_in_bytes); void svm_fifo_init (svm_fifo_t * f, u32 size); + +/** + * Allocate a fifo chunk on heap + * + * If the chunk is allocated on a fifo segment, this should be called + * with the segment's heap pushed. + * + * @param size chunk size in bytes. Will be rounded to the next highest + * power-of-two + * @return new chunk or 0 if alloc failed + */ +svm_fifo_chunk_t *svm_fifo_chunk_alloc (u32 size); + /** * Grow fifo size by adding chunk to chunk list * @@ -343,6 +356,24 @@ void svm_fifo_init (svm_fifo_t * f, u32 size); */ void svm_fifo_add_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c); void svm_fifo_free (svm_fifo_t * f); +/** + * Cleanup fifo chunk lookup rb tree + * + * The rb tree is allocated in segment heap so this should be called + * with it pushed. + * + * @param f fifo to cleanup + */ +void svm_fifo_free_chunk_lookup (svm_fifo_t * f); +/** + * Cleanup fifo ooo data + * + * The ooo data is allocated in producer process memory. The fifo + * segment heap should not be pushed. + * + * @param f fifo to cleanup + */ +void svm_fifo_free_ooo_data (svm_fifo_t * f); int svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, const u8 * copy_from_here); diff --git a/src/vnet/session/segment_manager.c b/src/vnet/session/segment_manager.c index 47c7f563542..9a8af3b42fc 100644 --- a/src/vnet/session/segment_manager.c +++ b/src/vnet/session/segment_manager.c @@ -627,8 +627,8 @@ alloc_check: void segment_manager_dealloc_fifos (svm_fifo_t * rx_fifo, svm_fifo_t * tx_fifo) { - fifo_segment_t *fs; segment_manager_t *sm; + fifo_segment_t *fs; u32 segment_index; if (!rx_fifo || !tx_fifo) @@ -669,6 +669,19 @@ segment_manager_dealloc_fifos (svm_fifo_t * rx_fifo, svm_fifo_t * tx_fifo) segment_manager_segment_reader_unlock (sm); } +int +segment_manager_grow_fifo (segment_manager_t * sm, svm_fifo_t * f, u32 size) +{ + fifo_segment_t *fs; + int rv; + + fs = segment_manager_get_segment_w_lock (sm, f->segment_index); + rv = fifo_segment_grow_fifo (fs, f, size); + segment_manager_segment_reader_unlock (sm); + + return rv; +} + u32 segment_manager_evt_q_expected_size (u32 q_len) { diff --git a/src/vnet/session/segment_manager.h b/src/vnet/session/segment_manager.h index 4f30b7a8d2c..cbf8e353522 100644 --- a/src/vnet/session/segment_manager.h +++ b/src/vnet/session/segment_manager.h @@ -94,7 +94,7 @@ void segment_manager_del_segment (segment_manager_t * sm, fifo_segment_t *segment_manager_get_segment (segment_manager_t * sm, u32 segment_index); fifo_segment_t *segment_manager_get_segment_w_handle (u64 sh); -fifo_segment_t *segment_manager_get_segment_w_lock (segment_manager_t *, +fifo_segment_t *segment_manager_get_segment_w_lock (segment_manager_t * sm, u32 segment_index); int segment_manager_add_first_segment (segment_manager_t * sm, u32 segment_size); @@ -114,6 +114,17 @@ int segment_manager_try_alloc_fifos (fifo_segment_t * fs, svm_fifo_t ** tx_fifo); void segment_manager_dealloc_fifos (svm_fifo_t * rx_fifo, svm_fifo_t * tx_fifo); + +/** + * Grows fifo owned by segment manager + * + * @param sm segment manager that owns the fifo + * @param f fifo to be grown + * @param size amount of bytes to add to fifo + * @return 0 on success, negative number otherwise + */ +int segment_manager_grow_fifo (segment_manager_t * sm, svm_fifo_t * f, + u32 size); u8 segment_manager_has_fifos (segment_manager_t * sm); svm_msg_q_t *segment_manager_alloc_queue (fifo_segment_t * fs, diff --git a/src/vppinfra/rbtree.c b/src/vppinfra/rbtree.c index 95e9d10967b..5bb2e875e86 100644 --- a/src/vppinfra/rbtree.c +++ b/src/vppinfra/rbtree.c @@ -401,8 +401,8 @@ rb_tree_n_nodes (rb_tree_t * rt) void rb_tree_free_nodes (rb_tree_t * rt) { - rb_node_t *n; - pool_flush (n, rt->nodes,;); + pool_free (rt->nodes); + rt->root = RBTREE_TNIL_INDEX; } void |