diff options
Diffstat (limited to 'src/svm')
| -rw-r--r-- | src/svm/fifo_segment.c | 130 | ||||
| -rw-r--r-- | src/svm/fifo_segment.h | 33 | ||||
| -rw-r--r-- | src/svm/svm_fifo.c | 40 | ||||
| -rw-r--r-- | src/svm/svm_fifo.h | 31 |
4 files changed, 217 insertions, 17 deletions
diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c index 6bf2634a290..9c332d6f1fa 100644 --- a/src/svm/fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -237,7 +237,7 @@ fs_free_list_for_size (u32 size) } static inline int -fs_fifo_size_is_valid (u32 size) +fs_chunk_size_is_valid (u32 size) { /* * 4K minimum. It's not likely that anything good will happen @@ -293,7 +293,7 @@ fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes, void *oldheap; int fl_index; - if (!fs_fifo_size_is_valid (data_bytes)) + if (!fs_chunk_size_is_valid (data_bytes)) { clib_warning ("fifo size out of range %d", data_bytes); return 0; @@ -304,7 +304,7 @@ fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes, sh = fs->ssvm.sh; ssvm_lock_non_recursive (sh, 1); - fsh = (fifo_segment_header_t *) sh->opaque[0]; + fsh = fs->h; vec_validate_init_empty (fsh->free_fifos, fl_index, 0); f = fsh->free_fifos[fl_index]; @@ -363,7 +363,7 @@ done: * Free fifo allocated in fifo segment */ void -fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f) +fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) { fifo_segment_header_t *fsh; ssvm_shared_header_t *sh; @@ -374,8 +374,8 @@ fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f) if (--f->refcnt > 0) return; - sh = s->ssvm.sh; - fsh = (fifo_segment_header_t *) sh->opaque[0]; + sh = fs->ssvm.sh; + fsh = fs->h; fl_index = f->freelist_index; ASSERT (fl_index < vec_len (fsh->free_fifos)); @@ -399,6 +399,29 @@ fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f) f->prev = 0; fsh->free_fifos[fl_index] = f; + /* If fifo has more chunks, free them */ + if (f->flags & SVM_FIFO_F_MULTI_CHUNK) + { + svm_fifo_chunk_t *cur, *next; + void *oldheap; + + next = f->start_chunk->next; + while (next != f->start_chunk) + { + cur = next; + next = next->next; + fl_index = fs_free_list_for_size (cur->length); + cur->next = fsh->free_chunks[fl_index]; + fsh->free_chunks[fl_index] = cur; + } + oldheap = ssvm_push_heap (sh); + svm_fifo_free_chunk_lookup (f); + ssvm_pop_heap (oldheap); + } + + /* not allocated on segment heap */ + svm_fifo_free_ooo_data (f); + if (CLIB_DEBUG) { f->master_session_index = ~0; @@ -431,13 +454,13 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0) return; - if (!fs_fifo_size_is_valid (rx_fifo_size)) + if (!fs_chunk_size_is_valid (rx_fifo_size)) { clib_warning ("rx fifo_size out of range %d", rx_fifo_size); return; } - if (!fs_fifo_size_is_valid (tx_fifo_size)) + if (!fs_chunk_size_is_valid (tx_fifo_size)) { clib_warning ("tx fifo_size out of range %d", tx_fifo_size); return; @@ -502,6 +525,51 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, ssvm_pop_heap (oldheap); } +int +fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, u32 chunk_size) +{ + ssvm_shared_header_t *sh; + svm_fifo_chunk_t *c; + void *oldheap; + int fl_index; + + if (!fs_chunk_size_is_valid (chunk_size)) + { + clib_warning ("chunk size out of range %d", chunk_size); + return 0; + } + + fl_index = fs_free_list_for_size (chunk_size); + + sh = fs->ssvm.sh; + ssvm_lock_non_recursive (sh, 1); + + vec_validate_init_empty (fs->h->free_chunks, fl_index, 0); + c = fs->h->free_chunks[fl_index]; + + oldheap = ssvm_push_heap (sh); + + if (!c) + { + c = svm_fifo_chunk_alloc (chunk_size); + if (!c) + { + ssvm_pop_heap (oldheap); + return -1; + } + } + else + { + fs->h->free_chunks[fl_index] = c->next; + } + + svm_fifo_add_chunk (f, c); + + ssvm_pop_heap (oldheap); + ssvm_unlock_non_recursive (sh); + return 0; +} + /** * Get number of active fifos */ @@ -559,6 +627,52 @@ fifo_segment_num_free_fifos (fifo_segment_t * fs, u32 fifo_size_in_bytes) return count; } +u32 +fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size) +{ + u32 count = 0, rounded_size, fl_index; + fifo_segment_header_t *fsh; + svm_fifo_chunk_t *c; + int i; + + fsh = fs->h; + + /* Count all free chunks? */ + if (size == ~0) + { + for (i = 0; i < vec_len (fsh->free_chunks); i++) + { + c = fsh->free_chunks[i]; + if (c == 0) + continue; + + while (c) + { + c = c->next; + count++; + } + } + return count; + } + + rounded_size = (1 << (max_log2 (size))); + fl_index = fs_free_list_for_size (rounded_size); + + if (fl_index >= vec_len (fsh->free_chunks)) + return 0; + + c = fsh->free_chunks[fl_index]; + if (c == 0) + return 0; + + while (c) + { + c = c->next; + count++; + } + return count; +} + u8 fifo_segment_has_fifos (fifo_segment_t * fs) { diff --git a/src/svm/fifo_segment.h b/src/svm/fifo_segment.h index cc0ff3adf3a..6ff538fcc07 100644 --- a/src/svm/fifo_segment.h +++ b/src/svm/fifo_segment.h @@ -38,10 +38,11 @@ typedef enum fifo_segment_flags_ typedef struct { - svm_fifo_t *fifos; /**< Linked list of active RX fifos */ - svm_fifo_t **free_fifos; /**< Freelists, by fifo size */ - u32 n_active_fifos; /**< Number of active fifos */ - u8 flags; /**< Segment flags */ + svm_fifo_t *fifos; /**< Linked list of active RX fifos */ + svm_fifo_t **free_fifos; /**< Freelists by fifo size */ + svm_fifo_chunk_t **free_chunks; /**< Freelists by chunk size */ + u32 n_active_fifos; /**< Number of active fifos */ + u8 flags; /**< Segment flags */ } fifo_segment_header_t; typedef struct @@ -82,7 +83,7 @@ void fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size); /** * Allocate fifo in fifo segment * - * @param fs fifo segment + * @param fs fifo segment for fifo * @param data_bytes size of default fifo chunk in bytes * @param ftype fifo type @ref fifo_segment_ftype_t * @return new fifo or 0 if alloc failed @@ -94,7 +95,7 @@ svm_fifo_t *fifo_segment_alloc_fifo (fifo_segment_t * fs, /** * Free fifo allocated in fifo segment * - * @param fs fifo segment + * @param fs fifo segment for fifo * @param f fifo to be freed */ void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f); @@ -107,7 +108,7 @@ void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f); * is hit, the number of fifo pairs requested is updated by subtracting the * number of fifos that have been successfully allocated. * - * @param fs fifo segment + * @param fs fifo segment for fifo * @param rx_fifo_size data size of rx fifos * @param tx_fifo_size data size of tx fifos * @param n_fifo_pairs number of pairs requested. Prior to returning, this @@ -117,10 +118,28 @@ void fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs, u32 rx_fifo_size, u32 tx_fifo_size, u32 * n_fifo_pairs); +/** + * Grow fifo size by adding an additional chunk of memory + * + * @param fs fifo segment for fifo + * @param f fifo to be grown + * @param chunk_size number of bytes to be added to fifo + * @return 0 on success or a negative number otherwise + */ +int fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, + u32 chunk_size); u8 fifo_segment_has_fifos (fifo_segment_t * fs); svm_fifo_t *fifo_segment_get_fifo_list (fifo_segment_t * fs); u32 fifo_segment_num_fifos (fifo_segment_t * fs); u32 fifo_segment_num_free_fifos (fifo_segment_t * fs, u32 fifo_size_in_bytes); +/** + * Find number of free chunks of given size + * + * @param fs fifo segment + * @param size chunk size of interest or ~0 if all should be counted + * @return number of chunks of given size + */ +u32 fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size); void fifo_segment_main_init (fifo_segment_main_t * sm, u64 baseva, u32 timeout_in_seconds); diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index eafe497eec1..c8fd263c094 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -298,7 +298,9 @@ svm_fifo_init (svm_fifo_t * f, u32 size) f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = f->start_chunk; } -/** create an svm fifo, in the current heap. Fails vs blow up the process */ +/** + * Creates a fifo in the current heap. Fails vs blow up the process + */ svm_fifo_t * svm_fifo_create (u32 data_size_in_bytes) { @@ -317,6 +319,27 @@ svm_fifo_create (u32 data_size_in_bytes) return f; } +/** + * Creates a fifo chunk in the current heap + */ +svm_fifo_chunk_t * +svm_fifo_chunk_alloc (u32 size) +{ + svm_fifo_chunk_t *c; + u32 rounded_size; + + /* round chunk size to the next highest power-of-two */ + rounded_size = (1 << (max_log2 (size))); + c = clib_mem_alloc_aligned_or_null (sizeof (*c) + rounded_size, + CLIB_CACHE_LINE_BYTES); + if (c == 0) + return 0; + + clib_memset (c, 0, sizeof (*c)); + c->length = rounded_size; + return c; +} + static inline void svm_fifo_size_update (svm_fifo_t * f, svm_fifo_chunk_t * c) { @@ -462,13 +485,26 @@ svm_fifo_find_chunk (svm_fifo_t * f, u32 pos) } void +svm_fifo_free_chunk_lookup (svm_fifo_t * f) +{ + rb_tree_free_nodes (&f->chunk_lookup); +} + +void +svm_fifo_free_ooo_data (svm_fifo_t * f) +{ + pool_free (f->ooo_segments); +} + +void svm_fifo_free (svm_fifo_t * f) { ASSERT (f->refcnt > 0); if (--f->refcnt == 0) { - pool_free (f->ooo_segments); + /* ooo data is not allocated on segment heap */ + svm_fifo_free_chunk_lookup (f); clib_mem_free (f); } } diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index 4f2f61939b7..7c84fc82198 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -332,6 +332,19 @@ svm_fifo_unset_event (svm_fifo_t * f) svm_fifo_t *svm_fifo_create (u32 data_size_in_bytes); void svm_fifo_init (svm_fifo_t * f, u32 size); + +/** + * Allocate a fifo chunk on heap + * + * If the chunk is allocated on a fifo segment, this should be called + * with the segment's heap pushed. + * + * @param size chunk size in bytes. Will be rounded to the next highest + * power-of-two + * @return new chunk or 0 if alloc failed + */ +svm_fifo_chunk_t *svm_fifo_chunk_alloc (u32 size); + /** * Grow fifo size by adding chunk to chunk list * @@ -343,6 +356,24 @@ void svm_fifo_init (svm_fifo_t * f, u32 size); */ void svm_fifo_add_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c); void svm_fifo_free (svm_fifo_t * f); +/** + * Cleanup fifo chunk lookup rb tree + * + * The rb tree is allocated in segment heap so this should be called + * with it pushed. + * + * @param f fifo to cleanup + */ +void svm_fifo_free_chunk_lookup (svm_fifo_t * f); +/** + * Cleanup fifo ooo data + * + * The ooo data is allocated in producer process memory. The fifo + * segment heap should not be pushed. + * + * @param f fifo to cleanup + */ +void svm_fifo_free_ooo_data (svm_fifo_t * f); int svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes, const u8 * copy_from_here); |