diff options
| -rw-r--r-- | src/plugins/unittest/svm_fifo_test.c | 19 | ||||
| -rw-r--r-- | src/svm/fifo_segment.c | 79 | ||||
| -rw-r--r-- | src/svm/fifo_types.h | 19 | ||||
| -rw-r--r-- | src/svm/svm_fifo.h | 2 |
4 files changed, 62 insertions, 57 deletions
diff --git a/src/plugins/unittest/svm_fifo_test.c b/src/plugins/unittest/svm_fifo_test.c index a223e132d3d..6adba7e8e59 100644 --- a/src/plugins/unittest/svm_fifo_test.c +++ b/src/plugins/unittest/svm_fifo_test.c @@ -2103,6 +2103,7 @@ sfifo_test_fifo_segment_hello_world (int verbose) clib_memset (a, 0, sizeof (*a)); a->segment_name = "fifo-test1"; a->segment_size = 256 << 10; + a->segment_type = SSVM_SEGMENT_PRIVATE; rv = fifo_segment_create (sm, a); SFIFO_TEST (!rv, "svm_fifo_segment_create returned %d", rv); @@ -2153,6 +2154,9 @@ sfifo_test_fifo_segment_fifo_grow (int verbose) a->segment_name = "fifo-test1"; /* size chosen to be able to force multi chunk allocation lower */ a->segment_size = 256 << 10; + /* overhead that reduces the amount of space dedicated to fifos */ + a->segment_size += 1 << 14; + a->segment_type = SSVM_SEGMENT_PRIVATE; /* fifo allocation allocates chunks in batch */ n_batch = FIFO_SEGMENT_ALLOC_BATCH_SIZE; @@ -2490,6 +2494,7 @@ sfifo_test_fifo_segment_mempig (int verbose) a->segment_name = "fifo-test1"; a->segment_size = 256 << 10; + a->segment_type = SSVM_SEGMENT_PRIVATE; rv = fifo_segment_create (sm, a); @@ -2546,18 +2551,21 @@ approx_leq (uword a, uword b, u32 margin) static int sfifo_test_fifo_segment_prealloc (int verbose) { + u32 max_pairs, pairs_req, free_space, pair_mem, overhead; fifo_segment_create_args_t _a, *a = &_a; fifo_segment_main_t *sm = &segment_main; - u32 max_pairs, pairs_req, free_space, pair_mem; svm_fifo_t *f, *tf, *old; fifo_segment_t *fs; int rv, alloc; clib_memset (a, 0, sizeof (*a)); + /* Overhead due to segment internal headers and offsets. The magic 384 + * bytes are the fsh->n_reserved_bytes after seg init */ + overhead = (8 << 10) + 384; a->segment_name = "fifo-test-prealloc"; - a->segment_size = 256 << 10; - a->segment_type = SSVM_SEGMENT_MEMFD; + a->segment_size = (256 << 10) + overhead; + a->segment_type = SSVM_SEGMENT_PRIVATE; rv = fifo_segment_create (sm, a); SFIFO_TEST (!rv, "svm_fifo_segment_create returned %d", rv); @@ -2568,7 +2576,7 @@ sfifo_test_fifo_segment_prealloc (int verbose) * Prealloc chunks and headers */ free_space = fifo_segment_free_bytes (fs); - SFIFO_TEST (free_space <= 256 << 10, "free space expected %u is %u", + SFIFO_TEST (free_space - 4096 <= 256 << 10, "free space expected %u is %u", 256 << 10, free_space); rv = fifo_segment_prealloc_fifo_chunks (fs, 0, 4096, 50); SFIFO_TEST (rv == 0, "chunk prealloc should work"); @@ -2591,7 +2599,7 @@ sfifo_test_fifo_segment_prealloc (int verbose) rv = fifo_segment_free_bytes (fs); free_space -= sizeof (svm_fifo_t) * 50; /* Memory alloc alignment accounts for the difference */ - SFIFO_TEST (approx_leq (free_space, rv, 16), "free space expected %u is %u", + SFIFO_TEST (approx_leq (free_space, rv, 128), "free space expected %u is %u", free_space, rv); free_space = rv; @@ -2666,7 +2674,6 @@ sfifo_test_fifo_segment_prealloc (int verbose) */ fifo_segment_free_fifo (fs, old); fifo_segment_free_fifo (fs, tf); - close (fs->ssvm.fd); fifo_segment_delete (sm, fs); return 0; } diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c index d474b85f03b..b4c3cbe6c6d 100644 --- a/src/svm/fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -143,7 +143,7 @@ fss_chunk_freelist_unlock (fifo_segment_slice_t *fss) static inline int fss_chunk_fl_index_is_valid (fifo_segment_slice_t * fss, u32 fl_index) { - return (fl_index < fss->n_chunk_lens); + return (fl_index < FS_CHUNK_VEC_LEN); } static void @@ -239,57 +239,37 @@ fss_fl_chunk_bytes_sub (fifo_segment_slice_t * fss, uword size) int fifo_segment_init (fifo_segment_t * fs) { - u32 max_chunk_sz, align = 8, offset = 2 * 4096, vec_sz; - uword max_fifo, seg_start, seg_size; + u32 align = 8, offset = 2 * 4096, slices_sz; + uword max_fifo, seg_start, seg_sz; fifo_segment_header_t *fsh; - fifo_segment_slice_t *fss; ssvm_shared_header_t *sh; void *seg_data; - int i; /* TODO remove ssvm heap entirely */ sh = fs->ssvm.sh; seg_data = (u8 *) sh + offset; - seg_size = sh->ssvm_size - offset; + seg_sz = sh->ssvm_size - offset; + + fs->n_slices = clib_max (fs->n_slices, 1); + slices_sz = sizeof (fifo_segment_slice_t) * fs->n_slices; seg_start = round_pow2_u64 (pointer_to_uword (seg_data), align); fsh = uword_to_pointer (seg_start, void *); - memset (fsh, 0, sizeof (*fsh)); - fsh->byte_index = sizeof (*fsh); - fsh->max_byte_index = seg_size; - fs->max_byte_index = fsh->max_byte_index; - ASSERT (fsh->max_byte_index <= sh->ssvm_va + sh->ssvm_size); - - fs->h = sh->opaque[0] = fsh; - fs->n_slices = clib_max (fs->n_slices, 1); + memset (fsh, 0, sizeof (*fsh) + slices_sz); + fsh->byte_index = sizeof (*fsh) + slices_sz; + fsh->max_byte_index = seg_sz; fsh->n_slices = fs->n_slices; - max_fifo = - clib_min ((fsh_n_free_bytes (fsh) - 4096) / 2, FIFO_SEGMENT_MAX_FIFO_SIZE); - fsh->max_log2_chunk_size = max_log2 (max_fifo); - - fsh->slices = fsh_alloc (fsh, sizeof (*fss) * fs->n_slices); - memset (fsh->slices, 0, sizeof (*fss) * fs->n_slices); - max_chunk_sz = fsh->max_log2_chunk_size - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE; - max_chunk_sz += 1; - - for (i = 0; i < fs->n_slices; i++) - { - fss = fsh_slice_get (fsh, i); - vec_sz = max_chunk_sz * sizeof (*fss->free_chunks); - fss->free_chunks = fsh_alloc (fsh, vec_sz); - memset (fss->free_chunks, 0, vec_sz); - vec_sz = max_chunk_sz * sizeof (*fss->num_chunks); - fss->num_chunks = fsh_alloc (fsh, vec_sz); - memset (fss->num_chunks, 0, vec_sz); - - fss->chunk_lock = 0; - fss->n_chunk_lens = max_chunk_sz; - } - + max_fifo = clib_min ((seg_sz - slices_sz) / 2, FIFO_SEGMENT_MAX_FIFO_SIZE); + fsh->max_log2_fifo_size = min_log2 (max_fifo); fsh->n_cached_bytes = 0; fsh->n_reserved_bytes = fsh->byte_index; + ASSERT (fsh->max_byte_index <= sh->ssvm_size - offset); + + fs->max_byte_index = fsh->max_byte_index; + fs->h = sh->opaque[0] = fsh; + sh->ready = 1; return (0); } @@ -334,6 +314,7 @@ fifo_segment_create (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) int fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) { + fifo_segment_header_t *fsh; fifo_segment_t *fs; int rv; @@ -355,8 +336,15 @@ fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) } /* Fish the segment header */ - fs->h = fs->ssvm.sh->opaque[0]; + fsh = fs->h = fs->ssvm.sh->opaque[0]; + + /* Probably a segment without fifos */ + if (!fsh) + goto done; + fs->max_byte_index = fsh->max_byte_index; + +done: vec_add1 (a->new_segment_indices, fs - sm->segments); return (0); } @@ -401,7 +389,8 @@ fs_freelist_for_size (u32 size) { if (PREDICT_FALSE (size < FIFO_SEGMENT_MIN_FIFO_SIZE)) return 0; - return max_log2 (size) - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE; + return clib_min (max_log2 (size) - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE, + FS_CHUNK_VEC_LEN - 1); } static inline u32 @@ -417,8 +406,8 @@ fs_chunk_size_is_valid (fifo_segment_header_t * fsh, u32 size) * 4K minimum. It's not likely that anything good will happen * with a smaller FIFO. */ - return size >= FIFO_SEGMENT_MIN_FIFO_SIZE - && size <= (1ULL << fsh->max_log2_chunk_size); + return size >= FIFO_SEGMENT_MIN_FIFO_SIZE && + size <= (1ULL << fsh->max_log2_fifo_size); } svm_fifo_chunk_t * @@ -744,7 +733,7 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index, ASSERT (slice_index < fs->n_slices); - if (PREDICT_FALSE (data_bytes > 1 << fsh->max_log2_chunk_size)) + if (PREDICT_FALSE (data_bytes > 1 << fsh->max_log2_fifo_size)) return 0; fss = fsh_slice_get (fsh, slice_index); @@ -1018,7 +1007,7 @@ fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size) /* Count all free chunks? */ if (size == ~0) { - for (i = 0; i < fss->n_chunk_lens; i++) + for (i = 0; i < FS_CHUNK_VEC_LEN; i++) { c = fss->free_chunks[i]; if (c == 0) @@ -1036,7 +1025,7 @@ fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size) rounded_size = (1 << (max_log2 (size))); fl_index = fs_freelist_for_size (rounded_size); - if (fl_index >= fss->n_chunk_lens) + if (fl_index >= FS_CHUNK_VEC_LEN) return 0; c = fss->free_chunks[fl_index]; @@ -1267,7 +1256,7 @@ format_fifo_segment (u8 * s, va_list * args) for (slice_index = 0; slice_index < fs->n_slices; slice_index++) { fss = fsh_slice_get (fsh, slice_index); - for (i = 0; i < fss->n_chunk_lens; i++) + for (i = 0; i < FS_CHUNK_VEC_LEN; i++) { c = fss->free_chunks[i]; if (c == 0 && fss->num_chunks[i] == 0) diff --git a/src/svm/fifo_types.h b/src/svm/fifo_types.h index 9182a2ac3cf..e839e7eaab7 100644 --- a/src/svm/fifo_types.h +++ b/src/svm/fifo_types.h @@ -21,6 +21,14 @@ #include <vppinfra/rbtree.h> #include <vppinfra/lock.h> +#define FS_MIN_LOG2_CHUNK_SZ 12 /**< also min fifo size */ +#define FS_MAX_LOG2_CHUNK_SZ 22 /**< 4MB max chunk size */ +#define FS_CHUNK_VEC_LEN 11 /**< number of chunk sizes */ + +STATIC_ASSERT ((FS_MAX_LOG2_CHUNK_SZ - FS_MIN_LOG2_CHUNK_SZ) == + FS_CHUNK_VEC_LEN - 1, + "update chunk sizes"); + #define SVM_FIFO_TRACE (0) #define SVM_FIFO_MAX_EVT_SUBSCRIBERS 7 @@ -101,24 +109,23 @@ typedef struct _svm_fifo typedef struct fifo_segment_slice_ { + svm_fifo_chunk_t *free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */ svm_fifo_t *fifos; /**< Linked list of active RX fifos */ svm_fifo_t *free_fifos; /**< Freelists by fifo size */ - svm_fifo_chunk_t **free_chunks; /**< Freelists by chunk size */ - u32 *num_chunks; /**< Allocated chunks by chunk size */ uword n_fl_chunk_bytes; /**< Chunk bytes on freelist */ uword virtual_mem; /**< Slice sum of all fifo sizes */ - u32 n_chunk_lens; + u32 num_chunks[FS_CHUNK_VEC_LEN]; /**< Allocated chunks by chunk size */ + CLIB_CACHE_LINE_ALIGN_MARK (lock); u32 chunk_lock; } fifo_segment_slice_t; struct fifo_segment_header_ { - fifo_segment_slice_t *slices; /** Fixed array of slices */ uword n_cached_bytes; /**< Cached bytes */ u32 n_active_fifos; /**< Number of active fifos */ u32 n_reserved_bytes; /**< Bytes not to be allocated */ - u32 max_log2_chunk_size; /**< Max log2(chunk size) for fs */ + u32 max_log2_fifo_size; /**< Max log2(chunk size) for fs */ u8 flags; /**< Segment flags */ u8 n_slices; /**< Number of slices */ u8 high_watermark; /**< Memory pressure watermark high */ @@ -127,6 +134,8 @@ struct fifo_segment_header_ CLIB_CACHE_LINE_ALIGN_MARK (allocator); uword byte_index; uword max_byte_index; + CLIB_CACHE_LINE_ALIGN_MARK (slice); + fifo_segment_slice_t slices[0]; /** Fixed array of slices */ }; void fsh_virtual_mem_update (fifo_segment_header_t * fsh, u32 slice_index, diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index 2a054526865..5656e63d998 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -712,7 +712,7 @@ svm_fifo_size (svm_fifo_t * f) static inline void svm_fifo_set_size (svm_fifo_t * f, u32 size) { - if (size > (1 << f->fs_hdr->max_log2_chunk_size)) + if (size > (1 << f->fs_hdr->max_log2_fifo_size)) return; fsh_virtual_mem_update (f->fs_hdr, f->slice_index, (int) f->size - size); f->size = size; |