svm: separate private from shared fifo segment ptrs

Type: improvement

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: Idcc8d862b98dba4a67f829c1778377f35ad47b00

4 files changed, 62 insertions, 57 deletions

diff --git a/src/plugins/unittest/svm_fifo_test.c b/src/plugins/unittest/svm_fifo_test.c
index a223e132d3d..6adba7e8e59 100644
--- a/src/plugins/unittest/svm_fifo_test.c
+++ b/src/plugins/unittest/svm_fifo_test.c
@@ -2103,6 +2103,7 @@ sfifo_test_fifo_segment_hello_world (int verbose)
   clib_memset (a, 0, sizeof (*a));
   a->segment_name = "fifo-test1";
   a->segment_size = 256 << 10;
+  a->segment_type = SSVM_SEGMENT_PRIVATE;
 
   rv = fifo_segment_create (sm, a);
   SFIFO_TEST (!rv, "svm_fifo_segment_create returned %d", rv);
@@ -2153,6 +2154,9 @@ sfifo_test_fifo_segment_fifo_grow (int verbose)
   a->segment_name = "fifo-test1";
   /* size chosen to be able to force multi chunk allocation lower */
   a->segment_size = 256 << 10;
+  /* overhead that reduces the amount of space dedicated to fifos */
+  a->segment_size += 1 << 14;
+  a->segment_type = SSVM_SEGMENT_PRIVATE;
 
   /* fifo allocation allocates chunks in batch */
   n_batch = FIFO_SEGMENT_ALLOC_BATCH_SIZE;
@@ -2490,6 +2494,7 @@ sfifo_test_fifo_segment_mempig (int verbose)
 
   a->segment_name = "fifo-test1";
   a->segment_size = 256 << 10;
+  a->segment_type = SSVM_SEGMENT_PRIVATE;
 
   rv = fifo_segment_create (sm, a);
 
@@ -2546,18 +2551,21 @@ approx_leq (uword a, uword b, u32 margin)
 static int
 sfifo_test_fifo_segment_prealloc (int verbose)
 {
+  u32 max_pairs, pairs_req, free_space, pair_mem, overhead;
   fifo_segment_create_args_t _a, *a = &_a;
   fifo_segment_main_t *sm = &segment_main;
-  u32 max_pairs, pairs_req, free_space, pair_mem;
   svm_fifo_t *f, *tf, *old;
   fifo_segment_t *fs;
   int rv, alloc;
 
   clib_memset (a, 0, sizeof (*a));
 
+  /* Overhead due to segment internal headers and offsets. The magic 384
+   * bytes are the fsh->n_reserved_bytes after seg init */
+  overhead = (8 << 10) + 384;
   a->segment_name = "fifo-test-prealloc";
-  a->segment_size = 256 << 10;
-  a->segment_type = SSVM_SEGMENT_MEMFD;
+  a->segment_size = (256 << 10) + overhead;
+  a->segment_type = SSVM_SEGMENT_PRIVATE;
 
   rv = fifo_segment_create (sm, a);
   SFIFO_TEST (!rv, "svm_fifo_segment_create returned %d", rv);
@@ -2568,7 +2576,7 @@ sfifo_test_fifo_segment_prealloc (int verbose)
    * Prealloc chunks and headers
    */
   free_space = fifo_segment_free_bytes (fs);
-  SFIFO_TEST (free_space <= 256 << 10, "free space expected %u is %u",
+  SFIFO_TEST (free_space - 4096 <= 256 << 10, "free space expected %u is %u",
 	      256 << 10, free_space);
   rv = fifo_segment_prealloc_fifo_chunks (fs, 0, 4096, 50);
   SFIFO_TEST (rv == 0, "chunk prealloc should work");
@@ -2591,7 +2599,7 @@ sfifo_test_fifo_segment_prealloc (int verbose)
   rv = fifo_segment_free_bytes (fs);
   free_space -= sizeof (svm_fifo_t) * 50;
   /* Memory alloc alignment accounts for the difference */
-  SFIFO_TEST (approx_leq (free_space, rv, 16), "free space expected %u is %u",
+  SFIFO_TEST (approx_leq (free_space, rv, 128), "free space expected %u is %u",
 	      free_space, rv);
   free_space = rv;
 
@@ -2666,7 +2674,6 @@ sfifo_test_fifo_segment_prealloc (int verbose)
    */
   fifo_segment_free_fifo (fs, old);
   fifo_segment_free_fifo (fs, tf);
-  close (fs->ssvm.fd);
   fifo_segment_delete (sm, fs);
   return 0;
 }
diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c
index d474b85f03b..b4c3cbe6c6d 100644
--- a/src/svm/fifo_segment.c
+++ b/src/svm/fifo_segment.c
@@ -143,7 +143,7 @@ fss_chunk_freelist_unlock (fifo_segment_slice_t *fss)
 static inline int
 fss_chunk_fl_index_is_valid (fifo_segment_slice_t * fss, u32 fl_index)
 {
-  return (fl_index < fss->n_chunk_lens);
+  return (fl_index < FS_CHUNK_VEC_LEN);
 }
 
 static void
@@ -239,57 +239,37 @@ fss_fl_chunk_bytes_sub (fifo_segment_slice_t * fss, uword size)
 int
 fifo_segment_init (fifo_segment_t * fs)
 {
-  u32 max_chunk_sz, align = 8, offset = 2 * 4096, vec_sz;
-  uword max_fifo, seg_start, seg_size;
+  u32 align = 8, offset = 2 * 4096, slices_sz;
+  uword max_fifo, seg_start, seg_sz;
   fifo_segment_header_t *fsh;
-  fifo_segment_slice_t *fss;
   ssvm_shared_header_t *sh;
   void *seg_data;
-  int i;
 
   /* TODO remove ssvm heap entirely */
   sh = fs->ssvm.sh;
 
   seg_data = (u8 *) sh + offset;
-  seg_size = sh->ssvm_size - offset;
+  seg_sz = sh->ssvm_size - offset;
+
+  fs->n_slices = clib_max (fs->n_slices, 1);
+  slices_sz = sizeof (fifo_segment_slice_t) * fs->n_slices;
 
   seg_start = round_pow2_u64 (pointer_to_uword (seg_data), align);
   fsh = uword_to_pointer (seg_start, void *);
-  memset (fsh, 0, sizeof (*fsh));
-  fsh->byte_index = sizeof (*fsh);
-  fsh->max_byte_index = seg_size;
-  fs->max_byte_index = fsh->max_byte_index;
-  ASSERT (fsh->max_byte_index <= sh->ssvm_va + sh->ssvm_size);
-
-  fs->h = sh->opaque[0] = fsh;
-  fs->n_slices = clib_max (fs->n_slices, 1);
+  memset (fsh, 0, sizeof (*fsh) + slices_sz);
 
+  fsh->byte_index = sizeof (*fsh) + slices_sz;
+  fsh->max_byte_index = seg_sz;
   fsh->n_slices = fs->n_slices;
-  max_fifo =
-    clib_min ((fsh_n_free_bytes (fsh) - 4096) / 2, FIFO_SEGMENT_MAX_FIFO_SIZE);
-  fsh->max_log2_chunk_size = max_log2 (max_fifo);
-
-  fsh->slices = fsh_alloc (fsh, sizeof (*fss) * fs->n_slices);
-  memset (fsh->slices, 0, sizeof (*fss) * fs->n_slices);
-  max_chunk_sz = fsh->max_log2_chunk_size - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE;
-  max_chunk_sz += 1;
-
-  for (i = 0; i < fs->n_slices; i++)
-    {
-      fss = fsh_slice_get (fsh, i);
-      vec_sz = max_chunk_sz * sizeof (*fss->free_chunks);
-      fss->free_chunks = fsh_alloc (fsh, vec_sz);
-      memset (fss->free_chunks, 0, vec_sz);
-      vec_sz = max_chunk_sz * sizeof (*fss->num_chunks);
-      fss->num_chunks = fsh_alloc (fsh, vec_sz);
-      memset (fss->num_chunks, 0, vec_sz);
-
-      fss->chunk_lock = 0;
-      fss->n_chunk_lens = max_chunk_sz;
-    }
-
+  max_fifo = clib_min ((seg_sz - slices_sz) / 2, FIFO_SEGMENT_MAX_FIFO_SIZE);
+  fsh->max_log2_fifo_size = min_log2 (max_fifo);
   fsh->n_cached_bytes = 0;
   fsh->n_reserved_bytes = fsh->byte_index;
+  ASSERT (fsh->max_byte_index <= sh->ssvm_size - offset);
+
+  fs->max_byte_index = fsh->max_byte_index;
+  fs->h = sh->opaque[0] = fsh;
+
   sh->ready = 1;
   return (0);
 }
@@ -334,6 +314,7 @@ fifo_segment_create (fifo_segment_main_t * sm, fifo_segment_create_args_t * a)
 int
 fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a)
 {
+  fifo_segment_header_t *fsh;
   fifo_segment_t *fs;
   int rv;
 
@@ -355,8 +336,15 @@ fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a)
     }
 
   /* Fish the segment header */
-  fs->h = fs->ssvm.sh->opaque[0];
+  fsh = fs->h = fs->ssvm.sh->opaque[0];
+
+  /* Probably a segment without fifos */
+  if (!fsh)
+    goto done;
 
+  fs->max_byte_index = fsh->max_byte_index;
+
+done:
   vec_add1 (a->new_segment_indices, fs - sm->segments);
   return (0);
 }
@@ -401,7 +389,8 @@ fs_freelist_for_size (u32 size)
 {
   if (PREDICT_FALSE (size < FIFO_SEGMENT_MIN_FIFO_SIZE))
     return 0;
-  return max_log2 (size) - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE;
+  return clib_min (max_log2 (size) - FIFO_SEGMENT_MIN_LOG2_FIFO_SIZE,
+		   FS_CHUNK_VEC_LEN - 1);
 }
 
 static inline u32
@@ -417,8 +406,8 @@ fs_chunk_size_is_valid (fifo_segment_header_t * fsh, u32 size)
    * 4K minimum. It's not likely that anything good will happen
    * with a smaller FIFO.
    */
-  return size >= FIFO_SEGMENT_MIN_FIFO_SIZE
-    && size <= (1ULL << fsh->max_log2_chunk_size);
+  return size >= FIFO_SEGMENT_MIN_FIFO_SIZE &&
+	 size <= (1ULL << fsh->max_log2_fifo_size);
 }
 
 svm_fifo_chunk_t *
@@ -744,7 +733,7 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index,
 
   ASSERT (slice_index < fs->n_slices);
 
-  if (PREDICT_FALSE (data_bytes > 1 << fsh->max_log2_chunk_size))
+  if (PREDICT_FALSE (data_bytes > 1 << fsh->max_log2_fifo_size))
     return 0;
 
   fss = fsh_slice_get (fsh, slice_index);
@@ -1018,7 +1007,7 @@ fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size)
   /* Count all free chunks? */
   if (size == ~0)
     {
-      for (i = 0; i < fss->n_chunk_lens; i++)
+      for (i = 0; i < FS_CHUNK_VEC_LEN; i++)
 	{
 	  c = fss->free_chunks[i];
 	  if (c == 0)
@@ -1036,7 +1025,7 @@ fs_slice_num_free_chunks (fifo_segment_slice_t * fss, u32 size)
   rounded_size = (1 << (max_log2 (size)));
   fl_index = fs_freelist_for_size (rounded_size);
 
-  if (fl_index >= fss->n_chunk_lens)
+  if (fl_index >= FS_CHUNK_VEC_LEN)
     return 0;
 
   c = fss->free_chunks[fl_index];
@@ -1267,7 +1256,7 @@ format_fifo_segment (u8 * s, va_list * args)
   for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
     {
       fss = fsh_slice_get (fsh, slice_index);
-      for (i = 0; i < fss->n_chunk_lens; i++)
+      for (i = 0; i < FS_CHUNK_VEC_LEN; i++)
 	{
 	  c = fss->free_chunks[i];
 	  if (c == 0 && fss->num_chunks[i] == 0)
diff --git a/src/svm/fifo_types.h b/src/svm/fifo_types.h
index 9182a2ac3cf..e839e7eaab7 100644
--- a/src/svm/fifo_types.h
+++ b/src/svm/fifo_types.h
@@ -21,6 +21,14 @@
 #include <vppinfra/rbtree.h>
 #include <vppinfra/lock.h>
 
+#define FS_MIN_LOG2_CHUNK_SZ 12 /**< also min fifo size */
+#define FS_MAX_LOG2_CHUNK_SZ 22 /**< 4MB max chunk size */
+#define FS_CHUNK_VEC_LEN     11 /**< number of chunk sizes */
+
+STATIC_ASSERT ((FS_MAX_LOG2_CHUNK_SZ - FS_MIN_LOG2_CHUNK_SZ) ==
+		 FS_CHUNK_VEC_LEN - 1,
+	       "update chunk sizes");
+
 #define SVM_FIFO_TRACE 			(0)
 #define SVM_FIFO_MAX_EVT_SUBSCRIBERS	7
 
@@ -101,24 +109,23 @@ typedef struct _svm_fifo
 
 typedef struct fifo_segment_slice_
 {
+  svm_fifo_chunk_t *free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */
   svm_fifo_t *fifos;			/**< Linked list of active RX fifos */
   svm_fifo_t *free_fifos;		/**< Freelists by fifo size  */
-  svm_fifo_chunk_t **free_chunks;	/**< Freelists by chunk size */
-  u32 *num_chunks;			/**< Allocated chunks by chunk size */
   uword n_fl_chunk_bytes;		/**< Chunk bytes on freelist */
   uword virtual_mem;			/**< Slice sum of all fifo sizes */
-  u32 n_chunk_lens;
+  u32 num_chunks[FS_CHUNK_VEC_LEN];	/**< Allocated chunks by chunk size */
+
   CLIB_CACHE_LINE_ALIGN_MARK (lock);
   u32 chunk_lock;
 } fifo_segment_slice_t;
 
 struct fifo_segment_header_
 {
-  fifo_segment_slice_t *slices;		/** Fixed array of slices */
   uword n_cached_bytes;			/**< Cached bytes */
   u32 n_active_fifos;			/**< Number of active fifos */
   u32 n_reserved_bytes;			/**< Bytes not to be allocated */
-  u32 max_log2_chunk_size;		/**< Max log2(chunk size) for fs */
+  u32 max_log2_fifo_size;		/**< Max log2(chunk size) for fs */
   u8 flags;				/**< Segment flags */
   u8 n_slices;				/**< Number of slices */
   u8 high_watermark;			/**< Memory pressure watermark high */
@@ -127,6 +134,8 @@ struct fifo_segment_header_
   CLIB_CACHE_LINE_ALIGN_MARK (allocator);
   uword byte_index;
   uword max_byte_index;
+  CLIB_CACHE_LINE_ALIGN_MARK (slice);
+  fifo_segment_slice_t slices[0]; /** Fixed array of slices */
 };
 
 void fsh_virtual_mem_update (fifo_segment_header_t * fsh, u32 slice_index,
diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h
index 2a054526865..5656e63d998 100644
--- a/src/svm/svm_fifo.h
+++ b/src/svm/svm_fifo.h
@@ -712,7 +712,7 @@ svm_fifo_size (svm_fifo_t * f)
 static inline void
 svm_fifo_set_size (svm_fifo_t * f, u32 size)
 {
-  if (size > (1 << f->fs_hdr->max_log2_chunk_size))
+  if (size > (1 << f->fs_hdr->max_log2_fifo_size))
     return;
   fsh_virtual_mem_update (f->fs_hdr, f->slice_index, (int) f->size - size);
   f->size = size;