1 files changed, 114 insertions, 44 deletions

diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c
index b4c3cbe6c6d..19fd0520666 100644
--- a/src/svm/fifo_segment.c
+++ b/src/svm/fifo_segment.c
@@ -14,6 +14,7 @@
  */
 
 #include <svm/fifo_segment.h>
+#include <vppinfra/mem.h>
 
 static inline void *
 fsh_alloc_aligned (fifo_segment_header_t *fsh, uword size, uword align)
@@ -239,7 +240,7 @@ fss_fl_chunk_bytes_sub (fifo_segment_slice_t * fss, uword size)
 int
 fifo_segment_init (fifo_segment_t * fs)
 {
-  u32 align = 8, offset = 2 * 4096, slices_sz;
+  u32 align = 8, offset = 2 * 4096, slices_sz, i;
   uword max_fifo, seg_start, seg_sz;
   fifo_segment_header_t *fsh;
   ssvm_shared_header_t *sh;
@@ -270,6 +271,11 @@ fifo_segment_init (fifo_segment_t * fs)
   fs->max_byte_index = fsh->max_byte_index;
   fs->h = sh->opaque[0] = fsh;
 
+  vec_validate (fs->slices, fs->n_slices - 1);
+  for (i = 0; i < fs->n_slices; i++)
+    fs->slices[i].fifos =
+      clib_mem_bulk_init (sizeof (svm_fifo_t), CLIB_CACHE_LINE_BYTES, 32);
+
   sh->ready = 1;
   return (0);
 }
@@ -343,6 +349,9 @@ fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a)
     goto done;
 
   fs->max_byte_index = fsh->max_byte_index;
+  vec_validate (fs->slices, 0);
+  fs->slices[0].fifos =
+    clib_mem_bulk_init (sizeof (svm_fifo_t), CLIB_CACHE_LINE_BYTES, 32);
 
 done:
   vec_add1 (a->new_segment_indices, fs - sm->segments);
@@ -352,6 +361,7 @@ done:
 void
 fifo_segment_delete (fifo_segment_main_t * sm, fifo_segment_t * s)
 {
+  fifo_segment_cleanup (s);
   ssvm_delete (&s->ssvm);
   clib_memset (s, 0xfe, sizeof (*s));
   pool_put (sm->segments, s);
@@ -478,7 +488,7 @@ static int
 fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh,
 			      fifo_segment_slice_t * fss, u32 batch_size)
 {
-  svm_fifo_t *f;
+  svm_fifo_shared_t *f;
   uword size;
   u8 *fmem;
   int i;
@@ -494,7 +504,7 @@ fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh,
   /* Carve fifo hdr space */
   for (i = 0; i < batch_size; i++)
     {
-      f = (svm_fifo_t *) fmem;
+      f = (svm_fifo_shared_t *) fmem;
       memset (f, 0, sizeof (*f));
       f->next = fss->free_fifos;
       fss->free_fifos = f;
@@ -555,11 +565,10 @@ fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh,
   return fsh_try_alloc_chunk_batch (fsh, fss, fl_index, batch_size);
 }
 
-static svm_fifo_t *
-fsh_try_alloc_fifo_hdr (fifo_segment_header_t * fsh,
-			fifo_segment_slice_t * fss)
+static svm_fifo_shared_t *
+fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
 {
-  svm_fifo_t *f;
+  svm_fifo_shared_t *f;
 
   if (!fss->free_fifos)
     {
@@ -642,40 +651,41 @@ done:
  * - single fifo allocation
  * - grab multiple fifo chunks from freelists
  */
-static svm_fifo_t *
-fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss,
-		   u32 data_bytes)
+static svm_fifo_shared_t *
+fs_try_alloc_fifo (fifo_segment_header_t *fsh, u32 slice_index, u32 data_bytes)
 {
+  fifo_segment_slice_t *fss;
   u32 fl_index, min_size;
   svm_fifo_chunk_t *c;
-  svm_fifo_t *f = 0;
+  svm_fifo_shared_t *sf = 0;
 
+  fss = fsh_slice_get (fsh, slice_index);
   min_size = clib_max ((fsh->pct_first_alloc * data_bytes) / 100, 4096);
   fl_index = fs_freelist_for_size (min_size);
 
   if (!fss_chunk_fl_index_is_valid (fss, fl_index))
     return 0;
 
-  f = fsh_try_alloc_fifo_hdr (fsh, fss);
-  if (!f)
+  sf = fsh_try_alloc_fifo_hdr (fsh, fss);
+  if (!sf)
     return 0;
 
   c = fsh_try_alloc_chunk (fsh, fss, min_size);
   if (!c)
     {
-      f->next = fss->free_fifos;
-      fss->free_fifos = f;
+      sf->next = fss->free_fifos;
+      fss->free_fifos = sf;
       return 0;
     }
 
-  f->start_chunk = c;
+  sf->start_chunk = c;
   while (c->next)
     c = c->next;
-  f->end_chunk = c;
-  f->size = data_bytes;
-  f->fs_hdr = fsh;
+  sf->end_chunk = c;
+  sf->size = data_bytes;
+  sf->slice_index = slice_index;
 
-  return f;
+  return sf;
 }
 
 svm_fifo_chunk_t *
@@ -720,6 +730,36 @@ fsh_collect_chunks (fifo_segment_header_t * fsh, u32 slice_index,
   fsh_slice_collect_chunks (fsh, fss, c);
 }
 
+svm_fifo_t *
+fs_fifo_alloc (fifo_segment_t *fs, u32 slice_index)
+{
+  fifo_slice_private_t *pfss = &fs->slices[slice_index];
+  svm_fifo_t *f;
+
+  f = clib_mem_bulk_alloc (pfss->fifos);
+  clib_memset (f, 0, sizeof (*f));
+  return f;
+}
+
+void
+fs_fifo_free (fifo_segment_t *fs, svm_fifo_t *f)
+{
+  u32 slice_index = f->shr->slice_index;
+  fifo_slice_private_t *pfss;
+
+  pfss = &fs->slices[slice_index];
+  clib_mem_bulk_free (pfss->fifos, f);
+}
+
+void
+fifo_segment_cleanup (fifo_segment_t *fs)
+{
+  int slice_index;
+
+  for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
+    clib_mem_bulk_destroy (fs->slices[slice_index].fifos);
+}
+
 /**
  * Allocate fifo in fifo segment
  */
@@ -729,6 +769,7 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index,
 {
   fifo_segment_header_t *fsh = fs->h;
   fifo_segment_slice_t *fss;
+  svm_fifo_shared_t *sf;
   svm_fifo_t *f = 0;
 
   ASSERT (slice_index < fs->n_slices);
@@ -736,15 +777,18 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index,
   if (PREDICT_FALSE (data_bytes > 1 << fsh->max_log2_fifo_size))
     return 0;
 
-  fss = fsh_slice_get (fsh, slice_index);
-  f = fs_try_alloc_fifo (fsh, fss, data_bytes);
-  if (!f)
+  sf = fs_try_alloc_fifo (fsh, slice_index, data_bytes);
+  if (!sf)
     goto done;
 
-  f->slice_index = slice_index;
+  f = fs_fifo_alloc (fs, slice_index);
+  f->fs_hdr = fsh;
+  f->shr = sf;
 
   svm_fifo_init (f, data_bytes);
 
+  fss = fsh_slice_get (fsh, slice_index);
+
   /* If rx fifo type add to active fifos list. When cleaning up segment,
    * we need a list of active sessions that should be disconnected. Since
    * both rx and tx fifos keep pointers to the session, it's enough to track
@@ -762,6 +806,19 @@ done:
   return (f);
 }
 
+svm_fifo_t *
+fifo_segment_alloc_fifo_w_shared (fifo_segment_t *fs, svm_fifo_shared_t *sf)
+{
+  svm_fifo_t *f = fs_fifo_alloc (fs, 0);
+  f->fs_hdr = fs->h;
+  f->shr = sf;
+
+  f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
+  f->segment_index = SVM_FIFO_INVALID_INDEX;
+  f->refcnt = 1;
+  return f;
+}
+
 /**
  * Free fifo allocated in fifo segment
  */
@@ -770,13 +827,35 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f)
 {
   fifo_segment_header_t *fsh = fs->h;
   fifo_segment_slice_t *fss;
+  svm_fifo_shared_t *sf;
 
   ASSERT (f->refcnt > 0);
 
   if (--f->refcnt > 0)
     return;
 
-  fss = fsh_slice_get (fsh, f->slice_index);
+  /*
+   * Cleanup shared state
+   */
+
+  sf = f->shr;
+  fss = fsh_slice_get (fsh, sf->slice_index);
+
+  /* Free fifo chunks */
+  fsh_slice_collect_chunks (fsh, fss, sf->start_chunk);
+
+  sf->start_chunk = sf->end_chunk = 0;
+  sf->head_chunk = sf->tail_chunk = 0;
+
+  /* Add to free list */
+  sf->next = fss->free_fifos;
+  fss->free_fifos = sf;
+
+  fss->virtual_mem -= svm_fifo_size (f);
+
+  /*
+   *  Cleanup private state
+   */
 
   /* Remove from active list. Only rx fifos are tracked */
   if (f->flags & SVM_FIFO_F_LL_TRACKED)
@@ -785,28 +864,19 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f)
       f->flags &= ~SVM_FIFO_F_LL_TRACKED;
     }
 
-  /* Free fifo chunks */
-  fsh_slice_collect_chunks (fsh, fss, f->start_chunk);
-
-  f->start_chunk = f->end_chunk = 0;
-  f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0;
-
-  /* not allocated on segment heap */
   svm_fifo_free_chunk_lookup (f);
   svm_fifo_free_ooo_data (f);
 
   if (CLIB_DEBUG)
     {
-      f->master_session_index = ~0;
+      sf->master_session_index = ~0;
       f->master_thread_index = ~0;
     }
 
-  fss->virtual_mem -= svm_fifo_size (f);
-
-  /* Add to free list */
-  f->next = fss->free_fifos;
+  f->ooo_enq = f->ooo_deq = 0;
   f->prev = 0;
-  fss->free_fifos = f;
+
+  fs_fifo_free (fs, f);
 
   fsh_active_fifos_update (fsh, -1);
 }
@@ -820,12 +890,12 @@ fifo_segment_detach_fifo (fifo_segment_t * fs, svm_fifo_t * f)
 
   ASSERT (f->refcnt == 1);
 
-  fss = fsh_slice_get (fs->h, f->slice_index);
+  fss = fsh_slice_get (fs->h, f->shr->slice_index);
   fss->virtual_mem -= svm_fifo_size (f);
   if (f->flags & SVM_FIFO_F_LL_TRACKED)
     fss_fifo_del_active_list (fss, f);
 
-  c = f->start_chunk;
+  c = f->shr->start_chunk;
   while (c)
     {
       fl_index = fs_freelist_for_size (c->length);
@@ -842,13 +912,13 @@ fifo_segment_attach_fifo (fifo_segment_t * fs, svm_fifo_t * f,
   svm_fifo_chunk_t *c;
   u32 fl_index;
 
-  f->slice_index = slice_index;
-  fss = fsh_slice_get (fs->h, f->slice_index);
+  f->shr->slice_index = slice_index;
+  fss = fsh_slice_get (fs->h, f->shr->slice_index);
   fss->virtual_mem += svm_fifo_size (f);
   if (f->flags & SVM_FIFO_F_LL_TRACKED)
     fss_fifo_add_active_list (fss, f);
 
-  c = f->start_chunk;
+  c = f->shr->start_chunk;
   while (c)
     {
       fl_index = fs_freelist_for_size (c->length);
@@ -966,7 +1036,7 @@ fifo_segment_num_fifos (fifo_segment_t * fs)
 static u32
 fs_slice_num_free_fifos (fifo_segment_slice_t * fss)
 {
-  svm_fifo_t *f;
+  svm_fifo_shared_t *f;
   u32 count = 0;
 
   f = fss->free_fifos;