svm: fifo segment support for chunk allocation

Change-Id: Ie96706b4d8bcb32d2d5f065bc765f95f4e9369e7
Signed-off-by: Florin Coras <fcoras@cisco.com>

4 files changed, 217 insertions, 17 deletions

diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c
index 6bf2634a290..9c332d6f1fa 100644
--- a/src/svm/fifo_segment.c
+++ b/src/svm/fifo_segment.c
@@ -237,7 +237,7 @@ fs_free_list_for_size (u32 size)
 }
 
 static inline int
-fs_fifo_size_is_valid (u32 size)
+fs_chunk_size_is_valid (u32 size)
 {
   /*
    * 4K minimum. It's not likely that anything good will happen
@@ -293,7 +293,7 @@ fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes,
   void *oldheap;
   int fl_index;
 
-  if (!fs_fifo_size_is_valid (data_bytes))
+  if (!fs_chunk_size_is_valid (data_bytes))
     {
       clib_warning ("fifo size out of range %d", data_bytes);
       return 0;
@@ -304,7 +304,7 @@ fifo_segment_alloc_fifo (fifo_segment_t * fs, u32 data_bytes,
   sh = fs->ssvm.sh;
   ssvm_lock_non_recursive (sh, 1);
 
-  fsh = (fifo_segment_header_t *) sh->opaque[0];
+  fsh = fs->h;
   vec_validate_init_empty (fsh->free_fifos, fl_index, 0);
   f = fsh->free_fifos[fl_index];
 
@@ -363,7 +363,7 @@ done:
  * Free fifo allocated in fifo segment
  */
 void
-fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f)
+fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f)
 {
   fifo_segment_header_t *fsh;
   ssvm_shared_header_t *sh;
@@ -374,8 +374,8 @@ fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f)
   if (--f->refcnt > 0)
     return;
 
-  sh = s->ssvm.sh;
-  fsh = (fifo_segment_header_t *) sh->opaque[0];
+  sh = fs->ssvm.sh;
+  fsh = fs->h;
   fl_index = f->freelist_index;
 
   ASSERT (fl_index < vec_len (fsh->free_fifos));
@@ -399,6 +399,29 @@ fifo_segment_free_fifo (fifo_segment_t * s, svm_fifo_t * f)
   f->prev = 0;
   fsh->free_fifos[fl_index] = f;
 
+  /* If fifo has more chunks, free them */
+  if (f->flags & SVM_FIFO_F_MULTI_CHUNK)
+    {
+      svm_fifo_chunk_t *cur, *next;
+      void *oldheap;
+
+      next = f->start_chunk->next;
+      while (next != f->start_chunk)
+	{
+	  cur = next;
+	  next = next->next;
+	  fl_index = fs_free_list_for_size (cur->length);
+	  cur->next = fsh->free_chunks[fl_index];
+	  fsh->free_chunks[fl_index] = cur;
+	}
+      oldheap = ssvm_push_heap (sh);
+      svm_fifo_free_chunk_lookup (f);
+      ssvm_pop_heap (oldheap);
+    }
+
+  /* not allocated on segment heap */
+  svm_fifo_free_ooo_data (f);
+
   if (CLIB_DEBUG)
     {
       f->master_session_index = ~0;
@@ -431,13 +454,13 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs,
   if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0)
     return;
 
-  if (!fs_fifo_size_is_valid (rx_fifo_size))
+  if (!fs_chunk_size_is_valid (rx_fifo_size))
     {
       clib_warning ("rx fifo_size out of range %d", rx_fifo_size);
       return;
     }
 
-  if (!fs_fifo_size_is_valid (tx_fifo_size))
+  if (!fs_chunk_size_is_valid (tx_fifo_size))
     {
       clib_warning ("tx fifo_size out of range %d", tx_fifo_size);
       return;
@@ -502,6 +525,51 @@ fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs,
   ssvm_pop_heap (oldheap);
 }
 
+int
+fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f, u32 chunk_size)
+{
+  ssvm_shared_header_t *sh;
+  svm_fifo_chunk_t *c;
+  void *oldheap;
+  int fl_index;
+
+  if (!fs_chunk_size_is_valid (chunk_size))
+    {
+      clib_warning ("chunk size out of range %d", chunk_size);
+      return 0;
+    }
+
+  fl_index = fs_free_list_for_size (chunk_size);
+
+  sh = fs->ssvm.sh;
+  ssvm_lock_non_recursive (sh, 1);
+
+  vec_validate_init_empty (fs->h->free_chunks, fl_index, 0);
+  c = fs->h->free_chunks[fl_index];
+
+  oldheap = ssvm_push_heap (sh);
+
+  if (!c)
+    {
+      c = svm_fifo_chunk_alloc (chunk_size);
+      if (!c)
+	{
+	  ssvm_pop_heap (oldheap);
+	  return -1;
+	}
+    }
+  else
+    {
+      fs->h->free_chunks[fl_index] = c->next;
+    }
+
+  svm_fifo_add_chunk (f, c);
+
+  ssvm_pop_heap (oldheap);
+  ssvm_unlock_non_recursive (sh);
+  return 0;
+}
+
 /**
  * Get number of active fifos
  */
@@ -559,6 +627,52 @@ fifo_segment_num_free_fifos (fifo_segment_t * fs, u32 fifo_size_in_bytes)
   return count;
 }
 
+u32
+fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size)
+{
+  u32 count = 0, rounded_size, fl_index;
+  fifo_segment_header_t *fsh;
+  svm_fifo_chunk_t *c;
+  int i;
+
+  fsh = fs->h;
+
+  /* Count all free chunks? */
+  if (size == ~0)
+    {
+      for (i = 0; i < vec_len (fsh->free_chunks); i++)
+	{
+	  c = fsh->free_chunks[i];
+	  if (c == 0)
+	    continue;
+
+	  while (c)
+	    {
+	      c = c->next;
+	      count++;
+	    }
+	}
+      return count;
+    }
+
+  rounded_size = (1 << (max_log2 (size)));
+  fl_index = fs_free_list_for_size (rounded_size);
+
+  if (fl_index >= vec_len (fsh->free_chunks))
+    return 0;
+
+  c = fsh->free_chunks[fl_index];
+  if (c == 0)
+    return 0;
+
+  while (c)
+    {
+      c = c->next;
+      count++;
+    }
+  return count;
+}
+
 u8
 fifo_segment_has_fifos (fifo_segment_t * fs)
 {
diff --git a/src/svm/fifo_segment.h b/src/svm/fifo_segment.h
index cc0ff3adf3a..6ff538fcc07 100644
--- a/src/svm/fifo_segment.h
+++ b/src/svm/fifo_segment.h
@@ -38,10 +38,11 @@ typedef enum fifo_segment_flags_
 
 typedef struct
 {
-  svm_fifo_t *fifos;		/**< Linked list of active RX fifos */
-  svm_fifo_t **free_fifos;	/**< Freelists, by fifo size  */
-  u32 n_active_fifos;		/**< Number of active fifos */
-  u8 flags;			/**< Segment flags */
+  svm_fifo_t *fifos;			/**< Linked list of active RX fifos */
+  svm_fifo_t **free_fifos;		/**< Freelists by fifo size  */
+  svm_fifo_chunk_t **free_chunks;	/**< Freelists by chunk size */
+  u32 n_active_fifos;			/**< Number of active fifos */
+  u8 flags;				/**< Segment flags */
 } fifo_segment_header_t;
 
 typedef struct
@@ -82,7 +83,7 @@ void fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size);
 /**
  * Allocate fifo in fifo segment
  *
- * @param fs		fifo segment
+ * @param fs		fifo segment for fifo
  * @param data_bytes	size of default fifo chunk in bytes
  * @param ftype		fifo type @ref fifo_segment_ftype_t
  * @return		new fifo or 0 if alloc failed
@@ -94,7 +95,7 @@ svm_fifo_t *fifo_segment_alloc_fifo (fifo_segment_t * fs,
 /**
  * Free fifo allocated in fifo segment
  *
- * @param fs		fifo segment
+ * @param fs		fifo segment for fifo
  * @param f		fifo to be freed
  */
 void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f);
@@ -107,7 +108,7 @@ void fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f);
  * is hit, the number of fifo pairs requested is updated by subtracting the
  * number of fifos that have been successfully allocated.
  *
- * @param fs		fifo segment
+ * @param fs		fifo segment for fifo
  * @param rx_fifo_size	data size of rx fifos
  * @param tx_fifo_size	data size of tx fifos
  * @param n_fifo_pairs	number of pairs requested. Prior to returning, this
@@ -117,10 +118,28 @@ void fifo_segment_preallocate_fifo_pairs (fifo_segment_t * fs,
 					  u32 rx_fifo_size,
 					  u32 tx_fifo_size,
 					  u32 * n_fifo_pairs);
+/**
+ * Grow fifo size by adding an additional chunk of memory
+ *
+ * @param fs		fifo segment for fifo
+ * @param f		fifo to be grown
+ * @param chunk_size	number of bytes to be added to fifo
+ * @return		0 on success or a negative number otherwise
+ */
+int fifo_segment_grow_fifo (fifo_segment_t * fs, svm_fifo_t * f,
+			    u32 chunk_size);
 u8 fifo_segment_has_fifos (fifo_segment_t * fs);
 svm_fifo_t *fifo_segment_get_fifo_list (fifo_segment_t * fs);
 u32 fifo_segment_num_fifos (fifo_segment_t * fs);
 u32 fifo_segment_num_free_fifos (fifo_segment_t * fs, u32 fifo_size_in_bytes);
+/**
+ * Find number of free chunks of given size
+ *
+ * @param fs	fifo segment
+ * @param size	chunk size of interest or ~0 if all should be counted
+ * @return	number of chunks of given size
+ */
+u32 fifo_segment_num_free_chunks (fifo_segment_t * fs, u32 size);
 
 void fifo_segment_main_init (fifo_segment_main_t * sm, u64 baseva,
 			     u32 timeout_in_seconds);
diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index eafe497eec1..c8fd263c094 100644
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -298,7 +298,9 @@ svm_fifo_init (svm_fifo_t * f, u32 size)
   f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = f->start_chunk;
 }
 
-/** create an svm fifo, in the current heap. Fails vs blow up the process */
+/**
+ * Creates a fifo in the current heap. Fails vs blow up the process
+ */
 svm_fifo_t *
 svm_fifo_create (u32 data_size_in_bytes)
 {
@@ -317,6 +319,27 @@ svm_fifo_create (u32 data_size_in_bytes)
   return f;
 }
 
+/**
+ * Creates a fifo chunk in the current heap
+ */
+svm_fifo_chunk_t *
+svm_fifo_chunk_alloc (u32 size)
+{
+  svm_fifo_chunk_t *c;
+  u32 rounded_size;
+
+  /* round chunk size to the next highest power-of-two */
+  rounded_size = (1 << (max_log2 (size)));
+  c = clib_mem_alloc_aligned_or_null (sizeof (*c) + rounded_size,
+				      CLIB_CACHE_LINE_BYTES);
+  if (c == 0)
+    return 0;
+
+  clib_memset (c, 0, sizeof (*c));
+  c->length = rounded_size;
+  return c;
+}
+
 static inline void
 svm_fifo_size_update (svm_fifo_t * f, svm_fifo_chunk_t * c)
 {
@@ -462,13 +485,26 @@ svm_fifo_find_chunk (svm_fifo_t * f, u32 pos)
 }
 
 void
+svm_fifo_free_chunk_lookup (svm_fifo_t * f)
+{
+  rb_tree_free_nodes (&f->chunk_lookup);
+}
+
+void
+svm_fifo_free_ooo_data (svm_fifo_t * f)
+{
+  pool_free (f->ooo_segments);
+}
+
+void
 svm_fifo_free (svm_fifo_t * f)
 {
   ASSERT (f->refcnt > 0);
 
   if (--f->refcnt == 0)
     {
-      pool_free (f->ooo_segments);
+      /* ooo data is not allocated on segment heap */
+      svm_fifo_free_chunk_lookup (f);
       clib_mem_free (f);
     }
 }
diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h
index 4f2f61939b7..7c84fc82198 100644
--- a/src/svm/svm_fifo.h
+++ b/src/svm/svm_fifo.h
@@ -332,6 +332,19 @@ svm_fifo_unset_event (svm_fifo_t * f)
 
 svm_fifo_t *svm_fifo_create (u32 data_size_in_bytes);
 void svm_fifo_init (svm_fifo_t * f, u32 size);
+
+/**
+ * Allocate a fifo chunk on heap
+ *
+ * If the chunk is allocated on a fifo segment, this should be called
+ * with the segment's heap pushed.
+ *
+ * @param size	chunk size in bytes. Will be rounded to the next highest
+ * 		power-of-two
+ * @return	new chunk or 0 if alloc failed
+ */
+svm_fifo_chunk_t *svm_fifo_chunk_alloc (u32 size);
+
 /**
  * Grow fifo size by adding chunk to chunk list
  *
@@ -343,6 +356,24 @@ void svm_fifo_init (svm_fifo_t * f, u32 size);
  */
 void svm_fifo_add_chunk (svm_fifo_t * f, svm_fifo_chunk_t * c);
 void svm_fifo_free (svm_fifo_t * f);
+/**
+ * Cleanup fifo chunk lookup rb tree
+ *
+ * The rb tree is allocated in segment heap so this should be called
+ * with it pushed.
+ *
+ * @param f 	fifo to cleanup
+ */
+void svm_fifo_free_chunk_lookup (svm_fifo_t * f);
+/**
+ * Cleanup fifo ooo data
+ *
+ * The ooo data is allocated in producer process memory. The fifo
+ * segment heap should not be pushed.
+ *
+ * @param f	fifo to cleanup
+ */
+void svm_fifo_free_ooo_data (svm_fifo_t * f);
 
 int svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
 			     const u8 * copy_from_here);