svm: allow indirect fifo data chunks

Fifos can use multiple memory chunks for simple read/write operations.
Adding/removing chunks after assignment not yet supported.

Change-Id: I2aceab6aea78059d74e0d3a9993c40d5196d077b
Signed-off-by: Florin Coras <fcoras@cisco.com>

3 files changed, 167 insertions, 165 deletions

diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index 918ad77956e..de1997452d5 100644
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -113,8 +113,7 @@ svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose)
 #endif
 
   dummy_fifo = svm_fifo_create (f->size);
-  clib_memset (f->data, 0xFF, f->nitems);
-
+  clib_memset (f->head_chunk->data, 0xFF, f->nitems);
   vec_validate (data, f->nitems);
   for (i = 0; i < vec_len (data); i++)
     data[i] = i;
@@ -204,6 +203,26 @@ format_svm_fifo (u8 * s, va_list * args)
   return s;
 }
 
+void
+svm_fifo_init (svm_fifo_t * f, u32 size)
+{
+  f->size = size;
+  /*
+   * usable size of the fifo set to rounded_data_size - 1
+   * to differentiate between free fifo and empty fifo.
+   */
+  f->nitems = f->size - 1;
+  f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
+  f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
+  f->segment_index = SVM_FIFO_INVALID_INDEX;
+  f->refcnt = 1;
+  f->head_chunk = &f->default_chunk;
+  f->tail_chunk = &f->default_chunk;
+  f->default_chunk.next = &f->default_chunk;
+  f->default_chunk.start_byte = 0;
+  f->default_chunk.length = f->size;
+}
+
 /** create an svm fifo, in the current heap. Fails vs blow up the process */
 svm_fifo_t *
 svm_fifo_create (u32 data_size_in_bytes)
@@ -219,17 +238,8 @@ svm_fifo_create (u32 data_size_in_bytes)
     return 0;
 
   clib_memset (f, 0, sizeof (*f));
-  f->size = rounded_data_size;
-  /*
-   * usable size of the fifo set to rounded_data_size - 1
-   * to differentiate between free fifo and empty fifo.
-   */
-  f->nitems = f->size - 1;
-  f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
-  f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
-  f->segment_index = SVM_FIFO_INVALID_INDEX;
-  f->refcnt = 1;
-  return (f);
+  svm_fifo_init (f, data_size_in_bytes);
+  return f;
 }
 
 void
@@ -296,7 +306,9 @@ ooo_segment_add (svm_fifo_t * f, u32 offset, u32 head, u32 tail, u32 length)
   u32 new_index, s_end_pos, s_index;
   u32 offset_pos, offset_end_pos;
 
-  ASSERT (offset + length <= ooo_segment_distance_from_tail (f, head, tail));
+  ASSERT (offset + length <= ooo_segment_distance_from_tail (f, head, tail)
+	  || head == tail);
+
   offset_pos = tail + offset;
   offset_end_pos = tail + offset + length;
 
@@ -469,11 +481,11 @@ ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued, u32 * tail)
   return bytes;
 }
 
-CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
-	       const u8 * copy_from_here)
+CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 len,
+	       const u8 * src)
 {
-  u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
-  u32 tail, head, free_count, tail_idx;
+  u32 n_chunk, to_copy, tail, head, free_count, tail_idx;
+  svm_fifo_chunk_t *c;
 
   f_load_head_tail_prod (f, &head, &tail);
 
@@ -486,50 +498,43 @@ CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
     return SVM_FIFO_FULL;
 
   /* number of bytes we're going to copy */
-  total_copy_bytes = free_count < max_bytes ? free_count : max_bytes;
+  to_copy = len = clib_min (free_count, len);
 
+  c = f->tail_chunk;
   tail_idx = tail % f->size;
+  ASSERT (tail_idx >= c->start_byte);
+  tail_idx -= c->start_byte;
+  n_chunk = c->length - tail_idx;
 
-  if (PREDICT_TRUE (copy_from_here != 0))
+  if (n_chunk < to_copy)
     {
-      first_copy_bytes = f->size - tail_idx;
-      if (first_copy_bytes < total_copy_bytes)
-	{
-	  clib_memcpy_fast (&f->data[tail_idx], copy_from_here,
-			    first_copy_bytes);
-	  /* number of bytes in second copy segment */
-	  second_copy_bytes = total_copy_bytes - first_copy_bytes;
-	  /* wrap around */
-	  clib_memcpy_fast (&f->data[0],
-			    copy_from_here + first_copy_bytes,
-			    second_copy_bytes);
-	}
-      else
+      clib_memcpy_fast (&c->data[tail_idx], src, n_chunk);
+      while ((to_copy -= n_chunk))
 	{
-	  clib_memcpy_fast (&f->data[tail_idx], copy_from_here,
-			    total_copy_bytes);
+	  c = c->next;
+	  n_chunk = clib_min (c->length, to_copy);
+	  clib_memcpy_fast (&c->data[0], src + (len - to_copy), n_chunk);
 	}
-      tail += total_copy_bytes;
+      f->tail_chunk = c;
     }
   else
     {
-      ASSERT (0);
-      /* Account for a zero-copy enqueue done elsewhere */
-      tail += max_bytes;
+      clib_memcpy_fast (&c->data[tail_idx], src, to_copy);
     }
+  tail += len;
 
-  svm_fifo_trace_add (f, head, total_copy_bytes, 2);
+  svm_fifo_trace_add (f, head, n_total, 2);
 
   /* collect out-of-order segments */
   if (PREDICT_FALSE (f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX))
-    total_copy_bytes += ooo_segment_try_collect (f, total_copy_bytes, &tail);
+    len += ooo_segment_try_collect (f, len, &tail);
 
-  ASSERT (total_copy_bytes <= free_count);
+  ASSERT (len <= free_count);
 
   /* store-rel: producer owned index (paired with load-acq in consumer) */
   clib_atomic_store_rel_n (&f->tail, tail);
 
-  return total_copy_bytes;
+  return len;
 }
 
 #ifndef CLIB_MARCH_VARIANT
@@ -550,11 +555,10 @@ svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
  * Returns -1 if none of the segment was copied due to lack of space
  */
 CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f,
-	       u32 offset, u32 required_bytes, u8 * copy_from_here)
+	       u32 offset, u32 len, u8 * src)
 {
-  u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
-  u32 tail_offset;
-  u32 tail, head, free_count, tail_offset_idx;
+  u32 to_copy, n_chunk, tail, head, free_count, tail_offset_idx;
+  svm_fifo_chunk_t *c;
 
   f_load_head_tail_prod (f, &head, &tail);
 
@@ -562,41 +566,35 @@ CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f,
   free_count = f_free_count (f, head, tail);
 
   /* will this request fit? */
-  if ((required_bytes + offset) > free_count)
+  if ((len + offset) > free_count)
     return -1;
 
   f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
 
-  ASSERT (required_bytes < f->nitems);
+  ASSERT (len < f->nitems);
+  svm_fifo_trace_add (f, offset, len, 1);
 
-  tail_offset = tail + offset;
-  tail_offset_idx = tail_offset % f->size;
+  ooo_segment_add (f, offset, head, tail, len);
 
-  svm_fifo_trace_add (f, offset, required_bytes, 1);
+  c = f->tail_chunk;
+  tail_offset_idx = (tail + offset) % f->size;
+  tail_offset_idx -= c->start_byte;
+  n_chunk = c->length - tail_offset_idx;
+  to_copy = len;
 
-  ooo_segment_add (f, offset, head, tail, required_bytes);
-
-  /* number of bytes we're going to copy */
-  total_copy_bytes = required_bytes;
-
-  /* number of bytes in first copy segment */
-  first_copy_bytes = f->size - tail_offset_idx;
-
-  if (first_copy_bytes < total_copy_bytes)
+  if (n_chunk < to_copy)
     {
-      clib_memcpy_fast (&f->data[tail_offset_idx], copy_from_here,
-			first_copy_bytes);
-
-      /* number of bytes in second copy segment */
-      second_copy_bytes = total_copy_bytes - first_copy_bytes;
-      /* wrap around */
-      clib_memcpy_fast (&f->data[0],
-			copy_from_here + first_copy_bytes, second_copy_bytes);
+      clib_memcpy_fast (&c->data[tail_offset_idx], src, n_chunk);
+      while ((to_copy -= n_chunk))
+	{
+	  c = c->next;
+	  n_chunk = clib_min (c->length, to_copy);
+	  clib_memcpy_fast (&c->data[0], src + (len - to_copy), n_chunk);
+	}
     }
   else
     {
-      clib_memcpy_fast (&f->data[tail_offset_idx], copy_from_here,
-			total_copy_bytes);
+      clib_memcpy_fast (&c->data[tail_offset_idx], src, len);
     }
 
   return 0;
@@ -616,28 +614,32 @@ svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 required_bytes,
 void
 svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len)
 {
-  u32 first_chunk;
+  u32 n_chunk;
   u32 head, tail, head_idx;
+  svm_fifo_chunk_t *c;
+
+  ASSERT (len <= f->nitems);
 
   f_load_head_tail_cons (f, &head, &tail);
+  c = f->head_chunk;
   head_idx = head % f->size;
-  first_chunk = f->size - (head_idx);
-  ASSERT (len <= f->nitems);
-  if (len <= first_chunk)
-    clib_memcpy_fast (&f->data[head_idx], data, len);
+  head_idx -= c->start_byte;
+  n_chunk = c->length - head_idx;
+  if (len <= n_chunk)
+    clib_memcpy_fast (&c->data[head_idx], data, len);
   else
     {
-      clib_memcpy_fast (&f->data[head_idx], data, first_chunk);
-      clib_memcpy_fast (&f->data[0], data + first_chunk, len - first_chunk);
+      clib_memcpy_fast (&c->data[head_idx], data, n_chunk);
+      clib_memcpy_fast (&c->next->data[0], data + n_chunk, len - n_chunk);
     }
 }
 #endif
 
-CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
-	       u8 * copy_here)
+CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 len,
+	       u8 * dst)
 {
-  u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
-  u32 tail, head, cursize, head_idx;
+  u32 to_copy, n_chunk, tail, head, cursize, head_idx;
+  svm_fifo_chunk_t *c;
 
   f_load_head_tail_cons (f, &head, &tail);
 
@@ -647,42 +649,35 @@ CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
   if (PREDICT_FALSE (cursize == 0))
     return -2;			/* nothing in the fifo */
 
-  /* number of bytes we're going to copy */
-  total_copy_bytes = (cursize < max_bytes) ? cursize : max_bytes;
+  to_copy = len = clib_min (cursize, len);
 
+  c = f->head_chunk;
   head_idx = head % f->size;
+  head_idx -= c->start_byte;
+  n_chunk = c->length - head_idx;
 
-  if (PREDICT_TRUE (copy_here != 0))
+  if (n_chunk < to_copy)
     {
-      /* number of bytes in first copy segment */
-      first_copy_bytes = f->size - head_idx;
-      if (first_copy_bytes < total_copy_bytes)
+      clib_memcpy_fast (dst, &c->data[head_idx], n_chunk);
+      while ((to_copy -= n_chunk))
 	{
-	  clib_memcpy_fast (copy_here, &f->data[head_idx], first_copy_bytes);
-	  /* number of bytes in second copy segment */
-	  second_copy_bytes = total_copy_bytes - first_copy_bytes;
-	  /* wrap around */
-	  clib_memcpy_fast (copy_here + first_copy_bytes,
-			    &f->data[0], second_copy_bytes);
+	  c = c->next;
+	  n_chunk = clib_min (c->length, to_copy);
+	  clib_memcpy_fast (dst + (len - to_copy), &c->data[0], n_chunk);
 	}
-      else
-	{
-	  clib_memcpy_fast (copy_here, &f->data[head_idx], total_copy_bytes);
-	}
-      head += total_copy_bytes;
+      f->head_chunk = c;
     }
   else
     {
-      ASSERT (0);
-      /* Account for a zero-copy dequeue done elsewhere */
-      head += max_bytes;
+      clib_memcpy_fast (dst, &c->data[head_idx], to_copy);
     }
+  head += len;
 
-  ASSERT (cursize >= total_copy_bytes);
+  ASSERT (cursize >= to_copy);
   /* store-rel: consumer owned index (paired with load-acq in producer) */
   clib_atomic_store_rel_n (&f->head, head);
 
-  return total_copy_bytes;
+  return len;
 }
 
 #ifndef CLIB_MARCH_VARIANT
@@ -696,11 +691,10 @@ svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
 #endif
 
 CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset,
-	       u32 max_bytes, u8 * copy_here)
+	       u32 len, u8 * dst)
 {
-  u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
-  u32 tail, head, cursize;
-  u32 relative_head_idx;
+  u32 to_copy, n_chunk, tail, head, cursize, head_idx;
+  svm_fifo_chunk_t *c;
 
   f_load_head_tail_cons (f, &head, &tail);
 
@@ -710,33 +704,29 @@ CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset,
   if (PREDICT_FALSE (cursize < relative_offset))
     return -2;			/* nothing in the fifo */
 
-  relative_head_idx = (head + relative_offset) % f->size;
+  to_copy = len = clib_min (cursize - relative_offset, len);
 
-  /* number of bytes we're going to copy */
-  total_copy_bytes = ((cursize - relative_offset) < max_bytes) ?
-    cursize - relative_offset : max_bytes;
+  c = f->head_chunk;
+  head_idx = (head + relative_offset) % f->size;
+  head_idx -= c->start_byte;
+  n_chunk = c->length - head_idx;
 
-  if (PREDICT_TRUE (copy_here != 0))
+  if (n_chunk < to_copy)
     {
-      /* number of bytes in first copy segment */
-      first_copy_bytes = f->size - relative_head_idx;
-      if (first_copy_bytes < total_copy_bytes)
-	{
-	  clib_memcpy_fast (copy_here, &f->data[relative_head_idx],
-			    first_copy_bytes);
-
-	  /* number of bytes in second copy segment */
-	  second_copy_bytes = total_copy_bytes - first_copy_bytes;
-	  clib_memcpy_fast (copy_here + first_copy_bytes, &f->data[0],
-			    second_copy_bytes);
-	}
-      else
+      clib_memcpy_fast (dst, &c->data[head_idx], n_chunk);
+      while ((to_copy -= n_chunk))
 	{
-	  clib_memcpy_fast (copy_here, &f->data[relative_head_idx],
-			    total_copy_bytes);
+	  c = c->next;
+	  n_chunk = clib_min (c->length, to_copy);
+	  clib_memcpy_fast (dst + (len - to_copy), &c->data[0], n_chunk);
 	}
+      f->head_chunk = c;
+    }
+  else
+    {
+      clib_memcpy_fast (dst, &c->data[head_idx], to_copy);
     }
-  return total_copy_bytes;
+  return len;
 }
 
 #ifndef CLIB_MARCH_VARIANT
@@ -805,14 +795,14 @@ svm_fifo_segments (svm_fifo_t * f, svm_fifo_segment_t * fs)
   if (tail < head)
     {
       fs[0].len = f->size - head_idx;
-      fs[0].data = f->data + head_idx;
+      fs[0].data = f->head_chunk->data + head_idx;
       fs[1].len = cursize - fs[0].len;
-      fs[1].data = f->data;
+      fs[1].data = f->head_chunk->data;
     }
   else
     {
       fs[0].len = cursize;
-      fs[0].data = f->data + head_idx;
+      fs[0].data = f->head_chunk->data + head_idx;
       fs[1].len = 0;
       fs[1].data = 0;
     }
@@ -828,7 +818,7 @@ svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs)
   head = f->head;
   head_idx = head % f->size;
 
-  ASSERT (fs[0].data == f->data + head_idx);
+  ASSERT (fs[0].data == f->head_chunk->data + head_idx);
   head += fs[0].len + fs[1].len;
   /* store-rel: consumer owned index (paired with load-acq in producer) */
   clib_atomic_store_rel_n (&f->head, head);
@@ -839,7 +829,7 @@ void
 svm_fifo_clone (svm_fifo_t * df, svm_fifo_t * sf)
 {
   u32 head, tail;
-  clib_memcpy_fast (df->data, sf->data, sf->size);
+  clib_memcpy_fast (df->head_chunk->data, sf->head_chunk->data, sf->size);
 
   f_load_head_tail_all_acq (sf, &head, &tail);
   clib_atomic_store_rel_n (&df->head, head);
diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h
index 94a05938d26..f311576f5af 100644
--- a/src/svm/svm_fifo.h
+++ b/src/svm/svm_fifo.h
@@ -60,11 +60,21 @@ typedef struct
   u32 action;
 } svm_fifo_trace_elem_t;
 
+typedef struct svm_fifo_chunk_
+{
+  u32 start_byte;
+  u32 length;
+  struct svm_fifo_chunk_ *next;
+  u8 data[0];
+} svm_fifo_chunk_t;
+
 typedef struct _svm_fifo
 {
   CLIB_CACHE_LINE_ALIGN_MARK (shared_first);
-  u32 size;			/**< size of the fifo(must be power of 2) */
+  u32 size;			/**< size of the fifo */
   u32 nitems;			/**< usable size(size-1) */
+  struct _svm_fifo *next;	/**< next in freelist/active chain */
+  struct _svm_fifo *prev;	/**< prev in active chain */
 
     CLIB_CACHE_LINE_ALIGN_MARK (shared_second);
   volatile u32 has_event;	/**< non-zero if deq event exists */
@@ -81,17 +91,17 @@ typedef struct _svm_fifo
 
     CLIB_CACHE_LINE_ALIGN_MARK (consumer);
   u32 head;
+  svm_fifo_chunk_t *head_chunk;
   volatile u32 want_tx_ntf;	/**< producer wants nudge */
   volatile u32 has_tx_ntf;
 
     CLIB_CACHE_LINE_ALIGN_MARK (producer);
   u32 tail;
+  svm_fifo_chunk_t *tail_chunk;
 
   ooo_segment_t *ooo_segments;	/**< Pool of ooo segments */
   u32 ooos_list_head;		/**< Head of out-of-order linked-list */
   u32 ooos_newest;		/**< Last segment to have been updated */
-  struct _svm_fifo *next;	/**< next in freelist/active chain */
-  struct _svm_fifo *prev;	/**< prev in active chain */
   volatile u8 n_subscribers;
   u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS];
 
@@ -99,7 +109,7 @@ typedef struct _svm_fifo
   svm_fifo_trace_elem_t *trace;
 #endif
 
-    CLIB_CACHE_LINE_ALIGN_MARK (data);
+  svm_fifo_chunk_t default_chunk;
 } svm_fifo_t;
 
 typedef enum
@@ -305,6 +315,7 @@ svm_fifo_unset_event (svm_fifo_t * f)
 }
 
 svm_fifo_t *svm_fifo_create (u32 data_size_in_bytes);
+void svm_fifo_init (svm_fifo_t * f, u32 size);
 void svm_fifo_free (svm_fifo_t * f);
 
 int svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
@@ -373,14 +384,16 @@ always_inline u8 *
 svm_fifo_head (svm_fifo_t * f)
 {
   /* load-relaxed: consumer owned index */
-  return (f->data + (f->head % f->size));
+  return (f->head_chunk->data
+	  + ((f->head % f->size) - f->head_chunk->start_byte));
 }
 
 always_inline u8 *
 svm_fifo_tail (svm_fifo_t * f)
 {
   /* load-relaxed: producer owned index */
-  return (f->data + (f->tail % f->size));
+  return (f->tail_chunk->data
+	  + ((f->tail % f->size) - f->tail_chunk->start_byte));
 }
 
 static inline void
diff --git a/src/svm/svm_fifo_segment.c b/src/svm/svm_fifo_segment.c
index 94bf6a5e73e..90bf593f74c 100644
--- a/src/svm/svm_fifo_segment.c
+++ b/src/svm/svm_fifo_segment.c
@@ -16,6 +16,23 @@
 #include <svm/svm_fifo_segment.h>
 
 static void
+fifo_init_for_segment (svm_fifo_segment_header_t * fsh, u8 * fifo_space,
+		       u32 size, u32 freelist_index)
+{
+  svm_fifo_t *f;
+
+  f = (svm_fifo_t *) fifo_space;
+  f->freelist_index = freelist_index;
+  f->default_chunk.next = &f->default_chunk;
+  f->default_chunk.start_byte = 0;
+  f->default_chunk.length = size;
+  f->head_chunk = &f->default_chunk;
+  f->tail_chunk = &f->default_chunk;
+  f->next = fsh->free_fifos[freelist_index];
+  fsh->free_fifos[freelist_index] = f;
+}
+
+static void
 allocate_new_fifo_chunk (svm_fifo_segment_header_t * fsh,
 			 u32 data_size_in_bytes, int chunk_size)
 {
@@ -43,14 +60,11 @@ allocate_new_fifo_chunk (svm_fifo_segment_header_t * fsh,
     return;
 
   /* Carve fifo space */
-  f = (svm_fifo_t *) fifo_space;
   for (i = 0; i < chunk_size; i++)
     {
-      f->freelist_index = freelist_index;
-      f->next = fsh->free_fifos[freelist_index];
-      fsh->free_fifos[freelist_index] = f;
+      fifo_init_for_segment (fsh, fifo_space, rounded_data_size,
+			     freelist_index);
       fifo_space += sizeof (*f) + rounded_data_size;
-      f = (svm_fifo_t *) fifo_space;
     }
 }
 
@@ -149,24 +163,18 @@ svm_fifo_segment_preallocate_fifo_pairs (svm_fifo_segment_private_t * s,
     }
 
   /* Carve rx fifo space */
-  f = (svm_fifo_t *) rx_fifo_space;
   for (i = 0; i < pairs_to_allocate; i++)
     {
-      f->freelist_index = rx_freelist_index;
-      f->next = fsh->free_fifos[rx_freelist_index];
-      fsh->free_fifos[rx_freelist_index] = f;
+      fifo_init_for_segment (fsh, rx_fifo_space, rx_rounded_data_size,
+			     rx_freelist_index);
       rx_fifo_space += sizeof (*f) + rx_rounded_data_size;
-      f = (svm_fifo_t *) rx_fifo_space;
     }
   /* Carve tx fifo space */
-  f = (svm_fifo_t *) tx_fifo_space;
   for (i = 0; i < pairs_to_allocate; i++)
     {
-      f->freelist_index = tx_freelist_index;
-      f->next = fsh->free_fifos[tx_freelist_index];
-      fsh->free_fifos[tx_freelist_index] = f;
+      fifo_init_for_segment (fsh, tx_fifo_space, tx_rounded_data_size,
+			     tx_freelist_index);
       tx_fifo_space += sizeof (*f) + tx_rounded_data_size;
-      f = (svm_fifo_t *) tx_fifo_space;
     }
 
   /* Account for the pairs allocated */
@@ -381,16 +389,7 @@ svm_fifo_segment_alloc_fifo (svm_fifo_segment_private_t * fs,
 	  fsh->free_fifos[freelist_index] = f->next;
 	  /* (re)initialize the fifo, as in svm_fifo_create */
 	  clib_memset (f, 0, sizeof (*f));
-	  f->size = (1 << (max_log2 (data_size_in_bytes)));
-	  /*
-	   * usable size of the fifo set to rounded_data_size - 1
-	   * to differentiate between free fifo and empty fifo.
-	   */
-	  f->nitems = f->size - 1;
-	  f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
-	  f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
-	  f->refcnt = 1;
-	  f->freelist_index = freelist_index;
+	  svm_fifo_init (f, data_size_in_bytes);
 	  goto found;
 	}
       break;