svm: fifo segment sptr for fifo hdr free list

With this there are no more pointers in data structures allocated on
fifo segments.

Type: improvement

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: Ibe584b7b6809fa360a105974655a91674db69ab6

4 files changed, 88 insertions, 49 deletions

diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c
index 4ffec7c86d0..fdcf84ea8d7 100644
--- a/src/svm/fifo_segment.c
+++ b/src/svm/fifo_segment.c
@@ -200,6 +200,32 @@ fss_chunk_free_list_pop (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss,
   return c;
 }
 
+static void
+fss_fifo_free_list_push (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss,
+			 svm_fifo_shared_t *sf)
+{
+  sf->next = fss->free_fifos;
+  fss->free_fifos = fs_sptr (fsh, sf);
+}
+
+static void
+fss_fifo_free_list_push_list (fifo_segment_header_t *fsh,
+			      fifo_segment_slice_t *fss,
+			      svm_fifo_shared_t *head, svm_fifo_shared_t *tail)
+{
+  tail->next = fss->free_fifos;
+  fss->free_fifos = fs_sptr (fsh, head);
+}
+
+svm_fifo_shared_t *
+fss_fifo_free_list_pop (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
+{
+  svm_fifo_shared_t *sf;
+  sf = fs_ptr (fsh, fss->free_fifos);
+  fss->free_fifos = sf->next;
+  return sf;
+}
+
 static inline void
 pfss_fifo_add_active_list (fifo_slice_private_t *pfss, svm_fifo_t *f)
 {
@@ -499,7 +525,7 @@ static int
 fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh,
 			      fifo_segment_slice_t * fss, u32 batch_size)
 {
-  svm_fifo_shared_t *f;
+  svm_fifo_shared_t *f, *head = 0, *tail;
   uword size;
   u8 *fmem;
   int i;
@@ -513,15 +539,18 @@ fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh,
     return -1;
 
   /* Carve fifo hdr space */
+  tail = f = (svm_fifo_shared_t *) fmem;
   for (i = 0; i < batch_size; i++)
     {
-      f = (svm_fifo_shared_t *) fmem;
-      memset (f, 0, sizeof (*f));
-      f->next = fss->free_fifos;
-      fss->free_fifos = f;
+      clib_memset (f, 0, sizeof (*f));
+      f->next = fs_sptr (fsh, head);
+      head = f;
       fmem += sizeof (*f);
+      f = (svm_fifo_shared_t *) fmem;
     }
 
+  fss_fifo_free_list_push_list (fsh, fss, head, tail);
+
   return 0;
 }
 
@@ -579,7 +608,7 @@ fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh,
 static svm_fifo_shared_t *
 fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
 {
-  svm_fifo_shared_t *f;
+  svm_fifo_shared_t *sf;
 
   if (!fss->free_fifos)
     {
@@ -588,10 +617,10 @@ fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
 	return 0;
     }
 
-  f = fss->free_fifos;
-  fss->free_fifos = f->next;
-  memset (f, 0, sizeof (*f));
-  return f;
+  sf = fss_fifo_free_list_pop (fsh, fss);
+  clib_memset (sf, 0, sizeof (*sf));
+
+  return sf;
 }
 
 static svm_fifo_chunk_t *
@@ -684,8 +713,7 @@ fs_try_alloc_fifo (fifo_segment_header_t *fsh, u32 slice_index, u32 data_bytes)
   c = fsh_try_alloc_chunk (fsh, fss, min_size);
   if (!c)
     {
-      sf->next = fss->free_fifos;
-      fss->free_fifos = sf;
+      fss_fifo_free_list_push (fsh, fss, sf);
       return 0;
     }
 
@@ -872,8 +900,9 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f)
   sf->head_chunk = sf->tail_chunk = 0;
 
   /* Add to free list */
-  sf->next = fss->free_fifos;
-  fss->free_fifos = sf;
+  fss_fifo_free_list_push (fsh, fss, sf);
+  //  sf->next = fss->free_fifos;
+  //  fss->free_fifos = fs_sptr (fsh, sf);
 
   fss->virtual_mem -= svm_fifo_size (f);
 
@@ -1131,18 +1160,18 @@ fifo_segment_num_fifos (fifo_segment_t * fs)
 }
 
 static u32
-fs_slice_num_free_fifos (fifo_segment_slice_t * fss)
+fs_slice_num_free_fifos (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
 {
   svm_fifo_shared_t *f;
   u32 count = 0;
 
-  f = fss->free_fifos;
+  f = fs_ptr (fsh, fss->free_fifos);
   if (f == 0)
     return 0;
 
   while (f)
     {
-      f = f->next;
+      f = fs_ptr (fsh, f->next);
       count++;
     }
   return count;
@@ -1159,7 +1188,7 @@ fifo_segment_num_free_fifos (fifo_segment_t * fs)
   for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
     {
       fss = fsh_slice_get (fsh, slice_index);
-      count += fs_slice_num_free_fifos (fss);
+      count += fs_slice_num_free_fifos (fsh, fss);
     }
   return count;
 }
diff --git a/src/svm/fifo_types.h b/src/svm/fifo_types.h
index bfd1a41a17b..592a7007c7d 100644
--- a/src/svm/fifo_types.h
+++ b/src/svm/fifo_types.h
@@ -33,18 +33,17 @@ STATIC_ASSERT ((FS_MAX_LOG2_CHUNK_SZ - FS_MIN_LOG2_CHUNK_SZ) ==
 #define SVM_FIFO_MAX_EVT_SUBSCRIBERS	7
 
 typedef struct fifo_segment_header_ fifo_segment_header_t;
-typedef struct svm_fifo_chunk_ svm_fifo_chunk_t;
-typedef svm_fifo_chunk_t *svm_fifo_chunk_ptr_t;
+typedef uword fs_sptr_t;
 
-struct svm_fifo_chunk_
+typedef struct svm_fifo_chunk_
 {
   u32 start_byte;		/**< chunk start byte */
   u32 length;			/**< length of chunk in bytes */
-  svm_fifo_chunk_ptr_t next;	/**< pointer to next chunk in linked-lists */
+  fs_sptr_t next;		/**< pointer to next chunk in linked-lists */
   rb_node_index_t enq_rb_index;	/**< enq node index if chunk in rbtree */
   rb_node_index_t deq_rb_index;	/**< deq node index if chunk in rbtree */
   u8 data[0];			/**< start of chunk data */
-};
+} svm_fifo_chunk_t;
 
 typedef struct
 {
@@ -64,25 +63,25 @@ typedef struct
 typedef struct svm_fifo_shr_
 {
   CLIB_CACHE_LINE_ALIGN_MARK (shared);
-  svm_fifo_chunk_ptr_t start_chunk; /**< first chunk in fifo chunk list */
-  svm_fifo_chunk_ptr_t end_chunk;   /**< end chunk in fifo chunk list */
+  fs_sptr_t start_chunk;	/**< first chunk in fifo chunk list */
+  fs_sptr_t end_chunk;		/**< end chunk in fifo chunk list */
   volatile u32 has_event;	/**< non-zero if deq event exists */
   u32 min_alloc;		/**< min chunk alloc if space available */
   u32 size;			/**< size of the fifo in bytes */
   u32 master_session_index;	/**< session layer session index */
   u32 client_session_index;	/**< app session index */
   u8 slice_index;		/**< segment slice for fifo */
-  struct svm_fifo_shr_ *next;	/**< next in freelist/active chain */
+  fs_sptr_t next;		/**< next in freelist */
 
   CLIB_CACHE_LINE_ALIGN_MARK (consumer);
-  svm_fifo_chunk_ptr_t head_chunk; /**< tracks chunk where head lands */
+  fs_sptr_t head_chunk;		/**< tracks chunk where head lands */
   u32 head;			/**< fifo head position/byte */
   volatile u32 want_deq_ntf;	/**< producer wants nudge */
   volatile u32 has_deq_ntf;
 
   CLIB_CACHE_LINE_ALIGN_MARK (producer);
   u32 tail;			/**< fifo tail position/byte */
-  svm_fifo_chunk_ptr_t tail_chunk; /**< tracks chunk where tail lands */
+  fs_sptr_t tail_chunk;		/**< tracks chunk where tail lands */
   volatile u8 n_subscribers;	/**< Number of subscribers for io events */
   u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS];
 } svm_fifo_shared_t;
@@ -95,10 +94,10 @@ typedef struct _svm_fifo
   rb_tree_t ooo_enq_lookup;	 /**< rbtree for ooo enq chunk lookup */
   rb_tree_t ooo_deq_lookup;	 /**< rbtree for ooo deq chunk lookup */
   svm_fifo_chunk_t *ooo_deq;	 /**< last chunk used for ooo dequeue */
-  svm_fifo_chunk_t *ooo_enq;	/**< last chunk used for ooo enqueue */
-  ooo_segment_t *ooo_segments;	/**< Pool of ooo segments */
-  u32 ooos_list_head;		/**< Head of out-of-order linked-list */
-  u32 ooos_newest;		/**< Last segment to have been updated */
+  svm_fifo_chunk_t *ooo_enq;	 /**< last chunk used for ooo enqueue */
+  ooo_segment_t *ooo_segments;	 /**< Pool of ooo segments */
+  u32 ooos_list_head;		 /**< Head of out-of-order linked-list */
+  u32 ooos_newest;		 /**< Last segment to have been updated */
 
   u8 flags;		  /**< fifo flags */
   u8 master_thread_index; /**< session layer thread index */
@@ -117,9 +116,8 @@ typedef struct _svm_fifo
 
 typedef struct fifo_segment_slice_
 {
-  svm_fifo_chunk_ptr_t
-    free_chunks[FS_CHUNK_VEC_LEN];	/**< Free chunks by size */
-  svm_fifo_shared_t *free_fifos;	/**< Freelists of fifo shared hdrs  */
+  fs_sptr_t free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */
+  fs_sptr_t free_fifos;			/**< Freelists of fifo shared hdrs  */
   uword n_fl_chunk_bytes;		/**< Chunk bytes on freelist */
   uword virtual_mem;			/**< Slice sum of all fifo sizes */
   u32 num_chunks[FS_CHUNK_VEC_LEN];	/**< Allocated chunks by chunk size */
@@ -157,16 +155,28 @@ struct fifo_segment_header_
 void fsh_virtual_mem_update (fifo_segment_header_t * fsh, u32 slice_index,
 			     int n_bytes);
 
+always_inline void *
+fs_ptr (fifo_segment_header_t *fsh, fs_sptr_t sp)
+{
+  return sp ? (void *) ((u8 *) fsh + sp) : 0;
+}
+
+always_inline fs_sptr_t
+fs_sptr (fifo_segment_header_t *fsh, void *p)
+{
+  return p ? (fs_sptr_t) ((u8 *) p - (u8 *) fsh) : 0;
+}
+
 always_inline svm_fifo_chunk_t *
-fs_chunk_ptr (fifo_segment_header_t *fsh, svm_fifo_chunk_ptr_t cp)
+fs_chunk_ptr (fifo_segment_header_t *fsh, fs_sptr_t cp)
 {
-  return cp ? (svm_fifo_chunk_t *) ((u8 *) fsh + pointer_to_uword (cp)) : 0;
+  return cp ? (svm_fifo_chunk_t *) ((u8 *) fsh + cp) : 0;
 }
 
-always_inline svm_fifo_chunk_ptr_t
+always_inline fs_sptr_t
 fs_chunk_sptr (fifo_segment_header_t *fsh, svm_fifo_chunk_t *c)
 {
-  return c ? (svm_fifo_chunk_ptr_t) ((u8 *) c - (u8 *) fsh) : 0;
+  return c ? (fs_sptr_t) ((u8 *) c - (u8 *) fsh) : 0;
 }
 
 #endif /* SRC_SVM_FIFO_TYPES_H_ */
diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index dc9e4fbed87..14eeb1c0802 100644
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -21,11 +21,11 @@
 #include <svm/fifo_segment.h>
 #include <vppinfra/cpu.h>
 
-#define F_INVALID_CPTR (svm_fifo_chunk_ptr_t) ~0ULL
+#define F_INVALID_CPTR (fs_sptr_t) ~0ULL
 
 CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t *f,
 	       svm_fifo_chunk_t *c, u32 tail_idx, const u8 *src, u32 len,
-	       svm_fifo_chunk_ptr_t *last)
+	       fs_sptr_t *last)
 {
   u32 n_chunk;
 
@@ -56,7 +56,7 @@ CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t *f,
 
 CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t *f,
 	       svm_fifo_chunk_t *c, u32 head_idx, u8 *dst, u32 len,
-	       svm_fifo_chunk_ptr_t *last)
+	       fs_sptr_t *last)
 {
   u32 n_chunk;
 
@@ -91,7 +91,7 @@ CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t *f,
 
 static inline void
 svm_fifo_copy_to_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx,
-			const u8 *src, u32 len, svm_fifo_chunk_ptr_t *last)
+			const u8 *src, u32 len, fs_sptr_t *last)
 {
   CLIB_MARCH_FN_SELECT (svm_fifo_copy_to_chunk) (f, c, tail_idx, src, len,
 						 last);
@@ -99,7 +99,7 @@ svm_fifo_copy_to_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx,
 
 static inline void
 svm_fifo_copy_from_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 head_idx,
-			  u8 *dst, u32 len, svm_fifo_chunk_ptr_t *last)
+			  u8 *dst, u32 len, fs_sptr_t *last)
 {
   CLIB_MARCH_FN_SELECT (svm_fifo_copy_from_chunk) (f, c, head_idx, dst, len,
 						   last);
@@ -906,7 +906,7 @@ int
 svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 len, u8 * src)
 {
   u32 tail, head, free_count, enq_pos;
-  svm_fifo_chunk_ptr_t last = F_INVALID_CPTR;
+  fs_sptr_t last = F_INVALID_CPTR;
 
   f_load_head_tail_prod (f, &head, &tail);
 
@@ -1141,7 +1141,7 @@ int
 svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 len, u8 * dst)
 {
   u32 tail, head, cursize, head_idx;
-  svm_fifo_chunk_ptr_t last = F_INVALID_CPTR;
+  fs_sptr_t last = F_INVALID_CPTR;
 
   f_load_head_tail_cons (f, &head, &tail);
 
diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h
index f5b60807c28..560628d2d07 100644
--- a/src/svm/svm_fifo.h
+++ b/src/svm/svm_fifo.h
@@ -196,19 +196,19 @@ f_tail_cptr (svm_fifo_t *f)
 }
 
 always_inline svm_fifo_chunk_t *
-f_cptr (svm_fifo_t *f, svm_fifo_chunk_ptr_t cp)
+f_cptr (svm_fifo_t *f, fs_sptr_t cp)
 {
   return fs_chunk_ptr (f->fs_hdr, cp);
 }
 
-always_inline svm_fifo_chunk_ptr_t
+always_inline fs_sptr_t
 f_csptr (svm_fifo_t *f, svm_fifo_chunk_t *c)
 {
   return fs_chunk_sptr (f->fs_hdr, c);
 }
 
 always_inline void
-f_csptr_link (svm_fifo_t *f, svm_fifo_chunk_ptr_t cp, svm_fifo_chunk_t *c)
+f_csptr_link (svm_fifo_t *f, fs_sptr_t cp, svm_fifo_chunk_t *c)
 {
   fs_chunk_ptr (f->fs_hdr, cp)->next = fs_chunk_sptr (f->fs_hdr, c);
 }