summaryrefslogtreecommitdiffstats
path: root/src/svm
diff options
context:
space:
mode:
authorFlorin Coras <fcoras@cisco.com>2020-12-29 16:55:32 -0800
committerDave Barach <openvpp@barachs.net>2020-12-30 16:45:54 +0000
commit17672aa49d3ebefb217d6f2c67f698b3e7e1f86c (patch)
tree657573730b1cd52c20da9ee7b0c0efe56cdbaaf2 /src/svm
parent9a45bd84d9f8e37f7914eb36d2ffdcd209aa15cb (diff)
svm: fifo segment sptr for fifo hdr free list
With this there are no more pointers in data structures allocated on fifo segments. Type: improvement Signed-off-by: Florin Coras <fcoras@cisco.com> Change-Id: Ibe584b7b6809fa360a105974655a91674db69ab6
Diffstat (limited to 'src/svm')
-rw-r--r--src/svm/fifo_segment.c65
-rw-r--r--src/svm/fifo_types.h52
-rw-r--r--src/svm/svm_fifo.c14
-rw-r--r--src/svm/svm_fifo.h6
4 files changed, 88 insertions, 49 deletions
diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c
index 4ffec7c86d0..fdcf84ea8d7 100644
--- a/src/svm/fifo_segment.c
+++ b/src/svm/fifo_segment.c
@@ -200,6 +200,32 @@ fss_chunk_free_list_pop (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss,
return c;
}
+static void
+fss_fifo_free_list_push (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss,
+ svm_fifo_shared_t *sf)
+{
+ sf->next = fss->free_fifos;
+ fss->free_fifos = fs_sptr (fsh, sf);
+}
+
+static void
+fss_fifo_free_list_push_list (fifo_segment_header_t *fsh,
+ fifo_segment_slice_t *fss,
+ svm_fifo_shared_t *head, svm_fifo_shared_t *tail)
+{
+ tail->next = fss->free_fifos;
+ fss->free_fifos = fs_sptr (fsh, head);
+}
+
+svm_fifo_shared_t *
+fss_fifo_free_list_pop (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
+{
+ svm_fifo_shared_t *sf;
+ sf = fs_ptr (fsh, fss->free_fifos);
+ fss->free_fifos = sf->next;
+ return sf;
+}
+
static inline void
pfss_fifo_add_active_list (fifo_slice_private_t *pfss, svm_fifo_t *f)
{
@@ -499,7 +525,7 @@ static int
fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh,
fifo_segment_slice_t * fss, u32 batch_size)
{
- svm_fifo_shared_t *f;
+ svm_fifo_shared_t *f, *head = 0, *tail;
uword size;
u8 *fmem;
int i;
@@ -513,15 +539,18 @@ fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh,
return -1;
/* Carve fifo hdr space */
+ tail = f = (svm_fifo_shared_t *) fmem;
for (i = 0; i < batch_size; i++)
{
- f = (svm_fifo_shared_t *) fmem;
- memset (f, 0, sizeof (*f));
- f->next = fss->free_fifos;
- fss->free_fifos = f;
+ clib_memset (f, 0, sizeof (*f));
+ f->next = fs_sptr (fsh, head);
+ head = f;
fmem += sizeof (*f);
+ f = (svm_fifo_shared_t *) fmem;
}
+ fss_fifo_free_list_push_list (fsh, fss, head, tail);
+
return 0;
}
@@ -579,7 +608,7 @@ fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh,
static svm_fifo_shared_t *
fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
{
- svm_fifo_shared_t *f;
+ svm_fifo_shared_t *sf;
if (!fss->free_fifos)
{
@@ -588,10 +617,10 @@ fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
return 0;
}
- f = fss->free_fifos;
- fss->free_fifos = f->next;
- memset (f, 0, sizeof (*f));
- return f;
+ sf = fss_fifo_free_list_pop (fsh, fss);
+ clib_memset (sf, 0, sizeof (*sf));
+
+ return sf;
}
static svm_fifo_chunk_t *
@@ -684,8 +713,7 @@ fs_try_alloc_fifo (fifo_segment_header_t *fsh, u32 slice_index, u32 data_bytes)
c = fsh_try_alloc_chunk (fsh, fss, min_size);
if (!c)
{
- sf->next = fss->free_fifos;
- fss->free_fifos = sf;
+ fss_fifo_free_list_push (fsh, fss, sf);
return 0;
}
@@ -872,8 +900,9 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f)
sf->head_chunk = sf->tail_chunk = 0;
/* Add to free list */
- sf->next = fss->free_fifos;
- fss->free_fifos = sf;
+ fss_fifo_free_list_push (fsh, fss, sf);
+ // sf->next = fss->free_fifos;
+ // fss->free_fifos = fs_sptr (fsh, sf);
fss->virtual_mem -= svm_fifo_size (f);
@@ -1131,18 +1160,18 @@ fifo_segment_num_fifos (fifo_segment_t * fs)
}
static u32
-fs_slice_num_free_fifos (fifo_segment_slice_t * fss)
+fs_slice_num_free_fifos (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss)
{
svm_fifo_shared_t *f;
u32 count = 0;
- f = fss->free_fifos;
+ f = fs_ptr (fsh, fss->free_fifos);
if (f == 0)
return 0;
while (f)
{
- f = f->next;
+ f = fs_ptr (fsh, f->next);
count++;
}
return count;
@@ -1159,7 +1188,7 @@ fifo_segment_num_free_fifos (fifo_segment_t * fs)
for (slice_index = 0; slice_index < fs->n_slices; slice_index++)
{
fss = fsh_slice_get (fsh, slice_index);
- count += fs_slice_num_free_fifos (fss);
+ count += fs_slice_num_free_fifos (fsh, fss);
}
return count;
}
diff --git a/src/svm/fifo_types.h b/src/svm/fifo_types.h
index bfd1a41a17b..592a7007c7d 100644
--- a/src/svm/fifo_types.h
+++ b/src/svm/fifo_types.h
@@ -33,18 +33,17 @@ STATIC_ASSERT ((FS_MAX_LOG2_CHUNK_SZ - FS_MIN_LOG2_CHUNK_SZ) ==
#define SVM_FIFO_MAX_EVT_SUBSCRIBERS 7
typedef struct fifo_segment_header_ fifo_segment_header_t;
-typedef struct svm_fifo_chunk_ svm_fifo_chunk_t;
-typedef svm_fifo_chunk_t *svm_fifo_chunk_ptr_t;
+typedef uword fs_sptr_t;
-struct svm_fifo_chunk_
+typedef struct svm_fifo_chunk_
{
u32 start_byte; /**< chunk start byte */
u32 length; /**< length of chunk in bytes */
- svm_fifo_chunk_ptr_t next; /**< pointer to next chunk in linked-lists */
+ fs_sptr_t next; /**< pointer to next chunk in linked-lists */
rb_node_index_t enq_rb_index; /**< enq node index if chunk in rbtree */
rb_node_index_t deq_rb_index; /**< deq node index if chunk in rbtree */
u8 data[0]; /**< start of chunk data */
-};
+} svm_fifo_chunk_t;
typedef struct
{
@@ -64,25 +63,25 @@ typedef struct
typedef struct svm_fifo_shr_
{
CLIB_CACHE_LINE_ALIGN_MARK (shared);
- svm_fifo_chunk_ptr_t start_chunk; /**< first chunk in fifo chunk list */
- svm_fifo_chunk_ptr_t end_chunk; /**< end chunk in fifo chunk list */
+ fs_sptr_t start_chunk; /**< first chunk in fifo chunk list */
+ fs_sptr_t end_chunk; /**< end chunk in fifo chunk list */
volatile u32 has_event; /**< non-zero if deq event exists */
u32 min_alloc; /**< min chunk alloc if space available */
u32 size; /**< size of the fifo in bytes */
u32 master_session_index; /**< session layer session index */
u32 client_session_index; /**< app session index */
u8 slice_index; /**< segment slice for fifo */
- struct svm_fifo_shr_ *next; /**< next in freelist/active chain */
+ fs_sptr_t next; /**< next in freelist */
CLIB_CACHE_LINE_ALIGN_MARK (consumer);
- svm_fifo_chunk_ptr_t head_chunk; /**< tracks chunk where head lands */
+ fs_sptr_t head_chunk; /**< tracks chunk where head lands */
u32 head; /**< fifo head position/byte */
volatile u32 want_deq_ntf; /**< producer wants nudge */
volatile u32 has_deq_ntf;
CLIB_CACHE_LINE_ALIGN_MARK (producer);
u32 tail; /**< fifo tail position/byte */
- svm_fifo_chunk_ptr_t tail_chunk; /**< tracks chunk where tail lands */
+ fs_sptr_t tail_chunk; /**< tracks chunk where tail lands */
volatile u8 n_subscribers; /**< Number of subscribers for io events */
u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS];
} svm_fifo_shared_t;
@@ -95,10 +94,10 @@ typedef struct _svm_fifo
rb_tree_t ooo_enq_lookup; /**< rbtree for ooo enq chunk lookup */
rb_tree_t ooo_deq_lookup; /**< rbtree for ooo deq chunk lookup */
svm_fifo_chunk_t *ooo_deq; /**< last chunk used for ooo dequeue */
- svm_fifo_chunk_t *ooo_enq; /**< last chunk used for ooo enqueue */
- ooo_segment_t *ooo_segments; /**< Pool of ooo segments */
- u32 ooos_list_head; /**< Head of out-of-order linked-list */
- u32 ooos_newest; /**< Last segment to have been updated */
+ svm_fifo_chunk_t *ooo_enq; /**< last chunk used for ooo enqueue */
+ ooo_segment_t *ooo_segments; /**< Pool of ooo segments */
+ u32 ooos_list_head; /**< Head of out-of-order linked-list */
+ u32 ooos_newest; /**< Last segment to have been updated */
u8 flags; /**< fifo flags */
u8 master_thread_index; /**< session layer thread index */
@@ -117,9 +116,8 @@ typedef struct _svm_fifo
typedef struct fifo_segment_slice_
{
- svm_fifo_chunk_ptr_t
- free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */
- svm_fifo_shared_t *free_fifos; /**< Freelists of fifo shared hdrs */
+ fs_sptr_t free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */
+ fs_sptr_t free_fifos; /**< Freelists of fifo shared hdrs */
uword n_fl_chunk_bytes; /**< Chunk bytes on freelist */
uword virtual_mem; /**< Slice sum of all fifo sizes */
u32 num_chunks[FS_CHUNK_VEC_LEN]; /**< Allocated chunks by chunk size */
@@ -157,16 +155,28 @@ struct fifo_segment_header_
void fsh_virtual_mem_update (fifo_segment_header_t * fsh, u32 slice_index,
int n_bytes);
+always_inline void *
+fs_ptr (fifo_segment_header_t *fsh, fs_sptr_t sp)
+{
+ return sp ? (void *) ((u8 *) fsh + sp) : 0;
+}
+
+always_inline fs_sptr_t
+fs_sptr (fifo_segment_header_t *fsh, void *p)
+{
+ return p ? (fs_sptr_t) ((u8 *) p - (u8 *) fsh) : 0;
+}
+
always_inline svm_fifo_chunk_t *
-fs_chunk_ptr (fifo_segment_header_t *fsh, svm_fifo_chunk_ptr_t cp)
+fs_chunk_ptr (fifo_segment_header_t *fsh, fs_sptr_t cp)
{
- return cp ? (svm_fifo_chunk_t *) ((u8 *) fsh + pointer_to_uword (cp)) : 0;
+ return cp ? (svm_fifo_chunk_t *) ((u8 *) fsh + cp) : 0;
}
-always_inline svm_fifo_chunk_ptr_t
+always_inline fs_sptr_t
fs_chunk_sptr (fifo_segment_header_t *fsh, svm_fifo_chunk_t *c)
{
- return c ? (svm_fifo_chunk_ptr_t) ((u8 *) c - (u8 *) fsh) : 0;
+ return c ? (fs_sptr_t) ((u8 *) c - (u8 *) fsh) : 0;
}
#endif /* SRC_SVM_FIFO_TYPES_H_ */
diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index dc9e4fbed87..14eeb1c0802 100644
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -21,11 +21,11 @@
#include <svm/fifo_segment.h>
#include <vppinfra/cpu.h>
-#define F_INVALID_CPTR (svm_fifo_chunk_ptr_t) ~0ULL
+#define F_INVALID_CPTR (fs_sptr_t) ~0ULL
CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t *f,
svm_fifo_chunk_t *c, u32 tail_idx, const u8 *src, u32 len,
- svm_fifo_chunk_ptr_t *last)
+ fs_sptr_t *last)
{
u32 n_chunk;
@@ -56,7 +56,7 @@ CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t *f,
CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t *f,
svm_fifo_chunk_t *c, u32 head_idx, u8 *dst, u32 len,
- svm_fifo_chunk_ptr_t *last)
+ fs_sptr_t *last)
{
u32 n_chunk;
@@ -91,7 +91,7 @@ CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t *f,
static inline void
svm_fifo_copy_to_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx,
- const u8 *src, u32 len, svm_fifo_chunk_ptr_t *last)
+ const u8 *src, u32 len, fs_sptr_t *last)
{
CLIB_MARCH_FN_SELECT (svm_fifo_copy_to_chunk) (f, c, tail_idx, src, len,
last);
@@ -99,7 +99,7 @@ svm_fifo_copy_to_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx,
static inline void
svm_fifo_copy_from_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 head_idx,
- u8 *dst, u32 len, svm_fifo_chunk_ptr_t *last)
+ u8 *dst, u32 len, fs_sptr_t *last)
{
CLIB_MARCH_FN_SELECT (svm_fifo_copy_from_chunk) (f, c, head_idx, dst, len,
last);
@@ -906,7 +906,7 @@ int
svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 len, u8 * src)
{
u32 tail, head, free_count, enq_pos;
- svm_fifo_chunk_ptr_t last = F_INVALID_CPTR;
+ fs_sptr_t last = F_INVALID_CPTR;
f_load_head_tail_prod (f, &head, &tail);
@@ -1141,7 +1141,7 @@ int
svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 len, u8 * dst)
{
u32 tail, head, cursize, head_idx;
- svm_fifo_chunk_ptr_t last = F_INVALID_CPTR;
+ fs_sptr_t last = F_INVALID_CPTR;
f_load_head_tail_cons (f, &head, &tail);
diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h
index f5b60807c28..560628d2d07 100644
--- a/src/svm/svm_fifo.h
+++ b/src/svm/svm_fifo.h
@@ -196,19 +196,19 @@ f_tail_cptr (svm_fifo_t *f)
}
always_inline svm_fifo_chunk_t *
-f_cptr (svm_fifo_t *f, svm_fifo_chunk_ptr_t cp)
+f_cptr (svm_fifo_t *f, fs_sptr_t cp)
{
return fs_chunk_ptr (f->fs_hdr, cp);
}
-always_inline svm_fifo_chunk_ptr_t
+always_inline fs_sptr_t
f_csptr (svm_fifo_t *f, svm_fifo_chunk_t *c)
{
return fs_chunk_sptr (f->fs_hdr, c);
}
always_inline void
-f_csptr_link (svm_fifo_t *f, svm_fifo_chunk_ptr_t cp, svm_fifo_chunk_t *c)
+f_csptr_link (svm_fifo_t *f, fs_sptr_t cp, svm_fifo_chunk_t *c)
{
fs_chunk_ptr (f->fs_hdr, cp)->next = fs_chunk_sptr (f->fs_hdr, c);
}