diff options
Diffstat (limited to 'src/svm')
| -rw-r--r-- | src/svm/fifo_segment.c | 65 | ||||
| -rw-r--r-- | src/svm/fifo_types.h | 52 | ||||
| -rw-r--r-- | src/svm/svm_fifo.c | 14 | ||||
| -rw-r--r-- | src/svm/svm_fifo.h | 6 |
4 files changed, 88 insertions, 49 deletions
diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c index 4ffec7c86d0..fdcf84ea8d7 100644 --- a/src/svm/fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -200,6 +200,32 @@ fss_chunk_free_list_pop (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, return c; } +static void +fss_fifo_free_list_push (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss, + svm_fifo_shared_t *sf) +{ + sf->next = fss->free_fifos; + fss->free_fifos = fs_sptr (fsh, sf); +} + +static void +fss_fifo_free_list_push_list (fifo_segment_header_t *fsh, + fifo_segment_slice_t *fss, + svm_fifo_shared_t *head, svm_fifo_shared_t *tail) +{ + tail->next = fss->free_fifos; + fss->free_fifos = fs_sptr (fsh, head); +} + +svm_fifo_shared_t * +fss_fifo_free_list_pop (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss) +{ + svm_fifo_shared_t *sf; + sf = fs_ptr (fsh, fss->free_fifos); + fss->free_fifos = sf->next; + return sf; +} + static inline void pfss_fifo_add_active_list (fifo_slice_private_t *pfss, svm_fifo_t *f) { @@ -499,7 +525,7 @@ static int fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, u32 batch_size) { - svm_fifo_shared_t *f; + svm_fifo_shared_t *f, *head = 0, *tail; uword size; u8 *fmem; int i; @@ -513,15 +539,18 @@ fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh, return -1; /* Carve fifo hdr space */ + tail = f = (svm_fifo_shared_t *) fmem; for (i = 0; i < batch_size; i++) { - f = (svm_fifo_shared_t *) fmem; - memset (f, 0, sizeof (*f)); - f->next = fss->free_fifos; - fss->free_fifos = f; + clib_memset (f, 0, sizeof (*f)); + f->next = fs_sptr (fsh, head); + head = f; fmem += sizeof (*f); + f = (svm_fifo_shared_t *) fmem; } + fss_fifo_free_list_push_list (fsh, fss, head, tail); + return 0; } @@ -579,7 +608,7 @@ fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh, static svm_fifo_shared_t * fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss) { - svm_fifo_shared_t *f; + svm_fifo_shared_t *sf; if (!fss->free_fifos) { @@ -588,10 +617,10 @@ fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss) return 0; } - f = fss->free_fifos; - fss->free_fifos = f->next; - memset (f, 0, sizeof (*f)); - return f; + sf = fss_fifo_free_list_pop (fsh, fss); + clib_memset (sf, 0, sizeof (*sf)); + + return sf; } static svm_fifo_chunk_t * @@ -684,8 +713,7 @@ fs_try_alloc_fifo (fifo_segment_header_t *fsh, u32 slice_index, u32 data_bytes) c = fsh_try_alloc_chunk (fsh, fss, min_size); if (!c) { - sf->next = fss->free_fifos; - fss->free_fifos = sf; + fss_fifo_free_list_push (fsh, fss, sf); return 0; } @@ -872,8 +900,9 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) sf->head_chunk = sf->tail_chunk = 0; /* Add to free list */ - sf->next = fss->free_fifos; - fss->free_fifos = sf; + fss_fifo_free_list_push (fsh, fss, sf); + // sf->next = fss->free_fifos; + // fss->free_fifos = fs_sptr (fsh, sf); fss->virtual_mem -= svm_fifo_size (f); @@ -1131,18 +1160,18 @@ fifo_segment_num_fifos (fifo_segment_t * fs) } static u32 -fs_slice_num_free_fifos (fifo_segment_slice_t * fss) +fs_slice_num_free_fifos (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss) { svm_fifo_shared_t *f; u32 count = 0; - f = fss->free_fifos; + f = fs_ptr (fsh, fss->free_fifos); if (f == 0) return 0; while (f) { - f = f->next; + f = fs_ptr (fsh, f->next); count++; } return count; @@ -1159,7 +1188,7 @@ fifo_segment_num_free_fifos (fifo_segment_t * fs) for (slice_index = 0; slice_index < fs->n_slices; slice_index++) { fss = fsh_slice_get (fsh, slice_index); - count += fs_slice_num_free_fifos (fss); + count += fs_slice_num_free_fifos (fsh, fss); } return count; } diff --git a/src/svm/fifo_types.h b/src/svm/fifo_types.h index bfd1a41a17b..592a7007c7d 100644 --- a/src/svm/fifo_types.h +++ b/src/svm/fifo_types.h @@ -33,18 +33,17 @@ STATIC_ASSERT ((FS_MAX_LOG2_CHUNK_SZ - FS_MIN_LOG2_CHUNK_SZ) == #define SVM_FIFO_MAX_EVT_SUBSCRIBERS 7 typedef struct fifo_segment_header_ fifo_segment_header_t; -typedef struct svm_fifo_chunk_ svm_fifo_chunk_t; -typedef svm_fifo_chunk_t *svm_fifo_chunk_ptr_t; +typedef uword fs_sptr_t; -struct svm_fifo_chunk_ +typedef struct svm_fifo_chunk_ { u32 start_byte; /**< chunk start byte */ u32 length; /**< length of chunk in bytes */ - svm_fifo_chunk_ptr_t next; /**< pointer to next chunk in linked-lists */ + fs_sptr_t next; /**< pointer to next chunk in linked-lists */ rb_node_index_t enq_rb_index; /**< enq node index if chunk in rbtree */ rb_node_index_t deq_rb_index; /**< deq node index if chunk in rbtree */ u8 data[0]; /**< start of chunk data */ -}; +} svm_fifo_chunk_t; typedef struct { @@ -64,25 +63,25 @@ typedef struct typedef struct svm_fifo_shr_ { CLIB_CACHE_LINE_ALIGN_MARK (shared); - svm_fifo_chunk_ptr_t start_chunk; /**< first chunk in fifo chunk list */ - svm_fifo_chunk_ptr_t end_chunk; /**< end chunk in fifo chunk list */ + fs_sptr_t start_chunk; /**< first chunk in fifo chunk list */ + fs_sptr_t end_chunk; /**< end chunk in fifo chunk list */ volatile u32 has_event; /**< non-zero if deq event exists */ u32 min_alloc; /**< min chunk alloc if space available */ u32 size; /**< size of the fifo in bytes */ u32 master_session_index; /**< session layer session index */ u32 client_session_index; /**< app session index */ u8 slice_index; /**< segment slice for fifo */ - struct svm_fifo_shr_ *next; /**< next in freelist/active chain */ + fs_sptr_t next; /**< next in freelist */ CLIB_CACHE_LINE_ALIGN_MARK (consumer); - svm_fifo_chunk_ptr_t head_chunk; /**< tracks chunk where head lands */ + fs_sptr_t head_chunk; /**< tracks chunk where head lands */ u32 head; /**< fifo head position/byte */ volatile u32 want_deq_ntf; /**< producer wants nudge */ volatile u32 has_deq_ntf; CLIB_CACHE_LINE_ALIGN_MARK (producer); u32 tail; /**< fifo tail position/byte */ - svm_fifo_chunk_ptr_t tail_chunk; /**< tracks chunk where tail lands */ + fs_sptr_t tail_chunk; /**< tracks chunk where tail lands */ volatile u8 n_subscribers; /**< Number of subscribers for io events */ u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS]; } svm_fifo_shared_t; @@ -95,10 +94,10 @@ typedef struct _svm_fifo rb_tree_t ooo_enq_lookup; /**< rbtree for ooo enq chunk lookup */ rb_tree_t ooo_deq_lookup; /**< rbtree for ooo deq chunk lookup */ svm_fifo_chunk_t *ooo_deq; /**< last chunk used for ooo dequeue */ - svm_fifo_chunk_t *ooo_enq; /**< last chunk used for ooo enqueue */ - ooo_segment_t *ooo_segments; /**< Pool of ooo segments */ - u32 ooos_list_head; /**< Head of out-of-order linked-list */ - u32 ooos_newest; /**< Last segment to have been updated */ + svm_fifo_chunk_t *ooo_enq; /**< last chunk used for ooo enqueue */ + ooo_segment_t *ooo_segments; /**< Pool of ooo segments */ + u32 ooos_list_head; /**< Head of out-of-order linked-list */ + u32 ooos_newest; /**< Last segment to have been updated */ u8 flags; /**< fifo flags */ u8 master_thread_index; /**< session layer thread index */ @@ -117,9 +116,8 @@ typedef struct _svm_fifo typedef struct fifo_segment_slice_ { - svm_fifo_chunk_ptr_t - free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */ - svm_fifo_shared_t *free_fifos; /**< Freelists of fifo shared hdrs */ + fs_sptr_t free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */ + fs_sptr_t free_fifos; /**< Freelists of fifo shared hdrs */ uword n_fl_chunk_bytes; /**< Chunk bytes on freelist */ uword virtual_mem; /**< Slice sum of all fifo sizes */ u32 num_chunks[FS_CHUNK_VEC_LEN]; /**< Allocated chunks by chunk size */ @@ -157,16 +155,28 @@ struct fifo_segment_header_ void fsh_virtual_mem_update (fifo_segment_header_t * fsh, u32 slice_index, int n_bytes); +always_inline void * +fs_ptr (fifo_segment_header_t *fsh, fs_sptr_t sp) +{ + return sp ? (void *) ((u8 *) fsh + sp) : 0; +} + +always_inline fs_sptr_t +fs_sptr (fifo_segment_header_t *fsh, void *p) +{ + return p ? (fs_sptr_t) ((u8 *) p - (u8 *) fsh) : 0; +} + always_inline svm_fifo_chunk_t * -fs_chunk_ptr (fifo_segment_header_t *fsh, svm_fifo_chunk_ptr_t cp) +fs_chunk_ptr (fifo_segment_header_t *fsh, fs_sptr_t cp) { - return cp ? (svm_fifo_chunk_t *) ((u8 *) fsh + pointer_to_uword (cp)) : 0; + return cp ? (svm_fifo_chunk_t *) ((u8 *) fsh + cp) : 0; } -always_inline svm_fifo_chunk_ptr_t +always_inline fs_sptr_t fs_chunk_sptr (fifo_segment_header_t *fsh, svm_fifo_chunk_t *c) { - return c ? (svm_fifo_chunk_ptr_t) ((u8 *) c - (u8 *) fsh) : 0; + return c ? (fs_sptr_t) ((u8 *) c - (u8 *) fsh) : 0; } #endif /* SRC_SVM_FIFO_TYPES_H_ */ diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index dc9e4fbed87..14eeb1c0802 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -21,11 +21,11 @@ #include <svm/fifo_segment.h> #include <vppinfra/cpu.h> -#define F_INVALID_CPTR (svm_fifo_chunk_ptr_t) ~0ULL +#define F_INVALID_CPTR (fs_sptr_t) ~0ULL CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx, const u8 *src, u32 len, - svm_fifo_chunk_ptr_t *last) + fs_sptr_t *last) { u32 n_chunk; @@ -56,7 +56,7 @@ CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t *f, CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t *f, svm_fifo_chunk_t *c, u32 head_idx, u8 *dst, u32 len, - svm_fifo_chunk_ptr_t *last) + fs_sptr_t *last) { u32 n_chunk; @@ -91,7 +91,7 @@ CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t *f, static inline void svm_fifo_copy_to_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx, - const u8 *src, u32 len, svm_fifo_chunk_ptr_t *last) + const u8 *src, u32 len, fs_sptr_t *last) { CLIB_MARCH_FN_SELECT (svm_fifo_copy_to_chunk) (f, c, tail_idx, src, len, last); @@ -99,7 +99,7 @@ svm_fifo_copy_to_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx, static inline void svm_fifo_copy_from_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 head_idx, - u8 *dst, u32 len, svm_fifo_chunk_ptr_t *last) + u8 *dst, u32 len, fs_sptr_t *last) { CLIB_MARCH_FN_SELECT (svm_fifo_copy_from_chunk) (f, c, head_idx, dst, len, last); @@ -906,7 +906,7 @@ int svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 len, u8 * src) { u32 tail, head, free_count, enq_pos; - svm_fifo_chunk_ptr_t last = F_INVALID_CPTR; + fs_sptr_t last = F_INVALID_CPTR; f_load_head_tail_prod (f, &head, &tail); @@ -1141,7 +1141,7 @@ int svm_fifo_peek (svm_fifo_t * f, u32 offset, u32 len, u8 * dst) { u32 tail, head, cursize, head_idx; - svm_fifo_chunk_ptr_t last = F_INVALID_CPTR; + fs_sptr_t last = F_INVALID_CPTR; f_load_head_tail_cons (f, &head, &tail); diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index f5b60807c28..560628d2d07 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -196,19 +196,19 @@ f_tail_cptr (svm_fifo_t *f) } always_inline svm_fifo_chunk_t * -f_cptr (svm_fifo_t *f, svm_fifo_chunk_ptr_t cp) +f_cptr (svm_fifo_t *f, fs_sptr_t cp) { return fs_chunk_ptr (f->fs_hdr, cp); } -always_inline svm_fifo_chunk_ptr_t +always_inline fs_sptr_t f_csptr (svm_fifo_t *f, svm_fifo_chunk_t *c) { return fs_chunk_sptr (f->fs_hdr, c); } always_inline void -f_csptr_link (svm_fifo_t *f, svm_fifo_chunk_ptr_t cp, svm_fifo_chunk_t *c) +f_csptr_link (svm_fifo_t *f, fs_sptr_t cp, svm_fifo_chunk_t *c) { fs_chunk_ptr (f->fs_hdr, cp)->next = fs_chunk_sptr (f->fs_hdr, c); } |