aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorFlorin Coras <fcoras@cisco.com>2019-04-09 18:29:14 -0700
committerDamjan Marion <dmarion@me.com>2019-04-18 13:16:18 +0000
commit0a846806742922d98f1aff9cf1370715e57a2ae5 (patch)
tree5214807e20731453254635ed2d9afa9649eb4469
parent5a8844bdbf4b055812cce2d7755a175b2cc90b75 (diff)
svm: allow indirect fifo data chunks
Fifos can use multiple memory chunks for simple read/write operations. Adding/removing chunks after assignment not yet supported. Change-Id: I2aceab6aea78059d74e0d3a9993c40d5196d077b Signed-off-by: Florin Coras <fcoras@cisco.com>
-rw-r--r--src/plugins/unittest/session_test.c2
-rw-r--r--src/plugins/unittest/tcp_test.c6
-rw-r--r--src/svm/svm_fifo.c258
-rw-r--r--src/svm/svm_fifo.h25
-rw-r--r--src/svm/svm_fifo_segment.c49
-rw-r--r--src/vnet/session/application_worker.c1
6 files changed, 171 insertions, 170 deletions
diff --git a/src/plugins/unittest/session_test.c b/src/plugins/unittest/session_test.c
index fa6a99ffbf0..542be604e5b 100644
--- a/src/plugins/unittest/session_test.c
+++ b/src/plugins/unittest/session_test.c
@@ -1817,7 +1817,7 @@ session_test_mq (vlib_main_t * vm, unformat_input_t * input)
s.rx_fifo = rx_fifo;
s.tx_fifo = tx_fifo;
s.session_state = SESSION_STATE_READY;
- counter = (u64 *) rx_fifo->data;
+ counter = (u64 *) rx_fifo->head_chunk->data;
start = vlib_time_now (vm);
pid = fork ();
diff --git a/src/plugins/unittest/tcp_test.c b/src/plugins/unittest/tcp_test.c
index c8d9d5fe247..f7bcad3e67e 100644
--- a/src/plugins/unittest/tcp_test.c
+++ b/src/plugins/unittest/tcp_test.c
@@ -740,7 +740,7 @@ fifo_prepare (u32 fifo_size)
f = svm_fifo_create (fifo_size);
/* Paint fifo data vector with -1's */
- clib_memset (f->data, 0xFF, fifo_size);
+ clib_memset (f->head_chunk->data, 0xFF, fifo_size);
return f;
}
@@ -1375,14 +1375,14 @@ tcp_test_fifo4 (vlib_main_t * vm, unformat_input_t * input)
static u32
fifo_pos (svm_fifo_t * f, u32 pos)
{
- return pos % f->nitems;
+ return pos;
}
static int
tcp_test_fifo5 (vlib_main_t * vm, unformat_input_t * input)
{
svm_fifo_t *f;
- u32 fifo_size = 400, j = 0, offset = 200;
+ u32 fifo_size = 401, j = 0, offset = 200;
int i, rv, verbose = 0;
u8 *test_data = 0, *data_buf = 0;
ooo_segment_t *ooo_seg;
diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index 918ad77956e..de1997452d5 100644
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -113,8 +113,7 @@ svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose)
#endif
dummy_fifo = svm_fifo_create (f->size);
- clib_memset (f->data, 0xFF, f->nitems);
-
+ clib_memset (f->head_chunk->data, 0xFF, f->nitems);
vec_validate (data, f->nitems);
for (i = 0; i < vec_len (data); i++)
data[i] = i;
@@ -204,6 +203,26 @@ format_svm_fifo (u8 * s, va_list * args)
return s;
}
+void
+svm_fifo_init (svm_fifo_t * f, u32 size)
+{
+ f->size = size;
+ /*
+ * usable size of the fifo set to rounded_data_size - 1
+ * to differentiate between free fifo and empty fifo.
+ */
+ f->nitems = f->size - 1;
+ f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
+ f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
+ f->segment_index = SVM_FIFO_INVALID_INDEX;
+ f->refcnt = 1;
+ f->head_chunk = &f->default_chunk;
+ f->tail_chunk = &f->default_chunk;
+ f->default_chunk.next = &f->default_chunk;
+ f->default_chunk.start_byte = 0;
+ f->default_chunk.length = f->size;
+}
+
/** create an svm fifo, in the current heap. Fails vs blow up the process */
svm_fifo_t *
svm_fifo_create (u32 data_size_in_bytes)
@@ -219,17 +238,8 @@ svm_fifo_create (u32 data_size_in_bytes)
return 0;
clib_memset (f, 0, sizeof (*f));
- f->size = rounded_data_size;
- /*
- * usable size of the fifo set to rounded_data_size - 1
- * to differentiate between free fifo and empty fifo.
- */
- f->nitems = f->size - 1;
- f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
- f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
- f->segment_index = SVM_FIFO_INVALID_INDEX;
- f->refcnt = 1;
- return (f);
+ svm_fifo_init (f, data_size_in_bytes);
+ return f;
}
void
@@ -296,7 +306,9 @@ ooo_segment_add (svm_fifo_t * f, u32 offset, u32 head, u32 tail, u32 length)
u32 new_index, s_end_pos, s_index;
u32 offset_pos, offset_end_pos;
- ASSERT (offset + length <= ooo_segment_distance_from_tail (f, head, tail));
+ ASSERT (offset + length <= ooo_segment_distance_from_tail (f, head, tail)
+ || head == tail);
+
offset_pos = tail + offset;
offset_end_pos = tail + offset + length;
@@ -469,11 +481,11 @@ ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued, u32 * tail)
return bytes;
}
-CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
- const u8 * copy_from_here)
+CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 len,
+ const u8 * src)
{
- u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
- u32 tail, head, free_count, tail_idx;
+ u32 n_chunk, to_copy, tail, head, free_count, tail_idx;
+ svm_fifo_chunk_t *c;
f_load_head_tail_prod (f, &head, &tail);
@@ -486,50 +498,43 @@ CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
return SVM_FIFO_FULL;
/* number of bytes we're going to copy */
- total_copy_bytes = free_count < max_bytes ? free_count : max_bytes;
+ to_copy = len = clib_min (free_count, len);
+ c = f->tail_chunk;
tail_idx = tail % f->size;
+ ASSERT (tail_idx >= c->start_byte);
+ tail_idx -= c->start_byte;
+ n_chunk = c->length - tail_idx;
- if (PREDICT_TRUE (copy_from_here != 0))
+ if (n_chunk < to_copy)
{
- first_copy_bytes = f->size - tail_idx;
- if (first_copy_bytes < total_copy_bytes)
- {
- clib_memcpy_fast (&f->data[tail_idx], copy_from_here,
- first_copy_bytes);
- /* number of bytes in second copy segment */
- second_copy_bytes = total_copy_bytes - first_copy_bytes;
- /* wrap around */
- clib_memcpy_fast (&f->data[0],
- copy_from_here + first_copy_bytes,
- second_copy_bytes);
- }
- else
+ clib_memcpy_fast (&c->data[tail_idx], src, n_chunk);
+ while ((to_copy -= n_chunk))
{
- clib_memcpy_fast (&f->data[tail_idx], copy_from_here,
- total_copy_bytes);
+ c = c->next;
+ n_chunk = clib_min (c->length, to_copy);
+ clib_memcpy_fast (&c->data[0], src + (len - to_copy), n_chunk);
}
- tail += total_copy_bytes;
+ f->tail_chunk = c;
}
else
{
- ASSERT (0);
- /* Account for a zero-copy enqueue done elsewhere */
- tail += max_bytes;
+ clib_memcpy_fast (&c->data[tail_idx], src, to_copy);
}
+ tail += len;
- svm_fifo_trace_add (f, head, total_copy_bytes, 2);
+ svm_fifo_trace_add (f, head, n_total, 2);
/* collect out-of-order segments */
if (PREDICT_FALSE (f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX))
- total_copy_bytes += ooo_segment_try_collect (f, total_copy_bytes, &tail);
+ len += ooo_segment_try_collect (f, len, &tail);
- ASSERT (total_copy_bytes <= free_count);
+ ASSERT (len <= free_count);
/* store-rel: producer owned index (paired with load-acq in consumer) */
clib_atomic_store_rel_n (&f->tail, tail);
- return total_copy_bytes;
+ return len;
}
#ifndef CLIB_MARCH_VARIANT
@@ -550,11 +555,10 @@ svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
* Returns -1 if none of the segment was copied due to lack of space
*/
CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f,
- u32 offset, u32 required_bytes, u8 * copy_from_here)
+ u32 offset, u32 len, u8 * src)
{
- u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
- u32 tail_offset;
- u32 tail, head, free_count, tail_offset_idx;
+ u32 to_copy, n_chunk, tail, head, free_count, tail_offset_idx;
+ svm_fifo_chunk_t *c;
f_load_head_tail_prod (f, &head, &tail);
@@ -562,41 +566,35 @@ CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f,
free_count = f_free_count (f, head, tail);
/* will this request fit? */
- if ((required_bytes + offset) > free_count)
+ if ((len + offset) > free_count)
return -1;
f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
- ASSERT (required_bytes < f->nitems);
+ ASSERT (len < f->nitems);
+ svm_fifo_trace_add (f, offset, len, 1);
- tail_offset = tail + offset;
- tail_offset_idx = tail_offset % f->size;
+ ooo_segment_add (f, offset, head, tail, len);
- svm_fifo_trace_add (f, offset, required_bytes, 1);
+ c = f->tail_chunk;
+ tail_offset_idx = (tail + offset) % f->size;
+ tail_offset_idx -= c->start_byte;
+ n_chunk = c->length - tail_offset_idx;
+ to_copy = len;
- ooo_segment_add (f, offset, head, tail, required_bytes);
-
- /* number of bytes we're going to copy */
- total_copy_bytes = required_bytes;
-
- /* number of bytes in first copy segment */
- first_copy_bytes = f->size - tail_offset_idx;
-
- if (first_copy_bytes < total_copy_bytes)
+ if (n_chunk < to_copy)
{
- clib_memcpy_fast (&f->data[tail_offset_idx], copy_from_here,
- first_copy_bytes);
-
- /* number of bytes in second copy segment */
- second_copy_bytes = total_copy_bytes - first_copy_bytes;
- /* wrap around */
- clib_memcpy_fast (&f->data[0],
- copy_from_here + first_copy_bytes, second_copy_bytes);
+ clib_memcpy_fast (&c->data[tail_offset_idx], src, n_chunk);
+ while ((to_copy -= n_chunk))
+ {
+ c = c->next;
+ n_chunk = clib_min (c->length, to_copy);
+ clib_memcpy_fast (&c->data[0], src + (len - to_copy), n_chunk);
+ }
}
else
{
- clib_memcpy_fast (&f->data[tail_offset_idx], copy_from_here,
- total_copy_bytes);
+ clib_memcpy_fast (&c->data[tail_offset_idx], src, len);
}
return 0;
@@ -616,28 +614,32 @@ svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 required_bytes,
void
svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len)
{
- u32 first_chunk;
+ u32 n_chunk;
u32 head, tail, head_idx;
+ svm_fifo_chunk_t *c;
+
+ ASSERT (len <= f->nitems);
f_load_head_tail_cons (f, &head, &tail);
+ c = f->head_chunk;
head_idx = head % f->size;
- first_chunk = f->size - (head_idx);
- ASSERT (len <= f->nitems);
- if (len <= first_chunk)
- clib_memcpy_fast (&f->data[head_idx], data, len);
+ head_idx -= c->start_byte;
+ n_chunk = c->length - head_idx;
+ if (len <= n_chunk)
+ clib_memcpy_fast (&c->data[head_idx], data, len);
else
{
- clib_memcpy_fast (&f->data[head_idx], data, first_chunk);
- clib_memcpy_fast (&f->data[0], data + first_chunk, len - first_chunk);
+ clib_memcpy_fast (&c->data[head_idx], data, n_chunk);
+ clib_memcpy_fast (&c->next->data[0], data + n_chunk, len - n_chunk);
}
}
#endif
-CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
- u8 * copy_here)
+CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 len,
+ u8 * dst)
{
- u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
- u32 tail, head, cursize, head_idx;
+ u32 to_copy, n_chunk, tail, head, cursize, head_idx;
+ svm_fifo_chunk_t *c;
f_load_head_tail_cons (f, &head, &tail);
@@ -647,42 +649,35 @@ CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
if (PREDICT_FALSE (cursize == 0))
return -2; /* nothing in the fifo */
- /* number of bytes we're going to copy */
- total_copy_bytes = (cursize < max_bytes) ? cursize : max_bytes;
+ to_copy = len = clib_min (cursize, len);
+ c = f->head_chunk;
head_idx = head % f->size;
+ head_idx -= c->start_byte;
+ n_chunk = c->length - head_idx;
- if (PREDICT_TRUE (copy_here != 0))
+ if (n_chunk < to_copy)
{
- /* number of bytes in first copy segment */
- first_copy_bytes = f->size - head_idx;
- if (first_copy_bytes < total_copy_bytes)
+ clib_memcpy_fast (dst, &c->data[head_idx], n_chunk);
+ while ((to_copy -= n_chunk))
{
- clib_memcpy_fast (copy_here, &f->data[head_idx], first_copy_bytes);
- /* number of bytes in second copy segment */
- second_copy_bytes = total_copy_bytes - first_copy_bytes;
- /* wrap around */
- clib_memcpy_fast (copy_here + first_copy_bytes,
- &f->data[0], second_copy_bytes);
+ c = c->next;
+ n_chunk = clib_min (c->length, to_copy);
+ clib_memcpy_fast (dst + (len - to_copy), &c->data[0], n_chunk);
}
- else
- {
- clib_memcpy_fast (copy_here, &f->data[head_idx], total_copy_bytes);
- }
- head += total_copy_bytes;
+ f->head_chunk = c;
}
else
{
- ASSERT (0);
- /* Account for a zero-copy dequeue done elsewhere */
- head += max_bytes;
+ clib_memcpy_fast (dst, &c->data[head_idx], to_copy);
}
+ head += len;
- ASSERT (cursize >= total_copy_bytes);
+ ASSERT (cursize >= to_copy);
/* store-rel: consumer owned index (paired with load-acq in producer) */
clib_atomic_store_rel_n (&f->head, head);
- return total_copy_bytes;
+ return len;
}
#ifndef CLIB_MARCH_VARIANT
@@ -696,11 +691,10 @@ svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
#endif
CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset,
- u32 max_bytes, u8 * copy_here)
+ u32 len, u8 * dst)
{
- u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
- u32 tail, head, cursize;
- u32 relative_head_idx;
+ u32 to_copy, n_chunk, tail, head, cursize, head_idx;
+ svm_fifo_chunk_t *c;
f_load_head_tail_cons (f, &head, &tail);
@@ -710,33 +704,29 @@ CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset,
if (PREDICT_FALSE (cursize < relative_offset))
return -2; /* nothing in the fifo */
- relative_head_idx = (head + relative_offset) % f->size;
+ to_copy = len = clib_min (cursize - relative_offset, len);
- /* number of bytes we're going to copy */
- total_copy_bytes = ((cursize - relative_offset) < max_bytes) ?
- cursize - relative_offset : max_bytes;
+ c = f->head_chunk;
+ head_idx = (head + relative_offset) % f->size;
+ head_idx -= c->start_byte;
+ n_chunk = c->length - head_idx;
- if (PREDICT_TRUE (copy_here != 0))
+ if (n_chunk < to_copy)
{
- /* number of bytes in first copy segment */
- first_copy_bytes = f->size - relative_head_idx;
- if (first_copy_bytes < total_copy_bytes)
- {
- clib_memcpy_fast (copy_here, &f->data[relative_head_idx],
- first_copy_bytes);
-
- /* number of bytes in second copy segment */
- second_copy_bytes = total_copy_bytes - first_copy_bytes;
- clib_memcpy_fast (copy_here + first_copy_bytes, &f->data[0],
- second_copy_bytes);
- }
- else
+ clib_memcpy_fast (dst, &c->data[head_idx], n_chunk);
+ while ((to_copy -= n_chunk))
{
- clib_memcpy_fast (copy_here, &f->data[relative_head_idx],
- total_copy_bytes);
+ c = c->next;
+ n_chunk = clib_min (c->length, to_copy);
+ clib_memcpy_fast (dst + (len - to_copy), &c->data[0], n_chunk);
}
+ f->head_chunk = c;
+ }
+ else
+ {
+ clib_memcpy_fast (dst, &c->data[head_idx], to_copy);
}
- return total_copy_bytes;
+ return len;
}
#ifndef CLIB_MARCH_VARIANT
@@ -805,14 +795,14 @@ svm_fifo_segments (svm_fifo_t * f, svm_fifo_segment_t * fs)
if (tail < head)
{
fs[0].len = f->size - head_idx;
- fs[0].data = f->data + head_idx;
+ fs[0].data = f->head_chunk->data + head_idx;
fs[1].len = cursize - fs[0].len;
- fs[1].data = f->data;
+ fs[1].data = f->head_chunk->data;
}
else
{
fs[0].len = cursize;
- fs[0].data = f->data + head_idx;
+ fs[0].data = f->head_chunk->data + head_idx;
fs[1].len = 0;
fs[1].data = 0;
}
@@ -828,7 +818,7 @@ svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs)
head = f->head;
head_idx = head % f->size;
- ASSERT (fs[0].data == f->data + head_idx);
+ ASSERT (fs[0].data == f->head_chunk->data + head_idx);
head += fs[0].len + fs[1].len;
/* store-rel: consumer owned index (paired with load-acq in producer) */
clib_atomic_store_rel_n (&f->head, head);
@@ -839,7 +829,7 @@ void
svm_fifo_clone (svm_fifo_t * df, svm_fifo_t * sf)
{
u32 head, tail;
- clib_memcpy_fast (df->data, sf->data, sf->size);
+ clib_memcpy_fast (df->head_chunk->data, sf->head_chunk->data, sf->size);
f_load_head_tail_all_acq (sf, &head, &tail);
clib_atomic_store_rel_n (&df->head, head);
diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h
index 94a05938d26..f311576f5af 100644
--- a/src/svm/svm_fifo.h
+++ b/src/svm/svm_fifo.h
@@ -60,11 +60,21 @@ typedef struct
u32 action;
} svm_fifo_trace_elem_t;
+typedef struct svm_fifo_chunk_
+{
+ u32 start_byte;
+ u32 length;
+ struct svm_fifo_chunk_ *next;
+ u8 data[0];
+} svm_fifo_chunk_t;
+
typedef struct _svm_fifo
{
CLIB_CACHE_LINE_ALIGN_MARK (shared_first);
- u32 size; /**< size of the fifo(must be power of 2) */
+ u32 size; /**< size of the fifo */
u32 nitems; /**< usable size(size-1) */
+ struct _svm_fifo *next; /**< next in freelist/active chain */
+ struct _svm_fifo *prev; /**< prev in active chain */
CLIB_CACHE_LINE_ALIGN_MARK (shared_second);
volatile u32 has_event; /**< non-zero if deq event exists */
@@ -81,17 +91,17 @@ typedef struct _svm_fifo
CLIB_CACHE_LINE_ALIGN_MARK (consumer);
u32 head;
+ svm_fifo_chunk_t *head_chunk;
volatile u32 want_tx_ntf; /**< producer wants nudge */
volatile u32 has_tx_ntf;
CLIB_CACHE_LINE_ALIGN_MARK (producer);
u32 tail;
+ svm_fifo_chunk_t *tail_chunk;
ooo_segment_t *ooo_segments; /**< Pool of ooo segments */
u32 ooos_list_head; /**< Head of out-of-order linked-list */
u32 ooos_newest; /**< Last segment to have been updated */
- struct _svm_fifo *next; /**< next in freelist/active chain */
- struct _svm_fifo *prev; /**< prev in active chain */
volatile u8 n_subscribers;
u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS];
@@ -99,7 +109,7 @@ typedef struct _svm_fifo
svm_fifo_trace_elem_t *trace;
#endif
- CLIB_CACHE_LINE_ALIGN_MARK (data);
+ svm_fifo_chunk_t default_chunk;
} svm_fifo_t;
typedef enum
@@ -305,6 +315,7 @@ svm_fifo_unset_event (svm_fifo_t * f)
}
svm_fifo_t *svm_fifo_create (u32 data_size_in_bytes);
+void svm_fifo_init (svm_fifo_t * f, u32 size);
void svm_fifo_free (svm_fifo_t * f);
int svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
@@ -373,14 +384,16 @@ always_inline u8 *
svm_fifo_head (svm_fifo_t * f)
{
/* load-relaxed: consumer owned index */
- return (f->data + (f->head % f->size));
+ return (f->head_chunk->data
+ + ((f->head % f->size) - f->head_chunk->start_byte));
}
always_inline u8 *
svm_fifo_tail (svm_fifo_t * f)
{
/* load-relaxed: producer owned index */
- return (f->data + (f->tail % f->size));
+ return (f->tail_chunk->data
+ + ((f->tail % f->size) - f->tail_chunk->start_byte));
}
static inline void
diff --git a/src/svm/svm_fifo_segment.c b/src/svm/svm_fifo_segment.c
index 94bf6a5e73e..90bf593f74c 100644
--- a/src/svm/svm_fifo_segment.c
+++ b/src/svm/svm_fifo_segment.c
@@ -16,6 +16,23 @@
#include <svm/svm_fifo_segment.h>
static void
+fifo_init_for_segment (svm_fifo_segment_header_t * fsh, u8 * fifo_space,
+ u32 size, u32 freelist_index)
+{
+ svm_fifo_t *f;
+
+ f = (svm_fifo_t *) fifo_space;
+ f->freelist_index = freelist_index;
+ f->default_chunk.next = &f->default_chunk;
+ f->default_chunk.start_byte = 0;
+ f->default_chunk.length = size;
+ f->head_chunk = &f->default_chunk;
+ f->tail_chunk = &f->default_chunk;
+ f->next = fsh->free_fifos[freelist_index];
+ fsh->free_fifos[freelist_index] = f;
+}
+
+static void
allocate_new_fifo_chunk (svm_fifo_segment_header_t * fsh,
u32 data_size_in_bytes, int chunk_size)
{
@@ -43,14 +60,11 @@ allocate_new_fifo_chunk (svm_fifo_segment_header_t * fsh,
return;
/* Carve fifo space */
- f = (svm_fifo_t *) fifo_space;
for (i = 0; i < chunk_size; i++)
{
- f->freelist_index = freelist_index;
- f->next = fsh->free_fifos[freelist_index];
- fsh->free_fifos[freelist_index] = f;
+ fifo_init_for_segment (fsh, fifo_space, rounded_data_size,
+ freelist_index);
fifo_space += sizeof (*f) + rounded_data_size;
- f = (svm_fifo_t *) fifo_space;
}
}
@@ -149,24 +163,18 @@ svm_fifo_segment_preallocate_fifo_pairs (svm_fifo_segment_private_t * s,
}
/* Carve rx fifo space */
- f = (svm_fifo_t *) rx_fifo_space;
for (i = 0; i < pairs_to_allocate; i++)
{
- f->freelist_index = rx_freelist_index;
- f->next = fsh->free_fifos[rx_freelist_index];
- fsh->free_fifos[rx_freelist_index] = f;
+ fifo_init_for_segment (fsh, rx_fifo_space, rx_rounded_data_size,
+ rx_freelist_index);
rx_fifo_space += sizeof (*f) + rx_rounded_data_size;
- f = (svm_fifo_t *) rx_fifo_space;
}
/* Carve tx fifo space */
- f = (svm_fifo_t *) tx_fifo_space;
for (i = 0; i < pairs_to_allocate; i++)
{
- f->freelist_index = tx_freelist_index;
- f->next = fsh->free_fifos[tx_freelist_index];
- fsh->free_fifos[tx_freelist_index] = f;
+ fifo_init_for_segment (fsh, tx_fifo_space, tx_rounded_data_size,
+ tx_freelist_index);
tx_fifo_space += sizeof (*f) + tx_rounded_data_size;
- f = (svm_fifo_t *) tx_fifo_space;
}
/* Account for the pairs allocated */
@@ -381,16 +389,7 @@ svm_fifo_segment_alloc_fifo (svm_fifo_segment_private_t * fs,
fsh->free_fifos[freelist_index] = f->next;
/* (re)initialize the fifo, as in svm_fifo_create */
clib_memset (f, 0, sizeof (*f));
- f->size = (1 << (max_log2 (data_size_in_bytes)));
- /*
- * usable size of the fifo set to rounded_data_size - 1
- * to differentiate between free fifo and empty fifo.
- */
- f->nitems = f->size - 1;
- f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
- f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
- f->refcnt = 1;
- f->freelist_index = freelist_index;
+ svm_fifo_init (f, data_size_in_bytes);
goto found;
}
break;
diff --git a/src/vnet/session/application_worker.c b/src/vnet/session/application_worker.c
index dd1088e2cbb..1dae5a55a45 100644
--- a/src/vnet/session/application_worker.c
+++ b/src/vnet/session/application_worker.c
@@ -375,7 +375,6 @@ app_worker_own_session (app_worker_t * app_wrk, session_t * s)
if (!svm_fifo_is_empty_cons (txf))
svm_fifo_clone (s->tx_fifo, txf);
-
segment_manager_dealloc_fifos (rxf, txf);
return 0;