summaryrefslogtreecommitdiffstats
path: root/src/svm/svm_fifo.c
diff options
context:
space:
mode:
authorFlorin Coras <fcoras@cisco.com>2019-04-09 18:29:14 -0700
committerDamjan Marion <dmarion@me.com>2019-04-18 13:16:18 +0000
commit0a846806742922d98f1aff9cf1370715e57a2ae5 (patch)
tree5214807e20731453254635ed2d9afa9649eb4469 /src/svm/svm_fifo.c
parent5a8844bdbf4b055812cce2d7755a175b2cc90b75 (diff)
svm: allow indirect fifo data chunks
Fifos can use multiple memory chunks for simple read/write operations. Adding/removing chunks after assignment not yet supported. Change-Id: I2aceab6aea78059d74e0d3a9993c40d5196d077b Signed-off-by: Florin Coras <fcoras@cisco.com>
Diffstat (limited to 'src/svm/svm_fifo.c')
-rw-r--r--src/svm/svm_fifo.c258
1 files changed, 124 insertions, 134 deletions
diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index 918ad77956e..de1997452d5 100644
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -113,8 +113,7 @@ svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose)
#endif
dummy_fifo = svm_fifo_create (f->size);
- clib_memset (f->data, 0xFF, f->nitems);
-
+ clib_memset (f->head_chunk->data, 0xFF, f->nitems);
vec_validate (data, f->nitems);
for (i = 0; i < vec_len (data); i++)
data[i] = i;
@@ -204,6 +203,26 @@ format_svm_fifo (u8 * s, va_list * args)
return s;
}
+void
+svm_fifo_init (svm_fifo_t * f, u32 size)
+{
+ f->size = size;
+ /*
+ * usable size of the fifo set to rounded_data_size - 1
+ * to differentiate between free fifo and empty fifo.
+ */
+ f->nitems = f->size - 1;
+ f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
+ f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
+ f->segment_index = SVM_FIFO_INVALID_INDEX;
+ f->refcnt = 1;
+ f->head_chunk = &f->default_chunk;
+ f->tail_chunk = &f->default_chunk;
+ f->default_chunk.next = &f->default_chunk;
+ f->default_chunk.start_byte = 0;
+ f->default_chunk.length = f->size;
+}
+
/** create an svm fifo, in the current heap. Fails vs blow up the process */
svm_fifo_t *
svm_fifo_create (u32 data_size_in_bytes)
@@ -219,17 +238,8 @@ svm_fifo_create (u32 data_size_in_bytes)
return 0;
clib_memset (f, 0, sizeof (*f));
- f->size = rounded_data_size;
- /*
- * usable size of the fifo set to rounded_data_size - 1
- * to differentiate between free fifo and empty fifo.
- */
- f->nitems = f->size - 1;
- f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
- f->ct_session_index = SVM_FIFO_INVALID_SESSION_INDEX;
- f->segment_index = SVM_FIFO_INVALID_INDEX;
- f->refcnt = 1;
- return (f);
+ svm_fifo_init (f, data_size_in_bytes);
+ return f;
}
void
@@ -296,7 +306,9 @@ ooo_segment_add (svm_fifo_t * f, u32 offset, u32 head, u32 tail, u32 length)
u32 new_index, s_end_pos, s_index;
u32 offset_pos, offset_end_pos;
- ASSERT (offset + length <= ooo_segment_distance_from_tail (f, head, tail));
+ ASSERT (offset + length <= ooo_segment_distance_from_tail (f, head, tail)
+ || head == tail);
+
offset_pos = tail + offset;
offset_end_pos = tail + offset + length;
@@ -469,11 +481,11 @@ ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued, u32 * tail)
return bytes;
}
-CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
- const u8 * copy_from_here)
+CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 len,
+ const u8 * src)
{
- u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
- u32 tail, head, free_count, tail_idx;
+ u32 n_chunk, to_copy, tail, head, free_count, tail_idx;
+ svm_fifo_chunk_t *c;
f_load_head_tail_prod (f, &head, &tail);
@@ -486,50 +498,43 @@ CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
return SVM_FIFO_FULL;
/* number of bytes we're going to copy */
- total_copy_bytes = free_count < max_bytes ? free_count : max_bytes;
+ to_copy = len = clib_min (free_count, len);
+ c = f->tail_chunk;
tail_idx = tail % f->size;
+ ASSERT (tail_idx >= c->start_byte);
+ tail_idx -= c->start_byte;
+ n_chunk = c->length - tail_idx;
- if (PREDICT_TRUE (copy_from_here != 0))
+ if (n_chunk < to_copy)
{
- first_copy_bytes = f->size - tail_idx;
- if (first_copy_bytes < total_copy_bytes)
- {
- clib_memcpy_fast (&f->data[tail_idx], copy_from_here,
- first_copy_bytes);
- /* number of bytes in second copy segment */
- second_copy_bytes = total_copy_bytes - first_copy_bytes;
- /* wrap around */
- clib_memcpy_fast (&f->data[0],
- copy_from_here + first_copy_bytes,
- second_copy_bytes);
- }
- else
+ clib_memcpy_fast (&c->data[tail_idx], src, n_chunk);
+ while ((to_copy -= n_chunk))
{
- clib_memcpy_fast (&f->data[tail_idx], copy_from_here,
- total_copy_bytes);
+ c = c->next;
+ n_chunk = clib_min (c->length, to_copy);
+ clib_memcpy_fast (&c->data[0], src + (len - to_copy), n_chunk);
}
- tail += total_copy_bytes;
+ f->tail_chunk = c;
}
else
{
- ASSERT (0);
- /* Account for a zero-copy enqueue done elsewhere */
- tail += max_bytes;
+ clib_memcpy_fast (&c->data[tail_idx], src, to_copy);
}
+ tail += len;
- svm_fifo_trace_add (f, head, total_copy_bytes, 2);
+ svm_fifo_trace_add (f, head, n_total, 2);
/* collect out-of-order segments */
if (PREDICT_FALSE (f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX))
- total_copy_bytes += ooo_segment_try_collect (f, total_copy_bytes, &tail);
+ len += ooo_segment_try_collect (f, len, &tail);
- ASSERT (total_copy_bytes <= free_count);
+ ASSERT (len <= free_count);
/* store-rel: producer owned index (paired with load-acq in consumer) */
clib_atomic_store_rel_n (&f->tail, tail);
- return total_copy_bytes;
+ return len;
}
#ifndef CLIB_MARCH_VARIANT
@@ -550,11 +555,10 @@ svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
* Returns -1 if none of the segment was copied due to lack of space
*/
CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f,
- u32 offset, u32 required_bytes, u8 * copy_from_here)
+ u32 offset, u32 len, u8 * src)
{
- u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
- u32 tail_offset;
- u32 tail, head, free_count, tail_offset_idx;
+ u32 to_copy, n_chunk, tail, head, free_count, tail_offset_idx;
+ svm_fifo_chunk_t *c;
f_load_head_tail_prod (f, &head, &tail);
@@ -562,41 +566,35 @@ CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f,
free_count = f_free_count (f, head, tail);
/* will this request fit? */
- if ((required_bytes + offset) > free_count)
+ if ((len + offset) > free_count)
return -1;
f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
- ASSERT (required_bytes < f->nitems);
+ ASSERT (len < f->nitems);
+ svm_fifo_trace_add (f, offset, len, 1);
- tail_offset = tail + offset;
- tail_offset_idx = tail_offset % f->size;
+ ooo_segment_add (f, offset, head, tail, len);
- svm_fifo_trace_add (f, offset, required_bytes, 1);
+ c = f->tail_chunk;
+ tail_offset_idx = (tail + offset) % f->size;
+ tail_offset_idx -= c->start_byte;
+ n_chunk = c->length - tail_offset_idx;
+ to_copy = len;
- ooo_segment_add (f, offset, head, tail, required_bytes);
-
- /* number of bytes we're going to copy */
- total_copy_bytes = required_bytes;
-
- /* number of bytes in first copy segment */
- first_copy_bytes = f->size - tail_offset_idx;
-
- if (first_copy_bytes < total_copy_bytes)
+ if (n_chunk < to_copy)
{
- clib_memcpy_fast (&f->data[tail_offset_idx], copy_from_here,
- first_copy_bytes);
-
- /* number of bytes in second copy segment */
- second_copy_bytes = total_copy_bytes - first_copy_bytes;
- /* wrap around */
- clib_memcpy_fast (&f->data[0],
- copy_from_here + first_copy_bytes, second_copy_bytes);
+ clib_memcpy_fast (&c->data[tail_offset_idx], src, n_chunk);
+ while ((to_copy -= n_chunk))
+ {
+ c = c->next;
+ n_chunk = clib_min (c->length, to_copy);
+ clib_memcpy_fast (&c->data[0], src + (len - to_copy), n_chunk);
+ }
}
else
{
- clib_memcpy_fast (&f->data[tail_offset_idx], copy_from_here,
- total_copy_bytes);
+ clib_memcpy_fast (&c->data[tail_offset_idx], src, len);
}
return 0;
@@ -616,28 +614,32 @@ svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 required_bytes,
void
svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len)
{
- u32 first_chunk;
+ u32 n_chunk;
u32 head, tail, head_idx;
+ svm_fifo_chunk_t *c;
+
+ ASSERT (len <= f->nitems);
f_load_head_tail_cons (f, &head, &tail);
+ c = f->head_chunk;
head_idx = head % f->size;
- first_chunk = f->size - (head_idx);
- ASSERT (len <= f->nitems);
- if (len <= first_chunk)
- clib_memcpy_fast (&f->data[head_idx], data, len);
+ head_idx -= c->start_byte;
+ n_chunk = c->length - head_idx;
+ if (len <= n_chunk)
+ clib_memcpy_fast (&c->data[head_idx], data, len);
else
{
- clib_memcpy_fast (&f->data[head_idx], data, first_chunk);
- clib_memcpy_fast (&f->data[0], data + first_chunk, len - first_chunk);
+ clib_memcpy_fast (&c->data[head_idx], data, n_chunk);
+ clib_memcpy_fast (&c->next->data[0], data + n_chunk, len - n_chunk);
}
}
#endif
-CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
- u8 * copy_here)
+CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 len,
+ u8 * dst)
{
- u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
- u32 tail, head, cursize, head_idx;
+ u32 to_copy, n_chunk, tail, head, cursize, head_idx;
+ svm_fifo_chunk_t *c;
f_load_head_tail_cons (f, &head, &tail);
@@ -647,42 +649,35 @@ CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
if (PREDICT_FALSE (cursize == 0))
return -2; /* nothing in the fifo */
- /* number of bytes we're going to copy */
- total_copy_bytes = (cursize < max_bytes) ? cursize : max_bytes;
+ to_copy = len = clib_min (cursize, len);
+ c = f->head_chunk;
head_idx = head % f->size;
+ head_idx -= c->start_byte;
+ n_chunk = c->length - head_idx;
- if (PREDICT_TRUE (copy_here != 0))
+ if (n_chunk < to_copy)
{
- /* number of bytes in first copy segment */
- first_copy_bytes = f->size - head_idx;
- if (first_copy_bytes < total_copy_bytes)
+ clib_memcpy_fast (dst, &c->data[head_idx], n_chunk);
+ while ((to_copy -= n_chunk))
{
- clib_memcpy_fast (copy_here, &f->data[head_idx], first_copy_bytes);
- /* number of bytes in second copy segment */
- second_copy_bytes = total_copy_bytes - first_copy_bytes;
- /* wrap around */
- clib_memcpy_fast (copy_here + first_copy_bytes,
- &f->data[0], second_copy_bytes);
+ c = c->next;
+ n_chunk = clib_min (c->length, to_copy);
+ clib_memcpy_fast (dst + (len - to_copy), &c->data[0], n_chunk);
}
- else
- {
- clib_memcpy_fast (copy_here, &f->data[head_idx], total_copy_bytes);
- }
- head += total_copy_bytes;
+ f->head_chunk = c;
}
else
{
- ASSERT (0);
- /* Account for a zero-copy dequeue done elsewhere */
- head += max_bytes;
+ clib_memcpy_fast (dst, &c->data[head_idx], to_copy);
}
+ head += len;
- ASSERT (cursize >= total_copy_bytes);
+ ASSERT (cursize >= to_copy);
/* store-rel: consumer owned index (paired with load-acq in producer) */
clib_atomic_store_rel_n (&f->head, head);
- return total_copy_bytes;
+ return len;
}
#ifndef CLIB_MARCH_VARIANT
@@ -696,11 +691,10 @@ svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
#endif
CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset,
- u32 max_bytes, u8 * copy_here)
+ u32 len, u8 * dst)
{
- u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
- u32 tail, head, cursize;
- u32 relative_head_idx;
+ u32 to_copy, n_chunk, tail, head, cursize, head_idx;
+ svm_fifo_chunk_t *c;
f_load_head_tail_cons (f, &head, &tail);
@@ -710,33 +704,29 @@ CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset,
if (PREDICT_FALSE (cursize < relative_offset))
return -2; /* nothing in the fifo */
- relative_head_idx = (head + relative_offset) % f->size;
+ to_copy = len = clib_min (cursize - relative_offset, len);
- /* number of bytes we're going to copy */
- total_copy_bytes = ((cursize - relative_offset) < max_bytes) ?
- cursize - relative_offset : max_bytes;
+ c = f->head_chunk;
+ head_idx = (head + relative_offset) % f->size;
+ head_idx -= c->start_byte;
+ n_chunk = c->length - head_idx;
- if (PREDICT_TRUE (copy_here != 0))
+ if (n_chunk < to_copy)
{
- /* number of bytes in first copy segment */
- first_copy_bytes = f->size - relative_head_idx;
- if (first_copy_bytes < total_copy_bytes)
- {
- clib_memcpy_fast (copy_here, &f->data[relative_head_idx],
- first_copy_bytes);
-
- /* number of bytes in second copy segment */
- second_copy_bytes = total_copy_bytes - first_copy_bytes;
- clib_memcpy_fast (copy_here + first_copy_bytes, &f->data[0],
- second_copy_bytes);
- }
- else
+ clib_memcpy_fast (dst, &c->data[head_idx], n_chunk);
+ while ((to_copy -= n_chunk))
{
- clib_memcpy_fast (copy_here, &f->data[relative_head_idx],
- total_copy_bytes);
+ c = c->next;
+ n_chunk = clib_min (c->length, to_copy);
+ clib_memcpy_fast (dst + (len - to_copy), &c->data[0], n_chunk);
}
+ f->head_chunk = c;
+ }
+ else
+ {
+ clib_memcpy_fast (dst, &c->data[head_idx], to_copy);
}
- return total_copy_bytes;
+ return len;
}
#ifndef CLIB_MARCH_VARIANT
@@ -805,14 +795,14 @@ svm_fifo_segments (svm_fifo_t * f, svm_fifo_segment_t * fs)
if (tail < head)
{
fs[0].len = f->size - head_idx;
- fs[0].data = f->data + head_idx;
+ fs[0].data = f->head_chunk->data + head_idx;
fs[1].len = cursize - fs[0].len;
- fs[1].data = f->data;
+ fs[1].data = f->head_chunk->data;
}
else
{
fs[0].len = cursize;
- fs[0].data = f->data + head_idx;
+ fs[0].data = f->head_chunk->data + head_idx;
fs[1].len = 0;
fs[1].data = 0;
}
@@ -828,7 +818,7 @@ svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs)
head = f->head;
head_idx = head % f->size;
- ASSERT (fs[0].data == f->data + head_idx);
+ ASSERT (fs[0].data == f->head_chunk->data + head_idx);
head += fs[0].len + fs[1].len;
/* store-rel: consumer owned index (paired with load-acq in producer) */
clib_atomic_store_rel_n (&f->head, head);
@@ -839,7 +829,7 @@ void
svm_fifo_clone (svm_fifo_t * df, svm_fifo_t * sf)
{
u32 head, tail;
- clib_memcpy_fast (df->data, sf->data, sf->size);
+ clib_memcpy_fast (df->head_chunk->data, sf->head_chunk->data, sf->size);
f_load_head_tail_all_acq (sf, &head, &tail);
clib_atomic_store_rel_n (&df->head, head);