svm: support for multi-segment enqueues

Type: feature

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I06c7022a6afbb146b23cbd3a430497ec9e8be73d

1 files changed, 81 insertions, 0 deletions

diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index d357eb6b8a9..f1ac8d49b92 100644
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -955,6 +955,87 @@ svm_fifo_enqueue_nocopy (svm_fifo_t * f, u32 len)
   clib_atomic_store_rel_n (&f->tail, tail);
 }
 
+int
+svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[],
+			   u32 n_segs, u8 allow_partial)
+{
+  u32 tail, head, free_count, len = 0, i;
+  svm_fifo_chunk_t *old_tail_c;
+
+  f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;
+
+  f_load_head_tail_prod (f, &head, &tail);
+
+  /* free space in fifo can only increase during enqueue: SPSC */
+  free_count = f_free_count (f, head, tail);
+
+  if (PREDICT_FALSE (free_count == 0))
+    return SVM_FIFO_EFULL;
+
+  for (i = 0; i < n_segs; i++)
+    len += segs[i].len;
+
+  old_tail_c = f->tail_chunk;
+
+  if (!allow_partial)
+    {
+      if (PREDICT_FALSE (free_count < len))
+	return SVM_FIFO_EFULL;
+
+      if (f_pos_gt (tail + len, f_chunk_end (f->end_chunk)))
+	{
+	  if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, len)))
+	    return SVM_FIFO_EGROW;
+	}
+
+      for (i = 0; i < n_segs; i++)
+	{
+	  svm_fifo_copy_to_chunk (f, f->tail_chunk, tail, segs[i].data,
+				  segs[i].len, &f->tail_chunk);
+	  tail += segs[i].len;
+	}
+    }
+  else
+    {
+      len = clib_min (free_count, len);
+
+      if (f_pos_gt (tail + len, f_chunk_end (f->end_chunk)))
+	{
+	  if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, len)))
+	    {
+	      len = f_chunk_end (f->end_chunk) - tail;
+	      if (!len)
+		return SVM_FIFO_EGROW;
+	    }
+	}
+
+      i = 0;
+      while (len)
+	{
+	  u32 to_copy = clib_min (segs[i].len, len);
+	  svm_fifo_copy_to_chunk (f, f->tail_chunk, tail, segs[i].data,
+				  to_copy, &f->tail_chunk);
+	  len -= to_copy;
+	  tail += to_copy;
+	  i++;
+	}
+    }
+
+  /* collect out-of-order segments */
+  if (PREDICT_FALSE (f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX))
+    {
+      len += ooo_segment_try_collect (f, len, &tail);
+      /* Tail chunk might've changed even if nothing was collected */
+      f->tail_chunk = f_lookup_clear_enq_chunks (f, old_tail_c, tail);
+      f->ooo_enq = 0;
+    }
+
+  /* store-rel: producer owned index (paired with load-acq in consumer) */
+  clib_atomic_store_rel_n (&f->tail, tail);
+
+  return len;
+}
+
 always_inline svm_fifo_chunk_t *
 f_unlink_chunks (svm_fifo_t * f, u32 end_pos, u8 maybe_ooo)
 {