vlib: add buffer cloning support

Change-Id: I50070611af15b2b4cc29664a8bee4f821ac3c835
Signed-off-by: Damjan Marion <damarion@cisco.com>

2 files changed, 39 insertions, 13 deletions

diff --git a/src/vnet/devices/dpdk/buffer.c b/src/vnet/devices/dpdk/buffer.c
index 007093e493a..f95d4cb5c38 100644
--- a/src/vnet/devices/dpdk/buffer.c
+++ b/src/vnet/devices/dpdk/buffer.c
@@ -79,20 +79,46 @@
 STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
 	       "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
 
+static_always_inline void
+dpdk_rte_pktmbuf_free (vlib_main_t * vm, vlib_buffer_t * b)
+{
+  vlib_buffer_t *hb = b;
+  struct rte_mbuf *mb;
+  u32 next, flags;
+  mb = rte_mbuf_from_vlib_buffer (hb);
+
+next:
+  flags = b->flags;
+  next = b->next_buffer;
+  mb = rte_mbuf_from_vlib_buffer (b);
+
+  if (PREDICT_FALSE (b->n_add_refs))
+    {
+      rte_mbuf_refcnt_update (mb, b->n_add_refs);
+      b->n_add_refs = 0;
+    }
+
+  rte_pktmbuf_free_seg (mb);
+
+  if (flags & VLIB_BUFFER_NEXT_PRESENT)
+    {
+      b = vlib_get_buffer (vm, next);
+      goto next;
+    }
+}
+
 static void
 del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
 {
   u32 i;
-  struct rte_mbuf *mb;
   vlib_buffer_t *b;
 
   for (i = 0; i < vec_len (f->buffers); i++)
     {
       b = vlib_get_buffer (vm, f->buffers[i]);
-      mb = rte_mbuf_from_vlib_buffer (b);
-      ASSERT (rte_mbuf_refcnt_read (mb) == 1);
-      rte_pktmbuf_free (mb);
+      dpdk_rte_pktmbuf_free (vm, b);
     }
+
   vec_free (f->name);
   vec_free (f->buffers);
 }
@@ -325,7 +351,6 @@ vlib_buffer_free_inline (vlib_main_t * vm,
   for (i = 0; i < n_buffers; i++)
     {
       vlib_buffer_t *b;
-      struct rte_mbuf *mb;
 
       b = vlib_get_buffer (vm, buffers[i]);
 
@@ -351,11 +376,7 @@ vlib_buffer_free_inline (vlib_main_t * vm,
       else
 	{
 	  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
-	    {
-	      mb = rte_mbuf_from_vlib_buffer (b);
-	      ASSERT (rte_mbuf_refcnt_read (mb) == 1);
-	      rte_pktmbuf_free (mb);
-	    }
+	    dpdk_rte_pktmbuf_free (vm, b);
 	}
     }
   if (vec_len (bm->announce_list))
diff --git a/src/vnet/devices/dpdk/device.c b/src/vnet/devices/dpdk/device.c
index c9d9a567b7a..17397900c59 100644
--- a/src/vnet/devices/dpdk/device.c
+++ b/src/vnet/devices/dpdk/device.c
@@ -168,13 +168,11 @@ dpdk_validate_rte_mbuf (vlib_main_t * vm, vlib_buffer_t * b,
 	{
 	  b2 = vlib_get_buffer (vm, b2->next_buffer);
 	  mb = rte_mbuf_from_vlib_buffer (b2);
-	  last_mb->next = mb;
-	  last_mb = mb;
 	  rte_pktmbuf_reset (mb);
 	}
     }
 
-  first_mb = mb = rte_mbuf_from_vlib_buffer (b);
+  last_mb = first_mb = mb = rte_mbuf_from_vlib_buffer (b);
   first_mb->nb_segs = 1;
   mb->data_len = b->current_length;
   mb->pkt_len = maybe_multiseg ? vlib_buffer_length_in_chain (vm, b) :
@@ -185,10 +183,17 @@ dpdk_validate_rte_mbuf (vlib_main_t * vm, vlib_buffer_t * b,
     {
       b = vlib_get_buffer (vm, b->next_buffer);
       mb = rte_mbuf_from_vlib_buffer (b);
+      last_mb->next = mb;
+      last_mb = mb;
       mb->data_len = b->current_length;
       mb->pkt_len = b->current_length;
       mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
       first_mb->nb_segs++;
+      if (PREDICT_FALSE (b->n_add_refs))
+	{
+	  rte_mbuf_refcnt_update (mb, b->n_add_refs);
+	  b->n_add_refs = 0;
+	}
     }
 }