dpdk: remove duplicate code in buffers.c

Change-Id: Idc17b4a32d40012556d5d8550942db0372ebf23d
Signed-off-by: Damjan Marion <damarion@cisco.com>

3 files changed, 78 insertions, 174 deletions

diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index 0b0e6054ec2..ea4960e2a6a 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -307,10 +307,10 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm,
 #define BUFFERS_PER_COPY (sizeof (vlib_copy_unit_t) / sizeof (u32))
 
 /* Make sure we have at least given number of unaligned buffers. */
-static void
-fill_unaligned (vlib_main_t * vm,
-		vlib_buffer_free_list_t * free_list,
-		uword n_unaligned_buffers)
+void
+vlib_buffer_free_list_fill_unaligned (vlib_main_t * vm,
+				      vlib_buffer_free_list_t * free_list,
+				      uword n_unaligned_buffers)
 {
   word la = vec_len (free_list->aligned_buffers);
   word lu = vec_len (free_list->unaligned_buffers);
@@ -333,8 +333,8 @@ fill_unaligned (vlib_main_t * vm,
 }
 
 /* After free aligned buffers may not contain even sized chunks. */
-static void
-trim_aligned (vlib_buffer_free_list_t * f)
+void
+vlib_buffer_free_list_trim_aligned (vlib_buffer_free_list_t * f)
 {
   uword l, n_trim;
 
@@ -361,15 +361,15 @@ trim_aligned (vlib_buffer_free_list_t * f)
     }
 }
 
-static void
-merge_free_lists (vlib_buffer_free_list_t * dst,
-		  vlib_buffer_free_list_t * src)
+void
+vlib_buffer_merge_free_lists (vlib_buffer_free_list_t * dst,
+			      vlib_buffer_free_list_t * src)
 {
   uword l;
   u32 *d;
 
-  trim_aligned (src);
-  trim_aligned (dst);
+  vlib_buffer_free_list_trim_aligned (src);
+  vlib_buffer_free_list_trim_aligned (dst);
 
   l = vec_len (src->aligned_buffers);
   if (l > 0)
@@ -388,16 +388,6 @@ merge_free_lists (vlib_buffer_free_list_t * dst,
     }
 }
 
-always_inline u32
-vlib_buffer_get_free_list_with_size (vlib_main_t * vm, u32 size)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-
-  size = vlib_buffer_round_size (size);
-  uword *p = hash_get (bm->free_list_by_size, size);
-  return p ? p[0] : ~0;
-}
-
 /* Add buffer free list. */
 static u32
 vlib_buffer_create_free_list_helper (vlib_main_t * vm,
@@ -537,8 +527,9 @@ vlib_buffer_delete_free_list_internal (vlib_main_t * vm, u32 free_list_index)
   merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes);
   if (merge_index != ~0 && merge_index != free_list_index)
     {
-      merge_free_lists (pool_elt_at_index (bm->buffer_free_list_pool,
-					   merge_index), f);
+      vlib_buffer_merge_free_lists (pool_elt_at_index
+				    (bm->buffer_free_list_pool, merge_index),
+				    f);
     }
 
   del_free_list (vm, f);
@@ -567,7 +558,7 @@ fill_free_list (vlib_main_t * vm,
   u32 *bi;
   u32 n_remaining, n_alloc, n_this_chunk;
 
-  trim_aligned (fl);
+  vlib_buffer_free_list_trim_aligned (fl);
 
   /* Already have enough free buffers on free list? */
   n = min_free_buffers - vec_len (fl->aligned_buffers);
@@ -666,7 +657,8 @@ alloc_from_free_list (vlib_main_t * vm,
   else
     n_unaligned_end = copy_alignment (dst + n_alloc_buffers);
 
-  fill_unaligned (vm, free_list, n_unaligned_start + n_unaligned_end);
+  vlib_buffer_free_list_fill_unaligned (vm, free_list,
+					n_unaligned_start + n_unaligned_end);
 
   u_len = vec_len (free_list->unaligned_buffers);
   u_src = free_list->unaligned_buffers + u_len - 1;
@@ -779,29 +771,6 @@ vlib_buffer_alloc_from_free_list_internal (vlib_main_t * vm,
   return alloc_from_free_list (vm, f, buffers, n_buffers);
 }
 
-always_inline void
-add_buffer_to_free_list (vlib_main_t * vm,
-			 vlib_buffer_free_list_t * f,
-			 u32 buffer_index, u8 do_init)
-{
-  vlib_buffer_t *b;
-  b = vlib_get_buffer (vm, buffer_index);
-  if (PREDICT_TRUE (do_init))
-    vlib_buffer_init_for_free_list (b, f);
-  vec_add1_aligned (f->aligned_buffers, buffer_index,
-		    sizeof (vlib_copy_unit_t));
-}
-
-always_inline vlib_buffer_free_list_t *
-buffer_get_free_list (vlib_main_t * vm, vlib_buffer_t * b, u32 * index)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-  u32 i;
-
-  *index = i = b->free_list_index;
-  return pool_elt_at_index (bm->buffer_free_list_pool, i);
-}
-
 void *
 vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp)
 {
@@ -845,7 +814,7 @@ vlib_buffer_free_inline (vlib_main_t * vm,
     vlib_buffer_t *b0;
 
     b0 = vlib_get_buffer (vm, bi0);
-    fl = buffer_get_free_list (vm, b0, &fi);
+    fl = vlib_buffer_get_buffer_free_list (vm, b0, &fi);
     if (fl->buffers_added_to_freelist_function)
       vec_add1 (announce_list, fl);
   }
@@ -926,7 +895,7 @@ again:
       fl0 = pool_elt_at_index (bm->buffer_free_list_pool, fi0);
       fl1 = pool_elt_at_index (bm->buffer_free_list_pool, fi1);
 
-      add_buffer_to_free_list (vm, fl0, bi0, free0);
+      vlib_buffer_add_to_free_list (vm, fl0, bi0, free0);
       if (PREDICT_FALSE (fl0->buffers_added_to_freelist_function != 0))
 	{
 	  int i;
@@ -946,7 +915,7 @@ again:
 	}
 
     no_fl1:
-      add_buffer_to_free_list (vm, fl1, bi1, free1);
+      vlib_buffer_add_to_free_list (vm, fl1, bi1, free1);
 
       /* Possibly change current free list. */
       if (fi0 != fi && fi1 != fi)
@@ -1003,7 +972,7 @@ again:
 
       fl0 = pool_elt_at_index (bm->buffer_free_list_pool, fi0);
 
-      add_buffer_to_free_list (vm, fl0, bi0, free0);
+      vlib_buffer_add_to_free_list (vm, fl0, bi0, free0);
       if (PREDICT_FALSE (fl0->buffers_added_to_freelist_function != 0))
 	{
 	  int i;
diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index 15d93c1657f..543a903c54d 100644
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -350,6 +350,41 @@ vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index)
 u32 vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
 					 char *fmt, ...);
 
+
+/* After free aligned buffers may not contain even sized chunks. */
+void vlib_buffer_free_list_trim_aligned (vlib_buffer_free_list_t * f);
+
+/* Merge two free lists */
+void vlib_buffer_merge_free_lists (vlib_buffer_free_list_t * dst,
+				   vlib_buffer_free_list_t * src);
+
+/* Make sure we have at least given number of unaligned buffers. */
+void vlib_buffer_free_list_fill_unaligned (vlib_main_t * vm,
+					   vlib_buffer_free_list_t *
+					   free_list,
+					   uword n_unaligned_buffers);
+
+always_inline u32
+vlib_buffer_get_free_list_with_size (vlib_main_t * vm, u32 size)
+{
+  vlib_buffer_main_t *bm = vm->buffer_main;
+
+  size = vlib_buffer_round_size (size);
+  uword *p = hash_get (bm->free_list_by_size, size);
+  return p ? p[0] : ~0;
+}
+
+always_inline vlib_buffer_free_list_t *
+vlib_buffer_get_buffer_free_list (vlib_main_t * vm, vlib_buffer_t * b,
+				  u32 * index)
+{
+  vlib_buffer_main_t *bm = vm->buffer_main;
+  u32 i;
+
+  *index = i = b->free_list_index;
+  return pool_elt_at_index (bm->buffer_free_list_pool, i);
+}
+
 always_inline vlib_buffer_free_list_t *
 vlib_buffer_get_free_list (vlib_main_t * vm, u32 free_list_index)
 {
@@ -675,6 +710,19 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * _dst,
 }
 
 always_inline void
+vlib_buffer_add_to_free_list (vlib_main_t * vm,
+			      vlib_buffer_free_list_t * f,
+			      u32 buffer_index, u8 do_init)
+{
+  vlib_buffer_t *b;
+  b = vlib_get_buffer (vm, buffer_index);
+  if (PREDICT_TRUE (do_init))
+    vlib_buffer_init_for_free_list (b, f);
+  vec_add1_aligned (f->aligned_buffers, buffer_index,
+		    sizeof (vlib_copy_unit_t));
+}
+
+always_inline void
 vlib_buffer_init_two_for_free_list (vlib_buffer_t * _dst0,
 				    vlib_buffer_t * _dst1,
 				    vlib_buffer_free_list_t * fl)
diff --git a/src/vnet/devices/dpdk/buffer.c b/src/vnet/devices/dpdk/buffer.c
index 214a916263c..038f46d9859 100644
--- a/src/vnet/devices/dpdk/buffer.c
+++ b/src/vnet/devices/dpdk/buffer.c
@@ -81,98 +81,6 @@ STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
 
 #define BUFFERS_PER_COPY (sizeof (vlib_copy_unit_t) / sizeof (u32))
 
-/* Make sure we have at least given number of unaligned buffers. */
-static void
-fill_unaligned (vlib_main_t * vm,
-		vlib_buffer_free_list_t * free_list,
-		uword n_unaligned_buffers)
-{
-  word la = vec_len (free_list->aligned_buffers);
-  word lu = vec_len (free_list->unaligned_buffers);
-
-  /* Aligned come in aligned copy-sized chunks. */
-  ASSERT (la % BUFFERS_PER_COPY == 0);
-
-  ASSERT (la >= n_unaligned_buffers);
-
-  while (lu < n_unaligned_buffers)
-    {
-      /* Copy 4 buffers from end of aligned vector to unaligned vector. */
-      vec_add (free_list->unaligned_buffers,
-	       free_list->aligned_buffers + la - BUFFERS_PER_COPY,
-	       BUFFERS_PER_COPY);
-      la -= BUFFERS_PER_COPY;
-      lu += BUFFERS_PER_COPY;
-    }
-  _vec_len (free_list->aligned_buffers) = la;
-}
-
-/* After free aligned buffers may not contain even sized chunks. */
-static void
-trim_aligned (vlib_buffer_free_list_t * f)
-{
-  uword l, n_trim;
-
-  /* Add unaligned to aligned before trim. */
-  l = vec_len (f->unaligned_buffers);
-  if (l > 0)
-    {
-      vec_add_aligned (f->aligned_buffers, f->unaligned_buffers, l,
-		       /* align */ sizeof (vlib_copy_unit_t));
-
-      _vec_len (f->unaligned_buffers) = 0;
-    }
-
-  /* Remove unaligned buffers from end of aligned vector and save for next trim. */
-  l = vec_len (f->aligned_buffers);
-  n_trim = l % BUFFERS_PER_COPY;
-  if (n_trim)
-    {
-      /* Trim aligned -> unaligned. */
-      vec_add (f->unaligned_buffers, f->aligned_buffers + l - n_trim, n_trim);
-
-      /* Remove from aligned. */
-      _vec_len (f->aligned_buffers) = l - n_trim;
-    }
-}
-
-static void
-merge_free_lists (vlib_buffer_free_list_t * dst,
-		  vlib_buffer_free_list_t * src)
-{
-  uword l;
-  u32 *d;
-
-  trim_aligned (src);
-  trim_aligned (dst);
-
-  l = vec_len (src->aligned_buffers);
-  if (l > 0)
-    {
-      vec_add2_aligned (dst->aligned_buffers, d, l,
-			/* align */ sizeof (vlib_copy_unit_t));
-      clib_memcpy (d, src->aligned_buffers, l * sizeof (d[0]));
-      vec_free (src->aligned_buffers);
-    }
-
-  l = vec_len (src->unaligned_buffers);
-  if (l > 0)
-    {
-      vec_add (dst->unaligned_buffers, src->unaligned_buffers, l);
-      vec_free (src->unaligned_buffers);
-    }
-}
-
-always_inline u32
-dpdk_buffer_get_free_list_with_size (vlib_main_t * vm, u32 size)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-
-  size = vlib_buffer_round_size (size);
-  uword *p = hash_get (bm->free_list_by_size, size);
-  return p ? p[0] : ~0;
-}
-
 static void
 del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
 {
@@ -212,11 +120,12 @@ dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index)
 
   f = vlib_buffer_get_free_list (vm, free_list_index);
 
-  merge_index = dpdk_buffer_get_free_list_with_size (vm, f->n_data_bytes);
+  merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes);
   if (merge_index != ~0 && merge_index != free_list_index)
     {
-      merge_free_lists (pool_elt_at_index (bm->buffer_free_list_pool,
-					   merge_index), f);
+      vlib_buffer_merge_free_lists (pool_elt_at_index
+				    (bm->buffer_free_list_pool, merge_index),
+				    f);
     }
 
   del_free_list (vm, f);
@@ -253,7 +162,7 @@ fill_free_list (vlib_main_t * vm,
   if (PREDICT_FALSE (rmp == 0))
     return 0;
 
-  trim_aligned (fl);
+  vlib_buffer_free_list_trim_aligned (fl);
 
   /* Already have enough free buffers on free list? */
   n = min_free_buffers - vec_len (fl->aligned_buffers);
@@ -333,7 +242,8 @@ alloc_from_free_list (vlib_main_t * vm,
   else
     n_unaligned_end = copy_alignment (dst + n_alloc_buffers);
 
-  fill_unaligned (vm, free_list, n_unaligned_start + n_unaligned_end);
+  vlib_buffer_free_list_fill_unaligned (vm, free_list,
+					n_unaligned_start + n_unaligned_end);
 
   u_len = vec_len (free_list->unaligned_buffers);
   u_src = free_list->unaligned_buffers + u_len - 1;
@@ -442,29 +352,6 @@ dpdk_buffer_alloc_from_free_list (vlib_main_t * vm,
   return alloc_from_free_list (vm, f, buffers, n_buffers);
 }
 
-always_inline void
-add_buffer_to_free_list (vlib_main_t * vm,
-			 vlib_buffer_free_list_t * f,
-			 u32 buffer_index, u8 do_init)
-{
-  vlib_buffer_t *b;
-  b = vlib_get_buffer (vm, buffer_index);
-  if (PREDICT_TRUE (do_init))
-    vlib_buffer_init_for_free_list (b, f);
-  vec_add1_aligned (f->aligned_buffers, buffer_index,
-		    sizeof (vlib_copy_unit_t));
-}
-
-always_inline vlib_buffer_free_list_t *
-buffer_get_free_list (vlib_main_t * vm, vlib_buffer_t * b, u32 * index)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-  u32 i;
-
-  *index = i = b->free_list_index;
-  return pool_elt_at_index (bm->buffer_free_list_pool, i);
-}
-
 static_always_inline void
 vlib_buffer_free_inline (vlib_main_t * vm,
 			 u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
@@ -491,14 +378,14 @@ vlib_buffer_free_inline (vlib_main_t * vm,
 
       b = vlib_get_buffer (vm, buffers[i]);
 
-      fl = buffer_get_free_list (vm, b, &fi);
+      fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
 
       /* The only current use of this callback: multicast recycle */
       if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0))
 	{
 	  int j;
 
-	  add_buffer_to_free_list
+	  vlib_buffer_add_to_free_list
 	    (vm, fl, buffers[i], (b->flags & VLIB_BUFFER_RECYCLE) == 0);
 
 	  for (j = 0; j < vec_len (bm->announce_list); j++)