vlib: make vlib_buffer_alloc inline function

Currently, every alloc involves callback function call. With this change
callback function is called only if there is no empty buffers on the
free list.

Change-Id: I2238c19ece7ce182c49ba0f2485add52f335f3b6
Signed-off-by: Damjan Marion <damarion@cisco.com>

6 files changed, 84 insertions, 167 deletions

diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c
index 3ac9c69a9d2..235a130d937 100644
--- a/src/plugins/dpdk/buffer.c
+++ b/src/plugins/dpdk/buffer.c
@@ -196,9 +196,10 @@ dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index)
 #endif
 
 /* Make sure free list has at least given number of free buffers. */
-static uword
-fill_free_list (vlib_main_t * vm,
-		vlib_buffer_free_list_t * fl, uword min_free_buffers)
+uword
+CLIB_MULTIARCH_FN (dpdk_buffer_fill_free_list) (vlib_main_t * vm,
+						vlib_buffer_free_list_t * fl,
+						uword min_free_buffers)
 {
   dpdk_main_t *dm = &dpdk_main;
   vlib_buffer_t *b0, *b1, *b2, *b3;
@@ -303,60 +304,6 @@ fill_free_list (vlib_main_t * vm,
   return n;
 }
 
-static u32
-alloc_from_free_list (vlib_main_t * vm,
-		      vlib_buffer_free_list_t * free_list,
-		      u32 * alloc_buffers, u32 n_alloc_buffers)
-{
-  u32 *dst, *src;
-  uword len, n_filled;
-
-  dst = alloc_buffers;
-
-  n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
-  if (n_filled == 0)
-    return 0;
-
-  len = vec_len (free_list->buffers);
-  ASSERT (len >= n_alloc_buffers);
-
-  src = free_list->buffers + len - n_alloc_buffers;
-  clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32));
-
-  _vec_len (free_list->buffers) -= n_alloc_buffers;
-
-  return n_alloc_buffers;
-}
-
-/* Allocate a given number of buffers into given array.
-   Returns number actually allocated which will be either zero or
-   number requested. */
-u32
-CLIB_MULTIARCH_FN (dpdk_buffer_alloc) (vlib_main_t * vm, u32 * buffers,
-				       u32 n_buffers)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-
-  return alloc_from_free_list
-    (vm,
-     pool_elt_at_index (bm->buffer_free_list_pool,
-			VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX),
-     buffers, n_buffers);
-}
-
-
-u32
-CLIB_MULTIARCH_FN (dpdk_buffer_alloc_from_free_list) (vlib_main_t * vm,
-						      u32 * buffers,
-						      u32 n_buffers,
-						      u32 free_list_index)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-  vlib_buffer_free_list_t *f;
-  f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
-  return alloc_from_free_list (vm, f, buffers, n_buffers);
-}
-
 static_always_inline void
 dpdk_prefetch_buffer_by_index (vlib_main_t * vm, u32 bi)
 {
@@ -733,8 +680,7 @@ VLIB_INIT_FUNCTION (dpdk_buffer_init);
 
 /* *INDENT-OFF* */
 VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = {
-  .vlib_buffer_alloc_cb = &dpdk_buffer_alloc,
-  .vlib_buffer_alloc_from_free_list_cb = &dpdk_buffer_alloc_from_free_list,
+  .vlib_buffer_fill_free_list_cb = &dpdk_buffer_fill_free_list,
   .vlib_buffer_free_cb = &dpdk_buffer_free,
   .vlib_buffer_free_no_next_cb = &dpdk_buffer_free_no_next,
   .vlib_packet_template_init_cb = &dpdk_packet_template_init,
@@ -743,12 +689,8 @@ VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = {
 /* *INDENT-ON* */
 
 #if __x86_64__
-vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx512;
-vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx2;
-vlib_buffer_alloc_from_free_list_cb_t __clib_weak
-  dpdk_buffer_alloc_from_free_list_avx512;
-vlib_buffer_alloc_from_free_list_cb_t __clib_weak
-  dpdk_buffer_alloc_from_free_list_avx2;
+vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx512;
+vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx2;
 vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx512;
 vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx2;
 vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx512;
@@ -758,19 +700,15 @@ static void __clib_constructor
 dpdk_input_multiarch_select (void)
 {
   vlib_buffer_callbacks_t *cb = &__dpdk_buffer_callbacks;
-  if (dpdk_buffer_alloc_avx512 && clib_cpu_supports_avx512f ())
+  if (dpdk_buffer_fill_free_list_avx512 && clib_cpu_supports_avx512f ())
     {
-      cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx512;
-      cb->vlib_buffer_alloc_from_free_list_cb =
-	dpdk_buffer_alloc_from_free_list_avx512;
+      cb->vlib_buffer_fill_free_list_cb = dpdk_buffer_fill_free_list_avx512;
       cb->vlib_buffer_free_cb = dpdk_buffer_free_avx512;
       cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_avx512;
     }
-  else if (dpdk_buffer_alloc_avx2 && clib_cpu_supports_avx2 ())
+  else if (dpdk_buffer_fill_free_list_avx2 && clib_cpu_supports_avx2 ())
     {
-      cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx2;
-      cb->vlib_buffer_alloc_from_free_list_cb =
-	dpdk_buffer_alloc_from_free_list_avx2;
+      cb->vlib_buffer_fill_free_list_cb = dpdk_buffer_fill_free_list_avx2;
       cb->vlib_buffer_free_cb = dpdk_buffer_free_avx2;
       cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_avx2;
     }
diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index f00e8854ac1..880cd95a067 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -294,7 +294,7 @@ vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm;
 
 /* When dubugging validate that given buffers are either known allocated
    or known free. */
-static void
+void
 vlib_buffer_validate_alloc_free (vlib_main_t * vm,
 				 u32 * buffers,
 				 uword n_buffers,
@@ -306,6 +306,9 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm,
   if (CLIB_DEBUG == 0)
     return;
 
+  if (vlib_buffer_callbacks)
+    return;
+
   is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED;
   b = buffers;
   for (i = 0; i < n_buffers; i++)
@@ -509,8 +512,9 @@ vlib_buffer_delete_free_list_internal (vlib_main_t * vm, u32 free_list_index)
 
 /* Make sure free list has at least given number of free buffers. */
 static uword
-fill_free_list (vlib_main_t * vm,
-		vlib_buffer_free_list_t * fl, uword min_free_buffers)
+vlib_buffer_fill_free_list_internal (vlib_main_t * vm,
+				     vlib_buffer_free_list_t * fl,
+				     uword min_free_buffers)
 {
   vlib_buffer_t *buffers, *b;
   vlib_buffer_free_list_t *mfl;
@@ -596,63 +600,6 @@ fill_free_list (vlib_main_t * vm,
   return n_alloc;
 }
 
-static u32
-alloc_from_free_list (vlib_main_t * vm,
-		      vlib_buffer_free_list_t * free_list,
-		      u32 * alloc_buffers, u32 n_alloc_buffers)
-{
-  u32 *dst, *src;
-  uword len;
-  uword n_filled;
-
-  dst = alloc_buffers;
-
-  n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
-  if (n_filled == 0)
-    return 0;
-
-  len = vec_len (free_list->buffers);
-  ASSERT (len >= n_alloc_buffers);
-
-  src = free_list->buffers + len - n_alloc_buffers;
-  clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32));
-
-  _vec_len (free_list->buffers) -= n_alloc_buffers;
-
-  /* Verify that buffers are known free. */
-  vlib_buffer_validate_alloc_free (vm, alloc_buffers,
-				   n_alloc_buffers, VLIB_BUFFER_KNOWN_FREE);
-
-  return n_alloc_buffers;
-}
-
-
-/* Allocate a given number of buffers into given array.
-   Returns number actually allocated which will be either zero or
-   number requested. */
-static u32
-vlib_buffer_alloc_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-
-  return alloc_from_free_list
-    (vm,
-     pool_elt_at_index (bm->buffer_free_list_pool,
-			VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX),
-     buffers, n_buffers);
-}
-
-static u32
-vlib_buffer_alloc_from_free_list_internal (vlib_main_t * vm,
-					   u32 * buffers,
-					   u32 n_buffers, u32 free_list_index)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-  vlib_buffer_free_list_t *f;
-  f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
-  return alloc_from_free_list (vm, f, buffers, n_buffers);
-}
-
 void *
 vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp)
 {
@@ -1087,9 +1034,7 @@ vlib_buffer_main_init (struct vlib_main_t * vm)
       return 0;
     }
 
-  bm->cb.vlib_buffer_alloc_cb = &vlib_buffer_alloc_internal;
-  bm->cb.vlib_buffer_alloc_from_free_list_cb =
-    &vlib_buffer_alloc_from_free_list_internal;
+  bm->cb.vlib_buffer_fill_free_list_cb = &vlib_buffer_fill_free_list_internal;
   bm->cb.vlib_buffer_free_cb = &vlib_buffer_free_internal;
   bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal;
   bm->cb.vlib_buffer_delete_free_list_cb =
diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h
index 21bed48ac14..c3c4def0991 100644
--- a/src/vlib/buffer.h
+++ b/src/vlib/buffer.h
@@ -380,12 +380,9 @@ typedef struct vlib_buffer_free_list_t
   uword buffer_init_function_opaque;
 } __attribute__ ((aligned (16))) vlib_buffer_free_list_t;
 
-typedef u32 (vlib_buffer_alloc_cb_t) (struct vlib_main_t * vm, u32 * buffers,
-				      u32 n_buffers);
-typedef u32 (vlib_buffer_alloc_from_free_list_cb_t) (struct vlib_main_t * vm,
-						     u32 * buffers,
-						     u32 n_buffers,
-						     u32 free_list_index);
+typedef uword (vlib_buffer_fill_free_list_cb_t) (struct vlib_main_t * vm,
+						 vlib_buffer_free_list_t * fl,
+						 uword min_free_buffers);
 typedef void (vlib_buffer_free_cb_t) (struct vlib_main_t * vm, u32 * buffers,
 				      u32 n_buffers);
 typedef void (vlib_buffer_free_no_next_cb_t) (struct vlib_main_t * vm,
@@ -393,8 +390,7 @@ typedef void (vlib_buffer_free_no_next_cb_t) (struct vlib_main_t * vm,
 
 typedef struct
 {
-  vlib_buffer_alloc_cb_t *vlib_buffer_alloc_cb;
-  vlib_buffer_alloc_from_free_list_cb_t *vlib_buffer_alloc_from_free_list_cb;
+  vlib_buffer_fill_free_list_cb_t *vlib_buffer_fill_free_list_cb;
   vlib_buffer_free_cb_t *vlib_buffer_free_cb;
   vlib_buffer_free_no_next_cb_t *vlib_buffer_free_no_next_cb;
   void (*vlib_packet_template_init_cb) (struct vlib_main_t * vm, void *t,
diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index c7b5321ffac..08f5b3e9010 100644
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -218,6 +218,11 @@ typedef enum
   VLIB_BUFFER_KNOWN_ALLOCATED,
 } vlib_buffer_known_state_t;
 
+void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
+				      uword n_buffers,
+				      vlib_buffer_known_state_t
+				      expected_state);
+
 always_inline vlib_buffer_known_state_t
 vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
 {
@@ -245,24 +250,6 @@ vlib_buffer_set_known_state (vlib_main_t * vm,
 u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
 			  uword follow_chain);
 
-/** \brief Allocate buffers into supplied array
-
-    @param vm - (vlib_main_t *) vlib main data structure pointer
-    @param buffers - (u32 * ) buffer index array
-    @param n_buffers - (u32) number of buffers requested
-    @return - (u32) number of buffers actually allocated, may be
-    less than the number requested or zero
-*/
-always_inline u32
-vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-
-  ASSERT (bm->cb.vlib_buffer_alloc_cb);
-
-  return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers);
-}
-
 always_inline u32
 vlib_buffer_round_size (u32 size)
 {
@@ -301,11 +288,62 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
 				  u32 n_buffers, u32 free_list_index)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
+  vlib_buffer_free_list_t *fl;
+  u32 *src;
+  uword len;
+
+  ASSERT (bm->cb.vlib_buffer_fill_free_list_cb);
+
+  fl = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
+
+  len = vec_len (fl->buffers);
+
+  if (PREDICT_FALSE (len < n_buffers))
+    {
+      bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers);
+      len = vec_len (fl->buffers);
 
-  ASSERT (bm->cb.vlib_buffer_alloc_from_free_list_cb);
+      /* even if fill free list didn't manage to refill free list
+         we should give what we have */
+      n_buffers = clib_min (len, n_buffers);
 
-  return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers,
-						     free_list_index);
+      /* following code is intentionaly duplicated to allow compiler
+         to optimize fast path when n_buffers is constant value */
+      src = fl->buffers + len - n_buffers;
+      clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+      _vec_len (fl->buffers) -= n_buffers;
+
+      /* Verify that buffers are known free. */
+      vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
+				       VLIB_BUFFER_KNOWN_FREE);
+
+      return n_buffers;
+    }
+
+  src = fl->buffers + len - n_buffers;
+  clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+  _vec_len (fl->buffers) -= n_buffers;
+
+  /* Verify that buffers are known free. */
+  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
+				   VLIB_BUFFER_KNOWN_FREE);
+
+  return n_buffers;
+}
+
+/** \brief Allocate buffers into supplied array
+
+    @param vm - (vlib_main_t *) vlib main data structure pointer
+    @param buffers - (u32 * ) buffer index array
+    @param n_buffers - (u32) number of buffers requested
+    @return - (u32) number of buffers actually allocated, may be
+    less than the number requested or zero
+*/
+always_inline u32
+vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
+{
+  return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
+					   VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
 }
 
 /** \brief Free buffers
diff --git a/src/vlib/mc.c b/src/vlib/mc.c
index e57962c94de..a289871f570 100644
--- a/src/vlib/mc.c
+++ b/src/vlib/mc.c
@@ -109,7 +109,7 @@ mc_seq_cmp (u32 x, u32 y)
 void *
 mc_get_vlib_buffer (vlib_main_t * vm, u32 n_bytes, u32 * bi_return)
 {
-  u32 n_alloc, bi;
+  u32 n_alloc, bi = 0;
   vlib_buffer_t *b;
 
   n_alloc = vlib_buffer_alloc (vm, &bi, 1);
diff --git a/src/vnet/dns/dns.c b/src/vnet/dns/dns.c
index cf76439ac8e..c4ba85105fc 100644
--- a/src/vnet/dns/dns.c
+++ b/src/vnet/dns/dns.c
@@ -2653,7 +2653,7 @@ vnet_send_dns4_reply (dns_main_t * dm, dns_pending_request_t * pr,
 		      dns_cache_entry_t * ep, vlib_buffer_t * b0)
 {
   vlib_main_t *vm = dm->vlib_main;
-  u32 bi;
+  u32 bi = 0;
   fib_prefix_t prefix;
   fib_node_index_t fei;
   u32 sw_if_index, fib_index;