From c8a26c6d239554bac96c481c840b3f5b3d8a17af Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Fri, 24 Nov 2017 20:15:23 +0100 Subject: vlib: make vlib_buffer_alloc inline function Currently, every alloc involves callback function call. With this change callback function is called only if there is no empty buffers on the free list. Change-Id: I2238c19ece7ce182c49ba0f2485add52f335f3b6 Signed-off-by: Damjan Marion --- src/vlib/buffer_funcs.h | 80 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 21 deletions(-) (limited to 'src/vlib/buffer_funcs.h') diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index c7b5321ffac..08f5b3e9010 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -218,6 +218,11 @@ typedef enum VLIB_BUFFER_KNOWN_ALLOCATED, } vlib_buffer_known_state_t; +void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers, + uword n_buffers, + vlib_buffer_known_state_t + expected_state); + always_inline vlib_buffer_known_state_t vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index) { @@ -245,24 +250,6 @@ vlib_buffer_set_known_state (vlib_main_t * vm, u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index, uword follow_chain); -/** \brief Allocate buffers into supplied array - - @param vm - (vlib_main_t *) vlib main data structure pointer - @param buffers - (u32 * ) buffer index array - @param n_buffers - (u32) number of buffers requested - @return - (u32) number of buffers actually allocated, may be - less than the number requested or zero -*/ -always_inline u32 -vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - - ASSERT (bm->cb.vlib_buffer_alloc_cb); - - return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers); -} - always_inline u32 vlib_buffer_round_size (u32 size) { @@ -301,11 +288,62 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm, u32 n_buffers, u32 free_list_index) { vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_free_list_t *fl; + u32 *src; + uword len; + + ASSERT (bm->cb.vlib_buffer_fill_free_list_cb); + + fl = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); + + len = vec_len (fl->buffers); + + if (PREDICT_FALSE (len < n_buffers)) + { + bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers); + len = vec_len (fl->buffers); - ASSERT (bm->cb.vlib_buffer_alloc_from_free_list_cb); + /* even if fill free list didn't manage to refill free list + we should give what we have */ + n_buffers = clib_min (len, n_buffers); - return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers, - free_list_index); + /* following code is intentionaly duplicated to allow compiler + to optimize fast path when n_buffers is constant value */ + src = fl->buffers + len - n_buffers; + clib_memcpy (buffers, src, n_buffers * sizeof (u32)); + _vec_len (fl->buffers) -= n_buffers; + + /* Verify that buffers are known free. */ + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); + + return n_buffers; + } + + src = fl->buffers + len - n_buffers; + clib_memcpy (buffers, src, n_buffers * sizeof (u32)); + _vec_len (fl->buffers) -= n_buffers; + + /* Verify that buffers are known free. */ + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); + + return n_buffers; +} + +/** \brief Allocate buffers into supplied array + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param buffers - (u32 * ) buffer index array + @param n_buffers - (u32) number of buffers requested + @return - (u32) number of buffers actually allocated, may be + less than the number requested or zero +*/ +always_inline u32 +vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +{ + return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers, + VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); } /** \brief Free buffers -- cgit 1.2.3-korg