diff options
| -rw-r--r-- | src/plugins/dpdk/buffer.c | 84 | ||||
| -rw-r--r-- | src/vlib/buffer.c | 71 | ||||
| -rw-r--r-- | src/vlib/buffer.h | 12 | ||||
| -rw-r--r-- | src/vlib/buffer_funcs.h | 80 | ||||
| -rw-r--r-- | src/vlib/mc.c | 2 | ||||
| -rw-r--r-- | src/vnet/dns/dns.c | 2 |
6 files changed, 84 insertions, 167 deletions
diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c index 3ac9c69a9d2..235a130d937 100644 --- a/src/plugins/dpdk/buffer.c +++ b/src/plugins/dpdk/buffer.c @@ -196,9 +196,10 @@ dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index) #endif /* Make sure free list has at least given number of free buffers. */ -static uword -fill_free_list (vlib_main_t * vm, - vlib_buffer_free_list_t * fl, uword min_free_buffers) +uword +CLIB_MULTIARCH_FN (dpdk_buffer_fill_free_list) (vlib_main_t * vm, + vlib_buffer_free_list_t * fl, + uword min_free_buffers) { dpdk_main_t *dm = &dpdk_main; vlib_buffer_t *b0, *b1, *b2, *b3; @@ -303,60 +304,6 @@ fill_free_list (vlib_main_t * vm, return n; } -static u32 -alloc_from_free_list (vlib_main_t * vm, - vlib_buffer_free_list_t * free_list, - u32 * alloc_buffers, u32 n_alloc_buffers) -{ - u32 *dst, *src; - uword len, n_filled; - - dst = alloc_buffers; - - n_filled = fill_free_list (vm, free_list, n_alloc_buffers); - if (n_filled == 0) - return 0; - - len = vec_len (free_list->buffers); - ASSERT (len >= n_alloc_buffers); - - src = free_list->buffers + len - n_alloc_buffers; - clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32)); - - _vec_len (free_list->buffers) -= n_alloc_buffers; - - return n_alloc_buffers; -} - -/* Allocate a given number of buffers into given array. - Returns number actually allocated which will be either zero or - number requested. */ -u32 -CLIB_MULTIARCH_FN (dpdk_buffer_alloc) (vlib_main_t * vm, u32 * buffers, - u32 n_buffers) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - - return alloc_from_free_list - (vm, - pool_elt_at_index (bm->buffer_free_list_pool, - VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX), - buffers, n_buffers); -} - - -u32 -CLIB_MULTIARCH_FN (dpdk_buffer_alloc_from_free_list) (vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, - u32 free_list_index) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *f; - f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); - return alloc_from_free_list (vm, f, buffers, n_buffers); -} - static_always_inline void dpdk_prefetch_buffer_by_index (vlib_main_t * vm, u32 bi) { @@ -733,8 +680,7 @@ VLIB_INIT_FUNCTION (dpdk_buffer_init); /* *INDENT-OFF* */ VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = { - .vlib_buffer_alloc_cb = &dpdk_buffer_alloc, - .vlib_buffer_alloc_from_free_list_cb = &dpdk_buffer_alloc_from_free_list, + .vlib_buffer_fill_free_list_cb = &dpdk_buffer_fill_free_list, .vlib_buffer_free_cb = &dpdk_buffer_free, .vlib_buffer_free_no_next_cb = &dpdk_buffer_free_no_next, .vlib_packet_template_init_cb = &dpdk_packet_template_init, @@ -743,12 +689,8 @@ VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = { /* *INDENT-ON* */ #if __x86_64__ -vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx512; -vlib_buffer_alloc_cb_t __clib_weak dpdk_buffer_alloc_avx2; -vlib_buffer_alloc_from_free_list_cb_t __clib_weak - dpdk_buffer_alloc_from_free_list_avx512; -vlib_buffer_alloc_from_free_list_cb_t __clib_weak - dpdk_buffer_alloc_from_free_list_avx2; +vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx512; +vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx2; vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx512; vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx2; vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx512; @@ -758,19 +700,15 @@ static void __clib_constructor dpdk_input_multiarch_select (void) { vlib_buffer_callbacks_t *cb = &__dpdk_buffer_callbacks; - if (dpdk_buffer_alloc_avx512 && clib_cpu_supports_avx512f ()) + if (dpdk_buffer_fill_free_list_avx512 && clib_cpu_supports_avx512f ()) { - cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx512; - cb->vlib_buffer_alloc_from_free_list_cb = - dpdk_buffer_alloc_from_free_list_avx512; + cb->vlib_buffer_fill_free_list_cb = dpdk_buffer_fill_free_list_avx512; cb->vlib_buffer_free_cb = dpdk_buffer_free_avx512; cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_avx512; } - else if (dpdk_buffer_alloc_avx2 && clib_cpu_supports_avx2 ()) + else if (dpdk_buffer_fill_free_list_avx2 && clib_cpu_supports_avx2 ()) { - cb->vlib_buffer_alloc_cb = dpdk_buffer_alloc_avx2; - cb->vlib_buffer_alloc_from_free_list_cb = - dpdk_buffer_alloc_from_free_list_avx2; + cb->vlib_buffer_fill_free_list_cb = dpdk_buffer_fill_free_list_avx2; cb->vlib_buffer_free_cb = dpdk_buffer_free_avx2; cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_avx2; } diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index f00e8854ac1..880cd95a067 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -294,7 +294,7 @@ vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm; /* When dubugging validate that given buffers are either known allocated or known free. */ -static void +void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers, uword n_buffers, @@ -306,6 +306,9 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm, if (CLIB_DEBUG == 0) return; + if (vlib_buffer_callbacks) + return; + is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED; b = buffers; for (i = 0; i < n_buffers; i++) @@ -509,8 +512,9 @@ vlib_buffer_delete_free_list_internal (vlib_main_t * vm, u32 free_list_index) /* Make sure free list has at least given number of free buffers. */ static uword -fill_free_list (vlib_main_t * vm, - vlib_buffer_free_list_t * fl, uword min_free_buffers) +vlib_buffer_fill_free_list_internal (vlib_main_t * vm, + vlib_buffer_free_list_t * fl, + uword min_free_buffers) { vlib_buffer_t *buffers, *b; vlib_buffer_free_list_t *mfl; @@ -596,63 +600,6 @@ fill_free_list (vlib_main_t * vm, return n_alloc; } -static u32 -alloc_from_free_list (vlib_main_t * vm, - vlib_buffer_free_list_t * free_list, - u32 * alloc_buffers, u32 n_alloc_buffers) -{ - u32 *dst, *src; - uword len; - uword n_filled; - - dst = alloc_buffers; - - n_filled = fill_free_list (vm, free_list, n_alloc_buffers); - if (n_filled == 0) - return 0; - - len = vec_len (free_list->buffers); - ASSERT (len >= n_alloc_buffers); - - src = free_list->buffers + len - n_alloc_buffers; - clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32)); - - _vec_len (free_list->buffers) -= n_alloc_buffers; - - /* Verify that buffers are known free. */ - vlib_buffer_validate_alloc_free (vm, alloc_buffers, - n_alloc_buffers, VLIB_BUFFER_KNOWN_FREE); - - return n_alloc_buffers; -} - - -/* Allocate a given number of buffers into given array. - Returns number actually allocated which will be either zero or - number requested. */ -static u32 -vlib_buffer_alloc_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - - return alloc_from_free_list - (vm, - pool_elt_at_index (bm->buffer_free_list_pool, - VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX), - buffers, n_buffers); -} - -static u32 -vlib_buffer_alloc_from_free_list_internal (vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, u32 free_list_index) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *f; - f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); - return alloc_from_free_list (vm, f, buffers, n_buffers); -} - void * vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp) { @@ -1087,9 +1034,7 @@ vlib_buffer_main_init (struct vlib_main_t * vm) return 0; } - bm->cb.vlib_buffer_alloc_cb = &vlib_buffer_alloc_internal; - bm->cb.vlib_buffer_alloc_from_free_list_cb = - &vlib_buffer_alloc_from_free_list_internal; + bm->cb.vlib_buffer_fill_free_list_cb = &vlib_buffer_fill_free_list_internal; bm->cb.vlib_buffer_free_cb = &vlib_buffer_free_internal; bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal; bm->cb.vlib_buffer_delete_free_list_cb = diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h index 21bed48ac14..c3c4def0991 100644 --- a/src/vlib/buffer.h +++ b/src/vlib/buffer.h @@ -380,12 +380,9 @@ typedef struct vlib_buffer_free_list_t uword buffer_init_function_opaque; } __attribute__ ((aligned (16))) vlib_buffer_free_list_t; -typedef u32 (vlib_buffer_alloc_cb_t) (struct vlib_main_t * vm, u32 * buffers, - u32 n_buffers); -typedef u32 (vlib_buffer_alloc_from_free_list_cb_t) (struct vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, - u32 free_list_index); +typedef uword (vlib_buffer_fill_free_list_cb_t) (struct vlib_main_t * vm, + vlib_buffer_free_list_t * fl, + uword min_free_buffers); typedef void (vlib_buffer_free_cb_t) (struct vlib_main_t * vm, u32 * buffers, u32 n_buffers); typedef void (vlib_buffer_free_no_next_cb_t) (struct vlib_main_t * vm, @@ -393,8 +390,7 @@ typedef void (vlib_buffer_free_no_next_cb_t) (struct vlib_main_t * vm, typedef struct { - vlib_buffer_alloc_cb_t *vlib_buffer_alloc_cb; - vlib_buffer_alloc_from_free_list_cb_t *vlib_buffer_alloc_from_free_list_cb; + vlib_buffer_fill_free_list_cb_t *vlib_buffer_fill_free_list_cb; vlib_buffer_free_cb_t *vlib_buffer_free_cb; vlib_buffer_free_no_next_cb_t *vlib_buffer_free_no_next_cb; void (*vlib_packet_template_init_cb) (struct vlib_main_t * vm, void *t, diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index c7b5321ffac..08f5b3e9010 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -218,6 +218,11 @@ typedef enum VLIB_BUFFER_KNOWN_ALLOCATED, } vlib_buffer_known_state_t; +void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers, + uword n_buffers, + vlib_buffer_known_state_t + expected_state); + always_inline vlib_buffer_known_state_t vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index) { @@ -245,24 +250,6 @@ vlib_buffer_set_known_state (vlib_main_t * vm, u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index, uword follow_chain); -/** \brief Allocate buffers into supplied array - - @param vm - (vlib_main_t *) vlib main data structure pointer - @param buffers - (u32 * ) buffer index array - @param n_buffers - (u32) number of buffers requested - @return - (u32) number of buffers actually allocated, may be - less than the number requested or zero -*/ -always_inline u32 -vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - - ASSERT (bm->cb.vlib_buffer_alloc_cb); - - return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers); -} - always_inline u32 vlib_buffer_round_size (u32 size) { @@ -301,11 +288,62 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm, u32 n_buffers, u32 free_list_index) { vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_free_list_t *fl; + u32 *src; + uword len; + + ASSERT (bm->cb.vlib_buffer_fill_free_list_cb); + + fl = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); + + len = vec_len (fl->buffers); + + if (PREDICT_FALSE (len < n_buffers)) + { + bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers); + len = vec_len (fl->buffers); - ASSERT (bm->cb.vlib_buffer_alloc_from_free_list_cb); + /* even if fill free list didn't manage to refill free list + we should give what we have */ + n_buffers = clib_min (len, n_buffers); - return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers, - free_list_index); + /* following code is intentionaly duplicated to allow compiler + to optimize fast path when n_buffers is constant value */ + src = fl->buffers + len - n_buffers; + clib_memcpy (buffers, src, n_buffers * sizeof (u32)); + _vec_len (fl->buffers) -= n_buffers; + + /* Verify that buffers are known free. */ + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); + + return n_buffers; + } + + src = fl->buffers + len - n_buffers; + clib_memcpy (buffers, src, n_buffers * sizeof (u32)); + _vec_len (fl->buffers) -= n_buffers; + + /* Verify that buffers are known free. */ + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); + + return n_buffers; +} + +/** \brief Allocate buffers into supplied array + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param buffers - (u32 * ) buffer index array + @param n_buffers - (u32) number of buffers requested + @return - (u32) number of buffers actually allocated, may be + less than the number requested or zero +*/ +always_inline u32 +vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +{ + return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers, + VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); } /** \brief Free buffers diff --git a/src/vlib/mc.c b/src/vlib/mc.c index e57962c94de..a289871f570 100644 --- a/src/vlib/mc.c +++ b/src/vlib/mc.c @@ -109,7 +109,7 @@ mc_seq_cmp (u32 x, u32 y) void * mc_get_vlib_buffer (vlib_main_t * vm, u32 n_bytes, u32 * bi_return) { - u32 n_alloc, bi; + u32 n_alloc, bi = 0; vlib_buffer_t *b; n_alloc = vlib_buffer_alloc (vm, &bi, 1); diff --git a/src/vnet/dns/dns.c b/src/vnet/dns/dns.c index cf76439ac8e..c4ba85105fc 100644 --- a/src/vnet/dns/dns.c +++ b/src/vnet/dns/dns.c @@ -2653,7 +2653,7 @@ vnet_send_dns4_reply (dns_main_t * dm, dns_pending_request_t * pr, dns_cache_entry_t * ep, vlib_buffer_t * b0) { vlib_main_t *vm = dm->vlib_main; - u32 bi; + u32 bi = 0; fib_prefix_t prefix; fib_node_index_t fei; u32 sw_if_index, fib_index; |