diff options
Diffstat (limited to 'src/vlib')
| -rw-r--r-- | src/vlib/buffer.c | 71 | ||||
| -rw-r--r-- | src/vlib/buffer.h | 12 | ||||
| -rw-r--r-- | src/vlib/buffer_funcs.h | 80 | ||||
| -rw-r--r-- | src/vlib/mc.c | 2 |
4 files changed, 72 insertions, 93 deletions
diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index f00e8854ac1..880cd95a067 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -294,7 +294,7 @@ vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm; /* When dubugging validate that given buffers are either known allocated or known free. */ -static void +void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers, uword n_buffers, @@ -306,6 +306,9 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm, if (CLIB_DEBUG == 0) return; + if (vlib_buffer_callbacks) + return; + is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED; b = buffers; for (i = 0; i < n_buffers; i++) @@ -509,8 +512,9 @@ vlib_buffer_delete_free_list_internal (vlib_main_t * vm, u32 free_list_index) /* Make sure free list has at least given number of free buffers. */ static uword -fill_free_list (vlib_main_t * vm, - vlib_buffer_free_list_t * fl, uword min_free_buffers) +vlib_buffer_fill_free_list_internal (vlib_main_t * vm, + vlib_buffer_free_list_t * fl, + uword min_free_buffers) { vlib_buffer_t *buffers, *b; vlib_buffer_free_list_t *mfl; @@ -596,63 +600,6 @@ fill_free_list (vlib_main_t * vm, return n_alloc; } -static u32 -alloc_from_free_list (vlib_main_t * vm, - vlib_buffer_free_list_t * free_list, - u32 * alloc_buffers, u32 n_alloc_buffers) -{ - u32 *dst, *src; - uword len; - uword n_filled; - - dst = alloc_buffers; - - n_filled = fill_free_list (vm, free_list, n_alloc_buffers); - if (n_filled == 0) - return 0; - - len = vec_len (free_list->buffers); - ASSERT (len >= n_alloc_buffers); - - src = free_list->buffers + len - n_alloc_buffers; - clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32)); - - _vec_len (free_list->buffers) -= n_alloc_buffers; - - /* Verify that buffers are known free. */ - vlib_buffer_validate_alloc_free (vm, alloc_buffers, - n_alloc_buffers, VLIB_BUFFER_KNOWN_FREE); - - return n_alloc_buffers; -} - - -/* Allocate a given number of buffers into given array. - Returns number actually allocated which will be either zero or - number requested. */ -static u32 -vlib_buffer_alloc_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - - return alloc_from_free_list - (vm, - pool_elt_at_index (bm->buffer_free_list_pool, - VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX), - buffers, n_buffers); -} - -static u32 -vlib_buffer_alloc_from_free_list_internal (vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, u32 free_list_index) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *f; - f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); - return alloc_from_free_list (vm, f, buffers, n_buffers); -} - void * vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp) { @@ -1087,9 +1034,7 @@ vlib_buffer_main_init (struct vlib_main_t * vm) return 0; } - bm->cb.vlib_buffer_alloc_cb = &vlib_buffer_alloc_internal; - bm->cb.vlib_buffer_alloc_from_free_list_cb = - &vlib_buffer_alloc_from_free_list_internal; + bm->cb.vlib_buffer_fill_free_list_cb = &vlib_buffer_fill_free_list_internal; bm->cb.vlib_buffer_free_cb = &vlib_buffer_free_internal; bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal; bm->cb.vlib_buffer_delete_free_list_cb = diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h index 21bed48ac14..c3c4def0991 100644 --- a/src/vlib/buffer.h +++ b/src/vlib/buffer.h @@ -380,12 +380,9 @@ typedef struct vlib_buffer_free_list_t uword buffer_init_function_opaque; } __attribute__ ((aligned (16))) vlib_buffer_free_list_t; -typedef u32 (vlib_buffer_alloc_cb_t) (struct vlib_main_t * vm, u32 * buffers, - u32 n_buffers); -typedef u32 (vlib_buffer_alloc_from_free_list_cb_t) (struct vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, - u32 free_list_index); +typedef uword (vlib_buffer_fill_free_list_cb_t) (struct vlib_main_t * vm, + vlib_buffer_free_list_t * fl, + uword min_free_buffers); typedef void (vlib_buffer_free_cb_t) (struct vlib_main_t * vm, u32 * buffers, u32 n_buffers); typedef void (vlib_buffer_free_no_next_cb_t) (struct vlib_main_t * vm, @@ -393,8 +390,7 @@ typedef void (vlib_buffer_free_no_next_cb_t) (struct vlib_main_t * vm, typedef struct { - vlib_buffer_alloc_cb_t *vlib_buffer_alloc_cb; - vlib_buffer_alloc_from_free_list_cb_t *vlib_buffer_alloc_from_free_list_cb; + vlib_buffer_fill_free_list_cb_t *vlib_buffer_fill_free_list_cb; vlib_buffer_free_cb_t *vlib_buffer_free_cb; vlib_buffer_free_no_next_cb_t *vlib_buffer_free_no_next_cb; void (*vlib_packet_template_init_cb) (struct vlib_main_t * vm, void *t, diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index c7b5321ffac..08f5b3e9010 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -218,6 +218,11 @@ typedef enum VLIB_BUFFER_KNOWN_ALLOCATED, } vlib_buffer_known_state_t; +void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers, + uword n_buffers, + vlib_buffer_known_state_t + expected_state); + always_inline vlib_buffer_known_state_t vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index) { @@ -245,24 +250,6 @@ vlib_buffer_set_known_state (vlib_main_t * vm, u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index, uword follow_chain); -/** \brief Allocate buffers into supplied array - - @param vm - (vlib_main_t *) vlib main data structure pointer - @param buffers - (u32 * ) buffer index array - @param n_buffers - (u32) number of buffers requested - @return - (u32) number of buffers actually allocated, may be - less than the number requested or zero -*/ -always_inline u32 -vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - - ASSERT (bm->cb.vlib_buffer_alloc_cb); - - return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers); -} - always_inline u32 vlib_buffer_round_size (u32 size) { @@ -301,11 +288,62 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm, u32 n_buffers, u32 free_list_index) { vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_free_list_t *fl; + u32 *src; + uword len; + + ASSERT (bm->cb.vlib_buffer_fill_free_list_cb); + + fl = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); + + len = vec_len (fl->buffers); + + if (PREDICT_FALSE (len < n_buffers)) + { + bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers); + len = vec_len (fl->buffers); - ASSERT (bm->cb.vlib_buffer_alloc_from_free_list_cb); + /* even if fill free list didn't manage to refill free list + we should give what we have */ + n_buffers = clib_min (len, n_buffers); - return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers, - free_list_index); + /* following code is intentionaly duplicated to allow compiler + to optimize fast path when n_buffers is constant value */ + src = fl->buffers + len - n_buffers; + clib_memcpy (buffers, src, n_buffers * sizeof (u32)); + _vec_len (fl->buffers) -= n_buffers; + + /* Verify that buffers are known free. */ + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); + + return n_buffers; + } + + src = fl->buffers + len - n_buffers; + clib_memcpy (buffers, src, n_buffers * sizeof (u32)); + _vec_len (fl->buffers) -= n_buffers; + + /* Verify that buffers are known free. */ + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); + + return n_buffers; +} + +/** \brief Allocate buffers into supplied array + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param buffers - (u32 * ) buffer index array + @param n_buffers - (u32) number of buffers requested + @return - (u32) number of buffers actually allocated, may be + less than the number requested or zero +*/ +always_inline u32 +vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +{ + return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers, + VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); } /** \brief Free buffers diff --git a/src/vlib/mc.c b/src/vlib/mc.c index e57962c94de..a289871f570 100644 --- a/src/vlib/mc.c +++ b/src/vlib/mc.c @@ -109,7 +109,7 @@ mc_seq_cmp (u32 x, u32 y) void * mc_get_vlib_buffer (vlib_main_t * vm, u32 n_bytes, u32 * bi_return) { - u32 n_alloc, bi; + u32 n_alloc, bi = 0; vlib_buffer_t *b; n_alloc = vlib_buffer_alloc (vm, &bi, 1); |