diff options
Diffstat (limited to 'src/vlib')
| -rw-r--r-- | src/vlib/buffer.c | 47 | ||||
| -rw-r--r-- | src/vlib/buffer.h | 67 | ||||
| -rw-r--r-- | src/vlib/buffer_funcs.h | 113 |
3 files changed, 119 insertions, 108 deletions
diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index 400488d0352..6b20a542ca1 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -71,10 +71,16 @@ format_vlib_buffer (u8 * s, va_list * args) { vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); u32 indent = format_get_indent (s); + u8 *a = 0; - s = format (s, "current data %d, length %d, free-list %d, clone-count %u", - b->current_data, b->current_length, - vlib_buffer_get_free_list_index (b), b->n_add_refs); +#define _(bit, name, v) \ + if (v && (b->flags & VLIB_BUFFER_##name)) \ + a = format (a, "%s ", v); + foreach_vlib_buffer_flag +#undef _ + s = format (s, "current data %d, length %d, free-list %d, clone-count %u", + b->current_data, b->current_length, + vlib_buffer_get_free_list_index (b), b->n_add_refs); if (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID) s = format (s, ", totlen-nifb %d", @@ -83,6 +89,10 @@ format_vlib_buffer (u8 * s, va_list * args) if (b->flags & VLIB_BUFFER_IS_TRACED) s = format (s, ", trace 0x%x", b->trace_index); + if (a) + s = format (s, "\n%U%v", format_white_space, indent, a); + vec_free (a); + while (b->flags & VLIB_BUFFER_NEXT_PRESENT) { vlib_main_t *vm = vlib_get_main (); @@ -349,7 +359,7 @@ vlib_buffer_merge_free_lists (vlib_buffer_free_list_t * dst, } /* Add buffer free list. */ -static u32 +static vlib_buffer_free_list_index_t vlib_buffer_create_free_list_helper (vlib_main_t * vm, u32 n_data_bytes, u32 is_public, u32 is_default, u8 * name) @@ -362,7 +372,7 @@ vlib_buffer_create_free_list_helper (vlib_main_t * vm, if (!is_default && pool_elts (bm->buffer_free_list_pool) == 0) { - u32 default_free_free_list_index; + vlib_buffer_free_list_index_t default_free_free_list_index; /* *INDENT-OFF* */ default_free_free_list_index = @@ -417,7 +427,7 @@ vlib_buffer_create_free_list_helper (vlib_main_t * vm, return f->index; } -u32 +vlib_buffer_free_list_index_t vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes, char *fmt, ...) { @@ -434,7 +444,7 @@ vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes, name); } -u32 +vlib_buffer_free_list_index_t vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes, char *fmt, ...) { @@ -473,20 +483,22 @@ del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) /* Add buffer free list. */ void -vlib_buffer_delete_free_list_internal (vlib_main_t * vm, u32 free_list_index) +vlib_buffer_delete_free_list_internal (vlib_main_t * vm, + vlib_buffer_free_list_index_t index) { vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_free_list_t *f; - u32 merge_index; + vlib_buffer_free_list_index_t merge_index; int i; ASSERT (vlib_get_thread_index () == 0); - f = vlib_buffer_get_free_list (vm, free_list_index); + f = vlib_buffer_get_free_list (vm, index); ASSERT (vec_len (f->buffers) == f->n_alloc); merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes); - if (merge_index != ~0 && merge_index != free_list_index) + if (merge_index != (vlib_buffer_free_list_index_t) ~ 0 + && merge_index != index) { vlib_buffer_merge_free_lists (pool_elt_at_index (bm->buffer_free_list_pool, merge_index), @@ -503,7 +515,7 @@ vlib_buffer_delete_free_list_internal (vlib_main_t * vm, u32 free_list_index) for (i = 1; i < vec_len (vlib_mains); i++) { bm = vlib_mains[i]->buffer_main; - f = vlib_buffer_get_free_list (vlib_mains[i], free_list_index);; + f = vlib_buffer_get_free_list (vlib_mains[i], index);; memset (f, 0xab, sizeof (f[0])); pool_put (bm->buffer_free_list_pool, f); } @@ -615,7 +627,7 @@ vlib_buffer_free_inline (vlib_main_t * vm, { vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_free_list_t *fl; - u32 fi; + vlib_buffer_free_list_index_t fi; int i; u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers, u32 follow_buffer_next); @@ -818,7 +830,7 @@ vlib_packet_template_get_packet_helper (vlib_main_t * vm, /* Append given data to end of buffer, possibly allocating new buffers. */ u32 vlib_buffer_add_data (vlib_main_t * vm, - u32 free_list_index, + vlib_buffer_free_list_index_t free_list_index, u32 buffer_index, void *data, u32 n_data_bytes) { u32 n_buffer_bytes, n_left, n_left_this_buffer, bi; @@ -875,10 +887,11 @@ out_of_buffers: u16 vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, - u32 free_list_index, + vlib_buffer_free_list_index_t + free_list_index, vlib_buffer_t * first, - vlib_buffer_t ** last, - void *data, u16 data_len) + vlib_buffer_t ** last, void *data, + u16 data_len) { vlib_buffer_t *l = *last; u32 n_buffer_bytes = diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h index c3c4def0991..a1c2db377ef 100644 --- a/src/vlib/buffer.h +++ b/src/vlib/buffer.h @@ -51,6 +51,8 @@ #define VLIB_BUFFER_DATA_SIZE (2048) #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE +typedef u8 vlib_buffer_free_list_index_t; + /** \file vlib buffer structure definition and a few select access methods. This structure and the buffer allocation @@ -58,6 +60,40 @@ of typing to make it so. */ +/** + * Buffer Flags + */ +#define foreach_vlib_buffer_flag \ + _( 0, NON_DEFAULT_FREELIST, "non-default-fl") \ + _( 1, IS_TRACED, 0) \ + _( 2, NEXT_PRESENT, 0) \ + _( 3, IS_RECYCLED, "is-recycled") \ + _( 4, TOTAL_LENGTH_VALID, 0) \ + _( 5, REPL_FAIL, "repl-fail") \ + _( 6, RECYCLE, "recycle") \ + _( 7, EXT_HDR_VALID, "ext-hdr-valid") + +/* NOTE: only buffer generic flags should be defined here, please consider + using user flags. i.e. src/vnet/buffer.h */ + +enum +{ +#define _(bit, name, v) VLIB_BUFFER_##name = (1 << (bit)), + foreach_vlib_buffer_flag +#undef _ +}; + +enum +{ +#define _(bit, name, v) VLIB_BUFFER_LOG2_##name = (bit), + foreach_vlib_buffer_flag +#undef _ +}; + + /* User defined buffer flags. */ +#define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n)) +#define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n)) + /* VLIB buffer representation. */ typedef struct { @@ -79,30 +115,11 @@ typedef struct <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says <br> VLIB_BUFFER_REPL_FAIL: packet replication failure <br> VLIB_BUFFER_RECYCLE: as it says - <br> VLIB_BUFFER_FLOW_REPORT: buffer is a flow report, <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header, set to avoid adding it to a flow report <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N */ -/* any change to the following line requres update of - * vlib_buffer_get_free_list_index(...) and - * vlib_buffer_set_free_list_index(...) functions */ -#define VLIB_BUFFER_FREE_LIST_INDEX_MASK ((1 << 5) - 1) - -#define VLIB_BUFFER_IS_TRACED (1 << 5) -#define VLIB_BUFFER_LOG2_NEXT_PRESENT (6) -#define VLIB_BUFFER_NEXT_PRESENT (1 << VLIB_BUFFER_LOG2_NEXT_PRESENT) -#define VLIB_BUFFER_IS_RECYCLED (1 << 7) -#define VLIB_BUFFER_TOTAL_LENGTH_VALID (1 << 8) -#define VLIB_BUFFER_REPL_FAIL (1 << 9) -#define VLIB_BUFFER_RECYCLE (1 << 10) -#define VLIB_BUFFER_FLOW_REPORT (1 << 11) -#define VLIB_BUFFER_EXT_HDR_VALID (1 << 12) - - /* User defined buffer flags. */ -#define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n)) -#define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n)) STRUCT_MARK (template_end); @@ -143,7 +160,10 @@ typedef struct /**< Only valid for first buffer in chain. Current length plus total length given here give total number of bytes in buffer chain. */ - u32 align_pad; /**< available */ + vlib_buffer_free_list_index_t free_list_index; /** < only used if + VLIB_BUFFER_NON_DEFAULT_FREELIST + flag is set */ + u8 align_pad[3]; /**< available */ u32 opaque2[12]; /**< More opaque data, see ../vnet/vnet/buffer.h */ /***** end of second cache line */ @@ -337,7 +357,7 @@ typedef struct vlib_buffer_free_list_t vlib_buffer_t buffer_init_template; /* Our index into vlib_main_t's buffer_free_list_pool. */ - u32 index; + vlib_buffer_free_list_index_t index; /* Number of data bytes for buffers in this free list. */ u32 n_data_bytes; @@ -400,7 +420,8 @@ typedef struct min_n_buffers_each_physmem_alloc, u8 * name); void (*vlib_buffer_delete_free_list_cb) (struct vlib_main_t * vm, - u32 free_list_index); + vlib_buffer_free_list_index_t + free_list_index); } vlib_buffer_callbacks_t; extern vlib_buffer_callbacks_t *vlib_buffer_callbacks; @@ -479,7 +500,7 @@ typedef struct u32 next_index; /* Free list to use to allocate new buffers. */ - u32 free_list_index; + vlib_buffer_free_list_index_t free_list_index; } tx; struct diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index b73fb266d64..cda44d6aa20 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -254,22 +254,26 @@ vlib_buffer_round_size (u32 size) return round_pow2 (size, sizeof (vlib_buffer_t)); } -always_inline u32 +always_inline vlib_buffer_free_list_index_t vlib_buffer_get_free_list_index (vlib_buffer_t * b) { - return b->flags & VLIB_BUFFER_FREE_LIST_INDEX_MASK; + if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NON_DEFAULT_FREELIST)) + return b->free_list_index; + + return 0; } always_inline void -vlib_buffer_set_free_list_index (vlib_buffer_t * b, u32 index) +vlib_buffer_set_free_list_index (vlib_buffer_t * b, + vlib_buffer_free_list_index_t index) { - /* if there is an need for more free lists we should consider - storig data in the 2nd cacheline */ - ASSERT (VLIB_BUFFER_FREE_LIST_INDEX_MASK & 1); - ASSERT (index <= VLIB_BUFFER_FREE_LIST_INDEX_MASK); - - b->flags &= ~VLIB_BUFFER_FREE_LIST_INDEX_MASK; - b->flags |= index & VLIB_BUFFER_FREE_LIST_INDEX_MASK; + if (PREDICT_FALSE (index)) + { + b->flags |= VLIB_BUFFER_NON_DEFAULT_FREELIST; + b->free_list_index = index; + } + else + b->flags &= ~VLIB_BUFFER_NON_DEFAULT_FREELIST; } /** \brief Allocate buffers from specific freelist into supplied array @@ -283,7 +287,8 @@ vlib_buffer_set_free_list_index (vlib_buffer_t * b, u32 index) always_inline u32 vlib_buffer_alloc_from_free_list (vlib_main_t * vm, u32 * buffers, - u32 n_buffers, u32 free_list_index) + u32 n_buffers, + vlib_buffer_free_list_index_t index) { vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_free_list_t *fl; @@ -292,7 +297,7 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm, ASSERT (bm->cb.vlib_buffer_fill_free_list_cb); - fl = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); + fl = pool_elt_at_index (bm->buffer_free_list_pool, index); len = vec_len (fl->buffers); @@ -429,10 +434,12 @@ vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index) } /* Add/delete buffer free lists. */ -u32 vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes, - char *fmt, ...); +vlib_buffer_free_list_index_t vlib_buffer_create_free_list (vlib_main_t * vm, + u32 n_data_bytes, + char *fmt, ...); always_inline void -vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index) +vlib_buffer_delete_free_list (vlib_main_t * vm, + vlib_buffer_free_list_index_t free_list_index) { vlib_buffer_main_t *bm = vm->buffer_main; @@ -442,8 +449,12 @@ vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index) } /* Find already existing public free list with given size or create one. */ -u32 vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes, - char *fmt, ...); +vlib_buffer_free_list_index_t vlib_buffer_get_or_create_free_list (vlib_main_t + * vm, + u32 + n_data_bytes, + char *fmt, + ...); /* Merge two free lists */ void vlib_buffer_merge_free_lists (vlib_buffer_free_list_t * dst, @@ -455,7 +466,7 @@ void vlib_buffer_free_list_fill_unaligned (vlib_main_t * vm, free_list, uword n_unaligned_buffers); -always_inline u32 +always_inline vlib_buffer_free_list_index_t vlib_buffer_get_free_list_with_size (vlib_main_t * vm, u32 size) { vlib_buffer_main_t *bm = vm->buffer_main; @@ -467,17 +478,18 @@ vlib_buffer_get_free_list_with_size (vlib_main_t * vm, u32 size) always_inline vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list (vlib_main_t * vm, vlib_buffer_t * b, - u32 * index) + vlib_buffer_free_list_index_t * index) { vlib_buffer_main_t *bm = vm->buffer_main; - u32 i; + vlib_buffer_free_list_index_t i; *index = i = vlib_buffer_get_free_list_index (b); return pool_elt_at_index (bm->buffer_free_list_pool, i); } always_inline vlib_buffer_free_list_t * -vlib_buffer_get_free_list (vlib_main_t * vm, u32 free_list_index) +vlib_buffer_get_free_list (vlib_main_t * vm, + vlib_buffer_free_list_index_t free_list_index) { vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_free_list_t *f; @@ -491,10 +503,10 @@ vlib_buffer_get_free_list (vlib_main_t * vm, u32 free_list_index) } always_inline u32 -vlib_buffer_free_list_buffer_size (vlib_main_t * vm, u32 free_list_index) +vlib_buffer_free_list_buffer_size (vlib_main_t * vm, + vlib_buffer_free_list_index_t index) { - vlib_buffer_free_list_t *f = - vlib_buffer_get_free_list (vm, free_list_index); + vlib_buffer_free_list_t *f = vlib_buffer_get_free_list (vm, index); return f->n_data_bytes; } @@ -525,7 +537,7 @@ vlib_copy_buffers (u32 * dst, u32 * src, u32 n) /* Append given data to end of buffer, possibly allocating new buffers. */ u32 vlib_buffer_add_data (vlib_main_t * vm, - u32 free_list_index, + vlib_buffer_free_list_index_t free_list_index, u32 buffer_index, void *data, u32 n_data_bytes); /* duplicate all buffers in chain */ @@ -772,7 +784,7 @@ vlib_buffer_chain_increase_length (vlib_buffer_t * first, * Returns the number of copied bytes. */ always_inline u16 vlib_buffer_chain_append_data (vlib_main_t * vm, - u32 free_list_index, + vlib_buffer_free_list_index_t free_list_index, vlib_buffer_t * first, vlib_buffer_t * last, void *data, u16 data_len) { @@ -795,10 +807,11 @@ vlib_buffer_chain_append_data (vlib_main_t * vm, * chained and points to the last buffer in the chain. */ u16 vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, - u32 free_list_index, + vlib_buffer_free_list_index_t + free_list_index, vlib_buffer_t * first, - vlib_buffer_t ** last, - void *data, u16 data_len); + vlib_buffer_t ** last, void *data, + u16 data_len); void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * first); format_function_t format_vlib_buffer, format_vlib_buffer_and_data, @@ -814,7 +827,7 @@ typedef struct u32 min_n_buffers_each_physmem_alloc; /* Buffer free list for this template. */ - u32 free_list_index; + vlib_buffer_free_list_index_t free_list_index; u32 *free_buffers; } vlib_packet_template_t; @@ -895,6 +908,7 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst, /* Not in the first 16 octets. */ dst->n_add_refs = src->n_add_refs; + vlib_buffer_set_free_list_index (dst, fl->index); /* Make sure it really worked. */ #define _(f) ASSERT (dst->f == src->f); @@ -936,43 +950,6 @@ vlib_buffer_add_to_free_list (vlib_main_t * vm, } } -always_inline void -vlib_buffer_init_two_for_free_list (vlib_buffer_t * dst0, - vlib_buffer_t * dst1, - vlib_buffer_free_list_t * fl) -{ - vlib_buffer_t *src = &fl->buffer_init_template; - - /* Make sure buffer template is sane. */ - ASSERT (fl->index == vlib_buffer_get_free_list_index (src)); - - clib_memcpy (STRUCT_MARK_PTR (dst0, template_start), - STRUCT_MARK_PTR (src, template_start), - STRUCT_OFFSET_OF (vlib_buffer_t, template_end) - - STRUCT_OFFSET_OF (vlib_buffer_t, template_start)); - - clib_memcpy (STRUCT_MARK_PTR (dst1, template_start), - STRUCT_MARK_PTR (src, template_start), - STRUCT_OFFSET_OF (vlib_buffer_t, template_end) - - STRUCT_OFFSET_OF (vlib_buffer_t, template_start)); - - /* Not in the first 16 octets. */ - dst0->n_add_refs = src->n_add_refs; - dst1->n_add_refs = src->n_add_refs; - - /* Make sure it really worked. */ -#define _(f) ASSERT (dst0->f == src->f); ASSERT( dst1->f == src->f) - _(current_data); - _(current_length); - _(flags); -#undef _ - - ASSERT (dst0->total_length_not_including_first_buffer == 0); - ASSERT (dst1->total_length_not_including_first_buffer == 0); - ASSERT (dst0->n_add_refs == 0); - ASSERT (dst1->n_add_refs == 0); -} - #if CLIB_DEBUG > 0 extern u32 *vlib_buffer_state_validation_lock; extern uword *vlib_buffer_state_validation_hash; @@ -1065,7 +1042,7 @@ vlib_buffer_chain_compress (vlib_main_t * vm, return; } /* probe free list to find allocated buffer size to avoid overfill */ - u32 index; + vlib_buffer_free_list_index_t index; vlib_buffer_free_list_t *free_list = vlib_buffer_get_buffer_free_list (vm, first, &index); |