diff options
Diffstat (limited to 'vlib')
-rw-r--r-- | vlib/vlib/buffer.h | 45 | ||||
-rw-r--r-- | vlib/vlib/buffer_funcs.h | 37 | ||||
-rw-r--r-- | vlib/vlib/dpdk_buffer.c | 51 |
3 files changed, 80 insertions, 53 deletions
diff --git a/vlib/vlib/buffer.h b/vlib/vlib/buffer.h index 6322481b..9c148ef2 100644 --- a/vlib/vlib/buffer.h +++ b/vlib/vlib/buffer.h @@ -45,7 +45,16 @@ #include <vppinfra/serialize.h> #include <vppinfra/vector.h> #include <vlib/error.h> /* for vlib_error_t */ + +#if DPDK > 0 +#include <rte_config.h> +#define VLIB_BUFFER_DATA_SIZE (2048) +#define VLIB_BUFFER_PRE_DATA_SIZE RTE_PKTMBUF_HEADROOM +#else #include <vlib/config.h> /* for __PRE_DATA_SIZE */ +#define VLIB_BUFFER_DATA_SIZE (512) +#define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE +#endif #ifdef CLIB_HAVE_VEC128 typedef u8x16 vlib_copy_unit_t; @@ -62,6 +71,7 @@ typedef uword vlib_copy_unit_t; /* VLIB buffer representation. */ typedef struct { + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); /* Offset within data[] that we are currently processing. If negative current header points into predata area. */ i16 current_data; /**< signed offset in data[], pre_data[] @@ -124,23 +134,25 @@ typedef struct { u32 opaque[8]; /**< Opaque data used by sub-graphs for their own purposes. See .../vnet/vnet/buffer.h */ - /***** end of first cache line */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline1); u32 opaque2[16]; /**< More opaque data, in its own cache line */ /***** end of second cache line */ - u8 pre_data [__PRE_DATA_SIZE]; /**< Space for inserting data - before buffer start. - Packet rewrite string will be - rewritten backwards and may extend - back before buffer->data[0]. - Must come directly before packet data. - */ - -#define VLIB_BUFFER_PRE_DATA_SIZE (ARRAY_LEN (((vlib_buffer_t *)0)->pre_data)) + CLIB_CACHE_LINE_ALIGN_MARK(cacheline2); + u8 pre_data [VLIB_BUFFER_PRE_DATA_SIZE]; /**< Space for inserting data + before buffer start. + Packet rewrite string will be + rewritten backwards and may extend + back before buffer->data[0]. + Must come directly before packet data. + */ + u8 data[0]; /**< Packet data. Hardware DMA here */ } vlib_buffer_t; /* Must be a multiple of 64B. */ +#define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE) + /** \brief Prefetch buffer metadata. The first 64 bytes of buffer contains most header information @@ -283,13 +295,7 @@ typedef struct { initializing static data for each packet generated. */ vlib_buffer_free_list_t * buffer_free_list_pool; #define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX (0) - -#if DPDK == 1 -/* must be same as dpdk buffer size */ -#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES (2048) -#else -#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES (512) -#endif +#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES VLIB_BUFFER_DATA_SIZE /* Hash table mapping buffer size (rounded to next unit of sizeof (vlib_buffer_t)) to free list index. */ @@ -357,6 +363,11 @@ serialize_vlib_buffer_n_bytes (serialize_main_t * m) return sm->tx.n_total_data_bytes + s->current_buffer_index + vec_len (s->overflow_buffer); } +#if DPDK > 0 +#define rte_mbuf_from_vlib_buffer(x) (((struct rte_mbuf *)x) - 1) +#define vlib_buffer_from_rte_mbuf(x) ((vlib_buffer_t *)(x+1)) +#endif + /* */ diff --git a/vlib/vlib/buffer_funcs.h b/vlib/vlib/buffer_funcs.h index eea417a9..f7bdb12d 100644 --- a/vlib/vlib/buffer_funcs.h +++ b/vlib/vlib/buffer_funcs.h @@ -254,7 +254,7 @@ u8 * vlib_validate_buffers (vlib_main_t * vm, clib_error_t * vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs, - unsigned mbuf_size, unsigned socket_id); + unsigned socket_id); /** \brief Allocate buffers into supplied array @@ -425,11 +425,9 @@ vlib_buffer_chain_init(vlib_buffer_t *first) first->flags &= ~VLIB_BUFFER_NEXT_PRESENT; first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; #if DPDK == 1 - (((struct rte_mbuf *) first) - 1)->nb_segs = 1; - (((struct rte_mbuf *) first) - 1)->next = 0; - (((struct rte_mbuf *) first) - 1)->pkt_len = 0; - (((struct rte_mbuf *) first) - 1)->data_len = 0; - (((struct rte_mbuf *) first) - 1)->data_off = RTE_PKTMBUF_HEADROOM + first->current_data; + struct rte_mbuf * mb = rte_mbuf_from_vlib_buffer(first); + rte_pktmbuf_reset(mb); + mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + first->current_data; #endif } @@ -446,11 +444,17 @@ vlib_buffer_chain_buffer(vlib_main_t *vm, next_buffer->current_length = 0; next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT; #if DPDK == 1 - (((struct rte_mbuf *) first) - 1)->nb_segs++; - (((struct rte_mbuf *) last) - 1)->next = (((struct rte_mbuf *) next_buffer) - 1); - (((struct rte_mbuf *) next_buffer) - 1)->data_len = 0; - (((struct rte_mbuf *) next_buffer) - 1)->data_off = RTE_PKTMBUF_HEADROOM + next_buffer->current_data; - (((struct rte_mbuf *) next_buffer) - 1)->next = 0; + struct rte_mbuf * mb; + mb = rte_mbuf_from_vlib_buffer(first); + mb->nb_segs++; + + mb = rte_mbuf_from_vlib_buffer(last); + mb->next = rte_mbuf_from_vlib_buffer(next_buffer); + + mb = rte_mbuf_from_vlib_buffer(next_buffer); + mb->data_len = 0; + mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + next_buffer->current_data; + mb->next = 0; #endif return next_buffer; } @@ -468,8 +472,10 @@ vlib_buffer_chain_increase_length(vlib_buffer_t *first, if (first != last) first->total_length_not_including_first_buffer += len; #if DPDK == 1 - (((struct rte_mbuf *) first) - 1)->pkt_len += len; - (((struct rte_mbuf *) last) - 1)->data_len += len; + struct rte_mbuf * mb_first = rte_mbuf_from_vlib_buffer(first); + struct rte_mbuf * mb_last = rte_mbuf_from_vlib_buffer(last); + mb_first->pkt_len += len; + mb_last->data_len += len; #endif } @@ -589,6 +595,11 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * _dst, vlib_buffer_union_t * dst = (vlib_buffer_union_t *) _dst; vlib_buffer_union_t * src = (vlib_buffer_union_t *) &fl->buffer_init_template; + /* Make sure vlib_buffer_t is cacheline aligned and sized */ + ASSERT(STRUCT_OFFSET_OF(vlib_buffer_t, cacheline0) == 0); + ASSERT(STRUCT_OFFSET_OF(vlib_buffer_t, cacheline1) == CLIB_CACHE_LINE_BYTES); + ASSERT(STRUCT_OFFSET_OF(vlib_buffer_t, cacheline2) == CLIB_CACHE_LINE_BYTES * 2); + /* Make sure buffer template is sane. */ ASSERT (fl->index == fl->buffer_init_template.free_list_index); diff --git a/vlib/vlib/dpdk_buffer.c b/vlib/vlib/dpdk_buffer.c index 04a6447d..db1fde18 100644 --- a/vlib/vlib/dpdk_buffer.c +++ b/vlib/vlib/dpdk_buffer.c @@ -66,12 +66,10 @@ #include <vlib/vlib.h> -phys_addr_t __attribute__ ((weak)) rte_mem_virt2phy(); -int __attribute__ ((weak)) rte_eal_has_hugepages(); -unsigned __attribute__ ((weak)) rte_socket_id(); -struct rte_mempool * __attribute__ ((weak)) rte_mempool_create(); -void __attribute__ ((weak)) rte_pktmbuf_init(); -void __attribute__ ((weak)) rte_pktmbuf_pool_init(); +#pragma weak rte_mem_virt2phy +#pragma weak rte_eal_has_hugepages +#pragma weak rte_socket_id +#pragma weak rte_pktmbuf_pool_create uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, vlib_buffer_t * b_first) { @@ -400,13 +398,13 @@ del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) for (i = 0; i < vec_len (f->unaligned_buffers); i++) { b = vlib_get_buffer (vm, f->unaligned_buffers[i]); - mb = ((struct rte_mbuf *)b)-1; + mb = rte_mbuf_from_vlib_buffer(b); ASSERT(rte_mbuf_refcnt_read(mb) == 1); rte_pktmbuf_free (mb); } for (i = 0; i < vec_len (f->aligned_buffers); i++) { b = vlib_get_buffer (vm, f->aligned_buffers[i]); - mb = ((struct rte_mbuf *)b)-1; + mb = rte_mbuf_from_vlib_buffer(b); ASSERT(rte_mbuf_refcnt_read(mb) == 1); rte_pktmbuf_free (mb); } @@ -487,7 +485,7 @@ fill_free_list (vlib_main_t * vm, mb->data_off = RTE_PKTMBUF_HEADROOM; mb->nb_segs = 1; - b = (vlib_buffer_t *)(mb+1); + b = vlib_buffer_from_rte_mbuf(mb); bi = vlib_get_buffer_index (vm, b); vec_add1_aligned (fl->aligned_buffers, bi, sizeof (vlib_copy_unit_t)); @@ -726,7 +724,7 @@ vlib_buffer_free_inline (vlib_main_t * vm, { if (PREDICT_TRUE (b->clone_count == 0)) { - mb = ((struct rte_mbuf *)b)-1; + mb = rte_mbuf_from_vlib_buffer(b); ASSERT(rte_mbuf_refcnt_read(mb) == 1); rte_pktmbuf_free (mb); } @@ -820,7 +818,7 @@ vlib_packet_template_get_packet (vlib_main_t * vm, /* Fix up mbuf header length fields */ struct rte_mbuf * mb; - mb = ((struct rte_mbuf *)b) - 1; + mb = rte_mbuf_from_vlib_buffer(b); mb->data_len = b->current_length; mb->pkt_len = b->current_length; @@ -916,22 +914,26 @@ vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * b_first) { vlib_buffer_t *b = b_first, *prev = b_first; - struct rte_mbuf *mb_first = ((struct rte_mbuf *) b) - 1; + struct rte_mbuf *mb_prev, *mb, *mb_first; - mb_first->pkt_len = mb_first-> data_len = b_first->current_length; + mb_first = rte_mbuf_from_vlib_buffer(b_first); + + mb_first->pkt_len = mb_first->data_len = b_first->current_length; while (b->flags & VLIB_BUFFER_NEXT_PRESENT) { b = vlib_get_buffer(vm, b->next_buffer); + mb = rte_mbuf_from_vlib_buffer(b); + mb_prev = rte_mbuf_from_vlib_buffer(prev); mb_first->nb_segs++; mb_first->pkt_len += b->current_length; - (((struct rte_mbuf *) prev) - 1)->next = (((struct rte_mbuf *) b) - 1); - (((struct rte_mbuf *) b) - 1)->data_len = b->current_length; + mb_prev->next = mb; + mb->data_len = b->current_length; prev = b; } } clib_error_t * vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs, - unsigned mbuf_size, unsigned socket_id) + unsigned socket_id) { vlib_buffer_main_t * bm = vm->buffer_main; vlib_physmem_main_t * vpm = &vm->physmem_main; @@ -939,7 +941,7 @@ vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs, uword new_start, new_size; int i; - if (!rte_mempool_create) + if (!rte_pktmbuf_pool_create) return clib_error_return (0, "not linked with DPDK"); vec_validate_aligned(bm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES); @@ -949,12 +951,15 @@ vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs, return 0; u8 * pool_name = format(0, "mbuf_pool_socket%u%c",socket_id, 0); - rmp = rte_mempool_create((char *) pool_name, - num_mbufs, mbuf_size, 512, - sizeof(struct rte_pktmbuf_pool_private), - rte_pktmbuf_pool_init, NULL, - rte_pktmbuf_init, NULL, - socket_id, 0); + + rmp = rte_pktmbuf_pool_create((char *) pool_name, /* pool name */ + num_mbufs, /* number of mbufs */ + 512, /* cache size */ + VLIB_BUFFER_HDR_SIZE, /* priv size */ + VLIB_BUFFER_PRE_DATA_SIZE + + VLIB_BUFFER_DATA_SIZE, /* dataroom size */ + socket_id); /* cpu socket */ + vec_free(pool_name); if (rmp) |