summaryrefslogtreecommitdiffstats
path: root/vlib
diff options
context:
space:
mode:
Diffstat (limited to 'vlib')
-rw-r--r--vlib/vlib/buffer.h45
-rw-r--r--vlib/vlib/buffer_funcs.h37
-rw-r--r--vlib/vlib/dpdk_buffer.c51
3 files changed, 80 insertions, 53 deletions
diff --git a/vlib/vlib/buffer.h b/vlib/vlib/buffer.h
index 6322481b696..9c148ef2f21 100644
--- a/vlib/vlib/buffer.h
+++ b/vlib/vlib/buffer.h
@@ -45,7 +45,16 @@
#include <vppinfra/serialize.h>
#include <vppinfra/vector.h>
#include <vlib/error.h> /* for vlib_error_t */
+
+#if DPDK > 0
+#include <rte_config.h>
+#define VLIB_BUFFER_DATA_SIZE (2048)
+#define VLIB_BUFFER_PRE_DATA_SIZE RTE_PKTMBUF_HEADROOM
+#else
#include <vlib/config.h> /* for __PRE_DATA_SIZE */
+#define VLIB_BUFFER_DATA_SIZE (512)
+#define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
+#endif
#ifdef CLIB_HAVE_VEC128
typedef u8x16 vlib_copy_unit_t;
@@ -62,6 +71,7 @@ typedef uword vlib_copy_unit_t;
/* VLIB buffer representation. */
typedef struct {
+ CLIB_CACHE_LINE_ALIGN_MARK(cacheline0);
/* Offset within data[] that we are currently processing.
If negative current header points into predata area. */
i16 current_data; /**< signed offset in data[], pre_data[]
@@ -124,23 +134,25 @@ typedef struct {
u32 opaque[8]; /**< Opaque data used by sub-graphs for their own purposes.
See .../vnet/vnet/buffer.h
*/
- /***** end of first cache line */
+ CLIB_CACHE_LINE_ALIGN_MARK(cacheline1);
u32 opaque2[16]; /**< More opaque data, in its own cache line */
/***** end of second cache line */
- u8 pre_data [__PRE_DATA_SIZE]; /**< Space for inserting data
- before buffer start.
- Packet rewrite string will be
- rewritten backwards and may extend
- back before buffer->data[0].
- Must come directly before packet data.
- */
-
-#define VLIB_BUFFER_PRE_DATA_SIZE (ARRAY_LEN (((vlib_buffer_t *)0)->pre_data))
+ CLIB_CACHE_LINE_ALIGN_MARK(cacheline2);
+ u8 pre_data [VLIB_BUFFER_PRE_DATA_SIZE]; /**< Space for inserting data
+ before buffer start.
+ Packet rewrite string will be
+ rewritten backwards and may extend
+ back before buffer->data[0].
+ Must come directly before packet data.
+ */
+
u8 data[0]; /**< Packet data. Hardware DMA here */
} vlib_buffer_t; /* Must be a multiple of 64B. */
+#define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
+
/** \brief Prefetch buffer metadata.
The first 64 bytes of buffer contains most header information
@@ -283,13 +295,7 @@ typedef struct {
initializing static data for each packet generated. */
vlib_buffer_free_list_t * buffer_free_list_pool;
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX (0)
-
-#if DPDK == 1
-/* must be same as dpdk buffer size */
-#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES (2048)
-#else
-#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES (512)
-#endif
+#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES VLIB_BUFFER_DATA_SIZE
/* Hash table mapping buffer size (rounded to next unit of
sizeof (vlib_buffer_t)) to free list index. */
@@ -357,6 +363,11 @@ serialize_vlib_buffer_n_bytes (serialize_main_t * m)
return sm->tx.n_total_data_bytes + s->current_buffer_index + vec_len (s->overflow_buffer);
}
+#if DPDK > 0
+#define rte_mbuf_from_vlib_buffer(x) (((struct rte_mbuf *)x) - 1)
+#define vlib_buffer_from_rte_mbuf(x) ((vlib_buffer_t *)(x+1))
+#endif
+
/*
*/
diff --git a/vlib/vlib/buffer_funcs.h b/vlib/vlib/buffer_funcs.h
index eea417a94f0..f7bdb12d680 100644
--- a/vlib/vlib/buffer_funcs.h
+++ b/vlib/vlib/buffer_funcs.h
@@ -254,7 +254,7 @@ u8 * vlib_validate_buffers (vlib_main_t * vm,
clib_error_t *
vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs,
- unsigned mbuf_size, unsigned socket_id);
+ unsigned socket_id);
/** \brief Allocate buffers into supplied array
@@ -425,11 +425,9 @@ vlib_buffer_chain_init(vlib_buffer_t *first)
first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
#if DPDK == 1
- (((struct rte_mbuf *) first) - 1)->nb_segs = 1;
- (((struct rte_mbuf *) first) - 1)->next = 0;
- (((struct rte_mbuf *) first) - 1)->pkt_len = 0;
- (((struct rte_mbuf *) first) - 1)->data_len = 0;
- (((struct rte_mbuf *) first) - 1)->data_off = RTE_PKTMBUF_HEADROOM + first->current_data;
+ struct rte_mbuf * mb = rte_mbuf_from_vlib_buffer(first);
+ rte_pktmbuf_reset(mb);
+ mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + first->current_data;
#endif
}
@@ -446,11 +444,17 @@ vlib_buffer_chain_buffer(vlib_main_t *vm,
next_buffer->current_length = 0;
next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
#if DPDK == 1
- (((struct rte_mbuf *) first) - 1)->nb_segs++;
- (((struct rte_mbuf *) last) - 1)->next = (((struct rte_mbuf *) next_buffer) - 1);
- (((struct rte_mbuf *) next_buffer) - 1)->data_len = 0;
- (((struct rte_mbuf *) next_buffer) - 1)->data_off = RTE_PKTMBUF_HEADROOM + next_buffer->current_data;
- (((struct rte_mbuf *) next_buffer) - 1)->next = 0;
+ struct rte_mbuf * mb;
+ mb = rte_mbuf_from_vlib_buffer(first);
+ mb->nb_segs++;
+
+ mb = rte_mbuf_from_vlib_buffer(last);
+ mb->next = rte_mbuf_from_vlib_buffer(next_buffer);
+
+ mb = rte_mbuf_from_vlib_buffer(next_buffer);
+ mb->data_len = 0;
+ mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + next_buffer->current_data;
+ mb->next = 0;
#endif
return next_buffer;
}
@@ -468,8 +472,10 @@ vlib_buffer_chain_increase_length(vlib_buffer_t *first,
if (first != last)
first->total_length_not_including_first_buffer += len;
#if DPDK == 1
- (((struct rte_mbuf *) first) - 1)->pkt_len += len;
- (((struct rte_mbuf *) last) - 1)->data_len += len;
+ struct rte_mbuf * mb_first = rte_mbuf_from_vlib_buffer(first);
+ struct rte_mbuf * mb_last = rte_mbuf_from_vlib_buffer(last);
+ mb_first->pkt_len += len;
+ mb_last->data_len += len;
#endif
}
@@ -589,6 +595,11 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * _dst,
vlib_buffer_union_t * dst = (vlib_buffer_union_t *) _dst;
vlib_buffer_union_t * src = (vlib_buffer_union_t *) &fl->buffer_init_template;
+ /* Make sure vlib_buffer_t is cacheline aligned and sized */
+ ASSERT(STRUCT_OFFSET_OF(vlib_buffer_t, cacheline0) == 0);
+ ASSERT(STRUCT_OFFSET_OF(vlib_buffer_t, cacheline1) == CLIB_CACHE_LINE_BYTES);
+ ASSERT(STRUCT_OFFSET_OF(vlib_buffer_t, cacheline2) == CLIB_CACHE_LINE_BYTES * 2);
+
/* Make sure buffer template is sane. */
ASSERT (fl->index == fl->buffer_init_template.free_list_index);
diff --git a/vlib/vlib/dpdk_buffer.c b/vlib/vlib/dpdk_buffer.c
index 04a6447d633..db1fde18c77 100644
--- a/vlib/vlib/dpdk_buffer.c
+++ b/vlib/vlib/dpdk_buffer.c
@@ -66,12 +66,10 @@
#include <vlib/vlib.h>
-phys_addr_t __attribute__ ((weak)) rte_mem_virt2phy();
-int __attribute__ ((weak)) rte_eal_has_hugepages();
-unsigned __attribute__ ((weak)) rte_socket_id();
-struct rte_mempool * __attribute__ ((weak)) rte_mempool_create();
-void __attribute__ ((weak)) rte_pktmbuf_init();
-void __attribute__ ((weak)) rte_pktmbuf_pool_init();
+#pragma weak rte_mem_virt2phy
+#pragma weak rte_eal_has_hugepages
+#pragma weak rte_socket_id
+#pragma weak rte_pktmbuf_pool_create
uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, vlib_buffer_t * b_first)
{
@@ -400,13 +398,13 @@ del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
for (i = 0; i < vec_len (f->unaligned_buffers); i++) {
b = vlib_get_buffer (vm, f->unaligned_buffers[i]);
- mb = ((struct rte_mbuf *)b)-1;
+ mb = rte_mbuf_from_vlib_buffer(b);
ASSERT(rte_mbuf_refcnt_read(mb) == 1);
rte_pktmbuf_free (mb);
}
for (i = 0; i < vec_len (f->aligned_buffers); i++) {
b = vlib_get_buffer (vm, f->aligned_buffers[i]);
- mb = ((struct rte_mbuf *)b)-1;
+ mb = rte_mbuf_from_vlib_buffer(b);
ASSERT(rte_mbuf_refcnt_read(mb) == 1);
rte_pktmbuf_free (mb);
}
@@ -487,7 +485,7 @@ fill_free_list (vlib_main_t * vm,
mb->data_off = RTE_PKTMBUF_HEADROOM;
mb->nb_segs = 1;
- b = (vlib_buffer_t *)(mb+1);
+ b = vlib_buffer_from_rte_mbuf(mb);
bi = vlib_get_buffer_index (vm, b);
vec_add1_aligned (fl->aligned_buffers, bi, sizeof (vlib_copy_unit_t));
@@ -726,7 +724,7 @@ vlib_buffer_free_inline (vlib_main_t * vm,
{
if (PREDICT_TRUE (b->clone_count == 0))
{
- mb = ((struct rte_mbuf *)b)-1;
+ mb = rte_mbuf_from_vlib_buffer(b);
ASSERT(rte_mbuf_refcnt_read(mb) == 1);
rte_pktmbuf_free (mb);
}
@@ -820,7 +818,7 @@ vlib_packet_template_get_packet (vlib_main_t * vm,
/* Fix up mbuf header length fields */
struct rte_mbuf * mb;
- mb = ((struct rte_mbuf *)b) - 1;
+ mb = rte_mbuf_from_vlib_buffer(b);
mb->data_len = b->current_length;
mb->pkt_len = b->current_length;
@@ -916,22 +914,26 @@ vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm,
void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * b_first)
{
vlib_buffer_t *b = b_first, *prev = b_first;
- struct rte_mbuf *mb_first = ((struct rte_mbuf *) b) - 1;
+ struct rte_mbuf *mb_prev, *mb, *mb_first;
- mb_first->pkt_len = mb_first-> data_len = b_first->current_length;
+ mb_first = rte_mbuf_from_vlib_buffer(b_first);
+
+ mb_first->pkt_len = mb_first->data_len = b_first->current_length;
while (b->flags & VLIB_BUFFER_NEXT_PRESENT) {
b = vlib_get_buffer(vm, b->next_buffer);
+ mb = rte_mbuf_from_vlib_buffer(b);
+ mb_prev = rte_mbuf_from_vlib_buffer(prev);
mb_first->nb_segs++;
mb_first->pkt_len += b->current_length;
- (((struct rte_mbuf *) prev) - 1)->next = (((struct rte_mbuf *) b) - 1);
- (((struct rte_mbuf *) b) - 1)->data_len = b->current_length;
+ mb_prev->next = mb;
+ mb->data_len = b->current_length;
prev = b;
}
}
clib_error_t *
vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs,
- unsigned mbuf_size, unsigned socket_id)
+ unsigned socket_id)
{
vlib_buffer_main_t * bm = vm->buffer_main;
vlib_physmem_main_t * vpm = &vm->physmem_main;
@@ -939,7 +941,7 @@ vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs,
uword new_start, new_size;
int i;
- if (!rte_mempool_create)
+ if (!rte_pktmbuf_pool_create)
return clib_error_return (0, "not linked with DPDK");
vec_validate_aligned(bm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES);
@@ -949,12 +951,15 @@ vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs,
return 0;
u8 * pool_name = format(0, "mbuf_pool_socket%u%c",socket_id, 0);
- rmp = rte_mempool_create((char *) pool_name,
- num_mbufs, mbuf_size, 512,
- sizeof(struct rte_pktmbuf_pool_private),
- rte_pktmbuf_pool_init, NULL,
- rte_pktmbuf_init, NULL,
- socket_id, 0);
+
+ rmp = rte_pktmbuf_pool_create((char *) pool_name, /* pool name */
+ num_mbufs, /* number of mbufs */
+ 512, /* cache size */
+ VLIB_BUFFER_HDR_SIZE, /* priv size */
+ VLIB_BUFFER_PRE_DATA_SIZE
+ + VLIB_BUFFER_DATA_SIZE, /* dataroom size */
+ socket_id); /* cpu socket */
+
vec_free(pool_name);
if (rmp)