Use rte_mempool private data for storing vlib_buffer_t

Change-Id: If3fc88a35bc0b736376113a39667caea42802ea1
Signed-off-by: Damjan Marion <damarion@cisco.com>

18 files changed, 126 insertions, 112 deletions

diff --git a/build-data/platforms/virl.mk b/build-data/platforms/virl.mk
index 29b47dad87f..f47f3691ae9 100644
--- a/build-data/platforms/virl.mk
+++ b/build-data/platforms/virl.mk
@@ -24,8 +24,6 @@ vpp_configure_args_virl = --with-dpdk
 vnet_configure_args_virl = --with-dpdk --with-virl
 
 # Set these parameters carefully. The vlib_buffer_t is 128 bytes, i.e.
-# dpdk_headroom = uiotarball_headroom = vlib_pre_data + 128
-dpdk_configure_args_virl = --with-headroom=256
 vlib_configure_args_virl = --with-pre-data=128
 
 # Override default -march and CONFIG_RTE_MACHINE settings
diff --git a/build-data/platforms/vpp.mk b/build-data/platforms/vpp.mk
index 6b1ba82f12a..50c85dcc7c1 100644
--- a/build-data/platforms/vpp.mk
+++ b/build-data/platforms/vpp.mk
@@ -24,8 +24,6 @@ vpp_configure_args_vpp = --with-dpdk
 vnet_configure_args_vpp = --with-dpdk
 
 # Set these parameters carefully. The vlib_buffer_t is 128 bytes, i.e.
-# dpdk_headroom = uiotarball_headroom = vlib_pre_data + 128
-dpdk_configure_args_vpp = --with-headroom=256
 vlib_configure_args_vpp = --with-pre-data=128
 
 
diff --git a/dpdk/Makefile b/dpdk/Makefile
index a07e86159b2..dabbf6adf48 100644
--- a/dpdk/Makefile
+++ b/dpdk/Makefile
@@ -16,7 +16,7 @@ SHELL := /bin/bash
 
 DPDK_BUILD_DIR        ?= $(CURDIR)/_build
 DPDK_INSTALL_DIR      ?= $(CURDIR)/_install
-DPDK_PKTMBUF_HEADROOM ?= 256
+DPDK_PKTMBUF_HEADROOM ?= 128
 DPDK_DOWNLOAD_DIR     ?= $(HOME)/Downloads
 DPDK_MARCH            ?= native
 DPDK_DEBUG            ?= n
diff --git a/vlib/vlib/buffer.h b/vlib/vlib/buffer.h
index 6322481b696..9c148ef2f21 100644
--- a/vlib/vlib/buffer.h
+++ b/vlib/vlib/buffer.h
@@ -45,7 +45,16 @@
 #include <vppinfra/serialize.h>
 #include <vppinfra/vector.h>
 #include <vlib/error.h>		/* for vlib_error_t */
+
+#if DPDK > 0
+#include <rte_config.h>
+#define VLIB_BUFFER_DATA_SIZE		(2048)
+#define VLIB_BUFFER_PRE_DATA_SIZE	RTE_PKTMBUF_HEADROOM
+#else
 #include <vlib/config.h>        /* for __PRE_DATA_SIZE */
+#define VLIB_BUFFER_DATA_SIZE		(512)
+#define VLIB_BUFFER_PRE_DATA_SIZE	__PRE_DATA_SIZE
+#endif
 
 #ifdef CLIB_HAVE_VEC128
 typedef u8x16 vlib_copy_unit_t;
@@ -62,6 +71,7 @@ typedef uword vlib_copy_unit_t;
     
 /* VLIB buffer representation. */
 typedef struct {
+  CLIB_CACHE_LINE_ALIGN_MARK(cacheline0);
   /* Offset within data[] that we are currently processing.
      If negative current header points into predata area. */ 
   i16 current_data;  /**< signed offset in data[], pre_data[]  
@@ -124,23 +134,25 @@ typedef struct {
   u32 opaque[8]; /**< Opaque data used by sub-graphs for their own purposes. 
                     See .../vnet/vnet/buffer.h
                  */
-  /***** end of first cache line */
+  CLIB_CACHE_LINE_ALIGN_MARK(cacheline1);
 
   u32 opaque2[16];  /**< More opaque data, in its own cache line */
 
   /***** end of second cache line */
-  u8 pre_data [__PRE_DATA_SIZE]; /**< Space for inserting data 
-                                     before buffer start.  
-                                     Packet rewrite string will be
-                                     rewritten backwards and may extend 
-                                     back before buffer->data[0].
-                                     Must come directly before packet data.
-                                 */
-
-#define VLIB_BUFFER_PRE_DATA_SIZE (ARRAY_LEN (((vlib_buffer_t *)0)->pre_data))
+  CLIB_CACHE_LINE_ALIGN_MARK(cacheline2);
+  u8 pre_data [VLIB_BUFFER_PRE_DATA_SIZE]; /**< Space for inserting data
+                                               before buffer start.
+                                               Packet rewrite string will be
+                                               rewritten backwards and may extend
+                                               back before buffer->data[0].
+                                               Must come directly before packet data.
+                                            */
+
   u8 data[0]; /**< Packet data. Hardware DMA here */
 } vlib_buffer_t;  /* Must be a multiple of 64B. */
 
+#define VLIB_BUFFER_HDR_SIZE  (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
+
 /** \brief Prefetch buffer metadata.
     The first 64 bytes of buffer contains most header information
 
@@ -283,13 +295,7 @@ typedef struct {
      initializing static data for each packet generated. */
   vlib_buffer_free_list_t * buffer_free_list_pool;
 #define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX (0)
-
-#if DPDK == 1
-/* must be same as dpdk buffer size */
-#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES (2048)
-#else
-#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES (512)
-#endif
+#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES VLIB_BUFFER_DATA_SIZE
 
   /* Hash table mapping buffer size (rounded to next unit of
      sizeof (vlib_buffer_t)) to free list index. */
@@ -357,6 +363,11 @@ serialize_vlib_buffer_n_bytes (serialize_main_t * m)
   return sm->tx.n_total_data_bytes + s->current_buffer_index + vec_len (s->overflow_buffer);
 }
 
+#if DPDK > 0
+#define rte_mbuf_from_vlib_buffer(x) (((struct rte_mbuf *)x) - 1)
+#define vlib_buffer_from_rte_mbuf(x) ((vlib_buffer_t *)(x+1))
+#endif
+
 /*
  */
 
diff --git a/vlib/vlib/buffer_funcs.h b/vlib/vlib/buffer_funcs.h
index eea417a94f0..f7bdb12d680 100644
--- a/vlib/vlib/buffer_funcs.h
+++ b/vlib/vlib/buffer_funcs.h
@@ -254,7 +254,7 @@ u8 * vlib_validate_buffers (vlib_main_t * vm,
 
 clib_error_t *
 vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs,
-                        unsigned mbuf_size, unsigned socket_id);
+                        unsigned socket_id);
 
 /** \brief Allocate buffers into supplied array
 
@@ -425,11 +425,9 @@ vlib_buffer_chain_init(vlib_buffer_t *first)
   first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
   first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
 #if DPDK == 1
-  (((struct rte_mbuf *) first) - 1)->nb_segs = 1;
-  (((struct rte_mbuf *) first) - 1)->next = 0;
-  (((struct rte_mbuf *) first) - 1)->pkt_len = 0;
-  (((struct rte_mbuf *) first) - 1)->data_len = 0;
-  (((struct rte_mbuf *) first) - 1)->data_off = RTE_PKTMBUF_HEADROOM + first->current_data;
+  struct rte_mbuf * mb = rte_mbuf_from_vlib_buffer(first);
+  rte_pktmbuf_reset(mb);
+  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + first->current_data;
 #endif
 }
 
@@ -446,11 +444,17 @@ vlib_buffer_chain_buffer(vlib_main_t *vm,
   next_buffer->current_length = 0;
   next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
 #if DPDK == 1
-  (((struct rte_mbuf *) first) - 1)->nb_segs++;
-  (((struct rte_mbuf *) last) - 1)->next = (((struct rte_mbuf *) next_buffer) - 1);
-  (((struct rte_mbuf *) next_buffer) - 1)->data_len = 0;
-  (((struct rte_mbuf *) next_buffer) - 1)->data_off = RTE_PKTMBUF_HEADROOM + next_buffer->current_data;
-  (((struct rte_mbuf *) next_buffer) - 1)->next = 0;
+  struct rte_mbuf * mb;
+  mb = rte_mbuf_from_vlib_buffer(first);
+  mb->nb_segs++;
+
+  mb = rte_mbuf_from_vlib_buffer(last);
+  mb->next = rte_mbuf_from_vlib_buffer(next_buffer);
+
+  mb = rte_mbuf_from_vlib_buffer(next_buffer);
+  mb->data_len = 0;
+  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + next_buffer->current_data;
+  mb->next = 0;
 #endif
   return next_buffer;
 }
@@ -468,8 +472,10 @@ vlib_buffer_chain_increase_length(vlib_buffer_t *first,
   if (first != last)
     first->total_length_not_including_first_buffer += len;
 #if DPDK == 1
-  (((struct rte_mbuf *) first) - 1)->pkt_len += len;
-  (((struct rte_mbuf *) last) - 1)->data_len += len;
+  struct rte_mbuf * mb_first = rte_mbuf_from_vlib_buffer(first);
+  struct rte_mbuf * mb_last = rte_mbuf_from_vlib_buffer(last);
+  mb_first->pkt_len += len;
+  mb_last->data_len += len;
 #endif
 }
 
@@ -589,6 +595,11 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * _dst,
   vlib_buffer_union_t * dst = (vlib_buffer_union_t *) _dst;
   vlib_buffer_union_t * src = (vlib_buffer_union_t *) &fl->buffer_init_template;
 
+  /* Make sure vlib_buffer_t is cacheline aligned and sized */
+  ASSERT(STRUCT_OFFSET_OF(vlib_buffer_t, cacheline0) == 0);
+  ASSERT(STRUCT_OFFSET_OF(vlib_buffer_t, cacheline1) == CLIB_CACHE_LINE_BYTES);
+  ASSERT(STRUCT_OFFSET_OF(vlib_buffer_t, cacheline2) == CLIB_CACHE_LINE_BYTES * 2);
+
   /* Make sure buffer template is sane. */
   ASSERT (fl->index == fl->buffer_init_template.free_list_index);
 
diff --git a/vlib/vlib/dpdk_buffer.c b/vlib/vlib/dpdk_buffer.c
index 04a6447d633..db1fde18c77 100644
--- a/vlib/vlib/dpdk_buffer.c
+++ b/vlib/vlib/dpdk_buffer.c
@@ -66,12 +66,10 @@
 
 #include <vlib/vlib.h>
 
-phys_addr_t          __attribute__ ((weak)) rte_mem_virt2phy();
-int                  __attribute__ ((weak)) rte_eal_has_hugepages();
-unsigned             __attribute__ ((weak)) rte_socket_id();
-struct rte_mempool * __attribute__ ((weak)) rte_mempool_create();
-void                 __attribute__ ((weak)) rte_pktmbuf_init();
-void                 __attribute__ ((weak)) rte_pktmbuf_pool_init();
+#pragma weak rte_mem_virt2phy
+#pragma weak rte_eal_has_hugepages
+#pragma weak rte_socket_id
+#pragma weak rte_pktmbuf_pool_create
 
 uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, vlib_buffer_t * b_first)
 {
@@ -400,13 +398,13 @@ del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
 
   for (i = 0; i < vec_len (f->unaligned_buffers); i++) {
       b = vlib_get_buffer (vm, f->unaligned_buffers[i]);
-      mb = ((struct rte_mbuf *)b)-1;
+      mb = rte_mbuf_from_vlib_buffer(b);
       ASSERT(rte_mbuf_refcnt_read(mb) == 1);
       rte_pktmbuf_free (mb);
   }
   for (i = 0; i < vec_len (f->aligned_buffers); i++) {
       b = vlib_get_buffer (vm, f->aligned_buffers[i]);
-      mb = ((struct rte_mbuf *)b)-1;
+      mb = rte_mbuf_from_vlib_buffer(b);
       ASSERT(rte_mbuf_refcnt_read(mb) == 1);
       rte_pktmbuf_free (mb);
   }
@@ -487,7 +485,7 @@ fill_free_list (vlib_main_t * vm,
       mb->data_off = RTE_PKTMBUF_HEADROOM;
       mb->nb_segs = 1;
 
-      b = (vlib_buffer_t *)(mb+1);
+      b = vlib_buffer_from_rte_mbuf(mb);
       bi = vlib_get_buffer_index (vm, b);
       
       vec_add1_aligned (fl->aligned_buffers, bi, sizeof (vlib_copy_unit_t));
@@ -726,7 +724,7 @@ vlib_buffer_free_inline (vlib_main_t * vm,
         {
 	  if (PREDICT_TRUE (b->clone_count == 0))
 	    {
-	      mb = ((struct rte_mbuf *)b)-1;
+	      mb = rte_mbuf_from_vlib_buffer(b);
 	      ASSERT(rte_mbuf_refcnt_read(mb) == 1);
 	      rte_pktmbuf_free (mb);
 	    }
@@ -820,7 +818,7 @@ vlib_packet_template_get_packet (vlib_main_t * vm,
 
   /* Fix up mbuf header length fields */
   struct rte_mbuf * mb;
-  mb = ((struct rte_mbuf *)b) - 1;
+  mb = rte_mbuf_from_vlib_buffer(b);
   mb->data_len = b->current_length;
   mb->pkt_len = b->current_length;
 
@@ -916,22 +914,26 @@ vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm,
 void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * b_first)
 {
   vlib_buffer_t *b = b_first, *prev = b_first;
-  struct rte_mbuf *mb_first = ((struct rte_mbuf *) b) - 1;
+  struct rte_mbuf *mb_prev, *mb, *mb_first;
 
-  mb_first->pkt_len = mb_first-> data_len = b_first->current_length;
+  mb_first = rte_mbuf_from_vlib_buffer(b_first);
+
+  mb_first->pkt_len = mb_first->data_len = b_first->current_length;
   while (b->flags & VLIB_BUFFER_NEXT_PRESENT) {
       b = vlib_get_buffer(vm, b->next_buffer);
+      mb = rte_mbuf_from_vlib_buffer(b);
+      mb_prev = rte_mbuf_from_vlib_buffer(prev);
       mb_first->nb_segs++;
       mb_first->pkt_len += b->current_length;
-      (((struct rte_mbuf *) prev) - 1)->next = (((struct rte_mbuf *) b) - 1);
-      (((struct rte_mbuf *) b) - 1)->data_len = b->current_length;
+      mb_prev->next = mb;
+      mb->data_len = b->current_length;
       prev = b;
   }
 }
 
 clib_error_t *
 vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs,
-                        unsigned mbuf_size, unsigned socket_id)
+                        unsigned socket_id)
 {
   vlib_buffer_main_t * bm = vm->buffer_main;
   vlib_physmem_main_t * vpm = &vm->physmem_main;
@@ -939,7 +941,7 @@ vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs,
   uword new_start, new_size;
   int i;
 
-  if (!rte_mempool_create)
+  if (!rte_pktmbuf_pool_create)
     return clib_error_return (0, "not linked with DPDK");
 
   vec_validate_aligned(bm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES);
@@ -949,12 +951,15 @@ vlib_buffer_pool_create(vlib_main_t * vm, unsigned num_mbufs,
     return 0;
 
   u8 * pool_name = format(0, "mbuf_pool_socket%u%c",socket_id, 0);
-  rmp = rte_mempool_create((char *) pool_name,
-                           num_mbufs, mbuf_size, 512,
-                           sizeof(struct rte_pktmbuf_pool_private),
-                           rte_pktmbuf_pool_init, NULL,
-                           rte_pktmbuf_init, NULL,
-                           socket_id, 0);
+
+  rmp = rte_pktmbuf_pool_create((char *) pool_name,         /* pool name */
+				 num_mbufs,                 /* number of mbufs */
+				 512,                       /* cache size */
+				 VLIB_BUFFER_HDR_SIZE,      /* priv size */
+				 VLIB_BUFFER_PRE_DATA_SIZE
+				 + VLIB_BUFFER_DATA_SIZE,   /* dataroom size */
+				 socket_id);                /* cpu socket */
+
   vec_free(pool_name);
 
   if (rmp)
diff --git a/vnet/vnet/devices/af_packet/node.c b/vnet/vnet/devices/af_packet/node.c
index 15a96f442f9..efe0e1c2586 100644
--- a/vnet/vnet/devices/af_packet/node.c
+++ b/vnet/vnet/devices/af_packet/node.c
@@ -94,11 +94,6 @@ buffer_add_to_chain(vlib_main_t *vm, u32 bi, u32 first_bi, u32 prev_bi)
   vlib_buffer_t * b = vlib_get_buffer (vm, bi);
   vlib_buffer_t * first_b = vlib_get_buffer (vm, first_bi);
   vlib_buffer_t * prev_b = vlib_get_buffer (vm, prev_bi);
-#if DPDK > 0
-  struct rte_mbuf * mbuf = ((struct rte_mbuf *) b) - 1;
-  struct rte_mbuf * first_mbuf = ((struct rte_mbuf *) first_b) - 1;
-  struct rte_mbuf * prev_mbuf = ((struct rte_mbuf *) prev_b) - 1;
-#endif
 
   /* update first buffer */
   first_b->total_length_not_including_first_buffer +=  b->current_length;
@@ -111,6 +106,9 @@ buffer_add_to_chain(vlib_main_t *vm, u32 bi, u32 first_bi, u32 prev_bi)
   b->next_buffer = 0;
 
 #if DPDK > 0
+  struct rte_mbuf * mbuf = rte_mbuf_from_vlib_buffer(b);
+  struct rte_mbuf * first_mbuf = rte_mbuf_from_vlib_buffer(first_b);
+  struct rte_mbuf * prev_mbuf = rte_mbuf_from_vlib_buffer(prev_b);
   first_mbuf->nb_segs++;
   prev_mbuf->next = mbuf;
   mbuf->data_len = b->current_length;
diff --git a/vnet/vnet/devices/dpdk/device.c b/vnet/vnet/devices/dpdk/device.c
index 02703cc59b2..dcf95803c84 100644
--- a/vnet/vnet/devices/dpdk/device.c
+++ b/vnet/vnet/devices/dpdk/device.c
@@ -87,7 +87,7 @@ static struct rte_mbuf * dpdk_replicate_packet_mb (vlib_buffer_t * b)
   unsigned socket_id = rte_socket_id();
 
   ASSERT (bm->pktmbuf_pools[socket_id]);
-  pkt_mb = ((struct rte_mbuf *)b)-1;
+  pkt_mb = rte_mbuf_from_vlib_buffer(b);
   nb_segs = pkt_mb->nb_segs;
   for (nb_segs_left = nb_segs; nb_segs_left; nb_segs_left--)
     {
@@ -159,7 +159,7 @@ dpdk_tx_trace_buffer (dpdk_main_t * dm,
   dpdk_tx_dma_trace_t * t0;
   struct rte_mbuf * mb;
 
-  mb = ((struct rte_mbuf *)buffer)-1;
+  mb = rte_mbuf_from_vlib_buffer(buffer);
 
   t0 = vlib_add_trace (vm, node, buffer, sizeof (t0[0]));
   t0->queue_index = queue_id;
@@ -541,7 +541,7 @@ dpdk_interface_tx (vlib_main_t * vm,
         {
           u32 bi0 = from[n_packets];
           vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
-          struct rte_mbuf *mb0 = ((struct rte_mbuf *)b0) - 1;
+          struct rte_mbuf *mb0 = rte_mbuf_from_vlib_buffer(b0);
           rte_pktmbuf_free (mb0);
         }
       return n_on_ring;
@@ -584,9 +584,9 @@ dpdk_interface_tx (vlib_main_t * vm,
       pref0 = vlib_get_buffer (vm, pi0);
       pref1 = vlib_get_buffer (vm, pi1);
 
-      prefmb0 = ((struct rte_mbuf *)pref0) - 1;
-      prefmb1 = ((struct rte_mbuf *)pref1) - 1;
-      
+      prefmb0 = rte_mbuf_from_vlib_buffer(pref0);
+      prefmb1 = rte_mbuf_from_vlib_buffer(pref1);
+
       CLIB_PREFETCH(prefmb0, CLIB_CACHE_LINE_BYTES, LOAD);
       CLIB_PREFETCH(pref0, CLIB_CACHE_LINE_BYTES, LOAD);
       CLIB_PREFETCH(prefmb1, CLIB_CACHE_LINE_BYTES, LOAD);
@@ -599,8 +599,8 @@ dpdk_interface_tx (vlib_main_t * vm,
       b0 = vlib_get_buffer (vm, bi0);
       b1 = vlib_get_buffer (vm, bi1);
 
-      mb0 = ((struct rte_mbuf *)b0) - 1;
-      mb1 = ((struct rte_mbuf *)b1) - 1;
+      mb0 = rte_mbuf_from_vlib_buffer(b0);
+      mb1 = rte_mbuf_from_vlib_buffer(b1);
 
       any_clone = b0->clone_count | b1->clone_count;
       if (PREDICT_FALSE(any_clone != 0))
@@ -701,7 +701,7 @@ dpdk_interface_tx (vlib_main_t * vm,
       
       b0 = vlib_get_buffer (vm, bi0);
 
-      mb0 = ((struct rte_mbuf *)b0) - 1;
+      mb0 = rte_mbuf_from_vlib_buffer(b0);
       if (PREDICT_FALSE(b0->clone_count != 0))
     {
       struct rte_mbuf * mb0_new = dpdk_replicate_packet_mb (b0);
@@ -921,7 +921,7 @@ dpdk_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
           vlib_buffer_main_t * bm = vm->buffer_main;
           memset(&conf, 0, sizeof(conf));
           snprintf(conf.name, RTE_KNI_NAMESIZE, "vpp%u", xd->kni_port_id);
-          conf.mbuf_size = MBUF_SIZE;
+          conf.mbuf_size = VLIB_BUFFER_DATA_SIZE;
           memset(&ops, 0, sizeof(ops));
           ops.port_id = xd->kni_port_id;
           ops.change_mtu = kni_change_mtu;
diff --git a/vnet/vnet/devices/dpdk/dpdk.h b/vnet/vnet/devices/dpdk/dpdk.h
index 656f39ea681..14f7b3e8518 100644
--- a/vnet/vnet/devices/dpdk/dpdk.h
+++ b/vnet/vnet/devices/dpdk/dpdk.h
@@ -62,7 +62,6 @@
 #define always_inline static inline __attribute__ ((__always_inline__))
 #endif
 
-#define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)
 #define NB_MBUF   (32<<10)
 
 extern vnet_device_class_t dpdk_device_class;
diff --git a/vnet/vnet/devices/dpdk/init.c b/vnet/vnet/devices/dpdk/init.c
index 8fe95ae5f09..8bb253a3a66 100644
--- a/vnet/vnet/devices/dpdk/init.c
+++ b/vnet/vnet/devices/dpdk/init.c
@@ -1405,13 +1405,13 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input)
   rte_dump_physmem_layout(stdout);
 
   /* main thread 1st */
-  error = vlib_buffer_pool_create(vm, dm->num_mbufs, MBUF_SIZE, rte_socket_id());
+  error = vlib_buffer_pool_create(vm, dm->num_mbufs, rte_socket_id());
   if (error)
     return error;
 
   for (i = 0; i < RTE_MAX_LCORE; i++)
     {
-      error = vlib_buffer_pool_create(vm, dm->num_mbufs, MBUF_SIZE,
+      error = vlib_buffer_pool_create(vm, dm->num_mbufs,
                                       rte_lcore_to_socket_id(i));
       if (error)
         return error;
diff --git a/vnet/vnet/devices/dpdk/node.c b/vnet/vnet/devices/dpdk/node.c
index 72d564ba8b2..4f5a84ae29a 100644
--- a/vnet/vnet/devices/dpdk/node.c
+++ b/vnet/vnet/devices/dpdk/node.c
@@ -366,7 +366,7 @@ void dpdk_rx_trace (dpdk_main_t * dm,
       n_left -= 1;
 
       b0 = vlib_get_buffer (vm, bi0);
-      mb = ((struct rte_mbuf *)b0) - 1;
+      mb = rte_mbuf_from_vlib_buffer(b0);
       dpdk_rx_next_and_error_from_mb_flags_x1 (xd, mb, b0,
 					       &next0, &error0);
       vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0);
@@ -602,20 +602,20 @@ static inline u32 dpdk_device_input ( dpdk_main_t * dm,
           if (PREDICT_TRUE(n_buffers > 2))
           {
               struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2];
-              vlib_buffer_t *bp = (vlib_buffer_t *)(pfmb+1);
+              vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
               CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, STORE);
               CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
           }
 
           ASSERT(mb);
 
-          b0 = (vlib_buffer_t *)(mb+1);
+          b0 = vlib_buffer_from_rte_mbuf(mb);
 
           /* check whether EFD is looking for packets to discard */
           if (PREDICT_FALSE(efd_discard_burst))
             {
               vlib_thread_main_t * tm = vlib_get_thread_main();
-              
+
               if (PREDICT_TRUE(cntr_type = is_efd_discardable(tm, b0, mb)))
                 {
                   rte_pktmbuf_free(mb);
@@ -633,7 +633,7 @@ static inline u32 dpdk_device_input ( dpdk_main_t * dm,
           if (PREDICT_FALSE(mb->nb_segs > 1))
             {
               struct rte_mbuf *pfmb = mb->next;
-              vlib_buffer_t *bp = (vlib_buffer_t *)(pfmb+1);
+              vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
               CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
               CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
 	      b_chain = b0;
@@ -693,7 +693,7 @@ static inline u32 dpdk_device_input ( dpdk_main_t * dm,
 	    {
 	      ASSERT(mb_seg != 0);
 
-	      b_seg = (vlib_buffer_t *)(mb_seg+1);
+	      b_seg = vlib_buffer_from_rte_mbuf(mb_seg);
 	      vlib_buffer_init_for_free_list (b_seg, fl);
               b_seg->clone_count = 0;
 
@@ -1280,13 +1280,13 @@ void dpdk_io_thread (vlib_worker_thread_t * w,
               if (PREDICT_TRUE(n_buffers > 1))
                 {
                   struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2];
-                  vlib_buffer_t *bp = (vlib_buffer_t *)(pfmb+1);
+                  vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
                   CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
                   CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
                   CLIB_PREFETCH (bp->data, CLIB_CACHE_LINE_BYTES, LOAD);
                 }
                 
-              b0 = (vlib_buffer_t *)(mb+1);
+              b0 = vlib_buffer_from_rte_mbuf(mb);
 
               /* check whether EFD is looking for packets to discard */
               if (PREDICT_FALSE(efd_discard_burst))
@@ -1310,7 +1310,7 @@ void dpdk_io_thread (vlib_worker_thread_t * w,
               if (PREDICT_FALSE(mb->nb_segs > 1))
                 {
                   struct rte_mbuf *pfmb = mb->next;
-                  vlib_buffer_t *bp = (vlib_buffer_t *)(pfmb+1);
+                  vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
                   CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
                   CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
                   b_chain = b0;
@@ -1361,7 +1361,7 @@ void dpdk_io_thread (vlib_worker_thread_t * w,
                 {
                   ASSERT(mb_seg != 0);
  
-                  b_seg = (vlib_buffer_t *)(mb_seg+1);
+                  b_seg = vlib_buffer_from_rte_mbuf(mb_seg);
                   vlib_buffer_init_for_free_list (b_seg, fl);
                   b_seg->clone_count = 0;
  
@@ -1681,14 +1681,14 @@ dpdk_io_input (vlib_main_t * vm,
           if (PREDICT_TRUE(n_buffers > 1))
             {
               struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2];
-              vlib_buffer_t *bp = (vlib_buffer_t *)(pfmb+1);
+              vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
               CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
               CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
               CLIB_PREFETCH (bp->data, CLIB_CACHE_LINE_BYTES, LOAD);
             }
-                
-          b0 = (vlib_buffer_t *)(mb+1);
-                
+
+          b0 = vlib_buffer_from_rte_mbuf(mb);
+
           /* check whether EFD is looking for packets to discard */
           if (PREDICT_FALSE(efd_discard_burst))
             {
@@ -1711,7 +1711,7 @@ dpdk_io_input (vlib_main_t * vm,
           if (PREDICT_FALSE(mb->nb_segs > 1))
             {
               struct rte_mbuf *pfmb = mb->next;
-              vlib_buffer_t *bp = (vlib_buffer_t *)(pfmb+1);
+              vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
               CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
               CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
               b_chain = b0;
@@ -1762,7 +1762,7 @@ dpdk_io_input (vlib_main_t * vm,
             {
               ASSERT(mb_seg != 0);
  
-              b_seg = (vlib_buffer_t *)(mb_seg+1);
+              b_seg = vlib_buffer_from_rte_mbuf(mb_seg);
               vlib_buffer_init_for_free_list (b_seg, fl);
               b_seg->clone_count = 0;
  
diff --git a/vnet/vnet/devices/ssvm/ssvm_eth.h b/vnet/vnet/devices/ssvm/ssvm_eth.h
index 70d895b9ad4..8f1f88964c2 100644
--- a/vnet/vnet/devices/ssvm/ssvm_eth.h
+++ b/vnet/vnet/devices/ssvm/ssvm_eth.h
@@ -35,7 +35,7 @@ extern vnet_device_class_t ssvm_eth_device_class;
 extern vlib_node_registration_t ssvm_eth_input_node;
 
 #define SSVM_BUFFER_SIZE  \
-  (VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES + VLIB_BUFFER_PRE_DATA_SIZE)
+  (VLIB_BUFFER_DATA_SIZE + VLIB_BUFFER_PRE_DATA_SIZE)
 #define SSVM_PACKET_TYPE 1
 
 typedef struct {
diff --git a/vnet/vnet/dpdk_replication.h b/vnet/vnet/dpdk_replication.h
index b25558f9b89..6259c449af0 100644
--- a/vnet/vnet/dpdk_replication.h
+++ b/vnet/vnet/dpdk_replication.h
@@ -49,7 +49,7 @@ vlib_dpdk_clone_buffer (vlib_main_t * vm, vlib_buffer_t * b)
         return 0;
 
       src_buf = b;
-      rv = dst_buf = (vlib_buffer_t *)(rte_mbufs[0] + 1);
+      rv = dst_buf = vlib_buffer_from_rte_mbuf(rte_mbufs[0]);
       vlib_buffer_init_for_free_list (dst_buf, fl);
       copy_src = b->data + src_buf->current_data;
       copy_dst = dst_buf->data + src_buf->current_data;
@@ -75,7 +75,7 @@ vlib_dpdk_clone_buffer (vlib_main_t * vm, vlib_buffer_t * b)
           if (i < new_buffers_needed - 1)
             {
               src_buf = vlib_get_buffer (vm, src_buf->next_buffer);
-              dst_buf = (vlib_buffer_t *)(rte_mbufs[i+1] + 1);
+              dst_buf = vlib_buffer_from_rte_mbuf(rte_mbufs[i+1]);
               vlib_buffer_init_for_free_list (dst_buf, fl);
               copy_src = src_buf->data;
               copy_dst = dst_buf->data;
@@ -87,7 +87,7 @@ vlib_dpdk_clone_buffer (vlib_main_t * vm, vlib_buffer_t * b)
   if (rte_mempool_get_bulk (rmp, (void **)rte_mbufs, 1) < 0)
     return 0;
 
-  rv = (vlib_buffer_t *)(rte_mbufs[0] + 1);
+  rv = vlib_buffer_from_rte_mbuf(rte_mbufs[0]);
   vlib_buffer_init_for_free_list (rv, fl);
 
   memcpy(rv->data + b->current_data, b->data + b->current_data, 
diff --git a/vnet/vnet/ip/ip.h b/vnet/vnet/ip/ip.h
index 76a2552ffa4..de46ad38c5f 100644
--- a/vnet/vnet/ip/ip.h
+++ b/vnet/vnet/ip/ip.h
@@ -162,7 +162,7 @@ ip_incremental_checksum_buffer (vlib_main_t * vm, vlib_buffer_t * first_buffer,
 #if DPDK > 0
 {
   u32 n_bytes_left = n_bytes_to_checksum;
-  struct rte_mbuf * mb = ((struct rte_mbuf *)first_buffer)-1;
+  struct rte_mbuf * mb = rte_mbuf_from_vlib_buffer(first_buffer);
   u8 nb_segs = mb->nb_segs;
   ASSERT(mb->data_len >= first_buffer_offset);
   void * h;
diff --git a/vnet/vnet/ip/ip6_forward.c b/vnet/vnet/ip/ip6_forward.c
index 1d0e21e528c..a478bab5fc4 100644
--- a/vnet/vnet/ip/ip6_forward.c
+++ b/vnet/vnet/ip/ip6_forward.c
@@ -1445,7 +1445,7 @@ u16 ip6_tcp_udp_icmp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, ip6
 #if DPDK > 0
   if (p0) 
   {
-    struct rte_mbuf *mb = ((struct rte_mbuf *)p0)-1;
+    struct rte_mbuf *mb = rte_mbuf_from_vlib_buffer(p0);
     u8 nb_segs = mb->nb_segs;
 
     n_this_buffer = (p0->current_length > headers_size ?
diff --git a/vnet/vnet/pg/input.c b/vnet/vnet/pg/input.c
index 4ec61ca7b96..38402c2e989 100644
--- a/vnet/vnet/pg/input.c
+++ b/vnet/vnet/pg/input.c
@@ -61,7 +61,7 @@ pg_set_mbuf_metadata (pg_main_t * pg, u32 * buffers, u32 n_alloc)
   for (i = 0; i < n_alloc; i++)
     {
       b = vlib_get_buffer (vm, buffers[i]);
-      mb = ((struct rte_mbuf *)b) - 1;
+      mb = rte_mbuf_from_vlib_buffer(b);
 
       delta = vlib_buffer_length_in_chain (vm, b) - (i16) mb->pkt_len;
       new_data_len = (u16)((i16) mb->data_len + delta);
@@ -1473,9 +1473,9 @@ pg_stream_fill (pg_main_t * pg, pg_stream_t * s, u32 n_buffers)
           ({
             vlib_buffer_t * b;
             struct rte_mbuf *mb;
-            
+
             b = vlib_get_buffer(vm, bi0[0]);
-            mb = (struct rte_mbuf *)b - 1;
+            mb = rte_mbuf_from_vlib_buffer(b);
             ASSERT(rte_mbuf_refcnt_read(mb) == 1);
           }));
         }
diff --git a/vnet/vnet/unix/tapcli.c b/vnet/vnet/unix/tapcli.c
index 5b0ac9378ee..7776b9ee837 100644
--- a/vnet/vnet/unix/tapcli.c
+++ b/vnet/vnet/unix/tapcli.c
@@ -220,7 +220,7 @@ static uword tapcli_rx_iface(vlib_main_t * vm,
                             tapcli_interface_t * ti)
 {
   tapcli_main_t * tm = &tapcli_main;
-  const uword buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES;
+  const uword buffer_size = VLIB_BUFFER_DATA_SIZE;
   u32 n_trace = vlib_get_trace_count (vm, node);
   u8 set_trace = 0;
 
@@ -434,7 +434,7 @@ static clib_error_t *
 tapcli_config (vlib_main_t * vm, unformat_input_t * input)
 {
   tapcli_main_t *tm = &tapcli_main;
-  const uword buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES;
+  const uword buffer_size = VLIB_BUFFER_DATA_SIZE;
 
   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
     {
diff --git a/vnet/vnet/unix/tuntap.c b/vnet/vnet/unix/tuntap.c
index 77c60fd6ee2..7ea0b70348d 100644
--- a/vnet/vnet/unix/tuntap.c
+++ b/vnet/vnet/unix/tuntap.c
@@ -200,12 +200,11 @@ tuntap_rx (vlib_main_t * vm,
   tuntap_main_t * tm = &tuntap_main;
   vlib_buffer_t * b;
   u32 bi;
+  const uword buffer_size = VLIB_BUFFER_DATA_SIZE;
 #if DPDK == 0
-  const uword buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES;
   u32 free_list_index = VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX;
 #else
   dpdk_main_t * dm = &dpdk_main;
-  const uword buffer_size = MBUF_SIZE;
   u32 free_list_index = dm->vlib_buffer_free_list_index;
 #endif
 
@@ -262,7 +261,7 @@ tuntap_rx (vlib_main_t * vm,
 #endif
 	b = vlib_get_buffer (vm, tm->rx_buffers[i_rx]);
 #if DPDK == 1
-        mb = (((struct rte_mbuf *)b)-1);
+	mb = rte_mbuf_from_vlib_buffer(b);
 #endif
 	b->flags = 0;
 	b->current_data = 0;
@@ -445,12 +444,7 @@ tuntap_config (vlib_main_t * vm, unformat_input_t * input)
   u8 * name;
   int flags = IFF_TUN | IFF_NO_PI;
   int is_enabled = 0, is_ether = 0, have_normal_interface = 0;
-#if DPDK == 0
-  const uword buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES;
-#else
-  const uword buffer_size = MBUF_SIZE;
-#endif
-
+  const uword buffer_size = VLIB_BUFFER_DATA_SIZE;
 
   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
     {