vlib: store buffer memory information in the buffer_main

Currently, buffer index is calculated as a offset to the physmem
region shifted by log2_cacheline size.

When DPDK is used we "hack" physmem data with information taken from
dpdk mempool. This makes physmem code not usable with DPDK.

This change makes buffer memory start and size independent of physmem
basically allowing physmem to be used when DPDK plugin is loaded.

Change-Id: Ieb399d398f147583b9baab467152a352d58c9c31
Signed-off-by: Damjan Marion <damarion@cisco.com>

9 files changed, 96 insertions, 114 deletions

diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c
index fd1d84153db..aa73eb6c008 100644
--- a/src/plugins/dpdk/buffer.c
+++ b/src/plugins/dpdk/buffer.c
@@ -431,7 +431,6 @@ vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
 			 unsigned socket_id)
 {
   dpdk_main_t *dm = &dpdk_main;
-  vlib_physmem_main_t *vpm = &vm->physmem_main;
   struct rte_mempool *rmp;
   int i;
 
@@ -453,63 +452,10 @@ vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
   if (rmp)
     {
       {
-	uword this_pool_end;
-	uword this_pool_start;
-	uword this_pool_size;
-	uword save_vpm_start, save_vpm_end, save_vpm_size;
 	struct rte_mempool_memhdr *memhdr;
 
-	this_pool_start = ~0;
-	this_pool_end = 0;
-
 	STAILQ_FOREACH (memhdr, &rmp->mem_list, next)
-	{
-	  if (((uword) (memhdr->addr + memhdr->len)) > this_pool_end)
-	    this_pool_end = (uword) (memhdr->addr + memhdr->len);
-	  if (((uword) memhdr->addr) < this_pool_start)
-	    this_pool_start = (uword) (memhdr->addr);
-	}
-	ASSERT (this_pool_start < ~0 && this_pool_end > 0);
-	this_pool_size = this_pool_end - this_pool_start;
-
-	if (CLIB_DEBUG > 1)
-	  {
-	    clib_warning ("%s: pool start %llx pool end %llx pool size %lld",
-			  pool_name, this_pool_start, this_pool_end,
-			  this_pool_size);
-	    clib_warning
-	      ("before: virtual.start %llx virtual.end %llx virtual.size %lld",
-	       vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
-	  }
-
-	save_vpm_start = vpm->virtual.start;
-	save_vpm_end = vpm->virtual.end;
-	save_vpm_size = vpm->virtual.size;
-
-	if ((this_pool_start < vpm->virtual.start) || vpm->virtual.start == 0)
-	  vpm->virtual.start = this_pool_start;
-	if (this_pool_end > vpm->virtual.end)
-	  vpm->virtual.end = this_pool_end;
-
-	vpm->virtual.size = vpm->virtual.end - vpm->virtual.start;
-
-	if (CLIB_DEBUG > 1)
-	  {
-	    clib_warning
-	      ("after: virtual.start %llx virtual.end %llx virtual.size %lld",
-	       vpm->virtual.start, vpm->virtual.end, vpm->virtual.size);
-	  }
-
-	/* check if fits into buffer index range */
-	if ((u64) vpm->virtual.size >
-	    ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
-	  {
-	    clib_warning ("physmem: virtual size out of range!");
-	    vpm->virtual.start = save_vpm_start;
-	    vpm->virtual.end = save_vpm_end;
-	    vpm->virtual.size = save_vpm_size;
-	    rmp = 0;
-	  }
+	  vlib_buffer_add_mem_range (vm, (uword) memhdr->addr, memhdr->len);
       }
       if (rmp)
 	{
@@ -564,7 +510,8 @@ buffer_state_validation_init (vlib_main_t * vm)
 VLIB_INIT_FUNCTION (buffer_state_validation_init);
 #endif
 
-static vlib_buffer_callbacks_t callbacks = {
+/* *INDENT-OFF* */
+VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = {
   .vlib_buffer_alloc_cb = &dpdk_buffer_alloc,
   .vlib_buffer_alloc_from_free_list_cb = &dpdk_buffer_alloc_from_free_list,
   .vlib_buffer_free_cb = &dpdk_buffer_free,
@@ -572,15 +519,7 @@ static vlib_buffer_callbacks_t callbacks = {
   .vlib_packet_template_init_cb = &dpdk_packet_template_init,
   .vlib_buffer_delete_free_list_cb = &dpdk_buffer_delete_free_list,
 };
-
-static clib_error_t *
-dpdk_buffer_init (vlib_main_t * vm)
-{
-  vlib_buffer_cb_register (vm, &callbacks);
-  return 0;
-}
-
-VLIB_INIT_FUNCTION (dpdk_buffer_init);
+/* *INDENT-ON* */
 
 /** @endcond */
 /*
diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index e50064ae622..b2a095cfb1a 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -46,6 +46,8 @@
 #include <vlib/vlib.h>
 #include <vlib/unix/unix.h>
 
+vlib_buffer_callbacks_t *vlib_buffer_callbacks = 0;
+
 uword
 vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
 				       vlib_buffer_t * b_first)
@@ -584,11 +586,6 @@ alloc_from_free_list (vlib_main_t * vm,
 
   dst = alloc_buffers;
 
-  /* wait with buffer memory allocation as long as possible
-     in case external buffer manager takes over */
-  if (PREDICT_FALSE (vm->os_physmem_alloc_aligned == 0))
-    unix_physmem_init (vm, 0 /* fail_if_physical_memory_not_present */ );
-
   n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
   if (n_filled == 0)
     return 0;
@@ -944,6 +941,36 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
   return copied;
 }
 
+void
+vlib_buffer_add_mem_range (vlib_main_t * vm, uword start, uword size)
+{
+  vlib_buffer_main_t *bm = vm->buffer_main;
+
+  if (bm->buffer_mem_size == 0)
+    {
+      bm->buffer_mem_start = start;
+      bm->buffer_mem_size = size;
+    }
+  else if (start < bm->buffer_mem_start)
+    {
+      bm->buffer_mem_size += bm->buffer_mem_start - start;
+      bm->buffer_mem_start = start;
+      if (size > bm->buffer_mem_size)
+	bm->buffer_mem_size = size;
+    }
+  else if (start > bm->buffer_mem_start)
+    {
+      uword new_size = start - bm->buffer_mem_start + size;
+      if (new_size > bm->buffer_mem_size)
+	bm->buffer_mem_size = new_size;
+    }
+
+  if ((u64) bm->buffer_mem_size >
+      ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
+    {
+      clib_panic ("buffer memory size out of range!");
+    }
+}
 
 static u8 *
 format_vlib_buffer_free_list (u8 * s, va_list * va)
@@ -1011,6 +1038,7 @@ void
 vlib_buffer_cb_init (struct vlib_main_t *vm)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
+
   bm->cb.vlib_buffer_alloc_cb = &vlib_buffer_alloc_internal;
   bm->cb.vlib_buffer_alloc_from_free_list_cb =
     &vlib_buffer_alloc_from_free_list_internal;
@@ -1018,25 +1046,6 @@ vlib_buffer_cb_init (struct vlib_main_t *vm)
   bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal;
   bm->cb.vlib_buffer_delete_free_list_cb =
     &vlib_buffer_delete_free_list_internal;
-  bm->extern_buffer_mgmt = 0;
-}
-
-int
-vlib_buffer_cb_register (struct vlib_main_t *vm, vlib_buffer_callbacks_t * cb)
-{
-  vlib_buffer_main_t *bm = vm->buffer_main;
-  if (bm->extern_buffer_mgmt)
-    return -1;
-
-#define _(x) bm->cb.x = cb->x
-  _(vlib_buffer_alloc_cb);
-  _(vlib_buffer_alloc_from_free_list_cb);
-  _(vlib_buffer_free_cb);
-  _(vlib_buffer_free_no_next_cb);
-  _(vlib_buffer_delete_free_list_cb);
-#undef _
-  bm->extern_buffer_mgmt = 1;
-  return 0;
 }
 
 /** @endcond */
diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h
index 18e2437d775..b20538b7f8c 100644
--- a/src/vlib/buffer.h
+++ b/src/vlib/buffer.h
@@ -388,12 +388,20 @@ typedef struct
 					   u32 free_list_index);
 } vlib_buffer_callbacks_t;
 
+extern vlib_buffer_callbacks_t *vlib_buffer_callbacks;
+
 typedef struct
 {
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  /* Virtual memory address and size of buffer memory, used for calculating
+     buffer index */
+  uword buffer_mem_start;
+  uword buffer_mem_size;
+
   /* Buffer free callback, for subversive activities */
-  u32 (*buffer_free_callback) (struct vlib_main_t * vm,
-			       u32 * buffers,
-			       u32 n_buffers, u32 follow_buffer_next);
+    u32 (*buffer_free_callback) (struct vlib_main_t * vm,
+				 u32 * buffers,
+				 u32 n_buffers, u32 follow_buffer_next);
   /* Pool of buffer free lists.
      Multiple free lists exist for packet generator which uses
      separate free lists for each packet stream --- so as to avoid
@@ -417,12 +425,12 @@ typedef struct
 
   /* Callbacks */
   vlib_buffer_callbacks_t cb;
-  int extern_buffer_mgmt;
+  int callbacks_registered;
 } vlib_buffer_main_t;
 
+void vlib_buffer_add_mem_range (struct vlib_main_t *vm, uword start,
+				uword size);
 void vlib_buffer_cb_init (struct vlib_main_t *vm);
-int vlib_buffer_cb_register (struct vlib_main_t *vm,
-			     vlib_buffer_callbacks_t * cb);
 
 typedef struct
 {
@@ -498,6 +506,18 @@ serialize_vlib_buffer_n_bytes (serialize_main_t * m)
 
 #endif /* included_vlib_buffer_h */
 
+#define VLIB_BUFFER_REGISTER_CALLBACKS(x,...)                           \
+    __VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks;       \
+static void __vlib_add_buffer_callbacks_t_##x (void)                    \
+    __attribute__((__constructor__)) ;                                  \
+static void __vlib_add_buffer_callbacks_t_##x (void)                    \
+{                                                                       \
+    if (vlib_buffer_callbacks)                                          \
+      clib_panic ("vlib buffer callbacks already registered");          \
+    vlib_buffer_callbacks = &__##x##_buffer_callbacks;                  \
+}                                                                       \
+__VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks
+
 /*
  * fd.io coding-style-patch-verification: ON
  *
diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index 328660a3428..79e3e69c919 100644
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -56,8 +56,11 @@
 always_inline vlib_buffer_t *
 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
 {
-  return vlib_physmem_at_offset (&vm->physmem_main, ((uword) buffer_index)
-				 << CLIB_LOG2_CACHE_LINE_BYTES);
+  vlib_buffer_main_t *bm = vm->buffer_main;
+  uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
+  ASSERT (offset < bm->buffer_mem_size);
+
+  return uword_to_pointer (bm->buffer_mem_start + offset, void *);
 }
 
 /** \brief Translate buffer pointer into buffer index
@@ -66,10 +69,14 @@ vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
     @param p - (void *) buffer pointer
     @return - (u32) buffer index
 */
+
 always_inline u32
 vlib_get_buffer_index (vlib_main_t * vm, void *p)
 {
-  uword offset = vlib_physmem_offset_of (&vm->physmem_main, p);
+  vlib_buffer_main_t *bm = vm->buffer_main;
+  uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
+  ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
+  ASSERT (offset < bm->buffer_mem_size);
   ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
   return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
 }
diff --git a/src/vlib/main.c b/src/vlib/main.c
index 19d70232e6e..73548fbea72 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -43,6 +43,7 @@
 #include <vlib/threads.h>
 #include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>
 
+#include <vlib/unix/unix.h>
 #include <vlib/unix/cj.h>
 
 CJ_GLOBAL_LOG_PROTOTYPE;
@@ -466,7 +467,7 @@ vlib_put_next_frame (vlib_main_t * vm,
   vlib_frame_t *f;
   u32 n_vectors_in_frame;
 
-  if (vm->buffer_main->extern_buffer_mgmt == 0 && CLIB_DEBUG > 0)
+  if (vm->buffer_main->callbacks_registered == 0 && CLIB_DEBUG > 0)
     vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
 
   nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
@@ -1712,7 +1713,22 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
     vm->name = "VLIB";
 
   vec_validate (vm->buffer_main, 0);
-  vlib_buffer_cb_init (vm);
+  if (vlib_buffer_callbacks)
+    {
+      /* external plugin has registered own buffer callbacks
+         so we just copy them */
+      vlib_buffer_main_t *bm = vm->buffer_main;
+      clib_memcpy (&bm->cb, vlib_buffer_callbacks,
+		   sizeof (vlib_buffer_callbacks_t));
+      bm->callbacks_registered = 1;
+    }
+  else
+    {
+      vlib_physmem_main_t *vpm = &vm->physmem_main;
+      vlib_buffer_cb_init (vm);
+      unix_physmem_init (vm, 0 /* fail_if_physical_memory_not_present */ );
+      vlib_buffer_add_mem_range (vm, vpm->virtual.start, vpm->virtual.size);
+    }
 
   if ((error = vlib_thread_init (vm)))
     {
diff --git a/src/vlib/unix/physmem.c b/src/vlib/unix/physmem.c
index 8d10ad2e88d..933cc63b7f4 100644
--- a/src/vlib/unix/physmem.c
+++ b/src/vlib/unix/physmem.c
@@ -45,14 +45,10 @@ static void *
 unix_physmem_alloc_aligned (vlib_physmem_main_t * vpm, uword n_bytes,
 			    uword alignment)
 {
-  vlib_main_t *vm = vlib_get_main ();
   physmem_main_t *pm = &physmem_main;
   uword lo_offset, hi_offset;
   uword *to_free = 0;
 
-  if (vm->buffer_main->extern_buffer_mgmt)
-    clib_warning ("unsafe alloc!");
-
   /* IO memory is always at least cache aligned. */
   alignment = clib_max (alignment, CLIB_CACHE_LINE_BYTES);
 
@@ -270,11 +266,6 @@ show_physmem (vlib_main_t * vm,
 	      unformat_input_t * input, vlib_cli_command_t * cmd)
 {
   physmem_main_t *pm = &physmem_main;
-  if (vm->buffer_main->extern_buffer_mgmt)
-    {
-      vlib_cli_output (vm, "Not supported with external buffer management.");
-      return 0;
-    }
 
   if (pm->heap)
     vlib_cli_output (vm, "%U", format_mheap, pm->heap, /* verbose */ 1);
diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c
index 597ae060045..c3738a6a2aa 100644
--- a/src/vnet/pg/input.c
+++ b/src/vnet/pg/input.c
@@ -1214,7 +1214,7 @@ pg_stream_fill_helper (pg_main_t * pg,
    * Historically, the pg maintained its own free lists and
    * device drivers tx paths would return pkts.
    */
-  if (vm->buffer_main->extern_buffer_mgmt == 0 &&
+  if (vm->buffer_main->callbacks_registered == 0 &&
       !(s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE))
     f->buffer_init_function = pg_buffer_init;
   f->buffer_init_function_opaque =
@@ -1238,7 +1238,7 @@ pg_stream_fill_helper (pg_main_t * pg,
   n_alloc = n_allocated;
 
   /* Reinitialize buffers */
-  if (vm->buffer_main->extern_buffer_mgmt == 0 || CLIB_DEBUG > 0
+  if (vm->buffer_main->callbacks_registered == 0 || CLIB_DEBUG > 0
       || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE))
     init_buffers_inline
       (vm, s,
@@ -1246,7 +1246,7 @@ pg_stream_fill_helper (pg_main_t * pg,
        n_alloc, (bi - s->buffer_indices) * s->buffer_bytes /* data offset */ ,
        s->buffer_bytes,
        /* set_data */
-       vm->buffer_main->extern_buffer_mgmt != 0
+       vm->buffer_main->callbacks_registered != 0
        || (s->flags & PG_STREAM_FLAGS_DISABLE_BUFFER_RECYCLE) != 0);
 
   if (next_buffers)
diff --git a/src/vnet/pg/stream.c b/src/vnet/pg/stream.c
index 05d820a3807..a540b32bba4 100644
--- a/src/vnet/pg/stream.c
+++ b/src/vnet/pg/stream.c
@@ -438,7 +438,7 @@ pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
     pg_buffer_index_t *bi;
     int n;
 
-    if (vm->buffer_main->extern_buffer_mgmt)
+    if (vm->buffer_main->callbacks_registered)
       s->buffer_bytes = VLIB_BUFFER_DATA_SIZE;
 
     if (!s->buffer_bytes)
diff --git a/src/vnet/replication.c b/src/vnet/replication.c
index 233a8c2f100..1c6f28d2493 100644
--- a/src/vnet/replication.c
+++ b/src/vnet/replication.c
@@ -214,7 +214,7 @@ replication_recycle_callback (vlib_main_t * vm, vlib_buffer_free_list_t * fl)
 	  b0->flags |= VLIB_BUFFER_IS_RECYCLED;
 
 #if (CLIB_DEBUG > 0)
-	  if (vm->buffer_main->extern_buffer_mgmt == 0)
+	  if (vm->buffer_main->callbacks_registered == 0)
 	    vlib_buffer_set_known_state (vm, bi0,
 					 VLIB_BUFFER_KNOWN_ALLOCATED);
 #endif