buffers: keep buffer_main in vlib_main_t

Change-Id: I3bb1d9f83dd08f4b93acd4a281bfec0674e39c2e
Signed-off-by: Damjan Marion <damarion@cisco.com>

8 files changed, 48 insertions, 45 deletions

diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c
index 768d66153e5..37966ba9759 100644
--- a/src/plugins/dpdk/buffer.c
+++ b/src/plugins/dpdk/buffer.c
@@ -286,9 +286,9 @@ static_always_inline void
 vlib_buffer_free_inline (vlib_main_t * vm,
 			 u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm = vm->buffer_main;
   vlib_buffer_t *bufp[n_buffers], **b = bufp;
-  u32 thread_index = vlib_get_thread_index ();
+  u32 thread_index = vm->thread_index;
   int i = 0;
   u32 simple_mask = VLIB_BUFFER_NEXT_PRESENT;
   u32 n_left, *bi;
diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c
index 3171ba22f7d..6e45b573f2c 100644
--- a/src/plugins/memif/memif.c
+++ b/src/plugins/memif/memif.c
@@ -338,7 +338,7 @@ memif_init_regions_and_queues (memif_if_t * mif)
     {
       vlib_buffer_pool_t *bp;
       /* *INDENT-OFF* */
-      vec_foreach (bp, buffer_main.buffer_pools)
+      vec_foreach (bp, vm->buffer_main->buffer_pools)
 	{
 	  vlib_physmem_map_t *pm;
 	  pm = vlib_physmem_get_map (vm, bp->physmem_map_index);
diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index 0251f52bd87..547c26445af 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -53,8 +53,6 @@ vlib_buffer_callbacks_t *vlib_buffer_callbacks = 0;
    */
 static u32 vlib_buffer_physmem_sz = 14 << 20;
 
-vlib_buffer_main_t buffer_main;
-
 /* logging */
 static vlib_log_class_t buffer_log_default;
 
@@ -197,7 +195,7 @@ vlib_validate_buffer_helper (vlib_main_t * vm,
       vlib_buffer_known_state_t k;
       u8 *msg, *result;
 
-      k = vlib_buffer_is_known (b->next_buffer);
+      k = vlib_buffer_is_known (vm, b->next_buffer);
       if (k != VLIB_BUFFER_KNOWN_ALLOCATED)
 	return format (0, "next 0x%x: %U",
 		       b->next_buffer, format_vlib_buffer_known_state, k);
@@ -255,7 +253,7 @@ vlib_validate_buffers (vlib_main_t * vm,
 	  goto done;
 	}
 
-      k = vlib_buffer_is_known (bi);
+      k = vlib_buffer_is_known (vm, bi);
       if (k != known_state)
 	{
 	  msg = format (0, "is %U; expected %U",
@@ -329,7 +327,7 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm,
 
       bi = b[0];
       b += 1;
-      known = vlib_buffer_is_known (bi);
+      known = vlib_buffer_is_known (vm, bi);
       if (known != expected_state)
 	{
 	  ASSERT (0);
@@ -339,8 +337,8 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm,
 	     format_vlib_buffer_known_state, known, bi);
 	}
 
-      vlib_buffer_set_known_state
-	(bi, is_free ? VLIB_BUFFER_KNOWN_FREE : VLIB_BUFFER_KNOWN_ALLOCATED);
+      vlib_buffer_set_known_state (vm, bi, is_free ? VLIB_BUFFER_KNOWN_FREE :
+				   VLIB_BUFFER_KNOWN_ALLOCATED);
     }
 }
 
@@ -350,7 +348,7 @@ vlib_buffer_create_free_list_helper (vlib_main_t * vm,
 				     u32 n_data_bytes,
 				     u32 is_public, u32 is_default, u8 * name)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm = vm->buffer_main;
   vlib_buffer_free_list_t *f;
   int i;
 
@@ -435,7 +433,7 @@ vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes,
 static void
 del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
 {
-  vlib_buffer_pool_t *bp = vlib_buffer_pool_get (f->buffer_pool_index);
+  vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, f->buffer_pool_index);
 
   vec_add_aligned (bp->buffers, f->buffers, vec_len (f->buffers),
 		   CLIB_CACHE_LINE_BYTES);
@@ -487,7 +485,7 @@ vlib_buffer_fill_free_list_internal (vlib_main_t * vm,
 				     uword min_free_buffers)
 {
   vlib_buffer_t *b;
-  vlib_buffer_pool_t *bp = vlib_buffer_pool_get (fl->buffer_pool_index);
+  vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, fl->buffer_pool_index);
   int n;
   u32 *bi;
   u32 n_alloc = 0;
@@ -530,7 +528,7 @@ vlib_buffer_fill_free_list_internal (vlib_main_t * vm,
       bi[0] = vlib_get_buffer_index (vm, b);
 
       if (CLIB_DEBUG > 0)
-	vlib_buffer_set_known_state (bi[0], VLIB_BUFFER_KNOWN_FREE);
+	vlib_buffer_set_known_state (vm, bi[0], VLIB_BUFFER_KNOWN_FREE);
 
       clib_memset (b, 0, sizeof (vlib_buffer_t));
       vlib_buffer_init_for_free_list (b, fl);
@@ -548,7 +546,7 @@ done:
 void *
 vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm = vm->buffer_main;
   void *rv = bm->buffer_free_callback;
 
   bm->buffer_free_callback = fp;
@@ -587,7 +585,7 @@ static_always_inline void
 vlib_buffer_free_inline (vlib_main_t * vm,
 			 u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm = vm->buffer_main;
   vlib_buffer_t *p, *b0, *b1, *b2, *b3;
   int i = 0;
   u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
@@ -792,7 +790,7 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
 u8
 vlib_buffer_register_physmem_map (vlib_main_t * vm, u32 physmem_map_index)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm = vm->buffer_main;
   vlib_buffer_pool_t *p;
   vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index);
   uword start = pointer_to_uword (m->base);
@@ -895,7 +893,7 @@ VLIB_CLI_COMMAND (show_buffers_command, static) = {
 clib_error_t *
 vlib_buffer_main_init (struct vlib_main_t * vm)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm;
   clib_error_t *error;
   u32 physmem_map_index;
   u8 pool_index;
@@ -903,6 +901,9 @@ vlib_buffer_main_init (struct vlib_main_t * vm)
 
   buffer_log_default = vlib_log_register_class ("buffer", 0);
 
+  bm = vm->buffer_main = clib_mem_alloc (sizeof (bm[0]));
+  clib_memset (bm, 0, sizeof (bm[0]));
+
   if (vlib_buffer_callbacks)
     {
       /* external plugin has registered own buffer callbacks
@@ -941,7 +942,7 @@ retry:
     return error;
 
   pool_index = vlib_buffer_register_physmem_map (vm, physmem_map_index);
-  vlib_buffer_pool_t *bp = vlib_buffer_pool_get (pool_index);
+  vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, pool_index);
   clib_spinlock_init (&bp->lock);
   bp->buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES +
     sizeof (vlib_buffer_t);
diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h
index 56d855101d5..672131b935d 100644
--- a/src/vlib/buffer.h
+++ b/src/vlib/buffer.h
@@ -458,16 +458,7 @@ typedef struct
   int callbacks_registered;
 } vlib_buffer_main_t;
 
-extern vlib_buffer_main_t buffer_main;
-
-static_always_inline vlib_buffer_pool_t *
-vlib_buffer_pool_get (u8 buffer_pool_index)
-{
-  vlib_buffer_main_t *bm = &buffer_main;
-  return vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
-}
-
-u8 vlib_buffer_register_physmem_map (struct vlib_main_t * vm,
+u8 vlib_buffer_register_physmem_map (struct vlib_main_t *vm,
 				     u32 physmem_map_index);
 
 clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm);
diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index 41b5164e0d8..9095bd3387f 100644
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -57,7 +57,7 @@
 always_inline vlib_buffer_t *
 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm = vm->buffer_main;
   uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
   ASSERT (offset < bm->buffer_mem_size);
 
@@ -88,8 +88,9 @@ static_always_inline void
 vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count,
 			      i32 offset)
 {
+  uword buffer_mem_start = vm->buffer_main->buffer_mem_start;
 #ifdef CLIB_HAVE_VEC256
-  u64x4 off = u64x4_splat (buffer_main.buffer_mem_start + offset);
+  u64x4 off = u64x4_splat (buffer_mem_start + offset);
   /* if count is not const, compiler will not unroll while loop
      se we maintain two-in-parallel variant */
   while (count >= 8)
@@ -111,7 +112,7 @@ vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count,
       /* shift and add to get vlib_buffer_t pointer */
       u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
 #elif defined (CLIB_HAVE_VEC128)
-      u64x2 off = u64x2_splat (buffer_main.buffer_mem_start + offset);
+      u64x2 off = u64x2_splat (buffer_mem_start + offset);
       u32x4 bi4 = u32x4_load_unaligned (bi);
       u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4);
 #if defined (__aarch64__)
@@ -165,7 +166,7 @@ vlib_get_buffers (vlib_main_t * vm, u32 * bi, vlib_buffer_t ** b, int count)
 always_inline u32
 vlib_get_buffer_index (vlib_main_t * vm, void *p)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm = vm->buffer_main;
   uword offset = pointer_to_uword (p) - bm->buffer_mem_start;
   ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start);
   ASSERT (offset < bm->buffer_mem_size);
@@ -187,7 +188,7 @@ vlib_get_buffer_indices_with_offset (vlib_main_t * vm, void **b, u32 * bi,
 {
 #ifdef CLIB_HAVE_VEC256
   u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
-  u64x4 off4 = u64x4_splat (buffer_main.buffer_mem_start - offset);
+  u64x4 off4 = u64x4_splat (vm->buffer_main->buffer_mem_start - offset);
 
   while (count >= 8)
     {
@@ -368,9 +369,9 @@ void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
 				      expected_state);
 
 always_inline vlib_buffer_known_state_t
-vlib_buffer_is_known (u32 buffer_index)
+vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm = vm->buffer_main;
 
   clib_spinlock_lock (&bm->buffer_known_hash_lockp);
   uword *p = hash_get (bm->buffer_known_hash, buffer_index);
@@ -379,10 +380,10 @@ vlib_buffer_is_known (u32 buffer_index)
 }
 
 always_inline void
-vlib_buffer_set_known_state (u32 buffer_index,
+vlib_buffer_set_known_state (vlib_main_t * vm, u32 buffer_index,
 			     vlib_buffer_known_state_t state)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm = vm->buffer_main;
 
   clib_spinlock_lock (&bm->buffer_known_hash_lockp);
   hash_set (bm->buffer_known_hash, buffer_index, state);
@@ -414,8 +415,8 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
 				  u32 n_buffers,
 				  vlib_buffer_free_list_index_t index)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
   vlib_buffer_free_list_t *fl;
+  vlib_buffer_main_t *bm = vm->buffer_main;
   u32 *src;
   uword len;
 
@@ -518,7 +519,7 @@ vlib_buffer_free (vlib_main_t * vm,
 		  /* number of buffers to free */
 		  u32 n_buffers)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm = vm->buffer_main;
 
   ASSERT (bm->cb.vlib_buffer_free_cb);
 
@@ -539,7 +540,7 @@ vlib_buffer_free_no_next (vlib_main_t * vm,
 			  /* number of buffers to free */
 			  u32 n_buffers)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm = vm->buffer_main;
 
   ASSERT (bm->cb.vlib_buffer_free_no_next_cb);
 
@@ -616,7 +617,7 @@ always_inline void
 vlib_buffer_delete_free_list (vlib_main_t * vm,
 			      vlib_buffer_free_list_index_t free_list_index)
 {
-  vlib_buffer_main_t *bm = &buffer_main;
+  vlib_buffer_main_t *bm = vm->buffer_main;
 
   ASSERT (bm->cb.vlib_buffer_delete_free_list_cb);
 
@@ -1003,12 +1004,19 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst,
   ASSERT (dst->n_add_refs == 0);
 }
 
+static_always_inline vlib_buffer_pool_t *
+vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index)
+{
+  vlib_buffer_main_t *bm = vm->buffer_main;
+  return vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
+}
+
 always_inline void
 vlib_buffer_add_to_free_list (vlib_main_t * vm,
 			      vlib_buffer_free_list_t * f,
 			      u32 buffer_index, u8 do_init)
 {
-  vlib_buffer_pool_t *bp = vlib_buffer_pool_get (f->buffer_pool_index);
+  vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, f->buffer_pool_index);
   vlib_buffer_t *b;
   b = vlib_get_buffer (vm, buffer_index);
   if (PREDICT_TRUE (do_init))
diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c
index d31b3c94d62..28a5140f3eb 100644
--- a/src/vlib/linux/pci.c
+++ b/src/vlib/linux/pci.c
@@ -967,7 +967,7 @@ add_device_vfio (vlib_main_t * vm, linux_pci_device_t * p,
     {
       vlib_buffer_pool_t *bp;
       /* *INDENT-OFF* */
-      vec_foreach (bp, buffer_main.buffer_pools)
+      vec_foreach (bp, vm->buffer_main->buffer_pools)
 	{
 	  u32 i;
 	  vlib_physmem_map_t *pm;
diff --git a/src/vlib/main.c b/src/vlib/main.c
index 650fa281746..160050486c5 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -463,7 +463,7 @@ vlib_put_next_frame (vlib_main_t * vm,
   vlib_frame_t *f;
   u32 n_vectors_in_frame;
 
-  if (buffer_main.callbacks_registered == 0 && CLIB_DEBUG > 0)
+  if (vm->buffer_main->callbacks_registered == 0 && CLIB_DEBUG > 0)
     vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
 
   nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
diff --git a/src/vlib/main.h b/src/vlib/main.h
index 8a25ce8b4cc..91661fdaaad 100644
--- a/src/vlib/main.h
+++ b/src/vlib/main.h
@@ -119,6 +119,9 @@ typedef struct vlib_main_t
   /* Pool of buffer free lists. */
   vlib_buffer_free_list_t *buffer_free_list_pool;
 
+  /* buffer main structure. */
+  vlib_buffer_main_t *buffer_main;
+
   /* physical memory main structure. */
   vlib_physmem_main_t physmem_main;