1 files changed, 85 insertions, 108 deletions

diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index 71f84d377af..674f15d5dc6 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -43,10 +43,11 @@
  * Allocate/free network buffers.
  */
 
-#include <vppinfra/linux/sysfs.h>
+#include <vppinfra/bitmap.h>
+#include <vppinfra/unix.h>
 #include <vlib/vlib.h>
 #include <vlib/unix/unix.h>
-#include <vpp/stats/stat_segment.h>
+#include <vlib/stats/stats.h>
 
 #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA 16384
 #define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV 8192
@@ -58,22 +59,8 @@ STATIC_ASSERT_FITS_IN (vlib_buffer_t, ref_count, 16);
 STATIC_ASSERT_FITS_IN (vlib_buffer_t, buffer_pool_index, 16);
 #endif
 
-/* Make sure that buffer template size is not accidentally changed */
-STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64);
-
 u16 __vlib_buffer_external_hdr_size = 0;
 
-static void
-buffer_gauges_update_cached_fn (stat_segment_directory_entry_t * e,
-				u32 index);
-
-static void
-buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e,
-				   u32 index);
-
-static void
-buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e, u32 index);
-
 uword
 vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm,
 				       vlib_buffer_t * b_first)
@@ -107,7 +94,7 @@ format_vlib_buffer_no_chain (u8 * s, va_list * args)
 		"ref-count %u", b->current_data, b->current_length,
 		b->buffer_pool_index, b->ref_count);
 
-  if (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)
+  if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
     s = format (s, ", totlen-nifb %d",
 		b->total_length_not_including_first_buffer);
 
@@ -485,26 +472,28 @@ static uword
 vlib_buffer_alloc_size (uword ext_hdr_size, uword data_size)
 {
   uword alloc_size = ext_hdr_size + sizeof (vlib_buffer_t) + data_size;
-  alloc_size = CLIB_CACHE_LINE_ROUND (alloc_size);
+  alloc_size = round_pow2 (alloc_size, VLIB_BUFFER_ALIGN);
 
-  /* in case when we have even number of cachelines, we add one more for
+  /* in case when we have even number of 'cachelines', we add one more for
    * better cache occupancy */
-  alloc_size |= CLIB_CACHE_LINE_BYTES;
+  alloc_size |= VLIB_BUFFER_ALIGN;
 
   return alloc_size;
 }
 
 u8
-vlib_buffer_pool_create (vlib_main_t * vm, char *name, u32 data_size,
-			 u32 physmem_map_index)
+vlib_buffer_pool_create (vlib_main_t *vm, u32 data_size, u32 physmem_map_index,
+			 char *fmt, ...)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
   vlib_buffer_pool_t *bp;
   vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index);
   uword start = pointer_to_uword (m->base);
   uword size = (uword) m->n_pages << m->log2_page_size;
-  uword i, j;
-  u32 alloc_size, n_alloc_per_page;
+  uword page_mask = ~pow2_mask (m->log2_page_size);
+  u8 *p;
+  u32 alloc_size;
+  va_list va;
 
   if (vec_len (bm->buffer_pools) >= 255)
     return ~0;
@@ -542,48 +531,57 @@ vlib_buffer_pool_create (vlib_main_t * vm, char *name, u32 data_size,
   bp->buffer_template.buffer_pool_index = bp->index;
   bp->buffer_template.ref_count = 1;
   bp->physmem_map_index = physmem_map_index;
-  bp->name = format (0, "%s%c", name, 0);
   bp->data_size = data_size;
   bp->numa_node = m->numa_node;
+  bp->log2_page_size = m->log2_page_size;
+
+  va_start (va, fmt);
+  bp->name = va_format (0, fmt, &va);
+  va_end (va);
 
   vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1,
 			CLIB_CACHE_LINE_BYTES);
 
   alloc_size = vlib_buffer_alloc_size (bm->ext_hdr_size, data_size);
-  n_alloc_per_page = (1ULL << m->log2_page_size) / alloc_size;
+  bp->alloc_size = alloc_size;
 
   /* preallocate buffer indices memory */
-  bp->n_buffers = m->n_pages * n_alloc_per_page;
-  bp->buffers = clib_mem_alloc_aligned (bp->n_buffers * sizeof (u32),
-					CLIB_CACHE_LINE_BYTES);
+  bp->buffers = clib_mem_alloc_aligned (
+    round_pow2 ((size / alloc_size) * sizeof (u32), CLIB_CACHE_LINE_BYTES),
+    CLIB_CACHE_LINE_BYTES);
 
   clib_spinlock_init (&bp->lock);
 
-  for (j = 0; j < m->n_pages; j++)
-    for (i = 0; i < n_alloc_per_page; i++)
-      {
-	u8 *p;
-	u32 bi;
-
-	p = m->base + (j << m->log2_page_size) + i * alloc_size;
-	p += bm->ext_hdr_size;
-
-	/*
-	 * Waste 1 buffer (maximum) so that 0 is never a valid buffer index.
-	 * Allows various places to ASSERT (bi != 0). Much easier
-	 * than debugging downstream crashes in successor nodes.
-	 */
-	if (p == m->base)
-	  continue;
+  p = m->base;
 
-	vlib_buffer_copy_template ((vlib_buffer_t *) p, &bp->buffer_template);
+  /* start with naturally aligned address */
+  p += alloc_size - (uword) p % alloc_size;
 
-	bi = vlib_get_buffer_index (vm, (vlib_buffer_t *) p);
+  /*
+   * Waste 1 buffer (maximum) so that 0 is never a valid buffer index.
+   * Allows various places to ASSERT (bi != 0). Much easier
+   * than debugging downstream crashes in successor nodes.
+   */
+  if (p == m->base)
+    p += alloc_size;
 
-	bp->buffers[bp->n_avail++] = bi;
+  for (; p < (u8 *) m->base + size - alloc_size; p += alloc_size)
+    {
+      vlib_buffer_t *b;
+      u32 bi;
+
+      /* skip if buffer spans across page boundary */
+      if (((uword) p & page_mask) != ((uword) (p + alloc_size) & page_mask))
+	continue;
+
+      b = (vlib_buffer_t *) (p + bm->ext_hdr_size);
+      b->template = bp->buffer_template;
+      bi = vlib_get_buffer_index (vm, b);
+      bp->buffers[bp->n_avail++] = bi;
+      vlib_get_buffer (vm, bi);
+    }
 
-	vlib_get_buffer (vm, bi);
-      }
+  bp->n_buffers = bp->n_avail;
 
   return bp->index;
 }
@@ -601,14 +599,13 @@ format_vlib_buffer_pool (u8 * s, va_list * va)
 		   "Pool Name", "Index", "NUMA", "Size", "Data Size",
 		   "Total", "Avail", "Cached", "Used");
 
-  /* *INDENT-OFF* */
   vec_foreach (bpt, bp->threads)
     cached += bpt->n_cached;
-  /* *INDENT-ON* */
 
-  s = format (s, "%-20s%=6d%=6d%=6u%=11u%=6u%=8u%=8u%=8u",
-	      bp->name, bp->index, bp->numa_node, bp->data_size +
-	      sizeof (vlib_buffer_t) + vm->buffer_main->ext_hdr_size,
+  s = format (s, "%-20v%=6d%=6d%=6u%=11u%=6u%=8u%=8u%=8u", bp->name, bp->index,
+	      bp->numa_node,
+	      bp->data_size + sizeof (vlib_buffer_t) +
+		vm->buffer_main->ext_hdr_size,
 	      bp->data_size, bp->n_buffers, bp->n_avail, cached,
 	      bp->n_buffers - bp->n_avail - cached);
 
@@ -638,34 +635,26 @@ show_buffers (vlib_main_t *vm, unformat_input_t *input,
   return 0;
 }
 
-/* *INDENT-OFF* */
 VLIB_CLI_COMMAND (show_buffers_command, static) = {
   .path = "show buffers",
   .short_help = "Show packet buffer allocation",
   .function = show_buffers,
 };
-/* *INDENT-ON* */
 
 clib_error_t *
-vlib_buffer_worker_init (vlib_main_t * vm)
+vlib_buffer_num_workers_change (vlib_main_t *vm)
 {
   vlib_buffer_main_t *bm = vm->buffer_main;
   vlib_buffer_pool_t *bp;
 
-  /* *INDENT-OFF* */
   vec_foreach (bp, bm->buffer_pools)
-    {
-      clib_spinlock_lock (&bp->lock);
-      vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1,
-			    CLIB_CACHE_LINE_BYTES);
-      clib_spinlock_unlock (&bp->lock);
-    }
-  /* *INDENT-ON* */
+    vec_validate_aligned (bp->threads, vlib_get_n_threads () - 1,
+			  CLIB_CACHE_LINE_BYTES);
 
   return 0;
 }
 
-VLIB_WORKER_INIT_FUNCTION (vlib_buffer_worker_init);
+VLIB_NUM_WORKERS_CHANGE_FN (vlib_buffer_num_workers_change);
 
 static clib_error_t *
 vlib_buffer_main_init_numa_alloc (struct vlib_main_t *vm, u32 numa_node,
@@ -711,7 +700,6 @@ vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node,
   vlib_buffer_main_t *bm = vm->buffer_main;
   u32 physmem_map_index;
   clib_error_t *error;
-  u8 *name = 0;
 
   if (bm->log2_page_size == CLIB_MEM_PAGE_SZ_UNKNOWN)
     {
@@ -742,14 +730,12 @@ vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node,
     return error;
 
 buffer_pool_create:
-  name = format (name, "default-numa-%d%c", numa_node, 0);
-  *index = vlib_buffer_pool_create (vm, (char *) name,
-				    vlib_buffer_get_default_data_size (vm),
-				    physmem_map_index);
+  *index =
+    vlib_buffer_pool_create (vm, vlib_buffer_get_default_data_size (vm),
+			     physmem_map_index, "default-numa-%d", numa_node);
 
   if (*index == (u8) ~ 0)
     error = clib_error_return (0, "maximum number of buffer pools reached");
-  vec_free (name);
 
 
   return error;
@@ -776,10 +762,8 @@ buffer_get_cached (vlib_buffer_pool_t * bp)
 
   clib_spinlock_lock (&bp->lock);
 
-  /* *INDENT-OFF* */
   vec_foreach (bpt, bp->threads)
     cached += bpt->n_cached;
-  /* *INDENT-ON* */
 
   clib_spinlock_unlock (&bp->lock);
 
@@ -798,37 +782,39 @@ buffer_get_by_index (vlib_buffer_main_t * bm, u32 index)
 }
 
 static void
-buffer_gauges_update_used_fn (stat_segment_directory_entry_t * e, u32 index)
+buffer_gauges_collect_used_fn (vlib_stats_collector_data_t *d)
 {
   vlib_main_t *vm = vlib_get_main ();
-  vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index);
+  vlib_buffer_pool_t *bp =
+    buffer_get_by_index (vm->buffer_main, d->private_data);
   if (!bp)
     return;
 
-  e->value = bp->n_buffers - bp->n_avail - buffer_get_cached (bp);
+  d->entry->value = bp->n_buffers - bp->n_avail - buffer_get_cached (bp);
 }
 
 static void
-buffer_gauges_update_available_fn (stat_segment_directory_entry_t * e,
-				   u32 index)
+buffer_gauges_collect_available_fn (vlib_stats_collector_data_t *d)
 {
   vlib_main_t *vm = vlib_get_main ();
-  vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index);
+  vlib_buffer_pool_t *bp =
+    buffer_get_by_index (vm->buffer_main, d->private_data);
   if (!bp)
     return;
 
-  e->value = bp->n_avail;
+  d->entry->value = bp->n_avail;
 }
 
 static void
-buffer_gauges_update_cached_fn (stat_segment_directory_entry_t * e, u32 index)
+buffer_gauges_collect_cached_fn (vlib_stats_collector_data_t *d)
 {
   vlib_main_t *vm = vlib_get_main ();
-  vlib_buffer_pool_t *bp = buffer_get_by_index (vm->buffer_main, index);
+  vlib_buffer_pool_t *bp =
+    buffer_get_by_index (vm->buffer_main, d->private_data);
   if (!bp)
     return;
 
-  e->value = buffer_get_cached (bp);
+  d->entry->value = buffer_get_cached (bp);
 }
 
 clib_error_t *
@@ -849,13 +835,8 @@ vlib_buffer_main_init (struct vlib_main_t * vm)
 
   clib_spinlock_init (&bm->buffer_known_hash_lockp);
 
-  if ((err = clib_sysfs_read ("/sys/devices/system/node/online", "%U",
-			      unformat_bitmap_list, &bmp)))
-    clib_error_free (err);
-
-  if ((err = clib_sysfs_read ("/sys/devices/system/node/has_memory", "%U",
-			      unformat_bitmap_list, &bmp_has_memory)))
-    clib_error_free (err);
+  bmp = os_get_online_cpu_node_bitmap ();
+  bmp_has_memory = os_get_cpu_with_memory_bitmap ();
 
   if (bmp && bmp_has_memory)
     bmp = clib_bitmap_and (bmp, bmp_has_memory);
@@ -868,7 +849,6 @@ vlib_buffer_main_init (struct vlib_main_t * vm)
     clib_panic ("system have more than %u NUMA nodes",
 		VLIB_BUFFER_MAX_NUMA_NODES);
 
-  /* *INDENT-OFF* */
   clib_bitmap_foreach (numa_node, bmp)
     {
       u8 *index = bm->default_buffer_pool_index_for_numa + numa_node;
@@ -883,7 +863,6 @@ vlib_buffer_main_init (struct vlib_main_t * vm)
       if (first_valid_buffer_pool_index == 0xff)
         first_valid_buffer_pool_index = index[0];
     }
-  /* *INDENT-ON* */
 
   if (first_valid_buffer_pool_index == (u8) ~ 0)
     {
@@ -891,34 +870,32 @@ vlib_buffer_main_init (struct vlib_main_t * vm)
       goto done;
     }
 
-  /* *INDENT-OFF* */
   clib_bitmap_foreach (numa_node, bmp)
     {
       if (bm->default_buffer_pool_index_for_numa[numa_node]  == (u8) ~0)
 	bm->default_buffer_pool_index_for_numa[numa_node] =
 	  first_valid_buffer_pool_index;
     }
-  /* *INDENT-ON* */
 
   vec_foreach (bp, bm->buffer_pools)
   {
+    vlib_stats_collector_reg_t reg = { .private_data = bp - bm->buffer_pools };
     if (bp->n_buffers == 0)
       continue;
 
-    vec_reset_length (name);
-    name = format (name, "/buffer-pools/%s/cached%c", bp->name, 0);
-    stat_segment_register_gauge (name, buffer_gauges_update_cached_fn,
-				 bp - bm->buffer_pools);
+    reg.entry_index =
+      vlib_stats_add_gauge ("/buffer-pools/%v/cached", bp->name);
+    reg.collect_fn = buffer_gauges_collect_cached_fn;
+    vlib_stats_register_collector_fn (&reg);
 
-    vec_reset_length (name);
-    name = format (name, "/buffer-pools/%s/used%c", bp->name, 0);
-    stat_segment_register_gauge (name, buffer_gauges_update_used_fn,
-				 bp - bm->buffer_pools);
+    reg.entry_index = vlib_stats_add_gauge ("/buffer-pools/%v/used", bp->name);
+    reg.collect_fn = buffer_gauges_collect_used_fn;
+    vlib_stats_register_collector_fn (&reg);
 
-    vec_reset_length (name);
-    name = format (name, "/buffer-pools/%s/available%c", bp->name, 0);
-    stat_segment_register_gauge (name, buffer_gauges_update_available_fn,
-				 bp - bm->buffer_pools);
+    reg.entry_index =
+      vlib_stats_add_gauge ("/buffer-pools/%v/available", bp->name);
+    reg.collect_fn = buffer_gauges_collect_available_fn;
+    vlib_stats_register_collector_fn (&reg);
   }
 
 done: