STATS: Dynamically mapped shared memory segment

Move from using a hash to a vector with offsets into shared memory.
Limit exposure of VPP data structures and include files to
external stats library and applications.

Change-Id: Ic06129f12d10cf4c4946a86d9bc734eacff2c7da
Signed-off-by: Ole Troan <ot@cisco.com>

4 files changed, 388 insertions, 353 deletions

diff --git a/src/vpp/stats/stat_segment.c b/src/vpp/stats/stat_segment.c
index dcaeb5078b3..7bf6624e075 100644
--- a/src/vpp/stats/stat_segment.c
+++ b/src/vpp/stats/stat_segment.c
@@ -12,47 +12,64 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+#include <vppinfra/mem.h>
 #include <vpp/stats/stats.h>
+#undef HAVE_MEMFD_CREATE
+#include <vppinfra/linux/syscall.h>
 
+/*
+ *  Used only by VPP writers
+ */
 void
 vlib_stat_segment_lock (void)
 {
   stats_main_t *sm = &stats_main;
-  vlib_main_t *vm = vlib_get_main ();
-  f64 deadman;
-
-  /* 3ms is WAY long enough to be reasonably sure something is wrong */
-  deadman = vlib_time_now (vm) + 3e-3;
-
-  while (__sync_lock_test_and_set (&((*sm->stat_segment_lockp)->lock), 1))
-    {
-      if (vlib_time_now (vm) >= deadman)
-	{
-	  clib_warning ("BUG: stat segment lock held too long...");
-	  break;
-	}
-    }
+  clib_spinlock_lock (sm->stat_segment_lockp);
+  sm->shared_header->in_progress = 1;
 }
 
 void
 vlib_stat_segment_unlock (void)
 {
   stats_main_t *sm = &stats_main;
+  sm->shared_header->epoch++;
+  sm->shared_header->in_progress = 0;
   clib_spinlock_unlock (sm->stat_segment_lockp);
 }
 
+/*
+ * Change heap to the stats shared memory segment
+ */
 void *
 vlib_stats_push_heap (void)
 {
   stats_main_t *sm = &stats_main;
-  ssvm_private_t *ssvmp = &sm->stat_segment;
-  ssvm_shared_header_t *shared_header;
 
-  ASSERT (ssvmp && ssvmp->sh);
+  ASSERT (sm && sm->shared_header);
+  return clib_mem_set_heap (sm->heap);
+}
 
-  shared_header = ssvmp->sh;
+/* Name to vector index hash */
+static u32
+lookup_or_create_hash_index (void *oldheap, char *name, u32 next_vector_index)
+{
+  stats_main_t *sm = &stats_main;
+  u32 index;
+  hash_pair_t *hp;
 
-  return ssvm_push_heap (shared_header);
+  hp = hash_get_pair (sm->directory_vector_by_name, name);
+  if (!hp)
+    {
+      hash_set (sm->directory_vector_by_name, name, next_vector_index);
+      index = next_vector_index;
+    }
+  else
+    {
+      index = hp->value[0];
+    }
+
+  return index;
 }
 
 void
@@ -60,250 +77,216 @@ vlib_stats_pop_heap (void *cm_arg, void *oldheap, stat_directory_type_t type)
 {
   vlib_simple_counter_main_t *cm = (vlib_simple_counter_main_t *) cm_arg;
   stats_main_t *sm = &stats_main;
-  ssvm_private_t *ssvmp = &sm->stat_segment;
-  ssvm_shared_header_t *shared_header;
+  stat_segment_shared_header_t *shared_header = sm->shared_header;
   char *stat_segment_name;
-  stat_segment_directory_entry_t *ep;
-  uword *p;
-
-  ASSERT (ssvmp && ssvmp->sh);
-
-  shared_header = ssvmp->sh;
+  stat_segment_directory_entry_t e = { 0 };
 
   /* Not all counters have names / hash-table entries */
-  if (cm->name || cm->stat_segment_name)
+  if (!cm->name && !cm->stat_segment_name)
     {
-      hash_pair_t *hp;
-      u8 *name_copy;
+      clib_mem_set_heap (oldheap);
+      return;
+    }
 
-      stat_segment_name = cm->stat_segment_name ?
-	cm->stat_segment_name : cm->name;
+  ASSERT (shared_header);
 
-      vlib_stat_segment_lock ();
+  vlib_stat_segment_lock ();
 
-      /* Update hash table. The name must be copied into the segment */
-      hp = hash_get_pair (sm->counter_vector_by_name, stat_segment_name);
-      if (hp)
-	{
-	  name_copy = (u8 *) hp->key;
-	  ep = (stat_segment_directory_entry_t *) (hp->value[0]);
-	  hash_unset_mem (sm->counter_vector_by_name, stat_segment_name);
-	  vec_free (name_copy);
-	  clib_mem_free (ep);
-	}
-      name_copy = format (0, "%s%c", stat_segment_name, 0);
-      ep = clib_mem_alloc (sizeof (*ep));
-      ep->type = type;
-      ep->value = cm->counters;
-      hash_set_mem (sm->counter_vector_by_name, name_copy, ep);
-
-      /* Reset the client hash table pointer, since it WILL change! */
-      shared_header->opaque[STAT_SEGMENT_OPAQUE_DIR]
-	= sm->counter_vector_by_name;
-
-      /* Warn clients to refresh any pointers they might be holding */
-      shared_header->opaque[STAT_SEGMENT_OPAQUE_EPOCH] = (void *)
-	((u64) shared_header->opaque[STAT_SEGMENT_OPAQUE_EPOCH] + 1);
-      vlib_stat_segment_unlock ();
+  /* Lookup hash-table is on the main heap */
+  stat_segment_name =
+    cm->stat_segment_name ? cm->stat_segment_name : cm->name;
+  u32 next_vector_index = vec_len (sm->directory_vector);
+  clib_mem_set_heap (oldheap);	/* Exit stats segment */
+  u32 vector_index = lookup_or_create_hash_index (oldheap, stat_segment_name,
+						  next_vector_index);
+  /* Back to stats segment */
+  clib_mem_set_heap (sm->heap);	/* Re-enter stat segment */
+
+
+  /* Update the vector */
+  if (vector_index == next_vector_index)
+    {				/* New */
+      strncpy (e.name, stat_segment_name, 128 - 1);
+      e.type = type;
+      vec_add1 (sm->directory_vector, e);
+      vector_index++;
     }
-  ssvm_pop_heap (oldheap);
+
+  stat_segment_directory_entry_t *ep = &sm->directory_vector[vector_index];
+  ep->offset = stat_segment_offset (shared_header, cm->counters);	/* Vector of threads of vectors of counters */
+  u64 *offset_vector =
+    ep->offset_vector ? stat_segment_pointer (shared_header,
+					      ep->offset_vector) : 0;
+
+  /* Update the 2nd dimension offset vector */
+  int i;
+  vec_validate (offset_vector, vec_len (cm->counters) - 1);
+  for (i = 0; i < vec_len (cm->counters); i++)
+    offset_vector[i] = stat_segment_offset (shared_header, cm->counters[i]);
+  ep->offset_vector = stat_segment_offset (shared_header, offset_vector);
+  sm->directory_vector[vector_index].offset =
+    stat_segment_offset (shared_header, cm->counters);
+
+  /* Reset the client hash table pointer, since it WILL change! */
+  shared_header->directory_offset =
+    stat_segment_offset (shared_header, sm->directory_vector);
+
+  vlib_stat_segment_unlock ();
+  clib_mem_set_heap (oldheap);
 }
 
 void
-vlib_stats_register_error_index (u8 * name, u64 index)
+vlib_stats_register_error_index (u8 * name, u64 * em_vec, u64 index)
 {
   stats_main_t *sm = &stats_main;
-  ssvm_private_t *ssvmp = &sm->stat_segment;
-  ssvm_shared_header_t *shared_header;
-  stat_segment_directory_entry_t *ep;
+  stat_segment_shared_header_t *shared_header = sm->shared_header;
+  stat_segment_directory_entry_t e;
   hash_pair_t *hp;
-  u8 *name_copy;
-  uword *p;
-
-  ASSERT (ssvmp && ssvmp->sh);
 
-  shared_header = ssvmp->sh;
+  ASSERT (shared_header);
 
   vlib_stat_segment_lock ();
-  /* Update hash table. The name must be copied into the segment */
-  hp = hash_get_pair (sm->counter_vector_by_name, name);
-  if (hp)
-    {
-      name_copy = (u8 *) hp->key;
-      ep = (stat_segment_directory_entry_t *) (hp->value[0]);
-      hash_unset_mem (sm->counter_vector_by_name, name);
-      vec_free (name_copy);
-      clib_mem_free (ep);
-    }
 
-  ep = clib_mem_alloc (sizeof (*ep));
-  ep->type = STAT_DIR_TYPE_ERROR_INDEX;
-  ep->value = (void *) index;
-
-  hash_set_mem (sm->counter_vector_by_name, name, ep);
-
-  /* Reset the client hash table pointer, since it WILL change! */
-  shared_header->opaque[STAT_SEGMENT_OPAQUE_DIR] = sm->counter_vector_by_name;
+  memcpy (e.name, name, vec_len (name));
+  e.name[vec_len (name)] = '\0';
+  e.type = STAT_DIR_TYPE_ERROR_INDEX;
+  e.offset = index;
+  vec_add1 (sm->directory_vector, e);
 
   /* Warn clients to refresh any pointers they might be holding */
-  shared_header->opaque[STAT_SEGMENT_OPAQUE_EPOCH] = (void *)
-    ((u64) shared_header->opaque[STAT_SEGMENT_OPAQUE_EPOCH] + 1);
+  shared_header->directory_offset =
+    stat_segment_offset (shared_header, sm->directory_vector);
+
   vlib_stat_segment_unlock ();
 }
 
-void
-vlib_stats_pop_heap2 (u64 * counter_vector, u32 thread_index, void *oldheap)
+static void
+stat_validate_counter_vector (stat_segment_directory_entry_t * ep, u32 max)
 {
   stats_main_t *sm = &stats_main;
-  ssvm_private_t *ssvmp = &sm->stat_segment;
-  ssvm_shared_header_t *shared_header;
-  stat_segment_directory_entry_t *ep;
-  hash_pair_t *hp;
-  u8 *error_vector_name;
-  u8 *name_copy;
-  uword *p;
-
-  ASSERT (ssvmp && ssvmp->sh);
-
-  shared_header = ssvmp->sh;
-
-  vlib_stat_segment_lock ();
-  error_vector_name = format (0, "/err/%d/counter_vector%c", thread_index, 0);
+  stat_segment_shared_header_t *shared_header = sm->shared_header;
+  counter_t **counters = 0;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  int i;
+  u64 *offset_vector = 0;
 
-  /* Update hash table. The name must be copied into the segment */
-  hp = hash_get_pair (sm->counter_vector_by_name, error_vector_name);
-  if (hp)
+  vec_validate_aligned (counters, tm->n_vlib_mains - 1,
+			CLIB_CACHE_LINE_BYTES);
+  for (i = 0; i < tm->n_vlib_mains; i++)
     {
-      name_copy = (u8 *) hp->key;
-      ep = (stat_segment_directory_entry_t *) (hp->value[0]);
-      hash_unset_mem (sm->counter_vector_by_name, error_vector_name);
-      vec_free (name_copy);
-      clib_mem_free (ep);
+      vec_validate_aligned (counters[i], max, CLIB_CACHE_LINE_BYTES);
+      vec_add1 (offset_vector,
+		stat_segment_offset (shared_header, counters[i]));
     }
+  ep->offset = stat_segment_offset (shared_header, counters);
+  ep->offset_vector = stat_segment_offset (shared_header, offset_vector);
+}
 
-  ep = clib_mem_alloc (sizeof (*ep));
-  ep->type = STAT_DIR_TYPE_VECTOR_POINTER;
-  ep->value = counter_vector;
+void
+vlib_stats_pop_heap2 (u64 * error_vector, u32 thread_index, void *oldheap)
+{
+  stats_main_t *sm = &stats_main;
+  stat_segment_shared_header_t *shared_header = sm->shared_header;
 
-  hash_set_mem (sm->counter_vector_by_name, error_vector_name, ep);
+  ASSERT (shared_header);
+
+  vlib_stat_segment_lock ();
 
   /* Reset the client hash table pointer, since it WILL change! */
-  shared_header->opaque[STAT_SEGMENT_OPAQUE_DIR] = sm->counter_vector_by_name;
+  shared_header->error_offset =
+    stat_segment_offset (shared_header, error_vector);
+  shared_header->directory_offset =
+    stat_segment_offset (shared_header, sm->directory_vector);
 
-  /* Warn clients to refresh any pointers they might be holding */
-  shared_header->opaque[STAT_SEGMENT_OPAQUE_EPOCH] = (void *)
-    ((u64) shared_header->opaque[STAT_SEGMENT_OPAQUE_EPOCH] + 1);
   vlib_stat_segment_unlock ();
-  ssvm_pop_heap (oldheap);
+  clib_mem_set_heap (oldheap);
 }
 
 clib_error_t *
 vlib_map_stat_segment_init (void)
 {
   stats_main_t *sm = &stats_main;
-  ssvm_private_t *ssvmp = &sm->stat_segment;
-  ssvm_shared_header_t *shared_header;
+  stat_segment_shared_header_t *shared_header;
   stat_segment_directory_entry_t *ep;
+
   f64 *scalar_data;
   u8 *name;
   void *oldheap;
   u32 *lock;
   int rv;
-  u64 memory_size;
+  ssize_t memory_size;
+
+
+  int mfd;
+  char *mem_name = "stat_segment_test";
+  void *memaddr;
 
   memory_size = sm->memory_size;
   if (memory_size == 0)
     memory_size = STAT_SEGMENT_DEFAULT_SIZE;
 
-  ssvmp->ssvm_size = memory_size;
-  ssvmp->i_am_master = 1;
-  ssvmp->my_pid = getpid ();
-  ssvmp->name = format (0, "/stats%c", 0);
-  ssvmp->requested_va = 0;
-
-  rv = ssvm_master_init (ssvmp, SSVM_SEGMENT_MEMFD);
-
-  if (rv)
-    return clib_error_return (0, "stat segment ssvm init failure");
-  shared_header = ssvmp->sh;
-
-  oldheap = ssvm_push_heap (shared_header);
-
-  /* Set up the name to counter-vector hash table */
-  sm->counter_vector_by_name = hash_create_string (0, sizeof (uword));
-
+  /* Create shared memory segment */
+  if ((mfd = memfd_create (mem_name, 0)) < 0)
+    return clib_error_return (0, "stat segment memfd_create failure");
+
+  /* Set size */
+  if ((ftruncate (mfd, memory_size)) == -1)
+    return clib_error_return (0, "stat segment ftruncate failure");
+
+  if ((memaddr =
+       mmap (NULL, memory_size, PROT_READ | PROT_WRITE, MAP_SHARED, mfd,
+	     0)) == MAP_FAILED)
+    return clib_error_return (0, "stat segment mmap failure");
+
+  void *heap;
+#if USE_DLMALLOC == 0
+  heap = mheap_alloc_with_flags (((u8 *) memaddr) + getpagesize (),
+				 memory_size - getpagesize (),
+				 MHEAP_FLAG_DISABLE_VM |
+				 MHEAP_FLAG_THREAD_SAFE);
+#else
+  heap =
+    create_mspace_with_base (((u8 *) memaddr) + getpagesize (),
+			     memory_size - getpagesize (), 1 /* locked */ );
+  mspace_disable_expand (heap);
+#endif
+
+  sm->heap = heap;
+  sm->memfd = mfd;
+
+  sm->directory_vector_by_name = hash_create_string (0, sizeof (uword));
+  sm->shared_header = shared_header = memaddr;
   sm->stat_segment_lockp = clib_mem_alloc (sizeof (clib_spinlock_t));
-
-  /* Save the hash table address in the shared segment, for clients */
   clib_spinlock_init (sm->stat_segment_lockp);
-  shared_header->opaque[STAT_SEGMENT_OPAQUE_LOCK] = sm->stat_segment_lockp;
-  shared_header->opaque[STAT_SEGMENT_OPAQUE_EPOCH] = (void *) 1;
-
-  /* Set up a few scalar stats */
-
-  scalar_data = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
-					CLIB_CACHE_LINE_BYTES);
-  sm->vector_rate_ptr = (scalar_data + 0);
-  sm->input_rate_ptr = (scalar_data + 1);
-  sm->last_runtime_ptr = (scalar_data + 2);
-  sm->last_runtime_stats_clear_ptr = (scalar_data + 3);
-  sm->heartbeat_ptr = (scalar_data + 4);
-
-  name = format (0, "/sys/vector_rate%c", 0);
-  ep = clib_mem_alloc (sizeof (*ep));
-  ep->type = STAT_DIR_TYPE_SCALAR_POINTER;
-  ep->value = sm->vector_rate_ptr;
-
-  hash_set_mem (sm->counter_vector_by_name, name, ep);
-
-  name = format (0, "/sys/input_rate%c", 0);
-  ep = clib_mem_alloc (sizeof (*ep));
-  ep->type = STAT_DIR_TYPE_SCALAR_POINTER;
-  ep->value = sm->input_rate_ptr;
-
-  hash_set_mem (sm->counter_vector_by_name, name, ep);
 
-  name = format (0, "/sys/last_update%c", 0);
-  ep = clib_mem_alloc (sizeof (*ep));
-  ep->type = STAT_DIR_TYPE_SCALAR_POINTER;
-  ep->value = sm->last_runtime_ptr;
+  oldheap = clib_mem_set_heap (sm->heap);
 
-  hash_set_mem (sm->counter_vector_by_name, name, ep);
-
-  name = format (0, "/sys/last_stats_clear%c", 0);
-  ep = clib_mem_alloc (sizeof (*ep));
-  ep->type = STAT_DIR_TYPE_SCALAR_POINTER;
-  ep->value = sm->last_runtime_stats_clear_ptr;
-
-  hash_set_mem (sm->counter_vector_by_name, name, ep);
-
-  name = format (0, "/sys/heartbeat%c", 0);
-  ep = clib_mem_alloc (sizeof (*ep));
-  ep->type = STAT_DIR_TYPE_SCALAR_POINTER;
-  ep->value = sm->heartbeat_ptr;
-
-  hash_set_mem (sm->counter_vector_by_name, name, ep);
+  /* Set up the name to counter-vector hash table */
+  sm->directory_vector = 0;
 
+  shared_header->epoch = 1;
 
-  /* Publish the hash table */
-  shared_header->opaque[STAT_SEGMENT_OPAQUE_DIR] = sm->counter_vector_by_name;
+  /* Scalar stats and node counters */
+  vec_validate (sm->directory_vector, STAT_COUNTERS - 1);
+#define _(E,t,n,p)							\
+  strcpy(sm->directory_vector[STAT_COUNTER_##E].name,  "/sys" #p "/" #n); \
+  sm->directory_vector[STAT_COUNTER_##E].type = STAT_DIR_TYPE_##t;
+  foreach_stat_segment_counter_name
+#undef _
+    /* Save the vector offset in the shared segment, for clients */
+    shared_header->directory_offset =
+    stat_segment_offset (shared_header, sm->directory_vector);
 
-  ssvm_pop_heap (oldheap);
+  clib_mem_set_heap (oldheap);
 
   return 0;
 }
 
-typedef struct
-{
-  u8 *name;
-  stat_segment_directory_entry_t *dir_entry;
-} show_stat_segment_t;
-
 static int
 name_sort_cmp (void *a1, void *a2)
 {
-  show_stat_segment_t *n1 = a1;
-  show_stat_segment_t *n2 = a2;
+  stat_segment_directory_entry_t *n1 = a1;
+  stat_segment_directory_entry_t *n2 = a2;
 
   return strcmp ((char *) n1->name, (char *) n2->name);
 }
@@ -316,30 +299,21 @@ format_stat_dir_entry (u8 * s, va_list * args)
   char *type_name;
   char *format_string;
 
-  format_string = "%-10s %20llx";
+  format_string = "%-74s %-10s %10lld";
 
   switch (ep->type)
     {
-    case STAT_DIR_TYPE_SCALAR_POINTER:
+    case STAT_DIR_TYPE_SCALAR_INDEX:
       type_name = "ScalarPtr";
       break;
 
-    case STAT_DIR_TYPE_VECTOR_POINTER:
-      type_name = "VectorPtr";
-      break;
-
     case STAT_DIR_TYPE_COUNTER_VECTOR_SIMPLE:
     case STAT_DIR_TYPE_COUNTER_VECTOR_COMBINED:
       type_name = "CMainPtr";
       break;
 
-    case STAT_DIR_TYPE_SERIALIZED_NODES:
-      type_name = "SerNodesPtr";
-      break;
-
     case STAT_DIR_TYPE_ERROR_INDEX:
       type_name = "ErrIndex";
-      format_string = "%-10s %20lld";
       break;
 
     default:
@@ -347,7 +321,7 @@ format_stat_dir_entry (u8 * s, va_list * args)
       break;
     }
 
-  return format (s, format_string, type_name, ep->value);
+  return format (s, format_string, ep->name, type_name, ep->offset);
 }
 
 static clib_error_t *
@@ -356,13 +330,10 @@ show_stat_segment_command_fn (vlib_main_t * vm,
 			      vlib_cli_command_t * cmd)
 {
   stats_main_t *sm = &stats_main;
-  ssvm_private_t *ssvmp = &sm->stat_segment;
-  ssvm_shared_header_t *shared_header;
   counter_t *counter;
   hash_pair_t *p;
-  show_stat_segment_t *show_data = 0;
-  show_stat_segment_t *this;
-  int i;
+  stat_segment_directory_entry_t *show_data, *this;
+  int i, j;
 
   int verbose = 0;
   u8 *s;
@@ -370,40 +341,25 @@ show_stat_segment_command_fn (vlib_main_t * vm,
   if (unformat (input, "verbose"))
     verbose = 1;
 
+  /* Lock even as reader, as this command doesn't handle epoch changes */
   vlib_stat_segment_lock ();
-
-  /* *INDENT-OFF* */
-  hash_foreach_pair (p, sm->counter_vector_by_name,
-  ({
-    vec_add2 (show_data, this, 1);
-
-    this->name = (u8 *) (p->key);
-    this->dir_entry = (stat_segment_directory_entry_t *)(p->value[0]);
-  }));
-  /* *INDENT-ON* */
-
+  show_data = vec_dup (sm->directory_vector);
   vlib_stat_segment_unlock ();
 
   vec_sort_with_function (show_data, name_sort_cmp);
 
-  vlib_cli_output (vm, "%-60s %10s %20s", "Name", "Type", "Value");
+  vlib_cli_output (vm, "%-74s %10s %10s", "Name", "Type", "Value");
 
   for (i = 0; i < vec_len (show_data); i++)
     {
-      this = vec_elt_at_index (show_data, i);
-
-      vlib_cli_output (vm, "%-60s %31U",
-		       this->name, format_stat_dir_entry, this->dir_entry);
+      vlib_cli_output (vm, "%-100U", format_stat_dir_entry,
+		       vec_elt_at_index (show_data, i));
     }
 
   if (verbose)
     {
-      ASSERT (ssvmp && ssvmp->sh);
-
-      shared_header = ssvmp->sh;
-
-      vlib_cli_output (vm, "%U", format_mheap,
-		       shared_header->heap, 0 /* verbose */ );
+      ASSERT (sm->heap);
+      vlib_cli_output (vm, "%U", format_mheap, sm->heap, 0 /* verbose */ );
     }
 
   return 0;
@@ -418,71 +374,95 @@ VLIB_CLI_COMMAND (show_stat_segment_command, static) =
 };
 /* *INDENT-ON* */
 
+/*
+ * Node performance counters:
+ * total_calls [threads][node-index]
+ * total_vectors
+ * total_calls
+ * total suspends
+ */
+
 static inline void
-update_serialized_nodes (stats_main_t * sm)
+update_node_counters (stats_main_t * sm)
 {
-  int i;
   vlib_main_t *vm = vlib_mains[0];
-  ssvm_private_t *ssvmp = &sm->stat_segment;
-  ssvm_shared_header_t *shared_header;
-  void *oldheap;
-  stat_segment_directory_entry_t *ep;
-  hash_pair_t *hp;
-  u8 *name_copy;
-
-  ASSERT (ssvmp && ssvmp->sh);
-
-  vec_reset_length (sm->serialized_nodes);
-
-  shared_header = ssvmp->sh;
-
-  oldheap = ssvm_push_heap (shared_header);
+  vlib_main_t **stat_vms = 0;
+  vlib_node_t ***node_dups = 0;
+  int i, j;
+  stat_segment_shared_header_t *shared_header = sm->shared_header;
+  static u32 no_max_nodes = 0;
 
-  vlib_stat_segment_lock ();
   vlib_node_get_nodes (0 /* vm, for barrier sync */ ,
 		       (u32) ~ 0 /* all threads */ ,
 		       1 /* include stats */ ,
 		       0 /* barrier sync */ ,
-		       &sm->node_dups, &sm->stat_vms);
+		       &node_dups, &stat_vms);
 
-  sm->serialized_nodes = vlib_node_serialize (vm, sm->node_dups,
-					      sm->serialized_nodes,
-					      0 /* include nexts */ ,
-					      1 /* include stats */ );
+  u32 l = vec_len (node_dups[0]);
 
-  hp = hash_get_pair (sm->counter_vector_by_name, "serialized_nodes");
-  if (hp)
+  /*
+   * Extend performance nodes if necessary
+   */
+  if (l > no_max_nodes)
     {
-      name_copy = (u8 *) hp->key;
-      ep = (stat_segment_directory_entry_t *) (hp->value[0]);
+      void *oldheap = clib_mem_set_heap (sm->heap);
+      vlib_stat_segment_lock ();
 
-      if (ep->value != sm->serialized_nodes)
-	{
-	  ep->value = sm->serialized_nodes;
-	  /* Warn clients to refresh any pointers they might be holding */
-	  shared_header->opaque[STAT_SEGMENT_OPAQUE_EPOCH] = (void *)
-	    ((u64) shared_header->opaque[STAT_SEGMENT_OPAQUE_EPOCH] + 1);
-	}
+      stat_validate_counter_vector (&sm->directory_vector
+				    [STAT_COUNTER_NODE_CLOCKS], l);
+      stat_validate_counter_vector (&sm->directory_vector
+				    [STAT_COUNTER_NODE_VECTORS], l);
+      stat_validate_counter_vector (&sm->directory_vector
+				    [STAT_COUNTER_NODE_CALLS], l);
+      stat_validate_counter_vector (&sm->directory_vector
+				    [STAT_COUNTER_NODE_SUSPENDS], l);
+
+      vlib_stat_segment_unlock ();
+      clib_mem_set_heap (oldheap);
+      no_max_nodes = l;
     }
-  else
+
+  for (j = 0; j < vec_len (node_dups); j++)
     {
-      name_copy = format (0, "%s%c", "serialized_nodes", 0);
-      ep = clib_mem_alloc (sizeof (*ep));
-      ep->type = STAT_DIR_TYPE_SERIALIZED_NODES;
-      ep->value = sm->serialized_nodes;
-      hash_set_mem (sm->counter_vector_by_name, name_copy, ep);
-
-      /* Reset the client hash table pointer */
-      shared_header->opaque[STAT_SEGMENT_OPAQUE_DIR]
-	= sm->counter_vector_by_name;
-
-      /* Warn clients to refresh any pointers they might be holding */
-      shared_header->opaque[STAT_SEGMENT_OPAQUE_EPOCH] = (void *)
-	((u64) shared_header->opaque[STAT_SEGMENT_OPAQUE_EPOCH] + 1);
-    }
+      vlib_node_t **nodes = node_dups[j];
+      u32 l = vec_len (nodes);
 
-  vlib_stat_segment_unlock ();
-  ssvm_pop_heap (oldheap);
+      for (i = 0; i < vec_len (nodes); i++)
+	{
+	  counter_t **counters;
+	  counter_t *c;
+	  vlib_node_t *n = nodes[i];
+
+	  counters =
+	    stat_segment_pointer (shared_header,
+				  sm->directory_vector
+				  [STAT_COUNTER_NODE_CLOCKS].offset);
+	  c = counters[j];
+	  c[n->index] = n->stats_total.clocks - n->stats_last_clear.clocks;
+
+	  counters =
+	    stat_segment_pointer (shared_header,
+				  sm->directory_vector
+				  [STAT_COUNTER_NODE_VECTORS].offset);
+	  c = counters[j];
+	  c[n->index] = n->stats_total.vectors - n->stats_last_clear.vectors;
+
+	  counters =
+	    stat_segment_pointer (shared_header,
+				  sm->directory_vector
+				  [STAT_COUNTER_NODE_CALLS].offset);
+	  c = counters[j];
+	  c[n->index] = n->stats_total.calls - n->stats_last_clear.calls;
+
+	  counters =
+	    stat_segment_pointer (shared_header,
+				  sm->directory_vector
+				  [STAT_COUNTER_NODE_SUSPENDS].offset);
+	  c = counters[j];
+	  c[n->index] =
+	    n->stats_total.suspends - n->stats_last_clear.suspends;
+	}
+    }
 }
 
 /*
@@ -515,25 +495,27 @@ do_stat_segment_updates (stats_main_t * sm)
     }
   vector_rate /= (f64) (i - start);
 
-  *sm->vector_rate_ptr = vector_rate / ((f64) (vec_len (vlib_mains) - start));
+  sm->directory_vector[STAT_COUNTER_VECTOR_RATE].value =
+    vector_rate / ((f64) (vec_len (vlib_mains) - start));
 
   /*
    * Compute the aggregate input rate
    */
   now = vlib_time_now (vm);
-  dt = now - sm->last_runtime_ptr[0];
+  dt = now - sm->directory_vector[STAT_COUNTER_LAST_UPDATE].value;
   input_packets = vnet_get_aggregate_rx_packets ();
-  *sm->input_rate_ptr = (f64) (input_packets - sm->last_input_packets) / dt;
-  sm->last_runtime_ptr[0] = now;
+  sm->directory_vector[STAT_COUNTER_INPUT_RATE].value =
+    (f64) (input_packets - sm->last_input_packets) / dt;
+  sm->directory_vector[STAT_COUNTER_LAST_UPDATE].value = now;
   sm->last_input_packets = input_packets;
-  sm->last_runtime_stats_clear_ptr[0] =
+  sm->directory_vector[STAT_COUNTER_LAST_STATS_CLEAR].value =
     vm->node_main.time_last_runtime_stats_clear;
 
-  if (sm->serialize_nodes)
-    update_serialized_nodes (sm);
+  if (sm->node_counters_enabled)
+    update_node_counters (sm);
 
   /* Heartbeat, so clients detect we're still here */
-  (*sm->heartbeat_ptr)++;
+  sm->directory_vector[STAT_COUNTER_HEARTBEAT].value++;
 }
 
 static clib_error_t *
@@ -546,10 +528,10 @@ statseg_config (vlib_main_t * vm, unformat_input_t * input)
     {
       if (unformat (input, "size %U", unformat_memory_size, &sm->memory_size))
 	;
-      else if (unformat (input, "serialize-nodes on"))
-	sm->serialize_nodes = 1;
-      else if (unformat (input, "serialize-nodes off"))
-	sm->serialize_nodes = 0;
+      else if (unformat (input, "per-node-counters on"))
+	sm->node_counters_enabled = 1;
+      else if (unformat (input, "per-node-counters off"))
+	sm->node_counters_enabled = 0;
       else
 	return clib_error_return (0, "unknown input `%U'",
 				  format_unformat_error, input);
diff --git a/src/vpp/stats/stat_segment.h b/src/vpp/stats/stat_segment.h
new file mode 100644
index 00000000000..0efd45ac5f0
--- /dev/null
+++ b/src/vpp/stats/stat_segment.h
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2018 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef included_stat_segment_h
+#define included_stat_segment_h
+
+#include <stdatomic.h>
+
+/* Default socket to exchange segment fd */
+#define STAT_SEGMENT_SOCKET_FILE "/run/vpp/stats.sock"
+
+typedef enum
+{
+  STAT_DIR_TYPE_ILLEGAL = 0,
+  STAT_DIR_TYPE_SCALAR_INDEX,
+  STAT_DIR_TYPE_COUNTER_VECTOR_SIMPLE,
+  STAT_DIR_TYPE_COUNTER_VECTOR_COMBINED,
+  STAT_DIR_TYPE_ERROR_INDEX,
+} stat_directory_type_t;
+
+typedef enum
+{
+ STAT_COUNTER_VECTOR_RATE = 0,
+ STAT_COUNTER_INPUT_RATE,
+ STAT_COUNTER_LAST_UPDATE,
+ STAT_COUNTER_LAST_STATS_CLEAR,
+ STAT_COUNTER_HEARTBEAT,
+ STAT_COUNTER_NODE_CLOCKS,
+ STAT_COUNTER_NODE_VECTORS,
+ STAT_COUNTER_NODE_CALLS,
+ STAT_COUNTER_NODE_SUSPENDS,
+ STAT_COUNTERS
+} stat_segment_counter_t;
+
+#define foreach_stat_segment_counter_name			\
+  _(VECTOR_RATE, SCALAR_INDEX, vector_rate,)			\
+  _(INPUT_RATE, SCALAR_INDEX, input_rate,)			\
+  _(LAST_UPDATE, SCALAR_INDEX, last_update,)			\
+  _(LAST_STATS_CLEAR, SCALAR_INDEX, last_stats_clear,)		\
+  _(HEARTBEAT, SCALAR_INDEX, heartbeat,)			\
+  _(NODE_CLOCKS, COUNTER_VECTOR_SIMPLE, clocks, /node)		\
+  _(NODE_VECTORS, COUNTER_VECTOR_SIMPLE, vectors, /node)	\
+  _(NODE_CALLS, COUNTER_VECTOR_SIMPLE, calls, /node)		\
+  _(NODE_SUSPENDS, COUNTER_VECTOR_SIMPLE, suspends, /node)
+
+typedef struct
+{
+  stat_directory_type_t type;
+  union {
+    uint64_t offset;
+    uint64_t index;
+    uint64_t value;
+  };
+  uint64_t offset_vector;
+  char name[128]; // TODO change this to pointer to "somewhere"
+} stat_segment_directory_entry_t;
+
+/* Default stat segment 32m */
+#define STAT_SEGMENT_DEFAULT_SIZE	(32<<20)
+
+/*
+ * Shared header first in the shared memory segment.
+ */
+typedef struct
+{
+  atomic_int_fast64_t epoch;
+  atomic_int_fast64_t in_progress;
+  atomic_int_fast64_t directory_offset;
+  atomic_int_fast64_t error_offset;
+  atomic_int_fast64_t stats_offset;
+} stat_segment_shared_header_t;
+
+static inline uint64_t
+stat_segment_offset (void *start, void *data)
+{
+  return (char *) data - (char *) start;
+}
+
+static inline void *
+stat_segment_pointer (void *start, uint64_t offset)
+{
+  return ((char *) start + offset);
+}
+
+#endif
diff --git a/src/vpp/stats/stats.c b/src/vpp/stats/stats.c
index d0753741997..5eea25e4464 100644
--- a/src/vpp/stats/stats.c
+++ b/src/vpp/stats/stats.c
@@ -2348,7 +2348,6 @@ static clib_error_t *
 stats_socket_accept_ready (clib_file_t * uf)
 {
   stats_main_t *sm = &stats_main;
-  ssvm_private_t *ssvmp = &sm->stat_segment;
   clib_error_t *err;
   clib_socket_t client = { 0 };
 
@@ -2360,7 +2359,7 @@ stats_socket_accept_ready (clib_file_t * uf)
     }
 
   /* Send the fd across and close */
-  err = clib_socket_sendmsg (&client, 0, 0, &ssvmp->fd, 1);
+  err = clib_socket_sendmsg (&client, 0, 0, &sm->memfd, 1);
   if (err)
     clib_error_report (err);
   clib_socket_close (&client);
diff --git a/src/vpp/stats/stats.h b/src/vpp/stats/stats.h
index 911706cbb09..f21451d352b 100644
--- a/src/vpp/stats/stats.h
+++ b/src/vpp/stats/stats.h
@@ -26,11 +26,7 @@
 #include <vlib/unix/unix.h>
 #include <vlibmemory/api.h>
 #include <vlibapi/api_helper_macros.h>
-#include <svm/queue.h>
-#include <svm/ssvm.h>
-
-/* Default socket to exchange segment fd */
-#define STAT_SEGMENT_SOCKET_FILE "/run/vpp/stats.sock"
+#include <vpp/stats/stat_segment.h>
 
 typedef struct
 {
@@ -99,7 +95,6 @@ typedef struct
 
 } vpe_client_stats_registration_t;
 
-
 typedef struct
 {
   void *mheap;
@@ -162,32 +157,18 @@ typedef struct
   vpe_client_registration_t **clients_tmp;
 
   /* statistics segment */
-  ssvm_private_t stat_segment;
-  uword *counter_vector_by_name;
+  uword *directory_vector_by_name;
+  stat_segment_directory_entry_t *directory_vector;
   clib_spinlock_t *stat_segment_lockp;
   clib_socket_t *socket;
   u8 *socket_name;
-  uword memory_size;
-  u8 serialize_nodes;
-
-  /* Pointers to scalar stats maintained by the stat thread */
-  f64 *input_rate_ptr;
-  f64 *last_runtime_ptr;
-  f64 *last_runtime_stats_clear_ptr;
-  f64 *vector_rate_ptr;
-  f64 *heartbeat_ptr;
-  u64 last_input_packets;
-
-  /* Pointers to vector stats maintained by the stat thread */
-  u8 *serialized_nodes;
-  vlib_main_t **stat_vms;
-  vlib_node_t ***node_dups;
+  ssize_t memory_size;
+  u8 node_counters_enabled;
+  void *heap;
+  stat_segment_shared_header_t *shared_header;	/* pointer to shared memory segment */
+  int memfd;
 
-  f64 *vectors_per_node;
-  f64 *vector_rate_in;
-  f64 *vector_rate_out;
-  f64 *vector_rate_drop;
-  f64 *vector_rate_punt;
+  u64 last_input_packets;
 
   /* convenience */
   vlib_main_t *vlib_main;
@@ -198,30 +179,6 @@ typedef struct
 
 extern stats_main_t stats_main;
 
-/* Default stat segment 32m */
-#define STAT_SEGMENT_DEFAULT_SIZE	(32<<20)
-
-#define STAT_SEGMENT_OPAQUE_LOCK	0
-#define STAT_SEGMENT_OPAQUE_DIR		1
-#define STAT_SEGMENT_OPAQUE_EPOCH	2
-
-typedef enum
-{
-  STAT_DIR_TYPE_ILLEGAL = 0,
-  STAT_DIR_TYPE_SCALAR_POINTER,
-  STAT_DIR_TYPE_VECTOR_POINTER,
-  STAT_DIR_TYPE_COUNTER_VECTOR_SIMPLE,
-  STAT_DIR_TYPE_COUNTER_VECTOR_COMBINED,
-  STAT_DIR_TYPE_ERROR_INDEX,
-  STAT_DIR_TYPE_SERIALIZED_NODES,
-} stat_directory_type_t;
-
-typedef struct
-{
-  stat_directory_type_t type;
-  void *value;
-} stat_segment_directory_entry_t;
-
 void do_stat_segment_updates (stats_main_t * sm);
 
 #endif /* __included_stats_h__ */