stats: memory heap counters

- Add counters for the main-heap
- Add additional counters per heap:
  STAT_MEM_TOTAL
  STAT_MEM_USED,
  STAT_MEM_FREE,
  STAT_MEM_USED_MMAP,
  STAT_MEM_TOTAL_ALLOC,
  STAT_MEM_FREE_CHUNKS,
  STAT_MEM_RELEASABLE,

The per-heap counters are organised as a two dimensional vector.
total, used and free are directly available via symlinks.

vpp_get_stats ls "^/mem/"
/mem/stat segment
/mem/stat segment/total
/mem/stat segment/used
/mem/stat segment/free
/mem/main heap
/mem/main heap/total
/mem/main heap/used
/mem/main heap/free

vpp_get_stats dump "^/mem/main\ heap$"
[0 @ 0]: 1073741776 packets /mem/main heap
[1 @ 0]: 91586688 packets /mem/main heap
[2 @ 0]: 982155088 packets /mem/main heap
[3 @ 0]: 0 packets /mem/main heap
[4 @ 0]: 1073741776 packets /mem/main heap
[5 @ 0]: 433 packets /mem/main heap
[6 @ 0]: 981708688 packets /mem/main heap

Type: feature
Signed-off-by: Ole Troan <ot@cisco.com>
Change-Id: I36725dde3b4b3befd27a8b4d3ba931f2d3b627cc

7 files changed, 246 insertions, 94 deletions

diff --git a/src/vlib/stat_weak_inlines.h b/src/vlib/stat_weak_inlines.h
index a1311e864b3..a68566d0fdd 100644
--- a/src/vlib/stat_weak_inlines.h
+++ b/src/vlib/stat_weak_inlines.h
@@ -63,4 +63,10 @@ vlib_stats_delete_cm (void *notused)
 {
 }
 
+void vlib_stats_register_mem_heap (void *) __attribute__ ((weak));
+void
+vlib_stats_register_mem_heap (void *notused)
+{
+}
+
 #endif
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 7c796f5d0ea..499a626e470 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -654,6 +654,7 @@ start_workers (vlib_main_t * vm)
   u32 n_vlib_mains = tm->n_vlib_mains;
   u32 worker_thread_index;
   clib_mem_heap_t *main_heap = clib_mem_get_per_cpu_heap ();
+  vlib_stats_register_mem_heap (main_heap);
 
   vec_reset_length (vlib_worker_threads);
 
diff --git a/src/vpp/CMakeLists.txt b/src/vpp/CMakeLists.txt
index 07983502096..0b9a505f158 100644
--- a/src/vpp/CMakeLists.txt
+++ b/src/vpp/CMakeLists.txt
@@ -61,6 +61,7 @@ set(VPP_SOURCES
   app/vpe_cli.c
   app/version.c
   stats/stat_segment.c
+  stats/stat_segment_provider.c
   api/api.c
   api/json_format.c
   api/custom_dump.c
diff --git a/src/vpp/stats/stat_segment.c b/src/vpp/stats/stat_segment.c
index c8445ba5814..efbc2e42ffc 100644
--- a/src/vpp/stats/stat_segment.c
+++ b/src/vpp/stats/stat_segment.c
@@ -333,21 +333,32 @@ vlib_stats_register_error_index (void *oldheap, u8 * name, u64 * em_vec,
   vlib_stat_segment_unlock ();
 }
 
+/*
+ * Creates a two dimensional vector with the maximum valid index specified in
+ * both dimensions as arguments.
+ * Must be called on the stat segment heap.
+ */
 static void
-stat_validate_counter_vector (stat_segment_directory_entry_t * ep, u32 max)
+stat_validate_counter_vector2 (stat_segment_directory_entry_t *ep, u32 max1,
+			       u32 max2)
 {
   counter_t **counters = ep->data;
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
   int i;
-
-  vec_validate_aligned (counters, tm->n_vlib_mains - 1,
-			CLIB_CACHE_LINE_BYTES);
-  for (i = 0; i < tm->n_vlib_mains; i++)
-    vec_validate_aligned (counters[i], max, CLIB_CACHE_LINE_BYTES);
+  vec_validate_aligned (counters, max1, CLIB_CACHE_LINE_BYTES);
+  for (i = 0; i <= max1; i++)
+    vec_validate_aligned (counters[i], max2, CLIB_CACHE_LINE_BYTES);
 
   ep->data = counters;
 }
 
+static void
+stat_validate_counter_vector (stat_segment_directory_entry_t *ep, u32 max)
+{
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  ASSERT (tm->n_vlib_mains > 0);
+  stat_validate_counter_vector2 (ep, tm->n_vlib_mains, max);
+}
+
 always_inline void
 stat_set_simple_counter (stat_segment_directory_entry_t * ep,
 			 u32 thread_index, u32 index, u64 value)
@@ -382,6 +393,41 @@ vlib_stats_pop_heap2 (u64 * error_vector, u32 thread_index, void *oldheap,
   clib_mem_set_heap (oldheap);
 }
 
+/*
+ * Create a new entry and add name to directory hash.
+ * Returns ~0 if name exists.
+ * Called from main heap.
+ */
+u32
+stat_segment_new_entry (u8 *name, stat_directory_type_t t)
+{
+  stat_segment_main_t *sm = &stat_segment_main;
+  stat_segment_shared_header_t *shared_header = sm->shared_header;
+  void *oldheap;
+  stat_segment_directory_entry_t e;
+
+  ASSERT (shared_header);
+
+  u32 vector_index = lookup_hash_index (name);
+  if (vector_index != STAT_SEGMENT_INDEX_INVALID) /* Already registered */
+    return ~0;
+
+  memset (&e, 0, sizeof (e));
+  e.type = t;
+  memcpy (e.name, name, vec_len (name));
+
+  oldheap = vlib_stats_push_heap (NULL);
+  vlib_stat_segment_lock ();
+  vector_index = vlib_stats_create_counter (&e, oldheap);
+
+  shared_header->directory_vector = sm->directory_vector;
+
+  vlib_stat_segment_unlock ();
+  clib_mem_set_heap (oldheap);
+
+  return vector_index;
+}
+
 clib_error_t *
 vlib_map_stat_segment_init (void)
 {
@@ -419,9 +465,9 @@ vlib_map_stat_segment_init (void)
 
   sys_page_sz = clib_mem_get_page_size ();
 
-  heap = clib_mem_create_heap (((u8 *) memaddr) + sys_page_sz, memory_size
-			       - sys_page_sz, 1 /* locked */ ,
-			       "stat segment");
+  heap =
+    clib_mem_create_heap (((u8 *) memaddr) + sys_page_sz,
+			  memory_size - sys_page_sz, 1 /* locked */, mem_name);
   sm->heap = heap;
   sm->memfd = mfd;
 
@@ -453,11 +499,7 @@ vlib_map_stat_segment_init (void)
 
   clib_mem_set_heap (oldheap);
 
-  /* Total shared memory size */
-  clib_mem_usage_t usage;
-  clib_mem_get_heap_usage (sm->heap, &usage);
-  sm->directory_vector[STAT_COUNTER_MEM_STATSEG_TOTAL].value =
-    usage.bytes_total;
+  vlib_stats_register_mem_heap (heap);
 
   return 0;
 }
@@ -504,6 +546,10 @@ format_stat_dir_entry (u8 * s, va_list * args)
       type_name = "empty";
       break;
 
+    case STAT_DIR_TYPE_SYMLINK:
+      type_name = "Symlink";
+      break;
+
     default:
       type_name = "illegal!";
       break;
@@ -771,12 +817,6 @@ do_stat_segment_updates (vlib_main_t *vm, stat_segment_main_t *sm)
   sm->directory_vector[STAT_COUNTER_LAST_STATS_CLEAR].value =
     vm->node_main.time_last_runtime_stats_clear;
 
-  /* Stats segment memory heap counter */
-  clib_mem_usage_t usage;
-  clib_mem_get_heap_usage (sm->heap, &usage);
-  sm->directory_vector[STAT_COUNTER_MEM_STATSEG_USED].value =
-    usage.bytes_used;
-
   if (sm->node_counters_enabled)
     update_node_counters (sm);
 
@@ -879,56 +919,40 @@ stat_segment_collector_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
   return 0;			/* or not */
 }
 
-static clib_error_t *
-statseg_init (vlib_main_t * vm)
+/*
+ * Add a data provider (via callback) for a given stats entry.
+ * TODO: Add support for per-provider interval.
+ */
+void
+stat_segment_poll_add (u32 vector_index, stat_segment_update_fn update_fn,
+		       u32 caller_index, u32 interval)
 {
   stat_segment_main_t *sm = &stat_segment_main;
+  stat_segment_gauges_pool_t *gauge;
 
-  /* set default socket file name when statseg config stanza is empty. */
-  if (!vec_len (sm->socket_name))
-    sm->socket_name = format (0, "%s/%s%c", vlib_unix_get_runtime_dir (),
-			      STAT_SEGMENT_SOCKET_FILENAME, 0);
-  return stats_segment_socket_init ();
-}
+  pool_get (sm->gauges, gauge);
+  gauge->fn = update_fn;
+  gauge->caller_index = caller_index;
+  gauge->directory_index = vector_index;
 
-/* *INDENT-OFF* */
-VLIB_INIT_FUNCTION (statseg_init) =
-{
-  .runs_after = VLIB_INITS("unix_input_init"),
-};
-/* *INDENT-ON* */
+  return;
+}
 
+/*
+ * Create an scalar entry with a data provider.
+ * Deprecated, replace with stat_segment_new_entry + stat_segment_pool_add
+ */
 clib_error_t *
 stat_segment_register_gauge (u8 * name, stat_segment_update_fn update_fn,
 			     u32 caller_index)
 {
   stat_segment_main_t *sm = &stat_segment_main;
-  stat_segment_shared_header_t *shared_header = sm->shared_header;
-  void *oldheap;
-  stat_segment_directory_entry_t e;
   stat_segment_gauges_pool_t *gauge;
 
-  ASSERT (shared_header);
-
-  u32 vector_index = lookup_hash_index (name);
-
-  if (vector_index != STAT_SEGMENT_INDEX_INVALID)	/* Already registered */
+  u32 vector_index = stat_segment_new_entry (name, STAT_DIR_TYPE_SCALAR_INDEX);
+  if (vector_index == ~0) /* Already registered */
     return clib_error_return (0, "%v is already registered", name);
 
-  memset (&e, 0, sizeof (e));
-  e.type = STAT_DIR_TYPE_SCALAR_INDEX;
-  memcpy (e.name, name, vec_len (name));
-
-  oldheap = vlib_stats_push_heap (NULL);
-  vlib_stat_segment_lock ();
-  vector_index = vlib_stats_create_counter (&e, oldheap);
-
-  shared_header->directory_vector = sm->directory_vector;
-
-  vlib_stat_segment_unlock ();
-  clib_mem_set_heap (oldheap);
-
-  /* Back on our own heap */
   pool_get (sm->gauges, gauge);
   gauge->fn = update_fn;
   gauge->caller_index = caller_index;
@@ -940,33 +964,11 @@ stat_segment_register_gauge (u8 * name, stat_segment_update_fn update_fn,
 clib_error_t *
 stat_segment_register_state_counter (u8 * name, u32 * index)
 {
-  stat_segment_main_t *sm = &stat_segment_main;
-  stat_segment_shared_header_t *shared_header = sm->shared_header;
-  void *oldheap;
-  stat_segment_directory_entry_t e;
-
-  ASSERT (shared_header);
   ASSERT (vlib_get_thread_index () == 0);
 
-  u32 vector_index = lookup_hash_index (name);
-
-  if (vector_index != STAT_SEGMENT_INDEX_INVALID)	/* Already registered */
+  u32 vector_index = stat_segment_new_entry (name, STAT_DIR_TYPE_SCALAR_INDEX);
+  if (vector_index == ~0) /* Already registered */
     return clib_error_return (0, "%v is already registered", name);
-
-  memset (&e, 0, sizeof (e));
-  e.type = STAT_DIR_TYPE_SCALAR_INDEX;
-  memcpy (e.name, name, vec_len (name));
-
-  oldheap = vlib_stats_push_heap (NULL);
-  vlib_stat_segment_lock ();
-
-  vector_index = vlib_stats_create_counter (&e, oldheap);
-
-  shared_header->directory_vector = sm->directory_vector;
-
-  vlib_stat_segment_unlock ();
-  clib_mem_set_heap (oldheap);
-
   *index = vector_index;
   return 0;
 }
@@ -1124,7 +1126,6 @@ statseg_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
 
 VNET_SW_INTERFACE_ADD_DEL_FUNCTION (statseg_sw_interface_add_del);
 
-/* *INDENT-OFF* */
 VLIB_REGISTER_NODE (stat_segment_collector, static) =
 {
 .function = stat_segment_collector_process,
@@ -1132,7 +1133,21 @@ VLIB_REGISTER_NODE (stat_segment_collector, static) =
 .type = VLIB_NODE_TYPE_PROCESS,
 };
 
-/* *INDENT-ON* */
+static clib_error_t *
+statseg_init (vlib_main_t *vm)
+{
+  stat_segment_main_t *sm = &stat_segment_main;
+
+  /* set default socket file name when statseg config stanza is empty. */
+  if (!vec_len (sm->socket_name))
+    sm->socket_name = format (0, "%s/%s%c", vlib_unix_get_runtime_dir (),
+			      STAT_SEGMENT_SOCKET_FILENAME, 0);
+  return stats_segment_socket_init ();
+}
+
+VLIB_INIT_FUNCTION (statseg_init) = {
+  .runs_after = VLIB_INITS ("unix_input_init"),
+};
 
 /*
  * fd.io coding-style-patch-verification: ON
diff --git a/src/vpp/stats/stat_segment.h b/src/vpp/stats/stat_segment.h
index 1d1ff31c294..c6056b1fade 100644
--- a/src/vpp/stats/stat_segment.h
+++ b/src/vpp/stats/stat_segment.h
@@ -35,8 +35,6 @@ typedef enum
  STAT_COUNTER_NODE_SUSPENDS,
  STAT_COUNTER_INTERFACE_NAMES,
  STAT_COUNTER_NODE_NAMES,
- STAT_COUNTER_MEM_STATSEG_TOTAL,
- STAT_COUNTER_MEM_STATSEG_USED,
  STAT_COUNTERS
 } stat_segment_counter_t;
 
@@ -58,8 +56,6 @@ typedef enum
   _ (HEARTBEAT, SCALAR_INDEX, heartbeat, /sys)                                \
   _ (INTERFACE_NAMES, NAME_VECTOR, names, /if)                                \
   _ (NODE_NAMES, NAME_VECTOR, names, /sys/node)                               \
-  _ (MEM_STATSEG_TOTAL, SCALAR_INDEX, total, /mem/statseg)                    \
-  _ (MEM_STATSEG_USED, SCALAR_INDEX, used, /mem/statseg)                      \
   foreach_stat_segment_node_counter_name
 /* clang-format on */
 
@@ -117,5 +113,14 @@ stat_segment_register_state_counter(u8 *name, u32 *index);
 clib_error_t *
 stat_segment_deregister_state_counter(u32 index);
 void stat_segment_set_state_counter (u32 index, u64 value);
+void stat_segment_poll_add (u32 vector_index, stat_segment_update_fn update_fn,
+			    u32 caller_index, u32 interval);
+
+u32 stat_segment_new_entry (u8 *name, stat_directory_type_t t);
+void vlib_stats_register_mem_heap (clib_mem_heap_t *heap);
+void vlib_stat_segment_lock (void);
+void vlib_stat_segment_unlock (void);
+void vlib_stats_register_symlink (void *oldheap, u8 *name, u32 index1,
+				  u32 index2, u8 lock);
 
 #endif
diff --git a/src/vpp/stats/stat_segment_provider.c b/src/vpp/stats/stat_segment_provider.c
new file mode 100644
index 00000000000..2212ffe98dd
--- /dev/null
+++ b/src/vpp/stats/stat_segment_provider.c
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2021 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Counters handled by the stats module directly.
+ */
+
+#include <stdbool.h>
+#include <vppinfra/mem.h>
+#include <vppinfra/vec.h>
+#include <vlib/vlib.h>
+#include <vlib/counter.h>
+#include "stat_segment.h"
+
+clib_mem_heap_t **memory_heaps_vec;
+u32 mem_vector_index;
+bool initialized = false;
+
+enum
+{
+  STAT_MEM_TOTAL = 0,
+  STAT_MEM_USED,
+  STAT_MEM_FREE,
+  STAT_MEM_USED_MMAP,
+  STAT_MEM_TOTAL_ALLOC,
+  STAT_MEM_FREE_CHUNKS,
+  STAT_MEM_RELEASABLE,
+} stat_mem_usage_e;
+
+/*
+ * Called from the stats periodic process to update memory counters.
+ */
+static void
+stat_provider_mem_usage_update_fn (stat_segment_directory_entry_t *e,
+				   u32 index)
+{
+  clib_mem_usage_t usage;
+  clib_mem_heap_t *heap;
+  counter_t **counters = e->data;
+  counter_t *cb;
+
+  heap = vec_elt (memory_heaps_vec, index);
+  clib_mem_get_heap_usage (heap, &usage);
+  cb = counters[0];
+  cb[STAT_MEM_TOTAL] = usage.bytes_total;
+  cb[STAT_MEM_USED] = usage.bytes_used;
+  cb[STAT_MEM_FREE] = usage.bytes_free;
+  cb[STAT_MEM_USED_MMAP] = usage.bytes_used_mmap;
+  cb[STAT_MEM_TOTAL_ALLOC] = usage.bytes_max;
+  cb[STAT_MEM_FREE_CHUNKS] = usage.bytes_free_reclaimed;
+  cb[STAT_MEM_RELEASABLE] = usage.bytes_overhead;
+}
+
+static counter_t **
+stat_validate_counter_vector3 (counter_t **counters, u32 max1, u32 max2)
+{
+  stat_segment_main_t *sm = &stat_segment_main;
+  int i;
+  void *oldheap = clib_mem_set_heap (sm->heap);
+  vec_validate_aligned (counters, max1, CLIB_CACHE_LINE_BYTES);
+  for (i = 0; i <= max1; i++)
+    vec_validate_aligned (counters[i], max2, CLIB_CACHE_LINE_BYTES);
+  clib_mem_set_heap (oldheap);
+  return counters;
+}
+
+/*
+ * Provide memory heap counters.
+ * Two dimensional array of heap index and per-heap gauges.
+ */
+void
+vlib_stats_register_mem_heap (clib_mem_heap_t *heap)
+{
+  stat_segment_main_t *sm = &stat_segment_main;
+  vec_add1 (memory_heaps_vec, heap);
+  u32 heap_index = vec_len (memory_heaps_vec) - 1;
+
+  /* Memory counters provider */
+  u8 *s = format (0, "/mem/%s", heap->name);
+  u8 *s_used = format (0, "/mem/%s/used", heap->name);
+  u8 *s_total = format (0, "/mem/%s/total", heap->name);
+  u8 *s_free = format (0, "/mem/%s/free", heap->name);
+  mem_vector_index =
+    stat_segment_new_entry (s, STAT_DIR_TYPE_COUNTER_VECTOR_SIMPLE);
+  vec_free (s);
+  if (mem_vector_index == ~0)
+    ASSERT (0);
+
+  vlib_stat_segment_lock ();
+  stat_segment_directory_entry_t *ep = &sm->directory_vector[mem_vector_index];
+  ep->data = stat_validate_counter_vector3 (ep->data, 0, STAT_MEM_RELEASABLE);
+
+  /* Create symlink */
+  void *oldheap = clib_mem_set_heap (sm->heap);
+  vlib_stats_register_symlink (oldheap, s_total, mem_vector_index,
+			       STAT_MEM_TOTAL, 0);
+  vlib_stats_register_symlink (oldheap, s_used, mem_vector_index,
+			       STAT_MEM_USED, 0);
+  vlib_stats_register_symlink (oldheap, s_free, mem_vector_index,
+			       STAT_MEM_FREE, 0);
+  vlib_stat_segment_unlock ();
+  clib_mem_set_heap (oldheap);
+  vec_free (s_used);
+  vec_free (s_total);
+  vec_free (s_free);
+
+  stat_segment_poll_add (mem_vector_index, stat_provider_mem_usage_update_fn,
+			 heap_index, 10);
+}
diff --git a/src/vppinfra/mem_dlmalloc.c b/src/vppinfra/mem_dlmalloc.c
index bc6561a738e..f8d4ca19fd8 100644
--- a/src/vppinfra/mem_dlmalloc.c
+++ b/src/vppinfra/mem_dlmalloc.c
@@ -474,14 +474,17 @@ clib_mem_get_heap_usage (clib_mem_heap_t * heap, clib_mem_usage_t * usage)
 {
   struct dlmallinfo mi = mspace_mallinfo (heap->mspace);
 
-  /* TODO: Fill in some more values */
+  usage->bytes_total = mi.arena; /* non-mmapped space allocated from system */
+  usage->bytes_used = mi.uordblks;	    /* total allocated space */
+  usage->bytes_free = mi.fordblks;	    /* total free space */
+  usage->bytes_used_mmap = mi.hblkhd;	    /* space in mmapped regions */
+  usage->bytes_max = mi.usmblks;	    /* maximum total allocated space */
+  usage->bytes_free_reclaimed = mi.ordblks; /* number of free chunks */
+  usage->bytes_overhead = mi.keepcost; /* releasable (via malloc_trim) space */
+
+  /* Not supported */
+  usage->bytes_used_sbrk = 0;
   usage->object_count = 0;
-  usage->bytes_total = mi.arena;
-  usage->bytes_overhead = 0;
-  usage->bytes_max = 0;
-  usage->bytes_used = mi.uordblks;
-  usage->bytes_free = mi.fordblks;
-  usage->bytes_free_reclaimed = 0;
 }
 
 /* Call serial number for debugger breakpoints. */