aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBenoît Ganne <bganne@cisco.com>2021-03-09 15:37:49 +0100
committerDamjan Marion <dmarion@me.com>2021-08-27 10:05:31 +0000
commite09a2337b82d2dcb5b7379a9581477af291d1a75 (patch)
tree619da9a205808a35efbd9d2c21bdc010a447e2d9
parenta13100f3aa21a39de991df9fff84cd91e4be3b80 (diff)
bufmon: add buffer monitoring plugin
This plugin allow to keep track of buffer usage in VPP graph nodes. The main use is to detect buffer leakages. Type: feature Change-Id: Iadcf4ab98207fab6e2fa375060879bc2a25b711e Signed-off-by: Benoît Ganne <bganne@cisco.com>
-rw-r--r--MAINTAINERS5
-rw-r--r--src/plugins/bufmon/CMakeLists.txt17
-rw-r--r--src/plugins/bufmon/FEATURE.yaml8
-rw-r--r--src/plugins/bufmon/bufmon.c313
-rw-r--r--src/plugins/bufmon/bufmon_doc.md24
-rw-r--r--src/vlib/buffer.c34
-rw-r--r--src/vlib/buffer.h26
-rw-r--r--src/vlib/buffer_funcs.h19
8 files changed, 422 insertions, 24 deletions
diff --git a/MAINTAINERS b/MAINTAINERS
index e0183dc2bd7..1417fa0333e 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -764,6 +764,11 @@ I: srtp
M: Florin Coras <fcoras@cisco.com>
F: src/plugins/srtp/
+Plugin - bufmon
+I: bufmon
+M: Benoît Ganne <bganne@cisco.com>
+F: src/plugins/bufmon/
+
cJSON
I: cjson
M: Ole Troan <ot@cisco.com>
diff --git a/src/plugins/bufmon/CMakeLists.txt b/src/plugins/bufmon/CMakeLists.txt
new file mode 100644
index 00000000000..bf929598f92
--- /dev/null
+++ b/src/plugins/bufmon/CMakeLists.txt
@@ -0,0 +1,17 @@
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_vpp_plugin(bufmon
+ SOURCES
+ bufmon.c
+)
diff --git a/src/plugins/bufmon/FEATURE.yaml b/src/plugins/bufmon/FEATURE.yaml
new file mode 100644
index 00000000000..639643e5a28
--- /dev/null
+++ b/src/plugins/bufmon/FEATURE.yaml
@@ -0,0 +1,8 @@
+---
+name: Buffers monitoring plugin
+maintainer: Benoît Ganne <bganne@cisco.com>
+features:
+ - monitor buffer utiization in VPP graph nodes
+description: "monitor buffer utiization in VPP graph nodes"
+state: production
+properties: [CLI, MULTITHREAD]
diff --git a/src/plugins/bufmon/bufmon.c b/src/plugins/bufmon/bufmon.c
new file mode 100644
index 00000000000..2a35acce482
--- /dev/null
+++ b/src/plugins/bufmon/bufmon.c
@@ -0,0 +1,313 @@
+#include <vlib/vlib.h>
+#include <vnet/plugin/plugin.h>
+#include <vpp/app/version.h>
+
+typedef struct
+{
+ u64 in;
+ u64 out;
+ u64 alloc;
+ u64 free;
+} bufmon_per_node_data_t;
+
+typedef struct
+{
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+ bufmon_per_node_data_t *pnd;
+ u32 cur_node;
+} bufmon_per_thread_data_t;
+
+typedef struct
+{
+ bufmon_per_thread_data_t *ptd;
+ int enabled;
+} bufmon_main_t;
+
+static bufmon_main_t bufmon_main;
+
+static u32
+bufmon_alloc_free_callback (vlib_main_t *vm, u32 n_buffers, const int is_free)
+{
+ bufmon_main_t *bm = &bufmon_main;
+ bufmon_per_thread_data_t *ptd;
+ bufmon_per_node_data_t *pnd;
+ u32 cur_node;
+
+ if (PREDICT_FALSE (vm->thread_index >= vec_len (bm->ptd)))
+ {
+ clib_warning ("bufmon: thread index %d unknown for buffer %s (%d)",
+ vm->thread_index, is_free ? "free" : "alloc", n_buffers);
+ return n_buffers;
+ }
+
+ ptd = vec_elt_at_index (bm->ptd, vm->thread_index);
+
+ cur_node = ptd->cur_node;
+ if (cur_node >= vec_len (ptd->pnd))
+ {
+ cur_node = vlib_get_current_process_node_index (vm);
+ vec_validate_aligned (ptd->pnd, cur_node, CLIB_CACHE_LINE_BYTES);
+ }
+
+ pnd = vec_elt_at_index (ptd->pnd, cur_node);
+
+ if (is_free)
+ pnd->free += n_buffers;
+ else
+ pnd->alloc += n_buffers;
+
+ return n_buffers;
+}
+
+static u32
+bufmon_alloc_callback (vlib_main_t *vm, u8 buffer_pool_index, u32 *buffers,
+ u32 n_buffers)
+{
+ return bufmon_alloc_free_callback (vm, n_buffers, 0 /* is_free */);
+}
+
+static u32
+bufmon_free_callback (vlib_main_t *vm, u8 buffer_pool_index, u32 *buffers,
+ u32 n_buffers)
+{
+ return bufmon_alloc_free_callback (vm, n_buffers, 1 /* is_free */);
+}
+
+static u32
+bufmon_count_buffers (vlib_main_t *vm, vlib_frame_t *frame)
+{
+ vlib_buffer_t *b[VLIB_FRAME_SIZE];
+ u32 *from = vlib_frame_vector_args (frame);
+ const u32 n = frame->n_vectors;
+ u32 nc = 0;
+ u32 i;
+
+ vlib_get_buffers (vm, from, b, n);
+
+ for (i = 0; i < n; i++)
+ {
+ const vlib_buffer_t *cb = b[i];
+ while (cb->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ nc++;
+ cb = vlib_get_buffer (vm, cb->next_buffer);
+ }
+ }
+
+ return n + nc;
+}
+
+static uword
+bufmon_dispatch_wrapper (vlib_main_t *vm, vlib_node_runtime_t *node,
+ vlib_frame_t *frame)
+{
+ vlib_node_main_t *nm = &vm->node_main;
+ bufmon_main_t *bm = &bufmon_main;
+ bufmon_per_thread_data_t *ptd;
+ bufmon_per_node_data_t *pnd;
+ int pending_frames;
+ uword rv;
+
+ vec_validate_aligned (bm->ptd, vm->thread_index, CLIB_CACHE_LINE_BYTES);
+ ptd = vec_elt_at_index (bm->ptd, vm->thread_index);
+ vec_validate_aligned (ptd->pnd, node->node_index, CLIB_CACHE_LINE_BYTES);
+ pnd = vec_elt_at_index (ptd->pnd, node->node_index);
+
+ if (frame)
+ pnd->in += bufmon_count_buffers (vm, frame);
+
+ pending_frames = vec_len (nm->pending_frames);
+ ptd->cur_node = node->node_index;
+
+ rv = node->function (vm, node, frame);
+
+ ptd->cur_node = ~0;
+ for (; pending_frames < vec_len (nm->pending_frames); pending_frames++)
+ {
+ vlib_pending_frame_t *p =
+ vec_elt_at_index (nm->pending_frames, pending_frames);
+ pnd->out += bufmon_count_buffers (vm, vlib_get_frame (vm, p->frame));
+ }
+
+ return rv;
+}
+
+static void
+bufmon_unregister_callbacks (vlib_main_t *vm)
+{
+ vlib_buffer_set_alloc_free_callback (vm, 0, 0);
+ foreach_vlib_main ()
+ vlib_node_set_dispatch_wrapper (this_vlib_main, 0);
+}
+
+static clib_error_t *
+bufmon_register_callbacks (vlib_main_t *vm)
+{
+ if (vlib_buffer_set_alloc_free_callback (vm, bufmon_alloc_callback,
+ bufmon_free_callback))
+ goto err0;
+
+ foreach_vlib_main ()
+ if (vlib_node_set_dispatch_wrapper (this_vlib_main,
+ bufmon_dispatch_wrapper))
+ goto err1;
+
+ return 0;
+
+err1:
+ foreach_vlib_main ()
+ vlib_node_set_dispatch_wrapper (this_vlib_main, 0);
+err0:
+ vlib_buffer_set_alloc_free_callback (vm, 0, 0);
+ return clib_error_return (0, "failed to register callback");
+}
+
+static clib_error_t *
+bufmon_enable_disable (vlib_main_t *vm, int enable)
+{
+ bufmon_main_t *bm = &bufmon_main;
+
+ if (enable)
+ {
+ if (bm->enabled)
+ return 0;
+ clib_error_t *error = bufmon_register_callbacks (vm);
+ if (error)
+ return error;
+ bm->enabled = 1;
+ }
+ else
+ {
+ if (!bm->enabled)
+ return 0;
+ bufmon_unregister_callbacks (vm);
+ bm->enabled = 0;
+ }
+
+ return 0;
+}
+
+static clib_error_t *
+set_buffer_traces (vlib_main_t *vm, unformat_input_t *input,
+ vlib_cli_command_t *cmd)
+{
+ unformat_input_t _line_input, *line_input = &_line_input;
+ int on = 1;
+
+ if (unformat_user (input, unformat_line_input, line_input))
+ {
+ while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+ {
+ if (unformat (line_input, "on"))
+ on = 1;
+ else if (unformat (line_input, "off"))
+ on = 0;
+ else
+ {
+ unformat_free (line_input);
+ return clib_error_return (0, "unknown input `%U'",
+ format_unformat_error, line_input);
+ }
+ }
+ unformat_free (line_input);
+ }
+
+ return bufmon_enable_disable (vm, on);
+}
+
+VLIB_CLI_COMMAND (set_buffer_traces_command, static) = {
+ .path = "set buffer traces",
+ .short_help = "set buffer traces [on|off]",
+ .function = set_buffer_traces,
+};
+
+static clib_error_t *
+show_buffer_traces (vlib_main_t *vm, unformat_input_t *input,
+ vlib_cli_command_t *cmd)
+{
+ unformat_input_t _line_input, *line_input = &_line_input;
+ const bufmon_main_t *bm = &bufmon_main;
+ const bufmon_per_thread_data_t *ptd;
+ const bufmon_per_node_data_t *pnd;
+ int verbose = 0;
+ int status = 0;
+
+ if (unformat_user (input, unformat_line_input, line_input))
+ {
+ while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+ {
+ if (unformat (line_input, "verbose"))
+ verbose = 1;
+ else if (unformat (line_input, "status"))
+ status = 1;
+ else
+ {
+ unformat_free (line_input);
+ return clib_error_return (0, "unknown input `%U'",
+ format_unformat_error, line_input);
+ }
+ }
+ unformat_free (line_input);
+ }
+
+ if (status)
+ {
+ vlib_cli_output (vm, "buffers tracing is %s",
+ bm->enabled ? "on" : "off");
+ return 0;
+ }
+
+ vlib_cli_output (vm, "%U\n\n", format_vlib_buffer_pool_all, vm);
+ vlib_cli_output (vm, "%30s%20s%20s%20s%20s%20s", "Node", "Allocated",
+ "Freed", "In", "Out", "Buffered");
+ vec_foreach (ptd, bm->ptd)
+ {
+ vec_foreach (pnd, ptd->pnd)
+ {
+ const u64 in = pnd->alloc + pnd->in;
+ const u64 out = pnd->free + pnd->out;
+ const i64 buffered = in - out;
+ if (0 == in && 0 == out)
+ continue; /* skip nodes w/o activity */
+ if (0 == buffered && !verbose)
+ continue; /* if not verbose, skip nodes w/o buffered buffers */
+ vlib_cli_output (vm, "%30U%20lu%20lu%20lu%20lu%20ld",
+ format_vlib_node_name, vm, pnd - ptd->pnd,
+ pnd->alloc, pnd->free, pnd->in, pnd->out, buffered);
+ }
+ }
+
+ return 0;
+}
+
+VLIB_CLI_COMMAND (show_buffer_traces_command, static) = {
+ .path = "show buffer traces",
+ .short_help = "show buffer traces [status|verbose]",
+ .function = show_buffer_traces,
+};
+
+static clib_error_t *
+clear_buffer_traces (vlib_main_t *vm, unformat_input_t *input,
+ vlib_cli_command_t *cmd)
+{
+ const bufmon_main_t *bm = &bufmon_main;
+ const bufmon_per_thread_data_t *ptd;
+ const bufmon_per_node_data_t *pnd;
+
+ vec_foreach (ptd, bm->ptd)
+ vec_foreach (pnd, ptd->pnd)
+ vec_reset_length (pnd);
+
+ return 0;
+}
+
+VLIB_CLI_COMMAND (clear_buffers_trace_command, static) = {
+ .path = "clear buffer traces",
+ .short_help = "clear buffer traces",
+ .function = clear_buffer_traces,
+};
+
+VLIB_PLUGIN_REGISTER () = {
+ .version = VPP_BUILD_VER,
+ .description = "Buffers monitoring plugin",
+};
diff --git a/src/plugins/bufmon/bufmon_doc.md b/src/plugins/bufmon/bufmon_doc.md
new file mode 100644
index 00000000000..bfa88f40a7d
--- /dev/null
+++ b/src/plugins/bufmon/bufmon_doc.md
@@ -0,0 +1,24 @@
+# Buffers monitoring plugin {#bufmon_doc}
+
+This plugin enables to track buffer utilization in the VPP graph nodes. The
+main use is to detect buffer leakage.
+It works by keeping track of number of buffer allocations and free in graph
+nodes and also of number of buffers received in input frames and in output
+frames.
+The formula to compute the number of "buffered" buffers in a node is simply:
+ #buffered = #alloc + #input - #free - #output
+Note: monitoring will impact performances.
+
+## Basic usage
+1. Turn buffer traces on:
+```
+~# vppctl set buffer traces on
+```
+2. Monitor buffer usage:
+```
+~# vppctl show buffer traces verbose
+```
+3. Turn buffer traces off:
+```
+~# vppctl set buffer traces off
+```
diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index adaafa36f5d..71f84d377af 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -615,20 +615,26 @@ format_vlib_buffer_pool (u8 * s, va_list * va)
return s;
}
-static clib_error_t *
-show_buffers (vlib_main_t * vm,
- unformat_input_t * input, vlib_cli_command_t * cmd)
+u8 *
+format_vlib_buffer_pool_all (u8 *s, va_list *va)
{
+ vlib_main_t *vm = va_arg (*va, vlib_main_t *);
vlib_buffer_main_t *bm = vm->buffer_main;
vlib_buffer_pool_t *bp;
- vlib_cli_output (vm, "%U", format_vlib_buffer_pool, vm, 0);
+ s = format (s, "%U", format_vlib_buffer_pool, vm, 0);
- /* *INDENT-OFF* */
vec_foreach (bp, bm->buffer_pools)
- vlib_cli_output (vm, "%U", format_vlib_buffer_pool, vm, bp);
- /* *INDENT-ON* */
+ s = format (s, "\n%U", format_vlib_buffer_pool, vm, bp);
+
+ return s;
+}
+static clib_error_t *
+show_buffers (vlib_main_t *vm, unformat_input_t *input,
+ vlib_cli_command_t *cmd)
+{
+ vlib_cli_output (vm, "%U", format_vlib_buffer_pool_all, vm);
return 0;
}
@@ -971,6 +977,20 @@ vlib_buffer_alloc_may_fail (vlib_main_t * vm, u32 n_buffers)
}
#endif
+__clib_export int
+vlib_buffer_set_alloc_free_callback (
+ vlib_main_t *vm, vlib_buffer_alloc_free_callback_t *alloc_callback_fn,
+ vlib_buffer_alloc_free_callback_t *free_callback_fn)
+{
+ vlib_buffer_main_t *bm = vm->buffer_main;
+ if ((alloc_callback_fn && bm->alloc_callback_fn) ||
+ (free_callback_fn && bm->free_callback_fn))
+ return 1;
+ bm->alloc_callback_fn = alloc_callback_fn;
+ bm->free_callback_fn = free_callback_fn;
+ return 0;
+}
+
/** @endcond */
/*
* fd.io coding-style-patch-verification: ON
diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h
index 349b7324996..b548adf4be8 100644
--- a/src/vlib/buffer.h
+++ b/src/vlib/buffer.h
@@ -472,6 +472,10 @@ typedef struct
#define VLIB_BUFFER_MAX_NUMA_NODES 32
+typedef u32 (vlib_buffer_alloc_free_callback_t) (struct vlib_main_t *vm,
+ u8 buffer_pool_index,
+ u32 *buffers, u32 n_buffers);
+
typedef struct
{
CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
@@ -481,12 +485,9 @@ typedef struct
uword buffer_mem_size;
vlib_buffer_pool_t *buffer_pools;
- /* Hash table mapping buffer index into number
- 0 => allocated but free, 1 => allocated and not-free.
- If buffer index is not in hash table then this buffer
- has never been allocated. */
- uword *buffer_known_hash;
- clib_spinlock_t buffer_known_hash_lockp;
+ vlib_buffer_alloc_free_callback_t *alloc_callback_fn;
+ vlib_buffer_alloc_free_callback_t *free_callback_fn;
+
u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES];
/* config */
@@ -495,12 +496,25 @@ typedef struct
u32 default_data_size;
clib_mem_page_sz_t log2_page_size;
+ /* Hash table mapping buffer index into number
+ 0 => allocated but free, 1 => allocated and not-free.
+ If buffer index is not in hash table then this buffer
+ has never been allocated. */
+ uword *buffer_known_hash;
+ clib_spinlock_t buffer_known_hash_lockp;
+
/* logging */
vlib_log_class_t log_default;
} vlib_buffer_main_t;
clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm);
+format_function_t format_vlib_buffer_pool_all;
+
+int vlib_buffer_set_alloc_free_callback (
+ struct vlib_main_t *vm, vlib_buffer_alloc_free_callback_t *alloc_callback_fn,
+ vlib_buffer_alloc_free_callback_t *free_callback_fn);
+
extern u16 __vlib_buffer_external_hdr_size;
#define VLIB_BUFFER_SET_EXT_HDR_SIZE(x) \
static void __clib_constructor \
diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index 89a765ee0d3..77964fde821 100644
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -626,11 +626,7 @@ vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
src = bpt->cached_buffers + len - n_buffers;
vlib_buffer_copy_indices (dst, src, n_buffers);
bpt->n_cached -= n_buffers;
-
- if (CLIB_DEBUG > 0)
- vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
- VLIB_BUFFER_KNOWN_FREE);
- return n_buffers;
+ goto done;
}
/* alloc bigger than cache - take buffers directly from main pool */
@@ -638,11 +634,7 @@ vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
{
n_buffers = vlib_buffer_pool_get (vm, buffer_pool_index, buffers,
n_buffers);
-
- if (CLIB_DEBUG > 0)
- vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
- VLIB_BUFFER_KNOWN_FREE);
- return n_buffers;
+ goto done;
}
/* take everything available in the cache */
@@ -670,11 +662,13 @@ vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
n_buffers -= n_left;
+done:
/* Verify that buffers are known free. */
if (CLIB_DEBUG > 0)
vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
VLIB_BUFFER_KNOWN_FREE);
-
+ if (PREDICT_FALSE (bm->alloc_callback_fn != 0))
+ bm->alloc_callback_fn (vm, buffer_pool_index, buffers, n_buffers);
return n_buffers;
}
@@ -776,6 +770,7 @@ static_always_inline void
vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index,
u32 * buffers, u32 n_buffers)
{
+ vlib_buffer_main_t *bm = vm->buffer_main;
vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
vlib_buffer_pool_thread_t *bpt = vec_elt_at_index (bp->threads,
vm->thread_index);
@@ -784,6 +779,8 @@ vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index,
if (CLIB_DEBUG > 0)
vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
VLIB_BUFFER_KNOWN_ALLOCATED);
+ if (PREDICT_FALSE (bm->free_callback_fn != 0))
+ bm->free_callback_fn (vm, buffer_pool_index, buffers, n_buffers);
n_cached = bpt->n_cached;
n_empty = VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ - n_cached;