From 7cd468a3d7dee7d6c92f69a0bb7061ae208ec727 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 19 Dec 2016 23:05:39 +0100 Subject: Reorganize source tree to use single autotools instance Change-Id: I7b51f88292e057c6443b12224486f2d0c9f8ae23 Signed-off-by: Damjan Marion --- src/vlib/threads_cli.c | 579 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 579 insertions(+) create mode 100644 src/vlib/threads_cli.c (limited to 'src/vlib/threads_cli.c') diff --git a/src/vlib/threads_cli.c b/src/vlib/threads_cli.c new file mode 100644 index 00000000..ee632279 --- /dev/null +++ b/src/vlib/threads_cli.c @@ -0,0 +1,579 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define _GNU_SOURCE + +#include +#include + +#include +#include + +#if DPDK==1 +#include +#include +#include +#include +#include +#endif + +static u8 * +format_sched_policy_and_priority (u8 * s, va_list * args) +{ + long i = va_arg (*args, long); + struct sched_param sched_param; + u8 *t = 0; + + switch (sched_getscheduler (i)) + { +#define _(v,f,str) case SCHED_POLICY_##f: t = (u8 *) str; break; + foreach_sched_policy +#undef _ + } + if (sched_getparam (i, &sched_param) == 0) + return format (s, "%s (%d)", t, sched_param.sched_priority); + else + return format (s, "%s (n/a)", t); +} + +static clib_error_t * +show_threads_fn (vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) +{ + vlib_worker_thread_t *w; + int i; + + vlib_cli_output (vm, "%-7s%-20s%-12s%-8s%-25s%-7s%-7s%-7s%-10s", + "ID", "Name", "Type", "LWP", "Sched Policy (Priority)", + "lcore", "Core", "Socket", "State"); + +#if !defined(__powerpc64__) + for (i = 0; i < vec_len (vlib_worker_threads); i++) + { + w = vlib_worker_threads + i; + u8 *line = NULL; + + line = format (line, "%-7d%-20s%-12s%-8d", + i, + w->name ? w->name : (u8 *) "", + w->registration ? w->registration->name : "", w->lwp); + + line = format (line, "%-25U", format_sched_policy_and_priority, w->lwp); + + int lcore = -1; + cpu_set_t cpuset; + CPU_ZERO (&cpuset); + int ret = -1; + + ret = + pthread_getaffinity_np (w->thread_id, sizeof (cpu_set_t), &cpuset); + if (!ret) + { + int c; + for (c = 0; c < CPU_SETSIZE; c++) + if (CPU_ISSET (c, &cpuset)) + { + if (lcore > -1) + { + lcore = -2; + break; + } + lcore = c; + } + } + else + { + lcore = w->lcore_id; + } + + if (lcore > -1) + { + const char *sys_cpu_path = "/sys/devices/system/cpu/cpu"; + int socket_id = -1; + int core_id = -1; + u8 *p = 0; + + p = format (p, "%s%u/topology/core_id%c", sys_cpu_path, lcore, 0); + vlib_sysfs_read ((char *) p, "%d", &core_id); + + vec_reset_length (p); + p = + format (p, + "%s%u/topology/physical_package_id%c", + sys_cpu_path, lcore, 0); + vlib_sysfs_read ((char *) p, "%d", &socket_id); + vec_free (p); + + line = format (line, "%-7u%-7u%-7u%", lcore, core_id, socket_id); +#if DPDK==1 + ASSERT (lcore <= RTE_MAX_LCORE); + switch (lcore_config[lcore].state) + { + case WAIT: + line = format (line, "wait"); + break; + case RUNNING: + line = format (line, "running"); + break; + case FINISHED: + line = format (line, "finished"); + break; + default: + line = format (line, "unknown"); + } +#endif + } + else + { + line = + format (line, "%-7s%-7s%-7s%", (lcore == -2) ? "M" : "n/a", "n/a", + "n/a"); + } + + vlib_cli_output (vm, "%v", line); + vec_free (line); + } +#endif + + return 0; +} + + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (show_threads_command, static) = { + .path = "show threads", + .short_help = "Show threads", + .function = show_threads_fn, +}; +/* *INDENT-ON* */ + +/* + * Trigger threads to grab frame queue trace data + */ +static clib_error_t * +trace_frame_queue (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + clib_error_t *error = NULL; + frame_queue_trace_t *fqt; + frame_queue_nelt_counter_t *fqh; + vlib_thread_main_t *tm = vlib_get_thread_main (); + vlib_frame_queue_main_t *fqm; + u32 num_fq; + u32 fqix; + u32 enable = 2; + u32 index = ~(u32) 0; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "on")) + enable = 1; + else if (unformat (line_input, "off")) + enable = 0; + else if (unformat (line_input, "index %u"), &index) + ; + else + return clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + } + + unformat_free (line_input); + + if (enable > 1) + return clib_error_return (0, "expecting on or off"); + + if (vec_len (tm->frame_queue_mains) == 0) + return clib_error_return (0, "no worker handoffs exist"); + + if (index > vec_len (tm->frame_queue_mains) - 1) + return clib_error_return (0, + "expecting valid worker handoff queue index"); + + fqm = vec_elt_at_index (tm->frame_queue_mains, index); + + num_fq = vec_len (fqm->vlib_frame_queues); + if (num_fq == 0) + { + vlib_cli_output (vm, "No frame queues exist\n"); + return error; + } + + // Allocate storage for trace if necessary + vec_validate_aligned (fqm->frame_queue_traces, num_fq - 1, + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (fqm->frame_queue_histogram, num_fq - 1, + CLIB_CACHE_LINE_BYTES); + + for (fqix = 0; fqix < num_fq; fqix++) + { + fqt = &fqm->frame_queue_traces[fqix]; + fqh = &fqm->frame_queue_histogram[fqix]; + + memset (fqt->n_vectors, 0xff, sizeof (fqt->n_vectors)); + fqt->written = 0; + memset (fqh, 0, sizeof (*fqh)); + fqm->vlib_frame_queues[fqix]->trace = enable; + } + return error; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (cmd_trace_frame_queue,static) = { + .path = "trace frame-queue", + .short_help = "trace frame-queue (on|off)", + .function = trace_frame_queue, + .is_mp_safe = 1, +}; +/* *INDENT-ON* */ + + +/* + * Adding two counters and compute percent of total + * Round up, e.g. 0.000001 => 1% + */ +static u32 +compute_percent (u64 * two_counters, u64 total) +{ + if (total == 0) + { + return 0; + } + else + { + return (((two_counters[0] + two_counters[1]) * 100) + + (total - 1)) / total; + } +} + +/* + * Display frame queue trace data gathered by threads. + */ +static clib_error_t * +show_frame_queue_internal (vlib_main_t * vm, + vlib_frame_queue_main_t * fqm, u32 histogram) +{ + clib_error_t *error = NULL; + frame_queue_trace_t *fqt; + frame_queue_nelt_counter_t *fqh; + u32 num_fq; + u32 fqix; + + num_fq = vec_len (fqm->frame_queue_traces); + if (num_fq == 0) + { + vlib_cli_output (vm, "No trace data for frame queues\n"); + return error; + } + + if (histogram) + { + vlib_cli_output (vm, "0-1 2-3 4-5 6-7 8-9 10-11 12-13 14-15 " + "16-17 18-19 20-21 22-23 24-25 26-27 28-29 30-31\n"); + } + + for (fqix = 0; fqix < num_fq; fqix++) + { + fqt = &(fqm->frame_queue_traces[fqix]); + + vlib_cli_output (vm, "Thread %d %v\n", fqix, + vlib_worker_threads[fqix].name); + + if (fqt->written == 0) + { + vlib_cli_output (vm, " no trace data\n"); + continue; + } + + if (histogram) + { + fqh = &(fqm->frame_queue_histogram[fqix]); + u32 nelt; + u64 total = 0; + + for (nelt = 0; nelt < FRAME_QUEUE_MAX_NELTS; nelt++) + { + total += fqh->count[nelt]; + } + + /* + * Print in pairs to condense the output. + * Allow entries with 0 counts to be clearly identified, by rounding up. + * Any non-zero value will be displayed as at least one percent. This + * also means the sum of percentages can be > 100, but that is fine. The + * histogram is counted from the last time "trace frame on" was issued. + */ + vlib_cli_output (vm, + "%3d%% %3d%% %3d%% %3d%% %3d%% %3d%% %3d%% %3d%% " + "%3d%% %3d%% %3d%% %3d%% %3d%% %3d%% %3d%% %3d%%\n", + compute_percent (&fqh->count[0], total), + compute_percent (&fqh->count[2], total), + compute_percent (&fqh->count[4], total), + compute_percent (&fqh->count[6], total), + compute_percent (&fqh->count[8], total), + compute_percent (&fqh->count[10], total), + compute_percent (&fqh->count[12], total), + compute_percent (&fqh->count[14], total), + compute_percent (&fqh->count[16], total), + compute_percent (&fqh->count[18], total), + compute_percent (&fqh->count[20], total), + compute_percent (&fqh->count[22], total), + compute_percent (&fqh->count[24], total), + compute_percent (&fqh->count[26], total), + compute_percent (&fqh->count[28], total), + compute_percent (&fqh->count[30], total)); + } + else + { + vlib_cli_output (vm, + " vector-threshold %d ring size %d in use %d\n", + fqt->threshold, fqt->nelts, fqt->n_in_use); + vlib_cli_output (vm, " head %12d head_hint %12d tail %12d\n", + fqt->head, fqt->head_hint, fqt->tail); + vlib_cli_output (vm, + " %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d\n", + fqt->n_vectors[0], fqt->n_vectors[1], + fqt->n_vectors[2], fqt->n_vectors[3], + fqt->n_vectors[4], fqt->n_vectors[5], + fqt->n_vectors[6], fqt->n_vectors[7], + fqt->n_vectors[8], fqt->n_vectors[9], + fqt->n_vectors[10], fqt->n_vectors[11], + fqt->n_vectors[12], fqt->n_vectors[13], + fqt->n_vectors[14], fqt->n_vectors[15]); + + if (fqt->nelts > 16) + { + vlib_cli_output (vm, + " %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d\n", + fqt->n_vectors[16], fqt->n_vectors[17], + fqt->n_vectors[18], fqt->n_vectors[19], + fqt->n_vectors[20], fqt->n_vectors[21], + fqt->n_vectors[22], fqt->n_vectors[23], + fqt->n_vectors[24], fqt->n_vectors[25], + fqt->n_vectors[26], fqt->n_vectors[27], + fqt->n_vectors[28], fqt->n_vectors[29], + fqt->n_vectors[30], fqt->n_vectors[31]); + } + } + + } + return error; +} + +static clib_error_t * +show_frame_queue_trace (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + vlib_thread_main_t *tm = vlib_get_thread_main (); + vlib_frame_queue_main_t *fqm; + clib_error_t *error; + + vec_foreach (fqm, tm->frame_queue_mains) + { + vlib_cli_output (vm, "Worker handoff queue index %u (next node '%U'):", + fqm - tm->frame_queue_mains, + format_vlib_node_name, vm, fqm->node_index); + error = show_frame_queue_internal (vm, fqm, 0); + if (error) + return error; + } + return 0; +} + +static clib_error_t * +show_frame_queue_histogram (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + vlib_thread_main_t *tm = vlib_get_thread_main (); + vlib_frame_queue_main_t *fqm; + clib_error_t *error; + + vec_foreach (fqm, tm->frame_queue_mains) + { + vlib_cli_output (vm, "Worker handoff queue index %u (next node '%U'):", + fqm - tm->frame_queue_mains, + format_vlib_node_name, vm, fqm->node_index); + error = show_frame_queue_internal (vm, fqm, 1); + if (error) + return error; + } + return 0; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (cmd_show_frame_queue_trace,static) = { + .path = "show frame-queue", + .short_help = "show frame-queue trace", + .function = show_frame_queue_trace, +}; +/* *INDENT-ON* */ + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (cmd_show_frame_queue_histogram,static) = { + .path = "show frame-queue histogram", + .short_help = "show frame-queue histogram", + .function = show_frame_queue_histogram, +}; +/* *INDENT-ON* */ + + +/* + * Modify the number of elements on the frame_queues + */ +static clib_error_t * +test_frame_queue_nelts (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + vlib_thread_main_t *tm = vlib_get_thread_main (); + vlib_frame_queue_main_t *fqm; + clib_error_t *error = NULL; + u32 num_fq; + u32 fqix; + u32 nelts = 0; + u32 index = ~(u32) 0; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "nelts %u", &nelts)) + ; + else if (unformat (line_input, "index %u", &index)) + ; + else + return clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + } + + unformat_free (line_input); + + if (index > vec_len (tm->frame_queue_mains) - 1) + return clib_error_return (0, + "expecting valid worker handoff queue index"); + + fqm = vec_elt_at_index (tm->frame_queue_mains, index); + + if ((nelts != 4) && (nelts != 8) && (nelts != 16) && (nelts != 32)) + { + return clib_error_return (0, "expecting 4,8,16,32"); + } + + num_fq = vec_len (fqm->vlib_frame_queues); + if (num_fq == 0) + { + vlib_cli_output (vm, "No frame queues exist\n"); + return error; + } + + for (fqix = 0; fqix < num_fq; fqix++) + { + fqm->vlib_frame_queues[fqix]->nelts = nelts; + } + + return error; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (cmd_test_frame_queue_nelts,static) = { + .path = "test frame-queue nelts", + .short_help = "test frame-queue nelts (4,8,16,32)", + .function = test_frame_queue_nelts, +}; +/* *INDENT-ON* */ + + +/* + * Modify the max number of packets pulled off the frame queues + */ +static clib_error_t * +test_frame_queue_threshold (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + vlib_thread_main_t *tm = vlib_get_thread_main (); + vlib_frame_queue_main_t *fqm; + clib_error_t *error = NULL; + u32 num_fq; + u32 fqix; + u32 threshold = ~(u32) 0; + u32 index = ~(u32) 0; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "threshold %u", &threshold)) + ; + else if (unformat (line_input, "index %u", &index)) + ; + else + return clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + } + + unformat_free (line_input); + + if (index > vec_len (tm->frame_queue_mains) - 1) + return clib_error_return (0, + "expecting valid worker handoff queue index"); + + fqm = vec_elt_at_index (tm->frame_queue_mains, index); + + + if (threshold == ~(u32) 0) + { + vlib_cli_output (vm, "expecting threshold value\n"); + return error; + } + + if (threshold == 0) + threshold = ~0; + + num_fq = vec_len (fqm->vlib_frame_queues); + if (num_fq == 0) + { + vlib_cli_output (vm, "No frame queues exist\n"); + return error; + } + + for (fqix = 0; fqix < num_fq; fqix++) + { + fqm->vlib_frame_queues[fqix]->vector_threshold = threshold; + } + + return error; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (cmd_test_frame_queue_threshold,static) = { + .path = "test frame-queue threshold", + .short_help = "test frame-queue threshold N (0=no limit)", + .function = test_frame_queue_threshold, +}; +/* *INDENT-ON* */ + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ -- cgit 1.2.3-korg From 878c609889dcdc58538d40d8b3f662320f88573d Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Wed, 4 Jan 2017 13:19:27 +0100 Subject: vlib: add buffer and thread callbacks Change-Id: I8e2e8f94a884ab2f9909d0c83ba00edd38cdab77 Signed-off-by: Damjan Marion --- src/plugins/flowperpkt/flowperpkt.c | 2 +- src/vlib.am | 1 + src/vlib/buffer.c | 736 +++--------------------------------- src/vlib/buffer.h | 45 ++- src/vlib/buffer_funcs.h | 82 ++-- src/vlib/buffer_serialize.c | 248 ++++++++++++ src/vlib/main.c | 7 +- src/vlib/threads.c | 112 +++--- src/vlib/threads.h | 17 +- src/vlib/threads_cli.c | 25 -- src/vlib/unix/physmem.c | 15 +- src/vnet.am | 2 + src/vnet/devices/dpdk/buffer.c | 729 +++++++++++++++++++++++++++++++++++ src/vnet/devices/dpdk/cli.c | 4 +- src/vnet/devices/dpdk/device.c | 7 +- src/vnet/devices/dpdk/dpdk.h | 3 + src/vnet/devices/dpdk/dpdk_priv.h | 3 + src/vnet/devices/dpdk/init.c | 6 +- src/vnet/devices/dpdk/thread.c | 85 +++++ src/vnet/sr/sr_replicate.c | 7 +- 20 files changed, 1304 insertions(+), 832 deletions(-) create mode 100644 src/vlib/buffer_serialize.c create mode 100644 src/vnet/devices/dpdk/buffer.c create mode 100644 src/vnet/devices/dpdk/thread.c (limited to 'src/vlib/threads_cli.c') diff --git a/src/plugins/flowperpkt/flowperpkt.c b/src/plugins/flowperpkt/flowperpkt.c index fb71d5b0..cc351599 100644 --- a/src/plugins/flowperpkt/flowperpkt.c +++ b/src/plugins/flowperpkt/flowperpkt.c @@ -643,7 +643,7 @@ flowperpkt_init (vlib_main_t * vm) vec_free (name); /* Decide how many worker threads we have */ - num_threads = 1 /* main thread */ + tm->n_eal_threads; + num_threads = 1 /* main thread */ + tm->n_threads; /* Allocate per worker thread vectors */ vec_validate (fm->ipv4_buffers_per_worker, num_threads - 1); diff --git a/src/vlib.am b/src/vlib.am index 0154d841..c21f88c4 100644 --- a/src/vlib.am +++ b/src/vlib.am @@ -23,6 +23,7 @@ vlib/config.h: libvlib_la_SOURCES = \ vlib/buffer.c \ + vlib/buffer_serialize.c \ vlib/cli.c \ vlib/cli.h \ vlib/config.h \ diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index 4bf6d125..0b0e6054 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -38,50 +38,13 @@ */ /** - * @cond (!DPDK) * @file * * Allocate/free network buffers. */ -#if DPDK > 0 -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#endif - #include -#if DPDK > 0 -#pragma weak rte_mem_virt2phy -#pragma weak rte_eal_has_hugepages -#pragma weak rte_socket_id -#pragma weak rte_pktmbuf_pool_create -#endif - uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, vlib_buffer_t * b_first) @@ -103,7 +66,6 @@ u8 * format_vlib_buffer (u8 * s, va_list * args) { vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); -#if DPDK > 0 uword indent = format_get_indent (s); s = format (s, "current data %d, length %d, free-list %d", @@ -126,18 +88,6 @@ format_vlib_buffer (u8 * s, va_list * args) format_white_space, indent, next_buffer, b->current_length); } -#else - - s = format (s, "current data %d, length %d, free-list %d", - b->current_data, b->current_length, b->free_list_index); - - if (b->flags & VLIB_BUFFER_IS_TRACED) - s = format (s, ", trace 0x%x", b->trace_index); - - if (b->flags & VLIB_BUFFER_NEXT_PRESENT) - s = format (s, ", next-buffer 0x%x", b->next_buffer); -#endif - return s; } @@ -153,7 +103,6 @@ format_vlib_buffer_and_data (u8 * s, va_list * args) return s; } -#if DPDK == 0 static u8 * format_vlib_buffer_known_state (u8 * s, va_list * args) { @@ -181,7 +130,6 @@ format_vlib_buffer_known_state (u8 * s, va_list * args) return format (s, "%s", t); } -#endif u8 * format_vlib_buffer_contents (u8 * s, va_list * va) @@ -200,7 +148,6 @@ format_vlib_buffer_contents (u8 * s, va_list * va) return s; } -#if DPDK == 0 static u8 * vlib_validate_buffer_helper (vlib_main_t * vm, u32 bi, @@ -217,11 +164,10 @@ vlib_validate_buffer_helper (vlib_main_t * vm, if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE) return format (0, "current data %d before pre-data", b->current_data); -#if DPDK == 0 + if (b->current_data + b->current_length > fl->n_data_bytes) return format (0, "%d-%d beyond end of buffer %d", b->current_data, b->current_length, fl->n_data_bytes); -#endif if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT)) { @@ -311,14 +257,12 @@ done: hash_free (hash); return result; } -#endif vlib_main_t **vlib_mains; -#if DPDK == 0 /* When dubugging validate that given buffers are either known allocated or known free. */ -static void +static void __attribute__ ((unused)) vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers, uword n_buffers, @@ -359,7 +303,6 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm, is_free ? VLIB_BUFFER_KNOWN_FREE : VLIB_BUFFER_KNOWN_ALLOCATED); } } -#endif #define BUFFERS_PER_COPY (sizeof (vlib_copy_unit_t) / sizeof (u32)) @@ -463,7 +406,6 @@ vlib_buffer_create_free_list_helper (vlib_main_t * vm, { vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_free_list_t *f; -#if DPDK > 0 int i; ASSERT (os_get_cpu_number () == 0); @@ -519,47 +461,6 @@ vlib_buffer_create_free_list_helper (vlib_main_t * vm, wf->unaligned_buffers = 0; wf->n_alloc = 0; } -#else - - if (!is_default && pool_elts (bm->buffer_free_list_pool) == 0) - { - u32 default_free_free_list_index; - - default_free_free_list_index = vlib_buffer_create_free_list_helper (vm, - /* default buffer size */ - VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES, - /* is_public */ - 1, - /* is_default */ - 1, - (u8 - *) - "default"); - ASSERT (default_free_free_list_index == - VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); - - if (n_data_bytes == VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES && is_public) - return default_free_free_list_index; - } - - pool_get_aligned (bm->buffer_free_list_pool, f, CLIB_CACHE_LINE_BYTES); - - memset (f, 0, sizeof (f[0])); - f->index = f - bm->buffer_free_list_pool; - f->n_data_bytes = vlib_buffer_round_size (n_data_bytes); - f->min_n_buffers_each_physmem_alloc = 256; - f->name = clib_mem_is_heap_object (name) ? name : format (0, "%s", name); - - /* Setup free buffer template. */ - f->buffer_init_template.free_list_index = f->index; - - if (is_public) - { - uword *p = hash_get (bm->free_list_by_size, f->n_data_bytes); - if (!p) - hash_set (bm->free_list_by_size, f->n_data_bytes, f->index); - } -#endif return f->index; } @@ -609,50 +510,30 @@ static void del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) { u32 i; -#if DPDK > 0 - struct rte_mbuf *mb; - vlib_buffer_t *b; - - for (i = 0; i < vec_len (f->unaligned_buffers); i++) - { - b = vlib_get_buffer (vm, f->unaligned_buffers[i]); - mb = rte_mbuf_from_vlib_buffer (b); - ASSERT (rte_mbuf_refcnt_read (mb) == 1); - rte_pktmbuf_free (mb); - } - for (i = 0; i < vec_len (f->aligned_buffers); i++) - { - b = vlib_get_buffer (vm, f->aligned_buffers[i]); - mb = rte_mbuf_from_vlib_buffer (b); - ASSERT (rte_mbuf_refcnt_read (mb) == 1); - rte_pktmbuf_free (mb); - } - vec_free (f->name); -#else for (i = 0; i < vec_len (f->buffer_memory_allocated); i++) vm->os_physmem_free (f->buffer_memory_allocated[i]); vec_free (f->name); vec_free (f->buffer_memory_allocated); -#endif vec_free (f->unaligned_buffers); vec_free (f->aligned_buffers); } /* Add buffer free list. */ void -vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index) +vlib_buffer_delete_free_list_internal (vlib_main_t * vm, u32 free_list_index) { vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_free_list_t *f; u32 merge_index; -#if DPDK > 0 int i; ASSERT (os_get_cpu_number () == 0); f = vlib_buffer_get_free_list (vm, free_list_index); + ASSERT (vec_len (f->unaligned_buffers) + vec_len (f->aligned_buffers) == + f->n_alloc); merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes); if (merge_index != ~0 && merge_index != free_list_index) { @@ -674,26 +555,6 @@ vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index) memset (f, 0xab, sizeof (f[0])); pool_put (bm->buffer_free_list_pool, f); } -#else - - f = vlib_buffer_get_free_list (vm, free_list_index); - - ASSERT (vec_len (f->unaligned_buffers) + vec_len (f->aligned_buffers) == - f->n_alloc); - merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes); - if (merge_index != ~0 && merge_index != free_list_index) - { - merge_free_lists (pool_elt_at_index (bm->buffer_free_list_pool, - merge_index), f); - } - - del_free_list (vm, f); - - /* Poison it. */ - memset (f, 0xab, sizeof (f[0])); - - pool_put (bm->buffer_free_list_pool, f); -#endif } /* Make sure free list has at least given number of free buffers. */ @@ -701,63 +562,6 @@ static uword fill_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * fl, uword min_free_buffers) { -#if DPDK > 0 - vlib_buffer_t *b; - int n, i; - u32 bi; - u32 n_remaining = 0, n_alloc = 0; - unsigned socket_id = rte_socket_id ? rte_socket_id () : 0; - struct rte_mempool *rmp = vm->buffer_main->pktmbuf_pools[socket_id]; - struct rte_mbuf *mb; - - /* Too early? */ - if (PREDICT_FALSE (rmp == 0)) - return 0; - - trim_aligned (fl); - - /* Already have enough free buffers on free list? */ - n = min_free_buffers - vec_len (fl->aligned_buffers); - if (n <= 0) - return min_free_buffers; - - /* Always allocate round number of buffers. */ - n = round_pow2 (n, BUFFERS_PER_COPY); - - /* Always allocate new buffers in reasonably large sized chunks. */ - n = clib_max (n, fl->min_n_buffers_each_physmem_alloc); - - vec_validate (vm->mbuf_alloc_list, n - 1); - - if (rte_mempool_get_bulk (rmp, vm->mbuf_alloc_list, n) < 0) - return 0; - - _vec_len (vm->mbuf_alloc_list) = n; - - for (i = 0; i < n; i++) - { - mb = vm->mbuf_alloc_list[i]; - - ASSERT (rte_mbuf_refcnt_read (mb) == 0); - rte_mbuf_refcnt_set (mb, 1); - - b = vlib_buffer_from_rte_mbuf (mb); - bi = vlib_get_buffer_index (vm, b); - - vec_add1_aligned (fl->aligned_buffers, bi, sizeof (vlib_copy_unit_t)); - n_alloc++; - n_remaining--; - - vlib_buffer_init_for_free_list (b, fl); - - if (fl->buffer_init_function) - fl->buffer_init_function (vm, fl, &bi, 1); - } - - fl->n_alloc += n; - - return n; -#else vlib_buffer_t *buffers, *b; int n, n_bytes, i; u32 *bi; @@ -824,7 +628,6 @@ fill_free_list (vlib_main_t * vm, fl->buffer_init_function (vm, fl, bi, n_this_chunk); } return n_alloc; -#endif } always_inline uword @@ -833,6 +636,7 @@ copy_alignment (u32 * x) return (pointer_to_uword (x) / sizeof (x[0])) % BUFFERS_PER_COPY; } + static u32 alloc_from_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * free_list, @@ -842,10 +646,6 @@ alloc_from_free_list (vlib_main_t * vm, uword u_len, n_left; uword n_unaligned_start, n_unaligned_end, n_filled; -#if DPDK == 0 - ASSERT (os_get_cpu_number () == 0); - -#endif n_left = n_alloc_buffers; dst = alloc_buffers; n_unaligned_start = ((BUFFERS_PER_COPY - copy_alignment (dst)) @@ -945,25 +745,21 @@ alloc_from_free_list (vlib_main_t * vm, else _vec_len (free_list->unaligned_buffers) = u_len; -#if DPDK == 0 /* Verify that buffers are known free. */ vlib_buffer_validate_alloc_free (vm, alloc_buffers, n_alloc_buffers, VLIB_BUFFER_KNOWN_FREE); -#endif return n_alloc_buffers; } + /* Allocate a given number of buffers into given array. Returns number actually allocated which will be either zero or number requested. */ -u32 -vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +static u32 +vlib_buffer_alloc_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers) { vlib_buffer_main_t *bm = vm->buffer_main; -#if DPDK == 0 - ASSERT (os_get_cpu_number () == 0); -#endif return alloc_from_free_list (vm, @@ -972,10 +768,10 @@ vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) buffers, n_buffers); } -u32 -vlib_buffer_alloc_from_free_list (vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, u32 free_list_index) +static u32 +vlib_buffer_alloc_from_free_list_internal (vlib_main_t * vm, + u32 * buffers, + u32 n_buffers, u32 free_list_index) { vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_free_list_t *f; @@ -1016,81 +812,10 @@ vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp) return rv; } -#if DPDK == 0 -void vnet_buffer_free_dpdk_mb (vlib_buffer_t * b) __attribute__ ((weak)); -void -vnet_buffer_free_dpdk_mb (vlib_buffer_t * b) -{ -} - -#endif static_always_inline void vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, u32 follow_buffer_next) { -#if DPDK > 0 - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *fl; - u32 fi; - int i; - u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers, - u32 follow_buffer_next); - - cb = bm->buffer_free_callback; - - if (PREDICT_FALSE (cb != 0)) - n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next); - - if (!n_buffers) - return; - - for (i = 0; i < n_buffers; i++) - { - vlib_buffer_t *b; - struct rte_mbuf *mb; - - b = vlib_get_buffer (vm, buffers[i]); - - fl = buffer_get_free_list (vm, b, &fi); - - /* The only current use of this callback: multicast recycle */ - if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0)) - { - int j; - - add_buffer_to_free_list - (vm, fl, buffers[i], (b->flags & VLIB_BUFFER_RECYCLE) == 0); - - for (j = 0; j < vec_len (bm->announce_list); j++) - { - if (fl == bm->announce_list[j]) - goto already_announced; - } - vec_add1 (bm->announce_list, fl); - already_announced: - ; - } - else - { - if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0)) - { - mb = rte_mbuf_from_vlib_buffer (b); - ASSERT (rte_mbuf_refcnt_read (mb) == 1); - rte_pktmbuf_free (mb); - } - } - } - if (vec_len (bm->announce_list)) - { - vlib_buffer_free_list_t *fl; - for (i = 0; i < vec_len (bm->announce_list); i++) - { - fl = bm->announce_list[i]; - fl->buffers_added_to_freelist_function (vm, fl); - } - _vec_len (bm->announce_list) = 0; - } -#else vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_free_list_t *fl; static u32 *next_to_free[2]; /* smp bad */ @@ -1315,26 +1040,25 @@ again: } _vec_len (announce_list) = 0; } -#endif } -void -vlib_buffer_free (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +static void +vlib_buffer_free_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers) { vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ 1); } -void -vlib_buffer_free_no_next (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +static void +vlib_buffer_free_no_next_internal (vlib_main_t * vm, u32 * buffers, + u32 n_buffers) { vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ 0); } -#if DPDK == 0 /* Copy template packet data into buffers as they are allocated. */ -static void +static void __attribute__ ((unused)) vlib_packet_template_buffer_init (vlib_main_t * vm, vlib_buffer_free_list_t * fl, u32 * buffers, u32 n_buffers) @@ -1352,7 +1076,6 @@ vlib_packet_template_buffer_init (vlib_main_t * vm, b->current_length); } } -#endif void vlib_packet_template_init (vlib_main_t * vm, @@ -1362,28 +1085,22 @@ vlib_packet_template_init (vlib_main_t * vm, uword min_n_buffers_each_physmem_alloc, char *fmt, ...) { -#if DPDK > 0 + vlib_buffer_main_t *bm = vm->buffer_main; va_list va; __attribute__ ((unused)) u8 *name; + vlib_buffer_free_list_t *fl; va_start (va, fmt); name = va_format (0, fmt, &va); va_end (va); - vlib_worker_thread_barrier_sync (vm); - memset (t, 0, sizeof (t[0])); - - vec_add (t->packet_data, packet_data, n_packet_data_bytes); + if (bm->cb.vlib_packet_template_init_cb) + bm->cb.vlib_packet_template_init_cb (vm, (void *) t, packet_data, + n_packet_data_bytes, + min_n_buffers_each_physmem_alloc, + name); - vlib_worker_thread_barrier_release (vm); -#else - vlib_buffer_free_list_t *fl; - va_list va; - u8 *name; - - va_start (va, fmt); - name = va_format (0, fmt, &va); - va_end (va); + vlib_worker_thread_barrier_sync (vm); memset (t, 0, sizeof (t[0])); @@ -1406,7 +1123,7 @@ vlib_packet_template_init (vlib_main_t * vm, fl->buffer_init_template.current_data = 0; fl->buffer_init_template.current_length = n_packet_data_bytes; fl->buffer_init_template.flags = 0; -#endif + vlib_worker_thread_barrier_release (vm); } void * @@ -1429,7 +1146,6 @@ vlib_packet_template_get_packet (vlib_main_t * vm, return b->data; } -#if DPDK == 0 void vlib_packet_template_get_packet_helper (vlib_main_t * vm, vlib_packet_template_t * t) @@ -1447,7 +1163,6 @@ vlib_packet_template_get_packet_helper (vlib_main_t * vm, _vec_len (t->free_buffers) = n_alloc; } -#endif /* Append given data to end of buffer, possibly allocating new buffers. */ u32 vlib_buffer_add_data (vlib_main_t * vm, @@ -1541,328 +1256,11 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, return copied; } -#if DPDK > 0 -clib_error_t * -vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, - unsigned socket_id) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_physmem_main_t *vpm = &vm->physmem_main; - struct rte_mempool *rmp; - int i; - - if (!rte_pktmbuf_pool_create) - return clib_error_return (0, "not linked with DPDK"); - - vec_validate_aligned (bm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES); - - /* pool already exists, nothing to do */ - if (bm->pktmbuf_pools[socket_id]) - return 0; - - u8 *pool_name = format (0, "mbuf_pool_socket%u%c", socket_id, 0); - - rmp = rte_pktmbuf_pool_create ((char *) pool_name, /* pool name */ - num_mbufs, /* number of mbufs */ - 512, /* cache size */ - VLIB_BUFFER_HDR_SIZE, /* priv size */ - VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE, /* dataroom size */ - socket_id); /* cpu socket */ - - if (rmp) - { - { - uword this_pool_end; - uword this_pool_start; - uword this_pool_size; - uword save_vpm_start, save_vpm_end, save_vpm_size; - struct rte_mempool_memhdr *memhdr; - - this_pool_start = ~0ULL; - this_pool_end = 0LL; - - STAILQ_FOREACH (memhdr, &rmp->mem_list, next) - { - if (((uword) (memhdr->addr + memhdr->len)) > this_pool_end) - this_pool_end = (uword) (memhdr->addr + memhdr->len); - if (((uword) memhdr->addr) < this_pool_start) - this_pool_start = (uword) (memhdr->addr); - } - ASSERT (this_pool_start < ~0ULL && this_pool_end > 0); - this_pool_size = this_pool_end - this_pool_start; - - if (CLIB_DEBUG > 1) - { - clib_warning ("%s: pool start %llx pool end %llx pool size %lld", - pool_name, this_pool_start, this_pool_end, - this_pool_size); - clib_warning - ("before: virtual.start %llx virtual.end %llx virtual.size %lld", - vpm->virtual.start, vpm->virtual.end, vpm->virtual.size); - } - - save_vpm_start = vpm->virtual.start; - save_vpm_end = vpm->virtual.end; - save_vpm_size = vpm->virtual.size; - - if ((this_pool_start < vpm->virtual.start) || vpm->virtual.start == 0) - vpm->virtual.start = this_pool_start; - if (this_pool_end > vpm->virtual.end) - vpm->virtual.end = this_pool_end; - - vpm->virtual.size = vpm->virtual.end - vpm->virtual.start; - - if (CLIB_DEBUG > 1) - { - clib_warning - ("after: virtual.start %llx virtual.end %llx virtual.size %lld", - vpm->virtual.start, vpm->virtual.end, vpm->virtual.size); - } - - /* check if fits into buffer index range */ - if ((u64) vpm->virtual.size > - ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES))) - { - clib_warning ("physmem: virtual size out of range!"); - vpm->virtual.start = save_vpm_start; - vpm->virtual.end = save_vpm_end; - vpm->virtual.size = save_vpm_size; - rmp = 0; - } - } - if (rmp) - { - bm->pktmbuf_pools[socket_id] = rmp; - vec_free (pool_name); - return 0; - } - } - - vec_free (pool_name); - - /* no usable pool for this socket, try to use pool from another one */ - for (i = 0; i < vec_len (bm->pktmbuf_pools); i++) - { - if (bm->pktmbuf_pools[i]) - { - clib_warning - ("WARNING: Failed to allocate mempool for CPU socket %u. " - "Threads running on socket %u will use socket %u mempool.", - socket_id, socket_id, i); - bm->pktmbuf_pools[socket_id] = bm->pktmbuf_pools[i]; - return 0; - } - } - - return clib_error_return (0, "failed to allocate mempool on socket %u", - socket_id); -} -#endif - -static void -vlib_serialize_tx (serialize_main_header_t * m, serialize_stream_t * s) -{ - vlib_main_t *vm; - vlib_serialize_buffer_main_t *sm; - uword n, n_bytes_to_write; - vlib_buffer_t *last; - - n_bytes_to_write = s->current_buffer_index; - sm = - uword_to_pointer (s->data_function_opaque, - vlib_serialize_buffer_main_t *); - vm = sm->vlib_main; - - ASSERT (sm->tx.max_n_data_bytes_per_chain > 0); - if (serialize_stream_is_end_of_stream (s) - || sm->tx.n_total_data_bytes + n_bytes_to_write > - sm->tx.max_n_data_bytes_per_chain) - { - vlib_process_t *p = vlib_get_current_process (vm); - - last = vlib_get_buffer (vm, sm->last_buffer); - last->current_length = n_bytes_to_write; - - vlib_set_next_frame_buffer (vm, &p->node_runtime, sm->tx.next_index, - sm->first_buffer); - - sm->first_buffer = sm->last_buffer = ~0; - sm->tx.n_total_data_bytes = 0; - } - - else if (n_bytes_to_write == 0 && s->n_buffer_bytes == 0) - { - ASSERT (sm->first_buffer == ~0); - ASSERT (sm->last_buffer == ~0); - n = - vlib_buffer_alloc_from_free_list (vm, &sm->first_buffer, 1, - sm->tx.free_list_index); - if (n != 1) - serialize_error (m, - clib_error_create - ("vlib_buffer_alloc_from_free_list fails")); - sm->last_buffer = sm->first_buffer; - s->n_buffer_bytes = - vlib_buffer_free_list_buffer_size (vm, sm->tx.free_list_index); - } - - if (n_bytes_to_write > 0) - { - vlib_buffer_t *prev = vlib_get_buffer (vm, sm->last_buffer); - n = - vlib_buffer_alloc_from_free_list (vm, &sm->last_buffer, 1, - sm->tx.free_list_index); - if (n != 1) - serialize_error (m, - clib_error_create - ("vlib_buffer_alloc_from_free_list fails")); - sm->tx.n_total_data_bytes += n_bytes_to_write; - prev->current_length = n_bytes_to_write; - prev->next_buffer = sm->last_buffer; - prev->flags |= VLIB_BUFFER_NEXT_PRESENT; - } - - if (sm->last_buffer != ~0) - { - last = vlib_get_buffer (vm, sm->last_buffer); - s->buffer = vlib_buffer_get_current (last); - s->current_buffer_index = 0; - ASSERT (last->current_data == s->current_buffer_index); - } -} - -static void -vlib_serialize_rx (serialize_main_header_t * m, serialize_stream_t * s) -{ - vlib_main_t *vm; - vlib_serialize_buffer_main_t *sm; - vlib_buffer_t *last; - - sm = - uword_to_pointer (s->data_function_opaque, - vlib_serialize_buffer_main_t *); - vm = sm->vlib_main; - - if (serialize_stream_is_end_of_stream (s)) - return; - - if (sm->last_buffer != ~0) - { - last = vlib_get_buffer (vm, sm->last_buffer); - - if (last->flags & VLIB_BUFFER_NEXT_PRESENT) - sm->last_buffer = last->next_buffer; - else - { - vlib_buffer_free (vm, &sm->first_buffer, /* count */ 1); - sm->first_buffer = sm->last_buffer = ~0; - } - } - - if (sm->last_buffer == ~0) - { - while (clib_fifo_elts (sm->rx.buffer_fifo) == 0) - { - sm->rx.ready_one_time_event = - vlib_process_create_one_time_event (vm, vlib_current_process (vm), - ~0); - vlib_process_wait_for_one_time_event (vm, /* no event data */ 0, - sm->rx.ready_one_time_event); - } - - clib_fifo_sub1 (sm->rx.buffer_fifo, sm->first_buffer); - sm->last_buffer = sm->first_buffer; - } - - ASSERT (sm->last_buffer != ~0); - - last = vlib_get_buffer (vm, sm->last_buffer); - s->current_buffer_index = 0; - s->buffer = vlib_buffer_get_current (last); - s->n_buffer_bytes = last->current_length; -} - -static void -serialize_open_vlib_helper (serialize_main_t * m, - vlib_main_t * vm, - vlib_serialize_buffer_main_t * sm, uword is_read) -{ - /* Initialize serialize main but save overflow buffer for re-use between calls. */ - { - u8 *save = m->stream.overflow_buffer; - memset (m, 0, sizeof (m[0])); - m->stream.overflow_buffer = save; - if (save) - _vec_len (save) = 0; - } - - sm->first_buffer = sm->last_buffer = ~0; - if (is_read) - clib_fifo_reset (sm->rx.buffer_fifo); - else - sm->tx.n_total_data_bytes = 0; - sm->vlib_main = vm; - m->header.data_function = is_read ? vlib_serialize_rx : vlib_serialize_tx; - m->stream.data_function_opaque = pointer_to_uword (sm); -} - -void -serialize_open_vlib_buffer (serialize_main_t * m, vlib_main_t * vm, - vlib_serialize_buffer_main_t * sm) -{ - serialize_open_vlib_helper (m, vm, sm, /* is_read */ 0); -} - -void -unserialize_open_vlib_buffer (serialize_main_t * m, vlib_main_t * vm, - vlib_serialize_buffer_main_t * sm) -{ - serialize_open_vlib_helper (m, vm, sm, /* is_read */ 1); -} - -u32 -serialize_close_vlib_buffer (serialize_main_t * m) -{ - vlib_serialize_buffer_main_t *sm - = uword_to_pointer (m->stream.data_function_opaque, - vlib_serialize_buffer_main_t *); - vlib_buffer_t *last; - serialize_stream_t *s = &m->stream; - - last = vlib_get_buffer (sm->vlib_main, sm->last_buffer); - last->current_length = s->current_buffer_index; - - if (vec_len (s->overflow_buffer) > 0) - { - sm->last_buffer - = vlib_buffer_add_data (sm->vlib_main, sm->tx.free_list_index, - sm->last_buffer == ~0 ? 0 : sm->last_buffer, - s->overflow_buffer, - vec_len (s->overflow_buffer)); - _vec_len (s->overflow_buffer) = 0; - } - - return sm->first_buffer; -} - -void -unserialize_close_vlib_buffer (serialize_main_t * m) -{ - vlib_serialize_buffer_main_t *sm - = uword_to_pointer (m->stream.data_function_opaque, - vlib_serialize_buffer_main_t *); - if (sm->first_buffer != ~0) - vlib_buffer_free_one (sm->vlib_main, sm->first_buffer); - clib_fifo_reset (sm->rx.buffer_fifo); - if (m->stream.overflow_buffer) - _vec_len (m->stream.overflow_buffer) = 0; -} static u8 * format_vlib_buffer_free_list (u8 * s, va_list * va) { vlib_buffer_free_list_t *f = va_arg (*va, vlib_buffer_free_list_t *); -#if DPDK > 0 u32 threadnum = va_arg (*va, u32); uword bytes_alloc, bytes_free, n_free, size; @@ -1877,21 +1275,6 @@ format_vlib_buffer_free_list (u8 * s, va_list * va) bytes_free = size * n_free; s = format (s, "%7d%30s%12d%12d%=12U%=12U%=12d%=12d", threadnum, -#else - uword bytes_alloc, bytes_free, n_free, size; - - if (!f) - return format (s, "%=30s%=12s%=12s%=12s%=12s%=12s%=12s", - "Name", "Index", "Size", "Alloc", "Free", "#Alloc", - "#Free"); - - size = sizeof (vlib_buffer_t) + f->n_data_bytes; - n_free = vec_len (f->aligned_buffers) + vec_len (f->unaligned_buffers); - bytes_alloc = size * f->n_alloc; - bytes_free = size * n_free; - - s = format (s, "%30s%12d%12d%=12U%=12U%=12d%=12d", -#endif f->name, f->index, f->n_data_bytes, format_memory_size, bytes_alloc, format_memory_size, bytes_free, f->n_alloc, n_free); @@ -1903,7 +1286,6 @@ static clib_error_t * show_buffers (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { -#if DPDK > 0 vlib_buffer_main_t *bm; vlib_buffer_free_list_t *f; vlib_main_t *curr_vm; @@ -1926,18 +1308,6 @@ show_buffers (vlib_main_t * vm, } while (vm_index < vec_len (vlib_mains)); -#else - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *f; - - vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, 0); - /* *INDENT-OFF* */ - pool_foreach (f, bm->buffer_free_list_pool, ({ - vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, f); - })); -/* *INDENT-ON* */ - -#endif return 0; } @@ -1949,34 +1319,38 @@ VLIB_CLI_COMMAND (show_buffers_command, static) = { }; /* *INDENT-ON* */ -#if DPDK > 0 -#if CLIB_DEBUG > 0 - -u32 *vlib_buffer_state_validation_lock; -uword *vlib_buffer_state_validation_hash; -void *vlib_buffer_state_heap; - -static clib_error_t * -buffer_state_validation_init (vlib_main_t * vm) +void +vlib_buffer_cb_init (struct vlib_main_t *vm) { - void *oldheap; - - vlib_buffer_state_heap = mheap_alloc (0, 10 << 20); - - oldheap = clib_mem_set_heap (vlib_buffer_state_heap); + vlib_buffer_main_t *bm = vm->buffer_main; + bm->cb.vlib_buffer_alloc_cb = &vlib_buffer_alloc_internal; + bm->cb.vlib_buffer_alloc_from_free_list_cb = + &vlib_buffer_alloc_from_free_list_internal; + bm->cb.vlib_buffer_free_cb = &vlib_buffer_free_internal; + bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal; + bm->cb.vlib_buffer_delete_free_list_cb = + &vlib_buffer_delete_free_list_internal; + bm->extern_buffer_mgmt = 0; +} - vlib_buffer_state_validation_hash = hash_create (0, sizeof (uword)); - vec_validate_aligned (vlib_buffer_state_validation_lock, 0, - CLIB_CACHE_LINE_BYTES); - clib_mem_set_heap (oldheap); +int +vlib_buffer_cb_register (struct vlib_main_t *vm, vlib_buffer_callbacks_t * cb) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + if (bm->extern_buffer_mgmt) + return -1; + +#define _(x) bm->cb.x = cb->x + _(vlib_buffer_alloc_cb); + _(vlib_buffer_alloc_from_free_list_cb); + _(vlib_buffer_free_cb); + _(vlib_buffer_free_no_next_cb); + _(vlib_buffer_delete_free_list_cb); +#undef _ + bm->extern_buffer_mgmt = 1; return 0; } -VLIB_INIT_FUNCTION (buffer_state_validation_init); -#endif -#endif - - /** @endcond */ /* * fd.io coding-style-patch-verification: ON diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h index 5f1e62f0..d270c08a 100644 --- a/src/vlib/buffer.h +++ b/src/vlib/buffer.h @@ -46,15 +46,9 @@ #include #include /* for vlib_error_t */ -#if DPDK > 0 -#include -#define VLIB_BUFFER_DATA_SIZE (2048) -#define VLIB_BUFFER_PRE_DATA_SIZE RTE_PKTMBUF_HEADROOM -#else #include /* for __PRE_DATA_SIZE */ -#define VLIB_BUFFER_DATA_SIZE (512) +#define VLIB_BUFFER_DATA_SIZE (2048) #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE -#endif #if defined (CLIB_HAVE_VEC128) || defined (__aarch64__) typedef u8x16 vlib_copy_unit_t; @@ -296,6 +290,27 @@ typedef struct vlib_buffer_free_list_t uword buffer_init_function_opaque; } __attribute__ ((aligned (16))) vlib_buffer_free_list_t; +typedef struct +{ + u32 (*vlib_buffer_alloc_cb) (struct vlib_main_t * vm, u32 * buffers, + u32 n_buffers); + u32 (*vlib_buffer_alloc_from_free_list_cb) (struct vlib_main_t * vm, + u32 * buffers, u32 n_buffers, + u32 free_list_index); + void (*vlib_buffer_free_cb) (struct vlib_main_t * vm, u32 * buffers, + u32 n_buffers); + void (*vlib_buffer_free_no_next_cb) (struct vlib_main_t * vm, u32 * buffers, + u32 n_buffers); + void (*vlib_packet_template_init_cb) (struct vlib_main_t * vm, void *t, + void *packet_data, + uword n_packet_data_bytes, + uword + min_n_buffers_each_physmem_alloc, + u8 * name); + void (*vlib_buffer_delete_free_list_cb) (struct vlib_main_t * vm, + u32 free_list_index); +} vlib_buffer_callbacks_t; + typedef struct { /* Buffer free callback, for subversive activities */ @@ -323,12 +338,15 @@ typedef struct /* List of free-lists needing Blue Light Special announcements */ vlib_buffer_free_list_t **announce_list; - /* Vector of rte_mempools per socket */ -#if DPDK == 1 - struct rte_mempool **pktmbuf_pools; -#endif + /* Callbacks */ + vlib_buffer_callbacks_t cb; + int extern_buffer_mgmt; } vlib_buffer_main_t; +void vlib_buffer_cb_init (struct vlib_main_t *vm); +int vlib_buffer_cb_register (struct vlib_main_t *vm, + vlib_buffer_callbacks_t * cb); + typedef struct { struct vlib_main_t *vlib_main; @@ -385,11 +403,6 @@ serialize_vlib_buffer_n_bytes (serialize_main_t * m) vec_len (s->overflow_buffer); } -#if DPDK > 0 -#define rte_mbuf_from_vlib_buffer(x) (((struct rte_mbuf *)x) - 1) -#define vlib_buffer_from_rte_mbuf(x) ((vlib_buffer_t *)(x+1)) -#endif - /* */ diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index 75716eca..15d93c16 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -195,8 +195,6 @@ do { \ } while (0) #endif -#if DPDK == 0 - typedef enum { /* Index is unknown. */ @@ -232,8 +230,6 @@ vlib_buffer_set_known_state (vlib_main_t * vm, u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index, uword follow_chain); -#endif /* DPDK == 0 */ - clib_error_t *vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, unsigned socket_id); @@ -245,7 +241,15 @@ clib_error_t *vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ -u32 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers); +always_inline u32 +vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + + ASSERT (bm->cb.vlib_buffer_alloc_cb); + + return bm->cb.vlib_buffer_alloc_cb (vm, buffers, n_buffers); +} always_inline u32 vlib_buffer_round_size (u32 size) @@ -261,9 +265,18 @@ vlib_buffer_round_size (u32 size) @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ -u32 vlib_buffer_alloc_from_free_list (vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, u32 free_list_index); +always_inline u32 +vlib_buffer_alloc_from_free_list (vlib_main_t * vm, + u32 * buffers, + u32 n_buffers, u32 free_list_index) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + + ASSERT (bm->cb.vlib_buffer_alloc_from_free_list_cb); + + return bm->cb.vlib_buffer_alloc_from_free_list_cb (vm, buffers, n_buffers, + free_list_index); +} /** \brief Free buffers Frees the entire buffer chain for each buffer @@ -273,11 +286,19 @@ u32 vlib_buffer_alloc_from_free_list (vlib_main_t * vm, @param n_buffers - (u32) number of buffers to free */ -void vlib_buffer_free (vlib_main_t * vm, - /* pointer to first buffer */ - u32 * buffers, - /* number of buffers to free */ - u32 n_buffers); +always_inline void +vlib_buffer_free (vlib_main_t * vm, + /* pointer to first buffer */ + u32 * buffers, + /* number of buffers to free */ + u32 n_buffers) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + + ASSERT (bm->cb.vlib_buffer_free_cb); + + return bm->cb.vlib_buffer_free_cb (vm, buffers, n_buffers); +} /** \brief Free buffers, does not free the buffer chain for each buffer @@ -286,11 +307,19 @@ void vlib_buffer_free (vlib_main_t * vm, @param n_buffers - (u32) number of buffers to free */ -void vlib_buffer_free_no_next (vlib_main_t * vm, - /* pointer to first buffer */ - u32 * buffers, - /* number of buffers to free */ - u32 n_buffers); +always_inline void +vlib_buffer_free_no_next (vlib_main_t * vm, + /* pointer to first buffer */ + u32 * buffers, + /* number of buffers to free */ + u32 n_buffers) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + + ASSERT (bm->cb.vlib_buffer_free_no_next_cb); + + return bm->cb.vlib_buffer_free_no_next_cb (vm, buffers, n_buffers); +} /** \brief Free one buffer Shorthand to free a single buffer chain. @@ -307,7 +336,15 @@ vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index) /* Add/delete buffer free lists. */ u32 vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes, char *fmt, ...); -void vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index); +always_inline void +vlib_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + + ASSERT (bm->cb.vlib_buffer_delete_free_list_cb); + + bm->cb.vlib_buffer_delete_free_list_cb (vm, free_list_index); +} /* Find already existing public free list with given size or create one. */ u32 vlib_buffer_get_or_create_free_list (vlib_main_t * vm, u32 n_data_bytes, @@ -453,11 +490,6 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b) return fd; } -/* - * vlib_buffer_chain_* functions provide a way to create long buffers. - * When DPDK is enabled, the 'hidden' DPDK header is taken care of transparently. - */ - /* Initializes the buffer as an empty packet with no chained buffers. */ always_inline void vlib_buffer_chain_init (vlib_buffer_t * first) @@ -537,8 +569,6 @@ typedef struct /* Vector of packet data. */ u8 *packet_data; - /* Note: the next three fields are unused if DPDK == 1 */ - /* Number of buffers to allocate in each call to physmem allocator. */ u32 min_n_buffers_each_physmem_alloc; diff --git a/src/vlib/buffer_serialize.c b/src/vlib/buffer_serialize.c new file mode 100644 index 00000000..96a5f0a0 --- /dev/null +++ b/src/vlib/buffer_serialize.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * buffer.c: allocate/free network buffers. + * + * Copyright (c) 2008 Eliot Dresselhaus + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include + +static void +vlib_serialize_tx (serialize_main_header_t * m, serialize_stream_t * s) +{ + vlib_main_t *vm; + vlib_serialize_buffer_main_t *sm; + uword n, n_bytes_to_write; + vlib_buffer_t *last; + + n_bytes_to_write = s->current_buffer_index; + sm = + uword_to_pointer (s->data_function_opaque, + vlib_serialize_buffer_main_t *); + vm = sm->vlib_main; + + ASSERT (sm->tx.max_n_data_bytes_per_chain > 0); + if (serialize_stream_is_end_of_stream (s) + || sm->tx.n_total_data_bytes + n_bytes_to_write > + sm->tx.max_n_data_bytes_per_chain) + { + vlib_process_t *p = vlib_get_current_process (vm); + + last = vlib_get_buffer (vm, sm->last_buffer); + last->current_length = n_bytes_to_write; + + vlib_set_next_frame_buffer (vm, &p->node_runtime, sm->tx.next_index, + sm->first_buffer); + + sm->first_buffer = sm->last_buffer = ~0; + sm->tx.n_total_data_bytes = 0; + } + + else if (n_bytes_to_write == 0 && s->n_buffer_bytes == 0) + { + ASSERT (sm->first_buffer == ~0); + ASSERT (sm->last_buffer == ~0); + n = + vlib_buffer_alloc_from_free_list (vm, &sm->first_buffer, 1, + sm->tx.free_list_index); + if (n != 1) + serialize_error (m, + clib_error_create + ("vlib_buffer_alloc_from_free_list fails")); + sm->last_buffer = sm->first_buffer; + s->n_buffer_bytes = + vlib_buffer_free_list_buffer_size (vm, sm->tx.free_list_index); + } + + if (n_bytes_to_write > 0) + { + vlib_buffer_t *prev = vlib_get_buffer (vm, sm->last_buffer); + n = + vlib_buffer_alloc_from_free_list (vm, &sm->last_buffer, 1, + sm->tx.free_list_index); + if (n != 1) + serialize_error (m, + clib_error_create + ("vlib_buffer_alloc_from_free_list fails")); + sm->tx.n_total_data_bytes += n_bytes_to_write; + prev->current_length = n_bytes_to_write; + prev->next_buffer = sm->last_buffer; + prev->flags |= VLIB_BUFFER_NEXT_PRESENT; + } + + if (sm->last_buffer != ~0) + { + last = vlib_get_buffer (vm, sm->last_buffer); + s->buffer = vlib_buffer_get_current (last); + s->current_buffer_index = 0; + ASSERT (last->current_data == s->current_buffer_index); + } +} + +static void +vlib_serialize_rx (serialize_main_header_t * m, serialize_stream_t * s) +{ + vlib_main_t *vm; + vlib_serialize_buffer_main_t *sm; + vlib_buffer_t *last; + + sm = + uword_to_pointer (s->data_function_opaque, + vlib_serialize_buffer_main_t *); + vm = sm->vlib_main; + + if (serialize_stream_is_end_of_stream (s)) + return; + + if (sm->last_buffer != ~0) + { + last = vlib_get_buffer (vm, sm->last_buffer); + + if (last->flags & VLIB_BUFFER_NEXT_PRESENT) + sm->last_buffer = last->next_buffer; + else + { + vlib_buffer_free (vm, &sm->first_buffer, /* count */ 1); + sm->first_buffer = sm->last_buffer = ~0; + } + } + + if (sm->last_buffer == ~0) + { + while (clib_fifo_elts (sm->rx.buffer_fifo) == 0) + { + sm->rx.ready_one_time_event = + vlib_process_create_one_time_event (vm, vlib_current_process (vm), + ~0); + vlib_process_wait_for_one_time_event (vm, /* no event data */ 0, + sm->rx.ready_one_time_event); + } + + clib_fifo_sub1 (sm->rx.buffer_fifo, sm->first_buffer); + sm->last_buffer = sm->first_buffer; + } + + ASSERT (sm->last_buffer != ~0); + + last = vlib_get_buffer (vm, sm->last_buffer); + s->current_buffer_index = 0; + s->buffer = vlib_buffer_get_current (last); + s->n_buffer_bytes = last->current_length; +} + +static void +serialize_open_vlib_helper (serialize_main_t * m, + vlib_main_t * vm, + vlib_serialize_buffer_main_t * sm, uword is_read) +{ + /* Initialize serialize main but save overflow buffer for re-use between calls. */ + { + u8 *save = m->stream.overflow_buffer; + memset (m, 0, sizeof (m[0])); + m->stream.overflow_buffer = save; + if (save) + _vec_len (save) = 0; + } + + sm->first_buffer = sm->last_buffer = ~0; + if (is_read) + clib_fifo_reset (sm->rx.buffer_fifo); + else + sm->tx.n_total_data_bytes = 0; + sm->vlib_main = vm; + m->header.data_function = is_read ? vlib_serialize_rx : vlib_serialize_tx; + m->stream.data_function_opaque = pointer_to_uword (sm); +} + +void +serialize_open_vlib_buffer (serialize_main_t * m, vlib_main_t * vm, + vlib_serialize_buffer_main_t * sm) +{ + serialize_open_vlib_helper (m, vm, sm, /* is_read */ 0); +} + +void +unserialize_open_vlib_buffer (serialize_main_t * m, vlib_main_t * vm, + vlib_serialize_buffer_main_t * sm) +{ + serialize_open_vlib_helper (m, vm, sm, /* is_read */ 1); +} + +u32 +serialize_close_vlib_buffer (serialize_main_t * m) +{ + vlib_serialize_buffer_main_t *sm + = uword_to_pointer (m->stream.data_function_opaque, + vlib_serialize_buffer_main_t *); + vlib_buffer_t *last; + serialize_stream_t *s = &m->stream; + + last = vlib_get_buffer (sm->vlib_main, sm->last_buffer); + last->current_length = s->current_buffer_index; + + if (vec_len (s->overflow_buffer) > 0) + { + sm->last_buffer + = vlib_buffer_add_data (sm->vlib_main, sm->tx.free_list_index, + sm->last_buffer == ~0 ? 0 : sm->last_buffer, + s->overflow_buffer, + vec_len (s->overflow_buffer)); + _vec_len (s->overflow_buffer) = 0; + } + + return sm->first_buffer; +} + +void +unserialize_close_vlib_buffer (serialize_main_t * m) +{ + vlib_serialize_buffer_main_t *sm + = uword_to_pointer (m->stream.data_function_opaque, + vlib_serialize_buffer_main_t *); + if (sm->first_buffer != ~0) + vlib_buffer_free_one (sm->vlib_main, sm->first_buffer); + clib_fifo_reset (sm->rx.buffer_fifo); + if (m->stream.overflow_buffer) + _vec_len (m->stream.overflow_buffer) = 0; +} + +/** @endcond */ +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vlib/main.c b/src/vlib/main.c index 6c6cad98..09f34bbd 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -465,7 +465,7 @@ vlib_put_next_frame (vlib_main_t * vm, vlib_frame_t *f; u32 n_vectors_in_frame; - if (DPDK == 0 && CLIB_DEBUG > 0) + if (vm->buffer_main->extern_buffer_mgmt == 0 && CLIB_DEBUG > 0) vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left); nf = vlib_node_runtime_get_next_frame (vm, r, next_index); @@ -1012,8 +1012,8 @@ dispatch_node (vlib_main_t * vm, /* When in interrupt mode and vector rate crosses threshold switch to polling mode. */ - if ((DPDK == 0 && dispatch_state == VLIB_NODE_STATE_INTERRUPT) - || (DPDK == 0 && dispatch_state == VLIB_NODE_STATE_POLLING + if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT) + || (dispatch_state == VLIB_NODE_STATE_POLLING && (node->flags & VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE))) { @@ -1615,6 +1615,7 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) vm->name = "VLIB"; vec_validate (vm->buffer_main, 0); + vlib_buffer_cb_init (vm); if ((error = vlib_thread_init (vm))) { diff --git a/src/vlib/threads.c b/src/vlib/threads.c index c5e58bc0..b3bbd30e 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -22,29 +22,10 @@ #include #include - -#if DPDK==1 -#include -#include -#include -#include -#include -#endif DECLARE_CJ_GLOBAL_LOG; #define FRAME_QUEUE_NELTS 32 - -#if DPDK==1 -/* - * Weak definitions of DPDK symbols used in this file. - * Needed for linking test programs without DPDK libs. - */ -unsigned __thread __attribute__ ((weak)) RTE_PER_LCORE (_lcore_id); -struct lcore_config __attribute__ ((weak)) lcore_config[]; -unsigned __attribute__ ((weak)) rte_socket_id (); -int __attribute__ ((weak)) rte_eal_remote_launch (); -#endif u32 vl (void *p) { @@ -194,14 +175,17 @@ vlib_thread_init (vlib_main_t * vm) tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1); /* pin main thread to main_lcore */ -#if DPDK==0 - { - cpu_set_t cpuset; - CPU_ZERO (&cpuset); - CPU_SET (tm->main_lcore, &cpuset); - pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset); - } -#endif + if (tm->cb.vlib_thread_set_lcore_cb) + { + tm->cb.vlib_thread_set_lcore_cb (0, tm->main_lcore); + } + else + { + cpu_set_t cpuset; + CPU_ZERO (&cpuset); + CPU_SET (tm->main_lcore, &cpuset); + pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset); + } /* as many threads as stacks... */ vec_validate_aligned (vlib_worker_threads, vec_len (vlib_thread_stacks) - 1, @@ -520,32 +504,29 @@ vlib_worker_thread_bootstrap_fn (void *arg) return rv; } -static int -vlib_launch_thread (void *fp, vlib_worker_thread_t * w, unsigned lcore_id) +static clib_error_t * +vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned lcore_id) { + vlib_thread_main_t *tm = &vlib_thread_main; void *(*fp_arg) (void *) = fp; w->lcore_id = lcore_id; -#if DPDK==1 - if (!w->registration->use_pthreads) - if (rte_eal_remote_launch) /* do we have dpdk linked */ - return rte_eal_remote_launch (fp, (void *) w, lcore_id); - else - return -1; + if (tm->cb.vlib_launch_thread_cb && !w->registration->use_pthreads) + return tm->cb.vlib_launch_thread_cb (fp, (void *) w, lcore_id); else -#endif { - int ret; pthread_t worker; cpu_set_t cpuset; CPU_ZERO (&cpuset); CPU_SET (lcore_id, &cpuset); - ret = pthread_create (&worker, NULL /* attr */ , fp_arg, (void *) w); - if (ret == 0) - return pthread_setaffinity_np (worker, sizeof (cpu_set_t), &cpuset); - else - return ret; + if (pthread_create (&worker, NULL /* attr */ , fp_arg, (void *) w)) + return clib_error_return_unix (0, "pthread_create"); + + if (pthread_setaffinity_np (worker, sizeof (cpu_set_t), &cpuset)) + return clib_error_return_unix (0, "pthread_setaffinity_np"); + + return 0; } } @@ -769,6 +750,7 @@ start_workers (vlib_main_t * vm) for (i = 0; i < vec_len (tm->registrations); i++) { + clib_error_t *err; int j; tr = tm->registrations[i]; @@ -778,22 +760,24 @@ start_workers (vlib_main_t * vm) for (j = 0; j < tr->count; j++) { w = vlib_worker_threads + worker_thread_index++; - if (vlib_launch_thread (vlib_worker_thread_bootstrap_fn, w, 0) < - 0) - clib_warning ("Couldn't start '%s' pthread ", tr->name); + err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn, + w, 0); + if (err) + clib_error_report (err); } } else { uword c; - /* *INDENT-OFF* */ - clib_bitmap_foreach (c, tr->coremask, ({ - w = vlib_worker_threads + worker_thread_index++; - if (vlib_launch_thread (vlib_worker_thread_bootstrap_fn, w, c) < 0) - clib_warning ("Couldn't start DPDK lcore %d", c); - - })); -/* *INDENT-ON* */ + /* *INDENT-OFF* */ + clib_bitmap_foreach (c, tr->coremask, ({ + w = vlib_worker_threads + worker_thread_index++; + err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn, + w, c); + if (err) + clib_error_report (err); + })); + /* *INDENT-ON* */ } } vlib_worker_thread_barrier_sync (vm); @@ -1105,7 +1089,7 @@ cpu_config (vlib_main_t * vm, unformat_input_t * input) { tm->n_thread_stacks += tr->count; tm->n_pthreads += tr->count * tr->use_pthreads; - tm->n_eal_threads += tr->count * (tr->use_pthreads == 0); + tm->n_threads += tr->count * (tr->use_pthreads == 0); tr = tr->next; } @@ -1423,6 +1407,7 @@ void vlib_worker_thread_fn (void *arg) { vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; + vlib_thread_main_t *tm = vlib_get_thread_main (); vlib_main_t *vm = vlib_get_main (); ASSERT (vm->cpu_index == os_get_cpu_number ()); @@ -1431,12 +1416,9 @@ vlib_worker_thread_fn (void *arg) clib_time_init (&vm->clib_time); clib_mem_set_heap (w->thread_mheap); -#if DPDK > 0 /* Wait until the dpdk init sequence is complete */ - vlib_thread_main_t *tm = vlib_get_thread_main (); - while (tm->worker_thread_release == 0) + while (tm->extern_thread_mgmt && tm->worker_thread_release == 0) vlib_worker_thread_barrier_check (); -#endif vlib_worker_thread_internal (vm); } @@ -1475,6 +1457,20 @@ vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts) return (fqm - tm->frame_queue_mains); } + +int +vlib_thread_cb_register (struct vlib_main_t *vm, vlib_thread_callbacks_t * cb) +{ + vlib_thread_main_t *tm = vlib_get_thread_main (); + + if (tm->extern_thread_mgmt) + return -1; + + tm->cb.vlib_launch_thread_cb = cb->vlib_launch_thread_cb; + tm->extern_thread_mgmt = 1; + return 0; +} + clib_error_t * threads_init (vlib_main_t * vm) { diff --git a/src/vlib/threads.h b/src/vlib/threads.h index 34ab5be8..75a5a281 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -263,6 +263,13 @@ typedef enum SCHED_POLICY_N, } sched_policy_t; +typedef struct +{ + clib_error_t *(*vlib_launch_thread_cb) (void *fp, vlib_worker_thread_t * w, + unsigned lcore_id); + clib_error_t *(*vlib_thread_set_lcore_cb) (u32 thread, u16 lcore); +} vlib_thread_callbacks_t; + typedef struct { /* Link list of registrations, built by constructors */ @@ -290,8 +297,8 @@ typedef struct /* Number of pthreads */ u32 n_pthreads; - /* Number of DPDK eal threads */ - u32 n_eal_threads; + /* Number of threads */ + u32 n_threads; /* Number of cores to skip, must match the core mask */ u32 skip_cores; @@ -320,6 +327,9 @@ typedef struct /* scheduling policy priority */ u32 sched_priority; + /* callbacks */ + vlib_thread_callbacks_t cb; + int extern_thread_mgmt; } vlib_thread_main_t; extern vlib_thread_main_t vlib_thread_main; @@ -459,6 +469,9 @@ vlib_get_worker_handoff_queue_elt (u32 frame_queue_index, return elt; } +int vlib_thread_cb_register (struct vlib_main_t *vm, + vlib_thread_callbacks_t * cb); + #endif /* included_vlib_threads_h */ /* diff --git a/src/vlib/threads_cli.c b/src/vlib/threads_cli.c index ee632279..b64028c4 100644 --- a/src/vlib/threads_cli.c +++ b/src/vlib/threads_cli.c @@ -20,14 +20,6 @@ #include #include -#if DPDK==1 -#include -#include -#include -#include -#include -#endif - static u8 * format_sched_policy_and_priority (u8 * s, va_list * args) { @@ -116,23 +108,6 @@ show_threads_fn (vlib_main_t * vm, vec_free (p); line = format (line, "%-7u%-7u%-7u%", lcore, core_id, socket_id); -#if DPDK==1 - ASSERT (lcore <= RTE_MAX_LCORE); - switch (lcore_config[lcore].state) - { - case WAIT: - line = format (line, "wait"); - break; - case RUNNING: - line = format (line, "running"); - break; - case FINISHED: - line = format (line, "finished"); - break; - default: - line = format (line, "unknown"); - } -#endif } else { diff --git a/src/vlib/unix/physmem.c b/src/vlib/unix/physmem.c index 80ab7b9d..8d10ad2e 100644 --- a/src/vlib/unix/physmem.c +++ b/src/vlib/unix/physmem.c @@ -45,13 +45,13 @@ static void * unix_physmem_alloc_aligned (vlib_physmem_main_t * vpm, uword n_bytes, uword alignment) { + vlib_main_t *vm = vlib_get_main (); physmem_main_t *pm = &physmem_main; uword lo_offset, hi_offset; uword *to_free = 0; -#if DPDK > 0 - clib_warning ("unsafe alloc!"); -#endif + if (vm->buffer_main->extern_buffer_mgmt) + clib_warning ("unsafe alloc!"); /* IO memory is always at least cache aligned. */ alignment = clib_max (alignment, CLIB_CACHE_LINE_BYTES); @@ -269,16 +269,17 @@ static clib_error_t * show_physmem (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { -#if DPDK > 0 - vlib_cli_output (vm, "Not supported with DPDK drivers."); -#else physmem_main_t *pm = &physmem_main; + if (vm->buffer_main->extern_buffer_mgmt) + { + vlib_cli_output (vm, "Not supported with external buffer management."); + return 0; + } if (pm->heap) vlib_cli_output (vm, "%U", format_mheap, pm->heap, /* verbose */ 1); else vlib_cli_output (vm, "No physmem allocated."); -#endif return 0; } diff --git a/src/vnet.am b/src/vnet.am index 665a16ea..47c5eda7 100644 --- a/src/vnet.am +++ b/src/vnet.am @@ -761,11 +761,13 @@ nobase_include_HEADERS += \ ######################################## if WITH_DPDK libvnet_la_SOURCES += \ + vnet/devices/dpdk/buffer.c \ vnet/devices/dpdk/dpdk_priv.h \ vnet/devices/dpdk/device.c \ vnet/devices/dpdk/format.c \ vnet/devices/dpdk/init.c \ vnet/devices/dpdk/node.c \ + vnet/devices/dpdk/thread.c \ vnet/devices/dpdk/hqos.c \ vnet/devices/dpdk/cli.c \ vnet/devices/dpdk/dpdk_api.c diff --git a/src/vnet/devices/dpdk/buffer.c b/src/vnet/devices/dpdk/buffer.c new file mode 100644 index 00000000..214a9162 --- /dev/null +++ b/src/vnet/devices/dpdk/buffer.c @@ -0,0 +1,729 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * buffer.c: allocate/free network buffers. + * + * Copyright (c) 2008 Eliot Dresselhaus + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/** + * @file + * + * Allocate/free network buffers. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + +STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM, + "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM"); + +#define BUFFERS_PER_COPY (sizeof (vlib_copy_unit_t) / sizeof (u32)) + +/* Make sure we have at least given number of unaligned buffers. */ +static void +fill_unaligned (vlib_main_t * vm, + vlib_buffer_free_list_t * free_list, + uword n_unaligned_buffers) +{ + word la = vec_len (free_list->aligned_buffers); + word lu = vec_len (free_list->unaligned_buffers); + + /* Aligned come in aligned copy-sized chunks. */ + ASSERT (la % BUFFERS_PER_COPY == 0); + + ASSERT (la >= n_unaligned_buffers); + + while (lu < n_unaligned_buffers) + { + /* Copy 4 buffers from end of aligned vector to unaligned vector. */ + vec_add (free_list->unaligned_buffers, + free_list->aligned_buffers + la - BUFFERS_PER_COPY, + BUFFERS_PER_COPY); + la -= BUFFERS_PER_COPY; + lu += BUFFERS_PER_COPY; + } + _vec_len (free_list->aligned_buffers) = la; +} + +/* After free aligned buffers may not contain even sized chunks. */ +static void +trim_aligned (vlib_buffer_free_list_t * f) +{ + uword l, n_trim; + + /* Add unaligned to aligned before trim. */ + l = vec_len (f->unaligned_buffers); + if (l > 0) + { + vec_add_aligned (f->aligned_buffers, f->unaligned_buffers, l, + /* align */ sizeof (vlib_copy_unit_t)); + + _vec_len (f->unaligned_buffers) = 0; + } + + /* Remove unaligned buffers from end of aligned vector and save for next trim. */ + l = vec_len (f->aligned_buffers); + n_trim = l % BUFFERS_PER_COPY; + if (n_trim) + { + /* Trim aligned -> unaligned. */ + vec_add (f->unaligned_buffers, f->aligned_buffers + l - n_trim, n_trim); + + /* Remove from aligned. */ + _vec_len (f->aligned_buffers) = l - n_trim; + } +} + +static void +merge_free_lists (vlib_buffer_free_list_t * dst, + vlib_buffer_free_list_t * src) +{ + uword l; + u32 *d; + + trim_aligned (src); + trim_aligned (dst); + + l = vec_len (src->aligned_buffers); + if (l > 0) + { + vec_add2_aligned (dst->aligned_buffers, d, l, + /* align */ sizeof (vlib_copy_unit_t)); + clib_memcpy (d, src->aligned_buffers, l * sizeof (d[0])); + vec_free (src->aligned_buffers); + } + + l = vec_len (src->unaligned_buffers); + if (l > 0) + { + vec_add (dst->unaligned_buffers, src->unaligned_buffers, l); + vec_free (src->unaligned_buffers); + } +} + +always_inline u32 +dpdk_buffer_get_free_list_with_size (vlib_main_t * vm, u32 size) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + + size = vlib_buffer_round_size (size); + uword *p = hash_get (bm->free_list_by_size, size); + return p ? p[0] : ~0; +} + +static void +del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) +{ + u32 i; + struct rte_mbuf *mb; + vlib_buffer_t *b; + + for (i = 0; i < vec_len (f->unaligned_buffers); i++) + { + b = vlib_get_buffer (vm, f->unaligned_buffers[i]); + mb = rte_mbuf_from_vlib_buffer (b); + ASSERT (rte_mbuf_refcnt_read (mb) == 1); + rte_pktmbuf_free (mb); + } + for (i = 0; i < vec_len (f->aligned_buffers); i++) + { + b = vlib_get_buffer (vm, f->aligned_buffers[i]); + mb = rte_mbuf_from_vlib_buffer (b); + ASSERT (rte_mbuf_refcnt_read (mb) == 1); + rte_pktmbuf_free (mb); + } + vec_free (f->name); + vec_free (f->unaligned_buffers); + vec_free (f->aligned_buffers); +} + +/* Add buffer free list. */ +static void +dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_free_list_t *f; + u32 merge_index; + int i; + + ASSERT (os_get_cpu_number () == 0); + + f = vlib_buffer_get_free_list (vm, free_list_index); + + merge_index = dpdk_buffer_get_free_list_with_size (vm, f->n_data_bytes); + if (merge_index != ~0 && merge_index != free_list_index) + { + merge_free_lists (pool_elt_at_index (bm->buffer_free_list_pool, + merge_index), f); + } + + del_free_list (vm, f); + + /* Poison it. */ + memset (f, 0xab, sizeof (f[0])); + + pool_put (bm->buffer_free_list_pool, f); + + for (i = 1; i < vec_len (vlib_mains); i++) + { + bm = vlib_mains[i]->buffer_main; + f = vlib_buffer_get_free_list (vlib_mains[i], free_list_index);; + memset (f, 0xab, sizeof (f[0])); + pool_put (bm->buffer_free_list_pool, f); + } +} + +/* Make sure free list has at least given number of free buffers. */ +static uword +fill_free_list (vlib_main_t * vm, + vlib_buffer_free_list_t * fl, uword min_free_buffers) +{ + dpdk_main_t *dm = &dpdk_main; + vlib_buffer_t *b; + int n, i; + u32 bi; + u32 n_remaining = 0, n_alloc = 0; + unsigned socket_id = rte_socket_id (); + struct rte_mempool *rmp = dm->pktmbuf_pools[socket_id]; + struct rte_mbuf *mb; + + /* Too early? */ + if (PREDICT_FALSE (rmp == 0)) + return 0; + + trim_aligned (fl); + + /* Already have enough free buffers on free list? */ + n = min_free_buffers - vec_len (fl->aligned_buffers); + if (n <= 0) + return min_free_buffers; + + /* Always allocate round number of buffers. */ + n = round_pow2 (n, BUFFERS_PER_COPY); + + /* Always allocate new buffers in reasonably large sized chunks. */ + n = clib_max (n, fl->min_n_buffers_each_physmem_alloc); + + vec_validate (vm->mbuf_alloc_list, n - 1); + + if (rte_mempool_get_bulk (rmp, vm->mbuf_alloc_list, n) < 0) + return 0; + + _vec_len (vm->mbuf_alloc_list) = n; + + for (i = 0; i < n; i++) + { + mb = vm->mbuf_alloc_list[i]; + + ASSERT (rte_mbuf_refcnt_read (mb) == 0); + rte_mbuf_refcnt_set (mb, 1); + + b = vlib_buffer_from_rte_mbuf (mb); + bi = vlib_get_buffer_index (vm, b); + + vec_add1_aligned (fl->aligned_buffers, bi, sizeof (vlib_copy_unit_t)); + n_alloc++; + n_remaining--; + + vlib_buffer_init_for_free_list (b, fl); + + if (fl->buffer_init_function) + fl->buffer_init_function (vm, fl, &bi, 1); + } + + fl->n_alloc += n; + + return n; +} + +always_inline uword +copy_alignment (u32 * x) +{ + return (pointer_to_uword (x) / sizeof (x[0])) % BUFFERS_PER_COPY; +} + +static u32 +alloc_from_free_list (vlib_main_t * vm, + vlib_buffer_free_list_t * free_list, + u32 * alloc_buffers, u32 n_alloc_buffers) +{ + u32 *dst, *u_src; + uword u_len, n_left; + uword n_unaligned_start, n_unaligned_end, n_filled; + + n_left = n_alloc_buffers; + dst = alloc_buffers; + n_unaligned_start = ((BUFFERS_PER_COPY - copy_alignment (dst)) + & (BUFFERS_PER_COPY - 1)); + + n_filled = fill_free_list (vm, free_list, n_alloc_buffers); + if (n_filled == 0) + return 0; + + n_left = n_filled < n_left ? n_filled : n_left; + n_alloc_buffers = n_left; + + if (n_unaligned_start >= n_left) + { + n_unaligned_start = n_left; + n_unaligned_end = 0; + } + else + n_unaligned_end = copy_alignment (dst + n_alloc_buffers); + + fill_unaligned (vm, free_list, n_unaligned_start + n_unaligned_end); + + u_len = vec_len (free_list->unaligned_buffers); + u_src = free_list->unaligned_buffers + u_len - 1; + + if (n_unaligned_start) + { + uword n_copy = n_unaligned_start; + if (n_copy > n_left) + n_copy = n_left; + n_left -= n_copy; + + while (n_copy > 0) + { + *dst++ = *u_src--; + n_copy--; + u_len--; + } + + /* Now dst should be aligned. */ + if (n_left > 0) + ASSERT (pointer_to_uword (dst) % sizeof (vlib_copy_unit_t) == 0); + } + + /* Aligned copy. */ + { + vlib_copy_unit_t *d, *s; + uword n_copy; + + if (vec_len (free_list->aligned_buffers) < + ((n_left / BUFFERS_PER_COPY) * BUFFERS_PER_COPY)) + abort (); + + n_copy = n_left / BUFFERS_PER_COPY; + n_left = n_left % BUFFERS_PER_COPY; + + /* Remove buffers from aligned free list. */ + _vec_len (free_list->aligned_buffers) -= n_copy * BUFFERS_PER_COPY; + + s = (vlib_copy_unit_t *) vec_end (free_list->aligned_buffers); + d = (vlib_copy_unit_t *) dst; + + /* Fast path loop. */ + while (n_copy >= 4) + { + d[0] = s[0]; + d[1] = s[1]; + d[2] = s[2]; + d[3] = s[3]; + n_copy -= 4; + s += 4; + d += 4; + } + + while (n_copy >= 1) + { + d[0] = s[0]; + n_copy -= 1; + s += 1; + d += 1; + } + + dst = (void *) d; + } + + /* Unaligned copy. */ + ASSERT (n_unaligned_end == n_left); + while (n_left > 0) + { + *dst++ = *u_src--; + n_left--; + u_len--; + } + + if (!free_list->unaligned_buffers) + ASSERT (u_len == 0); + else + _vec_len (free_list->unaligned_buffers) = u_len; + + return n_alloc_buffers; +} + +/* Allocate a given number of buffers into given array. + Returns number actually allocated which will be either zero or + number requested. */ +u32 +dpdk_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + + return alloc_from_free_list + (vm, + pool_elt_at_index (bm->buffer_free_list_pool, + VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX), + buffers, n_buffers); +} + + +u32 +dpdk_buffer_alloc_from_free_list (vlib_main_t * vm, + u32 * buffers, + u32 n_buffers, u32 free_list_index) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_free_list_t *f; + f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index); + return alloc_from_free_list (vm, f, buffers, n_buffers); +} + +always_inline void +add_buffer_to_free_list (vlib_main_t * vm, + vlib_buffer_free_list_t * f, + u32 buffer_index, u8 do_init) +{ + vlib_buffer_t *b; + b = vlib_get_buffer (vm, buffer_index); + if (PREDICT_TRUE (do_init)) + vlib_buffer_init_for_free_list (b, f); + vec_add1_aligned (f->aligned_buffers, buffer_index, + sizeof (vlib_copy_unit_t)); +} + +always_inline vlib_buffer_free_list_t * +buffer_get_free_list (vlib_main_t * vm, vlib_buffer_t * b, u32 * index) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + u32 i; + + *index = i = b->free_list_index; + return pool_elt_at_index (bm->buffer_free_list_pool, i); +} + +static_always_inline void +vlib_buffer_free_inline (vlib_main_t * vm, + u32 * buffers, u32 n_buffers, u32 follow_buffer_next) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_free_list_t *fl; + u32 fi; + int i; + u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers, + u32 follow_buffer_next); + + cb = bm->buffer_free_callback; + + if (PREDICT_FALSE (cb != 0)) + n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next); + + if (!n_buffers) + return; + + for (i = 0; i < n_buffers; i++) + { + vlib_buffer_t *b; + struct rte_mbuf *mb; + + b = vlib_get_buffer (vm, buffers[i]); + + fl = buffer_get_free_list (vm, b, &fi); + + /* The only current use of this callback: multicast recycle */ + if (PREDICT_FALSE (fl->buffers_added_to_freelist_function != 0)) + { + int j; + + add_buffer_to_free_list + (vm, fl, buffers[i], (b->flags & VLIB_BUFFER_RECYCLE) == 0); + + for (j = 0; j < vec_len (bm->announce_list); j++) + { + if (fl == bm->announce_list[j]) + goto already_announced; + } + vec_add1 (bm->announce_list, fl); + already_announced: + ; + } + else + { + if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0)) + { + mb = rte_mbuf_from_vlib_buffer (b); + ASSERT (rte_mbuf_refcnt_read (mb) == 1); + rte_pktmbuf_free (mb); + } + } + } + if (vec_len (bm->announce_list)) + { + vlib_buffer_free_list_t *fl; + for (i = 0; i < vec_len (bm->announce_list); i++) + { + fl = bm->announce_list[i]; + fl->buffers_added_to_freelist_function (vm, fl); + } + _vec_len (bm->announce_list) = 0; + } +} + +static void +dpdk_buffer_free (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +{ + vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ + 1); +} + +static void +dpdk_buffer_free_no_next (vlib_main_t * vm, u32 * buffers, u32 n_buffers) +{ + vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ + 0); +} + +static void +dpdk_packet_template_init (vlib_main_t * vm, + void *vt, + void *packet_data, + uword n_packet_data_bytes, + uword min_n_buffers_each_physmem_alloc, u8 * name) +{ + vlib_packet_template_t *t = (vlib_packet_template_t *) vt; + + vlib_worker_thread_barrier_sync (vm); + memset (t, 0, sizeof (t[0])); + + vec_add (t->packet_data, packet_data, n_packet_data_bytes); + + vlib_worker_thread_barrier_release (vm); +} + +clib_error_t * +vlib_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, + unsigned socket_id) +{ + dpdk_main_t *dm = &dpdk_main; + vlib_physmem_main_t *vpm = &vm->physmem_main; + struct rte_mempool *rmp; + int i; + + vec_validate_aligned (dm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES); + + /* pool already exists, nothing to do */ + if (dm->pktmbuf_pools[socket_id]) + return 0; + + u8 *pool_name = format (0, "mbuf_pool_socket%u%c", socket_id, 0); + + rmp = rte_pktmbuf_pool_create ((char *) pool_name, /* pool name */ + num_mbufs, /* number of mbufs */ + 512, /* cache size */ + VLIB_BUFFER_HDR_SIZE, /* priv size */ + VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE, /* dataroom size */ + socket_id); /* cpu socket */ + + if (rmp) + { + { + uword this_pool_end; + uword this_pool_start; + uword this_pool_size; + uword save_vpm_start, save_vpm_end, save_vpm_size; + struct rte_mempool_memhdr *memhdr; + + this_pool_start = ~0ULL; + this_pool_end = 0LL; + + STAILQ_FOREACH (memhdr, &rmp->mem_list, next) + { + if (((uword) (memhdr->addr + memhdr->len)) > this_pool_end) + this_pool_end = (uword) (memhdr->addr + memhdr->len); + if (((uword) memhdr->addr) < this_pool_start) + this_pool_start = (uword) (memhdr->addr); + } + ASSERT (this_pool_start < ~0ULL && this_pool_end > 0); + this_pool_size = this_pool_end - this_pool_start; + + if (CLIB_DEBUG > 1) + { + clib_warning ("%s: pool start %llx pool end %llx pool size %lld", + pool_name, this_pool_start, this_pool_end, + this_pool_size); + clib_warning + ("before: virtual.start %llx virtual.end %llx virtual.size %lld", + vpm->virtual.start, vpm->virtual.end, vpm->virtual.size); + } + + save_vpm_start = vpm->virtual.start; + save_vpm_end = vpm->virtual.end; + save_vpm_size = vpm->virtual.size; + + if ((this_pool_start < vpm->virtual.start) || vpm->virtual.start == 0) + vpm->virtual.start = this_pool_start; + if (this_pool_end > vpm->virtual.end) + vpm->virtual.end = this_pool_end; + + vpm->virtual.size = vpm->virtual.end - vpm->virtual.start; + + if (CLIB_DEBUG > 1) + { + clib_warning + ("after: virtual.start %llx virtual.end %llx virtual.size %lld", + vpm->virtual.start, vpm->virtual.end, vpm->virtual.size); + } + + /* check if fits into buffer index range */ + if ((u64) vpm->virtual.size > + ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES))) + { + clib_warning ("physmem: virtual size out of range!"); + vpm->virtual.start = save_vpm_start; + vpm->virtual.end = save_vpm_end; + vpm->virtual.size = save_vpm_size; + rmp = 0; + } + } + if (rmp) + { + dm->pktmbuf_pools[socket_id] = rmp; + vec_free (pool_name); + return 0; + } + } + + vec_free (pool_name); + + /* no usable pool for this socket, try to use pool from another one */ + for (i = 0; i < vec_len (dm->pktmbuf_pools); i++) + { + if (dm->pktmbuf_pools[i]) + { + clib_warning + ("WARNING: Failed to allocate mempool for CPU socket %u. " + "Threads running on socket %u will use socket %u mempool.", + socket_id, socket_id, i); + dm->pktmbuf_pools[socket_id] = dm->pktmbuf_pools[i]; + return 0; + } + } + + return clib_error_return (0, "failed to allocate mempool on socket %u", + socket_id); +} + +#if CLIB_DEBUG > 0 + +u32 *vlib_buffer_state_validation_lock; +uword *vlib_buffer_state_validation_hash; +void *vlib_buffer_state_heap; + +static clib_error_t * +buffer_state_validation_init (vlib_main_t * vm) +{ + void *oldheap; + + vlib_buffer_state_heap = mheap_alloc (0, 10 << 20); + + oldheap = clib_mem_set_heap (vlib_buffer_state_heap); + + vlib_buffer_state_validation_hash = hash_create (0, sizeof (uword)); + vec_validate_aligned (vlib_buffer_state_validation_lock, 0, + CLIB_CACHE_LINE_BYTES); + clib_mem_set_heap (oldheap); + return 0; +} + +VLIB_INIT_FUNCTION (buffer_state_validation_init); +#endif + +static vlib_buffer_callbacks_t callbacks = { + .vlib_buffer_alloc_cb = &dpdk_buffer_alloc, + .vlib_buffer_alloc_from_free_list_cb = &dpdk_buffer_alloc_from_free_list, + .vlib_buffer_free_cb = &dpdk_buffer_free, + .vlib_buffer_free_no_next_cb = &dpdk_buffer_free_no_next, + .vlib_packet_template_init_cb = &dpdk_packet_template_init, + .vlib_buffer_delete_free_list_cb = &dpdk_buffer_delete_free_list, +}; + +static clib_error_t * +dpdk_buffer_init (vlib_main_t * vm) +{ + vlib_buffer_cb_register (vm, &callbacks); + return 0; +} + +VLIB_INIT_FUNCTION (dpdk_buffer_init); + +/** @endcond */ +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vnet/devices/dpdk/cli.c b/src/vnet/devices/dpdk/cli.c index 538a00fd..22bd4b4f 100644 --- a/src/vnet/devices/dpdk/cli.c +++ b/src/vnet/devices/dpdk/cli.c @@ -164,9 +164,9 @@ show_dpdk_buffer (vlib_main_t * vm, unformat_input_t * input, struct rte_mempool *rmp; int i; - for (i = 0; i < vec_len (vm->buffer_main->pktmbuf_pools); i++) + for (i = 0; i < vec_len (dpdk_main.pktmbuf_pools); i++) { - rmp = vm->buffer_main->pktmbuf_pools[i]; + rmp = dpdk_main.pktmbuf_pools[i]; if (rmp) { unsigned count = rte_mempool_avail_count (rmp); diff --git a/src/vnet/devices/dpdk/device.c b/src/vnet/devices/dpdk/device.c index b22fbf2e..0deab6aa 100644 --- a/src/vnet/devices/dpdk/device.c +++ b/src/vnet/devices/dpdk/device.c @@ -87,19 +87,18 @@ dpdk_set_mc_filter (vnet_hw_interface_t * hi, struct rte_mbuf * dpdk_replicate_packet_mb (vlib_buffer_t * b) { - vlib_main_t *vm = vlib_get_main (); - vlib_buffer_main_t *bm = vm->buffer_main; + dpdk_main_t *dm = &dpdk_main; struct rte_mbuf **mbufs = 0, *s, *d; u8 nb_segs; unsigned socket_id = rte_socket_id (); int i; - ASSERT (bm->pktmbuf_pools[socket_id]); + ASSERT (dm->pktmbuf_pools[socket_id]); s = rte_mbuf_from_vlib_buffer (b); nb_segs = s->nb_segs; vec_validate (mbufs, nb_segs - 1); - if (rte_pktmbuf_alloc_bulk (bm->pktmbuf_pools[socket_id], mbufs, nb_segs)) + if (rte_pktmbuf_alloc_bulk (dm->pktmbuf_pools[socket_id], mbufs, nb_segs)) { vec_free (mbufs); return 0; diff --git a/src/vnet/devices/dpdk/dpdk.h b/src/vnet/devices/dpdk/dpdk.h index e0436031..066ec6fa 100644 --- a/src/vnet/devices/dpdk/dpdk.h +++ b/src/vnet/devices/dpdk/dpdk.h @@ -425,6 +425,9 @@ typedef struct vlib_main_t *vlib_main; vnet_main_t *vnet_main; dpdk_config_main_t *conf; + + /* mempool */ + struct rte_mempool **pktmbuf_pools; } dpdk_main_t; dpdk_main_t dpdk_main; diff --git a/src/vnet/devices/dpdk/dpdk_priv.h b/src/vnet/devices/dpdk/dpdk_priv.h index 0c81dbc3..dd40ff48 100644 --- a/src/vnet/devices/dpdk/dpdk_priv.h +++ b/src/vnet/devices/dpdk/dpdk_priv.h @@ -13,6 +13,9 @@ * limitations under the License. */ +#define rte_mbuf_from_vlib_buffer(x) (((struct rte_mbuf *)x) - 1) +#define vlib_buffer_from_rte_mbuf(x) ((vlib_buffer_t *)(x+1)) + #define DPDK_NB_RX_DESC_DEFAULT 1024 #define DPDK_NB_TX_DESC_DEFAULT 1024 #define DPDK_NB_RX_DESC_VIRTIO 256 diff --git a/src/vnet/devices/dpdk/init.c b/src/vnet/devices/dpdk/init.c index 60689463..4c040d20 100755 --- a/src/vnet/devices/dpdk/init.c +++ b/src/vnet/devices/dpdk/init.c @@ -64,8 +64,6 @@ static struct rte_eth_conf port_conf_template = { clib_error_t * dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd) { - vlib_main_t *vm = vlib_get_main (); - vlib_buffer_main_t *bm = vm->buffer_main; int rv; int j; @@ -107,7 +105,7 @@ dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd) rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc, xd->cpu_socket, 0, - bm-> + dm-> pktmbuf_pools[xd->cpu_socket_id_by_queue [j]]); @@ -115,7 +113,7 @@ dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd) if (rv < 0) rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc, SOCKET_ID_ANY, 0, - bm-> + dm-> pktmbuf_pools[xd->cpu_socket_id_by_queue [j]]); if (rv < 0) diff --git a/src/vnet/devices/dpdk/thread.c b/src/vnet/devices/dpdk/thread.c new file mode 100644 index 00000000..475dd142 --- /dev/null +++ b/src/vnet/devices/dpdk/thread.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +static clib_error_t * +dpdk_launch_thread (void *fp, vlib_worker_thread_t * w, unsigned lcore_id) +{ + int r; + r = rte_eal_remote_launch (fp, (void *) w, lcore_id); + if (r) + return clib_error_return (0, "Failed to launch thread %u", lcore_id); + return 0; +} + +static clib_error_t * +dpdk_thread_set_lcore (u32 thread, u16 lcore) +{ + return 0; +} + +static vlib_thread_callbacks_t callbacks = { + .vlib_launch_thread_cb = &dpdk_launch_thread, + .vlib_thread_set_lcore_cb = &dpdk_thread_set_lcore, +}; + +static clib_error_t * +dpdk_thread_init (vlib_main_t * vm) +{ + vlib_thread_cb_register (vm, &callbacks); + return 0; +} + +VLIB_INIT_FUNCTION (dpdk_thread_init); + +/** @endcond */ +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vnet/sr/sr_replicate.c b/src/vnet/sr/sr_replicate.c index 5f9de504..fa5a68c3 100644 --- a/src/vnet/sr/sr_replicate.c +++ b/src/vnet/sr/sr_replicate.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -142,6 +143,7 @@ static uword sr_replicate_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { + dpdk_main_t *dm = &dpdk_main; u32 n_left_from, *from, *to_next; sr_replicate_next_t next_index; int pkts_replicated = 0; @@ -149,7 +151,6 @@ sr_replicate_node_fn (vlib_main_t * vm, int no_buffer_drops = 0; vlib_buffer_free_list_t *fl; unsigned socket_id = rte_socket_id (); - vlib_buffer_main_t *bm = vm->buffer_main; from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -246,13 +247,13 @@ sr_replicate_node_fn (vlib_main_t * vm, vlib_buffer_t *clone0_c, *clone_b0; t0 = vec_elt_at_index (sm->tunnels, pol0->tunnel_indices[i]); - hdr_mb0 = rte_pktmbuf_alloc (bm->pktmbuf_pools[socket_id]); + hdr_mb0 = rte_pktmbuf_alloc (dm->pktmbuf_pools[socket_id]); if (i < (num_replicas - 1)) { /* Not the last tunnel to process */ clone0 = rte_pktmbuf_clone - (orig_mb0, bm->pktmbuf_pools[socket_id]); + (orig_mb0, dm->pktmbuf_pools[socket_id]); if (clone0 == 0) goto clone_fail; nb_seg = 0; -- cgit 1.2.3-korg From 2d41f169c818219b5e8b23ed32e28ef334f39c35 Mon Sep 17 00:00:00 2001 From: Matus Fabian Date: Tue, 21 Feb 2017 22:27:13 -0800 Subject: fix trace frame-queue unformat of index Change-Id: Id891af5ef3c4afe877282b34cd03fc43886940a3 Signed-off-by: Matus Fabian --- src/vlib/threads_cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/vlib/threads_cli.c') diff --git a/src/vlib/threads_cli.c b/src/vlib/threads_cli.c index b64028c4..54cc1aed 100644 --- a/src/vlib/threads_cli.c +++ b/src/vlib/threads_cli.c @@ -160,7 +160,7 @@ trace_frame_queue (vlib_main_t * vm, unformat_input_t * input, enable = 1; else if (unformat (line_input, "off")) enable = 0; - else if (unformat (line_input, "index %u"), &index) + else if (unformat (line_input, "index %u", &index)) ; else return clib_error_return (0, "parse error: '%U'", -- cgit 1.2.3-korg From a9a20e7f69f4a91a4d5267ab5ce14125bdc7d6c6 Mon Sep 17 00:00:00 2001 From: Billy McFall Date: Wed, 15 Feb 2017 11:39:12 -0500 Subject: VPP-635: CLI Memory leak with invalid parameter In the CLI parsing, below is a common pattern: /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) return 0; while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) { if (unformat (line_input, "x")) x = 1; : else return clib_error_return (0, "unknown input `%U'", format_unformat_error, line_input); } unformat_free (line_input); The 'else' returns if an unknown string is encountered. There a memory leak because the 'unformat_free(line_input)' is not called. There is a large number of instances of this pattern. Replaced the previous pattern with: /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) return 0; while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) { if (unformat (line_input, "x")) x = 1; : else { error = clib_error_return (0, "unknown input `%U'", format_unformat_error, line_input); goto done: } } /* ...Remaining code... */ done: unformat_free (line_input); return error; } In multiple files, 'unformat_free (line_input);' was never called, so there was a memory leak whether an invalid string was entered or not. Also, there were multiple instance where: error = clib_error_return (0, "unknown input `%U'", format_unformat_error, line_input); used 'input' as the last parameter instead of 'line_input'. The result is that output did not contain the substring in error, instead just an empty string. Fixed all of those as well. There are a lot of file, and very mind numbing work, so tried to keep it to a pattern to avoid mistakes. Change-Id: I8902f0c32a47dd7fb3bb3471a89818571702f1d2 Signed-off-by: Billy McFall Signed-off-by: Dave Barach --- build-root/emacs-lisp/tunnel-c-skel.el | 19 ++- src/plugins/ila/ila.c | 25 ++- src/plugins/lb/cli.c | 99 ++++++----- src/plugins/sixrd/sixrd.c | 42 +++-- src/plugins/snat/snat.c | 139 +++++++++++----- src/vlib/threads_cli.c | 79 ++++++--- src/vlib/trace.c | 13 +- src/vlib/unix/cli.c | 22 ++- src/vnet/devices/af_packet/cli.c | 56 +++++-- src/vnet/devices/dpdk/cli.c | 290 +++++++++++++++++++++++---------- src/vnet/devices/dpdk/ipsec/cli.c | 15 +- src/vnet/devices/netmap/cli.c | 54 ++++-- src/vnet/devices/virtio/vhost-user.c | 62 +++++-- src/vnet/gre/interface.c | 35 ++-- src/vnet/ip/ip4_source_check.c | 6 +- src/vnet/ip/ip4_test.c | 15 +- src/vnet/ip/ip6_neighbor.c | 27 ++- src/vnet/ip/lookup.c | 34 ++-- src/vnet/ipsec-gre/interface.c | 34 ++-- src/vnet/ipsec/ipsec_cli.c | 177 +++++++++++++------- src/vnet/l2/l2_patch.c | 26 ++- src/vnet/l2/l2_xcrw.c | 34 +++- src/vnet/l2tp/l2tp.c | 39 +++-- src/vnet/lisp-cp/lisp_cli.c | 139 ++++++++++++---- src/vnet/lisp-gpe/interface.c | 58 +++++-- src/vnet/lisp-gpe/lisp_gpe.c | 13 +- src/vnet/map/map.c | 186 +++++++++++++++------ src/vnet/mpls/mpls.c | 2 + src/vnet/mpls/mpls_tunnel.c | 19 ++- src/vnet/pg/cli.c | 39 +++-- src/vnet/policer/node_funcs.c | 19 ++- src/vnet/policer/policer.c | 13 +- src/vnet/unix/tapcli.c | 57 +++++-- src/vnet/vxlan-gpe/vxlan_gpe.c | 62 +++++-- src/vnet/vxlan/vxlan.c | 81 ++++++--- src/vpp/app/l2t.c | 9 +- src/vpp/app/vpe_cli.c | 24 ++- 37 files changed, 1487 insertions(+), 576 deletions(-) (limited to 'src/vlib/threads_cli.c') diff --git a/build-root/emacs-lisp/tunnel-c-skel.el b/build-root/emacs-lisp/tunnel-c-skel.el index aa260e53..a1b1757d 100644 --- a/build-root/emacs-lisp/tunnel-c-skel.el +++ b/build-root/emacs-lisp/tunnel-c-skel.el @@ -288,6 +288,7 @@ static clib_error_t * vlib_cli_command_t * cmd) { unformat_input_t _line_input, * line_input = &_line_input; + clib_error_t *error = 0; ip4_address_t src, dst; u8 is_add = 1; u8 src_set = 0; @@ -322,13 +323,19 @@ static clib_error_t * { encap_fib_index = fib_index_from_fib_id (tmp); if (encap_fib_index == ~0) - return clib_error_return (0, \"nonexistent encap fib id %d\", tmp); + { + unformat_free (line_input); + return clib_error_return (0, \"nonexistent encap fib id %d\", tmp); + } } else if (unformat (line_input, \"decap-vrf-id %d\", &tmp)) { decap_fib_index = fib_index_from_fib_id (tmp); if (decap_fib_index == ~0) - return clib_error_return (0, \"nonexistent decap fib id %d\", tmp); + { + unformat_free (line_input); + return clib_error_return (0, \"nonexistent decap fib id %d\", tmp); + } } else if (unformat (line_input, \"decap-next %U\", unformat_decap_next, &decap_next_index)) @@ -346,8 +353,12 @@ static clib_error_t * * in the " ENCAP_STACK " header */ else - return clib_error_return (0, \"parse error: '%U'\", - format_unformat_error, line_input); + { + error = clib_error_return (0, \"parse error: '%U'\", + format_unformat_error, line_input); + unformat_free (line_input); + return error; + } } unformat_free (line_input); diff --git a/src/plugins/ila/ila.c b/src/plugins/ila/ila.c index e0f3907f..52c7ea55 100644 --- a/src/plugins/ila/ila.c +++ b/src/plugins/ila/ila.c @@ -949,6 +949,7 @@ ila_entry_command_fn (vlib_main_t * vm, ila_add_del_entry_args_t args = { 0 }; u8 next_hop_set = 0; int ret; + clib_error_t *error = 0; args.type = ILA_TYPE_IID; args.csum_mode = ILA_CSUM_MODE_NO_ACTION; @@ -986,19 +987,29 @@ ila_entry_command_fn (vlib_main_t * vm, else if (unformat (line_input, "del")) args.is_del = 1; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (!next_hop_set) - return clib_error_return (0, "Specified a next hop"); + { + error = clib_error_return (0, "Specified a next hop"); + goto done; + } if ((ret = ila_add_del_entry (&args))) - return clib_error_return (0, "ila_add_del_entry returned error %d", ret); + { + error = clib_error_return (0, "ila_add_del_entry returned error %d", ret); + goto done; + } - return NULL; +done: + unformat_free (line_input); + + return error; } VLIB_CLI_COMMAND (ila_entry_command, static) = diff --git a/src/plugins/lb/cli.c b/src/plugins/lb/cli.c index b59c6426..6452a875 100644 --- a/src/plugins/lb/cli.c +++ b/src/plugins/lb/cli.c @@ -28,13 +28,16 @@ lb_vip_command_fn (vlib_main_t * vm, int ret; u32 gre4 = 0; lb_vip_type_t type; + clib_error_t *error = 0; if (!unformat_user (input, unformat_line_input, line_input)) return 0; - if (!unformat(line_input, "%U", unformat_ip46_prefix, &prefix, &plen, IP46_TYPE_ANY, &plen)) - return clib_error_return (0, "invalid vip prefix: '%U'", - format_unformat_error, line_input); + if (!unformat(line_input, "%U", unformat_ip46_prefix, &prefix, &plen, IP46_TYPE_ANY, &plen)) { + error = clib_error_return (0, "invalid vip prefix: '%U'", + format_unformat_error, line_input); + goto done; + } while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) { @@ -46,13 +49,13 @@ lb_vip_command_fn (vlib_main_t * vm, gre4 = 1; else if (unformat(line_input, "encap gre6")) gre4 = 0; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + else { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (ip46_prefix_is_ip4(&prefix, plen)) { type = (gre4)?LB_VIP_TYPE_IP4_GRE4:LB_VIP_TYPE_IP4_GRE6; @@ -65,17 +68,25 @@ lb_vip_command_fn (vlib_main_t * vm, u32 index; if (!del) { if ((ret = lb_vip_add(&prefix, plen, type, new_len, &index))) { - return clib_error_return (0, "lb_vip_add error %d", ret); + error = clib_error_return (0, "lb_vip_add error %d", ret); + goto done; } else { vlib_cli_output(vm, "lb_vip_add ok %d", index); } } else { - if ((ret = lb_vip_find_index(&prefix, plen, &index))) - return clib_error_return (0, "lb_vip_find_index error %d", ret); - else if ((ret = lb_vip_del(index))) - return clib_error_return (0, "lb_vip_del error %d", ret); + if ((ret = lb_vip_find_index(&prefix, plen, &index))) { + error = clib_error_return (0, "lb_vip_find_index error %d", ret); + goto done; + } else if ((ret = lb_vip_del(index))) { + error = clib_error_return (0, "lb_vip_del error %d", ret); + goto done; + } } - return NULL; + +done: + unformat_free (line_input); + + return error; } VLIB_CLI_COMMAND (lb_vip_command, static) = @@ -96,16 +107,21 @@ lb_as_command_fn (vlib_main_t * vm, u32 vip_index; u8 del = 0; int ret; + clib_error_t *error = 0; if (!unformat_user (input, unformat_line_input, line_input)) return 0; - if (!unformat(line_input, "%U", unformat_ip46_prefix, &vip_prefix, &vip_plen, IP46_TYPE_ANY)) - return clib_error_return (0, "invalid as address: '%U'", - format_unformat_error, line_input); + if (!unformat(line_input, "%U", unformat_ip46_prefix, &vip_prefix, &vip_plen, IP46_TYPE_ANY)) { + error = clib_error_return (0, "invalid as address: '%U'", + format_unformat_error, line_input); + goto done; + } - if ((ret = lb_vip_find_index(&vip_prefix, vip_plen, &vip_index))) - return clib_error_return (0, "lb_vip_find_index error %d", ret); + if ((ret = lb_vip_find_index(&vip_prefix, vip_plen, &vip_index))) { + error = clib_error_return (0, "lb_vip_find_index error %d", ret); + goto done; + } while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) { @@ -114,15 +130,15 @@ lb_as_command_fn (vlib_main_t * vm, } else if (unformat(line_input, "del")) { del = 1; } else { - vec_free(as_array); - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; } } if (!vec_len(as_array)) { - vec_free(as_array); - return clib_error_return (0, "No AS address provided"); + error = clib_error_return (0, "No AS address provided"); + goto done; } lb_garbage_collection(); @@ -130,18 +146,21 @@ lb_as_command_fn (vlib_main_t * vm, if (del) { if ((ret = lb_vip_del_ass(vip_index, as_array, vec_len(as_array)))) { - vec_free(as_array); - return clib_error_return (0, "lb_vip_del_ass error %d", ret); + error = clib_error_return (0, "lb_vip_del_ass error %d", ret); + goto done; } } else { if ((ret = lb_vip_add_ass(vip_index, as_array, vec_len(as_array)))) { - vec_free(as_array); - return clib_error_return (0, "lb_vip_add_ass error %d", ret); + error = clib_error_return (0, "lb_vip_add_ass error %d", ret); + goto done; } } +done: + unformat_free (line_input); vec_free(as_array); - return 0; + + return error; } VLIB_CLI_COMMAND (lb_as_command, static) = @@ -163,6 +182,7 @@ lb_conf_command_fn (vlib_main_t * vm, u32 per_cpu_sticky_buckets_log2 = 0; u32 flow_timeout = lbm->flow_timeout; int ret; + clib_error_t *error = 0; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -181,19 +201,24 @@ lb_conf_command_fn (vlib_main_t * vm, per_cpu_sticky_buckets = 1 << per_cpu_sticky_buckets_log2; } else if (unformat(line_input, "timeout %d", &flow_timeout)) ; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + else { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - lb_garbage_collection(); - if ((ret = lb_conf(&ip4, &ip6, per_cpu_sticky_buckets, flow_timeout))) - return clib_error_return (0, "lb_conf error %d", ret); + if ((ret = lb_conf(&ip4, &ip6, per_cpu_sticky_buckets, flow_timeout))) { + error = clib_error_return (0, "lb_conf error %d", ret); + goto done; + } - return NULL; +done: + unformat_free (line_input); + + return error; } VLIB_CLI_COMMAND (lb_conf_command, static) = diff --git a/src/plugins/sixrd/sixrd.c b/src/plugins/sixrd/sixrd.c index 71fc181f..67a9a3ad 100644 --- a/src/plugins/sixrd/sixrd.c +++ b/src/plugins/sixrd/sixrd.c @@ -192,6 +192,7 @@ sixrd_add_domain_command_fn (vlib_main_t *vm, u32 num_m_args = 0; /* Optional arguments */ u32 mtu = 0; + clib_error_t *error = 0; /* Get a line of input. */ if (!unformat_user(input, unformat_line_input, line_input)) @@ -205,19 +206,25 @@ sixrd_add_domain_command_fn (vlib_main_t *vm, num_m_args++; else if (unformat(line_input, "mtu %d", &mtu)) num_m_args++; - else - return clib_error_return(0, "unknown input `%U'", - format_unformat_error, input); + else { + error = clib_error_return(0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free(line_input); - if (num_m_args < 3) - return clib_error_return(0, "mandatory argument(s) missing"); + if (num_m_args < 3) { + error = clib_error_return(0, "mandatory argument(s) missing"); + goto done; + } sixrd_create_domain(&ip6_prefix, ip6_prefix_len, &ip4_prefix, ip4_prefix_len, &ip4_src, &sixrd_domain_index, mtu); - return 0; +done: + unformat_free (line_input); + + return error; } static clib_error_t * @@ -228,6 +235,7 @@ sixrd_del_domain_command_fn (vlib_main_t *vm, unformat_input_t _line_input, *line_input = &_line_input; u32 num_m_args = 0; u32 sixrd_domain_index; + clib_error_t *error = 0; /* Get a line of input. */ if (! unformat_user(input, unformat_line_input, line_input)) @@ -236,18 +244,24 @@ sixrd_del_domain_command_fn (vlib_main_t *vm, while (unformat_check_input(line_input) != UNFORMAT_END_OF_INPUT) { if (unformat(line_input, "index %d", &sixrd_domain_index)) num_m_args++; - else - return clib_error_return(0, "unknown input `%U'", - format_unformat_error, input); + else { + error = clib_error_return(0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free(line_input); - if (num_m_args != 1) - return clib_error_return(0, "mandatory argument(s) missing"); + if (num_m_args != 1) { + error = clib_error_return(0, "mandatory argument(s) missing"); + goto done; + } sixrd_delete_domain(sixrd_domain_index); - return 0; +done: + unformat_free (line_input); + + return error; } static u8 * diff --git a/src/plugins/snat/snat.c b/src/plugins/snat/snat.c index 73854a7a..8c2bacdb 100644 --- a/src/plugins/snat/snat.c +++ b/src/plugins/snat/snat.c @@ -1705,6 +1705,7 @@ add_address_command_fn (vlib_main_t * vm, int i, count; int is_add = 1; int rv = 0; + clib_error_t *error = 0; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -1721,19 +1722,27 @@ add_address_command_fn (vlib_main_t * vm, else if (unformat (line_input, "del")) is_add = 0; else - return clib_error_return (0, "unknown input '%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (sm->static_mapping_only) - return clib_error_return (0, "static mapping only mode"); + { + error = clib_error_return (0, "static mapping only mode"); + goto done; + } start_host_order = clib_host_to_net_u32 (start_addr.as_u32); end_host_order = clib_host_to_net_u32 (end_addr.as_u32); if (end_host_order < start_host_order) - return clib_error_return (0, "end address less than start address"); + { + error = clib_error_return (0, "end address less than start address"); + goto done; + } count = (end_host_order - start_host_order) + 1; @@ -1755,11 +1764,11 @@ add_address_command_fn (vlib_main_t * vm, switch (rv) { case VNET_API_ERROR_NO_SUCH_ENTRY: - return clib_error_return (0, "S-NAT address not exist."); - break; + error = clib_error_return (0, "S-NAT address not exist."); + goto done; case VNET_API_ERROR_UNSPECIFIED: - return clib_error_return (0, "S-NAT address used in static mapping."); - break; + error = clib_error_return (0, "S-NAT address used in static mapping."); + goto done; default: break; } @@ -1767,7 +1776,10 @@ add_address_command_fn (vlib_main_t * vm, increment_v4_address (&this_addr); } - return 0; +done: + unformat_free (line_input); + + return error; } VLIB_CLI_COMMAND (add_address_command, static) = { @@ -1807,10 +1819,12 @@ snat_feature_command_fn (vlib_main_t * vm, else if (unformat (line_input, "del")) is_del = 1; else - return clib_error_return (0, "unknown input '%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (vec_len (inside_sw_if_indices)) { @@ -1830,6 +1844,8 @@ snat_feature_command_fn (vlib_main_t * vm, } } +done: + unformat_free (line_input); vec_free (inside_sw_if_indices); vec_free (outside_sw_if_indices); @@ -1923,13 +1939,18 @@ add_static_mapping_command_fn (vlib_main_t * vm, else if (unformat (line_input, "del")) is_add = 0; else - return clib_error_return (0, "unknown input: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "unknown input: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (!addr_only && !proto_set) - return clib_error_return (0, "missing protocol"); + { + error = clib_error_return (0, "missing protocol"); + goto done; + } rv = snat_add_static_mapping(l_addr, e_addr, (u16) l_port, (u16) e_port, vrf_id, addr_only, sw_if_index, proto, is_add); @@ -1937,22 +1958,27 @@ add_static_mapping_command_fn (vlib_main_t * vm, switch (rv) { case VNET_API_ERROR_INVALID_VALUE: - return clib_error_return (0, "External port already in use."); - break; + error = clib_error_return (0, "External port already in use."); + goto done; case VNET_API_ERROR_NO_SUCH_ENTRY: if (is_add) - return clib_error_return (0, "External addres must be allocated."); + error = clib_error_return (0, "External addres must be allocated."); else - return clib_error_return (0, "Mapping not exist."); - break; + error = clib_error_return (0, "Mapping not exist."); + goto done; case VNET_API_ERROR_NO_SUCH_FIB: - return clib_error_return (0, "No such VRF id."); + error = clib_error_return (0, "No such VRF id."); + goto done; case VNET_API_ERROR_VALUE_EXIST: - return clib_error_return (0, "Mapping already exist."); + error = clib_error_return (0, "Mapping already exist."); + goto done; default: break; } +done: + unformat_free (line_input); + return error; } @@ -1985,6 +2011,7 @@ set_workers_command_fn (vlib_main_t * vm, unformat_input_t _line_input, *line_input = &_line_input; uword *bitmap = 0; int rv = 0; + clib_error_t *error = 0; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -1995,13 +2022,18 @@ set_workers_command_fn (vlib_main_t * vm, if (unformat (line_input, "%U", unformat_bitmap_list, &bitmap)) ; else - return clib_error_return (0, "unknown input '%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (bitmap == 0) - return clib_error_return (0, "List of workers must be specified."); + { + error = clib_error_return (0, "List of workers must be specified."); + goto done; + } rv = snat_set_workers(bitmap); @@ -2010,17 +2042,20 @@ set_workers_command_fn (vlib_main_t * vm, switch (rv) { case VNET_API_ERROR_INVALID_WORKER: - return clib_error_return (0, "Invalid worker(s)."); - break; + error = clib_error_return (0, "Invalid worker(s)."); + goto done; case VNET_API_ERROR_FEATURE_DISABLED: - return clib_error_return (0, + error = clib_error_return (0, "Supported only if 2 or more workes available."); - break; + goto done; default: break; } - return 0; +done: + unformat_free (line_input); + + return error; } /*? @@ -2047,6 +2082,7 @@ snat_ipfix_logging_enable_disable_command_fn (vlib_main_t * vm, u32 src_port = 0; u8 enable = 1; int rv = 0; + clib_error_t *error = 0; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -2061,17 +2097,25 @@ snat_ipfix_logging_enable_disable_command_fn (vlib_main_t * vm, else if (unformat (line_input, "disable")) enable = 0; else - return clib_error_return (0, "unknown input '%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); rv = snat_ipfix_logging_enable_disable (enable, domain_id, (u16) src_port); if (rv) - return clib_error_return (0, "ipfix logging enable failed"); + { + error = clib_error_return (0, "ipfix logging enable failed"); + goto done; + } - return 0; +done: + unformat_free (line_input); + + return error; } /*? @@ -2604,6 +2648,7 @@ snat_add_interface_address_command_fn (vlib_main_t * vm, u32 sw_if_index; int rv; int is_del = 0; + clib_error_t *error = 0; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -2617,8 +2662,11 @@ snat_add_interface_address_command_fn (vlib_main_t * vm, else if (unformat (line_input, "del")) is_del = 1; else - return clib_error_return (0, "unknown input '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + goto done; + } } rv = snat_add_interface_address (sm, sw_if_index, is_del); @@ -2629,10 +2677,15 @@ snat_add_interface_address_command_fn (vlib_main_t * vm, break; default: - return clib_error_return (0, "snat_add_interface_address returned %d", - rv); + error = clib_error_return (0, "snat_add_interface_address returned %d", + rv); + goto done; } - return 0; + +done: + unformat_free (line_input); + + return error; } VLIB_CLI_COMMAND (snat_add_interface_address_command, static) = { diff --git a/src/vlib/threads_cli.c b/src/vlib/threads_cli.c index 54cc1aed..36f8109e 100644 --- a/src/vlib/threads_cli.c +++ b/src/vlib/threads_cli.c @@ -163,21 +163,31 @@ trace_frame_queue (vlib_main_t * vm, unformat_input_t * input, else if (unformat (line_input, "index %u", &index)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (enable > 1) - return clib_error_return (0, "expecting on or off"); + { + error = clib_error_return (0, "expecting on or off"); + goto done; + } if (vec_len (tm->frame_queue_mains) == 0) - return clib_error_return (0, "no worker handoffs exist"); + { + error = clib_error_return (0, "no worker handoffs exist"); + goto done; + } if (index > vec_len (tm->frame_queue_mains) - 1) - return clib_error_return (0, - "expecting valid worker handoff queue index"); + { + error = clib_error_return (0, + "expecting valid worker handoff queue index"); + goto done; + } fqm = vec_elt_at_index (tm->frame_queue_mains, index); @@ -185,7 +195,7 @@ trace_frame_queue (vlib_main_t * vm, unformat_input_t * input, if (num_fq == 0) { vlib_cli_output (vm, "No frame queues exist\n"); - return error; + goto done; } // Allocate storage for trace if necessary @@ -204,6 +214,10 @@ trace_frame_queue (vlib_main_t * vm, unformat_input_t * input, memset (fqh, 0, sizeof (*fqh)); fqm->vlib_frame_queues[fqix]->trace = enable; } + +done: + unformat_free (line_input); + return error; } @@ -432,28 +446,33 @@ test_frame_queue_nelts (vlib_main_t * vm, unformat_input_t * input, else if (unformat (line_input, "index %u", &index)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (index > vec_len (tm->frame_queue_mains) - 1) - return clib_error_return (0, - "expecting valid worker handoff queue index"); + { + error = clib_error_return (0, + "expecting valid worker handoff queue index"); + goto done; + } fqm = vec_elt_at_index (tm->frame_queue_mains, index); if ((nelts != 4) && (nelts != 8) && (nelts != 16) && (nelts != 32)) { - return clib_error_return (0, "expecting 4,8,16,32"); + error = clib_error_return (0, "expecting 4,8,16,32"); + goto done; } num_fq = vec_len (fqm->vlib_frame_queues); if (num_fq == 0) { vlib_cli_output (vm, "No frame queues exist\n"); - return error; + goto done; } for (fqix = 0; fqix < num_fq; fqix++) @@ -461,6 +480,9 @@ test_frame_queue_nelts (vlib_main_t * vm, unformat_input_t * input, fqm->vlib_frame_queues[fqix]->nelts = nelts; } +done: + unformat_free (line_input); + return error; } @@ -499,15 +521,19 @@ test_frame_queue_threshold (vlib_main_t * vm, unformat_input_t * input, else if (unformat (line_input, "index %u", &index)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (index > vec_len (tm->frame_queue_mains) - 1) - return clib_error_return (0, - "expecting valid worker handoff queue index"); + { + error = clib_error_return (0, + "expecting valid worker handoff queue index"); + goto done; + } fqm = vec_elt_at_index (tm->frame_queue_mains, index); @@ -515,7 +541,7 @@ test_frame_queue_threshold (vlib_main_t * vm, unformat_input_t * input, if (threshold == ~(u32) 0) { vlib_cli_output (vm, "expecting threshold value\n"); - return error; + goto done; } if (threshold == 0) @@ -525,7 +551,7 @@ test_frame_queue_threshold (vlib_main_t * vm, unformat_input_t * input, if (num_fq == 0) { vlib_cli_output (vm, "No frame queues exist\n"); - return error; + goto done; } for (fqix = 0; fqix < num_fq; fqix++) @@ -533,6 +559,9 @@ test_frame_queue_threshold (vlib_main_t * vm, unformat_input_t * input, fqm->vlib_frame_queues[fqix]->vector_threshold = threshold; } +done: + unformat_free (line_input); + return error; } diff --git a/src/vlib/trace.c b/src/vlib/trace.c index dcdb837f..6d487ae1 100644 --- a/src/vlib/trace.c +++ b/src/vlib/trace.c @@ -372,6 +372,7 @@ cli_add_trace_buffer (vlib_main_t * vm, vlib_trace_node_t *tn; u32 node_index, add; u8 verbose = 0; + clib_error_t *error = 0; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -384,8 +385,11 @@ cli_add_trace_buffer (vlib_main_t * vm, else if (unformat (line_input, "verbose")) verbose = 1; else - return clib_error_create ("expected NODE COUNT, got `%U'", - format_unformat_error, line_input); + { + error = clib_error_create ("expected NODE COUNT, got `%U'", + format_unformat_error, line_input); + goto done; + } } /* *INDENT-OFF* */ @@ -403,7 +407,10 @@ cli_add_trace_buffer (vlib_main_t * vm, })); /* *INDENT-ON* */ - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ diff --git a/src/vlib/unix/cli.c b/src/vlib/unix/cli.c index 69fca6ec..88e2453c 100644 --- a/src/vlib/unix/cli.c +++ b/src/vlib/unix/cli.c @@ -2835,6 +2835,7 @@ unix_cli_set_terminal_pager (vlib_main_t * vm, unix_cli_main_t *cm = &unix_cli_main; unix_cli_file_t *cf; unformat_input_t _line_input, *line_input = &_line_input; + clib_error_t *error = 0; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -2852,13 +2853,17 @@ unix_cli_set_terminal_pager (vlib_main_t * vm, "Pager limit set to %u lines; note, this is global.\n", um->cli_pager_buffer_limit); else - return clib_error_return (0, "unknown parameter: `%U`", - format_unformat_error, line_input); + { + error = clib_error_return (0, "unknown parameter: `%U`", + format_unformat_error, line_input); + goto done; + } } +done: unformat_free (line_input); - return 0; + return error; } /*? @@ -2886,6 +2891,7 @@ unix_cli_set_terminal_history (vlib_main_t * vm, unix_cli_file_t *cf; unformat_input_t _line_input, *line_input = &_line_input; u32 limit; + clib_error_t *error = 0; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -2901,8 +2907,11 @@ unix_cli_set_terminal_history (vlib_main_t * vm, else if (unformat (line_input, "limit %u", &cf->history_limit)) ; else - return clib_error_return (0, "unknown parameter: `%U`", - format_unformat_error, line_input); + { + error = clib_error_return (0, "unknown parameter: `%U`", + format_unformat_error, line_input); + goto done; + } /* If we reduced history size, or turned it off, purge the history */ limit = cf->has_history ? cf->history_limit : 0; @@ -2914,9 +2923,10 @@ unix_cli_set_terminal_history (vlib_main_t * vm, } } +done: unformat_free (line_input); - return 0; + return error; } /*? diff --git a/src/vnet/devices/af_packet/cli.c b/src/vnet/devices/af_packet/cli.c index 6baa26e1..d4aa7016 100644 --- a/src/vnet/devices/af_packet/cli.c +++ b/src/vnet/devices/af_packet/cli.c @@ -49,6 +49,7 @@ af_packet_create_command_fn (vlib_main_t * vm, unformat_input_t * input, u8 *hw_addr_ptr = 0; u32 sw_if_index; int r; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -63,29 +64,47 @@ af_packet_create_command_fn (vlib_main_t * vm, unformat_input_t * input, (line_input, "hw-addr %U", unformat_ethernet_address, hwaddr)) hw_addr_ptr = hwaddr; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (host_if_name == NULL) - return clib_error_return (0, "missing host interface name"); + { + error = clib_error_return (0, "missing host interface name"); + goto done; + } r = af_packet_create_if (vm, host_if_name, hw_addr_ptr, &sw_if_index); - vec_free (host_if_name); if (r == VNET_API_ERROR_SYSCALL_ERROR_1) - return clib_error_return (0, "%s (errno %d)", strerror (errno), errno); + { + error = clib_error_return (0, "%s (errno %d)", strerror (errno), errno); + goto done; + } if (r == VNET_API_ERROR_INVALID_INTERFACE) - return clib_error_return (0, "Invalid interface name"); + { + error = clib_error_return (0, "Invalid interface name"); + goto done; + } if (r == VNET_API_ERROR_SUBIF_ALREADY_EXISTS) - return clib_error_return (0, "Interface elready exists"); + { + error = clib_error_return (0, "Interface elready exists"); + goto done; + } vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main (), sw_if_index); - return 0; + +done: + vec_free (host_if_name); + unformat_free (line_input); + + return error; } /*? @@ -124,6 +143,7 @@ af_packet_delete_command_fn (vlib_main_t * vm, unformat_input_t * input, { unformat_input_t _line_input, *line_input = &_line_input; u8 *host_if_name = NULL; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -134,18 +154,26 @@ af_packet_delete_command_fn (vlib_main_t * vm, unformat_input_t * input, if (unformat (line_input, "name %s", &host_if_name)) ; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (host_if_name == NULL) - return clib_error_return (0, "missing host interface name"); + { + error = clib_error_return (0, "missing host interface name"); + goto done; + } af_packet_delete_if (vm, host_if_name); + +done: vec_free (host_if_name); + unformat_free (line_input); - return 0; + return error; } /*? diff --git a/src/vnet/devices/dpdk/cli.c b/src/vnet/devices/dpdk/cli.c index d133cfd9..1fc665ac 100644 --- a/src/vnet/devices/dpdk/cli.c +++ b/src/vnet/devices/dpdk/cli.c @@ -398,7 +398,7 @@ set_dpdk_if_desc (vlib_main_t * vm, unformat_input_t * input, u32 hw_if_index = (u32) ~ 0; u32 nb_rx_desc = (u32) ~ 0; u32 nb_tx_desc = (u32) ~ 0; - clib_error_t *rv; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -414,25 +414,37 @@ set_dpdk_if_desc (vlib_main_t * vm, unformat_input_t * input, else if (unformat (line_input, "rx %d", &nb_rx_desc)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); + { + error = clib_error_return (0, "please specify valid interface name"); + goto done; + } hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); xd = vec_elt_at_index (dm->devices, hw->dev_instance); if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0) - return clib_error_return (0, "number of descriptors can be set only for " - "physical devices"); + { + error = + clib_error_return (0, + "number of descriptors can be set only for " + "physical devices"); + goto done; + } if ((nb_rx_desc == (u32) ~ 0 || nb_rx_desc == xd->nb_rx_desc) && (nb_tx_desc == (u32) ~ 0 || nb_tx_desc == xd->nb_tx_desc)) - return clib_error_return (0, "nothing changed"); + { + error = clib_error_return (0, "nothing changed"); + goto done; + } if (nb_rx_desc != (u32) ~ 0) xd->nb_rx_desc = nb_rx_desc; @@ -440,9 +452,12 @@ set_dpdk_if_desc (vlib_main_t * vm, unformat_input_t * input, if (nb_tx_desc != (u32) ~ 0) xd->nb_tx_desc = nb_tx_desc; - rv = dpdk_port_setup (dm, xd); + error = dpdk_port_setup (dm, xd); + +done: + unformat_free (line_input); - return rv; + return error; } /* *INDENT-OFF* */ @@ -523,6 +538,7 @@ set_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input, u32 queue = (u32) 0; u32 cpu = (u32) ~ 0; int i; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -538,18 +554,25 @@ set_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input, else if (unformat (line_input, "thread %d", &cpu)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); + { + error = clib_error_return (0, "please specify valid interface name"); + goto done; + } if (cpu < dm->input_cpu_first_index || cpu >= (dm->input_cpu_first_index + dm->input_cpu_count)) - return clib_error_return (0, "please specify valid thread id"); + { + error = clib_error_return (0, "please specify valid thread id"); + goto done; + } hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); xd = vec_elt_at_index (dm->devices, hw->dev_instance); @@ -563,7 +586,7 @@ set_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input, queue == dq->queue_id) { if (cpu == i) /* nothing to do */ - return 0; + goto done; vec_del1(dm->devices_by_cpu[i], dq - dm->devices_by_cpu[i]); vec_add2(dm->devices_by_cpu[cpu], dq, 1); @@ -586,13 +609,18 @@ set_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input, vlib_node_set_state (vlib_mains[cpu], dpdk_input_node.index, VLIB_NODE_STATE_POLLING); - return 0; + goto done; } } /* *INDENT-ON* */ } - return clib_error_return (0, "not found"); + error = clib_error_return (0, "not found"); + +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -653,6 +681,7 @@ set_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input, u32 hw_if_index = (u32) ~ 0; u32 cpu = (u32) ~ 0; int i; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -666,18 +695,22 @@ set_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input, else if (unformat (line_input, "thread %d", &cpu)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (hw_if_index == (u32) ~ 0) return clib_error_return (0, "please specify valid interface name"); if (cpu < dm->hqos_cpu_first_index || cpu >= (dm->hqos_cpu_first_index + dm->hqos_cpu_count)) - return clib_error_return (0, "please specify valid thread id"); + { + error = clib_error_return (0, "please specify valid thread id"); + goto done; + } hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); xd = vec_elt_at_index (dm->devices, hw->dev_instance); @@ -689,7 +722,7 @@ set_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input, if (hw_if_index == dm->devices[dq->device].vlib_hw_if_index) { if (cpu == i) /* nothing to do */ - return 0; + goto done; vec_del1 (dm->devices_by_hqos_cpu[i], dq - dm->devices_by_hqos_cpu[i]); @@ -703,12 +736,17 @@ set_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input, vec_sort_with_function (dm->devices_by_hqos_cpu[cpu], dpdk_device_queue_sort); - return 0; + goto done; } } } - return clib_error_return (0, "not found"); + error = clib_error_return (0, "not found"); + +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -732,6 +770,7 @@ set_dpdk_if_hqos_pipe (vlib_main_t * vm, unformat_input_t * input, u32 pipe_id = (u32) ~ 0; u32 profile_id = (u32) ~ 0; int rv; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -749,14 +788,18 @@ set_dpdk_if_hqos_pipe (vlib_main_t * vm, unformat_input_t * input, else if (unformat (line_input, "profile %d", &profile_id)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); + { + error = clib_error_return (0, "please specify valid interface name"); + goto done; + } hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); xd = vec_elt_at_index (dm->devices, hw->dev_instance); @@ -765,9 +808,15 @@ set_dpdk_if_hqos_pipe (vlib_main_t * vm, unformat_input_t * input, rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id, profile_id); if (rv) - return clib_error_return (0, "pipe configuration failed"); + { + error = clib_error_return (0, "pipe configuration failed"); + goto done; + } - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -797,6 +846,7 @@ set_dpdk_if_hqos_subport (vlib_main_t * vm, unformat_input_t * input, .tc_period = 10, }; int rv; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -829,23 +879,33 @@ set_dpdk_if_hqos_subport (vlib_main_t * vm, unformat_input_t * input, else if (unformat (line_input, "period %d", &p.tc_period)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); + { + error = clib_error_return (0, "please specify valid interface name"); + goto done; + } hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); xd = vec_elt_at_index (dm->devices, hw->dev_instance); rv = rte_sched_subport_config (xd->hqos_ht->hqos, subport_id, &p); if (rv) - return clib_error_return (0, "subport configuration failed"); + { + error = clib_error_return (0, "subport configuration failed"); + goto done; + } - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -872,6 +932,7 @@ set_dpdk_if_hqos_tctbl (vlib_main_t * vm, unformat_input_t * input, u32 queue = (u32) ~ 0; u32 entry = (u32) ~ 0; u32 val, i; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -889,20 +950,33 @@ set_dpdk_if_hqos_tctbl (vlib_main_t * vm, unformat_input_t * input, else if (unformat (line_input, "queue %d", &queue)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); + { + error = clib_error_return (0, "please specify valid interface name"); + goto done; + } if (entry >= 64) - return clib_error_return (0, "invalid entry"); + { + error = clib_error_return (0, "invalid entry"); + goto done; + } if (tc >= RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE) - return clib_error_return (0, "invalid traffic class"); + { + error = clib_error_return (0, "invalid traffic class"); + goto done; + } if (queue >= RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS) - return clib_error_return (0, "invalid traffic class"); + { + error = clib_error_return (0, "invalid traffic class"); + goto done; + } hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); xd = vec_elt_at_index (dm->devices, hw->dev_instance); @@ -911,7 +985,10 @@ set_dpdk_if_hqos_tctbl (vlib_main_t * vm, unformat_input_t * input, uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers"); /* Should never happen, shut up Coverity warning */ if (p == 0) - return clib_error_return (0, "no worker registrations?"); + { + error = clib_error_return (0, "no worker registrations?"); + goto done; + } vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0]; int worker_thread_first = tr->first_index; @@ -921,7 +998,10 @@ set_dpdk_if_hqos_tctbl (vlib_main_t * vm, unformat_input_t * input, for (i = 0; i < worker_thread_count; i++) xd->hqos_wt[worker_thread_first + i].hqos_tc_table[entry] = val; - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -939,6 +1019,7 @@ set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input, unformat_input_t _line_input, *line_input = &_line_input; vlib_thread_main_t *tm = vlib_get_thread_main (); dpdk_main_t *dm = &dpdk_main; + clib_error_t *error = NULL; /* Device specific data */ struct rte_eth_dev_info dev_info; @@ -984,15 +1065,19 @@ set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input, else if (unformat (line_input, "mask %llx", &mask)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - /* Get interface */ if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); + { + error = clib_error_return (0, "please specify valid interface name"); + goto done; + } hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); xd = vec_elt_at_index (dm->devices, hw->dev_instance); @@ -1019,7 +1104,7 @@ set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input, if (devconf->hqos_enabled == 0) { vlib_cli_output (vm, "HQoS disabled for this interface"); - return 0; + goto done; } n_subports_per_port = devconf->hqos.port.n_subports_per_port; @@ -1028,27 +1113,39 @@ set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input, /* Validate packet field configuration: id, offset and mask */ if (id >= 3) - return clib_error_return (0, "invalid packet field id"); + { + error = clib_error_return (0, "invalid packet field id"); + goto done; + } switch (id) { case 0: if (dpdk_hqos_validate_mask (mask, n_subports_per_port) != 0) - return clib_error_return (0, "invalid subport ID mask " - "(n_subports_per_port = %u)", - n_subports_per_port); + { + error = clib_error_return (0, "invalid subport ID mask " + "(n_subports_per_port = %u)", + n_subports_per_port); + goto done; + } break; case 1: if (dpdk_hqos_validate_mask (mask, n_pipes_per_subport) != 0) - return clib_error_return (0, "invalid pipe ID mask " - "(n_pipes_per_subport = %u)", - n_pipes_per_subport); + { + error = clib_error_return (0, "invalid pipe ID mask " + "(n_pipes_per_subport = %u)", + n_pipes_per_subport); + goto done; + } break; case 2: default: if (dpdk_hqos_validate_mask (mask, tctbl_size) != 0) - return clib_error_return (0, "invalid TC table index mask " - "(TC table size = %u)", tctbl_size); + { + error = clib_error_return (0, "invalid TC table index mask " + "(TC table size = %u)", tctbl_size); + goto done; + } } /* Propagate packet field configuration to all workers */ @@ -1075,7 +1172,10 @@ set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input, __builtin_ctzll (mask); } - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -1106,6 +1206,7 @@ show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input, dpdk_device_config_t *devconf = 0; vlib_thread_registration_t *tr; uword *p = 0; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -1117,14 +1218,18 @@ show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input, &hw_if_index)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify interface name!!"); + { + error = clib_error_return (0, "please specify interface name!!"); + goto done; + } hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); xd = vec_elt_at_index (dm->devices, hw->dev_instance); @@ -1151,7 +1256,7 @@ show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input, if (devconf->hqos_enabled == 0) { vlib_cli_output (vm, "HQoS disabled for this interface"); - return 0; + goto done; } /* Detect the set of worker threads */ @@ -1159,7 +1264,10 @@ show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input, /* Should never happen, shut up Coverity warning */ if (p == 0) - return clib_error_return (0, "no worker registrations?"); + { + error = clib_error_return (0, "no worker registrations?"); + goto done; + } tr = (vlib_thread_registration_t *) p[0]; @@ -1284,7 +1392,10 @@ show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input, } #endif - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -1315,6 +1426,7 @@ show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input, u32 qindex; struct rte_sched_queue_stats stats; u16 qlen; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -1339,14 +1451,18 @@ show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input, ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify interface name!!"); + { + error = clib_error_return (0, "please specify interface name!!"); + goto done; + } hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); xd = vec_elt_at_index (dm->devices, hw->dev_instance); @@ -1373,7 +1489,7 @@ show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input, if (devconf->hqos_enabled == 0) { vlib_cli_output (vm, "HQoS disabled for this interface"); - return 0; + goto done; } /* @@ -1386,7 +1502,10 @@ show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input, if (rte_sched_queue_read_stats (xd->hqos_ht->hqos, qindex, &stats, &qlen) != 0) - return clib_error_return (0, "failed to read stats"); + { + error = clib_error_return (0, "failed to read stats"); + goto done; + } vlib_cli_output (vm, "%=24s%=16s", "Stats Parameter", "Value"); vlib_cli_output (vm, "%=24s%=16d", "Packets", stats.n_pkts); @@ -1399,7 +1518,10 @@ show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input, vlib_cli_output (vm, "%=24s%=16d", "Bytes dropped", stats.n_bytes_dropped); - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ diff --git a/src/vnet/devices/dpdk/ipsec/cli.c b/src/vnet/devices/dpdk/ipsec/cli.c index 93df4a64..f9d3a5d0 100644 --- a/src/vnet/devices/dpdk/ipsec/cli.c +++ b/src/vnet/devices/dpdk/ipsec/cli.c @@ -111,6 +111,7 @@ lcore_cryptodev_map_fn (vlib_main_t * vm, unformat_input_t * input, { unformat_input_t _line_input, *line_input = &_line_input; u16 detail = 0; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -120,15 +121,19 @@ lcore_cryptodev_map_fn (vlib_main_t * vm, unformat_input_t * input, if (unformat (line_input, "verbose")) detail = 1; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - dpdk_ipsec_show_mapping (vm, detail); - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ diff --git a/src/vnet/devices/netmap/cli.c b/src/vnet/devices/netmap/cli.c index 6157f27c..71363294 100644 --- a/src/vnet/devices/netmap/cli.c +++ b/src/vnet/devices/netmap/cli.c @@ -37,6 +37,7 @@ netmap_create_command_fn (vlib_main_t * vm, unformat_input_t * input, u8 is_pipe = 0; u8 is_master = 0; u32 sw_if_index = ~0; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -57,30 +58,48 @@ netmap_create_command_fn (vlib_main_t * vm, unformat_input_t * input, else if (unformat (line_input, "slave")) is_master = 0; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (host_if_name == NULL) - return clib_error_return (0, "missing host interface name"); + { + error = clib_error_return (0, "missing host interface name"); + goto done; + } r = netmap_create_if (vm, host_if_name, hw_addr_ptr, is_pipe, is_master, &sw_if_index); if (r == VNET_API_ERROR_SYSCALL_ERROR_1) - return clib_error_return (0, "%s (errno %d)", strerror (errno), errno); + { + error = clib_error_return (0, "%s (errno %d)", strerror (errno), errno); + goto done; + } if (r == VNET_API_ERROR_INVALID_INTERFACE) - return clib_error_return (0, "Invalid interface name"); + { + error = clib_error_return (0, "Invalid interface name"); + goto done; + } if (r == VNET_API_ERROR_SUBIF_ALREADY_EXISTS) - return clib_error_return (0, "Interface already exists"); + { + error = clib_error_return (0, "Interface already exists"); + goto done; + } vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main (), sw_if_index); - return 0; + +done: + unformat_free (line_input); + + return error; } /*? @@ -144,6 +163,7 @@ netmap_delete_command_fn (vlib_main_t * vm, unformat_input_t * input, { unformat_input_t _line_input, *line_input = &_line_input; u8 *host_if_name = NULL; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -154,17 +174,25 @@ netmap_delete_command_fn (vlib_main_t * vm, unformat_input_t * input, if (unformat (line_input, "name %s", &host_if_name)) ; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (host_if_name == NULL) - return clib_error_return (0, "missing host interface name"); + { + error = clib_error_return (0, "missing host interface name"); + goto done; + } netmap_delete_if (vm, host_if_name); - return 0; +done: + unformat_free (line_input); + + return error; } /*? diff --git a/src/vnet/devices/virtio/vhost-user.c b/src/vnet/devices/virtio/vhost-user.c index 315daa77..c43f6e67 100644 --- a/src/vnet/devices/virtio/vhost-user.c +++ b/src/vnet/devices/virtio/vhost-user.c @@ -2682,6 +2682,7 @@ vhost_user_connect_command_fn (vlib_main_t * vm, u32 custom_dev_instance = ~0; u8 hwaddr[6]; u8 *hw = NULL; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -2704,10 +2705,12 @@ vhost_user_connect_command_fn (vlib_main_t * vm, renumber = 1; } else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); vnet_main_t *vnm = vnet_get_main (); @@ -2716,14 +2719,18 @@ vhost_user_connect_command_fn (vlib_main_t * vm, is_server, &sw_if_index, feature_mask, renumber, custom_dev_instance, hw))) { - vec_free (sock_filename); - return clib_error_return (0, "vhost_user_create_if returned %d", rv); + error = clib_error_return (0, "vhost_user_create_if returned %d", rv); + goto done; } - vec_free (sock_filename); vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main (), sw_if_index); - return 0; + +done: + vec_free (sock_filename); + unformat_free (line_input); + + return error; } clib_error_t * @@ -2734,6 +2741,7 @@ vhost_user_delete_command_fn (vlib_main_t * vm, unformat_input_t _line_input, *line_input = &_line_input; u32 sw_if_index = ~0; vnet_main_t *vnm = vnet_get_main (); + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -2751,15 +2759,25 @@ vhost_user_delete_command_fn (vlib_main_t * vm, vnet_get_sup_hw_interface (vnm, sw_if_index); if (hwif == NULL || vhost_user_dev_class.index != hwif->dev_class_index) - return clib_error_return (0, "Not a vhost interface"); + { + error = clib_error_return (0, "Not a vhost interface"); + goto done; + } } else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); + vhost_user_delete_if (vnm, vm, sw_if_index); - return 0; + +done: + unformat_free (line_input); + + return error; } int @@ -3286,6 +3304,7 @@ vhost_thread_command_fn (vlib_main_t * vm, u32 sw_if_index; u8 del = 0; int rv; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -3295,9 +3314,9 @@ vhost_thread_command_fn (vlib_main_t * vm, (line_input, "%U %d", unformat_vnet_sw_interface, vnet_get_main (), &sw_if_index, &worker_thread_index)) { - unformat_free (line_input); - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; } if (unformat (line_input, "del")) @@ -3305,9 +3324,16 @@ vhost_thread_command_fn (vlib_main_t * vm, if ((rv = vhost_user_thread_placement (sw_if_index, worker_thread_index, del))) - return clib_error_return (0, "vhost_user_thread_placement returned %d", - rv); - return 0; + { + error = clib_error_return (0, "vhost_user_thread_placement returned %d", + rv); + goto done; + } + +done: + unformat_free (line_input); + + return error; } diff --git a/src/vnet/gre/interface.c b/src/vnet/gre/interface.c index d624587d..d4476ac4 100644 --- a/src/vnet/gre/interface.c +++ b/src/vnet/gre/interface.c @@ -491,6 +491,7 @@ create_gre_tunnel_command_fn (vlib_main_t * vm, u32 num_m_args = 0; u8 is_add = 1; u32 sw_if_index; + clib_error_t *error = NULL; /* Get a line of input. */ if (! unformat_user (input, unformat_line_input, line_input)) @@ -508,16 +509,24 @@ create_gre_tunnel_command_fn (vlib_main_t * vm, else if (unformat (line_input, "teb")) teb = 1; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (num_m_args < 2) - return clib_error_return (0, "mandatory argument(s) missing"); + { + error = clib_error_return (0, "mandatory argument(s) missing"); + goto done; + } if (memcmp (&src, &dst, sizeof(src)) == 0) - return clib_error_return (0, "src and dst are identical"); + { + error = clib_error_return (0, "src and dst are identical"); + goto done; + } memset (a, 0, sizeof (*a)); a->outer_fib_id = outer_fib_id; @@ -536,15 +545,21 @@ create_gre_tunnel_command_fn (vlib_main_t * vm, vlib_cli_output(vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main(), sw_if_index); break; case VNET_API_ERROR_INVALID_VALUE: - return clib_error_return (0, "GRE tunnel already exists..."); + error = clib_error_return (0, "GRE tunnel already exists..."); + goto done; case VNET_API_ERROR_NO_SUCH_FIB: - return clib_error_return (0, "outer fib ID %d doesn't exist\n", - outer_fib_id); + error = clib_error_return (0, "outer fib ID %d doesn't exist\n", + outer_fib_id); + goto done; default: - return clib_error_return (0, "vnet_gre_add_del_tunnel returned %d", rv); + error = clib_error_return (0, "vnet_gre_add_del_tunnel returned %d", rv); + goto done; } - return 0; +done: + unformat_free (line_input); + + return error; } VLIB_CLI_COMMAND (create_gre_tunnel_command, static) = { diff --git a/src/vnet/ip/ip4_source_check.c b/src/vnet/ip/ip4_source_check.c index d461cc88..3af32f2e 100644 --- a/src/vnet/ip/ip4_source_check.c +++ b/src/vnet/ip/ip4_source_check.c @@ -399,6 +399,8 @@ set_ip_source_check (vlib_main_t * vm, vnet_feature_enable_disable ("ip4-unicast", feature_name, sw_if_index, is_del == 0, &config, sizeof (config)); done: + unformat_free (line_input); + return error; } @@ -531,7 +533,9 @@ ip_source_check_accept (vlib_main_t * vm, } done: - return (error); + unformat_free (line_input); + + return error; } /*? diff --git a/src/vnet/ip/ip4_test.c b/src/vnet/ip/ip4_test.c index 45d17113..73dabfdc 100644 --- a/src/vnet/ip/ip4_test.c +++ b/src/vnet/ip/ip4_test.c @@ -143,8 +143,11 @@ thrash (vlib_main_t * vm, else if (unformat (line_input, "verbose")) verbose = 1; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } } @@ -178,7 +181,7 @@ thrash (vlib_main_t * vm, if (p == 0) { vlib_cli_output (vm, "Couldn't map fib id %d to fib index\n", table_id); - return 0; + goto done; } table_index = p[0]; @@ -294,7 +297,11 @@ thrash (vlib_main_t * vm, pool_free (tm->route_pool); } - return 0; + +done: + unformat_free (line_input); + + return error; } /*? diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c index 7229591e..6b53137f 100644 --- a/src/vnet/ip/ip6_neighbor.c +++ b/src/vnet/ip/ip6_neighbor.c @@ -2923,7 +2923,10 @@ ip6_neighbor_cmd (vlib_main_t * vm, unformat_input_t * main_input, else if (unformat (line_input, "ra-lifetime")) { if (!unformat (line_input, "%d", &ra_lifetime)) - return (error = unformat_parse_error (line_input)); + { + error = unformat_parse_error (line_input); + goto done; + } use_lifetime = 1; break; } @@ -2931,13 +2934,19 @@ ip6_neighbor_cmd (vlib_main_t * vm, unformat_input_t * main_input, { if (!unformat (line_input, "%d %d", &ra_initial_count, &ra_initial_interval)) - return (error = unformat_parse_error (line_input)); + { + error = unformat_parse_error (line_input); + goto done; + } break; } else if (unformat (line_input, "ra-interval")) { if (!unformat (line_input, "%d", &ra_max_interval)) - return (error = unformat_parse_error (line_input)); + { + error = unformat_parse_error (line_input); + goto done; + } if (!unformat (line_input, "%d", &ra_min_interval)) ra_min_interval = 0; @@ -2949,7 +2958,10 @@ ip6_neighbor_cmd (vlib_main_t * vm, unformat_input_t * main_input, break; } else - return (unformat_parse_error (line_input)); + { + error = unformat_parse_error (line_input); + goto done; + } } if (add_radv_info) @@ -3006,7 +3018,10 @@ ip6_neighbor_cmd (vlib_main_t * vm, unformat_input_t * main_input, else if (unformat (line_input, "no-onlink")) no_onlink = 1; else - return (unformat_parse_error (line_input)); + { + error = unformat_parse_error (line_input); + goto done; + } } ip6_neighbor_ra_prefix (vm, sw_if_index, @@ -3018,9 +3033,9 @@ ip6_neighbor_cmd (vlib_main_t * vm, unformat_input_t * main_input, off_link, no_autoconfig, no_onlink, is_no); } +done: unformat_free (line_input); -done: return error; } diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c index 0ef0e7a6..807b87b6 100644 --- a/src/vnet/ip/lookup.c +++ b/src/vnet/ip/lookup.c @@ -568,8 +568,6 @@ vnet_ip_route_cmd (vlib_main_t * vm, } } - unformat_free (line_input); - if (vec_len (prefixs) == 0) { error = @@ -704,6 +702,7 @@ done: vec_free (dpos); vec_free (prefixs); vec_free (rpaths); + unformat_free (line_input); return error; } @@ -872,8 +871,6 @@ vnet_ip_mroute_cmd (vlib_main_t * vm, } } - unformat_free (line_input); - if (~0 == table_id) { /* @@ -970,6 +967,8 @@ vnet_ip_mroute_cmd (vlib_main_t * vm, (scount * gcount) / (timet[1] - timet[0])); done: + unformat_free (line_input); + return error; } @@ -1149,24 +1148,37 @@ probe_neighbor_address (vlib_main_t * vm, is_ip4 = 0; } else - return clib_error_return (0, "unknown input '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (sw_if_index == ~0) - return clib_error_return (0, "Interface required, not set."); + { + error = clib_error_return (0, "Interface required, not set."); + goto done; + } if (address_set == 0) - return clib_error_return (0, "ip address required, not set."); + { + error = clib_error_return (0, "ip address required, not set."); + goto done; + } if (address_set > 1) - return clib_error_return (0, "Multiple ip addresses not supported."); + { + error = clib_error_return (0, "Multiple ip addresses not supported."); + goto done; + } if (is_ip4) error = ip4_probe_neighbor_wait (vm, &a4, sw_if_index, retry_count); else error = ip6_probe_neighbor_wait (vm, &a6, sw_if_index, retry_count); +done: + unformat_free (line_input); + return error; } diff --git a/src/vnet/ipsec-gre/interface.c b/src/vnet/ipsec-gre/interface.c index 3b6e4ac2..0772ce73 100644 --- a/src/vnet/ipsec-gre/interface.c +++ b/src/vnet/ipsec-gre/interface.c @@ -232,6 +232,7 @@ create_ipsec_gre_tunnel_command_fn (vlib_main_t * vm, vnet_ipsec_gre_add_del_tunnel_args_t _a, *a = &_a; int rv; u32 sw_if_index; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -250,16 +251,24 @@ create_ipsec_gre_tunnel_command_fn (vlib_main_t * vm, else if (unformat (line_input, "remote-sa %d", &rsa)) num_m_args++; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (num_m_args < 4) - return clib_error_return (0, "mandatory argument(s) missing"); + { + error = clib_error_return (0, "mandatory argument(s) missing"); + goto done; + } if (memcmp (&src, &dst, sizeof (src)) == 0) - return clib_error_return (0, "src and dst are identical"); + { + error = clib_error_return (0, "src and dst are identical"); + goto done; + } memset (a, 0, sizeof (*a)); a->is_add = is_add; @@ -277,14 +286,19 @@ create_ipsec_gre_tunnel_command_fn (vlib_main_t * vm, vnet_get_main (), sw_if_index); break; case VNET_API_ERROR_INVALID_VALUE: - return clib_error_return (0, "GRE tunnel already exists..."); + error = clib_error_return (0, "GRE tunnel already exists..."); + goto done; default: - return clib_error_return (0, - "vnet_ipsec_gre_add_del_tunnel returned %d", - rv); + error = clib_error_return (0, + "vnet_ipsec_gre_add_del_tunnel returned %d", + rv); + goto done; } - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ diff --git a/src/vnet/ipsec/ipsec_cli.c b/src/vnet/ipsec/ipsec_cli.c index 3c1e26f2..0e034402 100644 --- a/src/vnet/ipsec/ipsec_cli.c +++ b/src/vnet/ipsec/ipsec_cli.c @@ -32,6 +32,7 @@ set_interface_spd_command_fn (vlib_main_t * vm, u32 sw_if_index = (u32) ~ 0; u32 spd_id; int is_add = 1; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -43,14 +44,18 @@ set_interface_spd_command_fn (vlib_main_t * vm, else if (unformat (line_input, "del")) is_add = 0; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - - unformat_free (line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } ipsec_set_interface_spd (vm, sw_if_index, spd_id, is_add); - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -72,7 +77,7 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm, ipsec_sa_t sa; int is_add = ~0; u8 *ck = 0, *ik = 0; - clib_error_t *err = 0; + clib_error_t *error = NULL; memset (&sa, 0, sizeof (sa)); @@ -90,8 +95,11 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm, else if (unformat (line_input, "esp")) sa.protocol = IPSEC_PROTOCOL_ESP; else if (unformat (line_input, "ah")) - //sa.protocol = IPSEC_PROTOCOL_AH; - return clib_error_return (0, "unsupported security protocol 'AH'"); + { + //sa.protocol = IPSEC_PROTOCOL_AH; + error = clib_error_return (0, "unsupported security protocol 'AH'"); + goto done; + } else if (unformat (line_input, "crypto-key %U", unformat_hex_string, &ck)) sa.crypto_key_len = vec_len (ck); @@ -102,8 +110,12 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm, { if (sa.crypto_alg < IPSEC_CRYPTO_ALG_AES_CBC_128 || sa.crypto_alg >= IPSEC_CRYPTO_N_ALG) - return clib_error_return (0, "unsupported crypto-alg: '%U'", - format_ipsec_crypto_alg, sa.crypto_alg); + { + error = clib_error_return (0, "unsupported crypto-alg: '%U'", + format_ipsec_crypto_alg, + sa.crypto_alg); + goto done; + } } else if (unformat (line_input, "integ-key %U", unformat_hex_string, &ik)) @@ -113,8 +125,12 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm, { if (sa.integ_alg < IPSEC_INTEG_ALG_SHA1_96 || sa.integ_alg >= IPSEC_INTEG_N_ALG) - return clib_error_return (0, "unsupported integ-alg: '%U'", - format_ipsec_integ_alg, sa.integ_alg); + { + error = clib_error_return (0, "unsupported integ-alg: '%U'", + format_ipsec_integ_alg, + sa.integ_alg); + goto done; + } } else if (unformat (line_input, "tunnel-src %U", unformat_ip4_address, &sa.tunnel_src_addr.ip4)) @@ -135,12 +151,13 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm, sa.is_tunnel_ip6 = 1; } else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (sa.crypto_key_len > sizeof (sa.crypto_key)) sa.crypto_key_len = sizeof (sa.crypto_key); @@ -156,14 +173,17 @@ ipsec_sa_add_del_command_fn (vlib_main_t * vm, if (is_add) { ASSERT (im->cb.check_support_cb); - err = im->cb.check_support_cb (&sa); - if (err) - return err; + error = im->cb.check_support_cb (&sa); + if (error) + goto done; } ipsec_add_del_sa (vm, &sa, is_add); - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -183,6 +203,7 @@ ipsec_spd_add_del_command_fn (vlib_main_t * vm, unformat_input_t _line_input, *line_input = &_line_input; u32 spd_id = ~0; int is_add = ~0; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -196,18 +217,25 @@ ipsec_spd_add_del_command_fn (vlib_main_t * vm, else if (unformat (line_input, "%u", &spd_id)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (spd_id == ~0) - return clib_error_return (0, "please specify SPD ID"); + { + error = clib_error_return (0, "please specify SPD ID"); + goto done; + } ipsec_add_del_spd (vm, spd_id, is_add); - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -230,6 +258,7 @@ ipsec_policy_add_del_command_fn (vlib_main_t * vm, int is_add = 0; int is_ip_any = 1; u32 tmp, tmp2; + clib_error_t *error = NULL; memset (&p, 0, sizeof (p)); p.lport.stop = p.rport.stop = ~0; @@ -262,7 +291,10 @@ ipsec_policy_add_del_command_fn (vlib_main_t * vm, &p.policy)) { if (p.policy == IPSEC_POLICY_ACTION_RESOLVE) - return clib_error_return (0, "unsupported action: 'resolve'"); + { + error = clib_error_return (0, "unsupported action: 'resolve'"); + goto done; + } } else if (unformat (line_input, "sa %u", &p.sa_id)) ; @@ -300,19 +332,24 @@ ipsec_policy_add_del_command_fn (vlib_main_t * vm, p.rport.stop = tmp2; } else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - ipsec_add_del_policy (vm, &p, is_add); if (is_ip_any) { p.is_ipv6 = 1; ipsec_add_del_policy (vm, &p, is_add); } - return 0; + +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -332,6 +369,7 @@ set_ipsec_sa_key_command_fn (vlib_main_t * vm, unformat_input_t _line_input, *line_input = &_line_input; ipsec_sa_t sa; u8 *ck = 0, *ik = 0; + clib_error_t *error = NULL; memset (&sa, 0, sizeof (sa)); @@ -349,12 +387,13 @@ set_ipsec_sa_key_command_fn (vlib_main_t * vm, if (unformat (line_input, "integ-key %U", unformat_hex_string, &ik)) sa.integ_key_len = vec_len (ik); else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (sa.crypto_key_len > sizeof (sa.crypto_key)) sa.crypto_key_len = sizeof (sa.crypto_key); @@ -369,7 +408,10 @@ set_ipsec_sa_key_command_fn (vlib_main_t * vm, ipsec_set_sa_key (vm, &sa); - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -649,6 +691,7 @@ create_ipsec_tunnel_command_fn (vlib_main_t * vm, ipsec_add_del_tunnel_args_t a; int rv; u32 num_m_args = 0; + clib_error_t *error = NULL; memset (&a, 0, sizeof (a)); a.is_add = 1; @@ -673,13 +716,18 @@ create_ipsec_tunnel_command_fn (vlib_main_t * vm, else if (unformat (line_input, "del")) a.is_add = 0; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (num_m_args < 4) - return clib_error_return (0, "mandatory argument(s) missing"); + { + error = clib_error_return (0, "mandatory argument(s) missing"); + goto done; + } rv = ipsec_add_del_tunnel_if (&a); @@ -689,16 +737,21 @@ create_ipsec_tunnel_command_fn (vlib_main_t * vm, break; case VNET_API_ERROR_INVALID_VALUE: if (a.is_add) - return clib_error_return (0, - "IPSec tunnel interface already exists..."); + error = clib_error_return (0, + "IPSec tunnel interface already exists..."); else - return clib_error_return (0, "IPSec tunnel interface not exists..."); + error = clib_error_return (0, "IPSec tunnel interface not exists..."); + goto done; default: - return clib_error_return (0, "ipsec_register_interface returned %d", - rv); + error = clib_error_return (0, "ipsec_register_interface returned %d", + rv); + goto done; } - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -720,6 +773,7 @@ set_interface_key_command_fn (vlib_main_t * vm, u32 hw_if_index = (u32) ~ 0; u32 alg; u8 *key = 0; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -748,25 +802,38 @@ set_interface_key_command_fn (vlib_main_t * vm, else if (unformat (line_input, "%U", unformat_hex_string, &key)) ; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (type == IPSEC_IF_SET_KEY_TYPE_NONE) - return clib_error_return (0, "unknown key type"); + { + error = clib_error_return (0, "unknown key type"); + goto done; + } if (alg > 0 && vec_len (key) == 0) - return clib_error_return (0, "key is not specified"); + { + error = clib_error_return (0, "key is not specified"); + goto done; + } if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "interface not specified"); + { + error = clib_error_return (0, "interface not specified"); + goto done; + } ipsec_set_interface_key (im->vnet_main, hw_if_index, type, alg, key); + +done: vec_free (key); + unformat_free (line_input); - return 0; + return error; } /* *INDENT-OFF* */ diff --git a/src/vnet/l2/l2_patch.c b/src/vnet/l2/l2_patch.c index 5e4691f4..ff3d2f3a 100644 --- a/src/vnet/l2/l2_patch.c +++ b/src/vnet/l2/l2_patch.c @@ -315,6 +315,7 @@ test_patch_command_fn (vlib_main_t * vm, int rx_set = 0; int tx_set = 0; int is_add = 1; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -335,10 +336,16 @@ test_patch_command_fn (vlib_main_t * vm, } if (rx_set == 0) - return clib_error_return (0, "rx interface not set"); + { + error = clib_error_return (0, "rx interface not set"); + goto done; + } if (tx_set == 0) - return clib_error_return (0, "tx interface not set"); + { + error = clib_error_return (0, "tx interface not set"); + goto done; + } rv = vnet_l2_patch_add_del (rx_sw_if_index, tx_sw_if_index, is_add); @@ -348,17 +355,24 @@ test_patch_command_fn (vlib_main_t * vm, break; case VNET_API_ERROR_INVALID_SW_IF_INDEX: - return clib_error_return (0, "rx interface not a physical port"); + error = clib_error_return (0, "rx interface not a physical port"); + goto done; case VNET_API_ERROR_INVALID_SW_IF_INDEX_2: - return clib_error_return (0, "tx interface not a physical port"); + error = clib_error_return (0, "tx interface not a physical port"); + goto done; default: - return clib_error_return + error = clib_error_return (0, "WARNING: vnet_l2_patch_add_del returned %d", rv); + goto done; } - return 0; + +done: + unformat_free (line_input); + + return error; } /*? diff --git a/src/vnet/l2/l2_xcrw.c b/src/vnet/l2/l2_xcrw.c index 70610a85..d08a5d8f 100644 --- a/src/vnet/l2/l2_xcrw.c +++ b/src/vnet/l2/l2_xcrw.c @@ -409,6 +409,7 @@ set_l2_xcrw_command_fn (vlib_main_t * vm, u8 *rw = 0; vnet_main_t *vnm = vnet_get_main (); int rv; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) @@ -416,8 +417,11 @@ set_l2_xcrw_command_fn (vlib_main_t * vm, if (!unformat (line_input, "%U", unformat_vnet_sw_interface, vnm, &l2_sw_if_index)) - return clib_error_return (0, "unknown input '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + goto done; + } while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) { @@ -436,7 +440,10 @@ set_l2_xcrw_command_fn (vlib_main_t * vm, } if (next_node_index == ~0) - return clib_error_return (0, "next node not specified"); + { + error = clib_error_return (0, "next node not specified"); + goto done; + } if (tx_fib_id != ~0) { @@ -448,7 +455,11 @@ set_l2_xcrw_command_fn (vlib_main_t * vm, p = hash_get (ip4_main.fib_index_by_table_id, tx_fib_id); if (p == 0) - return clib_error_return (0, "nonexistent tx_fib_id %d", tx_fib_id); + { + error = + clib_error_return (0, "nonexistent tx_fib_id %d", tx_fib_id); + goto done; + } tx_fib_index = p[0]; } @@ -463,16 +474,21 @@ set_l2_xcrw_command_fn (vlib_main_t * vm, break; case VNET_API_ERROR_INVALID_SW_IF_INDEX: - return clib_error_return (0, "%U not cross-connected", - format_vnet_sw_if_index_name, - vnm, l2_sw_if_index); + error = clib_error_return (0, "%U not cross-connected", + format_vnet_sw_if_index_name, + vnm, l2_sw_if_index); + goto done; + default: - return clib_error_return (0, "vnet_configure_l2_xcrw returned %d", rv); + error = clib_error_return (0, "vnet_configure_l2_xcrw returned %d", rv); + goto done; } +done: vec_free (rw); + unformat_free (line_input); - return 0; + return error; } /*? diff --git a/src/vnet/l2tp/l2tp.c b/src/vnet/l2tp/l2tp.c index a4531dab..2d323397 100644 --- a/src/vnet/l2tp/l2tp.c +++ b/src/vnet/l2tp/l2tp.c @@ -427,6 +427,7 @@ create_l2tpv3_tunnel_command_fn (vlib_main_t * vm, u32 sw_if_index; u32 encap_fib_id = ~0; u32 encap_fib_index = ~0; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -455,18 +456,22 @@ create_l2tpv3_tunnel_command_fn (vlib_main_t * vm, else if (unformat (line_input, "l2-sublayer-present")) l2_sublayer_present = 1; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (encap_fib_id != ~0) { uword *p; ip6_main_t *im = &ip6_main; if (!(p = hash_get (im->fib_index_by_table_id, encap_fib_id))) - return clib_error_return (0, "No fib with id %d", encap_fib_id); + { + error = clib_error_return (0, "No fib with id %d", encap_fib_id); + goto done; + } encap_fib_index = p[0]; } else @@ -475,9 +480,15 @@ create_l2tpv3_tunnel_command_fn (vlib_main_t * vm, } if (our_address_set == 0) - return clib_error_return (0, "our address not specified"); + { + error = clib_error_return (0, "our address not specified"); + goto done; + } if (client_address_set == 0) - return clib_error_return (0, "client address not specified"); + { + error = clib_error_return (0, "client address not specified"); + goto done; + } rv = create_l2tpv3_ipv6_tunnel (lm, &client_address, &our_address, local_session_id, remote_session_id, @@ -491,16 +502,22 @@ create_l2tpv3_tunnel_command_fn (vlib_main_t * vm, vnet_get_main (), sw_if_index); break; case VNET_API_ERROR_INVALID_VALUE: - return clib_error_return (0, "session already exists..."); + error = clib_error_return (0, "session already exists..."); + goto done; case VNET_API_ERROR_NO_SUCH_ENTRY: - return clib_error_return (0, "session does not exist..."); + error = clib_error_return (0, "session does not exist..."); + goto done; default: - return clib_error_return (0, "l2tp_session_add_del returned %d", rv); + error = clib_error_return (0, "l2tp_session_add_del returned %d", rv); + goto done; } - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ diff --git a/src/vnet/lisp-cp/lisp_cli.c b/src/vnet/lisp-cp/lisp_cli.c index 25d11c61..05df9fb6 100644 --- a/src/vnet/lisp-cp/lisp_cli.c +++ b/src/vnet/lisp-cp/lisp_cli.c @@ -25,6 +25,7 @@ lisp_show_adjacencies_command_fn (vlib_main_t * vm, vlib_cli_output (vm, "%s %40s\n", "leid", "reid"); unformat_input_t _line_input, *line_input = &_line_input; u32 vni = ~0; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -38,14 +39,14 @@ lisp_show_adjacencies_command_fn (vlib_main_t * vm, { vlib_cli_output (vm, "parse error: '%U'", format_unformat_error, line_input); - return 0; + goto done; } } if (~0 == vni) { vlib_cli_output (vm, "error: no vni specified!"); - return 0; + goto done; } adjs = vnet_lisp_adjacencies_get_by_vni (vni); @@ -57,7 +58,10 @@ lisp_show_adjacencies_command_fn (vlib_main_t * vm, } vec_free (adjs); - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -77,6 +81,7 @@ lisp_add_del_map_server_command_fn (vlib_main_t * vm, u8 is_add = 1, ip_set = 0; ip_address_t ip; unformat_input_t _line_input, *line_input = &_line_input; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -94,14 +99,14 @@ lisp_add_del_map_server_command_fn (vlib_main_t * vm, { vlib_cli_output (vm, "parse error: '%U'", format_unformat_error, line_input); - return 0; + goto done; } } if (!ip_set) { vlib_cli_output (vm, "map-server ip address not set!"); - return 0; + goto done; } rv = vnet_lisp_add_del_map_server (&ip, is_add); @@ -109,7 +114,10 @@ lisp_add_del_map_server_command_fn (vlib_main_t * vm, vlib_cli_output (vm, "failed to %s map-server!", is_add ? "add" : "delete"); - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -191,7 +199,7 @@ lisp_add_del_local_eid_command_fn (vlib_main_t * vm, unformat_input_t * input, if (key && (0 == key_id)) { vlib_cli_output (vm, "invalid key_id!"); - return 0; + goto done;; } gid_address_copy (&a->eid, &eid); @@ -213,6 +221,8 @@ done: vec_free (locator_set_name); gid_address_free (&a->eid); vec_free (a->key); + unformat_free (line_input); + return error; } @@ -233,6 +243,7 @@ lisp_eid_table_map_command_fn (vlib_main_t * vm, u8 is_add = 1, is_l2 = 0; u32 vni = 0, dp_id = 0; unformat_input_t _line_input, *line_input = &_line_input; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -250,11 +261,16 @@ lisp_eid_table_map_command_fn (vlib_main_t * vm, is_l2 = 1; else { - return unformat_parse_error (line_input); + error = unformat_parse_error (line_input); + goto done; } } vnet_lisp_eid_table_map (vni, dp_id, is_l2, is_add); - return 0; + +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -479,7 +495,7 @@ lisp_add_del_adjacency_command_fn (vlib_main_t * vm, unformat_input_t * input, != ip_prefix_version (leid_ippref))) { clib_warning ("remote and local EIDs are of different types!"); - return error; + goto done; } memset (a, 0, sizeof (a[0])); @@ -512,6 +528,7 @@ lisp_map_request_mode_command_fn (vlib_main_t * vm, { unformat_input_t _i, *i = &_i; map_request_mode_t mr_mode = _MR_MODE_MAX; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, i)) @@ -533,12 +550,15 @@ lisp_map_request_mode_command_fn (vlib_main_t * vm, if (_MR_MODE_MAX == mr_mode) { clib_warning ("No LISP map request mode entered!"); - return 0; + goto done; } vnet_lisp_set_map_request_mode (mr_mode); + done: - return 0; + unformat_free (i); + + return error; } /* *INDENT-OFF* */ @@ -630,7 +650,10 @@ lisp_pitr_set_locator_set_command_fn (vlib_main_t * vm, else if (unformat (line_input, "disable")) is_add = 0; else - return clib_error_return (0, "parse error"); + { + error = clib_error_return (0, "parse error"); + goto done; + } } if (!locator_name_set) @@ -648,6 +671,8 @@ lisp_pitr_set_locator_set_command_fn (vlib_main_t * vm, done: if (locator_set_name) vec_free (locator_set_name); + unformat_free (line_input); + return error; } @@ -771,6 +796,7 @@ lisp_show_eid_table_command_fn (vlib_main_t * vm, gid_address_t eid; u8 print_all = 1; u8 filter = 0; + clib_error_t *error = NULL; memset (&eid, 0, sizeof (eid)); @@ -787,8 +813,11 @@ lisp_show_eid_table_command_fn (vlib_main_t * vm, else if (unformat (line_input, "remote")) filter = 2; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } vlib_cli_output (vm, "%-35s%-20s%-30s%-20s%-s", @@ -818,7 +847,7 @@ lisp_show_eid_table_command_fn (vlib_main_t * vm, { mi = gid_dictionary_lookup (&lcm->mapping_index_by_gid, &eid); if ((u32) ~ 0 == mi) - return 0; + goto done; mapit = pool_elt_at_index (lcm->mapping_pool, mi); locator_set_t *ls = pool_elt_at_index (lcm->locator_set_pool, @@ -827,14 +856,17 @@ lisp_show_eid_table_command_fn (vlib_main_t * vm, if (filter && !((1 == filter && ls->local) || (2 == filter && !ls->local))) { - return 0; + goto done; } vlib_cli_output (vm, "%U,", format_eid_entry, lcm->vnet_main, lcm, mapit, ls); } - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -853,6 +885,7 @@ lisp_enable_disable_command_fn (vlib_main_t * vm, unformat_input_t * input, unformat_input_t _line_input, *line_input = &_line_input; u8 is_enabled = 0; u8 is_set = 0; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -869,16 +902,24 @@ lisp_enable_disable_command_fn (vlib_main_t * vm, unformat_input_t * input, is_set = 1; else { - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; } } if (!is_set) - return clib_error_return (0, "state not set"); + { + error = clib_error_return (0, "state not set"); + goto done; + } vnet_lisp_enable_disable (is_enabled); - return 0; + +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -897,6 +938,7 @@ lisp_map_register_enable_disable_command_fn (vlib_main_t * vm, unformat_input_t _line_input, *line_input = &_line_input; u8 is_enabled = 0; u8 is_set = 0; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -915,18 +957,22 @@ lisp_map_register_enable_disable_command_fn (vlib_main_t * vm, { vlib_cli_output (vm, "parse error: '%U'", format_unformat_error, line_input); - return 0; + goto done; } } if (!is_set) { vlib_cli_output (vm, "state not set!"); - return 0; + goto done; } vnet_lisp_map_register_enable_disable (is_enabled); - return 0; + +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -945,6 +991,7 @@ lisp_rloc_probe_enable_disable_command_fn (vlib_main_t * vm, unformat_input_t _line_input, *line_input = &_line_input; u8 is_enabled = 0; u8 is_set = 0; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -963,18 +1010,22 @@ lisp_rloc_probe_enable_disable_command_fn (vlib_main_t * vm, { vlib_cli_output (vm, "parse error: '%U'", format_unformat_error, line_input); - return 0; + goto done; } } if (!is_set) { vlib_cli_output (vm, "state not set!"); - return 0; + goto done; } vnet_lisp_rloc_probe_enable_disable (is_enabled); - return 0; + +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -1022,6 +1073,7 @@ lisp_show_eid_table_map_command_fn (vlib_main_t * vm, lisp_cp_main_t *lcm = vnet_lisp_cp_get_main (); uword *vni_table = 0; u8 is_l2 = 0; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -1040,14 +1092,17 @@ lisp_show_eid_table_map_command_fn (vlib_main_t * vm, is_l2 = 0; } else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } if (!vni_table) { vlib_cli_output (vm, "Error: expected l2|l3 param!\n"); - return 0; + goto done; } vlib_cli_output (vm, "%=10s%=10s", "VNI", is_l2 ? "BD" : "VRF"); @@ -1059,7 +1114,10 @@ lisp_show_eid_table_map_command_fn (vlib_main_t * vm, })); /* *INDENT-ON* */ - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ @@ -1131,6 +1189,8 @@ done: vec_free (locators); if (locator_set_name) vec_free (locator_set_name); + unformat_free (line_input); + return error; } @@ -1205,6 +1265,8 @@ lisp_add_del_locator_in_set_command_fn (vlib_main_t * vm, done: vec_free (locators); vec_free (locator_set_name); + unformat_free (line_input); + return error; } @@ -1322,6 +1384,8 @@ lisp_add_del_map_resolver_command_fn (vlib_main_t * vm, } done: + unformat_free (line_input); + return error; } @@ -1372,11 +1436,11 @@ lisp_add_del_mreq_itr_rlocs_command_fn (vlib_main_t * vm, is_add ? "add" : "delete"); } +done: vec_free (locator_set_name); + unformat_free (line_input); -done: return error; - } /* *INDENT-OFF* */ @@ -1438,7 +1502,10 @@ lisp_use_petr_set_locator_set_command_fn (vlib_main_t * vm, else if (unformat (line_input, "disable")) is_add = 0; else - return clib_error_return (0, "parse error"); + { + error = clib_error_return (0, "parse error"); + goto done; + } } if (!ip_set) @@ -1454,6 +1521,8 @@ lisp_use_petr_set_locator_set_command_fn (vlib_main_t * vm, } done: + unformat_free (line_input); + return error; } diff --git a/src/vnet/lisp-gpe/interface.c b/src/vnet/lisp-gpe/interface.c index 2142e095..19ac22e7 100644 --- a/src/vnet/lisp-gpe/interface.c +++ b/src/vnet/lisp-gpe/interface.c @@ -794,6 +794,7 @@ lisp_gpe_add_del_iface_command_fn (vlib_main_t * vm, unformat_input_t * input, u32 table_id, vni, bd_id; u8 vni_is_set = 0, vrf_is_set = 0, bd_index_is_set = 0; u8 nsh_iface = 0; + clib_error_t *error = NULL; if (vnet_lisp_gpe_enable_disable_status () == 0) { @@ -828,8 +829,9 @@ lisp_gpe_add_del_iface_command_fn (vlib_main_t * vm, unformat_input_t * input, } else { - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; } } @@ -839,7 +841,8 @@ lisp_gpe_add_del_iface_command_fn (vlib_main_t * vm, unformat_input_t * input, { if (~0 == lisp_gpe_add_nsh_iface (&lisp_gpe_main)) { - return clib_error_return (0, "NSH interface not created"); + error = clib_error_return (0, "NSH interface not created"); + goto done; } } else @@ -850,21 +853,34 @@ lisp_gpe_add_del_iface_command_fn (vlib_main_t * vm, unformat_input_t * input, } if (vrf_is_set && bd_index_is_set) - return clib_error_return (0, - "Cannot set both vrf and brdige domain index!"); + { + error = clib_error_return + (0, "Cannot set both vrf and brdige domain index!"); + goto done; + } if (!vni_is_set) - return clib_error_return (0, "vni must be set!"); + { + error = clib_error_return (0, "vni must be set!"); + goto done; + } if (!vrf_is_set && !bd_index_is_set) - return clib_error_return (0, "vrf or bridge domain index must be set!"); + { + error = + clib_error_return (0, "vrf or bridge domain index must be set!"); + goto done; + } if (bd_index_is_set) { if (is_add) { if (~0 == lisp_gpe_tenant_l2_iface_add_or_lock (vni, bd_id)) - return clib_error_return (0, "L2 interface not created"); + { + error = clib_error_return (0, "L2 interface not created"); + goto done; + } } else lisp_gpe_tenant_l2_iface_unlock (vni); @@ -874,13 +890,35 @@ lisp_gpe_add_del_iface_command_fn (vlib_main_t * vm, unformat_input_t * input, if (is_add) { if (~0 == lisp_gpe_tenant_l3_iface_add_or_lock (vni, table_id)) - return clib_error_return (0, "L3 interface not created"); + { + error = clib_error_return (0, "L3 interface not created"); + goto done; + } } else lisp_gpe_tenant_l3_iface_unlock (vni); } - return (NULL); + if (nsh_iface) + { + if (is_add) + { + if (~0 == lisp_gpe_add_nsh_iface (&lisp_gpe_main)) + { + error = clib_error_return (0, "NSH interface not created"); + goto done; + } + else + { + lisp_gpe_del_nsh_iface (&lisp_gpe_main); + } + } + } + +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ diff --git a/src/vnet/lisp-gpe/lisp_gpe.c b/src/vnet/lisp-gpe/lisp_gpe.c index 1f8afdae..f2fbcbd5 100644 --- a/src/vnet/lisp-gpe/lisp_gpe.c +++ b/src/vnet/lisp-gpe/lisp_gpe.c @@ -218,6 +218,7 @@ lisp_gpe_enable_disable_command_fn (vlib_main_t * vm, unformat_input_t _line_input, *line_input = &_line_input; u8 is_en = 1; vnet_lisp_gpe_enable_disable_args_t _a, *a = &_a; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -231,12 +232,18 @@ lisp_gpe_enable_disable_command_fn (vlib_main_t * vm, is_en = 0; else { - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; } } a->is_en = is_en; - return vnet_lisp_gpe_enable_disable (a); + error = vnet_lisp_gpe_enable_disable (a); + +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ diff --git a/src/vnet/map/map.c b/src/vnet/map/map.c index aeec6a94..a2d28118 100644 --- a/src/vnet/map/map.c +++ b/src/vnet/map/map.c @@ -465,6 +465,8 @@ map_security_check_command_fn (vlib_main_t * vm, { unformat_input_t _line_input, *line_input = &_line_input; map_main_t *mm = &map_main; + clib_error_t *error = NULL; + /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -476,11 +478,17 @@ map_security_check_command_fn (vlib_main_t * vm, else if (unformat (line_input, "on")) mm->sec_check = true; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } + +done: unformat_free (line_input); - return 0; + + return error; } static clib_error_t * @@ -490,6 +498,8 @@ map_security_check_frag_command_fn (vlib_main_t * vm, { unformat_input_t _line_input, *line_input = &_line_input; map_main_t *mm = &map_main; + clib_error_t *error = NULL; + /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -501,11 +511,17 @@ map_security_check_frag_command_fn (vlib_main_t * vm, else if (unformat (line_input, "on")) mm->sec_check_frag = true; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } + +done: unformat_free (line_input); - return 0; + + return error; } static clib_error_t * @@ -523,6 +539,7 @@ map_add_domain_command_fn (vlib_main_t * vm, u32 mtu = 0; u8 flags = 0; ip6_src_len = 128; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -559,20 +576,28 @@ map_add_domain_command_fn (vlib_main_t * vm, else if (unformat (line_input, "map-t")) flags |= MAP_DOMAIN_TRANSLATION; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (num_m_args < 3) - return clib_error_return (0, "mandatory argument(s) missing"); + { + error = clib_error_return (0, "mandatory argument(s) missing"); + goto done; + } map_create_domain (&ip4_prefix, ip4_prefix_len, &ip6_prefix, ip6_prefix_len, &ip6_src, ip6_src_len, ea_bits_len, psid_offset, psid_length, &map_domain_index, mtu, flags); - return 0; +done: + unformat_free (line_input); + + return error; } static clib_error_t * @@ -582,6 +607,7 @@ map_del_domain_command_fn (vlib_main_t * vm, unformat_input_t _line_input, *line_input = &_line_input; u32 num_m_args = 0; u32 map_domain_index; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -592,17 +618,25 @@ map_del_domain_command_fn (vlib_main_t * vm, if (unformat (line_input, "index %d", &map_domain_index)) num_m_args++; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (num_m_args != 1) - return clib_error_return (0, "mandatory argument(s) missing"); + { + error = clib_error_return (0, "mandatory argument(s) missing"); + goto done; + } map_delete_domain (map_domain_index); - return 0; +done: + unformat_free (line_input); + + return error; } static clib_error_t * @@ -613,6 +647,7 @@ map_add_rule_command_fn (vlib_main_t * vm, ip6_address_t tep; u32 num_m_args = 0; u32 psid = 0, map_domain_index; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -628,19 +663,29 @@ map_add_rule_command_fn (vlib_main_t * vm, if (unformat (line_input, "ip6-dst %U", unformat_ip6_address, &tep)) num_m_args++; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (num_m_args != 3) - return clib_error_return (0, "mandatory argument(s) missing"); + { + error = clib_error_return (0, "mandatory argument(s) missing"); + goto done; + } if (map_add_del_psid (map_domain_index, psid, &tep, 1) != 0) { - return clib_error_return (0, "Failing to add Mapping Rule"); + error = clib_error_return (0, "Failing to add Mapping Rule"); + goto done; } - return 0; + +done: + unformat_free (line_input); + + return error; } #if MAP_SKIP_IP6_LOOKUP @@ -653,6 +698,7 @@ map_pre_resolve_command_fn (vlib_main_t * vm, ip4_address_t ip4nh; ip6_address_t ip6nh; map_main_t *mm = &map_main; + clib_error_t *error = NULL; memset (&ip4nh, 0, sizeof (ip4nh)); memset (&ip6nh, 0, sizeof (ip6nh)); @@ -669,14 +715,19 @@ map_pre_resolve_command_fn (vlib_main_t * vm, if (unformat (line_input, "ip6-nh %U", unformat_ip6_address, &ip6nh)) mm->preresolve_ip6 = ip6nh; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); map_pre_resolve (&ip4nh, &ip6nh); - return 0; +done: + unformat_free (line_input); + + return error; } #endif @@ -688,6 +739,7 @@ map_icmp_relay_source_address_command_fn (vlib_main_t * vm, unformat_input_t _line_input, *line_input = &_line_input; ip4_address_t icmp_src_address; map_main_t *mm = &map_main; + clib_error_t *error = NULL; mm->icmp4_src_address.as_u32 = 0; @@ -701,12 +753,17 @@ map_icmp_relay_source_address_command_fn (vlib_main_t * vm, (line_input, "%U", unformat_ip4_address, &icmp_src_address)) mm->icmp4_src_address = icmp_src_address; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } + +done: unformat_free (line_input); - return 0; + return error; } static clib_error_t * @@ -717,6 +774,7 @@ map_icmp_unreachables_command_fn (vlib_main_t * vm, unformat_input_t _line_input, *line_input = &_line_input; map_main_t *mm = &map_main; int num_m_args = 0; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -730,16 +788,21 @@ map_icmp_unreachables_command_fn (vlib_main_t * vm, else if (unformat (line_input, "off")) mm->icmp6_enabled = false; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (num_m_args != 1) - return clib_error_return (0, "mandatory argument(s) missing"); + error = clib_error_return (0, "mandatory argument(s) missing"); - return 0; +done: + unformat_free (line_input); + + return error; } static clib_error_t * @@ -748,6 +811,7 @@ map_fragment_command_fn (vlib_main_t * vm, { unformat_input_t _line_input, *line_input = &_line_input; map_main_t *mm = &map_main; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -760,12 +824,17 @@ map_fragment_command_fn (vlib_main_t * vm, else if (unformat (line_input, "outer")) mm->frag_inner = false; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } + +done: unformat_free (line_input); - return 0; + return error; } static clib_error_t * @@ -775,6 +844,7 @@ map_fragment_df_command_fn (vlib_main_t * vm, { unformat_input_t _line_input, *line_input = &_line_input; map_main_t *mm = &map_main; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -787,12 +857,17 @@ map_fragment_df_command_fn (vlib_main_t * vm, else if (unformat (line_input, "off")) mm->frag_ignore_df = false; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } + +done: unformat_free (line_input); - return 0; + return error; } static clib_error_t * @@ -803,6 +878,7 @@ map_traffic_class_command_fn (vlib_main_t * vm, unformat_input_t _line_input, *line_input = &_line_input; map_main_t *mm = &map_main; u32 tc = 0; + clib_error_t *error = NULL; mm->tc_copy = false; @@ -817,12 +893,17 @@ map_traffic_class_command_fn (vlib_main_t * vm, else if (unformat (line_input, "%x", &tc)) mm->tc = tc & 0xff; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } + +done: unformat_free (line_input); - return 0; + return error; } static u8 * @@ -922,6 +1003,7 @@ show_map_domain_command_fn (vlib_main_t * vm, unformat_input_t * input, map_domain_t *d; bool counters = false; u32 map_domain_index = ~0; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -934,10 +1016,12 @@ show_map_domain_command_fn (vlib_main_t * vm, unformat_input_t * input, else if (unformat (line_input, "index %d", &map_domain_index)) ; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); if (pool_elts (mm->domains) == 0) vlib_cli_output (vm, "No MAP domains are configured..."); @@ -952,15 +1036,19 @@ show_map_domain_command_fn (vlib_main_t * vm, unformat_input_t * input, { if (pool_is_free_index (mm->domains, map_domain_index)) { - return clib_error_return (0, "MAP domain does not exists %d", - map_domain_index); + error = clib_error_return (0, "MAP domain does not exists %d", + map_domain_index); + goto done; } d = pool_elt_at_index (mm->domains, map_domain_index); vlib_cli_output (vm, "%U", format_map_domain, d, counters); } - return 0; +done: + unformat_free (line_input); + + return error; } static clib_error_t * diff --git a/src/vnet/mpls/mpls.c b/src/vnet/mpls/mpls.c index 0e610e17..7ae4aa00 100644 --- a/src/vnet/mpls/mpls.c +++ b/src/vnet/mpls/mpls.c @@ -470,6 +470,8 @@ vnet_mpls_local_label (vlib_main_t * vm, } done: + unformat_free (line_input); + return error; } diff --git a/src/vnet/mpls/mpls_tunnel.c b/src/vnet/mpls/mpls_tunnel.c index 8d1e30a3..e488271d 100644 --- a/src/vnet/mpls/mpls_tunnel.c +++ b/src/vnet/mpls/mpls_tunnel.c @@ -535,6 +535,7 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm, fib_route_path_t rpath, *rpaths = NULL; mpls_label_t out_label = MPLS_LABEL_INVALID, *labels = NULL; u32 sw_if_index; + clib_error_t *error = NULL; memset(&rpath, 0, sizeof(rpath)); @@ -595,8 +596,11 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm, else if (unformat (line_input, "l2-only")) l2_only = 1; else - return clib_error_return (0, "unknown input '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + goto done; + } } if (is_del) @@ -606,17 +610,22 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm, else { if (0 == vec_len(labels)) - return clib_error_return (0, "No Output Labels '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "No Output Labels '%U'", + format_unformat_error, line_input); + goto done; + } vec_add1(rpaths, rpath); vnet_mpls_tunnel_add(rpaths, labels, l2_only, &sw_if_index); } +done: vec_free(labels); vec_free(rpaths); + unformat_free (line_input); - return (NULL); + return error; } /*? diff --git a/src/vnet/pg/cli.c b/src/vnet/pg/cli.c index f5896b43..3c249a7b 100644 --- a/src/vnet/pg/cli.c +++ b/src/vnet/pg/cli.c @@ -547,21 +547,30 @@ pg_capture_cmd_fn (vlib_main_t * vm, else { error = clib_error_create ("unknown input `%U'", - format_unformat_error, input); - return error; + format_unformat_error, line_input); + goto done; } } if (!hi) - return clib_error_return (0, "Please specify interface name"); + { + error = clib_error_return (0, "Please specify interface name"); + goto done; + } if (hi->dev_class_index != pg_dev_class.index) - return clib_error_return (0, "Please specify packet-generator interface"); + { + error = + clib_error_return (0, "Please specify packet-generator interface"); + goto done; + } if (!pcap_file_name && is_disable == 0) - return clib_error_return (0, "Please specify pcap file name"); + { + error = clib_error_return (0, "Please specify pcap file name"); + goto done; + } - unformat_free (line_input); pg_capture_args_t _a, *a = &_a; @@ -572,6 +581,10 @@ pg_capture_cmd_fn (vlib_main_t * vm, a->count = count; error = pg_capture (a); + +done: + unformat_free (line_input); + return error; } @@ -590,6 +603,7 @@ create_pg_if_cmd_fn (vlib_main_t * vm, pg_main_t *pg = &pg_main; unformat_input_t _line_input, *line_input = &_line_input; u32 if_id; + clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) return 0; @@ -600,14 +614,19 @@ create_pg_if_cmd_fn (vlib_main_t * vm, ; else - return clib_error_create ("unknown input `%U'", - format_unformat_error, input); + { + error = clib_error_create ("unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } + pg_interface_add_or_get (pg, if_id); + +done: unformat_free (line_input); - pg_interface_add_or_get (pg, if_id); - return 0; + return error; } /* *INDENT-OFF* */ diff --git a/src/vnet/policer/node_funcs.c b/src/vnet/policer/node_funcs.c index 1f4997ff..457dd09f 100644 --- a/src/vnet/policer/node_funcs.c +++ b/src/vnet/policer/node_funcs.c @@ -447,6 +447,7 @@ test_policer_command_fn (vlib_main_t * vm, int rx_set = 0; int is_add = 1; int is_show = 0; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -468,7 +469,10 @@ test_policer_command_fn (vlib_main_t * vm, } if (rx_set == 0) - return clib_error_return (0, "interface not set"); + { + error = clib_error_return (0, "interface not set"); + goto done; + } if (is_show) { @@ -477,12 +481,13 @@ test_policer_command_fn (vlib_main_t * vm, policer = pool_elt_at_index (pm->policers, pi); vlib_cli_output (vm, "%U", format_policer_instance, policer); - return 0; + goto done; } if (is_add && config_name == 0) { - return clib_error_return (0, "policer config name required"); + error = clib_error_return (0, "policer config name required"); + goto done; } rv = test_policer_add_del (rx_sw_if_index, config_name, is_add); @@ -493,11 +498,15 @@ test_policer_command_fn (vlib_main_t * vm, break; default: - return clib_error_return + error = clib_error_return (0, "WARNING: vnet_vnet_policer_add_del returned %d", rv); + goto done; } - return 0; +done: + unformat_free (line_input); + + return error; } /* *INDENT-OFF* */ diff --git a/src/vnet/policer/policer.c b/src/vnet/policer/policer.c index 290a6af5..cd754e29 100644 --- a/src/vnet/policer/policer.c +++ b/src/vnet/policer/policer.c @@ -413,6 +413,7 @@ configure_policer_command_fn (vlib_main_t * vm, u8 is_add = 1; u8 *name = 0; u32 pi; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -433,13 +434,19 @@ configure_policer_command_fn (vlib_main_t * vm, foreach_config_param #undef _ else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } + error = policer_add_del (vm, name, &c, &pi, is_add); + +done: unformat_free (line_input); - return policer_add_del (vm, name, &c, &pi, is_add); + return error; } /* *INDENT-OFF* */ diff --git a/src/vnet/unix/tapcli.c b/src/vnet/unix/tapcli.c index 48e81b50..25c930c6 100644 --- a/src/vnet/unix/tapcli.c +++ b/src/vnet/unix/tapcli.c @@ -1308,6 +1308,7 @@ tap_connect_command_fn (vlib_main_t * vm, int ip6_address_set = 0; u32 ip4_mask_width = 0; u32 ip6_mask_width = 0; + clib_error_t *error = NULL; if (tm->is_disabled) return clib_error_return (0, "device disabled..."); @@ -1336,12 +1337,18 @@ tap_connect_command_fn (vlib_main_t * vm, else if (unformat (line_input, "%s", &intfc_name)) ; else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } } if (intfc_name == 0) - return clib_error_return (0, "interface name must be specified"); + { + error = clib_error_return (0, "interface name must be specified"); + goto done; + } memset (ap, 0, sizeof (*ap)); @@ -1367,48 +1374,64 @@ tap_connect_command_fn (vlib_main_t * vm, switch (rv) { case VNET_API_ERROR_SYSCALL_ERROR_1: - return clib_error_return (0, "Couldn't open /dev/net/tun"); + error = clib_error_return (0, "Couldn't open /dev/net/tun"); + goto done; case VNET_API_ERROR_SYSCALL_ERROR_2: - return clib_error_return (0, "Error setting flags on '%s'", intfc_name); - + error = clib_error_return (0, "Error setting flags on '%s'", intfc_name); + goto done; + case VNET_API_ERROR_SYSCALL_ERROR_3: - return clib_error_return (0, "Couldn't open provisioning socket"); + error = clib_error_return (0, "Couldn't open provisioning socket"); + goto done; case VNET_API_ERROR_SYSCALL_ERROR_4: - return clib_error_return (0, "Couldn't get if_index"); + error = clib_error_return (0, "Couldn't get if_index"); + goto done; case VNET_API_ERROR_SYSCALL_ERROR_5: - return clib_error_return (0, "Couldn't bind provisioning socket"); + error = clib_error_return (0, "Couldn't bind provisioning socket"); + goto done; case VNET_API_ERROR_SYSCALL_ERROR_6: - return clib_error_return (0, "Couldn't set device non-blocking flag"); + error = clib_error_return (0, "Couldn't set device non-blocking flag"); + goto done; case VNET_API_ERROR_SYSCALL_ERROR_7: - return clib_error_return (0, "Couldn't set device MTU"); + error = clib_error_return (0, "Couldn't set device MTU"); + goto done; case VNET_API_ERROR_SYSCALL_ERROR_8: - return clib_error_return (0, "Couldn't get interface flags"); + error = clib_error_return (0, "Couldn't get interface flags"); + goto done; case VNET_API_ERROR_SYSCALL_ERROR_9: - return clib_error_return (0, "Couldn't set intfc admin state up"); + error = clib_error_return (0, "Couldn't set intfc admin state up"); + goto done; case VNET_API_ERROR_SYSCALL_ERROR_10: - return clib_error_return (0, "Couldn't set intfc address/mask"); + error = clib_error_return (0, "Couldn't set intfc address/mask"); + goto done; case VNET_API_ERROR_INVALID_REGISTRATION: - return clib_error_return (0, "Invalid registration"); + error = clib_error_return (0, "Invalid registration"); + goto done; case 0: break; default: - return clib_error_return (0, "Unknown error: %d", rv); + error = clib_error_return (0, "Unknown error: %d", rv); + goto done; } vlib_cli_output(vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main(), sw_if_index); - return 0; + +done: + unformat_free (line_input); + + return error; } VLIB_CLI_COMMAND (tap_connect_command, static) = { diff --git a/src/vnet/vxlan-gpe/vxlan_gpe.c b/src/vnet/vxlan-gpe/vxlan_gpe.c index b97510c4..2cba596f 100644 --- a/src/vnet/vxlan-gpe/vxlan_gpe.c +++ b/src/vnet/vxlan-gpe/vxlan_gpe.c @@ -454,6 +454,7 @@ vxlan_gpe_add_del_tunnel_command_fn (vlib_main_t * vm, u32 tmp; vnet_vxlan_gpe_add_del_tunnel_args_t _a, * a = &_a; u32 sw_if_index; + clib_error_t *error = NULL; /* Get a line of input. */ if (! unformat_user (input, unformat_line_input, line_input)) @@ -494,7 +495,10 @@ vxlan_gpe_add_del_tunnel_command_fn (vlib_main_t * vm, encap_fib_index = ip4_fib_index_from_table_id (tmp); if (encap_fib_index == ~0) - return clib_error_return (0, "nonexistent encap fib id %d", tmp); + { + error = clib_error_return (0, "nonexistent encap fib id %d", tmp); + goto done; + } } else if (unformat (line_input, "decap-vrf-id %d", &tmp)) { @@ -504,7 +508,10 @@ vxlan_gpe_add_del_tunnel_command_fn (vlib_main_t * vm, decap_fib_index = ip4_fib_index_from_table_id (tmp); if (decap_fib_index == ~0) - return clib_error_return (0, "nonexistent decap fib id %d", tmp); + { + error = clib_error_return (0, "nonexistent decap fib id %d", tmp); + goto done; + } } else if (unformat (line_input, "vni %d", &vni)) vni_set = 1; @@ -517,27 +524,43 @@ vxlan_gpe_add_del_tunnel_command_fn (vlib_main_t * vm, else if (unformat(line_input, "next-nsh")) protocol = VXLAN_GPE_PROTOCOL_NSH; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (local_set == 0) - return clib_error_return (0, "tunnel local address not specified"); + { + error = clib_error_return (0, "tunnel local address not specified"); + goto done; + } if (remote_set == 0) - return clib_error_return (0, "tunnel remote address not specified"); + { + error = clib_error_return (0, "tunnel remote address not specified"); + goto done; + } if (ipv4_set && ipv6_set) - return clib_error_return (0, "both IPv4 and IPv6 addresses specified"); + { + error = clib_error_return (0, "both IPv4 and IPv6 addresses specified"); + goto done; + } if ((ipv4_set && memcmp(&local.ip4, &remote.ip4, sizeof(local.ip4)) == 0) || (ipv6_set && memcmp(&local.ip6, &remote.ip6, sizeof(local.ip6)) == 0)) - return clib_error_return (0, "src and dst addresses are identical"); + { + error = clib_error_return (0, "src and dst addresses are identical"); + goto done; + } if (vni_set == 0) - return clib_error_return (0, "vni not specified"); + { + error = clib_error_return (0, "vni not specified"); + goto done; + } memset (a, 0, sizeof (*a)); @@ -558,20 +581,27 @@ vxlan_gpe_add_del_tunnel_command_fn (vlib_main_t * vm, vlib_cli_output(vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main(), sw_if_index); break; case VNET_API_ERROR_INVALID_DECAP_NEXT: - return clib_error_return (0, "invalid decap-next..."); + error = clib_error_return (0, "invalid decap-next..."); + goto done; case VNET_API_ERROR_TUNNEL_EXIST: - return clib_error_return (0, "tunnel already exists..."); + error = clib_error_return (0, "tunnel already exists..."); + goto done; case VNET_API_ERROR_NO_SUCH_ENTRY: - return clib_error_return (0, "tunnel does not exist..."); + error = clib_error_return (0, "tunnel does not exist..."); + goto done; default: - return clib_error_return + error = clib_error_return (0, "vnet_vxlan_gpe_add_del_tunnel returned %d", rv); + goto done; } - return 0; +done: + unformat_free (line_input); + + return error; } VLIB_CLI_COMMAND (create_vxlan_gpe_tunnel_command, static) = { diff --git a/src/vnet/vxlan/vxlan.c b/src/vnet/vxlan/vxlan.c index 849fc25d..eedc16f8 100644 --- a/src/vnet/vxlan/vxlan.c +++ b/src/vnet/vxlan/vxlan.c @@ -657,6 +657,7 @@ vxlan_add_del_tunnel_command_fn (vlib_main_t * vm, int rv; vnet_vxlan_add_del_tunnel_args_t _a, * a = &_a; u32 tunnel_sw_if_index; + clib_error_t *error = NULL; /* Cant "universally zero init" (={0}) due to GCC bug 53119 */ memset(&src, 0, sizeof src); @@ -715,7 +716,10 @@ vxlan_add_del_tunnel_command_fn (vlib_main_t * vm, { encap_fib_index = fib_table_find (fib_ip_proto (ipv6_set), tmp); if (encap_fib_index == ~0) - return clib_error_return (0, "nonexistent encap-vrf-id %d", tmp); + { + error = clib_error_return (0, "nonexistent encap-vrf-id %d", tmp); + goto done; + } } else if (unformat (line_input, "decap-next %U", unformat_decap_next, &decap_next_index, ipv4_set)) @@ -723,41 +727,72 @@ vxlan_add_del_tunnel_command_fn (vlib_main_t * vm, else if (unformat (line_input, "vni %d", &vni)) { if (vni >> 24) - return clib_error_return (0, "vni %d out of range", vni); + { + error = clib_error_return (0, "vni %d out of range", vni); + goto done; + } } else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + goto done; + } } - unformat_free (line_input); - if (src_set == 0) - return clib_error_return (0, "tunnel src address not specified"); + { + error = clib_error_return (0, "tunnel src address not specified"); + goto done; + } if (dst_set == 0) - return clib_error_return (0, "tunnel dst address not specified"); + { + error = clib_error_return (0, "tunnel dst address not specified"); + goto done; + } if (grp_set && !ip46_address_is_multicast(&dst)) - return clib_error_return (0, "tunnel group address not multicast"); + { + error = clib_error_return (0, "tunnel group address not multicast"); + goto done; + } if (grp_set == 0 && ip46_address_is_multicast(&dst)) - return clib_error_return (0, "dst address must be unicast"); + { + error = clib_error_return (0, "dst address must be unicast"); + goto done; + } if (grp_set && mcast_sw_if_index == ~0) - return clib_error_return (0, "tunnel nonexistent multicast device"); + { + error = clib_error_return (0, "tunnel nonexistent multicast device"); + goto done; + } if (ipv4_set && ipv6_set) - return clib_error_return (0, "both IPv4 and IPv6 addresses specified"); + { + error = clib_error_return (0, "both IPv4 and IPv6 addresses specified"); + goto done; + } if (ip46_address_cmp(&src, &dst) == 0) - return clib_error_return (0, "src and dst addresses are identical"); + { + error = clib_error_return (0, "src and dst addresses are identical"); + goto done; + } if (decap_next_index == ~0) - return clib_error_return (0, "next node not found"); + { + error = clib_error_return (0, "next node not found"); + goto done; + } if (vni == 0) - return clib_error_return (0, "vni not specified"); + { + error = clib_error_return (0, "vni not specified"); + goto done; + } memset (a, 0, sizeof (*a)); @@ -779,17 +814,23 @@ vxlan_add_del_tunnel_command_fn (vlib_main_t * vm, break; case VNET_API_ERROR_TUNNEL_EXIST: - return clib_error_return (0, "tunnel already exists..."); + error = clib_error_return (0, "tunnel already exists..."); + goto done; case VNET_API_ERROR_NO_SUCH_ENTRY: - return clib_error_return (0, "tunnel does not exist..."); + error = clib_error_return (0, "tunnel does not exist..."); + goto done; default: - return clib_error_return + error = clib_error_return (0, "vnet_vxlan_add_del_tunnel returned %d", rv); + goto done; } - return 0; +done: + unformat_free (line_input); + + return error; } /*? @@ -912,6 +953,8 @@ set_ip_vxlan_bypass (u32 is_ip6, vnet_int_vxlan_bypass_mode (sw_if_index, is_ip6, is_enable); done: + unformat_free (line_input); + return error; } diff --git a/src/vpp/app/l2t.c b/src/vpp/app/l2t.c index 45dd2807..e1eda155 100644 --- a/src/vpp/app/l2t.c +++ b/src/vpp/app/l2t.c @@ -254,6 +254,7 @@ l2tp_session_add_command_fn (vlib_main_t * vm, u32 local_session_id = 1, remote_session_id = 1; int our_address_set = 0, client_address_set = 0; int l2_sublayer_present = 0; + clib_error_t *error = NULL; /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) @@ -290,8 +291,12 @@ l2tp_session_add_command_fn (vlib_main_t * vm, else if (unformat (line_input, "l2-sublayer-present")) l2_sublayer_present = 1; else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); + { + error = clib_error_return (0, "parse error: '%U'", + format_unformat_error, line_input); + unformat_free (line_input); + return error; + } } unformat_free (line_input); diff --git a/src/vpp/app/vpe_cli.c b/src/vpp/app/vpe_cli.c index a26bf71f..94bdc84c 100644 --- a/src/vpp/app/vpe_cli.c +++ b/src/vpp/app/vpe_cli.c @@ -36,6 +36,7 @@ virtual_ip_cmd_fn_command_fn (vlib_main_t * vm, mac_addr_t *mac_addrs = 0; u32 sw_if_index; u32 i; + clib_error_t *error = NULL; next_hops = NULL; rpaths = NULL; @@ -49,7 +50,11 @@ virtual_ip_cmd_fn_command_fn (vlib_main_t * vm, if (!unformat (line_input, "%U %U", unformat_ip4_address, &prefix.fp_addr.ip4, unformat_vnet_sw_interface, vnm, &sw_if_index)) - goto barf; + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) { @@ -67,13 +72,18 @@ virtual_ip_cmd_fn_command_fn (vlib_main_t * vm, } else { - barf: - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; } } + if (vec_len (mac_addrs) == 0 || vec_len (mac_addrs) != vec_len (next_hops)) - goto barf; + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } /* Create / delete special interface route /32's */ @@ -100,10 +110,12 @@ virtual_ip_cmd_fn_command_fn (vlib_main_t * vm, &prefix, FIB_SOURCE_CLI, FIB_ENTRY_FLAG_NONE, rpaths); +done: vec_free (mac_addrs); vec_free (next_hops); + unformat_free (line_input); - return 0; + return error; } /* *INDENT-OFF* */ -- cgit 1.2.3-korg From 3b64d6334b4e8d0759cff043a55042f88d1ccb0e Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Fri, 8 Sep 2017 12:26:12 +0200 Subject: vlib: move linux-specific code to vlib/linux Change-Id: Id79d2c2be7a98e15416a537c890a8f2dd6d4464d Signed-off-by: Damjan Marion --- src/plugins/dpdk/device/init.c | 1 + src/plugins/memif/memif.c | 1 + src/plugins/memif/private.h | 30 -- src/vlib.am | 7 +- src/vlib/linux/pci.c | 666 +++++++++++++++++++++++++++++++++ src/vlib/linux/physmem.c | 411 ++++++++++++++++++++ src/vlib/linux/syscall.h | 58 +++ src/vlib/linux/sysfs.c | 250 +++++++++++++ src/vlib/linux/sysfs.h | 44 +++ src/vlib/pci/linux_pci.c | 665 -------------------------------- src/vlib/threads_cli.c | 1 + src/vlib/unix/physmem.c | 439 ---------------------- src/vlib/unix/unix.h | 17 - src/vlib/unix/util.c | 219 ----------- src/vnet/devices/af_packet/af_packet.c | 1 + 15 files changed, 1438 insertions(+), 1372 deletions(-) create mode 100644 src/vlib/linux/pci.c create mode 100644 src/vlib/linux/physmem.c create mode 100644 src/vlib/linux/syscall.h create mode 100644 src/vlib/linux/sysfs.c create mode 100644 src/vlib/linux/sysfs.h delete mode 100644 src/vlib/pci/linux_pci.c delete mode 100644 src/vlib/unix/physmem.c (limited to 'src/vlib/threads_cli.c') diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index e23542f7..4ef3b676 100755 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c index 7e2d947f..4c387b92 100644 --- a/src/plugins/memif/memif.c +++ b/src/plugins/memif/memif.c @@ -33,6 +33,7 @@ #include #include +#include #include #include #include diff --git a/src/plugins/memif/private.h b/src/plugins/memif/private.h index 985ac5ec..b5f2f8ff 100644 --- a/src/plugins/memif/private.h +++ b/src/plugins/memif/private.h @@ -228,24 +228,6 @@ int memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args); int memif_delete_if (vlib_main_t * vm, memif_if_t * mif); clib_error_t *memif_plugin_api_hookup (vlib_main_t * vm); -#ifndef __NR_memfd_create -#if defined __x86_64__ -#define __NR_memfd_create 319 -#elif defined __arm__ -#define __NR_memfd_create 385 -#elif defined __aarch64__ -#define __NR_memfd_create 279 -#else -#error "__NR_memfd_create unknown for this architecture" -#endif -#endif - -static inline int -memfd_create (const char *name, unsigned int flags) -{ - return syscall (__NR_memfd_create, name, flags); -} - static_always_inline void * memif_get_buffer (memif_if_t * mif, memif_ring_t * ring, u16 slot) { @@ -253,18 +235,6 @@ memif_get_buffer (memif_if_t * mif, memif_ring_t * ring, u16 slot) return mif->regions[region].shm + ring->desc[slot].offset; } -#ifndef F_LINUX_SPECIFIC_BASE -#define F_LINUX_SPECIFIC_BASE 1024 -#endif -#define MFD_ALLOW_SEALING 0x0002U -#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9) -#define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10) - -#define F_SEAL_SEAL 0x0001 /* prevent further seals from being set */ -#define F_SEAL_SHRINK 0x0002 /* prevent file from shrinking */ -#define F_SEAL_GROW 0x0004 /* prevent file from growing */ -#define F_SEAL_WRITE 0x0008 /* prevent writes */ - /* memif.c */ clib_error_t *memif_init_regions_and_queues (memif_if_t * mif); clib_error_t *memif_connect (memif_if_t * mif); diff --git a/src/vlib.am b/src/vlib.am index cab90e2d..41d68690 100644 --- a/src/vlib.am +++ b/src/vlib.am @@ -32,13 +32,15 @@ libvlib_la_SOURCES = \ vlib/format.c \ vlib/i2c.c \ vlib/init.c \ + vlib/linux/pci.c \ + vlib/linux/physmem.c \ + vlib/linux/sysfs.c \ vlib/main.c \ vlib/mc.c \ vlib/node.c \ vlib/node_cli.c \ vlib/node_format.c \ vlib/pci/pci.c \ - vlib/pci/linux_pci.c \ vlib/threads.c \ vlib/threads_cli.c \ vlib/trace.c @@ -58,6 +60,8 @@ nobase_include_HEADERS += \ vlib/global_funcs.h \ vlib/i2c.h \ vlib/init.h \ + vlib/linux/sysfs.h \ + vlib/linux/syscall.h \ vlib/main.h \ vlib/mc.h \ vlib/node_funcs.h \ @@ -79,7 +83,6 @@ libvlib_la_SOURCES += \ vlib/unix/mc_socket.c \ vlib/unix/plugin.c \ vlib/unix/plugin.h \ - vlib/unix/physmem.c \ vlib/unix/util.c nobase_include_HEADERS += \ diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c new file mode 100644 index 00000000..cd2affdc --- /dev/null +++ b/src/vlib/linux/pci.c @@ -0,0 +1,666 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * pci.c: Linux user space PCI bus management. + * + * Copyright (c) 2008 Eliot Dresselhaus + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct +{ + /* /sys/bus/pci/devices/... directory name for this device. */ + u8 *dev_dir_name; + + /* Resource file descriptors. */ + int *resource_fds; + + /* File descriptor for config space read/write. */ + int config_fd; + + /* File descriptor for /dev/uio%d */ + int uio_fd; + + /* Minor device for uio device. */ + u32 uio_minor; + + /* Index given by unix_file_add. */ + u32 unix_file_index; + +} linux_pci_device_t; + +/* Pool of PCI devices. */ +typedef struct +{ + vlib_main_t *vlib_main; + linux_pci_device_t *linux_pci_devices; +} linux_pci_main_t; + +extern linux_pci_main_t linux_pci_main; + +/* Call to allocate/initialize the pci subsystem. + This is not an init function so that users can explicitly enable + pci only when it's needed. */ +clib_error_t *pci_bus_init (vlib_main_t * vm); + +clib_error_t *vlib_pci_bind_to_uio (vlib_pci_device_t * d, + char *uio_driver_name); + +linux_pci_main_t linux_pci_main; + +clib_error_t * +vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name) +{ + clib_error_t *error = 0; + u8 *s = 0, *driver_name = 0; + DIR *dir = 0; + struct dirent *e; + int fd, clear_driver_override = 0; + u8 *dev_dir_name = format (0, "/sys/bus/pci/devices/%U", + format_vlib_pci_addr, &d->bus_address); + + s = format (s, "%v/driver%c", dev_dir_name, 0); + driver_name = vlib_sysfs_link_to_name ((char *) s); + vec_reset_length (s); + + if (driver_name && + ((strcmp ("vfio-pci", (char *) driver_name) == 0) || + (strcmp ("uio_pci_generic", (char *) driver_name) == 0) || + (strcmp ("igb_uio", (char *) driver_name) == 0))) + goto done; + + /* walk trough all linux interfaces and if interface belonging to + this device is founf check if interface is admin up */ + dir = opendir ("/sys/class/net"); + s = format (s, "%U%c", format_vlib_pci_addr, &d->bus_address, 0); + + if (!dir) + { + error = clib_error_return (0, "Skipping PCI device %U: failed to " + "read /sys/class/net", + format_vlib_pci_addr, &d->bus_address); + goto done; + } + + fd = socket (PF_INET, SOCK_DGRAM, 0); + if (fd < 0) + { + error = clib_error_return_unix (0, "socket"); + goto done; + } + + while ((e = readdir (dir))) + { + struct ifreq ifr; + struct ethtool_drvinfo drvinfo; + + if (e->d_name[0] == '.') /* skip . and .. */ + continue; + + memset (&ifr, 0, sizeof ifr); + memset (&drvinfo, 0, sizeof drvinfo); + ifr.ifr_data = (char *) &drvinfo; + strncpy (ifr.ifr_name, e->d_name, IFNAMSIZ - 1); + drvinfo.cmd = ETHTOOL_GDRVINFO; + if (ioctl (fd, SIOCETHTOOL, &ifr) < 0) + { + /* Some interfaces (eg "lo") don't support this ioctl */ + if ((errno != ENOTSUP) && (errno != ENODEV)) + clib_unix_warning ("ioctl fetch intf %s bus info error", + e->d_name); + continue; + } + + if (strcmp ((char *) s, drvinfo.bus_info)) + continue; + + memset (&ifr, 0, sizeof (ifr)); + strncpy (ifr.ifr_name, e->d_name, IFNAMSIZ - 1); + if (ioctl (fd, SIOCGIFFLAGS, &ifr) < 0) + { + error = clib_error_return_unix (0, "ioctl fetch intf %s flags", + e->d_name); + close (fd); + goto done; + } + + if (ifr.ifr_flags & IFF_UP) + { + error = clib_error_return (0, "Skipping PCI device %U as host " + "interface %s is up", + format_vlib_pci_addr, &d->bus_address, + e->d_name); + close (fd); + goto done; + } + } + + close (fd); + vec_reset_length (s); + + s = format (s, "%v/driver/unbind%c", dev_dir_name, 0); + vlib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address); + vec_reset_length (s); + + s = format (s, "%v/driver_override%c", dev_dir_name, 0); + if (access ((char *) s, F_OK) == 0) + { + vlib_sysfs_write ((char *) s, "%s", uio_driver_name); + clear_driver_override = 1; + } + else + { + vec_reset_length (s); + s = format (s, "/sys/bus/pci/drivers/%s/new_id%c", uio_driver_name, 0); + vlib_sysfs_write ((char *) s, "0x%04x 0x%04x", d->vendor_id, + d->device_id); + } + vec_reset_length (s); + + s = format (s, "/sys/bus/pci/drivers/%s/bind%c", uio_driver_name, 0); + vlib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address); + vec_reset_length (s); + + if (clear_driver_override) + { + s = format (s, "%v/driver_override%c", dev_dir_name, 0); + vlib_sysfs_write ((char *) s, "%c", 0); + vec_reset_length (s); + } + +done: + closedir (dir); + vec_free (s); + vec_free (dev_dir_name); + vec_free (driver_name); + return error; +} + + +static clib_error_t * +scan_uio_dir (void *arg, u8 * path_name, u8 * file_name) +{ + linux_pci_device_t *l = arg; + unformat_input_t input; + + unformat_init_string (&input, (char *) file_name, vec_len (file_name)); + + if (!unformat (&input, "uio%d", &l->uio_minor)) + abort (); + + unformat_free (&input); + return 0; +} + +static clib_error_t * +linux_pci_uio_read_ready (unix_file_t * uf) +{ + vlib_pci_main_t *pm = &pci_main; + vlib_pci_device_t *d; + int __attribute__ ((unused)) rv; + + u32 icount; + rv = read (uf->file_descriptor, &icount, 4); + + d = pool_elt_at_index (pm->pci_devs, uf->private_data); + + if (d->interrupt_handler) + d->interrupt_handler (d); + + vlib_pci_intr_enable (d); + + return /* no error */ 0; +} + +static clib_error_t * +linux_pci_uio_error_ready (unix_file_t * uf) +{ + u32 error_index = (u32) uf->private_data; + + return clib_error_return (0, "pci device %d: error", error_index); +} + +static void +add_device (vlib_pci_device_t * dev, linux_pci_device_t * pdev) +{ + vlib_pci_main_t *pm = &pci_main; + linux_pci_main_t *lpm = &linux_pci_main; + linux_pci_device_t *l; + + pool_get (lpm->linux_pci_devices, l); + l[0] = pdev[0]; + + l->dev_dir_name = vec_dup (l->dev_dir_name); + + dev->os_handle = l - lpm->linux_pci_devices; + + { + u8 *uio_dir = format (0, "%s/uio", l->dev_dir_name); + foreach_directory_file ((char *) uio_dir, scan_uio_dir, l, /* scan_dirs */ + 1); + vec_free (uio_dir); + } + + { + char *uio_name = (char *) format (0, "/dev/uio%d%c", l->uio_minor, 0); + l->uio_fd = open (uio_name, O_RDWR); + if (l->uio_fd < 0) + clib_unix_error ("open `%s'", uio_name); + vec_free (uio_name); + } + + { + unix_file_t template = { 0 }; + unix_main_t *um = &unix_main; + + template.read_function = linux_pci_uio_read_ready; + template.file_descriptor = l->uio_fd; + template.error_function = linux_pci_uio_error_ready; + template.private_data = dev - pm->pci_devs; + + l->unix_file_index = unix_file_add (um, &template); + } +} + +static void +linux_pci_device_free (linux_pci_device_t * l) +{ + int i; + for (i = 0; i < vec_len (l->resource_fds); i++) + if (l->resource_fds[i] > 0) + close (l->resource_fds[i]); + if (l->config_fd > 0) + close (l->config_fd); + if (l->uio_fd > 0) + close (l->uio_fd); + vec_free (l->resource_fds); + vec_free (l->dev_dir_name); +} + +/* Configuration space read/write. */ +clib_error_t * +vlib_pci_read_write_config (vlib_pci_device_t * dev, + vlib_read_or_write_t read_or_write, + uword address, void *data, u32 n_bytes) +{ + linux_pci_main_t *lpm = &linux_pci_main; + linux_pci_device_t *p; + int n; + + p = pool_elt_at_index (lpm->linux_pci_devices, dev->os_handle); + + if (read_or_write == VLIB_READ) + n = pread (p->config_fd, data, n_bytes, address); + else + n = pwrite (p->config_fd, data, n_bytes, address); + + if (n != n_bytes) + return clib_error_return_unix (0, "%s", + read_or_write == VLIB_READ + ? "read" : "write"); + + return 0; +} + +static clib_error_t * +os_map_pci_resource_internal (uword os_handle, + u32 resource, u8 * addr, void **result) +{ + linux_pci_main_t *pm = &linux_pci_main; + linux_pci_device_t *p; + struct stat stat_buf; + u8 *file_name; + int fd; + clib_error_t *error; + int flags = MAP_SHARED; + + error = 0; + p = pool_elt_at_index (pm->linux_pci_devices, os_handle); + + file_name = format (0, "%v/resource%d%c", p->dev_dir_name, resource, 0); + fd = open ((char *) file_name, O_RDWR); + if (fd < 0) + { + error = clib_error_return_unix (0, "open `%s'", file_name); + goto done; + } + + if (fstat (fd, &stat_buf) < 0) + { + error = clib_error_return_unix (0, "fstat `%s'", file_name); + goto done; + } + + vec_validate (p->resource_fds, resource); + p->resource_fds[resource] = fd; + if (addr != 0) + flags |= MAP_FIXED; + + *result = mmap (addr, + /* size */ stat_buf.st_size, + PROT_READ | PROT_WRITE, flags, + /* file */ fd, + /* offset */ 0); + if (*result == (void *) -1) + { + error = clib_error_return_unix (0, "mmap `%s'", file_name); + goto done; + } + +done: + if (error) + { + if (fd >= 0) + close (fd); + } + vec_free (file_name); + return error; +} + +clib_error_t * +vlib_pci_map_resource (vlib_pci_device_t * dev, u32 resource, void **result) +{ + return (os_map_pci_resource_internal + (dev->os_handle, resource, 0 /* addr */ , + result)); +} + +clib_error_t * +vlib_pci_map_resource_fixed (vlib_pci_device_t * dev, + u32 resource, u8 * addr, void **result) +{ + return (os_map_pci_resource_internal + (dev->os_handle, resource, addr, result)); +} + +void +vlib_pci_free_device (vlib_pci_device_t * dev) +{ + linux_pci_main_t *pm = &linux_pci_main; + linux_pci_device_t *l; + + l = pool_elt_at_index (pm->linux_pci_devices, dev->os_handle); + linux_pci_device_free (l); + pool_put (pm->linux_pci_devices, l); +} + +pci_device_registration_t * __attribute__ ((unused)) +pci_device_next_registered (pci_device_registration_t * r) +{ + uword i; + + /* Null vendor id marks end of initialized list. */ + for (i = 0; r->supported_devices[i].vendor_id != 0; i++) + ; + + return clib_elf_section_data_next (r, i * sizeof (r->supported_devices[0])); +} + +static clib_error_t * +init_device_from_registered (vlib_main_t * vm, + vlib_pci_device_t * dev, + linux_pci_device_t * pdev) +{ + vlib_pci_main_t *pm = &pci_main; + pci_device_registration_t *r; + pci_device_id_t *i; + clib_error_t *error; + + r = pm->pci_device_registrations; + + while (r) + { + for (i = r->supported_devices; i->vendor_id != 0; i++) + if (i->vendor_id == dev->vendor_id && i->device_id == dev->device_id) + { + error = vlib_pci_bind_to_uio (dev, "uio_pci_generic"); + if (error) + { + clib_error_report (error); + continue; + } + + add_device (dev, pdev); + dev->interrupt_handler = r->interrupt_handler; + return r->init_function (vm, dev); + } + r = r->next_registration; + } + /* No driver, close the PCI config-space FD */ + close (pdev->config_fd); + return 0; +} + +static clib_error_t * +init_device (vlib_main_t * vm, + vlib_pci_device_t * dev, linux_pci_device_t * pdev) +{ + return init_device_from_registered (vm, dev, pdev); +} + +static clib_error_t * +scan_device (void *arg, u8 * dev_dir_name, u8 * ignored) +{ + vlib_main_t *vm = arg; + vlib_pci_main_t *pm = &pci_main; + int fd; + u8 *f; + clib_error_t *error = 0; + vlib_pci_device_t *dev; + linux_pci_device_t pdev = { 0 }; + u32 tmp; + + f = format (0, "%v/config%c", dev_dir_name, 0); + fd = open ((char *) f, O_RDWR); + + /* Try read-only access if write fails. */ + if (fd < 0) + fd = open ((char *) f, O_RDONLY); + + if (fd < 0) + { + error = clib_error_return_unix (0, "open `%s'", f); + goto done; + } + + pool_get (pm->pci_devs, dev); + + /* You can only read more that 64 bytes of config space as root; so we try to + read the full space but fall back to just the first 64 bytes. */ + if (read (fd, &dev->config_data, sizeof (dev->config_data)) != + sizeof (dev->config_data) + && read (fd, &dev->config0, + sizeof (dev->config0)) != sizeof (dev->config0)) + { + pool_put (pm->pci_devs, dev); + error = clib_error_return_unix (0, "read `%s'", f); + close (fd); + goto done; + } + + { + static pci_config_header_t all_ones; + if (all_ones.vendor_id == 0) + memset (&all_ones, ~0, sizeof (all_ones)); + + if (!memcmp (&dev->config0.header, &all_ones, sizeof (all_ones))) + { + pool_put (pm->pci_devs, dev); + error = clib_error_return (0, "invalid PCI config for `%s'", f); + close (fd); + goto done; + } + } + + if (dev->config0.header.header_type == 0) + pci_config_type0_little_to_host (&dev->config0); + else + pci_config_type1_little_to_host (&dev->config1); + + /* Parse bus, dev, function from directory name. */ + { + unformat_input_t input; + + unformat_init_string (&input, (char *) dev_dir_name, + vec_len (dev_dir_name)); + + if (!unformat (&input, "/sys/bus/pci/devices/%U", + unformat_vlib_pci_addr, &dev->bus_address)) + abort (); + + unformat_free (&input); + + } + + + pdev.config_fd = fd; + pdev.dev_dir_name = dev_dir_name; + + hash_set (pm->pci_dev_index_by_pci_addr, dev->bus_address.as_u32, + dev - pm->pci_devs); + + vec_reset_length (f); + f = format (f, "%v/vpd%c", dev_dir_name, 0); + fd = open ((char *) f, O_RDONLY); + if (fd >= 0) + { + while (1) + { + u8 tag[3]; + u8 *data = 0; + int len; + + if (read (fd, &tag, 3) != 3) + break; + + if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91) + break; + + len = (tag[2] << 8) | tag[1]; + vec_validate (data, len); + + if (read (fd, data, len) != len) + { + vec_free (data); + break; + } + if (tag[0] == 0x82) + dev->product_name = data; + else if (tag[0] == 0x90) + dev->vpd_r = data; + else if (tag[0] == 0x91) + dev->vpd_w = data; + + data = 0; + } + close (fd); + } + + dev->numa_node = -1; + vec_reset_length (f); + f = format (f, "%v/numa_node%c", dev_dir_name, 0); + vlib_sysfs_read ((char *) f, "%u", &dev->numa_node); + + vec_reset_length (f); + f = format (f, "%v/class%c", dev_dir_name, 0); + vlib_sysfs_read ((char *) f, "0x%x", &tmp); + dev->device_class = tmp >> 8; + + vec_reset_length (f); + f = format (f, "%v/vendor%c", dev_dir_name, 0); + vlib_sysfs_read ((char *) f, "0x%x", &tmp); + dev->vendor_id = tmp; + + vec_reset_length (f); + f = format (f, "%v/device%c", dev_dir_name, 0); + vlib_sysfs_read ((char *) f, "0x%x", &tmp); + dev->device_id = tmp; + + error = init_device (vm, dev, &pdev); + + vec_reset_length (f); + f = format (f, "%v/driver%c", dev_dir_name, 0); + dev->driver_name = vlib_sysfs_link_to_name ((char *) f); + +done: + vec_free (f); + return error; +} + +clib_error_t * +linux_pci_init (vlib_main_t * vm) +{ + vlib_pci_main_t *pm = &pci_main; + clib_error_t *error; + + pm->vlib_main = vm; + + if ((error = vlib_call_init_function (vm, unix_input_init))) + return error; + + ASSERT (sizeof (vlib_pci_addr_t) == sizeof (u32)); + pm->pci_dev_index_by_pci_addr = hash_create (0, sizeof (uword)); + + error = foreach_directory_file ("/sys/bus/pci/devices", scan_device, vm, + /* scan_dirs */ 0); + + /* Complain and continue. might not be root, etc. */ + if (error) + clib_error_report (error); + + return error; +} + +VLIB_INIT_FUNCTION (linux_pci_init); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vlib/linux/physmem.c b/src/vlib/linux/physmem.c new file mode 100644 index 00000000..6731295c --- /dev/null +++ b/src/vlib/linux/physmem.c @@ -0,0 +1,411 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * physmem.c: Unix physical memory + * + * Copyright (c) 2008 Eliot Dresselhaus + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +static void * +unix_physmem_alloc_aligned (vlib_main_t * vm, vlib_physmem_region_index_t idx, + uword n_bytes, uword alignment) +{ + vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx); + uword lo_offset, hi_offset; + uword *to_free = 0; + + if (pr->heap == 0) + return 0; + + /* IO memory is always at least cache aligned. */ + alignment = clib_max (alignment, CLIB_CACHE_LINE_BYTES); + + while (1) + { + mheap_get_aligned (pr->heap, n_bytes, + /* align */ alignment, + /* align offset */ 0, + &lo_offset); + + /* Allocation failed? */ + if (lo_offset == ~0) + break; + + if (pr->flags & VLIB_PHYSMEM_F_FAKE) + break; + + /* Make sure allocation does not span DMA physical chunk boundary. */ + hi_offset = lo_offset + n_bytes - 1; + + if ((lo_offset >> pr->log2_page_size) == + (hi_offset >> pr->log2_page_size)) + break; + + /* Allocation would span chunk boundary, queue it to be freed as soon as + we find suitable chunk. */ + vec_add1 (to_free, lo_offset); + } + + if (to_free != 0) + { + uword i; + for (i = 0; i < vec_len (to_free); i++) + mheap_put (pr->heap, to_free[i]); + vec_free (to_free); + } + + return lo_offset != ~0 ? pr->heap + lo_offset : 0; +} + +static void +unix_physmem_free (vlib_main_t * vm, vlib_physmem_region_index_t idx, void *x) +{ + vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx); + /* Return object to region's heap. */ + mheap_put (pr->heap, x - pr->heap); +} + +static u64 +get_page_paddr (int fd, uword addr) +{ + int pagesize = sysconf (_SC_PAGESIZE); + u64 seek, pagemap = 0; + + seek = ((u64) addr / pagesize) * sizeof (u64); + if (lseek (fd, seek, SEEK_SET) != seek) + { + clib_unix_warning ("lseek to 0x%llx", seek); + return 0; + } + if (read (fd, &pagemap, sizeof (pagemap)) != (sizeof (pagemap))) + { + clib_unix_warning ("read ptbits"); + return 0; + } + if ((pagemap & (1ULL << 63)) == 0) + return 0; + + pagemap &= pow2_mask (55); + + return pagemap * pagesize; +} + +static clib_error_t * +unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size, + u8 numa_node, u32 flags, + vlib_physmem_region_index_t * idx) +{ + vlib_physmem_main_t *vpm = &vm->physmem_main; + vlib_physmem_region_t *pr; + clib_error_t *error = 0; + int pagemap_fd = -1; + u8 *mount_dir = 0; + u8 *filename = 0; + struct stat st; + int old_mpol; + int mmap_flags; + struct bitmask *old_mask = numa_allocate_nodemask (); + + if (geteuid () != 0 && (flags & VLIB_PHYSMEM_F_FAKE) == 0) + return clib_error_return (0, "not allowed"); + + pool_get (vpm->regions, pr); + + if ((pr - vpm->regions) >= 256) + { + error = clib_error_return (0, "maximum number of regions reached"); + goto error; + } + + pr->index = pr - vpm->regions; + pr->fd = -1; + pr->flags = flags; + + if (get_mempolicy (&old_mpol, old_mask->maskp, old_mask->size + 1, NULL, 0) + == -1) + { + error = clib_error_return_unix (0, "get_mempolicy"); + goto error; + } + + if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) + { + if ((pagemap_fd = open ((char *) "/proc/self/pagemap", O_RDONLY)) == -1) + { + error = clib_error_return_unix (0, "open '/proc/self/pagemap'"); + goto error; + } + + mount_dir = format (0, "%s/physmem_region%d%c", + vlib_unix_get_runtime_dir (), pr->index, 0); + filename = format (0, "%s/mem%c", mount_dir, 0); + + unlink ((char *) mount_dir); + + error = vlib_unix_recursive_mkdir ((char *) mount_dir); + if (error) + goto error; + + if (mount ("none", (char *) mount_dir, "hugetlbfs", 0, NULL)) + { + error = clib_error_return_unix (0, "mount hugetlb directory '%s'", + mount_dir); + goto error; + } + + if ((pr->fd = open ((char *) filename, O_CREAT | O_RDWR, 0755)) == -1) + { + error = clib_error_return_unix (0, "open"); + goto error; + } + + mmap_flags = MAP_SHARED | MAP_HUGETLB | MAP_LOCKED; + } + else + { + if ((pr->fd = memfd_create (name, MFD_ALLOW_SEALING)) == -1) + return clib_error_return_unix (0, "memfd_create"); + + if ((fcntl (pr->fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1) + { + error = + clib_error_return_unix (0, "fcntl (F_ADD_SEALS, F_SEAL_SHRINK)"); + goto error; + } + mmap_flags = MAP_SHARED; + } + + if (fstat (pr->fd, &st)) + { + error = clib_error_return_unix (0, "fstat"); + goto error; + } + + pr->log2_page_size = min_log2 (st.st_blksize); + pr->n_pages = ((size - 1) >> pr->log2_page_size) + 1; + size = pr->n_pages * (1 << pr->log2_page_size); + + if ((ftruncate (pr->fd, size)) == -1) + { + error = clib_error_return_unix (0, "ftruncate length: %d", size); + goto error; + } + + if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) + { + error = vlib_sysfs_prealloc_hugepages (numa_node, + 1 << (pr->log2_page_size - 10), + pr->n_pages); + if (error) + goto error; + } + + numa_set_preferred (numa_node); + + pr->mem = mmap (0, size, (PROT_READ | PROT_WRITE), mmap_flags, pr->fd, 0); + + if (pr->mem == MAP_FAILED) + { + pr->mem = 0; + error = clib_error_return_unix (0, "mmap"); + goto error; + } + + if (set_mempolicy (old_mpol, old_mask->maskp, old_mask->size + 1) == -1) + { + error = clib_error_return_unix (0, "set_mempolicy"); + goto error; + } + + pr->size = pr->n_pages << pr->log2_page_size; + pr->page_mask = (1 << pr->log2_page_size) - 1; + pr->numa_node = numa_node; + pr->name = format (0, "%s", name); + + if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) + { + int i; + for (i = 0; i < pr->n_pages; i++) + { + void *ptr = pr->mem + (i << pr->log2_page_size); + int node; + move_pages (0, 1, &ptr, 0, &node, 0); + if (numa_node != node) + { + clib_warning + ("physmem page for region \'%s\' allocated on the wrong" + " numa node (requested %u actual %u)", pr->name, + pr->numa_node, node, i); + break; + } + } + } + + if (flags & VLIB_PHYSMEM_F_INIT_MHEAP) + { + pr->heap = mheap_alloc_with_flags (pr->mem, pr->size, + /* Don't want mheap mmap/munmap with IO memory. */ + MHEAP_FLAG_DISABLE_VM | + MHEAP_FLAG_THREAD_SAFE); + fformat (stdout, "%U", format_mheap, pr->heap, /* verbose */ 1); + } + + if (flags & VLIB_PHYSMEM_F_HAVE_BUFFERS) + { + vlib_buffer_add_mem_range (vm, pointer_to_uword (pr->mem), pr->size); + } + + *idx = pr->index; + + if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) + { + int i; + for (i = 0; i < pr->n_pages; i++) + { + uword vaddr = + pointer_to_uword (pr->mem) + (((u64) i) << pr->log2_page_size); + u64 page_paddr = get_page_paddr (pagemap_fd, vaddr); + vec_add1 (pr->page_table, page_paddr); + } + } + + goto done; + +error: + if (pr->fd > -1) + close (pr->fd); + + if (pr->mem) + munmap (pr->mem, size); + + memset (pr, 0, sizeof (*pr)); + pool_put (vpm->regions, pr); + +done: + if (mount_dir) + { + umount2 ((char *) mount_dir, MNT_DETACH); + rmdir ((char *) mount_dir); + vec_free (mount_dir); + } + numa_free_cpumask (old_mask); + vec_free (filename); + if (pagemap_fd > -1) + close (pagemap_fd); + return error; +} + +static void +unix_physmem_region_free (vlib_main_t * vm, vlib_physmem_region_index_t idx) +{ + vlib_physmem_main_t *vpm = &vm->physmem_main; + vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx); + + if (pr->fd > 0) + close (pr->fd); + munmap (pr->mem, pr->size); + vec_free (pr->name); + pool_put (vpm->regions, pr); +} + +clib_error_t * +unix_physmem_init (vlib_main_t * vm) +{ + clib_error_t *error = 0; + + /* Avoid multiple calls. */ + if (vm->os_physmem_alloc_aligned) + return error; + + vm->os_physmem_alloc_aligned = unix_physmem_alloc_aligned; + vm->os_physmem_free = unix_physmem_free; + vm->os_physmem_region_alloc = unix_physmem_region_alloc; + vm->os_physmem_region_free = unix_physmem_region_free; + + return error; +} + +static clib_error_t * +show_physmem (vlib_main_t * vm, + unformat_input_t * input, vlib_cli_command_t * cmd) +{ + vlib_physmem_main_t *vpm = &vm->physmem_main; + vlib_physmem_region_t *pr; + + /* *INDENT-OFF* */ + pool_foreach (pr, vpm->regions, ( + { + vlib_cli_output (vm, "index %u name '%s' page-size %uKB num-pages %d " + "numa-node %u fd %d\n", + pr->index, pr->name, (1 << (pr->log2_page_size -10)), + pr->n_pages, pr->numa_node, pr->fd); + if (pr->heap) + vlib_cli_output (vm, " %U", format_mheap, pr->heap, /* verbose */ 1); + else + vlib_cli_output (vm, " no heap\n"); + })); + /* *INDENT-ON* */ + return 0; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (show_physmem_command, static) = { + .path = "show physmem", + .short_help = "Show physical memory allocation", + .function = show_physmem, +}; +/* *INDENT-ON* */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vlib/linux/syscall.h b/src/vlib/linux/syscall.h new file mode 100644 index 00000000..9e37997e --- /dev/null +++ b/src/vlib/linux/syscall.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef included_linux_syscall_h +#define included_linux_syscall_h + +#ifndef __NR_memfd_create +#if defined __x86_64__ +#define __NR_memfd_create 319 +#elif defined __arm__ +#define __NR_memfd_create 385 +#elif defined __aarch64__ +#define __NR_memfd_create 279 +#else +#error "__NR_memfd_create unknown for this architecture" +#endif +#endif + +static inline int +memfd_create (const char *name, unsigned int flags) +{ + return syscall (__NR_memfd_create, name, flags); +} + +#ifndef F_LINUX_SPECIFIC_BASE +#define F_LINUX_SPECIFIC_BASE 1024 +#endif +#define MFD_ALLOW_SEALING 0x0002U +#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9) +#define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10) + +#define F_SEAL_SEAL 0x0001 /* prevent further seals from being set */ +#define F_SEAL_SHRINK 0x0002 /* prevent file from shrinking */ +#define F_SEAL_GROW 0x0004 /* prevent file from growing */ +#define F_SEAL_WRITE 0x0008 /* prevent writes */ + + +#endif /* included_linux_syscall_h */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vlib/linux/sysfs.c b/src/vlib/linux/sysfs.c new file mode 100644 index 00000000..f92f9ef5 --- /dev/null +++ b/src/vlib/linux/sysfs.c @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include + +clib_error_t * +vlib_sysfs_write (char *file_name, char *fmt, ...) +{ + u8 *s; + int fd; + clib_error_t *error = 0; + + fd = open (file_name, O_WRONLY); + if (fd < 0) + return clib_error_return_unix (0, "open `%s'", file_name); + + va_list va; + va_start (va, fmt); + s = va_format (0, fmt, &va); + va_end (va); + + if (write (fd, s, vec_len (s)) < 0) + error = clib_error_return_unix (0, "write `%s'", file_name); + + vec_free (s); + close (fd); + return error; +} + +clib_error_t * +vlib_sysfs_read (char *file_name, char *fmt, ...) +{ + unformat_input_t input; + u8 *s = 0; + int fd; + ssize_t sz; + uword result; + + fd = open (file_name, O_RDONLY); + if (fd < 0) + return clib_error_return_unix (0, "open `%s'", file_name); + + vec_validate (s, 4095); + + sz = read (fd, s, vec_len (s)); + if (sz < 0) + { + close (fd); + vec_free (s); + return clib_error_return_unix (0, "read `%s'", file_name); + } + + _vec_len (s) = sz; + unformat_init_vector (&input, s); + + va_list va; + va_start (va, fmt); + result = va_unformat (&input, fmt, &va); + va_end (va); + + vec_free (s); + close (fd); + + if (result == 0) + return clib_error_return (0, "unformat error"); + + return 0; +} + +u8 * +vlib_sysfs_link_to_name (char *link) +{ + char *p, buffer[64]; + unformat_input_t in; + u8 *s = 0; + int r; + + r = readlink (link, buffer, sizeof (buffer) - 1); + + if (r < 0) + return 0; + + buffer[r] = 0; + p = strrchr (buffer, '/'); + + if (!p) + return 0; + + unformat_init_string (&in, p + 1, strlen (p + 1)); + if (unformat (&in, "%s", &s) != 1) + clib_unix_warning ("no string?"); + unformat_free (&in); + + return s; +} + +clib_error_t * +vlib_sysfs_set_nr_hugepages (unsigned int numa_node, int page_size, int nr) +{ + clib_error_t *error = 0; + struct stat sb; + u8 *p = 0; + + p = format (p, "/sys/devices/system/node/node%u%c", numa_node, 0); + + if (stat ((char *) p, &sb) == 0) + { + if (S_ISDIR (sb.st_mode) == 0) + { + error = clib_error_return (0, "'%s' is not directory", p); + goto done; + } + } + else if (numa_node == 0) + { + vec_reset_length (p); + p = format (p, "/sys/kernel/mm%c", 0); + if (stat ((char *) p, &sb) < 0 || S_ISDIR (sb.st_mode) == 0) + { + error = clib_error_return (0, "'%s' does not exist or it is not " + "directory", p); + goto done; + } + } + else + { + error = clib_error_return (0, "'%s' does not exist", p); + goto done; + } + + _vec_len (p) -= 1; + p = format (p, "/hugepages/hugepages-%ukB/nr_hugepages%c", page_size, 0); + vlib_sysfs_write ((char *) p, "%d", nr); + +done: + vec_free (p); + return error; +} + + +static clib_error_t * +vlib_sysfs_get_xxx_hugepages (char *type, unsigned int numa_node, + int page_size, int *val) +{ + clib_error_t *error = 0; + struct stat sb; + u8 *p = 0; + + p = format (p, "/sys/devices/system/node/node%u%c", numa_node, 0); + + if (stat ((char *) p, &sb) == 0) + { + if (S_ISDIR (sb.st_mode) == 0) + { + error = clib_error_return (0, "'%s' is not directory", p); + goto done; + } + } + else if (numa_node == 0) + { + vec_reset_length (p); + p = format (p, "/sys/kernel/mm%c", 0); + if (stat ((char *) p, &sb) < 0 || S_ISDIR (sb.st_mode) == 0) + { + error = clib_error_return (0, "'%s' does not exist or it is not " + "directory", p); + goto done; + } + } + else + { + error = clib_error_return (0, "'%s' does not exist", p); + goto done; + } + + _vec_len (p) -= 1; + p = format (p, "/hugepages/hugepages-%ukB/%s_hugepages%c", page_size, + type, 0); + error = vlib_sysfs_read ((char *) p, "%d", val); + +done: + vec_free (p); + return error; +} + +clib_error_t * +vlib_sysfs_get_free_hugepages (unsigned int numa_node, int page_size, int *v) +{ + return vlib_sysfs_get_xxx_hugepages ("free", numa_node, page_size, v); +} + +clib_error_t * +vlib_sysfs_get_nr_hugepages (unsigned int numa_node, int page_size, int *v) +{ + return vlib_sysfs_get_xxx_hugepages ("nr", numa_node, page_size, v); +} + +clib_error_t * +vlib_sysfs_get_surplus_hugepages (unsigned int numa_node, int page_size, + int *v) +{ + return vlib_sysfs_get_xxx_hugepages ("surplus", numa_node, page_size, v); +} + +clib_error_t * +vlib_sysfs_prealloc_hugepages (unsigned int numa_node, int page_size, int nr) +{ + clib_error_t *error = 0; + int n, needed; + error = vlib_sysfs_get_free_hugepages (numa_node, page_size, &n); + if (error) + return error; + needed = nr - n; + if (needed <= 0) + return 0; + + error = vlib_sysfs_get_nr_hugepages (numa_node, page_size, &n); + if (error) + return error; + clib_warning ("pre-allocating %u additional %uK hugepages on numa node %u", + needed, page_size, numa_node); + return vlib_sysfs_set_nr_hugepages (numa_node, page_size, n + needed); +} + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vlib/linux/sysfs.h b/src/vlib/linux/sysfs.h new file mode 100644 index 00000000..14b71317 --- /dev/null +++ b/src/vlib/linux/sysfs.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef included_linux_sysfs_h +#define included_linux_sysfs_h + +clib_error_t *vlib_sysfs_write (char *file_name, char *fmt, ...); + +clib_error_t *vlib_sysfs_read (char *file_name, char *fmt, ...); + +u8 *vlib_sysfs_link_to_name (char *link); + +clib_error_t *vlib_sysfs_set_nr_hugepages (unsigned int numa_node, + int page_size, int nr); +clib_error_t *vlib_sysfs_get_nr_hugepages (unsigned int numa_node, + int page_size, int *v); +clib_error_t *vlib_sysfs_get_free_hugepages (unsigned int numa_node, + int page_size, int *v); +clib_error_t *vlib_sysfs_get_surplus_hugepages (unsigned int numa_node, + int page_size, int *v); +clib_error_t *vlib_sysfs_prealloc_hugepages (unsigned int numa_node, + int page_size, int nr); + +#endif /* included_linux_sysfs_h */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vlib/pci/linux_pci.c b/src/vlib/pci/linux_pci.c deleted file mode 100644 index 2d3c0a88..00000000 --- a/src/vlib/pci/linux_pci.c +++ /dev/null @@ -1,665 +0,0 @@ -/* - * Copyright (c) 2016 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - * pci.c: Linux user space PCI bus management. - * - * Copyright (c) 2008 Eliot Dresselhaus - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -typedef struct -{ - /* /sys/bus/pci/devices/... directory name for this device. */ - u8 *dev_dir_name; - - /* Resource file descriptors. */ - int *resource_fds; - - /* File descriptor for config space read/write. */ - int config_fd; - - /* File descriptor for /dev/uio%d */ - int uio_fd; - - /* Minor device for uio device. */ - u32 uio_minor; - - /* Index given by unix_file_add. */ - u32 unix_file_index; - -} linux_pci_device_t; - -/* Pool of PCI devices. */ -typedef struct -{ - vlib_main_t *vlib_main; - linux_pci_device_t *linux_pci_devices; -} linux_pci_main_t; - -extern linux_pci_main_t linux_pci_main; - -/* Call to allocate/initialize the pci subsystem. - This is not an init function so that users can explicitly enable - pci only when it's needed. */ -clib_error_t *pci_bus_init (vlib_main_t * vm); - -clib_error_t *vlib_pci_bind_to_uio (vlib_pci_device_t * d, - char *uio_driver_name); - -linux_pci_main_t linux_pci_main; - -clib_error_t * -vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name) -{ - clib_error_t *error = 0; - u8 *s = 0, *driver_name = 0; - DIR *dir = 0; - struct dirent *e; - int fd, clear_driver_override = 0; - u8 *dev_dir_name = format (0, "/sys/bus/pci/devices/%U", - format_vlib_pci_addr, &d->bus_address); - - s = format (s, "%v/driver%c", dev_dir_name, 0); - driver_name = vlib_sysfs_link_to_name ((char *) s); - vec_reset_length (s); - - if (driver_name && - ((strcmp ("vfio-pci", (char *) driver_name) == 0) || - (strcmp ("uio_pci_generic", (char *) driver_name) == 0) || - (strcmp ("igb_uio", (char *) driver_name) == 0))) - goto done; - - /* walk trough all linux interfaces and if interface belonging to - this device is founf check if interface is admin up */ - dir = opendir ("/sys/class/net"); - s = format (s, "%U%c", format_vlib_pci_addr, &d->bus_address, 0); - - if (!dir) - { - error = clib_error_return (0, "Skipping PCI device %U: failed to " - "read /sys/class/net", - format_vlib_pci_addr, &d->bus_address); - goto done; - } - - fd = socket (PF_INET, SOCK_DGRAM, 0); - if (fd < 0) - { - error = clib_error_return_unix (0, "socket"); - goto done; - } - - while ((e = readdir (dir))) - { - struct ifreq ifr; - struct ethtool_drvinfo drvinfo; - - if (e->d_name[0] == '.') /* skip . and .. */ - continue; - - memset (&ifr, 0, sizeof ifr); - memset (&drvinfo, 0, sizeof drvinfo); - ifr.ifr_data = (char *) &drvinfo; - strncpy (ifr.ifr_name, e->d_name, IFNAMSIZ - 1); - drvinfo.cmd = ETHTOOL_GDRVINFO; - if (ioctl (fd, SIOCETHTOOL, &ifr) < 0) - { - /* Some interfaces (eg "lo") don't support this ioctl */ - if ((errno != ENOTSUP) && (errno != ENODEV)) - clib_unix_warning ("ioctl fetch intf %s bus info error", - e->d_name); - continue; - } - - if (strcmp ((char *) s, drvinfo.bus_info)) - continue; - - memset (&ifr, 0, sizeof (ifr)); - strncpy (ifr.ifr_name, e->d_name, IFNAMSIZ - 1); - if (ioctl (fd, SIOCGIFFLAGS, &ifr) < 0) - { - error = clib_error_return_unix (0, "ioctl fetch intf %s flags", - e->d_name); - close (fd); - goto done; - } - - if (ifr.ifr_flags & IFF_UP) - { - error = clib_error_return (0, "Skipping PCI device %U as host " - "interface %s is up", - format_vlib_pci_addr, &d->bus_address, - e->d_name); - close (fd); - goto done; - } - } - - close (fd); - vec_reset_length (s); - - s = format (s, "%v/driver/unbind%c", dev_dir_name, 0); - vlib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address); - vec_reset_length (s); - - s = format (s, "%v/driver_override%c", dev_dir_name, 0); - if (access ((char *) s, F_OK) == 0) - { - vlib_sysfs_write ((char *) s, "%s", uio_driver_name); - clear_driver_override = 1; - } - else - { - vec_reset_length (s); - s = format (s, "/sys/bus/pci/drivers/%s/new_id%c", uio_driver_name, 0); - vlib_sysfs_write ((char *) s, "0x%04x 0x%04x", d->vendor_id, - d->device_id); - } - vec_reset_length (s); - - s = format (s, "/sys/bus/pci/drivers/%s/bind%c", uio_driver_name, 0); - vlib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address); - vec_reset_length (s); - - if (clear_driver_override) - { - s = format (s, "%v/driver_override%c", dev_dir_name, 0); - vlib_sysfs_write ((char *) s, "%c", 0); - vec_reset_length (s); - } - -done: - closedir (dir); - vec_free (s); - vec_free (dev_dir_name); - vec_free (driver_name); - return error; -} - - -static clib_error_t * -scan_uio_dir (void *arg, u8 * path_name, u8 * file_name) -{ - linux_pci_device_t *l = arg; - unformat_input_t input; - - unformat_init_string (&input, (char *) file_name, vec_len (file_name)); - - if (!unformat (&input, "uio%d", &l->uio_minor)) - abort (); - - unformat_free (&input); - return 0; -} - -static clib_error_t * -linux_pci_uio_read_ready (unix_file_t * uf) -{ - vlib_pci_main_t *pm = &pci_main; - vlib_pci_device_t *d; - int __attribute__ ((unused)) rv; - - u32 icount; - rv = read (uf->file_descriptor, &icount, 4); - - d = pool_elt_at_index (pm->pci_devs, uf->private_data); - - if (d->interrupt_handler) - d->interrupt_handler (d); - - vlib_pci_intr_enable (d); - - return /* no error */ 0; -} - -static clib_error_t * -linux_pci_uio_error_ready (unix_file_t * uf) -{ - u32 error_index = (u32) uf->private_data; - - return clib_error_return (0, "pci device %d: error", error_index); -} - -static void -add_device (vlib_pci_device_t * dev, linux_pci_device_t * pdev) -{ - vlib_pci_main_t *pm = &pci_main; - linux_pci_main_t *lpm = &linux_pci_main; - linux_pci_device_t *l; - - pool_get (lpm->linux_pci_devices, l); - l[0] = pdev[0]; - - l->dev_dir_name = vec_dup (l->dev_dir_name); - - dev->os_handle = l - lpm->linux_pci_devices; - - { - u8 *uio_dir = format (0, "%s/uio", l->dev_dir_name); - foreach_directory_file ((char *) uio_dir, scan_uio_dir, l, /* scan_dirs */ - 1); - vec_free (uio_dir); - } - - { - char *uio_name = (char *) format (0, "/dev/uio%d%c", l->uio_minor, 0); - l->uio_fd = open (uio_name, O_RDWR); - if (l->uio_fd < 0) - clib_unix_error ("open `%s'", uio_name); - vec_free (uio_name); - } - - { - unix_file_t template = { 0 }; - unix_main_t *um = &unix_main; - - template.read_function = linux_pci_uio_read_ready; - template.file_descriptor = l->uio_fd; - template.error_function = linux_pci_uio_error_ready; - template.private_data = dev - pm->pci_devs; - - l->unix_file_index = unix_file_add (um, &template); - } -} - -static void -linux_pci_device_free (linux_pci_device_t * l) -{ - int i; - for (i = 0; i < vec_len (l->resource_fds); i++) - if (l->resource_fds[i] > 0) - close (l->resource_fds[i]); - if (l->config_fd > 0) - close (l->config_fd); - if (l->uio_fd > 0) - close (l->uio_fd); - vec_free (l->resource_fds); - vec_free (l->dev_dir_name); -} - -/* Configuration space read/write. */ -clib_error_t * -vlib_pci_read_write_config (vlib_pci_device_t * dev, - vlib_read_or_write_t read_or_write, - uword address, void *data, u32 n_bytes) -{ - linux_pci_main_t *lpm = &linux_pci_main; - linux_pci_device_t *p; - int n; - - p = pool_elt_at_index (lpm->linux_pci_devices, dev->os_handle); - - if (read_or_write == VLIB_READ) - n = pread (p->config_fd, data, n_bytes, address); - else - n = pwrite (p->config_fd, data, n_bytes, address); - - if (n != n_bytes) - return clib_error_return_unix (0, "%s", - read_or_write == VLIB_READ - ? "read" : "write"); - - return 0; -} - -static clib_error_t * -os_map_pci_resource_internal (uword os_handle, - u32 resource, u8 * addr, void **result) -{ - linux_pci_main_t *pm = &linux_pci_main; - linux_pci_device_t *p; - struct stat stat_buf; - u8 *file_name; - int fd; - clib_error_t *error; - int flags = MAP_SHARED; - - error = 0; - p = pool_elt_at_index (pm->linux_pci_devices, os_handle); - - file_name = format (0, "%v/resource%d%c", p->dev_dir_name, resource, 0); - fd = open ((char *) file_name, O_RDWR); - if (fd < 0) - { - error = clib_error_return_unix (0, "open `%s'", file_name); - goto done; - } - - if (fstat (fd, &stat_buf) < 0) - { - error = clib_error_return_unix (0, "fstat `%s'", file_name); - goto done; - } - - vec_validate (p->resource_fds, resource); - p->resource_fds[resource] = fd; - if (addr != 0) - flags |= MAP_FIXED; - - *result = mmap (addr, - /* size */ stat_buf.st_size, - PROT_READ | PROT_WRITE, flags, - /* file */ fd, - /* offset */ 0); - if (*result == (void *) -1) - { - error = clib_error_return_unix (0, "mmap `%s'", file_name); - goto done; - } - -done: - if (error) - { - if (fd >= 0) - close (fd); - } - vec_free (file_name); - return error; -} - -clib_error_t * -vlib_pci_map_resource (vlib_pci_device_t * dev, u32 resource, void **result) -{ - return (os_map_pci_resource_internal - (dev->os_handle, resource, 0 /* addr */ , - result)); -} - -clib_error_t * -vlib_pci_map_resource_fixed (vlib_pci_device_t * dev, - u32 resource, u8 * addr, void **result) -{ - return (os_map_pci_resource_internal - (dev->os_handle, resource, addr, result)); -} - -void -vlib_pci_free_device (vlib_pci_device_t * dev) -{ - linux_pci_main_t *pm = &linux_pci_main; - linux_pci_device_t *l; - - l = pool_elt_at_index (pm->linux_pci_devices, dev->os_handle); - linux_pci_device_free (l); - pool_put (pm->linux_pci_devices, l); -} - -pci_device_registration_t * __attribute__ ((unused)) -pci_device_next_registered (pci_device_registration_t * r) -{ - uword i; - - /* Null vendor id marks end of initialized list. */ - for (i = 0; r->supported_devices[i].vendor_id != 0; i++) - ; - - return clib_elf_section_data_next (r, i * sizeof (r->supported_devices[0])); -} - -static clib_error_t * -init_device_from_registered (vlib_main_t * vm, - vlib_pci_device_t * dev, - linux_pci_device_t * pdev) -{ - vlib_pci_main_t *pm = &pci_main; - pci_device_registration_t *r; - pci_device_id_t *i; - clib_error_t *error; - - r = pm->pci_device_registrations; - - while (r) - { - for (i = r->supported_devices; i->vendor_id != 0; i++) - if (i->vendor_id == dev->vendor_id && i->device_id == dev->device_id) - { - error = vlib_pci_bind_to_uio (dev, "uio_pci_generic"); - if (error) - { - clib_error_report (error); - continue; - } - - add_device (dev, pdev); - dev->interrupt_handler = r->interrupt_handler; - return r->init_function (vm, dev); - } - r = r->next_registration; - } - /* No driver, close the PCI config-space FD */ - close (pdev->config_fd); - return 0; -} - -static clib_error_t * -init_device (vlib_main_t * vm, - vlib_pci_device_t * dev, linux_pci_device_t * pdev) -{ - return init_device_from_registered (vm, dev, pdev); -} - -static clib_error_t * -scan_device (void *arg, u8 * dev_dir_name, u8 * ignored) -{ - vlib_main_t *vm = arg; - vlib_pci_main_t *pm = &pci_main; - int fd; - u8 *f; - clib_error_t *error = 0; - vlib_pci_device_t *dev; - linux_pci_device_t pdev = { 0 }; - u32 tmp; - - f = format (0, "%v/config%c", dev_dir_name, 0); - fd = open ((char *) f, O_RDWR); - - /* Try read-only access if write fails. */ - if (fd < 0) - fd = open ((char *) f, O_RDONLY); - - if (fd < 0) - { - error = clib_error_return_unix (0, "open `%s'", f); - goto done; - } - - pool_get (pm->pci_devs, dev); - - /* You can only read more that 64 bytes of config space as root; so we try to - read the full space but fall back to just the first 64 bytes. */ - if (read (fd, &dev->config_data, sizeof (dev->config_data)) != - sizeof (dev->config_data) - && read (fd, &dev->config0, - sizeof (dev->config0)) != sizeof (dev->config0)) - { - pool_put (pm->pci_devs, dev); - error = clib_error_return_unix (0, "read `%s'", f); - close (fd); - goto done; - } - - { - static pci_config_header_t all_ones; - if (all_ones.vendor_id == 0) - memset (&all_ones, ~0, sizeof (all_ones)); - - if (!memcmp (&dev->config0.header, &all_ones, sizeof (all_ones))) - { - pool_put (pm->pci_devs, dev); - error = clib_error_return (0, "invalid PCI config for `%s'", f); - close (fd); - goto done; - } - } - - if (dev->config0.header.header_type == 0) - pci_config_type0_little_to_host (&dev->config0); - else - pci_config_type1_little_to_host (&dev->config1); - - /* Parse bus, dev, function from directory name. */ - { - unformat_input_t input; - - unformat_init_string (&input, (char *) dev_dir_name, - vec_len (dev_dir_name)); - - if (!unformat (&input, "/sys/bus/pci/devices/%U", - unformat_vlib_pci_addr, &dev->bus_address)) - abort (); - - unformat_free (&input); - - } - - - pdev.config_fd = fd; - pdev.dev_dir_name = dev_dir_name; - - hash_set (pm->pci_dev_index_by_pci_addr, dev->bus_address.as_u32, - dev - pm->pci_devs); - - vec_reset_length (f); - f = format (f, "%v/vpd%c", dev_dir_name, 0); - fd = open ((char *) f, O_RDONLY); - if (fd >= 0) - { - while (1) - { - u8 tag[3]; - u8 *data = 0; - int len; - - if (read (fd, &tag, 3) != 3) - break; - - if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91) - break; - - len = (tag[2] << 8) | tag[1]; - vec_validate (data, len); - - if (read (fd, data, len) != len) - { - vec_free (data); - break; - } - if (tag[0] == 0x82) - dev->product_name = data; - else if (tag[0] == 0x90) - dev->vpd_r = data; - else if (tag[0] == 0x91) - dev->vpd_w = data; - - data = 0; - } - close (fd); - } - - dev->numa_node = -1; - vec_reset_length (f); - f = format (f, "%v/numa_node%c", dev_dir_name, 0); - vlib_sysfs_read ((char *) f, "%u", &dev->numa_node); - - vec_reset_length (f); - f = format (f, "%v/class%c", dev_dir_name, 0); - vlib_sysfs_read ((char *) f, "0x%x", &tmp); - dev->device_class = tmp >> 8; - - vec_reset_length (f); - f = format (f, "%v/vendor%c", dev_dir_name, 0); - vlib_sysfs_read ((char *) f, "0x%x", &tmp); - dev->vendor_id = tmp; - - vec_reset_length (f); - f = format (f, "%v/device%c", dev_dir_name, 0); - vlib_sysfs_read ((char *) f, "0x%x", &tmp); - dev->device_id = tmp; - - error = init_device (vm, dev, &pdev); - - vec_reset_length (f); - f = format (f, "%v/driver%c", dev_dir_name, 0); - dev->driver_name = vlib_sysfs_link_to_name ((char *) f); - -done: - vec_free (f); - return error; -} - -clib_error_t * -linux_pci_init (vlib_main_t * vm) -{ - vlib_pci_main_t *pm = &pci_main; - clib_error_t *error; - - pm->vlib_main = vm; - - if ((error = vlib_call_init_function (vm, unix_input_init))) - return error; - - ASSERT (sizeof (vlib_pci_addr_t) == sizeof (u32)); - pm->pci_dev_index_by_pci_addr = hash_create (0, sizeof (uword)); - - error = foreach_directory_file ("/sys/bus/pci/devices", scan_device, vm, - /* scan_dirs */ 0); - - /* Complain and continue. might not be root, etc. */ - if (error) - clib_error_report (error); - - return error; -} - -VLIB_INIT_FUNCTION (linux_pci_init); - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vlib/threads_cli.c b/src/vlib/threads_cli.c index 36f8109e..f8d5d8f9 100644 --- a/src/vlib/threads_cli.c +++ b/src/vlib/threads_cli.c @@ -18,6 +18,7 @@ #include #include +#include #include static u8 * diff --git a/src/vlib/unix/physmem.c b/src/vlib/unix/physmem.c deleted file mode 100644 index d5d5d6c8..00000000 --- a/src/vlib/unix/physmem.c +++ /dev/null @@ -1,439 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - * physmem.c: Unix physical memory - * - * Copyright (c) 2008 Eliot Dresselhaus - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#ifndef __NR_memfd_create -#if defined __x86_64__ -#define __NR_memfd_create 319 -#elif defined __arm__ -#define __NR_memfd_create 385 -#elif defined __aarch64__ -#define __NR_memfd_create 279 -#else -#error "__NR_memfd_create unknown for this architecture" -#endif -#endif - -static inline int -memfd_create (const char *name, unsigned int flags) -{ - return syscall (__NR_memfd_create, name, flags); -} - -#ifndef F_LINUX_SPECIFIC_BASE -#define F_LINUX_SPECIFIC_BASE 1024 -#endif -#define MFD_ALLOW_SEALING 0x0002U -#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9) -#define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10) - -#define F_SEAL_SEAL 0x0001 /* prevent further seals from being set */ -#define F_SEAL_SHRINK 0x0002 /* prevent file from shrinking */ -#define F_SEAL_GROW 0x0004 /* prevent file from growing */ -#define F_SEAL_WRITE 0x0008 /* prevent writes */ - -static void * -unix_physmem_alloc_aligned (vlib_main_t * vm, vlib_physmem_region_index_t idx, - uword n_bytes, uword alignment) -{ - vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx); - uword lo_offset, hi_offset; - uword *to_free = 0; - - if (pr->heap == 0) - return 0; - - /* IO memory is always at least cache aligned. */ - alignment = clib_max (alignment, CLIB_CACHE_LINE_BYTES); - - while (1) - { - mheap_get_aligned (pr->heap, n_bytes, - /* align */ alignment, - /* align offset */ 0, - &lo_offset); - - /* Allocation failed? */ - if (lo_offset == ~0) - break; - - if (pr->flags & VLIB_PHYSMEM_F_FAKE) - break; - - /* Make sure allocation does not span DMA physical chunk boundary. */ - hi_offset = lo_offset + n_bytes - 1; - - if ((lo_offset >> pr->log2_page_size) == - (hi_offset >> pr->log2_page_size)) - break; - - /* Allocation would span chunk boundary, queue it to be freed as soon as - we find suitable chunk. */ - vec_add1 (to_free, lo_offset); - } - - if (to_free != 0) - { - uword i; - for (i = 0; i < vec_len (to_free); i++) - mheap_put (pr->heap, to_free[i]); - vec_free (to_free); - } - - return lo_offset != ~0 ? pr->heap + lo_offset : 0; -} - -static void -unix_physmem_free (vlib_main_t * vm, vlib_physmem_region_index_t idx, void *x) -{ - vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx); - /* Return object to region's heap. */ - mheap_put (pr->heap, x - pr->heap); -} - -static u64 -get_page_paddr (int fd, uword addr) -{ - int pagesize = sysconf (_SC_PAGESIZE); - u64 seek, pagemap = 0; - - seek = ((u64) addr / pagesize) * sizeof (u64); - if (lseek (fd, seek, SEEK_SET) != seek) - { - clib_unix_warning ("lseek to 0x%llx", seek); - return 0; - } - if (read (fd, &pagemap, sizeof (pagemap)) != (sizeof (pagemap))) - { - clib_unix_warning ("read ptbits"); - return 0; - } - if ((pagemap & (1ULL << 63)) == 0) - return 0; - - pagemap &= pow2_mask (55); - - return pagemap * pagesize; -} - -static clib_error_t * -unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size, - u8 numa_node, u32 flags, - vlib_physmem_region_index_t * idx) -{ - vlib_physmem_main_t *vpm = &vm->physmem_main; - vlib_physmem_region_t *pr; - clib_error_t *error = 0; - int pagemap_fd = -1; - u8 *mount_dir = 0; - u8 *filename = 0; - struct stat st; - int old_mpol; - int mmap_flags; - struct bitmask *old_mask = numa_allocate_nodemask (); - - if (geteuid () != 0 && (flags & VLIB_PHYSMEM_F_FAKE) == 0) - return clib_error_return (0, "not allowed"); - - pool_get (vpm->regions, pr); - - if ((pr - vpm->regions) >= 256) - { - error = clib_error_return (0, "maximum number of regions reached"); - goto error; - } - - pr->index = pr - vpm->regions; - pr->fd = -1; - pr->flags = flags; - - if (get_mempolicy (&old_mpol, old_mask->maskp, old_mask->size + 1, NULL, 0) - == -1) - { - error = clib_error_return_unix (0, "get_mempolicy"); - goto error; - } - - if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) - { - if ((pagemap_fd = open ((char *) "/proc/self/pagemap", O_RDONLY)) == -1) - { - error = clib_error_return_unix (0, "open '/proc/self/pagemap'"); - goto error; - } - - mount_dir = format (0, "%s/physmem_region%d%c", - vlib_unix_get_runtime_dir (), pr->index, 0); - filename = format (0, "%s/mem%c", mount_dir, 0); - - unlink ((char *) mount_dir); - - error = vlib_unix_recursive_mkdir ((char *) mount_dir); - if (error) - goto error; - - if (mount ("none", (char *) mount_dir, "hugetlbfs", 0, NULL)) - { - error = clib_error_return_unix (0, "mount hugetlb directory '%s'", - mount_dir); - goto error; - } - - if ((pr->fd = open ((char *) filename, O_CREAT | O_RDWR, 0755)) == -1) - { - error = clib_error_return_unix (0, "open"); - goto error; - } - - mmap_flags = MAP_SHARED | MAP_HUGETLB | MAP_LOCKED; - } - else - { - if ((pr->fd = memfd_create (name, MFD_ALLOW_SEALING)) == -1) - return clib_error_return_unix (0, "memfd_create"); - - if ((fcntl (pr->fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1) - { - error = - clib_error_return_unix (0, "fcntl (F_ADD_SEALS, F_SEAL_SHRINK)"); - goto error; - } - mmap_flags = MAP_SHARED; - } - - if (fstat (pr->fd, &st)) - { - error = clib_error_return_unix (0, "fstat"); - goto error; - } - - pr->log2_page_size = min_log2 (st.st_blksize); - pr->n_pages = ((size - 1) >> pr->log2_page_size) + 1; - size = pr->n_pages * (1 << pr->log2_page_size); - - if ((ftruncate (pr->fd, size)) == -1) - { - error = clib_error_return_unix (0, "ftruncate length: %d", size); - goto error; - } - - if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) - { - error = vlib_sysfs_prealloc_hugepages (numa_node, - 1 << (pr->log2_page_size - 10), - pr->n_pages); - if (error) - goto error; - } - - numa_set_preferred (numa_node); - - pr->mem = mmap (0, size, (PROT_READ | PROT_WRITE), mmap_flags, pr->fd, 0); - - if (pr->mem == MAP_FAILED) - { - pr->mem = 0; - error = clib_error_return_unix (0, "mmap"); - goto error; - } - - if (set_mempolicy (old_mpol, old_mask->maskp, old_mask->size + 1) == -1) - { - error = clib_error_return_unix (0, "set_mempolicy"); - goto error; - } - - pr->size = pr->n_pages << pr->log2_page_size; - pr->page_mask = (1 << pr->log2_page_size) - 1; - pr->numa_node = numa_node; - pr->name = format (0, "%s", name); - - if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) - { - int i; - for (i = 0; i < pr->n_pages; i++) - { - void *ptr = pr->mem + (i << pr->log2_page_size); - int node; - move_pages (0, 1, &ptr, 0, &node, 0); - if (numa_node != node) - { - clib_warning - ("physmem page for region \'%s\' allocated on the wrong" - " numa node (requested %u actual %u)", pr->name, - pr->numa_node, node, i); - break; - } - } - } - - if (flags & VLIB_PHYSMEM_F_INIT_MHEAP) - { - pr->heap = mheap_alloc_with_flags (pr->mem, pr->size, - /* Don't want mheap mmap/munmap with IO memory. */ - MHEAP_FLAG_DISABLE_VM | - MHEAP_FLAG_THREAD_SAFE); - fformat (stdout, "%U", format_mheap, pr->heap, /* verbose */ 1); - } - - if (flags & VLIB_PHYSMEM_F_HAVE_BUFFERS) - { - vlib_buffer_add_mem_range (vm, pointer_to_uword (pr->mem), pr->size); - } - - *idx = pr->index; - - if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) - { - int i; - for (i = 0; i < pr->n_pages; i++) - { - uword vaddr = - pointer_to_uword (pr->mem) + (((u64) i) << pr->log2_page_size); - u64 page_paddr = get_page_paddr (pagemap_fd, vaddr); - vec_add1 (pr->page_table, page_paddr); - } - } - - goto done; - -error: - if (pr->fd > -1) - close (pr->fd); - - if (pr->mem) - munmap (pr->mem, size); - - memset (pr, 0, sizeof (*pr)); - pool_put (vpm->regions, pr); - -done: - if (mount_dir) - { - umount2 ((char *) mount_dir, MNT_DETACH); - rmdir ((char *) mount_dir); - vec_free (mount_dir); - } - numa_free_cpumask (old_mask); - vec_free (filename); - if (pagemap_fd > -1) - close (pagemap_fd); - return error; -} - -static void -unix_physmem_region_free (vlib_main_t * vm, vlib_physmem_region_index_t idx) -{ - vlib_physmem_main_t *vpm = &vm->physmem_main; - vlib_physmem_region_t *pr = vlib_physmem_get_region (vm, idx); - - if (pr->fd > 0) - close (pr->fd); - munmap (pr->mem, pr->size); - vec_free (pr->name); - pool_put (vpm->regions, pr); -} - -clib_error_t * -unix_physmem_init (vlib_main_t * vm) -{ - clib_error_t *error = 0; - - /* Avoid multiple calls. */ - if (vm->os_physmem_alloc_aligned) - return error; - - vm->os_physmem_alloc_aligned = unix_physmem_alloc_aligned; - vm->os_physmem_free = unix_physmem_free; - vm->os_physmem_region_alloc = unix_physmem_region_alloc; - vm->os_physmem_region_free = unix_physmem_region_free; - - return error; -} - -static clib_error_t * -show_physmem (vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - vlib_physmem_main_t *vpm = &vm->physmem_main; - vlib_physmem_region_t *pr; - - /* *INDENT-OFF* */ - pool_foreach (pr, vpm->regions, ( - { - vlib_cli_output (vm, "index %u name '%s' page-size %uKB num-pages %d " - "numa-node %u fd %d\n", - pr->index, pr->name, (1 << (pr->log2_page_size -10)), - pr->n_pages, pr->numa_node, pr->fd); - if (pr->heap) - vlib_cli_output (vm, " %U", format_mheap, pr->heap, /* verbose */ 1); - else - vlib_cli_output (vm, " no heap\n"); - })); - /* *INDENT-ON* */ - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (show_physmem_command, static) = { - .path = "show physmem", - .short_help = "Show physical memory allocation", - .function = show_physmem, -}; -/* *INDENT-ON* */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vlib/unix/unix.h b/src/vlib/unix/unix.h index b5a33427..1b0d8b9d 100644 --- a/src/vlib/unix/unix.h +++ b/src/vlib/unix/unix.h @@ -217,23 +217,6 @@ extern u8 **vlib_thread_stacks; /* utils */ -clib_error_t *vlib_sysfs_write (char *file_name, char *fmt, ...); - -clib_error_t *vlib_sysfs_read (char *file_name, char *fmt, ...); - -u8 *vlib_sysfs_link_to_name (char *link); - -clib_error_t *vlib_sysfs_set_nr_hugepages (unsigned int numa_node, - int page_size, int nr); -clib_error_t *vlib_sysfs_get_nr_hugepages (unsigned int numa_node, - int page_size, int *v); -clib_error_t *vlib_sysfs_get_free_hugepages (unsigned int numa_node, - int page_size, int *v); -clib_error_t *vlib_sysfs_get_surplus_hugepages (unsigned int numa_node, - int page_size, int *v); -clib_error_t *vlib_sysfs_prealloc_hugepages (unsigned int numa_node, - int page_size, int nr); - clib_error_t *foreach_directory_file (char *dir_name, clib_error_t * (*f) (void *arg, u8 * path_name, diff --git a/src/vlib/unix/util.c b/src/vlib/unix/util.c index 0e252aca..5472751e 100644 --- a/src/vlib/unix/util.c +++ b/src/vlib/unix/util.c @@ -98,225 +98,6 @@ foreach_directory_file (char *dir_name, return error; } -clib_error_t * -vlib_sysfs_write (char *file_name, char *fmt, ...) -{ - u8 *s; - int fd; - clib_error_t *error = 0; - - fd = open (file_name, O_WRONLY); - if (fd < 0) - return clib_error_return_unix (0, "open `%s'", file_name); - - va_list va; - va_start (va, fmt); - s = va_format (0, fmt, &va); - va_end (va); - - if (write (fd, s, vec_len (s)) < 0) - error = clib_error_return_unix (0, "write `%s'", file_name); - - vec_free (s); - close (fd); - return error; -} - -clib_error_t * -vlib_sysfs_read (char *file_name, char *fmt, ...) -{ - unformat_input_t input; - u8 *s = 0; - int fd; - ssize_t sz; - uword result; - - fd = open (file_name, O_RDONLY); - if (fd < 0) - return clib_error_return_unix (0, "open `%s'", file_name); - - vec_validate (s, 4095); - - sz = read (fd, s, vec_len (s)); - if (sz < 0) - { - close (fd); - vec_free (s); - return clib_error_return_unix (0, "read `%s'", file_name); - } - - _vec_len (s) = sz; - unformat_init_vector (&input, s); - - va_list va; - va_start (va, fmt); - result = va_unformat (&input, fmt, &va); - va_end (va); - - vec_free (s); - close (fd); - - if (result == 0) - return clib_error_return (0, "unformat error"); - - return 0; -} - -u8 * -vlib_sysfs_link_to_name (char *link) -{ - char *p, buffer[64]; - unformat_input_t in; - u8 *s = 0; - int r; - - r = readlink (link, buffer, sizeof (buffer) - 1); - - if (r < 0) - return 0; - - buffer[r] = 0; - p = strrchr (buffer, '/'); - - if (!p) - return 0; - - unformat_init_string (&in, p + 1, strlen (p + 1)); - if (unformat (&in, "%s", &s) != 1) - clib_unix_warning ("no string?"); - unformat_free (&in); - - return s; -} - -clib_error_t * -vlib_sysfs_set_nr_hugepages (unsigned int numa_node, int page_size, int nr) -{ - clib_error_t *error = 0; - struct stat sb; - u8 *p = 0; - - p = format (p, "/sys/devices/system/node/node%u%c", numa_node, 0); - - if (stat ((char *) p, &sb) == 0) - { - if (S_ISDIR (sb.st_mode) == 0) - { - error = clib_error_return (0, "'%s' is not directory", p); - goto done; - } - } - else if (numa_node == 0) - { - vec_reset_length (p); - p = format (p, "/sys/kernel/mm%c", 0); - if (stat ((char *) p, &sb) < 0 || S_ISDIR (sb.st_mode) == 0) - { - error = clib_error_return (0, "'%s' does not exist or it is not " - "directory", p); - goto done; - } - } - else - { - error = clib_error_return (0, "'%s' does not exist", p); - goto done; - } - - _vec_len (p) -= 1; - p = format (p, "/hugepages/hugepages-%ukB/nr_hugepages%c", page_size, 0); - vlib_sysfs_write ((char *) p, "%d", nr); - -done: - vec_free (p); - return error; -} - - -static clib_error_t * -vlib_sysfs_get_xxx_hugepages (char *type, unsigned int numa_node, - int page_size, int *val) -{ - clib_error_t *error = 0; - struct stat sb; - u8 *p = 0; - - p = format (p, "/sys/devices/system/node/node%u%c", numa_node, 0); - - if (stat ((char *) p, &sb) == 0) - { - if (S_ISDIR (sb.st_mode) == 0) - { - error = clib_error_return (0, "'%s' is not directory", p); - goto done; - } - } - else if (numa_node == 0) - { - vec_reset_length (p); - p = format (p, "/sys/kernel/mm%c", 0); - if (stat ((char *) p, &sb) < 0 || S_ISDIR (sb.st_mode) == 0) - { - error = clib_error_return (0, "'%s' does not exist or it is not " - "directory", p); - goto done; - } - } - else - { - error = clib_error_return (0, "'%s' does not exist", p); - goto done; - } - - _vec_len (p) -= 1; - p = format (p, "/hugepages/hugepages-%ukB/%s_hugepages%c", page_size, - type, 0); - error = vlib_sysfs_read ((char *) p, "%d", val); - -done: - vec_free (p); - return error; -} - -clib_error_t * -vlib_sysfs_get_free_hugepages (unsigned int numa_node, int page_size, int *v) -{ - return vlib_sysfs_get_xxx_hugepages ("free", numa_node, page_size, v); -} - -clib_error_t * -vlib_sysfs_get_nr_hugepages (unsigned int numa_node, int page_size, int *v) -{ - return vlib_sysfs_get_xxx_hugepages ("nr", numa_node, page_size, v); -} - -clib_error_t * -vlib_sysfs_get_surplus_hugepages (unsigned int numa_node, int page_size, - int *v) -{ - return vlib_sysfs_get_xxx_hugepages ("surplus", numa_node, page_size, v); -} - -clib_error_t * -vlib_sysfs_prealloc_hugepages (unsigned int numa_node, int page_size, int nr) -{ - clib_error_t *error = 0; - int n, needed; - error = vlib_sysfs_get_free_hugepages (numa_node, page_size, &n); - if (error) - return error; - needed = nr - n; - if (needed <= 0) - return 0; - - error = vlib_sysfs_get_nr_hugepages (numa_node, page_size, &n); - if (error) - return error; - clib_warning ("pre-allocating %u additional %uK hugepages on numa node %u", - needed, page_size, numa_node); - return vlib_sysfs_set_nr_hugepages (numa_node, page_size, n + needed); -} - clib_error_t * vlib_unix_recursive_mkdir (char *path) { diff --git a/src/vnet/devices/af_packet/af_packet.c b/src/vnet/devices/af_packet/af_packet.c index ea52878d..e7e69214 100644 --- a/src/vnet/devices/af_packet/af_packet.c +++ b/src/vnet/devices/af_packet/af_packet.c @@ -26,6 +26,7 @@ #include #include +#include #include #include -- cgit 1.2.3-korg From 01914ce45729833cec88c65689de9a0336cd40cc Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Thu, 14 Sep 2017 19:04:50 +0200 Subject: vppinfra: add clib_mem_vm_ext_alloc function Change-Id: Iff33694fc42cc3bcc73cf1372339053a6365039c Signed-off-by: Damjan Marion --- src/plugins/dpdk/device/init.c | 6 +- src/plugins/memif/memif.c | 21 ++- src/vlib.am | 5 +- src/vlib/linux/pci.c | 25 ++-- src/vlib/linux/physmem.c | 192 ++++-------------------- src/vlib/linux/syscall.h | 58 -------- src/vlib/linux/sysfs.c | 250 ------------------------------- src/vlib/linux/sysfs.h | 44 ------ src/vlib/threads.c | 6 +- src/vlib/threads_cli.c | 6 +- src/vnet/devices/af_packet/af_packet.c | 4 +- src/vppinfra.am | 5 +- src/vppinfra/linux/mem.c | 260 +++++++++++++++++++++++++++++++++ src/vppinfra/linux/syscall.h | 56 +++++++ src/vppinfra/linux/sysfs.c | 250 +++++++++++++++++++++++++++++++ src/vppinfra/linux/sysfs.h | 46 ++++++ src/vppinfra/mem.h | 94 ++++++++++-- src/vppinfra/vm_linux_kernel.h | 78 ---------- src/vppinfra/vm_standalone.h | 74 ---------- src/vppinfra/vm_unix.h | 106 -------------- 20 files changed, 761 insertions(+), 825 deletions(-) delete mode 100644 src/vlib/linux/syscall.h delete mode 100644 src/vlib/linux/sysfs.c delete mode 100644 src/vlib/linux/sysfs.h create mode 100644 src/vppinfra/linux/mem.c create mode 100644 src/vppinfra/linux/syscall.h create mode 100644 src/vppinfra/linux/sysfs.c create mode 100644 src/vppinfra/linux/sysfs.h delete mode 100644 src/vppinfra/vm_linux_kernel.h delete mode 100644 src/vppinfra/vm_standalone.h delete mode 100644 src/vppinfra/vm_unix.h (limited to 'src/vlib/threads_cli.c') diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index 95176fb8..ee61f94e 100755 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include @@ -1040,7 +1040,7 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) mem = mem_by_socket[c]; page_size = 1024; - e = vlib_sysfs_get_free_hugepages(c, page_size * 1024, &pages_avail); + e = clib_sysfs_get_free_hugepages(c, page_size * 1024, &pages_avail); if (e != 0 || pages_avail < 0 || page_size * pages_avail < mem) use_1g = 0; @@ -1049,7 +1049,7 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) clib_error_free (e); page_size = 2; - e = vlib_sysfs_get_free_hugepages(c, page_size * 1024, &pages_avail); + e = clib_sysfs_get_free_hugepages(c, page_size * 1024, &pages_avail); if (e != 0 || pages_avail < 0 || page_size * pages_avail < mem) use_2m = 0; diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c index 8fec409a..6a609a57 100644 --- a/src/plugins/memif/memif.c +++ b/src/plugins/memif/memif.c @@ -33,7 +33,7 @@ #include #include -#include +#include #include #include #include @@ -267,6 +267,8 @@ memif_init_regions_and_queues (memif_if_t * mif) int i, j; u64 buffer_offset; memif_region_t *r; + clib_mem_vm_alloc_t alloc = { 0 }; + clib_error_t *err; vec_validate_aligned (mif->regions, 0, CLIB_CACHE_LINE_BYTES); r = vec_elt_at_index (mif->regions, 0); @@ -279,18 +281,15 @@ memif_init_regions_and_queues (memif_if_t * mif) mif->run.buffer_size * (1 << mif->run.log2_ring_size) * (mif->run.num_s2m_rings + mif->run.num_m2s_rings); - if ((r->fd = memfd_create ("memif region 0", MFD_ALLOW_SEALING)) == -1) - return clib_error_return_unix (0, "memfd_create"); - - if ((fcntl (r->fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1) - return clib_error_return_unix (0, "fcntl (F_ADD_SEALS, F_SEAL_SHRINK)"); + alloc.name = "memif region"; + alloc.size = r->region_size; + alloc.flags = CLIB_MEM_VM_F_SHARED; - if ((ftruncate (r->fd, r->region_size)) == -1) - return clib_error_return_unix (0, "ftruncate"); + err = clib_mem_vm_ext_alloc (&alloc); + if (err) + return err; - if ((r->shm = mmap (NULL, r->region_size, PROT_READ | PROT_WRITE, - MAP_SHARED, r->fd, 0)) == MAP_FAILED) - return clib_error_return_unix (0, "mmap"); + r->fd = alloc.fd; for (i = 0; i < mif->run.num_s2m_rings; i++) { diff --git a/src/vlib.am b/src/vlib.am index 41d68690..067e4afc 100644 --- a/src/vlib.am +++ b/src/vlib.am @@ -13,7 +13,7 @@ lib_LTLIBRARIES += libvlib.la -libvlib_la_LIBADD = libvppinfra.la -ldl -lpthread -lnuma +libvlib_la_LIBADD = libvppinfra.la -ldl -lpthread libvlib_la_DEPENDENCIES = libvppinfra.la BUILT_SOURCES += vlib/config.h @@ -34,7 +34,6 @@ libvlib_la_SOURCES = \ vlib/init.c \ vlib/linux/pci.c \ vlib/linux/physmem.c \ - vlib/linux/sysfs.c \ vlib/main.c \ vlib/mc.c \ vlib/node.c \ @@ -60,8 +59,6 @@ nobase_include_HEADERS += \ vlib/global_funcs.h \ vlib/i2c.h \ vlib/init.h \ - vlib/linux/sysfs.h \ - vlib/linux/syscall.h \ vlib/main.h \ vlib/mc.h \ vlib/node_funcs.h \ diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c index 4ce19190..790f168a 100644 --- a/src/vlib/linux/pci.c +++ b/src/vlib/linux/pci.c @@ -37,10 +37,11 @@ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#include + #include #include #include -#include #include #include @@ -104,7 +105,7 @@ vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name) format_vlib_pci_addr, &d->bus_address); s = format (s, "%v/driver%c", dev_dir_name, 0); - driver_name = vlib_sysfs_link_to_name ((char *) s); + driver_name = clib_sysfs_link_to_name ((char *) s); vec_reset_length (s); if (driver_name && @@ -183,32 +184,32 @@ vlib_pci_bind_to_uio (vlib_pci_device_t * d, char *uio_driver_name) vec_reset_length (s); s = format (s, "%v/driver/unbind%c", dev_dir_name, 0); - vlib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address); + clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address); vec_reset_length (s); s = format (s, "%v/driver_override%c", dev_dir_name, 0); if (access ((char *) s, F_OK) == 0) { - vlib_sysfs_write ((char *) s, "%s", uio_driver_name); + clib_sysfs_write ((char *) s, "%s", uio_driver_name); clear_driver_override = 1; } else { vec_reset_length (s); s = format (s, "/sys/bus/pci/drivers/%s/new_id%c", uio_driver_name, 0); - vlib_sysfs_write ((char *) s, "0x%04x 0x%04x", d->vendor_id, + clib_sysfs_write ((char *) s, "0x%04x 0x%04x", d->vendor_id, d->device_id); } vec_reset_length (s); s = format (s, "/sys/bus/pci/drivers/%s/bind%c", uio_driver_name, 0); - vlib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address); + clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, &d->bus_address); vec_reset_length (s); if (clear_driver_override) { s = format (s, "%v/driver_override%c", dev_dir_name, 0); - vlib_sysfs_write ((char *) s, "%c", 0); + clib_sysfs_write ((char *) s, "%c", 0); vec_reset_length (s); } @@ -602,28 +603,28 @@ scan_device (void *arg, u8 * dev_dir_name, u8 * ignored) dev->numa_node = -1; vec_reset_length (f); f = format (f, "%v/numa_node%c", dev_dir_name, 0); - vlib_sysfs_read ((char *) f, "%u", &dev->numa_node); + clib_sysfs_read ((char *) f, "%u", &dev->numa_node); vec_reset_length (f); f = format (f, "%v/class%c", dev_dir_name, 0); - vlib_sysfs_read ((char *) f, "0x%x", &tmp); + clib_sysfs_read ((char *) f, "0x%x", &tmp); dev->device_class = tmp >> 8; vec_reset_length (f); f = format (f, "%v/vendor%c", dev_dir_name, 0); - vlib_sysfs_read ((char *) f, "0x%x", &tmp); + clib_sysfs_read ((char *) f, "0x%x", &tmp); dev->vendor_id = tmp; vec_reset_length (f); f = format (f, "%v/device%c", dev_dir_name, 0); - vlib_sysfs_read ((char *) f, "0x%x", &tmp); + clib_sysfs_read ((char *) f, "0x%x", &tmp); dev->device_id = tmp; error = init_device (vm, dev, &pdev); vec_reset_length (f); f = format (f, "%v/driver%c", dev_dir_name, 0); - dev->driver_name = vlib_sysfs_link_to_name ((char *) f); + dev->driver_name = clib_sysfs_link_to_name ((char *) f); done: vec_free (f); diff --git a/src/vlib/linux/physmem.c b/src/vlib/linux/physmem.c index d8c5dc9b..3cc42a06 100644 --- a/src/vlib/linux/physmem.c +++ b/src/vlib/linux/physmem.c @@ -43,14 +43,12 @@ #include #include #include -#include -#include +#include +#include #include #include #include -#include -#include static void * unix_physmem_alloc_aligned (vlib_main_t * vm, vlib_physmem_region_index_t idx, @@ -111,31 +109,6 @@ unix_physmem_free (vlib_main_t * vm, vlib_physmem_region_index_t idx, void *x) mheap_put (pr->heap, x - pr->heap); } -static u64 -get_page_paddr (int fd, uword addr) -{ - int pagesize = sysconf (_SC_PAGESIZE); - u64 seek, pagemap = 0; - - seek = ((u64) addr / pagesize) * sizeof (u64); - if (lseek (fd, seek, SEEK_SET) != seek) - { - clib_unix_warning ("lseek to 0x%llx", seek); - return 0; - } - if (read (fd, &pagemap, sizeof (pagemap)) != (sizeof (pagemap))) - { - clib_unix_warning ("read ptbits"); - return 0; - } - if ((pagemap & (1ULL << 63)) == 0) - return 0; - - pagemap &= pow2_mask (55); - - return pagemap * pagesize; -} - static clib_error_t * unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size, u8 numa_node, u32 flags, @@ -144,13 +117,8 @@ unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size, vlib_physmem_main_t *vpm = &vm->physmem_main; vlib_physmem_region_t *pr; clib_error_t *error = 0; - int pagemap_fd = -1; - u8 *mount_dir = 0; - u8 *filename = 0; - struct stat st; - int old_mpol; - int mmap_flags; - struct bitmask *old_mask = numa_allocate_nodemask (); + clib_mem_vm_alloc_t alloc = { 0 }; + if (geteuid () != 0 && (flags & VLIB_PHYSMEM_F_FAKE) == 0) return clib_error_return (0, "not allowed"); @@ -163,113 +131,32 @@ unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size, goto error; } - pr->index = pr - vpm->regions; - pr->fd = -1; - pr->flags = flags; - - if (get_mempolicy (&old_mpol, old_mask->maskp, old_mask->size + 1, NULL, 0) - == -1) - { - if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) - { - error = clib_error_return_unix (0, "get_mempolicy"); - goto error; - } - else - old_mpol = -1; - } + alloc.name = name; + alloc.size = size; + alloc.numa_node = numa_node; + alloc.flags = CLIB_MEM_VM_F_SHARED; if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) { - if ((pagemap_fd = open ((char *) "/proc/self/pagemap", O_RDONLY)) == -1) - { - error = clib_error_return_unix (0, "open '/proc/self/pagemap'"); - goto error; - } - - mount_dir = format (0, "%s/physmem_region%d%c", - vlib_unix_get_runtime_dir (), pr->index, 0); - filename = format (0, "%s/mem%c", mount_dir, 0); - - unlink ((char *) mount_dir); - - error = vlib_unix_recursive_mkdir ((char *) mount_dir); - if (error) - goto error; - - if (mount ("none", (char *) mount_dir, "hugetlbfs", 0, NULL)) - { - error = clib_error_return_unix (0, "mount hugetlb directory '%s'", - mount_dir); - goto error; - } - - if ((pr->fd = open ((char *) filename, O_CREAT | O_RDWR, 0755)) == -1) - { - error = clib_error_return_unix (0, "open"); - goto error; - } - - mmap_flags = MAP_SHARED | MAP_HUGETLB | MAP_LOCKED; + alloc.flags |= CLIB_MEM_VM_F_HUGETLB; + alloc.flags |= CLIB_MEM_VM_F_HUGETLB_PREALLOC; + alloc.flags |= CLIB_MEM_VM_F_NUMA_FORCE; } else { - if ((pr->fd = memfd_create (name, MFD_ALLOW_SEALING)) == -1) - return clib_error_return_unix (0, "memfd_create"); - - if ((fcntl (pr->fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1) - { - error = - clib_error_return_unix (0, "fcntl (F_ADD_SEALS, F_SEAL_SHRINK)"); - goto error; - } - mmap_flags = MAP_SHARED; - } - - if (fstat (pr->fd, &st)) - { - error = clib_error_return_unix (0, "fstat"); - goto error; - } - - pr->log2_page_size = min_log2 (st.st_blksize); - pr->n_pages = ((size - 1) >> pr->log2_page_size) + 1; - size = pr->n_pages * (1 << pr->log2_page_size); - - if ((ftruncate (pr->fd, size)) == -1) - { - error = clib_error_return_unix (0, "ftruncate length: %d", size); - goto error; - } - - if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) - { - error = vlib_sysfs_prealloc_hugepages (numa_node, - 1 << (pr->log2_page_size - 10), - pr->n_pages); - if (error) - goto error; - } - - if (old_mpol != -1) - numa_set_preferred (numa_node); - - pr->mem = mmap (0, size, (PROT_READ | PROT_WRITE), mmap_flags, pr->fd, 0); - - if (pr->mem == MAP_FAILED) - { - pr->mem = 0; - error = clib_error_return_unix (0, "mmap"); - goto error; + alloc.flags |= CLIB_MEM_VM_F_NUMA_PREFER; } - if (old_mpol != -1 && - set_mempolicy (old_mpol, old_mask->maskp, old_mask->size + 1) == -1) - { - error = clib_error_return_unix (0, "set_mempolicy"); - goto error; - } + error = clib_mem_vm_ext_alloc (&alloc); + if (error) + goto error; + pr->index = pr - vpm->regions; + pr->flags = flags; + pr->fd = alloc.fd; + pr->mem = alloc.addr; + pr->log2_page_size = alloc.log2_page_size; + pr->n_pages = alloc.n_pages; pr->size = pr->n_pages << pr->log2_page_size; pr->page_mask = (1 << pr->log2_page_size) - 1; pr->numa_node = numa_node; @@ -285,13 +172,14 @@ unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size, move_pages (0, 1, &ptr, 0, &node, 0); if (numa_node != node) { - clib_warning - ("physmem page for region \'%s\' allocated on the wrong" - " numa node (requested %u actual %u)", pr->name, - pr->numa_node, node, i); + clib_warning ("physmem page for region \'%s\' allocated on the" + " wrong numa node (requested %u actual %u)", + pr->name, pr->numa_node, node, i); break; } } + pr->page_table = clib_mem_vm_get_paddr (pr->mem, pr->log2_page_size, + pr->n_pages); } if (flags & VLIB_PHYSMEM_F_INIT_MHEAP) @@ -309,41 +197,13 @@ unix_physmem_region_alloc (vlib_main_t * vm, char *name, u32 size, *idx = pr->index; - if ((flags & VLIB_PHYSMEM_F_FAKE) == 0) - { - int i; - for (i = 0; i < pr->n_pages; i++) - { - uword vaddr = - pointer_to_uword (pr->mem) + (((u64) i) << pr->log2_page_size); - u64 page_paddr = get_page_paddr (pagemap_fd, vaddr); - vec_add1 (pr->page_table, page_paddr); - } - } - goto done; error: - if (pr->fd > -1) - close (pr->fd); - - if (pr->mem) - munmap (pr->mem, size); - memset (pr, 0, sizeof (*pr)); pool_put (vpm->regions, pr); done: - if (mount_dir) - { - umount2 ((char *) mount_dir, MNT_DETACH); - rmdir ((char *) mount_dir); - vec_free (mount_dir); - } - numa_free_cpumask (old_mask); - vec_free (filename); - if (pagemap_fd > -1) - close (pagemap_fd); return error; } diff --git a/src/vlib/linux/syscall.h b/src/vlib/linux/syscall.h deleted file mode 100644 index 9e37997e..00000000 --- a/src/vlib/linux/syscall.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2017 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef included_linux_syscall_h -#define included_linux_syscall_h - -#ifndef __NR_memfd_create -#if defined __x86_64__ -#define __NR_memfd_create 319 -#elif defined __arm__ -#define __NR_memfd_create 385 -#elif defined __aarch64__ -#define __NR_memfd_create 279 -#else -#error "__NR_memfd_create unknown for this architecture" -#endif -#endif - -static inline int -memfd_create (const char *name, unsigned int flags) -{ - return syscall (__NR_memfd_create, name, flags); -} - -#ifndef F_LINUX_SPECIFIC_BASE -#define F_LINUX_SPECIFIC_BASE 1024 -#endif -#define MFD_ALLOW_SEALING 0x0002U -#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9) -#define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10) - -#define F_SEAL_SEAL 0x0001 /* prevent further seals from being set */ -#define F_SEAL_SHRINK 0x0002 /* prevent file from shrinking */ -#define F_SEAL_GROW 0x0004 /* prevent file from growing */ -#define F_SEAL_WRITE 0x0008 /* prevent writes */ - - -#endif /* included_linux_syscall_h */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vlib/linux/sysfs.c b/src/vlib/linux/sysfs.c deleted file mode 100644 index f92f9ef5..00000000 --- a/src/vlib/linux/sysfs.c +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Copyright (c) 2017 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include -#include -#include -#include - -clib_error_t * -vlib_sysfs_write (char *file_name, char *fmt, ...) -{ - u8 *s; - int fd; - clib_error_t *error = 0; - - fd = open (file_name, O_WRONLY); - if (fd < 0) - return clib_error_return_unix (0, "open `%s'", file_name); - - va_list va; - va_start (va, fmt); - s = va_format (0, fmt, &va); - va_end (va); - - if (write (fd, s, vec_len (s)) < 0) - error = clib_error_return_unix (0, "write `%s'", file_name); - - vec_free (s); - close (fd); - return error; -} - -clib_error_t * -vlib_sysfs_read (char *file_name, char *fmt, ...) -{ - unformat_input_t input; - u8 *s = 0; - int fd; - ssize_t sz; - uword result; - - fd = open (file_name, O_RDONLY); - if (fd < 0) - return clib_error_return_unix (0, "open `%s'", file_name); - - vec_validate (s, 4095); - - sz = read (fd, s, vec_len (s)); - if (sz < 0) - { - close (fd); - vec_free (s); - return clib_error_return_unix (0, "read `%s'", file_name); - } - - _vec_len (s) = sz; - unformat_init_vector (&input, s); - - va_list va; - va_start (va, fmt); - result = va_unformat (&input, fmt, &va); - va_end (va); - - vec_free (s); - close (fd); - - if (result == 0) - return clib_error_return (0, "unformat error"); - - return 0; -} - -u8 * -vlib_sysfs_link_to_name (char *link) -{ - char *p, buffer[64]; - unformat_input_t in; - u8 *s = 0; - int r; - - r = readlink (link, buffer, sizeof (buffer) - 1); - - if (r < 0) - return 0; - - buffer[r] = 0; - p = strrchr (buffer, '/'); - - if (!p) - return 0; - - unformat_init_string (&in, p + 1, strlen (p + 1)); - if (unformat (&in, "%s", &s) != 1) - clib_unix_warning ("no string?"); - unformat_free (&in); - - return s; -} - -clib_error_t * -vlib_sysfs_set_nr_hugepages (unsigned int numa_node, int page_size, int nr) -{ - clib_error_t *error = 0; - struct stat sb; - u8 *p = 0; - - p = format (p, "/sys/devices/system/node/node%u%c", numa_node, 0); - - if (stat ((char *) p, &sb) == 0) - { - if (S_ISDIR (sb.st_mode) == 0) - { - error = clib_error_return (0, "'%s' is not directory", p); - goto done; - } - } - else if (numa_node == 0) - { - vec_reset_length (p); - p = format (p, "/sys/kernel/mm%c", 0); - if (stat ((char *) p, &sb) < 0 || S_ISDIR (sb.st_mode) == 0) - { - error = clib_error_return (0, "'%s' does not exist or it is not " - "directory", p); - goto done; - } - } - else - { - error = clib_error_return (0, "'%s' does not exist", p); - goto done; - } - - _vec_len (p) -= 1; - p = format (p, "/hugepages/hugepages-%ukB/nr_hugepages%c", page_size, 0); - vlib_sysfs_write ((char *) p, "%d", nr); - -done: - vec_free (p); - return error; -} - - -static clib_error_t * -vlib_sysfs_get_xxx_hugepages (char *type, unsigned int numa_node, - int page_size, int *val) -{ - clib_error_t *error = 0; - struct stat sb; - u8 *p = 0; - - p = format (p, "/sys/devices/system/node/node%u%c", numa_node, 0); - - if (stat ((char *) p, &sb) == 0) - { - if (S_ISDIR (sb.st_mode) == 0) - { - error = clib_error_return (0, "'%s' is not directory", p); - goto done; - } - } - else if (numa_node == 0) - { - vec_reset_length (p); - p = format (p, "/sys/kernel/mm%c", 0); - if (stat ((char *) p, &sb) < 0 || S_ISDIR (sb.st_mode) == 0) - { - error = clib_error_return (0, "'%s' does not exist or it is not " - "directory", p); - goto done; - } - } - else - { - error = clib_error_return (0, "'%s' does not exist", p); - goto done; - } - - _vec_len (p) -= 1; - p = format (p, "/hugepages/hugepages-%ukB/%s_hugepages%c", page_size, - type, 0); - error = vlib_sysfs_read ((char *) p, "%d", val); - -done: - vec_free (p); - return error; -} - -clib_error_t * -vlib_sysfs_get_free_hugepages (unsigned int numa_node, int page_size, int *v) -{ - return vlib_sysfs_get_xxx_hugepages ("free", numa_node, page_size, v); -} - -clib_error_t * -vlib_sysfs_get_nr_hugepages (unsigned int numa_node, int page_size, int *v) -{ - return vlib_sysfs_get_xxx_hugepages ("nr", numa_node, page_size, v); -} - -clib_error_t * -vlib_sysfs_get_surplus_hugepages (unsigned int numa_node, int page_size, - int *v) -{ - return vlib_sysfs_get_xxx_hugepages ("surplus", numa_node, page_size, v); -} - -clib_error_t * -vlib_sysfs_prealloc_hugepages (unsigned int numa_node, int page_size, int nr) -{ - clib_error_t *error = 0; - int n, needed; - error = vlib_sysfs_get_free_hugepages (numa_node, page_size, &n); - if (error) - return error; - needed = nr - n; - if (needed <= 0) - return 0; - - error = vlib_sysfs_get_nr_hugepages (numa_node, page_size, &n); - if (error) - return error; - clib_warning ("pre-allocating %u additional %uK hugepages on numa node %u", - needed, page_size, numa_node); - return vlib_sysfs_set_nr_hugepages (numa_node, page_size, n + needed); -} - - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vlib/linux/sysfs.h b/src/vlib/linux/sysfs.h deleted file mode 100644 index 14b71317..00000000 --- a/src/vlib/linux/sysfs.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2017 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef included_linux_sysfs_h -#define included_linux_sysfs_h - -clib_error_t *vlib_sysfs_write (char *file_name, char *fmt, ...); - -clib_error_t *vlib_sysfs_read (char *file_name, char *fmt, ...); - -u8 *vlib_sysfs_link_to_name (char *link); - -clib_error_t *vlib_sysfs_set_nr_hugepages (unsigned int numa_node, - int page_size, int nr); -clib_error_t *vlib_sysfs_get_nr_hugepages (unsigned int numa_node, - int page_size, int *v); -clib_error_t *vlib_sysfs_get_free_hugepages (unsigned int numa_node, - int page_size, int *v); -clib_error_t *vlib_sysfs_get_surplus_hugepages (unsigned int numa_node, - int page_size, int *v); -clib_error_t *vlib_sysfs_prealloc_hugepages (unsigned int numa_node, - int page_size, int nr); - -#endif /* included_linux_sysfs_h */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 2d9ce84a..f9c7043c 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -289,7 +289,7 @@ sort_registrations_by_no_clone (void *a0, void *a1) } static uword * -vlib_sysfs_list_to_bitmap (char *filename) +clib_sysfs_list_to_bitmap (char *filename) { FILE *fp; uword *r = 0; @@ -331,9 +331,9 @@ vlib_thread_init (vlib_main_t * vm) /* get bitmaps of active cpu cores and sockets */ tm->cpu_core_bitmap = - vlib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online"); + clib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online"); tm->cpu_socket_bitmap = - vlib_sysfs_list_to_bitmap ("/sys/devices/system/node/online"); + clib_sysfs_list_to_bitmap ("/sys/devices/system/node/online"); avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap); diff --git a/src/vlib/threads_cli.c b/src/vlib/threads_cli.c index f8d5d8f9..02bdea5c 100644 --- a/src/vlib/threads_cli.c +++ b/src/vlib/threads_cli.c @@ -15,10 +15,10 @@ #define _GNU_SOURCE #include +#include #include #include -#include #include static u8 * @@ -98,14 +98,14 @@ show_threads_fn (vlib_main_t * vm, u8 *p = 0; p = format (p, "%s%u/topology/core_id%c", sys_cpu_path, lcore, 0); - vlib_sysfs_read ((char *) p, "%d", &core_id); + clib_sysfs_read ((char *) p, "%d", &core_id); vec_reset_length (p); p = format (p, "%s%u/topology/physical_package_id%c", sys_cpu_path, lcore, 0); - vlib_sysfs_read ((char *) p, "%d", &socket_id); + clib_sysfs_read ((char *) p, "%d", &socket_id); vec_free (p); line = format (line, "%-7u%-7u%-7u%", lcore, core_id, socket_id); diff --git a/src/vnet/devices/af_packet/af_packet.c b/src/vnet/devices/af_packet/af_packet.c index 62bb228f..32696014 100644 --- a/src/vnet/devices/af_packet/af_packet.c +++ b/src/vnet/devices/af_packet/af_packet.c @@ -24,9 +24,9 @@ #include #include +#include #include #include -#include #include #include @@ -75,7 +75,7 @@ af_packet_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, { s = format (0, "/sys/class/net/%s/mtu%c", apif->host_if_name, 0); - error = vlib_sysfs_write ((char *) s, "%d", hi->max_packet_bytes); + error = clib_sysfs_write ((char *) s, "%d", hi->max_packet_bytes); vec_free (s); if (error) diff --git a/src/vppinfra.am b/src/vppinfra.am index a5769a0d..daca9954 100644 --- a/src/vppinfra.am +++ b/src/vppinfra.am @@ -188,6 +188,8 @@ nobase_include_HEADERS = \ vppinfra/graph.h \ vppinfra/hash.h \ vppinfra/heap.h \ + vppinfra/linux/sysfs.h \ + vppinfra/linux/syscall.h \ vppinfra/lock.h \ vppinfra/longjmp.h \ vppinfra/macros.h \ @@ -233,7 +235,6 @@ nobase_include_HEADERS = \ vppinfra/vector_neon.h \ vppinfra/vector_sse2.h \ vppinfra/valgrind.h \ - vppinfra/vm_unix.h \ vppinfra/xxhash.h \ vppinfra/xy.h \ vppinfra/zvec.h @@ -291,6 +292,8 @@ CLIB_CORE = \ libvppinfra_la_SOURCES = \ $(CLIB_CORE) \ vppinfra/elf_clib.c \ + vppinfra/linux/mem.c \ + vppinfra/linux/sysfs.c \ vppinfra/socket.c \ vppinfra/timer.c \ vppinfra/unix-formats.c \ diff --git a/src/vppinfra/linux/mem.c b/src/vppinfra/linux/mem.c new file mode 100644 index 00000000..665ddf61 --- /dev/null +++ b/src/vppinfra/linux/mem.c @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifndef F_LINUX_SPECIFIC_BASE +#define F_LINUX_SPECIFIC_BASE 1024 +#endif + +#ifndef F_ADD_SEALS +#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9) +#define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10) + +#define F_SEAL_SEAL 0x0001 /* prevent further seals from being set */ +#define F_SEAL_SHRINK 0x0002 /* prevent file from shrinking */ +#define F_SEAL_GROW 0x0004 /* prevent file from growing */ +#define F_SEAL_WRITE 0x0008 /* prevent writes */ +#endif + +int +clib_mem_vm_get_log2_page_size (int fd) +{ + struct stat st = { 0 }; + if (fstat (fd, &st)) + return 0; + return min_log2 (st.st_blksize); +} + +clib_error_t * +clib_mem_vm_ext_alloc (clib_mem_vm_alloc_t * a) +{ + int fd = -1; + clib_error_t *err = 0; + void *addr = 0; + u8 *filename = 0; + int mmap_flags = MAP_SHARED; + int log2_page_size; + int n_pages; + int old_mpol = -1; + u64 old_mask[16] = { 0 }; + + /* save old numa mem policy if needed */ + if (a->flags & (CLIB_MEM_VM_F_NUMA_PREFER | CLIB_MEM_VM_F_NUMA_FORCE)) + { + int rv; + rv = + get_mempolicy (&old_mpol, old_mask, sizeof (old_mask) * 8 + 1, 0, 0); + + if (rv == -1) + { + if ((a->flags & CLIB_MEM_VM_F_NUMA_FORCE) != 0) + { + err = clib_error_return_unix (0, "get_mempolicy"); + goto error; + } + else + old_mpol = -1; + } + } + + /* if we are creating shared segment, we need file descriptor */ + if (a->flags & CLIB_MEM_VM_F_SHARED) + { + /* if hugepages are needed we need to create mount point */ + if (a->flags & CLIB_MEM_VM_F_HUGETLB) + { + char *mount_dir; + char template[] = "/tmp/hugepage_mount.XXXXXX"; + + mount_dir = mkdtemp (template); + if (mount_dir == 0) + return clib_error_return_unix (0, "mkdtemp \'%s\'", template); + + if (mount ("none", (char *) mount_dir, "hugetlbfs", 0, NULL)) + { + err = clib_error_return_unix (0, "mount hugetlb directory '%s'", + mount_dir); + goto error; + } + + filename = format (0, "%s/%s%c", mount_dir, a->name, 0); + + if ((fd = open ((char *) filename, O_CREAT | O_RDWR, 0755)) == -1) + { + err = clib_error_return_unix (0, "open"); + goto error; + } + umount2 ((char *) mount_dir, MNT_DETACH); + rmdir ((char *) mount_dir); + mmap_flags |= MAP_LOCKED; + } + else + { + if ((fd = memfd_create (a->name, MFD_ALLOW_SEALING)) == -1) + { + err = clib_error_return_unix (0, "memfd_create"); + goto error; + } + + if ((fcntl (fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1) + { + err = clib_error_return_unix (0, "fcntl (F_ADD_SEALS)"); + goto error; + } + } + log2_page_size = clib_mem_vm_get_log2_page_size (fd); + } + else /* not CLIB_MEM_VM_F_SHARED */ + { + if (a->flags & CLIB_MEM_VM_F_HUGETLB) + { + mmap_flags |= MAP_HUGETLB | MAP_PRIVATE | MAP_ANONYMOUS; + log2_page_size = 21; + } + else + { + mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS; + log2_page_size = min_log2 (sysconf (_SC_PAGESIZE)); + } + } + + n_pages = ((a->size - 1) >> log2_page_size) + 1; + + + if (a->flags & CLIB_MEM_VM_F_HUGETLB_PREALLOC) + { + err = clib_sysfs_prealloc_hugepages (a->numa_node, + 1 << (log2_page_size - 10), + n_pages); + if (err) + goto error; + + } + + if (fd != -1) + if ((ftruncate (fd, a->size)) == -1) + { + err = clib_error_return_unix (0, "ftruncate"); + goto error; + } + + if (old_mpol != -1) + { + int rv; + u64 mask[16] = { 0 }; + mask[0] = 1 << a->numa_node; + rv = set_mempolicy (MPOL_BIND, mask, sizeof (mask) * 8 + 1); + if (rv) + { + err = clib_error_return_unix (0, "set_mempolicy"); + goto error; + } + } + + addr = mmap (0, a->size, (PROT_READ | PROT_WRITE), mmap_flags, fd, 0); + if (addr == MAP_FAILED) + { + err = clib_error_return_unix (0, "mmap"); + goto error; + } + + /* re-apply ole numa memory policy */ + if (old_mpol != -1 && + set_mempolicy (old_mpol, old_mask, sizeof (old_mask) * 8 + 1) == -1) + { + err = clib_error_return_unix (0, "set_mempolicy"); + goto error; + } + + a->log2_page_size = log2_page_size; + a->n_pages = n_pages; + a->addr = addr; + a->fd = fd; + goto done; + +error: + if (fd != -1) + close (fd); + +done: + vec_free (filename); + return err; +} + +u64 * +clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages) +{ + int pagesize = sysconf (_SC_PAGESIZE); + int fd; + int i; + u64 *r = 0; + + if ((fd = open ((char *) "/proc/self/pagemap", O_RDONLY)) == -1) + return 0; + + for (i = 0; i < n_pages; i++) + { + u64 seek, pagemap = 0; + uword vaddr = pointer_to_uword (mem) + (((u64) i) << log2_page_size); + seek = ((u64) vaddr / pagesize) * sizeof (u64); + if (lseek (fd, seek, SEEK_SET) != seek) + goto done; + + if (read (fd, &pagemap, sizeof (pagemap)) != (sizeof (pagemap))) + goto done; + + if ((pagemap & (1ULL << 63)) == 0) + goto done; + + pagemap &= pow2_mask (55); + vec_add1 (r, pagemap * pagesize); + } + +done: + close (fd); + if (vec_len (r) != n_pages) + { + vec_free (r); + return 0; + } + return r; +} + + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vppinfra/linux/syscall.h b/src/vppinfra/linux/syscall.h new file mode 100644 index 00000000..f8ec5919 --- /dev/null +++ b/src/vppinfra/linux/syscall.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef included_linux_syscall_h +#define included_linux_syscall_h + +#include +#include + +static inline long +set_mempolicy (int mode, const unsigned long *nodemask, unsigned long maxnode) +{ + return syscall (__NR_set_mempolicy, mode, nodemask, maxnode); +} + +static inline int +get_mempolicy (int *mode, unsigned long *nodemask, unsigned long maxnode, + void *addr, unsigned long flags) +{ + return syscall (__NR_get_mempolicy, mode, nodemask, maxnode, addr, flags); +} + +static inline long +move_pages (int pid, unsigned long count, void **pages, const int *nodes, + int *status, int flags) +{ + return syscall (__NR_move_pages, pid, count, pages, nodes, status, flags); +} + +static inline int +memfd_create (const char *name, unsigned int flags) +{ + return syscall (__NR_memfd_create, name, flags); +} + +#endif /* included_linux_syscall_h */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vppinfra/linux/sysfs.c b/src/vppinfra/linux/sysfs.c new file mode 100644 index 00000000..5f611e6a --- /dev/null +++ b/src/vppinfra/linux/sysfs.c @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include +#include + +clib_error_t * +clib_sysfs_write (char *file_name, char *fmt, ...) +{ + u8 *s; + int fd; + clib_error_t *error = 0; + + fd = open (file_name, O_WRONLY); + if (fd < 0) + return clib_error_return_unix (0, "open `%s'", file_name); + + va_list va; + va_start (va, fmt); + s = va_format (0, fmt, &va); + va_end (va); + + if (write (fd, s, vec_len (s)) < 0) + error = clib_error_return_unix (0, "write `%s'", file_name); + + vec_free (s); + close (fd); + return error; +} + +clib_error_t * +clib_sysfs_read (char *file_name, char *fmt, ...) +{ + unformat_input_t input; + u8 *s = 0; + int fd; + ssize_t sz; + uword result; + + fd = open (file_name, O_RDONLY); + if (fd < 0) + return clib_error_return_unix (0, "open `%s'", file_name); + + vec_validate (s, 4095); + + sz = read (fd, s, vec_len (s)); + if (sz < 0) + { + close (fd); + vec_free (s); + return clib_error_return_unix (0, "read `%s'", file_name); + } + + _vec_len (s) = sz; + unformat_init_vector (&input, s); + + va_list va; + va_start (va, fmt); + result = va_unformat (&input, fmt, &va); + va_end (va); + + vec_free (s); + close (fd); + + if (result == 0) + return clib_error_return (0, "unformat error"); + + return 0; +} + +u8 * +clib_sysfs_link_to_name (char *link) +{ + char *p, buffer[64]; + unformat_input_t in; + u8 *s = 0; + int r; + + r = readlink (link, buffer, sizeof (buffer) - 1); + + if (r < 0) + return 0; + + buffer[r] = 0; + p = strrchr (buffer, '/'); + + if (!p) + return 0; + + unformat_init_string (&in, p + 1, strlen (p + 1)); + if (unformat (&in, "%s", &s) != 1) + clib_unix_warning ("no string?"); + unformat_free (&in); + + return s; +} + +clib_error_t * +clib_sysfs_set_nr_hugepages (int numa_node, int page_size, int nr) +{ + clib_error_t *error = 0; + struct stat sb; + u8 *p = 0; + + p = format (p, "/sys/devices/system/node/node%u%c", numa_node, 0); + + if (stat ((char *) p, &sb) == 0) + { + if (S_ISDIR (sb.st_mode) == 0) + { + error = clib_error_return (0, "'%s' is not directory", p); + goto done; + } + } + else if (numa_node == 0) + { + vec_reset_length (p); + p = format (p, "/sys/kernel/mm%c", 0); + if (stat ((char *) p, &sb) < 0 || S_ISDIR (sb.st_mode) == 0) + { + error = clib_error_return (0, "'%s' does not exist or it is not " + "directory", p); + goto done; + } + } + else + { + error = clib_error_return (0, "'%s' does not exist", p); + goto done; + } + + _vec_len (p) -= 1; + p = format (p, "/hugepages/hugepages-%ukB/nr_hugepages%c", page_size, 0); + clib_sysfs_write ((char *) p, "%d", nr); + +done: + vec_free (p); + return error; +} + + +static clib_error_t * +clib_sysfs_get_xxx_hugepages (char *type, int numa_node, + int page_size, int *val) +{ + clib_error_t *error = 0; + struct stat sb; + u8 *p = 0; + + p = format (p, "/sys/devices/system/node/node%u%c", numa_node, 0); + + if (stat ((char *) p, &sb) == 0) + { + if (S_ISDIR (sb.st_mode) == 0) + { + error = clib_error_return (0, "'%s' is not directory", p); + goto done; + } + } + else if (numa_node == 0) + { + vec_reset_length (p); + p = format (p, "/sys/kernel/mm%c", 0); + if (stat ((char *) p, &sb) < 0 || S_ISDIR (sb.st_mode) == 0) + { + error = clib_error_return (0, "'%s' does not exist or it is not " + "directory", p); + goto done; + } + } + else + { + error = clib_error_return (0, "'%s' does not exist", p); + goto done; + } + + _vec_len (p) -= 1; + p = format (p, "/hugepages/hugepages-%ukB/%s_hugepages%c", page_size, + type, 0); + error = clib_sysfs_read ((char *) p, "%d", val); + +done: + vec_free (p); + return error; +} + +clib_error_t * +clib_sysfs_get_free_hugepages (int numa_node, int page_size, int *v) +{ + return clib_sysfs_get_xxx_hugepages ("free", numa_node, page_size, v); +} + +clib_error_t * +clib_sysfs_get_nr_hugepages (int numa_node, int page_size, int *v) +{ + return clib_sysfs_get_xxx_hugepages ("nr", numa_node, page_size, v); +} + +clib_error_t * +clib_sysfs_get_surplus_hugepages (int numa_node, int page_size, int *v) +{ + return clib_sysfs_get_xxx_hugepages ("surplus", numa_node, page_size, v); +} + +clib_error_t * +clib_sysfs_prealloc_hugepages (int numa_node, int page_size, int nr) +{ + clib_error_t *error = 0; + int n, needed; + error = clib_sysfs_get_free_hugepages (numa_node, page_size, &n); + if (error) + return error; + needed = nr - n; + if (needed <= 0) + return 0; + + error = clib_sysfs_get_nr_hugepages (numa_node, page_size, &n); + if (error) + return error; + clib_warning ("pre-allocating %u additional %uK hugepages on numa node %u", + needed, page_size, numa_node); + return clib_sysfs_set_nr_hugepages (numa_node, page_size, n + needed); +} + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vppinfra/linux/sysfs.h b/src/vppinfra/linux/sysfs.h new file mode 100644 index 00000000..6c80cf95 --- /dev/null +++ b/src/vppinfra/linux/sysfs.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef included_linux_sysfs_h +#define included_linux_sysfs_h + +#include + +clib_error_t *clib_sysfs_write (char *file_name, char *fmt, ...); + +clib_error_t *clib_sysfs_read (char *file_name, char *fmt, ...); + +u8 *clib_sysfs_link_to_name (char *link); + +clib_error_t *clib_sysfs_set_nr_hugepages (int numa_node, + int page_size, int nr); +clib_error_t *clib_sysfs_get_nr_hugepages (int numa_node, + int page_size, int *v); +clib_error_t *clib_sysfs_get_free_hugepages (int numa_node, + int page_size, int *v); +clib_error_t *clib_sysfs_get_surplus_hugepages (int numa_node, + int page_size, int *v); +clib_error_t *clib_sysfs_prealloc_hugepages (int numa_node, + int page_size, int nr); + +#endif /* included_linux_sysfs_h */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index 63c5ac16..69ab8803 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -39,8 +39,11 @@ #define _included_clib_mem_h #include +#include +#include #include /* uword, etc */ +#include #include #include #include /* memcpy, memset */ @@ -264,19 +267,90 @@ void clib_mem_usage (clib_mem_usage_t * usage); u8 *format_clib_mem_usage (u8 * s, va_list * args); -/* Include appropriate VM functions depending on whether - we are compiling for linux kernel, for Unix or standalone. */ -#ifdef CLIB_LINUX_KERNEL -#include -#endif +/* Allocate virtual address space. */ +always_inline void * +clib_mem_vm_alloc (uword size) +{ + void *mmap_addr; + uword flags = MAP_PRIVATE; -#ifdef CLIB_UNIX -#include +#ifdef MAP_ANONYMOUS + flags |= MAP_ANONYMOUS; #endif -#ifdef CLIB_STANDALONE -#include -#endif + mmap_addr = mmap (0, size, PROT_READ | PROT_WRITE, flags, -1, 0); + if (mmap_addr == (void *) -1) + mmap_addr = 0; + + return mmap_addr; +} + +always_inline void +clib_mem_vm_free (void *addr, uword size) +{ + munmap (addr, size); +} + +always_inline void * +clib_mem_vm_unmap (void *addr, uword size) +{ + void *mmap_addr; + uword flags = MAP_PRIVATE | MAP_FIXED; + + /* To unmap we "map" with no protection. If we actually called + munmap then other callers could steal the address space. By + changing to PROT_NONE the kernel can free up the pages which is + really what we want "unmap" to mean. */ + mmap_addr = mmap (addr, size, PROT_NONE, flags, -1, 0); + if (mmap_addr == (void *) -1) + mmap_addr = 0; + + return mmap_addr; +} + +always_inline void * +clib_mem_vm_map (void *addr, uword size) +{ + void *mmap_addr; + uword flags = MAP_PRIVATE | MAP_FIXED; + + mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0); + if (mmap_addr == (void *) -1) + mmap_addr = 0; + + return mmap_addr; +} + +typedef struct +{ +#define CLIB_MEM_VM_F_SHARED (1 << 0) +#define CLIB_MEM_VM_F_HUGETLB (1 << 1) +#define CLIB_MEM_VM_F_NUMA_PREFER (1 << 2) +#define CLIB_MEM_VM_F_NUMA_FORCE (1 << 3) +#define CLIB_MEM_VM_F_HUGETLB_PREALLOC (1 << 4) + u32 flags; /**< vm allocation flags: +
CLIB_MEM_VM_F_SHARED: request shared memory, file + destiptor will be provided on successful allocation. +
CLIB_MEM_VM_F_HUGETLB: request hugepages. +
CLIB_MEM_VM_F_NUMA_PREFER: numa_node field contains valid + numa node preference. +
CLIB_MEM_VM_F_NUMA_FORCE: fail if setting numa policy fails. +
CLIB_MEM_VM_F_HUGETLB_PREALLOC: pre-allocate hugepages if + number of available pages is not sufficient. + */ + char *name; /**< Name for memory allocation, set by caller. */ + uword size; /**< Allocation size, set by caller. */ + int numa_node; /**< numa node preference. Valid if CLIB_MEM_VM_F_NUMA_PREFER set. */ + void *addr; /**< Pointer to allocated memory, set on successful allocation. */ + int fd; /**< File desriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set. */ + int log2_page_size; /* Page size in log2 format, set on successful allocation. */ + int n_pages; /* Number of pages. */ +} clib_mem_vm_alloc_t; + +clib_error_t *clib_mem_vm_ext_alloc (clib_mem_vm_alloc_t * a); +int clib_mem_vm_get_log2_page_size (int fd); +u64 *clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages); + #include /* clib_panic */ diff --git a/src/vppinfra/vm_linux_kernel.h b/src/vppinfra/vm_linux_kernel.h deleted file mode 100644 index fd9e6148..00000000 --- a/src/vppinfra/vm_linux_kernel.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#ifndef included_vm_linux_kernel_h -#define included_vm_linux_kernel_h - -#include -#include /* for GFP_* */ -#include /* for PAGE_KERNEL */ - -/* Allocate virtual address space. */ -always_inline void * -clib_mem_vm_alloc (uword size) -{ - return vmalloc (size); -} - -always_inline void -clib_mem_vm_free (void *addr, uword size) -{ - vfree (addr); -} - -always_inline void * -clib_mem_vm_unmap (void *addr, uword size) -{ - return 0; -} - -always_inline void * -clib_mem_vm_map (void *addr, uword size) -{ - return addr; -} - -#endif /* included_vm_linux_kernel_h */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vppinfra/vm_standalone.h b/src/vppinfra/vm_standalone.h deleted file mode 100644 index 2cd431bc..00000000 --- a/src/vppinfra/vm_standalone.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#ifndef included_vm_standalone_h -#define included_vm_standalone_h - -/* Stubs for standalone "system" which has no VM support. */ - -always_inline void * -clib_mem_vm_alloc (uword size) -{ - return 0; -} - -always_inline void -clib_mem_vm_free (void *addr, uword size) -{ -} - -always_inline void * -clib_mem_vm_unmap (void *addr, uword size) -{ - return 0; -} - -always_inline void * -clib_mem_vm_map (void *addr, uword size) -{ - return addr; -} - -#endif /* included_vm_standalone_h */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vppinfra/vm_unix.h b/src/vppinfra/vm_unix.h deleted file mode 100644 index 07e86516..00000000 --- a/src/vppinfra/vm_unix.h +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#ifndef included_vm_unix_h -#define included_vm_unix_h - -#include -#include - -/* Allocate virtual address space. */ -always_inline void * -clib_mem_vm_alloc (uword size) -{ - void *mmap_addr; - uword flags = MAP_PRIVATE; - -#ifdef MAP_ANONYMOUS - flags |= MAP_ANONYMOUS; -#endif - - mmap_addr = mmap (0, size, PROT_READ | PROT_WRITE, flags, -1, 0); - if (mmap_addr == (void *) -1) - mmap_addr = 0; - - return mmap_addr; -} - -always_inline void -clib_mem_vm_free (void *addr, uword size) -{ - munmap (addr, size); -} - -always_inline void * -clib_mem_vm_unmap (void *addr, uword size) -{ - void *mmap_addr; - uword flags = MAP_PRIVATE | MAP_FIXED; - - /* To unmap we "map" with no protection. If we actually called - munmap then other callers could steal the address space. By - changing to PROT_NONE the kernel can free up the pages which is - really what we want "unmap" to mean. */ - mmap_addr = mmap (addr, size, PROT_NONE, flags, -1, 0); - if (mmap_addr == (void *) -1) - mmap_addr = 0; - - return mmap_addr; -} - -always_inline void * -clib_mem_vm_map (void *addr, uword size) -{ - void *mmap_addr; - uword flags = MAP_PRIVATE | MAP_FIXED; - - mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0); - if (mmap_addr == (void *) -1) - mmap_addr = 0; - - return mmap_addr; -} - -#endif /* included_vm_unix_h */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ -- cgit 1.2.3-korg