From 0f8ecf0eb267d580da4d963a0e0420eb15238d78 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 27 Jun 2016 08:30:30 +0200 Subject: Decouple worker thread code from dpdk, enable worker threads in vpp_lite Change-Id: I28616f1a89f2da95484438ec1a1db64845f15ef6 Signed-off-by: Damjan Marion --- vlib/Makefile.am | 1 + vlib/vlib/node.h | 18 ++ vlib/vlib/threads.c | 245 ++++++++++++++++++++------ vlib/vlib/threads.h | 12 +- vlib/vlib/threads_cli.c | 362 +++++++++++++++++++++++++++++++++++++++ vnet/Makefile.am | 4 +- vnet/vnet/devices/dpdk/cli.c | 277 ------------------------------ vnet/vnet/devices/dpdk/dpdk.h | 23 --- vnet/vnet/devices/dpdk/init.c | 2 +- vnet/vnet/devices/dpdk/threads.c | 269 ----------------------------- vnet/vnet/devices/dpdk/threads.h | 27 --- vnet/vnet/handoff.c | 7 + 12 files changed, 593 insertions(+), 654 deletions(-) create mode 100644 vlib/vlib/threads_cli.c delete mode 100644 vnet/vnet/devices/dpdk/threads.c delete mode 100644 vnet/vnet/devices/dpdk/threads.h diff --git a/vlib/Makefile.am b/vlib/Makefile.am index 274756719c1..07a34c563c5 100644 --- a/vlib/Makefile.am +++ b/vlib/Makefile.am @@ -41,6 +41,7 @@ libvlib_la_SOURCES = \ vlib/pci/pci.c \ vlib/pci/linux_pci.c \ vlib/threads.c \ + vlib/threads_cli.c \ vlib/trace.c if WITH_DPDK diff --git a/vlib/vlib/node.h b/vlib/vlib/node.h index a54f4e83e62..e9842275b0e 100644 --- a/vlib/vlib/node.h +++ b/vlib/vlib/node.h @@ -663,4 +663,22 @@ typedef struct { vlib_node_registration_t * node_registrations; } vlib_node_main_t; + +#define FRAME_QUEUE_MAX_NELTS 32 +typedef struct { + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + u64 head; + u64 head_hint; + u64 tail; + u32 n_in_use; + u32 nelts; + u32 written; + u32 threshold; + i32 n_vectors[FRAME_QUEUE_MAX_NELTS]; +} frame_queue_trace_t; + +typedef struct { + u64 count[FRAME_QUEUE_MAX_NELTS]; +} frame_queue_nelt_counter_t; + #endif /* included_vlib_node_h */ diff --git a/vlib/vlib/threads.c b/vlib/vlib/threads.c index 1808f36fe50..d2ce449f6ce 100644 --- a/vlib/vlib/threads.c +++ b/vlib/vlib/threads.c @@ -51,11 +51,6 @@ u32 vl(void *p) return vec_len (p); } -void debug_hex_bytes (u8 *s, u32 n) -{ - fformat (stderr, "%U\n", format_hex_bytes, s, n); -} - vlib_thread_main_t vlib_thread_main; uword @@ -1150,66 +1145,210 @@ void vlib_worker_thread_barrier_release(vlib_main_t * vm) } } -static clib_error_t * -show_threads_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) +/* + * Check the frame queue to see if any frames are available. + * If so, pull the packets off the frames and put them to + * the handoff node. + */ +static inline int vlib_frame_queue_dequeue_internal (vlib_main_t *vm) { - vlib_worker_thread_t * w; - int i; + u32 thread_id = vm->cpu_index; + vlib_frame_queue_t *fq = vlib_frame_queues[thread_id]; + vlib_frame_queue_elt_t *elt; + u32 * from, * to; + vlib_frame_t * f; + int msg_type; + int processed = 0; + u32 n_left_to_node; + u32 vectors = 0; + vlib_thread_main_t *tm = vlib_get_thread_main(); - vlib_cli_output (vm, "%-7s%-20s%-12s%-8s%-7s%-7s%-7s%-10s", - "ID", "Name", "Type", "LWP", - "lcore", "Core", "Socket", "State"); + ASSERT (fq); + ASSERT(vm == vlib_mains[thread_id]); -#if !defined(__powerpc64__) - for (i = 0; i < vec_len(vlib_worker_threads); i++) + if (PREDICT_FALSE (tm->handoff_dispatch_node_index == ~0)) + return 0; + /* + * Gather trace data for frame queues + */ + if (PREDICT_FALSE(fq->trace)) { - w = vlib_worker_threads + i; - u8 * line = NULL; + frame_queue_trace_t *fqt; + frame_queue_nelt_counter_t *fqh; + u32 elix; + + fqt = &tm->frame_queue_traces[thread_id]; + + fqt->nelts = fq->nelts; + fqt->head = fq->head; + fqt->head_hint = fq->head_hint; + fqt->tail = fq->tail; + fqt->threshold = fq->vector_threshold; + fqt->n_in_use = fqt->tail - fqt->head; + if (fqt->n_in_use >= fqt->nelts){ + // if beyond max then use max + fqt->n_in_use = fqt->nelts-1; + } - line = format(line, "%-7d%-20s%-12s%-8d", - i, - w->name ? w->name : (u8 *) "", - w->registration ? w->registration->name : "", - w->lwp); + /* Record the number of elements in use in the histogram */ + fqh = &tm->frame_queue_histogram[thread_id]; + fqh->count[ fqt->n_in_use ]++; -#if DPDK==1 - int lcore = w->dpdk_lcore_id; - if (lcore > -1) + /* Record a snapshot of the elements in use */ + for (elix=0; elixnelts; elix++) { + elt = fq->elts + ((fq->head+1 + elix) & (fq->nelts-1)); + if (1 || elt->valid) + { + fqt->n_vectors[elix] = elt->n_vectors; + } + } + fqt->written = 1; + } + + while (1) + { + if (fq->head == fq->tail) { - line = format(line, "%-7u%-7u%-7u", - lcore, - lcore_config[lcore].core_id, - lcore_config[lcore].socket_id); + fq->head_hint = fq->head; + return processed; + } - switch(lcore_config[lcore].state) - { - case WAIT: - line = format(line, "wait"); - break; - case RUNNING: - line = format(line, "running"); - break; - case FINISHED: - line = format(line, "finished"); - break; - default: - line = format(line, "unknown"); - } + elt = fq->elts + ((fq->head+1) & (fq->nelts-1)); + + if (!elt->valid) + { + fq->head_hint = fq->head; + return processed; + } + + from = elt->buffer_index; + msg_type = elt->msg_type; + + ASSERT (msg_type == VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME); + ASSERT (elt->n_vectors <= VLIB_FRAME_SIZE); + + f = vlib_get_frame_to_node (vm, tm->handoff_dispatch_node_index); + + to = vlib_frame_vector_args (f); + + n_left_to_node = elt->n_vectors; + + while (n_left_to_node >= 4) + { + to[0] = from[0]; + to[1] = from[1]; + to[2] = from[2]; + to[3] = from[3]; + to += 4; + from += 4; + n_left_to_node -= 4; + } + + while (n_left_to_node > 0) + { + to[0] = from[0]; + to++; + from++; + n_left_to_node--; + } + + vectors += elt->n_vectors; + f->n_vectors = elt->n_vectors; + vlib_put_frame_to_node (vm, tm->handoff_dispatch_node_index, f); + + elt->valid = 0; + elt->n_vectors = 0; + elt->msg_type = 0xfefefefe; + CLIB_MEMORY_BARRIER(); + fq->head++; + processed++; + + /* + * Limit the number of packets pushed into the graph + */ + if (vectors >= fq->vector_threshold) + { + fq->head_hint = fq->head; + return processed; } -#endif - vlib_cli_output(vm, "%v", line); - vec_free(line); } -#endif + ASSERT(0); + return processed; +} - return 0; +static_always_inline void +vlib_worker_thread_internal (vlib_main_t *vm) +{ + vlib_node_main_t * nm = &vm->node_main; + u64 cpu_time_now = clib_cpu_time_now (); + + while (1) + { + vlib_worker_thread_barrier_check (); + + vlib_frame_queue_dequeue_internal (vm); + + vlib_node_runtime_t * n; + vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]) + { + cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT, + VLIB_NODE_STATE_POLLING, /* frame */ 0, + cpu_time_now); + } + + if (_vec_len (nm->pending_frames)) + { + int i; + cpu_time_now = clib_cpu_time_now (); + for (i = 0; i < _vec_len (nm->pending_frames); i++) { + vlib_pending_frame_t *p; + + p = nm->pending_frames + i; + + cpu_time_now = dispatch_pending_node (vm, p, cpu_time_now); + } + _vec_len (nm->pending_frames) = 0; + } + vlib_increment_main_loop_counter (vm); + + /* Record time stamp in case there are no enabled nodes and above + calls do not update time stamp. */ + cpu_time_now = clib_cpu_time_now (); + } } +void vlib_worker_thread_fn (void * arg) +{ + vlib_worker_thread_t * w = (vlib_worker_thread_t *) arg; + vlib_thread_main_t * tm = vlib_get_thread_main(); + vlib_main_t * vm = vlib_get_main(); + + ASSERT(vm->cpu_index == os_get_cpu_number()); + + vlib_worker_thread_init (w); + clib_time_init (&vm->clib_time); + clib_mem_set_heap (w->thread_mheap); + + /* Wait until the dpdk init sequence is complete */ + while (tm->worker_thread_release == 0) + vlib_worker_thread_barrier_check (); + + vlib_worker_thread_internal(vm); +} -VLIB_CLI_COMMAND (show_threads_command, static) = { - .path = "show threads", - .short_help = "Show threads", - .function = show_threads_fn, +VLIB_REGISTER_THREAD (worker_thread_reg, static) = { + .name = "workers", + .short_name = "wk", + .function = vlib_worker_thread_fn, }; + +clib_error_t *threads_init (vlib_main_t *vm) +{ + vlib_thread_main_t * tm = vlib_get_thread_main(); + + tm->handoff_dispatch_node_index = ~0; + + return 0; +} + +VLIB_INIT_FUNCTION (threads_init); diff --git a/vlib/vlib/threads.h b/vlib/vlib/threads.h index f81be798626..e3b902ae518 100644 --- a/vlib/vlib/threads.h +++ b/vlib/vlib/threads.h @@ -298,7 +298,17 @@ typedef struct { uword * cpu_socket_bitmap; vlib_efd_t efd; - + + /* handoff node index */ + u32 handoff_dispatch_node_index; + + /* for frame queue tracing */ + frame_queue_trace_t *frame_queue_traces; + frame_queue_nelt_counter_t *frame_queue_histogram; + + /* worker thread initialization barrier */ + volatile u32 worker_thread_release; + } vlib_thread_main_t; vlib_thread_main_t vlib_thread_main; diff --git a/vlib/vlib/threads_cli.c b/vlib/vlib/threads_cli.c new file mode 100644 index 00000000000..26cb2213d95 --- /dev/null +++ b/vlib/vlib/threads_cli.c @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +static clib_error_t * +show_threads_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + vlib_worker_thread_t * w; + int i; + + vlib_cli_output (vm, "%-7s%-20s%-12s%-8s%-7s%-7s%-7s%-10s", + "ID", "Name", "Type", "LWP", + "lcore", "Core", "Socket", "State"); + +#if !defined(__powerpc64__) + for (i = 0; i < vec_len(vlib_worker_threads); i++) + { + w = vlib_worker_threads + i; + u8 * line = NULL; + + line = format(line, "%-7d%-20s%-12s%-8d", + i, + w->name ? w->name : (u8 *) "", + w->registration ? w->registration->name : "", + w->lwp); + +#if DPDK==1 + int lcore = w->dpdk_lcore_id; + if (lcore > -1) + { + line = format(line, "%-7u%-7u%-7u", + lcore, + lcore_config[lcore].core_id, + lcore_config[lcore].socket_id); + + switch(lcore_config[lcore].state) + { + case WAIT: + line = format(line, "wait"); + break; + case RUNNING: + line = format(line, "running"); + break; + case FINISHED: + line = format(line, "finished"); + break; + default: + line = format(line, "unknown"); + } + } +#endif + vlib_cli_output(vm, "%v", line); + vec_free(line); + } +#endif + + return 0; +} + + +VLIB_CLI_COMMAND (show_threads_command, static) = { + .path = "show threads", + .short_help = "Show threads", + .function = show_threads_fn, +}; + +/* + * Trigger threads to grab frame queue trace data + */ +static clib_error_t * +trace_frame_queue (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + clib_error_t * error = NULL; + frame_queue_trace_t *fqt; + frame_queue_nelt_counter_t *fqh; + vlib_thread_main_t *tm = vlib_get_thread_main(); + u32 num_fq; + u32 fqix; + u32 enable = 0; + + if (unformat(input, "on")) { + enable = 1; + } else if (unformat(input, "off")) { + enable = 0; + } else { + return clib_error_return(0, "expecting on or off"); + } + + num_fq = vec_len(vlib_frame_queues); + if (num_fq == 0) + { + vlib_cli_output(vm, "No frame queues exist\n"); + return error; + } + + // Allocate storage for trace if necessary + vec_validate_aligned(tm->frame_queue_traces, num_fq-1, CLIB_CACHE_LINE_BYTES); + vec_validate_aligned(tm->frame_queue_histogram, num_fq-1, CLIB_CACHE_LINE_BYTES); + + for (fqix=0; fqixframe_queue_traces[fqix]; + fqh = &tm->frame_queue_histogram[fqix]; + + memset(fqt->n_vectors, 0xff, sizeof(fqt->n_vectors)); + fqt->written = 0; + memset(fqh, 0, sizeof(*fqh)); + vlib_frame_queues[fqix]->trace = enable; + } + return error; +} + +VLIB_CLI_COMMAND (cmd_trace_frame_queue,static) = { + .path = "trace frame-queue", + .short_help = "trace frame-queue (on|off)", + .function = trace_frame_queue, + .is_mp_safe = 1, +}; + + +/* + * Adding two counters and compute percent of total + * Round up, e.g. 0.000001 => 1% + */ +static u32 +compute_percent (u64 *two_counters, u64 total) +{ + if (total == 0) + { + return 0; + } + else + { + return (((two_counters[0] + two_counters[1]) * 100) + (total-1)) / total; + } +} + +/* + * Display frame queue trace data gathered by threads. + */ +static clib_error_t * +show_frame_queue_internal (vlib_main_t *vm, + u32 histogram) +{ + vlib_thread_main_t *tm = vlib_get_thread_main(); + clib_error_t * error = NULL; + frame_queue_trace_t *fqt; + frame_queue_nelt_counter_t *fqh; + u32 num_fq; + u32 fqix; + + num_fq = vec_len(tm->frame_queue_traces); + if (num_fq == 0) + { + vlib_cli_output(vm, "No trace data for frame queues\n"); + return error; + } + + if (histogram) + { + vlib_cli_output(vm, "0-1 2-3 4-5 6-7 8-9 10-11 12-13 14-15 " + "16-17 18-19 20-21 22-23 24-25 26-27 28-29 30-31\n"); + } + + for (fqix=0; fqixframe_queue_traces[fqix]); + + vlib_cli_output(vm, "Thread %d %v\n", fqix, vlib_worker_threads[fqix].name); + + if (fqt->written == 0) + { + vlib_cli_output(vm, " no trace data\n"); + continue; + } + + if (histogram) + { + fqh = &(tm->frame_queue_histogram[fqix]); + u32 nelt; + u64 total = 0; + + for (nelt=0; neltcount[nelt]; + } + + /* + * Print in pairs to condense the output. + * Allow entries with 0 counts to be clearly identified, by rounding up. + * Any non-zero value will be displayed as at least one percent. This + * also means the sum of percentages can be > 100, but that is fine. The + * histogram is counted from the last time "trace frame on" was issued. + */ + vlib_cli_output(vm, + "%3d%% %3d%% %3d%% %3d%% %3d%% %3d%% %3d%% %3d%% " + "%3d%% %3d%% %3d%% %3d%% %3d%% %3d%% %3d%% %3d%%\n", + compute_percent(&fqh->count[ 0], total), + compute_percent(&fqh->count[ 2], total), + compute_percent(&fqh->count[ 4], total), + compute_percent(&fqh->count[ 6], total), + compute_percent(&fqh->count[ 8], total), + compute_percent(&fqh->count[10], total), + compute_percent(&fqh->count[12], total), + compute_percent(&fqh->count[14], total), + compute_percent(&fqh->count[16], total), + compute_percent(&fqh->count[18], total), + compute_percent(&fqh->count[20], total), + compute_percent(&fqh->count[22], total), + compute_percent(&fqh->count[24], total), + compute_percent(&fqh->count[26], total), + compute_percent(&fqh->count[28], total), + compute_percent(&fqh->count[30], total)); + } + else + { + vlib_cli_output(vm, " vector-threshold %d ring size %d in use %d\n", + fqt->threshold, fqt->nelts, fqt->n_in_use); + vlib_cli_output(vm, " head %12d head_hint %12d tail %12d\n", + fqt->head, fqt->head_hint, fqt->tail); + vlib_cli_output(vm, " %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d\n", + fqt->n_vectors[0], fqt->n_vectors[1], fqt->n_vectors[2], fqt->n_vectors[3], + fqt->n_vectors[4], fqt->n_vectors[5], fqt->n_vectors[6], fqt->n_vectors[7], + fqt->n_vectors[8], fqt->n_vectors[9], fqt->n_vectors[10], fqt->n_vectors[11], + fqt->n_vectors[12], fqt->n_vectors[13], fqt->n_vectors[14], fqt->n_vectors[15]); + + if (fqt->nelts > 16) + { + vlib_cli_output(vm, " %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d\n", + fqt->n_vectors[16], fqt->n_vectors[17], fqt->n_vectors[18], fqt->n_vectors[19], + fqt->n_vectors[20], fqt->n_vectors[21], fqt->n_vectors[22], fqt->n_vectors[23], + fqt->n_vectors[24], fqt->n_vectors[25], fqt->n_vectors[26], fqt->n_vectors[27], + fqt->n_vectors[28], fqt->n_vectors[29], fqt->n_vectors[30], fqt->n_vectors[31]); + } + } + + } + return error; +} + +static clib_error_t * +show_frame_queue_trace (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + return show_frame_queue_internal (vm, 0); +} + +static clib_error_t * +show_frame_queue_histogram (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + return show_frame_queue_internal (vm, 1); +} + +VLIB_CLI_COMMAND (cmd_show_frame_queue_trace,static) = { + .path = "show frame-queue", + .short_help = "show frame-queue trace", + .function = show_frame_queue_trace, +}; + +VLIB_CLI_COMMAND (cmd_show_frame_queue_histogram,static) = { + .path = "show frame-queue histogram", + .short_help = "show frame-queue histogram", + .function = show_frame_queue_histogram, +}; + + +/* + * Modify the number of elements on the frame_queues + */ +static clib_error_t * +test_frame_queue_nelts (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + clib_error_t * error = NULL; + u32 num_fq; + u32 fqix; + u32 nelts = 0; + + unformat(input, "%d", &nelts); + if ((nelts != 4) && (nelts != 8) && (nelts != 16) && (nelts != 32)) { + return clib_error_return(0, "expecting 4,8,16,32"); + } + + num_fq = vec_len(vlib_frame_queues); + if (num_fq == 0) + { + vlib_cli_output(vm, "No frame queues exist\n"); + return error; + } + + for (fqix=0; fqixnelts = nelts; + } + + return error; +} + +VLIB_CLI_COMMAND (cmd_test_frame_queue_nelts,static) = { + .path = "test frame-queue nelts", + .short_help = "test frame-queue nelts (4,8,16,32)", + .function = test_frame_queue_nelts, +}; + + +/* + * Modify the max number of packets pulled off the frame queues + */ +static clib_error_t * +test_frame_queue_threshold (vlib_main_t *vm, unformat_input_t *input, + vlib_cli_command_t *cmd) +{ + clib_error_t * error = NULL; + u32 num_fq; + u32 fqix; + u32 threshold = 0; + + if (unformat(input, "%d", &threshold)) { + } else { + vlib_cli_output(vm, "expecting threshold value\n"); + return error; + } + + if (threshold == 0) + threshold = ~0; + + num_fq = vec_len(vlib_frame_queues); + if (num_fq == 0) + { + vlib_cli_output(vm, "No frame queues exist\n"); + return error; + } + + for (fqix=0; fqixvector_threshold = threshold; + } + + return error; +} + +VLIB_CLI_COMMAND (cmd_test_frame_queue_threshold,static) = { + .path = "test frame-queue threshold", + .short_help = "test frame-queue threshold N (0=no limit)", + .function = test_frame_queue_threshold, +}; + diff --git a/vnet/Makefile.am b/vnet/Makefile.am index 2d623c3229d..dcea38fb8dc 100644 --- a/vnet/Makefile.am +++ b/vnet/Makefile.am @@ -593,13 +593,11 @@ libvnet_la_SOURCES += \ vnet/devices/dpdk/format.c \ vnet/devices/dpdk/init.c \ vnet/devices/dpdk/node.c \ - vnet/devices/dpdk/threads.c \ vnet/devices/dpdk/vhost_user.c \ vnet/devices/dpdk/cli.c nobase_include_HEADERS += \ - vnet/devices/dpdk/dpdk.h \ - vnet/devices/dpdk/threads.h + vnet/devices/dpdk/dpdk.h else libvnet_la_SOURCES += \ vnet/devices/nic/ixge.c \ diff --git a/vnet/vnet/devices/dpdk/cli.c b/vnet/vnet/devices/dpdk/cli.c index 9063cad890a..bcc3d2f91a5 100644 --- a/vnet/vnet/devices/dpdk/cli.c +++ b/vnet/vnet/devices/dpdk/cli.c @@ -281,283 +281,6 @@ show_dpdk_device_stats (vlib_main_t * vm, dpdk_device_t * xd) device_queue_sz); } - -/* - * Trigger threads to grab frame queue trace data - */ -static clib_error_t * -trace_frame_queue (vlib_main_t *vm, unformat_input_t *input, - vlib_cli_command_t *cmd) -{ - clib_error_t * error = NULL; - frame_queue_trace_t *fqt; - frame_queue_nelt_counter_t *fqh; - u32 num_fq; - u32 fqix; - u32 enable = 0; - - if (unformat(input, "on")) { - enable = 1; - } else if (unformat(input, "off")) { - enable = 0; - } else { - return clib_error_return(0, "expecting on or off"); - } - - num_fq = vec_len(vlib_frame_queues); - if (num_fq == 0) - { - vlib_cli_output(vm, "No frame queues exist\n"); - return error; - } - - // Allocate storage for trace if necessary - vec_validate_aligned(dpdk_main.frame_queue_traces, num_fq-1, CLIB_CACHE_LINE_BYTES); - vec_validate_aligned(dpdk_main.frame_queue_histogram, num_fq-1, CLIB_CACHE_LINE_BYTES); - - for (fqix=0; fqixn_vectors, 0xff, sizeof(fqt->n_vectors)); - fqt->written = 0; - memset(fqh, 0, sizeof(*fqh)); - vlib_frame_queues[fqix]->trace = enable; - } - return error; -} - -VLIB_CLI_COMMAND (cmd_trace_frame_queue,static) = { - .path = "trace frame-queue", - .short_help = "trace frame-queue (on|off)", - .function = trace_frame_queue, - .is_mp_safe = 1, -}; - - -/* - * Adding two counters and compute percent of total - * Round up, e.g. 0.000001 => 1% - */ -static u32 -compute_percent (u64 *two_counters, u64 total) -{ - if (total == 0) - { - return 0; - } - else - { - return (((two_counters[0] + two_counters[1]) * 100) + (total-1)) / total; - } -} - -/* - * Display frame queue trace data gathered by threads. - */ -static clib_error_t * -show_frame_queue_internal (vlib_main_t *vm, - u32 histogram) -{ - clib_error_t * error = NULL; - frame_queue_trace_t *fqt; - frame_queue_nelt_counter_t *fqh; - u32 num_fq; - u32 fqix; - - num_fq = vec_len(dpdk_main.frame_queue_traces); - if (num_fq == 0) - { - vlib_cli_output(vm, "No trace data for frame queues\n"); - return error; - } - - if (histogram) - { - vlib_cli_output(vm, "0-1 2-3 4-5 6-7 8-9 10-11 12-13 14-15 " - "16-17 18-19 20-21 22-23 24-25 26-27 28-29 30-31\n"); - } - - for (fqix=0; fqixwritten == 0) - { - vlib_cli_output(vm, " no trace data\n"); - continue; - } - - if (histogram) - { - fqh = &(dpdk_main.frame_queue_histogram[fqix]); - u32 nelt; - u64 total = 0; - - for (nelt=0; neltcount[nelt]; - } - - /* - * Print in pairs to condense the output. - * Allow entries with 0 counts to be clearly identified, by rounding up. - * Any non-zero value will be displayed as at least one percent. This - * also means the sum of percentages can be > 100, but that is fine. The - * histogram is counted from the last time "trace frame on" was issued. - */ - vlib_cli_output(vm, - "%3d%% %3d%% %3d%% %3d%% %3d%% %3d%% %3d%% %3d%% " - "%3d%% %3d%% %3d%% %3d%% %3d%% %3d%% %3d%% %3d%%\n", - compute_percent(&fqh->count[ 0], total), - compute_percent(&fqh->count[ 2], total), - compute_percent(&fqh->count[ 4], total), - compute_percent(&fqh->count[ 6], total), - compute_percent(&fqh->count[ 8], total), - compute_percent(&fqh->count[10], total), - compute_percent(&fqh->count[12], total), - compute_percent(&fqh->count[14], total), - compute_percent(&fqh->count[16], total), - compute_percent(&fqh->count[18], total), - compute_percent(&fqh->count[20], total), - compute_percent(&fqh->count[22], total), - compute_percent(&fqh->count[24], total), - compute_percent(&fqh->count[26], total), - compute_percent(&fqh->count[28], total), - compute_percent(&fqh->count[30], total)); - } - else - { - vlib_cli_output(vm, " vector-threshold %d ring size %d in use %d\n", - fqt->threshold, fqt->nelts, fqt->n_in_use); - vlib_cli_output(vm, " head %12d head_hint %12d tail %12d\n", - fqt->head, fqt->head_hint, fqt->tail); - vlib_cli_output(vm, " %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d\n", - fqt->n_vectors[0], fqt->n_vectors[1], fqt->n_vectors[2], fqt->n_vectors[3], - fqt->n_vectors[4], fqt->n_vectors[5], fqt->n_vectors[6], fqt->n_vectors[7], - fqt->n_vectors[8], fqt->n_vectors[9], fqt->n_vectors[10], fqt->n_vectors[11], - fqt->n_vectors[12], fqt->n_vectors[13], fqt->n_vectors[14], fqt->n_vectors[15]); - - if (fqt->nelts > 16) - { - vlib_cli_output(vm, " %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d\n", - fqt->n_vectors[16], fqt->n_vectors[17], fqt->n_vectors[18], fqt->n_vectors[19], - fqt->n_vectors[20], fqt->n_vectors[21], fqt->n_vectors[22], fqt->n_vectors[23], - fqt->n_vectors[24], fqt->n_vectors[25], fqt->n_vectors[26], fqt->n_vectors[27], - fqt->n_vectors[28], fqt->n_vectors[29], fqt->n_vectors[30], fqt->n_vectors[31]); - } - } - - } - return error; -} - -static clib_error_t * -show_frame_queue_trace (vlib_main_t *vm, unformat_input_t *input, - vlib_cli_command_t *cmd) -{ - return show_frame_queue_internal (vm, 0); -} - -static clib_error_t * -show_frame_queue_histogram (vlib_main_t *vm, unformat_input_t *input, - vlib_cli_command_t *cmd) -{ - return show_frame_queue_internal (vm, 1); -} - -VLIB_CLI_COMMAND (cmd_show_frame_queue_trace,static) = { - .path = "show frame-queue", - .short_help = "show frame-queue trace", - .function = show_frame_queue_trace, -}; - -VLIB_CLI_COMMAND (cmd_show_frame_queue_histogram,static) = { - .path = "show frame-queue histogram", - .short_help = "show frame-queue histogram", - .function = show_frame_queue_histogram, -}; - - -/* - * Modify the number of elements on the frame_queues - */ -static clib_error_t * -test_frame_queue_nelts (vlib_main_t *vm, unformat_input_t *input, - vlib_cli_command_t *cmd) -{ - clib_error_t * error = NULL; - u32 num_fq; - u32 fqix; - u32 nelts = 0; - - unformat(input, "%d", &nelts); - if ((nelts != 4) && (nelts != 8) && (nelts != 16) && (nelts != 32)) { - return clib_error_return(0, "expecting 4,8,16,32"); - } - - num_fq = vec_len(vlib_frame_queues); - if (num_fq == 0) - { - vlib_cli_output(vm, "No frame queues exist\n"); - return error; - } - - for (fqix=0; fqixnelts = nelts; - } - - return error; -} - -VLIB_CLI_COMMAND (cmd_test_frame_queue_nelts,static) = { - .path = "test frame-queue nelts", - .short_help = "test frame-queue nelts (4,8,16,32)", - .function = test_frame_queue_nelts, -}; - - -/* - * Modify the max number of packets pulled off the frame queues - */ -static clib_error_t * -test_frame_queue_threshold (vlib_main_t *vm, unformat_input_t *input, - vlib_cli_command_t *cmd) -{ - clib_error_t * error = NULL; - u32 num_fq; - u32 fqix; - u32 threshold = 0; - - if (unformat(input, "%d", &threshold)) { - } else { - vlib_cli_output(vm, "expecting threshold value\n"); - return error; - } - - if (threshold == 0) - threshold = ~0; - - num_fq = vec_len(vlib_frame_queues); - if (num_fq == 0) - { - vlib_cli_output(vm, "No frame queues exist\n"); - return error; - } - - for (fqix=0; fqixvector_threshold = threshold; - } - - return error; -} - -VLIB_CLI_COMMAND (cmd_test_frame_queue_threshold,static) = { - .path = "test frame-queue threshold", - .short_help = "test frame-queue threshold N (0=no limit)", - .function = test_frame_queue_threshold, -}; - static void show_efd_config (vlib_main_t * vm) { diff --git a/vnet/vnet/devices/dpdk/dpdk.h b/vnet/vnet/devices/dpdk/dpdk.h index 1c3d0bb860f..5d2a2db7cf5 100644 --- a/vnet/vnet/devices/dpdk/dpdk.h +++ b/vnet/vnet/devices/dpdk/dpdk.h @@ -260,22 +260,6 @@ typedef struct { u8 need_txlock; /* Used by VNET_DPDK_DEV_VHOST_USER */ } dpdk_device_t; -#define MAX_NELTS 32 -typedef struct { - CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); - u64 head; - u64 head_hint; - u64 tail; - u32 n_in_use; - u32 nelts; - u32 written; - u32 threshold; - i32 n_vectors[MAX_NELTS]; -} frame_queue_trace_t; - -typedef struct { - u64 count[MAX_NELTS]; -} frame_queue_nelt_counter_t; #define DPDK_TX_RING_SIZE (4 * 1024) @@ -393,9 +377,6 @@ typedef struct { /* Ethernet input node index */ u32 ethernet_input_node_index; - /* dpdk i/o thread initialization barrier */ - volatile u32 worker_thread_release; - /* pcap tracing [only works if (CLIB_DEBUG > 0)] */ int tx_pcap_enable; pcap_main_t pcap_main; @@ -430,10 +411,6 @@ typedef struct { f64 link_state_poll_interval; f64 stat_poll_interval; - /* for frame queue tracing */ - frame_queue_trace_t *frame_queue_traces; - frame_queue_nelt_counter_t *frame_queue_histogram; - /* Sleep for this many MS after each device poll */ u32 poll_sleep; diff --git a/vnet/vnet/devices/dpdk/init.c b/vnet/vnet/devices/dpdk/init.c index 1f739494dd1..9054f5e2ea5 100644 --- a/vnet/vnet/devices/dpdk/init.c +++ b/vnet/vnet/devices/dpdk/init.c @@ -1565,7 +1565,7 @@ dpdk_process (vlib_main_t * vm, dpdk_vhost_user_process_init(&vu_state); - dm->worker_thread_release = 1; + tm->worker_thread_release = 1; f64 now = vlib_time_now (vm); vec_foreach (xd, dm->devices) diff --git a/vnet/vnet/devices/dpdk/threads.c b/vnet/vnet/devices/dpdk/threads.c deleted file mode 100644 index 48ee13d1eac..00000000000 --- a/vnet/vnet/devices/dpdk/threads.c +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include /* enumerate all vlib messages */ - -#define vl_typedefs /* define message structures */ -#include -#undef vl_typedefs - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) -#define vl_printfun -#include -#undef vl_printfun - - -/* - * Check the frame queue to see if any frames are available. - * If so, pull the packets off the frames and put them to - * the handoff node. - */ -static inline int vlib_frame_queue_dequeue_internal (vlib_main_t *vm) -{ - u32 thread_id = vm->cpu_index; - vlib_frame_queue_t *fq = vlib_frame_queues[thread_id]; - vlib_frame_queue_elt_t *elt; - u32 * from, * to; - vlib_frame_t * f; - int msg_type; - int processed = 0; - u32 n_left_to_node; - u32 vectors = 0; - - ASSERT (fq); - ASSERT(vm == vlib_mains[thread_id]); - - /* - * Gather trace data for frame queues - */ - if (PREDICT_FALSE(fq->trace)) - { - frame_queue_trace_t *fqt; - frame_queue_nelt_counter_t *fqh; - u32 elix; - - fqt = &dpdk_main.frame_queue_traces[thread_id]; - - fqt->nelts = fq->nelts; - fqt->head = fq->head; - fqt->head_hint = fq->head_hint; - fqt->tail = fq->tail; - fqt->threshold = fq->vector_threshold; - fqt->n_in_use = fqt->tail - fqt->head; - if (fqt->n_in_use >= fqt->nelts){ - // if beyond max then use max - fqt->n_in_use = fqt->nelts-1; - } - - /* Record the number of elements in use in the histogram */ - fqh = &dpdk_main.frame_queue_histogram[thread_id]; - fqh->count[ fqt->n_in_use ]++; - - /* Record a snapshot of the elements in use */ - for (elix=0; elixnelts; elix++) { - elt = fq->elts + ((fq->head+1 + elix) & (fq->nelts-1)); - if (1 || elt->valid) - { - fqt->n_vectors[elix] = elt->n_vectors; - } - } - fqt->written = 1; - } - - while (1) - { - if (fq->head == fq->tail) - { - fq->head_hint = fq->head; - return processed; - } - - elt = fq->elts + ((fq->head+1) & (fq->nelts-1)); - - if (!elt->valid) - { - fq->head_hint = fq->head; - return processed; - } - - from = elt->buffer_index; - msg_type = elt->msg_type; - - ASSERT (msg_type == VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME); - ASSERT (elt->n_vectors <= VLIB_FRAME_SIZE); - - f = vlib_get_frame_to_node - (vm, 1 ? handoff_dispatch_node.index : ethernet_input_node.index); - - to = vlib_frame_vector_args (f); - - n_left_to_node = elt->n_vectors; - - while (n_left_to_node >= 4) - { - to[0] = from[0]; - to[1] = from[1]; - to[2] = from[2]; - to[3] = from[3]; - to += 4; - from += 4; - n_left_to_node -= 4; - } - - while (n_left_to_node > 0) - { - to[0] = from[0]; - to++; - from++; - n_left_to_node--; - } - - vectors += elt->n_vectors; - f->n_vectors = elt->n_vectors; - vlib_put_frame_to_node - (vm, 1 ? handoff_dispatch_node.index : ethernet_input_node.index, f); - - elt->valid = 0; - elt->n_vectors = 0; - elt->msg_type = 0xfefefefe; - CLIB_MEMORY_BARRIER(); - fq->head++; - processed++; - - /* - * Limit the number of packets pushed into the graph - */ - if (vectors >= fq->vector_threshold) - { - fq->head_hint = fq->head; - return processed; - } - } - ASSERT(0); - return processed; -} - -int dpdk_frame_queue_dequeue (vlib_main_t *vm) -{ - return vlib_frame_queue_dequeue_internal (vm); -} - -/* - * dpdk_worker_thread - Contains the main loop of a worker thread. - * - * w - * Information for the current thread - * callback - * If not null, this function will be called once during each main loop. - */ -static_always_inline void -dpdk_worker_thread_internal (vlib_main_t *vm, - dpdk_worker_thread_callback_t callback) -{ - vlib_node_main_t * nm = &vm->node_main; - u64 cpu_time_now = clib_cpu_time_now (); - - while (1) - { - vlib_worker_thread_barrier_check (); - - vlib_frame_queue_dequeue_internal (vm); - - /* Invoke callback if supplied */ - if (PREDICT_FALSE(callback != NULL)) - callback(vm); - - vlib_node_runtime_t * n; - vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]) - { - cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT, - VLIB_NODE_STATE_POLLING, /* frame */ 0, - cpu_time_now); - } - - if (_vec_len (nm->pending_frames)) - { - int i; - cpu_time_now = clib_cpu_time_now (); - for (i = 0; i < _vec_len (nm->pending_frames); i++) { - vlib_pending_frame_t *p; - - p = nm->pending_frames + i; - - cpu_time_now = dispatch_pending_node (vm, p, cpu_time_now); - } - _vec_len (nm->pending_frames) = 0; - } - vlib_increment_main_loop_counter (vm); - - /* Record time stamp in case there are no enabled nodes and above - calls do not update time stamp. */ - cpu_time_now = clib_cpu_time_now (); - } -} - -void dpdk_worker_thread (vlib_worker_thread_t * w, - dpdk_worker_thread_callback_t callback) -{ - vlib_main_t *vm; - dpdk_main_t * dm = &dpdk_main; - - vm = vlib_get_main(); - - ASSERT(vm->cpu_index == os_get_cpu_number()); - - clib_time_init (&vm->clib_time); - clib_mem_set_heap (w->thread_mheap); - - /* Wait until the dpdk init sequence is complete */ - while (dm->worker_thread_release == 0) - vlib_worker_thread_barrier_check (); - - dpdk_worker_thread_internal(vm, callback); -} - -void dpdk_worker_thread_fn (void * arg) -{ - vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; - vlib_worker_thread_init (w); - dpdk_worker_thread (w, 0); -} - -#if VIRL == 0 -VLIB_REGISTER_THREAD (worker_thread_reg, static) = { - .name = "workers", - .short_name = "wk", - .function = dpdk_worker_thread_fn, -}; -#endif - -static clib_error_t * -dpdk_thread_init (vlib_main_t *vm) -{ - return (0); -} - -VLIB_INIT_FUNCTION(dpdk_thread_init); diff --git a/vnet/vnet/devices/dpdk/threads.h b/vnet/vnet/devices/dpdk/threads.h deleted file mode 100644 index c7d301bee21..00000000000 --- a/vnet/vnet/devices/dpdk/threads.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_dpdk_threads_h__ -#define __included_dpdk_threads_h__ - -#include - -typedef void (*dpdk_worker_thread_callback_t) (vlib_main_t *vm); - -void dpdk_worker_thread (vlib_worker_thread_t * w, - dpdk_worker_thread_callback_t callback); - -int dpdk_frame_queue_dequeue (vlib_main_t *vm); - -#endif /* __included_dpdk_threads_h__ */ diff --git a/vnet/vnet/handoff.c b/vnet/vnet/handoff.c index af8068a8729..28968c9020a 100644 --- a/vnet/vnet/handoff.c +++ b/vnet/vnet/handoff.c @@ -508,8 +508,12 @@ clib_error_t *handoff_init (vlib_main_t *vm) { handoff_main_t * hm = &handoff_main; vlib_thread_main_t * tm = vlib_get_thread_main(); + clib_error_t * error; uword * p; + if ((error = vlib_call_init_function (vm, threads_init))) + return error; + vlib_thread_registration_t * tr; /* Only the standard vnet worker threads are supported */ p = hash_get_mem (tm->thread_registrations_by_name, "workers"); @@ -526,6 +530,9 @@ clib_error_t *handoff_init (vlib_main_t *vm) hm->vlib_main = vm; hm->vnet_main = &vnet_main; + ASSERT (tm->handoff_dispatch_node_index == ~0); + tm->handoff_dispatch_node_index = handoff_dispatch_node.index; + return 0; } -- cgit 1.2.3-korg