From 7cd468a3d7dee7d6c92f69a0bb7061ae208ec727 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 19 Dec 2016 23:05:39 +0100 Subject: Reorganize source tree to use single autotools instance Change-Id: I7b51f88292e057c6443b12224486f2d0c9f8ae23 Signed-off-by: Damjan Marion --- src/vnet/hdlc/node.c | 351 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 351 insertions(+) create mode 100644 src/vnet/hdlc/node.c (limited to 'src/vnet/hdlc/node.c') diff --git a/src/vnet/hdlc/node.c b/src/vnet/hdlc/node.c new file mode 100644 index 00000000..4fe0296a --- /dev/null +++ b/src/vnet/hdlc/node.c @@ -0,0 +1,351 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * hdlc_node.c: hdlc packet processing + * + * Copyright (c) 2010 Eliot Dresselhaus + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include + +#define foreach_hdlc_input_next \ + _ (PUNT, "error-punt") \ + _ (DROP, "error-drop") + +typedef enum { +#define _(s,n) HDLC_INPUT_NEXT_##s, + foreach_hdlc_input_next +#undef _ + HDLC_INPUT_N_NEXT, +} hdlc_input_next_t; + +typedef struct { + u8 packet_data[32]; +} hdlc_input_trace_t; + +static u8 * format_hdlc_input_trace (u8 * s, va_list * va) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *); + hdlc_input_trace_t * t = va_arg (*va, hdlc_input_trace_t *); + + s = format (s, "%U", format_hdlc_header, t->packet_data); + + return s; +} + +typedef struct { + /* Sparse vector mapping hdlc protocol in network byte order + to next index. */ + u16 * next_by_protocol; + + u32 * sparse_index_by_next_index; +} hdlc_input_runtime_t; + +static uword +hdlc_input (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + hdlc_input_runtime_t * rt = (void *) node->runtime_data; + u32 n_left_from, next_index, i_next, * from, * to_next; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + if (node->flags & VLIB_NODE_FLAG_TRACE) + vlib_trace_frame_buffers_only (vm, node, + from, + n_left_from, + sizeof (from[0]), + sizeof (hdlc_input_trace_t)); + + next_index = node->cached_next_index; + i_next = vec_elt (rt->sparse_index_by_next_index, next_index); + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, + to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + u32 bi0, bi1; + vlib_buffer_t * b0, * b1; + hdlc_header_t * h0, * h1; + u32 i0, i1, len0, len1, protocol0, protocol1, enqueue_code; + + /* Prefetch next iteration. */ + { + vlib_buffer_t * b2, * b3; + + b2 = vlib_get_buffer (vm, from[2]); + b3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (b2, LOAD); + vlib_prefetch_buffer_header (b3, LOAD); + + CLIB_PREFETCH (b2->data, sizeof (h0[0]), LOAD); + CLIB_PREFETCH (b3->data, sizeof (h1[0]), LOAD); + } + + bi0 = from[0]; + bi1 = from[1]; + to_next[0] = bi0; + to_next[1] = bi1; + from += 2; + to_next += 2; + n_left_to_next -= 2; + n_left_from -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + h0 = (void *) (b0->data + b0->current_data); + h1 = (void *) (b1->data + b1->current_data); + + protocol0 = h0->protocol; + protocol1 = h1->protocol; + + /* Add padding bytes for OSI protocols. */ + len0 = sizeof (h0[0]); + len1 = sizeof (h1[0]); + + len0 += protocol0 == clib_host_to_net_u16 (HDLC_PROTOCOL_osi); + len1 += protocol1 == clib_host_to_net_u16 (HDLC_PROTOCOL_osi); + + b0->current_data += len0; + b1->current_data += len1; + + b0->current_length -= len0; + b1->current_length -= len1; + + /* Index sparse array with network byte order. */ + sparse_vec_index2 (rt->next_by_protocol, protocol0, protocol1, &i0, &i1); + + b0->error = node->errors[i0 == SPARSE_VEC_INVALID_INDEX ? HDLC_ERROR_UNKNOWN_PROTOCOL : HDLC_ERROR_NONE]; + b1->error = node->errors[i1 == SPARSE_VEC_INVALID_INDEX ? HDLC_ERROR_UNKNOWN_PROTOCOL : HDLC_ERROR_NONE]; + + enqueue_code = (i0 != i_next) + 2*(i1 != i_next); + + if (PREDICT_FALSE (enqueue_code != 0)) + { + switch (enqueue_code) + { + case 1: + /* A B A */ + to_next[-2] = bi1; + to_next -= 1; + n_left_to_next += 1; + vlib_set_next_frame_buffer (vm, node, vec_elt (rt->next_by_protocol, i0), bi0); + break; + + case 2: + /* A A B */ + to_next -= 1; + n_left_to_next += 1; + vlib_set_next_frame_buffer (vm, node, vec_elt (rt->next_by_protocol, i1), bi1); + break; + + case 3: + /* A B B or A B C */ + to_next -= 2; + n_left_to_next += 2; + vlib_set_next_frame_buffer (vm, node, vec_elt (rt->next_by_protocol, i0), bi0); + vlib_set_next_frame_buffer (vm, node, vec_elt (rt->next_by_protocol, i1), bi1); + if (i0 == i1) + { + vlib_put_next_frame (vm, node, next_index, + n_left_to_next); + i_next = i1; + next_index = vec_elt (rt->next_by_protocol, i_next); + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + } + } + } + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t * b0; + hdlc_header_t * h0; + u32 i0, len0, protocol0; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + h0 = (void *) (b0->data + b0->current_data); + + protocol0 = h0->protocol; + + /* Add padding bytes for OSI protocols. */ + len0 = sizeof (h0[0]); + len0 += protocol0 == clib_host_to_net_u16 (HDLC_PROTOCOL_osi); + + b0->current_data += len0; + b0->current_length -= len0; + + i0 = sparse_vec_index (rt->next_by_protocol, protocol0); + + b0->error = node->errors[i0 == SPARSE_VEC_INVALID_INDEX ? HDLC_ERROR_UNKNOWN_PROTOCOL : HDLC_ERROR_NONE]; + + /* Sent packet to wrong next? */ + if (PREDICT_FALSE (i0 != i_next)) + { + /* Return old frame; remove incorrectly enqueued packet. */ + vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1); + + /* Send to correct next. */ + i_next = i0; + next_index = vec_elt (rt->next_by_protocol, i_next); + vlib_get_next_frame (vm, node, next_index, + to_next, n_left_to_next); + + to_next[0] = bi0; + to_next += 1; + n_left_to_next -= 1; + } + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + return from_frame->n_vectors; +} + +static char * hdlc_error_strings[] = { +#define hdlc_error(n,s) s, +#include "error.def" +#undef hdlc_error +}; + +VLIB_REGISTER_NODE (hdlc_input_node) = { + .function = hdlc_input, + .name = "hdlc-input", + /* Takes a vector of packets. */ + .vector_size = sizeof (u32), + + .runtime_data_bytes = sizeof (hdlc_input_runtime_t), + + .n_errors = HDLC_N_ERROR, + .error_strings = hdlc_error_strings, + + .n_next_nodes = HDLC_INPUT_N_NEXT, + .next_nodes = { +#define _(s,n) [HDLC_INPUT_NEXT_##s] = n, + foreach_hdlc_input_next +#undef _ + }, + + .format_buffer = format_hdlc_header_with_length, + .format_trace = format_hdlc_input_trace, + .unformat_buffer = unformat_hdlc_header, +}; + +static clib_error_t * hdlc_input_init (vlib_main_t * vm) +{ + hdlc_input_runtime_t * rt; + + { + clib_error_t * error = vlib_call_init_function (vm, hdlc_init); + if (error) + clib_error_report (error); + } + + hdlc_setup_node (vm, hdlc_input_node.index); + + rt = vlib_node_get_runtime_data (vm, hdlc_input_node.index); + + rt->next_by_protocol = sparse_vec_new + (/* elt bytes */ sizeof (rt->next_by_protocol[0]), + /* bits in index */ BITS (((hdlc_header_t *) 0)->protocol)); + + vec_validate (rt->sparse_index_by_next_index, HDLC_INPUT_NEXT_DROP); + vec_validate (rt->sparse_index_by_next_index, HDLC_INPUT_NEXT_PUNT); + rt->sparse_index_by_next_index[HDLC_INPUT_NEXT_DROP] + = SPARSE_VEC_INVALID_INDEX; + rt->sparse_index_by_next_index[HDLC_INPUT_NEXT_PUNT] + = SPARSE_VEC_INVALID_INDEX; + + return 0; +} + +VLIB_INIT_FUNCTION (hdlc_input_init); + +void +hdlc_register_input_protocol (vlib_main_t * vm, + hdlc_protocol_t protocol, + u32 node_index) +{ + hdlc_main_t * em = &hdlc_main; + hdlc_protocol_info_t * pi; + hdlc_input_runtime_t * rt; + u16 * n; + u32 i; + + { + clib_error_t * error = vlib_call_init_function (vm, hdlc_input_init); + if (error) + clib_error_report (error); + } + + pi = hdlc_get_protocol_info (em, protocol); + pi->node_index = node_index; + pi->next_index = vlib_node_add_next (vm, + hdlc_input_node.index, + node_index); + + /* Setup hdlc protocol -> next index sparse vector mapping. */ + rt = vlib_node_get_runtime_data (vm, hdlc_input_node.index); + n = sparse_vec_validate (rt->next_by_protocol, clib_host_to_net_u16 (protocol)); + n[0] = pi->next_index; + + /* Rebuild next index -> sparse index inverse mapping when sparse vector + is updated. */ + vec_validate (rt->sparse_index_by_next_index, pi->next_index); + for (i = 1; i < vec_len (rt->next_by_protocol); i++) + rt->sparse_index_by_next_index[rt->next_by_protocol[i]] = i; +} -- cgit 1.2.3-korg From e9f929b52ddb741ec1e4cb2d92c6be1e798933a0 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Thu, 16 Mar 2017 11:32:09 +0100 Subject: vlib: make runtime_data thread-local Change-Id: I4aa3e7e42fb81211de1aed07dc7befee87a1e18b Signed-off-by: Damjan Marion --- src/vlib/init.h | 1 + src/vlib/main.h | 1 + src/vlib/node.c | 4 +- src/vlib/node.h | 81 +++++++++++++++++++++------------------- src/vlib/threads.c | 61 ++++++++++++++++++++++++++++-- src/vnet/gre/node.c | 26 ++++++++++--- src/vnet/hdlc/node.c | 27 +++++++++----- src/vnet/l2/l2_input_classify.c | 15 ++++++++ src/vnet/l2/l2_output_classify.c | 16 ++++++++ src/vnet/l2tp/l2tp.c | 10 +++++ src/vnet/mpls/node.c | 13 +++++++ src/vnet/ppp/node.c | 27 +++++++++----- src/vnet/tcp/tcp_syn_filter4.c | 23 +++++++----- src/vnet/udp/udp_local.c | 74 ++++++++++++++++++++++-------------- 14 files changed, 273 insertions(+), 106 deletions(-) (limited to 'src/vnet/hdlc/node.c') diff --git a/src/vlib/init.h b/src/vlib/init.h index 4fa5b304..12db3f90 100644 --- a/src/vlib/init.h +++ b/src/vlib/init.h @@ -109,6 +109,7 @@ static void __vlib_add_##tag##_function_##x (void) \ } #define VLIB_INIT_FUNCTION(x) VLIB_DECLARE_INIT_FUNCTION(x,init) +#define VLIB_WORKER_INIT_FUNCTION(x) VLIB_DECLARE_INIT_FUNCTION(x,worker_init) #define VLIB_MAIN_LOOP_ENTER_FUNCTION(x) \ VLIB_DECLARE_INIT_FUNCTION(x,main_loop_enter) diff --git a/src/vlib/main.h b/src/vlib/main.h index a6d50b39..98bc823d 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -162,6 +162,7 @@ typedef struct vlib_main_t /* List of init functions to call, setup by constructors */ _vlib_init_function_list_elt_t *init_function_registrations; + _vlib_init_function_list_elt_t *worker_init_function_registrations; _vlib_init_function_list_elt_t *main_loop_enter_function_registrations; _vlib_init_function_list_elt_t *main_loop_exit_function_registrations; _vlib_init_function_list_elt_t *api_init_function_registrations; diff --git a/src/vlib/node.c b/src/vlib/node.c index c419a13a..dc0a4de5 100644 --- a/src/vlib/node.c +++ b/src/vlib/node.c @@ -434,9 +434,7 @@ register_node (vlib_main_t * vm, vlib_node_registration_t * r) rt->errors[i] = vlib_error_set (n->index, i); STATIC_ASSERT_SIZEOF (vlib_node_runtime_t, 128); - ASSERT (vec_len (n->runtime_data) <= - sizeof (vlib_node_runtime_t) - - STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data)); + ASSERT (vec_len (n->runtime_data) <= VLIB_NODE_RUNTIME_DATA_SIZE); if (vec_len (n->runtime_data) > 0) clib_memcpy (rt->runtime_data, n->runtime_data, diff --git a/src/vlib/node.h b/src/vlib/node.h index b624e9d6..2a532cc3 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -411,65 +411,68 @@ typedef struct typedef struct vlib_node_runtime_t { - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - /* Node function to call. */ - vlib_node_function_t *function; + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); /**< cacheline mark */ - /* Vector of errors for this node. */ - vlib_error_t *errors; + vlib_node_function_t *function; /**< Node function to call. */ - /* Number of clock cycles. */ - u32 clocks_since_last_overflow; + vlib_error_t *errors; /**< Vector of errors for this node. */ - /* Maximum clock cycle for an invocation. */ - u32 max_clock; + u32 clocks_since_last_overflow; /**< Number of clock cycles. */ - /* Number of vectors in the recorded max_clock. */ - u32 max_clock_n; + u32 max_clock; /**< Maximum clock cycle for an + invocation. */ - /* Number of calls. */ - u32 calls_since_last_overflow; + u32 max_clock_n; /**< Number of vectors in the recorded + max_clock. */ - /* Number of vector elements processed by this node. */ - u32 vectors_since_last_overflow; + u32 calls_since_last_overflow; /**< Number of calls. */ - /* Start of next frames for this node. */ - u32 next_frame_index; + u32 vectors_since_last_overflow; /**< Number of vector elements + processed by this node. */ - /* Node index. */ - u32 node_index; + u32 next_frame_index; /**< Start of next frames for this + node. */ - /* For input nodes: decremented on each main loop interation until it reaches zero - and function is called. Allows some input nodes to be called - more than others. */ - u32 input_main_loops_per_call; + u32 node_index; /**< Node index. */ - /* Saved main loop counter of last dispatch of this node. */ - u32 main_loop_count_last_dispatch; + u32 input_main_loops_per_call; /**< For input nodes: decremented + on each main loop interation until + it reaches zero and function is + called. Allows some input nodes to + be called more than others. */ + + u32 main_loop_count_last_dispatch; /**< Saved main loop counter of last + dispatch of this node. */ u32 main_loop_vector_stats[2]; - /* Copy of main node flags. */ - u16 flags; + u16 flags; /**< Copy of main node flags. */ - /* Input node state. */ - u16 state; + u16 state; /**< Input node state. */ u16 n_next_nodes; - /* Next frame index that vector arguments were last enqueued to - last time this node ran. Set to zero before first run - of this node. */ - u16 cached_next_index; - - /* CPU this node runs on */ - u16 cpu_index; - - /* Function dependent node-runtime. */ - u8 runtime_data[0]; + u16 cached_next_index; /**< Next frame index that vector + arguments were last enqueued to + last time this node ran. Set to + zero before first run of this + node. */ + + u16 cpu_index; /**< CPU this node runs on */ + + u8 runtime_data[0]; /**< Function dependent + node-runtime data. This data is + thread local, and it is not + cloned from main thread. It needs + to be initialized for each thread + before it is used unless + runtime_data template exists in + vlib_node_t. */ } vlib_node_runtime_t; +#define VLIB_NODE_RUNTIME_DATA_SIZE (sizeof (vlib_node_runtime_t) - STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data)) + typedef struct { /* Number of allocated frames for this scalar/vector size. */ diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 07dbff33..3756c3fa 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -633,6 +633,8 @@ start_workers (vlib_main_t * vm) vm_clone->cpu_index = worker_thread_index; vm_clone->heap_base = w->thread_mheap; vm_clone->mbuf_alloc_list = 0; + vm_clone->init_functions_called = + hash_create (0, /* value bytes */ 0); memset (&vm_clone->random_buffer, 0, sizeof (vm_clone->random_buffer)); @@ -674,11 +676,33 @@ start_workers (vlib_main_t * vm) } nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] = vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]); + vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) + { + vlib_node_t *n = vlib_get_node (vm, rt->node_index); + rt->cpu_index = vm_clone->cpu_index; + /* copy initial runtime_data from node */ + if (n->runtime_data_bytes > 0) + clib_memcpy (rt->runtime_data, n->runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); + else if (CLIB_DEBUG > 0) + memset (rt->runtime_data, 0xfe, + VLIB_NODE_RUNTIME_DATA_SIZE); + } nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] = vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]); vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) + { + vlib_node_t *n = vlib_get_node (vm, rt->node_index); rt->cpu_index = vm_clone->cpu_index; + /* copy initial runtime_data from node */ + if (n->runtime_data_bytes > 0) + clib_memcpy (rt->runtime_data, n->runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); + else if (CLIB_DEBUG > 0) + memset (rt->runtime_data, 0xfe, + VLIB_NODE_RUNTIME_DATA_SIZE); + } nm_clone->processes = vec_dup (nm->processes); @@ -926,26 +950,51 @@ vlib_worker_thread_node_runtime_update (void) clib_mem_free (old_nodes_clone[j]); vec_free (old_nodes_clone); - vec_free (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]); + /* re-clone internal nodes */ + old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]; nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] = vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]); - /* clone input node runtime */ - old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]; + vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]) + { + vlib_node_t *n = vlib_get_node (vm, rt->node_index); + rt->cpu_index = vm_clone->cpu_index; + /* copy runtime_data, will be overwritten later for existing rt */ + clib_memcpy (rt->runtime_data, n->runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); + } + + for (j = 0; j < vec_len (old_rt); j++) + { + rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); + rt->state = old_rt[j].state; + clib_memcpy (rt->runtime_data, old_rt[j].runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); + } + vec_free (old_rt); + + /* re-clone input nodes */ + old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]; nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] = vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]); vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) { + vlib_node_t *n = vlib_get_node (vm, rt->node_index); rt->cpu_index = vm_clone->cpu_index; + /* copy runtime_data, will be overwritten later for existing rt */ + clib_memcpy (rt->runtime_data, n->runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); } for (j = 0; j < vec_len (old_rt); j++) { rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); rt->state = old_rt[j].state; + clib_memcpy (rt->runtime_data, old_rt[j].runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); } vec_free (old_rt); @@ -1342,6 +1391,7 @@ vlib_worker_thread_fn (void *arg) vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; vlib_thread_main_t *tm = vlib_get_thread_main (); vlib_main_t *vm = vlib_get_main (); + clib_error_t *e; ASSERT (vm->cpu_index == os_get_cpu_number ()); @@ -1349,6 +1399,11 @@ vlib_worker_thread_fn (void *arg) clib_time_init (&vm->clib_time); clib_mem_set_heap (w->thread_mheap); + e = vlib_call_init_exit_functions + (vm, vm->worker_init_function_registrations, 1 /* call_once */ ); + if (e) + clib_error_report (e); + /* Wait until the dpdk init sequence is complete */ while (tm->extern_thread_mgmt && tm->worker_thread_release == 0) vlib_worker_thread_barrier_check (); diff --git a/src/vnet/gre/node.c b/src/vnet/gre/node.c index 86f7a6ee..dd16db5e 100644 --- a/src/vnet/gre/node.c +++ b/src/vnet/gre/node.c @@ -448,7 +448,6 @@ gre_register_input_protocol (vlib_main_t * vm, { gre_main_t * em = &gre_main; gre_protocol_info_t * pi; - gre_input_runtime_t * rt; u16 * n; { @@ -464,10 +463,13 @@ gre_register_input_protocol (vlib_main_t * vm, node_index); /* Setup gre protocol -> next index sparse vector mapping. */ - rt = vlib_node_get_runtime_data (vm, gre_input_node.index); - n = sparse_vec_validate (rt->next_by_protocol, - clib_host_to_net_u16 (protocol)); - n[0] = pi->next_index; + foreach_vlib_main ({ + gre_input_runtime_t * rt; + rt = vlib_node_get_runtime_data (this_vlib_main, gre_input_node.index); + n = sparse_vec_validate (rt->next_by_protocol, + clib_host_to_net_u16 (protocol)); + n[0] = pi->next_index; + }); } static void @@ -529,3 +531,17 @@ static clib_error_t * gre_input_init (vlib_main_t * vm) } VLIB_INIT_FUNCTION (gre_input_init); + +static clib_error_t * gre_input_worker_init (vlib_main_t * vm) +{ + gre_input_runtime_t * rt; + + rt = vlib_node_get_runtime_data (vm, gre_input_node.index); + + rt->next_by_protocol = sparse_vec_new + (/* elt bytes */ sizeof (rt->next_by_protocol[0]), + /* bits in index */ BITS (((gre_header_t *) 0)->protocol)); + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (gre_input_worker_init); diff --git a/src/vnet/hdlc/node.c b/src/vnet/hdlc/node.c index 4fe0296a..57e04c85 100644 --- a/src/vnet/hdlc/node.c +++ b/src/vnet/hdlc/node.c @@ -285,18 +285,9 @@ VLIB_REGISTER_NODE (hdlc_input_node) = { .unformat_buffer = unformat_hdlc_header, }; -static clib_error_t * hdlc_input_init (vlib_main_t * vm) +static clib_error_t * hdlc_input_runtime_init (vlib_main_t * vm) { hdlc_input_runtime_t * rt; - - { - clib_error_t * error = vlib_call_init_function (vm, hdlc_init); - if (error) - clib_error_report (error); - } - - hdlc_setup_node (vm, hdlc_input_node.index); - rt = vlib_node_get_runtime_data (vm, hdlc_input_node.index); rt->next_by_protocol = sparse_vec_new @@ -313,7 +304,23 @@ static clib_error_t * hdlc_input_init (vlib_main_t * vm) return 0; } +static clib_error_t * hdlc_input_init (vlib_main_t * vm) +{ + + { + clib_error_t * error = vlib_call_init_function (vm, hdlc_init); + if (error) + clib_error_report (error); + } + + hdlc_setup_node (vm, hdlc_input_node.index); + hdlc_input_runtime_init (vm); + + return 0; +} + VLIB_INIT_FUNCTION (hdlc_input_init); +VLIB_WORKER_INIT_FUNCTION (hdlc_input_runtime_init); void hdlc_register_input_protocol (vlib_main_t * vm, diff --git a/src/vnet/l2/l2_input_classify.c b/src/vnet/l2/l2_input_classify.c index 497df192..485b9abd 100644 --- a/src/vnet/l2/l2_input_classify.c +++ b/src/vnet/l2/l2_input_classify.c @@ -505,6 +505,21 @@ l2_input_classify_init (vlib_main_t * vm) VLIB_INIT_FUNCTION (l2_input_classify_init); +clib_error_t * +l2_input_classify_worker_init (vlib_main_t * vm) +{ + l2_input_classify_main_t *cm = &l2_input_classify_main; + l2_input_classify_runtime_t *rt; + + rt = vlib_node_get_runtime_data (vm, l2_input_classify_node.index); + + rt->l2cm = cm; + rt->vcm = cm->vnet_classify_main; + + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (l2_input_classify_worker_init); /** Enable/disable l2 input classification on a specific interface. */ void diff --git a/src/vnet/l2/l2_output_classify.c b/src/vnet/l2/l2_output_classify.c index 832be1a1..c1bdaddc 100644 --- a/src/vnet/l2/l2_output_classify.c +++ b/src/vnet/l2/l2_output_classify.c @@ -505,6 +505,22 @@ l2_output_classify_init (vlib_main_t * vm) VLIB_INIT_FUNCTION (l2_output_classify_init); +clib_error_t * +l2_output_classify_worker_init (vlib_main_t * vm) +{ + l2_output_classify_main_t *cm = &l2_output_classify_main; + l2_output_classify_runtime_t *rt; + + rt = vlib_node_get_runtime_data (vm, l2_output_classify_node.index); + + rt->l2cm = cm; + rt->vcm = cm->vnet_classify_main; + + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (l2_output_classify_worker_init); + /** Enable/disable l2 input classification on a specific interface. */ void vnet_l2_output_classify_enable_disable (u32 sw_if_index, int enable_disable) diff --git a/src/vnet/l2tp/l2tp.c b/src/vnet/l2tp/l2tp.c index 2d323397..cb94d7e7 100644 --- a/src/vnet/l2tp/l2tp.c +++ b/src/vnet/l2tp/l2tp.c @@ -747,6 +747,16 @@ l2tp_init (vlib_main_t * vm) VLIB_INIT_FUNCTION (l2tp_init); +clib_error_t * +l2tp_worker_init (vlib_main_t * vm) +{ + l2tp_encap_init (vm); + + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (l2tp_worker_init); + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vnet/mpls/node.c b/src/vnet/mpls/node.c index 18100912..03bfaf56 100644 --- a/src/vnet/mpls/node.c +++ b/src/vnet/mpls/node.c @@ -301,3 +301,16 @@ static clib_error_t * mpls_input_init (vlib_main_t * vm) } VLIB_INIT_FUNCTION (mpls_input_init); + +static clib_error_t * mpls_input_worker_init (vlib_main_t * vm) +{ + mpls_input_runtime_t * rt; + rt = vlib_node_get_runtime_data (vm, mpls_input_node.index); + rt->last_label = (u32) ~0; + rt->last_inner_fib_index = 0; + rt->last_outer_fib_index = 0; + rt->mpls_main = &mpls_main; + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (mpls_input_worker_init); diff --git a/src/vnet/ppp/node.c b/src/vnet/ppp/node.c index 4f1f6a71..2f6e0c33 100644 --- a/src/vnet/ppp/node.c +++ b/src/vnet/ppp/node.c @@ -295,18 +295,10 @@ VLIB_REGISTER_NODE (ppp_input_node) = { /* *INDENT-ON* */ static clib_error_t * -ppp_input_init (vlib_main_t * vm) +ppp_input_runtime_init (vlib_main_t * vm) { ppp_input_runtime_t *rt; - { - clib_error_t *error = vlib_call_init_function (vm, ppp_init); - if (error) - clib_error_report (error); - } - - ppp_setup_node (vm, ppp_input_node.index); - rt = vlib_node_get_runtime_data (vm, ppp_input_node.index); rt->next_by_protocol = sparse_vec_new @@ -323,7 +315,24 @@ ppp_input_init (vlib_main_t * vm) return 0; } +static clib_error_t * +ppp_input_init (vlib_main_t * vm) +{ + + { + clib_error_t *error = vlib_call_init_function (vm, ppp_init); + if (error) + clib_error_report (error); + } + + ppp_setup_node (vm, ppp_input_node.index); + ppp_input_runtime_init (vm); + + return 0; +} + VLIB_INIT_FUNCTION (ppp_input_init); +VLIB_WORKER_INIT_FUNCTION (ppp_input_runtime_init); void ppp_register_input_protocol (vlib_main_t * vm, diff --git a/src/vnet/tcp/tcp_syn_filter4.c b/src/vnet/tcp/tcp_syn_filter4.c index c7605a30..9b2a8ac7 100644 --- a/src/vnet/tcp/tcp_syn_filter4.c +++ b/src/vnet/tcp/tcp_syn_filter4.c @@ -450,18 +450,21 @@ syn_filter_enable_disable (u32 sw_if_index, int enable_disable) if (enable_disable) { - vlib_main_t *vm = vlib_get_main (); syn_filter4_runtime_t *rt; - rt = vlib_node_get_runtime_data (vm, syn_filter4_node.index); - vec_validate (rt->syn_counts, 1023); - /* - * Given perfect disperson / optimal hashing results: - * Allow 128k (successful) syns/sec. 1024, buckets each of which - * absorb 128 syns before filtering. Reset table once a second. - * Reality bites, lets try resetting once every 100ms. - */ - rt->reset_interval = 0.1; /* reset interval in seconds */ + /* *INDENT-OFF* */ + foreach_vlib_main ({ + rt = vlib_node_get_runtime_data (this_vlib_main, syn_filter4_node.index); + vec_validate (rt->syn_counts, 1023); + /* + * Given perfect disperson / optimal hashing results: + * Allow 128k (successful) syns/sec. 1024, buckets each of which + * absorb 128 syns before filtering. Reset table once a second. + * Reality bites, lets try resetting once every 100ms. + */ + rt->reset_interval = 0.1; /* reset interval in seconds */ + }); + /* *INDENT-ON* */ } rv = vnet_feature_enable_disable ("ip4-local", "syn-filter-4", diff --git a/src/vnet/udp/udp_local.c b/src/vnet/udp/udp_local.c index 6b239f73..3a60b29b 100644 --- a/src/vnet/udp/udp_local.c +++ b/src/vnet/udp/udp_local.c @@ -520,11 +520,15 @@ udp_register_dst_port (vlib_main_t * vm, : udp6_input_node.index, node_index); /* Setup udp protocol -> next index sparse vector mapping. */ - rt = vlib_node_get_runtime_data - (vm, is_ip4 ? udp4_input_node.index : udp6_input_node.index); - n = sparse_vec_validate (rt->next_by_dst_port, - clib_host_to_net_u16 (dst_port)); - n[0] = pi->next_index; + /* *INDENT-OFF* */ + foreach_vlib_main({ + rt = vlib_node_get_runtime_data + (this_vlib_main, is_ip4 ? udp4_input_node.index : udp6_input_node.index); + n = sparse_vec_validate (rt->next_by_dst_port, + clib_host_to_net_u16 (dst_port)); + n[0] = pi->next_index; + }); + /* *INDENT-ON* */ } void @@ -541,11 +545,15 @@ udp_unregister_dst_port (vlib_main_t * vm, udp_dst_port_t dst_port, u8 is_ip4) return; /* Kill the mapping. Don't bother killing the pi, it may be back. */ - rt = vlib_node_get_runtime_data - (vm, is_ip4 ? udp4_input_node.index : udp6_input_node.index); - n = sparse_vec_validate (rt->next_by_dst_port, - clib_host_to_net_u16 (dst_port)); - n[0] = SPARSE_VEC_INVALID_INDEX; + /* *INDENT-OFF* */ + foreach_vlib_main({ + rt = vlib_node_get_runtime_data + (this_vlib_main, is_ip4 ? udp4_input_node.index : udp6_input_node.index); + n = sparse_vec_validate (rt->next_by_dst_port, + clib_host_to_net_u16 (dst_port)); + n[0] = SPARSE_VEC_INVALID_INDEX; + }); + /* *INDENT-ON* */ } void @@ -604,10 +612,27 @@ udp_setup_node (vlib_main_t * vm, u32 node_index) pn->unformat_edit = unformat_pg_udp_header; } +static void +udp_local_node_runtime_init (vlib_main_t * vm) +{ + udp_input_runtime_t *rt; + + rt = vlib_node_get_runtime_data (vm, udp4_input_node.index); + rt->next_by_dst_port = sparse_vec_new + ( /* elt bytes */ sizeof (rt->next_by_dst_port[0]), + /* bits in index */ BITS (((udp_header_t *) 0)->dst_port)); + rt->punt_unknown = 0; + + rt = vlib_node_get_runtime_data (vm, udp6_input_node.index); + rt->next_by_dst_port = sparse_vec_new + ( /* elt bytes */ sizeof (rt->next_by_dst_port[0]), + /* bits in index */ BITS (((udp_header_t *) 0)->dst_port)); + rt->punt_unknown = 0; +} + clib_error_t * udp_local_init (vlib_main_t * vm) { - udp_input_runtime_t *rt; udp_main_t *um = &udp_main; int i; @@ -628,27 +653,13 @@ udp_local_init (vlib_main_t * vm) udp_setup_node (vm, udp4_input_node.index); udp_setup_node (vm, udp6_input_node.index); - rt = vlib_node_get_runtime_data (vm, udp4_input_node.index); - - rt->next_by_dst_port = sparse_vec_new - ( /* elt bytes */ sizeof (rt->next_by_dst_port[0]), - /* bits in index */ BITS (((udp_header_t *) 0)->dst_port)); - - rt->punt_unknown = 0; + udp_local_node_runtime_init (vm); #define _(n,s) add_dst_port (um, UDP_DST_PORT_##s, #s, 1 /* is_ip4 */); foreach_udp4_dst_port #undef _ - rt = vlib_node_get_runtime_data (vm, udp6_input_node.index); - - rt->next_by_dst_port = sparse_vec_new - ( /* elt bytes */ sizeof (rt->next_by_dst_port[0]), - /* bits in index */ BITS (((udp_header_t *) 0)->dst_port)); - - rt->punt_unknown = 0; - #define _(n,s) add_dst_port (um, UDP_DST_PORT_##s, #s, 0 /* is_ip4 */); - foreach_udp6_dst_port + foreach_udp6_dst_port #undef _ ip4_register_protocol (IP_PROTOCOL_UDP, udp4_input_node.index); /* Note: ip6 differs from ip4, UDP is hotwired to ip6-udp-lookup */ @@ -657,6 +668,15 @@ udp_local_init (vlib_main_t * vm) VLIB_INIT_FUNCTION (udp_local_init); +clib_error_t * +udp_local_worker_init (vlib_main_t * vm) +{ + udp_local_node_runtime_init (vm); + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (udp_local_worker_init); + /* * fd.io coding-style-patch-verification: ON * -- cgit 1.2.3-korg