From 7cd468a3d7dee7d6c92f69a0bb7061ae208ec727 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 19 Dec 2016 23:05:39 +0100 Subject: Reorganize source tree to use single autotools instance Change-Id: I7b51f88292e057c6443b12224486f2d0c9f8ae23 Signed-off-by: Damjan Marion --- src/vnet/l2/l2_input_classify.c | 655 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 655 insertions(+) create mode 100644 src/vnet/l2/l2_input_classify.c (limited to 'src/vnet/l2/l2_input_classify.c') diff --git a/src/vnet/l2/l2_input_classify.c b/src/vnet/l2/l2_input_classify.c new file mode 100644 index 00000000..497df192 --- /dev/null +++ b/src/vnet/l2/l2_input_classify.c @@ -0,0 +1,655 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * l2_classify.c + */ + +#include +#include + +/** + * @file + * @brief L2 input classifier. + * + * @sa @ref vnet/vnet/classify/vnet_classify.c + * @sa @ref vnet/vnet/classify/vnet_classify.h + */ + +/** + * @brief l2_input_classifier packet trace record. + */ +typedef struct +{ + /** interface handle for the ith packet */ + u32 sw_if_index; + /** graph arc index selected for this packet */ + u32 next_index; + /** classifier table which provided the final result */ + u32 table_index; + /** offset in classifier heap of the corresponding session */ + u32 session_offset; +} l2_input_classify_trace_t; + +/** + * @brief vlib node runtime. + */ +typedef struct +{ + /** use-case independent main object pointer */ + vnet_classify_main_t *vcm; + /** l2 input classifier main object pointer */ + l2_input_classify_main_t *l2cm; +} l2_input_classify_runtime_t; + +/** Packet trace format function. */ +static u8 * +format_l2_input_classify_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + l2_input_classify_trace_t *t = va_arg (*args, l2_input_classify_trace_t *); + + s = format (s, "l2-classify: sw_if_index %d, table %d, offset %x, next %d", + t->sw_if_index, t->table_index, t->session_offset, + t->next_index); + return s; +} + +/** l2 input classifier main data structure. */ +l2_input_classify_main_t l2_input_classify_main; + +vlib_node_registration_t l2_input_classify_node; + +#define foreach_l2_input_classify_error \ +_(MISS, "Classify misses") \ +_(HIT, "Classify hits") \ +_(CHAIN_HIT, "Classify hits after chain walk") \ +_(DROP, "L2 Classify Drops") + +typedef enum +{ +#define _(sym,str) L2_INPUT_CLASSIFY_ERROR_##sym, + foreach_l2_input_classify_error +#undef _ + L2_INPUT_CLASSIFY_N_ERROR, +} l2_input_classify_error_t; + +static char *l2_input_classify_error_strings[] = { +#define _(sym,string) string, + foreach_l2_input_classify_error +#undef _ +}; + +/** + * @brief l2 input classifier node. + * @node l2-input-classify + * + * This is the l2 input classifier dispatch node + * + * @param vm vlib_main_t corresponding to the current thread. + * @param node vlib_node_runtime_t data for this node. + * @param frame vlib_frame_t whose contents should be dispatched. + * + * @par Graph mechanics: buffer metadata, next index usage + * + * @em Uses: + * - (l2_input_classify_runtime_t *) + * rt->classify_table_index_by_sw_if_index + * - Head of the per-interface, per-protocol classifier table chain + * for a specific interface. + * - @c ~0 => send pkts to the next feature in the L2 feature chain. + * - vnet_buffer(b)->sw_if_index[VLIB_RX] + * - Indicates the @c sw_if_index value of the interface that the + * packet was received on. + * - vnet_buffer(b0)->l2.feature_bitmap + * - Used to steer packets across l2 features enabled on the interface + * - (vnet_classify_entry_t) e0->next_index + * - Used to steer traffic when the classifier hits on a session + * - (vnet_classify_entry_t) e0->advance + * - Signed quantity applied via vlib_buffer_advance + * when the classifier hits on a session + * - (vnet_classify_table_t) t0->miss_next_index + * - Used to steer traffic when the classifier misses + * + * @em Sets: + * - vnet_buffer (b0)->l2_classify.table_index + * - Classifier table index of the first classifier table in + * the classifier table chain + * - vnet_buffer (b0)->l2_classify.hash + * - Bounded-index extensible hash corresponding to the + * masked fields in the current packet + * - vnet_buffer (b0)->l2.feature_bitmap + * - Used to steer packets across l2 features enabled on the interface + * - vnet_buffer (b0)->l2_classify.opaque_index + * - Copied from the classifier session object upon classifier hit + * + * @em Counters: + * - L2_INPUT_CLASSIFY_ERROR_MISS Classifier misses + * - L2_INPUT_CLASSIFY_ERROR_HIT Classifier hits + * - L2_INPUT_CLASSIFY_ERROR_CHAIN_HIT + * Classifier hits in other than the first table + */ + +static uword +l2_input_classify_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + u32 n_left_from, *from, *to_next; + l2_input_classify_next_t next_index; + l2_input_classify_main_t *cm = &l2_input_classify_main; + vnet_classify_main_t *vcm = cm->vnet_classify_main; + l2_input_classify_runtime_t *rt = + (l2_input_classify_runtime_t *) node->runtime_data; + u32 feature_bitmap; + u32 hits = 0; + u32 misses = 0; + u32 chain_hits = 0; + f64 now; + u32 n_next_nodes; + + n_next_nodes = node->n_next_nodes; + + now = vlib_time_now (vm); + + n_left_from = frame->n_vectors; + from = vlib_frame_vector_args (frame); + + /* First pass: compute hash */ + + while (n_left_from > 2) + { + vlib_buffer_t *b0, *b1; + u32 bi0, bi1; + ethernet_header_t *h0, *h1; + u32 sw_if_index0, sw_if_index1; + u16 type0, type1; + int type_index0, type_index1; + vnet_classify_table_t *t0, *t1; + u32 table_index0, table_index1; + u64 hash0, hash1; + + + /* prefetch next iteration */ + { + vlib_buffer_t *p1, *p2; + + p1 = vlib_get_buffer (vm, from[1]); + p2 = vlib_get_buffer (vm, from[2]); + + vlib_prefetch_buffer_header (p1, STORE); + CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE); + vlib_prefetch_buffer_header (p2, STORE); + CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); + } + + bi0 = from[0]; + b0 = vlib_get_buffer (vm, bi0); + h0 = vlib_buffer_get_current (b0); + + bi1 = from[1]; + b1 = vlib_get_buffer (vm, bi1); + h1 = vlib_buffer_get_current (b1); + + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + vnet_buffer (b0)->l2_classify.table_index = ~0; + + sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX]; + vnet_buffer (b1)->l2_classify.table_index = ~0; + + /* Select classifier table based on ethertype */ + type0 = clib_net_to_host_u16 (h0->type); + type1 = clib_net_to_host_u16 (h1->type); + + type_index0 = (type0 == ETHERNET_TYPE_IP4) + ? L2_INPUT_CLASSIFY_TABLE_IP4 : L2_INPUT_CLASSIFY_TABLE_OTHER; + type_index0 = (type0 == ETHERNET_TYPE_IP6) + ? L2_INPUT_CLASSIFY_TABLE_IP6 : type_index0; + + type_index1 = (type1 == ETHERNET_TYPE_IP4) + ? L2_INPUT_CLASSIFY_TABLE_IP4 : L2_INPUT_CLASSIFY_TABLE_OTHER; + type_index1 = (type1 == ETHERNET_TYPE_IP6) + ? L2_INPUT_CLASSIFY_TABLE_IP6 : type_index1; + + vnet_buffer (b0)->l2_classify.table_index = + table_index0 = + rt->l2cm->classify_table_index_by_sw_if_index + [type_index0][sw_if_index0]; + + if (table_index0 != ~0) + { + t0 = pool_elt_at_index (vcm->tables, table_index0); + + vnet_buffer (b0)->l2_classify.hash = hash0 = + vnet_classify_hash_packet (t0, (u8 *) h0); + vnet_classify_prefetch_bucket (t0, hash0); + } + + vnet_buffer (b1)->l2_classify.table_index = + table_index1 = + rt->l2cm->classify_table_index_by_sw_if_index + [type_index1][sw_if_index1]; + + if (table_index1 != ~0) + { + t1 = pool_elt_at_index (vcm->tables, table_index1); + + vnet_buffer (b1)->l2_classify.hash = hash1 = + vnet_classify_hash_packet (t1, (u8 *) h1); + vnet_classify_prefetch_bucket (t1, hash1); + } + + from += 2; + n_left_from -= 2; + } + + while (n_left_from > 0) + { + vlib_buffer_t *b0; + u32 bi0; + ethernet_header_t *h0; + u32 sw_if_index0; + u16 type0; + u32 type_index0; + vnet_classify_table_t *t0; + u32 table_index0; + u64 hash0; + + bi0 = from[0]; + b0 = vlib_get_buffer (vm, bi0); + h0 = vlib_buffer_get_current (b0); + + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + vnet_buffer (b0)->l2_classify.table_index = ~0; + + /* Select classifier table based on ethertype */ + type0 = clib_net_to_host_u16 (h0->type); + + type_index0 = (type0 == ETHERNET_TYPE_IP4) + ? L2_INPUT_CLASSIFY_TABLE_IP4 : L2_INPUT_CLASSIFY_TABLE_OTHER; + type_index0 = (type0 == ETHERNET_TYPE_IP6) + ? L2_INPUT_CLASSIFY_TABLE_IP6 : type_index0; + + vnet_buffer (b0)->l2_classify.table_index = + table_index0 = rt->l2cm->classify_table_index_by_sw_if_index + [type_index0][sw_if_index0]; + + if (table_index0 != ~0) + { + t0 = pool_elt_at_index (vcm->tables, table_index0); + + vnet_buffer (b0)->l2_classify.hash = hash0 = + vnet_classify_hash_packet (t0, (u8 *) h0); + vnet_classify_prefetch_bucket (t0, hash0); + } + from++; + n_left_from--; + } + + next_index = node->cached_next_index; + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + /* Not enough load/store slots to dual loop... */ + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 next0 = ~0; /* next l2 input feature, please... */ + ethernet_header_t *h0; + u32 table_index0; + u64 hash0; + vnet_classify_table_t *t0; + vnet_classify_entry_t *e0; + + if (PREDICT_TRUE (n_left_from > 2)) + { + vlib_buffer_t *p2 = vlib_get_buffer (vm, from[2]); + u64 phash2; + u32 table_index2; + vnet_classify_table_t *tp2; + + /* + * Prefetch table entry two ahead. Buffer / data + * were prefetched above... + */ + table_index2 = vnet_buffer (p2)->l2_classify.table_index; + + if (PREDICT_TRUE (table_index2 != ~0)) + { + tp2 = pool_elt_at_index (vcm->tables, table_index2); + phash2 = vnet_buffer (p2)->l2_classify.hash; + vnet_classify_prefetch_entry (tp2, phash2); + } + } + + /* speculatively enqueue b0 to the current next frame */ + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + h0 = vlib_buffer_get_current (b0); + table_index0 = vnet_buffer (b0)->l2_classify.table_index; + e0 = 0; + vnet_buffer (b0)->l2_classify.opaque_index = ~0; + + /* Remove ourself from the feature bitmap */ + feature_bitmap = vnet_buffer (b0)->l2.feature_bitmap + & ~L2INPUT_FEAT_INPUT_CLASSIFY; + + /* save for next feature graph nodes */ + vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap; + + if (PREDICT_TRUE (table_index0 != ~0)) + { + hash0 = vnet_buffer (b0)->l2_classify.hash; + t0 = pool_elt_at_index (vcm->tables, table_index0); + + e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now); + if (e0) + { + vnet_buffer (b0)->l2_classify.opaque_index + = e0->opaque_index; + vlib_buffer_advance (b0, e0->advance); + next0 = (e0->next_index < n_next_nodes) ? + e0->next_index : next0; + hits++; + } + else + { + while (1) + { + if (t0->next_table_index != ~0) + t0 = pool_elt_at_index (vcm->tables, + t0->next_table_index); + else + { + next0 = (t0->miss_next_index < n_next_nodes) ? + t0->miss_next_index : next0; + misses++; + break; + } + + hash0 = vnet_classify_hash_packet (t0, (u8 *) h0); + e0 = + vnet_classify_find_entry (t0, (u8 *) h0, hash0, now); + if (e0) + { + vnet_buffer (b0)->l2_classify.opaque_index + = e0->opaque_index; + vlib_buffer_advance (b0, e0->advance); + next0 = (e0->next_index < n_next_nodes) ? + e0->next_index : next0; + hits++; + chain_hits++; + break; + } + } + } + } + + if (PREDICT_FALSE (next0 == 0)) + b0->error = node->errors[L2_INPUT_CLASSIFY_ERROR_DROP]; + + if (PREDICT_TRUE (next0 == ~0)) + { + // Determine the next node + next0 = + feat_bitmap_get_next_node_index (cm->feat_next_node_index, + feature_bitmap); + } + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + l2_input_classify_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + t->table_index = table_index0; + t->next_index = next0; + t->session_offset = e0 ? vnet_classify_get_offset (t0, e0) : 0; + } + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + vlib_node_increment_counter (vm, node->node_index, + L2_INPUT_CLASSIFY_ERROR_MISS, misses); + vlib_node_increment_counter (vm, node->node_index, + L2_INPUT_CLASSIFY_ERROR_HIT, hits); + vlib_node_increment_counter (vm, node->node_index, + L2_INPUT_CLASSIFY_ERROR_CHAIN_HIT, chain_hits); + return frame->n_vectors; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (l2_input_classify_node) = { + .function = l2_input_classify_node_fn, + .name = "l2-input-classify", + .vector_size = sizeof (u32), + .format_trace = format_l2_input_classify_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(l2_input_classify_error_strings), + .error_strings = l2_input_classify_error_strings, + + .runtime_data_bytes = sizeof (l2_input_classify_runtime_t), + + .n_next_nodes = L2_INPUT_CLASSIFY_N_NEXT, + + /* edit / add dispositions here */ + .next_nodes = { + [L2_INPUT_CLASSIFY_NEXT_DROP] = "error-drop", + [L2_INPUT_CLASSIFY_NEXT_ETHERNET_INPUT] = "ethernet-input-not-l2", + [L2_INPUT_CLASSIFY_NEXT_IP4_INPUT] = "ip4-input", + [L2_INPUT_CLASSIFY_NEXT_IP6_INPUT] = "ip6-input", + [L2_INPUT_CLASSIFY_NEXT_LI] = "li-hit", + }, +}; +/* *INDENT-ON* */ + +VLIB_NODE_FUNCTION_MULTIARCH (l2_input_classify_node, + l2_input_classify_node_fn); + +/** l2 input classsifier feature initialization. */ +clib_error_t * +l2_input_classify_init (vlib_main_t * vm) +{ + l2_input_classify_main_t *cm = &l2_input_classify_main; + l2_input_classify_runtime_t *rt; + + rt = vlib_node_get_runtime_data (vm, l2_input_classify_node.index); + + cm->vlib_main = vm; + cm->vnet_main = vnet_get_main (); + cm->vnet_classify_main = &vnet_classify_main; + + /* Initialize the feature next-node indexes */ + feat_bitmap_init_next_nodes (vm, + l2_input_classify_node.index, + L2INPUT_N_FEAT, + l2input_get_feat_names (), + cm->feat_next_node_index); + rt->l2cm = cm; + rt->vcm = cm->vnet_classify_main; + + return 0; +} + +VLIB_INIT_FUNCTION (l2_input_classify_init); + + +/** Enable/disable l2 input classification on a specific interface. */ +void +vnet_l2_input_classify_enable_disable (u32 sw_if_index, int enable_disable) +{ + l2input_intf_bitmap_enable (sw_if_index, L2INPUT_FEAT_INPUT_CLASSIFY, + (u32) enable_disable); +} + +/** @brief Set l2 per-protocol, per-interface input classification tables. + * + * @param sw_if_index interface handle + * @param ip4_table_index ip4 classification table index, or ~0 + * @param ip6_table_index ip6 classification table index, or ~0 + * @param other_table_index non-ip4, non-ip6 classification table index, + * or ~0 + * @returns 0 on success, VNET_API_ERROR_NO_SUCH_TABLE, TABLE2, TABLE3 + * if the indicated (non-~0) table does not exist. + */ + +int +vnet_l2_input_classify_set_tables (u32 sw_if_index, + u32 ip4_table_index, + u32 ip6_table_index, u32 other_table_index) +{ + l2_input_classify_main_t *cm = &l2_input_classify_main; + vnet_classify_main_t *vcm = cm->vnet_classify_main; + + /* Assume that we've validated sw_if_index in the API layer */ + + if (ip4_table_index != ~0 && + pool_is_free_index (vcm->tables, ip4_table_index)) + return VNET_API_ERROR_NO_SUCH_TABLE; + + if (ip6_table_index != ~0 && + pool_is_free_index (vcm->tables, ip6_table_index)) + return VNET_API_ERROR_NO_SUCH_TABLE2; + + if (other_table_index != ~0 && + pool_is_free_index (vcm->tables, other_table_index)) + return VNET_API_ERROR_NO_SUCH_TABLE3; + + vec_validate + (cm->classify_table_index_by_sw_if_index[L2_INPUT_CLASSIFY_TABLE_IP4], + sw_if_index); + + vec_validate + (cm->classify_table_index_by_sw_if_index[L2_INPUT_CLASSIFY_TABLE_IP6], + sw_if_index); + + vec_validate + (cm->classify_table_index_by_sw_if_index[L2_INPUT_CLASSIFY_TABLE_OTHER], + sw_if_index); + + cm->classify_table_index_by_sw_if_index[L2_INPUT_CLASSIFY_TABLE_IP4] + [sw_if_index] = ip4_table_index; + + cm->classify_table_index_by_sw_if_index[L2_INPUT_CLASSIFY_TABLE_IP6] + [sw_if_index] = ip6_table_index; + + cm->classify_table_index_by_sw_if_index[L2_INPUT_CLASSIFY_TABLE_OTHER] + [sw_if_index] = other_table_index; + + return 0; +} + +static clib_error_t * +int_l2_input_classify_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + vnet_main_t *vnm = vnet_get_main (); + u32 sw_if_index = ~0; + u32 ip4_table_index = ~0; + u32 ip6_table_index = ~0; + u32 other_table_index = ~0; + int rv; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "intfc %U", unformat_vnet_sw_interface, + vnm, &sw_if_index)) + ; + else if (unformat (input, "ip4-table %d", &ip4_table_index)) + ; + else if (unformat (input, "ip6-table %d", &ip6_table_index)) + ; + else if (unformat (input, "other-table %d", &other_table_index)) + ; + else + break; + } + + if (sw_if_index == ~0) + return clib_error_return (0, "interface must be specified"); + + + if (ip4_table_index == ~0 && ip6_table_index == ~0 + && other_table_index == ~0) + { + vlib_cli_output (vm, "L2 classification disabled"); + vnet_l2_input_classify_enable_disable (sw_if_index, 0 /* enable */ ); + return 0; + } + + rv = vnet_l2_input_classify_set_tables (sw_if_index, ip4_table_index, + ip6_table_index, other_table_index); + switch (rv) + { + case 0: + vnet_l2_input_classify_enable_disable (sw_if_index, 1 /* enable */ ); + break; + + default: + return clib_error_return (0, "vnet_l2_input_classify_set_tables: %d", + rv); + break; + } + + return 0; +} + +/*? + * Configure l2 input classification. + * + * @cliexpar + * @cliexstart{set interface l2 input classify intfc [ip4-table ] [ip6-table ] [other-table ]} + * @cliexend + * @todo This is incomplete. This needs a detailed description and a + * practical example. + ?*/ +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (int_l2_input_classify_cli, static) = { + .path = "set interface l2 input classify", + .short_help = + "set interface l2 input classify intfc [ip4-table ]\n" + " [ip6-table ] [other-table ]", + .function = int_l2_input_classify_command_fn, +}; +/* *INDENT-ON* */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ -- cgit 1.2.3-korg From e9f929b52ddb741ec1e4cb2d92c6be1e798933a0 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Thu, 16 Mar 2017 11:32:09 +0100 Subject: vlib: make runtime_data thread-local Change-Id: I4aa3e7e42fb81211de1aed07dc7befee87a1e18b Signed-off-by: Damjan Marion --- src/vlib/init.h | 1 + src/vlib/main.h | 1 + src/vlib/node.c | 4 +- src/vlib/node.h | 81 +++++++++++++++++++++------------------- src/vlib/threads.c | 61 ++++++++++++++++++++++++++++-- src/vnet/gre/node.c | 26 ++++++++++--- src/vnet/hdlc/node.c | 27 +++++++++----- src/vnet/l2/l2_input_classify.c | 15 ++++++++ src/vnet/l2/l2_output_classify.c | 16 ++++++++ src/vnet/l2tp/l2tp.c | 10 +++++ src/vnet/mpls/node.c | 13 +++++++ src/vnet/ppp/node.c | 27 +++++++++----- src/vnet/tcp/tcp_syn_filter4.c | 23 +++++++----- src/vnet/udp/udp_local.c | 74 ++++++++++++++++++++++-------------- 14 files changed, 273 insertions(+), 106 deletions(-) (limited to 'src/vnet/l2/l2_input_classify.c') diff --git a/src/vlib/init.h b/src/vlib/init.h index 4fa5b304..12db3f90 100644 --- a/src/vlib/init.h +++ b/src/vlib/init.h @@ -109,6 +109,7 @@ static void __vlib_add_##tag##_function_##x (void) \ } #define VLIB_INIT_FUNCTION(x) VLIB_DECLARE_INIT_FUNCTION(x,init) +#define VLIB_WORKER_INIT_FUNCTION(x) VLIB_DECLARE_INIT_FUNCTION(x,worker_init) #define VLIB_MAIN_LOOP_ENTER_FUNCTION(x) \ VLIB_DECLARE_INIT_FUNCTION(x,main_loop_enter) diff --git a/src/vlib/main.h b/src/vlib/main.h index a6d50b39..98bc823d 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -162,6 +162,7 @@ typedef struct vlib_main_t /* List of init functions to call, setup by constructors */ _vlib_init_function_list_elt_t *init_function_registrations; + _vlib_init_function_list_elt_t *worker_init_function_registrations; _vlib_init_function_list_elt_t *main_loop_enter_function_registrations; _vlib_init_function_list_elt_t *main_loop_exit_function_registrations; _vlib_init_function_list_elt_t *api_init_function_registrations; diff --git a/src/vlib/node.c b/src/vlib/node.c index c419a13a..dc0a4de5 100644 --- a/src/vlib/node.c +++ b/src/vlib/node.c @@ -434,9 +434,7 @@ register_node (vlib_main_t * vm, vlib_node_registration_t * r) rt->errors[i] = vlib_error_set (n->index, i); STATIC_ASSERT_SIZEOF (vlib_node_runtime_t, 128); - ASSERT (vec_len (n->runtime_data) <= - sizeof (vlib_node_runtime_t) - - STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data)); + ASSERT (vec_len (n->runtime_data) <= VLIB_NODE_RUNTIME_DATA_SIZE); if (vec_len (n->runtime_data) > 0) clib_memcpy (rt->runtime_data, n->runtime_data, diff --git a/src/vlib/node.h b/src/vlib/node.h index b624e9d6..2a532cc3 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -411,65 +411,68 @@ typedef struct typedef struct vlib_node_runtime_t { - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - /* Node function to call. */ - vlib_node_function_t *function; + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); /**< cacheline mark */ - /* Vector of errors for this node. */ - vlib_error_t *errors; + vlib_node_function_t *function; /**< Node function to call. */ - /* Number of clock cycles. */ - u32 clocks_since_last_overflow; + vlib_error_t *errors; /**< Vector of errors for this node. */ - /* Maximum clock cycle for an invocation. */ - u32 max_clock; + u32 clocks_since_last_overflow; /**< Number of clock cycles. */ - /* Number of vectors in the recorded max_clock. */ - u32 max_clock_n; + u32 max_clock; /**< Maximum clock cycle for an + invocation. */ - /* Number of calls. */ - u32 calls_since_last_overflow; + u32 max_clock_n; /**< Number of vectors in the recorded + max_clock. */ - /* Number of vector elements processed by this node. */ - u32 vectors_since_last_overflow; + u32 calls_since_last_overflow; /**< Number of calls. */ - /* Start of next frames for this node. */ - u32 next_frame_index; + u32 vectors_since_last_overflow; /**< Number of vector elements + processed by this node. */ - /* Node index. */ - u32 node_index; + u32 next_frame_index; /**< Start of next frames for this + node. */ - /* For input nodes: decremented on each main loop interation until it reaches zero - and function is called. Allows some input nodes to be called - more than others. */ - u32 input_main_loops_per_call; + u32 node_index; /**< Node index. */ - /* Saved main loop counter of last dispatch of this node. */ - u32 main_loop_count_last_dispatch; + u32 input_main_loops_per_call; /**< For input nodes: decremented + on each main loop interation until + it reaches zero and function is + called. Allows some input nodes to + be called more than others. */ + + u32 main_loop_count_last_dispatch; /**< Saved main loop counter of last + dispatch of this node. */ u32 main_loop_vector_stats[2]; - /* Copy of main node flags. */ - u16 flags; + u16 flags; /**< Copy of main node flags. */ - /* Input node state. */ - u16 state; + u16 state; /**< Input node state. */ u16 n_next_nodes; - /* Next frame index that vector arguments were last enqueued to - last time this node ran. Set to zero before first run - of this node. */ - u16 cached_next_index; - - /* CPU this node runs on */ - u16 cpu_index; - - /* Function dependent node-runtime. */ - u8 runtime_data[0]; + u16 cached_next_index; /**< Next frame index that vector + arguments were last enqueued to + last time this node ran. Set to + zero before first run of this + node. */ + + u16 cpu_index; /**< CPU this node runs on */ + + u8 runtime_data[0]; /**< Function dependent + node-runtime data. This data is + thread local, and it is not + cloned from main thread. It needs + to be initialized for each thread + before it is used unless + runtime_data template exists in + vlib_node_t. */ } vlib_node_runtime_t; +#define VLIB_NODE_RUNTIME_DATA_SIZE (sizeof (vlib_node_runtime_t) - STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data)) + typedef struct { /* Number of allocated frames for this scalar/vector size. */ diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 07dbff33..3756c3fa 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -633,6 +633,8 @@ start_workers (vlib_main_t * vm) vm_clone->cpu_index = worker_thread_index; vm_clone->heap_base = w->thread_mheap; vm_clone->mbuf_alloc_list = 0; + vm_clone->init_functions_called = + hash_create (0, /* value bytes */ 0); memset (&vm_clone->random_buffer, 0, sizeof (vm_clone->random_buffer)); @@ -674,11 +676,33 @@ start_workers (vlib_main_t * vm) } nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] = vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]); + vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) + { + vlib_node_t *n = vlib_get_node (vm, rt->node_index); + rt->cpu_index = vm_clone->cpu_index; + /* copy initial runtime_data from node */ + if (n->runtime_data_bytes > 0) + clib_memcpy (rt->runtime_data, n->runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); + else if (CLIB_DEBUG > 0) + memset (rt->runtime_data, 0xfe, + VLIB_NODE_RUNTIME_DATA_SIZE); + } nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] = vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]); vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) + { + vlib_node_t *n = vlib_get_node (vm, rt->node_index); rt->cpu_index = vm_clone->cpu_index; + /* copy initial runtime_data from node */ + if (n->runtime_data_bytes > 0) + clib_memcpy (rt->runtime_data, n->runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); + else if (CLIB_DEBUG > 0) + memset (rt->runtime_data, 0xfe, + VLIB_NODE_RUNTIME_DATA_SIZE); + } nm_clone->processes = vec_dup (nm->processes); @@ -926,26 +950,51 @@ vlib_worker_thread_node_runtime_update (void) clib_mem_free (old_nodes_clone[j]); vec_free (old_nodes_clone); - vec_free (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]); + /* re-clone internal nodes */ + old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]; nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] = vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]); - /* clone input node runtime */ - old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]; + vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]) + { + vlib_node_t *n = vlib_get_node (vm, rt->node_index); + rt->cpu_index = vm_clone->cpu_index; + /* copy runtime_data, will be overwritten later for existing rt */ + clib_memcpy (rt->runtime_data, n->runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); + } + + for (j = 0; j < vec_len (old_rt); j++) + { + rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); + rt->state = old_rt[j].state; + clib_memcpy (rt->runtime_data, old_rt[j].runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); + } + vec_free (old_rt); + + /* re-clone input nodes */ + old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]; nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] = vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]); vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) { + vlib_node_t *n = vlib_get_node (vm, rt->node_index); rt->cpu_index = vm_clone->cpu_index; + /* copy runtime_data, will be overwritten later for existing rt */ + clib_memcpy (rt->runtime_data, n->runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); } for (j = 0; j < vec_len (old_rt); j++) { rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); rt->state = old_rt[j].state; + clib_memcpy (rt->runtime_data, old_rt[j].runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); } vec_free (old_rt); @@ -1342,6 +1391,7 @@ vlib_worker_thread_fn (void *arg) vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; vlib_thread_main_t *tm = vlib_get_thread_main (); vlib_main_t *vm = vlib_get_main (); + clib_error_t *e; ASSERT (vm->cpu_index == os_get_cpu_number ()); @@ -1349,6 +1399,11 @@ vlib_worker_thread_fn (void *arg) clib_time_init (&vm->clib_time); clib_mem_set_heap (w->thread_mheap); + e = vlib_call_init_exit_functions + (vm, vm->worker_init_function_registrations, 1 /* call_once */ ); + if (e) + clib_error_report (e); + /* Wait until the dpdk init sequence is complete */ while (tm->extern_thread_mgmt && tm->worker_thread_release == 0) vlib_worker_thread_barrier_check (); diff --git a/src/vnet/gre/node.c b/src/vnet/gre/node.c index 86f7a6ee..dd16db5e 100644 --- a/src/vnet/gre/node.c +++ b/src/vnet/gre/node.c @@ -448,7 +448,6 @@ gre_register_input_protocol (vlib_main_t * vm, { gre_main_t * em = &gre_main; gre_protocol_info_t * pi; - gre_input_runtime_t * rt; u16 * n; { @@ -464,10 +463,13 @@ gre_register_input_protocol (vlib_main_t * vm, node_index); /* Setup gre protocol -> next index sparse vector mapping. */ - rt = vlib_node_get_runtime_data (vm, gre_input_node.index); - n = sparse_vec_validate (rt->next_by_protocol, - clib_host_to_net_u16 (protocol)); - n[0] = pi->next_index; + foreach_vlib_main ({ + gre_input_runtime_t * rt; + rt = vlib_node_get_runtime_data (this_vlib_main, gre_input_node.index); + n = sparse_vec_validate (rt->next_by_protocol, + clib_host_to_net_u16 (protocol)); + n[0] = pi->next_index; + }); } static void @@ -529,3 +531,17 @@ static clib_error_t * gre_input_init (vlib_main_t * vm) } VLIB_INIT_FUNCTION (gre_input_init); + +static clib_error_t * gre_input_worker_init (vlib_main_t * vm) +{ + gre_input_runtime_t * rt; + + rt = vlib_node_get_runtime_data (vm, gre_input_node.index); + + rt->next_by_protocol = sparse_vec_new + (/* elt bytes */ sizeof (rt->next_by_protocol[0]), + /* bits in index */ BITS (((gre_header_t *) 0)->protocol)); + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (gre_input_worker_init); diff --git a/src/vnet/hdlc/node.c b/src/vnet/hdlc/node.c index 4fe0296a..57e04c85 100644 --- a/src/vnet/hdlc/node.c +++ b/src/vnet/hdlc/node.c @@ -285,18 +285,9 @@ VLIB_REGISTER_NODE (hdlc_input_node) = { .unformat_buffer = unformat_hdlc_header, }; -static clib_error_t * hdlc_input_init (vlib_main_t * vm) +static clib_error_t * hdlc_input_runtime_init (vlib_main_t * vm) { hdlc_input_runtime_t * rt; - - { - clib_error_t * error = vlib_call_init_function (vm, hdlc_init); - if (error) - clib_error_report (error); - } - - hdlc_setup_node (vm, hdlc_input_node.index); - rt = vlib_node_get_runtime_data (vm, hdlc_input_node.index); rt->next_by_protocol = sparse_vec_new @@ -313,7 +304,23 @@ static clib_error_t * hdlc_input_init (vlib_main_t * vm) return 0; } +static clib_error_t * hdlc_input_init (vlib_main_t * vm) +{ + + { + clib_error_t * error = vlib_call_init_function (vm, hdlc_init); + if (error) + clib_error_report (error); + } + + hdlc_setup_node (vm, hdlc_input_node.index); + hdlc_input_runtime_init (vm); + + return 0; +} + VLIB_INIT_FUNCTION (hdlc_input_init); +VLIB_WORKER_INIT_FUNCTION (hdlc_input_runtime_init); void hdlc_register_input_protocol (vlib_main_t * vm, diff --git a/src/vnet/l2/l2_input_classify.c b/src/vnet/l2/l2_input_classify.c index 497df192..485b9abd 100644 --- a/src/vnet/l2/l2_input_classify.c +++ b/src/vnet/l2/l2_input_classify.c @@ -505,6 +505,21 @@ l2_input_classify_init (vlib_main_t * vm) VLIB_INIT_FUNCTION (l2_input_classify_init); +clib_error_t * +l2_input_classify_worker_init (vlib_main_t * vm) +{ + l2_input_classify_main_t *cm = &l2_input_classify_main; + l2_input_classify_runtime_t *rt; + + rt = vlib_node_get_runtime_data (vm, l2_input_classify_node.index); + + rt->l2cm = cm; + rt->vcm = cm->vnet_classify_main; + + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (l2_input_classify_worker_init); /** Enable/disable l2 input classification on a specific interface. */ void diff --git a/src/vnet/l2/l2_output_classify.c b/src/vnet/l2/l2_output_classify.c index 832be1a1..c1bdaddc 100644 --- a/src/vnet/l2/l2_output_classify.c +++ b/src/vnet/l2/l2_output_classify.c @@ -505,6 +505,22 @@ l2_output_classify_init (vlib_main_t * vm) VLIB_INIT_FUNCTION (l2_output_classify_init); +clib_error_t * +l2_output_classify_worker_init (vlib_main_t * vm) +{ + l2_output_classify_main_t *cm = &l2_output_classify_main; + l2_output_classify_runtime_t *rt; + + rt = vlib_node_get_runtime_data (vm, l2_output_classify_node.index); + + rt->l2cm = cm; + rt->vcm = cm->vnet_classify_main; + + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (l2_output_classify_worker_init); + /** Enable/disable l2 input classification on a specific interface. */ void vnet_l2_output_classify_enable_disable (u32 sw_if_index, int enable_disable) diff --git a/src/vnet/l2tp/l2tp.c b/src/vnet/l2tp/l2tp.c index 2d323397..cb94d7e7 100644 --- a/src/vnet/l2tp/l2tp.c +++ b/src/vnet/l2tp/l2tp.c @@ -747,6 +747,16 @@ l2tp_init (vlib_main_t * vm) VLIB_INIT_FUNCTION (l2tp_init); +clib_error_t * +l2tp_worker_init (vlib_main_t * vm) +{ + l2tp_encap_init (vm); + + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (l2tp_worker_init); + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vnet/mpls/node.c b/src/vnet/mpls/node.c index 18100912..03bfaf56 100644 --- a/src/vnet/mpls/node.c +++ b/src/vnet/mpls/node.c @@ -301,3 +301,16 @@ static clib_error_t * mpls_input_init (vlib_main_t * vm) } VLIB_INIT_FUNCTION (mpls_input_init); + +static clib_error_t * mpls_input_worker_init (vlib_main_t * vm) +{ + mpls_input_runtime_t * rt; + rt = vlib_node_get_runtime_data (vm, mpls_input_node.index); + rt->last_label = (u32) ~0; + rt->last_inner_fib_index = 0; + rt->last_outer_fib_index = 0; + rt->mpls_main = &mpls_main; + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (mpls_input_worker_init); diff --git a/src/vnet/ppp/node.c b/src/vnet/ppp/node.c index 4f1f6a71..2f6e0c33 100644 --- a/src/vnet/ppp/node.c +++ b/src/vnet/ppp/node.c @@ -295,18 +295,10 @@ VLIB_REGISTER_NODE (ppp_input_node) = { /* *INDENT-ON* */ static clib_error_t * -ppp_input_init (vlib_main_t * vm) +ppp_input_runtime_init (vlib_main_t * vm) { ppp_input_runtime_t *rt; - { - clib_error_t *error = vlib_call_init_function (vm, ppp_init); - if (error) - clib_error_report (error); - } - - ppp_setup_node (vm, ppp_input_node.index); - rt = vlib_node_get_runtime_data (vm, ppp_input_node.index); rt->next_by_protocol = sparse_vec_new @@ -323,7 +315,24 @@ ppp_input_init (vlib_main_t * vm) return 0; } +static clib_error_t * +ppp_input_init (vlib_main_t * vm) +{ + + { + clib_error_t *error = vlib_call_init_function (vm, ppp_init); + if (error) + clib_error_report (error); + } + + ppp_setup_node (vm, ppp_input_node.index); + ppp_input_runtime_init (vm); + + return 0; +} + VLIB_INIT_FUNCTION (ppp_input_init); +VLIB_WORKER_INIT_FUNCTION (ppp_input_runtime_init); void ppp_register_input_protocol (vlib_main_t * vm, diff --git a/src/vnet/tcp/tcp_syn_filter4.c b/src/vnet/tcp/tcp_syn_filter4.c index c7605a30..9b2a8ac7 100644 --- a/src/vnet/tcp/tcp_syn_filter4.c +++ b/src/vnet/tcp/tcp_syn_filter4.c @@ -450,18 +450,21 @@ syn_filter_enable_disable (u32 sw_if_index, int enable_disable) if (enable_disable) { - vlib_main_t *vm = vlib_get_main (); syn_filter4_runtime_t *rt; - rt = vlib_node_get_runtime_data (vm, syn_filter4_node.index); - vec_validate (rt->syn_counts, 1023); - /* - * Given perfect disperson / optimal hashing results: - * Allow 128k (successful) syns/sec. 1024, buckets each of which - * absorb 128 syns before filtering. Reset table once a second. - * Reality bites, lets try resetting once every 100ms. - */ - rt->reset_interval = 0.1; /* reset interval in seconds */ + /* *INDENT-OFF* */ + foreach_vlib_main ({ + rt = vlib_node_get_runtime_data (this_vlib_main, syn_filter4_node.index); + vec_validate (rt->syn_counts, 1023); + /* + * Given perfect disperson / optimal hashing results: + * Allow 128k (successful) syns/sec. 1024, buckets each of which + * absorb 128 syns before filtering. Reset table once a second. + * Reality bites, lets try resetting once every 100ms. + */ + rt->reset_interval = 0.1; /* reset interval in seconds */ + }); + /* *INDENT-ON* */ } rv = vnet_feature_enable_disable ("ip4-local", "syn-filter-4", diff --git a/src/vnet/udp/udp_local.c b/src/vnet/udp/udp_local.c index 6b239f73..3a60b29b 100644 --- a/src/vnet/udp/udp_local.c +++ b/src/vnet/udp/udp_local.c @@ -520,11 +520,15 @@ udp_register_dst_port (vlib_main_t * vm, : udp6_input_node.index, node_index); /* Setup udp protocol -> next index sparse vector mapping. */ - rt = vlib_node_get_runtime_data - (vm, is_ip4 ? udp4_input_node.index : udp6_input_node.index); - n = sparse_vec_validate (rt->next_by_dst_port, - clib_host_to_net_u16 (dst_port)); - n[0] = pi->next_index; + /* *INDENT-OFF* */ + foreach_vlib_main({ + rt = vlib_node_get_runtime_data + (this_vlib_main, is_ip4 ? udp4_input_node.index : udp6_input_node.index); + n = sparse_vec_validate (rt->next_by_dst_port, + clib_host_to_net_u16 (dst_port)); + n[0] = pi->next_index; + }); + /* *INDENT-ON* */ } void @@ -541,11 +545,15 @@ udp_unregister_dst_port (vlib_main_t * vm, udp_dst_port_t dst_port, u8 is_ip4) return; /* Kill the mapping. Don't bother killing the pi, it may be back. */ - rt = vlib_node_get_runtime_data - (vm, is_ip4 ? udp4_input_node.index : udp6_input_node.index); - n = sparse_vec_validate (rt->next_by_dst_port, - clib_host_to_net_u16 (dst_port)); - n[0] = SPARSE_VEC_INVALID_INDEX; + /* *INDENT-OFF* */ + foreach_vlib_main({ + rt = vlib_node_get_runtime_data + (this_vlib_main, is_ip4 ? udp4_input_node.index : udp6_input_node.index); + n = sparse_vec_validate (rt->next_by_dst_port, + clib_host_to_net_u16 (dst_port)); + n[0] = SPARSE_VEC_INVALID_INDEX; + }); + /* *INDENT-ON* */ } void @@ -604,10 +612,27 @@ udp_setup_node (vlib_main_t * vm, u32 node_index) pn->unformat_edit = unformat_pg_udp_header; } +static void +udp_local_node_runtime_init (vlib_main_t * vm) +{ + udp_input_runtime_t *rt; + + rt = vlib_node_get_runtime_data (vm, udp4_input_node.index); + rt->next_by_dst_port = sparse_vec_new + ( /* elt bytes */ sizeof (rt->next_by_dst_port[0]), + /* bits in index */ BITS (((udp_header_t *) 0)->dst_port)); + rt->punt_unknown = 0; + + rt = vlib_node_get_runtime_data (vm, udp6_input_node.index); + rt->next_by_dst_port = sparse_vec_new + ( /* elt bytes */ sizeof (rt->next_by_dst_port[0]), + /* bits in index */ BITS (((udp_header_t *) 0)->dst_port)); + rt->punt_unknown = 0; +} + clib_error_t * udp_local_init (vlib_main_t * vm) { - udp_input_runtime_t *rt; udp_main_t *um = &udp_main; int i; @@ -628,27 +653,13 @@ udp_local_init (vlib_main_t * vm) udp_setup_node (vm, udp4_input_node.index); udp_setup_node (vm, udp6_input_node.index); - rt = vlib_node_get_runtime_data (vm, udp4_input_node.index); - - rt->next_by_dst_port = sparse_vec_new - ( /* elt bytes */ sizeof (rt->next_by_dst_port[0]), - /* bits in index */ BITS (((udp_header_t *) 0)->dst_port)); - - rt->punt_unknown = 0; + udp_local_node_runtime_init (vm); #define _(n,s) add_dst_port (um, UDP_DST_PORT_##s, #s, 1 /* is_ip4 */); foreach_udp4_dst_port #undef _ - rt = vlib_node_get_runtime_data (vm, udp6_input_node.index); - - rt->next_by_dst_port = sparse_vec_new - ( /* elt bytes */ sizeof (rt->next_by_dst_port[0]), - /* bits in index */ BITS (((udp_header_t *) 0)->dst_port)); - - rt->punt_unknown = 0; - #define _(n,s) add_dst_port (um, UDP_DST_PORT_##s, #s, 0 /* is_ip4 */); - foreach_udp6_dst_port + foreach_udp6_dst_port #undef _ ip4_register_protocol (IP_PROTOCOL_UDP, udp4_input_node.index); /* Note: ip6 differs from ip4, UDP is hotwired to ip6-udp-lookup */ @@ -657,6 +668,15 @@ udp_local_init (vlib_main_t * vm) VLIB_INIT_FUNCTION (udp_local_init); +clib_error_t * +udp_local_worker_init (vlib_main_t * vm) +{ + udp_local_node_runtime_init (vm); + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (udp_local_worker_init); + /* * fd.io coding-style-patch-verification: ON * -- cgit 1.2.3-korg From beb0b2e346c63e21ffe892ae0e04b67bb10fba5e Mon Sep 17 00:00:00 2001 From: John Lo Date: Sat, 22 Jul 2017 00:21:36 -0400 Subject: Improve L2 Input/Output Feature Infrastructure and Usage Simplify L2 output feature infra to unify with L2 input feature infra using the newly improved feature bitmap mechanism. Updated all L2 features to use the more efficient infra functions. Change-Id: If8f463826b0af0717129befe92a27ea8cfc40449 Signed-off-by: John Lo --- src/plugins/acl/fa_node.c | 7 +-- src/vnet/ethernet/arp.c | 8 +-- src/vnet/l2/feat_bitmap.h | 18 +++++- src/vnet/l2/l2_classify.h | 8 +-- src/vnet/l2/l2_efp_filter.c | 54 +++--------------- src/vnet/l2/l2_input.c | 37 ++++++------- src/vnet/l2/l2_input_acl.c | 9 +-- src/vnet/l2/l2_input_classify.c | 22 +++----- src/vnet/l2/l2_input_vtr.c | 35 ++---------- src/vnet/l2/l2_learn.c | 14 +---- src/vnet/l2/l2_output.c | 115 +++++++++++++++++++++++---------------- src/vnet/l2/l2_output.h | 80 +-------------------------- src/vnet/l2/l2_output_acl.c | 24 ++------ src/vnet/l2/l2_output_classify.c | 30 +++------- src/vnet/l2/l2_rw.c | 27 +++------ src/vnet/policer/node_funcs.c | 11 +--- 16 files changed, 160 insertions(+), 339 deletions(-) (limited to 'src/vnet/l2/l2_input_classify.c') diff --git a/src/plugins/acl/fa_node.c b/src/plugins/acl/fa_node.c index 0bbc7423..c483044d 100644 --- a/src/plugins/acl/fa_node.c +++ b/src/plugins/acl/fa_node.c @@ -931,7 +931,6 @@ acl_fa_node_fn (vlib_main_t * vm, u32 pkts_acl_permit = 0; u32 pkts_restart_session_timer = 0; u32 trace_bitmap = 0; - u32 feature_bitmap0; acl_main_t *am = &acl_main; fa_5tuple_t fa_5tuple, kv_sess; clib_bihash_kv_40_8_t value_sess; @@ -977,8 +976,6 @@ acl_fa_node_fn (vlib_main_t * vm, sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; else sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX]; - if (is_l2_path) - feature_bitmap0 = vnet_buffer (b0)->l2.feature_bitmap; /* * Extract the L3/L4 matching info into a 5-tuple structure, @@ -1089,9 +1086,7 @@ acl_fa_node_fn (vlib_main_t * vm, if (action > 0) { if (is_l2_path) - next0 = - feat_bitmap_get_next_node_index (l2_feat_next_node_index, - feature_bitmap0); + next0 = vnet_l2_feature_next (b0, l2_feat_next_node_index, 0); else vnet_feature_next (sw_if_index0, &next0, b0); } diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c index df681750..1bce3328 100644 --- a/src/vnet/ethernet/arp.c +++ b/src/vnet/ethernet/arp.c @@ -2291,12 +2291,8 @@ arp_term_l2bd (vlib_main_t * vm, next_l2_feature: { - u32 feature_bitmap0 = - vnet_buffer (p0)->l2.feature_bitmap & ~L2INPUT_FEAT_ARP_TERM; - vnet_buffer (p0)->l2.feature_bitmap = feature_bitmap0; - next0 = - feat_bitmap_get_next_node_index (arp_term_next_node_index, - feature_bitmap0); + next0 = vnet_l2_feature_next (p0, arp_term_next_node_index, + L2INPUT_FEAT_ARP_TERM); vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, pi0, next0); diff --git a/src/vnet/l2/feat_bitmap.h b/src/vnet/l2/feat_bitmap.h index c6e02ecc..5940ff7e 100644 --- a/src/vnet/l2/feat_bitmap.h +++ b/src/vnet/l2/feat_bitmap.h @@ -75,8 +75,8 @@ feat_bitmap_init_next_nodes (vlib_main_t * vm, u32 node_index, /* the current gr Return the graph node index for the feature corresponding to the first set bit in the bitmap. */ -always_inline - u32 feat_bitmap_get_next_node_index (u32 * next_nodes, u32 bitmap) +always_inline u32 +feat_bitmap_get_next_node_index (u32 * next_nodes, u32 bitmap) { u32 first_bit; @@ -85,6 +85,20 @@ always_inline return next_nodes[first_bit]; } +/** + Return the graph node index for the feature corresponding to the next + set bit after clearing the current feature bit in the feature_bitmap + of the current packet. +*/ +always_inline u32 +vnet_l2_feature_next (vlib_buffer_t * b, u32 * next_nodes, u32 feat_bit) +{ + vnet_buffer (b)->l2.feature_bitmap &= ~feat_bit; + u32 fb = vnet_buffer (b)->l2.feature_bitmap; + ASSERT (fb != 0); + return feat_bitmap_get_next_node_index (next_nodes, fb); +} + #endif /* included_vnet_l2_feat_bitmap_h */ /* diff --git a/src/vnet/l2/l2_classify.h b/src/vnet/l2/l2_classify.h index 184187ff..100c584a 100644 --- a/src/vnet/l2/l2_classify.h +++ b/src/vnet/l2/l2_classify.h @@ -68,15 +68,13 @@ typedef enum typedef struct _l2_classify_main { - /* Next nodes for each feature */ - u32 feat_next_node_index[32]; + /* Next nodes for L2 input and output features */ + u32 l2_inp_feat_next[32]; + u32 l2_out_feat_next[32]; /* Per-address-family classifier table vectors */ u32 *classify_table_index_by_sw_if_index[L2_INPUT_CLASSIFY_N_TABLES]; - /* Next nodes for features and output interfaces */ - l2_output_next_nodes_st next_nodes; - /* convenience variables */ vlib_main_t *vlib_main; vnet_main_t *vnet_main; diff --git a/src/vnet/l2/l2_efp_filter.c b/src/vnet/l2/l2_efp_filter.c index f9ba8f2f..faf78153 100644 --- a/src/vnet/l2/l2_efp_filter.c +++ b/src/vnet/l2/l2_efp_filter.c @@ -43,9 +43,8 @@ */ typedef struct { - - /* Next nodes for features and output interfaces */ - l2_output_next_nodes_st next_nodes; + /* Next nodes for L2 output features */ + u32 l2_out_feat_next[32]; /* convenience variables */ vlib_main_t *vlib_main; @@ -180,11 +179,6 @@ l2_efp_filter_node_fn (vlib_main_t * vm, vlib_node_t *n = vlib_get_node (vm, l2_efp_filter_node.index); u32 node_counter_base_index = n->error_heap_index; vlib_error_main_t *em = &vm->error_main; - u32 cached_sw_if_index = ~0; - u32 cached_next_index = ~0; - - /* invalidate cache to begin with */ - cached_sw_if_index = ~0; from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; /* number of packets to process */ @@ -203,7 +197,6 @@ l2_efp_filter_node_fn (vlib_main_t * vm, vlib_buffer_t *b0, *b1; u32 next0, next1; u32 sw_if_index0, sw_if_index1; - u32 feature_bitmap0, feature_bitmap1; u16 first_ethertype0, first_ethertype1; u16 outer_id0, inner_id0, outer_id1, inner_id1; u32 match_flags0, match_flags1; @@ -267,29 +260,11 @@ l2_efp_filter_node_fn (vlib_main_t * vm, em->counters[node_counter_base_index + L2_EFP_FILTER_ERROR_L2_EFP_FILTER] += 2; - /* Remove ourself from the feature bitmap */ - feature_bitmap0 = - vnet_buffer (b0)->l2.feature_bitmap & ~L2OUTPUT_FEAT_EFP_FILTER; - feature_bitmap1 = - vnet_buffer (b1)->l2.feature_bitmap & ~L2OUTPUT_FEAT_EFP_FILTER; - /* Determine next node */ - l2_output_dispatch (msm->vlib_main, - msm->vnet_main, - node, - l2_efp_filter_node.index, - &cached_sw_if_index, - &cached_next_index, - &msm->next_nodes, - b0, sw_if_index0, feature_bitmap0, &next0); - l2_output_dispatch (msm->vlib_main, - msm->vnet_main, - node, - l2_efp_filter_node.index, - &cached_sw_if_index, - &cached_next_index, - &msm->next_nodes, - b1, sw_if_index1, feature_bitmap1, &next1); + next0 = vnet_l2_feature_next (b0, msm->l2_out_feat_next, + L2OUTPUT_FEAT_EFP_FILTER); + next1 = vnet_l2_feature_next (b1, msm->l2_out_feat_next, + L2OUTPUT_FEAT_EFP_FILTER); /* perform the efp filter check on two packets */ @@ -394,7 +369,6 @@ l2_efp_filter_node_fn (vlib_main_t * vm, vlib_buffer_t *b0; u32 next0; u32 sw_if_index0; - u32 feature_bitmap0; u16 first_ethertype0; u16 outer_id0, inner_id0; u32 match_flags0; @@ -422,19 +396,9 @@ l2_efp_filter_node_fn (vlib_main_t * vm, em->counters[node_counter_base_index + L2_EFP_FILTER_ERROR_L2_EFP_FILTER] += 1; - /* Remove ourself from the feature bitmap */ - feature_bitmap0 = - vnet_buffer (b0)->l2.feature_bitmap & ~L2OUTPUT_FEAT_EFP_FILTER; - /* Determine next node */ - l2_output_dispatch (msm->vlib_main, - msm->vnet_main, - node, - l2_efp_filter_node.index, - &cached_sw_if_index, - &cached_next_index, - &msm->next_nodes, - b0, sw_if_index0, feature_bitmap0, &next0); + next0 = vnet_l2_feature_next (b0, msm->l2_out_feat_next, + L2OUTPUT_FEAT_EFP_FILTER); /* perform the efp filter check on one packet */ @@ -528,7 +492,7 @@ VLIB_NODE_FUNCTION_MULTIARCH (l2_efp_filter_node, l2_efp_filter_node_fn) l2_efp_filter_node.index, L2OUTPUT_N_FEAT, l2output_get_feat_names (), - mp->next_nodes.feat_next_node_index); + mp->l2_out_feat_next); return 0; } diff --git a/src/vnet/l2/l2_input.c b/src/vnet/l2/l2_input.c index aa156213..26c832ad 100644 --- a/src/vnet/l2/l2_input.c +++ b/src/vnet/l2/l2_input.c @@ -115,10 +115,7 @@ typedef enum static_always_inline void -classify_and_dispatch (vlib_main_t * vm, - vlib_node_runtime_t * node, - u32 thread_index, - l2input_main_t * msm, vlib_buffer_t * b0, u32 * next0) +classify_and_dispatch (l2input_main_t * msm, vlib_buffer_t * b0, u32 * next0) { /* * Load L2 input feature struct @@ -187,12 +184,7 @@ classify_and_dispatch (vlib_main_t * vm, } - if (config->xconnect) - { - /* Set the output interface */ - vnet_buffer (b0)->sw_if_index[VLIB_TX] = config->output_sw_if_index; - } - else + if (config->bridge) { /* Do bridge-domain processing */ bd_index0 = config->bd_index; @@ -220,6 +212,13 @@ classify_and_dispatch (vlib_main_t * vm, */ feat_mask = feat_mask & bd_config->feature_bitmap; } + else if (config->xconnect) + { + /* Set the output interface */ + vnet_buffer (b0)->sw_if_index[VLIB_TX] = config->output_sw_if_index; + } + else + feat_mask = L2INPUT_FEAT_DROP; /* mask out features from bitmap using packet type and bd config */ feature_bitmap = config->feature_bitmap & feat_mask; @@ -240,7 +239,6 @@ l2input_node_inline (vlib_main_t * vm, u32 n_left_from, *from, *to_next; l2input_next_t next_index; l2input_main_t *msm = &l2input_main; - u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; /* number of packets to process */ @@ -353,10 +351,10 @@ l2input_node_inline (vlib_main_t * vm, vlib_node_increment_counter (vm, l2input_node.index, L2INPUT_ERROR_L2INPUT, 4); - classify_and_dispatch (vm, node, thread_index, msm, b0, &next0); - classify_and_dispatch (vm, node, thread_index, msm, b1, &next1); - classify_and_dispatch (vm, node, thread_index, msm, b2, &next2); - classify_and_dispatch (vm, node, thread_index, msm, b3, &next3); + classify_and_dispatch (msm, b0, &next0); + classify_and_dispatch (msm, b1, &next1); + classify_and_dispatch (msm, b2, &next2); + classify_and_dispatch (msm, b3, &next3); /* verify speculative enqueues, maybe switch current next frame */ /* if next0==next1==next_index then nothing special needs to be done */ @@ -396,7 +394,7 @@ l2input_node_inline (vlib_main_t * vm, vlib_node_increment_counter (vm, l2input_node.index, L2INPUT_ERROR_L2INPUT, 1); - classify_and_dispatch (vm, node, thread_index, msm, b0, &next0); + classify_and_dispatch (msm, b0, &next0); /* verify speculative enqueue, maybe switch current next frame */ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, @@ -574,8 +572,8 @@ set_int_l2_mode (vlib_main_t * vm, vnet_main_t * vnet_main, /* */ } /* Make sure vector is big enough */ - vec_validate_init_empty (l2om->next_nodes.output_node_index_vec, - sw_if_index, L2OUTPUT_NEXT_DROP); + vec_validate_init_empty (l2om->output_node_index_vec, sw_if_index, + L2OUTPUT_NEXT_DROP); /* Initialize the l2-input configuration for the interface */ if (mode == MODE_L3) @@ -594,8 +592,7 @@ set_int_l2_mode (vlib_main_t * vm, vnet_main_t * vnet_main, /* */ /* Make sure any L2-output packet to this interface now in L3 mode is * dropped. This may happen if L2 FIB MAC entry is stale */ - l2om->next_nodes.output_node_index_vec[sw_if_index] = - L2OUTPUT_NEXT_BAD_INTF; + l2om->output_node_index_vec[sw_if_index] = L2OUTPUT_NEXT_BAD_INTF; } else { diff --git a/src/vnet/l2/l2_input_acl.c b/src/vnet/l2/l2_input_acl.c index 104fcd15..84030888 100644 --- a/src/vnet/l2/l2_input_acl.c +++ b/src/vnet/l2/l2_input_acl.c @@ -269,14 +269,11 @@ l2_inacl_node_fn (vlib_main_t * vm, e0 = 0; t0 = 0; - /* Feature bitmap update */ - vnet_buffer (b0)->l2.feature_bitmap &= ~L2INPUT_FEAT_ACL; - vnet_buffer (b0)->l2_classify.opaque_index = ~0; + /* Determine the next node */ - next0 = feat_bitmap_get_next_node_index (msm->feat_next_node_index, - vnet_buffer (b0)-> - l2.feature_bitmap); + next0 = vnet_l2_feature_next (b0, msm->feat_next_node_index, + L2INPUT_FEAT_ACL); if (PREDICT_TRUE (table_index0 != ~0)) { diff --git a/src/vnet/l2/l2_input_classify.c b/src/vnet/l2/l2_input_classify.c index 485b9abd..ee8042a0 100644 --- a/src/vnet/l2/l2_input_classify.c +++ b/src/vnet/l2/l2_input_classify.c @@ -152,7 +152,6 @@ l2_input_classify_node_fn (vlib_main_t * vm, vnet_classify_main_t *vcm = cm->vnet_classify_main; l2_input_classify_runtime_t *rt = (l2_input_classify_runtime_t *) node->runtime_data; - u32 feature_bitmap; u32 hits = 0; u32 misses = 0; u32 chain_hits = 0; @@ -354,13 +353,6 @@ l2_input_classify_node_fn (vlib_main_t * vm, e0 = 0; vnet_buffer (b0)->l2_classify.opaque_index = ~0; - /* Remove ourself from the feature bitmap */ - feature_bitmap = vnet_buffer (b0)->l2.feature_bitmap - & ~L2INPUT_FEAT_INPUT_CLASSIFY; - - /* save for next feature graph nodes */ - vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap; - if (PREDICT_TRUE (table_index0 != ~0)) { hash0 = vnet_buffer (b0)->l2_classify.hash; @@ -412,13 +404,13 @@ l2_input_classify_node_fn (vlib_main_t * vm, if (PREDICT_FALSE (next0 == 0)) b0->error = node->errors[L2_INPUT_CLASSIFY_ERROR_DROP]; + /* Determine the next node and remove ourself from bitmap */ if (PREDICT_TRUE (next0 == ~0)) - { - // Determine the next node - next0 = - feat_bitmap_get_next_node_index (cm->feat_next_node_index, - feature_bitmap); - } + next0 = vnet_l2_feature_next (b0, cm->l2_inp_feat_next, + L2INPUT_FEAT_INPUT_CLASSIFY); + else + vnet_buffer (b0)->l2.feature_bitmap &= + ~L2INPUT_FEAT_INPUT_CLASSIFY; if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) && (b0->flags & VLIB_BUFFER_IS_TRACED))) @@ -496,7 +488,7 @@ l2_input_classify_init (vlib_main_t * vm) l2_input_classify_node.index, L2INPUT_N_FEAT, l2input_get_feat_names (), - cm->feat_next_node_index); + cm->l2_inp_feat_next); rt->l2cm = cm; rt->vcm = cm->vnet_classify_main; diff --git a/src/vnet/l2/l2_input_vtr.c b/src/vnet/l2/l2_input_vtr.c index ded23095..9470752f 100644 --- a/src/vnet/l2/l2_input_vtr.c +++ b/src/vnet/l2/l2_input_vtr.c @@ -111,7 +111,6 @@ l2_invtr_node_fn (vlib_main_t * vm, vlib_buffer_t *b0, *b1; u32 next0, next1; u32 sw_if_index0, sw_if_index1; - u32 feature_bitmap0, feature_bitmap1; /* Prefetch next iteration. */ { @@ -160,23 +159,11 @@ l2_invtr_node_fn (vlib_main_t * vm, sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX]; - /* process 2 packets */ - - /* Remove ourself from the feature bitmap */ - feature_bitmap0 = - vnet_buffer (b0)->l2.feature_bitmap & ~L2INPUT_FEAT_VTR; - feature_bitmap1 = - vnet_buffer (b1)->l2.feature_bitmap & ~L2INPUT_FEAT_VTR; - - /* save for next feature graph nodes */ - vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap0; - vnet_buffer (b1)->l2.feature_bitmap = feature_bitmap1; - /* Determine the next node */ - next0 = feat_bitmap_get_next_node_index (msm->feat_next_node_index, - feature_bitmap0); - next1 = feat_bitmap_get_next_node_index (msm->feat_next_node_index, - feature_bitmap1); + next0 = vnet_l2_feature_next (b0, msm->feat_next_node_index, + L2INPUT_FEAT_VTR); + next1 = vnet_l2_feature_next (b1, msm->feat_next_node_index, + L2INPUT_FEAT_VTR); l2_output_config_t *config0; l2_output_config_t *config1; @@ -264,7 +251,6 @@ l2_invtr_node_fn (vlib_main_t * vm, vlib_buffer_t *b0; u32 next0; u32 sw_if_index0; - u32 feature_bitmap0; /* speculatively enqueue b0 to the current next frame */ bi0 = from[0]; @@ -278,18 +264,9 @@ l2_invtr_node_fn (vlib_main_t * vm, sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - /* process 1 packet */ - - /* Remove ourself from the feature bitmap */ - feature_bitmap0 = - vnet_buffer (b0)->l2.feature_bitmap & ~L2INPUT_FEAT_VTR; - - /* save for next feature graph nodes */ - vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap0; - /* Determine the next node */ - next0 = feat_bitmap_get_next_node_index (msm->feat_next_node_index, - feature_bitmap0); + next0 = vnet_l2_feature_next (b0, msm->feat_next_node_index, + L2INPUT_FEAT_VTR); l2_output_config_t *config0; config0 = vec_elt_at_index (l2output_main.configs, sw_if_index0); diff --git a/src/vnet/l2/l2_learn.c b/src/vnet/l2/l2_learn.c index b9904d3e..65406292 100644 --- a/src/vnet/l2/l2_learn.c +++ b/src/vnet/l2/l2_learn.c @@ -116,19 +116,9 @@ l2learn_process (vlib_node_runtime_t * node, u32 * bucket0, l2fib_entry_result_t * result0, u32 * next0, u8 timestamp) { - u32 feature_bitmap; - /* Set up the default next node (typically L2FWD) */ - - /* Remove ourself from the feature bitmap */ - feature_bitmap = vnet_buffer (b0)->l2.feature_bitmap & ~L2INPUT_FEAT_LEARN; - - /* Save for next feature graph nodes */ - vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap; - - /* Determine the next node */ - *next0 = feat_bitmap_get_next_node_index (msm->feat_next_node_index, - feature_bitmap); + *next0 = vnet_l2_feature_next (b0, msm->feat_next_node_index, + L2INPUT_FEAT_LEARN); /* Check mac table lookup result */ if (PREDICT_TRUE (result0->fields.sw_if_index == sw_if_index0)) diff --git a/src/vnet/l2/l2_output.c b/src/vnet/l2/l2_output.c index 51d5e145..b3537a35 100644 --- a/src/vnet/l2/l2_output.c +++ b/src/vnet/l2/l2_output.c @@ -98,6 +98,57 @@ split_horizon_violation (u8 shg1, u8 shg2) } } +/** Determine the next L2 node based on the output feature bitmap */ +static_always_inline void +l2_output_dispatch (vlib_buffer_t * b0, vlib_node_runtime_t * node, + u32 * cached_sw_if_index, u32 * cached_next_index, + u32 sw_if_index, u32 feature_bitmap, u32 * next0) +{ + /* + * The output feature bitmap always have at least the L2 output bit set + * for a normal L2 interface (or 0 if the interface is changed from L2 + * to L3 mode). So if the feature bitmap is 0 or just have L2 output bits set, + * we know there is no more feature and will just output packets on interface. + * Otherwise, get the index of the next feature node. + */ + if (PREDICT_FALSE ((feature_bitmap & ~L2OUTPUT_FEAT_OUTPUT) != 0)) + { + /* Save bitmap for the next feature graph nodes */ + vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap; + + /* Determine the next node */ + *next0 = + feat_bitmap_get_next_node_index (l2output_main.l2_out_feat_next, + feature_bitmap); + } + else + { + /* + * There are no features. Send packet to TX node for sw_if_index0 + * This is a little tricky in that the output interface next node indexes + * are not precomputed at init time. + */ + + if (sw_if_index == *cached_sw_if_index) + { + /* We hit in the one-entry cache. Use it. */ + *next0 = *cached_next_index; + } + else + { + /* Look up the output TX node for the sw_if_index */ + *next0 = vec_elt (l2output_main.output_node_index_vec, sw_if_index); + + if (PREDICT_FALSE (*next0 == L2OUTPUT_NEXT_DROP)) + b0->error = node->errors[L2OUTPUT_ERROR_MAPPING_DROP]; + + /* Update the one-entry cache */ + *cached_sw_if_index = sw_if_index; + *cached_next_index = *next0; + } + } +} + static_always_inline void l2output_vtr (vlib_node_runtime_t * node, l2_output_config_t * config, u32 feature_bitmap, vlib_buffer_t * b, u32 * next) @@ -239,41 +290,18 @@ l2output_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, feature_bitmap3 = config3->feature_bitmap; /* Determine next node */ - l2_output_dispatch (msm->vlib_main, - msm->vnet_main, - node, - l2output_node.index, - &cached_sw_if_index, - &cached_next_index, - &msm->next_nodes, - b0, sw_if_index0, feature_bitmap0, &next0); - - l2_output_dispatch (msm->vlib_main, - msm->vnet_main, - node, - l2output_node.index, - &cached_sw_if_index, - &cached_next_index, - &msm->next_nodes, - b1, sw_if_index1, feature_bitmap1, &next1); - - l2_output_dispatch (msm->vlib_main, - msm->vnet_main, - node, - l2output_node.index, - &cached_sw_if_index, - &cached_next_index, - &msm->next_nodes, - b2, sw_if_index2, feature_bitmap2, &next2); - - l2_output_dispatch (msm->vlib_main, - msm->vnet_main, - node, - l2output_node.index, - &cached_sw_if_index, - &cached_next_index, - &msm->next_nodes, - b3, sw_if_index3, feature_bitmap3, &next3); + l2_output_dispatch (b0, node, &cached_sw_if_index, + &cached_next_index, sw_if_index0, + feature_bitmap0, &next0); + l2_output_dispatch (b1, node, &cached_sw_if_index, + &cached_next_index, sw_if_index1, + feature_bitmap1, &next1); + l2_output_dispatch (b2, node, &cached_sw_if_index, + &cached_next_index, sw_if_index2, + feature_bitmap2, &next2); + l2_output_dispatch (b3, node, &cached_sw_if_index, + &cached_next_index, sw_if_index3, + feature_bitmap3, &next3); l2output_vtr (node, config0, feature_bitmap0, b0, &next0); l2output_vtr (node, config1, feature_bitmap1, b1, &next1); @@ -401,14 +429,9 @@ l2output_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, feature_bitmap0 = config0->feature_bitmap; /* Determine next node */ - l2_output_dispatch (msm->vlib_main, - msm->vnet_main, - node, - l2output_node.index, - &cached_sw_if_index, - &cached_next_index, - &msm->next_nodes, - b0, sw_if_index0, feature_bitmap0, &next0); + l2_output_dispatch (b0, node, &cached_sw_if_index, + &cached_next_index, sw_if_index0, + feature_bitmap0, &next0); l2output_vtr (node, config0, feature_bitmap0, b0, &next0); @@ -598,10 +621,10 @@ VLIB_NODE_FUNCTION_MULTIARCH (l2output_node, l2output_node_fn) l2output_node.index, L2OUTPUT_N_FEAT, l2output_get_feat_names (), - mp->next_nodes.feat_next_node_index); + mp->l2_out_feat_next); /* Initialize the output node mapping table */ - vec_validate_init_empty (mp->next_nodes.output_node_index_vec, 100, + vec_validate_init_empty (mp->output_node_index_vec, 100, L2OUTPUT_NEXT_DROP); return 0; @@ -621,7 +644,7 @@ l2output_create_output_node_mapping (vlib_main_t * vlib_main, /* dynamically create graph node arc */ u32 next = vlib_node_add_next (vlib_main, l2output_node.index, hw0->output_node_index); - l2output_main.next_nodes.output_node_index_vec[sw_if_index] = next; + l2output_main.output_node_index_vec[sw_if_index] = next; } /* Get a pointer to the config for the given interface */ diff --git a/src/vnet/l2/l2_output.h b/src/vnet/l2/l2_output.h index 82cefd2c..6da3e303 100644 --- a/src/vnet/l2/l2_output.h +++ b/src/vnet/l2/l2_output.h @@ -52,11 +52,6 @@ typedef struct } l2_output_config_t; - -/* - * The set of next nodes for features and interface output. - * Each output feature node should include this. - */ typedef struct { /* @@ -70,15 +65,7 @@ typedef struct * array of next node index for each output feature, indexed * by l2output_feat_t. Used to determine next feature node. */ - u32 feat_next_node_index[32]; - -} l2_output_next_nodes_st; - - -typedef struct -{ - /* Next nodes for features and output interfaces */ - l2_output_next_nodes_st next_nodes; + u32 l2_out_feat_next[32]; /* config vector indexed by sw_if_index */ l2_output_config_t *configs; @@ -163,71 +150,6 @@ void l2output_create_output_node_mapping (vlib_main_t * vlib_main, vnet_main_t * vnet_main, u32 sw_if_index); -/** Determine the next L2 node based on the output feature bitmap */ -always_inline void -l2_output_dispatch (vlib_main_t * vlib_main, - vnet_main_t * vnet_main, - vlib_node_runtime_t * node, - u32 node_index, - u32 * cached_sw_if_index, - u32 * cached_next_index, - l2_output_next_nodes_st * next_nodes, - vlib_buffer_t * b0, - u32 sw_if_index, u32 feature_bitmap, u32 * next0) -{ - /* - * The output feature bitmap always have at least the output feature bit set - * for a normal L2 interface (or all 0's if the interface is changed from L2 - * to L3 mode). So if next_nodes specified is that from the l2-output node and - * the bitmap is all clear except output feature bit, we know there is no more - * feature and will fall through to output packet. If next_nodes is from a L2 - * output feature node (and not l2-output), we always want to get the node for - * the next L2 output feature, including the last feature being interface- - * output node to output packet. - */ - if ((next_nodes != &l2output_main.next_nodes) - || ((feature_bitmap & ~L2OUTPUT_FEAT_OUTPUT) != 0)) - { - /* There are some features to execute */ - ASSERT (feature_bitmap != 0); - - /* Save bitmap for the next feature graph nodes */ - vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap; - - /* Determine the next node */ - *next0 = - feat_bitmap_get_next_node_index (next_nodes->feat_next_node_index, - feature_bitmap); - } - else - { - /* - * There are no features. Send packet to TX node for sw_if_index0 - * This is a little tricky in that the output interface next node indexes - * are not precomputed at init time. - */ - - if (sw_if_index == *cached_sw_if_index) - { - /* We hit in the one-entry cache. Use it. */ - *next0 = *cached_next_index; - } - else - { - /* Look up the output TX node for the sw_if_index */ - *next0 = vec_elt (l2output_main.next_nodes.output_node_index_vec, - sw_if_index); - - if (*next0 == L2OUTPUT_NEXT_DROP) - b0->error = node->errors[L2OUTPUT_ERROR_MAPPING_DROP]; - - /* Update the one-entry cache */ - *cached_sw_if_index = sw_if_index; - *cached_next_index = *next0; - } - } -} - /** Get a pointer to the config for the given interface */ l2_output_config_t *l2output_intf_config (u32 sw_if_index); diff --git a/src/vnet/l2/l2_output_acl.c b/src/vnet/l2/l2_output_acl.c index 1d1971a5..7d051326 100644 --- a/src/vnet/l2/l2_output_acl.c +++ b/src/vnet/l2/l2_output_acl.c @@ -34,8 +34,8 @@ typedef struct { - /* Next nodes for features and output interfaces */ - l2_output_next_nodes_st next_nodes; + /* Next nodes for L2 output features */ + u32 l2_out_feat_next[32]; /* convenience variables */ vlib_main_t *vlib_main; @@ -108,8 +108,6 @@ l2_outacl_node_fn (vlib_main_t * vm, vlib_node_t *n = vlib_get_node (vm, l2_outacl_node.index); u32 node_counter_base_index = n->error_heap_index; vlib_error_main_t *em = &vm->error_main; - u32 cached_sw_if_index = (u32) ~ 0; - u32 cached_next_index = (u32) ~ 0; from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; /* number of packets to process */ @@ -201,7 +199,6 @@ l2_outacl_node_fn (vlib_main_t * vm, u32 next0; u32 sw_if_index0; ethernet_header_t *h0; - u32 feature_bitmap0; /* speculatively enqueue b0 to the current next frame */ bi0 = from[0]; @@ -234,20 +231,9 @@ l2_outacl_node_fn (vlib_main_t * vm, * Dummy for now, just go to next feature node */ - - /* Remove ourself from the feature bitmap */ - feature_bitmap0 = - vnet_buffer (b0)->l2.feature_bitmap & ~L2OUTPUT_FEAT_ACL; - /* Determine next node */ - l2_output_dispatch (msm->vlib_main, - msm->vnet_main, - node, - l2_outacl_node.index, - &cached_sw_if_index, - &cached_next_index, - &msm->next_nodes, - b0, sw_if_index0, feature_bitmap0, &next0); + next0 = vnet_l2_feature_next (b0, msm->l2_out_feat_next, + L2OUTPUT_FEAT_ACL); /* verify speculative enqueue, maybe switch current next frame */ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, @@ -295,7 +281,7 @@ VLIB_NODE_FUNCTION_MULTIARCH (l2_outacl_node, l2_outacl_node_fn) l2_outacl_node.index, L2OUTPUT_N_FEAT, l2output_get_feat_names (), - mp->next_nodes.feat_next_node_index); + mp->l2_out_feat_next); return 0; } diff --git a/src/vnet/l2/l2_output_classify.c b/src/vnet/l2/l2_output_classify.c index 869b0656..a49abec2 100644 --- a/src/vnet/l2/l2_output_classify.c +++ b/src/vnet/l2/l2_output_classify.c @@ -144,14 +144,11 @@ l2_output_classify_node_fn (vlib_main_t * vm, vnet_classify_main_t *vcm = cm->vnet_classify_main; l2_output_classify_runtime_t *rt = (l2_output_classify_runtime_t *) node->runtime_data; - u32 feature_bitmap0; u32 hits = 0; u32 misses = 0; u32 chain_hits = 0; f64 now; u32 n_next_nodes; - u32 cached_sw_if_index = (u32) ~ 0; - u32 cached_next_index = (u32) ~ 0; u32 sw_if_index0; n_next_nodes = node->n_next_nodes; @@ -347,12 +344,6 @@ l2_output_classify_node_fn (vlib_main_t * vm, table_index0 = vnet_buffer (b0)->l2_classify.table_index; e0 = 0; vnet_buffer (b0)->l2_classify.opaque_index = ~0; - /* Remove ourself from the feature bitmap */ - feature_bitmap0 = vnet_buffer (b0)->l2.feature_bitmap - & ~L2OUTPUT_FEAT_OUTPUT_CLASSIFY; - - /* save for next feature graph nodes */ - vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap0; if (PREDICT_TRUE (table_index0 != ~0)) { @@ -405,20 +396,13 @@ l2_output_classify_node_fn (vlib_main_t * vm, if (PREDICT_FALSE (next0 == 0)) b0->error = node->errors[L2_OUTPUT_CLASSIFY_ERROR_DROP]; + /* Determine the next node and remove ourself from bitmap */ if (PREDICT_FALSE (next0 == ~0)) - { - sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX]; - - /* Determine next node */ - l2_output_dispatch (cm->vlib_main, - cm->vnet_main, - node, - l2_output_classify_node.index, - &cached_sw_if_index, - &cached_next_index, - &cm->next_nodes, - b0, sw_if_index0, feature_bitmap0, &next0); - } + next0 = vnet_l2_feature_next (b0, cm->l2_out_feat_next, + L2OUTPUT_FEAT_OUTPUT_CLASSIFY); + else + vnet_buffer (b0)->l2.feature_bitmap &= + ~L2OUTPUT_FEAT_OUTPUT_CLASSIFY; if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) && (b0->flags & VLIB_BUFFER_IS_TRACED))) @@ -493,7 +477,7 @@ l2_output_classify_init (vlib_main_t * vm) l2_output_classify_node.index, L2OUTPUT_N_FEAT, l2output_get_feat_names (), - cm->next_nodes.feat_next_node_index); + cm->l2_out_feat_next); rt->l2cm = cm; rt->vcm = cm->vnet_classify_main; diff --git a/src/vnet/l2/l2_rw.c b/src/vnet/l2/l2_rw.c index c54509d0..fec04774 100644 --- a/src/vnet/l2/l2_rw.c +++ b/src/vnet/l2/l2_rw.c @@ -179,8 +179,8 @@ l2_rw_node_fn (vlib_main_t * vm, while (n_left_from >= 4 && n_left_to_next >= 2) { - u32 bi0, next0, sw_if_index0, feature_bitmap0, rwe_index0; - u32 bi1, next1, sw_if_index1, feature_bitmap1, rwe_index1; + u32 bi0, next0, sw_if_index0, rwe_index0; + u32 bi1, next1, sw_if_index1, rwe_index1; vlib_buffer_t *b0, *b1; ethernet_header_t *h0, *h1; l2_rw_config_t *config0, *config1; @@ -273,16 +273,10 @@ l2_rw_node_fn (vlib_main_t * vm, } /* Update feature bitmap and get next feature index */ - feature_bitmap0 = - vnet_buffer (b0)->l2.feature_bitmap & ~L2INPUT_FEAT_RW; - feature_bitmap1 = - vnet_buffer (b1)->l2.feature_bitmap & ~L2INPUT_FEAT_RW; - vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap0; - vnet_buffer (b1)->l2.feature_bitmap = feature_bitmap1; - next0 = feat_bitmap_get_next_node_index (rw->feat_next_node_index, - feature_bitmap0); - next1 = feat_bitmap_get_next_node_index (rw->feat_next_node_index, - feature_bitmap1); + next0 = vnet_l2_feature_next (b0, rw->feat_next_node_index, + L2INPUT_FEAT_RW); + next1 = vnet_l2_feature_next (b1, rw->feat_next_node_index, + L2INPUT_FEAT_RW); vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, n_left_to_next, @@ -291,7 +285,7 @@ l2_rw_node_fn (vlib_main_t * vm, while (n_left_from > 0 && n_left_to_next > 0) { - u32 bi0, next0, sw_if_index0, feature_bitmap0, rwe_index0; + u32 bi0, next0, sw_if_index0, rwe_index0; vlib_buffer_t *b0; ethernet_header_t *h0; l2_rw_config_t *config0; @@ -341,11 +335,8 @@ l2_rw_node_fn (vlib_main_t * vm, } /* Update feature bitmap and get next feature index */ - feature_bitmap0 = - vnet_buffer (b0)->l2.feature_bitmap & ~L2INPUT_FEAT_RW; - vnet_buffer (b0)->l2.feature_bitmap = feature_bitmap0; - next0 = feat_bitmap_get_next_node_index (rw->feat_next_node_index, - feature_bitmap0); + next0 = vnet_l2_feature_next (b0, rw->feat_next_node_index, + L2INPUT_FEAT_RW); vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, diff --git a/src/vnet/policer/node_funcs.c b/src/vnet/policer/node_funcs.c index 457dd09f..fd031d02 100644 --- a/src/vnet/policer/node_funcs.c +++ b/src/vnet/policer/node_funcs.c @@ -736,14 +736,9 @@ policer_classify_inline (vlib_main_t * vm, if (tid == POLICER_CLASSIFY_TABLE_L2) { - /* Feature bitmap update */ - vnet_buffer (b0)->l2.feature_bitmap &= - ~L2INPUT_FEAT_POLICER_CLAS; - /* Determine the next node */ - next0 = - feat_bitmap_get_next_node_index (pcm->feat_next_node_index, - vnet_buffer (b0)-> - l2.feature_bitmap); + /* Feature bitmap update and determine the next node */ + next0 = vnet_l2_feature_next (b0, pcm->feat_next_node_index, + L2INPUT_FEAT_POLICER_CLAS); } else vnet_get_config_data (pcm->vnet_config_main[tid], -- cgit 1.2.3-korg