From 7cd468a3d7dee7d6c92f69a0bb7061ae208ec727 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 19 Dec 2016 23:05:39 +0100 Subject: Reorganize source tree to use single autotools instance Change-Id: I7b51f88292e057c6443b12224486f2d0c9f8ae23 Signed-off-by: Damjan Marion --- src/vnet/ethernet/node.c | 1368 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1368 insertions(+) create mode 100755 src/vnet/ethernet/node.c (limited to 'src/vnet/ethernet/node.c') diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c new file mode 100755 index 00000000..6d57da31 --- /dev/null +++ b/src/vnet/ethernet/node.c @@ -0,0 +1,1368 @@ +/* + * Copyright (c) 2015 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * ethernet_node.c: ethernet packet processing + * + * Copyright (c) 2008 Eliot Dresselhaus + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include + + +#define foreach_ethernet_input_next \ + _ (PUNT, "error-punt") \ + _ (DROP, "error-drop") \ + _ (LLC, "llc-input") + +typedef enum +{ +#define _(s,n) ETHERNET_INPUT_NEXT_##s, + foreach_ethernet_input_next +#undef _ + ETHERNET_INPUT_N_NEXT, +} ethernet_input_next_t; + +typedef struct +{ + u8 packet_data[32]; +} ethernet_input_trace_t; + +static u8 * +format_ethernet_input_trace (u8 * s, va_list * va) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *); + ethernet_input_trace_t *t = va_arg (*va, ethernet_input_trace_t *); + + s = format (s, "%U", format_ethernet_header, t->packet_data); + + return s; +} + +vlib_node_registration_t ethernet_input_node; + +typedef enum +{ + ETHERNET_INPUT_VARIANT_ETHERNET, + ETHERNET_INPUT_VARIANT_ETHERNET_TYPE, + ETHERNET_INPUT_VARIANT_NOT_L2, +} ethernet_input_variant_t; + + +// Parse the ethernet header to extract vlan tags and innermost ethertype +static_always_inline void +parse_header (ethernet_input_variant_t variant, + vlib_buffer_t * b0, + u16 * type, + u16 * orig_type, + u16 * outer_id, u16 * inner_id, u32 * match_flags) +{ + u8 vlan_count; + + if (variant == ETHERNET_INPUT_VARIANT_ETHERNET + || variant == ETHERNET_INPUT_VARIANT_NOT_L2) + { + ethernet_header_t *e0; + + e0 = (void *) (b0->data + b0->current_data); + + vnet_buffer (b0)->ethernet.start_of_ethernet_header = b0->current_data; + + vlib_buffer_advance (b0, sizeof (e0[0])); + + *type = clib_net_to_host_u16 (e0->type); + } + else if (variant == ETHERNET_INPUT_VARIANT_ETHERNET_TYPE) + { + // here when prior node was LLC/SNAP processing + u16 *e0; + + e0 = (void *) (b0->data + b0->current_data); + + vlib_buffer_advance (b0, sizeof (e0[0])); + + *type = clib_net_to_host_u16 (e0[0]); + } + + // save for distinguishing between dot1q and dot1ad later + *orig_type = *type; + + // default the tags to 0 (used if there is no corresponding tag) + *outer_id = 0; + *inner_id = 0; + + *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_0_TAG; + vlan_count = 0; + + // check for vlan encaps + if (ethernet_frame_is_tagged (*type)) + { + ethernet_vlan_header_t *h0; + u16 tag; + + *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_1_TAG; + + h0 = (void *) (b0->data + b0->current_data); + + tag = clib_net_to_host_u16 (h0->priority_cfi_and_id); + + *outer_id = tag & 0xfff; + + *type = clib_net_to_host_u16 (h0->type); + + vlib_buffer_advance (b0, sizeof (h0[0])); + vlan_count = 1; + + if (*type == ETHERNET_TYPE_VLAN) + { + // Double tagged packet + *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_2_TAG; + + h0 = (void *) (b0->data + b0->current_data); + + tag = clib_net_to_host_u16 (h0->priority_cfi_and_id); + + *inner_id = tag & 0xfff; + + *type = clib_net_to_host_u16 (h0->type); + + vlib_buffer_advance (b0, sizeof (h0[0])); + vlan_count = 2; + + if (*type == ETHERNET_TYPE_VLAN) + { + // More than double tagged packet + *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_3_TAG; + vlan_count = 3; // "unknown" number, aka, 3-or-more + } + } + } + ethernet_buffer_set_vlan_count (b0, vlan_count); +} + +// Determine the subinterface for this packet, given the result of the +// vlan table lookups and vlan header parsing. Check the most specific +// matches first. +static_always_inline void +identify_subint (vnet_hw_interface_t * hi, + vlib_buffer_t * b0, + u32 match_flags, + main_intf_t * main_intf, + vlan_intf_t * vlan_intf, + qinq_intf_t * qinq_intf, + u32 * new_sw_if_index, u8 * error0, u32 * is_l2) +{ + u32 matched; + + matched = eth_identify_subint (hi, b0, match_flags, + main_intf, vlan_intf, qinq_intf, + new_sw_if_index, error0, is_l2); + + if (matched) + { + + // Perform L3 my-mac filter + // A unicast packet arriving on an L3 interface must have a dmac matching the interface mac. + // This is required for promiscuous mode, else we will forward packets we aren't supposed to. + if (!(*is_l2)) + { + ethernet_header_t *e0; + e0 = + (void *) (b0->data + + vnet_buffer (b0)->ethernet.start_of_ethernet_header); + + if (!(ethernet_address_cast (e0->dst_address))) + { + if (!eth_mac_equal ((u8 *) e0, hi->hw_address)) + { + *error0 = ETHERNET_ERROR_L3_MAC_MISMATCH; + } + } + } + + // Check for down subinterface + *error0 = (*new_sw_if_index) != ~0 ? (*error0) : ETHERNET_ERROR_DOWN; + } +} + +static_always_inline void +determine_next_node (ethernet_main_t * em, + ethernet_input_variant_t variant, + u32 is_l20, + u32 type0, vlib_buffer_t * b0, u8 * error0, u8 * next0) +{ + if (PREDICT_FALSE (*error0 != ETHERNET_ERROR_NONE)) + { + // some error occurred + *next0 = ETHERNET_INPUT_NEXT_DROP; + } + else if (is_l20) + { + *next0 = em->l2_next; + // record the L2 len and reset the buffer so the L2 header is preserved + u32 eth_start = vnet_buffer (b0)->ethernet.start_of_ethernet_header; + vnet_buffer (b0)->l2.l2_len = b0->current_data - eth_start; + vlib_buffer_advance (b0, -ethernet_buffer_header_size (b0)); + + // check for common IP/MPLS ethertypes + } + else if (type0 == ETHERNET_TYPE_IP4) + { + *next0 = em->l3_next.input_next_ip4; + } + else if (type0 == ETHERNET_TYPE_IP6) + { + *next0 = em->l3_next.input_next_ip6; + } + else if (type0 == ETHERNET_TYPE_MPLS_UNICAST) + { + *next0 = em->l3_next.input_next_mpls; + + } + else if (em->redirect_l3) + { + // L3 Redirect is on, the cached common next nodes will be + // pointing to the redirect node, catch the uncommon types here + *next0 = em->redirect_l3_next; + } + else + { + // uncommon ethertype, check table + u32 i0; + i0 = sparse_vec_index (em->l3_next.input_next_by_type, type0); + *next0 = vec_elt (em->l3_next.input_next_by_type, i0); + *error0 = + i0 == + SPARSE_VEC_INVALID_INDEX ? ETHERNET_ERROR_UNKNOWN_TYPE : *error0; + + // The table is not populated with LLC values, so check that now. + // If variant is variant_ethernet then we came from LLC processing. Don't + // go back there; drop instead using by keeping the drop/bad table result. + if ((type0 < 0x600) && (variant == ETHERNET_INPUT_VARIANT_ETHERNET)) + { + *next0 = ETHERNET_INPUT_NEXT_LLC; + } + } +} + +static_always_inline uword +ethernet_input_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame, + ethernet_input_variant_t variant) +{ + vnet_main_t *vnm = vnet_get_main (); + ethernet_main_t *em = ðernet_main; + vlib_node_runtime_t *error_node; + u32 n_left_from, next_index, *from, *to_next; + u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; + u32 cpu_index = os_get_cpu_number (); + u32 cached_sw_if_index = ~0; + u32 cached_is_l2 = 0; /* shut up gcc */ + + if (variant != ETHERNET_INPUT_VARIANT_ETHERNET) + error_node = vlib_node_get_runtime (vm, ethernet_input_node.index); + else + error_node = node; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + if (node->flags & VLIB_NODE_FLAG_TRACE) + vlib_trace_frame_buffers_only (vm, node, + from, + n_left_from, + sizeof (from[0]), + sizeof (ethernet_input_trace_t)); + + next_index = node->cached_next_index; + stats_sw_if_index = node->runtime_data[0]; + stats_n_packets = stats_n_bytes = 0; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + u32 bi0, bi1; + vlib_buffer_t *b0, *b1; + u8 next0, next1, error0, error1; + u16 type0, orig_type0, type1, orig_type1; + u16 outer_id0, inner_id0, outer_id1, inner_id1; + u32 match_flags0, match_flags1; + u32 old_sw_if_index0, new_sw_if_index0, len0, old_sw_if_index1, + new_sw_if_index1, len1; + vnet_hw_interface_t *hi0, *hi1; + main_intf_t *main_intf0, *main_intf1; + vlan_intf_t *vlan_intf0, *vlan_intf1; + qinq_intf_t *qinq_intf0, *qinq_intf1; + u32 is_l20, is_l21; + ethernet_header_t *e0, *e1; + + /* Prefetch next iteration. */ + { + vlib_buffer_t *b2, *b3; + + b2 = vlib_get_buffer (vm, from[2]); + b3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (b2, STORE); + vlib_prefetch_buffer_header (b3, STORE); + + CLIB_PREFETCH (b2->data, sizeof (ethernet_header_t), LOAD); + CLIB_PREFETCH (b3->data, sizeof (ethernet_header_t), LOAD); + } + + bi0 = from[0]; + bi1 = from[1]; + to_next[0] = bi0; + to_next[1] = bi1; + from += 2; + to_next += 2; + n_left_to_next -= 2; + n_left_from -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + error0 = error1 = ETHERNET_ERROR_NONE; + e0 = vlib_buffer_get_current (b0); + type0 = clib_net_to_host_u16 (e0->type); + e1 = vlib_buffer_get_current (b1); + type1 = clib_net_to_host_u16 (e1->type); + + /* Speed-path for the untagged case */ + if (PREDICT_TRUE (variant == ETHERNET_INPUT_VARIANT_ETHERNET + && !ethernet_frame_is_tagged (type0) + && !ethernet_frame_is_tagged (type1))) + { + main_intf_t *intf0; + subint_config_t *subint0; + u32 sw_if_index0, sw_if_index1; + + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX]; + is_l20 = cached_is_l2; + + /* This is probably wholly unnecessary */ + if (PREDICT_FALSE (sw_if_index0 != sw_if_index1)) + goto slowpath; + + if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0)) + { + cached_sw_if_index = sw_if_index0; + hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); + intf0 = vec_elt_at_index (em->main_intfs, hi0->hw_if_index); + subint0 = &intf0->untagged_subint; + cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2; + } + + vnet_buffer (b0)->ethernet.start_of_ethernet_header = + b0->current_data; + vnet_buffer (b1)->ethernet.start_of_ethernet_header = + b1->current_data; + + if (PREDICT_TRUE (is_l20 != 0)) + { + next0 = em->l2_next; + vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t); + next1 = em->l2_next; + vnet_buffer (b1)->l2.l2_len = sizeof (ethernet_header_t); + } + else + { + determine_next_node (em, variant, 0, type0, b0, + &error0, &next0); + vlib_buffer_advance (b0, sizeof (ethernet_header_t)); + determine_next_node (em, variant, 0, type1, b1, + &error1, &next1); + vlib_buffer_advance (b1, sizeof (ethernet_header_t)); + } + goto ship_it01; + } + + /* Slow-path for the tagged case */ + slowpath: + parse_header (variant, + b0, + &type0, + &orig_type0, &outer_id0, &inner_id0, &match_flags0); + + parse_header (variant, + b1, + &type1, + &orig_type1, &outer_id1, &inner_id1, &match_flags1); + + old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + old_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX]; + + eth_vlan_table_lookups (em, + vnm, + old_sw_if_index0, + orig_type0, + outer_id0, + inner_id0, + &hi0, + &main_intf0, &vlan_intf0, &qinq_intf0); + + eth_vlan_table_lookups (em, + vnm, + old_sw_if_index1, + orig_type1, + outer_id1, + inner_id1, + &hi1, + &main_intf1, &vlan_intf1, &qinq_intf1); + + identify_subint (hi0, + b0, + match_flags0, + main_intf0, + vlan_intf0, + qinq_intf0, &new_sw_if_index0, &error0, &is_l20); + + identify_subint (hi1, + b1, + match_flags1, + main_intf1, + vlan_intf1, + qinq_intf1, &new_sw_if_index1, &error1, &is_l21); + + // Save RX sw_if_index for later nodes + vnet_buffer (b0)->sw_if_index[VLIB_RX] = + error0 != + ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0; + vnet_buffer (b1)->sw_if_index[VLIB_RX] = + error1 != + ETHERNET_ERROR_NONE ? old_sw_if_index1 : new_sw_if_index1; + + // Check if there is a stat to take (valid and non-main sw_if_index for pkt 0 or pkt 1) + if (((new_sw_if_index0 != ~0) + && (new_sw_if_index0 != old_sw_if_index0)) + || ((new_sw_if_index1 != ~0) + && (new_sw_if_index1 != old_sw_if_index1))) + { + + len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data + - vnet_buffer (b0)->ethernet.start_of_ethernet_header; + len1 = vlib_buffer_length_in_chain (vm, b1) + b1->current_data + - vnet_buffer (b1)->ethernet.start_of_ethernet_header; + + stats_n_packets += 2; + stats_n_bytes += len0 + len1; + + if (PREDICT_FALSE + (!(new_sw_if_index0 == stats_sw_if_index + && new_sw_if_index1 == stats_sw_if_index))) + { + stats_n_packets -= 2; + stats_n_bytes -= len0 + len1; + + if (new_sw_if_index0 != old_sw_if_index0 + && new_sw_if_index0 != ~0) + vlib_increment_combined_counter (vnm-> + interface_main.combined_sw_if_counters + + + VNET_INTERFACE_COUNTER_RX, + cpu_index, + new_sw_if_index0, 1, + len0); + if (new_sw_if_index1 != old_sw_if_index1 + && new_sw_if_index1 != ~0) + vlib_increment_combined_counter (vnm-> + interface_main.combined_sw_if_counters + + + VNET_INTERFACE_COUNTER_RX, + cpu_index, + new_sw_if_index1, 1, + len1); + + if (new_sw_if_index0 == new_sw_if_index1) + { + if (stats_n_packets > 0) + { + vlib_increment_combined_counter + (vnm->interface_main.combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + cpu_index, + stats_sw_if_index, + stats_n_packets, stats_n_bytes); + stats_n_packets = stats_n_bytes = 0; + } + stats_sw_if_index = new_sw_if_index0; + } + } + } + + if (variant == ETHERNET_INPUT_VARIANT_NOT_L2) + is_l20 = is_l21 = 0; + + determine_next_node (em, variant, is_l20, type0, b0, &error0, + &next0); + determine_next_node (em, variant, is_l21, type1, b1, &error1, + &next1); + + b0->error = error_node->errors[error0]; + b1->error = error_node->errors[error1]; + + ship_it01: + // verify speculative enqueue + vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, + n_left_to_next, bi0, bi1, next0, + next1); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u8 error0, next0; + u16 type0, orig_type0; + u16 outer_id0, inner_id0; + u32 match_flags0; + u32 old_sw_if_index0, new_sw_if_index0, len0; + vnet_hw_interface_t *hi0; + main_intf_t *main_intf0; + vlan_intf_t *vlan_intf0; + qinq_intf_t *qinq_intf0; + ethernet_header_t *e0; + u32 is_l20; + + // Prefetch next iteration + if (n_left_from > 1) + { + vlib_buffer_t *p2; + + p2 = vlib_get_buffer (vm, from[1]); + vlib_prefetch_buffer_header (p2, STORE); + CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); + } + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + error0 = ETHERNET_ERROR_NONE; + e0 = vlib_buffer_get_current (b0); + type0 = clib_net_to_host_u16 (e0->type); + + /* Speed-path for the untagged case */ + if (PREDICT_TRUE (variant == ETHERNET_INPUT_VARIANT_ETHERNET + && !ethernet_frame_is_tagged (type0))) + { + main_intf_t *intf0; + subint_config_t *subint0; + u32 sw_if_index0; + + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + is_l20 = cached_is_l2; + + if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0)) + { + cached_sw_if_index = sw_if_index0; + hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); + intf0 = vec_elt_at_index (em->main_intfs, hi0->hw_if_index); + subint0 = &intf0->untagged_subint; + cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2; + } + + vnet_buffer (b0)->ethernet.start_of_ethernet_header = + b0->current_data; + + if (PREDICT_TRUE (is_l20 != 0)) + { + next0 = em->l2_next; + vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t); + } + else + { + determine_next_node (em, variant, 0, type0, b0, + &error0, &next0); + vlib_buffer_advance (b0, sizeof (ethernet_header_t)); + } + goto ship_it0; + } + + /* Slow-path for the tagged case */ + parse_header (variant, + b0, + &type0, + &orig_type0, &outer_id0, &inner_id0, &match_flags0); + + old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + + eth_vlan_table_lookups (em, + vnm, + old_sw_if_index0, + orig_type0, + outer_id0, + inner_id0, + &hi0, + &main_intf0, &vlan_intf0, &qinq_intf0); + + identify_subint (hi0, + b0, + match_flags0, + main_intf0, + vlan_intf0, + qinq_intf0, &new_sw_if_index0, &error0, &is_l20); + + // Save RX sw_if_index for later nodes + vnet_buffer (b0)->sw_if_index[VLIB_RX] = + error0 != + ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0; + + // Increment subinterface stats + // Note that interface-level counters have already been incremented + // prior to calling this function. Thus only subinterface counters + // are incremented here. + // + // Interface level counters include packets received on the main + // interface and all subinterfaces. Subinterface level counters + // include only those packets received on that subinterface + // Increment stats if the subint is valid and it is not the main intf + if ((new_sw_if_index0 != ~0) + && (new_sw_if_index0 != old_sw_if_index0)) + { + + len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data + - vnet_buffer (b0)->ethernet.start_of_ethernet_header; + + stats_n_packets += 1; + stats_n_bytes += len0; + + // Batch stat increments from the same subinterface so counters + // don't need to be incremented for every packet. + if (PREDICT_FALSE (new_sw_if_index0 != stats_sw_if_index)) + { + stats_n_packets -= 1; + stats_n_bytes -= len0; + + if (new_sw_if_index0 != ~0) + vlib_increment_combined_counter + (vnm->interface_main.combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + cpu_index, new_sw_if_index0, 1, len0); + if (stats_n_packets > 0) + { + vlib_increment_combined_counter + (vnm->interface_main.combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + cpu_index, + stats_sw_if_index, stats_n_packets, stats_n_bytes); + stats_n_packets = stats_n_bytes = 0; + } + stats_sw_if_index = new_sw_if_index0; + } + } + + if (variant == ETHERNET_INPUT_VARIANT_NOT_L2) + is_l20 = 0; + + determine_next_node (em, variant, is_l20, type0, b0, &error0, + &next0); + + b0->error = error_node->errors[error0]; + + // verify speculative enqueue + ship_it0: + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + // Increment any remaining batched stats + if (stats_n_packets > 0) + { + vlib_increment_combined_counter + (vnm->interface_main.combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + node->runtime_data[0] = stats_sw_if_index; + } + + return from_frame->n_vectors; +} + +static uword +ethernet_input (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * from_frame) +{ + return ethernet_input_inline (vm, node, from_frame, + ETHERNET_INPUT_VARIANT_ETHERNET); +} + +static uword +ethernet_input_type (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * from_frame) +{ + return ethernet_input_inline (vm, node, from_frame, + ETHERNET_INPUT_VARIANT_ETHERNET_TYPE); +} + +static uword +ethernet_input_not_l2 (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * from_frame) +{ + return ethernet_input_inline (vm, node, from_frame, + ETHERNET_INPUT_VARIANT_NOT_L2); +} + + +// Return the subinterface config struct for the given sw_if_index +// Also return via parameter the appropriate match flags for the +// configured number of tags. +// On error (unsupported or not ethernet) return 0. +static subint_config_t * +ethernet_sw_interface_get_config (vnet_main_t * vnm, + u32 sw_if_index, + u32 * flags, u32 * unsupported) +{ + ethernet_main_t *em = ðernet_main; + vnet_hw_interface_t *hi; + vnet_sw_interface_t *si; + main_intf_t *main_intf; + vlan_table_t *vlan_table; + qinq_table_t *qinq_table; + subint_config_t *subint = 0; + + hi = vnet_get_sup_hw_interface (vnm, sw_if_index); + + if (!hi || (hi->hw_class_index != ethernet_hw_interface_class.index)) + { + *unsupported = 0; + goto done; // non-ethernet interface + } + + // ensure there's an entry for the main intf (shouldn't really be necessary) + vec_validate (em->main_intfs, hi->hw_if_index); + main_intf = vec_elt_at_index (em->main_intfs, hi->hw_if_index); + + // Locate the subint for the given ethernet config + si = vnet_get_sw_interface (vnm, sw_if_index); + + if (si->sub.eth.flags.default_sub) + { + subint = &main_intf->default_subint; + *flags = SUBINT_CONFIG_MATCH_0_TAG | + SUBINT_CONFIG_MATCH_1_TAG | + SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG; + } + else if ((si->sub.eth.flags.no_tags) || (si->sub.eth.raw_flags == 0)) + { + // if no flags are set then this is a main interface + // so treat as untagged + subint = &main_intf->untagged_subint; + *flags = SUBINT_CONFIG_MATCH_0_TAG; + } + else + { + // one or two tags + // first get the vlan table + if (si->sub.eth.flags.dot1ad) + { + if (main_intf->dot1ad_vlans == 0) + { + // Allocate a vlan table from the pool + pool_get (em->vlan_pool, vlan_table); + main_intf->dot1ad_vlans = vlan_table - em->vlan_pool; + } + else + { + // Get ptr to existing vlan table + vlan_table = + vec_elt_at_index (em->vlan_pool, main_intf->dot1ad_vlans); + } + } + else + { // dot1q + if (main_intf->dot1q_vlans == 0) + { + // Allocate a vlan table from the pool + pool_get (em->vlan_pool, vlan_table); + main_intf->dot1q_vlans = vlan_table - em->vlan_pool; + } + else + { + // Get ptr to existing vlan table + vlan_table = + vec_elt_at_index (em->vlan_pool, main_intf->dot1q_vlans); + } + } + + if (si->sub.eth.flags.one_tag) + { + *flags = si->sub.eth.flags.exact_match ? + SUBINT_CONFIG_MATCH_1_TAG : + (SUBINT_CONFIG_MATCH_1_TAG | + SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG); + + if (si->sub.eth.flags.outer_vlan_id_any) + { + // not implemented yet + *unsupported = 1; + goto done; + } + else + { + // a single vlan, a common case + subint = + &vlan_table->vlans[si->sub.eth. + outer_vlan_id].single_tag_subint; + } + + } + else + { + // Two tags + *flags = si->sub.eth.flags.exact_match ? + SUBINT_CONFIG_MATCH_2_TAG : + (SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG); + + if (si->sub.eth.flags.outer_vlan_id_any + && si->sub.eth.flags.inner_vlan_id_any) + { + // not implemented yet + *unsupported = 1; + goto done; + } + + if (si->sub.eth.flags.inner_vlan_id_any) + { + // a specific outer and "any" inner + // don't need a qinq table for this + subint = + &vlan_table->vlans[si->sub.eth. + outer_vlan_id].inner_any_subint; + if (si->sub.eth.flags.exact_match) + { + *flags = SUBINT_CONFIG_MATCH_2_TAG; + } + else + { + *flags = SUBINT_CONFIG_MATCH_2_TAG | + SUBINT_CONFIG_MATCH_3_TAG; + } + } + else + { + // a specific outer + specifc innner vlan id, a common case + + // get the qinq table + if (vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs == 0) + { + // Allocate a qinq table from the pool + pool_get (em->qinq_pool, qinq_table); + vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs = + qinq_table - em->qinq_pool; + } + else + { + // Get ptr to existing qinq table + qinq_table = + vec_elt_at_index (em->qinq_pool, + vlan_table->vlans[si->sub. + eth.outer_vlan_id]. + qinqs); + } + subint = &qinq_table->vlans[si->sub.eth.inner_vlan_id].subint; + } + } + } + +done: + return subint; +} + +clib_error_t * +ethernet_sw_interface_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags) +{ + subint_config_t *subint; + u32 dummy_flags; + u32 dummy_unsup; + clib_error_t *error = 0; + + // Find the config for this subinterface + subint = + ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags, + &dummy_unsup); + + if (subint == 0) + { + // not implemented yet or not ethernet + goto done; + } + + subint->sw_if_index = + ((flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ? sw_if_index : ~0); + +done: + return error; +} + +VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ethernet_sw_interface_up_down); + + +// Set the L2/L3 mode for the subinterface +void +ethernet_sw_interface_set_l2_mode (vnet_main_t * vnm, u32 sw_if_index, u32 l2) +{ + subint_config_t *subint; + u32 dummy_flags; + u32 dummy_unsup; + int is_port; + vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, sw_if_index); + + is_port = !(sw->type == VNET_SW_INTERFACE_TYPE_SUB); + + // Find the config for this subinterface + subint = + ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags, + &dummy_unsup); + + if (subint == 0) + { + // unimplemented or not ethernet + goto done; + } + + // Double check that the config we found is for our interface (or the interface is down) + ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0)); + + if (l2) + { + subint->flags |= SUBINT_CONFIG_L2; + if (is_port) + subint->flags |= + SUBINT_CONFIG_MATCH_0_TAG | SUBINT_CONFIG_MATCH_1_TAG + | SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG; + } + else + { + subint->flags &= ~SUBINT_CONFIG_L2; + if (is_port) + subint->flags &= + ~(SUBINT_CONFIG_MATCH_1_TAG | SUBINT_CONFIG_MATCH_2_TAG + | SUBINT_CONFIG_MATCH_3_TAG); + } + +done: + return; +} + +/* + * Set the L2/L3 mode for the subinterface regardless of port + */ +void +ethernet_sw_interface_set_l2_mode_noport (vnet_main_t * vnm, + u32 sw_if_index, u32 l2) +{ + subint_config_t *subint; + u32 dummy_flags; + u32 dummy_unsup; + + /* Find the config for this subinterface */ + subint = + ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags, + &dummy_unsup); + + if (subint == 0) + { + /* unimplemented or not ethernet */ + goto done; + } + + /* + * Double check that the config we found is for our interface (or the + * interface is down) + */ + ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0)); + + if (l2) + { + subint->flags |= SUBINT_CONFIG_L2; + } + else + { + subint->flags &= ~SUBINT_CONFIG_L2; + } + +done: + return; +} + +static clib_error_t * +ethernet_sw_interface_add_del (vnet_main_t * vnm, + u32 sw_if_index, u32 is_create) +{ + clib_error_t *error = 0; + subint_config_t *subint; + u32 match_flags; + u32 unsupported = 0; + + // Find the config for this subinterface + subint = + ethernet_sw_interface_get_config (vnm, sw_if_index, &match_flags, + &unsupported); + + if (subint == 0) + { + // not implemented yet or not ethernet + if (unsupported) + { + // this is the NYI case + error = clib_error_return (0, "not implemented yet"); + } + goto done; + } + + if (!is_create) + { + subint->flags = 0; + return error; + } + + // Initialize the subint + if (subint->flags & SUBINT_CONFIG_VALID) + { + // Error vlan already in use + error = clib_error_return (0, "vlan is already in use"); + } + else + { + // Note that config is L3 by defaulty + subint->flags = SUBINT_CONFIG_VALID | match_flags; + subint->sw_if_index = ~0; // because interfaces are initially down + } + +done: + return error; +} + +VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ethernet_sw_interface_add_del); + +static char *ethernet_error_strings[] = { +#define ethernet_error(n,c,s) s, +#include "error.def" +#undef ethernet_error +}; + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (ethernet_input_node) = { + .function = ethernet_input, + .name = "ethernet-input", + /* Takes a vector of packets. */ + .vector_size = sizeof (u32), + .n_errors = ETHERNET_N_ERROR, + .error_strings = ethernet_error_strings, + .n_next_nodes = ETHERNET_INPUT_N_NEXT, + .next_nodes = { +#define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n, + foreach_ethernet_input_next +#undef _ + }, + .format_buffer = format_ethernet_header_with_length, + .format_trace = format_ethernet_input_trace, + .unformat_buffer = unformat_ethernet_header, +}; +/* *INDENT-ON* */ + +/* *INDENT-OFF* */ +VLIB_NODE_FUNCTION_MULTIARCH (ethernet_input_node, ethernet_input) +/* *INDENT-ON* */ + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (ethernet_input_type_node, static) = { + .function = ethernet_input_type, + .name = "ethernet-input-type", + /* Takes a vector of packets. */ + .vector_size = sizeof (u32), + .n_next_nodes = ETHERNET_INPUT_N_NEXT, + .next_nodes = { +#define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n, + foreach_ethernet_input_next +#undef _ + }, +}; +/* *INDENT-ON* */ + +/* *INDENT-OFF* */ +VLIB_NODE_FUNCTION_MULTIARCH (ethernet_input_type_node, ethernet_input_type) +/* *INDENT-ON* */ + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (ethernet_input_not_l2_node, static) = { + .function = ethernet_input_not_l2, + .name = "ethernet-input-not-l2", + /* Takes a vector of packets. */ + .vector_size = sizeof (u32), + .n_next_nodes = ETHERNET_INPUT_N_NEXT, + .next_nodes = { +#define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n, + foreach_ethernet_input_next +#undef _ + }, +}; +/* *INDENT-ON* */ + + +/* *INDENT-OFF* */ +VLIB_NODE_FUNCTION_MULTIARCH (ethernet_input_not_l2_node, + ethernet_input_not_l2) +/* *INDENT-ON* */ + + +void +ethernet_set_rx_redirect (vnet_main_t * vnm, + vnet_hw_interface_t * hi, u32 enable) +{ + // Insure all packets go to ethernet-input (i.e. untagged ipv4 packets + // don't go directly to ip4-input) + vnet_hw_interface_rx_redirect_to_node + (vnm, hi->hw_if_index, enable ? ethernet_input_node.index : ~0); +} + + +/* + * Initialization and registration for the next_by_ethernet structure + */ + +clib_error_t * +next_by_ethertype_init (next_by_ethertype_t * l3_next) +{ + l3_next->input_next_by_type = sparse_vec_new + ( /* elt bytes */ sizeof (l3_next->input_next_by_type[0]), + /* bits in index */ BITS (((ethernet_header_t *) 0)->type)); + + vec_validate (l3_next->sparse_index_by_input_next_index, + ETHERNET_INPUT_NEXT_DROP); + vec_validate (l3_next->sparse_index_by_input_next_index, + ETHERNET_INPUT_NEXT_PUNT); + l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_DROP] = + SPARSE_VEC_INVALID_INDEX; + l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_PUNT] = + SPARSE_VEC_INVALID_INDEX; + + /* + * Make sure we don't wipe out an ethernet registration by mistake + * Can happen if init function ordering constraints are missing. + */ + if (CLIB_DEBUG > 0) + { + ethernet_main_t *em = ðernet_main; + ASSERT (em->next_by_ethertype_register_called == 0); + } + + return 0; +} + +// Add an ethertype -> next index mapping to the structure +clib_error_t * +next_by_ethertype_register (next_by_ethertype_t * l3_next, + u32 ethertype, u32 next_index) +{ + u32 i; + u16 *n; + ethernet_main_t *em = ðernet_main; + + if (CLIB_DEBUG > 0) + { + ethernet_main_t *em = ðernet_main; + em->next_by_ethertype_register_called = 1; + } + + /* Setup ethernet type -> next index sparse vector mapping. */ + n = sparse_vec_validate (l3_next->input_next_by_type, ethertype); + n[0] = next_index; + + /* Rebuild next index -> sparse index inverse mapping when sparse vector + is updated. */ + vec_validate (l3_next->sparse_index_by_input_next_index, next_index); + for (i = 1; i < vec_len (l3_next->input_next_by_type); i++) + l3_next-> + sparse_index_by_input_next_index[l3_next->input_next_by_type[i]] = i; + + // do not allow the cached next index's to be updated if L3 + // redirect is enabled, as it will have overwritten them + if (!em->redirect_l3) + { + // Cache common ethertypes directly + if (ethertype == ETHERNET_TYPE_IP4) + { + l3_next->input_next_ip4 = next_index; + } + else if (ethertype == ETHERNET_TYPE_IP6) + { + l3_next->input_next_ip6 = next_index; + } + else if (ethertype == ETHERNET_TYPE_MPLS_UNICAST) + { + l3_next->input_next_mpls = next_index; + } + } + return 0; +} + + +static clib_error_t * +ethernet_input_init (vlib_main_t * vm) +{ + ethernet_main_t *em = ðernet_main; + __attribute__ ((unused)) vlan_table_t *invalid_vlan_table; + __attribute__ ((unused)) qinq_table_t *invalid_qinq_table; + + ethernet_setup_node (vm, ethernet_input_node.index); + ethernet_setup_node (vm, ethernet_input_type_node.index); + ethernet_setup_node (vm, ethernet_input_not_l2_node.index); + + next_by_ethertype_init (&em->l3_next); + + // Initialize pools and vector for vlan parsing + vec_validate (em->main_intfs, 10); // 10 main interfaces + pool_alloc (em->vlan_pool, 10); + pool_alloc (em->qinq_pool, 1); + + // The first vlan pool will always be reserved for an invalid table + pool_get (em->vlan_pool, invalid_vlan_table); // first id = 0 + // The first qinq pool will always be reserved for an invalid table + pool_get (em->qinq_pool, invalid_qinq_table); // first id = 0 + + return 0; +} + +VLIB_INIT_FUNCTION (ethernet_input_init); + +void +ethernet_register_input_type (vlib_main_t * vm, + ethernet_type_t type, u32 node_index) +{ + ethernet_main_t *em = ðernet_main; + ethernet_type_info_t *ti; + u32 i; + + { + clib_error_t *error = vlib_call_init_function (vm, ethernet_init); + if (error) + clib_error_report (error); + } + + ti = ethernet_get_type_info (em, type); + ti->node_index = node_index; + ti->next_index = vlib_node_add_next (vm, + ethernet_input_node.index, node_index); + i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index); + ASSERT (i == ti->next_index); + + i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index); + ASSERT (i == ti->next_index); + + // Add the L3 node for this ethertype to the next nodes structure + next_by_ethertype_register (&em->l3_next, type, ti->next_index); + + // Call the registration functions for other nodes that want a mapping + l2bvi_register_input_type (vm, type, node_index); +} + +void +ethernet_register_l2_input (vlib_main_t * vm, u32 node_index) +{ + ethernet_main_t *em = ðernet_main; + u32 i; + + em->l2_next = + vlib_node_add_next (vm, ethernet_input_node.index, node_index); + + /* + * Even if we never use these arcs, we have to align the next indices... + */ + i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index); + + ASSERT (i == em->l2_next); + + i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index); + ASSERT (i == em->l2_next); +} + +// Register a next node for L3 redirect, and enable L3 redirect +void +ethernet_register_l3_redirect (vlib_main_t * vm, u32 node_index) +{ + ethernet_main_t *em = ðernet_main; + u32 i; + + em->redirect_l3 = 1; + em->redirect_l3_next = vlib_node_add_next (vm, + ethernet_input_node.index, + node_index); + /* + * Change the cached next nodes to the redirect node + */ + em->l3_next.input_next_ip4 = em->redirect_l3_next; + em->l3_next.input_next_ip6 = em->redirect_l3_next; + em->l3_next.input_next_mpls = em->redirect_l3_next; + + /* + * Even if we never use these arcs, we have to align the next indices... + */ + i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index); + + ASSERT (i == em->redirect_l3_next); + + i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index); + + ASSERT (i == em->redirect_l3_next); +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ -- cgit 1.2.3-korg From 1904c47d5f79f8e9dcc6c9ae2043e7d6be8bd6d8 Mon Sep 17 00:00:00 2001 From: John Lo Date: Fri, 10 Mar 2017 17:15:22 -0500 Subject: Add MAC address check in ethernet-input node if interface in L3 mode Interface can be in promiscuous mode if more than one of its sub- interface is in L2 mode. In promiscuous mode, L3 interface need to verify DMAC of packet to match that of the interface and drop if not. This check was done on sub-interface only and now also added to main interface path. Fix incorrect MAC addresses in the flow-per-pkt plugin test, which caused it to fail. Fix MAC address usage in BFD tests. Change-Id: I12a17ec05c7ab298ad10d400c90d082c97eca521 Signed-off-by: John Lo Signed-off-by: Klement Sekera --- src/vnet/ethernet/node.c | 23 +++++++++++++++++------ test/test_bfd.py | 11 ++++++++++- test/test_flowperpkt.py | 2 +- 3 files changed, 28 insertions(+), 8 deletions(-) (limited to 'src/vnet/ethernet/node.c') diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index 6d57da31..b699e381 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -294,6 +294,7 @@ ethernet_input_inline (vlib_main_t * vm, u32 cpu_index = os_get_cpu_number (); u32 cached_sw_if_index = ~0; u32 cached_is_l2 = 0; /* shut up gcc */ + vnet_hw_interface_t *hi = NULL; /* used for main interface only */ if (variant != ETHERNET_INPUT_VARIANT_ETHERNET) error_node = vlib_node_get_runtime (vm, ethernet_input_node.index); @@ -386,11 +387,12 @@ ethernet_input_inline (vlib_main_t * vm, if (PREDICT_FALSE (sw_if_index0 != sw_if_index1)) goto slowpath; + /* Now sw_if_index0 == sw_if_index1 */ if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0)) { cached_sw_if_index = sw_if_index0; - hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); - intf0 = vec_elt_at_index (em->main_intfs, hi0->hw_if_index); + hi = vnet_get_sup_hw_interface (vnm, sw_if_index0); + intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index); subint0 = &intf0->untagged_subint; cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2; } @@ -409,6 +411,12 @@ ethernet_input_inline (vlib_main_t * vm, } else { + if (!ethernet_address_cast (e0->dst_address) && + !eth_mac_equal ((u8 *) e0, hi->hw_address)) + error0 = ETHERNET_ERROR_L3_MAC_MISMATCH; + if (!ethernet_address_cast (e1->dst_address) && + !eth_mac_equal ((u8 *) e1, hi->hw_address)) + error1 = ETHERNET_ERROR_L3_MAC_MISMATCH; determine_next_node (em, variant, 0, type0, b0, &error0, &next0); vlib_buffer_advance (b0, sizeof (ethernet_header_t)); @@ -540,10 +548,10 @@ ethernet_input_inline (vlib_main_t * vm, determine_next_node (em, variant, is_l21, type1, b1, &error1, &next1); + ship_it01: b0->error = error_node->errors[error0]; b1->error = error_node->errors[error1]; - ship_it01: // verify speculative enqueue vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, @@ -603,8 +611,8 @@ ethernet_input_inline (vlib_main_t * vm, if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0)) { cached_sw_if_index = sw_if_index0; - hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); - intf0 = vec_elt_at_index (em->main_intfs, hi0->hw_if_index); + hi = vnet_get_sup_hw_interface (vnm, sw_if_index0); + intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index); subint0 = &intf0->untagged_subint; cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2; } @@ -619,6 +627,9 @@ ethernet_input_inline (vlib_main_t * vm, } else { + if (!ethernet_address_cast (e0->dst_address) && + !eth_mac_equal ((u8 *) e0, hi->hw_address)) + error0 = ETHERNET_ERROR_L3_MAC_MISMATCH; determine_next_node (em, variant, 0, type0, b0, &error0, &next0); vlib_buffer_advance (b0, sizeof (ethernet_header_t)); @@ -705,10 +716,10 @@ ethernet_input_inline (vlib_main_t * vm, determine_next_node (em, variant, is_l20, type0, b0, &error0, &next0); + ship_it0: b0->error = error_node->errors[error0]; // verify speculative enqueue - ship_it0: vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0); diff --git a/test/test_bfd.py b/test/test_bfd.py index 5460a2b5..4e3f688b 100644 --- a/test/test_bfd.py +++ b/test/test_bfd.py @@ -1090,6 +1090,9 @@ class BFD4TestCase(VppTestCase): self.assertNotEqual(p[IP].src, self.loopback0.local_ip4, "BFD ECHO src IP equal to loopback IP") self.logger.debug(ppp("Looping back packet:", p)) + self.assert_equal(p[Ether].dst, self.pg0.remote_mac, + "ECHO packet destination MAC address") + p[Ether].dst = self.pg0.local_mac self.pg0.add_stream(p) self.pg_start() elif p.haslayer(BFD): @@ -1159,6 +1162,7 @@ class BFD4TestCase(VppTestCase): self.logger.debug(ppp("Got packet:", p)) if p[UDP].dport == BFD.udp_dport_echo: self.logger.debug(ppp("Looping back packet:", p)) + p[Ether].dst = self.pg0.local_mac self.pg0.add_stream(p) self.pg_start() break @@ -1190,6 +1194,7 @@ class BFD4TestCase(VppTestCase): self.logger.debug(ppp("Got packet:", p)) if p[UDP].dport == BFD.udp_dport_echo: self.logger.debug(ppp("Looping back packet:", p)) + p[Ether].dst = self.pg0.local_mac self.pg0.add_stream(p) self.pg_start() break @@ -1230,6 +1235,7 @@ class BFD4TestCase(VppTestCase): else: self.logger.debug(ppp("Got followup echo packet:", p)) self.logger.debug(ppp("Looping back first echo packet:", p)) + echo_packet[Ether].dst = self.pg0.local_mac self.pg0.add_stream(echo_packet) self.pg_start() elif p.haslayer(BFD): @@ -1278,6 +1284,7 @@ class BFD4TestCase(VppTestCase): timeout_at = time.time() + self.vpp_session.detect_mult * \ self.test_session.required_min_echo_rx / USEC_IN_SEC p[BFD_vpp_echo].checksum = getrandbits(64) + p[Ether].dst = self.pg0.local_mac self.logger.debug(ppp("Looping back modified echo packet:", p)) self.pg0.add_stream(p) self.pg_start() @@ -1523,7 +1530,6 @@ class BFD6TestCase(VppTestCase): self.assert_equal(udp_sport_tx, udp_sport_rx, "UDP source port (== " "ECHO packet identifier for test purposes)") - @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_echo(self): """ echo function used """ bfd_session_up(self) @@ -1554,6 +1560,9 @@ class BFD6TestCase(VppTestCase): self.assertNotEqual(p[IPv6].src, self.loopback0.local_ip6, "BFD ECHO src IP equal to loopback IP") self.logger.debug(ppp("Looping back packet:", p)) + self.assert_equal(p[Ether].dst, self.pg0.remote_mac, + "ECHO packet destination MAC address") + p[Ether].dst = self.pg0.local_mac self.pg0.add_stream(p) self.pg_start() elif p.haslayer(BFD): diff --git a/test/test_flowperpkt.py b/test/test_flowperpkt.py index f16bfb7e..b13d0c63 100644 --- a/test/test_flowperpkt.py +++ b/test/test_flowperpkt.py @@ -43,7 +43,7 @@ class TestFlowperpkt(VppTestCase): for size in packet_sizes: info = self.create_packet_info(src_if, dst_if) payload = self.info_to_payload(info) - p = (Ether(src=src_if.local_mac, dst=dst_if.remote_mac) / + p = (Ether(src=src_if.remote_mac, dst=src_if.local_mac) / IP(src=src_if.remote_ip4, dst=dst_if.remote_ip4) / UDP(sport=1234, dport=4321) / Raw(payload)) -- cgit 1.2.3-korg From 586afd762bfa149f5ca167bd5fd5a0cd59ce94fe Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Wed, 5 Apr 2017 19:18:20 +0200 Subject: Use thread local storage for thread index This patch deprecates stack-based thread identification, Also removes requirement that thread stacks are adjacent. Finally, possibly annoying for some folks, it renames all occurences of cpu_index and cpu_number with thread index. Using word "cpu" is misleading here as thread can be migrated ti different CPU, and also it is not related to linux cpu index. Change-Id: I68cdaf661e701d2336fc953dcb9978d10a70f7c1 Signed-off-by: Damjan Marion --- src/examples/srv6-sample-localsid/node.c | 4 +- src/plugins/dpdk/buffer.c | 2 +- src/plugins/dpdk/device/device.c | 8 +- src/plugins/dpdk/device/dpdk_priv.h | 8 +- src/plugins/dpdk/device/init.c | 2 +- src/plugins/dpdk/device/node.c | 32 +++--- src/plugins/dpdk/hqos/hqos.c | 16 +-- src/plugins/dpdk/ipsec/cli.c | 8 +- src/plugins/dpdk/ipsec/crypto_node.c | 4 +- src/plugins/dpdk/ipsec/esp.h | 4 +- src/plugins/dpdk/ipsec/esp_decrypt.c | 4 +- src/plugins/dpdk/ipsec/esp_encrypt.c | 5 +- src/plugins/dpdk/ipsec/ipsec.c | 2 +- src/plugins/dpdk/ipsec/ipsec.h | 4 +- src/plugins/dpdk/main.c | 2 +- src/plugins/flowperpkt/l2_node.c | 2 +- src/plugins/flowperpkt/node.c | 2 +- src/plugins/ioam/export-common/ioam_export.h | 6 +- .../ioam/ip6/ioam_cache_tunnel_select_node.c | 16 +-- src/plugins/ixge/ixge.c | 2 +- src/plugins/lb/lb.c | 8 +- src/plugins/lb/node.c | 22 ++-- src/plugins/lb/refcount.c | 8 +- src/plugins/lb/refcount.h | 4 +- src/plugins/memif/node.c | 35 +++--- src/plugins/snat/in2out.c | 110 +++++++++--------- src/plugins/snat/out2in.c | 102 ++++++++--------- src/plugins/snat/snat.h | 10 +- src/vlib/buffer.c | 6 +- src/vlib/buffer_funcs.h | 4 +- src/vlib/cli.c | 6 +- src/vlib/counter.h | 16 +-- src/vlib/error.c | 2 +- src/vlib/global_funcs.h | 2 +- src/vlib/main.c | 14 +-- src/vlib/main.h | 2 +- src/vlib/node.c | 2 +- src/vlib/node.h | 6 +- src/vlib/node_funcs.h | 8 +- src/vlib/threads.c | 69 ++++------- src/vlib/threads.h | 21 ++-- src/vlib/unix/cj.c | 7 +- src/vlib/unix/cj.h | 2 +- src/vlib/unix/main.c | 43 +++---- src/vnet/adj/adj_l2.c | 4 +- src/vnet/adj/adj_midchain.c | 8 +- src/vnet/adj/adj_nsh.c | 4 +- src/vnet/classify/vnet_classify.c | 16 +-- src/vnet/cop/ip4_whitelist.c | 8 +- src/vnet/cop/ip6_whitelist.c | 8 +- src/vnet/devices/af_packet/node.c | 20 ++-- src/vnet/devices/devices.c | 61 +++++----- src/vnet/devices/devices.h | 18 +-- src/vnet/devices/netmap/node.c | 24 ++-- src/vnet/devices/ssvm/node.c | 6 +- src/vnet/devices/virtio/vhost-user.c | 127 +++++++++++---------- src/vnet/dpo/lookup_dpo.c | 20 ++-- src/vnet/dpo/replicate_dpo.c | 12 +- src/vnet/ethernet/arp.c | 2 +- src/vnet/ethernet/interface.c | 7 +- src/vnet/ethernet/node.c | 14 +-- src/vnet/gre/node.c | 8 +- src/vnet/interface.h | 2 +- src/vnet/interface_output.c | 53 ++++----- src/vnet/ip/ip4_forward.c | 34 +++--- src/vnet/ip/ip4_input.c | 8 +- src/vnet/ip/ip6_forward.c | 24 ++-- src/vnet/ip/ip6_input.c | 8 +- src/vnet/ip/ip6_neighbor.c | 4 +- src/vnet/ipsec/esp.h | 8 +- src/vnet/ipsec/esp_decrypt.c | 13 ++- src/vnet/ipsec/esp_encrypt.c | 13 ++- src/vnet/ipsec/ikev2.c | 64 ++++++----- src/vnet/ipsec/ipsec.h | 12 +- src/vnet/ipsec/ipsec_if.c | 2 +- src/vnet/l2/l2_bvi.h | 2 +- src/vnet/l2/l2_input.c | 14 +-- src/vnet/l2/l2_output.c | 6 +- src/vnet/l2tp/decap.c | 2 +- src/vnet/l2tp/encap.c | 2 +- src/vnet/l2tp/l2tp.c | 6 +- src/vnet/lisp-gpe/decap.c | 16 +-- src/vnet/lldp/lldp_input.c | 2 +- src/vnet/map/ip4_map.c | 14 +-- src/vnet/map/ip4_map_t.c | 12 +- src/vnet/map/ip6_map.c | 19 +-- src/vnet/map/ip6_map_t.c | 12 +- src/vnet/mpls/mpls_input.c | 8 +- src/vnet/mpls/mpls_lookup.c | 20 ++-- src/vnet/mpls/mpls_output.c | 10 +- src/vnet/pg/input.c | 4 +- src/vnet/replication.c | 20 ++-- src/vnet/replication.h | 2 +- src/vnet/session/node.c | 2 +- src/vnet/sr/sr_localsid.c | 44 +++---- src/vnet/tcp/builtin_client.c | 2 +- src/vnet/tcp/tcp.c | 8 +- src/vnet/tcp/tcp_debug.h | 2 +- src/vnet/tcp/tcp_input.c | 10 +- src/vnet/tcp/tcp_output.c | 20 ++-- src/vnet/udp/udp_input.c | 2 +- src/vnet/unix/tapcli.c | 2 +- src/vnet/unix/tuntap.c | 4 +- src/vnet/vxlan-gpe/decap.c | 10 +- src/vnet/vxlan-gpe/encap.c | 12 +- src/vnet/vxlan/decap.c | 10 +- src/vnet/vxlan/encap.c | 12 +- src/vpp/stats/stats.c | 14 +-- src/vpp/stats/stats.h | 2 +- 109 files changed, 790 insertions(+), 791 deletions(-) (limited to 'src/vnet/ethernet/node.c') diff --git a/src/examples/srv6-sample-localsid/node.c b/src/examples/srv6-sample-localsid/node.c index 7bae9cd7..e83e2352 100644 --- a/src/examples/srv6-sample-localsid/node.c +++ b/src/examples/srv6-sample-localsid/node.c @@ -114,7 +114,7 @@ srv6_localsid_sample_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_fram from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; next_index = node->cached_next_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -168,7 +168,7 @@ srv6_localsid_sample_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_fram /* This increments the SRv6 per LocalSID counters.*/ vlib_increment_combined_counter (((next0 == SRV6_SAMPLE_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : &(sm->sr_ls_valid_counters)), - cpu_index, + thread_index, ls0 - sm->localsids, 1, vlib_buffer_length_in_chain (vm, b0)); diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c index 2765c292..c80b3fa8 100644 --- a/src/plugins/dpdk/buffer.c +++ b/src/plugins/dpdk/buffer.c @@ -132,7 +132,7 @@ dpdk_buffer_delete_free_list (vlib_main_t * vm, u32 free_list_index) u32 merge_index; int i; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); f = vlib_buffer_get_free_list (vm, free_list_index); diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index 50b26689..91661246 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -243,7 +243,7 @@ static_always_inline ASSERT (ring->tx_tail == 0); n_retry = 16; - queue_id = vm->cpu_index; + queue_id = vm->thread_index; do { @@ -266,7 +266,7 @@ static_always_inline { /* no wrap, transmit in one burst */ dpdk_device_hqos_per_worker_thread_t *hqos = - &xd->hqos_wt[vm->cpu_index]; + &xd->hqos_wt[vm->thread_index]; ASSERT (hqos->swq != NULL); @@ -332,7 +332,7 @@ dpdk_buffer_recycle (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp) { dpdk_main_t *dm = &dpdk_main; - u32 my_cpu = vm->cpu_index; + u32 my_cpu = vm->thread_index; struct rte_mbuf *mb_new; if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0) @@ -376,7 +376,7 @@ dpdk_interface_tx (vlib_main_t * vm, tx_ring_hdr_t *ring; u32 n_on_ring; - my_cpu = vm->cpu_index; + my_cpu = vm->thread_index; queue_id = my_cpu; diff --git a/src/plugins/dpdk/device/dpdk_priv.h b/src/plugins/dpdk/device/dpdk_priv.h index dd40ff48..52b4ca4b 100644 --- a/src/plugins/dpdk/device/dpdk_priv.h +++ b/src/plugins/dpdk/device/dpdk_priv.h @@ -79,7 +79,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now) { vlib_simple_counter_main_t *cm; vnet_main_t *vnm = vnet_get_main (); - u32 my_cpu = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u64 rxerrors, last_rxerrors; /* only update counters for PMD interfaces */ @@ -96,7 +96,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now) cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, VNET_INTERFACE_COUNTER_RX_NO_BUF); - vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index, + vlib_increment_simple_counter (cm, thread_index, xd->vlib_sw_if_index, xd->stats.rx_nombuf - xd->last_stats.rx_nombuf); } @@ -107,7 +107,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now) cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, VNET_INTERFACE_COUNTER_RX_MISS); - vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index, + vlib_increment_simple_counter (cm, thread_index, xd->vlib_sw_if_index, xd->stats.imissed - xd->last_stats.imissed); } @@ -119,7 +119,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now) cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, VNET_INTERFACE_COUNTER_RX_ERROR); - vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index, + vlib_increment_simple_counter (cm, thread_index, xd->vlib_sw_if_index, rxerrors - last_rxerrors); } diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index 538db6cb..7eaf8da7 100755 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -324,7 +324,7 @@ dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd) int rv; int j; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) { diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index e740fd18..b10e0fad 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -283,7 +283,7 @@ dpdk_buffer_init_from_template (void *d0, void *d1, void *d2, void *d3, */ static_always_inline u32 dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, - vlib_node_runtime_t * node, u32 cpu_index, u16 queue_id, + vlib_node_runtime_t * node, u32 thread_index, u16 queue_id, int maybe_multiseg) { u32 n_buffers; @@ -294,7 +294,7 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, uword n_rx_bytes = 0; u32 n_trace, trace_cnt __attribute__ ((unused)); vlib_buffer_free_list_t *fl; - vlib_buffer_t *bt = vec_elt_at_index (dm->buffer_templates, cpu_index); + vlib_buffer_t *bt = vec_elt_at_index (dm->buffer_templates, thread_index); if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0) return 0; @@ -306,7 +306,7 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, return 0; } - vec_reset_length (xd->d_trace_buffers[cpu_index]); + vec_reset_length (xd->d_trace_buffers[thread_index]); trace_cnt = n_trace = vlib_get_trace_count (vm, node); if (n_trace > 0) @@ -318,7 +318,7 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, { struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index++]; vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb); - vec_add1 (xd->d_trace_buffers[cpu_index], + vec_add1 (xd->d_trace_buffers[thread_index], vlib_get_buffer_index (vm, b)); } } @@ -546,20 +546,22 @@ dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, vlib_put_next_frame (vm, node, next_index, n_left_to_next); } - if (PREDICT_FALSE (vec_len (xd->d_trace_buffers[cpu_index]) > 0)) + if (PREDICT_FALSE (vec_len (xd->d_trace_buffers[thread_index]) > 0)) { - dpdk_rx_trace (dm, node, xd, queue_id, xd->d_trace_buffers[cpu_index], - vec_len (xd->d_trace_buffers[cpu_index])); - vlib_set_trace_count (vm, node, n_trace - - vec_len (xd->d_trace_buffers[cpu_index])); + dpdk_rx_trace (dm, node, xd, queue_id, + xd->d_trace_buffers[thread_index], + vec_len (xd->d_trace_buffers[thread_index])); + vlib_set_trace_count (vm, node, + n_trace - + vec_len (xd->d_trace_buffers[thread_index])); } vlib_increment_combined_counter (vnet_get_main ()->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, xd->vlib_sw_if_index, mb_index, n_rx_bytes); + thread_index, xd->vlib_sw_if_index, mb_index, n_rx_bytes); - vnet_device_increment_rx_packets (cpu_index, mb_index); + vnet_device_increment_rx_packets (thread_index, mb_index); return mb_index; } @@ -630,19 +632,19 @@ dpdk_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f) dpdk_device_t *xd; uword n_rx_packets = 0; dpdk_device_and_queue_t *dq; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); /* * Poll all devices on this cpu for input/interrupts. */ /* *INDENT-OFF* */ - vec_foreach (dq, dm->devices_by_cpu[cpu_index]) + vec_foreach (dq, dm->devices_by_cpu[thread_index]) { xd = vec_elt_at_index(dm->devices, dq->device); if (xd->flags & DPDK_DEVICE_FLAG_MAYBE_MULTISEG) - n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id, /* maybe_multiseg */ 1); + n_rx_packets += dpdk_device_input (dm, xd, node, thread_index, dq->queue_id, /* maybe_multiseg */ 1); else - n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id, /* maybe_multiseg */ 0); + n_rx_packets += dpdk_device_input (dm, xd, node, thread_index, dq->queue_id, /* maybe_multiseg */ 0); } /* *INDENT-ON* */ diff --git a/src/plugins/dpdk/hqos/hqos.c b/src/plugins/dpdk/hqos/hqos.c index a288fca7..8b251beb 100644 --- a/src/plugins/dpdk/hqos/hqos.c +++ b/src/plugins/dpdk/hqos/hqos.c @@ -397,7 +397,7 @@ static_always_inline void dpdk_hqos_thread_internal_hqos_dbg_bypass (vlib_main_t * vm) { dpdk_main_t *dm = &dpdk_main; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; u32 dev_pos; dev_pos = 0; @@ -405,12 +405,12 @@ dpdk_hqos_thread_internal_hqos_dbg_bypass (vlib_main_t * vm) { vlib_worker_thread_barrier_check (); - u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]); + u32 n_devs = vec_len (dm->devices_by_hqos_cpu[thread_index]); if (dev_pos >= n_devs) dev_pos = 0; dpdk_device_and_queue_t *dq = - vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos); + vec_elt_at_index (dm->devices_by_hqos_cpu[thread_index], dev_pos); dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device); dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht; @@ -479,7 +479,7 @@ static_always_inline void dpdk_hqos_thread_internal (vlib_main_t * vm) { dpdk_main_t *dm = &dpdk_main; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; u32 dev_pos; dev_pos = 0; @@ -487,7 +487,7 @@ dpdk_hqos_thread_internal (vlib_main_t * vm) { vlib_worker_thread_barrier_check (); - u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]); + u32 n_devs = vec_len (dm->devices_by_hqos_cpu[thread_index]); if (PREDICT_FALSE (n_devs == 0)) { dev_pos = 0; @@ -497,7 +497,7 @@ dpdk_hqos_thread_internal (vlib_main_t * vm) dev_pos = 0; dpdk_device_and_queue_t *dq = - vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos); + vec_elt_at_index (dm->devices_by_hqos_cpu[thread_index], dev_pos); dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device); dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht; @@ -586,7 +586,7 @@ dpdk_hqos_thread (vlib_worker_thread_t * w) vm = vlib_get_main (); - ASSERT (vm->cpu_index == os_get_cpu_number ()); + ASSERT (vm->thread_index == vlib_get_thread_index ()); clib_time_init (&vm->clib_time); clib_mem_set_heap (w->thread_mheap); @@ -595,7 +595,7 @@ dpdk_hqos_thread (vlib_worker_thread_t * w) while (tm->worker_thread_release == 0) vlib_worker_thread_barrier_check (); - if (vec_len (dm->devices_by_hqos_cpu[vm->cpu_index]) == 0) + if (vec_len (dm->devices_by_hqos_cpu[vm->thread_index]) == 0) return clib_error ("current I/O TX thread does not have any devices assigned to it"); diff --git a/src/plugins/dpdk/ipsec/cli.c b/src/plugins/dpdk/ipsec/cli.c index cd0a6037..3ae8c9b8 100644 --- a/src/plugins/dpdk/ipsec/cli.c +++ b/src/plugins/dpdk/ipsec/cli.c @@ -42,8 +42,8 @@ dpdk_ipsec_show_mapping (vlib_main_t * vm, u16 detail_display) for (i = 0; i < tm->n_vlib_mains; i++) { uword key, data; - u32 cpu_index = vlib_mains[i]->cpu_index; - crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index]; + u32 thread_index = vlib_mains[i]->thread_index; + crypto_worker_main_t *cwm = &dcm->workers_main[thread_index]; u8 *s = 0; if (skip_master) @@ -57,7 +57,7 @@ dpdk_ipsec_show_mapping (vlib_main_t * vm, u16 detail_display) i32 last_cdev = -1; crypto_qp_data_t *qpd; - s = format (s, "%u\t", cpu_index); + s = format (s, "%u\t", thread_index); /* *INDENT-OFF* */ vec_foreach (qpd, cwm->qp_data) @@ -95,7 +95,7 @@ dpdk_ipsec_show_mapping (vlib_main_t * vm, u16 detail_display) cap.sym.auth.algo = p_key->auth_algo; check_algo_is_supported (&cap, auth_str); vlib_cli_output (vm, "%u\t%10s\t%15s\t%3s\t%u\t%u\n", - vlib_mains[i]->cpu_index, cipher_str, auth_str, + vlib_mains[i]->thread_index, cipher_str, auth_str, p_key->is_outbound ? "out" : "in", cwm->qp_data[data].dev_id, cwm->qp_data[data].qp_id); diff --git a/src/plugins/dpdk/ipsec/crypto_node.c b/src/plugins/dpdk/ipsec/crypto_node.c index dc3452b2..a3c45902 100644 --- a/src/plugins/dpdk/ipsec/crypto_node.c +++ b/src/plugins/dpdk/ipsec/crypto_node.c @@ -171,9 +171,9 @@ static uword dpdk_crypto_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); dpdk_crypto_main_t *dcm = &dpdk_crypto_main; - crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index]; + crypto_worker_main_t *cwm = &dcm->workers_main[thread_index]; crypto_qp_data_t *qpd; u32 n_deq = 0; diff --git a/src/plugins/dpdk/ipsec/esp.h b/src/plugins/dpdk/ipsec/esp.h index 320295b1..56f0c756 100644 --- a/src/plugins/dpdk/ipsec/esp.h +++ b/src/plugins/dpdk/ipsec/esp.h @@ -170,9 +170,9 @@ static_always_inline int create_sym_sess (ipsec_sa_t * sa, crypto_sa_session_t * sa_sess, u8 is_outbound) { - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); dpdk_crypto_main_t *dcm = &dpdk_crypto_main; - crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index]; + crypto_worker_main_t *cwm = &dcm->workers_main[thread_index]; struct rte_crypto_sym_xform cipher_xform = { 0 }; struct rte_crypto_sym_xform auth_xform = { 0 }; struct rte_crypto_sym_xform *xfs; diff --git a/src/plugins/dpdk/ipsec/esp_decrypt.c b/src/plugins/dpdk/ipsec/esp_decrypt.c index 286e03f8..bab76e3b 100644 --- a/src/plugins/dpdk/ipsec/esp_decrypt.c +++ b/src/plugins/dpdk/ipsec/esp_decrypt.c @@ -88,7 +88,7 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, { u32 n_left_from, *from, *to_next, next_index; ipsec_main_t *im = &ipsec_main; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); dpdk_crypto_main_t * dcm = &dpdk_crypto_main; dpdk_esp_main_t * em = &dpdk_esp_main; u32 i; @@ -104,7 +104,7 @@ dpdk_esp_decrypt_node_fn (vlib_main_t * vm, return n_left_from; } - crypto_worker_main_t *cwm = vec_elt_at_index(dcm->workers_main, cpu_index); + crypto_worker_main_t *cwm = vec_elt_at_index(dcm->workers_main, thread_index); u32 n_qps = vec_len(cwm->qp_data); struct rte_crypto_op ** cops_to_enq[n_qps]; u32 n_cop_qp[n_qps], * bi_to_enq[n_qps]; diff --git a/src/plugins/dpdk/ipsec/esp_encrypt.c b/src/plugins/dpdk/ipsec/esp_encrypt.c index 5b03de73..f996d7df 100644 --- a/src/plugins/dpdk/ipsec/esp_encrypt.c +++ b/src/plugins/dpdk/ipsec/esp_encrypt.c @@ -93,7 +93,7 @@ dpdk_esp_encrypt_node_fn (vlib_main_t * vm, { u32 n_left_from, *from, *to_next, next_index; ipsec_main_t *im = &ipsec_main; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); dpdk_crypto_main_t *dcm = &dpdk_crypto_main; dpdk_esp_main_t *em = &dpdk_esp_main; u32 i; @@ -111,7 +111,8 @@ dpdk_esp_encrypt_node_fn (vlib_main_t * vm, return n_left_from; } - crypto_worker_main_t *cwm = vec_elt_at_index (dcm->workers_main, cpu_index); + crypto_worker_main_t *cwm = + vec_elt_at_index (dcm->workers_main, thread_index); u32 n_qps = vec_len (cwm->qp_data); struct rte_crypto_op **cops_to_enq[n_qps]; u32 n_cop_qp[n_qps], *bi_to_enq[n_qps]; diff --git a/src/plugins/dpdk/ipsec/ipsec.c b/src/plugins/dpdk/ipsec/ipsec.c index b0aaaaec..5d8f4fba 100644 --- a/src/plugins/dpdk/ipsec/ipsec.c +++ b/src/plugins/dpdk/ipsec/ipsec.c @@ -289,7 +289,7 @@ dpdk_ipsec_process (vlib_main_t * vm, vlib_node_runtime_t * rt, if (!map) { clib_warning ("unable to create hash table for worker %u", - vlib_mains[i]->cpu_index); + vlib_mains[i]->thread_index); goto error; } cwm->algo_qp_map = map; diff --git a/src/plugins/dpdk/ipsec/ipsec.h b/src/plugins/dpdk/ipsec/ipsec.h index 28bffc80..f0f793c0 100644 --- a/src/plugins/dpdk/ipsec/ipsec.h +++ b/src/plugins/dpdk/ipsec/ipsec.h @@ -95,8 +95,8 @@ static_always_inline void crypto_alloc_cops () { dpdk_crypto_main_t *dcm = &dpdk_crypto_main; - u32 cpu_index = os_get_cpu_number (); - crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index]; + u32 thread_index = vlib_get_thread_index (); + crypto_worker_main_t *cwm = &dcm->workers_main[thread_index]; unsigned socket_id = rte_socket_id (); crypto_qp_data_t *qpd; diff --git a/src/plugins/dpdk/main.c b/src/plugins/dpdk/main.c index 7ee2a785..942b8b2d 100644 --- a/src/plugins/dpdk/main.c +++ b/src/plugins/dpdk/main.c @@ -39,7 +39,7 @@ rte_delay_us_override (unsigned us) * thread then do not intercept. (Must not be called from an * independent pthread). */ - if (os_get_cpu_number () == 0) + if (vlib_get_thread_index () == 0) { /* * We're in the vlib main thread or a vlib process. Make sure diff --git a/src/plugins/flowperpkt/l2_node.c b/src/plugins/flowperpkt/l2_node.c index 1c2f681e..fdaf81d1 100644 --- a/src/plugins/flowperpkt/l2_node.c +++ b/src/plugins/flowperpkt/l2_node.c @@ -102,7 +102,7 @@ add_to_flow_record_l2 (vlib_main_t * vm, u8 * src_mac, u8 * dst_mac, u16 ethertype, u64 timestamp, u16 length, int do_flush) { - u32 my_cpu_number = vm->cpu_index; + u32 my_cpu_number = vm->thread_index; flow_report_main_t *frm = &flow_report_main; ip4_header_t *ip; udp_header_t *udp; diff --git a/src/plugins/flowperpkt/node.c b/src/plugins/flowperpkt/node.c index f77f087d..0277682d 100644 --- a/src/plugins/flowperpkt/node.c +++ b/src/plugins/flowperpkt/node.c @@ -101,7 +101,7 @@ add_to_flow_record_ipv4 (vlib_main_t * vm, u32 src_address, u32 dst_address, u8 tos, u64 timestamp, u16 length, int do_flush) { - u32 my_cpu_number = vm->cpu_index; + u32 my_cpu_number = vm->thread_index; flow_report_main_t *frm = &flow_report_main; ip4_header_t *ip; udp_header_t *udp; diff --git a/src/plugins/ioam/export-common/ioam_export.h b/src/plugins/ioam/export-common/ioam_export.h index 2bf3fd54..9de0d13b 100644 --- a/src/plugins/ioam/export-common/ioam_export.h +++ b/src/plugins/ioam/export-common/ioam_export.h @@ -477,8 +477,8 @@ do { \ from = vlib_frame_vector_args (F); \ n_left_from = (F)->n_vectors; \ next_index = (N)->cached_next_index; \ - while (__sync_lock_test_and_set ((EM)->lockp[(VM)->cpu_index], 1)); \ - my_buf = ioam_export_get_my_buffer (EM, (VM)->cpu_index); \ + while (__sync_lock_test_and_set ((EM)->lockp[(VM)->thread_index], 1)); \ + my_buf = ioam_export_get_my_buffer (EM, (VM)->thread_index); \ my_buf->touched_at = vlib_time_now (VM); \ while (n_left_from > 0) \ { \ @@ -620,7 +620,7 @@ do { \ } \ vlib_node_increment_counter (VM, export_node.index, \ EXPORT_ERROR_RECORDED, pkts_recorded); \ - *(EM)->lockp[(VM)->cpu_index] = 0; \ + *(EM)->lockp[(VM)->thread_index] = 0; \ } while(0) #endif /* __included_ioam_export_h__ */ diff --git a/src/plugins/ioam/ip6/ioam_cache_tunnel_select_node.c b/src/plugins/ioam/ip6/ioam_cache_tunnel_select_node.c index a56dc040..0cf742c9 100644 --- a/src/plugins/ioam/ip6/ioam_cache_tunnel_select_node.c +++ b/src/plugins/ioam/ip6/ioam_cache_tunnel_select_node.c @@ -396,7 +396,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, clib_net_to_host_u32 (tcp0->seq_number) + 1, no_of_responses, now, - vm->cpu_index, &pool_index0)) + vm->thread_index, &pool_index0)) { cache_ts_added++; } @@ -419,7 +419,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, e2e = (ioam_e2e_cache_option_t *) ((u8 *) hbh0 + cm->rewrite_pool_index_offset); - e2e->pool_id = (u8) vm->cpu_index; + e2e->pool_id = (u8) vm->thread_index; e2e->pool_index = pool_index0; ioam_e2e_id_rewrite_handler ((ioam_e2e_id_option_t *) ((u8 *) e2e + @@ -455,7 +455,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, clib_net_to_host_u32 (tcp1->seq_number) + 1, no_of_responses, now, - vm->cpu_index, &pool_index1)) + vm->thread_index, &pool_index1)) { cache_ts_added++; } @@ -479,7 +479,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, e2e = (ioam_e2e_cache_option_t *) ((u8 *) hbh1 + cm->rewrite_pool_index_offset); - e2e->pool_id = (u8) vm->cpu_index; + e2e->pool_id = (u8) vm->thread_index; e2e->pool_index = pool_index1; ioam_e2e_id_rewrite_handler ((ioam_e2e_id_option_t *) ((u8 *) e2e + @@ -562,7 +562,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, clib_net_to_host_u32 (tcp0->seq_number) + 1, no_of_responses, now, - vm->cpu_index, &pool_index0)) + vm->thread_index, &pool_index0)) { cache_ts_added++; } @@ -585,7 +585,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, e2e = (ioam_e2e_cache_option_t *) ((u8 *) hbh0 + cm->rewrite_pool_index_offset); - e2e->pool_id = (u8) vm->cpu_index; + e2e->pool_id = (u8) vm->thread_index; e2e->pool_index = pool_index0; ioam_e2e_id_rewrite_handler ((ioam_e2e_id_option_t *) ((u8 *) e2e + @@ -701,7 +701,7 @@ expired_cache_ts_timer_callback (u32 * expired_timers) ioam_cache_main_t *cm = &ioam_cache_main; int i; u32 pool_index; - u32 thread_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 count = 0; for (i = 0; i < vec_len (expired_timers); i++) @@ -724,7 +724,7 @@ ioam_cache_ts_timer_tick_node_fn (vlib_main_t * vm, vlib_frame_t * f) { ioam_cache_main_t *cm = &ioam_cache_main; - u32 my_thread_index = os_get_cpu_number (); + u32 my_thread_index = vlib_get_thread_index (); struct timespec ts, tsrem; tw_timer_expire_timers_16t_2w_512sl (&cm->timer_wheels[my_thread_index], diff --git a/src/plugins/ixge/ixge.c b/src/plugins/ixge/ixge.c index f3c5cc09..08f5b692 100644 --- a/src/plugins/ixge/ixge.c +++ b/src/plugins/ixge/ixge.c @@ -1887,7 +1887,7 @@ done: vlib_increment_combined_counter (vnet_main. interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - 0 /* cpu_index */ , + 0 /* thread_index */ , xd->vlib_sw_if_index, n_packets, dq->rx.n_bytes); diff --git a/src/plugins/lb/lb.c b/src/plugins/lb/lb.c index add81236..addc2a42 100644 --- a/src/plugins/lb/lb.c +++ b/src/plugins/lb/lb.c @@ -63,11 +63,11 @@ u8 *format_lb_main (u8 * s, va_list * args) s = format(s, " #vips: %u\n", pool_elts(lbm->vips)); s = format(s, " #ass: %u\n", pool_elts(lbm->ass) - 1); - u32 cpu_index; - for(cpu_index = 0; cpu_index < tm->n_vlib_mains; cpu_index++ ) { - lb_hash_t *h = lbm->per_cpu[cpu_index].sticky_ht; + u32 thread_index; + for(thread_index = 0; thread_index < tm->n_vlib_mains; thread_index++ ) { + lb_hash_t *h = lbm->per_cpu[thread_index].sticky_ht; if (h) { - s = format(s, "core %d\n", cpu_index); + s = format(s, "core %d\n", thread_index); s = format(s, " timeout: %ds\n", h->timeout); s = format(s, " usage: %d / %d\n", lb_hash_elts(h, lb_hash_time_now(vlib_get_main())), lb_hash_size(h)); } diff --git a/src/plugins/lb/node.c b/src/plugins/lb/node.c index 8b763c53..3171148b 100644 --- a/src/plugins/lb/node.c +++ b/src/plugins/lb/node.c @@ -60,10 +60,10 @@ format_lb_trace (u8 * s, va_list * args) return s; } -lb_hash_t *lb_get_sticky_table(u32 cpu_index) +lb_hash_t *lb_get_sticky_table(u32 thread_index) { lb_main_t *lbm = &lb_main; - lb_hash_t *sticky_ht = lbm->per_cpu[cpu_index].sticky_ht; + lb_hash_t *sticky_ht = lbm->per_cpu[thread_index].sticky_ht; //Check if size changed if (PREDICT_FALSE(sticky_ht && (lbm->per_cpu_sticky_buckets != lb_hash_nbuckets(sticky_ht)))) { @@ -71,8 +71,8 @@ lb_hash_t *lb_get_sticky_table(u32 cpu_index) lb_hash_bucket_t *b; u32 i; lb_hash_foreach_entry(sticky_ht, b, i) { - vlib_refcount_add(&lbm->as_refcount, cpu_index, b->value[i], -1); - vlib_refcount_add(&lbm->as_refcount, cpu_index, 0, 1); + vlib_refcount_add(&lbm->as_refcount, thread_index, b->value[i], -1); + vlib_refcount_add(&lbm->as_refcount, thread_index, 0, 1); } lb_hash_free(sticky_ht); @@ -81,8 +81,8 @@ lb_hash_t *lb_get_sticky_table(u32 cpu_index) //Create if necessary if (PREDICT_FALSE(sticky_ht == NULL)) { - lbm->per_cpu[cpu_index].sticky_ht = lb_hash_alloc(lbm->per_cpu_sticky_buckets, lbm->flow_timeout); - sticky_ht = lbm->per_cpu[cpu_index].sticky_ht; + lbm->per_cpu[thread_index].sticky_ht = lb_hash_alloc(lbm->per_cpu_sticky_buckets, lbm->flow_timeout); + sticky_ht = lbm->per_cpu[thread_index].sticky_ht; clib_warning("Regenerated sticky table %p", sticky_ht); } @@ -153,10 +153,10 @@ lb_node_fn (vlib_main_t * vm, { lb_main_t *lbm = &lb_main; u32 n_left_from, *from, next_index, *to_next, n_left_to_next; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); u32 lb_time = lb_hash_time_now(vm); - lb_hash_t *sticky_ht = lb_get_sticky_table(cpu_index); + lb_hash_t *sticky_ht = lb_get_sticky_table(thread_index); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; next_index = node->cached_next_index; @@ -240,9 +240,9 @@ lb_node_fn (vlib_main_t * vm, //Configuration may be changed, vectors resized, etc... //Dereference previously used - vlib_refcount_add(&lbm->as_refcount, cpu_index, + vlib_refcount_add(&lbm->as_refcount, thread_index, lb_hash_available_value(sticky_ht, hash0, available_index0), -1); - vlib_refcount_add(&lbm->as_refcount, cpu_index, + vlib_refcount_add(&lbm->as_refcount, thread_index, asindex0, 1); //Add sticky entry @@ -260,7 +260,7 @@ lb_node_fn (vlib_main_t * vm, } vlib_increment_simple_counter(&lbm->vip_counters[counter], - cpu_index, + thread_index, vnet_buffer (p0)->ip.adj_index[VLIB_TX], 1); diff --git a/src/plugins/lb/refcount.c b/src/plugins/lb/refcount.c index 22415c88..6f01ab5a 100644 --- a/src/plugins/lb/refcount.c +++ b/src/plugins/lb/refcount.c @@ -31,10 +31,10 @@ u64 vlib_refcount_get(vlib_refcount_t *r, u32 index) { u64 count = 0; vlib_thread_main_t *tm = vlib_get_thread_main (); - u32 cpu_index; - for (cpu_index = 0; cpu_index < tm->n_vlib_mains; cpu_index++) { - if (r->per_cpu[cpu_index].length > index) - count += r->per_cpu[cpu_index].counters[index]; + u32 thread_index; + for (thread_index = 0; thread_index < tm->n_vlib_mains; thread_index++) { + if (r->per_cpu[thread_index].length > index) + count += r->per_cpu[thread_index].counters[index]; } return count; } diff --git a/src/plugins/lb/refcount.h b/src/plugins/lb/refcount.h index 8c26e7be..dcfcb3fe 100644 --- a/src/plugins/lb/refcount.h +++ b/src/plugins/lb/refcount.h @@ -45,9 +45,9 @@ typedef struct { void __vlib_refcount_resize(vlib_refcount_per_cpu_t *per_cpu, u32 size); static_always_inline -void vlib_refcount_add(vlib_refcount_t *r, u32 cpu_index, u32 counter_index, i32 v) +void vlib_refcount_add(vlib_refcount_t *r, u32 thread_index, u32 counter_index, i32 v) { - vlib_refcount_per_cpu_t *per_cpu = &r->per_cpu[cpu_index]; + vlib_refcount_per_cpu_t *per_cpu = &r->per_cpu[thread_index]; if (PREDICT_FALSE(counter_index >= per_cpu->length)) __vlib_refcount_resize(per_cpu, clib_max(counter_index + 16, per_cpu->length * 2)); diff --git a/src/plugins/memif/node.c b/src/plugins/memif/node.c index 659d5dfb..cee1f3d1 100644 --- a/src/plugins/memif/node.c +++ b/src/plugins/memif/node.c @@ -94,7 +94,7 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u32 n_rx_bytes = 0; u32 *to_next = 0; u32 n_free_bufs; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 bi0, bi1; vlib_buffer_t *b0, *b1; u16 ring_size = 1 << mif->log2_ring_size; @@ -105,14 +105,15 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (mif->per_interface_next_index != ~0) next_index = mif->per_interface_next_index; - n_free_bufs = vec_len (nm->rx_buffers[cpu_index]); + n_free_bufs = vec_len (nm->rx_buffers[thread_index]); if (PREDICT_FALSE (n_free_bufs < ring_size)) { - vec_validate (nm->rx_buffers[cpu_index], ring_size + n_free_bufs - 1); + vec_validate (nm->rx_buffers[thread_index], + ring_size + n_free_bufs - 1); n_free_bufs += - vlib_buffer_alloc (vm, &nm->rx_buffers[cpu_index][n_free_bufs], + vlib_buffer_alloc (vm, &nm->rx_buffers[thread_index][n_free_bufs], ring_size); - _vec_len (nm->rx_buffers[cpu_index]) = n_free_bufs; + _vec_len (nm->rx_buffers[thread_index]) = n_free_bufs; } head = ring->head; @@ -158,15 +159,15 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, CLIB_CACHE_LINE_BYTES, LOAD); } /* get empty buffer */ - u32 last_buf = vec_len (nm->rx_buffers[cpu_index]) - 1; - bi0 = nm->rx_buffers[cpu_index][last_buf]; - bi1 = nm->rx_buffers[cpu_index][last_buf - 1]; - _vec_len (nm->rx_buffers[cpu_index]) -= 2; + u32 last_buf = vec_len (nm->rx_buffers[thread_index]) - 1; + bi0 = nm->rx_buffers[thread_index][last_buf]; + bi1 = nm->rx_buffers[thread_index][last_buf - 1]; + _vec_len (nm->rx_buffers[thread_index]) -= 2; if (last_buf > 4) { - memif_prefetch (vm, nm->rx_buffers[cpu_index][last_buf - 2]); - memif_prefetch (vm, nm->rx_buffers[cpu_index][last_buf - 3]); + memif_prefetch (vm, nm->rx_buffers[thread_index][last_buf - 2]); + memif_prefetch (vm, nm->rx_buffers[thread_index][last_buf - 3]); } /* enqueue buffer */ @@ -256,9 +257,9 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, while (num_slots && n_left_to_next) { /* get empty buffer */ - u32 last_buf = vec_len (nm->rx_buffers[cpu_index]) - 1; - bi0 = nm->rx_buffers[cpu_index][last_buf]; - _vec_len (nm->rx_buffers[cpu_index]) = last_buf; + u32 last_buf = vec_len (nm->rx_buffers[thread_index]) - 1; + bi0 = nm->rx_buffers[thread_index][last_buf]; + _vec_len (nm->rx_buffers[thread_index]) = last_buf; /* enqueue buffer */ to_next[0] = bi0; @@ -315,7 +316,7 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, ring->tail = head; vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters - + VNET_INTERFACE_COUNTER_RX, cpu_index, + + VNET_INTERFACE_COUNTER_RX, thread_index, mif->hw_if_index, n_rx_packets, n_rx_bytes); @@ -327,7 +328,7 @@ memif_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { u32 n_rx_packets = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); memif_main_t *nm = &memif_main; memif_if_t *mif; @@ -337,7 +338,7 @@ memif_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, if (mif->flags & MEMIF_IF_FLAG_ADMIN_UP && mif->flags & MEMIF_IF_FLAG_CONNECTED && (mif->if_index % nm->input_cpu_count) == - (cpu_index - nm->input_cpu_first_index)) + (thread_index - nm->input_cpu_first_index)) { if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE) n_rx_packets += diff --git a/src/plugins/snat/in2out.c b/src/plugins/snat/in2out.c index b4961365..e5ee965f 100644 --- a/src/plugins/snat/in2out.c +++ b/src/plugins/snat/in2out.c @@ -212,7 +212,7 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, snat_session_t ** sessionp, vlib_node_runtime_t * node, u32 next0, - u32 cpu_index) + u32 thread_index) { snat_user_t *u; snat_user_key_t user_key; @@ -246,27 +246,27 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0)) { /* no, make a new one */ - pool_get (sm->per_thread_data[cpu_index].users, u); + pool_get (sm->per_thread_data[thread_index].users, u); memset (u, 0, sizeof (*u)); u->addr = ip0->src_address; u->fib_index = rx_fib_index0; - pool_get (sm->per_thread_data[cpu_index].list_pool, per_user_list_head_elt); + pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt); u->sessions_per_user_list_head_index = per_user_list_head_elt - - sm->per_thread_data[cpu_index].list_pool; + sm->per_thread_data[thread_index].list_pool; - clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_init (sm->per_thread_data[thread_index].list_pool, u->sessions_per_user_list_head_index); - kv0.value = u - sm->per_thread_data[cpu_index].users; + kv0.value = u - sm->per_thread_data[thread_index].users; /* add user */ clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */); } else { - u = pool_elt_at_index (sm->per_thread_data[cpu_index].users, + u = pool_elt_at_index (sm->per_thread_data[thread_index].users, value0.value); } @@ -276,25 +276,25 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, /* Remove the oldest dynamic translation */ do { oldest_per_user_translation_list_index = - clib_dlist_remove_head (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove_head (sm->per_thread_data[thread_index].list_pool, u->sessions_per_user_list_head_index); ASSERT (oldest_per_user_translation_list_index != ~0); /* add it back to the end of the LRU list */ - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, u->sessions_per_user_list_head_index, oldest_per_user_translation_list_index); /* Get the list element */ oldest_per_user_translation_list_elt = - pool_elt_at_index (sm->per_thread_data[cpu_index].list_pool, + pool_elt_at_index (sm->per_thread_data[thread_index].list_pool, oldest_per_user_translation_list_index); /* Get the session index from the list element */ session_index = oldest_per_user_translation_list_elt->value; /* Get the session */ - s = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, session_index); } while (snat_is_session_static (s)); @@ -346,7 +346,7 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, } /* Create a new session */ - pool_get (sm->per_thread_data[cpu_index].sessions, s); + pool_get (sm->per_thread_data[thread_index].sessions, s); memset (s, 0, sizeof (*s)); s->outside_address_index = address_index; @@ -362,22 +362,22 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, } /* Create list elts */ - pool_get (sm->per_thread_data[cpu_index].list_pool, + pool_get (sm->per_thread_data[thread_index].list_pool, per_user_translation_list_elt); - clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_init (sm->per_thread_data[thread_index].list_pool, per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool); + sm->per_thread_data[thread_index].list_pool); per_user_translation_list_elt->value = - s - sm->per_thread_data[cpu_index].sessions; + s - sm->per_thread_data[thread_index].sessions; s->per_user_index = per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool; + sm->per_thread_data[thread_index].list_pool; s->per_user_list_head_index = u->sessions_per_user_list_head_index; - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s->per_user_list_head_index, per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool); + sm->per_thread_data[thread_index].list_pool); } s->in2out = *key0; @@ -388,12 +388,12 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, /* Add to translation hashes */ kv0.key = s->in2out.as_u64; - kv0.value = s - sm->per_thread_data[cpu_index].sessions; + kv0.value = s - sm->per_thread_data[thread_index].sessions; if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */)) clib_warning ("in2out key add failed"); kv0.key = s->out2in.as_u64; - kv0.value = s - sm->per_thread_data[cpu_index].sessions; + kv0.value = s - sm->per_thread_data[thread_index].sessions; if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */)) clib_warning ("out2in key add failed"); @@ -403,7 +403,7 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0, worker_by_out_key.port = s->out2in.port; worker_by_out_key.fib_index = s->out2in.fib_index; kv0.key = worker_by_out_key.as_u64; - kv0.value = cpu_index; + kv0.value = thread_index; clib_bihash_add_del_8_8 (&sm->worker_by_out, &kv0, 1); /* log NAT event */ @@ -465,7 +465,7 @@ snat_in2out_error_t icmp_get_key(icmp46_header_t *icmp0, * * @param[in,out] sm SNAT main * @param[in,out] node SNAT node runtime - * @param[in] cpu_index CPU index + * @param[in] thread_index thread index * @param[in,out] b0 buffer containing packet to be translated * @param[out] p_key address and port before NAT translation * @param[out] p_value address and port after NAT translation @@ -473,7 +473,7 @@ snat_in2out_error_t icmp_get_key(icmp46_header_t *icmp0, * @param d optional parameter */ u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d) @@ -524,13 +524,13 @@ u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node, } next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, - &s0, node, next0, cpu_index); + &s0, node, next0, thread_index); if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP)) goto out; } else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value0.value); out: @@ -548,7 +548,7 @@ out: * * @param[in] sm SNAT main * @param[in,out] node SNAT node runtime - * @param[in] cpu_index CPU index + * @param[in] thread_index thread index * @param[in,out] b0 buffer containing packet to be translated * @param[out] p_key address and port before NAT translation * @param[out] p_value address and port after NAT translation @@ -556,7 +556,7 @@ out: * @param d optional parameter */ u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d) @@ -624,7 +624,7 @@ static inline u32 icmp_in2out (snat_main_t *sm, u32 rx_fib_index0, vlib_node_runtime_t * node, u32 next0, - u32 cpu_index, + u32 thread_index, void *d) { snat_session_key_t key0, sm0; @@ -641,7 +641,7 @@ static inline u32 icmp_in2out (snat_main_t *sm, echo0 = (icmp_echo_header_t *)(icmp0+1); - next0_tmp = sm->icmp_match_in2out_cb(sm, node, cpu_index, b0, + next0_tmp = sm->icmp_match_in2out_cb(sm, node, thread_index, b0, &key0, &sm0, &dont_translate, d); if (next0_tmp != ~0) next0 = next0_tmp; @@ -847,11 +847,11 @@ static inline u32 icmp_in2out_slow_path (snat_main_t *sm, vlib_node_runtime_t * node, u32 next0, f64 now, - u32 cpu_index, + u32 thread_index, snat_session_t ** p_s0) { next0 = icmp_in2out(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, cpu_index, p_s0); + next0, thread_index, p_s0); snat_session_t * s0 = *p_s0; if (PREDICT_TRUE(next0 != SNAT_IN2OUT_NEXT_DROP && s0)) { @@ -862,9 +862,9 @@ static inline u32 icmp_in2out_slow_path (snat_main_t *sm, /* Per-user LRU list maintenance for dynamic translations */ if (!snat_is_session_static (s0)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s0->per_user_list_head_index, s0->per_user_index); } @@ -884,7 +884,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, snat_runtime_t * rt = (snat_runtime_t *)node->runtime_data; f64 now = vlib_time_now (vm); u32 stats_node_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); stats_node_index = is_slow_path ? snat_in2out_slowpath_node.index : snat_in2out_node.index; @@ -977,7 +977,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, { next0 = icmp_in2out_slow_path (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, - node, next0, now, cpu_index, &s0); + node, next0, now, thread_index, &s0); goto trace00; } } @@ -1006,7 +1006,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, goto trace00; next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, - &s0, node, next0, cpu_index); + &s0, node, next0, thread_index); if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP)) goto trace00; } @@ -1017,7 +1017,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, } } else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value0.value); old_addr0 = ip0->src_address.as_u32; @@ -1063,9 +1063,9 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s0)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s0->per_user_list_head_index, s0->per_user_index); } @@ -1081,7 +1081,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, t->next_index = next0; t->session_index = ~0; if (s0) - t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; + t->session_index = s0 - sm->per_thread_data[thread_index].sessions; } pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP; @@ -1117,7 +1117,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, { next1 = icmp_in2out_slow_path (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, - next1, now, cpu_index, &s1); + next1, now, thread_index, &s1); goto trace01; } } @@ -1146,7 +1146,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, goto trace01; next1 = slow_path (sm, b1, ip1, rx_fib_index1, &key1, - &s1, node, next1, cpu_index); + &s1, node, next1, thread_index); if (PREDICT_FALSE (next1 == SNAT_IN2OUT_NEXT_DROP)) goto trace01; } @@ -1157,7 +1157,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, } } else - s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value1.value); old_addr1 = ip1->src_address.as_u32; @@ -1203,9 +1203,9 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s1)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s1->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s1->per_user_list_head_index, s1->per_user_index); } @@ -1220,7 +1220,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, t->next_index = next1; t->session_index = ~0; if (s1) - t->session_index = s1 - sm->per_thread_data[cpu_index].sessions; + t->session_index = s1 - sm->per_thread_data[thread_index].sessions; } pkts_processed += next1 != SNAT_IN2OUT_NEXT_DROP; @@ -1292,7 +1292,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, { next0 = icmp_in2out_slow_path (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, now, cpu_index, &s0); + next0, now, thread_index, &s0); goto trace0; } } @@ -1321,7 +1321,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, goto trace0; next0 = slow_path (sm, b0, ip0, rx_fib_index0, &key0, - &s0, node, next0, cpu_index); + &s0, node, next0, thread_index); if (PREDICT_FALSE (next0 == SNAT_IN2OUT_NEXT_DROP)) goto trace0; @@ -1333,7 +1333,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, } } else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value0.value); old_addr0 = ip0->src_address.as_u32; @@ -1379,9 +1379,9 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s0)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s0->per_user_list_head_index, s0->per_user_index); } @@ -1397,7 +1397,7 @@ snat_in2out_node_fn_inline (vlib_main_t * vm, t->next_index = next0; t->session_index = ~0; if (s0) - t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; + t->session_index = s0 - sm->per_thread_data[thread_index].sessions; } pkts_processed += next0 != SNAT_IN2OUT_NEXT_DROP; @@ -2010,7 +2010,7 @@ snat_in2out_worker_handoff_fn (vlib_main_t * vm, u32 n_left_to_next_worker = 0, *to_next_worker = 0; u32 next_worker_index = 0; u32 current_worker_index = ~0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); ASSERT (vec_len (sm->workers)); @@ -2048,7 +2048,7 @@ snat_in2out_worker_handoff_fn (vlib_main_t * vm, next_worker_index = sm->worker_in2out_cb(ip0, rx_fib_index0); - if (PREDICT_FALSE (next_worker_index != cpu_index)) + if (PREDICT_FALSE (next_worker_index != thread_index)) { do_handoff = 1; diff --git a/src/plugins/snat/out2in.c b/src/plugins/snat/out2in.c index 656e42db..5d308d78 100644 --- a/src/plugins/snat/out2in.c +++ b/src/plugins/snat/out2in.c @@ -129,7 +129,7 @@ create_session_for_static_mapping (snat_main_t *sm, snat_session_key_t in2out, snat_session_key_t out2in, vlib_node_runtime_t * node, - u32 cpu_index) + u32 thread_index) { snat_user_t *u; snat_user_key_t user_key; @@ -146,36 +146,36 @@ create_session_for_static_mapping (snat_main_t *sm, if (clib_bihash_search_8_8 (&sm->user_hash, &kv0, &value0)) { /* no, make a new one */ - pool_get (sm->per_thread_data[cpu_index].users, u); + pool_get (sm->per_thread_data[thread_index].users, u); memset (u, 0, sizeof (*u)); u->addr = in2out.addr; u->fib_index = in2out.fib_index; - pool_get (sm->per_thread_data[cpu_index].list_pool, + pool_get (sm->per_thread_data[thread_index].list_pool, per_user_list_head_elt); u->sessions_per_user_list_head_index = per_user_list_head_elt - - sm->per_thread_data[cpu_index].list_pool; + sm->per_thread_data[thread_index].list_pool; - clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_init (sm->per_thread_data[thread_index].list_pool, u->sessions_per_user_list_head_index); - kv0.value = u - sm->per_thread_data[cpu_index].users; + kv0.value = u - sm->per_thread_data[thread_index].users; /* add user */ clib_bihash_add_del_8_8 (&sm->user_hash, &kv0, 1 /* is_add */); /* add non-traslated packets worker lookup */ - kv0.value = cpu_index; + kv0.value = thread_index; clib_bihash_add_del_8_8 (&sm->worker_by_in, &kv0, 1); } else { - u = pool_elt_at_index (sm->per_thread_data[cpu_index].users, + u = pool_elt_at_index (sm->per_thread_data[thread_index].users, value0.value); } - pool_get (sm->per_thread_data[cpu_index].sessions, s); + pool_get (sm->per_thread_data[thread_index].sessions, s); memset (s, 0, sizeof (*s)); s->outside_address_index = ~0; @@ -183,22 +183,22 @@ create_session_for_static_mapping (snat_main_t *sm, u->nstaticsessions++; /* Create list elts */ - pool_get (sm->per_thread_data[cpu_index].list_pool, + pool_get (sm->per_thread_data[thread_index].list_pool, per_user_translation_list_elt); - clib_dlist_init (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_init (sm->per_thread_data[thread_index].list_pool, per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool); + sm->per_thread_data[thread_index].list_pool); per_user_translation_list_elt->value = - s - sm->per_thread_data[cpu_index].sessions; + s - sm->per_thread_data[thread_index].sessions; s->per_user_index = - per_user_translation_list_elt - sm->per_thread_data[cpu_index].list_pool; + per_user_translation_list_elt - sm->per_thread_data[thread_index].list_pool; s->per_user_list_head_index = u->sessions_per_user_list_head_index; - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s->per_user_list_head_index, per_user_translation_list_elt - - sm->per_thread_data[cpu_index].list_pool); + sm->per_thread_data[thread_index].list_pool); s->in2out = in2out; s->out2in = out2in; @@ -206,12 +206,12 @@ create_session_for_static_mapping (snat_main_t *sm, /* Add to translation hashes */ kv0.key = s->in2out.as_u64; - kv0.value = s - sm->per_thread_data[cpu_index].sessions; + kv0.value = s - sm->per_thread_data[thread_index].sessions; if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 1 /* is_add */)) clib_warning ("in2out key add failed"); kv0.key = s->out2in.as_u64; - kv0.value = s - sm->per_thread_data[cpu_index].sessions; + kv0.value = s - sm->per_thread_data[thread_index].sessions; if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 1 /* is_add */)) clib_warning ("out2in key add failed"); @@ -298,7 +298,7 @@ is_interface_addr(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0, * * @param[in,out] sm SNAT main * @param[in,out] node SNAT node runtime - * @param[in] cpu_index CPU index + * @param[in] thread_index thread index * @param[in,out] b0 buffer containing packet to be translated * @param[out] p_key address and port before NAT translation * @param[out] p_value address and port after NAT translation @@ -306,7 +306,7 @@ is_interface_addr(snat_main_t *sm, vlib_node_runtime_t *node, u32 sw_if_index0, * @param d optional parameter */ u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d) @@ -366,7 +366,7 @@ u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node, /* Create session initiated by host from external network */ s0 = create_session_for_static_mapping(sm, b0, sm0, key0, - node, cpu_index); + node, thread_index); if (!s0) { @@ -375,7 +375,7 @@ u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node, } } else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value0.value); out: @@ -393,7 +393,7 @@ out: * * @param[in] sm SNAT main * @param[in,out] node SNAT node runtime - * @param[in] cpu_index CPU index + * @param[in] thread_index thread index * @param[in,out] b0 buffer containing packet to be translated * @param[out] p_key address and port before NAT translation * @param[out] p_value address and port after NAT translation @@ -401,7 +401,7 @@ out: * @param d optional parameter */ u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d) @@ -460,7 +460,7 @@ static inline u32 icmp_out2in (snat_main_t *sm, u32 rx_fib_index0, vlib_node_runtime_t * node, u32 next0, - u32 cpu_index, + u32 thread_index, void *d) { snat_session_key_t key0, sm0; @@ -477,7 +477,7 @@ static inline u32 icmp_out2in (snat_main_t *sm, echo0 = (icmp_echo_header_t *)(icmp0+1); - next0_tmp = sm->icmp_match_out2in_cb(sm, node, cpu_index, b0, + next0_tmp = sm->icmp_match_out2in_cb(sm, node, thread_index, b0, &key0, &sm0, &dont_translate, d); if (next0_tmp != ~0) next0 = next0_tmp; @@ -589,11 +589,11 @@ static inline u32 icmp_out2in_slow_path (snat_main_t *sm, u32 rx_fib_index0, vlib_node_runtime_t * node, u32 next0, f64 now, - u32 cpu_index, + u32 thread_index, snat_session_t ** p_s0) { next0 = icmp_out2in(sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, cpu_index, p_s0); + next0, thread_index, p_s0); snat_session_t * s0 = *p_s0; if (PREDICT_TRUE(next0 != SNAT_OUT2IN_NEXT_DROP && s0)) { @@ -604,9 +604,9 @@ static inline u32 icmp_out2in_slow_path (snat_main_t *sm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s0)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s0->per_user_list_head_index, s0->per_user_index); } @@ -624,7 +624,7 @@ snat_out2in_node_fn (vlib_main_t * vm, u32 pkts_processed = 0; snat_main_t * sm = &snat_main; f64 now = vlib_time_now (vm); - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -712,7 +712,7 @@ snat_out2in_node_fn (vlib_main_t * vm, { next0 = icmp_out2in_slow_path (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, now, cpu_index, &s0); + next0, now, thread_index, &s0); goto trace0; } @@ -743,7 +743,7 @@ snat_out2in_node_fn (vlib_main_t * vm, /* Create session initiated by host from external network */ s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node, - cpu_index); + thread_index); if (!s0) { b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; @@ -752,7 +752,7 @@ snat_out2in_node_fn (vlib_main_t * vm, } } else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value0.value); old_addr0 = ip0->dst_address.as_u32; @@ -796,9 +796,9 @@ snat_out2in_node_fn (vlib_main_t * vm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s0)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s0->per_user_list_head_index, s0->per_user_index); } @@ -813,7 +813,7 @@ snat_out2in_node_fn (vlib_main_t * vm, t->next_index = next0; t->session_index = ~0; if (s0) - t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; + t->session_index = s0 - sm->per_thread_data[thread_index].sessions; } pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP; @@ -847,7 +847,7 @@ snat_out2in_node_fn (vlib_main_t * vm, { next1 = icmp_out2in_slow_path (sm, b1, ip1, icmp1, sw_if_index1, rx_fib_index1, node, - next1, now, cpu_index, &s1); + next1, now, thread_index, &s1); goto trace1; } @@ -878,7 +878,7 @@ snat_out2in_node_fn (vlib_main_t * vm, /* Create session initiated by host from external network */ s1 = create_session_for_static_mapping(sm, b1, sm1, key1, node, - cpu_index); + thread_index); if (!s1) { b1->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; @@ -887,7 +887,7 @@ snat_out2in_node_fn (vlib_main_t * vm, } } else - s1 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s1 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value1.value); old_addr1 = ip1->dst_address.as_u32; @@ -931,9 +931,9 @@ snat_out2in_node_fn (vlib_main_t * vm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s1)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s1->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s1->per_user_list_head_index, s1->per_user_index); } @@ -948,7 +948,7 @@ snat_out2in_node_fn (vlib_main_t * vm, t->next_index = next1; t->session_index = ~0; if (s1) - t->session_index = s1 - sm->per_thread_data[cpu_index].sessions; + t->session_index = s1 - sm->per_thread_data[thread_index].sessions; } pkts_processed += next1 != SNAT_OUT2IN_NEXT_DROP; @@ -1016,7 +1016,7 @@ snat_out2in_node_fn (vlib_main_t * vm, { next0 = icmp_out2in_slow_path (sm, b0, ip0, icmp0, sw_if_index0, rx_fib_index0, node, - next0, now, cpu_index, &s0); + next0, now, thread_index, &s0); goto trace00; } @@ -1048,7 +1048,7 @@ snat_out2in_node_fn (vlib_main_t * vm, /* Create session initiated by host from external network */ s0 = create_session_for_static_mapping(sm, b0, sm0, key0, node, - cpu_index); + thread_index); if (!s0) { b0->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION]; @@ -1057,7 +1057,7 @@ snat_out2in_node_fn (vlib_main_t * vm, } } else - s0 = pool_elt_at_index (sm->per_thread_data[cpu_index].sessions, + s0 = pool_elt_at_index (sm->per_thread_data[thread_index].sessions, value0.value); old_addr0 = ip0->dst_address.as_u32; @@ -1101,9 +1101,9 @@ snat_out2in_node_fn (vlib_main_t * vm, /* Per-user LRU list maintenance for dynamic translation */ if (!snat_is_session_static (s0)) { - clib_dlist_remove (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_remove (sm->per_thread_data[thread_index].list_pool, s0->per_user_index); - clib_dlist_addtail (sm->per_thread_data[cpu_index].list_pool, + clib_dlist_addtail (sm->per_thread_data[thread_index].list_pool, s0->per_user_list_head_index, s0->per_user_index); } @@ -1118,7 +1118,7 @@ snat_out2in_node_fn (vlib_main_t * vm, t->next_index = next0; t->session_index = ~0; if (s0) - t->session_index = s0 - sm->per_thread_data[cpu_index].sessions; + t->session_index = s0 - sm->per_thread_data[thread_index].sessions; } pkts_processed += next0 != SNAT_OUT2IN_NEXT_DROP; @@ -1599,7 +1599,7 @@ snat_out2in_worker_handoff_fn (vlib_main_t * vm, u32 n_left_to_next_worker = 0, *to_next_worker = 0; u32 next_worker_index = 0; u32 current_worker_index = ~0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); ASSERT (vec_len (sm->workers)); @@ -1637,7 +1637,7 @@ snat_out2in_worker_handoff_fn (vlib_main_t * vm, next_worker_index = sm->worker_out2in_cb(ip0, rx_fib_index0); - if (PREDICT_FALSE (next_worker_index != cpu_index)) + if (PREDICT_FALSE (next_worker_index != thread_index)) { do_handoff = 1; diff --git a/src/plugins/snat/snat.h b/src/plugins/snat/snat.h index 017825c0..f4e1c5c0 100644 --- a/src/plugins/snat/snat.h +++ b/src/plugins/snat/snat.h @@ -221,7 +221,7 @@ struct snat_main_s; typedef u32 snat_icmp_match_function_t (struct snat_main_s *sm, vlib_node_runtime_t *node, - u32 cpu_index, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, @@ -402,22 +402,22 @@ typedef struct { } tcp_udp_header_t; u32 icmp_match_in2out_fast(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d); u32 icmp_match_in2out_slow(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d); u32 icmp_match_out2in_fast(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d); u32 icmp_match_out2in_slow(snat_main_t *sm, vlib_node_runtime_t *node, - u32 cpu_index, vlib_buffer_t *b0, + u32 thread_index, vlib_buffer_t *b0, snat_session_key_t *p_key, snat_session_key_t *p_value, u8 *p_dont_translate, void *d); diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index a517a597..be3b41ef 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -299,7 +299,7 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm, if (CLIB_DEBUG == 0) return; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); /* smp disaster check */ if (vec_len (vlib_mains) > 1) @@ -355,7 +355,7 @@ vlib_buffer_create_free_list_helper (vlib_main_t * vm, vlib_buffer_free_list_t *f; int i; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); if (!is_default && pool_elts (bm->buffer_free_list_pool) == 0) { @@ -474,7 +474,7 @@ vlib_buffer_delete_free_list_internal (vlib_main_t * vm, u32 free_list_index) u32 merge_index; int i; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); f = vlib_buffer_get_free_list (vm, free_list_index); diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index 394c336a..328660a3 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -209,7 +209,7 @@ always_inline vlib_buffer_known_state_t vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index) { vlib_buffer_main_t *bm = vm->buffer_main; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); uword *p = hash_get (bm->buffer_known_hash, buffer_index); return p ? p[0] : VLIB_BUFFER_UNKNOWN; @@ -221,7 +221,7 @@ vlib_buffer_set_known_state (vlib_main_t * vm, vlib_buffer_known_state_t state) { vlib_buffer_main_t *bm = vm->buffer_main; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); hash_set (bm->buffer_known_hash, buffer_index, state); } diff --git a/src/vlib/cli.c b/src/vlib/cli.c index f853f655..3cc95076 100644 --- a/src/vlib/cli.c +++ b/src/vlib/cli.c @@ -709,7 +709,7 @@ test_heap_validate (vlib_main_t * vm, unformat_input_t * input, { /* *INDENT-OFF* */ foreach_vlib_main({ - heap = clib_per_cpu_mheaps[this_vlib_main->cpu_index]; + heap = clib_per_cpu_mheaps[this_vlib_main->thread_index]; mheap = mheap_header(heap); mheap->flags |= MHEAP_FLAG_VALIDATE; // Turn off small object cache because it delays detection of errors @@ -722,7 +722,7 @@ test_heap_validate (vlib_main_t * vm, unformat_input_t * input, { /* *INDENT-OFF* */ foreach_vlib_main({ - heap = clib_per_cpu_mheaps[this_vlib_main->cpu_index]; + heap = clib_per_cpu_mheaps[this_vlib_main->thread_index]; mheap = mheap_header(heap); mheap->flags &= ~MHEAP_FLAG_VALIDATE; mheap->flags |= MHEAP_FLAG_SMALL_OBJECT_CACHE; @@ -733,7 +733,7 @@ test_heap_validate (vlib_main_t * vm, unformat_input_t * input, { /* *INDENT-OFF* */ foreach_vlib_main({ - heap = clib_per_cpu_mheaps[this_vlib_main->cpu_index]; + heap = clib_per_cpu_mheaps[this_vlib_main->thread_index]; mheap = mheap_header(heap); mheap_validate(heap); }); diff --git a/src/vlib/counter.h b/src/vlib/counter.h index 17a85217..60e2055d 100644 --- a/src/vlib/counter.h +++ b/src/vlib/counter.h @@ -70,17 +70,17 @@ u32 vlib_simple_counter_n_counters (const vlib_simple_counter_main_t * cm); /** Increment a simple counter @param cm - (vlib_simple_counter_main_t *) simple counter main pointer - @param cpu_index - (u32) the current cpu index + @param thread_index - (u32) the current cpu index @param index - (u32) index of the counter to increment @param increment - (u64) quantitiy to add to the counter */ always_inline void vlib_increment_simple_counter (vlib_simple_counter_main_t * cm, - u32 cpu_index, u32 index, u64 increment) + u32 thread_index, u32 index, u64 increment) { counter_t *my_counters; - my_counters = cm->counters[cpu_index]; + my_counters = cm->counters[thread_index]; my_counters[index] += increment; } @@ -201,7 +201,7 @@ void vlib_clear_combined_counters (vlib_combined_counter_main_t * cm); /** Increment a combined counter @param cm - (vlib_combined_counter_main_t *) comined counter main pointer - @param cpu_index - (u32) the current cpu index + @param thread_index - (u32) the current cpu index @param index - (u32) index of the counter to increment @param packet_increment - (u64) number of packets to add to the counter @param byte_increment - (u64) number of bytes to add to the counter @@ -209,13 +209,13 @@ void vlib_clear_combined_counters (vlib_combined_counter_main_t * cm); always_inline void vlib_increment_combined_counter (vlib_combined_counter_main_t * cm, - u32 cpu_index, + u32 thread_index, u32 index, u64 n_packets, u64 n_bytes) { vlib_counter_t *my_counters; /* Use this CPU's counter array */ - my_counters = cm->counters[cpu_index]; + my_counters = cm->counters[thread_index]; my_counters[index].packets += n_packets; my_counters[index].bytes += n_bytes; @@ -224,14 +224,14 @@ vlib_increment_combined_counter (vlib_combined_counter_main_t * cm, /** Pre-fetch a per-thread combined counter for the given object index */ always_inline void vlib_prefetch_combined_counter (const vlib_combined_counter_main_t * cm, - u32 cpu_index, u32 index) + u32 thread_index, u32 index) { vlib_counter_t *cpu_counters; /* * This CPU's index is assumed to already be in cache */ - cpu_counters = cm->counters[cpu_index]; + cpu_counters = cm->counters[thread_index]; CLIB_PREFETCH (cpu_counters + index, CLIB_CACHE_LINE_BYTES, STORE); } diff --git a/src/vlib/error.c b/src/vlib/error.c index a2c23176..e4ed4ee3 100644 --- a/src/vlib/error.c +++ b/src/vlib/error.c @@ -149,7 +149,7 @@ vlib_register_errors (vlib_main_t * vm, vlib_node_t *n = vlib_get_node (vm, node_index); uword l; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); /* Free up any previous error strings. */ if (n->n_errors > 0) diff --git a/src/vlib/global_funcs.h b/src/vlib/global_funcs.h index f51ec381..9dd01fbf 100644 --- a/src/vlib/global_funcs.h +++ b/src/vlib/global_funcs.h @@ -23,7 +23,7 @@ always_inline vlib_main_t * vlib_get_main (void) { vlib_main_t *vm; - vm = vlib_mains[os_get_cpu_number ()]; + vm = vlib_mains[vlib_get_thread_index ()]; ASSERT (vm); return vm; } diff --git a/src/vlib/main.c b/src/vlib/main.c index b22203f0..422d3e26 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -136,18 +136,18 @@ vlib_frame_alloc_to_node (vlib_main_t * vm, u32 to_node_index, else { f = clib_mem_alloc_aligned_no_fail (n, VLIB_FRAME_ALIGN); - f->cpu_index = vm->cpu_index; + f->thread_index = vm->thread_index; fi = vlib_frame_index_no_check (vm, f); } /* Poison frame when debugging. */ if (CLIB_DEBUG > 0) { - u32 save_cpu_index = f->cpu_index; + u32 save_thread_index = f->thread_index; memset (f, 0xfe, n); - f->cpu_index = save_cpu_index; + f->thread_index = save_thread_index; } /* Insert magic number. */ @@ -517,7 +517,7 @@ vlib_put_next_frame (vlib_main_t * vm, * a dangling frame reference. Each thread has its own copy of * the next_frames vector. */ - if (0 && r->cpu_index != next_runtime->cpu_index) + if (0 && r->thread_index != next_runtime->thread_index) { nf->frame_index = ~0; nf->flags &= ~(VLIB_FRAME_PENDING | VLIB_FRAME_IS_ALLOCATED); @@ -866,7 +866,7 @@ vlib_elog_main_loop_event (vlib_main_t * vm, : evm->node_call_elog_event_types, node_index), /* track */ - (vm->cpu_index ? &vlib_worker_threads[vm->cpu_index]. + (vm->thread_index ? &vlib_worker_threads[vm->thread_index]. elog_track : &em->default_track), /* data to log */ n_vectors); } @@ -963,7 +963,7 @@ dispatch_node (vlib_main_t * vm, vm->cpu_time_last_node_dispatch = last_time_stamp; - if (1 /* || vm->cpu_index == node->cpu_index */ ) + if (1 /* || vm->thread_index == node->thread_index */ ) { vlib_main_t *stat_vm; @@ -1029,7 +1029,7 @@ dispatch_node (vlib_main_t * vm, { u32 node_name, vector_length, is_polling; } *ed; - vlib_worker_thread_t *w = vlib_worker_threads + vm->cpu_index; + vlib_worker_thread_t *w = vlib_worker_threads + vm->thread_index; #endif if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT diff --git a/src/vlib/main.h b/src/vlib/main.h index 0197b4f3..329bf073 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -156,7 +156,7 @@ typedef struct vlib_main_t uword *init_functions_called; /* to compare with node runtime */ - u32 cpu_index; + u32 thread_index; void **mbuf_alloc_list; diff --git a/src/vlib/node.c b/src/vlib/node.c index dc0a4de5..bbd3a42e 100644 --- a/src/vlib/node.c +++ b/src/vlib/node.c @@ -99,7 +99,7 @@ vlib_node_runtime_update (vlib_main_t * vm, u32 node_index, u32 next_index) vlib_pending_frame_t *pf; i32 i, j, n_insert; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); vlib_worker_thread_barrier_sync (vm); diff --git a/src/vlib/node.h b/src/vlib/node.h index fc7e7da2..1e2f4c38 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -344,8 +344,8 @@ typedef struct vlib_frame_t /* Number of vector elements currently in frame. */ u16 n_vectors; - /* Owner cpuid / heap id */ - u16 cpu_index; + /* Owner thread / heap id */ + u16 thread_index; /* Scalar and vector arguments to next node. */ u8 arguments[0]; @@ -459,7 +459,7 @@ typedef struct vlib_node_runtime_t zero before first run of this node. */ - u16 cpu_index; /**< CPU this node runs on */ + u16 thread_index; /**< thread this node runs on */ u8 runtime_data[0]; /**< Function dependent node-runtime data. This data is diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h index 1f7d94e1..54e36874 100644 --- a/src/vlib/node_funcs.h +++ b/src/vlib/node_funcs.h @@ -201,9 +201,9 @@ always_inline vlib_frame_t * vlib_get_frame_no_check (vlib_main_t * vm, uword frame_index) { vlib_frame_t *f; - u32 cpu_index = frame_index & VLIB_CPU_MASK; + u32 thread_index = frame_index & VLIB_CPU_MASK; u32 offset = frame_index & VLIB_OFFSET_MASK; - vm = vlib_mains[cpu_index]; + vm = vlib_mains[thread_index]; f = vm->heap_base + offset; return f; } @@ -215,10 +215,10 @@ vlib_frame_index_no_check (vlib_main_t * vm, vlib_frame_t * f) ASSERT (((uword) f & VLIB_CPU_MASK) == 0); - vm = vlib_mains[f->cpu_index]; + vm = vlib_mains[f->thread_index]; i = ((u8 *) f - (u8 *) vm->heap_base); - return i | f->cpu_index; + return i | f->thread_index; } always_inline vlib_frame_t * diff --git a/src/vlib/threads.c b/src/vlib/threads.c index ef3a24d3..4a111f8d 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -35,27 +35,12 @@ vl (void *p) vlib_worker_thread_t *vlib_worker_threads; vlib_thread_main_t vlib_thread_main; +__thread uword vlib_thread_index = 0; + uword os_get_cpu_number (void) { - void *sp; - uword n; - u32 len; - - len = vec_len (vlib_thread_stacks); - if (len == 0) - return 0; - - /* Get any old stack address. */ - sp = &sp; - - n = ((uword) sp - (uword) vlib_thread_stacks[0]) - >> VLIB_LOG2_THREAD_STACK_SIZE; - - /* "processes" have their own stacks, and they always run in thread 0 */ - n = n >= len ? 0 : n; - - return n; + return vlib_thread_index; } uword @@ -275,21 +260,6 @@ vlib_thread_init (vlib_main_t * vm) return 0; } -vlib_worker_thread_t * -vlib_alloc_thread (vlib_main_t * vm) -{ - vlib_worker_thread_t *w; - - if (vec_len (vlib_worker_threads) >= vec_len (vlib_thread_stacks)) - { - clib_warning ("out of worker threads... Quitting..."); - exit (1); - } - vec_add2 (vlib_worker_threads, w, 1); - w->thread_stack = vlib_thread_stacks[w - vlib_worker_threads]; - return w; -} - vlib_frame_queue_t * vlib_frame_queue_alloc (int nelts) { @@ -427,7 +397,7 @@ vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index, f64 b4 = vlib_time_now_ticks (vm, before); vlib_worker_thread_barrier_check (vm, b4); /* Bad idea. Dequeue -> enqueue -> dequeue -> trouble */ - // vlib_frame_queue_dequeue (vm->cpu_index, vm, nm); + // vlib_frame_queue_dequeue (vm->thread_index, vm, nm); } elt = fq->elts + (new_tail & (fq->nelts - 1)); @@ -497,6 +467,8 @@ vlib_worker_thread_bootstrap_fn (void *arg) w->lwp = syscall (SYS_gettid); w->thread_id = pthread_self (); + vlib_thread_index = w - vlib_worker_threads; + rv = (void *) clib_calljmp ((uword (*)(uword)) w->thread_function, (uword) arg, w->thread_stack + VLIB_THREAD_STACK_SIZE); @@ -610,7 +582,9 @@ start_workers (vlib_main_t * vm) mheap_alloc (0 /* use VM */ , tr->mheap_size); else w->thread_mheap = main_heap; - w->thread_stack = vlib_thread_stacks[w - vlib_worker_threads]; + + w->thread_stack = + vlib_thread_stack_init (w - vlib_worker_threads); w->thread_function = tr->function; w->thread_function_arg = w; w->instance_id = k; @@ -630,7 +604,7 @@ start_workers (vlib_main_t * vm) vm_clone = clib_mem_alloc (sizeof (*vm_clone)); clib_memcpy (vm_clone, vlib_mains[0], sizeof (*vm_clone)); - vm_clone->cpu_index = worker_thread_index; + vm_clone->thread_index = worker_thread_index; vm_clone->heap_base = w->thread_mheap; vm_clone->mbuf_alloc_list = 0; vm_clone->init_functions_called = @@ -679,7 +653,7 @@ start_workers (vlib_main_t * vm) vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) { vlib_node_t *n = vlib_get_node (vm, rt->node_index); - rt->cpu_index = vm_clone->cpu_index; + rt->thread_index = vm_clone->thread_index; /* copy initial runtime_data from node */ if (n->runtime_data && n->runtime_data_bytes > 0) clib_memcpy (rt->runtime_data, n->runtime_data, @@ -692,7 +666,7 @@ start_workers (vlib_main_t * vm) vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) { vlib_node_t *n = vlib_get_node (vm, rt->node_index); - rt->cpu_index = vm_clone->cpu_index; + rt->thread_index = vm_clone->thread_index; /* copy initial runtime_data from node */ if (n->runtime_data && n->runtime_data_bytes > 0) clib_memcpy (rt->runtime_data, n->runtime_data, @@ -756,7 +730,8 @@ start_workers (vlib_main_t * vm) mheap_alloc (0 /* use VM */ , tr->mheap_size); else w->thread_mheap = main_heap; - w->thread_stack = vlib_thread_stacks[w - vlib_worker_threads]; + w->thread_stack = + vlib_thread_stack_init (w - vlib_worker_threads); w->thread_function = tr->function; w->thread_function_arg = w; w->instance_id = j; @@ -827,7 +802,7 @@ vlib_worker_thread_node_runtime_update (void) uword n_calls, uword n_vectors, uword n_clocks); - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); if (vec_len (vlib_mains) == 1) return; @@ -835,7 +810,7 @@ vlib_worker_thread_node_runtime_update (void) vm = vlib_mains[0]; nm = &vm->node_main; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); ASSERT (*vlib_worker_threads->wait_at_barrier == 1); /* @@ -955,7 +930,7 @@ vlib_worker_thread_node_runtime_update (void) vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]) { vlib_node_t *n = vlib_get_node (vm, rt->node_index); - rt->cpu_index = vm_clone->cpu_index; + rt->thread_index = vm_clone->thread_index; /* copy runtime_data, will be overwritten later for existing rt */ if (n->runtime_data && n->runtime_data_bytes > 0) clib_memcpy (rt->runtime_data, n->runtime_data, @@ -981,7 +956,7 @@ vlib_worker_thread_node_runtime_update (void) vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) { vlib_node_t *n = vlib_get_node (vm, rt->node_index); - rt->cpu_index = vm_clone->cpu_index; + rt->thread_index = vm_clone->thread_index; /* copy runtime_data, will be overwritten later for existing rt */ if (n->runtime_data && n->runtime_data_bytes > 0) clib_memcpy (rt->runtime_data, n->runtime_data, @@ -1180,7 +1155,7 @@ vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which) if (vlib_mains == 0) return; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); vlib_worker_thread_barrier_sync (vm); switch (which) @@ -1212,7 +1187,7 @@ vlib_worker_thread_barrier_sync (vlib_main_t * vm) vlib_worker_threads[0].barrier_sync_count++; - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); deadline = vlib_time_now (vm) + BARRIER_SYNC_TIMEOUT; @@ -1260,7 +1235,7 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm) int vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm) { - u32 thread_id = vm->cpu_index; + u32 thread_id = vm->thread_index; vlib_frame_queue_t *fq = fqm->vlib_frame_queues[thread_id]; vlib_frame_queue_elt_t *elt; u32 *from, *to; @@ -1393,7 +1368,7 @@ vlib_worker_thread_fn (void *arg) vlib_main_t *vm = vlib_get_main (); clib_error_t *e; - ASSERT (vm->cpu_index == os_get_cpu_number ()); + ASSERT (vm->thread_index == vlib_get_thread_index ()); vlib_worker_thread_init (w); clib_time_init (&vm->clib_time); diff --git a/src/vlib/threads.h b/src/vlib/threads.h index eca4fc26..101d3d4a 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -153,8 +153,6 @@ typedef struct /* Called early, in thread 0's context */ clib_error_t *vlib_thread_init (vlib_main_t * vm); -vlib_worker_thread_t *vlib_alloc_thread (vlib_main_t * vm); - int vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t * frame, vlib_frame_queue_msg_type_t type); @@ -183,12 +181,19 @@ u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts); void vlib_worker_thread_barrier_sync (vlib_main_t * vm); void vlib_worker_thread_barrier_release (vlib_main_t * vm); +extern __thread uword vlib_thread_index; +static_always_inline uword +vlib_get_thread_index (void) +{ + return vlib_thread_index; +} + always_inline void vlib_smp_unsafe_warning (void) { if (CLIB_DEBUG > 0) { - if (os_get_cpu_number ()) + if (vlib_get_thread_index ()) fformat (stderr, "%s: SMP unsafe warning...\n", __FUNCTION__); } } @@ -331,21 +336,21 @@ vlib_num_workers () } always_inline u32 -vlib_get_worker_cpu_index (u32 worker_index) +vlib_get_worker_thread_index (u32 worker_index) { return worker_index + 1; } always_inline u32 -vlib_get_worker_index (u32 cpu_index) +vlib_get_worker_index (u32 thread_index) { - return cpu_index - 1; + return thread_index - 1; } always_inline u32 vlib_get_current_worker_index () { - return os_get_cpu_number () - 1; + return vlib_get_thread_index () - 1; } static inline void @@ -467,6 +472,8 @@ vlib_get_worker_handoff_queue_elt (u32 frame_queue_index, return elt; } +u8 *vlib_thread_stack_init (uword thread_index); + int vlib_thread_cb_register (struct vlib_main_t *vm, vlib_thread_callbacks_t * cb); diff --git a/src/vlib/unix/cj.c b/src/vlib/unix/cj.c index 33ba163a..7c1e9475 100644 --- a/src/vlib/unix/cj.c +++ b/src/vlib/unix/cj.c @@ -48,7 +48,7 @@ cj_log (u32 type, void *data0, void *data1) r = (cj_record_t *) & (cjm->records[new_tail & (cjm->num_records - 1)]); r->time = vlib_time_now (cjm->vlib_main); - r->cpu = os_get_cpu_number (); + r->thread_index = vlib_get_thread_index (); r->type = type; r->data[0] = pointer_to_uword (data0); r->data[1] = pointer_to_uword (data1); @@ -133,7 +133,8 @@ static inline void cj_dump_one_record (cj_record_t * r) { fprintf (stderr, "[%d]: %10.6f T%02d %llx %llx\n", - r->cpu, r->time, r->type, (long long unsigned int) r->data[0], + r->thread_index, r->time, r->type, + (long long unsigned int) r->data[0], (long long unsigned int) r->data[1]); } @@ -161,7 +162,7 @@ cj_dump_internal (u8 filter0_enable, u64 filter0, index = (cjm->tail + 1) & (cjm->num_records - 1); r = &(cjm->records[index]); - if (r->cpu != (u32) ~ 0) + if (r->thread_index != (u32) ~ 0) { /* Yes, dump from tail + 1 to the end */ for (i = index; i < cjm->num_records; i++) diff --git a/src/vlib/unix/cj.h b/src/vlib/unix/cj.h index 67626afe..d0a1d46e 100644 --- a/src/vlib/unix/cj.h +++ b/src/vlib/unix/cj.h @@ -23,7 +23,7 @@ typedef struct { f64 time; - u32 cpu; + u32 thread_index; u32 type; u64 data[2]; } cj_record_t; diff --git a/src/vlib/unix/main.c b/src/vlib/unix/main.c index 6b96cc0d..db5ddd64 100644 --- a/src/vlib/unix/main.c +++ b/src/vlib/unix/main.c @@ -510,13 +510,28 @@ thread0 (uword arg) return i; } +u8 * +vlib_thread_stack_init (uword thread_index) +{ + vec_validate (vlib_thread_stacks, thread_index); + vlib_thread_stacks[thread_index] = clib_mem_alloc_aligned + (VLIB_THREAD_STACK_SIZE, VLIB_THREAD_STACK_SIZE); + + /* + * Disallow writes to the bottom page of the stack, to + * catch stack overflows. + */ + if (mprotect (vlib_thread_stacks[thread_index], + clib_mem_get_page_size (), PROT_READ) < 0) + clib_unix_warning ("thread stack"); + return vlib_thread_stacks[thread_index]; +} + int vlib_unix_main (int argc, char *argv[]) { vlib_main_t *vm = &vlib_global_main; /* one and only time for this! */ - vlib_thread_main_t *tm = &vlib_thread_main; unformat_input_t input; - u8 *thread_stacks; clib_error_t *e; int i; @@ -548,29 +563,9 @@ vlib_unix_main (int argc, char *argv[]) } unformat_free (&input); - /* - * allocate n x VLIB_THREAD_STACK_SIZE stacks, aligned to a - * VLIB_THREAD_STACK_SIZE boundary - * See also: os_get_cpu_number() in vlib/vlib/threads.c - */ - thread_stacks = clib_mem_alloc_aligned - ((uword) tm->n_thread_stacks * VLIB_THREAD_STACK_SIZE, - VLIB_THREAD_STACK_SIZE); - - vec_validate (vlib_thread_stacks, tm->n_thread_stacks - 1); - for (i = 0; i < vec_len (vlib_thread_stacks); i++) - { - vlib_thread_stacks[i] = thread_stacks; - - /* - * Disallow writes to the bottom page of the stack, to - * catch stack overflows. - */ - if (mprotect (thread_stacks, clib_mem_get_page_size (), PROT_READ) < 0) - clib_unix_warning ("thread stack"); + vlib_thread_stack_init (0); - thread_stacks += VLIB_THREAD_STACK_SIZE; - } + vlib_thread_index = 0; i = clib_calljmp (thread0, (uword) vm, (void *) (vlib_thread_stacks[0] + diff --git a/src/vnet/adj/adj_l2.c b/src/vnet/adj/adj_l2.c index f68e54e0..20d70dd4 100644 --- a/src/vnet/adj/adj_l2.c +++ b/src/vnet/adj/adj_l2.c @@ -52,7 +52,7 @@ adj_l2_rewrite_inline (vlib_main_t * vm, { u32 * from = vlib_frame_vector_args (frame); u32 n_left_from, n_left_to_next, * to_next, next_index; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); ethernet_main_t * em = ðernet_main; n_left_from = frame->n_vectors; @@ -93,7 +93,7 @@ adj_l2_rewrite_inline (vlib_main_t * vm, vnet_buffer(p0)->sw_if_index[VLIB_TX] = adj0->rewrite_header.sw_if_index; vlib_increment_combined_counter(&adjacency_counters, - cpu_index, + thread_index, adj_index0, /* packet increment */ 0, /* byte increment */ rw_len0); diff --git a/src/vnet/adj/adj_midchain.c b/src/vnet/adj/adj_midchain.c index e8087f08..5756de43 100644 --- a/src/vnet/adj/adj_midchain.c +++ b/src/vnet/adj/adj_midchain.c @@ -49,7 +49,7 @@ adj_midchain_tx_inline (vlib_main_t * vm, u32 next_index; vnet_main_t *vnm = vnet_get_main (); vnet_interface_main_t *im = &vnm->interface_main; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; /* Vector of buffer / pkt indices we're supposed to process */ from = vlib_frame_vector_args (frame); @@ -124,13 +124,13 @@ adj_midchain_tx_inline (vlib_main_t * vm, { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, + thread_index, adj0->rewrite_header.sw_if_index, 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, + thread_index, adj1->rewrite_header.sw_if_index, 1, vlib_buffer_length_in_chain (vm, b1)); @@ -181,7 +181,7 @@ adj_midchain_tx_inline (vlib_main_t * vm, { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, + thread_index, adj0->rewrite_header.sw_if_index, 1, vlib_buffer_length_in_chain (vm, b0)); diff --git a/src/vnet/adj/adj_nsh.c b/src/vnet/adj/adj_nsh.c index 9a0f9d8b..128570b0 100644 --- a/src/vnet/adj/adj_nsh.c +++ b/src/vnet/adj/adj_nsh.c @@ -53,7 +53,7 @@ adj_nsh_rewrite_inline (vlib_main_t * vm, { u32 * from = vlib_frame_vector_args (frame); u32 n_left_from, n_left_to_next, * to_next, next_index; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); n_left_from = frame->n_vectors; next_index = node->cached_next_index; @@ -94,7 +94,7 @@ adj_nsh_rewrite_inline (vlib_main_t * vm, vnet_buffer(p0)->ip.save_rewrite_length = rw_len0; vlib_increment_combined_counter(&adjacency_counters, - cpu_index, + thread_index, adj_index0, /* packet increment */ 0, /* byte increment */ rw_len0); diff --git a/src/vnet/classify/vnet_classify.c b/src/vnet/classify/vnet_classify.c index 98842a48..70a189b0 100644 --- a/src/vnet/classify/vnet_classify.c +++ b/src/vnet/classify/vnet_classify.c @@ -251,12 +251,12 @@ static inline void make_working_copy vnet_classify_entry_##size##_t * working_copy##size = 0; foreach_size_in_u32x4; #undef _ - u32 cpu_number = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); - if (cpu_number >= vec_len (t->working_copies)) + if (thread_index >= vec_len (t->working_copies)) { oldheap = clib_mem_set_heap (t->mheap); - vec_validate (t->working_copies, cpu_number); + vec_validate (t->working_copies, thread_index); clib_mem_set_heap (oldheap); } @@ -265,7 +265,7 @@ static inline void make_working_copy * updates from multiple threads will not result in sporadic, spurious * lookup failures. */ - working_copy = t->working_copies[cpu_number]; + working_copy = t->working_copies[thread_index]; t->saved_bucket.as_u64 = b->as_u64; oldheap = clib_mem_set_heap (t->mheap); @@ -290,7 +290,7 @@ static inline void make_working_copy default: abort(); } - t->working_copies[cpu_number] = working_copy; + t->working_copies[thread_index] = working_copy; } _vec_len(working_copy) = (1<log2_pages)*t->entries_per_page; @@ -318,7 +318,7 @@ static inline void make_working_copy working_bucket.offset = vnet_classify_get_offset (t, working_copy); CLIB_MEMORY_BARRIER(); b->as_u64 = working_bucket.as_u64; - t->working_copies[cpu_number] = working_copy; + t->working_copies[thread_index] = working_copy; } static vnet_classify_entry_t * @@ -387,7 +387,7 @@ int vnet_classify_add_del (vnet_classify_table_t * t, int i; u64 hash, new_hash; u32 new_log2_pages; - u32 cpu_number = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); u8 * key_minus_skip; ASSERT ((add_v->flags & VNET_CLASSIFY_ENTRY_FREE) == 0); @@ -498,7 +498,7 @@ int vnet_classify_add_del (vnet_classify_table_t * t, new_log2_pages = t->saved_bucket.log2_pages + 1; expand_again: - working_copy = t->working_copies[cpu_number]; + working_copy = t->working_copies[thread_index]; new_v = split_and_rehash (t, working_copy, new_log2_pages); if (new_v == 0) diff --git a/src/vnet/cop/ip4_whitelist.c b/src/vnet/cop/ip4_whitelist.c index 6ef3d7d7..1b5e336b 100644 --- a/src/vnet/cop/ip4_whitelist.c +++ b/src/vnet/cop/ip4_whitelist.c @@ -60,7 +60,7 @@ ip4_cop_whitelist_node_fn (vlib_main_t * vm, cop_feature_type_t next_index; cop_main_t *cm = &cop_main; vlib_combined_counter_main_t * vcm = &load_balance_main.lbm_via_counters; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -177,12 +177,12 @@ ip4_cop_whitelist_node_fn (vlib_main_t * vm, dpo1 = load_balance_get_bucket_i(lb1, 0); vlib_increment_combined_counter - (vcm, cpu_index, lb_index0, 1, + (vcm, thread_index, lb_index0, 1, vlib_buffer_length_in_chain (vm, b0) + sizeof(ethernet_header_t)); vlib_increment_combined_counter - (vcm, cpu_index, lb_index1, 1, + (vcm, thread_index, lb_index1, 1, vlib_buffer_length_in_chain (vm, b1) + sizeof(ethernet_header_t)); @@ -273,7 +273,7 @@ ip4_cop_whitelist_node_fn (vlib_main_t * vm, dpo0 = load_balance_get_bucket_i(lb0, 0); vlib_increment_combined_counter - (vcm, cpu_index, lb_index0, 1, + (vcm, thread_index, lb_index0, 1, vlib_buffer_length_in_chain (vm, b0) + sizeof(ethernet_header_t)); diff --git a/src/vnet/cop/ip6_whitelist.c b/src/vnet/cop/ip6_whitelist.c index c2e16ccf..f3fe62e3 100644 --- a/src/vnet/cop/ip6_whitelist.c +++ b/src/vnet/cop/ip6_whitelist.c @@ -61,7 +61,7 @@ ip6_cop_whitelist_node_fn (vlib_main_t * vm, cop_main_t *cm = &cop_main; ip6_main_t * im6 = &ip6_main; vlib_combined_counter_main_t * vcm = &load_balance_main.lbm_via_counters; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -153,12 +153,12 @@ ip6_cop_whitelist_node_fn (vlib_main_t * vm, dpo1 = load_balance_get_bucket_i(lb1, 0); vlib_increment_combined_counter - (vcm, cpu_index, lb_index0, 1, + (vcm, thread_index, lb_index0, 1, vlib_buffer_length_in_chain (vm, b0) + sizeof(ethernet_header_t)); vlib_increment_combined_counter - (vcm, cpu_index, lb_index1, 1, + (vcm, thread_index, lb_index1, 1, vlib_buffer_length_in_chain (vm, b1) + sizeof(ethernet_header_t)); @@ -233,7 +233,7 @@ ip6_cop_whitelist_node_fn (vlib_main_t * vm, dpo0 = load_balance_get_bucket_i(lb0, 0); vlib_increment_combined_counter - (vcm, cpu_index, lb_index0, 1, + (vcm, thread_index, lb_index0, 1, vlib_buffer_length_in_chain (vm, b0) + sizeof(ethernet_header_t)); diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c index ba337f3f..76980102 100644 --- a/src/vnet/devices/af_packet/node.c +++ b/src/vnet/devices/af_packet/node.c @@ -124,7 +124,7 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, u32 frame_num = apif->rx_req->tp_frame_nr; u8 *block_start = apif->rx_ring + block * block_size; uword n_trace = vlib_get_trace_count (vm, node); - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); u32 min_bufs = apif->rx_req->tp_frame_size / n_buffer_bytes; @@ -132,15 +132,15 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, if (apif->per_interface_next_index != ~0) next_index = apif->per_interface_next_index; - n_free_bufs = vec_len (apm->rx_buffers[cpu_index]); + n_free_bufs = vec_len (apm->rx_buffers[thread_index]); if (PREDICT_FALSE (n_free_bufs < VLIB_FRAME_SIZE)) { - vec_validate (apm->rx_buffers[cpu_index], + vec_validate (apm->rx_buffers[thread_index], VLIB_FRAME_SIZE + n_free_bufs - 1); n_free_bufs += - vlib_buffer_alloc (vm, &apm->rx_buffers[cpu_index][n_free_bufs], + vlib_buffer_alloc (vm, &apm->rx_buffers[thread_index][n_free_bufs], VLIB_FRAME_SIZE); - _vec_len (apm->rx_buffers[cpu_index]) = n_free_bufs; + _vec_len (apm->rx_buffers[thread_index]) = n_free_bufs; } rx_frame = apif->next_rx_frame; @@ -163,11 +163,11 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, { /* grab free buffer */ u32 last_empty_buffer = - vec_len (apm->rx_buffers[cpu_index]) - 1; + vec_len (apm->rx_buffers[thread_index]) - 1; prev_bi0 = bi0; - bi0 = apm->rx_buffers[cpu_index][last_empty_buffer]; + bi0 = apm->rx_buffers[thread_index][last_empty_buffer]; b0 = vlib_get_buffer (vm, bi0); - _vec_len (apm->rx_buffers[cpu_index]) = last_empty_buffer; + _vec_len (apm->rx_buffers[thread_index]) = last_empty_buffer; n_free_bufs--; /* copy data */ @@ -236,9 +236,9 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_increment_combined_counter (vnet_get_main ()->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number (), apif->hw_if_index, n_rx_packets, n_rx_bytes); + vlib_get_thread_index (), apif->hw_if_index, n_rx_packets, n_rx_bytes); - vnet_device_increment_rx_packets (cpu_index, n_rx_packets); + vnet_device_increment_rx_packets (thread_index, n_rx_packets); return n_rx_packets; } diff --git a/src/vnet/devices/devices.c b/src/vnet/devices/devices.c index 41645220..5e5e812c 100644 --- a/src/vnet/devices/devices.c +++ b/src/vnet/devices/devices.c @@ -104,7 +104,7 @@ vnet_device_queue_sort (void *a1, void *a2) void vnet_device_input_assign_thread (u32 hw_if_index, - u16 queue_id, uword cpu_index) + u16 queue_id, uword thread_index) { vnet_main_t *vnm = vnet_get_main (); vnet_device_main_t *vdm = &vnet_device_main; @@ -115,19 +115,19 @@ vnet_device_input_assign_thread (u32 hw_if_index, ASSERT (hw->input_node_index > 0); - if (vdm->first_worker_cpu_index == 0) - cpu_index = 0; + if (vdm->first_worker_thread_index == 0) + thread_index = 0; - if (cpu_index != 0 && - (cpu_index < vdm->first_worker_cpu_index || - cpu_index > vdm->last_worker_cpu_index)) + if (thread_index != 0 && + (thread_index < vdm->first_worker_thread_index || + thread_index > vdm->last_worker_thread_index)) { - cpu_index = vdm->next_worker_cpu_index++; - if (vdm->next_worker_cpu_index > vdm->last_worker_cpu_index) - vdm->next_worker_cpu_index = vdm->first_worker_cpu_index; + thread_index = vdm->next_worker_thread_index++; + if (vdm->next_worker_thread_index > vdm->last_worker_thread_index) + vdm->next_worker_thread_index = vdm->first_worker_thread_index; } - vm = vlib_mains[cpu_index]; + vm = vlib_mains[thread_index]; rt = vlib_node_get_runtime_data (vm, hw->input_node_index); vec_add2 (rt->devices_and_queues, dq, 1); @@ -136,33 +136,33 @@ vnet_device_input_assign_thread (u32 hw_if_index, dq->queue_id = queue_id; vec_sort_with_function (rt->devices_and_queues, vnet_device_queue_sort); - vec_validate (hw->input_node_cpu_index_by_queue, queue_id); - hw->input_node_cpu_index_by_queue[queue_id] = cpu_index; + vec_validate (hw->input_node_thread_index_by_queue, queue_id); + hw->input_node_thread_index_by_queue[queue_id] = thread_index; } static int vnet_device_input_unassign_thread (u32 hw_if_index, u16 queue_id, - uword cpu_index) + uword thread_index) { vnet_main_t *vnm = vnet_get_main (); vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); vnet_device_input_runtime_t *rt; vnet_device_and_queue_t *dq; - uword old_cpu_index; + uword old_thread_index; - if (hw->input_node_cpu_index_by_queue == 0) + if (hw->input_node_thread_index_by_queue == 0) return VNET_API_ERROR_INVALID_INTERFACE; - if (vec_len (hw->input_node_cpu_index_by_queue) < queue_id + 1) + if (vec_len (hw->input_node_thread_index_by_queue) < queue_id + 1) return VNET_API_ERROR_INVALID_INTERFACE; - old_cpu_index = hw->input_node_cpu_index_by_queue[queue_id]; + old_thread_index = hw->input_node_thread_index_by_queue[queue_id]; - if (old_cpu_index == cpu_index) + if (old_thread_index == thread_index) return 0; rt = - vlib_node_get_runtime_data (vlib_mains[old_cpu_index], + vlib_node_get_runtime_data (vlib_mains[old_thread_index], hw->input_node_index); vec_foreach (dq, rt->devices_and_queues) @@ -240,7 +240,7 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input, vnet_device_main_t *vdm = &vnet_device_main; u32 hw_if_index = (u32) ~ 0; u32 queue_id = (u32) 0; - u32 cpu_index = (u32) ~ 0; + u32 thread_index = (u32) ~ 0; int rv; if (!unformat_user (input, unformat_line_input, line_input)) @@ -253,10 +253,10 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input, ; else if (unformat (line_input, "queue %d", &queue_id)) ; - else if (unformat (line_input, "main", &cpu_index)) - cpu_index = 0; - else if (unformat (line_input, "worker %d", &cpu_index)) - cpu_index += vdm->first_worker_cpu_index; + else if (unformat (line_input, "main", &thread_index)) + thread_index = 0; + else if (unformat (line_input, "worker %d", &thread_index)) + thread_index += vdm->first_worker_thread_index; else { error = clib_error_return (0, "parse error: '%U'", @@ -271,16 +271,17 @@ set_device_placement (vlib_main_t * vm, unformat_input_t * input, if (hw_if_index == (u32) ~ 0) return clib_error_return (0, "please specify valid interface name"); - if (cpu_index > vdm->last_worker_cpu_index) + if (thread_index > vdm->last_worker_thread_index) return clib_error_return (0, "please specify valid worker thread or main"); - rv = vnet_device_input_unassign_thread (hw_if_index, queue_id, cpu_index); + rv = + vnet_device_input_unassign_thread (hw_if_index, queue_id, thread_index); if (rv) return clib_error_return (0, "not found"); - vnet_device_input_assign_thread (hw_if_index, queue_id, cpu_index); + vnet_device_input_assign_thread (hw_if_index, queue_id, thread_index); return 0; } @@ -326,9 +327,9 @@ vnet_device_init (vlib_main_t * vm) tr = p ? (vlib_thread_registration_t *) p[0] : 0; if (tr && tr->count > 0) { - vdm->first_worker_cpu_index = tr->first_index; - vdm->next_worker_cpu_index = tr->first_index; - vdm->last_worker_cpu_index = tr->first_index + tr->count - 1; + vdm->first_worker_thread_index = tr->first_index; + vdm->next_worker_thread_index = tr->first_index; + vdm->last_worker_thread_index = tr->first_index + tr->count - 1; } return 0; } diff --git a/src/vnet/devices/devices.h b/src/vnet/devices/devices.h index bbb29fe3..966f8302 100644 --- a/src/vnet/devices/devices.h +++ b/src/vnet/devices/devices.h @@ -50,9 +50,9 @@ typedef struct typedef struct { vnet_device_per_worker_data_t *workers; - uword first_worker_cpu_index; - uword last_worker_cpu_index; - uword next_worker_cpu_index; + uword first_worker_thread_index; + uword last_worker_thread_index; + uword next_worker_thread_index; } vnet_device_main_t; typedef struct @@ -80,7 +80,7 @@ vnet_set_device_input_node (u32 hw_if_index, u32 node_index) } void vnet_device_input_assign_thread (u32 hw_if_index, u16 queue_id, - uword cpu_index); + uword thread_index); static inline u64 vnet_get_aggregate_rx_packets (void) @@ -95,12 +95,12 @@ vnet_get_aggregate_rx_packets (void) } static inline void -vnet_device_increment_rx_packets (u32 cpu_index, u64 count) +vnet_device_increment_rx_packets (u32 thread_index, u64 count) { vnet_device_main_t *vdm = &vnet_device_main; vnet_device_per_worker_data_t *pwd; - pwd = vec_elt_at_index (vdm->workers, cpu_index); + pwd = vec_elt_at_index (vdm->workers, thread_index); pwd->aggregate_rx_packets += count; } @@ -117,9 +117,9 @@ vnet_device_input_set_interrupt_pending (vnet_main_t * vnm, u32 hw_if_index, { vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); - ASSERT (queue_id < vec_len (hw->input_node_cpu_index_by_queue)); - u32 cpu_index = hw->input_node_cpu_index_by_queue[queue_id]; - vlib_node_set_interrupt_pending (vlib_mains[cpu_index], + ASSERT (queue_id < vec_len (hw->input_node_thread_index_by_queue)); + u32 thread_index = hw->input_node_thread_index_by_queue[queue_id]; + vlib_node_set_interrupt_pending (vlib_mains[thread_index], hw->input_node_index); } diff --git a/src/vnet/devices/netmap/node.c b/src/vnet/devices/netmap/node.c index 68ea7832..e120eeae 100644 --- a/src/vnet/devices/netmap/node.c +++ b/src/vnet/devices/netmap/node.c @@ -98,22 +98,22 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, u32 n_free_bufs; struct netmap_ring *ring; int cur_ring; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); if (nif->per_interface_next_index != ~0) next_index = nif->per_interface_next_index; - n_free_bufs = vec_len (nm->rx_buffers[cpu_index]); + n_free_bufs = vec_len (nm->rx_buffers[thread_index]); if (PREDICT_FALSE (n_free_bufs < VLIB_FRAME_SIZE)) { - vec_validate (nm->rx_buffers[cpu_index], + vec_validate (nm->rx_buffers[thread_index], VLIB_FRAME_SIZE + n_free_bufs - 1); n_free_bufs += - vlib_buffer_alloc (vm, &nm->rx_buffers[cpu_index][n_free_bufs], + vlib_buffer_alloc (vm, &nm->rx_buffers[thread_index][n_free_bufs], VLIB_FRAME_SIZE); - _vec_len (nm->rx_buffers[cpu_index]) = n_free_bufs; + _vec_len (nm->rx_buffers[thread_index]) = n_free_bufs; } cur_ring = nif->first_rx_ring; @@ -163,11 +163,11 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_buffer_t *b0; /* grab free buffer */ u32 last_empty_buffer = - vec_len (nm->rx_buffers[cpu_index]) - 1; + vec_len (nm->rx_buffers[thread_index]) - 1; prev_bi0 = bi0; - bi0 = nm->rx_buffers[cpu_index][last_empty_buffer]; + bi0 = nm->rx_buffers[thread_index][last_empty_buffer]; b0 = vlib_get_buffer (vm, bi0); - _vec_len (nm->rx_buffers[cpu_index]) = last_empty_buffer; + _vec_len (nm->rx_buffers[thread_index]) = last_empty_buffer; n_free_bufs--; /* copy data */ @@ -247,9 +247,9 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_increment_combined_counter (vnet_get_main ()->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number (), nif->hw_if_index, n_rx_packets, n_rx_bytes); + vlib_get_thread_index (), nif->hw_if_index, n_rx_packets, n_rx_bytes); - vnet_device_increment_rx_packets (cpu_index, n_rx_packets); + vnet_device_increment_rx_packets (thread_index, n_rx_packets); return n_rx_packets; } @@ -260,7 +260,7 @@ netmap_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, { int i; u32 n_rx_packets = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); netmap_main_t *nm = &netmap_main; netmap_if_t *nmi; @@ -269,7 +269,7 @@ netmap_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, nmi = vec_elt_at_index (nm->interfaces, i); if (nmi->is_admin_up && (i % nm->input_cpu_count) == - (cpu_index - nm->input_cpu_first_index)) + (thread_index - nm->input_cpu_first_index)) n_rx_packets += netmap_device_input_fn (vm, node, frame, nmi); } diff --git a/src/vnet/devices/ssvm/node.c b/src/vnet/devices/ssvm/node.c index a6c9dfd7..539b4161 100644 --- a/src/vnet/devices/ssvm/node.c +++ b/src/vnet/devices/ssvm/node.c @@ -89,7 +89,7 @@ ssvm_eth_device_input (ssvm_eth_main_t * em, ethernet_header_t *eh0; u16 type0; u32 n_rx_bytes = 0, l3_offset0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 trace_cnt __attribute__ ((unused)) = vlib_get_trace_count (vm, node); volatile u32 *lock; u32 *elt_indices; @@ -284,10 +284,10 @@ out: vlib_increment_combined_counter (vnet_get_main ()->interface_main.combined_sw_if_counters - + VNET_INTERFACE_COUNTER_RX, cpu_index, + + VNET_INTERFACE_COUNTER_RX, thread_index, intfc->vlib_hw_if_index, rx_queue_index, n_rx_bytes); - vnet_device_increment_rx_packets (cpu_index, rx_queue_index); + vnet_device_increment_rx_packets (thread_index, rx_queue_index); return rx_queue_index; } diff --git a/src/vnet/devices/virtio/vhost-user.c b/src/vnet/devices/virtio/vhost-user.c index 00807dc0..5e720f65 100644 --- a/src/vnet/devices/virtio/vhost-user.c +++ b/src/vnet/devices/virtio/vhost-user.c @@ -331,7 +331,7 @@ vhost_user_tx_thread_placement (vhost_user_intf_t * vui) { //Let's try to assign one queue to each thread u32 qid = 0; - u32 cpu_index = 0; + u32 thread_index = 0; vui->use_tx_spinlock = 0; while (1) { @@ -341,20 +341,21 @@ vhost_user_tx_thread_placement (vhost_user_intf_t * vui) if (!rxvq->started || !rxvq->enabled) continue; - vui->per_cpu_tx_qid[cpu_index] = qid; - cpu_index++; - if (cpu_index == vlib_get_thread_main ()->n_vlib_mains) + vui->per_cpu_tx_qid[thread_index] = qid; + thread_index++; + if (thread_index == vlib_get_thread_main ()->n_vlib_mains) return; } //We need to loop, meaning the spinlock has to be used vui->use_tx_spinlock = 1; - if (cpu_index == 0) + if (thread_index == 0) { //Could not find a single valid one - for (cpu_index = 0; - cpu_index < vlib_get_thread_main ()->n_vlib_mains; cpu_index++) + for (thread_index = 0; + thread_index < vlib_get_thread_main ()->n_vlib_mains; + thread_index++) { - vui->per_cpu_tx_qid[cpu_index] = 0; + vui->per_cpu_tx_qid[thread_index] = 0; } return; } @@ -368,7 +369,7 @@ vhost_user_rx_thread_placement () vhost_user_intf_t *vui; vhost_cpu_t *vhc; u32 *workers = 0; - u32 cpu_index; + u32 thread_index; vlib_main_t *vm; //Let's list all workers cpu indexes @@ -400,9 +401,9 @@ vhost_user_rx_thread_placement () continue; i %= vec_len (vui_workers); - cpu_index = vui_workers[i]; + thread_index = vui_workers[i]; i++; - vhc = &vum->cpus[cpu_index]; + vhc = &vum->cpus[thread_index]; iaq.qid = qid; iaq.vhost_iface_index = vui - vum->vhost_user_interfaces; @@ -429,14 +430,14 @@ vhost_user_rx_thread_placement () vhc->operation_mode = mode; } - for (cpu_index = vum->input_cpu_first_index; - cpu_index < vum->input_cpu_first_index + vum->input_cpu_count; - cpu_index++) + for (thread_index = vum->input_cpu_first_index; + thread_index < vum->input_cpu_first_index + vum->input_cpu_count; + thread_index++) { vlib_node_state_t state = VLIB_NODE_STATE_POLLING; - vhc = &vum->cpus[cpu_index]; - vm = vlib_mains ? vlib_mains[cpu_index] : &vlib_global_main; + vhc = &vum->cpus[thread_index]; + vm = vlib_mains ? vlib_mains[thread_index] : &vlib_global_main; switch (vhc->operation_mode) { case VHOST_USER_INTERRUPT_MODE: @@ -532,7 +533,7 @@ vhost_user_set_interrupt_pending (vhost_user_intf_t * vui, u32 ifq) { vhost_user_main_t *vum = &vhost_user_main; vhost_cpu_t *vhc; - u32 cpu_index; + u32 thread_index; vhost_iface_and_queue_t *vhiq; vlib_main_t *vm; u32 ifq2; @@ -553,8 +554,8 @@ vhost_user_set_interrupt_pending (vhost_user_intf_t * vui, u32 ifq) if ((vhiq->vhost_iface_index == (ifq >> 8)) && (VHOST_VRING_IDX_TX (vhiq->qid) == (ifq & 0xff))) { - cpu_index = vhc - vum->cpus; - vm = vlib_mains ? vlib_mains[cpu_index] : &vlib_global_main; + thread_index = vhc - vum->cpus; + vm = vlib_mains ? vlib_mains[thread_index] : &vlib_global_main; /* * Convert RX virtqueue number in the lower byte to vring * queue index for the input node process. Top bytes contain @@ -1592,7 +1593,7 @@ vhost_user_if_input (vlib_main_t * vm, u32 n_trace = vlib_get_trace_count (vm, node); u16 qsz_mask; u32 map_hint = 0; - u16 cpu_index = os_get_cpu_number (); + u16 thread_index = vlib_get_thread_index (); u16 copy_len = 0; { @@ -1651,32 +1652,32 @@ vhost_user_if_input (vlib_main_t * vm, * in the loop and come back later. This is not an issue as for big packet, * processing cost really comes from the memory copy. */ - if (PREDICT_FALSE (vum->cpus[cpu_index].rx_buffers_len < n_left + 1)) + if (PREDICT_FALSE (vum->cpus[thread_index].rx_buffers_len < n_left + 1)) { - u32 curr_len = vum->cpus[cpu_index].rx_buffers_len; - vum->cpus[cpu_index].rx_buffers_len += + u32 curr_len = vum->cpus[thread_index].rx_buffers_len; + vum->cpus[thread_index].rx_buffers_len += vlib_buffer_alloc_from_free_list (vm, - vum->cpus[cpu_index].rx_buffers + + vum->cpus[thread_index].rx_buffers + curr_len, VHOST_USER_RX_BUFFERS_N - curr_len, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); if (PREDICT_FALSE - (vum->cpus[cpu_index].rx_buffers_len < + (vum->cpus[thread_index].rx_buffers_len < VHOST_USER_RX_BUFFER_STARVATION)) { /* In case of buffer starvation, discard some packets from the queue * and log the event. * We keep doing best effort for the remaining packets. */ - u32 flush = (n_left + 1 > vum->cpus[cpu_index].rx_buffers_len) ? - n_left + 1 - vum->cpus[cpu_index].rx_buffers_len : 1; + u32 flush = (n_left + 1 > vum->cpus[thread_index].rx_buffers_len) ? + n_left + 1 - vum->cpus[thread_index].rx_buffers_len : 1; flush = vhost_user_rx_discard_packet (vm, vui, txvq, flush); n_left -= flush; vlib_increment_simple_counter (vnet_main. interface_main.sw_if_counters + VNET_INTERFACE_COUNTER_DROP, - os_get_cpu_number (), + vlib_get_thread_index (), vui->sw_if_index, flush); vlib_error_count (vm, vhost_user_input_node.index, @@ -1696,7 +1697,7 @@ vhost_user_if_input (vlib_main_t * vm, u32 desc_data_offset; vring_desc_t *desc_table = txvq->desc; - if (PREDICT_FALSE (vum->cpus[cpu_index].rx_buffers_len <= 1)) + if (PREDICT_FALSE (vum->cpus[thread_index].rx_buffers_len <= 1)) { /* Not enough rx_buffers * Note: We yeld on 1 so we don't need to do an additional @@ -1707,17 +1708,18 @@ vhost_user_if_input (vlib_main_t * vm, } desc_current = txvq->avail->ring[txvq->last_avail_idx & qsz_mask]; - vum->cpus[cpu_index].rx_buffers_len--; - bi_current = (vum->cpus[cpu_index].rx_buffers) - [vum->cpus[cpu_index].rx_buffers_len]; + vum->cpus[thread_index].rx_buffers_len--; + bi_current = (vum->cpus[thread_index].rx_buffers) + [vum->cpus[thread_index].rx_buffers_len]; b_head = b_current = vlib_get_buffer (vm, bi_current); to_next[0] = bi_current; //We do that now so we can forget about bi_current to_next++; n_left_to_next--; vlib_prefetch_buffer_with_index (vm, - (vum->cpus[cpu_index].rx_buffers) - [vum->cpus[cpu_index]. + (vum-> + cpus[thread_index].rx_buffers) + [vum->cpus[thread_index]. rx_buffers_len - 1], LOAD); /* Just preset the used descriptor id and length for later */ @@ -1791,7 +1793,7 @@ vhost_user_if_input (vlib_main_t * vm, (b_current->current_length == VLIB_BUFFER_DATA_SIZE)) { if (PREDICT_FALSE - (vum->cpus[cpu_index].rx_buffers_len == 0)) + (vum->cpus[thread_index].rx_buffers_len == 0)) { /* Cancel speculation */ to_next--; @@ -1805,17 +1807,18 @@ vhost_user_if_input (vlib_main_t * vm, * but valid. */ vhost_user_input_rewind_buffers (vm, - &vum->cpus[cpu_index], + &vum->cpus + [thread_index], b_head); n_left = 0; goto stop; } /* Get next output */ - vum->cpus[cpu_index].rx_buffers_len--; + vum->cpus[thread_index].rx_buffers_len--; u32 bi_next = - (vum->cpus[cpu_index].rx_buffers)[vum->cpus - [cpu_index].rx_buffers_len]; + (vum->cpus[thread_index].rx_buffers)[vum->cpus + [thread_index].rx_buffers_len]; b_current->next_buffer = bi_next; b_current->flags |= VLIB_BUFFER_NEXT_PRESENT; bi_current = bi_next; @@ -1823,7 +1826,7 @@ vhost_user_if_input (vlib_main_t * vm, } /* Prepare a copy order executed later for the data */ - vhost_copy_t *cpy = &vum->cpus[cpu_index].copy[copy_len]; + vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len]; copy_len++; u32 desc_data_l = desc_table[desc_current].len - desc_data_offset; @@ -1880,7 +1883,7 @@ vhost_user_if_input (vlib_main_t * vm, if (PREDICT_FALSE (copy_len >= VHOST_USER_RX_COPY_THRESHOLD)) { if (PREDICT_FALSE - (vhost_user_input_copy (vui, vum->cpus[cpu_index].copy, + (vhost_user_input_copy (vui, vum->cpus[thread_index].copy, copy_len, &map_hint))) { clib_warning @@ -1905,7 +1908,7 @@ vhost_user_if_input (vlib_main_t * vm, /* Do the memory copies */ if (PREDICT_FALSE - (vhost_user_input_copy (vui, vum->cpus[cpu_index].copy, + (vhost_user_input_copy (vui, vum->cpus[thread_index].copy, copy_len, &map_hint))) { clib_warning ("Memory mapping error on interface hw_if_index=%d " @@ -1933,9 +1936,9 @@ vhost_user_if_input (vlib_main_t * vm, vlib_increment_combined_counter (vnet_main.interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number (), vui->sw_if_index, n_rx_packets, n_rx_bytes); + vlib_get_thread_index (), vui->sw_if_index, n_rx_packets, n_rx_bytes); - vnet_device_increment_rx_packets (cpu_index, n_rx_packets); + vnet_device_increment_rx_packets (thread_index, n_rx_packets); return n_rx_packets; } @@ -1946,15 +1949,15 @@ vhost_user_input (vlib_main_t * vm, { vhost_user_main_t *vum = &vhost_user_main; uword n_rx_packets = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); vhost_iface_and_queue_t *vhiq; vhost_user_intf_t *vui; vhost_cpu_t *vhc; - vhc = &vum->cpus[cpu_index]; + vhc = &vum->cpus[thread_index]; if (PREDICT_TRUE (vhc->operation_mode == VHOST_USER_POLLING_MODE)) { - vec_foreach (vhiq, vum->cpus[cpu_index].rx_queues) + vec_foreach (vhiq, vum->cpus[thread_index].rx_queues) { vui = &vum->vhost_user_interfaces[vhiq->vhost_iface_index]; n_rx_packets += vhost_user_if_input (vm, vum, vui, vhiq->qid, node); @@ -2096,7 +2099,7 @@ vhost_user_tx (vlib_main_t * vm, vhost_user_vring_t *rxvq; u16 qsz_mask; u8 error; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 map_hint = 0; u8 retry = 8; u16 copy_len; @@ -2116,7 +2119,7 @@ vhost_user_tx (vlib_main_t * vm, qid = VHOST_VRING_IDX_RX (*vec_elt_at_index - (vui->per_cpu_tx_qid, os_get_cpu_number ())); + (vui->per_cpu_tx_qid, vlib_get_thread_index ())); rxvq = &vui->vrings[qid]; if (PREDICT_FALSE (vui->use_tx_spinlock)) vhost_user_vring_lock (vui, qid); @@ -2143,10 +2146,10 @@ retry: if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { - vum->cpus[cpu_index].current_trace = + vum->cpus[thread_index].current_trace = vlib_add_trace (vm, node, b0, - sizeof (*vum->cpus[cpu_index].current_trace)); - vhost_user_tx_trace (vum->cpus[cpu_index].current_trace, + sizeof (*vum->cpus[thread_index].current_trace)); + vhost_user_tx_trace (vum->cpus[thread_index].current_trace, vui, qid / 2, b0, rxvq); } @@ -2188,14 +2191,14 @@ retry: { // Get a header from the header array virtio_net_hdr_mrg_rxbuf_t *hdr = - &vum->cpus[cpu_index].tx_headers[tx_headers_len]; + &vum->cpus[thread_index].tx_headers[tx_headers_len]; tx_headers_len++; hdr->hdr.flags = 0; hdr->hdr.gso_type = 0; hdr->num_buffers = 1; //This is local, no need to check // Prepare a copy order executed later for the header - vhost_copy_t *cpy = &vum->cpus[cpu_index].copy[copy_len]; + vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len]; copy_len++; cpy->len = vui->virtio_net_hdr_sz; cpy->dst = buffer_map_addr; @@ -2220,7 +2223,7 @@ retry: else if (vui->virtio_net_hdr_sz == 12) //MRG is available { virtio_net_hdr_mrg_rxbuf_t *hdr = - &vum->cpus[cpu_index].tx_headers[tx_headers_len - 1]; + &vum->cpus[thread_index].tx_headers[tx_headers_len - 1]; //Move from available to used buffer rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id = @@ -2282,7 +2285,7 @@ retry: } { - vhost_copy_t *cpy = &vum->cpus[cpu_index].copy[copy_len]; + vhost_copy_t *cpy = &vum->cpus[thread_index].copy[copy_len]; copy_len++; cpy->len = bytes_left; cpy->len = (cpy->len > buffer_len) ? buffer_len : cpy->len; @@ -2325,8 +2328,8 @@ retry: if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { - vum->cpus[cpu_index].current_trace->hdr = - vum->cpus[cpu_index].tx_headers[tx_headers_len - 1]; + vum->cpus[thread_index].current_trace->hdr = + vum->cpus[thread_index].tx_headers[tx_headers_len - 1]; } n_left--; //At the end for error counting when 'goto done' is invoked @@ -2336,7 +2339,7 @@ retry: done: //Do the memory copies if (PREDICT_FALSE - (vhost_user_tx_copy (vui, vum->cpus[cpu_index].copy, + (vhost_user_tx_copy (vui, vum->cpus[thread_index].copy, copy_len, &map_hint))) { clib_warning ("Memory mapping error on interface hw_if_index=%d " @@ -2386,7 +2389,7 @@ done3: vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters + VNET_INTERFACE_COUNTER_DROP, - os_get_cpu_number (), vui->sw_if_index, n_left); + vlib_get_thread_index (), vui->sw_if_index, n_left); } vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors); @@ -2773,11 +2776,11 @@ vhost_user_send_interrupt_process (vlib_main_t * vm, case ~0: vec_foreach (vhc, vum->cpus) { - u32 cpu_index = vhc - vum->cpus; + u32 thread_index = vhc - vum->cpus; f64 next_timeout; next_timeout = timeout; - vec_foreach (vhiq, vum->cpus[cpu_index].rx_queues) + vec_foreach (vhiq, vum->cpus[thread_index].rx_queues) { vui = &vum->vhost_user_interfaces[vhiq->vhost_iface_index]; vhost_user_vring_t *rxvq = diff --git a/src/vnet/dpo/lookup_dpo.c b/src/vnet/dpo/lookup_dpo.c index e94e871c..97ad0a44 100644 --- a/src/vnet/dpo/lookup_dpo.c +++ b/src/vnet/dpo/lookup_dpo.c @@ -266,7 +266,7 @@ lookup_dpo_ip4_inline (vlib_main_t * vm, int table_from_interface) { u32 n_left_from, next_index, * from, * to_next; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters; from = vlib_frame_vector_args (from_frame); @@ -407,10 +407,10 @@ lookup_dpo_ip4_inline (vlib_main_t * vm, vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, b1)); if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) @@ -511,7 +511,7 @@ lookup_dpo_ip4_inline (vlib_main_t * vm, vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) @@ -606,7 +606,7 @@ lookup_dpo_ip6_inline (vlib_main_t * vm, { vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters; u32 n_left_from, next_index, * from, * to_next; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -749,10 +749,10 @@ lookup_dpo_ip6_inline (vlib_main_t * vm, vnet_buffer(b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, b1)); if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) @@ -853,7 +853,7 @@ lookup_dpo_ip6_inline (vlib_main_t * vm, vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) @@ -930,7 +930,7 @@ lookup_dpo_mpls_inline (vlib_main_t * vm, int table_from_interface) { u32 n_left_from, next_index, * from, * to_next; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters; from = vlib_frame_vector_args (from_frame); @@ -994,7 +994,7 @@ lookup_dpo_mpls_inline (vlib_main_t * vm, vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) diff --git a/src/vnet/dpo/replicate_dpo.c b/src/vnet/dpo/replicate_dpo.c index a9f334be..e25ceae9 100644 --- a/src/vnet/dpo/replicate_dpo.c +++ b/src/vnet/dpo/replicate_dpo.c @@ -627,7 +627,7 @@ replicate_inline (vlib_main_t * vm, vlib_combined_counter_main_t * cm = &replicate_main.repm_counters; replicate_main_t * rm = &replicate_main; u32 n_left_from, * from, * to_next, next_index; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -657,12 +657,12 @@ replicate_inline (vlib_main_t * vm, rep0 = replicate_get(repi0); vlib_increment_combined_counter( - cm, cpu_index, repi0, 1, + cm, thread_index, repi0, 1, vlib_buffer_length_in_chain(vm, b0)); - vec_validate (rm->clones[cpu_index], rep0->rep_n_buckets - 1); + vec_validate (rm->clones[thread_index], rep0->rep_n_buckets - 1); - num_cloned = vlib_buffer_clone (vm, bi0, rm->clones[cpu_index], rep0->rep_n_buckets, 128); + num_cloned = vlib_buffer_clone (vm, bi0, rm->clones[thread_index], rep0->rep_n_buckets, 128); if (num_cloned != rep0->rep_n_buckets) { @@ -673,7 +673,7 @@ replicate_inline (vlib_main_t * vm, for (bucket = 0; bucket < num_cloned; bucket++) { - ci0 = rm->clones[cpu_index][bucket]; + ci0 = rm->clones[thread_index][bucket]; c0 = vlib_get_buffer(vm, ci0); to_next[0] = ci0; @@ -700,7 +700,7 @@ replicate_inline (vlib_main_t * vm, vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); } } - vec_reset_length (rm->clones[cpu_index]); + vec_reset_length (rm->clones[thread_index]); } vlib_put_next_frame (vm, node, next_index, n_left_to_next); diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c index ee757505..c74a097e 100644 --- a/src/vnet/ethernet/arp.c +++ b/src/vnet/ethernet/arp.c @@ -1771,7 +1771,7 @@ set_ip4_over_ethernet_rpc_callback (vnet_arp_set_ip4_over_ethernet_rpc_args_t * a) { vnet_main_t *vm = vnet_get_main (); - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); if (a->flags & ETHERNET_ARP_ARGS_REMOVE) vnet_arp_unset_ip4_over_ethernet_internal (vm, a); diff --git a/src/vnet/ethernet/interface.c b/src/vnet/ethernet/interface.c index 9894e3c8..335e3f9f 100644 --- a/src/vnet/ethernet/interface.c +++ b/src/vnet/ethernet/interface.c @@ -362,7 +362,7 @@ simulated_ethernet_interface_tx (vlib_main_t * vm, u32 next_index = VNET_SIMULATED_ETHERNET_TX_NEXT_ETHERNET_INPUT; u32 i, next_node_index, bvi_flag, sw_if_index; u32 n_pkts = 0, n_bytes = 0; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; vnet_main_t *vnm = vnet_get_main (); vnet_interface_main_t *im = &vnm->interface_main; vlib_node_main_t *nm = &vm->node_main; @@ -420,8 +420,9 @@ simulated_ethernet_interface_tx (vlib_main_t * vm, /* increment TX interface stat */ vlib_increment_combined_counter (im->combined_sw_if_counters + - VNET_INTERFACE_COUNTER_TX, cpu_index, - sw_if_index, n_pkts, n_bytes); + VNET_INTERFACE_COUNTER_TX, + thread_index, sw_if_index, n_pkts, + n_bytes); } return n_left_from; diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index b699e381..f7787ed2 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -291,7 +291,7 @@ ethernet_input_inline (vlib_main_t * vm, vlib_node_runtime_t *error_node; u32 n_left_from, next_index, *from, *to_next; u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 cached_sw_if_index = ~0; u32 cached_is_l2 = 0; /* shut up gcc */ vnet_hw_interface_t *hi = NULL; /* used for main interface only */ @@ -510,7 +510,7 @@ ethernet_input_inline (vlib_main_t * vm, interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, new_sw_if_index0, 1, len0); if (new_sw_if_index1 != old_sw_if_index1 @@ -519,7 +519,7 @@ ethernet_input_inline (vlib_main_t * vm, interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, new_sw_if_index1, 1, len1); @@ -530,7 +530,7 @@ ethernet_input_inline (vlib_main_t * vm, vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = stats_n_bytes = 0; @@ -696,13 +696,13 @@ ethernet_input_inline (vlib_main_t * vm, vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, new_sw_if_index0, 1, len0); + thread_index, new_sw_if_index0, 1, len0); if (stats_n_packets > 0) { vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = stats_n_bytes = 0; } @@ -734,7 +734,7 @@ ethernet_input_inline (vlib_main_t * vm, vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } diff --git a/src/vnet/gre/node.c b/src/vnet/gre/node.c index 2683586e..acf15f24 100644 --- a/src/vnet/gre/node.c +++ b/src/vnet/gre/node.c @@ -75,7 +75,7 @@ gre_input (vlib_main_t * vm, u64 cached_tunnel_key6[4]; u32 cached_tunnel_sw_if_index = 0, tunnel_sw_if_index = 0; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); u32 len; vnet_interface_main_t *im = &gm->vnet_main->interface_main; @@ -257,7 +257,7 @@ gre_input (vlib_main_t * vm, len = vlib_buffer_length_in_chain (vm, b0); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, tunnel_sw_if_index, 1 /* packets */, len /* bytes */); @@ -324,7 +324,7 @@ drop0: len = vlib_buffer_length_in_chain (vm, b1); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, tunnel_sw_if_index, 1 /* packets */, len /* bytes */); @@ -502,7 +502,7 @@ drop1: len = vlib_buffer_length_in_chain (vm, b0); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, + thread_index, tunnel_sw_if_index, 1 /* packets */, len /* bytes */); diff --git a/src/vnet/interface.h b/src/vnet/interface.h index a1ea2d61..08f08b10 100644 --- a/src/vnet/interface.h +++ b/src/vnet/interface.h @@ -468,7 +468,7 @@ typedef struct vnet_hw_interface_t u32 input_node_index; /* input node cpu index by queue */ - u32 *input_node_cpu_index_by_queue; + u32 *input_node_thread_index_by_queue; } vnet_hw_interface_t; diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c index 03f2cdca..663dc309 100644 --- a/src/vnet/interface_output.c +++ b/src/vnet/interface_output.c @@ -196,7 +196,7 @@ slow_path (vlib_main_t * vm, */ static_always_inline void incr_output_stats (vnet_main_t * vnm, - u32 cpu_index, + u32 thread_index, u32 length, u32 sw_if_index, u32 * last_sw_if_index, u32 * n_packets, u32 * n_bytes) @@ -216,7 +216,7 @@ incr_output_stats (vnet_main_t * vnm, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, + thread_index, *last_sw_if_index, *n_packets, *n_bytes); } @@ -240,7 +240,7 @@ vnet_interface_output_node_flatten (vlib_main_t * vm, u32 n_left_to_tx, *from, *from_end, *to_tx; u32 n_bytes, n_buffers, n_packets; u32 last_sw_if_index; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; n_buffers = frame->n_vectors; @@ -266,7 +266,7 @@ vnet_interface_output_node_flatten (vlib_main_t * vm, cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, VNET_INTERFACE_COUNTER_TX_ERROR); - vlib_increment_simple_counter (cm, cpu_index, + vlib_increment_simple_counter (cm, thread_index, rt->sw_if_index, n_buffers); return vlib_error_drop_buffers (vm, node, from, /* buffer stride */ 1, @@ -341,18 +341,18 @@ vnet_interface_output_node_flatten (vlib_main_t * vm, from += 1; to_tx += n_buffers; n_left_to_tx -= n_buffers; - incr_output_stats (vnm, cpu_index, n_slow_bytes, + incr_output_stats (vnm, thread_index, n_slow_bytes, vnet_buffer (b)->sw_if_index[VLIB_TX], &last_sw_if_index, &n_packets, &n_bytes); } } else { - incr_output_stats (vnm, cpu_index, + incr_output_stats (vnm, thread_index, vlib_buffer_length_in_chain (vm, b0), vnet_buffer (b0)->sw_if_index[VLIB_TX], &last_sw_if_index, &n_packets, &n_bytes); - incr_output_stats (vnm, cpu_index, + incr_output_stats (vnm, thread_index, vlib_buffer_length_in_chain (vm, b0), vnet_buffer (b1)->sw_if_index[VLIB_TX], &last_sw_if_index, &n_packets, &n_bytes); @@ -396,7 +396,7 @@ vnet_interface_output_node_flatten (vlib_main_t * vm, to_tx += n_buffers; n_left_to_tx -= n_buffers; } - incr_output_stats (vnm, cpu_index, + incr_output_stats (vnm, thread_index, vlib_buffer_length_in_chain (vm, b0), vnet_buffer (b0)->sw_if_index[VLIB_TX], &last_sw_if_index, &n_packets, &n_bytes); @@ -408,7 +408,7 @@ vnet_interface_output_node_flatten (vlib_main_t * vm, } /* Final update of interface stats. */ - incr_output_stats (vnm, cpu_index, 0, ~0, /* ~0 will flush stats */ + incr_output_stats (vnm, thread_index, 0, ~0, /* ~0 will flush stats */ &last_sw_if_index, &n_packets, &n_bytes); return n_buffers; @@ -428,7 +428,7 @@ vnet_interface_output_node (vlib_main_t * vm, u32 n_left_to_tx, *from, *from_end, *to_tx; u32 n_bytes, n_buffers, n_packets; u32 n_bytes_b0, n_bytes_b1, n_bytes_b2, n_bytes_b3; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; vnet_interface_main_t *im = &vnm->interface_main; u32 next_index = VNET_INTERFACE_OUTPUT_NEXT_TX; u32 current_config_index = ~0; @@ -458,7 +458,7 @@ vnet_interface_output_node (vlib_main_t * vm, cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, VNET_INTERFACE_COUNTER_TX_ERROR); - vlib_increment_simple_counter (cm, cpu_index, + vlib_increment_simple_counter (cm, thread_index, rt->sw_if_index, n_buffers); return vlib_error_drop_buffers (vm, node, from, @@ -558,7 +558,7 @@ vnet_interface_output_node (vlib_main_t * vm, { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, tx_swif0, 1, + thread_index, tx_swif0, 1, n_bytes_b0); } @@ -567,7 +567,7 @@ vnet_interface_output_node (vlib_main_t * vm, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, tx_swif1, 1, + thread_index, tx_swif1, 1, n_bytes_b1); } @@ -576,7 +576,7 @@ vnet_interface_output_node (vlib_main_t * vm, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, tx_swif2, 1, + thread_index, tx_swif2, 1, n_bytes_b2); } if (PREDICT_FALSE (tx_swif3 != rt->sw_if_index)) @@ -584,7 +584,7 @@ vnet_interface_output_node (vlib_main_t * vm, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, tx_swif3, 1, + thread_index, tx_swif3, 1, n_bytes_b3); } } @@ -623,7 +623,7 @@ vnet_interface_output_node (vlib_main_t * vm, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, tx_swif0, 1, + thread_index, tx_swif0, 1, n_bytes_b0); } } @@ -634,7 +634,7 @@ vnet_interface_output_node (vlib_main_t * vm, /* Update main interface stats. */ vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, + thread_index, rt->sw_if_index, n_packets, n_bytes); return n_buffers; } @@ -893,7 +893,7 @@ process_drop_punt (vlib_main_t * vm, u32 current_sw_if_index, n_errors_current_sw_if_index; u64 current_counter; vlib_simple_counter_main_t *cm; - u32 cpu_index = vm->cpu_index; + u32 thread_index = vm->thread_index; static vlib_error_t memory[VNET_ERROR_N_DISPOSITION]; static char memory_init[VNET_ERROR_N_DISPOSITION]; @@ -965,19 +965,19 @@ process_drop_punt (vlib_main_t * vm, current_counter -= 2; n_errors_current_sw_if_index -= 2; - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index1, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index1, 1); /* Increment super-interface drop/punt counters for sub-interfaces. */ sw_if0 = vnet_get_sw_interface (vnm, sw_if_index0); vlib_increment_simple_counter - (cm, cpu_index, sw_if0->sup_sw_if_index, + (cm, thread_index, sw_if0->sup_sw_if_index, sw_if0->sup_sw_if_index != sw_if_index0); sw_if1 = vnet_get_sw_interface (vnm, sw_if_index1); vlib_increment_simple_counter - (cm, cpu_index, sw_if1->sup_sw_if_index, + (cm, thread_index, sw_if1->sup_sw_if_index, sw_if1->sup_sw_if_index != sw_if_index1); em->counters[current_counter_index] = current_counter; @@ -1013,11 +1013,12 @@ process_drop_punt (vlib_main_t * vm, sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; /* Increment drop/punt counters. */ - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); /* Increment super-interface drop/punt counters for sub-interfaces. */ sw_if0 = vnet_get_sw_interface (vnm, sw_if_index0); - vlib_increment_simple_counter (cm, cpu_index, sw_if0->sup_sw_if_index, + vlib_increment_simple_counter (cm, thread_index, + sw_if0->sup_sw_if_index, sw_if0->sup_sw_if_index != sw_if_index0); if (PREDICT_FALSE (e0 != current_error)) @@ -1041,12 +1042,12 @@ process_drop_punt (vlib_main_t * vm, { vnet_sw_interface_t *si; - vlib_increment_simple_counter (cm, cpu_index, current_sw_if_index, + vlib_increment_simple_counter (cm, thread_index, current_sw_if_index, n_errors_current_sw_if_index); si = vnet_get_sw_interface (vnm, current_sw_if_index); if (si->sup_sw_if_index != current_sw_if_index) - vlib_increment_simple_counter (cm, cpu_index, si->sup_sw_if_index, + vlib_increment_simple_counter (cm, thread_index, si->sup_sw_if_index, n_errors_current_sw_if_index); } diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c index ee1703e7..fdfe7f63 100644 --- a/src/vnet/ip/ip4_forward.c +++ b/src/vnet/ip/ip4_forward.c @@ -75,7 +75,7 @@ ip4_lookup_inline (vlib_main_t * vm, vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters; u32 n_left_from, n_left_to_next, *from, *to_next; ip_lookup_next_t next; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -292,19 +292,19 @@ ip4_lookup_inline (vlib_main_t * vm, vnet_buffer (p3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lb_index0, 1, + (cm, thread_index, lb_index0, 1, vlib_buffer_length_in_chain (vm, p0) + sizeof (ethernet_header_t)); vlib_increment_combined_counter - (cm, cpu_index, lb_index1, 1, + (cm, thread_index, lb_index1, 1, vlib_buffer_length_in_chain (vm, p1) + sizeof (ethernet_header_t)); vlib_increment_combined_counter - (cm, cpu_index, lb_index2, 1, + (cm, thread_index, lb_index2, 1, vlib_buffer_length_in_chain (vm, p2) + sizeof (ethernet_header_t)); vlib_increment_combined_counter - (cm, cpu_index, lb_index3, 1, + (cm, thread_index, lb_index3, 1, vlib_buffer_length_in_chain (vm, p3) + sizeof (ethernet_header_t)); @@ -392,7 +392,7 @@ ip4_lookup_inline (vlib_main_t * vm, vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); from += 1; to_next += 1; @@ -479,7 +479,7 @@ ip4_load_balance (vlib_main_t * vm, vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters; u32 n_left_from, n_left_to_next, *from, *to_next; ip_lookup_next_t next; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -584,9 +584,9 @@ ip4_load_balance (vlib_main_t * vm, vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); vlib_validate_buffer_enqueue_x2 (vm, node, next, to_next, n_left_to_next, @@ -639,7 +639,7 @@ ip4_load_balance (vlib_main_t * vm, vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_validate_buffer_enqueue_x1 (vm, node, next, to_next, n_left_to_next, @@ -2330,7 +2330,7 @@ ip4_rewrite_inline (vlib_main_t * vm, n_left_from = frame->n_vectors; next_index = node->cached_next_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -2379,9 +2379,9 @@ ip4_rewrite_inline (vlib_main_t * vm, if (do_counters) { vlib_prefetch_combined_counter (&adjacency_counters, - cpu_index, adj_index0); + thread_index, adj_index0); vlib_prefetch_combined_counter (&adjacency_counters, - cpu_index, adj_index1); + thread_index, adj_index1); } ip0 = vlib_buffer_get_current (p0); @@ -2527,13 +2527,13 @@ ip4_rewrite_inline (vlib_main_t * vm, { vlib_increment_combined_counter (&adjacency_counters, - cpu_index, + thread_index, adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); vlib_increment_combined_counter (&adjacency_counters, - cpu_index, + thread_index, adj_index1, 1, vlib_buffer_length_in_chain (vm, p1) + rw_len1); } @@ -2618,7 +2618,7 @@ ip4_rewrite_inline (vlib_main_t * vm, if (do_counters) vlib_prefetch_combined_counter (&adjacency_counters, - cpu_index, adj_index0); + thread_index, adj_index0); /* Guess we are only writing on simple Ethernet header. */ vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t)); @@ -2637,7 +2637,7 @@ ip4_rewrite_inline (vlib_main_t * vm, if (do_counters) vlib_increment_combined_counter (&adjacency_counters, - cpu_index, adj_index0, 1, + thread_index, adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); /* Check MTU of outgoing interface. */ diff --git a/src/vnet/ip/ip4_input.c b/src/vnet/ip/ip4_input.c index ba200a9f..3b08f4b0 100644 --- a/src/vnet/ip/ip4_input.c +++ b/src/vnet/ip/ip4_input.c @@ -85,7 +85,7 @@ ip4_input_inline (vlib_main_t * vm, vlib_node_runtime_t *error_node = vlib_node_get_runtime (vm, ip4_input_node.index); vlib_simple_counter_main_t *cm; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -178,8 +178,8 @@ ip4_input_inline (vlib_main_t * vm, vnet_feature_arc_start (arc0, sw_if_index0, &next0, p0); vnet_feature_arc_start (arc1, sw_if_index1, &next1, p1); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index1, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index1, 1); /* Punt packets with options or wrong version. */ if (PREDICT_FALSE (ip0->ip_version_and_header_length != 0x45)) @@ -299,7 +299,7 @@ ip4_input_inline (vlib_main_t * vm, vnet_buffer (p0)->ip.adj_index[VLIB_RX] = ~0; vnet_feature_arc_start (arc0, sw_if_index0, &next0, p0); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); /* Punt packets with options or wrong version. */ if (PREDICT_FALSE (ip0->ip_version_and_header_length != 0x45)) diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index c120f12c..c2fc4f87 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -74,7 +74,7 @@ ip6_lookup_inline (vlib_main_t * vm, vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters; u32 n_left_from, n_left_to_next, *from, *to_next; ip_lookup_next_t next; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -185,9 +185,9 @@ ip6_lookup_inline (vlib_main_t * vm, vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); from += 2; to_next += 2; @@ -291,7 +291,7 @@ ip6_lookup_inline (vlib_main_t * vm, vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); from += 1; to_next += 1; @@ -703,7 +703,7 @@ ip6_load_balance (vlib_main_t * vm, vlib_combined_counter_main_t *cm = &load_balance_main.lbm_via_counters; u32 n_left_from, n_left_to_next, *from, *to_next; ip_lookup_next_t next; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); ip6_main_t *im = &ip6_main; from = vlib_frame_vector_args (frame); @@ -824,9 +824,9 @@ ip6_load_balance (vlib_main_t * vm, vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); vlib_validate_buffer_enqueue_x2 (vm, node, next, to_next, n_left_to_next, @@ -886,7 +886,7 @@ ip6_load_balance (vlib_main_t * vm, } vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_validate_buffer_enqueue_x1 (vm, node, next, to_next, n_left_to_next, @@ -1897,7 +1897,7 @@ ip6_rewrite_inline (vlib_main_t * vm, n_left_from = frame->n_vectors; next_index = node->cached_next_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -2019,11 +2019,11 @@ ip6_rewrite_inline (vlib_main_t * vm, { vlib_increment_combined_counter (&adjacency_counters, - cpu_index, adj_index0, 1, + thread_index, adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); vlib_increment_combined_counter (&adjacency_counters, - cpu_index, adj_index1, 1, + thread_index, adj_index1, 1, vlib_buffer_length_in_chain (vm, p1) + rw_len1); } @@ -2156,7 +2156,7 @@ ip6_rewrite_inline (vlib_main_t * vm, { vlib_increment_combined_counter (&adjacency_counters, - cpu_index, adj_index0, 1, + thread_index, adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); } diff --git a/src/vnet/ip/ip6_input.c b/src/vnet/ip/ip6_input.c index 20306088..ffdc4727 100644 --- a/src/vnet/ip/ip6_input.c +++ b/src/vnet/ip/ip6_input.c @@ -82,7 +82,7 @@ ip6_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) vlib_node_runtime_t *error_node = vlib_node_get_runtime (vm, ip6_input_node.index); vlib_simple_counter_main_t *cm; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -171,8 +171,8 @@ ip6_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) vnet_feature_arc_start (arc0, sw_if_index0, &next0, p0); vnet_feature_arc_start (arc1, sw_if_index1, &next1, p1); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index1, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index1, 1); error0 = error1 = IP6_ERROR_NONE; @@ -270,7 +270,7 @@ ip6_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) vnet_buffer (p0)->ip.adj_index[VLIB_RX] = ~0; vnet_feature_arc_start (arc0, sw_if_index0, &next0, p0); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); error0 = IP6_ERROR_NONE; /* Version != 6? Drop it. */ diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c index 5d1fb6f8..2af546df 100644 --- a/src/vnet/ip/ip6_neighbor.c +++ b/src/vnet/ip/ip6_neighbor.c @@ -581,7 +581,7 @@ vnet_set_ip6_ethernet_neighbor (vlib_main_t * vm, u32 next_index; pending_resolution_t *pr, *mc; - if (os_get_cpu_number ()) + if (vlib_get_thread_index ()) { set_unset_ip6_neighbor_rpc (vm, sw_if_index, a, link_layer_address, 1 /* set new neighbor */ , is_static, @@ -722,7 +722,7 @@ vnet_unset_ip6_ethernet_neighbor (vlib_main_t * vm, uword *p; int rv = 0; - if (os_get_cpu_number ()) + if (vlib_get_thread_index ()) { set_unset_ip6_neighbor_rpc (vm, sw_if_index, a, link_layer_address, 0 /* unset */ , 0, 0); diff --git a/src/vnet/ipsec/esp.h b/src/vnet/ipsec/esp.h index 50cac806..799003b9 100644 --- a/src/vnet/ipsec/esp.h +++ b/src/vnet/ipsec/esp.h @@ -282,8 +282,8 @@ hmac_calc (ipsec_integ_alg_t alg, u8 * data, int data_len, u8 * signature, u8 use_esn, u32 seq_hi) { esp_main_t *em = &esp_main; - u32 cpu_index = os_get_cpu_number (); - HMAC_CTX *ctx = &(em->per_thread_data[cpu_index].hmac_ctx); + u32 thread_index = vlib_get_thread_index (); + HMAC_CTX *ctx = &(em->per_thread_data[thread_index].hmac_ctx); const EVP_MD *md = NULL; unsigned int len; @@ -292,10 +292,10 @@ hmac_calc (ipsec_integ_alg_t alg, if (PREDICT_FALSE (em->esp_integ_algs[alg].md == 0)) return 0; - if (PREDICT_FALSE (alg != em->per_thread_data[cpu_index].last_integ_alg)) + if (PREDICT_FALSE (alg != em->per_thread_data[thread_index].last_integ_alg)) { md = em->esp_integ_algs[alg].md; - em->per_thread_data[cpu_index].last_integ_alg = alg; + em->per_thread_data[thread_index].last_integ_alg = alg; } HMAC_Init (ctx, key, key_len, md); diff --git a/src/vnet/ipsec/esp_decrypt.c b/src/vnet/ipsec/esp_decrypt.c index 7289b260..925d2b45 100644 --- a/src/vnet/ipsec/esp_decrypt.c +++ b/src/vnet/ipsec/esp_decrypt.c @@ -85,8 +85,8 @@ esp_decrypt_aes_cbc (ipsec_crypto_alg_t alg, u8 * in, u8 * out, size_t in_len, u8 * key, u8 * iv) { esp_main_t *em = &esp_main; - u32 cpu_index = os_get_cpu_number (); - EVP_CIPHER_CTX *ctx = &(em->per_thread_data[cpu_index].decrypt_ctx); + u32 thread_index = vlib_get_thread_index (); + EVP_CIPHER_CTX *ctx = &(em->per_thread_data[thread_index].decrypt_ctx); const EVP_CIPHER *cipher = NULL; int out_len; @@ -95,10 +95,11 @@ esp_decrypt_aes_cbc (ipsec_crypto_alg_t alg, if (PREDICT_FALSE (em->esp_crypto_algs[alg].type == 0)) return; - if (PREDICT_FALSE (alg != em->per_thread_data[cpu_index].last_decrypt_alg)) + if (PREDICT_FALSE + (alg != em->per_thread_data[thread_index].last_decrypt_alg)) { cipher = em->esp_crypto_algs[alg].type; - em->per_thread_data[cpu_index].last_decrypt_alg = alg; + em->per_thread_data[thread_index].last_decrypt_alg = alg; } EVP_DecryptInit_ex (ctx, cipher, NULL, key, iv); @@ -117,11 +118,11 @@ esp_decrypt_node_fn (vlib_main_t * vm, u32 *recycle = 0; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); ipsec_alloc_empty_buffers (vm, im); - u32 *empty_buffers = im->empty_buffers[cpu_index]; + u32 *empty_buffers = im->empty_buffers[thread_index]; if (PREDICT_FALSE (vec_len (empty_buffers) < n_left_from)) { diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c index 44ae2297..b2bc4e0b 100644 --- a/src/vnet/ipsec/esp_encrypt.c +++ b/src/vnet/ipsec/esp_encrypt.c @@ -88,8 +88,8 @@ esp_encrypt_aes_cbc (ipsec_crypto_alg_t alg, u8 * in, u8 * out, size_t in_len, u8 * key, u8 * iv) { esp_main_t *em = &esp_main; - u32 cpu_index = os_get_cpu_number (); - EVP_CIPHER_CTX *ctx = &(em->per_thread_data[cpu_index].encrypt_ctx); + u32 thread_index = vlib_get_thread_index (); + EVP_CIPHER_CTX *ctx = &(em->per_thread_data[thread_index].encrypt_ctx); const EVP_CIPHER *cipher = NULL; int out_len; @@ -98,10 +98,11 @@ esp_encrypt_aes_cbc (ipsec_crypto_alg_t alg, if (PREDICT_FALSE (em->esp_crypto_algs[alg].type == IPSEC_CRYPTO_ALG_NONE)) return; - if (PREDICT_FALSE (alg != em->per_thread_data[cpu_index].last_encrypt_alg)) + if (PREDICT_FALSE + (alg != em->per_thread_data[thread_index].last_encrypt_alg)) { cipher = em->esp_crypto_algs[alg].type; - em->per_thread_data[cpu_index].last_encrypt_alg = alg; + em->per_thread_data[thread_index].last_encrypt_alg = alg; } EVP_EncryptInit_ex (ctx, cipher, NULL, key, iv); @@ -119,11 +120,11 @@ esp_encrypt_node_fn (vlib_main_t * vm, n_left_from = from_frame->n_vectors; ipsec_main_t *im = &ipsec_main; u32 *recycle = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); ipsec_alloc_empty_buffers (vm, im); - u32 *empty_buffers = im->empty_buffers[cpu_index]; + u32 *empty_buffers = im->empty_buffers[thread_index]; if (PREDICT_FALSE (vec_len (empty_buffers) < n_left_from)) { diff --git a/src/vnet/ipsec/ikev2.c b/src/vnet/ipsec/ikev2.c index 2c1074d8..3f9978a7 100644 --- a/src/vnet/ipsec/ikev2.c +++ b/src/vnet/ipsec/ikev2.c @@ -303,16 +303,16 @@ static void ikev2_delete_sa (ikev2_sa_t * sa) { ikev2_main_t *km = &ikev2_main; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); uword *p; ikev2_sa_free_all_vec (sa); - p = hash_get (km->per_thread_data[cpu_index].sa_by_rspi, sa->rspi); + p = hash_get (km->per_thread_data[thread_index].sa_by_rspi, sa->rspi); if (p) { - hash_unset (km->per_thread_data[cpu_index].sa_by_rspi, sa->rspi); - pool_put (km->per_thread_data[cpu_index].sas, sa); + hash_unset (km->per_thread_data[thread_index].sa_by_rspi, sa->rspi); + pool_put (km->per_thread_data[thread_index].sas, sa); } } @@ -776,29 +776,31 @@ ikev2_initial_contact_cleanup (ikev2_sa_t * sa) ikev2_sa_t *tmp; u32 i, *delete = 0; ikev2_child_sa_t *c; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); if (!sa->initial_contact) return; /* find old IKE SAs with the same authenticated identity */ /* *INDENT-OFF* */ - pool_foreach (tmp, km->per_thread_data[cpu_index].sas, ({ + pool_foreach (tmp, km->per_thread_data[thread_index].sas, ({ if (tmp->i_id.type != sa->i_id.type || vec_len(tmp->i_id.data) != vec_len(sa->i_id.data) || memcmp(sa->i_id.data, tmp->i_id.data, vec_len(sa->i_id.data))) continue; if (sa->rspi != tmp->rspi) - vec_add1(delete, tmp - km->per_thread_data[cpu_index].sas); + vec_add1(delete, tmp - km->per_thread_data[thread_index].sas); })); /* *INDENT-ON* */ for (i = 0; i < vec_len (delete); i++) { - tmp = pool_elt_at_index (km->per_thread_data[cpu_index].sas, delete[i]); - vec_foreach (c, tmp->childs) - ikev2_delete_tunnel_interface (km->vnet_main, tmp, c); + tmp = + pool_elt_at_index (km->per_thread_data[thread_index].sas, delete[i]); + vec_foreach (c, + tmp->childs) ikev2_delete_tunnel_interface (km->vnet_main, + tmp, c); ikev2_delete_sa (tmp); } @@ -1922,10 +1924,10 @@ ikev2_retransmit_sa_init (ike_header_t * ike, { ikev2_main_t *km = &ikev2_main; ikev2_sa_t *sa; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); /* *INDENT-OFF* */ - pool_foreach (sa, km->per_thread_data[cpu_index].sas, ({ + pool_foreach (sa, km->per_thread_data[thread_index].sas, ({ if (sa->ispi == clib_net_to_host_u64(ike->ispi) && sa->iaddr.as_u32 == iaddr.as_u32 && sa->raddr.as_u32 == raddr.as_u32) @@ -2036,7 +2038,7 @@ ikev2_node_fn (vlib_main_t * vm, u32 n_left_from, *from, *to_next; ikev2_next_t next_index; ikev2_main_t *km = &ikev2_main; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -2134,11 +2136,14 @@ ikev2_node_fn (vlib_main_t * vm, if (sa0->state == IKEV2_STATE_SA_INIT) { /* add SA to the pool */ - pool_get (km->per_thread_data[cpu_index].sas, sa0); + pool_get (km->per_thread_data[thread_index].sas, + sa0); clib_memcpy (sa0, &sa, sizeof (*sa0)); - hash_set (km->per_thread_data[cpu_index].sa_by_rspi, + hash_set (km-> + per_thread_data[thread_index].sa_by_rspi, sa0->rspi, - sa0 - km->per_thread_data[cpu_index].sas); + sa0 - + km->per_thread_data[thread_index].sas); } else { @@ -2169,11 +2174,11 @@ ikev2_node_fn (vlib_main_t * vm, if (sa0->state == IKEV2_STATE_SA_INIT) { /* add SA to the pool */ - pool_get (km->per_thread_data[cpu_index].sas, sa0); + pool_get (km->per_thread_data[thread_index].sas, sa0); clib_memcpy (sa0, &sa, sizeof (*sa0)); - hash_set (km->per_thread_data[cpu_index].sa_by_rspi, + hash_set (km->per_thread_data[thread_index].sa_by_rspi, sa0->rspi, - sa0 - km->per_thread_data[cpu_index].sas); + sa0 - km->per_thread_data[thread_index].sas); } else { @@ -2184,12 +2189,13 @@ ikev2_node_fn (vlib_main_t * vm, else if (ike0->exchange == IKEV2_EXCHANGE_IKE_AUTH) { uword *p; - p = hash_get (km->per_thread_data[cpu_index].sa_by_rspi, + p = hash_get (km->per_thread_data[thread_index].sa_by_rspi, clib_net_to_host_u64 (ike0->rspi)); if (p) { - sa0 = pool_elt_at_index (km->per_thread_data[cpu_index].sas, - p[0]); + sa0 = + pool_elt_at_index (km->per_thread_data[thread_index].sas, + p[0]); r = ikev2_retransmit_resp (sa0, ike0); if (r == 1) @@ -2240,12 +2246,13 @@ ikev2_node_fn (vlib_main_t * vm, else if (ike0->exchange == IKEV2_EXCHANGE_INFORMATIONAL) { uword *p; - p = hash_get (km->per_thread_data[cpu_index].sa_by_rspi, + p = hash_get (km->per_thread_data[thread_index].sa_by_rspi, clib_net_to_host_u64 (ike0->rspi)); if (p) { - sa0 = pool_elt_at_index (km->per_thread_data[cpu_index].sas, - p[0]); + sa0 = + pool_elt_at_index (km->per_thread_data[thread_index].sas, + p[0]); r = ikev2_retransmit_resp (sa0, ike0); if (r == 1) @@ -2305,12 +2312,13 @@ ikev2_node_fn (vlib_main_t * vm, else if (ike0->exchange == IKEV2_EXCHANGE_CREATE_CHILD_SA) { uword *p; - p = hash_get (km->per_thread_data[cpu_index].sa_by_rspi, + p = hash_get (km->per_thread_data[thread_index].sa_by_rspi, clib_net_to_host_u64 (ike0->rspi)); if (p) { - sa0 = pool_elt_at_index (km->per_thread_data[cpu_index].sas, - p[0]); + sa0 = + pool_elt_at_index (km->per_thread_data[thread_index].sas, + p[0]); r = ikev2_retransmit_resp (sa0, ike0); if (r == 1) diff --git a/src/vnet/ipsec/ipsec.h b/src/vnet/ipsec/ipsec.h index 58f0f145..c884e360 100644 --- a/src/vnet/ipsec/ipsec.h +++ b/src/vnet/ipsec/ipsec.h @@ -324,21 +324,21 @@ int ipsec_set_interface_key (vnet_main_t * vnm, u32 hw_if_index, always_inline void ipsec_alloc_empty_buffers (vlib_main_t * vm, ipsec_main_t * im) { - u32 cpu_index = os_get_cpu_number (); - uword l = vec_len (im->empty_buffers[cpu_index]); + u32 thread_index = vlib_get_thread_index (); + uword l = vec_len (im->empty_buffers[thread_index]); uword n_alloc = 0; if (PREDICT_FALSE (l < VLIB_FRAME_SIZE)) { - if (!im->empty_buffers[cpu_index]) + if (!im->empty_buffers[thread_index]) { - vec_alloc (im->empty_buffers[cpu_index], 2 * VLIB_FRAME_SIZE); + vec_alloc (im->empty_buffers[thread_index], 2 * VLIB_FRAME_SIZE); } - n_alloc = vlib_buffer_alloc (vm, im->empty_buffers[cpu_index] + l, + n_alloc = vlib_buffer_alloc (vm, im->empty_buffers[thread_index] + l, 2 * VLIB_FRAME_SIZE - l); - _vec_len (im->empty_buffers[cpu_index]) = l + n_alloc; + _vec_len (im->empty_buffers[thread_index]) = l + n_alloc; } } diff --git a/src/vnet/ipsec/ipsec_if.c b/src/vnet/ipsec/ipsec_if.c index dc882004..ed124894 100644 --- a/src/vnet/ipsec/ipsec_if.c +++ b/src/vnet/ipsec/ipsec_if.c @@ -99,7 +99,7 @@ static int ipsec_add_del_tunnel_if_rpc_callback (ipsec_add_del_tunnel_args_t * a) { vnet_main_t *vnm = vnet_get_main (); - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); return ipsec_add_del_tunnel_if_internal (vnm, a); } diff --git a/src/vnet/l2/l2_bvi.h b/src/vnet/l2/l2_bvi.h index dd1130a6..e21a1616 100644 --- a/src/vnet/l2/l2_bvi.h +++ b/src/vnet/l2/l2_bvi.h @@ -97,7 +97,7 @@ l2_to_bvi (vlib_main_t * vlib_main, vlib_increment_combined_counter (vnet_main->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - vlib_main->cpu_index, + vlib_main->thread_index, vnet_buffer (b0)->sw_if_index[VLIB_RX], 1, vlib_buffer_length_in_chain (vlib_main, b0)); return TO_BVI_ERR_OK; diff --git a/src/vnet/l2/l2_input.c b/src/vnet/l2/l2_input.c index 041ff38d..e5d6878a 100644 --- a/src/vnet/l2/l2_input.c +++ b/src/vnet/l2/l2_input.c @@ -117,7 +117,7 @@ typedef enum static_always_inline void classify_and_dispatch (vlib_main_t * vm, vlib_node_runtime_t * node, - u32 cpu_index, + u32 thread_index, l2input_main_t * msm, vlib_buffer_t * b0, u32 * next0) { /* @@ -237,7 +237,7 @@ l2input_node_inline (vlib_main_t * vm, u32 n_left_from, *from, *to_next; l2input_next_t next_index; l2input_main_t *msm = &l2input_main; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; /* number of packets to process */ @@ -350,10 +350,10 @@ l2input_node_inline (vlib_main_t * vm, vlib_node_increment_counter (vm, l2input_node.index, L2INPUT_ERROR_L2INPUT, 4); - classify_and_dispatch (vm, node, cpu_index, msm, b0, &next0); - classify_and_dispatch (vm, node, cpu_index, msm, b1, &next1); - classify_and_dispatch (vm, node, cpu_index, msm, b2, &next2); - classify_and_dispatch (vm, node, cpu_index, msm, b3, &next3); + classify_and_dispatch (vm, node, thread_index, msm, b0, &next0); + classify_and_dispatch (vm, node, thread_index, msm, b1, &next1); + classify_and_dispatch (vm, node, thread_index, msm, b2, &next2); + classify_and_dispatch (vm, node, thread_index, msm, b3, &next3); /* verify speculative enqueues, maybe switch current next frame */ /* if next0==next1==next_index then nothing special needs to be done */ @@ -393,7 +393,7 @@ l2input_node_inline (vlib_main_t * vm, vlib_node_increment_counter (vm, l2input_node.index, L2INPUT_ERROR_L2INPUT, 1); - classify_and_dispatch (vm, node, cpu_index, msm, b0, &next0); + classify_and_dispatch (vm, node, thread_index, msm, b0, &next0); /* verify speculative enqueue, maybe switch current next frame */ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, diff --git a/src/vnet/l2/l2_output.c b/src/vnet/l2/l2_output.c index 00f22571..e17b2a16 100644 --- a/src/vnet/l2/l2_output.c +++ b/src/vnet/l2/l2_output.c @@ -643,11 +643,11 @@ l2output_create_output_node_mapping (vlib_main_t * vlib_main, vnet_main_t * vnet hw0 = vnet_get_sup_hw_interface (vnet_main, sw_if_index); - uword cpu_number; + uword thread_index; - cpu_number = os_get_cpu_number (); + thread_index = vlib_get_thread_index (); - if (cpu_number) + if (thread_index) { u32 oldflags; diff --git a/src/vnet/l2tp/decap.c b/src/vnet/l2tp/decap.c index e8986935..46104129 100644 --- a/src/vnet/l2tp/decap.c +++ b/src/vnet/l2tp/decap.c @@ -149,7 +149,7 @@ last_stage (vlib_main_t * vm, vlib_node_runtime_t * node, u32 bi) /* per-mapping byte stats include the ethernet header */ vlib_increment_combined_counter (&lm->counter_main, - os_get_cpu_number (), + vlib_get_thread_index (), counter_index, 1 /* packet_increment */ , vlib_buffer_length_in_chain (vm, b) + sizeof (ethernet_header_t)); diff --git a/src/vnet/l2tp/encap.c b/src/vnet/l2tp/encap.c index ed7a9580..dcdfde4b 100644 --- a/src/vnet/l2tp/encap.c +++ b/src/vnet/l2tp/encap.c @@ -124,7 +124,7 @@ last_stage (vlib_main_t * vm, vlib_node_runtime_t * node, u32 bi) /* per-mapping byte stats include the ethernet header */ vlib_increment_combined_counter (&lm->counter_main, - os_get_cpu_number (), + vlib_get_thread_index (), counter_index, 1 /* packet_increment */ , vlib_buffer_length_in_chain (vm, b)); diff --git a/src/vnet/l2tp/l2tp.c b/src/vnet/l2tp/l2tp.c index cb94d7e7..3dedc447 100644 --- a/src/vnet/l2tp/l2tp.c +++ b/src/vnet/l2tp/l2tp.c @@ -157,7 +157,7 @@ test_counters_command_fn (vlib_main_t * vm, u32 session_index; u32 counter_index; u32 nincr = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); /* *INDENT-OFF* */ pool_foreach (session, lm->sessions, @@ -167,11 +167,11 @@ test_counters_command_fn (vlib_main_t * vm, session_index_to_counter_index (session_index, SESSION_COUNTER_USER_TO_NETWORK); vlib_increment_combined_counter (&lm->counter_main, - cpu_index, + thread_index, counter_index, 1/*pkt*/, 1111 /*bytes*/); vlib_increment_combined_counter (&lm->counter_main, - cpu_index, + thread_index, counter_index+1, 1/*pkt*/, 2222 /*bytes*/); nincr++; diff --git a/src/vnet/lisp-gpe/decap.c b/src/vnet/lisp-gpe/decap.c index d887a95f..68769710 100644 --- a/src/vnet/lisp-gpe/decap.c +++ b/src/vnet/lisp-gpe/decap.c @@ -103,7 +103,7 @@ next_index_to_iface (lisp_gpe_main_t * lgm, u32 next_index) } static_always_inline void -incr_decap_stats (vnet_main_t * vnm, u32 cpu_index, u32 length, +incr_decap_stats (vnet_main_t * vnm, u32 thread_index, u32 length, u32 sw_if_index, u32 * last_sw_if_index, u32 * n_packets, u32 * n_bytes) { @@ -122,7 +122,7 @@ incr_decap_stats (vnet_main_t * vnm, u32 cpu_index, u32 length, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, *last_sw_if_index, + thread_index, *last_sw_if_index, *n_packets, *n_bytes); } *last_sw_if_index = sw_if_index; @@ -150,11 +150,11 @@ static uword lisp_gpe_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame, u8 is_v4) { - u32 n_left_from, next_index, *from, *to_next, cpu_index; + u32 n_left_from, next_index, *from, *to_next, thread_index; u32 n_bytes = 0, n_packets = 0, last_sw_if_index = ~0, drops = 0; lisp_gpe_main_t *lgm = vnet_lisp_gpe_get_main (); - cpu_index = os_get_cpu_number (); + thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -267,7 +267,7 @@ lisp_gpe_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (si0) { - incr_decap_stats (lgm->vnet_main, cpu_index, + incr_decap_stats (lgm->vnet_main, thread_index, vlib_buffer_length_in_chain (vm, b0), si0[0], &last_sw_if_index, &n_packets, &n_bytes); vnet_buffer (b0)->sw_if_index[VLIB_RX] = si0[0]; @@ -282,7 +282,7 @@ lisp_gpe_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (si1) { - incr_decap_stats (lgm->vnet_main, cpu_index, + incr_decap_stats (lgm->vnet_main, thread_index, vlib_buffer_length_in_chain (vm, b1), si1[0], &last_sw_if_index, &n_packets, &n_bytes); vnet_buffer (b1)->sw_if_index[VLIB_RX] = si1[0]; @@ -397,7 +397,7 @@ lisp_gpe_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (si0) { - incr_decap_stats (lgm->vnet_main, cpu_index, + incr_decap_stats (lgm->vnet_main, thread_index, vlib_buffer_length_in_chain (vm, b0), si0[0], &last_sw_if_index, &n_packets, &n_bytes); vnet_buffer (b0)->sw_if_index[VLIB_RX] = si0[0]; @@ -430,7 +430,7 @@ lisp_gpe_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, } /* flush iface stats */ - incr_decap_stats (lgm->vnet_main, cpu_index, 0, ~0, &last_sw_if_index, + incr_decap_stats (lgm->vnet_main, thread_index, 0, ~0, &last_sw_if_index, &n_packets, &n_bytes); vlib_node_increment_counter (vm, lisp_gpe_ip4_input_node.index, LISP_GPE_ERROR_NO_TUNNEL, drops); diff --git a/src/vnet/lldp/lldp_input.c b/src/vnet/lldp/lldp_input.c index 762743d0..e88f6fdb 100644 --- a/src/vnet/lldp/lldp_input.c +++ b/src/vnet/lldp/lldp_input.c @@ -35,7 +35,7 @@ typedef struct static void lldp_rpc_update_peer_cb (const lldp_intf_update_t * a) { - ASSERT (os_get_cpu_number () == 0); + ASSERT (vlib_get_thread_index () == 0); lldp_intf_t *n = lldp_get_intf (&lldp_main, a->hw_if_index); if (!n) diff --git a/src/vnet/map/ip4_map.c b/src/vnet/map/ip4_map.c index 1a20d704..e39b6f14 100644 --- a/src/vnet/map/ip4_map.c +++ b/src/vnet/map/ip4_map.c @@ -248,7 +248,7 @@ ip4_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) next_index = node->cached_next_index; map_main_t *mm = &map_main; vlib_combined_counter_main_t *cm = mm->domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -377,7 +377,7 @@ ip4_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) ip40) ? IP4_MAP_NEXT_IP6_REWRITE : next0; vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, map_domain_index0, 1, clib_net_to_host_u16 (ip6h0->payload_length) + @@ -409,7 +409,7 @@ ip4_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) ip41) ? IP4_MAP_NEXT_IP6_REWRITE : next1; vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, map_domain_index1, 1, clib_net_to_host_u16 (ip6h1->payload_length) + @@ -520,7 +520,7 @@ ip4_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) ip40) ? IP4_MAP_NEXT_IP6_REWRITE : next0; vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, map_domain_index0, 1, clib_net_to_host_u16 (ip6h0->payload_length) + @@ -564,7 +564,7 @@ ip4_map_reass (vlib_main_t * vm, next_index = node->cached_next_index; map_main_t *mm = &map_main; vlib_combined_counter_main_t *cm = mm->domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 *fragments_to_drop = NULL; u32 *fragments_to_loopback = NULL; @@ -694,8 +694,8 @@ ip4_map_reass (vlib_main_t * vm, { if (error0 == MAP_ERROR_NONE) vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, map_domain_index0, - 1, + thread_index, + map_domain_index0, 1, clib_net_to_host_u16 (ip60->payload_length) + 40); next0 = diff --git a/src/vnet/map/ip4_map_t.c b/src/vnet/map/ip4_map_t.c index b63d76bf..5f2bcbf9 100644 --- a/src/vnet/map/ip4_map_t.c +++ b/src/vnet/map/ip4_map_t.c @@ -477,7 +477,7 @@ ip4_map_t_icmp (vlib_main_t * vm, n_left_from = frame->n_vectors; next_index = node->cached_next_index; vlib_combined_counter_main_t *cm = map_main.domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -520,7 +520,7 @@ ip4_map_t_icmp (vlib_main_t * vm, if (PREDICT_TRUE (error0 == MAP_ERROR_NONE)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, vnet_buffer (p0)->map_t. map_domain_index, 1, len0); } @@ -1051,7 +1051,7 @@ ip4_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) n_left_from = frame->n_vectors; next_index = node->cached_next_index; vlib_combined_counter_main_t *cm = map_main.domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -1158,7 +1158,7 @@ ip4_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) (error0 == MAP_ERROR_NONE && next0 != IP4_MAPT_NEXT_MAPT_ICMP)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, vnet_buffer (p0)->map_t. map_domain_index, 1, clib_net_to_host_u16 (ip40-> @@ -1169,7 +1169,7 @@ ip4_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) (error1 == MAP_ERROR_NONE && next1 != IP4_MAPT_NEXT_MAPT_ICMP)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, vnet_buffer (p1)->map_t. map_domain_index, 1, clib_net_to_host_u16 (ip41-> @@ -1252,7 +1252,7 @@ ip4_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) (error0 == MAP_ERROR_NONE && next0 != IP4_MAPT_NEXT_MAPT_ICMP)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_TX, - cpu_index, + thread_index, vnet_buffer (p0)->map_t. map_domain_index, 1, clib_net_to_host_u16 (ip40-> diff --git a/src/vnet/map/ip6_map.c b/src/vnet/map/ip6_map.c index f7eb768f..63ada962 100644 --- a/src/vnet/map/ip6_map.c +++ b/src/vnet/map/ip6_map.c @@ -172,7 +172,7 @@ ip6_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) vlib_node_get_runtime (vm, ip6_map_node.index); map_main_t *mm = &map_main; vlib_combined_counter_main_t *cm = mm->domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -319,7 +319,7 @@ ip6_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) IP6_MAP_NEXT_IP4_REWRITE : next0; } vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, map_domain_index0, 1, clib_net_to_host_u16 (ip40->length)); @@ -352,7 +352,7 @@ ip6_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) IP6_MAP_NEXT_IP4_REWRITE : next1; } vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, map_domain_index1, 1, clib_net_to_host_u16 (ip41->length)); @@ -505,7 +505,7 @@ ip6_map (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) IP6_MAP_NEXT_IP4_REWRITE : next0; } vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, map_domain_index0, 1, clib_net_to_host_u16 (ip40->length)); @@ -820,7 +820,7 @@ ip6_map_ip4_reass (vlib_main_t * vm, vlib_node_get_runtime (vm, ip6_map_ip4_reass_node.index); map_main_t *mm = &map_main; vlib_combined_counter_main_t *cm = mm->domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 *fragments_to_drop = NULL; u32 *fragments_to_loopback = NULL; @@ -958,8 +958,8 @@ ip6_map_ip4_reass (vlib_main_t * vm, { if (error0 == MAP_ERROR_NONE) vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, map_domain_index0, - 1, + thread_index, + map_domain_index0, 1, clib_net_to_host_u16 (ip40->length)); next0 = @@ -1015,7 +1015,7 @@ ip6_map_icmp_relay (vlib_main_t * vm, vlib_node_runtime_t *error_node = vlib_node_get_runtime (vm, ip6_map_icmp_relay_node.index); map_main_t *mm = &map_main; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u16 *fragment_ids, *fid; from = vlib_frame_vector_args (frame); @@ -1143,7 +1143,8 @@ ip6_map_icmp_relay (vlib_main_t * vm, ip_csum_t sum = ip_incremental_checksum (0, new_icmp40, nlen - 20); new_icmp40->checksum = ~ip_csum_fold (sum); - vlib_increment_simple_counter (&mm->icmp_relayed, cpu_index, 0, 1); + vlib_increment_simple_counter (&mm->icmp_relayed, thread_index, 0, + 1); error: if (PREDICT_FALSE (p0->flags & VLIB_BUFFER_IS_TRACED)) diff --git a/src/vnet/map/ip6_map_t.c b/src/vnet/map/ip6_map_t.c index eb3996c2..99151678 100644 --- a/src/vnet/map/ip6_map_t.c +++ b/src/vnet/map/ip6_map_t.c @@ -448,7 +448,7 @@ ip6_map_t_icmp (vlib_main_t * vm, n_left_from = frame->n_vectors; next_index = node->cached_next_index; vlib_combined_counter_main_t *cm = map_main.domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -493,7 +493,7 @@ ip6_map_t_icmp (vlib_main_t * vm, if (PREDICT_TRUE (error0 == MAP_ERROR_NONE)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, vnet_buffer (p0)-> map_t.map_domain_index, 1, len0); @@ -1051,7 +1051,7 @@ ip6_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) vlib_node_runtime_t *error_node = vlib_node_get_runtime (vm, ip6_map_t_node.index); vlib_combined_counter_main_t *cm = map_main.domain_counters; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -1218,7 +1218,7 @@ ip6_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) (error0 == MAP_ERROR_NONE && next0 != IP6_MAPT_NEXT_MAPT_ICMP)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, vnet_buffer (p0)-> map_t.map_domain_index, 1, clib_net_to_host_u16 @@ -1229,7 +1229,7 @@ ip6_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) (error1 == MAP_ERROR_NONE && next1 != IP6_MAPT_NEXT_MAPT_ICMP)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, vnet_buffer (p1)-> map_t.map_domain_index, 1, clib_net_to_host_u16 @@ -1403,7 +1403,7 @@ ip6_map_t (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) (error0 == MAP_ERROR_NONE && next0 != IP6_MAPT_NEXT_MAPT_ICMP)) { vlib_increment_combined_counter (cm + MAP_DOMAIN_COUNTER_RX, - cpu_index, + thread_index, vnet_buffer (p0)-> map_t.map_domain_index, 1, clib_net_to_host_u16 diff --git a/src/vnet/mpls/mpls_input.c b/src/vnet/mpls/mpls_input.c index 893c4511..1b9bdd05 100644 --- a/src/vnet/mpls/mpls_input.c +++ b/src/vnet/mpls/mpls_input.c @@ -76,7 +76,7 @@ mpls_input_inline (vlib_main_t * vm, u32 n_left_from, next_index, * from, * to_next; mpls_input_runtime_t * rt; mpls_main_t * mm; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); vlib_simple_counter_main_t * cm; vnet_main_t * vnm = vnet_get_main(); @@ -151,7 +151,7 @@ mpls_input_inline (vlib_main_t * vm, next0 = MPLS_INPUT_NEXT_LOOKUP; vnet_feature_arc_start(mm->input_feature_arc_index, sw_if_index0, &next0, b0); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); } if (PREDICT_FALSE(h1[3] == 0)) @@ -164,7 +164,7 @@ mpls_input_inline (vlib_main_t * vm, next1 = MPLS_INPUT_NEXT_LOOKUP; vnet_feature_arc_start(mm->input_feature_arc_index, sw_if_index1, &next1, b1); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index1, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index1, 1); } if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) @@ -215,7 +215,7 @@ mpls_input_inline (vlib_main_t * vm, { next0 = MPLS_INPUT_NEXT_LOOKUP; vnet_feature_arc_start(mm->input_feature_arc_index, sw_if_index0, &next0, b0); - vlib_increment_simple_counter (cm, cpu_index, sw_if_index0, 1); + vlib_increment_simple_counter (cm, thread_index, sw_if_index0, 1); } if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) diff --git a/src/vnet/mpls/mpls_lookup.c b/src/vnet/mpls/mpls_lookup.c index 475bb204..ace6a70f 100644 --- a/src/vnet/mpls/mpls_lookup.c +++ b/src/vnet/mpls/mpls_lookup.c @@ -67,7 +67,7 @@ mpls_lookup (vlib_main_t * vm, vlib_combined_counter_main_t * cm = &load_balance_main.lbm_to_counters; u32 n_left_from, next_index, * from, * to_next; mpls_main_t * mm = &mpls_main; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -220,16 +220,16 @@ mpls_lookup (vlib_main_t * vm, vnet_buffer (b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, b1)); vlib_increment_combined_counter - (cm, cpu_index, lbi2, 1, + (cm, thread_index, lbi2, 1, vlib_buffer_length_in_chain (vm, b2)); vlib_increment_combined_counter - (cm, cpu_index, lbi3, 1, + (cm, thread_index, lbi3, 1, vlib_buffer_length_in_chain (vm, b3)); /* @@ -351,7 +351,7 @@ mpls_lookup (vlib_main_t * vm, vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, b0)); /* @@ -440,7 +440,7 @@ mpls_load_balance (vlib_main_t * vm, { vlib_combined_counter_main_t * cm = &load_balance_main.lbm_via_counters; u32 n_left_from, n_left_to_next, * from, * to_next; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); u32 next; from = vlib_frame_vector_args (frame); @@ -536,10 +536,10 @@ mpls_load_balance (vlib_main_t * vm, vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_increment_combined_counter - (cm, cpu_index, lbi1, 1, + (cm, thread_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1)); if (PREDICT_FALSE(p0->flags & VLIB_BUFFER_IS_TRACED)) @@ -597,7 +597,7 @@ mpls_load_balance (vlib_main_t * vm, vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; vlib_increment_combined_counter - (cm, cpu_index, lbi0, 1, + (cm, thread_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0)); vlib_validate_buffer_enqueue_x1 (vm, node, next, diff --git a/src/vnet/mpls/mpls_output.c b/src/vnet/mpls/mpls_output.c index 08018fd1..d90dec21 100644 --- a/src/vnet/mpls/mpls_output.c +++ b/src/vnet/mpls/mpls_output.c @@ -64,12 +64,12 @@ mpls_output_inline (vlib_main_t * vm, vlib_frame_t * from_frame, int is_midchain) { - u32 n_left_from, next_index, * from, * to_next, cpu_index; + u32 n_left_from, next_index, * from, * to_next, thread_index; vlib_node_runtime_t * error_node; u32 n_left_to_next; mpls_main_t *mm; - cpu_index = os_get_cpu_number(); + thread_index = vlib_get_thread_index(); error_node = vlib_node_get_runtime (vm, mpls_output_node.index); from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -137,13 +137,13 @@ mpls_output_inline (vlib_main_t * vm, /* Bump the adj counters for packet and bytes */ vlib_increment_combined_counter (&adjacency_counters, - cpu_index, + thread_index, adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); vlib_increment_combined_counter (&adjacency_counters, - cpu_index, + thread_index, adj_index1, 1, vlib_buffer_length_in_chain (vm, p1) + rw_len1); @@ -245,7 +245,7 @@ mpls_output_inline (vlib_main_t * vm, vlib_increment_combined_counter (&adjacency_counters, - cpu_index, + thread_index, adj_index0, 1, vlib_buffer_length_in_chain (vm, p0) + rw_len0); diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c index 2649798b..597ae060 100644 --- a/src/vnet/pg/input.c +++ b/src/vnet/pg/input.c @@ -893,7 +893,7 @@ pg_generate_set_lengths (pg_main_t * pg, vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number (), + vlib_get_thread_index (), si->sw_if_index, n_buffers, length_sum); } @@ -1266,7 +1266,7 @@ pg_stream_fill_helper (pg_main_t * pg, l += vlib_buffer_index_length_in_chain (vm, buffers[i]); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number (), + vlib_get_thread_index (), si->sw_if_index, n_alloc, l); s->current_replay_packet_index += n_alloc; s->current_replay_packet_index %= diff --git a/src/vnet/replication.c b/src/vnet/replication.c index 86d922b5..233a8c2f 100644 --- a/src/vnet/replication.c +++ b/src/vnet/replication.c @@ -31,16 +31,16 @@ replication_prep (vlib_main_t * vm, { replication_main_t *rm = &replication_main; replication_context_t *ctx; - uword cpu_number = vm->cpu_index; + uword thread_index = vm->thread_index; ip4_header_t *ip; u32 ctx_id; /* Allocate a context, reserve context 0 */ - if (PREDICT_FALSE (rm->contexts[cpu_number] == 0)) - pool_get_aligned (rm->contexts[cpu_number], ctx, CLIB_CACHE_LINE_BYTES); + if (PREDICT_FALSE (rm->contexts[thread_index] == 0)) + pool_get_aligned (rm->contexts[thread_index], ctx, CLIB_CACHE_LINE_BYTES); - pool_get_aligned (rm->contexts[cpu_number], ctx, CLIB_CACHE_LINE_BYTES); - ctx_id = ctx - rm->contexts[cpu_number]; + pool_get_aligned (rm->contexts[thread_index], ctx, CLIB_CACHE_LINE_BYTES); + ctx_id = ctx - rm->contexts[thread_index]; /* Save state from vlib buffer */ ctx->saved_free_list_index = b0->free_list_index; @@ -94,11 +94,11 @@ replication_recycle (vlib_main_t * vm, vlib_buffer_t * b0, u32 is_last) { replication_main_t *rm = &replication_main; replication_context_t *ctx; - uword cpu_number = vm->cpu_index; + uword thread_index = vm->thread_index; ip4_header_t *ip; /* Get access to the replication context */ - ctx = pool_elt_at_index (rm->contexts[cpu_number], b0->recycle_count); + ctx = pool_elt_at_index (rm->contexts[thread_index], b0->recycle_count); /* Restore vnet buffer state */ clib_memcpy (vnet_buffer (b0), ctx->vnet_buffer, @@ -133,7 +133,7 @@ replication_recycle (vlib_main_t * vm, vlib_buffer_t * b0, u32 is_last) b0->flags &= ~VLIB_BUFFER_RECYCLE; /* Free context back to its pool */ - pool_put (rm->contexts[cpu_number], ctx); + pool_put (rm->contexts[thread_index], ctx); } return ctx; @@ -160,7 +160,7 @@ replication_recycle_callback (vlib_main_t * vm, vlib_buffer_free_list_t * fl) replication_main_t *rm = &replication_main; replication_context_t *ctx; u32 feature_node_index = 0; - uword cpu_number = vm->cpu_index; + uword thread_index = vm->thread_index; /* * All buffers in the list are destined to the same recycle node. @@ -172,7 +172,7 @@ replication_recycle_callback (vlib_main_t * vm, vlib_buffer_free_list_t * fl) { bi0 = fl->buffers[0]; b0 = vlib_get_buffer (vm, bi0); - ctx = pool_elt_at_index (rm->contexts[cpu_number], b0->recycle_count); + ctx = pool_elt_at_index (rm->contexts[thread_index], b0->recycle_count); feature_node_index = ctx->recycle_node_index; } diff --git a/src/vnet/replication.h b/src/vnet/replication.h index 5dc554c9..ce4b3ff1 100644 --- a/src/vnet/replication.h +++ b/src/vnet/replication.h @@ -100,7 +100,7 @@ replication_get_ctx (vlib_buffer_t * b0) replication_main_t *rm = &replication_main; return replication_is_recycled (b0) ? - pool_elt_at_index (rm->contexts[os_get_cpu_number ()], + pool_elt_at_index (rm->contexts[vlib_get_thread_index ()], b0->recycle_count) : 0; } diff --git a/src/vnet/session/node.c b/src/vnet/session/node.c index b86e87d9..dd211c51 100644 --- a/src/vnet/session/node.c +++ b/src/vnet/session/node.c @@ -311,7 +311,7 @@ session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, unix_shared_memory_queue_t *q; application_t *app; int n_tx_packets = 0; - u32 my_thread_index = vm->cpu_index; + u32 my_thread_index = vm->thread_index; int i, rv; f64 now = vlib_time_now (vm); diff --git a/src/vnet/sr/sr_localsid.c b/src/vnet/sr/sr_localsid.c index 2e3d56de..6d72a506 100755 --- a/src/vnet/sr/sr_localsid.c +++ b/src/vnet/sr/sr_localsid.c @@ -887,7 +887,7 @@ sr_localsid_d_fn (vlib_main_t * vm, vlib_node_runtime_t * node, from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; next_index = node->cached_next_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -974,26 +974,26 @@ sr_localsid_d_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_increment_combined_counter (((next0 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls0 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b0)); + &(sm->sr_ls_valid_counters)), thread_index, ls0 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter (((next1 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls1 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b1)); + &(sm->sr_ls_valid_counters)), thread_index, ls1 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b1)); vlib_increment_combined_counter (((next2 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls2 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b2)); + &(sm->sr_ls_valid_counters)), thread_index, ls2 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b2)); vlib_increment_combined_counter (((next3 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls3 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b3)); + &(sm->sr_ls_valid_counters)), thread_index, ls3 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b3)); vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, n_left_to_next, bi0, bi1, bi2, bi3, @@ -1062,8 +1062,8 @@ sr_localsid_d_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_increment_combined_counter (((next0 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls0 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b0)); + &(sm->sr_ls_valid_counters)), thread_index, ls0 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b0)); vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0); @@ -1103,7 +1103,7 @@ sr_localsid_fn (vlib_main_t * vm, vlib_node_runtime_t * node, from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; next_index = node->cached_next_index; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); while (n_left_from > 0) { @@ -1205,26 +1205,26 @@ sr_localsid_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_increment_combined_counter (((next0 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls0 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b0)); + &(sm->sr_ls_valid_counters)), thread_index, ls0 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b0)); vlib_increment_combined_counter (((next1 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls1 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b1)); + &(sm->sr_ls_valid_counters)), thread_index, ls1 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b1)); vlib_increment_combined_counter (((next2 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls2 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b2)); + &(sm->sr_ls_valid_counters)), thread_index, ls2 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b2)); vlib_increment_combined_counter (((next3 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls3 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b3)); + &(sm->sr_ls_valid_counters)), thread_index, ls3 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b3)); vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, n_left_to_next, bi0, bi1, bi2, bi3, @@ -1295,8 +1295,8 @@ sr_localsid_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_increment_combined_counter (((next0 == SR_LOCALSID_NEXT_ERROR) ? &(sm->sr_ls_invalid_counters) : - &(sm->sr_ls_valid_counters)), cpu_index, ls0 - sm->localsids, 1, - vlib_buffer_length_in_chain (vm, b0)); + &(sm->sr_ls_valid_counters)), thread_index, ls0 - sm->localsids, + 1, vlib_buffer_length_in_chain (vm, b0)); vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0); diff --git a/src/vnet/tcp/builtin_client.c b/src/vnet/tcp/builtin_client.c index e3705060..c1567aa0 100644 --- a/src/vnet/tcp/builtin_client.c +++ b/src/vnet/tcp/builtin_client.c @@ -174,7 +174,7 @@ tclient_thread_fn (void *arg) pthread_sigmask (SIG_SETMASK, &s, 0); } - clib_per_cpu_mheaps[os_get_cpu_number ()] = clib_per_cpu_mheaps[0]; + clib_per_cpu_mheaps[vlib_get_thread_index ()] = clib_per_cpu_mheaps[0]; while (1) { diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c index b2a371e2..b6c34828 100644 --- a/src/vnet/tcp/tcp.c +++ b/src/vnet/tcp/tcp.c @@ -646,10 +646,10 @@ const static transport_proto_vft_t tcp6_proto = { void tcp_timer_keep_handler (u32 conn_index) { - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); tcp_connection_t *tc; - tc = tcp_connection_get (conn_index, cpu_index); + tc = tcp_connection_get (conn_index, thread_index); tc->timers[TCP_TIMER_KEEP] = TCP_TIMER_HANDLE_INVALID; tcp_connection_close (tc); @@ -675,10 +675,10 @@ tcp_timer_establish_handler (u32 conn_index) void tcp_timer_waitclose_handler (u32 conn_index) { - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); tcp_connection_t *tc; - tc = tcp_connection_get (conn_index, cpu_index); + tc = tcp_connection_get (conn_index, thread_index); tc->timers[TCP_TIMER_WAITCLOSE] = TCP_TIMER_HANDLE_INVALID; /* Session didn't come back with a close(). Send FIN either way diff --git a/src/vnet/tcp/tcp_debug.h b/src/vnet/tcp/tcp_debug.h index 0090e15e..eaca672c 100644 --- a/src/vnet/tcp/tcp_debug.h +++ b/src/vnet/tcp/tcp_debug.h @@ -343,7 +343,7 @@ typedef enum _tcp_dbg_evt } \ else \ { \ - u32 _thread_index = os_get_cpu_number (); \ + u32 _thread_index = vlib_get_thread_index (); \ _tc = tcp_connection_get (_tc_index, _thread_index); \ } \ ELOG_TYPE_DECLARE (_e) = \ diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c index a8224dc2..7e9fa47b 100644 --- a/src/vnet/tcp/tcp_input.c +++ b/src/vnet/tcp/tcp_input.c @@ -1142,7 +1142,7 @@ tcp46_established_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame, int is_ip4) { u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index, errors = 0; + u32 my_thread_index = vm->thread_index, errors = 0; tcp_main_t *tm = vnet_get_tcp_main (); from = vlib_frame_vector_args (from_frame); @@ -1332,7 +1332,7 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { tcp_main_t *tm = vnet_get_tcp_main (); u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index, errors = 0; + u32 my_thread_index = vm->thread_index, errors = 0; u8 sst = is_ip4 ? SESSION_TYPE_IP4_TCP : SESSION_TYPE_IP6_TCP; from = vlib_frame_vector_args (from_frame); @@ -1634,7 +1634,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { tcp_main_t *tm = vnet_get_tcp_main (); u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index, errors = 0; + u32 my_thread_index = vm->thread_index, errors = 0; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -1989,7 +1989,7 @@ tcp46_listen_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame, int is_ip4) { u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index; + u32 my_thread_index = vm->thread_index; tcp_main_t *tm = vnet_get_tcp_main (); u8 sst = is_ip4 ? SESSION_TYPE_IP4_TCP : SESSION_TYPE_IP6_TCP; @@ -2243,7 +2243,7 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame, int is_ip4) { u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index; + u32 my_thread_index = vm->thread_index; tcp_main_t *tm = vnet_get_tcp_main (); from = vlib_frame_vector_args (from_frame); diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index ea157bd7..e18bfad7 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -387,8 +387,8 @@ tcp_make_options (tcp_connection_t * tc, tcp_options_t * opts, #define tcp_get_free_buffer_index(tm, bidx) \ do { \ u32 *my_tx_buffers, n_free_buffers; \ - u32 cpu_index = os_get_cpu_number(); \ - my_tx_buffers = tm->tx_buffers[cpu_index]; \ + u32 thread_index = vlib_get_thread_index(); \ + my_tx_buffers = tm->tx_buffers[thread_index]; \ if (PREDICT_FALSE(vec_len (my_tx_buffers) == 0)) \ { \ n_free_buffers = 32; /* TODO config or macro */ \ @@ -396,7 +396,7 @@ do { \ _vec_len(my_tx_buffers) = vlib_buffer_alloc_from_free_list ( \ tm->vlib_main, my_tx_buffers, n_free_buffers, \ VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); \ - tm->tx_buffers[cpu_index] = my_tx_buffers; \ + tm->tx_buffers[thread_index] = my_tx_buffers; \ } \ /* buffer shortage */ \ if (PREDICT_FALSE (vec_len (my_tx_buffers) == 0)) \ @@ -408,8 +408,8 @@ do { \ #define tcp_return_buffer(tm) \ do { \ u32 *my_tx_buffers; \ - u32 cpu_index = os_get_cpu_number(); \ - my_tx_buffers = tm->tx_buffers[cpu_index]; \ + u32 thread_index = vlib_get_thread_index(); \ + my_tx_buffers = tm->tx_buffers[thread_index]; \ _vec_len (my_tx_buffers) +=1; \ } while (0) @@ -942,7 +942,7 @@ tcp_send_ack (tcp_connection_t * tc) void tcp_timer_delack_handler (u32 index) { - u32 thread_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); tcp_connection_t *tc; tc = tcp_connection_get (index, thread_index); @@ -1022,7 +1022,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) { tcp_main_t *tm = vnet_get_tcp_main (); vlib_main_t *vm = vlib_get_main (); - u32 thread_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); tcp_connection_t *tc; vlib_buffer_t *b; u32 bi, snd_space, n_bytes; @@ -1152,7 +1152,7 @@ tcp_timer_persist_handler (u32 index) { tcp_main_t *tm = vnet_get_tcp_main (); vlib_main_t *vm = vlib_get_main (); - u32 thread_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); tcp_connection_t *tc; vlib_buffer_t *b; u32 bi, n_bytes; @@ -1313,7 +1313,7 @@ tcp46_output_inline (vlib_main_t * vm, vlib_frame_t * from_frame, int is_ip4) { u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index; + u32 my_thread_index = vm->thread_index; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; @@ -1524,7 +1524,7 @@ tcp46_send_reset_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame, u8 is_ip4) { u32 n_left_from, next_index, *from, *to_next; - u32 my_thread_index = vm->cpu_index; + u32 my_thread_index = vm->thread_index; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; diff --git a/src/vnet/udp/udp_input.c b/src/vnet/udp/udp_input.c index 4b22109b..810278e6 100644 --- a/src/vnet/udp/udp_input.c +++ b/src/vnet/udp/udp_input.c @@ -70,7 +70,7 @@ udp4_uri_input_node_fn (vlib_main_t * vm, udp4_uri_input_next_t next_index; udp_uri_main_t *um = vnet_get_udp_main (); session_manager_main_t *smm = vnet_get_session_manager_main (); - u32 my_thread_index = vm->cpu_index; + u32 my_thread_index = vm->thread_index; u8 my_enqueue_epoch; u32 *session_indices_to_enqueue; static u32 serial_number; diff --git a/src/vnet/unix/tapcli.c b/src/vnet/unix/tapcli.c index fb1a8bac..0fc62f6c 100644 --- a/src/vnet/unix/tapcli.c +++ b/src/vnet/unix/tapcli.c @@ -366,7 +366,7 @@ static uword tapcli_rx_iface(vlib_main_t * vm, vlib_increment_combined_counter ( vnet_main.interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number(), ti->sw_if_index, + vlib_get_thread_index(), ti->sw_if_index, 1, n_bytes_in_packet); if (PREDICT_FALSE(n_trace > 0)) { diff --git a/src/vnet/unix/tuntap.c b/src/vnet/unix/tuntap.c index 2cfcc92f..ac674653 100644 --- a/src/vnet/unix/tuntap.c +++ b/src/vnet/unix/tuntap.c @@ -189,7 +189,7 @@ tuntap_tx (vlib_main_t * vm, /* Update tuntap interface output stats. */ vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - vm->cpu_index, + vm->thread_index, tm->sw_if_index, n_packets, n_bytes); @@ -297,7 +297,7 @@ tuntap_rx (vlib_main_t * vm, vlib_increment_combined_counter (vnet_main.interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - os_get_cpu_number(), + vlib_get_thread_index(), tm->sw_if_index, 1, n_bytes_in_packet); diff --git a/src/vnet/vxlan-gpe/decap.c b/src/vnet/vxlan-gpe/decap.c index 22ab4b62..d4fe4231 100644 --- a/src/vnet/vxlan-gpe/decap.c +++ b/src/vnet/vxlan-gpe/decap.c @@ -115,7 +115,7 @@ vxlan_gpe_input (vlib_main_t * vm, vxlan4_gpe_tunnel_key_t last_key4; vxlan6_gpe_tunnel_key_t last_key6; u32 pkts_decapsulated = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; if (is_ip4) @@ -342,7 +342,7 @@ vxlan_gpe_input (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; stats_sw_if_index = sw_if_index0; @@ -427,7 +427,7 @@ vxlan_gpe_input (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len1; stats_sw_if_index = sw_if_index1; @@ -588,7 +588,7 @@ vxlan_gpe_input (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; stats_sw_if_index = sw_if_index0; @@ -615,7 +615,7 @@ vxlan_gpe_input (vlib_main_t * vm, if (stats_n_packets) { vlib_increment_combined_counter ( - im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, cpu_index, + im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } diff --git a/src/vnet/vxlan-gpe/encap.c b/src/vnet/vxlan-gpe/encap.c index 3a486e56..67ed94b4 100644 --- a/src/vnet/vxlan-gpe/encap.c +++ b/src/vnet/vxlan-gpe/encap.c @@ -151,7 +151,7 @@ vxlan_gpe_encap (vlib_main_t * vm, vnet_main_t * vnm = ngm->vnet_main; vnet_interface_main_t * im = &vnm->interface_main; u32 pkts_encapsulated = 0; - u32 cpu_index = os_get_cpu_number (); + u32 thread_index = vlib_get_thread_index (); u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; from = vlib_frame_vector_args (from_frame); @@ -253,7 +253,7 @@ vxlan_gpe_encap (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_sw_if_index = sw_if_index0; stats_n_packets = 2; stats_n_bytes = len0 + len1; @@ -262,10 +262,10 @@ vxlan_gpe_encap (vlib_main_t * vm, { vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, sw_if_index0, 1, len0); + thread_index, sw_if_index0, 1, len0); vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, sw_if_index1, 1, len1); + thread_index, sw_if_index1, 1, len1); } } @@ -335,7 +335,7 @@ vxlan_gpe_encap (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter ( im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; stats_sw_if_index = sw_if_index0; @@ -359,7 +359,7 @@ vxlan_gpe_encap (vlib_main_t * vm, if (stats_n_packets) { vlib_increment_combined_counter ( - im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, cpu_index, + im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } diff --git a/src/vnet/vxlan/decap.c b/src/vnet/vxlan/decap.c index 514b2c99..2acb1f6f 100644 --- a/src/vnet/vxlan/decap.c +++ b/src/vnet/vxlan/decap.c @@ -81,7 +81,7 @@ vxlan_input (vlib_main_t * vm, vxlan4_tunnel_key_t last_key4; vxlan6_tunnel_key_t last_key6; u32 pkts_decapsulated = 0; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; if (is_ip4) @@ -314,7 +314,7 @@ vxlan_input (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; @@ -468,7 +468,7 @@ vxlan_input (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len1; @@ -674,7 +674,7 @@ vxlan_input (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; @@ -711,7 +711,7 @@ vxlan_input (vlib_main_t * vm, { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } diff --git a/src/vnet/vxlan/encap.c b/src/vnet/vxlan/encap.c index 5b63064a..4cfbbc23 100644 --- a/src/vnet/vxlan/encap.c +++ b/src/vnet/vxlan/encap.c @@ -77,7 +77,7 @@ vxlan_encap_inline (vlib_main_t * vm, vnet_interface_main_t * im = &vnm->interface_main; u32 pkts_encapsulated = 0; u16 old_l0 = 0, old_l1 = 0; - u32 cpu_index = os_get_cpu_number(); + u32 thread_index = vlib_get_thread_index(); u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; u32 sw_if_index0 = 0, sw_if_index1 = 0; u32 next0 = 0, next1 = 0; @@ -301,7 +301,7 @@ vxlan_encap_inline (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, stats_sw_if_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_sw_if_index = sw_if_index0; stats_n_packets = 2; @@ -311,10 +311,10 @@ vxlan_encap_inline (vlib_main_t * vm, { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, sw_if_index0, 1, len0); + thread_index, sw_if_index0, 1, len0); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, sw_if_index1, 1, len1); + thread_index, sw_if_index1, 1, len1); } } @@ -464,7 +464,7 @@ vxlan_encap_inline (vlib_main_t * vm, if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, stats_sw_if_index, + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; @@ -496,7 +496,7 @@ vxlan_encap_inline (vlib_main_t * vm, { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, - cpu_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); + thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } diff --git a/src/vpp/stats/stats.c b/src/vpp/stats/stats.c index 042d02e2..4309cd51 100644 --- a/src/vpp/stats/stats.c +++ b/src/vpp/stats/stats.c @@ -66,14 +66,14 @@ _(VNET_IP6_NBR_COUNTERS, vnet_ip6_nbr_counters) void dslock (stats_main_t * sm, int release_hint, int tag) { - u32 thread_id; + u32 thread_index; data_structure_lock_t *l = sm->data_structure_lock; if (PREDICT_FALSE (l == 0)) return; - thread_id = os_get_cpu_number (); - if (l->lock && l->thread_id == thread_id) + thread_index = vlib_get_thread_index (); + if (l->lock && l->thread_index == thread_index) { l->count++; return; @@ -85,7 +85,7 @@ dslock (stats_main_t * sm, int release_hint, int tag) while (__sync_lock_test_and_set (&l->lock, 1)) /* zzzz */ ; l->tag = tag; - l->thread_id = thread_id; + l->thread_index = thread_index; l->count = 1; } @@ -99,14 +99,14 @@ stats_dslock_with_hint (int hint, int tag) void dsunlock (stats_main_t * sm) { - u32 thread_id; + u32 thread_index; data_structure_lock_t *l = sm->data_structure_lock; if (PREDICT_FALSE (l == 0)) return; - thread_id = os_get_cpu_number (); - ASSERT (l->lock && l->thread_id == thread_id); + thread_index = vlib_get_thread_index (); + ASSERT (l->lock && l->thread_index == thread_index); l->count--; if (l->count == 0) { diff --git a/src/vpp/stats/stats.h b/src/vpp/stats/stats.h index 118115be..024dc78e 100644 --- a/src/vpp/stats/stats.h +++ b/src/vpp/stats/stats.h @@ -30,7 +30,7 @@ typedef struct { volatile u32 lock; volatile u32 release_hint; - u32 thread_id; + u32 thread_index; u32 count; int tag; } data_structure_lock_t; -- cgit 1.2.3-korg From 0f26c5a0138ac86d7ebd197c31a09d8d624c35fe Mon Sep 17 00:00:00 2001 From: Neale Ranns Date: Wed, 1 Mar 2017 15:12:11 -0800 Subject: MPLS Mcast 1 - interface-DPO Used in the Data-plane to change a packet's input interface 2 - MPLS multicast FIB entry Same as a unicast entry but it links to a replicate not a load-balance DPO 3 - Multicast MPLS tunnel Update MPLS tunnels to use a FIB path-list to describe the endpoint[s]. Use the path-list to generate the forwarding chain (DPOs) to link to . 4 - Resolve a path via a local label (of an mLDP LSP) For IP multicast entries to use an LSP in the replication list, we need to decribe the 'resolve-via-label' where the label is that of a multicast LSP. 5 - MPLS disposition path sets RPF-ID For a interface-less LSP (i.e. mLDP not RSVP-TE) at the tail of the LSP we still need to perform an RPF check. An MPLS disposition DPO performs the MPLS pop validation checks and sets the RPF-ID in the packet. 6 - RPF check with per-entry RPF-ID An RPF-ID is used instead of a real interface SW if index in the case the IP traffic arrives from an LSP that does not have an associated interface. Change-Id: Ib92e177be919147bafeb599729abf3d1abc2f4b3 Signed-off-by: Neale Ranns --- src/plugins/dpdk/device/node.c | 2 +- src/vat/api_format.c | 133 ++--- src/vnet.am | 2 + src/vnet/adj/adj.c | 13 +- src/vnet/adj/adj.h | 6 + src/vnet/adj/adj_internal.h | 14 +- src/vnet/adj/adj_mcast.c | 134 ++++- src/vnet/adj/adj_mcast.h | 27 + src/vnet/adj/adj_midchain.c | 62 ++- src/vnet/adj/adj_nbr.c | 2 - src/vnet/buffer.h | 3 + src/vnet/devices/ssvm/node.c | 2 +- src/vnet/dhcp/dhcp6_proxy_node.c | 1 + src/vnet/dpo/dpo.c | 10 + src/vnet/dpo/dpo.h | 8 +- src/vnet/dpo/interface_dpo.c | 416 ++++++++++++++++ src/vnet/dpo/interface_dpo.h | 67 +++ src/vnet/dpo/lookup_dpo.c | 211 +++++++- src/vnet/dpo/lookup_dpo.h | 20 + src/vnet/dpo/mpls_disposition.c | 364 ++++++++++++++ src/vnet/dpo/mpls_disposition.h | 85 ++++ src/vnet/dpo/mpls_label_dpo.c | 6 +- src/vnet/dpo/replicate_dpo.c | 48 +- src/vnet/dpo/replicate_dpo.h | 2 + src/vnet/ethernet/arp.c | 1 + src/vnet/ethernet/interface.c | 2 +- src/vnet/ethernet/node.c | 4 +- src/vnet/ethernet/types.def | 4 +- src/vnet/fib/fib_api.h | 4 + src/vnet/fib/fib_entry.c | 47 +- src/vnet/fib/fib_entry.h | 13 +- src/vnet/fib/fib_entry_src.c | 154 +++--- src/vnet/fib/fib_internal.h | 1 + src/vnet/fib/fib_path.c | 222 +++++++-- src/vnet/fib/fib_path.h | 17 +- src/vnet/fib/fib_path_ext.c | 4 +- src/vnet/fib/fib_path_ext.h | 3 +- src/vnet/fib/fib_path_list.c | 270 ++++++---- src/vnet/fib/fib_path_list.h | 22 +- src/vnet/fib/fib_table.c | 47 +- src/vnet/fib/fib_test.c | 345 +++++++++++-- src/vnet/fib/fib_test.h | 111 +++++ src/vnet/fib/fib_types.c | 15 +- src/vnet/fib/fib_types.h | 60 ++- src/vnet/fib/mpls_fib.c | 15 +- src/vnet/handoff.h | 10 +- src/vnet/interface.c | 2 +- src/vnet/ip/ip.api | 3 + src/vnet/ip/ip4_forward.c | 20 + src/vnet/ip/ip6_forward.c | 23 + src/vnet/ip/ip6_neighbor.c | 1 + src/vnet/ip/ip_api.c | 98 ++-- src/vnet/ip/lookup.c | 3 +- src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c | 1 + src/vnet/mfib/ip4_mfib.c | 1 + src/vnet/mfib/ip6_mfib.c | 1 + src/vnet/mfib/mfib_entry.c | 395 +++++++++------ src/vnet/mfib/mfib_entry.h | 20 +- src/vnet/mfib/mfib_forward.c | 29 +- src/vnet/mfib/mfib_table.c | 8 +- src/vnet/mfib/mfib_table.h | 1 + src/vnet/mfib/mfib_test.c | 127 ++++- src/vnet/mpls/mpls.api | 87 ++-- src/vnet/mpls/mpls.c | 17 +- src/vnet/mpls/mpls_api.c | 97 ++-- src/vnet/mpls/mpls_input.c | 2 +- src/vnet/mpls/mpls_lookup.c | 236 ++++++--- src/vnet/mpls/mpls_tunnel.c | 883 ++++++++++++++++++++++----------- src/vnet/mpls/mpls_tunnel.h | 57 ++- src/vnet/mpls/mpls_types.h | 20 + src/vnet/srp/interface.c | 2 +- test/test_ip_mcast.py | 1 + test/test_mpls.py | 277 ++++++++++- test/vpp_ip_route.py | 38 +- test/vpp_mpls_tunnel_interface.py | 46 ++ test/vpp_papi_provider.py | 16 +- 76 files changed, 4393 insertions(+), 1128 deletions(-) create mode 100644 src/vnet/dpo/interface_dpo.c create mode 100644 src/vnet/dpo/interface_dpo.h create mode 100644 src/vnet/dpo/mpls_disposition.c create mode 100644 src/vnet/dpo/mpls_disposition.h create mode 100644 src/vnet/fib/fib_test.h create mode 100644 test/vpp_mpls_tunnel_interface.py (limited to 'src/vnet/ethernet/node.c') diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index b10e0fad..0549ba5d 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -52,7 +52,7 @@ always_inline int vlib_buffer_is_mpls (vlib_buffer_t * b) { ethernet_header_t *h = (ethernet_header_t *) vlib_buffer_get_current (b); - return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST)); + return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)); } always_inline u32 diff --git a/src/vat/api_format.c b/src/vat/api_format.c index 61b8e1d8..107aa012 100644 --- a/src/vat/api_format.c +++ b/src/vat/api_format.c @@ -16369,32 +16369,82 @@ api_netmap_delete (vat_main_t * vam) return ret; } -static void vl_api_mpls_tunnel_details_t_handler - (vl_api_mpls_tunnel_details_t * mp) +static void +vl_api_mpls_fib_path_print (vat_main_t * vam, vl_api_fib_path2_t * fp) +{ + if (fp->afi == IP46_TYPE_IP6) + print (vam->ofp, + " weight %d, sw_if_index %d, is_local %d, is_drop %d, " + "is_unreach %d, is_prohitbit %d, afi %d, next_hop %U", + ntohl (fp->weight), ntohl (fp->sw_if_index), fp->is_local, + fp->is_drop, fp->is_unreach, fp->is_prohibit, fp->afi, + format_ip6_address, fp->next_hop); + else if (fp->afi == IP46_TYPE_IP4) + print (vam->ofp, + " weight %d, sw_if_index %d, is_local %d, is_drop %d, " + "is_unreach %d, is_prohitbit %d, afi %d, next_hop %U", + ntohl (fp->weight), ntohl (fp->sw_if_index), fp->is_local, + fp->is_drop, fp->is_unreach, fp->is_prohibit, fp->afi, + format_ip4_address, fp->next_hop); +} + +static void +vl_api_mpls_fib_path_json_print (vat_json_node_t * node, + vl_api_fib_path2_t * fp) +{ + struct in_addr ip4; + struct in6_addr ip6; + + vat_json_object_add_uint (node, "weight", ntohl (fp->weight)); + vat_json_object_add_uint (node, "sw_if_index", ntohl (fp->sw_if_index)); + vat_json_object_add_uint (node, "is_local", fp->is_local); + vat_json_object_add_uint (node, "is_drop", fp->is_drop); + vat_json_object_add_uint (node, "is_unreach", fp->is_unreach); + vat_json_object_add_uint (node, "is_prohibit", fp->is_prohibit); + vat_json_object_add_uint (node, "next_hop_afi", fp->afi); + if (fp->afi == IP46_TYPE_IP4) + { + clib_memcpy (&ip4, &fp->next_hop, sizeof (ip4)); + vat_json_object_add_ip4 (node, "next_hop", ip4); + } + else if (fp->afi == IP46_TYPE_IP6) + { + clib_memcpy (&ip6, &fp->next_hop, sizeof (ip6)); + vat_json_object_add_ip6 (node, "next_hop", ip6); + } +} + +static void +vl_api_mpls_tunnel_details_t_handler (vl_api_mpls_tunnel_details_t * mp) { vat_main_t *vam = &vat_main; - i32 len = mp->mt_next_hop_n_labels; + int count = ntohl (mp->mt_count); + vl_api_fib_path2_t *fp; i32 i; - print (vam->ofp, "[%d]: via %U %d labels ", - mp->tunnel_index, - format_ip4_address, mp->mt_next_hop, - ntohl (mp->mt_next_hop_sw_if_index)); - for (i = 0; i < len; i++) + print (vam->ofp, "[%d]: sw_if_index %d via:", + ntohl (mp->mt_tunnel_index), ntohl (mp->mt_sw_if_index)); + fp = mp->mt_paths; + for (i = 0; i < count; i++) { - print (vam->ofp, "%u ", ntohl (mp->mt_next_hop_out_labels[i])); + vl_api_mpls_fib_path_print (vam, fp); + fp++; } + print (vam->ofp, ""); } -static void vl_api_mpls_tunnel_details_t_handler_json - (vl_api_mpls_tunnel_details_t * mp) +#define vl_api_mpls_tunnel_details_t_endian vl_noop_handler +#define vl_api_mpls_tunnel_details_t_print vl_noop_handler + +static void +vl_api_mpls_tunnel_details_t_handler_json (vl_api_mpls_tunnel_details_t * mp) { vat_main_t *vam = &vat_main; vat_json_node_t *node = NULL; - struct in_addr ip4; + int count = ntohl (mp->mt_count); + vl_api_fib_path2_t *fp; i32 i; - i32 len = mp->mt_next_hop_n_labels; if (VAT_JSON_ARRAY != vam->json_tree.type) { @@ -16404,17 +16454,17 @@ static void vl_api_mpls_tunnel_details_t_handler_json node = vat_json_array_add (&vam->json_tree); vat_json_init_object (node); - vat_json_object_add_uint (node, "tunnel_index", ntohl (mp->tunnel_index)); - clib_memcpy (&ip4, &(mp->mt_next_hop), sizeof (ip4)); - vat_json_object_add_ip4 (node, "next_hop", ip4); - vat_json_object_add_uint (node, "next_hop_sw_if_index", - ntohl (mp->mt_next_hop_sw_if_index)); - vat_json_object_add_uint (node, "l2_only", ntohl (mp->mt_l2_only)); - vat_json_object_add_uint (node, "label_count", len); - for (i = 0; i < len; i++) + vat_json_object_add_uint (node, "tunnel_index", + ntohl (mp->mt_tunnel_index)); + vat_json_object_add_uint (node, "sw_if_index", ntohl (mp->mt_sw_if_index)); + + vat_json_object_add_uint (node, "l2_only", mp->mt_l2_only); + + fp = mp->mt_paths; + for (i = 0; i < count; i++) { - vat_json_object_add_uint (node, "label", - ntohl (mp->mt_next_hop_out_labels[i])); + vl_api_mpls_fib_path_json_print (node, fp); + fp++; } } @@ -16453,6 +16503,7 @@ api_mpls_tunnel_dump (vat_main_t * vam) #define vl_api_mpls_fib_details_t_endian vl_noop_handler #define vl_api_mpls_fib_details_t_print vl_noop_handler + static void vl_api_mpls_fib_details_t_handler (vl_api_mpls_fib_details_t * mp) { @@ -16467,20 +16518,7 @@ vl_api_mpls_fib_details_t_handler (vl_api_mpls_fib_details_t * mp) fp = mp->path; for (i = 0; i < count; i++) { - if (fp->afi == IP46_TYPE_IP6) - print (vam->ofp, - " weight %d, sw_if_index %d, is_local %d, is_drop %d, " - "is_unreach %d, is_prohitbit %d, afi %d, next_hop %U", - ntohl (fp->weight), ntohl (fp->sw_if_index), fp->is_local, - fp->is_drop, fp->is_unreach, fp->is_prohibit, fp->afi, - format_ip6_address, fp->next_hop); - else if (fp->afi == IP46_TYPE_IP4) - print (vam->ofp, - " weight %d, sw_if_index %d, is_local %d, is_drop %d, " - "is_unreach %d, is_prohitbit %d, afi %d, next_hop %U", - ntohl (fp->weight), ntohl (fp->sw_if_index), fp->is_local, - fp->is_drop, fp->is_unreach, fp->is_prohibit, fp->afi, - format_ip4_address, fp->next_hop); + vl_api_mpls_fib_path_print (vam, fp); fp++; } } @@ -16491,8 +16529,6 @@ static void vl_api_mpls_fib_details_t_handler_json vat_main_t *vam = &vat_main; int count = ntohl (mp->count); vat_json_node_t *node = NULL; - struct in_addr ip4; - struct in6_addr ip6; vl_api_fib_path2_t *fp; int i; @@ -16511,23 +16547,8 @@ static void vl_api_mpls_fib_details_t_handler_json fp = mp->path; for (i = 0; i < count; i++) { - vat_json_object_add_uint (node, "weight", ntohl (fp->weight)); - vat_json_object_add_uint (node, "sw_if_index", ntohl (fp->sw_if_index)); - vat_json_object_add_uint (node, "is_local", fp->is_local); - vat_json_object_add_uint (node, "is_drop", fp->is_drop); - vat_json_object_add_uint (node, "is_unreach", fp->is_unreach); - vat_json_object_add_uint (node, "is_prohibit", fp->is_prohibit); - vat_json_object_add_uint (node, "next_hop_afi", fp->afi); - if (fp->afi == IP46_TYPE_IP4) - { - clib_memcpy (&ip4, &fp->next_hop, sizeof (ip4)); - vat_json_object_add_ip4 (node, "next_hop", ip4); - } - else if (fp->afi == IP46_TYPE_IP6) - { - clib_memcpy (&ip6, &fp->next_hop, sizeof (ip6)); - vat_json_object_add_ip6 (node, "next_hop", ip6); - } + vl_api_mpls_fib_path_json_print (node, fp); + fp++; } } diff --git a/src/vnet.am b/src/vnet.am index 643ae92e..bed4902b 100644 --- a/src/vnet.am +++ b/src/vnet.am @@ -990,6 +990,8 @@ libvnet_la_SOURCES += \ vnet/dpo/lookup_dpo.c \ vnet/dpo/classify_dpo.c \ vnet/dpo/replicate_dpo.c \ + vnet/dpo/interface_dpo.c \ + vnet/dpo/mpls_disposition.c \ vnet/dpo/mpls_label_dpo.c nobase_include_HEADERS += \ diff --git a/src/vnet/adj/adj.c b/src/vnet/adj/adj.c index 90182006..36dfe500 100644 --- a/src/vnet/adj/adj.c +++ b/src/vnet/adj/adj.c @@ -67,6 +67,10 @@ adj_alloc (fib_protocol_t proto) adj->lookup_next_index = 0; adj->ia_delegates = NULL; + /* lest it become a midchain in the future */ + memset(&adj->sub_type.midchain.next_dpo, 0, + sizeof(adj->sub_type.midchain.next_dpo)); + ip4_main.lookup_main.adjacency_heap = adj_pool; ip6_main.lookup_main.adjacency_heap = adj_pool; @@ -118,6 +122,9 @@ format_ip_adjacency (u8 * s, va_list * args) case IP_LOOKUP_NEXT_MCAST: s = format (s, "%U", format_adj_mcast, adj_index, 0); break; + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: + s = format (s, "%U", format_adj_mcast_midchain, adj_index, 0); + break; default: break; } @@ -180,6 +187,7 @@ adj_last_lock_gone (ip_adjacency_t *adj) adj->rewrite_header.sw_if_index); break; case IP_LOOKUP_NEXT_MCAST: + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: adj_mcast_remove(adj->ia_nh_proto, adj->rewrite_header.sw_if_index); break; @@ -338,6 +346,7 @@ adj_walk (u32 sw_if_index, FOR_EACH_FIB_IP_PROTOCOL(proto) { adj_nbr_walk(sw_if_index, proto, cb, ctx); + adj_mcast_walk(sw_if_index, proto, cb, ctx); } } @@ -544,9 +553,9 @@ adj_show (vlib_main_t * vm, * [@0] * [@1] glean: loop0 * [@2] ipv4 via 1.0.0.2 loop0: IP4: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc - * [@3] mpls via 1.0.0.2 loop0: MPLS_UNICAST: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc + * [@3] mpls via 1.0.0.2 loop0: MPLS: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc * [@4] ipv4 via 1.0.0.3 loop0: IP4: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc - * [@5] mpls via 1.0.0.3 loop0: MPLS_UNICAST: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc + * [@5] mpls via 1.0.0.3 loop0: MPLS: 00:00:22:aa:bb:cc -> 00:00:11:aa:bb:cc * @cliexend ?*/ VLIB_CLI_COMMAND (adj_show_command, static) = { diff --git a/src/vnet/adj/adj.h b/src/vnet/adj/adj.h index 32997c91..ed5eb1f1 100644 --- a/src/vnet/adj/adj.h +++ b/src/vnet/adj/adj.h @@ -81,6 +81,10 @@ typedef enum /** Multicast Adjacency. */ IP_LOOKUP_NEXT_MCAST, + /** Multicast Midchain Adjacency. An Adjacency for sending macst packets + * on a tunnel/virtual interface */ + IP_LOOKUP_NEXT_MCAST_MIDCHAIN, + IP_LOOKUP_N_NEXT, } __attribute__ ((packed)) ip_lookup_next_t; @@ -107,6 +111,7 @@ typedef enum [IP_LOOKUP_NEXT_REWRITE] = "ip4-rewrite", \ [IP_LOOKUP_NEXT_MCAST] = "ip4-rewrite-mcast", \ [IP_LOOKUP_NEXT_MIDCHAIN] = "ip4-midchain", \ + [IP_LOOKUP_NEXT_MCAST_MIDCHAIN] = "ip4-mcast-midchain", \ [IP_LOOKUP_NEXT_ICMP_ERROR] = "ip4-icmp-error", \ } @@ -119,6 +124,7 @@ typedef enum [IP_LOOKUP_NEXT_REWRITE] = "ip6-rewrite", \ [IP_LOOKUP_NEXT_MCAST] = "ip6-rewrite-mcast", \ [IP_LOOKUP_NEXT_MIDCHAIN] = "ip6-midchain", \ + [IP_LOOKUP_NEXT_MCAST_MIDCHAIN] = "ip6-mcast-midchain", \ [IP_LOOKUP_NEXT_ICMP_ERROR] = "ip6-icmp-error", \ [IP6_LOOKUP_NEXT_HOP_BY_HOP] = "ip6-hop-by-hop", \ [IP6_LOOKUP_NEXT_ADD_HOP_BY_HOP] = "ip6-add-hop-by-hop", \ diff --git a/src/vnet/adj/adj_internal.h b/src/vnet/adj/adj_internal.h index 30668625..2c123c54 100644 --- a/src/vnet/adj/adj_internal.h +++ b/src/vnet/adj/adj_internal.h @@ -17,6 +17,7 @@ #define __ADJ_INTERNAL_H__ #include +#include #include #include #include @@ -87,11 +88,14 @@ adj_get_index (ip_adjacency_t *adj) return (adj - adj_pool); } -extern void adj_nbr_update_rewrite_internal (ip_adjacency_t *adj, - ip_lookup_next_t adj_next_index, - u32 complete_next_index, - u32 next_index, - u8 *rewrite); +extern void adj_nbr_update_rewrite_internal(ip_adjacency_t *adj, + ip_lookup_next_t adj_next_index, + u32 complete_next_index, + u32 next_index, + u8 *rewrite); +extern void adj_midchain_setup(adj_index_t adj_index, + adj_midchain_fixup_t fixup, + adj_flags_t flags); extern ip_adjacency_t * adj_alloc(fib_protocol_t proto); diff --git a/src/vnet/adj/adj_mcast.c b/src/vnet/adj/adj_mcast.c index 4f678e43..755abfd4 100644 --- a/src/vnet/adj/adj_mcast.c +++ b/src/vnet/adj/adj_mcast.c @@ -13,7 +13,7 @@ * limitations under the License. */ -#include +#include #include #include #include @@ -129,6 +129,59 @@ adj_mcast_update_rewrite (adj_index_t adj_index, adj->rewrite_header.dst_mcast_mask = clib_host_to_net_u32(mask); } +/** + * adj_mcast_midchain_update_rewrite + * + * Update the adjacency's rewrite string. A NULL string implies the + * rewirte is reset (i.e. when ARP/ND etnry is gone). + * NB: the adj being updated may be handling traffic in the DP. + */ +void +adj_mcast_midchain_update_rewrite (adj_index_t adj_index, + adj_midchain_fixup_t fixup, + adj_flags_t flags, + u8 *rewrite, + u8 offset, + u32 mask) +{ + ip_adjacency_t *adj; + + ASSERT(ADJ_INDEX_INVALID != adj_index); + + adj = adj_get(adj_index); + + /* + * one time only update. since we don't support chainging the tunnel + * src,dst, this is all we need. + */ + ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_MCAST); + /* + * tunnels can always provide a rewrite. + */ + ASSERT(NULL != rewrite); + + adj_midchain_setup(adj_index, fixup, flags); + + /* + * update the adj's rewrite string and build the arc + * from the rewrite node to the interface's TX node + */ + adj_nbr_update_rewrite_internal(adj, IP_LOOKUP_NEXT_MCAST_MIDCHAIN, + adj_get_mcast_node(adj->ia_nh_proto), + vnet_tx_node_index_for_sw_interface( + vnet_get_main(), + adj->rewrite_header.sw_if_index), + rewrite); + + /* + * set the fields corresponding to the mcast IP address rewrite + * The mask must be stored in network byte order, since the packet's + * IP address will also be in network order. + */ + adj->rewrite_header.dst_mcast_offset = offset; + adj->rewrite_header.dst_mcast_mask = clib_host_to_net_u32(mask); +} + void adj_mcast_remove (fib_protocol_t proto, u32 sw_if_index) @@ -260,6 +313,24 @@ adj_mcast_interface_delete (vnet_main_t * vnm, VNET_SW_INTERFACE_ADD_DEL_FUNCTION(adj_mcast_interface_delete); +/** + * @brief Walk the multicast Adjacencies on a given interface + */ +void +adj_mcast_walk (u32 sw_if_index, + fib_protocol_t proto, + adj_walk_cb_t cb, + void *ctx) +{ + if (vec_len(adj_mcasts[proto]) > sw_if_index) + { + if (ADJ_INDEX_INVALID != adj_mcasts[proto][sw_if_index]) + { + cb(adj_mcasts[proto][sw_if_index], ctx); + } + } +} + u8* format_adj_mcast (u8* s, va_list *ap) { @@ -269,6 +340,8 @@ format_adj_mcast (u8* s, va_list *ap) s = format(s, "%U-mcast: ", format_fib_protocol, adj->ia_nh_proto); + if (adj->rewrite_header.flags & VNET_REWRITE_HAS_FEATURES) + s = format(s, "[features] "); s = format (s, "%U", format_vnet_rewrite, &adj->rewrite_header, sizeof (adj->rewrite_data), 0); @@ -276,6 +349,28 @@ format_adj_mcast (u8* s, va_list *ap) return (s); } +u8* +format_adj_mcast_midchain (u8* s, va_list *ap) +{ + index_t index = va_arg(*ap, index_t); + CLIB_UNUSED(u32 indent) = va_arg(*ap, u32); + vnet_main_t * vnm = vnet_get_main(); + ip_adjacency_t * adj = adj_get(index); + + s = format(s, "%U-mcast-midchain: ", + format_fib_protocol, adj->ia_nh_proto); + s = format (s, "%U", + format_vnet_rewrite, + vnm->vlib_main, &adj->rewrite_header, + sizeof (adj->rewrite_data), 0); + s = format (s, "\n%Ustacked-on:\n%U%U", + format_white_space, indent, + format_white_space, indent+2, + format_dpo_id, &adj->sub_type.midchain.next_dpo, indent+2); + + return (s); +} + static void adj_dpo_lock (dpo_id_t *dpo) @@ -293,6 +388,11 @@ const static dpo_vft_t adj_mcast_dpo_vft = { .dv_unlock = adj_dpo_unlock, .dv_format = format_adj_mcast, }; +const static dpo_vft_t adj_mcast_midchain_dpo_vft = { + .dv_lock = adj_dpo_lock, + .dv_unlock = adj_dpo_unlock, + .dv_format = format_adj_mcast_midchain, +}; /** * @brief The per-protocol VLIB graph nodes that are assigned to a mcast @@ -319,6 +419,31 @@ const static char* const * const adj_mcast_nodes[DPO_PROTO_NUM] = [DPO_PROTO_MPLS] = NULL, }; +/** + * @brief The per-protocol VLIB graph nodes that are assigned to a mcast + * object. + * + * this means that these graph nodes are ones from which a mcast is the + * parent object in the DPO-graph. + */ +const static char* const adj_mcast_midchain_ip4_nodes[] = +{ + "ip4-mcast-midchain", + NULL, +}; +const static char* const adj_mcast_midchain_ip6_nodes[] = +{ + "ip6-mcast-midchain", + NULL, +}; + +const static char* const * const adj_mcast_midchain_nodes[DPO_PROTO_NUM] = +{ + [DPO_PROTO_IP4] = adj_mcast_midchain_ip4_nodes, + [DPO_PROTO_IP6] = adj_mcast_midchain_ip6_nodes, + [DPO_PROTO_MPLS] = NULL, +}; + /** * @brief Return the size of the adj DB. * This is only for testing purposes so an efficient implementation is not needed @@ -349,5 +474,10 @@ adj_mcast_db_size (void) void adj_mcast_module_init (void) { - dpo_register(DPO_ADJACENCY_MCAST, &adj_mcast_dpo_vft, adj_mcast_nodes); + dpo_register(DPO_ADJACENCY_MCAST, + &adj_mcast_dpo_vft, + adj_mcast_nodes); + dpo_register(DPO_ADJACENCY_MCAST_MIDCHAIN, + &adj_mcast_midchain_dpo_vft, + adj_mcast_midchain_nodes); } diff --git a/src/vnet/adj/adj_mcast.h b/src/vnet/adj/adj_mcast.h index 40d44313..bfb0d6f6 100644 --- a/src/vnet/adj/adj_mcast.h +++ b/src/vnet/adj/adj_mcast.h @@ -26,6 +26,7 @@ #define __ADJ_MCAST_H__ #include +#include /** * @brief @@ -68,10 +69,36 @@ extern void adj_mcast_update_rewrite(adj_index_t adj_index, u8 offset, u32 mask); +/** + * @brief + * Update the rewrite string for an existing adjacecny and + * Convert the adjacency into a midchain + * + * @param + * The index of the adj to update + * + * @param + * The new rewrite + */ +extern void adj_mcast_midchain_update_rewrite(adj_index_t adj_index, + adj_midchain_fixup_t fixup, + adj_flags_t flags, + u8 *rewrite, + u8 offset, + u32 mask); +/** + * @brief Walk the multicast Adjacencies on a given interface + */ +extern void adj_mcast_walk (u32 sw_if_index, + fib_protocol_t adj_nh_proto, + adj_walk_cb_t cb, + void *ctx); + /** * @brief Format/display a mcast adjacency. */ extern u8* format_adj_mcast(u8* s, va_list *ap); +extern u8* format_adj_mcast_midchain(u8* s, va_list *ap); /** * @brief Get the sze of the mcast adj DB. Test purposes only. diff --git a/src/vnet/adj/adj_midchain.c b/src/vnet/adj/adj_midchain.c index 5756de43..a93a1c3e 100644 --- a/src/vnet/adj/adj_midchain.c +++ b/src/vnet/adj/adj_midchain.c @@ -346,7 +346,7 @@ adj_get_midchain_node (vnet_link_t link) static u8 adj_midchain_get_feature_arc_index_for_link_type (const ip_adjacency_t *adj) { - u8 arc = (u8) ~0; + u8 arc = (u8) ~0; switch (adj->ia_link) { case VNET_LINK_IP4: @@ -393,17 +393,14 @@ adj_nbr_midchain_get_tx_node (ip_adjacency_t *adj) } /** - * adj_nbr_midchain_update_rewrite + * adj_midchain_setup * - * Update the adjacency's rewrite string. A NULL string implies the - * rewrite is reset (i.e. when ARP/ND etnry is gone). - * NB: the adj being updated may be handling traffic in the DP. + * Setup the adj as a mid-chain */ void -adj_nbr_midchain_update_rewrite (adj_index_t adj_index, - adj_midchain_fixup_t fixup, - adj_flags_t flags, - u8 *rewrite) +adj_midchain_setup (adj_index_t adj_index, + adj_midchain_fixup_t fixup, + adj_flags_t flags) { u32 feature_index, tx_node; ip_adjacency_t *adj; @@ -413,16 +410,6 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index, adj = adj_get(adj_index); - /* - * one time only update. since we don't support chainging the tunnel - * src,dst, this is all we need. - */ - ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP); - /* - * tunnels can always provide a rewrite. - */ - ASSERT(NULL != rewrite); - adj->sub_type.midchain.fixup_func = fixup; adj->ia_flags |= flags; @@ -447,6 +434,38 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index, dpo_stack_from_node(tx_node, &adj->sub_type.midchain.next_dpo, drop_dpo_get(vnet_link_to_dpo_proto(adj->ia_link))); +} + +/** + * adj_nbr_midchain_update_rewrite + * + * Update the adjacency's rewrite string. A NULL string implies the + * rewrite is reset (i.e. when ARP/ND etnry is gone). + * NB: the adj being updated may be handling traffic in the DP. + */ +void +adj_nbr_midchain_update_rewrite (adj_index_t adj_index, + adj_midchain_fixup_t fixup, + adj_flags_t flags, + u8 *rewrite) +{ + ip_adjacency_t *adj; + + ASSERT(ADJ_INDEX_INVALID != adj_index); + + adj = adj_get(adj_index); + + /* + * one time only update. since we don't support chainging the tunnel + * src,dst, this is all we need. + */ + ASSERT(adj->lookup_next_index == IP_LOOKUP_NEXT_ARP); + /* + * tunnels can always provide a rewrite. + */ + ASSERT(NULL != rewrite); + + adj_midchain_setup(adj_index, fixup, flags); /* * update the rewirte with the workers paused. @@ -454,7 +473,7 @@ adj_nbr_midchain_update_rewrite (adj_index_t adj_index, adj_nbr_update_rewrite_internal(adj, IP_LOOKUP_NEXT_MIDCHAIN, adj_get_midchain_node(adj->ia_link), - tx_node, + adj_nbr_midchain_get_tx_node(adj), rewrite); } @@ -496,7 +515,8 @@ adj_nbr_midchain_stack (adj_index_t adj_index, adj = adj_get(adj_index); - ASSERT(IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index); + ASSERT((IP_LOOKUP_NEXT_MIDCHAIN == adj->lookup_next_index) || + (IP_LOOKUP_NEXT_MCAST_MIDCHAIN == adj->lookup_next_index)); dpo_stack_from_node(adj_nbr_midchain_get_tx_node(adj), &adj->sub_type.midchain.next_dpo, diff --git a/src/vnet/adj/adj_nbr.c b/src/vnet/adj/adj_nbr.c index ddacb030..3d450d1f 100644 --- a/src/vnet/adj/adj_nbr.c +++ b/src/vnet/adj/adj_nbr.c @@ -195,8 +195,6 @@ adj_nbr_alloc (fib_protocol_t nh_proto, adj->ia_link = link_type; adj->ia_nh_proto = nh_proto; adj->rewrite_header.sw_if_index = sw_if_index; - memset(&adj->sub_type.midchain.next_dpo, 0, - sizeof(adj->sub_type.midchain.next_dpo)); adj_nbr_evaluate_feature (adj_get_index(adj)); return (adj); diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h index ea3ce093..ed869d1f 100644 --- a/src/vnet/buffer.h +++ b/src/vnet/buffer.h @@ -130,6 +130,9 @@ typedef struct /* Rewrite length */ u32 save_rewrite_length; + + /* MFIB RPF ID */ + u32 rpf_id; }; /* ICMP */ diff --git a/src/vnet/devices/ssvm/node.c b/src/vnet/devices/ssvm/node.c index 539b4161..b7a8db05 100644 --- a/src/vnet/devices/ssvm/node.c +++ b/src/vnet/devices/ssvm/node.c @@ -210,7 +210,7 @@ ssvm_eth_device_input (ssvm_eth_main_t * em, next0 = SSVM_ETH_INPUT_NEXT_IP4_INPUT; else if (type0 == ETHERNET_TYPE_IP6) next0 = SSVM_ETH_INPUT_NEXT_IP6_INPUT; - else if (type0 == ETHERNET_TYPE_MPLS_UNICAST) + else if (type0 == ETHERNET_TYPE_MPLS) next0 = SSVM_ETH_INPUT_NEXT_MPLS_INPUT; l3_offset0 = ((next0 == SSVM_ETH_INPUT_NEXT_IP4_INPUT || diff --git a/src/vnet/dhcp/dhcp6_proxy_node.c b/src/vnet/dhcp/dhcp6_proxy_node.c index 524cb095..de73154d 100644 --- a/src/vnet/dhcp/dhcp6_proxy_node.c +++ b/src/vnet/dhcp/dhcp6_proxy_node.c @@ -883,6 +883,7 @@ dhcp6_proxy_set_server (ip46_address_t *addr, mfib_table_entry_update(rx_fib_index, &all_dhcp_servers, MFIB_SOURCE_DHCP, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_ACCEPT_ALL_ITF); mfib_table_lock(rx_fib_index, FIB_PROTOCOL_IP6); } diff --git a/src/vnet/dpo/dpo.c b/src/vnet/dpo/dpo.c index d8e075a7..dfc2bd92 100644 --- a/src/vnet/dpo/dpo.c +++ b/src/vnet/dpo/dpo.c @@ -37,6 +37,8 @@ #include #include #include +#include +#include /** * Array of char* names for the DPO types and protos @@ -182,6 +184,12 @@ dpo_set (dpo_id_t *dpo, case IP_LOOKUP_NEXT_MIDCHAIN: dpo->dpoi_type = DPO_ADJACENCY_MIDCHAIN; break; + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: + dpo->dpoi_type = DPO_ADJACENCY_MCAST_MIDCHAIN; + break; + case IP_LOOKUP_NEXT_MCAST: + dpo->dpoi_type = DPO_ADJACENCY_MCAST; + break; default: break; } @@ -453,6 +461,8 @@ dpo_module_init (vlib_main_t * vm) lookup_dpo_module_init(); ip_null_dpo_module_init(); replicate_module_init(); + interface_dpo_module_init(); + mpls_disp_dpo_module_init(); return (NULL); } diff --git a/src/vnet/dpo/dpo.h b/src/vnet/dpo/dpo.h index 48b92d3d..5aa4e2d2 100644 --- a/src/vnet/dpo/dpo.h +++ b/src/vnet/dpo/dpo.h @@ -108,12 +108,15 @@ typedef enum dpo_type_t_ { DPO_ADJACENCY_MIDCHAIN, DPO_ADJACENCY_GLEAN, DPO_ADJACENCY_MCAST, + DPO_ADJACENCY_MCAST_MIDCHAIN, DPO_RECEIVE, DPO_LOOKUP, DPO_LISP_CP, DPO_CLASSIFY, DPO_MPLS_LABEL, + DPO_MPLS_DISPOSITION, DPO_MFIB_ENTRY, + DPO_INTERFACE, DPO_LAST, } __attribute__((packed)) dpo_type_t; @@ -129,6 +132,7 @@ typedef enum dpo_type_t_ { [DPO_ADJACENCY_MIDCHAIN] = "dpo-adjacency-midcahin", \ [DPO_ADJACENCY_GLEAN] = "dpo-glean", \ [DPO_ADJACENCY_MCAST] = "dpo-adj-mcast", \ + [DPO_ADJACENCY_MCAST_MIDCHAIN] = "dpo-adj-mcast-midchain", \ [DPO_RECEIVE] = "dpo-receive", \ [DPO_LOOKUP] = "dpo-lookup", \ [DPO_LOAD_BALANCE] = "dpo-load-balance", \ @@ -136,7 +140,9 @@ typedef enum dpo_type_t_ { [DPO_LISP_CP] = "dpo-lisp-cp", \ [DPO_CLASSIFY] = "dpo-classify", \ [DPO_MPLS_LABEL] = "dpo-mpls-label", \ - [DPO_MFIB_ENTRY] = "dpo-mfib_entry" \ + [DPO_MPLS_DISPOSITION] = "dpo-mpls-diposition", \ + [DPO_MFIB_ENTRY] = "dpo-mfib_entry", \ + [DPO_INTERFACE] = "dpo-interface" \ } /** diff --git a/src/vnet/dpo/interface_dpo.c b/src/vnet/dpo/interface_dpo.c new file mode 100644 index 00000000..50ca756f --- /dev/null +++ b/src/vnet/dpo/interface_dpo.c @@ -0,0 +1,416 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +/* + * The 'DB' of interface DPOs. + * There is only one per-interface per-protocol, so this is a per-interface + * vector + */ +static index_t *interface_dpo_db[DPO_PROTO_NUM]; + +static interface_dpo_t * +interface_dpo_alloc (void) +{ + interface_dpo_t *ido; + + pool_get(interface_dpo_pool, ido); + + return (ido); +} + +static inline interface_dpo_t * +interface_dpo_get_from_dpo (const dpo_id_t *dpo) +{ + ASSERT(DPO_INTERFACE == dpo->dpoi_type); + + return (interface_dpo_get(dpo->dpoi_index)); +} + +static inline index_t +interface_dpo_get_index (interface_dpo_t *ido) +{ + return (ido - interface_dpo_pool); +} + +static void +interface_dpo_lock (dpo_id_t *dpo) +{ + interface_dpo_t *ido; + + ido = interface_dpo_get_from_dpo(dpo); + ido->ido_locks++; +} + +static void +interface_dpo_unlock (dpo_id_t *dpo) +{ + interface_dpo_t *ido; + + ido = interface_dpo_get_from_dpo(dpo); + ido->ido_locks--; + + if (0 == ido->ido_locks) + { + interface_dpo_db[ido->ido_proto][ido->ido_sw_if_index] = + INDEX_INVALID; + pool_put(interface_dpo_pool, ido); + } +} + +/* + * interface_dpo_add_or_lock + * + * Add/create and lock a new or lock an existing for the interface DPO + * on the interface and protocol given + */ +void +interface_dpo_add_or_lock (dpo_proto_t proto, + u32 sw_if_index, + dpo_id_t *dpo) +{ + interface_dpo_t *ido; + + vec_validate_init_empty(interface_dpo_db[proto], + sw_if_index, + INDEX_INVALID); + + if (INDEX_INVALID == interface_dpo_db[proto][sw_if_index]) + { + ido = interface_dpo_alloc(); + + ido->ido_sw_if_index = sw_if_index; + ido->ido_proto = proto; + + interface_dpo_db[proto][sw_if_index] = + interface_dpo_get_index(ido); + } + else + { + ido = interface_dpo_get(interface_dpo_db[proto][sw_if_index]); + } + + dpo_set(dpo, DPO_INTERFACE, proto, interface_dpo_get_index(ido)); +} + + +static clib_error_t * +interface_dpo_interface_state_change (vnet_main_t * vnm, + u32 sw_if_index, + u32 flags) +{ + /* + */ + return (NULL); +} + +VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION( + interface_dpo_interface_state_change); + +/** + * @brief Registered callback for HW interface state changes + */ +static clib_error_t * +interface_dpo_hw_interface_state_change (vnet_main_t * vnm, + u32 hw_if_index, + u32 flags) +{ + return (NULL); +} + +VNET_HW_INTERFACE_LINK_UP_DOWN_FUNCTION( + interface_dpo_hw_interface_state_change); + +static clib_error_t * +interface_dpo_interface_delete (vnet_main_t * vnm, + u32 sw_if_index, + u32 is_add) +{ + return (NULL); +} + +VNET_SW_INTERFACE_ADD_DEL_FUNCTION( + interface_dpo_interface_delete); + +u8* +format_interface_dpo (u8* s, va_list *ap) +{ + index_t index = va_arg(*ap, index_t); + CLIB_UNUSED(u32 indent) = va_arg(*ap, u32); + vnet_main_t * vnm = vnet_get_main(); + interface_dpo_t *ido = interface_dpo_get(index); + + return (format(s, "%U-dpo: %U", + format_vnet_sw_interface_name, + vnm, + vnet_get_sw_interface(vnm, ido->ido_sw_if_index), + format_dpo_proto, ido->ido_proto)); +} + +static void +interface_dpo_mem_show (void) +{ + fib_show_memory_usage("Interface", + pool_elts(interface_dpo_pool), + pool_len(interface_dpo_pool), + sizeof(interface_dpo_t)); +} + + +const static dpo_vft_t interface_dpo_vft = { + .dv_lock = interface_dpo_lock, + .dv_unlock = interface_dpo_unlock, + .dv_format = format_interface_dpo, + .dv_mem_show = interface_dpo_mem_show, +}; + +/** + * @brief The per-protocol VLIB graph nodes that are assigned to a glean + * object. + * + * this means that these graph nodes are ones from which a glean is the + * parent object in the DPO-graph. + */ +const static char* const interface_dpo_ip4_nodes[] = +{ + "interface-dpo-ip4", + NULL, +}; +const static char* const interface_dpo_ip6_nodes[] = +{ + "interface-dpo-ip4", + NULL, +}; + +const static char* const * const interface_dpo_nodes[DPO_PROTO_NUM] = +{ + [DPO_PROTO_IP4] = interface_dpo_ip4_nodes, + [DPO_PROTO_IP6] = interface_dpo_ip6_nodes, + [DPO_PROTO_MPLS] = NULL, +}; + +void +interface_dpo_module_init (void) +{ + dpo_register(DPO_INTERFACE, + &interface_dpo_vft, + interface_dpo_nodes); +} + +/** + * @brief Interface DPO trace data + */ +typedef struct interface_dpo_trace_t_ +{ + u32 sw_if_index; +} interface_dpo_trace_t; + +typedef enum interface_dpo_next_t_ +{ + INTERFACE_DPO_DROP = 0, + INTERFACE_DPO_INPUT = 1, +} interface_dpo_next_t; + +always_inline uword +interface_dpo_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + u32 n_left_from, next_index, * from, * to_next; + u32 cpu_index = os_get_cpu_number(); + vnet_interface_main_t *im; + + im = &vnet_get_main ()->interface_main; + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next > 2) + { + const interface_dpo_t *ido0, *ido1; + u32 bi0, idoi0, bi1, idoi1; + vlib_buffer_t *b0, *b1; + + bi0 = from[0]; + to_next[0] = bi0; + bi1 = from[1]; + to_next[1] = bi1; + from += 2; + to_next += 2; + n_left_from -= 2; + n_left_to_next -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + idoi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX]; + idoi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX]; + ido0 = interface_dpo_get(idoi0); + ido1 = interface_dpo_get(idoi1); + + vnet_buffer(b0)->sw_if_index[VLIB_RX] = ido0->ido_sw_if_index; + vnet_buffer(b1)->sw_if_index[VLIB_RX] = ido1->ido_sw_if_index; + + vlib_increment_combined_counter (im->combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + cpu_index, + ido0->ido_sw_if_index, + 1, + vlib_buffer_length_in_chain (vm, b0)); + vlib_increment_combined_counter (im->combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + cpu_index, + ido1->ido_sw_if_index, + 1, + vlib_buffer_length_in_chain (vm, b1)); + + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + interface_dpo_trace_t *tr0; + + tr0 = vlib_add_trace (vm, node, b0, sizeof (*tr0)); + tr0->sw_if_index = ido0->ido_sw_if_index; + } + if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED)) + { + interface_dpo_trace_t *tr1; + + tr1 = vlib_add_trace (vm, node, b1, sizeof (*tr1)); + tr1->sw_if_index = ido1->ido_sw_if_index; + } + + vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, + n_left_to_next, bi0, bi1, + INTERFACE_DPO_INPUT, + INTERFACE_DPO_INPUT); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + const interface_dpo_t * ido0; + vlib_buffer_t * b0; + u32 bi0, idoi0; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + idoi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX]; + ido0 = interface_dpo_get(idoi0); + + /* Swap the RX interface of the packet to the one the + * interface DPR represents */ + vnet_buffer(b0)->sw_if_index[VLIB_RX] = ido0->ido_sw_if_index; + + /* Bump the interface's RX coutners */ + vlib_increment_combined_counter (im->combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + cpu_index, + ido0->ido_sw_if_index, + 1, + vlib_buffer_length_in_chain (vm, b0)); + + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + interface_dpo_trace_t *tr; + + tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); + tr->sw_if_index = ido0->ido_sw_if_index; + } + + vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, + n_left_to_next, bi0, + INTERFACE_DPO_INPUT); + } + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + return from_frame->n_vectors; +} + +static u8 * +format_interface_dpo_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + interface_dpo_trace_t * t = va_arg (*args, interface_dpo_trace_t *); + uword indent = format_get_indent (s); + s = format (s, "%U sw_if_index:%d", + format_white_space, indent, + t->sw_if_index); + return s; +} + +static uword +interface_dpo_ip4 (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return (interface_dpo_inline(vm, node, from_frame)); +} + +static uword +interface_dpo_ip6 (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return (interface_dpo_inline(vm, node, from_frame)); +} + +VLIB_REGISTER_NODE (interface_dpo_ip4_node) = { + .function = interface_dpo_ip4, + .name = "interface-dpo-ip4", + .vector_size = sizeof (u32), + .format_trace = format_interface_dpo_trace, + + .n_next_nodes = 2, + .next_nodes = { + [INTERFACE_DPO_DROP] = "ip4-drop", + [INTERFACE_DPO_INPUT] = "ip4-input", + }, +}; + +VLIB_NODE_FUNCTION_MULTIARCH (interface_dpo_ip4_node, + interface_dpo_ip4) + +VLIB_REGISTER_NODE (interface_dpo_ip6_node) = { + .function = interface_dpo_ip6, + .name = "interface-dpo-ip6", + .vector_size = sizeof (u32), + .format_trace = format_interface_dpo_trace, + + .n_next_nodes = 2, + .next_nodes = { + [INTERFACE_DPO_DROP] = "ip6-drop", + [INTERFACE_DPO_INPUT] = "ip6-input", + }, +}; + +VLIB_NODE_FUNCTION_MULTIARCH (interface_dpo_ip6_node, + interface_dpo_ip6) + diff --git a/src/vnet/dpo/interface_dpo.h b/src/vnet/dpo/interface_dpo.h new file mode 100644 index 00000000..1538dfbb --- /dev/null +++ b/src/vnet/dpo/interface_dpo.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * @brief + * The data-path object representing interfaceing the packet, i.e. it's for-us + */ + +#ifndef __INTERFACE_DPO_H__ +#define __INTERFACE_DPO_H__ + +#include + +typedef struct interface_dpo_t_ +{ + /** + * The Software interface index that the packets will be given + * as the ingress/rx interface + */ + u32 ido_sw_if_index; + + /** + * next VLIB node. A '-input' node. + */ + u32 ido_next_node; + + /** + * DPO protocol that the packets will have as they 'ingress' + * on this interface + */ + dpo_proto_t ido_proto; + + /** + * number of locks. + */ + u16 ido_locks; +} interface_dpo_t; + +extern void interface_dpo_add_or_lock (dpo_proto_t proto, + u32 sw_if_index, + dpo_id_t *dpo); + +extern void interface_dpo_module_init(void); + +/** + * @brief pool of all interface DPOs + */ +interface_dpo_t *interface_dpo_pool; + +static inline interface_dpo_t * +interface_dpo_get (index_t index) +{ + return (pool_elt_at_index(interface_dpo_pool, index)); +} + +#endif diff --git a/src/vnet/dpo/lookup_dpo.c b/src/vnet/dpo/lookup_dpo.c index 97ad0a44..e5b00a79 100644 --- a/src/vnet/dpo/lookup_dpo.c +++ b/src/vnet/dpo/lookup_dpo.c @@ -21,8 +21,12 @@ #include #include #include +#include +#include +#include static const char *const lookup_input_names[] = LOOKUP_INPUTS; +static const char *const lookup_cast_names[] = LOOKUP_CASTS; /** * @brief Enumeration of the lookup subtypes @@ -31,6 +35,7 @@ typedef enum lookup_sub_type_t_ { LOOKUP_SUB_TYPE_SRC, LOOKUP_SUB_TYPE_DST, + LOOKUP_SUB_TYPE_DST_MCAST, LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE, } lookup_sub_type_t; #define LOOKUP_SUB_TYPE_NUM (LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE+1) @@ -67,6 +72,7 @@ lookup_dpo_get_index (lookup_dpo_t *lkd) static void lookup_dpo_add_or_lock_i (fib_node_index_t fib_index, dpo_proto_t proto, + lookup_cast_t cast, lookup_input_t input, lookup_table_t table_config, dpo_id_t *dpo) @@ -79,6 +85,7 @@ lookup_dpo_add_or_lock_i (fib_node_index_t fib_index, lkd->lkd_proto = proto; lkd->lkd_input = input; lkd->lkd_table = table_config; + lkd->lkd_cast = cast; /* * use the input type to select the lookup sub-type @@ -100,6 +107,10 @@ lookup_dpo_add_or_lock_i (fib_node_index_t fib_index, type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST]; break; } + if (LOOKUP_MULTICAST == cast) + { + type = lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST]; + } } if (0 == type) @@ -115,20 +126,29 @@ lookup_dpo_add_or_lock_i (fib_node_index_t fib_index, void lookup_dpo_add_or_lock_w_fib_index (fib_node_index_t fib_index, dpo_proto_t proto, + lookup_cast_t cast, lookup_input_t input, lookup_table_t table_config, dpo_id_t *dpo) { if (LOOKUP_TABLE_FROM_CONFIG == table_config) { - fib_table_lock(fib_index, dpo_proto_to_fib(proto)); + if (LOOKUP_UNICAST == cast) + { + fib_table_lock(fib_index, dpo_proto_to_fib(proto)); + } + else + { + mfib_table_lock(fib_index, dpo_proto_to_fib(proto)); + } } - lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo); + lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo); } void lookup_dpo_add_or_lock_w_table_id (u32 table_id, dpo_proto_t proto, + lookup_cast_t cast, lookup_input_t input, lookup_table_t table_config, dpo_id_t *dpo) @@ -137,13 +157,22 @@ lookup_dpo_add_or_lock_w_table_id (u32 table_id, if (LOOKUP_TABLE_FROM_CONFIG == table_config) { - fib_index = - fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto), - table_id); + if (LOOKUP_UNICAST == cast) + { + fib_index = + fib_table_find_or_create_and_lock(dpo_proto_to_fib(proto), + table_id); + } + else + { + fib_index = + mfib_table_find_or_create_and_lock(dpo_proto_to_fib(proto), + table_id); + } } ASSERT(FIB_NODE_INDEX_INVALID != fib_index); - lookup_dpo_add_or_lock_i(fib_index, proto, input, table_config, dpo); + lookup_dpo_add_or_lock_i(fib_index, proto, cast, input, table_config, dpo); } u8* @@ -156,16 +185,29 @@ format_lookup_dpo (u8 *s, va_list *args) if (LOOKUP_TABLE_FROM_INPUT_INTERFACE == lkd->lkd_table) { - s = format(s, "%s lookup in interface's %U table", + s = format(s, "%s,%s lookup in interface's %U table", lookup_input_names[lkd->lkd_input], + lookup_cast_names[lkd->lkd_cast], format_dpo_proto, lkd->lkd_proto); } else { - s = format(s, "%s lookup in %U", - lookup_input_names[lkd->lkd_input], - format_fib_table_name, lkd->lkd_fib_index, - dpo_proto_to_fib(lkd->lkd_proto)); + if (LOOKUP_UNICAST == lkd->lkd_cast) + { + s = format(s, "%s,%s lookup in %U", + lookup_input_names[lkd->lkd_input], + lookup_cast_names[lkd->lkd_cast], + format_fib_table_name, lkd->lkd_fib_index, + dpo_proto_to_fib(lkd->lkd_proto)); + } + else + { + s = format(s, "%s,%s lookup in %U", + lookup_input_names[lkd->lkd_input], + lookup_cast_names[lkd->lkd_cast], + format_mfib_table_name, lkd->lkd_fib_index, + dpo_proto_to_fib(lkd->lkd_proto)); + } } return (s); } @@ -193,8 +235,16 @@ lookup_dpo_unlock (dpo_id_t *dpo) { if (LOOKUP_TABLE_FROM_CONFIG == lkd->lkd_table) { - fib_table_unlock(lkd->lkd_fib_index, - dpo_proto_to_fib(lkd->lkd_proto)); + if (LOOKUP_UNICAST == lkd->lkd_cast) + { + fib_table_unlock(lkd->lkd_fib_index, + dpo_proto_to_fib(lkd->lkd_proto)); + } + else + { + mfib_table_unlock(lkd->lkd_fib_index, + dpo_proto_to_fib(lkd->lkd_proto)); + } } pool_put(lookup_dpo_pool, lkd); } @@ -1069,6 +1119,123 @@ VLIB_REGISTER_NODE (lookup_mpls_dst_itf_node) = { }; VLIB_NODE_FUNCTION_MULTIARCH (lookup_mpls_dst_itf_node, lookup_mpls_dst_itf) +typedef enum lookup_ip_dst_mcast_next_t_ { + LOOKUP_IP_DST_MCAST_NEXT_RPF, + LOOKUP_IP_DST_MCAST_N_NEXT, +} mfib_forward_lookup_next_t; + +always_inline uword +lookup_dpo_ip_dst_mcast_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame, + int is_v4) +{ + u32 n_left_from, next_index, * from, * to_next; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = LOOKUP_IP_DST_MCAST_NEXT_RPF; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next); + + /* while (n_left_from >= 4 && n_left_to_next >= 2) */ + /* } */ + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0, lkdi0, fib_index0, next0; + const lookup_dpo_t * lkd0; + fib_node_index_t mfei0; + vlib_buffer_t * b0; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + /* dst lookup was done by mpls lookup */ + lkdi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX]; + lkd0 = lookup_dpo_get(lkdi0); + fib_index0 = lkd0->lkd_fib_index; + next0 = LOOKUP_IP_DST_MCAST_NEXT_RPF; + + if (is_v4) + { + ip4_header_t * ip0; + + ip0 = vlib_buffer_get_current (b0); + mfei0 = ip4_mfib_table_lookup(ip4_mfib_get(fib_index0), + &ip0->src_address, + &ip0->dst_address, + 64); + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + lookup_trace_t *tr = vlib_add_trace (vm, node, + b0, sizeof (*tr)); + tr->fib_index = fib_index0; + tr->lbi = mfei0; + tr->addr.ip4 = ip0->dst_address; + } + } + else + { + ip6_header_t * ip0; + + ip0 = vlib_buffer_get_current (b0); + mfei0 = ip6_mfib_table_lookup2(ip6_mfib_get(fib_index0), + &ip0->src_address, + &ip0->dst_address); + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + lookup_trace_t *tr = vlib_add_trace (vm, node, + b0, sizeof (*tr)); + tr->fib_index = fib_index0; + tr->lbi = mfei0; + tr->addr.ip6 = ip0->dst_address; + } + } + + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = mfei0; + + vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + return from_frame->n_vectors; +} + +always_inline uword +lookup_ip4_dst_mcast (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame) +{ + return (lookup_dpo_ip_dst_mcast_inline(vm, node, from_frame, 1)); +} + +VLIB_REGISTER_NODE (lookup_ip4_dst_mcast_node) = { + .function = lookup_ip4_dst_mcast, + .name = "lookup-ip4-dst-mcast", + .vector_size = sizeof (u32), + + .format_trace = format_lookup_trace, + .n_next_nodes = LOOKUP_IP_DST_MCAST_N_NEXT, + .next_nodes = { + [LOOKUP_IP_DST_MCAST_NEXT_RPF] = "ip4-mfib-forward-rpf", + }, +}; +VLIB_NODE_FUNCTION_MULTIARCH (lookup_ip4_dst_mcast_node, + lookup_ip4_dst_mcast) + static void lookup_dpo_mem_show (void) { @@ -1129,6 +1296,22 @@ const static char* const * const lookup_dst_nodes[DPO_PROTO_NUM] = [DPO_PROTO_MPLS] = lookup_dst_mpls_nodes, }; +const static char* const lookup_dst_mcast_ip4_nodes[] = +{ + "lookup-ip4-dst-mcast", + NULL, +}; +const static char* const lookup_dst_mcast_ip6_nodes[] = +{ + "lookup-ip6-dst-mcast", + NULL, +}; +const static char* const * const lookup_dst_mcast_nodes[DPO_PROTO_NUM] = +{ + [DPO_PROTO_IP4] = lookup_dst_mcast_ip4_nodes, + [DPO_PROTO_IP6] = lookup_dst_mcast_ip6_nodes, +}; + const static char* const lookup_dst_from_interface_ip4_nodes[] = { "lookup-ip4-dst-itf", @@ -1168,6 +1351,8 @@ lookup_dpo_module_init (void) dpo_register_new_type(&lkd_vft, lookup_src_nodes); lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST] = dpo_register_new_type(&lkd_vft, lookup_dst_nodes); + lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_MCAST] = + dpo_register_new_type(&lkd_vft, lookup_dst_mcast_nodes); lookup_dpo_sub_types[LOOKUP_SUB_TYPE_DST_TABLE_FROM_INTERFACE] = dpo_register_new_type(&lkd_vft, lookup_dst_from_interface_nodes); } diff --git a/src/vnet/dpo/lookup_dpo.h b/src/vnet/dpo/lookup_dpo.h index ff283388..7dfd0385 100644 --- a/src/vnet/dpo/lookup_dpo.h +++ b/src/vnet/dpo/lookup_dpo.h @@ -46,6 +46,19 @@ typedef enum lookup_table_t_ { [LOOKUP_INPUT_DST_ADDR] = "table-configured", \ } +/** + * Switch to use the packet's source or destination address for lookup + */ +typedef enum lookup_cast_t_ { + LOOKUP_UNICAST, + LOOKUP_MULTICAST, +} __attribute__ ((packed)) lookup_cast_t; + +#define LOOKUP_CASTS { \ + [LOOKUP_UNICAST] = "unicast", \ + [LOOKUP_MULTICAST] = "multicast", \ +} + /** * A representation of an MPLS label for imposition in the data-path */ @@ -73,6 +86,11 @@ typedef struct lookup_dpo_t */ lookup_table_t lkd_table; + /** + * Unicast of rmulticast FIB lookup + */ + lookup_cast_t lkd_cast; + /** * Number of locks */ @@ -81,11 +99,13 @@ typedef struct lookup_dpo_t extern void lookup_dpo_add_or_lock_w_fib_index(fib_node_index_t fib_index, dpo_proto_t proto, + lookup_cast_t cast, lookup_input_t input, lookup_table_t table, dpo_id_t *dpo); extern void lookup_dpo_add_or_lock_w_table_id(u32 table_id, dpo_proto_t proto, + lookup_cast_t cast, lookup_input_t input, lookup_table_t table, dpo_id_t *dpo); diff --git a/src/vnet/dpo/mpls_disposition.c b/src/vnet/dpo/mpls_disposition.c new file mode 100644 index 00000000..5dc33fcf --- /dev/null +++ b/src/vnet/dpo/mpls_disposition.c @@ -0,0 +1,364 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +/* + * pool of all MPLS Label DPOs + */ +mpls_disp_dpo_t *mpls_disp_dpo_pool; + +static mpls_disp_dpo_t * +mpls_disp_dpo_alloc (void) +{ + mpls_disp_dpo_t *mdd; + + pool_get_aligned(mpls_disp_dpo_pool, mdd, CLIB_CACHE_LINE_BYTES); + memset(mdd, 0, sizeof(*mdd)); + + dpo_reset(&mdd->mdd_dpo); + + return (mdd); +} + +static index_t +mpls_disp_dpo_get_index (mpls_disp_dpo_t *mdd) +{ + return (mdd - mpls_disp_dpo_pool); +} + +index_t +mpls_disp_dpo_create (dpo_proto_t payload_proto, + fib_rpf_id_t rpf_id, + const dpo_id_t *dpo) +{ + mpls_disp_dpo_t *mdd; + + mdd = mpls_disp_dpo_alloc(); + + mdd->mdd_payload_proto = payload_proto; + mdd->mdd_rpf_id = rpf_id; + + dpo_stack(DPO_MPLS_DISPOSITION, + mdd->mdd_payload_proto, + &mdd->mdd_dpo, + dpo); + + return (mpls_disp_dpo_get_index(mdd)); +} + +u8* +format_mpls_disp_dpo (u8 *s, va_list *args) +{ + index_t index = va_arg (*args, index_t); + u32 indent = va_arg (*args, u32); + mpls_disp_dpo_t *mdd; + + mdd = mpls_disp_dpo_get(index); + + s = format(s, "mpls-disposition:[%d]:[%U]", + index, + format_dpo_proto, mdd->mdd_payload_proto); + + s = format(s, "\n%U", format_white_space, indent); + s = format(s, "%U", format_dpo_id, &mdd->mdd_dpo, indent+2); + + return (s); +} + +static void +mpls_disp_dpo_lock (dpo_id_t *dpo) +{ + mpls_disp_dpo_t *mdd; + + mdd = mpls_disp_dpo_get(dpo->dpoi_index); + + mdd->mdd_locks++; +} + +static void +mpls_disp_dpo_unlock (dpo_id_t *dpo) +{ + mpls_disp_dpo_t *mdd; + + mdd = mpls_disp_dpo_get(dpo->dpoi_index); + + mdd->mdd_locks--; + + if (0 == mdd->mdd_locks) + { + dpo_reset(&mdd->mdd_dpo); + pool_put(mpls_disp_dpo_pool, mdd); + } +} + +/** + * @brief A struct to hold tracing information for the MPLS label disposition + * node. + */ +typedef struct mpls_label_disposition_trace_t_ +{ + index_t mdd; +} mpls_label_disposition_trace_t; + +always_inline uword +mpls_label_disposition_inline (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * from_frame, + u8 payload_is_ip4, + u8 payload_is_ip6) +{ + u32 n_left_from, next_index, * from, * to_next; + + from = vlib_frame_vector_args (from_frame); + n_left_from = from_frame->n_vectors; + + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame(vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + mpls_disp_dpo_t *mdd0, *mdd1; + u32 bi0, mddi0, bi1, mddi1; + vlib_buffer_t * b0, *b1; + u32 next0, next1; + + bi0 = to_next[0] = from[0]; + bi1 = to_next[1] = from[1]; + + /* Prefetch next iteration. */ + { + vlib_buffer_t * p2, * p3; + + p2 = vlib_get_buffer (vm, from[2]); + p3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (p2, STORE); + vlib_prefetch_buffer_header (p3, STORE); + + CLIB_PREFETCH (p2->data, sizeof (ip6_header_t), STORE); + CLIB_PREFETCH (p3->data, sizeof (ip6_header_t), STORE); + } + + from += 2; + to_next += 2; + n_left_from -= 2; + n_left_to_next -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + /* dst lookup was done by ip4 lookup */ + mddi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX]; + mddi1 = vnet_buffer(b1)->ip.adj_index[VLIB_TX]; + mdd0 = mpls_disp_dpo_get(mddi0); + mdd1 = mpls_disp_dpo_get(mddi1); + + if (payload_is_ip4) + { + /* + * decrement the TTL on ingress to the LSP + */ + } + else if (payload_is_ip6) + { + /* + * decrement the TTL on ingress to the LSP + */ + } + + next0 = mdd0->mdd_dpo.dpoi_next_node; + next1 = mdd1->mdd_dpo.dpoi_next_node; + vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mdd0->mdd_dpo.dpoi_index; + vnet_buffer(b1)->ip.adj_index[VLIB_TX] = mdd1->mdd_dpo.dpoi_index; + vnet_buffer(b0)->ip.rpf_id = mdd0->mdd_rpf_id; + vnet_buffer(b1)->ip.rpf_id = mdd1->mdd_rpf_id; + + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_label_disposition_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + + tr->mdd = mddi0; + } + if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_label_disposition_trace_t *tr = + vlib_add_trace (vm, node, b1, sizeof (*tr)); + tr->mdd = mddi1; + } + + vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, + n_left_to_next, + bi0, bi1, next0, next1); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + mpls_disp_dpo_t *mdd0; + vlib_buffer_t * b0; + u32 bi0, mddi0; + u32 next0; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + /* dst lookup was done by ip4 lookup */ + mddi0 = vnet_buffer(b0)->ip.adj_index[VLIB_TX]; + mdd0 = mpls_disp_dpo_get(mddi0); + + if (payload_is_ip4) + { + /* + * decrement the TTL on ingress to the LSP + */ + } + else if (payload_is_ip6) + { + /* + * decrement the TTL on ingress to the LSP + */ + } + else + { + } + + next0 = mdd0->mdd_dpo.dpoi_next_node; + vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mdd0->mdd_dpo.dpoi_index; + vnet_buffer(b0)->ip.rpf_id = mdd0->mdd_rpf_id; + + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_label_disposition_trace_t *tr = + vlib_add_trace (vm, node, b0, sizeof (*tr)); + tr->mdd = mddi0; + } + + vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, + n_left_to_next, bi0, next0); + } + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + return from_frame->n_vectors; +} + +static u8 * +format_mpls_label_disposition_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + CLIB_UNUSED (mpls_label_disposition_trace_t * t); + + t = va_arg (*args, mpls_label_disposition_trace_t *); + + s = format(s, "disp:%d", t->mdd); + return (s); +} + +static uword +ip4_mpls_label_disposition (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return (mpls_label_disposition_inline(vm, node, frame, 1, 0)); +} + +VLIB_REGISTER_NODE (ip4_mpls_label_disposition_node) = { + .function = ip4_mpls_label_disposition, + .name = "ip4-mpls-label-disposition", + .vector_size = sizeof (u32), + + .format_trace = format_mpls_label_disposition_trace, + .n_next_nodes = 1, + .next_nodes = { + [0] = "ip4-drop", + } +}; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_mpls_label_disposition_node, + ip4_mpls_label_disposition) + +static uword +ip6_mpls_label_disposition (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return (mpls_label_disposition_inline(vm, node, frame, 0, 1)); +} + +VLIB_REGISTER_NODE (ip6_mpls_label_disposition_node) = { + .function = ip6_mpls_label_disposition, + .name = "ip6-mpls-label-disposition", + .vector_size = sizeof (u32), + + .format_trace = format_mpls_label_disposition_trace, + .n_next_nodes = 1, + .next_nodes = { + [0] = "ip6-drop", + } +}; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_mpls_label_disposition_node, + ip6_mpls_label_disposition) + +static void +mpls_disp_dpo_mem_show (void) +{ + fib_show_memory_usage("MPLS label", + pool_elts(mpls_disp_dpo_pool), + pool_len(mpls_disp_dpo_pool), + sizeof(mpls_disp_dpo_t)); +} + +const static dpo_vft_t mdd_vft = { + .dv_lock = mpls_disp_dpo_lock, + .dv_unlock = mpls_disp_dpo_unlock, + .dv_format = format_mpls_disp_dpo, + .dv_mem_show = mpls_disp_dpo_mem_show, +}; + +const static char* const mpls_label_disp_ip4_nodes[] = +{ + "ip4-mpls-label-disposition", + NULL, +}; +const static char* const mpls_label_disp_ip6_nodes[] = +{ + "ip6-mpls-label-disposition", + NULL, +}; +const static char* const * const mpls_label_disp_nodes[DPO_PROTO_NUM] = +{ + [DPO_PROTO_IP4] = mpls_label_disp_ip4_nodes, + [DPO_PROTO_IP6] = mpls_label_disp_ip6_nodes, +}; + + +void +mpls_disp_dpo_module_init (void) +{ + dpo_register(DPO_MPLS_DISPOSITION, &mdd_vft, mpls_label_disp_nodes); +} diff --git a/src/vnet/dpo/mpls_disposition.h b/src/vnet/dpo/mpls_disposition.h new file mode 100644 index 00000000..9c015083 --- /dev/null +++ b/src/vnet/dpo/mpls_disposition.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MPLS_DISP_DPO_H__ +#define __MPLS_DISP_DPO_H__ + +#include +#include +#include +#include + +/** + * A representation of an MPLS label for imposition in the data-path + */ +typedef struct mpls_disp_dpo_t +{ + /** + * Next DPO in the graph + */ + dpo_id_t mdd_dpo; + + /** + * The protocol of the payload/packets that are being encapped + */ + dpo_proto_t mdd_payload_proto; + + /** + * RPF-ID (if this is an mcast disposition) + */ + fib_rpf_id_t mdd_rpf_id; + + /** + * Number of locks/users of the label + */ + u16 mdd_locks; +} mpls_disp_dpo_t; + +/** + * @brief Assert that the MPLS label object is less than a cache line in size. + * Should this get any bigger then we will need to reconsider how many labels + * can be pushed in one object. + */ +_Static_assert((sizeof(mpls_disp_dpo_t) <= CLIB_CACHE_LINE_BYTES), + "MPLS Disposition DPO is larger than one cache line."); + +/** + * @brief Create an MPLS label object + * + * @param payload_proto The ptocool of the payload packets that will + * be imposed with this label header. + * @param dpo The parent of the created MPLS label object + */ +extern index_t mpls_disp_dpo_create(dpo_proto_t payload_proto, + fib_rpf_id_t rpf_id, + const dpo_id_t *dpo); + +extern u8* format_mpls_disp_dpo(u8 *s, va_list *args); + + +/* + * Encapsulation violation for fast data-path access + */ +extern mpls_disp_dpo_t *mpls_disp_dpo_pool; + +static inline mpls_disp_dpo_t * +mpls_disp_dpo_get (index_t index) +{ + return (pool_elt_at_index(mpls_disp_dpo_pool, index)); +} + +extern void mpls_disp_dpo_module_init(void); + +#endif diff --git a/src/vnet/dpo/mpls_label_dpo.c b/src/vnet/dpo/mpls_label_dpo.c index be9b2850..4d84b900 100644 --- a/src/vnet/dpo/mpls_label_dpo.c +++ b/src/vnet/dpo/mpls_label_dpo.c @@ -562,7 +562,7 @@ VLIB_REGISTER_NODE (mpls_label_imposition_node) = { .format_trace = format_mpls_label_imposition_trace, .n_next_nodes = 1, .next_nodes = { - [0] = "error-drop", + [0] = "mpls-drop", } }; VLIB_NODE_FUNCTION_MULTIARCH (mpls_label_imposition_node, @@ -584,7 +584,7 @@ VLIB_REGISTER_NODE (ip4_mpls_label_imposition_node) = { .format_trace = format_mpls_label_imposition_trace, .n_next_nodes = 1, .next_nodes = { - [0] = "error-drop", + [0] = "ip4-drop", } }; VLIB_NODE_FUNCTION_MULTIARCH (ip4_mpls_label_imposition_node, @@ -606,7 +606,7 @@ VLIB_REGISTER_NODE (ip6_mpls_label_imposition_node) = { .format_trace = format_mpls_label_imposition_trace, .n_next_nodes = 1, .next_nodes = { - [0] = "error-drop", + [0] = "ip6-drop", } }; VLIB_NODE_FUNCTION_MULTIARCH (ip6_mpls_label_imposition_node, diff --git a/src/vnet/dpo/replicate_dpo.c b/src/vnet/dpo/replicate_dpo.c index e25ceae9..9fdb9a05 100644 --- a/src/vnet/dpo/replicate_dpo.c +++ b/src/vnet/dpo/replicate_dpo.c @@ -17,6 +17,7 @@ #include #include #include +#include #undef REP_DEBUG @@ -106,6 +107,7 @@ replicate_format (index_t repi, dpo_id_t *buckets; u32 i; + repi &= ~MPLS_IS_REPLICATE; rep = replicate_get(repi); vlib_get_combined_counter(&(replicate_main.repm_counters), repi, &to); buckets = replicate_get_buckets(rep); @@ -187,6 +189,7 @@ replicate_set_bucket (index_t repi, replicate_t *rep; dpo_id_t *buckets; + repi &= ~MPLS_IS_REPLICATE; rep = replicate_get(repi); buckets = replicate_get_buckets(rep); @@ -199,11 +202,13 @@ int replicate_is_drop (const dpo_id_t *dpo) { replicate_t *rep; + index_t repi; if (DPO_REPLICATE != dpo->dpoi_type) return (0); - rep = replicate_get(dpo->dpoi_index); + repi = dpo->dpoi_index & ~MPLS_IS_REPLICATE; + rep = replicate_get(repi); if (1 == rep->rep_n_buckets) { @@ -218,6 +223,7 @@ replicate_get_bucket (index_t repi, { replicate_t *rep; + repi &= ~MPLS_IS_REPLICATE; rep = replicate_get(repi); return (replicate_get_bucket_i(rep, bucket)); @@ -288,9 +294,11 @@ replicate_multipath_update (const dpo_id_t *dpo, dpo_id_t *tmp_dpo; u32 ii, n_buckets; replicate_t *rep; + index_t repi; ASSERT(DPO_REPLICATE == dpo->dpoi_type); - rep = replicate_get(dpo->dpoi_index); + repi = dpo->dpoi_index & ~MPLS_IS_REPLICATE; + rep = replicate_get(repi); nhs = replicate_multipath_next_hop_fixup(next_hops, rep->rep_proto); n_buckets = vec_len(nhs); @@ -718,7 +726,7 @@ format_replicate_trace (u8 * s, va_list * args) s = format (s, "replicate: %d via %U", t->rep_index, - format_dpo_id, &t->dpo); + format_dpo_id, &t->dpo, 0); return s; } @@ -731,7 +739,7 @@ ip4_replicate (vlib_main_t * vm, } /** - * @brief + * @brief IP4 replication node */ VLIB_REGISTER_NODE (ip4_replicate_node) = { .function = ip4_replicate, @@ -744,7 +752,7 @@ VLIB_REGISTER_NODE (ip4_replicate_node) = { .format_trace = format_replicate_trace, .n_next_nodes = 1, .next_nodes = { - [0] = "error-drop", + [0] = "ip4-drop", }, }; @@ -757,7 +765,7 @@ ip6_replicate (vlib_main_t * vm, } /** - * @brief + * @brief IPv6 replication node */ VLIB_REGISTER_NODE (ip6_replicate_node) = { .function = ip6_replicate, @@ -770,7 +778,33 @@ VLIB_REGISTER_NODE (ip6_replicate_node) = { .format_trace = format_replicate_trace, .n_next_nodes = 1, .next_nodes = { - [0] = "error-drop", + [0] = "ip6-drop", + }, +}; + +static uword +mpls_replicate (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + return (replicate_inline (vm, node, frame)); +} + +/** + * @brief MPLS replication node + */ +VLIB_REGISTER_NODE (mpls_replicate_node) = { + .function = mpls_replicate, + .name = "mpls-replicate", + .vector_size = sizeof (u32), + + .n_errors = ARRAY_LEN(replicate_dpo_error_strings), + .error_strings = replicate_dpo_error_strings, + + .format_trace = format_replicate_trace, + .n_next_nodes = 1, + .next_nodes = { + [0] = "mpls-drop", }, }; diff --git a/src/vnet/dpo/replicate_dpo.h b/src/vnet/dpo/replicate_dpo.h index 77273015..7383184a 100644 --- a/src/vnet/dpo/replicate_dpo.h +++ b/src/vnet/dpo/replicate_dpo.h @@ -25,6 +25,7 @@ #include #include #include +#include /** * replicate main @@ -119,6 +120,7 @@ extern replicate_t *replicate_pool; static inline replicate_t* replicate_get (index_t repi) { + repi &= ~MPLS_IS_REPLICATE; return (pool_elt_at_index(replicate_pool, repi)); } diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c index c74a097e..dd509193 100644 --- a/src/vnet/ethernet/arp.c +++ b/src/vnet/ethernet/arp.c @@ -507,6 +507,7 @@ arp_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai) case IP_LOOKUP_NEXT_PUNT: case IP_LOOKUP_NEXT_LOCAL: case IP_LOOKUP_NEXT_REWRITE: + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: case IP_LOOKUP_NEXT_MIDCHAIN: case IP_LOOKUP_NEXT_ICMP_ERROR: case IP_LOOKUP_N_NEXT: diff --git a/src/vnet/ethernet/interface.c b/src/vnet/ethernet/interface.c index 335e3f9f..9ac30bc6 100644 --- a/src/vnet/ethernet/interface.c +++ b/src/vnet/ethernet/interface.c @@ -115,7 +115,7 @@ ethernet_build_rewrite (vnet_main_t * vnm, #define _(a,b) case VNET_LINK_##a: type = ETHERNET_TYPE_##b; break _(IP4, IP4); _(IP6, IP6); - _(MPLS, MPLS_UNICAST); + _(MPLS, MPLS); _(ARP, ARP); #undef _ default: diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index f7787ed2..5305012f 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -249,7 +249,7 @@ determine_next_node (ethernet_main_t * em, { *next0 = em->l3_next.input_next_ip6; } - else if (type0 == ETHERNET_TYPE_MPLS_UNICAST) + else if (type0 == ETHERNET_TYPE_MPLS) { *next0 = em->l3_next.input_next_mpls; @@ -1252,7 +1252,7 @@ next_by_ethertype_register (next_by_ethertype_t * l3_next, { l3_next->input_next_ip6 = next_index; } - else if (ethertype == ETHERNET_TYPE_MPLS_UNICAST) + else if (ethertype == ETHERNET_TYPE_MPLS) { l3_next->input_next_mpls = next_index; } diff --git a/src/vnet/ethernet/types.def b/src/vnet/ethernet/types.def index 643f3152..7dab8ee1 100644 --- a/src/vnet/ethernet/types.def +++ b/src/vnet/ethernet/types.def @@ -85,8 +85,8 @@ ethernet_type (0x876D, SECURE_DATA) ethernet_type (0x8808, MAC_CONTROL) ethernet_type (0x8809, SLOW_PROTOCOLS) ethernet_type (0x880B, PPP) -ethernet_type (0x8847, MPLS_UNICAST) -ethernet_type (0x8848, MPLS_MULTICAST) +ethernet_type (0x8847, MPLS) +ethernet_type (0x8848, MPLS_UPSTREAM_ASSIGNED) ethernet_type (0x8863, PPPOE_DISCOVERY) ethernet_type (0x8864, PPPOE_SESSION) ethernet_type (0x886D, INTEL_ANS) diff --git a/src/vnet/fib/fib_api.h b/src/vnet/fib/fib_api.h index f8275317..10d0cb58 100644 --- a/src/vnet/fib/fib_api.h +++ b/src/vnet/fib/fib_api.h @@ -24,6 +24,7 @@ add_del_route_check (fib_protocol_t table_proto, fib_protocol_t next_hop_table_proto, u32 next_hop_table_id, u8 create_missing_tables, + u8 is_rpf_id, u32 * fib_index, u32 * next_hop_fib_index); int @@ -33,10 +34,13 @@ add_del_route_t_handler (u8 is_multipath, u8 is_unreach, u8 is_prohibit, u8 is_local, + u8 is_multicast, u8 is_classify, u32 classify_table_index, u8 is_resolve_host, u8 is_resolve_attached, + u8 is_interface_rx, + u8 is_rpf_id, u32 fib_index, const fib_prefix_t * prefix, u8 next_hop_proto_is_ip4, diff --git a/src/vnet/fib/fib_entry.c b/src/vnet/fib/fib_entry.c index dac1fce9..6f811aa1 100644 --- a/src/vnet/fib/fib_entry.c +++ b/src/vnet/fib/fib_entry.c @@ -75,13 +75,7 @@ fib_entry_get_default_chain_type (const fib_entry_t *fib_entry) return (FIB_FORW_CHAIN_TYPE_UNICAST_IP6); case FIB_PROTOCOL_MPLS: if (MPLS_EOS == fib_entry->fe_prefix.fp_eos) - /* - * If the entry being asked is a eos-MPLS label entry, - * then use the payload-protocol field, that we stashed there - * for just this purpose - */ - return (fib_forw_chain_type_from_dpo_proto( - fib_entry->fe_prefix.fp_payload_proto)); + return (FIB_FORW_CHAIN_TYPE_MPLS_EOS); else return (FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS); } @@ -370,6 +364,35 @@ fib_entry_contribute_urpf (fib_node_index_t entry_index, return (fib_path_list_contribute_urpf(fib_entry->fe_parent, urpf)); } +/* + * If the client is request a chain for multicast forwarding then swap + * the chain type to one that can provide such transport. + */ +static fib_forward_chain_type_t +fib_entry_chain_type_mcast_to_ucast (fib_forward_chain_type_t fct) +{ + switch (fct) + { + case FIB_FORW_CHAIN_TYPE_MCAST_IP4: + case FIB_FORW_CHAIN_TYPE_MCAST_IP6: + /* + * we can only transport IP multicast packets if there is an + * LSP. + */ + fct = FIB_FORW_CHAIN_TYPE_MPLS_EOS; + break; + case FIB_FORW_CHAIN_TYPE_MPLS_EOS: + case FIB_FORW_CHAIN_TYPE_UNICAST_IP4: + case FIB_FORW_CHAIN_TYPE_UNICAST_IP6: + case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS: + case FIB_FORW_CHAIN_TYPE_ETHERNET: + case FIB_FORW_CHAIN_TYPE_NSH: + break; + } + + return (fct); +} + /* * fib_entry_contribute_forwarding * @@ -385,6 +408,11 @@ fib_entry_contribute_forwarding (fib_node_index_t fib_entry_index, fib_entry = fib_entry_get(fib_entry_index); + /* + * mfib children ask for mcast chains. fix these to the appropriate ucast types. + */ + fct = fib_entry_chain_type_mcast_to_ucast(fct); + if (fct == fib_entry_get_default_chain_type(fib_entry)) { dpo_copy(dpo, &fib_entry->fe_lb); @@ -414,6 +442,11 @@ fib_entry_contribute_forwarding (fib_node_index_t fib_entry_index, dpo_copy(dpo, &fed->fd_dpo); } + /* + * don't allow the special index indicating replicate.vs.load-balance + * to escape to the clients + */ + dpo->dpoi_index &= ~MPLS_IS_REPLICATE; } const dpo_id_t * diff --git a/src/vnet/fib/fib_entry.h b/src/vnet/fib/fib_entry.h index a3f75e60..b17a0b64 100644 --- a/src/vnet/fib/fib_entry.h +++ b/src/vnet/fib/fib_entry.h @@ -192,6 +192,11 @@ typedef enum fib_entry_attribute_t_ { * The prefix/address is local to this device */ FIB_ENTRY_ATTRIBUTE_LOCAL, + /** + * The prefix/address is a multicast prefix. + * this aplies only to MPLS. IP multicast is handled by mfib + */ + FIB_ENTRY_ATTRIBUTE_MULTICAST, /** * The prefix/address exempted from loose uRPF check * To be used with caution @@ -200,7 +205,7 @@ typedef enum fib_entry_attribute_t_ { /** * Marker. add new entries before this one. */ - FIB_ENTRY_ATTRIBUTE_LAST = FIB_ENTRY_ATTRIBUTE_URPF_EXEMPT, + FIB_ENTRY_ATTRIBUTE_LAST = FIB_ENTRY_ATTRIBUTE_MULTICAST, } fib_entry_attribute_t; /** @@ -215,7 +220,8 @@ typedef enum fib_entry_attribute_t_ { [FIB_ENTRY_ATTRIBUTE_DROP] = "drop", \ [FIB_ENTRY_ATTRIBUTE_EXCLUSIVE] = "exclusive", \ [FIB_ENTRY_ATTRIBUTE_LOCAL] = "local", \ - [FIB_ENTRY_ATTRIBUTE_URPF_EXEMPT] = "uRPF-exempt" \ + [FIB_ENTRY_ATTRIBUTE_URPF_EXEMPT] = "uRPF-exempt", \ + [FIB_ENTRY_ATTRIBUTE_MULTICAST] = "multicast", \ } #define FOR_EACH_FIB_ATTRIBUTE(_item) \ @@ -232,6 +238,7 @@ typedef enum fib_entry_flag_t_ { FIB_ENTRY_FLAG_LOCAL = (1 << FIB_ENTRY_ATTRIBUTE_LOCAL), FIB_ENTRY_FLAG_IMPORT = (1 << FIB_ENTRY_ATTRIBUTE_IMPORT), FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT = (1 << FIB_ENTRY_ATTRIBUTE_URPF_EXEMPT), + FIB_ENTRY_FLAG_MULTICAST = (1 << FIB_ENTRY_ATTRIBUTE_MULTICAST), } __attribute__((packed)) fib_entry_flag_t; /** @@ -396,7 +403,7 @@ typedef struct fib_entry_t_ { * paint the header straight on without the need to check the packet * type to derive the EOS bit value. */ - dpo_id_t fe_lb; // [FIB_FORW_CHAIN_MPLS_NUM]; + dpo_id_t fe_lb; /** * Vector of source infos. * Most entries will only have 1 source. So we optimise for memory usage, diff --git a/src/vnet/fib/fib_entry_src.c b/src/vnet/fib/fib_entry_src.c index aa1d5a24..a700282e 100644 --- a/src/vnet/fib/fib_entry_src.c +++ b/src/vnet/fib/fib_entry_src.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -229,8 +230,6 @@ fib_forward_chain_type_t fib_entry_chain_type_fixup (const fib_entry_t *entry, fib_forward_chain_type_t fct) { - ASSERT(FIB_FORW_CHAIN_TYPE_MPLS_EOS == fct); - /* * The EOS chain is a tricky since one cannot know the adjacency * to link to without knowing what the packets payload protocol @@ -238,6 +237,11 @@ fib_entry_chain_type_fixup (const fib_entry_t *entry, */ fib_forward_chain_type_t dfct; + if (FIB_FORW_CHAIN_TYPE_MPLS_EOS != fct) + { + return (fct); + } + dfct = fib_entry_get_default_chain_type(entry); if (FIB_FORW_CHAIN_TYPE_MPLS_EOS == dfct) @@ -303,7 +307,12 @@ fib_entry_src_collect_forwarding (fib_node_index_t pl_index, * found a matching extension. stack it to obtain the forwarding * info for this path. */ - ctx->next_hops = fib_path_ext_stack(path_ext, ctx->fib_entry, ctx->fct, ctx->next_hops); + ctx->next_hops = + fib_path_ext_stack(path_ext, + ctx->fct, + fib_entry_chain_type_fixup(ctx->fib_entry, + ctx->fct), + ctx->next_hops); } else { @@ -355,6 +364,9 @@ fib_entry_src_collect_forwarding (fib_node_index_t pl_index, fib_entry_chain_type_fixup(ctx->fib_entry, ctx->fct), &nh->path_dpo); + fib_path_stack_mpls_disp(path_index, + ctx->fib_entry->fe_prefix.fp_payload_proto, + &nh->path_dpo); break; } @@ -424,50 +436,70 @@ fib_entry_src_mk_lb (fib_entry_t *fib_entry, /* * first time create */ - flow_hash_config_t fhc; - - fhc = fib_table_get_flow_hash_config(fib_entry->fe_fib_index, - dpo_proto_to_fib(lb_proto)); - dpo_set(dpo_lb, - DPO_LOAD_BALANCE, - lb_proto, - load_balance_create(0, lb_proto, fhc)); + if (esrc->fes_entry_flags & FIB_ENTRY_FLAG_MULTICAST) + { + dpo_set(dpo_lb, + DPO_REPLICATE, + lb_proto, + MPLS_IS_REPLICATE | replicate_create(0, lb_proto)); + } + else + { + flow_hash_config_t fhc; + + fhc = fib_table_get_flow_hash_config(fib_entry->fe_fib_index, + dpo_proto_to_fib(lb_proto)); + dpo_set(dpo_lb, + DPO_LOAD_BALANCE, + lb_proto, + load_balance_create(0, lb_proto, fhc)); + } } - load_balance_multipath_update(dpo_lb, - ctx.next_hops, - fib_entry_calc_lb_flags(&ctx)); - vec_free(ctx.next_hops); - - /* - * if this entry is sourced by the uRPF-exempt source then we - * append the always present local0 interface (index 0) to the - * uRPF list so it is not empty. that way packets pass the loose check. - */ - index_t ui = fib_path_list_get_urpf(esrc->fes_pl); - - if ((fib_entry_is_sourced(fib_entry_get_index(fib_entry), - FIB_SOURCE_URPF_EXEMPT) || - (esrc->fes_entry_flags & FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT))&& - (0 == fib_urpf_check_size(ui))) + if (esrc->fes_entry_flags & FIB_ENTRY_FLAG_MULTICAST) { - /* - * The uRPF list we get from the path-list is shared by all - * other users of the list, but the uRPF exemption applies - * only to this prefix. So we need our own list. - */ - ui = fib_urpf_list_alloc_and_lock(); - fib_urpf_list_append(ui, 0); - fib_urpf_list_bake(ui); - load_balance_set_urpf(dpo_lb->dpoi_index, ui); - fib_urpf_list_unlock(ui); + /* + * MPLS multicast + */ + replicate_multipath_update(dpo_lb, ctx.next_hops); } else { - load_balance_set_urpf(dpo_lb->dpoi_index, ui); + load_balance_multipath_update(dpo_lb, + ctx.next_hops, + fib_entry_calc_lb_flags(&ctx)); + vec_free(ctx.next_hops); + + /* + * if this entry is sourced by the uRPF-exempt source then we + * append the always present local0 interface (index 0) to the + * uRPF list so it is not empty. that way packets pass the loose check. + */ + index_t ui = fib_path_list_get_urpf(esrc->fes_pl); + + if ((fib_entry_is_sourced(fib_entry_get_index(fib_entry), + FIB_SOURCE_URPF_EXEMPT) || + (esrc->fes_entry_flags & FIB_ENTRY_FLAG_LOOSE_URPF_EXEMPT))&& + (0 == fib_urpf_check_size(ui))) + { + /* + * The uRPF list we get from the path-list is shared by all + * other users of the list, but the uRPF exemption applies + * only to this prefix. So we need our own list. + */ + ui = fib_urpf_list_alloc_and_lock(); + fib_urpf_list_append(ui, 0); + fib_urpf_list_bake(ui); + load_balance_set_urpf(dpo_lb->dpoi_index, ui); + fib_urpf_list_unlock(ui); + } + else + { + load_balance_set_urpf(dpo_lb->dpoi_index, ui); + } + load_balance_set_fib_entry_flags(dpo_lb->dpoi_index, + fib_entry_get_flags_i(fib_entry)); } - load_balance_set_fib_entry_flags(dpo_lb->dpoi_index, - fib_entry_get_flags_i(fib_entry)); } void @@ -887,21 +919,6 @@ fib_entry_src_action_remove (fib_entry_t *fib_entry, return (sflags); } -static inline int -fib_route_recurses_via_self (const fib_prefix_t *prefix, - const fib_route_path_t *rpath) -{ - /* - * not all zeros next hop && - * is recursive path && - * nexthop is same as the route's address - */ - return ((!ip46_address_is_zero(&rpath->frp_addr)) && - (~0 == rpath->frp_sw_if_index) && - (0 == ip46_address_cmp(&rpath->frp_addr, &prefix->fp_addr))); - -} - /* * fib_route_attached_cross_table * @@ -962,14 +979,14 @@ fib_entry_src_flags_2_path_list_flags (fib_entry_flag_t eflags) { plf |= FIB_PATH_LIST_FLAG_DROP; } - if (eflags & FIB_ENTRY_FLAG_LOCAL) - { - plf |= FIB_PATH_LIST_FLAG_LOCAL; - } if (eflags & FIB_ENTRY_FLAG_EXCLUSIVE) { plf |= FIB_PATH_LIST_FLAG_EXCLUSIVE; } + if (eflags & FIB_ENTRY_FLAG_LOCAL) + { + plf |= FIB_PATH_LIST_FLAG_LOCAL; + } return (plf); } @@ -980,25 +997,6 @@ fib_entry_flags_update (const fib_entry_t *fib_entry, fib_path_list_flags_t *pl_flags, fib_entry_src_t *esrc) { - /* - * don't allow the addition of a recursive looped path for prefix - * via itself. - */ - if (fib_route_recurses_via_self(&fib_entry->fe_prefix, rpath)) - { - /* - * force the install of a drop path-list. - * we want the entry to have some path-list, mainly so - * the dodgy path can be rmeoved when the source stops playing - * silly buggers. - */ - *pl_flags |= FIB_PATH_LIST_FLAG_DROP; - } - else - { - *pl_flags &= ~FIB_PATH_LIST_FLAG_DROP; - } - if ((esrc->fes_src == FIB_SOURCE_API) || (esrc->fes_src == FIB_SOURCE_CLI)) { diff --git a/src/vnet/fib/fib_internal.h b/src/vnet/fib/fib_internal.h index 2d980bcc..8abc0e07 100644 --- a/src/vnet/fib/fib_internal.h +++ b/src/vnet/fib/fib_internal.h @@ -25,6 +25,7 @@ #undef FIB_DEBUG extern void fib_prefix_from_mpls_label(mpls_label_t label, + mpls_eos_bit_t eos, fib_prefix_t *prf); extern int fib_route_path_cmp(const fib_route_path_t *rpath1, diff --git a/src/vnet/fib/fib_path.c b/src/vnet/fib/fib_path.c index 6b202a97..f81f4170 100644 --- a/src/vnet/fib/fib_path.c +++ b/src/vnet/fib/fib_path.c @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include @@ -66,6 +68,10 @@ typedef enum fib_path_type_t_ { * deag. Link to a lookup adj in the next table */ FIB_PATH_TYPE_DEAG, + /** + * interface receive. + */ + FIB_PATH_TYPE_INTF_RX, /** * receive. it's for-us. */ @@ -88,6 +94,7 @@ typedef enum fib_path_type_t_ { [FIB_PATH_TYPE_SPECIAL] = "special", \ [FIB_PATH_TYPE_EXCLUSIVE] = "exclusive", \ [FIB_PATH_TYPE_DEAG] = "deag", \ + [FIB_PATH_TYPE_INTF_RX] = "intf-rx", \ [FIB_PATH_TYPE_RECEIVE] = "receive", \ } @@ -220,10 +227,16 @@ typedef struct fib_path_t_ { * The next-hop */ ip46_address_t fp_ip; - /** - * The local label to resolve through. - */ - mpls_label_t fp_local_label; + struct { + /** + * The local label to resolve through. + */ + mpls_label_t fp_local_label; + /** + * The EOS bit of the resolving label + */ + mpls_eos_bit_t fp_eos; + }; } fp_nh; /** * The FIB table index in which to find the next-hop. @@ -254,6 +267,10 @@ typedef struct fib_path_t_ { * The FIB index in which to perfom the next lookup */ fib_node_index_t fp_tbl_id; + /** + * The RPF-ID to tag the packets with + */ + fib_rpf_id_t fp_rpf_id; } deag; struct { } special; @@ -273,6 +290,12 @@ typedef struct fib_path_t_ { */ ip46_address_t fp_addr; } receive; + struct { + /** + * The interface on which the packets will be input. + */ + u32 fp_interface; + } intf_rx; }; STRUCT_MARK(path_hash_end); @@ -444,9 +467,11 @@ format_fib_path (u8 * s, va_list * args) case FIB_PATH_TYPE_RECURSIVE: if (FIB_PROTOCOL_MPLS == path->fp_nh_proto) { - s = format (s, "via %U", + s = format (s, "via %U %U", format_mpls_unicast_label, - path->recursive.fp_nh.fp_local_label); + path->recursive.fp_nh.fp_local_label, + format_mpls_eos_bit, + path->recursive.fp_nh.fp_eos); } else { @@ -465,6 +490,7 @@ format_fib_path (u8 * s, va_list * args) break; case FIB_PATH_TYPE_RECEIVE: + case FIB_PATH_TYPE_INTF_RX: case FIB_PATH_TYPE_SPECIAL: case FIB_PATH_TYPE_DEAG: case FIB_PATH_TYPE_EXCLUSIVE: @@ -736,6 +762,7 @@ fib_path_unresolve (fib_path_t *path) break; case FIB_PATH_TYPE_SPECIAL: case FIB_PATH_TYPE_RECEIVE: + case FIB_PATH_TYPE_INTF_RX: case FIB_PATH_TYPE_DEAG: /* * these hold only the path's DPO, which is reset below. @@ -754,16 +781,24 @@ fib_path_unresolve (fib_path_t *path) } static fib_forward_chain_type_t -fib_path_proto_to_chain_type (fib_protocol_t proto) +fib_path_to_chain_type (const fib_path_t *path) { - switch (proto) + switch (path->fp_nh_proto) { case FIB_PROTOCOL_IP4: return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4); case FIB_PROTOCOL_IP6: return (FIB_FORW_CHAIN_TYPE_UNICAST_IP6); case FIB_PROTOCOL_MPLS: - return (FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS); + if (FIB_PATH_TYPE_RECURSIVE == path->fp_type && + MPLS_EOS == path->recursive.fp_nh.fp_eos) + { + return (FIB_FORW_CHAIN_TYPE_MPLS_EOS); + } + else + { + return (FIB_FORW_CHAIN_TYPE_MPLS_EOS); + } } return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4); } @@ -793,7 +828,7 @@ fib_path_back_walk_notify (fib_node_t *node, */ fib_path_recursive_adj_update( path, - fib_path_proto_to_chain_type(path->fp_nh_proto), + fib_path_to_chain_type(path), &path->fp_dpo); } if ((FIB_NODE_BW_REASON_FLAG_ADJ_UPDATE & ctx->fnbw_reason) || @@ -931,6 +966,8 @@ FIXME comment path->fp_oper_flags |= FIB_PATH_OPER_FLAG_DROP; } break; + case FIB_PATH_TYPE_INTF_RX: + ASSERT(0); case FIB_PATH_TYPE_DEAG: /* * FIXME When VRF delete is allowed this will need a poke. @@ -986,6 +1023,14 @@ fib_path_route_flags_to_cfg_flags (const fib_route_path_t *rpath) cfg_flags |= FIB_PATH_CFG_FLAG_LOCAL; if (rpath->frp_flags & FIB_ROUTE_PATH_ATTACHED) cfg_flags |= FIB_PATH_CFG_FLAG_ATTACHED; + if (rpath->frp_flags & FIB_ROUTE_PATH_INTF_RX) + cfg_flags |= FIB_PATH_CFG_FLAG_INTF_RX; + if (rpath->frp_flags & FIB_ROUTE_PATH_RPF_ID) + cfg_flags |= FIB_PATH_CFG_FLAG_RPF_ID; + if (rpath->frp_flags & FIB_ROUTE_PATH_EXCLUSIVE) + cfg_flags |= FIB_PATH_CFG_FLAG_EXCLUSIVE; + if (rpath->frp_flags & FIB_ROUTE_PATH_DROP) + cfg_flags |= FIB_PATH_CFG_FLAG_DROP; return (cfg_flags); } @@ -998,8 +1043,6 @@ fib_path_route_flags_to_cfg_flags (const fib_route_path_t *rpath) */ fib_node_index_t fib_path_create (fib_node_index_t pl_index, - fib_protocol_t nh_proto, - fib_path_cfg_flags_t flags, const fib_route_path_t *rpath) { fib_path_t *path; @@ -1012,7 +1055,7 @@ fib_path_create (fib_node_index_t pl_index, dpo_reset(&path->fp_dpo); path->fp_pl_index = pl_index; - path->fp_nh_proto = nh_proto; + path->fp_nh_proto = rpath->frp_proto; path->fp_via_fib = FIB_NODE_INDEX_INVALID; path->fp_weight = rpath->frp_weight; if (0 == path->fp_weight) @@ -1023,8 +1066,7 @@ fib_path_create (fib_node_index_t pl_index, */ path->fp_weight = 1; } - path->fp_cfg_flags = flags; - path->fp_cfg_flags |= fib_path_route_flags_to_cfg_flags(rpath); + path->fp_cfg_flags = fib_path_route_flags_to_cfg_flags(rpath); /* * deduce the path's tpye from the parementers and save what is needed. @@ -1035,6 +1077,17 @@ fib_path_create (fib_node_index_t pl_index, path->receive.fp_interface = rpath->frp_sw_if_index; path->receive.fp_addr = rpath->frp_addr; } + else if (path->fp_cfg_flags & FIB_PATH_CFG_FLAG_INTF_RX) + { + path->fp_type = FIB_PATH_TYPE_INTF_RX; + path->intf_rx.fp_interface = rpath->frp_sw_if_index; + } + else if (path->fp_cfg_flags & FIB_PATH_CFG_FLAG_RPF_ID) + { + path->fp_type = FIB_PATH_TYPE_DEAG; + path->deag.fp_tbl_id = rpath->frp_fib_index; + path->deag.fp_rpf_id = rpath->frp_rpf_id; + } else if (~0 != rpath->frp_sw_if_index) { if (ip46_address_is_zero(&rpath->frp_addr)) @@ -1069,6 +1122,7 @@ fib_path_create (fib_node_index_t pl_index, if (FIB_PROTOCOL_MPLS == path->fp_nh_proto) { path->recursive.fp_nh.fp_local_label = rpath->frp_local_label; + path->recursive.fp_nh.fp_eos = rpath->frp_eos; } else { @@ -1238,17 +1292,13 @@ fib_path_cmp_i (const fib_path_t *path1, res = ip46_address_cmp(&path1->attached_next_hop.fp_nh, &path2->attached_next_hop.fp_nh); if (0 == res) { - res = vnet_sw_interface_compare( - vnet_get_main(), - path1->attached_next_hop.fp_interface, - path2->attached_next_hop.fp_interface); + res = (path1->attached_next_hop.fp_interface - + path2->attached_next_hop.fp_interface); } break; case FIB_PATH_TYPE_ATTACHED: - res = vnet_sw_interface_compare( - vnet_get_main(), - path1->attached.fp_interface, - path2->attached.fp_interface); + res = (path1->attached.fp_interface - + path2->attached.fp_interface); break; case FIB_PATH_TYPE_RECURSIVE: res = ip46_address_cmp(&path1->recursive.fp_nh, @@ -1261,6 +1311,13 @@ fib_path_cmp_i (const fib_path_t *path1, break; case FIB_PATH_TYPE_DEAG: res = (path1->deag.fp_tbl_id - path2->deag.fp_tbl_id); + if (0 == res) + { + res = (path1->deag.fp_rpf_id - path2->deag.fp_rpf_id); + } + break; + case FIB_PATH_TYPE_INTF_RX: + res = (path1->intf_rx.fp_interface - path2->intf_rx.fp_interface); break; case FIB_PATH_TYPE_SPECIAL: case FIB_PATH_TYPE_RECEIVE: @@ -1336,22 +1393,22 @@ fib_path_cmp_w_route_path (fib_node_index_t path_index, &rpath->frp_addr); if (0 == res) { - res = vnet_sw_interface_compare( - vnet_get_main(), - path->attached_next_hop.fp_interface, - rpath->frp_sw_if_index); + res = (path->attached_next_hop.fp_interface - + rpath->frp_sw_if_index); } break; case FIB_PATH_TYPE_ATTACHED: - res = vnet_sw_interface_compare( - vnet_get_main(), - path->attached.fp_interface, - rpath->frp_sw_if_index); + res = (path->attached.fp_interface - rpath->frp_sw_if_index); break; case FIB_PATH_TYPE_RECURSIVE: if (FIB_PROTOCOL_MPLS == path->fp_nh_proto) { res = path->recursive.fp_nh.fp_local_label - rpath->frp_local_label; + + if (res == 0) + { + res = path->recursive.fp_nh.fp_eos - rpath->frp_eos; + } } else { @@ -1364,9 +1421,16 @@ fib_path_cmp_w_route_path (fib_node_index_t path_index, res = (path->recursive.fp_tbl_id - rpath->frp_fib_index); } break; + case FIB_PATH_TYPE_INTF_RX: + res = (path->intf_rx.fp_interface - rpath->frp_sw_if_index); + break; case FIB_PATH_TYPE_DEAG: res = (path->deag.fp_tbl_id - rpath->frp_fib_index); - break; + if (0 == res) + { + res = (path->deag.fp_rpf_id - rpath->frp_rpf_id); + } + break; case FIB_PATH_TYPE_SPECIAL: case FIB_PATH_TYPE_RECEIVE: case FIB_PATH_TYPE_EXCLUSIVE: @@ -1465,6 +1529,7 @@ fib_path_recursive_loop_detect (fib_node_index_t path_index, case FIB_PATH_TYPE_SPECIAL: case FIB_PATH_TYPE_DEAG: case FIB_PATH_TYPE_RECEIVE: + case FIB_PATH_TYPE_INTF_RX: case FIB_PATH_TYPE_EXCLUSIVE: /* * these path types cannot be part of a loop, since they are the leaves @@ -1563,7 +1628,9 @@ fib_path_resolve (fib_node_index_t path_index) if (FIB_PROTOCOL_MPLS == path->fp_nh_proto) { - fib_prefix_from_mpls_label(path->recursive.fp_nh.fp_local_label, &pfx); + fib_prefix_from_mpls_label(path->recursive.fp_nh.fp_local_label, + path->recursive.fp_nh.fp_eos, + &pfx); } else { @@ -1592,7 +1659,7 @@ fib_path_resolve (fib_node_index_t path_index) */ fib_path_recursive_adj_update( path, - fib_path_proto_to_chain_type(path->fp_nh_proto), + fib_path_to_chain_type(path), &path->fp_dpo); break; @@ -1605,16 +1672,25 @@ fib_path_resolve (fib_node_index_t path_index) drop_dpo_get(fib_proto_to_dpo(path->fp_nh_proto))); break; case FIB_PATH_TYPE_DEAG: + { /* * Resolve via a lookup DPO. * FIXME. control plane should add routes with a table ID */ - lookup_dpo_add_or_lock_w_fib_index(path->deag.fp_tbl_id, - fib_proto_to_dpo(path->fp_nh_proto), - LOOKUP_INPUT_DST_ADDR, - LOOKUP_TABLE_FROM_CONFIG, - &path->fp_dpo); + lookup_cast_t cast; + + cast = (path->fp_cfg_flags & FIB_PATH_CFG_FLAG_RPF_ID ? + LOOKUP_MULTICAST : + LOOKUP_UNICAST); + + lookup_dpo_add_or_lock_w_fib_index(path->deag.fp_tbl_id, + fib_proto_to_dpo(path->fp_nh_proto), + cast, + LOOKUP_INPUT_DST_ADDR, + LOOKUP_TABLE_FROM_CONFIG, + &path->fp_dpo); break; + } case FIB_PATH_TYPE_RECEIVE: /* * Resolve via a receive DPO. @@ -1624,6 +1700,15 @@ fib_path_resolve (fib_node_index_t path_index) &path->receive.fp_addr, &path->fp_dpo); break; + case FIB_PATH_TYPE_INTF_RX: { + /* + * Resolve via a receive DPO. + */ + interface_dpo_add_or_lock(fib_proto_to_dpo(path->fp_nh_proto), + path->intf_rx.fp_interface, + &path->fp_dpo); + break; + } case FIB_PATH_TYPE_EXCLUSIVE: /* * Resolve via the user provided DPO @@ -1652,6 +1737,7 @@ fib_path_get_resolving_interface (fib_node_index_t path_index) return (path->receive.fp_interface); case FIB_PATH_TYPE_RECURSIVE: return (fib_entry_get_resolving_interface(path->fp_via_fib)); + case FIB_PATH_TYPE_INTF_RX: case FIB_PATH_TYPE_SPECIAL: case FIB_PATH_TYPE_DEAG: case FIB_PATH_TYPE_EXCLUSIVE: @@ -1743,6 +1829,7 @@ fib_path_contribute_urpf (fib_node_index_t path_index, case FIB_PATH_TYPE_DEAG: case FIB_PATH_TYPE_RECEIVE: + case FIB_PATH_TYPE_INTF_RX: /* * these path types don't link to an adj */ @@ -1750,6 +1837,44 @@ fib_path_contribute_urpf (fib_node_index_t path_index, } } +void +fib_path_stack_mpls_disp (fib_node_index_t path_index, + dpo_proto_t payload_proto, + dpo_id_t *dpo) +{ + fib_path_t *path; + + path = fib_path_get(path_index); + + ASSERT(path); + + switch (path->fp_type) + { + case FIB_PATH_TYPE_DEAG: + { + dpo_id_t tmp = DPO_INVALID; + + dpo_copy(&tmp, dpo); + dpo_set(dpo, + DPO_MPLS_DISPOSITION, + payload_proto, + mpls_disp_dpo_create(payload_proto, + path->deag.fp_rpf_id, + &tmp)); + dpo_reset(&tmp); + break; + } + case FIB_PATH_TYPE_RECEIVE: + case FIB_PATH_TYPE_ATTACHED: + case FIB_PATH_TYPE_ATTACHED_NEXT_HOP: + case FIB_PATH_TYPE_RECURSIVE: + case FIB_PATH_TYPE_INTF_RX: + case FIB_PATH_TYPE_EXCLUSIVE: + case FIB_PATH_TYPE_SPECIAL: + break; + } +} + void fib_path_contribute_forwarding (fib_node_index_t path_index, fib_forward_chain_type_t fct, @@ -1769,7 +1894,7 @@ fib_path_contribute_forwarding (fib_node_index_t path_index, * This then represents the path's 'native' protocol; IP. * For all others will need to go find something else. */ - if (fib_path_proto_to_chain_type(path->fp_nh_proto) == fct) + if (fib_path_to_chain_type(path) == fct) { dpo_copy(dpo, &path->fp_dpo); } @@ -1813,10 +1938,10 @@ fib_path_contribute_forwarding (fib_node_index_t path_index, case FIB_FORW_CHAIN_TYPE_UNICAST_IP4: case FIB_FORW_CHAIN_TYPE_UNICAST_IP6: case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS: - fib_path_recursive_adj_update(path, fct, dpo); - break; case FIB_FORW_CHAIN_TYPE_MCAST_IP4: case FIB_FORW_CHAIN_TYPE_MCAST_IP6: + fib_path_recursive_adj_update(path, fct, dpo); + break; case FIB_FORW_CHAIN_TYPE_ETHERNET: case FIB_FORW_CHAIN_TYPE_NSH: ASSERT(0); @@ -1829,13 +1954,14 @@ fib_path_contribute_forwarding (fib_node_index_t path_index, case FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS: lookup_dpo_add_or_lock_w_table_id(MPLS_FIB_DEFAULT_TABLE_ID, DPO_PROTO_MPLS, + LOOKUP_UNICAST, LOOKUP_INPUT_DST_ADDR, LOOKUP_TABLE_FROM_CONFIG, dpo); break; + case FIB_FORW_CHAIN_TYPE_MPLS_EOS: case FIB_FORW_CHAIN_TYPE_UNICAST_IP4: case FIB_FORW_CHAIN_TYPE_UNICAST_IP6: - case FIB_FORW_CHAIN_TYPE_MPLS_EOS: dpo_copy(dpo, &path->fp_dpo); break; case FIB_FORW_CHAIN_TYPE_MCAST_IP4: @@ -1870,7 +1996,7 @@ fib_path_contribute_forwarding (fib_node_index_t path_index, ai = adj_mcast_add_or_lock(path->fp_nh_proto, fib_forw_chain_type_to_link_type(fct), path->attached.fp_interface); - dpo_set(dpo, DPO_ADJACENCY_MCAST, + dpo_set(dpo, DPO_ADJACENCY, fib_forw_chain_type_to_dpo_proto(fct), ai); adj_unlock(ai); @@ -1878,6 +2004,14 @@ fib_path_contribute_forwarding (fib_node_index_t path_index, break; } break; + case FIB_PATH_TYPE_INTF_RX: + /* + * Create the adj needed for sending IP multicast traffic + */ + interface_dpo_add_or_lock(fib_forw_chain_type_to_dpo_proto(fct), + path->attached.fp_interface, + dpo); + break; case FIB_PATH_TYPE_RECEIVE: case FIB_PATH_TYPE_SPECIAL: dpo_copy(dpo, &path->fp_dpo); diff --git a/src/vnet/fib/fib_path.h b/src/vnet/fib/fib_path.h index 14efc1ab..334be6f5 100644 --- a/src/vnet/fib/fib_path.h +++ b/src/vnet/fib/fib_path.h @@ -69,6 +69,14 @@ typedef enum fib_path_cfg_attribute_t_ { /** * The path is a for-us path */ + FIB_PATH_CFG_ATTRIBUTE_INTF_RX, + /** + * The path is a deag with rpf-id + */ + FIB_PATH_CFG_ATTRIBUTE_RPF_ID, + /** + * The path is an interface recieve + */ FIB_PATH_CFG_ATTRIBUTE_LOCAL, /** * Marker. Add new types before this one, then update it. @@ -88,6 +96,8 @@ typedef enum fib_path_cfg_attribute_t_ { [FIB_PATH_CFG_ATTRIBUTE_RESOLVE_ATTACHED] = "resolve-attached", \ [FIB_PATH_CFG_ATTRIBUTE_LOCAL] = "local", \ [FIB_PATH_CFG_ATTRIBUTE_ATTACHED] = "attached", \ + [FIB_PATH_CFG_ATTRIBUTE_INTF_RX] = "interface-rx", \ + [FIB_PATH_CFG_ATTRIBUTE_RPF_ID] = "rpf-id", \ } #define FOR_EACH_FIB_PATH_CFG_ATTRIBUTE(_item) \ @@ -106,6 +116,8 @@ typedef enum fib_path_cfg_flags_t_ { FIB_PATH_CFG_FLAG_RESOLVE_ATTACHED = (1 << FIB_PATH_CFG_ATTRIBUTE_RESOLVE_ATTACHED), FIB_PATH_CFG_FLAG_LOCAL = (1 << FIB_PATH_CFG_ATTRIBUTE_LOCAL), FIB_PATH_CFG_FLAG_ATTACHED = (1 << FIB_PATH_CFG_ATTRIBUTE_ATTACHED), + FIB_PATH_CFG_FLAG_INTF_RX = (1 << FIB_PATH_CFG_ATTRIBUTE_INTF_RX), + FIB_PATH_CFG_FLAG_RPF_ID = (1 << FIB_PATH_CFG_ATTRIBUTE_RPF_ID), } __attribute__ ((packed)) fib_path_cfg_flags_t; @@ -117,8 +129,6 @@ extern u8 *fib_path_adj_format(fib_node_index_t pi, extern u8 * format_fib_path(u8 * s, va_list * args); extern fib_node_index_t fib_path_create(fib_node_index_t pl_index, - fib_protocol_t nh_proto, - fib_path_cfg_flags_t flags, const fib_route_path_t *path); extern fib_node_index_t fib_path_create_special(fib_node_index_t pl_index, fib_protocol_t nh_proto, @@ -145,6 +155,9 @@ extern load_balance_path_t * fib_path_append_nh_for_multipath_hash( fib_node_index_t path_index, fib_forward_chain_type_t fct, load_balance_path_t *hash_key); +extern void fib_path_stack_mpls_disp(fib_node_index_t path_index, + dpo_proto_t payload_proto, + dpo_id_t *dpo); extern void fib_path_contribute_forwarding(fib_node_index_t path_index, fib_forward_chain_type_t type, dpo_id_t *dpo); diff --git a/src/vnet/fib/fib_path_ext.c b/src/vnet/fib/fib_path_ext.c index f75b5626..08293bcf 100644 --- a/src/vnet/fib/fib_path_ext.c +++ b/src/vnet/fib/fib_path_ext.c @@ -103,8 +103,8 @@ fib_path_ext_is_imp_null (fib_path_ext_t *path_ext) load_balance_path_t * fib_path_ext_stack (fib_path_ext_t *path_ext, - const fib_entry_t *entry, fib_forward_chain_type_t child_fct, + fib_forward_chain_type_t imp_null_fct, load_balance_path_t *nhs) { fib_forward_chain_type_t parent_fct; @@ -129,7 +129,7 @@ fib_path_ext_stack (fib_path_ext_t *path_ext, */ if (fib_path_ext_is_imp_null(path_ext)) { - parent_fct = fib_entry_chain_type_fixup(entry, child_fct); + parent_fct = imp_null_fct; } else { diff --git a/src/vnet/fib/fib_path_ext.h b/src/vnet/fib/fib_path_ext.h index cf8f8df0..d617700d 100644 --- a/src/vnet/fib/fib_path_ext.h +++ b/src/vnet/fib/fib_path_ext.h @@ -18,6 +18,7 @@ #include #include +#include /** * A path extension is a per-entry addition to the forwarding information @@ -61,8 +62,8 @@ extern void fib_path_ext_resolve(fib_path_ext_t *path_ext, fib_node_index_t path_list_index); extern load_balance_path_t *fib_path_ext_stack(fib_path_ext_t *path_ext, - const struct fib_entry_t_ *entry, fib_forward_chain_type_t fct, + fib_forward_chain_type_t imp_null_fct, load_balance_path_t *nhs); #endif diff --git a/src/vnet/fib/fib_path_list.c b/src/vnet/fib/fib_path_list.c index b9a391b3..ea6565dd 100644 --- a/src/vnet/fib/fib_path_list.c +++ b/src/vnet/fib/fib_path_list.c @@ -40,13 +40,6 @@ typedef struct fib_path_list_t_ { */ fib_path_list_flags_t fpl_flags; - /** - * The next-hop protocol for the paths in this path list. - * Note that fixing the proto here means we don't support a mix of - * v4 and v6 paths. ho hum. - */ - fib_protocol_t fpl_nh_proto; - /** * Vector of paths indicies for all configured paths. * For shareable path-lists this list MUST not change. @@ -57,6 +50,11 @@ typedef struct fib_path_list_t_ { * the RPF list calculated for this path list */ fib_node_index_t fpl_urpf; + + /** + * Hash table of paths. valid only with INDEXED flag + */ + uword *fpl_db; } fib_path_list_t; /* @@ -131,7 +129,6 @@ format_fib_path_list (u8 * s, va_list * args) s = format (s, " index:%u", fib_path_list_get_index(path_list)); s = format (s, " locks:%u", path_list->fpl_node.fn_locks); - s = format (s, " proto:%U", format_fib_protocol, path_list->fpl_nh_proto); if (FIB_PATH_LIST_FLAG_NONE != path_list->fpl_flags) { @@ -155,26 +152,6 @@ format_fib_path_list (u8 * s, va_list * args) return (s); } -u8 * -fib_path_list_adjs_format (fib_node_index_t path_list_index, - u32 indent, - u8 * s) -{ - fib_path_list_t *path_list; - u32 i; - - path_list = fib_path_list_get(path_list_index); - - vec_foreach_index (i, path_list->fpl_paths) - { - s = fib_path_adj_format(path_list->fpl_paths[i], - indent, s); - } - - return (s); -} - - u8 * fib_path_list_format (fib_node_index_t path_list_index, u8 * s) @@ -648,27 +625,6 @@ fib_path_list_is_looped (fib_node_index_t path_list_index) return (path_list->fpl_flags & FIB_PATH_LIST_FLAG_LOOPED); } -static fib_path_cfg_flags_t -fib_path_list_flags_2_path_flags (fib_path_list_flags_t plf) -{ - fib_path_cfg_flags_t pf = FIB_PATH_CFG_FLAG_NONE; - - if (plf & FIB_PATH_LIST_FLAG_LOCAL) - { - pf |= FIB_PATH_CFG_FLAG_LOCAL; - } - if (plf & FIB_PATH_LIST_FLAG_DROP) - { - pf |= FIB_PATH_CFG_FLAG_DROP; - } - if (plf & FIB_PATH_LIST_FLAG_EXCLUSIVE) - { - pf |= FIB_PATH_CFG_FLAG_EXCLUSIVE; - } - - return (pf); -} - static fib_path_list_flags_t fib_path_list_flags_fixup (fib_path_list_flags_t flags) { @@ -695,18 +651,15 @@ fib_path_list_create (fib_path_list_flags_t flags, flags = fib_path_list_flags_fixup(flags); path_list = fib_path_list_alloc(&path_list_index); path_list->fpl_flags = flags; - /* - * we'll assume for now all paths are the same next-hop protocol - */ - path_list->fpl_nh_proto = rpaths[0].frp_proto; - vec_foreach_index(i, rpaths) + if (NULL != rpaths) { - vec_add1(path_list->fpl_paths, - fib_path_create(path_list_index, - path_list->fpl_nh_proto, - fib_path_list_flags_2_path_flags(flags), - &rpaths[i])); + vec_foreach_index(i, rpaths) + { + vec_add1(path_list->fpl_paths, + fib_path_create(path_list_index, + &rpaths[i])); + } } /* @@ -748,6 +701,27 @@ fib_path_list_create (fib_path_list_flags_t flags, return (path_list_index); } +static fib_path_cfg_flags_t +fib_path_list_flags_2_path_flags (fib_path_list_flags_t plf) +{ + fib_path_cfg_flags_t pf = FIB_PATH_CFG_FLAG_NONE; + + if (plf & FIB_PATH_LIST_FLAG_DROP) + { + pf |= FIB_PATH_CFG_FLAG_DROP; + } + if (plf & FIB_PATH_LIST_FLAG_EXCLUSIVE) + { + pf |= FIB_PATH_CFG_FLAG_EXCLUSIVE; + } + if (plf & FIB_PATH_LIST_FLAG_LOCAL) + { + pf |= FIB_PATH_CFG_FLAG_LOCAL; + } + + return (pf); +} + fib_node_index_t fib_path_list_create_special (fib_protocol_t nh_proto, fib_path_list_flags_t flags, @@ -758,11 +732,10 @@ fib_path_list_create_special (fib_protocol_t nh_proto, path_list = fib_path_list_alloc(&path_list_index); path_list->fpl_flags = flags; - path_list->fpl_nh_proto = nh_proto; path_index = fib_path_create_special(path_list_index, - path_list->fpl_nh_proto, + nh_proto, fib_path_list_flags_2_path_flags(flags), dpo); vec_add1(path_list->fpl_paths, path_index); @@ -775,6 +748,30 @@ fib_path_list_create_special (fib_protocol_t nh_proto, return (path_list_index); } +/* + * return the index info the path-lists's vector of paths, of the matching path. + * ~0 if not found + */ +u32 +fib_path_list_find_rpath (fib_node_index_t path_list_index, + const fib_route_path_t *rpath) +{ + fib_path_list_t *path_list; + u32 ii; + + path_list = fib_path_list_get(path_list_index); + + vec_foreach_index (ii, path_list->fpl_paths) + { + if (!fib_path_cmp_w_route_path(path_list->fpl_paths[ii], rpath)) + { + return (ii); + } + } + return (~0); +} + + /* * fib_path_list_copy_and_path_add * @@ -782,13 +779,62 @@ fib_path_list_create_special (fib_protocol_t nh_proto, * The path-list returned could either have been newly created, or * can be a shared path-list from the data-base. */ +fib_node_index_t +fib_path_list_path_add (fib_node_index_t path_list_index, + const fib_route_path_t *rpaths) +{ + fib_node_index_t new_path_index, *orig_path_index; + fib_path_list_t *path_list; + + /* + * alloc the new list before we retrieve the old one, lest + * the alloc result in a realloc + */ + path_list = fib_path_list_get(path_list_index); + + ASSERT(1 == vec_len(rpaths)); + ASSERT(!(path_list->fpl_flags & FIB_PATH_LIST_FLAG_SHARED)); + + FIB_PATH_LIST_DBG(orig_path_list, "path-add"); + + new_path_index = fib_path_create(path_list_index, + rpaths); + + vec_foreach (orig_path_index, path_list->fpl_paths) + { + /* + * don't add duplicate paths + */ + if (0 == fib_path_cmp(new_path_index, *orig_path_index)) + { + return (*orig_path_index); + } + } + + /* + * Add the new path - no sort, no sharing, no key.. + */ + vec_add1(path_list->fpl_paths, new_path_index); + + FIB_PATH_LIST_DBG(path_list, "path-added"); + + /* + * no shared path list requested. resolve and use the one + * just created. + */ + fib_path_resolve(new_path_index); + + return (new_path_index); +} + fib_node_index_t fib_path_list_copy_and_path_add (fib_node_index_t orig_path_list_index, - fib_path_list_flags_t flags, - const fib_route_path_t *rpaths) + fib_path_list_flags_t flags, + const fib_route_path_t *rpaths) { fib_node_index_t path_index, new_path_index, *orig_path_index; fib_path_list_t *path_list, *orig_path_list; + fib_node_index_t exist_path_list_index; fib_node_index_t path_list_index; fib_node_index_t pi; @@ -806,13 +852,11 @@ fib_path_list_copy_and_path_add (fib_node_index_t orig_path_list_index, flags = fib_path_list_flags_fixup(flags); path_list->fpl_flags = flags; - path_list->fpl_nh_proto = orig_path_list->fpl_nh_proto; + vec_validate(path_list->fpl_paths, vec_len(orig_path_list->fpl_paths)); pi = 0; new_path_index = fib_path_create(path_list_index, - path_list->fpl_nh_proto, - fib_path_list_flags_2_path_flags(flags), rpaths); vec_foreach (orig_path_index, orig_path_list->fpl_paths) @@ -845,46 +889,79 @@ fib_path_list_copy_and_path_add (fib_node_index_t orig_path_list_index, FIB_PATH_LIST_DBG(path_list, "path-added"); /* - * If a shared path list is requested, consult the DB for a match + * check for a matching path-list in the DB. + * If we find one then we can return the existing one and destroy the + * new one just created. */ - if (path_list->fpl_flags & FIB_PATH_LIST_FLAG_SHARED) + exist_path_list_index = fib_path_list_db_find(path_list); + if (FIB_NODE_INDEX_INVALID != exist_path_list_index) { - fib_node_index_t exist_path_list_index; - /* - * check for a matching path-list in the DB. - * If we find one then we can return the existing one and destroy the - * new one just created. - */ - exist_path_list_index = fib_path_list_db_find(path_list); - if (FIB_NODE_INDEX_INVALID != exist_path_list_index) - { - fib_path_list_destroy(path_list); + fib_path_list_destroy(path_list); - path_list_index = exist_path_list_index; - } - else - { - /* - * if there was not a matching path-list, then this - * new one will need inserting into the DB and resolving. - */ - fib_path_list_db_insert(path_list_index); - - path_list = fib_path_list_resolve(path_list); - } + path_list_index = exist_path_list_index; } else { - /* - * no shared path list requested. resolve and use the one - * just created. - */ - path_list = fib_path_list_resolve(path_list); + /* + * if there was not a matching path-list, then this + * new one will need inserting into the DB and resolving. + */ + fib_path_list_db_insert(path_list_index); + + path_list = fib_path_list_resolve(path_list); } return (path_list_index); } +/* + * fib_path_list_path_remove + */ +fib_node_index_t +fib_path_list_path_remove (fib_node_index_t path_list_index, + const fib_route_path_t *rpaths) +{ + fib_node_index_t match_path_index, tmp_path_index; + fib_path_list_t *path_list; + fib_node_index_t pi; + + path_list = fib_path_list_get(path_list_index); + + ASSERT(1 == vec_len(rpaths)); + ASSERT(!(path_list->fpl_flags & FIB_PATH_LIST_FLAG_SHARED)); + + FIB_PATH_LIST_DBG(orig_path_list, "path-remove"); + + /* + * create a representation of the path to be removed, so it + * can be used as a comparison object during the copy. + */ + tmp_path_index = fib_path_create(path_list_index, + rpaths); + match_path_index = FIB_NODE_INDEX_INVALID; + + vec_foreach_index (pi, path_list->fpl_paths) + { + if (0 == fib_path_cmp(tmp_path_index, + path_list->fpl_paths[pi])) + { + /* + * match - remove it + */ + match_path_index = path_list->fpl_paths[pi]; + fib_path_destroy(match_path_index); + vec_del1(path_list->fpl_paths, pi); + } + } + + /* + * done with the temporary now + */ + fib_path_destroy(tmp_path_index); + + return (match_path_index); +} + /* * fib_path_list_copy_and_path_remove * @@ -911,7 +988,6 @@ fib_path_list_copy_and_path_remove (fib_node_index_t orig_path_list_index, FIB_PATH_LIST_DBG(orig_path_list, "copy-remove"); path_list->fpl_flags = flags; - path_list->fpl_nh_proto = orig_path_list->fpl_nh_proto; /* * allocate as many paths as we might need in one go, rather than * using vec_add to do a few at a time. @@ -927,8 +1003,6 @@ fib_path_list_copy_and_path_remove (fib_node_index_t orig_path_list_index, * can be used as a comparison object during the copy. */ tmp_path_index = fib_path_create(path_list_index, - path_list->fpl_nh_proto, - fib_path_list_flags_2_path_flags(flags), rpaths); vec_foreach (orig_path_index, orig_path_list->fpl_paths) diff --git a/src/vnet/fib/fib_path_list.h b/src/vnet/fib/fib_path_list.h index b4971add..9d246211 100644 --- a/src/vnet/fib/fib_path_list.h +++ b/src/vnet/fib/fib_path_list.h @@ -38,6 +38,11 @@ typedef enum fib_path_list_attribute_t_ { * be searched for each route update. */ FIB_PATH_LIST_ATTRIBUTE_SHARED = FIB_PATH_LIST_ATTRIBUTE_FIRST, + /** + * Indexed means the path-list keeps a hash table of all paths for + * fast lookup. The lookup result is the fib_node_index of the path. + */ + FIB_PATH_LIST_ATTRIBUTE_INDEXED, /** * explicit drop path-list. Used when the entry source needs to * force a drop, despite the fact the path info is present. @@ -73,6 +78,7 @@ typedef enum fib_path_list_attribute_t_ { typedef enum fib_path_list_flags_t_ { FIB_PATH_LIST_FLAG_NONE = 0, FIB_PATH_LIST_FLAG_SHARED = (1 << FIB_PATH_LIST_ATTRIBUTE_SHARED), + FIB_PATH_LIST_FLAG_INDEXED = (1 << FIB_PATH_LIST_ATTRIBUTE_INDEXED), FIB_PATH_LIST_FLAG_DROP = (1 << FIB_PATH_LIST_ATTRIBUTE_DROP), FIB_PATH_LIST_FLAG_LOCAL = (1 << FIB_PATH_LIST_ATTRIBUTE_LOCAL), FIB_PATH_LIST_FLAG_EXCLUSIVE = (1 << FIB_PATH_LIST_ATTRIBUTE_EXCLUSIVE), @@ -83,10 +89,11 @@ typedef enum fib_path_list_flags_t_ { #define FIB_PATH_LIST_ATTRIBUTES { \ [FIB_PATH_LIST_ATTRIBUTE_SHARED] = "shared", \ + [FIB_PATH_LIST_ATTRIBUTE_INDEXED] = "indexed", \ [FIB_PATH_LIST_ATTRIBUTE_RESOLVED] = "resolved", \ [FIB_PATH_LIST_ATTRIBUTE_DROP] = "drop", \ [FIB_PATH_LIST_ATTRIBUTE_EXCLUSIVE] = "exclusive", \ - [FIB_PATH_LIST_ATTRIBUTE_LOCAL] = "local", \ + [FIB_PATH_LIST_ATTRIBUTE_LOCAL] = "local", \ [FIB_PATH_LIST_ATTRIBUTE_LOOPED] = "looped", \ [FIB_PATH_LIST_ATTRIBUTE_NO_URPF] = "no-uRPF", \ } @@ -110,6 +117,13 @@ extern fib_node_index_t fib_path_list_copy_and_path_remove( fib_node_index_t pl_index, fib_path_list_flags_t flags, const fib_route_path_t *path); +extern fib_node_index_t fib_path_list_path_add ( + fib_node_index_t path_list_index, + const fib_route_path_t *rpaths); +extern fib_node_index_t fib_path_list_path_remove ( + fib_node_index_t path_list_index, + const fib_route_path_t *rpaths); + extern u32 fib_path_list_get_n_paths(fib_node_index_t pl_index); extern void fib_path_list_contribute_forwarding(fib_node_index_t path_list_index, @@ -137,11 +151,11 @@ extern int fib_path_list_is_looped(fib_node_index_t path_list_index); extern fib_protocol_t fib_path_list_get_proto(fib_node_index_t path_list_index); extern u8 * fib_path_list_format(fib_node_index_t pl_index, u8 * s); -extern u8 * fib_path_list_adjs_format(fib_node_index_t pl_index, - u32 indent, - u8 * s); extern index_t fib_path_list_lb_map_add_or_lock(fib_node_index_t pl_index, const fib_node_index_t *pis); +extern u32 fib_path_list_find_rpath (fib_node_index_t path_list_index, + const fib_route_path_t *rpath); + /** * A callback function type for walking a path-list's paths */ diff --git a/src/vnet/fib/fib_table.c b/src/vnet/fib/fib_table.c index 6c3162e7..b31f35e3 100644 --- a/src/vnet/fib/fib_table.c +++ b/src/vnet/fib/fib_table.c @@ -475,8 +475,21 @@ fib_table_entry_special_remove (u32 fib_index, */ static void fib_table_route_path_fixup (const fib_prefix_t *prefix, + fib_entry_flag_t eflags, fib_route_path_t *path) { + /* + * not all zeros next hop && + * is recursive path && + * nexthop is same as the route's address + */ + if ((!ip46_address_is_zero(&path->frp_addr)) && + (~0 == path->frp_sw_if_index) && + (0 == ip46_address_cmp(&path->frp_addr, &prefix->fp_addr))) + { + /* Prefix recurses via itse;f */ + path->frp_flags |= FIB_ROUTE_PATH_DROP; + } if (fib_prefix_is_host(prefix) && ip46_address_is_zero(&path->frp_addr) && path->frp_sw_if_index != ~0) @@ -484,7 +497,19 @@ fib_table_route_path_fixup (const fib_prefix_t *prefix, path->frp_addr = prefix->fp_addr; path->frp_flags |= FIB_ROUTE_PATH_ATTACHED; } -} + if (eflags & FIB_ENTRY_FLAG_DROP) + { + path->frp_flags |= FIB_ROUTE_PATH_DROP; + } + if (eflags & FIB_ENTRY_FLAG_LOCAL) + { + path->frp_flags |= FIB_ROUTE_PATH_LOCAL; + } + if (eflags & FIB_ENTRY_FLAG_EXCLUSIVE) + { + path->frp_flags |= FIB_ROUTE_PATH_EXCLUSIVE; + } +} fib_node_index_t fib_table_entry_path_add (u32 fib_index, @@ -536,7 +561,7 @@ fib_table_entry_path_add2 (u32 fib_index, for (ii = 0; ii < vec_len(rpath); ii++) { - fib_table_route_path_fixup(prefix, &rpath[ii]); + fib_table_route_path_fixup(prefix, flags, &rpath[ii]); } if (FIB_NODE_INDEX_INVALID == fib_entry_index) @@ -583,11 +608,6 @@ fib_table_entry_path_remove2 (u32 fib_index, fib_table = fib_table_get(fib_index, prefix->fp_proto); fib_entry_index = fib_table_lookup_exact_match_i(fib_table, prefix); - for (ii = 0; ii < vec_len(rpath); ii++) - { - fib_table_route_path_fixup(prefix, &rpath[ii]); - } - if (FIB_NODE_INDEX_INVALID == fib_entry_index) { /* @@ -605,6 +625,15 @@ fib_table_entry_path_remove2 (u32 fib_index, fib_entry_lock(fib_entry_index); was_sourced = fib_entry_is_sourced(fib_entry_index, source); + for (ii = 0; ii < vec_len(rpath); ii++) + { + fib_table_route_path_fixup( + prefix, + fib_entry_get_flags_for_source(fib_entry_index, + source), + &rpath[ii]); + } + src_flag = fib_entry_path_remove(fib_entry_index, source, rpath); if (!(FIB_ENTRY_SRC_FLAG_ADDED & src_flag)) @@ -661,7 +690,6 @@ fib_table_entry_path_remove (u32 fib_index, }; fib_route_path_t *paths = NULL; - fib_table_route_path_fixup(prefix, &path); vec_add1(paths, path); fib_table_entry_path_remove2(fib_index, prefix, source, paths); @@ -692,7 +720,7 @@ fib_table_entry_update (u32 fib_index, for (ii = 0; ii < vec_len(paths); ii++) { - fib_table_route_path_fixup(prefix, &paths[ii]); + fib_table_route_path_fixup(prefix, flags, &paths[ii]); } /* * sort the paths provided by the control plane. this means @@ -750,7 +778,6 @@ fib_table_entry_update_one_path (u32 fib_index, }; fib_route_path_t *paths = NULL; - fib_table_route_path_fixup(prefix, &path); vec_add1(paths, path); fib_entry_index = diff --git a/src/vnet/fib/fib_test.c b/src/vnet/fib/fib_test.c index 3c9b8a38..e4a8a70e 100644 --- a/src/vnet/fib/fib_test.c +++ b/src/vnet/fib/fib_test.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include @@ -271,6 +273,7 @@ typedef enum fib_test_lb_bucket_type_t_ { FT_LB_O_LB, FT_LB_SPECIAL, FT_LB_ADJ, + FT_LB_INTF, } fib_test_lb_bucket_type_t; typedef struct fib_test_lb_bucket_t_ { @@ -315,6 +318,31 @@ typedef struct fib_test_lb_bucket_t_ { }; } fib_test_lb_bucket_t; +typedef enum fib_test_rep_bucket_type_t_ { + FT_REP_LABEL_O_ADJ, + FT_REP_DISP_MFIB_LOOKUP, + FT_REP_INTF, +} fib_test_rep_bucket_type_t; + +typedef struct fib_test_rep_bucket_t_ { + fib_test_rep_bucket_type_t type; + + union + { + struct + { + mpls_eos_bit_t eos; + mpls_label_t label; + u8 ttl; + adj_index_t adj; + } label_o_adj; + struct + { + adj_index_t adj; + } adj; + }; +} fib_test_rep_bucket_t; + #define FIB_TEST_LB(_cond, _comment, _args...) \ { \ if (!FIB_TEST_I(_cond, _comment, ##_args)) { \ @@ -322,7 +350,83 @@ typedef struct fib_test_lb_bucket_t_ { } \ } -static int +int +fib_test_validate_rep_v (const replicate_t *rep, + u16 n_buckets, + va_list ap) +{ + const fib_test_rep_bucket_t *exp; + const dpo_id_t *dpo; + int bucket; + + FIB_TEST_LB((n_buckets == rep->rep_n_buckets), + "n_buckets = %d", rep->rep_n_buckets); + + for (bucket = 0; bucket < n_buckets; bucket++) + { + exp = va_arg(ap, fib_test_rep_bucket_t*); + + dpo = replicate_get_bucket_i(rep, bucket); + + switch (exp->type) + { + case FT_REP_LABEL_O_ADJ: + { + const mpls_label_dpo_t *mld; + mpls_label_t hdr; + FIB_TEST_LB((DPO_MPLS_LABEL == dpo->dpoi_type), + "bucket %d stacks on %U", + bucket, + format_dpo_type, dpo->dpoi_type); + + mld = mpls_label_dpo_get(dpo->dpoi_index); + hdr = clib_net_to_host_u32(mld->mld_hdr[0].label_exp_s_ttl); + + FIB_TEST_LB((vnet_mpls_uc_get_label(hdr) == + exp->label_o_adj.label), + "bucket %d stacks on label %d", + bucket, + exp->label_o_adj.label); + + FIB_TEST_LB((vnet_mpls_uc_get_s(hdr) == + exp->label_o_adj.eos), + "bucket %d stacks on label %d %U", + bucket, + exp->label_o_adj.label, + format_mpls_eos_bit, exp->label_o_adj.eos); + + FIB_TEST_LB((DPO_ADJACENCY_INCOMPLETE == mld->mld_dpo.dpoi_type), + "bucket %d label stacks on %U", + bucket, + format_dpo_type, mld->mld_dpo.dpoi_type); + + FIB_TEST_LB((exp->label_o_adj.adj == mld->mld_dpo.dpoi_index), + "bucket %d label stacks on adj %d", + bucket, + exp->label_o_adj.adj); + } + break; + case FT_REP_INTF: + FIB_TEST_LB((DPO_INTERFACE == dpo->dpoi_type), + "bucket %d stacks on %U", + bucket, + format_dpo_type, dpo->dpoi_type); + + FIB_TEST_LB((exp->adj.adj == dpo->dpoi_index), + "bucket %d stacks on adj %d", + bucket, + exp->adj.adj); + break; + case FT_REP_DISP_MFIB_LOOKUP: +// ASSERT(0); + break; + } + } + + return (!0); +} + +int fib_test_validate_lb_v (const load_balance_t *lb, u16 n_buckets, va_list ap) @@ -484,6 +588,16 @@ fib_test_validate_lb_v (const load_balance_t *lb, bucket, exp->adj.adj); break; + case FT_LB_INTF: + FIB_TEST_I((DPO_INTERFACE == dpo->dpoi_type), + "bucket %d stacks on %U", + bucket, + format_dpo_type, dpo->dpoi_type); + FIB_TEST_LB((exp->adj.adj == dpo->dpoi_index), + "bucket %d stacks on adj %d", + bucket, + exp->adj.adj); + break; case FT_LB_O_LB: FIB_TEST_I((DPO_LOAD_BALANCE == dpo->dpoi_type), "bucket %d stacks on %U", @@ -509,14 +623,13 @@ fib_test_validate_lb_v (const load_balance_t *lb, return (!0); } -static int +int fib_test_validate_entry (fib_node_index_t fei, fib_forward_chain_type_t fct, u16 n_buckets, ...) { dpo_id_t dpo = DPO_INVALID; - const load_balance_t *lb; fib_prefix_t pfx; index_t fw_lbi; u32 fib_index; @@ -529,47 +642,59 @@ fib_test_validate_entry (fib_node_index_t fei, fib_index = fib_entry_get_fib_index(fei); fib_entry_contribute_forwarding(fei, fct, &dpo); - FIB_TEST_LB((DPO_LOAD_BALANCE == dpo.dpoi_type), - "Entry links to %U", - format_dpo_type, dpo.dpoi_type); - lb = load_balance_get(dpo.dpoi_index); - - res = fib_test_validate_lb_v(lb, n_buckets, ap); + if (DPO_REPLICATE == dpo.dpoi_type) + { + const replicate_t *rep; - /* - * ensure that the LB contributed by the entry is the - * same as the LB in the forwarding tables - */ - if (fct == fib_entry_get_default_chain_type(fib_entry_get(fei))) + rep = replicate_get(dpo.dpoi_index); + res = fib_test_validate_rep_v(rep, n_buckets, ap); + } + else { - switch (pfx.fp_proto) - { - case FIB_PROTOCOL_IP4: - fw_lbi = ip4_fib_forwarding_lookup(fib_index, &pfx.fp_addr.ip4); - break; - case FIB_PROTOCOL_IP6: - fw_lbi = ip6_fib_table_fwding_lookup(&ip6_main, fib_index, &pfx.fp_addr.ip6); - break; - case FIB_PROTOCOL_MPLS: - { - mpls_unicast_header_t hdr = { - .label_exp_s_ttl = 0, - }; + const load_balance_t *lb; + + FIB_TEST_LB((DPO_LOAD_BALANCE == dpo.dpoi_type), + "Entry links to %U", + format_dpo_type, dpo.dpoi_type); - vnet_mpls_uc_set_label(&hdr.label_exp_s_ttl, pfx.fp_label); - vnet_mpls_uc_set_s(&hdr.label_exp_s_ttl, pfx.fp_eos); - hdr.label_exp_s_ttl = clib_host_to_net_u32(hdr.label_exp_s_ttl); + lb = load_balance_get(dpo.dpoi_index); + res = fib_test_validate_lb_v(lb, n_buckets, ap); - fw_lbi = mpls_fib_table_forwarding_lookup(fib_index, &hdr); + /* + * ensure that the LB contributed by the entry is the + * same as the LB in the forwarding tables + */ + if (fct == fib_entry_get_default_chain_type(fib_entry_get(fei))) + { + switch (pfx.fp_proto) + { + case FIB_PROTOCOL_IP4: + fw_lbi = ip4_fib_forwarding_lookup(fib_index, &pfx.fp_addr.ip4); + break; + case FIB_PROTOCOL_IP6: + fw_lbi = ip6_fib_table_fwding_lookup(&ip6_main, fib_index, &pfx.fp_addr.ip6); break; + case FIB_PROTOCOL_MPLS: + { + mpls_unicast_header_t hdr = { + .label_exp_s_ttl = 0, + }; + + vnet_mpls_uc_set_label(&hdr.label_exp_s_ttl, pfx.fp_label); + vnet_mpls_uc_set_s(&hdr.label_exp_s_ttl, pfx.fp_eos); + hdr.label_exp_s_ttl = clib_host_to_net_u32(hdr.label_exp_s_ttl); + + fw_lbi = mpls_fib_table_forwarding_lookup(fib_index, &hdr); + break; + } + default: + fw_lbi = 0; } - default: - fw_lbi = 0; + FIB_TEST_LB((fw_lbi == dpo.dpoi_index), + "Contributed LB = FW LB: %U\n %U", + format_load_balance, fw_lbi, 0, + format_load_balance, dpo.dpoi_index, 0); } - FIB_TEST_LB((fw_lbi == dpo.dpoi_index), - "Contributed LB = FW LB: %U\n %U", - format_load_balance, fw_lbi, 0, - format_load_balance, dpo.dpoi_index, 0); } dpo_reset(&dpo); @@ -1289,6 +1414,7 @@ fib_test_v4 (void) lookup_dpo_add_or_lock_w_fib_index(fib_index, DPO_PROTO_IP4, + LOOKUP_UNICAST, LOOKUP_INPUT_DST_ADDR, LOOKUP_TABLE_FROM_CONFIG, &ex_dpo); @@ -2605,7 +2731,6 @@ fib_test_v4 (void) NULL, FIB_ROUTE_PATH_FLAG_NONE); - fei = fib_table_lookup(fib_index, &pfx_5_5_5_6_s_32); dpo1 = fib_entry_contribute_ip_forwarding(fei); @@ -7493,6 +7618,7 @@ lfib_test (void) fib_route_path_t *rpaths = NULL, rpath = { .frp_proto = FIB_PROTOCOL_MPLS, .frp_local_label = 1200, + .frp_eos = MPLS_NON_EOS, .frp_sw_if_index = ~0, // recurive .frp_fib_index = 0, // Default MPLS fib .frp_weight = 1, @@ -7607,6 +7733,146 @@ lfib_test (void) dpo_reset(&ip_1200); + /* + * An rx-interface route. + * like the tail of an mcast LSP + */ + dpo_id_t idpo = DPO_INVALID; + + interface_dpo_add_or_lock(DPO_PROTO_IP4, + tm->hw[0]->sw_if_index, + &idpo); + + fib_prefix_t pfx_2500 = { + .fp_len = 21, + .fp_proto = FIB_PROTOCOL_MPLS, + .fp_label = 2500, + .fp_eos = MPLS_EOS, + .fp_payload_proto = DPO_PROTO_IP4, + }; + fib_test_lb_bucket_t rx_intf_0 = { + .type = FT_LB_INTF, + .adj = { + .adj = idpo.dpoi_index, + }, + }; + + lfe = fib_table_entry_update_one_path(fib_index, + &pfx_2500, + FIB_SOURCE_API, + FIB_ENTRY_FLAG_NONE, + FIB_PROTOCOL_IP4, + NULL, + tm->hw[0]->sw_if_index, + ~0, // invalid fib index + 0, + NULL, + FIB_ROUTE_PATH_INTF_RX); + FIB_TEST(fib_test_validate_entry(lfe, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + 1, + &rx_intf_0), + "2500 rx-interface 0"); + fib_table_entry_delete(fib_index, &pfx_2500, FIB_SOURCE_API); + + /* + * An MPLS mulicast entry + */ + fib_prefix_t pfx_3500 = { + .fp_len = 21, + .fp_proto = FIB_PROTOCOL_MPLS, + .fp_label = 3500, + .fp_eos = MPLS_EOS, + .fp_payload_proto = DPO_PROTO_IP4, + }; + fib_test_rep_bucket_t mc_0 = { + .type = FT_REP_LABEL_O_ADJ, + .label_o_adj = { + .adj = ai_mpls_10_10_10_1, + .label = 3300, + .eos = MPLS_EOS, + }, + }; + fib_test_rep_bucket_t mc_intf_0 = { + .type = FT_REP_INTF, + .adj = { + .adj = idpo.dpoi_index, + }, + }; + mpls_label_t *l3300 = NULL; + vec_add1(l3300, 3300); + + lfe = fib_table_entry_update_one_path(lfib_index, + &pfx_3500, + FIB_SOURCE_API, + FIB_ENTRY_FLAG_MULTICAST, + FIB_PROTOCOL_IP4, + &nh_10_10_10_1, + tm->hw[0]->sw_if_index, + ~0, // invalid fib index + 1, + l3300, + FIB_ROUTE_PATH_FLAG_NONE); + FIB_TEST(fib_test_validate_entry(lfe, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + 1, + &mc_0), + "3500 via replicate over 10.10.10.1"); + + /* + * MPLS Bud-node. Add a replication via an interface-receieve path + */ + lfe = fib_table_entry_path_add(lfib_index, + &pfx_3500, + FIB_SOURCE_API, + FIB_ENTRY_FLAG_MULTICAST, + FIB_PROTOCOL_IP4, + NULL, + tm->hw[0]->sw_if_index, + ~0, // invalid fib index + 0, + NULL, + FIB_ROUTE_PATH_INTF_RX); + FIB_TEST(fib_test_validate_entry(lfe, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + 2, + &mc_0, + &mc_intf_0), + "3500 via replicate over 10.10.10.1 and interface-rx"); + + /* + * Add a replication via an interface-free for-us path + */ + fib_test_rep_bucket_t mc_disp = { + .type = FT_REP_DISP_MFIB_LOOKUP, + .adj = { + .adj = idpo.dpoi_index, + }, + }; + lfe = fib_table_entry_path_add(lfib_index, + &pfx_3500, + FIB_SOURCE_API, + FIB_ENTRY_FLAG_MULTICAST, + FIB_PROTOCOL_IP4, + NULL, + 5, // rpf-id + 0, // default table + 0, + NULL, + FIB_ROUTE_PATH_RPF_ID); + FIB_TEST(fib_test_validate_entry(lfe, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + 3, + &mc_0, + &mc_disp, + &mc_intf_0), + "3500 via replicate over 10.10.10.1 and interface-rx"); + + + + fib_table_entry_delete(fib_index, &pfx_3500, FIB_SOURCE_API); + dpo_reset(&idpo); + /* * cleanup */ @@ -7617,6 +7883,9 @@ lfib_test (void) FIB_TEST(lb_count == pool_elts(load_balance_pool), "Load-balance resources freed %d of %d", lb_count, pool_elts(load_balance_pool)); + FIB_TEST(0 == pool_elts(interface_dpo_pool), + "interface_dpo resources freed %d of %d", + 0, pool_elts(interface_dpo_pool)); return (0); } diff --git a/src/vnet/fib/fib_test.h b/src/vnet/fib/fib_test.h new file mode 100644 index 00000000..b98680bf --- /dev/null +++ b/src/vnet/fib/fib_test.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __FIB_TEST_H__ +#define __FIB_TEST_H__ + +#include + +typedef enum fib_test_lb_bucket_type_t_ { + FT_LB_LABEL_O_ADJ, + FT_LB_LABEL_STACK_O_ADJ, + FT_LB_LABEL_O_LB, + FT_LB_O_LB, + FT_LB_SPECIAL, + FT_LB_ADJ, + FT_LB_INTF, +} fib_test_lb_bucket_type_t; + +typedef struct fib_test_lb_bucket_t_ { + fib_test_lb_bucket_type_t type; + + union + { + struct + { + mpls_eos_bit_t eos; + mpls_label_t label; + u8 ttl; + adj_index_t adj; + } label_o_adj; + struct + { + mpls_eos_bit_t eos; + mpls_label_t label_stack[8]; + u8 label_stack_size; + u8 ttl; + adj_index_t adj; + } label_stack_o_adj; + struct + { + mpls_eos_bit_t eos; + mpls_label_t label; + u8 ttl; + index_t lb; + } label_o_lb; + struct + { + index_t adj; + } adj; + struct + { + index_t lb; + } lb; + struct + { + index_t adj; + } special; + }; +} fib_test_lb_bucket_t; + +typedef enum fib_test_rep_bucket_type_t_ { + FT_REP_LABEL_O_ADJ, + FT_REP_INTF, +} fib_test_rep_bucket_type_t; + +typedef struct fib_test_rep_bucket_t_ { + fib_test_rep_bucket_type_t type; + + union + { + struct + { + mpls_eos_bit_t eos; + mpls_label_t label; + u8 ttl; + adj_index_t adj; + } label_o_adj; + struct + { + adj_index_t adj; + } adj; + }; +} fib_test_rep_bucket_t; + + +extern int fib_test_validate_rep_v(const replicate_t *rep, + u16 n_buckets, + va_list ap); + +extern int fib_test_validate_lb_v(const load_balance_t *lb, + u16 n_buckets, + va_list ap); + +extern int fib_test_validate_entry(fib_node_index_t fei, + fib_forward_chain_type_t fct, + u16 n_buckets, + ...); + +#endif diff --git a/src/vnet/fib/fib_types.c b/src/vnet/fib/fib_types.c index 2837a59d..8165f3eb 100644 --- a/src/vnet/fib/fib_types.c +++ b/src/vnet/fib/fib_types.c @@ -66,12 +66,13 @@ fib_prefix_from_ip46_addr (const ip46_address_t *addr, void fib_prefix_from_mpls_label (mpls_label_t label, + mpls_eos_bit_t eos, fib_prefix_t *pfx) { pfx->fp_proto = FIB_PROTOCOL_MPLS; pfx->fp_len = 21; pfx->fp_label = label; - pfx->fp_eos = MPLS_NON_EOS; + pfx->fp_eos = eos; } int @@ -194,17 +195,7 @@ fib_route_path_cmp (const fib_route_path_t *rpath1, if (0 != res) return (res); - if (~0 != rpath1->frp_sw_if_index && - ~0 != rpath2->frp_sw_if_index) - { - res = vnet_sw_interface_compare(vnet_get_main(), - rpath1->frp_sw_if_index, - rpath2->frp_sw_if_index); - } - else - { - res = rpath1->frp_sw_if_index - rpath2->frp_sw_if_index; - } + res = (rpath1->frp_sw_if_index - rpath2->frp_sw_if_index); if (0 != res) return (res); diff --git a/src/vnet/fib/fib_types.h b/src/vnet/fib/fib_types.h index 1c5299a9..4cb73e8a 100644 --- a/src/vnet/fib/fib_types.h +++ b/src/vnet/fib/fib_types.h @@ -286,8 +286,36 @@ typedef enum fib_route_path_flags_t_ * Attached path */ FIB_ROUTE_PATH_ATTACHED = (1 << 3), + /** + * A Drop path - resolve the path on the drop DPO + */ + FIB_ROUTE_PATH_DROP = (1 << 4), + /** + * Don't resolve the path, use the DPO the client provides + */ + FIB_ROUTE_PATH_EXCLUSIVE = (1 << 5), + /** + * A path that result in received traffic being recieved/recirculated + * so that it appears to have arrived on the new interface + */ + FIB_ROUTE_PATH_INTF_RX = (1 << 6), + /** + * A local path with a RPF-ID => multicast traffic + */ + FIB_ROUTE_PATH_RPF_ID = (1 << 7), } fib_route_path_flags_t; +/** + * An RPF-ID is numerical value that is used RPF validate. An entry + * has-a RPF-ID, when a packet egress from (e.g. an LSP) it gains an + * RPF-ID, these two are compared for the RPF check. + * This replaces the interfce based chack (since the LSP has no associated + * interface. + */ +typedef u32 fib_rpf_id_t; + +#define MFIB_RPF_ID_NONE (0) + /** * @brief * A representation of a path as described by a route producer. @@ -321,17 +349,29 @@ typedef struct fib_route_path_t_ { */ ip46_address_t frp_addr; - /** - * The MPLS local Label to reursively resolve through. - * This is valid when the path type is MPLS. - */ - mpls_label_t frp_local_label; + struct { + /** + * The MPLS local Label to reursively resolve through. + * This is valid when the path type is MPLS. + */ + mpls_label_t frp_local_label; + /** + * EOS bit for the resolving label + */ + mpls_eos_bit_t frp_eos; + }; + }; + union { + /** + * The interface. + * Will be invalid for recursive paths. + */ + u32 frp_sw_if_index; + /** + * The RPF-ID + */ + fib_rpf_id_t frp_rpf_id; }; - /** - * The interface. - * Will be invalid for recursive paths. - */ - u32 frp_sw_if_index; /** * The FIB index to lookup the nexthop * Only valid for recursive paths. diff --git a/src/vnet/fib/mpls_fib.c b/src/vnet/fib/mpls_fib.c index 4b2b76ea..19f9f3c1 100644 --- a/src/vnet/fib/mpls_fib.c +++ b/src/vnet/fib/mpls_fib.c @@ -165,6 +165,7 @@ mpls_fib_create_with_table_id (u32 table_id) lookup_dpo_add_or_lock_w_fib_index(0, // unused DPO_PROTO_IP4, + LOOKUP_UNICAST, LOOKUP_INPUT_DST_ADDR, LOOKUP_TABLE_FROM_INPUT_INTERFACE, &dpo); @@ -179,6 +180,7 @@ mpls_fib_create_with_table_id (u32 table_id) lookup_dpo_add_or_lock_w_fib_index(0, //unsued DPO_PROTO_MPLS, + LOOKUP_UNICAST, LOOKUP_INPUT_DST_ADDR, LOOKUP_TABLE_FROM_INPUT_INTERFACE, &dpo); @@ -197,6 +199,7 @@ mpls_fib_create_with_table_id (u32 table_id) lookup_dpo_add_or_lock_w_fib_index(0, //unused DPO_PROTO_IP6, + LOOKUP_UNICAST, LOOKUP_INPUT_DST_ADDR, LOOKUP_TABLE_FROM_INPUT_INTERFACE, &dpo); @@ -210,6 +213,7 @@ mpls_fib_create_with_table_id (u32 table_id) prefix.fp_eos = MPLS_NON_EOS; lookup_dpo_add_or_lock_w_fib_index(0, // unsued DPO_PROTO_MPLS, + LOOKUP_UNICAST, LOOKUP_INPUT_DST_ADDR, LOOKUP_TABLE_FROM_INPUT_INTERFACE, &dpo); @@ -320,8 +324,15 @@ mpls_fib_forwarding_table_update (mpls_fib_t *mf, { mpls_label_t key; - ASSERT(DPO_LOAD_BALANCE == dpo->dpoi_type); - + ASSERT((DPO_LOAD_BALANCE == dpo->dpoi_type) || + (DPO_REPLICATE == dpo->dpoi_type)); + if (CLIB_DEBUG > 0) + { + if (DPO_REPLICATE == dpo->dpoi_type) + ASSERT(dpo->dpoi_index & MPLS_IS_REPLICATE); + if (DPO_LOAD_BALANCE == dpo->dpoi_type) + ASSERT(!(dpo->dpoi_index & MPLS_IS_REPLICATE)); + } key = mpls_fib_entry_mk_key(label, eos); mf->mf_lbs[key] = dpo->dpoi_index; diff --git a/src/vnet/handoff.h b/src/vnet/handoff.h index 815206a9..04ba8bfb 100644 --- a/src/vnet/handoff.h +++ b/src/vnet/handoff.h @@ -150,7 +150,7 @@ eth_get_sym_key (ethernet_header_t * h0) ip->dst_address.as_u64[0] ^ ip->dst_address.as_u64[1] ^ ip->protocol); } - else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST)) + else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) { hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1)); } @@ -179,8 +179,7 @@ eth_get_sym_key (ethernet_header_t * h0) ip->dst_address.as_u64[0] ^ ip->dst_address.as_u64[1] ^ ip->protocol); } - else if (outer->type == - clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST)) + else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) { hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1)); } @@ -210,7 +209,7 @@ eth_get_key (ethernet_header_t * h0) { hash_key = ipv6_get_key ((ip6_header_t *) (h0 + 1)); } - else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST)) + else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) { hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1)); } @@ -230,8 +229,7 @@ eth_get_key (ethernet_header_t * h0) { hash_key = ipv6_get_key ((ip6_header_t *) (outer + 1)); } - else if (outer->type == - clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST)) + else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) { hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1)); } diff --git a/src/vnet/interface.c b/src/vnet/interface.c index 2a1e70e8..45417b2f 100644 --- a/src/vnet/interface.c +++ b/src/vnet/interface.c @@ -1360,7 +1360,7 @@ vnet_link_to_l3_proto (vnet_link_t link) case VNET_LINK_IP6: return (VNET_L3_PACKET_TYPE_IP6); case VNET_LINK_MPLS: - return (VNET_L3_PACKET_TYPE_MPLS_UNICAST); + return (VNET_L3_PACKET_TYPE_MPLS); case VNET_LINK_ARP: return (VNET_L3_PACKET_TYPE_ARP); case VNET_LINK_ETHERNET: diff --git a/src/vnet/ip/ip.api b/src/vnet/ip/ip.api index 5c2df32c..6af1714f 100644 --- a/src/vnet/ip/ip.api +++ b/src/vnet/ip/ip.api @@ -478,6 +478,7 @@ define ip_mroute_add_del u32 table_id; u32 entry_flags; u32 itf_flags; + u32 rpf_id; u16 grp_address_length; u8 create_vrf_if_needed; u8 is_add; @@ -518,6 +519,8 @@ manual_endian manual_print define ip_mfib_details { u32 context; u32 table_id; + u32 entry_flags; + u32 rpf_id; u8 address_length; u8 grp_address[4]; u8 src_address[4]; diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c index fdfe7f63..9fdf9b3c 100644 --- a/src/vnet/ip/ip4_forward.c +++ b/src/vnet/ip/ip4_forward.c @@ -2752,6 +2752,16 @@ ip4_rewrite_mcast (vlib_main_t * vm, return ip4_rewrite_inline (vm, node, frame, 0, 0, 1); } +static uword +ip4_mcast_midchain (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + if (adj_are_counters_enabled ()) + return ip4_rewrite_inline (vm, node, frame, 1, 1, 1); + else + return ip4_rewrite_inline (vm, node, frame, 0, 1, 1); +} + /* *INDENT-OFF* */ VLIB_REGISTER_NODE (ip4_rewrite_node) = { .function = ip4_rewrite, @@ -2778,6 +2788,16 @@ VLIB_REGISTER_NODE (ip4_rewrite_mcast_node) = { }; VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_mcast_node, ip4_rewrite_mcast) +VLIB_REGISTER_NODE (ip4_mcast_midchain_node, static) = { + .function = ip4_mcast_midchain, + .name = "ip4-mcast-midchain", + .vector_size = sizeof (u32), + + .format_trace = format_ip4_rewrite_trace, + .sibling_of = "ip4-rewrite", +}; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_mcast_midchain_node, ip4_mcast_midchain) + VLIB_REGISTER_NODE (ip4_midchain_node) = { .function = ip4_midchain, .name = "ip4-midchain", diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index c2fc4f87..a369f79f 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -2246,6 +2246,16 @@ ip6_midchain (vlib_main_t * vm, return ip6_rewrite_inline (vm, node, frame, 0, 1, 0); } +static uword +ip6_mcast_midchain (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + if (adj_are_counters_enabled ()) + return ip6_rewrite_inline (vm, node, frame, 1, 1, 1); + else + return ip6_rewrite_inline (vm, node, frame, 1, 1, 1); +} + /* *INDENT-OFF* */ VLIB_REGISTER_NODE (ip6_midchain_node) = { @@ -2290,6 +2300,19 @@ VLIB_REGISTER_NODE (ip6_rewrite_mcast_node) = VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_mcast_node, ip6_rewrite_mcast); +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (ip6_mcast_midchain_node, static) = +{ + .function = ip6_mcast_midchain, + .name = "ip6-mcast-midchain", + .vector_size = sizeof (u32), + .format_trace = format_ip6_rewrite_trace, + .sibling_of = "ip6-rewrite", +}; +/* *INDENT-ON* */ + +VLIB_NODE_FUNCTION_MULTIARCH (ip6_mcast_midchain_node, ip6_mcast_midchain); + /* * Hop-by-Hop handling */ diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c index 2af546df..58b997aa 100644 --- a/src/vnet/ip/ip6_neighbor.c +++ b/src/vnet/ip/ip6_neighbor.c @@ -557,6 +557,7 @@ ip6_ethernet_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai) case IP_LOOKUP_NEXT_PUNT: case IP_LOOKUP_NEXT_LOCAL: case IP_LOOKUP_NEXT_REWRITE: + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: case IP_LOOKUP_NEXT_MIDCHAIN: case IP_LOOKUP_NEXT_ICMP_ERROR: case IP_LOOKUP_N_NEXT: diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c index b9f1782b..9c9cb4a4 100644 --- a/src/vnet/ip/ip_api.c +++ b/src/vnet/ip/ip_api.c @@ -438,17 +438,20 @@ vl_api_ip6_fib_dump_t_handler (vl_api_ip6_fib_dump_t * mp) } static void -send_ip_mfib_details (vpe_api_main_t * am, - unix_shared_memory_queue_t * q, - u32 table_id, - mfib_prefix_t * pfx, - fib_route_path_encode_t * api_rpaths, u32 context) +send_ip_mfib_details (unix_shared_memory_queue_t * q, + u32 context, u32 table_id, fib_node_index_t mfei) { + fib_route_path_encode_t *api_rpath, *api_rpaths = NULL; vl_api_ip_mfib_details_t *mp; - fib_route_path_encode_t *api_rpath; + mfib_entry_t *mfib_entry; vl_api_fib_path_t *fp; + mfib_prefix_t pfx; int path_count; + mfib_entry = mfib_entry_get (mfei); + mfib_entry_get_prefix (mfei, &pfx); + mfib_entry_encode (mfei, &api_rpaths); + path_count = vec_len (api_rpaths); mp = vl_msg_api_alloc (sizeof (*mp) + path_count * sizeof (*fp)); if (!mp) @@ -457,12 +460,14 @@ send_ip_mfib_details (vpe_api_main_t * am, mp->_vl_msg_id = ntohs (VL_API_IP_FIB_DETAILS); mp->context = context; + mp->rpf_id = mfib_entry->mfe_rpf_id; + mp->entry_flags = mfib_entry->mfe_flags; mp->table_id = htonl (table_id); - mp->address_length = pfx->fp_len; - memcpy (mp->grp_address, &pfx->fp_grp_addr.ip4, - sizeof (pfx->fp_grp_addr.ip4)); - memcpy (mp->src_address, &pfx->fp_src_addr.ip4, - sizeof (pfx->fp_src_addr.ip4)); + mp->address_length = pfx.fp_len; + memcpy (mp->grp_address, &pfx.fp_grp_addr.ip4, + sizeof (pfx.fp_grp_addr.ip4)); + memcpy (mp->src_address, &pfx.fp_src_addr.ip4, + sizeof (pfx.fp_src_addr.ip4)); mp->count = htonl (path_count); fp = mp->path; @@ -475,6 +480,7 @@ send_ip_mfib_details (vpe_api_main_t * am, copy_fib_next_hop (api_rpath, fp); fp++; } + vec_free (api_rpaths); vl_msg_api_send_shmem (q, (u8 *) & mp); } @@ -497,13 +503,10 @@ vl_api_ip_mfib_table_dump_walk (fib_node_index_t fei, void *arg) static void vl_api_ip_mfib_dump_t_handler (vl_api_ip_mfib_dump_t * mp) { - vpe_api_main_t *am = &vpe_api_main; unix_shared_memory_queue_t *q; ip4_main_t *im = &ip4_main; mfib_table_t *mfib_table; fib_node_index_t *mfeip; - mfib_prefix_t pfx; - fib_route_path_encode_t *api_rpaths = NULL; vl_api_ip_mfib_dump_ctc_t ctx = { .entries = NULL, }; @@ -524,21 +527,16 @@ vl_api_ip_mfib_dump_t_handler (vl_api_ip_mfib_dump_t * mp) vec_foreach (mfeip, ctx.entries) { - mfib_entry_get_prefix (*mfeip, &pfx); - mfib_entry_encode (*mfeip, &api_rpaths); - send_ip_mfib_details (am, q, + send_ip_mfib_details (q, mp->context, mfib_table->mft_table_id, - &pfx, api_rpaths, - mp->context); + *mfeip); } - vec_reset_length (api_rpaths); vec_reset_length (ctx.entries); })); /* *INDENT-ON* */ vec_free (ctx.entries); - vec_free (api_rpaths); } static void @@ -705,10 +703,13 @@ add_del_route_t_handler (u8 is_multipath, u8 is_unreach, u8 is_prohibit, u8 is_local, + u8 is_multicast, u8 is_classify, u32 classify_table_index, u8 is_resolve_host, u8 is_resolve_attached, + u8 is_interface_rx, + u8 is_rpf_id, u32 fib_index, const fib_prefix_t * prefix, u8 next_hop_proto_is_ip4, @@ -731,16 +732,24 @@ add_del_route_t_handler (u8 is_multipath, .frp_label_stack = next_hop_out_label_stack, }; fib_route_path_t *paths = NULL; + fib_entry_flag_t entry_flags = FIB_ENTRY_FLAG_NONE; if (MPLS_LABEL_INVALID != next_hop_via_label) { path.frp_proto = FIB_PROTOCOL_MPLS; path.frp_local_label = next_hop_via_label; + path.frp_eos = MPLS_NON_EOS; } if (is_resolve_host) path_flags |= FIB_ROUTE_PATH_RESOLVE_VIA_HOST; if (is_resolve_attached) path_flags |= FIB_ROUTE_PATH_RESOLVE_VIA_ATTACHED; + if (is_interface_rx) + path_flags |= FIB_ROUTE_PATH_INTF_RX; + if (is_rpf_id) + path_flags |= FIB_ROUTE_PATH_RPF_ID; + if (is_multicast) + entry_flags |= FIB_ENTRY_FLAG_MULTICAST; path.frp_flags = path_flags; @@ -754,8 +763,7 @@ add_del_route_t_handler (u8 is_multipath, if (is_add) fib_table_entry_path_add2 (fib_index, prefix, - FIB_SOURCE_API, - FIB_ENTRY_FLAG_NONE, paths); + FIB_SOURCE_API, entry_flags, paths); else fib_table_entry_path_remove2 (fib_index, prefix, FIB_SOURCE_API, paths); @@ -826,8 +834,7 @@ add_del_route_t_handler (u8 is_multipath, { vec_add1 (paths, path); fib_table_entry_update (fib_index, - prefix, - FIB_SOURCE_API, FIB_ENTRY_FLAG_NONE, paths); + prefix, FIB_SOURCE_API, entry_flags, paths); vec_free (paths); } else @@ -847,7 +854,7 @@ add_del_route_check (fib_protocol_t table_proto, fib_protocol_t next_hop_table_proto, u32 next_hop_table_id, u8 create_missing_tables, - u32 * fib_index, u32 * next_hop_fib_index) + u8 is_rpf_id, u32 * fib_index, u32 * next_hop_fib_index) { vnet_main_t *vnm = vnet_get_main (); @@ -866,7 +873,7 @@ add_del_route_check (fib_protocol_t table_proto, } } - if (~0 != ntohl (next_hop_sw_if_index)) + if (!is_rpf_id && ~0 != ntohl (next_hop_sw_if_index)) { if (pool_is_free_index (vnm->interface_main.sw_interfaces, ntohl (next_hop_sw_if_index))) @@ -876,16 +883,27 @@ add_del_route_check (fib_protocol_t table_proto, } else { - *next_hop_fib_index = fib_table_find (next_hop_table_proto, - ntohl (next_hop_table_id)); + if (is_rpf_id) + *next_hop_fib_index = mfib_table_find (next_hop_table_proto, + ntohl (next_hop_table_id)); + else + *next_hop_fib_index = fib_table_find (next_hop_table_proto, + ntohl (next_hop_table_id)); if (~0 == *next_hop_fib_index) { if (create_missing_tables) { - *next_hop_fib_index = - fib_table_find_or_create_and_lock (next_hop_table_proto, - ntohl (next_hop_table_id)); + if (is_rpf_id) + *next_hop_fib_index = + mfib_table_find_or_create_and_lock (next_hop_table_proto, + ntohl + (next_hop_table_id)); + else + *next_hop_fib_index = + fib_table_find_or_create_and_lock (next_hop_table_proto, + ntohl + (next_hop_table_id)); } else { @@ -910,7 +928,7 @@ ip4_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp) mp->next_hop_sw_if_index, FIB_PROTOCOL_IP4, mp->next_hop_table_id, - mp->create_vrf_if_needed, + mp->create_vrf_if_needed, 0, &fib_index, &next_hop_fib_index); if (0 != rv) @@ -943,11 +961,11 @@ ip4_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp) mp->is_drop, mp->is_unreach, mp->is_prohibit, - mp->is_local, + mp->is_local, 0, mp->is_classify, mp->classify_table_index, mp->is_resolve_host, - mp->is_resolve_attached, + mp->is_resolve_attached, 0, 0, fib_index, &pfx, 1, &nh, ntohl (mp->next_hop_sw_if_index), @@ -969,7 +987,7 @@ ip6_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp) mp->next_hop_sw_if_index, FIB_PROTOCOL_IP6, mp->next_hop_table_id, - mp->create_vrf_if_needed, + mp->create_vrf_if_needed, 0, &fib_index, &next_hop_fib_index); if (0 != rv) @@ -1002,11 +1020,11 @@ ip6_add_del_route_t_handler (vl_api_ip_add_del_route_t * mp) mp->is_drop, mp->is_unreach, mp->is_prohibit, - mp->is_local, + mp->is_local, 0, mp->is_classify, mp->classify_table_index, mp->is_resolve_host, - mp->is_resolve_attached, + mp->is_resolve_attached, 0, 0, fib_index, &pfx, 0, &nh, ntohl (mp->next_hop_sw_if_index), next_hop_fib_index, @@ -1075,6 +1093,7 @@ mroute_add_del_handler (u8 is_add, u32 fib_index, const mfib_prefix_t * prefix, u32 entry_flags, + fib_rpf_id_t rpf_id, u32 next_hop_sw_if_index, u32 itf_flags) { stats_dslock_with_hint (1 /* release hint */ , 2 /* tag */ ); @@ -1091,7 +1110,7 @@ mroute_add_del_handler (u8 is_add, if (!is_local && ~0 == next_hop_sw_if_index) { mfib_table_entry_update (fib_index, prefix, - MFIB_SOURCE_API, entry_flags); + MFIB_SOURCE_API, rpf_id, entry_flags); } else { @@ -1152,6 +1171,7 @@ api_mroute_add_del_t_handler (vl_api_ip_mroute_add_del_t * mp) mp->is_local, fib_index, &pfx, ntohl (mp->entry_flags), + ntohl (mp->rpf_id), ntohl (mp->next_hop_sw_if_index), ntohl (mp->itf_flags))); } diff --git a/src/vnet/ip/lookup.c b/src/vnet/ip/lookup.c index ec9a1f97..597de06b 100755 --- a/src/vnet/ip/lookup.c +++ b/src/vnet/ip/lookup.c @@ -450,6 +450,7 @@ vnet_ip_route_cmd (vlib_main_t * vm, unformat_mpls_unicast_label, &rpath.frp_local_label)) { rpath.frp_weight = 1; + rpath.frp_eos = MPLS_NON_EOS; rpath.frp_proto = FIB_PROTOCOL_MPLS; rpath.frp_sw_if_index = ~0; vec_add1 (rpaths, rpath); @@ -923,7 +924,7 @@ vnet_ip_mroute_cmd (vlib_main_t * vm, else if (eflags) { mfib_table_entry_update (fib_index, &pfx, MFIB_SOURCE_CLI, - eflags); + MFIB_RPF_ID_NONE, eflags); } else { diff --git a/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c b/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c index efa724e0..d2954e96 100644 --- a/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c +++ b/src/vnet/lisp-gpe/lisp_gpe_fwd_entry.c @@ -88,6 +88,7 @@ ip_dst_fib_add_route (u32 dst_fib_index, const ip_prefix_t * dst_prefix) (ip_prefix_version (dst_prefix) == IP6 ? DPO_PROTO_IP6 : DPO_PROTO_IP4), + LOOKUP_UNICAST, LOOKUP_INPUT_SRC_ADDR, LOOKUP_TABLE_FROM_CONFIG, &src_lkup_dpo); diff --git a/src/vnet/mfib/ip4_mfib.c b/src/vnet/mfib/ip4_mfib.c index 164cafa1..3ed7cba7 100644 --- a/src/vnet/mfib/ip4_mfib.c +++ b/src/vnet/mfib/ip4_mfib.c @@ -72,6 +72,7 @@ ip4_create_mfib_with_table_id (u32 table_id) mfib_table_entry_update(mfib_table->mft_index, &prefix, MFIB_SOURCE_DEFAULT_ROUTE, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_DROP); } diff --git a/src/vnet/mfib/ip6_mfib.c b/src/vnet/mfib/ip6_mfib.c index 991b91c6..116fee22 100644 --- a/src/vnet/mfib/ip6_mfib.c +++ b/src/vnet/mfib/ip6_mfib.c @@ -195,6 +195,7 @@ ip6_create_mfib_with_table_id (u32 table_id) mfib_table_entry_update(mfib_table->mft_index, &all_zeros, MFIB_SOURCE_DEFAULT_ROUTE, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_DROP); /* diff --git a/src/vnet/mfib/mfib_entry.c b/src/vnet/mfib/mfib_entry.c index 1aa8e086..847f25e7 100644 --- a/src/vnet/mfib/mfib_entry.c +++ b/src/vnet/mfib/mfib_entry.c @@ -48,6 +48,15 @@ #define MFIB_ENTRY_DBG(_e, _fmt, _args...) #endif +/** + * MFIB extensions to each path + */ +typedef struct mfib_path_ext_t_ +{ + mfib_itf_flags_t mfpe_flags; + fib_node_index_t mfpe_path; +} mfib_path_ext_t; + /** * The source of an MFIB entry */ @@ -58,22 +67,39 @@ typedef struct mfib_entry_src_t_ */ mfib_source_t mfes_src; + /** + * Route flags + */ + mfib_entry_flags_t mfes_flags; + /** * The path-list of forwarding interfaces */ fib_node_index_t mfes_pl; /** - * Route flags + * RPF-ID */ - mfib_entry_flags_t mfes_flags; + fib_rpf_id_t mfes_rpf_id; + + /** + * Hash table of path extensions + */ + mfib_path_ext_t *mfes_exts; /** - * The hash table of all interfaces + * The hash table of all interfaces. + * This is forwarding time information derived from the paths + * and their extensions. */ mfib_itf_t *mfes_itfs; } mfib_entry_src_t; +/** + * Pool of path extensions + */ +static mfib_path_ext_t *mfib_path_ext_pool; + /** * String names for each source */ @@ -123,6 +149,24 @@ format_mfib_entry_dpo (u8 * s, va_list * args) MFIB_ENTRY_FORMAT_BRIEF)); } +static inline mfib_path_ext_t * +mfib_entry_path_ext_get (index_t mi) +{ + return (pool_elt_at_index(mfib_path_ext_pool, mi)); +} + +static u8 * +format_mfib_entry_path_ext (u8 * s, va_list * args) +{ + mfib_path_ext_t *path_ext; + index_t mpi = va_arg(*args, index_t); + + path_ext = mfib_entry_path_ext_get(mpi); + return (format(s, "path:%d flags:%U", + path_ext->mfpe_path, + format_mfib_itf_flags, path_ext->mfpe_flags)); +} + u8 * format_mfib_entry (u8 * s, va_list * args) { @@ -141,6 +185,8 @@ format_mfib_entry (u8 * s, va_list * args) if (level >= MFIB_ENTRY_FORMAT_DETAIL) { + fib_node_index_t path_index, mpi; + s = format (s, "\n"); s = format (s, " fib:%d", mfib_entry->mfe_fib_index); s = format (s, " index:%d", mfib_entry_get_index(mfib_entry)); @@ -153,6 +199,14 @@ format_mfib_entry (u8 * s, va_list * args) { s = fib_path_list_format(msrc->mfes_pl, s); } + s = format (s, " Extensions:\n", + mfib_source_names[msrc->mfes_src]); + hash_foreach(path_index, mpi, msrc->mfes_exts, + ({ + s = format(s, " %U\n", format_mfib_entry_path_ext, mpi); + })); + s = format (s, " Interface-Forwarding:\n", + mfib_source_names[msrc->mfes_src]); hash_foreach(sw_if_index, mfi, msrc->mfes_itfs, ({ s = format(s, " %U\n", format_mfib_itf, mfi); @@ -165,7 +219,7 @@ format_mfib_entry (u8 * s, va_list * args) ({ s = format(s, "\n %U", format_mfib_itf, mfi); })); - + s = format(s, "\n RPF-ID:%d", mfib_entry->mfe_rpf_id); s = format(s, "\n %U-chain\n %U", format_fib_forw_chain_type, mfib_entry_get_default_chain_type(mfib_entry), @@ -314,13 +368,6 @@ mfib_entry_src_remove (mfib_entry_t *mfib_entry, } } -static int -mfib_entry_src_n_itfs (const mfib_entry_src_t *msrc) -{ - return (hash_elts(msrc->mfes_itfs)); -} - - static void mfib_entry_last_lock_gone (fib_node_t *node) { @@ -338,7 +385,6 @@ mfib_entry_last_lock_gone (fib_node_t *node) mfib_entry_src_flush(msrc); } - fib_path_list_unlock(mfib_entry->mfe_parent); vec_free(mfib_entry->mfe_srcs); fib_node_deinit(&mfib_entry->mfe_node); @@ -417,10 +463,9 @@ mfib_entry_alloc (u32 fib_index, mfib_entry->mfe_flags = 0; mfib_entry->mfe_fib_index = fib_index; mfib_entry->mfe_prefix = *prefix; - mfib_entry->mfe_parent = FIB_NODE_INDEX_INVALID; - mfib_entry->mfe_sibling = FIB_NODE_INDEX_INVALID; mfib_entry->mfe_srcs = NULL; mfib_entry->mfe_itfs = NULL; + mfib_entry->mfe_rpf_id = MFIB_RPF_ID_NONE; dpo_reset(&mfib_entry->mfe_rep); @@ -431,10 +476,57 @@ mfib_entry_alloc (u32 fib_index, return (mfib_entry); } +static inline mfib_path_ext_t * +mfib_entry_path_ext_find (mfib_path_ext_t *exts, + fib_node_index_t path_index) +{ + uword *p; + + p = hash_get(exts, path_index); + + if (NULL != p) + { + return (mfib_entry_path_ext_get(p[0])); + } + + return (NULL); +} + +static mfib_path_ext_t* +mfib_path_ext_add (mfib_entry_src_t *msrc, + fib_node_index_t path_index, + mfib_itf_flags_t mfi_flags) +{ + mfib_path_ext_t *path_ext; + + pool_get(mfib_path_ext_pool, path_ext); + + path_ext->mfpe_flags = mfi_flags; + path_ext->mfpe_path = path_index; + + hash_set(msrc->mfes_exts, path_index, + path_ext - mfib_path_ext_pool); + + return (path_ext); +} + +static void +mfib_path_ext_remove (mfib_entry_src_t *msrc, + fib_node_index_t path_index) +{ + mfib_path_ext_t *path_ext; + + path_ext = mfib_entry_path_ext_find(msrc->mfes_exts, path_index); + + hash_unset(msrc->mfes_exts, path_index); + pool_put(mfib_path_ext_pool, path_ext); +} + typedef struct mfib_entry_collect_forwarding_ctx_t_ { load_balance_path_t * next_hops; fib_forward_chain_type_t fct; + mfib_entry_src_t *msrc; } mfib_entry_collect_forwarding_ctx_t; static int @@ -455,6 +547,20 @@ mfib_entry_src_collect_forwarding (fib_node_index_t pl_index, return (!0); } + /* + * If the path is not forwarding to use it + */ + mfib_path_ext_t *path_ext; + + path_ext = mfib_entry_path_ext_find(ctx->msrc->mfes_exts, + path_index); + + if (NULL != path_ext && + !(path_ext->mfpe_flags & MFIB_ITF_FLAG_FORWARD)) + { + return (!0); + } + switch (ctx->fct) { case FIB_FORW_CHAIN_TYPE_MCAST_IP4: @@ -483,46 +589,61 @@ mfib_entry_src_collect_forwarding (fib_node_index_t pl_index, } static void -mfib_entry_stack (mfib_entry_t *mfib_entry) +mfib_entry_stack (mfib_entry_t *mfib_entry, + mfib_entry_src_t *msrc) { dpo_proto_t dp; dp = fib_proto_to_dpo(mfib_entry_get_proto(mfib_entry)); - if (FIB_NODE_INDEX_INVALID != mfib_entry->mfe_parent) + if (NULL != msrc && + FIB_NODE_INDEX_INVALID != msrc->mfes_pl) { mfib_entry_collect_forwarding_ctx_t ctx = { .next_hops = NULL, .fct = mfib_entry_get_default_chain_type(mfib_entry), + .msrc = msrc, }; - fib_path_list_walk(mfib_entry->mfe_parent, + fib_path_list_walk(msrc->mfes_pl, mfib_entry_src_collect_forwarding, &ctx); if (!(MFIB_ENTRY_FLAG_EXCLUSIVE & mfib_entry->mfe_flags)) { - /* - * each path contirbutes a next-hop. form a replicate - * from those choices. - */ - if (!dpo_id_is_valid(&mfib_entry->mfe_rep) || - dpo_is_drop(&mfib_entry->mfe_rep)) + if (NULL == ctx.next_hops) { - dpo_id_t tmp_dpo = DPO_INVALID; - - dpo_set(&tmp_dpo, - DPO_REPLICATE, dp, - replicate_create(0, dp)); - + /* + * no next-hops, stack directly on the drop + */ dpo_stack(DPO_MFIB_ENTRY, dp, &mfib_entry->mfe_rep, - &tmp_dpo); - - dpo_reset(&tmp_dpo); + drop_dpo_get(dp)); + } + else + { + /* + * each path contirbutes a next-hop. form a replicate + * from those choices. + */ + if (!dpo_id_is_valid(&mfib_entry->mfe_rep) || + dpo_is_drop(&mfib_entry->mfe_rep)) + { + dpo_id_t tmp_dpo = DPO_INVALID; + + dpo_set(&tmp_dpo, + DPO_REPLICATE, dp, + replicate_create(0, dp)); + + dpo_stack(DPO_MFIB_ENTRY, dp, + &mfib_entry->mfe_rep, + &tmp_dpo); + + dpo_reset(&tmp_dpo); + } + replicate_multipath_update(&mfib_entry->mfe_rep, + ctx.next_hops); } - replicate_multipath_update(&mfib_entry->mfe_rep, - ctx.next_hops); } else { @@ -548,11 +669,11 @@ mfib_entry_stack (mfib_entry_t *mfib_entry) } } -static void -mfib_entry_forwarding_path_add (mfib_entry_src_t *msrc, - const fib_route_path_t *rpath) +static fib_node_index_t +mfib_entry_src_path_add (mfib_entry_src_t *msrc, + const fib_route_path_t *rpath) { - fib_node_index_t old_pl_index; + fib_node_index_t path_index; fib_route_path_t *rpaths; ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_flags)); @@ -563,32 +684,26 @@ mfib_entry_forwarding_path_add (mfib_entry_src_t *msrc, rpaths = NULL; vec_add1(rpaths, rpath[0]); - old_pl_index = msrc->mfes_pl; - if (FIB_NODE_INDEX_INVALID == msrc->mfes_pl) { - msrc->mfes_pl = - fib_path_list_create(FIB_PATH_LIST_FLAG_NO_URPF, - rpaths); - } - else - { - msrc->mfes_pl = - fib_path_list_copy_and_path_add(msrc->mfes_pl, - FIB_PATH_LIST_FLAG_NO_URPF, - rpaths); + /* A non-shared path-list */ + msrc->mfes_pl = fib_path_list_create(FIB_PATH_LIST_FLAG_NO_URPF, + NULL); + fib_path_list_lock(msrc->mfes_pl); } - fib_path_list_lock(msrc->mfes_pl); - fib_path_list_unlock(old_pl_index); + + path_index = fib_path_list_path_add(msrc->mfes_pl, rpaths); vec_free(rpaths); + + return (path_index); } -static int -mfib_entry_forwarding_path_remove (mfib_entry_src_t *msrc, - const fib_route_path_t *rpath) +static fib_node_index_t +mfib_entry_src_path_remove (mfib_entry_src_t *msrc, + const fib_route_path_t *rpath) { - fib_node_index_t old_pl_index; + fib_node_index_t path_index; fib_route_path_t *rpaths; ASSERT(!(MFIB_ENTRY_FLAG_EXCLUSIVE & msrc->mfes_flags)); @@ -599,56 +714,31 @@ mfib_entry_forwarding_path_remove (mfib_entry_src_t *msrc, rpaths = NULL; vec_add1(rpaths, rpath[0]); - old_pl_index = msrc->mfes_pl; - - msrc->mfes_pl = - fib_path_list_copy_and_path_remove(msrc->mfes_pl, - FIB_PATH_LIST_FLAG_NONE, - rpaths); - - fib_path_list_lock(msrc->mfes_pl); - fib_path_list_unlock(old_pl_index); + path_index = fib_path_list_path_remove(msrc->mfes_pl, rpaths); vec_free(rpaths); - return (FIB_NODE_INDEX_INVALID != msrc->mfes_pl); + return (path_index); } static void mfib_entry_recalculate_forwarding (mfib_entry_t *mfib_entry) { - fib_node_index_t old_pl_index; mfib_entry_src_t *bsrc; - old_pl_index = mfib_entry->mfe_parent; - /* * copy the forwarding data from the bast source */ bsrc = mfib_entry_get_best_src(mfib_entry); - if (NULL == bsrc) - { - mfib_entry->mfe_parent = FIB_NODE_INDEX_INVALID; - } - else + if (NULL != bsrc) { - mfib_entry->mfe_parent = bsrc->mfes_pl; mfib_entry->mfe_flags = bsrc->mfes_flags; mfib_entry->mfe_itfs = bsrc->mfes_itfs; + mfib_entry->mfe_rpf_id = bsrc->mfes_rpf_id; } - /* - * re-stack the entry on the best forwarding info. - */ - if (old_pl_index != mfib_entry->mfe_parent || - FIB_NODE_INDEX_INVALID == old_pl_index) - { - mfib_entry_stack(mfib_entry); - - fib_path_list_lock(mfib_entry->mfe_parent); - fib_path_list_unlock(old_pl_index); - } + mfib_entry_stack(mfib_entry, bsrc); } @@ -656,6 +746,7 @@ fib_node_index_t mfib_entry_create (u32 fib_index, mfib_source_t source, const mfib_prefix_t *prefix, + fib_rpf_id_t rpf_id, mfib_entry_flags_t entry_flags) { fib_node_index_t mfib_entry_index; @@ -666,6 +757,7 @@ mfib_entry_create (u32 fib_index, &mfib_entry_index); msrc = mfib_entry_src_find_or_create(mfib_entry, source); msrc->mfes_flags = entry_flags; + msrc->mfes_rpf_id = rpf_id; mfib_entry_recalculate_forwarding(mfib_entry); @@ -682,13 +774,14 @@ static int mfib_entry_src_ok_for_delete (const mfib_entry_src_t *msrc) { return ((MFIB_ENTRY_FLAG_NONE == msrc->mfes_flags && - 0 == mfib_entry_src_n_itfs(msrc))); + 0 == fib_path_list_get_n_paths(msrc->mfes_pl))); } int mfib_entry_update (fib_node_index_t mfib_entry_index, mfib_source_t source, mfib_entry_flags_t entry_flags, + fib_rpf_id_t rpf_id, index_t repi) { mfib_entry_t *mfib_entry; @@ -697,6 +790,7 @@ mfib_entry_update (fib_node_index_t mfib_entry_index, mfib_entry = mfib_entry_get(mfib_entry_index); msrc = mfib_entry_src_find_or_create(mfib_entry, source); msrc->mfes_flags = entry_flags; + msrc->mfes_rpf_id = rpf_id; if (INDEX_INVALID != repi) { @@ -768,55 +862,79 @@ mfib_entry_path_update (fib_node_index_t mfib_entry_index, const fib_route_path_t *rpath, mfib_itf_flags_t itf_flags) { + fib_node_index_t path_index; + mfib_path_ext_t *path_ext; + mfib_itf_flags_t old, new; mfib_entry_t *mfib_entry; mfib_entry_src_t *msrc; - mfib_itf_t *mfib_itf; mfib_entry = mfib_entry_get(mfib_entry_index); ASSERT(NULL != mfib_entry); msrc = mfib_entry_src_find_or_create(mfib_entry, source); /* - * search for the interface in the current set + * add the path to the path-list. If it's a duplicate we'll get + * back the original path. + */ + path_index = mfib_entry_src_path_add(msrc, rpath); + + /* + * find the path extension for that path */ - mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs, - rpath[0].frp_sw_if_index); + path_ext = mfib_entry_path_ext_find(msrc->mfes_exts, path_index); - if (NULL == mfib_itf) + if (NULL == path_ext) { - /* - * this is a path we do not yet have. If it is forwarding then we - * add it to the replication set - */ - if (itf_flags & MFIB_ITF_FLAG_FORWARD) - { - mfib_entry_forwarding_path_add(msrc, rpath); - } - /* - * construct a new ITF for this entry's list - */ - mfib_entry_itf_add(msrc, - rpath[0].frp_sw_if_index, - mfib_itf_create(rpath[0].frp_sw_if_index, - itf_flags)); + old = MFIB_ITF_FLAG_NONE; + path_ext = mfib_path_ext_add(msrc, path_index, itf_flags); } else { - int was_forwarding = !!(mfib_itf->mfi_flags & MFIB_ITF_FLAG_FORWARD); - int is_forwarding = !!(itf_flags & MFIB_ITF_FLAG_FORWARD); + old = path_ext->mfpe_flags; + path_ext->mfpe_flags = itf_flags; + } - if (!was_forwarding && is_forwarding) - { - mfib_entry_forwarding_path_add(msrc, rpath); - } - else if (was_forwarding && !is_forwarding) + /* + * Has the path changed its contribution to the input interface set. + * Which only paths with interfaces can do... + */ + if (~0 != rpath[0].frp_sw_if_index) + { + mfib_itf_t *mfib_itf; + + new = itf_flags; + + if (old != new) { - mfib_entry_forwarding_path_remove(msrc, rpath); + if (MFIB_ITF_FLAG_NONE == new) + { + /* + * no more interface flags on this path, remove + * from the data-plane set + */ + mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index); + } + else if (MFIB_ITF_FLAG_NONE == old) + { + /* + * This interface is now contributing + */ + mfib_entry_itf_add(msrc, + rpath[0].frp_sw_if_index, + mfib_itf_create(rpath[0].frp_sw_if_index, + itf_flags)); + } + else + { + /* + * change of flag contributions + */ + mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs, + rpath[0].frp_sw_if_index); + /* Seen by packets inflight */ + mfib_itf->mfi_flags = new; + } } - /* - * packets in flight see these updates. - */ - mfib_itf->mfi_flags = itf_flags; } mfib_entry_recalculate_forwarding(mfib_entry); @@ -833,9 +951,9 @@ mfib_entry_path_remove (fib_node_index_t mfib_entry_index, mfib_source_t source, const fib_route_path_t *rpath) { + fib_node_index_t path_index; mfib_entry_t *mfib_entry; mfib_entry_src_t *msrc; - mfib_itf_t *mfib_itf; mfib_entry = mfib_entry_get(mfib_entry_index); ASSERT(NULL != mfib_entry); @@ -850,33 +968,23 @@ mfib_entry_path_remove (fib_node_index_t mfib_entry_index, } /* - * search for the interface in the current set + * remove the path from the path-list. If it's not there we'll get + * back invalid */ - mfib_itf = mfib_entry_itf_find(msrc->mfes_itfs, - rpath[0].frp_sw_if_index); + path_index = mfib_entry_src_path_remove(msrc, rpath); - if (NULL == mfib_itf) + if (FIB_NODE_INDEX_INVALID != path_index) { /* - * removing a path that does not exist + * don't need the extension, nor the interface anymore */ - return (mfib_entry_ok_for_delete(mfib_entry)); - } - - /* - * we have this path. If it is forwarding then we - * remove it to the replication set - */ - if (mfib_itf->mfi_flags & MFIB_ITF_FLAG_FORWARD) - { - mfib_entry_forwarding_path_remove(msrc, rpath); + mfib_path_ext_remove(msrc, path_index); + if (~0 != rpath[0].frp_sw_if_index) + { + mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index); + } } - /* - * remove the interface/path from this entry's list - */ - mfib_entry_itf_remove(msrc, rpath[0].frp_sw_if_index); - if (mfib_entry_src_ok_for_delete(msrc)) { /* @@ -1057,11 +1165,14 @@ mfib_entry_encode (fib_node_index_t mfib_entry_index, fib_route_path_encode_t **api_rpaths) { mfib_entry_t *mfib_entry; + mfib_entry_src_t *bsrc; mfib_entry = mfib_entry_get(mfib_entry_index); - if (FIB_NODE_INDEX_INVALID != mfib_entry->mfe_parent) + bsrc = mfib_entry_get_best_src(mfib_entry); + + if (FIB_NODE_INDEX_INVALID != bsrc->mfes_pl) { - fib_path_list_walk(mfib_entry->mfe_parent, + fib_path_list_walk(bsrc->mfes_pl, fib_path_encode, api_rpaths); } diff --git a/src/vnet/mfib/mfib_entry.h b/src/vnet/mfib/mfib_entry.h index dc8f49aa..4f62b18e 100644 --- a/src/vnet/mfib/mfib_entry.h +++ b/src/vnet/mfib/mfib_entry.h @@ -42,17 +42,6 @@ typedef struct mfib_entry_t_ { * The index of the FIB table this entry is in */ u32 mfe_fib_index; - /** - * the path-list for which this entry is a child. This is also the path-list - * that is contributing forwarding for this entry. - */ - fib_node_index_t mfe_parent; - /** - * index of this entry in the parent's child list. - * This is set when this entry is added as a child, but can also - * be changed by the parent as it manages its list. - */ - u32 mfe_sibling; /** * A vector of sources contributing forwarding @@ -65,7 +54,7 @@ typedef struct mfib_entry_t_ { CLIB_CACHE_LINE_ALIGN_MARK(cacheline1); /** - * The Replicate DPO used for forwarding. + * The DPO used for forwarding; replicate, drop, etc.. */ dpo_id_t mfe_rep; @@ -74,6 +63,11 @@ typedef struct mfib_entry_t_ { */ mfib_entry_flags_t mfe_flags; + /** + * RPF-ID used when the packets ingress not from an interface + */ + fib_rpf_id_t mfe_rpf_id; + /** * A hash table of interfaces */ @@ -90,11 +84,13 @@ extern u8 *format_mfib_entry(u8 * s, va_list * args); extern fib_node_index_t mfib_entry_create(u32 fib_index, mfib_source_t source, const mfib_prefix_t *prefix, + fib_rpf_id_t rpf_id, mfib_entry_flags_t entry_flags); extern int mfib_entry_update(fib_node_index_t fib_entry_index, mfib_source_t source, mfib_entry_flags_t entry_flags, + fib_rpf_id_t rpf_id, index_t rep_dpo); extern void mfib_entry_path_update(fib_node_index_t fib_entry_index, diff --git a/src/vnet/mfib/mfib_forward.c b/src/vnet/mfib/mfib_forward.c index 5fe0a57c..3d8f4f98 100644 --- a/src/vnet/mfib/mfib_forward.c +++ b/src/vnet/mfib/mfib_forward.c @@ -380,13 +380,27 @@ mfib_forward_rpf (vlib_main_t * vm, * for the case of throughput traffic that is not replicated * to the host stack nor sets local flags */ - if (PREDICT_TRUE(NULL != mfi0)) + + /* + * If the mfib entry has a configured RPF-ID check that + * in preference to an interface based RPF + */ + if (MFIB_RPF_ID_NONE != mfe0->mfe_rpf_id) { - iflags0 = mfi0->mfi_flags; + iflags0 = (mfe0->mfe_rpf_id == vnet_buffer(b0)->ip.rpf_id ? + MFIB_ITF_FLAG_ACCEPT : + MFIB_ITF_FLAG_NONE); } else { - iflags0 = MFIB_ITF_FLAG_NONE; + if (PREDICT_TRUE(NULL != mfi0)) + { + iflags0 = mfi0->mfi_flags; + } + else + { + iflags0 = MFIB_ITF_FLAG_NONE; + } } eflags0 = mfe0->mfe_flags; @@ -436,17 +450,16 @@ mfib_forward_rpf (vlib_main_t * vm, { mfib_forward_rpf_trace_t *t0; - t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0])); + t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); t0->entry_index = mfei0; + t0->itf_flags = iflags0; if (NULL == mfi0) { t0->sw_if_index = ~0; - t0->itf_flags = MFIB_ITF_FLAG_NONE; } else { t0->sw_if_index = mfi0->mfi_sw_if_index; - t0->itf_flags = mfi0->mfi_flags; } } vlib_validate_buffer_enqueue_x1 (vm, node, next, @@ -478,7 +491,7 @@ VLIB_REGISTER_NODE (ip4_mfib_forward_rpf_node, static) = { .n_next_nodes = MFIB_FORWARD_RPF_N_NEXT, .next_nodes = { - [MFIB_FORWARD_RPF_NEXT_DROP] = "error-drop", + [MFIB_FORWARD_RPF_NEXT_DROP] = "ip4-drop", }, }; @@ -503,7 +516,7 @@ VLIB_REGISTER_NODE (ip6_mfib_forward_rpf_node, static) = { .n_next_nodes = MFIB_FORWARD_RPF_N_NEXT, .next_nodes = { - [MFIB_FORWARD_RPF_NEXT_DROP] = "error-drop", + [MFIB_FORWARD_RPF_NEXT_DROP] = "ip6-drop", }, }; diff --git a/src/vnet/mfib/mfib_table.c b/src/vnet/mfib/mfib_table.c index 3b4bd985..7ffe8941 100644 --- a/src/vnet/mfib/mfib_table.c +++ b/src/vnet/mfib/mfib_table.c @@ -165,6 +165,7 @@ fib_node_index_t mfib_table_entry_update (u32 fib_index, const mfib_prefix_t *prefix, mfib_source_t source, + fib_rpf_id_t rpf_id, mfib_entry_flags_t entry_flags) { fib_node_index_t mfib_entry_index; @@ -181,7 +182,8 @@ mfib_table_entry_update (u32 fib_index, * update to a non-existing entry with non-zero flags */ mfib_entry_index = mfib_entry_create(fib_index, source, - prefix, entry_flags); + prefix, rpf_id, + entry_flags); mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index); } @@ -198,6 +200,7 @@ mfib_table_entry_update (u32 fib_index, if (mfib_entry_update(mfib_entry_index, source, entry_flags, + rpf_id, INDEX_INVALID)) { /* @@ -230,6 +233,7 @@ mfib_table_entry_path_update (u32 fib_index, mfib_entry_index = mfib_entry_create(fib_index, source, prefix, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_NONE); mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index); @@ -304,6 +308,7 @@ mfib_table_entry_special_add (u32 fib_index, mfib_entry_index = mfib_entry_create(fib_index, source, prefix, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_NONE); mfib_table_entry_insert(mfib_table, prefix, mfib_entry_index); @@ -311,6 +316,7 @@ mfib_table_entry_special_add (u32 fib_index, mfib_entry_update(mfib_entry_index, source, (MFIB_ENTRY_FLAG_EXCLUSIVE | entry_flags), + MFIB_RPF_ID_NONE, rep_dpo); return (mfib_entry_index); diff --git a/src/vnet/mfib/mfib_table.h b/src/vnet/mfib/mfib_table.h index 95239f7c..83aa04ef 100644 --- a/src/vnet/mfib/mfib_table.h +++ b/src/vnet/mfib/mfib_table.h @@ -122,6 +122,7 @@ extern fib_node_index_t mfib_table_lookup_exact_match(u32 fib_index, extern fib_node_index_t mfib_table_entry_update(u32 fib_index, const mfib_prefix_t *prefix, mfib_source_t source, + fib_rpf_id_t rpf_id, mfib_entry_flags_t flags); /** diff --git a/src/vnet/mfib/mfib_test.c b/src/vnet/mfib/mfib_test.c index 36a303e8..7c92ae99 100644 --- a/src/vnet/mfib/mfib_test.c +++ b/src/vnet/mfib/mfib_test.c @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include #include @@ -201,8 +203,8 @@ mfib_test_validate_rep_v (const replicate_t *rep, if (DPO_RECEIVE != dt) { MFIB_TEST_REP((ai == dpo->dpoi_index), - "bucket %d stacks on %U", - bucket, + "bucket %d [exp:%d] stacks on %U", + bucket, ai, format_dpo_id, dpo, 0); } } @@ -734,6 +736,7 @@ mfib_test_i (fib_protocol_t PROTO, mfib_table_entry_update(fib_index, pfx_s_g, MFIB_SOURCE_API, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_SIGNAL); MFIB_TEST(mfib_test_entry(mfei, MFIB_ENTRY_FLAG_SIGNAL, @@ -824,6 +827,7 @@ mfib_test_i (fib_protocol_t PROTO, mfib_table_entry_update(fib_index, pfx_s_g, MFIB_SOURCE_API, + MFIB_RPF_ID_NONE, (MFIB_ENTRY_FLAG_SIGNAL | MFIB_ENTRY_FLAG_CONNECTED)); MFIB_TEST(mfib_test_entry(mfei, @@ -965,6 +969,7 @@ mfib_test_i (fib_protocol_t PROTO, mfib_table_entry_update(fib_index, pfx_s_g, MFIB_SOURCE_API, + MFIB_RPF_ID_NONE, MFIB_ENTRY_FLAG_NONE); mfei = mfib_table_lookup_exact_match(fib_index, pfx_s_g); @@ -1073,6 +1078,117 @@ mfib_test_i (fib_protocol_t PROTO, MFIB_SOURCE_SRv6); dpo_reset(&td); + /* + * A Multicast LSP. This a mLDP head-end + */ + fib_node_index_t ai_mpls_10_10_10_1, lfei; + ip46_address_t nh_10_10_10_1 = { + .ip4 = { + .as_u32 = clib_host_to_net_u32(0x0a0a0a01), + }, + }; + ai_mpls_10_10_10_1 = adj_nbr_add_or_lock(FIB_PROTOCOL_IP4, + VNET_LINK_MPLS, + &nh_10_10_10_1, + tm->hw[0]->sw_if_index); + + fib_prefix_t pfx_3500 = { + .fp_len = 21, + .fp_proto = FIB_PROTOCOL_MPLS, + .fp_label = 3500, + .fp_eos = MPLS_EOS, + .fp_payload_proto = DPO_PROTO_IP4, + }; + fib_test_rep_bucket_t mc_0 = { + .type = FT_REP_LABEL_O_ADJ, + .label_o_adj = { + .adj = ai_mpls_10_10_10_1, + .label = 3300, + .eos = MPLS_EOS, + }, + }; + mpls_label_t *l3300 = NULL; + vec_add1(l3300, 3300); + + /* + * MPLS enable an interface so we get the MPLS table created + */ + mpls_sw_interface_enable_disable(&mpls_main, + tm->hw[0]->sw_if_index, + 1); + + lfei = fib_table_entry_update_one_path(0, // default MPLS Table + &pfx_3500, + FIB_SOURCE_API, + FIB_ENTRY_FLAG_MULTICAST, + FIB_PROTOCOL_IP4, + &nh_10_10_10_1, + tm->hw[0]->sw_if_index, + ~0, // invalid fib index + 1, + l3300, + FIB_ROUTE_PATH_FLAG_NONE); + MFIB_TEST(fib_test_validate_entry(lfei, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + 1, + &mc_0), + "3500 via replicate over 10.10.10.1"); + + /* + * An (S,G) that resolves via the mLDP head-end + */ + fib_route_path_t path_via_mldp = { + .frp_proto = FIB_PROTOCOL_MPLS, + .frp_local_label = pfx_3500.fp_label, + .frp_eos = MPLS_EOS, + .frp_sw_if_index = 0xffffffff, + .frp_fib_index = 0, + .frp_weight = 1, + .frp_flags = FIB_ROUTE_PATH_FLAG_NONE, + }; + dpo_id_t mldp_dpo = DPO_INVALID; + + fib_entry_contribute_forwarding(lfei, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + &mldp_dpo); + + mfei = mfib_table_entry_path_update(fib_index, + pfx_s_g, + MFIB_SOURCE_API, + &path_via_mldp, + MFIB_ITF_FLAG_FORWARD); + + MFIB_TEST(mfib_test_entry(mfei, + MFIB_ENTRY_FLAG_NONE, + 1, + DPO_REPLICATE, mldp_dpo.dpoi_index), + "%U over-mLDP replicate OK", + format_mfib_prefix, pfx_s_g); + + /* + * add a for-us path. this tests two types of non-attached paths on one entry + */ + mfei = mfib_table_entry_path_update(fib_index, + pfx_s_g, + MFIB_SOURCE_API, + &path_for_us, + MFIB_ITF_FLAG_FORWARD); + MFIB_TEST(mfib_test_entry(mfei, + MFIB_ENTRY_FLAG_NONE, + 2, + DPO_REPLICATE, mldp_dpo.dpoi_index, + DPO_RECEIVE, 0), + "%U mLDP+for-us replicate OK", + format_mfib_prefix, pfx_s_g); + + mfib_table_entry_delete(fib_index, + pfx_s_g, + MFIB_SOURCE_API); + fib_table_entry_delete(0, + &pfx_3500, + FIB_SOURCE_API); + dpo_reset(&mldp_dpo); + /* * Unlock the table - it's the last lock so should be gone thereafter */ @@ -1086,6 +1202,13 @@ mfib_test_i (fib_protocol_t PROTO, adj_unlock(ai_2); adj_unlock(ai_3); + /* + * MPLS disable the interface + */ + mpls_sw_interface_enable_disable(&mpls_main, + tm->hw[0]->sw_if_index, + 0); + /* * test we've leaked no resources */ diff --git a/src/vnet/mpls/mpls.api b/src/vnet/mpls/mpls.api index 2e3bfaf5..a1e1270a 100644 --- a/src/vnet/mpls/mpls.api +++ b/src/vnet/mpls/mpls.api @@ -55,6 +55,7 @@ define mpls_ip_bind_unbind_reply @param context - sender context, to match reply w/ request @param mt_is_add - Is this a route add or delete @param mt_sw_if_index - The SW interface index of the tunnel to delete + @param mt_is_multicast - Is the tunnel's underlying LSP multicast @param mt_next_hop_proto_is_ip4 - The next-hop is IPV4 @param mt_next_hop_weight - The weight, for UCMP @param mt_next_hop[16] - the nextop address @@ -70,6 +71,7 @@ define mpls_tunnel_add_del u32 mt_sw_if_index; u8 mt_is_add; u8 mt_l2_only; + u8 mt_is_multicast; u8 mt_next_hop_proto_is_ip4; u8 mt_next_hop_weight; u8 mt_next_hop[16]; @@ -102,30 +104,43 @@ define mpls_tunnel_dump i32 tunnel_index; }; -/** \brief mpls eth tunnel operational state response - @param tunnel_index - eth tunnel identifier - @param intfc_address - interface ipv4 addr - @param mask_width - interface ipv4 addr mask - @param hw_if_index - interface id - @param l2_only - - @param tunnel_dst_mac - - @param tx_sw_if_index - - @param encap_index - reference to mpls label table - @param nlabels - number of resolved labels - @param labels - resolved labels +/** \brief FIB path + @param sw_if_index - index of the interface + @param weight - The weight, for UCMP + @param is_local - local if non-zero, else remote + @param is_drop - Drop the packet + @param is_unreach - Drop the packet and rate limit send ICMP unreachable + @param is_prohibit - Drop the packet and rate limit send ICMP prohibited + @param afi - the afi of the next hop, IP46_TYPE_IP4=1, IP46_TYPE_IP6=2 + @param next_hop[16] - the next hop address + + WARNING: this type is replicated, pending cleanup completion + +*/ +typeonly manual_print manual_endian define fib_path2 +{ + u32 sw_if_index; + u32 weight; + u8 is_local; + u8 is_drop; + u8 is_unreach; + u8 is_prohibit; + u8 afi; + u8 next_hop[16]; + u32 labels[16]; +}; + +/** \brief mpls tunnel details */ -define mpls_tunnel_details +manual_endian manual_print define mpls_tunnel_details { u32 context; - u32 tunnel_index; - u8 mt_l2_only; u8 mt_sw_if_index; - u8 mt_next_hop_proto_is_ip4; - u8 mt_next_hop[16]; - u32 mt_next_hop_sw_if_index; - u32 mt_next_hop_table_id; - u32 mt_next_hop_n_labels; - u32 mt_next_hop_out_labels[mt_next_hop_n_labels]; + u8 mt_tunnel_index; + u8 mt_l2_only; + u8 mt_is_multicast; + u32 mt_count; + vl_api_fib_path2_t mt_paths[mt_count]; }; /** \brief MPLS Route Add / del route @@ -140,10 +155,14 @@ define mpls_tunnel_details create them @param mr_is_add - Is this a route add or delete @param mr_is_classify - Is this route result a classify + @param mr_is_multicast - Is this a multicast route @param mr_is_multipath - Is this route update a multipath - i.e. is this a path addition to an existing route @param mr_is_resolve_host - Recurse resolution constraint via a host prefix @param mr_is_resolve_attached - Recurse resolution constraint via attached prefix + @param mr_is_interface_rx - Interface Receive path + @param mr_is_interface_rx - RPF-ID Receive path. The next-hop interface + is used as the RPF-ID @param mr_next_hop_proto_is_ip4 - The next-hop is IPV4 @param mr_next_hop_weight - The weight, for UCMP @param mr_next_hop[16] - the nextop address @@ -164,9 +183,12 @@ define mpls_route_add_del u8 mr_create_table_if_needed; u8 mr_is_add; u8 mr_is_classify; + u8 mr_is_multicast; u8 mr_is_multipath; u8 mr_is_resolve_host; u8 mr_is_resolve_attached; + u8 mr_is_interface_rx; + u8 mr_is_rpf_id; u8 mr_next_hop_proto_is_ip4; u8 mr_next_hop_weight; u8 mr_next_hop[16]; @@ -187,31 +209,6 @@ define mpls_route_add_del_reply i32 retval; }; -/** \brief FIB path - @param sw_if_index - index of the interface - @param weight - The weight, for UCMP - @param is_local - local if non-zero, else remote - @param is_drop - Drop the packet - @param is_unreach - Drop the packet and rate limit send ICMP unreachable - @param is_prohibit - Drop the packet and rate limit send ICMP prohibited - @param afi - the afi of the next hop, IP46_TYPE_IP4=1, IP46_TYPE_IP6=2 - @param next_hop[16] - the next hop address - - WARNING: this type is replicated, pending cleanup completion - -*/ -typeonly manual_print manual_endian define fib_path2 -{ - u32 sw_if_index; - u32 weight; - u8 is_local; - u8 is_drop; - u8 is_unreach; - u8 is_prohibit; - u8 afi; - u8 next_hop[16]; -}; - /** \brief Dump MPLS fib table @param client_index - opaque cookie to identify the sender */ diff --git a/src/vnet/mpls/mpls.c b/src/vnet/mpls/mpls.c index 482577b1..451b15cf 100644 --- a/src/vnet/mpls/mpls.c +++ b/src/vnet/mpls/mpls.c @@ -286,7 +286,15 @@ vnet_mpls_local_label (vlib_main_t * vm, rpath.frp_proto = FIB_PROTOCOL_IP4; vec_add1(rpaths, rpath); } - + else if (unformat (line_input, "rx-ip4 %U", + unformat_vnet_sw_interface, vnm, + &rpath.frp_sw_if_index)) + { + rpath.frp_weight = 1; + rpath.frp_proto = FIB_PROTOCOL_IP4; + rpath.frp_flags = FIB_ROUTE_PATH_INTF_RX; + vec_add1(rpaths, rpath); + } else if (unformat (line_input, "via %U %U", unformat_ip6_address, &rpath.frp_addr.ip6, @@ -512,10 +520,3 @@ mpls_init (vlib_main_t * vm) } VLIB_INIT_FUNCTION (mpls_init); - -mpls_main_t * mpls_get_main (vlib_main_t * vm) -{ - vlib_call_init_function (vm, mpls_init); - return &mpls_main; -} - diff --git a/src/vnet/mpls/mpls_api.c b/src/vnet/mpls/mpls_api.c index f1aef6c9..6bfc491d 100644 --- a/src/vnet/mpls/mpls_api.c +++ b/src/vnet/mpls/mpls_api.c @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -163,6 +164,7 @@ mpls_route_add_del_t_handler (vnet_main_t * vnm, dpo_proto_to_fib (pfx.fp_payload_proto), mp->mr_next_hop_table_id, mp->mr_create_table_if_needed, + mp->mr_is_rpf_id, &fib_index, &next_hop_fib_index); if (0 != rv) @@ -192,10 +194,13 @@ mpls_route_add_del_t_handler (vnet_main_t * vnm, 0, // mp->is_unreach, 0, // mp->is_prohibit, 0, // mp->is_local, + mp->mr_is_multicast, mp->mr_is_classify, mp->mr_classify_table_index, mp->mr_is_resolve_host, mp->mr_is_resolve_attached, + mp->mr_is_interface_rx, + mp->mr_is_rpf_id, fib_index, &pfx, mp->mr_next_hop_proto_is_ip4, &nh, ntohl (mp->mr_next_hop_sw_if_index), @@ -229,46 +234,54 @@ vl_api_mpls_tunnel_add_del_t_handler (vl_api_mpls_tunnel_add_del_t * mp) int rv = 0; u32 tunnel_sw_if_index; int ii; + fib_route_path_t rpath, *rpaths = NULL; + + memset (&rpath, 0, sizeof (rpath)); stats_dslock_with_hint (1 /* release hint */ , 5 /* tag */ ); - if (mp->mt_is_add) + if (mp->mt_next_hop_proto_is_ip4) { - fib_route_path_t rpath, *rpaths = NULL; - mpls_label_t *label_stack = NULL; - - memset (&rpath, 0, sizeof (rpath)); - - if (mp->mt_next_hop_proto_is_ip4) - { - rpath.frp_proto = FIB_PROTOCOL_IP4; - clib_memcpy (&rpath.frp_addr.ip4, - mp->mt_next_hop, sizeof (rpath.frp_addr.ip4)); - } - else - { - rpath.frp_proto = FIB_PROTOCOL_IP6; - clib_memcpy (&rpath.frp_addr.ip6, - mp->mt_next_hop, sizeof (rpath.frp_addr.ip6)); - } - rpath.frp_sw_if_index = ntohl (mp->mt_next_hop_sw_if_index); + rpath.frp_proto = FIB_PROTOCOL_IP4; + clib_memcpy (&rpath.frp_addr.ip4, + mp->mt_next_hop, sizeof (rpath.frp_addr.ip4)); + } + else + { + rpath.frp_proto = FIB_PROTOCOL_IP6; + clib_memcpy (&rpath.frp_addr.ip6, + mp->mt_next_hop, sizeof (rpath.frp_addr.ip6)); + } + rpath.frp_sw_if_index = ntohl (mp->mt_next_hop_sw_if_index); + rpath.frp_weight = 1; + if (mp->mt_is_add) + { for (ii = 0; ii < mp->mt_next_hop_n_out_labels; ii++) - vec_add1 (label_stack, ntohl (mp->mt_next_hop_out_label_stack[ii])); + vec_add1 (rpath.frp_label_stack, + ntohl (mp->mt_next_hop_out_label_stack[ii])); + } - vec_add1 (rpaths, rpath); + vec_add1 (rpaths, rpath); - vnet_mpls_tunnel_add (rpaths, label_stack, - mp->mt_l2_only, &tunnel_sw_if_index); - vec_free (rpaths); - vec_free (label_stack); + tunnel_sw_if_index = ntohl (mp->mt_sw_if_index); + + if (mp->mt_is_add) + { + if (~0 == tunnel_sw_if_index) + tunnel_sw_if_index = vnet_mpls_tunnel_create (mp->mt_l2_only, + mp->mt_is_multicast); + vnet_mpls_tunnel_path_add (tunnel_sw_if_index, rpaths); } else { tunnel_sw_if_index = ntohl (mp->mt_sw_if_index); - vnet_mpls_tunnel_del (tunnel_sw_if_index); + if (!vnet_mpls_tunnel_path_remove (tunnel_sw_if_index, rpaths)) + vnet_mpls_tunnel_del (tunnel_sw_if_index); } + vec_free (rpaths); + stats_dsunlock (); /* *INDENT-OFF* */ @@ -289,10 +302,12 @@ typedef struct mpls_tunnel_send_walk_ctx_t_ static void send_mpls_tunnel_entry (u32 mti, void *arg) { + fib_route_path_encode_t *api_rpaths, *api_rpath; mpls_tunnel_send_walk_ctx_t *ctx; vl_api_mpls_tunnel_details_t *mp; const mpls_tunnel_t *mt; - u32 nlabels; + vl_api_fib_path2_t *fp; + u32 n; ctx = arg; @@ -300,18 +315,34 @@ send_mpls_tunnel_entry (u32 mti, void *arg) return; mt = mpls_tunnel_get (mti); - nlabels = vec_len (mt->mt_label_stack); + n = fib_path_list_get_n_paths (mt->mt_path_list); + + mp = vl_msg_api_alloc (sizeof (*mp) + n * sizeof (vl_api_fib_path2_t)); + memset (mp, 0, sizeof (*mp) + n * sizeof (vl_api_fib_path2_t)); - mp = vl_msg_api_alloc (sizeof (*mp) + nlabels * sizeof (u32)); - memset (mp, 0, sizeof (*mp)); mp->_vl_msg_id = ntohs (VL_API_MPLS_TUNNEL_DETAILS); mp->context = ctx->context; - mp->tunnel_index = ntohl (mti); - memcpy (mp->mt_next_hop_out_labels, - mt->mt_label_stack, nlabels * sizeof (u32)); + mp->mt_tunnel_index = ntohl (mti); + mp->mt_count = ntohl (n); + + fib_path_list_walk (mt->mt_path_list, fib_path_encode, &api_rpaths); + + fp = mp->mt_paths; + vec_foreach (api_rpath, api_rpaths) + { + memset (fp, 0, sizeof (*fp)); + + fp->weight = htonl (api_rpath->rpath.frp_weight); + fp->sw_if_index = htonl (api_rpath->rpath.frp_sw_if_index); + copy_fib_next_hop (api_rpath, fp); + fp++; + } // FIXME + // memcpy (mp->mt_next_hop_out_labels, + // mt->mt_label_stack, nlabels * sizeof (u32)); + vl_msg_api_send_shmem (ctx->q, (u8 *) & mp); } diff --git a/src/vnet/mpls/mpls_input.c b/src/vnet/mpls/mpls_input.c index 1b9bdd05..86ad8bba 100644 --- a/src/vnet/mpls/mpls_input.c +++ b/src/vnet/mpls/mpls_input.c @@ -291,7 +291,7 @@ mpls_setup_nodes (vlib_main_t * vm) rt->last_outer_fib_index = 0; rt->mpls_main = &mpls_main; - ethernet_register_input_type (vm, ETHERNET_TYPE_MPLS_UNICAST, + ethernet_register_input_type (vm, ETHERNET_TYPE_MPLS, mpls_input_node.index); } diff --git a/src/vnet/mpls/mpls_lookup.c b/src/vnet/mpls/mpls_lookup.c index ace6a70f..3c6be7e8 100644 --- a/src/vnet/mpls/mpls_lookup.c +++ b/src/vnet/mpls/mpls_lookup.c @@ -20,8 +20,17 @@ #include #include #include +#include -vlib_node_registration_t mpls_lookup_node; +/** + * Static MPLS VLIB forwarding node + */ +static vlib_node_registration_t mpls_lookup_node; + +/** + * The arc/edge from the MPLS lookup node to the MPLS replicate node + */ +static u32 mpls_lookup_to_replicate_edge; typedef struct { u32 next_index; @@ -156,81 +165,123 @@ mpls_lookup (vlib_main_t * vm, lbi2 = mpls_fib_table_forwarding_lookup (lfib_index2, h2); lbi3 = mpls_fib_table_forwarding_lookup (lfib_index3, h3); - lb0 = load_balance_get(lbi0); - lb1 = load_balance_get(lbi1); - lb2 = load_balance_get(lbi2); - lb3 = load_balance_get(lbi3); - hash_c0 = vnet_buffer(b0)->ip.flow_hash = 0; hash_c1 = vnet_buffer(b1)->ip.flow_hash = 0; hash_c2 = vnet_buffer(b2)->ip.flow_hash = 0; hash_c3 = vnet_buffer(b3)->ip.flow_hash = 0; - if (PREDICT_FALSE(lb0->lb_n_buckets > 1)) + if (MPLS_IS_REPLICATE & lbi0) { - hash_c0 = vnet_buffer (b0)->ip.flow_hash = - mpls_compute_flow_hash(h0, lb0->lb_hash_config); + next0 = mpls_lookup_to_replicate_edge; + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = + (lbi0 & ~MPLS_IS_REPLICATE); } - if (PREDICT_FALSE(lb1->lb_n_buckets > 1)) + else { - hash_c1 = vnet_buffer (b1)->ip.flow_hash = - mpls_compute_flow_hash(h1, lb1->lb_hash_config); + lb0 = load_balance_get(lbi0); + + if (PREDICT_FALSE(lb0->lb_n_buckets > 1)) + { + hash_c0 = vnet_buffer (b0)->ip.flow_hash = + mpls_compute_flow_hash(h0, lb0->lb_hash_config); + } + ASSERT (lb0->lb_n_buckets > 0); + ASSERT (is_pow2 (lb0->lb_n_buckets)); + dpo0 = load_balance_get_bucket_i(lb0, + (hash_c0 & + (lb0->lb_n_buckets_minus_1))); + next0 = dpo0->dpoi_next_node; + + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; + + vlib_increment_combined_counter + (cm, thread_index, lbi0, 1, + vlib_buffer_length_in_chain (vm, b0)); } - if (PREDICT_FALSE(lb2->lb_n_buckets > 1)) + if (MPLS_IS_REPLICATE & lbi1) { - hash_c2 = vnet_buffer (b2)->ip.flow_hash = - mpls_compute_flow_hash(h2, lb2->lb_hash_config); + next1 = mpls_lookup_to_replicate_edge; + vnet_buffer (b1)->ip.adj_index[VLIB_TX] = + (lbi1 & ~MPLS_IS_REPLICATE); } - if (PREDICT_FALSE(lb3->lb_n_buckets > 1)) + else { - hash_c3 = vnet_buffer (b3)->ip.flow_hash = - mpls_compute_flow_hash(h3, lb3->lb_hash_config); - } - - ASSERT (lb0->lb_n_buckets > 0); - ASSERT (is_pow2 (lb0->lb_n_buckets)); - ASSERT (lb1->lb_n_buckets > 0); - ASSERT (is_pow2 (lb1->lb_n_buckets)); - ASSERT (lb2->lb_n_buckets > 0); - ASSERT (is_pow2 (lb2->lb_n_buckets)); - ASSERT (lb3->lb_n_buckets > 0); - ASSERT (is_pow2 (lb3->lb_n_buckets)); - - dpo0 = load_balance_get_bucket_i(lb0, - (hash_c0 & - (lb0->lb_n_buckets_minus_1))); - dpo1 = load_balance_get_bucket_i(lb1, - (hash_c1 & - (lb1->lb_n_buckets_minus_1))); - dpo2 = load_balance_get_bucket_i(lb2, - (hash_c2 & - (lb2->lb_n_buckets_minus_1))); - dpo3 = load_balance_get_bucket_i(lb3, - (hash_c3 & - (lb3->lb_n_buckets_minus_1))); + lb1 = load_balance_get(lbi1); - next0 = dpo0->dpoi_next_node; - next1 = dpo1->dpoi_next_node; - next2 = dpo2->dpoi_next_node; - next3 = dpo3->dpoi_next_node; + if (PREDICT_FALSE(lb1->lb_n_buckets > 1)) + { + hash_c1 = vnet_buffer (b1)->ip.flow_hash = + mpls_compute_flow_hash(h1, lb1->lb_hash_config); + } + ASSERT (lb1->lb_n_buckets > 0); + ASSERT (is_pow2 (lb1->lb_n_buckets)); + dpo1 = load_balance_get_bucket_i(lb1, + (hash_c1 & + (lb1->lb_n_buckets_minus_1))); + next1 = dpo1->dpoi_next_node; + + vnet_buffer (b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; + + vlib_increment_combined_counter + (cm, thread_index, lbi1, 1, + vlib_buffer_length_in_chain (vm, b1)); + } + if (MPLS_IS_REPLICATE & lbi2) + { + next2 = mpls_lookup_to_replicate_edge; + vnet_buffer (b2)->ip.adj_index[VLIB_TX] = + (lbi2 & ~MPLS_IS_REPLICATE); + } + else + { + lb2 = load_balance_get(lbi2); - vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; - vnet_buffer (b1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; - vnet_buffer (b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index; - vnet_buffer (b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index; + if (PREDICT_FALSE(lb2->lb_n_buckets > 1)) + { + hash_c2 = vnet_buffer (b2)->ip.flow_hash = + mpls_compute_flow_hash(h2, lb2->lb_hash_config); + } + ASSERT (lb2->lb_n_buckets > 0); + ASSERT (is_pow2 (lb2->lb_n_buckets)); + dpo2 = load_balance_get_bucket_i(lb2, + (hash_c2 & + (lb2->lb_n_buckets_minus_1))); + next2 = dpo2->dpoi_next_node; + + vnet_buffer (b2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index; + + vlib_increment_combined_counter + (cm, thread_index, lbi2, 1, + vlib_buffer_length_in_chain (vm, b2)); + } + if (MPLS_IS_REPLICATE & lbi3) + { + next3 = mpls_lookup_to_replicate_edge; + vnet_buffer (b3)->ip.adj_index[VLIB_TX] = + (lbi3 & ~MPLS_IS_REPLICATE); + } + else + { + lb3 = load_balance_get(lbi3); - vlib_increment_combined_counter - (cm, thread_index, lbi0, 1, - vlib_buffer_length_in_chain (vm, b0)); - vlib_increment_combined_counter - (cm, thread_index, lbi1, 1, - vlib_buffer_length_in_chain (vm, b1)); - vlib_increment_combined_counter - (cm, thread_index, lbi2, 1, - vlib_buffer_length_in_chain (vm, b2)); - vlib_increment_combined_counter - (cm, thread_index, lbi3, 1, - vlib_buffer_length_in_chain (vm, b3)); + if (PREDICT_FALSE(lb3->lb_n_buckets > 1)) + { + hash_c3 = vnet_buffer (b3)->ip.flow_hash = + mpls_compute_flow_hash(h3, lb3->lb_hash_config); + } + ASSERT (lb3->lb_n_buckets > 0); + ASSERT (is_pow2 (lb3->lb_n_buckets)); + dpo3 = load_balance_get_bucket_i(lb3, + (hash_c3 & + (lb3->lb_n_buckets_minus_1))); + next3 = dpo3->dpoi_next_node; + + vnet_buffer (b3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index; + + vlib_increment_combined_counter + (cm, thread_index, lbi3, 1, + vlib_buffer_length_in_chain (vm, b3)); + } /* * before we pop the label copy th values we need to maintain. @@ -331,31 +382,41 @@ mpls_lookup (vlib_main_t * vm, vnet_buffer(b0)->sw_if_index[VLIB_RX]); lbi0 = mpls_fib_table_forwarding_lookup(lfib_index0, h0); - lb0 = load_balance_get(lbi0); - hash_c0 = vnet_buffer(b0)->ip.flow_hash = 0; - if (PREDICT_FALSE(lb0->lb_n_buckets > 1)) + + if (MPLS_IS_REPLICATE & lbi0) { - hash_c0 = vnet_buffer (b0)->ip.flow_hash = - mpls_compute_flow_hash(h0, lb0->lb_hash_config); + next0 = mpls_lookup_to_replicate_edge; + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = + (lbi0 & ~MPLS_IS_REPLICATE); } + else + { + lb0 = load_balance_get(lbi0); - ASSERT (lb0->lb_n_buckets > 0); - ASSERT (is_pow2 (lb0->lb_n_buckets)); + if (PREDICT_FALSE(lb0->lb_n_buckets > 1)) + { + hash_c0 = vnet_buffer (b0)->ip.flow_hash = + mpls_compute_flow_hash(h0, lb0->lb_hash_config); + } - dpo0 = load_balance_get_bucket_i(lb0, - (hash_c0 & - (lb0->lb_n_buckets_minus_1))); + ASSERT (lb0->lb_n_buckets > 0); + ASSERT (is_pow2 (lb0->lb_n_buckets)); - next0 = dpo0->dpoi_next_node; - vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; + dpo0 = load_balance_get_bucket_i(lb0, + (hash_c0 & + (lb0->lb_n_buckets_minus_1))); - vlib_increment_combined_counter - (cm, thread_index, lbi0, 1, - vlib_buffer_length_in_chain (vm, b0)); + next0 = dpo0->dpoi_next_node; + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; + + vlib_increment_combined_counter + (cm, thread_index, lbi0, 1, + vlib_buffer_length_in_chain (vm, b0)); + } /* - * before we pop the label copy th values we need to maintain. + * before we pop the label copy, values we need to maintain. * The label header is in network byte order. * last byte is the TTL. * bits 2 to 4 inclusive are the EXP bits @@ -398,7 +459,7 @@ static char * mpls_error_strings[] = { #undef mpls_error }; -VLIB_REGISTER_NODE (mpls_lookup_node) = { +VLIB_REGISTER_NODE (mpls_lookup_node, static) = { .function = mpls_lookup, .name = "mpls-lookup", /* Takes a vector of packets. */ @@ -621,3 +682,22 @@ VLIB_REGISTER_NODE (mpls_load_balance_node) = { }; VLIB_NODE_FUNCTION_MULTIARCH (mpls_load_balance_node, mpls_load_balance) + + +static clib_error_t * +mpls_lookup_init (vlib_main_t * vm) +{ + clib_error_t * error; + + if ((error = vlib_call_init_function (vm, mpls_init))) + return error; + + mpls_lookup_to_replicate_edge = + vlib_node_add_named_next(vm, + mpls_lookup_node.index, + "mpls-replicate"); + + return (NULL); +} + +VLIB_INIT_FUNCTION (mpls_lookup_init); diff --git a/src/vnet/mpls/mpls_tunnel.c b/src/vnet/mpls/mpls_tunnel.c index ac6fdcdf..1254dd9d 100644 --- a/src/vnet/mpls/mpls_tunnel.c +++ b/src/vnet/mpls/mpls_tunnel.c @@ -18,9 +18,12 @@ #include #include #include +#include #include #include #include +#include +#include /** * @brief pool of tunnel instances @@ -37,6 +40,11 @@ static u32 * mpls_tunnel_free_hw_if_indices; */ static u32 *mpls_tunnel_db; +/** + * @brief MPLS tunnel flags strings + */ +static const char *mpls_tunnel_attribute_names[] = MPLS_TUNNEL_ATTRIBUTES; + /** * @brief Get a tunnel object from a SW interface index */ @@ -44,103 +52,178 @@ static mpls_tunnel_t* mpls_tunnel_get_from_sw_if_index (u32 sw_if_index) { if ((vec_len(mpls_tunnel_db) < sw_if_index) || - (~0 == mpls_tunnel_db[sw_if_index])) - return (NULL); + (~0 == mpls_tunnel_db[sw_if_index])) + return (NULL); return (pool_elt_at_index(mpls_tunnel_pool, - mpls_tunnel_db[sw_if_index])); + mpls_tunnel_db[sw_if_index])); } /** - * @brief Return true if the label stack is imp-null only + * @brief Build a rewrite string for the MPLS tunnel. */ -static fib_forward_chain_type_t -mpls_tunnel_get_fwd_chain_type (const mpls_tunnel_t *mt) +static u8* +mpls_tunnel_build_rewrite_i (void) { - if ((1 == vec_len(mt->mt_label_stack)) && - (mt->mt_label_stack[0] == MPLS_IETF_IMPLICIT_NULL_LABEL)) - { - /* - * the only label in the label stack is implicit null - * we need to build an IP chain. - */ - if (FIB_PROTOCOL_IP4 == fib_path_list_get_proto(mt->mt_path_list)) - { - return (FIB_FORW_CHAIN_TYPE_UNICAST_IP4); - } - else - { - return (FIB_FORW_CHAIN_TYPE_UNICAST_IP6); - } - } - else - { - return (FIB_FORW_CHAIN_TYPE_MPLS_NON_EOS); - } + /* + * passing the adj code a NULL rewirte means 'i don't have one cos + * t'other end is unresolved'. That's not the case here. For the mpls + * tunnel there are just no bytes of encap to apply in the adj. We'll impose + * the label stack once we choose a path. So return a zero length rewrite. + */ + u8 *rewrite = NULL; + + vec_validate(rewrite, 0); + vec_reset_length(rewrite); + + return (rewrite); } /** * @brief Build a rewrite string for the MPLS tunnel. - * - * We have choices here; - * 1 - have an Adjacency with a zero length string and stack it on - * MPLS label objects - * 2 - put the label header rewrites in the adjacency string. - * - * We choose 2 since it results in fewer graph nodes in the egress path */ static u8* mpls_tunnel_build_rewrite (vnet_main_t * vnm, - u32 sw_if_index, - vnet_link_t link_type, - const void *dst_address) + u32 sw_if_index, + vnet_link_t link_type, + const void *dst_address) { - mpls_unicast_header_t *muh; - mpls_tunnel_t *mt; - u8 *rewrite; - u32 mti, ii; + return (mpls_tunnel_build_rewrite_i()); +} - rewrite = NULL; - mti = mpls_tunnel_db[sw_if_index]; - mt = pool_elt_at_index(mpls_tunnel_pool, mti); +typedef struct mpls_tunnel_collect_forwarding_ctx_t_ +{ + load_balance_path_t * next_hops; + const mpls_tunnel_t *mt; + fib_forward_chain_type_t fct; +} mpls_tunnel_collect_forwarding_ctx_t; + +static int +mpls_tunnel_collect_forwarding (fib_node_index_t pl_index, + fib_node_index_t path_index, + void *arg) +{ + mpls_tunnel_collect_forwarding_ctx_t *ctx; + fib_path_ext_t *path_ext; + int have_path_ext; + + ctx = arg; /* - * The vector must be allocated as u8 so the length is correct + * if the path is not resolved, don't include it. */ - ASSERT(0 < vec_len(mt->mt_label_stack)); - vec_validate(rewrite, (sizeof(*muh) * vec_len(mt->mt_label_stack)) - 1); - ASSERT(rewrite); - muh = (mpls_unicast_header_t *)rewrite; + if (!fib_path_is_resolved(path_index)) + { + return (!0); + } /* - * The last (inner most) label in the stack may be EOS, all the rest Non-EOS + * get the matching path-extension for the path being visited. */ - for (ii = 0; ii < vec_len(mt->mt_label_stack)-1; ii++) + have_path_ext = 0; + vec_foreach(path_ext, ctx->mt->mt_path_exts) { - vnet_mpls_uc_set_label(&muh[ii].label_exp_s_ttl, mt->mt_label_stack[ii]); - vnet_mpls_uc_set_ttl(&muh[ii].label_exp_s_ttl, 255); - vnet_mpls_uc_set_exp(&muh[ii].label_exp_s_ttl, 0); - vnet_mpls_uc_set_s(&muh[ii].label_exp_s_ttl, MPLS_NON_EOS); - muh[ii].label_exp_s_ttl = clib_host_to_net_u32(muh[ii].label_exp_s_ttl); + if (path_ext->fpe_path_index == path_index) + { + have_path_ext = 1; + break; + } } - vnet_mpls_uc_set_label(&muh[ii].label_exp_s_ttl, mt->mt_label_stack[ii]); - vnet_mpls_uc_set_ttl(&muh[ii].label_exp_s_ttl, 255); - vnet_mpls_uc_set_exp(&muh[ii].label_exp_s_ttl, 0); - - if ((VNET_LINK_MPLS == link_type) && - (mt->mt_label_stack[ii] != MPLS_IETF_IMPLICIT_NULL_LABEL)) + if (have_path_ext) { - vnet_mpls_uc_set_s(&muh[ii].label_exp_s_ttl, MPLS_NON_EOS); + /* + * found a matching extension. stack it to obtain the forwarding + * info for this path. + */ + ctx->next_hops = fib_path_ext_stack(path_ext, + ctx->fct, + ctx->fct, + ctx->next_hops); } else + ASSERT(0); + /* + * else + * There should be a path-extenios associated with each path + */ + + return (!0); +} + +static void +mpls_tunnel_mk_lb (mpls_tunnel_t *mt, + vnet_link_t linkt, + fib_forward_chain_type_t fct, + dpo_id_t *dpo_lb) +{ + dpo_proto_t lb_proto; + + /* + * If the entry has path extensions then we construct a load-balance + * by stacking the extensions on the forwarding chains of the paths. + * Otherwise we use the load-balance of the path-list + */ + mpls_tunnel_collect_forwarding_ctx_t ctx = { + .mt = mt, + .next_hops = NULL, + .fct = fct, + }; + + /* + * As an optimisation we allocate the vector of next-hops to be sized + * equal to the maximum nuber of paths we will need, which is also the + * most likely number we will need, since in most cases the paths are 'up'. + */ + vec_validate(ctx.next_hops, fib_path_list_get_n_paths(mt->mt_path_list)); + vec_reset_length(ctx.next_hops); + + lb_proto = vnet_link_to_dpo_proto(linkt); + + fib_path_list_walk(mt->mt_path_list, + mpls_tunnel_collect_forwarding, + &ctx); + + if (!dpo_id_is_valid(dpo_lb)) { - vnet_mpls_uc_set_s(&muh[ii].label_exp_s_ttl, MPLS_EOS); + /* + * first time create + */ + if (mt->mt_flags & MPLS_TUNNEL_FLAG_MCAST) + { + dpo_set(dpo_lb, + DPO_REPLICATE, + lb_proto, + replicate_create(0, lb_proto)); + } + else + { + flow_hash_config_t fhc; + + fhc = 0; // FIXME + /* fhc = fib_table_get_flow_hash_config(fib_entry->fe_fib_index, */ + /* dpo_proto_to_fib(lb_proto)); */ + dpo_set(dpo_lb, + DPO_LOAD_BALANCE, + lb_proto, + load_balance_create(0, lb_proto, fhc)); + } } - muh[ii].label_exp_s_ttl = clib_host_to_net_u32(muh[ii].label_exp_s_ttl); - - return (rewrite); + if (mt->mt_flags & MPLS_TUNNEL_FLAG_MCAST) + { + /* + * MPLS multicast + */ + replicate_multipath_update(dpo_lb, ctx.next_hops); + } + else + { + load_balance_multipath_update(dpo_lb, + ctx.next_hops, + LOAD_BALANCE_FLAG_NONE); + vec_free(ctx.next_hops); + } } /** @@ -161,45 +244,47 @@ mpls_tunnel_stack (adj_index_t ai) mt = mpls_tunnel_get_from_sw_if_index(sw_if_index); if (NULL == mt) - return; + return; /* - * find the adjacency that is contributed by the FIB path-list - * that this tunnel resovles via, and use it as the next adj - * in the midchain + * while we're stacking the adj, remove the tunnel from the child list + * of the path list. this breaks a circular dependency of walk updates + * where the create of adjacencies in the children can lead to walks + * that get back here. */ - if (vnet_hw_interface_get_flags(vnet_get_main(), - mt->mt_hw_if_index) & - VNET_HW_INTERFACE_FLAG_LINK_UP) - { - dpo_id_t dpo = DPO_INVALID; + fib_path_list_lock(mt->mt_path_list); - fib_path_list_contribute_forwarding(mt->mt_path_list, - mpls_tunnel_get_fwd_chain_type(mt), - &dpo); - - if (DPO_LOAD_BALANCE == dpo.dpoi_type) - { - /* - * we don't support multiple paths, so no need to load-balance. - * pull the first and only choice and stack directly on that. - */ - load_balance_t *lb; - - lb = load_balance_get (dpo.dpoi_index); + fib_path_list_child_remove(mt->mt_path_list, + mt->mt_sibling_index); - ASSERT(1 == lb->lb_n_buckets); + /* + * Construct the DPO (load-balance or replicate) that we can stack + * the tunnel's midchain on + */ + if (vnet_hw_interface_get_flags(vnet_get_main(), + mt->mt_hw_if_index) & + VNET_HW_INTERFACE_FLAG_LINK_UP) + { + dpo_id_t dpo = DPO_INVALID; - dpo_copy(&dpo, load_balance_get_bucket_i (lb, 0)); - } + mpls_tunnel_mk_lb(mt, + adj->ia_link, + FIB_FORW_CHAIN_TYPE_MPLS_EOS, + &dpo); - adj_nbr_midchain_stack(ai, &dpo); - dpo_reset(&dpo); + adj_nbr_midchain_stack(ai, &dpo); + dpo_reset(&dpo); } else { - adj_nbr_midchain_unstack(ai); + adj_nbr_midchain_unstack(ai); } + + mt->mt_sibling_index = fib_path_list_child_add(mt->mt_path_list, + FIB_NODE_TYPE_MPLS_TUNNEL, + mt - mpls_tunnel_pool); + + fib_path_list_lock(mt->mt_path_list); } /** @@ -207,7 +292,7 @@ mpls_tunnel_stack (adj_index_t ai) */ static adj_walk_rc_t mpls_adj_walk_cb (adj_index_t ai, - void *ctx) + void *ctx) { mpls_tunnel_stack(ai); @@ -224,17 +309,17 @@ mpls_tunnel_restack (mpls_tunnel_t *mt) */ FOR_EACH_FIB_PROTOCOL(proto) { - adj_nbr_walk(mt->mt_sw_if_index, - proto, - mpls_adj_walk_cb, - NULL); + adj_nbr_walk(mt->mt_sw_if_index, + proto, + mpls_adj_walk_cb, + NULL); } } static clib_error_t * mpls_tunnel_admin_up_down (vnet_main_t * vnm, - u32 hw_if_index, - u32 flags) + u32 hw_if_index, + u32 flags) { vnet_hw_interface_t * hi; mpls_tunnel_t *mt; @@ -244,13 +329,13 @@ mpls_tunnel_admin_up_down (vnet_main_t * vnm, mt = mpls_tunnel_get_from_sw_if_index(hi->sw_if_index); if (NULL == mt) - return (NULL); + return (NULL); if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) - vnet_hw_interface_set_flags (vnm, hw_if_index, - VNET_HW_INTERFACE_FLAG_LINK_UP); + vnet_hw_interface_set_flags (vnm, hw_if_index, + VNET_HW_INTERFACE_FLAG_LINK_UP); else - vnet_hw_interface_set_flags (vnm, hw_if_index, 0 /* down */); + vnet_hw_interface_set_flags (vnm, hw_if_index, 0 /* down */); mpls_tunnel_restack(mt); @@ -263,22 +348,58 @@ mpls_tunnel_admin_up_down (vnet_main_t * vnm, */ static void mpls_tunnel_fixup (vlib_main_t *vm, - ip_adjacency_t *adj, - vlib_buffer_t *b0) + ip_adjacency_t *adj, + vlib_buffer_t *b0) { + /* + * A no-op w.r.t. the header. but reset the 'have we pushed any + * MPLS labels onto the packet' flag. That way when we enter the + * tunnel we'll get a TTL set to 255 + */ + vnet_buffer(b0)->mpls.first = 0; } static void mpls_tunnel_update_adj (vnet_main_t * vnm, - u32 sw_if_index, - adj_index_t ai) + u32 sw_if_index, + adj_index_t ai) { - adj_nbr_midchain_update_rewrite( - ai, mpls_tunnel_fixup, - ADJ_FLAG_NONE, - mpls_tunnel_build_rewrite(vnm, sw_if_index, - adj_get_link_type(ai), - NULL)); + ip_adjacency_t *adj; + + ASSERT(ADJ_INDEX_INVALID != ai); + + adj = adj_get(ai); + + switch (adj->lookup_next_index) + { + case IP_LOOKUP_NEXT_ARP: + case IP_LOOKUP_NEXT_GLEAN: + adj_nbr_midchain_update_rewrite(ai, mpls_tunnel_fixup, + ADJ_FLAG_NONE, + mpls_tunnel_build_rewrite_i()); + break; + case IP_LOOKUP_NEXT_MCAST: + /* + * Construct a partial rewrite from the known ethernet mcast dest MAC + * There's no MAC fixup, so the last 2 parameters are 0 + */ + adj_mcast_midchain_update_rewrite(ai, mpls_tunnel_fixup, + ADJ_FLAG_NONE, + mpls_tunnel_build_rewrite_i(), + 0, 0); + break; + + case IP_LOOKUP_NEXT_DROP: + case IP_LOOKUP_NEXT_PUNT: + case IP_LOOKUP_NEXT_LOCAL: + case IP_LOOKUP_NEXT_REWRITE: + case IP_LOOKUP_NEXT_MIDCHAIN: + case IP_LOOKUP_NEXT_MCAST_MIDCHAIN: + case IP_LOOKUP_NEXT_ICMP_ERROR: + case IP_LOOKUP_N_NEXT: + ASSERT (0); + break; + } mpls_tunnel_stack(ai); } @@ -312,7 +433,7 @@ typedef struct mpls_tunnel_trace_t_ static u8 * format_mpls_tunnel_tx_trace (u8 * s, - va_list * args) + va_list * args) { CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); @@ -327,8 +448,8 @@ format_mpls_tunnel_tx_trace (u8 * s, */ static uword mpls_tunnel_tx (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) + vlib_node_runtime_t * node, + vlib_frame_t * frame) { u32 next_index; u32 * from, * to_next, n_left_from, n_left_to_next; @@ -355,32 +476,32 @@ mpls_tunnel_tx (vlib_main_t * vm, * FIXME DUAL LOOP */ while (n_left_from > 0 && n_left_to_next > 0) - { - vlib_buffer_t * b0; - u32 bi0; + { + vlib_buffer_t * b0; + u32 bi0; - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; - b0 = vlib_get_buffer(vm, bi0); + b0 = vlib_get_buffer(vm, bi0); - vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mt->mt_l2_adj; + vnet_buffer(b0)->ip.adj_index[VLIB_TX] = mt->mt_l2_adj; - if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) - { - mpls_tunnel_trace_t *tr = vlib_add_trace (vm, node, - b0, sizeof (*tr)); - tr->tunnel_id = rd->dev_instance; - } + if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) + { + mpls_tunnel_trace_t *tr = vlib_add_trace (vm, node, + b0, sizeof (*tr)); + tr->tunnel_id = rd->dev_instance; + } - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, mt->mt_l2_tx_arc); - } + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, mt->mt_l2_tx_arc); + } vlib_put_next_frame (vm, node, next_index, n_left_to_next); } @@ -417,13 +538,13 @@ mpls_tunnel_get (u32 mti) */ void mpls_tunnel_walk (mpls_tunnel_walk_cb_t cb, - void *ctx) + void *ctx) { u32 mti; pool_foreach_index(mti, mpls_tunnel_pool, ({ - cb(mti, ctx); + cb(mti, ctx); })); } @@ -435,25 +556,22 @@ vnet_mpls_tunnel_del (u32 sw_if_index) mt = mpls_tunnel_get_from_sw_if_index(sw_if_index); if (NULL == mt) - return; - - fib_path_list_child_remove(mt->mt_path_list, - mt->mt_sibling_index); - if (ADJ_INDEX_INVALID != mt->mt_l2_adj) - adj_unlock(mt->mt_l2_adj); + return; - vec_free(mt->mt_label_stack); + if (FIB_NODE_INDEX_INVALID != mt->mt_path_list) + fib_path_list_child_remove(mt->mt_path_list, + mt->mt_sibling_index); + if (ADJ_INDEX_INVALID != mt->mt_l2_adj) + adj_unlock(mt->mt_l2_adj); vec_add1 (mpls_tunnel_free_hw_if_indices, mt->mt_hw_if_index); pool_put(mpls_tunnel_pool, mt); mpls_tunnel_db[sw_if_index] = ~0; } -void -vnet_mpls_tunnel_add (fib_route_path_t *rpaths, - mpls_label_t *label_stack, - u8 l2_only, - u32 *sw_if_index) +u32 +vnet_mpls_tunnel_create (u8 l2_only, + u8 is_multicast) { vnet_hw_interface_t * hi; mpls_tunnel_t *mt; @@ -466,28 +584,33 @@ vnet_mpls_tunnel_add (fib_route_path_t *rpaths, mti = mt - mpls_tunnel_pool; fib_node_init(&mt->mt_node, FIB_NODE_TYPE_MPLS_TUNNEL); mt->mt_l2_adj = ADJ_INDEX_INVALID; + mt->mt_path_list = FIB_NODE_INDEX_INVALID; + mt->mt_sibling_index = FIB_NODE_INDEX_INVALID; + + if (is_multicast) + mt->mt_flags |= MPLS_TUNNEL_FLAG_MCAST; /* * Create a new, or re=use and old, tunnel HW interface */ if (vec_len (mpls_tunnel_free_hw_if_indices) > 0) { - mt->mt_hw_if_index = - mpls_tunnel_free_hw_if_indices[vec_len(mpls_tunnel_free_hw_if_indices)-1]; - _vec_len (mpls_tunnel_free_hw_if_indices) -= 1; - hi = vnet_get_hw_interface (vnm, mt->mt_hw_if_index); - hi->hw_instance = mti; - hi->dev_instance = mti; + mt->mt_hw_if_index = + mpls_tunnel_free_hw_if_indices[vec_len(mpls_tunnel_free_hw_if_indices)-1]; + _vec_len (mpls_tunnel_free_hw_if_indices) -= 1; + hi = vnet_get_hw_interface (vnm, mt->mt_hw_if_index); + hi->hw_instance = mti; + hi->dev_instance = mti; } - else + else { - mt->mt_hw_if_index = vnet_register_interface( - vnm, - mpls_tunnel_class.index, - mti, - mpls_tunnel_hw_interface_class.index, - mti); - hi = vnet_get_hw_interface(vnm, mt->mt_hw_if_index); + mt->mt_hw_if_index = vnet_register_interface( + vnm, + mpls_tunnel_class.index, + mti, + mpls_tunnel_hw_interface_class.index, + mti); + hi = vnet_get_hw_interface(vnm, mt->mt_hw_if_index); } /* @@ -497,43 +620,218 @@ vnet_mpls_tunnel_add (fib_route_path_t *rpaths, vec_validate_init_empty(mpls_tunnel_db, mt->mt_sw_if_index, ~0); mpls_tunnel_db[mt->mt_sw_if_index] = mti; + if (l2_only) + { + mt->mt_l2_adj = + adj_nbr_add_or_lock(fib_path_list_get_proto(mt->mt_path_list), + VNET_LINK_ETHERNET, + &zero_addr, + mt->mt_sw_if_index); + + mt->mt_l2_tx_arc = vlib_node_add_named_next(vlib_get_main(), + hi->tx_node_index, + "adj-l2-midchain"); + } + + return (mt->mt_sw_if_index); +} + +/* + * mpls_tunnel_path_ext_add + * + * append a path extension to the entry's list + */ +static void +mpls_tunnel_path_ext_append (mpls_tunnel_t *mt, + const fib_route_path_t *rpath) +{ + if (NULL != rpath->frp_label_stack) + { + fib_path_ext_t *path_ext; + + vec_add2(mt->mt_path_exts, path_ext, 1); + + fib_path_ext_init(path_ext, mt->mt_path_list, rpath); + } +} + +/* + * mpls_tunnel_path_ext_insert + * + * insert, sorted, a path extension to the entry's list. + * It's not strictly necessary in sort the path extensions, since each + * extension has the path index to which it resolves. However, by being + * sorted the load-balance produced has a deterministic order, not an order + * based on the sequence of extension additions. this is a considerable benefit. + */ +static void +mpls_tunnel_path_ext_insert (mpls_tunnel_t *mt, + const fib_route_path_t *rpath) +{ + if (0 == vec_len(mt->mt_path_exts)) + return (mpls_tunnel_path_ext_append(mt, rpath)); + + if (NULL != rpath->frp_label_stack) + { + fib_path_ext_t path_ext; + int i = 0; + + fib_path_ext_init(&path_ext, mt->mt_path_list, rpath); + + while (i < vec_len(mt->mt_path_exts) && + (fib_path_ext_cmp(&mt->mt_path_exts[i], rpath) < 0)) + { + i++; + } + + vec_insert_elts(mt->mt_path_exts, &path_ext, 1, i); + } +} + +void +vnet_mpls_tunnel_path_add (u32 sw_if_index, + fib_route_path_t *rpaths) +{ + mpls_tunnel_t *mt; + u32 mti; + + mt = mpls_tunnel_get_from_sw_if_index(sw_if_index); + + if (NULL == mt) + return; + + mti = mt - mpls_tunnel_pool; + /* * construct a path-list from the path provided */ - mt->mt_path_list = fib_path_list_create(FIB_PATH_LIST_FLAG_SHARED, rpaths); - mt->mt_sibling_index = fib_path_list_child_add(mt->mt_path_list, - FIB_NODE_TYPE_MPLS_TUNNEL, - mti); + if (FIB_NODE_INDEX_INVALID == mt->mt_path_list) + { + mt->mt_path_list = fib_path_list_create(FIB_PATH_LIST_FLAG_SHARED, rpaths); + mt->mt_sibling_index = fib_path_list_child_add(mt->mt_path_list, + FIB_NODE_TYPE_MPLS_TUNNEL, + mti); + } + else + { + fib_node_index_t old_pl_index; + fib_path_ext_t *path_ext; + + old_pl_index = mt->mt_path_list; + + mt->mt_path_list = + fib_path_list_copy_and_path_add(old_pl_index, + FIB_PATH_LIST_FLAG_SHARED, + rpaths); + + fib_path_list_child_remove(old_pl_index, + mt->mt_sibling_index); + mt->mt_sibling_index = fib_path_list_child_add(mt->mt_path_list, + FIB_NODE_TYPE_MPLS_TUNNEL, + mti); + /* + * re-resolve all the path-extensions with the new path-list + */ + vec_foreach(path_ext, mt->mt_path_exts) + { + fib_path_ext_resolve(path_ext, mt->mt_path_list); + } + } + mpls_tunnel_path_ext_insert(mt, rpaths); + mpls_tunnel_restack(mt); +} + +int +vnet_mpls_tunnel_path_remove (u32 sw_if_index, + fib_route_path_t *rpaths) +{ + mpls_tunnel_t *mt; + u32 mti; - mt->mt_label_stack = vec_dup(label_stack); + mt = mpls_tunnel_get_from_sw_if_index(sw_if_index); - if (l2_only) + if (NULL == mt) + return (0); + + mti = mt - mpls_tunnel_pool; + + /* + * construct a path-list from the path provided + */ + if (FIB_NODE_INDEX_INVALID == mt->mt_path_list) { - mt->mt_l2_adj = - adj_nbr_add_or_lock(fib_path_list_get_proto(mt->mt_path_list), - VNET_LINK_ETHERNET, - &zero_addr, - mt->mt_sw_if_index); - - mt->mt_l2_tx_arc = vlib_node_add_named_next(vlib_get_main(), - hi->tx_node_index, - "adj-l2-midchain"); + /* can't remove a path if we have onoe */ + return (0); } - - *sw_if_index = mt->mt_sw_if_index; + else + { + fib_node_index_t old_pl_index; + fib_path_ext_t *path_ext; + + old_pl_index = mt->mt_path_list; + + mt->mt_path_list = + fib_path_list_copy_and_path_remove(old_pl_index, + FIB_PATH_LIST_FLAG_SHARED, + rpaths); + + fib_path_list_child_remove(old_pl_index, + mt->mt_sibling_index); + + if (FIB_NODE_INDEX_INVALID == mt->mt_path_list) + { + /* no paths left */ + return (0); + } + else + { + mt->mt_sibling_index = + fib_path_list_child_add(mt->mt_path_list, + FIB_NODE_TYPE_MPLS_TUNNEL, + mti); + } + /* + * find the matching path extension and remove it + */ + vec_foreach(path_ext, mt->mt_path_exts) + { + if (!fib_path_ext_cmp(path_ext, rpaths)) + { + /* + * delete the element moving the remaining elements down 1 position. + * this preserves the sorted order. + */ + vec_free(path_ext->fpe_label_stack); + vec_delete(mt->mt_path_exts, 1, + (path_ext - mt->mt_path_exts)); + break; + } + } + /* + * re-resolve all the path-extensions with the new path-list + */ + vec_foreach(path_ext, mt->mt_path_exts) + { + fib_path_ext_resolve(path_ext, mt->mt_path_list); + } + + mpls_tunnel_restack(mt); + } + + return (fib_path_list_get_n_paths(mt->mt_path_list)); } + static clib_error_t * vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) + unformat_input_t * input, + vlib_cli_command_t * cmd) { unformat_input_t _line_input, * line_input = &_line_input; vnet_main_t * vnm = vnet_get_main(); - u8 is_del = 0; - u8 l2_only = 0; + u8 is_del = 0, l2_only = 0, is_multicast =0; fib_route_path_t rpath, *rpaths = NULL; - mpls_label_t out_label = MPLS_LABEL_INVALID, *labels = NULL; + mpls_label_t out_label = MPLS_LABEL_INVALID; u32 sw_if_index; clib_error_t *error = NULL; @@ -541,87 +839,89 @@ vnet_create_mpls_tunnel_command_fn (vlib_main_t * vm, /* Get a line of input. */ if (! unformat_user (input, unformat_line_input, line_input)) - return 0; + return 0; while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) { - if (unformat (line_input, "del %U", - unformat_vnet_sw_interface, vnm, - &sw_if_index)) - is_del = 1; - else if (unformat (line_input, "add")) - is_del = 0; - else if (unformat (line_input, "out-label %U", - unformat_mpls_unicast_label, &out_label)) - { - vec_add1(labels, out_label); - } - else if (unformat (line_input, "via %U %U", - unformat_ip4_address, - &rpath.frp_addr.ip4, - unformat_vnet_sw_interface, vnm, - &rpath.frp_sw_if_index)) - { - rpath.frp_weight = 1; - rpath.frp_proto = FIB_PROTOCOL_IP4; - } - - else if (unformat (line_input, "via %U %U", - unformat_ip6_address, - &rpath.frp_addr.ip6, - unformat_vnet_sw_interface, vnm, - &rpath.frp_sw_if_index)) - { - rpath.frp_weight = 1; - rpath.frp_proto = FIB_PROTOCOL_IP6; - } - else if (unformat (line_input, "via %U", - unformat_ip6_address, - &rpath.frp_addr.ip6)) - { - rpath.frp_fib_index = 0; - rpath.frp_weight = 1; - rpath.frp_sw_if_index = ~0; - rpath.frp_proto = FIB_PROTOCOL_IP6; - } - else if (unformat (line_input, "via %U", - unformat_ip4_address, - &rpath.frp_addr.ip4)) - { - rpath.frp_fib_index = 0; - rpath.frp_weight = 1; - rpath.frp_sw_if_index = ~0; - rpath.frp_proto = FIB_PROTOCOL_IP4; - } - else if (unformat (line_input, "l2-only")) - l2_only = 1; - else - { - error = clib_error_return (0, "unknown input '%U'", - format_unformat_error, line_input); - goto done; - } + if (unformat (line_input, "del %U", + unformat_vnet_sw_interface, vnm, + &sw_if_index)) + is_del = 1; + else if (unformat (line_input, "add")) + is_del = 0; + else if (unformat (line_input, "out-label %U", + unformat_mpls_unicast_label, &out_label)) + { + vec_add1(rpath.frp_label_stack, out_label); + } + else if (unformat (line_input, "via %U %U", + unformat_ip4_address, + &rpath.frp_addr.ip4, + unformat_vnet_sw_interface, vnm, + &rpath.frp_sw_if_index)) + { + rpath.frp_weight = 1; + rpath.frp_proto = FIB_PROTOCOL_IP4; + } + + else if (unformat (line_input, "via %U %U", + unformat_ip6_address, + &rpath.frp_addr.ip6, + unformat_vnet_sw_interface, vnm, + &rpath.frp_sw_if_index)) + { + rpath.frp_weight = 1; + rpath.frp_proto = FIB_PROTOCOL_IP6; + } + else if (unformat (line_input, "via %U", + unformat_ip6_address, + &rpath.frp_addr.ip6)) + { + rpath.frp_fib_index = 0; + rpath.frp_weight = 1; + rpath.frp_sw_if_index = ~0; + rpath.frp_proto = FIB_PROTOCOL_IP6; + } + else if (unformat (line_input, "via %U", + unformat_ip4_address, + &rpath.frp_addr.ip4)) + { + rpath.frp_fib_index = 0; + rpath.frp_weight = 1; + rpath.frp_sw_if_index = ~0; + rpath.frp_proto = FIB_PROTOCOL_IP4; + } + else if (unformat (line_input, "l2-only")) + l2_only = 1; + else if (unformat (line_input, "multicast")) + is_multicast = 1; + else + { + error = clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + goto done; + } } if (is_del) { - vnet_mpls_tunnel_del(sw_if_index); + vnet_mpls_tunnel_del(sw_if_index); } else { - if (0 == vec_len(labels)) - { - error = clib_error_return (0, "No Output Labels '%U'", - format_unformat_error, line_input); - goto done; - } - - vec_add1(rpaths, rpath); - vnet_mpls_tunnel_add(rpaths, labels, l2_only, &sw_if_index); + if (0 == vec_len(rpath.frp_label_stack)) + { + error = clib_error_return (0, "No Output Labels '%U'", + format_unformat_error, line_input); + goto done; + } + + vec_add1(rpaths, rpath); + sw_if_index = vnet_mpls_tunnel_create(l2_only, is_multicast); + vnet_mpls_tunnel_path_add(sw_if_index, rpaths); } done: - vec_free(labels); vec_free(rpaths); unformat_free (line_input); @@ -638,7 +938,7 @@ done: ?*/ VLIB_CLI_COMMAND (create_mpls_tunnel_command, static) = { .path = "mpls tunnel", - .short_help = + .short_help = "mpls tunnel via [addr] [interface] [out-labels]", .function = vnet_create_mpls_tunnel_command_fn, }; @@ -647,19 +947,28 @@ static u8 * format_mpls_tunnel (u8 * s, va_list * args) { mpls_tunnel_t *mt = va_arg (*args, mpls_tunnel_t *); - int ii; + mpls_tunnel_attribute_t attr; + fib_path_ext_t *path_ext; s = format(s, "mpls_tunnel%d: sw_if_index:%d hw_if_index:%d", - mt - mpls_tunnel_pool, - mt->mt_sw_if_index, - mt->mt_hw_if_index); - s = format(s, "\n label-stack:\n "); - for (ii = 0; ii < vec_len(mt->mt_label_stack); ii++) - { - s = format(s, "%d, ", mt->mt_label_stack[ii]); + mt - mpls_tunnel_pool, + mt->mt_sw_if_index, + mt->mt_hw_if_index); + if (MPLS_TUNNEL_FLAG_NONE != mt->mt_flags) { + s = format(s, " \n flags:"); + FOR_EACH_MPLS_TUNNEL_ATTRIBUTE(attr) { + if ((1<mt_flags) { + s = format (s, "%s,", mpls_tunnel_attribute_names[attr]); + } + } } s = format(s, "\n via:\n"); s = fib_path_list_format(mt->mt_path_list, s); + s = format(s, " Extensions:"); + vec_foreach(path_ext, mt->mt_path_exts) + { + s = format(s, "\n %U", format_fib_path_ext, path_ext); + } s = format(s, "\n"); return (s); @@ -667,42 +976,42 @@ format_mpls_tunnel (u8 * s, va_list * args) static clib_error_t * show_mpls_tunnel_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) + unformat_input_t * input, + vlib_cli_command_t * cmd) { mpls_tunnel_t * mt; u32 mti = ~0; if (pool_elts (mpls_tunnel_pool) == 0) - vlib_cli_output (vm, "No MPLS tunnels configured..."); + vlib_cli_output (vm, "No MPLS tunnels configured..."); while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { - if (unformat (input, "%d", &mti)) - ; - else - break; + if (unformat (input, "%d", &mti)) + ; + else + break; } if (~0 == mti) { - pool_foreach (mt, mpls_tunnel_pool, - ({ - vlib_cli_output (vm, "[@%d] %U", - mt - mpls_tunnel_pool, - format_mpls_tunnel, mt); - })); + pool_foreach (mt, mpls_tunnel_pool, + ({ + vlib_cli_output (vm, "[@%d] %U", + mt - mpls_tunnel_pool, + format_mpls_tunnel, mt); + })); } else { - if (pool_is_free_index(mpls_tunnel_pool, mti)) - return clib_error_return (0, "Not atunnel index %d", mti); + if (pool_is_free_index(mpls_tunnel_pool, mti)) + return clib_error_return (0, "Not atunnel index %d", mti); - mt = pool_elt_at_index(mpls_tunnel_pool, mti); + mt = pool_elt_at_index(mpls_tunnel_pool, mti); - vlib_cli_output (vm, "[@%d] %U", - mt - mpls_tunnel_pool, - format_mpls_tunnel, mt); + vlib_cli_output (vm, "[@%d] %U", + mt - mpls_tunnel_pool, + format_mpls_tunnel, mt); } return 0; @@ -715,7 +1024,7 @@ show_mpls_tunnel_command_fn (vlib_main_t * vm, * @cliexstart{sh mpls tunnel 2} * [@2] mpls_tunnel2: sw_if_index:5 hw_if_index:5 * label-stack: - * 3, + * 3, * via: * index:26 locks:1 proto:ipv4 uPRF-list:26 len:1 itfs:[2, ] * index:26 pl-index:26 ipv4 weight=1 attached-nexthop: oper-flags:resolved, @@ -743,7 +1052,7 @@ mpls_tunnel_from_fib_node (fib_node_t *node) */ static fib_node_back_walk_rc_t mpls_tunnel_back_walk (fib_node_t *node, - fib_node_back_walk_ctx_t *ctx) + fib_node_back_walk_ctx_t *ctx) { mpls_tunnel_restack(mpls_tunnel_from_fib_node(node)); diff --git a/src/vnet/mpls/mpls_tunnel.h b/src/vnet/mpls/mpls_tunnel.h index ee56c0fc..0b55d0db 100644 --- a/src/vnet/mpls/mpls_tunnel.h +++ b/src/vnet/mpls/mpls_tunnel.h @@ -17,6 +17,31 @@ #define __MPLS_TUNNEL_H__ #include +#include + +typedef enum mpls_tunnel_attribute_t_ +{ + MPLS_TUNNEL_ATTRIBUTE_FIRST = 0, + /** + * @brief The tunnel has an underlying multicast LSP + */ + MPLS_TUNNEL_ATTRIBUTE_MCAST = MPLS_TUNNEL_ATTRIBUTE_FIRST, + MPLS_TUNNEL_ATTRIBUTE_LAST = MPLS_TUNNEL_ATTRIBUTE_MCAST, +} mpls_tunnel_attribute_t; + +#define MPLS_TUNNEL_ATTRIBUTES { \ + [MPLS_TUNNEL_ATTRIBUTE_MCAST] = "multicast", \ +} +#define FOR_EACH_MPLS_TUNNEL_ATTRIBUTE(_item) \ + for (_item = MPLS_TUNNEL_ATTRIBUTE_FIRST; \ + _item < MPLS_TUNNEL_ATTRIBUTE_LAST; \ + _item++) + +typedef enum mpls_tunnel_flag_t_ { + MPLS_TUNNEL_FLAG_NONE = 0, + MPLS_TUNNEL_FLAG_MCAST = (1 << MPLS_TUNNEL_ATTRIBUTE_MCAST), +} __attribute__ ((packed)) mpls_tunnel_flags_t; + /** * @brief A uni-directional MPLS tunnel @@ -28,6 +53,11 @@ typedef struct mpls_tunnel_t_ */ fib_node_t mt_node; + /** + * @brief Tunnel flags + */ + mpls_tunnel_flags_t mt_flags; + /** * @brief If the tunnel is an L2 tunnel, this is the link type ETHERNET * adjacency @@ -50,9 +80,9 @@ typedef struct mpls_tunnel_t_ u32 mt_sibling_index; /** - * @brief The Label stack to apply to egress packets + * A vector of path extensions o hold the label stack for each path */ - mpls_label_t *mt_label_stack; + fib_path_ext_t *mt_path_exts; /** * @brief Flag to indicate the tunnel is only for L2 traffic, that is @@ -74,12 +104,27 @@ typedef struct mpls_tunnel_t_ /** * @brief Create a new MPLS tunnel + * @return the SW Interface index of the newly created tuneel */ -extern void vnet_mpls_tunnel_add (fib_route_path_t *rpath, - mpls_label_t *label_stack, - u8 l2_only, - u32 *sw_if_index); +extern u32 vnet_mpls_tunnel_create (u8 l2_only, + u8 is_multicast); +/** + * @brief Add a path to an MPLS tunnel + */ +extern void vnet_mpls_tunnel_path_add (u32 sw_if_index, + fib_route_path_t *rpath); + +/** + * @brief remove a path from a tunnel. + * @return the number of remaining paths. 0 implies the tunnel can be deleted + */ +extern int vnet_mpls_tunnel_path_remove (u32 sw_if_index, + fib_route_path_t *rpath); + +/** + * @brief Delete an MPLS tunnel + */ extern void vnet_mpls_tunnel_del (u32 sw_if_index); extern const mpls_tunnel_t *mpls_tunnel_get(u32 index); diff --git a/src/vnet/mpls/mpls_types.h b/src/vnet/mpls/mpls_types.h index d7c629df..b1075cdd 100644 --- a/src/vnet/mpls/mpls_types.h +++ b/src/vnet/mpls/mpls_types.h @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef __MPLS_TYPES_H__ #define __MPLS_TYPES_H__ @@ -36,4 +50,10 @@ (((_lbl) > MPLS_IETF_MIN_UNRES_LABEL) && \ ((_lbl) <= MPLS_IETF_MAX_UNRES_LABEL)) +/** + * The top bit of the index, which is the result of the MPLS lookup + * is used to determine if the DPO is a load-balance or a replicate + */ +#define MPLS_IS_REPLICATE 0x80000000 + #endif diff --git a/src/vnet/srp/interface.c b/src/vnet/srp/interface.c index d427cc3c..44e2b0d6 100644 --- a/src/vnet/srp/interface.c +++ b/src/vnet/srp/interface.c @@ -58,7 +58,7 @@ srp_build_rewrite (vnet_main_t * vnm, #define _(a,b) case VNET_LINK_##a: type = ETHERNET_TYPE_##b; break _ (IP4, IP4); _ (IP6, IP6); - _ (MPLS, MPLS_UNICAST); + _ (MPLS, MPLS); _ (ARP, ARP); #undef _ default: diff --git a/test/test_ip_mcast.py b/test/test_ip_mcast.py index 36d597a7..c1397d70 100644 --- a/test/test_ip_mcast.py +++ b/test/test_ip_mcast.py @@ -622,6 +622,7 @@ class TestIPMcast(VppTestCase): (MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT | MRouteItfFlags.MFIB_ITF_FLAG_NEGATE_SIGNAL)) + self.vapi.cli("clear trace") tx = self._mcast_connected_send_stream("232.1.1.1") signals = self.vapi.mfib_signal_dump() diff --git a/test/test_mpls.py b/test/test_mpls.py index fc832644..700b7091 100644 --- a/test/test_mpls.py +++ b/test/test_mpls.py @@ -5,7 +5,9 @@ import socket from framework import VppTestCase, VppTestRunner from vpp_ip_route import VppIpRoute, VppRoutePath, VppMplsRoute, \ - VppMplsIpBind + VppMplsIpBind, VppIpMRoute, VppMRoutePath, \ + MRouteItfFlags, MRouteEntryFlags +from vpp_mpls_tunnel_interface import VppMPLSTunnelInterface from scapy.packet import Raw from scapy.layers.l2 import Ether @@ -21,7 +23,7 @@ class TestMPLS(VppTestCase): super(TestMPLS, self).setUp() # create 2 pg interfaces - self.create_pg_interfaces(range(2)) + self.create_pg_interfaces(range(4)) # setup both interfaces # assign them different tables. @@ -53,10 +55,12 @@ class TestMPLS(VppTestCase): mpls_labels, mpls_ttl=255, ping=0, - ip_itf=None): + ip_itf=None, + dst_ip=None, + n=257): self.reset_packet_infos() pkts = [] - for i in range(0, 257): + for i in range(0, n): info = self.create_packet_info(src_if, src_if) payload = self.info_to_payload(info) p = Ether(dst=src_if.local_mac, src=src_if.remote_mac) @@ -67,9 +71,14 @@ class TestMPLS(VppTestCase): else: p = p / MPLS(label=mpls_labels[ii], ttl=mpls_ttl, s=0) if not ping: - p = (p / IP(src=src_if.local_ip4, dst=src_if.remote_ip4) / - UDP(sport=1234, dport=1234) / - Raw(payload)) + if not dst_ip: + p = (p / IP(src=src_if.local_ip4, dst=src_if.remote_ip4) / + UDP(sport=1234, dport=1234) / + Raw(payload)) + else: + p = (p / IP(src=src_if.local_ip4, dst=dst_ip) / + UDP(sport=1234, dport=1234) / + Raw(payload)) else: p = (p / IP(src=ip_itf.remote_ip4, dst=ip_itf.local_ip4) / @@ -254,6 +263,13 @@ class TestMPLS(VppTestCase): except: raise + def send_and_assert_no_replies(self, intf, pkts, remark): + intf.add_stream(pkts) + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + for i in self.pg_interfaces: + i.assert_nothing_captured(remark=remark) + def test_swap(self): """ MPLS label swap tests """ @@ -278,7 +294,7 @@ class TestMPLS(VppTestCase): self.pg_start() rx = self.pg0.get_capture() - self.verify_capture_labelled_ip4(self.pg0, rx, tx, [33]) + self.verify_capture_labelled(self.pg0, rx, tx, [33]) # # A simple MPLS xconnect - non-eos label in label out @@ -358,7 +374,7 @@ class TestMPLS(VppTestCase): self.pg_start() rx = self.pg0.get_capture() - self.verify_capture_labelled_ip4(self.pg0, rx, tx, [33, 44, 45]) + self.verify_capture_labelled(self.pg0, rx, tx, [33, 44, 45], num=2) # # A recursive non-EOS x-connect, which resolves through another @@ -576,25 +592,19 @@ class TestMPLS(VppTestCase): # # Create a tunnel with a single out label # - nh_addr = socket.inet_pton(socket.AF_INET, self.pg0.remote_ip4) - - reply = self.vapi.mpls_tunnel_add_del( - 0xffffffff, # don't know the if index yet - 1, # IPv4 next-hop - nh_addr, - self.pg0.sw_if_index, - 0, # next-hop-table-id - 1, # next-hop-weight - 2, # num-out-labels, - [44, 46]) - self.vapi.sw_interface_set_flags(reply.sw_if_index, admin_up_down=1) + mpls_tun = VppMPLSTunnelInterface(self, + [VppRoutePath(self.pg0.remote_ip4, + self.pg0.sw_if_index, + labels=[44, 46])]) + mpls_tun.add_vpp_config() + mpls_tun.admin_up() # # add an unlabelled route through the new tunnel # route_10_0_0_3 = VppIpRoute(self, "10.0.0.3", 32, [VppRoutePath("0.0.0.0", - reply.sw_if_index)]) + mpls_tun._sw_if_index)]) route_10_0_0_3.add_vpp_config() self.vapi.cli("clear trace") @@ -738,6 +748,229 @@ class TestMPLS(VppTestCase): route_35_eos.remove_vpp_config() route_34_eos.remove_vpp_config() + def test_interface_rx(self): + """ MPLS Interface Receive """ + + # + # Add a non-recursive route that will forward the traffic + # post-interface-rx + # + route_10_0_0_1 = VppIpRoute(self, "10.0.0.1", 32, + table_id=1, + paths=[VppRoutePath(self.pg1.remote_ip4, + self.pg1.sw_if_index)]) + route_10_0_0_1.add_vpp_config() + + # + # An interface receive label that maps traffic to RX on interface + # pg1 + # by injecting the packet in on pg0, which is in table 0 + # doing an interface-rx on pg1 and matching a route in table 1 + # if the packet egresses, then we must have swapped to pg1 + # so as to have matched the route in table 1 + # + route_34_eos = VppMplsRoute(self, 34, 1, + [VppRoutePath("0.0.0.0", + self.pg1.sw_if_index, + is_interface_rx=1)]) + route_34_eos.add_vpp_config() + + # + # ping an interface in the default table + # PG0 is in the default table + # + self.vapi.cli("clear trace") + tx = self.create_stream_labelled_ip4(self.pg0, [34], n=257, + dst_ip="10.0.0.1") + self.pg0.add_stream(tx) + + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + + rx = self.pg1.get_capture(257) + self.verify_capture_ip4(self.pg1, rx, tx) + + def test_mcast_mid_point(self): + """ MPLS Multicast Mid Point """ + + # + # Add a non-recursive route that will forward the traffic + # post-interface-rx + # + route_10_0_0_1 = VppIpRoute(self, "10.0.0.1", 32, + table_id=1, + paths=[VppRoutePath(self.pg1.remote_ip4, + self.pg1.sw_if_index)]) + route_10_0_0_1.add_vpp_config() + + # + # Add a mcast entry that replicate to pg2 and pg3 + # and replicate to a interface-rx (like a bud node would) + # + route_3400_eos = VppMplsRoute(self, 3400, 1, + [VppRoutePath(self.pg2.remote_ip4, + self.pg2.sw_if_index, + labels=[3401]), + VppRoutePath(self.pg3.remote_ip4, + self.pg3.sw_if_index, + labels=[3402]), + VppRoutePath("0.0.0.0", + self.pg1.sw_if_index, + is_interface_rx=1)], + is_multicast=1) + route_3400_eos.add_vpp_config() + + # + # ping an interface in the default table + # PG0 is in the default table + # + self.vapi.cli("clear trace") + tx = self.create_stream_labelled_ip4(self.pg0, [3400], n=257, + dst_ip="10.0.0.1") + self.pg0.add_stream(tx) + + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + + rx = self.pg1.get_capture(257) + self.verify_capture_ip4(self.pg1, rx, tx) + + rx = self.pg2.get_capture(257) + self.verify_capture_labelled(self.pg2, rx, tx, [3401]) + rx = self.pg3.get_capture(257) + self.verify_capture_labelled(self.pg3, rx, tx, [3402]) + + def test_mcast_head(self): + """ MPLS Multicast Head-end """ + + # + # Create a multicast tunnel with two replications + # + mpls_tun = VppMPLSTunnelInterface(self, + [VppRoutePath(self.pg2.remote_ip4, + self.pg2.sw_if_index, + labels=[42]), + VppRoutePath(self.pg3.remote_ip4, + self.pg3.sw_if_index, + labels=[43])], + is_multicast=1) + mpls_tun.add_vpp_config() + mpls_tun.admin_up() + + # + # add an unlabelled route through the new tunnel + # + route_10_0_0_3 = VppIpRoute(self, "10.0.0.3", 32, + [VppRoutePath("0.0.0.0", + mpls_tun._sw_if_index)]) + route_10_0_0_3.add_vpp_config() + + self.vapi.cli("clear trace") + tx = self.create_stream_ip4(self.pg0, "10.0.0.3") + self.pg0.add_stream(tx) + + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + + rx = self.pg2.get_capture(257) + self.verify_capture_tunneled_ip4(self.pg0, rx, tx, [42]) + rx = self.pg3.get_capture(257) + self.verify_capture_tunneled_ip4(self.pg0, rx, tx, [43]) + + # + # An an IP multicast route via the tunnel + # A (*,G). + # one accepting interface, pg0, 1 forwarding interface via the tunnel + # + route_232_1_1_1 = VppIpMRoute( + self, + "0.0.0.0", + "232.1.1.1", 32, + MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE, + [VppMRoutePath(self.pg0.sw_if_index, + MRouteItfFlags.MFIB_ITF_FLAG_ACCEPT), + VppMRoutePath(mpls_tun._sw_if_index, + MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)]) + route_232_1_1_1.add_vpp_config() + + self.vapi.cli("clear trace") + tx = self.create_stream_ip4(self.pg0, "232.1.1.1") + self.pg0.add_stream(tx) + + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + + rx = self.pg2.get_capture(257) + self.verify_capture_tunneled_ip4(self.pg0, rx, tx, [42]) + rx = self.pg3.get_capture(257) + self.verify_capture_tunneled_ip4(self.pg0, rx, tx, [43]) + + def test_mcast_tail(self): + """ MPLS Multicast Tail """ + + # + # Add a multicast route that will forward the traffic + # post-disposition + # + route_232_1_1_1 = VppIpMRoute( + self, + "0.0.0.0", + "232.1.1.1", 32, + MRouteEntryFlags.MFIB_ENTRY_FLAG_NONE, + table_id=1, + paths=[VppMRoutePath(self.pg1.sw_if_index, + MRouteItfFlags.MFIB_ITF_FLAG_FORWARD)]) + route_232_1_1_1.add_vpp_config() + + # + # An interface receive label that maps traffic to RX on interface + # pg1 + # by injecting the packet in on pg0, which is in table 0 + # doing an rpf-id and matching a route in table 1 + # if the packet egresses, then we must have matched the route in + # table 1 + # + route_34_eos = VppMplsRoute(self, 34, 1, + [VppRoutePath("0.0.0.0", + self.pg1.sw_if_index, + nh_table_id=1, + rpf_id=55)], + is_multicast=1) + + route_34_eos.add_vpp_config() + + # + # Drop due to interface lookup miss + # + self.vapi.cli("clear trace") + tx = self.create_stream_labelled_ip4(self.pg0, [34], + dst_ip="232.1.1.1", n=1) + self.send_and_assert_no_replies(self.pg0, tx, "RPF-ID drop none") + + # + # set the RPF-ID of the enrtry to match the input packet's + # + route_232_1_1_1.update_rpf_id(55) + + self.vapi.cli("clear trace") + tx = self.create_stream_labelled_ip4(self.pg0, [34], + dst_ip="232.1.1.1", n=257) + self.pg0.add_stream(tx) + + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + + rx = self.pg1.get_capture(257) + self.verify_capture_ip4(self.pg1, rx, tx) + + # + # set the RPF-ID of the enrtry to not match the input packet's + # + route_232_1_1_1.update_rpf_id(56) + tx = self.create_stream_labelled_ip4(self.pg0, [34], + dst_ip="232.1.1.1") + self.send_and_assert_no_replies(self.pg0, tx, "RPF-ID drop 56") + class TestMPLSDisabled(VppTestCase): """ MPLS disabled """ diff --git a/test/vpp_ip_route.py b/test/vpp_ip_route.py index faf5f801..d6146f28 100644 --- a/test/vpp_ip_route.py +++ b/test/vpp_ip_route.py @@ -55,15 +55,24 @@ class VppRoutePath(object): nh_table_id=0, labels=[], nh_via_label=MPLS_LABEL_INVALID, - is_ip6=0): + is_ip6=0, + rpf_id=0, + is_interface_rx=0): self.nh_itf = nh_sw_if_index self.nh_table_id = nh_table_id self.nh_via_label = nh_via_label self.nh_labels = labels + self.weight = 1 + self.rpf_id = rpf_id if is_ip6: self.nh_addr = inet_pton(AF_INET6, nh_addr) else: self.nh_addr = inet_pton(AF_INET, nh_addr) + self.is_interface_rx = is_interface_rx + self.is_rpf_id = 0 + if rpf_id != 0: + self.is_rpf_id = 1 + self.nh_itf = rpf_id class VppMRoutePath(VppRoutePath): @@ -176,13 +185,15 @@ class VppIpMRoute(VppObject): """ def __init__(self, test, src_addr, grp_addr, - grp_addr_len, e_flags, paths, table_id=0, is_ip6=0): + grp_addr_len, e_flags, paths, table_id=0, + rpf_id=0, is_ip6=0): self._test = test self.paths = paths self.grp_addr_len = grp_addr_len self.table_id = table_id self.e_flags = e_flags self.is_ip6 = is_ip6 + self.rpf_id = rpf_id if is_ip6: self.grp_addr = inet_pton(AF_INET6, grp_addr) @@ -199,6 +210,7 @@ class VppIpMRoute(VppObject): self.e_flags, path.nh_itf, path.nh_i_flags, + rpf_id=self.rpf_id, table_id=self.table_id, is_ipv6=self.is_ip6) self._test.registry.register(self, self._test.logger) @@ -226,6 +238,18 @@ class VppIpMRoute(VppObject): table_id=self.table_id, is_ipv6=self.is_ip6) + def update_rpf_id(self, rpf_id): + self.rpf_id = rpf_id + self._test.vapi.ip_mroute_add_del(self.src_addr, + self.grp_addr, + self.grp_addr_len, + self.e_flags, + 0xffffffff, + 0, + rpf_id=self.rpf_id, + table_id=self.table_id, + is_ipv6=self.is_ip6) + def update_path_flags(self, itf, flags): for path in self.paths: if path.nh_itf == itf: @@ -342,14 +366,17 @@ class VppMplsRoute(VppObject): MPLS Route/LSP """ - def __init__(self, test, local_label, eos_bit, paths, table_id=0): + def __init__(self, test, local_label, eos_bit, paths, table_id=0, + is_multicast=0): self._test = test self.paths = paths self.local_label = local_label self.eos_bit = eos_bit self.table_id = table_id + self.is_multicast = is_multicast def add_vpp_config(self): + is_multipath = len(self.paths) > 1 for path in self.paths: self._test.vapi.mpls_route_add_del( self.local_label, @@ -357,7 +384,11 @@ class VppMplsRoute(VppObject): 1, path.nh_addr, path.nh_itf, + is_multicast=self.is_multicast, + is_multipath=is_multipath, table_id=self.table_id, + is_interface_rx=path.is_interface_rx, + is_rpf_id=path.is_rpf_id, next_hop_out_label_stack=path.nh_labels, next_hop_n_out_labels=len( path.nh_labels), @@ -372,6 +403,7 @@ class VppMplsRoute(VppObject): 1, path.nh_addr, path.nh_itf, + is_rpf_id=path.is_rpf_id, table_id=self.table_id, is_add=0) diff --git a/test/vpp_mpls_tunnel_interface.py b/test/vpp_mpls_tunnel_interface.py new file mode 100644 index 00000000..f2001574 --- /dev/null +++ b/test/vpp_mpls_tunnel_interface.py @@ -0,0 +1,46 @@ + +from vpp_interface import VppInterface +from vpp_ip_route import VppRoutePath +import socket + + +class VppMPLSTunnelInterface(VppInterface): + """ + VPP MPLS Tunnel interface + """ + + def __init__(self, test, paths, is_multicast=0): + """ Create MPLS Tunnel interface """ + self._sw_if_index = 0 + super(VppMPLSTunnelInterface, self).__init__(test) + self._test = test + self.t_paths = paths + self.is_multicast = is_multicast + + def add_vpp_config(self): + self._sw_if_index = 0xffffffff + for path in self.t_paths: + reply = self.test.vapi.mpls_tunnel_add_del( + self._sw_if_index, + 1, # IPv4 next-hop + path.nh_addr, + path.nh_itf, + path.nh_table_id, + path.weight, + next_hop_out_label_stack=path.nh_labels, + next_hop_n_out_labels=len(path.nh_labels), + is_multicast=self.is_multicast) + self._sw_if_index = reply.sw_if_index + + def remove_vpp_config(self): + for path in self.t_paths: + reply = self.test.vapi.mpls_tunnel_add_del( + self.sw_if_index, + 1, # IPv4 next-hop + path.nh_addr, + path.nh_itf, + path.nh_table_id, + path.weight, + next_hop_out_label_stack=path.nh_labels, + next_hop_n_out_labels=len(path.nh_labels), + is_add=0) diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py index e8025dff..ceb684b7 100644 --- a/test/vpp_papi_provider.py +++ b/test/vpp_papi_provider.py @@ -849,6 +849,9 @@ class VppPapiProvider(object): create_vrf_if_needed=0, is_resolve_host=0, is_resolve_attached=0, + is_interface_rx=0, + is_rpf_id=0, + is_multicast=0, is_add=1, is_drop=0, is_multipath=0, @@ -872,6 +875,7 @@ class VppPapiProvider(object): :param is_local: (Default value = 0) :param is_classify: (Default value = 0) :param is_multipath: (Default value = 0) + :param is_multicast: (Default value = 0) :param is_resolve_host: (Default value = 0) :param is_resolve_attached: (Default value = 0) :param not_last: (Default value = 0) @@ -889,8 +893,11 @@ class VppPapiProvider(object): 'mr_is_add': is_add, 'mr_is_classify': is_classify, 'mr_is_multipath': is_multipath, + 'mr_is_multicast': is_multicast, 'mr_is_resolve_host': is_resolve_host, 'mr_is_resolve_attached': is_resolve_attached, + 'mr_is_interface_rx': is_interface_rx, + 'mr_is_rpf_id': is_rpf_id, 'mr_next_hop_proto_is_ip4': next_hop_proto_is_ip4, 'mr_next_hop_weight': next_hop_weight, 'mr_next_hop': next_hop_address, @@ -936,7 +943,8 @@ class VppPapiProvider(object): next_hop_via_label=MPLS_LABEL_INVALID, create_vrf_if_needed=0, is_add=1, - l2_only=0): + l2_only=0, + is_multicast=0): """ :param dst_address_length: @@ -956,8 +964,8 @@ class VppPapiProvider(object): :param is_multipath: (Default value = 0) :param is_resolve_host: (Default value = 0) :param is_resolve_attached: (Default value = 0) - :param not_last: (Default value = 0) :param next_hop_weight: (Default value = 1) + :param is_multicast: (Default value = 0) """ return self.api( @@ -965,6 +973,7 @@ class VppPapiProvider(object): {'mt_sw_if_index': tun_sw_if_index, 'mt_is_add': is_add, 'mt_l2_only': l2_only, + 'mt_is_multicast': is_multicast, 'mt_next_hop_proto_is_ip4': next_hop_proto_is_ip4, 'mt_next_hop_weight': next_hop_weight, 'mt_next_hop': next_hop_address, @@ -1469,6 +1478,7 @@ class VppPapiProvider(object): e_flags, next_hop_sw_if_index, i_flags, + rpf_id=0, table_id=0, create_vrf_if_needed=0, is_add=1, @@ -1481,6 +1491,8 @@ class VppPapiProvider(object): {'next_hop_sw_if_index': next_hop_sw_if_index, 'entry_flags': e_flags, 'itf_flags': i_flags, + 'table_id': table_id, + 'rpf_id': rpf_id, 'create_vrf_if_needed': create_vrf_if_needed, 'is_add': is_add, 'is_ipv6': is_ipv6, -- cgit 1.2.3-korg From 9e3e361eb1d400e79303126f245ae6152ae029cd Mon Sep 17 00:00:00 2001 From: Hongjun Ni Date: Wed, 26 Apr 2017 18:40:55 +0800 Subject: Fix mac check issue for vitual tunnel interface with no mac address Change-Id: I6ce21317fcaa25781199f4329be815f076ab8b09 Signed-off-by: Hongjun Ni --- src/vnet/ethernet/node.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/vnet/ethernet/node.c') diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index 5305012f..13474a43 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -412,9 +412,11 @@ ethernet_input_inline (vlib_main_t * vm, else { if (!ethernet_address_cast (e0->dst_address) && + (hi->hw_address != 0) && !eth_mac_equal ((u8 *) e0, hi->hw_address)) error0 = ETHERNET_ERROR_L3_MAC_MISMATCH; if (!ethernet_address_cast (e1->dst_address) && + (hi->hw_address != 0) && !eth_mac_equal ((u8 *) e1, hi->hw_address)) error1 = ETHERNET_ERROR_L3_MAC_MISMATCH; determine_next_node (em, variant, 0, type0, b0, @@ -628,6 +630,7 @@ ethernet_input_inline (vlib_main_t * vm, else { if (!ethernet_address_cast (e0->dst_address) && + (hi->hw_address != 0) && !eth_mac_equal ((u8 *) e0, hi->hw_address)) error0 = ETHERNET_ERROR_L3_MAC_MISMATCH; determine_next_node (em, variant, 0, type0, b0, -- cgit 1.2.3-korg From 30d0fd4804cf3526eea155cf600f2d3de2629038 Mon Sep 17 00:00:00 2001 From: Neale Ranns Date: Tue, 30 May 2017 07:30:04 -0700 Subject: Packets recieved on VLAN-0 map to the main interface Change-Id: I21b1ad39275495d4d006023b58f630a213445854 Signed-off-by: Neale Ranns --- src/vnet/ethernet/arp.c | 194 ++++++++--------------------------------------- src/vnet/ethernet/node.c | 2 + test/test_ip4.py | 46 +++++++++++ test/test_neighbor.py | 62 ++++++++++++++- 4 files changed, 138 insertions(+), 166 deletions(-) (limited to 'src/vnet/ethernet/node.c') diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c index 624c48c2..619628b3 100644 --- a/src/vnet/ethernet/arp.c +++ b/src/vnet/ethernet/arp.c @@ -830,21 +830,11 @@ unset_random_arp_entry (void) static int arp_unnumbered (vlib_buffer_t * p0, - u32 pi0, - ethernet_header_t * eth0, u32 input_sw_if_index, u32 conn_sw_if_index) { - vlib_main_t *vm = vlib_get_main (); vnet_main_t *vnm = vnet_get_main (); vnet_interface_main_t *vim = &vnm->interface_main; vnet_sw_interface_t *si; - vnet_hw_interface_t *hi; - u32 *buffers = 0; - vlib_buffer_t *b0; - int i; - u8 dst_mac_address[6]; - i16 header_size; - ethernet_arp_header_t *arp0; /* verify that the input interface is unnumbered to the connected. * the connected interface is the interface on which the subnet is @@ -860,92 +850,6 @@ arp_unnumbered (vlib_buffer_t * p0, return 0; } - /* Save the dst mac address */ - clib_memcpy (dst_mac_address, eth0->dst_address, sizeof (dst_mac_address)); - - vec_insert (buffers, 1, 0); - buffers[0] = pi0; - - for (i = 0; i < vec_len (buffers); i++) - { - b0 = vlib_get_buffer (vm, buffers[i]); - arp0 = vlib_buffer_get_current (b0); - - hi = vnet_get_sup_hw_interface (vnm, input_sw_if_index); - si = vnet_get_sw_interface (vnm, input_sw_if_index); - - /* For decoration, most likely */ - vnet_buffer (b0)->sw_if_index[VLIB_TX] = hi->sw_if_index; - - /* Fix ARP pkt src address */ - clib_memcpy (arp0->ip4_over_ethernet[0].ethernet, hi->hw_address, 6); - - /* Build L2 encaps for this swif */ - header_size = sizeof (ethernet_header_t); - if (si->sub.eth.flags.one_tag) - header_size += 4; - else if (si->sub.eth.flags.two_tags) - header_size += 8; - - vlib_buffer_advance (b0, -header_size); - eth0 = vlib_buffer_get_current (b0); - - if (si->sub.eth.flags.one_tag) - { - ethernet_vlan_header_t *outer = (void *) (eth0 + 1); - - eth0->type = si->sub.eth.flags.dot1ad ? - clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD) : - clib_host_to_net_u16 (ETHERNET_TYPE_VLAN); - outer->priority_cfi_and_id = - clib_host_to_net_u16 (si->sub.eth.outer_vlan_id); - outer->type = clib_host_to_net_u16 (ETHERNET_TYPE_ARP); - - } - else if (si->sub.eth.flags.two_tags) - { - ethernet_vlan_header_t *outer = (void *) (eth0 + 1); - ethernet_vlan_header_t *inner = (void *) (outer + 1); - - eth0->type = si->sub.eth.flags.dot1ad ? - clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD) : - clib_host_to_net_u16 (ETHERNET_TYPE_VLAN); - outer->priority_cfi_and_id = - clib_host_to_net_u16 (si->sub.eth.outer_vlan_id); - outer->type = clib_host_to_net_u16 (ETHERNET_TYPE_VLAN); - inner->priority_cfi_and_id = - clib_host_to_net_u16 (si->sub.eth.inner_vlan_id); - inner->type = clib_host_to_net_u16 (ETHERNET_TYPE_ARP); - - } - else - { - eth0->type = clib_host_to_net_u16 (ETHERNET_TYPE_ARP); - } - - /* Restore the original dst address, set src address */ - clib_memcpy (eth0->dst_address, dst_mac_address, - sizeof (eth0->dst_address)); - clib_memcpy (eth0->src_address, hi->hw_address, - sizeof (eth0->src_address)); - - /* Transmit replicas */ - if (i > 0) - { - vlib_frame_t *f = - vlib_get_frame_to_node (vm, hi->output_node_index); - u32 *to_next = vlib_frame_vector_args (f); - to_next[0] = buffers[i]; - f->n_vectors = 1; - vlib_put_frame_to_node (vm, hi->output_node_index, f); - } - } - - /* The regular path outputs the original pkt.. */ - vnet_buffer (p0)->sw_if_index[VLIB_TX] = input_sw_if_index; - - vec_free (buffers); - return !0; } @@ -990,7 +894,7 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) vlib_buffer_t *p0; vnet_hw_interface_t *hw_if0; ethernet_arp_header_t *arp0; - ethernet_header_t *eth0; + ethernet_header_t *eth_rx, *eth_tx; ip4_address_t *if_addr0, proxy_src; u32 pi0, error0, next0, sw_if_index0, conn_sw_if_index0, fib_index0; u8 is_request0, dst_is_local0, is_unnum0, is_vrrp_reply0; @@ -998,6 +902,8 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) fib_node_index_t dst_fei, src_fei; fib_prefix_t pfx0; fib_entry_flag_t src_flags, dst_flags; + ip_adjacency_t *adj0 = NULL; + adj_index_t ai; pi0 = from[0]; to_next[0] = pi0; @@ -1009,6 +915,8 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) p0 = vlib_get_buffer (vm, pi0); arp0 = vlib_buffer_get_current (p0); + /* Fill in ethernet header. */ + eth_rx = ethernet_buffer_get_header (p0); is_request0 = arp0->opcode == clib_host_to_net_u16 (ETHERNET_ARP_OPCODE_request); @@ -1075,6 +983,7 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) error0 = ETHERNET_ARP_ERROR_l3_src_address_not_local; goto drop2; } + if (sw_if_index0 != fib_entry_get_resolving_interface (src_fei)) { /* @@ -1096,9 +1005,6 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) fib_entry_get_prefix (dst_fei, &pfx0); if_addr0 = &pfx0.fp_addr.ip4; - /* Fill in ethernet header. */ - eth0 = ethernet_buffer_get_header (p0); - is_vrrp_reply0 = ((arp0->opcode == clib_host_to_net_u16 (ETHERNET_ARP_OPCODE_reply)) @@ -1110,8 +1016,9 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) /* Trash ARP packets whose ARP-level source addresses do not match their L2-frame-level source addresses, unless it's a reply from a VRRP virtual router */ - if (memcmp (eth0->src_address, arp0->ip4_over_ethernet[0].ethernet, - sizeof (eth0->src_address)) && !is_vrrp_reply0) + if (memcmp + (eth_rx->src_address, arp0->ip4_over_ethernet[0].ethernet, + sizeof (eth_rx->src_address)) && !is_vrrp_reply0) { error0 = ETHERNET_ARP_ERROR_l2_address_mismatch; goto drop2; @@ -1130,6 +1037,20 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) /* Send a reply. */ send_reply: + ai = fib_entry_get_adj (src_fei); + if (ADJ_INDEX_INVALID != ai) + { + adj0 = adj_get (ai); + } + else + { + error0 = ETHERNET_ARP_ERROR_missing_interface_address; + goto drop2; + } + /* Figure out how much to rewind current data from adjacency. */ + vlib_buffer_advance (p0, -adj0->rewrite_header.data_bytes); + eth_tx = vlib_buffer_get_current (p0); + vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0; hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); @@ -1149,68 +1070,19 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) /* Hardware must be ethernet-like. */ ASSERT (vec_len (hw_if0->hw_address) == 6); - clib_memcpy (eth0->dst_address, eth0->src_address, 6); - clib_memcpy (eth0->src_address, hw_if0->hw_address, 6); + /* the rx nd tx ethernet headers wil overlap in the case + * when we received a tagged VLAN=0 packet, but we are sending + * back untagged */ + memmove (eth_tx->dst_address, eth_rx->src_address, 6); + clib_memcpy (eth_tx->src_address, hw_if0->hw_address, 6); - /* Figure out how much to rewind current data from adjacency. */ - /* get the adj from the destination's covering connected */ if (NULL == pa) { if (is_unnum0) { - if (!arp_unnumbered (p0, pi0, eth0, - sw_if_index0, conn_sw_if_index0)) + if (!arp_unnumbered (p0, sw_if_index0, conn_sw_if_index0)) goto drop2; } - else - { - ip_adjacency_t *adj0 = NULL; - adj_index_t ai; - - if (FIB_ENTRY_FLAG_ATTACHED & src_flags) - { - /* - * If the source is attached use the adj from that source. - */ - ai = fib_entry_get_adj (src_fei); - if (ADJ_INDEX_INVALID != ai) - { - adj0 = adj_get (ai); - } - } - else - { - /* - * Get the glean adj from the cover. This is presumably interface - * sourced, and therefre needs to be a glean adj. - */ - ai = fib_entry_get_adj_for_source - (ip4_fib_table_lookup - (ip4_fib_get (fib_index0), - &arp0->ip4_over_ethernet[1].ip4, 31), - FIB_SOURCE_INTERFACE); - - if (ADJ_INDEX_INVALID != ai) - { - adj0 = adj_get (ai); - - if (adj0->lookup_next_index == IP_LOOKUP_NEXT_GLEAN) - { - adj0 = NULL; - } - } - } - if (NULL != adj0) - { - vlib_buffer_advance (p0, - -adj0->rewrite_header.data_bytes); - } - else - { - error0 = ETHERNET_ARP_ERROR_missing_interface_address; - goto drop2; - } - } } /* We are going to reply to this request, so learn the sender */ @@ -1256,20 +1128,14 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) if ((this_addr >= lo_addr && this_addr <= hi_addr) && (fib_index0 == pa->fib_index)) { - eth0 = ethernet_buffer_get_header (p0); proxy_src.as_u32 = arp0->ip4_over_ethernet[1].ip4.data_u32; /* - * Rewind buffer, direct code above not to - * think too hard about it. + * change the interface address to the proxied */ if_addr0 = &proxy_src; is_unnum0 = 0; - i32 ethernet_start = - vnet_buffer (p0)->ethernet.start_of_ethernet_header; - i32 rewind = p0->current_data - ethernet_start; - vlib_buffer_advance (p0, -rewind); n_proxy_arp_replies_sent++; goto send_reply; } diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index 13474a43..8967cebe 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -142,6 +142,8 @@ parse_header (ethernet_input_variant_t variant, tag = clib_net_to_host_u16 (h0->priority_cfi_and_id); *outer_id = tag & 0xfff; + if (0 == *outer_id) + *match_flags &= ~SUBINT_CONFIG_MATCH_1_TAG; *type = clib_net_to_host_u16 (h0->type); diff --git a/test/test_ip4.py b/test/test_ip4.py index ddfd2187..2f666f10 100644 --- a/test/test_ip4.py +++ b/test/test_ip4.py @@ -928,5 +928,51 @@ class TestIPLoadBalance(VppTestCase): [self.pg1, self.pg2, self.pg3, self.pg4]) + +class TestIPVlan0(VppTestCase): + """ IPv4 VLAN-0 """ + + def setUp(self): + super(TestIPVlan0, self).setUp() + + self.create_pg_interfaces(range(2)) + + for i in self.pg_interfaces: + i.admin_up() + i.config_ip4() + i.resolve_arp() + i.enable_mpls() + + def tearDown(self): + super(TestIPVlan0, self).tearDown() + for i in self.pg_interfaces: + i.disable_mpls() + i.unconfig_ip4() + i.admin_down() + + def send_and_expect(self, input, pkts, output): + input.add_stream(pkts) + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + rx = output.get_capture(len(pkts)) + + def test_ip_vlan_0(self): + """ IP VLAN-0 """ + + pkts = (Ether(src=self.pg0.remote_mac, + dst=self.pg0.local_mac) / + Dot1Q(vlan=0) / + IP(dst=self.pg1.remote_ip4, + src=self.pg0.remote_ip4) / + UDP(sport=1234, dport=1234) / + Raw('\xa5' * 100)) * 65 + + # + # Expect that packets sent on VLAN-0 are forwarded on the + # main interface. + # + self.send_and_expect(self.pg0, pkts, self.pg1) + + if __name__ == '__main__': unittest.main(testRunner=VppTestRunner) diff --git a/test/test_neighbor.py b/test/test_neighbor.py index d4f77294..1167b26b 100644 --- a/test/test_neighbor.py +++ b/test/test_neighbor.py @@ -8,7 +8,7 @@ from vpp_neighbor import VppNeighbor, find_nbr from vpp_ip_route import VppIpRoute, VppRoutePath, find_route from scapy.packet import Raw -from scapy.layers.l2 import Ether, ARP +from scapy.layers.l2 import Ether, ARP, Dot1Q from scapy.layers.inet import IP, UDP from scapy.contrib.mpls import MPLS @@ -142,7 +142,7 @@ class ARPTestCase(VppTestCase): # # Generate some hosts on the LAN # - self.pg1.generate_remote_hosts(9) + self.pg1.generate_remote_hosts(10) # # Send IP traffic to one of these unresolved hosts. @@ -286,6 +286,12 @@ class ARPTestCase(VppTestCase): hwsrc=self.pg2.remote_mac, pdst=self.pg1.local_ip4, psrc=self.pg2.remote_hosts[3].ip4)) + pt = (Ether(dst="ff:ff:ff:ff:ff:ff", src=self.pg2.remote_mac) / + Dot1Q(vlan=0) / + ARP(op="who-has", + hwsrc=self.pg2.remote_mac, + pdst=self.pg1.local_ip4, + psrc=self.pg2.remote_hosts[3].ip4)) self.send_and_assert_no_replies(self.pg2, p, "interface not IP enabled") @@ -311,6 +317,17 @@ class ARPTestCase(VppTestCase): self.pg_enable_capture(self.pg_interfaces) self.pg_start() + rx = self.pg2.get_capture(1) + self.verify_arp_resp(rx[0], + self.pg2.local_mac, + self.pg2.remote_mac, + self.pg1.local_ip4, + self.pg2.remote_hosts[3].ip4) + + self.pg2.add_stream(pt) + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + rx = self.pg2.get_capture(1) self.verify_arp_resp(rx[0], self.pg2.local_mac, @@ -455,6 +472,29 @@ class ARPTestCase(VppTestCase): self.pg1.local_ip4, self.pg1.remote_hosts[8].ip4) + # + # Send an ARP request from one of the so-far unlearned remote hosts + # with a VLAN0 tag + # + p = (Ether(dst="ff:ff:ff:ff:ff:ff", + src=self.pg1._remote_hosts[9].mac) / + Dot1Q(vlan=0) / + ARP(op="who-has", + hwsrc=self.pg1._remote_hosts[9].mac, + pdst=self.pg1.local_ip4, + psrc=self.pg1._remote_hosts[9].ip4)) + + self.pg1.add_stream(p) + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + + rx = self.pg1.get_capture(1) + self.verify_arp_resp(rx[0], + self.pg1.local_mac, + self.pg1._remote_hosts[9].mac, + self.pg1.local_ip4, + self.pg1._remote_hosts[9].ip4) + # # ERROR Cases # 1 - don't respond to ARP request for address not within the @@ -563,6 +603,13 @@ class ARPTestCase(VppTestCase): hwsrc=self.pg0.remote_mac, pdst="10.10.10.3", psrc=self.pg0.remote_ip4)) + arp_req_pg0_tagged = (Ether(src=self.pg0.remote_mac, + dst="ff:ff:ff:ff:ff:ff") / + Dot1Q(vlan=0) / + ARP(op="who-has", + hwsrc=self.pg0.remote_mac, + pdst="10.10.10.3", + psrc=self.pg0.remote_ip4)) arp_req_pg1 = (Ether(src=self.pg1.remote_mac, dst="ff:ff:ff:ff:ff:ff") / ARP(op="who-has", @@ -620,6 +667,17 @@ class ARPTestCase(VppTestCase): self.pg_enable_capture(self.pg_interfaces) self.pg_start() + rx = self.pg0.get_capture(1) + self.verify_arp_resp(rx[0], + self.pg0.local_mac, + self.pg0.remote_mac, + "10.10.10.3", + self.pg0.remote_ip4) + + self.pg0.add_stream(arp_req_pg0_tagged) + self.pg_enable_capture(self.pg_interfaces) + self.pg_start() + rx = self.pg0.get_capture(1) self.verify_arp_resp(rx[0], self.pg0.local_mac, -- cgit 1.2.3-korg From 6f7ebf900e6c65b73f8bb8f60ab5190cc2c893bc Mon Sep 17 00:00:00 2001 From: Eyal Bari Date: Tue, 13 Jun 2017 12:09:37 +0300 Subject: ETH:fix l2_len/vlan count mismatch for > 2 tags l2_len was not updated for the third tag as the ethernet node retracts by the vlan count after parse_header (using ethernet_buffer_header_size) it ends up pointing before the ethernet header + some minor cleanups Change-Id: I4ccaedd33928912e5d837376f146503b27071741 Signed-off-by: Eyal Bari --- src/vnet/ethernet/node.c | 5 ++++- src/vnet/l2/l2_input_vtr.c | 15 +++------------ src/vnet/l2/l2_vtr.c | 3 --- 3 files changed, 7 insertions(+), 16 deletions(-) (limited to 'src/vnet/ethernet/node.c') diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index 8967cebe..4529ca6a 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -165,11 +165,12 @@ parse_header (ethernet_input_variant_t variant, vlib_buffer_advance (b0, sizeof (h0[0])); vlan_count = 2; - if (*type == ETHERNET_TYPE_VLAN) { // More than double tagged packet *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_3_TAG; + + vlib_buffer_advance (b0, sizeof (h0[0])); vlan_count = 3; // "unknown" number, aka, 3-or-more } } @@ -239,6 +240,8 @@ determine_next_node (ethernet_main_t * em, // record the L2 len and reset the buffer so the L2 header is preserved u32 eth_start = vnet_buffer (b0)->ethernet.start_of_ethernet_header; vnet_buffer (b0)->l2.l2_len = b0->current_data - eth_start; + ASSERT (vnet_buffer (b0)->l2.l2_len == + ethernet_buffer_header_size (b0)); vlib_buffer_advance (b0, -ethernet_buffer_header_size (b0)); // check for common IP/MPLS ethertypes diff --git a/src/vnet/l2/l2_input_vtr.c b/src/vnet/l2/l2_input_vtr.c index 60a39631..ded23095 100644 --- a/src/vnet/l2/l2_input_vtr.c +++ b/src/vnet/l2/l2_input_vtr.c @@ -188,10 +188,7 @@ l2_invtr_node_fn (vlib_main_t * vm, if (config0->output_vtr.push_and_pop_bytes) { /* perform the tag rewrite on two packets */ - if (l2_vtr_process - (b0, - &(vec_elt_at_index - (l2output_main.configs, sw_if_index0)->input_vtr))) + if (l2_vtr_process (b0, &config0->input_vtr)) { /* Drop packet */ next0 = L2_INVTR_NEXT_DROP; @@ -212,10 +209,7 @@ l2_invtr_node_fn (vlib_main_t * vm, { if (config1->output_vtr.push_and_pop_bytes) { - if (l2_vtr_process - (b1, - &(vec_elt_at_index - (l2output_main.configs, sw_if_index1)->input_vtr))) + if (l2_vtr_process (b1, &config1->input_vtr)) { /* Drop packet */ next1 = L2_INVTR_NEXT_DROP; @@ -305,10 +299,7 @@ l2_invtr_node_fn (vlib_main_t * vm, if (config0->output_vtr.push_and_pop_bytes) { /* perform the tag rewrite on one packet */ - if (l2_vtr_process - (b0, - &(vec_elt_at_index - (l2output_main.configs, sw_if_index0)->input_vtr))) + if (l2_vtr_process (b0, &config0->input_vtr)) { /* Drop packet */ next0 = L2_INVTR_NEXT_DROP; diff --git a/src/vnet/l2/l2_vtr.c b/src/vnet/l2/l2_vtr.c index 3c5365f9..02a68991 100644 --- a/src/vnet/l2/l2_vtr.c +++ b/src/vnet/l2/l2_vtr.c @@ -229,9 +229,6 @@ l2vtr_configure (vlib_main_t * vlib_main, vnet_main_t * vnet_main, u32 sw_if_ind } in_config->pop_bytes = 8; in_config->push_bytes = 0; - - out_config->push_bytes = in_config->pop_bytes; - out_config->pop_bytes = in_config->push_bytes; break; case L2_VTR_PUSH_1: -- cgit 1.2.3-korg From 9a1ae1ac76ca43e799e8281da1177c186e49d4f7 Mon Sep 17 00:00:00 2001 From: Eyal Bari Date: Tue, 13 Jun 2017 08:42:35 +0300 Subject: ETH:optimized double is_tagged check a double version of is_tagged, uses "free lanes" in _mm_cmpeq_epi16 to check a second tag this code was not yet tested for performance Change-Id: I640017e1cc75c85a33d196ee911a7e4a512d9849 Signed-off-by: Eyal Bari --- src/vnet/ethernet/node.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) (limited to 'src/vnet/ethernet/node.c') diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index 4529ca6a..ce5b2c5b 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -285,6 +285,29 @@ determine_next_node (ethernet_main_t * em, } } +static_always_inline int +ethernet_frame_is_any_tagged (u16 type0, u16 type1) +{ +#if __SSE4_2__ + const __m128i ethertype_mask = _mm_set_epi16 (ETHERNET_TYPE_VLAN, + ETHERNET_TYPE_DOT1AD, + ETHERNET_TYPE_VLAN_9100, + ETHERNET_TYPE_VLAN_9200, + /* duplicate for type1 */ + ETHERNET_TYPE_VLAN, + ETHERNET_TYPE_DOT1AD, + ETHERNET_TYPE_VLAN_9100, + ETHERNET_TYPE_VLAN_9200); + + __m128i r = + _mm_set_epi16 (type0, type0, type0, type0, type1, type1, type1, type1); + r = _mm_cmpeq_epi16 (ethertype_mask, r); + return !_mm_test_all_zeros (r, r); +#else + return ethernet_frame_is_tagged (type0) || ethernet_frame_istagged (type1); +#endif +} + static_always_inline uword ethernet_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -377,8 +400,7 @@ ethernet_input_inline (vlib_main_t * vm, /* Speed-path for the untagged case */ if (PREDICT_TRUE (variant == ETHERNET_INPUT_VARIANT_ETHERNET - && !ethernet_frame_is_tagged (type0) - && !ethernet_frame_is_tagged (type1))) + && !ethernet_frame_is_any_tagged (type0, type1))) { main_intf_t *intf0; subint_config_t *subint0; -- cgit 1.2.3-korg From 38cfe98d4e0da61f9d39de8dba9f142ad845821c Mon Sep 17 00:00:00 2001 From: Christophe Fontaine Date: Mon, 10 Jul 2017 15:21:10 +0200 Subject: Fix typo for non x86 platforms. Change-Id: Ic31b388cb972fb5f8a3fc42a5412401c3ee3e487 Signed-off-by: Christophe Fontaine --- src/vnet/ethernet/node.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/vnet/ethernet/node.c') diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index ce5b2c5b..d9fdff48 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -304,7 +304,7 @@ ethernet_frame_is_any_tagged (u16 type0, u16 type1) r = _mm_cmpeq_epi16 (ethertype_mask, r); return !_mm_test_all_zeros (r, r); #else - return ethernet_frame_is_tagged (type0) || ethernet_frame_istagged (type1); + return ethernet_frame_is_tagged (type0) || ethernet_frame_is_tagged (type1); #endif } -- cgit 1.2.3-korg From 072401e8096c648b91f958bd911f64ce24fecff9 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Thu, 13 Jul 2017 18:53:27 +0200 Subject: Introduce l{2,3,4}_hdr_offset fields in the buffer metadata To save space in the first cacheline following is changed: - total_length_not_including_first_buffer moved to the 2nd cacheline. This field is used only when VLIB_BUFFER_TOTAL_LENGTH_VALID and VLIB_BUFFER_NEXT_PRESENT are both set. - free_list_index is now stored in 4bits inside flags, which allows up to 16 free lists. In case we need more we can store index in the 2nd cachelin Change-Id: Ic8521350819391af470d31d3fa1013e67ecb7681 Signed-off-by: Damjan Marion --- src/plugins/dpdk/device/node.c | 8 ++++++- src/vlib/buffer.c | 16 ++++++++----- src/vlib/buffer.h | 40 +++++++++++++++++--------------- src/vlib/buffer_funcs.h | 50 +++++++++++++++++++++++++++++----------- src/vnet/bfd/bfd_udp.c | 4 ++-- src/vnet/buffer.h | 14 +++-------- src/vnet/dhcp/dhcp4_proxy_node.c | 2 +- src/vnet/dhcp/dhcp6_proxy_node.c | 2 +- src/vnet/ethernet/ethernet.h | 3 +-- src/vnet/ethernet/node.c | 23 ++++++++---------- src/vnet/ip/ip4_forward.c | 6 ++--- src/vnet/ip/ip6_forward.c | 6 ++--- src/vnet/ip/ip6_neighbor.c | 19 +++++++-------- src/vnet/l2/l2_bvi.h | 2 +- src/vnet/lisp-cp/control.c | 2 +- src/vnet/replication.c | 6 ++--- 16 files changed, 111 insertions(+), 92 deletions(-) (limited to 'src/vnet/ethernet/node.c') diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index 69acc529..74fb8da1 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -208,7 +208,13 @@ dpdk_process_subseq_segs (vlib_main_t * vm, vlib_buffer_t * b, mb_seg = mb->next; b_chain = b; - while ((mb->nb_segs > 1) && (nb_seg < mb->nb_segs)) + if (mb->nb_segs < 2) + return; + + b->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; + b->total_length_not_including_first_buffer = 0; + + while (nb_seg < mb->nb_segs) { ASSERT (mb_seg != 0); diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index b2a095cf..53b60c16 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -72,8 +72,8 @@ format_vlib_buffer (u8 * s, va_list * args) uword indent = format_get_indent (s); s = format (s, "current data %d, length %d, free-list %d, clone-count %u", - b->current_data, b->current_length, b->free_list_index, - b->n_add_refs); + b->current_data, b->current_length, + vlib_buffer_get_free_list_index (b), b->n_add_refs); if (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID) s = format (s, ", totlen-nifb %d", @@ -163,10 +163,14 @@ vlib_validate_buffer_helper (vlib_main_t * vm, vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_free_list_t *fl; - if (pool_is_free_index (bm->buffer_free_list_pool, b->free_list_index)) - return format (0, "unknown free list 0x%x", b->free_list_index); + if (pool_is_free_index + (bm->buffer_free_list_pool, vlib_buffer_get_free_list_index (b))) + return format (0, "unknown free list 0x%x", + vlib_buffer_get_free_list_index (b)); - fl = pool_elt_at_index (bm->buffer_free_list_pool, b->free_list_index); + fl = + pool_elt_at_index (bm->buffer_free_list_pool, + vlib_buffer_get_free_list_index (b)); if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE) return format (0, "current data %d before pre-data", b->current_data); @@ -388,7 +392,7 @@ vlib_buffer_create_free_list_helper (vlib_main_t * vm, f->name = clib_mem_is_vec (name) ? name : format (0, "%s", name); /* Setup free buffer template. */ - f->buffer_init_template.free_list_index = f->index; + vlib_buffer_set_free_list_index (&f->buffer_init_template, f->index); f->buffer_init_template.n_add_refs = 0; if (is_public) diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h index b20538b7..c810db4e 100644 --- a/src/vlib/buffer.h +++ b/src/vlib/buffer.h @@ -72,6 +72,7 @@ typedef struct the end of this buffer. */ u32 flags; /**< buffer flags: +
VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,
VLIB_BUFFER_IS_TRACED: trace this buffer.
VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says @@ -82,28 +83,26 @@ typedef struct set to avoid adding it to a flow report
VLIB_BUFFER_FLAG_USER(n): user-defined bit N */ -#define VLIB_BUFFER_IS_TRACED (1 << 0) -#define VLIB_BUFFER_LOG2_NEXT_PRESENT (1) + +/* any change to the following line requres update of + * vlib_buffer_get_free_list_index(...) and + * vlib_buffer_set_free_list_index(...) functions */ +#define VLIB_BUFFER_FREE_LIST_INDEX_MASK ((1 << 4) - 1) + +#define VLIB_BUFFER_IS_TRACED (1 << 4) +#define VLIB_BUFFER_LOG2_NEXT_PRESENT (5) #define VLIB_BUFFER_NEXT_PRESENT (1 << VLIB_BUFFER_LOG2_NEXT_PRESENT) -#define VLIB_BUFFER_IS_RECYCLED (1 << 2) -#define VLIB_BUFFER_TOTAL_LENGTH_VALID (1 << 3) -#define VLIB_BUFFER_REPL_FAIL (1 << 4) -#define VLIB_BUFFER_RECYCLE (1 << 5) -#define VLIB_BUFFER_FLOW_REPORT (1 << 6) -#define VLIB_BUFFER_EXT_HDR_VALID (1 << 7) +#define VLIB_BUFFER_IS_RECYCLED (1 << 6) +#define VLIB_BUFFER_TOTAL_LENGTH_VALID (1 << 7) +#define VLIB_BUFFER_REPL_FAIL (1 << 8) +#define VLIB_BUFFER_RECYCLE (1 << 9) +#define VLIB_BUFFER_FLOW_REPORT (1 << 10) +#define VLIB_BUFFER_EXT_HDR_VALID (1 << 11) /* User defined buffer flags. */ #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n)) #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n)) - u32 free_list_index; /**< Buffer free list that this buffer was - allocated from and will be freed to. - */ - - u32 total_length_not_including_first_buffer; - /**< Only valid for first buffer in chain. Current length plus - total length given here give total number of bytes in buffer chain. - */ STRUCT_MARK (template_end); u32 next_buffer; /**< Next buffer for this linked-list of buffers. @@ -128,7 +127,7 @@ typedef struct Before allocating any of it, discussion required! */ - u32 opaque[8]; /**< Opaque data used by sub-graphs for their own purposes. + u32 opaque[10]; /**< Opaque data used by sub-graphs for their own purposes. See .../vnet/vnet/buffer.h */ CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); @@ -137,7 +136,12 @@ typedef struct if VLIB_PACKET_IS_TRACED flag is set. */ u32 recycle_count; /**< Used by L2 path recycle code */ - u32 opaque2[14]; /**< More opaque data, currently unused */ + + u32 total_length_not_including_first_buffer; + /**< Only valid for first buffer in chain. Current length plus + total length given here give total number of bytes in buffer chain. + */ + u32 opaque2[13]; /**< More opaque data, currently unused */ /***** end of second cache line */ CLIB_CACHE_LINE_ALIGN_MARK (cacheline2); diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index 97442e12..1aaac0b2 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -106,12 +106,15 @@ uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, always_inline uword vlib_buffer_length_in_chain (vlib_main_t * vm, vlib_buffer_t * b) { - uword l = b->current_length + b->total_length_not_including_first_buffer; - if (PREDICT_FALSE ((b->flags & (VLIB_BUFFER_NEXT_PRESENT - | VLIB_BUFFER_TOTAL_LENGTH_VALID)) - == VLIB_BUFFER_NEXT_PRESENT)) - return vlib_buffer_length_in_chain_slow_path (vm, b); - return l; + uword len = b->current_length; + + if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0)) + return len; + + if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)) + return len + b->total_length_not_including_first_buffer; + + return vlib_buffer_length_in_chain_slow_path (vm, b); } /** \brief Get length in bytes of the buffer index buffer chain @@ -261,6 +264,24 @@ vlib_buffer_round_size (u32 size) return round_pow2 (size, sizeof (vlib_buffer_t)); } +always_inline u32 +vlib_buffer_get_free_list_index (vlib_buffer_t * b) +{ + return b->flags & VLIB_BUFFER_FREE_LIST_INDEX_MASK; +} + +always_inline void +vlib_buffer_set_free_list_index (vlib_buffer_t * b, u32 index) +{ + /* if there is an need for more free lists we should consider + storig data in the 2nd cacheline */ + ASSERT (VLIB_BUFFER_FREE_LIST_INDEX_MASK & 1); + ASSERT (index <= VLIB_BUFFER_FREE_LIST_INDEX_MASK); + + b->flags &= ~VLIB_BUFFER_FREE_LIST_INDEX_MASK; + b->flags |= index & VLIB_BUFFER_FREE_LIST_INDEX_MASK; +} + /** \brief Allocate buffers from specific freelist into supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @@ -381,7 +402,7 @@ vlib_buffer_get_buffer_free_list (vlib_main_t * vm, vlib_buffer_t * b, vlib_buffer_main_t *bm = vm->buffer_main; u32 i; - *index = i = b->free_list_index; + *index = i = vlib_buffer_get_free_list_index (b); return pool_elt_at_index (bm->buffer_free_list_pool, i); } @@ -569,7 +590,8 @@ vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers, } n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers, - s->free_list_index); + vlib_buffer_get_free_list_index + (s)); if (PREDICT_FALSE (n_buffers == 0)) { buffers[0] = src_buffer; @@ -581,7 +603,8 @@ vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers, vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]); d->current_data = s->current_data; d->current_length = head_end_offset; - d->free_list_index = s->free_list_index; + vlib_buffer_set_free_list_index (d, + vlib_buffer_get_free_list_index (s)); d->total_length_not_including_first_buffer = s->total_length_not_including_first_buffer + s->current_length - head_end_offset; @@ -615,7 +638,8 @@ vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head, vlib_buffer_t * tail) { ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); - ASSERT (head->free_list_index == tail->free_list_index); + ASSERT (vlib_buffer_get_free_list_index (head) == + vlib_buffer_get_free_list_index (tail)); head->flags |= VLIB_BUFFER_NEXT_PRESENT; head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; @@ -791,7 +815,7 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst, CLIB_CACHE_LINE_BYTES * 2); /* Make sure buffer template is sane. */ - ASSERT (fl->index == fl->buffer_init_template.free_list_index); + ASSERT (fl->index == vlib_buffer_get_free_list_index (src)); clib_memcpy (STRUCT_MARK_PTR (dst, template_start), STRUCT_MARK_PTR (src, template_start), @@ -806,7 +830,6 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst, _(current_data); _(current_length); _(flags); - _(free_list_index); #undef _ ASSERT (dst->total_length_not_including_first_buffer == 0); ASSERT (dst->n_add_refs == 0); @@ -832,7 +855,7 @@ vlib_buffer_init_two_for_free_list (vlib_buffer_t * dst0, vlib_buffer_t *src = &fl->buffer_init_template; /* Make sure buffer template is sane. */ - ASSERT (fl->index == fl->buffer_init_template.free_list_index); + ASSERT (fl->index == vlib_buffer_get_free_list_index (src)); clib_memcpy (STRUCT_MARK_PTR (dst0, template_start), STRUCT_MARK_PTR (src, template_start), @@ -853,7 +876,6 @@ vlib_buffer_init_two_for_free_list (vlib_buffer_t * dst0, _(current_data); _(current_length); _(flags); - _(free_list_index); #undef _ ASSERT (dst0->total_length_not_including_first_buffer == 0); diff --git a/src/vnet/bfd/bfd_udp.c b/src/vnet/bfd/bfd_udp.c index 346c5495..06b843c6 100644 --- a/src/vnet/bfd/bfd_udp.c +++ b/src/vnet/bfd/bfd_udp.c @@ -843,7 +843,7 @@ bfd_udp4_find_headers (vlib_buffer_t * b, ip4_header_t ** ip4, udp_header_t ** udp) { /* sanity check first */ - const i32 start = vnet_buffer (b)->ip.start_of_ip_header; + const i32 start = vnet_buffer (b)->l3_hdr_offset; if (start < 0 && start < sizeof (b->pre_data)) { BFD_ERR ("Start of ip header is before pre_data, ignoring"); @@ -1000,7 +1000,7 @@ bfd_udp6_find_headers (vlib_buffer_t * b, ip6_header_t ** ip6, udp_header_t ** udp) { /* sanity check first */ - const i32 start = vnet_buffer (b)->ip.start_of_ip_header; + const i32 start = vnet_buffer (b)->l3_hdr_offset; if (start < 0 && start < sizeof (b->pre_data)) { BFD_ERR ("Start of ip header is before pre_data, ignoring"); diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h index 9aba34da..8647db00 100644 --- a/src/vnet/buffer.h +++ b/src/vnet/buffer.h @@ -71,7 +71,6 @@ #define VNET_BUFFER_SPAN_CLONE (1 << LOG2_VNET_BUFFER_SPAN_CLONE) #define foreach_buffer_opaque_union_subtype \ -_(ethernet) \ _(ip) \ _(swt) \ _(l2) \ @@ -100,16 +99,12 @@ _(tcp) typedef struct { u32 sw_if_index[VLIB_N_RX_TX]; + i16 l2_hdr_offset; + i16 l3_hdr_offset; + i16 l4_hdr_offset; union { - /* Ethernet. */ - struct - { - /* Saved value of current header by ethernet-input. */ - i32 start_of_ethernet_header; - } ethernet; - /* IP4/6 buffer opaque. */ struct { @@ -143,9 +138,6 @@ typedef struct u8 code; u32 data; } icmp; - - /* IP header offset from vlib_buffer.data - saved by ip*_local nodes */ - i32 start_of_ip_header; }; } ip; diff --git a/src/vnet/dhcp/dhcp4_proxy_node.c b/src/vnet/dhcp/dhcp4_proxy_node.c index 26e1e65c..1b59cdea 100644 --- a/src/vnet/dhcp/dhcp4_proxy_node.c +++ b/src/vnet/dhcp/dhcp4_proxy_node.c @@ -231,7 +231,7 @@ dhcp_proxy_to_server_input (vlib_main_t * vm, o = (dhcp_option_t *) (((uword) o) + (o->length + 2)); } - fl = vlib_buffer_get_free_list (vm, b0->free_list_index); + fl = vlib_buffer_get_free_list (vm, vlib_buffer_get_free_list_index (b0)); // start write at (option*)o, some packets have padding if (((u8 *)o - (u8 *)b0->data + VPP_DHCP_OPTION82_SIZE) > fl->n_data_bytes) { diff --git a/src/vnet/dhcp/dhcp6_proxy_node.c b/src/vnet/dhcp/dhcp6_proxy_node.c index 885313a5..e109cc4c 100644 --- a/src/vnet/dhcp/dhcp6_proxy_node.c +++ b/src/vnet/dhcp/dhcp6_proxy_node.c @@ -306,7 +306,7 @@ dhcpv6_proxy_to_server_input (vlib_main_t * vm, copy_ip6_address(&r1->link_addr, ia0); link_address_set: - fl = vlib_buffer_get_free_list (vm, b0->free_list_index); + fl = vlib_buffer_get_free_list (vm, vlib_buffer_get_free_list_index (b0)); if ((b0->current_length+sizeof(*id1)+sizeof(*vss1)+sizeof(*cmac)) > fl->n_data_bytes) diff --git a/src/vnet/ethernet/ethernet.h b/src/vnet/ethernet/ethernet.h index dcc656a7..2fc5b804 100644 --- a/src/vnet/ethernet/ethernet.h +++ b/src/vnet/ethernet/ethernet.h @@ -344,8 +344,7 @@ ethernet_setup_node (vlib_main_t * vm, u32 node_index) always_inline ethernet_header_t * ethernet_buffer_get_header (vlib_buffer_t * b) { - return (void *) - (b->data + vnet_buffer (b)->ethernet.start_of_ethernet_header); + return (void *) (b->data + vnet_buffer (b)->l2_hdr_offset); } /** Returns the number of VLAN headers in the current Ethernet frame in the diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index d9fdff48..421d501a 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -101,7 +101,7 @@ parse_header (ethernet_input_variant_t variant, e0 = (void *) (b0->data + b0->current_data); - vnet_buffer (b0)->ethernet.start_of_ethernet_header = b0->current_data; + vnet_buffer (b0)->l2_hdr_offset = b0->current_data; vlib_buffer_advance (b0, sizeof (e0[0])); @@ -205,9 +205,7 @@ identify_subint (vnet_hw_interface_t * hi, if (!(*is_l2)) { ethernet_header_t *e0; - e0 = - (void *) (b0->data + - vnet_buffer (b0)->ethernet.start_of_ethernet_header); + e0 = (void *) (b0->data + vnet_buffer (b0)->l2_hdr_offset); if (!(ethernet_address_cast (e0->dst_address))) { @@ -238,7 +236,7 @@ determine_next_node (ethernet_main_t * em, { *next0 = em->l2_next; // record the L2 len and reset the buffer so the L2 header is preserved - u32 eth_start = vnet_buffer (b0)->ethernet.start_of_ethernet_header; + u32 eth_start = vnet_buffer (b0)->l2_hdr_offset; vnet_buffer (b0)->l2.l2_len = b0->current_data - eth_start; ASSERT (vnet_buffer (b0)->l2.l2_len == ethernet_buffer_header_size (b0)); @@ -424,10 +422,8 @@ ethernet_input_inline (vlib_main_t * vm, cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2; } - vnet_buffer (b0)->ethernet.start_of_ethernet_header = - b0->current_data; - vnet_buffer (b1)->ethernet.start_of_ethernet_header = - b1->current_data; + vnet_buffer (b0)->l2_hdr_offset = b0->current_data; + vnet_buffer (b1)->l2_hdr_offset = b1->current_data; if (PREDICT_TRUE (is_l20 != 0)) { @@ -519,9 +515,9 @@ ethernet_input_inline (vlib_main_t * vm, { len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data - - vnet_buffer (b0)->ethernet.start_of_ethernet_header; + - vnet_buffer (b0)->l2_hdr_offset; len1 = vlib_buffer_length_in_chain (vm, b1) + b1->current_data - - vnet_buffer (b1)->ethernet.start_of_ethernet_header; + - vnet_buffer (b1)->l2_hdr_offset; stats_n_packets += 2; stats_n_bytes += len0 + len1; @@ -646,8 +642,7 @@ ethernet_input_inline (vlib_main_t * vm, cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2; } - vnet_buffer (b0)->ethernet.start_of_ethernet_header = - b0->current_data; + vnet_buffer (b0)->l2_hdr_offset = b0->current_data; if (PREDICT_TRUE (is_l20 != 0)) { @@ -710,7 +705,7 @@ ethernet_input_inline (vlib_main_t * vm, { len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data - - vnet_buffer (b0)->ethernet.start_of_ethernet_header; + - vnet_buffer (b0)->l2_hdr_offset; stats_n_packets += 1; stats_n_bytes += len0; diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c index 8263e01c..b8dfa847 100755 --- a/src/vnet/ip/ip4_forward.c +++ b/src/vnet/ip/ip4_forward.c @@ -1585,8 +1585,8 @@ ip4_local_inline (vlib_main_t * vm, ip0 = vlib_buffer_get_current (p0); ip1 = vlib_buffer_get_current (p1); - vnet_buffer (p0)->ip.start_of_ip_header = p0->current_data; - vnet_buffer (p1)->ip.start_of_ip_header = p1->current_data; + vnet_buffer (p0)->l3_hdr_offset = p0->current_data; + vnet_buffer (p1)->l3_hdr_offset = p1->current_data; sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX]; sw_if_index1 = vnet_buffer (p1)->sw_if_index[VLIB_RX]; @@ -1788,7 +1788,7 @@ ip4_local_inline (vlib_main_t * vm, ip0 = vlib_buffer_get_current (p0); - vnet_buffer (p0)->ip.start_of_ip_header = p0->current_data; + vnet_buffer (p0)->l3_hdr_offset = p0->current_data; sw_if_index0 = vnet_buffer (p0)->sw_if_index[VLIB_RX]; diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index 4b574b9a..2b8c2bd2 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -1362,8 +1362,8 @@ ip6_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) ip0 = vlib_buffer_get_current (p0); ip1 = vlib_buffer_get_current (p1); - vnet_buffer (p0)->ip.start_of_ip_header = p0->current_data; - vnet_buffer (p1)->ip.start_of_ip_header = p1->current_data; + vnet_buffer (p0)->l3_hdr_offset = p0->current_data; + vnet_buffer (p1)->l3_hdr_offset = p1->current_data; type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol]; type1 = lm->builtin_protocol_by_ip_protocol[ip1->protocol]; @@ -1493,7 +1493,7 @@ ip6_local (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) ip0 = vlib_buffer_get_current (p0); - vnet_buffer (p0)->ip.start_of_ip_header = p0->current_data; + vnet_buffer (p0)->l3_hdr_offset = p0->current_data; type0 = lm->builtin_protocol_by_ip_protocol[ip0->protocol]; next0 = lm->local_next_by_ip_protocol[ip0->protocol]; diff --git a/src/vnet/ip/ip6_neighbor.c b/src/vnet/ip/ip6_neighbor.c index b8f6f9b1..68a8cbbc 100644 --- a/src/vnet/ip/ip6_neighbor.c +++ b/src/vnet/ip/ip6_neighbor.c @@ -1479,9 +1479,8 @@ icmp6_router_solicitation (vlib_main_t * vm, sizeof (icmp6_router_advertisement_header_t); vlib_buffer_add_data (vm, - p0->free_list_index, - bi0, - (void *) &rh, + vlib_buffer_get_free_list_index + (p0), bi0, (void *) &rh, sizeof (icmp6_router_advertisement_header_t)); @@ -1499,9 +1498,8 @@ icmp6_router_solicitation (vlib_main_t * vm, eth_if0->address, 6); vlib_buffer_add_data (vm, - p0->free_list_index, - bi0, - (void *) &h, + vlib_buffer_get_free_list_index + (p0), bi0, (void *) &h, sizeof (icmp6_neighbor_discovery_ethernet_link_layer_address_option_t)); @@ -1525,9 +1523,8 @@ icmp6_router_solicitation (vlib_main_t * vm, sizeof (icmp6_neighbor_discovery_mtu_option_t); vlib_buffer_add_data (vm, - p0->free_list_index, - bi0, - (void *) &h, + vlib_buffer_get_free_list_index + (p0), bi0, (void *) &h, sizeof (icmp6_neighbor_discovery_mtu_option_t)); } @@ -1579,7 +1576,7 @@ icmp6_router_solicitation (vlib_main_t * vm, payload_length += sizeof( icmp6_neighbor_discovery_prefix_information_option_t); vlib_buffer_add_data (vm, - p0->free_list_index, + vlib_buffer_get_free_list_index (p0), bi0, (void *)&h, sizeof(icmp6_neighbor_discovery_prefix_information_option_t)); @@ -2326,7 +2323,7 @@ ip6_neighbor_send_mldpv2_report (u32 sw_if_index) num_addr_records++; vlib_buffer_add_data - (vm, b0->free_list_index, bo0, + (vm, vlib_buffer_get_free_list_index (b0), bo0, (void *)&rr, sizeof(icmp6_multicast_address_record_t)); payload_length += sizeof( icmp6_multicast_address_record_t); diff --git a/src/vnet/l2/l2_bvi.h b/src/vnet/l2/l2_bvi.h index e21a1616..662ec402 100644 --- a/src/vnet/l2/l2_bvi.h +++ b/src/vnet/l2/l2_bvi.h @@ -57,7 +57,7 @@ l2_to_bvi (vlib_main_t * vlib_main, } /* Save L2 header position which may be changed due to packet replication */ - vnet_buffer (b0)->ethernet.start_of_ethernet_header = b0->current_data; + vnet_buffer (b0)->l2_hdr_offset = b0->current_data; /* Strip L2 header */ l2_len = vnet_buffer (b0)->l2.l2_len; diff --git a/src/vnet/lisp-cp/control.c b/src/vnet/lisp-cp/control.c index 22b5c82c..d8a1372d 100644 --- a/src/vnet/lisp-cp/control.c +++ b/src/vnet/lisp-cp/control.c @@ -3706,7 +3706,7 @@ send_map_reply (lisp_cp_main_t * lcm, u32 mi, ip_address_t * dst, static void find_ip_header (vlib_buffer_t * b, u8 ** ip_hdr) { - const i32 start = vnet_buffer (b)->ip.start_of_ip_header; + const i32 start = vnet_buffer (b)->l3_hdr_offset; if (start < 0 && start < -sizeof (b->pre_data)) { *ip_hdr = 0; diff --git a/src/vnet/replication.c b/src/vnet/replication.c index 1c6f28d2..0fdca0bf 100644 --- a/src/vnet/replication.c +++ b/src/vnet/replication.c @@ -43,12 +43,12 @@ replication_prep (vlib_main_t * vm, ctx_id = ctx - rm->contexts[thread_index]; /* Save state from vlib buffer */ - ctx->saved_free_list_index = b0->free_list_index; + ctx->saved_free_list_index = vlib_buffer_get_free_list_index (b0); ctx->current_data = b0->current_data; /* Set up vlib buffer hooks */ b0->recycle_count = ctx_id; - b0->free_list_index = rm->recycle_list_index; + vlib_buffer_set_free_list_index (b0, rm->recycle_list_index); b0->flags |= VLIB_BUFFER_RECYCLE; /* Save feature state */ @@ -129,7 +129,7 @@ replication_recycle (vlib_main_t * vm, vlib_buffer_t * b0, u32 is_last) * This is the last replication in the list. * Restore original buffer free functionality. */ - b0->free_list_index = ctx->saved_free_list_index; + vlib_buffer_set_free_list_index (b0, ctx->saved_free_list_index); b0->flags &= ~VLIB_BUFFER_RECYCLE; /* Free context back to its pool */ -- cgit 1.2.3-korg From 15ac81c16fba83033090299413a3a2dbb848a0f9 Mon Sep 17 00:00:00 2001 From: Pavel Kotucek Date: Tue, 20 Jun 2017 14:00:26 +0200 Subject: P2P Ethernet Change-Id: Idb97e573961b3bc2acdeef77582314590795f8c3 Signed-off-by: Pavel Kotucek --- src/vat/api_format.c | 17 +- src/vnet.am | 1 + src/vnet/api_errno.h | 3 +- src/vnet/devices/devices.c | 6 + src/vnet/ethernet/ethernet.h | 1 + src/vnet/ethernet/interface.c | 54 +++- src/vnet/ethernet/node.c | 17 +- src/vnet/ethernet/p2p_ethernet.api | 1 + src/vnet/ethernet/p2p_ethernet.c | 165 +++++++++- src/vnet/ethernet/p2p_ethernet.h | 42 ++- src/vnet/ethernet/p2p_ethernet_api.c | 17 +- src/vnet/ethernet/p2p_ethernet_input.c | 247 +++++++++++++++ src/vnet/interface.c | 4 + src/vnet/interface.h | 15 + src/vnet/interface_funcs.h | 3 +- test/test_p2p_ethernet.py | 538 +++++++++++++++++++++++++++++++++ test/vpp_papi_provider.py | 25 ++ test/vpp_sub_interface.py | 23 ++ 18 files changed, 1145 insertions(+), 34 deletions(-) create mode 100644 src/vnet/ethernet/p2p_ethernet_input.c create mode 100644 test/test_p2p_ethernet.py (limited to 'src/vnet/ethernet/node.c') diff --git a/src/vat/api_format.c b/src/vat/api_format.c index 5a0c4580..6a2d36de 100644 --- a/src/vat/api_format.c +++ b/src/vat/api_format.c @@ -19161,6 +19161,7 @@ api_p2p_ethernet_add (vat_main_t * vam) unformat_input_t *i = vam->input; vl_api_p2p_ethernet_add_t *mp; u32 parent_if_index = ~0; + u32 sub_id = ~0; u8 remote_mac[6]; u8 mac_set = 0; int ret; @@ -19176,6 +19177,8 @@ api_p2p_ethernet_add (vat_main_t * vam) if (unformat (i, "remote_mac %U", unformat_ethernet_address, remote_mac)) mac_set++; + else if (unformat (i, "sub_id %d", &sub_id)) + ; else { clib_warning ("parse error '%U'", format_unformat_error, i); @@ -19193,9 +19196,15 @@ api_p2p_ethernet_add (vat_main_t * vam) errmsg ("missing remote mac address"); return -99; } + if (sub_id == ~0) + { + errmsg ("missing sub-interface id"); + return -99; + } M (P2P_ETHERNET_ADD, mp); mp->parent_if_index = ntohl (parent_if_index); + mp->subif_id = ntohl (sub_id); clib_memcpy (mp->remote_mac, remote_mac, sizeof (remote_mac)); S (mp); @@ -20094,11 +20103,10 @@ _(l2_xconnect_dump, "") \ _(sw_interface_set_mtu, " | sw_if_index mtu ") \ _(ip_neighbor_dump, "[ip6] | sw_if_index ") \ _(sw_interface_get_table, " | sw_if_index [ipv6]") \ -_(p2p_ethernet_add, " | sw_if_index remote_mac ") \ +_(p2p_ethernet_add, " | sw_if_index remote_mac sub_id ") \ _(p2p_ethernet_del, " | sw_if_index remote_mac ") \ -_(lldp_config, "system-name tx-hold tx-interval ") \ -_(sw_interface_set_lldp, \ - " | sw_if_index [port-desc ] [disable]") +_(lldp_config, "system-name tx-hold tx-interval ") \ +_(sw_interface_set_lldp, " | sw_if_index [port-desc ] [disable]") /* List of command functions, CLI names map directly to functions */ #define foreach_cli_function \ @@ -20122,7 +20130,6 @@ _(search_node_table, "usage: search_node_table ...") \ _(set, "usage: set ") \ _(script, "usage: script ") \ _(unset, "usage: unset ") - #define _(N,n) \ static void vl_api_##n##_t_handler_uni \ (vl_api_##n##_t * mp) \ diff --git a/src/vnet.am b/src/vnet.am index 060e3f38..ad84c028 100644 --- a/src/vnet.am +++ b/src/vnet.am @@ -115,6 +115,7 @@ libvnet_la_SOURCES += \ vnet/ethernet/pg.c \ vnet/ethernet/sfp.c \ vnet/ethernet/p2p_ethernet.c \ + vnet/ethernet/p2p_ethernet_input.c \ vnet/ethernet/p2p_ethernet_api.c nobase_include_HEADERS += \ diff --git a/src/vnet/api_errno.h b/src/vnet/api_errno.h index b22bb3a8..747c65e7 100644 --- a/src/vnet/api_errno.h +++ b/src/vnet/api_errno.h @@ -112,7 +112,8 @@ _(BD_ALREADY_EXISTS, -119, "Bridge domain already exists") \ _(BD_IN_USE, -120, "Bridge domain has member interfaces") \ _(BD_NOT_MODIFIABLE, -121, "Bridge domain 0 can't be deleted/modified") \ _(BD_ID_EXCEED_MAX, -122, "Bridge domain ID exceed 16M limit") \ -_(UNSUPPORTED, -123, "Unsupported") +_(UNSUPPORTED, -123, "Unsupported") \ +_(SUBIF_DOESNT_EXIST, -124, "Subinterface doesn't exist") typedef enum { diff --git a/src/vnet/devices/devices.c b/src/vnet/devices/devices.c index f64c6e0d..2eb8e30e 100644 --- a/src/vnet/devices/devices.c +++ b/src/vnet/devices/devices.c @@ -77,6 +77,12 @@ VNET_FEATURE_INIT (span_input, static) = { .runs_before = VNET_FEATURES ("ethernet-input"), }; +VNET_FEATURE_INIT (p2p_ethernet_node, static) = { + .arc_name = "device-input", + .node_name = "p2p-ethernet-input", + .runs_before = VNET_FEATURES ("ethernet-input"), +}; + VNET_FEATURE_INIT (ethernet_input, static) = { .arc_name = "device-input", .node_name = "ethernet-input", diff --git a/src/vnet/ethernet/ethernet.h b/src/vnet/ethernet/ethernet.h index d9ab8c10..9ca256c9 100644 --- a/src/vnet/ethernet/ethernet.h +++ b/src/vnet/ethernet/ethernet.h @@ -169,6 +169,7 @@ typedef struct #define SUBINT_CONFIG_MATCH_3_TAG (1<<3) #define SUBINT_CONFIG_VALID (1<<4) #define SUBINT_CONFIG_L2 (1<<5) +#define SUBINT_CONFIG_P2P (1<<6) } subint_config_t; diff --git a/src/vnet/ethernet/interface.c b/src/vnet/ethernet/interface.c index 9ac30bc6..3e78a49d 100644 --- a/src/vnet/ethernet/interface.c +++ b/src/vnet/ethernet/interface.c @@ -89,7 +89,10 @@ ethernet_build_rewrite (vnet_main_t * vnm, ethernet_type_t type; uword n_bytes = sizeof (h[0]); u8 *rewrite = NULL; + u8 is_p2p = 0; + if (sub_sw->type == VNET_SW_INTERFACE_TYPE_P2P) + is_p2p = 1; if (sub_sw != sup_sw) { if (sub_sw->sub.eth.flags.one_tag) @@ -100,13 +103,24 @@ ethernet_build_rewrite (vnet_main_t * vnm, { n_bytes += 2 * (sizeof (ethernet_vlan_header_t)); } - // Check for encaps that are not supported for L3 interfaces - if (!(sub_sw->sub.eth.flags.exact_match) || - (sub_sw->sub.eth.flags.default_sub) || - (sub_sw->sub.eth.flags.outer_vlan_id_any) || - (sub_sw->sub.eth.flags.inner_vlan_id_any)) + else if (PREDICT_FALSE (is_p2p)) { - return 0; + n_bytes = sizeof (ethernet_header_t); + } + if (PREDICT_FALSE (!is_p2p)) + { + // Check for encaps that are not supported for L3 interfaces + if (!(sub_sw->sub.eth.flags.exact_match) || + (sub_sw->sub.eth.flags.default_sub) || + (sub_sw->sub.eth.flags.outer_vlan_id_any) || + (sub_sw->sub.eth.flags.inner_vlan_id_any)) + { + return 0; + } + } + else + { + n_bytes = sizeof (ethernet_header_t); } } @@ -126,12 +140,20 @@ ethernet_build_rewrite (vnet_main_t * vnm, h = (ethernet_header_t *) rewrite; ei = pool_elt_at_index (em->interfaces, hw->hw_instance); clib_memcpy (h->src_address, ei->address, sizeof (h->src_address)); - if (dst_address) - clib_memcpy (h->dst_address, dst_address, sizeof (h->dst_address)); + if (is_p2p) + { + clib_memcpy (h->dst_address, sub_sw->p2p.client_mac, + sizeof (h->dst_address)); + } else - memset (h->dst_address, ~0, sizeof (h->dst_address)); /* broadcast */ + { + if (dst_address) + clib_memcpy (h->dst_address, dst_address, sizeof (h->dst_address)); + else + memset (h->dst_address, ~0, sizeof (h->dst_address)); /* broadcast */ + } - if (sub_sw->sub.eth.flags.one_tag) + if (PREDICT_FALSE (!is_p2p) && sub_sw->sub.eth.flags.one_tag) { ethernet_vlan_header_t *outer = (void *) (h + 1); @@ -143,7 +165,7 @@ ethernet_build_rewrite (vnet_main_t * vnm, outer->type = clib_host_to_net_u16 (type); } - else if (sub_sw->sub.eth.flags.two_tags) + else if (PREDICT_FALSE (!is_p2p) && sub_sw->sub.eth.flags.two_tags) { ethernet_vlan_header_t *outer = (void *) (h + 1); ethernet_vlan_header_t *inner = (void *) (outer + 1); @@ -174,7 +196,12 @@ ethernet_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, u32 ai) adj = adj_get (ai); - if (FIB_PROTOCOL_IP4 == adj->ia_nh_proto) + vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index); + if (si->type == VNET_SW_INTERFACE_TYPE_P2P) + { + default_update_adjacency (vnm, sw_if_index, ai); + } + else if (FIB_PROTOCOL_IP4 == adj->ia_nh_proto) { arp_update_adjacency (vnm, sw_if_index, ai); } @@ -719,7 +746,8 @@ vnet_delete_sub_interface (u32 sw_if_index) vnet_interface_main_t *im = &vnm->interface_main; vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index); - if (si->type == VNET_SW_INTERFACE_TYPE_SUB) + if (si->type == VNET_SW_INTERFACE_TYPE_SUB || + si->type == VNET_SW_INTERFACE_TYPE_P2P) { vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index); u64 sup_and_sub_key = diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index 421d501a..f216216d 100755 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -823,7 +824,21 @@ ethernet_sw_interface_get_config (vnet_main_t * vnm, // Locate the subint for the given ethernet config si = vnet_get_sw_interface (vnm, sw_if_index); - if (si->sub.eth.flags.default_sub) + if (si->type == VNET_SW_INTERFACE_TYPE_P2P) + { + p2p_ethernet_main_t *p2pm = &p2p_main; + u32 p2pe_sw_if_index = + p2p_ethernet_lookup (hi->hw_if_index, si->p2p.client_mac); + if (p2pe_sw_if_index == ~0) + { + pool_get (p2pm->p2p_subif_pool, subint); + si->p2p.pool_index = subint - p2pm->p2p_subif_pool; + } + else + subint = vec_elt_at_index (p2pm->p2p_subif_pool, si->p2p.pool_index); + *flags = SUBINT_CONFIG_P2P; + } + else if (si->sub.eth.flags.default_sub) { subint = &main_intf->default_subint; *flags = SUBINT_CONFIG_MATCH_0_TAG | diff --git a/src/vnet/ethernet/p2p_ethernet.api b/src/vnet/ethernet/p2p_ethernet.api index 72a73423..8fb66376 100644 --- a/src/vnet/ethernet/p2p_ethernet.api +++ b/src/vnet/ethernet/p2p_ethernet.api @@ -18,6 +18,7 @@ define p2p_ethernet_add u32 client_index; u32 context; u32 parent_if_index; + u32 subif_id; u8 remote_mac[6]; }; diff --git a/src/vnet/ethernet/p2p_ethernet.c b/src/vnet/ethernet/p2p_ethernet.c index 3c077318..e3f667b5 100644 --- a/src/vnet/ethernet/p2p_ethernet.c +++ b/src/vnet/ethernet/p2p_ethernet.c @@ -18,12 +18,152 @@ #include #include #include +#include + +p2p_ethernet_main_t p2p_main; + +static void +create_p2pe_key (p2p_key_t * p2pe_key, u32 parent_if_index, u8 * client_mac) +{ + clib_memcpy (p2pe_key->mac, client_mac, 6); + p2pe_key->pad1 = 0; + p2pe_key->hw_if_index = parent_if_index; + p2pe_key->pad2 = 0; +} + +u32 +p2p_ethernet_lookup (u32 parent_if_index, u8 * client_mac) +{ + p2p_ethernet_main_t *p2pm = &p2p_main; + p2p_key_t p2pe_key; + uword *p; + + create_p2pe_key (&p2pe_key, parent_if_index, client_mac); + p = hash_get_mem (p2pm->p2p_ethernet_by_key, &p2pe_key); + if (p) + return p[0]; + + return ~0; +} int p2p_ethernet_add_del (vlib_main_t * vm, u32 parent_if_index, - u8 * client_mac, int is_add) + u8 * client_mac, u32 p2pe_subif_id, int is_add, + u32 * p2pe_if_index) { - return 0; + vnet_main_t *vnm = vnet_get_main (); + p2p_ethernet_main_t *p2pm = &p2p_main; + vnet_interface_main_t *im = &vnm->interface_main; + + u32 p2pe_sw_if_index = ~0; + p2pe_sw_if_index = p2p_ethernet_lookup (parent_if_index, client_mac); + + if (p2pe_if_index) + *p2pe_if_index = ~0; + + if (is_add) + { + if (p2pe_sw_if_index == ~0) + { + vnet_hw_interface_t *hi; + + hi = vnet_get_hw_interface (vnm, parent_if_index); + if (hi->bond_info == VNET_HW_INTERFACE_BOND_INFO_SLAVE) + return VNET_API_ERROR_BOND_SLAVE_NOT_ALLOWED; + + u64 sup_and_sub_key = + ((u64) (hi->sw_if_index) << 32) | (u64) p2pe_subif_id; + uword *p; + p = hash_get_mem (im->sw_if_index_by_sup_and_sub, &sup_and_sub_key); + if (p) + { + if (CLIB_DEBUG > 0) + clib_warning + ("p2p ethernet sub-interface on sw_if_index %d with sub id %d already exists\n", + hi->sw_if_index, p2pe_subif_id); + return VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + } + vnet_sw_interface_t template = { + .type = VNET_SW_INTERFACE_TYPE_P2P, + .flood_class = VNET_FLOOD_CLASS_NORMAL, + .sup_sw_if_index = hi->sw_if_index, + .sub.id = p2pe_subif_id + }; + + clib_memcpy (template.p2p.client_mac, client_mac, + sizeof (template.p2p.client_mac)); + + if (vnet_create_sw_interface (vnm, &template, &p2pe_sw_if_index)) + return VNET_API_ERROR_SUBIF_CREATE_FAILED; + + vnet_interface_main_t *im = &vnm->interface_main; + sup_and_sub_key = + ((u64) (hi->sw_if_index) << 32) | (u64) p2pe_subif_id; + u64 *kp = clib_mem_alloc (sizeof (*kp)); + + *kp = sup_and_sub_key; + hash_set (hi->sub_interface_sw_if_index_by_id, p2pe_subif_id, + p2pe_sw_if_index); + hash_set_mem (im->sw_if_index_by_sup_and_sub, kp, p2pe_sw_if_index); + + p2p_key_t *p_p2pe_key; + p_p2pe_key = clib_mem_alloc (sizeof (*p_p2pe_key)); + create_p2pe_key (p_p2pe_key, parent_if_index, client_mac); + hash_set_mem (p2pm->p2p_ethernet_by_key, p_p2pe_key, + p2pe_sw_if_index); + + if (p2pe_if_index) + *p2pe_if_index = p2pe_sw_if_index; + + vec_validate (p2pm->p2p_ethernet_by_sw_if_index, parent_if_index); + if (p2pm->p2p_ethernet_by_sw_if_index[parent_if_index] == 0) + { + vnet_feature_enable_disable ("device-input", + "p2p-ethernet-input", + parent_if_index, 1, 0, 0); + /* Set promiscuous mode on the l2 interface */ + ethernet_set_flags (vnm, parent_if_index, + ETHERNET_INTERFACE_FLAG_ACCEPT_ALL); + + } + p2pm->p2p_ethernet_by_sw_if_index[parent_if_index]++; + /* set the interface mode */ + set_int_l2_mode (vm, vnm, MODE_L3, p2pe_subif_id, 0, 0, 0, 0); + return 0; + } + return VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + } + else + { + if (p2pe_sw_if_index == ~0) + return VNET_API_ERROR_SUBIF_DOESNT_EXIST; + else + { + int rv = 0; + rv = vnet_delete_sub_interface (p2pe_sw_if_index); + if (!rv) + { + vec_validate (p2pm->p2p_ethernet_by_sw_if_index, + parent_if_index); + if (p2pm->p2p_ethernet_by_sw_if_index[parent_if_index] == 1) + { + vnet_feature_enable_disable ("device-input", + "p2p-ethernet-input", + parent_if_index, 0, 0, 0); + /* Disable promiscuous mode on the l2 interface */ + ethernet_set_flags (vnm, parent_if_index, 0); + } + p2pm->p2p_ethernet_by_sw_if_index[parent_if_index]--; + + /* Remove p2p_ethernet from hash map */ + p2p_key_t *p_p2pe_key; + p_p2pe_key = clib_mem_alloc (sizeof (*p_p2pe_key)); + create_p2pe_key (p_p2pe_key, parent_if_index, client_mac); + hash_unset_mem (p2pm->p2p_ethernet_by_key, p_p2pe_key); + } + return rv; + } + } } static clib_error_t * @@ -35,6 +175,7 @@ vnet_p2p_ethernet_add_del (vlib_main_t * vm, unformat_input_t * input, int is_add = 1; int remote_mac = 0; u32 hw_if_index = ~0; + u32 sub_id = ~0; u8 client_mac[6]; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) @@ -44,6 +185,8 @@ vnet_p2p_ethernet_add_del (vlib_main_t * vm, unformat_input_t * input, ; else if (unformat (input, "%U", unformat_ethernet_address, &client_mac)) remote_mac = 1; + else if (unformat (input, "sub-id %d", &sub_id)) + ; else if (unformat (input, "del")) is_add = 0; else @@ -54,9 +197,11 @@ vnet_p2p_ethernet_add_del (vlib_main_t * vm, unformat_input_t * input, return clib_error_return (0, "Please specify parent interface ..."); if (!remote_mac) return clib_error_return (0, "Please specify client MAC address ..."); + if (sub_id == ~0 && is_add) + return clib_error_return (0, "Please specify sub-interface id ..."); u32 rv; - rv = p2p_ethernet_add_del (vm, hw_if_index, client_mac, is_add); + rv = p2p_ethernet_add_del (vm, hw_if_index, client_mac, sub_id, is_add, 0); switch (rv) { case VNET_API_ERROR_BOND_SLAVE_NOT_ALLOWED: @@ -77,17 +222,21 @@ vnet_p2p_ethernet_add_del (vlib_main_t * vm, unformat_input_t * input, return 0; } -/* *INDENT-OFF* */ VLIB_CLI_COMMAND (p2p_ethernet_add_del_command, static) = { - .path = "p2p_ethernet ", - .function = vnet_p2p_ethernet_add_del, - .short_help = "p2p_ethernet [del]",}; -/* *INDENT-ON* */ +.path = "p2p_ethernet ",.function = vnet_p2p_ethernet_add_del,.short_help = + "p2p_ethernet [sub-id | del]",}; static clib_error_t * p2p_ethernet_init (vlib_main_t * vm) { + p2p_ethernet_main_t *p2pm = &p2p_main; + + p2pm->vlib_main = vm; + p2pm->vnet_main = vnet_get_main (); + p2pm->p2p_ethernet_by_key = + hash_create_mem (0, sizeof (p2p_key_t), sizeof (uword)); + return 0; } diff --git a/src/vnet/ethernet/p2p_ethernet.h b/src/vnet/ethernet/p2p_ethernet.h index 31b93d82..bb1e2896 100644 --- a/src/vnet/ethernet/p2p_ethernet.h +++ b/src/vnet/ethernet/p2p_ethernet.h @@ -18,6 +18,46 @@ #include #include -int p2p_ethernet_add_del (vlib_main_t * vm, u32 parent_if_index, u8 * client_mac, int is_add); + +typedef struct { + /** + * Hash mapping parent sw_if_index and client mac address to p2p_ethernet sub-interface + */ + uword * p2p_ethernet_by_key; + + u32 *p2p_ethernet_by_sw_if_index; + + // Pool of p2p subifs; + subint_config_t *p2p_subif_pool; + + /* convenience */ + vlib_main_t * vlib_main; + vnet_main_t * vnet_main; +} p2p_ethernet_main_t; + +extern p2p_ethernet_main_t p2p_main; + +typedef struct +{ + u32 sw_if_index; + u32 p2pe_sw_if_index; + u8 client_mac[6]; +} p2p_ethernet_trace_t; + +/** + * @brief Key struct for P2P Ethernet + * Key fields: parent sw_if_index and client mac address + * all fields in NET byte order + */ + +typedef struct { + u8 mac[6]; + u16 pad1; // padding for u64 mac address + u32 hw_if_index; + u32 pad2; // padding for u64 +} p2p_key_t; + +u32 p2p_ethernet_lookup (u32 parent_sw_if_index, u8* client_mac); +int p2p_ethernet_add_del (vlib_main_t * vm, u32 parent_if_index, u8 * client_mac, u32 sub_id, int is_add, u32 *p2pe_if_index); #endif /* included_vnet_p2p_ethernet_h */ diff --git a/src/vnet/ethernet/p2p_ethernet_api.c b/src/vnet/ethernet/p2p_ethernet_api.c index 1d9eaeb0..f2c730b4 100644 --- a/src/vnet/ethernet/p2p_ethernet_api.c +++ b/src/vnet/ethernet/p2p_ethernet_api.c @@ -51,12 +51,21 @@ vl_api_p2p_ethernet_add_t_handler (vl_api_p2p_ethernet_add_t * mp) int rv; u32 parent_if_index = htonl (mp->parent_if_index); + u32 sub_id = htonl (mp->subif_id); + u32 p2pe_if_index; u8 remote_mac[6]; clib_memcpy (remote_mac, mp->remote_mac, 6); - rv = p2p_ethernet_add_del (vm, parent_if_index, remote_mac, 1); - - REPLY_MACRO (VL_API_P2P_ETHERNET_ADD_REPLY); + rv = + p2p_ethernet_add_del (vm, parent_if_index, remote_mac, sub_id, 1, + &p2pe_if_index); + + /* *INDENT-OFF* */ + REPLY_MACRO2(VL_API_P2P_ETHERNET_ADD_REPLY, + ({ + rmp->sw_if_index = htonl(p2pe_if_index); + })); + /* *INDENT-ON* */ } void @@ -70,7 +79,7 @@ vl_api_p2p_ethernet_del_t_handler (vl_api_p2p_ethernet_del_t * mp) u8 remote_mac[6]; clib_memcpy (remote_mac, mp->remote_mac, 6); - rv = p2p_ethernet_add_del (vm, parent_if_index, remote_mac, 0); + rv = p2p_ethernet_add_del (vm, parent_if_index, remote_mac, ~0, 0, 0); REPLY_MACRO (VL_API_P2P_ETHERNET_DEL_REPLY); } diff --git a/src/vnet/ethernet/p2p_ethernet_input.c b/src/vnet/ethernet/p2p_ethernet_input.c new file mode 100644 index 00000000..a58b832a --- /dev/null +++ b/src/vnet/ethernet/p2p_ethernet_input.c @@ -0,0 +1,247 @@ +/* + * node.c: p2p ethernet vpp node + * + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include +#include + +vlib_node_registration_t p2p_ethernet_input_node; + +/* packet trace format function */ +u8 * +format_p2p_ethernet_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + p2p_ethernet_trace_t *t = va_arg (*args, p2p_ethernet_trace_t *); + + vnet_main_t *vnm = &vnet_main; + s = format (s, "P2P ethernet: %U -> %U", + format_vnet_sw_if_index_name, vnm, t->sw_if_index, + format_vnet_sw_if_index_name, vnm, t->p2pe_sw_if_index); + + return s; +} + +#define foreach_p2p_ethernet_error \ +_(HITS, "P2P ethernet incoming packets processed") + +typedef enum +{ +#define _(sym,str) P2PE_ERROR_##sym, + foreach_p2p_ethernet_error +#undef _ + P2PE_N_ERROR, +} p2p_ethernet_error_t; + +static char *p2p_ethernet_error_strings[] = { +#define _(sym,string) string, + foreach_p2p_ethernet_error +#undef _ +}; + +static uword +p2p_ethernet_input_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + u32 n_trace = vlib_get_trace_count (vm, node); + u32 n_left_from, *from, *to_next; + u32 next_index; + u32 n_p2p_ethernet_packets = 0; + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + u32 bi0, bi1; + vlib_buffer_t *b0, *b1; + u32 next0 = 0, next1 = 0; + u32 sw_if_index0, sw_if_index1; + ethernet_header_t *en0, *en1; + u32 rx0, rx1; + + bi0 = from[0]; + bi1 = from[1]; + to_next[0] = bi0; + to_next[1] = bi1; + from += 2; + to_next += 2; + n_left_to_next -= 2; + n_left_from -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + en0 = vlib_buffer_get_current (b0); + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + en1 = vlib_buffer_get_current (b1); + sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX]; + + vnet_feature_next (sw_if_index0, &next0, b0); + vnet_feature_next (sw_if_index1, &next1, b1); + + rx0 = p2p_ethernet_lookup (sw_if_index0, en0->src_address); + rx1 = p2p_ethernet_lookup (sw_if_index1, en1->src_address); + + if (rx0 != ~0) + { + /* Send pkt to p2p_ethernet RX interface */ + vnet_buffer (b0)->sw_if_index[VLIB_RX] = rx0; + n_p2p_ethernet_packets += 1; + + if (PREDICT_FALSE (n_trace > 0)) + { + p2p_ethernet_trace_t *t0; + vlib_trace_buffer (vm, node, next_index, b0, + 1 /* follow_chain */ ); + vlib_set_trace_count (vm, node, --n_trace); + t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); + t0->sw_if_index = sw_if_index0; + t0->p2pe_sw_if_index = rx0; + } + } + if (rx1 != ~0) + { + /* Send pkt to p2p_ethernet RX interface */ + vnet_buffer (b1)->sw_if_index[VLIB_RX] = rx1; + n_p2p_ethernet_packets += 1; + + if (PREDICT_FALSE (n_trace > 0)) + { + p2p_ethernet_trace_t *t1; + vlib_trace_buffer (vm, node, next_index, b1, + 1 /* follow_chain */ ); + vlib_set_trace_count (vm, node, --n_trace); + t1 = vlib_add_trace (vm, node, b1, sizeof (*t1)); + t1->sw_if_index = sw_if_index1; + t1->p2pe_sw_if_index = rx1; + } + } + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi1, next1); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 next0 = 0; + u32 sw_if_index0; + ethernet_header_t *en0; + u32 rx0; + + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + en0 = vlib_buffer_get_current (b0); + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + + vnet_feature_next (sw_if_index0, &next0, b0); + + rx0 = p2p_ethernet_lookup (sw_if_index0, en0->src_address); + if (rx0 != ~0) + { + /* Send pkt to p2p_ethernet RX interface */ + vnet_buffer (b0)->sw_if_index[VLIB_RX] = rx0; + n_p2p_ethernet_packets += 1; + + if (PREDICT_FALSE (n_trace > 0)) + { + p2p_ethernet_trace_t *t0; + vlib_trace_buffer (vm, node, next_index, b0, + 1 /* follow_chain */ ); + vlib_set_trace_count (vm, node, --n_trace); + t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); + t0->sw_if_index = sw_if_index0; + t0->p2pe_sw_if_index = rx0; + } + } + else + { + if (PREDICT_FALSE (n_trace > 0)) + { + node->flags |= VLIB_NODE_FLAG_TRACE; + } + } + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + + vlib_node_increment_counter (vm, p2p_ethernet_input_node.index, + P2PE_ERROR_HITS, n_p2p_ethernet_packets); + return frame->n_vectors; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (p2p_ethernet_input_node) = { + .function = p2p_ethernet_input_node_fn, + .name = "p2p-ethernet-input", + .vector_size = sizeof (u32), + .format_trace = format_p2p_ethernet_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(p2p_ethernet_error_strings), + .error_strings = p2p_ethernet_error_strings, + + .n_next_nodes = 1, + + /* edit / add dispositions here */ + .next_nodes = { + [0] = "error-drop", + }, +}; +/* *INDENT-ON* */ + +VLIB_NODE_FUNCTION_MULTIARCH (p2p_ethernet_input_node, + p2p_ethernet_input_node_fn) +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vnet/interface.c b/src/vnet/interface.c index dad1f315..721259a7 100644 --- a/src/vnet/interface.c +++ b/src/vnet/interface.c @@ -1151,6 +1151,10 @@ vnet_hw_interface_compare (vnet_main_t * vnm, int vnet_sw_interface_is_p2p (vnet_main_t * vnm, u32 sw_if_index) { + vnet_sw_interface_t *si = vnet_get_sw_interface (vnm, sw_if_index); + if (si->type == VNET_SW_INTERFACE_TYPE_P2P) + return 1; + vnet_hw_interface_t *hw = vnet_get_sup_hw_interface (vnm, sw_if_index); vnet_hw_interface_class_t *hc = vnet_get_hw_interface_class (vnm, hw->hw_class_index); diff --git a/src/vnet/interface.h b/src/vnet/interface.h index fb75ff34..5ca489db 100644 --- a/src/vnet/interface.h +++ b/src/vnet/interface.h @@ -505,6 +505,7 @@ typedef enum /* A sub-interface. */ VNET_SW_INTERFACE_TYPE_SUB, + VNET_SW_INTERFACE_TYPE_P2P, } vnet_sw_interface_type_t; typedef struct @@ -538,6 +539,17 @@ typedef struct } eth; } vnet_sub_interface_t; +typedef struct +{ + /* + * Subinterface ID. A number 0-N to uniquely identify + * this subinterface under the main interface + */ + u32 id; + u32 pool_index; + u8 client_mac[6]; +} vnet_p2p_sub_interface_t; + typedef enum { /* Always flood */ @@ -594,6 +606,9 @@ typedef struct /* VNET_SW_INTERFACE_TYPE_SUB. */ vnet_sub_interface_t sub; + + /* VNET_SW_INTERFACE_TYPE_P2P. */ + vnet_p2p_sub_interface_t p2p; }; vnet_flood_class_t flood_class; diff --git a/src/vnet/interface_funcs.h b/src/vnet/interface_funcs.h index 26eef9b9..142bef57 100644 --- a/src/vnet/interface_funcs.h +++ b/src/vnet/interface_funcs.h @@ -73,7 +73,8 @@ always_inline vnet_sw_interface_t * vnet_get_sup_sw_interface (vnet_main_t * vnm, u32 sw_if_index) { vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, sw_if_index); - if (sw->type == VNET_SW_INTERFACE_TYPE_SUB) + if (sw->type == VNET_SW_INTERFACE_TYPE_SUB || + sw->type == VNET_SW_INTERFACE_TYPE_P2P) sw = vnet_get_sw_interface (vnm, sw->sup_sw_if_index); return sw; } diff --git a/test/test_p2p_ethernet.py b/test/test_p2p_ethernet.py new file mode 100644 index 00000000..37a1d18b --- /dev/null +++ b/test/test_p2p_ethernet.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python +import random +import unittest +import datetime +import re + +from scapy.packet import Raw +from scapy.layers.l2 import Ether +from scapy.layers.inet import IP, UDP +from scapy.layers.inet6 import IPv6 + +from framework import VppTestCase, VppTestRunner, running_extended_tests +from vpp_sub_interface import VppP2PSubint +from vpp_ip_route import VppIpRoute, VppRoutePath +from util import mactobinary + + +class P2PEthernetAPI(VppTestCase): + """P2P Ethernet tests""" + + p2p_sub_ifs = [] + + @classmethod + def setUpClass(cls): + super(P2PEthernetAPI, cls).setUpClass() + + # Create pg interfaces + cls.create_pg_interfaces(range(4)) + + # Set up all interfaces + for i in cls.pg_interfaces: + i.admin_up() + + def create_p2p_ethernet(self, parent_if, sub_id, remote_mac): + p2p = VppP2PSubint(self, parent_if, sub_id, mactobinary(remote_mac)) + self.p2p_sub_ifs.append(p2p) + + def delete_p2p_ethernet(self, parent_if, remote_mac): + self.vapi.delete_p2pethernet_subif(parent_if.sw_if_index, + mactobinary(remote_mac)) + + def test_api(self): + """delete/create p2p subif""" + self.logger.info("FFP_TEST_START_0000") + + self.create_p2p_ethernet(self.pg0, 1, "de:ad:00:00:00:01") + self.create_p2p_ethernet(self.pg0, 2, "de:ad:00:00:00:02") + intfs = self.vapi.cli("show interface") + + self.assertNotEqual(intfs.find('pg0.1'), -1) + self.assertNotEqual(intfs.find('pg0.2'), -1) + self.assertEqual(intfs.find('pg0.5'), -1) + + # create pg2.5 subif + self.create_p2p_ethernet(self.pg0, 5, "de:ad:00:00:00:ff") + intfs = self.vapi.cli("show interface") + self.assertNotEqual(intfs.find('pg0.5'), -1) + # delete pg2.5 subif + self.delete_p2p_ethernet(self.pg0, "de:ad:00:00:00:ff") + + intfs = self.vapi.cli("show interface") + + self.assertNotEqual(intfs.find('pg0.1'), -1) + self.assertNotEqual(intfs.find('pg0.2'), -1) + self.assertEqual(intfs.find('pg0.5'), -1) + + self.logger.info("FFP_TEST_FINISH_0000") + + def test_p2p_subif_creation_1k(self): + """create 1k of p2p subifs""" + self.logger.info("FFP_TEST_START_0001") + + macs = [] + clients = 1000 + mac = int("dead00000000", 16) + + for i in range(1, clients+1): + try: + macs.append(':'.join(re.findall('..', '{:02x}'.format(mac+i)))) + self.vapi.create_p2pethernet_subif(self.pg2.sw_if_index, + mactobinary(macs[i-1]), + i) + except Exception: + print "Failed to create subif %d %s" % (i, macs[i-1]) + raise + + intfs = self.vapi.cli("show interface").split("\n") + count = 0 + for intf in intfs: + if intf.startswith('pg2.'): + count += 1 + self.assertEqual(count, clients) + + self.logger.info("FFP_TEST_FINISH_0001") + + @unittest.skipUnless(running_extended_tests(), "part of extended tests") + def test_p2p_subif_creation_10k(self): + """create 100k of p2p subifs""" + self.logger.info("FFP_TEST_START_0001") + + macs = [] + clients = 100000 + mac = int("dead00000000", 16) + + s_time = datetime.datetime.now() + for i in range(1, clients+1): + if i % 1000 == 0: + e_time = datetime.datetime.now() + print "Created 1000 subifs in %s secs" % (e_time - s_time) + s_time = e_time + try: + macs.append(':'.join(re.findall('..', '{:02x}'.format(mac+i)))) + self.vapi.create_p2pethernet_subif(self.pg3.sw_if_index, + mactobinary(macs[i-1]), + i) + except Exception: + print "Failed to create subif %d %s" % (i, macs[i-1]) + raise + + intfs = self.vapi.cli("show interface").split("\n") + count = 0 + for intf in intfs: + if intf.startswith('pg3.'): + count += 1 + self.assertEqual(count, clients) + + self.logger.info("FFP_TEST_FINISH_0001") + + +class P2PEthernetIPV6(VppTestCase): + """P2P Ethernet IPv6 tests""" + + p2p_sub_ifs = [] + packets = [] + + @classmethod + def setUpClass(cls): + super(P2PEthernetIPV6, cls).setUpClass() + + # Create pg interfaces + cls.create_pg_interfaces(range(3)) + + # Packet sizes + cls.pg_if_packet_sizes = [64, 512, 1518, 9018] + + # Set up all interfaces + for i in cls.pg_interfaces: + i.admin_up() + + cls.pg0.generate_remote_hosts(3) + cls.pg0.configure_ipv6_neighbors() + + cls.pg1.config_ip6() + cls.pg1.generate_remote_hosts(3) + cls.pg1.configure_ipv6_neighbors() + cls.pg1.disable_ipv6_ra() + + def setUp(self): + super(P2PEthernetIPV6, self).setUp() + for p in self.packets: + self.packets.remove(p) + self.create_p2p_ethernet(self.pg0, 1, self.pg0._remote_hosts[0].mac) + self.create_p2p_ethernet(self.pg0, 2, self.pg0._remote_hosts[1].mac) + self.p2p_sub_ifs[0].config_ip6() + self.p2p_sub_ifs[1].config_ip6() + self.vapi.cli("trace add p2p-ethernet-input 50") + + def tearDown(self): + self.delete_p2p_ethernet(self.pg0, self.pg0._remote_hosts[0].mac) + self.delete_p2p_ethernet(self.pg0, self.pg0._remote_hosts[1].mac) + super(P2PEthernetIPV6, self).tearDown() + + def create_p2p_ethernet(self, parent_if, sub_id, remote_mac): + p2p = VppP2PSubint(self, parent_if, sub_id, mactobinary(remote_mac)) + p2p.admin_up() + p2p.config_ip6() + p2p.disable_ipv6_ra() + self.p2p_sub_ifs.append(p2p) + + def delete_p2p_ethernet(self, parent_if, remote_mac): + self.vapi.delete_p2pethernet_subif(parent_if.sw_if_index, + mactobinary(remote_mac)) + + def create_stream(self, src_mac=None, dst_mac=None, + src_ip=None, dst_ip=None, size=None): + pkt_size = size + if size is None: + pkt_size = random.choice(self.pg_if_packet_sizes) + p = Ether(src=src_mac, dst=dst_mac) + p /= IPv6(src=src_ip, dst=dst_ip) + p /= (UDP(sport=1234, dport=4321) / Raw('\xa5' * 20)) + self.extend_packet(p, pkt_size) + return p + + def send_packets(self, src_if=None, dst_if=None, packets=None, count=None): + self.pg_enable_capture([dst_if]) + if packets is None: + packets = self.packets + src_if.add_stream(packets) + self.pg_start() + if count is None: + count = len(packets) + return dst_if.get_capture(count) + + def verify_counters(self, counter_id, expected_value): + counters = self.vapi.cli("sh errors").split('\n') + counter_value = -1 + for i in range(1, len(counters)-1): + results = counters[i].split() + if results[1] == counter_id: + counter_value = int(results[0]) + break + self.assertEqual(counter_value, expected_value) + + def test_no_p2p_subif(self): + """standard routing without p2p subinterfaces""" + self.logger.info("FFP_TEST_START_0001") + + route_8000 = VppIpRoute(self, "8000::", 64, + [VppRoutePath(self.pg0.remote_ip6, + self.pg0.sw_if_index, + is_ip6=1)], + is_ip6=1) + route_8000.add_vpp_config() + + self.packets = [(Ether(dst=self.pg1.local_mac, + src=self.pg1.remote_mac) / + IPv6(src="3001::1", dst="8000::100") / + UDP(sport=1234, dport=1234) / + Raw('\xa5' * 100))] + self.send_packets(self.pg1, self.pg0) + + self.logger.info("FFP_TEST_FINISH_0001") + + def test_ip6_rx_p2p_subif(self): + """receive ipv6 packet via p2p subinterface""" + self.logger.info("FFP_TEST_START_0002") + + route_9001 = VppIpRoute(self, "9001::", 64, + [VppRoutePath(self.pg1.remote_ip6, + self.pg1.sw_if_index, + is_ip6=1)], + is_ip6=1) + route_9001.add_vpp_config() + + self.packets.append( + self.create_stream(src_mac=self.pg0._remote_hosts[0].mac, + dst_mac=self.pg0.local_mac, + src_ip=self.p2p_sub_ifs[0].remote_ip6, + dst_ip="9001::100")) + + self.send_packets(self.pg0, self.pg1, self.packets) + self.verify_counters('p2p-ethernet-input', 1) + + route_9001.remove_vpp_config() + self.logger.info("FFP_TEST_FINISH_0002") + + def test_ip6_rx_p2p_subif_route(self): + """route rx ip6 packet not matching p2p subinterface""" + self.logger.info("FFP_TEST_START_0003") + + self.pg0.config_ip6() + + route_3 = VppIpRoute(self, "9000::", 64, + [VppRoutePath(self.pg1._remote_hosts[0].ip6, + self.pg1.sw_if_index, + is_ip6=1)], + is_ip6=1) + route_3.add_vpp_config() + + self.packets.append( + self.create_stream(src_mac="02:03:00:00:ff:ff", + dst_mac=self.pg0.local_mac, + src_ip="a000::100", + dst_ip="9000::100")) + + self.send_packets(self.pg0, self.pg1) + + self.pg0.unconfig_ip6() + + route_3.remove_vpp_config() + + self.logger.info("FFP_TEST_FINISH_0003") + + def test_ip6_rx_p2p_subif_drop(self): + """drop rx packet not matching p2p subinterface""" + self.logger.info("FFP_TEST_START_0004") + + route_9001 = VppIpRoute(self, "9000::", 64, + [VppRoutePath(self.pg1._remote_hosts[0].ip6, + self.pg1.sw_if_index, + is_ip6=1)], + is_ip6=1) + route_9001.add_vpp_config() + + self.packets.append( + self.create_stream(src_mac="02:03:00:00:ff:ff", + dst_mac=self.pg0.local_mac, + src_ip="a000::100", + dst_ip="9000::100")) + + # no packet received + self.send_packets(self.pg0, self.pg1, count=0) + self.logger.info("FFP_TEST_FINISH_0004") + + def test_ip6_tx_p2p_subif(self): + """send packet via p2p subinterface""" + self.logger.info("FFP_TEST_START_0005") + + route_8000 = VppIpRoute(self, "8000::", 64, + [VppRoutePath(self.pg0.remote_ip6, + self.pg0.sw_if_index, + is_ip6=1)], + is_ip6=1) + route_8000.add_vpp_config() + route_8001 = VppIpRoute(self, "8001::", 64, + [VppRoutePath(self.p2p_sub_ifs[0].remote_ip6, + self.p2p_sub_ifs[0].sw_if_index, + is_ip6=1)], + is_ip6=1) + route_8001.add_vpp_config() + route_8002 = VppIpRoute(self, "8002::", 64, + [VppRoutePath(self.p2p_sub_ifs[1].remote_ip6, + self.p2p_sub_ifs[1].sw_if_index, + is_ip6=1)], + is_ip6=1) + route_8002.add_vpp_config() + + for i in range(0, 3): + self.packets.append( + self.create_stream(src_mac=self.pg1.remote_mac, + dst_mac=self.pg1.local_mac, + src_ip=self.pg1.remote_ip6, + dst_ip="800%d::100" % i)) + + self.send_packets(self.pg1, self.pg0, count=3) + + route_8000.remove_vpp_config() + route_8001.remove_vpp_config() + route_8002.remove_vpp_config() + + self.logger.info("FFP_TEST_FINISH_0005") + + def test_ip6_tx_p2p_subif_drop(self): + """drop tx ip6 packet not matching p2p subinterface""" + self.logger.info("FFP_TEST_START_0006") + + self.packets.append( + self.create_stream(src_mac="02:03:00:00:ff:ff", + dst_mac=self.pg0.local_mac, + src_ip="a000::100", + dst_ip="9000::100")) + + # no packet received + self.send_packets(self.pg0, self.pg1, count=0) + self.logger.info("FFP_TEST_FINISH_0006") + + +class P2PEthernetIPV4(VppTestCase): + """P2P Ethernet IPv4 tests""" + + p2p_sub_ifs = [] + packets = [] + + @classmethod + def setUpClass(cls): + super(P2PEthernetIPV4, cls).setUpClass() + + # Create pg interfaces + cls.create_pg_interfaces(range(3)) + + # Packet sizes + cls.pg_if_packet_sizes = [64, 512, 1518, 9018] + + # Set up all interfaces + for i in cls.pg_interfaces: + i.admin_up() + + cls.pg0.config_ip4() + cls.pg0.generate_remote_hosts(5) + cls.pg0.configure_ipv4_neighbors() + + cls.pg1.config_ip4() + cls.pg1.generate_remote_hosts(5) + cls.pg1.configure_ipv4_neighbors() + + def setUp(self): + super(P2PEthernetIPV4, self).setUp() + for p in self.packets: + self.packets.remove(p) + self.create_p2p_ethernet(self.pg0, 1, self.pg0._remote_hosts[0].mac) + self.create_p2p_ethernet(self.pg0, 2, self.pg0._remote_hosts[1].mac) + self.p2p_sub_ifs[0].config_ip4() + self.p2p_sub_ifs[1].config_ip4() + self.vapi.cli("trace add p2p-ethernet-input 50") + + def tearDown(self): + self.delete_p2p_ethernet(self.pg0, self.pg0._remote_hosts[0].mac) + self.delete_p2p_ethernet(self.pg0, self.pg0._remote_hosts[1].mac) + super(P2PEthernetIPV4, self).tearDown() + + def create_stream(self, src_mac=None, dst_mac=None, + src_ip=None, dst_ip=None, size=None): + pkt_size = size + if size is None: + pkt_size = random.choice(self.pg_if_packet_sizes) + p = Ether(src=src_mac, dst=dst_mac) + p /= IP(src=src_ip, dst=dst_ip) + p /= (UDP(sport=1234, dport=4321) / Raw('\xa5' * 20)) + self.extend_packet(p, pkt_size) + return p + + def send_packets(self, src_if=None, dst_if=None, packets=None, count=None): + self.pg_enable_capture([dst_if]) + if packets is None: + packets = self.packets + src_if.add_stream(packets) + self.pg_start() + if count is None: + count = len(packets) + return dst_if.get_capture(count) + + def verify_counters(self, counter_id, expected_value): + counters = self.vapi.cli("sh errors").split('\n') + counter_value = -1 + for i in range(1, len(counters)-1): + results = counters[i].split() + if results[1] == counter_id: + counter_value = int(results[0]) + break + self.assertEqual(counter_value, expected_value) + + def create_p2p_ethernet(self, parent_if, sub_id, remote_mac): + p2p = VppP2PSubint(self, parent_if, sub_id, mactobinary(remote_mac)) + p2p.admin_up() + p2p.config_ip4() + self.p2p_sub_ifs.append(p2p) + + def delete_p2p_ethernet(self, parent_if, remote_mac): + self.vapi.delete_p2pethernet_subif(parent_if.sw_if_index, + mactobinary(remote_mac)) + + def test_ip4_rx_p2p_subif(self): + """receive ipv4 packet via p2p subinterface""" + self.logger.info("FFP_TEST_START_0002") + + route_9000 = VppIpRoute(self, "9.0.0.0", 16, + [VppRoutePath(self.pg1.remote_ip4, + self.pg1.sw_if_index)]) + route_9000.add_vpp_config() + + self.packets.append( + self.create_stream(src_mac=self.pg0._remote_hosts[0].mac, + dst_mac=self.pg0.local_mac, + src_ip=self.p2p_sub_ifs[0].remote_ip4, + dst_ip="9.0.0.100")) + + self.send_packets(self.pg0, self.pg1, self.packets) + + self.verify_counters('p2p-ethernet-input', 1) + + route_9000.remove_vpp_config() + self.logger.info("FFP_TEST_FINISH_0002") + + def test_ip4_rx_p2p_subif_route(self): + """route rx packet not matching p2p subinterface""" + self.logger.info("FFP_TEST_START_0003") + + route_9001 = VppIpRoute(self, "9.0.0.0", 24, + [VppRoutePath(self.pg1.remote_ip4, + self.pg1.sw_if_index)]) + route_9001.add_vpp_config() + + self.packets.append( + self.create_stream(src_mac="02:01:00:00:ff:ff", + dst_mac=self.pg0.local_mac, + src_ip="8.0.0.100", + dst_ip="9.0.0.100")) + + self.send_packets(self.pg0, self.pg1) + + route_9001.remove_vpp_config() + + self.logger.info("FFP_TEST_FINISH_0003") + + def test_ip4_tx_p2p_subif(self): + """send ip4 packet via p2p subinterface""" + self.logger.info("FFP_TEST_START_0005") + + route_9100 = VppIpRoute(self, "9.1.0.100", 24, + [VppRoutePath(self.pg0.remote_ip4, + self.pg0.sw_if_index, + )]) + route_9100.add_vpp_config() + route_9200 = VppIpRoute(self, "9.2.0.100", 24, + [VppRoutePath(self.p2p_sub_ifs[0].remote_ip4, + self.p2p_sub_ifs[0].sw_if_index, + )]) + route_9200.add_vpp_config() + route_9300 = VppIpRoute(self, "9.3.0.100", 24, + [VppRoutePath(self.p2p_sub_ifs[1].remote_ip4, + self.p2p_sub_ifs[1].sw_if_index + )]) + route_9300.add_vpp_config() + + for i in range(0, 3): + self.packets.append( + self.create_stream(src_mac=self.pg1.remote_mac, + dst_mac=self.pg1.local_mac, + src_ip=self.pg1.remote_ip4, + dst_ip="9.%d.0.100" % (i+1))) + + self.send_packets(self.pg1, self.pg0) + + # route_7000.remove_vpp_config() + route_9100.remove_vpp_config() + route_9200.remove_vpp_config() + route_9300.remove_vpp_config() + + self.logger.info("FFP_TEST_FINISH_0005") + + def test_ip4_tx_p2p_subif_drop(self): + """drop tx ip4 packet not matching p2p subinterface""" + self.logger.info("FFP_TEST_START_0006") + + self.packets.append( + self.create_stream(src_mac="02:01:00:00:ff:ff", + dst_mac=self.pg0.local_mac, + src_ip="8.0.0.100", + dst_ip="9.0.0.100")) + + # no packet received + self.send_packets(self.pg0, self.pg1, count=0) + self.logger.info("FFP_TEST_FINISH_0006") + + +if __name__ == '__main__': + unittest.main(testRunner=VppTestRunner) diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py index 204d9e31..801a6c2d 100644 --- a/test/vpp_papi_provider.py +++ b/test/vpp_papi_provider.py @@ -601,6 +601,19 @@ class VppPapiProvider(object): 'outer_vlan_id': outer_vlan, 'inner_vlan_id': inner_vlan}) + def create_p2pethernet_subif(self, sw_if_index, remote_mac, subif_id): + """Create p2p ethernet subinterface + + :param sw_if_index: main (parent) interface + :param remote_mac: client (remote) mac address + + """ + return self.api( + self.papi.p2p_ethernet_add, + {'parent_if_index': sw_if_index, + 'remote_mac': remote_mac, + 'subif_id': subif_id}) + def delete_subif(self, sw_if_index): """Delete subinterface @@ -609,6 +622,18 @@ class VppPapiProvider(object): return self.api(self.papi.delete_subif, {'sw_if_index': sw_if_index}) + def delete_p2pethernet_subif(self, sw_if_index, remote_mac): + """Delete p2p ethernet subinterface + + :param sw_if_index: main (parent) interface + :param remote_mac: client (remote) mac address + + """ + return self.api( + self.papi.p2p_ethernet_del, + {'parent_if_index': sw_if_index, + 'remote_mac': remote_mac}) + def create_vlan_subif(self, sw_if_index, vlan): """ diff --git a/test/vpp_sub_interface.py b/test/vpp_sub_interface.py index dcd82da2..cabee88d 100644 --- a/test/vpp_sub_interface.py +++ b/test/vpp_sub_interface.py @@ -188,3 +188,26 @@ class VppDot1ADSubint(VppSubInterface): def remove_dot1_layer(self, packet): return self.remove_dot1ad_layer(packet, self.outer_vlan, self.inner_vlan) + + +class VppP2PSubint(VppSubInterface): + + def __init__(self, test, parent, sub_id, remote_mac): + r = test.vapi.create_p2pethernet_subif(parent.sw_if_index, + remote_mac, sub_id) + self._sw_if_index = r.sw_if_index + super(VppP2PSubint, self).__init__(test, parent, sub_id) + + def add_dot1_layer(self, packet): + return packet + + def remove_dot1_layer(self, packet): + return packet + + def create_arp_req(self): + packet = VppPGInterface.create_arp_req(self) + return packet + + def create_ndp_req(self): + packet = VppPGInterface.create_ndp_req(self) + return packet -- cgit 1.2.3-korg