/* * Copyright (c) 2018 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * ethernet_node.c: ethernet packet processing * * Copyright (c) 2008 Eliot Dresselhaus * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include <vlib/vlib.h> #include <vnet/pg/pg.h> #include <vnet/ethernet/ethernet.h> #include <vnet/ethernet/p2p_ethernet.h> #include <vnet/devices/pipe/pipe.h> #include <vppinfra/sparse_vec.h> #include <vnet/l2/l2_bvi.h> #define foreach_ethernet_input_next \ _ (PUNT, "error-punt") \ _ (DROP, "error-drop") \ _ (LLC, "llc-input") \ _ (IP4_INPUT, "ip4-input") \ _ (IP4_INPUT_NCS, "ip4-input-no-checksum") typedef enum { #define _(s,n) ETHERNET_INPUT_NEXT_##s, foreach_ethernet_input_next #undef _ ETHERNET_INPUT_N_NEXT, } ethernet_input_next_t; typedef struct { u8 packet_data[32]; u16 frame_flags; ethernet_input_frame_t frame_data; } ethernet_input_trace_t; static u8 * format_ethernet_input_trace (u8 * s, va_list * va) { CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *); ethernet_input_trace_t *t = va_arg (*va, ethernet_input_trace_t *); u32 indent = format_get_indent (s); if (t->frame_flags) { s = format (s, "frame: flags 0x%x", t->frame_flags); if (t->frame_flags & ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX) s = format (s, ", hw-if-index %u, sw-if-index %u", t->frame_data.hw_if_index, t->frame_data.sw_if_index); s = format (s, "\n%U", format_white_space, indent); } s = format (s, "%U", format_ethernet_header, t->packet_data); return s; } extern vlib_node_registration_t ethernet_input_node; typedef enum { ETHERNET_INPUT_VARIANT_ETHERNET, ETHERNET_INPUT_VARIANT_ETHERNET_TYPE, ETHERNET_INPUT_VARIANT_NOT_L2, } ethernet_input_variant_t; // Parse the ethernet header to extract vlan tags and innermost ethertype static_always_inline void parse_header (ethernet_input_variant_t variant, vlib_buffer_t * b0, u16 * type, u16 * orig_type, u16 * outer_id, u16 * inner_id, u32 * match_flags) { u8 vlan_count; if (variant == ETHERNET_INPUT_VARIANT_ETHERNET || variant == ETHERNET_INPUT_VARIANT_NOT_L2) { ethernet_header_t *e0; e0 = (void *) (b0->data + b0->current_data); vnet_buffer (b0)->l2_hdr_offset = b0->current_data; b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID; vlib_buffer_advance (b0, sizeof (e0[0])); *type = clib_net_to_host_u16 (e0->type); } else if (variant == ETHERNET_INPUT_VARIANT_ETHERNET_TYPE) { // here when prior node was LLC/SNAP processing u16 *e0; e0 = (void *) (b0->data + b0->current_data); vlib_buffer_advance (b0, sizeof (e0[0])); *type = clib_net_to_host_u16 (e0[0]); } // save for distinguishing between dot1q and dot1ad later *orig_type = *type; // default the tags to 0 (used if there is no corresponding tag) *outer_id = 0; *inner_id = 0; *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_0_TAG; vlan_count = 0; // check for vlan encaps if (ethernet_frame_is_tagged (*type)) { ethernet_vlan_header_t *h0; u16 tag; *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_1_TAG; h0 = (void *) (b0->data + b0->current_data); tag = clib_net_to_host_u16 (h0->priority_cfi_and_id); *outer_id = tag & 0xfff; if (0 == *outer_id) *match_flags &= ~SUBINT_CONFIG_MATCH_1_TAG; *type = clib_net_to_host_u16 (h0->type); vlib_buffer_advance (b0, sizeof (h0[0])); vlan_count = 1; if (*type == ETHERNET_TYPE_VLAN) { // Double tagged packet *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_2_TAG; h0 = (void *) (b0->data + b0->current_data); tag = clib_net_to_host_u16 (h0->priority_cfi_and_id); *inner_id = tag & 0xfff; *type = clib_net_to_host_u16 (h0->type); vlib_buffer_advance (b0, sizeof (h0[0])); vlan_count = 2; if (*type == ETHERNET_TYPE_VLAN) { // More than double tagged packet *match_flags = SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_3_TAG; vlib_buffer_advance (b0, sizeof (h0[0])); vlan_count = 3; // "unknown" number, aka, 3-or-more } } } ethernet_buffer_set_vlan_count (b0, vlan_count); } // Determine the subinterface for this packet, given the result of the // vlan table lookups and vlan header parsing. Check the most specific // matches first. static_always_inline void identify_subint (vnet_hw_interface_t * hi, vlib_buffer_t * b0, u32 match_flags, main_intf_t * main_intf, vlan_intf_t * vlan_intf, qinq_intf_t * qinq_intf, u32 * new_sw_if_index, u8 * error0, u32 * is_l2) { u32 matched; matched = eth_identify_subint (hi, match_flags, main_intf, vlan_intf, qinq_intf, new_sw_if_index, error0, is_l2); if (matched) { // Perform L3 my-mac filter // A unicast packet arriving on an L3 interface must have a dmac matching the interface mac. // This is required for promiscuous mode, else we will forward packets we aren't supposed to. if (!(*is_l2)) { ethernet_header_t *e0; e0 = (void *) (b0->data + vnet_buffer (b0)->l2_hdr_offset); if (!(ethernet_address_cast (e0->dst_address))) { if (!ethernet_mac_address_equal ((u8 *) e0, hi->hw_address)) { *error0 = ETHERNET_ERROR_L3_MAC_MISMATCH; } } } // Check for down subinterface *error0 = (*new_sw_if_index) != ~0 ? (*error0) : ETHERNET_ERROR_DOWN; } } static_always_inline void determine_next_node (ethernet_main_t * em, ethernet_input_variant_t variant, u32 is_l20, u32 type0, vlib_buffer_t * b0, u8 * error0, u8 * next0) { vnet_buffer (b0)->l3_hdr_offset = b0->current_data; b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID; if (PREDICT_FALSE (*error0 != ETHERNET_ERROR_NONE)) { // some error occurred *next0 = ETHERNET_INPUT_NEXT_DROP; } else if (is_l20) { // record the L2 len and reset the buffer so the L2 header is preserved u32 eth_start = vnet_buffer (b0)->l2_hdr_offset; vnet_buffer (b0)->l2.l2_len = b0->current_data - eth_start; *next0 = em->l2_next; ASSERT (vnet_buffer (b0)->l2.l2_len == ethernet_buffer_header_size (b0)); vlib_buffer_advance (b0, -(vnet_buffer (b0)->l2.l2_len)); // check for common IP/MPLS ethertypes } else if (type0 == ETHERNET_TYPE_IP4) { *next0 = em->l3_next.input_next_ip4; } else if (type0 == ETHERNET_TYPE_IP6) { *next0 = em->l3_next.input_next_ip6; } else if (type0 == ETHERNET_TYPE_MPLS) { *next0 = em->l3_next.input_next_mpls; } else if (em->redirect_l3) { // L3 Redirect is on, the cached common next nodes will be // pointing to the redirect node, catch the uncommon types here *next0 = em->redirect_l3_next; } else { // uncommon ethertype, check table u32 i0; i0 = sparse_vec_index (em->l3_next.input_next_by_type, type0); *next0 = vec_elt (em->l3_next.input_next_by_type, i0); *error0 = i0 == SPARSE_VEC_INVALID_INDEX ? ETHERNET_ERROR_UNKNOWN_TYPE : *error0; // The table is not populated with LLC values, so check that now. // If variant is variant_ethernet then we came from LLC processing. Don't // go back there; drop instead using by keeping the drop/bad table result. if ((type0 < 0x600) && (variant == ETHERNET_INPUT_VARIANT_ETHERNET)) { *next0 = ETHERNET_INPUT_NEXT_LLC; } } } /* following vector code relies on following assumptions */ STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, current_data, 0); STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, current_length, 2); STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, flags, 4); STATIC_ASSERT (STRUCT_OFFSET_OF (vnet_buffer_opaque_t, l2_hdr_offset) == STRUCT_OFFSET_OF (vnet_buffer_opaque_t, l3_hdr_offset) - 2, "l3_hdr_offset must follow l2_hdr_offset"); static_always_inline void eth_input_adv_and_flags_x4 (vlib_buffer_t ** b, int is_l3) { i16 adv = sizeof (ethernet_header_t); u32 flags = VNET_BUFFER_F_L2_HDR_OFFSET_VALID | VNET_BUFFER_F_L3_HDR_OFFSET_VALID; #ifdef CLIB_HAVE_VEC256 /* to reduce number of small loads/stores we are loading first 64 bits of each buffer metadata into 256-bit register so we can advance current_data, current_length and flags. Observed saving of this code is ~2 clocks per packet */ u64x4 r, radv; /* vector if signed 16 bit integers used in signed vector add operation to advnce current_data and current_length */ u32x8 flags4 = { 0, flags, 0, flags, 0, flags, 0, flags }; i16x16 adv4 = { adv, -adv, 0, 0, adv, -adv, 0, 0, adv, -adv, 0, 0, adv, -adv, 0, 0 }; /* load 4 x 64 bits */ r = u64x4_gather (b[0], b[1], b[2], b[3]); /* set flags */ r |= (u64x4) flags4; /* advance buffer */ radv = (u64x4) ((i16x16) r + adv4); /* write 4 x 64 bits */ u64x4_scatter (is_l3 ? radv : r, b[0], b[1], b[2], b[3]); /* use old current_data as l2_hdr_offset and new current_data as l3_hdr_offset */ r = (u64x4) u16x16_blend (r, radv << 16, 0xaa); /* store both l2_hdr_offset and l3_hdr_offset in single store operation */ u32x8_scatter_one ((u32x8) r, 0, &vnet_buffer (b[0])->l2_hdr_offset); u32x8_scatter_one ((u32x8) r, 2, &vnet_buffer (b[1])->l2_hdr_offset); u32x8_scatter_one ((u32x8) r, 4, &vnet_buffer (b[2])->l2_hdr_offset); u32x8_scatter_one ((u32x8) r, 6, &vnet_buffer (b[3])->l2_hdr_offset); if (is_l3) { ASSERT (b[0]->current_data == vnet_buffer (b[0])->l3_hdr_offset); ASSERT (b[1]->current_data == vnet_buffer (b[1])->l3_hdr_offset); ASSERT (b[2]->current_data == vnet_buffer (b[2])->l3_hdr_offset); ASSERT (b[3]->current_data == vnet_buffer (b[3])->l3_hdr_offset); ASSERT (b[0]->current_data - vnet_buffer (b[0])->l2_hdr_offset == adv); ASSERT (b[1]->current_data - vnet_buffer (b[1])->l2_hdr_offset == adv); ASSERT (b[2]->current_data - vnet_buffer (b[2])->l2_hdr_offset == adv); ASSERT (b[3]->current_data - vnet_buffer (b[3])->l2_hdr_offset == adv); } else { ASSERT (b[0]->current_data == vnet_buffer (b[0])->l2_hdr_offset); ASSERT (b[1]->current_data == vnet_buffer (b[1])->l2_hdr_offset); ASSERT (b[2]->current_data == vnet_buffer (b[2])->l2_hdr_offset); ASSERT (b[3]->current_data == vnet_buffer (b[3])->l2_hdr_offset); ASSERT (b[0]->current_data - vnet_buffer (b[0])->l3_hdr_offset == -adv); ASSERT (b[1]->current_data - vnet_buffer (b[1])->l3_hdr_offset == -adv); ASSERT (b[2]->current_data - vnet_buffer (b[2])->l3_hdr_offset == -adv); ASSERT (b[3]->current_data - vnet_buffer (b[3])->l3_hdr_offset == -adv); } #else vnet_buffer (b[0])->l2_hdr_offset = b[0]->current_data; vnet_buffer (b[1])->l2_hdr_offset = b[1]->current_data; vnet_buffer (b[2])->l2_hdr_offset = b[2]->current_data; vnet_buffer (b[3])->l2_hdr_offset = b[3]->current_data; vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data + adv; vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data + adv; vnet_buffer (b[2])->l3_hdr_offset = b[2]->current_data + adv; vnet_buffer (b[3])->l3_hdr_offset = b[3]->current_data + adv; if (is_l3) { vlib_buffer_advance (b[0], adv); vlib_buffer_advance (b[1], adv); vlib_buffer_advance (b[2], adv); vlib_buffer_advance (b[3], adv); } b[0]->flags |= flags; b[1]->flags |= flags; b[2]->flags |= flags; b[3]->flags |= flags; #endif if (!is_l3) { vnet_buffer (b[0])->l2.l2_len = adv; vnet_buffer (b[1])->l2.l2_len = adv; vnet_buffer (b[2])->l2.l2_len = adv; vnet_buffer (b[3])->l2.l2_len = adv; } } static_always_inline void eth_input_adv_and_flags_x1 (vlib_buffer_t ** b, int is_l3) { i16 adv = sizeof (ethernet_header_t); u32 flags = VNET_BUFFER_F_L2_HDR_OFFSET_VALID | VNET_BUFFER_F_L3_HDR_OFFSET_VALID; vnet_buffer (b[0])->l2_hdr_offset = b[0]->current_data; vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data + adv; if (is_l3) vlib_buffer_advance (b[0], adv); b[0]->flags |= flags; if (!is_l3) vnet_buffer (b[0])->l2.l2_len = adv; } static_always_inline void eth_input_get_etype_and_tags (vlib_buffer_t ** b, u16 * etype, u64 * tags, u64 * dmacs, int offset, int dmac_check) { ethernet_header_t *e; e = vlib_buffer_get_current (b[offset]); #ifdef CLIB_HAVE_VEC128 u64x2 r = u64x2_load_unaligned (((u8 *) & e->type) - 6); etype[offset] = ((u16x8) r)[3]; tags[offset] = r[1]; #else etype[offset] = e->type; tags[offset] = *(u64 *) (e + 1); #endif if (dmac_check) dmacs[offset] = *(u64 *) e; } static_always_inline u16 eth_input_next_by_type (u16 etype) { ethernet_main_t *em = ðernet_main; return (etype < 0x600) ? ETHERNET_INPUT_NEXT_LLC : vec_elt (em->l3_next.input_next_by_type, sparse_vec_index (em->l3_next.input_next_by_type, etype)); } typedef struct { u64 tag, mask; u32 sw_if_index; u16 type, len, next; i16 adv; u8 err, n_tags; u64 n_packets, n_bytes; } eth_input_tag_lookup_t; static_always_inline void eth_input_update_if_counters (vlib_main_t * vm, vnet_main_t * vnm, eth_input_tag_lookup_t * l) { if (l->n_packets == 0 || l->sw_if_index == ~0) return; if (l->adv > 0) l->n_bytes += l->n_packets * l->len; vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, vm->thread_index, l->sw_if_index, l->n_packets, l->n_bytes); } static_always_inline void eth_input_tag_lookup (vlib_main_t * vm, vnet_main_t * vnm, vlib_node_runtime_t * node, vnet_hw_interface_t * hi, u64 tag, u16 * next, vlib_buffer_t * b, eth_input_tag_lookup_t * l, u8 dmac_bad, int is_dot1ad, int main_is_l3, int check_dmac) { ethernet_main_t *em = ðernet_main; if ((tag ^ l->tag) & l->mask) { main_intf_t *mif = vec_elt_at_index (em->main_intfs, hi->hw_if_index); vlan_intf_t *vif; qinq_intf_t *qif; vlan_table_t *vlan_table; qinq_table_t *qinq_table; u16 *t = (u16 *) & tag; u16 vlan1 = clib_net_to_host_u16 (t[0]) & 0xFFF; u16 vlan2 = clib_net_to_host_u16 (t[2]) & 0xFFF; u32 matched, is_l2, new_sw_if_index; vlan_table = vec_elt_at_index (em->vlan_pool, is_dot1ad ? mif->dot1ad_vlans : mif->dot1q_vlans); vif = &vlan_table->vlans[vlan1]; qinq_table = vec_elt_at_index (em->qinq_pool, vif->qinqs); qif = &qinq_table->vlans[vlan2]; l->err = ETHERNET_ERROR_NONE; l->type = clib_net_to_host_u16 (t[1]); if (l->type == ETHERNET_TYPE_VLAN) { l->type = clib_net_to_host_u16 (t[3]); l->n_tags = 2; matched = eth_identify_subint (hi, SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_2_TAG, mif, vif, qif, &new_sw_if_index, &l->err, &is_l2); } else { l->n_tags = 1; if (vlan1 == 0) { new_sw_if_index = hi->sw_if_index; l->err = ETHERNET_ERROR_NONE; matched = 1; is_l2 = main_is_l3 == 0; } else matched = eth_identify_subint (hi, SUBINT_CONFIG_VALID | SUBINT_CONFIG_MATCH_1_TAG, mif, vif, qif, &new_sw_if_index, &l->err, &is_l2); } if (l->sw_if_index != new_sw_if_index) { eth_input_update_if_counters (vm, vnm, l); l->n_packets = 0; l->n_bytes = 0; l->sw_if_index = new_sw_if_index; } l->tag = tag; l->mask = (l->n_tags == 2) ? clib_net_to_host_u64 (0xffffffffffffffff) : clib_net_to_host_u64 (0xffffffff00000000); if (matched && l->sw_if_index == ~0) l->err = ETHERNET_ERROR_DOWN; l->len = sizeof (ethernet_header_t) + l->n_tags * sizeof (ethernet_vlan_header_t); if (main_is_l3) l->adv = is_l2 ? -(int) sizeof (ethernet_header_t) : l->n_tags * sizeof (ethernet_vlan_header_t); else l->adv = is_l2 ? 0 : l->len; if (PREDICT_FALSE (l->err != ETHERNET_ERROR_NONE)) l->next = ETHERNET_INPUT_NEXT_DROP; else if (is_l2) l->next = em->l2_next; else if (l->type == ETHERNET_TYPE_IP4) l->next = em->l3_next.input_next_ip4; else if (l->type == ETHERNET_TYPE_IP6) l->next = em->l3_next.input_next_ip6; else if (l->type == ETHERNET_TYPE_MPLS) l->next = em->l3_next.input_next_mpls; else if (em->redirect_l3) l->next = em->redirect_l3_next; else { l->next = eth_input_next_by_type (l->type); if (l->next == ETHERNET_INPUT_NEXT_PUNT) l->err = ETHERNET_ERROR_UNKNOWN_TYPE; } } if (check_dmac && l->adv > 0 && dmac_bad) { l->err = ETHERNET_ERROR_L3_MAC_MISMATCH; next[0] = ETHERNET_INPUT_NEXT_PUNT; } else next[0] = l->next; vlib_buffer_advance (b, l->adv); vnet_buffer (b)->l2.l2_len = l->len; vnet_buffer (b)->l3_hdr_offset = vnet_buffer (b)->l2_hdr_offset + l->len; if (l->err == ETHERNET_ERROR_NONE) { vnet_buffer (b)->sw_if_index[VLIB_RX] = l->sw_if_index; ethernet_buffer_set_vlan_count (b, l->n_tags); } else b->error = node->errors[l->err]; /* update counters */ l->n_packets += 1; l->n_bytes += vlib_buffer_length_in_chain (vm, b); } /* process frame of buffers, store ethertype into array and update buffer metadata fields depending on interface being l2 or l3 assuming that packets are untagged. For tagged packets those fields are updated later. Optionally store Destionation MAC address and tag data into arrays for further processing */ STATIC_ASSERT (VLIB_FRAME_SIZE % 8 == 0, "VLIB_FRAME_SIZE must be power of 8"); static_always_inline void eth_input_process_frame (vlib_main_t * vm, vlib_node_runtime_t * node, vnet_hw_interface_t * hi, u32 * buffer_indices, u32 n_packets, int main_is_l3, int ip4_cksum_ok, int dmac_check) { ethernet_main_t *em = ðernet_main; u16 nexts[VLIB_FRAME_SIZE], *next; u16 etypes[VLIB_FRAME_SIZE], *etype = etypes; u64 dmacs[VLIB_FRAME_SIZE], *dmac = dmacs; u8 dmacs_bad[VLIB_FRAME_SIZE]; u64 tags[VLIB_FRAME_SIZE], *tag = tags; u16 slowpath_indices[VLIB_FRAME_SIZE]; u16 n_slowpath, i; u16 next_ip4, next_ip6, next_mpls, next_l2; u16 et_ip4 = clib_host_to_net_u16 (ETHERNET_TYPE_IP4); u16 et_ip6 = clib_host_to_net_u16 (ETHERNET_TYPE_IP6); u16 et_mpls = clib_host_to_net_u16 (ETHERNET_TYPE_MPLS); u16 et_vlan = clib_host_to_net_u16 (ETHERNET_TYPE_VLAN); u16 et_dot1ad = clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD); i32 n_left = n_packets; vlib_buffer_t *b[20]; u32 *from; from = buffer_indices; while (n_left >= 20) { vlib_buffer_t **ph = b + 16, **pd = b + 8; vlib_get_buffers (vm, from, b, 4); vlib_get_buffers (vm, from + 8, pd, 4); vlib_get_buffers (vm, from + 16, ph, 4); vlib_prefetch_buffer_header (ph[0], LOAD); vlib_prefetch_buffer_data (pd[0], LOAD); eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check); vlib_prefetch_buffer_header (ph[1], LOAD); vlib_prefetch_buffer_data (pd[1], LOAD); eth_input_get_etype_and_tags (b, etype, tag, dmac, 1, dmac_check); vlib_prefetch_buffer_header (ph[2], LOAD); vlib_prefetch_buffer_data (pd[2], LOAD); eth_input_get_etype_and_tags (b, etype, tag, dmac, 2, dmac_check); vlib_prefetch_buffer_header (ph[3], LOAD); vlib_prefetch_buffer_data (pd[3], LOAD); eth_input_get_etype_and_tags (b, etype, tag, dmac, 3, dmac_check); eth_input_adv_and_flags_x4 (b, main_is_l3); /* next */ n_left -= 4; etype += 4; tag += 4; dmac += 4; from += 4; } while (n_left >= 4) { vlib_get_buffers (vm, from, b, 4); eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check); eth_input_get_etype_and_tags (b, etype, tag, dmac, 1, dmac_check); eth_input_get_etype_and_tags (b, etype, tag, dmac, 2, dmac_check); eth_input_get_etype_and_tags (b, etype, tag, dmac, 3, dmac_check); eth_input_adv_and_flags_x4 (b, main_is_l3); /* next */ n_left -= 4; etype += 4; tag += 4; dmac += 4; from += 4; } while (n_left) { vlib_get_buffers (vm, from, b, 1); eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check); eth_input_adv_and_flags_x1 (b, main_is_l3); /* next */ n_left -= 1; etype += 1; tag += 1; dmac += 4; from += 1; } if (dmac_check) { u64 mask = clib_net_to_host_u64 (0xFFFFFFFFFFFF0000); u64 igbit = clib_net_to_host_u64 (0x0100000000000000); u64 hwaddr = (*(u64 *) hi->hw_address) & mask; u64 *dmac = dmacs; u8 *dmac_bad = dmacs_bad; n_left = n_packets; #ifdef CLIB_HAVE_VEC256 u64x4 igbit4 = u64x4_splat (igbit); u64x4 mask4 = u64x4_splat (mask); u64x4 hwaddr4 = u64x4_splat (hwaddr); while (n_left >= 0) { u64x4 r0, r1; r0 = u64x4_load_unaligned (dmac + 0) & mask4; r1 = u64x4_load_unaligned (dmac + 4) & mask4; r0 = (r0 != hwaddr4) & ((r0 & igbit4) == 0); r1 = (r1 != hwaddr4) & ((r1 & igbit4) == 0); *(u32 *) (dmac_bad + 0) = u8x32_msb_mask ((u8x32) (r0)); *(u32 *) (dmac_bad + 4) = u8x32_msb_mask ((u8x32) (r1)); /* next */ dmac += 8; dmac_bad += 8; n_left -= 8; } #else while (n_left > 0) { u64 r0, r1, r2, r3; r0 = dmac[0] & mask; r1 = dmac[1] & mask; r2 = dmac[2] & mask; r3 = dmac[3] & mask; r0 = (r0 != hwaddr) && ((r0 & igbit) == 0); r1 = (r1 != hwaddr) && ((r1 & igbit) == 0); r2 = (r2 != hwaddr) && ((r2 & igbit) == 0); r3 = (r3 != hwaddr) && ((r3 & igbit) == 0); dmac_bad[0] = r0; dmac_bad[1] = r1; dmac_bad[2] = r2; dmac_bad[3] = r3; /* next */ dmac += 4; dmac_bad += 4; n_left -= 4; } #endif } next_ip4 = em->l3_next.input_next_ip4; next_ip6 = em->l3_next.input_next_ip6; next_mpls = em->l3_next.input_next_mpls; next_l2 = em->l2_next; if (next_ip4 == ETHERNET_INPUT_NEXT_IP4_INPUT && ip4_cksum_ok) next_ip4 = ETHERNET_INPUT_NEXT_IP4_INPUT_NCS; #ifdef CLIB_HAVE_VEC256 u16x16 et16_ip4 = u16x16_splat (et_ip4); u16x16 et16_ip6 = u16x16_splat (et_ip6); u16x16 et16_mpls = u16x16_splat (et_mpls); u16x16 et16_vlan = u16x16_splat (et_vlan); u16x16 et16_dot1ad = u16x16_splat (et_dot1ad); u16x16 next16_ip4 = u16x16_splat (next_ip4); u16x16 next16_ip6 = u16x16_splat (next_ip6); u16x16 next16_mpls = u16x16_splat (next_mpls); u16x16 next16_l2 = u16x16_splat (next_l2); u16x16 zero = { 0 }; u16x16 stairs = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; #endif etype = etypes; n_left = n_packets; next = nexts; n_slowpath = 0; i = 0; /* fastpath - in l3 mode hadles ip4, ip6 and mpls packets, other packets are considered as slowpath, in l2 mode all untagged packets are considered as fastpath */ while (n_left > 0) { #ifdef CLIB_HAVE_VEC256 if (n_left >= 16) { u16x16 r = zero; u16x16 e16 = u16x16_load_unaligned (etype); if (main_is_l3) { r += (e16 == et16_ip4) & next16_ip4; r += (e16 == et16_ip6) & next16_ip6; r += (e16 == et16_mpls) & next16_mpls; } else r = ((e16 != et16_vlan) & (e16 != et16_dot1ad)) & next16_l2; u16x16_store_unaligned (r, next); if (!u16x16_is_all_zero (r == zero)) { if (u16x16_is_all_zero (r)) { u16x16_store_unaligned (u16x16_splat (i) + stairs, slowpath_indices + n_slowpath); n_slowpath += 16; } else { for (int j = 0; j < 16; j++) if (next[j] == 0) slowpath_indices[n_slowpath++] = i + j; } } etype += 16; next += 16; n_left -= 16; i += 16; continue; } #endif if (main_is_l3 && etype[0] == et_ip4) next[0] = next_ip4; else if (main_is_l3 && etype[0] == et_ip6) next[0] = next_ip6; else if (main_is_l3 && etype[0] == et_mpls) next[0] = next_mpls; else if (main_is_l3 == 0 && etype[0] != et_vlan && etype[0] != et_dot1ad) next[0] = next_l2; else { next[0] = 0; slowpath_indices[n_slowpath++] = i; } etype += 1; next += 1; n_left -= 1; i += 1; } if (n_slowpath) { vnet_main_t *vnm = vnet_get_main (); n_left = n_slowpath; u16 *si = slowpath_indices; u32 last_unknown_etype = ~0; u32 last_unknown_next = ~0; eth_input_tag_lookup_t dot1ad_lookup, dot1q_lookup = { .mask = -1LL, .tag = tags[si[0]] ^ -1LL, .sw_if_index = ~0 }; clib_memcpy_fast (&dot1ad_lookup, &dot1q_lookup, sizeof (dot1q_lookup)); while (n_left) { i = si[0]; u16 etype = etypes[i]; if (etype == et_vlan) { vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]); eth_input_tag_lookup (vm, vnm, node, hi, tags[i], nexts + i, b, &dot1q_lookup, dmacs_bad[i], 0, main_is_l3, dmac_check); } else if (etype == et_dot1ad) { vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]); eth_input_tag_lookup (vm, vnm, node, hi, tags[i], nexts + i, b, &dot1ad_lookup, dmacs_bad[i], 1, main_is_l3, dmac_check); } else { /* untagged packet with not well known etyertype */ if (last_unknown_etype != etype) { last_unknown_etype = etype; etype = clib_host_to_net_u16 (etype); last_unknown_next = eth_input_next_by_type (etype); } if (dmac_check && main_is_l3 && dmacs_bad[i]) { vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]); b->error = node->errors[ETHERNET_ERROR_L3_MAC_MISMATCH]; nexts[i] = ETHERNET_INPUT_NEXT_PUNT; } else nexts[i] = last_unknown_next; } /* next */ n_left--; si++; } eth_input_update_if_counters (vm, vnm, &dot1q_lookup); eth_input_update_if_counters (vm, vnm, &dot1ad_lookup); } vlib_buffer_enqueue_to_next (vm, node, buffer_indices, nexts, n_packets); } static_always_inline void eth_input_single_int (vlib_main_t * vm, vlib_node_runtime_t * node, vnet_hw_interface_t * hi, u32 * from, u32 n_pkts, int ip4_cksum_ok) { ethernet_main_t *em = ðernet_main; ethernet_interface_t *ei; ei = pool_elt_at_index (em->interfaces, hi->hw_instance); main_intf_t *intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index); subint_config_t *subint0 = &intf0->untagged_subint; int main_is_l3 = (subint0->flags & SUBINT_CONFIG_L2) == 0; int promisc = (ei->flags & ETHERNET_INTERFACE_FLAG_ACCEPT_ALL) != 0; if (main_is_l3) { /* main interface is L3, we dont expect tagged packets and interface is not in promisc node, so we dont't need to check DMAC */ int is_l3 = 1; if (promisc == 0) eth_input_process_frame (vm, node, hi, from, n_pkts, is_l3, ip4_cksum_ok, 0); else /* subinterfaces and promisc mode so DMAC check is needed */ eth_input_process_frame (vm, node, hi, from, n_pkts, is_l3, ip4_cksum_ok, 1); return; } else { /* untagged packets are treated as L2 */ int is_l3 = 0; eth_input_process_frame (vm, node, hi, from, n_pkts, is_l3, ip4_cksum_ok, 1); return; } } static_always_inline void ethernet_input_trace (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame) { u32 *from, n_left; if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) == 0)) { from = vlib_frame_vector_args (from_frame); n_left = from_frame->n_vectors; while (n_left) { ethernet_input_trace_t *t0; vlib_buffer_t *b0 = vlib_get_buffer (vm, from[0]); if (b0->flags & VLIB_BUFFER_IS_TRACED) { t0 = vlib_add_trace (vm, node, b0, sizeof (ethernet_input_trace_t)); clib_memcpy_fast (t0->packet_data, b0->data + b0->current_data, sizeof (t0->packet_data)); t0->frame_flags = from_frame->flags; clib_memcpy_fast (&t0->frame_data, vlib_frame_scalar_args (from_frame), sizeof (ethernet_input_frame_t)); } from += 1; n_left -= 1; } } /* rx pcap capture if enabled */ if (PREDICT_FALSE (vm->pcap[VLIB_RX].pcap_enable)) { u32 bi0; from = vlib_frame_vector_args (from_frame); n_left = from_frame->n_vectors; while (n_left > 0) { vlib_buffer_t *b0; bi0 = from[0]; from++; b0 = vlib_get_buffer (vm, bi0); if (vm->pcap[VLIB_RX].pcap_sw_if_index == 0 || vm->pcap[VLIB_RX].pcap_sw_if_index == vnet_buffer (b0)->sw_if_index[VLIB_RX]) { pcap_add_buffer (&vm->pcap[VLIB_RX].pcap_main, vm, bi0, 512); } n_left--; } } } static_always_inline void ethernet_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u32 * from, u32 n_packets, ethernet_input_variant_t variant) { vnet_main_t *vnm = vnet_get_main (); ethernet_main_t *em = ðernet_main; vlib_node_runtime_t *error_node; u32 n_left_from, next_index, *to_next; u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; u32 thread_index = vm->thread_index; u32 cached_sw_if_index = ~0; u32 cached_is_l2 = 0; /* shut up gcc */ vnet_hw_interface_t *hi = NULL; /* used for main interface only */ if (variant != ETHERNET_INPUT_VARIANT_ETHERNET) error_node = vlib_node_get_runtime (vm, ethernet_input_node.index); else error_node = node; n_left_from = n_packets; next_index = node->cached_next_index; stats_sw_if_index = node->runtime_data[0]; stats_n_packets = stats_n_bytes = 0; while (n_left_from > 0) { u32 n_left_to_next; vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); while (n_left_from >= 4 && n_left_to_next >= 2) { u32 bi0, bi1; vlib_buffer_t *b0, *b1; u8 next0, next1, error0, error1; u16 type0, orig_type0, type1, orig_type1; u16 outer_id0, inner_id0, outer_id1, inner_id1; u32 match_flags0, match_flags1; u32 old_sw_if_index0, new_sw_if_index0, len0, old_sw_if_index1, new_sw_if_index1, len1; vnet_hw_interface_t *hi0, *hi1; main_intf_t *main_intf0, *main_intf1; vlan_intf_t *vlan_intf0, *vlan_intf1; qinq_intf_t *qinq_intf0, *qinq_intf1; u32 is_l20, is_l21; ethernet_header_t *e0, *e1; /* Prefetch next iteration. */ { vlib_buffer_t *b2, *b3; b2 = vlib_get_buffer (vm, from[2]); b3 = vlib_get_buffer (vm, from[3]); vlib_prefetch_buffer_header (b2, STORE); vlib_prefetch_buffer_header (b3, STORE); CLIB_PREFETCH (b2->data, sizeof (ethernet_header_t), LOAD); CLIB_PREFETCH (b3->data, sizeof (ethernet_header_t), LOAD); } bi0 = from[0]; bi1 = from[1]; to_next[0] = bi0; to_next[1] = bi1; from += 2; to_next += 2; n_left_to_next -= 2; n_left_from -= 2; b0 = vlib_get_buffer (vm, bi0); b1 = vlib_get_buffer (vm, bi1); error0 = error1 = ETHERNET_ERROR_NONE; e0 = vlib_buffer_get_current (b0); type0 = clib_net_to_host_u16 (e0->type); e1 = vlib_buffer_get_current (b1); type1 = clib_net_to_host_u16 (e1->type); /* Set the L2 header offset for all packets */ vnet_buffer (b0)->l2_hdr_offset = b0->current_data; vnet_buffer (b1)->l2_hdr_offset = b1->current_data; b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID; b1->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID; /* Speed-path for the untagged case */ if (PREDICT_TRUE (variant == ETHERNET_INPUT_VARIANT_ETHERNET && !ethernet_frame_is_any_tagged_x2 (type0, type1))) { main_intf_t *intf0; subint_config_t *subint0; u32 sw_if_index0, sw_if_index1; sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX]; is_l20 = cached_is_l2; /* This is probably wholly unnecessary */ if (PREDICT_FALSE (sw_if_index0 != sw_if_index1)) goto slowpath; /* Now sw_if_index0 == sw_if_index1 */ if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0)) { cached_sw_if_index = sw_if_index0; hi = vnet_get_sup_hw_interface (vnm, sw_if_index0); intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index); subint0 = &intf0->untagged_subint; cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2; } if (PREDICT_TRUE (is_l20 != 0)) { vnet_buffer (b0)->l3_hdr_offset = vnet_buffer (b0)->l2_hdr_offset + sizeof (ethernet_header_t); vnet_buffer (b1)->l3_hdr_offset = vnet_buffer (b1)->l2_hdr_offset + sizeof (ethernet_header_t); b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID; b1->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID; next0 = em->l2_next; vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t); next1 = em->l2_next; vnet_buffer (b1)->l2.l2_len = sizeof (ethernet_header_t); } else { if (!ethernet_address_cast (e0->dst_address) && (hi->hw_address != 0) && !ethernet_mac_address_equal ((u8 *) e0, hi->hw_address)) error0 = ETHERNET_ERROR_L3_MAC_MISMATCH; if (!ethernet_address_cast (e1->dst_address) && (hi->hw_address != 0) && !ethernet_mac_address_equal ((u8 *) e1, hi->hw_address)) error1 = ETHERNET_ERROR_L3_MAC_MISMATCH; vlib_buffer_advance (b0, sizeof (ethernet_header_t)); determine_next_node (em, variant, 0, type0, b0, &error0, &next0); vlib_buffer_advance (b1, sizeof (ethernet_header_t)); determine_next_node (em, variant, 0, type1, b1, &error1, &next1); } goto ship_it01; } /* Slow-path for the tagged case */ slowpath: parse_header (variant, b0, &type0, &orig_type0, &outer_id0, &inner_id0, &match_flags0); parse_header (variant, b1, &type1, &orig_type1, &outer_id1, &inner_id1, &match_flags1); old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; old_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX]; eth_vlan_table_lookups (em, vnm, old_sw_if_index0, orig_type0, outer_id0, inner_id0, &hi0, &main_intf0, &vlan_intf0, &qinq_intf0); eth_vlan_table_lookups (em, vnm, old_sw_if_index1, orig_type1, outer_id1, inner_id1, &hi1, &main_intf1, &vlan_intf1, &qinq_intf1); identify_subint (hi0, b0, match_flags0, main_intf0, vlan_intf0, qinq_intf0, &new_sw_if_index0, &error0, &is_l20); identify_subint (hi1, b1, match_flags1, main_intf1, vlan_intf1, qinq_intf1, &new_sw_if_index1, &error1, &is_l21); // Save RX sw_if_index for later nodes vnet_buffer (b0)->sw_if_index[VLIB_RX] = error0 != ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0; vnet_buffer (b1)->sw_if_index[VLIB_RX] = error1 != ETHERNET_ERROR_NONE ? old_sw_if_index1 : new_sw_if_index1; // Check if there is a stat to take (valid and non-main sw_if_index for pkt 0 or pkt 1) if (((new_sw_if_index0 != ~0) && (new_sw_if_index0 != old_sw_if_index0)) || ((new_sw_if_index1 != ~0) && (new_sw_if_index1 != old_sw_if_index1))) { len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data - vnet_buffer (b0)->l2_hdr_offset; len1 = vlib_buffer_length_in_chain (vm, b1) + b1->current_data - vnet_buffer (b1)->l2_hdr_offset; stats_n_packets += 2; stats_n_bytes += len0 + len1; if (PREDICT_FALSE (!(new_sw_if_index0 == stats_sw_if_index && new_sw_if_index1 == stats_sw_if_index))) { stats_n_packets -= 2; stats_n_bytes -= len0 + len1; if (new_sw_if_index0 != old_sw_if_index0 && new_sw_if_index0 != ~0) vlib_increment_combined_counter (vnm-> interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thread_index, new_sw_if_index0, 1, len0); if (new_sw_if_index1 != old_sw_if_index1 && new_sw_if_index1 != ~0) vlib_increment_combined_counter (vnm-> interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thread_index, new_sw_if_index1, 1, len1); if (new_sw_if_index0 == new_sw_if_index1) { if (stats_n_packets > 0) { vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = stats_n_bytes = 0; } stats_sw_if_index = new_sw_if_index0; } } } if (variant == ETHERNET_INPUT_VARIANT_NOT_L2) is_l20 = is_l21 = 0; determine_next_node (em, variant, is_l20, type0, b0, &error0, &next0); determine_next_node (em, variant, is_l21, type1, b1, &error1, &next1); ship_it01: b0->error = error_node->errors[error0]; b1->error = error_node->errors[error1]; // verify speculative enqueue vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1); } while (n_left_from > 0 && n_left_to_next > 0) { u32 bi0; vlib_buffer_t *b0; u8 error0, next0; u16 type0, orig_type0; u16 outer_id0, inner_id0; u32 match_flags0; u32 old_sw_if_index0, new_sw_if_index0, len0; vnet_hw_interface_t *hi0; main_intf_t *main_intf0; vlan_intf_t *vlan_intf0; qinq_intf_t *qinq_intf0; ethernet_header_t *e0; u32 is_l20; // Prefetch next iteration if (n_left_from > 1) { vlib_buffer_t *p2; p2 = vlib_get_buffer (vm, from[1]); vlib_prefetch_buffer_header (p2, STORE); CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); } bi0 = from[0]; to_next[0] = bi0; from += 1; to_next += 1; n_left_from -= 1; n_left_to_next -= 1; b0 = vlib_get_buffer (vm, bi0); error0 = ETHERNET_ERROR_NONE; e0 = vlib_buffer_get_current (b0); type0 = clib_net_to_host_u16 (e0->type); /* Set the L2 header offset for all packets */ vnet_buffer (b0)->l2_hdr_offset = b0->current_data; b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID; /* Speed-path for the untagged case */ if (PREDICT_TRUE (variant == ETHERNET_INPUT_VARIANT_ETHERNET && !ethernet_frame_is_tagged (type0))) { main_intf_t *intf0; subint_config_t *subint0; u32 sw_if_index0; sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; is_l20 = cached_is_l2; if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0)) { cached_sw_if_index = sw_if_index0; hi = vnet_get_sup_hw_interface (vnm, sw_if_index0); intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index); subint0 = &intf0->untagged_subint; cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2; } if (PREDICT_TRUE (is_l20 != 0)) { vnet_buffer (b0)->l3_hdr_offset = vnet_buffer (b0)->l2_hdr_offset + sizeof (ethernet_header_t); b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID; next0 = em->l2_next; vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t); } else { if (!ethernet_address_cast (e0->dst_address) && (hi->hw_address != 0) && !ethernet_mac_address_equal ((u8 *) e0, hi->hw_address)) error0 = ETHERNET_ERROR_L3_MAC_MISMATCH; vlib_buffer_advance (b0, sizeof (ethernet_header_t)); determine_next_node (em, variant, 0, type0, b0, &error0, &next0); } goto ship_it0; } /* Slow-path for the tagged case */ parse_header (variant, b0, &type0, &orig_type0, &outer_id0, &inner_id0, &match_flags0); old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; eth_vlan_table_lookups (em, vnm, old_sw_if_index0, orig_type0, outer_id0, inner_id0, &hi0, &main_intf0, &vlan_intf0, &qinq_intf0); identify_subint (hi0, b0, match_flags0, main_intf0, vlan_intf0, qinq_intf0, &new_sw_if_index0, &error0, &is_l20); // Save RX sw_if_index for later nodes vnet_buffer (b0)->sw_if_index[VLIB_RX] = error0 != ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0; // Increment subinterface stats // Note that interface-level counters have already been incremented // prior to calling this function. Thus only subinterface counters // are incremented here. // // Interface level counters include packets received on the main // interface and all subinterfaces. Subinterface level counters // include only those packets received on that subinterface // Increment stats if the subint is valid and it is not the main intf if ((new_sw_if_index0 != ~0) && (new_sw_if_index0 != old_sw_if_index0)) { len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data - vnet_buffer (b0)->l2_hdr_offset; stats_n_packets += 1; stats_n_bytes += len0; // Batch stat increments from the same subinterface so counters // don't need to be incremented for every packet. if (PREDICT_FALSE (new_sw_if_index0 != stats_sw_if_index)) { stats_n_packets -= 1; stats_n_bytes -= len0; if (new_sw_if_index0 != ~0) vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thread_index, new_sw_if_index0, 1, len0); if (stats_n_packets > 0) { vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = stats_n_bytes = 0; } stats_sw_if_index = new_sw_if_index0; } } if (variant == ETHERNET_INPUT_VARIANT_NOT_L2) is_l20 = 0; determine_next_node (em, variant, is_l20, type0, b0, &error0, &next0); ship_it0: b0->error = error_node->errors[error0]; // verify speculative enqueue vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0); } vlib_put_next_frame (vm, node, next_index, n_left_to_next); } // Increment any remaining batched stats if (stats_n_packets > 0) { vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters + VNET_INTERFACE_COUNTER_RX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } } VLIB_NODE_FN (ethernet_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { vnet_main_t *vnm = vnet_get_main (); u32 *from = vlib_frame_vector_args (frame); u32 n_packets = frame->n_vectors; ethernet_input_trace (vm, node, frame); if (frame->flags & ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX) { ethernet_input_frame_t *ef = vlib_frame_scalar_args (frame); int ip4_cksum_ok = (frame->flags & ETH_INPUT_FRAME_F_IP4_CKSUM_OK) != 0; vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, ef->hw_if_index); eth_input_single_int (vm, node, hi, from, n_packets, ip4_cksum_ok); } else ethernet_input_inline (vm, node, from, n_packets, ETHERNET_INPUT_VARIANT_ETHERNET); return n_packets; } VLIB_NODE_FN (ethernet_input_type_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame) { u32 *from = vlib_frame_vector_args (from_frame); u32 n_packets = from_frame->n_vectors; ethernet_input_trace (vm, node, from_frame); ethernet_input_inline (vm, node, from, n_packets, ETHERNET_INPUT_VARIANT_ETHERNET_TYPE); return n_packets; } VLIB_NODE_FN (ethernet_input_not_l2_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame) { u32 *from = vlib_frame_vector_args (from_frame); u32 n_packets = from_frame->n_vectors; ethernet_input_trace (vm, node, from_frame); ethernet_input_inline (vm, node, from, n_packets, ETHERNET_INPUT_VARIANT_NOT_L2); return n_packets; } // Return the subinterface config struct for the given sw_if_index // Also return via parameter the appropriate match flags for the // configured number of tags. // On error (unsupported or not ethernet) return 0. static subint_config_t * ethernet_sw_interface_get_config (vnet_main_t * vnm, u32 sw_if_index, u32 * flags, u32 * unsupported) { ethernet_main_t *em = ðernet_main; vnet_hw_interface_t *hi; vnet_sw_interface_t *si; main_intf_t *main_intf; vlan_table_t *vlan_table; qinq_table_t *qinq_table; subint_config_t *subint = 0; hi = vnet_get_sup_hw_interface (vnm, sw_if_index); if (!hi || (hi->hw_class_index != ethernet_hw_interface_class.index)) { *unsupported = 0; goto done; // non-ethernet interface } // ensure there's an entry for the main intf (shouldn't really be necessary) vec_validate (em->main_intfs, hi->hw_if_index); main_intf = vec_elt_at_index (em->main_intfs, hi->hw_if_index); // Locate the subint for the given ethernet config si = vnet_get_sw_interface (vnm, sw_if_index); if (si->type == VNET_SW_INTERFACE_TYPE_P2P) { p2p_ethernet_main_t *p2pm = &p2p_main; u32 p2pe_sw_if_index = p2p_ethernet_lookup (hi->hw_if_index, si->p2p.client_mac); if (p2pe_sw_if_index == ~0) { pool_get (p2pm->p2p_subif_pool, subint); si->p2p.pool_index = subint - p2pm->p2p_subif_pool; } else subint = vec_elt_at_index (p2pm->p2p_subif_pool, si->p2p.pool_index); *flags = SUBINT_CONFIG_P2P; } else if (si->type == VNET_SW_INTERFACE_TYPE_PIPE) { pipe_t *pipe; pipe = pipe_get (sw_if_index); subint = &pipe->subint; *flags = SUBINT_CONFIG_P2P; } else if (si->sub.eth.flags.default_sub) { subint = &main_intf->default_subint; *flags = SUBINT_CONFIG_MATCH_1_TAG | SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG; } else if ((si->sub.eth.flags.no_tags) || (si->sub.eth.raw_flags == 0)) { // if no flags are set then this is a main interface // so treat as untagged subint = &main_intf->untagged_subint; *flags = SUBINT_CONFIG_MATCH_0_TAG; } else { // one or two tags // first get the vlan table if (si->sub.eth.flags.dot1ad) { if (main_intf->dot1ad_vlans == 0) { // Allocate a vlan table from the pool pool_get (em->vlan_pool, vlan_table); main_intf->dot1ad_vlans = vlan_table - em->vlan_pool; } else { // Get ptr to existing vlan table vlan_table = vec_elt_at_index (em->vlan_pool, main_intf->dot1ad_vlans); } } else { // dot1q if (main_intf->dot1q_vlans == 0) { // Allocate a vlan table from the pool pool_get (em->vlan_pool, vlan_table); main_intf->dot1q_vlans = vlan_table - em->vlan_pool; } else { // Get ptr to existing vlan table vlan_table = vec_elt_at_index (em->vlan_pool, main_intf->dot1q_vlans); } } if (si->sub.eth.flags.one_tag) { *flags = si->sub.eth.flags.exact_match ? SUBINT_CONFIG_MATCH_1_TAG : (SUBINT_CONFIG_MATCH_1_TAG | SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG); if (si->sub.eth.flags.outer_vlan_id_any) { // not implemented yet *unsupported = 1; goto done; } else { // a single vlan, a common case subint = &vlan_table->vlans[si->sub.eth. outer_vlan_id].single_tag_subint; } } else { // Two tags *flags = si->sub.eth.flags.exact_match ? SUBINT_CONFIG_MATCH_2_TAG : (SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG); if (si->sub.eth.flags.outer_vlan_id_any && si->sub.eth.flags.inner_vlan_id_any) { // not implemented yet *unsupported = 1; goto done; } if (si->sub.eth.flags.inner_vlan_id_any) { // a specific outer and "any" inner // don't need a qinq table for this subint = &vlan_table->vlans[si->sub.eth. outer_vlan_id].inner_any_subint; if (si->sub.eth.flags.exact_match) { *flags = SUBINT_CONFIG_MATCH_2_TAG; } else { *flags = SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG; } } else { // a specific outer + specifc innner vlan id, a common case // get the qinq table if (vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs == 0) { // Allocate a qinq table from the pool pool_get (em->qinq_pool, qinq_table); vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs = qinq_table - em->qinq_pool; } else { // Get ptr to existing qinq table qinq_table = vec_elt_at_index (em->qinq_pool, vlan_table->vlans[si->sub. eth.outer_vlan_id]. qinqs); } subint = &qinq_table->vlans[si->sub.eth.inner_vlan_id].subint; } } } done: return subint; } static clib_error_t * ethernet_sw_interface_up_down (vnet_main_t * vnm, u32 sw_if_index, u32 flags) { subint_config_t *subint; u32 dummy_flags; u32 dummy_unsup; clib_error_t *error = 0; // Find the config for this subinterface subint = ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags, &dummy_unsup); if (subint == 0) { // not implemented yet or not ethernet goto done; } subint->sw_if_index = ((flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ? sw_if_index : ~0); done: return error; } VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION (ethernet_sw_interface_up_down); #ifndef CLIB_MARCH_VARIANT // Set the L2/L3 mode for the subinterface void ethernet_sw_interface_set_l2_mode (vnet_main_t * vnm, u32 sw_if_index, u32 l2) { subint_config_t *subint; u32 dummy_flags; u32 dummy_unsup; int is_port; vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, sw_if_index); is_port = !(sw->type == VNET_SW_INTERFACE_TYPE_SUB); // Find the config for this subinterface subint = ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags, &dummy_unsup); if (subint == 0) { // unimplemented or not ethernet goto done; } // Double check that the config we found is for our interface (or the interface is down) ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0)); if (l2) { subint->flags |= SUBINT_CONFIG_L2; if (is_port) subint->flags |= SUBINT_CONFIG_MATCH_0_TAG | SUBINT_CONFIG_MATCH_1_TAG | SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG; } else { subint->flags &= ~SUBINT_CONFIG_L2; if (is_port) subint->flags &= ~(SUBINT_CONFIG_MATCH_1_TAG | SUBINT_CONFIG_MATCH_2_TAG | SUBINT_CONFIG_MATCH_3_TAG); } done: return; } /* * Set the L2/L3 mode for the subinterface regardless of port */ void ethernet_sw_interface_set_l2_mode_noport (vnet_main_t * vnm, u32 sw_if_index, u32 l2) { subint_config_t *subint; u32 dummy_flags; u32 dummy_unsup; /* Find the config for this subinterface */ subint = ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags, &dummy_unsup); if (subint == 0) { /* unimplemented or not ethernet */ goto done; } /* * Double check that the config we found is for our interface (or the * interface is down) */ ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0)); if (l2) { subint->flags |= SUBINT_CONFIG_L2; } else { subint->flags &= ~SUBINT_CONFIG_L2; } done: return; } #endif static clib_error_t * ethernet_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_create) { clib_error_t *error = 0; subint_config_t *subint; u32 match_flags; u32 unsupported = 0; // Find the config for this subinterface subint = ethernet_sw_interface_get_config (vnm, sw_if_index, &match_flags, &unsupported); if (subint == 0) { // not implemented yet or not ethernet if (unsupported) { // this is the NYI case error = clib_error_return (0, "not implemented yet"); } goto done; } if (!is_create) { subint->flags = 0; return error; } // Initialize the subint if (subint->flags & SUBINT_CONFIG_VALID) { // Error vlan already in use error = clib_error_return (0, "vlan is already in use"); } else { // Note that config is L3 by default subint->flags = SUBINT_CONFIG_VALID | match_flags; subint->sw_if_index = ~0; // because interfaces are initially down } done: return error; } VNET_SW_INTERFACE_ADD_DEL_FUNCTION (ethernet_sw_interface_add_del); static char *ethernet_error_strings[] = { #define ethernet_error(n,c,s) s, #include "error.def" #undef ethernet_error }; /* *INDENT-OFF* */ VLIB_REGISTER_NODE (ethernet_input_node) = { .name = "ethernet-input", /* Takes a vector of packets. */ .vector_size = sizeof (u32), .scalar_size = sizeof (ethernet_input_frame_t), .n_errors = ETHERNET_N_ERROR, .error_strings = ethernet_error_strings, .n_next_nodes = ETHERNET_INPUT_N_NEXT, .next_nodes = { #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n, foreach_ethernet_input_next #undef _ }, .format_buffer = format_ethernet_header_with_length, .format_trace = format_ethernet_input_trace, .unformat_buffer = unformat_ethernet_header, }; VLIB_REGISTER_NODE (ethernet_input_type_node) = { .name = "ethernet-input-type", /* Takes a vector of packets. */ .vector_size = sizeof (u32), .n_next_nodes = ETHERNET_INPUT_N_NEXT, .next_nodes = { #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n, foreach_ethernet_input_next #undef _ }, }; VLIB_REGISTER_NODE (ethernet_input_not_l2_node) = { .name = "ethernet-input-not-l2", /* Takes a vector of packets. */ .vector_size = sizeof (u32), .n_next_nodes = ETHERNET_INPUT_N_NEXT, .next_nodes = { #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n, foreach_ethernet_input_next #undef _ }, }; /* *INDENT-ON* */ #ifndef CLIB_MARCH_VARIANT void ethernet_set_rx_redirect (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 enable) { // Insure all packets go to ethernet-input (i.e. untagged ipv4 packets // don't go directly to ip4-input) vnet_hw_interface_rx_redirect_to_node (vnm, hi->hw_if_index, enable ? ethernet_input_node.index : ~0); } /* * Initialization and registration for the next_by_ethernet structure */ clib_error_t * next_by_ethertype_init (next_by_ethertype_t * l3_next) { l3_next->input_next_by_type = sparse_vec_new ( /* elt bytes */ sizeof (l3_next->input_next_by_type[0]), /* bits in index */ BITS (((ethernet_header_t *) 0)->type)); vec_validate (l3_next->sparse_index_by_input_next_index, ETHERNET_INPUT_NEXT_DROP); vec_validate (l3_next->sparse_index_by_input_next_index, ETHERNET_INPUT_NEXT_PUNT); l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_DROP] = SPARSE_VEC_INVALID_INDEX; l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_PUNT] = SPARSE_VEC_INVALID_INDEX; /* * Make sure we don't wipe out an ethernet registration by mistake * Can happen if init function ordering constraints are missing. */ if (CLIB_DEBUG > 0) { ethernet_main_t *em = ðernet_main; ASSERT (em->next_by_ethertype_register_called == 0); } return 0; } // Add an ethertype -> next index mapping to the structure clib_error_t * next_by_ethertype_register (next_by_ethertype_t * l3_next, u32 ethertype, u32 next_index) { u32 i; u16 *n; ethernet_main_t *em = ðernet_main; if (CLIB_DEBUG > 0) { ethernet_main_t *em = ðernet_main; em->next_by_ethertype_register_called = 1; } /* Setup ethernet type -> next index sparse vector mapping. */ n = sparse_vec_validate (l3_next->input_next_by_type, ethertype); n[0] = next_index; /* Rebuild next index -> sparse index inverse mapping when sparse vector is updated. */ vec_validate (l3_next->sparse_index_by_input_next_index, next_index); for (i = 1; i < vec_len (l3_next->input_next_by_type); i++) l3_next-> sparse_index_by_input_next_index[l3_next->input_next_by_type[i]] = i; // do not allow the cached next index's to be updated if L3 // redirect is enabled, as it will have overwritten them if (!em->redirect_l3) { // Cache common ethertypes directly if (ethertype == ETHERNET_TYPE_IP4) { l3_next->input_next_ip4 = next_index; } else if (ethertype == ETHERNET_TYPE_IP6) { l3_next->input_next_ip6 = next_index; } else if (ethertype == ETHERNET_TYPE_MPLS) { l3_next->input_next_mpls = next_index; } } return 0; } static clib_error_t * ethernet_input_init (vlib_main_t * vm) { ethernet_main_t *em = ðernet_main; __attribute__ ((unused)) vlan_table_t *invalid_vlan_table; __attribute__ ((unused)) qinq_table_t *invalid_qinq_table; ethernet_setup_node (vm, ethernet_input_node.index); ethernet_setup_node (vm, ethernet_input_type_node.index); ethernet_setup_node (vm, ethernet_input_not_l2_node.index); next_by_ethertype_init (&em->l3_next); // Initialize pools and vector for vlan parsing vec_validate (em->main_intfs, 10); // 10 main interfaces pool_alloc (em->vlan_pool, 10); pool_alloc (em->qinq_pool, 1); // The first vlan pool will always be reserved for an invalid table pool_get (em->vlan_pool, invalid_vlan_table); // first id = 0 // The first qinq pool will always be reserved for an invalid table pool_get (em->qinq_pool, invalid_qinq_table); // first id = 0 return 0; } VLIB_INIT_FUNCTION (ethernet_input_init); void ethernet_register_input_type (vlib_main_t * vm, ethernet_type_t type, u32 node_index) { ethernet_main_t *em = ðernet_main; ethernet_type_info_t *ti; u32 i; { clib_error_t *error = vlib_call_init_function (vm, ethernet_init); if (error) clib_error_report (error); } ti = ethernet_get_type_info (em, type); ti->node_index = node_index; ti->next_index = vlib_node_add_next (vm, ethernet_input_node.index, node_index); i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index); ASSERT (i == ti->next_index); i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index); ASSERT (i == ti->next_index); // Add the L3 node for this ethertype to the next nodes structure next_by_ethertype_register (&em->l3_next, type, ti->next_index); // Call the registration functions for other nodes that want a mapping l2bvi_register_input_type (vm, type, node_index); } void ethernet_register_l2_input (vlib_main_t * vm, u32 node_index) { ethernet_main_t *em = ðernet_main; u32 i; em->l2_next = vlib_node_add_next (vm, ethernet_input_node.index, node_index); /* * Even if we never use these arcs, we have to align the next indices... */ i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index); ASSERT (i == em->l2_next); i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index); ASSERT (i == em->l2_next); } // Register a next node for L3 redirect, and enable L3 redirect void ethernet_register_l3_redirect (vlib_main_t * vm, u32 node_index) { ethernet_main_t *em = ðernet_main; u32 i; em->redirect_l3 = 1; em->redirect_l3_next = vlib_node_add_next (vm, ethernet_input_node.index, node_index); /* * Change the cached next nodes to the redirect node */ em->l3_next.input_next_ip4 = em->redirect_l3_next; em->l3_next.input_next_ip6 = em->redirect_l3_next; em->l3_next.input_next_mpls = em->redirect_l3_next; /* * Even if we never use these arcs, we have to align the next indices... */ i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index); ASSERT (i == em->redirect_l3_next); i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index); ASSERT (i == em->redirect_l3_next); } #endif /* * fd.io coding-style-patch-verification: ON * * Local Variables: * eval: (c-set-style "gnu") * End: */