From 943a4b0750a2bee04788cb153ca56fa07d5005d7 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Wed, 27 Oct 2021 15:17:47 -0700 Subject: interface: handoff refactor and optimizations - move existing ethernet hash functions to hash infra (no l4 awareness) - refactor code to use hash infra and add apis to request l4 aware hashing functions - hashing functions per interface - code cleanup Type: improvement Signed-off-by: Florin Coras Change-Id: Ia9f44db98d83f0f027aeb37718585a2e10ffd2c6 --- src/vnet/hash/crc32_handoff_eth.c | 341 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 341 insertions(+) create mode 100644 src/vnet/hash/crc32_handoff_eth.c (limited to 'src/vnet/hash/crc32_handoff_eth.c') diff --git a/src/vnet/hash/crc32_handoff_eth.c b/src/vnet/hash/crc32_handoff_eth.c new file mode 100644 index 00000000000..13fc12ce6ea --- /dev/null +++ b/src/vnet/hash/crc32_handoff_eth.c @@ -0,0 +1,341 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +static inline u64 +ipv4_get_key (ip4_header_t * ip) +{ + u64 hash_key; + + hash_key = *((u64 *) (&ip->address_pair)) ^ ip->protocol; + + return hash_key; +} + +static inline u64 +ipv6_get_key (ip6_header_t * ip) +{ + u64 hash_key; + + hash_key = ip->src_address.as_u64[0] ^ + rotate_left (ip->src_address.as_u64[1], 13) ^ + rotate_left (ip->dst_address.as_u64[0], 26) ^ + rotate_left (ip->dst_address.as_u64[1], 39) ^ ip->protocol; + + return hash_key; +} + +#define MPLS_BOTTOM_OF_STACK_BIT_MASK 0x00000100U +#define MPLS_LABEL_MASK 0xFFFFF000U + +static inline u64 +mpls_get_key (mpls_unicast_header_t * m) +{ + u64 hash_key; + u8 ip_ver; + + + /* find the bottom of the MPLS label stack. */ + if (PREDICT_TRUE (m->label_exp_s_ttl & + clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK))) + { + goto bottom_lbl_found; + } + m++; + + if (PREDICT_TRUE (m->label_exp_s_ttl & + clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK))) + { + goto bottom_lbl_found; + } + m++; + + if (m->label_exp_s_ttl & + clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)) + { + goto bottom_lbl_found; + } + m++; + + if (m->label_exp_s_ttl & + clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)) + { + goto bottom_lbl_found; + } + m++; + + if (m->label_exp_s_ttl & + clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)) + { + goto bottom_lbl_found; + } + + /* the bottom label was not found - use the last label */ + hash_key = m->label_exp_s_ttl & clib_net_to_host_u32 (MPLS_LABEL_MASK); + + return hash_key; + +bottom_lbl_found: + m++; + ip_ver = (*((u8 *) m) >> 4); + + /* find out if it is IPV4 or IPV6 header */ + if (PREDICT_TRUE (ip_ver == 4)) + { + hash_key = ipv4_get_key ((ip4_header_t *) m); + } + else if (PREDICT_TRUE (ip_ver == 6)) + { + hash_key = ipv6_get_key ((ip6_header_t *) m); + } + else + { + /* use the bottom label */ + hash_key = + (m - 1)->label_exp_s_ttl & clib_net_to_host_u32 (MPLS_LABEL_MASK); + } + + return hash_key; + +} + +static inline u64 +eth_get_sym_key (ethernet_header_t * h0) +{ + u64 hash_key; + + if (PREDICT_TRUE (h0->type) == clib_host_to_net_u16 (ETHERNET_TYPE_IP4)) + { + ip4_header_t *ip = (ip4_header_t *) (h0 + 1); + hash_key = + (u64) (ip->src_address.as_u32 ^ + ip->dst_address.as_u32 ^ ip->protocol); + } + else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) + { + ip6_header_t *ip = (ip6_header_t *) (h0 + 1); + hash_key = (u64) (ip->src_address.as_u64[0] ^ + ip->src_address.as_u64[1] ^ + ip->dst_address.as_u64[0] ^ + ip->dst_address.as_u64[1] ^ ip->protocol); + } + else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) + { + hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1)); + } + else + if (PREDICT_FALSE + ((h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) + || (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD)))) + { + ethernet_vlan_header_t *outer = (ethernet_vlan_header_t *) (h0 + 1); + + outer = (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) ? + outer + 1 : outer; + if (PREDICT_TRUE (outer->type) == + clib_host_to_net_u16 (ETHERNET_TYPE_IP4)) + { + ip4_header_t *ip = (ip4_header_t *) (outer + 1); + hash_key = + (u64) (ip->src_address.as_u32 ^ + ip->dst_address.as_u32 ^ ip->protocol); + } + else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) + { + ip6_header_t *ip = (ip6_header_t *) (outer + 1); + hash_key = + (u64) (ip->src_address.as_u64[0] ^ ip->src_address.as_u64[1] ^ + ip->dst_address.as_u64[0] ^ + ip->dst_address.as_u64[1] ^ ip->protocol); + } + else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) + { + hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1)); + } + else + { + hash_key = outer->type; + } + } + else + { + hash_key = 0; + } + + return hash_key; +} + +static inline u64 +eth_get_key (ethernet_header_t * h0) +{ + u64 hash_key; + + if (PREDICT_TRUE (h0->type) == clib_host_to_net_u16 (ETHERNET_TYPE_IP4)) + { + hash_key = ipv4_get_key ((ip4_header_t *) (h0 + 1)); + } + else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) + { + hash_key = ipv6_get_key ((ip6_header_t *) (h0 + 1)); + } + else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) + { + hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1)); + } + else if ((h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) || + (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD))) + { + ethernet_vlan_header_t *outer = (ethernet_vlan_header_t *) (h0 + 1); + + outer = (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) ? + outer + 1 : outer; + if (PREDICT_TRUE (outer->type) == + clib_host_to_net_u16 (ETHERNET_TYPE_IP4)) + { + hash_key = ipv4_get_key ((ip4_header_t *) (outer + 1)); + } + else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) + { + hash_key = ipv6_get_key ((ip6_header_t *) (outer + 1)); + } + else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) + { + hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1)); + } + else + { + hash_key = outer->type; + } + } + else + { + hash_key = 0; + } + + return hash_key; +} + +void +handoff_eth_crc32c_func (void **p, u32 *hash, u32 n_packets) +{ + u32 n_left_from = n_packets; + + while (n_left_from >= 8) + { + u64 key[4] = {}; + + clib_prefetch_load (p[4]); + clib_prefetch_load (p[5]); + clib_prefetch_load (p[6]); + clib_prefetch_load (p[7]); + + key[0] = eth_get_key ((ethernet_header_t *) p[0]); + key[1] = eth_get_key ((ethernet_header_t *) p[1]); + key[2] = eth_get_key ((ethernet_header_t *) p[2]); + key[3] = eth_get_key ((ethernet_header_t *) p[3]); + + hash[0] = clib_crc32c ((u8 *) &key[0], sizeof (key[0])); + hash[1] = clib_crc32c ((u8 *) &key[1], sizeof (key[1])); + hash[2] = clib_crc32c ((u8 *) &key[2], sizeof (key[2])); + hash[3] = clib_crc32c ((u8 *) &key[3], sizeof (key[3])); + + hash += 4; + n_left_from -= 4; + p += 4; + } + + while (n_left_from > 0) + { + u64 key; + + key = eth_get_key ((ethernet_header_t *) p[0]); + hash[0] = clib_crc32c ((u8 *) &key, sizeof (key)); + + hash += 1; + n_left_from -= 1; + p += 1; + } +} + +VNET_REGISTER_HASH_FUNCTION (handoff_eth_crc32c, static) = { + .name = "handoff-eth-crc32c", + .description = "Ethernet/IPv4/IPv6/MPLS headers", + .priority = 2, + .function[VNET_HASH_FN_TYPE_ETHERNET] = handoff_eth_crc32c_func, +}; + +void +handoff_eth_sym_crc32c_func (void **p, u32 *hash, u32 n_packets) +{ + u32 n_left_from = n_packets; + + while (n_left_from >= 8) + { + u64 key[4] = {}; + + clib_prefetch_load (p[4]); + clib_prefetch_load (p[5]); + clib_prefetch_load (p[6]); + clib_prefetch_load (p[7]); + + key[0] = eth_get_sym_key ((ethernet_header_t *) p[0]); + key[1] = eth_get_sym_key ((ethernet_header_t *) p[1]); + key[2] = eth_get_sym_key ((ethernet_header_t *) p[2]); + key[3] = eth_get_sym_key ((ethernet_header_t *) p[3]); + + hash[0] = clib_crc32c ((u8 *) &key[0], sizeof (key[0])); + hash[1] = clib_crc32c ((u8 *) &key[1], sizeof (key[1])); + hash[2] = clib_crc32c ((u8 *) &key[2], sizeof (key[2])); + hash[3] = clib_crc32c ((u8 *) &key[3], sizeof (key[3])); + + hash += 4; + n_left_from -= 4; + p += 4; + } + + while (n_left_from > 0) + { + u64 key; + + key = eth_get_sym_key ((ethernet_header_t *) p[0]); + hash[0] = clib_crc32c ((u8 *) &key, sizeof (key)); + + hash += 1; + n_left_from -= 1; + p += 1; + } +} + +VNET_REGISTER_HASH_FUNCTION (handoff_eth_sym_crc32c, static) = { + .name = "handoff-eth-sym-crc32c", + .description = "Ethernet/IPv4/IPv6/MPLS headers Symmetric", + .priority = 1, + .function[VNET_HASH_FN_TYPE_ETHERNET] = handoff_eth_sym_crc32c_func, +}; + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ -- cgit 1.2.3-korg