diff options
Diffstat (limited to 'src/vnet/hash')
| -rw-r--r-- | src/vnet/hash/FEATURE.yaml | 9 | ||||
| -rw-r--r-- | src/vnet/hash/cli.c | 33 | ||||
| -rw-r--r-- | src/vnet/hash/crc32_5tuple.c | 168 | ||||
| -rw-r--r-- | src/vnet/hash/handoff_eth.c | 352 | ||||
| -rw-r--r-- | src/vnet/hash/hash.c | 76 | ||||
| -rw-r--r-- | src/vnet/hash/hash.h | 59 | ||||
| -rw-r--r-- | src/vnet/hash/hash.rst | 90 | ||||
| -rw-r--r-- | src/vnet/hash/hash_eth.c | 326 |
8 files changed, 1113 insertions, 0 deletions
diff --git a/src/vnet/hash/FEATURE.yaml b/src/vnet/hash/FEATURE.yaml new file mode 100644 index 00000000000..d5b9a069c27 --- /dev/null +++ b/src/vnet/hash/FEATURE.yaml @@ -0,0 +1,9 @@ +--- +name: Hash infrastructure +maintainer: Mohsin Kazmi <mohsin.kazmi14@gmail.com>, Damjan Marion <damarion@cisco.com> +features: + - Ethernet + - IP +description: "Hash infrastructure" +state: development +properties: [CLI] diff --git a/src/vnet/hash/cli.c b/src/vnet/hash/cli.c new file mode 100644 index 00000000000..47d33b9872e --- /dev/null +++ b/src/vnet/hash/cli.c @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2021 Cisco Systems, Inc. + */ + +#include <vnet/vnet.h> +#include <vnet/hash/hash.h> + +static clib_error_t * +show_hash (vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd) +{ + clib_error_t *error = 0; + vnet_hash_main_t *hm = &vnet_hash_main; + vnet_hash_function_registration_t *hash; + + hash = hm->hash_registrations; + + vlib_cli_output (vm, "%-25s%-8s%s", "Name", "Prio", "Description"); + while (hash) + { + vlib_cli_output (vm, "%-25s%-8u%s", hash->name, hash->priority, + hash->description); + hash = hash->next; + } + + return (error); +} + +VLIB_CLI_COMMAND (cmd_show_hash, static) = { + .path = "show hash", + .short_help = "show hash", + .function = show_hash, +}; diff --git a/src/vnet/hash/crc32_5tuple.c b/src/vnet/hash/crc32_5tuple.c new file mode 100644 index 00000000000..2cdb19440c6 --- /dev/null +++ b/src/vnet/hash/crc32_5tuple.c @@ -0,0 +1,168 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2021 Cisco Systems, Inc. + */ + +#include <vnet/vnet.h> +#include <vnet/ethernet/ethernet.h> +#include <vnet/ip/ip4_packet.h> +#include <vnet/ip/ip6_packet.h> +#include <vnet/hash/hash.h> +#include <vppinfra/crc32.h> + +#ifdef clib_crc32c_uses_intrinsics + +static const u8 l4_mask_bits[256] = { + [IP_PROTOCOL_ICMP] = 16, [IP_PROTOCOL_IGMP] = 8, + [IP_PROTOCOL_TCP] = 32, [IP_PROTOCOL_UDP] = 32, + [IP_PROTOCOL_IPSEC_ESP] = 32, [IP_PROTOCOL_IPSEC_AH] = 32, + [IP_PROTOCOL_ICMP6] = 16, +}; + +static_always_inline u32 +compute_ip6_key (ip6_header_t *ip) +{ + u32 hash = 0, l4hdr; + u8 pr; + /* dst + src ip as u64 */ + hash = clib_crc32c_u64 (hash, *(u64u *) ((u8 *) ip + 8)); + hash = clib_crc32c_u64 (hash, *(u64u *) ((u8 *) ip + 16)); + hash = clib_crc32c_u64 (hash, *(u64u *) ((u8 *) ip + 24)); + hash = clib_crc32c_u64 (hash, *(u64u *) ((u8 *) ip + 32)); + pr = ip->protocol; + l4hdr = *(u32 *) ip6_next_header (ip) & pow2_mask (l4_mask_bits[pr]); + /* protocol + l4 hdr */ + return clib_crc32c_u64 (hash, ((u64) pr << 32) | l4hdr); +} + +static_always_inline u32 +compute_ip4_key (ip4_header_t *ip) +{ + u32 hash = 0, l4hdr; + u8 pr; + /* dst + src ip as u64 */ + hash = clib_crc32c_u64 (0, *(u64 *) ((u8 *) ip + 12)); + pr = ip->protocol; + l4hdr = *(u32 *) ip4_next_header (ip) & pow2_mask (l4_mask_bits[pr]); + /* protocol + l4 hdr */ + return clib_crc32c_u64 (hash, ((u64) pr << 32) | l4hdr); +} +static_always_inline u32 +compute_ip_key (void *p) +{ + if ((((u8 *) p)[0] & 0xf0) == 0x40) + return compute_ip4_key (p); + else if ((((u8 *) p)[0] & 0xf0) == 0x60) + return compute_ip6_key (p); + return 0; +} + +void +vnet_crc32c_5tuple_ip_func (void **p, u32 *hash, u32 n_packets) +{ + u32 n_left_from = n_packets; + + while (n_left_from >= 8) + { + clib_prefetch_load (p[4]); + clib_prefetch_load (p[5]); + clib_prefetch_load (p[6]); + clib_prefetch_load (p[7]); + + hash[0] = compute_ip_key (p[0]); + hash[1] = compute_ip_key (p[1]); + hash[2] = compute_ip_key (p[2]); + hash[3] = compute_ip_key (p[3]); + + hash += 4; + n_left_from -= 4; + p += 4; + } + + while (n_left_from > 0) + { + hash[0] = compute_ip_key (p[0]); + + hash += 1; + n_left_from -= 1; + p += 1; + } +} + +static_always_inline u32 +compute_ethernet_key (void *p) +{ + u16 ethertype = 0, l2hdr_sz = 0; + + ethernet_header_t *eh = (ethernet_header_t *) p; + ethertype = clib_net_to_host_u16 (eh->type); + l2hdr_sz = sizeof (ethernet_header_t); + + if (ethernet_frame_is_tagged (ethertype)) + { + ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1); + + ethertype = clib_net_to_host_u16 (vlan->type); + l2hdr_sz += sizeof (*vlan); + while (ethernet_frame_is_tagged (ethertype)) + { + vlan++; + ethertype = clib_net_to_host_u16 (vlan->type); + l2hdr_sz += sizeof (*vlan); + } + } + + if (ethertype == ETHERNET_TYPE_IP4) + { + ip4_header_t *ip4 = (ip4_header_t *) (p + l2hdr_sz); + return compute_ip4_key (ip4); + } + else if (ethertype == ETHERNET_TYPE_IP6) + { + ip6_header_t *ip6 = (ip6_header_t *) (p + l2hdr_sz); + return compute_ip6_key (ip6); + } + return 0; +} + +void +vnet_crc32c_5tuple_ethernet_func (void **p, u32 *hash, u32 n_packets) +{ + u32 n_left_from = n_packets; + + while (n_left_from >= 8) + { + clib_prefetch_load (p[4]); + clib_prefetch_load (p[5]); + clib_prefetch_load (p[6]); + clib_prefetch_load (p[7]); + + hash[0] = compute_ethernet_key (p[0]); + hash[1] = compute_ethernet_key (p[1]); + hash[2] = compute_ethernet_key (p[2]); + hash[3] = compute_ethernet_key (p[3]); + + hash += 4; + n_left_from -= 4; + p += 4; + } + + while (n_left_from > 0) + { + hash[0] = compute_ethernet_key (p[0]); + + hash += 1; + n_left_from -= 1; + p += 1; + } +} + +VNET_REGISTER_HASH_FUNCTION (crc32c_5tuple, static) = { + .name = "crc32c-5tuple", + .description = "IPv4/IPv6 header and TCP/UDP ports", + .priority = 50, + .function[VNET_HASH_FN_TYPE_ETHERNET] = vnet_crc32c_5tuple_ethernet_func, + .function[VNET_HASH_FN_TYPE_IP] = vnet_crc32c_5tuple_ip_func, +}; + +#endif diff --git a/src/vnet/hash/handoff_eth.c b/src/vnet/hash/handoff_eth.c new file mode 100644 index 00000000000..dc8db2ac413 --- /dev/null +++ b/src/vnet/hash/handoff_eth.c @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <vlib/vlib.h> +#include <vnet/ethernet/ethernet.h> +#include <vnet/hash/hash.h> +#include <vnet/ip/ip4_packet.h> +#include <vnet/ip/ip6_packet.h> +#include <vnet/mpls/packet.h> +#include <vppinfra/crc32.h> +#include <vppinfra/xxhash.h> + +always_inline u32 +ho_hash (u64 key) +{ +#ifdef clib_crc32c_uses_intrinsics + return clib_crc32c ((u8 *) &key, sizeof (key)); +#else + return clib_xxhash (key); +#endif +} + +static inline u64 +ipv4_get_key (ip4_header_t * ip) +{ + u64 hash_key; + + hash_key = *((u64 *) (&ip->address_pair)) ^ ip->protocol; + + return hash_key; +} + +static inline u64 +ipv6_get_key (ip6_header_t * ip) +{ + u64 hash_key; + + hash_key = ip->src_address.as_u64[0] ^ + rotate_left (ip->src_address.as_u64[1], 13) ^ + rotate_left (ip->dst_address.as_u64[0], 26) ^ + rotate_left (ip->dst_address.as_u64[1], 39) ^ ip->protocol; + + return hash_key; +} + +#define MPLS_BOTTOM_OF_STACK_BIT_MASK 0x00000100U +#define MPLS_LABEL_MASK 0xFFFFF000U + +static inline u64 +mpls_get_key (mpls_unicast_header_t * m) +{ + u64 hash_key; + u8 ip_ver; + + + /* find the bottom of the MPLS label stack. */ + if (PREDICT_TRUE (m->label_exp_s_ttl & + clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK))) + { + goto bottom_lbl_found; + } + m++; + + if (PREDICT_TRUE (m->label_exp_s_ttl & + clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK))) + { + goto bottom_lbl_found; + } + m++; + + if (m->label_exp_s_ttl & + clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)) + { + goto bottom_lbl_found; + } + m++; + + if (m->label_exp_s_ttl & + clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)) + { + goto bottom_lbl_found; + } + m++; + + if (m->label_exp_s_ttl & + clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)) + { + goto bottom_lbl_found; + } + + /* the bottom label was not found - use the last label */ + hash_key = m->label_exp_s_ttl & clib_net_to_host_u32 (MPLS_LABEL_MASK); + + return hash_key; + +bottom_lbl_found: + m++; + ip_ver = (*((u8 *) m) >> 4); + + /* find out if it is IPV4 or IPV6 header */ + if (PREDICT_TRUE (ip_ver == 4)) + { + hash_key = ipv4_get_key ((ip4_header_t *) m); + } + else if (PREDICT_TRUE (ip_ver == 6)) + { + hash_key = ipv6_get_key ((ip6_header_t *) m); + } + else + { + /* use the bottom label */ + hash_key = + (m - 1)->label_exp_s_ttl & clib_net_to_host_u32 (MPLS_LABEL_MASK); + } + + return hash_key; + +} + +static inline u64 +eth_get_sym_key (ethernet_header_t * h0) +{ + u64 hash_key; + + if (PREDICT_TRUE (h0->type) == clib_host_to_net_u16 (ETHERNET_TYPE_IP4)) + { + ip4_header_t *ip = (ip4_header_t *) (h0 + 1); + hash_key = + (u64) (ip->src_address.as_u32 ^ + ip->dst_address.as_u32 ^ ip->protocol); + } + else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) + { + ip6_header_t *ip = (ip6_header_t *) (h0 + 1); + hash_key = (u64) (ip->src_address.as_u64[0] ^ + ip->src_address.as_u64[1] ^ + ip->dst_address.as_u64[0] ^ + ip->dst_address.as_u64[1] ^ ip->protocol); + } + else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) + { + hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1)); + } + else + if (PREDICT_FALSE + ((h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) + || (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD)))) + { + ethernet_vlan_header_t *outer = (ethernet_vlan_header_t *) (h0 + 1); + + outer = (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) ? + outer + 1 : outer; + if (PREDICT_TRUE (outer->type) == + clib_host_to_net_u16 (ETHERNET_TYPE_IP4)) + { + ip4_header_t *ip = (ip4_header_t *) (outer + 1); + hash_key = + (u64) (ip->src_address.as_u32 ^ + ip->dst_address.as_u32 ^ ip->protocol); + } + else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) + { + ip6_header_t *ip = (ip6_header_t *) (outer + 1); + hash_key = + (u64) (ip->src_address.as_u64[0] ^ ip->src_address.as_u64[1] ^ + ip->dst_address.as_u64[0] ^ + ip->dst_address.as_u64[1] ^ ip->protocol); + } + else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) + { + hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1)); + } + else + { + hash_key = outer->type; + } + } + else + { + hash_key = 0; + } + + return hash_key; +} + +static inline u64 +eth_get_key (ethernet_header_t * h0) +{ + u64 hash_key; + + if (PREDICT_TRUE (h0->type) == clib_host_to_net_u16 (ETHERNET_TYPE_IP4)) + { + hash_key = ipv4_get_key ((ip4_header_t *) (h0 + 1)); + } + else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) + { + hash_key = ipv6_get_key ((ip6_header_t *) (h0 + 1)); + } + else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) + { + hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1)); + } + else if ((h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) || + (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD))) + { + ethernet_vlan_header_t *outer = (ethernet_vlan_header_t *) (h0 + 1); + + outer = (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) ? + outer + 1 : outer; + if (PREDICT_TRUE (outer->type) == + clib_host_to_net_u16 (ETHERNET_TYPE_IP4)) + { + hash_key = ipv4_get_key ((ip4_header_t *) (outer + 1)); + } + else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) + { + hash_key = ipv6_get_key ((ip6_header_t *) (outer + 1)); + } + else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS)) + { + hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1)); + } + else + { + hash_key = outer->type; + } + } + else + { + hash_key = 0; + } + + return hash_key; +} + +void +handoff_eth_func (void **p, u32 *hash, u32 n_packets) +{ + u32 n_left_from = n_packets; + + while (n_left_from >= 8) + { + u64 key[4] = {}; + + clib_prefetch_load (p[4]); + clib_prefetch_load (p[5]); + clib_prefetch_load (p[6]); + clib_prefetch_load (p[7]); + + key[0] = eth_get_key ((ethernet_header_t *) p[0]); + key[1] = eth_get_key ((ethernet_header_t *) p[1]); + key[2] = eth_get_key ((ethernet_header_t *) p[2]); + key[3] = eth_get_key ((ethernet_header_t *) p[3]); + + hash[0] = ho_hash (key[0]); + hash[1] = ho_hash (key[1]); + hash[2] = ho_hash (key[2]); + hash[3] = ho_hash (key[3]); + + hash += 4; + n_left_from -= 4; + p += 4; + } + + while (n_left_from > 0) + { + u64 key; + + key = eth_get_key ((ethernet_header_t *) p[0]); + hash[0] = ho_hash (key); + + hash += 1; + n_left_from -= 1; + p += 1; + } +} + +VNET_REGISTER_HASH_FUNCTION (handoff_eth, static) = { + .name = "handoff-eth", + .description = "Ethernet/IPv4/IPv6/MPLS headers", + .priority = 2, + .function[VNET_HASH_FN_TYPE_ETHERNET] = handoff_eth_func, +}; + +void +handoff_eth_sym_func (void **p, u32 *hash, u32 n_packets) +{ + u32 n_left_from = n_packets; + + while (n_left_from >= 8) + { + u64 key[4] = {}; + + clib_prefetch_load (p[4]); + clib_prefetch_load (p[5]); + clib_prefetch_load (p[6]); + clib_prefetch_load (p[7]); + + key[0] = eth_get_sym_key ((ethernet_header_t *) p[0]); + key[1] = eth_get_sym_key ((ethernet_header_t *) p[1]); + key[2] = eth_get_sym_key ((ethernet_header_t *) p[2]); + key[3] = eth_get_sym_key ((ethernet_header_t *) p[3]); + + hash[0] = ho_hash (key[0]); + hash[1] = ho_hash (key[1]); + hash[2] = ho_hash (key[2]); + hash[3] = ho_hash (key[3]); + + hash += 4; + n_left_from -= 4; + p += 4; + } + + while (n_left_from > 0) + { + u64 key; + + key = eth_get_sym_key ((ethernet_header_t *) p[0]); + hash[0] = ho_hash (key); + + hash += 1; + n_left_from -= 1; + p += 1; + } +} + +VNET_REGISTER_HASH_FUNCTION (handoff_eth_sym, static) = { + .name = "handoff-eth-sym", + .description = "Ethernet/IPv4/IPv6/MPLS headers Symmetric", + .priority = 1, + .function[VNET_HASH_FN_TYPE_ETHERNET] = handoff_eth_sym_func, +}; + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vnet/hash/hash.c b/src/vnet/hash/hash.c new file mode 100644 index 00000000000..31693c9889b --- /dev/null +++ b/src/vnet/hash/hash.c @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2021 Cisco Systems, Inc. + */ + +#include <vlib/vlib.h> +#include <vnet/vnet.h> +#include <vnet/interface.h> +#include <vnet/hash/hash.h> + +vnet_hash_main_t vnet_hash_main; + +u8 * +format_vnet_hash (u8 *s, va_list *args) +{ + vnet_hash_function_registration_t *hash = + va_arg (*args, vnet_hash_function_registration_t *); + + s = format (s, "[name: %s ", hash->name); + s = format (s, "priority: %u ", hash->priority); + s = format (s, "description: %s]", hash->description); + return s; +} + +/** + * select hash func with highest priority + */ +vnet_hash_fn_t +vnet_hash_default_function (vnet_hash_fn_type_t ftype) +{ + vnet_hash_function_registration_t *hash = vnet_hash_main.hash_registrations; + vnet_hash_function_registration_t *tmp_hash = hash; + while (hash) + { + if (hash->priority > tmp_hash->priority) + tmp_hash = hash; + hash = hash->next; + } + return tmp_hash->function[ftype]; +} + +vnet_hash_fn_t +vnet_hash_function_from_name (const char *name, vnet_hash_fn_type_t ftype) +{ + vnet_hash_function_registration_t *hash = vnet_hash_main.hash_registrations; + while (hash) + { + if (strcmp (hash->name, name) == 0) + break; + hash = hash->next; + } + if (!hash) + return (0); + return hash->function[ftype]; +} + +vnet_hash_function_registration_t * +vnet_hash_function_from_func (vnet_hash_fn_t fn, vnet_hash_fn_type_t ftype) +{ + vnet_hash_function_registration_t *hash = vnet_hash_main.hash_registrations; + while (hash) + { + if (hash->function[ftype] == fn) + break; + hash = hash->next; + } + return hash; +} + +static clib_error_t * +vnet_hash_init (vlib_main_t *vm) +{ + return (0); +} + +VLIB_INIT_FUNCTION (vnet_hash_init); diff --git a/src/vnet/hash/hash.h b/src/vnet/hash/hash.h new file mode 100644 index 00000000000..c1eb9475e28 --- /dev/null +++ b/src/vnet/hash/hash.h @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2021 Cisco Systems, Inc. + */ + +#ifndef __VNET_HASH_H__ +#define __VNET_HASH_H__ + +#include <vlib/vlib.h> + +#define foreach_vnet_hash_fn_types \ + _ (ETHERNET, 0, "hash-fn-ethernet") \ + _ (IP, 1, "hash-fn-ip") + +typedef enum +{ +#define _(f, n, s) VNET_HASH_FN_TYPE_##f, + foreach_vnet_hash_fn_types +#undef _ + VNET_HASH_FN_TYPE_N, +} vnet_hash_fn_type_t; + +typedef void (*vnet_hash_fn_t) (void **p, u32 *h, u32 n_packets); + +typedef struct vnet_hash_function_registration +{ + const char *name; + const char *description; + int priority; + vnet_hash_fn_t function[VNET_HASH_FN_TYPE_N]; + + struct vnet_hash_function_registration *next; +} vnet_hash_function_registration_t; + +typedef struct +{ + vnet_hash_function_registration_t *hash_registrations; +} vnet_hash_main_t; + +extern vnet_hash_main_t vnet_hash_main; + +#define VNET_REGISTER_HASH_FUNCTION(x, ...) \ + __VA_ARGS__ vnet_hash_function_registration_t __vnet_hash_function_##x; \ + static void __clib_constructor __vnet_hash_function_registration_##x (void) \ + { \ + vnet_hash_main_t *hm = &vnet_hash_main; \ + __vnet_hash_function_##x.next = hm->hash_registrations; \ + hm->hash_registrations = &__vnet_hash_function_##x; \ + } \ + __VA_ARGS__ vnet_hash_function_registration_t __vnet_hash_function_##x + +vnet_hash_fn_t vnet_hash_default_function (vnet_hash_fn_type_t ftype); +vnet_hash_fn_t vnet_hash_function_from_name (const char *name, + vnet_hash_fn_type_t ftype); +vnet_hash_function_registration_t * +vnet_hash_function_from_func (vnet_hash_fn_t fn, vnet_hash_fn_type_t ftype); +format_function_t format_vnet_hash; + +#endif diff --git a/src/vnet/hash/hash.rst b/src/vnet/hash/hash.rst new file mode 100644 index 00000000000..3db74e2f093 --- /dev/null +++ b/src/vnet/hash/hash.rst @@ -0,0 +1,90 @@ +.. _hash_doc: + +Hash Infra +========== + +Overview +________ + +Modern physical NICs uses packet flow hash for different purposes, i.e. Receive +Side Scaling, flow steering and interface bonding etc. NICs can also provide +packet flow hash prepended to data packet as metadata which can be used by +applications without recomputing the packet flow hash. + +As more and more services are deployed in virtualized environment, making use of +virtual interfaces to interconnect those services. + +The Hash Infrastructure +_______________________ + +VPP implements software based hashing functionality which can be used for different +purposes. It also provides users a centralized way to registry custom hash functions +based on traffic profile to be used in different vpp features i.e. Multi-TXQ, +software RSS or bonding driver. + +Data structures +^^^^^^^^^^^^^^^ + +Hashing infra provides two types of hashing functions: +``VNET_HASH_FN_TYPE_ETHERNET`` and ``VNET_HASH_FN_TYPE_IP`` for ethernet traffic and +IP traffic respectively. +Hashing infra provides uniform signature to the functions to be implemented: + +.. code:: c + + void (*vnet_hash_fn_t) (void **p, u32 *h, u32 n_packets); + +Here ``**p`` is the array of pointers pointing to the beginning of packet headers +(either ethernet or ip). +``*h`` is an empty array of size n_packets. On return, it will contain hashes. +``n_packets`` is the number of packets pass to this function. + +Custom hashing functions can be registered through ``VNET_REGISTER_HASH_FUNCTION``. +Users need to provide a name, description, priority and hashing functions for +registration. + +Default hashing function is selected based on the highest priority among the registered +hashing functions. + +.. code:: c + + typedef struct vnet_hash_function_registration + { + const char *name; + const char *description; + int priority; + vnet_hash_fn_t function[VNET_HASH_FN_TYPE_N]; + + struct vnet_hash_function_registration *next; + } vnet_hash_function_registration_t; + +For example, ``crc32c_5tuple`` provides two hashing functions: for IP traffic and for +ethernet traffic. It uses 5 tuples from the flow to compute the crc32 hash on it. + +.. code:: c + + void vnet_crc32c_5tuple_ip_func (void **p, u32 *hash, u32 n_packets); + void vnet_crc32c_5tuple_ethernet_func (void **p, u32 *hash, u32 n_packets); + + VNET_REGISTER_HASH_FUNCTION (crc32c_5tuple, static) = { + .name = "crc32c-5tuple", + .description = "IPv4/IPv6 header and TCP/UDP ports", + .priority = 50, + .function[VNET_HASH_FN_TYPE_ETHERNET] = vnet_crc32c_5tuple_ethernet_func, + .function[VNET_HASH_FN_TYPE_IP] = vnet_crc32c_5tuple_ip_func, + }; + + +Users can see all the registered hash functions along with priority and description. + +Hash API +^^^^^^^^ + +There is no Hash API at the moment. + +Hash CLI +^^^^^^^^ + +:: + + show hash diff --git a/src/vnet/hash/hash_eth.c b/src/vnet/hash/hash_eth.c new file mode 100644 index 00000000000..1ac8b66a1bc --- /dev/null +++ b/src/vnet/hash/hash_eth.c @@ -0,0 +1,326 @@ +/* + *------------------------------------------------------------------ + * Copyright (c) 2021 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ + +#define _GNU_SOURCE +#include <stdint.h> +#include <vlib/vlib.h> +#include <vlib/unix/unix.h> +#include <vnet/ethernet/ethernet.h> +#include <vnet/ip/ip4_packet.h> +#include <vnet/ip/ip6_packet.h> +#include <vnet/ip/ip6_hop_by_hop_packet.h> +#include <vnet/tcp/tcp_packet.h> +#include <vppinfra/lb_hash_hash.h> +#include <vnet/hash/hash.h> + +static_always_inline u16 * +locate_ethertype (ethernet_header_t *eth) +{ + u16 *ethertype_p; + ethernet_vlan_header_t *vlan; + + if (!ethernet_frame_is_tagged (clib_net_to_host_u16 (eth->type))) + { + ethertype_p = ð->type; + } + else + { + vlan = (void *) (eth + 1); + ethertype_p = &vlan->type; + if (*ethertype_p == ntohs (ETHERNET_TYPE_VLAN)) + { + vlan++; + ethertype_p = &vlan->type; + } + } + return ethertype_p; +} + +static void +hash_eth_l2 (void **p, u32 *hash, u32 n_packets) +{ + u32 n_left_from = n_packets; + + while (n_left_from >= 8) + { + ethernet_header_t *eth = *p; + u64 *dst = (u64 *) ð->dst_address[0]; + u64 a = clib_mem_unaligned (dst, u64); + u32 *src = (u32 *) ð->src_address[2]; + u32 b = clib_mem_unaligned (src, u32); + + clib_prefetch_load (p[4]); + clib_prefetch_load (p[5]); + clib_prefetch_load (p[6]); + clib_prefetch_load (p[7]); + + hash[0] = lb_hash_hash_2_tuples (a, b); + hash[1] = lb_hash_hash_2_tuples (a, b); + hash[2] = lb_hash_hash_2_tuples (a, b); + hash[3] = lb_hash_hash_2_tuples (a, b); + + hash += 4; + n_left_from -= 4; + p += 4; + } + + while (n_left_from > 0) + { + ethernet_header_t *eth = *p; + u64 *dst = (u64 *) ð->dst_address[0]; + u64 a = clib_mem_unaligned (dst, u64); + u32 *src = (u32 *) ð->src_address[2]; + u32 b = clib_mem_unaligned (src, u32); + + hash[0] = lb_hash_hash_2_tuples (a, b); + + hash += 1; + n_left_from -= 1; + p += 1; + } +} + +static_always_inline u32 +hash_eth_l23_inline (void **p) +{ + ethernet_header_t *eth = *p; + u8 ip_version; + ip4_header_t *ip4; + u16 ethertype, *ethertype_p; + u32 *mac1, *mac2, *mac3; + u32 hash; + + ethertype_p = locate_ethertype (eth); + ethertype = clib_mem_unaligned (ethertype_p, u16); + + if ((ethertype != htons (ETHERNET_TYPE_IP4)) && + (ethertype != htons (ETHERNET_TYPE_IP6))) + { + hash_eth_l2 (p, &hash, 1); + return hash; + } + + ip4 = (ip4_header_t *) (ethertype_p + 1); + ip_version = (ip4->ip_version_and_header_length >> 4); + + if (ip_version == 0x4) + { + u32 a; + + mac1 = (u32 *) ð->dst_address[0]; + mac2 = (u32 *) ð->dst_address[4]; + mac3 = (u32 *) ð->src_address[2]; + + a = clib_mem_unaligned (mac1, u32) ^ clib_mem_unaligned (mac2, u32) ^ + clib_mem_unaligned (mac3, u32); + hash = lb_hash_hash_2_tuples ( + clib_mem_unaligned (&ip4->address_pair, u64), a); + return hash; + } + + if (ip_version == 0x6) + { + u64 a; + ip6_header_t *ip6 = (ip6_header_t *) (eth + 1); + + mac1 = (u32 *) ð->dst_address[0]; + mac2 = (u32 *) ð->dst_address[4]; + mac3 = (u32 *) ð->src_address[2]; + + a = clib_mem_unaligned (mac1, u32) ^ clib_mem_unaligned (mac2, u32) ^ + clib_mem_unaligned (mac3, u32); + hash = lb_hash_hash ( + clib_mem_unaligned (&ip6->src_address.as_uword[0], uword), + clib_mem_unaligned (&ip6->src_address.as_uword[1], uword), + clib_mem_unaligned (&ip6->dst_address.as_uword[0], uword), + clib_mem_unaligned (&ip6->dst_address.as_uword[1], uword), a); + return hash; + } + + hash_eth_l2 (p, &hash, 1); + return hash; +} + +static void +hash_eth_l23 (void **p, u32 *hash, u32 n_packets) +{ + u32 n_left_from = n_packets; + + while (n_left_from >= 8) + { + clib_prefetch_load (p[4]); + clib_prefetch_load (p[5]); + clib_prefetch_load (p[6]); + clib_prefetch_load (p[7]); + + hash[0] = hash_eth_l23_inline (&p[0]); + hash[1] = hash_eth_l23_inline (&p[1]); + hash[2] = hash_eth_l23_inline (&p[2]); + hash[3] = hash_eth_l23_inline (&p[3]); + + hash += 4; + n_left_from -= 4; + p += 4; + } + + while (n_left_from > 0) + { + hash[0] = hash_eth_l23_inline (&p[0]); + + hash += 1; + n_left_from -= 1; + p += 1; + } +} + +static_always_inline u32 +hash_eth_l34_inline (void **p) +{ + ethernet_header_t *eth = *p; + u8 ip_version; + uword is_tcp_udp; + ip4_header_t *ip4; + u16 ethertype, *ethertype_p; + u32 hash; + + ethertype_p = locate_ethertype (eth); + ethertype = clib_mem_unaligned (ethertype_p, u16); + + if ((ethertype != htons (ETHERNET_TYPE_IP4)) && + (ethertype != htons (ETHERNET_TYPE_IP6))) + { + hash_eth_l2 (p, &hash, 1); + return hash; + } + + ip4 = (ip4_header_t *) (ethertype_p + 1); + ip_version = (ip4->ip_version_and_header_length >> 4); + + if (ip_version == 0x4) + { + u32 a, t1, t2; + tcp_header_t *tcp = (void *) (ip4 + 1); + + is_tcp_udp = (ip4->protocol == IP_PROTOCOL_TCP) || + (ip4->protocol == IP_PROTOCOL_UDP); + t1 = is_tcp_udp ? clib_mem_unaligned (&tcp->src, u16) : 0; + t2 = is_tcp_udp ? clib_mem_unaligned (&tcp->dst, u16) : 0; + a = t1 ^ t2; + hash = lb_hash_hash_2_tuples ( + clib_mem_unaligned (&ip4->address_pair, u64), a); + return hash; + } + + if (ip_version == 0x6) + { + u64 a; + u32 t1, t2; + ip6_header_t *ip6 = (ip6_header_t *) (eth + 1); + tcp_header_t *tcp = (void *) (ip6 + 1); + + is_tcp_udp = 0; + if (PREDICT_TRUE ((ip6->protocol == IP_PROTOCOL_TCP) || + (ip6->protocol == IP_PROTOCOL_UDP))) + { + is_tcp_udp = 1; + tcp = (void *) (ip6 + 1); + } + else if (ip6->protocol == IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) + { + ip6_hop_by_hop_header_t *hbh = (ip6_hop_by_hop_header_t *) (ip6 + 1); + if ((hbh->protocol == IP_PROTOCOL_TCP) || + (hbh->protocol == IP_PROTOCOL_UDP)) + { + is_tcp_udp = 1; + tcp = (tcp_header_t *) ((u8 *) hbh + ((hbh->length + 1) << 3)); + } + } + t1 = is_tcp_udp ? clib_mem_unaligned (&tcp->src, u16) : 0; + t2 = is_tcp_udp ? clib_mem_unaligned (&tcp->dst, u16) : 0; + a = t1 ^ t2; + hash = lb_hash_hash ( + clib_mem_unaligned (&ip6->src_address.as_uword[0], uword), + clib_mem_unaligned (&ip6->src_address.as_uword[1], uword), + clib_mem_unaligned (&ip6->dst_address.as_uword[0], uword), + clib_mem_unaligned (&ip6->dst_address.as_uword[1], uword), a); + return hash; + } + + hash_eth_l2 (p, &hash, 1); + return hash; +} + +static void +hash_eth_l34 (void **p, u32 *hash, u32 n_packets) +{ + u32 n_left_from = n_packets; + + while (n_left_from >= 8) + { + clib_prefetch_load (p[4]); + clib_prefetch_load (p[5]); + clib_prefetch_load (p[6]); + clib_prefetch_load (p[7]); + + hash[0] = hash_eth_l34_inline (&p[0]); + hash[1] = hash_eth_l34_inline (&p[1]); + hash[2] = hash_eth_l34_inline (&p[2]); + hash[3] = hash_eth_l34_inline (&p[3]); + + hash += 4; + n_left_from -= 4; + p += 4; + } + + while (n_left_from > 0) + { + hash[0] = hash_eth_l34_inline (&p[0]); + + hash += 1; + n_left_from -= 1; + p += 1; + } +} + +VNET_REGISTER_HASH_FUNCTION (hash_eth_l2, static) = { + .name = "hash-eth-l2", + .description = "Hash ethernet L2 headers", + .priority = 50, + .function[VNET_HASH_FN_TYPE_ETHERNET] = hash_eth_l2, +}; + +VNET_REGISTER_HASH_FUNCTION (hash_eth_l23, static) = { + .name = "hash-eth-l23", + .description = "Hash ethernet L23 headers", + .priority = 50, + .function[VNET_HASH_FN_TYPE_ETHERNET] = hash_eth_l23, +}; + +VNET_REGISTER_HASH_FUNCTION (hash_eth_l34, static) = { + .name = "hash-eth-l34", + .description = "Hash ethernet L34 headers", + .priority = 50, + .function[VNET_HASH_FN_TYPE_ETHERNET] = hash_eth_l34, +}; + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ |