interface: handoff refactor and optimizations

- move existing ethernet hash functions to hash infra (no l4
awareness)
- refactor code to use hash infra and add apis to request l4 aware
hashing functions
- hashing functions per interface
- code cleanup

Type: improvement

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: Ia9f44db98d83f0f027aeb37718585a2e10ffd2c6

1 files changed, 341 insertions, 0 deletions

diff --git a/src/vnet/hash/crc32_handoff_eth.c b/src/vnet/hash/crc32_handoff_eth.c
new file mode 100644
index 00000000000..13fc12ce6ea
--- /dev/null
+++ b/src/vnet/hash/crc32_handoff_eth.c
@@ -0,0 +1,341 @@
+/*
+ * Copyright (c) 2021 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vlib/vlib.h>
+#include <vnet/ethernet/ethernet.h>
+#include <vnet/hash/hash.h>
+#include <vnet/ip/ip4_packet.h>
+#include <vnet/ip/ip6_packet.h>
+#include <vnet/mpls/packet.h>
+#include <vppinfra/crc32.h>
+
+static inline u64
+ipv4_get_key (ip4_header_t * ip)
+{
+  u64 hash_key;
+
+  hash_key = *((u64 *) (&ip->address_pair)) ^ ip->protocol;
+
+  return hash_key;
+}
+
+static inline u64
+ipv6_get_key (ip6_header_t * ip)
+{
+  u64 hash_key;
+
+  hash_key = ip->src_address.as_u64[0] ^
+    rotate_left (ip->src_address.as_u64[1], 13) ^
+    rotate_left (ip->dst_address.as_u64[0], 26) ^
+    rotate_left (ip->dst_address.as_u64[1], 39) ^ ip->protocol;
+
+  return hash_key;
+}
+
+#define MPLS_BOTTOM_OF_STACK_BIT_MASK   0x00000100U
+#define MPLS_LABEL_MASK                 0xFFFFF000U
+
+static inline u64
+mpls_get_key (mpls_unicast_header_t * m)
+{
+  u64 hash_key;
+  u8 ip_ver;
+
+
+  /* find the bottom of the MPLS label stack. */
+  if (PREDICT_TRUE (m->label_exp_s_ttl &
+		    clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)))
+    {
+      goto bottom_lbl_found;
+    }
+  m++;
+
+  if (PREDICT_TRUE (m->label_exp_s_ttl &
+		    clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK)))
+    {
+      goto bottom_lbl_found;
+    }
+  m++;
+
+  if (m->label_exp_s_ttl &
+      clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK))
+    {
+      goto bottom_lbl_found;
+    }
+  m++;
+
+  if (m->label_exp_s_ttl &
+      clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK))
+    {
+      goto bottom_lbl_found;
+    }
+  m++;
+
+  if (m->label_exp_s_ttl &
+      clib_net_to_host_u32 (MPLS_BOTTOM_OF_STACK_BIT_MASK))
+    {
+      goto bottom_lbl_found;
+    }
+
+  /* the bottom label was not found - use the last label */
+  hash_key = m->label_exp_s_ttl & clib_net_to_host_u32 (MPLS_LABEL_MASK);
+
+  return hash_key;
+
+bottom_lbl_found:
+  m++;
+  ip_ver = (*((u8 *) m) >> 4);
+
+  /* find out if it is IPV4 or IPV6 header */
+  if (PREDICT_TRUE (ip_ver == 4))
+    {
+      hash_key = ipv4_get_key ((ip4_header_t *) m);
+    }
+  else if (PREDICT_TRUE (ip_ver == 6))
+    {
+      hash_key = ipv6_get_key ((ip6_header_t *) m);
+    }
+  else
+    {
+      /* use the bottom label */
+      hash_key =
+	(m - 1)->label_exp_s_ttl & clib_net_to_host_u32 (MPLS_LABEL_MASK);
+    }
+
+  return hash_key;
+
+}
+
+static inline u64
+eth_get_sym_key (ethernet_header_t * h0)
+{
+  u64 hash_key;
+
+  if (PREDICT_TRUE (h0->type) == clib_host_to_net_u16 (ETHERNET_TYPE_IP4))
+    {
+      ip4_header_t *ip = (ip4_header_t *) (h0 + 1);
+      hash_key =
+	(u64) (ip->src_address.as_u32 ^
+	       ip->dst_address.as_u32 ^ ip->protocol);
+    }
+  else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6))
+    {
+      ip6_header_t *ip = (ip6_header_t *) (h0 + 1);
+      hash_key = (u64) (ip->src_address.as_u64[0] ^
+			ip->src_address.as_u64[1] ^
+			ip->dst_address.as_u64[0] ^
+			ip->dst_address.as_u64[1] ^ ip->protocol);
+    }
+  else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
+    {
+      hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1));
+    }
+  else
+    if (PREDICT_FALSE
+	((h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN))
+	 || (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD))))
+    {
+      ethernet_vlan_header_t *outer = (ethernet_vlan_header_t *) (h0 + 1);
+
+      outer = (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) ?
+	outer + 1 : outer;
+      if (PREDICT_TRUE (outer->type) ==
+	  clib_host_to_net_u16 (ETHERNET_TYPE_IP4))
+	{
+	  ip4_header_t *ip = (ip4_header_t *) (outer + 1);
+	  hash_key =
+	    (u64) (ip->src_address.as_u32 ^
+		   ip->dst_address.as_u32 ^ ip->protocol);
+	}
+      else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6))
+	{
+	  ip6_header_t *ip = (ip6_header_t *) (outer + 1);
+	  hash_key =
+	    (u64) (ip->src_address.as_u64[0] ^ ip->src_address.as_u64[1] ^
+		   ip->dst_address.as_u64[0] ^
+		   ip->dst_address.as_u64[1] ^ ip->protocol);
+	}
+      else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
+	{
+	  hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1));
+	}
+      else
+	{
+	  hash_key = outer->type;
+	}
+    }
+  else
+    {
+      hash_key = 0;
+    }
+
+  return hash_key;
+}
+
+static inline u64
+eth_get_key (ethernet_header_t * h0)
+{
+  u64 hash_key;
+
+  if (PREDICT_TRUE (h0->type) == clib_host_to_net_u16 (ETHERNET_TYPE_IP4))
+    {
+      hash_key = ipv4_get_key ((ip4_header_t *) (h0 + 1));
+    }
+  else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6))
+    {
+      hash_key = ipv6_get_key ((ip6_header_t *) (h0 + 1));
+    }
+  else if (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
+    {
+      hash_key = mpls_get_key ((mpls_unicast_header_t *) (h0 + 1));
+    }
+  else if ((h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) ||
+	   (h0->type == clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD)))
+    {
+      ethernet_vlan_header_t *outer = (ethernet_vlan_header_t *) (h0 + 1);
+
+      outer = (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_VLAN)) ?
+	outer + 1 : outer;
+      if (PREDICT_TRUE (outer->type) ==
+	  clib_host_to_net_u16 (ETHERNET_TYPE_IP4))
+	{
+	  hash_key = ipv4_get_key ((ip4_header_t *) (outer + 1));
+	}
+      else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6))
+	{
+	  hash_key = ipv6_get_key ((ip6_header_t *) (outer + 1));
+	}
+      else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS))
+	{
+	  hash_key = mpls_get_key ((mpls_unicast_header_t *) (outer + 1));
+	}
+      else
+	{
+	  hash_key = outer->type;
+	}
+    }
+  else
+    {
+      hash_key = 0;
+    }
+
+  return hash_key;
+}
+
+void
+handoff_eth_crc32c_func (void **p, u32 *hash, u32 n_packets)
+{
+  u32 n_left_from = n_packets;
+
+  while (n_left_from >= 8)
+    {
+      u64 key[4] = {};
+
+      clib_prefetch_load (p[4]);
+      clib_prefetch_load (p[5]);
+      clib_prefetch_load (p[6]);
+      clib_prefetch_load (p[7]);
+
+      key[0] = eth_get_key ((ethernet_header_t *) p[0]);
+      key[1] = eth_get_key ((ethernet_header_t *) p[1]);
+      key[2] = eth_get_key ((ethernet_header_t *) p[2]);
+      key[3] = eth_get_key ((ethernet_header_t *) p[3]);
+
+      hash[0] = clib_crc32c ((u8 *) &key[0], sizeof (key[0]));
+      hash[1] = clib_crc32c ((u8 *) &key[1], sizeof (key[1]));
+      hash[2] = clib_crc32c ((u8 *) &key[2], sizeof (key[2]));
+      hash[3] = clib_crc32c ((u8 *) &key[3], sizeof (key[3]));
+
+      hash += 4;
+      n_left_from -= 4;
+      p += 4;
+    }
+
+  while (n_left_from > 0)
+    {
+      u64 key;
+
+      key = eth_get_key ((ethernet_header_t *) p[0]);
+      hash[0] = clib_crc32c ((u8 *) &key, sizeof (key));
+
+      hash += 1;
+      n_left_from -= 1;
+      p += 1;
+    }
+}
+
+VNET_REGISTER_HASH_FUNCTION (handoff_eth_crc32c, static) = {
+  .name = "handoff-eth-crc32c",
+  .description = "Ethernet/IPv4/IPv6/MPLS headers",
+  .priority = 2,
+  .function[VNET_HASH_FN_TYPE_ETHERNET] = handoff_eth_crc32c_func,
+};
+
+void
+handoff_eth_sym_crc32c_func (void **p, u32 *hash, u32 n_packets)
+{
+  u32 n_left_from = n_packets;
+
+  while (n_left_from >= 8)
+    {
+      u64 key[4] = {};
+
+      clib_prefetch_load (p[4]);
+      clib_prefetch_load (p[5]);
+      clib_prefetch_load (p[6]);
+      clib_prefetch_load (p[7]);
+
+      key[0] = eth_get_sym_key ((ethernet_header_t *) p[0]);
+      key[1] = eth_get_sym_key ((ethernet_header_t *) p[1]);
+      key[2] = eth_get_sym_key ((ethernet_header_t *) p[2]);
+      key[3] = eth_get_sym_key ((ethernet_header_t *) p[3]);
+
+      hash[0] = clib_crc32c ((u8 *) &key[0], sizeof (key[0]));
+      hash[1] = clib_crc32c ((u8 *) &key[1], sizeof (key[1]));
+      hash[2] = clib_crc32c ((u8 *) &key[2], sizeof (key[2]));
+      hash[3] = clib_crc32c ((u8 *) &key[3], sizeof (key[3]));
+
+      hash += 4;
+      n_left_from -= 4;
+      p += 4;
+    }
+
+  while (n_left_from > 0)
+    {
+      u64 key;
+
+      key = eth_get_sym_key ((ethernet_header_t *) p[0]);
+      hash[0] = clib_crc32c ((u8 *) &key, sizeof (key));
+
+      hash += 1;
+      n_left_from -= 1;
+      p += 1;
+    }
+}
+
+VNET_REGISTER_HASH_FUNCTION (handoff_eth_sym_crc32c, static) = {
+  .name = "handoff-eth-sym-crc32c",
+  .description = "Ethernet/IPv4/IPv6/MPLS headers Symmetric",
+  .priority = 1,
+  .function[VNET_HASH_FN_TYPE_ETHERNET] = handoff_eth_sym_crc32c_func,
+};
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */