cnat: fixes & prepare maglev

Notable changes:
- ip[46]-cnat-snat is renamed to cnat-snat-ip[46]
- indent fixes
- common trace primitives
- bihash is now 40_56 with alias

Type: refactor

Change-Id: I0a82cfe3b40efd96473e51061d7135ffe412ddfc
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>

1 files changed, 184 insertions, 99 deletions

diff --git a/src/plugins/cnat/cnat_node.h b/src/plugins/cnat/cnat_node.h
index e2a9965b2a6..56a6c612e1b 100644
--- a/src/plugins/cnat/cnat_node.h
+++ b/src/plugins/cnat/cnat_node.h
@@ -17,9 +17,16 @@
 #define __CNAT_NODE_H__
 
 #include <vlibmemory/api.h>
+#include <vnet/dpo/load_balance.h>
+#include <vnet/dpo/load_balance_map.h>
+
 #include <cnat/cnat_session.h>
 #include <cnat/cnat_client.h>
 #include <cnat/cnat_inline.h>
+#include <cnat/cnat_translation.h>
+
+#include <vnet/ip/ip4_inlines.h>
+#include <vnet/ip/ip6_inlines.h>
 
 typedef uword (*cnat_node_sub_t) (vlib_main_t * vm,
 				  vlib_node_runtime_t * node,
@@ -27,6 +34,77 @@ typedef uword (*cnat_node_sub_t) (vlib_main_t * vm,
 				  cnat_node_ctx_t * ctx, int rv,
 				  cnat_session_t * session);
 
+typedef struct cnat_trace_element_t_
+{
+  cnat_session_t session;
+  cnat_translation_t tr;
+  u32 sw_if_index[VLIB_N_RX_TX];
+  u32 snat_policy_result;
+  u8 flags;
+} cnat_trace_element_t;
+
+typedef enum cnat_trace_element_flag_t_
+{
+  CNAT_TRACE_SESSION_FOUND = (1 << 0),
+  CNAT_TRACE_SESSION_CREATED = (1 << 1),
+  CNAT_TRACE_TRANSLATION_FOUND = (1 << 2),
+  CNAT_TRACE_NO_NAT = (1 << 3),
+} cnat_trace_element_flag_t;
+
+static_always_inline void
+cnat_add_trace (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b,
+		cnat_session_t *session, const cnat_translation_t *ct,
+		u8 flags)
+{
+  cnat_trace_element_t *t;
+  if (NULL != ct)
+    flags |= CNAT_TRACE_TRANSLATION_FOUND;
+
+  t = vlib_add_trace (vm, node, b, sizeof (*t));
+  t->sw_if_index[VLIB_RX] = vnet_buffer (b)->sw_if_index[VLIB_RX];
+  t->sw_if_index[VLIB_TX] = vnet_buffer (b)->sw_if_index[VLIB_TX];
+
+  if (flags & (CNAT_TRACE_SESSION_FOUND | CNAT_TRACE_SESSION_CREATED))
+    clib_memcpy (&t->session, session, sizeof (t->session));
+  if (flags & CNAT_TRACE_TRANSLATION_FOUND)
+    clib_memcpy (&t->tr, ct, sizeof (cnat_translation_t));
+  t->flags = flags;
+}
+
+static u8 *
+format_cnat_trace (u8 *s, va_list *args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  cnat_trace_element_t *t = va_arg (*args, cnat_trace_element_t *);
+  u32 indent = format_get_indent (s);
+  vnet_main_t *vnm = vnet_get_main ();
+
+  if (t->flags & CNAT_TRACE_SESSION_CREATED)
+    s = format (s, "created session");
+  else if (t->flags & CNAT_TRACE_SESSION_FOUND)
+    s = format (s, "found session");
+  else
+    s = format (s, "session not found");
+
+  if (t->flags & (CNAT_TRACE_NO_NAT))
+    s = format (s, " [policy:skip]");
+
+  s = format (s, "\n%Uin:%U out:%U ", format_white_space, indent,
+	      format_vnet_sw_if_index_name, vnm, t->sw_if_index[VLIB_RX],
+	      format_vnet_sw_if_index_name, vnm, t->sw_if_index[VLIB_TX]);
+
+  if (t->flags & (CNAT_TRACE_SESSION_CREATED | CNAT_TRACE_SESSION_FOUND))
+    s = format (s, "\n%U%U", format_white_space, indent, format_cnat_session,
+		&t->session, 1);
+
+  if (t->flags & CNAT_TRACE_TRANSLATION_FOUND)
+    s = format (s, "\n%Utranslation: %U", format_white_space, indent,
+		format_cnat_translation, &t->tr, 0);
+
+  return s;
+}
+
 static_always_inline u8
 icmp_type_is_error_message (u8 icmp_type)
 {
@@ -604,18 +682,25 @@ cnat_translation_ip6 (const cnat_session_t * session,
 }
 
 static_always_inline void
-cnat_session_make_key (vlib_buffer_t * b, ip_address_family_t af,
-		       clib_bihash_kv_40_48_t * bkey)
+cnat_session_make_key (vlib_buffer_t *b, ip_address_family_t af,
+		       cnat_session_location_t cs_loc, cnat_bihash_kv_t *bkey)
 {
   udp_header_t *udp;
   cnat_session_t *session = (cnat_session_t *) bkey;
+  u32 iph_offset = 0;
   session->key.cs_af = af;
-  session->key.__cs_pad[0] = 0;
-  session->key.__cs_pad[1] = 0;
+
+  session->key.cs_loc = cs_loc;
+  session->key.__cs_pad = 0;
+  if (cs_loc == CNAT_LOCATION_OUTPUT)
+    /* rewind buffer */
+    iph_offset = vnet_buffer (b)->ip.save_rewrite_length;
+
   if (AF_IP4 == af)
     {
       ip4_header_t *ip4;
-      ip4 = vlib_buffer_get_current (b);
+      ip4 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b) + iph_offset);
+
       if (PREDICT_FALSE (ip4->protocol == IP_PROTOCOL_ICMP))
 	{
 	  icmp46_header_t *icmp = (icmp46_header_t *) (ip4 + 1);
@@ -646,7 +731,8 @@ cnat_session_make_key (vlib_buffer_t * b, ip_address_family_t af,
 	  else
 	    goto error;
 	}
-      else
+      else if (ip4->protocol == IP_PROTOCOL_UDP ||
+	       ip4->protocol == IP_PROTOCOL_TCP)
 	{
 	  udp = (udp_header_t *) (ip4 + 1);
 	  ip46_address_set_ip4 (&session->key.cs_ip[VLIB_TX],
@@ -657,12 +743,13 @@ cnat_session_make_key (vlib_buffer_t * b, ip_address_family_t af,
 	  session->key.cs_port[VLIB_RX] = udp->src_port;
 	  session->key.cs_port[VLIB_TX] = udp->dst_port;
 	}
-
+      else
+	goto error;
     }
   else
     {
       ip6_header_t *ip6;
-      ip6 = vlib_buffer_get_current (b);
+      ip6 = (ip6_header_t *) ((u8 *) vlib_buffer_get_current (b) + iph_offset);
       if (PREDICT_FALSE (ip6->protocol == IP_PROTOCOL_ICMP6))
 	{
 	  icmp46_header_t *icmp = (icmp46_header_t *) (ip6 + 1);
@@ -693,7 +780,8 @@ cnat_session_make_key (vlib_buffer_t * b, ip_address_family_t af,
 	  else
 	    goto error;
 	}
-      else
+      else if (ip6->protocol == IP_PROTOCOL_UDP ||
+	       ip6->protocol == IP_PROTOCOL_TCP)
 	{
 	  udp = (udp_header_t *) (ip6 + 1);
 	  ip46_address_set_ip6 (&session->key.cs_ip[VLIB_TX],
@@ -704,6 +792,8 @@ cnat_session_make_key (vlib_buffer_t * b, ip_address_family_t af,
 	  session->key.cs_port[VLIB_TX] = udp->dst_port;
 	  session->key.cs_proto = ip6->protocol;
 	}
+      else
+	goto error;
     }
   return;
 
@@ -715,59 +805,64 @@ error:
 
 /**
  * Create NAT sessions
+ * rsession_location is the location the (return) session will be
+ * matched at
  */
-
 static_always_inline void
-cnat_session_create (cnat_session_t * session, cnat_node_ctx_t * ctx,
+cnat_session_create (cnat_session_t *session, cnat_node_ctx_t *ctx,
+		     cnat_session_location_t rsession_location,
 		     u8 rsession_flags)
 {
   cnat_client_t *cc;
-  clib_bihash_kv_40_48_t rkey;
+  cnat_bihash_kv_t rkey;
   cnat_session_t *rsession = (cnat_session_t *) & rkey;
-  clib_bihash_kv_40_48_t *bkey = (clib_bihash_kv_40_48_t *) session;
-  clib_bihash_kv_40_48_t rvalue;
+  cnat_bihash_kv_t *bkey = (cnat_bihash_kv_t *) session;
+  cnat_bihash_kv_t rvalue;
   int rv;
 
   session->value.cs_ts_index = cnat_timestamp_new (ctx->now);
-  clib_bihash_add_del_40_48 (&cnat_session_db, bkey, 1);
-
-  /* is this the first time we've seen this source address */
-  cc = (AF_IP4 == ctx->af ?
-	cnat_client_ip4_find (&session->value.cs_ip[VLIB_RX].ip4) :
-	cnat_client_ip6_find (&session->value.cs_ip[VLIB_RX].ip6));
+  cnat_bihash_add_del (&cnat_session_db, bkey, 1);
 
-  if (NULL == cc)
+  if (!(rsession_flags & CNAT_SESSION_FLAG_NO_CLIENT))
     {
-      ip_address_t addr;
-      uword *p;
-      u32 refcnt;
+      /* is this the first time we've seen this source address */
+      cc = (AF_IP4 == ctx->af ?
+	      cnat_client_ip4_find (&session->value.cs_ip[VLIB_RX].ip4) :
+	      cnat_client_ip6_find (&session->value.cs_ip[VLIB_RX].ip6));
 
-      addr.version = ctx->af;
-      ip46_address_copy (&addr.ip, &session->value.cs_ip[VLIB_RX]);
+      if (NULL == cc)
+	{
+	  ip_address_t addr;
+	  uword *p;
+	  u32 refcnt;
 
-      /* Throttle */
-      clib_spinlock_lock (&cnat_client_db.throttle_lock);
+	  addr.version = ctx->af;
+	  ip46_address_copy (&addr.ip, &session->value.cs_ip[VLIB_RX]);
 
-      p = hash_get_mem (cnat_client_db.throttle_mem, &addr);
-      if (p)
-	{
-	  refcnt = p[0] + 1;
-	  hash_set_mem (cnat_client_db.throttle_mem, &addr, refcnt);
-	}
-      else
-	hash_set_mem_alloc (&cnat_client_db.throttle_mem, &addr, 0);
+	  /* Throttle */
+	  clib_spinlock_lock (&cnat_client_db.throttle_lock);
 
-      clib_spinlock_unlock (&cnat_client_db.throttle_lock);
+	  p = hash_get_mem (cnat_client_db.throttle_mem, &addr);
+	  if (p)
+	    {
+	      refcnt = p[0] + 1;
+	      hash_set_mem (cnat_client_db.throttle_mem, &addr, refcnt);
+	    }
+	  else
+	    hash_set_mem_alloc (&cnat_client_db.throttle_mem, &addr, 0);
 
-      /* fire client create to the main thread */
-      if (!p)
-	vl_api_rpc_call_main_thread (cnat_client_learn, (u8 *) &addr,
-				     sizeof (addr));
-    }
-  else
-    {
-      /* Refcount reverse session */
-      cnat_client_cnt_session (cc);
+	  clib_spinlock_unlock (&cnat_client_db.throttle_lock);
+
+	  /* fire client create to the main thread */
+	  if (!p)
+	    vl_api_rpc_call_main_thread (cnat_client_learn, (u8 *) &addr,
+					 sizeof (addr));
+	}
+      else
+	{
+	  /* Refcount reverse session */
+	  cnat_client_cnt_session (cc);
+	}
     }
 
   /* create the reverse flow key */
@@ -776,14 +871,14 @@ cnat_session_create (cnat_session_t * session, cnat_node_ctx_t * ctx,
   ip46_address_copy (&rsession->key.cs_ip[VLIB_TX],
 		     &session->value.cs_ip[VLIB_RX]);
   rsession->key.cs_proto = session->key.cs_proto;
-  rsession->key.__cs_pad[0] = 0;
-  rsession->key.__cs_pad[1] = 0;
+  rsession->key.cs_loc = rsession_location;
+  rsession->key.__cs_pad = 0;
   rsession->key.cs_af = ctx->af;
   rsession->key.cs_port[VLIB_RX] = session->value.cs_port[VLIB_TX];
   rsession->key.cs_port[VLIB_TX] = session->value.cs_port[VLIB_RX];
 
   /* First search for existing reverse session */
-  rv = clib_bihash_search_inline_2_40_48 (&cnat_session_db, &rkey, &rvalue);
+  rv = cnat_bihash_search_i2 (&cnat_session_db, &rkey, &rvalue);
   if (!rv)
     {
       /* Reverse session already exists
@@ -802,15 +897,14 @@ cnat_session_create (cnat_session_t * session, cnat_node_ctx_t * ctx,
   rsession->value.cs_port[VLIB_TX] = session->key.cs_port[VLIB_RX];
   rsession->value.cs_port[VLIB_RX] = session->key.cs_port[VLIB_TX];
 
-  clib_bihash_add_del_40_48 (&cnat_session_db, &rkey, 1);
+  cnat_bihash_add_del (&cnat_session_db, &rkey, 1);
 }
 
 always_inline uword
-cnat_node_inline (vlib_main_t * vm,
-		  vlib_node_runtime_t * node,
-		  vlib_frame_t * frame,
-		  cnat_node_sub_t cnat_sub,
-		  ip_address_family_t af, u8 do_trace)
+cnat_node_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
+		  vlib_frame_t *frame, cnat_node_sub_t cnat_sub,
+		  ip_address_family_t af, cnat_session_location_t cs_loc,
+		  u8 do_trace)
 {
   u32 n_left, *from, thread_index;
   vlib_buffer_t *bufs[VLIB_FRAME_SIZE];
@@ -825,7 +919,7 @@ cnat_node_inline (vlib_main_t * vm,
   vlib_get_buffers (vm, from, bufs, n_left);
   now = vlib_time_now (vm);
   cnat_session_t *session[4];
-  clib_bihash_kv_40_48_t bkey[4], bvalue[4];
+  cnat_bihash_kv_t bkey[4], bvalue[4];
   u64 hash[4];
   int rv[4];
 
@@ -834,15 +928,15 @@ cnat_node_inline (vlib_main_t * vm,
   if (n_left >= 8)
     {
       /* Kickstart our state */
-      cnat_session_make_key (b[3], af, &bkey[3]);
-      cnat_session_make_key (b[2], af, &bkey[2]);
-      cnat_session_make_key (b[1], af, &bkey[1]);
-      cnat_session_make_key (b[0], af, &bkey[0]);
+      cnat_session_make_key (b[3], af, cs_loc, &bkey[3]);
+      cnat_session_make_key (b[2], af, cs_loc, &bkey[2]);
+      cnat_session_make_key (b[1], af, cs_loc, &bkey[1]);
+      cnat_session_make_key (b[0], af, cs_loc, &bkey[0]);
 
-      hash[3] = clib_bihash_hash_40_48 (&bkey[3]);
-      hash[2] = clib_bihash_hash_40_48 (&bkey[2]);
-      hash[1] = clib_bihash_hash_40_48 (&bkey[1]);
-      hash[0] = clib_bihash_hash_40_48 (&bkey[0]);
+      hash[3] = cnat_bihash_hash (&bkey[3]);
+      hash[2] = cnat_bihash_hash (&bkey[2]);
+      hash[1] = cnat_bihash_hash (&bkey[1]);
+      hash[0] = cnat_bihash_hash (&bkey[0]);
     }
 
   while (n_left >= 8)
@@ -855,53 +949,45 @@ cnat_node_inline (vlib_main_t * vm,
 	  vlib_prefetch_buffer_header (b[8], LOAD);
 	}
 
-      rv[3] =
-	clib_bihash_search_inline_2_with_hash_40_48 (&cnat_session_db,
-						     hash[3], &bkey[3],
-						     &bvalue[3]);
+      rv[3] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[3], &bkey[3],
+					  &bvalue[3]);
       session[3] = (cnat_session_t *) (rv[3] ? &bkey[3] : &bvalue[3]);
       next[3] = cnat_sub (vm, node, b[3], &ctx, rv[3], session[3]);
 
-      rv[2] =
-	clib_bihash_search_inline_2_with_hash_40_48 (&cnat_session_db,
-						     hash[2], &bkey[2],
-						     &bvalue[2]);
+      rv[2] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[2], &bkey[2],
+					  &bvalue[2]);
       session[2] = (cnat_session_t *) (rv[2] ? &bkey[2] : &bvalue[2]);
       next[2] = cnat_sub (vm, node, b[2], &ctx, rv[2], session[2]);
 
-      rv[1] =
-	clib_bihash_search_inline_2_with_hash_40_48 (&cnat_session_db,
-						     hash[1], &bkey[1],
-						     &bvalue[1]);
+      rv[1] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[1], &bkey[1],
+					  &bvalue[1]);
       session[1] = (cnat_session_t *) (rv[1] ? &bkey[1] : &bvalue[1]);
       next[1] = cnat_sub (vm, node, b[1], &ctx, rv[1], session[1]);
 
-      rv[0] =
-	clib_bihash_search_inline_2_with_hash_40_48 (&cnat_session_db,
-						     hash[0], &bkey[0],
-						     &bvalue[0]);
+      rv[0] = cnat_bihash_search_i2_hash (&cnat_session_db, hash[0], &bkey[0],
+					  &bvalue[0]);
       session[0] = (cnat_session_t *) (rv[0] ? &bkey[0] : &bvalue[0]);
       next[0] = cnat_sub (vm, node, b[0], &ctx, rv[0], session[0]);
 
-      cnat_session_make_key (b[7], af, &bkey[3]);
-      cnat_session_make_key (b[6], af, &bkey[2]);
-      cnat_session_make_key (b[5], af, &bkey[1]);
-      cnat_session_make_key (b[4], af, &bkey[0]);
+      cnat_session_make_key (b[7], af, cs_loc, &bkey[3]);
+      cnat_session_make_key (b[6], af, cs_loc, &bkey[2]);
+      cnat_session_make_key (b[5], af, cs_loc, &bkey[1]);
+      cnat_session_make_key (b[4], af, cs_loc, &bkey[0]);
 
-      hash[3] = clib_bihash_hash_40_48 (&bkey[3]);
-      hash[2] = clib_bihash_hash_40_48 (&bkey[2]);
-      hash[1] = clib_bihash_hash_40_48 (&bkey[1]);
-      hash[0] = clib_bihash_hash_40_48 (&bkey[0]);
+      hash[3] = cnat_bihash_hash (&bkey[3]);
+      hash[2] = cnat_bihash_hash (&bkey[2]);
+      hash[1] = cnat_bihash_hash (&bkey[1]);
+      hash[0] = cnat_bihash_hash (&bkey[0]);
 
-      clib_bihash_prefetch_bucket_40_48 (&cnat_session_db, hash[3]);
-      clib_bihash_prefetch_bucket_40_48 (&cnat_session_db, hash[2]);
-      clib_bihash_prefetch_bucket_40_48 (&cnat_session_db, hash[1]);
-      clib_bihash_prefetch_bucket_40_48 (&cnat_session_db, hash[0]);
+      cnat_bihash_prefetch_bucket (&cnat_session_db, hash[3]);
+      cnat_bihash_prefetch_bucket (&cnat_session_db, hash[2]);
+      cnat_bihash_prefetch_bucket (&cnat_session_db, hash[1]);
+      cnat_bihash_prefetch_bucket (&cnat_session_db, hash[0]);
 
-      clib_bihash_prefetch_data_40_48 (&cnat_session_db, hash[3]);
-      clib_bihash_prefetch_data_40_48 (&cnat_session_db, hash[2]);
-      clib_bihash_prefetch_data_40_48 (&cnat_session_db, hash[1]);
-      clib_bihash_prefetch_data_40_48 (&cnat_session_db, hash[0]);
+      cnat_bihash_prefetch_data (&cnat_session_db, hash[3]);
+      cnat_bihash_prefetch_data (&cnat_session_db, hash[2]);
+      cnat_bihash_prefetch_data (&cnat_session_db, hash[1]);
+      cnat_bihash_prefetch_data (&cnat_session_db, hash[0]);
 
       b += 4;
       next += 4;
@@ -910,9 +996,8 @@ cnat_node_inline (vlib_main_t * vm,
 
   while (n_left > 0)
     {
-      cnat_session_make_key (b[0], af, &bkey[0]);
-      rv[0] = clib_bihash_search_inline_2_40_48 (&cnat_session_db,
-						 &bkey[0], &bvalue[0]);
+      cnat_session_make_key (b[0], af, cs_loc, &bkey[0]);
+      rv[0] = cnat_bihash_search_i2 (&cnat_session_db, &bkey[0], &bvalue[0]);
 
       session[0] = (cnat_session_t *) (rv[0] ? &bkey[0] : &bvalue[0]);
       next[0] = cnat_sub (vm, node, b[0], &ctx, rv[0], session[0]);