nat: fix multi worker scenarios

Properly select worker from hash table in out2in direction, increase
number of worker threads in MW test to 4 to test these cases.

Type: fix
Change-Id: I76eda5761ff497b85b031dd913a64b7fcb53b33a
Signed-off-by: Klement Sekera <ksekera@cisco.com>

2 files changed, 56 insertions, 28 deletions

diff --git a/src/plugins/nat/nat44-ed/nat44_ed.c b/src/plugins/nat/nat44-ed/nat44_ed.c
index 99029bb83b8..58aade568b9 100644
--- a/src/plugins/nat/nat44-ed/nat44_ed.c
+++ b/src/plugins/nat/nat44-ed/nat44_ed.c
@@ -2603,7 +2603,6 @@ nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
   snat_main_t *sm = &snat_main;
   clib_bihash_kv_8_8_t kv, value;
   clib_bihash_kv_16_8_t kv16, value16;
-  snat_main_per_thread_data_t *tsm;
 
   u32 proto, next_worker_index = 0;
   u16 port;
@@ -2612,29 +2611,7 @@ nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
 
   proto = ip_proto_to_nat_proto (ip->protocol);
 
-  if (PREDICT_TRUE (proto == NAT_PROTOCOL_UDP || proto == NAT_PROTOCOL_TCP))
-    {
-      init_ed_k (&kv16, ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port,
-		 ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
-		 rx_fib_index, ip->protocol);
-
-      if (PREDICT_TRUE (
-	    !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
-	{
-	  tsm =
-	    vec_elt_at_index (sm->per_thread_data,
-			      ed_value_get_thread_index (&value16));
-	  vnet_buffer2 (b)->nat.cached_session_index =
-	    ed_value_get_session_index (&value16);
-	  next_worker_index = sm->first_worker_index + tsm->thread_index;
-	  nat_elog_debug_handoff (
-	    sm, "HANDOFF OUT2IN (session)", next_worker_index, rx_fib_index,
-	    clib_net_to_host_u32 (ip->src_address.as_u32),
-	    clib_net_to_host_u32 (ip->dst_address.as_u32));
-	  return next_worker_index;
-	}
-    }
-  else if (proto == NAT_PROTOCOL_ICMP)
+  if (PREDICT_FALSE (proto == NAT_PROTOCOL_ICMP))
     {
       ip4_address_t lookup_saddr, lookup_daddr;
       u16 lookup_sport, lookup_dport;
@@ -2648,10 +2625,7 @@ nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
 	  if (PREDICT_TRUE (
 		!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
 	    {
-	      tsm =
-		vec_elt_at_index (sm->per_thread_data,
-				  ed_value_get_thread_index (&value16));
-	      next_worker_index = sm->first_worker_index + tsm->thread_index;
+	      next_worker_index = ed_value_get_thread_index (&value16);
 	      nat_elog_debug_handoff (
 		sm, "HANDOFF OUT2IN (session)", next_worker_index,
 		rx_fib_index, clib_net_to_host_u32 (ip->src_address.as_u32),
@@ -2661,6 +2635,23 @@ nat44_ed_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip,
 	}
     }
 
+  init_ed_k (&kv16, ip->src_address, vnet_buffer (b)->ip.reass.l4_src_port,
+	     ip->dst_address, vnet_buffer (b)->ip.reass.l4_dst_port,
+	     rx_fib_index, ip->protocol);
+
+  if (PREDICT_TRUE (
+	!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
+    {
+      vnet_buffer2 (b)->nat.cached_session_index =
+	ed_value_get_session_index (&value16);
+      next_worker_index = ed_value_get_thread_index (&value16);
+      nat_elog_debug_handoff (sm, "HANDOFF OUT2IN (session)",
+			      next_worker_index, rx_fib_index,
+			      clib_net_to_host_u32 (ip->src_address.as_u32),
+			      clib_net_to_host_u32 (ip->dst_address.as_u32));
+      return next_worker_index;
+    }
+
   /* first try static mappings without port */
   if (PREDICT_FALSE (pool_elts (sm->static_mappings)))
     {
diff --git a/src/plugins/nat/nat44-ed/nat44_ed_out2in.c b/src/plugins/nat/nat44-ed/nat44_ed_out2in.c
index f46433c9e53..e8cf7c930fa 100644
--- a/src/plugins/nat/nat44-ed/nat44_ed_out2in.c
+++ b/src/plugins/nat/nat44-ed/nat44_ed_out2in.c
@@ -37,6 +37,15 @@ static char *nat_out2in_ed_error_strings[] = {
 #undef _
 };
 
+typedef enum
+{
+  NAT_ED_SP_REASON_NO_REASON,
+  NAT_ED_SP_REASON_LOOKUP_FAILED,
+  NAT_ED_SP_REASON_VRF_EXPIRED,
+  NAT_ED_SP_TCP_CLOSED,
+  NAT_ED_SP_SESS_EXPIRED,
+} nat_slow_path_reason_e;
+
 typedef struct
 {
   u32 sw_if_index;
@@ -49,9 +58,30 @@ typedef struct
   u8 is_slow_path;
   u8 translation_via_i2of;
   u8 lookup_skipped;
+  nat_slow_path_reason_e slow_path_reason;
 } nat44_ed_out2in_trace_t;
 
 static u8 *
+format_slow_path_reason (u8 *s, va_list *args)
+{
+  nat_slow_path_reason_e reason = va_arg (*args, nat_slow_path_reason_e);
+  switch (reason)
+    {
+    case NAT_ED_SP_REASON_NO_REASON:
+      return format (s, "no reason for slow path");
+    case NAT_ED_SP_REASON_LOOKUP_FAILED:
+      return format (s, "slow path because lookup failed");
+    case NAT_ED_SP_REASON_VRF_EXPIRED:
+      return format (s, "slow path because vrf expired");
+    case NAT_ED_SP_TCP_CLOSED:
+      return format (s, "slow path because tcp closed");
+    case NAT_ED_SP_SESS_EXPIRED:
+      return format (s, "slow path because session expired");
+    }
+  return format (s, "invalid reason value");
+}
+
+static u8 *
 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
 {
   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
@@ -85,6 +115,7 @@ format_nat44_ed_out2in_trace (u8 * s, va_list * args)
 	  s = format (s, "\n  search key %U", format_ed_session_kvp,
 		      &t->search_key);
 	}
+      s = format (s, "\n %U", format_slow_path_reason, t->slow_path_reason);
     }
 
   return s;
@@ -790,6 +821,7 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
       snat_session_t *s0 = 0;
       clib_bihash_kv_16_8_t kv0, value0;
       nat_translation_error_e translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
+      nat_slow_path_reason_e slow_path_reason = NAT_ED_SP_REASON_NO_REASON;
       nat_6t_flow_t *f = 0;
       nat_6t_t lookup;
       int lookup_skipped = 0;
@@ -891,6 +923,7 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
       if (clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
 	{
 	  // flow does not exist go slow path
+	  slow_path_reason = NAT_ED_SP_REASON_LOOKUP_FAILED;
 	  next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 	  goto trace0;
 	}
@@ -905,6 +938,7 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 	  // session is closed, go slow path
 	  nat_free_session_data (sm, s0, thread_index, 0);
 	  nat_ed_session_delete (sm, s0, thread_index, 1);
+	  slow_path_reason = NAT_ED_SP_REASON_VRF_EXPIRED;
 	  next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 	  goto trace0;
 	}
@@ -914,6 +948,7 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 	  if (now >= s0->tcp_closed_timestamp)
 	    {
 	      // session is closed, go slow path, freed in slow path
+	      slow_path_reason = NAT_ED_SP_TCP_CLOSED;
 	      next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 	    }
 	  else
@@ -934,6 +969,7 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 	  // session is closed, go slow path
 	  nat_free_session_data (sm, s0, thread_index, 0);
 	  nat_ed_session_delete (sm, s0, thread_index, 1);
+	  slow_path_reason = NAT_ED_SP_SESS_EXPIRED;
 	  next[0] = NAT_NEXT_OUT2IN_ED_SLOW_PATH;
 	  goto trace0;
 	}
@@ -1042,6 +1078,7 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 	  t->translation_error = translation_error;
 	  clib_memcpy (&t->search_key, &kv0, sizeof (t->search_key));
 	  t->lookup_skipped = lookup_skipped;
+	  t->slow_path_reason = slow_path_reason;
 
 	  if (s0)
 	    {