nat: fix forwarding handoff workaround

A special case when out2in packet needs to
be handoffed to other worker thread. We are
not able to determine which thread they belong
to in the first processing of nat handoff node.
These packets needs to go through out2in slowpath
before we are able to tell where to handoff them.

Type: fix
Ticket: VPP-1941

Change-Id: I7173bda970ce6a91d81f48fc72aa2457586a076f
Signed-off-by: Filip Varga <fivarga@cisco.com>

6 files changed, 170 insertions, 31 deletions

diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
index 0ca2042191d..91f14d682f4 100644
--- a/src/plugins/nat/nat.c
+++ b/src/plugins/nat/nat.c
@@ -4752,6 +4752,7 @@ VLIB_REGISTER_NODE (nat_default_node) = {
     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
+    [NAT_NEXT_OUT2IN_ED_HANDOFF] = "nat44-ed-out2in-handoff",
     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
   },
diff --git a/src/plugins/nat/nat.h b/src/plugins/nat/nat.h
index a713f7b51cf..daebe451f4f 100644
--- a/src/plugins/nat/nat.h
+++ b/src/plugins/nat/nat.h
@@ -88,6 +88,7 @@ typedef enum
   NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH,
   NAT_NEXT_OUT2IN_ED_FAST_PATH,
   NAT_NEXT_OUT2IN_ED_SLOW_PATH,
+  NAT_NEXT_OUT2IN_ED_HANDOFF,
   NAT_NEXT_IN2OUT_CLASSIFY,
   NAT_NEXT_OUT2IN_CLASSIFY,
   NAT_N_NEXT,
@@ -162,6 +163,19 @@ typedef enum
     NAT_IN2OUT_ED_N_ERROR,
 } nat_in2out_ed_error_t;
 
+#define foreach_nat44_handoff_error                       \
+_(CONGESTION_DROP, "congestion drop")                     \
+_(SAME_WORKER, "same worker")                             \
+_(DO_HANDOFF, "do handoff")
+
+typedef enum
+{
+#define _(sym,str) NAT44_HANDOFF_ERROR_##sym,
+  foreach_nat44_handoff_error
+#undef _
+    NAT44_HANDOFF_N_ERROR,
+} nat44_handoff_error_t;
+
 #define foreach_nat_out2in_ed_error                     \
 _(UNSUPPORTED_PROTOCOL, "unsupported protocol")         \
 _(OUT_OF_PORTS, "out of ports")                         \
diff --git a/src/plugins/nat/nat44_handoff.c b/src/plugins/nat/nat44_handoff.c
index 69cdc42870f..651c8d1f4ac 100644
--- a/src/plugins/nat/nat44_handoff.c
+++ b/src/plugins/nat/nat44_handoff.c
@@ -33,26 +33,12 @@ typedef struct
   u8 output;
 } nat44_handoff_trace_t;
 
-#define foreach_nat44_handoff_error                       \
-_(CONGESTION_DROP, "congestion drop")                     \
-_(SAME_WORKER, "same worker")                             \
-_(DO_HANDOFF, "do handoff")
-
-typedef enum
-{
-#define _(sym,str) NAT44_HANDOFF_ERROR_##sym,
-  foreach_nat44_handoff_error
-#undef _
-    NAT44_HANDOFF_N_ERROR,
-} nat44_handoff_error_t;
-
 static char *nat44_handoff_error_strings[] = {
 #define _(sym,string) string,
   foreach_nat44_handoff_error
 #undef _
 };
 
-
 static u8 *
 format_nat44_handoff_trace (u8 * s, va_list * args)
 {
diff --git a/src/plugins/nat/nat_ha.c b/src/plugins/nat/nat_ha.c
index 3ae572ba930..08b065dec1d 100644
--- a/src/plugins/nat/nat_ha.c
+++ b/src/plugins/nat/nat_ha.c
@@ -1057,7 +1057,7 @@ typedef enum
 #define _(sym,str) NAT_HA_HANDOFF_ERROR_##sym,
   foreach_nat_ha_handoff_error
 #undef _
-    NAT44_HANDOFF_N_ERROR,
+    NAT_HA_HANDOFF_N_ERROR,
 } nat_ha_handoff_error_t;
 
 static char *nat_ha_handoff_error_strings[] = {
diff --git a/src/plugins/nat/out2in_ed.c b/src/plugins/nat/out2in_ed.c
index b0e8892f77a..95e2ea972af 100644
--- a/src/plugins/nat/out2in_ed.c
+++ b/src/plugins/nat/out2in_ed.c
@@ -46,6 +46,11 @@ typedef struct
   u32 is_slow_path;
 } nat44_ed_out2in_trace_t;
 
+typedef struct
+{
+  u16 thread_next;
+} nat44_ed_out2in_handoff_trace_t;
+
 static u8 *
 format_nat44_ed_out2in_trace (u8 * s, va_list * args)
 {
@@ -513,16 +518,24 @@ create_bypass_for_fwd (snat_main_t * sm, vlib_buffer_t * b, ip4_header_t * ip,
   nat44_session_update_lru (sm, s, thread_index);
 }
 
-static inline void
-create_bypass_for_fwd_worker (snat_main_t * sm, vlib_buffer_t * b,
-			      ip4_header_t * ip, u32 rx_fib_index)
+static_always_inline int
+create_bypass_for_fwd_worker (snat_main_t * sm,
+			      vlib_buffer_t * b, ip4_header_t * ip,
+			      u32 rx_fib_index, u32 thread_index)
 {
-  ip4_header_t ip_wkr = {
+  ip4_header_t tmp = {
     .src_address = ip->dst_address,
   };
-  u32 thread_index = sm->worker_in2out_cb (&ip_wkr, rx_fib_index, 0);
+  u32 index = sm->worker_in2out_cb (&tmp, rx_fib_index, 0);
+
+  if (index != thread_index)
+    {
+      vnet_buffer2 (b)->nat.thread_next = index;
+      return 1;
+    }
 
   create_bypass_for_fwd (sm, b, ip, rx_fib_index, thread_index);
+  return 0;
 }
 
 #ifndef CLIB_MARCH_VARIANT
@@ -590,10 +603,19 @@ icmp_match_out2in_ed (snat_main_t * sm, vlib_node_runtime_t * node,
 	      else
 		{
 		  if (sm->num_workers > 1)
-		    create_bypass_for_fwd_worker (sm, b, ip, rx_fib_index);
+		    {
+		      if (create_bypass_for_fwd_worker (sm, b, ip,
+							rx_fib_index,
+							thread_index))
+			{
+			  next = NAT_NEXT_OUT2IN_ED_HANDOFF;
+			}
+		    }
 		  else
-		    create_bypass_for_fwd (sm, b, ip, rx_fib_index,
-					   thread_index);
+		    {
+		      create_bypass_for_fwd (sm, b, ip, rx_fib_index,
+					     thread_index);
+		    }
 		}
 	    }
 	  goto out;
@@ -1208,12 +1230,18 @@ nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
 		    }
 		  else
 		    {
-		      if (sm->num_workers > 1)
-			create_bypass_for_fwd_worker (sm, b0, ip0,
-						      rx_fib_index0);
+		      if ((sm->num_workers > 1)
+			  && create_bypass_for_fwd_worker (sm, b0, ip0,
+							   rx_fib_index0,
+							   thread_index))
+			{
+			  next[0] = NAT_NEXT_OUT2IN_ED_HANDOFF;
+			}
 		      else
-			create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
-					       thread_index);
+			{
+			  create_bypass_for_fwd (sm, b0, ip0, rx_fib_index0,
+						 thread_index);
+			}
 		    }
 		}
 	      goto trace0;
@@ -1383,6 +1411,88 @@ nat44_ed_out2in_slow_path_node_fn_inline (vlib_main_t * vm,
   return frame->n_vectors;
 }
 
+static inline uword
+nat_handoff_node_fn_inline (vlib_main_t * vm,
+			    vlib_node_runtime_t * node,
+			    vlib_frame_t * frame, u32 fq_index)
+{
+  u32 n_enq, n_left_from, *from;
+
+  u16 thread_indices[VLIB_FRAME_SIZE], *ti = thread_indices;
+  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
+
+  from = vlib_frame_vector_args (frame);
+  n_left_from = frame->n_vectors;
+
+  vlib_get_buffers (vm, from, b, n_left_from);
+
+  while (n_left_from >= 4)
+    {
+      if (PREDICT_TRUE (n_left_from >= 8))
+	{
+	  vlib_prefetch_buffer_header (b[4], LOAD);
+	  vlib_prefetch_buffer_header (b[5], LOAD);
+	  vlib_prefetch_buffer_header (b[6], LOAD);
+	  vlib_prefetch_buffer_header (b[7], LOAD);
+	  CLIB_PREFETCH (&b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+	  CLIB_PREFETCH (&b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+	  CLIB_PREFETCH (&b[6]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+	  CLIB_PREFETCH (&b[7]->data, CLIB_CACHE_LINE_BYTES, LOAD);
+	}
+
+      ti[0] = vnet_buffer2 (b[0])->nat.thread_next;
+      ti[1] = vnet_buffer2 (b[1])->nat.thread_next;
+      ti[2] = vnet_buffer2 (b[2])->nat.thread_next;
+      ti[3] = vnet_buffer2 (b[3])->nat.thread_next;
+
+      b += 4;
+      ti += 4;
+      n_left_from -= 4;
+    }
+
+  while (n_left_from > 0)
+    {
+      ti[0] = vnet_buffer2 (b[0])->nat.thread_next;
+
+      b += 1;
+      ti += 1;
+      n_left_from -= 1;
+    }
+
+  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+    {
+      u32 i;
+      b = bufs;
+      ti = thread_indices;
+
+      for (i = 0; i < frame->n_vectors; i++)
+	{
+	  if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
+	    {
+	      nat44_ed_out2in_handoff_trace_t *t =
+		vlib_add_trace (vm, node, b[0], sizeof (*t));
+	      t->thread_next = ti[0];
+	      b += 1;
+	      ti += 1;
+	    }
+	  else
+	    break;
+	}
+    }
+
+  n_enq = vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices,
+					 frame->n_vectors, 1);
+
+  if (n_enq < frame->n_vectors)
+    {
+      vlib_node_increment_counter (vm, node->node_index,
+				   NAT44_HANDOFF_ERROR_CONGESTION_DROP,
+				   frame->n_vectors - n_enq);
+    }
+
+  return frame->n_vectors;
+}
+
 VLIB_NODE_FN (nat44_ed_out2in_node) (vlib_main_t * vm,
 				     vlib_node_runtime_t * node,
 				     vlib_frame_t * frame)
@@ -1431,6 +1541,35 @@ VLIB_REGISTER_NODE (nat44_ed_out2in_slowpath_node) = {
 /* *INDENT-ON* */
 
 static u8 *
+format_nat44_ed_out2in_handoff_trace (u8 * s, va_list * args)
+{
+  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+  nat44_ed_out2in_handoff_trace_t *t =
+    va_arg (*args, nat44_ed_out2in_handoff_trace_t *);
+  return format (s, "out2in ed handoff thread_next index %d", t->thread_next);
+}
+
+VLIB_NODE_FN (nat44_ed_out2in_handoff_node) (vlib_main_t * vm,
+					     vlib_node_runtime_t * node,
+					     vlib_frame_t * frame)
+{
+  return nat_handoff_node_fn_inline (vm, node, frame,
+				     snat_main.ed_out2in_node_index);
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (nat44_ed_out2in_handoff_node) = {
+  .name = "nat44-ed-out2in-handoff",
+  .vector_size = sizeof (u32),
+  .sibling_of = "nat-default",
+  .format_trace = format_nat44_ed_out2in_handoff_trace,
+  .type = VLIB_NODE_TYPE_INTERNAL,
+  .n_errors = 0,
+};
+/* *INDENT-ON* */
+
+static u8 *
 format_nat_pre_trace (u8 * s, va_list * args)
 {
   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h
index 50515c435e2..9e997b81c52 100644
--- a/src/vnet/buffer.h
+++ b/src/vnet/buffer.h
@@ -452,11 +452,10 @@ typedef struct
   /* size of L4 prototol header */
   u16 gso_l4_hdr_sz;
 
-  /* The union below has a u64 alignment, so this space is unused */
-  u32 __unused2[1];
-
   struct
   {
+    u16 unused;
+    u16 thread_next;
     u32 arc_next;
     u32 ed_out2in_nat_session_index;
   } nat;