nat: handoff rewrite & fixes for multi-worker

Type: fix

Change-Id: Ib9164d8f6c681e8900e645306f3a2dc0ac0e40a8
Signed-off-by: Filip Varga <filipvarga89@gmail.com>

2 files changed, 163 insertions, 60 deletions

diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
index 1f27fb8c788..85c42027be0 100755
--- a/src/plugins/nat/nat.c
+++ b/src/plugins/nat/nat.c
@@ -2554,26 +2554,49 @@ snat_static_mapping_match (snat_main_t * sm,
 	{
 	  if (PREDICT_FALSE (lb != 0))
 	    *lb = m->affinity ? AFFINITY_LB_NAT : LB_NAT;
-	  if (m->affinity)
+	  if (m->affinity && !nat_affinity_find_and_lock (ext_host_addr[0],
+							  match.addr,
+							  match.protocol,
+							  match.port,
+							  &backend_index))
 	    {
-	      if (nat_affinity_find_and_lock (ext_host_addr[0], match.addr,
-					      match.protocol, match.port,
-					      &backend_index))
-		goto get_local;
-
 	      local = pool_elt_at_index (m->locals, backend_index);
 	      mapping->addr = local->addr;
 	      mapping->port = clib_host_to_net_u16 (local->port);
 	      mapping->fib_index = local->fib_index;
 	      goto end;
 	    }
-	get_local:
-          /* *INDENT-OFF* */
-          pool_foreach_index (i, m->locals,
-          ({
-            vec_add1 (tmp, i);
-          }));
-          /* *INDENT-ON* */
+	  // pick locals matching this worker
+	  if (PREDICT_FALSE (sm->num_workers > 1))
+	    {
+	      u32 thread_index = vlib_get_thread_index ();
+              /* *INDENT-OFF* */
+              pool_foreach_index (i, m->locals,
+              ({
+                local = pool_elt_at_index (m->locals, i);
+
+                ip4_header_t ip = {
+		  .src_address = local->addr,
+	        };
+
+	        if (sm->worker_in2out_cb (&ip, m->fib_index) ==
+		    thread_index)
+                  {
+                    vec_add1 (tmp, i);
+                  }
+              }));
+              /* *INDENT-ON* */
+	      ASSERT (vec_len (tmp) != 0);
+	    }
+	  else
+	    {
+              /* *INDENT-OFF* */
+              pool_foreach_index (i, m->locals,
+              ({
+                vec_add1 (tmp, i);
+              }));
+              /* *INDENT-ON* */
+	    }
 	  hi = vec_len (tmp) - 1;
 	  local = pool_elt_at_index (m->locals, tmp[hi]);
 	  rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
@@ -2586,15 +2609,6 @@ snat_static_mapping_match (snat_main_t * sm,
 	  local = pool_elt_at_index (m->locals, tmp[lo]);
 	  if (!(local->prefix >= rand))
 	    return 1;
-	  if (PREDICT_FALSE (sm->num_workers > 1))
-	    {
-	      ip4_header_t ip = {
-		.src_address = local->addr,
-	      };
-	      if (sm->worker_in2out_cb (&ip, m->fib_index) !=
-		  vlib_get_thread_index ())
-		goto get_local;
-	    }
 	  mapping->addr = local->addr;
 	  mapping->port = clib_host_to_net_u16 (local->port);
 	  mapping->fib_index = local->fib_index;
diff --git a/src/plugins/nat/nat44_handoff.c b/src/plugins/nat/nat44_handoff.c
index e4cb483a458..04590e409ac 100644
--- a/src/plugins/nat/nat44_handoff.c
+++ b/src/plugins/nat/nat44_handoff.c
@@ -68,35 +68,29 @@ format_nat44_handoff_trace (u8 * s, va_list * args)
 }
 
 static inline uword
-nat44_worker_handoff_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+nat44_worker_handoff_fn_inline (vlib_main_t * vm,
+				vlib_node_runtime_t * node,
 				vlib_frame_t * frame, u8 is_output,
 				u8 is_in2out)
 {
+  u32 n_enq, n_left_from, *from, do_handoff = 0, same_worker = 0;
+
+  u16 thread_indices[VLIB_FRAME_SIZE], *ti = thread_indices;
+  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
   snat_main_t *sm = &snat_main;
-  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
-  u32 n_enq, n_left_from, *from;
-  u16 thread_indices[VLIB_FRAME_SIZE], *ti;
-  u32 fq_index;
+
   snat_get_worker_function_t *get_worker;
-  u32 thread_index = vm->thread_index;
-  u32 do_handoff = 0, same_worker = 0;
+  u32 fq_index, thread_index = vm->thread_index;
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;
-  vlib_get_buffers (vm, from, bufs, n_left_from);
 
-  b = bufs;
-  ti = thread_indices;
-
-  ASSERT (vec_len (sm->workers));
+  vlib_get_buffers (vm, from, b, n_left_from);
 
   if (is_in2out)
     {
+      fq_index = is_output ? sm->fq_in2out_output_index : sm->fq_in2out_index;
       get_worker = sm->worker_in2out_cb;
-      if (is_output)
-	fq_index = sm->fq_in2out_output_index;
-      else
-	fq_index = sm->fq_in2out_index;
     }
   else
     {
@@ -104,45 +98,138 @@ nat44_worker_handoff_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       get_worker = sm->worker_out2in_cb;
     }
 
-  while (n_left_from > 0)
+  while (n_left_from >= 4)
     {
-      u32 sw_if_index0;
-      u32 rx_fib_index0;
-      ip4_header_t *ip0;
+      u32 sw_if_index0, sw_if_index1, sw_if_index2, sw_if_index3;
+      u32 rx_fib_index0 = 0, rx_fib_index1 = 0,
+	rx_fib_index2 = 0, rx_fib_index3 = 0;
+      ip4_header_t *ip0, *ip1, *ip2, *ip3;
+
+      if (PREDICT_TRUE (n_left_from >= 8))
+	{
+	  vlib_prefetch_buffer_header (b[4], STORE);
+	  vlib_prefetch_buffer_header (b[5], STORE);
+	  vlib_prefetch_buffer_header (b[6], STORE);
+	  vlib_prefetch_buffer_header (b[7], STORE);
+	  CLIB_PREFETCH (&b[4]->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  CLIB_PREFETCH (&b[5]->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  CLIB_PREFETCH (&b[6]->data, CLIB_CACHE_LINE_BYTES, STORE);
+	  CLIB_PREFETCH (&b[7]->data, CLIB_CACHE_LINE_BYTES, STORE);
+	}
 
-      sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
-      rx_fib_index0 = ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
       ip0 = vlib_buffer_get_current (b[0]);
+      ip1 = vlib_buffer_get_current (b[1]);
+      ip2 = vlib_buffer_get_current (b[2]);
+      ip3 = vlib_buffer_get_current (b[3]);
+
+      if (PREDICT_FALSE (is_in2out))
+	{
+	  sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
+	  sw_if_index1 = vnet_buffer (b[1])->sw_if_index[VLIB_RX];
+	  sw_if_index2 = vnet_buffer (b[2])->sw_if_index[VLIB_RX];
+	  sw_if_index3 = vnet_buffer (b[3])->sw_if_index[VLIB_RX];
+
+	  rx_fib_index0 =
+	    ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
+	  rx_fib_index1 =
+	    ip4_fib_table_get_index_for_sw_if_index (sw_if_index1);
+	  rx_fib_index2 =
+	    ip4_fib_table_get_index_for_sw_if_index (sw_if_index2);
+	  rx_fib_index3 =
+	    ip4_fib_table_get_index_for_sw_if_index (sw_if_index3);
+	}
+
       ti[0] = get_worker (ip0, rx_fib_index0);
+      ti[1] = get_worker (ip1, rx_fib_index1);
+      ti[2] = get_worker (ip2, rx_fib_index2);
+      ti[3] = get_worker (ip3, rx_fib_index3);
 
-      if (ti[0] != thread_index)
+      if (ti[0] == thread_index)
+	same_worker++;
+      else
 	do_handoff++;
+
+      if (ti[1] == thread_index)
+	same_worker++;
+      else
+	do_handoff++;
+
+      if (ti[2] == thread_index)
+	same_worker++;
       else
+	do_handoff++;
+
+      if (ti[3] == thread_index)
 	same_worker++;
+      else
+	do_handoff++;
+
+      b += 4;
+      ti += 4;
+      n_left_from -= 4;
+    }
 
-      if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
-			 && (b[0]->flags & VLIB_BUFFER_IS_TRACED)))
+  while (n_left_from > 0)
+    {
+      u32 sw_if_index0;
+      u32 rx_fib_index0 = 0;
+      ip4_header_t *ip0;
+
+      ip0 = vlib_buffer_get_current (b[0]);
+
+      if (PREDICT_FALSE (is_in2out))
 	{
-	  nat44_handoff_trace_t *t =
-	    vlib_add_trace (vm, node, b[0], sizeof (*t));
-	  t->next_worker_index = ti[0];
-	  t->trace_index = vlib_buffer_get_trace_index (b[0]);
-	  t->in2out = is_in2out;
+	  sw_if_index0 = vnet_buffer (b[0])->sw_if_index[VLIB_RX];
+	  rx_fib_index0 =
+	    ip4_fib_table_get_index_for_sw_if_index (sw_if_index0);
 	}
 
-      n_left_from -= 1;
-      ti += 1;
+      ti[0] = get_worker (ip0, rx_fib_index0);
+
+      if (ti[0] == thread_index)
+	same_worker++;
+      else
+	do_handoff++;
+
       b += 1;
+      ti += 1;
+      n_left_from -= 1;
+    }
+
+  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
+    {
+      u32 i;
+      b = bufs;
+      ti = thread_indices;
+
+      for (i = 0; i < frame->n_vectors; i++)
+	{
+	  if (b[0]->flags & VLIB_BUFFER_IS_TRACED)
+	    {
+	      nat44_handoff_trace_t *t =
+		vlib_add_trace (vm, node, b[0], sizeof (*t));
+	      t->next_worker_index = ti[0];
+	      t->trace_index = vlib_buffer_get_trace_index (b[0]);
+	      t->in2out = is_in2out;
+
+	      b++;
+	      ti++;
+	    }
+	  else
+	    break;
+	}
     }
 
-  n_enq =
-    vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices,
-				   frame->n_vectors, 1);
+  n_enq = vlib_buffer_enqueue_to_thread (vm, fq_index, from, thread_indices,
+					 frame->n_vectors, 1);
 
   if (n_enq < frame->n_vectors)
-    vlib_node_increment_counter (vm, node->node_index,
-				 NAT44_HANDOFF_ERROR_CONGESTION_DROP,
-				 frame->n_vectors - n_enq);
+    {
+      vlib_node_increment_counter (vm, node->node_index,
+				   NAT44_HANDOFF_ERROR_CONGESTION_DROP,
+				   frame->n_vectors - n_enq);
+    }
+
   vlib_node_increment_counter (vm, node->node_index,
 			       NAT44_HANDOFF_ERROR_SAME_WORKER, same_worker);
   vlib_node_increment_counter (vm, node->node_index,
@@ -150,6 +237,8 @@ nat44_worker_handoff_fn_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   return frame->n_vectors;
 }
 
+
+
 VLIB_NODE_FN (snat_in2out_worker_handoff_node) (vlib_main_t * vm,
 						vlib_node_runtime_t * node,
 						vlib_frame_t * frame)