nat: optimize flow matching in ED NAT

This saves 6 clocks in nat44-ed-in2out node. (112->106 per packet)

Type: improvement
Signed-off-by: Klement Sekera <ksekera@cisco.com>
Change-Id: I48e757e7f4b6b0d250a432a4659fe6955fc52a07

5 files changed, 87 insertions, 81 deletions

diff --git a/src/plugins/nat/nat44-ed/nat44_ed.c b/src/plugins/nat/nat44-ed/nat44_ed.c
index d9d35fc55bc..fcc3e6bf0b5 100644
--- a/src/plugins/nat/nat44-ed/nat44_ed.c
+++ b/src/plugins/nat/nat44-ed/nat44_ed.c
@@ -3318,15 +3318,15 @@ nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
       f->rewrite.dport = f->match.dport;
     }
   if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
-      f->rewrite.icmp_id != f->match.icmp_id)
+      f->rewrite.icmp_id != f->match.sport)
     {
       f->l4_csum_delta =
 	ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
-      f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.icmp_id);
+      f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
     }
   else
     {
-      f->rewrite.icmp_id = f->match.icmp_id;
+      f->rewrite.icmp_id = f->match.sport;
     }
   if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
     {
diff --git a/src/plugins/nat/nat44-ed/nat44_ed.h b/src/plugins/nat/nat44-ed/nat44_ed.h
index 10d1207fec8..902e5e34b14 100644
--- a/src/plugins/nat/nat44-ed/nat44_ed.h
+++ b/src/plugins/nat/nat44-ed/nat44_ed.h
@@ -204,15 +204,22 @@ typedef CLIB_PACKED(struct
 }) per_vrf_sessions_t;
 /* *INDENT-ON* */
 
-typedef struct
+typedef union
 {
-  ip4_address_t saddr, daddr;
-  u32 fib_index;
-  u16 sport, dport;
-  u16 icmp_id;
-  u8 proto;
+  struct
+  {
+    ip4_address_t saddr, daddr;
+    u16 sport; // ICMP id for ICMP case
+    u16 dport;
+    u32 fib_index : 24;
+    u8 proto;
+  };
+  u64 as_u64[2];
+  u64x2u as_u128;
 } nat_6t_t;
 
+STATIC_ASSERT_SIZEOF (nat_6t_t, 2 * sizeof (u64));
+
 typedef struct
 {
 #define NAT_FLOW_OP_SADDR_REWRITE   (1 << 1)
@@ -223,7 +230,15 @@ typedef struct
 #define NAT_FLOW_OP_TXFIB_REWRITE   (1 << 6)
   int ops;
   nat_6t_t match;
-  nat_6t_t rewrite;
+  struct
+  {
+    ip4_address_t saddr, daddr;
+    u16 sport;
+    u16 dport;
+    u32 fib_index;
+    u8 proto;
+    u16 icmp_id;
+  } rewrite;
   uword l3_csum_delta;
   uword l4_csum_delta;
 } nat_6t_flow_t;
diff --git a/src/plugins/nat/nat44-ed/nat44_ed_in2out.c b/src/plugins/nat/nat44-ed/nat44_ed_in2out.c
index 784dea0faf6..b99b3367329 100644
--- a/src/plugins/nat/nat44-ed/nat44_ed_in2out.c
+++ b/src/plugins/nat/nat44-ed/nat44_ed_in2out.c
@@ -918,10 +918,11 @@ nat44_ed_in2out_slowpath_unknown_proto (snat_main_t *sm, vlib_buffer_t *b,
 }
 
 static inline uword
-nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
-					  vlib_node_runtime_t * node,
-					  vlib_frame_t * frame,
-					  int is_output_feature)
+nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t *vm,
+					  vlib_node_runtime_t *node,
+					  vlib_frame_t *frame,
+					  int is_output_feature,
+					  int is_multi_worker)
 {
   u32 n_left_from, *from;
   snat_main_t *sm = &snat_main;
@@ -948,9 +949,7 @@ nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
       clib_bihash_kv_16_8_t kv0, value0;
       nat_translation_error_e translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
       nat_6t_flow_t *f = 0;
-      ip4_address_t lookup_saddr, lookup_daddr;
-      u16 lookup_sport, lookup_dport;
-      u8 lookup_protocol;
+      nat_6t_t lookup;
       int lookup_skipped = 0;
 
       b0 = *b;
@@ -981,6 +980,7 @@ nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
       rx_fib_index0 =
 	fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
+      lookup.fib_index = rx_fib_index0;
 
       if (PREDICT_FALSE (ip0->ttl == 1))
 	{
@@ -1016,8 +1016,8 @@ nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
 	      goto trace0;
 	    }
 	  int err = nat_get_icmp_session_lookup_values (
-	    b0, ip0, &lookup_saddr, &lookup_sport, &lookup_daddr,
-	    &lookup_dport, &lookup_protocol);
+	    b0, ip0, &lookup.saddr, &lookup.sport, &lookup.daddr,
+	    &lookup.dport, &lookup.proto);
 	  if (err != 0)
 	    {
 	      b0->error = node->errors[err];
@@ -1027,30 +1027,27 @@ nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
 	}
       else
 	{
-	  lookup_protocol = ip0->protocol;
-	  lookup_saddr = ip0->src_address;
-	  lookup_daddr = ip0->dst_address;
-	  lookup_sport = vnet_buffer (b0)->ip.reass.l4_src_port;
-	  lookup_dport = vnet_buffer (b0)->ip.reass.l4_dst_port;
+	  lookup.proto = ip0->protocol;
+	  lookup.saddr.as_u32 = ip0->src_address.as_u32;
+	  lookup.daddr.as_u32 = ip0->dst_address.as_u32;
+	  lookup.sport = vnet_buffer (b0)->ip.reass.l4_src_port;
+	  lookup.dport = vnet_buffer (b0)->ip.reass.l4_dst_port;
 	}
 
       /* there might be a stashed index in vnet_buffer2 from handoff or
        * classify node, see if it can be used */
-      if (!pool_is_free_index (tsm->sessions,
+      if (is_multi_worker &&
+	  !pool_is_free_index (tsm->sessions,
 			       vnet_buffer2 (b0)->nat.cached_session_index))
 	{
 	  s0 = pool_elt_at_index (tsm->sessions,
 				  vnet_buffer2 (b0)->nat.cached_session_index);
 	  if (PREDICT_TRUE (
-		nat_6t_flow_match (&s0->i2o, b0, lookup_saddr, lookup_sport,
-				   lookup_daddr, lookup_dport, lookup_protocol,
-				   rx_fib_index0)
+		nat_6t_t_eq (&s0->i2o.match, &lookup)
 		// for some hairpinning cases there are two "i2i" flows instead
 		// of i2o and o2i as both hosts are on inside
 		|| (s0->flags & SNAT_SESSION_FLAG_HAIRPINNING &&
-		    nat_6t_flow_match (
-		      &s0->o2i, b0, lookup_saddr, lookup_sport, lookup_daddr,
-		      lookup_dport, lookup_protocol, rx_fib_index0))))
+		    nat_6t_t_eq (&s0->o2i.match, &lookup))))
 	    {
 	      /* yes, this is the droid we're looking for */
 	      lookup_skipped = 1;
@@ -1059,8 +1056,8 @@ nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
 	  s0 = NULL;
 	}
 
-      init_ed_k (&kv0, ip0->src_address, lookup_sport, ip0->dst_address,
-		 lookup_dport, rx_fib_index0, lookup_protocol);
+      init_ed_k (&kv0, lookup.saddr, lookup.sport, lookup.daddr, lookup.dport,
+		 lookup.fib_index, lookup.proto);
 
       // lookup flow
       if (clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
@@ -1117,16 +1114,12 @@ nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t * vm,
 
       b0->flags |= VNET_BUFFER_F_IS_NATED;
 
-      if (nat_6t_flow_match (&s0->i2o, b0, lookup_saddr, lookup_sport,
-			     lookup_daddr, lookup_dport, lookup_protocol,
-			     rx_fib_index0))
+      if (nat_6t_t_eq (&s0->i2o.match, &lookup))
 	{
 	  f = &s0->i2o;
 	}
       else if (s0->flags & SNAT_SESSION_FLAG_HAIRPINNING &&
-	       nat_6t_flow_match (&s0->o2i, b0, lookup_saddr, lookup_sport,
-				  lookup_daddr, lookup_dport, lookup_protocol,
-				  rx_fib_index0))
+	       nat_6t_t_eq (&s0->o2i.match, &lookup))
 	{
 	  f = &s0->o2i;
 	}
@@ -1459,7 +1452,14 @@ VLIB_NODE_FN (nat44_ed_in2out_node) (vlib_main_t * vm,
 				     vlib_node_runtime_t * node,
 				     vlib_frame_t * frame)
 {
-  return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 0);
+  if (snat_main.num_workers > 1)
+    {
+      return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 0, 1);
+    }
+  else
+    {
+      return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 0, 0);
+    }
 }
 
 VLIB_REGISTER_NODE (nat44_ed_in2out_node) = {
@@ -1477,7 +1477,14 @@ VLIB_NODE_FN (nat44_ed_in2out_output_node) (vlib_main_t * vm,
 					    vlib_node_runtime_t * node,
 					    vlib_frame_t * frame)
 {
-  return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 1);
+  if (snat_main.num_workers > 1)
+    {
+      return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 1, 1);
+    }
+  else
+    {
+      return nat44_ed_in2out_fast_path_node_fn_inline (vm, node, frame, 1, 0);
+    }
 }
 
 VLIB_REGISTER_NODE (nat44_ed_in2out_output_node) = {
diff --git a/src/plugins/nat/nat44-ed/nat44_ed_inlines.h b/src/plugins/nat/nat44-ed/nat44_ed_inlines.h
index 0623940cb6d..d1cd85e2418 100644
--- a/src/plugins/nat/nat44-ed/nat44_ed_inlines.h
+++ b/src/plugins/nat/nat44-ed/nat44_ed_inlines.h
@@ -592,15 +592,9 @@ nat_6t_o2i_flow_init (snat_main_t *sm, u32 thread_idx, snat_session_t *s,
 }
 
 static_always_inline int
-nat_6t_flow_match (nat_6t_flow_t *f, vlib_buffer_t *b, ip4_address_t saddr,
-		   u16 sport, ip4_address_t daddr, u16 dport, u8 protocol,
-		   u32 fib_index)
-{
-  return f->match.daddr.as_u32 == daddr.as_u32 &&
-	 f->match.dport == vnet_buffer (b)->ip.reass.l4_dst_port &&
-	 f->match.proto == protocol && f->match.fib_index == fib_index &&
-	 f->match.saddr.as_u32 == saddr.as_u32 &&
-	 f->match.sport == vnet_buffer (b)->ip.reass.l4_src_port;
+nat_6t_t_eq (nat_6t_t *t1, nat_6t_t *t2)
+{
+  return t1->as_u64[0] == t2->as_u64[0] && t1->as_u64[1] == t2->as_u64[1];
 }
 
 static inline uword
diff --git a/src/plugins/nat/nat44-ed/nat44_ed_out2in.c b/src/plugins/nat/nat44-ed/nat44_ed_out2in.c
index 4d354d3e8ec..90f2ec32155 100644
--- a/src/plugins/nat/nat44-ed/nat44_ed_out2in.c
+++ b/src/plugins/nat/nat44-ed/nat44_ed_out2in.c
@@ -793,17 +793,12 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
       clib_bihash_kv_16_8_t kv0, value0;
       nat_translation_error_e translation_error = NAT_ED_TRNSL_ERR_SUCCESS;
       nat_6t_flow_t *f = 0;
-      ip4_address_t lookup_saddr, lookup_daddr;
-      u16 lookup_sport, lookup_dport;
-      u8 lookup_protocol;
+      nat_6t_t lookup;
       int lookup_skipped = 0;
 
       b0 = *b;
       b++;
 
-      lookup_sport = vnet_buffer (b0)->ip.reass.l4_src_port;
-      lookup_dport = vnet_buffer (b0)->ip.reass.l4_dst_port;
-
       /* Prefetch next iteration. */
       if (PREDICT_TRUE (n_left_from >= 2))
 	{
@@ -818,6 +813,9 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 
       next[0] = vnet_buffer2 (b0)->nat.arc_next;
 
+      lookup.sport = vnet_buffer (b0)->ip.reass.l4_src_port;
+      lookup.dport = vnet_buffer (b0)->ip.reass.l4_dst_port;
+
       vnet_buffer (b0)->snat.flags = 0;
       ip0 = vlib_buffer_get_current (b0);
 
@@ -825,6 +823,8 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
       rx_fib_index0 =
 	fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index0);
 
+      lookup.fib_index = rx_fib_index0;
+
       if (PREDICT_FALSE (ip0->ttl == 1))
 	{
 	  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
@@ -851,8 +851,8 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 	      goto trace0;
 	    }
 	  int err = nat_get_icmp_session_lookup_values (
-	    b0, ip0, &lookup_saddr, &lookup_sport, &lookup_daddr,
-	    &lookup_dport, &lookup_protocol);
+	    b0, ip0, &lookup.saddr, &lookup.sport, &lookup.daddr,
+	    &lookup.dport, &lookup.proto);
 	  if (err != 0)
 	    {
 	      b0->error = node->errors[err];
@@ -862,26 +862,22 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 	}
       else
 	{
-	  lookup_saddr.as_u32 = ip0->src_address.as_u32;
-	  lookup_daddr.as_u32 = ip0->dst_address.as_u32;
-	  lookup_protocol = ip0->protocol;
+	  lookup.saddr.as_u32 = ip0->src_address.as_u32;
+	  lookup.daddr.as_u32 = ip0->dst_address.as_u32;
+	  lookup.proto = ip0->protocol;
 	}
 
       /* there might be a stashed index in vnet_buffer2 from handoff or
        * classify node, see if it can be used */
-      if (!pool_is_free_index (tsm->sessions,
+      if (is_multi_worker &&
+	  !pool_is_free_index (tsm->sessions,
 			       vnet_buffer2 (b0)->nat.cached_session_index))
 	{
 	  s0 = pool_elt_at_index (tsm->sessions,
 				  vnet_buffer2 (b0)->nat.cached_session_index);
-	  if (PREDICT_TRUE (
-		nat_6t_flow_match (&s0->o2i, b0, lookup_saddr, lookup_sport,
-				   lookup_daddr, lookup_dport, lookup_protocol,
-				   rx_fib_index0) ||
-		(s0->flags & SNAT_SESSION_FLAG_TWICE_NAT &&
-		 nat_6t_flow_match (&s0->i2o, b0, lookup_saddr, lookup_sport,
-				    lookup_daddr, lookup_dport,
-				    lookup_protocol, rx_fib_index0))))
+	  if (PREDICT_TRUE (nat_6t_t_eq (&s0->o2i.match, &lookup)) ||
+	      (s0->flags & SNAT_SESSION_FLAG_TWICE_NAT &&
+	       nat_6t_t_eq (&s0->i2o.match, &lookup)))
 	    {
 	      /* yes, this is the droid we're looking for */
 	      lookup_skipped = 1;
@@ -890,8 +886,8 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 	  s0 = NULL;
 	}
 
-      init_ed_k (&kv0, lookup_saddr, lookup_sport, lookup_daddr, lookup_dport,
-		 rx_fib_index0, lookup_protocol);
+      init_ed_k (&kv0, lookup.saddr, lookup.sport, lookup.daddr, lookup.dport,
+		 lookup.fib_index, lookup.proto);
 
       // lookup flow
       if (clib_bihash_search_16_8 (&sm->flow_hash, &kv0, &value0))
@@ -944,16 +940,12 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 	  goto trace0;
 	}
 
-      if (nat_6t_flow_match (&s0->o2i, b0, lookup_saddr, lookup_sport,
-			     lookup_daddr, lookup_dport, lookup_protocol,
-			     rx_fib_index0))
+      if (nat_6t_t_eq (&s0->o2i.match, &lookup))
 	{
 	  f = &s0->o2i;
 	}
       else if (s0->flags & SNAT_SESSION_FLAG_TWICE_NAT &&
-	       nat_6t_flow_match (&s0->i2o, b0, lookup_saddr, lookup_sport,
-				  lookup_daddr, lookup_dport, lookup_protocol,
-				  rx_fib_index0))
+	       nat_6t_t_eq (&s0->i2o.match, &lookup))
 	{
 	  f = &s0->i2o;
 	}
@@ -979,9 +971,7 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm,
 	    }
 	  else
 	    {
-	      if (nat_6t_flow_match (&s0->i2o, b0, lookup_saddr, lookup_sport,
-				     lookup_daddr, lookup_dport,
-				     lookup_protocol, rx_fib_index0))
+	      if (nat_6t_t_eq (&s0->i2o.match, &lookup))
 		{
 		  f = &s0->i2o;
 		}