nat: deal with flows instead of sessions

This change introduces flow concept to endpoint-dependent NAT. Instead
of having a session and a plethora of special cases in code for e.g.
hairpinning, twice-nat and others, figure all this out and store it in
flow logic. Every flow has a match and a rewrite part. This unifies all
the NAT packet processing cases into one - match a flow and rewrite the
packet based on that flow. It also provides a cure for hairpinning
dilemma where one part of the flow is on one worker and another on
a different one. These cases are also sped up by not requiring
destination adress lookup every single time to be able to rewrite source
nat as this is now part of flow rewrite logic.

Type: improvement
Change-Id: Ib60c992e16792ea4d4129bc10202ebb99a73b5be
Signed-off-by: Klement Sekera <ksekera@cisco.com>

1 files changed, 425 insertions, 177 deletions

diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
index 81af143ab2c..245689db45d 100644
--- a/src/plugins/nat/nat.c
+++ b/src/plugins/nat/nat.c
@@ -37,6 +37,7 @@
 #include <nat/nat44-ei/nat44_ei.h>
 
 #include <vpp/app/version.h>
+#include <nat/lib/nat_inlines.h>
 
 snat_main_t snat_main;
 
@@ -152,11 +153,6 @@ VNET_FEATURE_INIT (ip4_snat_hairpin_dst, static) = {
   .node_name = "nat44-hairpin-dst",
   .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
 };
-VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_dst, static) = {
-  .arc_name = "ip4-unicast",
-  .node_name = "nat44-ed-hairpin-dst",
-  .runs_after = VNET_FEATURES ("acl-plugin-in-ip4-fa","ip4-sv-reassembly-feature"),
-};
 
 /* Hook up output features */
 VNET_FEATURE_INIT (ip4_snat_in2out_output, static) = {
@@ -186,12 +182,6 @@ VNET_FEATURE_INIT (ip4_nat44_ed_in2out_output, static) = {
   .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
   .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
 };
-VNET_FEATURE_INIT (ip4_nat44_ed_hairpin_src, static) = {
-  .arc_name = "ip4-output",
-  .node_name = "nat44-ed-hairpin-src",
-  .runs_after = VNET_FEATURES ("ip4-sv-reassembly-output-feature"),
-  .runs_before = VNET_FEATURES ("acl-plugin-out-ip4-fa"),
-};
 
 /* Hook up ip4-local features */
 VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
@@ -200,13 +190,6 @@ VNET_FEATURE_INIT (ip4_nat_hairpinning, static) =
   .node_name = "nat44-hairpinning",
   .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
 };
-VNET_FEATURE_INIT (ip4_nat44_ed_hairpinning, static) =
-{
-  .arc_name = "ip4-local",
-  .node_name = "nat44-ed-hairpinning",
-  .runs_before = VNET_FEATURES("ip4-local-end-of-arc"),
-};
-
 
 VLIB_PLUGIN_REGISTER () = {
     .version = VPP_BUILD_VER,
@@ -275,13 +258,13 @@ format_ed_session_kvp (u8 * s, va_list * args)
   u32 fib_index;
 
   split_ed_kv (v, &l_addr, &r_addr, &proto, &fib_index, &l_port, &r_port);
-  s =
-    format (s,
-	    "local %U:%d remote %U:%d proto %U fib %d thread-index %u session-index %u",
-	    format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
-	    format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
-	    format_ip_protocol, proto, fib_index,
-	    ed_value_get_session_index (v), ed_value_get_thread_index (v));
+  s = format (s,
+	      "local %U:%d remote %U:%d proto %U fib %d thread-index %u "
+	      "session-index %u",
+	      format_ip4_address, &l_addr, clib_net_to_host_u16 (l_port),
+	      format_ip4_address, &r_addr, clib_net_to_host_u16 (r_port),
+	      format_ip_protocol, proto, fib_index,
+	      ed_value_get_thread_index (v), ed_value_get_session_index (v));
 
   return s;
 }
@@ -291,39 +274,22 @@ nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
 		       u8 is_ha)
 {
   clib_bihash_kv_8_8_t kv;
-  u8 proto;
-  u16 r_port, l_port;
-  ip4_address_t *l_addr, *r_addr;
-  u32 fib_index = 0;
-  clib_bihash_kv_16_8_t ed_kv;
   snat_main_per_thread_data_t *tsm =
     vec_elt_at_index (sm->per_thread_data, thread_index);
 
   if (is_ed_session (s))
     {
       per_vrf_sessions_unregister_session (s, thread_index);
+
+      if (nat_ed_ses_i2o_flow_hash_add_del (sm, thread_index, s, 0))
+	nat_elog_warn ("flow hash del failed");
+
+      if (nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 0))
+	nat_elog_warn ("flow hash del failed");
     }
 
   if (is_fwd_bypass_session (s))
     {
-      if (snat_is_unk_proto_session (s))
-	{
-	  init_ed_k (&ed_kv, s->in2out.addr, 0, s->ext_host_addr, 0, 0,
-		     s->in2out.port);
-	}
-      else
-	{
-	  l_port = s->in2out.port;
-	  r_port = s->ext_host_port;
-	  l_addr = &s->in2out.addr;
-	  r_addr = &s->ext_host_addr;
-	  proto = nat_proto_to_ip_proto (s->nat_proto);
-	  fib_index = s->in2out.fib_index;
-	  init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index,
-		     proto);
-	}
-      if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
-	nat_elog_warn ("in2out_ed key del failed");
       return;
     }
 
@@ -333,36 +299,6 @@ nat_free_session_data (snat_main_t * sm, snat_session_t * s, u32 thread_index,
       if (is_affinity_sessions (s))
 	nat_affinity_unlock (s->ext_host_addr, s->out2in.addr,
 			     s->nat_proto, s->out2in.port);
-      l_addr = &s->out2in.addr;
-      r_addr = &s->ext_host_addr;
-      fib_index = s->out2in.fib_index;
-      if (snat_is_unk_proto_session (s))
-	{
-	  proto = s->in2out.port;
-	  r_port = 0;
-	  l_port = 0;
-	}
-      else
-	{
-	  proto = nat_proto_to_ip_proto (s->nat_proto);
-	  l_port = s->out2in.port;
-	  r_port = s->ext_host_port;
-	}
-      init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
-      if (clib_bihash_add_del_16_8 (&sm->out2in_ed, &ed_kv, 0))
-	nat_elog_warn ("out2in_ed key del failed");
-      l_addr = &s->in2out.addr;
-      fib_index = s->in2out.fib_index;
-      if (!snat_is_unk_proto_session (s))
-	l_port = s->in2out.port;
-      if (is_twice_nat_session (s))
-	{
-	  r_addr = &s->ext_host_nat_addr;
-	  r_port = s->ext_host_nat_port;
-	}
-      init_ed_k (&ed_kv, *l_addr, l_port, *r_addr, r_port, fib_index, proto);
-      if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, &ed_kv, 0))
-	nat_elog_warn ("in2out_ed key del failed");
 
       if (!is_ha)
 	nat_syslog_nat44_sdel (s->user_index, s->in2out.fib_index,
@@ -1985,18 +1921,14 @@ feature_set:
                                              sw_if_index, 1, 0, 0);
                 if (!is_inside)
                   {
-                    if (sm->endpoint_dependent)
-                      vnet_feature_enable_disable ("ip4-local",
-                                                   "nat44-ed-hairpinning",
-                                                   sw_if_index, 1, 0, 0);
-                    else
-                      vnet_feature_enable_disable ("ip4-local",
-                                                   "nat44-hairpinning",
-                                                   sw_if_index, 1, 0, 0);
-                  }
-              }
-            else
-              {
+		    if (!sm->endpoint_dependent)
+		      vnet_feature_enable_disable ("ip4-local",
+						   "nat44-hairpinning",
+						   sw_if_index, 1, 0, 0);
+		  }
+	      }
+	    else
+	      {
 		int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 0);
 		if (rv)
 		  return rv;
@@ -2005,41 +1937,38 @@ feature_set:
                 pool_put (sm->interfaces, i);
                 if (is_inside)
                   {
-                    if (sm->endpoint_dependent)
-                      vnet_feature_enable_disable ("ip4-local",
-                                                   "nat44-ed-hairpinning",
-                                                   sw_if_index, 0, 0, 0);
-                    else
-                      vnet_feature_enable_disable ("ip4-local",
-                                                   "nat44-hairpinning",
-                                                   sw_if_index, 0, 0, 0);
-                  }
-              }
-          }
-        else
-          {
-            if ((nat_interface_is_inside(i) && is_inside) ||
-                (nat_interface_is_outside(i) && !is_inside))
-              return 0;
+		    if (!sm->endpoint_dependent)
+		      vnet_feature_enable_disable ("ip4-local",
+						   "nat44-hairpinning",
+						   sw_if_index, 0, 0, 0);
+		  }
+	      }
+	  }
+	else
+	  {
+	    if ((nat_interface_is_inside (i) && is_inside) ||
+		(nat_interface_is_outside (i) && !is_inside))
+	      return 0;
 
-            if (sm->num_workers > 1)
-              {
-                del_feature_name = !is_inside ?  "nat44-in2out-worker-handoff" :
-                                                 "nat44-out2in-worker-handoff";
-                feature_name = "nat44-handoff-classify";
-              }
-            else if (sm->endpoint_dependent)
-              {
-                del_feature_name = !is_inside ?  "nat-pre-in2out" :
-                                                 "nat-pre-out2in";
+	    if (sm->num_workers > 1)
+	      {
+		del_feature_name = !is_inside ? "nat44-in2out-worker-handoff" :
+						"nat44-out2in-worker-handoff";
+		feature_name = "nat44-handoff-classify";
+	      }
+	    else if (sm->endpoint_dependent)
+	      {
+		del_feature_name =
+		  !is_inside ? "nat-pre-in2out" : "nat-pre-out2in";
 
-                feature_name = "nat44-ed-classify";
-              }
-            else
-              {
-                del_feature_name = !is_inside ?  "nat44-in2out" : "nat44-out2in";
-                feature_name = "nat44-classify";
-              }
+		feature_name = "nat44-ed-classify";
+	      }
+	    else
+	      {
+		del_feature_name =
+		  !is_inside ? "nat44-in2out" : "nat44-out2in";
+		feature_name = "nat44-classify";
+	      }
 
 	    int rv = ip4_sv_reass_enable_disable_with_refcnt (sw_if_index, 1);
 	    if (rv)
@@ -2050,17 +1979,14 @@ feature_set:
                                          sw_if_index, 1, 0, 0);
             if (!is_inside)
               {
-                if (sm->endpoint_dependent)
-                  vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
-                                               sw_if_index, 0, 0, 0);
-                else
-                  vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
-                                               sw_if_index, 0, 0, 0);
-              }
-            goto set_flags;
-          }
+		if (!sm->endpoint_dependent)
+		  vnet_feature_enable_disable (
+		    "ip4-local", "nat44-hairpinning", sw_if_index, 0, 0, 0);
+	      }
+	    goto set_flags;
+	  }
 
-        goto fib;
+	goto fib;
       }
   }
   /* *INDENT-ON* */
@@ -2085,10 +2011,7 @@ feature_set:
 
   if (is_inside && !sm->out2in_dpo)
     {
-      if (sm->endpoint_dependent)
-	vnet_feature_enable_disable ("ip4-local", "nat44-ed-hairpinning",
-				     sw_if_index, 1, 0, 0);
-      else
+      if (!sm->endpoint_dependent)
 	vnet_feature_enable_disable ("ip4-local", "nat44-hairpinning",
 				     sw_if_index, 1, 0, 0);
     }
@@ -2199,10 +2122,6 @@ feature_set:
 							    !is_del);
 	  if (rv)
 	    return rv;
-	  vnet_feature_enable_disable ("ip4-unicast", "nat44-ed-hairpin-dst",
-				       sw_if_index, !is_del, 0, 0);
-	  vnet_feature_enable_disable ("ip4-output", "nat44-ed-hairpin-src",
-				       sw_if_index, !is_del, 0, 0);
 	}
       else
 	{
@@ -2575,12 +2494,6 @@ nat44_set_node_indexes (snat_main_t * sm, vlib_main_t * vm)
   sm->hairpin_dst_node_index = node->index;
   node = vlib_get_node_by_name (vm, (u8 *) "nat44-hairpin-src");
   sm->hairpin_src_node_index = node->index;
-  node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpinning");
-  sm->ed_hairpinning_node_index = node->index;
-  node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-dst");
-  sm->ed_hairpin_dst_node_index = node->index;
-  node = vlib_get_node_by_name (vm, (u8 *) "nat44-ed-hairpin-src");
-  sm->ed_hairpin_src_node_index = node->index;
 }
 
 #define nat_init_simple_counter(c, n, sn) \
@@ -2778,8 +2691,8 @@ nat44_ed_plugin_enable (nat44_config_t c)
 
   if (sm->pat)
     {
-      sm->icmp_match_in2out_cb = icmp_match_in2out_ed;
-      sm->icmp_match_out2in_cb = icmp_match_out2in_ed;
+      sm->icmp_match_in2out_cb = NULL;
+      sm->icmp_match_out2in_cb = NULL;
     }
   else
     {
@@ -2907,8 +2820,6 @@ nat44_ed_plugin_disable ()
 
   vec_free (sm->max_translations_per_fib);
 
-  nat_affinity_disable ();
-
   nat44_ed_db_free ();
 
   nat44_addresses_free (&sm->addresses);
@@ -3258,8 +3169,8 @@ nat44_ed_get_worker_in2out_cb (ip4_header_t *ip, u32 rx_fib_index,
       init_ed_k (&kv16, ip->src_address, udp->src_port, ip->dst_address,
 		 udp->dst_port, fib_index, ip->protocol);
 
-      if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
-						  &kv16, &value16)))
+      if (PREDICT_TRUE (
+	    !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
 	{
 	  tsm =
 	    vec_elt_at_index (sm->per_thread_data,
@@ -3327,13 +3238,13 @@ nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
       init_ed_k (&kv16, ip->dst_address, udp->dst_port, ip->src_address,
 		 udp->src_port, rx_fib_index, ip->protocol);
 
-      if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
-						  &kv16, &value16)))
+      if (PREDICT_TRUE (
+	    !clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
 	{
 	  tsm =
 	    vec_elt_at_index (sm->per_thread_data,
 			      ed_value_get_thread_index (&value16));
-	  vnet_buffer2 (b)->nat.ed_out2in_nat_session_index =
+	  vnet_buffer2 (b)->nat.cached_session_index =
 	    ed_value_get_session_index (&value16);
 	  next_worker_index = sm->first_worker_index + tsm->thread_index;
 	  nat_elog_debug_handoff ("HANDOFF OUT2IN (session)",
@@ -3347,10 +3258,17 @@ nat44_ed_get_worker_out2in_cb (vlib_buffer_t * b, ip4_header_t * ip,
     }
   else if (proto == NAT_PROTOCOL_ICMP)
     {
-      if (!get_icmp_o2i_ed_key (b, ip, rx_fib_index, ~0, ~0, 0, 0, 0, &kv16))
+      ip4_address_t lookup_saddr, lookup_daddr;
+      u16 lookup_sport, lookup_dport;
+      u8 lookup_protocol;
+      if (!nat_get_icmp_session_lookup_values (
+	    b, ip, &lookup_saddr, &lookup_sport, &lookup_daddr, &lookup_dport,
+	    &lookup_protocol))
 	{
-	  if (PREDICT_TRUE (!clib_bihash_search_16_8 (&sm->out2in_ed,
-						      &kv16, &value16)))
+	  init_ed_k (&kv16, lookup_saddr, lookup_sport, lookup_daddr,
+		     lookup_dport, rx_fib_index, lookup_protocol);
+	  if (PREDICT_TRUE (
+		!clib_bihash_search_16_8 (&sm->flow_hash, &kv16, &value16)))
 	    {
 	      tsm =
 		vec_elt_at_index (sm->per_thread_data,
@@ -3558,9 +3476,6 @@ nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations,
   tsm->unk_proto_lru_head_index = head - tsm->lru_pool;
   clib_dlist_init (tsm->lru_pool, tsm->unk_proto_lru_head_index);
 
-  clib_bihash_init_16_8 (&tsm->in2out_ed, "in2out-ed", translation_buckets, 0);
-  clib_bihash_set_kvp_format_fn_16_8 (&tsm->in2out_ed, format_ed_session_kvp);
-
   // TODO: ED nat is not using these
   // before removal large refactor required
   pool_alloc (tsm->list_pool, translations);
@@ -3569,6 +3484,17 @@ nat44_ed_worker_db_init (snat_main_per_thread_data_t *tsm, u32 translations,
 }
 
 static void
+reinit_ed_flow_hash ()
+{
+  snat_main_t *sm = &snat_main;
+  // we expect 2 flows per session, so multiply translation_buckets by 2
+  clib_bihash_init_16_8 (
+    &sm->flow_hash, "ed-flow-hash",
+    clib_max (1, sm->num_workers) * 2 * sm->translation_buckets, 0);
+  clib_bihash_set_kvp_format_fn_16_8 (&sm->flow_hash, format_ed_session_kvp);
+}
+
+static void
 nat44_ed_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
 {
   snat_main_t *sm = &snat_main;
@@ -3576,8 +3502,7 @@ nat44_ed_db_init (u32 translations, u32 translation_buckets, u32 user_buckets)
   u32 static_mapping_buckets = 1024;
   u32 static_mapping_memory_size = 64 << 20;
 
-  clib_bihash_init_16_8 (&sm->out2in_ed, "out2in-ed", translation_buckets, 0);
-  clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed, format_ed_session_kvp);
+  reinit_ed_flow_hash ();
 
   clib_bihash_init_8_8 (&sm->static_mapping_by_local,
 			"static_mapping_by_local", static_mapping_buckets,
@@ -3607,7 +3532,6 @@ nat44_ed_worker_db_free (snat_main_per_thread_data_t *tsm)
   pool_free (tsm->sessions);
   pool_free (tsm->lru_pool);
 
-  clib_bihash_free_16_8 (&tsm->in2out_ed);
   vec_free (tsm->per_vrf_sessions_vec);
 
   // TODO: resolve static mappings (put only to !ED)
@@ -3623,7 +3547,7 @@ nat44_ed_db_free ()
   snat_main_per_thread_data_t *tsm;
 
   pool_free (sm->static_mappings);
-  clib_bihash_free_16_8 (&sm->out2in_ed);
+  clib_bihash_free_16_8 (&sm->flow_hash);
   clib_bihash_free_8_8 (&sm->static_mapping_by_local);
   clib_bihash_free_8_8 (&sm->static_mapping_by_external);
 
@@ -3642,11 +3566,7 @@ nat44_ed_sessions_clear ()
   snat_main_t *sm = &snat_main;
   snat_main_per_thread_data_t *tsm;
 
-  clib_bihash_free_16_8 (&sm->out2in_ed);
-  clib_bihash_init_16_8 (
-    &sm->out2in_ed, "out2in-ed",
-    clib_max (1, sm->num_workers) * sm->translation_buckets, 0);
-  clib_bihash_set_kvp_format_fn_16_8 (&sm->out2in_ed, format_ed_session_kvp);
+  reinit_ed_flow_hash ();
 
   if (sm->pat)
     {
@@ -3896,7 +3816,6 @@ nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
 		      u32 vrf_id, int is_in)
 {
   ip4_header_t ip;
-  clib_bihash_16_8_t *t;
   clib_bihash_kv_16_8_t kv, value;
   u32 fib_index = fib_table_find (FIB_PROTOCOL_IP4, vrf_id);
   snat_session_t *s;
@@ -3913,16 +3832,15 @@ nat44_del_ed_session (snat_main_t * sm, ip4_address_t * addr, u16 port,
   else
     tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
 
-  t = is_in ? &tsm->in2out_ed : &sm->out2in_ed;
   init_ed_k (&kv, *addr, port, *eh_addr, eh_port, fib_index, proto);
-  if (clib_bihash_search_16_8 (t, &kv, &value))
+  if (clib_bihash_search_16_8 (&sm->flow_hash, &kv, &value))
     {
       return VNET_API_ERROR_NO_SUCH_ENTRY;
     }
 
-  if (pool_is_free_index (tsm->sessions, value.value))
+  if (pool_is_free_index (tsm->sessions, ed_value_get_session_index (&value)))
     return VNET_API_ERROR_UNSPECIFIED;
-  s = pool_elt_at_index (tsm->sessions, value.value);
+  s = pool_elt_at_index (tsm->sessions, ed_value_get_session_index (&value));
   nat_free_session_data (sm, s, tsm - sm->per_thread_data, 0);
   nat_ed_session_delete (sm, s, tsm - sm->per_thread_data, 1);
   return 0;
@@ -3952,13 +3870,343 @@ VLIB_REGISTER_NODE (nat_default_node) = {
     [NAT_NEXT_IN2OUT_ED_OUTPUT_SLOW_PATH] = "nat44-ed-in2out-output-slowpath",
     [NAT_NEXT_OUT2IN_ED_FAST_PATH] = "nat44-ed-out2in",
     [NAT_NEXT_OUT2IN_ED_SLOW_PATH] = "nat44-ed-out2in-slowpath",
-    [NAT_NEXT_OUT2IN_ED_HANDOFF] = "nat44-ed-out2in-handoff",
     [NAT_NEXT_IN2OUT_CLASSIFY] = "nat44-in2out-worker-handoff",
     [NAT_NEXT_OUT2IN_CLASSIFY] = "nat44-out2in-worker-handoff",
   },
 };
 /* *INDENT-ON* */
 
+void
+nat_6t_l3_l4_csum_calc (nat_6t_flow_t *f)
+{
+  f->l3_csum_delta = 0;
+  f->l4_csum_delta = 0;
+  if (f->ops & NAT_FLOW_OP_SADDR_REWRITE &&
+      f->rewrite.saddr.as_u32 != f->match.saddr.as_u32)
+    {
+      f->l3_csum_delta =
+	ip_csum_add_even (f->l3_csum_delta, f->rewrite.saddr.as_u32);
+      f->l3_csum_delta =
+	ip_csum_sub_even (f->l3_csum_delta, f->match.saddr.as_u32);
+    }
+  else
+    {
+      f->rewrite.saddr.as_u32 = f->match.saddr.as_u32;
+    }
+  if (f->ops & NAT_FLOW_OP_DADDR_REWRITE &&
+      f->rewrite.daddr.as_u32 != f->match.daddr.as_u32)
+    {
+      f->l3_csum_delta =
+	ip_csum_add_even (f->l3_csum_delta, f->rewrite.daddr.as_u32);
+      f->l3_csum_delta =
+	ip_csum_sub_even (f->l3_csum_delta, f->match.daddr.as_u32);
+    }
+  else
+    {
+      f->rewrite.daddr.as_u32 = f->match.daddr.as_u32;
+    }
+  if (f->ops & NAT_FLOW_OP_SPORT_REWRITE && f->rewrite.sport != f->match.sport)
+    {
+      f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.sport);
+      f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.sport);
+    }
+  else
+    {
+      f->rewrite.sport = f->match.sport;
+    }
+  if (f->ops & NAT_FLOW_OP_DPORT_REWRITE && f->rewrite.dport != f->match.dport)
+    {
+      f->l4_csum_delta = ip_csum_add_even (f->l4_csum_delta, f->rewrite.dport);
+      f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.dport);
+    }
+  else
+    {
+      f->rewrite.dport = f->match.dport;
+    }
+  if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE &&
+      f->rewrite.icmp_id != f->match.icmp_id)
+    {
+      f->l4_csum_delta =
+	ip_csum_add_even (f->l4_csum_delta, f->rewrite.icmp_id);
+      f->l4_csum_delta = ip_csum_sub_even (f->l4_csum_delta, f->match.icmp_id);
+    }
+  else
+    {
+      f->rewrite.icmp_id = f->match.icmp_id;
+    }
+  if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
+    {
+    }
+  else
+    {
+      f->rewrite.fib_index = f->match.fib_index;
+    }
+}
+
+static_always_inline int nat_6t_flow_icmp_translate (snat_main_t *sm,
+						     vlib_buffer_t *b,
+						     ip4_header_t *ip,
+						     nat_6t_flow_t *f);
+
+static_always_inline void
+nat_6t_flow_ip4_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
+			   nat_6t_flow_t *f, nat_protocol_t proto,
+			   int is_icmp_inner_ip4)
+{
+  udp_header_t *udp = ip4_next_header (ip);
+  tcp_header_t *tcp = (tcp_header_t *) udp;
+
+  if ((NAT_PROTOCOL_TCP == proto || NAT_PROTOCOL_UDP == proto) &&
+      !vnet_buffer (b)->ip.reass.is_non_first_fragment)
+    {
+      if (!is_icmp_inner_ip4)
+	{ // regular case
+	  ip->src_address = f->rewrite.saddr;
+	  ip->dst_address = f->rewrite.daddr;
+	  udp->src_port = f->rewrite.sport;
+	  udp->dst_port = f->rewrite.dport;
+	}
+      else
+	{ // icmp inner ip4 - reversed saddr/daddr
+	  ip->src_address = f->rewrite.daddr;
+	  ip->dst_address = f->rewrite.saddr;
+	  udp->src_port = f->rewrite.dport;
+	  udp->dst_port = f->rewrite.sport;
+	}
+
+      if (NAT_PROTOCOL_TCP == proto)
+	{
+	  ip_csum_t tcp_sum = tcp->checksum;
+	  tcp_sum = ip_csum_sub_even (tcp_sum, f->l3_csum_delta);
+	  tcp_sum = ip_csum_sub_even (tcp_sum, f->l4_csum_delta);
+	  mss_clamping (sm->mss_clamping, tcp, &tcp_sum);
+	  tcp->checksum = ip_csum_fold (tcp_sum);
+	}
+      else if (proto == NAT_PROTOCOL_UDP && udp->checksum)
+	{
+	  ip_csum_t udp_sum = udp->checksum;
+	  udp_sum = ip_csum_sub_even (udp_sum, f->l3_csum_delta);
+	  udp_sum = ip_csum_sub_even (udp_sum, f->l4_csum_delta);
+	  udp->checksum = ip_csum_fold (udp_sum);
+	}
+    }
+  else
+    {
+      if (!is_icmp_inner_ip4)
+	{ // regular case
+	  ip->src_address = f->rewrite.saddr;
+	  ip->dst_address = f->rewrite.daddr;
+	}
+      else
+	{ // icmp inner ip4 - reversed saddr/daddr
+	  ip->src_address = f->rewrite.daddr;
+	  ip->dst_address = f->rewrite.saddr;
+	}
+    }
+
+  ip_csum_t ip_sum = ip->checksum;
+  ip_sum = ip_csum_sub_even (ip_sum, f->l3_csum_delta);
+  ip->checksum = ip_csum_fold (ip_sum);
+  ASSERT (ip->checksum == ip4_header_checksum (ip));
+}
+
+static_always_inline int
+nat_6t_flow_icmp_translate (snat_main_t *sm, vlib_buffer_t *b,
+			    ip4_header_t *ip, nat_6t_flow_t *f)
+{
+  if (IP_PROTOCOL_ICMP != ip->protocol)
+    return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
+
+  icmp46_header_t *icmp = ip4_next_header (ip);
+  icmp_echo_header_t *echo = (icmp_echo_header_t *) (icmp + 1);
+
+  if ((!vnet_buffer (b)->ip.reass.is_non_first_fragment))
+    {
+      if (icmp->checksum == 0)
+	icmp->checksum = 0xffff;
+
+      if (!icmp_type_is_error_message (icmp->type))
+	{
+	  if ((f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE) &&
+	      (f->rewrite.icmp_id != echo->identifier))
+	    {
+	      ip_csum_t sum = icmp->checksum;
+	      sum = ip_csum_update (sum, echo->identifier, f->rewrite.icmp_id,
+				    icmp_echo_header_t,
+				    identifier /* changed member */);
+	      echo->identifier = f->rewrite.icmp_id;
+	      icmp->checksum = ip_csum_fold (sum);
+	    }
+	}
+      else
+	{
+	  // errors are not fragmented
+	  ip4_header_t *inner_ip = (ip4_header_t *) (echo + 1);
+
+	  if (!ip4_header_checksum_is_valid (inner_ip))
+	    {
+	      return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
+	    }
+
+	  nat_protocol_t inner_proto =
+	    ip_proto_to_nat_proto (inner_ip->protocol);
+
+	  ip_csum_t icmp_sum = icmp->checksum;
+
+	  switch (inner_proto)
+	    {
+	    case NAT_PROTOCOL_UDP:
+	    case NAT_PROTOCOL_TCP:
+	      nat_6t_flow_ip4_translate (sm, b, inner_ip, f, inner_proto,
+					 1 /* is_icmp_inner_ip4 */);
+	      icmp_sum = ip_csum_sub_even (icmp_sum, f->l3_csum_delta);
+	      icmp->checksum = ip_csum_fold (icmp_sum);
+	      break;
+	    case NAT_PROTOCOL_ICMP:
+	      if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
+		{
+		  icmp46_header_t *inner_icmp = ip4_next_header (inner_ip);
+		  icmp_echo_header_t *inner_echo =
+		    (icmp_echo_header_t *) (inner_icmp + 1);
+		  if (f->rewrite.icmp_id != inner_echo->identifier)
+		    {
+		      ip_csum_t sum = icmp->checksum;
+		      sum = ip_csum_update (
+			sum, inner_echo->identifier, f->rewrite.icmp_id,
+			icmp_echo_header_t, identifier /* changed member */);
+		      icmp->checksum = ip_csum_fold (sum);
+		      ip_csum_t inner_sum = inner_icmp->checksum;
+		      inner_sum = ip_csum_update (
+			sum, inner_echo->identifier, f->rewrite.icmp_id,
+			icmp_echo_header_t, identifier /* changed member */);
+		      inner_icmp->checksum = ip_csum_fold (inner_sum);
+		      inner_echo->identifier = f->rewrite.icmp_id;
+		    }
+		}
+	      break;
+	    default:
+	      clib_warning ("unexpected NAT protocol value `%d'", inner_proto);
+	      return NAT_ED_TRNSL_ERR_TRANSLATION_FAILED;
+	    }
+	}
+    }
+  return NAT_ED_TRNSL_ERR_SUCCESS;
+}
+
+nat_translation_error_e
+nat_6t_flow_buf_translate (snat_main_t *sm, vlib_buffer_t *b, ip4_header_t *ip,
+			   nat_6t_flow_t *f, nat_protocol_t proto,
+			   int is_output_feature)
+{
+  if (!is_output_feature && f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
+    {
+      vnet_buffer (b)->sw_if_index[VLIB_TX] = f->rewrite.fib_index;
+    }
+
+  nat_6t_flow_ip4_translate (sm, b, ip, f, proto, 0 /* is_icmp_inner_ip4 */);
+
+  if (NAT_PROTOCOL_ICMP == proto)
+    {
+      return nat_6t_flow_icmp_translate (sm, b, ip, f);
+    }
+
+  return NAT_ED_TRNSL_ERR_SUCCESS;
+}
+
+u8 *
+format_nat_6t (u8 *s, va_list *args)
+{
+  nat_6t_t *t = va_arg (*args, nat_6t_t *);
+
+  s = format (s, "saddr %U sport %u daddr %U dport %u proto %U fib_idx %u",
+	      format_ip4_address, t->saddr.as_u8,
+	      clib_net_to_host_u16 (t->sport), format_ip4_address,
+	      t->daddr.as_u8, clib_net_to_host_u16 (t->dport),
+	      format_ip_protocol, t->proto, t->fib_index);
+  return s;
+}
+
+u8 *
+format_nat_ed_translation_error (u8 *s, va_list *args)
+{
+  nat_translation_error_e e = va_arg (*args, nat_translation_error_e);
+
+  switch (e)
+    {
+    case NAT_ED_TRNSL_ERR_SUCCESS:
+      s = format (s, "success");
+      break;
+    case NAT_ED_TRNSL_ERR_TRANSLATION_FAILED:
+      s = format (s, "translation-failed");
+      break;
+    case NAT_ED_TRNSL_ERR_FLOW_MISMATCH:
+      s = format (s, "flow-mismatch");
+      break;
+    }
+  return s;
+}
+
+u8 *
+format_nat_6t_flow (u8 *s, va_list *args)
+{
+  nat_6t_flow_t *f = va_arg (*args, nat_6t_flow_t *);
+
+  s = format (s, "match: %U ", format_nat_6t, &f->match);
+  int r = 0;
+  if (f->ops & NAT_FLOW_OP_SADDR_REWRITE)
+    {
+      s = format (s, "rewrite: saddr %U ", format_ip4_address,
+		  f->rewrite.saddr.as_u8);
+      r = 1;
+    }
+  if (f->ops & NAT_FLOW_OP_SPORT_REWRITE)
+    {
+      if (!r)
+	{
+	  s = format (s, "rewrite: ");
+	  r = 1;
+	}
+      s = format (s, "sport %u ", clib_net_to_host_u16 (f->rewrite.sport));
+    }
+  if (f->ops & NAT_FLOW_OP_DADDR_REWRITE)
+    {
+      if (!r)
+	{
+	  s = format (s, "rewrite: ");
+	  r = 1;
+	}
+      s = format (s, "daddr %U ", format_ip4_address, f->rewrite.daddr.as_u8);
+    }
+  if (f->ops & NAT_FLOW_OP_DPORT_REWRITE)
+    {
+      if (!r)
+	{
+	  s = format (s, "rewrite: ");
+	  r = 1;
+	}
+      s = format (s, "dport %u ", clib_net_to_host_u16 (f->rewrite.dport));
+    }
+  if (f->ops & NAT_FLOW_OP_ICMP_ID_REWRITE)
+    {
+      if (!r)
+	{
+	  s = format (s, "rewrite: ");
+	  r = 1;
+	}
+      s = format (s, "icmp-id %u ", clib_net_to_host_u16 (f->rewrite.icmp_id));
+    }
+  if (f->ops & NAT_FLOW_OP_TXFIB_REWRITE)
+    {
+      if (!r)
+	{
+	  s = format (s, "rewrite: ");
+	  r = 1;
+	}
+      s = format (s, "txfib %u ", f->rewrite.fib_index);
+    }
+  return s;
+}
+
 /*
  * fd.io coding-style-patch-verification: ON
  *