SNAT: Fallback to 3-tuple key for non TCP/UDP sessions (VPP-884)

Change-Id: I4868ff6e81c579b29d3ea066976ae145f8b83e9e
Signed-off-by: Matus Fabian <matfabia@cisco.com>

6 files changed, 824 insertions, 80 deletions

diff --git a/src/plugins/snat/in2out.c b/src/plugins/snat/in2out.c
index 661ddeb1185..d3fb92f5f1d 100644
--- a/src/plugins/snat/in2out.c
+++ b/src/plugins/snat/in2out.c
@@ -297,24 +297,49 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
                                  session_index);
       } while (snat_is_session_static (s));
 
-      /* Remove in2out, out2in keys */
-      kv0.key = s->in2out.as_u64;
-      if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
-          clib_warning ("in2out key delete failed");
-      kv0.key = s->out2in.as_u64;
-      if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
-          clib_warning ("out2in key delete failed");
-
-      /* log NAT event */
-      snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
-                                          s->out2in.addr.as_u32,
-                                          s->in2out.protocol,
-                                          s->in2out.port,
-                                          s->out2in.port,
-                                          s->in2out.fib_index);
-
-      snat_free_outside_address_and_port 
-        (sm, &s->out2in, s->outside_address_index);
+      if (snat_is_unk_proto_session (s))
+        {
+          clib_bihash_kv_16_8_t up_kv;
+          snat_unk_proto_ses_key_t key;
+
+          /* Remove from lookup tables */
+          key.l_addr = s->in2out.addr;
+          key.r_addr = s->ext_host_addr;
+          key.fib_index = s->in2out.fib_index;
+          key.proto = s->in2out.port;
+          up_kv.key[0] = key.as_u64[0];
+          up_kv.key[1] = key.as_u64[1];
+          if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &up_kv, 0))
+            clib_warning ("in2out key del failed");
+
+          key.l_addr = s->out2in.addr;
+          key.fib_index = s->out2in.fib_index;
+          up_kv.key[0] = key.as_u64[0];
+          up_kv.key[1] = key.as_u64[1];
+          if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &up_kv, 0))
+            clib_warning ("out2in key del failed");
+        }
+      else
+        {
+          /* Remove in2out, out2in keys */
+          kv0.key = s->in2out.as_u64;
+          if (clib_bihash_add_del_8_8 (&sm->in2out, &kv0, 0 /* is_add */))
+              clib_warning ("in2out key delete failed");
+          kv0.key = s->out2in.as_u64;
+          if (clib_bihash_add_del_8_8 (&sm->out2in, &kv0, 0 /* is_add */))
+              clib_warning ("out2in key delete failed");
+
+          /* log NAT event */
+          snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
+                                              s->out2in.addr.as_u32,
+                                              s->in2out.protocol,
+                                              s->in2out.port,
+                                              s->out2in.port,
+                                              s->in2out.fib_index);
+
+          snat_free_outside_address_and_port
+            (sm, &s->out2in, s->outside_address_index);
+        }
       s->outside_address_index = ~0;
 
       if (snat_alloc_outside_address_and_port (sm, rx_fib_index0, &key1,
@@ -383,6 +408,7 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
   s->out2in = key1;
   s->out2in.protocol = key0->protocol;
   s->out2in.fib_index = outside_fib_index;
+  s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
   *sessionp = s;
 
   /* Add to translation hashes */
@@ -971,49 +997,298 @@ static void
 snat_in2out_unknown_proto (snat_main_t *sm,
                            vlib_buffer_t * b,
                            ip4_header_t * ip,
-                           u32 rx_fib_index)
+                           u32 rx_fib_index,
+                           u32 thread_index,
+                           f64 now,
+                           vlib_main_t * vm)
 {
   clib_bihash_kv_8_8_t kv, value;
+  clib_bihash_kv_16_8_t s_kv, s_value;
   snat_static_mapping_t *m;
   snat_session_key_t m_key;
-  u32 old_addr, new_addr;
+  u32 old_addr, new_addr = 0;
   ip_csum_t sum;
+  snat_user_key_t u_key;
+  snat_user_t *u;
+  dlist_elt_t *head, *elt, *oldest;
+  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
+  u32 elt_index, head_index, ses_index, oldest_index;
+  snat_session_t * s;
+  snat_unk_proto_ses_key_t key;
+  u32 address_index = ~0;
+  int i;
+  u8 is_sm = 0;
+
+  old_addr = ip->src_address.as_u32;
 
-  m_key.addr = ip->src_address;
-  m_key.port = 0;
-  m_key.protocol = 0;
-  m_key.fib_index = rx_fib_index;
-  kv.key = m_key.as_u64;
-  if (clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
-    return;
+  key.l_addr = ip->src_address;
+  key.r_addr = ip->dst_address;
+  key.fib_index = rx_fib_index;
+  key.proto = ip->protocol;
+  key.rsvd[0] = key.rsvd[1] = key.rsvd[2] = 0;
+  s_kv.key[0] = key.as_u64[0];
+  s_kv.key[1] = key.as_u64[1];
 
-  m = pool_elt_at_index (sm->static_mappings, value.value);
+  if (!clib_bihash_search_16_8 (&sm->in2out_unk_proto, &s_kv, &s_value))
+    {
+      s = pool_elt_at_index (tsm->sessions, s_value.value);
+      new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
+    }
+  else
+    {
+      u_key.addr = ip->src_address;
+      u_key.fib_index = rx_fib_index;
+      kv.key = u_key.as_u64;
 
-  old_addr = ip->src_address.as_u32;
-  new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
+      /* Ever heard of the "user" = src ip4 address before? */
+      if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
+        {
+          /* no, make a new one */
+          pool_get (tsm->users, u);
+          memset (u, 0, sizeof (*u));
+          u->addr = ip->src_address;
+          u->fib_index = rx_fib_index;
+
+          pool_get (tsm->list_pool, head);
+          u->sessions_per_user_list_head_index = head - tsm->list_pool;
+
+          clib_dlist_init (tsm->list_pool,
+                           u->sessions_per_user_list_head_index);
+
+          kv.value = u - tsm->users;
+
+          /* add user */
+          clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
+        }
+      else
+        {
+          u = pool_elt_at_index (tsm->users, value.value);
+        }
+
+      m_key.addr = ip->src_address;
+      m_key.port = 0;
+      m_key.protocol = 0;
+      m_key.fib_index = rx_fib_index;
+      kv.key = m_key.as_u64;
+
+      /* Try to find static mapping first */
+      if (!clib_bihash_search_8_8 (&sm->static_mapping_by_local, &kv, &value))
+        {
+          m = pool_elt_at_index (sm->static_mappings, value.value);
+          new_addr = ip->src_address.as_u32 = m->external_addr.as_u32;
+          is_sm = 1;
+          goto create_ses;
+        }
+      /* Fallback to 3-tuple key */
+      else
+        {
+          /* Choose same out address as for TCP/UDP session to same destination */
+          if (!clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
+            {
+              head_index = u->sessions_per_user_list_head_index;
+              head = pool_elt_at_index (tsm->list_pool, head_index);
+              elt_index = head->next;
+              elt = pool_elt_at_index (tsm->list_pool, elt_index);
+              ses_index = elt->value;
+              while (ses_index != ~0)
+                {
+                  s =  pool_elt_at_index (tsm->sessions, ses_index);
+                  if (s->ext_host_addr.as_u32 == ip->dst_address.as_u32)
+                    {
+                      new_addr = ip->src_address.as_u32 = s->out2in.addr.as_u32;
+                      address_index = s->outside_address_index;
+
+                      key.fib_index = sm->outside_fib_index;
+                      key.l_addr.as_u32 = new_addr;
+                      s_kv.key[0] = key.as_u64[0];
+                      s_kv.key[1] = key.as_u64[1];
+                      if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
+                        break;
+
+                      goto create_ses;
+                    }
+                }
+            }
+          key.fib_index = sm->outside_fib_index;
+          for (i = 0; i < vec_len (sm->addresses); i++)
+            {
+              key.l_addr.as_u32 = sm->addresses[i].addr.as_u32;
+              s_kv.key[0] = key.as_u64[0];
+              s_kv.key[1] = key.as_u64[1];
+              if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
+                {
+                  new_addr = ip->src_address.as_u32 = key.l_addr.as_u32;
+                  address_index = i;
+                  goto create_ses;
+                }
+            }
+          return;
+        }
+
+create_ses:
+      /* Over quota? Recycle the least recently used dynamic translation */
+      if (u->nsessions >= sm->max_translations_per_user && !is_sm)
+        {
+          /* Remove the oldest dynamic translation */
+          do {
+              oldest_index = clib_dlist_remove_head (
+                tsm->list_pool, u->sessions_per_user_list_head_index);
+
+              ASSERT (oldest_index != ~0);
+
+              /* add it back to the end of the LRU list */
+              clib_dlist_addtail (tsm->list_pool,
+                                  u->sessions_per_user_list_head_index,
+                                  oldest_index);
+              /* Get the list element */
+              oldest = pool_elt_at_index (tsm->list_pool, oldest_index);
+
+              /* Get the session index from the list element */
+              ses_index = oldest->value;
+
+              /* Get the session */
+              s = pool_elt_at_index (tsm->sessions, ses_index);
+          } while (snat_is_session_static (s));
+
+          if (snat_is_unk_proto_session (s))
+            {
+              /* Remove from lookup tables */
+              key.l_addr = s->in2out.addr;
+              key.r_addr = s->ext_host_addr;
+              key.fib_index = s->in2out.fib_index;
+              key.proto = s->in2out.port;
+              s_kv.key[0] = key.as_u64[0];
+              s_kv.key[1] = key.as_u64[1];
+              if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 0))
+                clib_warning ("in2out key del failed");
+
+              key.l_addr = s->out2in.addr;
+              key.fib_index = s->out2in.fib_index;
+              s_kv.key[0] = key.as_u64[0];
+              s_kv.key[1] = key.as_u64[1];
+              if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 0))
+                clib_warning ("out2in key del failed");
+            }
+          else
+            {
+              /* log NAT event */
+              snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
+                                                  s->out2in.addr.as_u32,
+                                                  s->in2out.protocol,
+                                                  s->in2out.port,
+                                                  s->out2in.port,
+                                                  s->in2out.fib_index);
+
+              snat_free_outside_address_and_port (sm, &s->out2in,
+                                                  s->outside_address_index);
+
+              /* Remove in2out, out2in keys */
+              kv.key = s->in2out.as_u64;
+              if (clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0))
+                clib_warning ("in2out key del failed");
+              kv.key = s->out2in.as_u64;
+              if (clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0))
+                clib_warning ("out2in key del failed");
+            }
+        }
+      else
+        {
+          /* Create a new session */
+          pool_get (tsm->sessions, s);
+          memset (s, 0, sizeof (*s));
+
+          /* Create list elts */
+          pool_get (tsm->list_pool, elt);
+          clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
+          elt->value = s - tsm->sessions;
+          s->per_user_index = elt - tsm->list_pool;
+          s->per_user_list_head_index = u->sessions_per_user_list_head_index;
+          clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
+                              s->per_user_index);
+        }
+
+      s->ext_host_addr.as_u32 = ip->dst_address.as_u32;
+      s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
+      s->outside_address_index = address_index;
+      s->out2in.addr.as_u32 = new_addr;
+      s->out2in.fib_index = sm->outside_fib_index;
+      s->in2out.addr.as_u32 = old_addr;
+      s->in2out.fib_index = rx_fib_index;
+      s->in2out.port = s->out2in.port = ip->protocol;
+      if (is_sm)
+        {
+          u->nstaticsessions++;
+          s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
+        }
+      else
+        {
+          u->nsessions++;
+        }
+
+      /* Add to lookup tables */
+      key.l_addr.as_u32 = old_addr;
+      key.r_addr = ip->dst_address;
+      key.proto = ip->protocol;
+      key.fib_index = rx_fib_index;
+      s_kv.key[0] = key.as_u64[0];
+      s_kv.key[1] = key.as_u64[1];
+      s_kv.value = s - tsm->sessions;
+      if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 1))
+        clib_warning ("in2out key add failed");
+
+      key.l_addr.as_u32 = new_addr;
+      key.fib_index = sm->outside_fib_index;
+      s_kv.key[0] = key.as_u64[0];
+      s_kv.key[1] = key.as_u64[1];
+      if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 1))
+        clib_warning ("out2in key add failed");
+  }
+
+  /* Update IP checksum */
   sum = ip->checksum;
   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, src_address);
   ip->checksum = ip_csum_fold (sum);
 
+  /* Accounting */
+  s->last_heard = now;
+  s->total_pkts++;
+  s->total_bytes += vlib_buffer_length_in_chain (vm, b);
+  /* Per-user LRU list maintenance */
+  clib_dlist_remove (tsm->list_pool, s->per_user_index);
+  clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
+                      s->per_user_index);
+
   /* Hairpinning */
-  m_key.addr = ip->dst_address;
-  m_key.fib_index = sm->outside_fib_index;
-  kv.key = m_key.as_u64;
-  if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
+  old_addr = ip->dst_address.as_u32;
+  key.l_addr.as_u32 = ip->dst_address.as_u32;
+  key.r_addr.as_u32 = new_addr;
+  key.fib_index = sm->outside_fib_index;
+  s_kv.key[0] = key.as_u64[0];
+  s_kv.key[1] = key.as_u64[1];
+  if (clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
     {
-      vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
-      return;
-    }
-
-  m = pool_elt_at_index (sm->static_mappings, value.value);
+      m_key.addr = ip->dst_address;
+      m_key.fib_index = sm->outside_fib_index;
+      kv.key = m_key.as_u64;
+      if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
+        {
+          vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
+          return;
+        }
 
-  old_addr = ip->dst_address.as_u32;
-  new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
+      m = pool_elt_at_index (sm->static_mappings, value.value);
+      vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
+      new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
+    }
+  else
+    {
+      s = pool_elt_at_index (tsm->sessions, s_value.value);
+      vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
+      new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
+    }
   sum = ip->checksum;
   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
   ip->checksum = ip_csum_fold (sum);
-
-  vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
 }
 
 static inline uword
@@ -1115,7 +1390,8 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
             {
               if (PREDICT_FALSE (proto0 == ~0))
                 {
-                  snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0);
+                  snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
+                                             thread_index, now, vm);
                   goto trace00;
                 }
 
@@ -1258,7 +1534,8 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
             {
               if (PREDICT_FALSE (proto1 == ~0))
                 {
-                  snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1);
+                  snat_in2out_unknown_proto (sm, b1, ip1, rx_fib_index1,
+                                             thread_index, now, vm);
                   goto trace01;
                 }
 
@@ -1436,7 +1713,8 @@ snat_in2out_node_fn_inline (vlib_main_t * vm,
             {
               if (PREDICT_FALSE (proto0 == ~0))
                 {
-                  snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0);
+                  snat_in2out_unknown_proto (sm, b0, ip0, rx_fib_index0,
+                                             thread_index, now, vm);
                   goto trace0;
                 }
 
diff --git a/src/plugins/snat/out2in.c b/src/plugins/snat/out2in.c
index 5c12b473c63..cba42465e91 100644
--- a/src/plugins/snat/out2in.c
+++ b/src/plugins/snat/out2in.c
@@ -137,6 +137,9 @@ create_session_for_static_mapping (snat_main_t *sm,
   clib_bihash_kv_8_8_t kv0, value0;
   dlist_elt_t * per_user_translation_list_elt;
   dlist_elt_t * per_user_list_head_elt;
+  ip4_header_t *ip0;
+
+  ip0 = vlib_buffer_get_current (b0);
 
   user_key.addr = in2out.addr;
   user_key.fib_index = in2out.fib_index;
@@ -180,6 +183,7 @@ create_session_for_static_mapping (snat_main_t *sm,
 
   s->outside_address_index = ~0;
   s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
+  s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
   u->nstaticsessions++;
 
   /* Create list elts */
@@ -615,31 +619,134 @@ static void
 snat_out2in_unknown_proto (snat_main_t *sm,
                            vlib_buffer_t * b,
                            ip4_header_t * ip,
-                           u32 rx_fib_index)
+                           u32 rx_fib_index,
+                           u32 thread_index,
+                           f64 now,
+                           vlib_main_t * vm)
 {
   clib_bihash_kv_8_8_t kv, value;
+  clib_bihash_kv_16_8_t s_kv, s_value;
   snat_static_mapping_t *m;
   snat_session_key_t m_key;
   u32 old_addr, new_addr;
   ip_csum_t sum;
+  snat_unk_proto_ses_key_t key;
+  snat_session_t * s;
+  snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
+  snat_user_key_t u_key;
+  snat_user_t *u;
+  dlist_elt_t *head, *elt;
 
-  m_key.addr = ip->dst_address;
-  m_key.port = 0;
-  m_key.protocol = 0;
-  m_key.fib_index = rx_fib_index;
-  kv.key = m_key.as_u64;
-  if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
-    return;
+  old_addr = ip->dst_address.as_u32;
 
-  m = pool_elt_at_index (sm->static_mappings, value.value);
+  key.l_addr = ip->dst_address;
+  key.r_addr = ip->src_address;
+  key.fib_index = rx_fib_index;
+  key.proto = ip->protocol;
+  key.rsvd[0] = key.rsvd[1] = key.rsvd[2] = 0;
+  s_kv.key[0] = key.as_u64[0];
+  s_kv.key[1] = key.as_u64[1];
 
-  old_addr = ip->dst_address.as_u32;
-  new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
+  if (!clib_bihash_search_16_8 (&sm->out2in_unk_proto, &s_kv, &s_value))
+    {
+      s = pool_elt_at_index (tsm->sessions, s_value.value);
+      new_addr = ip->dst_address.as_u32 = s->in2out.addr.as_u32;
+    }
+  else
+    {
+      m_key.addr = ip->dst_address;
+      m_key.port = 0;
+      m_key.protocol = 0;
+      m_key.fib_index = rx_fib_index;
+      kv.key = m_key.as_u64;
+      if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
+        return;
+
+      m = pool_elt_at_index (sm->static_mappings, value.value);
+
+      new_addr = ip->dst_address.as_u32 = m->local_addr.as_u32;
+
+      u_key.addr = ip->src_address;
+      u_key.fib_index = m->fib_index;
+      kv.key = u_key.as_u64;
+
+      /* Ever heard of the "user" = src ip4 address before? */
+      if (clib_bihash_search_8_8 (&sm->user_hash, &kv, &value))
+        {
+          /* no, make a new one */
+          pool_get (tsm->users, u);
+          memset (u, 0, sizeof (*u));
+          u->addr = ip->src_address;
+          u->fib_index = rx_fib_index;
+
+          pool_get (tsm->list_pool, head);
+          u->sessions_per_user_list_head_index = head - tsm->list_pool;
+
+          clib_dlist_init (tsm->list_pool,
+                           u->sessions_per_user_list_head_index);
+
+          kv.value = u - tsm->users;
+
+          /* add user */
+          clib_bihash_add_del_8_8 (&sm->user_hash, &kv, 1);
+        }
+      else
+        {
+          u = pool_elt_at_index (tsm->users, value.value);
+        }
+
+      /* Create a new session */
+      pool_get (tsm->sessions, s);
+      memset (s, 0, sizeof (*s));
+
+      s->ext_host_addr.as_u32 = ip->src_address.as_u32;
+      s->flags |= SNAT_SESSION_FLAG_UNKNOWN_PROTO;
+      s->flags |= SNAT_SESSION_FLAG_STATIC_MAPPING;
+      s->outside_address_index = ~0;
+      s->out2in.addr.as_u32 = old_addr;
+      s->out2in.fib_index = rx_fib_index;
+      s->in2out.addr.as_u32 = new_addr;
+      s->in2out.fib_index = m->fib_index;
+      s->in2out.port = s->out2in.port = ip->protocol;
+      u->nstaticsessions++;
+
+      /* Create list elts */
+      pool_get (tsm->list_pool, elt);
+      clib_dlist_init (tsm->list_pool, elt - tsm->list_pool);
+      elt->value = s - tsm->sessions;
+      s->per_user_index = elt - tsm->list_pool;
+      s->per_user_list_head_index = u->sessions_per_user_list_head_index;
+      clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
+                          s->per_user_index);
+
+      /* Add to lookup tables */
+      s_kv.value = s - tsm->sessions;
+      if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto, &s_kv, 1))
+        clib_warning ("out2in key add failed");
+
+      key.l_addr = ip->dst_address;
+      key.fib_index = m->fib_index;
+      s_kv.key[0] = key.as_u64[0];
+      s_kv.key[1] = key.as_u64[1];
+      if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto, &s_kv, 1))
+        clib_warning ("in2out key add failed");
+   }
+
+  /* Update IP checksum */
   sum = ip->checksum;
   sum = ip_csum_update (sum, old_addr, new_addr, ip4_header_t, dst_address);
   ip->checksum = ip_csum_fold (sum);
 
-  vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
+  vnet_buffer(b)->sw_if_index[VLIB_TX] = s->in2out.fib_index;
+
+  /* Accounting */
+  s->last_heard = now;
+  s->total_pkts++;
+  s->total_bytes += vlib_buffer_length_in_chain (vm, b);
+  /* Per-user LRU list maintenance */
+  clib_dlist_remove (tsm->list_pool, s->per_user_index);
+  clib_dlist_addtail (tsm->list_pool, s->per_user_list_head_index,
+                      s->per_user_index);
 }
 
 static uword
@@ -735,7 +842,8 @@ snat_out2in_node_fn (vlib_main_t * vm,
 
           if (PREDICT_FALSE (proto0 == ~0))
             {
-              snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0);
+              snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
+                                        thread_index, now, vm);
               goto trace0;
             }
 
@@ -873,7 +981,8 @@ snat_out2in_node_fn (vlib_main_t * vm,
 
           if (PREDICT_FALSE (proto1 == ~0))
             {
-              snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1);
+              snat_out2in_unknown_proto(sm, b1, ip1, rx_fib_index1,
+                                        thread_index, now, vm);
               goto trace1;
             }
 
@@ -1035,7 +1144,8 @@ snat_out2in_node_fn (vlib_main_t * vm,
 
           if (PREDICT_FALSE (proto0 == ~0))
             {
-              snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0);
+              snat_out2in_unknown_proto(sm, b0, ip0, rx_fib_index0,
+                                        thread_index, now, vm);
               goto trace00;
             }
 
diff --git a/src/plugins/snat/snat.c b/src/plugins/snat/snat.c
index df0a5a07294..97eb061c16b 100644
--- a/src/plugins/snat/snat.c
+++ b/src/plugins/snat/snat.c
@@ -468,6 +468,31 @@ int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
                             continue;
                         }
 
+                      if (snat_is_unk_proto_session (s))
+                        {
+                          clib_bihash_kv_16_8_t up_kv;
+                          snat_unk_proto_ses_key_t up_key;
+                          up_key.l_addr = s->in2out.addr;
+                          up_key.r_addr = s->ext_host_addr;
+                          up_key.fib_index = s->in2out.fib_index;
+                          up_key.proto = s->in2out.port;
+                          up_key.rsvd[0] = up_key.rsvd[1] = up_key.rsvd[2] = 0;
+                          up_kv.key[0] = up_key.as_u64[0];
+                          up_kv.key[1] = up_key.as_u64[1];
+                          if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto,
+                                                        &up_kv, 0))
+                            clib_warning ("in2out key del failed");
+
+                          up_key.l_addr = s->out2in.addr;
+                          up_key.fib_index = s->out2in.fib_index;
+                          up_kv.key[0] = up_key.as_u64[0];
+                          up_kv.key[1] = up_key.as_u64[1];
+                          if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto,
+                                                        &up_kv, 0))
+                            clib_warning ("out2in key del failed");
+
+                          goto delete;
+                        }
                       /* log NAT event */
                       snat_ipfix_logging_nat44_ses_delete(s->in2out.addr.as_u32,
                                                           s->out2in.addr.as_u32,
@@ -477,9 +502,12 @@ int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
                                                           s->in2out.fib_index);
 
                       value.key = s->in2out.as_u64;
-                      clib_bihash_add_del_8_8 (&sm->in2out, &value, 0);
+                      if (clib_bihash_add_del_8_8 (&sm->in2out, &value, 0))
+                        clib_warning ("in2out key del failed");
                       value.key = s->out2in.as_u64;
-                      clib_bihash_add_del_8_8 (&sm->out2in, &value, 0);
+                      if (clib_bihash_add_del_8_8 (&sm->out2in, &value, 0))
+                        clib_warning ("out2in key del failed");
+delete:
                       pool_put (tsm->sessions, s);
 
                       clib_dlist_remove (tsm->list_pool, del_elt_index);
@@ -572,18 +600,44 @@ int snat_del_address (snat_main_t *sm, ip4_address_t addr, u8 delete_sm)
           pool_foreach (ses, tsm->sessions, ({
             if (ses->out2in.addr.as_u32 == addr.as_u32)
               {
-                /* log NAT event */
-                snat_ipfix_logging_nat44_ses_delete(ses->in2out.addr.as_u32,
-                                                    ses->out2in.addr.as_u32,
-                                                    ses->in2out.protocol,
-                                                    ses->in2out.port,
-                                                    ses->out2in.port,
-                                                    ses->in2out.fib_index);
+                if (snat_is_unk_proto_session (ses))
+                  {
+                    clib_bihash_kv_16_8_t up_kv;
+                    snat_unk_proto_ses_key_t up_key;
+                    up_key.l_addr = ses->in2out.addr;
+                    up_key.r_addr = ses->ext_host_addr;
+                    up_key.fib_index = ses->in2out.fib_index;
+                    up_key.proto = ses->in2out.port;
+                    up_key.rsvd[0] = up_key.rsvd[1] = up_key.rsvd[2] = 0;
+                    up_kv.key[0] = up_key.as_u64[0];
+                    up_kv.key[1] = up_key.as_u64[1];
+                    if (clib_bihash_add_del_16_8 (&sm->in2out_unk_proto,
+                                                  &up_kv, 0))
+                      clib_warning ("in2out key del failed");
+
+                    up_key.l_addr = ses->out2in.addr;
+                    up_key.fib_index = ses->out2in.fib_index;
+                    up_kv.key[0] = up_key.as_u64[0];
+                    up_kv.key[1] = up_key.as_u64[1];
+                    if (clib_bihash_add_del_16_8 (&sm->out2in_unk_proto,
+                                                  &up_kv, 0))
+                      clib_warning ("out2in key del failed");
+                  }
+                else
+                  {
+                    /* log NAT event */
+                    snat_ipfix_logging_nat44_ses_delete(ses->in2out.addr.as_u32,
+                                                        ses->out2in.addr.as_u32,
+                                                        ses->in2out.protocol,
+                                                        ses->in2out.port,
+                                                        ses->out2in.port,
+                                                        ses->in2out.fib_index);
+                    kv.key = ses->in2out.as_u64;
+                    clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0);
+                    kv.key = ses->out2in.as_u64;
+                    clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0);
+                  }
                 vec_add1 (ses_to_be_removed, ses - tsm->sessions);
-                kv.key = ses->in2out.as_u64;
-                clib_bihash_add_del_8_8 (&sm->in2out, &kv, 0);
-                kv.key = ses->out2in.as_u64;
-                clib_bihash_add_del_8_8 (&sm->out2in, &kv, 0);
                 clib_dlist_remove (tsm->list_pool, ses->per_user_index);
                 user_key.addr = ses->in2out.addr;
                 user_key.fib_index = ses->in2out.fib_index;
@@ -1575,6 +1629,12 @@ snat_config (vlib_main_t * vm, unformat_input_t * input)
 
           clib_bihash_init_8_8 (&sm->user_hash, "users", user_buckets,
                                 user_memory_size);
+
+          clib_bihash_init_16_8 (&sm->in2out_unk_proto, "in2out-unk-proto",
+                                 translation_buckets, translation_memory_size);
+
+          clib_bihash_init_16_8 (&sm->out2in_unk_proto, "out2in-unk-proto",
+                                 translation_buckets, translation_memory_size);
         }
       else
         {
@@ -1636,8 +1696,20 @@ u8 * format_snat_session (u8 * s, va_list * args)
   snat_main_t * sm __attribute__((unused)) = va_arg (*args, snat_main_t *);
   snat_session_t * sess = va_arg (*args, snat_session_t *);
 
-  s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
-  s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
+  if (snat_is_unk_proto_session (sess))
+    {
+      s = format (s, "  i2o %U proto %u fib %u\n",
+                  format_ip4_address, &sess->in2out.addr, sess->in2out.port,
+                  sess->in2out.fib_index);
+      s = format (s, "    o2i %U proto %u fib %u\n",
+                  format_ip4_address, &sess->out2in.addr, sess->out2in.port,
+                  sess->out2in.fib_index);
+    }
+  else
+    {
+      s = format (s, "  i2o %U\n", format_snat_key, &sess->in2out);
+      s = format (s, "    o2i %U\n", format_snat_key, &sess->out2in);
+    }
   s = format (s, "       last heard %.2f\n", sess->last_heard);
   s = format (s, "       total pkts %d, total bytes %lld\n",
               sess->total_pkts, sess->total_bytes);
diff --git a/src/plugins/snat/snat.h b/src/plugins/snat/snat.h
index 016c2ff55df..1bc5fcd7545 100644
--- a/src/plugins/snat/snat.h
+++ b/src/plugins/snat/snat.h
@@ -24,6 +24,7 @@
 #include <vnet/ip/icmp46_packet.h>
 #include <vnet/api_errno.h>
 #include <vppinfra/bihash_8_8.h>
+#include <vppinfra/bihash_16_8.h>
 #include <vppinfra/dlist.h>
 #include <vppinfra/error.h>
 #include <vlibapi/api.h>
@@ -56,6 +57,21 @@ typedef struct {
   {
     struct
     {
+      ip4_address_t l_addr;
+      ip4_address_t r_addr;
+      u32 fib_index;
+      u8 proto;
+      u8 rsvd[3];
+    };
+    u64 as_u64[2];
+  };
+} snat_unk_proto_ses_key_t;
+
+typedef struct {
+  union
+  {
+    struct
+    {
       ip4_address_t ext_host_addr;
       u16 ext_host_port;
       u16 out_port;
@@ -120,6 +136,7 @@ typedef enum {
 
 
 #define SNAT_SESSION_FLAG_STATIC_MAPPING 1
+#define SNAT_SESSION_FLAG_UNKNOWN_PROTO  2
 
 typedef CLIB_PACKED(struct {
   snat_session_key_t out2in;    /* 0-15 */
@@ -143,6 +160,9 @@ typedef CLIB_PACKED(struct {
   /* Outside address */
   u32 outside_address_index;    /* 64-67 */
 
+  /* External host address */
+  ip4_address_t ext_host_addr;  /* 68-71 */
+
 }) snat_session_t;
 
 
@@ -240,6 +260,10 @@ typedef struct snat_main_s {
   clib_bihash_8_8_t out2in;
   clib_bihash_8_8_t in2out;
 
+  /* Unknown protocol sessions lookup tables */
+  clib_bihash_16_8_t out2in_unk_proto;
+  clib_bihash_16_8_t in2out_unk_proto;
+
   /* Find-a-user => src address lookup */
   clib_bihash_8_8_t user_hash;
 
@@ -374,6 +398,12 @@ typedef struct {
 */
 #define snat_is_session_static(s) s->flags & SNAT_SESSION_FLAG_STATIC_MAPPING
 
+/** \brief Check if SNAT session for unknown protocol.
+    @param s SNAT session
+    @return 1 if SNAT session for unknown protocol otherwise 0
+*/
+#define snat_is_unk_proto_session(s) s->flags & SNAT_SESSION_FLAG_UNKNOWN_PROTO
+
 /* 
  * Why is this here? Because we don't need to touch this layer to
  * simply reply to an icmp. We need to change id to a unique
diff --git a/src/plugins/snat/snat_api.c b/src/plugins/snat/snat_api.c
index 638f576c354..237b080dd67 100644
--- a/src/plugins/snat/snat_api.c
+++ b/src/plugins/snat/snat_api.c
@@ -732,15 +732,24 @@ static void
     ntohs (VL_API_SNAT_USER_SESSION_DETAILS + sm->msg_id_base);
   rmp->is_ip4 = 1;
   clib_memcpy (rmp->outside_ip_address, (&s->out2in.addr), 4);
-  rmp->outside_port = s->out2in.port;
   clib_memcpy (rmp->inside_ip_address, (&s->in2out.addr), 4);
-  rmp->inside_port = s->in2out.port;
-  rmp->protocol = ntohs (snat_proto_to_ip_proto (s->in2out.protocol));
   rmp->is_static = s->flags & SNAT_SESSION_FLAG_STATIC_MAPPING ? 1 : 0;
   rmp->last_heard = clib_host_to_net_u64 ((u64) s->last_heard);
   rmp->total_bytes = clib_host_to_net_u64 (s->total_bytes);
   rmp->total_pkts = ntohl (s->total_pkts);
   rmp->context = context;
+  if (snat_is_unk_proto_session (s))
+    {
+      rmp->outside_port = 0;
+      rmp->inside_port = 0;
+      rmp->protocol = ntohs (s->in2out.port);
+    }
+  else
+    {
+      rmp->outside_port = s->out2in.port;
+      rmp->inside_port = s->in2out.port;
+      rmp->protocol = ntohs (snat_proto_to_ip_proto (s->in2out.protocol));
+    }
 
   vl_msg_api_send_shmem (q, (u8 *) & rmp);
 }
diff --git a/test/test_snat.py b/test/test_snat.py
index f876c5f0ab7..df81fb56ab5 100644
--- a/test/test_snat.py
+++ b/test/test_snat.py
@@ -1932,7 +1932,7 @@ class TestSNAT(MethodHolder):
             self.logger.error(ppp("Unexpected or invalid packet:", packet))
             raise
 
-    def test_hairpinning_unknown_proto(self):
+    def test_hairpinning_static_unknown_proto(self):
         """ 1:1 NAT translate packet with unknown protocol - hairpinning """
 
         host = self.pg0.remote_hosts[0]
@@ -1987,6 +1987,127 @@ class TestSNAT(MethodHolder):
             self.logger.error(ppp("Unexpected or invalid packet:", packet))
             raise
 
+    def test_unknown_proto(self):
+        """ SNAT translate packet with unknown protocol """
+        self.snat_add_address(self.snat_addr)
+        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
+                                                 is_inside=0)
+
+        # in2out
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+             TCP(sport=self.tcp_port_in, dport=20))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg1.get_capture(1)
+
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IP(src=self.pg0.remote_ip4, dst=self.pg1.remote_ip4) /
+             GRE() /
+             IP(src=self.pg2.remote_ip4, dst=self.pg3.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg1.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, self.snat_addr)
+            self.assertEqual(packet[IP].dst, self.pg1.remote_ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+        # out2in
+        p = (Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
+             IP(src=self.pg1.remote_ip4, dst=self.snat_addr) /
+             GRE() /
+             IP(src=self.pg3.remote_ip4, dst=self.pg2.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg1.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, self.pg1.remote_ip4)
+            self.assertEqual(packet[IP].dst, self.pg0.remote_ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+    def test_hairpinning_unknown_proto(self):
+        """ SNAT translate packet with unknown protocol - hairpinning """
+        host = self.pg0.remote_hosts[0]
+        server = self.pg0.remote_hosts[1]
+        host_in_port = 1234
+        host_out_port = 0
+        server_in_port = 5678
+        server_out_port = 8765
+        server_nat_ip = "10.0.0.11"
+
+        self.snat_add_address(self.snat_addr)
+        self.vapi.snat_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.snat_interface_add_del_feature(self.pg1.sw_if_index,
+                                                 is_inside=0)
+
+        # add static mapping for server
+        self.snat_add_static_mapping(server.ip4, server_nat_ip)
+
+        # host to server
+        p = (Ether(src=host.mac, dst=self.pg0.local_mac) /
+             IP(src=host.ip4, dst=server_nat_ip) /
+             TCP(sport=host_in_port, dport=server_out_port))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(1)
+
+        p = (Ether(dst=self.pg0.local_mac, src=host.mac) /
+             IP(src=host.ip4, dst=server_nat_ip) /
+             GRE() /
+             IP(src=self.pg2.remote_ip4, dst=self.pg3.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, self.snat_addr)
+            self.assertEqual(packet[IP].dst, server.ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+        # server to host
+        p = (Ether(dst=self.pg0.local_mac, src=server.mac) /
+             IP(src=server.ip4, dst=self.snat_addr) /
+             GRE() /
+             IP(src=self.pg3.remote_ip4, dst=self.pg2.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, server_nat_ip)
+            self.assertEqual(packet[IP].dst, host.ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
     def tearDown(self):
         super(TestSNAT, self).tearDown()
         if not self.vpp_dead:
@@ -3230,6 +3351,130 @@ class TestNAT64(MethodHolder):
                                   vrf1_pref64_len)
         self.verify_capture_in_ip6(capture, dst_ip, self.pg2.remote_ip6)
 
+    def _test_unknown_proto(self):
+        """ NAT64 translate packet with unknown protocol """
+
+        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
+                                                self.nat_addr_n)
+        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
+        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
+        remote_ip6 = self.compose_ip6(self.pg1.remote_ip4, '64:ff9b::', 96)
+
+        # in2out
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=self.pg0.remote_ip6, dst=remote_ip6) /
+             TCP(sport=self.tcp_port_in, dport=20))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg1.get_capture(1)
+
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=self.pg0.remote_ip6, dst=remote_ip6) /
+             GRE() /
+             IP(src=self.pg2.local_ip4, dst=self.pg2.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg1.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IP].src, self.nat_addr)
+            self.assertEqual(packet[IP].dst, self.pg1.remote_ip4)
+            self.assertTrue(packet.haslayer(GRE))
+            self.check_ip_checksum(packet)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+        # out2in
+        p = (Ether(dst=self.pg1.local_mac, src=self.pg1.remote_mac) /
+             IP(src=self.pg1.remote_ip4, dst=self.nat_addr) /
+             GRE() /
+             IP(src=self.pg2.remote_ip4, dst=self.pg2.local_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg1.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IPv6].src, remote_ip6)
+            self.assertEqual(packet[IPv6].dst, self.pgi0.remote_ip6)
+            self.assertTrue(packet.haslayer(GRE))
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+    def _test_hairpinning_unknown_proto(self):
+        """ NAT64 translate packet with unknown protocol - hairpinning """
+
+        client = self.pg0.remote_hosts[0]
+        server = self.pg0.remote_hosts[1]
+        server_tcp_in_port = 22
+        server_tcp_out_port = 4022
+        client_tcp_in_port = 1234
+        client_udp_in_port = 1235
+        nat_addr_ip6 = self.compose_ip6(self.nat_addr, '64:ff9b::', 96)
+
+        self.vapi.nat64_add_del_pool_addr_range(self.nat_addr_n,
+                                                self.nat_addr_n)
+        self.vapi.nat64_add_del_interface(self.pg0.sw_if_index)
+        self.vapi.nat64_add_del_interface(self.pg1.sw_if_index, is_inside=0)
+
+        self.vapi.nat64_add_del_static_bib(server.ip6n,
+                                           self.nat_addr_n,
+                                           server_tcp_in_port,
+                                           server_tcp_out_port,
+                                           IP_PROTOS.tcp)
+
+        # client to server
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=client.ip6, dst=nat_addr_ip6) /
+             TCP(sport=client_tcp_in_port, dport=server_tcp_out_port))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=client.ip6, dst=nat_addr_ip6) /
+             GRE() /
+             IP(src=self.pg2.local_ip4, dst=self.pg2.remote_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IPv6].src, nat_addr_ip6)
+            self.assertEqual(packet[IPv6].dst, server.ip6)
+            self.assertTrue(packet.haslayer(GRE))
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
+        # server to client
+        p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+             IPv6(src=server.ip6, dst=nat_addr_ip6) /
+             GRE() /
+             IP(src=self.pg2.remote_ip4, dst=self.pg2.local_ip4) /
+             TCP(sport=1234, dport=1234))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        p = self.pg0.get_capture(1)
+        packet = p[0]
+        try:
+            self.assertEqual(packet[IPv6].src, nat_addr_ip6)
+            self.assertEqual(packet[IPv6].dst, client.ip6)
+            self.assertTrue(packet.haslayer(GRE))
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", packet))
+            raise
+
     def nat64_get_ses_num(self):
         """
         Return number of active NAT64 sessions.