NAT44: multiple outside FIB tables (VPP-1314)

Change-Id: I56eb15f8fd2d3049845287dc3df7870582764f8b
Signed-off-by: Matus Fabian <matfabia@cisco.com>

5 files changed, 620 insertions, 62 deletions

diff --git a/src/plugins/nat/in2out.c b/src/plugins/nat/in2out.c
index 2a41b952892..ae9c836b773 100755
--- a/src/plugins/nat/in2out.c
+++ b/src/plugins/nat/in2out.c
@@ -191,6 +191,7 @@ snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
     return 0;
 
   fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
+  nat_outside_fib_t *outside_fib;
   fib_prefix_t pfx = {
     .fp_proto = FIB_PROTOCOL_IP4,
     .fp_len = 32,
@@ -210,10 +211,20 @@ snat_not_translate_fast (snat_main_t * sm, vlib_node_runtime_t *node,
       u32 sw_if_index = fib_entry_get_resolving_interface (fei);
       if (sw_if_index == ~0)
         {
-          fei = fib_table_lookup (sm->outside_fib_index, &pfx);
-          if (FIB_NODE_INDEX_INVALID != fei)
-            sw_if_index = fib_entry_get_resolving_interface (fei);
+          vec_foreach (outside_fib, sm->outside_fibs)
+            {
+              fei = fib_table_lookup (outside_fib->fib_index, &pfx);
+              if (FIB_NODE_INDEX_INVALID != fei)
+                {
+                  sw_if_index = fib_entry_get_resolving_interface (fei);
+                  if (sw_if_index != ~0)
+                    break;
+                }
+            }
         }
+      if (sw_if_index == ~0)
+        return 1;
+
       snat_interface_t *i;
       pool_foreach (i, sm->interfaces,
       ({
@@ -273,7 +284,7 @@ nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
   key0.addr = ip0->src_address;
   key0.port = src_port;
   key0.protocol = proto0;
-  key0.fib_index = sm->outside_fib_index;
+  key0.fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
   kv0.key = key0.as_u64;
 
   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].out2in, &kv0,
@@ -284,7 +295,6 @@ nat_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip0,
   key0.addr = ip0->dst_address;
   key0.port = dst_port;
   key0.protocol = proto0;
-  key0.fib_index = sm->inside_fib_index;
   kv0.key = key0.as_u64;
   if (!clib_bihash_search_8_8 (&sm->per_thread_data[thread_index].in2out, &kv0,
                                &value0))
@@ -315,10 +325,17 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
   clib_bihash_kv_8_8_t kv0;
   snat_session_key_t key1;
   u32 address_index = ~0;
-  u32 outside_fib_index;
-  uword * p;
   udp_header_t * udp0 = ip4_next_header (ip0);
   u8 is_sm = 0;
+  nat_outside_fib_t *outside_fib;
+  fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
+  fib_prefix_t pfx = {
+    .fp_proto = FIB_PROTOCOL_IP4,
+    .fp_len = 32,
+    .fp_addr = {
+        .ip4.as_u32 = ip0->dst_address.as_u32,
+    },
+  };
 
   if (PREDICT_FALSE (maximum_sessions_exceeded(sm, thread_index)))
     {
@@ -328,14 +345,6 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
       return SNAT_IN2OUT_NEXT_DROP;
     }
 
-  p = hash_get (sm->ip4_main->fib_index_by_table_id, sm->outside_vrf_id);
-  if (! p)
-    {
-      b0->error = node->errors[SNAT_IN2OUT_ERROR_BAD_OUTSIDE_FIB];
-      return SNAT_IN2OUT_NEXT_DROP;
-    }
-  outside_fib_index = p[0];
-
   key1.protocol = key0->protocol;
 
   u = nat_user_get_or_create (sm, &ip0->src_address, rx_fib_index0,
@@ -377,7 +386,30 @@ static u32 slow_path (snat_main_t *sm, vlib_buffer_t *b0,
   s->in2out = *key0;
   s->out2in = key1;
   s->out2in.protocol = key0->protocol;
-  s->out2in.fib_index = outside_fib_index;
+  s->out2in.fib_index = sm->outside_fib_index;
+  switch (vec_len (sm->outside_fibs))
+    {
+    case 0:
+      s->out2in.fib_index = sm->outside_fib_index;
+      break;
+    case 1:
+      s->out2in.fib_index = sm->outside_fibs[0].fib_index;
+      break;
+    default:
+      vec_foreach (outside_fib, sm->outside_fibs)
+        {
+          fei = fib_table_lookup (outside_fib->fib_index, &pfx);
+          if (FIB_NODE_INDEX_INVALID != fei)
+            {
+              if (fib_entry_get_resolving_interface (fei) != ~0)
+                {
+                  s->out2in.fib_index = outside_fib->fib_index;
+                  break;
+                }
+            }
+        }
+      break;
+    }
   s->ext_host_addr.as_u32 = ip0->dst_address.as_u32;
   s->ext_host_port = udp0->dst_port;
   *sessionp = s;
@@ -1001,7 +1033,7 @@ nat_hairpinning_sm_unknown_proto (snat_main_t * sm,
   u32 old_addr, new_addr;
   ip_csum_t sum;
 
-  make_sm_kv (&kv, &ip->dst_address, 0, sm->outside_fib_index, 0);
+  make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
     return;
 
@@ -1049,8 +1081,8 @@ nat_in2out_sm_unknown_proto (snat_main_t *sm,
   /* Hairpinning */
   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
     {
+      vnet_buffer(b)->sw_if_index[VLIB_TX] = m->fib_index;
       nat_hairpinning_sm_unknown_proto (sm, b, ip);
-      vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
     }
 
   return 0;
@@ -2359,6 +2391,15 @@ slow_path_ed (snat_main_t *sm,
   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
   nat_ed_ses_key_t *key = (nat_ed_ses_key_t *) kv->key;
   u32 proto = ip_proto_to_snat_proto (key->proto);
+  nat_outside_fib_t *outside_fib;
+  fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
+  fib_prefix_t pfx = {
+    .fp_proto = FIB_PROTOCOL_IP4,
+    .fp_len = 32,
+    .fp_addr = {
+        .ip4.as_u32 = key->r_addr.as_u32,
+    },
+  };
 
   if (PREDICT_FALSE (maximum_sessions_exceeded (sm, thread_index)))
     {
@@ -2418,12 +2459,36 @@ slow_path_ed (snat_main_t *sm,
   s->out2in = key1;
   s->out2in.protocol = key0.protocol;
 
+  switch (vec_len (sm->outside_fibs))
+    {
+    case 0:
+      s->out2in.fib_index = sm->outside_fib_index;
+      break;
+    case 1:
+      s->out2in.fib_index = sm->outside_fibs[0].fib_index;
+      break;
+    default:
+      vec_foreach (outside_fib, sm->outside_fibs)
+        {
+          fei = fib_table_lookup (outside_fib->fib_index, &pfx);
+          if (FIB_NODE_INDEX_INVALID != fei)
+            {
+              if (fib_entry_get_resolving_interface (fei) != ~0)
+                {
+                  s->out2in.fib_index = outside_fib->fib_index;
+                  break;
+                }
+            }
+        }
+      break;
+    }
+
   /* Add to lookup tables */
   kv->value = s - tsm->sessions;
   if (clib_bihash_add_del_16_8 (&tsm->in2out_ed, kv, 1))
     nat_log_notice ("in2out-ed key add failed");
 
-  make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, key1.fib_index,
+  make_ed_kv (kv, &key1.addr, &key->r_addr, key->proto, s->out2in.fib_index,
               key1.port, key->r_port);
   kv->value = s - tsm->sessions;
   if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, kv, 1))
@@ -2544,16 +2609,17 @@ nat44_ed_not_translate_output_feature (snat_main_t * sm, ip4_header_t * ip,
   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
   snat_interface_t *i;
   snat_session_t *s;
+  u32 fib_index = ip4_fib_table_get_index_for_sw_if_index (sw_if_index);
 
   /* src NAT check */
-  make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto,
-              sm->outside_fib_index, src_port, dst_port);
+  make_ed_kv (&kv, &ip->src_address, &ip->dst_address, proto, fib_index,
+              src_port, dst_port);
   if (!clib_bihash_search_16_8 (&tsm->out2in_ed, &kv, &value))
     return 1;
 
   /* dst NAT check */
-  make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto,
-              sm->inside_fib_index, dst_port, src_port);
+  make_ed_kv (&kv, &ip->dst_address, &ip->src_address, proto, fib_index,
+              dst_port, src_port);
   if (!clib_bihash_search_16_8 (&tsm->in2out_ed, &kv, &value))
   {
     s = pool_elt_at_index (tsm->sessions, value.value);
@@ -2688,7 +2754,7 @@ nat44_ed_hairpinning_unknown_proto (snat_main_t *sm,
               sm->outside_fib_index, 0, 0);
   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
     {
-      make_sm_kv (&kv, &ip->dst_address, 0, sm->outside_fib_index, 0);
+      make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
         return;
 
@@ -2729,10 +2795,42 @@ nat44_ed_in2out_unknown_proto (snat_main_t *sm,
   snat_main_per_thread_data_t *tsm = &sm->per_thread_data[thread_index];
   u32 elt_index, head_index, ses_index;
   snat_session_t * s;
-  u32 address_index = ~0;
+  u32 address_index = ~0, outside_fib_index = sm->outside_fib_index;
   int i;
   u8 is_sm = 0;
+  nat_outside_fib_t *outside_fib;
+  fib_node_index_t fei = FIB_NODE_INDEX_INVALID;
+  fib_prefix_t pfx = {
+    .fp_proto = FIB_PROTOCOL_IP4,
+    .fp_len = 32,
+    .fp_addr = {
+        .ip4.as_u32 = ip->dst_address.as_u32,
+    },
+  };
 
+  switch (vec_len (sm->outside_fibs))
+    {
+    case 0:
+      outside_fib_index = sm->outside_fib_index;
+      break;
+    case 1:
+      outside_fib_index = sm->outside_fibs[0].fib_index;
+      break;
+    default:
+      vec_foreach (outside_fib, sm->outside_fibs)
+        {
+          fei = fib_table_lookup (outside_fib->fib_index, &pfx);
+          if (FIB_NODE_INDEX_INVALID != fei)
+            {
+              if (fib_entry_get_resolving_interface (fei) != ~0)
+                {
+                  outside_fib_index = outside_fib->fib_index;
+                  break;
+                }
+            }
+        }
+      break;
+    }
   old_addr = ip->src_address.as_u32;
 
   make_ed_kv (&s_kv, &ip->src_address, &ip->dst_address, ip->protocol,
@@ -2799,7 +2897,7 @@ nat44_ed_in2out_unknown_proto (snat_main_t *sm,
                   address_index = s->outside_address_index;
 
                   make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address,
-                              ip->protocol, sm->outside_fib_index, 0, 0);
+                              ip->protocol, outside_fib_index, 0, 0);
                   if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
                     goto create_ses;
 
@@ -2810,7 +2908,7 @@ nat44_ed_in2out_unknown_proto (snat_main_t *sm,
           for (i = 0; i < vec_len (sm->addresses); i++)
             {
               make_ed_kv (&s_kv, &sm->addresses[i].addr, &ip->dst_address,
-                          ip->protocol, sm->outside_fib_index, 0, 0);
+                          ip->protocol, outside_fib_index, 0, 0);
               if (clib_bihash_search_16_8 (&tsm->out2in_ed, &s_kv, &s_value))
                 {
                   new_addr = ip->src_address.as_u32 =
@@ -2835,7 +2933,7 @@ create_ses:
       s->flags |= SNAT_SESSION_FLAG_ENDPOINT_DEPENDENT;
       s->outside_address_index = address_index;
       s->out2in.addr.as_u32 = new_addr;
-      s->out2in.fib_index = sm->outside_fib_index;
+      s->out2in.fib_index = outside_fib_index;
       s->in2out.addr.as_u32 = old_addr;
       s->in2out.fib_index = rx_fib_index;
       s->in2out.port = s->out2in.port = ip->protocol;
@@ -2851,7 +2949,7 @@ create_ses:
         nat_log_notice ("in2out key add failed");
 
       make_ed_kv (&s_kv, &s->out2in.addr, &ip->dst_address, ip->protocol,
-                  sm->outside_fib_index, 0, 0);
+                  outside_fib_index, 0, 0);
       s_kv.value = s - tsm->sessions;
       if (clib_bihash_add_del_16_8 (&tsm->out2in_ed, &s_kv, 1))
         nat_log_notice ("out2in key add failed");
@@ -2872,7 +2970,7 @@ create_ses:
     nat44_ed_hairpinning_unknown_proto(sm, b, ip);
 
   if (vnet_buffer(b)->sw_if_index[VLIB_TX] == ~0)
-    vnet_buffer(b)->sw_if_index[VLIB_TX] = sm->outside_fib_index;
+    vnet_buffer(b)->sw_if_index[VLIB_TX] = outside_fib_index;
 
   return s;
 }
@@ -4692,7 +4790,7 @@ is_hairpinning (snat_main_t *sm, ip4_address_t * dst_addr)
     }
 
   m_key.addr.as_u32 = dst_addr->as_u32;
-  m_key.fib_index = sm->outside_fib_index;
+  m_key.fib_index = 0;
   m_key.port = 0;
   m_key.protocol = 0;
   kv.key = m_key.as_u64;
diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
index f236568db28..8dce3b96c50 100755
--- a/src/plugins/nat/nat.c
+++ b/src/plugins/nat/nat.c
@@ -895,7 +895,7 @@ int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
   m_key.addr = e_addr;
   m_key.port = addr_only ? 0 : e_port;
   m_key.protocol = addr_only ? 0 : proto;
-  m_key.fib_index = sm->outside_fib_index;
+  m_key.fib_index = 0;
   kv.key = m_key.as_u64;
   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
     m = 0;
@@ -1031,7 +1031,7 @@ int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 
       m_key.addr = m->external_addr;
       m_key.port = m->external_port;
-      m_key.fib_index = sm->outside_fib_index;
+      m_key.fib_index = 0;
       kv.key = m_key.as_u64;
       kv.value = m - sm->static_mappings;
       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1);
@@ -1130,7 +1130,7 @@ int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 
       m_key.addr = m->external_addr;
       m_key.port = m->external_port;
-      m_key.fib_index = sm->outside_fib_index;
+      m_key.fib_index = 0;
       kv.key = m_key.as_u64;
       clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0);
 
@@ -1244,7 +1244,7 @@ int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
   m_key.addr = e_addr;
   m_key.port = e_port;
   m_key.protocol = proto;
-  m_key.fib_index = sm->outside_fib_index;
+  m_key.fib_index = 0;
   kv.key = m_key.as_u64;
   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
     m = 0;
@@ -1315,7 +1315,7 @@ int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
       m_key.addr = m->external_addr;
       m_key.port = m->external_port;
       m_key.protocol = m->proto;
-      m_key.fib_index = sm->outside_fib_index;
+      m_key.fib_index = 0;
       kv.key = m_key.as_u64;
       kv.value = m - sm->static_mappings;
       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 1))
@@ -1397,7 +1397,7 @@ int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
       m_key.addr = m->external_addr;
       m_key.port = m->external_port;
       m_key.protocol = m->proto;
-      m_key.fib_index = sm->outside_fib_index;
+      m_key.fib_index = 0;
       kv.key = m_key.as_u64;
       if (clib_bihash_add_del_8_8(&sm->static_mapping_by_external, &kv, 0))
         {
@@ -1586,6 +1586,9 @@ int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
   snat_address_t * ap;
   snat_static_mapping_t * m;
   snat_det_map_t * dm;
+  nat_outside_fib_t *outside_fib;
+  u32 fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
+                                                       sw_if_index);
 
   if (sm->out2in_dpo && !is_inside)
     return VNET_API_ERROR_UNSUPPORTED;
@@ -1618,6 +1621,31 @@ int snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
     sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
                                                       NAT_FQ_NELTS);
 
+  if (!is_inside)
+    {
+      vec_foreach (outside_fib, sm->outside_fibs)
+        {
+          if (outside_fib->fib_index == fib_index)
+            {
+              if (is_del)
+                {
+                  outside_fib->refcount--;
+                  if (!outside_fib->refcount)
+                    vec_del1 (sm->outside_fibs, outside_fib - sm->outside_fibs);
+                }
+              else
+                outside_fib->refcount++;
+              goto feature_set;
+            }
+        }
+      if (!is_del)
+        {
+          vec_add2 (sm->outside_fibs, outside_fib, 1);
+          outside_fib->refcount = 1;
+          outside_fib->fib_index = fib_index;
+        }
+    }
+feature_set:
   pool_foreach (i, sm->interfaces,
   ({
     if (i->sw_if_index == sw_if_index)
@@ -2112,13 +2140,16 @@ int snat_static_mapping_match (snat_main_t * sm,
   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
   u32 rand, lo = 0, hi, mid;
 
+  m_key.fib_index = match.fib_index;
   if (by_external)
-    mapping_hash = &sm->static_mapping_by_external;
+    {
+      mapping_hash = &sm->static_mapping_by_external;
+      m_key.fib_index = 0;
+    }
 
   m_key.addr = match.addr;
   m_key.port = clib_net_to_host_u16 (match.port);
   m_key.protocol = match.protocol;
-  m_key.fib_index = match.fib_index;
 
   kv.key = m_key.as_u64;
 
diff --git a/src/plugins/nat/nat.h b/src/plugins/nat/nat.h
index bd00a5246dd..97bbec21360 100644
--- a/src/plugins/nat/nat.h
+++ b/src/plugins/nat/nat.h
@@ -203,6 +203,11 @@ typedef struct {
 } snat_address_t;
 
 typedef struct {
+  u32 fib_index;
+  u32 refcount;
+} nat_outside_fib_t;
+
+typedef struct {
   u16 in_port;
   snat_det_out_key_t out;
   u8 state;
@@ -350,6 +355,9 @@ typedef struct snat_main_s {
   u8 psid_length;
   u16 psid;
 
+  /* vector of outside fibs */
+  nat_outside_fib_t * outside_fibs;
+
   /* Vector of twice NAT addresses for extenal hosts */
   snat_address_t * twice_nat_addresses;
 
diff --git a/src/plugins/nat/out2in.c b/src/plugins/nat/out2in.c
index 80465b071d4..1a1a1f1f639 100755
--- a/src/plugins/nat/out2in.c
+++ b/src/plugins/nat/out2in.c
@@ -626,7 +626,7 @@ nat_out2in_sm_unknown_proto (snat_main_t *sm,
   m_key.addr = ip->dst_address;
   m_key.port = 0;
   m_key.protocol = 0;
-  m_key.fib_index = rx_fib_index;
+  m_key.fib_index = 0;
   kv.key = m_key.as_u64;
   if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
     return 1;
@@ -1962,7 +1962,7 @@ nat44_ed_out2in_unknown_proto (snat_main_t *sm,
           return 0;
         }
 
-      make_sm_kv (&kv, &ip->dst_address, 0, rx_fib_index, 0);
+      make_sm_kv (&kv, &ip->dst_address, 0, 0, 0);
       if (clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
         {
           b->error = node->errors[SNAT_OUT2IN_ERROR_NO_TRANSLATION];
diff --git a/test/test_nat.py b/test/test_nat.py
index 4ae2850678d..973666a3f9e 100644
--- a/test/test_nat.py
+++ b/test/test_nat.py
@@ -2383,6 +2383,8 @@ class TestNAT44(MethodHolder):
         self.pg1.set_table_ip4(vrf_id2)
         self.pg0.config_ip4()
         self.pg1.config_ip4()
+        self.pg0.resolve_arp()
+        self.pg1.resolve_arp()
 
         self.nat44_add_address(nat_ip1, vrf_id=vrf_id1)
         self.nat44_add_address(nat_ip2, vrf_id=vrf_id2)
@@ -2391,28 +2393,34 @@ class TestNAT44(MethodHolder):
         self.vapi.nat44_interface_add_del_feature(self.pg2.sw_if_index,
                                                   is_inside=0)
 
-        # first VRF
-        pkts = self.create_stream_in(self.pg0, self.pg2)
-        self.pg0.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg2.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip1)
+        try:
+            # first VRF
+            pkts = self.create_stream_in(self.pg0, self.pg2)
+            self.pg0.add_stream(pkts)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            capture = self.pg2.get_capture(len(pkts))
+            self.verify_capture_out(capture, nat_ip1)
 
-        # second VRF
-        pkts = self.create_stream_in(self.pg1, self.pg2)
-        self.pg1.add_stream(pkts)
-        self.pg_enable_capture(self.pg_interfaces)
-        self.pg_start()
-        capture = self.pg2.get_capture(len(pkts))
-        self.verify_capture_out(capture, nat_ip2)
+            # second VRF
+            pkts = self.create_stream_in(self.pg1, self.pg2)
+            self.pg1.add_stream(pkts)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            capture = self.pg2.get_capture(len(pkts))
+            self.verify_capture_out(capture, nat_ip2)
 
-        self.pg0.unconfig_ip4()
-        self.pg1.unconfig_ip4()
-        self.pg0.set_table_ip4(0)
-        self.pg1.set_table_ip4(0)
-        self.vapi.ip_table_add_del(vrf_id1, is_add=0)
-        self.vapi.ip_table_add_del(vrf_id2, is_add=0)
+        finally:
+            self.pg0.unconfig_ip4()
+            self.pg1.unconfig_ip4()
+            self.pg0.set_table_ip4(0)
+            self.pg1.set_table_ip4(0)
+            self.pg0.config_ip4()
+            self.pg1.config_ip4()
+            self.pg0.resolve_arp()
+            self.pg1.resolve_arp()
+            self.vapi.ip_table_add_del(vrf_id1, is_add=0)
+            self.vapi.ip_table_add_del(vrf_id2, is_add=0)
 
     def test_vrf_feature_independent(self):
         """ NAT44 tenant VRF independent address pool mode """
@@ -3143,6 +3151,74 @@ class TestNAT44(MethodHolder):
                 self.verify_ipfix_max_fragments_ip4(data, 0,
                                                     self.pg0.remote_ip4n)
 
+    def test_multiple_outside_vrf(self):
+        """ Multiple outside VRF """
+        vrf_id1 = 1
+        vrf_id2 = 2
+
+        self.pg1.unconfig_ip4()
+        self.pg2.unconfig_ip4()
+        self.vapi.ip_table_add_del(vrf_id1, is_add=1)
+        self.vapi.ip_table_add_del(vrf_id2, is_add=1)
+        self.pg1.set_table_ip4(vrf_id1)
+        self.pg2.set_table_ip4(vrf_id2)
+        self.pg1.config_ip4()
+        self.pg2.config_ip4()
+        self.pg1.resolve_arp()
+        self.pg2.resolve_arp()
+
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg1.sw_if_index,
+                                                  is_inside=0)
+        self.vapi.nat44_interface_add_del_feature(self.pg2.sw_if_index,
+                                                  is_inside=0)
+
+        try:
+            # first VRF
+            pkts = self.create_stream_in(self.pg0, self.pg1)
+            self.pg0.add_stream(pkts)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            capture = self.pg1.get_capture(len(pkts))
+            self.verify_capture_out(capture, self.nat_addr)
+
+            pkts = self.create_stream_out(self.pg1, self.nat_addr)
+            self.pg1.add_stream(pkts)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            capture = self.pg0.get_capture(len(pkts))
+            self.verify_capture_in(capture, self.pg0)
+
+            self.tcp_port_in = 60303
+            self.udp_port_in = 60304
+            self.icmp_id_in = 60305
+
+            # second VRF
+            pkts = self.create_stream_in(self.pg0, self.pg2)
+            self.pg0.add_stream(pkts)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            capture = self.pg2.get_capture(len(pkts))
+            self.verify_capture_out(capture, self.nat_addr)
+
+            pkts = self.create_stream_out(self.pg2, self.nat_addr)
+            self.pg2.add_stream(pkts)
+            self.pg_enable_capture(self.pg_interfaces)
+            self.pg_start()
+            capture = self.pg0.get_capture(len(pkts))
+            self.verify_capture_in(capture, self.pg0)
+
+        finally:
+            self.pg1.unconfig_ip4()
+            self.pg2.unconfig_ip4()
+            self.pg1.set_table_ip4(0)
+            self.pg2.set_table_ip4(0)
+            self.pg1.config_ip4()
+            self.pg2.config_ip4()
+            self.pg1.resolve_arp()
+            self.pg2.resolve_arp()
+
     def tearDown(self):
         super(TestNAT44, self).tearDown()
         if not self.vpp_dead:
@@ -3183,7 +3259,7 @@ class TestNAT44EndpointDependent(MethodHolder):
             cls.ipfix_domain_id = 1
             cls.tcp_external_port = 80
 
-            cls.create_pg_interfaces(range(5))
+            cls.create_pg_interfaces(range(7))
             cls.interfaces = list(cls.pg_interfaces[0:3])
 
             for i in cls.interfaces:
@@ -3207,6 +3283,58 @@ class TestNAT44EndpointDependent(MethodHolder):
             cls.pg4._remote_hosts[1]._ip4 = cls.pg4._remote_hosts[0]._ip4
             cls.pg4.resolve_arp()
 
+            zero_ip4n = socket.inet_pton(socket.AF_INET, "0.0.0.0")
+            cls.vapi.ip_table_add_del(1, is_add=1)
+
+            cls.pg5._local_ip4 = "10.1.1.1"
+            cls.pg5._local_ip4n = socket.inet_pton(socket.AF_INET,
+                                                   cls.pg5.local_ip4)
+            cls.pg5._remote_hosts[0]._ip4 = "10.1.1.2"
+            cls.pg5._remote_hosts[0]._ip4n = socket.inet_pton(
+                socket.AF_INET, cls.pg5.remote_ip4)
+            cls.pg5.set_table_ip4(1)
+            cls.pg5.config_ip4()
+            cls.pg5.admin_up()
+            cls.vapi.ip_add_del_route(dst_address=cls.pg5.remote_ip4n,
+                                      dst_address_length=32,
+                                      table_id=1,
+                                      next_hop_sw_if_index=cls.pg5.sw_if_index,
+                                      next_hop_address=zero_ip4n)
+
+            cls.pg6._local_ip4 = "10.1.2.1"
+            cls.pg6._local_ip4n = socket.inet_pton(socket.AF_INET,
+                                                   cls.pg6.local_ip4)
+            cls.pg6._remote_hosts[0]._ip4 = "10.1.2.2"
+            cls.pg6._remote_hosts[0]._ip4n = socket.inet_pton(
+                socket.AF_INET, cls.pg6.remote_ip4)
+            cls.pg6.set_table_ip4(1)
+            cls.pg6.config_ip4()
+            cls.pg6.admin_up()
+            cls.vapi.ip_add_del_route(dst_address=cls.pg6.remote_ip4n,
+                                      dst_address_length=32,
+                                      table_id=1,
+                                      next_hop_sw_if_index=cls.pg6.sw_if_index,
+                                      next_hop_address=zero_ip4n)
+
+            cls.vapi.ip_add_del_route(dst_address=cls.pg6.remote_ip4n,
+                                      dst_address_length=16,
+                                      next_hop_address=zero_ip4n,
+                                      table_id=0,
+                                      next_hop_table_id=1)
+            cls.vapi.ip_add_del_route(dst_address=zero_ip4n,
+                                      dst_address_length=0,
+                                      next_hop_address=zero_ip4n,
+                                      table_id=1,
+                                      next_hop_table_id=0)
+            cls.vapi.ip_add_del_route(dst_address=zero_ip4n,
+                                      dst_address_length=0,
+                                      table_id=0,
+                                      next_hop_sw_if_index=cls.pg1.sw_if_index,
+                                      next_hop_address=cls.pg1.local_ip4n)
+
+            cls.pg5.resolve_arp()
+            cls.pg6.resolve_arp()
+
         except Exception:
             super(TestNAT44EndpointDependent, cls).tearDownClass()
             raise
@@ -4422,6 +4550,299 @@ class TestNAT44EndpointDependent(MethodHolder):
         capture = self.pg0.get_capture(len(pkts))
         self.verify_capture_in(capture, self.pg0)
 
+    def test_multiple_vrf(self):
+        """ Multiple VRF setup """
+        external_addr = '1.2.3.4'
+        external_port = 80
+        local_port = 8080
+        port = 0
+
+        self.vapi.nat44_forwarding_enable_disable(1)
+        self.nat44_add_address(self.nat_addr)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg0.sw_if_index,
+                                                  is_inside=0)
+        self.vapi.nat44_interface_add_del_output_feature(self.pg1.sw_if_index,
+                                                         is_inside=0)
+        self.vapi.nat44_interface_add_del_feature(self.pg5.sw_if_index)
+        self.vapi.nat44_interface_add_del_feature(self.pg5.sw_if_index,
+                                                  is_inside=0)
+        self.vapi.nat44_interface_add_del_feature(self.pg6.sw_if_index,
+                                                  is_inside=0)
+        self.nat44_add_static_mapping(self.pg5.remote_ip4, external_addr,
+                                      local_port, external_port, vrf_id=1,
+                                      proto=IP_PROTOS.tcp, out2in_only=1)
+        self.nat44_add_static_mapping(
+             self.pg0.remote_ip4, external_sw_if_index=self.pg0.sw_if_index,
+             local_port=local_port, vrf_id=0, external_port=external_port,
+             proto=IP_PROTOS.tcp, out2in_only=1)
+
+        # from client to service (both VRF1)
+        p = (Ether(src=self.pg6.remote_mac, dst=self.pg6.local_mac) /
+             IP(src=self.pg6.remote_ip4, dst=external_addr) /
+             TCP(sport=12345, dport=external_port))
+        self.pg6.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg5.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.dst, self.pg5.remote_ip4)
+            self.assertEqual(tcp.dport, local_port)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # from service back to client (both VRF1)
+        p = (Ether(src=self.pg5.remote_mac, dst=self.pg5.local_mac) /
+             IP(src=self.pg5.remote_ip4, dst=self.pg6.remote_ip4) /
+             TCP(sport=local_port, dport=12345))
+        self.pg5.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg6.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, external_addr)
+            self.assertEqual(tcp.sport, external_port)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # dynamic NAT from VRF1 to VRF0 (output-feature)
+        p = (Ether(src=self.pg5.remote_mac, dst=self.pg5.local_mac) /
+             IP(src=self.pg5.remote_ip4, dst=self.pg1.remote_ip4) /
+             TCP(sport=2345, dport=22))
+        self.pg5.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg1.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.nat_addr)
+            self.assertNotEqual(tcp.sport, 2345)
+            self.assert_packet_checksums_valid(p)
+            port = tcp.sport
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
+             IP(src=self.pg1.remote_ip4, dst=self.nat_addr) /
+             TCP(sport=22, dport=port))
+        self.pg1.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg5.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.dst, self.pg5.remote_ip4)
+            self.assertEqual(tcp.dport, 2345)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # from client VRF1 to service VRF0
+        p = (Ether(src=self.pg6.remote_mac, dst=self.pg6.local_mac) /
+             IP(src=self.pg6.remote_ip4, dst=self.pg0.local_ip4) /
+             TCP(sport=12346, dport=external_port))
+        self.pg6.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.dst, self.pg0.remote_ip4)
+            self.assertEqual(tcp.dport, local_port)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # from service VRF0 back to client VRF1
+        p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+             IP(src=self.pg0.remote_ip4, dst=self.pg6.remote_ip4) /
+             TCP(sport=local_port, dport=12346))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg6.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.pg0.local_ip4)
+            self.assertEqual(tcp.sport, external_port)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # from client VRF0 to service VRF1
+        p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+             IP(src=self.pg0.remote_ip4, dst=external_addr) /
+             TCP(sport=12347, dport=external_port))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg5.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.dst, self.pg5.remote_ip4)
+            self.assertEqual(tcp.dport, local_port)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # from service VRF1 back to client VRF0
+        p = (Ether(src=self.pg5.remote_mac, dst=self.pg5.local_mac) /
+             IP(src=self.pg5.remote_ip4, dst=self.pg0.remote_ip4) /
+             TCP(sport=local_port, dport=12347))
+        self.pg5.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, external_addr)
+            self.assertEqual(tcp.sport, external_port)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # from client to server (both VRF1, no translation)
+        p = (Ether(src=self.pg6.remote_mac, dst=self.pg6.local_mac) /
+             IP(src=self.pg6.remote_ip4, dst=self.pg5.remote_ip4) /
+             TCP(sport=12348, dport=local_port))
+        self.pg6.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg5.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.dst, self.pg5.remote_ip4)
+            self.assertEqual(tcp.dport, local_port)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # from server back to client (both VRF1, no translation)
+        p = (Ether(src=self.pg5.remote_mac, dst=self.pg5.local_mac) /
+             IP(src=self.pg5.remote_ip4, dst=self.pg6.remote_ip4) /
+             TCP(sport=local_port, dport=12348))
+        self.pg5.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg6.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.pg5.remote_ip4)
+            self.assertEqual(tcp.sport, local_port)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # from client VRF1 to server VRF0 (no translation)
+        p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+             IP(src=self.pg0.remote_ip4, dst=self.pg6.remote_ip4) /
+             TCP(sport=local_port, dport=12349))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg6.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.pg0.remote_ip4)
+            self.assertEqual(tcp.sport, local_port)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # from server VRF0 back to client VRF1 (no translation)
+        p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+             IP(src=self.pg0.remote_ip4, dst=self.pg6.remote_ip4) /
+             TCP(sport=local_port, dport=12349))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg6.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.pg0.remote_ip4)
+            self.assertEqual(tcp.sport, local_port)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # from client VRF0 to server VRF1 (no translation)
+        p = (Ether(src=self.pg0.remote_mac, dst=self.pg0.local_mac) /
+             IP(src=self.pg0.remote_ip4, dst=self.pg5.remote_ip4) /
+             TCP(sport=12344, dport=local_port))
+        self.pg0.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg5.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.dst, self.pg5.remote_ip4)
+            self.assertEqual(tcp.dport, local_port)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
+        # from server VRF1 back to client VRF0 (no translation)
+        p = (Ether(src=self.pg5.remote_mac, dst=self.pg5.local_mac) /
+             IP(src=self.pg5.remote_ip4, dst=self.pg0.remote_ip4) /
+             TCP(sport=local_port, dport=12344))
+        self.pg5.add_stream(p)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(1)
+        p = capture[0]
+        try:
+            ip = p[IP]
+            tcp = p[TCP]
+            self.assertEqual(ip.src, self.pg5.remote_ip4)
+            self.assertEqual(tcp.sport, local_port)
+            self.assert_packet_checksums_valid(p)
+        except:
+            self.logger.error(ppp("Unexpected or invalid packet:", p))
+            raise
+
     def tearDown(self):
         super(TestNAT44EndpointDependent, self).tearDown()
         if not self.vpp_dead: