NAT44: LB NAT - local backends in multiple VRFs (VPP-1345)

Add support for local backends in multiple VRFs for load-balancing NAT rules.

Change-Id: I64e6818bd67a7e69985003498cf1f16f7200c334
Signed-off-by: Matus Fabian <matfabia@cisco.com>

7 files changed, 53 insertions, 45 deletions

diff --git a/src/plugins/nat/nat.api b/src/plugins/nat/nat.api
index 4192cf19e3a..26d0fbac094 100644
--- a/src/plugins/nat/nat.api
+++ b/src/plugins/nat/nat.api
@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 
-option version = "2.6.0";
+option version = "3.0.0";
 
 /**
  * @file nat.api
@@ -595,6 +595,7 @@ typeonly manual_endian define nat44_lb_addr_port {
   u8 addr[4];
   u16 port;
   u8 probability;
+  u32 vrf_id;
 };
 
 /** \brief Add/delete NAT44 load-balancing static mapping rule
@@ -604,7 +605,6 @@ typeonly manual_endian define nat44_lb_addr_port {
     @param external_addr - external IPv4 address of the service
     @param external_port - external L4 port number of the service
     @param protocol - IP protocol number of the service
-    @param vrf_id - internal network VRF ID
     @param twice_nat - if 1 translate external host address and port
     @param self_twice_nat - if 1 translate external host address and port
                             whenever external host address equals
@@ -621,7 +621,6 @@ autoreply manual_endian define nat44_add_del_lb_static_mapping {
   u8 external_addr[4];
   u16 external_port;
   u8 protocol;
-  u32 vrf_id;
   u8 twice_nat;
   u8 self_twice_nat;
   u8 out2in_only;
@@ -645,7 +644,6 @@ define nat44_lb_static_mapping_dump {
     @param external_addr - external IPv4 address of the service
     @param external_port - external L4 port number of the service
     @param protocol - IP protocol number of the service
-    @param vrf_id - internal network VRF ID
     @param twice_nat - if 1 translate external host address and port
     @param self_twice_nat - if 1 translate external host address and port
                             whenever external host address equals
@@ -660,7 +658,6 @@ manual_endian define nat44_lb_static_mapping_details {
   u8 external_addr[4];
   u16 external_port;
   u8 protocol;
-  u32 vrf_id;
   u8 twice_nat;
   u8 self_twice_nat;
   u8 out2in_only;
diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
index f35120663fb..cdf05fd351a 100755
--- a/src/plugins/nat/nat.c
+++ b/src/plugins/nat/nat.c
@@ -1235,7 +1235,7 @@ int snat_add_static_mapping(ip4_address_t l_addr, ip4_address_t e_addr,
 }
 
 int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
-                                     snat_protocol_t proto, u32 vrf_id,
+                                     snat_protocol_t proto,
                                      nat44_lb_addr_port_t *locals, u8 is_add,
                                      twice_nat_type_t twice_nat, u8 out2in_only,
                                      u8 *tag)
@@ -1244,7 +1244,6 @@ int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
   snat_static_mapping_t *m;
   snat_session_key_t m_key;
   clib_bihash_kv_8_8_t kv, value;
-  u32 fib_index;
   snat_address_t *a = 0;
   int i;
   nat44_lb_addr_port_t *local;
@@ -1277,10 +1276,6 @@ int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
       if (vec_len (locals) < 2)
         return VNET_API_ERROR_INVALID_VALUE;
 
-      fib_index = fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4,
-                                                     vrf_id,
-                                                     FIB_SOURCE_PLUGIN_LOW);
-
       /* Find external address in allocated addresses and reserve port for
          address and port pair mapping when dynamic translations enabled */
       if (!(sm->static_mapping_only || out2in_only))
@@ -1323,8 +1318,6 @@ int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
       m->tag = vec_dup (tag);
       m->external_addr = e_addr;
       m->addr_only = 0;
-      m->vrf_id = vrf_id;
-      m->fib_index = fib_index;
       m->external_port = e_port;
       m->proto = proto;
       m->twice_nat = twice_nat;
@@ -1345,7 +1338,10 @@ int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
       m_key.fib_index = m->fib_index;
       for (i = 0; i < vec_len (locals); i++)
         {
+          locals[i].fib_index = fib_table_find_or_create_and_lock (
+            FIB_PROTOCOL_IP4, locals[i].vrf_id, FIB_SOURCE_PLUGIN_LOW);
           m_key.addr = locals[i].addr;
+          m_key.fib_index = locals[i].fib_index;
           if (!out2in_only)
             {
               m_key.port = locals[i].port;
@@ -1380,8 +1376,6 @@ int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
       if (!m)
         return VNET_API_ERROR_NO_SUCH_ENTRY;
 
-      fib_table_unlock (m->fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
-
       /* Free external address port */
       if (!(sm->static_mapping_only || out2in_only))
         {
@@ -1425,11 +1419,13 @@ int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
 
       vec_foreach (local, m->locals)
         {
+          fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
+                            FIB_SOURCE_PLUGIN_LOW);
           m_key.addr = local->addr;
           if (!out2in_only)
             {
               m_key.port = local->port;
-              m_key.fib_index = m->fib_index;
+              m_key.fib_index = local->fib_index;
               kv.key = m_key.as_u64;
               if (clib_bihash_add_del_8_8(&sm->static_mapping_by_local, &kv, 0))
                 {
@@ -2207,15 +2203,16 @@ get_local:
             }
           mapping->addr = m->locals[lo].addr;
           mapping->port = clib_host_to_net_u16 (m->locals[lo].port);
+          mapping->fib_index = m->locals[lo].fib_index;
         }
       else
         {
+          mapping->fib_index = m->fib_index;
           mapping->addr = m->local_addr;
           /* Address only mapping doesn't change port */
           mapping->port = m->addr_only ? match.port
             : clib_host_to_net_u16 (m->local_port);
         }
-      mapping->fib_index = m->fib_index;
       mapping->protocol = m->proto;
     }
   else
@@ -3083,17 +3080,16 @@ u8 * format_snat_static_mapping (u8 * s, va_list * args)
    {
       if (vec_len (m->locals))
         {
-          s = format (s, "%U vrf %d external %U:%d %s %s",
+          s = format (s, "%U external %U:%d %s %s",
                       format_snat_protocol, m->proto,
-                      m->vrf_id,
                       format_ip4_address, &m->external_addr, m->external_port,
                       m->twice_nat == TWICE_NAT ? "twice-nat" :
                       m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
                       m->out2in_only ? "out2in-only" : "");
           vec_foreach (local, m->locals)
-            s = format (s, "\n  local %U:%d probability %d\%",
+            s = format (s, "\n  local %U:%d vrf %d probability %d\%",
                         format_ip4_address, &local->addr, local->port,
-                        local->probability);
+                        local->vrf_id, local->probability);
         }
       else
         s = format (s, "%U local %U:%d external %U:%d vrf %d %s %s",
diff --git a/src/plugins/nat/nat.h b/src/plugins/nat/nat.h
index 97bbec21360..0d51109175f 100644
--- a/src/plugins/nat/nat.h
+++ b/src/plugins/nat/nat.h
@@ -231,6 +231,8 @@ typedef struct {
   u16 port;
   u8 probability;
   u8 prefix;
+  u32 vrf_id;
+  u32 fib_index;
 } nat44_lb_addr_port_t;
 
 typedef enum {
@@ -610,7 +612,7 @@ int snat_add_interface_address(snat_main_t *sm, u32 sw_if_index, int is_del,
 uword unformat_snat_protocol(unformat_input_t * input, va_list * args);
 u8 * format_snat_protocol(u8 * s, va_list * args);
 int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
-                                     snat_protocol_t proto, u32 vrf_id,
+                                     snat_protocol_t proto,
                                      nat44_lb_addr_port_t *locals, u8 is_add,
                                      twice_nat_type_t twice_nat, u8 out2in_only,
                                      u8 *tag);
diff --git a/src/plugins/nat/nat44_cli.c b/src/plugins/nat/nat44_cli.c
index aa733a06a5a..e4fd0caa302 100644
--- a/src/plugins/nat/nat44_cli.c
+++ b/src/plugins/nat/nat44_cli.c
@@ -767,11 +767,20 @@ add_lb_static_mapping_command_fn (vlib_main_t * vm,
 	  local.probability = (u8) probability;
 	  vec_add1 (locals, local);
 	}
+      else if (unformat (line_input, "local %U:%u vrf %u probability %u",
+			 unformat_ip4_address, &l_addr, &l_port, &vrf_id,
+			 &probability))
+	{
+	  memset (&local, 0, sizeof (local));
+	  local.addr = l_addr;
+	  local.port = (u16) l_port;
+	  local.probability = (u8) probability;
+	  local.vrf_id = vrf_id;
+	  vec_add1 (locals, local);
+	}
       else if (unformat (line_input, "external %U:%u", unformat_ip4_address,
 			 &e_addr, &e_port))
 	;
-      else if (unformat (line_input, "vrf %u", &vrf_id))
-	;
       else if (unformat (line_input, "protocol %U", unformat_snat_protocol,
 			 &proto))
 	proto_set = 1;
@@ -803,9 +812,8 @@ add_lb_static_mapping_command_fn (vlib_main_t * vm,
       goto done;
     }
 
-  rv = nat44_add_del_lb_static_mapping (e_addr, (u16) e_port, proto, vrf_id,
-					locals, is_add, twice_nat,
-					out2in_only, 0);
+  rv = nat44_add_del_lb_static_mapping (e_addr, (u16) e_port, proto, locals,
+					is_add, twice_nat, out2in_only, 0);
 
   switch (rv)
     {
@@ -1716,8 +1724,8 @@ VLIB_CLI_COMMAND (add_lb_static_mapping_command, static) = {
   .function = add_lb_static_mapping_command_fn,
   .short_help =
     "nat44 add load-balancing static mapping protocol tcp|udp "
-    "external <addr>:<port> local <addr>:<port> probability <n> "
-    "[twice-nat|self-twice-nat] [vrf <table-id>] [out2in-only] [del]",
+    "external <addr>:<port> local <addr>:<port> [vrf <table-id>] "
+    "probability <n> [twice-nat|self-twice-nat] [out2in-only] [del]",
 };
 
 /*?
diff --git a/src/plugins/nat/nat_api.c b/src/plugins/nat/nat_api.c
index 6d4d0d912e6..5887efe5fea 100644
--- a/src/plugins/nat/nat_api.c
+++ b/src/plugins/nat/nat_api.c
@@ -1341,6 +1341,7 @@ unformat_nat44_lb_addr_port (vl_api_nat44_lb_addr_port_t * addr_port_pairs,
       clib_memcpy (&lb_addr_port.addr, ap->addr, 4);
       lb_addr_port.port = clib_net_to_host_u16 (ap->port);
       lb_addr_port.probability = ap->probability;
+      lb_addr_port.vrf_id = clib_net_to_host_u32 (ap->vrf_id);
       vec_add1 (lb_addr_port_pairs, lb_addr_port);
     }
 
@@ -1360,7 +1361,7 @@ static void
   snat_protocol_t proto;
   u8 *tag = 0;
 
-  if (sm->deterministic)
+  if (!sm->endpoint_dependent)
     {
       rv = VNET_API_ERROR_UNSUPPORTED;
       goto send_reply;
@@ -1380,8 +1381,7 @@ static void
   rv =
     nat44_add_del_lb_static_mapping (e_addr,
 				     clib_net_to_host_u16 (mp->external_port),
-				     proto, clib_net_to_host_u32 (mp->vrf_id),
-				     locals, mp->is_add, twice_nat,
+				     proto, locals, mp->is_add, twice_nat,
 				     mp->out2in_only, tag);
 
   vec_free (locals);
@@ -1423,7 +1423,6 @@ send_nat44_lb_static_mapping_details (snat_static_mapping_t * m,
   clib_memcpy (rmp->external_addr, &(m->external_addr), 4);
   rmp->external_port = ntohs (m->external_port);
   rmp->protocol = snat_proto_to_ip_proto (m->proto);
-  rmp->vrf_id = ntohl (m->vrf_id);
   rmp->context = context;
   if (m->twice_nat == TWICE_NAT)
     rmp->twice_nat = 1;
@@ -1439,6 +1438,7 @@ send_nat44_lb_static_mapping_details (snat_static_mapping_t * m,
     clib_memcpy (locals->addr, &(ap->addr), 4);
     locals->port = htons (ap->port);
     locals->probability = ap->probability;
+    locals->vrf_id = ntohl (ap->vrf_id);
     locals++;
     rmp->local_num++;
   }
@@ -1454,7 +1454,7 @@ static void
   snat_main_t *sm = &snat_main;
   snat_static_mapping_t *m;
 
-  if (sm->deterministic)
+  if (!sm->endpoint_dependent)
     return;
 
   reg = vl_api_client_index_to_registration (mp->client_index);
diff --git a/test/test_nat.py b/test/test_nat.py
index 363df719a75..4bae7015145 100644
--- a/test/test_nat.py
+++ b/test/test_nat.py
@@ -108,7 +108,6 @@ class MethodHolder(VppTestCase):
                 lb_sm.external_addr,
                 lb_sm.external_port,
                 lb_sm.protocol,
-                vrf_id=lb_sm.vrf_id,
                 twice_nat=lb_sm.twice_nat,
                 self_twice_nat=lb_sm.self_twice_nat,
                 out2in_only=lb_sm.out2in_only,
@@ -3433,10 +3432,12 @@ class TestNAT44EndpointDependent(MethodHolder):
 
         locals = [{'addr': server1.ip4n,
                    'port': local_port,
-                   'probability': 70},
+                   'probability': 70,
+                   'vrf_id': 0},
                   {'addr': server2.ip4n,
                    'port': local_port,
-                   'probability': 30}]
+                   'probability': 30,
+                   'vrf_id': 0}]
 
         self.nat44_add_address(self.nat_addr)
         self.vapi.nat44_add_del_lb_static_mapping(external_addr_n,
@@ -3515,10 +3516,12 @@ class TestNAT44EndpointDependent(MethodHolder):
 
         locals = [{'addr': server1.ip4n,
                    'port': local_port,
-                   'probability': 90},
+                   'probability': 90,
+                   'vrf_id': 0},
                   {'addr': server2.ip4n,
                    'port': local_port,
-                   'probability': 10}]
+                   'probability': 10,
+                   'vrf_id': 0}]
 
         self.nat44_add_address(self.nat_addr)
         self.vapi.nat44_add_del_lb_static_mapping(external_addr_n,
@@ -3560,10 +3563,12 @@ class TestNAT44EndpointDependent(MethodHolder):
 
         locals = [{'addr': server1.ip4n,
                    'port': local_port,
-                   'probability': 70},
+                   'probability': 70,
+                   'vrf_id': 0},
                   {'addr': server2.ip4n,
                    'port': local_port,
-                   'probability': 30}]
+                   'probability': 30,
+                   'vrf_id': 0}]
 
         self.vapi.nat44_forwarding_enable_disable(1)
         self.vapi.nat44_add_del_lb_static_mapping(external_addr_n,
@@ -3998,10 +4003,12 @@ class TestNAT44EndpointDependent(MethodHolder):
         else:
             locals = [{'addr': server1.ip4n,
                        'port': port_in1,
-                       'probability': 50},
+                       'probability': 50,
+                       'vrf_id': 0},
                       {'addr': server2.ip4n,
                        'port': port_in2,
-                       'probability': 50}]
+                       'probability': 50,
+                       'vrf_id': 0}]
             out_addr_n = socket.inet_pton(socket.AF_INET, self.nat_addr)
             self.vapi.nat44_add_del_lb_static_mapping(out_addr_n,
                                                       port_out,
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index 6ea1d2f3b3a..50a94d7431b 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -1585,7 +1585,6 @@ class VppPapiProvider(object):
             external_addr,
             external_port,
             protocol,
-            vrf_id=0,
             twice_nat=0,
             self_twice_nat=0,
             out2in_only=0,
@@ -1605,7 +1604,6 @@ class VppPapiProvider(object):
              'external_addr': external_addr,
              'external_port': external_port,
              'protocol': protocol,
-             'vrf_id': vrf_id,
              'twice_nat': twice_nat,
              'self_twice_nat': self_twice_nat,
              'out2in_only': out2in_only,