nat: pick outside addr based on local addr

Use outside addresses more evenly by using local address to pick from
pool of addresses. This ensures stability from POV of remote host -
an internal host always gets translated using the same outside address,
so it doesn't appear to be "hopping". Also, this avoids all hosts
being translated using the first address, which helps avoid needless
recaptchas and the like.

Exact assignment depends on internal ordering of addresses - local address
is used to pick an offset into internal vector. If that address cannot be
used, a linear search is performed as a fallback mechanism to find a possible
translation.

Type: improvement
Signed-off-by: Klement Sekera <ksekera@cisco.com>
Change-Id: I7ccb1da1dda5537f5d30d2f4cb48024f4b51c1a4

6 files changed, 215 insertions, 107 deletions

diff --git a/src/plugins/nat/nat44-ed/nat44_ed_in2out.c b/src/plugins/nat/nat44-ed/nat44_ed_in2out.c
index b99b3367329..59355e49ccd 100644
--- a/src/plugins/nat/nat44-ed/nat44_ed_in2out.c
+++ b/src/plugins/nat/nat44-ed/nat44_ed_in2out.c
@@ -165,91 +165,112 @@ snat_not_translate_fast (snat_main_t *sm, vlib_node_runtime_t *node,
 }
 
 static int
-nat_ed_alloc_addr_and_port (snat_main_t *sm, u32 rx_fib_index, u32 nat_proto,
-			    u32 thread_index, ip4_address_t r_addr, u16 r_port,
-			    u8 proto, u16 port_per_thread,
-			    u32 snat_thread_index, snat_session_t *s,
-			    ip4_address_t *outside_addr, u16 *outside_port)
+nat_ed_alloc_addr_and_port_with_snat_address (
+  snat_main_t *sm, u32 nat_proto, u32 thread_index, snat_address_t *a,
+  u16 port_per_thread, u32 snat_thread_index, snat_session_t *s,
+  ip4_address_t *outside_addr, u16 *outside_port)
 {
-  int i;
-  snat_address_t *a, *ga = 0;
-
   const u16 port_thread_offset = (port_per_thread * snat_thread_index) + 1024;
 
-  for (i = 0; i < vec_len (sm->addresses); i++)
+  s->o2i.match.daddr = a->addr;
+  /* first try port suggested by caller */
+  u16 port = clib_net_to_host_u16 (*outside_port);
+  u16 port_offset = port - port_thread_offset;
+  if (port <= port_thread_offset ||
+      port > port_thread_offset + port_per_thread)
+    {
+      /* need to pick a different port, suggested port doesn't fit in
+       * this thread's port range */
+      port_offset = snat_random_port (0, port_per_thread - 1);
+      port = port_thread_offset + port_offset;
+    }
+  u16 attempts = ED_PORT_ALLOC_ATTEMPTS;
+  do
     {
-      a = sm->addresses + i;
-      switch (nat_proto)
+      if (NAT_PROTOCOL_ICMP == nat_proto)
 	{
-#define _(N, j, n, unused)                                                    \
+	  s->o2i.match.sport = clib_host_to_net_u16 (port);
+	}
+      s->o2i.match.dport = clib_host_to_net_u16 (port);
+      if (0 == nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 2))
+	{
+#define _(N, i, n, s)                                                         \
   case NAT_PROTOCOL_##N:                                                      \
-    if (a->fib_index == rx_fib_index)                                         \
-      {                                                                       \
-	s->o2i.match.daddr = a->addr;                                         \
-	/* first try port suggested by caller */                              \
-	u16 port = clib_net_to_host_u16 (*outside_port);                      \
-	u16 port_offset = port - port_thread_offset;                          \
-	if (port <= port_thread_offset ||                                     \
-	    port > port_thread_offset + port_per_thread)                      \
-	  {                                                                   \
-	    /* need to pick a different port, suggested port doesn't fit in   \
-	     * this thread's port range */                                    \
-	    port_offset = snat_random_port (0, port_per_thread - 1);          \
-	    port = port_thread_offset + port_offset;                          \
-	  }                                                                   \
-	u16 attempts = ED_PORT_ALLOC_ATTEMPTS;                                \
-	do                                                                    \
-	  {                                                                   \
-	    if (NAT_PROTOCOL_ICMP == nat_proto)                               \
-	      {                                                               \
-		s->o2i.match.sport = clib_host_to_net_u16 (port);             \
-	      }                                                               \
-	    s->o2i.match.dport = clib_host_to_net_u16 (port);                 \
-	    if (0 ==                                                          \
-		nat_ed_ses_o2i_flow_hash_add_del (sm, thread_index, s, 2))    \
-	      {                                                               \
-		++a->busy_##n##_port_refcounts[port];                         \
-		a->busy_##n##_ports_per_thread[thread_index]++;               \
-		a->busy_##n##_ports++;                                        \
-		*outside_addr = a->addr;                                      \
-		*outside_port = clib_host_to_net_u16 (port);                  \
-		return 0;                                                     \
-	      }                                                               \
-	    port_offset = snat_random_port (0, port_per_thread - 1);          \
-	    port = port_thread_offset + port_offset;                          \
-	    --attempts;                                                       \
-	  }                                                                   \
-	while (attempts > 0);                                                 \
-      }                                                                       \
-    else if (a->fib_index == ~0)                                              \
-      {                                                                       \
-	ga = a;                                                               \
-      }                                                                       \
+    ++a->busy_##n##_port_refcounts[port];                                     \
+    a->busy_##n##_ports_per_thread[thread_index]++;                           \
+    a->busy_##n##_ports++;                                                    \
     break;
-
-	  foreach_nat_protocol;
-	default:
-	  nat_elog_info (sm, "unknown protocol");
-	  return 1;
+	  switch (nat_proto)
+	    {
+	      foreach_nat_protocol;
+	    default:
+	      nat_elog_info (sm, "unknown protocol");
+	      return 1;
+	    }
+#undef _
+	  *outside_addr = a->addr;
+	  *outside_port = clib_host_to_net_u16 (port);
+	  return 0;
 	}
+      port_offset = snat_random_port (0, port_per_thread - 1);
+      port = port_thread_offset + port_offset;
+      --attempts;
     }
+  while (attempts > 0);
+  return 1;
+}
 
-  if (ga)
+static int
+nat_ed_alloc_addr_and_port (snat_main_t *sm, u32 rx_fib_index, u32 nat_proto,
+			    u32 thread_index, ip4_address_t s_addr,
+			    u16 port_per_thread, u32 snat_thread_index,
+			    snat_session_t *s, ip4_address_t *outside_addr,
+			    u16 *outside_port)
+{
+  int i;
+  snat_address_t *a, *ga = 0;
+
+  if (vec_len (sm->addresses) > 0)
     {
-      /* fake fib_index to reuse macro */
-      rx_fib_index = ~0;
-      a = ga;
-      switch (nat_proto)
+      int s_addr_offset = s_addr.as_u32 % vec_len (sm->addresses);
+
+      for (i = s_addr_offset; i < vec_len (sm->addresses); ++i)
 	{
-	  foreach_nat_protocol;
-	default:
-	  nat_elog_info (sm, "unknown protocol");
-	  return 1;
+	  a = sm->addresses + i;
+	  if (a->fib_index == rx_fib_index)
+	    {
+	      return nat_ed_alloc_addr_and_port_with_snat_address (
+		sm, nat_proto, thread_index, a, port_per_thread,
+		snat_thread_index, s, outside_addr, outside_port);
+	    }
+	  else if (a->fib_index == ~0)
+	    {
+	      ga = a;
+	    }
 	}
-    }
 
-#undef _
+      for (i = 0; i < s_addr_offset; ++i)
+	{
+	  a = sm->addresses + i;
+	  if (a->fib_index == rx_fib_index)
+	    {
+	      return nat_ed_alloc_addr_and_port_with_snat_address (
+		sm, nat_proto, thread_index, a, port_per_thread,
+		snat_thread_index, s, outside_addr, outside_port);
+	    }
+	  else if (a->fib_index == ~0)
+	    {
+	      ga = a;
+	    }
+	}
 
+      if (ga)
+	{
+	  return nat_ed_alloc_addr_and_port_with_snat_address (
+	    sm, nat_proto, thread_index, a, port_per_thread, snat_thread_index,
+	    s, outside_addr, outside_port);
+	}
+    }
   /* Totally out of translations to use... */
   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
   return 1;
@@ -423,7 +444,7 @@ slow_path_ed (snat_main_t *sm, vlib_buffer_t *b, ip4_address_t l_addr,
       nat_6t_flow_txfib_rewrite_set (&s->o2i, rx_fib_index);
 
       if (nat_ed_alloc_addr_and_port (
-	    sm, rx_fib_index, nat_proto, thread_index, daddr, dport, proto,
+	    sm, rx_fib_index, nat_proto, thread_index, l_addr,
 	    sm->port_per_thread, tsm->snat_thread_index, s, &outside_addr,
 	    &outside_port))
 	{
diff --git a/src/plugins/nat/nat44-ei/nat44_ei.c b/src/plugins/nat/nat44-ei/nat44_ei.c
index fdf90708a09..d361060a42e 100644
--- a/src/plugins/nat/nat44-ei/nat44_ei.c
+++ b/src/plugins/nat/nat44-ei/nat44_ei.c
@@ -1468,7 +1468,8 @@ nat44_ei_get_out2in_worker_index (vlib_buffer_t *b, ip4_header_t *ip0,
 static int
 nat44_ei_alloc_default_cb (nat44_ei_address_t *addresses, u32 fib_index,
 			   u32 thread_index, nat_protocol_t proto,
-			   ip4_address_t *addr, u16 *port, u16 port_per_thread,
+			   ip4_address_t s_addr, ip4_address_t *addr,
+			   u16 *port, u16 port_per_thread,
 			   u32 snat_thread_index)
 {
   nat44_ei_main_t *nm = &nat44_ei_main;
@@ -1476,11 +1477,16 @@ nat44_ei_alloc_default_cb (nat44_ei_address_t *addresses, u32 fib_index,
   u32 portnum;
   int i;
 
-  for (i = 0; i < vec_len (addresses); i++)
+  if (vec_len (addresses) > 0)
     {
-      a = addresses + i;
-      switch (proto)
+
+      int s_addr_offset = s_addr.as_u32 % vec_len (addresses);
+
+      for (i = s_addr_offset; i < vec_len (addresses); ++i)
 	{
+	  a = addresses + i;
+	  switch (proto)
+	    {
 #define _(N, j, n, s)                                                         \
   case NAT_PROTOCOL_##N:                                                      \
     if (a->busy_##n##_ports_per_thread[thread_index] < port_per_thread)       \
@@ -1509,41 +1515,39 @@ nat44_ei_alloc_default_cb (nat44_ei_address_t *addresses, u32 fib_index,
 	  }                                                                   \
       }                                                                       \
     break;
-	  foreach_nat_protocol
-#undef _
-	    default : nat_elog_info (nm, "unknown protocol");
-	  return 1;
+	      foreach_nat_protocol;
+	    default:
+	      nat_elog_info (nm, "unknown protocol");
+	      return 1;
+	    }
 	}
-    }
 
+      for (i = 0; i < s_addr_offset; ++i)
+	{
+	  a = addresses + i;
+	  switch (proto)
+	    {
+	      foreach_nat_protocol;
+	    default:
+	      nat_elog_info (nm, "unknown protocol");
+	      return 1;
+	    }
+	}
   if (ga)
     {
       a = ga;
+      // fake fib index to reuse macro
+      fib_index = ~0;
       switch (proto)
 	{
-#define _(N, j, n, s)                                                         \
-  case NAT_PROTOCOL_##N:                                                      \
-    while (1)                                                                 \
-      {                                                                       \
-	portnum =                                                             \
-	  (port_per_thread * snat_thread_index) +                             \
-	  nat_random_port (&nm->random_seed, 0, port_per_thread - 1) + 1024;  \
-	if (a->busy_##n##_port_refcounts[portnum])                            \
-	  continue;                                                           \
-	++a->busy_##n##_port_refcounts[portnum];                              \
-	a->busy_##n##_ports_per_thread[thread_index]++;                       \
-	a->busy_##n##_ports++;                                                \
-	*addr = a->addr;                                                      \
-	*port = clib_host_to_net_u16 (portnum);                               \
-	return 0;                                                             \
-      }
-	  break;
-	  foreach_nat_protocol
-#undef _
+	  foreach_nat_protocol;
 	    default : nat_elog_info (nm, "unknown protocol");
 	  return 1;
 	}
     }
+    }
+
+#undef _
 
   /* Totally out of translations to use... */
   nat_ipfix_logging_addresses_exhausted (thread_index, 0);
@@ -1553,8 +1557,8 @@ nat44_ei_alloc_default_cb (nat44_ei_address_t *addresses, u32 fib_index,
 static int
 nat44_ei_alloc_range_cb (nat44_ei_address_t *addresses, u32 fib_index,
 			 u32 thread_index, nat_protocol_t proto,
-			 ip4_address_t *addr, u16 *port, u16 port_per_thread,
-			 u32 snat_thread_index)
+			 ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
+			 u16 port_per_thread, u32 snat_thread_index)
 {
   nat44_ei_main_t *nm = &nat44_ei_main;
   nat44_ei_address_t *a = addresses;
@@ -1600,8 +1604,8 @@ exhausted:
 static int
 nat44_ei_alloc_mape_cb (nat44_ei_address_t *addresses, u32 fib_index,
 			u32 thread_index, nat_protocol_t proto,
-			ip4_address_t *addr, u16 *port, u16 port_per_thread,
-			u32 snat_thread_index)
+			ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
+			u16 port_per_thread, u32 snat_thread_index)
 {
   nat44_ei_main_t *nm = &nat44_ei_main;
   nat44_ei_address_t *a = addresses;
diff --git a/src/plugins/nat/nat44-ei/nat44_ei.h b/src/plugins/nat/nat44-ei/nat44_ei.h
index b9212df44bd..7bc1b304fb3 100644
--- a/src/plugins/nat/nat44-ei/nat44_ei.h
+++ b/src/plugins/nat/nat44-ei/nat44_ei.h
@@ -81,8 +81,8 @@ clib_error_t *nat44_ei_api_hookup (vlib_main_t *vm);
 /* NAT address and port allocation function */
 typedef int (nat44_ei_alloc_out_addr_and_port_function_t) (
   nat44_ei_address_t *addresses, u32 fib_index, u32 thread_index,
-  nat_protocol_t proto, ip4_address_t *addr, u16 *port, u16 port_per_thread,
-  u32 snat_thread_index);
+  nat_protocol_t proto, ip4_address_t s_addr, ip4_address_t *addr, u16 *port,
+  u16 port_per_thread, u32 snat_thread_index);
 
 typedef struct
 {
diff --git a/src/plugins/nat/nat44-ei/nat44_ei_in2out.c b/src/plugins/nat/nat44-ei/nat44_ei_in2out.c
index 80beb7a49eb..d1a962f5222 100644
--- a/src/plugins/nat/nat44-ei/nat44_ei_in2out.c
+++ b/src/plugins/nat/nat44-ei/nat44_ei_in2out.c
@@ -336,8 +336,8 @@ slow_path (nat44_ei_main_t *nm, vlib_buffer_t *b0, ip4_header_t *ip0,
     {
       /* Try to create dynamic translation */
       if (nm->alloc_addr_and_port (
-	    nm->addresses, rx_fib_index0, thread_index, nat_proto, &sm_addr,
-	    &sm_port, nm->port_per_thread,
+	    nm->addresses, rx_fib_index0, thread_index, nat_proto,
+	    ip0->src_address, &sm_addr, &sm_port, nm->port_per_thread,
 	    nm->per_thread_data[thread_index].snat_thread_index))
 	{
 	  b0->error = node->errors[NAT44_EI_IN2OUT_ERROR_OUT_OF_PORTS];
diff --git a/src/plugins/nat/test/test_nat44_ed.py b/src/plugins/nat/test/test_nat44_ed.py
index 7e85c938ee8..8c3169e338f 100644
--- a/src/plugins/nat/test/test_nat44_ed.py
+++ b/src/plugins/nat/test/test_nat44_ed.py
@@ -1903,6 +1903,45 @@ class TestNAT44ED(NAT44EDTestCase):
             self.logger.error(ppp("Unexpected or invalid packet:", p))
             raise
 
+    def test_outside_address_distribution(self):
+        """ Outside address distribution based on source address """
+
+        x = 100
+        nat_addresses = []
+
+        for i in range(1, x):
+            a = "10.0.0.%d" % i
+            nat_addresses.append(a)
+
+        self.nat_add_inside_interface(self.pg0)
+        self.nat_add_outside_interface(self.pg1)
+
+        self.vapi.nat44_add_del_address_range(
+            first_ip_address=nat_addresses[0],
+            last_ip_address=nat_addresses[-1],
+            vrf_id=0xFFFFFFFF, is_add=1, flags=0)
+
+        self.pg0.generate_remote_hosts(x)
+
+        pkts = []
+        for i in range(x):
+            p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+                 IP(src=self.pg0.remote_hosts[i].ip4,
+                     dst=self.pg1.remote_ip4) /
+                 UDP(sport=7000+i, dport=80+i))
+            pkts.append(p)
+
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        recvd = self.pg1.get_capture(len(pkts))
+        for (p_sent, p_recvd) in zip(pkts, recvd):
+            packed = socket.inet_aton(p_sent[IP].src)
+            numeric = struct.unpack("!L", packed)[0]
+            numeric = socket.htonl(numeric)
+            a = nat_addresses[(numeric-1) % len(nat_addresses)]
+            self.assertEqual(a, p_recvd[IP].src, "Packet not translated")
+
 
 class TestNAT44EDMW(TestNAT44ED):
     """ NAT44ED MW Test Case """
diff --git a/src/plugins/nat/test/test_nat44_ei.py b/src/plugins/nat/test/test_nat44_ei.py
index da6ca4293c0..dcd7f46f073 100644
--- a/src/plugins/nat/test/test_nat44_ei.py
+++ b/src/plugins/nat/test/test_nat44_ei.py
@@ -3719,6 +3719,50 @@ class TestNAT44EI(MethodHolder):
         self.logger.info(
             self.vapi.cli("show nat44 ei addr-port-assignment-alg"))
 
+    def test_outside_address_distribution(self):
+        """ Outside address distribution based on source address """
+
+        x = 100
+        nat_addresses = []
+
+        for i in range(1, x):
+            a = "10.0.0.%d" % i
+            nat_addresses.append(a)
+
+        flags = self.config_flags.NAT44_EI_IF_INSIDE
+        self.vapi.nat44_ei_interface_add_del_feature(
+            sw_if_index=self.pg0.sw_if_index,
+            flags=flags, is_add=1)
+        self.vapi.nat44_ei_interface_add_del_feature(
+            sw_if_index=self.pg1.sw_if_index,
+            is_add=1)
+
+        self.vapi.nat44_ei_add_del_address_range(
+            first_ip_address=nat_addresses[0],
+            last_ip_address=nat_addresses[-1],
+            vrf_id=0xFFFFFFFF, is_add=1)
+
+        self.pg0.generate_remote_hosts(x)
+
+        pkts = []
+        for i in range(x):
+            p = (Ether(dst=self.pg0.local_mac, src=self.pg0.remote_mac) /
+                 IP(src=self.pg0.remote_hosts[i].ip4,
+                     dst=self.pg1.remote_ip4) /
+                 UDP(sport=7000+i, dport=80+i))
+            pkts.append(p)
+
+        self.pg0.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        recvd = self.pg1.get_capture(len(pkts))
+        for (p_sent, p_recvd) in zip(pkts, recvd):
+            packed = socket.inet_aton(p_sent[IP].src)
+            numeric = struct.unpack("!L", packed)[0]
+            numeric = socket.htonl(numeric)
+            a = nat_addresses[(numeric-1) % len(nat_addresses)]
+            self.assertEqual(a, p_recvd[IP].src, "Packet not translated")
+
 
 class TestNAT44Out2InDPO(MethodHolder):
     """ NAT44EI Test Cases using out2in DPO """