NAT44: nat44_add_del_lb_static_mapping enhancements (VPP-1514)

Change-Id: I5419e06592b0402e911e132796368800321f355a
Signed-off-by: Matus Fabian <matfabia@cisco.com>

8 files changed, 540 insertions, 54 deletions

diff --git a/src/plugins/nat/nat.api b/src/plugins/nat/nat.api
index 6e2d8e57853..f41428b70bb 100644
--- a/src/plugins/nat/nat.api
+++ b/src/plugins/nat/nat.api
@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 
-option version = "4.0.0";
+option version = "4.1.0";
 
 /**
  * @file nat.api
@@ -775,10 +775,29 @@ autoreply manual_endian define nat44_add_del_lb_static_mapping {
   u8 out2in_only;
   u8 tag[64];
   u32 affinity;
-  u8 local_num;
+  u32 local_num;
   vl_api_nat44_lb_addr_port_t locals[local_num];
 };
 
+/** \brief Add/delete NAT44 load-balancing static mapping rule backend
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param is_add - 1 if add, 0 if delete
+    @param external_addr - external IPv4 address of the service
+    @param external_port - external L4 port number of the service
+    @param protocol - IP protocol number of the service
+    @param local - local network node
+*/
+autoreply define nat44_lb_static_mapping_add_del_local {
+  u32 client_index;
+  u32 context;
+  u8 is_add;
+  u8 external_addr[4];
+  u16 external_port;
+  u8 protocol;
+  vl_api_nat44_lb_addr_port_t local;
+};
+
 /** \brief Dump NAT44 load-balancing static mapping rules
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
@@ -815,7 +834,7 @@ manual_endian define nat44_lb_static_mapping_details {
   u8 out2in_only;
   u8 tag[64];
   u32 affinity;
-  u8 local_num;
+  u32 local_num;
   vl_api_nat44_lb_addr_port_t locals[local_num];
 };
 
diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
index 6bfea3c4162..2e1aa770581 100755
--- a/src/plugins/nat/nat.c
+++ b/src/plugins/nat/nat.c
@@ -665,7 +665,7 @@ snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
   snat_session_t *s;
   snat_static_map_resolve_t *rp, *rp_match = 0;
   nat44_lb_addr_port_t *local;
-  u8 find = 0;
+  u32 find = ~0;
 
   if (!sm->endpoint_dependent)
     {
@@ -759,13 +759,13 @@ snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 	  if (is_identity_static_mapping (m))
 	    {
               /* *INDENT-OFF* */
-              vec_foreach (local, m->locals)
-                {
-                  if (local->vrf_id == vrf_id)
-                    return VNET_API_ERROR_VALUE_EXIST;
-                }
+              pool_foreach (local, m->locals,
+              ({
+                if (local->vrf_id == vrf_id)
+                  return VNET_API_ERROR_VALUE_EXIST;
+              }));
               /* *INDENT-ON* */
-	      vec_add2 (m->locals, local, 1);
+	      pool_get (m->locals, local);
 	      local->vrf_id = vrf_id;
 	      local->fib_index =
 		fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
@@ -879,7 +879,7 @@ snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
       if (identity_nat)
 	{
 	  m->flags |= NAT_STATIC_MAPPING_FLAG_IDENTITY_NAT;
-	  vec_add2 (m->locals, local, 1);
+	  pool_get (m->locals, local);
 	  local->vrf_id = vrf_id;
 	  local->fib_index = fib_index;
 	}
@@ -979,19 +979,19 @@ snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 	  if (vrf_id == ~0)
 	    vrf_id = sm->inside_vrf_id;
 
-	  for (i = 0; i < vec_len (m->locals); i++)
-	    {
-	      if (m->locals[i].vrf_id == vrf_id)
-		{
-		  find = 1;
-		  break;
-		}
-	    }
-	  if (!find)
+          /* *INDENT-OFF* */
+          pool_foreach (local, m->locals,
+          ({
+	    if (local->vrf_id == vrf_id)
+              find = local - m->locals;
+	  }));
+          /* *INDENT-ON* */
+	  if (find == ~0)
 	    return VNET_API_ERROR_NO_SUCH_ENTRY;
 
-	  fib_index = m->locals[i].fib_index;
-	  vec_del1 (m->locals, i);
+	  local = pool_elt_at_index (m->locals, find);
+	  fib_index = local->fib_index;
+	  pool_put (m->locals, local);
 	}
       else
 	fib_index = m->fib_index;
@@ -1089,7 +1089,7 @@ snat_add_static_mapping (ip4_address_t l_addr, ip4_address_t e_addr,
 	}
 
       fib_table_unlock (fib_index, FIB_PROTOCOL_IP4, FIB_SOURCE_PLUGIN_LOW);
-      if (vec_len (m->locals))
+      if (pool_elts (m->locals))
 	return 0;
 
       m_key.addr = m->external_addr;
@@ -1258,7 +1258,8 @@ nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
 	    }
 	  locals[i].prefix = (i == 0) ? locals[i].probability :
 	    (locals[i - 1].prefix + locals[i].probability);
-	  vec_add1 (m->locals, locals[i]);
+	  pool_get (m->locals, local);
+	  *local = locals[i];
 	  if (sm->num_workers > 1)
 	    {
 	      ip4_header_t ip = {
@@ -1331,8 +1332,8 @@ nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
 	}
 
       /* *INDENT-OFF* */
-      vec_foreach (local, m->locals)
-        {
+      pool_foreach (local, m->locals,
+      ({
           fib_table_unlock (local->fib_index, FIB_PROTOCOL_IP4,
                             FIB_SOURCE_PLUGIN_LOW);
           m_key.addr = local->addr;
@@ -1361,7 +1362,7 @@ nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
 
           /* Delete sessions */
           u_key.addr = local->addr;
-          u_key.fib_index = m->fib_index;
+          u_key.fib_index = local->fib_index;
           kv.key = u_key.as_u64;
           if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
             {
@@ -1391,11 +1392,11 @@ nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
                     }
                 }
             }
-        }
+      }));
       /* *INDENT-ON* */
       if (m->affinity)
 	nat_affinity_flush_service (m->affinity_per_service_list_head_index);
-      vec_free (m->locals);
+      pool_free (m->locals);
       vec_free (m->tag);
       vec_free (m->workers);
 
@@ -1406,6 +1407,187 @@ nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
 }
 
 int
+nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
+				       ip4_address_t l_addr, u16 l_port,
+				       snat_protocol_t proto, u32 vrf_id,
+				       u8 probability, u8 is_add)
+{
+  snat_main_t *sm = &snat_main;
+  snat_static_mapping_t *m = 0;
+  snat_session_key_t m_key;
+  clib_bihash_kv_8_8_t kv, value;
+  nat44_lb_addr_port_t *local, *prev_local, *match_local = 0;
+  snat_main_per_thread_data_t *tsm;
+  snat_user_key_t u_key;
+  snat_user_t *u;
+  snat_session_t *s;
+  dlist_elt_t *head, *elt;
+  u32 elt_index, head_index, ses_index, *locals = 0;
+  uword *bitmap = 0;
+  int i;
+
+  if (!sm->endpoint_dependent)
+    return VNET_API_ERROR_FEATURE_DISABLED;
+
+  m_key.addr = e_addr;
+  m_key.port = e_port;
+  m_key.protocol = proto;
+  m_key.fib_index = 0;
+  kv.key = m_key.as_u64;
+  if (!clib_bihash_search_8_8 (&sm->static_mapping_by_external, &kv, &value))
+    m = pool_elt_at_index (sm->static_mappings, value.value);
+
+  if (!m)
+    return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+  if (!is_lb_static_mapping (m))
+    return VNET_API_ERROR_INVALID_VALUE;
+
+  /* *INDENT-OFF* */
+  pool_foreach (local, m->locals,
+  ({
+    if ((local->addr.as_u32 == l_addr.as_u32) && (local->port == l_port) &&
+        (local->vrf_id == vrf_id))
+      {
+        match_local = local;
+        break;
+      }
+  }));
+  /* *INDENT-ON* */
+
+  if (is_add)
+    {
+      if (match_local)
+	return VNET_API_ERROR_VALUE_EXIST;
+
+      pool_get (m->locals, local);
+      clib_memset (local, 0, sizeof (*local));
+      local->addr.as_u32 = l_addr.as_u32;
+      local->port = l_port;
+      local->probability = probability;
+      local->vrf_id = vrf_id;
+      local->fib_index =
+	fib_table_find_or_create_and_lock (FIB_PROTOCOL_IP4, vrf_id,
+					   FIB_SOURCE_PLUGIN_LOW);
+
+      if (!is_out2in_only_static_mapping (m))
+	{
+	  m_key.addr = l_addr;
+	  m_key.port = l_port;
+	  m_key.fib_index = local->fib_index;
+	  kv.key = m_key.as_u64;
+	  kv.value = m - sm->static_mappings;
+	  if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 1))
+	    nat_log_err ("static_mapping_by_local key add failed");
+	}
+    }
+  else
+    {
+      if (!match_local)
+	return VNET_API_ERROR_NO_SUCH_ENTRY;
+
+      if (pool_elts (m->locals) < 3)
+	return VNET_API_ERROR_UNSPECIFIED;
+
+      fib_table_unlock (match_local->fib_index, FIB_PROTOCOL_IP4,
+			FIB_SOURCE_PLUGIN_LOW);
+
+      if (!is_out2in_only_static_mapping (m))
+	{
+	  m_key.addr = l_addr;
+	  m_key.port = l_port;
+	  m_key.fib_index = match_local->fib_index;
+	  kv.key = m_key.as_u64;
+	  if (clib_bihash_add_del_8_8 (&sm->static_mapping_by_local, &kv, 0))
+	    nat_log_err ("static_mapping_by_local key del failed");
+	}
+
+      if (sm->num_workers > 1)
+	{
+	  ip4_header_t ip = {
+	    .src_address = local->addr,
+	  };
+	  tsm = vec_elt_at_index (sm->per_thread_data,
+				  sm->worker_in2out_cb (&ip, m->fib_index));
+	}
+      else
+	tsm = vec_elt_at_index (sm->per_thread_data, sm->num_workers);
+
+      /* Delete sessions */
+      u_key.addr = match_local->addr;
+      u_key.fib_index = match_local->fib_index;
+      kv.key = u_key.as_u64;
+      if (!clib_bihash_search_8_8 (&tsm->user_hash, &kv, &value))
+	{
+	  u = pool_elt_at_index (tsm->users, value.value);
+	  if (u->nstaticsessions)
+	    {
+	      head_index = u->sessions_per_user_list_head_index;
+	      head = pool_elt_at_index (tsm->list_pool, head_index);
+	      elt_index = head->next;
+	      elt = pool_elt_at_index (tsm->list_pool, elt_index);
+	      ses_index = elt->value;
+	      while (ses_index != ~0)
+		{
+		  s = pool_elt_at_index (tsm->sessions, ses_index);
+		  elt = pool_elt_at_index (tsm->list_pool, elt->next);
+		  ses_index = elt->value;
+
+		  if (!(is_lb_session (s)))
+		    continue;
+
+		  if ((s->in2out.addr.as_u32 != match_local->addr.as_u32) ||
+		      (clib_net_to_host_u16 (s->in2out.port) !=
+		       match_local->port))
+		    continue;
+
+		  nat_free_session_data (sm, s, tsm - sm->per_thread_data);
+		  nat44_delete_session (sm, s, tsm - sm->per_thread_data);
+		}
+	    }
+	}
+
+      pool_put (m->locals, match_local);
+    }
+
+  vec_free (m->workers);
+
+  /* *INDENT-OFF* */
+  pool_foreach (local, m->locals,
+  ({
+    vec_add1 (locals, local - m->locals);
+    if (sm->num_workers > 1)
+      {
+        ip4_header_t ip;
+        ip.src_address.as_u32 = local->addr.as_u32,
+        bitmap = clib_bitmap_set (bitmap,
+                                  sm->worker_in2out_cb (&ip, local->fib_index),
+                                  1);
+      }
+  }));
+  /* *INDENT-ON* */
+
+  local = pool_elt_at_index (m->locals, locals[0]);
+  local->prefix = local->probability;
+  for (i = 1; i < vec_len (locals); i++)
+    {
+      local = pool_elt_at_index (m->locals, locals[i]);
+      prev_local = pool_elt_at_index (m->locals, locals[i - 1]);
+      local->prefix = local->probability + prev_local->prefix;
+    }
+
+  /* Assign workers */
+  if (sm->num_workers > 1)
+    {
+      /* *INDENT-OFF* */
+      clib_bitmap_foreach (i, bitmap, ({ vec_add1(m->workers, i); }));
+      /* *INDENT-ON* */
+    }
+
+  return 0;
+}
+
+int
 snat_del_address (snat_main_t * sm, ip4_address_t addr, u8 delete_sm,
 		  u8 twice_nat)
 {
@@ -2095,8 +2277,9 @@ snat_static_mapping_match (snat_main_t * sm,
   snat_static_mapping_t *m;
   snat_session_key_t m_key;
   clib_bihash_8_8_t *mapping_hash = &sm->static_mapping_by_local;
-  u32 rand, lo = 0, hi, mid;
+  u32 rand, lo = 0, hi, mid, *tmp = 0, i;
   u8 backend_index;
+  nat44_lb_addr_port_t *local;
 
   m_key.fib_index = match.fib_index;
   if (by_external)
@@ -2136,42 +2319,52 @@ snat_static_mapping_match (snat_main_t * sm,
 					      &backend_index))
 		goto get_local;
 
-	      mapping->addr = m->locals[backend_index].addr;
-	      mapping->port =
-		clib_host_to_net_u16 (m->locals[backend_index].port);
-	      mapping->fib_index = m->locals[backend_index].fib_index;
+	      local = pool_elt_at_index (m->locals, backend_index);
+	      mapping->addr = local->addr;
+	      mapping->port = clib_host_to_net_u16 (local->port);
+	      mapping->fib_index = local->fib_index;
 	      goto end;
 	    }
 	get_local:
-	  hi = vec_len (m->locals) - 1;
-	  rand = 1 + (random_u32 (&sm->random_seed) % m->locals[hi].prefix);
+          /* *INDENT-OFF* */
+          pool_foreach_index (i, m->locals,
+          ({
+            vec_add1 (tmp, i);
+          }));
+          /* *INDENT-ON* */
+	  hi = vec_len (tmp) - 1;
+	  local = pool_elt_at_index (m->locals, tmp[hi]);
+	  rand = 1 + (random_u32 (&sm->random_seed) % local->prefix);
 	  while (lo < hi)
 	    {
 	      mid = ((hi - lo) >> 1) + lo;
-	      (rand > m->locals[mid].prefix) ? (lo = mid + 1) : (hi = mid);
+	      local = pool_elt_at_index (m->locals, tmp[mid]);
+	      (rand > local->prefix) ? (lo = mid + 1) : (hi = mid);
 	    }
-	  if (!(m->locals[lo].prefix >= rand))
+	  local = pool_elt_at_index (m->locals, tmp[lo]);
+	  if (!(local->prefix >= rand))
 	    return 1;
 	  if (PREDICT_FALSE (sm->num_workers > 1))
 	    {
 	      ip4_header_t ip = {
-		.src_address = m->locals[lo].addr,
+		.src_address = local->addr,
 	      };
 	      if (sm->worker_in2out_cb (&ip, m->fib_index) !=
 		  vlib_get_thread_index ())
 		goto get_local;
 	    }
-	  mapping->addr = m->locals[lo].addr;
-	  mapping->port = clib_host_to_net_u16 (m->locals[lo].port);
-	  mapping->fib_index = m->locals[lo].fib_index;
+	  mapping->addr = local->addr;
+	  mapping->port = clib_host_to_net_u16 (local->port);
+	  mapping->fib_index = local->fib_index;
 	  if (m->affinity)
 	    {
 	      if (nat_affinity_create_and_lock (ext_host_addr[0], match.addr,
 						match.protocol, match.port,
-						lo, m->affinity,
+						tmp[lo], m->affinity,
 						m->affinity_per_service_list_head_index))
 		nat_log_info ("create affinity record failed");
 	    }
+	  vec_free (tmp);
 	}
       else
 	{
diff --git a/src/plugins/nat/nat.h b/src/plugins/nat/nat.h
index 3ce83ea2602..e851e2f98f7 100644
--- a/src/plugins/nat/nat.h
+++ b/src/plugins/nat/nat.h
@@ -859,6 +859,11 @@ int nat44_add_del_lb_static_mapping (ip4_address_t e_addr, u16 e_port,
 				     twice_nat_type_t twice_nat,
 				     u8 out2in_only, u8 * tag, u32 affinity);
 
+int nat44_lb_static_mapping_add_del_local (ip4_address_t e_addr, u16 e_port,
+					   ip4_address_t l_addr, u16 l_port,
+					   snat_protocol_t proto, u32 vrf_id,
+					   u8 probability, u8 is_add);
+
 clib_error_t *snat_api_init (vlib_main_t * vm, snat_main_t * sm);
 
 /**
diff --git a/src/plugins/nat/nat44_cli.c b/src/plugins/nat/nat44_cli.c
index 0c5b80c394e..9aaa8498e22 100644
--- a/src/plugins/nat/nat44_cli.c
+++ b/src/plugins/nat/nat44_cli.c
@@ -948,6 +948,98 @@ done:
 }
 
 static clib_error_t *
+add_lb_backend_command_fn (vlib_main_t * vm,
+			   unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  snat_main_t *sm = &snat_main;
+  clib_error_t *error = 0;
+  ip4_address_t l_addr, e_addr;
+  u32 l_port = 0, e_port = 0, vrf_id = 0, probability = 0;
+  int is_add = 1;
+  int rv;
+  snat_protocol_t proto;
+  u8 proto_set = 0;
+
+  if (sm->deterministic)
+    return clib_error_return (0, UNSUPPORTED_IN_DET_MODE_STR);
+
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "local %U:%u probability %u",
+		    unformat_ip4_address, &l_addr, &l_port, &probability))
+	;
+      else if (unformat (line_input, "local %U:%u vrf %u probability %u",
+			 unformat_ip4_address, &l_addr, &l_port, &vrf_id,
+			 &probability))
+	;
+      else if (unformat (line_input, "external %U:%u", unformat_ip4_address,
+			 &e_addr, &e_port))
+	;
+      else if (unformat (line_input, "protocol %U", unformat_snat_protocol,
+			 &proto))
+	proto_set = 1;
+      else if (unformat (line_input, "del"))
+	is_add = 0;
+      else
+	{
+	  error = clib_error_return (0, "unknown input: '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+
+  if (!l_port || !e_port)
+    {
+      error = clib_error_return (0, "local or external must be set");
+      goto done;
+    }
+
+  if (!proto_set)
+    {
+      error = clib_error_return (0, "missing protocol");
+      goto done;
+    }
+
+  rv =
+    nat44_lb_static_mapping_add_del_local (e_addr, (u16) e_port, l_addr,
+					   l_port, proto, vrf_id, probability,
+					   is_add);
+
+  switch (rv)
+    {
+    case VNET_API_ERROR_INVALID_VALUE:
+      error = clib_error_return (0, "External is not load-balancing static "
+				 "mapping.");
+      goto done;
+    case VNET_API_ERROR_NO_SUCH_ENTRY:
+      error = clib_error_return (0, "Mapping or back-end not exist.");
+      goto done;
+    case VNET_API_ERROR_VALUE_EXIST:
+      error = clib_error_return (0, "Back-end already exist.");
+      goto done;
+    case VNET_API_ERROR_FEATURE_DISABLED:
+      error =
+	clib_error_return (0, "Available only for endpoint-dependent mode.");
+      goto done;
+    case VNET_API_ERROR_UNSPECIFIED:
+      error = clib_error_return (0, "At least two back-ends must remain");
+      goto done;
+    default:
+      break;
+    }
+
+done:
+  unformat_free (line_input);
+
+  return error;
+}
+
+static clib_error_t *
 nat44_show_static_mappings_command_fn (vlib_main_t * vm,
 				       unformat_input_t * input,
 				       vlib_cli_command_t * cmd)
@@ -1936,6 +2028,24 @@ VLIB_CLI_COMMAND (add_lb_static_mapping_command, static) = {
 
 /*?
  * @cliexpar
+ * @cliexstart{nat44 add load-balancing static mapping}
+ * Modify service load balancing using NAT44
+ * To add new back-end server 10.100.10.30:8080 for service load balancing
+ * static mapping with external IP address 1.2.3.4 and TCP port 80 use:
+ *  vpp# nat44 add load-balancing back-end protocol tcp external 1.2.3.4:80 local 10.100.10.30:8080 probability 25
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (add_lb_backend_command, static) = {
+  .path = "nat44 add load-balancing back-end",
+  .function = add_lb_backend_command_fn,
+  .short_help =
+    "nat44 add load-balancing back-end protocol tcp|udp "
+    "external <addr>:<port> local <addr>:<port> [vrf <table-id>] "
+    "probability <n> [del]",
+};
+
+/*?
+ * @cliexpar
  * @cliexstart{show nat44 static mappings}
  * Show NAT44 static mappings.
  * vpp# show nat44 static mappings
diff --git a/src/plugins/nat/nat_api.c b/src/plugins/nat/nat_api.c
index 378e7abe871..b1aa3243ab9 100644
--- a/src/plugins/nat/nat_api.c
+++ b/src/plugins/nat/nat_api.c
@@ -1196,6 +1196,7 @@ send_nat44_identity_mapping_details (snat_static_mapping_t * m, int index,
 {
   vl_api_nat44_identity_mapping_details_t *rmp;
   snat_main_t *sm = &snat_main;
+  nat44_lb_addr_port_t *local = pool_elt_at_index (m->locals, index);
 
   rmp = vl_msg_api_alloc (sizeof (*rmp));
   clib_memset (rmp, 0, sizeof (*rmp));
@@ -1205,7 +1206,7 @@ send_nat44_identity_mapping_details (snat_static_mapping_t * m, int index,
   clib_memcpy (rmp->ip_address, &(m->local_addr), 4);
   rmp->port = htons (m->local_port);
   rmp->sw_if_index = ~0;
-  rmp->vrf_id = htonl (m->locals[index].vrf_id);
+  rmp->vrf_id = htonl (local->vrf_id);
   rmp->protocol = snat_proto_to_ip_proto (m->proto);
   rmp->context = context;
   if (m->tag)
@@ -1260,8 +1261,10 @@ static void
   ({
       if (is_identity_static_mapping(m) && !is_lb_static_mapping (m))
         {
-          for (j = 0; j < vec_len (m->locals); j++)
+          pool_foreach_index (j, m->locals,
+          ({
             send_nat44_identity_mapping_details (m, j, reg, mp->context);
+          }));
         }
   }));
   /* *INDENT-ON* */
@@ -1555,7 +1558,7 @@ vl_api_nat44_user_session_dump_t_print (vl_api_nat44_user_session_dump_t * mp,
 
 static nat44_lb_addr_port_t *
 unformat_nat44_lb_addr_port (vl_api_nat44_lb_addr_port_t * addr_port_pairs,
-			     u8 addr_port_pair_num)
+			     u32 addr_port_pair_num)
 {
   u8 i;
   nat44_lb_addr_port_t *lb_addr_port_pairs = 0, lb_addr_port;
@@ -1594,7 +1597,9 @@ static void
       goto send_reply;
     }
 
-  locals = unformat_nat44_lb_addr_port (mp->locals, mp->local_num);
+  locals =
+    unformat_nat44_lb_addr_port (mp->locals,
+				 clib_net_to_host_u32 (mp->local_num));
   clib_memcpy (&e_addr, mp->external_addr, 4);
   proto = ip_proto_to_snat_proto (mp->protocol);
   if (mp->twice_nat)
@@ -1632,6 +1637,52 @@ static void *vl_api_nat44_add_del_lb_static_mapping_t_print
 }
 
 static void
+  vl_api_nat44_lb_static_mapping_add_del_local_t_handler
+  (vl_api_nat44_lb_static_mapping_add_del_local_t * mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat44_lb_static_mapping_add_del_local_reply_t *rmp;
+  int rv = 0;
+  ip4_address_t e_addr, l_addr;
+  snat_protocol_t proto;
+
+  if (!sm->endpoint_dependent)
+    {
+      rv = VNET_API_ERROR_UNSUPPORTED;
+      goto send_reply;
+    }
+
+  clib_memcpy (&e_addr, mp->external_addr, 4);
+  clib_memcpy (&l_addr, mp->local.addr, 4);
+  proto = ip_proto_to_snat_proto (mp->protocol);
+
+  rv =
+    nat44_lb_static_mapping_add_del_local (e_addr,
+					   clib_net_to_host_u16
+					   (mp->external_port), l_addr,
+					   clib_net_to_host_u16 (mp->
+								 local.port),
+					   proto,
+					   clib_net_to_host_u32 (mp->
+								 local.vrf_id),
+					   mp->local.probability, mp->is_add);
+
+send_reply:
+  REPLY_MACRO (VL_API_NAT44_LB_STATIC_MAPPING_ADD_DEL_LOCAL_REPLY);
+}
+
+static void *vl_api_nat44_lb_static_mapping_add_del_local_t_print
+  (vl_api_nat44_lb_static_mapping_add_del_local_t * mp, void *handle)
+{
+  u8 *s;
+
+  s = format (0, "SCRIPT: nat44_lb_static_mapping_add_del_local ");
+  s = format (s, "is_add %d", mp->is_add);
+
+  FINISH;
+}
+
+static void
 send_nat44_lb_static_mapping_details (snat_static_mapping_t * m,
 				      vl_api_registration_t * reg,
 				      u32 context)
@@ -1640,10 +1691,12 @@ send_nat44_lb_static_mapping_details (snat_static_mapping_t * m,
   snat_main_t *sm = &snat_main;
   nat44_lb_addr_port_t *ap;
   vl_api_nat44_lb_addr_port_t *locals;
+  u32 local_num = 0;
 
   rmp =
     vl_msg_api_alloc (sizeof (*rmp) +
-		      (vec_len (m->locals) * sizeof (nat44_lb_addr_port_t)));
+		      (pool_elts (m->locals) *
+		       sizeof (nat44_lb_addr_port_t)));
   clib_memset (rmp, 0, sizeof (*rmp));
   rmp->_vl_msg_id =
     ntohs (VL_API_NAT44_LB_STATIC_MAPPING_DETAILS + sm->msg_id_base);
@@ -1661,15 +1714,18 @@ send_nat44_lb_static_mapping_details (snat_static_mapping_t * m,
     strncpy ((char *) rmp->tag, (char *) m->tag, vec_len (m->tag));
 
   locals = (vl_api_nat44_lb_addr_port_t *) rmp->locals;
-  vec_foreach (ap, m->locals)
-  {
+  /* *INDENT-OFF* */
+  pool_foreach (ap, m->locals,
+  ({
     clib_memcpy (locals->addr, &(ap->addr), 4);
     locals->port = htons (ap->port);
     locals->probability = ap->probability;
     locals->vrf_id = ntohl (ap->vrf_id);
     locals++;
-    rmp->local_num++;
-  }
+    local_num++;
+  }));
+  /* *INDENT-ON* */
+  rmp->local_num = ntohl (local_num);
 
   vl_api_send_msg (reg, (u8 *) rmp);
 }
@@ -3159,6 +3215,8 @@ _(NAT44_INTERFACE_ADD_DEL_OUTPUT_FEATURE,                               \
 _(NAT44_INTERFACE_OUTPUT_FEATURE_DUMP,                                  \
   nat44_interface_output_feature_dump)                                  \
 _(NAT44_ADD_DEL_LB_STATIC_MAPPING, nat44_add_del_lb_static_mapping)     \
+_(NAT44_LB_STATIC_MAPPING_ADD_DEL_LOCAL,                                \
+  nat44_lb_static_mapping_add_del_local)                                \
 _(NAT44_LB_STATIC_MAPPING_DUMP, nat44_lb_static_mapping_dump)           \
 _(NAT44_DEL_SESSION, nat44_del_session)                                 \
 _(NAT44_FORWARDING_ENABLE_DISABLE, nat44_forwarding_enable_disable)     \
diff --git a/src/plugins/nat/nat_format.c b/src/plugins/nat/nat_format.c
index 8e5ac4cade4..8452940898d 100644
--- a/src/plugins/nat/nat_format.c
+++ b/src/plugins/nat/nat_format.c
@@ -231,8 +231,10 @@ format_snat_static_mapping (u8 * s, va_list * args)
 		    format_ip4_address, &m->local_addr, m->local_port);
 
       /* *INDENT-OFF* */
-      vec_foreach (local, m->locals)
+      pool_foreach (local, m->locals,
+      ({
         s = format (s, " vrf %d", local->vrf_id);
+      }));
       /* *INDENT-ON* */
 
       return s;
@@ -256,10 +258,16 @@ format_snat_static_mapping (u8 * s, va_list * args)
 		      m->twice_nat == TWICE_NAT ? "twice-nat" :
 		      m->twice_nat == TWICE_NAT_SELF ? "self-twice-nat" : "",
 		      is_out2in_only_static_mapping (m) ? "out2in-only" : "");
-	  vec_foreach (local, m->locals)
+
+          /* *INDENT-OFF* */
+          pool_foreach (local, m->locals,
+          ({
 	    s = format (s, "\n  local %U:%d vrf %d probability %d\%",
 			format_ip4_address, &local->addr, local->port,
 			local->vrf_id, local->probability);
+          }));
+          /* *INDENT-ON* */
+
 	}
       else
 	s = format (s, "%U local %U:%d external %U:%d vrf %d %s %s",
diff --git a/test/test_nat.py b/test/test_nat.py
index 9879b762cc9..516961739b8 100644
--- a/test/test_nat.py
+++ b/test/test_nat.py
@@ -4269,6 +4269,7 @@ class TestNAT44EndpointDependent(MethodHolder):
         local_port = 8080
         server1 = self.pg0.remote_hosts[0]
         server2 = self.pg0.remote_hosts[1]
+        server3 = self.pg0.remote_hosts[2]
 
         locals = [{'addr': server1.ip4n,
                    'port': local_port,
@@ -4309,6 +4310,65 @@ class TestNAT44EndpointDependent(MethodHolder):
                 server2_n += 1
         self.assertGreater(server1_n, server2_n)
 
+        # add new back-end
+        self.vapi.nat44_lb_static_mapping_add_del_local(external_addr_n,
+                                                        external_port,
+                                                        server3.ip4n,
+                                                        local_port,
+                                                        IP_PROTOS.tcp,
+                                                        20)
+        server1_n = 0
+        server2_n = 0
+        server3_n = 0
+        clients = ip4_range(self.pg1.remote_ip4, 60, 110)
+        pkts = []
+        for client in clients:
+            p = (Ether(src=self.pg1.remote_mac, dst=self.pg1.local_mac) /
+                 IP(src=client, dst=self.nat_addr) /
+                 TCP(sport=12346, dport=external_port))
+            pkts.append(p)
+        self.assertGreater(len(pkts), 0)
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        for p in capture:
+            if p[IP].dst == server1.ip4:
+                server1_n += 1
+            elif p[IP].dst == server2.ip4:
+                server2_n += 1
+            else:
+                server3_n += 1
+        self.assertGreater(server1_n, 0)
+        self.assertGreater(server2_n, 0)
+        self.assertGreater(server3_n, 0)
+
+        # remove one back-end
+        self.vapi.nat44_lb_static_mapping_add_del_local(external_addr_n,
+                                                        external_port,
+                                                        server2.ip4n,
+                                                        local_port,
+                                                        IP_PROTOS.tcp,
+                                                        10,
+                                                        is_add=0)
+        server1_n = 0
+        server2_n = 0
+        server3_n = 0
+        self.pg1.add_stream(pkts)
+        self.pg_enable_capture(self.pg_interfaces)
+        self.pg_start()
+        capture = self.pg0.get_capture(len(pkts))
+        for p in capture:
+            if p[IP].dst == server1.ip4:
+                server1_n += 1
+            elif p[IP].dst == server2.ip4:
+                server2_n += 1
+            else:
+                server3_n += 1
+        self.assertGreater(server1_n, 0)
+        self.assertEqual(server2_n, 0)
+        self.assertGreater(server3_n, 0)
+
     def test_static_lb_2(self):
         """ NAT44 local service load balancing (asymmetrical rule) """
         external_addr_n = socket.inet_pton(socket.AF_INET, self.nat_addr)
diff --git a/test/vpp_papi_provider.py b/test/vpp_papi_provider.py
index 0a33c1eb2ac..4ab4380692e 100644
--- a/test/vpp_papi_provider.py
+++ b/test/vpp_papi_provider.py
@@ -1668,6 +1668,39 @@ class VppPapiProvider(object):
              'local_num': local_num,
              'locals': locals})
 
+    def nat44_lb_static_mapping_add_del_local(
+           self,
+           external_addr,
+           external_port,
+           local_addr,
+           local_port,
+           protocol,
+           probability,
+           vrf_id=0,
+           is_add=1):
+        """Add/delete NAT44 load-balancing static mapping rule backend
+
+        :param external_addr: external IPv4 address of the servic
+        :param external_port: external L4 port number of the service
+        :param local_addr: IPv4 address of the internal node
+        :param local_port: L4 port number of the internal node
+        :param protocol: IP protocol number
+        :param probability: probability of the internal node
+        :param vrf_id: VRF id of the internal node
+        :param is_add: 1 if add, 0 if delete
+        """
+        return self.api(
+            self.papi.nat44_lb_static_mapping_add_del_local,
+            {'is_add': is_add,
+             'external_addr': external_addr,
+             'external_port': external_port,
+             'local': {
+                 'addr': local_addr,
+                 'port': local_port,
+                 'probability': probability,
+                 'vrf_id': vrf_id},
+             'protocol': protocol})
+
     def nat44_lb_static_mapping_dump(self):
         """Dump NAT44 load balancing static mappings