nat: configurable handoff frame queue size

Make number of worker handoff frame queue elements configurable as
a set nat frame-queue-nelts command. The default value is 64 which
is the same value that was previously hard-coded. The idea is that
allowing larger values can be useful in some cases, to avoid
congestion drops. Also add nat_set_fq_options API support and a
corresponding test case.

Type: improvement

Change-Id: I5c321eb2d7997f76fac2703d9c4a5b2516375db3
Signed-off-by: Elias Rudberg <elias.rudberg@bahnhof.net>

7 files changed, 171 insertions, 6 deletions

diff --git a/src/plugins/nat/nat.c b/src/plugins/nat/nat.c
index 57d3b2bfdd1..11664aba0b6 100644
--- a/src/plugins/nat/nat.c
+++ b/src/plugins/nat/nat.c
@@ -1839,13 +1839,15 @@ snat_interface_add_del (u32 sw_if_index, u8 is_inside, int is_del)
 	feature_name = is_inside ? "nat44-in2out" : "nat44-out2in";
     }
 
+  ASSERT (sm->frame_queue_nelts > 0);
+
   if (sm->fq_in2out_index == ~0 && sm->num_workers > 1)
-    sm->fq_in2out_index =
-      vlib_frame_queue_main_init (sm->in2out_node_index, NAT_FQ_NELTS);
+    sm->fq_in2out_index = vlib_frame_queue_main_init (sm->in2out_node_index,
+						      sm->frame_queue_nelts);
 
   if (sm->fq_out2in_index == ~0 && sm->num_workers > 1)
-    sm->fq_out2in_index =
-      vlib_frame_queue_main_init (sm->out2in_node_index, NAT_FQ_NELTS);
+    sm->fq_out2in_index = vlib_frame_queue_main_init (sm->out2in_node_index,
+						      sm->frame_queue_nelts);
 
   if (sm->endpoint_dependent)
     update_per_vrf_sessions_vec (fib_index, is_del);
@@ -2280,6 +2282,15 @@ snat_set_workers (uword * bitmap)
   return 0;
 }
 
+int
+snat_set_frame_queue_nelts (u32 frame_queue_nelts)
+{
+  fail_if_enabled ();
+  snat_main_t *sm = &snat_main;
+  sm->frame_queue_nelts = frame_queue_nelts;
+  return 0;
+}
+
 static void
 snat_update_outside_fib (ip4_main_t * im, uword opaque,
 			 u32 sw_if_index, u32 new_fib_index,
@@ -2710,6 +2721,9 @@ nat44_ed_plugin_enable (nat44_config_t c)
   vlib_zero_simple_counter (&sm->total_sessions, 0);
   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
 
+  if (!sm->frame_queue_nelts)
+    sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
+
   sm->enabled = 1;
   sm->rconfig = c;
 
diff --git a/src/plugins/nat/nat.h b/src/plugins/nat/nat.h
index 86f6342ab09..d7f60dd2145 100644
--- a/src/plugins/nat/nat.h
+++ b/src/plugins/nat/nat.h
@@ -36,8 +36,8 @@
 #include <nat/lib/lib.h>
 #include <nat/lib/inlines.h>
 
-/* number of worker handoff frame queue elements */
-#define NAT_FQ_NELTS 64
+/* default number of worker handoff frame queue elements */
+#define NAT_FQ_NELTS_DEFAULT 64
 
 /* NAT buffer flags */
 #define SNAT_FLAG_HAIRPINNING (1 << 0)
@@ -782,6 +782,9 @@ typedef struct snat_main_s
   /* pat - dynamic mapping enabled or conneciton tracking */
   u8 pat;
 
+  /* number of worker handoff frame queue elements */
+  u32 frame_queue_nelts;
+
   /* nat44 plugin enabled */
   u8 enabled;
 
@@ -1299,6 +1302,15 @@ clib_error_t *nat44_api_hookup (vlib_main_t * vm);
 int snat_set_workers (uword * bitmap);
 
 /**
+ * @brief Set NAT plugin number of frame queue elements
+ *
+ * @param frame_queue_nelts number of worker handoff frame queue elements
+ *
+ * @return 0 on success, non-zero value otherwise
+ */
+int snat_set_frame_queue_nelts (u32 frame_queue_nelts);
+
+/**
  * @brief Enable/disable NAT44 feature on the interface
  *
  * @param sw_if_index software index of the interface
diff --git a/src/plugins/nat/nat44-ei/nat44_ei.c b/src/plugins/nat/nat44-ei/nat44_ei.c
index 34288e14856..02403d0cd99 100644
--- a/src/plugins/nat/nat44-ei/nat44_ei.c
+++ b/src/plugins/nat/nat44-ei/nat44_ei.c
@@ -117,6 +117,9 @@ nat44_ei_plugin_enable (nat44_ei_config_t c)
   vlib_zero_simple_counter (&sm->total_sessions, 0);
   vlib_zero_simple_counter (&sm->user_limit_reached, 0);
 
+  if (!sm->frame_queue_nelts)
+    sm->frame_queue_nelts = NAT_FQ_NELTS_DEFAULT;
+
   sm->enabled = 1;
 
   return 0;
diff --git a/src/plugins/nat/nat44.api b/src/plugins/nat/nat44.api
index fd06c10149c..f7fe7fd0fac 100644
--- a/src/plugins/nat/nat44.api
+++ b/src/plugins/nat/nat44.api
@@ -390,6 +390,42 @@ define nat_get_timeouts_reply {
   u32 icmp;
 };
 
+/** \brief Set NAT handoff frame queue options
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param frame_queue_nelts - number of worker handoff frame queue elements
+*/
+autoreply define nat_set_fq_options {
+  option in_progress;
+  u32 client_index;
+  u32 context;
+  u32 frame_queue_nelts;
+};
+
+/** \brief Show NAT handoff frame queue options
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+*/
+define nat_show_fq_options
+{
+  option in_progress;
+  u32 client_index;
+  u32 context;
+};
+
+/** \brief Show NAT handoff frame queue options reply
+    @param context - sender context, to match reply w/ request
+    @param retval - return code for the request
+    @param frame_queue_nelts - number of worker handoff frame queue elements
+*/
+define nat_show_fq_options_reply
+{
+  option in_progress;
+  u32 context;
+  i32 retval;
+  u32 frame_queue_nelts;
+};
+
 /** \brief Set address and port assignment algorithm
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
diff --git a/src/plugins/nat/nat44_api.c b/src/plugins/nat/nat44_api.c
index c025db74f2e..05a79719261 100644
--- a/src/plugins/nat/nat44_api.c
+++ b/src/plugins/nat/nat44_api.c
@@ -334,6 +334,31 @@ vl_api_nat_get_timeouts_t_handler (vl_api_nat_get_timeouts_t * mp)
 }
 
 static void
+vl_api_nat_set_fq_options_t_handler (vl_api_nat_set_fq_options_t *mp)
+{
+  snat_main_t *sm = &snat_main;
+  vl_api_nat_set_fq_options_reply_t *rmp;
+  int rv = 0;
+  u32 frame_queue_nelts = ntohl (mp->frame_queue_nelts);
+  rv = snat_set_frame_queue_nelts (frame_queue_nelts);
+  REPLY_MACRO (VL_API_NAT_SET_FQ_OPTIONS_REPLY);
+}
+
+static void
+vl_api_nat_show_fq_options_t_handler (vl_api_nat_show_fq_options_t *mp)
+{
+  vl_api_nat_show_fq_options_reply_t *rmp;
+  snat_main_t *sm = &snat_main;
+  int rv = 0;
+  /* clang-format off */
+  REPLY_MACRO2_ZERO (VL_API_NAT_SHOW_FQ_OPTIONS_REPLY,
+  ({
+    rmp->frame_queue_nelts = htonl (sm->frame_queue_nelts);
+  }));
+  /* clang-format on */
+}
+
+static void
   vl_api_nat_set_addr_and_port_alloc_alg_t_handler
   (vl_api_nat_set_addr_and_port_alloc_alg_t * mp)
 {
diff --git a/src/plugins/nat/nat44_cli.c b/src/plugins/nat/nat44_cli.c
index c89963ec85a..68c53d05389 100644
--- a/src/plugins/nat/nat44_cli.c
+++ b/src/plugins/nat/nat44_cli.c
@@ -1892,6 +1892,42 @@ nat_show_timeouts_command_fn (vlib_main_t * vm,
 }
 
 static clib_error_t *
+set_frame_queue_nelts_command_fn (vlib_main_t *vm, unformat_input_t *input,
+				  vlib_cli_command_t *cmd)
+{
+  unformat_input_t _line_input, *line_input = &_line_input;
+  clib_error_t *error = 0;
+  u32 frame_queue_nelts = 0;
+  /* Get a line of input. */
+  if (!unformat_user (input, unformat_line_input, line_input))
+    return 0;
+  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (line_input, "%u", &frame_queue_nelts))
+	;
+      else
+	{
+	  error = clib_error_return (0, "unknown input '%U'",
+				     format_unformat_error, line_input);
+	  goto done;
+	}
+    }
+  if (!frame_queue_nelts)
+    {
+      error = clib_error_return (0, "frame_queue_nelts cannot be zero");
+      goto done;
+    }
+  if (snat_set_frame_queue_nelts (frame_queue_nelts) != 0)
+    {
+      error = clib_error_return (0, "snat_set_frame_queue_nelts failed");
+      goto done;
+    }
+done:
+  unformat_free (line_input);
+  return error;
+}
+
+static clib_error_t *
 nat44_debug_fib_expire_command_fn (vlib_main_t * vm,
 				   unformat_input_t * input,
 				   vlib_cli_command_t * cmd)
@@ -2067,6 +2103,18 @@ VLIB_CLI_COMMAND (nat_show_timeouts_command, static) = {
 
 /*?
  * @cliexpar
+ * @cliexstart{set nat frame-queue-nelts}
+ * Set number of worker handoff frame queue elements.
+ * @cliexend
+?*/
+VLIB_CLI_COMMAND (set_frame_queue_nelts_command, static) = {
+  .path = "set nat frame-queue-nelts",
+  .function = set_frame_queue_nelts_command_fn,
+  .short_help = "set nat frame-queue-nelts <number>",
+};
+
+/*?
+ * @cliexpar
  * @cliexstart{nat set logging level}
  * To set NAT logging level use:
  * Set nat logging level
diff --git a/src/plugins/nat/test/test_nat44_ei.py b/src/plugins/nat/test/test_nat44_ei.py
index 16a3376e9b2..e69e24b2911 100644
--- a/src/plugins/nat/test/test_nat44_ei.py
+++ b/src/plugins/nat/test/test_nat44_ei.py
@@ -858,6 +858,33 @@ class MethodHolder(VppTestCase):
             self.assertEqual(data, p[Raw].load)
 
 
+class TestNAT44EIAPI(MethodHolder):
+    """ NAT44EI API Test Cases """
+
+    fq_nelts = 512
+
+    def setUp(self):
+        super(TestNAT44EIAPI, self).setUp()
+        self.vapi.nat_set_fq_options(frame_queue_nelts=self.fq_nelts)
+        self.vapi.nat44_plugin_enable_disable(enable=1)
+
+    def tearDown(self):
+        super(TestNAT44EIAPI, self).tearDown()
+        if not self.vpp_dead:
+            self.vapi.nat44_plugin_enable_disable(enable=0)
+            self.vapi.cli("clear logging")
+
+    def test_show_frame_queue_nelts(self):
+        """ API test - worker handoff frame queue elements """
+        nat_config = self.vapi.nat_show_fq_options()
+        self.assertEqual(self.fq_nelts, nat_config.frame_queue_nelts)
+        self.vapi.nat44_plugin_enable_disable(enable=0)
+        self.vapi.cli("set nat frame-queue-nelts 256")
+        self.vapi.nat44_plugin_enable_disable(enable=1)
+        nat_config = self.vapi.nat_show_fq_options()
+        self.assertEqual(256, nat_config.frame_queue_nelts)
+
+
 class TestNAT44EI(MethodHolder):
     """ NAT44EI Test Cases """