gso: packet coalesce library

Type: feature

Change-Id: Ia19d3611e596d9ec47509889b34e8fe793a0ccc3
Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>

7 files changed, 113 insertions, 9 deletions

diff --git a/src/vnet/pg/cli.c b/src/vnet/pg/cli.c
index b3aaddfda2b..cb8b5bbb287 100644
--- a/src/vnet/pg/cli.c
+++ b/src/vnet/pg/cli.c
@@ -661,7 +661,7 @@ create_pg_if_cmd_fn (vlib_main_t * vm,
 {
   pg_main_t *pg = &pg_main;
   unformat_input_t _line_input, *line_input = &_line_input;
-  u32 if_id, gso_enabled = 0, gso_size = 0;
+  u32 if_id, gso_enabled = 0, gso_size = 0, coalesce_enabled = 0;
   clib_error_t *error = NULL;
 
   if (!unformat_user (input, unformat_line_input, line_input))
@@ -681,6 +681,8 @@ create_pg_if_cmd_fn (vlib_main_t * vm,
 	      error = clib_error_create ("gso enabled but gso size missing");
 	      goto done;
 	    }
+	  if (unformat (line_input, "coalesce-enabled"))
+	    coalesce_enabled = 1;
 	}
       else
 	{
@@ -690,7 +692,8 @@ create_pg_if_cmd_fn (vlib_main_t * vm,
 	}
     }
 
-  pg_interface_add_or_get (pg, if_id, gso_enabled, gso_size);
+  pg_interface_add_or_get (pg, if_id, gso_enabled, gso_size,
+			   coalesce_enabled);
 
 done:
   unformat_free (line_input);
@@ -701,7 +704,8 @@ done:
 /* *INDENT-OFF* */
 VLIB_CLI_COMMAND (create_pg_if_cmd, static) = {
   .path = "create packet-generator",
-  .short_help = "create packet-generator interface <interface name> [gso-enabled gso-size <size>]",
+  .short_help = "create packet-generator interface <interface name>"
+                " [gso-enabled gso-size <size> [coalesce-enabled]]",
   .function = create_pg_if_cmd_fn,
 };
 /* *INDENT-ON* */
diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c
index 6968cce19e7..785592f3618 100644
--- a/src/vnet/pg/input.c
+++ b/src/vnet/pg/input.c
@@ -54,6 +54,7 @@
 #include <vnet/ip/ip6_packet.h>
 #include <vnet/udp/udp_packet.h>
 #include <vnet/devices/devices.h>
+#include <vnet/gso/gro_func.h>
 
 static int
 validate_buffer_data2 (vlib_buffer_t * b, pg_stream_t * s,
@@ -1640,6 +1641,9 @@ pg_generate_packets (vlib_node_runtime_t * node,
 			    &next_index, 0);
     }
 
+  if (PREDICT_FALSE (pi->coalesce_enabled))
+    vnet_gro_flow_table_schedule_node_on_dispatcher (vm, pi->flow_table);
+
   while (n_packets_to_generate > 0)
     {
       u32 *head, *start, *end;
diff --git a/src/vnet/pg/output.c b/src/vnet/pg/output.c
index d8059fab186..042591a7709 100644
--- a/src/vnet/pg/output.c
+++ b/src/vnet/pg/output.c
@@ -42,6 +42,7 @@
 #include <vnet/vnet.h>
 #include <vnet/pg/pg.h>
 #include <vnet/ethernet/ethernet.h>
+#include <vnet/gso/gro_func.h>
 
 uword
 pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
@@ -50,6 +51,8 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
   u32 *buffers = vlib_frame_vector_args (frame);
   uword n_buffers = frame->n_vectors;
   uword n_left = n_buffers;
+  u32 to[GRO_TO_VECTOR_SIZE (n_buffers)];
+  uword n_to = 0;
   vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
   pg_interface_t *pif = pool_elt_at_index (pg->interfaces, rd->dev_instance);
 
@@ -57,6 +60,13 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
     while (clib_atomic_test_and_set (pif->lockp))
       ;
 
+  if (PREDICT_FALSE (pif->coalesce_enabled))
+    {
+      n_to = vnet_gro_inline (vm, pif->flow_table, buffers, n_left, to);
+      buffers = to;
+      n_left = n_to;
+    }
+
   while (n_left > 0)
     {
       n_left--;
@@ -84,7 +94,13 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
       pif->pcap_main.n_packets_to_capture)
     pcap_close (&pif->pcap_main);
 
-  vlib_buffer_free (vm, vlib_frame_vector_args (frame), n_buffers);
+  if (PREDICT_FALSE (pif->coalesce_enabled))
+    {
+      n_buffers = n_to;
+      vlib_buffer_free (vm, to, n_to);
+    }
+  else
+    vlib_buffer_free (vm, vlib_frame_vector_args (frame), n_buffers);
   if (PREDICT_FALSE (pif->lockp != 0))
     clib_atomic_release (pif->lockp);
 
diff --git a/src/vnet/pg/pg.api b/src/vnet/pg/pg.api
index 86343d5c8df..3a44f1d87a7 100644
--- a/src/vnet/pg/pg.api
+++ b/src/vnet/pg/pg.api
@@ -49,6 +49,21 @@ define pg_create_interface_reply
   vl_api_interface_index_t sw_if_index;
 };
 
+/** \brief PacketGenerator interface enable/disable packet coalesce
+    @param client_index - opaque cookie to identify the sender
+    @param context - sender context, to match reply w/ request
+    @param interface_id - interface index
+    @param coalesce_enabled - enable/disable packet coalesce on this interface
+*/
+autoreply define pg_interface_enable_disable_coalesce
+{
+  u32 client_index;
+  u32 context;
+  vl_api_interface_index_t sw_if_index;
+  bool coalesce_enabled;
+  option status="in_progress";
+};
+
 /** \brief PacketGenerator capture packets on given interface request
     @param client_index - opaque cookie to identify the sender
     @param context - sender context, to match reply w/ request
diff --git a/src/vnet/pg/pg.h b/src/vnet/pg/pg.h
index a6616d9ced7..06e61261b7d 100644
--- a/src/vnet/pg/pg.h
+++ b/src/vnet/pg/pg.h
@@ -45,6 +45,7 @@
 #include <vppinfra/fifo.h>	/* for buffer_fifo */
 #include <vppinfra/pcap.h>
 #include <vnet/interface.h>
+#include <vnet/gso/gro.h>
 
 extern vnet_device_class_t pg_dev_class;
 
@@ -305,6 +306,8 @@ typedef struct
   /* Identifies stream for this interface. */
   u32 id;
 
+  u8 coalesce_enabled;
+  gro_flow_table_t *flow_table;
   u8 gso_enabled;
   u32 gso_size;
   pcap_main_t pcap_main;
@@ -358,9 +361,14 @@ void pg_stream_change (pg_main_t * pg, pg_stream_t * s);
 void pg_stream_enable_disable (pg_main_t * pg, pg_stream_t * s,
 			       int is_enable);
 
+/* Enable/disable packet coalesce on given interface */
+void pg_interface_enable_disable_coalesce (pg_interface_t * pi, u8 enable,
+					   u32 tx_node_index);
+
 /* Find/create free packet-generator interface index. */
 u32 pg_interface_add_or_get (pg_main_t * pg, uword stream_index,
-			     u8 gso_enabled, u32 gso_size);
+			     u8 gso_enabled, u32 gso_size,
+			     u8 coalesce_enabled);
 
 always_inline pg_node_t *
 pg_get_node (uword node_index)
diff --git a/src/vnet/pg/pg_api.c b/src/vnet/pg/pg_api.c
index bb58a4f0cec..554e8ea31c1 100644
--- a/src/vnet/pg/pg_api.c
+++ b/src/vnet/pg/pg_api.c
@@ -44,7 +44,8 @@
 #define foreach_pg_api_msg                                              \
 _(PG_CREATE_INTERFACE, pg_create_interface)                             \
 _(PG_CAPTURE, pg_capture)                                               \
-_(PG_ENABLE_DISABLE, pg_enable_disable)
+_(PG_ENABLE_DISABLE, pg_enable_disable)                                 \
+_(PG_INTERFACE_ENABLE_DISABLE_COALESCE, pg_interface_enable_disable_coalesce)
 
 static void
 vl_api_pg_create_interface_t_handler (vl_api_pg_create_interface_t * mp)
@@ -55,7 +56,7 @@ vl_api_pg_create_interface_t_handler (vl_api_pg_create_interface_t * mp)
   pg_main_t *pg = &pg_main;
   u32 pg_if_id = pg_interface_add_or_get (pg, ntohl (mp->interface_id),
 					  mp->gso_enabled,
-					  ntohl (mp->gso_size));
+					  ntohl (mp->gso_size), 0);
   pg_interface_t *pi = pool_elt_at_index (pg->interfaces, pg_if_id);
 
   /* *INDENT-OFF* */
@@ -67,6 +68,41 @@ vl_api_pg_create_interface_t_handler (vl_api_pg_create_interface_t * mp)
 }
 
 static void
+  vl_api_pg_interface_enable_disable_coalesce_t_handler
+  (vl_api_pg_interface_enable_disable_coalesce_t * mp)
+{
+  vl_api_pg_interface_enable_disable_coalesce_reply_t *rmp;
+  int rv = 0;
+
+  VALIDATE_SW_IF_INDEX (mp);
+
+  u32 sw_if_index = ntohl (mp->sw_if_index);
+
+  pg_main_t *pg = &pg_main;
+  vnet_main_t *vnm = vnet_get_main ();
+  vnet_hw_interface_t *hw =
+    vnet_get_sup_hw_interface_api_visible_or_null (vnm, sw_if_index);
+
+  if (hw)
+    {
+      pg_interface_t *pi =
+	pool_elt_at_index (pg->interfaces, hw->dev_instance);
+      if (pi->gso_enabled)
+	pg_interface_enable_disable_coalesce (pi, mp->coalesce_enabled,
+					      hw->tx_node_index);
+      else
+	rv = VNET_API_ERROR_CANNOT_ENABLE_DISABLE_FEATURE;
+    }
+  else
+    {
+      rv = VNET_API_ERROR_NO_MATCHING_INTERFACE;
+    }
+
+  BAD_SW_IF_INDEX_LABEL;
+  REPLY_MACRO (VL_API_PG_INTERFACE_ENABLE_DISABLE_COALESCE_REPLY);
+}
+
+static void
 vl_api_pg_capture_t_handler (vl_api_pg_capture_t * mp)
 {
   vl_api_pg_capture_reply_t *rmp;
diff --git a/src/vnet/pg/stream.c b/src/vnet/pg/stream.c
index f09e9a44398..88c89371c6c 100644
--- a/src/vnet/pg/stream.c
+++ b/src/vnet/pg/stream.c
@@ -178,9 +178,26 @@ pg_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags)
   return 0;
 }
 
+void
+pg_interface_enable_disable_coalesce (pg_interface_t * pi, u8 enable,
+				      u32 tx_node_index)
+{
+  if (enable)
+    {
+      gro_flow_table_init (&pi->flow_table, 1 /* is_l2 */ ,
+			   tx_node_index);
+      pi->coalesce_enabled = 1;
+    }
+  else
+    {
+      pi->coalesce_enabled = 0;
+      gro_flow_table_free (pi->flow_table);
+    }
+}
+
 u32
 pg_interface_add_or_get (pg_main_t * pg, uword if_id, u8 gso_enabled,
-			 u32 gso_size)
+			 u32 gso_size, u8 coalesce_enabled)
 {
   vnet_main_t *vnm = vnet_get_main ();
   vlib_main_t *vm = vlib_get_main ();
@@ -219,6 +236,10 @@ pg_interface_add_or_get (pg_main_t * pg, uword if_id, u8 gso_enabled,
 	  hi->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO;
 	  pi->gso_enabled = 1;
 	  pi->gso_size = gso_size;
+	  if (coalesce_enabled)
+	    {
+	      pg_interface_enable_disable_coalesce (pi, 1, hi->tx_node_index);
+	    }
 	}
       pi->sw_if_index = hi->sw_if_index;
 
@@ -454,7 +475,7 @@ pg_stream_add (pg_main_t * pg, pg_stream_t * s_init)
   /* Find an interface to use. */
   s->pg_if_index =
     pg_interface_add_or_get (pg, s->if_id, 0 /* gso_enabled */ ,
-			     0 /* gso_size */ );
+			     0 /* gso_size */ , 0 /* coalesce_enabled */ );
 
   if (s->sw_if_index[VLIB_RX] == ~0)
     {