From f382b06febf2b26c37fd384824a1915e16517a2b Mon Sep 17 00:00:00 2001 From: Mohsin Kazmi Date: Tue, 11 Aug 2020 15:00:44 +0200 Subject: gso: packet coalesce library Type: feature Change-Id: Ia19d3611e596d9ec47509889b34e8fe793a0ccc3 Signed-off-by: Mohsin Kazmi --- src/vnet/pg/cli.c | 10 +++++++--- src/vnet/pg/input.c | 4 ++++ src/vnet/pg/output.c | 18 +++++++++++++++++- src/vnet/pg/pg.api | 15 +++++++++++++++ src/vnet/pg/pg.h | 10 +++++++++- src/vnet/pg/pg_api.c | 40 ++++++++++++++++++++++++++++++++++++++-- src/vnet/pg/stream.c | 25 +++++++++++++++++++++++-- 7 files changed, 113 insertions(+), 9 deletions(-) (limited to 'src/vnet/pg') diff --git a/src/vnet/pg/cli.c b/src/vnet/pg/cli.c index b3aaddfda2b..cb8b5bbb287 100644 --- a/src/vnet/pg/cli.c +++ b/src/vnet/pg/cli.c @@ -661,7 +661,7 @@ create_pg_if_cmd_fn (vlib_main_t * vm, { pg_main_t *pg = &pg_main; unformat_input_t _line_input, *line_input = &_line_input; - u32 if_id, gso_enabled = 0, gso_size = 0; + u32 if_id, gso_enabled = 0, gso_size = 0, coalesce_enabled = 0; clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) @@ -681,6 +681,8 @@ create_pg_if_cmd_fn (vlib_main_t * vm, error = clib_error_create ("gso enabled but gso size missing"); goto done; } + if (unformat (line_input, "coalesce-enabled")) + coalesce_enabled = 1; } else { @@ -690,7 +692,8 @@ create_pg_if_cmd_fn (vlib_main_t * vm, } } - pg_interface_add_or_get (pg, if_id, gso_enabled, gso_size); + pg_interface_add_or_get (pg, if_id, gso_enabled, gso_size, + coalesce_enabled); done: unformat_free (line_input); @@ -701,7 +704,8 @@ done: /* *INDENT-OFF* */ VLIB_CLI_COMMAND (create_pg_if_cmd, static) = { .path = "create packet-generator", - .short_help = "create packet-generator interface [gso-enabled gso-size ]", + .short_help = "create packet-generator interface " + " [gso-enabled gso-size [coalesce-enabled]]", .function = create_pg_if_cmd_fn, }; /* *INDENT-ON* */ diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c index 6968cce19e7..785592f3618 100644 --- a/src/vnet/pg/input.c +++ b/src/vnet/pg/input.c @@ -54,6 +54,7 @@ #include #include #include +#include static int validate_buffer_data2 (vlib_buffer_t * b, pg_stream_t * s, @@ -1640,6 +1641,9 @@ pg_generate_packets (vlib_node_runtime_t * node, &next_index, 0); } + if (PREDICT_FALSE (pi->coalesce_enabled)) + vnet_gro_flow_table_schedule_node_on_dispatcher (vm, pi->flow_table); + while (n_packets_to_generate > 0) { u32 *head, *start, *end; diff --git a/src/vnet/pg/output.c b/src/vnet/pg/output.c index d8059fab186..042591a7709 100644 --- a/src/vnet/pg/output.c +++ b/src/vnet/pg/output.c @@ -42,6 +42,7 @@ #include #include #include +#include uword pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) @@ -50,6 +51,8 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) u32 *buffers = vlib_frame_vector_args (frame); uword n_buffers = frame->n_vectors; uword n_left = n_buffers; + u32 to[GRO_TO_VECTOR_SIZE (n_buffers)]; + uword n_to = 0; vnet_interface_output_runtime_t *rd = (void *) node->runtime_data; pg_interface_t *pif = pool_elt_at_index (pg->interfaces, rd->dev_instance); @@ -57,6 +60,13 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) while (clib_atomic_test_and_set (pif->lockp)) ; + if (PREDICT_FALSE (pif->coalesce_enabled)) + { + n_to = vnet_gro_inline (vm, pif->flow_table, buffers, n_left, to); + buffers = to; + n_left = n_to; + } + while (n_left > 0) { n_left--; @@ -84,7 +94,13 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) pif->pcap_main.n_packets_to_capture) pcap_close (&pif->pcap_main); - vlib_buffer_free (vm, vlib_frame_vector_args (frame), n_buffers); + if (PREDICT_FALSE (pif->coalesce_enabled)) + { + n_buffers = n_to; + vlib_buffer_free (vm, to, n_to); + } + else + vlib_buffer_free (vm, vlib_frame_vector_args (frame), n_buffers); if (PREDICT_FALSE (pif->lockp != 0)) clib_atomic_release (pif->lockp); diff --git a/src/vnet/pg/pg.api b/src/vnet/pg/pg.api index 86343d5c8df..3a44f1d87a7 100644 --- a/src/vnet/pg/pg.api +++ b/src/vnet/pg/pg.api @@ -49,6 +49,21 @@ define pg_create_interface_reply vl_api_interface_index_t sw_if_index; }; +/** \brief PacketGenerator interface enable/disable packet coalesce + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param interface_id - interface index + @param coalesce_enabled - enable/disable packet coalesce on this interface +*/ +autoreply define pg_interface_enable_disable_coalesce +{ + u32 client_index; + u32 context; + vl_api_interface_index_t sw_if_index; + bool coalesce_enabled; + option status="in_progress"; +}; + /** \brief PacketGenerator capture packets on given interface request @param client_index - opaque cookie to identify the sender @param context - sender context, to match reply w/ request diff --git a/src/vnet/pg/pg.h b/src/vnet/pg/pg.h index a6616d9ced7..06e61261b7d 100644 --- a/src/vnet/pg/pg.h +++ b/src/vnet/pg/pg.h @@ -45,6 +45,7 @@ #include /* for buffer_fifo */ #include #include +#include extern vnet_device_class_t pg_dev_class; @@ -305,6 +306,8 @@ typedef struct /* Identifies stream for this interface. */ u32 id; + u8 coalesce_enabled; + gro_flow_table_t *flow_table; u8 gso_enabled; u32 gso_size; pcap_main_t pcap_main; @@ -358,9 +361,14 @@ void pg_stream_change (pg_main_t * pg, pg_stream_t * s); void pg_stream_enable_disable (pg_main_t * pg, pg_stream_t * s, int is_enable); +/* Enable/disable packet coalesce on given interface */ +void pg_interface_enable_disable_coalesce (pg_interface_t * pi, u8 enable, + u32 tx_node_index); + /* Find/create free packet-generator interface index. */ u32 pg_interface_add_or_get (pg_main_t * pg, uword stream_index, - u8 gso_enabled, u32 gso_size); + u8 gso_enabled, u32 gso_size, + u8 coalesce_enabled); always_inline pg_node_t * pg_get_node (uword node_index) diff --git a/src/vnet/pg/pg_api.c b/src/vnet/pg/pg_api.c index bb58a4f0cec..554e8ea31c1 100644 --- a/src/vnet/pg/pg_api.c +++ b/src/vnet/pg/pg_api.c @@ -44,7 +44,8 @@ #define foreach_pg_api_msg \ _(PG_CREATE_INTERFACE, pg_create_interface) \ _(PG_CAPTURE, pg_capture) \ -_(PG_ENABLE_DISABLE, pg_enable_disable) +_(PG_ENABLE_DISABLE, pg_enable_disable) \ +_(PG_INTERFACE_ENABLE_DISABLE_COALESCE, pg_interface_enable_disable_coalesce) static void vl_api_pg_create_interface_t_handler (vl_api_pg_create_interface_t * mp) @@ -55,7 +56,7 @@ vl_api_pg_create_interface_t_handler (vl_api_pg_create_interface_t * mp) pg_main_t *pg = &pg_main; u32 pg_if_id = pg_interface_add_or_get (pg, ntohl (mp->interface_id), mp->gso_enabled, - ntohl (mp->gso_size)); + ntohl (mp->gso_size), 0); pg_interface_t *pi = pool_elt_at_index (pg->interfaces, pg_if_id); /* *INDENT-OFF* */ @@ -66,6 +67,41 @@ vl_api_pg_create_interface_t_handler (vl_api_pg_create_interface_t * mp) /* *INDENT-ON* */ } +static void + vl_api_pg_interface_enable_disable_coalesce_t_handler + (vl_api_pg_interface_enable_disable_coalesce_t * mp) +{ + vl_api_pg_interface_enable_disable_coalesce_reply_t *rmp; + int rv = 0; + + VALIDATE_SW_IF_INDEX (mp); + + u32 sw_if_index = ntohl (mp->sw_if_index); + + pg_main_t *pg = &pg_main; + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hw = + vnet_get_sup_hw_interface_api_visible_or_null (vnm, sw_if_index); + + if (hw) + { + pg_interface_t *pi = + pool_elt_at_index (pg->interfaces, hw->dev_instance); + if (pi->gso_enabled) + pg_interface_enable_disable_coalesce (pi, mp->coalesce_enabled, + hw->tx_node_index); + else + rv = VNET_API_ERROR_CANNOT_ENABLE_DISABLE_FEATURE; + } + else + { + rv = VNET_API_ERROR_NO_MATCHING_INTERFACE; + } + + BAD_SW_IF_INDEX_LABEL; + REPLY_MACRO (VL_API_PG_INTERFACE_ENABLE_DISABLE_COALESCE_REPLY); +} + static void vl_api_pg_capture_t_handler (vl_api_pg_capture_t * mp) { diff --git a/src/vnet/pg/stream.c b/src/vnet/pg/stream.c index f09e9a44398..88c89371c6c 100644 --- a/src/vnet/pg/stream.c +++ b/src/vnet/pg/stream.c @@ -178,9 +178,26 @@ pg_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags) return 0; } +void +pg_interface_enable_disable_coalesce (pg_interface_t * pi, u8 enable, + u32 tx_node_index) +{ + if (enable) + { + gro_flow_table_init (&pi->flow_table, 1 /* is_l2 */ , + tx_node_index); + pi->coalesce_enabled = 1; + } + else + { + pi->coalesce_enabled = 0; + gro_flow_table_free (pi->flow_table); + } +} + u32 pg_interface_add_or_get (pg_main_t * pg, uword if_id, u8 gso_enabled, - u32 gso_size) + u32 gso_size, u8 coalesce_enabled) { vnet_main_t *vnm = vnet_get_main (); vlib_main_t *vm = vlib_get_main (); @@ -219,6 +236,10 @@ pg_interface_add_or_get (pg_main_t * pg, uword if_id, u8 gso_enabled, hi->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO; pi->gso_enabled = 1; pi->gso_size = gso_size; + if (coalesce_enabled) + { + pg_interface_enable_disable_coalesce (pi, 1, hi->tx_node_index); + } } pi->sw_if_index = hi->sw_if_index; @@ -454,7 +475,7 @@ pg_stream_add (pg_main_t * pg, pg_stream_t * s_init) /* Find an interface to use. */ s->pg_if_index = pg_interface_add_or_get (pg, s->if_id, 0 /* gso_enabled */ , - 0 /* gso_size */ ); + 0 /* gso_size */ , 0 /* coalesce_enabled */ ); if (s->sw_if_index[VLIB_RX] == ~0) { -- cgit 1.2.3-korg