From 0b42ac565b970c186a9ad734f980b440f56fb25b Mon Sep 17 00:00:00 2001 From: Mohammed Hawari Date: Wed, 12 May 2021 12:31:01 +0200 Subject: dpdk: lro support Change-Id: Ifa09dcdfd567ac3c2afab4bd3bd468260f766e1d Signed-off-by: Mohammed Hawari Type: improvement --- src/plugins/dpdk/device/dpdk.h | 14 +++--- src/plugins/dpdk/device/dpdk_priv.h | 1 + src/plugins/dpdk/device/format.c | 1 + src/plugins/dpdk/device/init.c | 27 +++++++++++- src/plugins/dpdk/device/node.c | 86 +++++++++++++++++++++++++++++++------ 5 files changed, 110 insertions(+), 19 deletions(-) diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h index 9df7f398ee6..dd43137aebe 100644 --- a/src/plugins/dpdk/device/dpdk.h +++ b/src/plugins/dpdk/device/dpdk.h @@ -251,11 +251,12 @@ typedef struct #define DPDK_LINK_POLL_INTERVAL (3.0) #define DPDK_MIN_LINK_POLL_INTERVAL (0.001) /* 1msec */ -#define foreach_dpdk_device_config_item \ - _ (num_rx_queues) \ - _ (num_tx_queues) \ - _ (num_rx_desc) \ - _ (num_tx_desc) \ +#define foreach_dpdk_device_config_item \ + _ (num_rx_queues) \ + _ (num_tx_queues) \ + _ (num_rx_desc) \ + _ (num_tx_desc) \ + _ (max_lro_pkt_size) \ _ (rss_fn) typedef enum @@ -301,6 +302,7 @@ typedef struct u8 *eal_init_args_str; u8 *uio_driver_name; u8 no_multi_seg; + u8 enable_lro; u8 enable_tcp_udp_checksum; u8 no_tx_checksum_offload; u8 enable_telemetry; @@ -347,7 +349,7 @@ typedef struct u32 buffers[DPDK_RX_BURST_SZ]; u16 next[DPDK_RX_BURST_SZ]; u16 etype[DPDK_RX_BURST_SZ]; - u16 flags[DPDK_RX_BURST_SZ]; + u32 flags[DPDK_RX_BURST_SZ]; vlib_buffer_t buffer_template; } dpdk_per_thread_data_t; diff --git a/src/plugins/dpdk/device/dpdk_priv.h b/src/plugins/dpdk/device/dpdk_priv.h index a5a8a2ad57d..7f595ae8e19 100644 --- a/src/plugins/dpdk/device/dpdk_priv.h +++ b/src/plugins/dpdk/device/dpdk_priv.h @@ -15,6 +15,7 @@ #define DPDK_NB_RX_DESC_DEFAULT 1024 #define DPDK_NB_TX_DESC_DEFAULT 1024 +#define DPDK_MAX_LRO_SIZE_DEFAULT 65536 #define DPDK_NB_RX_DESC_VIRTIO 256 #define DPDK_NB_TX_DESC_VIRTIO 256 diff --git a/src/plugins/dpdk/device/format.c b/src/plugins/dpdk/device/format.c index 71f0d89454b..eb98730113d 100644 --- a/src/plugins/dpdk/device/format.c +++ b/src/plugins/dpdk/device/format.c @@ -60,6 +60,7 @@ _ (PKT_RX_L4_CKSUM_GOOD, "L4 cksum of RX pkt. is valid") \ _ (PKT_RX_IEEE1588_PTP, "RX IEEE1588 L2 Ethernet PT Packet") \ _ (PKT_RX_IEEE1588_TMST, "RX IEEE1588 L2/L4 timestamped packet") \ + _ (PKT_RX_LRO, "LRO packet") \ _ (PKT_RX_QINQ_STRIPPED, "RX packet QinQ tags stripped") #define foreach_dpdk_pkt_type \ diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index f923da6c09e..8c49ab841fd 100644 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -372,6 +372,19 @@ dpdk_lib_init (dpdk_main_t * dm) xd->port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_TCP_CKSUM; } + if (dm->conf->enable_lro) + { + if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) + { + xd->port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_TCP_LRO; + if (devconf->max_lro_pkt_size) + xd->port_conf.rxmode.max_lro_pkt_size = + devconf->max_lro_pkt_size; + else + xd->port_conf.rxmode.max_lro_pkt_size = + DPDK_MAX_LRO_SIZE_DEFAULT; + } + } if (dm->conf->no_multi_seg) { xd->port_conf.txmode.offloads &= ~DEV_TX_OFFLOAD_MULTI_SEGS; @@ -474,9 +487,16 @@ dpdk_lib_init (dpdk_main_t * dm) xd->port_conf.intr_conf.rxq = 1; break; + case VNET_DPDK_PMD_MLX5: + if (dm->conf->no_tx_checksum_offload == 0) + { + xd->port_conf.txmode.offloads |= DEV_TX_OFFLOAD_TCP_CKSUM; + xd->port_conf.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM; + xd->flags |= DPDK_DEVICE_FLAG_TX_OFFLOAD | + DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM; + } case VNET_DPDK_PMD_CXGBE: case VNET_DPDK_PMD_MLX4: - case VNET_DPDK_PMD_MLX5: case VNET_DPDK_PMD_QEDE: case VNET_DPDK_PMD_BNXT: xd->port_type = port_type_from_speed_capa (&dev_info); @@ -1270,6 +1290,9 @@ dpdk_device_config (dpdk_config_main_t *conf, void *addr, else if (unformat (input, "rss-queues %U", unformat_bitmap_list, &devconf->rss_queues)) ; + else if (unformat (input, "max-lro-pkt-size %u", + &devconf->max_lro_pkt_size)) + ; else { error = clib_error_return (0, "unknown input `%U'", @@ -1376,6 +1399,8 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) else if (unformat (input, "no-multi-seg")) conf->no_multi_seg = 1; + else if (unformat (input, "enable-lro")) + conf->enable_lro = 1; else if (unformat (input, "max-simd-bitwidth %U", unformat_max_simd_bitwidth, &conf->max_simd_bitwidth)) ; diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index 25222856912..c725681caa5 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -36,9 +36,10 @@ static char *dpdk_error_strings[] = { #undef _ }; -/* make sure all flags we need are stored in lower 8 bits */ -STATIC_ASSERT ((PKT_RX_IP_CKSUM_BAD | PKT_RX_FDIR) < - 256, "dpdk flags not un lower byte, fix needed"); +/* make sure all flags we need are stored in lower 32 bits */ +STATIC_ASSERT ((u64) (PKT_RX_IP_CKSUM_BAD | PKT_RX_FDIR | PKT_RX_LRO) < + (1ULL << 32), + "dpdk flags not in lower word, fix needed"); static_always_inline uword dpdk_process_subseq_segs (vlib_main_t * vm, vlib_buffer_t * b, @@ -145,31 +146,30 @@ dpdk_prefetch_buffer_x4 (struct rte_mbuf *mb[]) xd->per_interface_next_index */ -static_always_inline u16 -dpdk_ol_flags_extract (struct rte_mbuf **mb, u16 * flags, int count) +static_always_inline u32 +dpdk_ol_flags_extract (struct rte_mbuf **mb, u32 *flags, int count) { - u16 rv = 0; + u32 rv = 0; int i; for (i = 0; i < count; i++) { /* all flags we are interested in are in lower 8 bits but that might change */ - flags[i] = (u16) mb[i]->ol_flags; + flags[i] = (u32) mb[i]->ol_flags; rv |= flags[i]; } return rv; } static_always_inline uword -dpdk_process_rx_burst (vlib_main_t * vm, dpdk_per_thread_data_t * ptd, - uword n_rx_packets, int maybe_multiseg, - u16 * or_flagsp) +dpdk_process_rx_burst (vlib_main_t *vm, dpdk_per_thread_data_t *ptd, + uword n_rx_packets, int maybe_multiseg, u32 *or_flagsp) { u32 n_left = n_rx_packets; vlib_buffer_t *b[4]; struct rte_mbuf **mb = ptd->mbufs; uword n_bytes = 0; - u16 *flags, or_flags = 0; + u32 *flags, or_flags = 0; vlib_buffer_t bt; mb = ptd->mbufs; @@ -277,6 +277,65 @@ dpdk_process_flow_offload (dpdk_device_t * xd, dpdk_per_thread_data_t * ptd, } } +static_always_inline u16 +dpdk_lro_find_l4_hdr_sz (vlib_buffer_t *b) +{ + u16 l4_hdr_sz = 0; + u16 current_offset = 0; + ethernet_header_t *e; + tcp_header_t *tcp; + u8 *data = vlib_buffer_get_current (b); + u16 ethertype; + e = (void *) data; + current_offset += sizeof (e[0]); + ethertype = clib_net_to_host_u16 (e->type); + if (ethernet_frame_is_tagged (ethertype)) + { + ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (e + 1); + ethertype = clib_net_to_host_u16 (vlan->type); + current_offset += sizeof (*vlan); + if (ethertype == ETHERNET_TYPE_VLAN) + { + vlan++; + current_offset += sizeof (*vlan); + ethertype = clib_net_to_host_u16 (vlan->type); + } + } + data += current_offset; + if (ethertype == ETHERNET_TYPE_IP4) + { + data += sizeof (ip4_header_t); + tcp = (void *) data; + l4_hdr_sz = tcp_header_bytes (tcp); + } + else + { + /* FIXME: extension headers...*/ + data += sizeof (ip6_header_t); + tcp = (void *) data; + l4_hdr_sz = tcp_header_bytes (tcp); + } + return l4_hdr_sz; +} + +static_always_inline void +dpdk_process_lro_offload (dpdk_device_t *xd, dpdk_per_thread_data_t *ptd, + uword n_rx_packets) +{ + uword n; + vlib_buffer_t *b0; + for (n = 0; n < n_rx_packets; n++) + { + b0 = vlib_buffer_from_rte_mbuf (ptd->mbufs[n]); + if (ptd->flags[n] & PKT_RX_LRO) + { + b0->flags |= VNET_BUFFER_F_GSO; + vnet_buffer2 (b0)->gso_size = ptd->mbufs[n]->tso_segsz; + vnet_buffer2 (b0)->gso_l4_hdr_sz = dpdk_lro_find_l4_hdr_sz (b0); + } + } +} + static_always_inline u32 dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, vlib_node_runtime_t * node, u32 thread_index, u16 queue_id) @@ -289,7 +348,7 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, struct rte_mbuf **mb; vlib_buffer_t *b0; u16 *next; - u16 or_flags; + u32 or_flags; u32 n; int single_next = 0; @@ -339,6 +398,9 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, else n_rx_bytes = dpdk_process_rx_burst (vm, ptd, n_rx_packets, 0, &or_flags); + if (PREDICT_FALSE ((or_flags & PKT_RX_LRO))) + dpdk_process_lro_offload (xd, ptd, n_rx_packets); + if (PREDICT_FALSE (or_flags & PKT_RX_FDIR)) { /* some packets will need to go to different next nodes */ -- cgit 1.2.3-korg