From 6809538e646bf86c000dc1faba60b0a4157ad898 Mon Sep 17 00:00:00 2001 From: Mohsin Kazmi Date: Wed, 10 Feb 2021 11:26:24 +0100 Subject: vlib: refactor checksum offload support Type: refactor This patch refactors the offload flags in vlib_buffer_t. There are two main reasons behind this refactoring. First, offload flags are insufficient to represent outer and inner headers offloads. Second, room for these flags in first cacheline of vlib_buffer_t is also limited. This patch introduces a generic offload flag in first cacheline. And detailed offload flags in 2nd cacheline of the structure for performance optimization. Change-Id: Icc363a142fb9208ec7113ab5bbfc8230181f6004 Signed-off-by: Mohsin Kazmi --- src/plugins/avf/output.c | 14 ++++++-------- src/plugins/dpdk/device/device.c | 21 +++++++++------------ src/plugins/gtpu/gtpu_decap.c | 16 +++++++++------- src/plugins/lisp/lisp-cp/packets.c | 2 +- src/plugins/vmxnet3/input.c | 21 ++++++++++++--------- src/plugins/wireguard/wireguard_input.c | 3 ++- 6 files changed, 39 insertions(+), 38 deletions(-) (limited to 'src/plugins') diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c index c53fc42a850..5bcb68cc95c 100644 --- a/src/plugins/avf/output.c +++ b/src/plugins/avf/output.c @@ -57,12 +57,12 @@ static_always_inline u64 avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso) { u64 flags = 0; - if (!is_tso && !(b->flags & ((VNET_BUFFER_F_OFFLOAD_IP_CKSUM | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)))) + if (!is_tso && !(b->flags & VNET_BUFFER_F_OFFLOAD)) return 0; - u32 is_tcp = is_tso || b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; - u32 is_udp = !is_tso && b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + + u32 oflags = vnet_buffer2 (b)->oflags; + u32 is_tcp = is_tso || oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; + u32 is_udp = !is_tso && oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; u32 is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4; u32 is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6; ASSERT (!is_tcp || !is_udp); @@ -173,9 +173,7 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq, { u16 next = txq->next; u64 bits = AVF_TXD_CMD_EOP | AVF_TXD_CMD_RSV; - const u32 offload_mask = VNET_BUFFER_F_OFFLOAD_IP_CKSUM | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM | - VNET_BUFFER_F_GSO; + const u32 offload_mask = VNET_BUFFER_F_OFFLOAD | VNET_BUFFER_F_GSO; u64 one_by_one_offload_flags = 0; int is_tso; u16 n_desc = 0; diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index ec33f6a4461..6466b443633 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -218,17 +218,20 @@ static_always_inline void dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b, struct rte_mbuf *mb) { - u32 ip_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM; - u32 tcp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; - u32 udp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4; u32 tso = b->flags & VNET_BUFFER_F_GSO, max_pkt_len; + u32 oflags, ip_cksum, tcp_cksum, udp_cksum; u64 ol_flags; /* Is there any work for us? */ - if (PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum | tso) == 0)) + if (PREDICT_TRUE (((b->flags & VNET_BUFFER_F_OFFLOAD) | tso) == 0)) return; + oflags = vnet_buffer2 (b)->oflags; + ip_cksum = oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM; + tcp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; + udp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; + mb->l2_len = vnet_buffer (b)->l3_hdr_offset - b->current_data; mb->l3_len = vnet_buffer (b)->l4_hdr_offset - vnet_buffer (b)->l3_hdr_offset; @@ -328,10 +331,7 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm, } if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) && - (or_flags & - (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM - | VNET_BUFFER_F_OFFLOAD_IP_CKSUM - | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)))) + (or_flags & VNET_BUFFER_F_OFFLOAD))) { dpdk_buffer_tx_offload (xd, b[0], mb[0]); dpdk_buffer_tx_offload (xd, b[1], mb[1]); @@ -388,10 +388,7 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm, } if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) && - (or_flags & - (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM - | VNET_BUFFER_F_OFFLOAD_IP_CKSUM - | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)))) + (or_flags & VNET_BUFFER_F_OFFLOAD))) { dpdk_buffer_tx_offload (xd, b[0], mb[0]); dpdk_buffer_tx_offload (xd, b[1], mb[1]); diff --git a/src/plugins/gtpu/gtpu_decap.c b/src/plugins/gtpu/gtpu_decap.c index e3bc476ed6d..974ae8a8190 100644 --- a/src/plugins/gtpu/gtpu_decap.c +++ b/src/plugins/gtpu/gtpu_decap.c @@ -1257,13 +1257,15 @@ static char *gtpu_flow_error_strings[] = { }; -#define gtpu_local_need_csum_check(_b) \ - (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED \ - || _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) - -#define gtpu_local_csum_is_valid(_b) \ - ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT \ - || _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) != 0) +#define gtpu_local_need_csum_check(_b) \ + (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED || \ + (_b->flags & VNET_BUFFER_F_OFFLOAD && \ + vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))) + +#define gtpu_local_csum_is_valid(_b) \ + ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT || \ + (_b->flags & VNET_BUFFER_F_OFFLOAD && \ + vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0) static_always_inline u8 gtpu_validate_udp_csum (vlib_main_t * vm, vlib_buffer_t *b) diff --git a/src/plugins/lisp/lisp-cp/packets.c b/src/plugins/lisp/lisp-cp/packets.c index 8e2386e72bd..3f4292b4841 100644 --- a/src/plugins/lisp/lisp-cp/packets.c +++ b/src/plugins/lisp/lisp-cp/packets.c @@ -182,7 +182,7 @@ pkt_push_udp_and_ip (vlib_main_t * vm, vlib_buffer_t * b, u16 sp, u16 dp, if (csum_offload) { ih = pkt_push_ip (vm, b, sip, dip, IP_PROTOCOL_UDP, 1); - b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM); vnet_buffer (b)->l3_hdr_offset = (u8 *) ih - b->data; vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data; uh->checksum = 0; diff --git a/src/plugins/vmxnet3/input.c b/src/plugins/vmxnet3/input.c index affc3691af1..be528b26494 100644 --- a/src/plugins/vmxnet3/input.c +++ b/src/plugins/vmxnet3/input.c @@ -79,6 +79,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, u16 gso_size) { u8 l4_hdr_sz = 0; + u32 oflags = 0; if (rx_comp->flags & VMXNET3_RXCF_IP4) { @@ -90,15 +91,15 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) + ip4_header_bytes (ip4); hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID | - VNET_BUFFER_F_L3_HDR_OFFSET_VALID | - VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4; + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4; /* checksum offload */ if (!(rx_comp->index & VMXNET3_RXCI_CNC)) { if (!(rx_comp->flags & VMXNET3_RXCF_IPC)) { - hb->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; ip4->checksum = 0; } if (!(rx_comp->flags & VMXNET3_RXCF_TUC)) @@ -108,7 +109,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, tcp_header_t *tcp = (tcp_header_t *) (hb->data + vnet_buffer (hb)->l4_hdr_offset); - hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; tcp->checksum = 0; } else if (rx_comp->flags & VMXNET3_RXCF_UDP) @@ -116,7 +117,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, udp_header_t *udp = (udp_header_t *) (hb->data + vnet_buffer (hb)->l4_hdr_offset); - hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; udp->checksum = 0; } } @@ -148,8 +149,8 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) + sizeof (ip6_header_t); hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID | - VNET_BUFFER_F_L3_HDR_OFFSET_VALID | - VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6; + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6; /* checksum offload */ if (!(rx_comp->index & VMXNET3_RXCI_CNC)) @@ -161,7 +162,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, tcp_header_t *tcp = (tcp_header_t *) (hb->data + vnet_buffer (hb)->l4_hdr_offset); - hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; tcp->checksum = 0; } else if (rx_comp->flags & VMXNET3_RXCF_UDP) @@ -169,7 +170,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, udp_header_t *udp = (udp_header_t *) (hb->data + vnet_buffer (hb)->l4_hdr_offset); - hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; udp->checksum = 0; } } @@ -194,6 +195,8 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, hb->flags |= VNET_BUFFER_F_GSO; } } + if (oflags) + vnet_buffer_offload_flags_set (hb, oflags); } static_always_inline uword diff --git a/src/plugins/wireguard/wireguard_input.c b/src/plugins/wireguard/wireguard_input.c index d9843d90c7d..2690cf5e097 100755 --- a/src/plugins/wireguard/wireguard_input.c +++ b/src/plugins/wireguard/wireguard_input.c @@ -368,7 +368,8 @@ VLIB_NODE_FN (wg_input_node) (vlib_main_t * vm, clib_memcpy (vlib_buffer_get_current (b[0]), decr_data, decr_len); b[0]->current_length = decr_len; - b[0]->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + vnet_buffer_offload_flags_clear (b[0], + VNET_BUFFER_OFFLOAD_F_UDP_CKSUM); wg_timers_any_authenticated_packet_received (peer); wg_timers_any_authenticated_packet_traversal (peer); -- cgit 1.2.3-korg