From 6809538e646bf86c000dc1faba60b0a4157ad898 Mon Sep 17 00:00:00 2001 From: Mohsin Kazmi Date: Wed, 10 Feb 2021 11:26:24 +0100 Subject: vlib: refactor checksum offload support Type: refactor This patch refactors the offload flags in vlib_buffer_t. There are two main reasons behind this refactoring. First, offload flags are insufficient to represent outer and inner headers offloads. Second, room for these flags in first cacheline of vlib_buffer_t is also limited. This patch introduces a generic offload flag in first cacheline. And detailed offload flags in 2nd cacheline of the structure for performance optimization. Change-Id: Icc363a142fb9208ec7113ab5bbfc8230181f6004 Signed-off-by: Mohsin Kazmi --- src/vnet/devices/af_packet/node.c | 13 +++++++++---- src/vnet/devices/virtio/device.c | 22 ++++++++++++---------- src/vnet/devices/virtio/node.c | 14 ++++++++------ src/vnet/devices/virtio/vhost_user_input.c | 11 ++++++++--- src/vnet/devices/virtio/vhost_user_output.c | 19 ++++++++----------- 5 files changed, 45 insertions(+), 34 deletions(-) (limited to 'src/vnet/devices') diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c index f4db6399a57..9f1782e2d13 100644 --- a/src/vnet/devices/af_packet/node.c +++ b/src/vnet/devices/af_packet/node.c @@ -120,6 +120,7 @@ static_always_inline void mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) { ethernet_header_t *eth = vlib_buffer_get_current (b); + u32 oflags = 0; if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP4) { ip4_header_t *ip4 = @@ -127,7 +128,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) b->flags |= VNET_BUFFER_F_IS_IP4; if (ip4->protocol == IP_PROTOCOL_TCP) { - b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b) + sizeof (ethernet_header_t) + ip4_header_bytes (ip4)); @@ -136,7 +137,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) } else if (ip4->protocol == IP_PROTOCOL_UDP) { - b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b) + sizeof (ethernet_header_t) + ip4_header_bytes (ip4)); @@ -146,6 +147,8 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t); vnet_buffer (b)->l4_hdr_offset = sizeof (ethernet_header_t) + ip4_header_bytes (ip4); + if (oflags) + vnet_buffer_offload_flags_set (b, oflags); } else if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP6) { @@ -165,7 +168,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) } if (ip6->protocol == IP_PROTOCOL_TCP) { - b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b) + sizeof (ethernet_header_t) + ip6_hdr_len); @@ -174,7 +177,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) } else if (ip6->protocol == IP_PROTOCOL_UDP) { - b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b) + sizeof (ethernet_header_t) + ip6_hdr_len); @@ -184,6 +187,8 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t); vnet_buffer (b)->l4_hdr_offset = sizeof (ethernet_header_t) + ip6_hdr_len; + if (oflags) + vnet_buffer_offload_flags_set (b, oflags); } } diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c index 3482336b2ad..fb996d551fa 100644 --- a/src/vnet/devices/virtio/device.c +++ b/src/vnet/devices/virtio/device.c @@ -282,6 +282,8 @@ static_always_inline void set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, const int is_l2) { + u32 oflags = vnet_buffer2 (b)->oflags; + if (b->flags & VNET_BUFFER_F_IS_IP4) { ip4_header_t *ip4; @@ -290,11 +292,11 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, 0 /* ip6 */ ); hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; hdr->csum_start = gho.l4_hdr_offset; // 0x22; - if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) { hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum); } - else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) + else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) { hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum); } @@ -305,7 +307,7 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, */ ip4 = (ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset); - if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) ip4->checksum = ip4_header_checksum (ip4); } else if (b->flags & VNET_BUFFER_F_IS_IP6) @@ -315,11 +317,11 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, 1 /* ip6 */ ); hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; hdr->csum_start = gho.l4_hdr_offset; // 0x36; - if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) { hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum); } - else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) + else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) { hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum); } @@ -330,6 +332,8 @@ static_always_inline void set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, const int is_l2) { + u32 oflags = vnet_buffer2 (b)->oflags; + if (b->flags & VNET_BUFFER_F_IS_IP4) { ip4_header_t *ip4; @@ -348,7 +352,7 @@ set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, * virtio devices do not support IP4 checksum offload. So driver takes care * of it while doing tx. */ - if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) ip4->checksum = ip4_header_checksum (ip4); } else if (b->flags & VNET_BUFFER_F_IS_IP6) @@ -392,8 +396,7 @@ add_buffer_to_slot (vlib_main_t * vm, vlib_node_runtime_t * node, goto done; } } - else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) + else if (b->flags & VNET_BUFFER_F_OFFLOAD) { if (csum_offload) set_checksum_offsets (b, hdr, is_l2); @@ -584,8 +587,7 @@ add_buffer_to_slot_packed (vlib_main_t * vm, vlib_node_runtime_t * node, goto done; } } - else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) + else if (b->flags & VNET_BUFFER_F_OFFLOAD) { if (csum_offload) set_checksum_offsets (b, hdr, is_l2); diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c index be2405bb3b9..98df322ea15 100644 --- a/src/vnet/devices/virtio/node.c +++ b/src/vnet/devices/virtio/node.c @@ -235,6 +235,7 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { u16 ethertype = 0, l2hdr_sz = 0; + u32 oflags = 0; if (type == VIRTIO_IF_TYPE_TUN) { @@ -280,11 +281,10 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, (ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz); vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4); *l4_proto = ip4->protocol; + oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; b0->flags |= - (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM); - b0->flags |= - (VNET_BUFFER_F_L2_HDR_OFFSET_VALID - | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID | + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID); } else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) @@ -301,7 +301,7 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, } if (*l4_proto == IP_PROTOCOL_TCP) { - b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b0) + vnet_buffer (b0)->l4_hdr_offset); @@ -309,12 +309,14 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, } else if (*l4_proto == IP_PROTOCOL_UDP) { - b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b0) + vnet_buffer (b0)->l4_hdr_offset); *l4_hdr_sz = sizeof (*udp); } + if (oflags) + vnet_buffer_offload_flags_set (b0, oflags); } } diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c index 62b59f69ba9..739125bbcfe 100644 --- a/src/vnet/devices/virtio/vhost_user_input.c +++ b/src/vnet/devices/virtio/vhost_user_input.c @@ -253,6 +253,7 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data, ethernet_header_t *eh = (ethernet_header_t *) b0_data; u16 ethertype = clib_net_to_host_u16 (eh->type); u16 l2hdr_sz = sizeof (ethernet_header_t); + u32 oflags = 0; if (ethernet_frame_is_tagged (ethertype)) { @@ -278,7 +279,8 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data, { ip4_header_t *ip4 = (ip4_header_t *) (b0_data + l2hdr_sz); l4_proto = ip4->protocol; - b0->flags |= VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM; + b0->flags |= VNET_BUFFER_F_IS_IP4; + oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; } else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) { @@ -292,12 +294,12 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data, tcp_header_t *tcp = (tcp_header_t *) (b0_data + vnet_buffer (b0)->l4_hdr_offset); l4_hdr_sz = tcp_header_bytes (tcp); - b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; } else if (l4_proto == IP_PROTOCOL_UDP) { l4_hdr_sz = sizeof (udp_header_t); - b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; } if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP) @@ -318,6 +320,9 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data, vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz; b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6); } + + if (oflags) + vnet_buffer_offload_flags_set (b0, oflags); } static_always_inline void diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c index 465c0ea0903..8dbddea46f5 100644 --- a/src/vnet/devices/virtio/vhost_user_output.c +++ b/src/vnet/devices/virtio/vhost_user_output.c @@ -225,10 +225,11 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b, generic_header_offset_t gho = { 0 }; int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4; int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6; + u32 oflags = vnet_buffer2 (b)->oflags; ASSERT (!(is_ip4 && is_ip6)); vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6); - if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) { ip4_header_t *ip4; @@ -238,13 +239,13 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b, } /* checksum offload */ - if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) { hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; hdr->csum_start = gho.l4_hdr_offset; hdr->csum_offset = offsetof (udp_header_t, checksum); } - else if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) + else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) { hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; hdr->csum_start = gho.l4_hdr_offset; @@ -254,7 +255,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b, /* GSO offload */ if (b->flags & VNET_BUFFER_F_GSO) { - if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) { if (is_ip4 && (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4))) @@ -270,7 +271,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b, } } else if ((vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)) && - (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) + (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) { hdr->gso_size = vnet_buffer2 (b)->gso_size; hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP; @@ -477,9 +478,7 @@ retry: hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE; hdr->num_buffers = 1; - or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) || - (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) || - (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM); + or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD); /* Guest supports csum offload and buffer requires checksum offload? */ if (or_flags && @@ -813,9 +812,7 @@ retry: hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE; hdr->num_buffers = 1; //This is local, no need to check - or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) || - (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) || - (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM); + or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD); /* Guest supports csum offload and buffer requires checksum offload? */ if (or_flags -- cgit 1.2.3-korg