summaryrefslogtreecommitdiffstats
path: root/src/vnet/devices
diff options
context:
space:
mode:
authorMohsin Kazmi <sykazmi@cisco.com>2021-02-10 11:26:24 +0100
committerDamjan Marion <dmarion@me.com>2021-02-15 20:32:56 +0000
commit6809538e646bf86c000dc1faba60b0a4157ad898 (patch)
tree18a228b96226932381f15e44b4972636de1c7fe1 /src/vnet/devices
parent99c6dc6a7a36c0be95da9afb3ad8830b24754d4e (diff)
vlib: refactor checksum offload support
Type: refactor This patch refactors the offload flags in vlib_buffer_t. There are two main reasons behind this refactoring. First, offload flags are insufficient to represent outer and inner headers offloads. Second, room for these flags in first cacheline of vlib_buffer_t is also limited. This patch introduces a generic offload flag in first cacheline. And detailed offload flags in 2nd cacheline of the structure for performance optimization. Change-Id: Icc363a142fb9208ec7113ab5bbfc8230181f6004 Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>
Diffstat (limited to 'src/vnet/devices')
-rw-r--r--src/vnet/devices/af_packet/node.c13
-rw-r--r--src/vnet/devices/virtio/device.c22
-rw-r--r--src/vnet/devices/virtio/node.c14
-rw-r--r--src/vnet/devices/virtio/vhost_user_input.c11
-rw-r--r--src/vnet/devices/virtio/vhost_user_output.c19
5 files changed, 45 insertions, 34 deletions
diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c
index f4db6399a57..9f1782e2d13 100644
--- a/src/vnet/devices/af_packet/node.c
+++ b/src/vnet/devices/af_packet/node.c
@@ -120,6 +120,7 @@ static_always_inline void
mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
{
ethernet_header_t *eth = vlib_buffer_get_current (b);
+ u32 oflags = 0;
if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP4)
{
ip4_header_t *ip4 =
@@ -127,7 +128,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
b->flags |= VNET_BUFFER_F_IS_IP4;
if (ip4->protocol == IP_PROTOCOL_TCP)
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) +
ip4_header_bytes (ip4));
@@ -136,7 +137,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
}
else if (ip4->protocol == IP_PROTOCOL_UDP)
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) +
ip4_header_bytes (ip4));
@@ -146,6 +147,8 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t);
vnet_buffer (b)->l4_hdr_offset =
sizeof (ethernet_header_t) + ip4_header_bytes (ip4);
+ if (oflags)
+ vnet_buffer_offload_flags_set (b, oflags);
}
else if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP6)
{
@@ -165,7 +168,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
}
if (ip6->protocol == IP_PROTOCOL_TCP)
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp =
(tcp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) + ip6_hdr_len);
@@ -174,7 +177,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
}
else if (ip6->protocol == IP_PROTOCOL_UDP)
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp =
(udp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) + ip6_hdr_len);
@@ -184,6 +187,8 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t);
vnet_buffer (b)->l4_hdr_offset =
sizeof (ethernet_header_t) + ip6_hdr_len;
+ if (oflags)
+ vnet_buffer_offload_flags_set (b, oflags);
}
}
diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c
index 3482336b2ad..fb996d551fa 100644
--- a/src/vnet/devices/virtio/device.c
+++ b/src/vnet/devices/virtio/device.c
@@ -282,6 +282,8 @@ static_always_inline void
set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
const int is_l2)
{
+ u32 oflags = vnet_buffer2 (b)->oflags;
+
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
ip4_header_t *ip4;
@@ -290,11 +292,11 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
0 /* ip6 */ );
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset; // 0x22;
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
}
@@ -305,7 +307,7 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
*/
ip4 =
(ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
}
else if (b->flags & VNET_BUFFER_F_IS_IP6)
@@ -315,11 +317,11 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
1 /* ip6 */ );
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset; // 0x36;
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
}
@@ -330,6 +332,8 @@ static_always_inline void
set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
const int is_l2)
{
+ u32 oflags = vnet_buffer2 (b)->oflags;
+
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
ip4_header_t *ip4;
@@ -348,7 +352,7 @@ set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
* virtio devices do not support IP4 checksum offload. So driver takes care
* of it while doing tx.
*/
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
}
else if (b->flags & VNET_BUFFER_F_IS_IP6)
@@ -392,8 +396,7 @@ add_buffer_to_slot (vlib_main_t * vm, vlib_node_runtime_t * node,
goto done;
}
}
- else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ else if (b->flags & VNET_BUFFER_F_OFFLOAD)
{
if (csum_offload)
set_checksum_offsets (b, hdr, is_l2);
@@ -584,8 +587,7 @@ add_buffer_to_slot_packed (vlib_main_t * vm, vlib_node_runtime_t * node,
goto done;
}
}
- else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ else if (b->flags & VNET_BUFFER_F_OFFLOAD)
{
if (csum_offload)
set_checksum_offsets (b, hdr, is_l2);
diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c
index be2405bb3b9..98df322ea15 100644
--- a/src/vnet/devices/virtio/node.c
+++ b/src/vnet/devices/virtio/node.c
@@ -235,6 +235,7 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
{
u16 ethertype = 0, l2hdr_sz = 0;
+ u32 oflags = 0;
if (type == VIRTIO_IF_TYPE_TUN)
{
@@ -280,11 +281,10 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
(ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
*l4_proto = ip4->protocol;
+ oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
b0->flags |=
- (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
- b0->flags |=
- (VNET_BUFFER_F_L2_HDR_OFFSET_VALID
- | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
@@ -301,7 +301,7 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
}
if (*l4_proto == IP_PROTOCOL_TCP)
{
- b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b0) +
vnet_buffer
(b0)->l4_hdr_offset);
@@ -309,12 +309,14 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
}
else if (*l4_proto == IP_PROTOCOL_UDP)
{
- b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b0) +
vnet_buffer
(b0)->l4_hdr_offset);
*l4_hdr_sz = sizeof (*udp);
}
+ if (oflags)
+ vnet_buffer_offload_flags_set (b0, oflags);
}
}
diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c
index 62b59f69ba9..739125bbcfe 100644
--- a/src/vnet/devices/virtio/vhost_user_input.c
+++ b/src/vnet/devices/virtio/vhost_user_input.c
@@ -253,6 +253,7 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
ethernet_header_t *eh = (ethernet_header_t *) b0_data;
u16 ethertype = clib_net_to_host_u16 (eh->type);
u16 l2hdr_sz = sizeof (ethernet_header_t);
+ u32 oflags = 0;
if (ethernet_frame_is_tagged (ethertype))
{
@@ -278,7 +279,8 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
{
ip4_header_t *ip4 = (ip4_header_t *) (b0_data + l2hdr_sz);
l4_proto = ip4->protocol;
- b0->flags |= VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ b0->flags |= VNET_BUFFER_F_IS_IP4;
+ oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
{
@@ -292,12 +294,12 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
tcp_header_t *tcp = (tcp_header_t *)
(b0_data + vnet_buffer (b0)->l4_hdr_offset);
l4_hdr_sz = tcp_header_bytes (tcp);
- b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
}
else if (l4_proto == IP_PROTOCOL_UDP)
{
l4_hdr_sz = sizeof (udp_header_t);
- b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
}
if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP)
@@ -318,6 +320,9 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
}
+
+ if (oflags)
+ vnet_buffer_offload_flags_set (b0, oflags);
}
static_always_inline void
diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c
index 465c0ea0903..8dbddea46f5 100644
--- a/src/vnet/devices/virtio/vhost_user_output.c
+++ b/src/vnet/devices/virtio/vhost_user_output.c
@@ -225,10 +225,11 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
generic_header_offset_t gho = { 0 };
int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
+ u32 oflags = vnet_buffer2 (b)->oflags;
ASSERT (!(is_ip4 && is_ip6));
vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6);
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
{
ip4_header_t *ip4;
@@ -238,13 +239,13 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
}
/* checksum offload */
- if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset;
hdr->csum_offset = offsetof (udp_header_t, checksum);
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset;
@@ -254,7 +255,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
/* GSO offload */
if (b->flags & VNET_BUFFER_F_GSO)
{
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
if (is_ip4 &&
(vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)))
@@ -270,7 +271,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
}
}
else if ((vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)) &&
- (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
{
hdr->gso_size = vnet_buffer2 (b)->gso_size;
hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
@@ -477,9 +478,7 @@ retry:
hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
hdr->num_buffers = 1;
- or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+ or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
/* Guest supports csum offload and buffer requires checksum offload? */
if (or_flags &&
@@ -813,9 +812,7 @@ retry:
hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
hdr->num_buffers = 1; //This is local, no need to check
- or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+ or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
/* Guest supports csum offload and buffer requires checksum offload? */
if (or_flags