aboutsummaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorMohsin Kazmi <sykazmi@cisco.com>2021-02-10 11:26:24 +0100
committerDamjan Marion <dmarion@me.com>2021-02-15 20:32:56 +0000
commit6809538e646bf86c000dc1faba60b0a4157ad898 (patch)
tree18a228b96226932381f15e44b4972636de1c7fe1 /src
parent99c6dc6a7a36c0be95da9afb3ad8830b24754d4e (diff)
vlib: refactor checksum offload support
Type: refactor This patch refactors the offload flags in vlib_buffer_t. There are two main reasons behind this refactoring. First, offload flags are insufficient to represent outer and inner headers offloads. Second, room for these flags in first cacheline of vlib_buffer_t is also limited. This patch introduces a generic offload flag in first cacheline. And detailed offload flags in 2nd cacheline of the structure for performance optimization. Change-Id: Icc363a142fb9208ec7113ab5bbfc8230181f6004 Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>
Diffstat (limited to 'src')
-rw-r--r--src/plugins/avf/output.c14
-rw-r--r--src/plugins/dpdk/device/device.c21
-rw-r--r--src/plugins/gtpu/gtpu_decap.c16
-rw-r--r--src/plugins/lisp/lisp-cp/packets.c2
-rw-r--r--src/plugins/vmxnet3/input.c21
-rwxr-xr-xsrc/plugins/wireguard/wireguard_input.c3
-rw-r--r--src/vnet/buffer.c20
-rw-r--r--src/vnet/buffer.h108
-rw-r--r--src/vnet/devices/af_packet/node.c13
-rw-r--r--src/vnet/devices/virtio/device.c22
-rw-r--r--src/vnet/devices/virtio/node.c14
-rw-r--r--src/vnet/devices/virtio/vhost_user_input.c11
-rw-r--r--src/vnet/devices/virtio/vhost_user_output.c19
-rw-r--r--src/vnet/gso/gro_func.h13
-rw-r--r--src/vnet/gso/node.c10
-rw-r--r--src/vnet/interface_format.c28
-rw-r--r--src/vnet/interface_funcs.h1
-rw-r--r--src/vnet/interface_output.c19
-rw-r--r--src/vnet/interface_output.h31
-rw-r--r--src/vnet/ip/ip4_forward.c9
-rw-r--r--src/vnet/ip/ip4_inlines.h2
-rw-r--r--src/vnet/ip/ip6_forward.c62
-rw-r--r--src/vnet/ip/ip_frag.c2
-rw-r--r--src/vnet/ipsec/ipsec_output.c75
-rw-r--r--src/vnet/pg/cli.c4
-rw-r--r--src/vnet/pg/input.c31
-rw-r--r--src/vnet/pg/pg.h5
-rw-r--r--src/vnet/tcp/tcp_output.c2
-rw-r--r--src/vnet/udp/udp_inlines.h2
-rw-r--r--src/vnet/vxlan-gbp/encap.c26
-rw-r--r--src/vnet/vxlan/encap.c26
31 files changed, 371 insertions, 261 deletions
diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c
index c53fc42a850..5bcb68cc95c 100644
--- a/src/plugins/avf/output.c
+++ b/src/plugins/avf/output.c
@@ -57,12 +57,12 @@ static_always_inline u64
avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso)
{
u64 flags = 0;
- if (!is_tso && !(b->flags & ((VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+ if (!is_tso && !(b->flags & VNET_BUFFER_F_OFFLOAD))
return 0;
- u32 is_tcp = is_tso || b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
- u32 is_udp = !is_tso && b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+
+ u32 oflags = vnet_buffer2 (b)->oflags;
+ u32 is_tcp = is_tso || oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
+ u32 is_udp = !is_tso && oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
u32 is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
u32 is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
ASSERT (!is_tcp || !is_udp);
@@ -173,9 +173,7 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
{
u16 next = txq->next;
u64 bits = AVF_TXD_CMD_EOP | AVF_TXD_CMD_RSV;
- const u32 offload_mask = VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_GSO;
+ const u32 offload_mask = VNET_BUFFER_F_OFFLOAD | VNET_BUFFER_F_GSO;
u64 one_by_one_offload_flags = 0;
int is_tso;
u16 n_desc = 0;
diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index ec33f6a4461..6466b443633 100644
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -218,17 +218,20 @@ static_always_inline void
dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b,
struct rte_mbuf *mb)
{
- u32 ip_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
- u32 tcp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
- u32 udp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
u32 tso = b->flags & VNET_BUFFER_F_GSO, max_pkt_len;
+ u32 oflags, ip_cksum, tcp_cksum, udp_cksum;
u64 ol_flags;
/* Is there any work for us? */
- if (PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum | tso) == 0))
+ if (PREDICT_TRUE (((b->flags & VNET_BUFFER_F_OFFLOAD) | tso) == 0))
return;
+ oflags = vnet_buffer2 (b)->oflags;
+ ip_cksum = oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
+ tcp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
+ udp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
+
mb->l2_len = vnet_buffer (b)->l3_hdr_offset - b->current_data;
mb->l3_len = vnet_buffer (b)->l4_hdr_offset -
vnet_buffer (b)->l3_hdr_offset;
@@ -328,10 +331,7 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm,
}
if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
- (or_flags &
- (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
- | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
- | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+ (or_flags & VNET_BUFFER_F_OFFLOAD)))
{
dpdk_buffer_tx_offload (xd, b[0], mb[0]);
dpdk_buffer_tx_offload (xd, b[1], mb[1]);
@@ -388,10 +388,7 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm,
}
if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
- (or_flags &
- (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
- | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
- | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+ (or_flags & VNET_BUFFER_F_OFFLOAD)))
{
dpdk_buffer_tx_offload (xd, b[0], mb[0]);
dpdk_buffer_tx_offload (xd, b[1], mb[1]);
diff --git a/src/plugins/gtpu/gtpu_decap.c b/src/plugins/gtpu/gtpu_decap.c
index e3bc476ed6d..974ae8a8190 100644
--- a/src/plugins/gtpu/gtpu_decap.c
+++ b/src/plugins/gtpu/gtpu_decap.c
@@ -1257,13 +1257,15 @@ static char *gtpu_flow_error_strings[] = {
};
-#define gtpu_local_need_csum_check(_b) \
- (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED \
- || _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
-
-#define gtpu_local_csum_is_valid(_b) \
- ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT \
- || _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) != 0)
+#define gtpu_local_need_csum_check(_b) \
+ (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED || \
+ (_b->flags & VNET_BUFFER_F_OFFLOAD && \
+ vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
+
+#define gtpu_local_csum_is_valid(_b) \
+ ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT || \
+ (_b->flags & VNET_BUFFER_F_OFFLOAD && \
+ vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0)
static_always_inline u8
gtpu_validate_udp_csum (vlib_main_t * vm, vlib_buffer_t *b)
diff --git a/src/plugins/lisp/lisp-cp/packets.c b/src/plugins/lisp/lisp-cp/packets.c
index 8e2386e72bd..3f4292b4841 100644
--- a/src/plugins/lisp/lisp-cp/packets.c
+++ b/src/plugins/lisp/lisp-cp/packets.c
@@ -182,7 +182,7 @@ pkt_push_udp_and_ip (vlib_main_t * vm, vlib_buffer_t * b, u16 sp, u16 dp,
if (csum_offload)
{
ih = pkt_push_ip (vm, b, sip, dip, IP_PROTOCOL_UDP, 1);
- b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
vnet_buffer (b)->l3_hdr_offset = (u8 *) ih - b->data;
vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data;
uh->checksum = 0;
diff --git a/src/plugins/vmxnet3/input.c b/src/plugins/vmxnet3/input.c
index affc3691af1..be528b26494 100644
--- a/src/plugins/vmxnet3/input.c
+++ b/src/plugins/vmxnet3/input.c
@@ -79,6 +79,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
u16 gso_size)
{
u8 l4_hdr_sz = 0;
+ u32 oflags = 0;
if (rx_comp->flags & VMXNET3_RXCF_IP4)
{
@@ -90,15 +91,15 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) +
ip4_header_bytes (ip4);
hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
- VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
- VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4;
+ VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4;
/* checksum offload */
if (!(rx_comp->index & VMXNET3_RXCI_CNC))
{
if (!(rx_comp->flags & VMXNET3_RXCF_IPC))
{
- hb->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
ip4->checksum = 0;
}
if (!(rx_comp->flags & VMXNET3_RXCF_TUC))
@@ -108,7 +109,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
tcp_header_t *tcp =
(tcp_header_t *) (hb->data +
vnet_buffer (hb)->l4_hdr_offset);
- hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp->checksum = 0;
}
else if (rx_comp->flags & VMXNET3_RXCF_UDP)
@@ -116,7 +117,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
udp_header_t *udp =
(udp_header_t *) (hb->data +
vnet_buffer (hb)->l4_hdr_offset);
- hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp->checksum = 0;
}
}
@@ -148,8 +149,8 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) +
sizeof (ip6_header_t);
hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
- VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
- VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6;
+ VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6;
/* checksum offload */
if (!(rx_comp->index & VMXNET3_RXCI_CNC))
@@ -161,7 +162,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
tcp_header_t *tcp =
(tcp_header_t *) (hb->data +
vnet_buffer (hb)->l4_hdr_offset);
- hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp->checksum = 0;
}
else if (rx_comp->flags & VMXNET3_RXCF_UDP)
@@ -169,7 +170,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
udp_header_t *udp =
(udp_header_t *) (hb->data +
vnet_buffer (hb)->l4_hdr_offset);
- hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp->checksum = 0;
}
}
@@ -194,6 +195,8 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
hb->flags |= VNET_BUFFER_F_GSO;
}
}
+ if (oflags)
+ vnet_buffer_offload_flags_set (hb, oflags);
}
static_always_inline uword
diff --git a/src/plugins/wireguard/wireguard_input.c b/src/plugins/wireguard/wireguard_input.c
index d9843d90c7d..2690cf5e097 100755
--- a/src/plugins/wireguard/wireguard_input.c
+++ b/src/plugins/wireguard/wireguard_input.c
@@ -368,7 +368,8 @@ VLIB_NODE_FN (wg_input_node) (vlib_main_t * vm,
clib_memcpy (vlib_buffer_get_current (b[0]), decr_data, decr_len);
b[0]->current_length = decr_len;
- b[0]->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ vnet_buffer_offload_flags_clear (b[0],
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
wg_timers_any_authenticated_packet_received (peer);
wg_timers_any_authenticated_packet_traversal (peer);
diff --git a/src/vnet/buffer.c b/src/vnet/buffer.c
index 3cb6b948d2b..b41036d8124 100644
--- a/src/vnet/buffer.c
+++ b/src/vnet/buffer.c
@@ -16,6 +16,18 @@
#include <vlib/vlib.h>
#include <vnet/buffer.h>
+u8 *
+format_vnet_buffer_offload (u8 *s, va_list *args)
+{
+ vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
+
+#define _(bit, name, ss, v) \
+ if (v && (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name)) \
+ s = format (s, "%s ", ss);
+ foreach_vnet_buffer_offload_flag
+#undef _
+ return s;
+}
u8 *
format_vnet_buffer (u8 * s, va_list * args)
@@ -29,7 +41,10 @@ format_vnet_buffer (u8 * s, va_list * args)
a = format (a, "%s ", ss);
foreach_vnet_buffer_flag
#undef _
- if (b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID)
+ if (b->flags & VNET_BUFFER_F_OFFLOAD) a =
+ format (a, "%U ", format_vnet_buffer_offload, b);
+
+ if (b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID)
a = format (a, "l2-hdr-offset %d ", vnet_buffer (b)->l2_hdr_offset);
if (b->flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID)
@@ -38,6 +53,9 @@ format_vnet_buffer (u8 * s, va_list * args)
if (b->flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID)
a = format (a, "l4-hdr-offset %d ", vnet_buffer (b)->l4_hdr_offset);
+ if (b->flags & VNET_BUFFER_F_GSO)
+ a = format (a, "gso gso-size %d", vnet_buffer2 (b)->gso_size);
+
if (b->flags & VNET_BUFFER_F_QOS_DATA_VALID)
a = format (a, "qos %d.%d ",
vnet_buffer2 (b)->qos.bits, vnet_buffer2 (b)->qos.source);
diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h
index 27aeb3b42ea..fb734d5acde 100644
--- a/src/vnet/buffer.h
+++ b/src/vnet/buffer.h
@@ -46,34 +46,34 @@
* Flags that are set in the high order bits of ((vlib_buffer*)b)->flags
*
*/
-#define foreach_vnet_buffer_flag \
- _( 1, L4_CHECKSUM_COMPUTED, "l4-cksum-computed", 1) \
- _( 2, L4_CHECKSUM_CORRECT, "l4-cksum-correct", 1) \
- _( 3, VLAN_2_DEEP, "vlan-2-deep", 1) \
- _( 4, VLAN_1_DEEP, "vlan-1-deep", 1) \
- _( 5, SPAN_CLONE, "span-clone", 1) \
- _( 6, LOOP_COUNTER_VALID, "loop-counter-valid", 0) \
- _( 7, LOCALLY_ORIGINATED, "local", 1) \
- _( 8, IS_IP4, "ip4", 1) \
- _( 9, IS_IP6, "ip6", 1) \
- _(10, OFFLOAD_IP_CKSUM, "offload-ip-cksum", 1) \
- _(11, OFFLOAD_TCP_CKSUM, "offload-tcp-cksum", 1) \
- _(12, OFFLOAD_UDP_CKSUM, "offload-udp-cksum", 1) \
- _(13, IS_NATED, "natted", 1) \
- _(14, L2_HDR_OFFSET_VALID, "l2_hdr_offset_valid", 0) \
- _(15, L3_HDR_OFFSET_VALID, "l3_hdr_offset_valid", 0) \
- _(16, L4_HDR_OFFSET_VALID, "l4_hdr_offset_valid", 0) \
- _(17, FLOW_REPORT, "flow-report", 1) \
- _(18, IS_DVR, "dvr", 1) \
- _(19, QOS_DATA_VALID, "qos-data-valid", 0) \
- _(20, GSO, "gso", 0) \
- _(21, AVAIL1, "avail1", 1) \
- _(22, AVAIL2, "avail2", 1) \
- _(23, AVAIL3, "avail3", 1) \
- _(24, AVAIL4, "avail4", 1) \
- _(25, AVAIL5, "avail5", 1) \
- _(26, AVAIL6, "avail6", 1) \
- _(27, AVAIL7, "avail7", 1)
+#define foreach_vnet_buffer_flag \
+ _ (1, L4_CHECKSUM_COMPUTED, "l4-cksum-computed", 1) \
+ _ (2, L4_CHECKSUM_CORRECT, "l4-cksum-correct", 1) \
+ _ (3, VLAN_2_DEEP, "vlan-2-deep", 1) \
+ _ (4, VLAN_1_DEEP, "vlan-1-deep", 1) \
+ _ (5, SPAN_CLONE, "span-clone", 1) \
+ _ (6, LOOP_COUNTER_VALID, "loop-counter-valid", 0) \
+ _ (7, LOCALLY_ORIGINATED, "local", 1) \
+ _ (8, IS_IP4, "ip4", 1) \
+ _ (9, IS_IP6, "ip6", 1) \
+ _ (10, OFFLOAD, "offload", 0) \
+ _ (11, IS_NATED, "natted", 1) \
+ _ (12, L2_HDR_OFFSET_VALID, "l2_hdr_offset_valid", 0) \
+ _ (13, L3_HDR_OFFSET_VALID, "l3_hdr_offset_valid", 0) \
+ _ (14, L4_HDR_OFFSET_VALID, "l4_hdr_offset_valid", 0) \
+ _ (15, FLOW_REPORT, "flow-report", 1) \
+ _ (16, IS_DVR, "dvr", 1) \
+ _ (17, QOS_DATA_VALID, "qos-data-valid", 0) \
+ _ (18, GSO, "gso", 0) \
+ _ (19, AVAIL1, "avail1", 1) \
+ _ (20, AVAIL2, "avail2", 1) \
+ _ (21, AVAIL3, "avail3", 1) \
+ _ (22, AVAIL4, "avail4", 1) \
+ _ (23, AVAIL5, "avail5", 1) \
+ _ (24, AVAIL6, "avail6", 1) \
+ _ (25, AVAIL7, "avail7", 1) \
+ _ (26, AVAIL8, "avail8", 1) \
+ _ (27, AVAIL9, "avail9", 1)
/*
* Please allocate the FIRST available bit, redefine
@@ -81,10 +81,10 @@
* VNET_BUFFER_FLAGS_ALL_AVAIL definition.
*/
-#define VNET_BUFFER_FLAGS_ALL_AVAIL \
- (VNET_BUFFER_F_AVAIL1 | VNET_BUFFER_F_AVAIL2 | VNET_BUFFER_F_AVAIL3 | \
- VNET_BUFFER_F_AVAIL4 | VNET_BUFFER_F_AVAIL5 | VNET_BUFFER_F_AVAIL6 | \
- VNET_BUFFER_F_AVAIL7)
+#define VNET_BUFFER_FLAGS_ALL_AVAIL \
+ (VNET_BUFFER_F_AVAIL1 | VNET_BUFFER_F_AVAIL2 | VNET_BUFFER_F_AVAIL3 | \
+ VNET_BUFFER_F_AVAIL4 | VNET_BUFFER_F_AVAIL5 | VNET_BUFFER_F_AVAIL6 | \
+ VNET_BUFFER_F_AVAIL7 | VNET_BUFFER_F_AVAIL8 | VNET_BUFFER_F_AVAIL9)
#define VNET_BUFFER_FLAGS_VLAN_BITS \
(VNET_BUFFER_F_VLAN_1_DEEP | VNET_BUFFER_F_VLAN_2_DEEP)
@@ -420,6 +420,21 @@ STATIC_ASSERT (sizeof (vnet_buffer_opaque_t) <=
#define vnet_buffer(b) ((vnet_buffer_opaque_t *) (b)->opaque)
+#define foreach_vnet_buffer_offload_flag \
+ _ (0, IP_CKSUM, "offload-ip-cksum", 1) \
+ _ (1, TCP_CKSUM, "offload-tcp-cksum", 1) \
+ _ (2, UDP_CKSUM, "offload-udp-cksum", 1) \
+ _ (3, OUTER_IP_CKSUM, "offload-outer-ip-cksum", 1) \
+ _ (4, OUTER_TCP_CKSUM, "offload-outer-tcp-cksum", 1) \
+ _ (5, OUTER_UDP_CKSUM, "offload-outer-udp-cksum", 1)
+
+enum
+{
+#define _(bit, name, s, v) VNET_BUFFER_OFFLOAD_F_##name = (1 << bit),
+ foreach_vnet_buffer_offload_flag
+#undef _
+};
+
/* Full cache line (64 bytes) of additional space */
typedef struct
{
@@ -452,12 +467,15 @@ typedef struct
* in case the egress interface is not GSO-enabled - then we need to perform
* the segmentation, and use this value to cut the payload appropriately.
*/
- u16 gso_size;
- /* size of L4 prototol header */
- u16 gso_l4_hdr_sz;
+ struct
+ {
+ u16 gso_size;
+ /* size of L4 prototol header */
+ u16 gso_l4_hdr_sz;
- /* The union below has a u64 alignment, so this space is unused */
- u32 __unused2[1];
+ /* offload flags */
+ u32 oflags;
+ };
struct
{
@@ -501,6 +519,22 @@ STATIC_ASSERT (sizeof (vnet_buffer_opaque2_t) <=
format_function_t format_vnet_buffer;
+format_function_t format_vnet_buffer_offload;
+
+static_always_inline void
+vnet_buffer_offload_flags_set (vlib_buffer_t *b, u32 oflags)
+{
+ vnet_buffer2 (b)->oflags |= oflags;
+ b->flags |= VNET_BUFFER_F_OFFLOAD;
+}
+
+static_always_inline void
+vnet_buffer_offload_flags_clear (vlib_buffer_t *b, u32 oflags)
+{
+ vnet_buffer2 (b)->oflags &= ~oflags;
+ if (0 == vnet_buffer2 (b)->oflags)
+ b->flags &= ~VNET_BUFFER_F_OFFLOAD;
+}
#endif /* included_vnet_buffer_h */
diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c
index f4db6399a57..9f1782e2d13 100644
--- a/src/vnet/devices/af_packet/node.c
+++ b/src/vnet/devices/af_packet/node.c
@@ -120,6 +120,7 @@ static_always_inline void
mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
{
ethernet_header_t *eth = vlib_buffer_get_current (b);
+ u32 oflags = 0;
if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP4)
{
ip4_header_t *ip4 =
@@ -127,7 +128,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
b->flags |= VNET_BUFFER_F_IS_IP4;
if (ip4->protocol == IP_PROTOCOL_TCP)
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) +
ip4_header_bytes (ip4));
@@ -136,7 +137,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
}
else if (ip4->protocol == IP_PROTOCOL_UDP)
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) +
ip4_header_bytes (ip4));
@@ -146,6 +147,8 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t);
vnet_buffer (b)->l4_hdr_offset =
sizeof (ethernet_header_t) + ip4_header_bytes (ip4);
+ if (oflags)
+ vnet_buffer_offload_flags_set (b, oflags);
}
else if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP6)
{
@@ -165,7 +168,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
}
if (ip6->protocol == IP_PROTOCOL_TCP)
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp =
(tcp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) + ip6_hdr_len);
@@ -174,7 +177,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
}
else if (ip6->protocol == IP_PROTOCOL_UDP)
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp =
(udp_header_t *) (vlib_buffer_get_current (b) +
sizeof (ethernet_header_t) + ip6_hdr_len);
@@ -184,6 +187,8 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t);
vnet_buffer (b)->l4_hdr_offset =
sizeof (ethernet_header_t) + ip6_hdr_len;
+ if (oflags)
+ vnet_buffer_offload_flags_set (b, oflags);
}
}
diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c
index 3482336b2ad..fb996d551fa 100644
--- a/src/vnet/devices/virtio/device.c
+++ b/src/vnet/devices/virtio/device.c
@@ -282,6 +282,8 @@ static_always_inline void
set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
const int is_l2)
{
+ u32 oflags = vnet_buffer2 (b)->oflags;
+
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
ip4_header_t *ip4;
@@ -290,11 +292,11 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
0 /* ip6 */ );
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset; // 0x22;
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
}
@@ -305,7 +307,7 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
*/
ip4 =
(ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
}
else if (b->flags & VNET_BUFFER_F_IS_IP6)
@@ -315,11 +317,11 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
1 /* ip6 */ );
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset; // 0x36;
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
}
@@ -330,6 +332,8 @@ static_always_inline void
set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
const int is_l2)
{
+ u32 oflags = vnet_buffer2 (b)->oflags;
+
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
ip4_header_t *ip4;
@@ -348,7 +352,7 @@ set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
* virtio devices do not support IP4 checksum offload. So driver takes care
* of it while doing tx.
*/
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
}
else if (b->flags & VNET_BUFFER_F_IS_IP6)
@@ -392,8 +396,7 @@ add_buffer_to_slot (vlib_main_t * vm, vlib_node_runtime_t * node,
goto done;
}
}
- else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ else if (b->flags & VNET_BUFFER_F_OFFLOAD)
{
if (csum_offload)
set_checksum_offsets (b, hdr, is_l2);
@@ -584,8 +587,7 @@ add_buffer_to_slot_packed (vlib_main_t * vm, vlib_node_runtime_t * node,
goto done;
}
}
- else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ else if (b->flags & VNET_BUFFER_F_OFFLOAD)
{
if (csum_offload)
set_checksum_offsets (b, hdr, is_l2);
diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c
index be2405bb3b9..98df322ea15 100644
--- a/src/vnet/devices/virtio/node.c
+++ b/src/vnet/devices/virtio/node.c
@@ -235,6 +235,7 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
{
u16 ethertype = 0, l2hdr_sz = 0;
+ u32 oflags = 0;
if (type == VIRTIO_IF_TYPE_TUN)
{
@@ -280,11 +281,10 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
(ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
*l4_proto = ip4->protocol;
+ oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
b0->flags |=
- (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
- b0->flags |=
- (VNET_BUFFER_F_L2_HDR_OFFSET_VALID
- | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
@@ -301,7 +301,7 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
}
if (*l4_proto == IP_PROTOCOL_TCP)
{
- b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b0) +
vnet_buffer
(b0)->l4_hdr_offset);
@@ -309,12 +309,14 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
}
else if (*l4_proto == IP_PROTOCOL_UDP)
{
- b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b0) +
vnet_buffer
(b0)->l4_hdr_offset);
*l4_hdr_sz = sizeof (*udp);
}
+ if (oflags)
+ vnet_buffer_offload_flags_set (b0, oflags);
}
}
diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c
index 62b59f69ba9..739125bbcfe 100644
--- a/src/vnet/devices/virtio/vhost_user_input.c
+++ b/src/vnet/devices/virtio/vhost_user_input.c
@@ -253,6 +253,7 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
ethernet_header_t *eh = (ethernet_header_t *) b0_data;
u16 ethertype = clib_net_to_host_u16 (eh->type);
u16 l2hdr_sz = sizeof (ethernet_header_t);
+ u32 oflags = 0;
if (ethernet_frame_is_tagged (ethertype))
{
@@ -278,7 +279,8 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
{
ip4_header_t *ip4 = (ip4_header_t *) (b0_data + l2hdr_sz);
l4_proto = ip4->protocol;
- b0->flags |= VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ b0->flags |= VNET_BUFFER_F_IS_IP4;
+ oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
{
@@ -292,12 +294,12 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
tcp_header_t *tcp = (tcp_header_t *)
(b0_data + vnet_buffer (b0)->l4_hdr_offset);
l4_hdr_sz = tcp_header_bytes (tcp);
- b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
}
else if (l4_proto == IP_PROTOCOL_UDP)
{
l4_hdr_sz = sizeof (udp_header_t);
- b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
}
if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP)
@@ -318,6 +320,9 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
}
+
+ if (oflags)
+ vnet_buffer_offload_flags_set (b0, oflags);
}
static_always_inline void
diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c
index 465c0ea0903..8dbddea46f5 100644
--- a/src/vnet/devices/virtio/vhost_user_output.c
+++ b/src/vnet/devices/virtio/vhost_user_output.c
@@ -225,10 +225,11 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
generic_header_offset_t gho = { 0 };
int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
+ u32 oflags = vnet_buffer2 (b)->oflags;
ASSERT (!(is_ip4 && is_ip6));
vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6);
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
{
ip4_header_t *ip4;
@@ -238,13 +239,13 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
}
/* checksum offload */
- if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset;
hdr->csum_offset = offsetof (udp_header_t, checksum);
}
- else if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
hdr->csum_start = gho.l4_hdr_offset;
@@ -254,7 +255,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
/* GSO offload */
if (b->flags & VNET_BUFFER_F_GSO)
{
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
if (is_ip4 &&
(vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)))
@@ -270,7 +271,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
}
}
else if ((vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)) &&
- (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
{
hdr->gso_size = vnet_buffer2 (b)->gso_size;
hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
@@ -477,9 +478,7 @@ retry:
hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
hdr->num_buffers = 1;
- or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+ or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
/* Guest supports csum offload and buffer requires checksum offload? */
if (or_flags &&
@@ -813,9 +812,7 @@ retry:
hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
hdr->num_buffers = 1; //This is local, no need to check
- or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+ or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
/* Guest supports csum offload and buffer requires checksum offload? */
if (or_flags
diff --git a/src/vnet/gso/gro_func.h b/src/vnet/gso/gro_func.h
index b821a034e71..239009d680b 100644
--- a/src/vnet/gso/gro_func.h
+++ b/src/vnet/gso/gro_func.h
@@ -147,7 +147,7 @@ gro_validate_checksum (vlib_main_t * vm, vlib_buffer_t * b0,
{
u32 flags = 0;
- if (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (b0->flags & VNET_BUFFER_F_OFFLOAD)
return VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
vlib_buffer_advance (b0, gho0->l3_hdr_offset);
if (is_ip4)
@@ -353,9 +353,9 @@ gro_fixup_header (vlib_main_t * vm, vlib_buffer_t * b0, u32 ack_number,
ip4->length =
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
gho0.l3_hdr_offset);
- b0->flags |=
- (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4 |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
+ b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4);
+ vnet_buffer_offload_flags_set (b0, (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
}
else if (gho0.gho_flags & GHO_F_IP6)
{
@@ -364,9 +364,8 @@ gro_fixup_header (vlib_main_t * vm, vlib_buffer_t * b0, u32 ack_number,
ip6->payload_length =
clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
gho0.l4_hdr_offset);
- b0->flags |=
- (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6 |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+ b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
+ vnet_buffer_offload_flags_set (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
}
tcp_header_t *tcp0 =
diff --git a/src/vnet/gso/node.c b/src/vnet/gso/node.c
index d0c5ca054e6..ecbdc188ec8 100644
--- a/src/vnet/gso/node.c
+++ b/src/vnet/gso/node.c
@@ -143,7 +143,8 @@ tso_segment_vxlan_tunnel_headers_fixup (vlib_main_t * vm, vlib_buffer_t * b,
{
udp->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
}
- b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ /* FIXME: it should be OUTER_UDP_CKSUM */
+ vnet_buffer_offload_flags_clear (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
}
}
@@ -281,7 +282,8 @@ tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags,
tcp->checksum = 0;
tcp->checksum =
ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6, &bogus);
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ vnet_buffer_offload_flags_clear (b0,
+ VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
}
}
else
@@ -295,8 +297,8 @@ tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags,
tcp->checksum = 0;
tcp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip4);
}
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
}
if (!is_l2 && ((gho->gho_flags & GHO_F_TUNNEL) == 0))
diff --git a/src/vnet/interface_format.c b/src/vnet/interface_format.c
index dc2edff7c67..036ca53174c 100644
--- a/src/vnet/interface_format.c
+++ b/src/vnet/interface_format.c
@@ -722,7 +722,7 @@ unformat_vnet_buffer_flags (unformat_input_t * input, va_list * args)
while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
{
/* Red herring, there is no such buffer flag */
- if (unformat (input, "avail8"))
+ if (unformat (input, "avail10"))
return 0;
#define _(bit,enum,str,verbose) \
else if (unformat (input, str)) \
@@ -741,6 +741,32 @@ unformat_vnet_buffer_flags (unformat_input_t * input, va_list * args)
}
uword
+unformat_vnet_buffer_offload_flags (unformat_input_t *input, va_list *args)
+{
+ u32 *flagp = va_arg (*args, u32 *);
+ int rv = 0;
+ u32 oflags = 0;
+
+ while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+ {
+ if (0)
+ ;
+#define _(bit, enum, str, verbose) \
+ else if (unformat (input, str)) \
+ { \
+ oflags |= (1 << bit); \
+ rv = 1; \
+ }
+ foreach_vnet_buffer_offload_flag
+#undef _
+ else break;
+ }
+ if (rv)
+ *flagp = oflags;
+ return rv;
+}
+
+uword
unformat_vnet_hw_interface (unformat_input_t * input, va_list * args)
{
vnet_main_t *vnm = va_arg (*args, vnet_main_t *);
diff --git a/src/vnet/interface_funcs.h b/src/vnet/interface_funcs.h
index dbb0549314a..186e0327884 100644
--- a/src/vnet/interface_funcs.h
+++ b/src/vnet/interface_funcs.h
@@ -457,6 +457,7 @@ format_function_t format_vnet_sw_interface_flags;
unformat_function_t unformat_vnet_sw_interface;
unformat_function_t unformat_vnet_hw_interface;
unformat_function_t unformat_vnet_buffer_flags;
+unformat_function_t unformat_vnet_buffer_offload_flags;
/* Parses interface flags (up, down, enable, disable, etc.) */
unformat_function_t unformat_vnet_hw_interface_flags;
diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c
index 3363f76b722..ff315f6abe0 100644
--- a/src/vnet/interface_output.c
+++ b/src/vnet/interface_output.c
@@ -337,28 +337,24 @@ vnet_interface_output_node_inline (vlib_main_t * vm,
if (do_tx_offloads)
{
- u32 vnet_buffer_offload_flags =
- (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
- if (or_flags & vnet_buffer_offload_flags)
+ if (or_flags & VNET_BUFFER_F_OFFLOAD)
{
- if (b[0]->flags & vnet_buffer_offload_flags)
+ if (b[0]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[0],
b[0]->flags & VNET_BUFFER_F_IS_IP4,
b[0]->flags & VNET_BUFFER_F_IS_IP6);
- if (b[1]->flags & vnet_buffer_offload_flags)
+ if (b[1]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[1],
b[1]->flags & VNET_BUFFER_F_IS_IP4,
b[1]->flags & VNET_BUFFER_F_IS_IP6);
- if (b[2]->flags & vnet_buffer_offload_flags)
+ if (b[2]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[2],
b[2]->flags & VNET_BUFFER_F_IS_IP4,
b[2]->flags & VNET_BUFFER_F_IS_IP6);
- if (b[3]->flags & vnet_buffer_offload_flags)
+ if (b[3]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[3],
b[3]->flags & VNET_BUFFER_F_IS_IP4,
@@ -406,10 +402,7 @@ vnet_interface_output_node_inline (vlib_main_t * vm,
if (do_tx_offloads)
{
- if (b[0]->flags &
- (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_IP_CKSUM))
+ if (b[0]->flags & VNET_BUFFER_F_OFFLOAD)
vnet_calc_checksums_inline
(vm, b[0],
b[0]->flags & VNET_BUFFER_F_IS_IP4,
diff --git a/src/vnet/interface_output.h b/src/vnet/interface_output.h
index 4b082b314ad..27aadc6c5fd 100644
--- a/src/vnet/interface_output.h
+++ b/src/vnet/interface_output.h
@@ -44,18 +44,17 @@
#include <vnet/gso/hdr_offset_parser.h>
static_always_inline void
-vnet_calc_ip4_checksums (vlib_main_t * vm, vlib_buffer_t * b,
- ip4_header_t * ip4, tcp_header_t * th,
- udp_header_t * uh)
+vnet_calc_ip4_checksums (vlib_main_t *vm, vlib_buffer_t *b, ip4_header_t *ip4,
+ tcp_header_t *th, udp_header_t *uh, u32 oflags)
{
- if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
ip4->checksum = ip4_header_checksum (ip4);
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
th->checksum = 0;
th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
}
- if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
uh->checksum = 0;
uh->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
@@ -63,17 +62,16 @@ vnet_calc_ip4_checksums (vlib_main_t * vm, vlib_buffer_t * b,
}
static_always_inline void
-vnet_calc_ip6_checksums (vlib_main_t * vm, vlib_buffer_t * b,
- ip6_header_t * ip6, tcp_header_t * th,
- udp_header_t * uh)
+vnet_calc_ip6_checksums (vlib_main_t *vm, vlib_buffer_t *b, ip6_header_t *ip6,
+ tcp_header_t *th, udp_header_t *uh, u32 oflags)
{
int bogus;
- if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
{
th->checksum = 0;
th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
}
- if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+ if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
{
uh->checksum = 0;
uh->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
@@ -88,6 +86,7 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b,
ip6_header_t *ip6;
tcp_header_t *th;
udp_header_t *uh;
+ u32 oflags = vnet_buffer2 (b)->oflags;
ASSERT (!(is_ip4 && is_ip6));
@@ -98,16 +97,16 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b,
if (is_ip4)
{
- vnet_calc_ip4_checksums (vm, b, ip4, th, uh);
+ vnet_calc_ip4_checksums (vm, b, ip4, th, uh, oflags);
}
else if (is_ip6)
{
- vnet_calc_ip6_checksums (vm, b, ip6, th, uh);
+ vnet_calc_ip6_checksums (vm, b, ip6, th, uh, oflags);
}
- b->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
- b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
- b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ vnet_buffer_offload_flags_clear (b, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
}
#endif
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index 332c483aa9d..9e8d164b51f 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -1417,9 +1417,10 @@ ip4_local_l4_csum_validate (vlib_main_t * vm, vlib_buffer_t * p,
}
}
-#define ip4_local_csum_is_offloaded(_b) \
- _b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM \
- || _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM
+#define ip4_local_csum_is_offloaded(_b) \
+ ((_b->flags & VNET_BUFFER_F_OFFLOAD) && \
+ (vnet_buffer2 (_b)->oflags & \
+ (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
#define ip4_local_need_csum_check(is_tcp_udp, _b) \
(is_tcp_udp && !(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED \
@@ -2066,7 +2067,7 @@ ip4_ttl_and_checksum_check (vlib_buffer_t * b, ip4_header_t * ip, u16 * next,
/* Verify checksum. */
ASSERT (ip4_header_checksum_is_valid (ip) ||
- (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
+ (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
}
diff --git a/src/vnet/ip/ip4_inlines.h b/src/vnet/ip/ip4_inlines.h
index 3075fbf42a1..00a47125b8a 100644
--- a/src/vnet/ip/ip4_inlines.h
+++ b/src/vnet/ip/ip4_inlines.h
@@ -125,7 +125,7 @@ vlib_buffer_push_ip4_custom (vlib_main_t * vm, vlib_buffer_t * b,
if (csum_offload)
{
ih->checksum = 0;
- b->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_IP_CKSUM);
}
else
ih->checksum = ip4_header_checksum (ih);
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index bf6aa7f3212..0c1bc3b33ec 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -1333,15 +1333,23 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
flags[0] = b[0]->flags;
flags[1] = b[1]->flags;
+ u32 oflags[2];
+ oflags[0] = vnet_buffer2 (b[0])->oflags;
+ oflags[1] = vnet_buffer2 (b[1])->oflags;
+
+ u32 l4_offload[2];
+ l4_offload[0] = (flags[0] & VNET_BUFFER_F_OFFLOAD) &&
+ (oflags[0] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
+ l4_offload[1] = (flags[1] & VNET_BUFFER_F_OFFLOAD) &&
+ (oflags[1] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
+
u32 good_l4_csum[2];
good_l4_csum[0] =
- flags[0] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
+ (flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[0];
good_l4_csum[1] =
- flags[1] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
+ (flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[1];
u32 udp_offset[2] = { };
u8 is_tcp_udp[2];
@@ -1515,11 +1523,15 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
u8 type = lm->builtin_protocol_by_ip_protocol[ip->protocol];
u32 flags = b[0]->flags;
- u32 good_l4_csum =
- flags & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
+ u32 oflags = vnet_buffer2 (b[0])->oflags;
+
+ u32 l4_offload = (flags & VNET_BUFFER_F_OFFLOAD) &&
+ (oflags & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
+
+ u32 good_l4_csum =
+ (flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload;
u32 udp_offset;
i16 len_diff = 0;
u8 is_tcp_udp = ip6_next_proto_is_tcp_udp (b[0], ip, &udp_offset);
@@ -1539,8 +1551,9 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UNKNOWN;
len_diff = type == IP_BUILTIN_PROTOCOL_UDP ? len_diff : 0;
- u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN && !good_l4_csum
- && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
+ u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN &&
+ !good_l4_csum &&
+ !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
if (PREDICT_FALSE (need_csum))
{
flags = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
@@ -1556,7 +1569,6 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
error = len_diff < 0 ? IP6_ERROR_UDP_LENGTH : error;
-
STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
IP6_ERROR_UDP_CHECKSUM,
"Wrong IP6 errors constants");
@@ -1882,8 +1894,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
if (p1->flags & VNET_BUFFER_F_GSO)
ip1_len = gso_mtu_sz (p1);
-
-
ip6_mtu_check (p0, ip0_len,
adj0[0].rewrite_header.max_l3_packet_bytes,
is_locally_originated0, &next0, is_midchain,
@@ -1892,18 +1902,15 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
adj1[0].rewrite_header.max_l3_packet_bytes,
is_locally_originated1, &next1, is_midchain,
&error1);
-
/* Don't adjust the buffer for hop count issue; icmp-error node
* wants to see the IP header */
if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
{
p0->current_data -= rw_len0;
p0->current_length += rw_len0;
-
tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
next0 = adj0[0].rewrite_header.next_index;
-
if (PREDICT_FALSE
(adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
vnet_feature_arc_start_w_cfg_index
@@ -2070,7 +2077,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
adj0[0].rewrite_header.max_l3_packet_bytes,
is_locally_originated0, &next0, is_midchain,
&error0);
-
/* Don't adjust the buffer for hop count issue; icmp-error node
* wants to see the IP header */
if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
@@ -2289,7 +2295,6 @@ format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
ip6_hop_by_hop_option_t *opt0, *limit0;
ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
u8 type0;
-
s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d",
hbh0->protocol, (hbh0->length + 1) << 3, total_len);
@@ -2312,9 +2317,8 @@ format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
}
else
{
- s =
- format (s, "\n unrecognized option %d length %d", type0,
- opt0->length);
+ s = format (s, "\n unrecognized option %d length %d", type0,
+ opt0->length);
}
opt0 =
(ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
@@ -2362,9 +2366,8 @@ format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
}
else
{
- s =
- format (s, "\n unrecognized option %d length %d", type0,
- opt0->length);
+ s = format (s, "\n unrecognized option %d length %d", type0,
+ opt0->length);
}
opt0 =
(ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
@@ -3012,8 +3015,7 @@ VLIB_CLI_COMMAND (show_ip6_local, static) =
#ifndef CLIB_MARCH_VARIANT
int
-vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
- u32 table_index)
+vnet_set_ip6_classify_intfc (vlib_main_t *vm, u32 sw_if_index, u32 table_index)
{
vnet_main_t *vnm = vnet_get_main ();
vnet_interface_main_t *im = &vnm->interface_main;
@@ -3044,17 +3046,13 @@ vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
sw_if_index);
-
-
if (table_index != (u32) ~ 0)
{
dpo_id_t dpo = DPO_INVALID;
-
dpo_set (&dpo,
DPO_CLASSIFY,
DPO_PROTO_IP6,
classify_dpo_create (DPO_PROTO_IP6, table_index));
-
fib_table_entry_special_dpo_add (fib_index,
&pfx,
FIB_SOURCE_CLASSIFY,
diff --git a/src/vnet/ip/ip_frag.c b/src/vnet/ip/ip_frag.c
index 9aa8777514f..33fbb9b2a9d 100644
--- a/src/vnet/ip/ip_frag.c
+++ b/src/vnet/ip/ip_frag.c
@@ -226,7 +226,7 @@ ip4_frag_do_fragment (vlib_main_t * vm, u32 from_bi, u16 mtu,
to_ip4->checksum = ip4_header_checksum (to_ip4);
/* we've just done the IP checksum .. */
- to_b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+ vnet_buffer_offload_flags_clear (to_b, VNET_BUFFER_OFFLOAD_F_IP_CKSUM);
rem -= len;
fo += len;
diff --git a/src/vnet/ipsec/ipsec_output.c b/src/vnet/ipsec/ipsec_output.c
index d09a027de29..e6cf837d796 100644
--- a/src/vnet/ipsec/ipsec_output.c
+++ b/src/vnet/ipsec/ipsec_output.c
@@ -309,45 +309,50 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
next_node_index = im->ah4_encrypt_node_index;
vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
- if (is_ipv6)
+ if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_OFFLOAD))
{
- if (PREDICT_FALSE
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM))
- {
- tcp0->checksum =
- ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0,
- &bogus);
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
- }
- if (PREDICT_FALSE
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
- {
- udp0->checksum =
- ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0,
- &bogus);
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
- }
- }
- else
- {
- if (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
- {
- ip0->checksum = ip4_header_checksum (ip0);
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
- }
- if (PREDICT_FALSE
- (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM))
+ u32 oflags = vnet_buffer2 (b0)->oflags;
+
+ /*
+ * Clearing offload flags before checksum is computed
+ * It guarantees the cache hit!
+ */
+ vnet_buffer_offload_flags_clear (b0, oflags);
+
+ if (is_ipv6)
{
- tcp0->checksum =
- ip4_tcp_udp_compute_checksum (vm, b0, ip0);
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ if (PREDICT_FALSE (oflags &
+ VNET_BUFFER_OFFLOAD_F_TCP_CKSUM))
+ {
+ tcp0->checksum = ip6_tcp_udp_icmp_compute_checksum (
+ vm, b0, ip6_0, &bogus);
+ }
+ if (PREDICT_FALSE (oflags &
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
+ {
+ udp0->checksum = ip6_tcp_udp_icmp_compute_checksum (
+ vm, b0, ip6_0, &bogus);
+ }
}
- if (PREDICT_FALSE
- (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+ else
{
- udp0->checksum =
- ip4_tcp_udp_compute_checksum (vm, b0, ip0);
- b0->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ if (PREDICT_FALSE (oflags &
+ VNET_BUFFER_OFFLOAD_F_IP_CKSUM))
+ {
+ ip0->checksum = ip4_header_checksum (ip0);
+ }
+ if (PREDICT_FALSE (oflags &
+ VNET_BUFFER_OFFLOAD_F_TCP_CKSUM))
+ {
+ tcp0->checksum =
+ ip4_tcp_udp_compute_checksum (vm, b0, ip0);
+ }
+ if (PREDICT_FALSE (oflags &
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
+ {
+ udp0->checksum =
+ ip4_tcp_udp_compute_checksum (vm, b0, ip0);
+ }
}
}
vlib_buffer_advance (b0, iph_offset);
diff --git a/src/vnet/pg/cli.c b/src/vnet/pg/cli.c
index 625198634f5..3ef7a959f20 100644
--- a/src/vnet/pg/cli.c
+++ b/src/vnet/pg/cli.c
@@ -379,7 +379,9 @@ new_stream (vlib_main_t * vm,
else if (unformat (input, "buffer-flags %U",
unformat_vnet_buffer_flags, &s.buffer_flags))
;
-
+ else if (unformat (input, "buffer-offload-flags %U",
+ unformat_vnet_buffer_offload_flags, &s.buffer_oflags))
+ ;
else if (unformat (input, "node %U",
unformat_vlib_node, vm, &s.node_index))
;
diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c
index 940bf124c56..d3501b3e3b6 100644
--- a/src/vnet/pg/input.c
+++ b/src/vnet/pg/input.c
@@ -1537,13 +1537,14 @@ pg_input_trace (pg_main_t * pg,
}
static_always_inline void
-fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
- int gso_enabled, u32 gso_size)
+fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers,
+ u32 buffer_oflags, int gso_enabled, u32 gso_size)
{
for (int i = 0; i < n_buffers; i++)
{
vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[i]);
u8 l4_proto = 0;
+ u32 oflags = 0;
ethernet_header_t *eh =
(ethernet_header_t *) vlib_buffer_get_current (b0);
@@ -1574,10 +1575,11 @@ fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
l4_proto = ip4->protocol;
b0->flags |=
- (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
- b0->flags |= (VNET_BUFFER_F_L2_HDR_OFFSET_VALID
- | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
- VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
+ (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
+ if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
+ oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
}
else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
{
@@ -1594,7 +1596,8 @@ fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
if (l4_proto == IP_PROTOCOL_TCP)
{
- b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
+ oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
/* only set GSO flag for chained buffers */
if (gso_enabled && (b0->flags & VLIB_BUFFER_NEXT_PRESENT))
@@ -1609,8 +1612,12 @@ fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
}
else if (l4_proto == IP_PROTOCOL_UDP)
{
- b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
+ oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
}
+
+ if (oflags)
+ vnet_buffer_offload_flags_set (b0, oflags);
}
}
@@ -1710,13 +1717,11 @@ pg_generate_packets (vlib_node_runtime_t * node,
vnet_buffer (b)->feature_arc_index = feature_arc_index;
}
- if (pi->gso_enabled ||
- (s->buffer_flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_IP_CKSUM)))
+ if (pi->gso_enabled || (s->buffer_flags & VNET_BUFFER_F_OFFLOAD))
{
fill_buffer_offload_flags (vm, to_next, n_this_frame,
- pi->gso_enabled, pi->gso_size);
+ s->buffer_oflags, pi->gso_enabled,
+ pi->gso_size);
}
n_trace = vlib_get_trace_count (vm, node);
diff --git a/src/vnet/pg/pg.h b/src/vnet/pg/pg.h
index da2c2b54347..da5af25b22e 100644
--- a/src/vnet/pg/pg.h
+++ b/src/vnet/pg/pg.h
@@ -125,9 +125,12 @@ typedef struct pg_stream_t
for max_packet_bytes. */
u32 buffer_bytes;
- /* Buffer flags to set in each packet e.g. checksum offload flags */
+ /* Buffer flags to set in each packet e.g. l2 valid flags */
u32 buffer_flags;
+ /* Buffer offload flags to set in each packet e.g. checksum offload flags */
+ u32 buffer_oflags;
+
/* Last packet length if packet size edit type is increment. */
u32 last_increment_packet_size;
diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index e2c2b231985..235a59be0b3 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -430,7 +430,7 @@ tcp_compute_checksum (tcp_connection_t * tc, vlib_buffer_t * b)
}
else
{
- b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
}
return checksum;
}
diff --git a/src/vnet/udp/udp_inlines.h b/src/vnet/udp/udp_inlines.h
index ec800f9a413..067289c9979 100644
--- a/src/vnet/udp/udp_inlines.h
+++ b/src/vnet/udp/udp_inlines.h
@@ -35,7 +35,7 @@ vlib_buffer_push_udp (vlib_buffer_t * b, u16 sp, u16 dp, u8 offload_csum)
uh->checksum = 0;
uh->length = clib_host_to_net_u16 (udp_len);
if (offload_csum)
- b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+ vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data;
b->flags |= VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
return uh;
diff --git a/src/vnet/vxlan-gbp/encap.c b/src/vnet/vxlan-gbp/encap.c
index 4514fdae7a7..2a4e8a8e312 100644
--- a/src/vnet/vxlan-gbp/encap.c
+++ b/src/vnet/vxlan-gbp/encap.c
@@ -102,14 +102,14 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
u8 const underlay_hdr_len = is_ip4 ?
sizeof (ip4_vxlan_gbp_header_t) : sizeof (ip6_vxlan_gbp_header_t);
u16 const l3_len = is_ip4 ? sizeof (ip4_header_t) : sizeof (ip6_header_t);
- u32 const csum_flags = is_ip4 ? VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
- VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID :
- VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
- u32 const inner_packet_csum_offload_flags =
- VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ u32 const csum_flags =
+ is_ip4 ? VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID :
+ VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
+ u32 const outer_packet_csum_offload_flags =
+ is_ip4 ? VNET_BUFFER_OFFLOAD_F_IP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM :
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
u32 const inner_packet_removed_flags =
VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_IS_IP6 |
VNET_BUFFER_F_L2_HDR_OFFSET_VALID | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
@@ -147,7 +147,7 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
n_left_from -= 2;
u32 or_flags = b[0]->flags | b[1]->flags;
- if (csum_offload && (or_flags & inner_packet_csum_offload_flags))
+ if (csum_offload && (or_flags & VNET_BUFFER_F_OFFLOAD))
{
/* Only calculate the non-GSO packet csum offload */
if ((b[0]->flags & VNET_BUFFER_F_GSO) == 0)
@@ -300,9 +300,13 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
b[0]->flags |= csum_flags;
vnet_buffer (b[0])->l3_hdr_offset = l3_0 - b[0]->data;
vnet_buffer (b[0])->l4_hdr_offset = (u8 *) udp0 - b[0]->data;
+ vnet_buffer_offload_flags_set (b[0],
+ outer_packet_csum_offload_flags);
b[1]->flags |= csum_flags;
vnet_buffer (b[1])->l3_hdr_offset = l3_1 - b[1]->data;
vnet_buffer (b[1])->l4_hdr_offset = (u8 *) udp1 - b[1]->data;
+ vnet_buffer_offload_flags_set (b[1],
+ outer_packet_csum_offload_flags);
}
/* IPv4 UDP checksum only if checksum offload is used */
else if (is_ip4)
@@ -387,7 +391,7 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
n_left_from -= 1;
n_left_to_next -= 1;
- if (csum_offload && (b[0]->flags & inner_packet_csum_offload_flags))
+ if (csum_offload && (b[0]->flags & VNET_BUFFER_F_OFFLOAD))
{
/* Only calculate the non-GSO packet csum offload */
if ((b[0]->flags & VNET_BUFFER_F_GSO) == 0)
@@ -477,6 +481,8 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
b[0]->flags |= csum_flags;
vnet_buffer (b[0])->l3_hdr_offset = l3_0 - b[0]->data;
vnet_buffer (b[0])->l4_hdr_offset = (u8 *) udp0 - b[0]->data;
+ vnet_buffer_offload_flags_set (b[0],
+ outer_packet_csum_offload_flags);
}
/* IPv4 UDP checksum only if checksum offload is used */
else if (is_ip4)
diff --git a/src/vnet/vxlan/encap.c b/src/vnet/vxlan/encap.c
index 787758a94f8..476e0f2b3b7 100644
--- a/src/vnet/vxlan/encap.c
+++ b/src/vnet/vxlan/encap.c
@@ -98,14 +98,14 @@ vxlan_encap_inline (vlib_main_t * vm,
u8 const underlay_hdr_len = is_ip4 ?
sizeof (ip4_vxlan_header_t) : sizeof (ip6_vxlan_header_t);
u16 const l3_len = is_ip4 ? sizeof (ip4_header_t) : sizeof (ip6_header_t);
- u32 const csum_flags = is_ip4 ? VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
- VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID :
- VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
- u32 const inner_packet_csum_offload_flags =
- VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
- VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ u32 const csum_flags =
+ is_ip4 ? VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID :
+ VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+ VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
+ u32 const outer_packet_csum_offload_flags =
+ is_ip4 ? VNET_BUFFER_OFFLOAD_F_IP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM :
+ VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
u32 const inner_packet_removed_flags =
VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_IS_IP6 |
VNET_BUFFER_F_L2_HDR_OFFSET_VALID | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
@@ -144,7 +144,7 @@ vxlan_encap_inline (vlib_main_t * vm,
b += 2;
u32 or_flags = b0->flags | b1->flags;
- if (csum_offload && (or_flags & inner_packet_csum_offload_flags))
+ if (csum_offload && (or_flags & VNET_BUFFER_F_OFFLOAD))
{
/* Only calculate the non-GSO packet csum offload */
if ((b0->flags & VNET_BUFFER_F_GSO) == 0)
@@ -284,9 +284,13 @@ vxlan_encap_inline (vlib_main_t * vm,
b0->flags |= csum_flags;
vnet_buffer (b0)->l3_hdr_offset = l3_0 - b0->data;
vnet_buffer (b0)->l4_hdr_offset = (u8 *) udp0 - b0->data;
+ vnet_buffer_offload_flags_set (b0,
+ outer_packet_csum_offload_flags);
b1->flags |= csum_flags;
vnet_buffer (b1)->l3_hdr_offset = l3_1 - b1->data;
vnet_buffer (b1)->l4_hdr_offset = (u8 *) udp1 - b1->data;
+ vnet_buffer_offload_flags_set (b1,
+ outer_packet_csum_offload_flags);
}
/* IPv4 UDP checksum only if checksum offload is used */
else if (is_ip4)
@@ -377,7 +381,7 @@ vxlan_encap_inline (vlib_main_t * vm,
vlib_buffer_t *b0 = b[0];
b += 1;
- if (csum_offload && (b0->flags & inner_packet_csum_offload_flags))
+ if (csum_offload && (b0->flags & VNET_BUFFER_F_OFFLOAD))
{
/* Only calculate the non-GSO packet csum offload */
if ((b0->flags & VNET_BUFFER_F_GSO) == 0)
@@ -459,6 +463,8 @@ vxlan_encap_inline (vlib_main_t * vm,
b0->flags |= csum_flags;
vnet_buffer (b0)->l3_hdr_offset = l3_0 - b0->data;
vnet_buffer (b0)->l4_hdr_offset = (u8 *) udp0 - b0->data;
+ vnet_buffer_offload_flags_set (b0,
+ outer_packet_csum_offload_flags);
}
/* IPv4 UDP checksum only if checksum offload is used */
else if (is_ip4)