diff options
34 files changed, 377 insertions, 264 deletions
diff --git a/extras/pg/checksum_offload.pg b/extras/pg/checksum_offload.pg index a2e3205a21e..bfbdad66891 100644 --- a/extras/pg/checksum_offload.pg +++ b/extras/pg/checksum_offload.pg @@ -15,7 +15,8 @@ packet-generator new { interface loop0 tx-interface loop1 node loop1-output - buffer-flags ip4 offload-ip-cksum offload-udp-cksum + buffer-flags ip4 offload + buffer-offload-flags offload-ip-cksum offload-udp-cksum data { IP4: 1.2.3 -> 4.5.6 UDP: 11.22.33.44 -> 11.22.34.44 ttl 2 checksum 13 diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c index c53fc42a850..5bcb68cc95c 100644 --- a/src/plugins/avf/output.c +++ b/src/plugins/avf/output.c @@ -57,12 +57,12 @@ static_always_inline u64 avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso) { u64 flags = 0; - if (!is_tso && !(b->flags & ((VNET_BUFFER_F_OFFLOAD_IP_CKSUM | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)))) + if (!is_tso && !(b->flags & VNET_BUFFER_F_OFFLOAD)) return 0; - u32 is_tcp = is_tso || b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; - u32 is_udp = !is_tso && b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + + u32 oflags = vnet_buffer2 (b)->oflags; + u32 is_tcp = is_tso || oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; + u32 is_udp = !is_tso && oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; u32 is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4; u32 is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6; ASSERT (!is_tcp || !is_udp); @@ -173,9 +173,7 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq, { u16 next = txq->next; u64 bits = AVF_TXD_CMD_EOP | AVF_TXD_CMD_RSV; - const u32 offload_mask = VNET_BUFFER_F_OFFLOAD_IP_CKSUM | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM | - VNET_BUFFER_F_GSO; + const u32 offload_mask = VNET_BUFFER_F_OFFLOAD | VNET_BUFFER_F_GSO; u64 one_by_one_offload_flags = 0; int is_tso; u16 n_desc = 0; diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index ec33f6a4461..6466b443633 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -218,17 +218,20 @@ static_always_inline void dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b, struct rte_mbuf *mb) { - u32 ip_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM; - u32 tcp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; - u32 udp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4; u32 tso = b->flags & VNET_BUFFER_F_GSO, max_pkt_len; + u32 oflags, ip_cksum, tcp_cksum, udp_cksum; u64 ol_flags; /* Is there any work for us? */ - if (PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum | tso) == 0)) + if (PREDICT_TRUE (((b->flags & VNET_BUFFER_F_OFFLOAD) | tso) == 0)) return; + oflags = vnet_buffer2 (b)->oflags; + ip_cksum = oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM; + tcp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; + udp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; + mb->l2_len = vnet_buffer (b)->l3_hdr_offset - b->current_data; mb->l3_len = vnet_buffer (b)->l4_hdr_offset - vnet_buffer (b)->l3_hdr_offset; @@ -328,10 +331,7 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm, } if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) && - (or_flags & - (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM - | VNET_BUFFER_F_OFFLOAD_IP_CKSUM - | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)))) + (or_flags & VNET_BUFFER_F_OFFLOAD))) { dpdk_buffer_tx_offload (xd, b[0], mb[0]); dpdk_buffer_tx_offload (xd, b[1], mb[1]); @@ -388,10 +388,7 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm, } if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) && - (or_flags & - (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM - | VNET_BUFFER_F_OFFLOAD_IP_CKSUM - | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)))) + (or_flags & VNET_BUFFER_F_OFFLOAD))) { dpdk_buffer_tx_offload (xd, b[0], mb[0]); dpdk_buffer_tx_offload (xd, b[1], mb[1]); diff --git a/src/plugins/gtpu/gtpu_decap.c b/src/plugins/gtpu/gtpu_decap.c index e3bc476ed6d..974ae8a8190 100644 --- a/src/plugins/gtpu/gtpu_decap.c +++ b/src/plugins/gtpu/gtpu_decap.c @@ -1257,13 +1257,15 @@ static char *gtpu_flow_error_strings[] = { }; -#define gtpu_local_need_csum_check(_b) \ - (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED \ - || _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) - -#define gtpu_local_csum_is_valid(_b) \ - ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT \ - || _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) != 0) +#define gtpu_local_need_csum_check(_b) \ + (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED || \ + (_b->flags & VNET_BUFFER_F_OFFLOAD && \ + vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))) + +#define gtpu_local_csum_is_valid(_b) \ + ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT || \ + (_b->flags & VNET_BUFFER_F_OFFLOAD && \ + vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0) static_always_inline u8 gtpu_validate_udp_csum (vlib_main_t * vm, vlib_buffer_t *b) diff --git a/src/plugins/lisp/lisp-cp/packets.c b/src/plugins/lisp/lisp-cp/packets.c index 8e2386e72bd..3f4292b4841 100644 --- a/src/plugins/lisp/lisp-cp/packets.c +++ b/src/plugins/lisp/lisp-cp/packets.c @@ -182,7 +182,7 @@ pkt_push_udp_and_ip (vlib_main_t * vm, vlib_buffer_t * b, u16 sp, u16 dp, if (csum_offload) { ih = pkt_push_ip (vm, b, sip, dip, IP_PROTOCOL_UDP, 1); - b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM); vnet_buffer (b)->l3_hdr_offset = (u8 *) ih - b->data; vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data; uh->checksum = 0; diff --git a/src/plugins/vmxnet3/input.c b/src/plugins/vmxnet3/input.c index affc3691af1..be528b26494 100644 --- a/src/plugins/vmxnet3/input.c +++ b/src/plugins/vmxnet3/input.c @@ -79,6 +79,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, u16 gso_size) { u8 l4_hdr_sz = 0; + u32 oflags = 0; if (rx_comp->flags & VMXNET3_RXCF_IP4) { @@ -90,15 +91,15 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) + ip4_header_bytes (ip4); hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID | - VNET_BUFFER_F_L3_HDR_OFFSET_VALID | - VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4; + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4; /* checksum offload */ if (!(rx_comp->index & VMXNET3_RXCI_CNC)) { if (!(rx_comp->flags & VMXNET3_RXCF_IPC)) { - hb->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; ip4->checksum = 0; } if (!(rx_comp->flags & VMXNET3_RXCF_TUC)) @@ -108,7 +109,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, tcp_header_t *tcp = (tcp_header_t *) (hb->data + vnet_buffer (hb)->l4_hdr_offset); - hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; tcp->checksum = 0; } else if (rx_comp->flags & VMXNET3_RXCF_UDP) @@ -116,7 +117,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, udp_header_t *udp = (udp_header_t *) (hb->data + vnet_buffer (hb)->l4_hdr_offset); - hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; udp->checksum = 0; } } @@ -148,8 +149,8 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) + sizeof (ip6_header_t); hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID | - VNET_BUFFER_F_L3_HDR_OFFSET_VALID | - VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6; + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6; /* checksum offload */ if (!(rx_comp->index & VMXNET3_RXCI_CNC)) @@ -161,7 +162,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, tcp_header_t *tcp = (tcp_header_t *) (hb->data + vnet_buffer (hb)->l4_hdr_offset); - hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; tcp->checksum = 0; } else if (rx_comp->flags & VMXNET3_RXCF_UDP) @@ -169,7 +170,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, udp_header_t *udp = (udp_header_t *) (hb->data + vnet_buffer (hb)->l4_hdr_offset); - hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; udp->checksum = 0; } } @@ -194,6 +195,8 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb, hb->flags |= VNET_BUFFER_F_GSO; } } + if (oflags) + vnet_buffer_offload_flags_set (hb, oflags); } static_always_inline uword diff --git a/src/plugins/wireguard/wireguard_input.c b/src/plugins/wireguard/wireguard_input.c index d9843d90c7d..2690cf5e097 100755 --- a/src/plugins/wireguard/wireguard_input.c +++ b/src/plugins/wireguard/wireguard_input.c @@ -368,7 +368,8 @@ VLIB_NODE_FN (wg_input_node) (vlib_main_t * vm, clib_memcpy (vlib_buffer_get_current (b[0]), decr_data, decr_len); b[0]->current_length = decr_len; - b[0]->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + vnet_buffer_offload_flags_clear (b[0], + VNET_BUFFER_OFFLOAD_F_UDP_CKSUM); wg_timers_any_authenticated_packet_received (peer); wg_timers_any_authenticated_packet_traversal (peer); diff --git a/src/vnet/buffer.c b/src/vnet/buffer.c index 3cb6b948d2b..b41036d8124 100644 --- a/src/vnet/buffer.c +++ b/src/vnet/buffer.c @@ -16,6 +16,18 @@ #include <vlib/vlib.h> #include <vnet/buffer.h> +u8 * +format_vnet_buffer_offload (u8 *s, va_list *args) +{ + vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *); + +#define _(bit, name, ss, v) \ + if (v && (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name)) \ + s = format (s, "%s ", ss); + foreach_vnet_buffer_offload_flag +#undef _ + return s; +} u8 * format_vnet_buffer (u8 * s, va_list * args) @@ -29,7 +41,10 @@ format_vnet_buffer (u8 * s, va_list * args) a = format (a, "%s ", ss); foreach_vnet_buffer_flag #undef _ - if (b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID) + if (b->flags & VNET_BUFFER_F_OFFLOAD) a = + format (a, "%U ", format_vnet_buffer_offload, b); + + if (b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID) a = format (a, "l2-hdr-offset %d ", vnet_buffer (b)->l2_hdr_offset); if (b->flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID) @@ -38,6 +53,9 @@ format_vnet_buffer (u8 * s, va_list * args) if (b->flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID) a = format (a, "l4-hdr-offset %d ", vnet_buffer (b)->l4_hdr_offset); + if (b->flags & VNET_BUFFER_F_GSO) + a = format (a, "gso gso-size %d", vnet_buffer2 (b)->gso_size); + if (b->flags & VNET_BUFFER_F_QOS_DATA_VALID) a = format (a, "qos %d.%d ", vnet_buffer2 (b)->qos.bits, vnet_buffer2 (b)->qos.source); diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h index 27aeb3b42ea..fb734d5acde 100644 --- a/src/vnet/buffer.h +++ b/src/vnet/buffer.h @@ -46,34 +46,34 @@ * Flags that are set in the high order bits of ((vlib_buffer*)b)->flags * */ -#define foreach_vnet_buffer_flag \ - _( 1, L4_CHECKSUM_COMPUTED, "l4-cksum-computed", 1) \ - _( 2, L4_CHECKSUM_CORRECT, "l4-cksum-correct", 1) \ - _( 3, VLAN_2_DEEP, "vlan-2-deep", 1) \ - _( 4, VLAN_1_DEEP, "vlan-1-deep", 1) \ - _( 5, SPAN_CLONE, "span-clone", 1) \ - _( 6, LOOP_COUNTER_VALID, "loop-counter-valid", 0) \ - _( 7, LOCALLY_ORIGINATED, "local", 1) \ - _( 8, IS_IP4, "ip4", 1) \ - _( 9, IS_IP6, "ip6", 1) \ - _(10, OFFLOAD_IP_CKSUM, "offload-ip-cksum", 1) \ - _(11, OFFLOAD_TCP_CKSUM, "offload-tcp-cksum", 1) \ - _(12, OFFLOAD_UDP_CKSUM, "offload-udp-cksum", 1) \ - _(13, IS_NATED, "natted", 1) \ - _(14, L2_HDR_OFFSET_VALID, "l2_hdr_offset_valid", 0) \ - _(15, L3_HDR_OFFSET_VALID, "l3_hdr_offset_valid", 0) \ - _(16, L4_HDR_OFFSET_VALID, "l4_hdr_offset_valid", 0) \ - _(17, FLOW_REPORT, "flow-report", 1) \ - _(18, IS_DVR, "dvr", 1) \ - _(19, QOS_DATA_VALID, "qos-data-valid", 0) \ - _(20, GSO, "gso", 0) \ - _(21, AVAIL1, "avail1", 1) \ - _(22, AVAIL2, "avail2", 1) \ - _(23, AVAIL3, "avail3", 1) \ - _(24, AVAIL4, "avail4", 1) \ - _(25, AVAIL5, "avail5", 1) \ - _(26, AVAIL6, "avail6", 1) \ - _(27, AVAIL7, "avail7", 1) +#define foreach_vnet_buffer_flag \ + _ (1, L4_CHECKSUM_COMPUTED, "l4-cksum-computed", 1) \ + _ (2, L4_CHECKSUM_CORRECT, "l4-cksum-correct", 1) \ + _ (3, VLAN_2_DEEP, "vlan-2-deep", 1) \ + _ (4, VLAN_1_DEEP, "vlan-1-deep", 1) \ + _ (5, SPAN_CLONE, "span-clone", 1) \ + _ (6, LOOP_COUNTER_VALID, "loop-counter-valid", 0) \ + _ (7, LOCALLY_ORIGINATED, "local", 1) \ + _ (8, IS_IP4, "ip4", 1) \ + _ (9, IS_IP6, "ip6", 1) \ + _ (10, OFFLOAD, "offload", 0) \ + _ (11, IS_NATED, "natted", 1) \ + _ (12, L2_HDR_OFFSET_VALID, "l2_hdr_offset_valid", 0) \ + _ (13, L3_HDR_OFFSET_VALID, "l3_hdr_offset_valid", 0) \ + _ (14, L4_HDR_OFFSET_VALID, "l4_hdr_offset_valid", 0) \ + _ (15, FLOW_REPORT, "flow-report", 1) \ + _ (16, IS_DVR, "dvr", 1) \ + _ (17, QOS_DATA_VALID, "qos-data-valid", 0) \ + _ (18, GSO, "gso", 0) \ + _ (19, AVAIL1, "avail1", 1) \ + _ (20, AVAIL2, "avail2", 1) \ + _ (21, AVAIL3, "avail3", 1) \ + _ (22, AVAIL4, "avail4", 1) \ + _ (23, AVAIL5, "avail5", 1) \ + _ (24, AVAIL6, "avail6", 1) \ + _ (25, AVAIL7, "avail7", 1) \ + _ (26, AVAIL8, "avail8", 1) \ + _ (27, AVAIL9, "avail9", 1) /* * Please allocate the FIRST available bit, redefine @@ -81,10 +81,10 @@ * VNET_BUFFER_FLAGS_ALL_AVAIL definition. */ -#define VNET_BUFFER_FLAGS_ALL_AVAIL \ - (VNET_BUFFER_F_AVAIL1 | VNET_BUFFER_F_AVAIL2 | VNET_BUFFER_F_AVAIL3 | \ - VNET_BUFFER_F_AVAIL4 | VNET_BUFFER_F_AVAIL5 | VNET_BUFFER_F_AVAIL6 | \ - VNET_BUFFER_F_AVAIL7) +#define VNET_BUFFER_FLAGS_ALL_AVAIL \ + (VNET_BUFFER_F_AVAIL1 | VNET_BUFFER_F_AVAIL2 | VNET_BUFFER_F_AVAIL3 | \ + VNET_BUFFER_F_AVAIL4 | VNET_BUFFER_F_AVAIL5 | VNET_BUFFER_F_AVAIL6 | \ + VNET_BUFFER_F_AVAIL7 | VNET_BUFFER_F_AVAIL8 | VNET_BUFFER_F_AVAIL9) #define VNET_BUFFER_FLAGS_VLAN_BITS \ (VNET_BUFFER_F_VLAN_1_DEEP | VNET_BUFFER_F_VLAN_2_DEEP) @@ -420,6 +420,21 @@ STATIC_ASSERT (sizeof (vnet_buffer_opaque_t) <= #define vnet_buffer(b) ((vnet_buffer_opaque_t *) (b)->opaque) +#define foreach_vnet_buffer_offload_flag \ + _ (0, IP_CKSUM, "offload-ip-cksum", 1) \ + _ (1, TCP_CKSUM, "offload-tcp-cksum", 1) \ + _ (2, UDP_CKSUM, "offload-udp-cksum", 1) \ + _ (3, OUTER_IP_CKSUM, "offload-outer-ip-cksum", 1) \ + _ (4, OUTER_TCP_CKSUM, "offload-outer-tcp-cksum", 1) \ + _ (5, OUTER_UDP_CKSUM, "offload-outer-udp-cksum", 1) + +enum +{ +#define _(bit, name, s, v) VNET_BUFFER_OFFLOAD_F_##name = (1 << bit), + foreach_vnet_buffer_offload_flag +#undef _ +}; + /* Full cache line (64 bytes) of additional space */ typedef struct { @@ -452,12 +467,15 @@ typedef struct * in case the egress interface is not GSO-enabled - then we need to perform * the segmentation, and use this value to cut the payload appropriately. */ - u16 gso_size; - /* size of L4 prototol header */ - u16 gso_l4_hdr_sz; + struct + { + u16 gso_size; + /* size of L4 prototol header */ + u16 gso_l4_hdr_sz; - /* The union below has a u64 alignment, so this space is unused */ - u32 __unused2[1]; + /* offload flags */ + u32 oflags; + }; struct { @@ -501,6 +519,22 @@ STATIC_ASSERT (sizeof (vnet_buffer_opaque2_t) <= format_function_t format_vnet_buffer; +format_function_t format_vnet_buffer_offload; + +static_always_inline void +vnet_buffer_offload_flags_set (vlib_buffer_t *b, u32 oflags) +{ + vnet_buffer2 (b)->oflags |= oflags; + b->flags |= VNET_BUFFER_F_OFFLOAD; +} + +static_always_inline void +vnet_buffer_offload_flags_clear (vlib_buffer_t *b, u32 oflags) +{ + vnet_buffer2 (b)->oflags &= ~oflags; + if (0 == vnet_buffer2 (b)->oflags) + b->flags &= ~VNET_BUFFER_F_OFFLOAD; +} #endif /* included_vnet_buffer_h */ diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c index f4db6399a57..9f1782e2d13 100644 --- a/src/vnet/devices/af_packet/node.c +++ b/src/vnet/devices/af_packet/node.c @@ -120,6 +120,7 @@ static_always_inline void mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) { ethernet_header_t *eth = vlib_buffer_get_current (b); + u32 oflags = 0; if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP4) { ip4_header_t *ip4 = @@ -127,7 +128,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) b->flags |= VNET_BUFFER_F_IS_IP4; if (ip4->protocol == IP_PROTOCOL_TCP) { - b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b) + sizeof (ethernet_header_t) + ip4_header_bytes (ip4)); @@ -136,7 +137,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) } else if (ip4->protocol == IP_PROTOCOL_UDP) { - b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b) + sizeof (ethernet_header_t) + ip4_header_bytes (ip4)); @@ -146,6 +147,8 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t); vnet_buffer (b)->l4_hdr_offset = sizeof (ethernet_header_t) + ip4_header_bytes (ip4); + if (oflags) + vnet_buffer_offload_flags_set (b, oflags); } else if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP6) { @@ -165,7 +168,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) } if (ip6->protocol == IP_PROTOCOL_TCP) { - b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b) + sizeof (ethernet_header_t) + ip6_hdr_len); @@ -174,7 +177,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) } else if (ip6->protocol == IP_PROTOCOL_UDP) { - b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b) + sizeof (ethernet_header_t) + ip6_hdr_len); @@ -184,6 +187,8 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t); vnet_buffer (b)->l4_hdr_offset = sizeof (ethernet_header_t) + ip6_hdr_len; + if (oflags) + vnet_buffer_offload_flags_set (b, oflags); } } diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c index 3482336b2ad..fb996d551fa 100644 --- a/src/vnet/devices/virtio/device.c +++ b/src/vnet/devices/virtio/device.c @@ -282,6 +282,8 @@ static_always_inline void set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, const int is_l2) { + u32 oflags = vnet_buffer2 (b)->oflags; + if (b->flags & VNET_BUFFER_F_IS_IP4) { ip4_header_t *ip4; @@ -290,11 +292,11 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, 0 /* ip6 */ ); hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; hdr->csum_start = gho.l4_hdr_offset; // 0x22; - if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) { hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum); } - else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) + else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) { hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum); } @@ -305,7 +307,7 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, */ ip4 = (ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset); - if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) ip4->checksum = ip4_header_checksum (ip4); } else if (b->flags & VNET_BUFFER_F_IS_IP6) @@ -315,11 +317,11 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, 1 /* ip6 */ ); hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; hdr->csum_start = gho.l4_hdr_offset; // 0x36; - if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) { hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum); } - else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) + else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) { hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum); } @@ -330,6 +332,8 @@ static_always_inline void set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, const int is_l2) { + u32 oflags = vnet_buffer2 (b)->oflags; + if (b->flags & VNET_BUFFER_F_IS_IP4) { ip4_header_t *ip4; @@ -348,7 +352,7 @@ set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, * virtio devices do not support IP4 checksum offload. So driver takes care * of it while doing tx. */ - if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) ip4->checksum = ip4_header_checksum (ip4); } else if (b->flags & VNET_BUFFER_F_IS_IP6) @@ -392,8 +396,7 @@ add_buffer_to_slot (vlib_main_t * vm, vlib_node_runtime_t * node, goto done; } } - else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) + else if (b->flags & VNET_BUFFER_F_OFFLOAD) { if (csum_offload) set_checksum_offsets (b, hdr, is_l2); @@ -584,8 +587,7 @@ add_buffer_to_slot_packed (vlib_main_t * vm, vlib_node_runtime_t * node, goto done; } } - else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) + else if (b->flags & VNET_BUFFER_F_OFFLOAD) { if (csum_offload) set_checksum_offsets (b, hdr, is_l2); diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c index be2405bb3b9..98df322ea15 100644 --- a/src/vnet/devices/virtio/node.c +++ b/src/vnet/devices/virtio/node.c @@ -235,6 +235,7 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { u16 ethertype = 0, l2hdr_sz = 0; + u32 oflags = 0; if (type == VIRTIO_IF_TYPE_TUN) { @@ -280,11 +281,10 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, (ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz); vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4); *l4_proto = ip4->protocol; + oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; b0->flags |= - (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM); - b0->flags |= - (VNET_BUFFER_F_L2_HDR_OFFSET_VALID - | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID | + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID); } else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) @@ -301,7 +301,7 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, } if (*l4_proto == IP_PROTOCOL_TCP) { - b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b0) + vnet_buffer (b0)->l4_hdr_offset); @@ -309,12 +309,14 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, } else if (*l4_proto == IP_PROTOCOL_UDP) { - b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b0) + vnet_buffer (b0)->l4_hdr_offset); *l4_hdr_sz = sizeof (*udp); } + if (oflags) + vnet_buffer_offload_flags_set (b0, oflags); } } diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c index 62b59f69ba9..739125bbcfe 100644 --- a/src/vnet/devices/virtio/vhost_user_input.c +++ b/src/vnet/devices/virtio/vhost_user_input.c @@ -253,6 +253,7 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data, ethernet_header_t *eh = (ethernet_header_t *) b0_data; u16 ethertype = clib_net_to_host_u16 (eh->type); u16 l2hdr_sz = sizeof (ethernet_header_t); + u32 oflags = 0; if (ethernet_frame_is_tagged (ethertype)) { @@ -278,7 +279,8 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data, { ip4_header_t *ip4 = (ip4_header_t *) (b0_data + l2hdr_sz); l4_proto = ip4->protocol; - b0->flags |= VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM; + b0->flags |= VNET_BUFFER_F_IS_IP4; + oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; } else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) { @@ -292,12 +294,12 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data, tcp_header_t *tcp = (tcp_header_t *) (b0_data + vnet_buffer (b0)->l4_hdr_offset); l4_hdr_sz = tcp_header_bytes (tcp); - b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; } else if (l4_proto == IP_PROTOCOL_UDP) { l4_hdr_sz = sizeof (udp_header_t); - b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; } if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP) @@ -318,6 +320,9 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data, vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz; b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6); } + + if (oflags) + vnet_buffer_offload_flags_set (b0, oflags); } static_always_inline void diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c index 465c0ea0903..8dbddea46f5 100644 --- a/src/vnet/devices/virtio/vhost_user_output.c +++ b/src/vnet/devices/virtio/vhost_user_output.c @@ -225,10 +225,11 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b, generic_header_offset_t gho = { 0 }; int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4; int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6; + u32 oflags = vnet_buffer2 (b)->oflags; ASSERT (!(is_ip4 && is_ip6)); vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6); - if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) { ip4_header_t *ip4; @@ -238,13 +239,13 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b, } /* checksum offload */ - if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) { hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; hdr->csum_start = gho.l4_hdr_offset; hdr->csum_offset = offsetof (udp_header_t, checksum); } - else if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) + else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) { hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; hdr->csum_start = gho.l4_hdr_offset; @@ -254,7 +255,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b, /* GSO offload */ if (b->flags & VNET_BUFFER_F_GSO) { - if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) { if (is_ip4 && (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4))) @@ -270,7 +271,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b, } } else if ((vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)) && - (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) + (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) { hdr->gso_size = vnet_buffer2 (b)->gso_size; hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP; @@ -477,9 +478,7 @@ retry: hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE; hdr->num_buffers = 1; - or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) || - (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) || - (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM); + or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD); /* Guest supports csum offload and buffer requires checksum offload? */ if (or_flags && @@ -813,9 +812,7 @@ retry: hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE; hdr->num_buffers = 1; //This is local, no need to check - or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) || - (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) || - (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM); + or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD); /* Guest supports csum offload and buffer requires checksum offload? */ if (or_flags diff --git a/src/vnet/gso/gro_func.h b/src/vnet/gso/gro_func.h index b821a034e71..239009d680b 100644 --- a/src/vnet/gso/gro_func.h +++ b/src/vnet/gso/gro_func.h @@ -147,7 +147,7 @@ gro_validate_checksum (vlib_main_t * vm, vlib_buffer_t * b0, { u32 flags = 0; - if (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) + if (b0->flags & VNET_BUFFER_F_OFFLOAD) return VNET_BUFFER_F_L4_CHECKSUM_CORRECT; vlib_buffer_advance (b0, gho0->l3_hdr_offset); if (is_ip4) @@ -353,9 +353,9 @@ gro_fixup_header (vlib_main_t * vm, vlib_buffer_t * b0, u32 ack_number, ip4->length = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - gho0.l3_hdr_offset); - b0->flags |= - (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4 | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_IP_CKSUM); + b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4); + vnet_buffer_offload_flags_set (b0, (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | + VNET_BUFFER_OFFLOAD_F_IP_CKSUM)); } else if (gho0.gho_flags & GHO_F_IP6) { @@ -364,9 +364,8 @@ gro_fixup_header (vlib_main_t * vm, vlib_buffer_t * b0, u32 ack_number, ip6->payload_length = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - gho0.l4_hdr_offset); - b0->flags |= - (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6 | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM); + b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6); + vnet_buffer_offload_flags_set (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM); } tcp_header_t *tcp0 = diff --git a/src/vnet/gso/node.c b/src/vnet/gso/node.c index d0c5ca054e6..ecbdc188ec8 100644 --- a/src/vnet/gso/node.c +++ b/src/vnet/gso/node.c @@ -143,7 +143,8 @@ tso_segment_vxlan_tunnel_headers_fixup (vlib_main_t * vm, vlib_buffer_t * b, { udp->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4); } - b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + /* FIXME: it should be OUTER_UDP_CKSUM */ + vnet_buffer_offload_flags_clear (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM); } } @@ -281,7 +282,8 @@ tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags, tcp->checksum = 0; tcp->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6, &bogus); - b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + vnet_buffer_offload_flags_clear (b0, + VNET_BUFFER_OFFLOAD_F_TCP_CKSUM); } } else @@ -295,8 +297,8 @@ tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags, tcp->checksum = 0; tcp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip4); } - b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; - b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM; + vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM | + VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)); } if (!is_l2 && ((gho->gho_flags & GHO_F_TUNNEL) == 0)) diff --git a/src/vnet/interface_format.c b/src/vnet/interface_format.c index dc2edff7c67..036ca53174c 100644 --- a/src/vnet/interface_format.c +++ b/src/vnet/interface_format.c @@ -722,7 +722,7 @@ unformat_vnet_buffer_flags (unformat_input_t * input, va_list * args) while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { /* Red herring, there is no such buffer flag */ - if (unformat (input, "avail8")) + if (unformat (input, "avail10")) return 0; #define _(bit,enum,str,verbose) \ else if (unformat (input, str)) \ @@ -741,6 +741,32 @@ unformat_vnet_buffer_flags (unformat_input_t * input, va_list * args) } uword +unformat_vnet_buffer_offload_flags (unformat_input_t *input, va_list *args) +{ + u32 *flagp = va_arg (*args, u32 *); + int rv = 0; + u32 oflags = 0; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (0) + ; +#define _(bit, enum, str, verbose) \ + else if (unformat (input, str)) \ + { \ + oflags |= (1 << bit); \ + rv = 1; \ + } + foreach_vnet_buffer_offload_flag +#undef _ + else break; + } + if (rv) + *flagp = oflags; + return rv; +} + +uword unformat_vnet_hw_interface (unformat_input_t * input, va_list * args) { vnet_main_t *vnm = va_arg (*args, vnet_main_t *); diff --git a/src/vnet/interface_funcs.h b/src/vnet/interface_funcs.h index dbb0549314a..186e0327884 100644 --- a/src/vnet/interface_funcs.h +++ b/src/vnet/interface_funcs.h @@ -457,6 +457,7 @@ format_function_t format_vnet_sw_interface_flags; unformat_function_t unformat_vnet_sw_interface; unformat_function_t unformat_vnet_hw_interface; unformat_function_t unformat_vnet_buffer_flags; +unformat_function_t unformat_vnet_buffer_offload_flags; /* Parses interface flags (up, down, enable, disable, etc.) */ unformat_function_t unformat_vnet_hw_interface_flags; diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c index 3363f76b722..ff315f6abe0 100644 --- a/src/vnet/interface_output.c +++ b/src/vnet/interface_output.c @@ -337,28 +337,24 @@ vnet_interface_output_node_inline (vlib_main_t * vm, if (do_tx_offloads) { - u32 vnet_buffer_offload_flags = - (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM | - VNET_BUFFER_F_OFFLOAD_IP_CKSUM); - if (or_flags & vnet_buffer_offload_flags) + if (or_flags & VNET_BUFFER_F_OFFLOAD) { - if (b[0]->flags & vnet_buffer_offload_flags) + if (b[0]->flags & VNET_BUFFER_F_OFFLOAD) vnet_calc_checksums_inline (vm, b[0], b[0]->flags & VNET_BUFFER_F_IS_IP4, b[0]->flags & VNET_BUFFER_F_IS_IP6); - if (b[1]->flags & vnet_buffer_offload_flags) + if (b[1]->flags & VNET_BUFFER_F_OFFLOAD) vnet_calc_checksums_inline (vm, b[1], b[1]->flags & VNET_BUFFER_F_IS_IP4, b[1]->flags & VNET_BUFFER_F_IS_IP6); - if (b[2]->flags & vnet_buffer_offload_flags) + if (b[2]->flags & VNET_BUFFER_F_OFFLOAD) vnet_calc_checksums_inline (vm, b[2], b[2]->flags & VNET_BUFFER_F_IS_IP4, b[2]->flags & VNET_BUFFER_F_IS_IP6); - if (b[3]->flags & vnet_buffer_offload_flags) + if (b[3]->flags & VNET_BUFFER_F_OFFLOAD) vnet_calc_checksums_inline (vm, b[3], b[3]->flags & VNET_BUFFER_F_IS_IP4, @@ -406,10 +402,7 @@ vnet_interface_output_node_inline (vlib_main_t * vm, if (do_tx_offloads) { - if (b[0]->flags & - (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM | - VNET_BUFFER_F_OFFLOAD_IP_CKSUM)) + if (b[0]->flags & VNET_BUFFER_F_OFFLOAD) vnet_calc_checksums_inline (vm, b[0], b[0]->flags & VNET_BUFFER_F_IS_IP4, diff --git a/src/vnet/interface_output.h b/src/vnet/interface_output.h index 4b082b314ad..27aadc6c5fd 100644 --- a/src/vnet/interface_output.h +++ b/src/vnet/interface_output.h @@ -44,18 +44,17 @@ #include <vnet/gso/hdr_offset_parser.h> static_always_inline void -vnet_calc_ip4_checksums (vlib_main_t * vm, vlib_buffer_t * b, - ip4_header_t * ip4, tcp_header_t * th, - udp_header_t * uh) +vnet_calc_ip4_checksums (vlib_main_t *vm, vlib_buffer_t *b, ip4_header_t *ip4, + tcp_header_t *th, udp_header_t *uh, u32 oflags) { - if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) ip4->checksum = ip4_header_checksum (ip4); - if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) { th->checksum = 0; th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4); } - if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) { uh->checksum = 0; uh->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4); @@ -63,17 +62,16 @@ vnet_calc_ip4_checksums (vlib_main_t * vm, vlib_buffer_t * b, } static_always_inline void -vnet_calc_ip6_checksums (vlib_main_t * vm, vlib_buffer_t * b, - ip6_header_t * ip6, tcp_header_t * th, - udp_header_t * uh) +vnet_calc_ip6_checksums (vlib_main_t *vm, vlib_buffer_t *b, ip6_header_t *ip6, + tcp_header_t *th, udp_header_t *uh, u32 oflags) { int bogus; - if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) { th->checksum = 0; th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus); } - if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) + if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) { uh->checksum = 0; uh->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus); @@ -88,6 +86,7 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b, ip6_header_t *ip6; tcp_header_t *th; udp_header_t *uh; + u32 oflags = vnet_buffer2 (b)->oflags; ASSERT (!(is_ip4 && is_ip6)); @@ -98,16 +97,16 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b, if (is_ip4) { - vnet_calc_ip4_checksums (vm, b, ip4, th, uh); + vnet_calc_ip4_checksums (vm, b, ip4, th, uh, oflags); } else if (is_ip6) { - vnet_calc_ip6_checksums (vm, b, ip6, th, uh); + vnet_calc_ip6_checksums (vm, b, ip6, th, uh, oflags); } - b->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; - b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; - b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM; + vnet_buffer_offload_flags_clear (b, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM | + VNET_BUFFER_OFFLOAD_F_UDP_CKSUM | + VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)); } #endif diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c index 332c483aa9d..9e8d164b51f 100644 --- a/src/vnet/ip/ip4_forward.c +++ b/src/vnet/ip/ip4_forward.c @@ -1417,9 +1417,10 @@ ip4_local_l4_csum_validate (vlib_main_t * vm, vlib_buffer_t * p, } } -#define ip4_local_csum_is_offloaded(_b) \ - _b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM \ - || _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM +#define ip4_local_csum_is_offloaded(_b) \ + ((_b->flags & VNET_BUFFER_F_OFFLOAD) && \ + (vnet_buffer2 (_b)->oflags & \ + (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))) #define ip4_local_need_csum_check(is_tcp_udp, _b) \ (is_tcp_udp && !(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED \ @@ -2066,7 +2067,7 @@ ip4_ttl_and_checksum_check (vlib_buffer_t * b, ip4_header_t * ip, u16 * next, /* Verify checksum. */ ASSERT (ip4_header_checksum_is_valid (ip) || - (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)); + (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)); } diff --git a/src/vnet/ip/ip4_inlines.h b/src/vnet/ip/ip4_inlines.h index 3075fbf42a1..00a47125b8a 100644 --- a/src/vnet/ip/ip4_inlines.h +++ b/src/vnet/ip/ip4_inlines.h @@ -125,7 +125,7 @@ vlib_buffer_push_ip4_custom (vlib_main_t * vm, vlib_buffer_t * b, if (csum_offload) { ih->checksum = 0; - b->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM; + vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_IP_CKSUM); } else ih->checksum = ip4_header_checksum (ih); diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index bf6aa7f3212..0c1bc3b33ec 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -1333,15 +1333,23 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node, flags[0] = b[0]->flags; flags[1] = b[1]->flags; + u32 oflags[2]; + oflags[0] = vnet_buffer2 (b[0])->oflags; + oflags[1] = vnet_buffer2 (b[1])->oflags; + + u32 l4_offload[2]; + l4_offload[0] = (flags[0] & VNET_BUFFER_F_OFFLOAD) && + (oflags[0] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | + VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)); + l4_offload[1] = (flags[1] & VNET_BUFFER_F_OFFLOAD) && + (oflags[1] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | + VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)); + u32 good_l4_csum[2]; good_l4_csum[0] = - flags[0] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM); + (flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[0]; good_l4_csum[1] = - flags[1] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM); + (flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[1]; u32 udp_offset[2] = { }; u8 is_tcp_udp[2]; @@ -1515,11 +1523,15 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u8 type = lm->builtin_protocol_by_ip_protocol[ip->protocol]; u32 flags = b[0]->flags; - u32 good_l4_csum = - flags & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM); + u32 oflags = vnet_buffer2 (b[0])->oflags; + + u32 l4_offload = (flags & VNET_BUFFER_F_OFFLOAD) && + (oflags & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | + VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)); + + u32 good_l4_csum = + (flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload; u32 udp_offset; i16 len_diff = 0; u8 is_tcp_udp = ip6_next_proto_is_tcp_udp (b[0], ip, &udp_offset); @@ -1539,8 +1551,9 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node, good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UNKNOWN; len_diff = type == IP_BUILTIN_PROTOCOL_UDP ? len_diff : 0; - u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN && !good_l4_csum - && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); + u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN && + !good_l4_csum && + !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED); if (PREDICT_FALSE (need_csum)) { flags = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]); @@ -1556,7 +1569,6 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node, error = len_diff < 0 ? IP6_ERROR_UDP_LENGTH : error; - STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP == IP6_ERROR_UDP_CHECKSUM, "Wrong IP6 errors constants"); @@ -1882,8 +1894,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm, if (p1->flags & VNET_BUFFER_F_GSO) ip1_len = gso_mtu_sz (p1); - - ip6_mtu_check (p0, ip0_len, adj0[0].rewrite_header.max_l3_packet_bytes, is_locally_originated0, &next0, is_midchain, @@ -1892,18 +1902,15 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm, adj1[0].rewrite_header.max_l3_packet_bytes, is_locally_originated1, &next1, is_midchain, &error1); - /* Don't adjust the buffer for hop count issue; icmp-error node * wants to see the IP header */ if (PREDICT_TRUE (error0 == IP6_ERROR_NONE)) { p0->current_data -= rw_len0; p0->current_length += rw_len0; - tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index; vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0; next0 = adj0[0].rewrite_header.next_index; - if (PREDICT_FALSE (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES)) vnet_feature_arc_start_w_cfg_index @@ -2070,7 +2077,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm, adj0[0].rewrite_header.max_l3_packet_bytes, is_locally_originated0, &next0, is_midchain, &error0); - /* Don't adjust the buffer for hop count issue; icmp-error node * wants to see the IP header */ if (PREDICT_TRUE (error0 == IP6_ERROR_NONE)) @@ -2289,7 +2295,6 @@ format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args) ip6_hop_by_hop_option_t *opt0, *limit0; ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main; u8 type0; - s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d", hbh0->protocol, (hbh0->length + 1) << 3, total_len); @@ -2312,9 +2317,8 @@ format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args) } else { - s = - format (s, "\n unrecognized option %d length %d", type0, - opt0->length); + s = format (s, "\n unrecognized option %d length %d", type0, + opt0->length); } opt0 = (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length + @@ -2362,9 +2366,8 @@ format_ip6_hop_by_hop_trace (u8 * s, va_list * args) } else { - s = - format (s, "\n unrecognized option %d length %d", type0, - opt0->length); + s = format (s, "\n unrecognized option %d length %d", type0, + opt0->length); } opt0 = (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length + @@ -3012,8 +3015,7 @@ VLIB_CLI_COMMAND (show_ip6_local, static) = #ifndef CLIB_MARCH_VARIANT int -vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index, - u32 table_index) +vnet_set_ip6_classify_intfc (vlib_main_t *vm, u32 sw_if_index, u32 table_index) { vnet_main_t *vnm = vnet_get_main (); vnet_interface_main_t *im = &vnm->interface_main; @@ -3044,17 +3046,13 @@ vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index, fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4, sw_if_index); - - if (table_index != (u32) ~ 0) { dpo_id_t dpo = DPO_INVALID; - dpo_set (&dpo, DPO_CLASSIFY, DPO_PROTO_IP6, classify_dpo_create (DPO_PROTO_IP6, table_index)); - fib_table_entry_special_dpo_add (fib_index, &pfx, FIB_SOURCE_CLASSIFY, diff --git a/src/vnet/ip/ip_frag.c b/src/vnet/ip/ip_frag.c index 9aa8777514f..33fbb9b2a9d 100644 --- a/src/vnet/ip/ip_frag.c +++ b/src/vnet/ip/ip_frag.c @@ -226,7 +226,7 @@ ip4_frag_do_fragment (vlib_main_t * vm, u32 from_bi, u16 mtu, to_ip4->checksum = ip4_header_checksum (to_ip4); /* we've just done the IP checksum .. */ - to_b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM; + vnet_buffer_offload_flags_clear (to_b, VNET_BUFFER_OFFLOAD_F_IP_CKSUM); rem -= len; fo += len; diff --git a/src/vnet/ipsec/ipsec_output.c b/src/vnet/ipsec/ipsec_output.c index d09a027de29..e6cf837d796 100644 --- a/src/vnet/ipsec/ipsec_output.c +++ b/src/vnet/ipsec/ipsec_output.c @@ -309,45 +309,50 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node, next_node_index = im->ah4_encrypt_node_index; vnet_buffer (b0)->ipsec.sad_index = p0->sa_index; - if (is_ipv6) + if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_OFFLOAD)) { - if (PREDICT_FALSE - (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)) - { - tcp0->checksum = - ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0, - &bogus); - b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; - } - if (PREDICT_FALSE - (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) - { - udp0->checksum = - ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0, - &bogus); - b0->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; - } - } - else - { - if (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) - { - ip0->checksum = ip4_header_checksum (ip0); - b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM; - } - if (PREDICT_FALSE - (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)) + u32 oflags = vnet_buffer2 (b0)->oflags; + + /* + * Clearing offload flags before checksum is computed + * It guarantees the cache hit! + */ + vnet_buffer_offload_flags_clear (b0, oflags); + + if (is_ipv6) { - tcp0->checksum = - ip4_tcp_udp_compute_checksum (vm, b0, ip0); - b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + if (PREDICT_FALSE (oflags & + VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)) + { + tcp0->checksum = ip6_tcp_udp_icmp_compute_checksum ( + vm, b0, ip6_0, &bogus); + } + if (PREDICT_FALSE (oflags & + VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) + { + udp0->checksum = ip6_tcp_udp_icmp_compute_checksum ( + vm, b0, ip6_0, &bogus); + } } - if (PREDICT_FALSE - (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)) + else { - udp0->checksum = - ip4_tcp_udp_compute_checksum (vm, b0, ip0); - b0->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + if (PREDICT_FALSE (oflags & + VNET_BUFFER_OFFLOAD_F_IP_CKSUM)) + { + ip0->checksum = ip4_header_checksum (ip0); + } + if (PREDICT_FALSE (oflags & + VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)) + { + tcp0->checksum = + ip4_tcp_udp_compute_checksum (vm, b0, ip0); + } + if (PREDICT_FALSE (oflags & + VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) + { + udp0->checksum = + ip4_tcp_udp_compute_checksum (vm, b0, ip0); + } } } vlib_buffer_advance (b0, iph_offset); diff --git a/src/vnet/pg/cli.c b/src/vnet/pg/cli.c index 625198634f5..3ef7a959f20 100644 --- a/src/vnet/pg/cli.c +++ b/src/vnet/pg/cli.c @@ -379,7 +379,9 @@ new_stream (vlib_main_t * vm, else if (unformat (input, "buffer-flags %U", unformat_vnet_buffer_flags, &s.buffer_flags)) ; - + else if (unformat (input, "buffer-offload-flags %U", + unformat_vnet_buffer_offload_flags, &s.buffer_oflags)) + ; else if (unformat (input, "node %U", unformat_vlib_node, vm, &s.node_index)) ; diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c index 940bf124c56..d3501b3e3b6 100644 --- a/src/vnet/pg/input.c +++ b/src/vnet/pg/input.c @@ -1537,13 +1537,14 @@ pg_input_trace (pg_main_t * pg, } static_always_inline void -fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers, - int gso_enabled, u32 gso_size) +fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers, + u32 buffer_oflags, int gso_enabled, u32 gso_size) { for (int i = 0; i < n_buffers; i++) { vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[i]); u8 l4_proto = 0; + u32 oflags = 0; ethernet_header_t *eh = (ethernet_header_t *) vlib_buffer_get_current (b0); @@ -1574,10 +1575,11 @@ fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers, vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4); l4_proto = ip4->protocol; b0->flags |= - (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM); - b0->flags |= (VNET_BUFFER_F_L2_HDR_OFFSET_VALID - | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | - VNET_BUFFER_F_L4_HDR_OFFSET_VALID); + (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID | + VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID); + if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) + oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; } else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) { @@ -1594,7 +1596,8 @@ fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers, if (l4_proto == IP_PROTOCOL_TCP) { - b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM) + oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; /* only set GSO flag for chained buffers */ if (gso_enabled && (b0->flags & VLIB_BUFFER_NEXT_PRESENT)) @@ -1609,8 +1612,12 @@ fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers, } else if (l4_proto == IP_PROTOCOL_UDP) { - b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM) + oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; } + + if (oflags) + vnet_buffer_offload_flags_set (b0, oflags); } } @@ -1710,13 +1717,11 @@ pg_generate_packets (vlib_node_runtime_t * node, vnet_buffer (b)->feature_arc_index = feature_arc_index; } - if (pi->gso_enabled || - (s->buffer_flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | - VNET_BUFFER_F_OFFLOAD_UDP_CKSUM | - VNET_BUFFER_F_OFFLOAD_IP_CKSUM))) + if (pi->gso_enabled || (s->buffer_flags & VNET_BUFFER_F_OFFLOAD)) { fill_buffer_offload_flags (vm, to_next, n_this_frame, - pi->gso_enabled, pi->gso_size); + s->buffer_oflags, pi->gso_enabled, + pi->gso_size); } n_trace = vlib_get_trace_count (vm, node); diff --git a/src/vnet/pg/pg.h b/src/vnet/pg/pg.h index da2c2b54347..da5af25b22e 100644 --- a/src/vnet/pg/pg.h +++ b/src/vnet/pg/pg.h @@ -125,9 +125,12 @@ typedef struct pg_stream_t for max_packet_bytes. */ u32 buffer_bytes; - /* Buffer flags to set in each packet e.g. checksum offload flags */ + /* Buffer flags to set in each packet e.g. l2 valid flags */ u32 buffer_flags; + /* Buffer offload flags to set in each packet e.g. checksum offload flags */ + u32 buffer_oflags; + /* Last packet length if packet size edit type is increment. */ u32 last_increment_packet_size; diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index e2c2b231985..235a59be0b3 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -430,7 +430,7 @@ tcp_compute_checksum (tcp_connection_t * tc, vlib_buffer_t * b) } else { - b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM); } return checksum; } diff --git a/src/vnet/udp/udp_inlines.h b/src/vnet/udp/udp_inlines.h index ec800f9a413..067289c9979 100644 --- a/src/vnet/udp/udp_inlines.h +++ b/src/vnet/udp/udp_inlines.h @@ -35,7 +35,7 @@ vlib_buffer_push_udp (vlib_buffer_t * b, u16 sp, u16 dp, u8 offload_csum) uh->checksum = 0; uh->length = clib_host_to_net_u16 (udp_len); if (offload_csum) - b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM); vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data; b->flags |= VNET_BUFFER_F_L4_HDR_OFFSET_VALID; return uh; diff --git a/src/vnet/vxlan-gbp/encap.c b/src/vnet/vxlan-gbp/encap.c index 4514fdae7a7..2a4e8a8e312 100644 --- a/src/vnet/vxlan-gbp/encap.c +++ b/src/vnet/vxlan-gbp/encap.c @@ -102,14 +102,14 @@ vxlan_gbp_encap_inline (vlib_main_t * vm, u8 const underlay_hdr_len = is_ip4 ? sizeof (ip4_vxlan_gbp_header_t) : sizeof (ip6_vxlan_gbp_header_t); u16 const l3_len = is_ip4 ? sizeof (ip4_header_t) : sizeof (ip6_header_t); - u32 const csum_flags = is_ip4 ? VNET_BUFFER_F_OFFLOAD_IP_CKSUM | - VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM | - VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID : - VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM | - VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID; - u32 const inner_packet_csum_offload_flags = - VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + u32 const csum_flags = + is_ip4 ? VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID : + VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID; + u32 const outer_packet_csum_offload_flags = + is_ip4 ? VNET_BUFFER_OFFLOAD_F_IP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM : + VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; u32 const inner_packet_removed_flags = VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | @@ -147,7 +147,7 @@ vxlan_gbp_encap_inline (vlib_main_t * vm, n_left_from -= 2; u32 or_flags = b[0]->flags | b[1]->flags; - if (csum_offload && (or_flags & inner_packet_csum_offload_flags)) + if (csum_offload && (or_flags & VNET_BUFFER_F_OFFLOAD)) { /* Only calculate the non-GSO packet csum offload */ if ((b[0]->flags & VNET_BUFFER_F_GSO) == 0) @@ -300,9 +300,13 @@ vxlan_gbp_encap_inline (vlib_main_t * vm, b[0]->flags |= csum_flags; vnet_buffer (b[0])->l3_hdr_offset = l3_0 - b[0]->data; vnet_buffer (b[0])->l4_hdr_offset = (u8 *) udp0 - b[0]->data; + vnet_buffer_offload_flags_set (b[0], + outer_packet_csum_offload_flags); b[1]->flags |= csum_flags; vnet_buffer (b[1])->l3_hdr_offset = l3_1 - b[1]->data; vnet_buffer (b[1])->l4_hdr_offset = (u8 *) udp1 - b[1]->data; + vnet_buffer_offload_flags_set (b[1], + outer_packet_csum_offload_flags); } /* IPv4 UDP checksum only if checksum offload is used */ else if (is_ip4) @@ -387,7 +391,7 @@ vxlan_gbp_encap_inline (vlib_main_t * vm, n_left_from -= 1; n_left_to_next -= 1; - if (csum_offload && (b[0]->flags & inner_packet_csum_offload_flags)) + if (csum_offload && (b[0]->flags & VNET_BUFFER_F_OFFLOAD)) { /* Only calculate the non-GSO packet csum offload */ if ((b[0]->flags & VNET_BUFFER_F_GSO) == 0) @@ -477,6 +481,8 @@ vxlan_gbp_encap_inline (vlib_main_t * vm, b[0]->flags |= csum_flags; vnet_buffer (b[0])->l3_hdr_offset = l3_0 - b[0]->data; vnet_buffer (b[0])->l4_hdr_offset = (u8 *) udp0 - b[0]->data; + vnet_buffer_offload_flags_set (b[0], + outer_packet_csum_offload_flags); } /* IPv4 UDP checksum only if checksum offload is used */ else if (is_ip4) diff --git a/src/vnet/vxlan/encap.c b/src/vnet/vxlan/encap.c index 787758a94f8..476e0f2b3b7 100644 --- a/src/vnet/vxlan/encap.c +++ b/src/vnet/vxlan/encap.c @@ -98,14 +98,14 @@ vxlan_encap_inline (vlib_main_t * vm, u8 const underlay_hdr_len = is_ip4 ? sizeof (ip4_vxlan_header_t) : sizeof (ip6_vxlan_header_t); u16 const l3_len = is_ip4 ? sizeof (ip4_header_t) : sizeof (ip6_header_t); - u32 const csum_flags = is_ip4 ? VNET_BUFFER_F_OFFLOAD_IP_CKSUM | - VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM | - VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID : - VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM | - VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID; - u32 const inner_packet_csum_offload_flags = - VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM | - VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + u32 const csum_flags = + is_ip4 ? VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID : + VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | + VNET_BUFFER_F_L4_HDR_OFFSET_VALID; + u32 const outer_packet_csum_offload_flags = + is_ip4 ? VNET_BUFFER_OFFLOAD_F_IP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM : + VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; u32 const inner_packet_removed_flags = VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | @@ -144,7 +144,7 @@ vxlan_encap_inline (vlib_main_t * vm, b += 2; u32 or_flags = b0->flags | b1->flags; - if (csum_offload && (or_flags & inner_packet_csum_offload_flags)) + if (csum_offload && (or_flags & VNET_BUFFER_F_OFFLOAD)) { /* Only calculate the non-GSO packet csum offload */ if ((b0->flags & VNET_BUFFER_F_GSO) == 0) @@ -284,9 +284,13 @@ vxlan_encap_inline (vlib_main_t * vm, b0->flags |= csum_flags; vnet_buffer (b0)->l3_hdr_offset = l3_0 - b0->data; vnet_buffer (b0)->l4_hdr_offset = (u8 *) udp0 - b0->data; + vnet_buffer_offload_flags_set (b0, + outer_packet_csum_offload_flags); b1->flags |= csum_flags; vnet_buffer (b1)->l3_hdr_offset = l3_1 - b1->data; vnet_buffer (b1)->l4_hdr_offset = (u8 *) udp1 - b1->data; + vnet_buffer_offload_flags_set (b1, + outer_packet_csum_offload_flags); } /* IPv4 UDP checksum only if checksum offload is used */ else if (is_ip4) @@ -377,7 +381,7 @@ vxlan_encap_inline (vlib_main_t * vm, vlib_buffer_t *b0 = b[0]; b += 1; - if (csum_offload && (b0->flags & inner_packet_csum_offload_flags)) + if (csum_offload && (b0->flags & VNET_BUFFER_F_OFFLOAD)) { /* Only calculate the non-GSO packet csum offload */ if ((b0->flags & VNET_BUFFER_F_GSO) == 0) @@ -459,6 +463,8 @@ vxlan_encap_inline (vlib_main_t * vm, b0->flags |= csum_flags; vnet_buffer (b0)->l3_hdr_offset = l3_0 - b0->data; vnet_buffer (b0)->l4_hdr_offset = (u8 *) udp0 - b0->data; + vnet_buffer_offload_flags_set (b0, + outer_packet_csum_offload_flags); } /* IPv4 UDP checksum only if checksum offload is used */ else if (is_ip4) diff --git a/test/test_offload.py b/test/test_offload.py index 8b045bfe131..9f6aec65482 100644 --- a/test/test_offload.py +++ b/test/test_offload.py @@ -39,7 +39,8 @@ class TestOffload(VppTestCase): " interface loop0\n" " tx-interface loop1\n" " node loop1-output\n" - " buffer-flags ip4 offload-ip-cksum offload-udp-cksum\n" + " buffer-flags ip4 offload\n" + " buffer-offload-flags offload-ip-cksum offload-udp-cksum\n" " data {\n" " IP4: 1.2.3 -> dead.0000.0001\n" " UDP: 11.22.33.44 -> 11.22.34.44\n" diff --git a/test/test_pcap.py b/test/test_pcap.py index 5c086ef21c8..c058c5467af 100644 --- a/test/test_pcap.py +++ b/test/test_pcap.py @@ -42,7 +42,8 @@ class TestPcap(VppTestCase): " interface loop0\n" " tx-interface loop1\n" " node loop1-output\n" - " buffer-flags ip4 offload-ip-cksum offload-udp-cksum\n" + " buffer-flags ip4 offload\n" + " buffer-offload-flags offload-ip-cksum offload-udp-cksum\n" " data {\n" " IP4: 1.2.3 -> dead.0000.0001\n" " UDP: 11.22.33.44 -> 11.22.34.44\n" |