diff options
Diffstat (limited to 'src/vnet/gso')
| -rw-r--r-- | src/vnet/gso/FEATURE.yaml | 3 | ||||
| -rw-r--r-- | src/vnet/gso/gso.c | 1 | ||||
| -rw-r--r-- | src/vnet/gso/gso.h | 93 | ||||
| -rw-r--r-- | src/vnet/gso/hdr_offset_parser.h | 408 | ||||
| -rw-r--r-- | src/vnet/gso/node.c | 224 |
5 files changed, 605 insertions, 124 deletions
diff --git a/src/vnet/gso/FEATURE.yaml b/src/vnet/gso/FEATURE.yaml index 4edca9c0c90..b3296d6efa6 100644 --- a/src/vnet/gso/FEATURE.yaml +++ b/src/vnet/gso/FEATURE.yaml @@ -4,9 +4,10 @@ maintainer: ayourtch@gmail.com sykazmi@cisco.com features: - Basic GSO support - GSO for VLAN tagged packets + - GSO for VXLAN tunnel - Provide inline function to get header offsets description: "Generic Segmentation Offload" missing: - - Tunnels i.e. VXLAN + - Thorough Testing, IPIP, GRE, Geneve, IPSec state: experimental properties: [API, CLI] diff --git a/src/vnet/gso/gso.c b/src/vnet/gso/gso.c index cf90d22696d..c741b17bdd4 100644 --- a/src/vnet/gso/gso.c +++ b/src/vnet/gso/gso.c @@ -16,6 +16,7 @@ #include <vlib/vlib.h> #include <vnet/vnet.h> #include <vppinfra/error.h> +#include <vnet/ethernet/ethernet.h> #include <vnet/feature/feature.h> #include <vnet/l2/l2_in_out_feat_arc.h> #include <vnet/gso/gso.h> diff --git a/src/vnet/gso/gso.h b/src/vnet/gso/gso.h index 0e46c36f74d..8e174dfd1f6 100644 --- a/src/vnet/gso/gso.h +++ b/src/vnet/gso/gso.h @@ -16,25 +16,10 @@ #ifndef included_gso_h #define included_gso_h -#include <vnet/ethernet/ethernet.h> -#include <vnet/ip/ip4_packet.h> -#include <vnet/ip/ip6_packet.h> -#include <vnet/udp/udp_packet.h> #include <vnet/vnet.h> typedef struct { - i16 l2_hdr_offset; - i16 l3_hdr_offset; - i16 l4_hdr_offset; - u16 l4_hdr_sz; - i16 outer_l2_hdr_offset; - i16 outer_l3_hdr_offset; - i16 outer_l4_hdr_offset; -} gso_header_offset_t; - -typedef struct -{ vlib_main_t *vlib_main; vnet_main_t *vnet_main; u16 msg_id_base; @@ -44,84 +29,6 @@ extern gso_main_t gso_main; int vnet_sw_interface_gso_enable_disable (u32 sw_if_index, u8 enable); -static_always_inline gso_header_offset_t -vnet_gso_header_offset_parser (vlib_buffer_t * b0, int is_ip6) -{ - gso_header_offset_t gho = { 0 }; - u8 l4_proto = 0; - u8 l4_hdr_sz = 0; - - if (PREDICT_TRUE ((b0->flags & (VNET_BUFFER_F_L2_HDR_OFFSET_VALID | - VNET_BUFFER_F_L3_HDR_OFFSET_VALID | - VNET_BUFFER_F_L4_HDR_OFFSET_VALID)) == - (VNET_BUFFER_F_L2_HDR_OFFSET_VALID | - VNET_BUFFER_F_L3_HDR_OFFSET_VALID | - VNET_BUFFER_F_L4_HDR_OFFSET_VALID))) - { - gho.l2_hdr_offset = vnet_buffer (b0)->l2_hdr_offset; - gho.l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset; - gho.l4_hdr_offset = vnet_buffer (b0)->l4_hdr_offset; - gho.l4_hdr_sz = vnet_buffer2 (b0)->gso_l4_hdr_sz; - return gho; - } - - ethernet_header_t *eh = (ethernet_header_t *) vlib_buffer_get_current (b0); - u16 ethertype = clib_net_to_host_u16 (eh->type); - u16 l2hdr_sz = sizeof (ethernet_header_t); - - if (ethernet_frame_is_tagged (ethertype)) - { - ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1); - - ethertype = clib_net_to_host_u16 (vlan->type); - l2hdr_sz += sizeof (*vlan); - if (ethertype == ETHERNET_TYPE_VLAN) - { - vlan++; - ethertype = clib_net_to_host_u16 (vlan->type); - l2hdr_sz += sizeof (*vlan); - } - } - - gho.l2_hdr_offset = b0->current_data; - gho.l3_hdr_offset = l2hdr_sz; - - if (PREDICT_TRUE (is_ip6 == 0)) - { - ip4_header_t *ip4 = - (ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz); - gho.l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4); - l4_proto = ip4->protocol; - } - else if (PREDICT_TRUE (is_ip6)) - { - ip6_header_t *ip6 = - (ip6_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz); - /* FIXME IPv6 EH traversal */ - gho.l4_hdr_offset = l2hdr_sz + sizeof (ip6_header_t); - l4_proto = ip6->protocol; - } - if (l4_proto == IP_PROTOCOL_TCP) - { - tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b0) + - gho.l4_hdr_offset); - l4_hdr_sz = tcp_header_bytes (tcp); - } - else if (l4_proto == IP_PROTOCOL_UDP) - { - udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b0) + - gho.l4_hdr_offset); - l4_hdr_sz = sizeof (*udp); - } - - if (b0->flags & (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_IS_IP6)) - { - gho.l4_hdr_sz = l4_hdr_sz; - } - - return gho; -} - #endif /* included_gso_h */ /* diff --git a/src/vnet/gso/hdr_offset_parser.h b/src/vnet/gso/hdr_offset_parser.h new file mode 100644 index 00000000000..8f9e0cda984 --- /dev/null +++ b/src/vnet/gso/hdr_offset_parser.h @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2019 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef included_hdr_offset_parser_h +#define included_hdr_offset_parser_h + +#include <vnet/ethernet/ethernet.h> +#include <vnet/ip/ip4_packet.h> +#include <vnet/ip/ip6_packet.h> +#include <vnet/udp/udp.h> +#include <vnet/udp/udp_packet.h> +#include <vnet/vnet.h> +#include <vnet/vxlan/vxlan_packet.h> + +#define foreach_gho_flag \ + _( 0, IP4) \ + _( 1, IP6) \ + _( 2, TCP) \ + _( 3, UDP) \ + _( 4, OUTER_IP4) \ + _( 5, OUTER_IP6) \ + _( 6, OUTER_TCP) \ + _( 7, OUTER_UDP) \ + _( 8, VXLAN_TUNNEL) \ + _( 9, GRE_TUNNEL) \ + _( 10, IPIP_TUNNEL) \ + _( 11, GENEVE_TUNNEL) + +typedef enum gho_flag_t_ +{ +#define _(bit, name) GHO_F_##name = (1 << bit), + foreach_gho_flag +#undef _ +} gho_flag_t; + +#define GHO_F_TUNNEL (GHO_F_VXLAN_TUNNEL | \ + GHO_F_GENEVE_TUNNEL | \ + GHO_F_IPIP_TUNNEL | \ + GHO_F_GRE_TUNNEL) + +#define GHO_F_OUTER_HDR (GHO_F_OUTER_IP4 | \ + GHO_F_OUTER_IP6 | \ + GHO_F_OUTER_TCP | \ + GHO_F_OUTER_UDP) + +#define GHO_F_INNER_HDR (GHO_F_IP4 | \ + GHO_F_IP6 | \ + GHO_F_UDP | \ + GHO_F_TCP) + +typedef struct +{ + i16 outer_l2_hdr_offset; + i16 outer_l3_hdr_offset; + i16 outer_l4_hdr_offset; + u16 outer_l4_hdr_sz; + u16 outer_hdr_sz; + i16 l2_hdr_offset; + i16 l3_hdr_offset; + i16 l4_hdr_offset; + u16 l4_hdr_sz; + u16 hdr_sz; + gho_flag_t gho_flags; +} generic_header_offset_t; + +static_always_inline u8 * +format_generic_header_offset (u8 * s, va_list * args) +{ + generic_header_offset_t *gho = va_arg (*args, generic_header_offset_t *); + + s = format (s, "\n\t"); + if (gho->gho_flags & GHO_F_TUNNEL) + { + if (gho->gho_flags & GHO_F_VXLAN_TUNNEL) + s = format (s, "vxlan-tunnel "); + else if (gho->gho_flags & GHO_F_IPIP_TUNNEL) + s = format (s, "ipip-tunnel "); + else if (gho->gho_flags & GHO_F_GRE_TUNNEL) + s = format (s, "gre-tunnel "); + else if (gho->gho_flags & GHO_F_GENEVE_TUNNEL) + s = format (s, "geneve-tunnel "); + + if (gho->gho_flags & GHO_F_OUTER_IP4) + s = format (s, "outer-ipv4 "); + else if (gho->gho_flags & GHO_F_OUTER_IP6) + s = format (s, "outer-ipv6 "); + + if (gho->gho_flags & GHO_F_OUTER_UDP) + s = format (s, "outer-udp "); + else if (gho->gho_flags & GHO_F_OUTER_TCP) + s = format (s, "outer-tcp "); + + s = format (s, "outer-hdr-sz %u outer-l2-hdr-offset %d " + "outer-l3-hdr-offset %d outer-l4-hdr-offset %d " + "outer-l4-hdr-sz %u\n\t", + gho->outer_hdr_sz, gho->outer_l2_hdr_offset, + gho->outer_l3_hdr_offset, gho->outer_l4_hdr_offset, + gho->outer_l4_hdr_sz); + } + + if (gho->gho_flags & GHO_F_IP4) + s = format (s, "ipv4 "); + else if (gho->gho_flags & GHO_F_IP6) + s = format (s, "ipv6 "); + + if (gho->gho_flags & GHO_F_TCP) + s = format (s, "tcp "); + else if (gho->gho_flags & GHO_F_UDP) + s = format (s, "udp "); + + s = format (s, "hdr-sz %u l2-hdr-offset %d " + "l3-hdr-offset %d l4-hdr-offset %d " + "l4-hdr-sz %u", + gho->hdr_sz, gho->l2_hdr_offset, gho->l3_hdr_offset, + gho->l4_hdr_offset, gho->l4_hdr_sz); + + return s; +} + +static_always_inline void +vnet_get_inner_header (vlib_buffer_t * b0, generic_header_offset_t * gho) +{ + if ((gho->gho_flags & GHO_F_TUNNEL) + && (gho->gho_flags & GHO_F_OUTER_HDR) + && (b0->current_data == gho->outer_l2_hdr_offset)) + vlib_buffer_advance (b0, gho->outer_hdr_sz); +} + +static_always_inline void +vnet_get_outer_header (vlib_buffer_t * b0, generic_header_offset_t * gho) +{ + if ((gho->gho_flags & GHO_F_TUNNEL) + && (gho->gho_flags & GHO_F_OUTER_HDR) + && (b0->current_data == gho->l2_hdr_offset)) + vlib_buffer_advance (b0, -gho->outer_hdr_sz); +} + +static_always_inline void +vnet_geneve_inner_header_parser_inline (vlib_buffer_t * b0, + generic_header_offset_t * gho) +{ + /* not supported yet */ + if ((gho->gho_flags & GHO_F_GENEVE_TUNNEL) == 0) + return; + + ASSERT (0); +} + +static_always_inline void +vnet_gre_inner_header_parser_inline (vlib_buffer_t * b0, + generic_header_offset_t * gho) +{ + /* not supported yet */ + if ((gho->gho_flags & GHO_F_GRE_TUNNEL) == 0) + return; + + ASSERT (0); +} + +static_always_inline void +vnet_ipip_inner_header_parser_inline (vlib_buffer_t * b0, + generic_header_offset_t * gho) +{ + /* not supported yet */ + if ((gho->gho_flags & GHO_F_IPIP_TUNNEL) == 0) + return; + + ASSERT (0); +} + +static_always_inline void +vnet_vxlan_inner_header_parser_inline (vlib_buffer_t * b0, + generic_header_offset_t * gho) +{ + u8 l4_proto = 0; + u8 l4_hdr_sz = 0; + + if ((gho->gho_flags & GHO_F_VXLAN_TUNNEL) == 0) + return; + + gho->outer_l2_hdr_offset = gho->l2_hdr_offset; + gho->outer_l3_hdr_offset = gho->l3_hdr_offset; + gho->outer_l4_hdr_offset = gho->l4_hdr_offset; + gho->outer_l4_hdr_sz = gho->l4_hdr_sz; + gho->outer_hdr_sz = gho->hdr_sz; + + gho->l2_hdr_offset = 0; + gho->l3_hdr_offset = 0; + gho->l4_hdr_offset = 0; + gho->l4_hdr_sz = 0; + gho->hdr_sz = 0; + + if (gho->gho_flags & GHO_F_IP4) + { + gho->gho_flags |= GHO_F_OUTER_IP4; + } + else if (gho->gho_flags & GHO_F_IP6) + { + gho->gho_flags |= GHO_F_OUTER_IP6; + } + + if (gho->gho_flags & GHO_F_UDP) + { + gho->gho_flags |= GHO_F_OUTER_UDP; + } + + gho->gho_flags &= ~GHO_F_INNER_HDR; + + vnet_get_inner_header (b0, gho); + + gho->l2_hdr_offset = b0->current_data; + + ethernet_header_t *eh = (ethernet_header_t *) vlib_buffer_get_current (b0); + u16 ethertype = clib_net_to_host_u16 (eh->type); + u16 l2hdr_sz = sizeof (ethernet_header_t); + + if (ethernet_frame_is_tagged (ethertype)) + { + ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1); + + ethertype = clib_net_to_host_u16 (vlan->type); + l2hdr_sz += sizeof (*vlan); + if (ethertype == ETHERNET_TYPE_VLAN) + { + vlan++; + ethertype = clib_net_to_host_u16 (vlan->type); + l2hdr_sz += sizeof (*vlan); + } + } + + gho->l3_hdr_offset = l2hdr_sz; + + if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4)) + { + ip4_header_t *ip4 = + (ip4_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset); + gho->l4_hdr_offset = gho->l3_hdr_offset + ip4_header_bytes (ip4); + l4_proto = ip4->protocol; + gho->gho_flags |= GHO_F_IP4; + } + else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) + { + ip6_header_t *ip6 = + (ip6_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset); + /* FIXME IPv6 EH traversal */ + gho->l4_hdr_offset = gho->l3_hdr_offset + sizeof (ip6_header_t); + l4_proto = ip6->protocol; + gho->gho_flags |= GHO_F_IP6; + } + if (l4_proto == IP_PROTOCOL_TCP) + { + tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b0) + + gho->l4_hdr_offset); + l4_hdr_sz = tcp_header_bytes (tcp); + + gho->gho_flags |= GHO_F_TCP; + + } + else if (l4_proto == IP_PROTOCOL_UDP) + { + udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b0) + + gho->l4_hdr_offset); + l4_hdr_sz = sizeof (*udp); + + gho->gho_flags |= GHO_F_UDP; + } + + gho->l4_hdr_sz = l4_hdr_sz; + gho->hdr_sz += gho->l4_hdr_offset + l4_hdr_sz; + + vnet_get_outer_header (b0, gho); +} + +static_always_inline void +vnet_generic_inner_header_parser_inline (vlib_buffer_t * b0, + generic_header_offset_t * gho) +{ + + if (gho->gho_flags & GHO_F_VXLAN_TUNNEL) + vnet_vxlan_inner_header_parser_inline (b0, gho); + else if (gho->gho_flags & GHO_F_IPIP_TUNNEL) + vnet_ipip_inner_header_parser_inline (b0, gho); + else if (gho->gho_flags & GHO_F_GRE_TUNNEL) + vnet_gre_inner_header_parser_inline (b0, gho); + else if (gho->gho_flags & GHO_F_GENEVE_TUNNEL) + vnet_geneve_inner_header_parser_inline (b0, gho); +} + +static_always_inline void +vnet_generic_outer_header_parser_inline (vlib_buffer_t * b0, + generic_header_offset_t * gho) +{ + u8 l4_proto = 0; + u8 l4_hdr_sz = 0; + + ethernet_header_t *eh = (ethernet_header_t *) vlib_buffer_get_current (b0); + u16 ethertype = clib_net_to_host_u16 (eh->type); + u16 l2hdr_sz = sizeof (ethernet_header_t); + + if (ethernet_frame_is_tagged (ethertype)) + { + ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1); + + ethertype = clib_net_to_host_u16 (vlan->type); + l2hdr_sz += sizeof (*vlan); + if (ethertype == ETHERNET_TYPE_VLAN) + { + vlan++; + ethertype = clib_net_to_host_u16 (vlan->type); + l2hdr_sz += sizeof (*vlan); + } + } + + gho->l2_hdr_offset = b0->current_data; + gho->l3_hdr_offset = l2hdr_sz; + + if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4)) + { + ip4_header_t *ip4 = + (ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz); + gho->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4); + l4_proto = ip4->protocol; + gho->gho_flags |= GHO_F_IP4; + } + else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) + { + ip6_header_t *ip6 = + (ip6_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz); + /* FIXME IPv6 EH traversal */ + gho->l4_hdr_offset = l2hdr_sz + sizeof (ip6_header_t); + l4_proto = ip6->protocol; + gho->gho_flags |= GHO_F_IP6; + } + if (l4_proto == IP_PROTOCOL_TCP) + { + tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b0) + + gho->l4_hdr_offset); + l4_hdr_sz = tcp_header_bytes (tcp); + + gho->gho_flags |= GHO_F_TCP; + } + else if (l4_proto == IP_PROTOCOL_UDP) + { + udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b0) + + gho->l4_hdr_offset); + l4_hdr_sz = sizeof (*udp); + + gho->gho_flags |= GHO_F_UDP; + + if (UDP_DST_PORT_vxlan == clib_net_to_host_u16 (udp->dst_port)) + { + gho->gho_flags |= GHO_F_VXLAN_TUNNEL; + gho->hdr_sz += sizeof (vxlan_header_t); + } + else if (UDP_DST_PORT_geneve == clib_net_to_host_u16 (udp->dst_port)) + { + gho->gho_flags |= GHO_F_GENEVE_TUNNEL; + } + } + else if ((l4_proto == IP_PROTOCOL_IP_IN_IP) + || (l4_proto == IP_PROTOCOL_IPV6)) + { + l4_hdr_sz = 0; + gho->gho_flags |= GHO_F_IPIP_TUNNEL; + } + else if (l4_proto == IP_PROTOCOL_GRE) + { + l4_hdr_sz = 0; + gho->gho_flags |= GHO_F_GRE_TUNNEL; + } + + gho->l4_hdr_sz = l4_hdr_sz; + gho->hdr_sz += gho->l4_hdr_offset + l4_hdr_sz; +} + +static_always_inline void +vnet_generic_header_offset_parser (vlib_buffer_t * b0, + generic_header_offset_t * gho) +{ + vnet_generic_outer_header_parser_inline (b0, gho); + + if (gho->gho_flags & GHO_F_TUNNEL) + { + vnet_generic_inner_header_parser_inline (b0, gho); + } +} + +#endif /* included_hdr_offset_parser_h */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vnet/gso/node.c b/src/vnet/gso/node.c index b3125fee4c2..663f9cc602d 100644 --- a/src/vnet/gso/node.c +++ b/src/vnet/gso/node.c @@ -19,6 +19,7 @@ #include <vnet/ethernet/ethernet.h> #include <vnet/feature/feature.h> #include <vnet/gso/gso.h> +#include <vnet/gso/hdr_offset_parser.h> #include <vnet/ip/icmp46_packet.h> #include <vnet/ip/ip4.h> #include <vnet/ip/ip6.h> @@ -29,6 +30,7 @@ typedef struct u32 flags; u16 gso_size; u8 gso_l4_hdr_sz; + generic_header_offset_t gho; } gso_trace_t; static u8 * @@ -40,32 +42,126 @@ format_gso_trace (u8 * s, va_list * args) if (t->flags & VNET_BUFFER_F_GSO) { - s = format (s, "gso_sz %d gso_l4_hdr_sz %d", - t->gso_size, t->gso_l4_hdr_sz); + s = format (s, "gso_sz %d gso_l4_hdr_sz %d %U", + t->gso_size, t->gso_l4_hdr_sz, format_generic_header_offset, + &t->gho); } else { - s = format (s, "non-gso buffer"); + s = + format (s, "non-gso buffer %U", format_generic_header_offset, + &t->gho); } return s; } +static_always_inline void +tso_segment_vxlan_tunnel_headers_fixup (vlib_main_t * vm, vlib_buffer_t * b, + generic_header_offset_t * gho) +{ + u8 proto = 0; + ip4_header_t *ip4 = 0; + ip6_header_t *ip6 = 0; + udp_header_t *udp = 0; + + ip4 = + (ip4_header_t *) (vlib_buffer_get_current (b) + gho->outer_l3_hdr_offset); + ip6 = + (ip6_header_t *) (vlib_buffer_get_current (b) + gho->outer_l3_hdr_offset); + udp = + (udp_header_t *) (vlib_buffer_get_current (b) + gho->outer_l4_hdr_offset); + + if (gho->gho_flags & GHO_F_OUTER_IP4) + { + proto = ip4->protocol; + ip4->length = + clib_host_to_net_u16 (b->current_length - gho->outer_l3_hdr_offset); + ip4->checksum = ip4_header_checksum (ip4); + } + else if (gho->gho_flags & GHO_F_OUTER_IP6) + { + proto = ip6->protocol; + ip6->payload_length = + clib_host_to_net_u16 (b->current_length - gho->outer_l4_hdr_offset); + } + if (proto == IP_PROTOCOL_UDP) + { + int bogus; + udp->length = + clib_host_to_net_u16 (b->current_length - gho->outer_l4_hdr_offset); + udp->checksum = 0; + if (gho->gho_flags & GHO_F_OUTER_IP6) + { + udp->checksum = + ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus); + } + else if (gho->gho_flags & GHO_F_OUTER_IP4) + { + udp->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4); + } + b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM; + } +} + +static_always_inline u16 +tso_segment_vxlan_tunnel_fixup (vlib_main_t * vm, + vnet_interface_per_thread_data_t * ptd, + vlib_buffer_t * sb0, + generic_header_offset_t * gho) +{ + u16 n_tx_bufs = vec_len (ptd->split_buffers); + u16 i = 0, n_tx_bytes = 0; + + while (i < n_tx_bufs) + { + vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[i]); + vnet_get_outer_header (b0, gho); + clib_memcpy_fast (vlib_buffer_get_current (b0), + vlib_buffer_get_current (sb0), gho->outer_hdr_sz); + + tso_segment_vxlan_tunnel_headers_fixup (vm, b0, gho); + n_tx_bytes += gho->outer_hdr_sz; + i++; + } + return n_tx_bytes; +} + static_always_inline u16 tso_alloc_tx_bufs (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd, vlib_buffer_t * b0, u32 n_bytes_b0, u16 l234_sz, - u16 gso_size, gso_header_offset_t * gho) + u16 gso_size, u16 first_data_size, + generic_header_offset_t * gho) { - u16 size = + u16 n_alloc, size; + u16 first_packet_length = l234_sz + first_data_size; + + /* + * size is the amount of data per segmented buffer except the 1st + * segmented buffer. + * l2_hdr_offset is an offset == current_data of vlib_buffer_t. + * l234_sz is hdr_sz from l2_hdr_offset. + */ + size = clib_min (gso_size, vlib_buffer_get_default_data_size (vm) - l234_sz - gho->l2_hdr_offset); - /* rounded-up division */ - u16 n_bufs = (n_bytes_b0 - l234_sz + (size - 1)) / size; - u16 n_alloc; + /* + * First segmented buffer length is calculated separately. + * As it may contain less data than gso_size (when gso_size is + * greater than current_length of 1st buffer from GSO chained + * buffers) and/or size calculated above. + */ + u16 n_bufs = 1; + + /* + * Total packet length minus first packet length including l234 header. + * rounded-up division + */ + ASSERT (n_bytes_b0 > first_packet_length); + n_bufs += ((n_bytes_b0 - first_packet_length + (size - 1)) / size); - ASSERT (n_bufs > 0); vec_validate (ptd->split_buffers, n_bufs - 1); n_alloc = vlib_buffer_alloc (vm, ptd->split_buffers, n_bufs); @@ -96,7 +192,7 @@ tso_init_buf_from_template (vlib_main_t * vm, vlib_buffer_t * nb0, vlib_buffer_t * b0, u16 template_data_sz, u16 gso_size, u8 ** p_dst_ptr, u16 * p_dst_left, u32 next_tcp_seq, u32 flags, - gso_header_offset_t * gho) + generic_header_offset_t * gho) { tso_init_buf_from_template_base (nb0, b0, flags, template_data_sz); @@ -112,8 +208,9 @@ tso_init_buf_from_template (vlib_main_t * vm, vlib_buffer_t * nb0, } static_always_inline void -tso_fixup_segmented_buf (vlib_buffer_t * b0, u8 tcp_flags, int is_ip6, - gso_header_offset_t * gho) +tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags, + int is_ip6, generic_header_offset_t * gho, + u32 do_csums) { ip4_header_t *ip4 = (ip4_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset); @@ -125,13 +222,37 @@ tso_fixup_segmented_buf (vlib_buffer_t * b0, u8 tcp_flags, int is_ip6, tcp->flags = tcp_flags; if (is_ip6) - ip6->payload_length = - clib_host_to_net_u16 (b0->current_length - - (gho->l4_hdr_offset - gho->l2_hdr_offset)); + { + ip6->payload_length = + clib_host_to_net_u16 (b0->current_length - + (gho->l4_hdr_offset - gho->l2_hdr_offset)); + if (do_csums && (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)) + { + int bogus = 0; + tcp->checksum = 0; + tcp->checksum = + ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6, &bogus); + b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + } + } else - ip4->length = - clib_host_to_net_u16 (b0->current_length - - (gho->l3_hdr_offset - gho->l2_hdr_offset)); + { + ip4->length = + clib_host_to_net_u16 (b0->current_length - + (gho->l3_hdr_offset - gho->l2_hdr_offset)); + if (do_csums) + { + if (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) + ip4->checksum = ip4_header_checksum (ip4); + if (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM) + { + tcp->checksum = 0; + tcp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip4); + } + b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM; + b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM; + } + } } /** @@ -145,13 +266,13 @@ tso_fixup_segmented_buf (vlib_buffer_t * b0, u8 tcp_flags, int is_ip6, static_always_inline u32 tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd, - u32 sbi0, vlib_buffer_t * sb0, gso_header_offset_t * gho, - u32 n_bytes_b0, int is_ip6) + u32 sbi0, vlib_buffer_t * sb0, + generic_header_offset_t * gho, u32 n_bytes_b0, int is_ip6, + u32 do_csums) { u32 n_tx_bytes = 0; u16 gso_size = vnet_buffer2 (sb0)->gso_size; - int l4_hdr_sz = gho->l4_hdr_sz; u8 save_tcp_flags = 0; u8 tcp_flags_no_fin_psh = 0; u32 next_tcp_seq = 0; @@ -166,12 +287,13 @@ tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd, u32 default_bflags = sb0->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT); - u16 l234_sz = gho->l4_hdr_offset + l4_hdr_sz - gho->l2_hdr_offset; + u16 l234_sz = gho->hdr_sz; int first_data_size = clib_min (gso_size, sb0->current_length - l234_sz); next_tcp_seq += first_data_size; if (PREDICT_FALSE - (!tso_alloc_tx_bufs (vm, ptd, sb0, n_bytes_b0, l234_sz, gso_size, gho))) + (!tso_alloc_tx_bufs + (vm, ptd, sb0, n_bytes_b0, l234_sz, gso_size, first_data_size, gho))) return 0; vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[0]); @@ -194,7 +316,8 @@ tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd, src_ptr = vlib_buffer_get_current (sb0) + l234_sz + first_data_size; src_left = sb0->current_length - l234_sz - first_data_size; - tso_fixup_segmented_buf (b0, tcp_flags_no_fin_psh, is_ip6, gho); + tso_fixup_segmented_buf (vm, b0, tcp_flags_no_fin_psh, is_ip6, gho, + do_csums); /* grab a second buffer and prepare the loop */ ASSERT (dbi < vec_len (ptd->split_buffers)); @@ -243,8 +366,8 @@ tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd, if (0 == dst_left && total_src_left) { n_tx_bytes += cdb0->current_length; - tso_fixup_segmented_buf (cdb0, tcp_flags_no_fin_psh, is_ip6, - gho); + tso_fixup_segmented_buf (vm, cdb0, tcp_flags_no_fin_psh, is_ip6, + gho, do_csums); ASSERT (dbi < vec_len (ptd->split_buffers)); cdb0 = vlib_get_buffer (vm, ptd->split_buffers[dbi++]); tso_init_buf_from_template (vm, cdb0, b0, l234_sz, @@ -253,7 +376,8 @@ tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd, } } - tso_fixup_segmented_buf (cdb0, save_tcp_flags, is_ip6, gho); + tso_fixup_segmented_buf (vm, cdb0, save_tcp_flags, is_ip6, gho, + do_csums); n_tx_bytes += cdb0->current_length; } @@ -372,6 +496,7 @@ vnet_gso_node_inline (vlib_main_t * vm, t0->flags = b[0]->flags & VNET_BUFFER_F_GSO; t0->gso_size = vnet_buffer2 (b[0])->gso_size; t0->gso_l4_hdr_sz = vnet_buffer2 (b[0])->gso_l4_hdr_sz; + vnet_generic_header_offset_parser (b[0], &t0->gho); } if (b[1]->flags & VLIB_BUFFER_IS_TRACED) { @@ -379,6 +504,7 @@ vnet_gso_node_inline (vlib_main_t * vm, t1->flags = b[1]->flags & VNET_BUFFER_F_GSO; t1->gso_size = vnet_buffer2 (b[1])->gso_size; t1->gso_l4_hdr_sz = vnet_buffer2 (b[1])->gso_l4_hdr_sz; + vnet_generic_header_offset_parser (b[1], &t1->gho); } if (b[2]->flags & VLIB_BUFFER_IS_TRACED) { @@ -386,6 +512,7 @@ vnet_gso_node_inline (vlib_main_t * vm, t2->flags = b[2]->flags & VNET_BUFFER_F_GSO; t2->gso_size = vnet_buffer2 (b[2])->gso_size; t2->gso_l4_hdr_sz = vnet_buffer2 (b[2])->gso_l4_hdr_sz; + vnet_generic_header_offset_parser (b[2], &t2->gho); } if (b[3]->flags & VLIB_BUFFER_IS_TRACED) { @@ -393,6 +520,7 @@ vnet_gso_node_inline (vlib_main_t * vm, t3->flags = b[3]->flags & VNET_BUFFER_F_GSO; t3->gso_size = vnet_buffer2 (b[3])->gso_size; t3->gso_l4_hdr_sz = vnet_buffer2 (b[3])->gso_l4_hdr_sz; + vnet_generic_header_offset_parser (b[3], &t3->gho); } from += 4; @@ -419,6 +547,7 @@ vnet_gso_node_inline (vlib_main_t * vm, vnet_hw_interface_t *hi0; u32 next0 = 0; u32 do_segmentation0 = 0; + u32 do_csums = 0; swif0 = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; if (PREDICT_FALSE (hi->sw_if_index != swif0)) @@ -444,6 +573,7 @@ vnet_gso_node_inline (vlib_main_t * vm, t0->flags = b[0]->flags & VNET_BUFFER_F_GSO; t0->gso_size = vnet_buffer2 (b[0])->gso_size; t0->gso_l4_hdr_sz = vnet_buffer2 (b[0])->gso_l4_hdr_sz; + vnet_generic_header_offset_parser (b[0], &t0->gho); } if (do_segmentation0) @@ -458,14 +588,39 @@ vnet_gso_node_inline (vlib_main_t * vm, to_next -= 1; n_left_to_next += 1; /* undo the counting. */ - gso_header_offset_t gho; + generic_header_offset_t gho = { 0 }; u32 n_bytes_b0 = vlib_buffer_length_in_chain (vm, b[0]); u32 n_tx_bytes = 0; - gho = vnet_gso_header_offset_parser (b[0], is_ip6); + vnet_generic_header_offset_parser (b[0], &gho); + + if (PREDICT_FALSE (gho.gho_flags & GHO_F_TUNNEL)) + { + if (PREDICT_FALSE + ((gho.gho_flags & GHO_F_VXLAN_TUNNEL) == 0)) + { + /* not supported yet */ + drop_one_buffer_and_count (vm, vnm, node, from - 1, + hi->sw_if_index, + VNET_INTERFACE_OUTPUT_ERROR_UNHANDLED_GSO_TYPE); + b += 1; + continue; + } + + vnet_get_inner_header (b[0], &gho); + + n_bytes_b0 -= gho.outer_hdr_sz; + /* + * In case of tunnel encapsulated packet, we will + * calculate the checksums for segmented inner packets. + */ + do_csums = 1; + is_ip6 = (gho.gho_flags & GHO_F_IP6) != 0; + } + n_tx_bytes = tso_segment_buffer (vm, ptd, bi0, b[0], &gho, n_bytes_b0, - is_ip6); + is_ip6, do_csums); if (PREDICT_FALSE (n_tx_bytes == 0)) { @@ -476,6 +631,13 @@ vnet_gso_node_inline (vlib_main_t * vm, continue; } + if (PREDICT_FALSE (gho.gho_flags & GHO_F_VXLAN_TUNNEL)) + { + vnet_get_outer_header (b[0], &gho); + n_tx_bytes += + tso_segment_vxlan_tunnel_fixup (vm, ptd, b[0], &gho); + } + u16 n_tx_bufs = vec_len (ptd->split_buffers); u32 *from_seg = ptd->split_buffers; @@ -489,6 +651,7 @@ vnet_gso_node_inline (vlib_main_t * vm, { sbi0 = to_next[0] = from_seg[0]; sb0 = vlib_get_buffer (vm, sbi0); + ASSERT (sb0->current_length > 0); to_next += 1; from_seg += 1; n_left_to_next -= 1; @@ -509,6 +672,7 @@ vnet_gso_node_inline (vlib_main_t * vm, { sbi0 = to_next[0] = from_seg[0]; sb0 = vlib_get_buffer (vm, sbi0); + ASSERT (sb0->current_length > 0); to_next += 1; from_seg += 1; n_left_to_next -= 1; |