diff options
Diffstat (limited to 'lib/librte_gso')
| -rw-r--r-- | lib/librte_gso/Makefile | 54 | ||||
| -rw-r--r-- | lib/librte_gso/gso_common.c | 153 | ||||
| -rw-r--r-- | lib/librte_gso/gso_common.h | 171 | ||||
| -rw-r--r-- | lib/librte_gso/gso_tcp4.c | 102 | ||||
| -rw-r--r-- | lib/librte_gso/gso_tcp4.h | 74 | ||||
| -rw-r--r-- | lib/librte_gso/gso_tunnel_tcp4.c | 126 | ||||
| -rw-r--r-- | lib/librte_gso/gso_tunnel_tcp4.h | 75 | ||||
| -rw-r--r-- | lib/librte_gso/rte_gso.c | 110 | ||||
| -rw-r--r-- | lib/librte_gso/rte_gso.h | 148 | ||||
| -rw-r--r-- | lib/librte_gso/rte_gso_version.map | 7 |
10 files changed, 1020 insertions, 0 deletions
diff --git a/lib/librte_gso/Makefile b/lib/librte_gso/Makefile new file mode 100644 index 00000000..ea5ad742 --- /dev/null +++ b/lib/librte_gso/Makefile @@ -0,0 +1,54 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_gso.a + +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 +LDLIBS += -lrte_eal -lrte_mbuf -lrte_ethdev -lrte_net +LDLIBS += -lrte_mempool + +EXPORT_MAP := rte_gso_version.map + +LIBABIVER := 1 + +#source files +SRCS-$(CONFIG_RTE_LIBRTE_GSO) += rte_gso.c +SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_common.c +SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_tcp4.c +SRCS-$(CONFIG_RTE_LIBRTE_GSO) += gso_tunnel_tcp4.c + +# install this header file +SYMLINK-$(CONFIG_RTE_LIBRTE_GSO)-include += rte_gso.h + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/lib/librte_gso/gso_common.c b/lib/librte_gso/gso_common.c new file mode 100644 index 00000000..ee75d4cd --- /dev/null +++ b/lib/librte_gso/gso_common.c @@ -0,0 +1,153 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdbool.h> +#include <errno.h> + +#include <rte_memcpy.h> +#include <rte_mempool.h> + +#include "gso_common.h" + +static inline void +hdr_segment_init(struct rte_mbuf *hdr_segment, struct rte_mbuf *pkt, + uint16_t pkt_hdr_offset) +{ + /* Copy MBUF metadata */ + hdr_segment->nb_segs = 1; + hdr_segment->port = pkt->port; + hdr_segment->ol_flags = pkt->ol_flags; + hdr_segment->packet_type = pkt->packet_type; + hdr_segment->pkt_len = pkt_hdr_offset; + hdr_segment->data_len = pkt_hdr_offset; + hdr_segment->tx_offload = pkt->tx_offload; + + /* Copy the packet header */ + rte_memcpy(rte_pktmbuf_mtod(hdr_segment, char *), + rte_pktmbuf_mtod(pkt, char *), + pkt_hdr_offset); +} + +static inline void +free_gso_segment(struct rte_mbuf **pkts, uint16_t nb_pkts) +{ + uint16_t i; + + for (i = 0; i < nb_pkts; i++) + rte_pktmbuf_free(pkts[i]); +} + +int +gso_do_segment(struct rte_mbuf *pkt, + uint16_t pkt_hdr_offset, + uint16_t pyld_unit_size, + struct rte_mempool *direct_pool, + struct rte_mempool *indirect_pool, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out) +{ + struct rte_mbuf *pkt_in; + struct rte_mbuf *hdr_segment, *pyld_segment, *prev_segment; + uint16_t pkt_in_data_pos, segment_bytes_remaining; + uint16_t pyld_len, nb_segs; + bool more_in_pkt, more_out_segs; + + pkt_in = pkt; + nb_segs = 0; + more_in_pkt = 1; + pkt_in_data_pos = pkt_hdr_offset; + + while (more_in_pkt) { + if (unlikely(nb_segs >= nb_pkts_out)) { + free_gso_segment(pkts_out, nb_segs); + return -EINVAL; + } + + /* Allocate a direct MBUF */ + hdr_segment = rte_pktmbuf_alloc(direct_pool); + if (unlikely(hdr_segment == NULL)) { + free_gso_segment(pkts_out, nb_segs); + return -ENOMEM; + } + /* Fill the packet header */ + hdr_segment_init(hdr_segment, pkt, pkt_hdr_offset); + + prev_segment = hdr_segment; + segment_bytes_remaining = pyld_unit_size; + more_out_segs = 1; + + while (more_out_segs && more_in_pkt) { + /* Allocate an indirect MBUF */ + pyld_segment = rte_pktmbuf_alloc(indirect_pool); + if (unlikely(pyld_segment == NULL)) { + rte_pktmbuf_free(hdr_segment); + free_gso_segment(pkts_out, nb_segs); + return -ENOMEM; + } + /* Attach to current MBUF segment of pkt */ + rte_pktmbuf_attach(pyld_segment, pkt_in); + + prev_segment->next = pyld_segment; + prev_segment = pyld_segment; + + pyld_len = segment_bytes_remaining; + if (pyld_len + pkt_in_data_pos > pkt_in->data_len) + pyld_len = pkt_in->data_len - pkt_in_data_pos; + + pyld_segment->data_off = pkt_in_data_pos + + pkt_in->data_off; + pyld_segment->data_len = pyld_len; + + /* Update header segment */ + hdr_segment->pkt_len += pyld_len; + hdr_segment->nb_segs++; + + pkt_in_data_pos += pyld_len; + segment_bytes_remaining -= pyld_len; + + /* Finish processing a MBUF segment of pkt */ + if (pkt_in_data_pos == pkt_in->data_len) { + pkt_in = pkt_in->next; + pkt_in_data_pos = 0; + if (pkt_in == NULL) + more_in_pkt = 0; + } + + /* Finish generating a GSO segment */ + if (segment_bytes_remaining == 0) + more_out_segs = 0; + } + pkts_out[nb_segs++] = hdr_segment; + } + return nb_segs; +} diff --git a/lib/librte_gso/gso_common.h b/lib/librte_gso/gso_common.h new file mode 100644 index 00000000..145ea495 --- /dev/null +++ b/lib/librte_gso/gso_common.h @@ -0,0 +1,171 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _GSO_COMMON_H_ +#define _GSO_COMMON_H_ + +#include <stdint.h> + +#include <rte_mbuf.h> +#include <rte_ip.h> +#include <rte_tcp.h> +#include <rte_udp.h> + +#define IS_FRAGMENTED(frag_off) (((frag_off) & IPV4_HDR_OFFSET_MASK) != 0 \ + || ((frag_off) & IPV4_HDR_MF_FLAG) == IPV4_HDR_MF_FLAG) + +#define TCP_HDR_PSH_MASK ((uint8_t)0x08) +#define TCP_HDR_FIN_MASK ((uint8_t)0x01) + +#define IS_IPV4_TCP(flag) (((flag) & (PKT_TX_TCP_SEG | PKT_TX_IPV4)) == \ + (PKT_TX_TCP_SEG | PKT_TX_IPV4)) + +#define IS_IPV4_VXLAN_TCP4(flag) (((flag) & (PKT_TX_TCP_SEG | PKT_TX_IPV4 | \ + PKT_TX_OUTER_IPV4 | PKT_TX_TUNNEL_VXLAN)) == \ + (PKT_TX_TCP_SEG | PKT_TX_IPV4 | PKT_TX_OUTER_IPV4 | \ + PKT_TX_TUNNEL_VXLAN)) + +#define IS_IPV4_GRE_TCP4(flag) (((flag) & (PKT_TX_TCP_SEG | PKT_TX_IPV4 | \ + PKT_TX_OUTER_IPV4 | PKT_TX_TUNNEL_GRE)) == \ + (PKT_TX_TCP_SEG | PKT_TX_IPV4 | PKT_TX_OUTER_IPV4 | \ + PKT_TX_TUNNEL_GRE)) + +/** + * Internal function which updates the UDP header of a packet, following + * segmentation. This is required to update the header's datagram length field. + * + * @param pkt + * The packet containing the UDP header. + * @param udp_offset + * The offset of the UDP header from the start of the packet. + */ +static inline void +update_udp_header(struct rte_mbuf *pkt, uint16_t udp_offset) +{ + struct udp_hdr *udp_hdr; + + udp_hdr = (struct udp_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + udp_offset); + udp_hdr->dgram_len = rte_cpu_to_be_16(pkt->pkt_len - udp_offset); +} + +/** + * Internal function which updates the TCP header of a packet, following + * segmentation. This is required to update the header's 'sent' sequence + * number, and also to clear 'PSH' and 'FIN' flags for non-tail segments. + * + * @param pkt + * The packet containing the TCP header. + * @param l4_offset + * The offset of the TCP header from the start of the packet. + * @param sent_seq + * The sent sequence number. + * @param non-tail + * Indicates whether or not this is a tail segment. + */ +static inline void +update_tcp_header(struct rte_mbuf *pkt, uint16_t l4_offset, uint32_t sent_seq, + uint8_t non_tail) +{ + struct tcp_hdr *tcp_hdr; + + tcp_hdr = (struct tcp_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + l4_offset); + tcp_hdr->sent_seq = rte_cpu_to_be_32(sent_seq); + if (likely(non_tail)) + tcp_hdr->tcp_flags &= (~(TCP_HDR_PSH_MASK | + TCP_HDR_FIN_MASK)); +} + +/** + * Internal function which updates the IPv4 header of a packet, following + * segmentation. This is required to update the header's 'total_length' field, + * to reflect the reduced length of the now-segmented packet. Furthermore, the + * header's 'packet_id' field must be updated to reflect the new ID of the + * now-segmented packet. + * + * @param pkt + * The packet containing the IPv4 header. + * @param l3_offset + * The offset of the IPv4 header from the start of the packet. + * @param id + * The new ID of the packet. + */ +static inline void +update_ipv4_header(struct rte_mbuf *pkt, uint16_t l3_offset, uint16_t id) +{ + struct ipv4_hdr *ipv4_hdr; + + ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + l3_offset); + ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len - l3_offset); + ipv4_hdr->packet_id = rte_cpu_to_be_16(id); +} + +/** + * Internal function which divides the input packet into small segments. + * Each of the newly-created segments is organized as a two-segment MBUF, + * where the first segment is a standard mbuf, which stores a copy of + * packet header, and the second is an indirect mbuf which points to a + * section of data in the input packet. + * + * @param pkt + * Packet to segment. + * @param pkt_hdr_offset + * Packet header offset, measured in bytes. + * @param pyld_unit_size + * The max payload length of a GSO segment. + * @param direct_pool + * MBUF pool used for allocating direct buffers for output segments. + * @param indirect_pool + * MBUF pool used for allocating indirect buffers for output segments. + * @param pkts_out + * Pointer array used to keep the mbuf addresses of output segments. If + * the memory space in pkts_out is insufficient, gso_do_segment() fails + * and returns -EINVAL. + * @param nb_pkts_out + * The max number of items that pkts_out can keep. + * + * @return + * - The number of segments created in the event of success. + * - Return -ENOMEM if run out of memory in MBUF pools. + * - Return -EINVAL for invalid parameters. + */ +int gso_do_segment(struct rte_mbuf *pkt, + uint16_t pkt_hdr_offset, + uint16_t pyld_unit_size, + struct rte_mempool *direct_pool, + struct rte_mempool *indirect_pool, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out); +#endif diff --git a/lib/librte_gso/gso_tcp4.c b/lib/librte_gso/gso_tcp4.c new file mode 100644 index 00000000..0c628cb1 --- /dev/null +++ b/lib/librte_gso/gso_tcp4.c @@ -0,0 +1,102 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "gso_common.h" +#include "gso_tcp4.h" + +static void +update_ipv4_tcp_headers(struct rte_mbuf *pkt, uint8_t ipid_delta, + struct rte_mbuf **segs, uint16_t nb_segs) +{ + struct ipv4_hdr *ipv4_hdr; + struct tcp_hdr *tcp_hdr; + uint32_t sent_seq; + uint16_t id, tail_idx, i; + uint16_t l3_offset = pkt->l2_len; + uint16_t l4_offset = l3_offset + pkt->l3_len; + + ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char*) + + l3_offset); + tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len); + id = rte_be_to_cpu_16(ipv4_hdr->packet_id); + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq); + tail_idx = nb_segs - 1; + + for (i = 0; i < nb_segs; i++) { + update_ipv4_header(segs[i], l3_offset, id); + update_tcp_header(segs[i], l4_offset, sent_seq, i < tail_idx); + id += ipid_delta; + sent_seq += (segs[i]->pkt_len - segs[i]->data_len); + } +} + +int +gso_tcp4_segment(struct rte_mbuf *pkt, + uint16_t gso_size, + uint8_t ipid_delta, + struct rte_mempool *direct_pool, + struct rte_mempool *indirect_pool, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out) +{ + struct ipv4_hdr *ipv4_hdr; + uint16_t pyld_unit_size, hdr_offset; + uint16_t frag_off; + int ret; + + /* Don't process the fragmented packet */ + ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + pkt->l2_len); + frag_off = rte_be_to_cpu_16(ipv4_hdr->fragment_offset); + if (unlikely(IS_FRAGMENTED(frag_off))) { + pkts_out[0] = pkt; + return 1; + } + + /* Don't process the packet without data */ + hdr_offset = pkt->l2_len + pkt->l3_len + pkt->l4_len; + if (unlikely(hdr_offset >= pkt->pkt_len)) { + pkts_out[0] = pkt; + return 1; + } + + pyld_unit_size = gso_size - hdr_offset; + + /* Segment the payload */ + ret = gso_do_segment(pkt, hdr_offset, pyld_unit_size, direct_pool, + indirect_pool, pkts_out, nb_pkts_out); + if (ret > 1) + update_ipv4_tcp_headers(pkt, ipid_delta, pkts_out, ret); + + return ret; +} diff --git a/lib/librte_gso/gso_tcp4.h b/lib/librte_gso/gso_tcp4.h new file mode 100644 index 00000000..1c574412 --- /dev/null +++ b/lib/librte_gso/gso_tcp4.h @@ -0,0 +1,74 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _GSO_TCP4_H_ +#define _GSO_TCP4_H_ + +#include <stdint.h> +#include <rte_mbuf.h> + +/** + * Segment an IPv4/TCP packet. This function doesn't check if the input + * packet has correct checksums, and doesn't update checksums for output + * GSO segments. Furthermore, it doesn't process IP fragment packets. + * + * @param pkt + * The packet mbuf to segment. + * @param gso_size + * The max length of a GSO segment, measured in bytes. + * @param ipid_delta + * The increasing unit of IP ids. + * @param direct_pool + * MBUF pool used for allocating direct buffers for output segments. + * @param indirect_pool + * MBUF pool used for allocating indirect buffers for output segments. + * @param pkts_out + * Pointer array used to store the MBUF addresses of output GSO + * segments, when the function succeeds. If the memory space in + * pkts_out is insufficient, it fails and returns -EINVAL. + * @param nb_pkts_out + * The max number of items that 'pkts_out' can keep. + * + * @return + * - The number of GSO segments filled in pkts_out on success. + * - Return -ENOMEM if run out of memory in MBUF pools. + * - Return -EINVAL for invalid parameters. + */ +int gso_tcp4_segment(struct rte_mbuf *pkt, + uint16_t gso_size, + uint8_t ip_delta, + struct rte_mempool *direct_pool, + struct rte_mempool *indirect_pool, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out); +#endif diff --git a/lib/librte_gso/gso_tunnel_tcp4.c b/lib/librte_gso/gso_tunnel_tcp4.c new file mode 100644 index 00000000..8d0cfd7a --- /dev/null +++ b/lib/librte_gso/gso_tunnel_tcp4.c @@ -0,0 +1,126 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "gso_common.h" +#include "gso_tunnel_tcp4.h" + +static void +update_tunnel_ipv4_tcp_headers(struct rte_mbuf *pkt, uint8_t ipid_delta, + struct rte_mbuf **segs, uint16_t nb_segs) +{ + struct ipv4_hdr *ipv4_hdr; + struct tcp_hdr *tcp_hdr; + uint32_t sent_seq; + uint16_t outer_id, inner_id, tail_idx, i; + uint16_t outer_ipv4_offset, inner_ipv4_offset; + uint16_t udp_gre_offset, tcp_offset; + uint8_t update_udp_hdr; + + outer_ipv4_offset = pkt->outer_l2_len; + udp_gre_offset = outer_ipv4_offset + pkt->outer_l3_len; + inner_ipv4_offset = udp_gre_offset + pkt->l2_len; + tcp_offset = inner_ipv4_offset + pkt->l3_len; + + /* Outer IPv4 header. */ + ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + outer_ipv4_offset); + outer_id = rte_be_to_cpu_16(ipv4_hdr->packet_id); + + /* Inner IPv4 header. */ + ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + inner_ipv4_offset); + inner_id = rte_be_to_cpu_16(ipv4_hdr->packet_id); + + tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len); + sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq); + tail_idx = nb_segs - 1; + + /* Only update UDP header for VxLAN packets. */ + update_udp_hdr = (pkt->ol_flags & PKT_TX_TUNNEL_VXLAN) ? 1 : 0; + + for (i = 0; i < nb_segs; i++) { + update_ipv4_header(segs[i], outer_ipv4_offset, outer_id); + if (update_udp_hdr) + update_udp_header(segs[i], udp_gre_offset); + update_ipv4_header(segs[i], inner_ipv4_offset, inner_id); + update_tcp_header(segs[i], tcp_offset, sent_seq, i < tail_idx); + outer_id++; + inner_id += ipid_delta; + sent_seq += (segs[i]->pkt_len - segs[i]->data_len); + } +} + +int +gso_tunnel_tcp4_segment(struct rte_mbuf *pkt, + uint16_t gso_size, + uint8_t ipid_delta, + struct rte_mempool *direct_pool, + struct rte_mempool *indirect_pool, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out) +{ + struct ipv4_hdr *inner_ipv4_hdr; + uint16_t pyld_unit_size, hdr_offset, frag_off; + int ret = 1; + + hdr_offset = pkt->outer_l2_len + pkt->outer_l3_len + pkt->l2_len; + inner_ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + + hdr_offset); + /* + * Don't process the packet whose MF bit or offset in the inner + * IPv4 header are non-zero. + */ + frag_off = rte_be_to_cpu_16(inner_ipv4_hdr->fragment_offset); + if (unlikely(IS_FRAGMENTED(frag_off))) { + pkts_out[0] = pkt; + return 1; + } + + hdr_offset += pkt->l3_len + pkt->l4_len; + /* Don't process the packet without data */ + if (hdr_offset >= pkt->pkt_len) { + pkts_out[0] = pkt; + return 1; + } + pyld_unit_size = gso_size - hdr_offset; + + /* Segment the payload */ + ret = gso_do_segment(pkt, hdr_offset, pyld_unit_size, direct_pool, + indirect_pool, pkts_out, nb_pkts_out); + if (ret <= 1) + return ret; + + update_tunnel_ipv4_tcp_headers(pkt, ipid_delta, pkts_out, ret); + + return ret; +} diff --git a/lib/librte_gso/gso_tunnel_tcp4.h b/lib/librte_gso/gso_tunnel_tcp4.h new file mode 100644 index 00000000..3c67f0cd --- /dev/null +++ b/lib/librte_gso/gso_tunnel_tcp4.h @@ -0,0 +1,75 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _GSO_TUNNEL_TCP4_H_ +#define _GSO_TUNNEL_TCP4_H_ + +#include <stdint.h> +#include <rte_mbuf.h> + +/** + * Segment a tunneling packet with inner TCP/IPv4 headers. This function + * doesn't check if the input packet has correct checksums, and doesn't + * update checksums for output GSO segments. Furthermore, it doesn't + * process IP fragment packets. + * + * @param pkt + * The packet mbuf to segment. + * @param gso_size + * The max length of a GSO segment, measured in bytes. + * @param ipid_delta + * The increasing unit of IP ids. + * @param direct_pool + * MBUF pool used for allocating direct buffers for output segments. + * @param indirect_pool + * MBUF pool used for allocating indirect buffers for output segments. + * @param pkts_out + * Pointer array used to store the MBUF addresses of output GSO + * segments, when it succeeds. If the memory space in pkts_out is + * insufficient, it fails and returns -EINVAL. + * @param nb_pkts_out + * The max number of items that 'pkts_out' can keep. + * + * @return + * - The number of GSO segments filled in pkts_out on success. + * - Return -ENOMEM if run out of memory in MBUF pools. + * - Return -EINVAL for invalid parameters. + */ +int gso_tunnel_tcp4_segment(struct rte_mbuf *pkt, + uint16_t gso_size, + uint8_t ipid_delta, + struct rte_mempool *direct_pool, + struct rte_mempool *indirect_pool, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out); +#endif diff --git a/lib/librte_gso/rte_gso.c b/lib/librte_gso/rte_gso.c new file mode 100644 index 00000000..f86e6541 --- /dev/null +++ b/lib/librte_gso/rte_gso.c @@ -0,0 +1,110 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <errno.h> + +#include <rte_log.h> +#include <rte_ethdev.h> + +#include "rte_gso.h" +#include "gso_common.h" +#include "gso_tcp4.h" +#include "gso_tunnel_tcp4.h" + +int +rte_gso_segment(struct rte_mbuf *pkt, + const struct rte_gso_ctx *gso_ctx, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out) +{ + struct rte_mempool *direct_pool, *indirect_pool; + struct rte_mbuf *pkt_seg; + uint64_t ol_flags; + uint16_t gso_size; + uint8_t ipid_delta; + int ret = 1; + + if (pkt == NULL || pkts_out == NULL || gso_ctx == NULL || + nb_pkts_out < 1 || + gso_ctx->gso_size < RTE_GSO_SEG_SIZE_MIN || + ((gso_ctx->gso_types & (DEV_TX_OFFLOAD_TCP_TSO | + DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO)) == 0)) + return -EINVAL; + + if (gso_ctx->gso_size >= pkt->pkt_len) { + pkt->ol_flags &= (~PKT_TX_TCP_SEG); + pkts_out[0] = pkt; + return 1; + } + + direct_pool = gso_ctx->direct_pool; + indirect_pool = gso_ctx->indirect_pool; + gso_size = gso_ctx->gso_size; + ipid_delta = (gso_ctx->flag != RTE_GSO_FLAG_IPID_FIXED); + ol_flags = pkt->ol_flags; + + if ((IS_IPV4_VXLAN_TCP4(pkt->ol_flags) && + (gso_ctx->gso_types & DEV_TX_OFFLOAD_VXLAN_TNL_TSO)) || + ((IS_IPV4_GRE_TCP4(pkt->ol_flags) && + (gso_ctx->gso_types & DEV_TX_OFFLOAD_GRE_TNL_TSO)))) { + pkt->ol_flags &= (~PKT_TX_TCP_SEG); + ret = gso_tunnel_tcp4_segment(pkt, gso_size, ipid_delta, + direct_pool, indirect_pool, + pkts_out, nb_pkts_out); + } else if (IS_IPV4_TCP(pkt->ol_flags) && + (gso_ctx->gso_types & DEV_TX_OFFLOAD_TCP_TSO)) { + pkt->ol_flags &= (~PKT_TX_TCP_SEG); + ret = gso_tcp4_segment(pkt, gso_size, ipid_delta, + direct_pool, indirect_pool, + pkts_out, nb_pkts_out); + } else { + /* unsupported packet, skip */ + pkts_out[0] = pkt; + RTE_LOG(DEBUG, GSO, "Unsupported packet type\n"); + return 1; + } + + if (ret > 1) { + pkt_seg = pkt; + while (pkt_seg) { + rte_mbuf_refcnt_update(pkt_seg, -1); + pkt_seg = pkt_seg->next; + } + } else if (ret < 0) { + /* Revert the ol_flags in the event of failure. */ + pkt->ol_flags = ol_flags; + } + + return ret; +} diff --git a/lib/librte_gso/rte_gso.h b/lib/librte_gso/rte_gso.h new file mode 100644 index 00000000..4b77176f --- /dev/null +++ b/lib/librte_gso/rte_gso.h @@ -0,0 +1,148 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_GSO_H_ +#define _RTE_GSO_H_ + +/** + * @file + * Interface to GSO library + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> +#include <rte_mbuf.h> + +/* Minimum GSO segment size. */ +#define RTE_GSO_SEG_SIZE_MIN (sizeof(struct ether_hdr) + \ + sizeof(struct ipv4_hdr) + sizeof(struct tcp_hdr) + 1) + +/* GSO flags for rte_gso_ctx. */ +#define RTE_GSO_FLAG_IPID_FIXED (1ULL << 0) +/**< Use fixed IP ids for output GSO segments. Setting + * 0 indicates using incremental IP ids. + */ + +/** + * GSO context structure. + */ +struct rte_gso_ctx { + struct rte_mempool *direct_pool; + /**< MBUF pool for allocating direct buffers, which are used + * to store packet headers for GSO segments. + */ + struct rte_mempool *indirect_pool; + /**< MBUF pool for allocating indirect buffers, which are used + * to locate packet payloads for GSO segments. The indirect + * buffer doesn't contain any data, but simply points to an + * offset within the packet to segment. + */ + uint64_t flag; + /**< flag that controls specific attributes of output segments, + * such as the type of IP ID generated (i.e. fixed or incremental). + */ + uint32_t gso_types; + /**< the bit mask of required GSO types. The GSO library + * uses the same macros as that of describing device TX + * offloading capabilities (i.e. DEV_TX_OFFLOAD_*_TSO) for + * gso_types. + * + * For example, if applications want to segment TCP/IPv4 + * packets, set DEV_TX_OFFLOAD_TCP_TSO in gso_types. + */ + uint16_t gso_size; + /**< maximum size of an output GSO segment, including packet + * header and payload, measured in bytes. Must exceed + * RTE_GSO_SEG_SIZE_MIN. + */ +}; + +/** + * Segmentation function, which supports processing of both single- and + * multi- MBUF packets. + * + * Note that we refer to the packets that are segmented from the input + * packet as 'GSO segments'. rte_gso_segment() doesn't check if the + * input packet has correct checksums, and doesn't update checksums for + * output GSO segments. Additionally, it doesn't process IP fragment + * packets. + * + * Before calling rte_gso_segment(), applications must set proper ol_flags + * for the packet. The GSO library uses the same macros as that of TSO. + * For example, set PKT_TX_TCP_SEG and PKT_TX_IPV4 in ol_flags to segment + * a TCP/IPv4 packet. If rte_gso_segment() succceds, the PKT_TX_TCP_SEG + * flag is removed for all GSO segments and the input packet. + * + * Each of the newly-created GSO segments is organized as a two-segment + * MBUF, where the first segment is a standard MBUF, which stores a copy + * of packet header, and the second is an indirect MBUF which points to + * a section of data in the input packet. Since each GSO segment has + * multiple MBUFs (i.e. typically 2 MBUFs), the driver of the interface which + * the GSO segments are sent to should support transmission of multi-segment + * packets. + * + * If the input packet is GSO'd, its mbuf refcnt reduces by 1. Therefore, + * when all GSO segments are freed, the input packet is freed automatically. + * + * If the memory space in pkts_out or MBUF pools is insufficient, this + * function fails, and it returns (-1) * errno. Otherwise, GSO succeeds, + * and this function returns the number of output GSO segments filled in + * pkts_out. + * + * @param pkt + * The packet mbuf to segment. + * @param ctx + * GSO context object pointer. + * @param pkts_out + * Pointer array used to store the MBUF addresses of output GSO + * segments, when rte_gso_segment() succeeds. + * @param nb_pkts_out + * The max number of items that pkts_out can keep. + * + * @return + * - The number of GSO segments filled in pkts_out on success. + * - Return -ENOMEM if run out of memory in MBUF pools. + * - Return -EINVAL for invalid parameters. + */ +int rte_gso_segment(struct rte_mbuf *pkt, + const struct rte_gso_ctx *ctx, + struct rte_mbuf **pkts_out, + uint16_t nb_pkts_out); +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_GSO_H_ */ diff --git a/lib/librte_gso/rte_gso_version.map b/lib/librte_gso/rte_gso_version.map new file mode 100644 index 00000000..e1fd453e --- /dev/null +++ b/lib/librte_gso/rte_gso_version.map @@ -0,0 +1,7 @@ +DPDK_17.11 { + global: + + rte_gso_segment; + + local: *; +}; |