diff options
Diffstat (limited to 'lib/librte_gro/rte_gro.c')
-rw-r--r-- | lib/librte_gro/rte_gro.c | 229 |
1 files changed, 136 insertions, 93 deletions
diff --git a/lib/librte_gro/rte_gro.c b/lib/librte_gro/rte_gro.c index 7853246a..6618f4d3 100644 --- a/lib/librte_gro/rte_gro.c +++ b/lib/librte_gro/rte_gro.c @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation */ #include <rte_malloc.h> @@ -37,6 +9,7 @@ #include "rte_gro.h" #include "gro_tcp4.h" +#include "gro_vxlan_tcp4.h" typedef void *(*gro_tbl_create_fn)(uint16_t socket_id, uint16_t max_flow_num, @@ -45,17 +18,33 @@ typedef void (*gro_tbl_destroy_fn)(void *tbl); typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl); static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = { - gro_tcp4_tbl_create, NULL}; + gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create, NULL}; static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = { - gro_tcp4_tbl_destroy, NULL}; + gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy, + NULL}; static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = { - gro_tcp4_tbl_pkt_count, NULL}; + gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count, + NULL}; + +#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \ + ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP)) + +#define IS_IPV4_VXLAN_TCP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \ + ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \ + ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \ + RTE_PTYPE_TUNNEL_VXLAN) && \ + ((ptype & RTE_PTYPE_INNER_L4_TCP) == \ + RTE_PTYPE_INNER_L4_TCP) && \ + (((ptype & RTE_PTYPE_INNER_L3_MASK) & \ + (RTE_PTYPE_INNER_L3_IPV4 | \ + RTE_PTYPE_INNER_L3_IPV4_EXT | \ + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN)) != 0)) /* - * GRO context structure, which is used to merge packets. It keeps - * many reassembly tables of desired GRO types. Applications need to - * create GRO context objects before using rte_gro_reassemble to - * perform GRO. + * GRO context structure. It keeps the table structures, which are + * used to merge packets, for different GRO types. Before using + * rte_gro_reassemble(), applications need to create the GRO context + * first. */ struct gro_ctx { /* GRO types to perform */ @@ -113,8 +102,6 @@ rte_gro_ctx_destroy(void *ctx) uint64_t gro_type_flag; uint8_t i; - if (gro_ctx == NULL) - return; for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { gro_type_flag = 1ULL << i; if ((gro_ctx->gro_types & gro_type_flag) == 0) @@ -131,62 +118,95 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts, uint16_t nb_pkts, const struct rte_gro_param *param) { - uint16_t i; - uint16_t nb_after_gro = nb_pkts; - uint32_t item_num; - /* allocate a reassembly table for TCP/IPv4 GRO */ struct gro_tcp4_tbl tcp_tbl; - struct gro_tcp4_key tcp_keys[RTE_GRO_MAX_BURST_ITEM_NUM]; + struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM]; struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} }; + /* Allocate a reassembly table for VXLAN GRO */ + struct gro_vxlan_tcp4_tbl vxlan_tbl; + struct gro_vxlan_tcp4_flow vxlan_flows[RTE_GRO_MAX_BURST_ITEM_NUM]; + struct gro_vxlan_tcp4_item vxlan_items[RTE_GRO_MAX_BURST_ITEM_NUM] = { + {{0}, 0, 0} }; + struct rte_mbuf *unprocess_pkts[nb_pkts]; - uint16_t unprocess_num = 0; + uint32_t item_num; int32_t ret; - uint64_t current_time; + uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts; + uint8_t do_tcp4_gro = 0, do_vxlan_gro = 0; - if ((param->gro_types & RTE_GRO_TCP_IPV4) == 0) + if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 | + RTE_GRO_TCP_IPV4)) == 0)) return nb_pkts; - /* get the actual number of packets */ + /* Get the maximum number of packets */ item_num = RTE_MIN(nb_pkts, (param->max_flow_num * - param->max_item_per_flow)); + param->max_item_per_flow)); item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM); - for (i = 0; i < item_num; i++) - tcp_keys[i].start_index = INVALID_ARRAY_INDEX; - - tcp_tbl.keys = tcp_keys; - tcp_tbl.items = tcp_items; - tcp_tbl.key_num = 0; - tcp_tbl.item_num = 0; - tcp_tbl.max_key_num = item_num; - tcp_tbl.max_item_num = item_num; + if (param->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) { + for (i = 0; i < item_num; i++) + vxlan_flows[i].start_index = INVALID_ARRAY_INDEX; + + vxlan_tbl.flows = vxlan_flows; + vxlan_tbl.items = vxlan_items; + vxlan_tbl.flow_num = 0; + vxlan_tbl.item_num = 0; + vxlan_tbl.max_flow_num = item_num; + vxlan_tbl.max_item_num = item_num; + do_vxlan_gro = 1; + } - current_time = rte_rdtsc(); + if (param->gro_types & RTE_GRO_TCP_IPV4) { + for (i = 0; i < item_num; i++) + tcp_flows[i].start_index = INVALID_ARRAY_INDEX; + + tcp_tbl.flows = tcp_flows; + tcp_tbl.items = tcp_items; + tcp_tbl.flow_num = 0; + tcp_tbl.item_num = 0; + tcp_tbl.max_flow_num = item_num; + tcp_tbl.max_item_num = item_num; + do_tcp4_gro = 1; + } for (i = 0; i < nb_pkts; i++) { - if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 | - RTE_PTYPE_L4_TCP)) == - (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) { - ret = gro_tcp4_reassemble(pkts[i], - &tcp_tbl, - current_time); + /* + * The timestamp is ignored, since all packets + * will be flushed from the tables. + */ + if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) && + do_vxlan_gro) { + ret = gro_vxlan_tcp4_reassemble(pkts[i], &vxlan_tbl, 0); + if (ret > 0) + /* Merge successfully */ + nb_after_gro--; + else if (ret < 0) + unprocess_pkts[unprocess_num++] = pkts[i]; + } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) && + do_tcp4_gro) { + ret = gro_tcp4_reassemble(pkts[i], &tcp_tbl, 0); if (ret > 0) /* merge successfully */ nb_after_gro--; - else if (ret < 0) { - unprocess_pkts[unprocess_num++] = - pkts[i]; - } + else if (ret < 0) + unprocess_pkts[unprocess_num++] = pkts[i]; } else unprocess_pkts[unprocess_num++] = pkts[i]; } - /* re-arrange GROed packets */ if (nb_after_gro < nb_pkts) { - i = gro_tcp4_tbl_timeout_flush(&tcp_tbl, current_time, - pkts, nb_pkts); + i = 0; + /* Flush all packets from the tables */ + if (do_vxlan_gro) { + i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tbl, + 0, pkts, nb_pkts); + } + if (do_tcp4_gro) { + i += gro_tcp4_tbl_timeout_flush(&tcp_tbl, 0, + &pkts[i], nb_pkts - i); + } + /* Copy unprocessed packets */ if (unprocess_num > 0) { memcpy(&pkts[i], unprocess_pkts, sizeof(struct rte_mbuf *) * @@ -202,31 +222,43 @@ rte_gro_reassemble(struct rte_mbuf **pkts, uint16_t nb_pkts, void *ctx) { - uint16_t i, unprocess_num = 0; struct rte_mbuf *unprocess_pkts[nb_pkts]; struct gro_ctx *gro_ctx = ctx; + void *tcp_tbl, *vxlan_tbl; uint64_t current_time; + uint16_t i, unprocess_num = 0; + uint8_t do_tcp4_gro, do_vxlan_gro; - if ((gro_ctx->gro_types & RTE_GRO_TCP_IPV4) == 0) + if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 | + RTE_GRO_TCP_IPV4)) == 0)) return nb_pkts; + tcp_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX]; + vxlan_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX]; + + do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) == + RTE_GRO_TCP_IPV4; + do_vxlan_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) == + RTE_GRO_IPV4_VXLAN_TCP_IPV4; + current_time = rte_rdtsc(); for (i = 0; i < nb_pkts; i++) { - if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 | - RTE_PTYPE_L4_TCP)) == - (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) { - if (gro_tcp4_reassemble(pkts[i], - gro_ctx->tbls - [RTE_GRO_TCP_IPV4_INDEX], + if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) && + do_vxlan_gro) { + if (gro_vxlan_tcp4_reassemble(pkts[i], vxlan_tbl, + current_time) < 0) + unprocess_pkts[unprocess_num++] = pkts[i]; + } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) && + do_tcp4_gro) { + if (gro_tcp4_reassemble(pkts[i], tcp_tbl, current_time) < 0) unprocess_pkts[unprocess_num++] = pkts[i]; } else unprocess_pkts[unprocess_num++] = pkts[i]; } if (unprocess_num > 0) { - memcpy(pkts, unprocess_pkts, - sizeof(struct rte_mbuf *) * + memcpy(pkts, unprocess_pkts, sizeof(struct rte_mbuf *) * unprocess_num); } @@ -242,17 +274,27 @@ rte_gro_timeout_flush(void *ctx, { struct gro_ctx *gro_ctx = ctx; uint64_t flush_timestamp; + uint16_t num = 0; gro_types = gro_types & gro_ctx->gro_types; flush_timestamp = rte_rdtsc() - timeout_cycles; - if (gro_types & RTE_GRO_TCP_IPV4) { - return gro_tcp4_tbl_timeout_flush( + if (gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) { + num = gro_vxlan_tcp4_tbl_timeout_flush(gro_ctx->tbls[ + RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX], + flush_timestamp, out, max_nb_out); + max_nb_out -= num; + } + + /* If no available space in 'out', stop flushing. */ + if ((gro_types & RTE_GRO_TCP_IPV4) && max_nb_out > 0) { + num += gro_tcp4_tbl_timeout_flush( gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX], flush_timestamp, - out, max_nb_out); + &out[num], max_nb_out); } - return 0; + + return num; } uint64_t @@ -260,19 +302,20 @@ rte_gro_get_pkt_count(void *ctx) { struct gro_ctx *gro_ctx = ctx; gro_tbl_pkt_count_fn pkt_count_fn; + uint64_t gro_types = gro_ctx->gro_types, flag; uint64_t item_num = 0; - uint64_t gro_type_flag; uint8_t i; - for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { - gro_type_flag = 1ULL << i; - if ((gro_ctx->gro_types & gro_type_flag) == 0) + for (i = 0; i < RTE_GRO_TYPE_MAX_NUM && gro_types; i++) { + flag = 1ULL << i; + if ((gro_types & flag) == 0) continue; + gro_types ^= flag; pkt_count_fn = tbl_pkt_count_fn[i]; - if (pkt_count_fn == NULL) - continue; - item_num += pkt_count_fn(gro_ctx->tbls[i]); + if (pkt_count_fn) + item_num += pkt_count_fn(gro_ctx->tbls[i]); } + return item_num; } |