New upstream version 18.02upstream/18.02

Change-Id: I89ed24cb2a49b78fe5be6970b99dd46c1499fcc3
Signed-off-by: Luca Boccassi <luca.boccassi@gmail.com>

1 files changed, 136 insertions, 93 deletions

diff --git a/lib/librte_gro/rte_gro.c b/lib/librte_gro/rte_gro.c
index 7853246a..6618f4d3 100644
--- a/lib/librte_gro/rte_gro.c
+++ b/lib/librte_gro/rte_gro.c
@@ -1,33 +1,5 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright(c) 2017 Intel Corporation. All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
  */
 
 #include <rte_malloc.h>
@@ -37,6 +9,7 @@
 
 #include "rte_gro.h"
 #include "gro_tcp4.h"
+#include "gro_vxlan_tcp4.h"
 
 typedef void *(*gro_tbl_create_fn)(uint16_t socket_id,
 		uint16_t max_flow_num,
@@ -45,17 +18,33 @@ typedef void (*gro_tbl_destroy_fn)(void *tbl);
 typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
 
 static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
-		gro_tcp4_tbl_create, NULL};
+		gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create, NULL};
 static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
-			gro_tcp4_tbl_destroy, NULL};
+			gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
+			NULL};
 static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
-			gro_tcp4_tbl_pkt_count, NULL};
+			gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
+			NULL};
+
+#define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
+		((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP))
+
+#define IS_IPV4_VXLAN_TCP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
+		((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
+		((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \
+		 RTE_PTYPE_TUNNEL_VXLAN) && \
+		 ((ptype & RTE_PTYPE_INNER_L4_TCP) == \
+		  RTE_PTYPE_INNER_L4_TCP) && \
+		  (((ptype & RTE_PTYPE_INNER_L3_MASK) & \
+		    (RTE_PTYPE_INNER_L3_IPV4 | \
+		     RTE_PTYPE_INNER_L3_IPV4_EXT | \
+		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN)) != 0))
 
 /*
- * GRO context structure, which is used to merge packets. It keeps
- * many reassembly tables of desired GRO types. Applications need to
- * create GRO context objects before using rte_gro_reassemble to
- * perform GRO.
+ * GRO context structure. It keeps the table structures, which are
+ * used to merge packets, for different GRO types. Before using
+ * rte_gro_reassemble(), applications need to create the GRO context
+ * first.
  */
 struct gro_ctx {
 	/* GRO types to perform */
@@ -113,8 +102,6 @@ rte_gro_ctx_destroy(void *ctx)
 	uint64_t gro_type_flag;
 	uint8_t i;
 
-	if (gro_ctx == NULL)
-		return;
 	for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
 		gro_type_flag = 1ULL << i;
 		if ((gro_ctx->gro_types & gro_type_flag) == 0)
@@ -131,62 +118,95 @@ rte_gro_reassemble_burst(struct rte_mbuf **pkts,
 		uint16_t nb_pkts,
 		const struct rte_gro_param *param)
 {
-	uint16_t i;
-	uint16_t nb_after_gro = nb_pkts;
-	uint32_t item_num;
-
 	/* allocate a reassembly table for TCP/IPv4 GRO */
 	struct gro_tcp4_tbl tcp_tbl;
-	struct gro_tcp4_key tcp_keys[RTE_GRO_MAX_BURST_ITEM_NUM];
+	struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
 	struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
 
+	/* Allocate a reassembly table for VXLAN GRO */
+	struct gro_vxlan_tcp4_tbl vxlan_tbl;
+	struct gro_vxlan_tcp4_flow vxlan_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
+	struct gro_vxlan_tcp4_item vxlan_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {
+		{{0}, 0, 0} };
+
 	struct rte_mbuf *unprocess_pkts[nb_pkts];
-	uint16_t unprocess_num = 0;
+	uint32_t item_num;
 	int32_t ret;
-	uint64_t current_time;
+	uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
+	uint8_t do_tcp4_gro = 0, do_vxlan_gro = 0;
 
-	if ((param->gro_types & RTE_GRO_TCP_IPV4) == 0)
+	if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
+					RTE_GRO_TCP_IPV4)) == 0))
 		return nb_pkts;
 
-	/* get the actual number of packets */
+	/* Get the maximum number of packets */
 	item_num = RTE_MIN(nb_pkts, (param->max_flow_num *
-			param->max_item_per_flow));
+				param->max_item_per_flow));
 	item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM);
 
-	for (i = 0; i < item_num; i++)
-		tcp_keys[i].start_index = INVALID_ARRAY_INDEX;
-
-	tcp_tbl.keys = tcp_keys;
-	tcp_tbl.items = tcp_items;
-	tcp_tbl.key_num = 0;
-	tcp_tbl.item_num = 0;
-	tcp_tbl.max_key_num = item_num;
-	tcp_tbl.max_item_num = item_num;
+	if (param->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) {
+		for (i = 0; i < item_num; i++)
+			vxlan_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+		vxlan_tbl.flows = vxlan_flows;
+		vxlan_tbl.items = vxlan_items;
+		vxlan_tbl.flow_num = 0;
+		vxlan_tbl.item_num = 0;
+		vxlan_tbl.max_flow_num = item_num;
+		vxlan_tbl.max_item_num = item_num;
+		do_vxlan_gro = 1;
+	}
 
-	current_time = rte_rdtsc();
+	if (param->gro_types & RTE_GRO_TCP_IPV4) {
+		for (i = 0; i < item_num; i++)
+			tcp_flows[i].start_index = INVALID_ARRAY_INDEX;
+
+		tcp_tbl.flows = tcp_flows;
+		tcp_tbl.items = tcp_items;
+		tcp_tbl.flow_num = 0;
+		tcp_tbl.item_num = 0;
+		tcp_tbl.max_flow_num = item_num;
+		tcp_tbl.max_item_num = item_num;
+		do_tcp4_gro = 1;
+	}
 
 	for (i = 0; i < nb_pkts; i++) {
-		if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 |
-					RTE_PTYPE_L4_TCP)) ==
-				(RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) {
-			ret = gro_tcp4_reassemble(pkts[i],
-					&tcp_tbl,
-					current_time);
+		/*
+		 * The timestamp is ignored, since all packets
+		 * will be flushed from the tables.
+		 */
+		if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) &&
+				do_vxlan_gro) {
+			ret = gro_vxlan_tcp4_reassemble(pkts[i], &vxlan_tbl, 0);
+			if (ret > 0)
+				/* Merge successfully */
+				nb_after_gro--;
+			else if (ret < 0)
+				unprocess_pkts[unprocess_num++] = pkts[i];
+		} else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) &&
+				do_tcp4_gro) {
+			ret = gro_tcp4_reassemble(pkts[i], &tcp_tbl, 0);
 			if (ret > 0)
 				/* merge successfully */
 				nb_after_gro--;
-			else if (ret < 0) {
-				unprocess_pkts[unprocess_num++] =
-					pkts[i];
-			}
+			else if (ret < 0)
+				unprocess_pkts[unprocess_num++] = pkts[i];
 		} else
 			unprocess_pkts[unprocess_num++] = pkts[i];
 	}
 
-	/* re-arrange GROed packets */
 	if (nb_after_gro < nb_pkts) {
-		i = gro_tcp4_tbl_timeout_flush(&tcp_tbl, current_time,
-				pkts, nb_pkts);
+		i = 0;
+		/* Flush all packets from the tables */
+		if (do_vxlan_gro) {
+			i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tbl,
+					0, pkts, nb_pkts);
+		}
+		if (do_tcp4_gro) {
+			i += gro_tcp4_tbl_timeout_flush(&tcp_tbl, 0,
+					&pkts[i], nb_pkts - i);
+		}
+		/* Copy unprocessed packets */
 		if (unprocess_num > 0) {
 			memcpy(&pkts[i], unprocess_pkts,
 					sizeof(struct rte_mbuf *) *
@@ -202,31 +222,43 @@ rte_gro_reassemble(struct rte_mbuf **pkts,
 		uint16_t nb_pkts,
 		void *ctx)
 {
-	uint16_t i, unprocess_num = 0;
 	struct rte_mbuf *unprocess_pkts[nb_pkts];
 	struct gro_ctx *gro_ctx = ctx;
+	void *tcp_tbl, *vxlan_tbl;
 	uint64_t current_time;
+	uint16_t i, unprocess_num = 0;
+	uint8_t do_tcp4_gro, do_vxlan_gro;
 
-	if ((gro_ctx->gro_types & RTE_GRO_TCP_IPV4) == 0)
+	if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
+					RTE_GRO_TCP_IPV4)) == 0))
 		return nb_pkts;
 
+	tcp_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX];
+	vxlan_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
+
+	do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
+		RTE_GRO_TCP_IPV4;
+	do_vxlan_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) ==
+		RTE_GRO_IPV4_VXLAN_TCP_IPV4;
+
 	current_time = rte_rdtsc();
 
 	for (i = 0; i < nb_pkts; i++) {
-		if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 |
-					RTE_PTYPE_L4_TCP)) ==
-				(RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) {
-			if (gro_tcp4_reassemble(pkts[i],
-						gro_ctx->tbls
-						[RTE_GRO_TCP_IPV4_INDEX],
+		if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) &&
+				do_vxlan_gro) {
+			if (gro_vxlan_tcp4_reassemble(pkts[i], vxlan_tbl,
+						current_time) < 0)
+				unprocess_pkts[unprocess_num++] = pkts[i];
+		} else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) &&
+				do_tcp4_gro) {
+			if (gro_tcp4_reassemble(pkts[i], tcp_tbl,
 						current_time) < 0)
 				unprocess_pkts[unprocess_num++] = pkts[i];
 		} else
 			unprocess_pkts[unprocess_num++] = pkts[i];
 	}
 	if (unprocess_num > 0) {
-		memcpy(pkts, unprocess_pkts,
-				sizeof(struct rte_mbuf *) *
+		memcpy(pkts, unprocess_pkts, sizeof(struct rte_mbuf *) *
 				unprocess_num);
 	}
 
@@ -242,17 +274,27 @@ rte_gro_timeout_flush(void *ctx,
 {
 	struct gro_ctx *gro_ctx = ctx;
 	uint64_t flush_timestamp;
+	uint16_t num = 0;
 
 	gro_types = gro_types & gro_ctx->gro_types;
 	flush_timestamp = rte_rdtsc() - timeout_cycles;
 
-	if (gro_types & RTE_GRO_TCP_IPV4) {
-		return gro_tcp4_tbl_timeout_flush(
+	if (gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) {
+		num = gro_vxlan_tcp4_tbl_timeout_flush(gro_ctx->tbls[
+				RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX],
+				flush_timestamp, out, max_nb_out);
+		max_nb_out -= num;
+	}
+
+	/* If no available space in 'out', stop flushing. */
+	if ((gro_types & RTE_GRO_TCP_IPV4) && max_nb_out > 0) {
+		num += gro_tcp4_tbl_timeout_flush(
 				gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX],
 				flush_timestamp,
-				out, max_nb_out);
+				&out[num], max_nb_out);
 	}
-	return 0;
+
+	return num;
 }
 
 uint64_t
@@ -260,19 +302,20 @@ rte_gro_get_pkt_count(void *ctx)
 {
 	struct gro_ctx *gro_ctx = ctx;
 	gro_tbl_pkt_count_fn pkt_count_fn;
+	uint64_t gro_types = gro_ctx->gro_types, flag;
 	uint64_t item_num = 0;
-	uint64_t gro_type_flag;
 	uint8_t i;
 
-	for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
-		gro_type_flag = 1ULL << i;
-		if ((gro_ctx->gro_types & gro_type_flag) == 0)
+	for (i = 0; i < RTE_GRO_TYPE_MAX_NUM && gro_types; i++) {
+		flag = 1ULL << i;
+		if ((gro_types & flag) == 0)
 			continue;
 
+		gro_types ^= flag;
 		pkt_count_fn = tbl_pkt_count_fn[i];
-		if (pkt_count_fn == NULL)
-			continue;
-		item_num += pkt_count_fn(gro_ctx->tbls[i]);
+		if (pkt_count_fn)
+			item_num += pkt_count_fn(gro_ctx->tbls[i]);
 	}
+
 	return item_num;
 }