New upstream version 17.08

Change-Id: I288b50990f52646089d6b1f3aaa6ba2f091a51d7
Signed-off-by: Luca Boccassi <luca.boccassi@gmail.com>

1 files changed, 93 insertions, 15 deletions

diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
index a7bc199f..e704a7f3 100644
--- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
+++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
@@ -87,6 +87,8 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq)
 		mb1 = rxep[1].mbuf;
 
 		/* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */
+		RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_physaddr) !=
+				offsetof(struct rte_mbuf, buf_addr) + 8);
 		vaddr0 = _mm_loadu_si128((__m128i *)&(mb0->buf_addr));
 		vaddr1 = _mm_loadu_si128((__m128i *)&(mb1->buf_addr));
 
@@ -214,32 +216,84 @@ desc_to_olflags_v(__m128i descs[4], __m128i mbuf_init, uint8_t vlan_flags,
 	 * appropriate flags means that we have to do a shift and blend for
 	 * each mbuf before we do the write.
 	 */
-#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
-
 	rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 8), 0x10);
 	rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 6), 0x10);
 	rearm2 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 4), 0x10);
 	rearm3 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vtag1, 2), 0x10);
 
-#else
-	rearm0 = _mm_slli_si128(vtag1, 14);
-	rearm1 = _mm_slli_si128(vtag1, 12);
-	rearm2 = _mm_slli_si128(vtag1, 10);
-	rearm3 = _mm_slli_si128(vtag1, 8);
-
-	rearm0 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm0, 48));
-	rearm1 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm1, 48));
-	rearm2 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm2, 48));
-	rearm3 = _mm_or_si128(mbuf_init, _mm_srli_epi64(rearm3, 48));
-
-#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
-
+	/* write the rearm data and the olflags in one write */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+			offsetof(struct rte_mbuf, rearm_data) + 8);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+			RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
 	_mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0);
 	_mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1);
 	_mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2);
 	_mm_store_si128((__m128i *)&rx_pkts[3]->rearm_data, rearm3);
 }
 
+static inline uint32_t get_packet_type(int index,
+				       uint32_t pkt_info,
+				       uint32_t etqf_check,
+				       uint32_t tunnel_check)
+{
+	if (etqf_check & (0x02 << (index * RTE_IXGBE_DESCS_PER_LOOP)))
+		return RTE_PTYPE_UNKNOWN;
+
+	if (tunnel_check & (0x02 << (index * RTE_IXGBE_DESCS_PER_LOOP))) {
+		pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL;
+		return ptype_table_tn[pkt_info];
+	}
+
+	pkt_info &= IXGBE_PACKET_TYPE_MASK_82599;
+	return ptype_table[pkt_info];
+}
+
+static inline void
+desc_to_ptype_v(__m128i descs[4], uint16_t pkt_type_mask,
+		struct rte_mbuf **rx_pkts)
+{
+	__m128i etqf_mask = _mm_set_epi64x(0x800000008000LL, 0x800000008000LL);
+	__m128i ptype_mask = _mm_set_epi32(
+		pkt_type_mask, pkt_type_mask, pkt_type_mask, pkt_type_mask);
+	__m128i tunnel_mask =
+		_mm_set_epi64x(0x100000001000LL, 0x100000001000LL);
+
+	uint32_t etqf_check, tunnel_check, pkt_info;
+
+	__m128i ptype0 = _mm_unpacklo_epi32(descs[0], descs[2]);
+	__m128i ptype1 = _mm_unpacklo_epi32(descs[1], descs[3]);
+
+	/* interleave low 32 bits,
+	 * now we have 4 ptypes in a XMM register
+	 */
+	ptype0 = _mm_unpacklo_epi32(ptype0, ptype1);
+
+	/* create a etqf bitmask based on the etqf bit. */
+	etqf_check = _mm_movemask_epi8(_mm_and_si128(ptype0, etqf_mask));
+
+	/* shift left by IXGBE_PACKET_TYPE_SHIFT, and apply ptype mask */
+	ptype0 = _mm_and_si128(_mm_srli_epi32(ptype0, IXGBE_PACKET_TYPE_SHIFT),
+			       ptype_mask);
+
+	/* create a tunnel bitmask based on the tunnel bit */
+	tunnel_check = _mm_movemask_epi8(
+		_mm_slli_epi32(_mm_and_si128(ptype0, tunnel_mask), 0x3));
+
+	pkt_info = _mm_extract_epi32(ptype0, 0);
+	rx_pkts[0]->packet_type =
+		get_packet_type(0, pkt_info, etqf_check, tunnel_check);
+	pkt_info = _mm_extract_epi32(ptype0, 1);
+	rx_pkts[1]->packet_type =
+		get_packet_type(1, pkt_info, etqf_check, tunnel_check);
+	pkt_info = _mm_extract_epi32(ptype0, 2);
+	rx_pkts[2]->packet_type =
+		get_packet_type(2, pkt_info, etqf_check, tunnel_check);
+	pkt_info = _mm_extract_epi32(ptype0, 3);
+	rx_pkts[3]->packet_type =
+		get_packet_type(3, pkt_info, etqf_check, tunnel_check);
+}
+
 /*
  * vPMD raw receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP)
  *
@@ -266,6 +320,15 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
 				-rxq->crc_len, /* sub crc on pkt_len */
 				0, 0            /* ignore pkt_type field */
 			);
+	/*
+	 * compile-time check the above crc_adjust layout is correct.
+	 * NOTE: the first field (lowest address) is given last in set_epi16
+	 * call above.
+	 */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+			offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+			offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
 	__m128i dd_check, eop_check;
 	__m128i mbuf_init;
 	uint8_t vlan_flags;
@@ -312,6 +375,19 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
 		0xFF, 0xFF,  /* skip 32 bit pkt_type */
 		0xFF, 0xFF
 		);
+	/*
+	 * Compile-time verify the shuffle mask
+	 * NOTE: some field positions already verified above, but duplicated
+	 * here for completeness in case of future modifications.
+	 */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+			offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+			offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
+			offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+			offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
 
 	mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer);
 
@@ -447,6 +523,8 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
 		_mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
 				pkt_mb1);
 
+		desc_to_ptype_v(descs, rxq->pkt_type_mask, &rx_pkts[pos]);
+
 		/* C.4 calc avaialbe number of desc */
 		var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
 		nb_pkts_recd += var;