dpdk1702-rc2 upstream files unchanged + mlx5 driver rc3

Signed-off-by: Ido Barnea <ibarnea@cisco.com>

12 files changed, 182 insertions, 10 deletions

diff --git a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic.h b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic.h
index b20056b8..4eac6663 100644
--- a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic.h
+++ b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic.h
@@ -38,6 +38,8 @@
 extern "C" {
 #endif
 
+#include <stdint.h>
+#include <rte_common.h>
 #include <emmintrin.h>
 #include "generic/rte_atomic.h"
 
@@ -59,6 +61,12 @@ extern "C" {
 
 #define rte_smp_rmb() rte_compiler_barrier()
 
+#define rte_io_mb() rte_mb()
+
+#define rte_io_wmb() rte_compiler_barrier()
+
+#define rte_io_rmb() rte_compiler_barrier()
+
 /*------------------------- 16 bit atomic operations -------------------------*/
 
 #ifndef RTE_FORCE_INTRINSICS
diff --git a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h
index 400d8a96..2e04c759 100644
--- a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h
+++ b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h
@@ -37,9 +37,17 @@
  * All rights reserved.
  */
 
+#ifndef _RTE_ATOMIC_X86_H_
+#error do not include this file directly, use <rte_atomic.h> instead
+#endif
+
 #ifndef _RTE_ATOMIC_I686_H_
 #define _RTE_ATOMIC_I686_H_
 
+#include <stdint.h>
+#include <rte_common.h>
+#include <rte_atomic.h>
+
 /*------------------------- 64 bit atomic operations -------------------------*/
 
 #ifndef RTE_FORCE_INTRINSICS
@@ -47,6 +55,7 @@ static inline int
 rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
 {
 	uint8_t res;
+	RTE_STD_C11
 	union {
 		struct {
 			uint32_t l32;
diff --git a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
index 4de66000..1a53a766 100644
--- a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
+++ b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
@@ -37,9 +37,17 @@
  * All rights reserved.
  */
 
+#ifndef _RTE_ATOMIC_X86_H_
+#error do not include this file directly, use <rte_atomic.h> instead
+#endif
+
 #ifndef _RTE_ATOMIC_X86_64_H_
 #define _RTE_ATOMIC_X86_64_H_
 
+#include <stdint.h>
+#include <rte_common.h>
+#include <rte_atomic.h>
+
 /*------------------------- 64 bit atomic operations -------------------------*/
 
 #ifndef RTE_FORCE_INTRINSICS
diff --git a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder.h b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder.h
index ffdb6ef5..251f11b4 100644
--- a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder.h
+++ b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder.h
@@ -38,6 +38,8 @@
 extern "C" {
 #endif
 
+#include <stdint.h>
+#include <rte_common.h>
 #include "generic/rte_byteorder.h"
 
 #ifndef RTE_BYTE_ORDER
diff --git a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h
index 51c306f8..14d64834 100644
--- a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h
+++ b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h
@@ -31,9 +31,16 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#ifndef _RTE_BYTEORDER_X86_H_
+#error do not include this file directly, use <rte_byteorder.h> instead
+#endif
+
 #ifndef _RTE_BYTEORDER_I686_H_
 #define _RTE_BYTEORDER_I686_H_
 
+#include <stdint.h>
+#include <rte_byteorder.h>
+
 /*
  * An architecture-optimized byte swap for a 64-bit value.
  *
diff --git a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h
index dda572bd..516ac052 100644
--- a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h
+++ b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h
@@ -31,9 +31,16 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#ifndef _RTE_BYTEORDER_X86_H_
+#error do not include this file directly, use <rte_byteorder.h> instead
+#endif
+
 #ifndef _RTE_BYTEORDER_X86_64_H_
 #define _RTE_BYTEORDER_X86_64_H_
 
+#include <stdint.h>
+#include <rte_common.h>
+
 /*
  * An architecture-optimized byte swap for a 64-bit value.
  *
diff --git a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_cycles.h b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_cycles.h
index 6e3c7d89..5eb6ce96 100644
--- a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_cycles.h
+++ b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_cycles.h
@@ -75,12 +75,14 @@ extern "C" {
 extern int rte_cycles_vmware_tsc_map;
 #include <rte_branch_prediction.h>
 #endif
+#include <rte_common.h>
 
 static inline uint64_t
 rte_rdtsc(void)
 {
 	union {
 		uint64_t tsc_64;
+		RTE_STD_C11
 		struct {
 			uint32_t lo_32;
 			uint32_t hi_32;
diff --git a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_io.h b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_io.h
new file mode 100644
index 00000000..c8d14043
--- /dev/null
+++ b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_io.h
@@ -0,0 +1,47 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Cavium networks. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Cavium networks nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_IO_X86_H_
+#define _RTE_IO_X86_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_io.h"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_IO_X86_H_ */
diff --git a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_memcpy.h b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_memcpy.h
index 413035e7..b9785e85 100644
--- a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_memcpy.h
+++ b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_memcpy.h
@@ -69,6 +69,8 @@ rte_memcpy(void *dst, const void *src, size_t n) __attribute__((always_inline));
 
 #ifdef RTE_MACHINE_CPUFLAG_AVX512F
 
+#define ALIGNMENT_MASK 0x3F
+
 /**
  * AVX512 implementation below
  */
@@ -189,7 +191,7 @@ rte_mov512blocks(uint8_t *dst, const uint8_t *src, size_t n)
 }
 
 static inline void *
-rte_memcpy(void *dst, const void *src, size_t n)
+rte_memcpy_generic(void *dst, const void *src, size_t n)
 {
 	uintptr_t dstu = (uintptr_t)dst;
 	uintptr_t srcu = (uintptr_t)src;
@@ -308,6 +310,8 @@ COPY_BLOCK_128_BACK63:
 
 #elif defined RTE_MACHINE_CPUFLAG_AVX2
 
+#define ALIGNMENT_MASK 0x1F
+
 /**
  * AVX2 implementation below
  */
@@ -387,7 +391,7 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n)
 }
 
 static inline void *
-rte_memcpy(void *dst, const void *src, size_t n)
+rte_memcpy_generic(void *dst, const void *src, size_t n)
 {
 	uintptr_t dstu = (uintptr_t)dst;
 	uintptr_t srcu = (uintptr_t)src;
@@ -499,6 +503,8 @@ COPY_BLOCK_128_BACK31:
 
 #else /* RTE_MACHINE_CPUFLAG */
 
+#define ALIGNMENT_MASK 0x0F
+
 /**
  * SSE & AVX implementation below
  */
@@ -594,7 +600,7 @@ rte_mov256(uint8_t *dst, const uint8_t *src)
  * - __m128i <xmm0> ~ <xmm8> must be pre-defined
  */
 #define MOVEUNALIGNED_LEFT47_IMM(dst, src, len, offset)                                                     \
-({                                                                                                          \
+__extension__ ({                                                                                            \
     int tmp;                                                                                                \
     while (len >= 128 + 16 - offset) {                                                                      \
         xmm0 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 0 * 16));                  \
@@ -655,7 +661,7 @@ rte_mov256(uint8_t *dst, const uint8_t *src)
  * - __m128i <xmm0> ~ <xmm8> used in MOVEUNALIGNED_LEFT47_IMM must be pre-defined
  */
 #define MOVEUNALIGNED_LEFT47(dst, src, len, offset)                   \
-({                                                                    \
+__extension__ ({                                                      \
     switch (offset) {                                                 \
     case 0x01: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x01); break;    \
     case 0x02: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x02); break;    \
@@ -677,7 +683,7 @@ rte_mov256(uint8_t *dst, const uint8_t *src)
 })
 
 static inline void *
-rte_memcpy(void *dst, const void *src, size_t n)
+rte_memcpy_generic(void *dst, const void *src, size_t n)
 {
 	__m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8;
 	uintptr_t dstu = (uintptr_t)dst;
@@ -821,6 +827,75 @@ COPY_BLOCK_64_BACK15:
 
 #endif /* RTE_MACHINE_CPUFLAG */
 
+static inline void *
+rte_memcpy_aligned(void *dst, const void *src, size_t n)
+{
+	void *ret = dst;
+
+	/* Copy size <= 16 bytes */
+	if (n < 16) {
+		if (n & 0x01) {
+			*(uint8_t *)dst = *(const uint8_t *)src;
+			src = (const uint8_t *)src + 1;
+			dst = (uint8_t *)dst + 1;
+		}
+		if (n & 0x02) {
+			*(uint16_t *)dst = *(const uint16_t *)src;
+			src = (const uint16_t *)src + 1;
+			dst = (uint16_t *)dst + 1;
+		}
+		if (n & 0x04) {
+			*(uint32_t *)dst = *(const uint32_t *)src;
+			src = (const uint32_t *)src + 1;
+			dst = (uint32_t *)dst + 1;
+		}
+		if (n & 0x08)
+			*(uint64_t *)dst = *(const uint64_t *)src;
+
+		return ret;
+	}
+
+	/* Copy 16 <= size <= 32 bytes */
+	if (n <= 32) {
+		rte_mov16((uint8_t *)dst, (const uint8_t *)src);
+		rte_mov16((uint8_t *)dst - 16 + n,
+				(const uint8_t *)src - 16 + n);
+
+		return ret;
+	}
+
+	/* Copy 32 < size <= 64 bytes */
+	if (n <= 64) {
+		rte_mov32((uint8_t *)dst, (const uint8_t *)src);
+		rte_mov32((uint8_t *)dst - 32 + n,
+				(const uint8_t *)src - 32 + n);
+
+		return ret;
+	}
+
+	/* Copy 64 bytes blocks */
+	for (; n >= 64; n -= 64) {
+		rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+		dst = (uint8_t *)dst + 64;
+		src = (const uint8_t *)src + 64;
+	}
+
+	/* Copy whatever left */
+	rte_mov64((uint8_t *)dst - 64 + n,
+			(const uint8_t *)src - 64 + n);
+
+	return ret;
+}
+
+static inline void *
+rte_memcpy(void *dst, const void *src, size_t n)
+{
+	if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK))
+		return rte_memcpy_aligned(dst, src, n);
+	else
+		return rte_memcpy_generic(dst, src, n);
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_prefetch.h b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_prefetch.h
index 5dac47eb..f464398f 100644
--- a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_prefetch.h
+++ b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_prefetch.h
@@ -38,6 +38,7 @@
 extern "C" {
 #endif
 
+#include <rte_common.h>
 #include "generic/rte_prefetch.h"
 
 static inline void rte_prefetch0(const volatile void *p)
diff --git a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_rtm.h b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_rtm.h
index 0649f794..ab099952 100644
--- a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_rtm.h
+++ b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_rtm.h
@@ -20,6 +20,7 @@
 /* Official RTM intrinsics interface matching gcc/icc, but works
    on older gcc compatible compilers and binutils. */
 
+#include <rte_common.h>
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_vect.h b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_vect.h
index b698797c..1b4b85dd 100644
--- a/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_vect.h
+++ b/src/dpdk/lib/librte_eal/common/include/arch/x86/rte_vect.h
@@ -31,8 +31,8 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifndef _RTE_VECT_H_
-#define _RTE_VECT_H_
+#ifndef _RTE_VECT_X86_H_
+#define _RTE_VECT_X86_H_
 
 /**
  * @file
@@ -40,6 +40,9 @@
  * RTE SSE/AVX related header.
  */
 
+#include <stdint.h>
+#include "generic/rte_vect.h"
+
 #if (defined(__ICC) || (__GNUC__ == 4 &&  __GNUC_MINOR__ < 4))
 
 #ifdef __SSE__
@@ -106,7 +109,8 @@ typedef union rte_ymm {
 #endif /* __AVX__ */
 
 #ifdef RTE_ARCH_I686
-#define _mm_cvtsi128_si64(a) ({ \
+#define _mm_cvtsi128_si64(a)    \
+__extension__ ({                \
 	rte_xmm_t m;            \
 	m.x = (a);              \
 	(m.u64[0]);             \
@@ -117,7 +121,8 @@ typedef union rte_ymm {
  * Prior to version 12.1 icc doesn't support _mm_set_epi64x.
  */
 #if (defined(__ICC) && __ICC < 1210)
-#define _mm_set_epi64x(a, b)  ({ \
+#define _mm_set_epi64x(a, b)     \
+__extension__ ({                 \
 	rte_xmm_t m;             \
 	m.u64[0] = b;            \
 	m.u64[1] = a;            \
@@ -129,4 +134,4 @@ typedef union rte_ymm {
 }
 #endif
 
-#endif /* _RTE_VECT_H_ */
+#endif /* _RTE_VECT_X86_H_ */