From 3d9b72106bd664b1267533e7278ff817f942e3c6 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 8 Dec 2016 14:07:29 +0100 Subject: Imported Upstream version 16.11 Change-Id: I1944c65ddc88a9ad70f8c0eb6731552b84fbcb77 Signed-off-by: Christian Ehrhardt --- .../common/include/arch/arm/rte_byteorder.h | 2 + .../common/include/arch/arm/rte_cycles_64.h | 33 ++++++++++++ .../common/include/arch/arm/rte_memcpy_32.h | 3 +- .../common/include/arch/arm/rte_prefetch_32.h | 1 + .../common/include/arch/arm/rte_prefetch_64.h | 1 + lib/librte_eal/common/include/arch/arm/rte_vect.h | 1 + .../common/include/arch/ppc_64/rte_atomic.h | 1 + .../common/include/arch/ppc_64/rte_byteorder.h | 1 + .../common/include/arch/ppc_64/rte_cycles.h | 2 + .../common/include/arch/ppc_64/rte_memcpy.h | 3 +- .../common/include/arch/ppc_64/rte_prefetch.h | 1 + .../common/include/arch/ppc_64/rte_vect.h | 60 ++++++++++++++++++++++ .../common/include/arch/x86/rte_atomic.h | 2 + .../common/include/arch/x86/rte_atomic_32.h | 9 ++++ .../common/include/arch/x86/rte_atomic_64.h | 8 +++ .../common/include/arch/x86/rte_byteorder.h | 2 + .../common/include/arch/x86/rte_byteorder_32.h | 7 +++ .../common/include/arch/x86/rte_byteorder_64.h | 7 +++ .../common/include/arch/x86/rte_cycles.h | 2 + .../common/include/arch/x86/rte_memcpy.h | 4 +- .../common/include/arch/x86/rte_prefetch.h | 1 + lib/librte_eal/common/include/arch/x86/rte_rtm.h | 1 + lib/librte_eal/common/include/arch/x86/rte_vect.h | 8 ++- 23 files changed, 154 insertions(+), 6 deletions(-) create mode 100644 lib/librte_eal/common/include/arch/ppc_64/rte_vect.h (limited to 'lib/librte_eal/common/include/arch') diff --git a/lib/librte_eal/common/include/arch/arm/rte_byteorder.h b/lib/librte_eal/common/include/arch/arm/rte_byteorder.h index 3f2dd1f2..1b312b30 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_byteorder.h +++ b/lib/librte_eal/common/include/arch/arm/rte_byteorder.h @@ -41,6 +41,8 @@ extern "C" { #endif +#include +#include #include "generic/rte_byteorder.h" /* fix missing __builtin_bswap16 for gcc older then 4.8 */ diff --git a/lib/librte_eal/common/include/arch/arm/rte_cycles_64.h b/lib/librte_eal/common/include/arch/arm/rte_cycles_64.h index 14f26120..867a9468 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_cycles_64.h +++ b/lib/librte_eal/common/include/arch/arm/rte_cycles_64.h @@ -45,6 +45,11 @@ extern "C" { * @return * The time base for this lcore. */ +#ifndef RTE_ARM_EAL_RDTSC_USE_PMU +/** + * This call is portable to any ARMv8 architecture, however, typically + * cntvct_el0 runs at <= 100MHz and it may be imprecise for some tasks. + */ static inline uint64_t rte_rdtsc(void) { @@ -53,6 +58,34 @@ rte_rdtsc(void) asm volatile("mrs %0, cntvct_el0" : "=r" (tsc)); return tsc; } +#else +/** + * This is an alternative method to enable rte_rdtsc() with high resolution + * PMU cycles counter.The cycle counter runs at cpu frequency and this scheme + * uses ARMv8 PMU subsystem to get the cycle counter at userspace, However, + * access to PMU cycle counter from user space is not enabled by default in + * arm64 linux kernel. + * It is possible to enable cycle counter at user space access by configuring + * the PMU from the privileged mode (kernel space). + * + * asm volatile("msr pmintenset_el1, %0" : : "r" ((u64)(0 << 31))); + * asm volatile("msr pmcntenset_el0, %0" :: "r" BIT(31)); + * asm volatile("msr pmuserenr_el0, %0" : : "r"(BIT(0) | BIT(2))); + * asm volatile("mrs %0, pmcr_el0" : "=r" (val)); + * val |= (BIT(0) | BIT(2)); + * isb(); + * asm volatile("msr pmcr_el0, %0" : : "r" (val)); + * + */ +static inline uint64_t +rte_rdtsc(void) +{ + uint64_t tsc; + + asm volatile("mrs %0, pmccntr_el0" : "=r"(tsc)); + return tsc; +} +#endif static inline uint64_t rte_rdtsc_precise(void) diff --git a/lib/librte_eal/common/include/arch/arm/rte_memcpy_32.h b/lib/librte_eal/common/include/arch/arm/rte_memcpy_32.h index da6c233a..c3a26192 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_memcpy_32.h +++ b/lib/librte_eal/common/include/arch/arm/rte_memcpy_32.h @@ -148,7 +148,8 @@ rte_mov256(uint8_t *dst, const uint8_t *src) } #define rte_memcpy(dst, src, n) \ - ({ (__builtin_constant_p(n)) ? \ + __extension__ ({ \ + (__builtin_constant_p(n)) ? \ memcpy((dst), (src), (n)) : \ rte_memcpy_func((dst), (src), (n)); }) diff --git a/lib/librte_eal/common/include/arch/arm/rte_prefetch_32.h b/lib/librte_eal/common/include/arch/arm/rte_prefetch_32.h index 5aeed22d..43cde172 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_prefetch_32.h +++ b/lib/librte_eal/common/include/arch/arm/rte_prefetch_32.h @@ -37,6 +37,7 @@ extern "C" { #endif +#include #include "generic/rte_prefetch.h" static inline void rte_prefetch0(const volatile void *p) diff --git a/lib/librte_eal/common/include/arch/arm/rte_prefetch_64.h b/lib/librte_eal/common/include/arch/arm/rte_prefetch_64.h index 3ed46a46..0d077ea6 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_prefetch_64.h +++ b/lib/librte_eal/common/include/arch/arm/rte_prefetch_64.h @@ -37,6 +37,7 @@ extern "C" { #endif +#include #include "generic/rte_prefetch.h" static inline void rte_prefetch0(const volatile void *p) diff --git a/lib/librte_eal/common/include/arch/arm/rte_vect.h b/lib/librte_eal/common/include/arch/arm/rte_vect.h index a33c0544..b86c2cf5 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_vect.h +++ b/lib/librte_eal/common/include/arch/arm/rte_vect.h @@ -33,6 +33,7 @@ #ifndef _RTE_VECT_ARM_H_ #define _RTE_VECT_ARM_H_ +#include #include "arm_neon.h" #ifdef __cplusplus diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h b/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h index 924e8940..fb4fccb4 100644 --- a/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h +++ b/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h @@ -46,6 +46,7 @@ extern "C" { #endif +#include #include "generic/rte_atomic.h" /** diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_byteorder.h b/lib/librte_eal/common/include/arch/ppc_64/rte_byteorder.h index 3c1734ed..544de3c2 100644 --- a/lib/librte_eal/common/include/arch/ppc_64/rte_byteorder.h +++ b/lib/librte_eal/common/include/arch/ppc_64/rte_byteorder.h @@ -42,6 +42,7 @@ extern "C" { #endif +#include #include "generic/rte_byteorder.h" /* diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_cycles.h b/lib/librte_eal/common/include/arch/ppc_64/rte_cycles.h index 64beddf9..8fa6fc60 100644 --- a/lib/librte_eal/common/include/arch/ppc_64/rte_cycles.h +++ b/lib/librte_eal/common/include/arch/ppc_64/rte_cycles.h @@ -40,6 +40,7 @@ extern "C" { #include "generic/rte_cycles.h" #include +#include /** * Read the time base register. @@ -52,6 +53,7 @@ rte_rdtsc(void) { union { uint64_t tsc_64; + RTE_STD_C11 struct { #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN uint32_t hi_32; diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_memcpy.h b/lib/librte_eal/common/include/arch/ppc_64/rte_memcpy.h index acf7aac2..ca9d1dc5 100644 --- a/lib/librte_eal/common/include/arch/ppc_64/rte_memcpy.h +++ b/lib/librte_eal/common/include/arch/ppc_64/rte_memcpy.h @@ -95,7 +95,8 @@ rte_mov256(uint8_t *dst, const uint8_t *src) } #define rte_memcpy(dst, src, n) \ - ({ (__builtin_constant_p(n)) ? \ + __extension__ ({ \ + (__builtin_constant_p(n)) ? \ memcpy((dst), (src), (n)) : \ rte_memcpy_func((dst), (src), (n)); }) diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h b/lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h index 9a1995ea..fd2e53b9 100644 --- a/lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h +++ b/lib/librte_eal/common/include/arch/ppc_64/rte_prefetch.h @@ -37,6 +37,7 @@ extern "C" { #endif +#include #include "generic/rte_prefetch.h" static inline void rte_prefetch0(const volatile void *p) diff --git a/lib/librte_eal/common/include/arch/ppc_64/rte_vect.h b/lib/librte_eal/common/include/arch/ppc_64/rte_vect.h new file mode 100644 index 00000000..05209e52 --- /dev/null +++ b/lib/librte_eal/common/include/arch/ppc_64/rte_vect.h @@ -0,0 +1,60 @@ +/* + * BSD LICENSE + * + * Copyright (C) IBM Corporation 2016. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of IBM Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef _RTE_VECT_PPC_64_H_ +#define _RTE_VECT_PPC_64_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef vector signed int xmm_t; + +#define XMM_SIZE (sizeof(xmm_t)) +#define XMM_MASK (XMM_SIZE - 1) + +typedef union rte_xmm { + xmm_t x; + uint8_t u8[XMM_SIZE / sizeof(uint8_t)]; + uint16_t u16[XMM_SIZE / sizeof(uint16_t)]; + uint32_t u32[XMM_SIZE / sizeof(uint32_t)]; + uint64_t u64[XMM_SIZE / sizeof(uint64_t)]; + double pd[XMM_SIZE / sizeof(double)]; +} __attribute__((aligned(16))) rte_xmm_t; + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_VECT_PPC_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_atomic.h b/lib/librte_eal/common/include/arch/x86/rte_atomic.h index b20056b8..00b1cdf5 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_atomic.h +++ b/lib/librte_eal/common/include/arch/x86/rte_atomic.h @@ -38,6 +38,8 @@ extern "C" { #endif +#include +#include #include #include "generic/rte_atomic.h" diff --git a/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h b/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h index 400d8a96..2e04c759 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h +++ b/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h @@ -37,9 +37,17 @@ * All rights reserved. */ +#ifndef _RTE_ATOMIC_X86_H_ +#error do not include this file directly, use instead +#endif + #ifndef _RTE_ATOMIC_I686_H_ #define _RTE_ATOMIC_I686_H_ +#include +#include +#include + /*------------------------- 64 bit atomic operations -------------------------*/ #ifndef RTE_FORCE_INTRINSICS @@ -47,6 +55,7 @@ static inline int rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src) { uint8_t res; + RTE_STD_C11 union { struct { uint32_t l32; diff --git a/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h index 4de66000..1a53a766 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h +++ b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h @@ -37,9 +37,17 @@ * All rights reserved. */ +#ifndef _RTE_ATOMIC_X86_H_ +#error do not include this file directly, use instead +#endif + #ifndef _RTE_ATOMIC_X86_64_H_ #define _RTE_ATOMIC_X86_64_H_ +#include +#include +#include + /*------------------------- 64 bit atomic operations -------------------------*/ #ifndef RTE_FORCE_INTRINSICS diff --git a/lib/librte_eal/common/include/arch/x86/rte_byteorder.h b/lib/librte_eal/common/include/arch/x86/rte_byteorder.h index ffdb6ef5..251f11b4 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_byteorder.h +++ b/lib/librte_eal/common/include/arch/x86/rte_byteorder.h @@ -38,6 +38,8 @@ extern "C" { #endif +#include +#include #include "generic/rte_byteorder.h" #ifndef RTE_BYTE_ORDER diff --git a/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h b/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h index 51c306f8..14d64834 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h +++ b/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h @@ -31,9 +31,16 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#ifndef _RTE_BYTEORDER_X86_H_ +#error do not include this file directly, use instead +#endif + #ifndef _RTE_BYTEORDER_I686_H_ #define _RTE_BYTEORDER_I686_H_ +#include +#include + /* * An architecture-optimized byte swap for a 64-bit value. * diff --git a/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h b/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h index dda572bd..516ac052 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h +++ b/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h @@ -31,9 +31,16 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#ifndef _RTE_BYTEORDER_X86_H_ +#error do not include this file directly, use instead +#endif + #ifndef _RTE_BYTEORDER_X86_64_H_ #define _RTE_BYTEORDER_X86_64_H_ +#include +#include + /* * An architecture-optimized byte swap for a 64-bit value. * diff --git a/lib/librte_eal/common/include/arch/x86/rte_cycles.h b/lib/librte_eal/common/include/arch/x86/rte_cycles.h index 6e3c7d89..5eb6ce96 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_cycles.h +++ b/lib/librte_eal/common/include/arch/x86/rte_cycles.h @@ -75,12 +75,14 @@ extern "C" { extern int rte_cycles_vmware_tsc_map; #include #endif +#include static inline uint64_t rte_rdtsc(void) { union { uint64_t tsc_64; + RTE_STD_C11 struct { uint32_t lo_32; uint32_t hi_32; diff --git a/lib/librte_eal/common/include/arch/x86/rte_memcpy.h b/lib/librte_eal/common/include/arch/x86/rte_memcpy.h index 413035e7..b3bfc235 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_memcpy.h +++ b/lib/librte_eal/common/include/arch/x86/rte_memcpy.h @@ -594,7 +594,7 @@ rte_mov256(uint8_t *dst, const uint8_t *src) * - __m128i ~ must be pre-defined */ #define MOVEUNALIGNED_LEFT47_IMM(dst, src, len, offset) \ -({ \ +__extension__ ({ \ int tmp; \ while (len >= 128 + 16 - offset) { \ xmm0 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 0 * 16)); \ @@ -655,7 +655,7 @@ rte_mov256(uint8_t *dst, const uint8_t *src) * - __m128i ~ used in MOVEUNALIGNED_LEFT47_IMM must be pre-defined */ #define MOVEUNALIGNED_LEFT47(dst, src, len, offset) \ -({ \ +__extension__ ({ \ switch (offset) { \ case 0x01: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x01); break; \ case 0x02: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x02); break; \ diff --git a/lib/librte_eal/common/include/arch/x86/rte_prefetch.h b/lib/librte_eal/common/include/arch/x86/rte_prefetch.h index 5dac47eb..f464398f 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_prefetch.h +++ b/lib/librte_eal/common/include/arch/x86/rte_prefetch.h @@ -38,6 +38,7 @@ extern "C" { #endif +#include #include "generic/rte_prefetch.h" static inline void rte_prefetch0(const volatile void *p) diff --git a/lib/librte_eal/common/include/arch/x86/rte_rtm.h b/lib/librte_eal/common/include/arch/x86/rte_rtm.h index 0649f794..ab099952 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_rtm.h +++ b/lib/librte_eal/common/include/arch/x86/rte_rtm.h @@ -20,6 +20,7 @@ /* Official RTM intrinsics interface matching gcc/icc, but works on older gcc compatible compilers and binutils. */ +#include #ifdef __cplusplus extern "C" { diff --git a/lib/librte_eal/common/include/arch/x86/rte_vect.h b/lib/librte_eal/common/include/arch/x86/rte_vect.h index b698797c..77f2e253 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_vect.h +++ b/lib/librte_eal/common/include/arch/x86/rte_vect.h @@ -40,6 +40,8 @@ * RTE SSE/AVX related header. */ +#include + #if (defined(__ICC) || (__GNUC__ == 4 && __GNUC_MINOR__ < 4)) #ifdef __SSE__ @@ -106,7 +108,8 @@ typedef union rte_ymm { #endif /* __AVX__ */ #ifdef RTE_ARCH_I686 -#define _mm_cvtsi128_si64(a) ({ \ +#define _mm_cvtsi128_si64(a) \ +__extension__ ({ \ rte_xmm_t m; \ m.x = (a); \ (m.u64[0]); \ @@ -117,7 +120,8 @@ typedef union rte_ymm { * Prior to version 12.1 icc doesn't support _mm_set_epi64x. */ #if (defined(__ICC) && __ICC < 1210) -#define _mm_set_epi64x(a, b) ({ \ +#define _mm_set_epi64x(a, b) \ +__extension__ ({ \ rte_xmm_t m; \ m.u64[0] = b; \ m.u64[1] = a; \ -- cgit 1.2.3-korg