From f239aed5e674965691846e8ce3f187dd47523689 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Wed, 16 Aug 2017 18:42:05 +0100 Subject: New upstream version 17.08 Change-Id: I288b50990f52646089d6b1f3aaa6ba2f091a51d7 Signed-off-by: Luca Boccassi --- lib/librte_hash/Makefile | 2 ++ lib/librte_hash/rte_cmp_arm64.h | 4 ++-- lib/librte_hash/rte_cmp_x86.h | 6 ------ lib/librte_hash/rte_crc_arm64.h | 8 ++------ lib/librte_hash/rte_cuckoo_hash.c | 27 ++++++++++++++++----------- lib/librte_hash/rte_cuckoo_hash.h | 2 +- lib/librte_hash/rte_fbk_hash.h | 2 +- lib/librte_hash/rte_hash_crc.h | 36 +++++++++++------------------------- lib/librte_hash/rte_thash.h | 13 +++++++++---- 9 files changed, 44 insertions(+), 56 deletions(-) (limited to 'lib/librte_hash') diff --git a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile index d856aa26..9cf13a04 100644 --- a/lib/librte_hash/Makefile +++ b/lib/librte_hash/Makefile @@ -49,8 +49,10 @@ SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_hash_crc.h ifeq ($(CONFIG_RTE_ARCH_ARM64),y) +ifneq ($(findstring RTE_MACHINE_CPUFLAG_CRC32,$(CFLAGS)),) SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_crc_arm64.h endif +endif SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h diff --git a/lib/librte_hash/rte_cmp_arm64.h b/lib/librte_hash/rte_cmp_arm64.h index 6fd937b1..950cef3b 100644 --- a/lib/librte_hash/rte_cmp_arm64.h +++ b/lib/librte_hash/rte_cmp_arm64.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015 Cavium networks. All rights reserved. + * Copyright(c) 2015 Cavium, Inc. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -14,7 +14,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * diff --git a/lib/librte_hash/rte_cmp_x86.h b/lib/librte_hash/rte_cmp_x86.h index e8c484d6..704c2dec 100644 --- a/lib/librte_hash/rte_cmp_x86.h +++ b/lib/librte_hash/rte_cmp_x86.h @@ -37,15 +37,9 @@ rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unu { const __m128i k1 = _mm_loadu_si128((const __m128i *) key1); const __m128i k2 = _mm_loadu_si128((const __m128i *) key2); -#ifdef RTE_MACHINE_CPUFLAG_SSE4_1 const __m128i x = _mm_xor_si128(k1, k2); return !_mm_test_all_zeros(x, x); -#else - const __m128i x = _mm_cmpeq_epi32(k1, k2); - - return _mm_movemask_epi8(x) != 0xffff; -#endif } static int diff --git a/lib/librte_hash/rte_crc_arm64.h b/lib/librte_hash/rte_crc_arm64.h index 2abe42ab..774428be 100644 --- a/lib/librte_hash/rte_crc_arm64.h +++ b/lib/librte_hash/rte_crc_arm64.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015 Cavium networks. All rights reserved. + * Copyright(c) 2015 Cavium, Inc. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -14,7 +14,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -52,7 +52,6 @@ extern "C" { static inline uint32_t crc32c_arm64_u8(uint8_t data, uint32_t init_val) { - asm(".arch armv8-a+crc"); __asm__ volatile( "crc32cb %w[crc], %w[crc], %w[value]" : [crc] "+r" (init_val) @@ -63,7 +62,6 @@ crc32c_arm64_u8(uint8_t data, uint32_t init_val) static inline uint32_t crc32c_arm64_u16(uint16_t data, uint32_t init_val) { - asm(".arch armv8-a+crc"); __asm__ volatile( "crc32ch %w[crc], %w[crc], %w[value]" : [crc] "+r" (init_val) @@ -74,7 +72,6 @@ crc32c_arm64_u16(uint16_t data, uint32_t init_val) static inline uint32_t crc32c_arm64_u32(uint32_t data, uint32_t init_val) { - asm(".arch armv8-a+crc"); __asm__ volatile( "crc32cw %w[crc], %w[crc], %w[value]" : [crc] "+r" (init_val) @@ -85,7 +82,6 @@ crc32c_arm64_u32(uint32_t data, uint32_t init_val) static inline uint32_t crc32c_arm64_u64(uint64_t data, uint32_t init_val) { - asm(".arch armv8-a+crc"); __asm__ volatile( "crc32cx %w[crc], %w[crc], %x[value]" : [crc] "+r" (init_val) diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c index 645c0cfa..87b25c01 100644 --- a/lib/librte_hash/rte_cuckoo_hash.c +++ b/lib/librte_hash/rte_cuckoo_hash.c @@ -52,11 +52,11 @@ #include #include #include -#include #include #include #include #include +#include #include "rte_hash.h" #include "rte_cuckoo_hash.h" @@ -538,8 +538,10 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, n_slots = rte_ring_mc_dequeue_burst(h->free_slots, cached_free_slots->objs, LCORE_CACHE_SIZE, NULL); - if (n_slots == 0) - return -ENOSPC; + if (n_slots == 0) { + ret = -ENOSPC; + goto failure; + } cached_free_slots->len += n_slots; } @@ -548,8 +550,10 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, cached_free_slots->len--; slot_id = cached_free_slots->objs[cached_free_slots->len]; } else { - if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0) - return -ENOSPC; + if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0) { + ret = -ENOSPC; + goto failure; + } } new_k = RTE_PTR_ADD(keys, (uintptr_t)slot_id * h->key_entry_size); @@ -569,7 +573,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, k->pdata = data; /* * Return index where key is stored, - * substracting the first dummy index + * subtracting the first dummy index */ return prim_bkt->key_idx[i] - 1; } @@ -589,7 +593,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, k->pdata = data; /* * Return index where key is stored, - * substracting the first dummy index + * subtracting the first dummy index */ return sec_bkt->key_idx[i] - 1; } @@ -659,6 +663,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, /* Error in addition, store new slot back in the ring and return error */ enqueue_slot_back(h, cached_free_slots, (void *)((uintptr_t) new_idx)); +failure: if (h->add_key == ADD_KEY_MULTIWRITER) rte_spinlock_unlock(h->multiwriter_lock); return ret; @@ -730,7 +735,7 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, *data = k->pdata; /* * Return index where key is stored, - * substracting the first dummy index + * subtracting the first dummy index */ return bkt->key_idx[i] - 1; } @@ -753,7 +758,7 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key, *data = k->pdata; /* * Return index where key is stored, - * substracting the first dummy index + * subtracting the first dummy index */ return bkt->key_idx[i] - 1; } @@ -847,7 +852,7 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, /* * Return index where key is stored, - * substracting the first dummy index + * subtracting the first dummy index */ ret = bkt->key_idx[i] - 1; bkt->key_idx[i] = EMPTY_SLOT; @@ -872,7 +877,7 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, /* * Return index where key is stored, - * substracting the first dummy index + * subtracting the first dummy index */ ret = bkt->key_idx[i] - 1; bkt->key_idx[i] = EMPTY_SLOT; diff --git a/lib/librte_hash/rte_cuckoo_hash.h b/lib/librte_hash/rte_cuckoo_hash.h index 1b8ffed8..f75392d2 100644 --- a/lib/librte_hash/rte_cuckoo_hash.h +++ b/lib/librte_hash/rte_cuckoo_hash.h @@ -58,7 +58,7 @@ #endif /* Hash function used if none is specified */ -#if defined(RTE_MACHINE_CPUFLAG_SSE4_2) || defined(RTE_MACHINE_CPUFLAG_CRC32) +#if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_CRC32) #include #define DEFAULT_HASH_FUNC rte_hash_crc #else diff --git a/lib/librte_hash/rte_fbk_hash.h b/lib/librte_hash/rte_fbk_hash.h index bd46048f..c39c0976 100644 --- a/lib/librte_hash/rte_fbk_hash.h +++ b/lib/librte_hash/rte_fbk_hash.h @@ -55,7 +55,7 @@ extern "C" { #include #ifndef RTE_FBK_HASH_FUNC_DEFAULT -#if defined(RTE_MACHINE_CPUFLAG_SSE4_2) || defined(RTE_MACHINE_CPUFLAG_CRC32) +#if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_CRC32) #include /** Default four-byte key hash function if none is specified. */ #define RTE_FBK_HASH_FUNC_DEFAULT rte_hash_crc_4byte diff --git a/lib/librte_hash/rte_hash_crc.h b/lib/librte_hash/rte_hash_crc.h index 0f485b85..ea6be522 100644 --- a/lib/librte_hash/rte_hash_crc.h +++ b/lib/librte_hash/rte_hash_crc.h @@ -387,7 +387,7 @@ crc32c_2words(uint64_t data, uint32_t init_val) return crc; } -#if defined(RTE_ARCH_I686) || defined(RTE_ARCH_X86_64) +#if defined(RTE_ARCH_X86) static inline uint32_t crc32c_sse42_u8(uint8_t data, uint32_t init_val) { @@ -453,7 +453,7 @@ crc32c_sse42_u64(uint64_t data, uint64_t init_val) static uint8_t crc32_alg = CRC32_SW; -#if defined(RTE_ARCH_ARM64) +#if defined(RTE_ARCH_ARM64) && defined(RTE_MACHINE_CPUFLAG_CRC32) #include "rte_crc_arm64.h" #else @@ -471,26 +471,12 @@ static uint8_t crc32_alg = CRC32_SW; static inline void rte_hash_crc_set_alg(uint8_t alg) { - switch (alg) { -#if defined(RTE_ARCH_I686) || defined(RTE_ARCH_X86_64) - case CRC32_SSE42_x64: - if (! rte_cpu_get_flag_enabled(RTE_CPUFLAG_EM64T)) - alg = CRC32_SSE42; -#if __GNUC__ >= 7 - __attribute__ ((fallthrough)); +#if defined(RTE_ARCH_X86) + if (alg == CRC32_SSE42_x64 && + !rte_cpu_get_flag_enabled(RTE_CPUFLAG_EM64T)) + alg = CRC32_SSE42; #endif - case CRC32_SSE42: - if (! rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_2)) - alg = CRC32_SW; -#if __GNUC__ >= 7 - __attribute__ ((fallthrough)); -#endif -#endif - case CRC32_SW: - crc32_alg = alg; - default: - break; - } + crc32_alg = alg; } /* Setting the best available algorithm */ @@ -515,7 +501,7 @@ rte_hash_crc_init_alg(void) static inline uint32_t rte_hash_crc_1byte(uint8_t data, uint32_t init_val) { -#if defined RTE_ARCH_I686 || defined RTE_ARCH_X86_64 +#if defined RTE_ARCH_X86 if (likely(crc32_alg & CRC32_SSE42)) return crc32c_sse42_u8(data, init_val); #endif @@ -538,7 +524,7 @@ rte_hash_crc_1byte(uint8_t data, uint32_t init_val) static inline uint32_t rte_hash_crc_2byte(uint16_t data, uint32_t init_val) { -#if defined RTE_ARCH_I686 || defined RTE_ARCH_X86_64 +#if defined RTE_ARCH_X86 if (likely(crc32_alg & CRC32_SSE42)) return crc32c_sse42_u16(data, init_val); #endif @@ -561,7 +547,7 @@ rte_hash_crc_2byte(uint16_t data, uint32_t init_val) static inline uint32_t rte_hash_crc_4byte(uint32_t data, uint32_t init_val) { -#if defined RTE_ARCH_I686 || defined RTE_ARCH_X86_64 +#if defined RTE_ARCH_X86 if (likely(crc32_alg & CRC32_SSE42)) return crc32c_sse42_u32(data, init_val); #endif @@ -589,7 +575,7 @@ rte_hash_crc_8byte(uint64_t data, uint32_t init_val) return crc32c_sse42_u64(data, init_val); #endif -#if defined RTE_ARCH_I686 || defined RTE_ARCH_X86_64 +#if defined RTE_ARCH_X86 if (likely(crc32_alg & CRC32_SSE42)) return crc32c_sse42_u64_mimic(data, init_val); #endif diff --git a/lib/librte_hash/rte_thash.h b/lib/librte_hash/rte_thash.h index a4886a8c..2fffd61d 100644 --- a/lib/librte_hash/rte_thash.h +++ b/lib/librte_hash/rte_thash.h @@ -56,11 +56,11 @@ extern "C" { #include #include -#ifdef __SSE3__ +#if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_NEON) #include #endif -#ifdef __SSE3__ +#ifdef RTE_ARCH_X86 /* Byte swap mask used for converting IPv6 address * 4-byte chunks to CPU byte order */ @@ -134,7 +134,7 @@ struct rte_ipv6_tuple { union rte_thash_tuple { struct rte_ipv4_tuple v4; struct rte_ipv6_tuple v6; -#ifdef __SSE3__ +#ifdef RTE_ARCH_X86 } __attribute__((aligned(XMM_SIZE))); #else }; @@ -169,13 +169,18 @@ rte_convert_rss_key(const uint32_t *orig, uint32_t *targ, int len) static inline void rte_thash_load_v6_addrs(const struct ipv6_hdr *orig, union rte_thash_tuple *targ) { -#ifdef __SSE3__ +#ifdef RTE_ARCH_X86 __m128i ipv6 = _mm_loadu_si128((const __m128i *)orig->src_addr); *(__m128i *)targ->v6.src_addr = _mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask); ipv6 = _mm_loadu_si128((const __m128i *)orig->dst_addr); *(__m128i *)targ->v6.dst_addr = _mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask); +#elif defined(RTE_MACHINE_CPUFLAG_NEON) + uint8x16_t ipv6 = vld1q_u8((uint8_t const *)orig->src_addr); + vst1q_u8((uint8_t *)targ->v6.src_addr, vrev32q_u8(ipv6)); + ipv6 = vld1q_u8((uint8_t const *)orig->dst_addr); + vst1q_u8((uint8_t *)targ->v6.dst_addr, vrev32q_u8(ipv6)); #else int i; for (i = 0; i < 4; i++) { -- cgit 1.2.3-korg