summaryrefslogtreecommitdiffstats
path: root/lib/librte_hash
diff options
context:
space:
mode:
Diffstat (limited to 'lib/librte_hash')
-rw-r--r--lib/librte_hash/Makefile2
-rw-r--r--lib/librte_hash/rte_cmp_arm64.h4
-rw-r--r--lib/librte_hash/rte_cmp_x86.h6
-rw-r--r--lib/librte_hash/rte_crc_arm64.h8
-rw-r--r--lib/librte_hash/rte_cuckoo_hash.c27
-rw-r--r--lib/librte_hash/rte_cuckoo_hash.h2
-rw-r--r--lib/librte_hash/rte_fbk_hash.h2
-rw-r--r--lib/librte_hash/rte_hash_crc.h36
-rw-r--r--lib/librte_hash/rte_thash.h13
9 files changed, 44 insertions, 56 deletions
diff --git a/lib/librte_hash/Makefile b/lib/librte_hash/Makefile
index d856aa26..9cf13a04 100644
--- a/lib/librte_hash/Makefile
+++ b/lib/librte_hash/Makefile
@@ -49,8 +49,10 @@ SRCS-$(CONFIG_RTE_LIBRTE_HASH) += rte_fbk_hash.c
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include := rte_hash.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_hash_crc.h
ifeq ($(CONFIG_RTE_ARCH_ARM64),y)
+ifneq ($(findstring RTE_MACHINE_CPUFLAG_CRC32,$(CFLAGS)),)
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_crc_arm64.h
endif
+endif
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_jhash.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_thash.h
SYMLINK-$(CONFIG_RTE_LIBRTE_HASH)-include += rte_fbk_hash.h
diff --git a/lib/librte_hash/rte_cmp_arm64.h b/lib/librte_hash/rte_cmp_arm64.h
index 6fd937b1..950cef3b 100644
--- a/lib/librte_hash/rte_cmp_arm64.h
+++ b/lib/librte_hash/rte_cmp_arm64.h
@@ -1,7 +1,7 @@
/*-
* BSD LICENSE
*
- * Copyright(c) 2015 Cavium networks. All rights reserved.
+ * Copyright(c) 2015 Cavium, Inc. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -14,7 +14,7 @@
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
- * * Neither the name of Cavium networks nor the names of its
+ * * Neither the name of Cavium, Inc nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
diff --git a/lib/librte_hash/rte_cmp_x86.h b/lib/librte_hash/rte_cmp_x86.h
index e8c484d6..704c2dec 100644
--- a/lib/librte_hash/rte_cmp_x86.h
+++ b/lib/librte_hash/rte_cmp_x86.h
@@ -37,15 +37,9 @@ rte_hash_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unu
{
const __m128i k1 = _mm_loadu_si128((const __m128i *) key1);
const __m128i k2 = _mm_loadu_si128((const __m128i *) key2);
-#ifdef RTE_MACHINE_CPUFLAG_SSE4_1
const __m128i x = _mm_xor_si128(k1, k2);
return !_mm_test_all_zeros(x, x);
-#else
- const __m128i x = _mm_cmpeq_epi32(k1, k2);
-
- return _mm_movemask_epi8(x) != 0xffff;
-#endif
}
static int
diff --git a/lib/librte_hash/rte_crc_arm64.h b/lib/librte_hash/rte_crc_arm64.h
index 2abe42ab..774428be 100644
--- a/lib/librte_hash/rte_crc_arm64.h
+++ b/lib/librte_hash/rte_crc_arm64.h
@@ -1,7 +1,7 @@
/*-
* BSD LICENSE
*
- * Copyright(c) 2015 Cavium networks. All rights reserved.
+ * Copyright(c) 2015 Cavium, Inc. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -14,7 +14,7 @@
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
- * * Neither the name of Cavium networks nor the names of its
+ * * Neither the name of Cavium, Inc nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
@@ -52,7 +52,6 @@ extern "C" {
static inline uint32_t
crc32c_arm64_u8(uint8_t data, uint32_t init_val)
{
- asm(".arch armv8-a+crc");
__asm__ volatile(
"crc32cb %w[crc], %w[crc], %w[value]"
: [crc] "+r" (init_val)
@@ -63,7 +62,6 @@ crc32c_arm64_u8(uint8_t data, uint32_t init_val)
static inline uint32_t
crc32c_arm64_u16(uint16_t data, uint32_t init_val)
{
- asm(".arch armv8-a+crc");
__asm__ volatile(
"crc32ch %w[crc], %w[crc], %w[value]"
: [crc] "+r" (init_val)
@@ -74,7 +72,6 @@ crc32c_arm64_u16(uint16_t data, uint32_t init_val)
static inline uint32_t
crc32c_arm64_u32(uint32_t data, uint32_t init_val)
{
- asm(".arch armv8-a+crc");
__asm__ volatile(
"crc32cw %w[crc], %w[crc], %w[value]"
: [crc] "+r" (init_val)
@@ -85,7 +82,6 @@ crc32c_arm64_u32(uint32_t data, uint32_t init_val)
static inline uint32_t
crc32c_arm64_u64(uint64_t data, uint32_t init_val)
{
- asm(".arch armv8-a+crc");
__asm__ volatile(
"crc32cx %w[crc], %w[crc], %x[value]"
: [crc] "+r" (init_val)
diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c
index 645c0cfa..87b25c01 100644
--- a/lib/librte_hash/rte_cuckoo_hash.c
+++ b/lib/librte_hash/rte_cuckoo_hash.c
@@ -52,11 +52,11 @@
#include <rte_errno.h>
#include <rte_string_fns.h>
#include <rte_cpuflags.h>
-#include <rte_log.h>
#include <rte_rwlock.h>
#include <rte_spinlock.h>
#include <rte_ring.h>
#include <rte_compat.h>
+#include <rte_pause.h>
#include "rte_hash.h"
#include "rte_cuckoo_hash.h"
@@ -538,8 +538,10 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
n_slots = rte_ring_mc_dequeue_burst(h->free_slots,
cached_free_slots->objs,
LCORE_CACHE_SIZE, NULL);
- if (n_slots == 0)
- return -ENOSPC;
+ if (n_slots == 0) {
+ ret = -ENOSPC;
+ goto failure;
+ }
cached_free_slots->len += n_slots;
}
@@ -548,8 +550,10 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
cached_free_slots->len--;
slot_id = cached_free_slots->objs[cached_free_slots->len];
} else {
- if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0)
- return -ENOSPC;
+ if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0) {
+ ret = -ENOSPC;
+ goto failure;
+ }
}
new_k = RTE_PTR_ADD(keys, (uintptr_t)slot_id * h->key_entry_size);
@@ -569,7 +573,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
k->pdata = data;
/*
* Return index where key is stored,
- * substracting the first dummy index
+ * subtracting the first dummy index
*/
return prim_bkt->key_idx[i] - 1;
}
@@ -589,7 +593,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
k->pdata = data;
/*
* Return index where key is stored,
- * substracting the first dummy index
+ * subtracting the first dummy index
*/
return sec_bkt->key_idx[i] - 1;
}
@@ -659,6 +663,7 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key,
/* Error in addition, store new slot back in the ring and return error */
enqueue_slot_back(h, cached_free_slots, (void *)((uintptr_t) new_idx));
+failure:
if (h->add_key == ADD_KEY_MULTIWRITER)
rte_spinlock_unlock(h->multiwriter_lock);
return ret;
@@ -730,7 +735,7 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
*data = k->pdata;
/*
* Return index where key is stored,
- * substracting the first dummy index
+ * subtracting the first dummy index
*/
return bkt->key_idx[i] - 1;
}
@@ -753,7 +758,7 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,
*data = k->pdata;
/*
* Return index where key is stored,
- * substracting the first dummy index
+ * subtracting the first dummy index
*/
return bkt->key_idx[i] - 1;
}
@@ -847,7 +852,7 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
/*
* Return index where key is stored,
- * substracting the first dummy index
+ * subtracting the first dummy index
*/
ret = bkt->key_idx[i] - 1;
bkt->key_idx[i] = EMPTY_SLOT;
@@ -872,7 +877,7 @@ __rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key,
/*
* Return index where key is stored,
- * substracting the first dummy index
+ * subtracting the first dummy index
*/
ret = bkt->key_idx[i] - 1;
bkt->key_idx[i] = EMPTY_SLOT;
diff --git a/lib/librte_hash/rte_cuckoo_hash.h b/lib/librte_hash/rte_cuckoo_hash.h
index 1b8ffed8..f75392d2 100644
--- a/lib/librte_hash/rte_cuckoo_hash.h
+++ b/lib/librte_hash/rte_cuckoo_hash.h
@@ -58,7 +58,7 @@
#endif
/* Hash function used if none is specified */
-#if defined(RTE_MACHINE_CPUFLAG_SSE4_2) || defined(RTE_MACHINE_CPUFLAG_CRC32)
+#if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_CRC32)
#include <rte_hash_crc.h>
#define DEFAULT_HASH_FUNC rte_hash_crc
#else
diff --git a/lib/librte_hash/rte_fbk_hash.h b/lib/librte_hash/rte_fbk_hash.h
index bd46048f..c39c0976 100644
--- a/lib/librte_hash/rte_fbk_hash.h
+++ b/lib/librte_hash/rte_fbk_hash.h
@@ -55,7 +55,7 @@ extern "C" {
#include <string.h>
#ifndef RTE_FBK_HASH_FUNC_DEFAULT
-#if defined(RTE_MACHINE_CPUFLAG_SSE4_2) || defined(RTE_MACHINE_CPUFLAG_CRC32)
+#if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_CRC32)
#include <rte_hash_crc.h>
/** Default four-byte key hash function if none is specified. */
#define RTE_FBK_HASH_FUNC_DEFAULT rte_hash_crc_4byte
diff --git a/lib/librte_hash/rte_hash_crc.h b/lib/librte_hash/rte_hash_crc.h
index 0f485b85..ea6be522 100644
--- a/lib/librte_hash/rte_hash_crc.h
+++ b/lib/librte_hash/rte_hash_crc.h
@@ -387,7 +387,7 @@ crc32c_2words(uint64_t data, uint32_t init_val)
return crc;
}
-#if defined(RTE_ARCH_I686) || defined(RTE_ARCH_X86_64)
+#if defined(RTE_ARCH_X86)
static inline uint32_t
crc32c_sse42_u8(uint8_t data, uint32_t init_val)
{
@@ -453,7 +453,7 @@ crc32c_sse42_u64(uint64_t data, uint64_t init_val)
static uint8_t crc32_alg = CRC32_SW;
-#if defined(RTE_ARCH_ARM64)
+#if defined(RTE_ARCH_ARM64) && defined(RTE_MACHINE_CPUFLAG_CRC32)
#include "rte_crc_arm64.h"
#else
@@ -471,26 +471,12 @@ static uint8_t crc32_alg = CRC32_SW;
static inline void
rte_hash_crc_set_alg(uint8_t alg)
{
- switch (alg) {
-#if defined(RTE_ARCH_I686) || defined(RTE_ARCH_X86_64)
- case CRC32_SSE42_x64:
- if (! rte_cpu_get_flag_enabled(RTE_CPUFLAG_EM64T))
- alg = CRC32_SSE42;
-#if __GNUC__ >= 7
- __attribute__ ((fallthrough));
+#if defined(RTE_ARCH_X86)
+ if (alg == CRC32_SSE42_x64 &&
+ !rte_cpu_get_flag_enabled(RTE_CPUFLAG_EM64T))
+ alg = CRC32_SSE42;
#endif
- case CRC32_SSE42:
- if (! rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_2))
- alg = CRC32_SW;
-#if __GNUC__ >= 7
- __attribute__ ((fallthrough));
-#endif
-#endif
- case CRC32_SW:
- crc32_alg = alg;
- default:
- break;
- }
+ crc32_alg = alg;
}
/* Setting the best available algorithm */
@@ -515,7 +501,7 @@ rte_hash_crc_init_alg(void)
static inline uint32_t
rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
{
-#if defined RTE_ARCH_I686 || defined RTE_ARCH_X86_64
+#if defined RTE_ARCH_X86
if (likely(crc32_alg & CRC32_SSE42))
return crc32c_sse42_u8(data, init_val);
#endif
@@ -538,7 +524,7 @@ rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
static inline uint32_t
rte_hash_crc_2byte(uint16_t data, uint32_t init_val)
{
-#if defined RTE_ARCH_I686 || defined RTE_ARCH_X86_64
+#if defined RTE_ARCH_X86
if (likely(crc32_alg & CRC32_SSE42))
return crc32c_sse42_u16(data, init_val);
#endif
@@ -561,7 +547,7 @@ rte_hash_crc_2byte(uint16_t data, uint32_t init_val)
static inline uint32_t
rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
{
-#if defined RTE_ARCH_I686 || defined RTE_ARCH_X86_64
+#if defined RTE_ARCH_X86
if (likely(crc32_alg & CRC32_SSE42))
return crc32c_sse42_u32(data, init_val);
#endif
@@ -589,7 +575,7 @@ rte_hash_crc_8byte(uint64_t data, uint32_t init_val)
return crc32c_sse42_u64(data, init_val);
#endif
-#if defined RTE_ARCH_I686 || defined RTE_ARCH_X86_64
+#if defined RTE_ARCH_X86
if (likely(crc32_alg & CRC32_SSE42))
return crc32c_sse42_u64_mimic(data, init_val);
#endif
diff --git a/lib/librte_hash/rte_thash.h b/lib/librte_hash/rte_thash.h
index a4886a8c..2fffd61d 100644
--- a/lib/librte_hash/rte_thash.h
+++ b/lib/librte_hash/rte_thash.h
@@ -56,11 +56,11 @@ extern "C" {
#include <rte_ip.h>
#include <rte_common.h>
-#ifdef __SSE3__
+#if defined(RTE_ARCH_X86) || defined(RTE_MACHINE_CPUFLAG_NEON)
#include <rte_vect.h>
#endif
-#ifdef __SSE3__
+#ifdef RTE_ARCH_X86
/* Byte swap mask used for converting IPv6 address
* 4-byte chunks to CPU byte order
*/
@@ -134,7 +134,7 @@ struct rte_ipv6_tuple {
union rte_thash_tuple {
struct rte_ipv4_tuple v4;
struct rte_ipv6_tuple v6;
-#ifdef __SSE3__
+#ifdef RTE_ARCH_X86
} __attribute__((aligned(XMM_SIZE)));
#else
};
@@ -169,13 +169,18 @@ rte_convert_rss_key(const uint32_t *orig, uint32_t *targ, int len)
static inline void
rte_thash_load_v6_addrs(const struct ipv6_hdr *orig, union rte_thash_tuple *targ)
{
-#ifdef __SSE3__
+#ifdef RTE_ARCH_X86
__m128i ipv6 = _mm_loadu_si128((const __m128i *)orig->src_addr);
*(__m128i *)targ->v6.src_addr =
_mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask);
ipv6 = _mm_loadu_si128((const __m128i *)orig->dst_addr);
*(__m128i *)targ->v6.dst_addr =
_mm_shuffle_epi8(ipv6, rte_thash_ipv6_bswap_mask);
+#elif defined(RTE_MACHINE_CPUFLAG_NEON)
+ uint8x16_t ipv6 = vld1q_u8((uint8_t const *)orig->src_addr);
+ vst1q_u8((uint8_t *)targ->v6.src_addr, vrev32q_u8(ipv6));
+ ipv6 = vld1q_u8((uint8_t const *)orig->dst_addr);
+ vst1q_u8((uint8_t *)targ->v6.dst_addr, vrev32q_u8(ipv6));
#else
int i;
for (i = 0; i < 4; i++) {