diff options
| -rw-r--r-- | src/vlib/buffer_funcs.c | 294 | ||||
| -rw-r--r-- | src/vppinfra/clib.h | 1 | ||||
| -rw-r--r-- | src/vppinfra/vector.h | 4 | ||||
| -rw-r--r-- | src/vppinfra/vector_avx2.h | 21 | ||||
| -rw-r--r-- | src/vppinfra/vector_avx512.h | 4 | ||||
| -rw-r--r-- | src/vppinfra/vector_funcs.h | 366 | ||||
| -rw-r--r-- | src/vppinfra/vector_sse42.h | 6 |
7 files changed, 266 insertions, 430 deletions
diff --git a/src/vlib/buffer_funcs.c b/src/vlib/buffer_funcs.c index f3023a3ba27..eaf141e5e9a 100644 --- a/src/vlib/buffer_funcs.c +++ b/src/vlib/buffer_funcs.c @@ -2,127 +2,222 @@ * Copyright(c) 2021 Cisco Systems, Inc. */ +#include <vppinfra/clib.h> +#include <vppinfra/vector_funcs.h> #include <vlib/vlib.h> -void __clib_section (".vlib_buffer_enqueue_to_next_fn") -CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_next_fn) -(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, - uword count) +typedef struct { - u32 *to_next, n_left_to_next, max; - u16 next_index; + uword used_elts[VLIB_FRAME_SIZE / 64]; + u32 uword_offset; +} extract_data_t; - next_index = nexts[0]; - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - max = clib_min (n_left_to_next, count); - - while (count) +static_always_inline u32 * +extract_unused_elts_x64 (u32 *elts, u16 *indices, u16 index, int n_left, + u64 *bmp, u32 *dst) +{ + u64 mask = 0; +#if defined(CLIB_HAVE_VEC128) + mask = clib_compare_u16_x64 (index, indices); + if (n_left == 64) { - u32 n_enqueued; - if ((nexts[0] != next_index) || n_left_to_next == 0) + if (mask == ~0ULL) { - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - next_index = nexts[0]; - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - max = clib_min (n_left_to_next, count); + clib_memcpy_u32 (dst, elts, 64); + *bmp = ~0ULL; + return dst + 64; } -#if defined(CLIB_HAVE_VEC512) - u16x32 next32 = CLIB_MEM_OVERFLOW_LOAD (u16x32_load_unaligned, nexts); - next32 = (next32 == u16x32_splat (next32[0])); - u64 bitmap = u16x32_msb_mask (next32); - n_enqueued = count_trailing_zeros (~bitmap); + } + else + mask &= pow2_mask (n_left); + + *bmp |= mask; + +#if defined(CLIB_HAVE_VEC512_COMPRESS) + u32x16u *ev = (u32x16u *) elts; + for (int i = 0; i < 4; i++) + { + int cnt = _popcnt32 ((u16) mask); + u32x16_compress_store (ev[i], mask, dst); + dst += cnt; + mask >>= 16; + } + +#elif defined(CLIB_HAVE_VEC256_COMPRESS) + u32x8u *ev = (u32x8u *) elts; + for (int i = 0; i < 8; i++) + { + int cnt = _popcnt32 ((u8) mask); + u32x8_compress_store (ev[i], mask, dst); + dst += cnt; + mask >>= 8; + } #elif defined(CLIB_HAVE_VEC256) - u16x16 next16 = CLIB_MEM_OVERFLOW_LOAD (u16x16_load_unaligned, nexts); - next16 = (next16 == u16x16_splat (next16[0])); - u64 bitmap = u8x32_msb_mask ((u8x32) next16); - n_enqueued = count_trailing_zeros (~bitmap) / 2; -#elif defined(CLIB_HAVE_VEC128) && defined(CLIB_HAVE_VEC128_MSB_MASK) - u16x8 next8 = CLIB_MEM_OVERFLOW_LOAD (u16x8_load_unaligned, nexts); - next8 = (next8 == u16x8_splat (next8[0])); - u64 bitmap = u8x16_msb_mask ((u8x16) next8); - n_enqueued = count_trailing_zeros (~bitmap) / 2; + while (mask) + { + u16 bit = count_trailing_zeros (mask); + mask = clear_lowest_set_bit (mask); + dst++[0] = elts[bit]; + } +#else + while (mask) + { + u16 bit = count_trailing_zeros (mask); + mask ^= 1ULL << bit; + dst++[0] = elts[bit]; + } +#endif #else - u16 x = 0; - if (count + 3 < max) + for (int i = 0; i < n_left; i++) + { + if (indices[i] == index) { - x |= next_index ^ nexts[1]; - x |= next_index ^ nexts[2]; - x |= next_index ^ nexts[3]; - n_enqueued = (x == 0) ? 4 : 1; + dst++[0] = elts[i]; + mask |= 1ULL << i; } - else - n_enqueued = 1; + } + *bmp |= mask; #endif + return dst; +} - if (PREDICT_FALSE (n_enqueued > max)) - n_enqueued = max; +static_always_inline u32 +extract_unused_elts_by_index (extract_data_t *d, u32 *elts, u16 *indices, + u16 index, int n_left, u32 *dst) +{ + u32 *dst0 = dst; + u64 *bmp = d->used_elts; + while (n_left >= 64) + { + dst = extract_unused_elts_x64 (elts, indices, index, 64, bmp, dst); -#ifdef CLIB_HAVE_VEC512 - if (n_enqueued >= 32) - { - vlib_buffer_copy_indices (to_next, buffers, 32); - nexts += 32; - to_next += 32; - buffers += 32; - n_left_to_next -= 32; - count -= 32; - max -= 32; - continue; - } -#endif + /* next */ + indices += 64; + elts += 64; + bmp++; + n_left -= 64; + } -#ifdef CLIB_HAVE_VEC256 - if (n_enqueued >= 16) - { - vlib_buffer_copy_indices (to_next, buffers, 16); - nexts += 16; - to_next += 16; - buffers += 16; - n_left_to_next -= 16; - count -= 16; - max -= 16; - continue; - } -#endif + if (n_left) + dst = extract_unused_elts_x64 (elts, indices, index, n_left, bmp, dst); -#ifdef CLIB_HAVE_VEC128 - if (n_enqueued >= 8) - { - vlib_buffer_copy_indices (to_next, buffers, 8); - nexts += 8; - to_next += 8; - buffers += 8; - n_left_to_next -= 8; - count -= 8; - max -= 8; - continue; - } -#endif + return dst - dst0; +} + +static_always_inline u32 +find_first_unused_elt (extract_data_t *d) +{ + u64 *ue = d->used_elts + d->uword_offset; + + while (PREDICT_FALSE (ue[0] == ~0)) + { + ue++; + d->uword_offset++; + } + + return d->uword_offset * 64 + count_trailing_zeros (~ue[0]); +} + +static_always_inline u32 +enqueue_one (vlib_main_t *vm, vlib_node_runtime_t *node, extract_data_t *d, + u16 next_index, u32 *buffers, u16 *nexts, u32 n_buffers, + u32 n_left, u32 *tmp) +{ + vlib_frame_t *f; + u32 n_extracted, n_free; + u32 *to; + + f = vlib_get_next_frame_internal (vm, node, next_index, 0); + + n_free = VLIB_FRAME_SIZE - f->n_vectors; + + /* if frame contains enough space for worst case scenario, we can avoid + * use of tmp */ + if (n_free >= n_left) + to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors; + else + to = tmp; + + n_extracted = extract_unused_elts_by_index (d, buffers, nexts, next_index, + n_buffers, to); + + if (to != tmp) + { + /* indices already written to frame, just close it */ + vlib_put_next_frame (vm, node, next_index, n_free - n_extracted); + } + else if (n_free >= n_extracted) + { + /* enough space in the existing frame */ + to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors; + vlib_buffer_copy_indices (to, tmp, n_extracted); + vlib_put_next_frame (vm, node, next_index, n_free - n_extracted); + } + else + { + /* full frame */ + to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors; + vlib_buffer_copy_indices (to, tmp, n_free); + vlib_put_next_frame (vm, node, next_index, 0); + + /* second frame */ + u32 n_2nd_frame = n_extracted - n_free; + f = vlib_get_next_frame_internal (vm, node, next_index, 1); + to = vlib_frame_vector_args (f); + vlib_buffer_copy_indices (to, tmp + n_free, n_2nd_frame); + vlib_put_next_frame (vm, node, next_index, + VLIB_FRAME_SIZE - n_2nd_frame); + } + + return n_left - n_extracted; +} + +void __clib_section (".vlib_buffer_enqueue_to_next_fn") +CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_next_fn) +(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, + uword count) +{ + u32 tmp[VLIB_FRAME_SIZE]; + u32 n_left; + u16 next_index; + + while (count >= VLIB_FRAME_SIZE) + { + extract_data_t d = {}; + n_left = VLIB_FRAME_SIZE; - if (n_enqueued >= 4) + next_index = nexts[0]; + n_left = enqueue_one (vm, node, &d, next_index, buffers, nexts, + VLIB_FRAME_SIZE, n_left, tmp); + + while (n_left) { - vlib_buffer_copy_indices (to_next, buffers, 4); - nexts += 4; - to_next += 4; - buffers += 4; - n_left_to_next -= 4; - count -= 4; - max -= 4; - continue; + next_index = nexts[find_first_unused_elt (&d)]; + n_left = enqueue_one (vm, node, &d, next_index, buffers, nexts, + VLIB_FRAME_SIZE, n_left, tmp); } - /* copy */ - to_next[0] = buffers[0]; + buffers += VLIB_FRAME_SIZE; + nexts += VLIB_FRAME_SIZE; + count -= VLIB_FRAME_SIZE; + } - /* next */ - nexts += 1; - to_next += 1; - buffers += 1; - n_left_to_next -= 1; - count -= 1; - max -= 1; + if (count) + { + extract_data_t d = {}; + next_index = nexts[0]; + n_left = count; + + n_left = enqueue_one (vm, node, &d, next_index, buffers, nexts, count, + n_left, tmp); + + while (n_left) + { + next_index = nexts[find_first_unused_elt (&d)]; + n_left = enqueue_one (vm, node, &d, next_index, buffers, nexts, + count, n_left, tmp); + } } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); } CLIB_MARCH_FN_REGISTRATION (vlib_buffer_enqueue_to_next_fn); @@ -161,7 +256,6 @@ next: } vlib_put_next_frame (vm, node, next_index, n_left_to_next); } - CLIB_MARCH_FN_REGISTRATION (vlib_buffer_enqueue_to_single_next_fn); u32 __clib_section (".vlib_buffer_enqueue_to_thread_fn") diff --git a/src/vppinfra/clib.h b/src/vppinfra/clib.h index 4629a7defb8..7cdf4a41aaa 100644 --- a/src/vppinfra/clib.h +++ b/src/vppinfra/clib.h @@ -369,6 +369,7 @@ void qsort (void *base, uword n, uword size, uword clib_backtrace (uword * callers, uword max_callers, uword n_frames_to_skip); +#include <vppinfra/byte_order.h> #endif /* included_clib_h */ /* diff --git a/src/vppinfra/vector.h b/src/vppinfra/vector.h index d5bc955a2e5..fed6fa3c1fd 100644 --- a/src/vppinfra/vector.h +++ b/src/vppinfra/vector.h @@ -189,10 +189,6 @@ foreach_vec #include <vppinfra/vector_neon.h> #endif -#if (defined(CLIB_HAVE_VEC128) || defined(CLIB_HAVE_VEC64)) -#include <vppinfra/vector_funcs.h> -#endif - /* this macro generate _splat inline functions for each scalar vector type */ #ifndef CLIB_VEC128_SPLAT_DEFINED #define _(t, s, c) \ diff --git a/src/vppinfra/vector_avx2.h b/src/vppinfra/vector_avx2.h index c24ed728c3c..584bd207b27 100644 --- a/src/vppinfra/vector_avx2.h +++ b/src/vppinfra/vector_avx2.h @@ -75,6 +75,10 @@ u32x8_permute (u32x8 v, u32x8 idx) return (u32x8) _mm256_permutevar8x32_epi32 ((__m256i) v, (__m256i) idx); } +#define u64x4_permute(v, m0, m1, m2, m3) \ + (u64x4) _mm256_permute4x64_epi64 ( \ + (__m256i) v, ((m0) | (m1) << 2 | (m2) << 4 | (m3) << 6)) + /* _extract_lo, _extract_hi */ /* *INDENT-OFF* */ #define _(t1,t2) \ @@ -101,8 +105,17 @@ _(u64x2, u64x4) #undef _ /* *INDENT-ON* */ +always_inline u8x32 +u16x16_pack (u16x16 lo, u16x16 hi) +{ + return (u8x32) _mm256_packus_epi16 ((__m256i) lo, (__m256i) hi); +} - +always_inline i8x32 +i16x16_pack (i16x16 lo, i16x16 hi) +{ + return (i8x32) _mm256_packs_epi16 ((__m256i) lo, (__m256i) hi); +} static_always_inline u32 u8x32_msb_mask (u8x32 v) @@ -110,6 +123,12 @@ u8x32_msb_mask (u8x32 v) return _mm256_movemask_epi8 ((__m256i) v); } +static_always_inline u32 +i8x32_msb_mask (i8x32 v) +{ + return _mm256_movemask_epi8 ((__m256i) v); +} + /* _from_ */ /* *INDENT-OFF* */ #define _(f,t,i) \ diff --git a/src/vppinfra/vector_avx512.h b/src/vppinfra/vector_avx512.h index a688baec7fb..2f5763e3c92 100644 --- a/src/vppinfra/vector_avx512.h +++ b/src/vppinfra/vector_avx512.h @@ -323,8 +323,12 @@ _ (u8x16, u16, _mm, __m128i, epi8) #endif #undef _ +#ifdef CLIB_HAVE_VEC256 #define CLIB_HAVE_VEC256_COMPRESS +#endif +#ifdef CLIB_HAVE_VEC512 #define CLIB_HAVE_VEC512_COMPRESS +#endif #ifndef __AVX512VBMI2__ static_always_inline u16x16 diff --git a/src/vppinfra/vector_funcs.h b/src/vppinfra/vector_funcs.h index db09de0f04c..c8670662910 100644 --- a/src/vppinfra/vector_funcs.h +++ b/src/vppinfra/vector_funcs.h @@ -1,334 +1,50 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* SPDX-License-Identifier: Apache-2.0 + * Copyright(c) 2021 Cisco Systems, Inc. */ -/* - Copyright (c) 2008 Eliot Dresselhaus - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ #ifndef included_vector_funcs_h #define included_vector_funcs_h +#include <vppinfra/clib.h> -#include <vppinfra/byte_order.h> - -/* Addition/subtraction. */ -#if CLIB_VECTOR_WORD_BITS == 128 -#define u8x_add u8x16_add -#define u16x_add u16x8_add -#define u32x_add u32x4_add -#define u64x_add u64x2_add -#define i8x_add i8x16_add -#define i16x_add i16x8_add -#define i32x_add i32x4_add -#define i64x_add i64x2_add -#define u8x_sub u8x16_sub -#define u16x_sub u16x8_sub -#define u32x_sub u32x4_sub -#define u64x_sub u64x2_sub -#define i8x_sub i8x16_sub -#define i16x_sub i16x8_sub -#define i32x_sub i32x4_sub -#define i64x_sub i64x2_sub -#endif - -#if CLIB_VECTOR_WORD_BITS == 64 -#define u8x_add u8x8_add -#define u16x_add u16x4_add -#define u32x_add u32x2_add -#define i8x_add i8x8_add -#define i16x_add i16x4_add -#define i32x_add i32x2_add -#define u8x_sub u8x8_sub -#define u16x_sub u16x4_sub -#define u32x_sub u32x2_sub -#define i8x_sub i8x8_sub -#define i16x_sub i16x4_sub -#define i32x_sub i32x2_sub -#endif - -/* Saturating addition/subtraction. */ -#if CLIB_VECTOR_WORD_BITS == 128 -#define u8x_add_saturate u8x16_add_saturate -#define u16x_add_saturate u16x8_add_saturate -#define i8x_add_saturate i8x16_add_saturate -#define i16x_add_saturate i16x8_add_saturate -#define u8x_sub_saturate u8x16_sub_saturate -#define u16x_sub_saturate u16x8_sub_saturate -#define i8x_sub_saturate i8x16_sub_saturate -#define i16x_sub_saturate i16x8_sub_saturate -#endif - -#if CLIB_VECTOR_WORD_BITS == 64 -#define u8x_add_saturate u8x8_add_saturate -#define u16x_add_saturate u16x4_add_saturate -#define i8x_add_saturate i8x8_add_saturate -#define i16x_add_saturate i16x4_add_saturate -#define u8x_sub_saturate u8x8_sub_saturate -#define u16x_sub_saturate u16x4_sub_saturate -#define i8x_sub_saturate i8x8_sub_saturate -#define i16x_sub_saturate i16x4_sub_saturate -#endif - -#define _vector_interleave(a,b,t) \ -do { \ - t _tmp_lo = t##_interleave_lo (a, b); \ - t _tmp_hi = t##_interleave_hi (a, b); \ - if (CLIB_ARCH_IS_LITTLE_ENDIAN) \ - (a) = _tmp_lo, (b) = _tmp_hi; \ - else \ - (a) = _tmp_hi, (b) = _tmp_lo; \ -} while (0) - -/* 128 bit interleaves. */ -#define u8x16_interleave(a,b) _vector_interleave(a,b,u8x16) -#define i8x16_interleave(a,b) _vector_interleave(a,b,i8x16) -#define u16x8_interleave(a,b) _vector_interleave(a,b,u16x8) -#define i16x8_interleave(a,b) _vector_interleave(a,b,i16x8) -#define u32x4_interleave(a,b) _vector_interleave(a,b,u32x4) -#define i32x4_interleave(a,b) _vector_interleave(a,b,i32x4) -#define u64x2_interleave(a,b) _vector_interleave(a,b,u64x2) -#define i64x2_interleave(a,b) _vector_interleave(a,b,i64x2) +/** \brief Compare 64 16-bit elemments with provied value and return bitmap -/* 64 bit interleaves. */ -#define u8x8_interleave(a,b) _vector_interleave(a,b,u8x8) -#define i8x8_interleave(a,b) _vector_interleave(a,b,i8x8) -#define u16x4_interleave(a,b) _vector_interleave(a,b,u16x4) -#define i16x4_interleave(a,b) _vector_interleave(a,b,i16x4) -#define u32x2_interleave(a,b) _vector_interleave(a,b,u32x2) -#define i32x2_interleave(a,b) _vector_interleave(a,b,i32x2) - -/* Word sized interleaves. */ -#if CLIB_VECTOR_WORD_BITS == 128 -#define u8x_interleave u8x16_interleave -#define u16x_interleave u16x8_interleave -#define u32x_interleave u32x4_interleave -#define u64x_interleave u64x2_interleave -#endif - -#if CLIB_VECTOR_WORD_BITS == 64 -#define u8x_interleave u8x8_interleave -#define u16x_interleave u16x4_interleave -#define u32x_interleave u32x2_interleave -#define u64x_interleave(a,b) /* do nothing */ -#endif - -/* Vector word sized shifts. */ -#if CLIB_VECTOR_WORD_BITS == 128 -#define u8x_shift_left u8x16_shift_left -#define i8x_shift_left i8x16_shift_left -#define u16x_shift_left u16x8_shift_left -#define i16x_shift_left i16x8_shift_left -#define u32x_shift_left u32x4_shift_left -#define i32x_shift_left i32x4_shift_left -#define u64x_shift_left u64x2_shift_left -#define i64x_shift_left i64x2_shift_left -#define u8x_shift_right u8x16_shift_right -#define i8x_shift_right i8x16_shift_right -#define u16x_shift_right u16x8_shift_right -#define i16x_shift_right i16x8_shift_right -#define u32x_shift_right u32x4_shift_right -#define i32x_shift_right i32x4_shift_right -#define u64x_shift_right u64x2_shift_right -#define i64x_shift_right i64x2_shift_right -#define u8x_rotate_left u8x16_rotate_left -#define i8x_rotate_left i8x16_rotate_left -#define u16x_rotate_left u16x8_rotate_left -#define i16x_rotate_left i16x8_rotate_left -#define u32x_rotate_left u32x4_rotate_left -#define i32x_rotate_left i32x4_rotate_left -#define u64x_rotate_left u64x2_rotate_left -#define i64x_rotate_left i64x2_rotate_left -#define u8x_rotate_right u8x16_rotate_right -#define i8x_rotate_right i8x16_rotate_right -#define u16x_rotate_right u16x8_rotate_right -#define i16x_rotate_right i16x8_rotate_right -#define u32x_rotate_right u32x4_rotate_right -#define i32x_rotate_right i32x4_rotate_right -#define u64x_rotate_right u64x2_rotate_right -#define i64x_rotate_right i64x2_rotate_right -#define u8x_ishift_left u8x16_ishift_left -#define i8x_ishift_left i8x16_ishift_left -#define u16x_ishift_left u16x8_ishift_left -#define i16x_ishift_left i16x8_ishift_left -#define u32x_ishift_left u32x4_ishift_left -#define i32x_ishift_left i32x4_ishift_left -#define u64x_ishift_left u64x2_ishift_left -#define i64x_ishift_left i64x2_ishift_left -#define u8x_ishift_right u8x16_ishift_right -#define i8x_ishift_right i8x16_ishift_right -#define u16x_ishift_right u16x8_ishift_right -#define i16x_ishift_right i16x8_ishift_right -#define u32x_ishift_right u32x4_ishift_right -#define i32x_ishift_right i32x4_ishift_right -#define u64x_ishift_right u64x2_ishift_right -#define i64x_ishift_right i64x2_ishift_right -#define u8x_irotate_left u8x16_irotate_left -#define i8x_irotate_left i8x16_irotate_left -#define u16x_irotate_left u16x8_irotate_left -#define i16x_irotate_left i16x8_irotate_left -#define u32x_irotate_left u32x4_irotate_left -#define i32x_irotate_left i32x4_irotate_left -#define u64x_irotate_left u64x2_irotate_left -#define i64x_irotate_left i64x2_irotate_left -#define u8x_irotate_right u8x16_irotate_right -#define i8x_irotate_right i8x16_irotate_right -#define u16x_irotate_right u16x8_irotate_right -#define i16x_irotate_right i16x8_irotate_right -#define u32x_irotate_right u32x4_irotate_right -#define i32x_irotate_right i32x4_irotate_right -#define u64x_irotate_right u64x2_irotate_right -#define i64x_irotate_right i64x2_irotate_right -#endif - -#if CLIB_VECTOR_WORD_BITS == 64 -#define u8x_shift_left u8x8_shift_left -#define i8x_shift_left i8x8_shift_left -#define u16x_shift_left u16x4_shift_left -#define i16x_shift_left i16x4_shift_left -#define u32x_shift_left u32x2_shift_left -#define i32x_shift_left i32x2_shift_left -#define u8x_shift_right u8x8_shift_right -#define i8x_shift_right i8x8_shift_right -#define u16x_shift_right u16x4_shift_right -#define i16x_shift_right i16x4_shift_right -#define u32x_shift_right u32x2_shift_right -#define i32x_shift_right i32x2_shift_right -#define u8x_rotate_left u8x8_rotate_left -#define i8x_rotate_left i8x8_rotate_left -#define u16x_rotate_left u16x4_rotate_left -#define i16x_rotate_left i16x4_rotate_left -#define u32x_rotate_left u32x2_rotate_left -#define i32x_rotate_left i32x2_rotate_left -#define u8x_rotate_right u8x8_rotate_right -#define i8x_rotate_right i8x8_rotate_right -#define u16x_rotate_right u16x4_rotate_right -#define i16x_rotate_right i16x4_rotate_right -#define u32x_rotate_right u32x2_rotate_right -#define i32x_rotate_right i32x2_rotate_right -#define u8x_ishift_left u8x8_ishift_left -#define i8x_ishift_left i8x8_ishift_left -#define u16x_ishift_left u16x4_ishift_left -#define i16x_ishift_left i16x4_ishift_left -#define u32x_ishift_left u32x2_ishift_left -#define i32x_ishift_left i32x2_ishift_left -#define u8x_ishift_right u8x8_ishift_right -#define i8x_ishift_right i8x8_ishift_right -#define u16x_ishift_right u16x4_ishift_right -#define i16x_ishift_right i16x4_ishift_right -#define u32x_ishift_right u32x2_ishift_right -#define i32x_ishift_right i32x2_ishift_right -#define u8x_irotate_left u8x8_irotate_left -#define i8x_irotate_left i8x8_irotate_left -#define u16x_irotate_left u16x4_irotate_left -#define i16x_irotate_left i16x4_irotate_left -#define u32x_irotate_left u32x2_irotate_left -#define i32x_irotate_left i32x2_irotate_left -#define u8x_irotate_right u8x8_irotate_right -#define i8x_irotate_right i8x8_irotate_right -#define u16x_irotate_right u16x4_irotate_right -#define i16x_irotate_right i16x4_irotate_right -#define u32x_irotate_right u32x2_irotate_right -#define i32x_irotate_right i32x2_irotate_right -#endif + @param v value to compare elements with + @param a array of 64 u16 elements + @return u64 bitmap where each bit represents result of comparison +*/ -#if CLIB_VECTOR_WORD_BITS == 128 -#define u8x_splat u8x16_splat -#define i8x_splat i8x16_splat -#define u16x_splat u16x8_splat -#define i16x_splat i16x8_splat -#define u32x_splat u32x4_splat -#define i32x_splat i32x4_splat -#define u64x_splat u64x2_splat -#define i64x_splat i64x2_splat -#endif +static_always_inline u64 +clib_compare_u16_x64 (u16 v, u16 *a) +{ + u64 mask = 0; +#if defined(CLIB_HAVE_VEC512) && !defined(__aarch64__) + u16x32 v32 = u16x32_splat (v); + u16x32u *av = (u16x32u *) a; + mask = ((u64) u16x32_is_equal_mask (av[0], v32) | + (u64) u16x32_is_equal_mask (av[1], v32) << 32); +#elif defined(CLIB_HAVE_VEC256) + u16x16 v16 = u16x16_splat (v); + u16x16u *av = (u16x16u *) a; + i8x32 x; + + x = i16x16_pack (v16 == av[0], v16 == av[1]); + mask = i8x32_msb_mask ((i8x32) u64x4_permute (x, 0, 2, 1, 3)); + x = i16x16_pack (v16 == av[2], v16 == av[3]); + mask |= (u64) i8x32_msb_mask ((i8x32) u64x4_permute (x, 0, 2, 1, 3)) << 32; +#elif defined(CLIB_HAVE_VEC128) && defined(CLIB_HAVE_VEC128_MSB_MASK) + u16x8 idx8 = u16x8_splat (v); + u16x8u *av = (u16x8u *) a; + mask = + ((u64) i8x16_msb_mask (i16x8_pack (idx8 == av[0], idx8 == av[1])) | + (u64) i8x16_msb_mask (i16x8_pack (idx8 == av[2], idx8 == av[3])) << 16 | + (u64) i8x16_msb_mask (i16x8_pack (idx8 == av[4], idx8 == av[5])) << 32 | + (u64) i8x16_msb_mask (i16x8_pack (idx8 == av[6], idx8 == av[7])) << 48); +#else + for (int i = 0; i < 64; i++) + if (a[i] == v) + mask |= 1ULL << i; +#endif + return mask; +} -#if CLIB_VECTOR_WORD_BITS == 64 -#define u8x_splat u8x8_splat -#define i8x_splat i8x8_splat -#define u16x_splat u16x4_splat -#define i16x_splat i16x4_splat -#define u32x_splat u32x2_splat -#define i32x_splat i32x2_splat #endif - -#define u32x4_transpose_step(x,y) \ -do { \ - u32x4 _x = (x); \ - u32x4 _y = (y); \ - (x) = u32x4_interleave_lo (_x, _y); \ - (y) = u32x4_interleave_hi (_x, _y); \ -} while (0) - -/* 4x4 transpose: x_ij -> x_ji */ -#define u32x4_transpose(x0,x1,x2,x3) \ -do { \ - u32x4 _x0 = (u32x4) (x0); \ - u32x4 _x1 = (u32x4) (x1); \ - u32x4 _x2 = (u32x4) (x2); \ - u32x4 _x3 = (u32x4) (x3); \ - u32x4_transpose_step (_x0, _x2); \ - u32x4_transpose_step (_x1, _x3); \ - u32x4_transpose_step (_x0, _x1); \ - u32x4_transpose_step (_x2, _x3); \ - (x0) = (u32x4) _x0; \ - (x1) = (u32x4) _x1; \ - (x2) = (u32x4) _x2; \ - (x3) = (u32x4) _x3; \ -} while (0) - -#define i32x4_transpose(x0,x1,x2,x3) \ -do { \ - u32x4 _x0 = (u32x4) (x0); \ - u32x4 _x1 = (u32x4) (x1); \ - u32x4 _x2 = (u32x4) (x2); \ - u32x4 _x3 = (u32x4) (x3); \ - u32x4_transpose_step (_x0, _x2); \ - u32x4_transpose_step (_x1, _x3); \ - u32x4_transpose_step (_x0, _x1); \ - u32x4_transpose_step (_x2, _x3); \ - (x0) = (i32x4) _x0; \ - (x1) = (i32x4) _x1; \ - (x2) = (i32x4) _x2; \ - (x3) = (i32x4) _x3; \ -} while (0) - -#undef _ - -#endif /* included_vector_funcs_h */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vppinfra/vector_sse42.h b/src/vppinfra/vector_sse42.h index 06952f117dd..f86fad39b02 100644 --- a/src/vppinfra/vector_sse42.h +++ b/src/vppinfra/vector_sse42.h @@ -613,6 +613,12 @@ u8x16_msb_mask (u8x16 v) return _mm_movemask_epi8 ((__m128i) v); } +static_always_inline u16 +i8x16_msb_mask (i8x16 v) +{ + return _mm_movemask_epi8 ((__m128i) v); +} + #define CLIB_HAVE_VEC128_MSB_MASK #undef _signed_binop |