diff options
Diffstat (limited to 'src/vppinfra/vector_avx2.h')
| -rw-r--r-- | src/vppinfra/vector_avx2.h | 118 |
1 files changed, 101 insertions, 17 deletions
diff --git a/src/vppinfra/vector_avx2.h b/src/vppinfra/vector_avx2.h index f38a3bdae73..866c82fcec3 100644 --- a/src/vppinfra/vector_avx2.h +++ b/src/vppinfra/vector_avx2.h @@ -19,7 +19,6 @@ #include <vppinfra/clib.h> #include <x86intrin.h> -/* *INDENT-OFF* */ #define foreach_avx2_vec256i \ _(i,8,32,epi8) _(i,16,16,epi16) _(i,32,8,epi32) _(i,64,4,epi64) #define foreach_avx2_vec256u \ @@ -67,7 +66,6 @@ t##s##x##c##_interleave_hi (t##s##x##c a, t##s##x##c b) \ foreach_avx2_vec256i foreach_avx2_vec256u #undef _ -/* *INDENT-ON* */ always_inline u32x8 u32x8_permute (u32x8 v, u32x8 idx) @@ -80,7 +78,6 @@ u32x8_permute (u32x8 v, u32x8 idx) (__m256i) v, ((m0) | (m1) << 2 | (m2) << 4 | (m3) << 6)) /* _extract_lo, _extract_hi */ -/* *INDENT-OFF* */ #define _(t1,t2) \ always_inline t1 \ t2##_extract_lo (t2 v) \ @@ -103,7 +100,6 @@ _(u16x8, u16x16) _(u32x4, u32x8) _(u64x2, u64x4) #undef _ -/* *INDENT-ON* */ /* 256 bit packs. */ #define _(f, t, fn) \ @@ -132,7 +128,6 @@ i8x32_msb_mask (i8x32 v) } /* _from_ */ -/* *INDENT-OFF* */ #define _(f,t,i) \ static_always_inline t \ t##_from_##f (f x) \ @@ -151,7 +146,6 @@ _ (i8x16, i16x16, epi8_epi16) _(i8x16, i32x8, epi8_epi32) _(i8x16, i64x4, epi8_epi64) #undef _ -/* *INDENT-ON* */ static_always_inline u64x4 u64x4_byte_swap (u64x4 v) @@ -183,15 +177,12 @@ u16x16_byte_swap (u16x16 v) return (u16x16) _mm256_shuffle_epi8 ((__m256i) v, (__m256i) swap); } -static_always_inline u8x32 -u8x32_shuffle (u8x32 v, u8x32 m) -{ - return (u8x32) _mm256_shuffle_epi8 ((__m256i) v, (__m256i) m); -} - #define u8x32_align_right(a, b, imm) \ (u8x32) _mm256_alignr_epi8 ((__m256i) a, (__m256i) b, imm) +#define u64x4_align_right(a, b, imm) \ + (u64x4) _mm256_alignr_epi64 ((__m256i) a, (__m256i) b, imm) + static_always_inline u32 u32x8_sum_elts (u32x8 sum8) { @@ -206,6 +197,36 @@ u32x8_hadd (u32x8 v1, u32x8 v2) return (u32x8) _mm256_hadd_epi32 ((__m256i) v1, (__m256i) v2); } +static_always_inline u32 +u32x8_hxor (u32x8 v) +{ + u32x4 v4; + v4 = u32x8_extract_lo (v) ^ u32x8_extract_hi (v); + v4 ^= (u32x4) u8x16_align_right (v4, v4, 8); + v4 ^= (u32x4) u8x16_align_right (v4, v4, 4); + return v4[0]; +} + +static_always_inline u8x32 +u8x32_xor3 (u8x32 a, u8x32 b, u8x32 c) +{ +#if __AVX512F__ + return (u8x32) _mm256_ternarylogic_epi32 ((__m256i) a, (__m256i) b, + (__m256i) c, 0x96); +#endif + return a ^ b ^ c; +} + +static_always_inline u8x32 +u8x32_reflect_u8x16 (u8x32 x) +{ + static const u8x32 mask = { + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + }; + return (u8x32) _mm256_shuffle_epi8 ((__m256i) x, (__m256i) mask); +} + static_always_inline u16x16 u16x16_mask_last (u16x16 v, u8 n_last) { @@ -308,11 +329,17 @@ u32x8_scatter_one (u32x8 r, int index, void *p) *(u32 *) p = r[index]; } -static_always_inline u8x32 -u8x32_is_greater (u8x32 v1, u8x32 v2) -{ - return (u8x32) _mm256_cmpgt_epi8 ((__m256i) v1, (__m256i) v2); -} +#define u32x8_gather_u32(base, indices, scale) \ + (u32x8) _mm256_i32gather_epi32 ((const int *) base, (__m256i) indices, scale) + +#ifdef __AVX512F__ +#define u32x8_scatter_u32(base, indices, v, scale) \ + _mm256_i32scatter_epi32 (base, (__m256i) indices, (__m256i) v, scale) +#else +#define u32x8_scatter_u32(base, indices, v, scale) \ + for (u32 i = 0; i < 8; i++) \ + *((u32u *) ((u8 *) base + (scale) * (indices)[i])) = (v)[i]; +#endif static_always_inline u8x32 u8x32_blend (u8x32 v1, u8x32 v2, u8x32 mask) @@ -321,6 +348,11 @@ u8x32_blend (u8x32 v1, u8x32 v2, u8x32 mask) (__m256i) mask); } +#define u8x32_word_shift_left(a, n) \ + (u8x32) _mm256_bslli_epi128 ((__m256i) a, n) +#define u8x32_word_shift_right(a, n) \ + (u8x32) _mm256_bsrli_epi128 ((__m256i) a, n) + #define u32x8_permute_lanes(a, b, m) \ (u32x8) _mm256_permute2x128_si256 ((__m256i) a, (__m256i) b, m) #define u64x4_permute_lanes(a, b, m) \ @@ -390,6 +422,58 @@ u64x4_transpose (u64x4 a[8]) a[3] = u64x4_permute_lanes (r[1], r[3], 0x31); } +static_always_inline u8x32 +u8x32_splat_u8x16 (u8x16 a) +{ + return (u8x32) _mm256_broadcastsi128_si256 ((__m128i) a); +} + +static_always_inline u32x8 +u32x8_splat_u32x4 (u32x4 a) +{ + return (u32x8) _mm256_broadcastsi128_si256 ((__m128i) a); +} + +static_always_inline u64x4 +u64x4_splat_u64x2 (u64x2 a) +{ + return (u64x4) _mm256_broadcastsi128_si256 ((__m128i) a); +} + +static_always_inline u8x32 +u8x32_load_partial (u8 *data, uword n) +{ +#if defined(CLIB_HAVE_VEC256_MASK_LOAD_STORE) + return u8x32_mask_load_zero (data, pow2_mask (n)); +#else + u8x32 r = {}; + if (n > 16) + { + r = u8x32_insert_lo (r, *(u8x16u *) data); + r = u8x32_insert_hi (r, u8x16_load_partial (data + 16, n - 16)); + } + else + r = u8x32_insert_lo (r, u8x16_load_partial (data, n)); + return r; +#endif +} + +static_always_inline void +u8x32_store_partial (u8x32 r, u8 *data, uword n) +{ +#if defined(CLIB_HAVE_VEC256_MASK_LOAD_STORE) + u8x32_mask_store (r, data, pow2_mask (n)); +#else + if (n > 16) + { + *(u8x16u *) data = u8x32_extract_lo (r); + u8x16_store_partial (u8x32_extract_hi (r), data + 16, n - 16); + } + else + u8x16_store_partial (u8x32_extract_lo (r), data, n); +#endif +} + #endif /* included_vector_avx2_h */ /* |