diff options
Diffstat (limited to 'src/vppinfra/vector_sse42.h')
| -rw-r--r-- | src/vppinfra/vector_sse42.h | 90 |
1 files changed, 62 insertions, 28 deletions
diff --git a/src/vppinfra/vector_sse42.h b/src/vppinfra/vector_sse42.h index 7e75ad28710..58d5da90125 100644 --- a/src/vppinfra/vector_sse42.h +++ b/src/vppinfra/vector_sse42.h @@ -41,7 +41,6 @@ #include <vppinfra/error_bootstrap.h> /* for ASSERT */ #include <x86intrin.h> -/* *INDENT-OFF* */ #define foreach_sse42_vec128i \ _(i,8,16,epi8) _(i,16,8,epi16) _(i,32,4,epi32) _(i,64,2,epi64x) #define foreach_sse42_vec128u \ @@ -92,7 +91,6 @@ t##s##x##c##_max (t##s##x##c a, t##s##x##c b) \ _(i,8,16,epi8) _(i,16,8,epi16) _(i,32,4,epi32) _(i,64,2,epi64) _(u,8,16,epu8) _(u,16,8,epu16) _(u,32,4,epu32) _(u,64,2,epu64) #undef _ -/* *INDENT-ON* */ #define CLIB_VEC128_SPLAT_DEFINED #define CLIB_HAVE_VEC128_UNALIGNED_LOAD_STORE @@ -411,26 +409,7 @@ u32x4_sum_elts (u32x4 sum4) return sum4[0]; } -static_always_inline u8x16 -u8x16_shuffle (u8x16 v, u8x16 m) -{ - return (u8x16) _mm_shuffle_epi8 ((__m128i) v, (__m128i) m); -} - -static_always_inline u32x4 -u32x4_shuffle (u32x4 v, const int a, const int b, const int c, const int d) -{ -#if defined(__clang__) || !__OPTIMIZE__ - u32x4 r = { v[a], v[b], v[c], v[d] }; - return r; -#else - return (u32x4) _mm_shuffle_epi32 ((__m128i) v, - a | b << 2 | c << 4 | d << 6); -#endif -} - /* _from_ */ -/* *INDENT-OFF* */ #define _(f,t,i) \ static_always_inline t \ t##_from_##f (f x) \ @@ -450,7 +429,6 @@ _(i16x8, i32x4, epi16_epi32) _(i16x8, i64x2, epi16_epi64) _(i32x4, i64x2, epi32_epi64) #undef _ -/* *INDENT-ON* */ static_always_inline u64x2 u64x2_gather (void *p0, void *p1) @@ -496,12 +474,6 @@ u32x4_scatter_one (u32x4 r, int index, void *p) } static_always_inline u8x16 -u8x16_is_greater (u8x16 v1, u8x16 v2) -{ - return (u8x16) _mm_cmpgt_epi8 ((__m128i) v1, (__m128i) v2); -} - -static_always_inline u8x16 u8x16_blend (u8x16 v1, u8x16 v2, u8x16 mask) { return (u8x16) _mm_blendv_epi8 ((__m128i) v1, (__m128i) v2, (__m128i) mask); @@ -517,6 +489,68 @@ u8x16_xor3 (u8x16 a, u8x16 b, u8x16 c) return a ^ b ^ c; } +static_always_inline u8x16 +u8x16_load_partial (u8 *data, uword n) +{ + u8x16 r = {}; +#if defined(CLIB_HAVE_VEC128_MASK_LOAD_STORE) + return u8x16_mask_load_zero (data, pow2_mask (n)); +#endif + if (n > 7) + { + u64x2 r; + r[1] = *(u64u *) (data + n - 8); + r >>= (16 - n) * 8; + r[0] = *(u64u *) data; + return (u8x16) r; + } + else if (n > 3) + { + u32x4 r = {}; + r[1] = *(u32u *) (data + n - 4); + r >>= (8 - n) * 8; + r[0] = *(u32u *) data; + return (u8x16) r; + } + else if (n > 1) + { + u16x8 r = {}; + r[1] = *(u16u *) (data + n - 2); + r >>= (4 - n) * 8; + r[0] = *(u16u *) data; + return (u8x16) r; + } + else if (n > 0) + r[0] = *data; + return r; +} + +static_always_inline void +u8x16_store_partial (u8x16 r, u8 *data, uword n) +{ +#if defined(CLIB_HAVE_VEC256_MASK_LOAD_STORE) + u8x16_mask_store (r, data, pow2_mask (n)); +#else + if (n > 7) + { + *(u64u *) (data + n - 8) = ((u64x2) r)[1] << ((16 - n) * 8); + *(u64u *) data = ((u64x2) r)[0]; + } + else if (n > 3) + { + *(u32u *) (data + n - 4) = ((u32x4) r)[1] << ((8 - n) * 8); + *(u32u *) data = ((u32x4) r)[0]; + } + else if (n > 1) + { + *(u16u *) (data + n - 2) = ((u16x8) r)[1] << ((4 - n) * 8); + *(u16u *) data = ((u16x8) r)[0]; + } + else if (n > 0) + data[0] = r[0]; +#endif +} + #endif /* included_vector_sse2_h */ /* |