aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLijian Zhang <Lijian.Zhang@arm.com>2018-10-31 13:35:20 +0800
committerDamjan Marion <dmarion@me.com>2018-11-07 12:03:34 +0000
commitf5942d5612d99c5ea1189cb9f8de6b6097b0456e (patch)
treed02d1de927e7c16f985ce8f784bb968973c67e30
parentc3baf62702b7b9d339f10da48a55039e7ddc6bc9 (diff)
Optimize xxx_zero_byte_mask NEON function
Optimize zero byte mask NEON functions below with less intrinsics, and get their outputs consistent with functions in vector_sse42.h always_inline u32 u64x2_zero_byte_mask (u64x2 input) always_inline u32 u32x4_zero_byte_mask (u32x4 input) always_inline u32 u16x8_zero_byte_mask (u16x8 input) always_inline u32 u8x16_zero_byte_mask (u8x16 input) always_inline u32 i64x2_zero_byte_mask (i64x2 input) always_inline u32 i32x4_zero_byte_mask (i32x4 input) always_inline u32 i16x8_zero_byte_mask (i16x8 input) always_inline u32 i8x16_zero_byte_mask (i8x16 input) Change-Id: I7f485915baeb37fa2dd484699b8769e0136f6574 Signed-off-by: Lijian Zhang <Lijian.Zhang@arm.com> Reviewed-by: Sirshak Das <Sirshak.Das@arm.com>
-rw-r--r--src/vppinfra/vector_neon.h51
1 files changed, 7 insertions, 44 deletions
diff --git a/src/vppinfra/vector_neon.h b/src/vppinfra/vector_neon.h
index 3ac41bbb72c..aef83657321 100644
--- a/src/vppinfra/vector_neon.h
+++ b/src/vppinfra/vector_neon.h
@@ -20,6 +20,8 @@
/* Arithmetic */
#define u16x8_sub_saturate(a,b) vsubq_u16(a,b)
#define i16x8_sub_saturate(a,b) vsubq_s16(a,b)
+/* Dummy. Aid making uniform macros */
+#define vreinterpretq_u8_u8(a) a
/* Converts all ones/zeros compare mask to bitmap. */
always_inline u32
@@ -37,50 +39,6 @@ u8x16_compare_byte_mask (u8x16 v)
return (u32) (vgetq_lane_u64 (x64, 0) + (vgetq_lane_u64 (x64, 1) << 8));
}
-always_inline u32
-u16x8_zero_byte_mask (u16x8 input)
-{
- u8x16 vall_one = vdupq_n_u8 (0x0);
- u8x16 res_values = { 0x01, 0x02, 0x04, 0x08,
- 0x10, 0x20, 0x40, 0x80,
- 0x01, 0x02, 0x04, 0x08,
- 0x10, 0x20, 0x40, 0x80
- };
-
- /* input --> [0x80, 0x40, 0x01, 0xf0, ... ] */
- u8x16 test_result =
- vreinterpretq_u8_u16 (vceqq_u16 (input, vreinterpretq_u16_u8 (vall_one)));
- u8x16 before_merge = vminq_u8 (test_result, res_values);
- /*before_merge--> [0x80, 0x00, 0x00, 0x10, ... ] */
- /* u8x16 --> [a,b,c,d, e,f,g,h, i,j,k,l, m,n,o,p] */
- /* pair add until we have 2 uint64_t */
- u16x8 merge1 = vpaddlq_u8 (before_merge);
- /* u16x8--> [a+b,c+d, e+f,g+h, i+j,k+l, m+n,o+p] */
- u32x4 merge2 = vpaddlq_u16 (merge1);
- /* u32x4--> [a+b+c+d, e+f+g+h, i+j+k+l, m+n+o+p] */
- u64x2 merge3 = vpaddlq_u32 (merge2);
- /* u64x2--> [a+b+c+d+e+f+g+h, i+j+k+l+m+n+o+p] */
- return (u32) (vgetq_lane_u64 (merge3, 1) << 8) + vgetq_lane_u64 (merge3, 0);
-}
-
-always_inline u32
-u8x16_zero_byte_mask (u8x16 input)
-{
- return u16x8_zero_byte_mask ((u16x8) input);
-}
-
-always_inline u32
-u32x4_zero_byte_mask (u32x4 input)
-{
- return u16x8_zero_byte_mask ((u16x8) input);
-}
-
-always_inline u32
-u64x2_zero_byte_mask (u64x2 input)
-{
- return u16x8_zero_byte_mask ((u16x8) input);
-}
-
/* *INDENT-OFF* */
#define foreach_neon_vec128i \
_(i,8,16,s8) _(i,16,8,s16) _(i,32,4,s32) _(i,64,2,s64)
@@ -113,6 +71,11 @@ t##s##x##c##_is_equal (t##s##x##c a, t##s##x##c b) \
static_always_inline int \
t##s##x##c##_is_all_equal (t##s##x##c v, t##s x) \
{ return t##s##x##c##_is_equal (v, t##s##x##c##_splat (x)); }; \
+\
+static_always_inline u32 \
+t##s##x##c##_zero_byte_mask (t##s##x##c x) \
+{ uint8x16_t v = vreinterpretq_u8_u##s (vceqq_##i (vdupq_n_##i(0), x)); \
+ return u8x16_compare_byte_mask (v); } \
foreach_neon_vec128i foreach_neon_vec128u