diff options
| author |   Dmitry Valter <d-valter@yandex-team.ru> | 2021-11-14 17:05:44 +0000 |
|---|---|---|
| committer |   Damjan Marion <dmarion@me.com> | 2021-11-15 12:57:26 +0000 |
| commit | 923325f0ef7c4042decfbf25628a56fca5c79253 (patch) | |
| tree | 0f2ae60580bd74524e572f2ea9ae902f6fd51aca | |
| parent | 23ff4ce21e71b697fb059ea82857413d993db9d0 (diff) | |
vppinfra: fix masks in AVX512 clib_count_equal_*
Mask result of uAxB_is_equal_mask when buffer is masked. Otherwise it
return vector length B as a result for zeroed words.
This bug caused crashes in error_drop in tests on Ice Lake.
Type: fix
Fixes: 7459be1b3626b608e60df574343a1432a068ebce
Change-Id: I56183e77f8a8ab6c530e79b465067958de84dceb
Signed-off-by: Dmitry Valter <d-valter@yandex-team.ru>
| -rw-r--r-- | src/vppinfra/vector/count_equal.h | 14 | ||||
| -rw-r--r-- | src/vppinfra/vector/test/count_equal.c | 57 |
2 files changed, 40 insertions, 31 deletions
diff --git a/src/vppinfra/vector/count_equal.h b/src/vppinfra/vector/count_equal.h index a2aeecd9ba0..ca2fbb7fd39 100644 --- a/src/vppinfra/vector/count_equal.h +++ b/src/vppinfra/vector/count_equal.h @@ -85,7 +85,8 @@ clib_count_equal_u32 (u32 *data, uword max_count) { u32 mask = pow2_mask (max_count - count); u32 bmp = - u32x16_is_equal_mask (u32x16_mask_load_zero (data, mask), splat); + u32x16_is_equal_mask (u32x16_mask_load_zero (data, mask), splat) & + mask; return count + count_trailing_zeros (~bmp); } #elif defined(CLIB_HAVE_VEC256) @@ -108,11 +109,12 @@ clib_count_equal_u32 (u32 *data, uword max_count) } if (count == max_count) return count; -#if defined(CxLIB_HAVE_VEC256_MASK_LOAD_STORE) +#if defined(CLIB_HAVE_VEC256_MASK_LOAD_STORE) else { u32 mask = pow2_mask (max_count - count); - u32 bmp = u32x8_is_equal_mask (u32x8_mask_load_zero (data, mask), splat); + u32 bmp = + u32x8_is_equal_mask (u32x8_mask_load_zero (data, mask), splat) & mask; return count + count_trailing_zeros (~bmp); } #endif @@ -243,7 +245,8 @@ clib_count_equal_u8 (u8 *data, uword max_count) else { u64 mask = pow2_mask (max_count - count); - u64 bmp = u8x64_is_equal_mask (u8x64_mask_load_zero (data, mask), splat); + u64 bmp = + u8x64_is_equal_mask (u8x64_mask_load_zero (data, mask), splat) & mask; return count + count_trailing_zeros (~bmp); } #endif @@ -265,7 +268,8 @@ clib_count_equal_u8 (u8 *data, uword max_count) else { u32 mask = pow2_mask (max_count - count); - u64 bmp = u8x32_msb_mask (u8x32_mask_load_zero (data, mask) == splat); + u64 bmp = + u8x32_msb_mask (u8x32_mask_load_zero (data, mask) == splat) & mask; return count + count_trailing_zeros (~bmp); } #endif diff --git a/src/vppinfra/vector/test/count_equal.c b/src/vppinfra/vector/test/count_equal.c index cd1c8a5c4d1..1ca9735af4b 100644 --- a/src/vppinfra/vector/test/count_equal.c +++ b/src/vppinfra/vector/test/count_equal.c @@ -35,37 +35,42 @@ \ mprotect (data, 1ULL < ps, PROT_NONE); \ \ - for (int i = 1; i <= (1 << ps) / sizeof (data[0]); i++) \ - data[-i] = 7; \ - \ - for (int i = 0; i < ARRAY_LEN (lengths); i++) \ + for (u8 d = 0; d < 255; d++) \ { \ - uword rv, len = lengths[i]; \ - \ - if ((rv = wfn_##type (data - len, len)) != len) \ + for (int i = 1; i <= (1 << ps) / sizeof (data[0]); i++) \ + data[-i] = d; \ + for (int i = 0; i < ARRAY_LEN (lengths); i++) \ { \ - err = clib_error_return ( \ - err, "testcase 1 failed for len %u (rv %u)", len, rv); \ - goto done; \ - } \ + uword rv, len = lengths[i]; \ \ - data[-1] = 8; \ - if (len > 1 && ((rv = wfn_##type (data - len, len)) != len - 1)) \ - { \ - err = clib_error_return ( \ - err, "testcase 2 failed for len %u (rv %u)", len, rv); \ - goto done; \ - } \ - data[-1] = 7; \ + if ((rv = wfn_##type (data - len, len)) != len) \ + { \ + err = clib_error_return ( \ + err, "testcase 1 failed for len %u data %u(rv %u)", len, d, \ + rv); \ + goto done; \ + } \ \ - data[-2] = 8; \ - if (len > 2 && ((rv = wfn_##type (data - len, len)) != len - 2)) \ - { \ - err = clib_error_return ( \ - err, "testcase 3 failed for len %u (rv %u)", len, rv); \ - goto done; \ + data[-1] = d + 1; \ + if (len > 1 && ((rv = wfn_##type (data - len, len)) != len - 1)) \ + { \ + err = clib_error_return ( \ + err, "testcase 2 failed for len %u data %u (rv %u)", len, \ + d, rv); \ + goto done; \ + } \ + data[-1] = d; \ + \ + data[-2] = d + 1; \ + if (len > 2 && ((rv = wfn_##type (data - len, len)) != len - 2)) \ + { \ + err = clib_error_return ( \ + err, "testcase 3 failed for len %u data %u (rv %u)", len, \ + d, rv); \ + goto done; \ + } \ + data[-2] = d; \ } \ - data[-2] = 7; \ } \ \ done: \ |