diff options
| -rw-r--r-- | src/vppinfra/vector/test/array_mask.c | 2 | ||||
| -rw-r--r-- | src/vppinfra/vector/test/compress.c | 8 | ||||
| -rw-r--r-- | src/vppinfra/vector/test/count_equal.c | 4 | ||||
| -rw-r--r-- | src/vppinfra/vector/test/index_to_ptr.c | 2 | ||||
| -rw-r--r-- | src/vppinfra/vector/test/ip_csum.c | 65 | ||||
| -rw-r--r-- | src/vppinfra/vector/test/mask_compare.c | 4 | ||||
| -rw-r--r-- | src/vppinfra/vector/test/memcpy_x86_64.c | 2 | ||||
| -rw-r--r-- | src/vppinfra/vector/test/test.c | 249 | ||||
| -rw-r--r-- | src/vppinfra/vector/test/test.h | 73 |
9 files changed, 386 insertions, 23 deletions
diff --git a/src/vppinfra/vector/test/array_mask.c b/src/vppinfra/vector/test/array_mask.c index a1f4da728d4..904bb214f55 100644 --- a/src/vppinfra/vector/test/array_mask.c +++ b/src/vppinfra/vector/test/array_mask.c @@ -6,7 +6,7 @@ #include <vppinfra/vector/test/test.h> #include <vppinfra/vector/array_mask.h> -__clib_test_fn void +__test_funct_fn void clib_array_mask_u32_wrapper (u32 *src, u32 mask, u32 n_elts) { clib_array_mask_u32 (src, mask, n_elts); diff --git a/src/vppinfra/vector/test/compress.c b/src/vppinfra/vector/test/compress.c index 9bc53ff1e41..4f3fd533340 100644 --- a/src/vppinfra/vector/test/compress.c +++ b/src/vppinfra/vector/test/compress.c @@ -6,25 +6,25 @@ #include <vppinfra/vector/test/test.h> #include <vppinfra/vector/compress.h> -__clib_test_fn u32 +__test_funct_fn u32 clib_compress_u64_wrapper (u64 *dst, u64 *src, u64 *mask, u32 n_elts) { return clib_compress_u64 (dst, src, mask, n_elts); } -__clib_test_fn u32 +__test_funct_fn u32 clib_compress_u32_wrapper (u32 *dst, u32 *src, u64 *mask, u32 n_elts) { return clib_compress_u32 (dst, src, mask, n_elts); } -__clib_test_fn u32 +__test_funct_fn u32 clib_compress_u16_wrapper (u16 *dst, u16 *src, u64 *mask, u32 n_elts) { return clib_compress_u16 (dst, src, mask, n_elts); } -__clib_test_fn u32 +__test_funct_fn u32 clib_compress_u8_wrapper (u8 *dst, u8 *src, u64 *mask, u32 n_elts) { return clib_compress_u8 (dst, src, mask, n_elts); diff --git a/src/vppinfra/vector/test/count_equal.c b/src/vppinfra/vector/test/count_equal.c index 1ca9735af4b..c57b0275984 100644 --- a/src/vppinfra/vector/test/count_equal.c +++ b/src/vppinfra/vector/test/count_equal.c @@ -9,8 +9,8 @@ #define foreach_clib_count_equal(type) \ typedef uword (wrapper_fn_##type) (type * a, uword maxcount); \ \ - __clib_test_fn uword clib_count_equal_##type##_wrapper (type *a, \ - uword maxcount) \ + __test_funct_fn uword clib_count_equal_##type##_wrapper (type *a, \ + uword maxcount) \ { \ return clib_count_equal_##type (a, maxcount); \ } \ diff --git a/src/vppinfra/vector/test/index_to_ptr.c b/src/vppinfra/vector/test/index_to_ptr.c index ae33020328a..7ee3b94cb7c 100644 --- a/src/vppinfra/vector/test/index_to_ptr.c +++ b/src/vppinfra/vector/test/index_to_ptr.c @@ -9,7 +9,7 @@ typedef void (wrapper_fn) (u32 *indices, void *base, u8 shift, void **ptrs, u32 n_elts); -__clib_test_fn void +__test_funct_fn void clib_index_to_ptr_u32_wrapper (u32 *indices, void *base, u8 shift, void **ptrs, u32 n_elts) { diff --git a/src/vppinfra/vector/test/ip_csum.c b/src/vppinfra/vector/test/ip_csum.c index 135d5ae63b2..cb33c036120 100644 --- a/src/vppinfra/vector/test/ip_csum.c +++ b/src/vppinfra/vector/test/ip_csum.c @@ -80,7 +80,7 @@ static clib_error_t * test_clib_ip_csum (clib_error_t *err) { u8 *buf; - buf = clib_mem_alloc_aligned (65536, CLIB_CACHE_LINE_BYTES); + buf = test_mem_alloc (65536); for (int i = 0; i < 65536; i++) buf[i] = 0xf0 + ((i * 7) & 0xf); @@ -110,11 +110,72 @@ test_clib_ip_csum (clib_error_t *err) } } done: - clib_mem_free (buf); + test_mem_free (buf); return err; } +void __test_perf_fn +perftest_ip4_hdr (int fd, test_perf_t *tp) +{ + u32 n = tp->n_ops; + u8 *data = test_mem_alloc_and_splat (20, n, (void *) &test1); + u16 *res = test_mem_alloc (n * sizeof (u16)); + + test_perf_event_enable (fd); + for (int i = 0; i < n; i++) + res[i] = clib_ip_csum (data + i * 20, 20); + test_perf_event_disable (fd); + + test_mem_free (data); + test_mem_free (res); +} + +void __test_perf_fn +perftest_tcp_payload (int fd, test_perf_t *tp) +{ + u32 n = tp->n_ops; + volatile uword *lenp = &tp->arg0; + u8 *data = test_mem_alloc_and_splat (20, n, (void *) &test1); + u16 *res = test_mem_alloc (n * sizeof (u16)); + + test_perf_event_enable (fd); + for (int i = 0; i < n; i++) + res[i] = clib_ip_csum (data + i * lenp[0], lenp[0]); + test_perf_event_disable (fd); + + test_mem_free (data); + test_mem_free (res); +} + +void __test_perf_fn +perftest_byte (int fd, test_perf_t *tp) +{ + volatile uword *np = &tp->n_ops; + u8 *data = test_mem_alloc_and_fill_inc_u8 (*np, 0, 0); + u16 *res = test_mem_alloc (sizeof (u16)); + + test_perf_event_enable (fd); + res[0] = clib_ip_csum (data, np[0]); + test_perf_event_disable (fd); + + test_mem_free (data); + test_mem_free (res); +} + REGISTER_TEST (clib_ip_csum) = { .name = "clib_ip_csum", .fn = test_clib_ip_csum, + .perf_tests = PERF_TESTS ( + { .name = "ip4_hdr", + .op_name = "IP4Hdr", + .n_ops = 1024, + .fn = perftest_ip4_hdr }, + { .name = "tcp_paylaad", + .op_name = "1460Byte", + .n_ops = 16, + .arg0 = 1460, + .fn = perftest_tcp_payload }, + { .name = "byte", .op_name = "Byte", .n_ops = 16384, .fn = perftest_byte } + + ), }; diff --git a/src/vppinfra/vector/test/mask_compare.c b/src/vppinfra/vector/test/mask_compare.c index 64df0ee084a..40cac1b6ce3 100644 --- a/src/vppinfra/vector/test/mask_compare.c +++ b/src/vppinfra/vector/test/mask_compare.c @@ -6,13 +6,13 @@ #include <vppinfra/vector/test/test.h> #include <vppinfra/vector/mask_compare.h> -__clib_test_fn void +__test_funct_fn void clib_mask_compare_u16_wrapper (u16 v, u16 *a, u64 *mask, u32 n_elts) { clib_mask_compare_u16 (v, a, mask, n_elts); } -__clib_test_fn void +__test_funct_fn void clib_mask_compare_u32_wrapper (u32 v, u32 *a, u64 *mask, u32 n_elts) { clib_mask_compare_u32 (v, a, mask, n_elts); diff --git a/src/vppinfra/vector/test/memcpy_x86_64.c b/src/vppinfra/vector/test/memcpy_x86_64.c index 78aab18d2e3..edb32d05dc6 100644 --- a/src/vppinfra/vector/test/memcpy_x86_64.c +++ b/src/vppinfra/vector/test/memcpy_x86_64.c @@ -8,7 +8,7 @@ #include <vppinfra/vector/test/test.h> #include <vppinfra/vector/mask_compare.h> -__clib_test_fn void +__test_funct_fn void wrapper (u8 *dst, u8 *src, uword n) { clib_memcpy_x86_64 (dst, src, n); diff --git a/src/vppinfra/vector/test/test.c b/src/vppinfra/vector/test/test.c index 1a8b9d6ea10..d098766481c 100644 --- a/src/vppinfra/vector/test/test.c +++ b/src/vppinfra/vector/test/test.c @@ -4,8 +4,9 @@ #include <vppinfra/format.h> #include <vppinfra/vector/test/test.h> +#include <vppinfra/error.h> -test_registration_t *test_registrations[CLIB_MARCH_TYPE_N_VARIANTS] = {}; +test_main_t test_main; int test_march_supported (clib_march_variant_type_t type) @@ -18,14 +19,12 @@ test_march_supported (clib_march_variant_type_t type) return 0; } -int -main (int argc, char *argv[]) +clib_error_t * +test_funct (test_main_t *tm) { - clib_mem_init (0, 64ULL << 20); - for (int i = 0; i < CLIB_MARCH_TYPE_N_VARIANTS; i++) { - test_registration_t *r = test_registrations[i]; + test_registration_t *r = tm->registrations[i]; if (r == 0 || test_march_supported (i) < 0) continue; @@ -51,3 +50,241 @@ main (int argc, char *argv[]) fformat (stdout, "\n"); return 0; } + +#define TEST_PERF_MAX_EVENTS 7 +typedef struct +{ + u64 config[TEST_PERF_MAX_EVENTS]; + u8 n_events; + format_function_t *format_fn; +} test_perf_event_bundle_t; + +static u8 * +format_test_perf_bundle_default (u8 *s, va_list *args) +{ + test_perf_event_bundle_t __clib_unused *b = + va_arg (*args, test_perf_event_bundle_t *); + test_perf_t *tp = va_arg (*args, test_perf_t *); + u64 *data = va_arg (*args, u64 *); + + if (data) + s = format (s, "%5.2f", (f64) data[1] / data[0]); + else + s = format (s, "%5s", "IPC"); + + if (data) + s = format (s, "%8.2f", (f64) data[0] / tp->n_ops); + else + s = format (s, "%8s", "Clks/Op"); + + if (data) + s = format (s, "%8.2f", (f64) data[1] / tp->n_ops); + else + s = format (s, "%8s", "Inst/Op"); + + if (data) + s = format (s, "%9.2f", (f64) data[2] / tp->n_ops); + else + s = format (s, "%9s", "Brnch/Op"); + + if (data) + s = format (s, "%10.2f", (f64) data[3] / tp->n_ops); + else + s = format (s, "%10s", "BrMiss/Op"); + return s; +} + +test_perf_event_bundle_t perf_bundles[] = { { + .config[0] = PERF_COUNT_HW_CPU_CYCLES, + .config[1] = PERF_COUNT_HW_INSTRUCTIONS, + .config[2] = PERF_COUNT_HW_BRANCH_INSTRUCTIONS, + .config[3] = PERF_COUNT_HW_BRANCH_MISSES, + .n_events = 4, + .format_fn = format_test_perf_bundle_default, +} }; + +#ifdef __linux__ +clib_error_t * +test_perf (test_main_t *tm) +{ + clib_error_t *err = 0; + test_perf_event_bundle_t *b = perf_bundles; + int group_fd = -1, fds[TEST_PERF_MAX_EVENTS]; + u64 count[TEST_PERF_MAX_EVENTS + 3] = {}; + struct perf_event_attr pe = { + .size = sizeof (struct perf_event_attr), + .type = PERF_TYPE_HARDWARE, + .disabled = 1, + .exclude_kernel = 1, + .exclude_hv = 1, + .pinned = 1, + .exclusive = 1, + .read_format = (PERF_FORMAT_GROUP | PERF_FORMAT_TOTAL_TIME_ENABLED | + PERF_FORMAT_TOTAL_TIME_RUNNING), + }; + + for (int i = 0; i < TEST_PERF_MAX_EVENTS; i++) + fds[i] = -1; + + for (int i = 0; i < b->n_events; i++) + { + pe.config = b->config[i]; + int fd = syscall (__NR_perf_event_open, &pe, /* pid */ 0, /* cpu */ -1, + /* group_fd */ group_fd, /* flags */ 0); + if (fd < 0) + { + err = clib_error_return_unix (0, "perf_event_open"); + goto done; + } + + if (group_fd == -1) + { + group_fd = fd; + pe.pinned = 0; + pe.exclusive = 0; + } + fds[i] = fd; + } + + for (int i = 0; i < CLIB_MARCH_TYPE_N_VARIANTS; i++) + { + test_registration_t *r = tm->registrations[i]; + + if (r == 0 || test_march_supported (i) < 0) + continue; + + fformat (stdout, "\nMultiarch Variant: %U\n", format_march_variant, i); + fformat (stdout, + "-------------------------------------------------------\n"); + while (r) + { + if (r->perf_tests) + { + test_perf_t *pt = r->perf_tests; + fformat (stdout, "%-22s%-12s%U\n", r->name, "OpType", + b->format_fn, b, pt, 0UL); + do + { + u32 read_size = (b->n_events + 3) * sizeof (u64); + for (int i = 0; i < tm->repeat; i++) + { + test_perf_event_reset (group_fd); + pt->fn (group_fd, pt); + if ((read (group_fd, &count, read_size) != read_size)) + { + err = clib_error_return_unix (0, "read"); + goto done; + } + if (count[1] != count[2]) + clib_warning ( + "perf counters were not running all the time." +#ifdef __x86_64__ + "\nConsider turning NMI watchdog off ('sysctl -w " + "kernel.nmi_watchdog=0')." +#endif + ); + fformat (stdout, " %-20s%-12s%U\n", pt->name, + pt->op_name ? pt->op_name : "", b->format_fn, b, + pt, count + 3); + } + } + while ((++pt)->fn); + } + r = r->next; + } + } + +done: + for (int i = 0; i < TEST_PERF_MAX_EVENTS; i++) + if (fds[i] != -1) + close (fds[i]); + return err; +} +#endif + +int +main (int argc, char *argv[]) +{ + test_main_t *tm = &test_main; + unformat_input_t _i = {}, *i = &_i; + clib_mem_init (0, 64ULL << 20); + clib_error_t *err; + int perf = 0; + + /* defaults */ + tm->repeat = 3; + + unformat_init_command_line (i, argv); + + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "perf")) + perf = 1; + else if (unformat (i, "repeat %d", &tm->repeat)) + ; + else + { + clib_warning ("unknown input '%U'", format_unformat_error, i); + exit (1); + } + } + + if (perf) + err = test_perf (tm); + else + err = test_funct (tm); + + if (err) + { + clib_error_report (err); + fformat (stderr, "\n"); + return 1; + } + return 0; +} + +void * +test_mem_alloc (uword size) +{ + void *rv; + size = round_pow2 (size, CLIB_CACHE_LINE_BYTES); + rv = clib_mem_alloc_aligned (size, CLIB_CACHE_LINE_BYTES); + clib_memset_u8 (rv, 0, size); + return rv; +} + +void * +test_mem_alloc_and_fill_inc_u8 (uword size, u8 start, u8 mask) +{ + u8 *rv; + mask = mask ? mask : 0xff; + size = round_pow2 (size, CLIB_CACHE_LINE_BYTES); + rv = clib_mem_alloc_aligned (size, CLIB_CACHE_LINE_BYTES); + for (uword i = 0; i < size; i++) + rv[i] = ((u8) i + start) & mask; + return rv; +} + +void * +test_mem_alloc_and_splat (uword elt_size, uword n_elts, void *elt) +{ + u8 *rv, *e; + uword data_size = elt_size * n_elts; + uword alloc_size = round_pow2 (data_size, CLIB_CACHE_LINE_BYTES); + e = rv = clib_mem_alloc_aligned (alloc_size, CLIB_CACHE_LINE_BYTES); + while (e - rv < data_size) + { + clib_memcpy_fast (e, elt, elt_size); + e += elt_size; + } + + if (data_size < alloc_size) + clib_memset_u8 (e, 0, alloc_size - data_size); + return rv; +} + +void +test_mem_free (void *p) +{ + clib_mem_free (p); +} diff --git a/src/vppinfra/vector/test/test.h b/src/vppinfra/vector/test/test.h index bc499fb24e8..02169c1d6de 100644 --- a/src/vppinfra/vector/test/test.h +++ b/src/vppinfra/vector/test/test.h @@ -6,20 +6,44 @@ #define included_test_test_h #include <vppinfra/cpu.h> +#ifdef __linux__ +#include <sys/ioctl.h> +#include <linux/perf_event.h> +#endif typedef clib_error_t *(test_fn_t) (clib_error_t *); +struct test_perf_; +typedef void (test_perf_fn_t) (int fd, struct test_perf_ *tp); + +typedef struct test_perf_ +{ + u64 n_ops; + u64 arg0; + char *op_name; + char *name; + test_perf_fn_t *fn; +} test_perf_t; + typedef struct test_registration_ { char *name; u8 multiarch : 1; test_fn_t *fn; + test_perf_t *perf_tests; + u32 n_perf_tests; struct test_registration_ *next; } test_registration_t; -extern test_registration_t *test_registrations[CLIB_MARCH_TYPE_N_VARIANTS]; +typedef struct +{ + test_registration_t *registrations[CLIB_MARCH_TYPE_N_VARIANTS]; + u32 repeat; +} test_main_t; +extern test_main_t test_main; -#define __clib_test_fn static __clib_noinline __clib_section (".test_wrapper") +#define __test_funct_fn static __clib_noinline __clib_section (".test_func") +#define __test_perf_fn static __clib_noinline __clib_section (".test_perf") #define REGISTER_TEST(x) \ test_registration_t CLIB_MARCH_SFX (__test_##x); \ @@ -27,9 +51,50 @@ extern test_registration_t *test_registrations[CLIB_MARCH_TYPE_N_VARIANTS]; void) \ { \ test_registration_t *r = &CLIB_MARCH_SFX (__test_##x); \ - r->next = test_registrations[CLIB_MARCH_SFX (CLIB_MARCH_VARIANT_TYPE)]; \ - test_registrations[CLIB_MARCH_SFX (CLIB_MARCH_VARIANT_TYPE)] = r; \ + r->next = \ + test_main.registrations[CLIB_MARCH_SFX (CLIB_MARCH_VARIANT_TYPE)]; \ + test_main.registrations[CLIB_MARCH_SFX (CLIB_MARCH_VARIANT_TYPE)] = r; \ } \ test_registration_t CLIB_MARCH_SFX (__test_##x) +#define PERF_TESTS(...) \ + (test_perf_t[]) \ + { \ + __VA_ARGS__, {} \ + } + +static_always_inline void +test_perf_event_ioctl (int fd, u32 req) +{ +#ifdef __x86_64__ + asm inline("syscall" + : + : "D"(fd), "S"(req), "a"(__NR_ioctl), "d"(PERF_IOC_FLAG_GROUP) + : "rcx", "r11" /* registers modified by kernel */); +#else + ioctl (fd, req, PERF_IOC_FLAG_GROUP); +#endif +} + +static_always_inline void +test_perf_event_reset (int fd) +{ + test_perf_event_ioctl (fd, PERF_EVENT_IOC_RESET); +} +static_always_inline void +test_perf_event_enable (int fd) +{ + test_perf_event_ioctl (fd, PERF_EVENT_IOC_ENABLE); +} +static_always_inline void +test_perf_event_disable (int fd) +{ + test_perf_event_ioctl (fd, PERF_EVENT_IOC_DISABLE); +} + +void *test_mem_alloc (uword size); +void *test_mem_alloc_and_fill_inc_u8 (uword size, u8 start, u8 mask); +void *test_mem_alloc_and_splat (uword elt_size, uword n_elts, void *elt); +void test_mem_free (void *p); + #endif |