From 812b32dd8f637118bf65de2cdff0e95b421a963b Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 28 May 2018 21:26:47 +0200 Subject: Add VLIB_NODE_FN() macro to simplify multiversioning of node functions Change-Id: Ibab5e27277f618ceb2d543b9d6a1a5f191e7d1db Signed-off-by: Damjan Marion --- src/plugins/acl.am | 10 +-- src/plugins/acl/dataplane_node.c | 142 +++++++-------------------------------- src/plugins/avf.am | 10 +-- src/plugins/avf/input.c | 20 +----- src/plugins/avf/output.c | 2 +- src/plugins/dpdk.am | 10 +-- src/plugins/dpdk/buffer.c | 4 +- src/plugins/dpdk/device/device.c | 6 +- src/plugins/dpdk/device/node.c | 22 +----- src/plugins/memif.am | 10 +-- src/plugins/memif/device.c | 4 +- src/plugins/memif/node.c | 25 ++----- src/vlib/node.c | 20 ++++++ src/vlib/node.h | 28 ++++++++ src/vnet.am | 16 ++--- src/vnet/bonding/node.c | 26 ++----- src/vnet/ip/ip4_input.c | 34 ++-------- src/vnet/l2/l2_output.c | 28 ++------ src/vppinfra/cpu.h | 27 +++++++- 19 files changed, 160 insertions(+), 284 deletions(-) diff --git a/src/plugins/acl.am b/src/plugins/acl.am index 7d723402615..8508a1dc025 100644 --- a/src/plugins/acl.am +++ b/src/plugins/acl.am @@ -34,7 +34,7 @@ acl_plugin_la_SOURCES = \ API_FILES += acl/acl.api if CPU_X86_64 -acl_multiversioning_files = \ +acl_multiversioning_sources = \ acl/dataplane_node.c @@ -42,10 +42,10 @@ if CC_SUPPORTS_AVX2 ############################################################### # AVX2 ############################################################### -libacl_plugin_avx2_la_SOURCES = $(acl_multiversioning_files) +libacl_plugin_avx2_la_SOURCES = $(acl_multiversioning_sources) libacl_plugin_avx2_la_CFLAGS = \ $(AM_CFLAGS) @CPU_AVX2_FLAGS@ \ - -DCLIB_MULTIARCH_VARIANT=avx2 + -DCLIB_MARCH_VARIANT=avx2 noinst_LTLIBRARIES += libacl_plugin_avx2.la acl_plugin_la_LIBADD += libacl_plugin_avx2.la endif @@ -54,10 +54,10 @@ if CC_SUPPORTS_AVX512 ############################################################### # AVX512 ############################################################### -libacl_plugin_avx512_la_SOURCES = $(acl_multiversioning_files) +libacl_plugin_avx512_la_SOURCES = $(acl_multiversioning_sources) libacl_plugin_avx512_la_CFLAGS = \ $(AM_CFLAGS) @CPU_AVX512_FLAGS@ \ - -DCLIB_MULTIARCH_VARIANT=avx512 + -DCLIB_MARCH_VARIANT=avx512 noinst_LTLIBRARIES += libacl_plugin_avx512.la acl_plugin_la_LIBADD += libacl_plugin_avx512.la endif diff --git a/src/plugins/acl/dataplane_node.c b/src/plugins/acl/dataplane_node.c index 5393091f26f..351cbbd8fb2 100644 --- a/src/plugins/acl/dataplane_node.c +++ b/src/plugins/acl/dataplane_node.c @@ -327,36 +327,10 @@ acl_fa_node_fn (vlib_main_t * vm, return frame->n_vectors; } -vlib_node_function_t __clib_weak acl_in_ip4_l2_node_fn_avx512; -vlib_node_function_t __clib_weak acl_in_ip4_l2_node_fn_avx2; - -vlib_node_function_t __clib_weak acl_out_ip4_l2_node_fn_avx512; -vlib_node_function_t __clib_weak acl_out_ip4_l2_node_fn_avx2; - -vlib_node_function_t __clib_weak acl_in_ip6_l2_node_fn_avx512; -vlib_node_function_t __clib_weak acl_in_ip6_l2_node_fn_avx2; - -vlib_node_function_t __clib_weak acl_out_ip6_l2_node_fn_avx512; -vlib_node_function_t __clib_weak acl_out_ip6_l2_node_fn_avx2; - -vlib_node_function_t __clib_weak acl_in_ip4_fa_node_fn_avx512; -vlib_node_function_t __clib_weak acl_in_ip4_fa_node_fn_avx2; - -vlib_node_function_t __clib_weak acl_out_ip4_fa_node_fn_avx512; -vlib_node_function_t __clib_weak acl_out_ip4_fa_node_fn_avx2; - -vlib_node_function_t __clib_weak acl_in_ip6_fa_node_fn_avx512; -vlib_node_function_t __clib_weak acl_in_ip6_fa_node_fn_avx2; - -vlib_node_function_t __clib_weak acl_out_ip6_fa_node_fn_avx512; -vlib_node_function_t __clib_weak acl_out_ip6_fa_node_fn_avx2; - - vlib_node_registration_t acl_in_l2_ip6_node; -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (acl_in_ip6_l2_node_fn) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) +VLIB_NODE_FN (acl_in_l2_ip6_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { acl_main_t *am = &acl_main; return acl_fa_node_fn (vm, node, frame, 1, 1, 1, @@ -365,10 +339,9 @@ CLIB_MULTIARCH_FN (acl_in_ip6_l2_node_fn) (vlib_main_t * vm, } vlib_node_registration_t acl_in_l2_ip4_node; -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (acl_in_ip4_l2_node_fn) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) +VLIB_NODE_FN (acl_in_l2_ip4_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { acl_main_t *am = &acl_main; return acl_fa_node_fn (vm, node, frame, 0, 1, 1, @@ -377,10 +350,9 @@ CLIB_MULTIARCH_FN (acl_in_ip4_l2_node_fn) (vlib_main_t * vm, } vlib_node_registration_t acl_out_l2_ip6_node; -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (acl_out_ip6_l2_node_fn) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) +VLIB_NODE_FN (acl_out_l2_ip6_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { acl_main_t *am = &acl_main; return acl_fa_node_fn (vm, node, frame, 1, 0, 1, @@ -389,10 +361,9 @@ CLIB_MULTIARCH_FN (acl_out_ip6_l2_node_fn) (vlib_main_t * vm, } vlib_node_registration_t acl_out_l2_ip4_node; -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (acl_out_ip4_l2_node_fn) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) +VLIB_NODE_FN (acl_out_l2_ip4_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { acl_main_t *am = &acl_main; return acl_fa_node_fn (vm, node, frame, 0, 0, 1, @@ -403,93 +374,38 @@ CLIB_MULTIARCH_FN (acl_out_ip4_l2_node_fn) (vlib_main_t * vm, /**** L3 processing path nodes ****/ vlib_node_registration_t acl_in_fa_ip6_node; -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (acl_in_ip6_fa_node_fn) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) +VLIB_NODE_FN (acl_in_fa_ip6_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { return acl_fa_node_fn (vm, node, frame, 1, 1, 0, 0, &acl_in_fa_ip6_node); } vlib_node_registration_t acl_in_fa_ip4_node; -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (acl_in_ip4_fa_node_fn) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) +VLIB_NODE_FN (acl_in_fa_ip4_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { return acl_fa_node_fn (vm, node, frame, 0, 1, 0, 0, &acl_in_fa_ip4_node); } vlib_node_registration_t acl_out_fa_ip6_node; -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (acl_out_ip6_fa_node_fn) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) +VLIB_NODE_FN (acl_out_fa_ip6_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { return acl_fa_node_fn (vm, node, frame, 1, 0, 0, 0, &acl_out_fa_ip6_node); } vlib_node_registration_t acl_out_fa_ip4_node; -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (acl_out_ip4_fa_node_fn) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) +VLIB_NODE_FN (acl_out_fa_ip4_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { return acl_fa_node_fn (vm, node, frame, 0, 0, 0, 0, &acl_out_fa_ip4_node); } - - -#if __x86_64__ -static void __clib_constructor -acl_plugin_multiarch_select (void) -{ - if (acl_in_ip4_l2_node_fn_avx512 && clib_cpu_supports_avx512f ()) - acl_in_l2_ip4_node.function = acl_in_ip4_l2_node_fn_avx512; - else if (acl_in_ip4_l2_node_fn_avx2 && clib_cpu_supports_avx2 ()) - acl_in_l2_ip4_node.function = acl_in_ip4_l2_node_fn_avx2; - - if (acl_out_ip4_l2_node_fn_avx512 && clib_cpu_supports_avx512f ()) - acl_out_l2_ip4_node.function = acl_out_ip4_l2_node_fn_avx512; - else if (acl_out_ip4_l2_node_fn_avx2 && clib_cpu_supports_avx2 ()) - acl_out_l2_ip4_node.function = acl_out_ip4_l2_node_fn_avx2; - - if (acl_in_ip6_l2_node_fn_avx512 && clib_cpu_supports_avx512f ()) - acl_in_l2_ip6_node.function = acl_in_ip6_l2_node_fn_avx512; - else if (acl_in_ip6_l2_node_fn_avx2 && clib_cpu_supports_avx2 ()) - acl_in_l2_ip6_node.function = acl_in_ip6_l2_node_fn_avx2; - - if (acl_out_ip6_l2_node_fn_avx512 && clib_cpu_supports_avx512f ()) - acl_out_l2_ip6_node.function = acl_out_ip6_l2_node_fn_avx512; - else if (acl_out_ip6_l2_node_fn_avx2 && clib_cpu_supports_avx2 ()) - acl_out_l2_ip6_node.function = acl_out_ip6_l2_node_fn_avx2; - - if (acl_in_ip4_fa_node_fn_avx512 && clib_cpu_supports_avx512f ()) - acl_in_fa_ip4_node.function = acl_in_ip4_fa_node_fn_avx512; - else if (acl_in_ip4_fa_node_fn_avx2 && clib_cpu_supports_avx2 ()) - acl_in_fa_ip4_node.function = acl_in_ip4_fa_node_fn_avx2; - - if (acl_out_ip4_fa_node_fn_avx512 && clib_cpu_supports_avx512f ()) - acl_out_fa_ip4_node.function = acl_out_ip4_fa_node_fn_avx512; - else if (acl_out_ip4_fa_node_fn_avx2 && clib_cpu_supports_avx2 ()) - acl_out_fa_ip4_node.function = acl_out_ip4_fa_node_fn_avx2; - - if (acl_in_ip6_fa_node_fn_avx512 && clib_cpu_supports_avx512f ()) - acl_in_fa_ip6_node.function = acl_in_ip6_fa_node_fn_avx512; - else if (acl_in_ip6_fa_node_fn_avx2 && clib_cpu_supports_avx2 ()) - acl_in_fa_ip6_node.function = acl_in_ip6_fa_node_fn_avx2; - - if (acl_out_ip6_fa_node_fn_avx512 && clib_cpu_supports_avx512f ()) - acl_out_fa_ip6_node.function = acl_out_ip6_fa_node_fn_avx512; - else if (acl_out_ip6_fa_node_fn_avx2 && clib_cpu_supports_avx2 ()) - acl_out_fa_ip6_node.function = acl_out_ip6_fa_node_fn_avx2; - -} -#endif - - - -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT static u8 * format_fa_5tuple (u8 * s, va_list * args) { @@ -549,7 +465,6 @@ static char *acl_fa_error_strings[] = { VLIB_REGISTER_NODE (acl_in_l2_ip6_node) = { - .function = acl_in_ip6_l2_node_fn, .name = "acl-plugin-in-ip6-l2", .vector_size = sizeof (u32), .format_trace = format_acl_plugin_trace, @@ -565,7 +480,6 @@ VLIB_REGISTER_NODE (acl_in_l2_ip6_node) = VLIB_REGISTER_NODE (acl_in_l2_ip4_node) = { - .function = acl_in_ip4_l2_node_fn, .name = "acl-plugin-in-ip4-l2", .vector_size = sizeof (u32), .format_trace = format_acl_plugin_trace, @@ -581,7 +495,6 @@ VLIB_REGISTER_NODE (acl_in_l2_ip4_node) = VLIB_REGISTER_NODE (acl_out_l2_ip6_node) = { - .function = acl_out_ip6_l2_node_fn, .name = "acl-plugin-out-ip6-l2", .vector_size = sizeof (u32), .format_trace = format_acl_plugin_trace, @@ -597,7 +510,6 @@ VLIB_REGISTER_NODE (acl_out_l2_ip6_node) = VLIB_REGISTER_NODE (acl_out_l2_ip4_node) = { - .function = acl_out_ip4_l2_node_fn, .name = "acl-plugin-out-ip4-l2", .vector_size = sizeof (u32), .format_trace = format_acl_plugin_trace, @@ -614,7 +526,6 @@ VLIB_REGISTER_NODE (acl_out_l2_ip4_node) = VLIB_REGISTER_NODE (acl_in_fa_ip6_node) = { - .function = acl_in_ip6_fa_node_fn, .name = "acl-plugin-in-ip6-fa", .vector_size = sizeof (u32), .format_trace = format_acl_plugin_trace, @@ -637,7 +548,6 @@ VNET_FEATURE_INIT (acl_in_ip6_fa_feature, static) = VLIB_REGISTER_NODE (acl_in_fa_ip4_node) = { - .function = acl_in_ip4_fa_node_fn, .name = "acl-plugin-in-ip4-fa", .vector_size = sizeof (u32), .format_trace = format_acl_plugin_trace, @@ -661,7 +571,6 @@ VNET_FEATURE_INIT (acl_in_ip4_fa_feature, static) = VLIB_REGISTER_NODE (acl_out_fa_ip6_node) = { - .function = acl_out_ip6_fa_node_fn, .name = "acl-plugin-out-ip6-fa", .vector_size = sizeof (u32), .format_trace = format_acl_plugin_trace, @@ -684,7 +593,6 @@ VNET_FEATURE_INIT (acl_out_ip6_fa_feature, static) = VLIB_REGISTER_NODE (acl_out_fa_ip4_node) = { - .function = acl_out_ip4_fa_node_fn, .name = "acl-plugin-out-ip4-fa", .vector_size = sizeof (u32), .format_trace = format_acl_plugin_trace, diff --git a/src/plugins/avf.am b/src/plugins/avf.am index 76231a16bcb..1c7885fdc44 100644 --- a/src/plugins/avf.am +++ b/src/plugins/avf.am @@ -25,7 +25,7 @@ avf_plugin_la_SOURCES = \ noinst_HEADERS += avf/avf.h if CPU_X86_64 -avf_multiversioning_files = \ +avf_multiversioning_sources = \ avf/input.c \ avf/output.c @@ -33,10 +33,10 @@ if CC_SUPPORTS_AVX2 ############################################################### # AVX2 ############################################################### -libavf_plugin_avx2_la_SOURCES = $(avf_multiversioning_files) +libavf_plugin_avx2_la_SOURCES = $(avf_multiversioning_sources) libavf_plugin_avx2_la_CFLAGS = \ $(AM_CFLAGS) @CPU_AVX2_FLAGS@ \ - -DCLIB_MULTIARCH_VARIANT=avx2 + -DCLIB_MARCH_VARIANT=avx2 noinst_LTLIBRARIES += libavf_plugin_avx2.la avf_plugin_la_LIBADD += libavf_plugin_avx2.la endif @@ -45,10 +45,10 @@ if CC_SUPPORTS_AVX512 ############################################################### # AVX512 ############################################################### -libavf_plugin_avx512_la_SOURCES = $(avf_multiversioning_files) +libavf_plugin_avx512_la_SOURCES = $(avf_multiversioning_sources) libavf_plugin_avx512_la_CFLAGS = \ $(AM_CFLAGS) @CPU_AVX512_FLAGS@ \ - -DCLIB_MULTIARCH_VARIANT=avx512 + -DCLIB_MARCH_VARIANT=avx512 noinst_LTLIBRARIES += libavf_plugin_avx512.la avf_plugin_la_LIBADD += libavf_plugin_avx512.la endif diff --git a/src/plugins/avf/input.c b/src/plugins/avf/input.c index 931bfa39de9..5b7f48a3ed7 100644 --- a/src/plugins/avf/input.c +++ b/src/plugins/avf/input.c @@ -375,8 +375,7 @@ avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, return n_rx_packets; } -uword -CLIB_MULTIARCH_FN (avf_input) (vlib_main_t * vm, vlib_node_runtime_t * node, +VLIB_NODE_FN (avf_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { u32 n_rx = 0; @@ -399,10 +398,9 @@ CLIB_MULTIARCH_FN (avf_input) (vlib_main_t * vm, vlib_node_runtime_t * node, return n_rx; } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT /* *INDENT-OFF* */ VLIB_REGISTER_NODE (avf_input_node) = { - .function = avf_input, .name = "avf-input", .sibling_of = "device-input", .format_trace = format_avf_input_trace, @@ -411,20 +409,6 @@ VLIB_REGISTER_NODE (avf_input_node) = { .n_errors = AVF_INPUT_N_ERROR, .error_strings = avf_input_error_strings, }; - -#if __x86_64__ -vlib_node_function_t __clib_weak avf_input_avx512; -vlib_node_function_t __clib_weak avf_input_avx2; -static void __clib_constructor -avf_input_multiarch_select (void) -{ - if (avf_input_avx512 && clib_cpu_supports_avx512f ()) - avf_input_node.function = avf_input_avx512; - else if (avf_input_avx2 && clib_cpu_supports_avx2 ()) - avf_input_node.function = avf_input_avx2; -} - -#endif #endif /* *INDENT-ON* */ diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c index 8e5fa6a43f8..ec4d7f6c0a2 100644 --- a/src/plugins/avf/output.c +++ b/src/plugins/avf/output.c @@ -159,7 +159,7 @@ CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm, return frame->n_vectors - n_left; } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT #if __x86_64__ vlib_node_function_t __clib_weak avf_interface_tx_avx512; vlib_node_function_t __clib_weak avf_interface_tx_avx2; diff --git a/src/plugins/dpdk.am b/src/plugins/dpdk.am index af7d96d7c17..6c15d0abd8e 100644 --- a/src/plugins/dpdk.am +++ b/src/plugins/dpdk.am @@ -57,7 +57,7 @@ dpdk_plugin_la_SOURCES = \ API_FILES += dpdk/api/dpdk.api if CPU_X86_64 -dpdk_multiversioning_files = \ +dpdk_multiversioning_sources = \ dpdk/buffer.c \ dpdk/device/node.c \ dpdk/device/device.c @@ -66,10 +66,10 @@ if CC_SUPPORTS_AVX2 ############################################################### # AVX2 ############################################################### -libdpdk_plugin_avx2_la_SOURCES = $(dpdk_multiversioning_files) +libdpdk_plugin_avx2_la_SOURCES = $(dpdk_multiversioning_sources) libdpdk_plugin_avx2_la_CFLAGS = \ $(AM_CFLAGS) @CPU_AVX2_FLAGS@ \ - -DCLIB_MULTIARCH_VARIANT=avx2 + -DCLIB_MARCH_VARIANT=avx2 noinst_LTLIBRARIES += libdpdk_plugin_avx2.la dpdk_plugin_la_LIBADD += libdpdk_plugin_avx2.la endif @@ -78,10 +78,10 @@ if CC_SUPPORTS_AVX512 ############################################################### # AVX512 ############################################################### -libdpdk_plugin_avx512_la_SOURCES = $(dpdk_multiversioning_files) +libdpdk_plugin_avx512_la_SOURCES = $(dpdk_multiversioning_sources) libdpdk_plugin_avx512_la_CFLAGS = \ $(AM_CFLAGS) @CPU_AVX512_FLAGS@ \ - -DCLIB_MULTIARCH_VARIANT=avx512 + -DCLIB_MARCH_VARIANT=avx512 noinst_LTLIBRARIES += libdpdk_plugin_avx512.la dpdk_plugin_la_LIBADD += libdpdk_plugin_avx512.la endif diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c index 452c47931ca..3324c049511 100644 --- a/src/plugins/dpdk/buffer.c +++ b/src/plugins/dpdk/buffer.c @@ -146,7 +146,7 @@ next: } } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT static void del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) { @@ -436,7 +436,7 @@ CLIB_MULTIARCH_FN (dpdk_buffer_free_no_next) (vlib_main_t * vm, u32 * buffers, 0); } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT static void dpdk_packet_template_init (vlib_main_t * vm, void *vt, diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index 5409fe4f50e..fe659fbeaf1 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -37,7 +37,7 @@ typedef enum DPDK_TX_FUNC_N_ERROR, } dpdk_tx_func_error_t; -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT static char *dpdk_tx_func_error_strings[] = { #define _(n,s) s, foreach_dpdk_tx_func_error @@ -517,7 +517,7 @@ CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm, return tx_pkts; } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT static void dpdk_clear_hw_interface_counters (u32 instance) { @@ -695,7 +695,7 @@ dpdk_interface_tx_multiarch_select (void) #define UP_DOWN_FLAG_EVENT 1 -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT uword admin_up_down_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index daccf733372..7f4b2cb8145 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -28,7 +28,7 @@ #include -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT static char *dpdk_error_strings[] = { #define _(n,s) s, foreach_dpdk_error @@ -636,8 +636,7 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, return n_rx_packets; } -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (dpdk_input) (vlib_main_t * vm, vlib_node_runtime_t * node, +VLIB_NODE_FN (dpdk_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f) { dpdk_main_t *dm = &dpdk_main; @@ -666,10 +665,9 @@ CLIB_MULTIARCH_FN (dpdk_input) (vlib_main_t * vm, vlib_node_runtime_t * node, return n_rx_packets; } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT /* *INDENT-OFF* */ VLIB_REGISTER_NODE (dpdk_input_node) = { - .function = dpdk_input, .type = VLIB_NODE_TYPE_INPUT, .name = "dpdk-input", .sibling_of = "device-input", @@ -684,20 +682,6 @@ VLIB_REGISTER_NODE (dpdk_input_node) = { .error_strings = dpdk_error_strings, }; /* *INDENT-ON* */ - -vlib_node_function_t __clib_weak dpdk_input_avx512; -vlib_node_function_t __clib_weak dpdk_input_avx2; - -#if __x86_64__ -static void __clib_constructor -dpdk_input_multiarch_select (void) -{ - if (dpdk_input_avx512 && clib_cpu_supports_avx512f ()) - dpdk_input_node.function = dpdk_input_avx512; - else if (dpdk_input_avx2 && clib_cpu_supports_avx2 ()) - dpdk_input_node.function = dpdk_input_avx2; -} -#endif #endif /* diff --git a/src/plugins/memif.am b/src/plugins/memif.am index 6e9aa779155..33a378f4a3f 100644 --- a/src/plugins/memif.am +++ b/src/plugins/memif.am @@ -36,7 +36,7 @@ nobase_apiinclude_HEADERS += \ API_FILES += memif/memif.api if CPU_X86_64 -memif_multiversioning_files = \ +memif_multiversioning_sources = \ memif/node.c \ memif/device.c @@ -44,10 +44,10 @@ if CC_SUPPORTS_AVX2 ############################################################### # AVX2 ############################################################### -libmemif_plugin_avx2_la_SOURCES = $(memif_multiversioning_files) +libmemif_plugin_avx2_la_SOURCES = $(memif_multiversioning_sources) libmemif_plugin_avx2_la_CFLAGS = \ $(AM_CFLAGS) @CPU_AVX2_FLAGS@ \ - -DCLIB_MULTIARCH_VARIANT=avx2 + -DCLIB_MARCH_VARIANT=avx2 noinst_LTLIBRARIES += libmemif_plugin_avx2.la memif_plugin_la_LIBADD += libmemif_plugin_avx2.la endif @@ -56,10 +56,10 @@ if CC_SUPPORTS_AVX512 ############################################################### # AVX512 ############################################################### -libmemif_plugin_avx512_la_SOURCES = $(memif_multiversioning_files) +libmemif_plugin_avx512_la_SOURCES = $(memif_multiversioning_sources) libmemif_plugin_avx512_la_CFLAGS = \ $(AM_CFLAGS) @CPU_AVX512_FLAGS@ \ - -DCLIB_MULTIARCH_VARIANT=avx512 + -DCLIB_MARCH_VARIANT=avx512 noinst_LTLIBRARIES += libmemif_plugin_avx512.la memif_plugin_la_LIBADD += libmemif_plugin_avx512.la endif diff --git a/src/plugins/memif/device.c b/src/plugins/memif/device.c index c70a4ea3549..14b9d547234 100644 --- a/src/plugins/memif/device.c +++ b/src/plugins/memif/device.c @@ -46,7 +46,7 @@ static __clib_unused char *memif_tx_func_error_strings[] = { #undef _ }; -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT u8 * format_memif_device_name (u8 * s, va_list * args) { @@ -500,7 +500,7 @@ memif_subif_add_del_function (vnet_main_t * vnm, return 0; } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT /* *INDENT-OFF* */ VNET_DEVICE_CLASS (memif_device_class) = { .name = "memif", diff --git a/src/plugins/memif/node.c b/src/plugins/memif/node.c index 029e25d814a..6e2807a2054 100644 --- a/src/plugins/memif/node.c +++ b/src/plugins/memif/node.c @@ -854,10 +854,10 @@ done: return n_rx_packets; } -uword -CLIB_MULTIARCH_FN (memif_input_fn) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) + +VLIB_NODE_FN (memif_input_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { u32 n_rx = 0; memif_main_t *mm = &memif_main; @@ -910,10 +910,9 @@ CLIB_MULTIARCH_FN (memif_input_fn) (vlib_main_t * vm, return n_rx; } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT /* *INDENT-OFF* */ VLIB_REGISTER_NODE (memif_input_node) = { - .function = memif_input_fn, .name = "memif-input", .sibling_of = "device-input", .format_trace = format_memif_input_trace, @@ -922,20 +921,6 @@ VLIB_REGISTER_NODE (memif_input_node) = { .n_errors = MEMIF_INPUT_N_ERROR, .error_strings = memif_input_error_strings, }; - -vlib_node_function_t __clib_weak memif_input_fn_avx512; -vlib_node_function_t __clib_weak memif_input_fn_avx2; - -#if __x86_64__ -static void __clib_constructor -memif_input_multiarch_select (void) -{ - if (memif_input_fn_avx512 && clib_cpu_supports_avx512f ()) - memif_input_node.function = memif_input_fn_avx512; - else if (memif_input_fn_avx2 && clib_cpu_supports_avx2 ()) - memif_input_node.function = memif_input_fn_avx2; -} -#endif #endif /* *INDENT-ON* */ diff --git a/src/vlib/node.c b/src/vlib/node.c index 17dd2ea87d4..cc1732bb90d 100644 --- a/src/vlib/node.c +++ b/src/vlib/node.c @@ -324,6 +324,26 @@ register_node (vlib_main_t * vm, vlib_node_registration_t * r) ASSERT (VLIB_NODE_TYPE_INTERNAL == zero.type); } + if (r->node_fn_registrations) + { + vlib_node_fn_registration_t *fnr = r->node_fn_registrations; + int priority = -1; + + /* to avoid confusion, please remove ".function " statiement from + CLIB_NODE_REGISTRATION() if using function function candidates */ + ASSERT (r->function == 0); + + while (fnr) + { + if (fnr->priority > priority) + { + priority = fnr->priority; + r->function = fnr->function; + } + fnr = fnr->next_registration; + } + } + ASSERT (r->function != 0); n = clib_mem_alloc_no_fail (sizeof (n[0])); diff --git a/src/vlib/node.h b/src/vlib/node.h index 7177bebeffb..67eaea3f0e8 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -75,11 +75,21 @@ typedef enum VLIB_N_NODE_TYPE, } vlib_node_type_t; +typedef struct _vlib_node_fn_registration +{ + vlib_node_function_t *function; + int priority; + struct _vlib_node_fn_registration *next_registration; +} vlib_node_fn_registration_t; + typedef struct _vlib_node_registration { /* Vector processing function for this node. */ vlib_node_function_t *function; + /* Node function candidate registration with priority */ + vlib_node_fn_registration_t *node_fn_registrations; + /* Node name. */ char *name; @@ -160,6 +170,24 @@ static void __vlib_rm_node_registration_##x (void) \ } \ __VA_ARGS__ vlib_node_registration_t x +#define VLIB_NODE_FN(node) \ +uword CLIB_MARCH_SFX (node##_fn)(); \ +static vlib_node_fn_registration_t \ + CLIB_MARCH_SFX(node##_fn_registration) = \ + { .function = &CLIB_MARCH_SFX (node##_fn), }; \ + \ +static void __clib_constructor \ +CLIB_MARCH_SFX (node##_multiarch_register) (void) \ +{ \ + extern vlib_node_registration_t node; \ + vlib_node_fn_registration_t *r; \ + r = & CLIB_MARCH_SFX (node##_fn_registration); \ + r->priority = CLIB_MARCH_FN_PRIORITY(); \ + r->next_registration = node.node_fn_registrations; \ + node.node_fn_registrations = r; \ +} \ +uword CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (node##_fn) + #if CLIB_DEBUG > 0 #define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn) #define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) diff --git a/src/vnet.am b/src/vnet.am index 47159fc4591..86f3ad00f33 100644 --- a/src/vnet.am +++ b/src/vnet.am @@ -21,7 +21,7 @@ libvnet_la_DEPENDENCIES = \ libsvmdb.la \ libsvm.la \ libvlibmemory.la -libvnet_multiversioning_files = +libvnet_multiversioning_sources = libvnet_la_LIBADD = $(libvnet_la_DEPENDENCIES) -lm -lpthread -ldl -lrt @@ -128,7 +128,7 @@ libvnet_la_SOURCES += \ vnet/ethernet/p2p_ethernet_input.c \ vnet/ethernet/p2p_ethernet_api.c -libvnet_multiversioning_files += \ +libvnet_multiversioning_sources += \ vnet/l2/l2_output.c nobase_include_HEADERS += \ @@ -294,7 +294,7 @@ nobase_include_HEADERS += \ vnet/bonding/node.h \ vnet/bonding/bond.api.h -libvnet_multiversioning_files = \ +libvnet_multiversioning_sources += \ vnet/bonding/node.c API_FILES += vnet/bonding/bond.api @@ -416,7 +416,7 @@ API_FILES += \ vnet/ip/rd_cp.api \ vnet/ip/punt.api -libvnet_multiversioning_files = \ +libvnet_multiversioning_sources += \ vnet/ip/ip4_input.c ######################################## @@ -1286,10 +1286,10 @@ if CC_SUPPORTS_AVX2 ############################################################### # AVX2 ############################################################### -libvnet_avx2_la_SOURCES = $(libvnet_multiversioning_files) +libvnet_avx2_la_SOURCES = $(libvnet_multiversioning_sources) libvnet_avx2_la_CFLAGS = \ $(AM_CFLAGS) @CPU_AVX2_FLAGS@ \ - -DCLIB_MULTIARCH_VARIANT=avx2 + -DCLIB_MARCH_VARIANT=avx2 noinst_LTLIBRARIES += libvnet_avx2.la libvnet_la_LIBADD += libvnet_avx2.la endif @@ -1298,10 +1298,10 @@ if CC_SUPPORTS_AVX512 ############################################################### # AVX512 ############################################################### -libvnet_avx512_la_SOURCES = $(libvnet_multiversioning_files) +libvnet_avx512_la_SOURCES = $(libvnet_multiversioning_sources) libvnet_avx512_la_CFLAGS = \ $(AM_CFLAGS) @CPU_AVX512_FLAGS@ \ - -DCLIB_MULTIARCH_VARIANT=avx512 + -DCLIB_MARCH_VARIANT=avx512 noinst_LTLIBRARIES += libvnet_avx512.la libvnet_la_LIBADD += libvnet_avx512.la endif diff --git a/src/vnet/bonding/node.c b/src/vnet/bonding/node.c index df2da5c66fc..7d2acf7d0f7 100644 --- a/src/vnet/bonding/node.c +++ b/src/vnet/bonding/node.c @@ -36,7 +36,7 @@ typedef enum BOND_INPUT_N_ERROR, } bond_input_error_t; -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT static char *bond_input_error_strings[] = { #define _(n,s) s, foreach_bond_input_error @@ -172,10 +172,9 @@ bond_update_next (vlib_main_t * vm, vlib_node_runtime_t * node, vnet_feature_next ( /* not used */ 0, next_index, b); } -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (bond_input_fn) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) +VLIB_NODE_FN (bond_input_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { u16 thread_index = vlib_get_thread_index (); u32 *from, n_left; @@ -367,7 +366,7 @@ CLIB_MULTIARCH_FN (bond_input_fn) (vlib_main_t * vm, return frame->n_vectors; } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT static clib_error_t * bond_input_init (vlib_main_t * vm) { @@ -376,7 +375,6 @@ bond_input_init (vlib_main_t * vm) /* *INDENT-OFF* */ VLIB_REGISTER_NODE (bond_input_node) = { - .function = bond_input_fn, .name = "bond-input", .vector_size = sizeof (u32), .format_buffer = format_ethernet_header_with_length, @@ -391,20 +389,6 @@ VLIB_REGISTER_NODE (bond_input_node) = { } }; -#if __x86_64__ -vlib_node_function_t __clib_weak bond_input_fn_avx512; -vlib_node_function_t __clib_weak bond_input_fn_avx2; -static void __clib_constructor -bond_input_multiarch_select (void) -{ - if (bond_input_fn_avx512 && clib_cpu_supports_avx512f ()) - bond_input_node.function = bond_input_fn_avx512; - else if (bond_input_fn_avx2 && clib_cpu_supports_avx2 ()) - bond_input_node.function = bond_input_fn_avx2; -} -#endif - - VLIB_INIT_FUNCTION (bond_input_init); VNET_FEATURE_INIT (bond_input, static) = diff --git a/src/vnet/ip/ip4_input.c b/src/vnet/ip/ip4_input.c index 7af2bee187a..b476f95ab8b 100644 --- a/src/vnet/ip/ip4_input.c +++ b/src/vnet/ip/ip4_input.c @@ -47,7 +47,7 @@ typedef struct u8 packet_data[64]; } ip4_input_trace_t; -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT static u8 * format_ip4_input_trace (u8 * s, va_list * va) { @@ -292,22 +292,20 @@ ip4_input_inline (vlib_main_t * vm, vnet_get_config_data (... &next0 ...); or @c error-drop */ -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (ip4_input) (vlib_main_t * vm, vlib_node_runtime_t * node, +VLIB_NODE_FN (ip4_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { return ip4_input_inline (vm, node, frame, /* verify_checksum */ 1); } -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (ip4_input_no_checksum) (vlib_main_t * vm, +VLIB_NODE_FN (ip4_input_no_checksum_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { return ip4_input_inline (vm, node, frame, /* verify_checksum */ 0); } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT char *ip4_error_strings[] = { #define _(sym,string) string, foreach_ip4_error @@ -316,7 +314,6 @@ char *ip4_error_strings[] = { /* *INDENT-OFF* */ VLIB_REGISTER_NODE (ip4_input_node) = { - .function = ip4_input, .name = "ip4-input", .vector_size = sizeof (u32), @@ -337,8 +334,7 @@ VLIB_REGISTER_NODE (ip4_input_node) = { .format_trace = format_ip4_input_trace, }; -VLIB_REGISTER_NODE (ip4_input_no_checksum_node,static) = { - .function = ip4_input_no_checksum, +VLIB_REGISTER_NODE (ip4_input_no_checksum_node) = { .name = "ip4-input-no-checksum", .vector_size = sizeof (u32), @@ -357,26 +353,6 @@ VLIB_REGISTER_NODE (ip4_input_no_checksum_node,static) = { }; /* *INDENT-ON* */ -#if __x86_64__ -vlib_node_function_t __clib_weak ip4_input_avx512; -vlib_node_function_t __clib_weak ip4_input_avx2; -vlib_node_function_t __clib_weak ip4_input_no_checksum_avx512; -vlib_node_function_t __clib_weak ip4_input_no_checksum_avx2; -static void __clib_constructor -ip4_input_multiarch_select (void) -{ - if (ip4_input_no_checksum_avx512 && clib_cpu_supports_avx512f ()) - ip4_input_no_checksum_node.function = ip4_input_no_checksum_avx512; - else if (ip4_input_no_checksum_avx2 && clib_cpu_supports_avx2 ()) - ip4_input_no_checksum_node.function = ip4_input_no_checksum_avx2; - - if (ip4_input_avx512 && clib_cpu_supports_avx512f ()) - ip4_input_node.function = ip4_input_avx512; - else if (ip4_input_avx2 && clib_cpu_supports_avx2 ()) - ip4_input_node.function = ip4_input_avx2; -} -#endif - static clib_error_t * ip4_init (vlib_main_t * vm) { diff --git a/src/vnet/l2/l2_output.c b/src/vnet/l2/l2_output.c index 306049bd0db..1fa042ffd54 100644 --- a/src/vnet/l2/l2_output.c +++ b/src/vnet/l2/l2_output.c @@ -27,7 +27,7 @@ #include -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT /* Feature graph node names */ static char *l2output_feat_names[] = { #define _(sym,name) name, @@ -76,7 +76,7 @@ typedef struct u8 raw[12]; /* raw data */ } l2output_trace_t; -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT /* packet trace format function */ static u8 * format_l2output_trace (u8 * s, va_list * args) @@ -275,10 +275,9 @@ l2output_process_batch (vlib_main_t * vm, vlib_node_runtime_t * node, l2_efp, l2_vtr, l2_pbb, 1, 1); } -uword CLIB_CPU_OPTIMIZED -CLIB_MULTIARCH_FN (l2output_node_fn) (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * frame) +VLIB_NODE_FN (l2output_node) (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * frame) { u32 n_left, *from; l2output_main_t *msm = &l2output_main; @@ -431,10 +430,9 @@ CLIB_MULTIARCH_FN (l2output_node_fn) (vlib_main_t * vm, return frame->n_vectors; } -#ifndef CLIB_MULTIARCH_VARIANT +#ifndef CLIB_MARCH_VARIANT /* *INDENT-OFF* */ VLIB_REGISTER_NODE (l2output_node) = { - .function = l2output_node_fn, .name = "l2-output", .vector_size = sizeof (u32), .format_trace = format_l2output_trace, @@ -451,20 +449,6 @@ VLIB_REGISTER_NODE (l2output_node) = { [L2OUTPUT_NEXT_BAD_INTF] = "l2-output-bad-intf", }, }; - -#if __x86_64__ -vlib_node_function_t __clib_weak l2output_node_fn_avx512; -vlib_node_function_t __clib_weak l2output_node_fn_avx2; -static void __clib_constructor -l2output_multiarch_select (void) -{ - if (l2output_node_fn_avx512 && clib_cpu_supports_avx512f ()) - l2output_node.function = l2output_node_fn_avx512; - else if (l2output_node_fn_avx2 && clib_cpu_supports_avx2 ()) - l2output_node.function = l2output_node_fn_avx2; -} -#endif - /* *INDENT-ON* */ diff --git a/src/vppinfra/cpu.h b/src/vppinfra/cpu.h index f670f3e964b..110815c9f87 100644 --- a/src/vppinfra/cpu.h +++ b/src/vppinfra/cpu.h @@ -51,14 +51,16 @@ return & fn; \ } -#ifdef CLIB_MULTIARCH_VARIANT +#ifdef CLIB_MARCH_VARIANT #define __CLIB_MULTIARCH_FN(a,b) a##_##b #define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b) -#define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MULTIARCH_VARIANT) +#define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT) #else #define CLIB_MULTIARCH_FN(fn) fn #endif +#define CLIB_MARCH_SFX CLIB_MULTIARCH_FN + #define foreach_x86_64_flags \ _ (sse3, 1, ecx, 0) \ _ (ssse3, 1, ecx, 9) \ @@ -165,6 +167,27 @@ clib_cpu_supports_aes () #endif } +static inline int +clib_cpu_march_priority_avx512 () +{ + if (clib_cpu_supports_avx512f ()) + return 20; + return -1; +} + +static inline int +clib_cpu_march_priority_avx2 () +{ + if (clib_cpu_supports_avx2 ()) + return 10; + return -1; +} + +#ifdef CLIB_MARCH_VARIANT +#define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)() +#else +#define CLIB_MARCH_FN_PRIORITY() 0 +#endif #endif /* included_clib_cpu_h */ format_function_t format_cpu_uarch; -- cgit 1.2.3-korg