summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/plugins/acl.am10
-rw-r--r--src/plugins/acl/dataplane_node.c142
-rw-r--r--src/plugins/avf.am10
-rw-r--r--src/plugins/avf/input.c20
-rw-r--r--src/plugins/avf/output.c2
-rw-r--r--src/plugins/dpdk.am10
-rw-r--r--src/plugins/dpdk/buffer.c4
-rw-r--r--src/plugins/dpdk/device/device.c6
-rw-r--r--src/plugins/dpdk/device/node.c22
-rw-r--r--src/plugins/memif.am10
-rw-r--r--src/plugins/memif/device.c4
-rw-r--r--src/plugins/memif/node.c25
-rw-r--r--src/vlib/node.c20
-rw-r--r--src/vlib/node.h28
-rw-r--r--src/vnet.am16
-rw-r--r--src/vnet/bonding/node.c26
-rw-r--r--src/vnet/ip/ip4_input.c34
-rw-r--r--src/vnet/l2/l2_output.c28
-rw-r--r--src/vppinfra/cpu.h27
19 files changed, 160 insertions, 284 deletions
diff --git a/src/plugins/acl.am b/src/plugins/acl.am
index 7d723402615..8508a1dc025 100644
--- a/src/plugins/acl.am
+++ b/src/plugins/acl.am
@@ -34,7 +34,7 @@ acl_plugin_la_SOURCES = \
API_FILES += acl/acl.api
if CPU_X86_64
-acl_multiversioning_files = \
+acl_multiversioning_sources = \
acl/dataplane_node.c
@@ -42,10 +42,10 @@ if CC_SUPPORTS_AVX2
###############################################################
# AVX2
###############################################################
-libacl_plugin_avx2_la_SOURCES = $(acl_multiversioning_files)
+libacl_plugin_avx2_la_SOURCES = $(acl_multiversioning_sources)
libacl_plugin_avx2_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX2_FLAGS@ \
- -DCLIB_MULTIARCH_VARIANT=avx2
+ -DCLIB_MARCH_VARIANT=avx2
noinst_LTLIBRARIES += libacl_plugin_avx2.la
acl_plugin_la_LIBADD += libacl_plugin_avx2.la
endif
@@ -54,10 +54,10 @@ if CC_SUPPORTS_AVX512
###############################################################
# AVX512
###############################################################
-libacl_plugin_avx512_la_SOURCES = $(acl_multiversioning_files)
+libacl_plugin_avx512_la_SOURCES = $(acl_multiversioning_sources)
libacl_plugin_avx512_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX512_FLAGS@ \
- -DCLIB_MULTIARCH_VARIANT=avx512
+ -DCLIB_MARCH_VARIANT=avx512
noinst_LTLIBRARIES += libacl_plugin_avx512.la
acl_plugin_la_LIBADD += libacl_plugin_avx512.la
endif
diff --git a/src/plugins/acl/dataplane_node.c b/src/plugins/acl/dataplane_node.c
index 5393091f26f..351cbbd8fb2 100644
--- a/src/plugins/acl/dataplane_node.c
+++ b/src/plugins/acl/dataplane_node.c
@@ -327,36 +327,10 @@ acl_fa_node_fn (vlib_main_t * vm,
return frame->n_vectors;
}
-vlib_node_function_t __clib_weak acl_in_ip4_l2_node_fn_avx512;
-vlib_node_function_t __clib_weak acl_in_ip4_l2_node_fn_avx2;
-
-vlib_node_function_t __clib_weak acl_out_ip4_l2_node_fn_avx512;
-vlib_node_function_t __clib_weak acl_out_ip4_l2_node_fn_avx2;
-
-vlib_node_function_t __clib_weak acl_in_ip6_l2_node_fn_avx512;
-vlib_node_function_t __clib_weak acl_in_ip6_l2_node_fn_avx2;
-
-vlib_node_function_t __clib_weak acl_out_ip6_l2_node_fn_avx512;
-vlib_node_function_t __clib_weak acl_out_ip6_l2_node_fn_avx2;
-
-vlib_node_function_t __clib_weak acl_in_ip4_fa_node_fn_avx512;
-vlib_node_function_t __clib_weak acl_in_ip4_fa_node_fn_avx2;
-
-vlib_node_function_t __clib_weak acl_out_ip4_fa_node_fn_avx512;
-vlib_node_function_t __clib_weak acl_out_ip4_fa_node_fn_avx2;
-
-vlib_node_function_t __clib_weak acl_in_ip6_fa_node_fn_avx512;
-vlib_node_function_t __clib_weak acl_in_ip6_fa_node_fn_avx2;
-
-vlib_node_function_t __clib_weak acl_out_ip6_fa_node_fn_avx512;
-vlib_node_function_t __clib_weak acl_out_ip6_fa_node_fn_avx2;
-
-
vlib_node_registration_t acl_in_l2_ip6_node;
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (acl_in_ip6_l2_node_fn) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame)
+VLIB_NODE_FN (acl_in_l2_ip6_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
{
acl_main_t *am = &acl_main;
return acl_fa_node_fn (vm, node, frame, 1, 1, 1,
@@ -365,10 +339,9 @@ CLIB_MULTIARCH_FN (acl_in_ip6_l2_node_fn) (vlib_main_t * vm,
}
vlib_node_registration_t acl_in_l2_ip4_node;
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (acl_in_ip4_l2_node_fn) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame)
+VLIB_NODE_FN (acl_in_l2_ip4_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
{
acl_main_t *am = &acl_main;
return acl_fa_node_fn (vm, node, frame, 0, 1, 1,
@@ -377,10 +350,9 @@ CLIB_MULTIARCH_FN (acl_in_ip4_l2_node_fn) (vlib_main_t * vm,
}
vlib_node_registration_t acl_out_l2_ip6_node;
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (acl_out_ip6_l2_node_fn) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame)
+VLIB_NODE_FN (acl_out_l2_ip6_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
{
acl_main_t *am = &acl_main;
return acl_fa_node_fn (vm, node, frame, 1, 0, 1,
@@ -389,10 +361,9 @@ CLIB_MULTIARCH_FN (acl_out_ip6_l2_node_fn) (vlib_main_t * vm,
}
vlib_node_registration_t acl_out_l2_ip4_node;
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (acl_out_ip4_l2_node_fn) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame)
+VLIB_NODE_FN (acl_out_l2_ip4_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
{
acl_main_t *am = &acl_main;
return acl_fa_node_fn (vm, node, frame, 0, 0, 1,
@@ -403,93 +374,38 @@ CLIB_MULTIARCH_FN (acl_out_ip4_l2_node_fn) (vlib_main_t * vm,
/**** L3 processing path nodes ****/
vlib_node_registration_t acl_in_fa_ip6_node;
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (acl_in_ip6_fa_node_fn) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame)
+VLIB_NODE_FN (acl_in_fa_ip6_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
{
return acl_fa_node_fn (vm, node, frame, 1, 1, 0, 0, &acl_in_fa_ip6_node);
}
vlib_node_registration_t acl_in_fa_ip4_node;
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (acl_in_ip4_fa_node_fn) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame)
+VLIB_NODE_FN (acl_in_fa_ip4_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
{
return acl_fa_node_fn (vm, node, frame, 0, 1, 0, 0, &acl_in_fa_ip4_node);
}
vlib_node_registration_t acl_out_fa_ip6_node;
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (acl_out_ip6_fa_node_fn) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame)
+VLIB_NODE_FN (acl_out_fa_ip6_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
{
return acl_fa_node_fn (vm, node, frame, 1, 0, 0, 0, &acl_out_fa_ip6_node);
}
vlib_node_registration_t acl_out_fa_ip4_node;
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (acl_out_ip4_fa_node_fn) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame)
+VLIB_NODE_FN (acl_out_fa_ip4_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
{
return acl_fa_node_fn (vm, node, frame, 0, 0, 0, 0, &acl_out_fa_ip4_node);
}
-
-
-#if __x86_64__
-static void __clib_constructor
-acl_plugin_multiarch_select (void)
-{
- if (acl_in_ip4_l2_node_fn_avx512 && clib_cpu_supports_avx512f ())
- acl_in_l2_ip4_node.function = acl_in_ip4_l2_node_fn_avx512;
- else if (acl_in_ip4_l2_node_fn_avx2 && clib_cpu_supports_avx2 ())
- acl_in_l2_ip4_node.function = acl_in_ip4_l2_node_fn_avx2;
-
- if (acl_out_ip4_l2_node_fn_avx512 && clib_cpu_supports_avx512f ())
- acl_out_l2_ip4_node.function = acl_out_ip4_l2_node_fn_avx512;
- else if (acl_out_ip4_l2_node_fn_avx2 && clib_cpu_supports_avx2 ())
- acl_out_l2_ip4_node.function = acl_out_ip4_l2_node_fn_avx2;
-
- if (acl_in_ip6_l2_node_fn_avx512 && clib_cpu_supports_avx512f ())
- acl_in_l2_ip6_node.function = acl_in_ip6_l2_node_fn_avx512;
- else if (acl_in_ip6_l2_node_fn_avx2 && clib_cpu_supports_avx2 ())
- acl_in_l2_ip6_node.function = acl_in_ip6_l2_node_fn_avx2;
-
- if (acl_out_ip6_l2_node_fn_avx512 && clib_cpu_supports_avx512f ())
- acl_out_l2_ip6_node.function = acl_out_ip6_l2_node_fn_avx512;
- else if (acl_out_ip6_l2_node_fn_avx2 && clib_cpu_supports_avx2 ())
- acl_out_l2_ip6_node.function = acl_out_ip6_l2_node_fn_avx2;
-
- if (acl_in_ip4_fa_node_fn_avx512 && clib_cpu_supports_avx512f ())
- acl_in_fa_ip4_node.function = acl_in_ip4_fa_node_fn_avx512;
- else if (acl_in_ip4_fa_node_fn_avx2 && clib_cpu_supports_avx2 ())
- acl_in_fa_ip4_node.function = acl_in_ip4_fa_node_fn_avx2;
-
- if (acl_out_ip4_fa_node_fn_avx512 && clib_cpu_supports_avx512f ())
- acl_out_fa_ip4_node.function = acl_out_ip4_fa_node_fn_avx512;
- else if (acl_out_ip4_fa_node_fn_avx2 && clib_cpu_supports_avx2 ())
- acl_out_fa_ip4_node.function = acl_out_ip4_fa_node_fn_avx2;
-
- if (acl_in_ip6_fa_node_fn_avx512 && clib_cpu_supports_avx512f ())
- acl_in_fa_ip6_node.function = acl_in_ip6_fa_node_fn_avx512;
- else if (acl_in_ip6_fa_node_fn_avx2 && clib_cpu_supports_avx2 ())
- acl_in_fa_ip6_node.function = acl_in_ip6_fa_node_fn_avx2;
-
- if (acl_out_ip6_fa_node_fn_avx512 && clib_cpu_supports_avx512f ())
- acl_out_fa_ip6_node.function = acl_out_ip6_fa_node_fn_avx512;
- else if (acl_out_ip6_fa_node_fn_avx2 && clib_cpu_supports_avx2 ())
- acl_out_fa_ip6_node.function = acl_out_ip6_fa_node_fn_avx2;
-
-}
-#endif
-
-
-
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
static u8 *
format_fa_5tuple (u8 * s, va_list * args)
{
@@ -549,7 +465,6 @@ static char *acl_fa_error_strings[] = {
VLIB_REGISTER_NODE (acl_in_l2_ip6_node) =
{
- .function = acl_in_ip6_l2_node_fn,
.name = "acl-plugin-in-ip6-l2",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -565,7 +480,6 @@ VLIB_REGISTER_NODE (acl_in_l2_ip6_node) =
VLIB_REGISTER_NODE (acl_in_l2_ip4_node) =
{
- .function = acl_in_ip4_l2_node_fn,
.name = "acl-plugin-in-ip4-l2",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -581,7 +495,6 @@ VLIB_REGISTER_NODE (acl_in_l2_ip4_node) =
VLIB_REGISTER_NODE (acl_out_l2_ip6_node) =
{
- .function = acl_out_ip6_l2_node_fn,
.name = "acl-plugin-out-ip6-l2",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -597,7 +510,6 @@ VLIB_REGISTER_NODE (acl_out_l2_ip6_node) =
VLIB_REGISTER_NODE (acl_out_l2_ip4_node) =
{
- .function = acl_out_ip4_l2_node_fn,
.name = "acl-plugin-out-ip4-l2",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -614,7 +526,6 @@ VLIB_REGISTER_NODE (acl_out_l2_ip4_node) =
VLIB_REGISTER_NODE (acl_in_fa_ip6_node) =
{
- .function = acl_in_ip6_fa_node_fn,
.name = "acl-plugin-in-ip6-fa",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -637,7 +548,6 @@ VNET_FEATURE_INIT (acl_in_ip6_fa_feature, static) =
VLIB_REGISTER_NODE (acl_in_fa_ip4_node) =
{
- .function = acl_in_ip4_fa_node_fn,
.name = "acl-plugin-in-ip4-fa",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -661,7 +571,6 @@ VNET_FEATURE_INIT (acl_in_ip4_fa_feature, static) =
VLIB_REGISTER_NODE (acl_out_fa_ip6_node) =
{
- .function = acl_out_ip6_fa_node_fn,
.name = "acl-plugin-out-ip6-fa",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
@@ -684,7 +593,6 @@ VNET_FEATURE_INIT (acl_out_ip6_fa_feature, static) =
VLIB_REGISTER_NODE (acl_out_fa_ip4_node) =
{
- .function = acl_out_ip4_fa_node_fn,
.name = "acl-plugin-out-ip4-fa",
.vector_size = sizeof (u32),
.format_trace = format_acl_plugin_trace,
diff --git a/src/plugins/avf.am b/src/plugins/avf.am
index 76231a16bcb..1c7885fdc44 100644
--- a/src/plugins/avf.am
+++ b/src/plugins/avf.am
@@ -25,7 +25,7 @@ avf_plugin_la_SOURCES = \
noinst_HEADERS += avf/avf.h
if CPU_X86_64
-avf_multiversioning_files = \
+avf_multiversioning_sources = \
avf/input.c \
avf/output.c
@@ -33,10 +33,10 @@ if CC_SUPPORTS_AVX2
###############################################################
# AVX2
###############################################################
-libavf_plugin_avx2_la_SOURCES = $(avf_multiversioning_files)
+libavf_plugin_avx2_la_SOURCES = $(avf_multiversioning_sources)
libavf_plugin_avx2_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX2_FLAGS@ \
- -DCLIB_MULTIARCH_VARIANT=avx2
+ -DCLIB_MARCH_VARIANT=avx2
noinst_LTLIBRARIES += libavf_plugin_avx2.la
avf_plugin_la_LIBADD += libavf_plugin_avx2.la
endif
@@ -45,10 +45,10 @@ if CC_SUPPORTS_AVX512
###############################################################
# AVX512
###############################################################
-libavf_plugin_avx512_la_SOURCES = $(avf_multiversioning_files)
+libavf_plugin_avx512_la_SOURCES = $(avf_multiversioning_sources)
libavf_plugin_avx512_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX512_FLAGS@ \
- -DCLIB_MULTIARCH_VARIANT=avx512
+ -DCLIB_MARCH_VARIANT=avx512
noinst_LTLIBRARIES += libavf_plugin_avx512.la
avf_plugin_la_LIBADD += libavf_plugin_avx512.la
endif
diff --git a/src/plugins/avf/input.c b/src/plugins/avf/input.c
index 931bfa39de9..5b7f48a3ed7 100644
--- a/src/plugins/avf/input.c
+++ b/src/plugins/avf/input.c
@@ -375,8 +375,7 @@ avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
return n_rx_packets;
}
-uword
-CLIB_MULTIARCH_FN (avf_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
+VLIB_NODE_FN (avf_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
u32 n_rx = 0;
@@ -399,10 +398,9 @@ CLIB_MULTIARCH_FN (avf_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
return n_rx;
}
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (avf_input_node) = {
- .function = avf_input,
.name = "avf-input",
.sibling_of = "device-input",
.format_trace = format_avf_input_trace,
@@ -411,20 +409,6 @@ VLIB_REGISTER_NODE (avf_input_node) = {
.n_errors = AVF_INPUT_N_ERROR,
.error_strings = avf_input_error_strings,
};
-
-#if __x86_64__
-vlib_node_function_t __clib_weak avf_input_avx512;
-vlib_node_function_t __clib_weak avf_input_avx2;
-static void __clib_constructor
-avf_input_multiarch_select (void)
-{
- if (avf_input_avx512 && clib_cpu_supports_avx512f ())
- avf_input_node.function = avf_input_avx512;
- else if (avf_input_avx2 && clib_cpu_supports_avx2 ())
- avf_input_node.function = avf_input_avx2;
-}
-
-#endif
#endif
/* *INDENT-ON* */
diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c
index 8e5fa6a43f8..ec4d7f6c0a2 100644
--- a/src/plugins/avf/output.c
+++ b/src/plugins/avf/output.c
@@ -159,7 +159,7 @@ CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm,
return frame->n_vectors - n_left;
}
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
#if __x86_64__
vlib_node_function_t __clib_weak avf_interface_tx_avx512;
vlib_node_function_t __clib_weak avf_interface_tx_avx2;
diff --git a/src/plugins/dpdk.am b/src/plugins/dpdk.am
index af7d96d7c17..6c15d0abd8e 100644
--- a/src/plugins/dpdk.am
+++ b/src/plugins/dpdk.am
@@ -57,7 +57,7 @@ dpdk_plugin_la_SOURCES = \
API_FILES += dpdk/api/dpdk.api
if CPU_X86_64
-dpdk_multiversioning_files = \
+dpdk_multiversioning_sources = \
dpdk/buffer.c \
dpdk/device/node.c \
dpdk/device/device.c
@@ -66,10 +66,10 @@ if CC_SUPPORTS_AVX2
###############################################################
# AVX2
###############################################################
-libdpdk_plugin_avx2_la_SOURCES = $(dpdk_multiversioning_files)
+libdpdk_plugin_avx2_la_SOURCES = $(dpdk_multiversioning_sources)
libdpdk_plugin_avx2_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX2_FLAGS@ \
- -DCLIB_MULTIARCH_VARIANT=avx2
+ -DCLIB_MARCH_VARIANT=avx2
noinst_LTLIBRARIES += libdpdk_plugin_avx2.la
dpdk_plugin_la_LIBADD += libdpdk_plugin_avx2.la
endif
@@ -78,10 +78,10 @@ if CC_SUPPORTS_AVX512
###############################################################
# AVX512
###############################################################
-libdpdk_plugin_avx512_la_SOURCES = $(dpdk_multiversioning_files)
+libdpdk_plugin_avx512_la_SOURCES = $(dpdk_multiversioning_sources)
libdpdk_plugin_avx512_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX512_FLAGS@ \
- -DCLIB_MULTIARCH_VARIANT=avx512
+ -DCLIB_MARCH_VARIANT=avx512
noinst_LTLIBRARIES += libdpdk_plugin_avx512.la
dpdk_plugin_la_LIBADD += libdpdk_plugin_avx512.la
endif
diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c
index 452c47931ca..3324c049511 100644
--- a/src/plugins/dpdk/buffer.c
+++ b/src/plugins/dpdk/buffer.c
@@ -146,7 +146,7 @@ next:
}
}
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
static void
del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f)
{
@@ -436,7 +436,7 @@ CLIB_MULTIARCH_FN (dpdk_buffer_free_no_next) (vlib_main_t * vm, u32 * buffers,
0);
}
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
static void
dpdk_packet_template_init (vlib_main_t * vm,
void *vt,
diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index 5409fe4f50e..fe659fbeaf1 100644
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -37,7 +37,7 @@ typedef enum
DPDK_TX_FUNC_N_ERROR,
} dpdk_tx_func_error_t;
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
static char *dpdk_tx_func_error_strings[] = {
#define _(n,s) s,
foreach_dpdk_tx_func_error
@@ -517,7 +517,7 @@ CLIB_MULTIARCH_FN (dpdk_interface_tx) (vlib_main_t * vm,
return tx_pkts;
}
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
static void
dpdk_clear_hw_interface_counters (u32 instance)
{
@@ -695,7 +695,7 @@ dpdk_interface_tx_multiarch_select (void)
#define UP_DOWN_FLAG_EVENT 1
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
uword
admin_up_down_process (vlib_main_t * vm,
vlib_node_runtime_t * rt, vlib_frame_t * f)
diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c
index daccf733372..7f4b2cb8145 100644
--- a/src/plugins/dpdk/device/node.c
+++ b/src/plugins/dpdk/device/node.c
@@ -28,7 +28,7 @@
#include <dpdk/device/dpdk_priv.h>
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
static char *dpdk_error_strings[] = {
#define _(n,s) s,
foreach_dpdk_error
@@ -636,8 +636,7 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd,
return n_rx_packets;
}
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (dpdk_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
+VLIB_NODE_FN (dpdk_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * f)
{
dpdk_main_t *dm = &dpdk_main;
@@ -666,10 +665,9 @@ CLIB_MULTIARCH_FN (dpdk_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
return n_rx_packets;
}
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (dpdk_input_node) = {
- .function = dpdk_input,
.type = VLIB_NODE_TYPE_INPUT,
.name = "dpdk-input",
.sibling_of = "device-input",
@@ -684,20 +682,6 @@ VLIB_REGISTER_NODE (dpdk_input_node) = {
.error_strings = dpdk_error_strings,
};
/* *INDENT-ON* */
-
-vlib_node_function_t __clib_weak dpdk_input_avx512;
-vlib_node_function_t __clib_weak dpdk_input_avx2;
-
-#if __x86_64__
-static void __clib_constructor
-dpdk_input_multiarch_select (void)
-{
- if (dpdk_input_avx512 && clib_cpu_supports_avx512f ())
- dpdk_input_node.function = dpdk_input_avx512;
- else if (dpdk_input_avx2 && clib_cpu_supports_avx2 ())
- dpdk_input_node.function = dpdk_input_avx2;
-}
-#endif
#endif
/*
diff --git a/src/plugins/memif.am b/src/plugins/memif.am
index 6e9aa779155..33a378f4a3f 100644
--- a/src/plugins/memif.am
+++ b/src/plugins/memif.am
@@ -36,7 +36,7 @@ nobase_apiinclude_HEADERS += \
API_FILES += memif/memif.api
if CPU_X86_64
-memif_multiversioning_files = \
+memif_multiversioning_sources = \
memif/node.c \
memif/device.c
@@ -44,10 +44,10 @@ if CC_SUPPORTS_AVX2
###############################################################
# AVX2
###############################################################
-libmemif_plugin_avx2_la_SOURCES = $(memif_multiversioning_files)
+libmemif_plugin_avx2_la_SOURCES = $(memif_multiversioning_sources)
libmemif_plugin_avx2_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX2_FLAGS@ \
- -DCLIB_MULTIARCH_VARIANT=avx2
+ -DCLIB_MARCH_VARIANT=avx2
noinst_LTLIBRARIES += libmemif_plugin_avx2.la
memif_plugin_la_LIBADD += libmemif_plugin_avx2.la
endif
@@ -56,10 +56,10 @@ if CC_SUPPORTS_AVX512
###############################################################
# AVX512
###############################################################
-libmemif_plugin_avx512_la_SOURCES = $(memif_multiversioning_files)
+libmemif_plugin_avx512_la_SOURCES = $(memif_multiversioning_sources)
libmemif_plugin_avx512_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX512_FLAGS@ \
- -DCLIB_MULTIARCH_VARIANT=avx512
+ -DCLIB_MARCH_VARIANT=avx512
noinst_LTLIBRARIES += libmemif_plugin_avx512.la
memif_plugin_la_LIBADD += libmemif_plugin_avx512.la
endif
diff --git a/src/plugins/memif/device.c b/src/plugins/memif/device.c
index c70a4ea3549..14b9d547234 100644
--- a/src/plugins/memif/device.c
+++ b/src/plugins/memif/device.c
@@ -46,7 +46,7 @@ static __clib_unused char *memif_tx_func_error_strings[] = {
#undef _
};
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
u8 *
format_memif_device_name (u8 * s, va_list * args)
{
@@ -500,7 +500,7 @@ memif_subif_add_del_function (vnet_main_t * vnm,
return 0;
}
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
/* *INDENT-OFF* */
VNET_DEVICE_CLASS (memif_device_class) = {
.name = "memif",
diff --git a/src/plugins/memif/node.c b/src/plugins/memif/node.c
index 029e25d814a..6e2807a2054 100644
--- a/src/plugins/memif/node.c
+++ b/src/plugins/memif/node.c
@@ -854,10 +854,10 @@ done:
return n_rx_packets;
}
-uword
-CLIB_MULTIARCH_FN (memif_input_fn) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame)
+
+VLIB_NODE_FN (memif_input_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
{
u32 n_rx = 0;
memif_main_t *mm = &memif_main;
@@ -910,10 +910,9 @@ CLIB_MULTIARCH_FN (memif_input_fn) (vlib_main_t * vm,
return n_rx;
}
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (memif_input_node) = {
- .function = memif_input_fn,
.name = "memif-input",
.sibling_of = "device-input",
.format_trace = format_memif_input_trace,
@@ -922,20 +921,6 @@ VLIB_REGISTER_NODE (memif_input_node) = {
.n_errors = MEMIF_INPUT_N_ERROR,
.error_strings = memif_input_error_strings,
};
-
-vlib_node_function_t __clib_weak memif_input_fn_avx512;
-vlib_node_function_t __clib_weak memif_input_fn_avx2;
-
-#if __x86_64__
-static void __clib_constructor
-memif_input_multiarch_select (void)
-{
- if (memif_input_fn_avx512 && clib_cpu_supports_avx512f ())
- memif_input_node.function = memif_input_fn_avx512;
- else if (memif_input_fn_avx2 && clib_cpu_supports_avx2 ())
- memif_input_node.function = memif_input_fn_avx2;
-}
-#endif
#endif
/* *INDENT-ON* */
diff --git a/src/vlib/node.c b/src/vlib/node.c
index 17dd2ea87d4..cc1732bb90d 100644
--- a/src/vlib/node.c
+++ b/src/vlib/node.c
@@ -324,6 +324,26 @@ register_node (vlib_main_t * vm, vlib_node_registration_t * r)
ASSERT (VLIB_NODE_TYPE_INTERNAL == zero.type);
}
+ if (r->node_fn_registrations)
+ {
+ vlib_node_fn_registration_t *fnr = r->node_fn_registrations;
+ int priority = -1;
+
+ /* to avoid confusion, please remove ".function " statiement from
+ CLIB_NODE_REGISTRATION() if using function function candidates */
+ ASSERT (r->function == 0);
+
+ while (fnr)
+ {
+ if (fnr->priority > priority)
+ {
+ priority = fnr->priority;
+ r->function = fnr->function;
+ }
+ fnr = fnr->next_registration;
+ }
+ }
+
ASSERT (r->function != 0);
n = clib_mem_alloc_no_fail (sizeof (n[0]));
diff --git a/src/vlib/node.h b/src/vlib/node.h
index 7177bebeffb..67eaea3f0e8 100644
--- a/src/vlib/node.h
+++ b/src/vlib/node.h
@@ -75,11 +75,21 @@ typedef enum
VLIB_N_NODE_TYPE,
} vlib_node_type_t;
+typedef struct _vlib_node_fn_registration
+{
+ vlib_node_function_t *function;
+ int priority;
+ struct _vlib_node_fn_registration *next_registration;
+} vlib_node_fn_registration_t;
+
typedef struct _vlib_node_registration
{
/* Vector processing function for this node. */
vlib_node_function_t *function;
+ /* Node function candidate registration with priority */
+ vlib_node_fn_registration_t *node_fn_registrations;
+
/* Node name. */
char *name;
@@ -160,6 +170,24 @@ static void __vlib_rm_node_registration_##x (void) \
} \
__VA_ARGS__ vlib_node_registration_t x
+#define VLIB_NODE_FN(node) \
+uword CLIB_MARCH_SFX (node##_fn)(); \
+static vlib_node_fn_registration_t \
+ CLIB_MARCH_SFX(node##_fn_registration) = \
+ { .function = &CLIB_MARCH_SFX (node##_fn), }; \
+ \
+static void __clib_constructor \
+CLIB_MARCH_SFX (node##_multiarch_register) (void) \
+{ \
+ extern vlib_node_registration_t node; \
+ vlib_node_fn_registration_t *r; \
+ r = & CLIB_MARCH_SFX (node##_fn_registration); \
+ r->priority = CLIB_MARCH_FN_PRIORITY(); \
+ r->next_registration = node.node_fn_registrations; \
+ node.node_fn_registrations = r; \
+} \
+uword CLIB_CPU_OPTIMIZED CLIB_MARCH_SFX (node##_fn)
+
#if CLIB_DEBUG > 0
#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn)
#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn)
diff --git a/src/vnet.am b/src/vnet.am
index 47159fc4591..86f3ad00f33 100644
--- a/src/vnet.am
+++ b/src/vnet.am
@@ -21,7 +21,7 @@ libvnet_la_DEPENDENCIES = \
libsvmdb.la \
libsvm.la \
libvlibmemory.la
-libvnet_multiversioning_files =
+libvnet_multiversioning_sources =
libvnet_la_LIBADD = $(libvnet_la_DEPENDENCIES) -lm -lpthread -ldl -lrt
@@ -128,7 +128,7 @@ libvnet_la_SOURCES += \
vnet/ethernet/p2p_ethernet_input.c \
vnet/ethernet/p2p_ethernet_api.c
-libvnet_multiversioning_files += \
+libvnet_multiversioning_sources += \
vnet/l2/l2_output.c
nobase_include_HEADERS += \
@@ -294,7 +294,7 @@ nobase_include_HEADERS += \
vnet/bonding/node.h \
vnet/bonding/bond.api.h
-libvnet_multiversioning_files = \
+libvnet_multiversioning_sources += \
vnet/bonding/node.c
API_FILES += vnet/bonding/bond.api
@@ -416,7 +416,7 @@ API_FILES += \
vnet/ip/rd_cp.api \
vnet/ip/punt.api
-libvnet_multiversioning_files = \
+libvnet_multiversioning_sources += \
vnet/ip/ip4_input.c
########################################
@@ -1286,10 +1286,10 @@ if CC_SUPPORTS_AVX2
###############################################################
# AVX2
###############################################################
-libvnet_avx2_la_SOURCES = $(libvnet_multiversioning_files)
+libvnet_avx2_la_SOURCES = $(libvnet_multiversioning_sources)
libvnet_avx2_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX2_FLAGS@ \
- -DCLIB_MULTIARCH_VARIANT=avx2
+ -DCLIB_MARCH_VARIANT=avx2
noinst_LTLIBRARIES += libvnet_avx2.la
libvnet_la_LIBADD += libvnet_avx2.la
endif
@@ -1298,10 +1298,10 @@ if CC_SUPPORTS_AVX512
###############################################################
# AVX512
###############################################################
-libvnet_avx512_la_SOURCES = $(libvnet_multiversioning_files)
+libvnet_avx512_la_SOURCES = $(libvnet_multiversioning_sources)
libvnet_avx512_la_CFLAGS = \
$(AM_CFLAGS) @CPU_AVX512_FLAGS@ \
- -DCLIB_MULTIARCH_VARIANT=avx512
+ -DCLIB_MARCH_VARIANT=avx512
noinst_LTLIBRARIES += libvnet_avx512.la
libvnet_la_LIBADD += libvnet_avx512.la
endif
diff --git a/src/vnet/bonding/node.c b/src/vnet/bonding/node.c
index df2da5c66fc..7d2acf7d0f7 100644
--- a/src/vnet/bonding/node.c
+++ b/src/vnet/bonding/node.c
@@ -36,7 +36,7 @@ typedef enum
BOND_INPUT_N_ERROR,
} bond_input_error_t;
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
static char *bond_input_error_strings[] = {
#define _(n,s) s,
foreach_bond_input_error
@@ -172,10 +172,9 @@ bond_update_next (vlib_main_t * vm, vlib_node_runtime_t * node,
vnet_feature_next ( /* not used */ 0, next_index, b);
}
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (bond_input_fn) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame)
+VLIB_NODE_FN (bond_input_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
{
u16 thread_index = vlib_get_thread_index ();
u32 *from, n_left;
@@ -367,7 +366,7 @@ CLIB_MULTIARCH_FN (bond_input_fn) (vlib_main_t * vm,
return frame->n_vectors;
}
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
static clib_error_t *
bond_input_init (vlib_main_t * vm)
{
@@ -376,7 +375,6 @@ bond_input_init (vlib_main_t * vm)
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (bond_input_node) = {
- .function = bond_input_fn,
.name = "bond-input",
.vector_size = sizeof (u32),
.format_buffer = format_ethernet_header_with_length,
@@ -391,20 +389,6 @@ VLIB_REGISTER_NODE (bond_input_node) = {
}
};
-#if __x86_64__
-vlib_node_function_t __clib_weak bond_input_fn_avx512;
-vlib_node_function_t __clib_weak bond_input_fn_avx2;
-static void __clib_constructor
-bond_input_multiarch_select (void)
-{
- if (bond_input_fn_avx512 && clib_cpu_supports_avx512f ())
- bond_input_node.function = bond_input_fn_avx512;
- else if (bond_input_fn_avx2 && clib_cpu_supports_avx2 ())
- bond_input_node.function = bond_input_fn_avx2;
-}
-#endif
-
-
VLIB_INIT_FUNCTION (bond_input_init);
VNET_FEATURE_INIT (bond_input, static) =
diff --git a/src/vnet/ip/ip4_input.c b/src/vnet/ip/ip4_input.c
index 7af2bee187a..b476f95ab8b 100644
--- a/src/vnet/ip/ip4_input.c
+++ b/src/vnet/ip/ip4_input.c
@@ -47,7 +47,7 @@ typedef struct
u8 packet_data[64];
} ip4_input_trace_t;
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
static u8 *
format_ip4_input_trace (u8 * s, va_list * va)
{
@@ -292,22 +292,20 @@ ip4_input_inline (vlib_main_t * vm,
<code> vnet_get_config_data (... &next0 ...); </code>
or @c error-drop
*/
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (ip4_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
+VLIB_NODE_FN (ip4_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
return ip4_input_inline (vm, node, frame, /* verify_checksum */ 1);
}
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (ip4_input_no_checksum) (vlib_main_t * vm,
+VLIB_NODE_FN (ip4_input_no_checksum_node) (vlib_main_t * vm,
vlib_node_runtime_t * node,
vlib_frame_t * frame)
{
return ip4_input_inline (vm, node, frame, /* verify_checksum */ 0);
}
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
char *ip4_error_strings[] = {
#define _(sym,string) string,
foreach_ip4_error
@@ -316,7 +314,6 @@ char *ip4_error_strings[] = {
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (ip4_input_node) = {
- .function = ip4_input,
.name = "ip4-input",
.vector_size = sizeof (u32),
@@ -337,8 +334,7 @@ VLIB_REGISTER_NODE (ip4_input_node) = {
.format_trace = format_ip4_input_trace,
};
-VLIB_REGISTER_NODE (ip4_input_no_checksum_node,static) = {
- .function = ip4_input_no_checksum,
+VLIB_REGISTER_NODE (ip4_input_no_checksum_node) = {
.name = "ip4-input-no-checksum",
.vector_size = sizeof (u32),
@@ -357,26 +353,6 @@ VLIB_REGISTER_NODE (ip4_input_no_checksum_node,static) = {
};
/* *INDENT-ON* */
-#if __x86_64__
-vlib_node_function_t __clib_weak ip4_input_avx512;
-vlib_node_function_t __clib_weak ip4_input_avx2;
-vlib_node_function_t __clib_weak ip4_input_no_checksum_avx512;
-vlib_node_function_t __clib_weak ip4_input_no_checksum_avx2;
-static void __clib_constructor
-ip4_input_multiarch_select (void)
-{
- if (ip4_input_no_checksum_avx512 && clib_cpu_supports_avx512f ())
- ip4_input_no_checksum_node.function = ip4_input_no_checksum_avx512;
- else if (ip4_input_no_checksum_avx2 && clib_cpu_supports_avx2 ())
- ip4_input_no_checksum_node.function = ip4_input_no_checksum_avx2;
-
- if (ip4_input_avx512 && clib_cpu_supports_avx512f ())
- ip4_input_node.function = ip4_input_avx512;
- else if (ip4_input_avx2 && clib_cpu_supports_avx2 ())
- ip4_input_node.function = ip4_input_avx2;
-}
-#endif
-
static clib_error_t *
ip4_init (vlib_main_t * vm)
{
diff --git a/src/vnet/l2/l2_output.c b/src/vnet/l2/l2_output.c
index 306049bd0db..1fa042ffd54 100644
--- a/src/vnet/l2/l2_output.c
+++ b/src/vnet/l2/l2_output.c
@@ -27,7 +27,7 @@
#include <vnet/l2/l2_output.h>
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
/* Feature graph node names */
static char *l2output_feat_names[] = {
#define _(sym,name) name,
@@ -76,7 +76,7 @@ typedef struct
u8 raw[12]; /* raw data */
} l2output_trace_t;
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
/* packet trace format function */
static u8 *
format_l2output_trace (u8 * s, va_list * args)
@@ -275,10 +275,9 @@ l2output_process_batch (vlib_main_t * vm, vlib_node_runtime_t * node,
l2_efp, l2_vtr, l2_pbb, 1, 1);
}
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (l2output_node_fn) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * frame)
+VLIB_NODE_FN (l2output_node) (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
{
u32 n_left, *from;
l2output_main_t *msm = &l2output_main;
@@ -431,10 +430,9 @@ CLIB_MULTIARCH_FN (l2output_node_fn) (vlib_main_t * vm,
return frame->n_vectors;
}
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (l2output_node) = {
- .function = l2output_node_fn,
.name = "l2-output",
.vector_size = sizeof (u32),
.format_trace = format_l2output_trace,
@@ -451,20 +449,6 @@ VLIB_REGISTER_NODE (l2output_node) = {
[L2OUTPUT_NEXT_BAD_INTF] = "l2-output-bad-intf",
},
};
-
-#if __x86_64__
-vlib_node_function_t __clib_weak l2output_node_fn_avx512;
-vlib_node_function_t __clib_weak l2output_node_fn_avx2;
-static void __clib_constructor
-l2output_multiarch_select (void)
-{
- if (l2output_node_fn_avx512 && clib_cpu_supports_avx512f ())
- l2output_node.function = l2output_node_fn_avx512;
- else if (l2output_node_fn_avx2 && clib_cpu_supports_avx2 ())
- l2output_node.function = l2output_node_fn_avx2;
-}
-#endif
-
/* *INDENT-ON* */
diff --git a/src/vppinfra/cpu.h b/src/vppinfra/cpu.h
index f670f3e964b..110815c9f87 100644
--- a/src/vppinfra/cpu.h
+++ b/src/vppinfra/cpu.h
@@ -51,14 +51,16 @@
return & fn; \
}
-#ifdef CLIB_MULTIARCH_VARIANT
+#ifdef CLIB_MARCH_VARIANT
#define __CLIB_MULTIARCH_FN(a,b) a##_##b
#define _CLIB_MULTIARCH_FN(a,b) __CLIB_MULTIARCH_FN(a,b)
-#define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MULTIARCH_VARIANT)
+#define CLIB_MULTIARCH_FN(fn) _CLIB_MULTIARCH_FN(fn,CLIB_MARCH_VARIANT)
#else
#define CLIB_MULTIARCH_FN(fn) fn
#endif
+#define CLIB_MARCH_SFX CLIB_MULTIARCH_FN
+
#define foreach_x86_64_flags \
_ (sse3, 1, ecx, 0) \
_ (ssse3, 1, ecx, 9) \
@@ -165,6 +167,27 @@ clib_cpu_supports_aes ()
#endif
}
+static inline int
+clib_cpu_march_priority_avx512 ()
+{
+ if (clib_cpu_supports_avx512f ())
+ return 20;
+ return -1;
+}
+
+static inline int
+clib_cpu_march_priority_avx2 ()
+{
+ if (clib_cpu_supports_avx2 ())
+ return 10;
+ return -1;
+}
+
+#ifdef CLIB_MARCH_VARIANT
+#define CLIB_MARCH_FN_PRIORITY() CLIB_MARCH_SFX(clib_cpu_march_priority)()
+#else
+#define CLIB_MARCH_FN_PRIORITY() 0
+#endif
#endif /* included_clib_cpu_h */
format_function_t format_cpu_uarch;