diff options
| author |   Damjan Marion <damarion@cisco.com> | 2016-05-11 23:07:18 +0200 |
|---|---|---|
| committer | Damjan Marion <damarion@cisco.com> | 2016-05-19 18:14:38 +0200 |
| commit | 1c80e831b728ab378949714d5059a0b5b1822a0a (patch) | |
| tree | e4f3ecfee143f7dee0e9905570d41ca3ee345c83 | |
| parent | 82e29c455833b5b12e04c89d2dec1106b499e6b0 (diff) | |
Add support for multiple microarchitectures in single binary
* compiler -march= parameter is changed from native to corei7
so code is always genereted with instructions which are available
on the Nehalem microarchitecture (up to SSE4.2)
* compiler -mtune= parameter is added so code is optimized for
corei7-avx which equals to Sandy Bridge microarchitecture
* set of macros is added which allows run-time detection of available
cpu instructions (e.g. clib_cpu_supports_avx())
* set of macros is added which allows us to clone graph node funcitons
where cloned function is optmized for different microarchitecture
Those macros are using following attributes:
__attribute__((flatten))
__attribute__((target("arch=core-avx2)))
I.e. If applied to foo_node_fn() macro will generate cloned
functions foo_node_fn_avx2() and foo_node_fn_avx512() (future)
It will also generate function void * foo_node_fn_multiarch_select()
which detects available instruction set and returns pointer to the
best matching function clone.
Change-Id: I2dce0ac92a5ede95fcb56f47f3d1f3c4c040bac0
Signed-off-by: Damjan Marion <damarion@cisco.com>
72 files changed, 439 insertions, 34 deletions
diff --git a/build-data/packages/dpdk.mk b/build-data/packages/dpdk.mk index 2bfc4b49..0a4c1610 100644 --- a/build-data/packages/dpdk.mk +++ b/build-data/packages/dpdk.mk @@ -4,6 +4,11 @@ ifeq ($(DPDK_MARCH),) DPDK_MARCH="native" endif +DPDK_TUNE = $(strip $($(PLATFORM)_mtune)) +ifeq ($(DPDK_TUNE),) + DPDK_MARCH="generic" +endif + ifneq (,$(findstring debug,$(TAG))) DPDK_DEBUG=y else @@ -14,6 +19,7 @@ DPDK_MAKE_ARGS = -C $(call find_source_fn,$(PACKAGE_SOURCE)) \ DPDK_BUILD_DIR=$(PACKAGE_BUILD_DIR) \ DPDK_INSTALL_DIR=$(PACKAGE_INSTALL_DIR) \ DPDK_MARCH=$(DPDK_MARCH) \ + DPDK_TUNE=$(DPDK_TUNE) \ DPDK_DEBUG=$(DPDK_DEBUG) diff --git a/build-data/platforms/vpp.mk b/build-data/platforms/vpp.mk index c381be19..ec0d874d 100644 --- a/build-data/platforms/vpp.mk +++ b/build-data/platforms/vpp.mk @@ -13,6 +13,9 @@ # vector packet processor vpp_arch = native +vpp_march = corei7 # Nehalem Instruction set +vpp_mtune = corei7-avx # Optimize for Sandy Bridge +vpp_dpdk_arch = corei7 vpp_native_tools = vppapigen vpp_uses_dpdk = yes @@ -40,9 +43,9 @@ vpp_debug_TAG_CFLAGS = -g -O0 -DCLIB_DEBUG -DFORTIFY_SOURCE=2 -march=$(MARCH) \ vpp_debug_TAG_LDFLAGS = -g -O0 -DCLIB_DEBUG -DFORTIFY_SOURCE=2 -march=$(MARCH) \ -fstack-protector-all -fPIC -Werror -vpp_TAG_CFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) \ +vpp_TAG_CFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) -mtune=$(MTUNE) \ -fstack-protector -fPIC -Werror -vpp_TAG_LDFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) \ +vpp_TAG_LDFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) -mtune=$(MTUNE) \ -fstack-protector -fPIC -Werror vpp_gcov_TAG_CFLAGS = -g -O0 -DCLIB_DEBUG -march=$(MARCH) \ diff --git a/build-data/platforms/vpp_lite.mk b/build-data/platforms/vpp_lite.mk index 4183700e..d35d2347 100644 --- a/build-data/platforms/vpp_lite.mk +++ b/build-data/platforms/vpp_lite.mk @@ -13,6 +13,8 @@ # vector packet processor vpp_lite_arch = native +vpp_lite_march = corei7 # Nehalem Instruction set +vpp_lite_mtune = corei7-avx # Optimize for Sandy Bridge vpp_lite_native_tools = vppapigen vpp_lite_uses_dpdk = no @@ -30,7 +32,7 @@ vpp_lite_debug_TAG_CFLAGS = -g -O0 -DCLIB_DEBUG -DFORTIFY_SOURCE=2 -march=$(MARC vpp_lite_debug_TAG_LDFLAGS = -g -O0 -DCLIB_DEBUG -DFORTIFY_SOURCE=2 -march=$(MARCH) \ -fstack-protector-all -fPIC -Werror -vpp_lite_TAG_CFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) \ +vpp_lite_TAG_CFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) -mtune=$(MTUNE) \ -fstack-protector -fPIC -Werror -vpp_lite_TAG_LDFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) \ +vpp_lite_TAG_LDFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) -mtune=$(MTUNE) \ -fstack-protector -fPIC -Werror diff --git a/build-root/Makefile b/build-root/Makefile index a60cbf0a..97fb43d4 100644 --- a/build-root/Makefile +++ b/build-root/Makefile @@ -220,6 +220,11 @@ else endif export MARCH +MTUNE = $(strip $($(PLATFORM)_mtune)) +ifeq ($(MTUNE),) + MTUNE = generic +endif + ###################################################################### # Generic build stuff ###################################################################### diff --git a/dpdk/Makefile b/dpdk/Makefile index 165058d9..b7e1e097 100644 --- a/dpdk/Makefile +++ b/dpdk/Makefile @@ -19,6 +19,7 @@ DPDK_INSTALL_DIR ?= $(CURDIR)/_install DPDK_PKTMBUF_HEADROOM ?= 128 DPDK_DOWNLOAD_DIR ?= $(HOME)/Downloads DPDK_MARCH ?= native +DPDK_TUNE ?= generic DPDK_DEBUG ?= n B := $(DPDK_BUILD_DIR) @@ -51,7 +52,7 @@ DPDK_CPU_LDFLAGS := DPDK_EXTRA_LDFLAGS := -g ifeq ($(DPDK_DEBUG),n) -DPDK_EXTRA_CFLAGS := -g +DPDK_EXTRA_CFLAGS := -g -mtune=$(DPDK_TUNE) else DPDK_EXTRA_CFLAGS := -g -O0 endif diff --git a/vlib/vlib/node.h b/vlib/vlib/node.h index 2caede6e..9b33a0a3 100644 --- a/vlib/vlib/node.h +++ b/vlib/vlib/node.h @@ -40,6 +40,7 @@ #ifndef included_vlib_node_h #define included_vlib_node_h +#include <vppinfra/cpu.h> #include <vppinfra/longjmp.h> #include <vppinfra/timing_wheel.h> #include <vlib/trace.h> /* for vlib_trace_filter_t */ @@ -149,6 +150,32 @@ static void __vlib_add_node_registration_##x (void) \ } \ __VA_ARGS__ vlib_node_registration_t x +#if CLIB_DEBUG > 0 +#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn) +#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) +#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn) +#else +#define VLIB_NODE_FUNCTION_CLONE_TEMPLATE(arch, fn, tgt) \ + uword \ + __attribute__ ((flatten)) \ + __attribute__ ((target (tgt))) \ + CLIB_CPU_OPTIMIZED \ + fn ## _ ## arch ( struct vlib_main_t * vm, \ + struct vlib_node_runtime_t * node, \ + struct vlib_frame_t * frame) \ + { return fn (vm, node, frame); } + +#define VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) \ + foreach_march_variant(VLIB_NODE_FUNCTION_CLONE_TEMPLATE, fn) + +#define VLIB_NODE_FUNCTION_MULTIARCH(node, fn) \ + VLIB_NODE_FUNCTION_MULTIARCH_CLONE(fn) \ + CLIB_MULTIARCH_SELECT_FN(fn, static inline) \ + static void __attribute__((__constructor__)) \ + __vlib_node_function_multiarch_select_##node (void) \ + { node.function = fn ## _multiarch_select(); } +#endif + always_inline vlib_node_registration_t * vlib_node_next_registered (vlib_node_registration_t * c) { diff --git a/vnet/vnet/classify/ip_classify.c b/vnet/vnet/classify/ip_classify.c index 8152f6ce..75e80ad6 100644 --- a/vnet/vnet/classify/ip_classify.c +++ b/vnet/vnet/classify/ip_classify.c @@ -329,6 +329,8 @@ VLIB_REGISTER_NODE (ip4_classify_node) = { .next_nodes = IP4_LOOKUP_NEXT_NODES, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_classify_node, ip4_classify) + static uword ip6_classify (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -350,6 +352,8 @@ VLIB_REGISTER_NODE (ip6_classify_node) = { .next_nodes = IP6_LOOKUP_NEXT_NODES, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_classify_node, ip6_classify) + static clib_error_t * ip_classify_init (vlib_main_t * vm) { diff --git a/vnet/vnet/cop/ip4_whitelist.c b/vnet/vnet/cop/ip4_whitelist.c index 6178e891..5578558c 100644 --- a/vnet/vnet/cop/ip4_whitelist.c +++ b/vnet/vnet/cop/ip4_whitelist.c @@ -348,6 +348,8 @@ VLIB_REGISTER_NODE (ip4_cop_whitelist_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_cop_whitelist_node, ip4_cop_whitelist_node_fn) + static clib_error_t * ip4_whitelist_init (vlib_main_t * vm) { diff --git a/vnet/vnet/cop/ip6_whitelist.c b/vnet/vnet/cop/ip6_whitelist.c index dc6a1ee5..4a8f33fb 100644 --- a/vnet/vnet/cop/ip6_whitelist.c +++ b/vnet/vnet/cop/ip6_whitelist.c @@ -286,6 +286,8 @@ VLIB_REGISTER_NODE (ip6_cop_whitelist_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_cop_whitelist_node, ip6_cop_whitelist_node_fn) + static clib_error_t * ip6_whitelist_init (vlib_main_t * vm) { diff --git a/vnet/vnet/cop/node1.c b/vnet/vnet/cop/node1.c index 3ee7006a..b448b531 100644 --- a/vnet/vnet/cop/node1.c +++ b/vnet/vnet/cop/node1.c @@ -282,6 +282,8 @@ VLIB_REGISTER_NODE (cop_input_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (cop_input_node, cop_input_node_fn) + #define foreach_cop_stub \ _(default-cop-whitelist, default_cop_whitelist) diff --git a/vnet/vnet/devices/af_packet/device.c b/vnet/vnet/devices/af_packet/device.c index f0e91487..0671d9e2 100644 --- a/vnet/vnet/devices/af_packet/device.c +++ b/vnet/vnet/devices/af_packet/device.c @@ -204,3 +204,6 @@ VNET_DEVICE_CLASS (af_packet_device_class) = { .subif_add_del_function = af_packet_subif_add_del_function, .no_flatten_output_chains = 1, }; + +VLIB_DEVICE_TX_FUNCTION_MULTIARCH (af_packet_device_class, + af_packet_interface_tx) diff --git a/vnet/vnet/devices/af_packet/node.c b/vnet/vnet/devices/af_packet/node.c index b622a221..0c608ea4 100644 --- a/vnet/vnet/devices/af_packet/node.c +++ b/vnet/vnet/devices/af_packet/node.c @@ -267,7 +267,6 @@ af_packet_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, return n_rx_packets; } - VLIB_REGISTER_NODE (af_packet_input_node) = { .function = af_packet_input_fn, .name = "af-packet-input", @@ -283,3 +282,6 @@ VLIB_REGISTER_NODE (af_packet_input_node) = { [AF_PACKET_INPUT_NEXT_ETHERNET_INPUT] = "ethernet-input", }, }; + +VLIB_NODE_FUNCTION_MULTIARCH (af_packet_input_node, af_packet_input_fn) + diff --git a/vnet/vnet/devices/dpdk/cli.c b/vnet/vnet/devices/dpdk/cli.c index 9ec19867..9063cad8 100644 --- a/vnet/vnet/devices/dpdk/cli.c +++ b/vnet/vnet/devices/dpdk/cli.c @@ -759,6 +759,7 @@ set_efd (vlib_main_t *vm, unformat_input_t *input, dpdk_main_t * dm = &dpdk_main; vlib_thread_main_t * tm = vlib_get_thread_main(); clib_error_t * error = NULL; + vlib_node_runtime_t * rt = vlib_node_get_runtime (vm, dpdk_input_node.index); if (unformat(input, "enable")) { if (unformat(input, "dpdk")) { @@ -844,6 +845,13 @@ set_efd (vlib_main_t *vm, unformat_input_t *input, format_unformat_error, input); } + if (dm->efd.enabled) + rt->function = dpdk_input_efd_multiarch_select(); + else if (dm->use_rss) + rt->function = dpdk_input_rss_multiarch_select(); + else + rt->function = dpdk_input_multiarch_select(); + return error; } diff --git a/vnet/vnet/devices/dpdk/device.c b/vnet/vnet/devices/dpdk/device.c index ab85dfb4..a38c8d19 100644 --- a/vnet/vnet/devices/dpdk/device.c +++ b/vnet/vnet/devices/dpdk/device.c @@ -1184,6 +1184,9 @@ VNET_DEVICE_CLASS (dpdk_device_class) = { .name_renumber = dpdk_device_renumber, }; +VLIB_DEVICE_TX_FUNCTION_MULTIARCH (dpdk_device_class, + dpdk_interface_tx) + void dpdk_set_flowcontrol_callback (vlib_main_t *vm, dpdk_flowcontrol_callback_t callback) { diff --git a/vnet/vnet/devices/dpdk/dpdk.h b/vnet/vnet/devices/dpdk/dpdk.h index 019d83f7..525cd8d1 100644 --- a/vnet/vnet/devices/dpdk/dpdk.h +++ b/vnet/vnet/devices/dpdk/dpdk.h @@ -584,8 +584,9 @@ dpdk_pmd_t dpdk_get_pmd_type (vnet_hw_interface_t *hi); i8 dpdk_get_cpu_socket (vnet_hw_interface_t *hi); -uword -dpdk_input_rss (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f); +void * dpdk_input_multiarch_select(); +void * dpdk_input_rss_multiarch_select(); +void * dpdk_input_efd_multiarch_select(); clib_error_t* dpdk_get_hw_interface_stats (u32 hw_if_index, struct rte_eth_stats* dest); diff --git a/vnet/vnet/devices/dpdk/init.c b/vnet/vnet/devices/dpdk/init.c index 63fa4c07..8ee59ff8 100644 --- a/vnet/vnet/devices/dpdk/init.c +++ b/vnet/vnet/devices/dpdk/init.c @@ -804,6 +804,7 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) clib_error_t * error = 0; dpdk_main_t * dm = &dpdk_main; vlib_thread_main_t * tm = vlib_get_thread_main(); + vlib_node_runtime_t * rt = vlib_node_get_runtime (vm, dpdk_input_node.index); u8 * s, * tmp = 0; u8 * pci_dev_id = 0; u8 * rte_cmd = 0, * ethname = 0; @@ -1251,10 +1252,9 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) } if (dm->use_rss) - { - vlib_node_runtime_t * rt = vlib_node_get_runtime (vm, dpdk_input_node.index); - rt->function = dpdk_input_rss; - } + rt->function = dpdk_input_rss_multiarch_select(); + else + rt->function = dpdk_input_multiarch_select(); done: return error; } diff --git a/vnet/vnet/devices/dpdk/node.c b/vnet/vnet/devices/dpdk/node.c index a7590a5c..ca94511b 100644 --- a/vnet/vnet/devices/dpdk/node.c +++ b/vnet/vnet/devices/dpdk/node.c @@ -251,6 +251,8 @@ VLIB_REGISTER_NODE (handoff_dispatch_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (handoff_dispatch_node, handoff_dispatch_node_fn) + clib_error_t *handoff_dispatch_init (vlib_main_t *vm) { handoff_dispatch_main_t * mp = &handoff_dispatch_main; @@ -488,7 +490,8 @@ static inline u32 dpdk_device_input ( dpdk_main_t * dm, dpdk_device_t * xd, vlib_node_runtime_t * node, u32 cpu_index, - u16 queue_id) + u16 queue_id, + int use_efd) { u32 n_buffers; u32 next_index = DPDK_RX_NEXT_ETHERNET_INPUT; @@ -510,7 +513,7 @@ static inline u32 dpdk_device_input ( dpdk_main_t * dm, if (n_buffers == 0) { /* check if EFD (dpdk) is enabled */ - if (PREDICT_FALSE(dm->efd.enabled)) + if (PREDICT_FALSE(use_efd && dm->efd.enabled)) { /* reset a few stats */ xd->efd_agent.last_poll_time = 0; @@ -546,7 +549,7 @@ static inline u32 dpdk_device_input ( dpdk_main_t * dm, /* Check for congestion if EFD (Early-Fast-Discard) is enabled * in any mode (e.g. dpdk, monitor, or drop_all) */ - if (PREDICT_FALSE(dm->efd.enabled)) + if (PREDICT_FALSE(use_efd && dm->efd.enabled)) { /* update EFD counters */ dpdk_efd_update_counters(xd, n_buffers, dm->efd.enabled); @@ -793,7 +796,7 @@ dpdk_input (vlib_main_t * vm, { xd = vec_elt_at_index(dm->devices, dq->device); ASSERT(dq->queue_id == 0); - n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, 0); + n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, 0, 0); } VIRL_SPEED_LIMIT() @@ -818,7 +821,7 @@ dpdk_input_rss (vlib_main_t * vm, vec_foreach (dq, dm->devices_by_cpu[cpu_index]) { xd = vec_elt_at_index(dm->devices, dq->device); - n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id); + n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id, 0); } VIRL_SPEED_LIMIT() @@ -826,6 +829,32 @@ dpdk_input_rss (vlib_main_t * vm, return n_rx_packets; } +uword +dpdk_input_efd (vlib_main_t * vm, + vlib_node_runtime_t * node, + vlib_frame_t * f) +{ + dpdk_main_t * dm = &dpdk_main; + dpdk_device_t * xd; + uword n_rx_packets = 0; + dpdk_device_and_queue_t * dq; + u32 cpu_index = os_get_cpu_number(); + + /* + * Poll all devices on this cpu for input/interrupts. + */ + vec_foreach (dq, dm->devices_by_cpu[cpu_index]) + { + xd = vec_elt_at_index(dm->devices, dq->device); + n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id, 1); + } + + VIRL_SPEED_LIMIT() + + return n_rx_packets; +} + + VLIB_REGISTER_NODE (dpdk_input_node) = { .function = dpdk_input, .type = VLIB_NODE_TYPE_INPUT, @@ -850,6 +879,17 @@ VLIB_REGISTER_NODE (dpdk_input_node) = { }, }; + +/* handle dpdk_input_rss alternative function */ +VLIB_NODE_FUNCTION_MULTIARCH_CLONE(dpdk_input) +VLIB_NODE_FUNCTION_MULTIARCH_CLONE(dpdk_input_rss) +VLIB_NODE_FUNCTION_MULTIARCH_CLONE(dpdk_input_efd) + +/* this macro defines dpdk_input_rss_multiarch_select() */ +CLIB_MULTIARCH_SELECT_FN(dpdk_input); +CLIB_MULTIARCH_SELECT_FN(dpdk_input_rss); +CLIB_MULTIARCH_SELECT_FN(dpdk_input_efd); + /* * Override the next nodes for the dpdk input nodes. * Must be invoked prior to VLIB_INIT_FUNCTION calls. diff --git a/vnet/vnet/devices/netmap/device.c b/vnet/vnet/devices/netmap/device.c index f04e0672..a966ffef 100644 --- a/vnet/vnet/devices/netmap/device.c +++ b/vnet/vnet/devices/netmap/device.c @@ -232,3 +232,6 @@ VNET_DEVICE_CLASS (netmap_device_class) = { .subif_add_del_function = netmap_subif_add_del_function, .no_flatten_output_chains = 1, }; + +VLIB_DEVICE_TX_FUNCTION_MULTIARCH(netmap_device_class, + netmap_interface_tx) diff --git a/vnet/vnet/devices/netmap/node.c b/vnet/vnet/devices/netmap/node.c index 3986c7e4..f4c39e62 100644 --- a/vnet/vnet/devices/netmap/node.c +++ b/vnet/vnet/devices/netmap/node.c @@ -270,7 +270,6 @@ netmap_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, return n_rx_packets; } - VLIB_REGISTER_NODE (netmap_input_node) = { .function = netmap_input_fn, .name = "netmap-input", @@ -287,3 +286,5 @@ VLIB_REGISTER_NODE (netmap_input_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (netmap_input_node, netmap_input_fn) + diff --git a/vnet/vnet/devices/ssvm/node.c b/vnet/vnet/devices/ssvm/node.c index 84625696..57b5fd22 100644 --- a/vnet/vnet/devices/ssvm/node.c +++ b/vnet/vnet/devices/ssvm/node.c @@ -334,3 +334,5 @@ VLIB_REGISTER_NODE (ssvm_eth_input_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ssvm_eth_input_node, ssvm_eth_input_node_fn) + diff --git a/vnet/vnet/devices/ssvm/ssvm_eth.c b/vnet/vnet/devices/ssvm/ssvm_eth.c index 49f2d5f8..7c65e212 100644 --- a/vnet/vnet/devices/ssvm/ssvm_eth.c +++ b/vnet/vnet/devices/ssvm/ssvm_eth.c @@ -474,3 +474,6 @@ VNET_DEVICE_CLASS (ssvm_eth_device_class) = { .rx_redirect_to_node = ssvm_eth_set_interface_next_node, .no_flatten_output_chains = 1, }; + +VLIB_DEVICE_TX_FUNCTION_MULTIARCH (ssvm_eth_device_class, + ssvm_eth_interface_tx) diff --git a/vnet/vnet/devices/virtio/vhost-user.c b/vnet/vnet/devices/virtio/vhost-user.c index 5902c42b..ef4993f9 100644 --- a/vnet/vnet/devices/virtio/vhost-user.c +++ b/vnet/vnet/devices/virtio/vhost-user.c @@ -1144,6 +1144,8 @@ VLIB_REGISTER_NODE (vhost_user_input_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (vhost_user_input_node, vhost_user_input) + static uword vhost_user_intfc_tx (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -1373,6 +1375,9 @@ VNET_DEVICE_CLASS (vhost_user_dev_class,static) = { .no_flatten_output_chains = 1, }; +VLIB_DEVICE_TX_FUNCTION_MULTIARCH (vhost_user_dev_class, + vhost_user_intfc_tx) + static uword vhost_user_process (vlib_main_t * vm, vlib_node_runtime_t * rt, diff --git a/vnet/vnet/ethernet/node.c b/vnet/vnet/ethernet/node.c index e8902fd7..226a66e9 100644 --- a/vnet/vnet/ethernet/node.c +++ b/vnet/vnet/ethernet/node.c @@ -945,6 +945,8 @@ VLIB_REGISTER_NODE (ethernet_input_node) = { .unformat_buffer = unformat_ethernet_header, }; +VLIB_NODE_FUNCTION_MULTIARCH (ethernet_input_node, ethernet_input) + VLIB_REGISTER_NODE (ethernet_input_type_node,static) = { .function = ethernet_input_type, .name = "ethernet-input-type", @@ -959,6 +961,8 @@ VLIB_REGISTER_NODE (ethernet_input_type_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ethernet_input_type_node, ethernet_input_type) + VLIB_REGISTER_NODE (ethernet_input_not_l2_node,static) = { .function = ethernet_input_not_l2, .name = "ethernet-input-not-l2", @@ -973,6 +977,8 @@ VLIB_REGISTER_NODE (ethernet_input_not_l2_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ethernet_input_not_l2_node, ethernet_input_not_l2) + void ethernet_set_rx_redirect (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 enable) diff --git a/vnet/vnet/gre/gre.c b/vnet/vnet/gre/gre.c index 075bd6fd..6d375159 100644 --- a/vnet/vnet/gre/gre.c +++ b/vnet/vnet/gre/gre.c @@ -454,6 +454,9 @@ VNET_DEVICE_CLASS (gre_device_class) = { #endif }; +VLIB_DEVICE_TX_FUNCTION_MULTIARCH (gre_device_class, + gre_interface_tx) + VNET_HW_INTERFACE_CLASS (gre_hw_interface_class) = { .name = "GRE", diff --git a/vnet/vnet/gre/node.c b/vnet/vnet/gre/node.c index a91f7e9e..e33b2b05 100644 --- a/vnet/vnet/gre/node.c +++ b/vnet/vnet/gre/node.c @@ -434,6 +434,8 @@ VLIB_REGISTER_NODE (gre_input_node) = { .unformat_buffer = unformat_gre_header, }; +VLIB_NODE_FUNCTION_MULTIARCH (gre_input_node, gre_input) + void gre_register_input_protocol (vlib_main_t * vm, gre_protocol_t protocol, diff --git a/vnet/vnet/interface.h b/vnet/vnet/interface.h index 2829a0cc..30dcf276 100644 --- a/vnet/vnet/interface.h +++ b/vnet/vnet/interface.h @@ -163,6 +163,32 @@ static void __vnet_add_device_class_registration_##x (void) \ } \ __VA_ARGS__ vnet_device_class_t x +#define VLIB_DEVICE_TX_FUNCTION_CLONE_TEMPLATE(arch, fn, tgt) \ + uword \ + __attribute__ ((flatten)) \ + __attribute__ ((target (tgt))) \ + CLIB_CPU_OPTIMIZED \ + fn ## _ ## arch ( vlib_main_t * vm, \ + vlib_node_runtime_t * node, \ + vlib_frame_t * frame) \ + { return fn (vm, node, frame); } + +#define VLIB_DEVICE_TX_FUNCTION_MULTIARCH_CLONE(fn) \ + foreach_march_variant(VLIB_DEVICE_TX_FUNCTION_CLONE_TEMPLATE, fn) + +#if CLIB_DEBUG > 0 +#define VLIB_MULTIARCH_CLONE_AND_SELECT_FN(fn,...) +#define VLIB_DEVICE_TX_FUNCTION_MULTIARCH(dev, fn) +#else +#define VLIB_DEVICE_TX_FUNCTION_MULTIARCH(dev, fn) \ + VLIB_DEVICE_TX_FUNCTION_MULTIARCH_CLONE(fn) \ + CLIB_MULTIARCH_SELECT_FN(fn, static inline) \ + static void __attribute__((__constructor__)) \ + __vlib_device_tx_function_multiarch_select_##dev (void) \ + { dev.tx_function = fn ## _multiarch_select(); } +#endif + + /* Layer-2 (e.g. Ethernet) interface class. */ typedef struct _vnet_hw_interface_class { /* Index into main vector. */ diff --git a/vnet/vnet/interface_output.c b/vnet/vnet/interface_output.c index 04c1b7f5..9f9fb707 100644 --- a/vnet/vnet/interface_output.c +++ b/vnet/vnet/interface_output.c @@ -1140,6 +1140,8 @@ VLIB_REGISTER_NODE (drop_buffers,static) = { .validate_frame = validate_error_frame, }; +VLIB_NODE_FUNCTION_MULTIARCH (drop_buffers, process_drop) + VLIB_REGISTER_NODE (punt_buffers,static) = { .function = process_punt, .flags = (VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH @@ -1150,12 +1152,16 @@ VLIB_REGISTER_NODE (punt_buffers,static) = { .validate_frame = validate_error_frame, }; +VLIB_NODE_FUNCTION_MULTIARCH (punt_buffers, process_punt) + VLIB_REGISTER_NODE (vnet_per_buffer_interface_output_node,static) = { .function = vnet_per_buffer_interface_output, .name = "interface-output", .vector_size = sizeof (u32), }; +VLIB_NODE_FUNCTION_MULTIARCH (vnet_per_buffer_interface_output_node, vnet_per_buffer_interface_output) + clib_error_t * vnet_per_buffer_interface_output_hw_interface_add_del (vnet_main_t * vnm, u32 hw_if_index, diff --git a/vnet/vnet/ip/ip4_forward.c b/vnet/vnet/ip/ip4_forward.c index a84b83bc..ae2f9eea 100644 --- a/vnet/vnet/ip/ip4_forward.c +++ b/vnet/vnet/ip/ip4_forward.c @@ -1354,6 +1354,8 @@ VLIB_REGISTER_NODE (ip4_lookup_node) = { .next_nodes = IP4_LOOKUP_NEXT_NODES, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_node, ip4_lookup) + static uword ip4_indirect (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -1373,6 +1375,8 @@ VLIB_REGISTER_NODE (ip4_indirect_node) = { .next_nodes = IP4_LOOKUP_NEXT_NODES, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_indirect_node, ip4_indirect) + /* Global IP4 main. */ ip4_main_t ip4_main; @@ -1612,6 +1616,8 @@ VLIB_REGISTER_NODE (ip4_drop_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_drop_node, ip4_drop) + VLIB_REGISTER_NODE (ip4_punt_node,static) = { .function = ip4_punt, .name = "ip4-punt", @@ -1625,6 +1631,8 @@ VLIB_REGISTER_NODE (ip4_punt_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_punt_node, ip4_punt) + VLIB_REGISTER_NODE (ip4_miss_node,static) = { .function = ip4_miss, .name = "ip4-miss", @@ -1638,6 +1646,8 @@ VLIB_REGISTER_NODE (ip4_miss_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_miss_node, ip4_miss) + /* Compute TCP/UDP/ICMP4 checksum in software. */ u16 ip4_tcp_udp_compute_checksum (vlib_main_t * vm, vlib_buffer_t * p0, @@ -2076,6 +2086,8 @@ VLIB_REGISTER_NODE (ip4_local_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_local_node, ip4_local) + void ip4_register_protocol (u32 protocol, u32 node_index) { vlib_main_t * vm = vlib_get_main(); @@ -2743,6 +2755,8 @@ VLIB_REGISTER_NODE (ip4_rewrite_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_node, ip4_rewrite_transit) + VLIB_REGISTER_NODE (ip4_rewrite_local_node,static) = { .function = ip4_rewrite_local, .name = "ip4-rewrite-local", @@ -2759,6 +2773,8 @@ VLIB_REGISTER_NODE (ip4_rewrite_local_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_rewrite_local_node, ip4_rewrite_local) + static clib_error_t * add_del_interface_table (vlib_main_t * vm, unformat_input_t * input, @@ -3032,6 +3048,8 @@ VLIB_REGISTER_NODE (ip4_lookup_multicast_node,static) = { .next_nodes = IP4_LOOKUP_NEXT_NODES, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_lookup_multicast_node, ip4_lookup_multicast) + VLIB_REGISTER_NODE (ip4_multicast_node,static) = { .function = ip4_drop, .name = "ip4-multicast", diff --git a/vnet/vnet/ip/ip4_hop_by_hop.c b/vnet/vnet/ip/ip4_hop_by_hop.c index ae46040a..177feb74 100644 --- a/vnet/vnet/ip/ip4_hop_by_hop.c +++ b/vnet/vnet/ip/ip4_hop_by_hop.c @@ -233,6 +233,8 @@ VLIB_REGISTER_NODE (ip4_hop_by_hop_node) = { .next_nodes = IP4_LOOKUP_NEXT_NODES, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_hop_by_hop_node, ip4_hop_by_hop_node_fn) + VLIB_REGISTER_NODE (ip4_add_hop_by_hop_node) = { .function = ip4_hop_by_hop_node_fn, .name = "ip4-add-hop-by-hop", diff --git a/vnet/vnet/ip/ip4_input.c b/vnet/vnet/ip/ip4_input.c index f31df0f6..60634250 100644 --- a/vnet/vnet/ip/ip4_input.c +++ b/vnet/vnet/ip/ip4_input.c @@ -371,6 +371,8 @@ VLIB_REGISTER_NODE (ip4_input_node) = { .format_trace = format_ip4_input_trace, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_input_node, ip4_input) + VLIB_REGISTER_NODE (ip4_input_no_checksum_node,static) = { .function = ip4_input_no_checksum, .name = "ip4-input-no-checksum", @@ -389,6 +391,8 @@ VLIB_REGISTER_NODE (ip4_input_no_checksum_node,static) = { .format_trace = format_ip4_input_trace, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_input_no_checksum_node, ip4_input_no_checksum) + static clib_error_t * ip4_init (vlib_main_t * vm) { clib_error_t * error; diff --git a/vnet/vnet/ip/ip4_source_check.c b/vnet/vnet/ip/ip4_source_check.c index 47e22f23..11e6678e 100644 --- a/vnet/vnet/ip/ip4_source_check.c +++ b/vnet/vnet/ip/ip4_source_check.c @@ -297,6 +297,9 @@ VLIB_REGISTER_NODE (ip4_check_source_reachable_via_any) = { .format_trace = format_ip4_source_check_trace, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_check_source_reachable_via_any, + ip4_source_check_reachable_via_any) + VLIB_REGISTER_NODE (ip4_check_source_reachable_via_rx) = { .function = ip4_source_check_reachable_via_rx, .name = "ip4-source-check-via-rx", @@ -311,6 +314,9 @@ VLIB_REGISTER_NODE (ip4_check_source_reachable_via_rx) = { .format_trace = format_ip4_source_check_trace, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_check_source_reachable_via_rx, + ip4_source_check_reachable_via_rx) + static clib_error_t * set_ip_source_check (vlib_main_t * vm, unformat_input_t * input, diff --git a/vnet/vnet/ip/ip6_forward.c b/vnet/vnet/ip/ip6_forward.c index a136da3e..7093c571 100644 --- a/vnet/vnet/ip/ip6_forward.c +++ b/vnet/vnet/ip/ip6_forward.c @@ -1260,6 +1260,8 @@ VLIB_REGISTER_NODE (ip6_lookup_node) = { .next_nodes = IP6_LOOKUP_NEXT_NODES, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_lookup_node, ip6_lookup) + static uword ip6_indirect (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -1278,6 +1280,8 @@ VLIB_REGISTER_NODE (ip6_indirect_node) = { .next_nodes = IP6_LOOKUP_NEXT_NODES, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_indirect_node, ip6_indirect) + typedef struct { /* Adjacency taken. */ u32 adj_index; @@ -1454,6 +1458,8 @@ VLIB_REGISTER_NODE (ip6_drop_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_drop_node, ip6_drop) + VLIB_REGISTER_NODE (ip6_punt_node,static) = { .function = ip6_punt, .name = "ip6-punt", @@ -1467,6 +1473,8 @@ VLIB_REGISTER_NODE (ip6_punt_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_punt_node, ip6_punt) + VLIB_REGISTER_NODE (ip6_miss_node,static) = { .function = ip6_miss, .name = "ip6-miss", @@ -1480,6 +1488,8 @@ VLIB_REGISTER_NODE (ip6_miss_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_miss_node, ip6_miss) + VLIB_REGISTER_NODE (ip6_multicast_node,static) = { .function = ip6_drop, .name = "ip6-multicast", @@ -1858,6 +1868,8 @@ VLIB_REGISTER_NODE (ip6_local_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_local_node, ip6_local) + void ip6_register_protocol (u32 protocol, u32 node_index) { vlib_main_t * vm = vlib_get_main(); @@ -2424,6 +2436,8 @@ VLIB_REGISTER_NODE (ip6_rewrite_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_node, ip6_rewrite_transit) + VLIB_REGISTER_NODE (ip6_rewrite_local_node,static) = { .function = ip6_rewrite_local, .name = "ip6-rewrite-local", @@ -2439,6 +2453,8 @@ VLIB_REGISTER_NODE (ip6_rewrite_local_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_rewrite_local_node, ip6_rewrite_local) + /* Global IP6 main. */ ip6_main_t ip6_main; diff --git a/vnet/vnet/ip/ip6_hop_by_hop.c b/vnet/vnet/ip/ip6_hop_by_hop.c index 16650dde..045dbc7f 100644 --- a/vnet/vnet/ip/ip6_hop_by_hop.c +++ b/vnet/vnet/ip/ip6_hop_by_hop.c @@ -628,6 +628,8 @@ VLIB_REGISTER_NODE (ip6_hop_by_hop_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_hop_by_hop_node, ip6_hop_by_hop_node_fn) + /* The main h-b-h tracer will be invoked, no need to do much here */ typedef struct { u32 next_index; @@ -849,6 +851,7 @@ VLIB_REGISTER_NODE (ip6_add_hop_by_hop_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_add_hop_by_hop_node, ip6_add_hop_by_hop_node_fn) /* The main h-b-h tracer was already invoked, no need to do much here */ typedef struct { @@ -1162,6 +1165,8 @@ VLIB_REGISTER_NODE (ip6_pop_hop_by_hop_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_pop_hop_by_hop_node, + ip6_pop_hop_by_hop_node_fn) static clib_error_t * ip6_hop_by_hop_init (vlib_main_t * vm) diff --git a/vnet/vnet/ip/ip6_input.c b/vnet/vnet/ip/ip6_input.c index f96a1cfb..2042cbd7 100644 --- a/vnet/vnet/ip/ip6_input.c +++ b/vnet/vnet/ip/ip6_input.c @@ -305,6 +305,8 @@ VLIB_REGISTER_NODE (ip6_input_node) = { .format_trace = format_ip6_input_trace, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_input_node, ip6_input) + static clib_error_t * ip6_init (vlib_main_t * vm) { ethernet_register_input_type (vm, ETHERNET_TYPE_IP6, diff --git a/vnet/vnet/ip/ip_input_acl.c b/vnet/vnet/ip/ip_input_acl.c index e905ed15..fcf8eeab 100644 --- a/vnet/vnet/ip/ip_input_acl.c +++ b/vnet/vnet/ip/ip_input_acl.c @@ -369,6 +369,8 @@ VLIB_REGISTER_NODE (ip4_inacl_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip4_inacl_node, ip4_inacl) + static uword ip6_inacl (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -392,6 +394,8 @@ VLIB_REGISTER_NODE (ip6_inacl_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ip6_inacl_node, ip6_inacl) + static clib_error_t * ip_inacl_init (vlib_main_t * vm) { diff --git a/vnet/vnet/ip/udp_local.c b/vnet/vnet/ip/udp_local.c index 253ad0f3..354dd4e8 100644 --- a/vnet/vnet/ip/udp_local.c +++ b/vnet/vnet/ip/udp_local.c @@ -327,6 +327,8 @@ VLIB_REGISTER_NODE (udp4_input_node) = { .unformat_buffer = unformat_udp_header, }; +VLIB_NODE_FUNCTION_MULTIARCH (udp4_input_node, udp4_input) + VLIB_REGISTER_NODE (udp6_input_node) = { .function = udp6_input, .name = "ip6-udp-lookup", @@ -350,6 +352,8 @@ VLIB_REGISTER_NODE (udp6_input_node) = { .unformat_buffer = unformat_udp_header, }; +VLIB_NODE_FUNCTION_MULTIARCH (udp6_input_node, udp6_input) + static void add_dst_port (udp_main_t * um, udp_dst_port_t dst_port, char * dst_port_name, u8 is_ip4) diff --git a/vnet/vnet/ipsec/esp_decrypt.c b/vnet/vnet/ipsec/esp_decrypt.c index 958a4d67..7af88b23 100644 --- a/vnet/vnet/ipsec/esp_decrypt.c +++ b/vnet/vnet/ipsec/esp_decrypt.c @@ -425,3 +425,5 @@ VLIB_REGISTER_NODE (esp_decrypt_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (esp_decrypt_node, esp_decrypt_node_fn) + diff --git a/vnet/vnet/ipsec/esp_encrypt.c b/vnet/vnet/ipsec/esp_encrypt.c index 39bbf2e4..d1dbcf96 100644 --- a/vnet/vnet/ipsec/esp_encrypt.c +++ b/vnet/vnet/ipsec/esp_encrypt.c @@ -387,3 +387,5 @@ VLIB_REGISTER_NODE (esp_encrypt_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (esp_encrypt_node, esp_encrypt_node_fn) + diff --git a/vnet/vnet/ipsec/ipsec_if_in.c b/vnet/vnet/ipsec/ipsec_if_in.c index 517f8bff..5a8a6858 100644 --- a/vnet/vnet/ipsec/ipsec_if_in.c +++ b/vnet/vnet/ipsec/ipsec_if_in.c @@ -148,4 +148,7 @@ VLIB_REGISTER_NODE (ipsec_if_input_node) = { [IPSEC_IF_INPUT_NEXT_ESP_DECRYPT] = "esp-decrypt", [IPSEC_IF_INPUT_NEXT_DROP] = "error-drop", }, -};
\ No newline at end of file +}; + +VLIB_NODE_FUNCTION_MULTIARCH (ipsec_if_input_node, ipsec_if_input_node_fn) + diff --git a/vnet/vnet/ipsec/ipsec_if_out.c b/vnet/vnet/ipsec/ipsec_if_out.c index 1e1dd528..9573c4f6 100644 --- a/vnet/vnet/ipsec/ipsec_if_out.c +++ b/vnet/vnet/ipsec/ipsec_if_out.c @@ -138,3 +138,5 @@ VLIB_REGISTER_NODE (ipsec_if_output_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ipsec_if_output_node, ipsec_if_output_node_fn) + diff --git a/vnet/vnet/ipsec/ipsec_input.c b/vnet/vnet/ipsec/ipsec_input.c index 3cd60ba1..09acd106 100644 --- a/vnet/vnet/ipsec/ipsec_input.c +++ b/vnet/vnet/ipsec/ipsec_input.c @@ -288,6 +288,9 @@ VLIB_REGISTER_NODE (ipsec_input_ip4_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ipsec_input_ip4_node, + ipsec_input_ip4_node_fn) + static vlib_node_registration_t ipsec_input_ip6_node; @@ -406,3 +409,7 @@ VLIB_REGISTER_NODE (ipsec_input_ip6_node,static) = { #undef _ }, }; + +VLIB_NODE_FUNCTION_MULTIARCH (ipsec_input_ip6_node, + ipsec_input_ip6_node_fn) + diff --git a/vnet/vnet/ipsec/ipsec_output.c b/vnet/vnet/ipsec/ipsec_output.c index 509b1e2f..93554681 100644 --- a/vnet/vnet/ipsec/ipsec_output.c +++ b/vnet/vnet/ipsec/ipsec_output.c @@ -405,6 +405,8 @@ VLIB_REGISTER_NODE (ipsec_output_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (ipsec_output_node, ipsec_output_node_fn) + #else /* IPSEC > 1 */ /* Dummy ipsec output node, in case when IPSec is disabled */ diff --git a/vnet/vnet/l2/l2_classify.c b/vnet/vnet/l2/l2_classify.c index 449dea72..765bc686 100644 --- a/vnet/vnet/l2/l2_classify.c +++ b/vnet/vnet/l2/l2_classify.c @@ -399,6 +399,8 @@ VLIB_REGISTER_NODE (l2_classify_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2_classify_node, l2_classify_node_fn) + clib_error_t *l2_classify_init (vlib_main_t *vm) { l2_classify_main_t * cm = &l2_classify_main; diff --git a/vnet/vnet/l2/l2_efp_filter.c b/vnet/vnet/l2/l2_efp_filter.c index b865c375..17b7eb3f 100644 --- a/vnet/vnet/l2/l2_efp_filter.c +++ b/vnet/vnet/l2/l2_efp_filter.c @@ -498,6 +498,8 @@ VLIB_REGISTER_NODE (l2_efp_filter_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2_efp_filter_node, l2_efp_filter_node_fn) + clib_error_t *l2_efp_filter_init (vlib_main_t *vm) { l2_efp_filter_main_t * mp = &l2_efp_filter_main; diff --git a/vnet/vnet/l2/l2_flood.c b/vnet/vnet/l2/l2_flood.c index ca8c171c..9f71677c 100644 --- a/vnet/vnet/l2/l2_flood.c +++ b/vnet/vnet/l2/l2_flood.c @@ -442,6 +442,8 @@ VLIB_REGISTER_NODE (l2flood_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2flood_node, l2flood_node_fn) + clib_error_t *l2flood_init (vlib_main_t *vm) { l2flood_main_t * mp = &l2flood_main; diff --git a/vnet/vnet/l2/l2_fwd.c b/vnet/vnet/l2/l2_fwd.c index 88a6b69c..5af83a75 100644 --- a/vnet/vnet/l2/l2_fwd.c +++ b/vnet/vnet/l2/l2_fwd.c @@ -367,6 +367,8 @@ VLIB_REGISTER_NODE (l2fwd_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2fwd_node, l2fwd_node_fn) + clib_error_t *l2fwd_init (vlib_main_t *vm) { l2fwd_main_t * mp = &l2fwd_main; diff --git a/vnet/vnet/l2/l2_input.c b/vnet/vnet/l2/l2_input.c index 870aff68..dd3a8b75 100644 --- a/vnet/vnet/l2/l2_input.c +++ b/vnet/vnet/l2/l2_input.c @@ -433,6 +433,8 @@ VLIB_REGISTER_NODE (l2input_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2input_node, l2input_node_fn) + clib_error_t *l2input_init (vlib_main_t *vm) { l2input_main_t * mp = &l2input_main; diff --git a/vnet/vnet/l2/l2_input_acl.c b/vnet/vnet/l2/l2_input_acl.c index 49df0f65..68c005f1 100644 --- a/vnet/vnet/l2/l2_input_acl.c +++ b/vnet/vnet/l2/l2_input_acl.c @@ -371,6 +371,8 @@ VLIB_REGISTER_NODE (l2_inacl_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2_inacl_node, l2_inacl_node_fn) + clib_error_t *l2_inacl_init (vlib_main_t *vm) { l2_inacl_main_t * mp = &l2_inacl_main; diff --git a/vnet/vnet/l2/l2_input_vtr.c b/vnet/vnet/l2/l2_input_vtr.c index bbef33a9..d0f2181b 100644 --- a/vnet/vnet/l2/l2_input_vtr.c +++ b/vnet/vnet/l2/l2_input_vtr.c @@ -293,6 +293,8 @@ VLIB_REGISTER_NODE (l2_invtr_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2_invtr_node, l2_invtr_node_fn) + clib_error_t *l2_invtr_init (vlib_main_t *vm) { l2_invtr_main_t * mp = &l2_invtr_main; diff --git a/vnet/vnet/l2/l2_learn.c b/vnet/vnet/l2/l2_learn.c index a891cb3c..0d94e43d 100644 --- a/vnet/vnet/l2/l2_learn.c +++ b/vnet/vnet/l2/l2_learn.c @@ -415,6 +415,7 @@ VLIB_REGISTER_NODE (l2learn_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2learn_node, l2learn_node_fn) clib_error_t *l2learn_init (vlib_main_t *vm) { diff --git a/vnet/vnet/l2/l2_output.c b/vnet/vnet/l2/l2_output.c index a550d075..f5b22117 100644 --- a/vnet/vnet/l2/l2_output.c +++ b/vnet/vnet/l2/l2_output.c @@ -414,6 +414,8 @@ VLIB_REGISTER_NODE (l2output_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2output_node, l2output_node_fn) + clib_error_t *l2output_init (vlib_main_t *vm) { l2output_main_t * mp = &l2output_main; diff --git a/vnet/vnet/l2/l2_output_acl.c b/vnet/vnet/l2/l2_output_acl.c index f75345f1..b222255f 100644 --- a/vnet/vnet/l2/l2_output_acl.c +++ b/vnet/vnet/l2/l2_output_acl.c @@ -274,6 +274,8 @@ VLIB_REGISTER_NODE (l2_outacl_node,static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2_outacl_node, l2_outacl_node_fn) + clib_error_t *l2_outacl_init (vlib_main_t *vm) { l2_outacl_main_t * mp = &l2_outacl_main; diff --git a/vnet/vnet/l2/l2_patch.c b/vnet/vnet/l2/l2_patch.c index 63be409d..0839142b 100644 --- a/vnet/vnet/l2/l2_patch.c +++ b/vnet/vnet/l2/l2_patch.c @@ -252,6 +252,8 @@ VLIB_REGISTER_NODE (l2_patch_node, static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2_patch_node, l2_patch_node_fn) + int vnet_l2_patch_add_del (u32 rx_sw_if_index, u32 tx_sw_if_index, int is_add) { l2_patch_main_t * l2pm = &l2_patch_main; diff --git a/vnet/vnet/l2/l2_rw.c b/vnet/vnet/l2/l2_rw.c index 93580c24..b66a7d34 100644 --- a/vnet/vnet/l2/l2_rw.c +++ b/vnet/vnet/l2/l2_rw.c @@ -604,3 +604,5 @@ VLIB_REGISTER_NODE (l2_rw_node) = { .next_nodes = { [L2_RW_NEXT_DROP] = "error-drop"}, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2_rw_node, l2_rw_node_fn) + diff --git a/vnet/vnet/l2/l2_xcrw.c b/vnet/vnet/l2/l2_xcrw.c index f5fe3ca1..00542a19 100644 --- a/vnet/vnet/l2/l2_xcrw.c +++ b/vnet/vnet/l2/l2_xcrw.c @@ -255,6 +255,8 @@ VLIB_REGISTER_NODE (l2_xcrw_node, static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2_xcrw_node, l2_xcrw_node_fn) + clib_error_t *l2_xcrw_init (vlib_main_t *vm) { l2_xcrw_main_t * mp = &l2_xcrw_main; diff --git a/vnet/vnet/l2tp/decap.c b/vnet/vnet/l2tp/decap.c index 1a2bc489..5f0d05c0 100644 --- a/vnet/vnet/l2tp/decap.c +++ b/vnet/vnet/l2tp/decap.c @@ -247,6 +247,8 @@ VLIB_REGISTER_NODE (l2t_decap_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2t_decap_node, l2t_decap_node_fn) + void l2tp_decap_init (void) { ip6_register_protocol (IP_PROTOCOL_L2TP, l2t_decap_node.index); diff --git a/vnet/vnet/l2tp/encap.c b/vnet/vnet/l2tp/encap.c index 8f26ab00..eca098cb 100644 --- a/vnet/vnet/l2tp/encap.c +++ b/vnet/vnet/l2tp/encap.c @@ -206,6 +206,8 @@ VLIB_REGISTER_NODE (l2t_encap_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2t_encap_node, l2t_encap_node_fn) + void l2tp_encap_init (vlib_main_t * vm) { l2tp_encap_runtime_t * rt; diff --git a/vnet/vnet/lawful-intercept/node.c b/vnet/vnet/lawful-intercept/node.c index 26514cc5..8701c323 100644 --- a/vnet/vnet/lawful-intercept/node.c +++ b/vnet/vnet/lawful-intercept/node.c @@ -272,6 +272,8 @@ VLIB_REGISTER_NODE (li_hit_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (li_hit_node, li_hit_node_fn) + #else #include <vlib/vlib.h> diff --git a/vnet/vnet/mpls-gre/interface.c b/vnet/vnet/mpls-gre/interface.c index 408ca750..75cd022a 100644 --- a/vnet/vnet/mpls-gre/interface.c +++ b/vnet/vnet/mpls-gre/interface.c @@ -259,6 +259,9 @@ VNET_DEVICE_CLASS (mpls_gre_device_class) = { #endif }; +VLIB_DEVICE_TX_FUNCTION_MULTIARCH (mpls_gre_device_class, + mpls_gre_interface_tx) + VNET_HW_INTERFACE_CLASS (mpls_gre_hw_interface_class) = { .name = "MPLS-GRE", .format_header = format_mpls_gre_header_with_length, @@ -510,6 +513,8 @@ VNET_DEVICE_CLASS (mpls_eth_device_class) = { #endif }; +VLIB_DEVICE_TX_FUNCTION_MULTIARCH (mpls_eth_device_class, + mpls_eth_interface_tx) VNET_HW_INTERFACE_CLASS (mpls_eth_hw_interface_class) = { .name = "MPLS-ETH", @@ -676,6 +681,8 @@ VLIB_REGISTER_NODE (mpls_post_rewrite_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (mpls_post_rewrite_node, mpls_post_rewrite) + static u8 * mpls_gre_rewrite (mpls_main_t *mm, mpls_gre_tunnel_t * t) { ip4_header_t * ip0; diff --git a/vnet/vnet/mpls-gre/node.c b/vnet/vnet/mpls-gre/node.c index 6bf5f814..474e2e2a 100644 --- a/vnet/vnet/mpls-gre/node.c +++ b/vnet/vnet/mpls-gre/node.c @@ -278,6 +278,8 @@ VLIB_REGISTER_NODE (mpls_input_node) = { .unformat_buffer = unformat_mpls_gre_header, }; +VLIB_NODE_FUNCTION_MULTIARCH (mpls_input_node, mpls_input) + static uword mpls_ethernet_input (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -310,6 +312,8 @@ VLIB_REGISTER_NODE (mpls_ethernet_input_node) = { .unformat_buffer = unformat_mpls_gre_header, }; +VLIB_NODE_FUNCTION_MULTIARCH (mpls_ethernet_input_node, mpls_ethernet_input) + static void mpls_setup_nodes (vlib_main_t * vm) { diff --git a/vnet/vnet/mpls-gre/policy_encap.c b/vnet/vnet/mpls-gre/policy_encap.c index b4439193..e3f4beb6 100644 --- a/vnet/vnet/mpls-gre/policy_encap.c +++ b/vnet/vnet/mpls-gre/policy_encap.c @@ -153,6 +153,8 @@ VLIB_REGISTER_NODE (mpls_policy_encap_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (mpls_policy_encap_node, mpls_policy_encap) + static clib_error_t * mpls_policy_encap_init (vlib_main_t * vm) { diff --git a/vnet/vnet/policer/node_funcs.c b/vnet/vnet/policer/node_funcs.c index 8a76e869..739ce455 100644 --- a/vnet/vnet/policer/node_funcs.c +++ b/vnet/vnet/policer/node_funcs.c @@ -324,6 +324,9 @@ VLIB_REGISTER_NODE (policer_by_sw_if_index_node, static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (policer_by_sw_if_index_node, + vnet_policer_by_sw_if_index); + int test_policer_add_del (u32 rx_sw_if_index, u8 *config_name, int is_add) diff --git a/vnet/vnet/sr/sr.c b/vnet/vnet/sr/sr.c index 46318a27..e0ef3181 100644 --- a/vnet/vnet/sr/sr.c +++ b/vnet/vnet/sr/sr.c @@ -653,6 +653,8 @@ VLIB_REGISTER_NODE (sr_rewrite_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (sr_rewrite_node, sr_rewrite) + static int ip6_delete_route_no_next_hop (ip6_address_t *dst_address_arg, u32 dst_address_length, u32 rx_table_id) @@ -1911,6 +1913,8 @@ VLIB_REGISTER_NODE (sr_fix_dst_addr_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (sr_fix_dst_addr_node, sr_fix_dst_addr) + static clib_error_t * sr_init (vlib_main_t * vm) { ip6_sr_main_t * sm = &sr_main; @@ -2560,6 +2564,8 @@ VLIB_REGISTER_NODE (sr_local_node, static) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (sr_local_node, sr_local) + ip6_sr_main_t * sr_get_main (vlib_main_t * vm) { vlib_call_init_function (vm, sr_init); diff --git a/vnet/vnet/sr/sr_replicate.c b/vnet/vnet/sr/sr_replicate.c index 80d04ba6..ccc01465 100644 --- a/vnet/vnet/sr/sr_replicate.c +++ b/vnet/vnet/sr/sr_replicate.c @@ -350,6 +350,8 @@ VLIB_REGISTER_NODE (sr_replicate_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (sr_replicate_node, sr_replicate_node_fn) + clib_error_t *sr_replicate_init (vlib_main_t *vm) { sr_replicate_main_t *msm = &sr_replicate_main; diff --git a/vnet/vnet/vxlan/decap.c b/vnet/vnet/vxlan/decap.c index bd61f4bb..00acd660 100644 --- a/vnet/vnet/vxlan/decap.c +++ b/vnet/vnet/vxlan/decap.c @@ -550,6 +550,8 @@ VLIB_REGISTER_NODE (vxlan4_input_node) = { // $$$$ .unformat_buffer = unformat_vxlan_header, }; +VLIB_NODE_FUNCTION_MULTIARCH (vxlan4_input_node, vxlan4_input) + VLIB_REGISTER_NODE (vxlan6_input_node) = { .function = vxlan6_input, .name = "vxlan6-input", @@ -570,3 +572,6 @@ VLIB_REGISTER_NODE (vxlan6_input_node) = { .format_trace = format_vxlan_rx_trace, // $$$$ .unformat_buffer = unformat_vxlan_header, }; + +VLIB_NODE_FUNCTION_MULTIARCH (vxlan6_input_node, vxlan6_input) + diff --git a/vnet/vnet/vxlan/encap.c b/vnet/vnet/vxlan/encap.c index ee5c7d5e..387a728a 100644 --- a/vnet/vnet/vxlan/encap.c +++ b/vnet/vnet/vxlan/encap.c @@ -586,3 +586,6 @@ VLIB_REGISTER_NODE (vxlan_encap_node) = { [VXLAN_ENCAP_NEXT_DROP] = "error-drop", }, }; + +VLIB_NODE_FUNCTION_MULTIARCH (vxlan_encap_node, vxlan_encap) + diff --git a/vpp/app/l2t_ip6.c b/vpp/app/l2t_ip6.c index ac24886a..8a1e23a9 100644 --- a/vpp/app/l2t_ip6.c +++ b/vpp/app/l2t_ip6.c @@ -274,6 +274,8 @@ static VLIB_REGISTER_NODE (sw6_ip6_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (sw6_ip6_node, ip6_l2t_node_fn) + static clib_error_t * l2tp_config (vlib_main_t * vm, unformat_input_t * input) { diff --git a/vpp/app/l2t_l2.c b/vpp/app/l2t_l2.c index 5c8327d5..f6385a5a 100644 --- a/vpp/app/l2t_l2.c +++ b/vpp/app/l2t_l2.c @@ -250,3 +250,5 @@ VLIB_REGISTER_NODE (l2t_l2_node) = { }, }; +VLIB_NODE_FUNCTION_MULTIARCH (l2t_l2_node, l2t_l2_node_fn) + diff --git a/vpp/app/version.c b/vpp/app/version.c index 588aadf2..22bec400 100644 --- a/vpp/app/version.c +++ b/vpp/app/version.c @@ -59,6 +59,7 @@ show_vpe_version_command_fn (vlib_main_t * vm, _("Compiler", "%s", vpe_compiler); _("CPU model name", "%U", format_cpu_model_name); _("CPU microarchitecture", "%U", format_cpu_uarch); + _("CPU flags", "%U", format_cpu_flags); _("Current PID", "%d", getpid()); #if DPDK > 0 _("DPDK Version", "%s", rte_version()); diff --git a/vpp/vnet/main.c b/vpp/vnet/main.c index f707994b..25400839 100644 --- a/vpp/vnet/main.c +++ b/vpp/vnet/main.c @@ -12,6 +12,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#include <vppinfra/cpu.h> #include <vlib/vlib.h> #include <vlib/unix/unix.h> #include <vnet/plugin/plugin.h> @@ -59,33 +61,32 @@ int main (int argc, char * argv[]) u32 size; void vlib_set_get_handoff_structure_cb (void *cb); -#if __x86_64__ && !defined(__clang__) - __builtin_cpu_init (); +#if __x86_64__ const char * msg = "ERROR: This binary requires CPU with %s extensions.\n"; #define _(a,b) \ - if (!__builtin_cpu_supports(a)) \ + if (!clib_cpu_supports_ ## a ()) \ { \ fprintf(stderr, msg, b); \ exit(1); \ } #if __AVX2__ - _("avx2", "AVX2") + _(avx2, "AVX2") #endif #if __AVX__ - _("avx", "AVX") + _(avx, "AVX") #endif #if __SSE4_2__ - _("sse4.2", "SSE4.2") + _(sse42, "SSE4.2") #endif #if __SSE4_1__ - _("sse4.1", "SSE4.1") + _(sse41, "SSE4.1") #endif #if __SSSE3__ - _("ssse3", "SSSE3") + _(ssse3, "SSSE3") #endif #if __SSE3__ - _("sse3", "SSE3") + _(sse3, "SSE3") #endif #undef _ #endif diff --git a/vppinfra/vppinfra/cpu.c b/vppinfra/vppinfra/cpu.c index f2dbaf1f..9008ee3d 100644 --- a/vppinfra/vppinfra/cpu.c +++ b/vppinfra/vppinfra/cpu.c @@ -16,10 +16,6 @@ #include <vppinfra/format.h> #include <vppinfra/cpu.h> -#if __x86_64__ -#include <cpuid.h> -#endif - #define foreach_x86_cpu_uarch \ _(0x06, 0x4f, "Broadwell", "Broadwell-EP/EX") \ _(0x06, 0x3d, "Broadwell", "Broadwell") \ @@ -108,4 +104,21 @@ format_cpu_model_name (u8 * s, va_list * args) #else /* ! __x86_64__ */ return format (s, "unknown"); #endif -}
\ No newline at end of file +} + +u8 * +format_cpu_flags (u8 * s, va_list * args) +{ +#if __x86_64__ +#define _(flag, func, reg, bit) \ + if (clib_cpu_supports_ ## flag()) \ + s = format (s, #flag " "); + foreach_x86_64_flags + return s; +#undef _ +#else /* ! __x86_64__ */ + return format (s, "unknown"); +#endif +} + + diff --git a/vppinfra/vppinfra/cpu.h b/vppinfra/vppinfra/cpu.h index 79cdf74f..961af709 100644 --- a/vppinfra/vppinfra/cpu.h +++ b/vppinfra/vppinfra/cpu.h @@ -16,7 +16,83 @@ #ifndef included_clib_cpu_h #define included_clib_cpu_h +#include <vppinfra/format.h> + +/* + * multiarchitecture support. Adding new entry will produce + * new graph node function variant optimized for specific cpu + * microarchitecture. + * Order is important for runtime selection, as 1st match wins... + */ + +#if __x86_64__ && CLIB_DEBUG == 0 +#define foreach_march_variant(macro, x) \ + macro(avx2, x, "arch=core-avx2") +#else +#define foreach_march_variant(macro, x) +#endif + + +#if __GNUC__ > 4 && !__clang__ +#define CLIB_CPU_OPTIMIZED __attribute__ ((optimize ("tree-vectorize"))) +#else +#define CLIB_CPU_OPTIMIZED +#endif + + +#define CLIB_MULTIARCH_ARCH_CHECK(arch, fn, tgt) \ + if (clib_cpu_supports_ ## arch()) \ + return & fn ## _ ##arch; + +#define CLIB_MULTIARCH_SELECT_FN(fn,...) \ + __VA_ARGS__ void * fn ## _multiarch_select(void) \ +{ \ + foreach_march_variant(CLIB_MULTIARCH_ARCH_CHECK, fn) \ + return & fn; \ +} + +#if __x86_64__ +#include "cpuid.h" + +#define foreach_x86_64_flags \ +_ (sse3, 1, ecx, 0) \ +_ (ssse3, 1, ecx, 9) \ +_ (sse41, 1, ecx, 19) \ +_ (sse42, 1, ecx, 20) \ +_ (avx, 1, ecx, 28) \ +_ (avx2, 7, ebx, 5) \ +_ (avx512f, 7, ebx, 16) \ +_ (aes, 1, ecx, 25) \ +_ (sha, 7, ebx, 29) + +static inline int +clib_get_cpuid(const u32 lev, u32 * eax, u32 *ebx, u32 * ecx, u32 * edx) +{ + if ((u32) __get_cpuid_max (0x80000000 & lev, 0) < lev) + return 0; + if (lev == 7) + __cpuid_count(lev, 0, *eax, *ebx, *ecx, *edx); + else + __cpuid(lev, *eax, *ebx, *ecx, *edx); + return 1; +} + + +#define _(flag, func, reg, bit) \ +static inline int \ +clib_cpu_supports_ ## flag() \ +{ \ + u32 __attribute__((unused)) eax, ebx = 0, ecx = 0, edx = 0; \ + clib_get_cpuid (func, &eax, &ebx, &ecx, &edx); \ + \ + return ((reg & (1 << bit)) != 0); \ +} + foreach_x86_64_flags +#undef _ +#endif + format_function_t format_cpu_uarch; format_function_t format_cpu_model_name; +format_function_t format_cpu_flags; -#endif
\ No newline at end of file +#endif |