diff options
Diffstat (limited to 'vpp')
| -rw-r--r-- | vpp/build-data/packages/vnet.mk | 6 | ||||
| -rw-r--r-- | vpp/build-data/packages/vpp.mk | 7 | ||||
| -rw-r--r-- | vpp/build-data/platforms/odp.mk | 55 | ||||
| -rw-r--r-- | vpp/vnet/Makefile.am | 13 | ||||
| -rw-r--r-- | vpp/vnet/configure.ac | 8 | ||||
| -rwxr-xr-x | vpp/vnet/vnet/devices/odp/cli.c | 147 | ||||
| -rwxr-xr-x | vpp/vnet/vnet/devices/odp/device.c | 226 | ||||
| -rwxr-xr-x | vpp/vnet/vnet/devices/odp/node.c | 334 | ||||
| -rwxr-xr-x | vpp/vnet/vnet/devices/odp/odp_packet.c | 354 | ||||
| -rwxr-xr-x | vpp/vnet/vnet/devices/odp/odp_packet.h | 74 | ||||
| -rw-r--r-- | vpp/vpp/Makefile.am | 6 | ||||
| -rw-r--r-- | vpp/vpp/configure.ac | 7 |
12 files changed, 1236 insertions, 1 deletions
diff --git a/vpp/build-data/packages/vnet.mk b/vpp/build-data/packages/vnet.mk index 399ca1b4..1e0605d3 100644 --- a/vpp/build-data/packages/vnet.mk +++ b/vpp/build-data/packages/vnet.mk @@ -16,6 +16,12 @@ vnet_LDFLAGS = $(call installed_libs_fn, \ vlib \ vlib-api) +#include ODP library only if ODP is being used. +ifeq ($($(PLATFORM)_uses_odp),yes) +vnet_CPPFLAGS += -I$($(PLATFORM)_odp_inc_dir) +vnet_LDFLAGS += -L$($(PLATFORM)_odp_lib_dir) +endif + ifeq ($($(PLATFORM)_enable_tests),yes) vnet_configure_args += --enable-tests endif diff --git a/vpp/build-data/packages/vpp.mk b/vpp/build-data/packages/vpp.mk index 6831c6b8..7a4f57ab 100644 --- a/vpp/build-data/packages/vpp.mk +++ b/vpp/build-data/packages/vpp.mk @@ -30,6 +30,13 @@ vpp_LDFLAGS = $(call installed_libs_fn, \ vlib-api \ vnet) +#include ODP library only if ODP is being used. +ifeq ($($(PLATFORM)_uses_odp),yes) +vpp_CPPFLAGS += -I$($(PLATFORM)_odp_inc_dir) +vpp_LDFLAGS += -L$($(PLATFORM)_odp_lib_dir) +export ODP_LIBS= $($(PLATFORM)_odp_libs) +endif + # include & link with openssl only if needed ifneq ($($(PLATFORM)_uses_openssl),no) vpp_CPPFLAGS += $(call installed_includes_fn, openssl) diff --git a/vpp/build-data/platforms/odp.mk b/vpp/build-data/platforms/odp.mk new file mode 100644 index 00000000..df2ae9e5 --- /dev/null +++ b/vpp/build-data/platforms/odp.mk @@ -0,0 +1,55 @@ +# Copyright (c) 2016 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# vector packet processor +odp_arch = native +ifeq ($(shell uname -m),x86_64) +odp_march = corei7 # Nehalem Instruction set +odp_mtune = corei7-avx # Optimize for Sandy Bridge +else +odp_march = native +odp_mtune = generic +endif +odp_native_tools = vppapigen + +odp_uses_dpdk = no +odp_uses_odp= yes +# Uncoment to enable building unit tests +#odp_enable_tests = yes + +odp_root_packages = vpp vlib vlib-api vnet svm vpp-api-test \ + vpp-api gmod + +vlib_configure_args_odp = --with-pre-data=128 + +#ODP configuration parameters +odp_odp_libs = -lodp-dpdk -ldpdk -lpcap +odp_odp_inc_dir=$(ODP_INST_PATH)/include +odp_odp_lib_dir=$(ODP_INST_PATH)/lib +vpp_configure_args_odp = --with-odplib +vnet_configure_args_odp = --with-odplib + +odp_debug_TAG_CFLAGS = -g -O0 -DCLIB_DEBUG -DFORTIFY_SOURCE=2 -march=$(MARCH) \ + -fstack-protector-all -fPIC -Werror +odp_debug_TAG_LDFLAGS = -g -O0 -DCLIB_DEBUG -DFORTIFY_SOURCE=2 -march=$(MARCH) \ + -fstack-protector-all -fPIC -Werror + +odp_TAG_CFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) -mtune=$(MTUNE) \ + -fstack-protector -fPIC -Werror +odp_TAG_LDFLAGS = -g -O2 -DFORTIFY_SOURCE=2 -march=$(MARCH) -mtune=$(MTUNE) \ + -fstack-protector -fPIC -Werror + +odp_gcov_TAG_CFLAGS = -g -O0 -DCLIB_DEBUG -march=$(MARCH) \ + -fPIC -Werror -fprofile-arcs -ftest-coverage +odp_gcov_TAG_LDFLAGS = -g -O0 -DCLIB_DEBUG -march=$(MARCH) \ + -fPIC -Werror -coverage diff --git a/vpp/vnet/Makefile.am b/vpp/vnet/Makefile.am index 4e4b2c0a..bb62ef52 100644 --- a/vpp/vnet/Makefile.am +++ b/vpp/vnet/Makefile.am @@ -784,6 +784,19 @@ libvnet_la_SOURCES += \ nobase_include_HEADERS += \ vnet/devices/netmap/netmap.h +######################################## +# ODP interface +######################################## +if WITH_ODPLIB +libvnet_la_SOURCES += \ + vnet/devices/odp/odp_packet.c \ + vnet/devices/odp/device.c \ + vnet/devices/odp/node.c \ + vnet/devices/odp/cli.c + +nobase_include_HEADERS += \ + vnet/devices/odp/odp_packet.h +endif ######################################## # Driver feature graph arc support diff --git a/vpp/vnet/configure.ac b/vpp/vnet/configure.ac index 6a5281b0..b6a13d67 100644 --- a/vpp/vnet/configure.ac +++ b/vpp/vnet/configure.ac @@ -27,6 +27,11 @@ AC_ARG_WITH(ipv6sr, [with_ipv6sr=0], [with_ipv6sr=1]) +AC_ARG_WITH(odplib, + AC_HELP_STRING([--with-odplib],[Use odplib]), + [with_odplib=1], + [with_odplib=0]) + AC_ARG_ENABLE(tests, AC_HELP_STRING([--enable-tests], [Build unit tests]), [enable_tests=1], @@ -44,6 +49,9 @@ AC_SUBST(IPSEC,[-DIPSEC=${with_ipsec}]) AM_CONDITIONAL(WITH_IPV6SR, test "$with_ipv6sr" = "1") AC_SUBST(IPV6SR,[-DIPV6SR=${with_ipv6sr}]) +AM_CONDITIONAL(WITH_ODPLIB, test "$with_odplib" = "1") +AC_SUBST(ODPLIB,[-DODPLIB=${with_odplib}]) + AM_CONDITIONAL(ENABLE_TESTS, test "$enable_tests" = "1") AC_OUTPUT([Makefile]) diff --git a/vpp/vnet/vnet/devices/odp/cli.c b/vpp/vnet/vnet/devices/odp/cli.c new file mode 100755 index 00000000..9280f1a7 --- /dev/null +++ b/vpp/vnet/vnet/devices/odp/cli.c @@ -0,0 +1,147 @@ +/* + *------------------------------------------------------------------ + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ + +#include <fcntl.h> /* for open */ +#include <sys/ioctl.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/uio.h> /* for iovec */ +#include <netinet/in.h> +#include <vlib/vlib.h> +#include <vlib/unix/unix.h> +#include <vnet/ip/ip.h> +#include <vnet/ethernet/ethernet.h> +#include <vnet/devices/odp/odp_packet.h> + +static clib_error_t * +odp_packet_create_command_fn (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + u8 *host_if_name = NULL; + u8 hwaddr[6]; + u8 *hw_addr_ptr = 0; + u32 sw_if_index; + u32 mode=0; + int r; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "name %s", &host_if_name)) + ; + else + if (unformat + (line_input, "hw-addr %U", unformat_ethernet_address, hwaddr)) + hw_addr_ptr = hwaddr; + else + if (unformat + (line_input, "mode %d", &mode)) + ; + else + return clib_error_return (0, "unknown input `%U'", + format_unformat_error, input); + } + unformat_free (line_input); + + if (host_if_name == NULL) + return clib_error_return (0, "missing host interface name"); + + r = odp_packet_create_if (vm, host_if_name, hw_addr_ptr, &sw_if_index, mode); + vec_free (host_if_name); + + if (r == VNET_API_ERROR_SYSCALL_ERROR_1) + return clib_error_return (0, "%s (errno %d)", strerror (errno), errno); + + if (r == VNET_API_ERROR_INVALID_INTERFACE) + return clib_error_return (0, "Invalid interface name"); + + if (r == VNET_API_ERROR_SUBIF_ALREADY_EXISTS) + return clib_error_return (0, "Interface elready exists"); + + vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main (), + sw_if_index); + + return 0; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (odp_packet_create_command, static) = { + .path = "create pktio-interface", + .short_help = "create pktio-interface name <interface name> [hw-addr <mac>]", + .function = odp_packet_create_command_fn, +}; +/* *INDENT-ON* */ + +static clib_error_t * +odp_packet_delete_command_fn (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + unformat_input_t _line_input, *line_input = &_line_input; + u8 *host_if_name = NULL; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "name %s", &host_if_name)) + ; + else + return clib_error_return (0, "unknown input `%U'", + format_unformat_error, input); + } + unformat_free (line_input); + + + if (host_if_name == NULL) + return clib_error_return (0, "missing host interface name"); + + + odp_packet_delete_if (vm, host_if_name); + vec_free(host_if_name); + + return 0; +} + +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (odp_packet_delete_command, static) = { + .path = "delete pktio-interface", + .short_help = "delete pktio-interface name <interface name>", + .function = odp_packet_delete_command_fn, +}; +/* *INDENT-ON* */ + +clib_error_t * +odp_packet_cli_init (vlib_main_t * vm) +{ + return 0; +} + +VLIB_INIT_FUNCTION (odp_packet_cli_init); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/vpp/vnet/vnet/devices/odp/device.c b/vpp/vnet/vnet/devices/odp/device.c new file mode 100755 index 00000000..807c00b1 --- /dev/null +++ b/vpp/vnet/vnet/devices/odp/device.c @@ -0,0 +1,226 @@ +/* + *------------------------------------------------------------------ + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ + +#include <linux/if_packet.h> +#include <vlib/vlib.h> +#include <vlib/unix/unix.h> +#include <vnet/ip/ip.h> +#include <vnet/ethernet/ethernet.h> +#include <vnet/devices/odp/odp_packet.h> + +#define foreach_odp_packet_tx_func_error \ +_(FRAME_NOT_READY, "tx frame not ready") \ +_(TXRING_EAGAIN, "tx sendto temporary failure") \ +_(TXRING_FATAL, "tx sendto fatal failure") \ +_(TXRING_OVERRUN, "tx ring overrun") + +typedef enum +{ +#define _(f,s) ODP_PACKET_TX_ERROR_##f, + foreach_odp_packet_tx_func_error +#undef _ + ODP_PACKET_TX_N_ERROR, +} odp_packet_tx_func_error_t; + +static char *odp_packet_tx_func_error_strings[] = { +#define _(n,s) s, + foreach_odp_packet_tx_func_error +#undef _ +}; + + +static u8 * +format_odp_packet_device_name (u8 * s, va_list * args) +{ + u32 i = va_arg (*args, u32); + odp_packet_main_t *om = &odp_packet_main; + odp_packet_if_t *oif = vec_elt_at_index (om->interfaces, i); + + s = format (s, "odp-%s", oif->host_if_name); + return s; +} + +static u8 * +format_odp_packet_device (u8 * s, va_list * args) +{ + s = format (s, "odp interface"); + return s; +} + +static u8 * +format_odp_packet_tx_trace (u8 * s, va_list * args) +{ + s = format (s, "Unimplemented..."); + return s; +} + +static uword +odp_packet_interface_tx (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + + odp_packet_main_t *om = &odp_packet_main; + u32 *buffers = vlib_frame_args (frame); + u32 n_left = frame->n_vectors; + vnet_interface_output_runtime_t *rd = (void *) node->runtime_data; + odp_packet_if_t *oif =pool_elt_at_index (om->interfaces, rd->dev_instance); + odp_pktout_queue_t pktout; + odp_packet_t pkt_tbl[VLIB_FRAME_SIZE]; + u32 sent=0, count=0; + + if (PREDICT_FALSE (oif->lockp != 0)) + { + while (__sync_lock_test_and_set (oif->lockp, 1)) + ; + } + + if (odp_pktout_queue(oif->pktio, &pktout, 1) != 1) + { + return -1; + } + + while (n_left > 0) + { + u32 len; + vlib_buffer_t *b0; + n_left--; + u32 bi = buffers[0]; + buffers++; + + do + { + b0 = vlib_get_buffer (vm, bi); + len = b0->current_length; + pkt_tbl[count] = odp_packet_alloc(om->pool, len); + + if (pkt_tbl[count] == ODP_PACKET_INVALID) + { + clib_warning("odp packet alloc failed"); + } + + clib_memcpy ((u8 *) (odp_packet_data(pkt_tbl[count])), + vlib_buffer_get_current (b0), len); + count++; + } + while ((bi = b0->next_buffer) && (count < VLIB_FRAME_SIZE)); + } + + CLIB_MEMORY_BARRIER (); + + sent = odp_pktout_send(pktout, pkt_tbl, count); + sent= sent > 0 ? sent : 0; + + if (odp_unlikely( sent < count )) + { + do + { + odp_packet_free(pkt_tbl[sent]); + } + while(++sent < count ); + } + + if (PREDICT_FALSE (oif->lockp != 0)) + *oif->lockp = 0; + + vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors); + + return frame->n_vectors; +} + +static void +odp_packet_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index, + u32 node_index) +{ + odp_packet_main_t *om = &odp_packet_main; + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); + odp_packet_if_t *oif = pool_elt_at_index (om->interfaces, hw->dev_instance); + + if (node_index == ~0) + { + oif->per_interface_next_index = node_index; + return; + } + + oif->per_interface_next_index = + vlib_node_add_next (vlib_get_main (), odp_packet_input_node.index, + node_index); + +} + +static void +odp_packet_clear_hw_interface_counters (u32 instance) +{ + /* Nothing for now */ +} + +static clib_error_t * +odp_packet_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, + u32 flags) +{ + odp_packet_main_t *om = &odp_packet_main; + vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); + odp_packet_if_t *oif = + pool_elt_at_index (om->interfaces, hw->dev_instance); + u32 hw_flags; + + oif->is_admin_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0; + + if (oif->is_admin_up) + hw_flags = VNET_HW_INTERFACE_FLAG_LINK_UP; + else + hw_flags = 0; + + vnet_hw_interface_set_flags (vnm, hw_if_index, hw_flags); + + return 0; +} + +static clib_error_t * +odp_packet_subif_add_del_function (vnet_main_t * vnm, + u32 hw_if_index, + struct vnet_sw_interface_t *st, int is_add) +{ +/* Nothing for now */ + return 0; +} + +/* *INDENT-OFF* */ +VNET_DEVICE_CLASS (odp_packet_device_class) = { + .name = "odp-packet", + .tx_function = odp_packet_interface_tx, + .format_device_name = format_odp_packet_device_name, + .format_device = format_odp_packet_device, + .format_tx_trace = format_odp_packet_tx_trace, + .tx_function_n_errors = ODP_PACKET_TX_N_ERROR, + .tx_function_error_strings = odp_packet_tx_func_error_strings, + .rx_redirect_to_node = odp_packet_set_interface_next_node, + .clear_counters = odp_packet_clear_hw_interface_counters, + .admin_up_down_function = odp_packet_interface_admin_up_down, + .subif_add_del_function = odp_packet_subif_add_del_function, +}; + +VLIB_DEVICE_TX_FUNCTION_MULTIARCH (odp_packet_device_class, + odp_packet_interface_tx) +/* *INDENT-ON* */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/vpp/vnet/vnet/devices/odp/node.c b/vpp/vnet/vnet/devices/odp/node.c new file mode 100755 index 00000000..e736e5f6 --- /dev/null +++ b/vpp/vnet/vnet/devices/odp/node.c @@ -0,0 +1,334 @@ +/* + *------------------------------------------------------------------ + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ + +#include <linux/if_packet.h> +#include <vlib/vlib.h> +#include <vlib/unix/unix.h> +#include <vnet/ip/ip.h> +#include <vnet/ethernet/ethernet.h> +#include <vnet/devices/devices.h> +#include <vnet/feature/feature.h> +#include <vnet/devices/odp/odp_packet.h> + +#define foreach_odp_packet_input_error + +typedef enum +{ +#define _(f,s) ODP_PACKET_INPUT_ERROR_##f, + foreach_odp_packet_input_error +#undef _ + ODP_PACKET_INPUT_N_ERROR, +} odp_packet_input_error_t; + +static char *odp_packet_input_error_strings[] = { +#define _(n,s) s, + foreach_odp_packet_input_error +#undef _ +}; + +typedef struct +{ + u32 next_index; + u32 hw_if_index; + odp_packet_t pkt; +} odp_packet_input_trace_t; + +static u8 * +format_odp_packet_input_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + odp_packet_input_trace_t *t = va_arg (*args, odp_packet_input_trace_t *); + + s = format (s, "odp_packet: hw_if_index %d next-index %d", + t->hw_if_index, t->next_index); + + return s; +} + +always_inline void +buffer_add_to_chain (vlib_main_t * vm, u32 bi, u32 first_bi, u32 prev_bi) +{ + vlib_buffer_t *b = vlib_get_buffer (vm, bi); + vlib_buffer_t *first_b = vlib_get_buffer (vm, first_bi); + vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_bi); + + /* update first buffer */ + first_b->total_length_not_including_first_buffer += b->current_length; + + /* update previous buffer */ + prev_b->next_buffer = bi; + prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT; + + /* update current buffer */ + b->next_buffer = 0; + + +} + +always_inline int +odp_packet_queue_mode (odp_pktio_t pktio, u32 mode, odp_packet_t pkt_tbl[]) +{ + u32 num_evts = 0, num_pkts = 0, i = 0; + odp_queue_t inq; + odp_event_t evt_tbl[VLIB_FRAME_SIZE]; + u64 sched_wait = odp_schedule_wait_time(ODP_TIME_MSEC_IN_NS * 100); + + if (pktio == ODP_PKTIO_INVALID) + { + clib_warning("odp_pktio_lookup() failed"); + return -1; + } + + inq = ODP_QUEUE_INVALID; + if ((mode == APPL_MODE_PKT_QUEUE) && + (odp_pktin_event_queue(pktio, &inq, 1) != 1)) + { + clib_warning("Error:no input queue"); + return -1; + } + + if (inq != ODP_QUEUE_INVALID) + num_evts = odp_queue_deq_multi(inq, evt_tbl, VLIB_FRAME_SIZE); + else + num_evts = odp_schedule_multi(NULL, sched_wait, evt_tbl, VLIB_FRAME_SIZE); + + /* convert events to packets, discarding any non-packet events */ + for (i = 0; i < num_evts; ++i) + { + if (odp_event_type(evt_tbl[i]) == ODP_EVENT_PACKET) + pkt_tbl[num_pkts++] = odp_packet_from_event(evt_tbl[i]); + else + odp_event_free(evt_tbl[i]); + } + + return num_pkts; + +} + +always_inline int +odp_packet_burst_mode (odp_pktio_t pktio, odp_pktin_queue_t pktin, odp_packet_t pkt_tbl[]) +{ + u32 num_pkts; + + if (odp_pktin_queue(pktio, &pktin, 1) != 1) + { + clib_warning("odp_pktio_open() failed: no pktin queue"); + return -1; + } + + num_pkts = odp_pktin_recv(pktin, pkt_tbl, VLIB_FRAME_SIZE); + + return num_pkts; + +} + +always_inline uword +odp_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * frame, odp_packet_if_t *oif) +{ + u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; + uword n_trace = vlib_get_trace_count (vm, node); + odp_packet_main_t *om = &odp_packet_main; + u32 n_rx_packets = 0; + u32 n_rx_bytes = 0; + u32 *to_next = 0; + u32 n_free_bufs; + u32 cpu_index = os_get_cpu_number(); + odp_pktin_queue_t pktin = { 0 }; + odp_packet_t pkt,pkt_tbl[VLIB_FRAME_SIZE]; + u32 pkts = 0, pkts_ok = 0; + u32 n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, + VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); + + if (oif->per_interface_next_index != ~0) + next_index = oif->per_interface_next_index; + + n_free_bufs = vec_len (om->rx_buffers[cpu_index]); + if (PREDICT_FALSE (n_free_bufs < VLIB_FRAME_SIZE)) + { + vec_validate (om->rx_buffers[cpu_index], + VLIB_FRAME_SIZE + n_free_bufs - 1); + n_free_bufs += + vlib_buffer_alloc (vm, &om->rx_buffers[cpu_index][n_free_bufs], + VLIB_FRAME_SIZE); + _vec_len (om->rx_buffers[cpu_index]) = n_free_bufs; + + } + + if ((oif->mode ==( APPL_MODE_PKT_QUEUE)) || + (oif->mode ==(APPL_MODE_PKT_SCHED))) + { + pkts = odp_packet_queue_mode(oif->pktio, oif->mode, pkt_tbl); + } + else + { + pkts = odp_packet_burst_mode(oif->pktio, pktin, pkt_tbl); + } + + if (pkts > 0) + { + u32 n_left_to_next,i = 0; + u32 next0 = next_index; + pkts_ok = drop_err_pkts(pkt_tbl, pkts); + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while((i < pkts_ok) && (n_left_to_next) && (n_free_bufs)) + { + vlib_buffer_t *first_b0 = 0; + u32 offset = 0; + u32 bi0 = 0, first_bi0 = 0, prev_bi0; + uint8_t *data_buf; + pkt = pkt_tbl[i]; + u32 data_len = odp_packet_len(pkt); + data_buf = malloc(data_len); + memset(data_buf, 0, data_len); + odp_packet_copy_to_mem(pkt, 0, data_len, data_buf); + + while (data_len && n_free_bufs) + { + vlib_buffer_t *b0; + /* grab free buffer */ + u32 last_empty_buffer = + vec_len (om->rx_buffers[cpu_index]) - 1; + prev_bi0 = bi0; + bi0 = om->rx_buffers[cpu_index][last_empty_buffer]; + b0 = vlib_get_buffer (vm, bi0); + _vec_len (om->rx_buffers[cpu_index]) = last_empty_buffer; + n_free_bufs--; + /* copy data */ + u32 bytes_to_copy = + data_len > n_buffer_bytes ? n_buffer_bytes : data_len; + b0->current_data = 0; + clib_memcpy (vlib_buffer_get_current (b0), + (u8 *) data_buf + offset, + bytes_to_copy); + + /* fill buffer header */ + b0->current_length = bytes_to_copy; + + if (offset == 0) + { + b0->total_length_not_including_first_buffer = 0; + b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; + vnet_buffer (b0)->sw_if_index[VLIB_RX] = + oif->sw_if_index; + vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; + first_bi0 = bi0; + first_b0 = vlib_get_buffer (vm, first_bi0); + } + else + { + buffer_add_to_chain (vm, bi0, first_bi0, prev_bi0); + } + + offset += bytes_to_copy; + data_len -= bytes_to_copy; + } + /* trace */ + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (first_b0); + if (PREDICT_FALSE (n_trace > 0)) + { + odp_packet_input_trace_t *tr; + vlib_trace_buffer (vm, node, next0, first_b0, 0); + vlib_set_trace_count (vm, node, --n_trace); + tr = vlib_add_trace (vm, node, first_b0, sizeof (*tr)); + tr->next_index = next0; + tr->hw_if_index = oif->hw_if_index; + } + + /* redirect if feature path enabled */ + vnet_feature_start_device_input_x1 (oif->sw_if_index, &next0, + first_b0, 0); + + /* enque and take next packet */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, + n_left_to_next, first_bi0, + next0); + + /* next packet */ + n_rx_packets++; + n_rx_bytes += odp_packet_len(pkt); + to_next[0] = first_bi0; + to_next += 1; + n_left_to_next--; + free(data_buf); + odp_packet_free(pkt_tbl[i]); + i++; + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + + } + + vlib_increment_combined_counter(vnet_get_main ()->interface_main.combined_sw_if_counters + + VNET_INTERFACE_COUNTER_RX, + os_get_cpu_number (), oif->hw_if_index, n_rx_packets, n_rx_bytes); + + return n_rx_packets; + +} + +static uword +odp_packet_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * frame) +{ + + int i; + u32 n_rx_packets = 0; + u32 cpu_index = os_get_cpu_number (); + odp_packet_main_t *om = &odp_packet_main; + odp_packet_if_t *oif; + + for (i = 0; i < vec_len (om->interfaces); i++) + { + oif = vec_elt_at_index (om->interfaces, i); + + if (oif->is_admin_up && + (i % om->input_cpu_count) == + (cpu_index - om->input_cpu_first_index)) + { + n_rx_packets += odp_packet_device_input_fn (vm, node, frame, oif); + } + } + + return n_rx_packets; +} + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (odp_packet_input_node) = { + .function = odp_packet_input_fn, + .name = "odp-packet-input", + .sibling_of = "device-input", + .format_trace = format_odp_packet_input_trace, + .type = VLIB_NODE_TYPE_INPUT, + .state = VLIB_NODE_STATE_DISABLED, + .n_errors = ODP_PACKET_INPUT_N_ERROR, + .error_strings = odp_packet_input_error_strings, +}; + +VLIB_NODE_FUNCTION_MULTIARCH (odp_packet_input_node, odp_packet_input_fn) +/* *INDENT-ON* */ + + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/vpp/vnet/vnet/devices/odp/odp_packet.c b/vpp/vnet/vnet/devices/odp/odp_packet.c new file mode 100755 index 00000000..5e092dc6 --- /dev/null +++ b/vpp/vnet/vnet/devices/odp/odp_packet.c @@ -0,0 +1,354 @@ +/* + *------------------------------------------------------------------ + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ + +#include <linux/if_ether.h> +#include <linux/if_packet.h> +#include <vlib/vlib.h> +#include <vlib/unix/unix.h> +#include <vnet/ip/ip.h> +#include <vnet/ethernet/ethernet.h> +#include <vnet/devices/odp/odp_packet.h> + +static u32 +odp_packet_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, + u32 flags) +{ + /* nothing for now */ + return 0; +} + +/** + * Drop packets which input parsing marked as containing errors. + * + * Frees packets with error and modifies pkt_tbl[] to only contain packets with + * no detected errors. + * + * @param pkt_tbl Array of packet + * @param len Length of pkt_tbl[] + * + * @return Number of packets with no detected error + */ +u32 +drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len) +{ + odp_packet_t pkt; + unsigned pkt_cnt = len; + unsigned i, j; + + for (i = 0, j = 0; i < len; ++i) + { + pkt = pkt_tbl[i]; + + if (odp_unlikely(odp_packet_has_error(pkt))) + { + odp_packet_free(pkt); /* Drop */ + pkt_cnt--; + } + else if (odp_unlikely(i != j++)) + { + pkt_tbl[j-1] = pkt; + } + } + + return pkt_cnt; +} + +static odp_pktio_t +create_pktio(const char *dev, odp_pool_t pool, u32 mode) +{ + odp_pktio_t pktio; + int ret; + odp_pktio_param_t pktio_param; + odp_pktin_queue_param_t pktin_param; + + odp_pktio_param_init(&pktio_param); + + switch(mode) + { + case APPL_MODE_PKT_BURST: + pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT; + break; + case APPL_MODE_PKT_QUEUE: + pktio_param.in_mode = ODP_PKTIN_MODE_QUEUE; + break; + case APPL_MODE_PKT_SCHED: + pktio_param.in_mode = ODP_PKTIN_MODE_SCHED; + break; + default: + clib_warning("Invalid mode\n"); + } + + /* Open a packet IO instance */ + pktio = odp_pktio_open(dev, pool, &pktio_param); + + if (pktio == ODP_PKTIO_INVALID) + { + clib_warning("Error: pktio create failed for %s",dev); + } + + odp_pktin_queue_param_init(&pktin_param); + + if (mode == APPL_MODE_PKT_SCHED) + pktin_param.queue_param.sched.sync = ODP_SCHED_SYNC_ATOMIC; + + if (odp_pktin_queue_config(pktio, &pktin_param)) + { + clib_warning("Error: pktin config failed"); + } + + if (odp_pktout_queue_config(pktio, NULL)) + { + clib_warning("Error: pktout config failed"); + } + + ret = odp_pktio_start(pktio); + if (ret != 0) + { + clib_warning("Error: unable to start"); + } + + return pktio; +} + +int +odp_worker_thread_enable () +{ + + /*If worker threads are enabled, switch to polling mode */ + foreach_vlib_main (( + { + vlib_node_set_state (this_vlib_main, + odp_packet_input_node.index, + VLIB_NODE_STATE_POLLING); + })); + return 0; +} + +int +odp_worker_thread_disable () +{ + foreach_vlib_main (( + { + vlib_node_set_state (this_vlib_main, + odp_packet_input_node.index, + VLIB_NODE_STATE_DISABLED); + })); + + return 0; +} + +u32 +odp_packet_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set, + u32 * sw_if_index , u32 mode) +{ + odp_packet_main_t *om = &odp_packet_main; + int ret = 0; + odp_packet_if_t *oif = 0; + u8 hw_addr[6]; + clib_error_t *error = 0; + vnet_sw_interface_t *sw; + vnet_main_t *vnm = vnet_get_main (); + uword *p; + u8 *host_if_name_dup=vec_dup(host_if_name); + vlib_thread_main_t *tm= vlib_get_thread_main(); + + p = mhash_get (&om->if_index_by_host_if_name, host_if_name); + if (p) + return VNET_API_ERROR_SUBIF_ALREADY_EXISTS; + + pool_get (om->interfaces, oif); + oif->if_index = oif - om->interfaces; + oif->host_if_name = host_if_name_dup; + oif->per_interface_next_index= ~0; + + /* Create a pktio instance */ + oif->pktio=create_pktio((char*)host_if_name, om->pool, mode); + oif->mode=mode; + om->if_count++; + + if (tm->n_vlib_mains > 1) + { + oif->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, + CLIB_CACHE_LINE_BYTES); + memset ((void *) oif->lockp, 0, CLIB_CACHE_LINE_BYTES); + } + + /*use configured or generate random MAC address */ + if (hw_addr_set) + clib_memcpy (hw_addr, hw_addr_set, 6); + else + { + f64 now = vlib_time_now (vm); + u32 rnd; + rnd = (u32) (now * 1e6); + rnd = random_u32 (&rnd); + + clib_memcpy (hw_addr + 2, &rnd, sizeof (rnd)); + hw_addr[0] = 2; + hw_addr[1] = 0xfe; + } + + error = ethernet_register_interface (vnm, odp_packet_device_class.index, + oif->if_index, hw_addr, &oif->hw_if_index, odp_packet_eth_flag_change); + + if (error) + { + memset (oif, 0, sizeof (*oif)); + pool_put (om->interfaces, oif); + clib_error_report (error); + ret = VNET_API_ERROR_SYSCALL_ERROR_1; + goto error; + } + + sw = vnet_get_hw_sw_interface (vnm, oif->hw_if_index); + oif->sw_if_index = sw->sw_if_index; + + vnet_hw_interface_set_flags (vnm, oif->hw_if_index, + VNET_HW_INTERFACE_FLAG_LINK_UP); + + mhash_set_mem (&om->if_index_by_host_if_name, host_if_name_dup, &oif->if_index, + 0); + if (sw_if_index) + *sw_if_index = oif->sw_if_index; + + if (tm->n_vlib_mains > 1 && pool_elts (om->interfaces) == 1) + { + /*Fixme :Workers support commented for now as vlib_buffer not thread safe*/ + //odp_worker_thread_enable (); + } + else + { + vlib_node_set_state (vm, odp_packet_input_node.index, + VLIB_NODE_STATE_POLLING); + } + return 0; + + error: + vec_free (host_if_name_dup); + + return ret; +} + +u32 +odp_packet_delete_if (vlib_main_t * vm, u8 * host_if_name) +{ + vnet_main_t *vnm = vnet_get_main (); + odp_packet_main_t *om = &odp_packet_main; + odp_packet_if_t *oif = 0; + uword *p; + vlib_thread_main_t *tm= vlib_get_thread_main(); + + p = mhash_get (&om->if_index_by_host_if_name, host_if_name); + + if (p == NULL) + { + clib_warning ("Host interface %s does not exist", host_if_name); + return VNET_API_ERROR_SYSCALL_ERROR_1; + } + + oif = pool_elt_at_index (om->interfaces, p[0]); + vnet_hw_interface_set_flags (vnm, oif->hw_if_index, 0); + + om->if_count--; + + odp_pktio_stop(odp_pktio_lookup((char*)host_if_name)); + odp_pktio_close(odp_pktio_lookup((char*)host_if_name)); + + vec_free (oif->host_if_name); + oif->host_if_name = NULL; + + mhash_unset (&om->if_index_by_host_if_name, host_if_name, &oif->if_index); + ethernet_delete_interface (vnm, oif->hw_if_index); + + pool_put(om->interfaces, oif); + + if (tm->n_vlib_mains > 1 && pool_elts (om->interfaces) == 0) + { + odp_pool_destroy(om->pool); + /*Fixme :Workers support commented for now*/ + // odp_worker_thread_disable (); + } + + return 0; + +} + +static clib_error_t * +odp_packet_init (vlib_main_t * vm) +{ + odp_packet_main_t *om = &odp_packet_main; + vlib_thread_main_t *tm = vlib_get_thread_main (); + vlib_thread_registration_t *tr; + uword *p; + odp_platform_init_t platform_params; + odp_pool_param_t params; + + memset(om, 0, sizeof(odp_packet_main_t)); + om->input_cpu_first_index = 0; + om->input_cpu_count = 1; + om->if_count = 0; + memset(&platform_params, 0, sizeof(platform_params)); + + if (odp_init_global(&om->instance, NULL, &platform_params)) + clib_warning("Error:ODP global init failed"); + + if (odp_init_local(om->instance, ODP_THREAD_CONTROL) != 0) + { + clib_warning("Error: ODP local init failed"); + odp_term_global(om->instance); + + } + /* Create packet pool */ + odp_pool_param_init(¶ms); + params.pkt.seg_len = SHM_PKT_POOL_BUF_SIZE; + params.pkt.len = SHM_PKT_POOL_BUF_SIZE; + params.type = ODP_POOL_PACKET; + params.pkt.num = SHM_PKT_POOL_NB_PKTS; + + om->pool = odp_pool_create(SHM_PKT_POOL_NAME, ¶ms); + + if (om->pool == ODP_POOL_INVALID) + { + clib_warning("Error: packet pool create failed"); + } + + /* find out which cpus will be used for input */ + p = hash_get_mem (tm->thread_registrations_by_name, "workers"); + tr = p ? (vlib_thread_registration_t *) p[0] : 0; + + if (tr && tr->count > 0) + { + om->input_cpu_first_index = tr->first_index; + om->input_cpu_count = tr->count; + } + + mhash_init_vec_string (&om->if_index_by_host_if_name, sizeof (uword)); + + vec_validate_aligned (om->rx_buffers, tm->n_vlib_mains - 1, + CLIB_CACHE_LINE_BYTES); + return 0; +} + +VLIB_INIT_FUNCTION (odp_packet_init); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/vpp/vnet/vnet/devices/odp/odp_packet.h b/vpp/vnet/vnet/devices/odp/odp_packet.h new file mode 100755 index 00000000..1d8e5e5b --- /dev/null +++ b/vpp/vnet/vnet/devices/odp/odp_packet.h @@ -0,0 +1,74 @@ +/* + *------------------------------------------------------------------ + * Copyright (c) 2016 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ + +#include <odp_api.h> + +#define SHM_PKT_POOL_BUF_SIZE 1856 +#define SHM_PKT_POOL_NB_PKTS 10240 +#define SHM_PKT_POOL_NAME "packet_pool" +#define APPL_MODE_PKT_BURST 0 +#define APPL_MODE_PKT_QUEUE 1 +#define APPL_MODE_PKT_SCHED 2 + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + u8 *host_if_name; + volatile u32 *lockp; + uword if_index; + odp_pktio_t pktio; + u32 hw_if_index; + u32 sw_if_index; + u32 next_rx_frame; + u32 next_tx_frame; + u32 per_interface_next_index; + u8 is_admin_up; + u32 mode; +} odp_packet_if_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + odp_packet_if_t *interfaces; + /* rx buffer cache */ + u32 **rx_buffers; + u32 input_cpu_first_index; + u32 input_cpu_count; + /* hash of host interface names */ + mhash_t if_index_by_host_if_name; + odp_instance_t instance; + odp_pool_t pool; + u32 if_count; +} odp_packet_main_t; + +odp_packet_main_t odp_packet_main; +extern vnet_device_class_t odp_packet_device_class; +extern vlib_node_registration_t odp_packet_input_node; + +u32 odp_packet_create_if (vlib_main_t * vm, u8 * host_if_name, + u8 * hw_addr_set, u32 * sw_if_index, u32 mode); +u32 odp_packet_delete_if (vlib_main_t * vm, u8 * host_if_name); + +u32 drop_err_pkts(odp_packet_t pkt_tbl[], u32 len); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/vpp/vpp/Makefile.am b/vpp/vpp/Makefile.am index 9ae06055..2d0c2bb9 100644 --- a/vpp/vpp/Makefile.am +++ b/vpp/vpp/Makefile.am @@ -100,6 +100,12 @@ vpp_LDADD += -lvnet vpp_LDADD += -lsvm -lsvmdb -lrt +if WITH_ODPLIB +vpp_LDFLAGS = -Wl,--whole-archive +vpp_LDADD += $(ODP_LIBS) +vpp_LDFLAGS += -Wl,--no-whole-archive +endif + vpp_LDADD += -lvnetplugin if WITH_DPDK diff --git a/vpp/vpp/configure.ac b/vpp/vpp/configure.ac index d36695cc..e1a5d7df 100644 --- a/vpp/vpp/configure.ac +++ b/vpp/vpp/configure.ac @@ -34,12 +34,17 @@ AC_ARG_WITH(ipv6sr, [with_ipv6sr=0], [with_ipv6sr=1]) +AC_ARG_WITH(odplib, + AC_HELP_STRING([--with-odplib],[Use odp]), + [with_odplib=1], + [with_odplib=0]) + AM_CONDITIONAL(WITH_DPDK, test "$with_dpdk" = "1") AM_CONDITIONAL(ENABLE_DPDK_SHARED, test "$enable_dpdk_shared" = "1") AC_SUBST(DPDK,["-DDPDK=${with_dpdk} -DDPDK_SHARED_LIB=${enable_dpdk_shared}"]) AM_CONDITIONAL(WITH_DPDK_CRYPTO, test "$with_dpdk_crypto" = "1") AC_SUBST(DPDK_CRYPTO,[-DDPDK_CRYPTO=${with_dpdk_crypto}]) - +AM_CONDITIONAL(WITH_ODPLIB, test "$with_odplib" = "1") AM_COND_IF( [ENABLE_DPDK_SHARED], |