diff options
Diffstat (limited to 'vnet/vnet/devices/dpdk')
| -rw-r--r-- | vnet/vnet/devices/dpdk/cli.c | 1296 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/device.c | 840 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/dpdk.h | 534 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/dpdk_priv.h | 132 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/format.c | 763 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/hqos.c | 775 | ||||
| -rwxr-xr-x | vnet/vnet/devices/dpdk/init.c | 1803 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/ipsec/cli.c | 141 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/ipsec/crypto_node.c | 210 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/ipsec/dir.dox | 18 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/ipsec/dpdk_crypto_ipsec_doc.md | 73 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/ipsec/esp.h | 295 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/ipsec/esp_decrypt.c | 583 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/ipsec/esp_encrypt.c | 598 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/ipsec/ipsec.c | 313 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/ipsec/ipsec.h | 227 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/node.c | 687 | ||||
| -rw-r--r-- | vnet/vnet/devices/dpdk/qos_doc.md | 404 |
18 files changed, 0 insertions, 9692 deletions
diff --git a/vnet/vnet/devices/dpdk/cli.c b/vnet/vnet/devices/dpdk/cli.c deleted file mode 100644 index 538a00fd975..00000000000 --- a/vnet/vnet/devices/dpdk/cli.c +++ /dev/null @@ -1,1296 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <vnet/vnet.h> -#include <vppinfra/vec.h> -#include <vppinfra/error.h> -#include <vppinfra/format.h> -#include <vppinfra/xxhash.h> - -#include <vnet/ethernet/ethernet.h> -#include <vnet/devices/dpdk/dpdk.h> -#include <vnet/classify/vnet_classify.h> -#include <vnet/mpls/packet.h> - -#include "dpdk_priv.h" - -static clib_error_t * -pcap_trace_command_fn (vlib_main_t * vm, - unformat_input_t * input, vlib_cli_command_t * cmd) -{ - dpdk_main_t *dm = &dpdk_main; - u8 *filename; - u32 max; - int matched = 0; - clib_error_t *error = 0; - - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "on")) - { - if (dm->tx_pcap_enable == 0) - { - if (dm->pcap_filename == 0) - dm->pcap_filename = format (0, "/tmp/vpe.pcap%c", 0); - - memset (&dm->pcap_main, 0, sizeof (dm->pcap_main)); - dm->pcap_main.file_name = (char *) dm->pcap_filename; - dm->pcap_main.n_packets_to_capture = 100; - if (dm->pcap_pkts_to_capture) - dm->pcap_main.n_packets_to_capture = dm->pcap_pkts_to_capture; - - dm->pcap_main.packet_type = PCAP_PACKET_TYPE_ethernet; - dm->tx_pcap_enable = 1; - matched = 1; - vlib_cli_output (vm, "pcap tx capture on..."); - } - else - { - vlib_cli_output (vm, "pcap tx capture already on..."); - } - matched = 1; - } - else if (unformat (input, "off")) - { - if (dm->tx_pcap_enable) - { - vlib_cli_output (vm, "captured %d pkts...", - dm->pcap_main.n_packets_captured + 1); - if (dm->pcap_main.n_packets_captured) - { - dm->pcap_main.n_packets_to_capture = - dm->pcap_main.n_packets_captured; - error = pcap_write (&dm->pcap_main); - if (error) - clib_error_report (error); - else - vlib_cli_output (vm, "saved to %s...", dm->pcap_filename); - } - } - else - { - vlib_cli_output (vm, "pcap tx capture already off..."); - } - - dm->tx_pcap_enable = 0; - matched = 1; - } - else if (unformat (input, "max %d", &max)) - { - dm->pcap_pkts_to_capture = max; - matched = 1; - } - - else if (unformat (input, "intfc %U", - unformat_vnet_sw_interface, dm->vnet_main, - &dm->pcap_sw_if_index)) - matched = 1; - else if (unformat (input, "intfc any")) - { - dm->pcap_sw_if_index = 0; - matched = 1; - } - else if (unformat (input, "file %s", &filename)) - { - u8 *chroot_filename; - /* Brain-police user path input */ - if (strstr ((char *) filename, "..") - || index ((char *) filename, '/')) - { - vlib_cli_output (vm, "illegal characters in filename '%s'", - filename); - continue; - } - - chroot_filename = format (0, "/tmp/%s%c", filename, 0); - vec_free (filename); - - if (dm->pcap_filename) - vec_free (dm->pcap_filename); - vec_add1 (filename, 0); - dm->pcap_filename = chroot_filename; - matched = 1; - } - else if (unformat (input, "status")) - { - if (dm->tx_pcap_enable == 0) - { - vlib_cli_output (vm, "pcap tx capture is off..."); - continue; - } - - vlib_cli_output (vm, "pcap tx capture: %d of %d pkts...", - dm->pcap_main.n_packets_captured, - dm->pcap_main.n_packets_to_capture); - matched = 1; - } - - else - break; - } - - if (matched == 0) - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); - - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (pcap_trace_command, static) = { - .path = "pcap tx trace", - .short_help = - "pcap tx trace on off max <nn> intfc <intfc> file <name> status", - .function = pcap_trace_command_fn, -}; -/* *INDENT-ON* */ - - -static clib_error_t * -show_dpdk_buffer (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - struct rte_mempool *rmp; - int i; - - for (i = 0; i < vec_len (vm->buffer_main->pktmbuf_pools); i++) - { - rmp = vm->buffer_main->pktmbuf_pools[i]; - if (rmp) - { - unsigned count = rte_mempool_avail_count (rmp); - unsigned free_count = rte_mempool_in_use_count (rmp); - - vlib_cli_output (vm, - "name=\"%s\" available = %7d allocated = %7d total = %7d\n", - rmp->name, (u32) count, (u32) free_count, - (u32) (count + free_count)); - } - else - { - vlib_cli_output (vm, "rte_mempool is NULL (!)\n"); - } - } - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_show_dpdk_bufferr,static) = { - .path = "show dpdk buffer", - .short_help = "show dpdk buffer state", - .function = show_dpdk_buffer, - .is_mp_safe = 1, -}; -/* *INDENT-ON* */ - -static clib_error_t * -test_dpdk_buffer (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - static u32 *allocated_buffers; - u32 n_alloc = 0; - u32 n_free = 0; - u32 first, actual_alloc; - - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "allocate %d", &n_alloc)) - ; - else if (unformat (input, "free %d", &n_free)) - ; - else - break; - } - - if (n_free) - { - if (vec_len (allocated_buffers) < n_free) - return clib_error_return (0, "Can't free %d, only %d allocated", - n_free, vec_len (allocated_buffers)); - - first = vec_len (allocated_buffers) - n_free; - vlib_buffer_free (vm, allocated_buffers + first, n_free); - _vec_len (allocated_buffers) = first; - } - if (n_alloc) - { - first = vec_len (allocated_buffers); - vec_validate (allocated_buffers, - vec_len (allocated_buffers) + n_alloc - 1); - - actual_alloc = vlib_buffer_alloc (vm, allocated_buffers + first, - n_alloc); - _vec_len (allocated_buffers) = first + actual_alloc; - - if (actual_alloc < n_alloc) - vlib_cli_output (vm, "WARNING: only allocated %d buffers", - actual_alloc); - } - - vlib_cli_output (vm, "Currently %d buffers allocated", - vec_len (allocated_buffers)); - - if (allocated_buffers && vec_len (allocated_buffers) == 0) - vec_free (allocated_buffers); - - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_test_dpdk_buffer,static) = { - .path = "test dpdk buffer", - .short_help = "test dpdk buffer [allocate <nn>][free <nn>]", - .function = test_dpdk_buffer, - .is_mp_safe = 1, -}; -/* *INDENT-ON* */ - -static clib_error_t * -set_dpdk_if_desc (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - dpdk_main_t *dm = &dpdk_main; - vnet_hw_interface_t *hw; - dpdk_device_t *xd; - u32 hw_if_index = (u32) ~ 0; - u32 nb_rx_desc = (u32) ~ 0; - u32 nb_tx_desc = (u32) ~ 0; - clib_error_t *rv; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat - (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main, - &hw_if_index)) - ; - else if (unformat (line_input, "tx %d", &nb_tx_desc)) - ; - else if (unformat (line_input, "rx %d", &nb_rx_desc)) - ; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - - unformat_free (line_input); - - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); - - hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); - xd = vec_elt_at_index (dm->devices, hw->dev_instance); - - if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0) - return clib_error_return (0, "number of descriptors can be set only for " - "physical devices"); - - if ((nb_rx_desc == (u32) ~ 0 || nb_rx_desc == xd->nb_rx_desc) && - (nb_tx_desc == (u32) ~ 0 || nb_tx_desc == xd->nb_tx_desc)) - return clib_error_return (0, "nothing changed"); - - if (nb_rx_desc != (u32) ~ 0) - xd->nb_rx_desc = nb_rx_desc; - - if (nb_tx_desc != (u32) ~ 0) - xd->nb_tx_desc = nb_tx_desc; - - rv = dpdk_port_setup (dm, xd); - - return rv; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_set_dpdk_if_desc,static) = { - .path = "set dpdk interface descriptors", - .short_help = "set dpdk interface descriptors <if-name> [rx <n>] [tx <n>]", - .function = set_dpdk_if_desc, -}; -/* *INDENT-ON* */ - -static clib_error_t * -show_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - vlib_thread_main_t *tm = vlib_get_thread_main (); - dpdk_main_t *dm = &dpdk_main; - dpdk_device_and_queue_t *dq; - int cpu; - - if (tm->n_vlib_mains == 1) - vlib_cli_output (vm, "All interfaces are handled by main thread"); - - for (cpu = 0; cpu < vec_len (dm->devices_by_cpu); cpu++) - { - if (vec_len (dm->devices_by_cpu[cpu])) - vlib_cli_output (vm, "Thread %u (%s at lcore %u):", cpu, - vlib_worker_threads[cpu].name, - vlib_worker_threads[cpu].lcore_id); - - /* *INDENT-OFF* */ - vec_foreach(dq, dm->devices_by_cpu[cpu]) - { - u32 hw_if_index = dm->devices[dq->device].vlib_hw_if_index; - vnet_hw_interface_t * hi = vnet_get_hw_interface(dm->vnet_main, hw_if_index); - vlib_cli_output(vm, " %v queue %u", hi->name, dq->queue_id); - } - /* *INDENT-ON* */ - } - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_show_dpdk_if_placement,static) = { - .path = "show dpdk interface placement", - .short_help = "show dpdk interface placement", - .function = show_dpdk_if_placement, -}; -/* *INDENT-ON* */ - -static int -dpdk_device_queue_sort (void *a1, void *a2) -{ - dpdk_device_and_queue_t *dq1 = a1; - dpdk_device_and_queue_t *dq2 = a2; - - if (dq1->device > dq2->device) - return 1; - else if (dq1->device < dq2->device) - return -1; - else if (dq1->queue_id > dq2->queue_id) - return 1; - else if (dq1->queue_id < dq2->queue_id) - return -1; - else - return 0; -} - -static clib_error_t * -set_dpdk_if_placement (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - dpdk_main_t *dm = &dpdk_main; - dpdk_device_and_queue_t *dq; - vnet_hw_interface_t *hw; - dpdk_device_t *xd; - u32 hw_if_index = (u32) ~ 0; - u32 queue = (u32) 0; - u32 cpu = (u32) ~ 0; - int i; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat - (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main, - &hw_if_index)) - ; - else if (unformat (line_input, "queue %d", &queue)) - ; - else if (unformat (line_input, "thread %d", &cpu)) - ; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - - unformat_free (line_input); - - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); - - if (cpu < dm->input_cpu_first_index || - cpu >= (dm->input_cpu_first_index + dm->input_cpu_count)) - return clib_error_return (0, "please specify valid thread id"); - - hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); - xd = vec_elt_at_index (dm->devices, hw->dev_instance); - - for (i = 0; i < vec_len (dm->devices_by_cpu); i++) - { - /* *INDENT-OFF* */ - vec_foreach(dq, dm->devices_by_cpu[i]) - { - if (hw_if_index == dm->devices[dq->device].vlib_hw_if_index && - queue == dq->queue_id) - { - if (cpu == i) /* nothing to do */ - return 0; - - vec_del1(dm->devices_by_cpu[i], dq - dm->devices_by_cpu[i]); - vec_add2(dm->devices_by_cpu[cpu], dq, 1); - dq->queue_id = queue; - dq->device = xd->device_index; - xd->cpu_socket_id_by_queue[queue] = - rte_lcore_to_socket_id(vlib_worker_threads[cpu].lcore_id); - - vec_sort_with_function(dm->devices_by_cpu[i], - dpdk_device_queue_sort); - - vec_sort_with_function(dm->devices_by_cpu[cpu], - dpdk_device_queue_sort); - - if (vec_len(dm->devices_by_cpu[i]) == 0) - vlib_node_set_state (vlib_mains[i], dpdk_input_node.index, - VLIB_NODE_STATE_DISABLED); - - if (vec_len(dm->devices_by_cpu[cpu]) == 1) - vlib_node_set_state (vlib_mains[cpu], dpdk_input_node.index, - VLIB_NODE_STATE_POLLING); - - return 0; - } - } - /* *INDENT-ON* */ - } - - return clib_error_return (0, "not found"); -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_set_dpdk_if_placement,static) = { - .path = "set dpdk interface placement", - .short_help = "set dpdk interface placement <if-name> [queue <n>] thread <n>", - .function = set_dpdk_if_placement, -}; -/* *INDENT-ON* */ - -static clib_error_t * -show_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - vlib_thread_main_t *tm = vlib_get_thread_main (); - dpdk_main_t *dm = &dpdk_main; - dpdk_device_and_queue_t *dq; - int cpu; - - if (tm->n_vlib_mains == 1) - vlib_cli_output (vm, "All interfaces are handled by main thread"); - - for (cpu = 0; cpu < vec_len (dm->devices_by_hqos_cpu); cpu++) - { - if (vec_len (dm->devices_by_hqos_cpu[cpu])) - vlib_cli_output (vm, "Thread %u (%s at lcore %u):", cpu, - vlib_worker_threads[cpu].name, - vlib_worker_threads[cpu].lcore_id); - - vec_foreach (dq, dm->devices_by_hqos_cpu[cpu]) - { - u32 hw_if_index = dm->devices[dq->device].vlib_hw_if_index; - vnet_hw_interface_t *hi = - vnet_get_hw_interface (dm->vnet_main, hw_if_index); - vlib_cli_output (vm, " %v queue %u", hi->name, dq->queue_id); - } - } - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_show_dpdk_if_hqos_placement, static) = { - .path = "show dpdk interface hqos placement", - .short_help = "show dpdk interface hqos placement", - .function = show_dpdk_if_hqos_placement, -}; -/* *INDENT-ON* */ - -static clib_error_t * -set_dpdk_if_hqos_placement (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - dpdk_main_t *dm = &dpdk_main; - dpdk_device_and_queue_t *dq; - vnet_hw_interface_t *hw; - dpdk_device_t *xd; - u32 hw_if_index = (u32) ~ 0; - u32 cpu = (u32) ~ 0; - int i; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat - (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main, - &hw_if_index)) - ; - else if (unformat (line_input, "thread %d", &cpu)) - ; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - - unformat_free (line_input); - - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); - - if (cpu < dm->hqos_cpu_first_index || - cpu >= (dm->hqos_cpu_first_index + dm->hqos_cpu_count)) - return clib_error_return (0, "please specify valid thread id"); - - hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); - xd = vec_elt_at_index (dm->devices, hw->dev_instance); - - for (i = 0; i < vec_len (dm->devices_by_hqos_cpu); i++) - { - vec_foreach (dq, dm->devices_by_hqos_cpu[i]) - { - if (hw_if_index == dm->devices[dq->device].vlib_hw_if_index) - { - if (cpu == i) /* nothing to do */ - return 0; - - vec_del1 (dm->devices_by_hqos_cpu[i], - dq - dm->devices_by_hqos_cpu[i]); - vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1); - dq->queue_id = 0; - dq->device = xd->device_index; - - vec_sort_with_function (dm->devices_by_hqos_cpu[i], - dpdk_device_queue_sort); - - vec_sort_with_function (dm->devices_by_hqos_cpu[cpu], - dpdk_device_queue_sort); - - return 0; - } - } - } - - return clib_error_return (0, "not found"); -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_placement, static) = { - .path = "set dpdk interface hqos placement", - .short_help = "set dpdk interface hqos placement <if-name> thread <n>", - .function = set_dpdk_if_hqos_placement, -}; -/* *INDENT-ON* */ - -static clib_error_t * -set_dpdk_if_hqos_pipe (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - dpdk_main_t *dm = &dpdk_main; - vnet_hw_interface_t *hw; - dpdk_device_t *xd; - u32 hw_if_index = (u32) ~ 0; - u32 subport_id = (u32) ~ 0; - u32 pipe_id = (u32) ~ 0; - u32 profile_id = (u32) ~ 0; - int rv; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat - (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main, - &hw_if_index)) - ; - else if (unformat (line_input, "subport %d", &subport_id)) - ; - else if (unformat (line_input, "pipe %d", &pipe_id)) - ; - else if (unformat (line_input, "profile %d", &profile_id)) - ; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - - unformat_free (line_input); - - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); - - hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); - xd = vec_elt_at_index (dm->devices, hw->dev_instance); - - rv = - rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id, - profile_id); - if (rv) - return clib_error_return (0, "pipe configuration failed"); - - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_pipe, static) = -{ - .path = "set dpdk interface hqos pipe", - .short_help = "set dpdk interface hqos pipe <if-name> subport <n> pipe <n> " - "profile <n>", - .function = set_dpdk_if_hqos_pipe, -}; -/* *INDENT-ON* */ - -static clib_error_t * -set_dpdk_if_hqos_subport (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - dpdk_main_t *dm = &dpdk_main; - vnet_hw_interface_t *hw; - dpdk_device_t *xd; - u32 hw_if_index = (u32) ~ 0; - u32 subport_id = (u32) ~ 0; - struct rte_sched_subport_params p = { - .tb_rate = 1250000000, /* 10GbE */ - .tb_size = 1000000, - .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000}, - .tc_period = 10, - }; - int rv; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat - (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main, - &hw_if_index)) - ; - else if (unformat (line_input, "subport %d", &subport_id)) - ; - else if (unformat (line_input, "rate %d", &p.tb_rate)) - { - p.tc_rate[0] = p.tb_rate; - p.tc_rate[1] = p.tb_rate; - p.tc_rate[2] = p.tb_rate; - p.tc_rate[3] = p.tb_rate; - } - else if (unformat (line_input, "bktsize %d", &p.tb_size)) - ; - else if (unformat (line_input, "tc0 %d", &p.tc_rate[0])) - ; - else if (unformat (line_input, "tc1 %d", &p.tc_rate[1])) - ; - else if (unformat (line_input, "tc2 %d", &p.tc_rate[2])) - ; - else if (unformat (line_input, "tc3 %d", &p.tc_rate[3])) - ; - else if (unformat (line_input, "period %d", &p.tc_period)) - ; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - - unformat_free (line_input); - - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); - - hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); - xd = vec_elt_at_index (dm->devices, hw->dev_instance); - - rv = rte_sched_subport_config (xd->hqos_ht->hqos, subport_id, &p); - if (rv) - return clib_error_return (0, "subport configuration failed"); - - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_subport, static) = { - .path = "set dpdk interface hqos subport", - .short_help = "set dpdk interface hqos subport <if-name> subport <n> " - "[rate <n>] [bktsize <n>] [tc0 <n>] [tc1 <n>] [tc2 <n>] [tc3 <n>] " - "[period <n>]", - .function = set_dpdk_if_hqos_subport, -}; -/* *INDENT-ON* */ - -static clib_error_t * -set_dpdk_if_hqos_tctbl (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - vlib_thread_main_t *tm = vlib_get_thread_main (); - dpdk_main_t *dm = &dpdk_main; - vnet_hw_interface_t *hw; - dpdk_device_t *xd; - u32 hw_if_index = (u32) ~ 0; - u32 tc = (u32) ~ 0; - u32 queue = (u32) ~ 0; - u32 entry = (u32) ~ 0; - u32 val, i; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat - (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main, - &hw_if_index)) - ; - else if (unformat (line_input, "entry %d", &entry)) - ; - else if (unformat (line_input, "tc %d", &tc)) - ; - else if (unformat (line_input, "queue %d", &queue)) - ; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - - unformat_free (line_input); - - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); - if (entry >= 64) - return clib_error_return (0, "invalid entry"); - if (tc >= RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE) - return clib_error_return (0, "invalid traffic class"); - if (queue >= RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS) - return clib_error_return (0, "invalid traffic class"); - - hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); - xd = vec_elt_at_index (dm->devices, hw->dev_instance); - - /* Detect the set of worker threads */ - uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers"); - /* Should never happen, shut up Coverity warning */ - if (p == 0) - return clib_error_return (0, "no worker registrations?"); - - vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0]; - int worker_thread_first = tr->first_index; - int worker_thread_count = tr->count; - - val = tc * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + queue; - for (i = 0; i < worker_thread_count; i++) - xd->hqos_wt[worker_thread_first + i].hqos_tc_table[entry] = val; - - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_tctbl, static) = { - .path = "set dpdk interface hqos tctbl", - .short_help = "set dpdk interface hqos tctbl <if-name> entry <n> tc <n> queue <n>", - .function = set_dpdk_if_hqos_tctbl, -}; -/* *INDENT-ON* */ - -static clib_error_t * -set_dpdk_if_hqos_pktfield (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - vlib_thread_main_t *tm = vlib_get_thread_main (); - dpdk_main_t *dm = &dpdk_main; - - /* Device specific data */ - struct rte_eth_dev_info dev_info; - dpdk_device_config_t *devconf = 0; - vnet_hw_interface_t *hw; - dpdk_device_t *xd; - u32 hw_if_index = (u32) ~ 0; - - /* Detect the set of worker threads */ - uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers"); - /* Should never happen, shut up Coverity warning */ - if (p == 0) - return clib_error_return (0, "no worker registrations?"); - - vlib_thread_registration_t *tr = (vlib_thread_registration_t *) p[0]; - int worker_thread_first = tr->first_index; - int worker_thread_count = tr->count; - - /* Packet field configuration */ - u64 mask = (u64) ~ 0; - u32 id = (u32) ~ 0; - u32 offset = (u32) ~ 0; - - /* HQoS params */ - u32 n_subports_per_port, n_pipes_per_subport, tctbl_size; - - u32 i; - - /* Parse input arguments */ - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat - (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main, - &hw_if_index)) - ; - else if (unformat (line_input, "id %d", &id)) - ; - else if (unformat (line_input, "offset %d", &offset)) - ; - else if (unformat (line_input, "mask %llx", &mask)) - ; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - - unformat_free (line_input); - - /* Get interface */ - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify valid interface name"); - - hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); - xd = vec_elt_at_index (dm->devices, hw->dev_instance); - - rte_eth_dev_info_get (xd->device_index, &dev_info); - if (dev_info.pci_dev) - { /* bonded interface has no pci info */ - vlib_pci_addr_t pci_addr; - - pci_addr.domain = dev_info.pci_dev->addr.domain; - pci_addr.bus = dev_info.pci_dev->addr.bus; - pci_addr.slot = dev_info.pci_dev->addr.devid; - pci_addr.function = dev_info.pci_dev->addr.function; - - p = - hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32); - } - - if (p) - devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]); - else - devconf = &dm->conf->default_devconf; - - if (devconf->hqos_enabled == 0) - { - vlib_cli_output (vm, "HQoS disabled for this interface"); - return 0; - } - - n_subports_per_port = devconf->hqos.port.n_subports_per_port; - n_pipes_per_subport = devconf->hqos.port.n_pipes_per_subport; - tctbl_size = RTE_DIM (devconf->hqos.tc_table); - - /* Validate packet field configuration: id, offset and mask */ - if (id >= 3) - return clib_error_return (0, "invalid packet field id"); - - switch (id) - { - case 0: - if (dpdk_hqos_validate_mask (mask, n_subports_per_port) != 0) - return clib_error_return (0, "invalid subport ID mask " - "(n_subports_per_port = %u)", - n_subports_per_port); - break; - case 1: - if (dpdk_hqos_validate_mask (mask, n_pipes_per_subport) != 0) - return clib_error_return (0, "invalid pipe ID mask " - "(n_pipes_per_subport = %u)", - n_pipes_per_subport); - break; - case 2: - default: - if (dpdk_hqos_validate_mask (mask, tctbl_size) != 0) - return clib_error_return (0, "invalid TC table index mask " - "(TC table size = %u)", tctbl_size); - } - - /* Propagate packet field configuration to all workers */ - for (i = 0; i < worker_thread_count; i++) - switch (id) - { - case 0: - xd->hqos_wt[worker_thread_first + i].hqos_field0_slabpos = offset; - xd->hqos_wt[worker_thread_first + i].hqos_field0_slabmask = mask; - xd->hqos_wt[worker_thread_first + i].hqos_field0_slabshr = - __builtin_ctzll (mask); - break; - case 1: - xd->hqos_wt[worker_thread_first + i].hqos_field1_slabpos = offset; - xd->hqos_wt[worker_thread_first + i].hqos_field1_slabmask = mask; - xd->hqos_wt[worker_thread_first + i].hqos_field1_slabshr = - __builtin_ctzll (mask); - break; - case 2: - default: - xd->hqos_wt[worker_thread_first + i].hqos_field2_slabpos = offset; - xd->hqos_wt[worker_thread_first + i].hqos_field2_slabmask = mask; - xd->hqos_wt[worker_thread_first + i].hqos_field2_slabshr = - __builtin_ctzll (mask); - } - - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_set_dpdk_if_hqos_pktfield, static) = { - .path = "set dpdk interface hqos pktfield", - .short_help = "set dpdk interface hqos pktfield <if-name> id <n> offset <n> " - "mask <n>", - .function = set_dpdk_if_hqos_pktfield, -}; -/* *INDENT-ON* */ - -static clib_error_t * -show_dpdk_if_hqos (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - vlib_thread_main_t *tm = vlib_get_thread_main (); - dpdk_main_t *dm = &dpdk_main; - vnet_hw_interface_t *hw; - dpdk_device_t *xd; - dpdk_device_config_hqos_t *cfg; - dpdk_device_hqos_per_hqos_thread_t *ht; - dpdk_device_hqos_per_worker_thread_t *wk; - u32 *tctbl; - u32 hw_if_index = (u32) ~ 0; - u32 profile_id, i; - struct rte_eth_dev_info dev_info; - dpdk_device_config_t *devconf = 0; - vlib_thread_registration_t *tr; - uword *p = 0; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat - (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main, - &hw_if_index)) - ; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - - unformat_free (line_input); - - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify interface name!!"); - - hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); - xd = vec_elt_at_index (dm->devices, hw->dev_instance); - - rte_eth_dev_info_get (xd->device_index, &dev_info); - if (dev_info.pci_dev) - { /* bonded interface has no pci info */ - vlib_pci_addr_t pci_addr; - - pci_addr.domain = dev_info.pci_dev->addr.domain; - pci_addr.bus = dev_info.pci_dev->addr.bus; - pci_addr.slot = dev_info.pci_dev->addr.devid; - pci_addr.function = dev_info.pci_dev->addr.function; - - p = - hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32); - } - - if (p) - devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]); - else - devconf = &dm->conf->default_devconf; - - if (devconf->hqos_enabled == 0) - { - vlib_cli_output (vm, "HQoS disabled for this interface"); - return 0; - } - - /* Detect the set of worker threads */ - p = hash_get_mem (tm->thread_registrations_by_name, "workers"); - - /* Should never happen, shut up Coverity warning */ - if (p == 0) - return clib_error_return (0, "no worker registrations?"); - - tr = (vlib_thread_registration_t *) p[0]; - - cfg = &devconf->hqos; - ht = xd->hqos_ht; - wk = &xd->hqos_wt[tr->first_index]; - tctbl = wk->hqos_tc_table; - - vlib_cli_output (vm, " Thread:"); - vlib_cli_output (vm, " Input SWQ size = %u packets", cfg->swq_size); - vlib_cli_output (vm, " Enqueue burst size = %u packets", - ht->hqos_burst_enq); - vlib_cli_output (vm, " Dequeue burst size = %u packets", - ht->hqos_burst_deq); - - vlib_cli_output (vm, - " Packet field 0: slab position = %4u, slab bitmask = 0x%016llx", - wk->hqos_field0_slabpos, wk->hqos_field0_slabmask); - vlib_cli_output (vm, - " Packet field 1: slab position = %4u, slab bitmask = 0x%016llx", - wk->hqos_field1_slabpos, wk->hqos_field1_slabmask); - vlib_cli_output (vm, - " Packet field 2: slab position = %4u, slab bitmask = 0x%016llx", - wk->hqos_field2_slabpos, wk->hqos_field2_slabmask); - vlib_cli_output (vm, " Packet field 2 translation table:"); - vlib_cli_output (vm, " [ 0 .. 15]: " - "%2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u", - tctbl[0], tctbl[1], tctbl[2], tctbl[3], - tctbl[4], tctbl[5], tctbl[6], tctbl[7], - tctbl[8], tctbl[9], tctbl[10], tctbl[11], - tctbl[12], tctbl[13], tctbl[14], tctbl[15]); - vlib_cli_output (vm, " [16 .. 31]: " - "%2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u", - tctbl[16], tctbl[17], tctbl[18], tctbl[19], - tctbl[20], tctbl[21], tctbl[22], tctbl[23], - tctbl[24], tctbl[25], tctbl[26], tctbl[27], - tctbl[28], tctbl[29], tctbl[30], tctbl[31]); - vlib_cli_output (vm, " [32 .. 47]: " - "%2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u", - tctbl[32], tctbl[33], tctbl[34], tctbl[35], - tctbl[36], tctbl[37], tctbl[38], tctbl[39], - tctbl[40], tctbl[41], tctbl[42], tctbl[43], - tctbl[44], tctbl[45], tctbl[46], tctbl[47]); - vlib_cli_output (vm, " [48 .. 63]: " - "%2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u %2u", - tctbl[48], tctbl[49], tctbl[50], tctbl[51], - tctbl[52], tctbl[53], tctbl[54], tctbl[55], - tctbl[56], tctbl[57], tctbl[58], tctbl[59], - tctbl[60], tctbl[61], tctbl[62], tctbl[63]); - - vlib_cli_output (vm, " Port:"); - vlib_cli_output (vm, " Rate = %u bytes/second", cfg->port.rate); - vlib_cli_output (vm, " MTU = %u bytes", cfg->port.mtu); - vlib_cli_output (vm, " Frame overhead = %u bytes", - cfg->port.frame_overhead); - vlib_cli_output (vm, " Number of subports = %u", - cfg->port.n_subports_per_port); - vlib_cli_output (vm, " Number of pipes per subport = %u", - cfg->port.n_pipes_per_subport); - vlib_cli_output (vm, - " Packet queue size: TC0 = %u, TC1 = %u, TC2 = %u, TC3 = %u packets", - cfg->port.qsize[0], cfg->port.qsize[1], cfg->port.qsize[2], - cfg->port.qsize[3]); - vlib_cli_output (vm, " Number of pipe profiles = %u", - cfg->port.n_pipe_profiles); - - for (profile_id = 0; profile_id < vec_len (cfg->pipe); profile_id++) - { - vlib_cli_output (vm, " Pipe profile %u:", profile_id); - vlib_cli_output (vm, " Rate = %u bytes/second", - cfg->pipe[profile_id].tb_rate); - vlib_cli_output (vm, " Token bucket size = %u bytes", - cfg->pipe[profile_id].tb_size); - vlib_cli_output (vm, - " Traffic class rate: TC0 = %u, TC1 = %u, TC2 = %u, TC3 = %u bytes/second", - cfg->pipe[profile_id].tc_rate[0], - cfg->pipe[profile_id].tc_rate[1], - cfg->pipe[profile_id].tc_rate[2], - cfg->pipe[profile_id].tc_rate[3]); - vlib_cli_output (vm, " TC period = %u milliseconds", - cfg->pipe[profile_id].tc_period); -#ifdef RTE_SCHED_SUBPORT_TC_OV - vlib_cli_output (vm, " TC3 oversubscription_weight = %u", - cfg->pipe[profile_id].tc_ov_weight); -#endif - - for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) - { - vlib_cli_output (vm, - " TC%u WRR weights: Q0 = %u, Q1 = %u, Q2 = %u, Q3 = %u", - i, cfg->pipe[profile_id].wrr_weights[i * 4], - cfg->pipe[profile_id].wrr_weights[i * 4 + 1], - cfg->pipe[profile_id].wrr_weights[i * 4 + 2], - cfg->pipe[profile_id].wrr_weights[i * 4 + 3]); - } - } - -#ifdef RTE_SCHED_RED - vlib_cli_output (vm, " Weighted Random Early Detection (WRED):"); - for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) - { - vlib_cli_output (vm, " TC%u min: G = %u, Y = %u, R = %u", i, - cfg->port.red_params[i][e_RTE_METER_GREEN].min_th, - cfg->port.red_params[i][e_RTE_METER_YELLOW].min_th, - cfg->port.red_params[i][e_RTE_METER_RED].min_th); - - vlib_cli_output (vm, " TC%u max: G = %u, Y = %u, R = %u", i, - cfg->port.red_params[i][e_RTE_METER_GREEN].max_th, - cfg->port.red_params[i][e_RTE_METER_YELLOW].max_th, - cfg->port.red_params[i][e_RTE_METER_RED].max_th); - - vlib_cli_output (vm, - " TC%u inverted probability: G = %u, Y = %u, R = %u", - i, cfg->port.red_params[i][e_RTE_METER_GREEN].maxp_inv, - cfg->port.red_params[i][e_RTE_METER_YELLOW].maxp_inv, - cfg->port.red_params[i][e_RTE_METER_RED].maxp_inv); - - vlib_cli_output (vm, " TC%u weight: R = %u, Y = %u, R = %u", i, - cfg->port.red_params[i][e_RTE_METER_GREEN].wq_log2, - cfg->port.red_params[i][e_RTE_METER_YELLOW].wq_log2, - cfg->port.red_params[i][e_RTE_METER_RED].wq_log2); - } -#endif - - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_show_dpdk_if_hqos, static) = { - .path = "show dpdk interface hqos", - .short_help = "show dpdk interface hqos <if-name>", - .function = show_dpdk_if_hqos, -}; - -/* *INDENT-ON* */ - -static clib_error_t * -show_dpdk_hqos_queue_stats (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - dpdk_main_t *dm = &dpdk_main; - u32 hw_if_index = (u32) ~ 0; - u32 subport = (u32) ~ 0; - u32 pipe = (u32) ~ 0; - u32 tc = (u32) ~ 0; - u32 tc_q = (u32) ~ 0; - vnet_hw_interface_t *hw; - dpdk_device_t *xd; - uword *p = 0; - struct rte_eth_dev_info dev_info; - dpdk_device_config_t *devconf = 0; - u32 qindex; - struct rte_sched_queue_stats stats; - u16 qlen; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat - (line_input, "%U", unformat_vnet_hw_interface, dm->vnet_main, - &hw_if_index)) - ; - - else if (unformat (line_input, "subport %d", &subport)) - ; - - else if (unformat (line_input, "pipe %d", &pipe)) - ; - - else if (unformat (line_input, "tc %d", &tc)) - ; - - else if (unformat (line_input, "tc_q %d", &tc_q)) - ; - - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - - unformat_free (line_input); - - if (hw_if_index == (u32) ~ 0) - return clib_error_return (0, "please specify interface name!!"); - - hw = vnet_get_hw_interface (dm->vnet_main, hw_if_index); - xd = vec_elt_at_index (dm->devices, hw->dev_instance); - - rte_eth_dev_info_get (xd->device_index, &dev_info); - if (dev_info.pci_dev) - { /* bonded interface has no pci info */ - vlib_pci_addr_t pci_addr; - - pci_addr.domain = dev_info.pci_dev->addr.domain; - pci_addr.bus = dev_info.pci_dev->addr.bus; - pci_addr.slot = dev_info.pci_dev->addr.devid; - pci_addr.function = dev_info.pci_dev->addr.function; - - p = - hash_get (dm->conf->device_config_index_by_pci_addr, pci_addr.as_u32); - } - - if (p) - devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]); - else - devconf = &dm->conf->default_devconf; - - if (devconf->hqos_enabled == 0) - { - vlib_cli_output (vm, "HQoS disabled for this interface"); - return 0; - } - - /* - * Figure out which queue to query. cf rte_sched_port_qindex. (Not sure why - * that method isn't made public by DPDK - how _should_ we get the queue ID?) - */ - qindex = subport * devconf->hqos.port.n_pipes_per_subport + pipe; - qindex = qindex * RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE + tc; - qindex = qindex * RTE_SCHED_QUEUES_PER_TRAFFIC_CLASS + tc_q; - - if (rte_sched_queue_read_stats (xd->hqos_ht->hqos, qindex, &stats, &qlen) != - 0) - return clib_error_return (0, "failed to read stats"); - - vlib_cli_output (vm, "%=24s%=16s", "Stats Parameter", "Value"); - vlib_cli_output (vm, "%=24s%=16d", "Packets", stats.n_pkts); - vlib_cli_output (vm, "%=24s%=16d", "Packets dropped", stats.n_pkts_dropped); -#ifdef RTE_SCHED_RED - vlib_cli_output (vm, "%=24s%=16d", "Packets dropped (RED)", - stats.n_pkts_red_dropped); -#endif - vlib_cli_output (vm, "%=24s%=16d", "Bytes", stats.n_bytes); - vlib_cli_output (vm, "%=24s%=16d", "Bytes dropped", stats.n_bytes_dropped); - - - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (cmd_show_dpdk_hqos_queue_stats, static) = { - .path = "show dpdk hqos queue", - .short_help = "show dpdk hqos queue <if-name> subport <subport> pipe <pipe> tc <tc> tc_q <tc_q>", - .function = show_dpdk_hqos_queue_stats, -}; -/* *INDENT-ON* */ - -clib_error_t * -dpdk_cli_init (vlib_main_t * vm) -{ - return 0; -} - -VLIB_INIT_FUNCTION (dpdk_cli_init); - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/device.c b/vnet/vnet/devices/dpdk/device.c deleted file mode 100644 index b22fbf2e69e..00000000000 --- a/vnet/vnet/devices/dpdk/device.c +++ /dev/null @@ -1,840 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <vnet/vnet.h> -#include <vppinfra/vec.h> -#include <vppinfra/format.h> -#include <vlib/unix/cj.h> -#include <assert.h> - -#include <vnet/ethernet/ethernet.h> -#include <vnet/devices/dpdk/dpdk.h> - -#include "dpdk_priv.h" -#include <vppinfra/error.h> - -#define foreach_dpdk_tx_func_error \ - _(BAD_RETVAL, "DPDK tx function returned an error") \ - _(RING_FULL, "Tx packet drops (ring full)") \ - _(PKT_DROP, "Tx packet drops (dpdk tx failure)") \ - _(REPL_FAIL, "Tx packet drops (replication failure)") - -typedef enum -{ -#define _(f,s) DPDK_TX_FUNC_ERROR_##f, - foreach_dpdk_tx_func_error -#undef _ - DPDK_TX_FUNC_N_ERROR, -} dpdk_tx_func_error_t; - -static char *dpdk_tx_func_error_strings[] = { -#define _(n,s) s, - foreach_dpdk_tx_func_error -#undef _ -}; - -clib_error_t * -dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address) -{ - int error; - dpdk_main_t *dm = &dpdk_main; - dpdk_device_t *xd = vec_elt_at_index (dm->devices, hi->dev_instance); - - error = rte_eth_dev_default_mac_addr_set (xd->device_index, - (struct ether_addr *) address); - - if (error) - { - return clib_error_return (0, "mac address set failed: %d", error); - } - else - { - return NULL; - } -} - -clib_error_t * -dpdk_set_mc_filter (vnet_hw_interface_t * hi, - struct ether_addr mc_addr_vec[], int naddr) -{ - int error; - dpdk_main_t *dm = &dpdk_main; - dpdk_device_t *xd = vec_elt_at_index (dm->devices, hi->dev_instance); - - error = rte_eth_dev_set_mc_addr_list (xd->device_index, mc_addr_vec, naddr); - - if (error) - { - return clib_error_return (0, "mc addr list failed: %d", error); - } - else - { - return NULL; - } -} - -struct rte_mbuf * -dpdk_replicate_packet_mb (vlib_buffer_t * b) -{ - vlib_main_t *vm = vlib_get_main (); - vlib_buffer_main_t *bm = vm->buffer_main; - struct rte_mbuf **mbufs = 0, *s, *d; - u8 nb_segs; - unsigned socket_id = rte_socket_id (); - int i; - - ASSERT (bm->pktmbuf_pools[socket_id]); - s = rte_mbuf_from_vlib_buffer (b); - nb_segs = s->nb_segs; - vec_validate (mbufs, nb_segs - 1); - - if (rte_pktmbuf_alloc_bulk (bm->pktmbuf_pools[socket_id], mbufs, nb_segs)) - { - vec_free (mbufs); - return 0; - } - - d = mbufs[0]; - d->nb_segs = s->nb_segs; - d->data_len = s->data_len; - d->pkt_len = s->pkt_len; - d->data_off = s->data_off; - clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len); - - for (i = 1; i < nb_segs; i++) - { - d->next = mbufs[i]; - d = mbufs[i]; - s = s->next; - d->data_len = s->data_len; - clib_memcpy (d->buf_addr, s->buf_addr, - RTE_PKTMBUF_HEADROOM + s->data_len); - } - - d = mbufs[0]; - vec_free (mbufs); - return d; -} - -static void -dpdk_tx_trace_buffer (dpdk_main_t * dm, - vlib_node_runtime_t * node, - dpdk_device_t * xd, - u16 queue_id, u32 buffer_index, vlib_buffer_t * buffer) -{ - vlib_main_t *vm = vlib_get_main (); - dpdk_tx_dma_trace_t *t0; - struct rte_mbuf *mb; - - mb = rte_mbuf_from_vlib_buffer (buffer); - - t0 = vlib_add_trace (vm, node, buffer, sizeof (t0[0])); - t0->queue_index = queue_id; - t0->device_index = xd->device_index; - t0->buffer_index = buffer_index; - clib_memcpy (&t0->mb, mb, sizeof (t0->mb)); - clib_memcpy (&t0->buffer, buffer, - sizeof (buffer[0]) - sizeof (buffer->pre_data)); - clib_memcpy (t0->buffer.pre_data, buffer->data + buffer->current_data, - sizeof (t0->buffer.pre_data)); -} - -static_always_inline void -dpdk_validate_rte_mbuf (vlib_main_t * vm, vlib_buffer_t * b, - int maybe_multiseg) -{ - struct rte_mbuf *mb, *first_mb, *last_mb; - - /* buffer is coming from non-dpdk source so we need to init - rte_mbuf header */ - if (PREDICT_FALSE ((b->flags & VNET_BUFFER_RTE_MBUF_VALID) == 0)) - { - vlib_buffer_t *b2 = b; - last_mb = mb = rte_mbuf_from_vlib_buffer (b2); - rte_pktmbuf_reset (mb); - while (maybe_multiseg && (b2->flags & VLIB_BUFFER_NEXT_PRESENT)) - { - b2 = vlib_get_buffer (vm, b2->next_buffer); - mb = rte_mbuf_from_vlib_buffer (b2); - last_mb->next = mb; - last_mb = mb; - rte_pktmbuf_reset (mb); - } - } - - first_mb = mb = rte_mbuf_from_vlib_buffer (b); - first_mb->nb_segs = 1; - mb->data_len = b->current_length; - mb->pkt_len = maybe_multiseg ? vlib_buffer_length_in_chain (vm, b) : - b->current_length; - mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data; - - while (maybe_multiseg && (b->flags & VLIB_BUFFER_NEXT_PRESENT)) - { - b = vlib_get_buffer (vm, b->next_buffer); - mb = rte_mbuf_from_vlib_buffer (b); - mb->data_len = b->current_length; - mb->pkt_len = b->current_length; - mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data; - first_mb->nb_segs++; - } -} - -/* - * This function calls the dpdk's tx_burst function to transmit the packets - * on the tx_vector. It manages a lock per-device if the device does not - * support multiple queues. It returns the number of packets untransmitted - * on the tx_vector. If all packets are transmitted (the normal case), the - * function returns 0. - * - * The function assumes there is at least one packet on the tx_vector. - */ -static_always_inline - u32 tx_burst_vector_internal (vlib_main_t * vm, - dpdk_device_t * xd, - struct rte_mbuf **tx_vector) -{ - dpdk_main_t *dm = &dpdk_main; - u32 n_packets; - u32 tx_head; - u32 tx_tail; - u32 n_retry; - int rv; - int queue_id; - tx_ring_hdr_t *ring; - - ring = vec_header (tx_vector, sizeof (*ring)); - - n_packets = ring->tx_head - ring->tx_tail; - - tx_head = ring->tx_head % xd->nb_tx_desc; - - /* - * Ensure rte_eth_tx_burst is not called with 0 packets, which can lead to - * unpredictable results. - */ - ASSERT (n_packets > 0); - - /* - * Check for tx_vector overflow. If this fails it is a system configuration - * error. The ring should be sized big enough to handle the largest un-flowed - * off burst from a traffic manager. A larger size also helps performance - * a bit because it decreases the probability of having to issue two tx_burst - * calls due to a ring wrap. - */ - ASSERT (n_packets < xd->nb_tx_desc); - ASSERT (ring->tx_tail == 0); - - n_retry = 16; - queue_id = vm->cpu_index; - - do - { - /* start the burst at the tail */ - tx_tail = ring->tx_tail % xd->nb_tx_desc; - - /* - * This device only supports one TX queue, - * and we're running multi-threaded... - */ - if (PREDICT_FALSE (xd->lockp != 0)) - { - queue_id = queue_id % xd->tx_q_used; - while (__sync_lock_test_and_set (xd->lockp[queue_id], 1)) - /* zzzz */ - queue_id = (queue_id + 1) % xd->tx_q_used; - } - - if (PREDICT_FALSE (xd->flags & DPDK_DEVICE_FLAG_HQOS)) /* HQoS ON */ - { - /* no wrap, transmit in one burst */ - dpdk_device_hqos_per_worker_thread_t *hqos = - &xd->hqos_wt[vm->cpu_index]; - - ASSERT (hqos->swq != NULL); - - dpdk_hqos_metadata_set (hqos, - &tx_vector[tx_tail], tx_head - tx_tail); - rv = rte_ring_sp_enqueue_burst (hqos->swq, - (void **) &tx_vector[tx_tail], - (uint16_t) (tx_head - tx_tail)); - } - else if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD)) - { - /* no wrap, transmit in one burst */ - rv = rte_eth_tx_burst (xd->device_index, - (uint16_t) queue_id, - &tx_vector[tx_tail], - (uint16_t) (tx_head - tx_tail)); - } - else - { - ASSERT (0); - rv = 0; - } - - if (PREDICT_FALSE (xd->lockp != 0)) - *xd->lockp[queue_id] = 0; - - if (PREDICT_FALSE (rv < 0)) - { - // emit non-fatal message, bump counter - vnet_main_t *vnm = dm->vnet_main; - vnet_interface_main_t *im = &vnm->interface_main; - u32 node_index; - - node_index = vec_elt_at_index (im->hw_interfaces, - xd->vlib_hw_if_index)->tx_node_index; - - vlib_error_count (vm, node_index, DPDK_TX_FUNC_ERROR_BAD_RETVAL, 1); - clib_warning ("rte_eth_tx_burst[%d]: error %d", xd->device_index, - rv); - return n_packets; // untransmitted packets - } - ring->tx_tail += (u16) rv; - n_packets -= (uint16_t) rv; - } - while (rv && n_packets && (n_retry > 0)); - - return n_packets; -} - -static_always_inline void -dpdk_prefetch_buffer_by_index (vlib_main_t * vm, u32 bi) -{ - vlib_buffer_t *b; - struct rte_mbuf *mb; - b = vlib_get_buffer (vm, bi); - mb = rte_mbuf_from_vlib_buffer (b); - CLIB_PREFETCH (mb, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); -} - -static_always_inline void -dpdk_buffer_recycle (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp) -{ - dpdk_main_t *dm = &dpdk_main; - u32 my_cpu = vm->cpu_index; - struct rte_mbuf *mb_new; - - if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0) - return; - - mb_new = dpdk_replicate_packet_mb (b); - if (PREDICT_FALSE (mb_new == 0)) - { - vlib_error_count (vm, node->node_index, - DPDK_TX_FUNC_ERROR_REPL_FAIL, 1); - b->flags |= VLIB_BUFFER_REPL_FAIL; - } - else - *mbp = mb_new; - - vec_add1 (dm->recycle[my_cpu], bi); -} - -/* - * Transmits the packets on the frame to the interface associated with the - * node. It first copies packets on the frame to a tx_vector containing the - * rte_mbuf pointers. It then passes this vector to tx_burst_vector_internal - * which calls the dpdk tx_burst function. - */ -static uword -dpdk_interface_tx (vlib_main_t * vm, - vlib_node_runtime_t * node, vlib_frame_t * f) -{ - dpdk_main_t *dm = &dpdk_main; - vnet_interface_output_runtime_t *rd = (void *) node->runtime_data; - dpdk_device_t *xd = vec_elt_at_index (dm->devices, rd->dev_instance); - u32 n_packets = f->n_vectors; - u32 n_left; - u32 *from; - struct rte_mbuf **tx_vector; - u16 i; - u16 nb_tx_desc = xd->nb_tx_desc; - int queue_id; - u32 my_cpu; - u32 tx_pkts = 0; - tx_ring_hdr_t *ring; - u32 n_on_ring; - - my_cpu = vm->cpu_index; - - queue_id = my_cpu; - - tx_vector = xd->tx_vectors[queue_id]; - ring = vec_header (tx_vector, sizeof (*ring)); - - n_on_ring = ring->tx_head - ring->tx_tail; - from = vlib_frame_vector_args (f); - - ASSERT (n_packets <= VLIB_FRAME_SIZE); - - if (PREDICT_FALSE (n_on_ring + n_packets > nb_tx_desc)) - { - /* - * Overflowing the ring should never happen. - * If it does then drop the whole frame. - */ - vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_RING_FULL, - n_packets); - - while (n_packets--) - { - u32 bi0 = from[n_packets]; - vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); - struct rte_mbuf *mb0 = rte_mbuf_from_vlib_buffer (b0); - rte_pktmbuf_free (mb0); - } - return n_on_ring; - } - - if (PREDICT_FALSE (dm->tx_pcap_enable)) - { - n_left = n_packets; - while (n_left > 0) - { - u32 bi0 = from[0]; - vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); - if (dm->pcap_sw_if_index == 0 || - dm->pcap_sw_if_index == vnet_buffer (b0)->sw_if_index[VLIB_TX]) - pcap_add_buffer (&dm->pcap_main, vm, bi0, 512); - from++; - n_left--; - } - } - - from = vlib_frame_vector_args (f); - n_left = n_packets; - i = ring->tx_head % nb_tx_desc; - - while (n_left >= 8) - { - u32 bi0, bi1, bi2, bi3; - struct rte_mbuf *mb0, *mb1, *mb2, *mb3; - vlib_buffer_t *b0, *b1, *b2, *b3; - u32 or_flags; - - dpdk_prefetch_buffer_by_index (vm, from[4]); - dpdk_prefetch_buffer_by_index (vm, from[5]); - dpdk_prefetch_buffer_by_index (vm, from[6]); - dpdk_prefetch_buffer_by_index (vm, from[7]); - - bi0 = from[0]; - bi1 = from[1]; - bi2 = from[2]; - bi3 = from[3]; - from += 4; - - b0 = vlib_get_buffer (vm, bi0); - b1 = vlib_get_buffer (vm, bi1); - b2 = vlib_get_buffer (vm, bi2); - b3 = vlib_get_buffer (vm, bi3); - - or_flags = b0->flags | b1->flags | b2->flags | b3->flags; - - if (or_flags & VLIB_BUFFER_NEXT_PRESENT) - { - dpdk_validate_rte_mbuf (vm, b0, 1); - dpdk_validate_rte_mbuf (vm, b1, 1); - dpdk_validate_rte_mbuf (vm, b2, 1); - dpdk_validate_rte_mbuf (vm, b3, 1); - } - else - { - dpdk_validate_rte_mbuf (vm, b0, 0); - dpdk_validate_rte_mbuf (vm, b1, 0); - dpdk_validate_rte_mbuf (vm, b2, 0); - dpdk_validate_rte_mbuf (vm, b3, 0); - } - - mb0 = rte_mbuf_from_vlib_buffer (b0); - mb1 = rte_mbuf_from_vlib_buffer (b1); - mb2 = rte_mbuf_from_vlib_buffer (b2); - mb3 = rte_mbuf_from_vlib_buffer (b3); - - if (PREDICT_FALSE (or_flags & VLIB_BUFFER_RECYCLE)) - { - dpdk_buffer_recycle (vm, node, b0, bi0, &mb0); - dpdk_buffer_recycle (vm, node, b1, bi1, &mb1); - dpdk_buffer_recycle (vm, node, b2, bi2, &mb2); - dpdk_buffer_recycle (vm, node, b3, bi3, &mb3); - - /* dont enqueue packets if replication failed as they must - be sent back to recycle */ - if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0)) - tx_vector[i++ % nb_tx_desc] = mb0; - if (PREDICT_TRUE ((b1->flags & VLIB_BUFFER_REPL_FAIL) == 0)) - tx_vector[i++ % nb_tx_desc] = mb1; - if (PREDICT_TRUE ((b2->flags & VLIB_BUFFER_REPL_FAIL) == 0)) - tx_vector[i++ % nb_tx_desc] = mb2; - if (PREDICT_TRUE ((b3->flags & VLIB_BUFFER_REPL_FAIL) == 0)) - tx_vector[i++ % nb_tx_desc] = mb3; - } - else - { - if (PREDICT_FALSE (i + 3 >= nb_tx_desc)) - { - tx_vector[i++ % nb_tx_desc] = mb0; - tx_vector[i++ % nb_tx_desc] = mb1; - tx_vector[i++ % nb_tx_desc] = mb2; - tx_vector[i++ % nb_tx_desc] = mb3; - i %= nb_tx_desc; - } - else - { - tx_vector[i++] = mb0; - tx_vector[i++] = mb1; - tx_vector[i++] = mb2; - tx_vector[i++] = mb3; - } - } - - - if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) - { - if (b0->flags & VLIB_BUFFER_IS_TRACED) - dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0); - if (b1->flags & VLIB_BUFFER_IS_TRACED) - dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi1, b1); - if (b2->flags & VLIB_BUFFER_IS_TRACED) - dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi2, b2); - if (b3->flags & VLIB_BUFFER_IS_TRACED) - dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi3, b3); - } - - n_left -= 4; - } - while (n_left > 0) - { - u32 bi0; - struct rte_mbuf *mb0; - vlib_buffer_t *b0; - - bi0 = from[0]; - from++; - - b0 = vlib_get_buffer (vm, bi0); - - dpdk_validate_rte_mbuf (vm, b0, 1); - - mb0 = rte_mbuf_from_vlib_buffer (b0); - dpdk_buffer_recycle (vm, node, b0, bi0, &mb0); - - if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) - if (b0->flags & VLIB_BUFFER_IS_TRACED) - dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0); - - if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0)) - { - tx_vector[i % nb_tx_desc] = mb0; - i++; - } - n_left--; - } - - /* account for additional packets in the ring */ - ring->tx_head += n_packets; - n_on_ring = ring->tx_head - ring->tx_tail; - - /* transmit as many packets as possible */ - n_packets = tx_burst_vector_internal (vm, xd, tx_vector); - - /* - * tx_pkts is the number of packets successfully transmitted - * This is the number originally on ring minus the number remaining on ring - */ - tx_pkts = n_on_ring - n_packets; - - { - /* If there is no callback then drop any non-transmitted packets */ - if (PREDICT_FALSE (n_packets)) - { - vlib_simple_counter_main_t *cm; - vnet_main_t *vnm = vnet_get_main (); - - cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, - VNET_INTERFACE_COUNTER_TX_ERROR); - - vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index, - n_packets); - - vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_PKT_DROP, - n_packets); - - while (n_packets--) - rte_pktmbuf_free (tx_vector[ring->tx_tail + n_packets]); - } - - /* Reset head/tail to avoid unnecessary wrap */ - ring->tx_head = 0; - ring->tx_tail = 0; - } - - /* Recycle replicated buffers */ - if (PREDICT_FALSE (vec_len (dm->recycle[my_cpu]))) - { - vlib_buffer_free (vm, dm->recycle[my_cpu], - vec_len (dm->recycle[my_cpu])); - _vec_len (dm->recycle[my_cpu]) = 0; - } - - ASSERT (ring->tx_head >= ring->tx_tail); - - return tx_pkts; -} - -static void -dpdk_clear_hw_interface_counters (u32 instance) -{ - dpdk_main_t *dm = &dpdk_main; - dpdk_device_t *xd = vec_elt_at_index (dm->devices, instance); - - /* - * Set the "last_cleared_stats" to the current stats, so that - * things appear to clear from a display perspective. - */ - dpdk_update_counters (xd, vlib_time_now (dm->vlib_main)); - - clib_memcpy (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats)); - clib_memcpy (xd->last_cleared_xstats, xd->xstats, - vec_len (xd->last_cleared_xstats) * - sizeof (xd->last_cleared_xstats[0])); - -} - -static clib_error_t * -dpdk_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags) -{ - vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index); - uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0; - dpdk_main_t *dm = &dpdk_main; - dpdk_device_t *xd = vec_elt_at_index (dm->devices, hif->dev_instance); - int rv = 0; - - if (is_up) - { - f64 now = vlib_time_now (dm->vlib_main); - - if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0) - rv = rte_eth_dev_start (xd->device_index); - - if (xd->flags & DPDK_DEVICE_FLAG_PROMISC) - rte_eth_promiscuous_enable (xd->device_index); - else - rte_eth_promiscuous_disable (xd->device_index); - - rte_eth_allmulticast_enable (xd->device_index); - xd->flags |= DPDK_DEVICE_FLAG_ADMIN_UP; - dpdk_update_counters (xd, now); - dpdk_update_link_state (xd, now); - } - else - { - xd->flags &= ~DPDK_DEVICE_FLAG_ADMIN_UP; - - rte_eth_allmulticast_disable (xd->device_index); - vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index, 0); - rte_eth_dev_stop (xd->device_index); - - /* For bonded interface, stop slave links */ - if (xd->pmd == VNET_DPDK_PMD_BOND) - { - u8 slink[16]; - int nlink = rte_eth_bond_slaves_get (xd->device_index, slink, 16); - while (nlink >= 1) - { - u8 dpdk_port = slink[--nlink]; - rte_eth_dev_stop (dpdk_port); - } - } - } - - if (rv < 0) - clib_warning ("rte_eth_dev_%s error: %d", is_up ? "start" : "stop", rv); - - return /* no error */ 0; -} - -/* - * Dynamically redirect all pkts from a specific interface - * to the specified node - */ -static void -dpdk_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index, - u32 node_index) -{ - dpdk_main_t *xm = &dpdk_main; - vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); - dpdk_device_t *xd = vec_elt_at_index (xm->devices, hw->dev_instance); - - /* Shut off redirection */ - if (node_index == ~0) - { - xd->per_interface_next_index = node_index; - return; - } - - xd->per_interface_next_index = - vlib_node_add_next (xm->vlib_main, dpdk_input_node.index, node_index); -} - - -static clib_error_t * -dpdk_subif_add_del_function (vnet_main_t * vnm, - u32 hw_if_index, - struct vnet_sw_interface_t *st, int is_add) -{ - dpdk_main_t *xm = &dpdk_main; - vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); - dpdk_device_t *xd = vec_elt_at_index (xm->devices, hw->dev_instance); - vnet_sw_interface_t *t = (vnet_sw_interface_t *) st; - int r, vlan_offload; - u32 prev_subifs = xd->num_subifs; - clib_error_t *err = 0; - - if (is_add) - xd->num_subifs++; - else if (xd->num_subifs) - xd->num_subifs--; - - if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0) - goto done; - - /* currently we program VLANS only for IXGBE VF and I40E VF */ - if ((xd->pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->pmd != VNET_DPDK_PMD_I40EVF)) - goto done; - - if (t->sub.eth.flags.no_tags == 1) - goto done; - - if ((t->sub.eth.flags.one_tag != 1) || (t->sub.eth.flags.exact_match != 1)) - { - xd->num_subifs = prev_subifs; - err = clib_error_return (0, "unsupported VLAN setup"); - goto done; - } - - vlan_offload = rte_eth_dev_get_vlan_offload (xd->device_index); - vlan_offload |= ETH_VLAN_FILTER_OFFLOAD; - - if ((r = rte_eth_dev_set_vlan_offload (xd->device_index, vlan_offload))) - { - xd->num_subifs = prev_subifs; - err = clib_error_return (0, "rte_eth_dev_set_vlan_offload[%d]: err %d", - xd->device_index, r); - goto done; - } - - - if ((r = - rte_eth_dev_vlan_filter (xd->device_index, t->sub.eth.outer_vlan_id, - is_add))) - { - xd->num_subifs = prev_subifs; - err = clib_error_return (0, "rte_eth_dev_vlan_filter[%d]: err %d", - xd->device_index, r); - goto done; - } - -done: - if (xd->num_subifs) - xd->flags |= DPDK_DEVICE_FLAG_HAVE_SUBIF; - else - xd->flags &= ~DPDK_DEVICE_FLAG_HAVE_SUBIF; - - return err; -} - -/* *INDENT-OFF* */ -VNET_DEVICE_CLASS (dpdk_device_class) = { - .name = "dpdk", - .tx_function = dpdk_interface_tx, - .tx_function_n_errors = DPDK_TX_FUNC_N_ERROR, - .tx_function_error_strings = dpdk_tx_func_error_strings, - .format_device_name = format_dpdk_device_name, - .format_device = format_dpdk_device, - .format_tx_trace = format_dpdk_tx_dma_trace, - .clear_counters = dpdk_clear_hw_interface_counters, - .admin_up_down_function = dpdk_interface_admin_up_down, - .subif_add_del_function = dpdk_subif_add_del_function, - .rx_redirect_to_node = dpdk_set_interface_next_node, - .mac_addr_change_function = dpdk_set_mac_address, -}; - -VLIB_DEVICE_TX_FUNCTION_MULTIARCH (dpdk_device_class, dpdk_interface_tx) -/* *INDENT-ON* */ - -#define UP_DOWN_FLAG_EVENT 1 - -uword -admin_up_down_process (vlib_main_t * vm, - vlib_node_runtime_t * rt, vlib_frame_t * f) -{ - clib_error_t *error = 0; - uword event_type; - uword *event_data = 0; - u32 sw_if_index; - u32 flags; - - while (1) - { - vlib_process_wait_for_event (vm); - - event_type = vlib_process_get_events (vm, &event_data); - - dpdk_main.admin_up_down_in_progress = 1; - - switch (event_type) - { - case UP_DOWN_FLAG_EVENT: - { - if (vec_len (event_data) == 2) - { - sw_if_index = event_data[0]; - flags = event_data[1]; - error = - vnet_sw_interface_set_flags (vnet_get_main (), sw_if_index, - flags); - clib_error_report (error); - } - } - break; - } - - vec_reset_length (event_data); - - dpdk_main.admin_up_down_in_progress = 0; - - } - return 0; /* or not */ -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (admin_up_down_process_node,static) = { - .function = admin_up_down_process, - .type = VLIB_NODE_TYPE_PROCESS, - .name = "admin-up-down-process", - .process_log2_n_stack_bytes = 17, // 256KB -}; -/* *INDENT-ON* */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/dpdk.h b/vnet/vnet/devices/dpdk/dpdk.h deleted file mode 100644 index d8f378d2b54..00000000000 --- a/vnet/vnet/devices/dpdk/dpdk.h +++ /dev/null @@ -1,534 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __included_dpdk_h__ -#define __included_dpdk_h__ - -/* $$$$ We should rename always_inline -> clib_always_inline */ -#undef always_inline - -#include <rte_config.h> - -#include <rte_common.h> -#include <rte_dev.h> -#include <rte_log.h> -#include <rte_memory.h> -#include <rte_memzone.h> -#include <rte_tailq.h> -#include <rte_eal.h> -#include <rte_per_lcore.h> -#include <rte_launch.h> -#include <rte_atomic.h> -#include <rte_cycles.h> -#include <rte_prefetch.h> -#include <rte_lcore.h> -#include <rte_per_lcore.h> -#include <rte_branch_prediction.h> -#include <rte_interrupts.h> -#include <rte_pci.h> -#include <rte_random.h> -#include <rte_debug.h> -#include <rte_ether.h> -#include <rte_ethdev.h> -#include <rte_ring.h> -#include <rte_mempool.h> -#include <rte_mbuf.h> -#include <rte_virtio_net.h> -#include <rte_version.h> -#include <rte_eth_bond.h> -#include <rte_sched.h> - -#include <vnet/unix/pcap.h> -#include <vnet/devices/devices.h> - -#if CLIB_DEBUG > 0 -#define always_inline static inline -#else -#define always_inline static inline __attribute__ ((__always_inline__)) -#endif - -#include <vlib/pci/pci.h> - -#define NB_MBUF (16<<10) - -extern vnet_device_class_t dpdk_device_class; -extern vlib_node_registration_t dpdk_input_node; -extern vlib_node_registration_t handoff_dispatch_node; - -#if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0) -#define foreach_dpdk_pmd \ - _ ("net_thunderx", THUNDERX) \ - _ ("net_e1000_em", E1000EM) \ - _ ("net_e1000_igb", IGB) \ - _ ("net_e1000_igb_vf", IGBVF) \ - _ ("net_ixgbe", IXGBE) \ - _ ("net_ixgbe_vf", IXGBEVF) \ - _ ("net_i40e", I40E) \ - _ ("net_i40e_vf", I40EVF) \ - _ ("net_virtio", VIRTIO) \ - _ ("net_enic", ENIC) \ - _ ("net_vmxnet3", VMXNET3) \ - _ ("net_af_packet", AF_PACKET) \ - _ ("rte_bond_pmd", BOND) \ - _ ("net_fm10k", FM10K) \ - _ ("net_cxgbe", CXGBE) \ - _ ("net_mlx5", MLX5) \ - _ ("net_dpaa2", DPAA2) -#else -#define foreach_dpdk_pmd \ - _ ("rte_nicvf_pmd", THUNDERX) \ - _ ("rte_em_pmd", E1000EM) \ - _ ("rte_igb_pmd", IGB) \ - _ ("rte_igbvf_pmd", IGBVF) \ - _ ("rte_ixgbe_pmd", IXGBE) \ - _ ("rte_ixgbevf_pmd", IXGBEVF) \ - _ ("rte_i40e_pmd", I40E) \ - _ ("rte_i40evf_pmd", I40EVF) \ - _ ("rte_virtio_pmd", VIRTIO) \ - _ ("rte_enic_pmd", ENIC) \ - _ ("rte_vmxnet3_pmd", VMXNET3) \ - _ ("AF_PACKET PMD", AF_PACKET) \ - _ ("rte_bond_pmd", BOND) \ - _ ("rte_pmd_fm10k", FM10K) \ - _ ("rte_cxgbe_pmd", CXGBE) \ - _ ("rte_dpaa2_dpni", DPAA2) -#endif - -typedef enum -{ - VNET_DPDK_PMD_NONE, -#define _(s,f) VNET_DPDK_PMD_##f, - foreach_dpdk_pmd -#undef _ - VNET_DPDK_PMD_UNKNOWN, /* must be last */ -} dpdk_pmd_t; - -typedef enum -{ - VNET_DPDK_PORT_TYPE_ETH_1G, - VNET_DPDK_PORT_TYPE_ETH_10G, - VNET_DPDK_PORT_TYPE_ETH_40G, - VNET_DPDK_PORT_TYPE_ETH_100G, - VNET_DPDK_PORT_TYPE_ETH_BOND, - VNET_DPDK_PORT_TYPE_ETH_SWITCH, - VNET_DPDK_PORT_TYPE_AF_PACKET, - VNET_DPDK_PORT_TYPE_UNKNOWN, -} dpdk_port_type_t; - -/* - * The header for the tx_vector in dpdk_device_t. - * Head and tail are indexes into the tx_vector and are of type - * u64 so they never overflow. - */ -typedef struct -{ - u64 tx_head; - u64 tx_tail; -} tx_ring_hdr_t; - -typedef struct -{ - struct rte_ring *swq; - - u64 hqos_field0_slabmask; - u32 hqos_field0_slabpos; - u32 hqos_field0_slabshr; - u64 hqos_field1_slabmask; - u32 hqos_field1_slabpos; - u32 hqos_field1_slabshr; - u64 hqos_field2_slabmask; - u32 hqos_field2_slabpos; - u32 hqos_field2_slabshr; - u32 hqos_tc_table[64]; -} dpdk_device_hqos_per_worker_thread_t; - -typedef struct -{ - struct rte_ring **swq; - struct rte_mbuf **pkts_enq; - struct rte_mbuf **pkts_deq; - struct rte_sched_port *hqos; - u32 hqos_burst_enq; - u32 hqos_burst_deq; - u32 pkts_enq_len; - u32 swq_pos; - u32 flush_count; -} dpdk_device_hqos_per_hqos_thread_t; - -typedef struct -{ - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - volatile u32 **lockp; - - /* Instance ID */ - u32 device_index; - - u32 vlib_hw_if_index; - u32 vlib_sw_if_index; - - /* next node index if we decide to steal the rx graph arc */ - u32 per_interface_next_index; - - /* dpdk rte_mbuf rx and tx vectors, VLIB_FRAME_SIZE */ - struct rte_mbuf ***tx_vectors; /* one per worker thread */ - struct rte_mbuf ***rx_vectors; - - /* vector of traced contexts, per device */ - u32 **d_trace_buffers; - - dpdk_pmd_t pmd:8; - i8 cpu_socket; - - u16 flags; -#define DPDK_DEVICE_FLAG_ADMIN_UP (1 << 0) -#define DPDK_DEVICE_FLAG_PROMISC (1 << 1) -#define DPDK_DEVICE_FLAG_PMD (1 << 2) -#define DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE (1 << 3) -#define DPDK_DEVICE_FLAG_MAYBE_MULTISEG (1 << 4) -#define DPDK_DEVICE_FLAG_HAVE_SUBIF (1 << 5) -#define DPDK_DEVICE_FLAG_HQOS (1 << 6) - - u16 nb_tx_desc; - CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); - - u8 *interface_name_suffix; - - /* number of sub-interfaces */ - u16 num_subifs; - - /* PMD related */ - u16 tx_q_used; - u16 rx_q_used; - u16 nb_rx_desc; - u16 *cpu_socket_id_by_queue; - struct rte_eth_conf port_conf; - struct rte_eth_txconf tx_conf; - - /* HQoS related */ - dpdk_device_hqos_per_worker_thread_t *hqos_wt; - dpdk_device_hqos_per_hqos_thread_t *hqos_ht; - - /* af_packet */ - u8 af_packet_port_id; - - struct rte_eth_link link; - f64 time_last_link_update; - - struct rte_eth_stats stats; - struct rte_eth_stats last_stats; - struct rte_eth_stats last_cleared_stats; - struct rte_eth_xstat *xstats; - struct rte_eth_xstat *last_cleared_xstats; - f64 time_last_stats_update; - dpdk_port_type_t port_type; -} dpdk_device_t; - -#define DPDK_STATS_POLL_INTERVAL (10.0) -#define DPDK_MIN_STATS_POLL_INTERVAL (0.001) /* 1msec */ - -#define DPDK_LINK_POLL_INTERVAL (3.0) -#define DPDK_MIN_LINK_POLL_INTERVAL (0.001) /* 1msec */ - -typedef struct -{ - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - - /* total input packet counter */ - u64 aggregate_rx_packets; -} dpdk_worker_t; - -typedef struct -{ - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - - /* total input packet counter */ - u64 aggregate_rx_packets; -} dpdk_hqos_thread_t; - -typedef struct -{ - u32 device; - u16 queue_id; -} dpdk_device_and_queue_t; - -#ifndef DPDK_HQOS_DBG_BYPASS -#define DPDK_HQOS_DBG_BYPASS 0 -#endif - -#ifndef HQOS_FLUSH_COUNT_THRESHOLD -#define HQOS_FLUSH_COUNT_THRESHOLD 100000 -#endif - -typedef struct dpdk_device_config_hqos_t -{ - u32 hqos_thread; - u32 hqos_thread_valid; - - u32 swq_size; - u32 burst_enq; - u32 burst_deq; - - u32 pktfield0_slabpos; - u32 pktfield1_slabpos; - u32 pktfield2_slabpos; - u64 pktfield0_slabmask; - u64 pktfield1_slabmask; - u64 pktfield2_slabmask; - u32 tc_table[64]; - - struct rte_sched_port_params port; - struct rte_sched_subport_params *subport; - struct rte_sched_pipe_params *pipe; - uint32_t *pipe_map; -} dpdk_device_config_hqos_t; - -int dpdk_hqos_validate_mask (u64 mask, u32 n); -void dpdk_device_config_hqos_pipe_profile_default (dpdk_device_config_hqos_t * - hqos, u32 pipe_profile_id); -void dpdk_device_config_hqos_default (dpdk_device_config_hqos_t * hqos); -clib_error_t *dpdk_port_setup_hqos (dpdk_device_t * xd, - dpdk_device_config_hqos_t * hqos); -void dpdk_hqos_metadata_set (dpdk_device_hqos_per_worker_thread_t * hqos, - struct rte_mbuf **pkts, u32 n_pkts); - -#define foreach_dpdk_device_config_item \ - _ (num_rx_queues) \ - _ (num_tx_queues) \ - _ (num_rx_desc) \ - _ (num_tx_desc) \ - _ (rss_fn) - -typedef struct -{ - vlib_pci_addr_t pci_addr; - u8 is_blacklisted; - u8 vlan_strip_offload; -#define DPDK_DEVICE_VLAN_STRIP_DEFAULT 0 -#define DPDK_DEVICE_VLAN_STRIP_OFF 1 -#define DPDK_DEVICE_VLAN_STRIP_ON 2 - -#define _(x) uword x; - foreach_dpdk_device_config_item -#undef _ - clib_bitmap_t * workers; - u32 hqos_enabled; - dpdk_device_config_hqos_t hqos; -} dpdk_device_config_t; - -typedef struct -{ - - /* Config stuff */ - u8 **eal_init_args; - u8 *eal_init_args_str; - u8 *uio_driver_name; - u8 no_multi_seg; - u8 enable_tcp_udp_checksum; - - /* Required config parameters */ - u8 coremask_set_manually; - u8 nchannels_set_manually; - u32 coremask; - u32 nchannels; - u32 num_mbufs; - u8 num_kni; /* while kni_init allows u32, port_id in callback fn is only u8 */ - - /* - * format interface names ala xxxEthernet%d/%d/%d instead of - * xxxEthernet%x/%x/%x. - */ - u8 interface_name_format_decimal; - - /* per-device config */ - dpdk_device_config_t default_devconf; - dpdk_device_config_t *dev_confs; - uword *device_config_index_by_pci_addr; - -} dpdk_config_main_t; - -dpdk_config_main_t dpdk_config_main; - -typedef struct -{ - - /* Devices */ - dpdk_device_t *devices; - dpdk_device_and_queue_t **devices_by_cpu; - dpdk_device_and_queue_t **devices_by_hqos_cpu; - - /* per-thread recycle lists */ - u32 **recycle; - - /* buffer flags template, configurable to enable/disable tcp / udp cksum */ - u32 buffer_flags_template; - - /* vlib buffer free list, must be same size as an rte_mbuf */ - u32 vlib_buffer_free_list_index; - - /* dpdk worker "threads" */ - dpdk_worker_t *workers; - - /* dpdk HQoS "threads" */ - dpdk_hqos_thread_t *hqos_threads; - - /* Ethernet input node index */ - u32 ethernet_input_node_index; - - /* pcap tracing [only works if (CLIB_DEBUG > 0)] */ - int tx_pcap_enable; - pcap_main_t pcap_main; - u8 *pcap_filename; - u32 pcap_sw_if_index; - u32 pcap_pkts_to_capture; - - /* hashes */ - uword *dpdk_device_by_kni_port_id; - uword *vu_sw_if_index_by_listener_fd; - uword *vu_sw_if_index_by_sock_fd; - u32 *vu_inactive_interfaces_device_index; - - /* - * flag indicating that a posted admin up/down - * (via post_sw_interface_set_flags) is in progress - */ - u8 admin_up_down_in_progress; - - u8 use_rss; - - /* which cpus are running dpdk-input */ - int input_cpu_first_index; - int input_cpu_count; - - /* which cpus are running I/O TX */ - int hqos_cpu_first_index; - int hqos_cpu_count; - - /* control interval of dpdk link state and stat polling */ - f64 link_state_poll_interval; - f64 stat_poll_interval; - - /* Sleep for this many MS after each device poll */ - u32 poll_sleep; - - /* convenience */ - vlib_main_t *vlib_main; - vnet_main_t *vnet_main; - dpdk_config_main_t *conf; -} dpdk_main_t; - -dpdk_main_t dpdk_main; - -typedef struct -{ - u32 buffer_index; - u16 device_index; - u8 queue_index; - struct rte_mbuf mb; - /* Copy of VLIB buffer; packet data stored in pre_data. */ - vlib_buffer_t buffer; -} dpdk_tx_dma_trace_t; - -typedef struct -{ - u32 buffer_index; - u16 device_index; - u16 queue_index; - struct rte_mbuf mb; - vlib_buffer_t buffer; /* Copy of VLIB buffer; pkt data stored in pre_data. */ - u8 data[256]; /* First 256 data bytes, used for hexdump */ -} dpdk_rx_dma_trace_t; - -void vnet_buffer_needs_dpdk_mb (vlib_buffer_t * b); - -clib_error_t *dpdk_set_mac_address (vnet_hw_interface_t * hi, char *address); - -clib_error_t *dpdk_set_mc_filter (vnet_hw_interface_t * hi, - struct ether_addr mc_addr_vec[], int naddr); - -void dpdk_thread_input (dpdk_main_t * dm, dpdk_device_t * xd); - -clib_error_t *dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd); - -u32 dpdk_interface_tx_vector (vlib_main_t * vm, u32 dev_instance); - -struct rte_mbuf *dpdk_replicate_packet_mb (vlib_buffer_t * b); -struct rte_mbuf *dpdk_zerocopy_replicate_packet_mb (vlib_buffer_t * b); - -#define foreach_dpdk_error \ - _(NONE, "no error") \ - _(RX_PACKET_ERROR, "Rx packet errors") \ - _(RX_BAD_FCS, "Rx bad fcs") \ - _(IP_CHECKSUM_ERROR, "Rx ip checksum errors") \ - _(RX_ALLOC_FAIL, "rx buf alloc from free list failed") \ - _(RX_ALLOC_NO_PHYSMEM, "rx buf alloc failed no physmem") \ - _(RX_ALLOC_DROP_PKTS, "rx packets dropped due to alloc error") - -typedef enum -{ -#define _(f,s) DPDK_ERROR_##f, - foreach_dpdk_error -#undef _ - DPDK_N_ERROR, -} dpdk_error_t; - -int dpdk_set_stat_poll_interval (f64 interval); -int dpdk_set_link_state_poll_interval (f64 interval); -void dpdk_update_link_state (dpdk_device_t * xd, f64 now); -void dpdk_device_lock_init (dpdk_device_t * xd); -void dpdk_device_lock_free (dpdk_device_t * xd); - -static inline u64 -vnet_get_aggregate_rx_packets (void) -{ - dpdk_main_t *dm = &dpdk_main; - u64 sum = 0; - dpdk_worker_t *dw; - - vec_foreach (dw, dm->workers) sum += dw->aggregate_rx_packets; - - return sum; -} - -void dpdk_rx_trace (dpdk_main_t * dm, - vlib_node_runtime_t * node, - dpdk_device_t * xd, - u16 queue_id, u32 * buffers, uword n_buffers); - -#define EFD_OPERATION_LESS_THAN 0 -#define EFD_OPERATION_GREATER_OR_EQUAL 1 - -format_function_t format_dpdk_device_name; -format_function_t format_dpdk_device; -format_function_t format_dpdk_tx_dma_trace; -format_function_t format_dpdk_rx_dma_trace; -format_function_t format_dpdk_rte_mbuf; -format_function_t format_dpdk_rx_rte_mbuf; -unformat_function_t unformat_socket_mem; -clib_error_t *unformat_rss_fn (unformat_input_t * input, uword * rss_fn); -clib_error_t *unformat_hqos (unformat_input_t * input, - dpdk_device_config_hqos_t * hqos); - -uword -admin_up_down_process (vlib_main_t * vm, - vlib_node_runtime_t * rt, vlib_frame_t * f); - -#endif /* __included_dpdk_h__ */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/dpdk_priv.h b/vnet/vnet/devices/dpdk/dpdk_priv.h deleted file mode 100644 index 0c81dbc3beb..00000000000 --- a/vnet/vnet/devices/dpdk/dpdk_priv.h +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define DPDK_NB_RX_DESC_DEFAULT 1024 -#define DPDK_NB_TX_DESC_DEFAULT 1024 -#define DPDK_NB_RX_DESC_VIRTIO 256 -#define DPDK_NB_TX_DESC_VIRTIO 256 - -#define I40E_DEV_ID_SFP_XL710 0x1572 -#define I40E_DEV_ID_QSFP_A 0x1583 -#define I40E_DEV_ID_QSFP_B 0x1584 -#define I40E_DEV_ID_QSFP_C 0x1585 -#define I40E_DEV_ID_10G_BASE_T 0x1586 -#define I40E_DEV_ID_VF 0x154C - -/* These args appear by themselves */ -#define foreach_eal_double_hyphen_predicate_arg \ -_(no-shconf) \ -_(no-hpet) \ -_(no-huge) \ -_(vmware-tsc-map) - -#define foreach_eal_single_hyphen_mandatory_arg \ -_(coremask, c) \ -_(nchannels, n) \ - -#define foreach_eal_single_hyphen_arg \ -_(blacklist, b) \ -_(mem-alloc-request, m) \ -_(force-ranks, r) - -/* These args are preceeded by "--" and followed by a single string */ -#define foreach_eal_double_hyphen_arg \ -_(huge-dir) \ -_(proc-type) \ -_(file-prefix) \ -_(vdev) - -static inline void -dpdk_get_xstats (dpdk_device_t * xd) -{ - int len; - if ((len = rte_eth_xstats_get (xd->device_index, NULL, 0)) > 0) - { - vec_validate (xd->xstats, len - 1); - vec_validate (xd->last_cleared_xstats, len - 1); - - len = - rte_eth_xstats_get (xd->device_index, xd->xstats, - vec_len (xd->xstats)); - - ASSERT (vec_len (xd->xstats) == len); - ASSERT (vec_len (xd->last_cleared_xstats) == len); - - _vec_len (xd->xstats) = len; - _vec_len (xd->last_cleared_xstats) = len; - - } -} - - -static inline void -dpdk_update_counters (dpdk_device_t * xd, f64 now) -{ - vlib_simple_counter_main_t *cm; - vnet_main_t *vnm = vnet_get_main (); - u32 my_cpu = os_get_cpu_number (); - u64 rxerrors, last_rxerrors; - - /* only update counters for PMD interfaces */ - if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0) - return; - - xd->time_last_stats_update = now ? now : xd->time_last_stats_update; - clib_memcpy (&xd->last_stats, &xd->stats, sizeof (xd->last_stats)); - rte_eth_stats_get (xd->device_index, &xd->stats); - - /* maybe bump interface rx no buffer counter */ - if (PREDICT_FALSE (xd->stats.rx_nombuf != xd->last_stats.rx_nombuf)) - { - cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, - VNET_INTERFACE_COUNTER_RX_NO_BUF); - - vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index, - xd->stats.rx_nombuf - - xd->last_stats.rx_nombuf); - } - - /* missed pkt counter */ - if (PREDICT_FALSE (xd->stats.imissed != xd->last_stats.imissed)) - { - cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, - VNET_INTERFACE_COUNTER_RX_MISS); - - vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index, - xd->stats.imissed - - xd->last_stats.imissed); - } - rxerrors = xd->stats.ierrors; - last_rxerrors = xd->last_stats.ierrors; - - if (PREDICT_FALSE (rxerrors != last_rxerrors)) - { - cm = vec_elt_at_index (vnm->interface_main.sw_if_counters, - VNET_INTERFACE_COUNTER_RX_ERROR); - - vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index, - rxerrors - last_rxerrors); - } - - dpdk_get_xstats (xd); -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/format.c b/vnet/vnet/devices/dpdk/format.c deleted file mode 100644 index ff7c7a5a41c..00000000000 --- a/vnet/vnet/devices/dpdk/format.c +++ /dev/null @@ -1,763 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <vnet/vnet.h> -#include <vppinfra/vec.h> -#include <vppinfra/format.h> -#include <vlib/unix/cj.h> -#include <assert.h> - -#include <vnet/ethernet/ethernet.h> -#include <vnet/devices/dpdk/dpdk.h> - -#include "dpdk_priv.h" -#include <vppinfra/error.h> - -#define foreach_dpdk_counter \ - _ (tx_frames_ok, opackets) \ - _ (tx_bytes_ok, obytes) \ - _ (tx_errors, oerrors) \ - _ (rx_frames_ok, ipackets) \ - _ (rx_bytes_ok, ibytes) \ - _ (rx_errors, ierrors) \ - _ (rx_missed, imissed) \ - _ (rx_no_bufs, rx_nombuf) - -#define foreach_dpdk_q_counter \ - _ (rx_frames_ok, q_ipackets) \ - _ (tx_frames_ok, q_opackets) \ - _ (rx_bytes_ok, q_ibytes) \ - _ (tx_bytes_ok, q_obytes) \ - _ (rx_errors, q_errors) - -#define foreach_dpdk_rss_hf \ - _(ETH_RSS_FRAG_IPV4, "ipv4-frag") \ - _(ETH_RSS_NONFRAG_IPV4_TCP, "ipv4-tcp") \ - _(ETH_RSS_NONFRAG_IPV4_UDP, "ipv4-udp") \ - _(ETH_RSS_NONFRAG_IPV4_SCTP, "ipv4-sctp") \ - _(ETH_RSS_NONFRAG_IPV4_OTHER, "ipv4-other") \ - _(ETH_RSS_IPV4, "ipv4") \ - _(ETH_RSS_IPV6_TCP_EX, "ipv6-tcp-ex") \ - _(ETH_RSS_IPV6_UDP_EX, "ipv6-udp-ex") \ - _(ETH_RSS_FRAG_IPV6, "ipv6-frag") \ - _(ETH_RSS_NONFRAG_IPV6_TCP, "ipv6-tcp") \ - _(ETH_RSS_NONFRAG_IPV6_UDP, "ipv6-udp") \ - _(ETH_RSS_NONFRAG_IPV6_SCTP, "ipv6-sctp") \ - _(ETH_RSS_NONFRAG_IPV6_OTHER, "ipv6-other") \ - _(ETH_RSS_L2_PAYLOAD, "l2-payload") \ - _(ETH_RSS_IPV6_EX, "ipv6-ex") \ - _(ETH_RSS_IPV6, "ipv6") - - -#define foreach_dpdk_rx_offload_caps \ - _(DEV_RX_OFFLOAD_VLAN_STRIP, "vlan-strip") \ - _(DEV_RX_OFFLOAD_IPV4_CKSUM, "ipv4-cksum") \ - _(DEV_RX_OFFLOAD_UDP_CKSUM , "udp-cksum") \ - _(DEV_RX_OFFLOAD_TCP_CKSUM , "tcp-cksum") \ - _(DEV_RX_OFFLOAD_TCP_LRO , "rcp-lro") \ - _(DEV_RX_OFFLOAD_QINQ_STRIP, "qinq-strip") - -#define foreach_dpdk_tx_offload_caps \ - _(DEV_TX_OFFLOAD_VLAN_INSERT, "vlan-insert") \ - _(DEV_TX_OFFLOAD_IPV4_CKSUM, "ipv4-cksum") \ - _(DEV_TX_OFFLOAD_UDP_CKSUM , "udp-cksum") \ - _(DEV_TX_OFFLOAD_TCP_CKSUM , "tcp-cksum") \ - _(DEV_TX_OFFLOAD_SCTP_CKSUM , "sctp-cksum") \ - _(DEV_TX_OFFLOAD_TCP_TSO , "tcp-tso") \ - _(DEV_TX_OFFLOAD_UDP_TSO , "udp-tso") \ - _(DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM, "outer-ipv4-cksum") \ - _(DEV_TX_OFFLOAD_QINQ_INSERT, "qinq-insert") - -#if RTE_VERSION < RTE_VERSION_NUM(16, 11, 0, 0) -/* New ol_flags bits added in DPDK-16.11 */ -#define PKT_RX_IP_CKSUM_GOOD (1ULL << 7) -#define PKT_RX_L4_CKSUM_GOOD (1ULL << 8) -#endif - -#define foreach_dpdk_pkt_rx_offload_flag \ - _ (PKT_RX_VLAN_PKT, "RX packet is a 802.1q VLAN packet") \ - _ (PKT_RX_RSS_HASH, "RX packet with RSS hash result") \ - _ (PKT_RX_FDIR, "RX packet with FDIR infos") \ - _ (PKT_RX_L4_CKSUM_BAD, "L4 cksum of RX pkt. is not OK") \ - _ (PKT_RX_IP_CKSUM_BAD, "IP cksum of RX pkt. is not OK") \ - _ (PKT_RX_VLAN_STRIPPED, "RX packet VLAN tag stripped") \ - _ (PKT_RX_IP_CKSUM_GOOD, "IP cksum of RX pkt. is valid") \ - _ (PKT_RX_L4_CKSUM_GOOD, "L4 cksum of RX pkt. is valid") \ - _ (PKT_RX_IEEE1588_PTP, "RX IEEE1588 L2 Ethernet PT Packet") \ - _ (PKT_RX_IEEE1588_TMST, "RX IEEE1588 L2/L4 timestamped packet") \ - _ (PKT_RX_QINQ_STRIPPED, "RX packet QinQ tags stripped") - -#if RTE_VERSION < RTE_VERSION_NUM(16, 11, 0, 0) -/* PTYPE added in DPDK-16.11 */ -#define RTE_PTYPE_L2_ETHER_VLAN 0x00000006 -#define RTE_PTYPE_L2_ETHER_QINQ 0x00000007 -#endif - -#define foreach_dpdk_pkt_type \ - _ (L2, ETHER, "Ethernet packet") \ - _ (L2, ETHER_TIMESYNC, "Ethernet packet for time sync") \ - _ (L2, ETHER_ARP, "ARP packet") \ - _ (L2, ETHER_LLDP, "LLDP (Link Layer Discovery Protocol) packet") \ - _ (L2, ETHER_NSH, "NSH (Network Service Header) packet") \ - _ (L2, ETHER_VLAN, "VLAN packet") \ - _ (L2, ETHER_QINQ, "QinQ packet") \ - _ (L3, IPV4, "IPv4 packet without extension headers") \ - _ (L3, IPV4_EXT, "IPv4 packet with extension headers") \ - _ (L3, IPV4_EXT_UNKNOWN, "IPv4 packet with or without extension headers") \ - _ (L3, IPV6, "IPv6 packet without extension headers") \ - _ (L3, IPV6_EXT, "IPv6 packet with extension headers") \ - _ (L3, IPV6_EXT_UNKNOWN, "IPv6 packet with or without extension headers") \ - _ (L4, TCP, "TCP packet") \ - _ (L4, UDP, "UDP packet") \ - _ (L4, FRAG, "Fragmented IP packet") \ - _ (L4, SCTP, "SCTP (Stream Control Transmission Protocol) packet") \ - _ (L4, ICMP, "ICMP packet") \ - _ (L4, NONFRAG, "Non-fragmented IP packet") \ - _ (TUNNEL, GRE, "GRE tunneling packet") \ - _ (TUNNEL, VXLAN, "VXLAN tunneling packet") \ - _ (TUNNEL, NVGRE, "NVGRE Tunneling packet") \ - _ (TUNNEL, GENEVE, "GENEVE Tunneling packet") \ - _ (TUNNEL, GRENAT, "Teredo, VXLAN or GRE Tunneling packet") \ - _ (INNER_L2, ETHER, "Inner Ethernet packet") \ - _ (INNER_L2, ETHER_VLAN, "Inner Ethernet packet with VLAN") \ - _ (INNER_L3, IPV4, "Inner IPv4 packet without extension headers") \ - _ (INNER_L3, IPV4_EXT, "Inner IPv4 packet with extension headers") \ - _ (INNER_L3, IPV4_EXT_UNKNOWN, "Inner IPv4 packet with or without extension headers") \ - _ (INNER_L3, IPV6, "Inner IPv6 packet without extension headers") \ - _ (INNER_L3, IPV6_EXT, "Inner IPv6 packet with extension headers") \ - _ (INNER_L3, IPV6_EXT_UNKNOWN, "Inner IPv6 packet with or without extension headers") \ - _ (INNER_L4, TCP, "Inner TCP packet") \ - _ (INNER_L4, UDP, "Inner UDP packet") \ - _ (INNER_L4, FRAG, "Inner fagmented IP packet") \ - _ (INNER_L4, SCTP, "Inner SCTP (Stream Control Transmission Protocol) packet") \ - _ (INNER_L4, ICMP, "Inner ICMP packet") \ - _ (INNER_L4, NONFRAG, "Inner non-fragmented IP packet") - -#define foreach_dpdk_pkt_tx_offload_flag \ - _ (PKT_TX_VLAN_PKT, "TX packet is a 802.1q VLAN packet") \ - _ (PKT_TX_IP_CKSUM, "IP cksum of TX pkt. computed by NIC") \ - _ (PKT_TX_TCP_CKSUM, "TCP cksum of TX pkt. computed by NIC") \ - _ (PKT_TX_SCTP_CKSUM, "SCTP cksum of TX pkt. computed by NIC") \ - _ (PKT_TX_IEEE1588_TMST, "TX IEEE1588 packet to timestamp") - -#define foreach_dpdk_pkt_offload_flag \ - foreach_dpdk_pkt_rx_offload_flag \ - foreach_dpdk_pkt_tx_offload_flag - -u8 * -format_dpdk_device_name (u8 * s, va_list * args) -{ - dpdk_main_t *dm = &dpdk_main; - char *devname_format; - char *device_name; - u32 i = va_arg (*args, u32); - struct rte_eth_dev_info dev_info; - u8 *ret; - - if (dm->conf->interface_name_format_decimal) - devname_format = "%s%d/%d/%d"; - else - devname_format = "%s%x/%x/%x"; - - switch (dm->devices[i].port_type) - { - case VNET_DPDK_PORT_TYPE_ETH_1G: - device_name = "GigabitEthernet"; - break; - - case VNET_DPDK_PORT_TYPE_ETH_10G: - device_name = "TenGigabitEthernet"; - break; - - case VNET_DPDK_PORT_TYPE_ETH_40G: - device_name = "FortyGigabitEthernet"; - break; - - case VNET_DPDK_PORT_TYPE_ETH_100G: - device_name = "HundredGigabitEthernet"; - break; - - case VNET_DPDK_PORT_TYPE_ETH_BOND: - return format (s, "BondEthernet%d", dm->devices[i].device_index); - - case VNET_DPDK_PORT_TYPE_ETH_SWITCH: - device_name = "EthernetSwitch"; - break; - - case VNET_DPDK_PORT_TYPE_AF_PACKET: - rte_eth_dev_info_get (i, &dev_info); - return format (s, "af_packet%d", dm->devices[i].af_packet_port_id); - - default: - case VNET_DPDK_PORT_TYPE_UNKNOWN: - device_name = "UnknownEthernet"; - break; - } - - rte_eth_dev_info_get (i, &dev_info); - - if (dev_info.pci_dev) - ret = format (s, devname_format, device_name, dev_info.pci_dev->addr.bus, - dev_info.pci_dev->addr.devid, - dev_info.pci_dev->addr.function); - else - ret = format (s, "%s%d", device_name, dm->devices[i].device_index); - - if (dm->devices[i].interface_name_suffix) - return format (ret, "/%s", dm->devices[i].interface_name_suffix); - return ret; -} - -static u8 * -format_dpdk_device_type (u8 * s, va_list * args) -{ - dpdk_main_t *dm = &dpdk_main; - char *dev_type; - u32 i = va_arg (*args, u32); - - switch (dm->devices[i].pmd) - { - case VNET_DPDK_PMD_E1000EM: - dev_type = "Intel 82540EM (e1000)"; - break; - - case VNET_DPDK_PMD_IGB: - dev_type = "Intel e1000"; - break; - - case VNET_DPDK_PMD_I40E: - dev_type = "Intel X710/XL710 Family"; - break; - - case VNET_DPDK_PMD_I40EVF: - dev_type = "Intel X710/XL710 Family VF"; - break; - - case VNET_DPDK_PMD_FM10K: - dev_type = "Intel FM10000 Family Ethernet Switch"; - break; - - case VNET_DPDK_PMD_IGBVF: - dev_type = "Intel e1000 VF"; - break; - - case VNET_DPDK_PMD_VIRTIO: - dev_type = "Red Hat Virtio"; - break; - - case VNET_DPDK_PMD_IXGBEVF: - dev_type = "Intel 82599 VF"; - break; - - case VNET_DPDK_PMD_IXGBE: - dev_type = "Intel 82599"; - break; - - case VNET_DPDK_PMD_ENIC: - dev_type = "Cisco VIC"; - break; - - case VNET_DPDK_PMD_CXGBE: - dev_type = "Chelsio T4/T5"; - break; - - case VNET_DPDK_PMD_MLX5: - dev_type = "Mellanox ConnectX-4 Family"; - break; - - case VNET_DPDK_PMD_VMXNET3: - dev_type = "VMware VMXNET3"; - break; - - case VNET_DPDK_PMD_AF_PACKET: - dev_type = "af_packet"; - break; - - case VNET_DPDK_PMD_BOND: - dev_type = "Ethernet Bonding"; - break; - - case VNET_DPDK_PMD_DPAA2: - dev_type = "NXP DPAA2 Mac"; - break; - - default: - case VNET_DPDK_PMD_UNKNOWN: - dev_type = "### UNKNOWN ###"; - break; - } - - return format (s, dev_type); -} - -static u8 * -format_dpdk_link_status (u8 * s, va_list * args) -{ - dpdk_device_t *xd = va_arg (*args, dpdk_device_t *); - struct rte_eth_link *l = &xd->link; - vnet_main_t *vnm = vnet_get_main (); - vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, xd->vlib_hw_if_index); - - s = format (s, "%s ", l->link_status ? "up" : "down"); - if (l->link_status) - { - u32 promisc = rte_eth_promiscuous_get (xd->device_index); - - s = format (s, "%s duplex ", (l->link_duplex == ETH_LINK_FULL_DUPLEX) ? - "full" : "half"); - s = format (s, "speed %u mtu %d %s\n", l->link_speed, - hi->max_packet_bytes, promisc ? " promisc" : ""); - } - else - s = format (s, "\n"); - - return s; -} - -#define _line_len 72 -#define _(v, str) \ -if (bitmap & v) { \ - if (format_get_indent (s) > next_split ) { \ - next_split += _line_len; \ - s = format(s,"\n%U", format_white_space, indent); \ - } \ - s = format(s, "%s ", str); \ -} - -static u8 * -format_dpdk_rss_hf_name (u8 * s, va_list * args) -{ - u64 bitmap = va_arg (*args, u64); - int next_split = _line_len; - int indent = format_get_indent (s); - - if (!bitmap) - return format (s, "none"); - - foreach_dpdk_rss_hf return s; -} - -static u8 * -format_dpdk_rx_offload_caps (u8 * s, va_list * args) -{ - u32 bitmap = va_arg (*args, u32); - int next_split = _line_len; - int indent = format_get_indent (s); - - if (!bitmap) - return format (s, "none"); - - foreach_dpdk_rx_offload_caps return s; -} - -static u8 * -format_dpdk_tx_offload_caps (u8 * s, va_list * args) -{ - u32 bitmap = va_arg (*args, u32); - int next_split = _line_len; - int indent = format_get_indent (s); - if (!bitmap) - return format (s, "none"); - - foreach_dpdk_tx_offload_caps return s; -} - -#undef _line_len -#undef _ - -u8 * -format_dpdk_device (u8 * s, va_list * args) -{ - u32 dev_instance = va_arg (*args, u32); - int verbose = va_arg (*args, int); - dpdk_main_t *dm = &dpdk_main; - dpdk_device_t *xd = vec_elt_at_index (dm->devices, dev_instance); - uword indent = format_get_indent (s); - f64 now = vlib_time_now (dm->vlib_main); - struct rte_eth_dev_info di; - - dpdk_update_counters (xd, now); - dpdk_update_link_state (xd, now); - - s = format (s, "%U\n%Ucarrier %U", - format_dpdk_device_type, xd->device_index, - format_white_space, indent + 2, format_dpdk_link_status, xd); - - rte_eth_dev_info_get (xd->device_index, &di); - - if (verbose > 1 && xd->flags & DPDK_DEVICE_FLAG_PMD) - { - struct rte_pci_device *pci; - struct rte_eth_rss_conf rss_conf; - int vlan_off; - int retval; - - rss_conf.rss_key = 0; - retval = rte_eth_dev_rss_hash_conf_get (xd->device_index, &rss_conf); - if (retval < 0) - clib_warning ("rte_eth_dev_rss_hash_conf_get returned %d", retval); - pci = di.pci_dev; - - if (pci) - s = - format (s, - "%Upci id: device %04x:%04x subsystem %04x:%04x\n" - "%Upci address: %04x:%02x:%02x.%02x\n", - format_white_space, indent + 2, pci->id.vendor_id, - pci->id.device_id, pci->id.subsystem_vendor_id, - pci->id.subsystem_device_id, format_white_space, indent + 2, - pci->addr.domain, pci->addr.bus, pci->addr.devid, - pci->addr.function); - s = - format (s, "%Umax rx packet len: %d\n", format_white_space, - indent + 2, di.max_rx_pktlen); - s = - format (s, "%Umax num of queues: rx %d tx %d\n", format_white_space, - indent + 2, di.max_rx_queues, di.max_tx_queues); - s = - format (s, "%Upromiscuous: unicast %s all-multicast %s\n", - format_white_space, indent + 2, - rte_eth_promiscuous_get (xd->device_index) ? "on" : "off", - rte_eth_promiscuous_get (xd->device_index) ? "on" : "off"); - vlan_off = rte_eth_dev_get_vlan_offload (xd->device_index); - s = format (s, "%Uvlan offload: strip %s filter %s qinq %s\n", - format_white_space, indent + 2, - vlan_off & ETH_VLAN_STRIP_OFFLOAD ? "on" : "off", - vlan_off & ETH_VLAN_FILTER_OFFLOAD ? "on" : "off", - vlan_off & ETH_VLAN_EXTEND_OFFLOAD ? "on" : "off"); - s = format (s, "%Urx offload caps: %U\n", - format_white_space, indent + 2, - format_dpdk_rx_offload_caps, di.rx_offload_capa); - s = format (s, "%Utx offload caps: %U\n", - format_white_space, indent + 2, - format_dpdk_tx_offload_caps, di.tx_offload_capa); - s = format (s, "%Urss active: %U\n" - "%Urss supported: %U\n", - format_white_space, indent + 2, - format_dpdk_rss_hf_name, rss_conf.rss_hf, - format_white_space, indent + 2, - format_dpdk_rss_hf_name, di.flow_type_rss_offloads); - } - - s = format (s, "%Urx queues %d, rx desc %d, tx queues %d, tx desc %d\n", - format_white_space, indent + 2, - xd->rx_q_used, xd->nb_rx_desc, xd->tx_q_used, xd->nb_tx_desc); - - if (xd->cpu_socket > -1) - s = format (s, "%Ucpu socket %d\n", - format_white_space, indent + 2, xd->cpu_socket); - - /* $$$ MIB counters */ - { -#define _(N, V) \ - if ((xd->stats.V - xd->last_cleared_stats.V) != 0) { \ - s = format (s, "\n%U%-40U%16Ld", \ - format_white_space, indent + 2, \ - format_c_identifier, #N, \ - xd->stats.V - xd->last_cleared_stats.V); \ - } \ - - foreach_dpdk_counter -#undef _ - } - - u8 *xs = 0; - u32 i = 0; - struct rte_eth_xstat *xstat, *last_xstat; - struct rte_eth_xstat_name *xstat_names = 0; - int len = rte_eth_xstats_get_names (xd->device_index, NULL, 0); - vec_validate (xstat_names, len - 1); - rte_eth_xstats_get_names (xd->device_index, xstat_names, len); - - ASSERT (vec_len (xd->xstats) == vec_len (xd->last_cleared_xstats)); - - /* *INDENT-OFF* */ - vec_foreach_index(i, xd->xstats) - { - u64 delta = 0; - xstat = vec_elt_at_index(xd->xstats, i); - last_xstat = vec_elt_at_index(xd->last_cleared_xstats, i); - - delta = xstat->value - last_xstat->value; - if (verbose == 2 || (verbose && delta)) - { - /* format_c_identifier doesn't like c strings inside vector */ - u8 * name = format(0,"%s", xstat_names[i].name); - xs = format(xs, "\n%U%-38U%16Ld", - format_white_space, indent + 4, - format_c_identifier, name, delta); - vec_free(name); - } - } - /* *INDENT-ON* */ - - vec_free (xstat_names); - - if (xs) - { - s = format (s, "\n%Uextended stats:%v", - format_white_space, indent + 2, xs); - vec_free (xs); - } - - return s; -} - -u8 * -format_dpdk_tx_dma_trace (u8 * s, va_list * va) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *); - CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main (); - dpdk_tx_dma_trace_t *t = va_arg (*va, dpdk_tx_dma_trace_t *); - dpdk_main_t *dm = &dpdk_main; - dpdk_device_t *xd = vec_elt_at_index (dm->devices, t->device_index); - uword indent = format_get_indent (s); - vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->vlib_sw_if_index); - - s = format (s, "%U tx queue %d", - format_vnet_sw_interface_name, vnm, sw, t->queue_index); - - s = format (s, "\n%Ubuffer 0x%x: %U", - format_white_space, indent, - t->buffer_index, format_vlib_buffer, &t->buffer); - - s = format (s, "\n%U%U", format_white_space, indent, - format_ethernet_header_with_length, t->buffer.pre_data, - sizeof (t->buffer.pre_data)); - - return s; -} - -u8 * -format_dpdk_rx_dma_trace (u8 * s, va_list * va) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *); - CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main (); - dpdk_rx_dma_trace_t *t = va_arg (*va, dpdk_rx_dma_trace_t *); - dpdk_main_t *dm = &dpdk_main; - dpdk_device_t *xd = vec_elt_at_index (dm->devices, t->device_index); - format_function_t *f; - uword indent = format_get_indent (s); - vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->vlib_sw_if_index); - - s = format (s, "%U rx queue %d", - format_vnet_sw_interface_name, vnm, sw, t->queue_index); - - s = format (s, "\n%Ubuffer 0x%x: %U", - format_white_space, indent, - t->buffer_index, format_vlib_buffer, &t->buffer); - - s = format (s, "\n%U%U", - format_white_space, indent, - format_dpdk_rte_mbuf, &t->mb, &t->data); - - if (vm->trace_main.verbose) - { - s = format (s, "\n%UPacket Dump%s", format_white_space, indent + 2, - t->mb.data_len > sizeof (t->data) ? " (truncated)" : ""); - s = format (s, "\n%U%U", format_white_space, indent + 4, - format_hexdump, &t->data, - t->mb.data_len > - sizeof (t->data) ? sizeof (t->data) : t->mb.data_len); - } - f = node->format_buffer; - if (!f) - f = format_hex_bytes; - s = format (s, "\n%U%U", format_white_space, indent, - f, t->buffer.pre_data, sizeof (t->buffer.pre_data)); - - return s; -} - - -static inline u8 * -format_dpdk_pkt_types (u8 * s, va_list * va) -{ - u32 *pkt_types = va_arg (*va, u32 *); - uword indent __attribute__ ((unused)) = format_get_indent (s) + 2; - - if (!*pkt_types) - return s; - - s = format (s, "Packet Types"); - -#define _(L, F, S) \ - if ((*pkt_types & RTE_PTYPE_##L##_MASK) == RTE_PTYPE_##L##_##F) \ - { \ - s = format (s, "\n%U%s (0x%04x) %s", format_white_space, indent, \ - "RTE_PTYPE_" #L "_" #F, RTE_PTYPE_##L##_##F, S); \ - } - - foreach_dpdk_pkt_type -#undef _ - return s; -} - -static inline u8 * -format_dpdk_pkt_offload_flags (u8 * s, va_list * va) -{ - u64 *ol_flags = va_arg (*va, u64 *); - uword indent = format_get_indent (s) + 2; - - if (!*ol_flags) - return s; - - s = format (s, "Packet Offload Flags"); - -#define _(F, S) \ - if (*ol_flags & F) \ - { \ - s = format (s, "\n%U%s (0x%04x) %s", \ - format_white_space, indent, #F, F, S); \ - } - - foreach_dpdk_pkt_offload_flag -#undef _ - return s; -} - -u8 * -format_dpdk_rte_mbuf_vlan (u8 * s, va_list * va) -{ - ethernet_vlan_header_tv_t *vlan_hdr = - va_arg (*va, ethernet_vlan_header_tv_t *); - - if (clib_net_to_host_u16 (vlan_hdr->type) == ETHERNET_TYPE_DOT1AD) - { - s = format (s, "%U 802.1q vlan ", - format_ethernet_vlan_tci, - clib_net_to_host_u16 (vlan_hdr->priority_cfi_and_id)); - vlan_hdr++; - } - - s = format (s, "%U", - format_ethernet_vlan_tci, - clib_net_to_host_u16 (vlan_hdr->priority_cfi_and_id)); - - return s; -} - -u8 * -format_dpdk_rte_mbuf (u8 * s, va_list * va) -{ - struct rte_mbuf *mb = va_arg (*va, struct rte_mbuf *); - ethernet_header_t *eth_hdr = va_arg (*va, ethernet_header_t *); - uword indent = format_get_indent (s) + 2; - - s = format (s, "PKT MBUF: port %d, nb_segs %d, pkt_len %d" - "\n%Ubuf_len %d, data_len %d, ol_flags 0x%x, data_off %d, phys_addr 0x%x" - "\n%Upacket_type 0x%x", - mb->port, mb->nb_segs, mb->pkt_len, - format_white_space, indent, - mb->buf_len, mb->data_len, mb->ol_flags, mb->data_off, - mb->buf_physaddr, format_white_space, indent, mb->packet_type); - - if (mb->ol_flags) - s = format (s, "\n%U%U", format_white_space, indent, - format_dpdk_pkt_offload_flags, &mb->ol_flags); - - if ((mb->ol_flags & PKT_RX_VLAN_PKT) && - ((mb->ol_flags & (PKT_RX_VLAN_STRIPPED | PKT_RX_QINQ_STRIPPED)) == 0)) - { - ethernet_vlan_header_tv_t *vlan_hdr = - ((ethernet_vlan_header_tv_t *) & (eth_hdr->type)); - s = format (s, " %U", format_dpdk_rte_mbuf_vlan, vlan_hdr); - } - - if (mb->packet_type) - s = format (s, "\n%U%U", format_white_space, indent, - format_dpdk_pkt_types, &mb->packet_type); - - return s; -} - -uword -unformat_socket_mem (unformat_input_t * input, va_list * va) -{ - uword **r = va_arg (*va, uword **); - int i = 0; - u32 mem; - - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, ",")) - hash_set (*r, i, 1024); - else if (unformat (input, "%u,", &mem)) - hash_set (*r, i, mem); - else if (unformat (input, "%u", &mem)) - hash_set (*r, i, mem); - else - { - unformat_put_input (input); - goto done; - } - i++; - } - -done: - return 1; -} - -clib_error_t * -unformat_rss_fn (unformat_input_t * input, uword * rss_fn) -{ - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (0) - ; -#undef _ -#define _(f, s) \ - else if (unformat (input, s)) \ - *rss_fn |= f; - - foreach_dpdk_rss_hf -#undef _ - else - { - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); - } - } - return 0; -} - -clib_error_t * -unformat_hqos (unformat_input_t * input, dpdk_device_config_hqos_t * hqos) -{ - clib_error_t *error = 0; - - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "hqos-thread %u", &hqos->hqos_thread)) - hqos->hqos_thread_valid = 1; - else - { - error = clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); - break; - } - } - - return error; -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/hqos.c b/vnet/vnet/devices/dpdk/hqos.c deleted file mode 100644 index d68bc48f80b..00000000000 --- a/vnet/vnet/devices/dpdk/hqos.c +++ /dev/null @@ -1,775 +0,0 @@ -/* - * Copyright(c) 2016 Intel Corporation. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <sys/stat.h> -#include <sys/mount.h> -#include <string.h> -#include <fcntl.h> - -#include <vppinfra/vec.h> -#include <vppinfra/error.h> -#include <vppinfra/format.h> -#include <vppinfra/bitmap.h> - -#include <vnet/vnet.h> -#include <vnet/ethernet/ethernet.h> -#include <vnet/devices/dpdk/dpdk.h> - -#include <vlib/unix/physmem.h> -#include <vlib/pci/pci.h> -#include <vlibmemory/api.h> -#include <vlibmemory/vl_memory_msg_enum.h> /* enumerate all vlib messages */ - -#define vl_typedefs /* define message structures */ -#include <vlibmemory/vl_memory_api_h.h> -#undef vl_typedefs - -/* instantiate all the print functions we know about */ -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) -#define vl_printfun -#include <vlibmemory/vl_memory_api_h.h> -#undef vl_printfun - -#include "dpdk_priv.h" - -dpdk_main_t dpdk_main; - -/*** - * - * HQoS default configuration values - * - ***/ - -static dpdk_device_config_hqos_t hqos_params_default = { - .hqos_thread_valid = 0, - - .swq_size = 4096, - .burst_enq = 256, - .burst_deq = 220, - - /* - * Packet field to identify the subport. - * - * Default value: Since only one subport is defined by default (see below: - * n_subports_per_port = 1), the subport ID is hardcoded to 0. - */ - .pktfield0_slabpos = 0, - .pktfield0_slabmask = 0, - - /* - * Packet field to identify the pipe. - * - * Default value: Assuming Ethernet/IPv4/UDP packets, UDP payload bits 12 .. 23 - */ - .pktfield1_slabpos = 40, - .pktfield1_slabmask = 0x0000000FFF000000LLU, - - /* Packet field used as index into TC translation table to identify the traffic - * class and queue. - * - * Default value: Assuming Ethernet/IPv4 packets, IPv4 DSCP field - */ - .pktfield2_slabpos = 8, - .pktfield2_slabmask = 0x00000000000000FCLLU, - .tc_table = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - }, - - /* port */ - .port = { - .name = NULL, /* Set at init */ - .socket = 0, /* Set at init */ - .rate = 1250000000, /* Assuming 10GbE port */ - .mtu = 14 + 1500, /* Assuming Ethernet/IPv4 pkt (Ethernet FCS not included) */ - .frame_overhead = RTE_SCHED_FRAME_OVERHEAD_DEFAULT, - .n_subports_per_port = 1, - .n_pipes_per_subport = 4096, - .qsize = {64, 64, 64, 64}, - .pipe_profiles = NULL, /* Set at config */ - .n_pipe_profiles = 1, - -#ifdef RTE_SCHED_RED - .red_params = { - /* Traffic Class 0 Colors Green / Yellow / Red */ - [0][0] = {.min_th = 48,.max_th = 64,.maxp_inv = - 10,.wq_log2 = 9}, - [0][1] = {.min_th = 40,.max_th = 64,.maxp_inv = - 10,.wq_log2 = 9}, - [0][2] = {.min_th = 32,.max_th = 64,.maxp_inv = - 10,.wq_log2 = 9}, - - /* Traffic Class 1 - Colors Green / Yellow / Red */ - [1][0] = {.min_th = 48,.max_th = 64,.maxp_inv = - 10,.wq_log2 = 9}, - [1][1] = {.min_th = 40,.max_th = 64,.maxp_inv = - 10,.wq_log2 = 9}, - [1][2] = {.min_th = 32,.max_th = 64,.maxp_inv = - 10,.wq_log2 = 9}, - - /* Traffic Class 2 - Colors Green / Yellow / Red */ - [2][0] = {.min_th = 48,.max_th = 64,.maxp_inv = - 10,.wq_log2 = 9}, - [2][1] = {.min_th = 40,.max_th = 64,.maxp_inv = - 10,.wq_log2 = 9}, - [2][2] = {.min_th = 32,.max_th = 64,.maxp_inv = - 10,.wq_log2 = 9}, - - /* Traffic Class 3 - Colors Green / Yellow / Red */ - [3][0] = {.min_th = 48,.max_th = 64,.maxp_inv = - 10,.wq_log2 = 9}, - [3][1] = {.min_th = 40,.max_th = 64,.maxp_inv = - 10,.wq_log2 = 9}, - [3][2] = {.min_th = 32,.max_th = 64,.maxp_inv = - 10,.wq_log2 = 9} - }, -#endif /* RTE_SCHED_RED */ - }, -}; - -static struct rte_sched_subport_params hqos_subport_params_default = { - .tb_rate = 1250000000, /* 10GbE line rate (measured in bytes/second) */ - .tb_size = 1000000, - .tc_rate = {1250000000, 1250000000, 1250000000, 1250000000}, - .tc_period = 10, -}; - -static struct rte_sched_pipe_params hqos_pipe_params_default = { - .tb_rate = 305175, /* 10GbE line rate divided by 4K pipes */ - .tb_size = 1000000, - .tc_rate = {305175, 305175, 305175, 305175}, - .tc_period = 40, -#ifdef RTE_SCHED_SUBPORT_TC_OV - .tc_ov_weight = 1, -#endif - .wrr_weights = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, -}; - -/*** - * - * HQoS configuration - * - ***/ - -int -dpdk_hqos_validate_mask (u64 mask, u32 n) -{ - int count = __builtin_popcountll (mask); - int pos_lead = sizeof (u64) * 8 - __builtin_clzll (mask); - int pos_trail = __builtin_ctzll (mask); - int count_expected = __builtin_popcount (n - 1); - - /* Handle the exceptions */ - if (n == 0) - return -1; /* Error */ - - if ((mask == 0) && (n == 1)) - return 0; /* OK */ - - if (((mask == 0) && (n != 1)) || ((mask != 0) && (n == 1))) - return -2; /* Error */ - - /* Check that mask is contiguous */ - if ((pos_lead - pos_trail) != count) - return -3; /* Error */ - - /* Check that mask contains the expected number of bits set */ - if (count != count_expected) - return -4; /* Error */ - - return 0; /* OK */ -} - -void -dpdk_device_config_hqos_pipe_profile_default (dpdk_device_config_hqos_t * - hqos, u32 pipe_profile_id) -{ - memcpy (&hqos->pipe[pipe_profile_id], &hqos_pipe_params_default, - sizeof (hqos_pipe_params_default)); -} - -void -dpdk_device_config_hqos_default (dpdk_device_config_hqos_t * hqos) -{ - struct rte_sched_subport_params *subport_params; - struct rte_sched_pipe_params *pipe_params; - u32 *pipe_map; - u32 i; - - memcpy (hqos, &hqos_params_default, sizeof (hqos_params_default)); - - /* pipe */ - vec_add2 (hqos->pipe, pipe_params, hqos->port.n_pipe_profiles); - - for (i = 0; i < vec_len (hqos->pipe); i++) - memcpy (&pipe_params[i], - &hqos_pipe_params_default, sizeof (hqos_pipe_params_default)); - - hqos->port.pipe_profiles = hqos->pipe; - - /* subport */ - vec_add2 (hqos->subport, subport_params, hqos->port.n_subports_per_port); - - for (i = 0; i < vec_len (hqos->subport); i++) - memcpy (&subport_params[i], - &hqos_subport_params_default, - sizeof (hqos_subport_params_default)); - - /* pipe profile */ - vec_add2 (hqos->pipe_map, - pipe_map, - hqos->port.n_subports_per_port * hqos->port.n_pipes_per_subport); - - for (i = 0; i < vec_len (hqos->pipe_map); i++) - pipe_map[i] = 0; -} - -/*** - * - * HQoS init - * - ***/ - -clib_error_t * -dpdk_port_setup_hqos (dpdk_device_t * xd, dpdk_device_config_hqos_t * hqos) -{ - vlib_thread_main_t *tm = vlib_get_thread_main (); - char name[32]; - u32 subport_id, i; - int rv; - - /* Detect the set of worker threads */ - int worker_thread_first = 0; - int worker_thread_count = 0; - - uword *p = hash_get_mem (tm->thread_registrations_by_name, "workers"); - vlib_thread_registration_t *tr = - p ? (vlib_thread_registration_t *) p[0] : 0; - - if (tr && tr->count > 0) - { - worker_thread_first = tr->first_index; - worker_thread_count = tr->count; - } - - /* Allocate the per-thread device data array */ - vec_validate_aligned (xd->hqos_wt, tm->n_vlib_mains - 1, - CLIB_CACHE_LINE_BYTES); - memset (xd->hqos_wt, 0, tm->n_vlib_mains * sizeof (xd->hqos_wt[0])); - - vec_validate_aligned (xd->hqos_ht, 0, CLIB_CACHE_LINE_BYTES); - memset (xd->hqos_ht, 0, sizeof (xd->hqos_ht[0])); - - /* Allocate space for one SWQ per worker thread in the I/O TX thread data structure */ - vec_validate (xd->hqos_ht->swq, worker_thread_count); - - /* SWQ */ - for (i = 0; i < worker_thread_count + 1; i++) - { - u32 swq_flags = RING_F_SP_ENQ | RING_F_SC_DEQ; - - snprintf (name, sizeof (name), "SWQ-worker%u-to-device%u", i, - xd->device_index); - xd->hqos_ht->swq[i] = - rte_ring_create (name, hqos->swq_size, xd->cpu_socket, swq_flags); - if (xd->hqos_ht->swq[i] == NULL) - return clib_error_return (0, - "SWQ-worker%u-to-device%u: rte_ring_create err", - i, xd->device_index); - } - - /* - * HQoS - */ - - /* HQoS port */ - snprintf (name, sizeof (name), "HQoS%u", xd->device_index); - hqos->port.name = strdup (name); - if (hqos->port.name == NULL) - return clib_error_return (0, "HQoS%u: strdup err", xd->device_index); - - hqos->port.socket = rte_eth_dev_socket_id (xd->device_index); - if (hqos->port.socket == SOCKET_ID_ANY) - hqos->port.socket = 0; - - xd->hqos_ht->hqos = rte_sched_port_config (&hqos->port); - if (xd->hqos_ht->hqos == NULL) - return clib_error_return (0, "HQoS%u: rte_sched_port_config err", - xd->device_index); - - /* HQoS subport */ - for (subport_id = 0; subport_id < hqos->port.n_subports_per_port; - subport_id++) - { - u32 pipe_id; - - rv = - rte_sched_subport_config (xd->hqos_ht->hqos, subport_id, - &hqos->subport[subport_id]); - if (rv) - return clib_error_return (0, - "HQoS%u subport %u: rte_sched_subport_config err (%d)", - xd->device_index, subport_id, rv); - - /* HQoS pipe */ - for (pipe_id = 0; pipe_id < hqos->port.n_pipes_per_subport; pipe_id++) - { - u32 pos = subport_id * hqos->port.n_pipes_per_subport + pipe_id; - u32 profile_id = hqos->pipe_map[pos]; - - rv = - rte_sched_pipe_config (xd->hqos_ht->hqos, subport_id, pipe_id, - profile_id); - if (rv) - return clib_error_return (0, - "HQoS%u subport %u pipe %u: rte_sched_pipe_config err (%d)", - xd->device_index, subport_id, pipe_id, - rv); - } - } - - /* Set up per-thread device data for the I/O TX thread */ - xd->hqos_ht->hqos_burst_enq = hqos->burst_enq; - xd->hqos_ht->hqos_burst_deq = hqos->burst_deq; - vec_validate (xd->hqos_ht->pkts_enq, 2 * hqos->burst_enq - 1); - vec_validate (xd->hqos_ht->pkts_deq, hqos->burst_deq - 1); - xd->hqos_ht->pkts_enq_len = 0; - xd->hqos_ht->swq_pos = 0; - xd->hqos_ht->flush_count = 0; - - /* Set up per-thread device data for each worker thread */ - for (i = 0; i < worker_thread_count + 1; i++) - { - u32 tid; - if (i) - tid = worker_thread_first + (i - 1); - else - tid = i; - - xd->hqos_wt[tid].swq = xd->hqos_ht->swq[i]; - xd->hqos_wt[tid].hqos_field0_slabpos = hqos->pktfield0_slabpos; - xd->hqos_wt[tid].hqos_field0_slabmask = hqos->pktfield0_slabmask; - xd->hqos_wt[tid].hqos_field0_slabshr = - __builtin_ctzll (hqos->pktfield0_slabmask); - xd->hqos_wt[tid].hqos_field1_slabpos = hqos->pktfield1_slabpos; - xd->hqos_wt[tid].hqos_field1_slabmask = hqos->pktfield1_slabmask; - xd->hqos_wt[tid].hqos_field1_slabshr = - __builtin_ctzll (hqos->pktfield1_slabmask); - xd->hqos_wt[tid].hqos_field2_slabpos = hqos->pktfield2_slabpos; - xd->hqos_wt[tid].hqos_field2_slabmask = hqos->pktfield2_slabmask; - xd->hqos_wt[tid].hqos_field2_slabshr = - __builtin_ctzll (hqos->pktfield2_slabmask); - memcpy (xd->hqos_wt[tid].hqos_tc_table, hqos->tc_table, - sizeof (hqos->tc_table)); - } - - return 0; -} - -/*** - * - * HQoS run-time - * - ***/ -/* - * dpdk_hqos_thread - Contains the main loop of an HQoS thread. - * - * w - * Information for the current thread - */ -static_always_inline void -dpdk_hqos_thread_internal_hqos_dbg_bypass (vlib_main_t * vm) -{ - dpdk_main_t *dm = &dpdk_main; - u32 cpu_index = vm->cpu_index; - u32 dev_pos; - - dev_pos = 0; - while (1) - { - vlib_worker_thread_barrier_check (); - - u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]); - if (dev_pos >= n_devs) - dev_pos = 0; - - dpdk_device_and_queue_t *dq = - vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos); - dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device); - - dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht; - u32 device_index = xd->device_index; - u16 queue_id = dq->queue_id; - - struct rte_mbuf **pkts_enq = hqos->pkts_enq; - u32 pkts_enq_len = hqos->pkts_enq_len; - u32 swq_pos = hqos->swq_pos; - u32 n_swq = vec_len (hqos->swq), i; - u32 flush_count = hqos->flush_count; - - for (i = 0; i < n_swq; i++) - { - /* Get current SWQ for this device */ - struct rte_ring *swq = hqos->swq[swq_pos]; - - /* Read SWQ burst to packet buffer of this device */ - pkts_enq_len += rte_ring_sc_dequeue_burst (swq, - (void **) - &pkts_enq[pkts_enq_len], - hqos->hqos_burst_enq); - - /* Get next SWQ for this device */ - swq_pos++; - if (swq_pos >= n_swq) - swq_pos = 0; - hqos->swq_pos = swq_pos; - - /* HWQ TX enqueue when burst available */ - if (pkts_enq_len >= hqos->hqos_burst_enq) - { - u32 n_pkts = rte_eth_tx_burst (device_index, - (uint16_t) queue_id, - pkts_enq, - (uint16_t) pkts_enq_len); - - for (; n_pkts < pkts_enq_len; n_pkts++) - rte_pktmbuf_free (pkts_enq[n_pkts]); - - pkts_enq_len = 0; - flush_count = 0; - break; - } - } - if (pkts_enq_len) - { - flush_count++; - if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD)) - { - rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len); - - pkts_enq_len = 0; - flush_count = 0; - } - } - hqos->pkts_enq_len = pkts_enq_len; - hqos->flush_count = flush_count; - - /* Advance to next device */ - dev_pos++; - } -} - -static_always_inline void -dpdk_hqos_thread_internal (vlib_main_t * vm) -{ - dpdk_main_t *dm = &dpdk_main; - u32 cpu_index = vm->cpu_index; - u32 dev_pos; - - dev_pos = 0; - while (1) - { - vlib_worker_thread_barrier_check (); - - u32 n_devs = vec_len (dm->devices_by_hqos_cpu[cpu_index]); - if (PREDICT_FALSE (n_devs == 0)) - { - dev_pos = 0; - continue; - } - if (dev_pos >= n_devs) - dev_pos = 0; - - dpdk_device_and_queue_t *dq = - vec_elt_at_index (dm->devices_by_hqos_cpu[cpu_index], dev_pos); - dpdk_device_t *xd = vec_elt_at_index (dm->devices, dq->device); - - dpdk_device_hqos_per_hqos_thread_t *hqos = xd->hqos_ht; - u32 device_index = xd->device_index; - u16 queue_id = dq->queue_id; - - struct rte_mbuf **pkts_enq = hqos->pkts_enq; - struct rte_mbuf **pkts_deq = hqos->pkts_deq; - u32 pkts_enq_len = hqos->pkts_enq_len; - u32 swq_pos = hqos->swq_pos; - u32 n_swq = vec_len (hqos->swq), i; - u32 flush_count = hqos->flush_count; - - /* - * SWQ dequeue and HQoS enqueue for current device - */ - for (i = 0; i < n_swq; i++) - { - /* Get current SWQ for this device */ - struct rte_ring *swq = hqos->swq[swq_pos]; - - /* Read SWQ burst to packet buffer of this device */ - pkts_enq_len += rte_ring_sc_dequeue_burst (swq, - (void **) - &pkts_enq[pkts_enq_len], - hqos->hqos_burst_enq); - - /* Get next SWQ for this device */ - swq_pos++; - if (swq_pos >= n_swq) - swq_pos = 0; - hqos->swq_pos = swq_pos; - - /* HQoS enqueue when burst available */ - if (pkts_enq_len >= hqos->hqos_burst_enq) - { - rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len); - - pkts_enq_len = 0; - flush_count = 0; - break; - } - } - if (pkts_enq_len) - { - flush_count++; - if (PREDICT_FALSE (flush_count == HQOS_FLUSH_COUNT_THRESHOLD)) - { - rte_sched_port_enqueue (hqos->hqos, pkts_enq, pkts_enq_len); - - pkts_enq_len = 0; - flush_count = 0; - } - } - hqos->pkts_enq_len = pkts_enq_len; - hqos->flush_count = flush_count; - - /* - * HQoS dequeue and HWQ TX enqueue for current device - */ - { - u32 pkts_deq_len, n_pkts; - - pkts_deq_len = rte_sched_port_dequeue (hqos->hqos, - pkts_deq, - hqos->hqos_burst_deq); - - for (n_pkts = 0; n_pkts < pkts_deq_len;) - n_pkts += rte_eth_tx_burst (device_index, - (uint16_t) queue_id, - &pkts_deq[n_pkts], - (uint16_t) (pkts_deq_len - n_pkts)); - } - - /* Advance to next device */ - dev_pos++; - } -} - -void -dpdk_hqos_thread (vlib_worker_thread_t * w) -{ - vlib_main_t *vm; - vlib_thread_main_t *tm = vlib_get_thread_main (); - dpdk_main_t *dm = &dpdk_main; - - vm = vlib_get_main (); - - ASSERT (vm->cpu_index == os_get_cpu_number ()); - - clib_time_init (&vm->clib_time); - clib_mem_set_heap (w->thread_mheap); - - /* Wait until the dpdk init sequence is complete */ - while (tm->worker_thread_release == 0) - vlib_worker_thread_barrier_check (); - - if (vec_len (dm->devices_by_hqos_cpu[vm->cpu_index]) == 0) - return - clib_error - ("current I/O TX thread does not have any devices assigned to it"); - - if (DPDK_HQOS_DBG_BYPASS) - dpdk_hqos_thread_internal_hqos_dbg_bypass (vm); - else - dpdk_hqos_thread_internal (vm); -} - -void -dpdk_hqos_thread_fn (void *arg) -{ - vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; - vlib_worker_thread_init (w); - dpdk_hqos_thread (w); -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_THREAD (hqos_thread_reg, static) = -{ - .name = "hqos-threads", - .short_name = "hqos-threads", - .function = dpdk_hqos_thread_fn, -}; -/* *INDENT-ON* */ - -/* - * HQoS run-time code to be called by the worker threads - */ -#define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr) \ -({ \ - u64 slab = *((u64 *) &byte_array[slab_pos]); \ - u64 val = (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr; \ - val; \ -}) - -#define RTE_SCHED_PORT_HIERARCHY(subport, pipe, traffic_class, queue, color) \ - ((((u64) (queue)) & 0x3) | \ - ((((u64) (traffic_class)) & 0x3) << 2) | \ - ((((u64) (color)) & 0x3) << 4) | \ - ((((u64) (subport)) & 0xFFFF) << 16) | \ - ((((u64) (pipe)) & 0xFFFFFFFF) << 32)) - -void -dpdk_hqos_metadata_set (dpdk_device_hqos_per_worker_thread_t * hqos, - struct rte_mbuf **pkts, u32 n_pkts) -{ - u32 i; - - for (i = 0; i < (n_pkts & (~0x3)); i += 4) - { - struct rte_mbuf *pkt0 = pkts[i]; - struct rte_mbuf *pkt1 = pkts[i + 1]; - struct rte_mbuf *pkt2 = pkts[i + 2]; - struct rte_mbuf *pkt3 = pkts[i + 3]; - - u8 *pkt0_data = rte_pktmbuf_mtod (pkt0, u8 *); - u8 *pkt1_data = rte_pktmbuf_mtod (pkt1, u8 *); - u8 *pkt2_data = rte_pktmbuf_mtod (pkt2, u8 *); - u8 *pkt3_data = rte_pktmbuf_mtod (pkt3, u8 *); - - u64 pkt0_subport = BITFIELD (pkt0_data, hqos->hqos_field0_slabpos, - hqos->hqos_field0_slabmask, - hqos->hqos_field0_slabshr); - u64 pkt0_pipe = BITFIELD (pkt0_data, hqos->hqos_field1_slabpos, - hqos->hqos_field1_slabmask, - hqos->hqos_field1_slabshr); - u64 pkt0_dscp = BITFIELD (pkt0_data, hqos->hqos_field2_slabpos, - hqos->hqos_field2_slabmask, - hqos->hqos_field2_slabshr); - u32 pkt0_tc = hqos->hqos_tc_table[pkt0_dscp & 0x3F] >> 2; - u32 pkt0_tc_q = hqos->hqos_tc_table[pkt0_dscp & 0x3F] & 0x3; - - u64 pkt1_subport = BITFIELD (pkt1_data, hqos->hqos_field0_slabpos, - hqos->hqos_field0_slabmask, - hqos->hqos_field0_slabshr); - u64 pkt1_pipe = BITFIELD (pkt1_data, hqos->hqos_field1_slabpos, - hqos->hqos_field1_slabmask, - hqos->hqos_field1_slabshr); - u64 pkt1_dscp = BITFIELD (pkt1_data, hqos->hqos_field2_slabpos, - hqos->hqos_field2_slabmask, - hqos->hqos_field2_slabshr); - u32 pkt1_tc = hqos->hqos_tc_table[pkt1_dscp & 0x3F] >> 2; - u32 pkt1_tc_q = hqos->hqos_tc_table[pkt1_dscp & 0x3F] & 0x3; - - u64 pkt2_subport = BITFIELD (pkt2_data, hqos->hqos_field0_slabpos, - hqos->hqos_field0_slabmask, - hqos->hqos_field0_slabshr); - u64 pkt2_pipe = BITFIELD (pkt2_data, hqos->hqos_field1_slabpos, - hqos->hqos_field1_slabmask, - hqos->hqos_field1_slabshr); - u64 pkt2_dscp = BITFIELD (pkt2_data, hqos->hqos_field2_slabpos, - hqos->hqos_field2_slabmask, - hqos->hqos_field2_slabshr); - u32 pkt2_tc = hqos->hqos_tc_table[pkt2_dscp & 0x3F] >> 2; - u32 pkt2_tc_q = hqos->hqos_tc_table[pkt2_dscp & 0x3F] & 0x3; - - u64 pkt3_subport = BITFIELD (pkt3_data, hqos->hqos_field0_slabpos, - hqos->hqos_field0_slabmask, - hqos->hqos_field0_slabshr); - u64 pkt3_pipe = BITFIELD (pkt3_data, hqos->hqos_field1_slabpos, - hqos->hqos_field1_slabmask, - hqos->hqos_field1_slabshr); - u64 pkt3_dscp = BITFIELD (pkt3_data, hqos->hqos_field2_slabpos, - hqos->hqos_field2_slabmask, - hqos->hqos_field2_slabshr); - u32 pkt3_tc = hqos->hqos_tc_table[pkt3_dscp & 0x3F] >> 2; - u32 pkt3_tc_q = hqos->hqos_tc_table[pkt3_dscp & 0x3F] & 0x3; - - u64 pkt0_sched = RTE_SCHED_PORT_HIERARCHY (pkt0_subport, - pkt0_pipe, - pkt0_tc, - pkt0_tc_q, - 0); - u64 pkt1_sched = RTE_SCHED_PORT_HIERARCHY (pkt1_subport, - pkt1_pipe, - pkt1_tc, - pkt1_tc_q, - 0); - u64 pkt2_sched = RTE_SCHED_PORT_HIERARCHY (pkt2_subport, - pkt2_pipe, - pkt2_tc, - pkt2_tc_q, - 0); - u64 pkt3_sched = RTE_SCHED_PORT_HIERARCHY (pkt3_subport, - pkt3_pipe, - pkt3_tc, - pkt3_tc_q, - 0); - - pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF; - pkt0->hash.sched.hi = pkt0_sched >> 32; - pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF; - pkt1->hash.sched.hi = pkt1_sched >> 32; - pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF; - pkt2->hash.sched.hi = pkt2_sched >> 32; - pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF; - pkt3->hash.sched.hi = pkt3_sched >> 32; - } - - for (; i < n_pkts; i++) - { - struct rte_mbuf *pkt = pkts[i]; - - u8 *pkt_data = rte_pktmbuf_mtod (pkt, u8 *); - - u64 pkt_subport = BITFIELD (pkt_data, hqos->hqos_field0_slabpos, - hqos->hqos_field0_slabmask, - hqos->hqos_field0_slabshr); - u64 pkt_pipe = BITFIELD (pkt_data, hqos->hqos_field1_slabpos, - hqos->hqos_field1_slabmask, - hqos->hqos_field1_slabshr); - u64 pkt_dscp = BITFIELD (pkt_data, hqos->hqos_field2_slabpos, - hqos->hqos_field2_slabmask, - hqos->hqos_field2_slabshr); - u32 pkt_tc = hqos->hqos_tc_table[pkt_dscp & 0x3F] >> 2; - u32 pkt_tc_q = hqos->hqos_tc_table[pkt_dscp & 0x3F] & 0x3; - - u64 pkt_sched = RTE_SCHED_PORT_HIERARCHY (pkt_subport, - pkt_pipe, - pkt_tc, - pkt_tc_q, - 0); - - pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF; - pkt->hash.sched.hi = pkt_sched >> 32; - } -} - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/init.c b/vnet/vnet/devices/dpdk/init.c deleted file mode 100755 index 693ca985130..00000000000 --- a/vnet/vnet/devices/dpdk/init.c +++ /dev/null @@ -1,1803 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <vnet/vnet.h> -#include <vppinfra/vec.h> -#include <vppinfra/error.h> -#include <vppinfra/format.h> -#include <vppinfra/bitmap.h> - -#include <vnet/ethernet/ethernet.h> -#include <vnet/devices/dpdk/dpdk.h> -#include <vlib/unix/physmem.h> -#include <vlib/pci/pci.h> - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <sys/stat.h> -#include <sys/mount.h> -#include <string.h> -#include <fcntl.h> - -#include "dpdk_priv.h" - -dpdk_main_t dpdk_main; - -/* force linker to link functions used by vlib and declared weak */ -void *vlib_weakly_linked_functions[] = { - &rte_pktmbuf_init, - &rte_pktmbuf_pool_init, -}; - -#define LINK_STATE_ELOGS 0 - -#define DEFAULT_HUGE_DIR "/run/vpp/hugepages" -#define VPP_RUN_DIR "/run/vpp" - -/* Port configuration, mildly modified Intel app values */ - -static struct rte_eth_conf port_conf_template = { - .rxmode = { - .split_hdr_size = 0, - .header_split = 0, /**< Header Split disabled */ - .hw_ip_checksum = 0, /**< IP checksum offload disabled */ - .hw_vlan_filter = 0, /**< VLAN filtering disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ - }, - .txmode = { - .mq_mode = ETH_MQ_TX_NONE, - }, -}; - -clib_error_t * -dpdk_port_setup (dpdk_main_t * dm, dpdk_device_t * xd) -{ - vlib_main_t *vm = vlib_get_main (); - vlib_buffer_main_t *bm = vm->buffer_main; - int rv; - int j; - - ASSERT (os_get_cpu_number () == 0); - - if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) - { - vnet_hw_interface_set_flags (dm->vnet_main, xd->vlib_hw_if_index, 0); - rte_eth_dev_stop (xd->device_index); - } - - rv = rte_eth_dev_configure (xd->device_index, xd->rx_q_used, - xd->tx_q_used, &xd->port_conf); - - if (rv < 0) - return clib_error_return (0, "rte_eth_dev_configure[%d]: err %d", - xd->device_index, rv); - - /* Set up one TX-queue per worker thread */ - for (j = 0; j < xd->tx_q_used; j++) - { - rv = rte_eth_tx_queue_setup (xd->device_index, j, xd->nb_tx_desc, - xd->cpu_socket, &xd->tx_conf); - - /* retry with any other CPU socket */ - if (rv < 0) - rv = rte_eth_tx_queue_setup (xd->device_index, j, xd->nb_tx_desc, - SOCKET_ID_ANY, &xd->tx_conf); - if (rv < 0) - break; - } - - if (rv < 0) - return clib_error_return (0, "rte_eth_tx_queue_setup[%d]: err %d", - xd->device_index, rv); - - for (j = 0; j < xd->rx_q_used; j++) - { - - rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc, - xd->cpu_socket, 0, - bm-> - pktmbuf_pools[xd->cpu_socket_id_by_queue - [j]]); - - /* retry with any other CPU socket */ - if (rv < 0) - rv = rte_eth_rx_queue_setup (xd->device_index, j, xd->nb_rx_desc, - SOCKET_ID_ANY, 0, - bm-> - pktmbuf_pools[xd->cpu_socket_id_by_queue - [j]]); - if (rv < 0) - return clib_error_return (0, "rte_eth_rx_queue_setup[%d]: err %d", - xd->device_index, rv); - } - - if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) - { - int rv; - rv = rte_eth_dev_start (xd->device_index); - if (rv < 0) - clib_warning ("rte_eth_dev_start %d returned %d", - xd->device_index, rv); - } - return 0; -} - -static u32 -dpdk_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags) -{ - dpdk_main_t *dm = &dpdk_main; - dpdk_device_t *xd = vec_elt_at_index (dm->devices, hi->dev_instance); - u32 old = 0; - - if (ETHERNET_INTERFACE_FLAG_CONFIG_PROMISC (flags)) - { - old = (xd->flags & DPDK_DEVICE_FLAG_PROMISC) != 0; - - if (flags & ETHERNET_INTERFACE_FLAG_ACCEPT_ALL) - xd->flags |= DPDK_DEVICE_FLAG_PROMISC; - else - xd->flags &= ~DPDK_DEVICE_FLAG_PROMISC; - - if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) - { - if (xd->flags & DPDK_DEVICE_FLAG_PROMISC) - rte_eth_promiscuous_enable (xd->device_index); - else - rte_eth_promiscuous_disable (xd->device_index); - } - } - else if (ETHERNET_INTERFACE_FLAG_CONFIG_MTU (flags)) - { - /* - * DAW-FIXME: The Cisco VIC firmware does not provide an api for a - * driver to dynamically change the mtu. If/when the - * VIC firmware gets fixed, then this should be removed. - */ - if (xd->pmd == VNET_DPDK_PMD_ENIC) - { - struct rte_eth_dev_info dev_info; - - /* - * Restore mtu to what has been set by CIMC in the firmware cfg. - */ - rte_eth_dev_info_get (xd->device_index, &dev_info); - hi->max_packet_bytes = dev_info.max_rx_pktlen; - - vlib_cli_output (vlib_get_main (), - "Cisco VIC mtu can only be changed " - "using CIMC then rebooting the server!"); - } - else - { - int rv; - - xd->port_conf.rxmode.max_rx_pkt_len = hi->max_packet_bytes; - - if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) - rte_eth_dev_stop (xd->device_index); - - rv = rte_eth_dev_configure - (xd->device_index, xd->rx_q_used, xd->tx_q_used, &xd->port_conf); - - if (rv < 0) - vlib_cli_output (vlib_get_main (), - "rte_eth_dev_configure[%d]: err %d", - xd->device_index, rv); - - rte_eth_dev_set_mtu (xd->device_index, hi->max_packet_bytes); - - if (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) - { - int rv = rte_eth_dev_start (xd->device_index); - if (rv < 0) - clib_warning ("rte_eth_dev_start %d returned %d", - xd->device_index, rv); - } - } - } - return old; -} - -void -dpdk_device_lock_init (dpdk_device_t * xd) -{ - int q; - vec_validate (xd->lockp, xd->tx_q_used - 1); - for (q = 0; q < xd->tx_q_used; q++) - { - xd->lockp[q] = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, - CLIB_CACHE_LINE_BYTES); - memset ((void *) xd->lockp[q], 0, CLIB_CACHE_LINE_BYTES); - } -} - -void -dpdk_device_lock_free (dpdk_device_t * xd) -{ - int q; - - for (q = 0; q < vec_len (xd->lockp); q++) - clib_mem_free ((void *) xd->lockp[q]); - vec_free (xd->lockp); - xd->lockp = 0; -} - -static clib_error_t * -dpdk_lib_init (dpdk_main_t * dm) -{ - u32 nports; - u32 nb_desc = 0; - int i; - clib_error_t *error; - vlib_main_t *vm = vlib_get_main (); - vlib_thread_main_t *tm = vlib_get_thread_main (); - vnet_sw_interface_t *sw; - vnet_hw_interface_t *hi; - dpdk_device_t *xd; - vlib_pci_addr_t last_pci_addr; - u32 last_pci_addr_port = 0; - vlib_thread_registration_t *tr, *tr_hqos; - uword *p, *p_hqos; - - u32 next_cpu = 0, next_hqos_cpu = 0; - u8 af_packet_port_id = 0; - last_pci_addr.as_u32 = ~0; - - dm->input_cpu_first_index = 0; - dm->input_cpu_count = 1; - - /* find out which cpus will be used for input */ - p = hash_get_mem (tm->thread_registrations_by_name, "workers"); - tr = p ? (vlib_thread_registration_t *) p[0] : 0; - - if (tr && tr->count > 0) - { - dm->input_cpu_first_index = tr->first_index; - dm->input_cpu_count = tr->count; - } - - vec_validate_aligned (dm->devices_by_cpu, tm->n_vlib_mains - 1, - CLIB_CACHE_LINE_BYTES); - - vec_validate_aligned (dm->workers, tm->n_vlib_mains - 1, - CLIB_CACHE_LINE_BYTES); - - dm->hqos_cpu_first_index = 0; - dm->hqos_cpu_count = 0; - - /* find out which cpus will be used for I/O TX */ - p_hqos = hash_get_mem (tm->thread_registrations_by_name, "hqos-threads"); - tr_hqos = p_hqos ? (vlib_thread_registration_t *) p_hqos[0] : 0; - - if (tr_hqos && tr_hqos->count > 0) - { - dm->hqos_cpu_first_index = tr_hqos->first_index; - dm->hqos_cpu_count = tr_hqos->count; - } - - vec_validate_aligned (dm->devices_by_hqos_cpu, tm->n_vlib_mains - 1, - CLIB_CACHE_LINE_BYTES); - - vec_validate_aligned (dm->hqos_threads, tm->n_vlib_mains - 1, - CLIB_CACHE_LINE_BYTES); - - nports = rte_eth_dev_count (); - if (nports < 1) - { - clib_warning ("DPDK drivers found no ports..."); - } - - if (CLIB_DEBUG > 0) - clib_warning ("DPDK drivers found %d ports...", nports); - - /* - * All buffers are all allocated from the same rte_mempool. - * Thus they all have the same number of data bytes. - */ - dm->vlib_buffer_free_list_index = - vlib_buffer_get_or_create_free_list (vm, - VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES, - "dpdk rx"); - - if (dm->conf->enable_tcp_udp_checksum) - dm->buffer_flags_template &= ~(IP_BUFFER_L4_CHECKSUM_CORRECT - | IP_BUFFER_L4_CHECKSUM_COMPUTED); - - for (i = 0; i < nports; i++) - { - u8 addr[6]; - u8 vlan_strip = 0; - int j; - struct rte_eth_dev_info dev_info; - clib_error_t *rv; - struct rte_eth_link l; - dpdk_device_config_t *devconf = 0; - vlib_pci_addr_t pci_addr; - uword *p = 0; - - rte_eth_dev_info_get (i, &dev_info); - if (dev_info.pci_dev) /* bonded interface has no pci info */ - { - pci_addr.domain = dev_info.pci_dev->addr.domain; - pci_addr.bus = dev_info.pci_dev->addr.bus; - pci_addr.slot = dev_info.pci_dev->addr.devid; - pci_addr.function = dev_info.pci_dev->addr.function; - p = - hash_get (dm->conf->device_config_index_by_pci_addr, - pci_addr.as_u32); - } - - if (p) - devconf = pool_elt_at_index (dm->conf->dev_confs, p[0]); - else - devconf = &dm->conf->default_devconf; - - /* Create vnet interface */ - vec_add2_aligned (dm->devices, xd, 1, CLIB_CACHE_LINE_BYTES); - xd->nb_rx_desc = DPDK_NB_RX_DESC_DEFAULT; - xd->nb_tx_desc = DPDK_NB_TX_DESC_DEFAULT; - xd->cpu_socket = (i8) rte_eth_dev_socket_id (i); - - /* Handle interface naming for devices with multiple ports sharing same PCI ID */ - if (dev_info.pci_dev) - { - struct rte_eth_dev_info di = { 0 }; - rte_eth_dev_info_get (i + 1, &di); - if (di.pci_dev && pci_addr.as_u32 != last_pci_addr.as_u32 && - memcmp (&dev_info.pci_dev->addr, &di.pci_dev->addr, - sizeof (struct rte_pci_addr)) == 0) - { - xd->interface_name_suffix = format (0, "0"); - last_pci_addr.as_u32 = pci_addr.as_u32; - last_pci_addr_port = i; - } - else if (pci_addr.as_u32 == last_pci_addr.as_u32) - { - xd->interface_name_suffix = - format (0, "%u", i - last_pci_addr_port); - } - else - { - last_pci_addr.as_u32 = ~0; - } - } - else - last_pci_addr.as_u32 = ~0; - - clib_memcpy (&xd->tx_conf, &dev_info.default_txconf, - sizeof (struct rte_eth_txconf)); - if (dm->conf->no_multi_seg) - { - xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS; - port_conf_template.rxmode.jumbo_frame = 0; - } - else - { - xd->tx_conf.txq_flags &= ~ETH_TXQ_FLAGS_NOMULTSEGS; - port_conf_template.rxmode.jumbo_frame = 1; - xd->flags |= DPDK_DEVICE_FLAG_MAYBE_MULTISEG; - } - - clib_memcpy (&xd->port_conf, &port_conf_template, - sizeof (struct rte_eth_conf)); - - xd->tx_q_used = clib_min (dev_info.max_tx_queues, tm->n_vlib_mains); - - if (devconf->num_tx_queues > 0 - && devconf->num_tx_queues < xd->tx_q_used) - xd->tx_q_used = clib_min (xd->tx_q_used, devconf->num_tx_queues); - - if (devconf->num_rx_queues > 1 && dm->use_rss == 0) - { - dm->use_rss = 1; - } - - if (devconf->num_rx_queues > 1 - && dev_info.max_rx_queues >= devconf->num_rx_queues) - { - xd->rx_q_used = devconf->num_rx_queues; - xd->port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; - if (devconf->rss_fn == 0) - xd->port_conf.rx_adv_conf.rss_conf.rss_hf = - ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP; - else - xd->port_conf.rx_adv_conf.rss_conf.rss_hf = devconf->rss_fn; - } - else - xd->rx_q_used = 1; - - xd->flags |= DPDK_DEVICE_FLAG_PMD; - - /* workaround for drivers not setting driver_name */ - if ((!dev_info.driver_name) && (dev_info.pci_dev)) -#if RTE_VERSION < RTE_VERSION_NUM(16, 11, 0, 0) - dev_info.driver_name = dev_info.pci_dev->driver->name; -#else - dev_info.driver_name = dev_info.pci_dev->driver->driver.name; -#endif - ASSERT (dev_info.driver_name); - - if (!xd->pmd) - { - - -#define _(s,f) else if (dev_info.driver_name && \ - !strcmp(dev_info.driver_name, s)) \ - xd->pmd = VNET_DPDK_PMD_##f; - if (0) - ; - foreach_dpdk_pmd -#undef _ - else - xd->pmd = VNET_DPDK_PMD_UNKNOWN; - - xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN; - xd->nb_rx_desc = DPDK_NB_RX_DESC_DEFAULT; - xd->nb_tx_desc = DPDK_NB_TX_DESC_DEFAULT; - - switch (xd->pmd) - { - /* 1G adapters */ - case VNET_DPDK_PMD_E1000EM: - case VNET_DPDK_PMD_IGB: - case VNET_DPDK_PMD_IGBVF: - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_1G; - break; - - /* 10G adapters */ - case VNET_DPDK_PMD_IXGBE: - case VNET_DPDK_PMD_IXGBEVF: - case VNET_DPDK_PMD_THUNDERX: - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G; - break; - case VNET_DPDK_PMD_DPAA2: - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G; - break; - - /* Cisco VIC */ - case VNET_DPDK_PMD_ENIC: - rte_eth_link_get_nowait (i, &l); - xd->flags |= DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE; - if (l.link_speed == 40000) - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G; - else - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G; - break; - - /* Intel Fortville */ - case VNET_DPDK_PMD_I40E: - case VNET_DPDK_PMD_I40EVF: - xd->flags |= DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE; - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G; - - switch (dev_info.pci_dev->id.device_id) - { - case I40E_DEV_ID_10G_BASE_T: - case I40E_DEV_ID_SFP_XL710: - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G; - break; - case I40E_DEV_ID_QSFP_A: - case I40E_DEV_ID_QSFP_B: - case I40E_DEV_ID_QSFP_C: - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G; - break; - case I40E_DEV_ID_VF: - rte_eth_link_get_nowait (i, &l); - xd->port_type = l.link_speed == 10000 ? - VNET_DPDK_PORT_TYPE_ETH_10G : VNET_DPDK_PORT_TYPE_ETH_40G; - break; - default: - xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN; - } - break; - - case VNET_DPDK_PMD_CXGBE: - switch (dev_info.pci_dev->id.device_id) - { - case 0x540d: /* T580-CR */ - case 0x5410: /* T580-LP-cr */ - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G; - break; - case 0x5403: /* T540-CR */ - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G; - break; - default: - xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN; - } - break; - - case VNET_DPDK_PMD_MLX5: - { - char *pn_100g[] = { "MCX415A-CCAT", "MCX416A-CCAT", 0 }; - char *pn_40g[] = { "MCX413A-BCAT", "MCX414A-BCAT", - "MCX415A-BCAT", "MCX416A-BCAT", "MCX4131A-BCAT", 0 - }; - char *pn_10g[] = { "MCX4111A-XCAT", "MCX4121A-XCAT", 0 }; - - vlib_pci_device_t *pd = vlib_get_pci_device (&pci_addr); - u8 *pn = 0; - char **c; - int found = 0; - pn = format (0, "%U%c", - format_vlib_pci_vpd, pd->vpd_r, "PN", 0); - - if (!pn) - break; - - c = pn_100g; - while (!found && c[0]) - { - if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0) - { - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_100G; - break; - } - c++; - } - - c = pn_40g; - while (!found && c[0]) - { - if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0) - { - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G; - break; - } - c++; - } - - c = pn_10g; - while (!found && c[0]) - { - if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0) - { - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G; - break; - } - c++; - } - - vec_free (pn); - } - - break; - /* Intel Red Rock Canyon */ - case VNET_DPDK_PMD_FM10K: - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_SWITCH; - break; - - /* virtio */ - case VNET_DPDK_PMD_VIRTIO: - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_1G; - xd->nb_rx_desc = DPDK_NB_RX_DESC_VIRTIO; - xd->nb_tx_desc = DPDK_NB_TX_DESC_VIRTIO; - break; - - /* vmxnet3 */ - case VNET_DPDK_PMD_VMXNET3: - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_1G; - xd->tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS; - break; - - case VNET_DPDK_PMD_AF_PACKET: - xd->port_type = VNET_DPDK_PORT_TYPE_AF_PACKET; - xd->af_packet_port_id = af_packet_port_id++; - break; - - case VNET_DPDK_PMD_BOND: - xd->flags |= DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE; - xd->port_type = VNET_DPDK_PORT_TYPE_ETH_BOND; - break; - - default: - xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN; - } - - if (devconf->num_rx_desc) - xd->nb_rx_desc = devconf->num_rx_desc; - - if (devconf->num_tx_desc) - xd->nb_tx_desc = devconf->num_tx_desc; - } - - /* - * Ensure default mtu is not > the mtu read from the hardware. - * Otherwise rte_eth_dev_configure() will fail and the port will - * not be available. - */ - if (ETHERNET_MAX_PACKET_BYTES > dev_info.max_rx_pktlen) - { - /* - * This device does not support the platforms's max frame - * size. Use it's advertised mru instead. - */ - xd->port_conf.rxmode.max_rx_pkt_len = dev_info.max_rx_pktlen; - } - else - { - xd->port_conf.rxmode.max_rx_pkt_len = ETHERNET_MAX_PACKET_BYTES; - - /* - * Some platforms do not account for Ethernet FCS (4 bytes) in - * MTU calculations. To interop with them increase mru but only - * if the device's settings can support it. - */ - if ((dev_info.max_rx_pktlen >= (ETHERNET_MAX_PACKET_BYTES + 4)) && - xd->port_conf.rxmode.hw_strip_crc) - { - /* - * Allow additional 4 bytes (for Ethernet FCS). These bytes are - * stripped by h/w and so will not consume any buffer memory. - */ - xd->port_conf.rxmode.max_rx_pkt_len += 4; - } - } - - if (xd->pmd == VNET_DPDK_PMD_AF_PACKET) - { - f64 now = vlib_time_now (vm); - u32 rnd; - rnd = (u32) (now * 1e6); - rnd = random_u32 (&rnd); - clib_memcpy (addr + 2, &rnd, sizeof (rnd)); - addr[0] = 2; - addr[1] = 0xfe; - } - else - rte_eth_macaddr_get (i, (struct ether_addr *) addr); - - if (xd->tx_q_used < tm->n_vlib_mains) - dpdk_device_lock_init (xd); - - xd->device_index = xd - dm->devices; - ASSERT (i == xd->device_index); - xd->per_interface_next_index = ~0; - - /* assign interface to input thread */ - dpdk_device_and_queue_t *dq; - int q; - - if (devconf->workers) - { - int i; - q = 0; - /* *INDENT-OFF* */ - clib_bitmap_foreach (i, devconf->workers, ({ - int cpu = dm->input_cpu_first_index + i; - unsigned lcore = vlib_worker_threads[cpu].lcore_id; - vec_validate(xd->cpu_socket_id_by_queue, q); - xd->cpu_socket_id_by_queue[q] = rte_lcore_to_socket_id(lcore); - vec_add2(dm->devices_by_cpu[cpu], dq, 1); - dq->device = xd->device_index; - dq->queue_id = q++; - })); - /* *INDENT-ON* */ - } - else - for (q = 0; q < xd->rx_q_used; q++) - { - int cpu = dm->input_cpu_first_index + next_cpu; - unsigned lcore = vlib_worker_threads[cpu].lcore_id; - - /* - * numa node for worker thread handling this queue - * needed for taking buffers from the right mempool - */ - vec_validate (xd->cpu_socket_id_by_queue, q); - xd->cpu_socket_id_by_queue[q] = rte_lcore_to_socket_id (lcore); - - /* - * construct vector of (device,queue) pairs for each worker thread - */ - vec_add2 (dm->devices_by_cpu[cpu], dq, 1); - dq->device = xd->device_index; - dq->queue_id = q; - - next_cpu++; - if (next_cpu == dm->input_cpu_count) - next_cpu = 0; - } - - - if (devconf->hqos_enabled) - { - xd->flags |= DPDK_DEVICE_FLAG_HQOS; - - if (devconf->hqos.hqos_thread_valid) - { - int cpu = dm->hqos_cpu_first_index + devconf->hqos.hqos_thread; - - if (devconf->hqos.hqos_thread >= dm->hqos_cpu_count) - return clib_error_return (0, "invalid HQoS thread index"); - - vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1); - dq->device = xd->device_index; - dq->queue_id = 0; - } - else - { - int cpu = dm->hqos_cpu_first_index + next_hqos_cpu; - - if (dm->hqos_cpu_count == 0) - return clib_error_return (0, "no HQoS threads available"); - - vec_add2 (dm->devices_by_hqos_cpu[cpu], dq, 1); - dq->device = xd->device_index; - dq->queue_id = 0; - - next_hqos_cpu++; - if (next_hqos_cpu == dm->hqos_cpu_count) - next_hqos_cpu = 0; - - devconf->hqos.hqos_thread_valid = 1; - devconf->hqos.hqos_thread = cpu; - } - } - - vec_validate_aligned (xd->tx_vectors, tm->n_vlib_mains, - CLIB_CACHE_LINE_BYTES); - for (j = 0; j < tm->n_vlib_mains; j++) - { - vec_validate_ha (xd->tx_vectors[j], xd->nb_tx_desc, - sizeof (tx_ring_hdr_t), CLIB_CACHE_LINE_BYTES); - vec_reset_length (xd->tx_vectors[j]); - } - - vec_validate_aligned (xd->rx_vectors, xd->rx_q_used, - CLIB_CACHE_LINE_BYTES); - for (j = 0; j < xd->rx_q_used; j++) - { - vec_validate_aligned (xd->rx_vectors[j], VLIB_FRAME_SIZE - 1, - CLIB_CACHE_LINE_BYTES); - vec_reset_length (xd->rx_vectors[j]); - } - - vec_validate_aligned (xd->d_trace_buffers, tm->n_vlib_mains, - CLIB_CACHE_LINE_BYTES); - - rv = dpdk_port_setup (dm, xd); - - if (rv) - return rv; - - if (devconf->hqos_enabled) - { - rv = dpdk_port_setup_hqos (xd, &devconf->hqos); - if (rv) - return rv; - } - - /* count the number of descriptors used for this device */ - nb_desc += xd->nb_rx_desc + xd->nb_tx_desc * xd->tx_q_used; - - error = ethernet_register_interface - (dm->vnet_main, dpdk_device_class.index, xd->device_index, - /* ethernet address */ addr, - &xd->vlib_hw_if_index, dpdk_flag_change); - if (error) - return error; - - sw = vnet_get_hw_sw_interface (dm->vnet_main, xd->vlib_hw_if_index); - xd->vlib_sw_if_index = sw->sw_if_index; - hi = vnet_get_hw_interface (dm->vnet_main, xd->vlib_hw_if_index); - - /* - * DAW-FIXME: The Cisco VIC firmware does not provide an api for a - * driver to dynamically change the mtu. If/when the - * VIC firmware gets fixed, then this should be removed. - */ - if (xd->pmd == VNET_DPDK_PMD_ENIC) - { - /* - * Initialize mtu to what has been set by CIMC in the firmware cfg. - */ - hi->max_packet_bytes = dev_info.max_rx_pktlen; - if (devconf->vlan_strip_offload != DPDK_DEVICE_VLAN_STRIP_OFF) - vlan_strip = 1; /* remove vlan tag from VIC port by default */ - else - clib_warning ("VLAN strip disabled for interface\n"); - } - else if (devconf->vlan_strip_offload == DPDK_DEVICE_VLAN_STRIP_ON) - vlan_strip = 1; - - if (vlan_strip) - { - int vlan_off; - vlan_off = rte_eth_dev_get_vlan_offload (xd->device_index); - vlan_off |= ETH_VLAN_STRIP_OFFLOAD; - xd->port_conf.rxmode.hw_vlan_strip = vlan_off; - if (rte_eth_dev_set_vlan_offload (xd->device_index, vlan_off) == 0) - clib_warning ("VLAN strip enabled for interface\n"); - else - clib_warning ("VLAN strip cannot be supported by interface\n"); - } - - hi->max_l3_packet_bytes[VLIB_RX] = hi->max_l3_packet_bytes[VLIB_TX] = - xd->port_conf.rxmode.max_rx_pkt_len - sizeof (ethernet_header_t); - - rte_eth_dev_set_mtu (xd->device_index, hi->max_packet_bytes); - } - - if (nb_desc > dm->conf->num_mbufs) - clib_warning ("%d mbufs allocated but total rx/tx ring size is %d\n", - dm->conf->num_mbufs, nb_desc); - - return 0; -} - -static void -dpdk_bind_devices_to_uio (dpdk_config_main_t * conf) -{ - vlib_pci_main_t *pm = &pci_main; - clib_error_t *error; - vlib_pci_device_t *d; - u8 *pci_addr = 0; - int num_whitelisted = vec_len (conf->dev_confs); - - /* *INDENT-OFF* */ - pool_foreach (d, pm->pci_devs, ({ - dpdk_device_config_t * devconf = 0; - vec_reset_length (pci_addr); - pci_addr = format (pci_addr, "%U%c", format_vlib_pci_addr, &d->bus_address, 0); - - if (d->device_class != PCI_CLASS_NETWORK_ETHERNET) - continue; - - if (num_whitelisted) - { - uword * p = hash_get (conf->device_config_index_by_pci_addr, d->bus_address.as_u32); - - if (!p) - continue; - - devconf = pool_elt_at_index (conf->dev_confs, p[0]); - } - - /* virtio */ - if (d->vendor_id == 0x1af4 && d->device_id == 0x1000) - ; - /* vmxnet3 */ - else if (d->vendor_id == 0x15ad && d->device_id == 0x07b0) - ; - /* all Intel devices */ - else if (d->vendor_id == 0x8086) - ; - /* Cisco VIC */ - else if (d->vendor_id == 0x1137 && d->device_id == 0x0043) - ; - /* Chelsio T4/T5 */ - else if (d->vendor_id == 0x1425 && (d->device_id & 0xe000) == 0x4000) - ; - else - { - clib_warning ("Unsupported Ethernet PCI device 0x%04x:0x%04x found " - "at PCI address %s\n", (u16) d->vendor_id, (u16) d->device_id, - pci_addr); - continue; - } - - error = vlib_pci_bind_to_uio (d, (char *) conf->uio_driver_name); - - if (error) - { - if (devconf == 0) - { - pool_get (conf->dev_confs, devconf); - hash_set (conf->device_config_index_by_pci_addr, d->bus_address.as_u32, - devconf - conf->dev_confs); - devconf->pci_addr.as_u32 = d->bus_address.as_u32; - } - devconf->is_blacklisted = 1; - clib_error_report (error); - } - })); - /* *INDENT-ON* */ - vec_free (pci_addr); -} - -static clib_error_t * -dpdk_device_config (dpdk_config_main_t * conf, vlib_pci_addr_t pci_addr, - unformat_input_t * input, u8 is_default) -{ - clib_error_t *error = 0; - uword *p; - dpdk_device_config_t *devconf; - unformat_input_t sub_input; - - if (is_default) - { - devconf = &conf->default_devconf; - } - else - { - p = hash_get (conf->device_config_index_by_pci_addr, pci_addr.as_u32); - - if (!p) - { - pool_get (conf->dev_confs, devconf); - hash_set (conf->device_config_index_by_pci_addr, pci_addr.as_u32, - devconf - conf->dev_confs); - } - else - return clib_error_return (0, - "duplicate configuration for PCI address %U", - format_vlib_pci_addr, &pci_addr); - } - - devconf->pci_addr.as_u32 = pci_addr.as_u32; - devconf->hqos_enabled = 0; - dpdk_device_config_hqos_default (&devconf->hqos); - - if (!input) - return 0; - - unformat_skip_white_space (input); - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "num-rx-queues %u", &devconf->num_rx_queues)) - ; - else if (unformat (input, "num-tx-queues %u", &devconf->num_tx_queues)) - ; - else if (unformat (input, "num-rx-desc %u", &devconf->num_rx_desc)) - ; - else if (unformat (input, "num-tx-desc %u", &devconf->num_tx_desc)) - ; - else if (unformat (input, "workers %U", unformat_bitmap_list, - &devconf->workers)) - ; - else - if (unformat - (input, "rss %U", unformat_vlib_cli_sub_input, &sub_input)) - { - error = unformat_rss_fn (&sub_input, &devconf->rss_fn); - if (error) - break; - } - else if (unformat (input, "vlan-strip-offload off")) - devconf->vlan_strip_offload = DPDK_DEVICE_VLAN_STRIP_OFF; - else if (unformat (input, "vlan-strip-offload on")) - devconf->vlan_strip_offload = DPDK_DEVICE_VLAN_STRIP_ON; - else - if (unformat - (input, "hqos %U", unformat_vlib_cli_sub_input, &sub_input)) - { - devconf->hqos_enabled = 1; - error = unformat_hqos (&sub_input, &devconf->hqos); - if (error) - break; - } - else if (unformat (input, "hqos")) - { - devconf->hqos_enabled = 1; - } - else - { - error = clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); - break; - } - } - - if (error) - return error; - - if (devconf->workers && devconf->num_rx_queues == 0) - devconf->num_rx_queues = clib_bitmap_count_set_bits (devconf->workers); - else if (devconf->workers && - clib_bitmap_count_set_bits (devconf->workers) != - devconf->num_rx_queues) - error = - clib_error_return (0, - "%U: number of worker threadds must be " - "equal to number of rx queues", format_vlib_pci_addr, - &pci_addr); - - return error; -} - -static clib_error_t * -dpdk_config (vlib_main_t * vm, unformat_input_t * input) -{ - clib_error_t *error = 0; - dpdk_main_t *dm = &dpdk_main; - dpdk_config_main_t *conf = &dpdk_config_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - dpdk_device_config_t *devconf; - vlib_pci_addr_t pci_addr; - unformat_input_t sub_input; - u8 *s, *tmp = 0; - u8 *rte_cmd = 0, *ethname = 0; - u32 log_level; - int ret, i; - int num_whitelisted = 0; - u8 no_pci = 0; - u8 no_huge = 0; - u8 huge_dir = 0; - u8 file_prefix = 0; - u8 *socket_mem = 0; - - conf->device_config_index_by_pci_addr = hash_create (0, sizeof (uword)); - - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - /* Prime the pump */ - if (unformat (input, "no-hugetlb")) - { - vec_add1 (conf->eal_init_args, (u8 *) "no-huge"); - no_huge = 1; - } - - else if (unformat (input, "enable-tcp-udp-checksum")) - conf->enable_tcp_udp_checksum = 1; - - else if (unformat (input, "decimal-interface-names")) - conf->interface_name_format_decimal = 1; - - else if (unformat (input, "no-multi-seg")) - conf->no_multi_seg = 1; - - else if (unformat (input, "dev default %U", unformat_vlib_cli_sub_input, - &sub_input)) - { - error = - dpdk_device_config (conf, (vlib_pci_addr_t) (u32) ~ 1, &sub_input, - 1); - - if (error) - return error; - } - else - if (unformat - (input, "dev %U %U", unformat_vlib_pci_addr, &pci_addr, - unformat_vlib_cli_sub_input, &sub_input)) - { - error = dpdk_device_config (conf, pci_addr, &sub_input, 0); - - if (error) - return error; - - num_whitelisted++; - } - else if (unformat (input, "dev %U", unformat_vlib_pci_addr, &pci_addr)) - { - error = dpdk_device_config (conf, pci_addr, 0, 0); - - if (error) - return error; - - num_whitelisted++; - } - else if (unformat (input, "num-mbufs %d", &conf->num_mbufs)) - ; - else if (unformat (input, "kni %d", &conf->num_kni)) - ; - else if (unformat (input, "uio-driver %s", &conf->uio_driver_name)) - ; - else if (unformat (input, "socket-mem %s", &socket_mem)) - ; - else if (unformat (input, "no-pci")) - { - no_pci = 1; - tmp = format (0, "--no-pci%c", 0); - vec_add1 (conf->eal_init_args, tmp); - } - else if (unformat (input, "poll-sleep %d", &dm->poll_sleep)) - ; - -#define _(a) \ - else if (unformat(input, #a)) \ - { \ - tmp = format (0, "--%s%c", #a, 0); \ - vec_add1 (conf->eal_init_args, tmp); \ - } - foreach_eal_double_hyphen_predicate_arg -#undef _ -#define _(a) \ - else if (unformat(input, #a " %s", &s)) \ - { \ - if (!strncmp(#a, "huge-dir", 8)) \ - huge_dir = 1; \ - else if (!strncmp(#a, "file-prefix", 11)) \ - file_prefix = 1; \ - tmp = format (0, "--%s%c", #a, 0); \ - vec_add1 (conf->eal_init_args, tmp); \ - vec_add1 (s, 0); \ - vec_add1 (conf->eal_init_args, s); \ - } - foreach_eal_double_hyphen_arg -#undef _ -#define _(a,b) \ - else if (unformat(input, #a " %s", &s)) \ - { \ - tmp = format (0, "-%s%c", #b, 0); \ - vec_add1 (conf->eal_init_args, tmp); \ - vec_add1 (s, 0); \ - vec_add1 (conf->eal_init_args, s); \ - } - foreach_eal_single_hyphen_arg -#undef _ -#define _(a,b) \ - else if (unformat(input, #a " %s", &s)) \ - { \ - tmp = format (0, "-%s%c", #b, 0); \ - vec_add1 (conf->eal_init_args, tmp); \ - vec_add1 (s, 0); \ - vec_add1 (conf->eal_init_args, s); \ - conf->a##_set_manually = 1; \ - } - foreach_eal_single_hyphen_mandatory_arg -#undef _ - else if (unformat (input, "default")) - ; - - else if (unformat_skip_white_space (input)) - ; - else - { - error = clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); - goto done; - } - } - - if (!conf->uio_driver_name) - conf->uio_driver_name = format (0, "igb_uio%c", 0); - - /* - * Use 1G huge pages if available. - */ - if (!no_huge && !huge_dir) - { - u32 x, *mem_by_socket = 0; - uword c = 0; - u8 use_1g = 1; - u8 use_2m = 1; - u8 less_than_1g = 1; - int rv; - - umount (DEFAULT_HUGE_DIR); - - /* Process "socket-mem" parameter value */ - if (vec_len (socket_mem)) - { - unformat_input_t in; - unformat_init_vector (&in, socket_mem); - while (unformat_check_input (&in) != UNFORMAT_END_OF_INPUT) - { - if (unformat (&in, "%u,", &x)) - ; - else if (unformat (&in, "%u", &x)) - ; - else if (unformat (&in, ",")) - x = 0; - else - break; - - vec_add1 (mem_by_socket, x); - - if (x > 1023) - less_than_1g = 0; - } - /* Note: unformat_free vec_frees(in.buffer), aka socket_mem... */ - unformat_free (&in); - socket_mem = 0; - } - else - { - /* *INDENT-OFF* */ - clib_bitmap_foreach (c, tm->cpu_socket_bitmap, ( - { - vec_validate(mem_by_socket, c); - mem_by_socket[c] = 256; /* default per-socket mem */ - } - )); - /* *INDENT-ON* */ - } - - /* check if available enough 1GB pages for each socket */ - /* *INDENT-OFF* */ - clib_bitmap_foreach (c, tm->cpu_socket_bitmap, ( - { - int pages_avail, page_size, mem; - - vec_validate(mem_by_socket, c); - mem = mem_by_socket[c]; - - page_size = 1024; - pages_avail = vlib_sysfs_get_free_hugepages(c, page_size * 1024); - - if (pages_avail < 0 || page_size * pages_avail < mem) - use_1g = 0; - - page_size = 2; - pages_avail = vlib_sysfs_get_free_hugepages(c, page_size * 1024); - - if (pages_avail < 0 || page_size * pages_avail < mem) - use_2m = 0; - })); - /* *INDENT-ON* */ - - if (mem_by_socket == 0) - { - error = clib_error_return (0, "mem_by_socket NULL"); - goto done; - } - _vec_len (mem_by_socket) = c + 1; - - /* regenerate socket_mem string */ - vec_foreach_index (x, mem_by_socket) - socket_mem = format (socket_mem, "%s%u", - socket_mem ? "," : "", mem_by_socket[x]); - socket_mem = format (socket_mem, "%c", 0); - - vec_free (mem_by_socket); - - rv = mkdir (VPP_RUN_DIR, 0755); - if (rv && errno != EEXIST) - { - error = clib_error_return (0, "mkdir '%s' failed errno %d", - VPP_RUN_DIR, errno); - goto done; - } - - rv = mkdir (DEFAULT_HUGE_DIR, 0755); - if (rv && errno != EEXIST) - { - error = clib_error_return (0, "mkdir '%s' failed errno %d", - DEFAULT_HUGE_DIR, errno); - goto done; - } - - if (use_1g && !(less_than_1g && use_2m)) - { - rv = - mount ("none", DEFAULT_HUGE_DIR, "hugetlbfs", 0, "pagesize=1G"); - } - else if (use_2m) - { - rv = mount ("none", DEFAULT_HUGE_DIR, "hugetlbfs", 0, NULL); - } - else - { - return clib_error_return (0, "not enough free huge pages"); - } - - if (rv) - { - error = clib_error_return (0, "mount failed %d", errno); - goto done; - } - - tmp = format (0, "--huge-dir%c", 0); - vec_add1 (conf->eal_init_args, tmp); - tmp = format (0, "%s%c", DEFAULT_HUGE_DIR, 0); - vec_add1 (conf->eal_init_args, tmp); - if (!file_prefix) - { - tmp = format (0, "--file-prefix%c", 0); - vec_add1 (conf->eal_init_args, tmp); - tmp = format (0, "vpp%c", 0); - vec_add1 (conf->eal_init_args, tmp); - } - } - - vec_free (rte_cmd); - vec_free (ethname); - - if (error) - return error; - - /* I'll bet that -c and -n must be the first and second args... */ - if (!conf->coremask_set_manually) - { - vlib_thread_registration_t *tr; - uword *coremask = 0; - int i; - - /* main thread core */ - coremask = clib_bitmap_set (coremask, tm->main_lcore, 1); - - for (i = 0; i < vec_len (tm->registrations); i++) - { - tr = tm->registrations[i]; - coremask = clib_bitmap_or (coremask, tr->coremask); - } - - vec_insert (conf->eal_init_args, 2, 1); - conf->eal_init_args[1] = (u8 *) "-c"; - tmp = format (0, "%U%c", format_bitmap_hex, coremask, 0); - conf->eal_init_args[2] = tmp; - clib_bitmap_free (coremask); - } - - if (!conf->nchannels_set_manually) - { - vec_insert (conf->eal_init_args, 2, 3); - conf->eal_init_args[3] = (u8 *) "-n"; - tmp = format (0, "%d", conf->nchannels); - conf->eal_init_args[4] = tmp; - } - - if (no_pci == 0 && geteuid () == 0) - dpdk_bind_devices_to_uio (conf); - -#define _(x) \ - if (devconf->x == 0 && conf->default_devconf.x > 0) \ - devconf->x = conf->default_devconf.x ; - - /* *INDENT-OFF* */ - pool_foreach (devconf, conf->dev_confs, ({ - - /* default per-device config items */ - foreach_dpdk_device_config_item - - /* add DPDK EAL whitelist/blacklist entry */ - if (num_whitelisted > 0 && devconf->is_blacklisted == 0) - { - tmp = format (0, "-w%c", 0); - vec_add1 (conf->eal_init_args, tmp); - tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0); - vec_add1 (conf->eal_init_args, tmp); - } - else if (num_whitelisted == 0 && devconf->is_blacklisted != 0) - { - tmp = format (0, "-b%c", 0); - vec_add1 (conf->eal_init_args, tmp); - tmp = format (0, "%U%c", format_vlib_pci_addr, &devconf->pci_addr, 0); - vec_add1 (conf->eal_init_args, tmp); - } - })); - /* *INDENT-ON* */ - -#undef _ - - /* set master-lcore */ - tmp = format (0, "--master-lcore%c", 0); - vec_add1 (conf->eal_init_args, tmp); - tmp = format (0, "%u%c", tm->main_lcore, 0); - vec_add1 (conf->eal_init_args, tmp); - - /* set socket-mem */ - tmp = format (0, "--socket-mem%c", 0); - vec_add1 (conf->eal_init_args, tmp); - tmp = format (0, "%s%c", socket_mem, 0); - vec_add1 (conf->eal_init_args, tmp); - - /* NULL terminate the "argv" vector, in case of stupidity */ - vec_add1 (conf->eal_init_args, 0); - _vec_len (conf->eal_init_args) -= 1; - - /* Set up DPDK eal and packet mbuf pool early. */ - - log_level = (CLIB_DEBUG > 0) ? RTE_LOG_DEBUG : RTE_LOG_NOTICE; - - rte_set_log_level (log_level); - - vm = vlib_get_main (); - - /* make copy of args as rte_eal_init tends to mess up with arg array */ - for (i = 1; i < vec_len (conf->eal_init_args); i++) - conf->eal_init_args_str = format (conf->eal_init_args_str, "%s ", - conf->eal_init_args[i]); - - ret = - rte_eal_init (vec_len (conf->eal_init_args), - (char **) conf->eal_init_args); - - /* lazy umount hugepages */ - umount2 (DEFAULT_HUGE_DIR, MNT_DETACH); - - if (ret < 0) - return clib_error_return (0, "rte_eal_init returned %d", ret); - - /* Dump the physical memory layout prior to creating the mbuf_pool */ - fprintf (stdout, "DPDK physical memory layout:\n"); - rte_dump_physmem_layout (stdout); - - /* main thread 1st */ - error = vlib_buffer_pool_create (vm, conf->num_mbufs, rte_socket_id ()); - if (error) - return error; - - for (i = 0; i < RTE_MAX_LCORE; i++) - { - error = vlib_buffer_pool_create (vm, conf->num_mbufs, - rte_lcore_to_socket_id (i)); - if (error) - return error; - } - -done: - return error; -} - -VLIB_CONFIG_FUNCTION (dpdk_config, "dpdk"); - -void -dpdk_update_link_state (dpdk_device_t * xd, f64 now) -{ - vnet_main_t *vnm = vnet_get_main (); - struct rte_eth_link prev_link = xd->link; - u32 hw_flags = 0; - u8 hw_flags_chg = 0; - - /* only update link state for PMD interfaces */ - if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0) - return; - - xd->time_last_link_update = now ? now : xd->time_last_link_update; - memset (&xd->link, 0, sizeof (xd->link)); - rte_eth_link_get_nowait (xd->device_index, &xd->link); - - if (LINK_STATE_ELOGS) - { - vlib_main_t *vm = vlib_get_main (); - ELOG_TYPE_DECLARE (e) = - { - .format = - "update-link-state: sw_if_index %d, admin_up %d," - "old link_state %d new link_state %d",.format_args = "i4i1i1i1",}; - - struct - { - u32 sw_if_index; - u8 admin_up; - u8 old_link_state; - u8 new_link_state; - } *ed; - ed = ELOG_DATA (&vm->elog_main, e); - ed->sw_if_index = xd->vlib_sw_if_index; - ed->admin_up = (xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) != 0; - ed->old_link_state = (u8) - vnet_hw_interface_is_link_up (vnm, xd->vlib_hw_if_index); - ed->new_link_state = (u8) xd->link.link_status; - } - - if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) && - ((xd->link.link_status != 0) ^ - vnet_hw_interface_is_link_up (vnm, xd->vlib_hw_if_index))) - { - hw_flags_chg = 1; - hw_flags |= (xd->link.link_status ? VNET_HW_INTERFACE_FLAG_LINK_UP : 0); - } - - if (hw_flags_chg || (xd->link.link_duplex != prev_link.link_duplex)) - { - hw_flags_chg = 1; - switch (xd->link.link_duplex) - { - case ETH_LINK_HALF_DUPLEX: - hw_flags |= VNET_HW_INTERFACE_FLAG_HALF_DUPLEX; - break; - case ETH_LINK_FULL_DUPLEX: - hw_flags |= VNET_HW_INTERFACE_FLAG_FULL_DUPLEX; - break; - default: - break; - } - } - if (hw_flags_chg || (xd->link.link_speed != prev_link.link_speed)) - { - hw_flags_chg = 1; - switch (xd->link.link_speed) - { - case ETH_SPEED_NUM_10M: - hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_10M; - break; - case ETH_SPEED_NUM_100M: - hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_100M; - break; - case ETH_SPEED_NUM_1G: - hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_1G; - break; - case ETH_SPEED_NUM_10G: - hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_10G; - break; - case ETH_SPEED_NUM_40G: - hw_flags |= VNET_HW_INTERFACE_FLAG_SPEED_40G; - break; - case 0: - break; - default: - clib_warning ("unknown link speed %d", xd->link.link_speed); - break; - } - } - if (hw_flags_chg) - { - if (LINK_STATE_ELOGS) - { - vlib_main_t *vm = vlib_get_main (); - - ELOG_TYPE_DECLARE (e) = - { - .format = - "update-link-state: sw_if_index %d, new flags %d",.format_args - = "i4i4",}; - - struct - { - u32 sw_if_index; - u32 flags; - } *ed; - ed = ELOG_DATA (&vm->elog_main, e); - ed->sw_if_index = xd->vlib_sw_if_index; - ed->flags = hw_flags; - } - vnet_hw_interface_set_flags (vnm, xd->vlib_hw_if_index, hw_flags); - } -} - -static uword -dpdk_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) -{ - clib_error_t *error; - vnet_main_t *vnm = vnet_get_main (); - dpdk_main_t *dm = &dpdk_main; - ethernet_main_t *em = ðernet_main; - dpdk_device_t *xd; - vlib_thread_main_t *tm = vlib_get_thread_main (); - int i; - - error = dpdk_lib_init (dm); - - /* - * Turn on the input node if we found some devices to drive - * and we're not running worker threads or i/o threads - */ - - if (error == 0 && vec_len (dm->devices) > 0) - { - if (tm->n_vlib_mains == 1) - vlib_node_set_state (vm, dpdk_input_node.index, - VLIB_NODE_STATE_POLLING); - else - for (i = 0; i < tm->n_vlib_mains; i++) - if (vec_len (dm->devices_by_cpu[i]) > 0) - vlib_node_set_state (vlib_mains[i], dpdk_input_node.index, - VLIB_NODE_STATE_POLLING); - } - - if (error) - clib_error_report (error); - - tm->worker_thread_release = 1; - - f64 now = vlib_time_now (vm); - vec_foreach (xd, dm->devices) - { - dpdk_update_link_state (xd, now); - } - - { - /* - * Extra set up for bond interfaces: - * 1. Setup MACs for bond interfaces and their slave links which was set - * in dpdk_port_setup() but needs to be done again here to take effect. - * 2. Set up info for bond interface related CLI support. - */ - int nports = rte_eth_dev_count (); - if (nports > 0) - { - for (i = 0; i < nports; i++) - { - struct rte_eth_dev_info dev_info; - rte_eth_dev_info_get (i, &dev_info); - if (!dev_info.driver_name) -#if RTE_VERSION < RTE_VERSION_NUM(16, 11, 0, 0) - dev_info.driver_name = dev_info.pci_dev->driver->name; -#else - dev_info.driver_name = dev_info.pci_dev->driver->driver.name; -#endif - ASSERT (dev_info.driver_name); - if (strncmp (dev_info.driver_name, "rte_bond_pmd", 12) == 0) - { - u8 addr[6]; - u8 slink[16]; - int nlink = rte_eth_bond_slaves_get (i, slink, 16); - if (nlink > 0) - { - vnet_hw_interface_t *bhi; - ethernet_interface_t *bei; - int rv; - - /* Get MAC of 1st slave link */ - rte_eth_macaddr_get (slink[0], - (struct ether_addr *) addr); - /* Set MAC of bounded interface to that of 1st slave link */ - rv = - rte_eth_bond_mac_address_set (i, - (struct ether_addr *) - addr); - if (rv < 0) - clib_warning ("Failed to set MAC address"); - - /* Populate MAC of bonded interface in VPP hw tables */ - bhi = - vnet_get_hw_interface (vnm, - dm->devices[i].vlib_hw_if_index); - bei = - pool_elt_at_index (em->interfaces, bhi->hw_instance); - clib_memcpy (bhi->hw_address, addr, 6); - clib_memcpy (bei->address, addr, 6); - /* Init l3 packet size allowed on bonded interface */ - bhi->max_packet_bytes = ETHERNET_MAX_PACKET_BYTES; - bhi->max_l3_packet_bytes[VLIB_RX] = - bhi->max_l3_packet_bytes[VLIB_TX] = - ETHERNET_MAX_PACKET_BYTES - sizeof (ethernet_header_t); - while (nlink >= 1) - { /* for all slave links */ - int slave = slink[--nlink]; - dpdk_device_t *sdev = &dm->devices[slave]; - vnet_hw_interface_t *shi; - vnet_sw_interface_t *ssi; - /* Add MAC to all slave links except the first one */ - if (nlink) - rte_eth_dev_mac_addr_add (slave, - (struct ether_addr *) - addr, 0); - /* Set slaves bitmap for bonded interface */ - bhi->bond_info = - clib_bitmap_set (bhi->bond_info, - sdev->vlib_hw_if_index, 1); - /* Set slave link flags on slave interface */ - shi = - vnet_get_hw_interface (vnm, sdev->vlib_hw_if_index); - ssi = - vnet_get_sw_interface (vnm, sdev->vlib_sw_if_index); - shi->bond_info = VNET_HW_INTERFACE_BOND_INFO_SLAVE; - ssi->flags |= VNET_SW_INTERFACE_FLAG_BOND_SLAVE; - - /* Set l3 packet size allowed as the lowest of slave */ - if (bhi->max_l3_packet_bytes[VLIB_RX] > - shi->max_l3_packet_bytes[VLIB_RX]) - bhi->max_l3_packet_bytes[VLIB_RX] = - bhi->max_l3_packet_bytes[VLIB_TX] = - shi->max_l3_packet_bytes[VLIB_RX]; - - /* Set max packet size allowed as the lowest of slave */ - if (bhi->max_packet_bytes > shi->max_packet_bytes) - bhi->max_packet_bytes = shi->max_packet_bytes; - } - } - } - } - } - } - - while (1) - { - /* - * check each time through the loop in case intervals are changed - */ - f64 min_wait = dm->link_state_poll_interval < dm->stat_poll_interval ? - dm->link_state_poll_interval : dm->stat_poll_interval; - - vlib_process_wait_for_event_or_clock (vm, min_wait); - - if (dm->admin_up_down_in_progress) - /* skip the poll if an admin up down is in progress (on any interface) */ - continue; - - vec_foreach (xd, dm->devices) - { - f64 now = vlib_time_now (vm); - if ((now - xd->time_last_stats_update) >= dm->stat_poll_interval) - dpdk_update_counters (xd, now); - if ((now - xd->time_last_link_update) >= dm->link_state_poll_interval) - dpdk_update_link_state (xd, now); - - } - } - - return 0; -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (dpdk_process_node,static) = { - .function = dpdk_process, - .type = VLIB_NODE_TYPE_PROCESS, - .name = "dpdk-process", - .process_log2_n_stack_bytes = 17, -}; -/* *INDENT-ON* */ - -int -dpdk_set_stat_poll_interval (f64 interval) -{ - if (interval < DPDK_MIN_STATS_POLL_INTERVAL) - return (VNET_API_ERROR_INVALID_VALUE); - - dpdk_main.stat_poll_interval = interval; - - return 0; -} - -int -dpdk_set_link_state_poll_interval (f64 interval) -{ - if (interval < DPDK_MIN_LINK_POLL_INTERVAL) - return (VNET_API_ERROR_INVALID_VALUE); - - dpdk_main.link_state_poll_interval = interval; - - return 0; -} - -clib_error_t * -dpdk_init (vlib_main_t * vm) -{ - dpdk_main_t *dm = &dpdk_main; - vlib_node_t *ei; - clib_error_t *error = 0; - vlib_thread_main_t *tm = vlib_get_thread_main (); - - /* verify that structs are cacheline aligned */ - STATIC_ASSERT (offsetof (dpdk_device_t, cacheline0) == 0, - "Cache line marker must be 1st element in dpdk_device_t"); - STATIC_ASSERT (offsetof (dpdk_device_t, cacheline1) == - CLIB_CACHE_LINE_BYTES, - "Data in cache line 0 is bigger than cache line size"); - STATIC_ASSERT (offsetof (dpdk_worker_t, cacheline0) == 0, - "Cache line marker must be 1st element in dpdk_worker_t"); - STATIC_ASSERT (offsetof (frame_queue_trace_t, cacheline0) == 0, - "Cache line marker must be 1st element in frame_queue_trace_t"); - - dm->vlib_main = vm; - dm->vnet_main = vnet_get_main (); - dm->conf = &dpdk_config_main; - - ei = vlib_get_node_by_name (vm, (u8 *) "ethernet-input"); - if (ei == 0) - return clib_error_return (0, "ethernet-input node AWOL"); - - dm->ethernet_input_node_index = ei->index; - - dm->conf->nchannels = 4; - dm->conf->num_mbufs = dm->conf->num_mbufs ? dm->conf->num_mbufs : NB_MBUF; - vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet"); - - dm->dpdk_device_by_kni_port_id = hash_create (0, sizeof (uword)); - dm->vu_sw_if_index_by_listener_fd = hash_create (0, sizeof (uword)); - dm->vu_sw_if_index_by_sock_fd = hash_create (0, sizeof (uword)); - - /* $$$ use n_thread_stacks since it's known-good at this point */ - vec_validate (dm->recycle, tm->n_thread_stacks - 1); - - /* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */ - dm->buffer_flags_template = - (VLIB_BUFFER_TOTAL_LENGTH_VALID | VNET_BUFFER_RTE_MBUF_VALID - | IP_BUFFER_L4_CHECKSUM_COMPUTED | IP_BUFFER_L4_CHECKSUM_CORRECT); - - dm->stat_poll_interval = DPDK_STATS_POLL_INTERVAL; - dm->link_state_poll_interval = DPDK_LINK_POLL_INTERVAL; - - /* init CLI */ - if ((error = vlib_call_init_function (vm, dpdk_cli_init))) - return error; - - return error; -} - -VLIB_INIT_FUNCTION (dpdk_init); - - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/ipsec/cli.c b/vnet/vnet/devices/dpdk/ipsec/cli.c deleted file mode 100644 index 3b634e036da..00000000000 --- a/vnet/vnet/devices/dpdk/ipsec/cli.c +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2016 Intel and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <vnet/vnet.h> -#include <vnet/devices/dpdk/ipsec/ipsec.h> - -static void -dpdk_ipsec_show_mapping (vlib_main_t * vm, u16 detail_display) -{ - dpdk_crypto_main_t *dcm = &dpdk_crypto_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - u32 i, skip_master; - - if (detail_display) - vlib_cli_output (vm, "worker\t%10s\t%15s\tdir\tdev\tqp\n", - "cipher", "auth"); - else - vlib_cli_output (vm, "worker\tcrypto device id(type)\n"); - - skip_master = vlib_num_workers () > 0; - - for (i = 0; i < tm->n_vlib_mains; i++) - { - uword key, data; - u32 cpu_index = vlib_mains[i]->cpu_index; - crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index]; - u8 *s = 0; - - if (skip_master) - { - skip_master = 0; - continue; - } - - if (!detail_display) - { - i32 last_cdev = -1; - crypto_qp_data_t *qpd; - - s = format (s, "%u\t", cpu_index); - - /* *INDENT-OFF* */ - vec_foreach (qpd, cwm->qp_data) - { - u32 dev_id = qpd->dev_id; - - if ((u16) last_cdev != dev_id) - { - struct rte_cryptodev_info cdev_info; - - rte_cryptodev_info_get (dev_id, &cdev_info); - - s = format(s, "%u(%s)\t", dev_id, cdev_info.feature_flags & - RTE_CRYPTODEV_FF_HW_ACCELERATED ? "HW" : "SW"); - } - last_cdev = dev_id; - } - /* *INDENT-ON* */ - vlib_cli_output (vm, "%s", s); - } - else - { - char cipher_str[15], auth_str[15]; - struct rte_cryptodev_capabilities cap; - crypto_worker_qp_key_t *p_key = (crypto_worker_qp_key_t *) & key; - /* *INDENT-OFF* */ - hash_foreach (key, data, cwm->algo_qp_map, - ({ - cap.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC; - cap.sym.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER; - cap.sym.cipher.algo = p_key->cipher_algo; - check_algo_is_supported (&cap, cipher_str); - cap.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC; - cap.sym.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH; - cap.sym.auth.algo = p_key->auth_algo; - check_algo_is_supported (&cap, auth_str); - vlib_cli_output (vm, "%u\t%10s\t%15s\t%3s\t%u\t%u\n", - vlib_mains[i]->cpu_index, cipher_str, auth_str, - p_key->is_outbound ? "out" : "in", - cwm->qp_data[data].dev_id, - cwm->qp_data[data].qp_id); - })); - /* *INDENT-ON* */ - } - } -} - -static clib_error_t * -lcore_cryptodev_map_fn (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) -{ - unformat_input_t _line_input, *line_input = &_line_input; - u16 detail = 0; - - if (!unformat_user (input, unformat_line_input, line_input)) - return 0; - - while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (line_input, "verbose")) - detail = 1; - else - return clib_error_return (0, "parse error: '%U'", - format_unformat_error, line_input); - } - - unformat_free (line_input); - - dpdk_ipsec_show_mapping (vm, detail); - - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (lcore_cryptodev_map, static) = { - .path = "show crypto device mapping", - .short_help = - "show cryptodev device mapping <verbose>", - .function = lcore_cryptodev_map_fn, -}; -/* *INDENT-ON* */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/ipsec/crypto_node.c b/vnet/vnet/devices/dpdk/ipsec/crypto_node.c deleted file mode 100644 index 7b32704ec05..00000000000 --- a/vnet/vnet/devices/dpdk/ipsec/crypto_node.c +++ /dev/null @@ -1,210 +0,0 @@ -/* - *------------------------------------------------------------------ - * crypto_node.c - DPDK Cryptodev input node - * - * Copyright (c) 2016 Intel and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------ - */ - -#include <vlib/vlib.h> -#include <vnet/ip/ip.h> -#include <vnet/ethernet/ethernet.h> -#include <vnet/ipsec/ipsec.h> - -#include <vnet/devices/dpdk/ipsec/ipsec.h> - -#define foreach_dpdk_crypto_input_next \ - _(DROP, "error-drop") \ - _(ENCRYPT_POST, "dpdk-esp-encrypt-post") \ - _(DECRYPT_POST, "dpdk-esp-decrypt-post") - -typedef enum -{ -#define _(f,s) DPDK_CRYPTO_INPUT_NEXT_##f, - foreach_dpdk_crypto_input_next -#undef _ - DPDK_CRYPTO_INPUT_N_NEXT, -} dpdk_crypto_input_next_t; - -#define foreach_dpdk_crypto_input_error \ - _(DQ_COPS, "Crypto ops dequeued") \ - _(COP_FAILED, "Crypto op failed") - -typedef enum -{ -#define _(f,s) DPDK_CRYPTO_INPUT_ERROR_##f, - foreach_dpdk_crypto_input_error -#undef _ - DPDK_CRYPTO_INPUT_N_ERROR, -} dpdk_crypto_input_error_t; - -static char *dpdk_crypto_input_error_strings[] = { -#define _(n, s) s, - foreach_dpdk_crypto_input_error -#undef _ -}; - -vlib_node_registration_t dpdk_crypto_input_node; - -typedef struct -{ - u32 cdev; - u32 qp; - u32 status; - u32 sa_idx; - u32 next_index; -} dpdk_crypto_input_trace_t; - -static u8 * -format_dpdk_crypto_input_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - dpdk_crypto_input_trace_t *t = va_arg (*args, dpdk_crypto_input_trace_t *); - - s = format (s, "dpdk_crypto: cryptodev-id %u queue-pair %u next-index %d", - t->cdev, t->qp, t->next_index); - - s = format (s, " status %u sa-idx %u\n", t->status, t->sa_idx); - - return s; -} - -static_always_inline u32 -dpdk_crypto_dequeue (vlib_main_t * vm, vlib_node_runtime_t * node, - crypto_qp_data_t * qpd) -{ - u32 n_deq, *to_next = 0, next_index, n_cops, def_next_index; - struct rte_crypto_op **cops = qpd->cops; - - if (qpd->inflights == 0) - return 0; - - if (qpd->is_outbound) - def_next_index = DPDK_CRYPTO_INPUT_NEXT_ENCRYPT_POST; - else - def_next_index = DPDK_CRYPTO_INPUT_NEXT_DECRYPT_POST; - - n_cops = rte_cryptodev_dequeue_burst (qpd->dev_id, qpd->qp_id, - cops, VLIB_FRAME_SIZE); - n_deq = n_cops; - next_index = def_next_index; - - qpd->inflights -= n_cops; - ASSERT (qpd->inflights >= 0); - - while (n_cops > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - while (n_cops > 0 && n_left_to_next > 0) - { - u32 bi0, next0; - vlib_buffer_t *b0 = 0; - struct rte_crypto_op *cop; - struct rte_crypto_sym_op *sym_cop; - - cop = cops[0]; - cops += 1; - n_cops -= 1; - n_left_to_next -= 1; - - next0 = def_next_index; - - if (PREDICT_FALSE (cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) - { - next0 = DPDK_CRYPTO_INPUT_NEXT_DROP; - vlib_node_increment_counter (vm, dpdk_crypto_input_node.index, - DPDK_CRYPTO_INPUT_ERROR_COP_FAILED, - 1); - } - cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; - - sym_cop = (struct rte_crypto_sym_op *) (cop + 1); - b0 = vlib_buffer_from_rte_mbuf (sym_cop->m_src); - bi0 = vlib_get_buffer_index (vm, b0); - - to_next[0] = bi0; - to_next += 1; - - if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) - { - vlib_trace_next_frame (vm, node, next0); - dpdk_crypto_input_trace_t *tr = - vlib_add_trace (vm, node, b0, sizeof (*tr)); - tr->cdev = qpd->dev_id; - tr->qp = qpd->qp_id; - tr->status = cop->status; - tr->next_index = next0; - tr->sa_idx = vnet_buffer (b0)->ipsec.sad_index; - } - - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, - n_left_to_next, bi0, next0); - } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - crypto_free_cop (qpd, qpd->cops, n_deq); - - vlib_node_increment_counter (vm, dpdk_crypto_input_node.index, - DPDK_CRYPTO_INPUT_ERROR_DQ_COPS, n_deq); - return n_deq; -} - -static uword -dpdk_crypto_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * frame) -{ - u32 cpu_index = os_get_cpu_number (); - dpdk_crypto_main_t *dcm = &dpdk_crypto_main; - crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index]; - crypto_qp_data_t *qpd; - u32 n_deq = 0; - - /* *INDENT-OFF* */ - vec_foreach (qpd, cwm->qp_data) - n_deq += dpdk_crypto_dequeue(vm, node, qpd); - /* *INDENT-ON* */ - - return n_deq; -} - -VLIB_REGISTER_NODE (dpdk_crypto_input_node) = -{ - .function = dpdk_crypto_input_fn,.name = "dpdk-crypto-input",.format_trace = - format_dpdk_crypto_input_trace,.type = VLIB_NODE_TYPE_INPUT,.state = - VLIB_NODE_STATE_DISABLED,.n_errors = - DPDK_CRYPTO_INPUT_N_ERROR,.error_strings = - dpdk_crypto_input_error_strings,.n_next_nodes = - DPDK_CRYPTO_INPUT_N_NEXT,.next_nodes = - { -#define _(s,n) [DPDK_CRYPTO_INPUT_NEXT_##s] = n, - foreach_dpdk_crypto_input_next -#undef _ - } -,}; - -#if DPDK_CRYPTO==1 -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_crypto_input_node, dpdk_crypto_input_fn) -#endif -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/ipsec/dir.dox b/vnet/vnet/devices/dpdk/ipsec/dir.dox deleted file mode 100644 index ffebfc4d62e..00000000000 --- a/vnet/vnet/devices/dpdk/ipsec/dir.dox +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2016 Intel and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - @dir vnet/vnet/devices/dpdk/ipsec - @brief IPSec ESP encrypt/decrypt using DPDK Cryptodev API -*/ diff --git a/vnet/vnet/devices/dpdk/ipsec/dpdk_crypto_ipsec_doc.md b/vnet/vnet/devices/dpdk/ipsec/dpdk_crypto_ipsec_doc.md deleted file mode 100644 index 8089696f4a0..00000000000 --- a/vnet/vnet/devices/dpdk/ipsec/dpdk_crypto_ipsec_doc.md +++ /dev/null @@ -1,73 +0,0 @@ -# VPP IPSec implementation using DPDK Cryptodev API {#dpdk_crypto_ipsec_doc} - -This document is meant to contain all related information about implementation and usability. - - -## VPP IPsec with DPDK Cryptodev - -DPDK Cryptodev is an asynchronous crypto API that supports both Hardware and Software implementations (for more details refer to [DPDK Cryptography Device Library documentation](http://dpdk.org/doc/guides/prog_guide/cryptodev_lib.html)). - -When DPDK Cryptodev support is enabled, the node graph is modified by adding and replacing some of the nodes. - -The following nodes are replaced: -* esp-encrypt -> dpdk-esp-encrypt -* esp-decrypt -> dpdk-esp-decrypt - -The following nodes are added: -* dpdk-crypto-input : polling input node, basically dequeuing from crypto devices. -* dpdk-esp-encrypt-post : internal node. -* dpdk-esp-decrypt-post : internal node. - - -### How to enable VPP IPSec with DPDK Cryptodev support - -To enable DPDK Cryptodev support (disabled by default), we need the following env option: - - vpp_uses_dpdk_cryptodev=yes - -A couple of ways to achive this: -* uncomment/add it in the platforms config (ie. build-data/platforms/vpp.mk) -* set the option when building vpp (ie. make vpp_uses_dpdk_cryptodev=yes build-release) - - -### Crypto Resources allocation - -VPP allocates crypto resources based on a best effort approach: -* first allocate Hardware crypto resources, then Software. -* if there are not enough crypto resources for all workers, all packets will be dropped if they reach ESP encrypt/decrypt nodes, displaying the warning: - - 0: dpdk_ipsec_init: not enough cryptodevs for ipsec - - -### Configuration example - -No especial IPsec configuration is required. - -Once DPDK Cryptodev is enabled, the user just needs to provide cryptodevs in the startup.conf. - -Example startup.conf: - -``` -dpdk { - socket-mem 1024,1024 - num-mbufs 131072 - dev 0000:81:00.0 - dev 0000:81:00.1 - dev 0000:85:01.0 - dev 0000:85:01.1 - vdev cryptodev_aesni_mb_pmd,socket_id=1 - vdev cryptodev_aesni_mb_pmd,socket_id=1 -} -``` - -In the above configuration: -* 0000:85:01.0 and 0000:85:01.1 are crypto BDFs and they require the same driver binding as DPDK Ethernet devices but they do not support any extra configuration options. -* Two AESNI-MB Software Cryptodev PMDs are created in NUMA node 1. - -For further details refer to [DPDK Crypto Device Driver documentation](http://dpdk.org/doc/guides/cryptodevs/index.html) - -### Operational data - -The following CLI command displays the Cryptodev/Worker mapping: - - show crypto device mapping [verbose] diff --git a/vnet/vnet/devices/dpdk/ipsec/esp.h b/vnet/vnet/devices/dpdk/ipsec/esp.h deleted file mode 100644 index 7ef90c49816..00000000000 --- a/vnet/vnet/devices/dpdk/ipsec/esp.h +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Copyright (c) 2016 Intel and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __DPDK_ESP_H__ -#define __DPDK_ESP_H__ - -#include <vnet/devices/dpdk/ipsec/ipsec.h> -#include <vnet/ipsec/ipsec.h> -#include <vnet/ipsec/esp.h> - -typedef struct -{ - enum rte_crypto_cipher_algorithm algo; - u8 key_len; - u8 iv_len; -} dpdk_esp_crypto_alg_t; - -typedef struct -{ - enum rte_crypto_auth_algorithm algo; - u8 trunc_size; -} dpdk_esp_integ_alg_t; - -typedef struct -{ - dpdk_esp_crypto_alg_t *esp_crypto_algs; - dpdk_esp_integ_alg_t *esp_integ_algs; -} dpdk_esp_main_t; - -dpdk_esp_main_t dpdk_esp_main; - -static_always_inline void -dpdk_esp_init () -{ - dpdk_esp_main_t *em = &dpdk_esp_main; - dpdk_esp_integ_alg_t *i; - dpdk_esp_crypto_alg_t *c; - - vec_validate (em->esp_crypto_algs, IPSEC_CRYPTO_N_ALG - 1); - - c = &em->esp_crypto_algs[IPSEC_CRYPTO_ALG_AES_CBC_128]; - c->algo = RTE_CRYPTO_CIPHER_AES_CBC; - c->key_len = 16; - c->iv_len = 16; - - c = &em->esp_crypto_algs[IPSEC_CRYPTO_ALG_AES_CBC_192]; - c->algo = RTE_CRYPTO_CIPHER_AES_CBC; - c->key_len = 24; - c->iv_len = 16; - - c = &em->esp_crypto_algs[IPSEC_CRYPTO_ALG_AES_CBC_256]; - c->algo = RTE_CRYPTO_CIPHER_AES_CBC; - c->key_len = 32; - c->iv_len = 16; - - c = &em->esp_crypto_algs[IPSEC_CRYPTO_ALG_AES_GCM_128]; - c->algo = RTE_CRYPTO_CIPHER_AES_GCM; - c->key_len = 16; - c->iv_len = 8; - - vec_validate (em->esp_integ_algs, IPSEC_INTEG_N_ALG - 1); - - i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA1_96]; - i->algo = RTE_CRYPTO_AUTH_SHA1_HMAC; - i->trunc_size = 12; - - i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA_256_96]; - i->algo = RTE_CRYPTO_AUTH_SHA256_HMAC; - i->trunc_size = 12; - - i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA_256_128]; - i->algo = RTE_CRYPTO_AUTH_SHA256_HMAC; - i->trunc_size = 16; - - i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA_384_192]; - i->algo = RTE_CRYPTO_AUTH_SHA384_HMAC; - i->trunc_size = 24; - - i = &em->esp_integ_algs[IPSEC_INTEG_ALG_SHA_512_256]; - i->algo = RTE_CRYPTO_AUTH_SHA512_HMAC; - i->trunc_size = 32; - - i = &em->esp_integ_algs[IPSEC_INTEG_ALG_AES_GCM_128]; - i->algo = RTE_CRYPTO_AUTH_AES_GCM; - i->trunc_size = 16; -} - -static_always_inline int -add_del_sa_sess (u32 sa_index, u8 is_add) -{ - dpdk_crypto_main_t *dcm = &dpdk_crypto_main; - crypto_worker_main_t *cwm; - u8 skip_master = vlib_num_workers () > 0; - - /* *INDENT-OFF* */ - vec_foreach (cwm, dcm->workers_main) - { - crypto_sa_session_t *sa_sess; - u8 is_outbound; - - if (skip_master) - { - skip_master = 0; - continue; - } - - for (is_outbound = 0; is_outbound < 2; is_outbound++) - { - if (is_add) - { - pool_get (cwm->sa_sess_d[is_outbound], sa_sess); - } - else - { - u8 dev_id; - - sa_sess = pool_elt_at_index (cwm->sa_sess_d[is_outbound], sa_index); - dev_id = cwm->qp_data[sa_sess->qp_index].dev_id; - - if (!sa_sess->sess) - continue; - - if (rte_cryptodev_sym_session_free(dev_id, sa_sess->sess)) - { - clib_warning("failed to free session"); - return -1; - } - memset(sa_sess, 0, sizeof(sa_sess[0])); - } - } - } - /* *INDENT-OFF* */ - - return 0; -} - -static_always_inline int -translate_crypto_algo(ipsec_crypto_alg_t crypto_algo, - struct rte_crypto_sym_xform *cipher_xform) -{ - switch (crypto_algo) - { - case IPSEC_CRYPTO_ALG_NONE: - cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_NULL; - break; - case IPSEC_CRYPTO_ALG_AES_CBC_128: - case IPSEC_CRYPTO_ALG_AES_CBC_192: - case IPSEC_CRYPTO_ALG_AES_CBC_256: - cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; - break; - case IPSEC_CRYPTO_ALG_AES_GCM_128: - cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_AES_GCM; - break; - default: - return -1; - } - - cipher_xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER; - - return 0; -} - -static_always_inline int -translate_integ_algo(ipsec_integ_alg_t integ_alg, - struct rte_crypto_sym_xform *auth_xform, int use_esn) -{ - switch (integ_alg) { - case IPSEC_INTEG_ALG_NONE: - auth_xform->auth.algo = RTE_CRYPTO_AUTH_NULL; - auth_xform->auth.digest_length = 0; - break; - case IPSEC_INTEG_ALG_SHA1_96: - auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC; - auth_xform->auth.digest_length = 12; - break; - case IPSEC_INTEG_ALG_SHA_256_96: - auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA256_HMAC; - auth_xform->auth.digest_length = 12; - break; - case IPSEC_INTEG_ALG_SHA_256_128: - auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA256_HMAC; - auth_xform->auth.digest_length = 16; - break; - case IPSEC_INTEG_ALG_SHA_384_192: - auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA384_HMAC; - auth_xform->auth.digest_length = 24; - break; - case IPSEC_INTEG_ALG_SHA_512_256: - auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA512_HMAC; - auth_xform->auth.digest_length = 32; - break; - case IPSEC_INTEG_ALG_AES_GCM_128: - auth_xform->auth.algo = RTE_CRYPTO_AUTH_AES_GCM; - auth_xform->auth.digest_length = 16; - auth_xform->auth.add_auth_data_length = use_esn? 12 : 8; - break; - default: - return -1; - } - - auth_xform->type = RTE_CRYPTO_SYM_XFORM_AUTH; - - return 0; -} - -static_always_inline int -create_sym_sess(ipsec_sa_t *sa, crypto_sa_session_t *sa_sess, u8 is_outbound) -{ - u32 cpu_index = os_get_cpu_number(); - dpdk_crypto_main_t * dcm = &dpdk_crypto_main; - crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index]; - struct rte_crypto_sym_xform cipher_xform = {0}; - struct rte_crypto_sym_xform auth_xform = {0}; - struct rte_crypto_sym_xform *xfs; - uword key = 0, *data; - crypto_worker_qp_key_t *p_key = (crypto_worker_qp_key_t *)&key; - - if (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) - { - sa->crypto_key_len -= 4; - clib_memcpy(&sa->salt, &sa->crypto_key[sa->crypto_key_len], 4); - } - else - { - sa->salt = (u32) rand(); - } - - cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; - cipher_xform.cipher.key.data = sa->crypto_key; - cipher_xform.cipher.key.length = sa->crypto_key_len; - - auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; - auth_xform.auth.key.data = sa->integ_key; - auth_xform.auth.key.length = sa->integ_key_len; - - if (translate_crypto_algo(sa->crypto_alg, &cipher_xform) < 0) - return -1; - p_key->cipher_algo = cipher_xform.cipher.algo; - - if (translate_integ_algo(sa->integ_alg, &auth_xform, sa->use_esn) < 0) - return -1; - p_key->auth_algo = auth_xform.auth.algo; - - if (is_outbound) - { - cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; - auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; - cipher_xform.next = &auth_xform; - xfs = &cipher_xform; - } - else - { - cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT; - auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_VERIFY; - auth_xform.next = &cipher_xform; - xfs = &auth_xform; - } - - p_key->is_outbound = is_outbound; - - data = hash_get(cwm->algo_qp_map, key); - if (!data) - return -1; - - sa_sess->sess = - rte_cryptodev_sym_session_create(cwm->qp_data[*data].dev_id, xfs); - - if (!sa_sess->sess) - return -1; - - sa_sess->qp_index = (u8)*data; - - return 0; -} - -#endif /* __DPDK_ESP_H__ */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/ipsec/esp_decrypt.c b/vnet/vnet/devices/dpdk/ipsec/esp_decrypt.c deleted file mode 100644 index 89ab9f9bc43..00000000000 --- a/vnet/vnet/devices/dpdk/ipsec/esp_decrypt.c +++ /dev/null @@ -1,583 +0,0 @@ -/* - * esp_decrypt.c : IPSec ESP Decrypt node using DPDK Cryptodev - * - * Copyright (c) 2016 Intel and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <vnet/vnet.h> -#include <vnet/api_errno.h> -#include <vnet/ip/ip.h> - -#include <vnet/ipsec/ipsec.h> -#include <vnet/devices/dpdk/ipsec/ipsec.h> -#include <vnet/devices/dpdk/ipsec/esp.h> - -#define foreach_esp_decrypt_next \ -_(DROP, "error-drop") \ -_(IP4_INPUT, "ip4-input") \ -_(IP6_INPUT, "ip6-input") - -#define _(v, s) ESP_DECRYPT_NEXT_##v, -typedef enum { - foreach_esp_decrypt_next -#undef _ - ESP_DECRYPT_N_NEXT, -} esp_decrypt_next_t; - -#define foreach_esp_decrypt_error \ - _(RX_PKTS, "ESP pkts received") \ - _(DECRYPTION_FAILED, "ESP decryption failed") \ - _(REPLAY, "SA replayed packet") \ - _(NOT_IP, "Not IP packet (dropped)") \ - _(ENQ_FAIL, "Enqueue failed (buffer full)") \ - _(NO_CRYPTODEV, "Cryptodev not configured") \ - _(BAD_LEN, "Invalid ciphertext length") \ - _(UNSUPPORTED, "Cipher/Auth not supported") - - -typedef enum { -#define _(sym,str) ESP_DECRYPT_ERROR_##sym, - foreach_esp_decrypt_error -#undef _ - ESP_DECRYPT_N_ERROR, -} esp_decrypt_error_t; - -static char * esp_decrypt_error_strings[] = { -#define _(sym,string) string, - foreach_esp_decrypt_error -#undef _ -}; - -vlib_node_registration_t dpdk_esp_decrypt_node; - -typedef struct { - ipsec_crypto_alg_t crypto_alg; - ipsec_integ_alg_t integ_alg; -} esp_decrypt_trace_t; - -/* packet trace format function */ -static u8 * format_esp_decrypt_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - esp_decrypt_trace_t * t = va_arg (*args, esp_decrypt_trace_t *); - - s = format (s, "esp: crypto %U integrity %U", - format_ipsec_crypto_alg, t->crypto_alg, - format_ipsec_integ_alg, t->integ_alg); - return s; -} - -static uword -dpdk_esp_decrypt_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * from_frame) -{ - u32 n_left_from, *from, *to_next, next_index; - ipsec_main_t *im = &ipsec_main; - u32 cpu_index = os_get_cpu_number(); - dpdk_crypto_main_t * dcm = &dpdk_crypto_main; - dpdk_esp_main_t * em = &dpdk_esp_main; - u32 i; - - from = vlib_frame_vector_args (from_frame); - n_left_from = from_frame->n_vectors; - - if (PREDICT_FALSE(!dcm->workers_main)) - { - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_NO_CRYPTODEV, n_left_from); - vlib_buffer_free(vm, from, n_left_from); - return n_left_from; - } - - crypto_worker_main_t *cwm = vec_elt_at_index(dcm->workers_main, cpu_index); - u32 n_qps = vec_len(cwm->qp_data); - struct rte_crypto_op ** cops_to_enq[n_qps]; - u32 n_cop_qp[n_qps], * bi_to_enq[n_qps]; - - for (i = 0; i < n_qps; i++) - { - bi_to_enq[i] = cwm->qp_data[i].bi; - cops_to_enq[i] = cwm->qp_data[i].cops; - } - - memset(n_cop_qp, 0, n_qps * sizeof(u32)); - - crypto_alloc_cops(); - - next_index = ESP_DECRYPT_NEXT_DROP; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0, sa_index0 = ~0, seq, icv_size, iv_size; - vlib_buffer_t * b0; - esp_header_t * esp0; - ipsec_sa_t * sa0; - struct rte_mbuf * mb0 = 0; - const int BLOCK_SIZE = 16; - crypto_sa_session_t * sa_sess; - void * sess; - u16 qp_index; - struct rte_crypto_op * cop = 0; - - bi0 = from[0]; - from += 1; - n_left_from -= 1; - - b0 = vlib_get_buffer (vm, bi0); - esp0 = vlib_buffer_get_current (b0); - - sa_index0 = vnet_buffer(b0)->ipsec.sad_index; - sa0 = pool_elt_at_index (im->sad, sa_index0); - - seq = clib_host_to_net_u32(esp0->seq); - - /* anti-replay check */ - if (sa0->use_anti_replay) - { - int rv = 0; - - if (PREDICT_TRUE(sa0->use_esn)) - rv = esp_replay_check_esn(sa0, seq); - else - rv = esp_replay_check(sa0, seq); - - if (PREDICT_FALSE(rv)) - { - clib_warning ("anti-replay SPI %u seq %u", sa0->spi, seq); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_REPLAY, 1); - to_next[0] = bi0; - to_next += 1; - n_left_to_next -= 1; - goto trace; - } - } - - if (PREDICT_FALSE(sa0->integ_alg == IPSEC_INTEG_ALG_NONE) || - PREDICT_FALSE(sa0->crypto_alg == IPSEC_CRYPTO_ALG_NONE)) - { - clib_warning ("SPI %u : only cipher + auth supported", sa0->spi); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_UNSUPPORTED, 1); - to_next[0] = bi0; - to_next += 1; - n_left_to_next -= 1; - goto trace; - } - - sa_sess = pool_elt_at_index(cwm->sa_sess_d[0], sa_index0); - - if (PREDICT_FALSE(!sa_sess->sess)) - { - int ret = create_sym_sess(sa0, sa_sess, 0); - ASSERT(ret == 0); - } - - sess = sa_sess->sess; - qp_index = sa_sess->qp_index; - - ASSERT (vec_len (vec_elt (cwm->qp_data, qp_index).free_cops) > 0); - cop = vec_pop (vec_elt (cwm->qp_data, qp_index).free_cops); - ASSERT (cop->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED); - - cops_to_enq[qp_index][0] = cop; - cops_to_enq[qp_index] += 1; - n_cop_qp[qp_index] += 1; - bi_to_enq[qp_index][0] = bi0; - bi_to_enq[qp_index] += 1; - - rte_crypto_op_attach_sym_session(cop, sess); - - icv_size = em->esp_integ_algs[sa0->integ_alg].trunc_size; - iv_size = em->esp_crypto_algs[sa0->crypto_alg].iv_len; - - /* Convert vlib buffer to mbuf */ - mb0 = rte_mbuf_from_vlib_buffer(b0); - mb0->data_len = b0->current_length; - mb0->pkt_len = b0->current_length; - mb0->data_off = RTE_PKTMBUF_HEADROOM + b0->current_data; - - /* Outer IP header has already been stripped */ - u16 payload_len = rte_pktmbuf_pkt_len(mb0) - sizeof (esp_header_t) - - iv_size - icv_size; - - if ((payload_len & (BLOCK_SIZE - 1)) || (payload_len <= 0)) - { - clib_warning ("payload %u not multiple of %d\n", - payload_len, BLOCK_SIZE); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_BAD_LEN, 1); - vec_add (vec_elt (cwm->qp_data, qp_index).free_cops, &cop, 1); - bi_to_enq[qp_index] -= 1; - cops_to_enq[qp_index] -= 1; - n_cop_qp[qp_index] -= 1; - to_next[0] = bi0; - to_next += 1; - n_left_to_next -= 1; - goto trace; - } - - struct rte_crypto_sym_op *sym_cop = (struct rte_crypto_sym_op *)(cop + 1); - - sym_cop->m_src = mb0; - sym_cop->cipher.data.offset = sizeof (esp_header_t) + iv_size; - sym_cop->cipher.data.length = payload_len; - - u8 *iv = rte_pktmbuf_mtod_offset(mb0, void*, sizeof (esp_header_t)); - dpdk_cop_priv_t * priv = (dpdk_cop_priv_t *)(sym_cop + 1); - - if (sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) - { - dpdk_gcm_cnt_blk *icb = &priv->cb; - icb->salt = sa0->salt; - clib_memcpy(icb->iv, iv, 8); - icb->cnt = clib_host_to_net_u32(1); - sym_cop->cipher.iv.data = (u8 *)icb; - sym_cop->cipher.iv.phys_addr = cop->phys_addr + - (uintptr_t)icb - (uintptr_t)cop; - sym_cop->cipher.iv.length = 16; - - u8 *aad = priv->aad; - clib_memcpy(aad, iv - sizeof(esp_header_t), 8); - sym_cop->auth.aad.data = aad; - sym_cop->auth.aad.phys_addr = cop->phys_addr + - (uintptr_t)aad - (uintptr_t)cop; - if (sa0->use_esn) - { - *((u32*)&aad[8]) = sa0->seq_hi; - sym_cop->auth.aad.length = 12; - } - else - { - sym_cop->auth.aad.length = 8; - } - - sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(mb0, void*, - rte_pktmbuf_pkt_len(mb0) - icv_size); - sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(mb0, - rte_pktmbuf_pkt_len(mb0) - icv_size); - sym_cop->auth.digest.length = icv_size; - - } - else - { - sym_cop->cipher.iv.data = rte_pktmbuf_mtod_offset(mb0, void*, - sizeof (esp_header_t)); - sym_cop->cipher.iv.phys_addr = rte_pktmbuf_mtophys_offset(mb0, - sizeof (esp_header_t)); - sym_cop->cipher.iv.length = iv_size; - - if (sa0->use_esn) - { - dpdk_cop_priv_t* priv = (dpdk_cop_priv_t*) (sym_cop + 1); - u8* payload_end = rte_pktmbuf_mtod_offset( - mb0, u8*, sizeof(esp_header_t) + iv_size + payload_len); - - clib_memcpy (priv->icv, payload_end, icv_size); - *((u32*) payload_end) = sa0->seq_hi; - sym_cop->auth.data.offset = 0; - sym_cop->auth.data.length = sizeof(esp_header_t) + iv_size - + payload_len + sizeof(sa0->seq_hi); - sym_cop->auth.digest.data = priv->icv; - sym_cop->auth.digest.phys_addr = cop->phys_addr - + (uintptr_t) priv->icv - (uintptr_t) cop; - sym_cop->auth.digest.length = icv_size; - } - else - { - sym_cop->auth.data.offset = 0; - sym_cop->auth.data.length = sizeof(esp_header_t) + - iv_size + payload_len; - - sym_cop->auth.digest.data = rte_pktmbuf_mtod_offset(mb0, void*, - rte_pktmbuf_pkt_len(mb0) - icv_size); - sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(mb0, - rte_pktmbuf_pkt_len(mb0) - icv_size); - sym_cop->auth.digest.length = icv_size; - } - } - -trace: - if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) - { - esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - tr->crypto_alg = sa0->crypto_alg; - tr->integ_alg = sa0->integ_alg; - } - } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_RX_PKTS, - from_frame->n_vectors); - crypto_qp_data_t *qpd; - /* *INDENT-OFF* */ - vec_foreach_index (i, cwm->qp_data) - { - u32 enq; - - qpd = vec_elt_at_index(cwm->qp_data, i); - enq = rte_cryptodev_enqueue_burst(qpd->dev_id, qpd->qp_id, - qpd->cops, n_cop_qp[i]); - qpd->inflights += enq; - - if (PREDICT_FALSE(enq < n_cop_qp[i])) - { - crypto_free_cop (qpd, &qpd->cops[enq], n_cop_qp[i] - enq); - vlib_buffer_free (vm, &qpd->bi[enq], n_cop_qp[i] - enq); - - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_ENQ_FAIL, - n_cop_qp[i] - enq); - } - } - /* *INDENT-ON* */ - - return from_frame->n_vectors; -} - -VLIB_REGISTER_NODE (dpdk_esp_decrypt_node) = { - .function = dpdk_esp_decrypt_node_fn, - .name = "dpdk-esp-decrypt", - .vector_size = sizeof (u32), - .format_trace = format_esp_decrypt_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(esp_decrypt_error_strings), - .error_strings = esp_decrypt_error_strings, - - .n_next_nodes = ESP_DECRYPT_N_NEXT, - .next_nodes = { -#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n, - foreach_esp_decrypt_next -#undef _ - }, -}; - -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_node, dpdk_esp_decrypt_node_fn) - -/* - * Decrypt Post Node - */ - -#define foreach_esp_decrypt_post_error \ - _(PKTS, "ESP post pkts") - -typedef enum { -#define _(sym,str) ESP_DECRYPT_POST_ERROR_##sym, - foreach_esp_decrypt_post_error -#undef _ - ESP_DECRYPT_POST_N_ERROR, -} esp_decrypt_post_error_t; - -static char * esp_decrypt_post_error_strings[] = { -#define _(sym,string) string, - foreach_esp_decrypt_post_error -#undef _ -}; - -vlib_node_registration_t dpdk_esp_decrypt_post_node; - -static u8 * format_esp_decrypt_post_trace (u8 * s, va_list * args) -{ - return s; -} - -static uword -dpdk_esp_decrypt_post_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * from_frame) -{ - u32 n_left_from, *from, *to_next = 0, next_index; - ipsec_sa_t * sa0; - u32 sa_index0 = ~0; - ipsec_main_t *im = &ipsec_main; - dpdk_esp_main_t *em = &dpdk_esp_main; - - from = vlib_frame_vector_args (from_frame); - n_left_from = from_frame->n_vectors; - - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - while (n_left_from > 0 && n_left_to_next > 0) - { - esp_footer_t * f0; - u32 bi0, next0, icv_size, iv_size; - vlib_buffer_t * b0 = 0; - ip4_header_t *ih4 = 0, *oh4 = 0; - ip6_header_t *ih6 = 0, *oh6 = 0; - u8 tunnel_mode = 1; - u8 transport_ip6 = 0; - - next0 = ESP_DECRYPT_NEXT_DROP; - - bi0 = from[0]; - from += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - - sa_index0 = vnet_buffer(b0)->ipsec.sad_index; - sa0 = pool_elt_at_index (im->sad, sa_index0); - - to_next[0] = bi0; - to_next += 1; - - icv_size = em->esp_integ_algs[sa0->integ_alg].trunc_size; - iv_size = em->esp_crypto_algs[sa0->crypto_alg].iv_len; - - if (sa0->use_anti_replay) - { - esp_header_t * esp0 = vlib_buffer_get_current (b0); - u32 seq; - seq = clib_host_to_net_u32(esp0->seq); - if (PREDICT_TRUE(sa0->use_esn)) - esp_replay_advance_esn(sa0, seq); - else - esp_replay_advance(sa0, seq); - } - - ih4 = (ip4_header_t *) (b0->data + sizeof(ethernet_header_t)); - vlib_buffer_advance (b0, sizeof (esp_header_t) + iv_size); - - b0->current_length -= (icv_size + 2); - b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; - f0 = (esp_footer_t *) ((u8 *) vlib_buffer_get_current (b0) + - b0->current_length); - b0->current_length -= f0->pad_length; - - /* transport mode */ - if (PREDICT_FALSE(!sa0->is_tunnel && !sa0->is_tunnel_ip6)) - { - tunnel_mode = 0; - - if (PREDICT_TRUE((ih4->ip_version_and_header_length & 0xF0) != 0x40)) - { - if (PREDICT_TRUE((ih4->ip_version_and_header_length & 0xF0) == 0x60)) - transport_ip6 = 1; - else - { - clib_warning("next header: 0x%x", f0->next_header); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_NOT_IP, 1); - goto trace; - } - } - } - - if (PREDICT_TRUE (tunnel_mode)) - { - if (PREDICT_TRUE(f0->next_header == IP_PROTOCOL_IP_IN_IP)) - next0 = ESP_DECRYPT_NEXT_IP4_INPUT; - else if (f0->next_header == IP_PROTOCOL_IPV6) - next0 = ESP_DECRYPT_NEXT_IP6_INPUT; - else - { - clib_warning("next header: 0x%x", f0->next_header); - vlib_node_increment_counter (vm, dpdk_esp_decrypt_node.index, - ESP_DECRYPT_ERROR_DECRYPTION_FAILED, - 1); - goto trace; - } - } - /* transport mode */ - else - { - if (PREDICT_FALSE(transport_ip6)) - { - ih6 = (ip6_header_t *) (b0->data + sizeof(ethernet_header_t)); - vlib_buffer_advance (b0, -sizeof(ip6_header_t)); - oh6 = vlib_buffer_get_current (b0); - memmove(oh6, ih6, sizeof(ip6_header_t)); - - next0 = ESP_DECRYPT_NEXT_IP6_INPUT; - oh6->protocol = f0->next_header; - oh6->payload_length = - clib_host_to_net_u16 ( - vlib_buffer_length_in_chain(vm, b0) - - sizeof (ip6_header_t)); - } - else - { - vlib_buffer_advance (b0, -sizeof(ip4_header_t)); - oh4 = vlib_buffer_get_current (b0); - memmove(oh4, ih4, sizeof(ip4_header_t)); - - next0 = ESP_DECRYPT_NEXT_IP4_INPUT; - oh4->ip_version_and_header_length = 0x45; - oh4->fragment_id = 0; - oh4->flags_and_fragment_offset = 0; - oh4->protocol = f0->next_header; - oh4->length = clib_host_to_net_u16 ( - vlib_buffer_length_in_chain (vm, b0)); - oh4->checksum = ip4_header_checksum (oh4); - } - } - - vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32)~0; - -trace: - if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) - { - esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - tr->crypto_alg = sa0->crypto_alg; - tr->integ_alg = sa0->integ_alg; - } - - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, bi0, next0); - } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - vlib_node_increment_counter (vm, dpdk_esp_decrypt_post_node.index, - ESP_DECRYPT_POST_ERROR_PKTS, - from_frame->n_vectors); - - return from_frame->n_vectors; -} - -VLIB_REGISTER_NODE (dpdk_esp_decrypt_post_node) = { - .function = dpdk_esp_decrypt_post_node_fn, - .name = "dpdk-esp-decrypt-post", - .vector_size = sizeof (u32), - .format_trace = format_esp_decrypt_post_trace, - .type = VLIB_NODE_TYPE_INTERNAL, - - .n_errors = ARRAY_LEN(esp_decrypt_post_error_strings), - .error_strings = esp_decrypt_post_error_strings, - - .n_next_nodes = ESP_DECRYPT_N_NEXT, - .next_nodes = { -#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n, - foreach_esp_decrypt_next -#undef _ - }, -}; - -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_decrypt_post_node, dpdk_esp_decrypt_post_node_fn) diff --git a/vnet/vnet/devices/dpdk/ipsec/esp_encrypt.c b/vnet/vnet/devices/dpdk/ipsec/esp_encrypt.c deleted file mode 100644 index 10bb4616eef..00000000000 --- a/vnet/vnet/devices/dpdk/ipsec/esp_encrypt.c +++ /dev/null @@ -1,598 +0,0 @@ -/* - * esp_encrypt.c : IPSec ESP encrypt node using DPDK Cryptodev - * - * Copyright (c) 2016 Intel and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <vnet/vnet.h> -#include <vnet/api_errno.h> -#include <vnet/ip/ip.h> - -#include <vnet/ipsec/ipsec.h> -#include <vnet/devices/dpdk/ipsec/ipsec.h> -#include <vnet/devices/dpdk/ipsec/esp.h> - -#define foreach_esp_encrypt_next \ -_(DROP, "error-drop") \ -_(IP4_LOOKUP, "ip4-lookup") \ -_(IP6_LOOKUP, "ip6-lookup") \ -_(INTERFACE_OUTPUT, "interface-output") - -#define _(v, s) ESP_ENCRYPT_NEXT_##v, -typedef enum -{ - foreach_esp_encrypt_next -#undef _ - ESP_ENCRYPT_N_NEXT, -} esp_encrypt_next_t; - -#define foreach_esp_encrypt_error \ - _(RX_PKTS, "ESP pkts received") \ - _(SEQ_CYCLED, "sequence number cycled") \ - _(ENQ_FAIL, "Enqueue failed (buffer full)") \ - _(NO_CRYPTODEV, "Cryptodev not configured") \ - _(UNSUPPORTED, "Cipher/Auth not supported") - - -typedef enum -{ -#define _(sym,str) ESP_ENCRYPT_ERROR_##sym, - foreach_esp_encrypt_error -#undef _ - ESP_ENCRYPT_N_ERROR, -} esp_encrypt_error_t; - -static char *esp_encrypt_error_strings[] = { -#define _(sym,string) string, - foreach_esp_encrypt_error -#undef _ -}; - -vlib_node_registration_t dpdk_esp_encrypt_node; - -typedef struct -{ - u32 spi; - u32 seq; - ipsec_crypto_alg_t crypto_alg; - ipsec_integ_alg_t integ_alg; -} esp_encrypt_trace_t; - -/* packet trace format function */ -static u8 * -format_esp_encrypt_trace (u8 * s, va_list * args) -{ - CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); - CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); - esp_encrypt_trace_t *t = va_arg (*args, esp_encrypt_trace_t *); - - s = format (s, "esp: spi %u seq %u crypto %U integrity %U", - t->spi, t->seq, - format_ipsec_crypto_alg, t->crypto_alg, - format_ipsec_integ_alg, t->integ_alg); - return s; -} - -static uword -dpdk_esp_encrypt_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * from_frame) -{ - u32 n_left_from, *from, *to_next, next_index; - ipsec_main_t *im = &ipsec_main; - u32 cpu_index = os_get_cpu_number (); - dpdk_crypto_main_t *dcm = &dpdk_crypto_main; - dpdk_esp_main_t *em = &dpdk_esp_main; - u32 i; - - from = vlib_frame_vector_args (from_frame); - n_left_from = from_frame->n_vectors; - - if (PREDICT_FALSE (!dcm->workers_main)) - { - /* Likely there are not enough cryptodevs, so drop frame */ - vlib_node_increment_counter (vm, dpdk_esp_encrypt_node.index, - ESP_ENCRYPT_ERROR_NO_CRYPTODEV, - n_left_from); - vlib_buffer_free (vm, from, n_left_from); - return n_left_from; - } - - crypto_worker_main_t *cwm = vec_elt_at_index (dcm->workers_main, cpu_index); - u32 n_qps = vec_len (cwm->qp_data); - struct rte_crypto_op **cops_to_enq[n_qps]; - u32 n_cop_qp[n_qps], *bi_to_enq[n_qps]; - - for (i = 0; i < n_qps; i++) - { - bi_to_enq[i] = cwm->qp_data[i].bi; - cops_to_enq[i] = cwm->qp_data[i].cops; - } - - memset (n_cop_qp, 0, n_qps * sizeof (u32)); - - crypto_alloc_cops (); - - next_index = ESP_ENCRYPT_NEXT_DROP; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0, next0; - vlib_buffer_t *b0 = 0; - u32 sa_index0; - ipsec_sa_t *sa0; - ip4_and_esp_header_t *ih0, *oh0 = 0; - ip6_and_esp_header_t *ih6_0, *oh6_0 = 0; - struct rte_mbuf *mb0 = 0; - esp_footer_t *f0; - u8 is_ipv6; - u8 ip_hdr_size; - u8 next_hdr_type; - u8 transport_mode = 0; - const int BLOCK_SIZE = 16; - u32 iv_size; - u16 orig_sz; - crypto_sa_session_t *sa_sess; - void *sess; - struct rte_crypto_op *cop = 0; - u16 qp_index; - - bi0 = from[0]; - from += 1; - n_left_from -= 1; - - b0 = vlib_get_buffer (vm, bi0); - sa_index0 = vnet_buffer (b0)->ipsec.sad_index; - sa0 = pool_elt_at_index (im->sad, sa_index0); - - if (PREDICT_FALSE (esp_seq_advance (sa0))) - { - clib_warning ("sequence number counter has cycled SPI %u", - sa0->spi); - vlib_node_increment_counter (vm, dpdk_esp_encrypt_node.index, - ESP_ENCRYPT_ERROR_SEQ_CYCLED, 1); - //TODO: rekey SA - to_next[0] = bi0; - to_next += 1; - n_left_to_next -= 1; - goto trace; - } - - sa_sess = pool_elt_at_index (cwm->sa_sess_d[1], sa_index0); - if (PREDICT_FALSE (!sa_sess->sess)) - { - int ret = create_sym_sess (sa0, sa_sess, 1); - ASSERT (ret == 0); - } - - qp_index = sa_sess->qp_index; - sess = sa_sess->sess; - - ASSERT (vec_len (vec_elt (cwm->qp_data, qp_index).free_cops) > 0); - cop = vec_pop (vec_elt (cwm->qp_data, qp_index).free_cops); - ASSERT (cop->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED); - - cops_to_enq[qp_index][0] = cop; - cops_to_enq[qp_index] += 1; - n_cop_qp[qp_index] += 1; - bi_to_enq[qp_index][0] = bi0; - bi_to_enq[qp_index] += 1; - - ssize_t adv; - iv_size = em->esp_crypto_algs[sa0->crypto_alg].iv_len; - ih0 = vlib_buffer_get_current (b0); - orig_sz = b0->current_length; - is_ipv6 = (ih0->ip4.ip_version_and_header_length & 0xF0) == 0x60; - /* is ipv6 */ - if (PREDICT_TRUE (sa0->is_tunnel)) - { - if (PREDICT_TRUE (!is_ipv6)) - adv = -sizeof (ip4_and_esp_header_t); - else - adv = -sizeof (ip6_and_esp_header_t); - } - else - { - adv = -sizeof (esp_header_t); - if (PREDICT_TRUE (!is_ipv6)) - orig_sz -= sizeof (ip4_header_t); - else - orig_sz -= sizeof (ip6_header_t); - } - - /*transport mode save the eth header before it is overwritten */ - if (PREDICT_FALSE (!sa0->is_tunnel)) - { - ethernet_header_t *ieh0 = (ethernet_header_t *) - ((u8 *) vlib_buffer_get_current (b0) - - sizeof (ethernet_header_t)); - ethernet_header_t *oeh0 = - (ethernet_header_t *) ((u8 *) ieh0 + (adv - iv_size)); - clib_memcpy (oeh0, ieh0, sizeof (ethernet_header_t)); - } - - vlib_buffer_advance (b0, adv - iv_size); - - /* XXX IP6/ip4 and IP4/IP6 not supported, only IP4/IP4 and IP6/IP6 */ - - /* is ipv6 */ - if (PREDICT_FALSE (is_ipv6)) - { - ih6_0 = (ip6_and_esp_header_t *) ih0; - ip_hdr_size = sizeof (ip6_header_t); - oh6_0 = vlib_buffer_get_current (b0); - - if (PREDICT_TRUE (sa0->is_tunnel)) - { - next_hdr_type = IP_PROTOCOL_IPV6; - oh6_0->ip6.ip_version_traffic_class_and_flow_label = - ih6_0->ip6.ip_version_traffic_class_and_flow_label; - } - else - { - next_hdr_type = ih6_0->ip6.protocol; - memmove (oh6_0, ih6_0, sizeof (ip6_header_t)); - } - - oh6_0->ip6.protocol = IP_PROTOCOL_IPSEC_ESP; - oh6_0->ip6.hop_limit = 254; - oh6_0->esp.spi = clib_net_to_host_u32 (sa0->spi); - oh6_0->esp.seq = clib_net_to_host_u32 (sa0->seq); - } - else - { - ip_hdr_size = sizeof (ip4_header_t); - oh0 = vlib_buffer_get_current (b0); - - if (PREDICT_TRUE (sa0->is_tunnel)) - { - next_hdr_type = IP_PROTOCOL_IP_IN_IP; - oh0->ip4.tos = ih0->ip4.tos; - } - else - { - next_hdr_type = ih0->ip4.protocol; - memmove (oh0, ih0, sizeof (ip4_header_t)); - } - - oh0->ip4.ip_version_and_header_length = 0x45; - oh0->ip4.fragment_id = 0; - oh0->ip4.flags_and_fragment_offset = 0; - oh0->ip4.ttl = 254; - oh0->ip4.protocol = IP_PROTOCOL_IPSEC_ESP; - oh0->esp.spi = clib_net_to_host_u32 (sa0->spi); - oh0->esp.seq = clib_net_to_host_u32 (sa0->seq); - } - - if (PREDICT_TRUE (sa0->is_tunnel && !sa0->is_tunnel_ip6)) - { - oh0->ip4.src_address.as_u32 = sa0->tunnel_src_addr.ip4.as_u32; - oh0->ip4.dst_address.as_u32 = sa0->tunnel_dst_addr.ip4.as_u32; - - /* in tunnel mode send it back to FIB */ - next0 = ESP_ENCRYPT_NEXT_IP4_LOOKUP; - vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; - } - else if (sa0->is_tunnel && sa0->is_tunnel_ip6) - { - oh6_0->ip6.src_address.as_u64[0] = - sa0->tunnel_src_addr.ip6.as_u64[0]; - oh6_0->ip6.src_address.as_u64[1] = - sa0->tunnel_src_addr.ip6.as_u64[1]; - oh6_0->ip6.dst_address.as_u64[0] = - sa0->tunnel_dst_addr.ip6.as_u64[0]; - oh6_0->ip6.dst_address.as_u64[1] = - sa0->tunnel_dst_addr.ip6.as_u64[1]; - - /* in tunnel mode send it back to FIB */ - next0 = ESP_ENCRYPT_NEXT_IP6_LOOKUP; - vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; - } - else - { - next0 = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT; - transport_mode = 1; - } - - ASSERT (sa0->crypto_alg < IPSEC_CRYPTO_N_ALG); - ASSERT (sa0->crypto_alg != IPSEC_CRYPTO_ALG_NONE); - - int blocks = 1 + (orig_sz + 1) / BLOCK_SIZE; - - /* pad packet in input buffer */ - u8 pad_bytes = BLOCK_SIZE * blocks - 2 - orig_sz; - u8 i; - u8 *padding = vlib_buffer_get_current (b0) + b0->current_length; - - for (i = 0; i < pad_bytes; ++i) - padding[i] = i + 1; - - f0 = vlib_buffer_get_current (b0) + b0->current_length + pad_bytes; - f0->pad_length = pad_bytes; - f0->next_header = next_hdr_type; - b0->current_length += pad_bytes + 2 + - em->esp_integ_algs[sa0->integ_alg].trunc_size; - - vnet_buffer (b0)->sw_if_index[VLIB_RX] = - vnet_buffer (b0)->sw_if_index[VLIB_RX]; - b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; - - struct rte_crypto_sym_op *sym_cop; - sym_cop = (struct rte_crypto_sym_op *) (cop + 1); - - dpdk_cop_priv_t *priv = (dpdk_cop_priv_t *) (sym_cop + 1); - - vnet_buffer (b0)->unused[0] = next0; - - mb0 = rte_mbuf_from_vlib_buffer (b0); - mb0->data_len = b0->current_length; - mb0->pkt_len = b0->current_length; - mb0->data_off = RTE_PKTMBUF_HEADROOM + b0->current_data; - - rte_crypto_op_attach_sym_session (cop, sess); - - sym_cop->m_src = mb0; - - dpdk_gcm_cnt_blk *icb = &priv->cb; - icb->salt = sa0->salt; - icb->iv[0] = sa0->seq; - icb->iv[1] = sa0->seq_hi; - - if (sa0->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) - { - icb->cnt = clib_host_to_net_u32 (1); - clib_memcpy (vlib_buffer_get_current (b0) + ip_hdr_size + - sizeof (esp_header_t), icb->iv, 8); - sym_cop->cipher.data.offset = - ip_hdr_size + sizeof (esp_header_t) + iv_size; - sym_cop->cipher.data.length = BLOCK_SIZE * blocks; - sym_cop->cipher.iv.length = 16; - } - else - { - sym_cop->cipher.data.offset = - ip_hdr_size + sizeof (esp_header_t); - sym_cop->cipher.data.length = BLOCK_SIZE * blocks + iv_size; - sym_cop->cipher.iv.length = iv_size; - } - - sym_cop->cipher.iv.data = (u8 *) icb; - sym_cop->cipher.iv.phys_addr = cop->phys_addr + (uintptr_t) icb - - (uintptr_t) cop; - - - ASSERT (sa0->integ_alg < IPSEC_INTEG_N_ALG); - ASSERT (sa0->integ_alg != IPSEC_INTEG_ALG_NONE); - - if (PREDICT_FALSE (sa0->integ_alg == IPSEC_INTEG_ALG_AES_GCM_128)) - { - u8 *aad = priv->aad; - clib_memcpy (aad, vlib_buffer_get_current (b0) + ip_hdr_size, - 8); - sym_cop->auth.aad.data = aad; - sym_cop->auth.aad.phys_addr = cop->phys_addr + - (uintptr_t) aad - (uintptr_t) cop; - - if (PREDICT_FALSE (sa0->use_esn)) - { - *((u32 *) & aad[8]) = sa0->seq_hi; - sym_cop->auth.aad.length = 12; - } - else - { - sym_cop->auth.aad.length = 8; - } - } - else - { - sym_cop->auth.data.offset = ip_hdr_size; - sym_cop->auth.data.length = b0->current_length - ip_hdr_size - - em->esp_integ_algs[sa0->integ_alg].trunc_size; - - if (PREDICT_FALSE (sa0->use_esn)) - { - u8 *payload_end = - vlib_buffer_get_current (b0) + b0->current_length; - *((u32 *) payload_end) = sa0->seq_hi; - sym_cop->auth.data.length += sizeof (sa0->seq_hi); - } - } - sym_cop->auth.digest.data = vlib_buffer_get_current (b0) + - b0->current_length - - em->esp_integ_algs[sa0->integ_alg].trunc_size; - sym_cop->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset (mb0, - b0->current_length - - - em->esp_integ_algs - [sa0->integ_alg].trunc_size); - sym_cop->auth.digest.length = - em->esp_integ_algs[sa0->integ_alg].trunc_size; - - - if (PREDICT_FALSE (is_ipv6)) - { - oh6_0->ip6.payload_length = - clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - - sizeof (ip6_header_t)); - } - else - { - oh0->ip4.length = - clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0)); - oh0->ip4.checksum = ip4_header_checksum (&oh0->ip4); - } - - if (transport_mode) - vlib_buffer_advance (b0, -sizeof (ethernet_header_t)); - - trace: - if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) - { - esp_encrypt_trace_t *tr = - vlib_add_trace (vm, node, b0, sizeof (*tr)); - tr->spi = sa0->spi; - tr->seq = sa0->seq - 1; - tr->crypto_alg = sa0->crypto_alg; - tr->integ_alg = sa0->integ_alg; - } - } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - vlib_node_increment_counter (vm, dpdk_esp_encrypt_node.index, - ESP_ENCRYPT_ERROR_RX_PKTS, - from_frame->n_vectors); - crypto_qp_data_t *qpd; - /* *INDENT-OFF* */ - vec_foreach_index (i, cwm->qp_data) - { - u32 enq; - - qpd = vec_elt_at_index(cwm->qp_data, i); - enq = rte_cryptodev_enqueue_burst(qpd->dev_id, qpd->qp_id, - qpd->cops, n_cop_qp[i]); - qpd->inflights += enq; - - if (PREDICT_FALSE(enq < n_cop_qp[i])) - { - crypto_free_cop (qpd, &qpd->cops[enq], n_cop_qp[i] - enq); - vlib_buffer_free (vm, &qpd->bi[enq], n_cop_qp[i] - enq); - - vlib_node_increment_counter (vm, dpdk_esp_encrypt_node.index, - ESP_ENCRYPT_ERROR_ENQ_FAIL, - n_cop_qp[i] - enq); - } - } - /* *INDENT-ON* */ - - return from_frame->n_vectors; -} - -VLIB_REGISTER_NODE (dpdk_esp_encrypt_node) = -{ - .function = dpdk_esp_encrypt_node_fn,.name = "dpdk-esp-encrypt",.flags = - VLIB_NODE_FLAG_IS_OUTPUT,.vector_size = sizeof (u32),.format_trace = - format_esp_encrypt_trace,.n_errors = - ARRAY_LEN (esp_encrypt_error_strings),.error_strings = - esp_encrypt_error_strings,.n_next_nodes = 1,.next_nodes = - { - [ESP_ENCRYPT_NEXT_DROP] = "error-drop",} -}; - -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_encrypt_node, dpdk_esp_encrypt_node_fn) -/* - * ESP Encrypt Post Node - */ -#define foreach_esp_encrypt_post_error \ - _(PKTS, "ESP post pkts") - typedef enum - { -#define _(sym,str) ESP_ENCRYPT_POST_ERROR_##sym, - foreach_esp_encrypt_post_error -#undef _ - ESP_ENCRYPT_POST_N_ERROR, - } esp_encrypt_post_error_t; - - static char *esp_encrypt_post_error_strings[] = { -#define _(sym,string) string, - foreach_esp_encrypt_post_error -#undef _ - }; - -vlib_node_registration_t dpdk_esp_encrypt_post_node; - -static u8 * -format_esp_encrypt_post_trace (u8 * s, va_list * args) -{ - return s; -} - -static uword -dpdk_esp_encrypt_post_node_fn (vlib_main_t * vm, - vlib_node_runtime_t * node, - vlib_frame_t * from_frame) -{ - u32 n_left_from, *from, *to_next = 0, next_index; - - from = vlib_frame_vector_args (from_frame); - n_left_from = from_frame->n_vectors; - - next_index = node->cached_next_index; - - while (n_left_from > 0) - { - u32 n_left_to_next; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - while (n_left_from > 0 && n_left_to_next > 0) - { - u32 bi0, next0; - vlib_buffer_t *b0 = 0; - - bi0 = from[0]; - from += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - - to_next[0] = bi0; - to_next += 1; - - next0 = vnet_buffer (b0)->unused[0]; - - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, bi0, - next0); - } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - vlib_node_increment_counter (vm, dpdk_esp_encrypt_post_node.index, - ESP_ENCRYPT_POST_ERROR_PKTS, - from_frame->n_vectors); - - return from_frame->n_vectors; -} - -VLIB_REGISTER_NODE (dpdk_esp_encrypt_post_node) = -{ - .function = dpdk_esp_encrypt_post_node_fn,.name = - "dpdk-esp-encrypt-post",.vector_size = sizeof (u32),.format_trace = - format_esp_encrypt_post_trace,.type = VLIB_NODE_TYPE_INTERNAL,.n_errors = - ARRAY_LEN (esp_encrypt_post_error_strings),.error_strings = - esp_encrypt_post_error_strings,.n_next_nodes = - ESP_ENCRYPT_N_NEXT,.next_nodes = - { -#define _(s,n) [ESP_ENCRYPT_NEXT_##s] = n, - foreach_esp_encrypt_next -#undef _ - } -}; - -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_esp_encrypt_post_node, - dpdk_esp_encrypt_post_node_fn) -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/ipsec/ipsec.c b/vnet/vnet/devices/dpdk/ipsec/ipsec.c deleted file mode 100644 index de253f02636..00000000000 --- a/vnet/vnet/devices/dpdk/ipsec/ipsec.c +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Copyright (c) 2016 Intel and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <vnet/vnet.h> -#include <vnet/ip/ip.h> -#include <vnet/api_errno.h> -#include <vnet/devices/dpdk/dpdk.h> -#include <vnet/devices/dpdk/ipsec/ipsec.h> -#include <vnet/devices/dpdk/ipsec/esp.h> -#include <vnet/ipsec/ipsec.h> - -#define DPDK_CRYPTO_NB_OBJS 2048 -#define DPDK_CRYPTO_CACHE_SIZE 512 -#define DPDK_CRYPTO_PRIV_SIZE 128 -#define DPDK_CRYPTO_N_QUEUE_DESC 512 -#define DPDK_CRYPTO_NB_COPS (1024 * 4) - -/* - * return: - * -1: update failed - * 0: already exist - * 1: mapped - */ -static int -update_qp_data (crypto_worker_main_t * cwm, - u8 cdev_id, u16 qp_id, u8 is_outbound, u16 * idx) -{ - crypto_qp_data_t *qpd; - - /* *INDENT-OFF* */ - vec_foreach_index (*idx, cwm->qp_data) - { - qpd = vec_elt_at_index(cwm->qp_data, *idx); - - if (qpd->dev_id == cdev_id && qpd->qp_id == qp_id && - qpd->is_outbound == is_outbound) - return 0; - } - /* *INDENT-ON* */ - - vec_add2 (cwm->qp_data, qpd, 1); - - qpd->dev_id = cdev_id; - qpd->qp_id = qp_id; - qpd->is_outbound = is_outbound; - - return 1; -} - -/* - * return: - * -1: error - * 0: already exist - * 1: mapped - */ -static int -add_mapping (crypto_worker_main_t * cwm, - u8 cdev_id, u16 qp, u8 is_outbound, - const struct rte_cryptodev_capabilities *cipher_cap, - const struct rte_cryptodev_capabilities *auth_cap) -{ - int mapped; - u16 qp_index; - uword key = 0, data, *ret; - crypto_worker_qp_key_t *p_key = (crypto_worker_qp_key_t *) & key; - - p_key->cipher_algo = (u8) cipher_cap->sym.cipher.algo; - p_key->auth_algo = (u8) auth_cap->sym.auth.algo; - p_key->is_outbound = is_outbound; - - ret = hash_get (cwm->algo_qp_map, key); - if (ret) - return 0; - - mapped = update_qp_data (cwm, cdev_id, qp, is_outbound, &qp_index); - if (mapped < 0) - return -1; - - data = (uword) qp_index; - - ret = hash_set (cwm->algo_qp_map, key, data); - if (!ret) - rte_panic ("Failed to insert hash table\n"); - - return mapped; -} - -/* - * return: - * 0: already exist - * 1: mapped - */ -static int -add_cdev_mapping (crypto_worker_main_t * cwm, - struct rte_cryptodev_info *dev_info, u8 cdev_id, - u16 qp, u8 is_outbound) -{ - const struct rte_cryptodev_capabilities *i, *j; - u32 mapped = 0; - - for (i = dev_info->capabilities; i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) - { - if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER) - continue; - - if (check_algo_is_supported (i, NULL) != 0) - continue; - - for (j = dev_info->capabilities; j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; - j++) - { - int status = 0; - - if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH) - continue; - - if (check_algo_is_supported (j, NULL) != 0) - continue; - - status = add_mapping (cwm, cdev_id, qp, is_outbound, i, j); - if (status == 1) - mapped += 1; - if (status < 0) - return status; - } - } - - return mapped; -} - -static int -check_cryptodev_queues () -{ - u32 n_qs = 0; - u8 cdev_id; - u32 n_req_qs = 2; - - if (vlib_num_workers () > 0) - n_req_qs = vlib_num_workers () * 2; - - for (cdev_id = 0; cdev_id < rte_cryptodev_count (); cdev_id++) - { - struct rte_cryptodev_info cdev_info; - - rte_cryptodev_info_get (cdev_id, &cdev_info); - - if (! - (cdev_info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) - continue; - - n_qs += cdev_info.max_nb_queue_pairs; - } - - if (n_qs >= n_req_qs) - return 0; - else - return -1; -} - -static clib_error_t * -dpdk_ipsec_init (vlib_main_t * vm) -{ - dpdk_crypto_main_t *dcm = &dpdk_crypto_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - struct rte_cryptodev_config dev_conf; - struct rte_cryptodev_qp_conf qp_conf; - struct rte_cryptodev_info cdev_info; - struct rte_mempool *rmp; - i32 dev_id, ret; - u32 i, skip_master; - - if (check_cryptodev_queues () < 0) - return clib_error_return (0, "not enough cryptodevs for ipsec"); - - vec_alloc (dcm->workers_main, tm->n_vlib_mains); - _vec_len (dcm->workers_main) = tm->n_vlib_mains; - - fprintf (stdout, "DPDK Cryptodevs info:\n"); - fprintf (stdout, "dev_id\tn_qp\tnb_obj\tcache_size\n"); - /* HW cryptodevs have higher dev_id, use HW first */ - for (dev_id = rte_cryptodev_count () - 1; dev_id >= 0; dev_id--) - { - u16 max_nb_qp, qp = 0; - skip_master = vlib_num_workers () > 0; - - rte_cryptodev_info_get (dev_id, &cdev_info); - - if (! - (cdev_info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING)) - continue; - - max_nb_qp = cdev_info.max_nb_queue_pairs; - - for (i = 0; i < tm->n_vlib_mains; i++) - { - u8 is_outbound; - crypto_worker_main_t *cwm; - uword *map; - - if (skip_master) - { - skip_master = 0; - continue; - } - - cwm = vec_elt_at_index (dcm->workers_main, i); - map = cwm->algo_qp_map; - - if (!map) - { - map = hash_create (0, sizeof (crypto_worker_qp_key_t)); - if (!map) - return clib_error_return (0, "unable to create hash table " - "for worker %u", - vlib_mains[i]->cpu_index); - cwm->algo_qp_map = map; - } - - for (is_outbound = 0; is_outbound < 2 && qp < max_nb_qp; - is_outbound++) - { - int mapped = add_cdev_mapping (cwm, &cdev_info, - dev_id, qp, is_outbound); - if (mapped > 0) - qp++; - - if (mapped < 0) - return clib_error_return (0, - "too many queues for one worker"); - } - } - - if (qp == 0) - continue; - - dev_conf.socket_id = rte_cryptodev_socket_id (dev_id); - dev_conf.nb_queue_pairs = cdev_info.max_nb_queue_pairs; - dev_conf.session_mp.nb_objs = DPDK_CRYPTO_NB_OBJS; - dev_conf.session_mp.cache_size = DPDK_CRYPTO_CACHE_SIZE; - - ret = rte_cryptodev_configure (dev_id, &dev_conf); - if (ret < 0) - return clib_error_return (0, "cryptodev %u config error", dev_id); - - qp_conf.nb_descriptors = DPDK_CRYPTO_N_QUEUE_DESC; - for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++) - { - ret = rte_cryptodev_queue_pair_setup (dev_id, qp, &qp_conf, - dev_conf.socket_id); - if (ret < 0) - return clib_error_return (0, "cryptodev %u qp %u setup error", - dev_id, qp); - } - fprintf (stdout, "%u\t%u\t%u\t%u\n", dev_id, dev_conf.nb_queue_pairs, - DPDK_CRYPTO_NB_OBJS, DPDK_CRYPTO_CACHE_SIZE); - } - - u32 socket_id = rte_socket_id (); - - vec_validate_aligned (dcm->cop_pools, socket_id, CLIB_CACHE_LINE_BYTES); - - /* pool already exists, nothing to do */ - if (dcm->cop_pools[socket_id]) - return 0; - - u8 *pool_name = format (0, "crypto_op_pool_socket%u%c", socket_id, 0); - - rmp = rte_crypto_op_pool_create ((char *) pool_name, - RTE_CRYPTO_OP_TYPE_SYMMETRIC, - DPDK_CRYPTO_NB_COPS * - (1 + vlib_num_workers ()), - DPDK_CRYPTO_CACHE_SIZE, - DPDK_CRYPTO_PRIV_SIZE, socket_id); - vec_free (pool_name); - - if (!rmp) - return clib_error_return (0, "failed to allocate mempool on socket %u", - socket_id); - dcm->cop_pools[socket_id] = rmp; - - dpdk_esp_init (); - - if (vec_len (vlib_mains) == 0) - vlib_node_set_state (&vlib_global_main, dpdk_crypto_input_node.index, - VLIB_NODE_STATE_POLLING); - else - for (i = 1; i < tm->n_vlib_mains; i++) - vlib_node_set_state (vlib_mains[i], dpdk_crypto_input_node.index, - VLIB_NODE_STATE_POLLING); - - return 0; -} - -VLIB_MAIN_LOOP_ENTER_FUNCTION (dpdk_ipsec_init); - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/ipsec/ipsec.h b/vnet/vnet/devices/dpdk/ipsec/ipsec.h deleted file mode 100644 index e6c7498c0d3..00000000000 --- a/vnet/vnet/devices/dpdk/ipsec/ipsec.h +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright (c) 2016 Intel and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __DPDK_IPSEC_H__ -#define __DPDK_IPSEC_H__ - -#include <vnet/vnet.h> - -#undef always_inline -#include <rte_crypto.h> -#include <rte_cryptodev.h> - -#if CLIB_DEBUG > 0 -#define always_inline static inline -#else -#define always_inline static inline __attribute__ ((__always_inline__)) -#endif - - -#define MAX_QP_PER_LCORE 16 - -typedef struct -{ - u32 salt; - u32 iv[2]; - u32 cnt; -} dpdk_gcm_cnt_blk; - -typedef struct -{ - dpdk_gcm_cnt_blk cb; - union - { - u8 aad[12]; - u8 icv[64]; - }; -} dpdk_cop_priv_t; - -typedef struct -{ - u8 cipher_algo; - u8 auth_algo; - u8 is_outbound; -} crypto_worker_qp_key_t; - -typedef struct -{ - u16 dev_id; - u16 qp_id; - u16 is_outbound; - i16 inflights; - u32 bi[VLIB_FRAME_SIZE]; - struct rte_crypto_op *cops[VLIB_FRAME_SIZE]; - struct rte_crypto_op **free_cops; -} crypto_qp_data_t; - -typedef struct -{ - u8 qp_index; - void *sess; -} crypto_sa_session_t; - -typedef struct -{ - crypto_sa_session_t *sa_sess_d[2]; - crypto_qp_data_t *qp_data; - uword *algo_qp_map; -} crypto_worker_main_t; - -typedef struct -{ - struct rte_mempool **cop_pools; - crypto_worker_main_t *workers_main; -} dpdk_crypto_main_t; - -dpdk_crypto_main_t dpdk_crypto_main; - -extern vlib_node_registration_t dpdk_crypto_input_node; - -#define CRYPTO_N_FREE_COPS (VLIB_FRAME_SIZE * 3) - -static_always_inline void -crypto_alloc_cops () -{ - dpdk_crypto_main_t *dcm = &dpdk_crypto_main; - u32 cpu_index = os_get_cpu_number (); - crypto_worker_main_t *cwm = &dcm->workers_main[cpu_index]; - unsigned socket_id = rte_socket_id (); - crypto_qp_data_t *qpd; - - /* *INDENT-OFF* */ - vec_foreach (qpd, cwm->qp_data) - { - u32 l = vec_len (qpd->free_cops); - - if (PREDICT_FALSE (l < VLIB_FRAME_SIZE)) - { - u32 n_alloc; - - if (PREDICT_FALSE (!qpd->free_cops)) - vec_alloc (qpd->free_cops, CRYPTO_N_FREE_COPS); - - n_alloc = rte_crypto_op_bulk_alloc (dcm->cop_pools[socket_id], - RTE_CRYPTO_OP_TYPE_SYMMETRIC, - &qpd->free_cops[l], - CRYPTO_N_FREE_COPS - l - 1); - - _vec_len (qpd->free_cops) = l + n_alloc; - } - } - /* *INDENT-ON* */ -} - -static_always_inline void -crypto_free_cop (crypto_qp_data_t * qpd, struct rte_crypto_op **cops, u32 n) -{ - u32 l = vec_len (qpd->free_cops); - - if (l + n >= CRYPTO_N_FREE_COPS) - { - l -= VLIB_FRAME_SIZE; - rte_mempool_put_bulk (cops[0]->mempool, - (void **) &qpd->free_cops[l], VLIB_FRAME_SIZE); - } - clib_memcpy (&qpd->free_cops[l], cops, sizeof (*cops) * n); - - _vec_len (qpd->free_cops) = l + n; -} - -static_always_inline int -check_algo_is_supported (const struct rte_cryptodev_capabilities *cap, - char *name) -{ - struct - { - uint8_t cipher_algo; - enum rte_crypto_sym_xform_type type; - union - { - enum rte_crypto_auth_algorithm auth; - enum rte_crypto_cipher_algorithm cipher; - }; - char *name; - } supported_algo[] = - { - { - .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher = - RTE_CRYPTO_CIPHER_NULL,.name = "NULL"}, - { - .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher = - RTE_CRYPTO_CIPHER_AES_CBC,.name = "AES_CBC"}, - { - .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher = - RTE_CRYPTO_CIPHER_AES_CTR,.name = "AES_CTR"}, - { - .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.cipher = - RTE_CRYPTO_CIPHER_3DES_CBC,.name = "3DES-CBC"}, - { - .type = RTE_CRYPTO_SYM_XFORM_CIPHER,.auth = - RTE_CRYPTO_CIPHER_AES_GCM,.name = "AES-GCM"}, - { - .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth = - RTE_CRYPTO_AUTH_SHA1_HMAC,.name = "HMAC-SHA1"}, - { - .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth = - RTE_CRYPTO_AUTH_SHA256_HMAC,.name = "HMAC-SHA256"}, - { - .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth = - RTE_CRYPTO_AUTH_SHA384_HMAC,.name = "HMAC-SHA384"}, - { - .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth = - RTE_CRYPTO_AUTH_SHA512_HMAC,.name = "HMAC-SHA512"}, - { - .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth = - RTE_CRYPTO_AUTH_AES_XCBC_MAC,.name = "AES-XCBC-MAC"}, - { - .type = RTE_CRYPTO_SYM_XFORM_AUTH,.auth = - RTE_CRYPTO_AUTH_AES_GCM,.name = "AES-GCM"}, - { - /* tail */ - .type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED},}; - uint32_t i = 0; - - if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) - return -1; - - while (supported_algo[i].type != RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED) - { - if (cap->sym.xform_type == supported_algo[i].type) - { - if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER && - cap->sym.cipher.algo == supported_algo[i].cipher) || - (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AUTH && - cap->sym.auth.algo == supported_algo[i].auth)) - { - if (name) - strcpy (name, supported_algo[i].name); - return 0; - } - } - - i++; - } - - return -1; -} - -#endif /* __DPDK_IPSEC_H__ */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/node.c b/vnet/vnet/devices/dpdk/node.c deleted file mode 100644 index e541cdbcbd2..00000000000 --- a/vnet/vnet/devices/dpdk/node.c +++ /dev/null @@ -1,687 +0,0 @@ -/* - * Copyright (c) 2015 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <vnet/vnet.h> -#include <vppinfra/vec.h> -#include <vppinfra/error.h> -#include <vppinfra/format.h> -#include <vppinfra/xxhash.h> - -#include <vnet/ethernet/ethernet.h> -#include <vnet/devices/dpdk/dpdk.h> -#include <vnet/classify/vnet_classify.h> -#include <vnet/mpls/packet.h> -#include <vnet/handoff.h> -#include <vnet/devices/devices.h> -#include <vnet/feature/feature.h> - -#include "dpdk_priv.h" - -static char *dpdk_error_strings[] = { -#define _(n,s) s, - foreach_dpdk_error -#undef _ -}; - -always_inline int -vlib_buffer_is_ip4 (vlib_buffer_t * b) -{ - ethernet_header_t *h = (ethernet_header_t *) b->data; - return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP4)); -} - -always_inline int -vlib_buffer_is_ip6 (vlib_buffer_t * b) -{ - ethernet_header_t *h = (ethernet_header_t *) b->data; - return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)); -} - -always_inline int -vlib_buffer_is_mpls (vlib_buffer_t * b) -{ - ethernet_header_t *h = (ethernet_header_t *) b->data; - return (h->type == clib_host_to_net_u16 (ETHERNET_TYPE_MPLS_UNICAST)); -} - -#if RTE_VERSION < RTE_VERSION_NUM(16, 11, 0, 0) -/* New ol_flags bits added in DPDK-16.11 */ -#define PKT_RX_IP_CKSUM_GOOD (1ULL << 7) -#endif - -always_inline u32 -dpdk_rx_next_from_etype (struct rte_mbuf * mb, vlib_buffer_t * b0) -{ - if (PREDICT_TRUE (vlib_buffer_is_ip4 (b0))) - if (PREDICT_TRUE ((mb->ol_flags & PKT_RX_IP_CKSUM_GOOD) != 0)) - return VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT; - else - return VNET_DEVICE_INPUT_NEXT_IP4_INPUT; - else if (PREDICT_TRUE (vlib_buffer_is_ip6 (b0))) - return VNET_DEVICE_INPUT_NEXT_IP6_INPUT; - else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0))) - return VNET_DEVICE_INPUT_NEXT_MPLS_INPUT; - else - return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; -} - -always_inline int -dpdk_mbuf_is_vlan (struct rte_mbuf *mb) -{ -#if RTE_VERSION >= RTE_VERSION_NUM(16, 11, 0, 0) - return (mb->packet_type & RTE_PTYPE_L2_ETHER_VLAN) == - RTE_PTYPE_L2_ETHER_VLAN; -#else - return - (mb->ol_flags & - (PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED | PKT_RX_QINQ_STRIPPED)) == - PKT_RX_VLAN_PKT; -#endif -} - -always_inline int -dpdk_mbuf_is_ip4 (struct rte_mbuf *mb) -{ - return RTE_ETH_IS_IPV4_HDR (mb->packet_type) != 0; -} - -always_inline int -dpdk_mbuf_is_ip6 (struct rte_mbuf *mb) -{ - return RTE_ETH_IS_IPV6_HDR (mb->packet_type) != 0; -} - -always_inline u32 -dpdk_rx_next_from_mb (struct rte_mbuf * mb, vlib_buffer_t * b0) -{ - if (PREDICT_FALSE (dpdk_mbuf_is_vlan (mb))) - return VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; - else if (PREDICT_TRUE (dpdk_mbuf_is_ip4 (mb))) - return VNET_DEVICE_INPUT_NEXT_IP4_NCS_INPUT; - else if (PREDICT_TRUE (dpdk_mbuf_is_ip6 (mb))) - return VNET_DEVICE_INPUT_NEXT_IP6_INPUT; - else if (PREDICT_TRUE (vlib_buffer_is_mpls (b0))) - return VNET_DEVICE_INPUT_NEXT_MPLS_INPUT; - else - return dpdk_rx_next_from_etype (mb, b0); -} - -always_inline void -dpdk_rx_error_from_mb (struct rte_mbuf *mb, u32 * next, u8 * error) -{ - if (mb->ol_flags & PKT_RX_IP_CKSUM_BAD) - { - *error = DPDK_ERROR_IP_CHECKSUM_ERROR; - *next = VNET_DEVICE_INPUT_NEXT_DROP; - } - else - *error = DPDK_ERROR_NONE; -} - -void -dpdk_rx_trace (dpdk_main_t * dm, - vlib_node_runtime_t * node, - dpdk_device_t * xd, - u16 queue_id, u32 * buffers, uword n_buffers) -{ - vlib_main_t *vm = vlib_get_main (); - u32 *b, n_left; - u32 next0; - - n_left = n_buffers; - b = buffers; - - while (n_left >= 1) - { - u32 bi0; - vlib_buffer_t *b0; - dpdk_rx_dma_trace_t *t0; - struct rte_mbuf *mb; - u8 error0; - - bi0 = b[0]; - n_left -= 1; - - b0 = vlib_get_buffer (vm, bi0); - mb = rte_mbuf_from_vlib_buffer (b0); - - if (PREDICT_FALSE (xd->per_interface_next_index != ~0)) - next0 = xd->per_interface_next_index; - else if (PREDICT_TRUE - ((xd->flags & DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE) != 0)) - next0 = dpdk_rx_next_from_mb (mb, b0); - else - next0 = dpdk_rx_next_from_etype (mb, b0); - - dpdk_rx_error_from_mb (mb, &next0, &error0); - - vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0); - t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0])); - t0->queue_index = queue_id; - t0->device_index = xd->device_index; - t0->buffer_index = bi0; - - clib_memcpy (&t0->mb, mb, sizeof (t0->mb)); - clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data)); - clib_memcpy (t0->buffer.pre_data, b0->data, - sizeof (t0->buffer.pre_data)); - clib_memcpy (&t0->data, mb->buf_addr + mb->data_off, sizeof (t0->data)); - - b += 1; - } -} - -static inline u32 -dpdk_rx_burst (dpdk_main_t * dm, dpdk_device_t * xd, u16 queue_id) -{ - u32 n_buffers; - u32 n_left; - u32 n_this_chunk; - - n_left = VLIB_FRAME_SIZE; - n_buffers = 0; - - if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD)) - { - while (n_left) - { - n_this_chunk = rte_eth_rx_burst (xd->device_index, queue_id, - xd->rx_vectors[queue_id] + - n_buffers, n_left); - n_buffers += n_this_chunk; - n_left -= n_this_chunk; - - /* Empirically, DPDK r1.8 produces vectors w/ 32 or fewer elts */ - if (n_this_chunk < 32) - break; - } - } - else - { - ASSERT (0); - } - - return n_buffers; -} - - -static_always_inline void -dpdk_process_subseq_segs (vlib_main_t * vm, vlib_buffer_t * b, - struct rte_mbuf *mb, vlib_buffer_free_list_t * fl) -{ - u8 nb_seg = 1; - struct rte_mbuf *mb_seg = 0; - vlib_buffer_t *b_seg, *b_chain = 0; - mb_seg = mb->next; - b_chain = b; - - while ((mb->nb_segs > 1) && (nb_seg < mb->nb_segs)) - { - ASSERT (mb_seg != 0); - - b_seg = vlib_buffer_from_rte_mbuf (mb_seg); - vlib_buffer_init_for_free_list (b_seg, fl); - - ASSERT ((b_seg->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); - ASSERT (b_seg->current_data == 0); - - /* - * The driver (e.g. virtio) may not put the packet data at the start - * of the segment, so don't assume b_seg->current_data == 0 is correct. - */ - b_seg->current_data = - (mb_seg->buf_addr + mb_seg->data_off) - (void *) b_seg->data; - - b_seg->current_length = mb_seg->data_len; - b->total_length_not_including_first_buffer += mb_seg->data_len; - - b_chain->flags |= VLIB_BUFFER_NEXT_PRESENT; - b_chain->next_buffer = vlib_get_buffer_index (vm, b_seg); - - b_chain = b_seg; - mb_seg = mb_seg->next; - nb_seg++; - } -} - -static_always_inline void -dpdk_prefetch_buffer (struct rte_mbuf *mb) -{ - vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb); - CLIB_PREFETCH (mb, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, STORE); -} - -/* - * This function is used when there are no worker threads. - * The main thread performs IO and forwards the packets. - */ -static_always_inline u32 -dpdk_device_input (dpdk_main_t * dm, dpdk_device_t * xd, - vlib_node_runtime_t * node, u32 cpu_index, u16 queue_id) -{ - u32 n_buffers; - u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; - u32 n_left_to_next, *to_next; - u32 mb_index; - vlib_main_t *vm = vlib_get_main (); - uword n_rx_bytes = 0; - u32 n_trace, trace_cnt __attribute__ ((unused)); - vlib_buffer_free_list_t *fl; - u32 buffer_flags_template; - - if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0) - return 0; - - n_buffers = dpdk_rx_burst (dm, xd, queue_id); - - if (n_buffers == 0) - { - return 0; - } - - buffer_flags_template = dm->buffer_flags_template; - - vec_reset_length (xd->d_trace_buffers[cpu_index]); - trace_cnt = n_trace = vlib_get_trace_count (vm, node); - - if (n_trace > 0) - { - u32 n = clib_min (n_trace, n_buffers); - mb_index = 0; - - while (n--) - { - struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index++]; - vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb); - vec_add1 (xd->d_trace_buffers[cpu_index], - vlib_get_buffer_index (vm, b)); - } - } - - fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); - - mb_index = 0; - - while (n_buffers > 0) - { - vlib_buffer_t *b0, *b1, *b2, *b3; - u32 bi0, next0, l3_offset0; - u32 bi1, next1, l3_offset1; - u32 bi2, next2, l3_offset2; - u32 bi3, next3, l3_offset3; - u8 error0, error1, error2, error3; - u64 or_ol_flags; - - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); - - while (n_buffers > 8 && n_left_to_next > 4) - { - struct rte_mbuf *mb0 = xd->rx_vectors[queue_id][mb_index]; - struct rte_mbuf *mb1 = xd->rx_vectors[queue_id][mb_index + 1]; - struct rte_mbuf *mb2 = xd->rx_vectors[queue_id][mb_index + 2]; - struct rte_mbuf *mb3 = xd->rx_vectors[queue_id][mb_index + 3]; - - dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 4]); - dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 5]); - dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 6]); - dpdk_prefetch_buffer (xd->rx_vectors[queue_id][mb_index + 7]); - - if (xd->flags & DPDK_DEVICE_FLAG_MAYBE_MULTISEG) - { - if (PREDICT_FALSE (mb0->nb_segs > 1)) - dpdk_prefetch_buffer (mb0->next); - if (PREDICT_FALSE (mb1->nb_segs > 1)) - dpdk_prefetch_buffer (mb1->next); - if (PREDICT_FALSE (mb2->nb_segs > 1)) - dpdk_prefetch_buffer (mb2->next); - if (PREDICT_FALSE (mb3->nb_segs > 1)) - dpdk_prefetch_buffer (mb3->next); - } - - ASSERT (mb0); - ASSERT (mb1); - ASSERT (mb2); - ASSERT (mb3); - - or_ol_flags = (mb0->ol_flags | mb1->ol_flags | - mb2->ol_flags | mb3->ol_flags); - b0 = vlib_buffer_from_rte_mbuf (mb0); - b1 = vlib_buffer_from_rte_mbuf (mb1); - b2 = vlib_buffer_from_rte_mbuf (mb2); - b3 = vlib_buffer_from_rte_mbuf (mb3); - - vlib_buffer_init_for_free_list (b0, fl); - vlib_buffer_init_for_free_list (b1, fl); - vlib_buffer_init_for_free_list (b2, fl); - vlib_buffer_init_for_free_list (b3, fl); - - bi0 = vlib_get_buffer_index (vm, b0); - bi1 = vlib_get_buffer_index (vm, b1); - bi2 = vlib_get_buffer_index (vm, b2); - bi3 = vlib_get_buffer_index (vm, b3); - - to_next[0] = bi0; - to_next[1] = bi1; - to_next[2] = bi2; - to_next[3] = bi3; - to_next += 4; - n_left_to_next -= 4; - - if (PREDICT_FALSE (xd->per_interface_next_index != ~0)) - { - next0 = next1 = next2 = next3 = xd->per_interface_next_index; - } - else if (PREDICT_TRUE - ((xd->flags & DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE) != 0)) - { - next0 = dpdk_rx_next_from_mb (mb0, b0); - next1 = dpdk_rx_next_from_mb (mb1, b1); - next2 = dpdk_rx_next_from_mb (mb2, b2); - next3 = dpdk_rx_next_from_mb (mb3, b3); - } - else - { - next0 = dpdk_rx_next_from_etype (mb0, b0); - next1 = dpdk_rx_next_from_etype (mb1, b1); - next2 = dpdk_rx_next_from_etype (mb2, b2); - next3 = dpdk_rx_next_from_etype (mb3, b3); - } - - if (PREDICT_FALSE (or_ol_flags & PKT_RX_IP_CKSUM_BAD)) - { - dpdk_rx_error_from_mb (mb0, &next0, &error0); - dpdk_rx_error_from_mb (mb1, &next1, &error1); - dpdk_rx_error_from_mb (mb2, &next2, &error2); - dpdk_rx_error_from_mb (mb3, &next3, &error3); - b0->error = node->errors[error0]; - b1->error = node->errors[error1]; - b2->error = node->errors[error2]; - b3->error = node->errors[error3]; - } - else - { - b0->error = b1->error = node->errors[DPDK_ERROR_NONE]; - b2->error = b3->error = node->errors[DPDK_ERROR_NONE]; - } - - l3_offset0 = device_input_next_node_advance[next0]; - l3_offset1 = device_input_next_node_advance[next1]; - l3_offset2 = device_input_next_node_advance[next2]; - l3_offset3 = device_input_next_node_advance[next3]; - - b0->current_data = l3_offset0 + mb0->data_off; - b1->current_data = l3_offset1 + mb1->data_off; - b2->current_data = l3_offset2 + mb2->data_off; - b3->current_data = l3_offset3 + mb3->data_off; - - b0->current_data -= RTE_PKTMBUF_HEADROOM; - b1->current_data -= RTE_PKTMBUF_HEADROOM; - b2->current_data -= RTE_PKTMBUF_HEADROOM; - b3->current_data -= RTE_PKTMBUF_HEADROOM; - - b0->current_length = mb0->data_len - l3_offset0; - b1->current_length = mb1->data_len - l3_offset1; - b2->current_length = mb2->data_len - l3_offset2; - b3->current_length = mb3->data_len - l3_offset3; - - b0->flags = buffer_flags_template; - b1->flags = buffer_flags_template; - b2->flags = buffer_flags_template; - b3->flags = buffer_flags_template; - - vnet_buffer (b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index; - vnet_buffer (b1)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index; - vnet_buffer (b2)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index; - vnet_buffer (b3)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index; - - vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; - vnet_buffer (b1)->sw_if_index[VLIB_TX] = (u32) ~ 0; - vnet_buffer (b2)->sw_if_index[VLIB_TX] = (u32) ~ 0; - vnet_buffer (b3)->sw_if_index[VLIB_TX] = (u32) ~ 0; - - n_rx_bytes += mb0->pkt_len; - n_rx_bytes += mb1->pkt_len; - n_rx_bytes += mb2->pkt_len; - n_rx_bytes += mb3->pkt_len; - - /* Process subsequent segments of multi-segment packets */ - if (xd->flags & DPDK_DEVICE_FLAG_MAYBE_MULTISEG) - { - dpdk_process_subseq_segs (vm, b0, mb0, fl); - dpdk_process_subseq_segs (vm, b1, mb1, fl); - dpdk_process_subseq_segs (vm, b2, mb2, fl); - dpdk_process_subseq_segs (vm, b3, mb3, fl); - } - - /* - * Turn this on if you run into - * "bad monkey" contexts, and you want to know exactly - * which nodes they've visited... See main.c... - */ - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3); - - /* Do we have any driver RX features configured on the interface? */ - vnet_feature_start_device_input_x4 (xd->vlib_sw_if_index, - &next0, &next1, &next2, &next3, - b0, b1, b2, b3, - l3_offset0, l3_offset1, - l3_offset2, l3_offset3); - - vlib_validate_buffer_enqueue_x4 (vm, node, next_index, - to_next, n_left_to_next, - bi0, bi1, bi2, bi3, - next0, next1, next2, next3); - n_buffers -= 4; - mb_index += 4; - } - while (n_buffers > 0 && n_left_to_next > 0) - { - struct rte_mbuf *mb0 = xd->rx_vectors[queue_id][mb_index]; - - ASSERT (mb0); - - b0 = vlib_buffer_from_rte_mbuf (mb0); - - /* Prefetch one next segment if it exists. */ - if (PREDICT_FALSE (mb0->nb_segs > 1)) - dpdk_prefetch_buffer (mb0->next); - - vlib_buffer_init_for_free_list (b0, fl); - - bi0 = vlib_get_buffer_index (vm, b0); - - to_next[0] = bi0; - to_next++; - n_left_to_next--; - - if (PREDICT_FALSE (xd->per_interface_next_index != ~0)) - next0 = xd->per_interface_next_index; - else if (PREDICT_TRUE - ((xd->flags & DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE) != 0)) - next0 = dpdk_rx_next_from_mb (mb0, b0); - else - next0 = dpdk_rx_next_from_etype (mb0, b0); - - dpdk_rx_error_from_mb (mb0, &next0, &error0); - b0->error = node->errors[error0]; - - l3_offset0 = device_input_next_node_advance[next0]; - - b0->current_data = l3_offset0; - b0->current_data += mb0->data_off - RTE_PKTMBUF_HEADROOM; - b0->current_length = mb0->data_len - l3_offset0; - - b0->flags = buffer_flags_template; - - vnet_buffer (b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index; - vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; - n_rx_bytes += mb0->pkt_len; - - /* Process subsequent segments of multi-segment packets */ - dpdk_process_subseq_segs (vm, b0, mb0, fl); - - /* - * Turn this on if you run into - * "bad monkey" contexts, and you want to know exactly - * which nodes they've visited... See main.c... - */ - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); - - /* Do we have any driver RX features configured on the interface? */ - vnet_feature_start_device_input_x1 (xd->vlib_sw_if_index, &next0, - b0, l3_offset0); - - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, - to_next, n_left_to_next, - bi0, next0); - n_buffers--; - mb_index++; - } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); - } - - if (PREDICT_FALSE (vec_len (xd->d_trace_buffers[cpu_index]) > 0)) - { - dpdk_rx_trace (dm, node, xd, queue_id, xd->d_trace_buffers[cpu_index], - vec_len (xd->d_trace_buffers[cpu_index])); - vlib_set_trace_count (vm, node, n_trace - - vec_len (xd->d_trace_buffers[cpu_index])); - } - - vlib_increment_combined_counter - (vnet_get_main ()->interface_main.combined_sw_if_counters - + VNET_INTERFACE_COUNTER_RX, - cpu_index, xd->vlib_sw_if_index, mb_index, n_rx_bytes); - - dpdk_worker_t *dw = vec_elt_at_index (dm->workers, cpu_index); - dw->aggregate_rx_packets += mb_index; - - return mb_index; -} - -static inline void -poll_rate_limit (dpdk_main_t * dm) -{ - /* Limit the poll rate by sleeping for N msec between polls */ - if (PREDICT_FALSE (dm->poll_sleep != 0)) - { - struct timespec ts, tsrem; - - ts.tv_sec = 0; - ts.tv_nsec = 1000 * 1000 * dm->poll_sleep; /* 1ms */ - - while (nanosleep (&ts, &tsrem) < 0) - { - ts = tsrem; - } - } -} - -/** \brief Main DPDK input node - @node dpdk-input - - This is the main DPDK input node: across each assigned interface, - call rte_eth_rx_burst(...) or similar to obtain a vector of - packets to process. Handle early packet discard. Derive @c - vlib_buffer_t metadata from <code>struct rte_mbuf</code> metadata, - Depending on the resulting metadata: adjust <code>b->current_data, - b->current_length </code> and dispatch directly to - ip4-input-no-checksum, or ip6-input. Trace the packet if required. - - @param vm vlib_main_t corresponding to the current thread - @param node vlib_node_runtime_t - @param f vlib_frame_t input-node, not used. - - @par Graph mechanics: buffer metadata, next index usage - - @em Uses: - - <code>struct rte_mbuf mb->ol_flags</code> - - PKT_RX_IP_CKSUM_BAD - - <code> RTE_ETH_IS_xxx_HDR(mb->packet_type) </code> - - packet classification result - - @em Sets: - - <code>b->error</code> if the packet is to be dropped immediately - - <code>b->current_data, b->current_length</code> - - adjusted as needed to skip the L2 header in direct-dispatch cases - - <code>vnet_buffer(b)->sw_if_index[VLIB_RX]</code> - - rx interface sw_if_index - - <code>vnet_buffer(b)->sw_if_index[VLIB_TX] = ~0</code> - - required by ipX-lookup - - <code>b->flags</code> - - to indicate multi-segment pkts (VLIB_BUFFER_NEXT_PRESENT), etc. - - <em>Next Nodes:</em> - - Static arcs to: error-drop, ethernet-input, - ip4-input-no-checksum, ip6-input, mpls-input - - per-interface redirection, controlled by - <code>xd->per_interface_next_index</code> -*/ - -static uword -dpdk_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * f) -{ - dpdk_main_t *dm = &dpdk_main; - dpdk_device_t *xd; - uword n_rx_packets = 0; - dpdk_device_and_queue_t *dq; - u32 cpu_index = os_get_cpu_number (); - - /* - * Poll all devices on this cpu for input/interrupts. - */ - /* *INDENT-OFF* */ - vec_foreach (dq, dm->devices_by_cpu[cpu_index]) - { - xd = vec_elt_at_index(dm->devices, dq->device); - n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id); - } - /* *INDENT-ON* */ - - poll_rate_limit (dm); - - return n_rx_packets; -} - -/* *INDENT-OFF* */ -VLIB_REGISTER_NODE (dpdk_input_node) = { - .function = dpdk_input, - .type = VLIB_NODE_TYPE_INPUT, - .name = "dpdk-input", - .sibling_of = "device-input", - - /* Will be enabled if/when hardware is detected. */ - .state = VLIB_NODE_STATE_DISABLED, - - .format_buffer = format_ethernet_header_with_length, - .format_trace = format_dpdk_rx_dma_trace, - - .n_errors = DPDK_N_ERROR, - .error_strings = dpdk_error_strings, -}; - -VLIB_NODE_FUNCTION_MULTIARCH (dpdk_input_node, dpdk_input); -/* *INDENT-ON* */ - -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/vnet/vnet/devices/dpdk/qos_doc.md b/vnet/vnet/devices/dpdk/qos_doc.md deleted file mode 100644 index 9bd0659d616..00000000000 --- a/vnet/vnet/devices/dpdk/qos_doc.md +++ /dev/null @@ -1,404 +0,0 @@ -# QoS Hierarchical Scheduler {#qos_doc} - -The Quality-of-Service (QoS) scheduler performs egress-traffic management by -prioritizing the transmission of the packets of different type services and -subcribers based on the Service Level Agreements (SLAs). The QoS scheduler can -be enabled on one or more NIC output interfaces depending upon the -requirement. - - -## Overview - -The QoS schdeuler supports a number of scheduling and shaping levels which -construct hierarchical-tree. The first level in the hierarchy is port (i.e. -the physical interface) that constitutes the root node of the tree. The -subsequent level is subport which represents the group of the -users/subscribers. The individual user/subscriber is represented by the pipe -at the next level. Each user can have different traffic type based on the -criteria of specific loss rate, jitter, and latency. These traffic types are -represented at the traffic-class level in the form of different traffic- -classes. The last level contains number of queues which are grouped together -to host the packets of the specific class type traffic. - -The QoS scheduler implementation requires flow classification, enqueue and -dequeue operations. The flow classification is mandatory stage for HQoS where -incoming packets are classified by mapping the packet fields information to -5-tuple (HQoS subport, pipe, traffic class, queue within traffic class, and -color) and storing that information in mbuf sched field. The enqueue operation -uses this information to determine the queue for storing the packet, and at -this stage, if the specific queue is full, QoS drops the packet. The dequeue -operation consists of scheduling the packet based on its length and available -credits, and handing over the scheduled packet to the output interface. - -For more information on QoS Scheduler, please refer DPDK Programmer's Guide- -http://dpdk.org/doc/guides/prog_guide/qos_framework.html - - -### QoS Schdeuler Parameters - -Following illustrates the default HQoS configuration for each 10GbE output -port: - -Single subport (subport 0): - - Subport rate set to 100% of port rate - - Each of the 4 traffic classes has rate set to 100% of port rate - -4K pipes per subport 0 (pipes 0 .. 4095) with identical configuration: - - Pipe rate set to 1/4K of port rate - - Each of the 4 traffic classes has rate set to 100% of pipe rate - - Within each traffic class, the byte-level WRR weights for the 4 queues are set to 1:1:1:1 - - -#### Port configuration - -``` -port { - rate 1250000000 /* Assuming 10GbE port */ - frame_overhead 24 /* Overhead fields per Ethernet frame: - * 7B (Preamble) + - * 1B (Start of Frame Delimiter (SFD)) + - * 4B (Frame Check Sequence (FCS)) + - * 12B (Inter Frame Gap (IFG)) - */ - mtu 1522 /* Assuming Ethernet/IPv4 pkt (FCS not included) */ - n_subports_per_port 1 /* Number of subports per output interface */ - n_pipes_per_subport 4096 /* Number of pipes (users/subscribers) */ - queue_sizes 64 64 64 64 /* Packet queue size for each traffic class. - * All queues within the same pipe traffic class - * have the same size. Queues from different - * pipes serving the same traffic class have - * the same size. */ -} -``` - - -#### Subport configuration - -``` -subport 0 { - tb_rate 1250000000 /* Subport level token bucket rate (bytes per second) */ - tb_size 1000000 /* Subport level token bucket size (bytes) */ - tc0_rate 1250000000 /* Subport level token bucket rate for traffic class 0 (bytes per second) */ - tc1_rate 1250000000 /* Subport level token bucket rate for traffic class 1 (bytes per second) */ - tc2_rate 1250000000 /* Subport level token bucket rate for traffic class 2 (bytes per second) */ - tc3_rate 1250000000 /* Subport level token bucket rate for traffic class 3 (bytes per second) */ - tc_period 10 /* Time interval for refilling the token bucket associated with traffic class (Milliseconds) */ - pipe 0 4095 profile 0 /* pipes (users/subscribers) configured with pipe profile 0 */ -} -``` - - -#### Pipe configuration - -``` -pipe_profile 0 { - tb_rate 305175 /* Pipe level token bucket rate (bytes per second) */ - tb_size 1000000 /* Pipe level token bucket size (bytes) */ - tc0_rate 305175 /* Pipe level token bucket rate for traffic class 0 (bytes per second) */ - tc1_rate 305175 /* Pipe level token bucket rate for traffic class 1 (bytes per second) */ - tc2_rate 305175 /* Pipe level token bucket rate for traffic class 2 (bytes per second) */ - tc3_rate 305175 /* Pipe level token bucket rate for traffic class 3 (bytes per second) */ - tc_period 40 /* Time interval for refilling the token bucket associated with traffic class at pipe level (Milliseconds) */ - tc3_oversubscription_weight 1 /* Weight traffic class 3 oversubscription */ - tc0_wrr_weights 1 1 1 1 /* Pipe queues WRR weights for traffic class 0 */ - tc1_wrr_weights 1 1 1 1 /* Pipe queues WRR weights for traffic class 1 */ - tc2_wrr_weights 1 1 1 1 /* Pipe queues WRR weights for traffic class 2 */ - tc3_wrr_weights 1 1 1 1 /* Pipe queues WRR weights for traffic class 3 */ -} -``` - - -#### Random Early Detection (RED) parameters per traffic class and color (Green / Yellow / Red) - -``` -red { - tc0_wred_min 48 40 32 /* Minimum threshold for traffic class 0 queue (min_th) in number of packets */ - tc0_wred_max 64 64 64 /* Maximum threshold for traffic class 0 queue (max_th) in number of packets */ - tc0_wred_inv_prob 10 10 10 /* Inverse of packet marking probability for traffic class 0 queue (maxp = 1 / maxp_inv) */ - tc0_wred_weight 9 9 9 /* Traffic Class 0 queue weight */ - tc1_wred_min 48 40 32 /* Minimum threshold for traffic class 1 queue (min_th) in number of packets */ - tc1_wred_max 64 64 64 /* Maximum threshold for traffic class 1 queue (max_th) in number of packets */ - tc1_wred_inv_prob 10 10 10 /* Inverse of packet marking probability for traffic class 1 queue (maxp = 1 / maxp_inv) */ - tc1_wred_weight 9 9 9 /* Traffic Class 1 queue weight */ - tc2_wred_min 48 40 32 /* Minimum threshold for traffic class 2 queue (min_th) in number of packets */ - tc2_wred_max 64 64 64 /* Maximum threshold for traffic class 2 queue (max_th) in number of packets */ - tc2_wred_inv_prob 10 10 10 /* Inverse of packet marking probability for traffic class 2 queue (maxp = 1 / maxp_inv) */ - tc2_wred_weight 9 9 9 /* Traffic Class 2 queue weight */ - tc3_wred_min 48 40 32 /* Minimum threshold for traffic class 3 queue (min_th) in number of packets */ - tc3_wred_max 64 64 64 /* Maximum threshold for traffic class 3 queue (max_th) in number of packets */ - tc3_wred_inv_prob 10 10 10 /* Inverse of packet marking probability for traffic class 3 queue (maxp = 1 / maxp_inv) */ - tc3_wred_weight 9 9 9 /* Traffic Class 3 queue weight */ -} -``` - - -### DPDK QoS Scheduler Integration in VPP - -The Hierarchical Quaity-of-Service (HQoS) scheduler object could be seen as -part of the logical NIC output interface. To enable HQoS on specific output -interface, vpp startup.conf file has to be configured accordingly. The output -interface that requires HQoS, should have "hqos" parameter specified in dpdk -section. Another optional parameter "hqos-thread" has been defined which can -be used to associate the output interface with specific hqos thread. In cpu -section of the config file, "corelist-hqos-threads" is introduced to assign -logical cpu cores to run the HQoS threads. A HQoS thread can run multiple HQoS -objects each associated with different output interfaces. All worker threads -instead of writing packets to NIC TX queue directly, write the packets to a -software queues. The hqos_threads read the software queues, and enqueue the -packets to HQoS objects, as well as dequeue packets from HQOS objects and -write them to NIC output interfaces. The worker threads need to be able to -send the packets to any output interface, therefore, each HQoS object -associated with NIC output interface should have software queues equal to -worker threads count. - -Following illustrates the sample startup configuration file with 4x worker -threads feeding 2x hqos threads that handle each QoS scheduler for 1x output -interface. - -``` -dpdk { - socket-mem 16384,16384 - - dev 0000:02:00.0 { - num-rx-queues 2 - hqos - } - dev 0000:06:00.0 { - num-rx-queues 2 - hqos - } - - num-mbufs 1000000 -} - -cpu { - main-core 0 - corelist-workers 1, 2, 3, 4 - corelist-hqos-threads 5, 6 -} -``` - - -### QoS scheduler CLI Commands - -Each QoS scheduler instance is initialised with default parameters required to -configure hqos port, subport, pipe and queues. Some of the parameters can be -re-configured in run-time through CLI commands. - - -#### Configuration - -Following commands can be used to configure QoS scheduler parameters. - -The command below can be used to set the subport level parameters such as -token bucket rate (bytes per seconds), token bucket size (bytes), traffic -class rates (bytes per seconds) and token update period (Milliseconds). - -``` -set dpdk interface hqos subport <if-name> subport <n> [rate <n>] - [bktsize <n>] [tc0 <n>] [tc1 <n>] [tc2 <n>] [tc3 <n>] [period <n>] -``` - -For setting the pipe profile, following command can be used. - -``` -set dpdk interface hqos pipe <if-name> subport <n> pipe <n> profile <n> -``` - -To assign QoS scheduler instance to the specific thread, following command can -be used. - -``` -set dpdk interface hqos placement <if-name> thread <n> -``` - -The command below is used to set the packet fields required for classifiying -the incoming packet. As a result of classification process, packet field -information will be mapped to 5 tuples (subport, pipe, traffic class, pipe, -color) and stored in packet mbuf. - -``` -set dpdk interface hqos pktfield <if-name> id <n> offset <n> mask <n> -``` - -The DSCP table entries used for idenfiying the traffic class and queue can be set using the command below; - -``` -set dpdk interface hqos tctbl <if-name> entry <n> tc <n> queue <n> -``` - - -#### Show Command - -The QoS Scheduler configuration can displayed using the command below. - -``` - vpp# show dpdk interface hqos TenGigabitEthernet2/0/0 - Thread: - Input SWQ size = 4096 packets - Enqueue burst size = 256 packets - Dequeue burst size = 220 packets - Packet field 0: slab position = 0, slab bitmask = 0x0000000000000000 - Packet field 1: slab position = 40, slab bitmask = 0x0000000fff000000 - Packet field 2: slab position = 8, slab bitmask = 0x00000000000000fc - Packet field 2 translation table: - [ 0 .. 15]: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 - [16 .. 31]: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 - [32 .. 47]: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 - [48 .. 63]: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 - Port: - Rate = 1250000000 bytes/second - MTU = 1514 bytes - Frame overhead = 24 bytes - Number of subports = 1 - Number of pipes per subport = 4096 - Packet queue size: TC0 = 64, TC1 = 64, TC2 = 64, TC3 = 64 packets - Number of pipe profiles = 1 - Pipe profile 0: - Rate = 305175 bytes/second - Token bucket size = 1000000 bytes - Traffic class rate: TC0 = 305175, TC1 = 305175, TC2 = 305175, TC3 = 305175 bytes/second - TC period = 40 milliseconds - TC0 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1 - TC1 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1 - TC2 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1 - TC3 WRR weights: Q0 = 1, Q1 = 1, Q2 = 1, Q3 = 1 -``` - -The QoS Scheduler placement over the logical cpu cores can be displayed using -below command. - -``` - vpp# show dpdk interface hqos placement - Thread 5 (vpp_hqos-threads_0 at lcore 5): - TenGigabitEthernet2/0/0 queue 0 - Thread 6 (vpp_hqos-threads_1 at lcore 6): - TenGigabitEthernet4/0/1 queue 0 -``` - - -### QoS Scheduler Binary APIs - -This section explans the available binary APIs for configuring QoS scheduler -parameters in run-time. - -The following API can be used to set the pipe profile of a pipe that belongs -to a given subport: - -``` -sw_interface_set_dpdk_hqos_pipe rx <intfc> | sw_if_index <id> - subport <subport-id> pipe <pipe-id> profile <profile-id> -``` - -The data structures used for set the pipe profile parameter are as follows; - -``` - /** \\brief DPDK interface HQoS pipe profile set request - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param sw_if_index - the interface - @param subport - subport ID - @param pipe - pipe ID within its subport - @param profile - pipe profile ID - */ - define sw_interface_set_dpdk_hqos_pipe { - u32 client_index; - u32 context; - u32 sw_if_index; - u32 subport; - u32 pipe; - u32 profile; - }; - - /** \\brief DPDK interface HQoS pipe profile set reply - @param context - sender context, to match reply w/ request - @param retval - request return code - */ - define sw_interface_set_dpdk_hqos_pipe_reply { - u32 context; - i32 retval; - }; -``` - -The following API can be used to set the subport level parameters, for -example- token bucket rate (bytes per seconds), token bucket size (bytes), -traffic class rate (bytes per seconds) and tokens update period. - -``` -sw_interface_set_dpdk_hqos_subport rx <intfc> | sw_if_index <id> - subport <subport-id> [rate <n>] [bktsize <n>] - [tc0 <n>] [tc1 <n>] [tc2 <n>] [tc3 <n>] [period <n>] -``` - -The data structures used for set the subport level parameter are as follows; - -``` - /** \\brief DPDK interface HQoS subport parameters set request - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param sw_if_index - the interface - @param subport - subport ID - @param tb_rate - subport token bucket rate (measured in bytes/second) - @param tb_size - subport token bucket size (measured in credits) - @param tc_rate - subport traffic class 0 .. 3 rates (measured in bytes/second) - @param tc_period - enforcement period for rates (measured in milliseconds) - */ - define sw_interface_set_dpdk_hqos_subport { - u32 client_index; - u32 context; - u32 sw_if_index; - u32 subport; - u32 tb_rate; - u32 tb_size; - u32 tc_rate[4]; - u32 tc_period; - }; - - /** \\brief DPDK interface HQoS subport parameters set reply - @param context - sender context, to match reply w/ request - @param retval - request return code - */ - define sw_interface_set_dpdk_hqos_subport_reply { - u32 context; - i32 retval; - }; -``` - -The following API can be used set the DSCP table entry. The DSCP table have -64 entries to map the packet DSCP field onto traffic class and hqos input -queue. - -``` -sw_interface_set_dpdk_hqos_tctbl rx <intfc> | sw_if_index <id> - entry <n> tc <n> queue <n> -``` - -The data structures used for setting DSCP table entries are given below. - -``` - /** \\brief DPDK interface HQoS tctbl entry set request - @param client_index - opaque cookie to identify the sender - @param context - sender context, to match reply w/ request - @param sw_if_index - the interface - @param entry - entry index ID - @param tc - traffic class (0 .. 3) - @param queue - traffic class queue (0 .. 3) - */ - define sw_interface_set_dpdk_hqos_tctbl { - u32 client_index; - u32 context; - u32 sw_if_index; - u32 entry; - u32 tc; - u32 queue; - }; - - /** \\brief DPDK interface HQoS tctbl entry set reply - @param context - sender context, to match reply w/ request - @param retval - request return code - */ - define sw_interface_set_dpdk_hqos_tctbl_reply { - u32 context; - i32 retval; - }; -``` |