From 6a07348f4a5312310c01fdd5af937ce26580eb2d Mon Sep 17 00:00:00 2001 From: Benoît Ganne Date: Thu, 13 Oct 2022 17:22:26 +0200 Subject: pci: add option to force uio binding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Type: improvement Change-Id: Ifea4badd58f7e2b5e792d7506f6747851a08587f Signed-off-by: Benoît Ganne --- docs/configuration/reference.rst | 12 ++++ src/plugins/dpdk/device/dpdk.h | 1 + src/plugins/dpdk/device/init.c | 5 +- src/plugins/vmxnet3/cli.c | 13 ++-- src/plugins/vmxnet3/vmxnet3.c | 3 +- src/plugins/vmxnet3/vmxnet3.h | 9 ++- src/vlib/linux/pci.c | 124 ++++++++++++++++++++------------------- src/vlib/pci/pci.h | 4 +- 8 files changed, 101 insertions(+), 70 deletions(-) diff --git a/docs/configuration/reference.rst b/docs/configuration/reference.rst index 84b2fd75887..84a951cb956 100644 --- a/docs/configuration/reference.rst +++ b/docs/configuration/reference.rst @@ -666,6 +666,18 @@ or auto (default) uio-driver vfio-pci +uio-bind-force +^^^^^^^^^^^^^^^^^^^^^^ + +Force VPP to rebind the interface(s) to the selected UIO driver, even if the +interface is up in Linux. +By default, VPP will refuse to bind an interface if it is up in Linux, +in case it is in active use. + +.. code-block:: console + + uio-bind-force + no-multi-seg ^^^^^^^^^^^^ diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h index 02cf6812d43..7569fc60a90 100644 --- a/src/plugins/dpdk/device/dpdk.h +++ b/src/plugins/dpdk/device/dpdk.h @@ -265,6 +265,7 @@ typedef struct u8 **eal_init_args; u8 *eal_init_args_str; u8 *uio_driver_name; + u8 uio_bind_force; u8 enable_telemetry; u16 max_simd_bitwidth; diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index 3f27a07f0ba..83ce2dc9cbc 100644 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -681,7 +681,8 @@ dpdk_bind_devices_to_uio (dpdk_config_main_t * conf) continue; } - error = vlib_pci_bind_to_uio (vm, addr, (char *) conf->uio_driver_name); + error = vlib_pci_bind_to_uio (vm, addr, (char *) conf->uio_driver_name, + conf->uio_bind_force); if (error) { @@ -1089,6 +1090,8 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) } else if (unformat (input, "uio-driver %s", &conf->uio_driver_name)) ; + else if (unformat (input, "uio-bind-force")) + conf->uio_bind_force = 1; else if (unformat (input, "socket-mem %s", &socket_mem)) ; else if (unformat (input, "no-pci")) diff --git a/src/plugins/vmxnet3/cli.c b/src/plugins/vmxnet3/cli.c index 039e9f3a75b..f751358262b 100644 --- a/src/plugins/vmxnet3/cli.c +++ b/src/plugins/vmxnet3/cli.c @@ -47,8 +47,10 @@ vmxnet3_create_command_fn (vlib_main_t * vm, unformat_input_t * input, args.enable_gso = 1; else if (unformat (line_input, "elog")) args.enable_elog = 1; + else if (unformat (line_input, "bind force")) + args.bind = VMXNET3_BIND_FORCE; else if (unformat (line_input, "bind")) - args.bind = 1; + args.bind = VMXNET3_BIND_DEFAULT; else if (unformat (line_input, "rx-queue-size %u", &size)) args.rxq_size = size; else if (unformat (line_input, "tx-queue-size %u", &size)) @@ -77,10 +79,11 @@ vmxnet3_create_command_fn (vlib_main_t * vm, unformat_input_t * input, /* *INDENT-OFF* */ VLIB_CLI_COMMAND (vmxnet3_create_command, static) = { .path = "create interface vmxnet3", - .short_help = "create interface vmxnet3 " - " [rx-queue-size ] [tx-queue-size ]" - " [num-tx-queues ] [num-rx-queues ] [bind]" - " [gso]", + .short_help = + "create interface vmxnet3 " + " [rx-queue-size ] [tx-queue-size ]" + " [num-tx-queues ] [num-rx-queues ] [bind [force]]" + " [gso]", .function = vmxnet3_create_command_fn, }; /* *INDENT-ON* */ diff --git a/src/plugins/vmxnet3/vmxnet3.c b/src/plugins/vmxnet3/vmxnet3.c index 770cb2d418d..be862d7127a 100644 --- a/src/plugins/vmxnet3/vmxnet3.c +++ b/src/plugins/vmxnet3/vmxnet3.c @@ -692,7 +692,8 @@ vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args) if (args->bind) { - error = vlib_pci_bind_to_uio (vm, &args->addr, (char *) "auto"); + error = vlib_pci_bind_to_uio (vm, &args->addr, (char *) "auto", + VMXNET3_BIND_FORCE == args->bind); if (error) { args->rv = VNET_API_ERROR_INVALID_INTERFACE; diff --git a/src/plugins/vmxnet3/vmxnet3.h b/src/plugins/vmxnet3/vmxnet3.h index 81aeec6f5c5..89602f8ee9e 100644 --- a/src/plugins/vmxnet3/vmxnet3.h +++ b/src/plugins/vmxnet3/vmxnet3.h @@ -606,6 +606,13 @@ typedef struct extern vmxnet3_main_t vmxnet3_main; +typedef enum +{ + VMXNET3_BIND_NONE = 0, + VMXNET3_BIND_DEFAULT = 1, + VMXNET3_BIND_FORCE = 2, +} __clib_packed vmxnet3_bind_t; + typedef struct { vlib_pci_addr_t addr; @@ -614,7 +621,7 @@ typedef struct u16 rxq_num; u16 txq_size; u16 txq_num; - u8 bind; + vmxnet3_bind_t bind; u8 enable_gso; /* return */ i32 rv; diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c index 9f0629f46bc..83e9dde2f16 100644 --- a/src/vlib/linux/pci.c +++ b/src/vlib/linux/pci.c @@ -453,8 +453,8 @@ directory_exists (char *path) } clib_error_t * -vlib_pci_bind_to_uio (vlib_main_t * vm, vlib_pci_addr_t * addr, - char *uio_drv_name) +vlib_pci_bind_to_uio (vlib_main_t *vm, vlib_pci_addr_t *addr, + char *uio_drv_name, int force) { clib_error_t *error = 0; u8 *s = 0, *driver_name = 0; @@ -523,76 +523,80 @@ vlib_pci_bind_to_uio (vlib_main_t * vm, vlib_pci_addr_t * addr, (strcmp ("igb_uio", (char *) driver_name) == 0))) goto done; - /* walk trough all linux interfaces and if interface belonging to - this device is founf check if interface is admin up */ - dir = opendir ("/sys/class/net"); - s = format (s, "%U%c", format_vlib_pci_addr, addr, 0); - - if (!dir) - { - error = clib_error_return (0, "Skipping PCI device %U: failed to " - "read /sys/class/net", - format_vlib_pci_addr, addr); - goto done; - } - - fd = socket (PF_INET, SOCK_DGRAM, 0); - if (fd < 0) - { - error = clib_error_return_unix (0, "socket"); - goto done; - } - - while ((e = readdir (dir))) + if (!force) { - struct ifreq ifr; - struct ethtool_drvinfo drvinfo; + /* walk trough all linux interfaces and if interface belonging to + this device is found check if interface is admin up */ + dir = opendir ("/sys/class/net"); + s = format (s, "%U%c", format_vlib_pci_addr, addr, 0); - if (e->d_name[0] == '.') /* skip . and .. */ - continue; - - clib_memset (&ifr, 0, sizeof ifr); - clib_memset (&drvinfo, 0, sizeof drvinfo); - ifr.ifr_data = (char *) &drvinfo; - clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1); - - drvinfo.cmd = ETHTOOL_GDRVINFO; - if (ioctl (fd, SIOCETHTOOL, &ifr) < 0) + if (!dir) { - /* Some interfaces (eg "lo") don't support this ioctl */ - if ((errno != ENOTSUP) && (errno != ENODEV)) - clib_unix_warning ("ioctl fetch intf %s bus info error", - e->d_name); - continue; + error = clib_error_return (0, + "Skipping PCI device %U: failed to " + "read /sys/class/net", + format_vlib_pci_addr, addr); + goto done; } - if (strcmp ((char *) s, drvinfo.bus_info)) - continue; - - clib_memset (&ifr, 0, sizeof (ifr)); - clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1); - - if (ioctl (fd, SIOCGIFFLAGS, &ifr) < 0) + fd = socket (PF_INET, SOCK_DGRAM, 0); + if (fd < 0) { - error = clib_error_return_unix (0, "ioctl fetch intf %s flags", - e->d_name); - close (fd); + error = clib_error_return_unix (0, "socket"); goto done; } - if (ifr.ifr_flags & IFF_UP) + while ((e = readdir (dir))) { - vlib_log (VLIB_LOG_LEVEL_WARNING, pci_main.log_default, - "Skipping PCI device %U as host " - "interface %s is up", format_vlib_pci_addr, addr, - e->d_name); - close (fd); - goto done; + struct ifreq ifr; + struct ethtool_drvinfo drvinfo; + + if (e->d_name[0] == '.') /* skip . and .. */ + continue; + + clib_memset (&ifr, 0, sizeof ifr); + clib_memset (&drvinfo, 0, sizeof drvinfo); + ifr.ifr_data = (char *) &drvinfo; + clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1); + + drvinfo.cmd = ETHTOOL_GDRVINFO; + if (ioctl (fd, SIOCETHTOOL, &ifr) < 0) + { + /* Some interfaces (eg "lo") don't support this ioctl */ + if ((errno != ENOTSUP) && (errno != ENODEV)) + clib_unix_warning ("ioctl fetch intf %s bus info error", + e->d_name); + continue; + } + + if (strcmp ((char *) s, drvinfo.bus_info)) + continue; + + clib_memset (&ifr, 0, sizeof (ifr)); + clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1); + + if (ioctl (fd, SIOCGIFFLAGS, &ifr) < 0) + { + error = clib_error_return_unix (0, "ioctl fetch intf %s flags", + e->d_name); + close (fd); + goto done; + } + + if (ifr.ifr_flags & IFF_UP) + { + vlib_log (VLIB_LOG_LEVEL_WARNING, pci_main.log_default, + "Skipping PCI device %U as host " + "interface %s is up", + format_vlib_pci_addr, addr, e->d_name); + close (fd); + goto done; + } } - } - close (fd); - vec_reset_length (s); + close (fd); + vec_reset_length (s); + } s = format (s, "%v/driver/unbind%c", dev_dir_name, 0); clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, addr); diff --git a/src/vlib/pci/pci.h b/src/vlib/pci/pci.h index 1dc4ce6ea36..4e9cf4aee9e 100644 --- a/src/vlib/pci/pci.h +++ b/src/vlib/pci/pci.h @@ -182,8 +182,8 @@ static void __vlib_rm_pci_device_registration_##x (void) \ } \ __VA_ARGS__ pci_device_registration_t x -clib_error_t *vlib_pci_bind_to_uio (vlib_main_t * vm, vlib_pci_addr_t * addr, - char *uio_driver_name); +clib_error_t *vlib_pci_bind_to_uio (vlib_main_t *vm, vlib_pci_addr_t *addr, + char *uio_driver_name, int force); /* Configuration space read/write. */ clib_error_t *vlib_pci_read_write_config (vlib_main_t * vm, -- cgit 1.2.3-korg