From 3cba9eaf8dce261928939aa112fae4354b51e229 Mon Sep 17 00:00:00 2001 From: Juraj Linkeš Date: Tue, 12 Oct 2021 10:03:04 +0200 Subject: vpp_device: bind to vfio-pci before running tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In rare cases, binding the whole /dev/vfio folder will result in unusable VFs: notice dpdk EAL: Cannot open /dev/vfio/151: Device or resource busy [0], section 4.3.1. provides some clues as to what's going on and how to avoid the failure. Mounting /dev/vfio reset the file descriptors of all devices under /dev/vfio. Vfio-pci creates a device when an interface is bound to it. The rare failure then occurs when /dev/vfio is mounted while a process is using the file descriptors result in that process using invalid file descriptors (or file descriptors belonging to a different VF). Fix the issue by binding i40e and ice VFs to vfio-pci before containers are created and make sure that the VFs are not unbound later in testing. Only bind DUT VFs since the TG uses the kernel driver. [0]: https://connect.redhat.com/sites/default/files/2021-03/Cloud Native Network Function Requirements.pdf Ticket: CSIT-1794 Change-Id: I83db91b29d16669fb034b141ad247f6f796fdf64 Signed-off-by: Juraj Linkeš --- resources/libraries/bash/function/device.sh | 116 +++++++++++++++++++--- resources/libraries/python/DUTSetup.py | 11 +- resources/libraries/robot/shared/interfaces.robot | 3 +- 3 files changed, 111 insertions(+), 19 deletions(-) (limited to 'resources/libraries') diff --git a/resources/libraries/bash/function/device.sh b/resources/libraries/bash/function/device.sh index 228a73b78c..1ad113ddba 100644 --- a/resources/libraries/bash/function/device.sh +++ b/resources/libraries/bash/function/device.sh @@ -30,6 +30,7 @@ function activate_wrapper () { enter_mutex || die get_available_interfaces "${1}" "${2}" || die + bind_dut_interfaces_to_vpp_driver || die start_topology_containers "${3}" || die bind_interfaces_to_containers || die set_env_variables || die @@ -38,6 +39,29 @@ function activate_wrapper () { } +function bind_dut_interfaces_to_vpp_driver () { + + # Bind DUT network interfaces to the driver that vpp will use + # + # Variables read: + # - DUT1_NETDEVS - List of network devices allocated to DUT1 container. + # Variables set: + # - NETDEV - Linux network interface. + # - DRIVER - Kernel driver to bind the interface to. + # - KRN_DRIVER - The original kernel driver of the network interface. + + for NETDEV in "${DUT1_NETDEVS[@]}"; do + get_pci_addr || die + get_krn_driver || die + if [[ ${KRN_DRIVER} == "iavf" ]]; then + DRIVER="vfio-pci" + ADDR=${PCI_ADDR} + bind_interfaces_to_driver || die + fi + done +} + + function bind_interfaces_to_containers () { # Bind linux network interface to container and create symlink for PCI @@ -51,36 +75,42 @@ function bind_interfaces_to_containers () { # - TG_NETDEVS - List of network devices allocated to TG container. # Variables set: # - NETDEV - Linux network interface. + # - KRN_DRIVER - Kernel driver of network device. set -exuo pipefail - for NETDEV in "${TG_NETDEVS[@]}"; do - get_pci_addr || die + for PCI_ADDR in "${TG_PCIDEVS[@]}"; do + get_netdev_name || die link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || { die "Reading symlink for PCI address failed!" } cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}" - sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || { - die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!" - } docker exec "${DCR_UUIDS[tg]}" ${cmd} || { die "Linking PCI address in container failed!" } + + sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || { + die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!" + } done - for NETDEV in "${DUT1_NETDEVS[@]}"; do - get_pci_addr || die + for PCI_ADDR in "${DUT1_PCIDEVS[@]}"; do link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || { die "Reading symlink for PCI address failed!" } cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}" - sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || { - die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!" - } docker exec "${DCR_UUIDS[dut1]}" ${cmd} || { die "Linking PCI address in container failed!" } + + get_krn_driver + if [[ ${KRN_DRIVER} != "vfio-pci" ]]; then + get_netdev_name || die + sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || { + die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!" + } + fi done } @@ -99,13 +129,22 @@ function bind_interfaces_to_driver () { pci_path="/sys/bus/pci/devices/${ADDR}" drv_path="/sys/bus/pci/drivers/${DRIVER}" if [ -d "${pci_path}/driver" ]; then - echo ${ADDR} | sudo tee ${pci_path}/driver/unbind || { + echo ${ADDR} | sudo tee ${pci_path}/driver/unbind > /dev/null || { die "Failed to unbind interface ${ADDR}!" } fi - echo ${ADDR} | sudo tee ${drv_path}/bind || { + + echo ${DRIVER} | sudo tee /sys/bus/pci/devices/${ADDR}/driver_override \ + > /dev/null || { + die "Failed to override driver to ${DRIVER} for ${ADDR}!" + } + + echo ${ADDR} | sudo tee ${drv_path}/bind > /dev/null || { die "Failed to bind interface ${ADDR}!" } + + echo | sudo tee /sys/bus/pci/devices/${ADDR}/driver_override > /dev/null \ + || die "Failed to reset driver override for ${ADDR}!" } @@ -415,6 +454,25 @@ function get_mac_addr () { } +function get_netdev_name () { + + # Get Linux network device name. + # + # Variables read: + # - PCI_ADDR - PCI address of the device. + # Variables set: + # - NETDEV - Linux network device name. + + set -exuo pipefail + + if [ -d /sys/bus/pci/devices/${PCI_ADDR}/net ]; then + NETDEV="$(basename /sys/bus/pci/devices/${PCI_ADDR}/net/*)" || { + die "Failed to get Linux interface name of ${PCI_ADDR}" + } + fi +} + + function get_csit_model () { # Get CSIT model name from linux network device name. @@ -467,6 +525,24 @@ function get_pci_addr () { } +function get_vfio_group () { + + # Get the VFIO group of a pci device. + # + # Variables read: + # - PCI_ADDR - PCI address of a device. + # Variables set: + # - VFIO_GROUP - The VFIO group of the PCI device. + + if [[ -d /sys/bus/pci/devices/${PCI_ADDR}/iommu_group ]]; then + VFIO_GROUP="$(basename\ + $(readlink /sys/bus/pci/devices/${PCI_ADDR}/iommu_group)\ + )" || { + die "PCI device ${PCI_ADDR} does not have an iommu group!" + } + fi +} + function get_vlan_filter () { # Get VLAN stripping filter from PF searched by mac adress. @@ -683,9 +759,19 @@ function start_topology_containers () { # Override access to PCI bus by attaching a filesystem mount to the # container. dcr_stc_params+="--mount type=tmpfs,destination=/sys/bus/pci/devices " - # Mount vfio to be able to bind to see bound interfaces. We cannot use - # --device=/dev/vfio as this does not see newly bound interfaces. - dcr_stc_params+="--volume /dev/vfio:/dev/vfio " + # Mount vfio devices to be able to use VFs inside the container. + vfio_bound="false" + for PCI_ADDR in ${DUT1_PCIDEVS[@]}; do + get_krn_driver + if [[ ${KRN_DRIVER} == "vfio-pci" ]]; then + get_vfio_group + dcr_stc_params+="--device /dev/vfio/${VFIO_GROUP} " + vfio_bound="true" + fi + done + if ! ${vfio_bound}; then + dcr_stc_params+="--volume /dev/vfio:/dev/vfio " + fi # Disable manipulation with hugepages by VPP. dcr_stc_params+="--volume /dev/null:/etc/sysctl.d/80-vpp.conf " # Mount docker.sock to be able to use docker deamon of the host. diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py index 16acfba7ad..c7a560262c 100644 --- a/resources/libraries/python/DUTSetup.py +++ b/resources/libraries/python/DUTSetup.py @@ -444,16 +444,21 @@ class DUTSetup: ) @staticmethod - def pci_driver_unbind_list(node, *pci_addrs): - """Unbind PCI devices from current driver on node. + def unbind_pci_devices_from_other_driver(node, driver, *pci_addrs): + """Unbind PCI devices from driver other than input driver on node. :param node: DUT node. + :param driver: Driver to not unbind from. If None or empty string, + will attempt to unbind from the current driver. :param pci_addrs: PCI device addresses. :type node: dict + :type driver: str :type pci_addrs: list """ for pci_addr in pci_addrs: - DUTSetup.pci_driver_unbind(node, pci_addr) + if not driver or \ + DUTSetup.get_pci_dev_driver(node, pci_addr) != driver: + DUTSetup.pci_driver_unbind(node, pci_addr) @staticmethod def pci_driver_bind(node, pci_addr, driver): diff --git a/resources/libraries/robot/shared/interfaces.robot b/resources/libraries/robot/shared/interfaces.robot index 0c669247d6..b33baaf214 100644 --- a/resources/libraries/robot/shared/interfaces.robot +++ b/resources/libraries/robot/shared/interfaces.robot @@ -137,7 +137,8 @@ | | Run Keyword If | ${index} >= 0 | Return From Keyword | | FOR | ${dut} | IN | @{duts} | | | Stop VPP Service | ${nodes['${dut}']} -| | | PCI Driver Unbind List | ${nodes['${dut}']} | @{${dut}_pf_pci} +| | | Unbind PCI Devices From Other Driver | ${nodes['${dut}']} | vfio-pci | +| | | ... | @{${dut}_pf_pci} | | | Run keyword | ${dut}.Add DPDK Dev | @{${dut}_pf_pci} | | | Run Keyword If | ${dpdk_no_tx_checksum_offload} | | | ... | ${dut}.Add DPDK No Tx Checksum Offload -- cgit 1.2.3-korg