aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJuraj Linkeš <juraj.linkes@pantheon.tech>2021-10-12 10:03:04 +0200
committerJuraj Linkeš <juraj.linkes@pantheon.tech>2021-11-08 11:46:27 +0100
commitfe2c403ce6dac0483a4621ebf5472d0716ac8a55 (patch)
treee6b600a78a6442556f52bccc4cba62319d955da6
parent3858b578e9e8d34ffd9d7767818d277364caa355 (diff)
vpp_device: bind to vfio-pci before running tests
In rare cases, binding the whole /dev/vfio folder will result in unusable VFs: notice dpdk EAL: Cannot open /dev/vfio/151: Device or resource busy [0], section 4.3.1. provides some clues as to what's going on and how to avoid the failure. Mounting /dev/vfio reset the file descriptors of all devices under /dev/vfio. Vfio-pci creates a device when an interface is bound to it. The rare failure then occurs when /dev/vfio is mounted while a process is using the file descriptors result in that process using invalid file descriptors (or file descriptors belonging to a different VF). Fix the issue by binding i40e and ice VFs to vfio-pci before containers are created and make sure that the VFs are not unbound later in testing. Only bind DUT VFs since the TG uses the kernel driver. [0]: https://connect.redhat.com/sites/default/files/2021-03/Cloud Native Network Function Requirements.pdf Ticket: CSIT-1794 Change-Id: I83db91b29d16669fb034b141ad247f6f796fdf64 Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
-rw-r--r--resources/libraries/bash/function/device.sh116
-rw-r--r--resources/libraries/python/DUTSetup.py11
-rw-r--r--resources/libraries/robot/shared/interfaces.robot3
3 files changed, 111 insertions, 19 deletions
diff --git a/resources/libraries/bash/function/device.sh b/resources/libraries/bash/function/device.sh
index 228a73b78c..1ad113ddba 100644
--- a/resources/libraries/bash/function/device.sh
+++ b/resources/libraries/bash/function/device.sh
@@ -30,6 +30,7 @@ function activate_wrapper () {
enter_mutex || die
get_available_interfaces "${1}" "${2}" || die
+ bind_dut_interfaces_to_vpp_driver || die
start_topology_containers "${3}" || die
bind_interfaces_to_containers || die
set_env_variables || die
@@ -38,6 +39,29 @@ function activate_wrapper () {
}
+function bind_dut_interfaces_to_vpp_driver () {
+
+ # Bind DUT network interfaces to the driver that vpp will use
+ #
+ # Variables read:
+ # - DUT1_NETDEVS - List of network devices allocated to DUT1 container.
+ # Variables set:
+ # - NETDEV - Linux network interface.
+ # - DRIVER - Kernel driver to bind the interface to.
+ # - KRN_DRIVER - The original kernel driver of the network interface.
+
+ for NETDEV in "${DUT1_NETDEVS[@]}"; do
+ get_pci_addr || die
+ get_krn_driver || die
+ if [[ ${KRN_DRIVER} == "iavf" ]]; then
+ DRIVER="vfio-pci"
+ ADDR=${PCI_ADDR}
+ bind_interfaces_to_driver || die
+ fi
+ done
+}
+
+
function bind_interfaces_to_containers () {
# Bind linux network interface to container and create symlink for PCI
@@ -51,36 +75,42 @@ function bind_interfaces_to_containers () {
# - TG_NETDEVS - List of network devices allocated to TG container.
# Variables set:
# - NETDEV - Linux network interface.
+ # - KRN_DRIVER - Kernel driver of network device.
set -exuo pipefail
- for NETDEV in "${TG_NETDEVS[@]}"; do
- get_pci_addr || die
+ for PCI_ADDR in "${TG_PCIDEVS[@]}"; do
+ get_netdev_name || die
link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
die "Reading symlink for PCI address failed!"
}
cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
- sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || {
- die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!"
- }
docker exec "${DCR_UUIDS[tg]}" ${cmd} || {
die "Linking PCI address in container failed!"
}
+
+ sudo ip link set ${NETDEV} netns ${DCR_CPIDS[tg]} || {
+ die "Moving interface to ${DCR_CPIDS[tg]} namespace failed!"
+ }
done
- for NETDEV in "${DUT1_NETDEVS[@]}"; do
- get_pci_addr || die
+ for PCI_ADDR in "${DUT1_PCIDEVS[@]}"; do
link_target=$(readlink -f /sys/bus/pci/devices/"${PCI_ADDR}") || {
die "Reading symlink for PCI address failed!"
}
cmd="ln -s ${link_target} /sys/bus/pci/devices/${PCI_ADDR}"
- sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || {
- die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!"
- }
docker exec "${DCR_UUIDS[dut1]}" ${cmd} || {
die "Linking PCI address in container failed!"
}
+
+ get_krn_driver
+ if [[ ${KRN_DRIVER} != "vfio-pci" ]]; then
+ get_netdev_name || die
+ sudo ip link set ${NETDEV} netns ${DCR_CPIDS[dut1]} || {
+ die "Moving interface to ${DCR_CPIDS[dut1]} namespace failed!"
+ }
+ fi
done
}
@@ -99,13 +129,22 @@ function bind_interfaces_to_driver () {
pci_path="/sys/bus/pci/devices/${ADDR}"
drv_path="/sys/bus/pci/drivers/${DRIVER}"
if [ -d "${pci_path}/driver" ]; then
- echo ${ADDR} | sudo tee ${pci_path}/driver/unbind || {
+ echo ${ADDR} | sudo tee ${pci_path}/driver/unbind > /dev/null || {
die "Failed to unbind interface ${ADDR}!"
}
fi
- echo ${ADDR} | sudo tee ${drv_path}/bind || {
+
+ echo ${DRIVER} | sudo tee /sys/bus/pci/devices/${ADDR}/driver_override \
+ > /dev/null || {
+ die "Failed to override driver to ${DRIVER} for ${ADDR}!"
+ }
+
+ echo ${ADDR} | sudo tee ${drv_path}/bind > /dev/null || {
die "Failed to bind interface ${ADDR}!"
}
+
+ echo | sudo tee /sys/bus/pci/devices/${ADDR}/driver_override > /dev/null \
+ || die "Failed to reset driver override for ${ADDR}!"
}
@@ -415,6 +454,25 @@ function get_mac_addr () {
}
+function get_netdev_name () {
+
+ # Get Linux network device name.
+ #
+ # Variables read:
+ # - PCI_ADDR - PCI address of the device.
+ # Variables set:
+ # - NETDEV - Linux network device name.
+
+ set -exuo pipefail
+
+ if [ -d /sys/bus/pci/devices/${PCI_ADDR}/net ]; then
+ NETDEV="$(basename /sys/bus/pci/devices/${PCI_ADDR}/net/*)" || {
+ die "Failed to get Linux interface name of ${PCI_ADDR}"
+ }
+ fi
+}
+
+
function get_csit_model () {
# Get CSIT model name from linux network device name.
@@ -467,6 +525,24 @@ function get_pci_addr () {
}
+function get_vfio_group () {
+
+ # Get the VFIO group of a pci device.
+ #
+ # Variables read:
+ # - PCI_ADDR - PCI address of a device.
+ # Variables set:
+ # - VFIO_GROUP - The VFIO group of the PCI device.
+
+ if [[ -d /sys/bus/pci/devices/${PCI_ADDR}/iommu_group ]]; then
+ VFIO_GROUP="$(basename\
+ $(readlink /sys/bus/pci/devices/${PCI_ADDR}/iommu_group)\
+ )" || {
+ die "PCI device ${PCI_ADDR} does not have an iommu group!"
+ }
+ fi
+}
+
function get_vlan_filter () {
# Get VLAN stripping filter from PF searched by mac adress.
@@ -683,9 +759,19 @@ function start_topology_containers () {
# Override access to PCI bus by attaching a filesystem mount to the
# container.
dcr_stc_params+="--mount type=tmpfs,destination=/sys/bus/pci/devices "
- # Mount vfio to be able to bind to see bound interfaces. We cannot use
- # --device=/dev/vfio as this does not see newly bound interfaces.
- dcr_stc_params+="--volume /dev/vfio:/dev/vfio "
+ # Mount vfio devices to be able to use VFs inside the container.
+ vfio_bound="false"
+ for PCI_ADDR in ${DUT1_PCIDEVS[@]}; do
+ get_krn_driver
+ if [[ ${KRN_DRIVER} == "vfio-pci" ]]; then
+ get_vfio_group
+ dcr_stc_params+="--device /dev/vfio/${VFIO_GROUP} "
+ vfio_bound="true"
+ fi
+ done
+ if ! ${vfio_bound}; then
+ dcr_stc_params+="--volume /dev/vfio:/dev/vfio "
+ fi
# Disable manipulation with hugepages by VPP.
dcr_stc_params+="--volume /dev/null:/etc/sysctl.d/80-vpp.conf "
# Mount docker.sock to be able to use docker deamon of the host.
diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index 9d0a3a8ff7..02cac03b1d 100644
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -444,16 +444,21 @@ class DUTSetup:
)
@staticmethod
- def pci_driver_unbind_list(node, *pci_addrs):
- """Unbind PCI devices from current driver on node.
+ def unbind_pci_devices_from_other_driver(node, driver, *pci_addrs):
+ """Unbind PCI devices from driver other than input driver on node.
:param node: DUT node.
+ :param driver: Driver to not unbind from. If None or empty string,
+ will attempt to unbind from the current driver.
:param pci_addrs: PCI device addresses.
:type node: dict
+ :type driver: str
:type pci_addrs: list
"""
for pci_addr in pci_addrs:
- DUTSetup.pci_driver_unbind(node, pci_addr)
+ if not driver or \
+ DUTSetup.get_pci_dev_driver(node, pci_addr) != driver:
+ DUTSetup.pci_driver_unbind(node, pci_addr)
@staticmethod
def pci_driver_bind(node, pci_addr, driver):
diff --git a/resources/libraries/robot/shared/interfaces.robot b/resources/libraries/robot/shared/interfaces.robot
index 0c669247d6..b33baaf214 100644
--- a/resources/libraries/robot/shared/interfaces.robot
+++ b/resources/libraries/robot/shared/interfaces.robot
@@ -137,7 +137,8 @@
| | Run Keyword If | ${index} >= 0 | Return From Keyword
| | FOR | ${dut} | IN | @{duts}
| | | Stop VPP Service | ${nodes['${dut}']}
-| | | PCI Driver Unbind List | ${nodes['${dut}']} | @{${dut}_pf_pci}
+| | | Unbind PCI Devices From Other Driver | ${nodes['${dut}']} | vfio-pci |
+| | | ... | @{${dut}_pf_pci}
| | | Run keyword | ${dut}.Add DPDK Dev | @{${dut}_pf_pci}
| | | Run Keyword If | ${dpdk_no_tx_checksum_offload}
| | | ... | ${dut}.Add DPDK No Tx Checksum Offload