diff options
| -rw-r--r-- | resources/libraries/python/DUTSetup.py | 8 | ||||
| -rw-r--r-- | resources/libraries/python/QATUtil.py | 103 | ||||
| -rw-r--r-- | resources/libraries/python/VppConfigGenerator.py | 15 | ||||
| -rw-r--r-- | resources/libraries/robot/shared/default.robot | 23 | ||||
| -rw-r--r-- | resources/libraries/robot/shared/suite_setup.robot | 8 | ||||
| -rw-r--r-- | topologies/available/lf_2n_spr_testbed23.yaml | 28 | ||||
| -rw-r--r-- | topologies/available/lf_3n_snr_testbed39.yaml | 16 | ||||
| -rw-r--r-- | topologies/available/lf_3na_spr_testbed21.yaml | 42 | ||||
| -rw-r--r-- | topologies/available/lf_3nb_spr_testbed22.yaml | 42 |
9 files changed, 166 insertions, 119 deletions
diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py index a9b549a935..9fc525cab7 100644 --- a/resources/libraries/python/DUTSetup.py +++ b/resources/libraries/python/DUTSetup.py @@ -300,19 +300,21 @@ class DUTSetup: return sriov_numvfs @staticmethod - def set_sriov_numvfs(node, pf_pci_addr, numvfs=0): + def set_sriov_numvfs(node, pf_pci_addr, path="devices", numvfs=0): """Init or reset SR-IOV virtual functions by setting its number on PCI device on DUT. Setting to zero removes all VFs. :param node: DUT node. :param pf_pci_addr: Physical Function PCI device address. + :param type: Either device or driver. :param numvfs: Number of VFs to initialize, 0 - removes the VFs. :type node: dict :type pf_pci_addr: str + :type type: str :type numvfs: int :raises RuntimeError: Failed to create VFs on PCI. """ - cmd = f"test -f /sys/bus/pci/devices/{pf_pci_addr}/sriov_numvfs" + cmd = f"test -f /sys/bus/pci/{path}/{pf_pci_addr}/sriov_numvfs" sriov_unsupported, _, _ = exec_cmd(node, cmd) # if sriov_numvfs doesn't exist, then sriov_unsupported != 0 if int(sriov_unsupported): @@ -328,7 +330,7 @@ class DUTSetup: pci = pf_pci_addr.replace(u":", r"\:") command = f"sh -c \"echo {numvfs} | " \ - f"tee /sys/bus/pci/devices/{pci}/sriov_numvfs\"" + f"tee /sys/bus/pci/{path}/{pci}/sriov_numvfs\"" message = f"Failed to create {numvfs} VFs on {pf_pci_addr} device " \ f"on {node[u'host']}" diff --git a/resources/libraries/python/QATUtil.py b/resources/libraries/python/QATUtil.py index 34a8e3b8d9..be8cc9561e 100644 --- a/resources/libraries/python/QATUtil.py +++ b/resources/libraries/python/QATUtil.py @@ -14,7 +14,7 @@ """QAT util library.""" from resources.libraries.python.DUTSetup import DUTSetup -from resources.libraries.python.topology import Topology +from resources.libraries.python.topology import NodeType, Topology from resources.libraries.python.VPPUtil import VPPUtil @@ -22,88 +22,53 @@ class QATUtil: """Contains methods for setting up QATs.""" @staticmethod - def crypto_device_verify(node, crypto_type, numvfs, force_init=True): - """Verify if Crypto QAT device virtual functions are initialized on all - DUTs. If parameter force initialization is set to True, then try to - initialize or remove VFs on QAT. + def crypto_device_verify_on_all_duts(nodes): + """Verify if Crypto QAT device and its virtual functions are initialized + on all DUTs. - :param node: DUT node. - :crypto_type: Crypto device type - HW_DH895xcc, HW_C3xxx, HW_C4xxx - or HW_4xxx. - :param numvfs: Number of VFs to initialize, 0 - disable the VFs. - :param force_init: If True then try to initialize to specific value. - :type node: dict - :type crypto_type: string - :type numvfs: int - :type force_init: bool - :returns: nothing - :raises RuntimeError: If QAT VFs are not created and force init is set - to False. + :param nodes: Nodes in the topology. + :type nodes: dict """ - pci_addr = Topology.get_cryptodev(node) - - if force_init: - # QAT is not initialized and we want to initialize with numvfs. - QATUtil.crypto_device_init(node, crypto_type, numvfs) - else: - raise RuntimeError( - f"QAT device failed to create VFs on {node[u'host']}" - ) + VPPUtil.stop_vpp_service_on_all_duts(nodes) - # QAT VF devices must be re-bound to vfio-pci driver before use. - pci_addr = Topology.get_cryptodev(node) - for i in range(numvfs): - DUTSetup.pci_vf_driver_unbind(node, pci_addr, i) - DUTSetup.pci_vf_driver_bind(node, pci_addr, i, "vfio-pci") + for node in nodes.values(): + if node["type"] == NodeType.DUT: + cryptodevs = Topology.get_cryptodev(node) + if not cryptodevs: + return + for device in cryptodevs.values(): + QATUtil.crypto_device_init(node, device) @staticmethod - def crypto_device_init(node, crypto_type, numvfs): + def crypto_device_init(node, device): """Init Crypto QAT device virtual functions on DUT. :param node: DUT node. - :crypto_type: Crypto device type - HW_DH895xcc, HW_C3xxx, HW_C4xxx - or HW_4xxx. - :param numvfs: Number of VFs to initialize, 0 - disable the VFs. + :device: Crypto device entry from topology file. :type node: dict - :type crypto_type: string - :type numvfs: int - :returns: nothing - :raises RuntimeError: If failed to stop VPP or QAT failed to initialize. + :type device: dict """ - if crypto_type == u"HW_DH895xcc": - kernel_mod = u"qat_dh895xcc" - kernel_drv = u"dh895xcc" - elif crypto_type == u"HW_C3xxx": - kernel_mod = u"qat_c3xxx" - kernel_drv = u"c3xxx" - elif crypto_type == u"HW_C4xxx": - kernel_mod = u"qat_c4xxx" - kernel_drv = u"c4xxx" - elif crypto_type == u"HW_4xxx": - kernel_mod = u"qat_4xxx" - kernel_drv = u"4xxx" - else: - raise RuntimeError( - f"Unsupported crypto device type on {node[u'host']}" - ) - - pci_addr = Topology.get_cryptodev(node) - - # QAT device must be re-bound to kernel driver before initialization. - DUTSetup.verify_kernel_module(node, kernel_mod, force_load=True) - - # Stop VPP to prevent deadlock. - VPPUtil.stop_vpp_service(node) + DUTSetup.verify_kernel_module(node, device["module"], force_load=True) current_driver = DUTSetup.get_pci_dev_driver( - node, pci_addr.replace(u":", r"\:") + node, device["pci_address"].replace(":", r"\:") ) if current_driver is not None: - DUTSetup.pci_driver_unbind(node, pci_addr) - + DUTSetup.pci_driver_unbind(node, device["pci_address"]) # Bind to kernel driver. - DUTSetup.pci_driver_bind(node, pci_addr, kernel_drv) + DUTSetup.pci_driver_bind(node, device["pci_address"], device["driver"]) # Initialize QAT VFs. - if numvfs > 0: - DUTSetup.set_sriov_numvfs(node, pci_addr, numvfs)
\ No newline at end of file + if int(device["numvfs"]) > 0: + DUTSetup.set_sriov_numvfs( + node, device["pci_address"], type="drivers/4xxx", + numvfs=device["numvfs"] + ) + + for cvf in range(int(device["numvfs"])): + DUTSetup.pci_vf_driver_unbind( + node, device["pci_address"], cvf + ) + DUTSetup.pci_vf_driver_bind( + node, device["pci_address"], cvf, "vfio-pci" + ) diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py index a57d24cde5..0628e8ee47 100644 --- a/resources/libraries/python/VppConfigGenerator.py +++ b/resources/libraries/python/VppConfigGenerator.py @@ -259,12 +259,13 @@ class VppConfigGenerator: :param count: Number of HW crypto devices to add. :type count: int """ - cryptodev = Topology.get_cryptodev(self._node) - for i in range(count): - cryptodev_config = re.sub(r"\d.\d$", f"1.{str(i)}", cryptodev) - path = [u"dpdk", f"dev {cryptodev_config}"] - self.add_config_item(self._nodeconfig, u"", path) - self.add_dpdk_uio_driver(u"vfio-pci") + cryptodevs = Topology.get_cryptodev(self._node) + for device in cryptodevs.values(): + for i in range(int(count/len(cryptodevs))): + addr = re.sub(r"\d.\d$", f"0.{i+1}", device["pci_address"]) + path = ["dpdk", f"dev {addr}"] + self.add_config_item(self._nodeconfig, "", path) + self.add_dpdk_uio_driver("vfio-pci") def add_dpdk_sw_cryptodev(self, sw_pmd_type, socket_id, count): """Add DPDK SW Crypto device configuration. @@ -740,4 +741,4 @@ class VppInitConfig: ) vpp_config.add_ip6_hash_buckets(2000000) vpp_config.add_ip6_heap_size("4G") - vpp_config.apply_config()
\ No newline at end of file + vpp_config.apply_config() diff --git a/resources/libraries/robot/shared/default.robot b/resources/libraries/robot/shared/default.robot index be68958f5f..c17b7ac441 100644 --- a/resources/libraries/robot/shared/default.robot +++ b/resources/libraries/robot/shared/default.robot @@ -96,29 +96,6 @@ | | # Parens are there to perform the call. | | Run Keyword If | $resetter | Evaluate | $resetter() -| Configure crypto device on all DUTs -| | [Documentation] | Verify if Crypto QAT device virtual functions are -| | ... | initialized on all DUTs. If parameter force_init is set to True, then -| | ... | try to initialize/disable. -| | -| | ... | *Arguments:* -| | ... | - crypto_type - Crypto device type - HW_DH895xcc, HW_C3xxx, HW_C4xxx -| | ... | or HW_4xxx; default value: HW_DH895xcc. Type: string -| | ... | - numvfs - Number of VFs to initialize, 0 - disable the VFs; default -| | ... | value: ${32} Type: integer -| | ... | - force_init - Force to initialize. Type: boolean -| | -| | ... | *Example:* -| | -| | ... | \| Configure crypto device on all DUTs \| HW_DH895xcc \| ${32} \| -| | -| | [Arguments] | ${crypto_type} | ${numvfs} | ${force_init}=${True} -| | -| | FOR | ${dut} | IN | @{duts} -| | | Crypto Device Verify | ${nodes['${dut}']} | ${crypto_type} -| | | ... | ${numvfs} | force_init=${force_init} -| | END - | Configure kernel module on all DUTs | | [Documentation] | Verify if specific kernel module is loaded on all DUTs. | | ... | If parameter force_load is set to True, then try to load. diff --git a/resources/libraries/robot/shared/suite_setup.robot b/resources/libraries/robot/shared/suite_setup.robot index 9c8603af3b..438a39fb0e 100644 --- a/resources/libraries/robot/shared/suite_setup.robot +++ b/resources/libraries/robot/shared/suite_setup.robot @@ -275,13 +275,7 @@ | | [Documentation] | | ... | Additional Setup for suites which uses QAT HW. | | -| | ${numvfs}= | Set Variable If -| | ... | '${crypto_type}' == 'HW_DH895xcc' | ${32} -| | ... | '${crypto_type}' == 'HW_C3xxx' | ${16} -| | ... | '${crypto_type}' == 'HW_C4xxx' | ${128} -| | ... | '${crypto_type}' == 'HW_4xxx' | ${16} -| | Configure crypto device on all DUTs | ${crypto_type} | numvfs=${numvfs} -| | ... | force_init=${True} +| | Crypto Device Verify on all DUTs | ${nodes} | Additional Suite Setup Action For nginx | | [Documentation] diff --git a/topologies/available/lf_2n_spr_testbed23.yaml b/topologies/available/lf_2n_spr_testbed23.yaml index bf11a03147..7d76f16788 100644 --- a/topologies/available/lf_2n_spr_testbed23.yaml +++ b/topologies/available/lf_2n_spr_testbed23.yaml @@ -24,7 +24,19 @@ nodes: pci_address: "0000:f2:01.0" driver: idxd model: Intel-DSA - cryptodev: "0000:f3:00.0" + cryptodev: + device1: + pci_address: "0000:76:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 + device2: + pci_address: "0000:7a:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 interfaces: port1: # s57-t23-tg1-c2/p1 - 200GE port1 Mellanox ConnectX7-2p200GE. @@ -139,7 +151,19 @@ nodes: pci_address: "0000:f2:01.0" driver: idxd model: Intel-DSA - cryptodev: "0000:f3:00.0" + cryptodev: + device1: + pci_address: "0000:76:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 + device2: + pci_address: "0000:7a:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 interfaces: port1: # s56-t23-sut1-c2/p1 - 200GE port1 Mellanox ConnectX7-2p200GE. diff --git a/topologies/available/lf_3n_snr_testbed39.yaml b/topologies/available/lf_3n_snr_testbed39.yaml index 6e69d18445..d2f1e0df73 100644 --- a/topologies/available/lf_3n_snr_testbed39.yaml +++ b/topologies/available/lf_3n_snr_testbed39.yaml @@ -55,7 +55,13 @@ nodes: port: 22 username: testuser password: Csit1234 - cryptodev: "0000:f4:00.0" + cryptodev: + device1: + pci_address: "0000:f4:00.0" + driver: "c4xxx" + model: "HW_C4xxx" + module: "qat_c4xxx" + numvfs: 8 uio_driver: vfio-pci interfaces: port1: @@ -97,7 +103,13 @@ nodes: port: 22 username: testuser password: Csit1234 - cryptodev: "0000:f4:00.0" + cryptodev: + device1: + pci_address: "0000:f4:00.0" + driver: "c4xxx" + model: "HW_C4xxx" + module: "qat_c4xxx" + numvfs: 8 uio_driver: vfio-pci interfaces: port1: diff --git a/topologies/available/lf_3na_spr_testbed21.yaml b/topologies/available/lf_3na_spr_testbed21.yaml index 73dc9bad44..1167ab096f 100644 --- a/topologies/available/lf_3na_spr_testbed21.yaml +++ b/topologies/available/lf_3na_spr_testbed21.yaml @@ -24,7 +24,19 @@ nodes: pci_address: "0000:f2:01.0" driver: idxd model: Intel-DSA - cryptodev: "0000:76:00.0" + cryptodev: + device1: + pci_address: "0000:76:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 + device2: + pci_address: "0000:7a:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 interfaces: port1: # s53-t21-tg1-c4/p1 - 200GE port1 Mellanox ConnectX7-2p200GE. @@ -107,7 +119,19 @@ nodes: pci_address: "0000:f2:01.0" driver: idxd model: Intel-DSA - cryptodev: "0000:76:00.0" + cryptodev: + device1: + pci_address: "0000:76:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 + device2: + pci_address: "0000:7a:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 interfaces: port1: # s52-t21-sut1-c4/p1 - 200GE port1 Mellanox ConnectX7-2p200GE. @@ -174,7 +198,19 @@ nodes: pci_address: "0000:f2:01.0" driver: idxd model: Intel-DSA - cryptodev: "0000:f3:00.0" + cryptodev: + device1: + pci_address: "0000:f3:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 + device2: + pci_address: "0000:f7:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 interfaces: port1: # s52-t21-sut2-c9/p1 - 200GE port1 Mellanox ConnectX7-2p200GE. diff --git a/topologies/available/lf_3nb_spr_testbed22.yaml b/topologies/available/lf_3nb_spr_testbed22.yaml index f8e9708283..4b6f378e8c 100644 --- a/topologies/available/lf_3nb_spr_testbed22.yaml +++ b/topologies/available/lf_3nb_spr_testbed22.yaml @@ -24,7 +24,19 @@ nodes: pci_address: "0000:f2:01.0" driver: idxd model: Intel-DSA - cryptodev: "0000:76:00.0" + cryptodev: + device1: + pci_address: "0000:76:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 + device2: + pci_address: "0000:7a:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 interfaces: port1: # s55-t22-tg1-c4/p1 - 100GE port1 Intel E810-CQDA2 2p100GE. @@ -75,7 +87,19 @@ nodes: pci_address: "0000:f2:01.0" driver: idxd model: Intel-DSA - cryptodev: "0000:76:00.0" + cryptodev: + device1: + pci_address: "0000:76:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 + device2: + pci_address: "0000:7a:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 interfaces: port1: # s54-t22-sut1-c4/p1 - 100GE port1 Intel E810-CQDA2 2p100GE. @@ -126,7 +150,19 @@ nodes: pci_address: "0000:f2:01.0" driver: idxd model: Intel-DSA - cryptodev: "0000:f3:00.0" + cryptodev: + device1: + pci_address: "0000:f3:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 + device2: + pci_address: "0000:f7:00.0" + driver: "4xxx" + model: "HW_4xxx" + module: "qat_4xxx" + numvfs: 8 interfaces: port1: # s54-t22-sut1-c9/p1 - 100GE port1 Intel E810-CQDA2 2p100GE. |