feat(core): QAT initialization refactor

Signed-off-by: pmikus <peter.mikus@protonmail.ch>
Change-Id: Idc91da55ab0b5c6b8eb53e8b4ce4da18a1ae891c

9 files changed, 166 insertions, 119 deletions

diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index a9b549a935..9fc525cab7 100644
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -300,19 +300,21 @@ class DUTSetup:
                 return sriov_numvfs
 
     @staticmethod
-    def set_sriov_numvfs(node, pf_pci_addr, numvfs=0):
+    def set_sriov_numvfs(node, pf_pci_addr, path="devices", numvfs=0):
         """Init or reset SR-IOV virtual functions by setting its number on PCI
         device on DUT. Setting to zero removes all VFs.
 
         :param node: DUT node.
         :param pf_pci_addr: Physical Function PCI device address.
+        :param type: Either device or driver.
         :param numvfs: Number of VFs to initialize, 0 - removes the VFs.
         :type node: dict
         :type pf_pci_addr: str
+        :type type: str
         :type numvfs: int
         :raises RuntimeError: Failed to create VFs on PCI.
         """
-        cmd = f"test -f /sys/bus/pci/devices/{pf_pci_addr}/sriov_numvfs"
+        cmd = f"test -f /sys/bus/pci/{path}/{pf_pci_addr}/sriov_numvfs"
         sriov_unsupported, _, _ = exec_cmd(node, cmd)
         # if sriov_numvfs doesn't exist, then sriov_unsupported != 0
         if int(sriov_unsupported):
@@ -328,7 +330,7 @@ class DUTSetup:
 
         pci = pf_pci_addr.replace(u":", r"\:")
         command = f"sh -c \"echo {numvfs} | " \
-            f"tee /sys/bus/pci/devices/{pci}/sriov_numvfs\""
+            f"tee /sys/bus/pci/{path}/{pci}/sriov_numvfs\""
         message = f"Failed to create {numvfs} VFs on {pf_pci_addr} device " \
             f"on {node[u'host']}"
 
diff --git a/resources/libraries/python/QATUtil.py b/resources/libraries/python/QATUtil.py
index 34a8e3b8d9..be8cc9561e 100644
--- a/resources/libraries/python/QATUtil.py
+++ b/resources/libraries/python/QATUtil.py
@@ -14,7 +14,7 @@
 """QAT util library."""
 
 from resources.libraries.python.DUTSetup import DUTSetup
-from resources.libraries.python.topology import Topology
+from resources.libraries.python.topology import NodeType, Topology
 from resources.libraries.python.VPPUtil import VPPUtil
 
 
@@ -22,88 +22,53 @@ class QATUtil:
     """Contains methods for setting up QATs."""
 
     @staticmethod
-    def crypto_device_verify(node, crypto_type, numvfs, force_init=True):
-        """Verify if Crypto QAT device virtual functions are initialized on all
-        DUTs. If parameter force initialization is set to True, then try to
-        initialize or remove VFs on QAT.
+    def crypto_device_verify_on_all_duts(nodes):
+        """Verify if Crypto QAT device and its virtual functions are initialized
+        on all DUTs.
 
-        :param node: DUT node.
-        :crypto_type: Crypto device type - HW_DH895xcc, HW_C3xxx, HW_C4xxx
-                      or HW_4xxx.
-        :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
-        :param force_init: If True then try to initialize to specific value.
-        :type node: dict
-        :type crypto_type: string
-        :type numvfs: int
-        :type force_init: bool
-        :returns: nothing
-        :raises RuntimeError: If QAT VFs are not created and force init is set
-                              to False.
+        :param nodes: Nodes in the topology.
+        :type nodes: dict
         """
-        pci_addr = Topology.get_cryptodev(node)
-
-        if force_init:
-            # QAT is not initialized and we want to initialize with numvfs.
-            QATUtil.crypto_device_init(node, crypto_type, numvfs)
-        else:
-            raise RuntimeError(
-                f"QAT device failed to create VFs on {node[u'host']}"
-            )
+        VPPUtil.stop_vpp_service_on_all_duts(nodes)
 
-        # QAT VF devices must be re-bound to vfio-pci driver before use.
-        pci_addr = Topology.get_cryptodev(node)
-        for i in range(numvfs):
-            DUTSetup.pci_vf_driver_unbind(node, pci_addr, i)
-            DUTSetup.pci_vf_driver_bind(node, pci_addr, i, "vfio-pci")
+        for node in nodes.values():
+            if node["type"] == NodeType.DUT:
+                cryptodevs = Topology.get_cryptodev(node)
+                if not cryptodevs:
+                    return
+                for device in cryptodevs.values():
+                    QATUtil.crypto_device_init(node, device)
 
     @staticmethod
-    def crypto_device_init(node, crypto_type, numvfs):
+    def crypto_device_init(node, device):
         """Init Crypto QAT device virtual functions on DUT.
 
         :param node: DUT node.
-        :crypto_type: Crypto device type - HW_DH895xcc, HW_C3xxx, HW_C4xxx
-                      or HW_4xxx.
-        :param numvfs: Number of VFs to initialize, 0 - disable the VFs.
+        :device: Crypto device entry from topology file.
         :type node: dict
-        :type crypto_type: string
-        :type numvfs: int
-        :returns: nothing
-        :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
+        :type device: dict
         """
-        if crypto_type == u"HW_DH895xcc":
-            kernel_mod = u"qat_dh895xcc"
-            kernel_drv = u"dh895xcc"
-        elif crypto_type == u"HW_C3xxx":
-            kernel_mod = u"qat_c3xxx"
-            kernel_drv = u"c3xxx"
-        elif crypto_type == u"HW_C4xxx":
-            kernel_mod = u"qat_c4xxx"
-            kernel_drv = u"c4xxx"
-        elif crypto_type == u"HW_4xxx":
-            kernel_mod = u"qat_4xxx"
-            kernel_drv = u"4xxx"
-        else:
-            raise RuntimeError(
-                f"Unsupported crypto device type on {node[u'host']}"
-            )
-
-        pci_addr = Topology.get_cryptodev(node)
-
-        # QAT device must be re-bound to kernel driver before initialization.
-        DUTSetup.verify_kernel_module(node, kernel_mod, force_load=True)
-
-        # Stop VPP to prevent deadlock.
-        VPPUtil.stop_vpp_service(node)
+        DUTSetup.verify_kernel_module(node, device["module"], force_load=True)
 
         current_driver = DUTSetup.get_pci_dev_driver(
-            node, pci_addr.replace(u":", r"\:")
+            node, device["pci_address"].replace(":", r"\:")
         )
         if current_driver is not None:
-            DUTSetup.pci_driver_unbind(node, pci_addr)
-
+            DUTSetup.pci_driver_unbind(node, device["pci_address"])
         # Bind to kernel driver.
-        DUTSetup.pci_driver_bind(node, pci_addr, kernel_drv)
+        DUTSetup.pci_driver_bind(node, device["pci_address"], device["driver"])
 
         # Initialize QAT VFs.
-        if numvfs > 0:
-            DUTSetup.set_sriov_numvfs(node, pci_addr, numvfs)
\ No newline at end of file
+        if int(device["numvfs"]) > 0:
+            DUTSetup.set_sriov_numvfs(
+                node, device["pci_address"], type="drivers/4xxx",
+                numvfs=device["numvfs"]
+            )
+
+        for cvf in range(int(device["numvfs"])):
+            DUTSetup.pci_vf_driver_unbind(
+                node, device["pci_address"], cvf
+            )
+            DUTSetup.pci_vf_driver_bind(
+                node, device["pci_address"], cvf, "vfio-pci"
+            )
diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py
index a57d24cde5..0628e8ee47 100644
--- a/resources/libraries/python/VppConfigGenerator.py
+++ b/resources/libraries/python/VppConfigGenerator.py
@@ -259,12 +259,13 @@ class VppConfigGenerator:
         :param count: Number of HW crypto devices to add.
         :type count: int
         """
-        cryptodev = Topology.get_cryptodev(self._node)
-        for i in range(count):
-            cryptodev_config = re.sub(r"\d.\d$", f"1.{str(i)}", cryptodev)
-            path = [u"dpdk", f"dev {cryptodev_config}"]
-            self.add_config_item(self._nodeconfig, u"", path)
-        self.add_dpdk_uio_driver(u"vfio-pci")
+        cryptodevs = Topology.get_cryptodev(self._node)
+        for device in cryptodevs.values():
+            for i in range(int(count/len(cryptodevs))):
+                addr = re.sub(r"\d.\d$", f"0.{i+1}", device["pci_address"])
+                path = ["dpdk", f"dev {addr}"]
+                self.add_config_item(self._nodeconfig, "", path)
+        self.add_dpdk_uio_driver("vfio-pci")
 
     def add_dpdk_sw_cryptodev(self, sw_pmd_type, socket_id, count):
         """Add DPDK SW Crypto device configuration.
@@ -740,4 +741,4 @@ class VppInitConfig:
                 )
                 vpp_config.add_ip6_hash_buckets(2000000)
                 vpp_config.add_ip6_heap_size("4G")
-                vpp_config.apply_config()
\ No newline at end of file
+                vpp_config.apply_config()
diff --git a/resources/libraries/robot/shared/default.robot b/resources/libraries/robot/shared/default.robot
index be68958f5f..c17b7ac441 100644
--- a/resources/libraries/robot/shared/default.robot
+++ b/resources/libraries/robot/shared/default.robot
@@ -96,29 +96,6 @@
 | | # Parens are there to perform the call.
 | | Run Keyword If | $resetter | Evaluate | $resetter()
 
-| Configure crypto device on all DUTs
-| | [Documentation] | Verify if Crypto QAT device virtual functions are
-| | ... | initialized on all DUTs. If parameter force_init is set to True, then
-| | ... | try to initialize/disable.
-| |
-| | ... | *Arguments:*
-| | ... | - crypto_type - Crypto device type - HW_DH895xcc, HW_C3xxx, HW_C4xxx
-| | ... | or HW_4xxx; default value: HW_DH895xcc. Type: string
-| | ... | - numvfs - Number of VFs to initialize, 0 - disable the VFs; default
-| | ... | value: ${32} Type: integer
-| | ... | - force_init - Force to initialize. Type: boolean
-| |
-| | ... | *Example:*
-| |
-| | ... | \| Configure crypto device on all DUTs \| HW_DH895xcc \| ${32} \|
-| |
-| | [Arguments] | ${crypto_type} | ${numvfs} | ${force_init}=${True}
-| |
-| | FOR | ${dut} | IN | @{duts}
-| | | Crypto Device Verify | ${nodes['${dut}']} | ${crypto_type}
-| | | ... | ${numvfs} | force_init=${force_init}
-| | END
-
 | Configure kernel module on all DUTs
 | | [Documentation] | Verify if specific kernel module is loaded on all DUTs.
 | | ... | If parameter force_load is set to True, then try to load.
diff --git a/resources/libraries/robot/shared/suite_setup.robot b/resources/libraries/robot/shared/suite_setup.robot
index 9c8603af3b..438a39fb0e 100644
--- a/resources/libraries/robot/shared/suite_setup.robot
+++ b/resources/libraries/robot/shared/suite_setup.robot
@@ -275,13 +275,7 @@
 | | [Documentation]
 | | ... | Additional Setup for suites which uses QAT HW.
 | |
-| | ${numvfs}= | Set Variable If
-| | ... | '${crypto_type}' == 'HW_DH895xcc' | ${32}
-| | ... | '${crypto_type}' == 'HW_C3xxx' | ${16}
-| | ... | '${crypto_type}' == 'HW_C4xxx' | ${128}
-| | ... | '${crypto_type}' == 'HW_4xxx' | ${16}
-| | Configure crypto device on all DUTs | ${crypto_type} | numvfs=${numvfs}
-| | ... | force_init=${True}
+| | Crypto Device Verify on all DUTs | ${nodes}
 
 | Additional Suite Setup Action For nginx
 | | [Documentation]
diff --git a/topologies/available/lf_2n_spr_testbed23.yaml b/topologies/available/lf_2n_spr_testbed23.yaml
index bf11a03147..7d76f16788 100644
--- a/topologies/available/lf_2n_spr_testbed23.yaml
+++ b/topologies/available/lf_2n_spr_testbed23.yaml
@@ -24,7 +24,19 @@ nodes:
         pci_address: "0000:f2:01.0"
         driver: idxd
         model: Intel-DSA
-    cryptodev: "0000:f3:00.0"
+    cryptodev:
+      device1:
+        pci_address: "0000:76:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
+      device2:
+        pci_address: "0000:7a:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
     interfaces:
       port1:
         # s57-t23-tg1-c2/p1 - 200GE port1 Mellanox ConnectX7-2p200GE.
@@ -139,7 +151,19 @@ nodes:
         pci_address: "0000:f2:01.0"
         driver: idxd
         model: Intel-DSA
-    cryptodev: "0000:f3:00.0"
+    cryptodev:
+      device1:
+        pci_address: "0000:76:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
+      device2:
+        pci_address: "0000:7a:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
     interfaces:
       port1:
         # s56-t23-sut1-c2/p1 - 200GE port1 Mellanox ConnectX7-2p200GE.
diff --git a/topologies/available/lf_3n_snr_testbed39.yaml b/topologies/available/lf_3n_snr_testbed39.yaml
index 6e69d18445..d2f1e0df73 100644
--- a/topologies/available/lf_3n_snr_testbed39.yaml
+++ b/topologies/available/lf_3n_snr_testbed39.yaml
@@ -55,7 +55,13 @@ nodes:
     port: 22
     username: testuser
     password: Csit1234
-    cryptodev: "0000:f4:00.0"
+    cryptodev:
+      device1:
+        pci_address: "0000:f4:00.0"
+        driver: "c4xxx"
+        model: "HW_C4xxx"
+        module: "qat_c4xxx"
+        numvfs: 8
     uio_driver: vfio-pci
     interfaces:
       port1:
@@ -97,7 +103,13 @@ nodes:
     port: 22
     username: testuser
     password: Csit1234
-    cryptodev: "0000:f4:00.0"
+    cryptodev:
+      device1:
+        pci_address: "0000:f4:00.0"
+        driver: "c4xxx"
+        model: "HW_C4xxx"
+        module: "qat_c4xxx"
+        numvfs: 8
     uio_driver: vfio-pci
     interfaces:
       port1:
diff --git a/topologies/available/lf_3na_spr_testbed21.yaml b/topologies/available/lf_3na_spr_testbed21.yaml
index 73dc9bad44..1167ab096f 100644
--- a/topologies/available/lf_3na_spr_testbed21.yaml
+++ b/topologies/available/lf_3na_spr_testbed21.yaml
@@ -24,7 +24,19 @@ nodes:
         pci_address: "0000:f2:01.0"
         driver: idxd
         model: Intel-DSA
-    cryptodev: "0000:76:00.0"
+    cryptodev:
+      device1:
+        pci_address: "0000:76:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
+      device2:
+        pci_address: "0000:7a:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
     interfaces:
       port1:
         # s53-t21-tg1-c4/p1 - 200GE port1 Mellanox ConnectX7-2p200GE.
@@ -107,7 +119,19 @@ nodes:
         pci_address: "0000:f2:01.0"
         driver: idxd
         model: Intel-DSA
-    cryptodev: "0000:76:00.0"
+    cryptodev:
+      device1:
+        pci_address: "0000:76:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
+      device2:
+        pci_address: "0000:7a:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
     interfaces:
       port1:
         # s52-t21-sut1-c4/p1 - 200GE port1 Mellanox ConnectX7-2p200GE.
@@ -174,7 +198,19 @@ nodes:
         pci_address: "0000:f2:01.0"
         driver: idxd
         model: Intel-DSA
-    cryptodev: "0000:f3:00.0"
+    cryptodev:
+      device1:
+        pci_address: "0000:f3:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
+      device2:
+        pci_address: "0000:f7:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
     interfaces:
       port1:
         # s52-t21-sut2-c9/p1 - 200GE port1 Mellanox ConnectX7-2p200GE.
diff --git a/topologies/available/lf_3nb_spr_testbed22.yaml b/topologies/available/lf_3nb_spr_testbed22.yaml
index f8e9708283..4b6f378e8c 100644
--- a/topologies/available/lf_3nb_spr_testbed22.yaml
+++ b/topologies/available/lf_3nb_spr_testbed22.yaml
@@ -24,7 +24,19 @@ nodes:
         pci_address: "0000:f2:01.0"
         driver: idxd
         model: Intel-DSA
-    cryptodev: "0000:76:00.0"
+    cryptodev:
+      device1:
+        pci_address: "0000:76:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
+      device2:
+        pci_address: "0000:7a:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
     interfaces:
       port1:
         # s55-t22-tg1-c4/p1 - 100GE port1 Intel E810-CQDA2 2p100GE.
@@ -75,7 +87,19 @@ nodes:
         pci_address: "0000:f2:01.0"
         driver: idxd
         model: Intel-DSA
-    cryptodev: "0000:76:00.0"
+    cryptodev:
+      device1:
+        pci_address: "0000:76:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
+      device2:
+        pci_address: "0000:7a:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
     interfaces:
       port1:
         # s54-t22-sut1-c4/p1 - 100GE port1 Intel E810-CQDA2 2p100GE.
@@ -126,7 +150,19 @@ nodes:
         pci_address: "0000:f2:01.0"
         driver: idxd
         model: Intel-DSA
-    cryptodev: "0000:f3:00.0"
+    cryptodev:
+      device1:
+        pci_address: "0000:f3:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
+      device2:
+        pci_address: "0000:f7:00.0"
+        driver: "4xxx"
+        model: "HW_4xxx"
+        module: "qat_4xxx"
+        numvfs: 8
     interfaces:
       port1:
         # s54-t22-sut1-c9/p1 - 100GE port1 Intel E810-CQDA2 2p100GE.