CSIT-1205 Create AVF driver test

- Add L1 KWs for SR-IOV handling (init Vfs, remove Vfs, ...)
- Cleanup L1 KWs for SR-IOV bind/unbind/pci_get/...
- Add L2 KWs for Test Setup/Teardown, L2patch, Create AVF interface
- Add sample L2patch test fox x710, xxv710

Change-Id: If17077877455a14043617d8ea0d06cbe47b469e3
Signed-off-by: Peter Mikus <pmikus@cisco.com>

1 files changed, 217 insertions, 107 deletions

diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index 8f9e94d560..84862d4167 100644
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -16,7 +16,7 @@
 from robot.api import logger
 
 from resources.libraries.python.topology import NodeType, Topology
-from resources.libraries.python.ssh import SSH
+from resources.libraries.python.ssh import SSH, exec_cmd_no_error
 from resources.libraries.python.constants import Constants
 from resources.libraries.python.VatExecutor import VatExecutor
 from resources.libraries.python.VPPUtil import VPPUtil
@@ -252,7 +252,7 @@ class DUTSetup(object):
     def crypto_device_verify(node, force_init=False, numvfs=32):
         """Verify if Crypto QAT device virtual functions are initialized on all
         DUTs. If parameter force initialization is set to True, then try to
-        initialize or disable QAT.
+        initialize or remove VFs on QAT.
 
         :param node: DUT node.
         :param force_init: If True then try to initialize to specific value.
@@ -261,37 +261,19 @@ class DUTSetup(object):
         :type force_init: bool
         :type numvfs: int
         :returns: nothing
-        :raises RuntimeError: If QAT is not initialized or failed to initialize.
+        :raises RuntimeError: If QAT VFs are not created and force init is set
+                              to False.
         """
+        pci_addr = Topology.get_cryptodev(node)
+        sriov_numvfs = DUTSetup.get_sriov_numvfs(node, pci_addr)
 
-        ssh = SSH()
-        ssh.connect(node)
-
-        cryptodev = Topology.get_cryptodev(node)
-        cmd = 'cat /sys/bus/pci/devices/{0}/sriov_numvfs'.\
-            format(cryptodev.replace(':', r'\:'))
-
-        # Try to read number of VFs from PCI address of QAT device
-        for _ in range(3):
-            ret_code, stdout, _ = ssh.exec_command(cmd)
-            if not int(ret_code):
-                try:
-                    sriov_numvfs = int(stdout)
-                except ValueError:
-                    logger.trace('Reading sriov_numvfs info failed on {0}'.
-                                 format(node['host']))
-                else:
-                    if sriov_numvfs != numvfs:
-                        if force_init:
-                            # QAT is not initialized and we want to initialize
-                            # with numvfs
-                            DUTSetup.crypto_device_init(node, numvfs)
-                        else:
-                            raise RuntimeError('QAT device {0} is not '
-                                               'initialized to {1} on host {2}'
-                                               .format(cryptodev, numvfs,
-                                                       node['host']))
-                    break
+        if sriov_numvfs != numvfs:
+            if force_init:
+                # QAT is not initialized and we want to initialize with numvfs
+                DUTSetup.crypto_device_init(node, numvfs)
+            else:
+                raise RuntimeError('QAT device failed to create VFs on {host}'.
+                                   format(host=node['host']))
 
     @staticmethod
     def crypto_device_init(node, numvfs):
@@ -304,33 +286,98 @@ class DUTSetup(object):
         :returns: nothing
         :raises RuntimeError: If failed to stop VPP or QAT failed to initialize.
         """
-        cryptodev = Topology.get_cryptodev(node)
+        pci_addr = Topology.get_cryptodev(node)
 
-        # QAT device must be re-bound to kernel driver before initialization
-        driver = 'dh895xcc'
-        kernel_module = 'qat_dh895xcc'
-        current_driver = DUTSetup.get_pci_dev_driver(
-            node, cryptodev.replace(':', r'\:'))
-
-        DUTSetup.kernel_module_verify(node, kernel_module, force_load=True)
+        # QAT device must be re-bound to kernel driver before initialization.
+        DUTSetup.verify_kernel_module(node, 'qat_dh895xcc', force_load=True)
 
+        # Stop VPP to prevent deadlock.
         VPPUtil.stop_vpp_service(node)
+
+        current_driver = DUTSetup.get_pci_dev_driver(
+            node, pci_addr.replace(':', r'\:'))
         if current_driver is not None:
-            DUTSetup.pci_driver_unbind(node, cryptodev)
-        DUTSetup.pci_driver_bind(node, cryptodev, driver)
+            DUTSetup.pci_driver_unbind(node, pci_addr)
 
-        ssh = SSH()
-        ssh.connect(node)
+        # Bind to kernel driver.
+        DUTSetup.pci_driver_bind(node, pci_addr, 'dh895xcc')
 
-        # Initialize QAT VFs
+        # Initialize QAT VFs.
         if numvfs > 0:
-            cmd = 'echo "{0}" | tee /sys/bus/pci/devices/{1}/sriov_numvfs'.\
-                format(numvfs, cryptodev.replace(':', r'\:'), timeout=180)
-            ret_code, _, _ = ssh.exec_command_sudo("sh -c '{0}'".format(cmd))
+            DUTSetup.set_sriov_numvfs(node, pci_addr, numvfs)
 
-            if int(ret_code):
-                raise RuntimeError('Failed to initialize {0} VFs on QAT device '
-                                   ' on host {1}'.format(numvfs, node['host']))
+    @staticmethod
+    def get_virtfn_pci_addr(node, pf_pci_addr, vf_id):
+        """Get PCI address of Virtual Function.
+
+        :param node: DUT node.
+        :param pf_pci_addr: Physical Function PCI address.
+        :param vf_id: Virtual Function number.
+        :type node: dict
+        :type pf_pci_addr: str
+        :type vf_id: int
+        :returns: Virtual Function PCI address.
+        :rtype: int
+        :raises RuntimeError: If failed to get Virtual Function PCI address.
+        """
+        command = "sh -c "\
+            "'basename $(readlink /sys/bus/pci/devices/{pci}/virtfn{vf_id})'".\
+            format(pci=pf_pci_addr, vf_id=vf_id)
+        message = 'Failed to get virtual function PCI address.'
+
+        stdout, _ = exec_cmd_no_error(node, command, timeout=30, sudo=True,
+                                      message=message)
+
+        return stdout.strip()
+
+    @staticmethod
+    def get_sriov_numvfs(node, pf_pci_addr):
+        """Get number of SR-IOV VFs.
+
+        :param node: DUT node.
+        :param pf_pci_addr: Physical Function PCI device address.
+        :type node: dict
+        :type pf_pci_addr: str
+        :returns: Number of VFs.
+        :rtype: int
+        :raises RuntimeError: If PCI device is not SR-IOV capable.
+        """
+        command = 'cat /sys/bus/pci/devices/{pci}/sriov_numvfs'.\
+            format(pci=pf_pci_addr.replace(':', r'\:'))
+        message = 'PCI device {pci} is not a SR-IOV device.'.\
+            format(pci=pf_pci_addr)
+
+        for _ in range(3):
+            stdout, _ = exec_cmd_no_error(node, command, timeout=30, sudo=True,
+                                          message=message)
+            try:
+                sriov_numvfs = int(stdout)
+            except ValueError:
+                logger.trace('Reading sriov_numvfs info failed on {host}'.
+                             format(host=node['host']))
+            else:
+                return sriov_numvfs
+
+    @staticmethod
+    def set_sriov_numvfs(node, pf_pci_addr, numvfs=0):
+        """Init or reset SR-IOV virtual functions by setting its number on PCI
+        device on DUT. Setting to zero removes all VFs.
+
+        :param node: DUT node.
+        :param pf_pci_addr: Physical Function PCI device address.
+        :param numvfs: Number of VFs to initialize, 0 - removes the VFs.
+        :type node: dict
+        :type pf_pci_addr: str
+        :type numvfs: int
+        :raises RuntimeError: Failed to create VFs on PCI.
+        """
+        command = "sh -c "\
+            "'echo {num} | tee /sys/bus/pci/devices/{pci}/sriov_numvfs'".\
+            format(num=numvfs, pci=pf_pci_addr.replace(':', r'\:'))
+        message = 'Failed to create {num} VFs on {pci} device on {host}'.\
+            format(num=numvfs, pci=pf_pci_addr, host=node['host'])
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
 
     @staticmethod
     def pci_driver_unbind(node, pci_addr):
@@ -340,20 +387,15 @@ class DUTSetup(object):
         :param pci_addr: PCI device address.
         :type node: dict
         :type pci_addr: str
-        :returns: nothing
         :raises RuntimeError: If PCI device unbind failed.
         """
+        command = "sh -c "\
+            "'echo {pci} | tee /sys/bus/pci/devices/{pcie}/driver/unbind'".\
+            format(pci=pci_addr, pcie=pci_addr.replace(':', r'\:'))
+        message = 'Failed to unbind PCI device {pci} on {host}'.\
+            format(pci=pci_addr, host=node['host'])
 
-        ssh = SSH()
-        ssh.connect(node)
-
-        ret_code, _, _ = ssh.exec_command_sudo(
-            "sh -c 'echo {0} | tee /sys/bus/pci/devices/{1}/driver/unbind'"
-            .format(pci_addr, pci_addr.replace(':', r'\:')), timeout=180)
-
-        if int(ret_code):
-            raise RuntimeError('Failed to unbind PCI device {0} from driver on '
-                               'host {1}'.format(pci_addr, node['host']))
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
 
     @staticmethod
     def pci_driver_bind(node, pci_addr, driver):
@@ -365,21 +407,92 @@ class DUTSetup(object):
         :type node: dict
         :type pci_addr: str
         :type driver: str
-        :returns: nothing
         :raises RuntimeError: If PCI device bind failed.
         """
+        message = 'Failed to bind PCI device {pci} to {driver} on host {host}'.\
+            format(pci=pci_addr, driver=driver, host=node['host'])
 
-        ssh = SSH()
-        ssh.connect(node)
+        command = "sh -c "\
+            "'echo {driver} | tee /sys/bus/pci/devices/{pci}/driver_override'".\
+            format(driver=driver, pci=pci_addr.replace(':', r'\:'))
 
-        ret_code, _, _ = ssh.exec_command_sudo(
-            "sh -c 'echo {0} | tee /sys/bus/pci/drivers/{1}/bind'".format(
-                pci_addr, driver), timeout=180)
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
 
-        if int(ret_code):
-            raise RuntimeError('Failed to bind PCI device {0} to {1} driver on '
-                               'host {2}'.format(pci_addr, driver,
-                                                 node['host']))
+        command = "sh -c "\
+            "'echo {pci} | tee /sys/bus/pci/drivers/{driver}/bind'".\
+            format(pci=pci_addr, driver=driver)
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
+
+        command = "sh -c "\
+            "'echo  | tee /sys/bus/pci/devices/{pci}/driver_override'".\
+            format(pci=pci_addr.replace(':', r'\:'))
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
+
+    @staticmethod
+    def pci_vf_driver_unbind(node, pf_pci_addr, vf_id):
+        """Unbind Virtual Function from driver on node.
+
+        :param node: DUT node.
+        :param pf_pci_addr: PCI device address.
+        :param vf_id: Virtual Function ID.
+        :type node: dict
+        :type pf_pci_addr: str
+        :type vf_id: int
+        :raises RuntimeError: If Virtual Function unbind failed.
+        """
+        vf_pci_addr = DUTSetup.get_virtfn_pci_addr(node, pf_pci_addr, vf_id)
+        vf_path = "/sys/bus/pci/devices/{pf_pci_addr}/virtfn{vf_id}".\
+            format(pf_pci_addr=pf_pci_addr.replace(':', r'\:'), vf_id=vf_id)
+
+        command = "sh -c "\
+            "'echo {vf_pci_addr} | tee {vf_path}/driver/unbind'".\
+            format(vf_pci_addr=vf_pci_addr, vf_path=vf_path)
+
+        message = 'Failed to unbind VF {vf_pci_addr} to on {host}'.\
+            format(vf_pci_addr=vf_pci_addr, host=node['host'])
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
+
+    @staticmethod
+    def pci_vf_driver_bind(node, pf_pci_addr, vf_id, driver):
+        """Bind Virtual Function to driver on node.
+
+        :param node: DUT node.
+        :param pf_pci_addr: PCI device address.
+        :param vf_id: Virtual Function ID.
+        :param driver: Driver to bind.
+        :type node: dict
+        :type pf_pci_addr: str
+        :type vf_id: int
+        :type driver: str
+        :raises RuntimeError: If PCI device bind failed.
+        """
+        vf_pci_addr = DUTSetup.get_virtfn_pci_addr(node, pf_pci_addr, vf_id)
+        vf_path = "/sys/bus/pci/devices/{pf_pci_addr}/virtfn{vf_id}".\
+            format(pf_pci_addr=pf_pci_addr.replace(':', r'\:'), vf_id=vf_id)
+
+        message = 'Failed to bind VF {vf_pci_addr} to {driver} on {host}'.\
+            format(vf_pci_addr=vf_pci_addr, driver=driver, host=node['host'])
+
+        command = "sh -c "\
+            "'echo {driver} | tee {vf_path}/driver_override'".\
+            format(driver=driver, vf_path=vf_path)
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
+
+        command = "sh -c "\
+            "'echo {vf_pci_addr} | tee /sys/bus/pci/drivers/{driver}/bind'".\
+            format(vf_pci_addr=vf_pci_addr, driver=driver)
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
+
+        command = "sh -c "\
+            "'echo  | tee {vf_path}/driver_override'".\
+            format(vf_path=vf_path)
+
+        exec_cmd_no_error(node, command, timeout=60, sudo=True, message=message)
 
     @staticmethod
     def get_pci_dev_driver(node, pci_addr):
@@ -442,7 +555,7 @@ class DUTSetup(object):
         return None
 
     @staticmethod
-    def kernel_module_verify(node, module, force_load=False):
+    def verify_kernel_module(node, module, force_load=False):
         """Verify if kernel module is loaded on node. If parameter force
         load is set to True, then try to load the modules.
 
@@ -454,22 +567,22 @@ class DUTSetup(object):
         :type force_load: bool
         :raises RuntimeError: If module is not loaded or failed to load.
         """
-        ssh = SSH()
-        ssh.connect(node)
+        command = 'grep -w {module} /proc/modules'.format(module=module)
+        message = 'Kernel module {module} is not loaded on host {host}'.\
+            format(module=module, host=node['host'])
 
-        cmd = 'grep -w {0} /proc/modules'.format(module)
-        ret_code, _, _ = ssh.exec_command(cmd)
-
-        if int(ret_code):
+        try:
+            exec_cmd_no_error(node, command, timeout=30, sudo=False,
+                              message=message)
+        except RuntimeError:
             if force_load:
                 # Module is not loaded and we want to load it
-                DUTSetup.kernel_module_load(node, module)
+                DUTSetup.load_kernel_module(node, module)
             else:
-                raise RuntimeError('Kernel module {0} is not loaded on host '
-                                   '{1}'.format(module, node['host']))
+                raise
 
     @staticmethod
-    def kernel_module_verify_on_all_duts(nodes, module, force_load=False):
+    def verify_kernel_module_on_all_duts(nodes, module, force_load=False):
         """Verify if kernel module is loaded on all DUTs. If parameter force
         load is set to True, then try to load the modules.
 
@@ -482,7 +595,7 @@ class DUTSetup(object):
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
-                DUTSetup.kernel_module_verify(node, module, force_load)
+                DUTSetup.verify_kernel_module(node, module, force_load)
 
     @staticmethod
     def verify_uio_driver_on_all_duts(nodes):
@@ -495,10 +608,10 @@ class DUTSetup(object):
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
                 uio_driver = Topology.get_uio_driver(node)
-                DUTSetup.kernel_module_verify(node, uio_driver, force_load=True)
+                DUTSetup.verify_kernel_module(node, uio_driver, force_load=True)
 
     @staticmethod
-    def kernel_module_load(node, module):
+    def load_kernel_module(node, module):
         """Load kernel module on node.
 
         :param node: DUT node.
@@ -508,15 +621,11 @@ class DUTSetup(object):
         :returns: nothing
         :raises RuntimeError: If loading failed.
         """
+        command = 'modprobe {module}'.format(module=module)
+        message = 'Failed to load {module} on host {host}'.\
+            format(module=module, host=node['host'])
 
-        ssh = SSH()
-        ssh.connect(node)
-
-        ret_code, _, _ = ssh.exec_command_sudo("modprobe {0}".format(module))
-
-        if int(ret_code):
-            raise RuntimeError('Failed to load {0} kernel module on host {1}'.
-                               format(module, node['host']))
+        exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
 
     @staticmethod
     def vpp_enable_traces_on_all_duts(nodes):
@@ -615,6 +724,21 @@ class DUTSetup(object):
                 ssh.disconnect(node)
 
     @staticmethod
+    def verify_vpp_on_dut(node):
+        """Verify that VPP is installed on DUT node.
+
+        :param node: DUT node.
+        :type node: dict
+        :raises RuntimeError: If failed to restart VPP, get VPP version
+            or get VPP interfaces.
+        """
+
+        logger.debug("Verify VPP on node {0}".format(node['host']))
+
+        DUTSetup.vpp_show_version_verbose(node)
+        DUTSetup.vpp_show_interfaces(node)
+
+    @staticmethod
     def verify_vpp_on_all_duts(nodes):
         """Verify that VPP is installed on all DUT nodes.
 
@@ -630,20 +754,6 @@ class DUTSetup(object):
             if node['type'] == NodeType.DUT:
                 DUTSetup.verify_vpp_on_dut(node)
 
-    @staticmethod
-    def verify_vpp_on_dut(node):
-        """Verify that VPP is installed on DUT node.
-
-        :param node: DUT node.
-        :type node: dict
-        :raises RuntimeError: If failed to restart VPP, get VPP version
-            or get VPP interfaces.
-        """
-
-        logger.debug("Verify VPP on node {0}".format(node['host']))
-
-        DUTSetup.vpp_show_version_verbose(node)
-        DUTSetup.vpp_show_interfaces(node)
 
     @staticmethod
     def get_huge_page_size(node):