CSIT-1493 VPP restart handling code

Change-Id: Ibe52125089f39e0ff17ec607a3ed00c61d52ab8c
Signed-off-by: Peter Mikus <pmikus@cisco.com>

6 files changed, 160 insertions, 145 deletions

diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index 67edefb930..e2fba066d3 100644
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -61,6 +61,39 @@ class DUTSetup(object):
                 DUTSetup.get_service_logs(node, service)
 
     @staticmethod
+    def restart_service(node, service):
+        """Restarts the named service on node.
+
+        :param node: Node in the topology.
+        :param service: Service unit name.
+        :type node: dict
+        :type service: str
+        """
+        if DUTSetup.running_in_container(node):
+            command = 'supervisorctl restart {name}'.format(name=service)
+        else:
+            command = 'service {name} restart'.format(name=service)
+        message = 'Node {host} failed to restart service {name}'.\
+            format(host=node['host'], name=service)
+
+        exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
+
+        DUTSetup.get_service_logs(node, service)
+
+    @staticmethod
+    def restart_service_on_all_duts(nodes, service):
+        """Retarts the named service on all DUTs.
+
+        :param node: Nodes in the topology.
+        :param service: Service unit name.
+        :type node: dict
+        :type service: str
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.restart_service(node, service)
+
+    @staticmethod
     def start_service(node, service):
         """Start up the named service on node.
 
@@ -69,6 +102,7 @@ class DUTSetup(object):
         :type node: dict
         :type service: str
         """
+        # TODO: change command to start once all parent function updated.
         if DUTSetup.running_in_container(node):
             command = 'supervisorctl restart {name}'.format(name=service)
         else:
@@ -609,7 +643,8 @@ class DUTSetup(object):
 
     @staticmethod
     def install_vpp_on_all_duts(nodes, vpp_pkg_dir):
-        """Install VPP on all DUT nodes.
+        """Install VPP on all DUT nodes. Start the VPP service in case of
+        systemd is not available or does not support autostart.
 
         :param nodes: Nodes in the topology.
         :param vpp_pkg_dir: Path to directory where VPP packages are stored.
@@ -634,6 +669,8 @@ class DUTSetup(object):
                                       format(dir=vpp_pkg_dir), timeout=120,
                                       sudo=True, message=message)
                     exec_cmd_no_error(node, 'dpkg -l | grep vpp', sudo=True)
+                    if DUTSetup.running_in_container(node):
+                        DUTSetup.restart_service(node, Constants.VPP_UNIT)
                 else:
                     exec_cmd_no_error(node, 'yum -y remove "*vpp*" || true',
                                       timeout=120, sudo=True)
@@ -641,6 +678,7 @@ class DUTSetup(object):
                                       format(dir=vpp_pkg_dir), timeout=120,
                                       sudo=True, message=message)
                     exec_cmd_no_error(node, 'rpm -qai *vpp*', sudo=True)
+                    DUTSetup.restart_service(node, Constants.VPP_UNIT)
 
     @staticmethod
     def running_in_container(node):
diff --git a/resources/libraries/python/KubernetesUtils.py b/resources/libraries/python/KubernetesUtils.py
index e932492e05..60e12863b4 100644
--- a/resources/libraries/python/KubernetesUtils.py
+++ b/resources/libraries/python/KubernetesUtils.py
@@ -483,7 +483,6 @@ class KubernetesUtils(object):
         vpp_config.set_node(kwargs['node'])
         vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
         vpp_config.add_unix_nodaemon()
-        vpp_config.add_dpdk_socketmem('1024,1024')
         vpp_config.add_heapsize('4G')
         vpp_config.add_ip_heap_size('4G')
         vpp_config.add_ip6_heap_size('4G')
@@ -500,7 +499,7 @@ class KubernetesUtils(object):
         if cpuset_cpus:
             corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
             vpp_config.add_cpu_corelist_workers(corelist_workers)
-        vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)
+        vpp_config.write_config(filename=kwargs['filename'])
 
     @staticmethod
     def create_kubernetes_vnf_startup_config(**kwargs):
@@ -536,4 +535,4 @@ class KubernetesUtils(object):
             corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
             vpp_config.add_cpu_corelist_workers(corelist_workers)
         vpp_config.add_plugin('disable', 'dpdk_plugin.so')
-        vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)
+        vpp_config.write_config(filename=kwargs['filename'])
diff --git a/resources/libraries/python/QemuUtils.py b/resources/libraries/python/QemuUtils.py
index 5404b51292..735feab403 100644
--- a/resources/libraries/python/QemuUtils.py
+++ b/resources/libraries/python/QemuUtils.py
@@ -212,7 +212,7 @@ class QemuUtils(object):
             vpp_config.add_dpdk_no_tx_checksum_offload()
         vpp_config.add_plugin('disable', 'default')
         vpp_config.add_plugin('enable', 'dpdk_plugin.so')
-        vpp_config.apply_config(startup, restart_vpp=False)
+        vpp_config.write_config(startup)
 
         # Create VPP running configuration.
         template = '{res}/{tpl}.exec'.format(res=Constants.RESOURCES_TPL_VM,
diff --git a/resources/libraries/python/VPPUtil.py b/resources/libraries/python/VPPUtil.py
index 726ba2bde6..5b7728412e 100644
--- a/resources/libraries/python/VPPUtil.py
+++ b/resources/libraries/python/VPPUtil.py
@@ -13,14 +13,12 @@
 
 """VPP util library."""
 
-import time
-
 from robot.api import logger
 
 from resources.libraries.python.Constants import Constants
 from resources.libraries.python.DUTSetup import DUTSetup
 from resources.libraries.python.PapiExecutor import PapiExecutor
-from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
+from resources.libraries.python.ssh import exec_cmd_no_error
 from resources.libraries.python.topology import NodeType
 from resources.libraries.python.VatExecutor import VatExecutor
 
@@ -58,55 +56,39 @@ class VPPUtil(object):
             exec_cmd_no_error(node, command, timeout=30, sudo=True)
 
     @staticmethod
-    def start_vpp_service(node, retries=120):
-        """Start VPP service on the specified node.
+    def restart_vpp_service(node):
+        """Restart VPP service on the specified topology node.
 
-        :param node: VPP node.
-        :param retries: Number of times (default 60) to re-try waiting.
+        :param node: Topology node.
         :type node: dict
-        :type retries: int
-        :raises RuntimeError: If VPP service fails to start.
         """
-        DUTSetup.start_service(node, Constants.VPP_UNIT)
-        # Sleep 1 second, up to <retry> times,
-        # and verify if VPP is running.
-        for _ in range(retries):
-            time.sleep(1)
-            command = 'vppctl show pci'
-            ret, stdout, _ = exec_cmd(node, command, timeout=30, sudo=True)
-            if not ret and 'Connection refused' not in stdout:
-                break
-        else:
-            raise RuntimeError('VPP failed to start on host {name}'.
-                               format(name=node['host']))
-        DUTSetup.get_service_logs(node, Constants.VPP_UNIT)
+        DUTSetup.restart_service(node, Constants.VPP_UNIT)
 
     @staticmethod
-    def start_vpp_service_on_all_duts(nodes):
-        """Start up the VPP service on all nodes.
+    def restart_vpp_service_on_all_duts(nodes):
+        """Restart VPP service on all DUT nodes.
 
-        :param nodes: Nodes in the topology.
+        :param nodes: Topology nodes.
         :type nodes: dict
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
-                VPPUtil.start_vpp_service(node)
+                VPPUtil.restart_vpp_service(node)
 
     @staticmethod
     def stop_vpp_service(node):
-        """Stop VPP service on the specified node.
+        """Stop VPP service on the specified topology node.
 
-        :param node: VPP node.
+        :param node: Topology node.
         :type node: dict
-        :raises RuntimeError: If VPP service fails to stop.
         """
         DUTSetup.stop_service(node, Constants.VPP_UNIT)
 
     @staticmethod
     def stop_vpp_service_on_all_duts(nodes):
-        """Stop VPP service on all nodes.
+        """Stop VPP service on all DUT nodes.
 
-        :param nodes: Nodes in the topology.
+        :param nodes: Topology nodes.
         :type nodes: dict
         """
         for node in nodes.values():
@@ -114,31 +96,60 @@ class VPPUtil(object):
                 VPPUtil.stop_vpp_service(node)
 
     @staticmethod
-    def verify_vpp_on_dut(node):
-        """Verify that VPP is installed on DUT node.
+    def verify_vpp_installed(node):
+        """Verify that VPP is installed on the specified topology node.
+
+        :param node: Topology node.
+        :type node: dict
+        """
+        cmd = 'command -v vpp'
+        exec_cmd_no_error(
+            node, cmd, message='VPP is not installed!')
+
+    @staticmethod
+    def verify_vpp_started(node):
+        """Verify that VPP is started on the specified topology node.
 
-        :param node: DUT node.
+        :param node: Topology node.
         :type node: dict
-        :raises RuntimeError: If failed to restart VPP, get VPP version
-            or get VPP interfaces.
         """
-        VPPUtil.vpp_show_version_verbose(node)
-        VPPUtil.vpp_show_interfaces(node)
+        cmd = ('vppctl show pci 2>&1 | '
+               'fgrep -v "Connection refused" | '
+               'fgrep -v "No such file or directory"')
+        exec_cmd_no_error(
+            node, cmd, sudo=True, message='VPP failed to start!', retries=120)
+
+    @staticmethod
+    def verify_vpp(node):
+        """Verify that VPP is installed and started on the specified topology
+        node.
+
+        :param node: Topology node.
+        :type node: dict
+        :raises RuntimeError: If VPP service fails to start.
+        """
+        VPPUtil.verify_vpp_installed(node)
+        try:
+            # Verify responsivness of vppctl.
+            VPPUtil.verify_vpp_started(node)
+            # Verify responsivness of PAPI.
+            VPPUtil.show_log(node)
+        finally:
+            DUTSetup.get_service_logs(node, Constants.VPP_UNIT)
 
     @staticmethod
     def verify_vpp_on_all_duts(nodes):
-        """Verify that VPP is installed on all DUT nodes.
+        """Verify that VPP is installed and started on all DUT nodes.
 
         :param nodes: Nodes in the topology.
         :type nodes: dict
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
-                VPPUtil.start_vpp_service(node)
-                VPPUtil.verify_vpp_on_dut(node)
+                VPPUtil.verify_vpp(node)
 
     @staticmethod
-    def vpp_show_version(node, verbose=False):
+    def vpp_show_version(node, verbose=True):
         """Run "show_version" PAPI command.
 
         :param node: Node to run command on.
@@ -149,7 +160,6 @@ class VPPUtil(object):
         :returns: VPP version.
         :rtype: str
         """
-
         with PapiExecutor(node) as papi_exec:
             data = papi_exec.add('show_version').execute_should_pass().\
                 verify_reply()
@@ -164,25 +174,15 @@ class VPPUtil(object):
         return data['version'].rstrip('\0x00')
 
     @staticmethod
-    def vpp_show_version_verbose(node):
-        """Run "show_version" API command and return verbose string of version
-        data.
-
-        :param node: Node to run command on.
-        :type node: dict
-        """
-        VPPUtil.vpp_show_version(node, verbose=True)
-
-    @staticmethod
     def show_vpp_version_on_all_duts(nodes):
-        """Show VPP version on all DUTs.
+        """Show VPP version verbose on all DUTs.
 
-        :param nodes: VPP nodes.
+        :param nodes: Nodes in the topology.
         :type nodes: dict
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
-                VPPUtil.vpp_show_version_verbose(node)
+                VPPUtil.vpp_show_version(node)
 
     @staticmethod
     def vpp_show_interfaces(node):
@@ -212,26 +212,6 @@ class VPPUtil(object):
                            json_out=False)
 
     @staticmethod
-    def vpp_api_trace_dump(node):
-        """Run "api trace custom-dump" CLI command.
-
-        :param node: Node to run command on.
-        :type node: dict
-        """
-        vat = VatExecutor()
-        vat.execute_script("api_trace_dump.vat", node, json_out=False)
-
-    @staticmethod
-    def vpp_api_trace_save(node):
-        """Run "api trace save" CLI command.
-
-        :param node: Node to run command on.
-        :type node: dict
-        """
-        vat = VatExecutor()
-        vat.execute_script("api_trace_save.vat", node, json_out=False)
-
-    @staticmethod
     def vpp_enable_traces_on_dut(node):
         """Enable vpp packet traces on the DUT node.
 
@@ -298,6 +278,19 @@ class VPPUtil(object):
                 VPPUtil.show_event_logger_on_dut(node)
 
     @staticmethod
+    def show_log(node):
+        """Show log on the specified topology node.
+
+        :param node: Topology node.
+        :type node: dict
+        :returns: VPP log data.
+        :rtype: list
+        """
+        with PapiExecutor(node) as papi_exec:
+            return papi_exec.add('cli_inband', cmd='show log').get_replies().\
+                verify_reply()["reply"]
+
+    @staticmethod
     def vpp_show_threads(node):
         """Show VPP threads on node.
 
diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py
index 350e1c0881..883197f978 100644
--- a/resources/libraries/python/VppConfigGenerator.py
+++ b/resources/libraries/python/VppConfigGenerator.py
@@ -14,13 +14,11 @@
 """VPP Configuration File Generator library."""
 
 import re
-import time
 
-from resources.libraries.python.ssh import SSH
-from resources.libraries.python.Constants import Constants
-from resources.libraries.python.DUTSetup import DUTSetup
+from resources.libraries.python.ssh import exec_cmd_no_error
 from resources.libraries.python.topology import NodeType
 from resources.libraries.python.topology import Topology
+from resources.libraries.python.VPPUtil import VPPUtil
 
 __all__ = ['VppConfigGenerator']
 
@@ -559,74 +557,51 @@ class VppConfigGenerator(object):
         path = ['session', 'local-endpoints-table-memory']
         self.add_config_item(self._nodeconfig, value, path)
 
-    def apply_config(self, filename=None, retries=60, restart_vpp=True):
-        """Generate and apply VPP configuration for node.
+    def write_config(self, filename=None):
+        """Generate and write VPP startup configuration to file.
 
         Use data from calls to this class to form a startup.conf file and
-        replace /etc/vpp/startup.conf with it on node.
+        replace /etc/vpp/startup.conf with it on topology node.
 
         :param filename: Startup configuration file name.
-        :param retries: Number of times (default 60) to re-try waiting.
-        :param restart_vpp: Whether to restart VPP.
         :type filename: str
-        :type retries: int
-        :type restart_vpp: bool.
-        :raises RuntimeError: If writing config file failed or restart of VPP
-            failed or backup of VPP startup.conf failed.
         """
         self.dump_config(self._nodeconfig)
 
-        ssh = SSH()
-        ssh.connect(self._node)
-
         if filename is None:
             filename = self._vpp_startup_conf
 
         if self._vpp_startup_conf_backup is not None:
-            ret, _, _ = \
-                ssh.exec_command('sudo cp {src} {dest}'.
-                                 format(src=self._vpp_startup_conf,
-                                        dest=self._vpp_startup_conf_backup))
-            if ret != 0:
-                raise RuntimeError('Backup of config file failed on node '
-                                   '{name}'.format(name=self._hostname))
-
-        ret, _, _ = \
-            ssh.exec_command('echo "{config}" | sudo tee {filename}'.
-                             format(config=self._vpp_config,
-                                    filename=filename))
-
-        if ret != 0:
-            raise RuntimeError('Writing config file failed to node {name}'.
-                               format(name=self._hostname))
-
-        if restart_vpp:
-            DUTSetup.start_service(self._node, Constants.VPP_UNIT)
-
-            # Sleep <waittime> seconds, up to <retry> times,
-            # and verify if VPP is running.
-            for _ in range(retries):
-                time.sleep(1)
-                ret, stdout, _ = \
-                    ssh.exec_command_sudo('vppctl show pci')
-                if ret == 0 and 'Connection refused' not in stdout:
-                    break
-            else:
-                raise RuntimeError('VPP failed to restart on node {name}'.
-                                   format(name=self._hostname))
+            cmd = ('cp {src} {dest}'.format(
+                src=self._vpp_startup_conf, dest=self._vpp_startup_conf_backup))
+            exec_cmd_no_error(
+                self._node, cmd, sudo=True, message='Copy config file failed!')
 
-    def restore_config(self):
-        """Restore VPP startup.conf from backup.
+        cmd = ('echo "{config}" | sudo tee {filename}'.format(
+            config=self._vpp_config, filename=filename))
+        exec_cmd_no_error(
+            self._node, cmd, message='Writing config file failed!')
+
+    def apply_config(self, filename=None, verify_vpp=True):
+        """Generate and write VPP startup configuration to file and restart VPP.
+
+        Use data from calls to this class to form a startup.conf file and
+        replace /etc/vpp/startup.conf with it on topology node.
 
-        :raises RuntimeError: When restoration of startup.conf file failed.
+        :param filename: Startup configuration file name.
+        :param verify_vpp: Verify VPP is running after restart.
+        :type filename: str
+        :type verify_vpp: bool
         """
+        self.write_config(filename=filename)
 
-        ssh = SSH()
-        ssh.connect(self._node)
+        VPPUtil.restart_vpp_service(self._node)
+        if verify_vpp:
+            VPPUtil.verify_vpp(self._node)
 
-        ret, _, _ = ssh.exec_command('sudo cp {src} {dest}'.
-                                     format(src=self._vpp_startup_conf_backup,
-                                            dest=self._vpp_startup_conf))
-        if ret != 0:
-            raise RuntimeError('Restoration of config file failed on node '
-                               '{name}'.format(name=self._hostname))
+    def restore_config(self):
+        """Restore VPP startup.conf from backup."""
+        cmd = ('cp {src} {dest}'.format(
+            src=self._vpp_startup_conf_backup, dest=self._vpp_startup_conf))
+        exec_cmd_no_error(
+            self._node, cmd, sudo=True, message='Copy config file failed!')
diff --git a/resources/libraries/python/ssh.py b/resources/libraries/python/ssh.py
index 966d1b0448..9c7adc44e1 100644
--- a/resources/libraries/python/ssh.py
+++ b/resources/libraries/python/ssh.py
@@ -452,10 +452,14 @@ def exec_cmd(node, cmd, timeout=600, sudo=False, disconnect=False):
 
 
 def exec_cmd_no_error(
-        node, cmd, timeout=600, sudo=False, message=None, disconnect=False):
+        node, cmd, timeout=600, sudo=False, message=None, disconnect=False,
+        retries=0):
     """Convenience function to ssh/exec/return out & err.
 
     Verifies that return code is zero.
+    Supports retries, timeout is related to each try separately then. There is
+    sleep(1) before each retry.
+    Disconnect (if enabled) is applied after each try.
 
     :param node: DUT node.
     :param cmd: Command to be executed.
@@ -463,21 +467,27 @@ def exec_cmd_no_error(
     :param sudo: Sudo privilege execution flag. Default: False.
     :param message: Error message in case of failure. Default: None.
     :param disconnect: Close the opened SSH connection if True.
+    :param retries: How many times to retry on failure.
     :type node: dict
     :type cmd: str or OptionString
     :type timeout: int
     :type sudo: bool
     :type message: str
     :type disconnect: bool
+    :type retries: int
     :returns: Stdout, Stderr.
     :rtype: tuple(str, str)
     :raises RuntimeError: If bash return code is not 0.
     """
-    ret_code, stdout, stderr = exec_cmd(
-        node, cmd, timeout=timeout, sudo=sudo, disconnect=disconnect)
-    msg = ('Command execution failed: "{cmd}"\n{stderr}'.
-           format(cmd=cmd, stderr=stderr) if message is None else message)
-    if ret_code != 0:
+    for _ in range(retries + 1):
+        ret_code, stdout, stderr = exec_cmd(
+            node, cmd, timeout=timeout, sudo=sudo, disconnect=disconnect)
+        if ret_code == 0:
+            break
+        sleep(1)
+    else:
+        msg = ('Command execution failed: "{cmd}"\n{stderr}'.
+               format(cmd=cmd, stderr=stderr) if message is None else message)
         raise RuntimeError(msg)
 
     return stdout, stderr