Create unified service operations

- Add ability to dump the logs from service unit (since last start)
- Cleanup service code to more streamlined way.

Change-Id: I6e332dce3229727dd36fd72a8ecfb9fc9efe4d63
Signed-off-by: Peter Mikus <pmikus@cisco.com>
Signed-off-by: Jan Gelety <jgelety@cisco.com>

4 files changed, 213 insertions, 50 deletions

diff --git a/resources/libraries/bash/dut_setup.sh b/resources/libraries/bash/dut_setup.sh
index 952b82bdc7..988ffe1ce2 100644
--- a/resources/libraries/bash/dut_setup.sh
+++ b/resources/libraries/bash/dut_setup.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2016 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -38,6 +38,8 @@ cmd 'sudo -S service vpp restart'
 echo "[Command_desc] SLEEP for three seconds, so that VPP is up for sure"
 cmd 'sleep 3'
 
+cmd "sudo journalctl --no-pager --unit=vpp --since=\"$(echo `systemctl show -p ActiveEnterTimestamp vpp` | awk '{print $2 $3}')\""
+
 cmd 'cat /proc/meminfo'
 
 cmd 'free -m'
@@ -48,14 +50,8 @@ cmd 'sudo dmidecode | grep UUID'
 
 cmd 'lspci -Dnn'
 
-if [ -f "/etc/redhat-release" ]; then
-    cmd 'tail -n 100 /var/log/messages'
-else
-    cmd 'tail -n 100 /var/log/syslog'
-fi
-
 echo "[Command_desc] Adding dpdk-input trace"
 cmd 'sudo vpp_api_test <<< "exec trace add dpdk-input 100"'
 
 echo "[Command_desc] Adding vhost-user-input trace"
-cmd 'sudo vpp_api_test <<< "exec trace add vhost-user-input 100"'
\ No newline at end of file
+cmd 'sudo vpp_api_test <<< "exec trace add vhost-user-input 100"'
diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index e0479cfb60..4cb4ad4040 100644
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -24,6 +24,63 @@ from resources.libraries.python.VPPUtil import VPPUtil
 
 class DUTSetup(object):
     """Contains methods for setting up DUTs."""
+
+    @staticmethod
+    def get_service_logs(node, service):
+        """Get specific service unit logs by journalctl from node.
+
+        :param node: Node in the topology.
+        :param service: Service unit name.
+        :type node: dict
+        :type service: str
+        """
+        ssh = SSH()
+        ssh.connect(node)
+        ret_code, _, _ = \
+            ssh.exec_command_sudo('journalctl --no-pager --unit={name} '
+                                  '--since="$(echo `systemctl show -p '
+                                  'ActiveEnterTimestamp {name}` | '
+                                  'awk \'{{print $2 $3}}\')"'.
+                                  format(name=service))
+        if int(ret_code) != 0:
+            raise RuntimeError('DUT {host} failed to get logs from unit {name}'.
+                               format(host=node['host'], name=service))
+
+    @staticmethod
+    def get_service_logs_on_all_duts(nodes, service):
+        """Get specific service unit logs by journalctl from all DUTs.
+
+        :param nodes: Nodes in the topology.
+        :param service: Service unit name.
+        :type nodes: dict
+        :type service: str
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.get_service_logs(node, service)
+
+    @staticmethod
+    def start_service(node, service):
+        """Start up the named service on node.
+
+        :param node: Node in the topology.
+        :param service: Service unit name.
+        :type node: dict
+        :type service: str
+        """
+        ssh = SSH()
+        ssh.connect(node)
+        # We are doing restart. With this we do not care if service
+        # was running or not.
+        ret_code, _, _ = \
+            ssh.exec_command_sudo('service {name} restart'.
+                                  format(name=service), timeout=120)
+        if int(ret_code) != 0:
+            raise RuntimeError('DUT {host} failed to start service {name}'.
+                               format(host=node['host'], name=service))
+
+        DUTSetup.get_service_logs(node, service)
+
     @staticmethod
     def start_vpp_service_on_all_duts(nodes):
         """Start up the VPP service on all nodes.
@@ -31,17 +88,9 @@ class DUTSetup(object):
         :param nodes: Nodes in the topology.
         :type nodes: dict
         """
-        ssh = SSH()
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
-                ssh.connect(node)
-                (ret_code, stdout, stderr) = \
-                    ssh.exec_command_sudo('service vpp restart', timeout=120)
-                if int(ret_code) != 0:
-                    logger.debug('stdout: {0}'.format(stdout))
-                    logger.debug('stderr: {0}'.format(stderr))
-                    raise Exception('DUT {0} failed to start VPP service'.
-                                    format(node['host']))
+                DUTSetup.start_service(node, Constants.VPP_UNIT)
 
     @staticmethod
     def vpp_show_version_verbose(node):
@@ -53,6 +102,12 @@ class DUTSetup(object):
         vat = VatExecutor()
         vat.execute_script("show_version_verbose.vat", node, json_out=False)
 
+        try:
+            vat.script_should_have_passed()
+        except AssertionError:
+            raise RuntimeError('Failed to get VPP version on host: {name}'.
+                               format(name=node['host']))
+
     @staticmethod
     def show_vpp_version_on_all_duts(nodes):
         """Show VPP version verbose on all DUTs.
@@ -65,6 +120,22 @@ class DUTSetup(object):
                 DUTSetup.vpp_show_version_verbose(node)
 
     @staticmethod
+    def vpp_show_interfaces(node):
+        """Run "show interface" CLI command.
+
+        :param node: Node to run command on.
+        :type node: dict
+        """
+        vat = VatExecutor()
+        vat.execute_script("show_interface.vat", node, json_out=False)
+
+        try:
+            vat.script_should_have_passed()
+        except AssertionError:
+            raise RuntimeError('Failed to get VPP interfaces on host: {name}'.
+                               format(name=node['host']))
+
+    @staticmethod
     def vpp_api_trace_save(node):
         """Run "api trace save" CLI command.
 
@@ -103,17 +174,15 @@ class DUTSetup(object):
         ssh = SSH()
         ssh.connect(node)
 
-        (ret_code, stdout, stderr) = \
+        ret_code, _, _ = \
             ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'.
                              format(Constants.REMOTE_FW_DIR,
                                     Constants.RESOURCES_LIB_SH), timeout=120)
         logger.trace(stdout)
         logger.trace(stderr)
         if int(ret_code) != 0:
-            logger.debug('DUT {0} setup script failed: "{1}"'.
-                         format(node['host'], stdout + stderr))
-            raise Exception('DUT test setup script failed at node {}'.
-                            format(node['host']))
+            raise RuntimeError('DUT test setup script failed at node {name}'.
+                               format(name=node['host']))
 
     @staticmethod
     def get_vpp_pid(node):
@@ -429,3 +498,106 @@ class DUTSetup(object):
         vat = VatExecutor()
         vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
         vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
+
+    @staticmethod
+    def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
+        """Install VPP on all DUT nodes.
+
+        :param nodes: Nodes in the topology.
+        :param vpp_pkg_dir: Path to directory where VPP packages are stored.
+        :param vpp_rpm_pkgs: List of VPP rpm packages to be installed.
+        :param vpp_deb_pkgs: List of VPP deb packages to be installed.
+        :type nodes: dict
+        :type vpp_pkg_dir: str
+        :type vpp_rpm_pkgs: list
+        :type vpp_deb_pkgs: list
+        :raises: RuntimeError if failed to remove or install VPP
+        """
+
+        logger.debug("Installing VPP")
+
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                logger.debug("Installing VPP on node {0}".format(node['host']))
+
+                ssh = SSH()
+                ssh.connect(node)
+
+                cmd = "[[ -f /etc/redhat-release ]]"
+                return_code, _, _ = ssh.exec_command(cmd)
+                if int(return_code) == 0:
+                    # workaroud - uninstall existing vpp installation until
+                    # start-testcase script is updated on all virl servers
+                    rpm_pkgs_remove = "vpp*"
+                    cmd_u = 'yum -y remove "{0}"'.format(rpm_pkgs_remove)
+                    r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
+                    if int(r_rcode) != 0:
+                        raise RuntimeError('Failed to remove previous VPP'
+                                           'installation on host {0}:\n{1}'
+                                           .format(node['host'], r_err))
+
+                    rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg)
+                                             for pkg in vpp_rpm_pkgs) + "*.rpm"
+                    cmd_i = "rpm -ivh {0}".format(rpm_pkgs)
+                    ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
+                    if int(ret_code) != 0:
+                        raise RuntimeError('Failed to install VPP on host {0}:'
+                                           '\n{1}'.format(node['host'], err))
+                    else:
+                        ssh.exec_command_sudo("rpm -qai vpp*")
+                        logger.info("VPP installed on node {0}".
+                                    format(node['host']))
+                else:
+                    # workaroud - uninstall existing vpp installation until
+                    # start-testcase script is updated on all virl servers
+                    deb_pkgs_remove = "vpp*"
+                    cmd_u = 'apt-get purge -y "{0}"'.format(deb_pkgs_remove)
+                    r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
+                    if int(r_rcode) != 0:
+                        raise RuntimeError('Failed to remove previous VPP'
+                                           'installation on host {0}:\n{1}'
+                                           .format(node['host'], r_err))
+                    deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg)
+                                             for pkg in vpp_deb_pkgs) + "*.deb"
+                    cmd_i = "dpkg -i --force-all {0}".format(deb_pkgs)
+                    ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
+                    if int(ret_code) != 0:
+                        raise RuntimeError('Failed to install VPP on host {0}:'
+                                           '\n{1}'.format(node['host'], err))
+                    else:
+                        ssh.exec_command_sudo("dpkg -l | grep vpp")
+                        logger.info("VPP installed on node {0}".
+                                    format(node['host']))
+
+                ssh.disconnect(node)
+
+    @staticmethod
+    def verify_vpp_on_all_duts(nodes):
+        """Verify that VPP is installed on all DUT nodes.
+
+        :param nodes: Nodes in the topology.
+        :type nodes: dict
+        """
+
+        logger.debug("Verify VPP on all DUTs")
+
+        DUTSetup.start_vpp_service_on_all_duts(nodes)
+
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.verify_vpp_on_dut(node)
+
+    @staticmethod
+    def verify_vpp_on_dut(node):
+        """Verify that VPP is installed on DUT node.
+
+        :param node: DUT node.
+        :type node: dict
+        :raises: RuntimeError if failed to restart VPP, get VPP version or
+        get VPP interfaces
+        """
+
+        logger.debug("Verify VPP on node {0}".format(node['host']))
+
+        DUTSetup.vpp_show_version_verbose(node)
+        DUTSetup.vpp_show_interfaces(node)
diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py
index 88776fe3f1..d3e598de7c 100644
--- a/resources/libraries/python/VppConfigGenerator.py
+++ b/resources/libraries/python/VppConfigGenerator.py
@@ -17,6 +17,8 @@ import re
 import time
 
 from resources.libraries.python.ssh import SSH
+from resources.libraries.python.constants import Constants
+from resources.libraries.python.DUTSetup import DUTSetup
 from resources.libraries.python.topology import NodeType
 from resources.libraries.python.topology import Topology
 
@@ -513,48 +515,38 @@ class VppConfigGenerator(object):
             filename = self._vpp_startup_conf
 
         if self._vpp_startup_conf_backup is not None:
-            (ret, _, _) = \
-                ssh.exec_command('sudo cp {0} {1}'.
-                                 format(self._vpp_startup_conf,
-                                        self._vpp_startup_conf_backup))
+            ret, _, _ = \
+                ssh.exec_command('sudo cp {src} {dest}'.
+                                 format(src=self._vpp_startup_conf,
+                                        dest=self._vpp_startup_conf_backup))
             if ret != 0:
-                raise RuntimeError('Backup of config file failed on node {}'.
-                                   format(self._hostname))
+                raise RuntimeError('Backup of config file failed on node '
+                                   '{name}'.format(name=self._hostname))
 
-        (ret, _, _) = \
+        ret, _, _ = \
             ssh.exec_command('echo "{config}" | sudo tee {filename}'.
                              format(config=self._vpp_config,
                                     filename=filename))
 
         if ret != 0:
-            raise RuntimeError('Writing config file failed to node {}'.
-                               format(self._hostname))
+            raise RuntimeError('Writing config file failed to node {name}'.
+                               format(name=self._hostname))
 
         if restart_vpp:
-            # Instead of restarting, we'll do separate start and stop
-            # actions. This way we don't care whether VPP was running
-            # to begin with.
-            ssh.exec_command('sudo service {} stop'
-                             .format(self._vpp_service_name))
-            (ret, _, _) = \
-                ssh.exec_command('sudo service {} start'
-                                 .format(self._vpp_service_name))
-            if ret != 0:
-                raise RuntimeError('Restarting VPP failed on node {}'.
-                                   format(self._hostname))
+            DUTSetup.start_service(self._node, Constants.VPP_UNIT)
 
             # Sleep <waittime> seconds, up to <retry> times,
             # and verify if VPP is running.
             for _ in range(retries):
                 time.sleep(waittime)
-                (ret, stdout, _) = \
+                ret, stdout, _ = \
                     ssh.exec_command('echo show hardware-interfaces | '
                                      'nc 0 5002 || echo "VPP not yet running"')
                 if ret == 0 and stdout != 'VPP not yet running':
                     break
             else:
-                raise RuntimeError('VPP failed to restart on node {}'.
-                                   format(self._hostname))
+                raise RuntimeError('VPP failed to restart on node {name}'.
+                                   format(name=self._hostname))
 
     def restore_config(self):
         """Restore VPP startup.conf from backup.
@@ -565,9 +557,9 @@ class VppConfigGenerator(object):
         ssh = SSH()
         ssh.connect(self._node)
 
-        (ret, _, _) = ssh.exec_command('sudo cp {0} {1}'.
-                                       format(self._vpp_startup_conf_backup,
-                                              self._vpp_startup_conf))
+        ret, _, _ = ssh.exec_command('sudo cp {src} {dest}'.
+                                     format(src=self._vpp_startup_conf_backup,
+                                            dest=self._vpp_startup_conf))
         if ret != 0:
-            raise RuntimeError('Restoration of config file failed on node {}'.
-                               format(self._hostname))
+            raise RuntimeError('Restoration of config file failed on node '
+                               '{name}'.format(name=self._hostname))
diff --git a/resources/libraries/python/constants.py b/resources/libraries/python/constants.py
index c3f1551c55..9507e66d85 100644
--- a/resources/libraries/python/constants.py
+++ b/resources/libraries/python/constants.py
@@ -29,6 +29,9 @@ class Constants(object):
     # OpenVPP VAT binary name
     VAT_BIN_NAME = 'vpp_api_test'
 
+    # VPP service unit name
+    VPP_UNIT = 'vpp'
+
     # QEMU version to install
     QEMU_INSTALL_VERSION = 'qemu-2.5.0'