Create unified service operations

- Add ability to dump the logs from service unit (since last start)
- Cleanup service code to more streamlined way.

Change-Id: I6e332dce3229727dd36fd72a8ecfb9fc9efe4d63
Signed-off-by: Peter Mikus <pmikus@cisco.com>

4 files changed, 96 insertions, 58 deletions

diff --git a/resources/libraries/bash/dut_setup.sh b/resources/libraries/bash/dut_setup.sh
index 952b82bdc7..988ffe1ce2 100644
--- a/resources/libraries/bash/dut_setup.sh
+++ b/resources/libraries/bash/dut_setup.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2016 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -38,6 +38,8 @@ cmd 'sudo -S service vpp restart'
 echo "[Command_desc] SLEEP for three seconds, so that VPP is up for sure"
 cmd 'sleep 3'
 
+cmd "sudo journalctl --no-pager --unit=vpp --since=\"$(echo `systemctl show -p ActiveEnterTimestamp vpp` | awk '{print $2 $3}')\""
+
 cmd 'cat /proc/meminfo'
 
 cmd 'free -m'
@@ -48,14 +50,8 @@ cmd 'sudo dmidecode | grep UUID'
 
 cmd 'lspci -Dnn'
 
-if [ -f "/etc/redhat-release" ]; then
-    cmd 'tail -n 100 /var/log/messages'
-else
-    cmd 'tail -n 100 /var/log/syslog'
-fi
-
 echo "[Command_desc] Adding dpdk-input trace"
 cmd 'sudo vpp_api_test <<< "exec trace add dpdk-input 100"'
 
 echo "[Command_desc] Adding vhost-user-input trace"
-cmd 'sudo vpp_api_test <<< "exec trace add vhost-user-input 100"'
\ No newline at end of file
+cmd 'sudo vpp_api_test <<< "exec trace add vhost-user-input 100"'
diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index 5b70bd6b5a..2741eaaf1f 100644
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -13,8 +13,6 @@
 
 """DUT setup library."""
 
-import os
-
 from robot.api import logger
 
 from resources.libraries.python.topology import NodeType, Topology
@@ -26,6 +24,63 @@ from resources.libraries.python.VPPUtil import VPPUtil
 
 class DUTSetup(object):
     """Contains methods for setting up DUTs."""
+
+    @staticmethod
+    def get_service_logs(node, service):
+        """Get specific service unit logs by journalctl from node.
+
+        :param node: Node in the topology.
+        :param service: Service unit name.
+        :type node: dict
+        :type service: str
+        """
+        ssh = SSH()
+        ssh.connect(node)
+        ret_code, _, _ = \
+            ssh.exec_command_sudo('journalctl --no-pager --unit={name} '
+                                  '--since="$(echo `systemctl show -p '
+                                  'ActiveEnterTimestamp {name}` | '
+                                  'awk \'{{print $2 $3}}\')"'.
+                                  format(name=service))
+        if int(ret_code) != 0:
+            raise RuntimeError('DUT {host} failed to get logs from unit {name}'.
+                               format(host=node['host'], name=service))
+
+    @staticmethod
+    def get_service_logs_on_all_duts(nodes, service):
+        """Get specific service unit logs by journalctl from all DUTs.
+
+        :param nodes: Nodes in the topology.
+        :param service: Service unit name.
+        :type nodes: dict
+        :type service: str
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.get_service_logs(node, service)
+
+    @staticmethod
+    def start_service(node, service):
+        """Start up the named service on node.
+
+        :param node: Node in the topology.
+        :param service: Service unit name.
+        :type node: dict
+        :type service: str
+        """
+        ssh = SSH()
+        ssh.connect(node)
+        # We are doing restart. With this we do not care if service
+        # was running or not.
+        ret_code, _, _ = \
+            ssh.exec_command_sudo('service {name} restart'.
+                                  format(name=service), timeout=120)
+        if int(ret_code) != 0:
+            raise RuntimeError('DUT {host} failed to start service {name}'.
+                               format(host=node['host'], name=service))
+
+        DUTSetup.get_service_logs(node, service)
+
     @staticmethod
     def start_vpp_service_on_all_duts(nodes):
         """Start up the VPP service on all nodes.
@@ -33,15 +88,9 @@ class DUTSetup(object):
         :param nodes: Nodes in the topology.
         :type nodes: dict
         """
-        ssh = SSH()
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
-                ssh.connect(node)
-                (ret_code, stdout, stderr) = \
-                    ssh.exec_command_sudo('service vpp restart', timeout=120)
-                if int(ret_code) != 0:
-                    raise Exception('DUT {0} failed to start VPP service'.
-                                    format(node['host']))
+                DUTSetup.start_service(node, Constants.VPP_UNIT)
 
     @staticmethod
     def vpp_show_version_verbose(node):
@@ -56,8 +105,8 @@ class DUTSetup(object):
         try:
             vat.script_should_have_passed()
         except AssertionError:
-            raise RuntimeError('Failed to get VPP version on host: {}'.
-                               format(node['host']))
+            raise RuntimeError('Failed to get VPP version on host: {name}'.
+                               format(name=node['host']))
 
     @staticmethod
     def show_vpp_version_on_all_duts(nodes):
@@ -83,8 +132,8 @@ class DUTSetup(object):
         try:
             vat.script_should_have_passed()
         except AssertionError:
-            raise RuntimeError('Failed to get VPP interfaces on host: {}'.
-                               format(node['host']))
+            raise RuntimeError('Failed to get VPP interfaces on host: {name}'.
+                               format(name=node['host']))
 
     @staticmethod
     def vpp_api_trace_save(node):
@@ -125,15 +174,13 @@ class DUTSetup(object):
         ssh = SSH()
         ssh.connect(node)
 
-        (ret_code, stdout, stderr) = \
+        ret_code, _, _ = \
             ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'.
                              format(Constants.REMOTE_FW_DIR,
                                     Constants.RESOURCES_LIB_SH), timeout=120)
         if int(ret_code) != 0:
-            logger.debug('DUT {0} setup script failed: "{1}"'.
-                         format(node['host'], stdout + stderr))
-            raise Exception('DUT test setup script failed at node {}'.
-                            format(node['host']))
+            raise RuntimeError('DUT test setup script failed at node {name}'.
+                               format(name=node['host']))
 
     @staticmethod
     def get_vpp_pid(node):
@@ -485,7 +532,7 @@ class DUTSetup(object):
                     if int(r_rcode) != 0:
                         raise RuntimeError('Failed to remove previous VPP'
                                            'installation on host {0}:\n{1}'
-                                           .format(node['host']), r_err)
+                                           .format(node['host'], r_err))
 
                     rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg)
                                              for pkg in vpp_rpm_pkgs) + "*.rpm"
@@ -493,7 +540,7 @@ class DUTSetup(object):
                     ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
                     if int(ret_code) != 0:
                         raise RuntimeError('Failed to install VPP on host {0}:'
-                                           '\n{1}'.format(node['host']), err)
+                                           '\n{1}'.format(node['host'], err))
                     else:
                         ssh.exec_command_sudo("rpm -qai vpp*")
                         logger.info("VPP installed on node {0}".
@@ -507,14 +554,14 @@ class DUTSetup(object):
                     if int(r_rcode) != 0:
                         raise RuntimeError('Failed to remove previous VPP'
                                            'installation on host {0}:\n{1}'
-                                           .format(node['host']), r_err)
+                                           .format(node['host'], r_err))
                     deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg)
                                              for pkg in vpp_deb_pkgs) + "*.deb"
                     cmd_i = "dpkg -i --force-all {0}".format(deb_pkgs)
                     ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
                     if int(ret_code) != 0:
                         raise RuntimeError('Failed to install VPP on host {0}:'
-                                           '\n{1}'.format(node['host']), err)
+                                           '\n{1}'.format(node['host'], err))
                     else:
                         ssh.exec_command_sudo("dpkg -l | grep vpp")
                         logger.info("VPP installed on node {0}".
diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py
index 4476cfa3ee..d740514994 100644
--- a/resources/libraries/python/VppConfigGenerator.py
+++ b/resources/libraries/python/VppConfigGenerator.py
@@ -17,6 +17,8 @@ import re
 import time
 
 from resources.libraries.python.ssh import SSH
+from resources.libraries.python.constants import Constants
+from resources.libraries.python.DUTSetup import DUTSetup
 from resources.libraries.python.topology import NodeType
 from resources.libraries.python.topology import Topology
 
@@ -503,48 +505,38 @@ class VppConfigGenerator(object):
             filename = self._vpp_startup_conf
 
         if self._vpp_startup_conf_backup is not None:
-            (ret, _, _) = \
-                ssh.exec_command('sudo cp {0} {1}'.
-                                 format(self._vpp_startup_conf,
-                                        self._vpp_startup_conf_backup))
+            ret, _, _ = \
+                ssh.exec_command('sudo cp {src} {dest}'.
+                                 format(src=self._vpp_startup_conf,
+                                        dest=self._vpp_startup_conf_backup))
             if ret != 0:
-                raise RuntimeError('Backup of config file failed on node {}'.
-                                   format(self._hostname))
+                raise RuntimeError('Backup of config file failed on node '
+                                   '{name}'.format(name=self._hostname))
 
-        (ret, _, _) = \
+        ret, _, _ = \
             ssh.exec_command('echo "{config}" | sudo tee {filename}'.
                              format(config=self._vpp_config,
                                     filename=filename))
 
         if ret != 0:
-            raise RuntimeError('Writing config file failed to node {}'.
-                               format(self._hostname))
+            raise RuntimeError('Writing config file failed to node {name}'.
+                               format(name=self._hostname))
 
         if restart_vpp:
-            # Instead of restarting, we'll do separate start and stop
-            # actions. This way we don't care whether VPP was running
-            # to begin with.
-            ssh.exec_command('sudo service {} stop'
-                             .format(self._vpp_service_name))
-            (ret, _, _) = \
-                ssh.exec_command('sudo service {} start'
-                                 .format(self._vpp_service_name))
-            if ret != 0:
-                raise RuntimeError('Restarting VPP failed on node {}'.
-                                   format(self._hostname))
+            DUTSetup.start_service(self._node, Constants.VPP_UNIT)
 
             # Sleep <waittime> seconds, up to <retry> times,
             # and verify if VPP is running.
             for _ in range(retries):
                 time.sleep(1)
-                (ret, stdout, _) = \
+                ret, stdout, _ = \
                     ssh.exec_command('echo show hardware-interfaces | '
                                      'nc 0 5002 || echo "VPP not yet running"')
                 if ret == 0 and 'VPP not yet running' not in stdout:
                     break
             else:
-                raise RuntimeError('VPP failed to restart on node {}'.
-                                   format(self._hostname))
+                raise RuntimeError('VPP failed to restart on node {name}'.
+                                   format(name=self._hostname))
 
     def restore_config(self):
         """Restore VPP startup.conf from backup.
@@ -555,9 +547,9 @@ class VppConfigGenerator(object):
         ssh = SSH()
         ssh.connect(self._node)
 
-        (ret, _, _) = ssh.exec_command('sudo cp {0} {1}'.
-                                       format(self._vpp_startup_conf_backup,
-                                              self._vpp_startup_conf))
+        ret, _, _ = ssh.exec_command('sudo cp {src} {dest}'.
+                                     format(src=self._vpp_startup_conf_backup,
+                                            dest=self._vpp_startup_conf))
         if ret != 0:
-            raise RuntimeError('Restoration of config file failed on node {}'.
-                               format(self._hostname))
+            raise RuntimeError('Restoration of config file failed on node '
+                               '{name}'.format(name=self._hostname))
diff --git a/resources/libraries/python/constants.py b/resources/libraries/python/constants.py
index c3f1551c55..9507e66d85 100644
--- a/resources/libraries/python/constants.py
+++ b/resources/libraries/python/constants.py
@@ -29,6 +29,9 @@ class Constants(object):
     # OpenVPP VAT binary name
     VAT_BIN_NAME = 'vpp_api_test'
 
+    # VPP service unit name
+    VPP_UNIT = 'vpp'
+
     # QEMU version to install
     QEMU_INSTALL_VERSION = 'qemu-2.5.0'