CSIT-1291 Improve service handling with container detection

Change-Id: Iaeac10676f46e0267041afee927bc6e3201146b2
Signed-off-by: Peter Mikus <pmikus@cisco.com>

2 files changed, 229 insertions, 187 deletions

diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index 84862d4167..7885ad0a16 100644
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -15,11 +15,9 @@
 
 from robot.api import logger
 
-from resources.libraries.python.topology import NodeType, Topology
-from resources.libraries.python.ssh import SSH, exec_cmd_no_error
 from resources.libraries.python.constants import Constants
-from resources.libraries.python.VatExecutor import VatExecutor
-from resources.libraries.python.VPPUtil import VPPUtil
+from resources.libraries.python.ssh import SSH, exec_cmd_no_error
+from resources.libraries.python.topology import NodeType, Topology
 
 
 class DUTSetup(object):
@@ -27,28 +25,30 @@ class DUTSetup(object):
 
     @staticmethod
     def get_service_logs(node, service):
-        """Get specific service unit logs by journalctl from node.
+        """Get specific service unit logs from node.
 
         :param node: Node in the topology.
         :param service: Service unit name.
         :type node: dict
         :type service: str
         """
-        ssh = SSH()
-        ssh.connect(node)
-        ret_code, _, _ = \
-            ssh.exec_command_sudo('journalctl --no-pager --unit={name} '
-                                  '--since="$(echo `systemctl show -p '
-                                  'ActiveEnterTimestamp {name}` | '
-                                  'awk \'{{print $2 $3}}\')"'.
-                                  format(name=service))
-        if int(ret_code):
-            raise RuntimeError('DUT {host} failed to get logs from unit {name}'.
-                               format(host=node['host'], name=service))
+        if DUTSetup.running_in_container(node):
+            command = 'echo $(< /var/log/supervisord.log)'
+        else:
+            command = ('journalctl --no-pager --unit={name} '
+                       '--since="$(echo `systemctl show -p '
+                       'ActiveEnterTimestamp {name}` | '
+                       'awk \'{{print $2 $3}}\')"'.
+                       format(name=service))
+        message = 'Node {host} failed to get logs from unit {name}'.\
+            format(host=node['host'], name=service)
+
+        exec_cmd_no_error(node, command, timeout=30, sudo=True,
+                          message=message)
 
     @staticmethod
     def get_service_logs_on_all_duts(nodes, service):
-        """Get specific service unit logs by journalctl from all DUTs.
+        """Get specific service unit logs from all DUTs.
 
         :param nodes: Nodes in the topology.
         :param service: Service unit name.
@@ -68,99 +68,62 @@ class DUTSetup(object):
         :type node: dict
         :type service: str
         """
-        ssh = SSH()
-        ssh.connect(node)
-        # We are doing restart. With this we do not care if service
-        # was running or not.
-        ret_code, _, _ = \
-            ssh.exec_command_sudo('service {name} restart'.
-                                  format(name=service), timeout=120)
-        if int(ret_code):
-            raise RuntimeError('DUT {host} failed to start service {name}'.
-                               format(host=node['host'], name=service))
-
-        DUTSetup.get_service_logs(node, service)
+        if DUTSetup.running_in_container(node):
+            command = 'supervisorctl restart {name}'.format(name=service)
+        else:
+            command = 'service {name} restart'.format(name=service)
+        message = 'Node {host} failed to start service {name}'.\
+            format(host=node['host'], name=service)
 
-    @staticmethod
-    def start_vpp_service_on_all_duts(nodes):
-        """Start up the VPP service on all nodes.
+        exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
 
-        :param nodes: Nodes in the topology.
-        :type nodes: dict
-        """
-        for node in nodes.values():
-            if node['type'] == NodeType.DUT:
-                DUTSetup.start_service(node, Constants.VPP_UNIT)
+        DUTSetup.get_service_logs(node, service)
 
     @staticmethod
-    def vpp_show_version_verbose(node):
-        """Run "show version verbose" CLI command.
+    def start_service_on_all_duts(nodes, service):
+        """Start up the named service on all DUTs.
 
-        :param node: Node to run command on.
+        :param node: Nodes in the topology.
+        :param service: Service unit name.
         :type node: dict
-        """
-        vat = VatExecutor()
-        vat.execute_script("show_version_verbose.vat", node, json_out=False)
-
-        try:
-            vat.script_should_have_passed()
-        except AssertionError:
-            raise RuntimeError('Failed to get VPP version on host: {name}'.
-                               format(name=node['host']))
-
-    @staticmethod
-    def show_vpp_version_on_all_duts(nodes):
-        """Show VPP version verbose on all DUTs.
-
-        :param nodes: VPP nodes
-        :type nodes: dict
+        :type service: str
         """
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
-                DUTSetup.vpp_show_version_verbose(node)
+                DUTSetup.start_service(node, service)
 
     @staticmethod
-    def vpp_show_interfaces(node):
-        """Run "show interface" CLI command.
+    def stop_service(node, service):
+        """Stop the named service on node.
 
-        :param node: Node to run command on.
+        :param node: Node in the topology.
+        :param service: Service unit name.
         :type node: dict
+        :type service: str
         """
-        vat = VatExecutor()
-        vat.execute_script("show_interface.vat", node, json_out=False)
-
-        try:
-            vat.script_should_have_passed()
-        except AssertionError:
-            raise RuntimeError('Failed to get VPP interfaces on host: {name}'.
-                               format(name=node['host']))
+        if DUTSetup.running_in_container(node):
+            command = 'supervisorctl stop {name}'.format(name=service)
+        else:
+            command = 'service {name} stop'.format(name=service)
+        message = 'Node {host} failed to stop service {name}'.\
+            format(host=node['host'], name=service)
 
-    @staticmethod
-    def vpp_api_trace_save(node):
-        """Run "api trace save" CLI command.
+        exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
 
-        :param node: Node to run command on.
-        :type node: dict
-        """
-        vat = VatExecutor()
-        vat.execute_script("api_trace_save.vat", node, json_out=False)
+        DUTSetup.get_service_logs(node, service)
 
     @staticmethod
-    def vpp_api_trace_dump(node):
-        """Run "api trace custom-dump" CLI command.
+    def stop_service_on_all_duts(nodes, service):
+        """Stop the named service on all DUTs.
 
-        :param node: Node to run command on.
+        :param node: Nodes in the topology.
+        :param service: Service unit name.
         :type node: dict
+        :type service: str
         """
-        vat = VatExecutor()
-        vat.execute_script("api_trace_dump.vat", node, json_out=False)
-
-    @staticmethod
-    def setup_all_duts(nodes):
-        """Prepare all DUTs in given topology for test execution."""
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
-                DUTSetup.setup_dut(node)
+                DUTSetup.stop_service(node, service)
 
     @staticmethod
     def setup_dut(node):
@@ -171,16 +134,24 @@ class DUTSetup(object):
 
         :raises Exception: If the DUT setup fails.
         """
-        ssh = SSH()
-        ssh.connect(node)
+        command = 'bash {0}/{1}/dut_setup.sh'.\
+            format(Constants.REMOTE_FW_DIR, Constants.RESOURCES_LIB_SH)
+        message = 'DUT test setup script failed at node {name}'.\
+            format(name=node['host'])
 
-        ret_code, _, _ = \
-            ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'.
-                             format(Constants.REMOTE_FW_DIR,
-                                    Constants.RESOURCES_LIB_SH), timeout=120)
-        if int(ret_code):
-            raise RuntimeError('DUT test setup script failed at node {name}'.
-                               format(name=node['host']))
+        exec_cmd_no_error(node, command, timeout=120, sudo=True,
+                          message=message)
+
+    @staticmethod
+    def setup_all_duts(nodes):
+        """Run script over SSH to setup all DUT nodes.
+
+        :param nodes: Topology nodes.
+        :type nodes: dict
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.setup_dut(node)
 
     @staticmethod
     def get_vpp_pid(node):
@@ -192,7 +163,6 @@ class DUTSetup(object):
         :rtype: int
         :raises RuntimeError: If it is not possible to get the PID.
         """
-
         ssh = SSH()
         ssh.connect(node)
 
@@ -230,7 +200,6 @@ class DUTSetup(object):
         :returns: PIDs
         :rtype: dict
         """
-
         pids = dict()
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
@@ -238,17 +207,6 @@ class DUTSetup(object):
         return pids
 
     @staticmethod
-    def vpp_show_crypto_device_mapping(node):
-        """Run "show crypto device mapping" CLI command.
-
-        :param node: Node to run command on.
-        :type node: dict
-        """
-        vat = VatExecutor()
-        vat.execute_script("show_crypto_device_mapping.vat", node,
-                           json_out=False)
-
-    @staticmethod
     def crypto_device_verify(node, force_init=False, numvfs=32):
         """Verify if Crypto QAT device virtual functions are initialized on all
         DUTs. If parameter force initialization is set to True, then try to
@@ -292,7 +250,7 @@ class DUTSetup(object):
         DUTSetup.verify_kernel_module(node, 'qat_dh895xcc', force_load=True)
 
         # Stop VPP to prevent deadlock.
-        VPPUtil.stop_vpp_service(node)
+        DUTSetup.stop_service(node, Constants.VPP_UNIT)
 
         current_driver = DUTSetup.get_pci_dev_driver(
             node, pci_addr.replace(':', r'\:'))
@@ -628,30 +586,6 @@ class DUTSetup(object):
         exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
 
     @staticmethod
-    def vpp_enable_traces_on_all_duts(nodes):
-        """Enable vpp packet traces on all DUTs in the given topology.
-
-        :param nodes: Nodes in the topology.
-        :type nodes: dict
-        """
-        for node in nodes.values():
-            if node['type'] == NodeType.DUT:
-                DUTSetup.vpp_enable_traces_on_dut(node)
-
-    @staticmethod
-    def vpp_enable_traces_on_dut(node):
-        """Enable vpp packet traces on the DUT node.
-
-        :param node: DUT node to set up.
-        :type node: dict
-        """
-
-        vat = VatExecutor()
-        vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
-        vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
-        vat.execute_script("enable_memif_traces.vat", node, json_out=False)
-
-    @staticmethod
     def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
         """Install VPP on all DUT nodes.
 
@@ -665,9 +599,6 @@ class DUTSetup(object):
         :type vpp_deb_pkgs: list
         :raises RuntimeError: If failed to remove or install VPP.
         """
-
-        logger.debug("Installing VPP")
-
         for node in nodes.values():
             if node['type'] == NodeType.DUT:
                 logger.debug("Installing VPP on node {0}".format(node['host']))
@@ -724,36 +655,23 @@ class DUTSetup(object):
                 ssh.disconnect(node)
 
     @staticmethod
-    def verify_vpp_on_dut(node):
-        """Verify that VPP is installed on DUT node.
+    def running_in_container(node):
+        """This method tests if topology node is running inside container.
 
-        :param node: DUT node.
+        :param node: Topology node.
         :type node: dict
-        :raises RuntimeError: If failed to restart VPP, get VPP version
-            or get VPP interfaces.
+        :returns: True if running in docker container, false if not or failed
+        to detect.
+        :rtype: bool
         """
-
-        logger.debug("Verify VPP on node {0}".format(node['host']))
-
-        DUTSetup.vpp_show_version_verbose(node)
-        DUTSetup.vpp_show_interfaces(node)
-
-    @staticmethod
-    def verify_vpp_on_all_duts(nodes):
-        """Verify that VPP is installed on all DUT nodes.
-
-        :param nodes: Nodes in the topology.
-        :type nodes: dict
-        """
-
-        logger.debug("Verify VPP on all DUTs")
-
-        DUTSetup.start_vpp_service_on_all_duts(nodes)
-
-        for node in nodes.values():
-            if node['type'] == NodeType.DUT:
-                DUTSetup.verify_vpp_on_dut(node)
-
+        command = "fgrep docker /proc/1/cgroup"
+        message = 'Failed to get cgroup settings.'
+        try:
+            exec_cmd_no_error(node, command, timeout=30, sudo=False,
+                              message=message)
+        except RuntimeError:
+            return False
+        return True
 
     @staticmethod
     def get_huge_page_size(node):
diff --git a/resources/libraries/python/VPPUtil.py b/resources/libraries/python/VPPUtil.py
index 3a0148a481..de3caad916 100644
--- a/resources/libraries/python/VPPUtil.py
+++ b/resources/libraries/python/VPPUtil.py
@@ -11,8 +11,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""VPP util library"""
-from resources.libraries.python.ssh import SSH
+"""VPP util library."""
+from resources.libraries.python.constants import Constants
+from resources.libraries.python.DUTSetup import DUTSetup
+from resources.libraries.python.ssh import exec_cmd_no_error
+from resources.libraries.python.topology import NodeType
+from resources.libraries.python.VatExecutor import VatExecutor
 
 
 class VPPUtil(object):
@@ -42,10 +46,30 @@ class VPPUtil(object):
         if additional_cmds:
             for cmd in additional_cmds:
                 def_setting_tb_displayed['Custom Setting: {}'.format(cmd)] = cmd
-        ssh = SSH()
-        ssh.connect(node)
-        for _, value in def_setting_tb_displayed.items():
-            ssh.exec_command_sudo('vppctl sh {}'.format(value))
+
+        for _, cmd in def_setting_tb_displayed.items():
+            command = 'vppctl sh {cmd}'.format(cmd=cmd)
+            exec_cmd_no_error(node, command, timeout=30, sudo=True)
+
+    @staticmethod
+    def start_vpp_service(node):
+        """Start VPP service on the specified node.
+
+        :param node: VPP node.
+        :type node: dict
+        :raises RuntimeError: If VPP service fails to start.
+        """
+        DUTSetup.start_service(node, Constants.VPP_UNIT)
+        DUTSetup.get_service_logs(node, Constants.VPP_UNIT)
+
+    @staticmethod
+    def start_vpp_service_on_all_duts(nodes):
+        """Start up the VPP service on all nodes.
+
+        :param nodes: Nodes in the topology.
+        :type nodes: dict
+        """
+        DUTSetup.start_service_on_all_duts(nodes, Constants.VPP_UNIT)
 
     @staticmethod
     def stop_vpp_service(node):
@@ -53,28 +77,128 @@ class VPPUtil(object):
 
         :param node: VPP node.
         :type node: dict
-        :raises RuntimeError: If VPP fails to stop.
+        :raises RuntimeError: If VPP service fails to stop.
+        """
+        DUTSetup.stop_service(node, Constants.VPP_UNIT)
+
+    @staticmethod
+    def verify_vpp_on_dut(node):
+        """Verify that VPP is installed on DUT node.
+
+        :param node: DUT node.
+        :type node: dict
+        :raises RuntimeError: If failed to restart VPP, get VPP version
+            or get VPP interfaces.
         """
+        VPPUtil.vpp_show_version_verbose(node)
+        VPPUtil.vpp_show_interfaces(node)
 
-        ssh = SSH()
-        ssh.connect(node)
-        cmd = "service vpp stop"
-        ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=80)
-        if int(ret_code) != 0:
-            raise RuntimeError("VPP service did not shut down gracefully.")
+    @staticmethod
+    def verify_vpp_on_all_duts(nodes):
+        """Verify that VPP is installed on all DUT nodes.
+
+        :param nodes: Nodes in the topology.
+        :type nodes: dict
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                DUTSetup.start_service(node, Constants.VPP_UNIT)
+                VPPUtil.vpp_show_version_verbose(node)
+                VPPUtil.vpp_show_interfaces(node)
 
     @staticmethod
-    def start_vpp_service(node):
-        """start VPP service on the specified node.
+    def vpp_show_version_verbose(node):
+        """Run "show version verbose" CLI command.
 
-        :param node: VPP node.
+        :param node: Node to run command on.
+        :type node: dict
+        """
+        vat = VatExecutor()
+        vat.execute_script("show_version_verbose.vat", node, json_out=False)
+
+        try:
+            vat.script_should_have_passed()
+        except AssertionError:
+            raise RuntimeError('Failed to get VPP version on host: {name}'.
+                               format(name=node['host']))
+
+    @staticmethod
+    def show_vpp_version_on_all_duts(nodes):
+        """Show VPP version verbose on all DUTs.
+
+        :param nodes: VPP nodes.
+        :type nodes: dict
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                VPPUtil.vpp_show_version_verbose(node)
+
+    @staticmethod
+    def vpp_show_interfaces(node):
+        """Run "show interface" CLI command.
+
+        :param node: Node to run command on.
+        :type node: dict
+        """
+        vat = VatExecutor()
+        vat.execute_script("show_interface.vat", node, json_out=False)
+
+        try:
+            vat.script_should_have_passed()
+        except AssertionError:
+            raise RuntimeError('Failed to get VPP interfaces on host: {name}'.
+                               format(name=node['host']))
+
+    @staticmethod
+    def vpp_show_crypto_device_mapping(node):
+        """Run "show crypto device mapping" CLI command.
+
+        :param node: Node to run command on.
+        :type node: dict
+        """
+        vat = VatExecutor()
+        vat.execute_script("show_crypto_device_mapping.vat", node,
+                           json_out=False)
+
+    @staticmethod
+    def vpp_api_trace_dump(node):
+        """Run "api trace custom-dump" CLI command.
+
+        :param node: Node to run command on.
+        :type node: dict
+        """
+        vat = VatExecutor()
+        vat.execute_script("api_trace_dump.vat", node, json_out=False)
+
+    @staticmethod
+    def vpp_api_trace_save(node):
+        """Run "api trace save" CLI command.
+
+        :param node: Node to run command on.
+        :type node: dict
+        """
+        vat = VatExecutor()
+        vat.execute_script("api_trace_save.vat", node, json_out=False)
+
+    @staticmethod
+    def vpp_enable_traces_on_dut(node):
+        """Enable vpp packet traces on the DUT node.
+
+        :param node: DUT node to set up.
         :type node: dict
-        :raises RuntimeError: If VPP fails to start.
         """
+        vat = VatExecutor()
+        vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
+        vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
+        vat.execute_script("enable_memif_traces.vat", node, json_out=False)
 
-        ssh = SSH()
-        ssh.connect(node)
-        cmd = "service vpp start"
-        ret_code, _, _ = ssh.exec_command_sudo(cmd)
-        if int(ret_code) != 0:
-            raise RuntimeError("VPP service did not start.")
+    @staticmethod
+    def vpp_enable_traces_on_all_duts(nodes):
+        """Enable vpp packet traces on all DUTs in the given topology.
+
+        :param nodes: Nodes in the topology.
+        :type nodes: dict
+        """
+        for node in nodes.values():
+            if node['type'] == NodeType.DUT:
+                VPPUtil.vpp_enable_traces_on_dut(node)