From 33d982ab77b174d8a9c5ff59679d5f664be17a05 Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Mon, 9 Apr 2018 15:08:41 +0200 Subject: Create unified service operations - Add ability to dump the logs from service unit (since last start) - Cleanup service code to more streamlined way. Change-Id: I6e332dce3229727dd36fd72a8ecfb9fc9efe4d63 Signed-off-by: Peter Mikus Signed-off-by: Jan Gelety --- resources/libraries/bash/dut_setup.sh | 12 +- resources/libraries/python/DUTSetup.py | 200 +++++++++++++++++++++-- resources/libraries/python/VppConfigGenerator.py | 48 +++--- resources/libraries/python/constants.py | 3 + 4 files changed, 213 insertions(+), 50 deletions(-) diff --git a/resources/libraries/bash/dut_setup.sh b/resources/libraries/bash/dut_setup.sh index 952b82bdc7..988ffe1ce2 100644 --- a/resources/libraries/bash/dut_setup.sh +++ b/resources/libraries/bash/dut_setup.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2016 Cisco and/or its affiliates. +# Copyright (c) 2018 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -38,6 +38,8 @@ cmd 'sudo -S service vpp restart' echo "[Command_desc] SLEEP for three seconds, so that VPP is up for sure" cmd 'sleep 3' +cmd "sudo journalctl --no-pager --unit=vpp --since=\"$(echo `systemctl show -p ActiveEnterTimestamp vpp` | awk '{print $2 $3}')\"" + cmd 'cat /proc/meminfo' cmd 'free -m' @@ -48,14 +50,8 @@ cmd 'sudo dmidecode | grep UUID' cmd 'lspci -Dnn' -if [ -f "/etc/redhat-release" ]; then - cmd 'tail -n 100 /var/log/messages' -else - cmd 'tail -n 100 /var/log/syslog' -fi - echo "[Command_desc] Adding dpdk-input trace" cmd 'sudo vpp_api_test <<< "exec trace add dpdk-input 100"' echo "[Command_desc] Adding vhost-user-input trace" -cmd 'sudo vpp_api_test <<< "exec trace add vhost-user-input 100"' \ No newline at end of file +cmd 'sudo vpp_api_test <<< "exec trace add vhost-user-input 100"' diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py index e0479cfb60..4cb4ad4040 100644 --- a/resources/libraries/python/DUTSetup.py +++ b/resources/libraries/python/DUTSetup.py @@ -24,6 +24,63 @@ from resources.libraries.python.VPPUtil import VPPUtil class DUTSetup(object): """Contains methods for setting up DUTs.""" + + @staticmethod + def get_service_logs(node, service): + """Get specific service unit logs by journalctl from node. + + :param node: Node in the topology. + :param service: Service unit name. + :type node: dict + :type service: str + """ + ssh = SSH() + ssh.connect(node) + ret_code, _, _ = \ + ssh.exec_command_sudo('journalctl --no-pager --unit={name} ' + '--since="$(echo `systemctl show -p ' + 'ActiveEnterTimestamp {name}` | ' + 'awk \'{{print $2 $3}}\')"'. + format(name=service)) + if int(ret_code) != 0: + raise RuntimeError('DUT {host} failed to get logs from unit {name}'. + format(host=node['host'], name=service)) + + @staticmethod + def get_service_logs_on_all_duts(nodes, service): + """Get specific service unit logs by journalctl from all DUTs. + + :param nodes: Nodes in the topology. + :param service: Service unit name. + :type nodes: dict + :type service: str + """ + for node in nodes.values(): + if node['type'] == NodeType.DUT: + DUTSetup.get_service_logs(node, service) + + @staticmethod + def start_service(node, service): + """Start up the named service on node. + + :param node: Node in the topology. + :param service: Service unit name. + :type node: dict + :type service: str + """ + ssh = SSH() + ssh.connect(node) + # We are doing restart. With this we do not care if service + # was running or not. + ret_code, _, _ = \ + ssh.exec_command_sudo('service {name} restart'. + format(name=service), timeout=120) + if int(ret_code) != 0: + raise RuntimeError('DUT {host} failed to start service {name}'. + format(host=node['host'], name=service)) + + DUTSetup.get_service_logs(node, service) + @staticmethod def start_vpp_service_on_all_duts(nodes): """Start up the VPP service on all nodes. @@ -31,17 +88,9 @@ class DUTSetup(object): :param nodes: Nodes in the topology. :type nodes: dict """ - ssh = SSH() for node in nodes.values(): if node['type'] == NodeType.DUT: - ssh.connect(node) - (ret_code, stdout, stderr) = \ - ssh.exec_command_sudo('service vpp restart', timeout=120) - if int(ret_code) != 0: - logger.debug('stdout: {0}'.format(stdout)) - logger.debug('stderr: {0}'.format(stderr)) - raise Exception('DUT {0} failed to start VPP service'. - format(node['host'])) + DUTSetup.start_service(node, Constants.VPP_UNIT) @staticmethod def vpp_show_version_verbose(node): @@ -53,6 +102,12 @@ class DUTSetup(object): vat = VatExecutor() vat.execute_script("show_version_verbose.vat", node, json_out=False) + try: + vat.script_should_have_passed() + except AssertionError: + raise RuntimeError('Failed to get VPP version on host: {name}'. + format(name=node['host'])) + @staticmethod def show_vpp_version_on_all_duts(nodes): """Show VPP version verbose on all DUTs. @@ -64,6 +119,22 @@ class DUTSetup(object): if node['type'] == NodeType.DUT: DUTSetup.vpp_show_version_verbose(node) + @staticmethod + def vpp_show_interfaces(node): + """Run "show interface" CLI command. + + :param node: Node to run command on. + :type node: dict + """ + vat = VatExecutor() + vat.execute_script("show_interface.vat", node, json_out=False) + + try: + vat.script_should_have_passed() + except AssertionError: + raise RuntimeError('Failed to get VPP interfaces on host: {name}'. + format(name=node['host'])) + @staticmethod def vpp_api_trace_save(node): """Run "api trace save" CLI command. @@ -103,17 +174,15 @@ class DUTSetup(object): ssh = SSH() ssh.connect(node) - (ret_code, stdout, stderr) = \ + ret_code, _, _ = \ ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'. format(Constants.REMOTE_FW_DIR, Constants.RESOURCES_LIB_SH), timeout=120) logger.trace(stdout) logger.trace(stderr) if int(ret_code) != 0: - logger.debug('DUT {0} setup script failed: "{1}"'. - format(node['host'], stdout + stderr)) - raise Exception('DUT test setup script failed at node {}'. - format(node['host'])) + raise RuntimeError('DUT test setup script failed at node {name}'. + format(name=node['host'])) @staticmethod def get_vpp_pid(node): @@ -429,3 +498,106 @@ class DUTSetup(object): vat = VatExecutor() vat.execute_script("enable_dpdk_traces.vat", node, json_out=False) vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False) + + @staticmethod + def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs): + """Install VPP on all DUT nodes. + + :param nodes: Nodes in the topology. + :param vpp_pkg_dir: Path to directory where VPP packages are stored. + :param vpp_rpm_pkgs: List of VPP rpm packages to be installed. + :param vpp_deb_pkgs: List of VPP deb packages to be installed. + :type nodes: dict + :type vpp_pkg_dir: str + :type vpp_rpm_pkgs: list + :type vpp_deb_pkgs: list + :raises: RuntimeError if failed to remove or install VPP + """ + + logger.debug("Installing VPP") + + for node in nodes.values(): + if node['type'] == NodeType.DUT: + logger.debug("Installing VPP on node {0}".format(node['host'])) + + ssh = SSH() + ssh.connect(node) + + cmd = "[[ -f /etc/redhat-release ]]" + return_code, _, _ = ssh.exec_command(cmd) + if int(return_code) == 0: + # workaroud - uninstall existing vpp installation until + # start-testcase script is updated on all virl servers + rpm_pkgs_remove = "vpp*" + cmd_u = 'yum -y remove "{0}"'.format(rpm_pkgs_remove) + r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90) + if int(r_rcode) != 0: + raise RuntimeError('Failed to remove previous VPP' + 'installation on host {0}:\n{1}' + .format(node['host'], r_err)) + + rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg) + for pkg in vpp_rpm_pkgs) + "*.rpm" + cmd_i = "rpm -ivh {0}".format(rpm_pkgs) + ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90) + if int(ret_code) != 0: + raise RuntimeError('Failed to install VPP on host {0}:' + '\n{1}'.format(node['host'], err)) + else: + ssh.exec_command_sudo("rpm -qai vpp*") + logger.info("VPP installed on node {0}". + format(node['host'])) + else: + # workaroud - uninstall existing vpp installation until + # start-testcase script is updated on all virl servers + deb_pkgs_remove = "vpp*" + cmd_u = 'apt-get purge -y "{0}"'.format(deb_pkgs_remove) + r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90) + if int(r_rcode) != 0: + raise RuntimeError('Failed to remove previous VPP' + 'installation on host {0}:\n{1}' + .format(node['host'], r_err)) + deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg) + for pkg in vpp_deb_pkgs) + "*.deb" + cmd_i = "dpkg -i --force-all {0}".format(deb_pkgs) + ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90) + if int(ret_code) != 0: + raise RuntimeError('Failed to install VPP on host {0}:' + '\n{1}'.format(node['host'], err)) + else: + ssh.exec_command_sudo("dpkg -l | grep vpp") + logger.info("VPP installed on node {0}". + format(node['host'])) + + ssh.disconnect(node) + + @staticmethod + def verify_vpp_on_all_duts(nodes): + """Verify that VPP is installed on all DUT nodes. + + :param nodes: Nodes in the topology. + :type nodes: dict + """ + + logger.debug("Verify VPP on all DUTs") + + DUTSetup.start_vpp_service_on_all_duts(nodes) + + for node in nodes.values(): + if node['type'] == NodeType.DUT: + DUTSetup.verify_vpp_on_dut(node) + + @staticmethod + def verify_vpp_on_dut(node): + """Verify that VPP is installed on DUT node. + + :param node: DUT node. + :type node: dict + :raises: RuntimeError if failed to restart VPP, get VPP version or + get VPP interfaces + """ + + logger.debug("Verify VPP on node {0}".format(node['host'])) + + DUTSetup.vpp_show_version_verbose(node) + DUTSetup.vpp_show_interfaces(node) diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py index 88776fe3f1..d3e598de7c 100644 --- a/resources/libraries/python/VppConfigGenerator.py +++ b/resources/libraries/python/VppConfigGenerator.py @@ -17,6 +17,8 @@ import re import time from resources.libraries.python.ssh import SSH +from resources.libraries.python.constants import Constants +from resources.libraries.python.DUTSetup import DUTSetup from resources.libraries.python.topology import NodeType from resources.libraries.python.topology import Topology @@ -513,48 +515,38 @@ class VppConfigGenerator(object): filename = self._vpp_startup_conf if self._vpp_startup_conf_backup is not None: - (ret, _, _) = \ - ssh.exec_command('sudo cp {0} {1}'. - format(self._vpp_startup_conf, - self._vpp_startup_conf_backup)) + ret, _, _ = \ + ssh.exec_command('sudo cp {src} {dest}'. + format(src=self._vpp_startup_conf, + dest=self._vpp_startup_conf_backup)) if ret != 0: - raise RuntimeError('Backup of config file failed on node {}'. - format(self._hostname)) + raise RuntimeError('Backup of config file failed on node ' + '{name}'.format(name=self._hostname)) - (ret, _, _) = \ + ret, _, _ = \ ssh.exec_command('echo "{config}" | sudo tee {filename}'. format(config=self._vpp_config, filename=filename)) if ret != 0: - raise RuntimeError('Writing config file failed to node {}'. - format(self._hostname)) + raise RuntimeError('Writing config file failed to node {name}'. + format(name=self._hostname)) if restart_vpp: - # Instead of restarting, we'll do separate start and stop - # actions. This way we don't care whether VPP was running - # to begin with. - ssh.exec_command('sudo service {} stop' - .format(self._vpp_service_name)) - (ret, _, _) = \ - ssh.exec_command('sudo service {} start' - .format(self._vpp_service_name)) - if ret != 0: - raise RuntimeError('Restarting VPP failed on node {}'. - format(self._hostname)) + DUTSetup.start_service(self._node, Constants.VPP_UNIT) # Sleep seconds, up to times, # and verify if VPP is running. for _ in range(retries): time.sleep(waittime) - (ret, stdout, _) = \ + ret, stdout, _ = \ ssh.exec_command('echo show hardware-interfaces | ' 'nc 0 5002 || echo "VPP not yet running"') if ret == 0 and stdout != 'VPP not yet running': break else: - raise RuntimeError('VPP failed to restart on node {}'. - format(self._hostname)) + raise RuntimeError('VPP failed to restart on node {name}'. + format(name=self._hostname)) def restore_config(self): """Restore VPP startup.conf from backup. @@ -565,9 +557,9 @@ class VppConfigGenerator(object): ssh = SSH() ssh.connect(self._node) - (ret, _, _) = ssh.exec_command('sudo cp {0} {1}'. - format(self._vpp_startup_conf_backup, - self._vpp_startup_conf)) + ret, _, _ = ssh.exec_command('sudo cp {src} {dest}'. + format(src=self._vpp_startup_conf_backup, + dest=self._vpp_startup_conf)) if ret != 0: - raise RuntimeError('Restoration of config file failed on node {}'. - format(self._hostname)) + raise RuntimeError('Restoration of config file failed on node ' + '{name}'.format(name=self._hostname)) diff --git a/resources/libraries/python/constants.py b/resources/libraries/python/constants.py index c3f1551c55..9507e66d85 100644 --- a/resources/libraries/python/constants.py +++ b/resources/libraries/python/constants.py @@ -29,6 +29,9 @@ class Constants(object): # OpenVPP VAT binary name VAT_BIN_NAME = 'vpp_api_test' + # VPP service unit name + VPP_UNIT = 'vpp' + # QEMU version to install QEMU_INSTALL_VERSION = 'qemu-2.5.0' -- cgit 1.2.3-korg