aboutsummaryrefslogtreecommitdiffstats
path: root/resources/libraries
diff options
context:
space:
mode:
authorPeter Mikus <pmikus@cisco.com>2018-04-09 15:08:41 +0200
committerJan Gelety <jgelety@cisco.com>2018-05-21 10:31:52 +0200
commit33d982ab77b174d8a9c5ff59679d5f664be17a05 (patch)
tree812d9741546f0efa615b6a8bf395d9b38eba36bb /resources/libraries
parent56f9e1430c3e4442b27f0c584b2855653bf31e99 (diff)
Create unified service operations
- Add ability to dump the logs from service unit (since last start) - Cleanup service code to more streamlined way. Change-Id: I6e332dce3229727dd36fd72a8ecfb9fc9efe4d63 Signed-off-by: Peter Mikus <pmikus@cisco.com> Signed-off-by: Jan Gelety <jgelety@cisco.com>
Diffstat (limited to 'resources/libraries')
-rw-r--r--resources/libraries/bash/dut_setup.sh12
-rw-r--r--resources/libraries/python/DUTSetup.py200
-rw-r--r--resources/libraries/python/VppConfigGenerator.py48
-rw-r--r--resources/libraries/python/constants.py3
4 files changed, 213 insertions, 50 deletions
diff --git a/resources/libraries/bash/dut_setup.sh b/resources/libraries/bash/dut_setup.sh
index 952b82bdc7..988ffe1ce2 100644
--- a/resources/libraries/bash/dut_setup.sh
+++ b/resources/libraries/bash/dut_setup.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright (c) 2016 Cisco and/or its affiliates.
+# Copyright (c) 2018 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
@@ -38,6 +38,8 @@ cmd 'sudo -S service vpp restart'
echo "[Command_desc] SLEEP for three seconds, so that VPP is up for sure"
cmd 'sleep 3'
+cmd "sudo journalctl --no-pager --unit=vpp --since=\"$(echo `systemctl show -p ActiveEnterTimestamp vpp` | awk '{print $2 $3}')\""
+
cmd 'cat /proc/meminfo'
cmd 'free -m'
@@ -48,14 +50,8 @@ cmd 'sudo dmidecode | grep UUID'
cmd 'lspci -Dnn'
-if [ -f "/etc/redhat-release" ]; then
- cmd 'tail -n 100 /var/log/messages'
-else
- cmd 'tail -n 100 /var/log/syslog'
-fi
-
echo "[Command_desc] Adding dpdk-input trace"
cmd 'sudo vpp_api_test <<< "exec trace add dpdk-input 100"'
echo "[Command_desc] Adding vhost-user-input trace"
-cmd 'sudo vpp_api_test <<< "exec trace add vhost-user-input 100"' \ No newline at end of file
+cmd 'sudo vpp_api_test <<< "exec trace add vhost-user-input 100"'
diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index e0479cfb60..4cb4ad4040 100644
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -24,6 +24,63 @@ from resources.libraries.python.VPPUtil import VPPUtil
class DUTSetup(object):
"""Contains methods for setting up DUTs."""
+
+ @staticmethod
+ def get_service_logs(node, service):
+ """Get specific service unit logs by journalctl from node.
+
+ :param node: Node in the topology.
+ :param service: Service unit name.
+ :type node: dict
+ :type service: str
+ """
+ ssh = SSH()
+ ssh.connect(node)
+ ret_code, _, _ = \
+ ssh.exec_command_sudo('journalctl --no-pager --unit={name} '
+ '--since="$(echo `systemctl show -p '
+ 'ActiveEnterTimestamp {name}` | '
+ 'awk \'{{print $2 $3}}\')"'.
+ format(name=service))
+ if int(ret_code) != 0:
+ raise RuntimeError('DUT {host} failed to get logs from unit {name}'.
+ format(host=node['host'], name=service))
+
+ @staticmethod
+ def get_service_logs_on_all_duts(nodes, service):
+ """Get specific service unit logs by journalctl from all DUTs.
+
+ :param nodes: Nodes in the topology.
+ :param service: Service unit name.
+ :type nodes: dict
+ :type service: str
+ """
+ for node in nodes.values():
+ if node['type'] == NodeType.DUT:
+ DUTSetup.get_service_logs(node, service)
+
+ @staticmethod
+ def start_service(node, service):
+ """Start up the named service on node.
+
+ :param node: Node in the topology.
+ :param service: Service unit name.
+ :type node: dict
+ :type service: str
+ """
+ ssh = SSH()
+ ssh.connect(node)
+ # We are doing restart. With this we do not care if service
+ # was running or not.
+ ret_code, _, _ = \
+ ssh.exec_command_sudo('service {name} restart'.
+ format(name=service), timeout=120)
+ if int(ret_code) != 0:
+ raise RuntimeError('DUT {host} failed to start service {name}'.
+ format(host=node['host'], name=service))
+
+ DUTSetup.get_service_logs(node, service)
+
@staticmethod
def start_vpp_service_on_all_duts(nodes):
"""Start up the VPP service on all nodes.
@@ -31,17 +88,9 @@ class DUTSetup(object):
:param nodes: Nodes in the topology.
:type nodes: dict
"""
- ssh = SSH()
for node in nodes.values():
if node['type'] == NodeType.DUT:
- ssh.connect(node)
- (ret_code, stdout, stderr) = \
- ssh.exec_command_sudo('service vpp restart', timeout=120)
- if int(ret_code) != 0:
- logger.debug('stdout: {0}'.format(stdout))
- logger.debug('stderr: {0}'.format(stderr))
- raise Exception('DUT {0} failed to start VPP service'.
- format(node['host']))
+ DUTSetup.start_service(node, Constants.VPP_UNIT)
@staticmethod
def vpp_show_version_verbose(node):
@@ -53,6 +102,12 @@ class DUTSetup(object):
vat = VatExecutor()
vat.execute_script("show_version_verbose.vat", node, json_out=False)
+ try:
+ vat.script_should_have_passed()
+ except AssertionError:
+ raise RuntimeError('Failed to get VPP version on host: {name}'.
+ format(name=node['host']))
+
@staticmethod
def show_vpp_version_on_all_duts(nodes):
"""Show VPP version verbose on all DUTs.
@@ -65,6 +120,22 @@ class DUTSetup(object):
DUTSetup.vpp_show_version_verbose(node)
@staticmethod
+ def vpp_show_interfaces(node):
+ """Run "show interface" CLI command.
+
+ :param node: Node to run command on.
+ :type node: dict
+ """
+ vat = VatExecutor()
+ vat.execute_script("show_interface.vat", node, json_out=False)
+
+ try:
+ vat.script_should_have_passed()
+ except AssertionError:
+ raise RuntimeError('Failed to get VPP interfaces on host: {name}'.
+ format(name=node['host']))
+
+ @staticmethod
def vpp_api_trace_save(node):
"""Run "api trace save" CLI command.
@@ -103,17 +174,15 @@ class DUTSetup(object):
ssh = SSH()
ssh.connect(node)
- (ret_code, stdout, stderr) = \
+ ret_code, _, _ = \
ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'.
format(Constants.REMOTE_FW_DIR,
Constants.RESOURCES_LIB_SH), timeout=120)
logger.trace(stdout)
logger.trace(stderr)
if int(ret_code) != 0:
- logger.debug('DUT {0} setup script failed: "{1}"'.
- format(node['host'], stdout + stderr))
- raise Exception('DUT test setup script failed at node {}'.
- format(node['host']))
+ raise RuntimeError('DUT test setup script failed at node {name}'.
+ format(name=node['host']))
@staticmethod
def get_vpp_pid(node):
@@ -429,3 +498,106 @@ class DUTSetup(object):
vat = VatExecutor()
vat.execute_script("enable_dpdk_traces.vat", node, json_out=False)
vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False)
+
+ @staticmethod
+ def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs):
+ """Install VPP on all DUT nodes.
+
+ :param nodes: Nodes in the topology.
+ :param vpp_pkg_dir: Path to directory where VPP packages are stored.
+ :param vpp_rpm_pkgs: List of VPP rpm packages to be installed.
+ :param vpp_deb_pkgs: List of VPP deb packages to be installed.
+ :type nodes: dict
+ :type vpp_pkg_dir: str
+ :type vpp_rpm_pkgs: list
+ :type vpp_deb_pkgs: list
+ :raises: RuntimeError if failed to remove or install VPP
+ """
+
+ logger.debug("Installing VPP")
+
+ for node in nodes.values():
+ if node['type'] == NodeType.DUT:
+ logger.debug("Installing VPP on node {0}".format(node['host']))
+
+ ssh = SSH()
+ ssh.connect(node)
+
+ cmd = "[[ -f /etc/redhat-release ]]"
+ return_code, _, _ = ssh.exec_command(cmd)
+ if int(return_code) == 0:
+ # workaroud - uninstall existing vpp installation until
+ # start-testcase script is updated on all virl servers
+ rpm_pkgs_remove = "vpp*"
+ cmd_u = 'yum -y remove "{0}"'.format(rpm_pkgs_remove)
+ r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
+ if int(r_rcode) != 0:
+ raise RuntimeError('Failed to remove previous VPP'
+ 'installation on host {0}:\n{1}'
+ .format(node['host'], r_err))
+
+ rpm_pkgs = "*.rpm ".join(str(vpp_pkg_dir + pkg)
+ for pkg in vpp_rpm_pkgs) + "*.rpm"
+ cmd_i = "rpm -ivh {0}".format(rpm_pkgs)
+ ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
+ if int(ret_code) != 0:
+ raise RuntimeError('Failed to install VPP on host {0}:'
+ '\n{1}'.format(node['host'], err))
+ else:
+ ssh.exec_command_sudo("rpm -qai vpp*")
+ logger.info("VPP installed on node {0}".
+ format(node['host']))
+ else:
+ # workaroud - uninstall existing vpp installation until
+ # start-testcase script is updated on all virl servers
+ deb_pkgs_remove = "vpp*"
+ cmd_u = 'apt-get purge -y "{0}"'.format(deb_pkgs_remove)
+ r_rcode, _, r_err = ssh.exec_command_sudo(cmd_u, timeout=90)
+ if int(r_rcode) != 0:
+ raise RuntimeError('Failed to remove previous VPP'
+ 'installation on host {0}:\n{1}'
+ .format(node['host'], r_err))
+ deb_pkgs = "*.deb ".join(str(vpp_pkg_dir + pkg)
+ for pkg in vpp_deb_pkgs) + "*.deb"
+ cmd_i = "dpkg -i --force-all {0}".format(deb_pkgs)
+ ret_code, _, err = ssh.exec_command_sudo(cmd_i, timeout=90)
+ if int(ret_code) != 0:
+ raise RuntimeError('Failed to install VPP on host {0}:'
+ '\n{1}'.format(node['host'], err))
+ else:
+ ssh.exec_command_sudo("dpkg -l | grep vpp")
+ logger.info("VPP installed on node {0}".
+ format(node['host']))
+
+ ssh.disconnect(node)
+
+ @staticmethod
+ def verify_vpp_on_all_duts(nodes):
+ """Verify that VPP is installed on all DUT nodes.
+
+ :param nodes: Nodes in the topology.
+ :type nodes: dict
+ """
+
+ logger.debug("Verify VPP on all DUTs")
+
+ DUTSetup.start_vpp_service_on_all_duts(nodes)
+
+ for node in nodes.values():
+ if node['type'] == NodeType.DUT:
+ DUTSetup.verify_vpp_on_dut(node)
+
+ @staticmethod
+ def verify_vpp_on_dut(node):
+ """Verify that VPP is installed on DUT node.
+
+ :param node: DUT node.
+ :type node: dict
+ :raises: RuntimeError if failed to restart VPP, get VPP version or
+ get VPP interfaces
+ """
+
+ logger.debug("Verify VPP on node {0}".format(node['host']))
+
+ DUTSetup.vpp_show_version_verbose(node)
+ DUTSetup.vpp_show_interfaces(node)
diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py
index 88776fe3f1..d3e598de7c 100644
--- a/resources/libraries/python/VppConfigGenerator.py
+++ b/resources/libraries/python/VppConfigGenerator.py
@@ -17,6 +17,8 @@ import re
import time
from resources.libraries.python.ssh import SSH
+from resources.libraries.python.constants import Constants
+from resources.libraries.python.DUTSetup import DUTSetup
from resources.libraries.python.topology import NodeType
from resources.libraries.python.topology import Topology
@@ -513,48 +515,38 @@ class VppConfigGenerator(object):
filename = self._vpp_startup_conf
if self._vpp_startup_conf_backup is not None:
- (ret, _, _) = \
- ssh.exec_command('sudo cp {0} {1}'.
- format(self._vpp_startup_conf,
- self._vpp_startup_conf_backup))
+ ret, _, _ = \
+ ssh.exec_command('sudo cp {src} {dest}'.
+ format(src=self._vpp_startup_conf,
+ dest=self._vpp_startup_conf_backup))
if ret != 0:
- raise RuntimeError('Backup of config file failed on node {}'.
- format(self._hostname))
+ raise RuntimeError('Backup of config file failed on node '
+ '{name}'.format(name=self._hostname))
- (ret, _, _) = \
+ ret, _, _ = \
ssh.exec_command('echo "{config}" | sudo tee {filename}'.
format(config=self._vpp_config,
filename=filename))
if ret != 0:
- raise RuntimeError('Writing config file failed to node {}'.
- format(self._hostname))
+ raise RuntimeError('Writing config file failed to node {name}'.
+ format(name=self._hostname))
if restart_vpp:
- # Instead of restarting, we'll do separate start and stop
- # actions. This way we don't care whether VPP was running
- # to begin with.
- ssh.exec_command('sudo service {} stop'
- .format(self._vpp_service_name))
- (ret, _, _) = \
- ssh.exec_command('sudo service {} start'
- .format(self._vpp_service_name))
- if ret != 0:
- raise RuntimeError('Restarting VPP failed on node {}'.
- format(self._hostname))
+ DUTSetup.start_service(self._node, Constants.VPP_UNIT)
# Sleep <waittime> seconds, up to <retry> times,
# and verify if VPP is running.
for _ in range(retries):
time.sleep(waittime)
- (ret, stdout, _) = \
+ ret, stdout, _ = \
ssh.exec_command('echo show hardware-interfaces | '
'nc 0 5002 || echo "VPP not yet running"')
if ret == 0 and stdout != 'VPP not yet running':
break
else:
- raise RuntimeError('VPP failed to restart on node {}'.
- format(self._hostname))
+ raise RuntimeError('VPP failed to restart on node {name}'.
+ format(name=self._hostname))
def restore_config(self):
"""Restore VPP startup.conf from backup.
@@ -565,9 +557,9 @@ class VppConfigGenerator(object):
ssh = SSH()
ssh.connect(self._node)
- (ret, _, _) = ssh.exec_command('sudo cp {0} {1}'.
- format(self._vpp_startup_conf_backup,
- self._vpp_startup_conf))
+ ret, _, _ = ssh.exec_command('sudo cp {src} {dest}'.
+ format(src=self._vpp_startup_conf_backup,
+ dest=self._vpp_startup_conf))
if ret != 0:
- raise RuntimeError('Restoration of config file failed on node {}'.
- format(self._hostname))
+ raise RuntimeError('Restoration of config file failed on node '
+ '{name}'.format(name=self._hostname))
diff --git a/resources/libraries/python/constants.py b/resources/libraries/python/constants.py
index c3f1551c55..9507e66d85 100644
--- a/resources/libraries/python/constants.py
+++ b/resources/libraries/python/constants.py
@@ -29,6 +29,9 @@ class Constants(object):
# OpenVPP VAT binary name
VAT_BIN_NAME = 'vpp_api_test'
+ # VPP service unit name
+ VPP_UNIT = 'vpp'
+
# QEMU version to install
QEMU_INSTALL_VERSION = 'qemu-2.5.0'