aboutsummaryrefslogtreecommitdiffstats
path: root/resources/libraries/python
diff options
context:
space:
mode:
authorPeter Mikus <pmikus@cisco.com>2019-04-24 11:31:30 +0000
committerVratko Polak <vrpolak@cisco.com>2019-05-06 10:30:11 +0000
commite001fdea995835f1ef75a5e21607ba02d78e4068 (patch)
tree03392169d3d404c55531940aa658fd73db2e54b1 /resources/libraries/python
parent1c54bfd3bdfafe2764394f6540fd35f708bf2239 (diff)
CSIT-1493 VPP restart handling code
Change-Id: Ibe52125089f39e0ff17ec607a3ed00c61d52ab8c Signed-off-by: Peter Mikus <pmikus@cisco.com>
Diffstat (limited to 'resources/libraries/python')
-rw-r--r--resources/libraries/python/DUTSetup.py40
-rw-r--r--resources/libraries/python/KubernetesUtils.py5
-rw-r--r--resources/libraries/python/QemuUtils.py2
-rw-r--r--resources/libraries/python/VPPUtil.py145
-rw-r--r--resources/libraries/python/VppConfigGenerator.py91
-rw-r--r--resources/libraries/python/ssh.py22
6 files changed, 160 insertions, 145 deletions
diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py
index 67edefb930..e2fba066d3 100644
--- a/resources/libraries/python/DUTSetup.py
+++ b/resources/libraries/python/DUTSetup.py
@@ -61,6 +61,39 @@ class DUTSetup(object):
DUTSetup.get_service_logs(node, service)
@staticmethod
+ def restart_service(node, service):
+ """Restarts the named service on node.
+
+ :param node: Node in the topology.
+ :param service: Service unit name.
+ :type node: dict
+ :type service: str
+ """
+ if DUTSetup.running_in_container(node):
+ command = 'supervisorctl restart {name}'.format(name=service)
+ else:
+ command = 'service {name} restart'.format(name=service)
+ message = 'Node {host} failed to restart service {name}'.\
+ format(host=node['host'], name=service)
+
+ exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message)
+
+ DUTSetup.get_service_logs(node, service)
+
+ @staticmethod
+ def restart_service_on_all_duts(nodes, service):
+ """Retarts the named service on all DUTs.
+
+ :param node: Nodes in the topology.
+ :param service: Service unit name.
+ :type node: dict
+ :type service: str
+ """
+ for node in nodes.values():
+ if node['type'] == NodeType.DUT:
+ DUTSetup.restart_service(node, service)
+
+ @staticmethod
def start_service(node, service):
"""Start up the named service on node.
@@ -69,6 +102,7 @@ class DUTSetup(object):
:type node: dict
:type service: str
"""
+ # TODO: change command to start once all parent function updated.
if DUTSetup.running_in_container(node):
command = 'supervisorctl restart {name}'.format(name=service)
else:
@@ -609,7 +643,8 @@ class DUTSetup(object):
@staticmethod
def install_vpp_on_all_duts(nodes, vpp_pkg_dir):
- """Install VPP on all DUT nodes.
+ """Install VPP on all DUT nodes. Start the VPP service in case of
+ systemd is not available or does not support autostart.
:param nodes: Nodes in the topology.
:param vpp_pkg_dir: Path to directory where VPP packages are stored.
@@ -634,6 +669,8 @@ class DUTSetup(object):
format(dir=vpp_pkg_dir), timeout=120,
sudo=True, message=message)
exec_cmd_no_error(node, 'dpkg -l | grep vpp', sudo=True)
+ if DUTSetup.running_in_container(node):
+ DUTSetup.restart_service(node, Constants.VPP_UNIT)
else:
exec_cmd_no_error(node, 'yum -y remove "*vpp*" || true',
timeout=120, sudo=True)
@@ -641,6 +678,7 @@ class DUTSetup(object):
format(dir=vpp_pkg_dir), timeout=120,
sudo=True, message=message)
exec_cmd_no_error(node, 'rpm -qai *vpp*', sudo=True)
+ DUTSetup.restart_service(node, Constants.VPP_UNIT)
@staticmethod
def running_in_container(node):
diff --git a/resources/libraries/python/KubernetesUtils.py b/resources/libraries/python/KubernetesUtils.py
index e932492e05..60e12863b4 100644
--- a/resources/libraries/python/KubernetesUtils.py
+++ b/resources/libraries/python/KubernetesUtils.py
@@ -483,7 +483,6 @@ class KubernetesUtils(object):
vpp_config.set_node(kwargs['node'])
vpp_config.add_unix_cli_listen(value='0.0.0.0:5002')
vpp_config.add_unix_nodaemon()
- vpp_config.add_dpdk_socketmem('1024,1024')
vpp_config.add_heapsize('4G')
vpp_config.add_ip_heap_size('4G')
vpp_config.add_ip6_heap_size('4G')
@@ -500,7 +499,7 @@ class KubernetesUtils(object):
if cpuset_cpus:
corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
vpp_config.add_cpu_corelist_workers(corelist_workers)
- vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)
+ vpp_config.write_config(filename=kwargs['filename'])
@staticmethod
def create_kubernetes_vnf_startup_config(**kwargs):
@@ -536,4 +535,4 @@ class KubernetesUtils(object):
corelist_workers = ','.join(str(cpu) for cpu in cpuset_cpus)
vpp_config.add_cpu_corelist_workers(corelist_workers)
vpp_config.add_plugin('disable', 'dpdk_plugin.so')
- vpp_config.apply_config(filename=kwargs['filename'], restart_vpp=False)
+ vpp_config.write_config(filename=kwargs['filename'])
diff --git a/resources/libraries/python/QemuUtils.py b/resources/libraries/python/QemuUtils.py
index 5404b51292..735feab403 100644
--- a/resources/libraries/python/QemuUtils.py
+++ b/resources/libraries/python/QemuUtils.py
@@ -212,7 +212,7 @@ class QemuUtils(object):
vpp_config.add_dpdk_no_tx_checksum_offload()
vpp_config.add_plugin('disable', 'default')
vpp_config.add_plugin('enable', 'dpdk_plugin.so')
- vpp_config.apply_config(startup, restart_vpp=False)
+ vpp_config.write_config(startup)
# Create VPP running configuration.
template = '{res}/{tpl}.exec'.format(res=Constants.RESOURCES_TPL_VM,
diff --git a/resources/libraries/python/VPPUtil.py b/resources/libraries/python/VPPUtil.py
index 726ba2bde6..5b7728412e 100644
--- a/resources/libraries/python/VPPUtil.py
+++ b/resources/libraries/python/VPPUtil.py
@@ -13,14 +13,12 @@
"""VPP util library."""
-import time
-
from robot.api import logger
from resources.libraries.python.Constants import Constants
from resources.libraries.python.DUTSetup import DUTSetup
from resources.libraries.python.PapiExecutor import PapiExecutor
-from resources.libraries.python.ssh import exec_cmd, exec_cmd_no_error
+from resources.libraries.python.ssh import exec_cmd_no_error
from resources.libraries.python.topology import NodeType
from resources.libraries.python.VatExecutor import VatExecutor
@@ -58,55 +56,39 @@ class VPPUtil(object):
exec_cmd_no_error(node, command, timeout=30, sudo=True)
@staticmethod
- def start_vpp_service(node, retries=120):
- """Start VPP service on the specified node.
+ def restart_vpp_service(node):
+ """Restart VPP service on the specified topology node.
- :param node: VPP node.
- :param retries: Number of times (default 60) to re-try waiting.
+ :param node: Topology node.
:type node: dict
- :type retries: int
- :raises RuntimeError: If VPP service fails to start.
"""
- DUTSetup.start_service(node, Constants.VPP_UNIT)
- # Sleep 1 second, up to <retry> times,
- # and verify if VPP is running.
- for _ in range(retries):
- time.sleep(1)
- command = 'vppctl show pci'
- ret, stdout, _ = exec_cmd(node, command, timeout=30, sudo=True)
- if not ret and 'Connection refused' not in stdout:
- break
- else:
- raise RuntimeError('VPP failed to start on host {name}'.
- format(name=node['host']))
- DUTSetup.get_service_logs(node, Constants.VPP_UNIT)
+ DUTSetup.restart_service(node, Constants.VPP_UNIT)
@staticmethod
- def start_vpp_service_on_all_duts(nodes):
- """Start up the VPP service on all nodes.
+ def restart_vpp_service_on_all_duts(nodes):
+ """Restart VPP service on all DUT nodes.
- :param nodes: Nodes in the topology.
+ :param nodes: Topology nodes.
:type nodes: dict
"""
for node in nodes.values():
if node['type'] == NodeType.DUT:
- VPPUtil.start_vpp_service(node)
+ VPPUtil.restart_vpp_service(node)
@staticmethod
def stop_vpp_service(node):
- """Stop VPP service on the specified node.
+ """Stop VPP service on the specified topology node.
- :param node: VPP node.
+ :param node: Topology node.
:type node: dict
- :raises RuntimeError: If VPP service fails to stop.
"""
DUTSetup.stop_service(node, Constants.VPP_UNIT)
@staticmethod
def stop_vpp_service_on_all_duts(nodes):
- """Stop VPP service on all nodes.
+ """Stop VPP service on all DUT nodes.
- :param nodes: Nodes in the topology.
+ :param nodes: Topology nodes.
:type nodes: dict
"""
for node in nodes.values():
@@ -114,31 +96,60 @@ class VPPUtil(object):
VPPUtil.stop_vpp_service(node)
@staticmethod
- def verify_vpp_on_dut(node):
- """Verify that VPP is installed on DUT node.
+ def verify_vpp_installed(node):
+ """Verify that VPP is installed on the specified topology node.
+
+ :param node: Topology node.
+ :type node: dict
+ """
+ cmd = 'command -v vpp'
+ exec_cmd_no_error(
+ node, cmd, message='VPP is not installed!')
+
+ @staticmethod
+ def verify_vpp_started(node):
+ """Verify that VPP is started on the specified topology node.
- :param node: DUT node.
+ :param node: Topology node.
:type node: dict
- :raises RuntimeError: If failed to restart VPP, get VPP version
- or get VPP interfaces.
"""
- VPPUtil.vpp_show_version_verbose(node)
- VPPUtil.vpp_show_interfaces(node)
+ cmd = ('vppctl show pci 2>&1 | '
+ 'fgrep -v "Connection refused" | '
+ 'fgrep -v "No such file or directory"')
+ exec_cmd_no_error(
+ node, cmd, sudo=True, message='VPP failed to start!', retries=120)
+
+ @staticmethod
+ def verify_vpp(node):
+ """Verify that VPP is installed and started on the specified topology
+ node.
+
+ :param node: Topology node.
+ :type node: dict
+ :raises RuntimeError: If VPP service fails to start.
+ """
+ VPPUtil.verify_vpp_installed(node)
+ try:
+ # Verify responsivness of vppctl.
+ VPPUtil.verify_vpp_started(node)
+ # Verify responsivness of PAPI.
+ VPPUtil.show_log(node)
+ finally:
+ DUTSetup.get_service_logs(node, Constants.VPP_UNIT)
@staticmethod
def verify_vpp_on_all_duts(nodes):
- """Verify that VPP is installed on all DUT nodes.
+ """Verify that VPP is installed and started on all DUT nodes.
:param nodes: Nodes in the topology.
:type nodes: dict
"""
for node in nodes.values():
if node['type'] == NodeType.DUT:
- VPPUtil.start_vpp_service(node)
- VPPUtil.verify_vpp_on_dut(node)
+ VPPUtil.verify_vpp(node)
@staticmethod
- def vpp_show_version(node, verbose=False):
+ def vpp_show_version(node, verbose=True):
"""Run "show_version" PAPI command.
:param node: Node to run command on.
@@ -149,7 +160,6 @@ class VPPUtil(object):
:returns: VPP version.
:rtype: str
"""
-
with PapiExecutor(node) as papi_exec:
data = papi_exec.add('show_version').execute_should_pass().\
verify_reply()
@@ -164,25 +174,15 @@ class VPPUtil(object):
return data['version'].rstrip('\0x00')
@staticmethod
- def vpp_show_version_verbose(node):
- """Run "show_version" API command and return verbose string of version
- data.
-
- :param node: Node to run command on.
- :type node: dict
- """
- VPPUtil.vpp_show_version(node, verbose=True)
-
- @staticmethod
def show_vpp_version_on_all_duts(nodes):
- """Show VPP version on all DUTs.
+ """Show VPP version verbose on all DUTs.
- :param nodes: VPP nodes.
+ :param nodes: Nodes in the topology.
:type nodes: dict
"""
for node in nodes.values():
if node['type'] == NodeType.DUT:
- VPPUtil.vpp_show_version_verbose(node)
+ VPPUtil.vpp_show_version(node)
@staticmethod
def vpp_show_interfaces(node):
@@ -212,26 +212,6 @@ class VPPUtil(object):
json_out=False)
@staticmethod
- def vpp_api_trace_dump(node):
- """Run "api trace custom-dump" CLI command.
-
- :param node: Node to run command on.
- :type node: dict
- """
- vat = VatExecutor()
- vat.execute_script("api_trace_dump.vat", node, json_out=False)
-
- @staticmethod
- def vpp_api_trace_save(node):
- """Run "api trace save" CLI command.
-
- :param node: Node to run command on.
- :type node: dict
- """
- vat = VatExecutor()
- vat.execute_script("api_trace_save.vat", node, json_out=False)
-
- @staticmethod
def vpp_enable_traces_on_dut(node):
"""Enable vpp packet traces on the DUT node.
@@ -298,6 +278,19 @@ class VPPUtil(object):
VPPUtil.show_event_logger_on_dut(node)
@staticmethod
+ def show_log(node):
+ """Show log on the specified topology node.
+
+ :param node: Topology node.
+ :type node: dict
+ :returns: VPP log data.
+ :rtype: list
+ """
+ with PapiExecutor(node) as papi_exec:
+ return papi_exec.add('cli_inband', cmd='show log').get_replies().\
+ verify_reply()["reply"]
+
+ @staticmethod
def vpp_show_threads(node):
"""Show VPP threads on node.
diff --git a/resources/libraries/python/VppConfigGenerator.py b/resources/libraries/python/VppConfigGenerator.py
index 350e1c0881..883197f978 100644
--- a/resources/libraries/python/VppConfigGenerator.py
+++ b/resources/libraries/python/VppConfigGenerator.py
@@ -14,13 +14,11 @@
"""VPP Configuration File Generator library."""
import re
-import time
-from resources.libraries.python.ssh import SSH
-from resources.libraries.python.Constants import Constants
-from resources.libraries.python.DUTSetup import DUTSetup
+from resources.libraries.python.ssh import exec_cmd_no_error
from resources.libraries.python.topology import NodeType
from resources.libraries.python.topology import Topology
+from resources.libraries.python.VPPUtil import VPPUtil
__all__ = ['VppConfigGenerator']
@@ -559,74 +557,51 @@ class VppConfigGenerator(object):
path = ['session', 'local-endpoints-table-memory']
self.add_config_item(self._nodeconfig, value, path)
- def apply_config(self, filename=None, retries=60, restart_vpp=True):
- """Generate and apply VPP configuration for node.
+ def write_config(self, filename=None):
+ """Generate and write VPP startup configuration to file.
Use data from calls to this class to form a startup.conf file and
- replace /etc/vpp/startup.conf with it on node.
+ replace /etc/vpp/startup.conf with it on topology node.
:param filename: Startup configuration file name.
- :param retries: Number of times (default 60) to re-try waiting.
- :param restart_vpp: Whether to restart VPP.
:type filename: str
- :type retries: int
- :type restart_vpp: bool.
- :raises RuntimeError: If writing config file failed or restart of VPP
- failed or backup of VPP startup.conf failed.
"""
self.dump_config(self._nodeconfig)
- ssh = SSH()
- ssh.connect(self._node)
-
if filename is None:
filename = self._vpp_startup_conf
if self._vpp_startup_conf_backup is not None:
- ret, _, _ = \
- ssh.exec_command('sudo cp {src} {dest}'.
- format(src=self._vpp_startup_conf,
- dest=self._vpp_startup_conf_backup))
- if ret != 0:
- raise RuntimeError('Backup of config file failed on node '
- '{name}'.format(name=self._hostname))
-
- ret, _, _ = \
- ssh.exec_command('echo "{config}" | sudo tee {filename}'.
- format(config=self._vpp_config,
- filename=filename))
-
- if ret != 0:
- raise RuntimeError('Writing config file failed to node {name}'.
- format(name=self._hostname))
-
- if restart_vpp:
- DUTSetup.start_service(self._node, Constants.VPP_UNIT)
-
- # Sleep <waittime> seconds, up to <retry> times,
- # and verify if VPP is running.
- for _ in range(retries):
- time.sleep(1)
- ret, stdout, _ = \
- ssh.exec_command_sudo('vppctl show pci')
- if ret == 0 and 'Connection refused' not in stdout:
- break
- else:
- raise RuntimeError('VPP failed to restart on node {name}'.
- format(name=self._hostname))
+ cmd = ('cp {src} {dest}'.format(
+ src=self._vpp_startup_conf, dest=self._vpp_startup_conf_backup))
+ exec_cmd_no_error(
+ self._node, cmd, sudo=True, message='Copy config file failed!')
- def restore_config(self):
- """Restore VPP startup.conf from backup.
+ cmd = ('echo "{config}" | sudo tee {filename}'.format(
+ config=self._vpp_config, filename=filename))
+ exec_cmd_no_error(
+ self._node, cmd, message='Writing config file failed!')
+
+ def apply_config(self, filename=None, verify_vpp=True):
+ """Generate and write VPP startup configuration to file and restart VPP.
+
+ Use data from calls to this class to form a startup.conf file and
+ replace /etc/vpp/startup.conf with it on topology node.
- :raises RuntimeError: When restoration of startup.conf file failed.
+ :param filename: Startup configuration file name.
+ :param verify_vpp: Verify VPP is running after restart.
+ :type filename: str
+ :type verify_vpp: bool
"""
+ self.write_config(filename=filename)
- ssh = SSH()
- ssh.connect(self._node)
+ VPPUtil.restart_vpp_service(self._node)
+ if verify_vpp:
+ VPPUtil.verify_vpp(self._node)
- ret, _, _ = ssh.exec_command('sudo cp {src} {dest}'.
- format(src=self._vpp_startup_conf_backup,
- dest=self._vpp_startup_conf))
- if ret != 0:
- raise RuntimeError('Restoration of config file failed on node '
- '{name}'.format(name=self._hostname))
+ def restore_config(self):
+ """Restore VPP startup.conf from backup."""
+ cmd = ('cp {src} {dest}'.format(
+ src=self._vpp_startup_conf_backup, dest=self._vpp_startup_conf))
+ exec_cmd_no_error(
+ self._node, cmd, sudo=True, message='Copy config file failed!')
diff --git a/resources/libraries/python/ssh.py b/resources/libraries/python/ssh.py
index 966d1b0448..9c7adc44e1 100644
--- a/resources/libraries/python/ssh.py
+++ b/resources/libraries/python/ssh.py
@@ -452,10 +452,14 @@ def exec_cmd(node, cmd, timeout=600, sudo=False, disconnect=False):
def exec_cmd_no_error(
- node, cmd, timeout=600, sudo=False, message=None, disconnect=False):
+ node, cmd, timeout=600, sudo=False, message=None, disconnect=False,
+ retries=0):
"""Convenience function to ssh/exec/return out & err.
Verifies that return code is zero.
+ Supports retries, timeout is related to each try separately then. There is
+ sleep(1) before each retry.
+ Disconnect (if enabled) is applied after each try.
:param node: DUT node.
:param cmd: Command to be executed.
@@ -463,21 +467,27 @@ def exec_cmd_no_error(
:param sudo: Sudo privilege execution flag. Default: False.
:param message: Error message in case of failure. Default: None.
:param disconnect: Close the opened SSH connection if True.
+ :param retries: How many times to retry on failure.
:type node: dict
:type cmd: str or OptionString
:type timeout: int
:type sudo: bool
:type message: str
:type disconnect: bool
+ :type retries: int
:returns: Stdout, Stderr.
:rtype: tuple(str, str)
:raises RuntimeError: If bash return code is not 0.
"""
- ret_code, stdout, stderr = exec_cmd(
- node, cmd, timeout=timeout, sudo=sudo, disconnect=disconnect)
- msg = ('Command execution failed: "{cmd}"\n{stderr}'.
- format(cmd=cmd, stderr=stderr) if message is None else message)
- if ret_code != 0:
+ for _ in range(retries + 1):
+ ret_code, stdout, stderr = exec_cmd(
+ node, cmd, timeout=timeout, sudo=sudo, disconnect=disconnect)
+ if ret_code == 0:
+ break
+ sleep(1)
+ else:
+ msg = ('Command execution failed: "{cmd}"\n{stderr}'.
+ format(cmd=cmd, stderr=stderr) if message is None else message)
raise RuntimeError(msg)
return stdout, stderr