diff options
| -rw-r--r-- | resources/libraries/bash/dut_setup.sh | 2 | ||||
| -rw-r--r-- | resources/libraries/python/DUTSetup.py | 246 | ||||
| -rw-r--r-- | resources/libraries/python/VPPUtil.py | 170 |
3 files changed, 230 insertions, 188 deletions
diff --git a/resources/libraries/bash/dut_setup.sh b/resources/libraries/bash/dut_setup.sh index 28b495c3fa..aee6386bd7 100644 --- a/resources/libraries/bash/dut_setup.sh +++ b/resources/libraries/bash/dut_setup.sh @@ -33,7 +33,7 @@ cmd 'ps aux | grep vpp' cmd 'cat /etc/vpp/startup.conf' -cmd 'sudo -S service vpp restart' +cmd 'if fgrep docker /proc/1/cgroup; then supervisorctl restart vpp; else sudo -S service vpp restart; fi' echo "[Command_desc] SLEEP for three seconds, so that VPP is up for sure" cmd 'sleep 3' diff --git a/resources/libraries/python/DUTSetup.py b/resources/libraries/python/DUTSetup.py index 84862d4167..7885ad0a16 100644 --- a/resources/libraries/python/DUTSetup.py +++ b/resources/libraries/python/DUTSetup.py @@ -15,11 +15,9 @@ from robot.api import logger -from resources.libraries.python.topology import NodeType, Topology -from resources.libraries.python.ssh import SSH, exec_cmd_no_error from resources.libraries.python.constants import Constants -from resources.libraries.python.VatExecutor import VatExecutor -from resources.libraries.python.VPPUtil import VPPUtil +from resources.libraries.python.ssh import SSH, exec_cmd_no_error +from resources.libraries.python.topology import NodeType, Topology class DUTSetup(object): @@ -27,28 +25,30 @@ class DUTSetup(object): @staticmethod def get_service_logs(node, service): - """Get specific service unit logs by journalctl from node. + """Get specific service unit logs from node. :param node: Node in the topology. :param service: Service unit name. :type node: dict :type service: str """ - ssh = SSH() - ssh.connect(node) - ret_code, _, _ = \ - ssh.exec_command_sudo('journalctl --no-pager --unit={name} ' - '--since="$(echo `systemctl show -p ' - 'ActiveEnterTimestamp {name}` | ' - 'awk \'{{print $2 $3}}\')"'. - format(name=service)) - if int(ret_code): - raise RuntimeError('DUT {host} failed to get logs from unit {name}'. - format(host=node['host'], name=service)) + if DUTSetup.running_in_container(node): + command = 'echo $(< /var/log/supervisord.log)' + else: + command = ('journalctl --no-pager --unit={name} ' + '--since="$(echo `systemctl show -p ' + 'ActiveEnterTimestamp {name}` | ' + 'awk \'{{print $2 $3}}\')"'. + format(name=service)) + message = 'Node {host} failed to get logs from unit {name}'.\ + format(host=node['host'], name=service) + + exec_cmd_no_error(node, command, timeout=30, sudo=True, + message=message) @staticmethod def get_service_logs_on_all_duts(nodes, service): - """Get specific service unit logs by journalctl from all DUTs. + """Get specific service unit logs from all DUTs. :param nodes: Nodes in the topology. :param service: Service unit name. @@ -68,99 +68,62 @@ class DUTSetup(object): :type node: dict :type service: str """ - ssh = SSH() - ssh.connect(node) - # We are doing restart. With this we do not care if service - # was running or not. - ret_code, _, _ = \ - ssh.exec_command_sudo('service {name} restart'. - format(name=service), timeout=120) - if int(ret_code): - raise RuntimeError('DUT {host} failed to start service {name}'. - format(host=node['host'], name=service)) - - DUTSetup.get_service_logs(node, service) + if DUTSetup.running_in_container(node): + command = 'supervisorctl restart {name}'.format(name=service) + else: + command = 'service {name} restart'.format(name=service) + message = 'Node {host} failed to start service {name}'.\ + format(host=node['host'], name=service) - @staticmethod - def start_vpp_service_on_all_duts(nodes): - """Start up the VPP service on all nodes. + exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message) - :param nodes: Nodes in the topology. - :type nodes: dict - """ - for node in nodes.values(): - if node['type'] == NodeType.DUT: - DUTSetup.start_service(node, Constants.VPP_UNIT) + DUTSetup.get_service_logs(node, service) @staticmethod - def vpp_show_version_verbose(node): - """Run "show version verbose" CLI command. + def start_service_on_all_duts(nodes, service): + """Start up the named service on all DUTs. - :param node: Node to run command on. + :param node: Nodes in the topology. + :param service: Service unit name. :type node: dict - """ - vat = VatExecutor() - vat.execute_script("show_version_verbose.vat", node, json_out=False) - - try: - vat.script_should_have_passed() - except AssertionError: - raise RuntimeError('Failed to get VPP version on host: {name}'. - format(name=node['host'])) - - @staticmethod - def show_vpp_version_on_all_duts(nodes): - """Show VPP version verbose on all DUTs. - - :param nodes: VPP nodes - :type nodes: dict + :type service: str """ for node in nodes.values(): if node['type'] == NodeType.DUT: - DUTSetup.vpp_show_version_verbose(node) + DUTSetup.start_service(node, service) @staticmethod - def vpp_show_interfaces(node): - """Run "show interface" CLI command. + def stop_service(node, service): + """Stop the named service on node. - :param node: Node to run command on. + :param node: Node in the topology. + :param service: Service unit name. :type node: dict + :type service: str """ - vat = VatExecutor() - vat.execute_script("show_interface.vat", node, json_out=False) - - try: - vat.script_should_have_passed() - except AssertionError: - raise RuntimeError('Failed to get VPP interfaces on host: {name}'. - format(name=node['host'])) + if DUTSetup.running_in_container(node): + command = 'supervisorctl stop {name}'.format(name=service) + else: + command = 'service {name} stop'.format(name=service) + message = 'Node {host} failed to stop service {name}'.\ + format(host=node['host'], name=service) - @staticmethod - def vpp_api_trace_save(node): - """Run "api trace save" CLI command. + exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message) - :param node: Node to run command on. - :type node: dict - """ - vat = VatExecutor() - vat.execute_script("api_trace_save.vat", node, json_out=False) + DUTSetup.get_service_logs(node, service) @staticmethod - def vpp_api_trace_dump(node): - """Run "api trace custom-dump" CLI command. + def stop_service_on_all_duts(nodes, service): + """Stop the named service on all DUTs. - :param node: Node to run command on. + :param node: Nodes in the topology. + :param service: Service unit name. :type node: dict + :type service: str """ - vat = VatExecutor() - vat.execute_script("api_trace_dump.vat", node, json_out=False) - - @staticmethod - def setup_all_duts(nodes): - """Prepare all DUTs in given topology for test execution.""" for node in nodes.values(): if node['type'] == NodeType.DUT: - DUTSetup.setup_dut(node) + DUTSetup.stop_service(node, service) @staticmethod def setup_dut(node): @@ -171,16 +134,24 @@ class DUTSetup(object): :raises Exception: If the DUT setup fails. """ - ssh = SSH() - ssh.connect(node) + command = 'bash {0}/{1}/dut_setup.sh'.\ + format(Constants.REMOTE_FW_DIR, Constants.RESOURCES_LIB_SH) + message = 'DUT test setup script failed at node {name}'.\ + format(name=node['host']) - ret_code, _, _ = \ - ssh.exec_command('sudo -Sn bash {0}/{1}/dut_setup.sh'. - format(Constants.REMOTE_FW_DIR, - Constants.RESOURCES_LIB_SH), timeout=120) - if int(ret_code): - raise RuntimeError('DUT test setup script failed at node {name}'. - format(name=node['host'])) + exec_cmd_no_error(node, command, timeout=120, sudo=True, + message=message) + + @staticmethod + def setup_all_duts(nodes): + """Run script over SSH to setup all DUT nodes. + + :param nodes: Topology nodes. + :type nodes: dict + """ + for node in nodes.values(): + if node['type'] == NodeType.DUT: + DUTSetup.setup_dut(node) @staticmethod def get_vpp_pid(node): @@ -192,7 +163,6 @@ class DUTSetup(object): :rtype: int :raises RuntimeError: If it is not possible to get the PID. """ - ssh = SSH() ssh.connect(node) @@ -230,7 +200,6 @@ class DUTSetup(object): :returns: PIDs :rtype: dict """ - pids = dict() for node in nodes.values(): if node['type'] == NodeType.DUT: @@ -238,17 +207,6 @@ class DUTSetup(object): return pids @staticmethod - def vpp_show_crypto_device_mapping(node): - """Run "show crypto device mapping" CLI command. - - :param node: Node to run command on. - :type node: dict - """ - vat = VatExecutor() - vat.execute_script("show_crypto_device_mapping.vat", node, - json_out=False) - - @staticmethod def crypto_device_verify(node, force_init=False, numvfs=32): """Verify if Crypto QAT device virtual functions are initialized on all DUTs. If parameter force initialization is set to True, then try to @@ -292,7 +250,7 @@ class DUTSetup(object): DUTSetup.verify_kernel_module(node, 'qat_dh895xcc', force_load=True) # Stop VPP to prevent deadlock. - VPPUtil.stop_vpp_service(node) + DUTSetup.stop_service(node, Constants.VPP_UNIT) current_driver = DUTSetup.get_pci_dev_driver( node, pci_addr.replace(':', r'\:')) @@ -628,30 +586,6 @@ class DUTSetup(object): exec_cmd_no_error(node, command, timeout=30, sudo=True, message=message) @staticmethod - def vpp_enable_traces_on_all_duts(nodes): - """Enable vpp packet traces on all DUTs in the given topology. - - :param nodes: Nodes in the topology. - :type nodes: dict - """ - for node in nodes.values(): - if node['type'] == NodeType.DUT: - DUTSetup.vpp_enable_traces_on_dut(node) - - @staticmethod - def vpp_enable_traces_on_dut(node): - """Enable vpp packet traces on the DUT node. - - :param node: DUT node to set up. - :type node: dict - """ - - vat = VatExecutor() - vat.execute_script("enable_dpdk_traces.vat", node, json_out=False) - vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False) - vat.execute_script("enable_memif_traces.vat", node, json_out=False) - - @staticmethod def install_vpp_on_all_duts(nodes, vpp_pkg_dir, vpp_rpm_pkgs, vpp_deb_pkgs): """Install VPP on all DUT nodes. @@ -665,9 +599,6 @@ class DUTSetup(object): :type vpp_deb_pkgs: list :raises RuntimeError: If failed to remove or install VPP. """ - - logger.debug("Installing VPP") - for node in nodes.values(): if node['type'] == NodeType.DUT: logger.debug("Installing VPP on node {0}".format(node['host'])) @@ -724,36 +655,23 @@ class DUTSetup(object): ssh.disconnect(node) @staticmethod - def verify_vpp_on_dut(node): - """Verify that VPP is installed on DUT node. + def running_in_container(node): + """This method tests if topology node is running inside container. - :param node: DUT node. + :param node: Topology node. :type node: dict - :raises RuntimeError: If failed to restart VPP, get VPP version - or get VPP interfaces. + :returns: True if running in docker container, false if not or failed + to detect. + :rtype: bool """ - - logger.debug("Verify VPP on node {0}".format(node['host'])) - - DUTSetup.vpp_show_version_verbose(node) - DUTSetup.vpp_show_interfaces(node) - - @staticmethod - def verify_vpp_on_all_duts(nodes): - """Verify that VPP is installed on all DUT nodes. - - :param nodes: Nodes in the topology. - :type nodes: dict - """ - - logger.debug("Verify VPP on all DUTs") - - DUTSetup.start_vpp_service_on_all_duts(nodes) - - for node in nodes.values(): - if node['type'] == NodeType.DUT: - DUTSetup.verify_vpp_on_dut(node) - + command = "fgrep docker /proc/1/cgroup" + message = 'Failed to get cgroup settings.' + try: + exec_cmd_no_error(node, command, timeout=30, sudo=False, + message=message) + except RuntimeError: + return False + return True @staticmethod def get_huge_page_size(node): diff --git a/resources/libraries/python/VPPUtil.py b/resources/libraries/python/VPPUtil.py index 3a0148a481..de3caad916 100644 --- a/resources/libraries/python/VPPUtil.py +++ b/resources/libraries/python/VPPUtil.py @@ -11,8 +11,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""VPP util library""" -from resources.libraries.python.ssh import SSH +"""VPP util library.""" +from resources.libraries.python.constants import Constants +from resources.libraries.python.DUTSetup import DUTSetup +from resources.libraries.python.ssh import exec_cmd_no_error +from resources.libraries.python.topology import NodeType +from resources.libraries.python.VatExecutor import VatExecutor class VPPUtil(object): @@ -42,10 +46,30 @@ class VPPUtil(object): if additional_cmds: for cmd in additional_cmds: def_setting_tb_displayed['Custom Setting: {}'.format(cmd)] = cmd - ssh = SSH() - ssh.connect(node) - for _, value in def_setting_tb_displayed.items(): - ssh.exec_command_sudo('vppctl sh {}'.format(value)) + + for _, cmd in def_setting_tb_displayed.items(): + command = 'vppctl sh {cmd}'.format(cmd=cmd) + exec_cmd_no_error(node, command, timeout=30, sudo=True) + + @staticmethod + def start_vpp_service(node): + """Start VPP service on the specified node. + + :param node: VPP node. + :type node: dict + :raises RuntimeError: If VPP service fails to start. + """ + DUTSetup.start_service(node, Constants.VPP_UNIT) + DUTSetup.get_service_logs(node, Constants.VPP_UNIT) + + @staticmethod + def start_vpp_service_on_all_duts(nodes): + """Start up the VPP service on all nodes. + + :param nodes: Nodes in the topology. + :type nodes: dict + """ + DUTSetup.start_service_on_all_duts(nodes, Constants.VPP_UNIT) @staticmethod def stop_vpp_service(node): @@ -53,28 +77,128 @@ class VPPUtil(object): :param node: VPP node. :type node: dict - :raises RuntimeError: If VPP fails to stop. + :raises RuntimeError: If VPP service fails to stop. + """ + DUTSetup.stop_service(node, Constants.VPP_UNIT) + + @staticmethod + def verify_vpp_on_dut(node): + """Verify that VPP is installed on DUT node. + + :param node: DUT node. + :type node: dict + :raises RuntimeError: If failed to restart VPP, get VPP version + or get VPP interfaces. """ + VPPUtil.vpp_show_version_verbose(node) + VPPUtil.vpp_show_interfaces(node) - ssh = SSH() - ssh.connect(node) - cmd = "service vpp stop" - ret_code, _, _ = ssh.exec_command_sudo(cmd, timeout=80) - if int(ret_code) != 0: - raise RuntimeError("VPP service did not shut down gracefully.") + @staticmethod + def verify_vpp_on_all_duts(nodes): + """Verify that VPP is installed on all DUT nodes. + + :param nodes: Nodes in the topology. + :type nodes: dict + """ + for node in nodes.values(): + if node['type'] == NodeType.DUT: + DUTSetup.start_service(node, Constants.VPP_UNIT) + VPPUtil.vpp_show_version_verbose(node) + VPPUtil.vpp_show_interfaces(node) @staticmethod - def start_vpp_service(node): - """start VPP service on the specified node. + def vpp_show_version_verbose(node): + """Run "show version verbose" CLI command. - :param node: VPP node. + :param node: Node to run command on. + :type node: dict + """ + vat = VatExecutor() + vat.execute_script("show_version_verbose.vat", node, json_out=False) + + try: + vat.script_should_have_passed() + except AssertionError: + raise RuntimeError('Failed to get VPP version on host: {name}'. + format(name=node['host'])) + + @staticmethod + def show_vpp_version_on_all_duts(nodes): + """Show VPP version verbose on all DUTs. + + :param nodes: VPP nodes. + :type nodes: dict + """ + for node in nodes.values(): + if node['type'] == NodeType.DUT: + VPPUtil.vpp_show_version_verbose(node) + + @staticmethod + def vpp_show_interfaces(node): + """Run "show interface" CLI command. + + :param node: Node to run command on. + :type node: dict + """ + vat = VatExecutor() + vat.execute_script("show_interface.vat", node, json_out=False) + + try: + vat.script_should_have_passed() + except AssertionError: + raise RuntimeError('Failed to get VPP interfaces on host: {name}'. + format(name=node['host'])) + + @staticmethod + def vpp_show_crypto_device_mapping(node): + """Run "show crypto device mapping" CLI command. + + :param node: Node to run command on. + :type node: dict + """ + vat = VatExecutor() + vat.execute_script("show_crypto_device_mapping.vat", node, + json_out=False) + + @staticmethod + def vpp_api_trace_dump(node): + """Run "api trace custom-dump" CLI command. + + :param node: Node to run command on. + :type node: dict + """ + vat = VatExecutor() + vat.execute_script("api_trace_dump.vat", node, json_out=False) + + @staticmethod + def vpp_api_trace_save(node): + """Run "api trace save" CLI command. + + :param node: Node to run command on. + :type node: dict + """ + vat = VatExecutor() + vat.execute_script("api_trace_save.vat", node, json_out=False) + + @staticmethod + def vpp_enable_traces_on_dut(node): + """Enable vpp packet traces on the DUT node. + + :param node: DUT node to set up. :type node: dict - :raises RuntimeError: If VPP fails to start. """ + vat = VatExecutor() + vat.execute_script("enable_dpdk_traces.vat", node, json_out=False) + vat.execute_script("enable_vhost_user_traces.vat", node, json_out=False) + vat.execute_script("enable_memif_traces.vat", node, json_out=False) - ssh = SSH() - ssh.connect(node) - cmd = "service vpp start" - ret_code, _, _ = ssh.exec_command_sudo(cmd) - if int(ret_code) != 0: - raise RuntimeError("VPP service did not start.") + @staticmethod + def vpp_enable_traces_on_all_duts(nodes): + """Enable vpp packet traces on all DUTs in the given topology. + + :param nodes: Nodes in the topology. + :type nodes: dict + """ + for node in nodes.values(): + if node['type'] == NodeType.DUT: + VPPUtil.vpp_enable_traces_on_dut(node) |