diff options
Diffstat (limited to 'resources/libraries/python/SetupFramework.py')
| -rw-r--r-- | resources/libraries/python/SetupFramework.py | 82 |
1 files changed, 55 insertions, 27 deletions
diff --git a/resources/libraries/python/SetupFramework.py b/resources/libraries/python/SetupFramework.py index 45447e923b..95ca8a7d51 100644 --- a/resources/libraries/python/SetupFramework.py +++ b/resources/libraries/python/SetupFramework.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2023 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -19,6 +19,7 @@ supposed to end up here. from os import environ, remove from tempfile import NamedTemporaryFile import threading +import traceback from robot.api import logger @@ -55,7 +56,8 @@ def pack_framework_dir(): run( [ u"tar", u"--sparse", u"--exclude-vcs", u"--exclude=output*.xml", - u"--exclude=./tmp", u"-zcf", file_name, u"." + u"--exclude=./tmp", u"--exclude=./env", u"--exclude=./.git", + u"-zcf", file_name, u"." ], msg=u"Could not pack testing framework" ) @@ -104,7 +106,7 @@ def extract_tarball_at_node(tarball, node): node, cmd, message=f"Failed to extract {tarball} at node {node[u'type']} " f"host {node[u'host']}, port {node[u'port']}", - timeout=30, include_reason=True + timeout=600, include_reason=True ) logger.console( f"Extracting tarball to {con.REMOTE_FW_DIR} on {node[u'type']} " @@ -115,9 +117,13 @@ def extract_tarball_at_node(tarball, node): def create_env_directory_at_node(node): """Create fresh virtualenv to a directory, install pip requirements. + Return stdout and stderr of the command, + so we see which installs are behaving weird (e.g. attempting download). + :param node: Node to create virtualenv on. :type node: dict - :returns: nothing + :returns: Stdout and stderr. + :rtype: str, str :raises RuntimeError: When failed to setup virtualenv. """ logger.console( @@ -126,9 +132,10 @@ def create_env_directory_at_node(node): ) cmd = f"cd {con.REMOTE_FW_DIR} && rm -rf env && virtualenv " \ f"-p $(which python3) --system-site-packages --never-download env " \ - f"&& source env/bin/activate && pip3 install -r requirements.txt" - exec_cmd_no_error( - node, cmd, timeout=100, include_reason=True, + f"&& source env/bin/activate && ANSIBLE_SKIP_CONFLICT_CHECK=1 " \ + f"pip3 install -r requirements.txt" + stdout, stderr = exec_cmd_no_error( + node, cmd, timeout=300, include_reason=True, message=f"Failed install at node {node[u'type']} host {node[u'host']}, " f"port {node[u'port']}" ) @@ -136,19 +143,22 @@ def create_env_directory_at_node(node): f"Virtualenv setup on {node[u'type']} host {node[u'host']}, " f"port {node[u'port']} done." ) + return stdout, stderr -def setup_node(node, tarball, remote_tarball, results=None): +def setup_node(node, tarball, remote_tarball, results=None, logs=None): """Copy a tarball to a node and extract it. :param node: A node where the tarball will be copied and extracted. :param tarball: Local path of tarball to be copied. :param remote_tarball: Remote path of the tarball. :param results: A list where to store the result of node setup, optional. + :param logs: A list where to store anything that should be logged. :type node: dict :type tarball: str :type remote_tarball: str :type results: list + :type logs: list :returns: True - success, False - error :rtype: bool """ @@ -156,12 +166,18 @@ def setup_node(node, tarball, remote_tarball, results=None): copy_tarball_to_node(tarball, node) extract_tarball_at_node(remote_tarball, node) if node[u"type"] == NodeType.TG: - create_env_directory_at_node(node) - except RuntimeError as exc: - logger.console( - f"Node {node[u'type']} host {node[u'host']}, port {node[u'port']} " - f"setup failed, error: {exc!r}" - ) + stdout, stderr = create_env_directory_at_node(node) + if isinstance(logs, list): + logs.append(f"{node[u'host']} Env stdout: {stdout}") + logs.append(f"{node[u'host']} Env stderr: {stderr}") + except Exception: + # any exception must result in result = False + # since this runs in a thread and can't be caught anywhere else + err_msg = f"Node {node[u'type']} host {node[u'host']}, " \ + f"port {node[u'port']} setup failed." + logger.console(err_msg) + if isinstance(logs, list): + logs.append(f"{err_msg} Exception: {traceback.format_exc()}") result = False else: logger.console( @@ -199,7 +215,7 @@ def delete_framework_dir(node): node, f"sudo rm -rf {con.REMOTE_FW_DIR}", message=f"Framework delete failed at node {node[u'type']} " f"host {node[u'host']}, port {node[u'port']}", - timeout=100, include_reason=True + timeout=100, include_reason=True, ) logger.console( f"Deleting framework directory on {node[u'type']} host {node[u'host']}," @@ -207,23 +223,26 @@ def delete_framework_dir(node): ) -def cleanup_node(node, results=None): +def cleanup_node(node, results=None, logs=None): """Delete a tarball from a node. :param node: A node where the tarball will be delete. :param results: A list where to store the result of node cleanup, optional. + :param logs: A list where to store anything that should be logged. :type node: dict :type results: list + :type logs: list :returns: True - success, False - error :rtype: bool """ try: delete_framework_dir(node) - except RuntimeError: - logger.error( - f"Cleanup of node {node[u'type']} host {node[u'host']}, " - f"port {node[u'port']} failed." - ) + except Exception: + err_msg = f"Cleanup of node {node[u'type']} host {node[u'host']}, " \ + f"port {node[u'port']} failed." + logger.console(err_msg) + if isinstance(logs, list): + logs.append(f"{err_msg} Exception: {traceback.format_exc()}") result = False else: logger.console( @@ -240,9 +259,9 @@ def cleanup_node(node, results=None): class SetupFramework: """Setup suite run on topology nodes. - Many VAT/CLI based tests need the scripts at remote hosts before executing - them. This class packs the whole testing directory and copies it over - to all nodes in topology under /tmp/ + Some tests need the scripts at remote hosts before executing them. + This class packs the whole testing directory and copies it over + to all nodes in topology under /tmp/. """ @staticmethod @@ -261,16 +280,17 @@ class SetupFramework: remote_tarball = f"{tarball}" results = list() + logs = list() threads = list() for node in nodes.values(): - args = node, tarball, remote_tarball, results + args = node, tarball, remote_tarball, results, logs thread = threading.Thread(target=setup_node, args=args) thread.start() threads.append(thread) logger.info( - f"Executing node setups in parallel, waiting for threads to end" + u"Executing node setups in parallel, waiting for threads to end." ) for thread in threads: @@ -278,6 +298,9 @@ class SetupFramework: logger.info(f"Results: {results}") + for log in logs: + logger.trace(log) + delete_local_tarball(tarball) if all(results): logger.console(u"All nodes are ready.") @@ -303,10 +326,12 @@ class CleanupFramework: """ results = list() + logs = list() threads = list() for node in nodes.values(): - thread = threading.Thread(target=cleanup_node, args=(node, results)) + thread = threading.Thread(target=cleanup_node, + args=(node, results, logs)) thread.start() threads.append(thread) @@ -319,6 +344,9 @@ class CleanupFramework: logger.info(f"Results: {results}") + for log in logs: + logger.trace(log) + if all(results): logger.console(u"All nodes cleaned up.") else: |