aboutsummaryrefslogtreecommitdiffstats
path: root/resources
diff options
context:
space:
mode:
authorJuraj Linkeš <juraj.linkes@pantheon.tech>2021-03-23 08:46:16 +0100
committerJuraj Linkeš <juraj.linkes@pantheon.tech>2021-03-24 15:24:17 +0100
commit80b2451e43972ef7a1a177b8632f3bf952f3a8c7 (patch)
treed11f6b67f6c4e0325812926fcacd3bb5a215cd29 /resources
parent7f7ac7dcd4cdf3e3834b7f81efadb8c9b6a080e4 (diff)
FrameworkSetup: catch all exceptions in threads
Any uncaught exception in a thread running setup_node or cleanup_node will prevent the result of that thread's execution from being passed to the main thread. Since the main thread assumes there will be a result for each thread, this leads to false positives. Catch all exceptions in those background threads to fix this. Change-Id: I64013dfc671e576290c8376a58c754ed21b9ba3e Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
Diffstat (limited to 'resources')
-rw-r--r--resources/libraries/python/SetupFramework.py47
1 files changed, 32 insertions, 15 deletions
diff --git a/resources/libraries/python/SetupFramework.py b/resources/libraries/python/SetupFramework.py
index 7e3b36cc77..c27dbe08c2 100644
--- a/resources/libraries/python/SetupFramework.py
+++ b/resources/libraries/python/SetupFramework.py
@@ -17,9 +17,9 @@ supposed to end up here.
"""
from os import environ, remove
-import socket # For catching socket.timeout.
from tempfile import NamedTemporaryFile
import threading
+import traceback
from robot.api import logger
@@ -140,17 +140,19 @@ def create_env_directory_at_node(node):
)
-def setup_node(node, tarball, remote_tarball, results=None):
+def setup_node(node, tarball, remote_tarball, results=None, logs=None):
"""Copy a tarball to a node and extract it.
:param node: A node where the tarball will be copied and extracted.
:param tarball: Local path of tarball to be copied.
:param remote_tarball: Remote path of the tarball.
:param results: A list where to store the result of node setup, optional.
+ :param logs: A list where to store anything that should be logged.
:type node: dict
:type tarball: str
:type remote_tarball: str
:type results: list
+ :type logs: list
:returns: True - success, False - error
:rtype: bool
"""
@@ -159,11 +161,14 @@ def setup_node(node, tarball, remote_tarball, results=None):
extract_tarball_at_node(remote_tarball, node)
if node[u"type"] == NodeType.TG:
create_env_directory_at_node(node)
- except (RuntimeError, socket.timeout) as exc:
- logger.console(
- f"Node {node[u'type']} host {node[u'host']}, port {node[u'port']} "
- f"setup failed, error: {exc!r}"
- )
+ except Exception:
+ # any exception must result in result = False
+ # since this runs in a thread and can't be caught anywhere else
+ err_msg = f"Node {node[u'type']} host {node[u'host']}, " \
+ f"port {node[u'port']} setup failed."
+ logger.console(err_msg)
+ if isinstance(logs, list):
+ logs.append(f"{err_msg} Exception: {traceback.format_exc()}")
result = False
else:
logger.console(
@@ -209,23 +214,26 @@ def delete_framework_dir(node):
)
-def cleanup_node(node, results=None):
+def cleanup_node(node, results=None, logs=None):
"""Delete a tarball from a node.
:param node: A node where the tarball will be delete.
:param results: A list where to store the result of node cleanup, optional.
+ :param logs: A list where to store anything that should be logged.
:type node: dict
:type results: list
+ :type logs: list
:returns: True - success, False - error
:rtype: bool
"""
try:
delete_framework_dir(node)
- except RuntimeError:
- logger.error(
- f"Cleanup of node {node[u'type']} host {node[u'host']}, "
- f"port {node[u'port']} failed."
- )
+ except Exception:
+ err_msg = f"Cleanup of node {node[u'type']} host {node[u'host']}, " \
+ f"port {node[u'port']} failed."
+ logger.console(err_msg)
+ if isinstance(logs, list):
+ logs.append(f"{err_msg} Exception: {traceback.format_exc()}")
result = False
else:
logger.console(
@@ -263,10 +271,11 @@ class SetupFramework:
remote_tarball = f"{tarball}"
results = list()
+ logs = list()
threads = list()
for node in nodes.values():
- args = node, tarball, remote_tarball, results
+ args = node, tarball, remote_tarball, results, logs
thread = threading.Thread(target=setup_node, args=args)
thread.start()
threads.append(thread)
@@ -280,6 +289,9 @@ class SetupFramework:
logger.info(f"Results: {results}")
+ for log in logs:
+ logger.trace(log)
+
delete_local_tarball(tarball)
if all(results):
logger.console(u"All nodes are ready.")
@@ -305,10 +317,12 @@ class CleanupFramework:
"""
results = list()
+ logs = list()
threads = list()
for node in nodes.values():
- thread = threading.Thread(target=cleanup_node, args=(node, results))
+ thread = threading.Thread(target=cleanup_node,
+ args=(node, results, logs))
thread.start()
threads.append(thread)
@@ -321,6 +335,9 @@ class CleanupFramework:
logger.info(f"Results: {results}")
+ for log in logs:
+ logger.trace(log)
+
if all(results):
logger.console(u"All nodes cleaned up.")
else: