summaryrefslogtreecommitdiffstats
path: root/test/run_tests.py
diff options
context:
space:
mode:
authorKlement Sekera <ksekera@cisco.com>2017-08-08 04:33:53 +0200
committerDave Wallace <dwallacelf@gmail.com>2017-08-10 00:14:59 +0000
commit909a6a1eb92ceacb983bcff088fb512352929c46 (patch)
tree0c99faec79006ae9c4e7802e992d698d342b88be /test/run_tests.py
parent52851e6aa9304054fd1059c8dd284abf8e532bf2 (diff)
make test: detect hung tests
Run tests in a forked process with a set of pipes to communicate keep-alives and overall result. This allows us to detect when e.g. vpp dies mid-API call causing the test to hang waiting for response (which will never come since vpp died). Support setting a (per test case) TIMEOUT make test option to set timeout, with a default timeout of 120 seconds. Example - fail the test suite if any test-case fails to finish within 300s: make TIMEOUT=300 test Change-Id: I0d04f26a7232968f4bf043adf5d5b508f5018717 Signed-off-by: Klement Sekera <ksekera@cisco.com>
Diffstat (limited to 'test/run_tests.py')
-rw-r--r--test/run_tests.py72
1 files changed, 69 insertions, 3 deletions
diff --git a/test/run_tests.py b/test/run_tests.py
index 1b9c677df26..6d477d880f3 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -2,10 +2,14 @@
import sys
import os
+import select
import unittest
import argparse
import importlib
+from multiprocessing import Process, Pipe
from framework import VppTestRunner
+from debug import spawn_gdb
+from log import global_logger
def add_from_dir(suite, directory):
@@ -39,12 +43,38 @@ def add_from_dir(suite, directory):
if method.startswith("test_"):
suite.addTest(cls(method))
+
+def test_runner_wrapper(keep_alive_pipe, result_pipe):
+ result = not VppTestRunner(
+ pipe=keep_alive_pipe,
+ verbosity=verbose,
+ failfast=failfast).run(suite).wasSuccessful()
+ result_pipe.send(result)
+ result_pipe.close()
+ keep_alive_pipe.close()
+
+
+def handle_core(vpp_binary, core_path):
+ try:
+ d = os.getenv("DEBUG")
+ except:
+ d = None
+ if d and d.lower() == "core":
+ spawn_gdb(vpp_binary, core_path, global_logger)
+
+
if __name__ == '__main__':
try:
verbose = int(os.getenv("V", 0))
except:
verbose = 0
+ default_test_timeout = 120
+ try:
+ test_timeout = int(os.getenv("TIMEOUT", default_test_timeout))
+ except:
+ test_timeout = default_test_timeout
+
parser = argparse.ArgumentParser(description="VPP unit tests")
parser.add_argument("-f", "--failfast", action='count',
help="fast failure flag")
@@ -56,7 +86,43 @@ if __name__ == '__main__':
suite = unittest.TestSuite()
for d in args.dir:
- print("Adding tests from directory tree %s" % d)
+ global_logger.info("Adding tests from directory tree %s" % d)
add_from_dir(suite, d)
- sys.exit(not VppTestRunner(verbosity=verbose,
- failfast=failfast).run(suite).wasSuccessful())
+ keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
+ result_parent_end, result_child_end = Pipe(duplex=False)
+
+ p = Process(target=test_runner_wrapper,
+ args=(keep_alive_child_end,
+ result_child_end))
+ p.start()
+ last_test_temp_dir = None
+ last_test_vpp_binary = None
+ last_test = None
+ result = None
+ while result is None:
+ readable = select.select([keep_alive_parent_end.fileno(),
+ result_parent_end.fileno(),
+ ],
+ [], [], test_timeout)[0]
+ if result_parent_end.fileno() in readable:
+ result = result_parent_end.recv()
+ elif keep_alive_parent_end.fileno() in readable:
+ while keep_alive_parent_end.poll():
+ last_test, last_test_vpp_binary, last_test_temp_dir =\
+ keep_alive_parent_end.recv()
+ else:
+ global_logger.critical("Timeout while waiting for child test "
+ "runner process (last test running was "
+ "`%s' in `%s')!" %
+ (last_test, last_test_temp_dir))
+ if last_test_temp_dir and last_test_vpp_binary:
+ core_path = "%s/core" % last_test_temp_dir
+ if os.path.isfile(core_path):
+ global_logger.error("Core-file exists in test temporary "
+ "directory: %s!" % core_path)
+ handle_core(last_test_vpp_binary, core_path)
+ p.terminate()
+ result = -1
+ keep_alive_parent_end.close()
+ result_parent_end.close()
+ sys.exit(result)