summaryrefslogtreecommitdiffstats
path: root/test/run_tests.py
diff options
context:
space:
mode:
authorKlement Sekera <ksekera@cisco.com>2017-10-05 10:26:03 +0200
committerDamjan Marion <dmarion.lists@gmail.com>2017-10-10 16:48:38 +0000
commitdf2b980dafe3912267536a8ec5198978702cea4a (patch)
treecbfbdf86257892a25e0a63a56fee6e71e03bf272 /test/run_tests.py
parentac1a7286500c18ac51beb66e4a2b99ea26c8c966 (diff)
make test: add RETRIES option
Change-Id: Ibe31e932bc997f0101a8947e01df90a90d1f100f Signed-off-by: Klement Sekera <ksekera@cisco.com>
Diffstat (limited to 'test/run_tests.py')
-rw-r--r--test/run_tests.py61
1 files changed, 53 insertions, 8 deletions
diff --git a/test/run_tests.py b/test/run_tests.py
index b07a923ade2..df6bf6cbaf4 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -13,14 +13,16 @@ from log import global_logger
from discover_tests import discover_tests
-def test_runner_wrapper(suite, keep_alive_pipe, result_pipe):
+def test_runner_wrapper(suite, keep_alive_pipe, result_pipe, failed_pipe):
result = not VppTestRunner(
- pipe=keep_alive_pipe,
+ keep_alive_pipe=keep_alive_pipe,
+ failed_pipe=failed_pipe,
verbosity=verbose,
failfast=failfast).run(suite).wasSuccessful()
result_pipe.send(result)
result_pipe.close()
keep_alive_pipe.close()
+ failed_pipe.close()
class add_to_suite_callback:
@@ -31,29 +33,54 @@ class add_to_suite_callback:
suite.addTest(cls(method))
+class Filter_by_class_list:
+ def __init__(self, class_list):
+ self.class_list = class_list
+
+ def __call__(self, file_name, class_name, func_name):
+ return class_name in self.class_list
+
+
+def suite_from_failed(suite, failed):
+ filter_cb = Filter_by_class_list(failed)
+ return VppTestRunner.filter_tests(suite, filter_cb)
+
+
def run_forked(suite):
keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
result_parent_end, result_child_end = Pipe(duplex=False)
+ failed_parent_end, failed_child_end = Pipe(duplex=False)
child = Process(target=test_runner_wrapper,
- args=(suite, keep_alive_child_end, result_child_end))
+ args=(suite, keep_alive_child_end, result_child_end,
+ failed_child_end))
child.start()
last_test_temp_dir = None
last_test_vpp_binary = None
last_test = None
result = None
+ failed = set()
while result is None:
readable = select.select([keep_alive_parent_end.fileno(),
result_parent_end.fileno(),
+ failed_parent_end.fileno(),
],
[], [], test_timeout)[0]
+ timeout = True
if result_parent_end.fileno() in readable:
result = result_parent_end.recv()
- elif keep_alive_parent_end.fileno() in readable:
+ timeout = False
+ if keep_alive_parent_end.fileno() in readable:
while keep_alive_parent_end.poll():
last_test, last_test_vpp_binary,\
last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
- else:
+ timeout = False
+ if failed_parent_end.fileno() in readable:
+ while failed_parent_end.poll():
+ failed_test = failed_parent_end.recv()
+ failed.add(failed_test.__name__)
+ timeout = False
+ if timeout:
global_logger.critical("Timeout while waiting for child test "
"runner process (last test running was "
"`%s' in `%s')!" %
@@ -81,7 +108,8 @@ def run_forked(suite):
result = -1
keep_alive_parent_end.close()
result_parent_end.close()
- return result
+ failed_parent_end.close()
+ return result, failed
if __name__ == '__main__':
@@ -114,11 +142,28 @@ if __name__ == '__main__':
suite = unittest.TestSuite()
cb = add_to_suite_callback(suite)
for d in args.dir:
- global_logger.info("Adding tests from directory tree %s" % d)
+ print("Adding tests from directory tree %s" % d)
discover_tests(d, cb)
+ try:
+ retries = int(os.getenv("RETRIES"))
+ except:
+ retries = 0
+ if retries is None:
+ retries = 0
+ attempts = retries + 1
+ if attempts > 1:
+ print("Perform %s attempts to pass the suite..." % attempts)
if debug is None or debug.lower() not in ["gdb", "gdbserver"]:
- sys.exit(run_forked(suite))
+ while True:
+ result, failed = run_forked(suite)
+ attempts = attempts - 1
+ print("%s test(s) failed, %s attempt(s) left" %
+ (len(failed), attempts))
+ if len(failed) > 0 and attempts > 0:
+ suite = suite_from_failed(suite, failed)
+ continue
+ sys.exit(result)
# don't fork if debugging..
sys.exit(not VppTestRunner(verbosity=verbose,