From b23ffd7ef216463c35b75c831e6a27e58971f4ec Mon Sep 17 00:00:00 2001 From: Klement Sekera Date: Mon, 31 May 2021 16:08:53 +0200 Subject: tests: make tests less make dependent Implement command line argument parsing instead of passing arguments via environment variables. Add script for running tests without having to invoke make. Deprecate running tests via make. Type: improvement Change-Id: I2e3054a61a2ae25d460e9be00be7d7705fbf943e Signed-off-by: Klement Sekera Signed-off-by: Dave Wallace --- test/run_tests.py | 110 +++++++++++++++++++----------------------------------- 1 file changed, 39 insertions(+), 71 deletions(-) (limited to 'test/run_tests.py') diff --git a/test/run_tests.py b/test/run_tests.py index 5d091ad253f..1d194ad96ca 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -8,22 +8,24 @@ import unittest import argparse import time import threading +import traceback import signal import re from multiprocessing import Process, Pipe, get_context from multiprocessing.queues import Queue from multiprocessing.managers import BaseManager import framework -from framework import VppTestRunner, running_extended_tests, VppTestCase, \ +from config import config, num_cpus, available_cpus, max_vpp_cpus +from framework import VppTestRunner, VppTestCase, \ get_testcase_doc_name, get_test_description, PASS, FAIL, ERROR, SKIP, \ TEST_RUN, SKIP_CPU_SHORTAGE from debug import spawn_gdb, start_vpp_in_gdb from log import get_parallel_logger, double_line_delim, RED, YELLOW, GREEN, \ colorize, single_line_delim from discover_tests import discover_tests +import sanity_run_vpp from subprocess import check_output, CalledProcessError from util import check_core_path, get_core_path, is_core_present -from cpu_config import num_cpus, max_vpp_cpus, available_cpus # timeout which controls how long the child has to finish after seeing # a core dump in test temporary directory. If this is exceeded, parent assumes @@ -126,9 +128,9 @@ def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue, VppTestCase.parallel_handler = logger.handlers[0] result = VppTestRunner(keep_alive_pipe=keep_alive_pipe, descriptions=descriptions, - verbosity=verbose, + verbosity=config.verbose, result_pipe=result_pipe, - failfast=failfast, + failfast=config.failfast, print_summary=False).run(suite) finished_pipe.send(result.wasSuccessful()) finished_pipe.close() @@ -241,8 +243,7 @@ def handle_failed_suite(logger, last_test_temp_dir, vpp_pid): if last_test_temp_dir: # Need to create link in case of a timeout or core dump without failure lttd = os.path.basename(last_test_temp_dir) - failed_dir = os.getenv('FAILED_DIR') - link_path = '%s%s-FAILED' % (failed_dir, lttd) + link_path = '%s%s-FAILED' % (config.failed_dir, lttd) if not os.path.exists(link_path): os.symlink(last_test_temp_dir, link_path) logger.error("Symlink to failed testcase directory: %s -> %s" @@ -272,8 +273,7 @@ def handle_failed_suite(logger, last_test_temp_dir, vpp_pid): except Exception as e: logger.exception("Unexpected error running `file' utility " "on core-file") - logger.error("gdb %s %s" % - (os.getenv('VPP_BIN', 'vpp'), core_path)) + logger.error(f"gdb {config.vpp_bin} {core_path}") if vpp_pid: # Copy api post mortem @@ -292,7 +292,7 @@ def check_and_handle_core(vpp_binary, tempdir, core_crash_test): print(single_line_delim) spawn_gdb(vpp_binary, get_core_path(tempdir)) print(single_line_delim) - elif compress_core: + elif config.compress_core: print("Compressing core-file in test directory `%s'" % tempdir) os.system("gzip %s" % get_core_path(tempdir)) @@ -312,7 +312,7 @@ def process_finished_testsuite(wrapped_testcase_suite, results.append(wrapped_testcase_suite.result) finished_testcase_suites.add(wrapped_testcase_suite) stop_run = False - if failfast and not wrapped_testcase_suite.was_successful(): + if config.failfast and not wrapped_testcase_suite.was_successful(): stop_run = True if not wrapped_testcase_suite.was_successful(): @@ -421,7 +421,7 @@ def run_forked(testcase_suites): continue fail = False - if wrapped_testcase_suite.last_heard + test_timeout < \ + if wrapped_testcase_suite.last_heard + config.timeout < \ time.time(): fail = True wrapped_testcase_suite.logger.critical( @@ -515,7 +515,7 @@ def run_forked(testcase_suites): raise finally: read_from_testcases.clear() - stdouterr_thread.join(test_timeout) + stdouterr_thread.join(config.timeout) manager.shutdown() handle_cores(failed_wrapped_testcases) @@ -566,11 +566,8 @@ class SplitToSuitesCallback: self.filtered.addTest(test_method) -test_option = "TEST" - - -def parse_test_option(): - f = os.getenv(test_option, None) +def parse_test_filter(test_filter): + f = test_filter filter_file_name = None filter_class_name = None filter_func_name = None @@ -807,46 +804,32 @@ def parse_results(results): return return_code, results_per_suite.rerun -def parse_digit_env(env_var, default): - value = os.getenv(env_var, default) - if value != default: - if value.isdigit(): - value = int(value) - else: - print('WARNING: unsupported value "%s" for env var "%s",' - 'defaulting to %s' % (value, env_var, default)) - value = default - return value - - if __name__ == '__main__': - verbose = parse_digit_env("V", 0) + print(f"Config is: {config}") - test_timeout = parse_digit_env("TIMEOUT", 600) # default = 10 minutes + if config.sanity: + print("Running sanity test case.") + try: + rc = sanity_run_vpp.main() + if rc != 0: + sys.exit(rc) + except Exception as e: + print(traceback.format_exc()) + print("Couldn't run sanity test case.") + sys.exit(-1) test_finished_join_timeout = 15 - retries = parse_digit_env("RETRIES", 0) + debug_gdb = config.debug in ["gdb", "gdbserver", "attach"] + debug_core = config.debug == "core" - debug = os.getenv("DEBUG", "n").lower() in ["gdb", "gdbserver", "attach"] - - debug_core = os.getenv("DEBUG", "").lower() == "core" - compress_core = framework.BoolEnvironmentVariable("CORE_COMPRESS") - - if os.getenv("VPP_IN_GDB", "n").lower() in ["1", "y", "yes"]: - start_vpp_in_gdb() - exit() - - step = framework.BoolEnvironmentVariable("STEP") - force_foreground = framework.BoolEnvironmentVariable("FORCE_FOREGROUND") - - run_interactive = debug or step or force_foreground + run_interactive = debug_gdb or config.step or config.force_foreground max_concurrent_tests = 0 print(f"OS reports {num_cpus} available cpu(s).") - test_jobs = os.getenv("TEST_JOBS", "1").lower() # default = 1 process + test_jobs = config.jobs if test_jobs == 'auto': if run_interactive: max_concurrent_tests = 1 @@ -856,15 +839,7 @@ if __name__ == '__main__': print(f"Running at most {max_concurrent_tests} python test " "processes concurrently.") else: - try: - test_jobs = int(test_jobs) - except ValueError as e: - raise ValueError("Invalid TEST_JOBS value specified, valid " - "values are a positive integer or 'auto'") from e - if test_jobs <= 0: - raise ValueError("Invalid TEST_JOBS value specified, valid " - "values are a positive integer or 'auto'") - max_concurrent_tests = int(test_jobs) + max_concurrent_tests = test_jobs print(f"Running at most {max_concurrent_tests} python test processes " "concurrently as set by 'TEST_JOBS'.") @@ -876,27 +851,20 @@ if __name__ == '__main__': 'STEP is set) in parallel (TEST_JOBS is more than 1) is not ' 'supported') - parser = argparse.ArgumentParser(description="VPP unit tests") - parser.add_argument("-f", "--failfast", action='store_true', - help="fast failure flag") - parser.add_argument("-d", "--dir", action='append', type=str, - help="directory containing test files " - "(may be specified multiple times)") - args = parser.parse_args() - failfast = args.failfast descriptions = True print("Running tests using custom test runner.") - filter_file, filter_class, filter_func = parse_test_option() + filter_file, filter_class, filter_func = \ + parse_test_filter(config.filter) - print("Active filters: file=%s, class=%s, function=%s" % ( + print("Selected filters: file=%s, class=%s, function=%s" % ( filter_file, filter_class, filter_func)) filter_cb = FilterByTestOption(filter_file, filter_class, filter_func) - ignore_path = os.getenv("VENV_PATH", None) + ignore_path = config.venv_dir cb = SplitToSuitesCallback(filter_cb) - for d in args.dir: + for d in config.test_src_dir: print("Adding tests from directory tree %s" % d) discover_tests(d, cb, ignore_path) @@ -925,10 +893,10 @@ if __name__ == '__main__': print("%s out of %s tests match specified filters" % ( tests_amount, tests_amount + cb.filtered.countTestCases())) - if not running_extended_tests: + if not config.extended: print("Not running extended tests (some tests will be skipped)") - attempts = retries + 1 + attempts = config.retries + 1 if attempts > 1: print("Perform %s attempts to pass the suite..." % attempts) @@ -945,8 +913,8 @@ if __name__ == '__main__': suite.assign_cpus([]) cpu_shortage = True full_suite.addTests(suites) - result = VppTestRunner(verbosity=verbose, - failfast=failfast, + result = VppTestRunner(verbosity=config.verbose, + failfast=config.failfast, print_summary=True).run(full_suite) was_successful = result.wasSuccessful() if not was_successful: -- cgit 1.2.3-korg