diff options
| -rw-r--r-- | test/Makefile | 8 | ||||
| -rw-r--r-- | test/config.py | 15 | ||||
| -rw-r--r-- | test/run_tests.py | 32 |
3 files changed, 52 insertions, 3 deletions
diff --git a/test/Makefile b/test/Makefile index 0a1b770963a..e26bf7f5a0f 100644 --- a/test/Makefile +++ b/test/Makefile @@ -276,7 +276,7 @@ endif EXTRA_ARGS=$(ARG0) $(ARG1) $(ARG2) $(ARG3) $(ARG4) $(ARG5) $(ARG6) $(ARG7) $(ARG8) $(ARG9) $(ARG10) $(ARG11) $(ARG12) $(ARG13) $(ARG14) $(ARG15) $(ARG16) $(ARG17) $(ARG18) $(ARG19) -RUN_TESTS_ARGS=--failed-dir=$(FAILED_DIR) --verbose=$(V) --jobs=$(TEST_JOBS) --filter=$(TEST) --retries=$(RETRIES) --venv-dir=$(VENV_PATH) --vpp-ws-dir=$(WS_ROOT) --vpp-tag=$(TAG) --rnd-seed=$(RND_SEED) --vpp-worker-count="$(VPP_WORKER_COUNT)" --keep-pcaps $(PLUGIN_PATH_ARGS) $(EXC_PLUGINS_ARG) $(TEST_PLUGIN_PATH_ARGS) $(EXTRA_ARGS) +RUN_TESTS_ARGS=--failed-dir=$(FAILED_DIR) --verbose=$(V) --jobs=$(TEST_JOBS) --filter=$(TEST) --skip-filter=$(SKIP_TESTS) --retries=$(RETRIES) --venv-dir=$(VENV_PATH) --vpp-ws-dir=$(WS_ROOT) --vpp-tag=$(TAG) --rnd-seed=$(RND_SEED) --vpp-worker-count="$(VPP_WORKER_COUNT)" --keep-pcaps $(PLUGIN_PATH_ARGS) $(EXC_PLUGINS_ARG) $(TEST_PLUGIN_PATH_ARGS) $(EXTRA_ARGS) RUN_SCRIPT_ARGS=--python-opts=$(PYTHON_OPTS) define retest-func @@ -606,6 +606,12 @@ help: @echo " 'test_icmp_error' in all files" @echo " (default: '')" @echo "" + @echo " SKIP_TESTS=<filter>,[<filter>],..." + @echo " Skip tests matching one or more comma-delimited" + @echo " filter expressions, even if they were selected by TEST" + @echo "" + @echo " (default: '')" + @echo "" @echo " VARIANT=<variant>" @echo " specify which march node variant to unit test" @echo " e.g. VARIANT=skx test the skx march variants" diff --git a/test/config.py b/test/config.py index e939f188c6c..2870c55df05 100644 --- a/test/config.py +++ b/test/config.py @@ -151,6 +151,21 @@ parser.add_argument( "--filter", action="store", metavar="FILTER_EXPRESSION", help=filter_help_string ) +skip_filter_help_string = """\ +expression consists of one or more filters separated by commas (',') +filter consists of 3 string selectors separated by dots ('.') + +The syntax is identical to the expression used to select the tests, +except this one one has the effect to skip the tests that match it. +""" + +parser.add_argument( + "--skip-filter", + action="store", + metavar="SKIP_FILTER_EXPR", + help=skip_filter_help_string, +) + default_retries = 0 parser.add_argument( diff --git a/test/run_tests.py b/test/run_tests.py index 66e0ee4db39..166f5d3cd48 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -685,8 +685,9 @@ def filter_tests(tests, filter_cb): class FilterByTestOption: - def __init__(self, filters): + def __init__(self, filters, skip_filters): self.filters = filters + self.skip_filters = skip_filters def __call__(self, file_name, class_name, func_name): def test_one( @@ -716,6 +717,21 @@ class FilterByTestOption: class_name, func_name, ): + for ( + x_filter_file_name, + x_filter_class_name, + x_filter_func_name, + ) in self.skip_filters: + if test_one( + x_filter_file_name, + x_filter_class_name, + x_filter_func_name, + file_name, + class_name, + func_name, + ): + # If the test matches the excluded tests, do not run it + return False return True return False @@ -976,6 +992,11 @@ if __name__ == "__main__": print("Running tests using custom test runner.") filters = [(parse_test_filter(f)) for f in config.filter.split(",")] + skip_filters = [(parse_test_filter(f)) for f in config.skip_filter.split(",")] + # Remove the (None, None, None) triplet that gets pushed by default + skip_filters = [ + triplet for triplet in skip_filters if triplet != (None, None, None) + ] print( "Selected filters: ", @@ -984,8 +1005,15 @@ if __name__ == "__main__": for filter_file, filter_class, filter_func in filters ), ) + print( + "Selected skip filters: ", + "|".join( + f"file={filter_file}, class={filter_class}, function={filter_func}" + for filter_file, filter_class, filter_func in skip_filters + ), + ) - filter_cb = FilterByTestOption(filters) + filter_cb = FilterByTestOption(filters, skip_filters) cb = SplitToSuitesCallback(filter_cb) for d in config.test_src_dir: |