tests: support multiple filter expressions

Support multiple comma-delimited filter expressions,
e.g. to run both bfd and ip4 tests, it's now possible to do:

make test TEST=bfd,ip4

Same goes for wildcards, e.g.:

make test TEST=bfd,..test_longest_prefix_match,..test_icmp_error

Type: improvement
Change-Id: I0cceaa443cb612dca955f301c7407959f9a71a6e
Signed-off-by: Klement Sekera <klement.sekera@gmail.com>

3 files changed, 202 insertions, 64 deletions

diff --git a/test/Makefile b/test/Makefile
index e5e997583db..44187408440 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -404,52 +404,165 @@ help:
 	@echo " test-shell-debug       - enter shell with test environment (debug build)"
 	@echo " test-refresh-deps      - refresh the Python dependencies for the tests"
 	@echo ""
-	@echo "Arguments controlling test runs:"
-	@echo ""
-	@echo " V=[0|1|2]              - set test verbosity level"
-	@echo "                          0=ERROR, 1=INFO, 2=DEBUG"
-	@echo " TEST_JOBS=[<n>|auto]   - use at most <n> parallel python processes for test execution, if auto, set to number of available cpus (default: 1)"
-	@echo " MAX_VPP_CPUS=[<n>|auto]- use at most <n> cpus for running vpp main and worker threads, if auto, set to number of available cpus (default: auto)"
-	@echo " CACHE_OUTPUT=[0|n|no]  - disable cache VPP stdout/stderr and log as one block after test finishes (default: yes)"
-	@echo " FAILFAST=[1|y|yes]     - fail fast if 1, otherwise complete all tests"
-	@echo " TIMEOUT=<timeout>      - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)"
-	@echo " RETRIES=<n>            - retry failed tests <n> times"
-	@echo " DEBUG=<type>           - set VPP debugging kind"
-	@echo "    DEBUG=core          - detect coredump and load it in gdb on crash"
-	@echo "    DEBUG=gdb           - allow easy debugging by printing VPP PID"
-	@echo "                          and waiting for user input before running"
-	@echo "                          and tearing down a testcase"
-	@echo "    DEBUG=gdbserver     - run gdb inside a gdb server, otherwise"
-	@echo "                          same as above"
-	@echo "    DEBUG=attach        - attach test case to already running vpp in gdb (see test-start-vpp-in-gdb)"
-	@echo " STEP=[1|y|yes]         - enable stepping through a testcase (for testcase debugging)"
-	@echo " SANITY=[0|n|no]        - disable sanity import of vpp-api/sanity vpp run before running tests"
-	@echo " EXTENDED_TESTS=[1|y|yes] - run extended tests"
-	@echo " TEST=<filter>          - filter the set of tests:"
-	@echo "    by file-name        - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py"
-	@echo "    by file-suffix      - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py"
-	@echo "    by wildcard         - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'"
-	@echo "                          e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name"
-	@echo "                               TEST='bfd.*.*' is equivalent to above example of filter by file-suffix"
-	@echo "                               TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class"
-	@echo "                               TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase"
-	@echo "                               TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes"
-	@echo " VARIANT=<variant>      - specify which march node variant to unit test"
-	@echo "                          e.g. VARIANT=skx test the skx march variants"
-	@echo "                          e.g. VARIANT=icl test the icl march variants"
-	@echo " COREDUMP_SIZE=<size>   - pass <size> as unix { coredump-size <size> } argument to vpp"
-	@echo "                          e.g. COREDUMP_SIZE=4g"
-	@echo "                               COREDUMP_SIZE=unlimited"
-	@echo " COREDUMP_COMPRESS=[1|y|yes] - compress core files if not debugging them"
-	@echo " EXTERN_TESTS=<path>    - path to out-of-tree test_<name>.py files containing test cases"
-	@echo " EXTERN_PLUGINS=<path>  - path to out-of-tree plugins to be loaded by vpp under test"
-	@echo " EXTERN_COV_DIR=<path>  - path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report"
-	@echo " PROFILE=[1|y|yes]      - enable profiling of test framework via cProfile module"
-	@echo " PROFILE_SORT_BY=opt    - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime)"
-	@echo " PROFILE_OUTPUT=file    - output profiling info to file - use absolute path (default: stdout)"
-	@echo " TEST_DEBUG=[1|y|yes]   - enable debugging of the test framework itself (expert)"
-	@echo " API_FUZZ=[1|y|yes]     - enable VPP api fuzz testing"
-	@echo " RND_SEED=<seed>        - Seed RND with given seed"
+	@echo "Environment variables controlling test runs:"
+	@echo ""
+	@echo "   V=[0|1|2]"
+	@echo "	      set test verbosity level: 0=ERROR, 1=INFO, 2=DEBUG"
+	@echo ""
+	@echo "   TEST_JOBS=[<n>|auto]"
+	@echo "       use at most <n> parallel python processes for test"
+	@echo "       execution, if auto, set to number of available cpus"
+	@echo "       (default: 1)"
+	@echo ""
+	@echo "   MAX_VPP_CPUS=[<n>|auto]"
+	@echo "       use at most <n> cpus for running vpp"
+	@echo "       'auto' sets to number of available cpus"
+	@echo "       (default: auto)"
+	@echo ""
+	@echo "   CACHE_OUTPUT=[0|n|no]"
+	@echo "       disable caching VPP stdout/stderr and logging it"
+	@echo "       as one block after test finishes"
+	@echo "       (default: yes)"
+	@echo ""
+	@echo "   FAILFAST=[1|y|yes]"
+	@echo "       if enabled, stop running tests on first failure"
+	@echo "       otherwise finish running whole suite"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   TIMEOUT=<timeout>"
+	@echo "       fail test suite if any single test takes longer"
+	@echo "       than <timeout> (in seconds) to finish"
+	@echo "       (default: 600)"
+	@echo ""
+	@echo "   RETRIES=<n>"
+	@echo "       retry failed tests <n> times"
+	@echo "       (default: 0)"
+	@echo ""
+	@echo "   DEBUG=<type>"
+	@echo "       configure VPP debugging:"
+	@echo "       DEBUG=core"
+	@echo "           detect coredump and load it in gdb on crash"
+	@echo ""
+	@echo "       DEBUG=gdb"
+	@echo "           print VPP PID and wait for user input before"
+	@echo "           running and tearing down a testcase, allowing"
+	@echo "           easy gdb attach"
+	@echo ""
+	@echo "       DEBUG=gdbserver"
+	@echo "           same as above, but run gdb inside a gdb server"
+	@echo ""
+	@echo "       DEBUG=attach"
+	@echo "           attach to existing vpp in running in gdb"
+	@echo "           (see test-start-vpp-in-gdb)"
+	@echo "       (default: none)"
+	@echo ""
+	@echo "   STEP=[1|y|yes]"
+	@echo "       enable stepping through a testcase"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   SANITY=[0|n|no]"
+	@echo "       disable sanity import of vpp-api/vpp sanity"
+	@echo "       run before running tests"
+	@echo "       (default: yes)"
+	@echo ""
+	@echo "   EXTENDED_TESTS=[1|y|yes]"
+	@echo "       run extended tests"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   TEST=<filter>,[<filter>],..."
+	@echo "       only run tests matching one or more comma-delimited"
+	@echo "       filter expressions"
+	@echo ""
+	@echo "       simple filter:"
+	@echo "           file name or file suffix select all tests from a file"
+	@echo "           examples:"
+	@echo "               TEST=test_bfd"
+	@echo "               TEST=bfd"
+	@echo "                    equivalent expressions selecting all"
+	@echo "                    tests defined in test_bfd.py"
+	@echo ""
+	@echo "       wildcard filter:"
+	@echo "           advanced filtering based on test file, test class"
+	@echo "           and test function"
+	@echo "           each filter expression is in the form of"
+	@echo "               <file>.<class>.<test function>"
+	@echo "           each of the tokens can be left empty or replaced"
+	@echo "           with '*' to select all objects available"
+	@echo "           examples:"
+	@echo "               TEST=test_bfd.*.*"
+	@echo "               TEST=test_bfd.."
+	@echo "               TEST=bfd.*.*"
+	@echo "               TEST=bfd.."
+	@echo "                    select all tests defined in test_bfd.py"
+	@echo "               TEST=bfd.BFDAPITestCase.*"
+	@echo "               TEST=bfd.BFDAPITestCase."
+	@echo "                    select all tests from test_bfd.py"
+	@echo "                    which are part of BFDAPITestCase class"
+	@echo "               TEST=bfd.BFDAPITestCase.test_add_bfd"
+	@echo "                    select a single test named test_add_bfd"
+	@echo "                    from test_bfd.py/BFDAPITestCase"
+	@echo "               TEST=..test_add_bfd"
+	@echo "               TEST=*.*.test_add_bfd"
+	@echo "                    select all test functions named test_add_bfd"
+	@echo "                    from all files/classes"
+	@echo "               TEST=bfd,ip4,..test_icmp_error"
+	@echo "                    select all test functions in test_bfd.py,"
+	@echo "                    test_ip4.py and all test functions named"
+	@echo "                    'test_icmp_error' in all files"
+	@echo "       (default: '')"
+	@echo ""
+	@echo "   VARIANT=<variant>"
+	@echo "       specify which march node variant to unit test"
+	@echo "           e.g. VARIANT=skx test the skx march variants"
+	@echo "           e.g. VARIANT=icl test the icl march variants"
+	@echo "       (default: '')"
+	@echo ""
+	@echo "   COREDUMP_SIZE=<size>"
+	@echo "       pass <size> as unix { coredump-size <size> } argument"
+	@echo "       to vpp, e.g. COREDUMP_SIZE=4g or COREDUMP_SIZE=unlimited"
+	@echo "       (default: '')"
+	@echo ""
+	@echo "   COREDUMP_COMPRESS=[1|y|yes]"
+	@echo "       if no debug option is set, compress any core files"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   EXTERN_TESTS=<path>"
+	@echo "       include out-of-tree test_*.py files under <path>"
+	@echo "       (default: '')"
+	@echo ""
+	@echo "   EXTERN_PLUGINS=<path>"
+	@echo "       load out-of-tree vpp plugins in <path>"
+	@echo "       (default: '')"
+	@echo ""
+	@echo "   EXTERN_COV_DIR=<path>"
+	@echo "       path to out-of-tree prefix, where source, object"
+	@echo "       and .gcda files can be found for coverage report"
+	@echo "       (default: '')"
+	@echo ""
+	@echo "   PROFILE=[1|y|yes]"
+	@echo "       enable profiling of test framework via cProfile module"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   PROFILE_SORT_BY=opt"
+	@echo "       sort profiling report by opt - see cProfile documentation"
+	@echo "       for possible values"
+	@echo "       (default: cumtime)"
+	@echo ""
+	@echo "   PROFILE_OUTPUT=file"
+	@echo "       output profiling info to file - use absolute path"
+	@echo "       (default: stdout)"
+	@echo ""
+	@echo "   TEST_DEBUG=[1|y|yes]"
+	@echo "       enable debugging of the test framework itself (expert)"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   API_FUZZ=[1|y|yes]"
+	@echo "       enable VPP api fuzz testing"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   RND_SEED=<seed>"
+	@echo "       random seed used by test framework"
+	@echo "       (default: time.time())"
 	@echo ""
 	@echo "Starting VPP in GDB for use with DEBUG=attach:"
 	@echo ""
diff --git a/test/config.py b/test/config.py
index d2f14c82e19..578cc40fa2a 100644
--- a/test/config.py
+++ b/test/config.py
@@ -122,7 +122,8 @@ parser.add_argument(
 )
 
 filter_help_string = """\
-expression consists of 3 string selectors separated by '.' separators:
+expression consists of one or more filters separated by commas (',')
+filter consists of 3 string selectors separated by dots ('.')
 
     <file>.<class>.<function>
 
@@ -142,6 +143,8 @@ examples:
    test_add_bfd from test_bfd.py/BFDAPITestCase
 4. '.*.test_add_bfd' selects all test functions named test_add_bfd
    from all files/classes
+5. 'bfd,ip4,..test_icmp_error' selects all test functions in test_bfd.py,
+   test_ip4.py and all test functions named 'test_icmp_error' in all files
 """
 parser.add_argument(
     "--filter", action="store", metavar="FILTER_EXPRESSION", help=filter_help_string
diff --git a/test/run_tests.py b/test/run_tests.py
index 85344ca15e2..3bbf1bc3153 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -631,7 +631,7 @@ def parse_test_filter(test_filter):
         if "." in f:
             parts = f.split(".")
             if len(parts) > 3:
-                raise Exception("Unrecognized %s option: %s" % (test_option, f))
+                raise Exception(f"Invalid test filter: {test_filter}")
             if len(parts) > 2:
                 if parts[2] not in ("*", ""):
                     filter_func_name = parts[2]
@@ -677,21 +677,40 @@ def filter_tests(tests, filter_cb):
 
 
 class FilterByTestOption:
-    def __init__(self, filter_file_name, filter_class_name, filter_func_name):
-        self.filter_file_name = filter_file_name
-        self.filter_class_name = filter_class_name
-        self.filter_func_name = filter_func_name
+    def __init__(self, filters):
+        self.filters = filters
 
     def __call__(self, file_name, class_name, func_name):
-        if self.filter_file_name:
-            fn_match = fnmatch.fnmatch(file_name, self.filter_file_name)
-            if not fn_match:
+        def test_one(
+            filter_file_name,
+            filter_class_name,
+            filter_func_name,
+            file_name,
+            class_name,
+            func_name,
+        ):
+            if filter_file_name:
+                fn_match = fnmatch.fnmatch(file_name, filter_file_name)
+                if not fn_match:
+                    return False
+            if filter_class_name and class_name != filter_class_name:
                 return False
-        if self.filter_class_name and class_name != self.filter_class_name:
-            return False
-        if self.filter_func_name and func_name != self.filter_func_name:
-            return False
-        return True
+            if filter_func_name and func_name != filter_func_name:
+                return False
+            return True
+
+        for filter_file_name, filter_class_name, filter_func_name in self.filters:
+            if test_one(
+                filter_file_name,
+                filter_class_name,
+                filter_func_name,
+                file_name,
+                class_name,
+                func_name,
+            ):
+                return True
+
+        return False
 
 
 class FilterByClassList:
@@ -938,14 +957,17 @@ if __name__ == "__main__":
     descriptions = True
 
     print("Running tests using custom test runner.")
-    filter_file, filter_class, filter_func = parse_test_filter(config.filter)
+    filters = [(parse_test_filter(f)) for f in config.filter.split(",")]
 
     print(
-        "Selected filters: file=%s, class=%s, function=%s"
-        % (filter_file, filter_class, filter_func)
+        "Selected filters: ",
+        "|".join(
+            f"file={filter_file}, class={filter_class}, function={filter_func}"
+            for filter_file, filter_class, filter_func in filters
+        ),
     )
 
-    filter_cb = FilterByTestOption(filter_file, filter_class, filter_func)
+    filter_cb = FilterByTestOption(filters)
 
     cb = SplitToSuitesCallback(filter_cb)
     for d in config.test_src_dir: