tests: support multiple filter expressions

Support multiple comma-delimited filter expressions,
e.g. to run both bfd and ip4 tests, it's now possible to do:

make test TEST=bfd,ip4

Same goes for wildcards, e.g.:

make test TEST=bfd,..test_longest_prefix_match,..test_icmp_error

Type: improvement
Change-Id: I0cceaa443cb612dca955f301c7407959f9a71a6e
Signed-off-by: Klement Sekera <klement.sekera@gmail.com>

1 files changed, 159 insertions, 46 deletions

diff --git a/test/Makefile b/test/Makefile
index e5e997583db..44187408440 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -404,52 +404,165 @@ help:
 	@echo " test-shell-debug       - enter shell with test environment (debug build)"
 	@echo " test-refresh-deps      - refresh the Python dependencies for the tests"
 	@echo ""
-	@echo "Arguments controlling test runs:"
-	@echo ""
-	@echo " V=[0|1|2]              - set test verbosity level"
-	@echo "                          0=ERROR, 1=INFO, 2=DEBUG"
-	@echo " TEST_JOBS=[<n>|auto]   - use at most <n> parallel python processes for test execution, if auto, set to number of available cpus (default: 1)"
-	@echo " MAX_VPP_CPUS=[<n>|auto]- use at most <n> cpus for running vpp main and worker threads, if auto, set to number of available cpus (default: auto)"
-	@echo " CACHE_OUTPUT=[0|n|no]  - disable cache VPP stdout/stderr and log as one block after test finishes (default: yes)"
-	@echo " FAILFAST=[1|y|yes]     - fail fast if 1, otherwise complete all tests"
-	@echo " TIMEOUT=<timeout>      - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)"
-	@echo " RETRIES=<n>            - retry failed tests <n> times"
-	@echo " DEBUG=<type>           - set VPP debugging kind"
-	@echo "    DEBUG=core          - detect coredump and load it in gdb on crash"
-	@echo "    DEBUG=gdb           - allow easy debugging by printing VPP PID"
-	@echo "                          and waiting for user input before running"
-	@echo "                          and tearing down a testcase"
-	@echo "    DEBUG=gdbserver     - run gdb inside a gdb server, otherwise"
-	@echo "                          same as above"
-	@echo "    DEBUG=attach        - attach test case to already running vpp in gdb (see test-start-vpp-in-gdb)"
-	@echo " STEP=[1|y|yes]         - enable stepping through a testcase (for testcase debugging)"
-	@echo " SANITY=[0|n|no]        - disable sanity import of vpp-api/sanity vpp run before running tests"
-	@echo " EXTENDED_TESTS=[1|y|yes] - run extended tests"
-	@echo " TEST=<filter>          - filter the set of tests:"
-	@echo "    by file-name        - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py"
-	@echo "    by file-suffix      - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py"
-	@echo "    by wildcard         - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'"
-	@echo "                          e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name"
-	@echo "                               TEST='bfd.*.*' is equivalent to above example of filter by file-suffix"
-	@echo "                               TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class"
-	@echo "                               TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase"
-	@echo "                               TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes"
-	@echo " VARIANT=<variant>      - specify which march node variant to unit test"
-	@echo "                          e.g. VARIANT=skx test the skx march variants"
-	@echo "                          e.g. VARIANT=icl test the icl march variants"
-	@echo " COREDUMP_SIZE=<size>   - pass <size> as unix { coredump-size <size> } argument to vpp"
-	@echo "                          e.g. COREDUMP_SIZE=4g"
-	@echo "                               COREDUMP_SIZE=unlimited"
-	@echo " COREDUMP_COMPRESS=[1|y|yes] - compress core files if not debugging them"
-	@echo " EXTERN_TESTS=<path>    - path to out-of-tree test_<name>.py files containing test cases"
-	@echo " EXTERN_PLUGINS=<path>  - path to out-of-tree plugins to be loaded by vpp under test"
-	@echo " EXTERN_COV_DIR=<path>  - path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report"
-	@echo " PROFILE=[1|y|yes]      - enable profiling of test framework via cProfile module"
-	@echo " PROFILE_SORT_BY=opt    - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime)"
-	@echo " PROFILE_OUTPUT=file    - output profiling info to file - use absolute path (default: stdout)"
-	@echo " TEST_DEBUG=[1|y|yes]   - enable debugging of the test framework itself (expert)"
-	@echo " API_FUZZ=[1|y|yes]     - enable VPP api fuzz testing"
-	@echo " RND_SEED=<seed>        - Seed RND with given seed"
+	@echo "Environment variables controlling test runs:"
+	@echo ""
+	@echo "   V=[0|1|2]"
+	@echo "	      set test verbosity level: 0=ERROR, 1=INFO, 2=DEBUG"
+	@echo ""
+	@echo "   TEST_JOBS=[<n>|auto]"
+	@echo "       use at most <n> parallel python processes for test"
+	@echo "       execution, if auto, set to number of available cpus"
+	@echo "       (default: 1)"
+	@echo ""
+	@echo "   MAX_VPP_CPUS=[<n>|auto]"
+	@echo "       use at most <n> cpus for running vpp"
+	@echo "       'auto' sets to number of available cpus"
+	@echo "       (default: auto)"
+	@echo ""
+	@echo "   CACHE_OUTPUT=[0|n|no]"
+	@echo "       disable caching VPP stdout/stderr and logging it"
+	@echo "       as one block after test finishes"
+	@echo "       (default: yes)"
+	@echo ""
+	@echo "   FAILFAST=[1|y|yes]"
+	@echo "       if enabled, stop running tests on first failure"
+	@echo "       otherwise finish running whole suite"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   TIMEOUT=<timeout>"
+	@echo "       fail test suite if any single test takes longer"
+	@echo "       than <timeout> (in seconds) to finish"
+	@echo "       (default: 600)"
+	@echo ""
+	@echo "   RETRIES=<n>"
+	@echo "       retry failed tests <n> times"
+	@echo "       (default: 0)"
+	@echo ""
+	@echo "   DEBUG=<type>"
+	@echo "       configure VPP debugging:"
+	@echo "       DEBUG=core"
+	@echo "           detect coredump and load it in gdb on crash"
+	@echo ""
+	@echo "       DEBUG=gdb"
+	@echo "           print VPP PID and wait for user input before"
+	@echo "           running and tearing down a testcase, allowing"
+	@echo "           easy gdb attach"
+	@echo ""
+	@echo "       DEBUG=gdbserver"
+	@echo "           same as above, but run gdb inside a gdb server"
+	@echo ""
+	@echo "       DEBUG=attach"
+	@echo "           attach to existing vpp in running in gdb"
+	@echo "           (see test-start-vpp-in-gdb)"
+	@echo "       (default: none)"
+	@echo ""
+	@echo "   STEP=[1|y|yes]"
+	@echo "       enable stepping through a testcase"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   SANITY=[0|n|no]"
+	@echo "       disable sanity import of vpp-api/vpp sanity"
+	@echo "       run before running tests"
+	@echo "       (default: yes)"
+	@echo ""
+	@echo "   EXTENDED_TESTS=[1|y|yes]"
+	@echo "       run extended tests"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   TEST=<filter>,[<filter>],..."
+	@echo "       only run tests matching one or more comma-delimited"
+	@echo "       filter expressions"
+	@echo ""
+	@echo "       simple filter:"
+	@echo "           file name or file suffix select all tests from a file"
+	@echo "           examples:"
+	@echo "               TEST=test_bfd"
+	@echo "               TEST=bfd"
+	@echo "                    equivalent expressions selecting all"
+	@echo "                    tests defined in test_bfd.py"
+	@echo ""
+	@echo "       wildcard filter:"
+	@echo "           advanced filtering based on test file, test class"
+	@echo "           and test function"
+	@echo "           each filter expression is in the form of"
+	@echo "               <file>.<class>.<test function>"
+	@echo "           each of the tokens can be left empty or replaced"
+	@echo "           with '*' to select all objects available"
+	@echo "           examples:"
+	@echo "               TEST=test_bfd.*.*"
+	@echo "               TEST=test_bfd.."
+	@echo "               TEST=bfd.*.*"
+	@echo "               TEST=bfd.."
+	@echo "                    select all tests defined in test_bfd.py"
+	@echo "               TEST=bfd.BFDAPITestCase.*"
+	@echo "               TEST=bfd.BFDAPITestCase."
+	@echo "                    select all tests from test_bfd.py"
+	@echo "                    which are part of BFDAPITestCase class"
+	@echo "               TEST=bfd.BFDAPITestCase.test_add_bfd"
+	@echo "                    select a single test named test_add_bfd"
+	@echo "                    from test_bfd.py/BFDAPITestCase"
+	@echo "               TEST=..test_add_bfd"
+	@echo "               TEST=*.*.test_add_bfd"
+	@echo "                    select all test functions named test_add_bfd"
+	@echo "                    from all files/classes"
+	@echo "               TEST=bfd,ip4,..test_icmp_error"
+	@echo "                    select all test functions in test_bfd.py,"
+	@echo "                    test_ip4.py and all test functions named"
+	@echo "                    'test_icmp_error' in all files"
+	@echo "       (default: '')"
+	@echo ""
+	@echo "   VARIANT=<variant>"
+	@echo "       specify which march node variant to unit test"
+	@echo "           e.g. VARIANT=skx test the skx march variants"
+	@echo "           e.g. VARIANT=icl test the icl march variants"
+	@echo "       (default: '')"
+	@echo ""
+	@echo "   COREDUMP_SIZE=<size>"
+	@echo "       pass <size> as unix { coredump-size <size> } argument"
+	@echo "       to vpp, e.g. COREDUMP_SIZE=4g or COREDUMP_SIZE=unlimited"
+	@echo "       (default: '')"
+	@echo ""
+	@echo "   COREDUMP_COMPRESS=[1|y|yes]"
+	@echo "       if no debug option is set, compress any core files"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   EXTERN_TESTS=<path>"
+	@echo "       include out-of-tree test_*.py files under <path>"
+	@echo "       (default: '')"
+	@echo ""
+	@echo "   EXTERN_PLUGINS=<path>"
+	@echo "       load out-of-tree vpp plugins in <path>"
+	@echo "       (default: '')"
+	@echo ""
+	@echo "   EXTERN_COV_DIR=<path>"
+	@echo "       path to out-of-tree prefix, where source, object"
+	@echo "       and .gcda files can be found for coverage report"
+	@echo "       (default: '')"
+	@echo ""
+	@echo "   PROFILE=[1|y|yes]"
+	@echo "       enable profiling of test framework via cProfile module"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   PROFILE_SORT_BY=opt"
+	@echo "       sort profiling report by opt - see cProfile documentation"
+	@echo "       for possible values"
+	@echo "       (default: cumtime)"
+	@echo ""
+	@echo "   PROFILE_OUTPUT=file"
+	@echo "       output profiling info to file - use absolute path"
+	@echo "       (default: stdout)"
+	@echo ""
+	@echo "   TEST_DEBUG=[1|y|yes]"
+	@echo "       enable debugging of the test framework itself (expert)"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   API_FUZZ=[1|y|yes]"
+	@echo "       enable VPP api fuzz testing"
+	@echo "       (default: no)"
+	@echo ""
+	@echo "   RND_SEED=<seed>"
+	@echo "       random seed used by test framework"
+	@echo "       (default: time.time())"
 	@echo ""
 	@echo "Starting VPP in GDB for use with DEBUG=attach:"
 	@echo ""