summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorKlement Sekera <ksekera@cisco.com>2017-02-03 07:29:43 +0100
committerDamjan Marion <dmarion.lists@gmail.com>2017-02-06 23:33:52 +0000
commit104543fa6a836df84b5b6d9b1909df3d5226a1e7 (patch)
treebe475e2fc32ea60598a5fbae06d1d6de1b267c04
parentbd69a5f24c6e83e9101f203dd124864fb2877a17 (diff)
make test: improve test filtering
Implement fine-grained test filtering by supporting more complicated filters beside the original file name suffix filter. Change-Id: If5a166d08cffe8c58cc6cf174e6df861c34dbaa6 Signed-off-by: Klement Sekera <ksekera@cisco.com>
-rw-r--r--Makefile2
-rw-r--r--test/Makefile12
-rw-r--r--test/framework.py72
3 files changed, 81 insertions, 5 deletions
diff --git a/Makefile b/Makefile
index b50d203914e..bbbb2acfb7f 100644
--- a/Makefile
+++ b/Makefile
@@ -115,7 +115,7 @@ help:
@echo " startup.conf file is present"
@echo " GDB=<path> - gdb binary to use for debugging"
@echo " PLATFORM=<name> - target platform. default is vpp"
- @echo " TEST=<name> - only run specific test"
+ @echo " TEST=<filter> - apply filter to test set, see test-help"
@echo ""
@echo "Current Argument Values:"
@echo " V = $(V)"
diff --git a/test/Makefile b/test/Makefile
index 7a18be1d587..5c0d48f0f07 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -35,7 +35,7 @@ $(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE)
@touch $@
define retest-func
- @bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python run_tests.py discover -p test_$(TEST)\"*.py\""
+ @bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python run_tests.py discover -p test_\"*.py\""
endef
test: reset verify-python-path $(PAPI_INSTALL_DONE)
@@ -111,7 +111,15 @@ help:
@echo " DEBUG=gdbserver - run gdb inside a gdb server, otherwise "
@echo " same as above"
@echo " STEP=[yes|no] - ease debugging by stepping through a testcase "
- @echo " TEST=<name> - only run specific test"
+ @echo " TEST=<filter> - filter the set of tests:"
+ @echo " by file-name - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py"
+ @echo " by file-suffix - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py"
+ @echo " by wildcard - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'"
+ @echo " e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name"
+ @echo " TEST='bfd.*.*' is equivalent to above example of filter by file-suffix"
+ @echo " TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class"
+ @echo " TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase"
+ @echo " TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes"
@echo ""
@echo "Creating test documentation"
@echo " test-doc - generate documentation for test framework"
diff --git a/test/framework.py b/test/framework.py
index 8ceb33c3a44..889a30469ba 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -698,7 +698,7 @@ class VppTestResult(unittest.TestResult):
class VppTestRunner(unittest.TextTestRunner):
"""
- A basic test runner implementation which prints results on standard error.
+ A basic test runner implementation which prints results to standard error.
"""
@property
def resultclass(self):
@@ -713,6 +713,67 @@ class VppTestRunner(unittest.TextTestRunner):
verbosity, failfast, buffer,
resultclass)
+ test_option = "TEST"
+
+ def parse_test_option(self):
+ try:
+ f = os.getenv(self.test_option)
+ except:
+ f = None
+ filter_file_name = None
+ filter_class_name = None
+ filter_func_name = None
+ if f:
+ if '.' in f:
+ parts = f.split('.')
+ if len(parts) > 3:
+ raise Exception("Unrecognized %s option: %s" %
+ (self.test_option, f))
+ if len(parts) > 2:
+ if parts[2] not in ('*', ''):
+ filter_func_name = parts[2]
+ if parts[1] not in ('*', ''):
+ filter_class_name = parts[1]
+ if parts[0] not in ('*', ''):
+ if parts[0].startswith('test_'):
+ filter_file_name = parts[0]
+ else:
+ filter_file_name = 'test_%s' % parts[0]
+ else:
+ if f.startswith('test_'):
+ filter_file_name = f
+ else:
+ filter_file_name = 'test_%s' % f
+ return filter_file_name, filter_class_name, filter_func_name
+
+ def filter_tests(self, tests, filter_file, filter_class, filter_func):
+ result = unittest.suite.TestSuite()
+ for t in tests:
+ if isinstance(t, unittest.suite.TestSuite):
+ # this is a bunch of tests, recursively filter...
+ x = self.filter_tests(t, filter_file, filter_class,
+ filter_func)
+ if x.countTestCases() > 0:
+ result.addTest(x)
+ elif isinstance(t, unittest.TestCase):
+ # this is a single test
+ parts = t.id().split('.')
+ # t.id() for common cases like this:
+ # test_classifier.TestClassifier.test_acl_ip
+ # apply filtering only if it is so
+ if len(parts) == 3:
+ if filter_file and filter_file != parts[0]:
+ continue
+ if filter_class and filter_class != parts[1]:
+ continue
+ if filter_func and filter_func != parts[2]:
+ continue
+ result.addTest(t)
+ else:
+ # unexpected object, don't touch it
+ result.addTest(t)
+ return result
+
def run(self, test):
"""
Run the tests
@@ -721,4 +782,11 @@ class VppTestRunner(unittest.TextTestRunner):
"""
print("Running tests using custom test runner") # debug message
- return super(VppTestRunner, self).run(test)
+ filter_file, filter_class, filter_func = self.parse_test_option()
+ print("Active filters: file=%s, class=%s, function=%s" % (
+ filter_file, filter_class, filter_func))
+ filtered = self.filter_tests(test, filter_file, filter_class,
+ filter_func)
+ print("%s out of %s tests match specified filters" % (
+ filtered.countTestCases(), test.countTestCases()))
+ return super(VppTestRunner, self).run(filtered)