make test: support out-of-tree tests

env EXTERN_TESTS="/path/to/extra/tests" make test

causes to run the default test set and tests collected from
test_*.py files under subtree specified in EXTERN_TESTS.

Change-Id: I58c5471dd6010730278a5b47d4318737d920bc28
Signed-off-by: Klement Sekera <ksekera@cisco.com>

2 files changed, 58 insertions, 4 deletions

diff --git a/test/Makefile b/test/Makefile
index 4338e09675e..c65eaae6bfe 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -23,10 +23,15 @@ verify-no-running-vpp:
 		false; \
 	fi
 
-UNITTEST_EXTRA_OPTS=""
+UNITTEST_EXTRA_OPTS=
+UNITTEST_FAILFAST_OPTS=
 
 ifeq ($(FAILFAST),1)
-UNITTEST_EXTRA_OPTS="-f"
+UNITTEST_EXTRA_OPTS=-f
+endif
+
+ifneq ($(EXTERN_TESTS),)
+UNITTEST_EXTRA_OPTS=$(UNITTEST_FAILFAST_OPTS) -d $(EXTERN_TESTS)
 endif
 
 PYTHON_VENV_PATH=$(VPP_PYTHON_PREFIX)/virtualenv
@@ -59,7 +64,7 @@ $(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE)
 	@touch $@
 
 define retest-func
-	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python run_tests.py discover $(UNITTEST_EXTRA_OPTS) -p test_\"*.py\""
+	@bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python run_tests.py -d $(TEST_DIR) $(UNITTEST_EXTRA_OPTS)"
 endef
 
 .PHONY: sanity
diff --git a/test/run_tests.py b/test/run_tests.py
index 8f2174b1463..39b093685da 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -1,12 +1,61 @@
 #!/usr/bin/env python
 
+import sys
 import os
 import unittest
+import argparse
+import importlib
 from framework import VppTestRunner
 
+
+def add_from_dir(suite, directory):
+    do_insert = True
+    for _f in os.listdir(directory):
+        f = "%s/%s" % (directory, _f)
+        if os.path.isdir(f):
+            add_from_dir(suite, f)
+            continue
+        if not os.path.isfile(f):
+            continue
+        if do_insert:
+            sys.path.insert(0, directory)
+            do_insert = False
+        if not _f.startswith("test_") or not _f.endswith(".py"):
+            continue
+        name = "".join(f.split("/")[-1].split(".")[:-1])
+        if name in sys.modules:
+            raise Exception("Duplicate test module `%s' found!" % name)
+        module = importlib.import_module(name)
+        for name, cls in module.__dict__.items():
+            if not isinstance(cls, type):
+                continue
+            if not issubclass(cls, unittest.TestCase):
+                continue
+            if name == "VppTestCase":
+                continue
+            for method in dir(cls):
+                if not callable(getattr(cls, method)):
+                    continue
+                if method.startswith("test_"):
+                    suite.addTest(cls(method))
+
 if __name__ == '__main__':
     try:
         verbose = int(os.getenv("V", 0))
     except:
         verbose = 0
-    unittest.main(testRunner=VppTestRunner, module=None, verbosity=verbose)
+
+    parser = argparse.ArgumentParser(description="VPP unit tests")
+    parser.add_argument("-f", "--failfast", action='count',
+                        help="fast failure flag")
+    parser.add_argument("-d", "--dir", action='append', type=str,
+                        help="directory containing test files "
+                             "(may be specified multiple times)")
+    args = parser.parse_args()
+    failfast = True if args.failfast == 1 else False
+
+    suite = unittest.TestSuite()
+    for d in args.dir:
+        print("Adding tests from directory tree %s" % d)
+        add_from_dir(suite, d)
+    VppTestRunner(verbosity=verbose, failfast=failfast).run(suite)