[TEST] - automatic test run

- skript find all test file
- start test function from test file

Change-Id: I3b37247c960afa6bf788cd14f1d8d240af3100c6
Signed-off-by: Andrej Kozemcak <andrej.kozemcak@pantheon.tech>

8 files changed, 167 insertions, 31 deletions

diff --git a/Makefile b/Makefile
index 564d084..d3801f7 100644
--- a/Makefile
+++ b/Makefile
@@ -249,7 +249,7 @@ build-plugins:
 	@# NEW INSTRUCTIONS TO BUILD-PLUGINS MUST BE DECLARED ON A NEW LINE WITH '@'
 
 test-plugins:
-	@test/run_test.py
+	@test/run_test.py --dir ./test/
 
 build-gnmi:
 	@mkdir -p $(BR)/build-gnmi/; cd $(BR)/build-gnmi/; \
diff --git a/test/framework.py b/test/framework.py
index 5c890dc..bd42b48 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -19,6 +19,7 @@ import unittest
 
 from topology import Topology
 import vppctl
+import sys
 
 
 class SweetcombTestCase(unittest.TestCase):
@@ -39,11 +40,71 @@ class SweetcombTestCase(unittest.TestCase):
         cls.netopeer_cli = cls.topology.get_netopeer_cli()
         cls.vppctl = vppctl.Vppctl()
 
-    def check_response(self, resps, expected_result, checks):
-        assert resps[1] == expected_result
+    def runTest(self):
+        pass
+
+
+class SweetcombTestResult(unittest.TestResult):
+
+    def __init__(self, stream=None, descriptions=None, verbosity=None,
+                 runner=None):
+        """
+        :param stream File descriptor to store where to report test results.
+            Set to the standard error stream by default.
+        :param descriptions Boolean variable to store information if to use
+            test case descriptions.
+        :param verbosity Integer variable to store required verbosity level.
+        """
+        super(SweetcombTestResult, self).__init__(stream, descriptions, verbosity)
+        self.stream = stream
+        self.descriptions = descriptions
+        self.verbosity = verbosity
+        self.result_string = None
+        self.runner = runner
+
+
+class SweetcombTestRunner(unittest.TextTestRunner):
+    """
+    A basic test runner implementation which prints results to standard error.
+    """
+
+    @property
+    def resultclass(self):
+        return SweetcombTestResult
+
+    def __init__(self, keep_alive_pipe=None, descriptions=True, verbosity=1,
+                 result_pipe=None, failfast=False, buffer=False,
+                 resultclass=None, print_summary=True, **kwargs):
+        # ignore stream setting here, use hard-coded stdout to be in sync
+        # with prints from VppTestCase methods ...
+        super(SweetcombTestRunner, self).__init__(sys.stdout, descriptions,
+                                            verbosity, failfast, buffer,
+                                            resultclass, **kwargs)
+        #KeepAliveReporter.pipe = keep_alive_pipe
+
+        self.orig_stream = self.stream
+        self.resultclass.test_framework_result_pipe = result_pipe
+
+        self.print_summary = print_summary
+
+    def _makeResult(self):
+        return self.resultclass(self.stream,
+                                self.descriptions,
+                                self.verbosity,
+                                self)
+
+    def run(self, test):
+        """
+        Run the tests
+
+        :param test:
+
+        """
+
+        result = super(SweetcombTestRunner, self).run(test)
+        if not self.print_summary:
+            self.stream = self.orig_stream
+            result.stream = self.orig_stream
+        return result
+
 
-        for key, val in checks.items():
-            for resp in resps:
-                r = str(resp).strip()
-                if r.find("<"+key+">") == 0:
-                    assert r[r.find("<"+key+">")+len("<"+key+">"):r.rfind("</"+key+">")] == val
diff --git a/test/run_test.py b/test/run_test.py
index a4cce6d..7acd152 100755
--- a/test/run_test.py
+++ b/test/run_test.py
@@ -19,23 +19,92 @@
 
 import unittest
 import util
+import argparse
+import os
+import importlib
+import sys
+import fnmatch
 
-from framework import SweetcombTestCase
-from test_ietf_interfaces import TestIetfInterfaces
-from test_oc_interfaces import TestOcInterfaces
 
+from framework import SweetcombTestCase, SweetcombTestRunner
 
-def suite():
-    suite = unittest.TestSuite()
-    suite.addTest(TestIetfInterfaces('test_ipv4'))
-    suite.addTest(TestIetfInterfaces('test_interface'))
-    suite.addTest(TestOcInterfaces('test_interface'))
-    suite.addTest(TestOcInterfaces('test_interface_ipv4'))
-    return suite
+
+class SplitToSuitesCallback:
+    def __init__(self):
+        self.suites = {}
+        self.suite_name = 'default'
+
+    def __call__(self, file_name, cls, method):
+        test_method = cls(method)
+
+        self.suite_name = file_name + cls.__name__
+        if self.suite_name not in self.suites:
+            self.suites[self.suite_name] = unittest.TestSuite()
+        self.suites[self.suite_name].addTest(test_method)
+
+
+def discover_tests(directory, callback, ignore_path):
+    do_insert = True
+    for _f in os.listdir(directory):
+        f = "%s/%s" % (directory, _f)
+        if os.path.isdir(f):
+            if ignore_path is not None and f.startswith(ignore_path):
+                continue
+            discover_tests(f, callback, ignore_path)
+            continue
+        if not os.path.isfile(f):
+            continue
+        if do_insert:
+            sys.path.insert(0, directory)
+            do_insert = False
+        if not _f.startswith("test_") or not _f.endswith(".py"):
+            continue
+        name = "".join(f.split("/")[-1].split(".")[:-1])
+        module = importlib.import_module(name)
+        for name, cls in module.__dict__.items():
+            if not isinstance(cls, type):
+                continue
+            if not issubclass(cls, unittest.TestCase):
+                continue
+            if name == "SweetcombTestCase" or name.startswith("Template"):
+                continue
+
+            for method in dir(cls):
+                if not callable(getattr(cls, method)):
+                    continue
+                if method.startswith("test_"):
+                    callback(_f, cls, method)
 
 
 if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Sweetcomb tests")
+    parser.add_argument("-d", "--dir", action='append', type=str,
+                        help="directory containing test files "
+                             "(may be specified multiple times)")
+    args = parser.parse_args()
+
     util.import_yang_modules()
-    runner = unittest.TextTestRunner()
-    runner.run(suite())
+
+    ddir = list()
+    if args.dir is None:
+        ddir.append(os.getcwd())
+    else:
+        ddir = args.dir
+
+    cb = SplitToSuitesCallback()
+
+    ignore_path = 'conf'
+    for d in ddir:
+        print("Adding tests from directory tree {}".format(d))
+        discover_tests(d, cb, ignore_path)
+
+    suites = []
+    for testcase_suite in cb.suites.values():
+        suites.append(testcase_suite)
+
+    full_suite = unittest.TestSuite()
+    #map(full_suite.addTests, suites)
+    for suite in suites:
+        full_suite.addTests(suite)
+    result = SweetcombTestRunner(print_summary=True).run(full_suite)
 
diff --git a/test/test_ietf_interfaces.py b/test/test_ietf_interfaces.py
index 3b08859..a4ebf78 100644
--- a/test/test_ietf_interfaces.py
+++ b/test/test_ietf_interfaces.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 #
 # Copyright (c) 2019 PANTHEON.tech.
 # Copyright (c) 2019 Cisco and/or its affiliates.
@@ -18,7 +19,7 @@
 import unittest
 
 import util
-from framework import SweetcombTestCase
+from framework import SweetcombTestCase, SweetcombTestRunner
 from ydk.models.ietf import ietf_interfaces
 from ydk.models.ietf import iana_if_type
 from ydk.services import CRUDService
@@ -116,3 +117,7 @@ class TestIetfInterfaces(SweetcombTestCase):
         a = self.vppctl.show_address(name)
 
         self.assertIsNone(a)
+
+
+if __name__ == '__main__':
+    unittest.main(testRunner=SweetcombTestRunner)
diff --git a/test/test_oc_interfaces.py b/test/test_oc_interfaces.py
index 3b94945..be87eed 100644
--- a/test/test_oc_interfaces.py
+++ b/test/test_oc_interfaces.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 #
 # Copyright (c) 2019 PANTHEON.tech.
 # Copyright (c) 2019 Cisco and/or its affiliates.
@@ -18,7 +19,7 @@
 import unittest
 
 import util
-from framework import SweetcombTestCase
+from framework import SweetcombTestCase, SweetcombTestRunner
 from ydk.models.openconfig import openconfig_interfaces
 from ydk.models.ietf import iana_if_type
 from ydk.services import CRUDService
@@ -48,8 +49,8 @@ class TestOcInterfaces(SweetcombTestCase):
 
         try:
             crud_service.create(self.netopeer_cli, interface)
-        except YError:
-            print("Error create services")
+        except YError as err:
+            print("Error create services: {}".format(err))
 
         p = self.vppctl.show_interface(name)
         self.assertIsNotNone(p)
@@ -92,3 +93,6 @@ class TestOcInterfaces(SweetcombTestCase):
             assert()
 
         a = self.vppctl.show_address(name)
+
+if __name__ == '__main__':
+    unittest.main(testRunner=SweetcombTestRunner)
diff --git a/test/topology.py b/test/topology.py
index a8a2f15..26dc622 100644
--- a/test/topology.py
+++ b/test/topology.py
@@ -25,6 +25,7 @@ import psutil
 import time
 from ydk.providers import NetconfServiceProvider
 
+
 class Topology:
     debug = False
 
@@ -68,7 +69,6 @@ class Topology:
         ip.addr('add', index=vpp1, address='192.168.0.2', prefixlen=24)
         ip.addr('add', index=vpp2, address='192.168.1.2', prefixlen=24)
 
-
     def _start_sysrepo(self):
         print("Start sysrepo deamon.")
         #TODO: Need property close.
diff --git a/test/util.py b/test/util.py
index 576bc43..0d4ba65 100644
--- a/test/util.py
+++ b/test/util.py
@@ -27,13 +27,11 @@ def ping(ip):
 def import_yang_modules():
     print("Import YANG models to sysrepo.")
 
-    #directory = '/root/src/sweetcomb/'
-    directory = './'
-    os.chdir(directory)
+    directory = os.getcwd()
     subprocess.run(["make", "uninstall-models"])
     subprocess.run(["make", "install-models"])
 
-    os.chdir(directory + "test/conf/")
+    os.chdir(directory + "/test/conf/")
     print("Import configuration to sysrepo datastore.")
     subprocess.run(["sysrepocfg", "--import=ietf-interfaces.xml",
                     "--datastore=startup", "--format=xml", "--leve=0",
diff --git a/test/vpp_controler.py b/test/vpp_controler.py
index 32d6d70..b29f00e 100644
--- a/test/vpp_controler.py
+++ b/test/vpp_controler.py
@@ -28,12 +28,11 @@ class Vpp_controler:
         self.cmd = "vpp"
         self.ccmd = "vppctl"
         self.rootPath = os.getcwd()
-        self.configuration = self.rootPath + "/vpp.conf"
+        self.configuration = self.rootPath + "/test/conf/vpp.conf"
         self.process = None
         self.debug = debug
 
     def __del__(self):
-        #self.kill()
         self.terminate()
 
     def _default_conf_vpp(self):