From e77922662052f0caec4129ee43ab9d176b806769 Mon Sep 17 00:00:00 2001 From: Andrej Kozemcak Date: Thu, 30 May 2019 13:53:07 +0200 Subject: [TEST] - automatic test run - skript find all test file - start test function from test file Change-Id: I3b37247c960afa6bf788cd14f1d8d240af3100c6 Signed-off-by: Andrej Kozemcak --- Makefile | 2 +- test/framework.py | 75 +++++++++++++++++++++++++++++++---- test/run_test.py | 93 ++++++++++++++++++++++++++++++++++++++------ test/test_ietf_interfaces.py | 7 +++- test/test_oc_interfaces.py | 10 +++-- test/topology.py | 2 +- test/util.py | 6 +-- test/vpp_controler.py | 3 +- 8 files changed, 167 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 564d084..d3801f7 100644 --- a/Makefile +++ b/Makefile @@ -249,7 +249,7 @@ build-plugins: @# NEW INSTRUCTIONS TO BUILD-PLUGINS MUST BE DECLARED ON A NEW LINE WITH '@' test-plugins: - @test/run_test.py + @test/run_test.py --dir ./test/ build-gnmi: @mkdir -p $(BR)/build-gnmi/; cd $(BR)/build-gnmi/; \ diff --git a/test/framework.py b/test/framework.py index 5c890dc..bd42b48 100644 --- a/test/framework.py +++ b/test/framework.py @@ -19,6 +19,7 @@ import unittest from topology import Topology import vppctl +import sys class SweetcombTestCase(unittest.TestCase): @@ -39,11 +40,71 @@ class SweetcombTestCase(unittest.TestCase): cls.netopeer_cli = cls.topology.get_netopeer_cli() cls.vppctl = vppctl.Vppctl() - def check_response(self, resps, expected_result, checks): - assert resps[1] == expected_result + def runTest(self): + pass + + +class SweetcombTestResult(unittest.TestResult): + + def __init__(self, stream=None, descriptions=None, verbosity=None, + runner=None): + """ + :param stream File descriptor to store where to report test results. + Set to the standard error stream by default. + :param descriptions Boolean variable to store information if to use + test case descriptions. + :param verbosity Integer variable to store required verbosity level. + """ + super(SweetcombTestResult, self).__init__(stream, descriptions, verbosity) + self.stream = stream + self.descriptions = descriptions + self.verbosity = verbosity + self.result_string = None + self.runner = runner + + +class SweetcombTestRunner(unittest.TextTestRunner): + """ + A basic test runner implementation which prints results to standard error. + """ + + @property + def resultclass(self): + return SweetcombTestResult + + def __init__(self, keep_alive_pipe=None, descriptions=True, verbosity=1, + result_pipe=None, failfast=False, buffer=False, + resultclass=None, print_summary=True, **kwargs): + # ignore stream setting here, use hard-coded stdout to be in sync + # with prints from VppTestCase methods ... + super(SweetcombTestRunner, self).__init__(sys.stdout, descriptions, + verbosity, failfast, buffer, + resultclass, **kwargs) + #KeepAliveReporter.pipe = keep_alive_pipe + + self.orig_stream = self.stream + self.resultclass.test_framework_result_pipe = result_pipe + + self.print_summary = print_summary + + def _makeResult(self): + return self.resultclass(self.stream, + self.descriptions, + self.verbosity, + self) + + def run(self, test): + """ + Run the tests + + :param test: + + """ + + result = super(SweetcombTestRunner, self).run(test) + if not self.print_summary: + self.stream = self.orig_stream + result.stream = self.orig_stream + return result + - for key, val in checks.items(): - for resp in resps: - r = str(resp).strip() - if r.find("<"+key+">") == 0: - assert r[r.find("<"+key+">")+len("<"+key+">"):r.rfind("")] == val diff --git a/test/run_test.py b/test/run_test.py index a4cce6d..7acd152 100755 --- a/test/run_test.py +++ b/test/run_test.py @@ -19,23 +19,92 @@ import unittest import util +import argparse +import os +import importlib +import sys +import fnmatch -from framework import SweetcombTestCase -from test_ietf_interfaces import TestIetfInterfaces -from test_oc_interfaces import TestOcInterfaces +from framework import SweetcombTestCase, SweetcombTestRunner -def suite(): - suite = unittest.TestSuite() - suite.addTest(TestIetfInterfaces('test_ipv4')) - suite.addTest(TestIetfInterfaces('test_interface')) - suite.addTest(TestOcInterfaces('test_interface')) - suite.addTest(TestOcInterfaces('test_interface_ipv4')) - return suite + +class SplitToSuitesCallback: + def __init__(self): + self.suites = {} + self.suite_name = 'default' + + def __call__(self, file_name, cls, method): + test_method = cls(method) + + self.suite_name = file_name + cls.__name__ + if self.suite_name not in self.suites: + self.suites[self.suite_name] = unittest.TestSuite() + self.suites[self.suite_name].addTest(test_method) + + +def discover_tests(directory, callback, ignore_path): + do_insert = True + for _f in os.listdir(directory): + f = "%s/%s" % (directory, _f) + if os.path.isdir(f): + if ignore_path is not None and f.startswith(ignore_path): + continue + discover_tests(f, callback, ignore_path) + continue + if not os.path.isfile(f): + continue + if do_insert: + sys.path.insert(0, directory) + do_insert = False + if not _f.startswith("test_") or not _f.endswith(".py"): + continue + name = "".join(f.split("/")[-1].split(".")[:-1]) + module = importlib.import_module(name) + for name, cls in module.__dict__.items(): + if not isinstance(cls, type): + continue + if not issubclass(cls, unittest.TestCase): + continue + if name == "SweetcombTestCase" or name.startswith("Template"): + continue + + for method in dir(cls): + if not callable(getattr(cls, method)): + continue + if method.startswith("test_"): + callback(_f, cls, method) if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Sweetcomb tests") + parser.add_argument("-d", "--dir", action='append', type=str, + help="directory containing test files " + "(may be specified multiple times)") + args = parser.parse_args() + util.import_yang_modules() - runner = unittest.TextTestRunner() - runner.run(suite()) + + ddir = list() + if args.dir is None: + ddir.append(os.getcwd()) + else: + ddir = args.dir + + cb = SplitToSuitesCallback() + + ignore_path = 'conf' + for d in ddir: + print("Adding tests from directory tree {}".format(d)) + discover_tests(d, cb, ignore_path) + + suites = [] + for testcase_suite in cb.suites.values(): + suites.append(testcase_suite) + + full_suite = unittest.TestSuite() + #map(full_suite.addTests, suites) + for suite in suites: + full_suite.addTests(suite) + result = SweetcombTestRunner(print_summary=True).run(full_suite) diff --git a/test/test_ietf_interfaces.py b/test/test_ietf_interfaces.py index 3b08859..a4ebf78 100644 --- a/test/test_ietf_interfaces.py +++ b/test/test_ietf_interfaces.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # # Copyright (c) 2019 PANTHEON.tech. # Copyright (c) 2019 Cisco and/or its affiliates. @@ -18,7 +19,7 @@ import unittest import util -from framework import SweetcombTestCase +from framework import SweetcombTestCase, SweetcombTestRunner from ydk.models.ietf import ietf_interfaces from ydk.models.ietf import iana_if_type from ydk.services import CRUDService @@ -116,3 +117,7 @@ class TestIetfInterfaces(SweetcombTestCase): a = self.vppctl.show_address(name) self.assertIsNone(a) + + +if __name__ == '__main__': + unittest.main(testRunner=SweetcombTestRunner) diff --git a/test/test_oc_interfaces.py b/test/test_oc_interfaces.py index 3b94945..be87eed 100644 --- a/test/test_oc_interfaces.py +++ b/test/test_oc_interfaces.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # # Copyright (c) 2019 PANTHEON.tech. # Copyright (c) 2019 Cisco and/or its affiliates. @@ -18,7 +19,7 @@ import unittest import util -from framework import SweetcombTestCase +from framework import SweetcombTestCase, SweetcombTestRunner from ydk.models.openconfig import openconfig_interfaces from ydk.models.ietf import iana_if_type from ydk.services import CRUDService @@ -48,8 +49,8 @@ class TestOcInterfaces(SweetcombTestCase): try: crud_service.create(self.netopeer_cli, interface) - except YError: - print("Error create services") + except YError as err: + print("Error create services: {}".format(err)) p = self.vppctl.show_interface(name) self.assertIsNotNone(p) @@ -92,3 +93,6 @@ class TestOcInterfaces(SweetcombTestCase): assert() a = self.vppctl.show_address(name) + +if __name__ == '__main__': + unittest.main(testRunner=SweetcombTestRunner) diff --git a/test/topology.py b/test/topology.py index a8a2f15..26dc622 100644 --- a/test/topology.py +++ b/test/topology.py @@ -25,6 +25,7 @@ import psutil import time from ydk.providers import NetconfServiceProvider + class Topology: debug = False @@ -68,7 +69,6 @@ class Topology: ip.addr('add', index=vpp1, address='192.168.0.2', prefixlen=24) ip.addr('add', index=vpp2, address='192.168.1.2', prefixlen=24) - def _start_sysrepo(self): print("Start sysrepo deamon.") #TODO: Need property close. diff --git a/test/util.py b/test/util.py index 576bc43..0d4ba65 100644 --- a/test/util.py +++ b/test/util.py @@ -27,13 +27,11 @@ def ping(ip): def import_yang_modules(): print("Import YANG models to sysrepo.") - #directory = '/root/src/sweetcomb/' - directory = './' - os.chdir(directory) + directory = os.getcwd() subprocess.run(["make", "uninstall-models"]) subprocess.run(["make", "install-models"]) - os.chdir(directory + "test/conf/") + os.chdir(directory + "/test/conf/") print("Import configuration to sysrepo datastore.") subprocess.run(["sysrepocfg", "--import=ietf-interfaces.xml", "--datastore=startup", "--format=xml", "--leve=0", diff --git a/test/vpp_controler.py b/test/vpp_controler.py index 32d6d70..b29f00e 100644 --- a/test/vpp_controler.py +++ b/test/vpp_controler.py @@ -28,12 +28,11 @@ class Vpp_controler: self.cmd = "vpp" self.ccmd = "vppctl" self.rootPath = os.getcwd() - self.configuration = self.rootPath + "/vpp.conf" + self.configuration = self.rootPath + "/test/conf/vpp.conf" self.process = None self.debug = debug def __del__(self): - #self.kill() self.terminate() def _default_conf_vpp(self): -- cgit