From f56b77a0764222cc45a9df572df901067a273356 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 3 Oct 2016 19:44:57 +0200 Subject: test: new test infrastructure Change-Id: I73ca19c431743f6b39669c583d9222a6559346ef Signed-off-by: Jan Gelety Signed-off-by: Juraj Sloboda Signed-off-by: Stefan Kobza Signed-off-by: Matej Klotton Signed-off-by: Maciek Konstantynowicz Signed-off-by: Damjan Marion --- test/run_tests.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 test/run_tests.py (limited to 'test/run_tests.py') diff --git a/test/run_tests.py b/test/run_tests.py new file mode 100644 index 00000000..8f2174b1 --- /dev/null +++ b/test/run_tests.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +import os +import unittest +from framework import VppTestRunner + +if __name__ == '__main__': + try: + verbose = int(os.getenv("V", 0)) + except: + verbose = 0 + unittest.main(testRunner=VppTestRunner, module=None, verbosity=verbose) -- cgit 1.2.3-korg From 993e0edf4e8078d10ddd5efa8156d62ce8f9d0c5 Mon Sep 17 00:00:00 2001 From: Klement Sekera Date: Thu, 16 Mar 2017 09:14:59 +0100 Subject: make test: support out-of-tree tests env EXTERN_TESTS="/path/to/extra/tests" make test causes to run the default test set and tests collected from test_*.py files under subtree specified in EXTERN_TESTS. Change-Id: I58c5471dd6010730278a5b47d4318737d920bc28 Signed-off-by: Klement Sekera --- Makefile | 5 +++-- test/Makefile | 11 ++++++++--- test/run_tests.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 61 insertions(+), 6 deletions(-) (limited to 'test/run_tests.py') diff --git a/Makefile b/Makefile index 44547067..36d37bf6 100644 --- a/Makefile +++ b/Makefile @@ -222,6 +222,7 @@ export VPP_PYTHON_PREFIX=$(BR)/python define test $(if $(filter-out $(3),retest),make -C $(BR) PLATFORM=$(1) TAG=$(2) vpp-install,) make -C test \ + TEST_DIR=$(WS_ROOT)/test \ VPP_TEST_BUILD_DIR=$(BR)/build-$(2)-native \ VPP_TEST_BIN=$(BR)/install-$(2)-native/vpp/bin/vpp \ VPP_TEST_PLUGIN_PATH=$(BR)/install-$(2)-native/vpp/lib64/vpp_plugins \ @@ -252,10 +253,10 @@ test-wipe: @make -C test wipe test-shell: bootstrap - $(call test,vpp_lite,vpp_lite,shell) + $(call test,vpp,vpp,shell) test-shell-debug: bootstrap - $(call test,vpp_lite,vpp_lite_debug,shell) + $(call test,vpp,vpp_debug,shell) test-doc: @make -C test doc diff --git a/test/Makefile b/test/Makefile index 4338e096..c65eaae6 100644 --- a/test/Makefile +++ b/test/Makefile @@ -23,10 +23,15 @@ verify-no-running-vpp: false; \ fi -UNITTEST_EXTRA_OPTS="" +UNITTEST_EXTRA_OPTS= +UNITTEST_FAILFAST_OPTS= ifeq ($(FAILFAST),1) -UNITTEST_EXTRA_OPTS="-f" +UNITTEST_EXTRA_OPTS=-f +endif + +ifneq ($(EXTERN_TESTS),) +UNITTEST_EXTRA_OPTS=$(UNITTEST_FAILFAST_OPTS) -d $(EXTERN_TESTS) endif PYTHON_VENV_PATH=$(VPP_PYTHON_PREFIX)/virtualenv @@ -59,7 +64,7 @@ $(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE) @touch $@ define retest-func - @bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python run_tests.py discover $(UNITTEST_EXTRA_OPTS) -p test_\"*.py\"" + @bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python run_tests.py -d $(TEST_DIR) $(UNITTEST_EXTRA_OPTS)" endef .PHONY: sanity diff --git a/test/run_tests.py b/test/run_tests.py index 8f2174b1..39b09368 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -1,12 +1,61 @@ #!/usr/bin/env python +import sys import os import unittest +import argparse +import importlib from framework import VppTestRunner + +def add_from_dir(suite, directory): + do_insert = True + for _f in os.listdir(directory): + f = "%s/%s" % (directory, _f) + if os.path.isdir(f): + add_from_dir(suite, f) + continue + if not os.path.isfile(f): + continue + if do_insert: + sys.path.insert(0, directory) + do_insert = False + if not _f.startswith("test_") or not _f.endswith(".py"): + continue + name = "".join(f.split("/")[-1].split(".")[:-1]) + if name in sys.modules: + raise Exception("Duplicate test module `%s' found!" % name) + module = importlib.import_module(name) + for name, cls in module.__dict__.items(): + if not isinstance(cls, type): + continue + if not issubclass(cls, unittest.TestCase): + continue + if name == "VppTestCase": + continue + for method in dir(cls): + if not callable(getattr(cls, method)): + continue + if method.startswith("test_"): + suite.addTest(cls(method)) + if __name__ == '__main__': try: verbose = int(os.getenv("V", 0)) except: verbose = 0 - unittest.main(testRunner=VppTestRunner, module=None, verbosity=verbose) + + parser = argparse.ArgumentParser(description="VPP unit tests") + parser.add_argument("-f", "--failfast", action='count', + help="fast failure flag") + parser.add_argument("-d", "--dir", action='append', type=str, + help="directory containing test files " + "(may be specified multiple times)") + args = parser.parse_args() + failfast = True if args.failfast == 1 else False + + suite = unittest.TestSuite() + for d in args.dir: + print("Adding tests from directory tree %s" % d) + add_from_dir(suite, d) + VppTestRunner(verbosity=verbose, failfast=failfast).run(suite) -- cgit 1.2.3-korg From 0c8ad446db72078a2255329c2cfaced517829c78 Mon Sep 17 00:00:00 2001 From: Klement Sekera Date: Fri, 24 Mar 2017 04:29:06 +0100 Subject: make test: properly propagate exit status Change-Id: Ie9f48a0d5e0a9cd08eb8f07d49149eee40f04131 Signed-off-by: Klement Sekera --- test/run_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'test/run_tests.py') diff --git a/test/run_tests.py b/test/run_tests.py index 39b09368..1b9c677d 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -58,4 +58,5 @@ if __name__ == '__main__': for d in args.dir: print("Adding tests from directory tree %s" % d) add_from_dir(suite, d) - VppTestRunner(verbosity=verbose, failfast=failfast).run(suite) + sys.exit(not VppTestRunner(verbosity=verbose, + failfast=failfast).run(suite).wasSuccessful()) -- cgit 1.2.3-korg From 909a6a1eb92ceacb983bcff088fb512352929c46 Mon Sep 17 00:00:00 2001 From: Klement Sekera Date: Tue, 8 Aug 2017 04:33:53 +0200 Subject: make test: detect hung tests Run tests in a forked process with a set of pipes to communicate keep-alives and overall result. This allows us to detect when e.g. vpp dies mid-API call causing the test to hang waiting for response (which will never come since vpp died). Support setting a (per test case) TIMEOUT make test option to set timeout, with a default timeout of 120 seconds. Example - fail the test suite if any test-case fails to finish within 300s: make TIMEOUT=300 test Change-Id: I0d04f26a7232968f4bf043adf5d5b508f5018717 Signed-off-by: Klement Sekera --- test/Makefile | 1 + test/debug.py | 23 ++++++++++++++++ test/framework.py | 42 +++++++++++++++++++++++++++-- test/hook.py | 29 +++++--------------- test/run_tests.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++--- test/sanity_run_vpp.py | 10 +++++-- 6 files changed, 148 insertions(+), 29 deletions(-) create mode 100644 test/debug.py (limited to 'test/run_tests.py') diff --git a/test/Makefile b/test/Makefile index 7416afc2..d48a6b5a 100644 --- a/test/Makefile +++ b/test/Makefile @@ -184,6 +184,7 @@ help: @echo "Arguments controlling test runs:" @echo " V=[0|1|2] - set test verbosity level" @echo " FAILFAST=[0|1] - fail fast if 1, complete all tests if 0" + @echo " TIMEOUT= - fail test suite if any single test takes longer than to finish" @echo " DEBUG= - set VPP debugging kind" @echo " DEBUG=core - detect coredump and load it in gdb on crash" @echo " DEBUG=gdb - allow easy debugging by printing VPP PID " diff --git a/test/debug.py b/test/debug.py new file mode 100644 index 00000000..4516b8c1 --- /dev/null +++ b/test/debug.py @@ -0,0 +1,23 @@ +""" debug utilities """ + +import os +import pexpect + +gdb_path = '/usr/bin/gdb' + + +def spawn_gdb(binary_path, core_path, logger): + if os.path.isfile(gdb_path) and os.access(gdb_path, os.X_OK): + # automatically attach gdb + gdb_cmdline = "%s %s %s" % (gdb_path, binary_path, core_path) + gdb = pexpect.spawn(gdb_cmdline) + gdb.interact() + try: + gdb.terminate(True) + except: + pass + if gdb.isalive(): + raise Exception("GDB refused to die...") + else: + logger.error("Debugger '%s' does not exist or is not an " + "executable.." % gdb_path) diff --git a/test/framework.py b/test/framework.py index fd493db3..58b76bbe 100644 --- a/test/framework.py +++ b/test/framework.py @@ -12,7 +12,7 @@ import resource import faulthandler from collections import deque from threading import Thread, Event -from inspect import getdoc +from inspect import getdoc, isclass from traceback import format_exception from logging import FileHandler, DEBUG, Formatter from scapy.packet import Raw @@ -92,6 +92,39 @@ def running_extended_tests(): return False +class KeepAliveReporter(object): + """ + Singleton object which reports test start to parent process + """ + _shared_state = {} + + def __init__(self): + self.__dict__ = self._shared_state + + @property + def pipe(self): + return self._pipe + + @pipe.setter + def pipe(self, pipe): + if hasattr(self, '_pipe'): + raise Exception("Internal error - pipe should only be set once.") + self._pipe = pipe + + def send_keep_alive(self, test): + """ + Write current test tmpdir & desc to keep-alive pipe to signal liveness + """ + if isclass(test): + desc = test.__name__ + else: + desc = test.shortDescription() + if not desc: + desc = str(test) + + self.pipe.send((desc, test.vpp_bin, test.tempdir)) + + class VppTestCase(unittest.TestCase): """This subclass is a base class for VPP test cases that are implemented as classes. It provides methods to create and run test case. @@ -257,6 +290,8 @@ class VppTestCase(unittest.TestCase): cls.vpp_dead = False cls.registry = VppObjectRegistry() cls.vpp_startup_failed = False + cls.reporter = KeepAliveReporter() + cls.reporter.send_keep_alive(cls) # need to catch exceptions here because if we raise, then the cleanup # doesn't get called and we might end with a zombie vpp try: @@ -394,6 +429,7 @@ class VppTestCase(unittest.TestCase): def setUp(self): """ Clear trace before running each test""" + self.reporter.send_keep_alive(self) self.logger.debug("--- setUp() for %s.%s(%s) called ---" % (self.__class__.__name__, self._testMethodName, self._testMethodDoc)) @@ -865,13 +901,15 @@ class VppTestRunner(unittest.TextTestRunner): """Class maintaining the results of the tests""" return VppTestResult - def __init__(self, stream=sys.stderr, descriptions=True, verbosity=1, + def __init__(self, pipe, stream=sys.stderr, descriptions=True, verbosity=1, failfast=False, buffer=False, resultclass=None): # ignore stream setting here, use hard-coded stdout to be in sync # with prints from VppTestCase methods ... super(VppTestRunner, self).__init__(sys.stdout, descriptions, verbosity, failfast, buffer, resultclass) + reporter = KeepAliveReporter() + reporter.pipe = pipe test_option = "TEST" diff --git a/test/hook.py b/test/hook.py index 247704ec..f34e0c5b 100644 --- a/test/hook.py +++ b/test/hook.py @@ -1,8 +1,8 @@ import signal import os -import pexpect import traceback -from log import * +from log import RED, single_line_delim, double_line_delim +from debug import spawn_gdb, gdb_path class Hook(object): @@ -60,31 +60,16 @@ class PollHook(Hook): self.testcase = testcase self.logger = testcase.logger - def spawn_gdb(self, gdb_path, core_path): - gdb_cmdline = gdb_path + ' ' + self.testcase.vpp_bin + ' ' + core_path - gdb = pexpect.spawn(gdb_cmdline) - gdb.interact() - try: - gdb.terminate(True) - except: - pass - if gdb.isalive(): - raise Exception("GDB refused to die...") - def on_crash(self, core_path): if self.testcase.debug_core: - gdb_path = '/usr/bin/gdb' - if os.path.isfile(gdb_path) and os.access(gdb_path, os.X_OK): - # automatically attach gdb - self.spawn_gdb(gdb_path, core_path) - return - else: + if not spawn_gdb(self.testcase.vpp_bin, core_path): self.logger.error( "Debugger '%s' does not exist or is not an executable.." % gdb_path) - - self.logger.critical('core file present, debug with: gdb ' + - self.testcase.vpp_bin + ' ' + core_path) + else: + return + self.logger.critical("Core file present, debug with: gdb %s %s" % + (self.testcase.vpp_bin, core_path)) def poll_vpp(self): """ diff --git a/test/run_tests.py b/test/run_tests.py index 1b9c677d..6d477d88 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -2,10 +2,14 @@ import sys import os +import select import unittest import argparse import importlib +from multiprocessing import Process, Pipe from framework import VppTestRunner +from debug import spawn_gdb +from log import global_logger def add_from_dir(suite, directory): @@ -39,12 +43,38 @@ def add_from_dir(suite, directory): if method.startswith("test_"): suite.addTest(cls(method)) + +def test_runner_wrapper(keep_alive_pipe, result_pipe): + result = not VppTestRunner( + pipe=keep_alive_pipe, + verbosity=verbose, + failfast=failfast).run(suite).wasSuccessful() + result_pipe.send(result) + result_pipe.close() + keep_alive_pipe.close() + + +def handle_core(vpp_binary, core_path): + try: + d = os.getenv("DEBUG") + except: + d = None + if d and d.lower() == "core": + spawn_gdb(vpp_binary, core_path, global_logger) + + if __name__ == '__main__': try: verbose = int(os.getenv("V", 0)) except: verbose = 0 + default_test_timeout = 120 + try: + test_timeout = int(os.getenv("TIMEOUT", default_test_timeout)) + except: + test_timeout = default_test_timeout + parser = argparse.ArgumentParser(description="VPP unit tests") parser.add_argument("-f", "--failfast", action='count', help="fast failure flag") @@ -56,7 +86,43 @@ if __name__ == '__main__': suite = unittest.TestSuite() for d in args.dir: - print("Adding tests from directory tree %s" % d) + global_logger.info("Adding tests from directory tree %s" % d) add_from_dir(suite, d) - sys.exit(not VppTestRunner(verbosity=verbose, - failfast=failfast).run(suite).wasSuccessful()) + keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False) + result_parent_end, result_child_end = Pipe(duplex=False) + + p = Process(target=test_runner_wrapper, + args=(keep_alive_child_end, + result_child_end)) + p.start() + last_test_temp_dir = None + last_test_vpp_binary = None + last_test = None + result = None + while result is None: + readable = select.select([keep_alive_parent_end.fileno(), + result_parent_end.fileno(), + ], + [], [], test_timeout)[0] + if result_parent_end.fileno() in readable: + result = result_parent_end.recv() + elif keep_alive_parent_end.fileno() in readable: + while keep_alive_parent_end.poll(): + last_test, last_test_vpp_binary, last_test_temp_dir =\ + keep_alive_parent_end.recv() + else: + global_logger.critical("Timeout while waiting for child test " + "runner process (last test running was " + "`%s' in `%s')!" % + (last_test, last_test_temp_dir)) + if last_test_temp_dir and last_test_vpp_binary: + core_path = "%s/core" % last_test_temp_dir + if os.path.isfile(core_path): + global_logger.error("Core-file exists in test temporary " + "directory: %s!" % core_path) + handle_core(last_test_vpp_binary, core_path) + p.terminate() + result = -1 + keep_alive_parent_end.close() + result_parent_end.close() + sys.exit(result) diff --git a/test/sanity_run_vpp.py b/test/sanity_run_vpp.py index 527b618f..156608c1 100644 --- a/test/sanity_run_vpp.py +++ b/test/sanity_run_vpp.py @@ -1,9 +1,10 @@ #!/usr/bin/env python from __future__ import print_function -from framework import VppTestCase -from hook import VppDiedError +from multiprocessing import Pipe from sys import exit +from hook import VppDiedError +from framework import VppTestCase, KeepAliveReporter class SanityTestCase(VppTestCase): @@ -13,6 +14,9 @@ class SanityTestCase(VppTestCase): if __name__ == '__main__': rc = 0 tc = SanityTestCase + x, y = Pipe() + reporter = KeepAliveReporter() + reporter.pipe = y try: tc.setUpClass() except VppDiedError: @@ -22,5 +26,7 @@ if __name__ == '__main__': tc.tearDownClass() except: pass + x.close() + y.close() exit(rc) -- cgit 1.2.3-korg From fa3eb7a993fe7538c51360960ef92dced0562342 Mon Sep 17 00:00:00 2001 From: Klement Sekera Date: Thu, 10 Aug 2017 06:50:27 +0200 Subject: make test: bump default test timeout to 10 minutes Change-Id: I25d88966376d712ff61f29227a45880a59e8ecf2 Signed-off-by: Klement Sekera --- test/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/run_tests.py') diff --git a/test/run_tests.py b/test/run_tests.py index 6d477d88..8b135ddc 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -69,7 +69,7 @@ if __name__ == '__main__': except: verbose = 0 - default_test_timeout = 120 + default_test_timeout = 600 # 10 minutes try: test_timeout = int(os.getenv("TIMEOUT", default_test_timeout)) except: -- cgit 1.2.3-korg From 3f6ff19a30e9fbe5befb4cc3521d1812e5612197 Mon Sep 17 00:00:00 2001 From: Klement Sekera Date: Fri, 11 Aug 2017 06:56:05 +0200 Subject: make test: fix broken DEBUG=gdb* options Change-Id: I5d80982eeab78a629760f567eda3b1539d96e3a8 Signed-off-by: Klement Sekera --- test/framework.py | 8 +++-- test/run_tests.py | 92 ++++++++++++++++++++++++++++++------------------------- 2 files changed, 56 insertions(+), 44 deletions(-) (limited to 'test/run_tests.py') diff --git a/test/framework.py b/test/framework.py index 58b76bbe..89d95cb3 100644 --- a/test/framework.py +++ b/test/framework.py @@ -115,6 +115,10 @@ class KeepAliveReporter(object): """ Write current test tmpdir & desc to keep-alive pipe to signal liveness """ + if self.pipe is None: + # if not running forked.. + return + if isclass(test): desc = test.__name__ else: @@ -901,8 +905,8 @@ class VppTestRunner(unittest.TextTestRunner): """Class maintaining the results of the tests""" return VppTestResult - def __init__(self, pipe, stream=sys.stderr, descriptions=True, verbosity=1, - failfast=False, buffer=False, resultclass=None): + def __init__(self, pipe=None, stream=sys.stderr, descriptions=True, + verbosity=1, failfast=False, buffer=False, resultclass=None): # ignore stream setting here, use hard-coded stdout to be in sync # with prints from VppTestCase methods ... super(VppTestRunner, self).__init__(sys.stdout, descriptions, diff --git a/test/run_tests.py b/test/run_tests.py index 8b135ddc..271f5c5c 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -44,7 +44,7 @@ def add_from_dir(suite, directory): suite.addTest(cls(method)) -def test_runner_wrapper(keep_alive_pipe, result_pipe): +def test_runner_wrapper(suite, keep_alive_pipe, result_pipe): result = not VppTestRunner( pipe=keep_alive_pipe, verbosity=verbose, @@ -54,47 +54,13 @@ def test_runner_wrapper(keep_alive_pipe, result_pipe): keep_alive_pipe.close() -def handle_core(vpp_binary, core_path): - try: - d = os.getenv("DEBUG") - except: - d = None - if d and d.lower() == "core": - spawn_gdb(vpp_binary, core_path, global_logger) - - -if __name__ == '__main__': - try: - verbose = int(os.getenv("V", 0)) - except: - verbose = 0 - - default_test_timeout = 600 # 10 minutes - try: - test_timeout = int(os.getenv("TIMEOUT", default_test_timeout)) - except: - test_timeout = default_test_timeout - - parser = argparse.ArgumentParser(description="VPP unit tests") - parser.add_argument("-f", "--failfast", action='count', - help="fast failure flag") - parser.add_argument("-d", "--dir", action='append', type=str, - help="directory containing test files " - "(may be specified multiple times)") - args = parser.parse_args() - failfast = True if args.failfast == 1 else False - - suite = unittest.TestSuite() - for d in args.dir: - global_logger.info("Adding tests from directory tree %s" % d) - add_from_dir(suite, d) +def run_forked(suite): keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False) result_parent_end, result_child_end = Pipe(duplex=False) - p = Process(target=test_runner_wrapper, - args=(keep_alive_child_end, - result_child_end)) - p.start() + child = Process(target=test_runner_wrapper, + args=(suite, keep_alive_child_end, result_child_end)) + child.start() last_test_temp_dir = None last_test_vpp_binary = None last_test = None @@ -120,9 +86,51 @@ if __name__ == '__main__': if os.path.isfile(core_path): global_logger.error("Core-file exists in test temporary " "directory: %s!" % core_path) - handle_core(last_test_vpp_binary, core_path) - p.terminate() + if d and d.lower() == "core": + spawn_gdb(last_test_vpp_binary, core_path, + global_logger) + child.terminate() result = -1 keep_alive_parent_end.close() result_parent_end.close() - sys.exit(result) + return result + + +if __name__ == '__main__': + + try: + verbose = int(os.getenv("V", 0)) + except: + verbose = 0 + + default_test_timeout = 600 # 10 minutes + try: + test_timeout = int(os.getenv("TIMEOUT", default_test_timeout)) + except: + test_timeout = default_test_timeout + + try: + debug = os.getenv("DEBUG") + except: + debug = None + + parser = argparse.ArgumentParser(description="VPP unit tests") + parser.add_argument("-f", "--failfast", action='count', + help="fast failure flag") + parser.add_argument("-d", "--dir", action='append', type=str, + help="directory containing test files " + "(may be specified multiple times)") + args = parser.parse_args() + failfast = True if args.failfast == 1 else False + + suite = unittest.TestSuite() + for d in args.dir: + global_logger.info("Adding tests from directory tree %s" % d) + add_from_dir(suite, d) + + if debug is None or debug.lower() not in ["gdb", "gdbserver"]: + sys.exit(run_forked(suite)) + + # don't fork if debugging.. + sys.exit(not VppTestRunner(verbosity=verbose, + failfast=failfast).run(suite).wasSuccessful()) -- cgit 1.2.3-korg From fcbf44448b85519b85616a6b87310462249c1c63 Mon Sep 17 00:00:00 2001 From: Klement Sekera Date: Thu, 17 Aug 2017 07:38:42 +0200 Subject: make test: separate test discovery code Separating test discovery code to it's own script file has the advantage of easily doing e.g. listing of all existing tests. Change-Id: I80dc280263cc7e33e7e13cb0d48b39bf08ece24d Signed-off-by: Klement Sekera --- test/discover_tests.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++ test/run_tests.py | 45 ++++++++++----------------------------- 2 files changed, 68 insertions(+), 34 deletions(-) create mode 100755 test/discover_tests.py (limited to 'test/run_tests.py') diff --git a/test/discover_tests.py b/test/discover_tests.py new file mode 100755 index 00000000..eea59410 --- /dev/null +++ b/test/discover_tests.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +import sys +import os +import unittest +import importlib +import argparse + + +def discover_tests(directory, callback): + do_insert = True + for _f in os.listdir(directory): + f = "%s/%s" % (directory, _f) + if os.path.isdir(f): + discover_tests(f, callback) + continue + if not os.path.isfile(f): + continue + if do_insert: + sys.path.insert(0, directory) + do_insert = False + if not _f.startswith("test_") or not _f.endswith(".py"): + continue + name = "".join(f.split("/")[-1].split(".")[:-1]) + if name in sys.modules: + raise Exception("Duplicate test module `%s' found!" % name) + module = importlib.import_module(name) + for name, cls in module.__dict__.items(): + if not isinstance(cls, type): + continue + if not issubclass(cls, unittest.TestCase): + continue + if name == "VppTestCase": + continue + for method in dir(cls): + if not callable(getattr(cls, method)): + continue + if method.startswith("test_"): + callback(_f, cls, method) + + +def print_callback(file_name, cls, method): + print("%s.%s.%s" % (file_name, cls.__name__, method)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Discover VPP unit tests") + parser.add_argument("-d", "--dir", action='append', type=str, + help="directory containing test files " + "(may be specified multiple times)") + args = parser.parse_args() + if args.dir is None: + args.dir = "." + + suite = unittest.TestSuite() + for d in args.dir: + discover_tests(d, print_callback) diff --git a/test/run_tests.py b/test/run_tests.py index 271f5c5c..9614080d 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -5,43 +5,11 @@ import os import select import unittest import argparse -import importlib from multiprocessing import Process, Pipe from framework import VppTestRunner from debug import spawn_gdb from log import global_logger - - -def add_from_dir(suite, directory): - do_insert = True - for _f in os.listdir(directory): - f = "%s/%s" % (directory, _f) - if os.path.isdir(f): - add_from_dir(suite, f) - continue - if not os.path.isfile(f): - continue - if do_insert: - sys.path.insert(0, directory) - do_insert = False - if not _f.startswith("test_") or not _f.endswith(".py"): - continue - name = "".join(f.split("/")[-1].split(".")[:-1]) - if name in sys.modules: - raise Exception("Duplicate test module `%s' found!" % name) - module = importlib.import_module(name) - for name, cls in module.__dict__.items(): - if not isinstance(cls, type): - continue - if not issubclass(cls, unittest.TestCase): - continue - if name == "VppTestCase": - continue - for method in dir(cls): - if not callable(getattr(cls, method)): - continue - if method.startswith("test_"): - suite.addTest(cls(method)) +from discover_tests import discover_tests def test_runner_wrapper(suite, keep_alive_pipe, result_pipe): @@ -54,6 +22,14 @@ def test_runner_wrapper(suite, keep_alive_pipe, result_pipe): keep_alive_pipe.close() +class add_to_suite_callback: + def __init__(self, suite): + self.suite = suite + + def __call__(self, file_name, cls, method): + suite.addTest(cls(method)) + + def run_forked(suite): keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False) result_parent_end, result_child_end = Pipe(duplex=False) @@ -124,9 +100,10 @@ if __name__ == '__main__': failfast = True if args.failfast == 1 else False suite = unittest.TestSuite() + cb = add_to_suite_callback(suite) for d in args.dir: global_logger.info("Adding tests from directory tree %s" % d) - add_from_dir(suite, d) + discover_tests(d, cb) if debug is None or debug.lower() not in ["gdb", "gdbserver"]: sys.exit(run_forked(suite)) -- cgit 1.2.3-korg From 76c37d2d8464827f7270b162bad8db344aa13a5d Mon Sep 17 00:00:00 2001 From: Dave Wallace Date: Sat, 30 Sep 2017 15:12:19 -0400 Subject: make test: Create link to failed test dir on timeout. (VPP-1011) - Also change default coredump configuration from "coredump-size unlimited" to "full-coredump" Change-Id: Iefedc2636f2d9696b7575b34e91dd7be49f601fa Signed-off-by: Dave Wallace (cherry picked from commit 981fadf928dadac683d2f629edf738aa91510af3) --- test/framework.py | 2 +- test/run_tests.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'test/run_tests.py') diff --git a/test/framework.py b/test/framework.py index 008bda3b..869a3402 100644 --- a/test/framework.py +++ b/test/framework.py @@ -205,7 +205,7 @@ class VppTestCase(unittest.TestCase): except: pass if coredump_size is None: - coredump_size = "coredump-size unlimited" + coredump_size = "full-coredump" cls.vpp_cmdline = [cls.vpp_bin, "unix", "{", "nodaemon", debug_cli, coredump_size, "}", "api-trace", "{", "on", "}", diff --git a/test/run_tests.py b/test/run_tests.py index 9614080d..999252d1 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -57,6 +57,12 @@ def run_forked(suite): "runner process (last test running was " "`%s' in `%s')!" % (last_test, last_test_temp_dir)) + failed_dir = os.getenv('VPP_TEST_FAILED_DIR') + lttd = last_test_temp_dir.split("/")[-1] + link_path = '%s%s-FAILED' % (failed_dir, lttd) + global_logger.error("Creating a link to the failed " + + "test: %s -> %s" % (link_path, lttd)) + os.symlink(last_test_temp_dir, link_path) if last_test_temp_dir and last_test_vpp_binary: core_path = "%s/core" % last_test_temp_dir if os.path.isfile(core_path): -- cgit 1.2.3-korg From 86c0446e28c4bf9e2dda7e9324de91f950720bcf Mon Sep 17 00:00:00 2001 From: Dave Wallace Date: Sat, 30 Sep 2017 22:04:21 -0400 Subject: make test: Copy api_post_mortem.$$ file tmp test dir for archiving. (VPP-1011) Change-Id: I4baf89ef383dbc2f309081a6b56b13ebcb8fc2df Signed-off-by: Dave Wallace (cherry picked from commit e2efd12b8418558cd8e701368287860409e8d265) --- test/framework.py | 10 +++++----- test/run_tests.py | 10 ++++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) (limited to 'test/run_tests.py') diff --git a/test/framework.py b/test/framework.py index 869a3402..600a0e54 100644 --- a/test/framework.py +++ b/test/framework.py @@ -126,7 +126,7 @@ class KeepAliveReporter(object): if not desc: desc = str(test) - self.pipe.send((desc, test.vpp_bin, test.tempdir)) + self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid)) class VppTestCase(unittest.TestCase): @@ -205,10 +205,10 @@ class VppTestCase(unittest.TestCase): except: pass if coredump_size is None: - coredump_size = "full-coredump" + coredump_size = "coredump-size unlimited" cls.vpp_cmdline = [cls.vpp_bin, "unix", - "{", "nodaemon", debug_cli, coredump_size, "}", - "api-trace", "{", "on", "}", + "{", "nodaemon", debug_cli, "full-coredump", + coredump_size, "}", "api-trace", "{", "on", "}", "api-segment", "{", "prefix", cls.shm_prefix, "}", "plugins", "{", "plugin", "dpdk_plugin.so", "{", "disable", "}", "}"] @@ -295,11 +295,11 @@ class VppTestCase(unittest.TestCase): cls.registry = VppObjectRegistry() cls.vpp_startup_failed = False cls.reporter = KeepAliveReporter() - cls.reporter.send_keep_alive(cls) # need to catch exceptions here because if we raise, then the cleanup # doesn't get called and we might end with a zombie vpp try: cls.run_vpp() + cls.reporter.send_keep_alive(cls) cls.vpp_stdout_deque = deque() cls.vpp_stderr_deque = deque() cls.pump_thread_stop_flag = Event() diff --git a/test/run_tests.py b/test/run_tests.py index 999252d1..b07a923a 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import sys +import shutil import os import select import unittest @@ -50,8 +51,8 @@ def run_forked(suite): result = result_parent_end.recv() elif keep_alive_parent_end.fileno() in readable: while keep_alive_parent_end.poll(): - last_test, last_test_vpp_binary, last_test_temp_dir =\ - keep_alive_parent_end.recv() + last_test, last_test_vpp_binary,\ + last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv() else: global_logger.critical("Timeout while waiting for child test " "runner process (last test running was " @@ -63,6 +64,11 @@ def run_forked(suite): global_logger.error("Creating a link to the failed " + "test: %s -> %s" % (link_path, lttd)) os.symlink(last_test_temp_dir, link_path) + api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid + if os.path.isfile(api_post_mortem_path): + global_logger.error("Copying api_post_mortem.%d to %s" % + (vpp_pid, last_test_temp_dir)) + shutil.copy2(api_post_mortem_path, last_test_temp_dir) if last_test_temp_dir and last_test_vpp_binary: core_path = "%s/core" % last_test_temp_dir if os.path.isfile(core_path): -- cgit 1.2.3-korg