summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorjuraj.linkes <juraj.linkes@pantheon.tech>2018-11-16 17:28:56 +0100
committerDamjan Marion <dmarion@me.com>2018-11-26 19:54:25 +0000
commitabec0129b17797170d59521e0ad8b5f6d5853643 (patch)
treee3f3d18d768b759f2c0a91743ca2ec6aa8babcc6
parent2152e1cf8b3420d94ba31fe25853977ad945f35a (diff)
Split non-parallel testsuite
Split one big suite into smaller suites when not running tests in parallel. This results in all tests being executed in one iteration. Change-Id: I0d3d357a95d9cc596b606d5911a5819e8ffdeee5 Signed-off-by: juraj.linkes <juraj.linkes@pantheon.tech>
-rw-r--r--test/framework.py32
-rw-r--r--test/run_tests.py27
-rw-r--r--test/sanity_run_vpp.py7
3 files changed, 40 insertions, 26 deletions
diff --git a/test/framework.py b/test/framework.py
index ca15202f905..54b7a2dfa5e 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -1020,7 +1020,7 @@ class VppTestResult(unittest.TestResult):
core_crash_test_cases_info = set()
current_test_case_info = None
- def __init__(self, stream, descriptions, verbosity):
+ def __init__(self, stream, descriptions, verbosity, runner):
"""
:param stream File descriptor to store where to report test results.
Set to the standard error stream by default.
@@ -1033,6 +1033,7 @@ class VppTestResult(unittest.TestResult):
self.descriptions = descriptions
self.verbosity = verbosity
self.result_string = None
+ self.runner = runner
def addSuccess(self, test):
"""
@@ -1216,9 +1217,16 @@ class VppTestResult(unittest.TestResult):
"""
Print errors from running the test case
"""
- self.stream.writeln()
- self.printErrorList('ERROR', self.errors)
- self.printErrorList('FAIL', self.failures)
+ if len(self.errors) > 0 or len(self.failures) > 0:
+ self.stream.writeln()
+ self.printErrorList('ERROR', self.errors)
+ self.printErrorList('FAIL', self.failures)
+
+ # ^^ that is the last output from unittest before summary
+ if not self.runner.print_summary:
+ devnull = unittest.runner._WritelnDecorator(open(os.devnull, 'w'))
+ self.stream = devnull
+ self.runner.stream = devnull
def printErrorList(self, flavour, errors):
"""
@@ -1248,7 +1256,7 @@ class VppTestRunner(unittest.TextTestRunner):
def __init__(self, keep_alive_pipe=None, descriptions=True, verbosity=1,
result_pipe=None, failfast=False, buffer=False,
- resultclass=None):
+ resultclass=None, print_summary=True):
# ignore stream setting here, use hard-coded stdout to be in sync
# with prints from VppTestCase methods ...
@@ -1257,7 +1265,16 @@ class VppTestRunner(unittest.TextTestRunner):
resultclass)
KeepAliveReporter.pipe = keep_alive_pipe
- VppTestResult.test_framework_result_pipe = result_pipe
+ self.orig_stream = self.stream
+ self.resultclass.test_framework_result_pipe = result_pipe
+
+ self.print_summary = print_summary
+
+ def _makeResult(self):
+ return self.resultclass(self.stream,
+ self.descriptions,
+ self.verbosity,
+ self)
def run(self, test):
"""
@@ -1269,6 +1286,9 @@ class VppTestRunner(unittest.TextTestRunner):
faulthandler.enable() # emit stack trace to stderr if killed by signal
result = super(VppTestRunner, self).run(test)
+ if not self.print_summary:
+ self.stream = self.orig_stream
+ result.stream = self.orig_stream
return result
diff --git a/test/run_tests.py b/test/run_tests.py
index b2de2e702fb..d72a74c2ee4 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -125,7 +125,8 @@ def test_runner_wrapper(suite, keep_alive_pipe, stdouterr_queue,
descriptions=descriptions,
verbosity=verbose,
result_pipe=result_pipe,
- failfast=failfast).run(suite)
+ failfast=failfast,
+ print_summary=False).run(suite)
finished_pipe.send(result.wasSuccessful())
finished_pipe.close()
keep_alive_pipe.close()
@@ -234,12 +235,9 @@ def handle_failed_suite(logger, last_test_temp_dir, vpp_pid):
failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
link_path = '%s%s-FAILED' % (failed_dir, lttd)
if not os.path.exists(link_path):
- logger.error("Creating a link to the failed test: %s -> %s" %
- (link_path, lttd))
os.symlink(last_test_temp_dir, link_path)
- else:
- logger.error("Link to the failed test already exists: %s -> %s" %
- (link_path, lttd))
+ logger.error("Symlink to failed testcase directory: %s -> %s"
+ % (link_path, lttd))
# Report core existence
core_path = get_core_path(last_test_temp_dir)
@@ -575,10 +573,7 @@ class AllResults(dict):
retval = 1
if retval != 0:
- if concurrent_tests == 1:
- self.rerun.append(result.suite_from_failed())
- else:
- self.rerun.append(result.testcase_suite)
+ self.rerun.append(result.testcase_suite)
return retval
@@ -767,13 +762,6 @@ if __name__ == '__main__':
tests_amount += testcase_suite.countTestCases()
suites.append(testcase_suite)
- if concurrent_tests == 1:
- new_suite = unittest.TestSuite()
- for suite in suites:
- new_suite.addTests(suite)
-
- suites = [new_suite]
-
print("%s out of %s tests match specified filters" % (
tests_amount, tests_amount + cb.filtered.countTestCases()))
@@ -786,8 +774,9 @@ if __name__ == '__main__':
if run_interactive:
# don't fork if requiring interactive terminal
- result = VppTestRunner(verbosity=verbose, failfast=failfast)\
- .run(suites[0])
+ result = VppTestRunner(verbosity=verbose,
+ failfast=failfast,
+ print_summary=True).run(suites[0])
was_successful = result.wasSuccessful()
if not was_successful:
for test_case_info in result.failed_test_cases_info:
diff --git a/test/sanity_run_vpp.py b/test/sanity_run_vpp.py
index 012b16db154..933e3a719fc 100644
--- a/test/sanity_run_vpp.py
+++ b/test/sanity_run_vpp.py
@@ -8,7 +8,7 @@ from framework import VppTestCase, KeepAliveReporter
class SanityTestCase(VppTestCase):
- """ Sanity test case - verify if VPP is able to start """
+ """ Sanity test case - verify thether VPP is able to start """
pass
if __name__ == '__main__':
@@ -29,4 +29,9 @@ if __name__ == '__main__':
x.close()
y.close()
+ if rc == 0:
+ print('Sanity test case passed\n')
+ else:
+ print('Sanity test case failed\n')
+
exit(rc)