diff options
| -rw-r--r-- | test/asf/asfframework.py | 110 | ||||
| -rw-r--r-- | test/framework.py | 110 | ||||
| -rw-r--r-- | test/run_tests.py | 125 | ||||
| -rw-r--r-- | test/test_result_code.py | 15 |
4 files changed, 204 insertions, 156 deletions
diff --git a/test/asf/asfframework.py b/test/asf/asfframework.py index 1997d9f32f5..d613ff2576e 100644 --- a/test/asf/asfframework.py +++ b/test/asf/asfframework.py @@ -46,6 +46,7 @@ from log import ( ) from vpp_object import VppObjectRegistry from util import ppp, is_core_present +from test_result_code import TestResultCode logger = logging.getLogger(__name__) @@ -53,13 +54,6 @@ logger = logging.getLogger(__name__) null_logger = logging.getLogger("VppTestCase") null_logger.addHandler(logging.NullHandler()) -PASS = 0 -FAIL = 1 -ERROR = 2 -SKIP = 3 -TEST_RUN = 4 -SKIP_CPU_SHORTAGE = 5 - if config.debug_framework: import debug_internal @@ -1407,6 +1401,7 @@ class VppTestResult(unittest.TestResult): self.stream = stream self.descriptions = descriptions self.verbosity = verbosity + self.result_code = TestResultCode.TEST_RUN self.result_string = None self.runner = runner self.printed = [] @@ -1418,15 +1413,25 @@ class VppTestResult(unittest.TestResult): :param test: """ - if self.current_test_case_info: - self.current_test_case_info.logger.debug( - "--- addSuccess() %s.%s(%s) called" - % (test.__class__.__name__, test._testMethodName, test._testMethodDoc) - ) + self.log_result("addSuccess", test) unittest.TestResult.addSuccess(self, test) self.result_string = colorize("OK", GREEN) - - self.send_result_through_pipe(test, PASS) + self.result_code = TestResultCode.PASS + self.send_result_through_pipe(test, self.result_code) + + def addExpectedFailure(self, test, err): + self.log_result("addExpectedFailure", test, err) + super().addExpectedFailure(test, err) + self.result_string = colorize("FAIL", GREEN) + self.result_code = TestResultCode.EXPECTED_FAIL + self.send_result_through_pipe(test, self.result_code) + + def addUnexpectedSuccess(self, test): + self.log_result("addUnexpectedSuccess", test) + super().addUnexpectedSuccess(test) + self.result_string = colorize("OK", RED) + self.result_code = TestResultCode.UNEXPECTED_PASS + self.send_result_through_pipe(test, self.result_code) def addSkip(self, test, reason): """ @@ -1436,23 +1441,15 @@ class VppTestResult(unittest.TestResult): :param reason: """ - if self.current_test_case_info: - self.current_test_case_info.logger.debug( - "--- addSkip() %s.%s(%s) called, reason is %s" - % ( - test.__class__.__name__, - test._testMethodName, - test._testMethodDoc, - reason, - ) - ) + self.log_result("addSkip", test, reason=reason) unittest.TestResult.addSkip(self, test, reason) self.result_string = colorize("SKIP", YELLOW) if reason == "not enough cpus": - self.send_result_through_pipe(test, SKIP_CPU_SHORTAGE) + self.result_code = TestResultCode.SKIP_CPU_SHORTAGE else: - self.send_result_through_pipe(test, SKIP) + self.result_code = TestResultCode.SKIP + self.send_result_through_pipe(test, self.result_code) def symlink_failed(self): if self.current_test_case_info: @@ -1484,7 +1481,7 @@ class VppTestResult(unittest.TestResult): if pipe: pipe.send((test.id(), result)) - def log_error(self, test, err, fn_name): + def log_result(self, fn, test, err=None, reason=None): if self.current_test_case_info: if isinstance(test, unittest.suite._ErrorHolder): test_name = test.description @@ -1494,25 +1491,29 @@ class VppTestResult(unittest.TestResult): test._testMethodName, test._testMethodDoc, ) + extra_msg = "" + if err: + extra_msg += f", error is {err}" + if reason: + extra_msg += f", reason is {reason}" self.current_test_case_info.logger.debug( - "--- %s() %s called, err is %s" % (fn_name, test_name, err) - ) - self.current_test_case_info.logger.debug( - "formatted exception is:\n%s" % "".join(format_exception(*err)) + f"--- {fn}() {test_name} called{extra_msg}" ) + if err: + self.current_test_case_info.logger.debug( + "formatted exception is:\n%s" % "".join(format_exception(*err)) + ) - def add_error(self, test, err, unittest_fn, error_type): - if error_type == FAIL: - self.log_error(test, err, "addFailure") + def add_error(self, test, err, unittest_fn, result_code): + self.result_code = result_code + if result_code == TestResultCode.FAIL: + self.log_result("addFailure", test, err=err) error_type_str = colorize("FAIL", RED) - elif error_type == ERROR: - self.log_error(test, err, "addError") + elif result_code == TestResultCode.ERROR: + self.log_result("addError", test, err=err) error_type_str = colorize("ERROR", RED) else: - raise Exception( - "Error type %s cannot be used to record an " - "error or a failure" % error_type - ) + raise Exception(f"Unexpected result code {result_code}") unittest_fn(self, test, err) if self.current_test_case_info: @@ -1535,7 +1536,7 @@ class VppTestResult(unittest.TestResult): else: self.result_string = "%s [no temp dir]" % error_type_str - self.send_result_through_pipe(test, error_type) + self.send_result_through_pipe(test, result_code) def addFailure(self, test, err): """ @@ -1545,7 +1546,7 @@ class VppTestResult(unittest.TestResult): :param err: error message """ - self.add_error(test, err, unittest.TestResult.addFailure, FAIL) + self.add_error(test, err, unittest.TestResult.addFailure, TestResultCode.FAIL) def addError(self, test, err): """ @@ -1555,7 +1556,7 @@ class VppTestResult(unittest.TestResult): :param err: error message """ - self.add_error(test, err, unittest.TestResult.addError, ERROR) + self.add_error(test, err, unittest.TestResult.addError, TestResultCode.ERROR) def getDescription(self, test): """ @@ -1634,23 +1635,40 @@ class VppTestResult(unittest.TestResult): """ unittest.TestResult.stopTest(self, test) + result_code_to_suffix = { + TestResultCode.PASS: "", + TestResultCode.FAIL: "", + TestResultCode.ERROR: "", + TestResultCode.SKIP: "", + TestResultCode.TEST_RUN: "", + TestResultCode.SKIP_CPU_SHORTAGE: "", + TestResultCode.EXPECTED_FAIL: " [EXPECTED FAIL]", + TestResultCode.UNEXPECTED_PASS: " [UNEXPECTED PASS]", + } + if self.verbosity > 0: self.stream.writeln(single_line_delim) self.stream.writeln( - "%-73s%s" % (self.getDescription(test), self.result_string) + "%-72s%s%s" + % ( + self.getDescription(test), + self.result_string, + result_code_to_suffix[self.result_code], + ) ) self.stream.writeln(single_line_delim) else: self.stream.writeln( - "%-68s %4.2f %s" + "%-67s %4.2f %s%s" % ( self.getDescription(test), time.time() - self.start_test, self.result_string, + result_code_to_suffix[self.result_code], ) ) - self.send_result_through_pipe(test, TEST_RUN) + self.send_result_through_pipe(test, TestResultCode.TEST_RUN) def printErrors(self): """ diff --git a/test/framework.py b/test/framework.py index f39794f22cf..cd39e1b311c 100644 --- a/test/framework.py +++ b/test/framework.py @@ -52,6 +52,7 @@ from scapy.layers.inet import IPerror, TCPerror, UDPerror, ICMPerror from scapy.layers.inet6 import ICMPv6DestUnreach, ICMPv6EchoRequest from scapy.layers.inet6 import ICMPv6EchoReply from vpp_running import use_running +from test_result_code import TestResultCode logger = logging.getLogger(__name__) @@ -60,13 +61,6 @@ logger = logging.getLogger(__name__) null_logger = logging.getLogger("VppTestCase") null_logger.addHandler(logging.NullHandler()) -PASS = 0 -FAIL = 1 -ERROR = 2 -SKIP = 3 -TEST_RUN = 4 -SKIP_CPU_SHORTAGE = 5 - if config.debug_framework: import debug_internal @@ -1666,6 +1660,7 @@ class VppTestResult(unittest.TestResult): self.stream = stream self.descriptions = descriptions self.verbosity = verbosity + self.result_code = TestResultCode.TEST_RUN self.result_string = None self.runner = runner self.printed = [] @@ -1677,15 +1672,25 @@ class VppTestResult(unittest.TestResult): :param test: """ - if self.current_test_case_info: - self.current_test_case_info.logger.debug( - "--- addSuccess() %s.%s(%s) called" - % (test.__class__.__name__, test._testMethodName, test._testMethodDoc) - ) + self.log_result("addSuccess", test) unittest.TestResult.addSuccess(self, test) self.result_string = colorize("OK", GREEN) - - self.send_result_through_pipe(test, PASS) + self.result_code = TestResultCode.PASS + self.send_result_through_pipe(test, self.result_code) + + def addExpectedFailure(self, test, err): + self.log_result("addExpectedFailure", test, err) + super().addExpectedFailure(test, err) + self.result_string = colorize("FAIL", GREEN) + self.result_code = TestResultCode.EXPECTED_FAIL + self.send_result_through_pipe(test, self.result_code) + + def addUnexpectedSuccess(self, test): + self.log_result("addUnexpectedSuccess", test) + super().addUnexpectedSuccess(test) + self.result_string = colorize("OK", RED) + self.result_code = TestResultCode.UNEXPECTED_PASS + self.send_result_through_pipe(test, self.result_code) def addSkip(self, test, reason): """ @@ -1695,23 +1700,15 @@ class VppTestResult(unittest.TestResult): :param reason: """ - if self.current_test_case_info: - self.current_test_case_info.logger.debug( - "--- addSkip() %s.%s(%s) called, reason is %s" - % ( - test.__class__.__name__, - test._testMethodName, - test._testMethodDoc, - reason, - ) - ) + self.log_result("addSkip", test, reason=reason) unittest.TestResult.addSkip(self, test, reason) self.result_string = colorize("SKIP", YELLOW) if reason == "not enough cpus": - self.send_result_through_pipe(test, SKIP_CPU_SHORTAGE) + self.result_code = TestResultCode.SKIP_CPU_SHORTAGE else: - self.send_result_through_pipe(test, SKIP) + self.result_code = TestResultCode.SKIP + self.send_result_through_pipe(test, self.result_code) def symlink_failed(self): if self.current_test_case_info: @@ -1743,7 +1740,7 @@ class VppTestResult(unittest.TestResult): if pipe: pipe.send((test.id(), result)) - def log_error(self, test, err, fn_name): + def log_result(self, fn, test, err=None, reason=None): if self.current_test_case_info: if isinstance(test, unittest.suite._ErrorHolder): test_name = test.description @@ -1753,25 +1750,29 @@ class VppTestResult(unittest.TestResult): test._testMethodName, test._testMethodDoc, ) + extra_msg = "" + if err: + extra_msg += f", error is {err}" + if reason: + extra_msg += f", reason is {reason}" self.current_test_case_info.logger.debug( - "--- %s() %s called, err is %s" % (fn_name, test_name, err) - ) - self.current_test_case_info.logger.debug( - "formatted exception is:\n%s" % "".join(format_exception(*err)) + f"--- {fn}() {test_name} called{extra_msg}" ) + if err: + self.current_test_case_info.logger.debug( + "formatted exception is:\n%s" % "".join(format_exception(*err)) + ) - def add_error(self, test, err, unittest_fn, error_type): - if error_type == FAIL: - self.log_error(test, err, "addFailure") + def add_error(self, test, err, unittest_fn, result_code): + self.result_code = result_code + if result_code == TestResultCode.FAIL: + self.log_result("addFailure", test, err=err) error_type_str = colorize("FAIL", RED) - elif error_type == ERROR: - self.log_error(test, err, "addError") + elif result_code == TestResultCode.ERROR: + self.log_result("addError", test, err=err) error_type_str = colorize("ERROR", RED) else: - raise Exception( - "Error type %s cannot be used to record an " - "error or a failure" % error_type - ) + raise Exception(f"Unexpected result code {result_code}") unittest_fn(self, test, err) if self.current_test_case_info: @@ -1794,7 +1795,7 @@ class VppTestResult(unittest.TestResult): else: self.result_string = "%s [no temp dir]" % error_type_str - self.send_result_through_pipe(test, error_type) + self.send_result_through_pipe(test, result_code) def addFailure(self, test, err): """ @@ -1804,7 +1805,7 @@ class VppTestResult(unittest.TestResult): :param err: error message """ - self.add_error(test, err, unittest.TestResult.addFailure, FAIL) + self.add_error(test, err, unittest.TestResult.addFailure, TestResultCode.FAIL) def addError(self, test, err): """ @@ -1814,7 +1815,7 @@ class VppTestResult(unittest.TestResult): :param err: error message """ - self.add_error(test, err, unittest.TestResult.addError, ERROR) + self.add_error(test, err, unittest.TestResult.addError, TestResultCode.ERROR) def getDescription(self, test): """ @@ -1907,23 +1908,40 @@ class VppTestResult(unittest.TestResult): """ unittest.TestResult.stopTest(self, test) + result_code_to_suffix = { + TestResultCode.PASS: "", + TestResultCode.FAIL: "", + TestResultCode.ERROR: "", + TestResultCode.SKIP: "", + TestResultCode.TEST_RUN: "", + TestResultCode.SKIP_CPU_SHORTAGE: "", + TestResultCode.EXPECTED_FAIL: " [EXPECTED FAIL]", + TestResultCode.UNEXPECTED_PASS: " [UNEXPECTED PASS]", + } + if self.verbosity > 0: self.stream.writeln(single_line_delim) self.stream.writeln( - "%-73s%s" % (self.getDescription(test), self.result_string) + "%-72s%s%s" + % ( + self.getDescription(test), + self.result_string, + result_code_to_suffix[self.result_code], + ) ) self.stream.writeln(single_line_delim) else: self.stream.writeln( - "%-68s %4.2f %s" + "%-67s %4.2f %s%s" % ( self.getDescription(test), time.time() - self.start_test, self.result_string, + result_code_to_suffix[self.result_code], ) ) - self.send_result_through_pipe(test, TEST_RUN) + self.send_result_through_pipe(test, TestResultCode.TEST_RUN) def printErrors(self): """ diff --git a/test/run_tests.py b/test/run_tests.py index 2aa7b028ffc..3cabab5c67e 100644 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -19,13 +19,8 @@ from framework import ( VppTestCase, get_testcase_doc_name, get_test_description, - PASS, - FAIL, - ERROR, - SKIP, - TEST_RUN, - SKIP_CPU_SHORTAGE, ) +from test_result_code import TestResultCode from debug import spawn_gdb from log import ( get_parallel_logger, @@ -70,12 +65,8 @@ StreamQueueManager.register("StreamQueue", StreamQueue) class TestResult(dict): def __init__(self, testcase_suite, testcases_by_id=None): super(TestResult, self).__init__() - self[PASS] = [] - self[FAIL] = [] - self[ERROR] = [] - self[SKIP] = [] - self[SKIP_CPU_SHORTAGE] = [] - self[TEST_RUN] = [] + for trc in list(TestResultCode): + self[trc] = [] self.crashed = False self.testcase_suite = testcase_suite self.testcases = [testcase for testcase in testcase_suite] @@ -83,13 +74,19 @@ class TestResult(dict): def was_successful(self): return ( - 0 == len(self[FAIL]) == len(self[ERROR]) - and len(self[PASS] + self[SKIP] + self[SKIP_CPU_SHORTAGE]) + 0 + == len(self[TestResultCode.FAIL]) + == len(self[TestResultCode.ERROR]) + == len(self[TestResultCode.UNEXPECTED_PASS]) + and len(self[TestResultCode.PASS]) + + len(self[TestResultCode.SKIP]) + + len(self[TestResultCode.SKIP_CPU_SHORTAGE]) + + len(self[TestResultCode.EXPECTED_FAIL]) == self.testcase_suite.countTestCases() ) def no_tests_run(self): - return 0 == len(self[TEST_RUN]) + return 0 == len(self[TestResultCode.TEST_RUN]) def process_result(self, test_id, result): self[result].append(test_id) @@ -98,7 +95,13 @@ class TestResult(dict): rerun_ids = set([]) for testcase in self.testcase_suite: tc_id = testcase.id() - if tc_id not in self[PASS] + self[SKIP] + self[SKIP_CPU_SHORTAGE]: + if ( + tc_id + not in self[TestResultCode.PASS] + + self[TestResultCode.SKIP] + + self[TestResultCode.SKIP_CPU_SHORTAGE] + + self[TestResultCode.EXPECTED_FAIL] + ): rerun_ids.add(tc_id) if rerun_ids: return suite_from_failed(self.testcase_suite, rerun_ids) @@ -518,7 +521,7 @@ def run_forked(testcase_suites): pass wrapped_testcase_suite.result.crashed = True wrapped_testcase_suite.result.process_result( - wrapped_testcase_suite.last_test_id, ERROR + wrapped_testcase_suite.last_test_id, TestResultCode.ERROR ) stop_run = ( process_finished_testsuite( @@ -735,20 +738,15 @@ class AllResults(dict): super(AllResults, self).__init__() self.all_testcases = 0 self.results_per_suite = [] - self[PASS] = 0 - self[FAIL] = 0 - self[ERROR] = 0 - self[SKIP] = 0 - self[SKIP_CPU_SHORTAGE] = 0 - self[TEST_RUN] = 0 + for trc in list(TestResultCode): + self[trc] = 0 self.rerun = [] self.testsuites_no_tests_run = [] def add_results(self, result): self.results_per_suite.append(result) - result_types = [PASS, FAIL, ERROR, SKIP, TEST_RUN, SKIP_CPU_SHORTAGE] - for result_type in result_types: - self[result_type] += len(result[result_type]) + for trc in list(TestResultCode): + self[trc] += len(result[trc]) def add_result(self, result): retval = 0 @@ -785,19 +783,29 @@ class AllResults(dict): indent_results( [ f"Scheduled tests: {self.all_testcases}", - f"Executed tests: {self[TEST_RUN]}", - f"Passed tests: {colorize(self[PASS], GREEN)}", - f"Skipped tests: {colorize(self[SKIP], YELLOW)}" - if self[SKIP] + f"Executed tests: {self[TestResultCode.TEST_RUN]}", + f"Passed tests: {colorize(self[TestResultCode.PASS], GREEN)}", + f"Expected failures: {colorize(self[TestResultCode.EXPECTED_FAIL], GREEN)}" + if self[TestResultCode.EXPECTED_FAIL] + else None, + f"Skipped tests: {colorize(self[TestResultCode.SKIP], YELLOW)}" + if self[TestResultCode.SKIP] else None, f"Not Executed tests: {colorize(self.not_executed, RED)}" if self.not_executed else None, - f"Failures: {colorize(self[FAIL], RED)}" if self[FAIL] else None, - f"Errors: {colorize(self[ERROR], RED)}" if self[ERROR] else None, + f"Failures: {colorize(self[TestResultCode.FAIL], RED)}" + if self[TestResultCode.FAIL] + else None, + f"Unexpected passes: {colorize(self[TestResultCode.UNEXPECTED_PASS], RED)}" + if self[TestResultCode.UNEXPECTED_PASS] + else None, + f"Errors: {colorize(self[TestResultCode.ERROR], RED)}" + if self[TestResultCode.ERROR] + else None, "Tests skipped due to lack of CPUS: " - f"{colorize(self[SKIP_CPU_SHORTAGE], YELLOW)}" - if self[SKIP_CPU_SHORTAGE] + f"{colorize(self[TestResultCode.SKIP_CPU_SHORTAGE], YELLOW)}" + if self[TestResultCode.SKIP_CPU_SHORTAGE] else None, ] ) @@ -805,43 +813,28 @@ class AllResults(dict): if self.all_failed > 0: print("FAILURES AND ERRORS IN TESTS:") for result in self.results_per_suite: - failed_testcase_ids = result[FAIL] - errored_testcase_ids = result[ERROR] old_testcase_name = None - if failed_testcase_ids: - for failed_test_id in failed_testcase_ids: + for tr_code, headline in ( + (TestResultCode.FAIL, "FAILURE"), + (TestResultCode.ERROR, "ERROR"), + (TestResultCode.UNEXPECTED_PASS, "UNEXPECTED PASS"), + ): + if not result[tr_code]: + continue + + for failed_test_id in result[tr_code]: new_testcase_name, test_name = result.get_testcase_names( failed_test_id ) if new_testcase_name != old_testcase_name: print( - " Testcase name: {}".format( - colorize(new_testcase_name, RED) - ) - ) - old_testcase_name = new_testcase_name - print( - " FAILURE: {} [{}]".format( - colorize(test_name, RED), failed_test_id - ) - ) - if errored_testcase_ids: - for errored_test_id in errored_testcase_ids: - new_testcase_name, test_name = result.get_testcase_names( - errored_test_id - ) - if new_testcase_name != old_testcase_name: - print( - " Testcase name: {}".format( - colorize(new_testcase_name, RED) - ) + f" Testcase name: {colorize(new_testcase_name, RED)}" ) old_testcase_name = new_testcase_name print( - " ERROR: {} [{}]".format( - colorize(test_name, RED), errored_test_id - ) + f" {headline}: {colorize(test_name, RED)} [{failed_test_id}]" ) + if self.testsuites_no_tests_run: print("TESTCASES WHERE NO TESTS WERE SUCCESSFULLY EXECUTED:") tc_classes = set() @@ -851,7 +844,7 @@ class AllResults(dict): for tc_class in tc_classes: print(" {}".format(colorize(tc_class, RED))) - if self[SKIP_CPU_SHORTAGE]: + if self[TestResultCode.SKIP_CPU_SHORTAGE]: print() print( colorize( @@ -865,11 +858,15 @@ class AllResults(dict): @property def not_executed(self): - return self.all_testcases - self[TEST_RUN] + return self.all_testcases - self[TestResultCode.TEST_RUN] @property def all_failed(self): - return self[FAIL] + self[ERROR] + return ( + self[TestResultCode.FAIL] + + self[TestResultCode.ERROR] + + self[TestResultCode.UNEXPECTED_PASS] + ) def parse_results(results): diff --git a/test/test_result_code.py b/test/test_result_code.py new file mode 100644 index 00000000000..93e8324a5a8 --- /dev/null +++ b/test/test_result_code.py @@ -0,0 +1,15 @@ +#!/usr/bin/python3 + +from enum import IntEnum, auto, unique + + +@unique +class TestResultCode(IntEnum): + PASS = auto() + FAIL = auto() + ERROR = auto() + SKIP = auto() + TEST_RUN = auto() + SKIP_CPU_SHORTAGE = auto() + EXPECTED_FAIL = auto() + UNEXPECTED_PASS = auto() |