Fix test summary and retries

There was an issue when tests crashed and weren't properly retried.

Change-Id: Id5ef828ecc9a8dc0f08c50183721db06e162e6c3
Signed-off-by: juraj.linkes <juraj.linkes@pantheon.tech>

2 files changed, 37 insertions, 24 deletions

diff --git a/test/framework.py b/test/framework.py
index 74cc7ff3a26..9ad964098c2 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -961,7 +961,7 @@ class VppTestResult(unittest.TestResult):
         self.verbosity = verbosity
         self.result_string = None
         self.printer = TestCasePrinter()
-        self.passed = 0
+        self.passed = []
 
     def addSuccess(self, test):
         """
@@ -975,7 +975,7 @@ class VppTestResult(unittest.TestResult):
                               % (test.__class__.__name__,
                                  test._testMethodName,
                                  test._testMethodDoc))
-        self.passed += 1
+        self.passed.append(test.id())
         unittest.TestResult.addSuccess(self, test)
         self.result_string = colorize("OK", GREEN)
 
diff --git a/test/run_tests.py b/test/run_tests.py
index 76b6a62b79d..89c7797d5ba 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -87,6 +87,7 @@ class TestCaseWrapper(object):
         self.last_test_vpp_binary = None
         self.last_test = None
         self.result = None
+        self.vpp_pid = None
         self.last_heard = time.time()
         self.core_detected_at = None
         self.failed_tests = []
@@ -122,7 +123,7 @@ def stdouterr_reader_wrapper(unread_testcases, finished_unread_testcases,
             read_testcase = None
 
 
-def run_forked(testcases):
+def run_forked(testcase_suites):
     wrapped_testcase_suites = set()
 
     # suites are unhashable, need to use list
@@ -133,8 +134,9 @@ def run_forked(testcases):
     manager = StreamQueueManager()
     manager.start()
     for i in range(concurrent_tests):
-        if len(testcases) > 0:
-            wrapped_testcase_suite = TestCaseWrapper(testcases.pop(0), manager)
+        if len(testcase_suites) > 0:
+            wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
+                                                     manager)
             wrapped_testcase_suites.add(wrapped_testcase_suite)
             unread_testcases.add(wrapped_testcase_suite)
             # time.sleep(1)
@@ -220,13 +222,17 @@ def run_forked(testcases):
 
             if fail:
                 failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
-                lttd = os.path.basename(
-                    wrapped_testcase_suite.last_test_temp_dir)
+                if wrapped_testcase_suite.last_test_temp_dir:
+                    lttd = os.path.basename(
+                        wrapped_testcase_suite.last_test_temp_dir)
+                else:
+                    lttd = None
                 link_path = '%s%s-FAILED' % (failed_dir, lttd)
                 wrapped_testcase_suite.logger.error(
                     "Creating a link to the failed test: %s -> %s" %
                     (link_path, lttd))
-                if not os.path.exists(link_path):
+                if not os.path.exists(link_path) \
+                        and wrapped_testcase_suite.last_test_temp_dir:
                     os.symlink(wrapped_testcase_suite.last_test_temp_dir,
                                link_path)
                 api_post_mortem_path = "/tmp/api_post_mortem.%d" % \
@@ -280,8 +286,8 @@ def run_forked(testcases):
             wrapped_testcase_suites.remove(finished_testcase)
             finished_unread_testcases.add(finished_testcase)
             finished_testcase.stdouterr_queue.put(None)
-            if len(testcases) > 0:
-                new_testcase = TestCaseWrapper(testcases.pop(0), manager)
+            if len(testcase_suites) > 0:
+                new_testcase = TestCaseWrapper(testcase_suites.pop(0), manager)
                 wrapped_testcase_suites.add(new_testcase)
                 unread_testcases.add(new_testcase)
 
@@ -385,14 +391,15 @@ class FilterByTestOption:
 
 
 class FilterByClassList:
-    def __init__(self, class_list):
-        self.class_list = class_list
+    def __init__(self, classes_with_filenames):
+        self.classes_with_filenames = classes_with_filenames
 
     def __call__(self, file_name, class_name, func_name):
-        return class_name in self.class_list
+        return '.'.join([file_name, class_name]) in self.classes_with_filenames
 
 
 def suite_from_failed(suite, failed):
+    failed = {x.rsplit('.', 1)[0] for x in failed}
     filter_cb = FilterByClassList(failed)
     suite = filter_tests(suite, filter_cb)
     return suite
@@ -442,11 +449,11 @@ class NonPassedResults(dict):
 
     def add_result(self, testcase_suite, result):
         retval = 0
-        self.all_testcases += result.testsRun
-        self.passed += result.passed
         if result:
-            # suite finished properly
-            if not result.wasSuccessful():
+            self.all_testcases += result.testsRun
+            self.passed += len(result.passed)
+            if not len(result.passed) + len(result.skipped) \
+                    == testcase_suite.countTestCases():
                 retval = 1
 
             self.add_results(result.failures, self.failures_id)
@@ -456,16 +463,22 @@ class NonPassedResults(dict):
                              self.expectedFailures_id)
             self.add_results(result.unexpectedSuccesses,
                              self.unexpectedSuccesses_id)
+        else:
+            retval = -1
 
         if retval != 0:
             if concurrent_tests == 1:
                 if result:
-                    rerun_classes = {x[0].__class__.__name__ for
-                                     x in result.errors}
-                    rerun_classes.update({x[0].__class__.__name__ for
-                                          x in result.failures})
-                    self.rerun.append(suite_from_failed(testcase_suite,
-                                                        rerun_classes))
+                    rerun_ids = set([])
+                    skipped = [x.id() for (x, _) in result.skipped]
+                    for testcase in testcase_suite:
+                        tc_id = testcase.id()
+                        if tc_id not in result.passed and \
+                                tc_id not in skipped:
+                            rerun_ids.add(tc_id)
+                    if len(rerun_ids) > 0:
+                        self.rerun.append(suite_from_failed(testcase_suite,
+                                                            rerun_ids))
                 else:
                     self.rerun.append(testcase_suite)
             else:
@@ -648,7 +661,7 @@ if __name__ == '__main__':
     if concurrent_tests == 1:
         new_suite = unittest.TestSuite()
         for suite in suites:
-            new_suite.addTest(suite)
+            new_suite.addTests(suite)
 
         suites = [new_suite]