tests: "force solo" testcase support

Some of the tests are time-sensitive, and at present require a non-trivial
modification in order to run at high concurrency.

Without these modifications, they intermittently fail, and require
the test retries.

Rather than setting them to the extended tests and forgetting
about them, put them into a "solo" set, which gets run in a
single-threaded mode after the rest of the tests are done.

Mark a few of the tests that showed errors during TEST_JOBS=48
as forced-solo.

Also, give a better diagnostic if the testcase misses a docstring
needed to represent it in the diagnostic outputs.

Type: fix

Change-Id: I33fe62eb17edc1885bd2c3523892051d52da6546
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>

1 files changed, 46 insertions, 6 deletions

diff --git a/test/run_tests.py b/test/run_tests.py
index 499d6df1290..66118ca6f41 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -324,6 +324,8 @@ def process_finished_testsuite(wrapped_testcase_suite,
 
 def run_forked(testcase_suites):
     wrapped_testcase_suites = set()
+    solo_testcase_suites = []
+    total_test_runners = 0
 
     # suites are unhashable, need to use list
     results = []
@@ -331,12 +333,29 @@ def run_forked(testcase_suites):
     finished_unread_testcases = set()
     manager = StreamQueueManager()
     manager.start()
-    for i in range(concurrent_tests):
+    total_test_runners = 0
+    while total_test_runners < concurrent_tests:
         if testcase_suites:
-            wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
+            a_suite = testcase_suites.pop(0)
+            if a_suite.force_solo:
+                solo_testcase_suites.append(a_suite)
+                continue
+            wrapped_testcase_suite = TestCaseWrapper(a_suite,
                                                      manager)
             wrapped_testcase_suites.add(wrapped_testcase_suite)
             unread_testcases.add(wrapped_testcase_suite)
+            total_test_runners = total_test_runners + 1
+        else:
+            break
+
+    while total_test_runners < 1 and solo_testcase_suites:
+        if solo_testcase_suites:
+            a_suite = solo_testcase_suites.pop(0)
+            wrapped_testcase_suite = TestCaseWrapper(a_suite,
+                                                     manager)
+            wrapped_testcase_suites.add(wrapped_testcase_suite)
+            unread_testcases.add(wrapped_testcase_suite)
+            total_test_runners = total_test_runners + 1
         else:
             break
 
@@ -448,14 +467,32 @@ def run_forked(testcase_suites):
                 wrapped_testcase_suites.remove(finished_testcase)
                 finished_unread_testcases.add(finished_testcase)
                 finished_testcase.stdouterr_queue.put(None)
+                total_test_runners = total_test_runners - 1
                 if stop_run:
                     while testcase_suites:
                         results.append(TestResult(testcase_suites.pop(0)))
                 elif testcase_suites:
-                    new_testcase = TestCaseWrapper(testcase_suites.pop(0),
-                                                   manager)
-                    wrapped_testcase_suites.add(new_testcase)
-                    unread_testcases.add(new_testcase)
+                    a_testcase = testcase_suites.pop(0)
+                    while a_testcase and a_testcase.force_solo:
+                        solo_testcase_suites.append(a_testcase)
+                        if testcase_suites:
+                            a_testcase = testcase_suites.pop(0)
+                        else:
+                            a_testcase = None
+                    if a_testcase:
+                        new_testcase = TestCaseWrapper(a_testcase,
+                                                       manager)
+                        wrapped_testcase_suites.add(new_testcase)
+                        total_test_runners = total_test_runners + 1
+                        unread_testcases.add(new_testcase)
+                else:
+                    if solo_testcase_suites and total_test_runners == 0:
+                        a_testcase = solo_testcase_suites.pop(0)
+                        new_testcase = TestCaseWrapper(a_testcase,
+                                                       manager)
+                        wrapped_testcase_suites.add(new_testcase)
+                        total_test_runners = total_test_runners + 1
+                        unread_testcases.add(new_testcase)
             time.sleep(0.1)
     except Exception:
         for wrapped_testcase_suite in wrapped_testcase_suites:
@@ -484,7 +521,10 @@ class SplitToSuitesCallback:
             self.suite_name = file_name + cls.__name__
             if self.suite_name not in self.suites:
                 self.suites[self.suite_name] = unittest.TestSuite()
+                self.suites[self.suite_name].force_solo = False
             self.suites[self.suite_name].addTest(test_method)
+            if test_method.force_solo():
+                self.suites[self.suite_name].force_solo = True
 
         else:
             self.filtered.addTest(test_method)