summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAndrew Yourtchenko <ayourtch@gmail.com>2020-08-26 14:33:54 +0000
committerOle Trøan <otroan@employees.org>2020-08-27 08:03:38 +0000
commita3b7c554c669afc627f9a1e32666211bb6fb6b25 (patch)
tree625a8fd262f823d0bd79ec779430a6df905368df
parentd135487aff499b3450fd8d3013b6af52d80e96a1 (diff)
tests: "force solo" testcase support
Some of the tests are time-sensitive, and at present require a non-trivial modification in order to run at high concurrency. Without these modifications, they intermittently fail, and require the test retries. Rather than setting them to the extended tests and forgetting about them, put them into a "solo" set, which gets run in a single-threaded mode after the rest of the tests are done. Mark a few of the tests that showed errors during TEST_JOBS=48 as forced-solo. Also, give a better diagnostic if the testcase misses a docstring needed to represent it in the diagnostic outputs. Type: fix Change-Id: I33fe62eb17edc1885bd2c3523892051d52da6546 Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>
-rw-r--r--src/plugins/dhcp/test/test_dhcp.py4
-rw-r--r--src/plugins/dhcp/test/test_dhcp6.py4
-rw-r--r--src/plugins/flowprobe/test/test_flowprobe.py4
-rw-r--r--src/plugins/memif/test/test_memif.py4
-rw-r--r--test/framework.py17
-rw-r--r--test/run_tests.py52
-rw-r--r--test/test_bfd.py24
-rw-r--r--test/test_ip6.py4
-rw-r--r--test/test_session.py4
-rwxr-xr-xtest/test_util.py9
10 files changed, 118 insertions, 8 deletions
diff --git a/src/plugins/dhcp/test/test_dhcp.py b/src/plugins/dhcp/test/test_dhcp.py
index 065683fbcda..266932c3830 100644
--- a/src/plugins/dhcp/test/test_dhcp.py
+++ b/src/plugins/dhcp/test/test_dhcp.py
@@ -36,6 +36,10 @@ class TestDHCP(VppTestCase):
""" DHCP Test Case """
@classmethod
+ def force_solo(cls):
+ return True
+
+ @classmethod
def setUpClass(cls):
super(TestDHCP, cls).setUpClass()
diff --git a/src/plugins/dhcp/test/test_dhcp6.py b/src/plugins/dhcp/test/test_dhcp6.py
index 7254496f48f..ebd43729e4b 100644
--- a/src/plugins/dhcp/test/test_dhcp6.py
+++ b/src/plugins/dhcp/test/test_dhcp6.py
@@ -224,6 +224,10 @@ class TestDHCPv6IANAControlPlane(VppTestCase):
""" DHCPv6 IA NA Control Plane Test Case """
@classmethod
+ def force_solo(cls):
+ return True
+
+ @classmethod
def setUpClass(cls):
super(TestDHCPv6IANAControlPlane, cls).setUpClass()
diff --git a/src/plugins/flowprobe/test/test_flowprobe.py b/src/plugins/flowprobe/test/test_flowprobe.py
index 092e8d3951d..4cf019ad47e 100644
--- a/src/plugins/flowprobe/test/test_flowprobe.py
+++ b/src/plugins/flowprobe/test/test_flowprobe.py
@@ -347,6 +347,10 @@ class Flowprobe(MethodHolder):
"""Template verification, timer tests"""
@classmethod
+ def force_solo(cls):
+ return True
+
+ @classmethod
def setUpClass(cls):
super(Flowprobe, cls).setUpClass()
diff --git a/src/plugins/memif/test/test_memif.py b/src/plugins/memif/test/test_memif.py
index f11dd89f975..244818cfc3a 100644
--- a/src/plugins/memif/test/test_memif.py
+++ b/src/plugins/memif/test/test_memif.py
@@ -17,6 +17,10 @@ class TestMemif(VppTestCase):
""" Memif Test Case """
@classmethod
+ def force_solo(cls):
+ return True
+
+ @classmethod
def setUpClass(cls):
# fork new process before client connects to VPP
cls.remote_test = RemoteClass(RemoteVppTestCase)
diff --git a/test/framework.py b/test/framework.py
index c9d72a768bb..ba5f4014798 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -299,6 +299,11 @@ class VppTestCase(unittest.TestCase):
return 0
@classmethod
+ def force_solo(cls):
+ """ if the test case class is timing-sensitive - return true """
+ return False
+
+ @classmethod
def instance(cls):
"""Return the instance of this testcase"""
return cls.test_instance
@@ -1412,9 +1417,19 @@ class VppTestResult(unittest.TestResult):
"""
def print_header(test):
+ test_doc = getdoc(test)
+ if not test_doc:
+ raise Exception("No doc string for test '%s'" % test.id())
+ test_title = test_doc.splitlines()[0]
+ test_title_colored = colorize(test_title, GREEN)
+ if test.force_solo():
+ # long live PEP-8 and 80 char width limitation...
+ c = YELLOW
+ test_title_colored = colorize("SOLO RUN: " + test_title, c)
+
if not hasattr(test.__class__, '_header_printed'):
print(double_line_delim)
- print(colorize(getdoc(test).splitlines()[0], GREEN))
+ print(test_title_colored)
print(double_line_delim)
test.__class__._header_printed = True
diff --git a/test/run_tests.py b/test/run_tests.py
index 499d6df1290..66118ca6f41 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -324,6 +324,8 @@ def process_finished_testsuite(wrapped_testcase_suite,
def run_forked(testcase_suites):
wrapped_testcase_suites = set()
+ solo_testcase_suites = []
+ total_test_runners = 0
# suites are unhashable, need to use list
results = []
@@ -331,12 +333,29 @@ def run_forked(testcase_suites):
finished_unread_testcases = set()
manager = StreamQueueManager()
manager.start()
- for i in range(concurrent_tests):
+ total_test_runners = 0
+ while total_test_runners < concurrent_tests:
if testcase_suites:
- wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
+ a_suite = testcase_suites.pop(0)
+ if a_suite.force_solo:
+ solo_testcase_suites.append(a_suite)
+ continue
+ wrapped_testcase_suite = TestCaseWrapper(a_suite,
manager)
wrapped_testcase_suites.add(wrapped_testcase_suite)
unread_testcases.add(wrapped_testcase_suite)
+ total_test_runners = total_test_runners + 1
+ else:
+ break
+
+ while total_test_runners < 1 and solo_testcase_suites:
+ if solo_testcase_suites:
+ a_suite = solo_testcase_suites.pop(0)
+ wrapped_testcase_suite = TestCaseWrapper(a_suite,
+ manager)
+ wrapped_testcase_suites.add(wrapped_testcase_suite)
+ unread_testcases.add(wrapped_testcase_suite)
+ total_test_runners = total_test_runners + 1
else:
break
@@ -448,14 +467,32 @@ def run_forked(testcase_suites):
wrapped_testcase_suites.remove(finished_testcase)
finished_unread_testcases.add(finished_testcase)
finished_testcase.stdouterr_queue.put(None)
+ total_test_runners = total_test_runners - 1
if stop_run:
while testcase_suites:
results.append(TestResult(testcase_suites.pop(0)))
elif testcase_suites:
- new_testcase = TestCaseWrapper(testcase_suites.pop(0),
- manager)
- wrapped_testcase_suites.add(new_testcase)
- unread_testcases.add(new_testcase)
+ a_testcase = testcase_suites.pop(0)
+ while a_testcase and a_testcase.force_solo:
+ solo_testcase_suites.append(a_testcase)
+ if testcase_suites:
+ a_testcase = testcase_suites.pop(0)
+ else:
+ a_testcase = None
+ if a_testcase:
+ new_testcase = TestCaseWrapper(a_testcase,
+ manager)
+ wrapped_testcase_suites.add(new_testcase)
+ total_test_runners = total_test_runners + 1
+ unread_testcases.add(new_testcase)
+ else:
+ if solo_testcase_suites and total_test_runners == 0:
+ a_testcase = solo_testcase_suites.pop(0)
+ new_testcase = TestCaseWrapper(a_testcase,
+ manager)
+ wrapped_testcase_suites.add(new_testcase)
+ total_test_runners = total_test_runners + 1
+ unread_testcases.add(new_testcase)
time.sleep(0.1)
except Exception:
for wrapped_testcase_suite in wrapped_testcase_suites:
@@ -484,7 +521,10 @@ class SplitToSuitesCallback:
self.suite_name = file_name + cls.__name__
if self.suite_name not in self.suites:
self.suites[self.suite_name] = unittest.TestSuite()
+ self.suites[self.suite_name].force_solo = False
self.suites[self.suite_name].addTest(test_method)
+ if test_method.force_solo():
+ self.suites[self.suite_name].force_solo = True
else:
self.filtered.addTest(test_method)
diff --git a/test/test_bfd.py b/test/test_bfd.py
index 67b62766c21..f66f75a3408 100644
--- a/test/test_bfd.py
+++ b/test/test_bfd.py
@@ -686,6 +686,10 @@ class BFD4TestCase(VppTestCase):
test_session = None
@classmethod
+ def force_solo(cls):
+ return True
+
+ @classmethod
def setUpClass(cls):
super(BFD4TestCase, cls).setUpClass()
cls.vapi.cli("set log class bfd level debug")
@@ -1494,6 +1498,10 @@ class BFD6TestCase(VppTestCase):
test_session = None
@classmethod
+ def force_solo(cls):
+ return True
+
+ @classmethod
def setUpClass(cls):
super(BFD6TestCase, cls).setUpClass()
cls.vapi.cli("set log class bfd level debug")
@@ -1705,6 +1713,10 @@ class BFDFIBTestCase(VppTestCase):
test_session = None
@classmethod
+ def force_solo(cls):
+ return True
+
+ @classmethod
def setUpClass(cls):
super(BFDFIBTestCase, cls).setUpClass()
@@ -1893,6 +1905,10 @@ class BFDSHA1TestCase(VppTestCase):
test_session = None
@classmethod
+ def force_solo(cls):
+ return True
+
+ @classmethod
def setUpClass(cls):
super(BFDSHA1TestCase, cls).setUpClass()
cls.vapi.cli("set log class bfd level debug")
@@ -2123,6 +2139,10 @@ class BFDAuthOnOffTestCase(VppTestCase):
test_session = None
@classmethod
+ def force_solo(cls):
+ return True
+
+ @classmethod
def setUpClass(cls):
super(BFDAuthOnOffTestCase, cls).setUpClass()
cls.vapi.cli("set log class bfd level debug")
@@ -2332,6 +2352,10 @@ class BFDCLITestCase(VppTestCase):
pg0 = None
@classmethod
+ def force_solo(cls):
+ return True
+
+ @classmethod
def setUpClass(cls):
super(BFDCLITestCase, cls).setUpClass()
cls.vapi.cli("set log class bfd level debug")
diff --git a/test/test_ip6.py b/test/test_ip6.py
index f8a2dc24e42..36ef5777207 100644
--- a/test/test_ip6.py
+++ b/test/test_ip6.py
@@ -165,6 +165,10 @@ class TestIPv6(TestIPv6ND):
""" IPv6 Test Case """
@classmethod
+ def force_solo(cls):
+ return True
+
+ @classmethod
def setUpClass(cls):
super(TestIPv6, cls).setUpClass()
diff --git a/test/test_session.py b/test/test_session.py
index 94218d08e8e..309bf6e32e4 100644
--- a/test/test_session.py
+++ b/test/test_session.py
@@ -121,6 +121,10 @@ class TestSvmFifoUnitTests(VppTestCase):
""" SVM Fifo Unit Tests Case """
@classmethod
+ def force_solo(cls):
+ return True
+
+ @classmethod
def setUpClass(cls):
super(TestSvmFifoUnitTests, cls).setUpClass()
diff --git a/test/test_util.py b/test/test_util.py
index 51cc739ca8d..8501881a065 100755
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -7,8 +7,15 @@ from vpp_papi import mac_pton, mac_ntop
class TestUtil (unittest.TestCase):
- """ MAC to binary and back """
+ """ Test framework utility tests """
+
+ @classmethod
+ def force_solo(cls):
+ """ if the test case class is timing-sensitive - return true """
+ return False
+
def test_mac_to_binary(self):
+ """ MAC to binary and back """
mac = 'aa:bb:cc:dd:ee:ff'
b = mac_pton(mac)
mac2 = mac_ntop(b)