tests: add generalized tags for tests, use them for run-solo tests

We have accumulated several scenarios in prod or wishlists
where it would be useful to have a general infra to say yes/no
about a certain test, and potentially make decisions based on that,
for example:

- runs solo (aka 'time-dependent')
- (wishlist) part of quick smoke-test set
- (wishlist) intermittent failure unrelated to timing
- (wishlist) test broken with a multi-worker config in vpp

Refactor the current "run-solo" code to allow for this extension.

Type: test

Change-Id: Ia5b3810e57c0543753c8e0dc4dc0cfb4a30b36ac
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>
Signed-off-by: Klement Sekera <ksekera@cisco.com>

5 files changed, 39 insertions, 18 deletions

diff --git a/test/framework.py b/test/framework.py
index f5775134ded..7ab5b453b8e 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -21,6 +21,7 @@ from threading import Thread, Event
 from inspect import getdoc, isclass
 from traceback import format_exception
 from logging import FileHandler, DEBUG, Formatter
+from enum import Enum
 
 import scapy.compat
 from scapy.packet import Raw
@@ -255,6 +256,22 @@ class KeepAliveReporter(object):
         self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid))
 
 
+class TestCaseTag(Enum):
+    RUN_SOLO = 1
+
+
+def create_tag_decorator(e):
+    def decorator(cls):
+        try:
+            cls.test_tags.append(e)
+        except AttributeError:
+            cls.test_tags = [e]
+        return cls
+    return decorator
+
+tag_run_solo = create_tag_decorator(TestCaseTag.RUN_SOLO)
+
+
 class VppTestCase(unittest.TestCase):
     """This subclass is a base class for VPP test cases that are implemented as
     classes. It provides methods to create and run test case.
@@ -279,11 +296,20 @@ class VppTestCase(unittest.TestCase):
             return 0
 
     @classmethod
-    def force_solo(cls):
-        """ if the test case class is timing-sensitive - return true """
+    def has_tag(cls, tag):
+        """ if the test case has a given tag - return true """
+        try:
+            return tag in cls.test_tags
+        except AttributeError:
+            pass
         return False
 
     @classmethod
+    def is_tagged_run_solo(cls):
+        """ if the test case class is timing-sensitive - return true """
+        return cls.has_tag(TestCaseTag.RUN_SOLO)
+
+    @classmethod
     def instance(cls):
         """Return the instance of this testcase"""
         return cls.test_instance
@@ -1404,7 +1430,7 @@ class VppTestResult(unittest.TestResult):
                 raise Exception("No doc string for test '%s'" % test.id())
             test_title = test_doc.splitlines()[0]
             test_title_colored = colorize(test_title, GREEN)
-            if test.force_solo():
+            if test.is_tagged_run_solo():
                 # long live PEP-8 and 80 char width limitation...
                 c = YELLOW
                 test_title_colored = colorize("SOLO RUN: " + test_title, c)
diff --git a/test/run_tests.py b/test/run_tests.py
index d5bdfc838df..a88a69a0eb2 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -337,7 +337,7 @@ def run_forked(testcase_suites):
     while total_test_runners < concurrent_tests:
         if testcase_suites:
             a_suite = testcase_suites.pop(0)
-            if a_suite.force_solo:
+            if a_suite.is_tagged_run_solo:
                 solo_testcase_suites.append(a_suite)
                 continue
             wrapped_testcase_suite = TestCaseWrapper(a_suite,
@@ -473,7 +473,7 @@ def run_forked(testcase_suites):
                         results.append(TestResult(testcase_suites.pop(0)))
                 elif testcase_suites:
                     a_testcase = testcase_suites.pop(0)
-                    while a_testcase and a_testcase.force_solo:
+                    while a_testcase and a_testcase.is_tagged_run_solo:
                         solo_testcase_suites.append(a_testcase)
                         if testcase_suites:
                             a_testcase = testcase_suites.pop(0)
@@ -520,10 +520,10 @@ class SplitToSuitesCallback:
             self.suite_name = file_name + cls.__name__
             if self.suite_name not in self.suites:
                 self.suites[self.suite_name] = unittest.TestSuite()
-                self.suites[self.suite_name].force_solo = False
+                self.suites[self.suite_name].is_tagged_run_solo = False
             self.suites[self.suite_name].addTest(test_method)
-            if test_method.force_solo():
-                self.suites[self.suite_name].force_solo = True
+            if test_method.is_tagged_run_solo():
+                self.suites[self.suite_name].is_tagged_run_solo = True
 
         else:
             self.filtered.addTest(test_method)
diff --git a/test/test_ip6.py b/test/test_ip6.py
index 5dd2bbcff64..0ef2dd1d2bd 100644
--- a/test/test_ip6.py
+++ b/test/test_ip6.py
@@ -19,7 +19,7 @@ from scapy.utils6 import in6_getnsma, in6_getnsmac, in6_ptop, in6_islladdr, \
     in6_mactoifaceid
 from six import moves
 
-from framework import VppTestCase, VppTestRunner
+from framework import VppTestCase, VppTestRunner, tag_run_solo
 from util import ppp, ip6_normalize, mk_ll_addr
 from vpp_papi import VppEnum
 from vpp_ip import DpoProto, VppIpPuntPolicer, VppIpPuntRedirect
@@ -162,14 +162,11 @@ class TestIPv6ND(VppTestCase):
         self.assertEqual(ip.dst, dip)
 
 
+@tag_run_solo
 class TestIPv6(TestIPv6ND):
     """ IPv6 Test Case """
 
     @classmethod
-    def force_solo(cls):
-        return True
-
-    @classmethod
     def setUpClass(cls):
         super(TestIPv6, cls).setUpClass()
 
diff --git a/test/test_session.py b/test/test_session.py
index 5b1bfb77989..6854cb8a8bd 100644
--- a/test/test_session.py
+++ b/test/test_session.py
@@ -3,6 +3,7 @@
 import unittest
 
 from framework import VppTestCase, VppTestRunner
+from framework import tag_run_solo
 from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
 
 
@@ -117,14 +118,11 @@ class TestSessionUnitTests(VppTestCase):
         self.vapi.session_enable_disable(is_enable=0)
 
 
+@tag_run_solo
 class TestSvmFifoUnitTests(VppTestCase):
     """ SVM Fifo Unit Tests Case """
 
     @classmethod
-    def force_solo(cls):
-        return True
-
-    @classmethod
     def setUpClass(cls):
         super(TestSvmFifoUnitTests, cls).setUpClass()
 
diff --git a/test/test_util.py b/test/test_util.py
index 8501881a065..eb20531505e 100755
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -10,7 +10,7 @@ class TestUtil (unittest.TestCase):
     """ Test framework utility tests """
 
     @classmethod
-    def force_solo(cls):
+    def is_tagged_run_solo(cls):
         """ if the test case class is timing-sensitive - return true """
         return False