tests: add generalized tags for tests, use them for run-solo tests

We have accumulated several scenarios in prod or wishlists
where it would be useful to have a general infra to say yes/no
about a certain test, and potentially make decisions based on that,
for example:

- runs solo (aka 'time-dependent')
- (wishlist) part of quick smoke-test set
- (wishlist) intermittent failure unrelated to timing
- (wishlist) test broken with a multi-worker config in vpp

Refactor the current "run-solo" code to allow for this extension.

Type: test

Change-Id: Ia5b3810e57c0543753c8e0dc4dc0cfb4a30b36ac
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>
Signed-off-by: Klement Sekera <ksekera@cisco.com>

1 files changed, 29 insertions, 3 deletions

diff --git a/test/framework.py b/test/framework.py
index f5775134ded..7ab5b453b8e 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -21,6 +21,7 @@ from threading import Thread, Event
 from inspect import getdoc, isclass
 from traceback import format_exception
 from logging import FileHandler, DEBUG, Formatter
+from enum import Enum
 
 import scapy.compat
 from scapy.packet import Raw
@@ -255,6 +256,22 @@ class KeepAliveReporter(object):
         self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid))
 
 
+class TestCaseTag(Enum):
+    RUN_SOLO = 1
+
+
+def create_tag_decorator(e):
+    def decorator(cls):
+        try:
+            cls.test_tags.append(e)
+        except AttributeError:
+            cls.test_tags = [e]
+        return cls
+    return decorator
+
+tag_run_solo = create_tag_decorator(TestCaseTag.RUN_SOLO)
+
+
 class VppTestCase(unittest.TestCase):
     """This subclass is a base class for VPP test cases that are implemented as
     classes. It provides methods to create and run test case.
@@ -279,11 +296,20 @@ class VppTestCase(unittest.TestCase):
             return 0
 
     @classmethod
-    def force_solo(cls):
-        """ if the test case class is timing-sensitive - return true """
+    def has_tag(cls, tag):
+        """ if the test case has a given tag - return true """
+        try:
+            return tag in cls.test_tags
+        except AttributeError:
+            pass
         return False
 
     @classmethod
+    def is_tagged_run_solo(cls):
+        """ if the test case class is timing-sensitive - return true """
+        return cls.has_tag(TestCaseTag.RUN_SOLO)
+
+    @classmethod
     def instance(cls):
         """Return the instance of this testcase"""
         return cls.test_instance
@@ -1404,7 +1430,7 @@ class VppTestResult(unittest.TestResult):
                 raise Exception("No doc string for test '%s'" % test.id())
             test_title = test_doc.splitlines()[0]
             test_title_colored = colorize(test_title, GREEN)
-            if test.force_solo():
+            if test.is_tagged_run_solo():
                 # long live PEP-8 and 80 char width limitation...
                 c = YELLOW
                 test_title_colored = colorize("SOLO RUN: " + test_title, c)