aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAndrew Yourtchenko <ayourtch@gmail.com>2021-01-14 10:19:08 +0000
committerAndrew Yourtchenko <ayourtch@gmail.com>2021-01-22 15:35:11 +0000
commit06f328129a01276858fff1086215478fa106dd8e (patch)
tree6d6c0091b4145466fdfeb8fbe6616ff1f09b4d08
parentb8f6122b4f4c828dee103d1f3116d27e6e3e6f3a (diff)
tests: add generalized tags for tests, use them for run-solo tests
We have accumulated several scenarios in prod or wishlists where it would be useful to have a general infra to say yes/no about a certain test, and potentially make decisions based on that, for example: - runs solo (aka 'time-dependent') - (wishlist) part of quick smoke-test set - (wishlist) intermittent failure unrelated to timing - (wishlist) test broken with a multi-worker config in vpp Refactor the current "run-solo" code to allow for this extension. Type: test Change-Id: Ia5b3810e57c0543753c8e0dc4dc0cfb4a30b36ac Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com> Signed-off-by: Klement Sekera <ksekera@cisco.com>
-rw-r--r--src/plugins/dhcp/test/test_dhcp.py6
-rw-r--r--src/plugins/dhcp/test/test_dhcp6.py6
-rw-r--r--src/plugins/flowprobe/test/test_flowprobe.py6
-rw-r--r--src/plugins/memif/test/test_memif.py6
-rw-r--r--src/vnet/bfd/test/test_bfd.py31
-rw-r--r--test/framework.py32
-rw-r--r--test/run_tests.py10
-rw-r--r--test/test_ip6.py7
-rw-r--r--test/test_session.py6
-rwxr-xr-xtest/test_util.py2
10 files changed, 54 insertions, 58 deletions
diff --git a/src/plugins/dhcp/test/test_dhcp.py b/src/plugins/dhcp/test/test_dhcp.py
index 266932c3830..e17b0049df7 100644
--- a/src/plugins/dhcp/test/test_dhcp.py
+++ b/src/plugins/dhcp/test/test_dhcp.py
@@ -6,6 +6,7 @@ import struct
import six
from framework import VppTestCase, VppTestRunner, running_extended_tests
+from framework import tag_run_solo
from vpp_neighbor import VppNeighbor
from vpp_ip_route import find_route, VppIpTable
from util import mk_ll_addr
@@ -32,14 +33,11 @@ DHCP6_CLIENT_PORT = 547
DHCP6_SERVER_PORT = 546
+@tag_run_solo
class TestDHCP(VppTestCase):
""" DHCP Test Case """
@classmethod
- def force_solo(cls):
- return True
-
- @classmethod
def setUpClass(cls):
super(TestDHCP, cls).setUpClass()
diff --git a/src/plugins/dhcp/test/test_dhcp6.py b/src/plugins/dhcp/test/test_dhcp6.py
index ebd43729e4b..e35e0e1ccda 100644
--- a/src/plugins/dhcp/test/test_dhcp6.py
+++ b/src/plugins/dhcp/test/test_dhcp6.py
@@ -9,6 +9,7 @@ from scapy.layers.inet6 import IPv6, Ether, UDP
from scapy.utils6 import in6_mactoifaceid
from framework import VppTestCase
+from framework import tag_run_solo
from vpp_papi import VppEnum
import util
import os
@@ -220,14 +221,11 @@ class TestDHCPv6DataPlane(VppTestCase):
self.vapi.dhcp6_clients_enable_disable(enable=0)
+@tag_run_solo
class TestDHCPv6IANAControlPlane(VppTestCase):
""" DHCPv6 IA NA Control Plane Test Case """
@classmethod
- def force_solo(cls):
- return True
-
- @classmethod
def setUpClass(cls):
super(TestDHCPv6IANAControlPlane, cls).setUpClass()
diff --git a/src/plugins/flowprobe/test/test_flowprobe.py b/src/plugins/flowprobe/test/test_flowprobe.py
index 4cf019ad47e..56724abe269 100644
--- a/src/plugins/flowprobe/test/test_flowprobe.py
+++ b/src/plugins/flowprobe/test/test_flowprobe.py
@@ -13,6 +13,7 @@ from scapy.layers.inet import IP, TCP, UDP
from scapy.layers.inet6 import IPv6
from framework import VppTestCase, VppTestRunner, running_extended_tests
+from framework import tag_run_solo
from vpp_object import VppObject
from vpp_pg_interface import CaptureTimeoutError
from util import ppp
@@ -343,14 +344,11 @@ class MethodHolder(VppTestCase):
return p
+@tag_run_solo
class Flowprobe(MethodHolder):
"""Template verification, timer tests"""
@classmethod
- def force_solo(cls):
- return True
-
- @classmethod
def setUpClass(cls):
super(Flowprobe, cls).setUpClass()
diff --git a/src/plugins/memif/test/test_memif.py b/src/plugins/memif/test/test_memif.py
index f2bbded320b..caaab87f1e8 100644
--- a/src/plugins/memif/test/test_memif.py
+++ b/src/plugins/memif/test/test_memif.py
@@ -5,6 +5,7 @@ from scapy.layers.l2 import Ether
from scapy.layers.inet import IP, ICMP
from framework import VppTestCase, VppTestRunner, running_extended_tests
+from framework import tag_run_solo
from remote_test import RemoteClass, RemoteVppTestCase
from vpp_memif import remove_all_memif_vpp_config, \
VppSocketFilename, VppMemif
@@ -12,14 +13,11 @@ from vpp_ip_route import VppIpRoute, VppRoutePath
from vpp_papi import VppEnum
+@tag_run_solo
class TestMemif(VppTestCase):
""" Memif Test Case """
@classmethod
- def force_solo(cls):
- return True
-
- @classmethod
def setUpClass(cls):
# fork new process before client connects to VPP
cls.remote_test = RemoteClass(RemoteVppTestCase)
diff --git a/src/vnet/bfd/test/test_bfd.py b/src/vnet/bfd/test/test_bfd.py
index 784cdadb24d..71904a89f7f 100644
--- a/src/vnet/bfd/test/test_bfd.py
+++ b/src/vnet/bfd/test/test_bfd.py
@@ -22,6 +22,7 @@ from scapy.packet import Raw
from bfd import VppBFDAuthKey, BFD, BFDAuthType, VppBFDUDPSession, \
BFDDiagCode, BFDState, BFD_vpp_echo
from framework import VppTestCase, VppTestRunner, running_extended_tests
+from framework import tag_run_solo
from util import ppp
from vpp_ip import DpoProto
from vpp_ip_route import VppIpRoute, VppRoutePath
@@ -677,6 +678,7 @@ def wait_for_bfd_packet(test, timeout=1, pcap_time_min=None, is_tunnel=False):
return p
+@tag_run_solo
class BFD4TestCase(VppTestCase):
"""Bidirectional Forwarding Detection (BFD)"""
@@ -686,10 +688,6 @@ class BFD4TestCase(VppTestCase):
test_session = None
@classmethod
- def force_solo(cls):
- return True
-
- @classmethod
def setUpClass(cls):
super(BFD4TestCase, cls).setUpClass()
cls.vapi.cli("set log class bfd level debug")
@@ -1489,6 +1487,7 @@ class BFD4TestCase(VppTestCase):
self.assertFalse(vpp_session.query_vpp_config())
+@tag_run_solo
class BFD6TestCase(VppTestCase):
"""Bidirectional Forwarding Detection (BFD) (IPv6) """
@@ -1498,10 +1497,6 @@ class BFD6TestCase(VppTestCase):
test_session = None
@classmethod
- def force_solo(cls):
- return True
-
- @classmethod
def setUpClass(cls):
super(BFD6TestCase, cls).setUpClass()
cls.vapi.cli("set log class bfd level debug")
@@ -1706,6 +1701,7 @@ class BFD6TestCase(VppTestCase):
self.assertFalse(vpp_session.query_vpp_config())
+@tag_run_solo
class BFDFIBTestCase(VppTestCase):
""" BFD-FIB interactions (IPv6) """
@@ -1713,10 +1709,6 @@ class BFDFIBTestCase(VppTestCase):
test_session = None
@classmethod
- def force_solo(cls):
- return True
-
- @classmethod
def setUpClass(cls):
super(BFDFIBTestCase, cls).setUpClass()
@@ -1896,6 +1888,7 @@ class BFDTunTestCase(VppTestCase):
bfd_session_down(self)
+@tag_run_solo
class BFDSHA1TestCase(VppTestCase):
"""Bidirectional Forwarding Detection (BFD) (SHA1 auth) """
@@ -1905,10 +1898,6 @@ class BFDSHA1TestCase(VppTestCase):
test_session = None
@classmethod
- def force_solo(cls):
- return True
-
- @classmethod
def setUpClass(cls):
super(BFDSHA1TestCase, cls).setUpClass()
cls.vapi.cli("set log class bfd level debug")
@@ -2131,6 +2120,7 @@ class BFDSHA1TestCase(VppTestCase):
bfd_session_up(self)
+@tag_run_solo
class BFDAuthOnOffTestCase(VppTestCase):
"""Bidirectional Forwarding Detection (BFD) (changing auth) """
@@ -2139,10 +2129,6 @@ class BFDAuthOnOffTestCase(VppTestCase):
test_session = None
@classmethod
- def force_solo(cls):
- return True
-
- @classmethod
def setUpClass(cls):
super(BFDAuthOnOffTestCase, cls).setUpClass()
cls.vapi.cli("set log class bfd level debug")
@@ -2347,15 +2333,12 @@ class BFDAuthOnOffTestCase(VppTestCase):
"number of bfd events")
+@tag_run_solo
class BFDCLITestCase(VppTestCase):
"""Bidirectional Forwarding Detection (BFD) (CLI) """
pg0 = None
@classmethod
- def force_solo(cls):
- return True
-
- @classmethod
def setUpClass(cls):
super(BFDCLITestCase, cls).setUpClass()
cls.vapi.cli("set log class bfd level debug")
diff --git a/test/framework.py b/test/framework.py
index f5775134ded..7ab5b453b8e 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -21,6 +21,7 @@ from threading import Thread, Event
from inspect import getdoc, isclass
from traceback import format_exception
from logging import FileHandler, DEBUG, Formatter
+from enum import Enum
import scapy.compat
from scapy.packet import Raw
@@ -255,6 +256,22 @@ class KeepAliveReporter(object):
self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid))
+class TestCaseTag(Enum):
+ RUN_SOLO = 1
+
+
+def create_tag_decorator(e):
+ def decorator(cls):
+ try:
+ cls.test_tags.append(e)
+ except AttributeError:
+ cls.test_tags = [e]
+ return cls
+ return decorator
+
+tag_run_solo = create_tag_decorator(TestCaseTag.RUN_SOLO)
+
+
class VppTestCase(unittest.TestCase):
"""This subclass is a base class for VPP test cases that are implemented as
classes. It provides methods to create and run test case.
@@ -279,11 +296,20 @@ class VppTestCase(unittest.TestCase):
return 0
@classmethod
- def force_solo(cls):
- """ if the test case class is timing-sensitive - return true """
+ def has_tag(cls, tag):
+ """ if the test case has a given tag - return true """
+ try:
+ return tag in cls.test_tags
+ except AttributeError:
+ pass
return False
@classmethod
+ def is_tagged_run_solo(cls):
+ """ if the test case class is timing-sensitive - return true """
+ return cls.has_tag(TestCaseTag.RUN_SOLO)
+
+ @classmethod
def instance(cls):
"""Return the instance of this testcase"""
return cls.test_instance
@@ -1404,7 +1430,7 @@ class VppTestResult(unittest.TestResult):
raise Exception("No doc string for test '%s'" % test.id())
test_title = test_doc.splitlines()[0]
test_title_colored = colorize(test_title, GREEN)
- if test.force_solo():
+ if test.is_tagged_run_solo():
# long live PEP-8 and 80 char width limitation...
c = YELLOW
test_title_colored = colorize("SOLO RUN: " + test_title, c)
diff --git a/test/run_tests.py b/test/run_tests.py
index d5bdfc838df..a88a69a0eb2 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -337,7 +337,7 @@ def run_forked(testcase_suites):
while total_test_runners < concurrent_tests:
if testcase_suites:
a_suite = testcase_suites.pop(0)
- if a_suite.force_solo:
+ if a_suite.is_tagged_run_solo:
solo_testcase_suites.append(a_suite)
continue
wrapped_testcase_suite = TestCaseWrapper(a_suite,
@@ -473,7 +473,7 @@ def run_forked(testcase_suites):
results.append(TestResult(testcase_suites.pop(0)))
elif testcase_suites:
a_testcase = testcase_suites.pop(0)
- while a_testcase and a_testcase.force_solo:
+ while a_testcase and a_testcase.is_tagged_run_solo:
solo_testcase_suites.append(a_testcase)
if testcase_suites:
a_testcase = testcase_suites.pop(0)
@@ -520,10 +520,10 @@ class SplitToSuitesCallback:
self.suite_name = file_name + cls.__name__
if self.suite_name not in self.suites:
self.suites[self.suite_name] = unittest.TestSuite()
- self.suites[self.suite_name].force_solo = False
+ self.suites[self.suite_name].is_tagged_run_solo = False
self.suites[self.suite_name].addTest(test_method)
- if test_method.force_solo():
- self.suites[self.suite_name].force_solo = True
+ if test_method.is_tagged_run_solo():
+ self.suites[self.suite_name].is_tagged_run_solo = True
else:
self.filtered.addTest(test_method)
diff --git a/test/test_ip6.py b/test/test_ip6.py
index 5dd2bbcff64..0ef2dd1d2bd 100644
--- a/test/test_ip6.py
+++ b/test/test_ip6.py
@@ -19,7 +19,7 @@ from scapy.utils6 import in6_getnsma, in6_getnsmac, in6_ptop, in6_islladdr, \
in6_mactoifaceid
from six import moves
-from framework import VppTestCase, VppTestRunner
+from framework import VppTestCase, VppTestRunner, tag_run_solo
from util import ppp, ip6_normalize, mk_ll_addr
from vpp_papi import VppEnum
from vpp_ip import DpoProto, VppIpPuntPolicer, VppIpPuntRedirect
@@ -162,14 +162,11 @@ class TestIPv6ND(VppTestCase):
self.assertEqual(ip.dst, dip)
+@tag_run_solo
class TestIPv6(TestIPv6ND):
""" IPv6 Test Case """
@classmethod
- def force_solo(cls):
- return True
-
- @classmethod
def setUpClass(cls):
super(TestIPv6, cls).setUpClass()
diff --git a/test/test_session.py b/test/test_session.py
index 5b1bfb77989..6854cb8a8bd 100644
--- a/test/test_session.py
+++ b/test/test_session.py
@@ -3,6 +3,7 @@
import unittest
from framework import VppTestCase, VppTestRunner
+from framework import tag_run_solo
from vpp_ip_route import VppIpTable, VppIpRoute, VppRoutePath
@@ -117,14 +118,11 @@ class TestSessionUnitTests(VppTestCase):
self.vapi.session_enable_disable(is_enable=0)
+@tag_run_solo
class TestSvmFifoUnitTests(VppTestCase):
""" SVM Fifo Unit Tests Case """
@classmethod
- def force_solo(cls):
- return True
-
- @classmethod
def setUpClass(cls):
super(TestSvmFifoUnitTests, cls).setUpClass()
diff --git a/test/test_util.py b/test/test_util.py
index 8501881a065..eb20531505e 100755
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -10,7 +10,7 @@ class TestUtil (unittest.TestCase):
""" Test framework utility tests """
@classmethod
- def force_solo(cls):
+ def is_tagged_run_solo(cls):
""" if the test case class is timing-sensitive - return true """
return False