diff options
| -rw-r--r-- | test/framework.py | 58 | ||||
| -rw-r--r-- | test/hook.py | 2 | ||||
| -rw-r--r-- | test/test_bfd.py | 14 | ||||
| -rw-r--r-- | test/test_flowprobe.py | 9 | ||||
| -rw-r--r-- | test/test_ikev2.py | 9 | ||||
| -rw-r--r-- | test/test_memif.py | 6 | ||||
| -rw-r--r-- | test/test_nat44_ed.py | 4 | ||||
| -rw-r--r-- | test/test_nat44_ei.py | 4 | ||||
| -rw-r--r-- | test/test_nat64.py | 5 | ||||
| -rw-r--r-- | test/test_neighbor.py | 3 | ||||
| -rw-r--r-- | test/test_wireguard.py | 10 |
11 files changed, 116 insertions, 8 deletions
diff --git a/test/framework.py b/test/framework.py index c85dec5dbdf..d130855bb55 100644 --- a/test/framework.py +++ b/test/framework.py @@ -147,6 +147,8 @@ class _PacketInfo(object): def pump_output(testclass): """pump output from vpp stdout/stderr to proper queues""" + if not hasattr(testclass, "vpp"): + return stdout_fragment = "" stderr_fragment = "" while not testclass.pump_thread_stop_flag.is_set(): @@ -212,6 +214,17 @@ def _is_distro_ubuntu2204(): is_distro_ubuntu2204 = _is_distro_ubuntu2204() +def _is_distro_debian11(): + with open("/etc/os-release") as f: + for line in f.readlines(): + if "bullseye" in line: + return True + return False + + +is_distro_debian11 = _is_distro_debian11() + + class KeepAliveReporter(object): """ Singleton object which reports test start to parent process @@ -237,7 +250,7 @@ class KeepAliveReporter(object): """ Write current test tmpdir & desc to keep-alive pipe to signal liveness """ - if self.pipe is None: + if not hasattr(test, "vpp") or self.pipe is None: # if not running forked.. return @@ -259,6 +272,8 @@ class TestCaseTag(Enum): FIXME_ASAN = 3 # marks suites broken on Ubuntu-22.04 FIXME_UBUNTU2204 = 4 + # marks suites broken on Debian-11 + FIXME_DEBIAN11 = 5 def create_tag_decorator(e): @@ -276,6 +291,7 @@ tag_run_solo = create_tag_decorator(TestCaseTag.RUN_SOLO) tag_fixme_vpp_workers = create_tag_decorator(TestCaseTag.FIXME_VPP_WORKERS) tag_fixme_asan = create_tag_decorator(TestCaseTag.FIXME_ASAN) tag_fixme_ubuntu2204 = create_tag_decorator(TestCaseTag.FIXME_UBUNTU2204) +tag_fixme_debian11 = create_tag_decorator(TestCaseTag.FIXME_DEBIAN11) class DummyVpp: @@ -358,6 +374,12 @@ class VppTestCase(CPUInterface, unittest.TestCase): cls = unittest.skip("Skipping @tag_fixme_ubuntu2204 tests")(cls) @classmethod + def skip_fixme_debian11(cls): + """if distro is Debian-11 and @tag_fixme_debian11 mark for skip""" + if cls.has_tag(TestCaseTag.FIXME_DEBIAN11): + cls = unittest.skip("Skipping @tag_fixme_debian11 tests")(cls) + + @classmethod def instance(cls): """Return the instance of this testcase""" return cls.test_instance @@ -556,6 +578,10 @@ class VppTestCase(CPUInterface, unittest.TestCase): @classmethod def run_vpp(cls): + if ( + is_distro_ubuntu2204 == True and cls.has_tag(TestCaseTag.FIXME_UBUNTU2204) + ) or (is_distro_debian11 == True and cls.has_tag(TestCaseTag.FIXME_DEBIAN11)): + return cls.logger.debug(f"Assigned cpus: {cls.cpus}") cmdline = cls.vpp_cmdline @@ -694,6 +720,8 @@ class VppTestCase(CPUInterface, unittest.TestCase): cls.attach_vpp() else: cls.run_vpp() + if not hasattr(cls, "vpp"): + return cls.reporter.send_keep_alive(cls, "setUpClass") VppTestResult.current_test_case_info = TestCaseInfo( cls.logger, cls.tempdir, cls.vpp.pid, config.vpp @@ -854,6 +882,8 @@ class VppTestCase(CPUInterface, unittest.TestCase): def tearDownClass(cls): """Perform final cleanup after running all tests in this test-case""" cls.logger.debug("--- tearDownClass() for %s called ---" % cls.__name__) + if not hasattr(cls, "vpp"): + return cls.reporter.send_keep_alive(cls, "tearDownClass") cls.quit() cls.file_handler.close() @@ -871,6 +901,8 @@ class VppTestCase(CPUInterface, unittest.TestCase): "--- tearDown() for %s.%s(%s) called ---" % (self.__class__.__name__, self._testMethodName, self._testMethodDoc) ) + if not hasattr(self, "vpp"): + return try: if not self.vpp_dead: @@ -904,6 +936,8 @@ class VppTestCase(CPUInterface, unittest.TestCase): def setUp(self): """Clear trace before running each test""" super(VppTestCase, self).setUp() + if not hasattr(self, "vpp"): + return self.reporter.send_keep_alive(self) if self.vpp_dead: raise VppDiedError( @@ -1008,6 +1042,9 @@ class VppTestCase(CPUInterface, unittest.TestCase): @classmethod def create_pg_ip4_interfaces(cls, interfaces, gso=0, gso_size=0): + if not hasattr(cls, "vpp"): + cls.pg_interfaces = [] + return cls.pg_interfaces pgmode = VppEnum.vl_api_pg_interface_mode_t return cls.create_pg_interfaces_internal( interfaces, gso, gso_size, pgmode.PG_API_MODE_IP4 @@ -1015,6 +1052,9 @@ class VppTestCase(CPUInterface, unittest.TestCase): @classmethod def create_pg_ip6_interfaces(cls, interfaces, gso=0, gso_size=0): + if not hasattr(cls, "vpp"): + cls.pg_interfaces = [] + return cls.pg_interfaces pgmode = VppEnum.vl_api_pg_interface_mode_t return cls.create_pg_interfaces_internal( interfaces, gso, gso_size, pgmode.PG_API_MODE_IP6 @@ -1022,6 +1062,9 @@ class VppTestCase(CPUInterface, unittest.TestCase): @classmethod def create_pg_interfaces(cls, interfaces, gso=0, gso_size=0): + if not hasattr(cls, "vpp"): + cls.pg_interfaces = [] + return cls.pg_interfaces pgmode = VppEnum.vl_api_pg_interface_mode_t return cls.create_pg_interfaces_internal( interfaces, gso, gso_size, pgmode.PG_API_MODE_ETHERNET @@ -1029,6 +1072,9 @@ class VppTestCase(CPUInterface, unittest.TestCase): @classmethod def create_pg_ethernet_interfaces(cls, interfaces, gso=0, gso_size=0): + if not hasattr(cls, "vpp"): + cls.pg_interfaces = [] + return cls.pg_interfaces pgmode = VppEnum.vl_api_pg_interface_mode_t return cls.create_pg_interfaces_internal( interfaces, gso, gso_size, pgmode.PG_API_MODE_ETHERNET @@ -1042,6 +1088,9 @@ class VppTestCase(CPUInterface, unittest.TestCase): :param count: number of interfaces created. :returns: List of created interfaces. """ + if not hasattr(cls, "vpp"): + cls.lo_interfaces = [] + return cls.lo_interfaces result = [VppLoInterface(cls) for i in range(count)] for intf in result: setattr(cls, intf.name, intf) @@ -1056,6 +1105,9 @@ class VppTestCase(CPUInterface, unittest.TestCase): :param count: number of interfaces created. :returns: List of created interfaces. """ + if not hasattr(cls, "vpp"): + cls.bvi_interfaces = [] + return cls.bvi_interfaces result = [VppBviInterface(cls) for i in range(count)] for intf in result: setattr(cls, intf.name, intf) @@ -1786,6 +1838,10 @@ class VppTestResult(unittest.TestResult): test_title = colorize(f"FIXME on Ubuntu-22.04: {test_title}", RED) test.skip_fixme_ubuntu2204() + if is_distro_debian11 == True and test.has_tag(TestCaseTag.FIXME_DEBIAN11): + test_title = colorize(f"FIXME on Debian-11: {test_title}", RED) + test.skip_fixme_debian11() + if hasattr(test, "vpp_worker_count"): if test.vpp_worker_count == 0: test_title += " [main thread only]" diff --git a/test/hook.py b/test/hook.py index 3429bdad1fe..58bbcaf9e7d 100644 --- a/test/hook.py +++ b/test/hook.py @@ -116,7 +116,7 @@ class PollHook(Hook): Poll the vpp status and throw an exception if it's not running :raises VppDiedError: exception if VPP is not running anymore """ - if self.test.vpp_dead: + if not hasattr(self.test, "vpp") or self.test.vpp_dead: # already dead, nothing to do return diff --git a/test/test_bfd.py b/test/test_bfd.py index 71133701420..67ddb4b97e1 100644 --- a/test/test_bfd.py +++ b/test/test_bfd.py @@ -30,7 +30,8 @@ from bfd import ( BFDState, BFD_vpp_echo, ) -from framework import tag_fixme_vpp_workers +from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, tag_fixme_debian11 +from framework import is_distro_ubuntu2204, is_distro_debian11 from framework import VppTestCase, VppTestRunner from framework import tag_run_solo from util import ppp @@ -818,6 +819,8 @@ def bfd_stats_diff(stats_before, stats_after): @tag_run_solo +@tag_fixme_ubuntu2204 +@tag_fixme_debian11 class BFD4TestCase(VppTestCase): """Bidirectional Forwarding Detection (BFD)""" @@ -828,6 +831,10 @@ class BFD4TestCase(VppTestCase): @classmethod def setUpClass(cls): + if (is_distro_ubuntu2204 == True or is_distro_debian11 == True) and not hasattr( + cls, "vpp" + ): + return super(BFD4TestCase, cls).setUpClass() cls.vapi.cli("set log class bfd level debug") try: @@ -1722,6 +1729,7 @@ class BFD4TestCase(VppTestCase): @tag_run_solo @tag_fixme_vpp_workers +@tag_fixme_ubuntu2204 class BFD6TestCase(VppTestCase): """Bidirectional Forwarding Detection (BFD) (IPv6)""" @@ -1732,6 +1740,8 @@ class BFD6TestCase(VppTestCase): @classmethod def setUpClass(cls): + if is_distro_ubuntu2204 == True and not hasattr(cls, "vpp"): + return super(BFD6TestCase, cls).setUpClass() cls.vapi.cli("set log class bfd level debug") try: @@ -1755,6 +1765,8 @@ class BFD6TestCase(VppTestCase): def setUp(self): super(BFD6TestCase, self).setUp() + if is_distro_ubuntu2204 == True and not hasattr(self, "vpp"): + return self.factory = AuthKeyFactory() self.vapi.want_bfd_events() self.pg0.enable_capture() diff --git a/test/test_flowprobe.py b/test/test_flowprobe.py index 141d7458c39..e3ff2245ad1 100644 --- a/test/test_flowprobe.py +++ b/test/test_flowprobe.py @@ -13,7 +13,8 @@ from scapy.layers.inet import IP, TCP, UDP from scapy.layers.inet6 import IPv6 from config import config -from framework import tag_fixme_vpp_workers +from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, tag_fixme_debian11 +from framework import is_distro_ubuntu2204, is_distro_debian11 from framework import VppTestCase, VppTestRunner from framework import tag_run_solo from vpp_object import VppObject @@ -151,6 +152,10 @@ class MethodHolder(VppTestCase): variables and configure VPP. """ super(MethodHolder, cls).setUpClass() + if (is_distro_ubuntu2204 == True or is_distro_debian11 == True) and not hasattr( + cls, "vpp" + ): + return try: # Create pg interfaces cls.create_pg_interfaces(range(9)) @@ -337,6 +342,8 @@ class MethodHolder(VppTestCase): @tag_run_solo @tag_fixme_vpp_workers +@tag_fixme_ubuntu2204 +@tag_fixme_debian11 class Flowprobe(MethodHolder): """Template verification, timer tests""" diff --git a/test/test_ikev2.py b/test/test_ikev2.py index ffddc79dbcf..f0fd2055379 100644 --- a/test/test_ikev2.py +++ b/test/test_ikev2.py @@ -19,7 +19,8 @@ from scapy.layers.inet import IP, UDP, Ether from scapy.layers.inet6 import IPv6 from scapy.packet import raw, Raw from scapy.utils import long_converter -from framework import tag_fixme_vpp_workers +from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, tag_fixme_debian11 +from framework import is_distro_ubuntu2204, is_distro_debian11 from framework import VppTestCase, VppTestRunner from vpp_ikev2 import Profile, IDType, AuthMethod from vpp_papi import VppEnum @@ -2177,6 +2178,8 @@ class TestResponderRekeyRepeatKEX(TestResponderRekeyRepeat): WITH_KEX = True +@tag_fixme_ubuntu2204 +@tag_fixme_debian11 class TestResponderVrf(TestResponderPsk, Ikev2Params): """test ikev2 responder - non-default table id""" @@ -2186,6 +2189,10 @@ class TestResponderVrf(TestResponderPsk, Ikev2Params): globals()["ikev2"] = _ikev2 super(IkePeer, cls).setUpClass() + if (is_distro_ubuntu2204 == True or is_distro_debian11 == True) and not hasattr( + cls, "vpp" + ): + return cls.create_pg_interfaces(range(1)) cls.vapi.cli("ip table add 1") cls.vapi.cli("set interface ip table pg0 1") diff --git a/test/test_memif.py b/test/test_memif.py index 9b15cd0005b..f0b60214442 100644 --- a/test/test_memif.py +++ b/test/test_memif.py @@ -5,7 +5,7 @@ from scapy.layers.l2 import Ether from scapy.layers.inet import IP, ICMP from framework import VppTestCase, VppTestRunner -from framework import tag_run_solo +from framework import tag_run_solo, tag_fixme_debian11, is_distro_debian11 from remote_test import RemoteClass, RemoteVppTestCase from vpp_memif import remove_all_memif_vpp_config, VppSocketFilename, VppMemif from vpp_ip_route import VppIpRoute, VppRoutePath @@ -13,6 +13,7 @@ from vpp_papi import VppEnum @tag_run_solo +@tag_fixme_debian11 class TestMemif(VppTestCase): """Memif Test Case""" @@ -36,6 +37,9 @@ class TestMemif(VppTestCase): cls.remote_test.start_remote() cls.remote_test.set_request_timeout(10) super(TestMemif, cls).setUpClass() + if is_distro_debian11 == True and not hasattr(cls, "vpp"): + cls.remote_test.quit_remote() + return cls.remote_test.setUpClass(cls.tempdir) cls.create_pg_interfaces(range(1)) for pg in cls.pg_interfaces: diff --git a/test/test_nat44_ed.py b/test/test_nat44_ed.py index d90afd27025..6b0eedcec20 100644 --- a/test/test_nat44_ed.py +++ b/test/test_nat44_ed.py @@ -5,6 +5,7 @@ from io import BytesIO from random import randint, choice import scapy.compat +from framework import tag_fixme_ubuntu2204, is_distro_ubuntu2204 from framework import VppTestCase, VppTestRunner from scapy.data import IP_PROTOS from scapy.layers.inet import IP, TCP, UDP, ICMP, GRE @@ -159,6 +160,8 @@ class TestNAT44ED(VppTestCase): @classmethod def setUpClass(cls): super().setUpClass() + if is_distro_ubuntu2204 == True and not hasattr(cls, "vpp"): + return cls.create_pg_interfaces(range(12)) cls.interfaces = list(cls.pg_interfaces[:4]) @@ -2550,6 +2553,7 @@ class TestNAT44ED(VppTestCase): out_if.unconfig() +@tag_fixme_ubuntu2204 class TestNAT44EDMW(TestNAT44ED): """NAT44ED MW Test Case""" diff --git a/test/test_nat44_ei.py b/test/test_nat44_ei.py index 9eb127aaf0b..259c445ea5b 100644 --- a/test/test_nat44_ei.py +++ b/test/test_nat44_ei.py @@ -8,6 +8,7 @@ import unittest from io import BytesIO import scapy.compat +from framework import tag_fixme_debian11, is_distro_debian11 from framework import VppTestCase, VppTestRunner from ipfix import IPFIX, Set, Template, Data, IPFIXDecoder from scapy.all import ( @@ -910,6 +911,7 @@ def get_nat44_ei_in2out_worker_index(ip, vpp_worker_count): return 1 + h % vpp_worker_count +@tag_fixme_debian11 class TestNAT44EI(MethodHolder): """NAT44EI Test Cases""" @@ -919,6 +921,8 @@ class TestNAT44EI(MethodHolder): @classmethod def setUpClass(cls): super(TestNAT44EI, cls).setUpClass() + if is_distro_debian11 == True and not hasattr(cls, "vpp"): + return cls.vapi.cli("set log class nat44-ei level debug") cls.tcp_port_in = 6303 diff --git a/test/test_nat64.py b/test/test_nat64.py index 214072addc9..05a2031f921 100644 --- a/test/test_nat64.py +++ b/test/test_nat64.py @@ -9,7 +9,7 @@ from io import BytesIO import scapy.compat from config import config -from framework import tag_fixme_vpp_workers +from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, is_distro_ubuntu2204 from framework import VppTestCase, VppTestRunner from ipfix import IPFIX, Set, Template, Data, IPFIXDecoder from scapy.data import IP_PROTOS @@ -34,6 +34,7 @@ from vpp_papi import VppEnum @tag_fixme_vpp_workers +@tag_fixme_ubuntu2204 class TestNAT64(VppTestCase): """NAT64 Test Cases""" @@ -49,6 +50,8 @@ class TestNAT64(VppTestCase): def setUpClass(cls): super(TestNAT64, cls).setUpClass() + if is_distro_ubuntu2204 == True and not hasattr(cls, "vpp"): + return cls.tcp_port_in = 6303 cls.tcp_port_out = 6303 cls.udp_port_in = 6304 diff --git a/test/test_neighbor.py b/test/test_neighbor.py index 58d98a36d92..f663e735d78 100644 --- a/test/test_neighbor.py +++ b/test/test_neighbor.py @@ -4,7 +4,7 @@ import unittest import os from socket import AF_INET, AF_INET6, inet_pton -from framework import tag_fixme_vpp_workers +from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, tag_fixme_debian11 from framework import VppTestCase, VppTestRunner from vpp_neighbor import VppNeighbor, find_nbr from vpp_ip_route import ( @@ -2196,6 +2196,7 @@ class NeighborStatsTestCase(VppTestCase): self.assertEqual(NUM_PKTS + 16, nd1.get_stats()["packets"]) +@tag_fixme_ubuntu2204 class NeighborAgeTestCase(VppTestCase): """ARP/ND Aging""" diff --git a/test/test_wireguard.py b/test/test_wireguard.py index 7055b7ab936..6da112af0d4 100644 --- a/test/test_wireguard.py +++ b/test/test_wireguard.py @@ -41,6 +41,8 @@ from vpp_interface import VppInterface from vpp_ip_route import VppIpRoute, VppRoutePath from vpp_object import VppObject from vpp_papi import VppEnum +from framework import tag_fixme_ubuntu2204, tag_fixme_debian11 +from framework import is_distro_ubuntu2204, is_distro_debian11 from framework import VppTestCase from re import compile import unittest @@ -493,6 +495,8 @@ class VppWgPeer(VppObject): self._test.assertEqual(rv.peer_index, self.index) +@tag_fixme_ubuntu2204 +@tag_fixme_debian11 class TestWg(VppTestCase): """Wireguard Test Case""" @@ -518,6 +522,10 @@ class TestWg(VppTestCase): @classmethod def setUpClass(cls): super(TestWg, cls).setUpClass() + if (is_distro_ubuntu2204 == True or is_distro_debian11 == True) and not hasattr( + cls, "vpp" + ): + return try: cls.create_pg_interfaces(range(3)) for i in cls.pg_interfaces: @@ -2227,6 +2235,8 @@ class TestWg(VppTestCase): wg1.remove_vpp_config() +@tag_fixme_ubuntu2204 +@tag_fixme_debian11 class WireguardHandoffTests(TestWg): """Wireguard Tests in multi worker setup""" |