tests: disable failing tests on ubuntu-22.04 debian-11

Type: test

Change-Id: I7b2314a731c83b3dcd69c999edb8ebed53839724
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

11 files changed, 116 insertions, 8 deletions

diff --git a/test/framework.py b/test/framework.py
index c85dec5dbdf..d130855bb55 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -147,6 +147,8 @@ class _PacketInfo(object):
 
 def pump_output(testclass):
     """pump output from vpp stdout/stderr to proper queues"""
+    if not hasattr(testclass, "vpp"):
+        return
     stdout_fragment = ""
     stderr_fragment = ""
     while not testclass.pump_thread_stop_flag.is_set():
@@ -212,6 +214,17 @@ def _is_distro_ubuntu2204():
 is_distro_ubuntu2204 = _is_distro_ubuntu2204()
 
 
+def _is_distro_debian11():
+    with open("/etc/os-release") as f:
+        for line in f.readlines():
+            if "bullseye" in line:
+                return True
+    return False
+
+
+is_distro_debian11 = _is_distro_debian11()
+
+
 class KeepAliveReporter(object):
     """
     Singleton object which reports test start to parent process
@@ -237,7 +250,7 @@ class KeepAliveReporter(object):
         """
         Write current test tmpdir & desc to keep-alive pipe to signal liveness
         """
-        if self.pipe is None:
+        if not hasattr(test, "vpp") or self.pipe is None:
             # if not running forked..
             return
 
@@ -259,6 +272,8 @@ class TestCaseTag(Enum):
     FIXME_ASAN = 3
     # marks suites broken on Ubuntu-22.04
     FIXME_UBUNTU2204 = 4
+    # marks suites broken on Debian-11
+    FIXME_DEBIAN11 = 5
 
 
 def create_tag_decorator(e):
@@ -276,6 +291,7 @@ tag_run_solo = create_tag_decorator(TestCaseTag.RUN_SOLO)
 tag_fixme_vpp_workers = create_tag_decorator(TestCaseTag.FIXME_VPP_WORKERS)
 tag_fixme_asan = create_tag_decorator(TestCaseTag.FIXME_ASAN)
 tag_fixme_ubuntu2204 = create_tag_decorator(TestCaseTag.FIXME_UBUNTU2204)
+tag_fixme_debian11 = create_tag_decorator(TestCaseTag.FIXME_DEBIAN11)
 
 
 class DummyVpp:
@@ -358,6 +374,12 @@ class VppTestCase(CPUInterface, unittest.TestCase):
             cls = unittest.skip("Skipping @tag_fixme_ubuntu2204 tests")(cls)
 
     @classmethod
+    def skip_fixme_debian11(cls):
+        """if distro is Debian-11 and @tag_fixme_debian11 mark for skip"""
+        if cls.has_tag(TestCaseTag.FIXME_DEBIAN11):
+            cls = unittest.skip("Skipping @tag_fixme_debian11 tests")(cls)
+
+    @classmethod
     def instance(cls):
         """Return the instance of this testcase"""
         return cls.test_instance
@@ -556,6 +578,10 @@ class VppTestCase(CPUInterface, unittest.TestCase):
 
     @classmethod
     def run_vpp(cls):
+        if (
+            is_distro_ubuntu2204 == True and cls.has_tag(TestCaseTag.FIXME_UBUNTU2204)
+        ) or (is_distro_debian11 == True and cls.has_tag(TestCaseTag.FIXME_DEBIAN11)):
+            return
         cls.logger.debug(f"Assigned cpus: {cls.cpus}")
         cmdline = cls.vpp_cmdline
 
@@ -694,6 +720,8 @@ class VppTestCase(CPUInterface, unittest.TestCase):
                 cls.attach_vpp()
             else:
                 cls.run_vpp()
+                if not hasattr(cls, "vpp"):
+                    return
             cls.reporter.send_keep_alive(cls, "setUpClass")
             VppTestResult.current_test_case_info = TestCaseInfo(
                 cls.logger, cls.tempdir, cls.vpp.pid, config.vpp
@@ -854,6 +882,8 @@ class VppTestCase(CPUInterface, unittest.TestCase):
     def tearDownClass(cls):
         """Perform final cleanup after running all tests in this test-case"""
         cls.logger.debug("--- tearDownClass() for %s called ---" % cls.__name__)
+        if not hasattr(cls, "vpp"):
+            return
         cls.reporter.send_keep_alive(cls, "tearDownClass")
         cls.quit()
         cls.file_handler.close()
@@ -871,6 +901,8 @@ class VppTestCase(CPUInterface, unittest.TestCase):
             "--- tearDown() for %s.%s(%s) called ---"
             % (self.__class__.__name__, self._testMethodName, self._testMethodDoc)
         )
+        if not hasattr(self, "vpp"):
+            return
 
         try:
             if not self.vpp_dead:
@@ -904,6 +936,8 @@ class VppTestCase(CPUInterface, unittest.TestCase):
     def setUp(self):
         """Clear trace before running each test"""
         super(VppTestCase, self).setUp()
+        if not hasattr(self, "vpp"):
+            return
         self.reporter.send_keep_alive(self)
         if self.vpp_dead:
             raise VppDiedError(
@@ -1008,6 +1042,9 @@ class VppTestCase(CPUInterface, unittest.TestCase):
 
     @classmethod
     def create_pg_ip4_interfaces(cls, interfaces, gso=0, gso_size=0):
+        if not hasattr(cls, "vpp"):
+            cls.pg_interfaces = []
+            return cls.pg_interfaces
         pgmode = VppEnum.vl_api_pg_interface_mode_t
         return cls.create_pg_interfaces_internal(
             interfaces, gso, gso_size, pgmode.PG_API_MODE_IP4
@@ -1015,6 +1052,9 @@ class VppTestCase(CPUInterface, unittest.TestCase):
 
     @classmethod
     def create_pg_ip6_interfaces(cls, interfaces, gso=0, gso_size=0):
+        if not hasattr(cls, "vpp"):
+            cls.pg_interfaces = []
+            return cls.pg_interfaces
         pgmode = VppEnum.vl_api_pg_interface_mode_t
         return cls.create_pg_interfaces_internal(
             interfaces, gso, gso_size, pgmode.PG_API_MODE_IP6
@@ -1022,6 +1062,9 @@ class VppTestCase(CPUInterface, unittest.TestCase):
 
     @classmethod
     def create_pg_interfaces(cls, interfaces, gso=0, gso_size=0):
+        if not hasattr(cls, "vpp"):
+            cls.pg_interfaces = []
+            return cls.pg_interfaces
         pgmode = VppEnum.vl_api_pg_interface_mode_t
         return cls.create_pg_interfaces_internal(
             interfaces, gso, gso_size, pgmode.PG_API_MODE_ETHERNET
@@ -1029,6 +1072,9 @@ class VppTestCase(CPUInterface, unittest.TestCase):
 
     @classmethod
     def create_pg_ethernet_interfaces(cls, interfaces, gso=0, gso_size=0):
+        if not hasattr(cls, "vpp"):
+            cls.pg_interfaces = []
+            return cls.pg_interfaces
         pgmode = VppEnum.vl_api_pg_interface_mode_t
         return cls.create_pg_interfaces_internal(
             interfaces, gso, gso_size, pgmode.PG_API_MODE_ETHERNET
@@ -1042,6 +1088,9 @@ class VppTestCase(CPUInterface, unittest.TestCase):
         :param count: number of interfaces created.
         :returns: List of created interfaces.
         """
+        if not hasattr(cls, "vpp"):
+            cls.lo_interfaces = []
+            return cls.lo_interfaces
         result = [VppLoInterface(cls) for i in range(count)]
         for intf in result:
             setattr(cls, intf.name, intf)
@@ -1056,6 +1105,9 @@ class VppTestCase(CPUInterface, unittest.TestCase):
         :param count: number of interfaces created.
         :returns: List of created interfaces.
         """
+        if not hasattr(cls, "vpp"):
+            cls.bvi_interfaces = []
+            return cls.bvi_interfaces
         result = [VppBviInterface(cls) for i in range(count)]
         for intf in result:
             setattr(cls, intf.name, intf)
@@ -1786,6 +1838,10 @@ class VppTestResult(unittest.TestResult):
                 test_title = colorize(f"FIXME on Ubuntu-22.04: {test_title}", RED)
                 test.skip_fixme_ubuntu2204()
 
+            if is_distro_debian11 == True and test.has_tag(TestCaseTag.FIXME_DEBIAN11):
+                test_title = colorize(f"FIXME on Debian-11: {test_title}", RED)
+                test.skip_fixme_debian11()
+
             if hasattr(test, "vpp_worker_count"):
                 if test.vpp_worker_count == 0:
                     test_title += " [main thread only]"
diff --git a/test/hook.py b/test/hook.py
index 3429bdad1fe..58bbcaf9e7d 100644
--- a/test/hook.py
+++ b/test/hook.py
@@ -116,7 +116,7 @@ class PollHook(Hook):
         Poll the vpp status and throw an exception if it's not running
         :raises VppDiedError: exception if VPP is not running anymore
         """
-        if self.test.vpp_dead:
+        if not hasattr(self.test, "vpp") or self.test.vpp_dead:
             # already dead, nothing to do
             return
 
diff --git a/test/test_bfd.py b/test/test_bfd.py
index 71133701420..67ddb4b97e1 100644
--- a/test/test_bfd.py
+++ b/test/test_bfd.py
@@ -30,7 +30,8 @@ from bfd import (
     BFDState,
     BFD_vpp_echo,
 )
-from framework import tag_fixme_vpp_workers
+from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, tag_fixme_debian11
+from framework import is_distro_ubuntu2204, is_distro_debian11
 from framework import VppTestCase, VppTestRunner
 from framework import tag_run_solo
 from util import ppp
@@ -818,6 +819,8 @@ def bfd_stats_diff(stats_before, stats_after):
 
 
 @tag_run_solo
+@tag_fixme_ubuntu2204
+@tag_fixme_debian11
 class BFD4TestCase(VppTestCase):
     """Bidirectional Forwarding Detection (BFD)"""
 
@@ -828,6 +831,10 @@ class BFD4TestCase(VppTestCase):
 
     @classmethod
     def setUpClass(cls):
+        if (is_distro_ubuntu2204 == True or is_distro_debian11 == True) and not hasattr(
+            cls, "vpp"
+        ):
+            return
         super(BFD4TestCase, cls).setUpClass()
         cls.vapi.cli("set log class bfd level debug")
         try:
@@ -1722,6 +1729,7 @@ class BFD4TestCase(VppTestCase):
 
 @tag_run_solo
 @tag_fixme_vpp_workers
+@tag_fixme_ubuntu2204
 class BFD6TestCase(VppTestCase):
     """Bidirectional Forwarding Detection (BFD) (IPv6)"""
 
@@ -1732,6 +1740,8 @@ class BFD6TestCase(VppTestCase):
 
     @classmethod
     def setUpClass(cls):
+        if is_distro_ubuntu2204 == True and not hasattr(cls, "vpp"):
+            return
         super(BFD6TestCase, cls).setUpClass()
         cls.vapi.cli("set log class bfd level debug")
         try:
@@ -1755,6 +1765,8 @@ class BFD6TestCase(VppTestCase):
 
     def setUp(self):
         super(BFD6TestCase, self).setUp()
+        if is_distro_ubuntu2204 == True and not hasattr(self, "vpp"):
+            return
         self.factory = AuthKeyFactory()
         self.vapi.want_bfd_events()
         self.pg0.enable_capture()
diff --git a/test/test_flowprobe.py b/test/test_flowprobe.py
index 141d7458c39..e3ff2245ad1 100644
--- a/test/test_flowprobe.py
+++ b/test/test_flowprobe.py
@@ -13,7 +13,8 @@ from scapy.layers.inet import IP, TCP, UDP
 from scapy.layers.inet6 import IPv6
 
 from config import config
-from framework import tag_fixme_vpp_workers
+from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, tag_fixme_debian11
+from framework import is_distro_ubuntu2204, is_distro_debian11
 from framework import VppTestCase, VppTestRunner
 from framework import tag_run_solo
 from vpp_object import VppObject
@@ -151,6 +152,10 @@ class MethodHolder(VppTestCase):
         variables and configure VPP.
         """
         super(MethodHolder, cls).setUpClass()
+        if (is_distro_ubuntu2204 == True or is_distro_debian11 == True) and not hasattr(
+            cls, "vpp"
+        ):
+            return
         try:
             # Create pg interfaces
             cls.create_pg_interfaces(range(9))
@@ -337,6 +342,8 @@ class MethodHolder(VppTestCase):
 
 @tag_run_solo
 @tag_fixme_vpp_workers
+@tag_fixme_ubuntu2204
+@tag_fixme_debian11
 class Flowprobe(MethodHolder):
     """Template verification, timer tests"""
 
diff --git a/test/test_ikev2.py b/test/test_ikev2.py
index ffddc79dbcf..f0fd2055379 100644
--- a/test/test_ikev2.py
+++ b/test/test_ikev2.py
@@ -19,7 +19,8 @@ from scapy.layers.inet import IP, UDP, Ether
 from scapy.layers.inet6 import IPv6
 from scapy.packet import raw, Raw
 from scapy.utils import long_converter
-from framework import tag_fixme_vpp_workers
+from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, tag_fixme_debian11
+from framework import is_distro_ubuntu2204, is_distro_debian11
 from framework import VppTestCase, VppTestRunner
 from vpp_ikev2 import Profile, IDType, AuthMethod
 from vpp_papi import VppEnum
@@ -2177,6 +2178,8 @@ class TestResponderRekeyRepeatKEX(TestResponderRekeyRepeat):
     WITH_KEX = True
 
 
+@tag_fixme_ubuntu2204
+@tag_fixme_debian11
 class TestResponderVrf(TestResponderPsk, Ikev2Params):
     """test ikev2 responder - non-default table id"""
 
@@ -2186,6 +2189,10 @@ class TestResponderVrf(TestResponderPsk, Ikev2Params):
 
         globals()["ikev2"] = _ikev2
         super(IkePeer, cls).setUpClass()
+        if (is_distro_ubuntu2204 == True or is_distro_debian11 == True) and not hasattr(
+            cls, "vpp"
+        ):
+            return
         cls.create_pg_interfaces(range(1))
         cls.vapi.cli("ip table add 1")
         cls.vapi.cli("set interface ip table pg0 1")
diff --git a/test/test_memif.py b/test/test_memif.py
index 9b15cd0005b..f0b60214442 100644
--- a/test/test_memif.py
+++ b/test/test_memif.py
@@ -5,7 +5,7 @@ from scapy.layers.l2 import Ether
 from scapy.layers.inet import IP, ICMP
 
 from framework import VppTestCase, VppTestRunner
-from framework import tag_run_solo
+from framework import tag_run_solo, tag_fixme_debian11, is_distro_debian11
 from remote_test import RemoteClass, RemoteVppTestCase
 from vpp_memif import remove_all_memif_vpp_config, VppSocketFilename, VppMemif
 from vpp_ip_route import VppIpRoute, VppRoutePath
@@ -13,6 +13,7 @@ from vpp_papi import VppEnum
 
 
 @tag_run_solo
+@tag_fixme_debian11
 class TestMemif(VppTestCase):
     """Memif Test Case"""
 
@@ -36,6 +37,9 @@ class TestMemif(VppTestCase):
         cls.remote_test.start_remote()
         cls.remote_test.set_request_timeout(10)
         super(TestMemif, cls).setUpClass()
+        if is_distro_debian11 == True and not hasattr(cls, "vpp"):
+            cls.remote_test.quit_remote()
+            return
         cls.remote_test.setUpClass(cls.tempdir)
         cls.create_pg_interfaces(range(1))
         for pg in cls.pg_interfaces:
diff --git a/test/test_nat44_ed.py b/test/test_nat44_ed.py
index d90afd27025..6b0eedcec20 100644
--- a/test/test_nat44_ed.py
+++ b/test/test_nat44_ed.py
@@ -5,6 +5,7 @@ from io import BytesIO
 from random import randint, choice
 
 import scapy.compat
+from framework import tag_fixme_ubuntu2204, is_distro_ubuntu2204
 from framework import VppTestCase, VppTestRunner
 from scapy.data import IP_PROTOS
 from scapy.layers.inet import IP, TCP, UDP, ICMP, GRE
@@ -159,6 +160,8 @@ class TestNAT44ED(VppTestCase):
     @classmethod
     def setUpClass(cls):
         super().setUpClass()
+        if is_distro_ubuntu2204 == True and not hasattr(cls, "vpp"):
+            return
 
         cls.create_pg_interfaces(range(12))
         cls.interfaces = list(cls.pg_interfaces[:4])
@@ -2550,6 +2553,7 @@ class TestNAT44ED(VppTestCase):
             out_if.unconfig()
 
 
+@tag_fixme_ubuntu2204
 class TestNAT44EDMW(TestNAT44ED):
     """NAT44ED MW Test Case"""
 
diff --git a/test/test_nat44_ei.py b/test/test_nat44_ei.py
index 9eb127aaf0b..259c445ea5b 100644
--- a/test/test_nat44_ei.py
+++ b/test/test_nat44_ei.py
@@ -8,6 +8,7 @@ import unittest
 from io import BytesIO
 
 import scapy.compat
+from framework import tag_fixme_debian11, is_distro_debian11
 from framework import VppTestCase, VppTestRunner
 from ipfix import IPFIX, Set, Template, Data, IPFIXDecoder
 from scapy.all import (
@@ -910,6 +911,7 @@ def get_nat44_ei_in2out_worker_index(ip, vpp_worker_count):
     return 1 + h % vpp_worker_count
 
 
+@tag_fixme_debian11
 class TestNAT44EI(MethodHolder):
     """NAT44EI Test Cases"""
 
@@ -919,6 +921,8 @@ class TestNAT44EI(MethodHolder):
     @classmethod
     def setUpClass(cls):
         super(TestNAT44EI, cls).setUpClass()
+        if is_distro_debian11 == True and not hasattr(cls, "vpp"):
+            return
         cls.vapi.cli("set log class nat44-ei level debug")
 
         cls.tcp_port_in = 6303
diff --git a/test/test_nat64.py b/test/test_nat64.py
index 214072addc9..05a2031f921 100644
--- a/test/test_nat64.py
+++ b/test/test_nat64.py
@@ -9,7 +9,7 @@ from io import BytesIO
 
 import scapy.compat
 from config import config
-from framework import tag_fixme_vpp_workers
+from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, is_distro_ubuntu2204
 from framework import VppTestCase, VppTestRunner
 from ipfix import IPFIX, Set, Template, Data, IPFIXDecoder
 from scapy.data import IP_PROTOS
@@ -34,6 +34,7 @@ from vpp_papi import VppEnum
 
 
 @tag_fixme_vpp_workers
+@tag_fixme_ubuntu2204
 class TestNAT64(VppTestCase):
     """NAT64 Test Cases"""
 
@@ -49,6 +50,8 @@ class TestNAT64(VppTestCase):
     def setUpClass(cls):
         super(TestNAT64, cls).setUpClass()
 
+        if is_distro_ubuntu2204 == True and not hasattr(cls, "vpp"):
+            return
         cls.tcp_port_in = 6303
         cls.tcp_port_out = 6303
         cls.udp_port_in = 6304
diff --git a/test/test_neighbor.py b/test/test_neighbor.py
index 58d98a36d92..f663e735d78 100644
--- a/test/test_neighbor.py
+++ b/test/test_neighbor.py
@@ -4,7 +4,7 @@ import unittest
 import os
 from socket import AF_INET, AF_INET6, inet_pton
 
-from framework import tag_fixme_vpp_workers
+from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, tag_fixme_debian11
 from framework import VppTestCase, VppTestRunner
 from vpp_neighbor import VppNeighbor, find_nbr
 from vpp_ip_route import (
@@ -2196,6 +2196,7 @@ class NeighborStatsTestCase(VppTestCase):
         self.assertEqual(NUM_PKTS + 16, nd1.get_stats()["packets"])
 
 
+@tag_fixme_ubuntu2204
 class NeighborAgeTestCase(VppTestCase):
     """ARP/ND Aging"""
 
diff --git a/test/test_wireguard.py b/test/test_wireguard.py
index 7055b7ab936..6da112af0d4 100644
--- a/test/test_wireguard.py
+++ b/test/test_wireguard.py
@@ -41,6 +41,8 @@ from vpp_interface import VppInterface
 from vpp_ip_route import VppIpRoute, VppRoutePath
 from vpp_object import VppObject
 from vpp_papi import VppEnum
+from framework import tag_fixme_ubuntu2204, tag_fixme_debian11
+from framework import is_distro_ubuntu2204, is_distro_debian11
 from framework import VppTestCase
 from re import compile
 import unittest
@@ -493,6 +495,8 @@ class VppWgPeer(VppObject):
         self._test.assertEqual(rv.peer_index, self.index)
 
 
+@tag_fixme_ubuntu2204
+@tag_fixme_debian11
 class TestWg(VppTestCase):
     """Wireguard Test Case"""
 
@@ -518,6 +522,10 @@ class TestWg(VppTestCase):
     @classmethod
     def setUpClass(cls):
         super(TestWg, cls).setUpClass()
+        if (is_distro_ubuntu2204 == True or is_distro_debian11 == True) and not hasattr(
+            cls, "vpp"
+        ):
+            return
         try:
             cls.create_pg_interfaces(range(3))
             for i in cls.pg_interfaces:
@@ -2227,6 +2235,8 @@ class TestWg(VppTestCase):
         wg1.remove_vpp_config()
 
 
+@tag_fixme_ubuntu2204
+@tag_fixme_debian11
 class WireguardHandoffTests(TestWg):
     """Wireguard Tests in multi worker setup"""