make test: split into basic and extended tests

Implement plumbing to allow decorating tests as extended, e.g.:

@unittest.skipUnless(running_extended_tests(), "part of extended tests")

both methods and classes can be decorated this way.

Change make test and make test-debug to run only non-extended tests.
Introduce make test-all and make test-all-debug to run the full suite.
Run full suite as part of make verify.
Decorate most BFD tests as extended.

Change-Id: I3bc64f59e9fe238f7f767d7e043dc165d03e9dfa
Signed-off-by: Klement Sekera <ksekera@cisco.com>

1 files changed, 33 insertions, 2 deletions

diff --git a/test/test_bfd.py b/test/test_bfd.py
index e7ebb214..5460a2b5 100644
--- a/test/test_bfd.py
+++ b/test/test_bfd.py
@@ -15,7 +15,7 @@ from scapy.layers.inet import UDP, IP
 from scapy.layers.inet6 import IPv6
 from bfd import VppBFDAuthKey, BFD, BFDAuthType, VppBFDUDPSession, \
     BFDDiagCode, BFDState, BFD_vpp_echo
-from framework import VppTestCase, VppTestRunner
+from framework import VppTestCase, VppTestRunner, running_extended_tests
 from vpp_pg_interface import CaptureTimeoutError
 from util import ppp
 from vpp_papi_provider import UnexpectedApiReturnValueError
@@ -40,6 +40,7 @@ class AuthKeyFactory(object):
                              conf_key_id=conf_key_id, key=key)
 
 
+@unittest.skipUnless(running_extended_tests(), "part of extended tests")
 class BFDAPITestCase(VppTestCase):
     """Bidirectional Forwarding Detection (BFD) - API"""
 
@@ -692,6 +693,7 @@ class BFD4TestCase(VppTestCase):
         bfd_session_up(self)
         bfd_session_down(self)
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_hold_up(self):
         """ hold BFD session up """
         bfd_session_up(self)
@@ -701,6 +703,7 @@ class BFD4TestCase(VppTestCase):
         self.assert_equal(len(self.vapi.collect_events()), 0,
                           "number of bfd events")
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_slow_timer(self):
         """ verify slow periodic control frames while session down """
         packet_count = 3
@@ -715,6 +718,7 @@ class BFD4TestCase(VppTestCase):
                 time_diff, 0.70, 1.05, "time between slow packets")
             prev_packet = next_packet
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_zero_remote_min_rx(self):
         """ no packets when zero remote required min rx interval """
         bfd_session_up(self)
@@ -739,6 +743,7 @@ class BFD4TestCase(VppTestCase):
         self.assert_equal(
             len(self.vapi.collect_events()), 0, "number of bfd events")
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_conn_down(self):
         """ verify session goes down after inactivity """
         bfd_session_up(self)
@@ -748,6 +753,7 @@ class BFD4TestCase(VppTestCase):
         e = self.vapi.wait_for_event(1, "bfd_udp_session_details")
         verify_event(self, e, expected_state=BFDState.down)
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_large_required_min_rx(self):
         """ large remote required min rx interval """
         bfd_session_up(self)
@@ -777,6 +783,7 @@ class BFD4TestCase(VppTestCase):
                 break
         self.assert_equal(count, 0, "number of packets received")
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_immediate_remote_min_rx_reduction(self):
         """ immediately honor remote required min rx reduction """
         self.vpp_session.remove_vpp_config()
@@ -809,6 +816,7 @@ class BFD4TestCase(VppTestCase):
                                  "time between BFD packets")
             reference_packet = p
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_modify_req_min_rx_double(self):
         """ modify session - double required min rx """
         bfd_session_up(self)
@@ -838,6 +846,7 @@ class BFD4TestCase(VppTestCase):
         self.assert_in_range(time_to_event, .9 * timeout,
                              1.1 * timeout, "session timeout")
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_modify_req_min_rx_halve(self):
         """ modify session - halve required min rx """
         self.vpp_session.modify_parameters(
@@ -878,6 +887,7 @@ class BFD4TestCase(VppTestCase):
                              "time before bfd session goes down")
         verify_event(self, e, expected_state=BFDState.down)
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_modify_detect_mult(self):
         """ modify detect multiplier """
         bfd_session_up(self)
@@ -901,6 +911,7 @@ class BFD4TestCase(VppTestCase):
         self.assertNotIn("P", p.sprintf("%BFD.flags%"),
                          "Poll bit not set in BFD packet")
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_queued_poll(self):
         """ test poll sequence queueing """
         bfd_session_up(self)
@@ -966,6 +977,7 @@ class BFD4TestCase(VppTestCase):
         self.assertNotIn("P", p.sprintf("%BFD.flags%"),
                          "Poll bit set in BFD packet")
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_poll_response(self):
         """ test correct response to control frame with poll bit set """
         bfd_session_up(self)
@@ -976,6 +988,7 @@ class BFD4TestCase(VppTestCase):
             self, pcap_time_min=time.time() - self.vpp_clock_offset)
         self.assertIn("F", final.sprintf("%BFD.flags%"))
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_no_periodic_if_remote_demand(self):
         """ no periodic frames outside poll sequence if remote demand set """
         bfd_session_up(self)
@@ -1093,6 +1106,7 @@ class BFD4TestCase(VppTestCase):
                                   "number of bfd events")
             self.test_session.send_packet()
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_echo_fail(self):
         """ session goes down if echo function fails """
         bfd_session_up(self)
@@ -1132,6 +1146,7 @@ class BFD4TestCase(VppTestCase):
         self.assert_equal(events[0].state, BFDState.down, BFDState)
         self.assertTrue(verified_diag, "Incorrect diagnostics code received")
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_echo_stop(self):
         """ echo function stops if peer sets required min echo rx zero """
         bfd_session_up(self)
@@ -1162,6 +1177,7 @@ class BFD4TestCase(VppTestCase):
             events = self.vapi.collect_events()
             self.assert_equal(len(events), 0, "number of bfd events")
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_echo_source_removed(self):
         """ echo function stops if echo source is removed """
         bfd_session_up(self)
@@ -1192,6 +1208,7 @@ class BFD4TestCase(VppTestCase):
             events = self.vapi.collect_events()
             self.assert_equal(len(events), 0, "number of bfd events")
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_stale_echo(self):
         """ stale echo packets don't keep a session up """
         bfd_session_up(self)
@@ -1243,6 +1260,7 @@ class BFD4TestCase(VppTestCase):
             self.test_session.send_packet()
         self.assertTrue(timeout_ok, "Expected timeout event didn't occur")
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_invalid_echo_checksum(self):
         """ echo packets with invalid checksum don't keep a session up """
         bfd_session_up(self)
@@ -1291,6 +1309,7 @@ class BFD4TestCase(VppTestCase):
             self.test_session.send_packet()
         self.assertTrue(timeout_ok, "Expected timeout event didn't occur")
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_admin_up_down(self):
         """ put session admin-up and admin-down """
         bfd_session_up(self)
@@ -1328,6 +1347,7 @@ class BFD4TestCase(VppTestCase):
         e = self.vapi.wait_for_event(1, "bfd_udp_session_details")
         verify_event(self, e, expected_state=BFDState.up)
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_config_change_remote_demand(self):
         """ configuration change while peer in demand mode """
         bfd_session_up(self)
@@ -1445,6 +1465,7 @@ class BFD6TestCase(VppTestCase):
         self.test_session.send_packet()
         self.assert_equal(self.vpp_session.state, BFDState.up, BFDState)
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_hold_up(self):
         """ hold BFD session up """
         bfd_session_up(self)
@@ -1502,6 +1523,7 @@ class BFD6TestCase(VppTestCase):
         self.assert_equal(udp_sport_tx, udp_sport_rx, "UDP source port (== "
                           "ECHO packet identifier for test purposes)")
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_echo(self):
         """ echo function used """
         bfd_session_up(self)
@@ -1596,6 +1618,7 @@ class BFDSHA1TestCase(VppTestCase):
             bfd_key_id=self.vpp_session.bfd_key_id)
         bfd_session_up(self)
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_hold_up(self):
         """ hold BFD session up """
         key = self.factory.create_random_key(self)
@@ -1614,6 +1637,7 @@ class BFDSHA1TestCase(VppTestCase):
             self.test_session.send_packet()
         self.assert_equal(self.vpp_session.state, BFDState.up, BFDState)
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_hold_up_meticulous(self):
         """ hold BFD session up - meticulous auth """
         key = self.factory.create_random_key(
@@ -1635,6 +1659,7 @@ class BFDSHA1TestCase(VppTestCase):
             self.test_session.send_packet()
         self.assert_equal(self.vpp_session.state, BFDState.up, BFDState)
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_send_bad_seq_number(self):
         """ session is not kept alive by msgs with bad sequence numbers"""
         key = self.factory.create_random_key(
@@ -1697,6 +1722,7 @@ class BFDSHA1TestCase(VppTestCase):
         wait_for_bfd_packet(self)
         self.assert_equal(self.vpp_session.state, BFDState.up, BFDState)
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_mismatch_auth(self):
         """ session is not brought down by unauthenticated msg """
         key = self.factory.create_random_key(self)
@@ -1711,6 +1737,7 @@ class BFDSHA1TestCase(VppTestCase):
                                             legitimate_test_session,
                                             rogue_test_session)
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_mismatch_bfd_key_id(self):
         """ session is not brought down by msg with non-existent key-id """
         key = self.factory.create_random_key(self)
@@ -1730,6 +1757,7 @@ class BFDSHA1TestCase(VppTestCase):
                                             legitimate_test_session,
                                             rogue_test_session)
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_mismatched_auth_type(self):
         """ session is not brought down by msg with wrong auth type """
         key = self.factory.create_random_key(self)
@@ -1746,6 +1774,7 @@ class BFDSHA1TestCase(VppTestCase):
             vpp_session, legitimate_test_session, rogue_test_session,
             {'auth_type': BFDAuthType.keyed_md5})
 
+    @unittest.skipUnless(running_extended_tests(), "part of extended tests")
     def test_restart(self):
         """ simulate remote peer restart and resynchronization """
         key = self.factory.create_random_key(
@@ -1761,7 +1790,7 @@ class BFDSHA1TestCase(VppTestCase):
         # don't send any packets for 2*detection_time
         detection_time = self.test_session.detect_mult *\
             self.vpp_session.required_min_rx / USEC_IN_SEC
-        self.sleep(2*detection_time, "simulating peer restart")
+        self.sleep(2 * detection_time, "simulating peer restart")
         events = self.vapi.collect_events()
         self.assert_equal(len(events), 1, "number of bfd events")
         verify_event(self, events[0], expected_state=BFDState.down)
@@ -1774,6 +1803,7 @@ class BFDSHA1TestCase(VppTestCase):
         bfd_session_up(self)
 
 
+@unittest.skipUnless(running_extended_tests(), "part of extended tests")
 class BFDAuthOnOffTestCase(VppTestCase):
     """Bidirectional Forwarding Detection (BFD) (changing auth) """
 
@@ -1981,6 +2011,7 @@ class BFDAuthOnOffTestCase(VppTestCase):
                           "number of bfd events")
 
 
+@unittest.skipUnless(running_extended_tests(), "part of extended tests")
 class BFDCLITestCase(VppTestCase):
     """Bidirectional Forwarding Detection (BFD) (CLI) """
     pg0 = None