diff options
| -rw-r--r-- | Makefile | 17 | ||||
| -rw-r--r-- | test/Makefile | 6 | ||||
| -rw-r--r-- | test/framework.py | 30 | ||||
| -rw-r--r-- | test/test_bfd.py | 35 |
4 files changed, 78 insertions, 10 deletions
@@ -82,8 +82,10 @@ help: @echo " run-release - run release binary" @echo " debug - run debug binary with debugger" @echo " debug-release - run release binary with debugger" - @echo " test - build and run functional tests" - @echo " test-debug - build and run functional tests (debug build)" + @echo " test - build and run (basic) functional tests" + @echo " test-debug - build and run (basic) functional tests (debug build)" + @echo " test-all - build and run (all) functional tests" + @echo " test-all-debug - build and run (all) functional tests (debug build)" @echo " test-wipe - wipe files generated by unit tests" @echo " retest - run functional tests" @echo " retest-debug - run functional tests (debug build)" @@ -221,6 +223,7 @@ define test VPP_TEST_PLUGIN_PATH=$(BR)/install-$(2)-native/vpp/lib64/vpp_plugins \ VPP_TEST_INSTALL_PATH=$(BR)/install-$(2)-native/ \ LD_LIBRARY_PATH=$(BR)/install-$(2)-native/vpp/lib64/ \ + EXTENDED_TESTS=$(EXTENDED_TESTS) \ $(3) endef @@ -230,6 +233,14 @@ test: bootstrap test-debug: bootstrap $(call test,vpp_lite,vpp_lite_debug,test) +test-all: bootstrap + $(eval EXTENDED_TESTS=yes) + $(call test,vpp_lite,vpp_lite,test) + +test-all-debug: bootstrap + $(eval EXTENDED_TESTS=yes) + $(call test,vpp_lite,vpp_lite_debug,test) + test-help: @make -C test help @@ -368,6 +379,6 @@ ifeq ($(OS_ID)-$(OS_VERSION_ID),ubuntu-16.04) endif $(call banner,"Building $(PKG) packages") @make pkg-$(PKG) - @make test + @make test-all diff --git a/test/Makefile b/test/Makefile index 52980b9e8b9..1bb3c6c28ae 100644 --- a/test/Makefile +++ b/test/Makefile @@ -119,8 +119,10 @@ checkstyle: verify-python-path help: @echo "Running tests:" @echo "" - @echo " test - build and run functional tests" - @echo " test-debug - build and run functional tests (debug build)" + @echo " test - build and run (basic) functional tests" + @echo " test-debug - build and run (basic) functional tests (debug build)" + @echo " test-all - build and run (all) functional tests" + @echo " test-all-debug - build and run (all) functional tests (debug build)" @echo " retest - run functional tests" @echo " retest-debug - run functional tests (debug build)" @echo " test-wipe - wipe (temporary) files generated by unit tests" diff --git a/test/framework.py b/test/framework.py index b7e6b4a06e7..6b1799a5544 100644 --- a/test/framework.py +++ b/test/framework.py @@ -78,6 +78,15 @@ def pump_output(testclass): # of properly terminating the loop +def running_extended_tests(): + try: + s = os.getenv("EXTENDED_TESTS") + return True if s.lower() in ("y", "yes", "1") else False + except: + return False + return False + + class VppTestCase(unittest.TestCase): """This subclass is a base class for VPP test cases that are implemented as classes. It provides methods to create and run test case. @@ -230,9 +239,6 @@ class VppTestCase(unittest.TestCase): cls.verbose = 0 cls.vpp_dead = False cls.registry = VppObjectRegistry() - print(double_line_delim) - print(colorize(getdoc(cls).splitlines()[0], YELLOW)) - print(double_line_delim) # need to catch exceptions here because if we raise, then the cleanup # doesn't get called and we might end with a zombie vpp try: @@ -613,6 +619,22 @@ class VppTestCase(unittest.TestCase): time.sleep(timeout) +class TestCasePrinter(object): + _shared_state = {} + + def __init__(self): + self.__dict__ = self._shared_state + if not hasattr(self, "_test_case_set"): + self._test_case_set = set() + + def print_test_case_heading_if_first_time(self, case): + if case.__class__ not in self._test_case_set: + print(double_line_delim) + print(colorize(getdoc(case.__class__).splitlines()[0], YELLOW)) + print(double_line_delim) + self._test_case_set.add(case.__class__) + + class VppTestResult(unittest.TestResult): """ @property result_string @@ -641,6 +663,7 @@ class VppTestResult(unittest.TestResult): self.descriptions = descriptions self.verbosity = verbosity self.result_string = None + self.printer = TestCasePrinter() def addSuccess(self, test): """ @@ -740,6 +763,7 @@ class VppTestResult(unittest.TestResult): :param test: """ + self.printer.print_test_case_heading_if_first_time(test) unittest.TestResult.startTest(self, test) if self.verbosity > 0: self.stream.writeln( diff --git a/test/test_bfd.py b/test/test_bfd.py index e7ebb214d9a..5460a2b5589 100644 --- a/test/test_bfd.py +++ b/test/test_bfd.py @@ -15,7 +15,7 @@ from scapy.layers.inet import UDP, IP from scapy.layers.inet6 import IPv6 from bfd import VppBFDAuthKey, BFD, BFDAuthType, VppBFDUDPSession, \ BFDDiagCode, BFDState, BFD_vpp_echo -from framework import VppTestCase, VppTestRunner +from framework import VppTestCase, VppTestRunner, running_extended_tests from vpp_pg_interface import CaptureTimeoutError from util import ppp from vpp_papi_provider import UnexpectedApiReturnValueError @@ -40,6 +40,7 @@ class AuthKeyFactory(object): conf_key_id=conf_key_id, key=key) +@unittest.skipUnless(running_extended_tests(), "part of extended tests") class BFDAPITestCase(VppTestCase): """Bidirectional Forwarding Detection (BFD) - API""" @@ -692,6 +693,7 @@ class BFD4TestCase(VppTestCase): bfd_session_up(self) bfd_session_down(self) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_hold_up(self): """ hold BFD session up """ bfd_session_up(self) @@ -701,6 +703,7 @@ class BFD4TestCase(VppTestCase): self.assert_equal(len(self.vapi.collect_events()), 0, "number of bfd events") + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_slow_timer(self): """ verify slow periodic control frames while session down """ packet_count = 3 @@ -715,6 +718,7 @@ class BFD4TestCase(VppTestCase): time_diff, 0.70, 1.05, "time between slow packets") prev_packet = next_packet + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_zero_remote_min_rx(self): """ no packets when zero remote required min rx interval """ bfd_session_up(self) @@ -739,6 +743,7 @@ class BFD4TestCase(VppTestCase): self.assert_equal( len(self.vapi.collect_events()), 0, "number of bfd events") + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_conn_down(self): """ verify session goes down after inactivity """ bfd_session_up(self) @@ -748,6 +753,7 @@ class BFD4TestCase(VppTestCase): e = self.vapi.wait_for_event(1, "bfd_udp_session_details") verify_event(self, e, expected_state=BFDState.down) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_large_required_min_rx(self): """ large remote required min rx interval """ bfd_session_up(self) @@ -777,6 +783,7 @@ class BFD4TestCase(VppTestCase): break self.assert_equal(count, 0, "number of packets received") + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_immediate_remote_min_rx_reduction(self): """ immediately honor remote required min rx reduction """ self.vpp_session.remove_vpp_config() @@ -809,6 +816,7 @@ class BFD4TestCase(VppTestCase): "time between BFD packets") reference_packet = p + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_modify_req_min_rx_double(self): """ modify session - double required min rx """ bfd_session_up(self) @@ -838,6 +846,7 @@ class BFD4TestCase(VppTestCase): self.assert_in_range(time_to_event, .9 * timeout, 1.1 * timeout, "session timeout") + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_modify_req_min_rx_halve(self): """ modify session - halve required min rx """ self.vpp_session.modify_parameters( @@ -878,6 +887,7 @@ class BFD4TestCase(VppTestCase): "time before bfd session goes down") verify_event(self, e, expected_state=BFDState.down) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_modify_detect_mult(self): """ modify detect multiplier """ bfd_session_up(self) @@ -901,6 +911,7 @@ class BFD4TestCase(VppTestCase): self.assertNotIn("P", p.sprintf("%BFD.flags%"), "Poll bit not set in BFD packet") + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_queued_poll(self): """ test poll sequence queueing """ bfd_session_up(self) @@ -966,6 +977,7 @@ class BFD4TestCase(VppTestCase): self.assertNotIn("P", p.sprintf("%BFD.flags%"), "Poll bit set in BFD packet") + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_poll_response(self): """ test correct response to control frame with poll bit set """ bfd_session_up(self) @@ -976,6 +988,7 @@ class BFD4TestCase(VppTestCase): self, pcap_time_min=time.time() - self.vpp_clock_offset) self.assertIn("F", final.sprintf("%BFD.flags%")) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_no_periodic_if_remote_demand(self): """ no periodic frames outside poll sequence if remote demand set """ bfd_session_up(self) @@ -1093,6 +1106,7 @@ class BFD4TestCase(VppTestCase): "number of bfd events") self.test_session.send_packet() + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_echo_fail(self): """ session goes down if echo function fails """ bfd_session_up(self) @@ -1132,6 +1146,7 @@ class BFD4TestCase(VppTestCase): self.assert_equal(events[0].state, BFDState.down, BFDState) self.assertTrue(verified_diag, "Incorrect diagnostics code received") + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_echo_stop(self): """ echo function stops if peer sets required min echo rx zero """ bfd_session_up(self) @@ -1162,6 +1177,7 @@ class BFD4TestCase(VppTestCase): events = self.vapi.collect_events() self.assert_equal(len(events), 0, "number of bfd events") + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_echo_source_removed(self): """ echo function stops if echo source is removed """ bfd_session_up(self) @@ -1192,6 +1208,7 @@ class BFD4TestCase(VppTestCase): events = self.vapi.collect_events() self.assert_equal(len(events), 0, "number of bfd events") + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_stale_echo(self): """ stale echo packets don't keep a session up """ bfd_session_up(self) @@ -1243,6 +1260,7 @@ class BFD4TestCase(VppTestCase): self.test_session.send_packet() self.assertTrue(timeout_ok, "Expected timeout event didn't occur") + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_invalid_echo_checksum(self): """ echo packets with invalid checksum don't keep a session up """ bfd_session_up(self) @@ -1291,6 +1309,7 @@ class BFD4TestCase(VppTestCase): self.test_session.send_packet() self.assertTrue(timeout_ok, "Expected timeout event didn't occur") + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_admin_up_down(self): """ put session admin-up and admin-down """ bfd_session_up(self) @@ -1328,6 +1347,7 @@ class BFD4TestCase(VppTestCase): e = self.vapi.wait_for_event(1, "bfd_udp_session_details") verify_event(self, e, expected_state=BFDState.up) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_config_change_remote_demand(self): """ configuration change while peer in demand mode """ bfd_session_up(self) @@ -1445,6 +1465,7 @@ class BFD6TestCase(VppTestCase): self.test_session.send_packet() self.assert_equal(self.vpp_session.state, BFDState.up, BFDState) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_hold_up(self): """ hold BFD session up """ bfd_session_up(self) @@ -1502,6 +1523,7 @@ class BFD6TestCase(VppTestCase): self.assert_equal(udp_sport_tx, udp_sport_rx, "UDP source port (== " "ECHO packet identifier for test purposes)") + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_echo(self): """ echo function used """ bfd_session_up(self) @@ -1596,6 +1618,7 @@ class BFDSHA1TestCase(VppTestCase): bfd_key_id=self.vpp_session.bfd_key_id) bfd_session_up(self) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_hold_up(self): """ hold BFD session up """ key = self.factory.create_random_key(self) @@ -1614,6 +1637,7 @@ class BFDSHA1TestCase(VppTestCase): self.test_session.send_packet() self.assert_equal(self.vpp_session.state, BFDState.up, BFDState) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_hold_up_meticulous(self): """ hold BFD session up - meticulous auth """ key = self.factory.create_random_key( @@ -1635,6 +1659,7 @@ class BFDSHA1TestCase(VppTestCase): self.test_session.send_packet() self.assert_equal(self.vpp_session.state, BFDState.up, BFDState) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_send_bad_seq_number(self): """ session is not kept alive by msgs with bad sequence numbers""" key = self.factory.create_random_key( @@ -1697,6 +1722,7 @@ class BFDSHA1TestCase(VppTestCase): wait_for_bfd_packet(self) self.assert_equal(self.vpp_session.state, BFDState.up, BFDState) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_mismatch_auth(self): """ session is not brought down by unauthenticated msg """ key = self.factory.create_random_key(self) @@ -1711,6 +1737,7 @@ class BFDSHA1TestCase(VppTestCase): legitimate_test_session, rogue_test_session) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_mismatch_bfd_key_id(self): """ session is not brought down by msg with non-existent key-id """ key = self.factory.create_random_key(self) @@ -1730,6 +1757,7 @@ class BFDSHA1TestCase(VppTestCase): legitimate_test_session, rogue_test_session) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_mismatched_auth_type(self): """ session is not brought down by msg with wrong auth type """ key = self.factory.create_random_key(self) @@ -1746,6 +1774,7 @@ class BFDSHA1TestCase(VppTestCase): vpp_session, legitimate_test_session, rogue_test_session, {'auth_type': BFDAuthType.keyed_md5}) + @unittest.skipUnless(running_extended_tests(), "part of extended tests") def test_restart(self): """ simulate remote peer restart and resynchronization """ key = self.factory.create_random_key( @@ -1761,7 +1790,7 @@ class BFDSHA1TestCase(VppTestCase): # don't send any packets for 2*detection_time detection_time = self.test_session.detect_mult *\ self.vpp_session.required_min_rx / USEC_IN_SEC - self.sleep(2*detection_time, "simulating peer restart") + self.sleep(2 * detection_time, "simulating peer restart") events = self.vapi.collect_events() self.assert_equal(len(events), 1, "number of bfd events") verify_event(self, events[0], expected_state=BFDState.down) @@ -1774,6 +1803,7 @@ class BFDSHA1TestCase(VppTestCase): bfd_session_up(self) +@unittest.skipUnless(running_extended_tests(), "part of extended tests") class BFDAuthOnOffTestCase(VppTestCase): """Bidirectional Forwarding Detection (BFD) (changing auth) """ @@ -1981,6 +2011,7 @@ class BFDAuthOnOffTestCase(VppTestCase): "number of bfd events") +@unittest.skipUnless(running_extended_tests(), "part of extended tests") class BFDCLITestCase(VppTestCase): """Bidirectional Forwarding Detection (BFD) (CLI) """ pg0 = None |