Regression: another fix for ipv6 STL test, add flag --no-dut-config to disable DUT config

Change-Id: I56c38324086cfc81d2ad0a451d528707f5357b51
Signed-off-by: Yaroslav Brustinov <ybrustin@cisco.com>

11 files changed, 75 insertions, 67 deletions

diff --git a/scripts/automation/regression/stateful_tests/trex_client_cfg_test.py b/scripts/automation/regression/stateful_tests/trex_client_cfg_test.py
index 158f59b9..baac8bf0 100644
--- a/scripts/automation/regression/stateful_tests/trex_client_cfg_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_client_cfg_test.py
@@ -23,10 +23,10 @@ class CTRexClientCfg_Test(CTRexNbarBase):
     def test_client_cfg_nbar(self):
         if self.is_loopback:
             self.skip('No NBAR on loopback')
-
-        self.router.configure_basic_interfaces()
-        self.router.config_pbr(mode = "config")
-        self.router.config_nbar_pd()
+        if not CTRexScenario.router_cfg['no_dut_config']:
+            self.router.configure_basic_interfaces()
+            self.router.config_pbr(mode = "config")
+            self.router.config_nbar_pd()
 
         mult = self.get_benchmark_param('multiplier')
         core = self.get_benchmark_param('cores')
@@ -51,8 +51,9 @@ class CTRexClientCfg_Test(CTRexNbarBase):
         if self.is_loopback:
             self.skip('Not relevant on loopback')
 
-        self.router.configure_basic_interfaces(vlan = True)
-        self.router.config_pbr(mode = "config", vlan = True)
+        if not CTRexScenario.router_cfg['no_dut_config']:
+            self.router.configure_basic_interfaces(vlan = True)
+            self.router.config_pbr(mode = "config", vlan = True)
 
         mult = self.get_benchmark_param('multiplier')
         core = self.get_benchmark_param('cores')
diff --git a/scripts/automation/regression/stateful_tests/trex_client_pkg_test.py b/scripts/automation/regression/stateful_tests/trex_client_pkg_test.py
index 892be966..8d5ddebf 100755
--- a/scripts/automation/regression/stateful_tests/trex_client_pkg_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_client_pkg_test.py
@@ -10,7 +10,7 @@ class CTRexClientPKG_Test(CTRexGeneral_Test):
 
     def setUp(self):
         CTRexGeneral_Test.setUp(self)
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces()
             self.router.config_pbr(mode = 'config')
         self.unzip_client_package()
diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py
index dda379f9..1f52de07 100755
--- a/scripts/automation/regression/stateful_tests/trex_general_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_general_test.py
@@ -403,7 +403,7 @@ class CTRexGeneral_Test(unittest.TestCase):
             self.trex.force_kill(confirm = False)
         if not self.is_loopback:
             print('')
-            if not self.stl_trex: # stateful
+            if not self.stl_trex and CTRexScenario.router_cfg['forceCleanConfig']:
                 self.router.load_clean_config()
             self.router.clear_counters()
             self.router.clear_packet_drop_stats()
diff --git a/scripts/automation/regression/stateful_tests/trex_imix_test.py b/scripts/automation/regression/stateful_tests/trex_imix_test.py
index 4cb01db3..650bc3e5 100755
--- a/scripts/automation/regression/stateful_tests/trex_imix_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_imix_test.py
@@ -1,5 +1,5 @@
 #!/router/bin/python
-from .trex_general_test import CTRexGeneral_Test
+from .trex_general_test import CTRexGeneral_Test, CTRexScenario
 from CPlatform import CStaticRouteConfig
 from .tests_exceptions import *
 #import sys
@@ -21,7 +21,7 @@ class CTRexIMIX_Test(CTRexGeneral_Test):
     def test_short_flow(self):
         """ short UDP flow with 64B packets, this test with small number of active flows """
         # test initializtion
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces()
             self.router.config_pbr(mode = "config")
 
@@ -50,7 +50,7 @@ class CTRexIMIX_Test(CTRexGeneral_Test):
     def test_short_flow_high_active(self):
         """ short UDP flow with 64B packets, this test with 8M  active flows """
         # test initializtion
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces()
             self.router.config_pbr(mode = "config")
 
@@ -82,7 +82,7 @@ class CTRexIMIX_Test(CTRexGeneral_Test):
     def test_short_flow_high_active2(self):
         """ short UDP flow with 64B packets, this test with 8M  active flows """
         # test initializtion
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces()
             self.router.config_pbr(mode = "config")
 
@@ -113,7 +113,7 @@ class CTRexIMIX_Test(CTRexGeneral_Test):
 
     def test_routing_imix_64(self):
         # test initializtion
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces()
             self.router.config_pbr(mode = "config")
 
@@ -160,7 +160,7 @@ class CTRexIMIX_Test(CTRexGeneral_Test):
 
     def test_routing_imix (self):
         # test initializtion
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces()
             self.router.config_pbr(mode = "config")
 
@@ -193,7 +193,7 @@ class CTRexIMIX_Test(CTRexGeneral_Test):
         if self.is_loopback:
             self.skip('In loopback mode the test is same as test_routing_imix')
         # test initializtion
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces()
 
             # Configure static routing based on benchmark data input
@@ -228,7 +228,7 @@ class CTRexIMIX_Test(CTRexGeneral_Test):
 
     def test_static_routing_imix_asymmetric (self):
         # test initializtion
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces()
 
             # Configure static routing based on benchmark data input
@@ -260,7 +260,7 @@ class CTRexIMIX_Test(CTRexGeneral_Test):
 
 
     def test_jumbo(self, duration = 100, **kwargs):
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces(mtu = 9216)
             self.router.config_pbr(mode = "config")
 
diff --git a/scripts/automation/regression/stateful_tests/trex_ipv6_test.py b/scripts/automation/regression/stateful_tests/trex_ipv6_test.py
index 4d6f7953..ae06bcd6 100755
--- a/scripts/automation/regression/stateful_tests/trex_ipv6_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_ipv6_test.py
@@ -1,5 +1,5 @@
 #!/router/bin/python
-from .trex_general_test import CTRexGeneral_Test
+from .trex_general_test import CTRexGeneral_Test, CTRexScenario
 from .tests_exceptions import *
 import time
 from nose.tools import assert_equal
@@ -19,7 +19,7 @@ class CTRexIPv6_Test(CTRexGeneral_Test):
         if self.is_virt_nics:
             self.skip('--ipv6 flag does not work correctly in with virtual NICs') # TODO: fix
         # test initializtion
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces()
 
             self.router.config_pbr(mode = "config")
@@ -57,10 +57,10 @@ class CTRexIPv6_Test(CTRexGeneral_Test):
         if self.is_loopback:
             self.skip('The test checks ipv6 drops by device and we are in loopback setup')
         # test initializtion
-        self.router.configure_basic_interfaces()
-
-        # NOT CONFIGURING IPv6 INTENTIONALLY TO GET DROPS!
-        self.router.config_pbr(mode = "config")
+        if not CTRexScenario.router_cfg['no_dut_config']:
+            self.router.configure_basic_interfaces()
+            # NOT CONFIGURING IPv6 INTENTIONALLY TO GET DROPS!
+            self.router.config_pbr(mode = "config")
         
         # same params as test_ipv6_simple
         mult = self.get_benchmark_param('multiplier', test_name = 'test_ipv6_simple')
diff --git a/scripts/automation/regression/stateful_tests/trex_nat_test.py b/scripts/automation/regression/stateful_tests/trex_nat_test.py
index 3da1dc79..75ab3b7b 100755
--- a/scripts/automation/regression/stateful_tests/trex_nat_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_nat_test.py
@@ -1,5 +1,5 @@
 #!/router/bin/python
-from .trex_general_test import CTRexGeneral_Test
+from .trex_general_test import CTRexGeneral_Test, CTRexScenario
 from .tests_exceptions import *
 import time
 from CPlatform import CStaticRouteConfig, CNatConfig
@@ -21,13 +21,12 @@ class CTRexNoNat_Test(CTRexGeneral_Test):#(unittest.TestCase):
 
     def test_nat_learning(self):
         # test initializtion
-        self.router.configure_basic_interfaces()
-
-        stat_route_dict = self.get_benchmark_param('stat_route_dict')
-        stat_route_obj = CStaticRouteConfig(stat_route_dict)
-        self.router.config_static_routing(stat_route_obj, mode = "config")
-
-        self.router.config_nat_verify()         # shutdown duplicate interfaces
+        if not CTRexScenario.router_cfg['no_dut_config']:
+            self.router.configure_basic_interfaces()
+            stat_route_dict = self.get_benchmark_param('stat_route_dict')
+            stat_route_obj = CStaticRouteConfig(stat_route_dict)
+            self.router.config_static_routing(stat_route_obj, mode = "config")
+            self.router.config_nat_verify()         # shutdown duplicate interfaces
 
 #       self.trex.set_yaml_file('cap2/http_simple.yaml')
         mult = self.get_benchmark_param('multiplier')
@@ -104,16 +103,14 @@ class CTRexNat_Test(CTRexGeneral_Test):#(unittest.TestCase):
 
     def nat_simple_helper(self, learn_mode=1, traffic_file='cap2/http_simple.yaml'):
         # test initializtion
-        self.router.configure_basic_interfaces()
-
-        
-        stat_route_dict = self.get_benchmark_param('stat_route_dict')
-        stat_route_obj = CStaticRouteConfig(stat_route_dict)
-        self.router.config_static_routing(stat_route_obj, mode = "config")
-
-        nat_dict = self.get_benchmark_param('nat_dict')
-        nat_obj  = CNatConfig(nat_dict)
-        self.router.config_nat(nat_obj)
+        if not CTRexScenario.router_cfg['no_dut_config']:
+            self.router.configure_basic_interfaces()
+            stat_route_dict = self.get_benchmark_param('stat_route_dict')
+            stat_route_obj = CStaticRouteConfig(stat_route_dict)
+            self.router.config_static_routing(stat_route_obj, mode = "config")
+            nat_dict = self.get_benchmark_param('nat_dict')
+            nat_obj  = CNatConfig(nat_dict)
+            self.router.config_nat(nat_obj)
 
 #       self.trex.set_yaml_file('cap2/http_simple.yaml')
         mult = self.get_benchmark_param('multiplier')
diff --git a/scripts/automation/regression/stateful_tests/trex_nbar_test.py b/scripts/automation/regression/stateful_tests/trex_nbar_test.py
index a98da9ac..59e8c0ae 100755
--- a/scripts/automation/regression/stateful_tests/trex_nbar_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_nbar_test.py
@@ -1,5 +1,5 @@
 #!/router/bin/python
-from .trex_general_test import CTRexGeneral_Test
+from .trex_general_test import CTRexGeneral_Test, CTRexScenario
 from .tests_exceptions import *
 from interfaces_e import IFType
 from nose.tools import nottest
@@ -56,10 +56,11 @@ class CTRexNbar_Test(CTRexNbarBase):
     def test_nbar_simple(self):
         # test initializtion
         deviation_compare_value = 0.03   # default value of deviation - 3%
-        self.router.configure_basic_interfaces()
 
-        self.router.config_pbr(mode = "config")
-        self.router.config_nbar_pd()
+        if not CTRexScenario.router_cfg['no_dut_config']:
+            self.router.configure_basic_interfaces()
+            self.router.config_pbr(mode = "config")
+            self.router.config_nbar_pd()
 
         mult = self.get_benchmark_param('multiplier')
         core = self.get_benchmark_param('cores')
@@ -89,9 +90,10 @@ class CTRexNbar_Test(CTRexNbarBase):
 
     # the name intentionally not matches nose default pattern, including the test should be specified explicitly
     def NBarLong(self):
-        self.router.configure_basic_interfaces()
-        self.router.config_pbr(mode = "config")
-        self.router.config_nbar_pd()
+        if not CTRexScenario.router_cfg['no_dut_config']:
+            self.router.configure_basic_interfaces()
+            self.router.config_pbr(mode = "config")
+            self.router.config_nbar_pd()
 
         mult = self.get_benchmark_param('multiplier')
         core = self.get_benchmark_param('cores')
diff --git a/scripts/automation/regression/stateful_tests/trex_rx_test.py b/scripts/automation/regression/stateful_tests/trex_rx_test.py
index 161856b1..d2050703 100755
--- a/scripts/automation/regression/stateful_tests/trex_rx_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_rx_test.py
@@ -1,5 +1,5 @@
 #!/router/bin/python
-from .trex_general_test import CTRexGeneral_Test
+from .trex_general_test import CTRexGeneral_Test, CTRexScenario
 from CPlatform import CStaticRouteConfig, CNatConfig
 from .tests_exceptions import *
 #import sys
@@ -87,7 +87,7 @@ class CTRexRx_Test(CTRexGeneral_Test):
             self.fail('Errors in rx_check: %s' % e)
 
     def test_rx_check_sfr(self):
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces()
             self.router.config_pbr(mode = 'config')
 
@@ -121,7 +121,7 @@ class CTRexRx_Test(CTRexGeneral_Test):
 
 
     def test_rx_check_http(self):
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             # TODO: skip as test_rx_check_http_negative will cover it
             #self.skip('This test is covered by test_rx_check_http_negative')
             self.router.configure_basic_interfaces()
@@ -155,7 +155,7 @@ class CTRexRx_Test(CTRexGeneral_Test):
 
 
     def test_rx_check_sfr_ipv6(self):
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces()
             self.router.config_pbr(mode = 'config')
             self.router.config_ipv6_pbr(mode = "config")
@@ -190,7 +190,7 @@ class CTRexRx_Test(CTRexGeneral_Test):
 
 
     def test_rx_check_http_ipv6(self):
-        if not self.is_loopback:
+        if not self.is_loopback and not CTRexScenario.router_cfg['no_dut_config']:
             self.router.configure_basic_interfaces()
             self.router.config_pbr(mode = "config")
             self.router.config_ipv6_pbr(mode = "config")
@@ -226,8 +226,9 @@ class CTRexRx_Test(CTRexGeneral_Test):
         if self.is_loopback:
             self.skip('This test uses NAT, not relevant for loopback')
 
-        self.router.configure_basic_interfaces()
-        self.router.config_pbr(mode = "config")
+        if not CTRexScenario.router_cfg['no_dut_config']:
+            self.router.configure_basic_interfaces()
+            self.router.config_pbr(mode = "config")
 
         core  = self.get_benchmark_param('cores')
         mult  = self.get_benchmark_param('multiplier')
diff --git a/scripts/automation/regression/stateless_tests/stl_general_test.py b/scripts/automation/regression/stateless_tests/stl_general_test.py
index 6470d8c2..952ed70e 100644
--- a/scripts/automation/regression/stateless_tests/stl_general_test.py
+++ b/scripts/automation/regression/stateless_tests/stl_general_test.py
@@ -73,14 +73,15 @@ class STLBasic_Test(CStlGeneral_Test):
         if not self.is_loopback:
             try:
                 sys.stdout.flush()
-                sys.stdout.write('Configuring DUT... ')
-                start_time = time.time()
-                if CTRexScenario.router_cfg['forceCleanConfig']:
-                    CTRexScenario.router.load_clean_config()
-                CTRexScenario.router.configure_basic_interfaces()
-                CTRexScenario.router.config_pbr(mode = "config")
-                CTRexScenario.router.config_ipv6_pbr(mode = "config")
-                sys.stdout.write('done. (%ss)\n' % int(time.time() - start_time))
+                if not CTRexScenario.router_cfg['no_dut_config']:
+                    sys.stdout.write('Configuring DUT... ')
+                    start_time = time.time()
+                    if CTRexScenario.router_cfg['forceCleanConfig']:
+                        CTRexScenario.router.load_clean_config()
+                    CTRexScenario.router.configure_basic_interfaces()
+                    CTRexScenario.router.config_pbr(mode = "config")
+                    CTRexScenario.router.config_ipv6_pbr(mode = "config")
+                    sys.stdout.write('done. (%ss)\n' % int(time.time() - start_time))
             except Exception as e:
                 print('')
                 CTRexScenario.stl_init_error = 'Could not configure device, err: %s' % e
diff --git a/scripts/automation/regression/stateless_tests/stl_ipv6_test.py b/scripts/automation/regression/stateless_tests/stl_ipv6_test.py
index b8173db5..1fe248a3 100755
--- a/scripts/automation/regression/stateless_tests/stl_ipv6_test.py
+++ b/scripts/automation/regression/stateless_tests/stl_ipv6_test.py
@@ -8,8 +8,8 @@ class STLIPv6_Test(CStlGeneral_Test):
     def setUp(self):
         CStlGeneral_Test.setUp(self)
         print('')
-        self.stl_trex.set_service_mode(ports = [0])
         self.stl_trex.reset()
+        self.stl_trex.set_service_mode(ports = [0])
 
     def tearDown(self):
         CStlGeneral_Test.tearDown(self)
diff --git a/scripts/automation/regression/trex_unit_test.py b/scripts/automation/regression/trex_unit_test.py
index de77385e..ac6203bf 100755
--- a/scripts/automation/regression/trex_unit_test.py
+++ b/scripts/automation/regression/trex_unit_test.py
@@ -70,13 +70,13 @@ def id_split(idval):
 # option to select wanted test by name without file, class etc.
 def new_Selector_wantMethod(self, method, orig_Selector_wantMethod = Selector.wantMethod):
     result = orig_Selector_wantMethod(self, method)
-    return result and (not CTRexScenario.test or CTRexScenario.test in getattr(method, '__name__', ''))
+    return result and (not CTRexScenario.test or filter(lambda t: t in getattr(method, '__name__', ''), CTRexScenario.test.split(',')))
 
 Selector.wantMethod = new_Selector_wantMethod
 
 def new_Selector_wantFunction(self, function, orig_Selector_wantFunction = Selector.wantFunction):
     result = orig_Selector_wantFunction(self, function)
-    return result and (not CTRexScenario.test or CTRexScenario.test in getattr(function, '__name__', ''))
+    return result and (not CTRexScenario.test or filter(lambda t: t in getattr(function, '__name__', ''), CTRexScenario.test.split(',')))
 
 Selector.wantFunction = new_Selector_wantFunction
 
@@ -342,7 +342,9 @@ class CTRexTestConfiguringPlugin(Plugin):
         parser.add_option('--trex-args', default = '',
                             help="Additional TRex arguments (--no-watchdog etc.).")
         parser.add_option('-t', '--test', type = str,
-                            help = 'Test name to run (without file, class etc.)')
+                            help = 'Test name to run (without file, class etc.). Can choose several names splitted by comma.')
+        parser.add_option('--no-dut-config', action = 'store_true',
+                            help = 'Skip the config of DUT to save time. Implies --skip-clean.')
 
 
     def configure(self, options, conf):
@@ -378,6 +380,9 @@ class CTRexTestConfiguringPlugin(Plugin):
         self.kill_running  = options.kill_running
         self.load_image    = options.load_image
         self.clean_config  = False if options.skip_clean_config else True
+        self.no_dut_config = options.no_dut_config
+        if self.no_dut_config:
+            self.clean_config = False
         self.server_logs   = options.server_logs
         if options.log_path:
             self.loggerPath = options.log_path
@@ -495,6 +500,7 @@ class CTRexTestConfiguringPlugin(Plugin):
                                             forceImageReload = self.load_image,
                                             silent_mode      = not self.telnet_verbose,
                                             forceCleanConfig = self.clean_config,
+                                            no_dut_config    = self.no_dut_config,
                                             tftp_config_dict = self.configuration.tftp)
         try:
             CustomLogger.setup_custom_logger('TRexLogger', self.loggerPath)