regression:

    corrections of benchmarks
    move packet builder tests to functional folder
    add CPU utilization boundries for more tests
    print Skipping message
    fix relative drop counting
API:
    convert hostname to IP at init of client side (trex_client.py)
various:
    move python path determination to external common file
    add functional tests running bash script
    add sudo check to t-rex-64

16 files changed, 50 insertions, 54 deletions

diff --git a/scripts/automation/regression/setups/kiwi02/benchmark.yaml b/scripts/automation/regression/setups/kiwi02/benchmark.yaml
index c387a994..ccd566cc 100644
--- a/scripts/automation/regression/setups/kiwi02/benchmark.yaml
+++ b/scripts/automation/regression/setups/kiwi02/benchmark.yaml
@@ -49,9 +49,9 @@ test_nat_simple  :
     dual_port_mask           : 1.0.0.0
     pool_start               : 200.0.0.0
     pool_netmask             : 255.255.255.0
-  multiplier        : 400
+  multiplier        : 10000
   cpu_to_core_ratio : 37270000
-  cores             : 4
+  cores             : 1
   exp_bw            : 1
   exp_latency       : 1
   allow_timeout_dev : YES
@@ -63,8 +63,8 @@ test_nat_learning  :
     dual_port_mask          : 1.0.0.0
     client_destination_mask : 255.0.0.0
     server_destination_mask : 255.0.0.0
-  multiplier        : 400 
-  cores             : 4
+  multiplier        : 10000 
+  cores             : 1
   nat_opened        : 100000
   cpu_to_core_ratio : 37270000
   exp_bw            : 1
@@ -78,7 +78,7 @@ test_routing_imix_64    :
   exp_latency       : 1
 
 test_routing_imix   :
-  multiplier        : 35
+  multiplier        : 32
   cores             : 2
   cpu_to_core_ratio : 8900
   exp_latency       : 1
@@ -90,7 +90,7 @@ test_static_routing_imix    :
     dual_port_mask          : 1.0.0.0
     client_destination_mask : 255.0.0.0
     server_destination_mask : 255.0.0.0
-  multiplier        : 35
+  multiplier        : 32
   cores             : 2
   cpu_to_core_ratio : 3766666
   exp_latency       : 1
@@ -102,7 +102,7 @@ test_static_routing_imix_asymmetric:
     dual_port_mask          : 1.0.0.0
     client_destination_mask : 255.0.0.0
     server_destination_mask : 255.0.0.0
-  multiplier        : 18
+  multiplier        : 16
   cores             : 1
   cpu_to_core_ratio : 3766666
   exp_latency       : 1
@@ -135,7 +135,11 @@ test_rx_check_http_ipv6:
     cores             : 2
     rx_sample_rate    : 32
 
+test_rx_check_http_negative:
+    multiplier        : 40000
+    cores             : 2
+    rx_sample_rate    : 32
 
 test_jumbo:
-    multiplier : 56
+    multiplier : 55
     cores      : 1
diff --git a/scripts/automation/regression/setups/trex-dan/benchmark.yaml b/scripts/automation/regression/setups/trex-dan/benchmark.yaml
index f65fcf90..ae814551 100644
--- a/scripts/automation/regression/setups/trex-dan/benchmark.yaml
+++ b/scripts/automation/regression/setups/trex-dan/benchmark.yaml
@@ -49,7 +49,7 @@ test_nat_simple  :
     dual_port_mask           : 1.0.0.0
     pool_start               : 200.0.0.0
     pool_netmask             : 255.255.255.0
-  multiplier        : 150 
+  multiplier        : 550 
   cores             : 1
   cpu_to_core_ratio : 37270000
   exp_bw            : 1
@@ -63,7 +63,7 @@ test_nat_learning  :
     dual_port_mask          : 1.0.0.0
     client_destination_mask : 255.0.0.0
     server_destination_mask : 255.0.0.0
-  multiplier        : 150 
+  multiplier        : 550 
   cores             : 1
   nat_opened        : 40000
   cpu_to_core_ratio : 270
diff --git a/scripts/automation/regression/setups/trex-dan/config.yaml b/scripts/automation/regression/setups/trex-dan/config.yaml
index ae60f9ad..5f91ea6a 100644
--- a/scripts/automation/regression/setups/trex-dan/config.yaml
+++ b/scripts/automation/regression/setups/trex-dan/config.yaml
@@ -34,7 +34,6 @@
 
 trex:
   hostname       : trex-dan
-#  version_path   : /auto/proj-pcube-b/apps/PL-b/tools/bp_sim2/v1.57/ #/auto/srg-sce-swinfra-usr/emb/users/danklei/Work/asr1k/emb/private/bpsim/main/scripts
   cores          : 2
   modes          : [VM]
 
diff --git a/scripts/automation/regression/setups/trex04/benchmark.yaml b/scripts/automation/regression/setups/trex04/benchmark.yaml
index 56193f46..e5459dce 100644
--- a/scripts/automation/regression/setups/trex04/benchmark.yaml
+++ b/scripts/automation/regression/setups/trex04/benchmark.yaml
@@ -20,7 +20,7 @@ test_routing_imix_64    :
   exp_latency       : 1
 
 test_routing_imix   :
-  multiplier        : 0.8
+  multiplier        : 0.5
   cores             : 1
   cpu_to_core_ratio : 1800
   exp_latency       : 1
diff --git a/scripts/automation/regression/unit_tests/pkt_bld_tests/payload_gen_test.py b/scripts/automation/regression/unit_tests/functional_tests/payload_gen_test.py
index 80d2b086..80d2b086 100755
--- a/scripts/automation/regression/unit_tests/pkt_bld_tests/payload_gen_test.py
+++ b/scripts/automation/regression/unit_tests/functional_tests/payload_gen_test.py
diff --git a/scripts/automation/regression/unit_tests/pkt_bld_tests/pkt_bld_general_test.py b/scripts/automation/regression/unit_tests/functional_tests/pkt_bld_general_test.py
index b630147b..b630147b 100755
--- a/scripts/automation/regression/unit_tests/pkt_bld_tests/pkt_bld_general_test.py
+++ b/scripts/automation/regression/unit_tests/functional_tests/pkt_bld_general_test.py
diff --git a/scripts/automation/regression/unit_tests/pkt_bld_tests/pkt_builder_test.py b/scripts/automation/regression/unit_tests/functional_tests/pkt_builder_test.py
index a0b87a43..fd157c8a 100755
--- a/scripts/automation/regression/unit_tests/pkt_bld_tests/pkt_builder_test.py
+++ b/scripts/automation/regression/unit_tests/functional_tests/pkt_builder_test.py
@@ -212,7 +212,7 @@ class CTRexPktBuilder_Test(pkt_bld_general_test.CGeneralPktBld_Test):
         # finally, set IP header len with relation to payload data
         self.pkt_bld.set_layer_attr("l3_ip", "len", len(self.pkt_bld.get_layer('l3_ip')))
 
-        filepath = "unit_tests/pkt_bld_tests/test.pcap"
+        filepath = "unit_tests/functional_tests/test.pcap"
         self.pkt_bld.dump_pkt_to_pcap(filepath)
         assert os.path.isfile(filepath)
         # remove pcap after creation - masked for now
diff --git a/scripts/automation/regression/unit_tests/pkt_bld_tests/test.pcap b/scripts/automation/regression/unit_tests/functional_tests/test.pcap
index da1830b8..e2b12565 100755
--- a/scripts/automation/regression/unit_tests/pkt_bld_tests/test.pcap
+++ b/scripts/automation/regression/unit_tests/functional_tests/test.pcap
diff --git a/scripts/automation/regression/unit_tests/pkt_bld_tests/test2.pcap b/scripts/automation/regression/unit_tests/functional_tests/test2.pcap
index 1d35d9c1..1d35d9c1 100755
--- a/scripts/automation/regression/unit_tests/pkt_bld_tests/test2.pcap
+++ b/scripts/automation/regression/unit_tests/functional_tests/test2.pcap
diff --git a/scripts/automation/regression/unit_tests/pkt_bld_tests/test_cmp.pcap b/scripts/automation/regression/unit_tests/functional_tests/test_cmp.pcap
index 4c92859f..4c92859f 100755
--- a/scripts/automation/regression/unit_tests/pkt_bld_tests/test_cmp.pcap
+++ b/scripts/automation/regression/unit_tests/functional_tests/test_cmp.pcap
diff --git a/scripts/automation/regression/unit_tests/pkt_bld_tests/vm_test.py b/scripts/automation/regression/unit_tests/functional_tests/vm_test.py
index 603c52db..603c52db 100755
--- a/scripts/automation/regression/unit_tests/pkt_bld_tests/vm_test.py
+++ b/scripts/automation/regression/unit_tests/functional_tests/vm_test.py
diff --git a/scripts/automation/regression/unit_tests/pkt_bld_tests/vm_variable_test.py b/scripts/automation/regression/unit_tests/functional_tests/vm_variable_test.py
index af56b1b1..af56b1b1 100755
--- a/scripts/automation/regression/unit_tests/pkt_bld_tests/vm_variable_test.py
+++ b/scripts/automation/regression/unit_tests/functional_tests/vm_variable_test.py
diff --git a/scripts/automation/regression/unit_tests/trex_general_test.py b/scripts/automation/regression/unit_tests/trex_general_test.py
index af3c897d..9bcccaab 100755
--- a/scripts/automation/regression/unit_tests/trex_general_test.py
+++ b/scripts/automation/regression/unit_tests/trex_general_test.py
@@ -213,7 +213,7 @@ class CTRexGeneral_Test(unittest.TestCase):
         if not test_name:
             test_name = self.get_name()
         if test_name not in self.benchmark:
-            self.skip('No data in benchmark.yaml for test %s, skipping.' % test_name)
+            self.skip('No data in benchmark.yaml for test: %s, param: %s. Skipping.' % (test_name, param))
         if sub_param:
             return self.benchmark[test_name][param].get(sub_param)
         else:
@@ -230,7 +230,7 @@ class CTRexGeneral_Test(unittest.TestCase):
             trex_tx_pckt    = trex_res.get_last_value("trex-global.data.m_total_tx_pkts")
             trex_drops      = trex_res.get_total_drops()
             trex_drop_rate  = trex_res.get_drop_rate()
-            if ( (trex_drops/trex_tx_pckt) > 0.001) and (trex_drop_rate > 0.0):     # deliberately mask kickoff drops when T-Rex first initiated
+            if ( trex_drops > 0.001 * trex_tx_pckt) and (trex_drop_rate > 0.0):     # deliberately mask kickoff drops when T-Rex first initiated
                 self.fail('Number of packet drops larger than 0.1% of all traffic')
 
             # check queue full, queue drop, allocation error
@@ -282,7 +282,8 @@ class CTRexGeneral_Test(unittest.TestCase):
         self.fail_reasons.append(reason)
 
     # skip running of the test, counts as 'passed' but prints 'skipped'
-    def skip(self, message = ''):
+    def skip(self, message = 'Unknown reason'):
+        print 'Skip: %s' % message
         self.skipping = True
         raise SkipTest(message)
 
diff --git a/scripts/automation/regression/unit_tests/trex_nat_test.py b/scripts/automation/regression/unit_tests/trex_nat_test.py
index 452f7ecf..9fe12507 100755
--- a/scripts/automation/regression/unit_tests/trex_nat_test.py
+++ b/scripts/automation/regression/unit_tests/trex_nat_test.py
@@ -67,8 +67,7 @@ class CTRexNoNat_Test(CTRexGeneral_Test):#(unittest.TestCase):
         self.check_results_gt (learning_stats, 'm_total_nat_open', expected_nat_opened)
 
         self.check_general_scenario_results(trex_res)
-
-        # self.check_CPU_benchmark(trex_res, 10)
+        self.check_CPU_benchmark(trex_res, minimal_cpu = 10, maximal_cpu = 85)
 
     def tearDown(self):
         CTRexGeneral_Test.tearDown(self)
@@ -128,7 +127,7 @@ class CTRexNat_Test(CTRexGeneral_Test):#(unittest.TestCase):
         if self.get_benchmark_param('allow_timeout_dev'):
             nat_timeout_ratio = trex_nat_stats['m_total_nat_time_out']/trex_nat_stats['m_total_nat_open']
             if nat_timeout_ratio > 0.005:
-                self.fail('TRex nat_timeout ratio %f > 0.005 (0.5%) and not as expected to be less than 0.5%' %(nat_timeout_ratio))
+                self.fail('TRex nat_timeout ratio %f > 0.5%%' % nat_timeout_ratio)
         else:
             self.check_results_eq (trex_nat_stats,'m_total_nat_time_out', 0.0)
         self.check_results_eq (trex_nat_stats,'m_total_nat_no_fid', 0.0)
@@ -142,7 +141,7 @@ class CTRexNat_Test(CTRexGeneral_Test):#(unittest.TestCase):
 #        test_norm_cpu = 2*(trex_tx_pckt/(core*cpu_util))
 #        print "test_norm_cpu is: ", test_norm_cpu
 
-        # self.check_CPU_benchmark(trex_res, 10)
+        self.check_CPU_benchmark(trex_res, minimal_cpu = 10, maximal_cpu = 85)
 
         #if ( abs((test_norm_cpu/self.get_benchmark_param('cpu_to_core_ratio')) - 1) > 0.03):
         #    raiseraise AbnormalResultError('Normalized bandwidth to CPU utilization ratio exceeds 3%')
diff --git a/scripts/automation/regression/unit_tests/trex_rx_test.py b/scripts/automation/regression/unit_tests/trex_rx_test.py
index a37615c4..4f404616 100755
--- a/scripts/automation/regression/unit_tests/trex_rx_test.py
+++ b/scripts/automation/regression/unit_tests/trex_rx_test.py
@@ -223,9 +223,7 @@ class CTRexRx_Test(CTRexGeneral_Test):
             self.skip('This test uses NAT, not relevant for loopback')
 
         self.router.configure_basic_interfaces()
-        stat_route_dict = self.get_benchmark_param('stat_route_dict')
-        stat_route_obj = CStaticRouteConfig(stat_route_dict)
-        self.router.config_static_routing(stat_route_obj, mode = "config")
+        self.router.config_pbr(mode = "config")
 
         core  = self.get_benchmark_param('cores')
         mult  = self.get_benchmark_param('multiplier')
@@ -234,48 +232,40 @@ class CTRexRx_Test(CTRexGeneral_Test):
         ret = self.trex.start_trex(
             c = core,
             m = mult,
-            #p  = True,
-            #nc = True,
+            p = True,
             rx_check = sample_rate,
-            d = 80,
+            d = 50,
             f = 'cap2/http_simple.yaml',
             l = 1000,
             k = 10,
             learn_verify = True,
             l_pkt_mode = 2)
 
-        print 'Run for 2 minutes, expect no errors'
-        trex_res = self.trex.sample_x_seconds(60)
+        print 'Run for 40 seconds, expect no errors'
+        trex_res = self.trex.sample_x_seconds(40)
         print ("\nLATEST RESULT OBJECT:")
         print trex_res
         self.check_general_scenario_results(trex_res)
-        self.check_CPU_benchmark(trex_res, 10)
+        self.check_CPU_benchmark(trex_res)
         self.check_rx_errors(trex_res)
 
-        try:
-        # TODO: add nat/zbf config for router
-            nat_dict = self.get_benchmark_param('nat_dict')
-            nat_obj  = CNatConfig(nat_dict)
-            self.router.config_nat(nat_obj)
-            self.router.config_nat_verify()
-            self.router.config_zbf()
-
-            print 'Run until finish, expect errors'
-            trex_res = self.trex.sample_to_run_finish()
-
-            self.router.config_no_zbf()
-            self.router.clear_nat_translations()
-            print ("\nLATEST RESULT OBJECT:")
-            print trex_res
-            nat_stats = self.router.get_nat_stats()
-            print nat_stats
-            self.check_general_scenario_results(trex_res)
-            self.check_CPU_benchmark(trex_res, 10)
-            self.check_rx_errors(trex_res)
+        print 'Run until finish, expect errors'
+        old_errors = copy.deepcopy(self.fail_reasons)
+        nat_dict = self.get_benchmark_param('nat_dict', test_name = 'test_nat_simple')
+        nat_obj  = CNatConfig(nat_dict)
+        self.router.config_nat(nat_obj)
+        self.router.config_zbf()
+        trex_res = self.trex.sample_to_run_finish()
+        self.router.config_no_nat(nat_obj)
+        self.router.config_no_zbf()
+        print ("\nLATEST RESULT OBJECT:")
+        print trex_res
+        self.check_rx_errors(trex_res)
+        if self.fail_reasons == old_errors:
             self.fail('Expected errors here, got none.')
-        except Exception as e:
-            print 'Got errors as expected: %s' % e
-            pass
+        else:
+            print 'Got errors as expected.'
+            self.fail_reasons = old_errors
 
     def tearDown(self):
         CTRexGeneral_Test.tearDown(self)
diff --git a/scripts/automation/trex_control_plane/client/trex_client.py b/scripts/automation/trex_control_plane/client/trex_client.py
index 1d94dc06..9e3944d4 100755
--- a/scripts/automation/trex_control_plane/client/trex_client.py
+++ b/scripts/automation/trex_control_plane/client/trex_client.py
@@ -61,14 +61,17 @@ class CTRexClient(object):
             socket errors, in case server could not be reached.
 
         """
-        self.trex_host          = trex_host
+        try:
+            self.trex_host          = socket.gethostbyname(trex_host)
+        except: # give it another try
+            self.trex_host          = socket.gethostbyname(trex_host)
         self.trex_daemon_port   = trex_daemon_port
         self.trex_zmq_port      = trex_zmq_port
         self.seq                = None
         self.verbose            = verbose
         self.result_obj         = CTRexResult(max_history_size)
         self.decoder            = JSONDecoder()
-        self.trex_server_path   = "http://{hostname}:{port}/".format( hostname = trex_host, port = trex_daemon_port )
+        self.trex_server_path   = "http://{hostname}:{port}/".format( hostname = self.trex_host, port = trex_daemon_port )
         self.__verbose_print("Connecting to TRex @ {trex_path} ...".format( trex_path = self.trex_server_path ) )
         self.history            = jsonrpclib.history.History()
         self.server             = jsonrpclib.Server(self.trex_server_path, history = self.history)