Merge trex-204

5 files changed, 34 insertions, 89 deletions

diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py
index 1a44970a..42720f70 100755
--- a/scripts/automation/regression/stateful_tests/trex_general_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_general_test.py
@@ -48,6 +48,7 @@ def tearDownModule(module):
 class CTRexGeneral_Test(unittest.TestCase):
     """This class defines the general stateful testcase of the T-Rex traffic generator"""
     def __init__ (self, *args, **kwargs):
+        sys.stdout.flush()
         unittest.TestCase.__init__(self, *args, **kwargs)
         if CTRexScenario.is_test_list:
             return
@@ -57,6 +58,7 @@ class CTRexGeneral_Test(unittest.TestCase):
         self.trex                  = CTRexScenario.trex
         self.trex_crashed          = CTRexScenario.trex_crashed
         self.modes                 = CTRexScenario.modes
+        self.GAManager             = CTRexScenario.GAManager
         self.skipping              = False
         self.fail_reasons          = []
         if not hasattr(self, 'unsupported_modes'):
@@ -135,15 +137,12 @@ class CTRexGeneral_Test(unittest.TestCase):
         if res[name] != float(val):
             self.fail('TRex results[%s]==%f and not as expected %f ' % (name, res[name], val))
 
-    def check_CPU_benchmark (self, trex_res, err = 10, minimal_cpu = None, maximal_cpu = 85):
+    def check_CPU_benchmark (self, trex_res, err = 25, minimal_cpu = 30, maximal_cpu = 85):
             #cpu_util = float(trex_res.get_last_value("trex-global.data.m_cpu_util"))
-            cpu_util = sum([float(x) for x in trex_res.get_value_list("trex-global.data.m_cpu_util")[-4:-1]]) / 3 # mean of 3 values before last
+            cpu_util = sum(trex_res.get_value_list("trex-global.data.m_cpu_util")[-4:-1]) / 3.0 # mean of 3 values before last
 
-            if minimal_cpu is None:
-                if '1G' in self.modes:
-                    minimal_cpu = 1
-                else:
-                    minimal_cpu = 30
+            if '1G' in self.modes:
+                minimal_cpu /= 10.0
 
             if not self.is_virt_nics:
                 if cpu_util > maximal_cpu:
@@ -151,23 +150,25 @@ class CTRexGeneral_Test(unittest.TestCase):
                 if cpu_util < minimal_cpu:
                     self.fail("CPU is too low (%s%%), can't verify performance in such low CPU%%." % cpu_util )
 
-            cores = self.get_benchmark_param('cores')
-            trex_tx_bps  = trex_res.get_last_value("trex-global.data.m_total_tx_bytes")
-            test_norm_cpu = 100.0*(trex_tx_bps/(cores*cpu_util))/1e6
+            test_norm_cpu  = sum(trex_res.get_value_list("trex-global.data.m_bw_per_core")[-4:-1]) / 3.0
 
-            print("TRex CPU utilization: %g%%, norm_cpu is : %d Mb/core" % (round(cpu_util), int(test_norm_cpu)))
+            print("TRex CPU utilization: %g%%, norm_cpu is : %g Gb/core" % (round(cpu_util, 2), round(test_norm_cpu)))
 
-            #expected_norm_cpu = self.get_benchmark_param('cpu_to_core_ratio')
+            expected_norm_cpu = self.get_benchmark_param('bw_per_core')
+            if not expected_norm_cpu:
+                expected_norm_cpu = 1
 
-            #calc_error_precent = abs(100.0*(test_norm_cpu/expected_norm_cpu)-100.0) 
-
-#           if calc_error_precent > err:
-#               msg ='Normalized bandwidth to CPU utilization ratio is %2.0f Mb/core expected %2.0f Mb/core more than %2.0f %% - ERROR' % (test_norm_cpu, expected_norm_cpu, err)
-#               raise AbnormalResultError(msg)
-#           else:
-#               msg ='Normalized bandwidth to CPU utilization ratio is %2.0f Mb/core expected %2.0f Mb/core less than %2.0f %% - OK' % (test_norm_cpu, expected_norm_cpu, err)
-#               print msg
+            calc_error_precent = abs(100.0 * test_norm_cpu / expected_norm_cpu - 100)
+            print('Err percent: %s' % calc_error_precent)
+            if calc_error_precent > err and cpu_util > 10:
+                self.fail('Excepted bw_per_core ratio: %s, got: %g' % (expected_norm_cpu, round(test_norm_cpu)))
 
+            # report benchmarks
+            if self.GAManager:
+                setup_test = '%s.%s' % (CTRexScenario.setup_name, self.get_name())
+                self.GAManager.gaAddAction(Event = 'stateful_test', action = setup_test, label = 'bw_per_core', value = int(test_norm_cpu))
+                self.GAManager.gaAddAction(Event = 'stateful_test', action = setup_test, label = 'bw_per_core_exp', value = int(expected_norm_cpu))
+                self.GAManager.emptyAndReportQ()
 
     def check_results_gt (self, res, name, val):
         if res is None:
@@ -341,7 +342,9 @@ class CTRexGeneral_Test(unittest.TestCase):
                 except Exception as e:
                     print("Can't get TRex log:", e)
             if len(self.fail_reasons):
+                sys.stdout.flush()
                 raise Exception('The test is failed, reasons:\n%s' % '\n'.join(self.fail_reasons))
+        sys.stdout.flush()
 
     def check_for_trex_crash(self):
         pass
diff --git a/scripts/automation/regression/stateful_tests/trex_imix_test.py b/scripts/automation/regression/stateful_tests/trex_imix_test.py
index 95a5471d..cafa1c55 100755
--- a/scripts/automation/regression/stateful_tests/trex_imix_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_imix_test.py
@@ -161,7 +161,7 @@ class CTRexIMIX_Test(CTRexGeneral_Test):
 
         self.check_general_scenario_results(trex_res)
 
-        self.check_CPU_benchmark(trex_res)
+        self.check_CPU_benchmark(trex_res, minimal_cpu = 25)
 
 
     def test_jumbo(self, duration = 100, **kwargs):
diff --git a/scripts/automation/regression/stateful_tests/trex_nat_test.py b/scripts/automation/regression/stateful_tests/trex_nat_test.py
index 512ad4e4..b2df684d 100755
--- a/scripts/automation/regression/stateful_tests/trex_nat_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_nat_test.py
@@ -54,9 +54,9 @@ class CTRexNoNat_Test(CTRexGeneral_Test):#(unittest.TestCase):
         learning_stats = trex_res.get_last_value("trex-global.data", ".*nat.*") # extract all nat data
 
         if self.get_benchmark_param('allow_timeout_dev'):
-            nat_timeout_ratio = learning_stats['m_total_nat_time_out']/learning_stats['m_total_nat_open']
+            nat_timeout_ratio = float(learning_stats['m_total_nat_time_out']) / learning_stats['m_total_nat_open']
             if nat_timeout_ratio > 0.005:
-                self.fail('TRex nat_timeout ratio %f > 0.005 (0.5%) and not as expected to be less than 0.5%' %(nat_timeout_ratio))
+                self.fail('TRex nat_timeout ratio %f > 0.5%%' % nat_timeout_ratio)
         else:
             self.check_results_eq (learning_stats, 'm_total_nat_time_out', 0.0)
         self.check_results_eq (learning_stats, 'm_total_nat_no_fid', 0.0)
@@ -128,7 +128,7 @@ class CTRexNat_Test(CTRexGeneral_Test):#(unittest.TestCase):
 
         trex_nat_stats = trex_res.get_last_value("trex-global.data", ".*nat.*") # extract all nat data
         if self.get_benchmark_param('allow_timeout_dev'):
-            nat_timeout_ratio = trex_nat_stats['m_total_nat_time_out']/trex_nat_stats['m_total_nat_open']
+            nat_timeout_ratio = float(trex_nat_stats['m_total_nat_time_out']) / trex_nat_stats['m_total_nat_open']
             if nat_timeout_ratio > 0.005:
                 self.fail('TRex nat_timeout ratio %f > 0.5%%' % nat_timeout_ratio)
         else:
diff --git a/scripts/automation/regression/stateful_tests/trex_nbar_test.py b/scripts/automation/regression/stateful_tests/trex_nbar_test.py
index 69c3f605..fa3f3485 100755
--- a/scripts/automation/regression/stateful_tests/trex_nbar_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_nbar_test.py
@@ -82,70 +82,10 @@ class CTRexNbar_Test(CTRexGeneral_Test):
         print("\nLATEST DUMP:")
         print(trex_res.get_latest_dump())
 
-
         self.check_general_scenario_results(trex_res, check_latency = False) # NBAR can cause latency
-        #       test_norm_cpu = 2*(trex_res.result['total-tx']/(core*trex_res.result['cpu_utilization']))
-        trex_tx_pckt  = trex_res.get_last_value("trex-global.data.m_total_tx_pkts")
-        cpu_util = trex_res.get_last_value("trex-global.data.m_cpu_util")
-        cpu_util_hist = trex_res.get_value_list("trex-global.data.m_cpu_util")
-        print("cpu util is:", cpu_util)
-        print(cpu_util_hist)
-        test_norm_cpu = 2 * trex_tx_pckt / (core * cpu_util)
-        print("test_norm_cpu is:", test_norm_cpu)
-
-        
-        if self.get_benchmark_param('cpu2core_custom_dev'):
-            # check this test by custom deviation
-            deviation_compare_value = self.get_benchmark_param('cpu2core_dev')
-            print("Comparing test with custom deviation value- {dev_val}%".format( dev_val = int(deviation_compare_value*100) ))
-
-        # need to be fixed !
-        #if ( abs((test_norm_cpu/self.get_benchmark_param('cpu_to_core_ratio')) - 1) > deviation_compare_value):
-        #    raise AbnormalResultError('Normalized bandwidth to CPU utilization ratio exceeds benchmark boundaries')
-
+        self.check_CPU_benchmark(trex_res)
         self.match_classification()
 
-        assert True
-
-    @nottest
-    def test_rx_check (self):
-        # test initializtion
-        self.router.configure_basic_interfaces()
-
-        self.router.config_pbr(mode = "config")
-        self.router.config_nbar_pd()
-
-        mult = self.get_benchmark_param('multiplier')
-        core = self.get_benchmark_param('cores')
-        sample_rate = self.get_benchmark_param('rx_sample_rate')
-
-        ret = self.trex.start_trex(
-            c = core,
-            m = mult,
-            p  = True,
-            nc = True,
-            rx_check = sample_rate,
-            d = 100,   
-            f = 'cap2/sfr.yaml',
-            l = 1000)
-
-        trex_res = self.trex.sample_to_run_finish()
-
-        # trex_res is a CTRexResult instance- and contains the summary of the test results
-        # you may see all the results keys by simply calling here for 'print trex_res.result'
-        print("\nLATEST RESULT OBJECT:")
-        print(trex_res)
-        print("\nLATEST DUMP:")
-        print(trex_res.get_latest_dump())
-
-        self.check_general_scenario_results(trex_res)
-
-        self.check_CPU_benchmark(trex_res, 10)
-
-#       if trex_res.result['rx_check_tx']==trex_res.result['rx_check_rx']:  # rx_check verification shoud pass
-#           assert trex_res.result['rx_check_verification'] == "OK"
-#       else:
-#           assert trex_res.result['rx_check_verification'] == "FAIL"
 
     # the name intentionally not matches nose default pattern, including the test should be specified explicitly
     def NBarLong(self):
diff --git a/scripts/automation/regression/stateful_tests/trex_rx_test.py b/scripts/automation/regression/stateful_tests/trex_rx_test.py
index 2f0a24f4..c097b180 100755
--- a/scripts/automation/regression/stateful_tests/trex_rx_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_rx_test.py
@@ -52,9 +52,10 @@ class CTRexRx_Test(CTRexGeneral_Test):
 
                 path = 'rx-check.data.stats.m_total_rx'
                 total_rx = trex_res.get_last_value(path)
-                if not total_rx:
+                if total_rx is None:
                     raise AbnormalResultError('No TRex results by path: %s' % path)
-
+                elif not total_rx:
+                    raise AbnormalResultError('Total rx_check (%s) packets is zero.' % path)
 
                 print('Total packets checked: %s' % total_rx)
                 print('Latency counters: %s' % latency_counters_display)
@@ -69,7 +70,7 @@ class CTRexRx_Test(CTRexGeneral_Test):
             error_tolerance = self.get_benchmark_param('error_tolerance')
             if not error_tolerance or not allow_error_tolerance:
                 error_tolerance = 0
-            error_percentage = float(total_errors) * 100 / total_rx
+            error_percentage = total_errors * 100.0 / total_rx
 
             if total_errors > 0:
                 if self.is_loopback or error_percentage > error_tolerance:
@@ -255,7 +256,8 @@ class CTRexRx_Test(CTRexGeneral_Test):
         self.router.config_zbf()
         trex_res = self.trex.sample_to_run_finish()
         self.router.config_no_zbf()
-        self.router.clear_nat_translations()
+        self.router.config_no_nat(nat_obj)
+        #self.router.clear_nat_translations()
         print("\nLATEST RESULT OBJECT:")
         print(trex_res)
         self.check_rx_errors(trex_res, allow_error_tolerance = False)