Merge branch 'master' into capture

8 files changed, 246 insertions, 18 deletions

diff --git a/scripts/automation/regression/setups/kiwi02/benchmark.yaml b/scripts/automation/regression/setups/kiwi02/benchmark.yaml
index 1eefccaf..b6055366 100644
--- a/scripts/automation/regression/setups/kiwi02/benchmark.yaml
+++ b/scripts/automation/regression/setups/kiwi02/benchmark.yaml
@@ -272,7 +272,7 @@ test_performance_vm_single_cpu:
     cfg:
         mult                    : "90%"
         mpps_per_core_golden    : 
-                                   min: 11.2
+                                   min: 10.0
                                    max: 13.1
 
 
@@ -280,7 +280,7 @@ test_performance_vm_single_cpu_cached:
     cfg:
         mult                    : "90%"
         mpps_per_core_golden    : 
-                                   min: 20.5
+                                   min: 18.0
                                    max: 25.0
 
 
@@ -289,7 +289,7 @@ test_performance_syn_attack_single_cpu:
      cfg:
         mult                    : "90%"
         mpps_per_core_golden    :
-                                  min: 9.3
+                                  min: 8.0
                                   max: 11.5
 
 test_performance_vm_multi_cpus:
@@ -297,7 +297,7 @@ test_performance_vm_multi_cpus:
         core_count             : 4
         mult                   : "90%"
         mpps_per_core_golden   :
-                                  min: 9.7
+                                  min: 8.0
                                   max: 12.5
 
 
@@ -306,7 +306,7 @@ test_performance_vm_multi_cpus_cached:
         core_count             : 4
         mult                   : "90%"
         mpps_per_core_golden   :
-                                  min: 19.0
+                                  min: 18.0
                                   max: 22.0
 
 test_performance_syn_attack_multi_cpus:
@@ -314,6 +314,6 @@ test_performance_syn_attack_multi_cpus:
         core_count             : 4
         mult                   : "90%"
         mpps_per_core_golden   :
-                                  min: 8.4
+                                  min: 8.0
                                   max: 10.5
 
diff --git a/scripts/automation/regression/setups/trex21/benchmark.yaml b/scripts/automation/regression/setups/trex21/benchmark.yaml
new file mode 100644
index 00000000..b366b3fb
--- /dev/null
+++ b/scripts/automation/regression/setups/trex21/benchmark.yaml
@@ -0,0 +1,155 @@
+################################################################
+####           TRex benchmark configuration file           ####
+################################################################
+
+### stateful ###
+
+test_jumbo:
+    multiplier          : 2.8
+    cores               : 1
+    bw_per_core         : 106.652
+
+
+test_routing_imix:
+    multiplier          : 0.5
+    cores               : 1
+    bw_per_core         : 11.577
+
+
+test_routing_imix_64:
+    multiplier          : 28
+    cores               : 1
+    bw_per_core         : 2.030
+
+
+test_static_routing_imix_asymmetric:
+    multiplier          : 0.8
+    cores               : 1
+    bw_per_core         : 13.742
+
+
+
+### stateless ###
+
+test_CPU_benchmark:
+    profiles:
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 64}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 64, stream_count: 10}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 64, stream_count: 100}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+# causes queue full
+#      - name            : stl/udp_for_benchmarks.py
+#        kwargs          : {packet_len: 64, stream_count: 1000}
+#        cpu_util        : 1
+#        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 128}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 256}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 512}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 1500}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 4000}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 9000}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 9000, stream_count: 10}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 9000, stream_count: 100}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+# not enough memory + queue full if memory increase
+#      - name            : stl/udp_for_benchmarks.py
+#        kwargs          : {packet_len: 9000, stream_count: 1000}
+#        cpu_util        : 1
+#        bw_per_core     : 1
+
+      - name            : stl/imix.py
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 64}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 128}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 256}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 512}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 1500}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 4000}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 9000}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/pcap.py
+        kwargs          : {ipg_usec: 4, loop_count: 0}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_rand_len_9k.py
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/hlt/hlt_udp_rand_len_9k.py
+        cpu_util        : 1
+        bw_per_core     : 1
+
+
diff --git a/scripts/automation/regression/setups/trex21/config.yaml b/scripts/automation/regression/setups/trex21/config.yaml
new file mode 100644
index 00000000..3cf135c3
--- /dev/null
+++ b/scripts/automation/regression/setups/trex21/config.yaml
@@ -0,0 +1,40 @@
+################################################################
+####         TRex nightly test configuration file          ####
+################################################################
+
+
+### TRex configuration:
+# hostname        - can be DNS name or IP for the TRex machine for ssh to the box
+# password        - root password for TRex machine
+# is_dual         - should the TRex inject with -p ?
+# version_path    - path to the TRex version and executable
+# cores           - how many cores should be used
+# latency         - rate of latency packets injected by the TRex
+# modes           - list of modes (tagging) of this setup (loopback etc.)
+# 	  * loopback  - Trex works via loopback. Router and TFTP configurations may be skipped.
+#     * VM        - Virtual OS (accept low CPU utilization in tests, latency can get spikes)
+#     * virt_nics - NICs are virtual (VMXNET3 etc.)
+
+### Router configuration:
+# hostname       - the router hostname as apears in ______# cli prefix
+# ip_address     - the router's ip that can be used to communicate with
+# image          - the desired imaged wished to be loaded as the router's running config
+# line_password  - router password when access via Telent
+# en_password    - router password when changing to "enable" mode
+# interfaces     - an array of client-server pairs, representing the interfaces configurations of the router
+# configurations - an array of configurations that could possibly loaded into the router during the test. 
+#                  The "clean" configuration is a mandatory configuration the router will load with to run the basic test bench
+
+### TFTP configuration:
+# hostname       - the tftp hostname
+# ip_address     - the tftp's ip address
+# images_path    - the tftp's relative path in which the router's images are located
+
+### Test_misc configuration:
+# expected_bw    - the "golden" bandwidth (in Gbps) results planned on receiving from the test
+
+trex:
+  hostname       : csi-trex-21
+  cores          : 1
+  modes          : ['loopback', 'virt_nics']
+
diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py
index 4453fd94..dc3f9943 100755
--- a/scripts/automation/regression/stateful_tests/trex_general_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_general_test.py
@@ -313,7 +313,7 @@ class CTRexGeneral_Test(unittest.TestCase):
                 # check that max latency does not exceed 1 msec
                 if self.configuration.trex['trex_name'] == '10.56.217.210': # temporary workaround for latency issue in kiwi02, remove it ASAP. http://trex-tgn.cisco.com/youtrack/issue/trex-194
                     allowed_latency = 8000
-                elif self.is_VM:
+                elif self.is_VM or self.is_virt_nics:
                     allowed_latency = 9999999
                 else: # no excuses, check 1ms
                     allowed_latency = 1000
@@ -321,7 +321,7 @@ class CTRexGeneral_Test(unittest.TestCase):
                     self.fail('LatencyError: Maximal latency exceeds %s (usec)' % allowed_latency)
 
                 # check that avg latency does not exceed 1 msec
-                if self.is_VM:
+                if self.is_VM or self.is_virt_nics:
                     allowed_latency = 9999999
                 else: # no excuses, check 1ms
                     allowed_latency = 1000
diff --git a/scripts/automation/regression/stateful_tests/trex_imix_test.py b/scripts/automation/regression/stateful_tests/trex_imix_test.py
index 5f52fab7..4cb01db3 100755
--- a/scripts/automation/regression/stateful_tests/trex_imix_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_imix_test.py
@@ -141,6 +141,8 @@ class CTRexIMIX_Test(CTRexGeneral_Test):
         self.check_general_scenario_results(trex_res)
         self.check_CPU_benchmark(trex_res)
 
+
+
     # the name intentionally not matches nose default pattern, including the test should be specified explicitly
     def dummy(self):
         ret = self.trex.start_trex(
diff --git a/scripts/automation/regression/stateless_tests/stl_rx_test.py b/scripts/automation/regression/stateless_tests/stl_rx_test.py
index 8812ac48..3d4ed977 100644
--- a/scripts/automation/regression/stateless_tests/stl_rx_test.py
+++ b/scripts/automation/regression/stateless_tests/stl_rx_test.py
@@ -32,6 +32,12 @@ class STLRX_Test(CStlGeneral_Test):
                         'latency_9k_max_average': 100,
                         'latency_9k_max_latency': 250,
                         },
+                'rte_i40evf_pmd': {
+                        'rate_percent': 80,
+                        'total_pkts': 1000,
+                        'rate_latency': 1,
+                        'latency_9k_enable': False,
+                        },
                 'rte_igb_pmd': {
                         'rate_percent': 80,
                         'total_pkts': 500,
@@ -526,19 +532,37 @@ class STLRX_Test(CStlGeneral_Test):
         except STLError as e:
             assert False , '{0}'.format(e)
 
+    def _run_fcs_stream (self,is_vm):
+        """ this test send 1 64 byte packet with latency and check that all counters are reported as 64 bytes"""
+        res=True
+        try:
+            all_ports=list(CTRexScenario.stl_ports_map['map'].keys());
+            for port in all_ports:
+                for l in [True,False]:
+                    print(" test port {0} latency : {1} ".format(port,l))
+                    self.send_1_burst(port,l,100)
+        except Exception as e:
+            if is_vm :
+                res=False
+            else:
+                raise e
+        return(res);
+
+
 
 
     def test_fcs_stream(self):
         """ this test send 1 64 byte packet with latency and check that all counters are reported as 64 bytes"""
 
-        if self.is_virt_nics:
-            self.skip('Skip this for virtual NICs')
-
-        all_ports=list(CTRexScenario.stl_ports_map['map'].keys());
-        for port in all_ports:
-            for l in [True,False]:
-                print(" test port {0} latency : {1} ".format(port,l))
-                self.send_1_burst(port,l,100)
+        is_vm=self.is_virt_nics # in case of VM and vSwitch there are drop of packets in some cases, let retry number of times 
+                                # in this case we just want to check functionality that packet of 64 is reported as 64 in all levels 
+        retry=1
+        if is_vm:
+            retry=4
+        for i in range(0,retry):
+            if self._run_fcs_stream (is_vm):
+                break;
+            print("==> retry  %d .." %(i));
 
 
     # this test adds more and more latency streams and re-test with incremental
diff --git a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
index 4e7deb93..490e3b7a 100755
--- a/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
+++ b/scripts/automation/trex_control_plane/stf/trex_stf_lib/trex_client.py
@@ -1110,6 +1110,13 @@ class CTRexResult(object):
         self.clear_results()
         self.latency_checked = True
         self.filtered_latency_amount = filtered_latency_amount
+        self.set_warmup_default()
+
+    def set_warmup_default (self):
+        self.set_warmup(0.96)
+
+    def set_warmup (self,new_warmup_max):
+        self.warmup_max = new_warmup_max
 
     def __repr__(self):
         return ("Is valid history?       {arg}\n".format( arg = self.is_valid_hist() ) +
@@ -1414,7 +1421,7 @@ class CTRexResult(object):
             self._current_tx_rate = CTRexResult.__get_value_by_path(latest_dump, "trex-global.data", "m_tx_(?!expected_)\w+")
             if not self._done_warmup and self._expected_tx_rate is not None:
                 # check for up to 4% change between expected and actual
-                if (self._current_tx_rate['m_tx_bps'] > 0.96 * self._expected_tx_rate['m_tx_expected_bps']):
+                if (self._current_tx_rate['m_tx_bps'] > self.warmup_max * self._expected_tx_rate['m_tx_expected_bps']):
                     self._done_warmup = True
                     latest_dump['warmup_barrier'] = True
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py
index cf594948..30a699c5 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_hltapi.py
@@ -1536,7 +1536,7 @@ def get_TOS(user_kwargs, kwargs):
     if user_args & (TOS2 - TOS1) and user_args & (TOS1 - TOS2):
         raise STLError('You have mixed %s and %s TOS parameters' % (TOS1, TOS2))
     if user_args & (TOS0 - TOS1 - TOS2):
-        return (kwargs['ip_precedence'] << 5) + (kwargs['ip_tos_field'] << 2) + kwargs['ip_mbz']
+        return (kwargs['ip_precedence'] << 5) + (kwargs['ip_tos_field'] << 1) + kwargs['ip_mbz']
     if user_args & (TOS1 - TOS2):
         return (kwargs['ip_precedence'] << 5) + (kwargs['ip_delay'] << 4) + (kwargs['ip_throughput'] << 3) + (kwargs['ip_reliability'] << 2) + (kwargs['ip_cost'] << 1) + kwargs['ip_reserved']
     return (kwargs['ip_dscp'] << 2) + kwargs['ip_cu']