Merge branch 'master' into rx_phase_2

7 files changed, 354 insertions, 22 deletions

diff --git a/scripts/automation/regression/setups/trex09/benchmark.yaml b/scripts/automation/regression/setups/trex09/benchmark.yaml
index 1d2b57c8..9fece63f 100644
--- a/scripts/automation/regression/setups/trex09/benchmark.yaml
+++ b/scripts/automation/regression/setups/trex09/benchmark.yaml
@@ -221,7 +221,7 @@ test_performance_vm_multi_cpus_cached:
         core_count             : 2
         mult                   : "90%"
         mpps_per_core_golden   :
-                                  min: 28.8
+                                  min: 26.8
                                   max: 29.5
 
 test_performance_syn_attack_multi_cpus:
diff --git a/scripts/automation/regression/setups/trex11/benchmark.yaml b/scripts/automation/regression/setups/trex11/benchmark.yaml
index a4969d2d..5ebcdd55 100644
--- a/scripts/automation/regression/setups/trex11/benchmark.yaml
+++ b/scripts/automation/regression/setups/trex11/benchmark.yaml
@@ -40,6 +40,19 @@ test_rx_check_http: &rx_http
     rx_sample_rate      : 128
     bw_per_core         : 49.464
 
+test_rx_check_http_ipv6:
+    <<                  : *rx_http
+    bw_per_core         : 49.237
+
+test_rx_check_sfr: &rx_sfr
+    multiplier          : 8
+    cores               : 1
+    rx_sample_rate      : 128
+    bw_per_core         : 20.9
+
+test_rx_check_sfr_ipv6:
+    <<                  : *rx_sfr
+    bw_per_core         : 23.9
 
 
 ### stateless ###
diff --git a/scripts/automation/regression/setups/trex14/BU/benchmark.yaml b/scripts/automation/regression/setups/trex14/BU/benchmark.yaml
new file mode 100644
index 00000000..04f13e79
--- /dev/null
+++ b/scripts/automation/regression/setups/trex14/BU/benchmark.yaml
@@ -0,0 +1,245 @@
+###############################################################
+####           TRex benchmark configuration file           ####
+###############################################################
+
+#### common templates ###
+
+stat_route_dict: &stat_route_dict
+    clients_start           : 16.0.0.1
+    servers_start           : 48.0.0.1
+    dual_port_mask          : 1.0.0.0
+    client_destination_mask : 255.0.0.0
+    server_destination_mask : 255.0.0.0
+
+nat_dict: &nat_dict
+    clients_net_start        : 16.0.0.0
+    client_acl_wildcard_mask : 0.0.0.255
+    dual_port_mask           : 1.0.0.0
+    pool_start               : 200.0.0.0
+    pool_netmask             : 255.255.255.0
+
+
+### stateful ###
+
+test_jumbo:
+    multiplier          : 17
+    cores               : 1
+    bw_per_core         : 543.232
+
+
+test_routing_imix:
+    multiplier          : 10
+    cores               : 1
+    bw_per_core         : 34.128
+
+
+test_routing_imix_64:
+    multiplier          : 430
+    cores               : 1
+    bw_per_core         : 5.893
+
+
+test_static_routing_imix: &test_static_routing_imix
+    stat_route_dict     : *stat_route_dict
+    multiplier          : 8
+    cores               : 1
+    bw_per_core         : 34.339
+
+test_static_routing_imix_asymmetric: *test_static_routing_imix
+
+
+test_ipv6_simple:
+    multiplier          : 9
+    cores               : 2
+    bw_per_core         : 19.064
+
+
+test_nat_simple_mode1: &test_nat_simple
+    stat_route_dict     : *stat_route_dict
+    nat_dict            : *nat_dict
+    multiplier          : 6000
+    cores               : 1
+    nat_opened          : 500000
+    allow_timeout_dev   : True
+    bw_per_core         : 44.445
+
+test_nat_simple_mode2: *test_nat_simple
+
+test_nat_simple_mode3: *test_nat_simple
+
+test_nat_learning: *test_nat_simple
+
+
+test_nbar_simple:
+    multiplier          : 7.5
+    cores               : 2
+    bw_per_core         : 17.174
+    nbar_classification:
+        http            : 32.58
+        rtp-audio       : 21.21
+        oracle_sqlnet   : 11.41
+        exchange        : 11.22
+        rtp             : 11.2
+        citrix          : 5.65
+        rtsp            : 2.87
+        dns             : 1.96
+        smtp            : 0.57
+        pop3            : 0.37
+        ssl             : 0.28
+        sctp            : 0.13
+        sip             : 0.09
+        unknown         : 0.45
+
+
+test_rx_check_http: &rx_http
+    multiplier          : 15000
+    cores               : 1
+    rx_sample_rate      : 16
+    bw_per_core         : 39.560
+
+test_rx_check_http_ipv6:
+    <<                  : *rx_http
+    bw_per_core         : 49.237
+
+test_rx_check_http_negative_disabled:
+    <<                  : *rx_http
+    stat_route_dict     : *stat_route_dict
+    nat_dict            : *nat_dict
+
+
+test_rx_check_sfr: &rx_sfr
+    multiplier          : 10
+    cores               : 3
+    rx_sample_rate      : 16
+    bw_per_core         : 16.082
+
+test_rx_check_sfr_ipv6:
+    <<                  : *rx_sfr
+    bw_per_core         : 19.198
+
+
+
+### stateless ###
+
+test_CPU_benchmark:
+    profiles:
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 64}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 64, stream_count: 10}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 64, stream_count: 100}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+# causes queue full
+#      - name            : stl/udp_for_benchmarks.py
+#        kwargs          : {packet_len: 64, stream_count: 1000}
+#        cpu_util        : 1
+#        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 128}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 256}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 512}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 1500}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 4000}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 9000}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 9000, stream_count: 10}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_for_benchmarks.py
+        kwargs          : {packet_len: 9000, stream_count: 100}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+# not enough memory + queue full if memory increase
+#      - name            : stl/udp_for_benchmarks.py
+#        kwargs          : {packet_len: 9000, stream_count: 1000}
+#        cpu_util        : 1
+#        bw_per_core     : 1
+
+      - name            : stl/imix.py
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 64}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 128}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 256}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 512}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 1500}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 4000}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_1pkt_tuple_gen.py
+        kwargs          : {packet_len: 9000}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/pcap.py
+        kwargs          : {ipg_usec: 2, loop_count: 0}
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/udp_rand_len_9k.py
+        cpu_util        : 1
+        bw_per_core     : 1
+
+      - name            : stl/hlt/hlt_udp_rand_len_9k.py
+        cpu_util        : 1
+        bw_per_core     : 1
+
+
diff --git a/scripts/automation/regression/setups/trex14/BU/config.yaml b/scripts/automation/regression/setups/trex14/BU/config.yaml
new file mode 100644
index 00000000..0fd6b70e
--- /dev/null
+++ b/scripts/automation/regression/setups/trex14/BU/config.yaml
@@ -0,0 +1,67 @@
+################################################################
+####         TRex nightly test configuration file          ####
+################################################################
+
+
+### TRex configuration:
+# hostname        - can be DNS name or IP for the TRex machine for ssh to the box
+# password        - root password for TRex machine
+# is_dual         - should the TRex inject with -p ?
+# version_path    - path to the TRex version and executable
+# cores           - how many cores should be used
+# latency         - rate of latency packets injected by the TRex
+# modes           - list of modes (tagging) of this setup (loopback etc.)
+# 	  * loopback  - Trex works via loopback. Router and TFTP configurations may be skipped.
+#     * VM        - Virtual OS (accept low CPU utilization in tests, latency can get spikes)
+#     * virt_nics - NICs are virtual (VMXNET3 etc.)
+
+### Router configuration:
+# hostname       - the router hostname as apears in ______# cli prefix
+# ip_address     - the router's ip that can be used to communicate with
+# image          - the desired imaged wished to be loaded as the router's running config
+# line_password  - router password when access via Telent
+# en_password    - router password when changing to "enable" mode
+# interfaces     - an array of client-server pairs, representing the interfaces configurations of the router
+# configurations - an array of configurations that could possibly loaded into the router during the test. 
+#                  The "clean" configuration is a mandatory configuration the router will load with to run the basic test bench
+
+### TFTP configuration:
+# hostname       - the tftp hostname
+# ip_address     - the tftp's ip address
+# images_path    - the tftp's relative path in which the router's images are located
+
+### Test_misc configuration:
+# expected_bw    - the "golden" bandwidth (in Gbps) results planned on receiving from the test
+
+trex:
+  hostname       : csi-trex-14
+  cores          : 4
+  modes          : []
+
+router:
+  model          : ASR1001x
+  hostname       : csi-asr-01
+  ip_address     : 10.56.216.103
+  image          : asr1001x-universalk9.03.17.00.S.156-1.S-std.SPA.bin
+  line_password  : cisco
+  en_password    : cisco
+  mgmt_interface : GigabitEthernet0
+  clean_config   : /Configurations/danklei/asr1001_TRex_clean_config.cfg
+  intf_masking   : 255.255.255.0
+  ipv6_mask      : 64
+  interfaces     :
+    - client : 
+        name          : Te0/0/0
+        src_mac_addr  : 0000.0001.0000
+        dest_mac_addr : 0000.0001.0000
+      server : 
+        name          : Te0/0/1
+        src_mac_addr  : 0000.0001.0000
+        dest_mac_addr : 0000.0001.0000
+      vrf_name      : null    
+
+tftp:
+  hostname       : ats-asr-srv-1
+  ip_address     : 10.56.217.7
+  root_dir       : /scratch/tftp/
+  images_path    : /asr1001x/
diff --git a/scripts/automation/regression/setups/trex14/benchmark.yaml b/scripts/automation/regression/setups/trex14/benchmark.yaml
index 04f13e79..0dc340b0 100644
--- a/scripts/automation/regression/setups/trex14/benchmark.yaml
+++ b/scripts/automation/regression/setups/trex14/benchmark.yaml
@@ -75,20 +75,19 @@ test_nbar_simple:
     cores               : 2
     bw_per_core         : 17.174
     nbar_classification:
-        http            : 32.58
-        rtp-audio       : 21.21
-        oracle_sqlnet   : 11.41
-        exchange        : 11.22
-        rtp             : 11.2
-        citrix          : 5.65
-        rtsp            : 2.87
-        dns             : 1.96
+        rtp             : 32.57
+        http            : 30.25
+        oracle_sqlnet   : 11.23
+        exchange        : 10.80
+        citrix          : 5.62
+        rtsp            : 2.84
+        dns             : 1.95
         smtp            : 0.57
-        pop3            : 0.37
-        ssl             : 0.28
+        pop3            : 0.36
+        ssl             : 0.17
         sctp            : 0.13
         sip             : 0.09
-        unknown         : 0.45
+        unknown         : 3.41
 
 
 test_rx_check_http: &rx_http
diff --git a/scripts/automation/regression/setups/trex14/config.yaml b/scripts/automation/regression/setups/trex14/config.yaml
index 0fd6b70e..ffb61763 100644
--- a/scripts/automation/regression/setups/trex14/config.yaml
+++ b/scripts/automation/regression/setups/trex14/config.yaml
@@ -36,28 +36,27 @@
 trex:
   hostname       : csi-trex-14
   cores          : 4
-  modes          : []
 
 router:
   model          : ASR1001x
   hostname       : csi-asr-01
-  ip_address     : 10.56.216.103
-  image          : asr1001x-universalk9.03.17.00.S.156-1.S-std.SPA.bin
+  ip_address     : 10.56.216.120
+  image          : asr1001x-universalk9.03.13.02.S.154-3.S2-ext.SPA.bin
   line_password  : cisco
   en_password    : cisco
   mgmt_interface : GigabitEthernet0
-  clean_config   : /Configurations/danklei/asr1001_TRex_clean_config.cfg
+  clean_config   : clean_config.cfg
   intf_masking   : 255.255.255.0
   ipv6_mask      : 64
   interfaces     :
     - client : 
         name          : Te0/0/0
-        src_mac_addr  : 0000.0001.0000
-        dest_mac_addr : 0000.0001.0000
+        src_mac_addr  : 0000.0001.0002
+        dest_mac_addr : 0000.0001.0001
       server : 
         name          : Te0/0/1
-        src_mac_addr  : 0000.0001.0000
-        dest_mac_addr : 0000.0001.0000
+        src_mac_addr  : 0000.0002.0002
+        dest_mac_addr : 0000.0002.0001
       vrf_name      : null    
 
 tftp:
diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py
index 1843af00..fe38ed34 100755
--- a/scripts/automation/regression/stateful_tests/trex_general_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_general_test.py
@@ -257,7 +257,7 @@ class CTRexGeneral_Test(unittest.TestCase):
                     allowed_latency = 1000
                 if max(trex_res.get_max_latency().values()) > allowed_latency:
                     self.fail('LatencyError: Maximal latency exceeds %s (usec)' % allowed_latency)
-    
+
                 # check that avg latency does not exceed 1 msec
                 if self.is_VM:
                     allowed_latency = 9999999
@@ -266,6 +266,15 @@ class CTRexGeneral_Test(unittest.TestCase):
                 if max(trex_res.get_avg_latency().values()) > allowed_latency:
                     self.fail('LatencyError: Average latency exceeds %s (usec)' % allowed_latency)
 
+                ports_names = trex_res.get_last_value('trex-latecny-v2.data', 'port\-\d+')
+                if not ports_names:
+                    raise AbnormalResultError('Could not find ports info in TRex results, path: trex-latecny-v2.data.port-*')
+                for port_name in ports_names:
+                    path = 'trex-latecny-v2.data.%s.hist.cnt' % port_name
+                    lat_count = trex_res.get_last_value(path)
+                    if lat_count == 0:
+                        self.fail('LatencyError: Number of latency packets received on %s is 0' % port_name)
+
             if not self.is_loopback:
                 # check router number of drops --> deliberately masked- need to be figured out!!!!!
                 pkt_drop_stats = self.router.get_drop_stats()
@@ -359,7 +368,7 @@ class CTRexGeneral_Test(unittest.TestCase):
                     print("Can't get TRex log:", e)
             if len(self.fail_reasons):
                 sys.stdout.flush()
-                raise Exception('The test is failed, reasons:\n%s' % '\n'.join(self.fail_reasons))
+                raise Exception('Test failed. Reasons:\n%s' % '\n'.join(self.fail_reasons))
         sys.stdout.flush()
 
     def check_for_trex_crash(self):