Merge branch 'master' into cpu_per_core

6 files changed, 179 insertions, 94 deletions

diff --git a/scripts/automation/regression/CPlatform.py b/scripts/automation/regression/CPlatform.py
index de1c22ce..256d7411 100755
--- a/scripts/automation/regression/CPlatform.py
+++ b/scripts/automation/regression/CPlatform.py
@@ -20,7 +20,7 @@ class CPlatform(object):
         self.tftp_cfg           = None
         self.config_history     = { 'basic_if_config' : False, 'tftp_server_config' : False }
 
-    def configure_basic_interfaces(self, mtu = 4000):
+    def configure_basic_interfaces(self, mtu = 9050):
 
         cache = CCommandCache()
         for dual_if in self.if_mngr.get_dual_if_list():
@@ -46,7 +46,7 @@ class CPlatform(object):
 
 
 
-    def configure_basic_filtered_interfaces(self, intf_list, mtu = 4000):
+    def configure_basic_filtered_interfaces(self, intf_list, mtu = 9050):
 
         cache = CCommandCache()
         for intf in intf_list:
diff --git a/scripts/automation/regression/stateless_tests/stl_rx_test.py b/scripts/automation/regression/stateless_tests/stl_rx_test.py
index 23ebf081..090261ff 100644
--- a/scripts/automation/regression/stateless_tests/stl_rx_test.py
+++ b/scripts/automation/regression/stateless_tests/stl_rx_test.py
@@ -9,11 +9,7 @@ class STLRX_Test(CStlGeneral_Test):
     """Tests for RX feature"""
 
     def setUp(self):
-        #if CTRexScenario.setup_name in ('trex08', 'trex09'):
-        #    self.skip('This test makes trex08 and trex09 sick. Fix those ASAP.')
-        if self.is_virt_nics:
-            self.skip('Skip this for virtual NICs for now')
-        per_driver_params = {"rte_vmxnet3_pmd": [1, 50, 1,False], "rte_ixgbe_pmd": [30, 5000, 1,True,200,400], "rte_i40e_pmd": [80, 5000, 1,True,100,250],
+        per_driver_params = {"rte_vmxnet3_pmd": [1, 50, 1,False], "rte_ixgbe_pmd": [30, 1000, 1,True,300,400], "rte_i40e_pmd": [80, 1000, 1,True,100,250],
                              "rte_igb_pmd": [80, 500, 1,False], "rte_em_pmd": [1, 50, 1,False], "rte_virtio_pmd": [1, 50, 1,False]}
 
         CStlGeneral_Test.setUp(self)
@@ -32,21 +28,38 @@ class STLRX_Test(CStlGeneral_Test):
             self.skip('port {0} does not support RX'.format(self.rx_port))
         self.cap = cap
 
-        self.rate_percent = per_driver_params[port_info['driver']][0]
-        self.total_pkts = per_driver_params[port_info['driver']][1]
-        if len(per_driver_params[port_info['driver']]) > 2:
-            self.rate_lat = per_driver_params[port_info['driver']][2]
+        drv_name = port_info['driver']
+        self.rate_percent = per_driver_params[drv_name][0]
+        self.total_pkts = per_driver_params[drv_name][1]
+        if len(per_driver_params[drv_name]) > 2:
+            self.rate_lat = per_driver_params[drv_name][2]
         else:
             self.rate_lat = self.rate_percent
+        self.lat_pps = 1000
         self.drops_expected = False
         self.c.reset(ports = [self.tx_port, self.rx_port])
 
+        vm = STLScVmRaw( [ STLVmFlowVar ( "ip_src",  min_value="10.0.0.1",
+                                          max_value="10.0.0.255", size=4, step=1,op="inc"),
+                           STLVmWrFlowVar (fv_name="ip_src", pkt_offset= "IP.src" ), # write ip to packet IP.src
+                           STLVmFixIpv4(offset = "IP")                                # fix checksum
+                          ]
+                         # Latency is bound to one core. We test that this option is not causing trouble
+                         ,split_by_field = "ip_src"
+                         ,cache_size =255 # Cache is ignored by latency flows. Need to test it is not crashing.
+                         );
+
         self.pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('Your_paylaod_comes_here'))
         self.large_pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('a'*1000))
         self.pkt_9k = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('a'*9000))
+        self.vm_pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")
+                                    / UDP(dport=12,sport=1025)/('Your_paylaod_comes_here')
+                                    , vm = vm)
+        self.vm_large_pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('a'*1000)
+                                          , vm = vm)
+        self.vm_9k_pkt = STLPktBuilder(pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12,sport=1025)/('a'*9000)
+                                       ,vm = vm)
 
-
-        drv_name=port_info['driver']
         self.latency_9k_enable=per_driver_params[drv_name][3]
         if self.latency_9k_enable:
             self.latency_9k_max_average = per_driver_params[drv_name][4]
@@ -124,7 +137,7 @@ class STLRX_Test(CStlGeneral_Test):
             tmp = 'TX pkts mismatch - got: {0}, expected: {1}'.format(tx_pkts, total_pkts)
             assert False, tmp
 
-        if tx_bytes != (total_pkts * (pkt_len + 4)): # + 4 for ethernet CRC
+        if tx_bytes != (total_pkts * pkt_len):
             pprint.pprint(flow_stats)
             tmp = 'TX bytes mismatch - got: {0}, expected: {1}'.format(tx_bytes, (total_pkts * pkt_len))
             assert False, tmp
@@ -136,7 +149,7 @@ class STLRX_Test(CStlGeneral_Test):
 
         if "rx_bytes" in self.cap:
             rx_bytes = flow_stats['rx_bytes'].get(self.rx_port, 0)
-            if rx_bytes != (total_pkts * (pkt_len + 4)) and not self.drops_expected: # +4 for ethernet CRC
+            if rx_bytes != (total_pkts * pkt_len) and not self.drops_expected:
                 pprint.pprint(flow_stats)
                 tmp = 'RX bytes mismatch - got: {0}, expected: {1}'.format(rx_bytes, (total_pkts * pkt_len))
                 assert False, tmp
@@ -157,7 +170,7 @@ class STLRX_Test(CStlGeneral_Test):
 
     # one stream on TX --> RX
     def test_one_stream(self):
-        total_pkts = self.total_pkts * 10
+        total_pkts = self.total_pkts
 
         try:
             s1 = STLStream(name = 'rx',
@@ -172,7 +185,7 @@ class STLRX_Test(CStlGeneral_Test):
 
             print("\ninjecting {0} packets on port {1}\n".format(total_pkts, self.tx_port))
 
-            exp = {'pg_id': 5, 'total_pkts': total_pkts, 'pkt_len': self.pkt.get_pkt_len()}
+            exp = {'pg_id': 5, 'total_pkts': total_pkts, 'pkt_len': s1.get_pkt_len()}
 
             self.__rx_iteration( [exp] )
 
@@ -182,6 +195,9 @@ class STLRX_Test(CStlGeneral_Test):
 
 
     def test_multiple_streams(self):
+        if self.is_virt_nics:
+            self.skip('Skip this for virtual NICs')
+
         num_latency_streams = 128
         num_flow_stat_streams = 127
         total_pkts = int(self.total_pkts / (num_latency_streams + num_flow_stat_streams))
@@ -200,7 +216,7 @@ class STLRX_Test(CStlGeneral_Test):
                                          flow_stats = STLFlowLatencyStats(pg_id = pg_id),
                                          mode = STLTXSingleBurst(total_pkts = total_pkts+pg_id, percentage = percent)))
 
-                exp.append({'pg_id': pg_id, 'total_pkts': total_pkts+pg_id, 'pkt_len': self.pkt.get_pkt_len()})
+                exp.append({'pg_id': pg_id, 'total_pkts': total_pkts+pg_id, 'pkt_len': streams[-1].get_pkt_len()})
 
             for pg_id in range(num_latency_streams + 1, num_latency_streams + num_flow_stat_streams):
 
@@ -209,7 +225,7 @@ class STLRX_Test(CStlGeneral_Test):
                                          flow_stats = STLFlowStats(pg_id = pg_id),
                                          mode = STLTXSingleBurst(total_pkts = total_pkts+pg_id, percentage = percent)))
 
-                exp.append({'pg_id': pg_id, 'total_pkts': total_pkts+pg_id, 'pkt_len': self.pkt.get_pkt_len()})
+                exp.append({'pg_id': pg_id, 'total_pkts': total_pkts+pg_id, 'pkt_len': streams[-1].get_pkt_len()})
 
             # add both streams to ports
             self.c.add_streams(streams, ports = [self.tx_port])
@@ -224,36 +240,42 @@ class STLRX_Test(CStlGeneral_Test):
         total_pkts = self.total_pkts
 
         try:
-            s1 = STLStream(name = 'rx',
-                           packet = self.pkt,
-                           flow_stats = STLFlowStats(pg_id = 5),
-                           mode = STLTXSingleBurst(total_pkts = total_pkts,
-                                                   percentage = self.rate_percent
-                                                   ))
-
-            s_lat = STLStream(name = 'rx',
-                           packet = self.pkt,
-                           flow_stats = STLFlowLatencyStats(pg_id = 5),
-                           mode = STLTXSingleBurst(total_pkts = total_pkts,
-                                                   percentage = self.rate_lat
-                                                   ))
-
-            print("\ninjecting {0} packets on port {1}\n".format(total_pkts, self.tx_port))
-
-            exp = {'pg_id': 5, 'total_pkts': total_pkts, 'pkt_len': self.pkt.get_pkt_len()}
-            exp_lat = {'pg_id': 5, 'total_pkts': total_pkts, 'pkt_len': self.pkt.get_pkt_len()}
-
-            self.c.add_streams([s1], ports = [self.tx_port])
-            for i in range(0, 10):
-                print("starting iteration {0}".format(i))
-                self.__rx_iteration( [exp] )
-
-            self.c.remove_all_streams(ports = [self.tx_port])
-            self.c.add_streams([s_lat], ports = [self.tx_port])
-            for i in range(0, 10):
-                print("starting iteration {0} latency".format(i))
-                self.__rx_iteration( [exp_lat] )
+            streams_data = [
+                {'name': 'Flow stat. No latency', 'pkt': self.pkt, 'lat': False},
+                {'name': 'Latency, no field engine', 'pkt': self.pkt, 'lat': True},
+                {'name': 'Latency, short packet with field engine', 'pkt': self.vm_pkt, 'lat': True},
+                {'name': 'Latency, large packet field engine', 'pkt': self.vm_large_pkt, 'lat': True}
+            ]
+            if self.latency_9k_enable:
+                streams_data.append({'name': 'Latency, 9k packet with field engine', 'pkt': self.vm_9k_pkt, 'lat': True})
 
+            streams = []
+            for data in streams_data:
+                if data['lat']:
+                    flow_stats = STLFlowLatencyStats(pg_id = 5)
+                    mode = STLTXSingleBurst(total_pkts = total_pkts, percentage = self.rate_percent)
+                else:
+                    flow_stats = STLFlowStats(pg_id = 5)
+                    mode = STLTXSingleBurst(total_pkts = total_pkts, pps = self.lat_pps)
+
+                s = STLStream(name = data['name'],
+                              packet = data['pkt'],
+                              flow_stats = flow_stats,
+                              mode = mode
+                              )
+                streams.append(s)
+
+            print("\ninjecting {0} packets on port {1}".format(total_pkts, self.tx_port))
+            exp = {'pg_id': 5, 'total_pkts': total_pkts}
+
+            for stream in streams:
+                self.c.add_streams([stream], ports = [self.tx_port])
+                print("Stream: {0}".format(stream.name))
+                exp['pkt_len'] = stream.get_pkt_len()
+                for i in range(0, 10):
+                    print("Iteration {0}".format(i))
+                    self.__rx_iteration( [exp] )
+                self.c.remove_all_streams(ports = [self.tx_port])
 
 
         except STLError as e:
@@ -313,8 +335,8 @@ class STLRX_Test(CStlGeneral_Test):
 
     # check low latency when you have stream of 9K stream 
     def test_9k_stream(self):
-
-        #self.skip('Skip due to bug trex-215')
+        if self.is_virt_nics:
+            self.skip('Skip this for virtual NICs')
 
         if self.latency_9k_enable == False:
            print("SKIP")
@@ -429,7 +451,7 @@ class STLRX_Test(CStlGeneral_Test):
                 self.check_stats (stats,ls['tx_pkts'][pid], pkts,"ls['tx_pkts'][pid]")
 
                 self.check_stats (stats,ls['tx_bytes']['total'], tbytes,"ls['tx_bytes']['total']")
-                self.check_stats (stats,ls['tx_bytes'][pid], pkts+1,"ls['tx_bytes'][pid]")
+                self.check_stats (stats,ls['tx_bytes'][pid], tbytes,"ls['tx_bytes'][pid]")
 
 
             return 0
@@ -441,17 +463,21 @@ class STLRX_Test(CStlGeneral_Test):
 
     def test_fcs_stream(self):
         """ this test send 1 64 byte packet with latency and check that all counters are reported as 64 bytes"""
-        #self.skip('Skip due to bug trex-213')
+
+        if self.is_virt_nics:
+            self.skip('Skip this for virtual NICs')
 
         all_ports=list(CTRexScenario.stl_ports_map['map'].keys());
         for port in all_ports:
             for l in [True,False]:
                 print(" test port {0} latency : {1} ".format(port,l))
-                self.send_1_burst(port,False,100)
+                self.send_1_burst(port,l,100)
 
     
     # this test adds more and more latency streams and re-test with incremental
     def test_incremental_latency_streams (self):
+        if self.is_virt_nics:
+            self.skip('Skip this for virtual NICs')
 
         total_pkts = self.total_pkts
         percent = 0.5
@@ -484,7 +510,7 @@ class STLRX_Test(CStlGeneral_Test):
 
                     print("port {0} : {1} streams at {2}% of line rate\n".format(self.tx_port, i, total_percent))
 
-                    exp.append({'pg_id': i, 'total_pkts': total_pkts, 'pkt_len': my_pkt.get_pkt_len()})
+                    exp.append({'pg_id': i, 'total_pkts': total_pkts, 'pkt_len': s1.get_pkt_len()})
 
                     self.__rx_iteration( exp )
 
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_flow_latency_stats.py b/scripts/automation/trex_control_plane/stl/examples/stl_flow_latency_stats.py
index ac0e212b..d8a99479 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_flow_latency_stats.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_flow_latency_stats.py
@@ -1,10 +1,12 @@
+# Example showing how to define stream for latency measurement, and how to parse the latency information
+
 import stl_path
 from trex_stl_lib.api import *
 
 import time
 import pprint
 
-def rx_example (tx_port, rx_port, burst_size, bw):
+def rx_example (tx_port, rx_port, burst_size, pps):
 
     print("\nGoing to inject {0} packets on port {1} - checking RX stats on port {2}\n".format(burst_size, tx_port, rx_port))
 
@@ -19,7 +21,7 @@ def rx_example (tx_port, rx_port, burst_size, bw):
                        packet = pkt,
                        flow_stats = STLFlowLatencyStats(pg_id = 5),
                        mode = STLTXSingleBurst(total_pkts = total_pkts,
-                                               percentage = bw))
+                                               pps = pps))
 
         # connect to server
         c.connect()
@@ -32,7 +34,7 @@ def rx_example (tx_port, rx_port, burst_size, bw):
 
         print("\nInjecting {0} packets on port {1}\n".format(total_pkts, tx_port))
 
-        rc = rx_iteration(c, tx_port, rx_port, total_pkts, pkt.get_pkt_len(), bw)
+        rc = rx_iteration(c, tx_port, rx_port, total_pkts, pkt.get_pkt_len())
         if not rc:
             passed = False
 
@@ -44,12 +46,12 @@ def rx_example (tx_port, rx_port, burst_size, bw):
         c.disconnect()
 
     if passed:
-        print("\nTest has passed :-)\n")
+        print("\nTest passed :-)\n")
     else:
-        print("\nTest has failed :-(\n")
+        print("\nTest failed :-(\n")
 
 # RX one iteration
-def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
+def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len):
     
     c.clear_stats()
 
@@ -58,7 +60,8 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
 
     stats = c.get_stats()
     flow_stats = stats['flow_stats'].get(5)
-    lat_stats = stats['latency'].get(5)
+    global_lat_stats = stats['latency']
+    lat_stats = global_lat_stats.get(5)
     if not flow_stats:
         print("no flow stats available")
         return False
@@ -74,6 +77,8 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
     dup = lat_stats['err_cntrs']['dup']
     sth = lat_stats['err_cntrs']['seq_too_high']
     stl = lat_stats['err_cntrs']['seq_too_low']
+    old_flow = global_lat_stats['global']['old_flow']
+    bad_hdr = global_lat_stats['global']['bad_hdr']
     lat = lat_stats['latency']
     jitter = lat['jitter']
     avg = lat['average']
@@ -89,6 +94,10 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
             return False
 
     print('Error counters: dropped:{0}, ooo:{1} dup:{2} seq too high:{3} seq too low:{4}'.format(drops, ooo, dup, sth, stl))
+    if old_flow:
+        print ('Packets arriving too late after flow stopped: {0}'.format(old_flow))
+    if bad_hdr:
+        print ('Latency packets with corrupted info: {0}'.format(bad_hdr))
     print('Latency info:')
     print("  Maximum latency(usec): {0}".format(tot_max))
     print("  Minimum latency(usec): {0}".format(tot_min))
@@ -131,5 +140,5 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
     return True
 
 # run the tests
-rx_example(tx_port = 1, rx_port = 0, burst_size = 500000, bw = 50)
+rx_example(tx_port = 0, rx_port = 1, burst_size = 1000, pps = 1000)
 
diff --git a/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py b/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py
index ed4902fa..3c630ece 100644
--- a/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py
+++ b/scripts/automation/trex_control_plane/stl/examples/stl_flow_stats.py
@@ -1,3 +1,5 @@
+# Example showing how to define stream for getting per flow statistics, and how to parse the received statistics
+
 import stl_path
 from trex_stl_lib.api import *
 
@@ -27,18 +29,14 @@ def rx_example (tx_port, rx_port, burst_size, bw):
         # prepare our ports
         c.reset(ports = [tx_port, rx_port])
 
-        # add both streams to ports
+        # add stream to port
         c.add_streams([s1], ports = [tx_port])
 
-        print("\ninjecting {0} packets on port {1}\n".format(total_pkts, tx_port))
-
-        for i in range(0, 10):
-            print("\nStarting iteration: {0}:".format(i))
-            rc = rx_iteration(c, tx_port, rx_port, total_pkts, pkt.get_pkt_len(), bw)
-            if not rc:
-                passed = False
-                break
+        print("\ngoing to inject {0} packets on port {1}\n".format(total_pkts, tx_port))
 
+        rc = rx_iteration(c, tx_port, rx_port, total_pkts, s1.get_pkt_len())
+        if not rc:
+            passed = False
 
     except STLError as e:
         passed = False
@@ -48,19 +46,21 @@ def rx_example (tx_port, rx_port, burst_size, bw):
         c.disconnect()
 
     if passed:
-        print("\nTest has passed :-)\n")
+        print("\nTest passed :-)\n")
     else:
-        print("\nTest has failed :-(\n")
+        print("\nTest failed :-(\n")
 
 # RX one iteration
-def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
-    
+def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len):
+    ret = True
+
     c.clear_stats()
 
     c.start(ports = [tx_port])
     c.wait_on_traffic(ports = [tx_port])
 
-    flow_stats = c.get_stats()['flow_stats'].get(5)
+    global_flow_stats = c.get_stats()['flow_stats']
+    flow_stats = global_flow_stats.get(5)
     if not flow_stats:
         print("no flow stats available")
         return False
@@ -78,26 +78,33 @@ def rx_iteration (c, tx_port, rx_port, total_pkts, pkt_len, bw):
     if tx_pkts != total_pkts:
         print("TX pkts mismatch - got: {0}, expected: {1}".format(tx_pkts, total_pkts))
         pprint.pprint(flow_stats)
-        return False
+        ret = False
     else:
         print("TX pkts match   - {0}".format(tx_pkts))
 
     if tx_bytes != (total_pkts * pkt_len):
         print("TX bytes mismatch - got: {0}, expected: {1}".format(tx_bytes, (total_pkts * pkt_len)))
         pprint.pprint(flow_stats)
-        return False
+        ret = False
     else:
         print("TX bytes match  - {0}".format(tx_bytes))
 
     if rx_pkts != total_pkts:
         print("RX pkts mismatch - got: {0}, expected: {1}".format(rx_pkts, total_pkts))
         pprint.pprint(flow_stats)
-        return False
+        ret = False
     else:
         print("RX pkts match   - {0}".format(rx_pkts))
 
-    return True
+
+    for field in ['rx_err', 'tx_err']:
+        for port in global_flow_stats['global'][field].keys():
+            if global_flow_stats['global'][field][port] != 0:
+                print ("\n{0} on port {1}: {2} - You should consider increasing rx_delay_ms value in wait_on_traffic"
+                       .format(field, port, global_flow_stats['global'][field][port]))
+
+    return ret
 
 # run the tests
-rx_example(tx_port = 1, rx_port = 2, burst_size = 500000, bw = 50)
+rx_example(tx_port = 0, rx_port = 1, burst_size = 500, bw = 50)
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
index dc0035b3..ae7c23f2 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
@@ -1287,7 +1287,7 @@ class STLClient(object):
 
             **total** and per port statistics contain dictionary with following format.
 
-            Most of the bytes counters (unless  specified otherwise) are in L2 layer including the FCS. e.g. minimum packet size in 64 bytes
+            Most of the bytes counters (unless specified otherwise) are in L2 layer, including the Ethernet FCS. e.g. minimum packet size is 64 bytes
 
             ===============================  ===============
             key                               Meaning
@@ -1306,21 +1306,34 @@ class STLClient(object):
 
             .. _flow_stats:
 
-            **flow_stats** contains dictionaries per packet group id (pg id). Each one with the following structure.
+            **flow_stats** contains :ref:`global dictionary <flow_stats_global>`, and dictionaries per packet group id (pg id). See structures below.
+
+            **per pg_id flow stat** dictionaries have following structure:
 
             =================   ===============
             key                 Meaning
             =================   ===============
-            rx_bps              Receivd bytes per second rate
-            rx_bps_l1           Receivd bytes per second rate, including layer one
+            rx_bps              Received bytes per second rate
+            rx_bps_l1           Received bytes per second rate, including layer one
             rx_bytes            Total number of received bytes
             rx_pkts             Total number of received packets
             rx_pps              Received packets per second
-            tx_bps              Transmitted bytes per second rate
-            tx_bps_l1           Transmitted bytes per second rate, including layer one
+            tx_bps              Transmit bytes per second rate
+            tx_bps_l1           Transmit bytes per second rate, including layer one
             tx_bytes            Total number of sent bytes
             tx_pkts             Total number of sent packets
-            tx_pps              Transmit packets per second
+            tx_pps              Transmit packets per second rate
+            =================   ===============
+
+            .. _flow_stats_global:
+
+            **global flow stats** dictionary has the following structure:
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            rx_err              Number of flow statistics packets received that we could not associate to any pg_id. This can happen if latency on the used setup is large. See :ref:`wait_on_traffic <wait_on_traffic>` rx_delay_ms parameter for details.
+            tx_err              Number of flow statistics packets transmitted that we could not associate to any pg_id. This is never expected. If you see this different than 0, please report.
             =================   ===============
 
             .. _global:
@@ -1343,7 +1356,9 @@ class STLClient(object):
 
             .. _latency:
 
-            **latency** contains dictionary per packet group id (pg id). Each one with the following structure.
+            **latency** contains :ref:`global dictionary <lat_stats_global>`, and dictionaries per packet group id (pg id). Each one with the following structure.
+
+            **per pg_id latency stat** dictionaries have following structure:
 
             ===========================          ===============
             key                                  Meaning
@@ -1397,7 +1412,16 @@ class STLClient(object):
             total_min           Minimum latency measured over the stream lifetime (in usec).
             =================   ===============
 
+            .. _lat_stats_global:
 
+            **global latency stats** dictionary has the following structure:
+
+            =================   ===============
+            key                 Meaning
+            =================   ===============
+            old_flow            Number of latency statistics packets received that we could not associate to any pg_id. This can happen if latency on the used setup is large. See :ref:`wait_on_traffic <wait_on_traffic>` rx_delay_ms parameter for details.
+            bad_hdr             Number of latency packets received with bad latency data. This can happen becuase of garbage packets in the network, or if the DUT causes packet corruption.
+            =================   ===============
 
         :raises:
           None
@@ -2292,6 +2316,8 @@ class STLClient(object):
     @__api_check(True)
     def wait_on_traffic (self, ports = None, timeout = 60, rx_delay_ms = 10):
         """
+            .. _wait_on_traffic:
+
             Block until traffic on specified port(s) has ended
 
             :parameters:
@@ -2302,12 +2328,11 @@ class STLClient(object):
                     timeout in seconds
 
                 rx_delay_ms : int
-                    time to wait until RX filters are removed
-                    this value should reflect the time it takes
-                    packets which were transmitted to arrive
+                    Time to wait (in milliseconds) after last packet was sent, until RX filters used for
+                    measuring flow statistics and latency are removed.
+                    This value should reflect the time it takes packets which were transmitted to arrive
                     to the destination.
-                    after this time the RX filters will be removed
-
+                    After this time, RX filters will be removed, and packets arriving for per flow statistics feature and latency flows will be counted as errors.
 
             :raises:
                 + :exc:`STLTimeoutError` - in case timeout has expired
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
index 678adb4e..af4d6f69 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
@@ -1093,6 +1093,13 @@ class CLatencyStats(CTRexStats):
             snapshot = {}
         output = {}
 
+        output['global'] = {}
+        for field in ['bad_hdr', 'old_flow']:
+            if 'global' in snapshot and field in snapshot['global']:
+                output['global'][field] = snapshot['global'][field]
+            else:
+                output['global'][field] = 0
+
         # we care only about the current active keys
         pg_ids = list(filter(is_intable, snapshot.keys()))
 
@@ -1190,6 +1197,14 @@ class CRxStats(CTRexStats):
         # copy timestamp field
         output['ts'] = current['ts']
 
+        # global (not per pg_id) error counters
+        output['global'] = {}
+        for field in ['rx_err', 'tx_err']:
+            output['global'][field] = {}
+            if 'global' in current and field in current['global']:
+                for port in current['global'][field]:
+                    output['global'][field][int(port)] = current['global'][field][port]
+
         # we care only about the current active keys
         pg_ids = list(filter(is_intable, current.keys()))
 
@@ -1337,6 +1352,9 @@ class CRxStats(CTRexStats):
         for pg_id, value in self.latest_stats.items():
             # skip non ints
             if not is_intable(pg_id):
+                # 'global' stats are in the same level of the pg_ids. We do want them to go to the user
+                if pg_id == 'global':
+                    stats[pg_id] = value
                 continue
             # bare counters
             stats[int(pg_id)] = {}