Latency stat python API work

6 files changed, 64 insertions, 97 deletions

diff --git a/scripts/automation/regression/stateless_tests/stl_rx_test.py b/scripts/automation/regression/stateless_tests/stl_rx_test.py
index 3b979744..a5bc115f 100644
--- a/scripts/automation/regression/stateless_tests/stl_rx_test.py
+++ b/scripts/automation/regression/stateless_tests/stl_rx_test.py
@@ -7,7 +7,7 @@ class STLRX_Test(CStlGeneral_Test):
     """Tests for RX feature"""
 
     def setUp(self):
-        per_driver_params = {"rte_vmxnet3_pmd": [1, 50], "rte_ixgbe_pmd": [30, 5000], "rte_i40e_pmd": [80, 5000],
+        per_driver_params = {"rte_vmxnet3_pmd": [1, 50], "rte_ixgbe_pmd": [10, 5000], "rte_i40e_pmd": [80, 5000],
                              "rte_igb_pmd": [80, 500], "rte_em_pmd": [1, 50], "rte_virtio_pmd": [1, 50]}
 
         CStlGeneral_Test.setUp(self)
@@ -19,8 +19,9 @@ class STLRX_Test(CStlGeneral_Test):
 
         port_info = self.c.get_port_info(ports = self.rx_port)[0]
         cap = port_info['rx']['caps']
-        if cap != 1:
-            self.skip('port {0} does not support RX'.format(self.rx_port))
+#        print cap; ????
+#        if cap != 1:
+#            self.skip('port {0} does not support RX'.format(self.rx_port))
 
         self.rate_percent = per_driver_params[port_info['driver']][0]
         self.total_pkts = per_driver_params[port_info['driver']][1]
@@ -36,14 +37,24 @@ class STLRX_Test(CStlGeneral_Test):
 
 
     def __verify_flow (self, pg_id, total_pkts, pkt_len):
-
         flow_stats = self.c.get_stats()['flow_stats'].get(pg_id)
+        latency_stats = self.c.get_stats()['latency'].get(pg_id)
+
         if not flow_stats:
             assert False, "no flow stats available"
 
         tx_pkts  = flow_stats['tx_pkts'].get(self.tx_port, 0)
         tx_bytes = flow_stats['tx_bytes'].get(self.tx_port, 0)
         rx_pkts  = flow_stats['rx_pkts'].get(self.rx_port, 0)
+        drops = latency_stats['err_cntrs']['dropped']
+        ooo = latency_stats['err_cntrs']['out_of_order']
+        jitter = latency_stats['jitter']
+        latency = latency_stats['latency']
+
+        if drops != 0 or ooo != 0:
+            pprint.pprint(latency_stats)
+            tmp='Dropped or out of order packets - dropped: {0}, ooo: {1}'.format(drops, ooo)
+            assert False, tmp
 
         if tx_pkts != total_pkts:
             pprint.pprint(flow_stats)
@@ -78,7 +89,7 @@ class STLRX_Test(CStlGeneral_Test):
         try:
             s1 = STLStream(name = 'rx',
                            packet = self.pkt,
-                           flow_stats = STLFlowStats(pg_id = 5),
+                           flow_stats = STLFlowLatencyStats(pg_id = 5),
                            mode = STLTXSingleBurst(total_pkts = total_pkts,
                                                    percentage = self.rate_percent
                                                    ))
@@ -99,13 +110,11 @@ class STLRX_Test(CStlGeneral_Test):
 
     # one simple stream on TX --> RX
     def test_multiple_streams(self):
-        num_streams = 10
+        num_streams = 128
         total_pkts = int(self.total_pkts / num_streams)
         if total_pkts == 0:
             total_pkts = 1
-        percent = int(self.rate_percent / num_streams)
-        if percent == 0:
-            percent = 1
+        percent = float(self.rate_percent) / num_streams
 
         try:
             streams = []
@@ -115,7 +124,7 @@ class STLRX_Test(CStlGeneral_Test):
 
                 streams.append(STLStream(name = 'rx {0}'.format(pg_id),
                                          packet = self.pkt,
-                                         flow_stats = STLFlowStats(pg_id = pg_id),
+                                         flow_stats = STLFlowLatencyStats(pg_id = pg_id),
                                          mode = STLTXSingleBurst(total_pkts = total_pkts+pg_id, percentage = percent)))
 
                 exp.append({'pg_id': pg_id, 'total_pkts': total_pkts+pg_id, 'pkt_len': self.pkt.get_pkt_len()})
@@ -135,7 +144,7 @@ class STLRX_Test(CStlGeneral_Test):
         try:
             s1 = STLStream(name = 'rx',
                            packet = self.pkt,
-                           flow_stats = STLFlowStats(pg_id = 5),
+                           flow_stats = STLFlowLatencyStats(pg_id = 5),
                            mode = STLTXSingleBurst(total_pkts = total_pkts,
                                                    percentage = self.rate_percent
                                                    ))
@@ -143,7 +152,7 @@ class STLRX_Test(CStlGeneral_Test):
             # add both streams to ports
             self.c.add_streams([s1], ports = [self.tx_port])
 
-            #print "\ninjecting {0} packets on port {1}\n".format(total_pkts, self.tx_port)
+            print "\ninjecting {0} packets on port {1}\n".format(total_pkts, self.tx_port)
 
             exp = {'pg_id': 5, 'total_pkts': total_pkts, 'pkt_len': self.pkt.get_pkt_len()}
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py
index 5c9faf0f..731ddb10 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_async_client.py
@@ -293,6 +293,9 @@ class CTRexAsyncClient():
         elif name == "flow_stats":
             self.event_handler.on_async_rx_stats_event(data, baseline)
 
+        elif name == "latency_stats":
+            self.event_handler.on_async_latency_stats_event(data, baseline)
+
         else:
             pass
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
index e7b46aea..88c64e3c 100755
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_client.py
@@ -187,6 +187,8 @@ class EventsHandler(object):
     def on_async_rx_stats_event (self, data, baseline):
         self.client.flow_stats.update(data, baseline)
 
+    def on_async_latency_stats_event (self, data, baseline):
+        self.client.latency_stats.update(data, baseline)
 
     # handles an async stats update from the subscriber
     def on_async_stats_update(self, dump_data, baseline):
@@ -551,6 +553,8 @@ class STLClient(object):
 
         self.flow_stats = trex_stl_stats.CRxStats(self.ports)
 
+        self.latency_stats = trex_stl_stats.CLatencyStats(self.ports)
+
         self.stats_generator = trex_stl_stats.CTRexInfoGenerator(self.global_stats,
                                                                  self.ports,
                                                                  self.flow_stats)
@@ -891,6 +895,7 @@ class STLClient(object):
         stats['total'] = total
 
         stats['flow_stats'] = self.flow_stats.get_stats()
+        stats['latency'] = self.latency_stats.get_stats()
 
         return stats
 
diff --git a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
index e30da00e..a65ef9e0 100644
--- a/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
+++ b/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_stats.py
@@ -831,6 +831,37 @@ class CPortStats(CTRexStats):
                 }
 
 
+class CLatencyStats(CTRexStats):
+    def __init__(self, ports):
+        super(CLatencyStats, self).__init__()
+
+    # for API
+    def get_stats (self):
+        return self.latest_stats
+
+    def process_snapshot (self, current):
+        output = {}
+
+        # we care only about the current active keys
+        pg_ids = list(filter(is_intable, current.keys()))
+
+        for pg_id in pg_ids:
+            current_pg = current.get(pg_id)
+            int_pg_id = int(pg_id)
+            output[int_pg_id] = {}
+            for field in ['err_cntrs', 'jitter', 'latency']:
+                output[int_pg_id][field] = current_pg[field]
+        return output
+
+    def update (self, snapshot, baseline):
+        # generate a new snapshot
+        if (snapshot is not None):
+            new_snapshot = self.process_snapshot(snapshot)
+        else:
+            return
+
+        self.latest_stats = new_snapshot
+        return True
 
 
 # RX stats objects - COMPLEX :-(
diff --git a/src/flow_stat.cpp b/src/flow_stat.cpp
index d63dc778..b2c3f487 100644
--- a/src/flow_stat.cpp
+++ b/src/flow_stat.cpp
@@ -1048,20 +1048,10 @@ bool CFlowStatRuleMgr::dump_json(std::string & s_json, std::string & l_json, boo
                 l_data_section[str_user_id]["jitter"] = user_id_info_p->get_jitter();
             }
             ///????? add last 10 samples
-            l_data_section[str_user_id]["err_cntrs"]["lost"]
+            l_data_section[str_user_id]["err_cntrs"]["dropped"]
                 = Json::Value::UInt64(user_id_info_p->get_seq_err_cnt());
             l_data_section[str_user_id]["err_cntrs"]["out_of_order"]
                 = Json::Value::UInt64(user_id_info_p->get_ooo_cnt());
-
-            //??? temp - remove
-#if 0
-            s_data_section[str_user_id]["tx_bytes"]["0"]
-                = Json::Value::UInt64(user_id_info_p->get_seq_err_cnt());
-            s_data_section[str_user_id]["tx_bytes"]["1"] = 0;
-            s_data_section[str_user_id]["rx_bytes"]["0"]
-                = Json::Value::UInt64(user_id_info_p->get_ooo_cnt());
-            s_data_section[str_user_id]["rx_bytes"]["1"] = 0;
-#endif
         }
     }
 
diff --git a/src/main_dpdk.cpp b/src/main_dpdk.cpp
index ccd1aa8d..b0fd3180 100644
--- a/src/main_dpdk.cpp
+++ b/src/main_dpdk.cpp
@@ -2008,8 +2008,8 @@ void CCoreEthIF::update_mac_addr(CGenNode * node,uint8_t *p){
 
 int CCoreEthIFStateless::send_node_flow_stat(rte_mbuf *m, CGenNodeStateless * node_sl, CCorePerPort *  lp_port
                                              , CVirtualIFPerSideStats  * lp_stats) {
-    //??? remove
-# if 0
+    // Defining this makes 10% percent packet loss. 1% packet reorder.
+# ifdef ERR_CNTRS_TEST
     static int temp=1;
     temp++;
 #endif
@@ -2038,20 +2038,15 @@ int CCoreEthIFStateless::send_node_flow_stat(rte_mbuf *m, CGenNodeStateless * no
 
         lp_stats->m_seq_num[hw_id_payload]++;
 
-        //??? remove
-#if 0
+#ifdef ERR_CNTRS_TEST
         if (temp % 10 == 0) {
             fsp_head->seq = lp_stats->m_seq_num[hw_id_payload]++;
         }
-
         if ((temp - 1) % 100 == 0) {
             fsp_head->seq = lp_stats->m_seq_num[hw_id_payload] - 4;
-            //            lp_stats->m_seq_num[hw_id_payload]--;
         }
 #endif
 
-
-
         if (rte_pktmbuf_is_contiguous(m)) {
             // We have only the const mbuf
             mi = CGlobalInfo::pktmbuf_alloc_small(get_socket_id());
@@ -2081,72 +2076,6 @@ int CCoreEthIFStateless::send_node_flow_stat(rte_mbuf *m, CGenNodeStateless * no
     return 0;
 }
 
-#if 0
-//??? remove
-// Maybe make it part of send_node somehow
-int CCoreEthIFStateless::send_node_flow_stat(CGenNodeStateless * node_sl) {
-    //??? remove
-    static int temp=1;
-    temp++;
-
-    uint16_t hw_id = node_sl->get_stat_hw_id();
-    tx_per_flow_t *lp_s;
-    /* check that we have mbuf  */
-    rte_mbuf_t *temp_m = node_sl->get_cache_mbuf();
-    rte_mbuf_t *m;
-
-    if (temp_m) {
-        /* cache case */
-        m = node_sl->alloc_flow_stat_mbuf(temp_m);
-    }else{
-        temp_m = node_sl->alloc_node_with_vm();
-        assert(temp_m);
-        m = node_sl->alloc_flow_stat_mbuf(temp_m);
-        rte_pktmbuf_free(temp_m);
-    }
-
-    pkt_dir_t dir=(pkt_dir_t)node_sl->get_mbuf_cache_dir();
-    CCorePerPort *  lp_port=&m_ports[dir];
-    CVirtualIFPerSideStats  * lp_stats = &m_stats[dir];
-
-    if (hw_id >= MAX_FLOW_STATS) {
-        // payload rule hw_ids are in the range right above ip id rules
-        uint16_t hw_id_payload = hw_id - MAX_FLOW_STATS;
-        if (hw_id_payload > max_stat_hw_id_seen_payload) {
-            max_stat_hw_id_seen_payload = hw_id_payload;
-        }
-        uint8_t *p = rte_pktmbuf_mtod(m, uint8_t*);
-        struct flow_stat_payload_header *fsp_head = (struct flow_stat_payload_header *)
-            (p + m->pkt_len - sizeof(struct flow_stat_payload_header));
-        fsp_head->seq = lp_stats->m_seq_num[hw_id_payload];
-        fsp_head->time_stamp = os_get_hr_tick_64();
-        lp_stats->m_seq_num[hw_id_payload]++;
-        // remove ???
-
-        if (temp % 10 == 0) {
-            fsp_head->seq = lp_stats->m_seq_num[hw_id_payload]++;
-        }
-#if 1
-        if ((temp - 1) % 100 == 0) {
-            fsp_head->seq = lp_stats->m_seq_num[hw_id_payload] - 4;
-            //            lp_stats->m_seq_num[hw_id_payload]--;
-        }
-#endif
-    } else {
-        // ip id rule
-        if (hw_id > max_stat_hw_id_seen) {
-            max_stat_hw_id_seen = hw_id;
-        }
-    }
-    lp_s = &lp_stats->m_tx_per_flow[hw_id];
-    lp_s->add_pkts(1);
-    lp_s->add_bytes(m->pkt_len);
-
-    send_pkt(lp_port,m,lp_stats);
-    return 0;
-}
-#endif
-
 int CCoreEthIFStateless::send_node(CGenNode * no) {
     /* if a node is marked as slow path - single IF to redirect it to slow path */
     if (no->get_is_slow_path()) {