more error counter

5 files changed, 172 insertions, 79 deletions

diff --git a/src/flow_stat.cpp b/src/flow_stat.cpp
index feb7276a..5fc3aadd 100644
--- a/src/flow_stat.cpp
+++ b/src/flow_stat.cpp
@@ -171,10 +171,16 @@ void CFlowStatUserIdInfoPayload::add_stream(uint8_t proto) {
 void CFlowStatUserIdInfoPayload::reset_hw_id() {
     CFlowStatUserIdInfo::reset_hw_id();
 
-    m_seq_error_base += m_rfc2544_info.m_seq_error;
+    m_seq_err_base += m_rfc2544_info.m_seq_err;
     m_out_of_order_base += m_rfc2544_info.m_out_of_order;
-    m_rfc2544_info.m_seq_error = 0;
+    m_dup_base += m_rfc2544_info.m_dup;
+    m_seq_err_ev_big_base += m_rfc2544_info.m_seq_err_ev_big;
+    m_seq_err_ev_low_base += m_rfc2544_info.m_seq_err_ev_low;
+    m_rfc2544_info.m_seq_err = 0;
     m_rfc2544_info.m_out_of_order = 0;
+    m_rfc2544_info.m_dup = 0;
+    m_rfc2544_info.m_seq_err_ev_big = 0;
+    m_rfc2544_info.m_seq_err_ev_low = 0;
 }
 
 /************** class CFlowStatUserIdMap ***************/
@@ -919,16 +925,9 @@ bool CFlowStatRuleMgr::dump_json(std::string & s_json, std::string & l_json, boo
     l_root["name"] = "latency_stats";
     l_root["type"] = 0;
 
-    //??? temp - just to be able to print in python
-    static int temp = 0;
-    temp++;
-    if (temp == 10) {
-        l_root["type"] = 1;
-        temp = 0;
-    }
-    
     if (baseline) {
         s_root["baseline"] = true;
+        l_root["baseline"] = true;
     }
 
     Json::Value &s_data_section = s_root["data"];
@@ -1051,20 +1050,26 @@ bool CFlowStatRuleMgr::dump_json(std::string & s_json, std::string & l_json, boo
                 user_id_info_p->set_seq_err_cnt(rfc2544_info[hw_id].get_seq_err_cnt());
                 user_id_info_p->set_ooo_cnt(rfc2544_info[hw_id].get_ooo_cnt());
                 l_data_section[str_user_id]["latency"] = lat_hist;
-                l_data_section[str_user_id]["latency"]["last_max"] = rfc2544_info[hw_id].get_last_max();
-                l_data_section[str_user_id]["jitter"] = rfc2544_info[hw_id].get_jitter();
+                l_data_section[str_user_id]["latency"]["last_max"] = rfc2544_info[hw_id].get_last_max_usec();
+                l_data_section[str_user_id]["jitter"] = rfc2544_info[hw_id].get_jitter_usec();
             } else {
                 // Not mapped to hw_id. Get saved info.
                 user_id_info_p->get_latency_json(lat_hist);
                 l_data_section[str_user_id]["latency"]["histogram"] = lat_hist;
                 l_data_section[str_user_id]["latency"]["last_max"] = 0;
-                l_data_section[str_user_id]["jitter"] = user_id_info_p->get_jitter();
+                l_data_section[str_user_id]["jitter"] = user_id_info_p->get_jitter_usec();
             }
-            ///????? add last 10 samples
+            //todo: add last 10 samples
             l_data_section[str_user_id]["err_cntrs"]["dropped"]
                 = Json::Value::UInt64(user_id_info_p->get_seq_err_cnt());
             l_data_section[str_user_id]["err_cntrs"]["out_of_order"]
                 = Json::Value::UInt64(user_id_info_p->get_ooo_cnt());
+            l_data_section[str_user_id]["err_cntrs"]["dup"]
+                = Json::Value::UInt64(user_id_info_p->get_dup_cnt());
+            l_data_section[str_user_id]["err_cntrs"]["seq_too_high"]
+                = Json::Value::UInt64(user_id_info_p->get_seq_err_big_cnt());
+            l_data_section[str_user_id]["err_cntrs"]["seq_too_low"]
+                = Json::Value::UInt64(user_id_info_p->get_seq_err_low_cnt());
         }
     }
 
diff --git a/src/flow_stat.h b/src/flow_stat.h
index f239044d..f9761454 100644
--- a/src/flow_stat.h
+++ b/src/flow_stat.h
@@ -73,19 +73,34 @@ class rfc2544_info_t_ {
         m_latency = json;
     }
 
-    inline void set_err_cntrs(uint64_t seq, uint64_t ooo) {
-        m_seq_error = seq;
+    inline void set_err_cntrs(uint64_t seq, uint64_t ooo, uint64_t dup, uint64_t seq_big, uint64_t seq_low) {
+        m_seq_err = seq;
         m_out_of_order = ooo;
+        m_dup = dup;
+        m_seq_err_ev_big = seq_big;
+        m_seq_err_ev_low = seq_low;
     }
 
     inline uint64_t get_seq_err_cnt() {
-        return m_seq_error;
+        return m_seq_err;
     }
 
     inline uint64_t get_ooo_cnt() {
         return m_out_of_order;
     }
 
+    inline uint64_t get_dup_cnt() {
+        return m_dup;
+    }
+
+    inline uint64_t get_seq_err_ev_big() {
+        return m_seq_err_ev_big;
+    }
+
+    inline uint64_t get_seq_err_ev_low() {
+        return m_seq_err_ev_low;
+    }
+
     inline double get_jitter() const {
         return m_jitter;
     }
@@ -94,6 +109,10 @@ class rfc2544_info_t_ {
         m_jitter = jitter;
     }
 
+    uint32_t get_jitter_usec(){
+        return (uint32_t)(m_jitter * 1000000.0);
+    }
+
     inline void set_last_max(dsec_t val) {
         m_last_max_latency = val;
     }
@@ -102,48 +121,70 @@ class rfc2544_info_t_ {
         return m_last_max_latency;
     }
 
+    inline uint32_t get_last_max_usec() {
+        return (uint32_t)(m_last_max_latency * 1000000.0);
+    }
+
     inline void clear() {
-        m_seq_error = 0;
+        m_seq_err = 0;
         m_out_of_order = 0;
+        m_dup = 0;
+        m_seq_err_ev_big = 0;
+        m_seq_err_ev_low = 0;
         m_jitter = 0;
         m_latency = Json::Value("");
     }
 
     inline rfc2544_info_t_ operator+ (const rfc2544_info_t_ &t_in) {
         rfc2544_info_t_ t_out;
-        t_out.m_seq_error = this->m_seq_error + t_in.m_seq_error;
+        t_out.m_seq_err = this->m_seq_err + t_in.m_seq_err;
         t_out.m_out_of_order = this->m_out_of_order + t_in.m_out_of_order;
+        t_out.m_dup = this->m_dup + t_in.m_dup;
+        t_out.m_seq_err_ev_big = this->m_seq_err_ev_big + t_in.m_seq_err_ev_big;
+        t_out.m_seq_err_ev_low = this->m_seq_err_ev_low + t_in.m_seq_err_ev_low;
         return t_out;
     }
 
     inline rfc2544_info_t_ operator- (const rfc2544_info_t_ &t_in) {
         rfc2544_info_t_ t_out;
-        t_out.m_seq_error = this->m_seq_error - t_in.m_seq_error;
+        t_out.m_seq_err = this->m_seq_err - t_in.m_seq_err;
         t_out.m_out_of_order = this->m_out_of_order - t_in.m_out_of_order;
+        t_out.m_dup = this->m_dup - t_in.m_dup;
+        t_out.m_seq_err_ev_big = this->m_seq_err_ev_big - t_in.m_seq_err_ev_big;
+        t_out.m_seq_err_ev_low = this->m_seq_err_ev_low - t_in.m_seq_err_ev_low;
         return t_out;
     }
 
     inline rfc2544_info_t_ operator+= (const rfc2544_info_t_ &t_in) {
-        m_seq_error += t_in.m_seq_error;
+        m_seq_err += t_in.m_seq_err;
         m_out_of_order += t_in.m_out_of_order;
+        m_dup += t_in.m_dup;
+        m_seq_err_ev_big += t_in.m_seq_err_ev_big;
+        m_seq_err_ev_low += t_in.m_seq_err_ev_low;
         return *this;
     }
 
     inline bool operator!= (const rfc2544_info_t_ &t_in) {
-        if ((m_jitter != t_in.m_jitter) || (m_seq_error != t_in.m_seq_error) || (m_out_of_order != t_in.m_out_of_order))
+        if ((m_jitter != t_in.m_jitter) || (m_seq_err != t_in.m_seq_err)
+            || (m_out_of_order != t_in.m_out_of_order) || (m_dup != t_in.m_dup)
+            || (m_seq_err_ev_big != t_in.m_seq_err_ev_big) || (m_seq_err_ev_low != t_in.m_seq_err_ev_low))
             return true;
         return false;
     }
 
     friend std::ostream& operator<<(std::ostream& os, const rfc2544_info_t_ &t) {
         os  << "jitter:" << t.m_jitter << " errors(seq:"
-            << t.m_seq_error << " out of order:" << t.m_out_of_order << ")";
+            << t.m_seq_err << " out of order:" << t.m_out_of_order
+            << " dup:" << t.m_dup << ")";
         return os;
     }
 
  private:
-    uint64_t m_seq_error;
+    uint64_t m_seq_err;
     uint64_t m_out_of_order;
+    uint64_t m_dup;
+    uint64_t m_seq_err_ev_big;
+    uint64_t m_seq_err_ev_low;
     double   m_jitter;
     dsec_t   m_last_max_latency;
     // json latency object. In case of stop/start, we calculate latency graph from scratch,
@@ -212,9 +253,6 @@ class tx_per_flow_t_ {
  private:
     uint64_t m_bytes;
     uint64_t m_pkts;
-    uint64_t m_seq_error_base;
-    uint64_t m_out_of_order_base;
-
 };
 
 typedef class rfc2544_info_t_ rfc2544_info_t;
@@ -283,8 +321,11 @@ class CFlowStatUserIdInfoPayload : public CFlowStatUserIdInfo {
 
     void clear() {
         m_rfc2544_info.clear();
-        m_seq_error_base = 0;
+        m_seq_err_base = 0;
         m_out_of_order_base = 0;
+        m_dup_base = 0;
+        m_seq_err_ev_big_base = 0;
+        m_seq_err_ev_low_base = 0;
     }
     inline void get_latency_json(Json::Value & json) const {
         json = m_rfc2544_info.m_latency;
@@ -295,19 +336,23 @@ class CFlowStatUserIdInfoPayload : public CFlowStatUserIdInfo {
     }
 
     inline double get_jitter() const {
-        return m_rfc2544_info.m_jitter;
+        return m_rfc2544_info.get_jitter();
     }
 
     inline void set_jitter(double jitter) {
-        m_rfc2544_info.m_jitter = jitter;
+        m_rfc2544_info.set_jitter(jitter);
+    }
+
+    uint32_t get_jitter_usec(){
+        return m_rfc2544_info.get_jitter_usec();
     }
 
     inline void set_seq_err_cnt(uint64_t cnt) {
-        m_rfc2544_info.m_seq_error = cnt;
+        m_rfc2544_info.m_seq_err = cnt;
     }
 
     inline uint64_t get_seq_err_cnt() const {
-        return m_rfc2544_info.m_seq_error + m_seq_error_base;
+        return m_rfc2544_info.m_seq_err + m_seq_err_base;
     }
 
     inline void set_ooo_cnt(uint64_t cnt) {
@@ -318,11 +363,38 @@ class CFlowStatUserIdInfoPayload : public CFlowStatUserIdInfo {
         return m_rfc2544_info.m_out_of_order + m_out_of_order_base;
     }
 
-    inline void reset_hw_id();    
+    inline void set_dup_cnt(uint64_t cnt) {
+        m_rfc2544_info.m_dup = cnt;
+    }
+
+    inline uint64_t get_dup_cnt() const {
+        return m_rfc2544_info.m_dup + m_dup_base;
+    }
+
+    inline void set_seq_err_big_cnt(uint64_t cnt) {
+        m_rfc2544_info.m_seq_err_ev_big = cnt;
+    }
+
+    inline uint64_t get_seq_err_big_cnt() const {
+        return m_rfc2544_info.m_seq_err_ev_big + m_seq_err_ev_big_base;
+    }
+
+    inline void set_seq_err_low_cnt(uint64_t cnt) {
+        m_rfc2544_info.m_seq_err_ev_low = cnt;
+    }
+
+    inline uint64_t get_seq_err_low_cnt() const {
+        return m_rfc2544_info.m_seq_err_ev_low + m_seq_err_ev_low_base;
+    }
+
+    inline void reset_hw_id();
  private:
     rfc2544_info_t m_rfc2544_info;
-    uint64_t m_seq_error_base;
+    uint64_t m_seq_err_base;
     uint64_t m_out_of_order_base;
+    uint64_t m_dup_base;
+    uint64_t m_seq_err_ev_big_base;
+    uint64_t m_seq_err_ev_low_base;
 };
 
 class CFlowStatUserIdMap {
diff --git a/src/gtest/trex_stateless_gtest.cpp b/src/gtest/trex_stateless_gtest.cpp
index e9c71df9..35254713 100644
--- a/src/gtest/trex_stateless_gtest.cpp
+++ b/src/gtest/trex_stateless_gtest.cpp
@@ -2880,7 +2880,7 @@ TEST_F(basic_stl, vm_enable0) {
 }
 
 #if 0
-// does not work with valgrind, because we need to free the dp->rx message queue in simulation mode
+// todo: does not work with valgrind, because we need to free the dp->rx message queue in simulation mode
 TEST_F(basic_stl, vm_enable0_flow_stat) {
 
     CEnableVm vm_test;
@@ -2900,7 +2900,7 @@ TEST_F(basic_stl, vm_enable1) {
 }
 
 #if 0
-//??? does not work. need to check
+//todo: does not work. need to check
 TEST_F(basic_stl, vm_enable1_flow_stat) {
 
     CEnableVm vm_test;
diff --git a/src/stateless/rx/trex_stateless_rx_core.cpp b/src/stateless/rx/trex_stateless_rx_core.cpp
index 4a7c0b29..463fa7d3 100644
--- a/src/stateless/rx/trex_stateless_rx_core.cpp
+++ b/src/stateless/rx/trex_stateless_rx_core.cpp
@@ -32,12 +32,14 @@ void CRxCoreStateless::create(const CRxSlCfg &cfg) {
     for (int i = 0; i < MAX_FLOW_STATS_PAYLOAD; i++) {
         // This is the seq num value we expect next packet to have.
         // Init value should match m_seq_num in CVirtualIFPerSideStats
-        m_per_flow_seq[i] = UINT32_MAX - 1;  // catch wrap around issues early
-        m_per_flow_hist[i].Create();
-        m_per_flow_jitter[i].reset();
-        m_per_flow_seq_error[i] = 0;
-        m_per_flow_out_of_order[i] = 0;
-        m_per_flow_dup[i] = 0;
+        m_rfc2544[i].seq = UINT32_MAX - 1;  // catch wrap around issues early
+        m_rfc2544[i].latency.Create();
+        m_rfc2544[i].jitter.reset();
+        m_rfc2544[i].seq_err = 0;
+        m_rfc2544[i].seq_err_events_too_big = 0;
+        m_rfc2544[i].seq_err_events_too_low = 0;
+        m_rfc2544[i].out_of_order = 0;
+        m_rfc2544[i].dup = 0;
     }
 }
 
@@ -138,47 +140,51 @@ void CRxCoreStateless::handle_rx_pkt(CLatencyManagerPerPortStl *lp, rte_mbuf_t *
                     if (fsp_head->magic == FLOW_STAT_PAYLOAD_MAGIC) {
                         hw_id = fsp_head->hw_id;
                         seq = fsp_head->seq;
-                        if (unlikely(seq != m_per_flow_seq[hw_id])) {
-                            if (seq < m_per_flow_seq[hw_id]) {
-                                if (m_per_flow_seq[hw_id] - seq > 100000) {
+                        if (unlikely(seq != m_rfc2544[hw_id].seq)) {
+                            if (seq < m_rfc2544[hw_id].seq) {
+                                if (m_rfc2544[hw_id].seq - seq > 100000) {
                                     // packet loss while we had wrap around
-                                    m_per_flow_seq_error[hw_id] += seq - m_per_flow_seq[hw_id];
-                                    m_per_flow_seq[hw_id] = seq + 1;
+                                    m_rfc2544[hw_id].seq_err += seq - m_rfc2544[hw_id].seq;
+                                    m_rfc2544[hw_id].seq_err_events_too_big++;
+                                    m_rfc2544[hw_id].seq = seq + 1;
                                 } else {
-                                    if (seq == (m_per_flow_seq[hw_id] - 1)) {
-                                        m_per_flow_dup[hw_id] += 1;
+                                    if (seq == (m_rfc2544[hw_id].seq - 1)) {
+                                        m_rfc2544[hw_id].dup += 1;
                                     } else {
-                                        m_per_flow_out_of_order[hw_id] += 1;
+                                         m_rfc2544[hw_id].out_of_order += 1;
                                         // We thought it was lost, but it was just out of order
-                                        m_per_flow_seq_error[hw_id] -= 1;
+                                        m_rfc2544[hw_id].seq_err -= 1;
+                                        m_rfc2544[hw_id].seq_err_events_too_low++;
                                     }
                                 }
                             } else {
-                                if (unlikely (m_per_flow_seq[hw_id] - seq > 100000)) {
+                                if (unlikely (m_rfc2544[hw_id].seq - seq > 100000)) {
                                     // packet reorder while we had wrap around
-                                    if (seq == (m_per_flow_seq[hw_id] - 1)) {
-                                        m_per_flow_dup[hw_id] += 1;
+                                    if (seq == (m_rfc2544[hw_id].seq - 1)) {
+                                        m_rfc2544[hw_id].dup += 1;
                                     } else {
-                                        m_per_flow_out_of_order[hw_id] += 1;
+                                         m_rfc2544[hw_id].out_of_order += 1;
                                         // We thought it was lost, but it was just out of order
-                                        m_per_flow_seq_error[hw_id] -= 1;
+                                        m_rfc2544[hw_id].seq_err -= 1;
+                                        m_rfc2544[hw_id].seq_err_events_too_low++;
                                     }
                                 } else {
-                                // seq > m_per_flow_seq[hw_id]. Assuming lost packets
-                                    m_per_flow_seq_error[hw_id] += seq - m_per_flow_seq[hw_id];
-                                    m_per_flow_seq[hw_id] = seq + 1;
+                                // seq > m_rfc2544[hw_id].seq. Assuming lost packets
+                                    m_rfc2544[hw_id].seq_err += seq - m_rfc2544[hw_id].seq;
+                                    m_rfc2544[hw_id].seq_err_events_too_big++;
+                                    m_rfc2544[hw_id].seq = seq + 1;
                                 }
                             }
                         } else {
-                            m_per_flow_seq[hw_id] = seq + 1;
+                            m_rfc2544[hw_id].seq = seq + 1;
                         }
                         lp->m_port.m_rx_pg_stat_payload[hw_id].add_pkts(1);
                         lp->m_port.m_rx_pg_stat_payload[hw_id].add_bytes(m->pkt_len);
                         uint64_t d = (os_get_hr_tick_64() - fsp_head->time_stamp );
                         dsec_t ctime = ptime_convert_hr_dsec(d);
-                        m_per_flow_hist[hw_id].Add(ctime);
-                        m_per_flow_last_max[hw_id].update(ctime);
-                        m_per_flow_jitter[hw_id].calc(ctime);
+                        m_rfc2544[hw_id].latency.Add(ctime);
+                        m_rfc2544[hw_id].last_max.update(ctime);
+                        m_rfc2544[hw_id].jitter.calc(ctime);
                     }
                 } else {
                     hw_id = get_hw_id(ip_id);
@@ -331,20 +337,24 @@ int CRxCoreStateless::get_rfc2544_info(rfc2544_info_t *rfc2544_info, int min, in
     Json::Reader reader;
 
     for (int hw_id = min; hw_id <= max; hw_id++) {
-        rfc2544_info[hw_id - min].set_err_cntrs(m_per_flow_seq_error[hw_id], m_per_flow_out_of_order[hw_id]);
-        rfc2544_info[hw_id - min].set_jitter(m_per_flow_jitter[hw_id].get_jitter());
-        m_per_flow_hist[hw_id].update();
-        m_per_flow_hist[hw_id].dump_json("", json_str);
+        rfc2544_info[hw_id - min].set_err_cntrs(m_rfc2544[hw_id].seq_err, m_rfc2544[hw_id].out_of_order
+                                                , m_rfc2544[hw_id].dup, m_rfc2544[hw_id].seq_err_events_too_big
+                                                , m_rfc2544[hw_id].seq_err_events_too_low);
+        rfc2544_info[hw_id - min].set_jitter(m_rfc2544[hw_id].jitter.get_jitter());
+        m_rfc2544[hw_id].latency.update();
+        m_rfc2544[hw_id].latency.dump_json("", json_str);
         // This is a hack. We need to make the dump_json return json object.
         reader.parse( json_str.c_str(), json);
         rfc2544_info[hw_id - min].set_latency_json(json);
-        rfc2544_info[hw_id - min].set_last_max(m_per_flow_last_max[hw_id].switchMax());
+        rfc2544_info[hw_id - min].set_last_max(m_rfc2544[hw_id].last_max.switchMax());
 
         if (reset) {
-            m_per_flow_seq_error[hw_id] = 0;
-            m_per_flow_out_of_order[hw_id] = 0;
-            m_per_flow_hist[hw_id].Reset();
-            m_per_flow_jitter[hw_id].reset();
+            m_rfc2544[hw_id].seq_err = 0;
+            m_rfc2544[hw_id].seq_err_events_too_big = 0;
+            m_rfc2544[hw_id].seq_err_events_too_low = 0;
+            m_rfc2544[hw_id].out_of_order = 0;
+            m_rfc2544[hw_id].latency.Reset();
+            m_rfc2544[hw_id].jitter.reset();
         }
     }
     return 0;
diff --git a/src/stateless/rx/trex_stateless_rx_core.h b/src/stateless/rx/trex_stateless_rx_core.h
index 604ad260..abe8e754 100644
--- a/src/stateless/rx/trex_stateless_rx_core.h
+++ b/src/stateless/rx/trex_stateless_rx_core.h
@@ -109,6 +109,18 @@ class CRxSlCfg {
     CPortLatencyHWBase * m_ports[TREX_MAX_PORTS];
 };
 
+struct per_flow_rfc2544_info {
+    uint32_t seq; // expected next seq num
+    CTimeHistogram  latency; // latency info
+    CJitter         jitter; 
+    uint64_t seq_err; // How many packet seq num gaps we saw (packets lost or out of order)
+    uint64_t seq_err_events_too_big; // How many packet seq num greater than expected events we had
+    uint64_t seq_err_events_too_low; // How many packet seq num lower than expected events we had
+    uint64_t out_of_order; // Packets we got with seq num lower than expected
+    uint64_t dup; // Packets we got with same seq num
+    CLastMax last_max; // maximum for last measurement period (reset whenever we read it).
+};
+
 class CRxCoreStateless {
     enum state_e {
         STATE_IDLE,
@@ -156,12 +168,6 @@ class CRxCoreStateless {
     CCpuUtlCp m_cpu_cp_u;
     // Used for acking "work" (go out of idle) messages from cp
     volatile bool m_ack_start_work_msg __rte_cache_aligned;
-    uint32_t m_per_flow_seq[MAX_FLOW_STATS_PAYLOAD]; // expected next seq num
-    CTimeHistogram  m_per_flow_hist[MAX_FLOW_STATS_PAYLOAD]; /* latency info */
-    CJitter         m_per_flow_jitter[MAX_FLOW_STATS_PAYLOAD];
-    uint64_t m_per_flow_seq_error[MAX_FLOW_STATS_PAYLOAD]; // How many packet seq num gaps we saw (packets lost or out of order)
-    uint64_t m_per_flow_out_of_order[MAX_FLOW_STATS_PAYLOAD]; // Packets we got with seq num lower than expected
-    uint64_t m_per_flow_dup[MAX_FLOW_STATS_PAYLOAD]; // Packets we got with same seq num
-    CLastMax m_per_flow_last_max[MAX_FLOW_STATS_PAYLOAD];
+    struct per_flow_rfc2544_info m_rfc2544[MAX_FLOW_STATS_PAYLOAD];
 };
 #endif