fix for RX CPU util issue:

see: https://trex-tgn.cisco.com/youtrack/issue/trex-350

Signed-off-by: imarom <imarom@cisco.com>

2 files changed, 117 insertions, 87 deletions

diff --git a/src/stateless/rx/trex_stateless_rx_core.cpp b/src/stateless/rx/trex_stateless_rx_core.cpp
index b75b0e8f..03351036 100644
--- a/src/stateless/rx/trex_stateless_rx_core.cpp
+++ b/src/stateless/rx/trex_stateless_rx_core.cpp
@@ -76,7 +76,7 @@ void CRxCoreStateless::create(const CRxSlCfg &cfg) {
     CMessagingManager * cp_rx = CMsgIns::Ins()->getCpRx();
 
     m_ring_from_cp = cp_rx->getRingCpToDp(0);
-    m_state = STATE_IDLE;
+    m_state = STATE_COLD;
 
     for (int i = 0; i < MAX_FLOW_STATS_PAYLOAD; i++) {
         m_rfc2544[i].create();
@@ -132,48 +132,23 @@ bool CRxCoreStateless::periodic_check_for_cp_messages() {
 
 }
 
-void
-CRxCoreStateless::periodic_check_for_dp_messages() {
-
-    for (int i = 0; i < m_tx_cores; i++) {
-        periodic_check_for_dp_messages_core(i);
-    }
-    
-}
-
-void
-CRxCoreStateless::periodic_check_for_dp_messages_core(uint32_t core_id) {
-
-    CNodeRing *ring = CMsgIns::Ins()->getRxDp()->getRingDpToCp(core_id);
-    
-    /* fast path */
-    if ( likely ( ring->isEmpty() ) ) {
-        return;
-    }
-
-    while (true) {
-        CGenNode *node = NULL;
-
-        if (ring->Dequeue(node) != 0) {
-            break;
-        }
-        
-        //assert(node);
-    }
-}
 
 void CRxCoreStateless::recalculate_next_state() {
     if (m_state == STATE_QUIT) {
         return;
     }
 
-    /* next state is determine by the question are there any ports with active features ? */
-    m_state = (are_any_features_active() ? STATE_WORKING : STATE_IDLE);
+    /* only latency requires the 'hot' state */
+    m_state = (is_latency_active() ? STATE_HOT : STATE_COLD);
 }
 
-bool CRxCoreStateless::are_any_features_active() {
+
+/**
+ * return true if any port has latency enabled
+ */
+bool CRxCoreStateless::is_latency_active() {
     for (int i = 0; i < m_max_ports; i++) {
-        if (m_rx_port_mngr[i].has_features_set()) {
+        if (m_rx_port_mngr[i].is_feature_set(RXPortManager::LATENCY)) {
             return true;
         }
     }
@@ -181,71 +156,107 @@ bool CRxCoreStateless::are_any_features_active() {
     return false;
 }
 
-void CRxCoreStateless::idle_state_loop() {
-    const int SHORT_DELAY_MS    = 2;
-    const int LONG_DELAY_MS     = 50;
-    const int DEEP_SLEEP_LIMIT  = 2000;
 
-    int counter = 0;
+/**
+ * when in hot state we busy poll as fast as possible 
+ * because of latency packets 
+ * 
+ */
+void CRxCoreStateless::hot_state_loop() {
+    
+    while (m_state == STATE_HOT) {
+        work_tick();
+        rte_pause();
+    }
+}
+
 
-    while (m_state == STATE_IDLE) {
-        bool had_msg = periodic_check_for_cp_messages();
-        if (had_msg) {
+/**
+ * on cold state loop we adapt to work actually done
+ * 
+ */
+void CRxCoreStateless::cold_state_loop() {
+    const uint32_t COLD_LIMIT_ITER = 1000;
+    const uint32_t COLD_SLEEP_MS  = 10;
+
+    int counter = 0;
+ 
+    while (m_state == STATE_COLD) {
+        bool did_something = work_tick();
+        if (did_something) {
             counter = 0;
+            /* we are hot - continue with no delay */
             continue;
-        } else {
-            flush_all_pending_pkts();
         }
-
-        /* enter deep sleep only if enough time had passed */
-        if (counter < DEEP_SLEEP_LIMIT) {
-            delay(SHORT_DELAY_MS);
+        
+        if (counter < COLD_LIMIT_ITER) {
+            /* hot stage */
             counter++;
+            rte_pause();
         } else {
-            delay(LONG_DELAY_MS);
+            /* cold stage */
+            delay(COLD_SLEEP_MS);
         }
+        
     }
 }
 
 /**
- * for each port handle the grat ARP mechansim
+ * init the 'IF' scheduler values
  * 
+ * @author imarom (2/20/2017)
  */
-void CRxCoreStateless::handle_grat_arp() {
-    for (int i = 0; i < m_max_ports; i++) {
-        m_rx_port_mngr[i].send_next_grat_arp();
-    }
-}
-
-void CRxCoreStateless::handle_work_stage() {
+void CRxCoreStateless::init_work_stage() {
     
     /* set the next sync time to */
-    dsec_t sync_time_sec = now_sec() + (1.0 / 1000);
-    dsec_t grat_arp_sec  = now_sec() + (double)CGlobalInfo::m_options.m_arp_ref_per;
-
-    while (m_state == STATE_WORKING) {
-        process_all_pending_pkts();
-
-        dsec_t now = now_sec();
+    m_sync_time_sec = now_sec() + (1.0 / 1000);
+    m_grat_arp_sec  = now_sec() + (double)CGlobalInfo::m_options.m_arp_ref_per;
+}
+    
+/**
+ * performs once tick of work
+ * return true if anything was done
+ */
+bool CRxCoreStateless::work_tick() {
+    bool did_something = false;
+    
+    /* if any packet arrived - mark as */
+    if (process_all_pending_pkts()) {
+        did_something = true;
+    }
+    
+    dsec_t now = now_sec();
         
-        /* until a scheduler is added here - dirty IFs */
+    /* until a scheduler is added here - dirty IFs */
+
+    if ( (now - m_sync_time_sec) > 0 ) {
 
-        if ( (now - sync_time_sec) > 0 ) {
-            periodic_check_for_cp_messages();
-            //periodic_check_for_dp_messages();
-            sync_time_sec = now + (1.0 / 1000);
+        if (periodic_check_for_cp_messages()) {
+            did_something = true;
         }
         
-        if ( (now - grat_arp_sec) > 0) {
-            handle_grat_arp();
-            grat_arp_sec = now + (double)CGlobalInfo::m_options.m_arp_ref_per;
-        }
-
-        rte_pause();
+        m_sync_time_sec = now + (1.0 / 1000);
+    }
         
+    if ( (now - m_grat_arp_sec) > 0) {
+        handle_grat_arp();
+        m_grat_arp_sec = now + (double)CGlobalInfo::m_options.m_arp_ref_per;
     }
+    
+    return did_something;
 }
 
+/**
+ * for each port handle the grat ARP mechansim
+ * 
+ */
+void CRxCoreStateless::handle_grat_arp() {
+    for (int i = 0; i < m_max_ports; i++) {
+        m_rx_port_mngr[i].send_next_grat_arp();
+    }
+}
+
+
 void CRxCoreStateless::start() {
     /* register a watchdog handle on current core */
     m_monitor.create("STL RX CORE", 1);
@@ -253,13 +264,18 @@ void CRxCoreStateless::start() {
 
     recalculate_next_state();
     
+    init_work_stage();
+    
     while (m_state != STATE_QUIT) {
+
         switch (m_state) {
-        case STATE_IDLE:
-            idle_state_loop();
+
+        case STATE_HOT:
+            hot_state_loop();
             break;
-        case STATE_WORKING:
-            handle_work_stage();
+            
+        case STATE_COLD:
+            cold_state_loop();
             break;
 
         default:
diff --git a/src/stateless/rx/trex_stateless_rx_core.h b/src/stateless/rx/trex_stateless_rx_core.h
index 954a5f04..8166ef52 100644
--- a/src/stateless/rx/trex_stateless_rx_core.h
+++ b/src/stateless/rx/trex_stateless_rx_core.h
@@ -105,11 +105,19 @@ class CRxCoreErrCntrs {
 class CRxCoreStateless {
     
     /**
-     * core states
+     * core states 
+     *  
+     * STATE_IDLE - only checking for messages periodically 
+     * STATE_COLD - will sleep until a packet arrives 
+     *              then it will move to a faster pace
+     *              until no packet arrives for some time
+     *  
+     * STATE_HOT  - 100% checking for packets (latency check)
      */
     enum state_e {
         STATE_IDLE,
-        STATE_WORKING,
+        STATE_COLD,
+        STATE_HOT,
         STATE_QUIT
     };
 
@@ -124,7 +132,7 @@ class CRxCoreStateless {
 
 
     void quit() {m_state = STATE_QUIT;}
-    bool is_working() const {return (m_state == STATE_WORKING);}
+    bool is_working() const {return (m_state != STATE_QUIT);}
     double get_cpu_util();
     void update_cpu_util();
 
@@ -159,14 +167,17 @@ class CRxCoreStateless {
 
     bool periodic_check_for_cp_messages();
 
-    void periodic_check_for_dp_messages();
-    void periodic_check_for_dp_messages_core(uint32_t core_id);
-    
     void tickle();
-    void idle_state_loop();
 
+    /* states */
+    //void idle_state_loop();
+    void hot_state_loop();
+    void cold_state_loop();
+    void init_work_stage();
+    bool work_tick();
+    
     void recalculate_next_state();
-    bool are_any_features_active();
+    bool is_latency_active();
 
     void handle_rx_queue_msgs(uint8_t thread_id, CNodeRing * r);
     void handle_work_stage();
@@ -190,6 +201,9 @@ class CRxCoreStateless {
     CCpuUtlDp        m_cpu_dp_u;
     CCpuUtlCp        m_cpu_cp_u;
 
+    dsec_t           m_sync_time_sec;
+    dsec_t           m_grat_arp_sec;
+    
     // Used for acking "work" (go out of idle) messages from cp
     volatile bool m_ack_start_work_msg __rte_cache_aligned;