From 4c83b9d445e68b3b1dba126648169a6cd05ac9b2 Mon Sep 17 00:00:00 2001 From: imarom Date: Mon, 20 Feb 2017 14:53:25 +0200 Subject: fix for RX CPU util issue: see: https://trex-tgn.cisco.com/youtrack/issue/trex-350 Signed-off-by: imarom --- src/stateless/rx/trex_stateless_rx_core.cpp | 174 +++++++++++++++------------- src/stateless/rx/trex_stateless_rx_core.h | 30 +++-- 2 files changed, 117 insertions(+), 87 deletions(-) diff --git a/src/stateless/rx/trex_stateless_rx_core.cpp b/src/stateless/rx/trex_stateless_rx_core.cpp index b75b0e8f..03351036 100644 --- a/src/stateless/rx/trex_stateless_rx_core.cpp +++ b/src/stateless/rx/trex_stateless_rx_core.cpp @@ -76,7 +76,7 @@ void CRxCoreStateless::create(const CRxSlCfg &cfg) { CMessagingManager * cp_rx = CMsgIns::Ins()->getCpRx(); m_ring_from_cp = cp_rx->getRingCpToDp(0); - m_state = STATE_IDLE; + m_state = STATE_COLD; for (int i = 0; i < MAX_FLOW_STATS_PAYLOAD; i++) { m_rfc2544[i].create(); @@ -132,48 +132,23 @@ bool CRxCoreStateless::periodic_check_for_cp_messages() { } -void -CRxCoreStateless::periodic_check_for_dp_messages() { - - for (int i = 0; i < m_tx_cores; i++) { - periodic_check_for_dp_messages_core(i); - } - -} - -void -CRxCoreStateless::periodic_check_for_dp_messages_core(uint32_t core_id) { - - CNodeRing *ring = CMsgIns::Ins()->getRxDp()->getRingDpToCp(core_id); - - /* fast path */ - if ( likely ( ring->isEmpty() ) ) { - return; - } - - while (true) { - CGenNode *node = NULL; - - if (ring->Dequeue(node) != 0) { - break; - } - - //assert(node); - } -} void CRxCoreStateless::recalculate_next_state() { if (m_state == STATE_QUIT) { return; } - /* next state is determine by the question are there any ports with active features ? */ - m_state = (are_any_features_active() ? STATE_WORKING : STATE_IDLE); + /* only latency requires the 'hot' state */ + m_state = (is_latency_active() ? STATE_HOT : STATE_COLD); } -bool CRxCoreStateless::are_any_features_active() { + +/** + * return true if any port has latency enabled + */ +bool CRxCoreStateless::is_latency_active() { for (int i = 0; i < m_max_ports; i++) { - if (m_rx_port_mngr[i].has_features_set()) { + if (m_rx_port_mngr[i].is_feature_set(RXPortManager::LATENCY)) { return true; } } @@ -181,71 +156,107 @@ bool CRxCoreStateless::are_any_features_active() { return false; } -void CRxCoreStateless::idle_state_loop() { - const int SHORT_DELAY_MS = 2; - const int LONG_DELAY_MS = 50; - const int DEEP_SLEEP_LIMIT = 2000; - int counter = 0; +/** + * when in hot state we busy poll as fast as possible + * because of latency packets + * + */ +void CRxCoreStateless::hot_state_loop() { + + while (m_state == STATE_HOT) { + work_tick(); + rte_pause(); + } +} + - while (m_state == STATE_IDLE) { - bool had_msg = periodic_check_for_cp_messages(); - if (had_msg) { +/** + * on cold state loop we adapt to work actually done + * + */ +void CRxCoreStateless::cold_state_loop() { + const uint32_t COLD_LIMIT_ITER = 1000; + const uint32_t COLD_SLEEP_MS = 10; + + int counter = 0; + + while (m_state == STATE_COLD) { + bool did_something = work_tick(); + if (did_something) { counter = 0; + /* we are hot - continue with no delay */ continue; - } else { - flush_all_pending_pkts(); } - - /* enter deep sleep only if enough time had passed */ - if (counter < DEEP_SLEEP_LIMIT) { - delay(SHORT_DELAY_MS); + + if (counter < COLD_LIMIT_ITER) { + /* hot stage */ counter++; + rte_pause(); } else { - delay(LONG_DELAY_MS); + /* cold stage */ + delay(COLD_SLEEP_MS); } + } } /** - * for each port handle the grat ARP mechansim + * init the 'IF' scheduler values * + * @author imarom (2/20/2017) */ -void CRxCoreStateless::handle_grat_arp() { - for (int i = 0; i < m_max_ports; i++) { - m_rx_port_mngr[i].send_next_grat_arp(); - } -} - -void CRxCoreStateless::handle_work_stage() { +void CRxCoreStateless::init_work_stage() { /* set the next sync time to */ - dsec_t sync_time_sec = now_sec() + (1.0 / 1000); - dsec_t grat_arp_sec = now_sec() + (double)CGlobalInfo::m_options.m_arp_ref_per; - - while (m_state == STATE_WORKING) { - process_all_pending_pkts(); - - dsec_t now = now_sec(); + m_sync_time_sec = now_sec() + (1.0 / 1000); + m_grat_arp_sec = now_sec() + (double)CGlobalInfo::m_options.m_arp_ref_per; +} + +/** + * performs once tick of work + * return true if anything was done + */ +bool CRxCoreStateless::work_tick() { + bool did_something = false; + + /* if any packet arrived - mark as */ + if (process_all_pending_pkts()) { + did_something = true; + } + + dsec_t now = now_sec(); - /* until a scheduler is added here - dirty IFs */ + /* until a scheduler is added here - dirty IFs */ + + if ( (now - m_sync_time_sec) > 0 ) { - if ( (now - sync_time_sec) > 0 ) { - periodic_check_for_cp_messages(); - //periodic_check_for_dp_messages(); - sync_time_sec = now + (1.0 / 1000); + if (periodic_check_for_cp_messages()) { + did_something = true; } - if ( (now - grat_arp_sec) > 0) { - handle_grat_arp(); - grat_arp_sec = now + (double)CGlobalInfo::m_options.m_arp_ref_per; - } - - rte_pause(); + m_sync_time_sec = now + (1.0 / 1000); + } + if ( (now - m_grat_arp_sec) > 0) { + handle_grat_arp(); + m_grat_arp_sec = now + (double)CGlobalInfo::m_options.m_arp_ref_per; } + + return did_something; } +/** + * for each port handle the grat ARP mechansim + * + */ +void CRxCoreStateless::handle_grat_arp() { + for (int i = 0; i < m_max_ports; i++) { + m_rx_port_mngr[i].send_next_grat_arp(); + } +} + + void CRxCoreStateless::start() { /* register a watchdog handle on current core */ m_monitor.create("STL RX CORE", 1); @@ -253,13 +264,18 @@ void CRxCoreStateless::start() { recalculate_next_state(); + init_work_stage(); + while (m_state != STATE_QUIT) { + switch (m_state) { - case STATE_IDLE: - idle_state_loop(); + + case STATE_HOT: + hot_state_loop(); break; - case STATE_WORKING: - handle_work_stage(); + + case STATE_COLD: + cold_state_loop(); break; default: diff --git a/src/stateless/rx/trex_stateless_rx_core.h b/src/stateless/rx/trex_stateless_rx_core.h index 954a5f04..8166ef52 100644 --- a/src/stateless/rx/trex_stateless_rx_core.h +++ b/src/stateless/rx/trex_stateless_rx_core.h @@ -105,11 +105,19 @@ class CRxCoreErrCntrs { class CRxCoreStateless { /** - * core states + * core states + * + * STATE_IDLE - only checking for messages periodically + * STATE_COLD - will sleep until a packet arrives + * then it will move to a faster pace + * until no packet arrives for some time + * + * STATE_HOT - 100% checking for packets (latency check) */ enum state_e { STATE_IDLE, - STATE_WORKING, + STATE_COLD, + STATE_HOT, STATE_QUIT }; @@ -124,7 +132,7 @@ class CRxCoreStateless { void quit() {m_state = STATE_QUIT;} - bool is_working() const {return (m_state == STATE_WORKING);} + bool is_working() const {return (m_state != STATE_QUIT);} double get_cpu_util(); void update_cpu_util(); @@ -159,14 +167,17 @@ class CRxCoreStateless { bool periodic_check_for_cp_messages(); - void periodic_check_for_dp_messages(); - void periodic_check_for_dp_messages_core(uint32_t core_id); - void tickle(); - void idle_state_loop(); + /* states */ + //void idle_state_loop(); + void hot_state_loop(); + void cold_state_loop(); + void init_work_stage(); + bool work_tick(); + void recalculate_next_state(); - bool are_any_features_active(); + bool is_latency_active(); void handle_rx_queue_msgs(uint8_t thread_id, CNodeRing * r); void handle_work_stage(); @@ -190,6 +201,9 @@ class CRxCoreStateless { CCpuUtlDp m_cpu_dp_u; CCpuUtlCp m_cpu_cp_u; + dsec_t m_sync_time_sec; + dsec_t m_grat_arp_sec; + // Used for acking "work" (go out of idle) messages from cp volatile bool m_ack_start_work_msg __rte_cache_aligned; -- cgit 1.2.3-korg