add not accurate timer wheel for better performance

Signed-off-by: Hanoh Haim <hhaim@cisco.com>

31 files changed, 845 insertions, 17 deletions

diff --git a/.gitignore b/.gitignore
index b3c8cfee..a8fb3f8b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -97,6 +97,26 @@ scripts/5.cap
 scripts/6.cap
 scripts/7.cap
 scripts/8.cap
+scripts/exp/rtsp_short1-0.erf
+scripts/exp/rtsp_short1_ipv6_rxcheck.erf
+scripts/exp/rtsp_short1_rxcheck.erf
+scripts/exp/rtsp_short1_v6-0.erf
+scripts/exp/rtsp_short2-0.erf
+scripts/exp/rtsp_short2_v6-0.erf
+scripts/exp/rtsp_short3-0.erf
+scripts/exp/rtsp_short3_v6-0.erf
+scripts/exp/sip_short1-0.erf
+scripts/exp/sip_short1_v6-0.erf
+scripts/exp/sip_short2-0.erf
+scripts/exp/sip_short2_v6-0.erf
+scripts/exp/sip_short3-0.erf
+scripts/exp/sip_short3_v6-0.erf
+scripts/exp/imix-0.erf
+scripts/exp/imix_v6-0.erf
+scripts/exp/limit_single_pkt-0.erf
+scripts/exp/pcap_mode1-0.erf
+
+
 
 
 
diff --git a/doc/visio_drawings/tw.xlsx b/doc/visio_drawings/tw.xlsx
index 2ddc86d3..31e69fe4 100644
--- a/doc/visio_drawings/tw.xlsx
+++ b/doc/visio_drawings/tw.xlsx
diff --git a/scripts/automation/trex_control_plane/stf/examples/stf_active_flow.py b/scripts/automation/trex_control_plane/stf/examples/stf_active_flow.py
index 0a72c9ac..8560a5db 100644
--- a/scripts/automation/trex_control_plane/stf/examples/stf_active_flow.py
+++ b/scripts/automation/trex_control_plane/stf/examples/stf_active_flow.py
@@ -14,8 +14,8 @@ def minimal_stateful_test(server,csv_file,a_active_flows):
     trex_client.start_trex(
             c = 7,
             m = 30000,
-#            f = 'cap2/cur_flow_single.yaml',
-            f = 'cap2/cur_flow.yaml',
+            f = 'cap2/cur_flow_single.yaml',
+#            f = 'cap2/cur_flow.yaml',
             d = 30,
             l = 1000,
             p=True,
@@ -39,6 +39,7 @@ def minimal_stateful_test(server,csv_file,a_active_flows):
         print("WARNING QUEU WAS FULL");
 
     tuple=(active_flows[-5],cpu_utl[-5],pps[-5],queue_full[-1])
+    print(tuple)
     file_writer = csv.writer(test_file)
     file_writer.writerow(tuple);
     
@@ -58,7 +59,7 @@ if __name__ == '__main__':
     max_flows = 8000000;
     min_flows = 100;
     active_flow = min_flows;
-    num_point = 10
+    num_point = 40
     factor = math.exp(math.log(max_flows/min_flows,math.e)/num_point);
     for i in range(num_point+1):
         print("<<=====================>>",i,math.floor(active_flow))
diff --git a/scripts/cfg/kiwi02_more_flows.yaml b/scripts/cfg/kiwi02_more_flows.yaml
index a156d4f4..42735fdb 100644
--- a/scripts/cfg/kiwi02_more_flows.yaml
+++ b/scripts/cfg/kiwi02_more_flows.yaml
@@ -10,6 +10,18 @@
                threads  : [1,2,3,4]
              - socket   : 1
                threads  : [8,9,10,11]
+  port_info:
+                 - dest_mac        :   [0x00,0x00,0x00,0x01,0x00,0x00]
+                   src_mac         :   [0x00,0x00,0x00,0x01,0x00,0x00]
+
+                 - dest_mac        :   [0x00,0x00,0x00,0x01,0x00,0x00]
+                   src_mac         :   [0x00,0x00,0x00,0x01,0x00,0x00]
+
+                 - dest_mac        :   [0x00,0x00,0x00,0x01,0x00,0x00]
+                   src_mac         :   [0x00,0x00,0x00,0x01,0x00,0x00]
+
+                 - dest_mac        :   [0x00,0x00,0x00,0x01,0x00,0x00]
+                   src_mac         :   [0x00,0x00,0x00,0x01,0x00,0x00]
 
   memory    :                                          
         dp_flows    : 4048576
diff --git a/scripts/exp/imix-0-ex.erf b/scripts/exp/imix-0-ex.erf
index 6f5f4f07..b044cb70 100755
--- a/scripts/exp/imix-0-ex.erf
+++ b/scripts/exp/imix-0-ex.erf
diff --git a/scripts/exp/imix-0.erf b/scripts/exp/imix-0.erf
index 6f5f4f07..b044cb70 100644
--- a/scripts/exp/imix-0.erf
+++ b/scripts/exp/imix-0.erf
diff --git a/scripts/exp/imix_v6-0-ex.erf b/scripts/exp/imix_v6-0-ex.erf
index d52d8541..589300c2 100755
--- a/scripts/exp/imix_v6-0-ex.erf
+++ b/scripts/exp/imix_v6-0-ex.erf
diff --git a/scripts/exp/imix_v6-0.erf b/scripts/exp/imix_v6-0.erf
index d52d8541..589300c2 100644
--- a/scripts/exp/imix_v6-0.erf
+++ b/scripts/exp/imix_v6-0.erf
diff --git a/scripts/exp/limit_single_pkt-0-ex.erf b/scripts/exp/limit_single_pkt-0-ex.erf
index 344a25d0..0ec672fa 100755
--- a/scripts/exp/limit_single_pkt-0-ex.erf
+++ b/scripts/exp/limit_single_pkt-0-ex.erf
diff --git a/scripts/exp/limit_single_pkt-0.erf b/scripts/exp/limit_single_pkt-0.erf
index 344a25d0..0ec672fa 100644
--- a/scripts/exp/limit_single_pkt-0.erf
+++ b/scripts/exp/limit_single_pkt-0.erf
diff --git a/scripts/exp/pcap_mode1-0.erf b/scripts/exp/pcap_mode1-0.erf
index 72e14bef..1f043784 100644
--- a/scripts/exp/pcap_mode1-0.erf
+++ b/scripts/exp/pcap_mode1-0.erf
diff --git a/scripts/exp/pcap_mode2-0-ex.erf b/scripts/exp/pcap_mode2-0-ex.erf
index dde7d7e2..3f6ff4c9 100755
--- a/scripts/exp/pcap_mode2-0-ex.erf
+++ b/scripts/exp/pcap_mode2-0-ex.erf
diff --git a/scripts/exp/rtsp_short1-0.erf b/scripts/exp/rtsp_short1-0.erf
index d75f8f98..990edbeb 100644
--- a/scripts/exp/rtsp_short1-0.erf
+++ b/scripts/exp/rtsp_short1-0.erf
diff --git a/scripts/exp/rtsp_short1_ipv6_rxcheck.erf b/scripts/exp/rtsp_short1_ipv6_rxcheck.erf
index 046e0a1e..a06fe3a7 100644
--- a/scripts/exp/rtsp_short1_ipv6_rxcheck.erf
+++ b/scripts/exp/rtsp_short1_ipv6_rxcheck.erf
diff --git a/scripts/exp/rtsp_short1_rxcheck.erf b/scripts/exp/rtsp_short1_rxcheck.erf
index dc195ac0..b5f17783 100644
--- a/scripts/exp/rtsp_short1_rxcheck.erf
+++ b/scripts/exp/rtsp_short1_rxcheck.erf
diff --git a/scripts/exp/rtsp_short1_v6-0.erf b/scripts/exp/rtsp_short1_v6-0.erf
index ba220161..2cb322ac 100644
--- a/scripts/exp/rtsp_short1_v6-0.erf
+++ b/scripts/exp/rtsp_short1_v6-0.erf
diff --git a/scripts/exp/rtsp_short2-0.erf b/scripts/exp/rtsp_short2-0.erf
index d75f8f98..990edbeb 100644
--- a/scripts/exp/rtsp_short2-0.erf
+++ b/scripts/exp/rtsp_short2-0.erf
diff --git a/scripts/exp/rtsp_short2_v6-0.erf b/scripts/exp/rtsp_short2_v6-0.erf
index ba220161..2cb322ac 100644
--- a/scripts/exp/rtsp_short2_v6-0.erf
+++ b/scripts/exp/rtsp_short2_v6-0.erf
diff --git a/scripts/exp/rtsp_short3-0.erf b/scripts/exp/rtsp_short3-0.erf
index 57668046..1b178df9 100644
--- a/scripts/exp/rtsp_short3-0.erf
+++ b/scripts/exp/rtsp_short3-0.erf
diff --git a/scripts/exp/rtsp_short3_v6-0.erf b/scripts/exp/rtsp_short3_v6-0.erf
index 49fa4c20..048c3b65 100644
--- a/scripts/exp/rtsp_short3_v6-0.erf
+++ b/scripts/exp/rtsp_short3_v6-0.erf
diff --git a/scripts/exp/sip_short1-0.erf b/scripts/exp/sip_short1-0.erf
index ee1ddd13..4f8f33ce 100644
--- a/scripts/exp/sip_short1-0.erf
+++ b/scripts/exp/sip_short1-0.erf
diff --git a/scripts/exp/sip_short1_v6-0.erf b/scripts/exp/sip_short1_v6-0.erf
index 573b5b8d..7ad1ad14 100644
--- a/scripts/exp/sip_short1_v6-0.erf
+++ b/scripts/exp/sip_short1_v6-0.erf
diff --git a/scripts/exp/sip_short2-0.erf b/scripts/exp/sip_short2-0.erf
index ee1ddd13..4f8f33ce 100644
--- a/scripts/exp/sip_short2-0.erf
+++ b/scripts/exp/sip_short2-0.erf
diff --git a/scripts/exp/sip_short2_v6-0.erf b/scripts/exp/sip_short2_v6-0.erf
index 573b5b8d..7ad1ad14 100644
--- a/scripts/exp/sip_short2_v6-0.erf
+++ b/scripts/exp/sip_short2_v6-0.erf
diff --git a/scripts/exp/sip_short3-0.erf b/scripts/exp/sip_short3-0.erf
index 5f21788d..0d6c4796 100644
--- a/scripts/exp/sip_short3-0.erf
+++ b/scripts/exp/sip_short3-0.erf
diff --git a/scripts/exp/sip_short3_v6-0.erf b/scripts/exp/sip_short3_v6-0.erf
index 35b95903..321754d1 100644
--- a/scripts/exp/sip_short3_v6-0.erf
+++ b/scripts/exp/sip_short3_v6-0.erf
diff --git a/src/bp_sim.cpp b/src/bp_sim.cpp
index 938d8f65..5950a80d 100755
--- a/src/bp_sim.cpp
+++ b/src/bp_sim.cpp
@@ -3413,7 +3413,7 @@ bool  CNodeGenerator::Create(CFlowGenListPerThread  *  parent){
    m_socket_id =0;
    m_realtime_his.Create();
    m_last_sync_time_sec = 0;
-
+   m_tw_level1_next_sec = 0;
    return(true);
 }
 
@@ -3527,7 +3527,7 @@ bool CFlowGenListPerThread::Create(uint32_t           thread_id,
                                                  0 ,
                                                  socket_id);
 
-     RC_HTW_t tw_res=m_tw.Create(TW_BUCKETS,TW_LEVELS); 
+     RC_HTW_t tw_res=m_tw.Create(TW_BUCKETS,TW_BUCKETS_LEVEL1_DIV); 
      if (tw_res != RC_HTW_OK){
          CHTimerWheelErrorStr err(tw_res);
          printf("Timer wheel configuration error,please look into the manual for details  \n");
@@ -3934,7 +3934,7 @@ inline bool CNodeGenerator::do_work_both(CGenNode * node,
             /* update bucket time */
             thread->m_cur_time_sec = node->m_time;
             if ( ON_TERMINATE ) {
-                thread->m_tw.on_tick((void*)thread,tw_on_tick_per_thread_cb_always);
+                thread->m_tw.on_tick_level0((void*)thread,tw_on_tick_per_thread_cb_always);
                 if ( thread->m_tw.is_any_events_left() ){
                     node->m_time += BUCKET_TIME_SEC;
                     m_p_queue.push(node);
@@ -3942,7 +3942,7 @@ inline bool CNodeGenerator::do_work_both(CGenNode * node,
                     thread->free_node(node);
                 }
             }else{
-                thread->m_tw.on_tick((void*)thread,tw_on_tick_per_thread_cb);
+                thread->m_tw.on_tick_level0((void*)thread,tw_on_tick_per_thread_cb);
                 node->m_time += BUCKET_TIME_SEC;;
                 m_p_queue.push(node);
             }
@@ -4058,6 +4058,7 @@ inline int CNodeGenerator::flush_file_realtime(dsec_t max_time,
     }else{
         add_exit_node(thread,max_time);
     }
+    m_scheduler_offset = offset;
 
     thread->m_cpu_dp_u.start_work1();
 
@@ -4169,6 +4170,8 @@ void CNodeGenerator::handle_time_strech(CGenNode * &node,
     /* fix the time offset */
     dsec_t dt = cur_time - n_time;
     offset += dt;
+    /* set new offset */
+    m_scheduler_offset = offset;
 
     /* check if flow sync message was delayed too much */
     if ( (cur_time - m_last_sync_time_sec) > SYNC_TIME_OUT ) {
@@ -4242,6 +4245,59 @@ int CNodeGenerator::flush_file(dsec_t max_time,
 }
 
 
+void CNodeGenerator::handle_batch_tw_level1(CGenNode *node, 
+                                            CFlowGenListPerThread *thread,
+                                            bool &exit_scheduler,
+                                            bool on_terminate) {
+
+    m_p_queue.pop();
+    /* update bucket time */
+    thread->m_cur_time_sec = node->m_time;
+
+    bool stop_loop=false;
+
+    while (!stop_loop) {
+        na_htw_state_num_t tw_state = thread->m_tw.on_tick_level1((void*)thread,tw_on_tick_per_thread_cb);
+        if ( (tw_state == TW_FIRST_FINISH) || (tw_state == TW_FIRST_FINISH_ANY)){
+            node->m_time += BUCKET_TIME_SEC_LEVEL1;
+            stop_loop=true;
+        }else{
+            switch (tw_state) {
+            case TW_FIRST_BATCH:
+                m_tw_level1_next_sec = node->m_time + BUCKET_TIME_SEC_LEVEL1;
+                node->m_time = now_sec()-m_scheduler_offset; /* spread if we can */
+                if (m_tw_level1_next_sec+m_scheduler_offset > now_sec() ) {
+                    stop_loop=true;
+                }
+                break;
+            case TW_NEXT_BATCH :
+                node->m_time = now_sec()-m_scheduler_offset; /* spread if we can */
+                if (m_tw_level1_next_sec+m_scheduler_offset > now_sec() ) {
+                    stop_loop=true;
+                }
+                break;
+            case TW_END_BATCH:
+                if (m_tw_level1_next_sec+m_scheduler_offset > now_sec() ) {
+                    node->m_time = m_tw_level1_next_sec;
+                }else{
+                    node->m_time = m_tw_level1_next_sec;  /* too late but we don't have anyting to do  */
+                }
+                stop_loop=true;
+              break;
+            default:
+                assert(0);
+            };
+        }
+    }
+
+    if ( on_terminate &&  
+         (thread->m_tw.is_any_events_left()==false) ){
+        thread->free_node(node);
+    }else{
+        m_p_queue.push(node);
+    }
+}
+
 
 void CNodeGenerator::handle_flow_pkt(CGenNode *node, CFlowGenListPerThread *thread) {
 
@@ -4387,6 +4443,10 @@ CNodeGenerator::handle_slow_messages(uint8_t type,
         handle_command(node, thread, exit_scheduler);
         break;
 
+    case CGenNode::TW_SYNC1:
+        handle_batch_tw_level1(node, thread, exit_scheduler,on_terminate);
+        break;
+
     default:
         assert(0);
     }
@@ -4677,6 +4737,26 @@ void CFlowGenListPerThread::handle_nat_msg(CGenNodeNatInfo * msg){
     }
 }
 
+
+void   CFlowGenListPerThread::no_memory_error(){
+    printf("--------\n");
+    printf("\n");
+    printf("\n");
+    printf("ERROR, not enough flow objects, try to enlarge the number of objects in trex_cfg file or reduce the bandwidth \n");
+    printf("See in the manual how to enlarge the number of objects.\n");
+    printf("\n");
+    printf("\n");
+    printf(" Check your active flows, 'Active-flows    :  6771863', If it too high reduce the multiplier \n");
+    printf(" or use --active-flows directive to reduce the number of flows\n");
+    printf(" If you don't have enough memory for flows you should add something like that in your config file            \n");                                
+    printf("\n");
+    printf(" memory    :             \n");                                
+    printf("      dp_flows    : 4048576 \n");
+    printf("--------\n");
+    exit(1);
+}
+
+
 bool CFlowGenListPerThread::check_msgs_from_rx() {
     if ( likely ( m_ring_from_rx->isEmpty() ) ) {
         return false;
@@ -4825,6 +4905,11 @@ void CFlowGenListPerThread::start_generate_stateful(std::string erf_file_name,
         node->m_type = CGenNode::TW_SYNC;
         node->m_time = m_cur_time_sec + BUCKET_TIME_SEC ;
         m_node_gen.add_node(node);
+
+        node= create_node() ;
+        node->m_type = CGenNode::TW_SYNC1;
+        node->m_time = m_cur_time_sec + BUCKET_TIME_SEC_LEVEL1 ;
+        m_node_gen.add_node(node);
     }
 
 
diff --git a/src/bp_sim.h b/src/bp_sim.h
index 9cdfd30a..4df1dcd9 100755
--- a/src/bp_sim.h
+++ b/src/bp_sim.h
@@ -370,6 +370,13 @@ public:
 #define CONST_9k_MBUF_SIZE   (MAX_PKT_ALIGN_BUF_9K + MBUF_PKT_PREFIX)
 
 
+#define TW_BUCKETS       (CGlobalInfo::m_options.get_tw_buckets())
+#define TW_BUCKETS_LEVEL1_DIV (16)
+#define TW_LEVELS        (CGlobalInfo::m_options.get_tw_levels())
+#define BUCKET_TIME_SEC (CGlobalInfo::m_options.get_tw_bucket_time_in_sec())
+#define BUCKET_TIME_SEC_LEVEL1 (CGlobalInfo::m_options.get_tw_bucket_level1_time_in_sec())
+
+
 class CPreviewMode {
 public:
     CPreviewMode(){
@@ -726,7 +733,7 @@ public:
         m_arp_ref_per = 120; // in seconds
         m_tw_buckets = 1024;
         m_tw_levels  = 3;
-        m_tw_bucket_time_sec = (20.0/1000000.0);
+        set_tw_bucket_time_in_usec(20.0);
         m_active_flows=0;
 
     }
@@ -775,6 +782,7 @@ public:
 
     CMacAddrCfg     m_mac_addr[TREX_MAX_PORTS];
     double          m_tw_bucket_time_sec;
+    double          m_tw_bucket_time_sec_level1;
     
 
     uint8_t *       get_src_mac_addr(int if_index){
@@ -819,8 +827,13 @@ public:
         return (m_tw_bucket_time_sec);
     }
 
+    inline double get_tw_bucket_level1_time_in_sec(void){
+        return (m_tw_bucket_time_sec_level1);
+    }
+
     void set_tw_bucket_time_in_usec(double usec){
-        m_tw_bucket_time_sec=(usec/1000000.0);
+        m_tw_bucket_time_sec= (usec/1000000.0);
+        m_tw_bucket_time_sec_level1 = (m_tw_bucket_time_sec*(double)m_tw_buckets)/((double)TW_BUCKETS_LEVEL1_DIV);
     }
 
     void     set_tw_buckets(uint16_t buckets){
@@ -1469,7 +1482,9 @@ public:
         EXIT_PORT_SCHED         =8,
         PCAP_PKT                =9,
         GRAT_ARP                =10,
-        TW_SYNC                 =11
+        TW_SYNC                 =11,
+        TW_SYNC1                =12,
+
     };
 
     /* flags MASKS*/
@@ -2215,6 +2230,8 @@ private:
     void handle_flow_sync(CGenNode *node, CFlowGenListPerThread *thread, bool &exit_scheduler);
     void handle_pcap_pkt(CGenNode *node, CFlowGenListPerThread *thread);
     void handle_maintenance(CFlowGenListPerThread *thread);
+    void handle_batch_tw_level1(CGenNode *node, CFlowGenListPerThread *thread,bool &exit_scheduler,bool on_terminate);
+
 
 public:
     pqueue_t                  m_p_queue;
@@ -2226,8 +2243,10 @@ public:
     uint64_t                  m_non_active;
     uint64_t                  m_limit;
     CTimeHistogram            m_realtime_his;
+    dsec_t                    m_scheduler_offset;
 
     dsec_t                    m_last_sync_time_sec;
+    dsec_t                    m_tw_level1_next_sec;
 };
 
 
@@ -3798,9 +3817,6 @@ private:
     bool           server_seq_init;  /* TCP seq been init for server? */
 };
 
-#define TW_BUCKETS       (CGlobalInfo::m_options.get_tw_buckets())
-#define TW_LEVELS        (CGlobalInfo::m_options.get_tw_levels())
-#define BUCKET_TIME_SEC (CGlobalInfo::m_options.get_tw_bucket_time_in_sec())
 
 
 
@@ -3957,6 +3973,8 @@ public:
     
 private:
     
+    FORCE_NO_INLINE void   no_memory_error();
+
     bool check_msgs_from_rx();
     
     void handle_nat_msg(CGenNodeNatInfo * msg);
@@ -4016,7 +4034,7 @@ public:
 
 public:
     CNodeGenerator                   m_node_gen;
-    CHTimerWheel                     m_tw;
+    CNATimerWheel                    m_tw;
 
 public:
     uint32_t                         m_cur_template;
@@ -4051,7 +4069,7 @@ private:
 inline CGenNode * CFlowGenListPerThread::create_node(void){
     CGenNode * res;
     if ( unlikely (rte_mempool_sc_get(m_node_pool, (void **)&res) <0) ){
-        rte_exit(EXIT_FAILURE, "cant allocate object , need more \n");
+        no_memory_error();
         return (0);
     }
     return (res);
diff --git a/src/gtest/bp_timer_gtest.cpp b/src/gtest/bp_timer_gtest.cpp
index 07f0e214..1e8e7069 100644
--- a/src/gtest/bp_timer_gtest.cpp
+++ b/src/gtest/bp_timer_gtest.cpp
@@ -216,9 +216,9 @@ TEST_F(gt_r_timer, timer7) {
 
     int i;
     for (i=0; i<150; i++) {
-        printf(" tick %d :",i);
+        //printf(" tick %d :",i);
         timer.on_tick((void *)&timer,my_test_on_tick_cb7);
-        printf(" \n");
+        //printf(" \n");
     }
 
     EXPECT_EQ( timer.Delete(),RC_HTW_OK);
@@ -655,4 +655,460 @@ TEST_F(gt_r_timer, timer18) {
 
 
 
+/////////////////////////////////////////////////////////////////
+/* test for NA class */
+
+class CNATimerWheelTest1Cfg {
+public:
+    uint32_t m_wheel_size;
+    uint32_t m_level1_div;
+    uint32_t m_start_tick;
+    uint32_t m_restart_tick;
+    uint32_t m_total_ticks;
+    int     m_verbose;
+    bool     m_dont_assert;
+};
+
+
+class CNATimerWheelTest1 : public CHTimerWheelBase {
+
+public:
+    bool Create(CNATimerWheelTest1Cfg & cfg);
+    void Delete();
+    void start_test();
+    virtual void on_tick(CMyTestObject *lpobj);
+
+private:
+    CNATimerWheelTest1Cfg m_cfg;
+    CNATimerWheel         m_timer;
+    CMyTestObject         m_event;
+    uint32_t              m_ticks;
+    uint32_t              m_total_ticks;
+    uint32_t              m_expected_total_ticks;
+    uint32_t              m_div_err;
+
+    uint32_t              m_expect_tick;
+    double                m_max_err;
+};
+
+void my_test_on_tick_cb18(void *userdata,CHTimerObj *tmr){
+    CHTimerWheelBase * lp=(CHTimerWheelBase *)userdata;
+    UNSAFE_CONTAINER_OF_PUSH
+    CMyTestObject *lpobj=(CMyTestObject *)((uint8_t*)tmr-offsetof (CMyTestObject,m_timer));
+    UNSAFE_CONTAINER_OF_POP
+    lp->on_tick(lpobj);
+}
+
+
+void CNATimerWheelTest1::on_tick(CMyTestObject *lpobj){
+    assert(lpobj->m_id==17);
+    m_total_ticks++;
+    if (m_cfg.m_verbose) {
+        printf(" [event(%d)-%d]",lpobj->m_timer.m_wheel,lpobj->m_id);
+    }
+    if (!m_cfg.m_dont_assert){
+        uint32_t expect_min=m_expect_tick;
+        if (expect_min>m_div_err) {
+            expect_min-=m_div_err*2;
+        }
+        double pre=abs(100.0-100.0*(double)m_ticks/(double)m_expect_tick);
+        if (pre>m_max_err){
+            m_max_err=pre;
+        }
+        if (pre>(200.0/(double)m_div_err)) {
+            printf(" =====>tick:%d expect [%d -%d] %f \n",m_ticks,expect_min,m_expect_tick+(m_div_err*2),pre);
+        }
+    }
+    m_timer.timer_start(&lpobj->m_timer,m_cfg.m_restart_tick);
+    m_expect_tick+=m_cfg.m_restart_tick;
+}
+
+
+void CNATimerWheelTest1::start_test(){
+
+    if (m_cfg.m_verbose) {
+        printf(" test start %d,restart: %d \n",m_cfg.m_start_tick,m_cfg.m_restart_tick);
+    }
+    int i;
+    m_expected_total_ticks=0;
+    uint32_t cnt=m_cfg.m_start_tick;
+    for (i=0; i<m_cfg.m_total_ticks; i++) {
+        if (i==cnt) {
+            m_expected_total_ticks++;
+            cnt+=m_cfg.m_restart_tick;
+        }
+    }
+
+    m_div_err =m_cfg.m_wheel_size/m_cfg.m_level1_div;
+    m_total_ticks=0;
+    m_event.m_id=17;
+    m_timer.timer_start(&m_event.m_timer,m_cfg.m_start_tick);
+
+    m_ticks=0;
+    m_expect_tick= m_cfg.m_start_tick;
+
+    for (i=0; i<m_cfg.m_total_ticks; i++) {
+        if (m_cfg.m_verbose) {
+          printf(" tick %d :",i);
+        }
+        m_ticks=i;
+        m_timer.on_tick_level0((void *)this,my_test_on_tick_cb18);
+        /* level 2 */
+        if ((i>=m_div_err) && (i%m_div_err==0)) {
+            int cnt_rerty=0;
+            while (true){
+                if (m_cfg.m_verbose>1) {
+                  printf("\n level1 - try %d \n",cnt_rerty);
+                }
+
+                na_htw_state_num_t state;
+                state = m_timer.on_tick_level1((void *)this,my_test_on_tick_cb18);
+                if (m_cfg.m_verbose>1) {
+                  printf("\n state - %lu \n",(ulong)state);
+                }
+
+                if ( state !=TW_NEXT_BATCH){
+                    break;
+                }
+                cnt_rerty++;
+            }
+            if (m_cfg.m_verbose>1) {
+               printf("\n level1 - stop %d \n",cnt_rerty);
+            }
+        }
+        if (m_cfg.m_verbose) {
+          printf(" \n");
+        }
+    }
+    if (m_cfg.m_verbose) {
+       printf(" %d == %d \n",m_expected_total_ticks,m_total_ticks);
+    }
+    if (!m_cfg.m_dont_assert){
+      //assert( (m_expected_total_ticks==m_total_ticks) || ((m_expected_total_ticks+1) ==m_total_ticks) );
+    } 
+}
+
+
+bool CNATimerWheelTest1::Create(CNATimerWheelTest1Cfg & cfg){
+    m_cfg = cfg;
+    m_max_err=0.0;
+    assert(m_timer.Create(m_cfg.m_wheel_size,m_cfg.m_level1_div)==RC_HTW_OK);
+    m_ticks=0;
+    return (true);
+}
+
+void CNATimerWheelTest1::Delete(){
+    //printf (" %f \n",m_max_err);
+    assert(m_timer.Delete()==RC_HTW_OK);
+}
+
+
+TEST_F(gt_r_timer, timer20) {
+
+    CNATimerWheelTest1 test;
+
+    CNATimerWheelTest1Cfg  cfg ={
+        .m_wheel_size    = 32,
+        .m_level1_div    = 4,
+        .m_start_tick    = 2,
+        .m_restart_tick  = 2,
+        .m_total_ticks   = 1024,
+        .m_verbose=0
+    };
+    test.Create(cfg);
+    test.start_test();
+    test.Delete();
+}
+
+TEST_F(gt_r_timer, timer21) {
+
+    CNATimerWheelTest1 test;
+
+    CNATimerWheelTest1Cfg  cfg ={
+        .m_wheel_size    = 32,
+        .m_level1_div    = 4,
+        .m_start_tick    = 2,
+        .m_restart_tick  = 34,
+        .m_total_ticks   = 100,
+        .m_verbose=0
+    };
+    test.Create(cfg);
+    test.start_test();
+    test.Delete();
+}
+
+
+TEST_F(gt_r_timer, timer22) {
+
+    CNATimerWheelTest1 test;
+
+    CNATimerWheelTest1Cfg  cfg ={
+        .m_wheel_size    = 32,
+        .m_level1_div    = 4,
+        .m_start_tick    = 2,
+        .m_restart_tick  = 55,
+        .m_total_ticks   = 1000,
+        .m_verbose=0,
+        .m_dont_assert =0
+    };
+    test.Create(cfg);
+    test.start_test();
+    test.Delete();
+}
+
+TEST_F(gt_r_timer, timer23) {
+
+    int i,j;
+
+    for (i=0; i<100; i++) {
+        for (j=1; j<100; j++) {
+            CNATimerWheelTest1 test;
+            CNATimerWheelTest1Cfg  cfg ={
+                .m_wheel_size    = 32,
+                .m_level1_div    = 4,
+                .m_start_tick    = (uint32_t)i,
+                .m_restart_tick  = (uint32_t)j,
+                .m_total_ticks   = 1000,
+                .m_verbose=0,
+                .m_dont_assert =0
+            };
+
+            cfg.m_total_ticks= (uint32_t)(i*2+j*10);
+            test.Create(cfg);
+            test.start_test();
+            test.Delete();
+        }
+    }
+}
+
+
+
+#if 0
+// too long, skip for now 
+TEST_F(gt_r_timer, timer24) {
+
+    int i,j;
+
+    for (i=0; i<2048; i++) {
+        printf(" %d \n",i);
+        for (j=1024; j<2048; j=j+7) {
+            CNATimerWheelTest1 test;
+            CNATimerWheelTest1Cfg  cfg ={
+                .m_wheel_size    = 1024,
+                .m_level1_div    = 32,
+                .m_start_tick    = (uint32_t)i,
+                .m_restart_tick  = (uint32_t)j,
+                .m_total_ticks   = 3000,
+                .m_verbose=0,
+                .m_dont_assert =0
+            };
+
+            cfg.m_total_ticks= (uint32_t)(i*2+j*10);
+            test.Create(cfg);
+            test.start_test();
+            test.Delete();
+        }
+    }
+}
+#endif
+
+/* very long flow, need to restart it */
+TEST_F(gt_r_timer, timer25) {
+
+
+        CNATimerWheelTest1 test;
+
+        CNATimerWheelTest1Cfg  cfg ={
+            .m_wheel_size    = 32,
+            .m_level1_div    = 4,
+            .m_start_tick    = 2,
+            .m_restart_tick  = 512,
+            .m_total_ticks   = 1000,
+            .m_verbose=0,
+            .m_dont_assert =0
+        };
+
+        test.Create(cfg);
+        test.start_test();
+        test.Delete();
+}
+
+
+
+////////////////////////////////////////////////////////
+
+class CNATimerWheelTest2Cfg {
+public:
+    uint32_t m_wheel_size;
+    uint32_t m_level1_div;
+    uint32_t m_number_of_con_event;
+    uint32_t m_total_ticks;
+    bool     m_random;
+    bool     m_burst;
+    int      m_verbose;
+    bool     m_dont_check;
+};
+
+class CNATimerWheelTest2 : public CHTimerWheelBase {
+
+public:
+    bool Create(CNATimerWheelTest2Cfg & cfg);
+    void Delete();
+    void start_test();
+    virtual void on_tick(CMyTestObject *lpobj);
+
+private:
+    CNATimerWheelTest2Cfg  m_cfg;
+    CNATimerWheel          m_timer;
+    uint32_t              m_ticks;
+    uint32_t              m_div_err;
+};
+
+bool CNATimerWheelTest2::Create(CNATimerWheelTest2Cfg & cfg){
+    m_cfg = cfg;
+    assert(m_timer.Create(m_cfg.m_wheel_size,m_cfg.m_level1_div)==RC_HTW_OK);
+    m_ticks=0;
+    return (true);
+}
+
+void CNATimerWheelTest2::Delete(){
+    assert(m_timer.Delete()==RC_HTW_OK);
+}
+
+
+void CNATimerWheelTest2::start_test(){
+
+    CMyTestObject *  m_events = new CMyTestObject[m_cfg.m_number_of_con_event]; 
+    int i;
+    for (i=0; i<m_cfg.m_number_of_con_event; i++) {
+        CMyTestObject * lp=&m_events[i];
+        lp->m_id=i+1;
+        if (m_cfg.m_random) {
+            lp->m_d_tick = ((rand() % m_cfg.m_number_of_con_event)+1);
+            if (m_cfg.m_verbose) {
+                printf(" flow %d : %d \n",i,lp->m_d_tick);
+            }
+        }else{
+            if (m_cfg.m_burst){
+                lp->m_d_tick = m_cfg.m_wheel_size*2; /* all in the same bucket */
+            }else{
+                lp->m_d_tick=i+1;
+            }
+        }
+        lp->m_t_tick=lp->m_d_tick;
+        m_timer.timer_start(&lp->m_timer,lp->m_d_tick);
+    }
+
+    m_div_err =m_cfg.m_wheel_size/m_cfg.m_level1_div;
+
+    for (i=0; i<m_cfg.m_total_ticks; i++) {
+        if (m_cfg.m_verbose) {
+          printf(" tick %d :",i);
+        }
+        m_ticks=i;
+        m_timer.on_tick_level0((void *)this,my_test_on_tick_cb18);
+
+        if ((i>=m_div_err) && (i%m_div_err==0)) {
+            int cnt_rerty=0;
+            while (true){
+                if (m_cfg.m_verbose>1) {
+                  printf("\n level1 - try %d \n",cnt_rerty);
+                }
+
+                na_htw_state_num_t state;
+                state = m_timer.on_tick_level1((void *)this,my_test_on_tick_cb18);
+                if (m_cfg.m_verbose>1) {
+                  printf("\n state - %lu \n",(ulong)state);
+                }
+
+                if ( state !=TW_NEXT_BATCH){
+                    break;
+                }
+
+                cnt_rerty++;
+            }
+            if (m_cfg.m_verbose>1) {
+               printf("\n level1 - stop %d \n",cnt_rerty);
+            }
+        }
+
+
+        if (m_cfg.m_verbose) {
+          printf(" \n");
+        }
+    }
+    delete []m_events;
+}
+
+
+void CNATimerWheelTest2::on_tick(CMyTestObject *lp){
+
+    if (!m_cfg.m_random && !m_cfg.m_burst) {
+        assert(lp->m_id==lp->m_d_tick);
+    }
+    if (m_cfg.m_verbose) {
+        printf(" [event %d ]",lp->m_id);
+    }
+    m_timer.timer_start(&lp->m_timer,lp->m_d_tick);
+    if (!m_cfg.m_dont_check){
+        double pre=abs(100.0-100.0*(double)m_ticks/(double)lp->m_t_tick);
+        if (pre>(200.0/(double)m_div_err)) {
+            printf(" =====>tick:%d  %f \n",m_ticks,pre);
+            assert(0);
+        }
+    }
+    lp->m_t_tick+=lp->m_d_tick;
+}
+
+
+TEST_F(gt_r_timer, timer30) {
+
+        CNATimerWheelTest2 test;
+        CNATimerWheelTest2Cfg  cfg ={
+            .m_wheel_size    = 32,
+            .m_level1_div    = 4,
+            .m_number_of_con_event   = 100,
+            .m_total_ticks =1000,
+            .m_random=false,
+            .m_burst=false,
+            .m_verbose =false
+        };
+        test.Create(cfg);
+        test.start_test();
+        test.Delete();
+}
+
+TEST_F(gt_r_timer, timer31) {
+
+        CNATimerWheelTest2 test;
+        CNATimerWheelTest2Cfg  cfg ={
+            .m_wheel_size    = 32,
+            .m_level1_div    = 4,
+            .m_number_of_con_event   = 500,
+            .m_total_ticks =5000,
+            .m_random=true,
+            .m_burst=false,
+            .m_verbose =false
+        };
+        test.Create(cfg);
+        test.start_test();
+        test.Delete();
+}
+
+TEST_F(gt_r_timer, timer32) {
+
+        CNATimerWheelTest2 test;
+        CNATimerWheelTest2Cfg  cfg ={
+            .m_wheel_size    = 32,
+            .m_level1_div    = 4,
+            .m_number_of_con_event   = 500,
+            .m_total_ticks =100,
+            .m_random=false,
+            .m_burst=true,
+            .m_verbose =0
+        };
+        test.Create(cfg);
+        test.start_test();
+        test.Delete();
+}
 
diff --git a/src/h_timer.cpp b/src/h_timer.cpp
index b3d86d46..4e52c3d2 100644
--- a/src/h_timer.cpp
+++ b/src/h_timer.cpp
@@ -266,6 +266,168 @@ RC_HTW_t CHTimerWheel::Delete(){
     return(RC_HTW_OK);
 }
 
+////////////////////////////////////////////////////////
+
+
+
+void CNATimerWheel::detach_all(void *userdata,htw_on_tick_cb_t cb){
+    #ifndef _DEBUG 
+    if (m_total_events==0) {
+        return;
+    }
+    #endif
+    int i;
+    uint32_t res=0;
+    for (i=0;i<HNA_TIMER_LEVELS; i++) {
+        CHTimerOneWheel * lp=&m_timer_w[i];
+        res=lp->detach_all(userdata,cb);
+        assert(m_total_events>=res);
+        m_total_events -=res;
+    }
+    assert(m_total_events==0);
+}
+
+
+void CNATimerWheel::on_tick_level0(void *userdata,htw_on_tick_cb_t cb){
+
+    CHTimerOneWheel * lp=&m_timer_w[0];
+    CHTimerObj * event;
+
+    while (  true ) {
+        event = lp->pop_event();
+        if (!event) {
+            break;
+        }
+        m_total_events--;
+        cb(userdata,event);
+   }
+   lp->timer_tick();
+   m_ticks[0]++;
+}
+
+/* almost always we will have burst here */
+na_htw_state_num_t CNATimerWheel::on_tick_level1(void *userdata,htw_on_tick_cb_t cb){
+
+    CHTimerOneWheel * lp=&m_timer_w[1];
+    CHTimerObj * event;
+    uint32_t cnt=0;
+
+    while (  true ) {
+        event = lp->pop_event();
+        if (!event) {
+            break;
+        }
+        if (event->m_ticks_left==0) {
+            m_total_events--;
+            cb(userdata,event);
+        }else{
+            timer_start_rest(event,event->m_ticks_left);
+        }
+        cnt++;
+        if (cnt>HNA_MAX_LEVEL1_EVENTS) {
+            /* need another batch */
+            na_htw_state_num_t old_state;
+            old_state=m_state;
+            m_state=TW_NEXT_BATCH;
+            if (old_state ==TW_FIRST_FINISH){
+               return(TW_FIRST_BATCH);
+            }else{
+               return(TW_NEXT_BATCH);
+            }
+        }
+   }
+   lp->timer_tick();
+   m_ticks[1]++;
+   if (m_state==TW_FIRST_FINISH) {
+       if (cnt>0) {
+           return (TW_FIRST_FINISH_ANY);
+       }else{
+           return (TW_FIRST_FINISH);
+       }
+   }else{
+       assert(m_state==TW_NEXT_BATCH);
+       m_state=TW_FIRST_FINISH;
+       return(TW_END_BATCH);
+   }
+}
+
+
+
+RC_HTW_t CNATimerWheel::timer_stop (CHTimerObj *tmr){
+    if ( tmr->is_running() ) {
+        assert(tmr->m_wheel<HNA_TIMER_LEVELS);
+        m_timer_w[tmr->m_wheel].timer_stop(tmr);
+        m_total_events--;
+    }
+    return (RC_HTW_OK);
+}
+
+
+
+RC_HTW_t CNATimerWheel::timer_start_rest(CHTimerObj  *tmr, 
+                                        htw_ticks_t  ticks){
+
+    htw_ticks_t nticks = (ticks+m_wheel_level1_err)>>m_wheel_level1_shift; 
+    if (nticks<m_wheel_size) {
+        if (nticks<2) {
+            nticks=2; /* not on the same bucket*/
+        }
+        tmr->m_ticks_left=0;
+        tmr->m_wheel=1;
+        m_timer_w[1].timer_start(tmr,nticks-1);
+    }else{
+        tmr->m_ticks_left = ticks - ((m_wheel_size-1)<<m_wheel_level1_shift);
+        tmr->m_wheel=1;
+        m_timer_w[1].timer_start(tmr,m_wheel_size-1);
+    }
+    return (RC_HTW_OK);
+}
+
+
+void CNATimerWheel::reset(){
+    m_wheel_shift=0;
+    m_total_events=0;
+    m_wheel_size=0;
+    m_wheel_mask=0;
+    m_wheel_level1_shift=0;
+    m_wheel_level1_err=0;
+    m_state=TW_FIRST_FINISH;
+    int i;
+    for (i=0; i<HNA_TIMER_LEVELS; i++) {
+        m_ticks[i]=0;
+    }
+
+}
+
+
+RC_HTW_t CNATimerWheel::Create(uint32_t wheel_size,
+                               uint8_t level1_div){
+    RC_HTW_t res;
+    int i;
+    for (i=0; i<HNA_TIMER_LEVELS; i++) {
+        res = m_timer_w[i].Create(wheel_size);
+        if ( res !=RC_HTW_OK ){
+            return (res);
+        }
+        m_ticks[i]=0;
+    }
+    m_wheel_shift = utl_log2_shift(wheel_size);
+    m_wheel_mask  = utl_mask_log2(wheel_size);
+    m_wheel_size  = wheel_size;
+    m_wheel_level1_shift = m_wheel_shift - utl_log2_shift((uint32_t)level1_div);
+    m_wheel_level1_err  = ((1<<(m_wheel_level1_shift))-1);
+    assert(m_wheel_shift>utl_log2_shift((uint32_t)level1_div));
+
+    return(RC_HTW_OK);
+}
+
+RC_HTW_t CNATimerWheel::Delete(){
+    int i;
+    for (i=0; i<HNA_TIMER_LEVELS; i++) {
+        m_timer_w[i].Delete();
+    }
+    return(RC_HTW_OK);
+}
 
 
 
diff --git a/src/h_timer.h b/src/h_timer.h
index 22343533..17ff44be 100644
--- a/src/h_timer.h
+++ b/src/h_timer.h
@@ -344,4 +344,78 @@ private:
 } ;
 
 
+
+
+#define HNA_TIMER_LEVELS  (2)
+#define HNA_MAX_LEVEL1_EVENTS (64) /* small bursts */
+
+typedef enum {
+    TW_FIRST_FINISH =17,
+    TW_FIRST_FINISH_ANY =18,
+    TW_FIRST_BATCH   =19,
+    TW_NEXT_BATCH   =20,
+    TW_END_BATCH    =21
+} NA_HTW_STATE_t;
+
+typedef uint8_t na_htw_state_num_t;
+
+
+/* two levels 0,1. level 1 would be less accurate */ 
+class CNATimerWheel {
+
+public:
+    CNATimerWheel(){
+        reset();
+    }
+
+    RC_HTW_t Create(uint32_t wheel_size,uint8_t level1_div);
+
+    RC_HTW_t Delete();
+
+
+    inline RC_HTW_t timer_start(CHTimerObj  *tmr, 
+                                htw_ticks_t  ticks){
+        m_total_events++;
+        if (likely(ticks<m_wheel_size)) {
+            tmr->m_ticks_left=0;
+            tmr->m_wheel=0;
+            return (m_timer_w[0].timer_start(tmr,ticks));
+        }
+        return ( timer_start_rest(tmr, ticks));
+    }
+
+    RC_HTW_t timer_stop (CHTimerObj *tmr);
+
+    void on_tick_level0(void *userdata,htw_on_tick_cb_t cb);
+
+    na_htw_state_num_t on_tick_level1(void *userdata,htw_on_tick_cb_t cb);
+
+    bool is_any_events_left(){
+        return(m_total_events>0?true:false);
+    }
+
+    /* iterate all, detach and call the callback */
+    void detach_all(void *userdata,htw_on_tick_cb_t cb);
+
+
+private:
+    void reset(void);
+
+    RC_HTW_t timer_start_rest(CHTimerObj  *tmr, 
+                              htw_ticks_t  ticks);
+
+private:
+    htw_ticks_t         m_ticks[HNA_TIMER_LEVELS];               
+    uint32_t            m_wheel_size;  //e.g. 256
+    uint32_t            m_wheel_mask;  //e.g 256-1
+    uint32_t            m_wheel_shift; // e.g 8
+    uint32_t            m_wheel_level1_shift; //e.g 16 
+    uint32_t            m_wheel_level1_err; //e.g 16 
+
+    uint64_t            m_total_events;
+    CHTimerOneWheel     m_timer_w[HNA_TIMER_LEVELS];
+    na_htw_state_num_t  m_state;
+} ;
+
+
 #endif