tcp: explicit use of timer wheel in timer apis

Type: refactor

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I06120d43c15b2fa3f437ef235bf5f83ec2beb45e

6 files changed, 190 insertions, 171 deletions

diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c
index 2ac938a1b1e..d85225b9172 100644
--- a/src/vnet/tcp/tcp.c
+++ b/src/vnet/tcp/tcp.c
@@ -208,10 +208,14 @@ tcp_half_open_connection_del (tcp_connection_t * tc)
 int
 tcp_half_open_connection_cleanup (tcp_connection_t * tc)
 {
+  tcp_worker_ctx_t *wrk;
+
   /* Make sure this is the owning thread */
   if (tc->c_thread_index != vlib_get_thread_index ())
     return 1;
-  tcp_timer_reset (tc, TCP_TIMER_RETRANSMIT_SYN);
+
+  wrk = tcp_get_worker (tc->c_thread_index);
+  tcp_timer_reset (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN);
   tcp_half_open_connection_del (tc);
   return 0;
 }
@@ -354,6 +358,8 @@ tcp_program_cleanup (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
 void
 tcp_connection_close (tcp_connection_t * tc)
 {
+  tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
+
   TCP_EVT (TCP_EVT_CLOSE, tc);
 
   /* Send/Program FIN if needed and switch state */
@@ -368,7 +374,8 @@ tcp_connection_close (tcp_connection_t * tc)
       tcp_connection_timers_reset (tc);
       tcp_send_fin (tc);
       tcp_connection_set_state (tc, TCP_STATE_FIN_WAIT_1);
-      tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.finwait1_time);
+      tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE,
+			tcp_cfg.finwait1_time);
       break;
     case TCP_STATE_ESTABLISHED:
       /* If closing with unread data, reset the connection */
@@ -379,7 +386,7 @@ tcp_connection_close (tcp_connection_t * tc)
 	  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
 	  session_transport_closed_notify (&tc->connection);
 	  tcp_program_cleanup (tcp_get_worker (tc->c_thread_index), tc);
-	  tcp_worker_stats_inc (tc->c_thread_index, rst_unread, 1);
+	  tcp_worker_stats_inc (wrk, rst_unread, 1);
 	  break;
 	}
       if (!transport_max_tx_dequeue (&tc->connection))
@@ -390,7 +397,8 @@ tcp_connection_close (tcp_connection_t * tc)
       /* Set a timer in case the peer stops responding. Otherwise the
        * connection will be stuck here forever. */
       ASSERT (tc->timers[TCP_TIMER_WAITCLOSE] == TCP_TIMER_HANDLE_INVALID);
-      tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.finwait1_time);
+      tcp_timer_set (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE,
+		     tcp_cfg.finwait1_time);
       break;
     case TCP_STATE_CLOSE_WAIT:
       if (!transport_max_tx_dequeue (&tc->connection))
@@ -398,13 +406,15 @@ tcp_connection_close (tcp_connection_t * tc)
 	  tcp_send_fin (tc);
 	  tcp_connection_timers_reset (tc);
 	  tcp_connection_set_state (tc, TCP_STATE_LAST_ACK);
-	  tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.lastack_time);
+	  tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE,
+			    tcp_cfg.lastack_time);
 	}
       else
 	tc->flags |= TCP_CONN_FINPNDG;
       break;
     case TCP_STATE_FIN_WAIT_1:
-      tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.finwait1_time);
+      tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE,
+			tcp_cfg.finwait1_time);
       break;
     case TCP_STATE_CLOSED:
       /* Cleanup should've been programmed already */
@@ -469,11 +479,11 @@ tcp_connection_timers_init (tcp_connection_t * tc)
 void
 tcp_connection_timers_reset (tcp_connection_t * tc)
 {
+  tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
   int i;
+
   for (i = 0; i < TCP_N_TIMERS; i++)
-    {
-      tcp_timer_reset (tc, i);
-    }
+    tcp_timer_reset (&wrk->timer_wheel, tc, i);
 }
 
 #if 0
@@ -956,7 +966,7 @@ tcp_timer_waitclose_handler (tcp_connection_t * tc)
 	  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
 	  session_transport_closed_notify (&tc->connection);
 	  tcp_program_cleanup (wrk, tc);
-	  tcp_workerp_stats_inc (wrk, to_closewait, 1);
+	  tcp_worker_stats_inc (wrk, to_closewait, 1);
 	  break;
 	}
 
@@ -969,8 +979,9 @@ tcp_timer_waitclose_handler (tcp_connection_t * tc)
       session_transport_closed_notify (&tc->connection);
 
       /* Make sure we don't wait in LAST ACK forever */
-      tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.lastack_time);
-      tcp_workerp_stats_inc (wrk, to_closewait2, 1);
+      tcp_timer_set (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE,
+		     tcp_cfg.lastack_time);
+      tcp_worker_stats_inc (wrk, to_closewait2, 1);
 
       /* Don't delete the connection yet */
       break;
@@ -992,21 +1003,21 @@ tcp_timer_waitclose_handler (tcp_connection_t * tc)
 	  tcp_program_cleanup (wrk, tc);
 	}
       session_transport_closed_notify (&tc->connection);
-      tcp_workerp_stats_inc (wrk, to_finwait1, 1);
+      tcp_worker_stats_inc (wrk, to_finwait1, 1);
       break;
     case TCP_STATE_LAST_ACK:
       tcp_connection_timers_reset (tc);
       tcp_connection_set_state (tc, TCP_STATE_CLOSED);
       session_transport_closed_notify (&tc->connection);
       tcp_program_cleanup (wrk, tc);
-      tcp_workerp_stats_inc (wrk, to_lastack, 1);
+      tcp_worker_stats_inc (wrk, to_lastack, 1);
       break;
     case TCP_STATE_CLOSING:
       tcp_connection_timers_reset (tc);
       tcp_connection_set_state (tc, TCP_STATE_CLOSED);
       session_transport_closed_notify (&tc->connection);
       tcp_program_cleanup (wrk, tc);
-      tcp_workerp_stats_inc (wrk, to_closing, 1);
+      tcp_worker_stats_inc (wrk, to_closing, 1);
       break;
     case TCP_STATE_FIN_WAIT_2:
       tcp_send_reset (tc);
@@ -1014,7 +1025,7 @@ tcp_timer_waitclose_handler (tcp_connection_t * tc)
       tcp_connection_set_state (tc, TCP_STATE_CLOSED);
       session_transport_closed_notify (&tc->connection);
       tcp_program_cleanup (wrk, tc);
-      tcp_workerp_stats_inc (wrk, to_finwait2, 1);
+      tcp_worker_stats_inc (wrk, to_finwait2, 1);
       break;
     case TCP_STATE_TIME_WAIT:
       tcp_connection_set_state (tc, TCP_STATE_CLOSED);
@@ -1186,7 +1197,7 @@ tcp_expired_timers_dispatch (u32 * expired_timers)
 
   wrk = tcp_get_worker (thread_index);
   n_expired = vec_len (expired_timers);
-  tcp_workerp_stats_inc (wrk, timer_expirations, n_expired);
+  tcp_worker_stats_inc (wrk, timer_expirations, n_expired);
   n_left = clib_fifo_elts (wrk->pending_timers);
 
   /*
diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h
index 708d7566eb2..f8c8cb342fe 100644
--- a/src/vnet/tcp/tcp.h
+++ b/src/vnet/tcp/tcp.h
@@ -115,17 +115,14 @@ typedef struct tcp_worker_ctx_
   tcp_cleanup_req_t *pending_cleanups;
 
   /** worker timer wheel */
-  tw_timer_wheel_16t_2w_512sl_t timer_wheel;
+  tcp_timer_wheel_t timer_wheel;
 
     CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
 
   tcp_wrk_stats_t stats;
 } tcp_worker_ctx_t;
 
-#define tcp_worker_stats_inc(_ti,_stat,_val) 		\
-  tcp_main.wrk_ctx[_ti].stats._stat += _val
-
-#define tcp_workerp_stats_inc(_wrk,_stat,_val) 		\
+#define tcp_worker_stats_inc(_wrk,_stat,_val) 		\
   _wrk->stats._stat += _val
 
 typedef struct tcp_iss_seed_
@@ -356,113 +353,6 @@ format_function_t format_tcp_rcv_sacks;
 format_function_t format_tcp_connection;
 format_function_t format_tcp_connection_id;
 
-always_inline void
-tcp_timer_set (tcp_connection_t * tc, u8 timer_id, u32 interval)
-{
-  ASSERT (tc->c_thread_index == vlib_get_thread_index ());
-  ASSERT (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID);
-  tc->timers[timer_id] =
-    tw_timer_start_16t_2w_512sl (&tcp_main.
-				 wrk_ctx[tc->c_thread_index].timer_wheel,
-				 tc->c_c_index, timer_id, interval);
-}
-
-always_inline void
-tcp_timer_reset (tcp_connection_t * tc, u8 timer_id)
-{
-  ASSERT (tc->c_thread_index == vlib_get_thread_index ());
-  if (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID)
-    return;
-
-  tw_timer_stop_16t_2w_512sl (&tcp_main.
-			      wrk_ctx[tc->c_thread_index].timer_wheel,
-			      tc->timers[timer_id]);
-  tc->timers[timer_id] = TCP_TIMER_HANDLE_INVALID;
-}
-
-always_inline void
-tcp_timer_update (tcp_connection_t * tc, u8 timer_id, u32 interval)
-{
-  ASSERT (tc->c_thread_index == vlib_get_thread_index ());
-  if (tc->timers[timer_id] != TCP_TIMER_HANDLE_INVALID)
-    tw_timer_update_16t_2w_512sl (&tcp_main.
-				  wrk_ctx[tc->c_thread_index].timer_wheel,
-				  tc->timers[timer_id], interval);
-  else
-    tc->timers[timer_id] =
-      tw_timer_start_16t_2w_512sl (&tcp_main.
-				   wrk_ctx[tc->c_thread_index].timer_wheel,
-				   tc->c_c_index, timer_id, interval);
-}
-
-always_inline void
-tcp_retransmit_timer_set (tcp_connection_t * tc)
-{
-  ASSERT (tc->snd_una != tc->snd_una_max);
-  tcp_timer_set (tc, TCP_TIMER_RETRANSMIT,
-		 clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
-}
-
-always_inline void
-tcp_retransmit_timer_reset (tcp_connection_t * tc)
-{
-  tcp_timer_reset (tc, TCP_TIMER_RETRANSMIT);
-}
-
-always_inline void
-tcp_retransmit_timer_force_update (tcp_connection_t * tc)
-{
-  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT,
-		    clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
-}
-
-always_inline void
-tcp_persist_timer_set (tcp_connection_t * tc)
-{
-  /* Reuse RTO. It's backed off in handler */
-  tcp_timer_set (tc, TCP_TIMER_PERSIST,
-		 clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
-}
-
-always_inline void
-tcp_persist_timer_update (tcp_connection_t * tc)
-{
-  u32 interval;
-
-  if (seq_leq (tc->snd_una, tc->snd_congestion + tc->burst_acked))
-    interval = 1;
-  else
-    interval = clib_max (tc->rto * TCP_TO_TIMER_TICK, 1);
-
-  tcp_timer_update (tc, TCP_TIMER_PERSIST, interval);
-}
-
-always_inline void
-tcp_persist_timer_reset (tcp_connection_t * tc)
-{
-  tcp_timer_reset (tc, TCP_TIMER_PERSIST);
-}
-
-always_inline void
-tcp_retransmit_timer_update (tcp_connection_t * tc)
-{
-  if (tc->snd_una == tc->snd_nxt)
-    {
-      tcp_retransmit_timer_reset (tc);
-      if (tc->snd_wnd < tc->snd_mss)
-	tcp_persist_timer_update (tc);
-    }
-  else
-    tcp_timer_update (tc, TCP_TIMER_RETRANSMIT,
-		      clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
-}
-
-always_inline u8
-tcp_timer_is_active (tcp_connection_t * tc, tcp_timers_e timer)
-{
-  return tc->timers[timer] != TCP_TIMER_HANDLE_INVALID;
-}
-
 #define tcp_validate_txf_size(_tc, _a) 					\
   ASSERT(_tc->state != TCP_STATE_ESTABLISHED 				\
 	 || transport_max_tx_dequeue (&_tc->connection) >= _a)
diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c
index e27cffb9444..713e11fd896 100755
--- a/src/vnet/tcp/tcp_input.c
+++ b/src/vnet/tcp/tcp_input.c
@@ -583,7 +583,7 @@ tcp_handle_postponed_dequeues (tcp_worker_ctx_t * wrk)
 
       /* If everything has been acked, stop retransmit timer
        * otherwise update. */
-      tcp_retransmit_timer_update (tc);
+      tcp_retransmit_timer_update (&wrk->timer_wheel, tc);
 
       /* Update pacer based on our new cwnd estimate */
       tcp_connection_tx_pacer_update (tc);
@@ -628,12 +628,18 @@ tcp_update_snd_wnd (tcp_connection_t * tc, u32 seq, u32 ack, u32 snd_wnd)
 	  /* Set persist timer if not set and we just got 0 wnd */
 	  if (!tcp_timer_is_active (tc, TCP_TIMER_PERSIST)
 	      && !tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT))
-	    tcp_persist_timer_set (tc);
+	    {
+	      tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
+	      tcp_persist_timer_set (&wrk->timer_wheel, tc);
+	    }
 	}
       else
 	{
 	  if (PREDICT_FALSE (tcp_timer_is_active (tc, TCP_TIMER_PERSIST)))
-	    tcp_persist_timer_reset (tc);
+	    {
+	      tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
+	      tcp_persist_timer_reset (&wrk->timer_wheel, tc);
+	    }
 
 	  if (PREDICT_FALSE (tcp_is_descheduled (tc)))
 	    tcp_reschedule (tc);
@@ -1137,7 +1143,8 @@ tcp_rcv_fin (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, vlib_buffer_t * b,
    * in CLOSE-WAIT, set timer (reuse WAITCLOSE). */
   tcp_connection_set_state (tc, TCP_STATE_CLOSE_WAIT);
   tcp_program_disconnect (wrk, tc);
-  tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time);
+  tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE,
+		    tcp_cfg.closewait_time);
   TCP_EVT (TCP_EVT_FIN_RCVD, tc);
   *error = TCP_ERROR_FIN_RCVD;
 }
@@ -1356,7 +1363,8 @@ in_order:
   if (tcp_can_delack (tc))
     {
       if (!tcp_timer_is_active (tc, TCP_TIMER_DELACK))
-	tcp_timer_set (tc, TCP_TIMER_DELACK, tcp_cfg.delack_time);
+	tcp_timer_set (&wrk->timer_wheel, tc, TCP_TIMER_DELACK,
+		       tcp_cfg.delack_time);
       goto done;
     }
 
@@ -2224,7 +2232,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  tc0->snd_wl2 = vnet_buffer (b0)->tcp.ack_number;
 
 	  /* Reset SYN-ACK retransmit and SYN_RCV establish timers */
-	  tcp_retransmit_timer_reset (tc0);
+	  tcp_retransmit_timer_reset (&wrk->timer_wheel, tc0);
 	  if (session_stream_accept_notify (&tc0->connection))
 	    {
 	      error0 = TCP_ERROR_MSG_QUEUE_FULL;
@@ -2258,7 +2266,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 		tcp_send_fin (tc0);
 	      /* If a fin was received and data was acked extend wait */
 	      else if ((tc0->flags & TCP_CONN_FINRCVD) && tc0->bytes_acked)
-		tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE,
+		tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE,
 				  tcp_cfg.closewait_time);
 	    }
 	  /* If FIN is ACKed */
@@ -2281,7 +2289,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      tcp_connection_set_state (tc0, TCP_STATE_FIN_WAIT_2);
 	      /* Enable waitclose because we're willing to wait for peer's
 	       * FIN but not indefinitely. */
-	      tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.finwait2_time);
+	      tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE,
+			     tcp_cfg.finwait2_time);
 
 	      /* Don't try to deq the FIN acked */
 	      if (tc0->burst_acked > 1)
@@ -2314,7 +2323,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  tcp_send_fin (tc0);
 	  tcp_connection_timers_reset (tc0);
 	  tcp_connection_set_state (tc0, TCP_STATE_LAST_ACK);
-	  tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.lastack_time);
+	  tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE,
+			 tcp_cfg.lastack_time);
 	  break;
 	case TCP_STATE_CLOSING:
 	  /* In addition to the processing for the ESTABLISHED state, if
@@ -2328,7 +2338,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
 	  tcp_connection_timers_reset (tc0);
 	  tcp_connection_set_state (tc0, TCP_STATE_TIME_WAIT);
-	  tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time);
+	  tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE,
+			 tcp_cfg.timewait_time);
 	  session_transport_closed_notify (&tc0->connection);
 	  goto drop;
 
@@ -2372,7 +2383,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	    goto drop;
 
 	  tcp_program_ack (tc0);
-	  tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time);
+	  tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE,
+			    tcp_cfg.timewait_time);
 	  goto drop;
 
 	  break;
@@ -2414,7 +2426,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  tcp_program_ack (tc0);
 	  tcp_connection_set_state (tc0, TCP_STATE_CLOSE_WAIT);
 	  tcp_program_disconnect (wrk, tc0);
-	  tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time);
+	  tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE,
+			    tcp_cfg.closewait_time);
 	  break;
 	case TCP_STATE_SYN_RCVD:
 	  /* Send FIN-ACK, enter LAST-ACK and because the app was not
@@ -2424,7 +2437,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  tc0->rcv_nxt += 1;
 	  tcp_send_fin (tc0);
 	  tcp_connection_set_state (tc0, TCP_STATE_LAST_ACK);
-	  tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.lastack_time);
+	  tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE,
+			 tcp_cfg.lastack_time);
 	  break;
 	case TCP_STATE_CLOSE_WAIT:
 	case TCP_STATE_CLOSING:
@@ -2440,7 +2454,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	       * sending it. Since we already received a fin, do not wait
 	       * for too long. */
 	      tc0->flags |= TCP_CONN_FINRCVD;
-	      tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE,
+	      tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE,
 				tcp_cfg.closewait_time);
 	    }
 	  else
@@ -2448,7 +2462,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      tcp_connection_set_state (tc0, TCP_STATE_CLOSING);
 	      tcp_program_ack (tc0);
 	      /* Wait for ACK for our FIN but not forever */
-	      tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE,
+	      tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE,
 				tcp_cfg.closing_time);
 	    }
 	  break;
@@ -2457,7 +2471,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  tc0->rcv_nxt += 1;
 	  tcp_connection_set_state (tc0, TCP_STATE_TIME_WAIT);
 	  tcp_connection_timers_reset (tc0);
-	  tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time);
+	  tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE,
+			 tcp_cfg.timewait_time);
 	  tcp_program_ack (tc0);
 	  session_transport_closed_notify (&tc0->connection);
 	  break;
@@ -2465,7 +2480,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  /* Remain in the TIME-WAIT state. Restart the time-wait
 	   * timeout.
 	   */
-	  tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time);
+	  tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE,
+			    tcp_cfg.timewait_time);
 	  break;
 	}
       error0 = TCP_ERROR_FIN_RCVD;
diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index aeeffa726c3..a68d5269abd 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -808,12 +808,12 @@ tcp_send_syn (tcp_connection_t * tc)
    * Setup retransmit and establish timers before requesting buffer
    * such that we can return if we've ran out.
    */
-  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
+  tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN,
 		    tc->rto * TCP_TO_TIMER_TICK);
 
   if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
     {
-      tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1);
+      tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN, 1);
       return;
     }
 
@@ -839,11 +839,11 @@ tcp_send_synack (tcp_connection_t * tc)
   vlib_buffer_t *b;
   u32 bi;
 
-  tcp_retransmit_timer_force_update (tc);
+  tcp_retransmit_timer_force_update (&wrk->timer_wheel, tc);
 
   if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
     {
-      tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
+      tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT, 1);
       return;
     }
 
@@ -874,7 +874,7 @@ tcp_send_fin (tcp_connection_t * tc)
   if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
     {
       /* Out of buffers so program fin retransmit ASAP */
-      tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
+      tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT, 1);
       if (fin_snt)
 	tc->snd_nxt += 1;
       else
@@ -887,7 +887,7 @@ tcp_send_fin (tcp_connection_t * tc)
   if ((tc->flags & TCP_CONN_SNDACK) && !tc->pending_dupacks)
     tc->flags &= ~TCP_CONN_SNDACK;
 
-  tcp_retransmit_timer_force_update (tc);
+  tcp_retransmit_timer_force_update (&wrk->timer_wheel, tc);
   b = vlib_get_buffer (vm, bi);
   tcp_init_buffer (vm, b);
   tcp_make_fin (tc, b);
@@ -1001,7 +1001,8 @@ tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b)
     }
   if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)))
     {
-      tcp_retransmit_timer_set (tc);
+      tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
+      tcp_retransmit_timer_set (&wrk->timer_wheel, tc);
       tc->rto_boff = 0;
     }
   tcp_trajectory_add_start (b, 3);
@@ -1252,7 +1253,7 @@ tcp_prepare_retransmit_segment (tcp_worker_ctx_t * wrk,
 
   tc->bytes_retrans += n_bytes;
   tc->segs_retrans += 1;
-  tcp_workerp_stats_inc (wrk, rxt_segs, 1);
+  tcp_worker_stats_inc (wrk, rxt_segs, 1);
   TCP_EVT (TCP_EVT_CC_RTX, tc, offset, n_bytes);
 
   return n_bytes;
@@ -1304,7 +1305,7 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc)
   vlib_buffer_t *b = 0;
   u32 bi, n_bytes;
 
-  tcp_workerp_stats_inc (wrk, tr_events, 1);
+  tcp_worker_stats_inc (wrk, tr_events, 1);
 
   /* Should be handled by a different handler */
   if (PREDICT_FALSE (tc->state == TCP_STATE_SYN_SENT))
@@ -1357,7 +1358,7 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc)
 	  session_transport_closed_notify (&tc->connection);
 	  tcp_connection_timers_reset (tc);
 	  tcp_program_cleanup (wrk, tc);
-	  tcp_workerp_stats_inc (wrk, tr_abort, 1);
+	  tcp_worker_stats_inc (wrk, tr_abort, 1);
 	  return;
 	}
 
@@ -1373,7 +1374,7 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc)
       n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, n_bytes, &b);
       if (!n_bytes)
 	{
-	  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
+	  tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT, 1);
 	  return;
 	}
 
@@ -1381,7 +1382,7 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc)
       tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
 
       tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
-      tcp_retransmit_timer_force_update (tc);
+      tcp_retransmit_timer_force_update (&wrk->timer_wheel, tc);
 
       tc->rto_boff += 1;
       if (tc->rto_boff == 1)
@@ -1409,13 +1410,13 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc)
 	  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
 	  tcp_connection_timers_reset (tc);
 	  tcp_program_cleanup (wrk, tc);
-	  tcp_workerp_stats_inc (wrk, tr_abort, 1);
+	  tcp_worker_stats_inc (wrk, tr_abort, 1);
 	  return;
 	}
 
       if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
 	{
-	  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
+	  tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT, 1);
 	  return;
 	}
 
@@ -1423,7 +1424,7 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc)
       if (tc->rto_boff > TCP_RTO_SYN_RETRIES)
 	tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
 
-      tcp_retransmit_timer_force_update (tc);
+      tcp_retransmit_timer_force_update (&wrk->timer_wheel, tc);
 
       b = vlib_get_buffer (vm, bi);
       tcp_init_buffer (vm, b);
@@ -1478,7 +1479,7 @@ tcp_timer_retransmit_syn_handler (tcp_connection_t * tc)
 
   if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
     {
-      tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1);
+      tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN, 1);
       return;
     }
 
@@ -1498,7 +1499,7 @@ tcp_timer_retransmit_syn_handler (tcp_connection_t * tc)
   tcp_push_ip_hdr (wrk, tc, b);
   tcp_enqueue_to_ip_lookup (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index);
 
-  tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN,
+  tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN,
 		    tc->rto * TCP_TO_TIMER_TICK);
 }
 
@@ -1529,7 +1530,7 @@ tcp_timer_persist_handler (tcp_connection_t * tc)
    * next time */
   if (!available_bytes)
     {
-      tcp_persist_timer_set (tc);
+      tcp_persist_timer_set (&wrk->timer_wheel, tc);
       return;
     }
 
@@ -1545,7 +1546,7 @@ tcp_timer_persist_handler (tcp_connection_t * tc)
    */
   if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1)))
     {
-      tcp_persist_timer_set (tc);
+      tcp_persist_timer_set (&wrk->timer_wheel, tc);
       return;
     }
 
@@ -1576,7 +1577,7 @@ tcp_timer_persist_handler (tcp_connection_t * tc)
   tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
 
   /* Just sent new data, enable retransmit */
-  tcp_retransmit_timer_update (tc);
+  tcp_retransmit_timer_update (&wrk->timer_wheel, tc);
 
   return;
 
@@ -2171,9 +2172,6 @@ tcp_output_handle_packet (tcp_connection_t * tc0, vlib_buffer_t * b0,
 	}
     }
 
-  if (!TCP_ALWAYS_ACK)
-    tcp_timer_reset (tc0, TCP_TIMER_DELACK);
-
   tc0->segs_out += 1;
 }
 
diff --git a/src/vnet/tcp/tcp_timer.h b/src/vnet/tcp/tcp_timer.h
index 08516b7eec2..d34fdcb146d 100644
--- a/src/vnet/tcp/tcp_timer.h
+++ b/src/vnet/tcp/tcp_timer.h
@@ -15,8 +15,109 @@
 #ifndef __included_tcp_timer_h__
 #define __included_tcp_timer_h__
 
-#include <vppinfra/tw_timer_16t_2w_512sl.h>
-#include <vppinfra/tw_timer_16t_1w_2048sl.h>
+#include <vnet/tcp/tcp_types.h>
+
+always_inline void
+tcp_timer_set (tcp_timer_wheel_t * tw, tcp_connection_t * tc, u8 timer_id,
+	       u32 interval)
+{
+  ASSERT (tc->c_thread_index == vlib_get_thread_index ());
+  ASSERT (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID);
+  tc->timers[timer_id] = tw_timer_start_16t_2w_512sl (tw, tc->c_c_index,
+						      timer_id, interval);
+}
+
+always_inline void
+tcp_timer_reset (tcp_timer_wheel_t * tw, tcp_connection_t * tc, u8 timer_id)
+{
+  ASSERT (tc->c_thread_index == vlib_get_thread_index ());
+  if (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID)
+    return;
+
+  tw_timer_stop_16t_2w_512sl (tw, tc->timers[timer_id]);
+  tc->timers[timer_id] = TCP_TIMER_HANDLE_INVALID;
+}
+
+always_inline void
+tcp_timer_update (tcp_timer_wheel_t * tw, tcp_connection_t * tc, u8 timer_id,
+		  u32 interval)
+{
+  ASSERT (tc->c_thread_index == vlib_get_thread_index ());
+  if (tc->timers[timer_id] != TCP_TIMER_HANDLE_INVALID)
+    tw_timer_update_16t_2w_512sl (tw, tc->timers[timer_id], interval);
+  else
+    tc->timers[timer_id] = tw_timer_start_16t_2w_512sl (tw, tc->c_c_index,
+							timer_id, interval);
+}
+
+always_inline void
+tcp_retransmit_timer_set (tcp_timer_wheel_t * tw, tcp_connection_t * tc)
+{
+  ASSERT (tc->snd_una != tc->snd_una_max);
+  tcp_timer_set (tw, tc, TCP_TIMER_RETRANSMIT,
+		 clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
+}
+
+always_inline void
+tcp_retransmit_timer_reset (tcp_timer_wheel_t * tw, tcp_connection_t * tc)
+{
+  tcp_timer_reset (tw, tc, TCP_TIMER_RETRANSMIT);
+}
+
+always_inline void
+tcp_retransmit_timer_force_update (tcp_timer_wheel_t * tw,
+				   tcp_connection_t * tc)
+{
+  tcp_timer_update (tw, tc, TCP_TIMER_RETRANSMIT,
+		    clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
+}
+
+always_inline void
+tcp_persist_timer_set (tcp_timer_wheel_t * tw, tcp_connection_t * tc)
+{
+  /* Reuse RTO. It's backed off in handler */
+  tcp_timer_set (tw, tc, TCP_TIMER_PERSIST,
+		 clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
+}
+
+always_inline void
+tcp_persist_timer_update (tcp_timer_wheel_t * tw, tcp_connection_t * tc)
+{
+  u32 interval;
+
+  if (seq_leq (tc->snd_una, tc->snd_congestion + tc->burst_acked))
+    interval = 1;
+  else
+    interval = clib_max (tc->rto * TCP_TO_TIMER_TICK, 1);
+
+  tcp_timer_update (tw, tc, TCP_TIMER_PERSIST, interval);
+}
+
+always_inline void
+tcp_persist_timer_reset (tcp_timer_wheel_t * tw, tcp_connection_t * tc)
+{
+  tcp_timer_reset (tw, tc, TCP_TIMER_PERSIST);
+}
+
+always_inline void
+tcp_retransmit_timer_update (tcp_timer_wheel_t * tw, tcp_connection_t * tc)
+{
+  if (tc->snd_una == tc->snd_nxt)
+    {
+      tcp_retransmit_timer_reset (tw, tc);
+      if (tc->snd_wnd < tc->snd_mss)
+	tcp_persist_timer_update (tw, tc);
+    }
+  else
+    tcp_timer_update (tw, tc, TCP_TIMER_RETRANSMIT,
+		      clib_max (tc->rto * TCP_TO_TIMER_TICK, 1));
+}
+
+always_inline u8
+tcp_timer_is_active (tcp_connection_t * tc, tcp_timers_e timer)
+{
+  return tc->timers[timer] != TCP_TIMER_HANDLE_INVALID;
+}
 
 #endif /* __included_tcp_timer_h__ */
 
diff --git a/src/vnet/tcp/tcp_types.h b/src/vnet/tcp/tcp_types.h
index ccb7ae86aed..cc4f4b3df3a 100644
--- a/src/vnet/tcp/tcp_types.h
+++ b/src/vnet/tcp/tcp_types.h
@@ -20,6 +20,7 @@
 #include <vppinfra/rbtree.h>
 #include <vnet/tcp/tcp_packet.h>
 #include <vnet/session/transport.h>
+#include <vppinfra/tw_timer_16t_2w_512sl.h>
 
 #define TCP_TICK 0.001			/**< TCP tick period (s) */
 #define THZ (u32) (1/TCP_TICK)		/**< TCP tick frequency */
@@ -440,6 +441,8 @@ tcp_get_connection_from_transport (transport_connection_t * tconn)
   return (tcp_connection_t *) tconn;
 }
 
+typedef tw_timer_wheel_16t_2w_512sl_t tcp_timer_wheel_t;
+
 #endif /* SRC_VNET_TCP_TCP_TYPES_H_ */
 
 /*