1 files changed, 147 insertions, 101 deletions

diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c
index e34f773d7d3..f24ddb3a879 100644
--- a/src/vnet/tcp/tcp.c
+++ b/src/vnet/tcp/tcp.c
@@ -1288,103 +1288,23 @@ tcp_session_tx_fifo_offset (transport_connection_t * trans_conn)
 }
 
 static void
-tcp_update_time (f64 now, u8 thread_index)
-{
-  tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
-
-  tcp_set_time_now (wrk);
-  tw_timer_expire_timers_16t_2w_512sl (&wrk->timer_wheel, now);
-  tcp_flush_frames_to_output (wrk);
-}
-
-static void
-tcp_session_flush_data (transport_connection_t * tconn)
-{
-  tcp_connection_t *tc = (tcp_connection_t *) tconn;
-  if (tc->flags & TCP_CONN_PSH_PENDING)
-    return;
-  tc->flags |= TCP_CONN_PSH_PENDING;
-  tc->psh_seq = tc->snd_una + transport_max_tx_dequeue (tconn) - 1;
-}
-
-/* *INDENT-OFF* */
-const static transport_proto_vft_t tcp_proto = {
-  .enable = vnet_tcp_enable_disable,
-  .start_listen = tcp_session_bind,
-  .stop_listen = tcp_session_unbind,
-  .push_header = tcp_session_push_header,
-  .get_connection = tcp_session_get_transport,
-  .get_listener = tcp_session_get_listener,
-  .get_half_open = tcp_half_open_session_get_transport,
-  .connect = tcp_session_open,
-  .close = tcp_session_close,
-  .cleanup = tcp_session_cleanup,
-  .reset = tcp_session_reset,
-  .send_mss = tcp_session_send_mss,
-  .send_space = tcp_session_send_space,
-  .update_time = tcp_update_time,
-  .tx_fifo_offset = tcp_session_tx_fifo_offset,
-  .flush_data = tcp_session_flush_data,
-  .custom_tx = tcp_session_custom_tx,
-  .format_connection = format_tcp_session,
-  .format_listener = format_tcp_listener_session,
-  .format_half_open = format_tcp_half_open_session,
-  .transport_options = {
-    .tx_type = TRANSPORT_TX_PEEK,
-    .service_type = TRANSPORT_SERVICE_VC,
-  },
-};
-/* *INDENT-ON* */
-
-void
-tcp_connection_tx_pacer_update (tcp_connection_t * tc)
-{
-  if (!transport_connection_is_tx_paced (&tc->connection))
-    return;
-
-  f64 srtt = clib_min ((f64) tc->srtt * TCP_TICK, tc->mrtt_us);
-
-  transport_connection_tx_pacer_update (&tc->connection,
-					tcp_cc_get_pacing_rate (tc),
-					srtt * CLIB_US_TIME_FREQ);
-}
-
-void
-tcp_connection_tx_pacer_reset (tcp_connection_t * tc, u32 window,
-			       u32 start_bucket)
-{
-  f64 srtt = clib_min ((f64) tc->srtt * TCP_TICK, tc->mrtt_us);
-  transport_connection_tx_pacer_reset (&tc->connection,
-				       tcp_cc_get_pacing_rate (tc),
-				       start_bucket,
-				       srtt * CLIB_US_TIME_FREQ);
-}
-
-static void
-tcp_timer_waitclose_handler (u32 conn_index, u32 thread_index)
+tcp_timer_waitclose_handler (tcp_connection_t * tc)
 {
-  tcp_connection_t *tc;
-
-  tc = tcp_connection_get (conn_index, thread_index);
-  if (!tc)
-    return;
-
   switch (tc->state)
     {
     case TCP_STATE_CLOSE_WAIT:
       tcp_connection_timers_reset (tc);
       session_transport_closed_notify (&tc->connection);
-
+      /* App never returned with a close */
       if (!(tc->flags & TCP_CONN_FINPNDG))
 	{
-	  clib_warning ("close-wait with fin sent");
 	  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
 	  tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.cleanup_time);
+	  tcp_worker_stats_inc (tc->c_thread_index, to_closewait, 1);
 	  break;
 	}
 
-      /* Session didn't come back with a close. Send FIN either way
-       * and switch to LAST_ACK. */
+      /* Send FIN either way and switch to LAST_ACK. */
       tcp_cong_recovery_off (tc);
       /* Make sure we don't try to send unsent data */
       tc->snd_nxt = tc->snd_una;
@@ -1393,7 +1313,7 @@ tcp_timer_waitclose_handler (u32 conn_index, u32 thread_index)
 
       /* Make sure we don't wait in LAST ACK forever */
       tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.lastack_time);
-      tcp_worker_stats_inc (thread_index, to_closewait, 1);
+      tcp_worker_stats_inc (tc->c_thread_index, to_closewait2, 1);
 
       /* Don't delete the connection yet */
       break;
@@ -1415,21 +1335,21 @@ tcp_timer_waitclose_handler (u32 conn_index, u32 thread_index)
 	  tcp_connection_set_state (tc, TCP_STATE_CLOSED);
 	  tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.cleanup_time);
 	}
-      tcp_worker_stats_inc (thread_index, to_finwait1, 1);
+      tcp_worker_stats_inc (tc->c_thread_index, to_finwait1, 1);
       break;
     case TCP_STATE_LAST_ACK:
       tcp_connection_timers_reset (tc);
       tcp_connection_set_state (tc, TCP_STATE_CLOSED);
       tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.cleanup_time);
       session_transport_closed_notify (&tc->connection);
-      tcp_worker_stats_inc (thread_index, to_lastack, 1);
+      tcp_worker_stats_inc (tc->c_thread_index, to_lastack, 1);
       break;
     case TCP_STATE_CLOSING:
       tcp_connection_timers_reset (tc);
       tcp_connection_set_state (tc, TCP_STATE_CLOSED);
       tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.cleanup_time);
       session_transport_closed_notify (&tc->connection);
-      tcp_worker_stats_inc (thread_index, to_closing, 1);
+      tcp_worker_stats_inc (tc->c_thread_index, to_closing, 1);
       break;
     case TCP_STATE_FIN_WAIT_2:
       tcp_send_reset (tc);
@@ -1437,7 +1357,7 @@ tcp_timer_waitclose_handler (u32 conn_index, u32 thread_index)
       tcp_connection_set_state (tc, TCP_STATE_CLOSED);
       session_transport_closed_notify (&tc->connection);
       tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.cleanup_time);
-      tcp_worker_stats_inc (thread_index, to_finwait2, 1);
+      tcp_worker_stats_inc (tc->c_thread_index, to_finwait2, 1);
       break;
     default:
       tcp_connection_del (tc);
@@ -1457,15 +1377,144 @@ static timer_expiration_handler *timer_expiration_handlers[TCP_N_TIMERS] =
 /* *INDENT-ON* */
 
 static void
+tcp_dispatch_pending_timers (tcp_worker_ctx_t * wrk)
+{
+  u32 n_timers, connection_index, timer_id, thread_index, timer_handle;
+  tcp_connection_t *tc;
+  int i;
+
+  if (!(n_timers = clib_fifo_elts (wrk->pending_timers)))
+    return;
+
+  thread_index = wrk->vm->thread_index;
+  for (i = 0; i < clib_min (n_timers, 32); i++)
+    {
+      clib_fifo_sub1 (wrk->pending_timers, timer_handle);
+      connection_index = timer_handle & 0x0FFFFFFF;
+      timer_id = timer_handle >> 28;
+
+      if (PREDICT_TRUE (timer_id != TCP_TIMER_RETRANSMIT_SYN))
+	tc = tcp_connection_get (connection_index, thread_index);
+      else
+	tc = tcp_half_open_connection_get (connection_index);
+
+      if (PREDICT_FALSE (!tc))
+	continue;
+
+      /* Skip timer if it was rearmed while pending dispatch */
+      if (PREDICT_FALSE (tc->timers[timer_id] != TCP_TIMER_HANDLE_INVALID))
+	continue;
+
+      (*timer_expiration_handlers[timer_id]) (tc);
+    }
+}
+
+/**
+ * Flush ip lookup tx frames populated by timer pops
+ */
+static void
+tcp_flush_frames_to_output (tcp_worker_ctx_t * wrk)
+{
+  if (wrk->ip_lookup_tx_frames[0])
+    {
+      vlib_put_frame_to_node (wrk->vm, ip4_lookup_node.index,
+			      wrk->ip_lookup_tx_frames[0]);
+      wrk->ip_lookup_tx_frames[0] = 0;
+    }
+  if (wrk->ip_lookup_tx_frames[1])
+    {
+      vlib_put_frame_to_node (wrk->vm, ip6_lookup_node.index,
+			      wrk->ip_lookup_tx_frames[1]);
+      wrk->ip_lookup_tx_frames[1] = 0;
+    }
+}
+
+static void
+tcp_update_time (f64 now, u8 thread_index)
+{
+  tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
+
+  tcp_set_time_now (wrk);
+  tw_timer_expire_timers_16t_2w_512sl (&wrk->timer_wheel, now);
+  tcp_dispatch_pending_timers (wrk);
+  tcp_flush_frames_to_output (wrk);
+}
+
+static void
+tcp_session_flush_data (transport_connection_t * tconn)
+{
+  tcp_connection_t *tc = (tcp_connection_t *) tconn;
+  if (tc->flags & TCP_CONN_PSH_PENDING)
+    return;
+  tc->flags |= TCP_CONN_PSH_PENDING;
+  tc->psh_seq = tc->snd_una + transport_max_tx_dequeue (tconn) - 1;
+}
+
+/* *INDENT-OFF* */
+const static transport_proto_vft_t tcp_proto = {
+  .enable = vnet_tcp_enable_disable,
+  .start_listen = tcp_session_bind,
+  .stop_listen = tcp_session_unbind,
+  .push_header = tcp_session_push_header,
+  .get_connection = tcp_session_get_transport,
+  .get_listener = tcp_session_get_listener,
+  .get_half_open = tcp_half_open_session_get_transport,
+  .connect = tcp_session_open,
+  .close = tcp_session_close,
+  .cleanup = tcp_session_cleanup,
+  .reset = tcp_session_reset,
+  .send_mss = tcp_session_send_mss,
+  .send_space = tcp_session_send_space,
+  .update_time = tcp_update_time,
+  .tx_fifo_offset = tcp_session_tx_fifo_offset,
+  .flush_data = tcp_session_flush_data,
+  .custom_tx = tcp_session_custom_tx,
+  .format_connection = format_tcp_session,
+  .format_listener = format_tcp_listener_session,
+  .format_half_open = format_tcp_half_open_session,
+  .transport_options = {
+    .tx_type = TRANSPORT_TX_PEEK,
+    .service_type = TRANSPORT_SERVICE_VC,
+  },
+};
+/* *INDENT-ON* */
+
+void
+tcp_connection_tx_pacer_update (tcp_connection_t * tc)
+{
+  if (!transport_connection_is_tx_paced (&tc->connection))
+    return;
+
+  f64 srtt = clib_min ((f64) tc->srtt * TCP_TICK, tc->mrtt_us);
+
+  transport_connection_tx_pacer_update (&tc->connection,
+					tcp_cc_get_pacing_rate (tc),
+					srtt * CLIB_US_TIME_FREQ);
+}
+
+void
+tcp_connection_tx_pacer_reset (tcp_connection_t * tc, u32 window,
+			       u32 start_bucket)
+{
+  f64 srtt = clib_min ((f64) tc->srtt * TCP_TICK, tc->mrtt_us);
+  transport_connection_tx_pacer_reset (&tc->connection,
+				       tcp_cc_get_pacing_rate (tc),
+				       start_bucket,
+				       srtt * CLIB_US_TIME_FREQ);
+}
+
+static void
 tcp_expired_timers_dispatch (u32 * expired_timers)
 {
   u32 thread_index = vlib_get_thread_index ();
   u32 connection_index, timer_id, n_expired;
+  tcp_worker_ctx_t *wrk;
   tcp_connection_t *tc;
   int i;
 
+  wrk = tcp_get_worker (thread_index);
   n_expired = vec_len (expired_timers);
-  tcp_worker_stats_inc (thread_index, timer_expirations, n_expired);
+  tcp_workerp_stats_inc (wrk, timer_expirations, n_expired);
 
   /*
    * Invalidate all timer handles before dispatching. This avoids dangling
@@ -1486,15 +1535,7 @@ tcp_expired_timers_dispatch (u32 * expired_timers)
       tc->timers[timer_id] = TCP_TIMER_HANDLE_INVALID;
     }
 
-  /*
-   * Dispatch expired timers
-   */
-  for (i = 0; i < n_expired; i++)
-    {
-      connection_index = expired_timers[i] & 0x0FFFFFFF;
-      timer_id = expired_timers[i] >> 28;
-      (*timer_expiration_handlers[timer_id]) (connection_index, thread_index);
-    }
+  clib_fifo_add (wrk->pending_timers, expired_timers, n_expired);
 }
 
 static void
@@ -2297,14 +2338,19 @@ show_tcp_stats_fn (vlib_main_t * vm, unformat_input_t * input,
   for (thread = 0; thread < vec_len (tm->wrk_ctx); thread++)
     {
       wrk = tcp_get_worker (thread);
-      vlib_cli_output (vm, "Thread %d:\n", thread);
+      vlib_cli_output (vm, "Thread %u:\n", thread);
+
+      if (clib_fifo_elts (wrk->pending_timers))
+	vlib_cli_output (vm, " %lu pending timers",
+			 clib_fifo_elts (wrk->pending_timers));
 
 #define _(name,type,str)					\
   if (wrk->stats.name)						\
-    vlib_cli_output (vm, " %ld %s", wrk->stats.name, str);
+    vlib_cli_output (vm, " %lu %s", wrk->stats.name, str);
       foreach_tcp_wrk_stat
 #undef _
     }
+
   return 0;
 }