tcp: time infra improvements

Type: improvement

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I998c0686f9f7dc556dda8b28e23bbed127d0aafc

12 files changed, 87 insertions, 47 deletions

diff --git a/src/plugins/unittest/tcp_test.c b/src/plugins/unittest/tcp_test.c
index eb2b6b68072..25b6744244e 100644
--- a/src/plugins/unittest/tcp_test.c
+++ b/src/plugins/unittest/tcp_test.c
@@ -1003,6 +1003,13 @@ tbt_seq_lt (u32 a, u32 b)
   return seq_lt (a, b);
 }
 
+static void
+tcp_test_set_time (u32 thread_index, u32 val)
+{
+  session_main.wrk[thread_index].last_vlib_time = val;
+  tcp_set_time_now (&tcp_main.wrk_ctx[thread_index], val);
+}
+
 static int
 tcp_test_delivery (vlib_main_t * vm, unformat_input_t * input)
 {
@@ -1031,7 +1038,7 @@ tcp_test_delivery (vlib_main_t * vm, unformat_input_t * input)
 
   /* Init data structures */
   memset (tc, 0, sizeof (*tc));
-  session_main.wrk[thread_index].last_vlib_time = 1;
+  tcp_test_set_time (thread_index, 1);
   transport_connection_tx_pacer_update (&tc->connection, rate, 1e6);
 
   tcp_bt_init (tc);
@@ -1056,7 +1063,7 @@ tcp_test_delivery (vlib_main_t * vm, unformat_input_t * input)
   TCP_TEST (!(bts->flags & TCP_BTS_IS_APP_LIMITED), "not app limited");
 
   /* 2) check delivery rate at time 2 */
-  session_main.wrk[thread_index].last_vlib_time = 2;
+  tcp_test_set_time (thread_index, 2);
   tc->snd_una = tc->snd_nxt = burst;
   tc->bytes_acked = burst;
 
@@ -1077,7 +1084,7 @@ tcp_test_delivery (vlib_main_t * vm, unformat_input_t * input)
   tc->snd_nxt += burst;
 
   /* 4) track second burst at time 3 */
-  session_main.wrk[thread_index].last_vlib_time = 3;
+  tcp_test_set_time (thread_index, 3);
   tcp_bt_track_tx (tc, burst);
   tc->snd_nxt += burst;
 
@@ -1094,7 +1101,7 @@ tcp_test_delivery (vlib_main_t * vm, unformat_input_t * input)
   TCP_TEST (bts->prev == bt->head, "prev should be head");
 
   /* 5) check delivery rate at time 4 */
-  session_main.wrk[thread_index].last_vlib_time = 4;
+  tcp_test_set_time (thread_index, 4);
   tc->snd_una = tc->snd_nxt;
   tc->bytes_acked = 2 * burst;
 
@@ -1124,17 +1131,17 @@ tcp_test_delivery (vlib_main_t * vm, unformat_input_t * input)
   tc->snd_nxt += burst;
 
   /* 2) track second burst at time 5 */
-  session_main.wrk[thread_index].last_vlib_time = 5;
+  tcp_test_set_time (thread_index, 5);
   tcp_bt_track_tx (tc, burst);
   tc->snd_nxt += burst;
 
   /* 3) track third burst at time 6 */
-  session_main.wrk[thread_index].last_vlib_time = 6;
+  tcp_test_set_time (thread_index, 6);
   tcp_bt_track_tx (tc, burst);
   tc->snd_nxt += burst;
 
   /* 4) track fourth burst at time 7 */
-  session_main.wrk[thread_index].last_vlib_time = 7;
+  tcp_test_set_time (thread_index, 7);
   /* Limited until last burst is acked */
   tc->app_limited = snd_una + 4 * burst - 1;
   tcp_bt_track_tx (tc, burst);
@@ -1147,7 +1154,7 @@ tcp_test_delivery (vlib_main_t * vm, unformat_input_t * input)
    * [snd_una + burst, snd_una + burst + 10]
    * [snd_una + 2 * burst + 10, snd_una + 2 * burst + 20]
    */
-  session_main.wrk[thread_index].last_vlib_time = 8;
+  tcp_test_set_time (thread_index, 8);
   tc->snd_una += 10;
   tc->bytes_acked = 10;
   sb->last_sacked_bytes = 20;
@@ -1192,7 +1199,7 @@ tcp_test_delivery (vlib_main_t * vm, unformat_input_t * input)
    * [snd_una + burst + 10, snd_una + 2 * burst + 10]
    * [snd_una + 2 * burst + 20, snd_una + 4 * burst]
    */
-  session_main.wrk[thread_index].last_vlib_time = 9;
+  tcp_test_set_time (thread_index, 9);
 
   tcp_bt_track_rxt (tc, snd_una + 10, snd_una + burst);
   TCP_TEST (tcp_bt_is_sane (bt), "tracker should be sane");
@@ -1238,7 +1245,7 @@ tcp_test_delivery (vlib_main_t * vm, unformat_input_t * input)
    * [snd_una + 2 * burst + 20, snd_una + 2 * burst + 30]
    * [snd_una + 2 * burst + 50, snd_una + 2 * burst + 60]
    */
-  session_main.wrk[thread_index].last_vlib_time = 10;
+  tcp_test_set_time (thread_index, 10);
   tc->snd_una = snd_una + 2 * burst;
   tc->bytes_acked = 2 * burst - 10;
   sb->last_sacked_bytes = 20;
@@ -1276,7 +1283,7 @@ tcp_test_delivery (vlib_main_t * vm, unformat_input_t * input)
   /*
    * 8) check delivery rate at time 11
    */
-  session_main.wrk[thread_index].last_vlib_time = 11;
+  tcp_test_set_time (thread_index, 11);
   tc->snd_una = tc->snd_nxt;
   tc->bytes_acked = 2 * burst;
   sb->last_sacked_bytes = 0;
@@ -1314,7 +1321,7 @@ tcp_test_delivery (vlib_main_t * vm, unformat_input_t * input)
   tcp_bt_track_tx (tc, burst);
   tc->snd_nxt += burst;
 
-  session_main.wrk[thread_index].last_vlib_time = 12;
+  tcp_test_set_time (thread_index, 12);
   tcp_bt_track_tx (tc, burst);
   tc->snd_nxt += burst;
 
diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h
index aba8a1c3fd1..55863163a89 100644
--- a/src/vnet/session/session.h
+++ b/src/vnet/session/session.h
@@ -684,6 +684,13 @@ session_add_pending_tx_buffer (u32 thread_index, u32 bi, u32 next_node)
   vec_add1 (wrk->pending_tx_nexts, next_node);
 }
 
+always_inline void
+session_wrk_update_time (session_worker_t *wrk, f64 now)
+{
+  wrk->last_vlib_time = now;
+  wrk->last_vlib_us_time = wrk->last_vlib_time * CLIB_US_TIME_FREQ;
+}
+
 fifo_segment_t *session_main_get_evt_q_segment (void);
 void session_node_enable_disable (u8 is_en);
 clib_error_t *vnet_session_enable_disable (vlib_main_t * vm, u8 is_en);
diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c
index f3713d00cd1..f8157cc6214 100644
--- a/src/vnet/session/session_node.c
+++ b/src/vnet/session/session_node.c
@@ -1431,8 +1431,7 @@ session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 
   SESSION_EVT (SESSION_EVT_DISPATCH_START, wrk);
 
-  wrk->last_vlib_time = vlib_time_now (vm);
-  wrk->last_vlib_us_time = wrk->last_vlib_time * CLIB_US_TIME_FREQ;
+  session_wrk_update_time (wrk, vlib_time_now (vm));
 
   /*
    *  Update transport time
diff --git a/src/vnet/session/transport.c b/src/vnet/session/transport.c
index 7cc2a8e04c5..4f6ac8b1c48 100644
--- a/src/vnet/session/transport.c
+++ b/src/vnet/session/transport.c
@@ -763,6 +763,12 @@ transport_connection_tx_pacer_update_bytes (transport_connection_t * tc,
 }
 
 void
+transport_update_pacer_time (u32 thread_index, clib_time_type_t now)
+{
+  session_wrk_update_time (session_main_get_worker (thread_index), now);
+}
+
+void
 transport_connection_reschedule (transport_connection_t * tc)
 {
   tc->flags &= ~TRANSPORT_CONNECTION_F_DESCHED;
diff --git a/src/vnet/session/transport.h b/src/vnet/session/transport.h
index 9a9b89fc9fd..efd2507ed4c 100644
--- a/src/vnet/session/transport.h
+++ b/src/vnet/session/transport.h
@@ -339,6 +339,14 @@ void
 transport_connection_tx_pacer_update_bytes (transport_connection_t * tc,
 					    u32 bytes);
 
+/**
+ * Request pacer time update
+ *
+ * @param thread_index	thread for which time is updated
+ * @param now		time now
+ */
+void transport_update_pacer_time (u32 thread_index, clib_time_type_t now);
+
 #endif /* SRC_VNET_SESSION_TRANSPORT_H_ */
 
 /*
diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c
index 72161ec8e17..2d384a65cfc 100644
--- a/src/vnet/tcp/tcp.c
+++ b/src/vnet/tcp/tcp.c
@@ -337,7 +337,7 @@ tcp_program_cleanup (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
   tcp_cleanup_req_t *req;
   clib_time_type_t now;
 
-  now = transport_time_now (tc->c_thread_index);
+  now = tcp_time_now_us (tc->c_thread_index);
   clib_fifo_add2 (wrk->pending_cleanups, req);
   req->connection_index = tc->c_c_index;
   req->free_time = now + tcp_cfg.cleanup_time;
@@ -675,7 +675,7 @@ tcp_init_snd_vars (tcp_connection_t * tc)
    * handshake may make it look as if time has flown in the opposite
    * direction for us.
    */
-  tcp_set_time_now (tcp_get_worker (vlib_get_thread_index ()));
+  tcp_update_time_now (tcp_get_worker (vlib_get_thread_index ()));
 
   tcp_init_rcv_mss (tc);
   tc->iss = tcp_generate_random_iss (tc);
@@ -1147,7 +1147,7 @@ tcp_update_time (f64 now, u8 thread_index)
 {
   tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index);
 
-  tcp_set_time_now (wrk);
+  tcp_set_time_now (wrk, now);
   tcp_handle_cleanups (wrk, now);
   tcp_timer_expire_timers (&wrk->timer_wheel, now);
   tcp_dispatch_pending_timers (wrk);
diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h
index 2725c1f51cf..418bc476cb9 100644
--- a/src/vnet/tcp/tcp.h
+++ b/src/vnet/tcp/tcp.h
@@ -91,15 +91,15 @@ typedef struct tcp_worker_ctx_
   /** convenience pointer to this thread's vlib main */
   vlib_main_t *vm;
 
+  /** Time used for high precision (us) measurements in seconds */
+  f64 time_us;
+
   /** Time measured in @ref TCP_TSTAMP_TICK used for time stamps */
-  u32 time_now;
+  u32 time_tstamp;
 
   /* Max timers to be handled per dispatch loop */
   u32 max_timers_per_loop;
 
-  /** Session layer edge indices to tcp output */
-  u32 tco_next_node[2];
-
   /* Fifo of pending timer expirations */
   u32 *pending_timers;
 
@@ -114,6 +114,9 @@ typedef struct tcp_worker_ctx_
   /* fifo of pending free requests */
   tcp_cleanup_req_t *pending_cleanups;
 
+  /** Session layer edge indices to tcp output */
+  u32 tco_next_node[2];
+
   /** worker timer wheel */
   tcp_timer_wheel_t timer_wheel;
 
diff --git a/src/vnet/tcp/tcp_cli.c b/src/vnet/tcp/tcp_cli.c
index 21634df69f7..6d7b7c8ce40 100644
--- a/src/vnet/tcp/tcp_cli.c
+++ b/src/vnet/tcp/tcp_cli.c
@@ -149,9 +149,8 @@ format_tcp_congestion (u8 * s, va_list * args)
 	      format_white_space, indent, tc->snd_congestion - tc->iss,
 	      tc->rcv_dupacks, tc->limited_transmit - tc->iss);
   s = format (s, "%Urxt_bytes %u rxt_delivered %u rxt_head %u rxt_ts %u\n",
-	      format_white_space, indent, tc->snd_rxt_bytes,
-	      tc->rxt_delivered, tc->rxt_head - tc->iss,
-	      tcp_time_now_w_thread (tc->c_thread_index) - tc->snd_rxt_ts);
+	      format_white_space, indent, tc->snd_rxt_bytes, tc->rxt_delivered,
+	      tc->rxt_head - tc->iss, tcp_tstamp (tc) - tc->snd_rxt_ts);
   if (tcp_in_fastrecovery (tc))
     prr_space = tcp_fastrecovery_prr_snd_space (tc);
   s = format (s, "%Uprr_start %u prr_delivered %u prr space %u\n",
@@ -202,7 +201,7 @@ format_tcp_vars (u8 * s, va_list * args)
   s = format (s, " tsval_recent %u\n", tc->tsval_recent);
   s = format (s, " tsecr %u tsecr_last_ack %u tsval_recent_age %u",
 	      tc->rcv_opts.tsecr, tc->tsecr_last_ack,
-	      tcp_time_now () - tc->tsval_recent_age);
+	      tcp_time_tstamp (tc->c_thread_index) - tc->tsval_recent_age);
   s = format (s, " snd_mss %u\n", tc->snd_mss);
   s = format (s, " rto %u rto_boff %u srtt %.1f us %.3f rttvar %.1f",
 	      tc->rto / 1000, tc->rto_boff, tc->srtt / 1000.0,
diff --git a/src/vnet/tcp/tcp_cubic.c b/src/vnet/tcp/tcp_cubic.c
index b8ac80a8feb..cc2ffeae9f0 100644
--- a/src/vnet/tcp/tcp_cubic.c
+++ b/src/vnet/tcp/tcp_cubic.c
@@ -51,7 +51,7 @@ STATIC_ASSERT (sizeof (cubic_data_t) <= TCP_CC_DATA_SZ, "cubic data len");
 static inline f64
 cubic_time (u32 thread_index)
 {
-  return transport_time_now (thread_index);
+  return tcp_time_now_us (thread_index);
 }
 
 /**
diff --git a/src/vnet/tcp/tcp_inlines.h b/src/vnet/tcp/tcp_inlines.h
index 45762671da8..a0121308008 100644
--- a/src/vnet/tcp/tcp_inlines.h
+++ b/src/vnet/tcp/tcp_inlines.h
@@ -187,16 +187,13 @@ tcp_is_lost_fin (tcp_connection_t * tc)
   return 0;
 }
 
+/**
+ * Time used to generate timestamps, not the timestamp
+ */
 always_inline u32
-tcp_time_now (void)
-{
-  return tcp_main.wrk_ctx[vlib_get_thread_index ()].time_now;
-}
-
-always_inline u32
-tcp_time_now_w_thread (u32 thread_index)
+tcp_time_tstamp (u32 thread_index)
 {
-  return tcp_main.wrk_ctx[thread_index].time_now;
+  return tcp_main.wrk_ctx[thread_index].time_tstamp;
 }
 
 /**
@@ -205,20 +202,34 @@ tcp_time_now_w_thread (u32 thread_index)
 always_inline u32
 tcp_tstamp (tcp_connection_t * tc)
 {
-  return (tcp_main.wrk_ctx[tc->c_thread_index].time_now -
+  return (tcp_main.wrk_ctx[tc->c_thread_index].time_tstamp -
 	  tc->timestamp_delta);
 }
 
 always_inline f64
 tcp_time_now_us (u32 thread_index)
 {
-  return transport_time_now (thread_index);
+  return tcp_main.wrk_ctx[thread_index].time_us;
 }
 
-always_inline u32
-tcp_set_time_now (tcp_worker_ctx_t * wrk)
+always_inline void
+tcp_set_time_now (tcp_worker_ctx_t *wrk, f64 now)
+{
+  /* TCP internal cache of time reference. Could use @ref transport_time_now
+   * but because @ref tcp_time_now_us is used per packet, caching might
+   * slightly improve efficiency. */
+  wrk->time_us = now;
+  wrk->time_tstamp = (u64) (now * TCP_TSTP_HZ);
+}
+
+always_inline void
+tcp_update_time_now (tcp_worker_ctx_t *wrk)
 {
-  return wrk->time_now = (u64) (vlib_time_now (wrk->vm) * TCP_TSTP_HZ);
+  f64 now = vlib_time_now (wrk->vm);
+
+  /* Both pacer and tcp us time need to be updated */
+  transport_update_pacer_time (wrk->vm->thread_index, now);
+  tcp_set_time_now (wrk, now);
 }
 
 always_inline tcp_connection_t *
@@ -359,7 +370,7 @@ tcp_init_w_buffer (tcp_connection_t * tc, vlib_buffer_t * b, u8 is_ip4)
   if (tcp_opts_tstamp (&tc->rcv_opts))
     {
       tc->tsval_recent = tc->rcv_opts.tsval;
-      tc->tsval_recent_age = tcp_time_now ();
+      tc->tsval_recent_age = tcp_time_tstamp (tc->c_thread_index);
     }
 
   if (tcp_opts_wscale (&tc->rcv_opts))
diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c
index 509732fa876..b64c236bd47 100644
--- a/src/vnet/tcp/tcp_input.c
+++ b/src/vnet/tcp/tcp_input.c
@@ -149,7 +149,7 @@ tcp_update_timestamp (tcp_connection_t * tc, u32 seq, u32 seq_end)
     {
       ASSERT (timestamp_leq (tc->tsval_recent, tc->rcv_opts.tsval));
       tc->tsval_recent = tc->rcv_opts.tsval;
-      tc->tsval_recent_age = tcp_time_now_w_thread (tc->c_thread_index);
+      tc->tsval_recent_age = tcp_time_tstamp (tc->c_thread_index);
     }
 }
 
@@ -288,7 +288,7 @@ tcp_segment_validate (tcp_worker_ctx_t * wrk, tcp_connection_t * tc0,
       /* If it just so happens that a segment updates tsval_recent for a
        * segment over 24 days old, invalidate tsval_recent. */
       if (timestamp_lt (tc0->tsval_recent_age + TCP_PAWS_IDLE,
-			tcp_time_now_w_thread (tc0->c_thread_index)))
+			tcp_time_tstamp (tc0->c_thread_index)))
 	{
 	  tc0->tsval_recent = tc0->rcv_opts.tsval;
 	  clib_warning ("paws failed: 24-day old segment");
@@ -1920,7 +1920,7 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       if (tcp_opts_tstamp (&new_tc0->rcv_opts))
 	{
 	  new_tc0->tsval_recent = new_tc0->rcv_opts.tsval;
-	  new_tc0->tsval_recent_age = tcp_time_now ();
+	  new_tc0->tsval_recent_age = tcp_time_tstamp (my_thread_index);
 	}
 
       if (tcp_opts_wscale (&new_tc0->rcv_opts))
@@ -2830,7 +2830,7 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
   u16 nexts[VLIB_FRAME_SIZE], *next;
 
-  tcp_set_time_now (tcp_get_worker (thread_index));
+  tcp_update_time_now (tcp_get_worker (thread_index));
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;
diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index 235a59be0b3..928e8249eb1 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -176,7 +176,7 @@ tcp_make_syn_options (tcp_connection_t * tc, tcp_options_t * opts)
   len += TCP_OPTION_LEN_WINDOW_SCALE;
 
   opts->flags |= TCP_OPTS_FLAG_TSTAMP;
-  opts->tsval = tcp_time_now ();
+  opts->tsval = tcp_time_tstamp (tc->c_thread_index);
   opts->tsecr = 0;
   len += TCP_OPTION_LEN_TIMESTAMP;
 
@@ -210,7 +210,7 @@ tcp_make_synack_options (tcp_connection_t * tc, tcp_options_t * opts)
   if (tcp_opts_tstamp (&tc->rcv_opts))
     {
       opts->flags |= TCP_OPTS_FLAG_TSTAMP;
-      opts->tsval = tcp_time_now ();
+      opts->tsval = tcp_time_tstamp (tc->c_thread_index);
       opts->tsecr = tc->tsval_recent;
       len += TCP_OPTION_LEN_TIMESTAMP;
     }
@@ -2188,7 +2188,7 @@ tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
   from = vlib_frame_vector_args (frame);
   n_left_from = frame->n_vectors;
-  tcp_set_time_now (tcp_get_worker (thread_index));
+  tcp_update_time_now (tcp_get_worker (thread_index));
 
   if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
     tcp46_output_trace_frame (vm, node, from, n_left_from);