session tcp: support pacer idle timeouts

Type: feature

To avoid excessive bursts, pacer must be provided with an estimated rtt
for the connection. That's used to compute an idle timeout, i.e., time
after which the bucket is reset to 1 mtu due to inactivity. For now,
idle timeout is computed as 5% of the rtt.

Change-Id: Ia0b752fe7b4ad0ce97b477fb886b0133a2321541
Signed-off-by: Florin Coras <fcoras@cisco.com>

4 files changed, 57 insertions, 47 deletions

diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c
index e1114b794e6..a072bfa0f77 100644
--- a/src/vnet/session/session_node.c
+++ b/src/vnet/session/session_node.c
@@ -873,16 +873,29 @@ session_tx_fifo_read_and_snd_i (session_worker_t * wrk,
       return SESSION_TX_NO_DATA;
     }
 
-  ctx->snd_space = transport_connection_snd_space (ctx->tc, ctx->snd_mss);
+  ctx->snd_space = transport_connection_snd_space (ctx->tc);
 
   /* This flow queue is "empty" so it should be re-evaluated before
    * the ones that have data to send. */
-  if (ctx->snd_space == 0)
+  if (!ctx->snd_space)
     {
       session_evt_add_head_old (wrk, elt);
       return SESSION_TX_NO_DATA;
     }
 
+  if (transport_connection_is_tx_paced (ctx->tc))
+    {
+      u32 snd_space = transport_connection_tx_pacer_burst (ctx->tc);
+      if (snd_space < TRANSPORT_PACER_MIN_BURST)
+	{
+	  session_evt_add_head_old (wrk, elt);
+	  return SESSION_TX_NO_DATA;
+	}
+      snd_space = clib_min (ctx->snd_space, snd_space);
+      ctx->snd_space = snd_space >= ctx->snd_mss ?
+	snd_space - snd_space % ctx->snd_mss : snd_space;
+    }
+
   /* Allow enqueuing of a new event */
   svm_fifo_unset_event (ctx->s->tx_fifo);
 
@@ -891,7 +904,7 @@ session_tx_fifo_read_and_snd_i (session_worker_t * wrk,
 
   if (PREDICT_FALSE (!ctx->max_len_to_snd))
     {
-      transport_connection_tx_pacer_reset_bucket (ctx->tc);
+      transport_connection_tx_pacer_reset_bucket (ctx->tc, 0);
       return SESSION_TX_NO_DATA;
     }
 
diff --git a/src/vnet/session/transport.c b/src/vnet/session/transport.c
index 902c7400af9..c8c58357afd 100644
--- a/src/vnet/session/transport.c
+++ b/src/vnet/session/transport.c
@@ -599,9 +599,9 @@ format_transport_pacer (u8 * s, va_list * args)
 
   now = transport_us_time_now (thread_index);
   diff = now - pacer->last_update;
-  s = format (s, "rate %lu bucket %lu t/p %.3f last_update %U",
+  s = format (s, "rate %lu bucket %lu t/p %.3f last_update %U idle %u",
 	      pacer->bytes_per_sec, pacer->bucket, pacer->tokens_per_period,
-	      format_clib_us_time, diff);
+	      format_clib_us_time, diff, pacer->idle_timeout_us);
   return s;
 }
 
@@ -611,15 +611,14 @@ spacer_max_burst (spacer_t * pacer, clib_us_time_t time_now)
   u64 n_periods = (time_now - pacer->last_update);
   u64 inc;
 
-  if (PREDICT_FALSE (n_periods > 5e4))
+  if (PREDICT_FALSE (n_periods > pacer->idle_timeout_us))
     {
       pacer->last_update = time_now;
       pacer->bucket = TRANSPORT_PACER_MIN_BURST;
       return TRANSPORT_PACER_MIN_BURST;
     }
 
-  if (n_periods > 0
-      && (inc = (f32) n_periods * pacer->tokens_per_period) > 10)
+  if ((inc = (f32) n_periods * pacer->tokens_per_period) > 10)
     {
       pacer->last_update = time_now;
       pacer->bucket = clib_min (pacer->bucket + inc, pacer->bytes_per_sec);
@@ -636,11 +635,14 @@ spacer_update_bucket (spacer_t * pacer, u32 bytes)
 }
 
 static inline void
-spacer_set_pace_rate (spacer_t * pacer, u64 rate_bytes_per_sec)
+spacer_set_pace_rate (spacer_t * pacer, u64 rate_bytes_per_sec,
+		      clib_us_time_t rtt)
 {
   ASSERT (rate_bytes_per_sec != 0);
   pacer->bytes_per_sec = rate_bytes_per_sec;
   pacer->tokens_per_period = rate_bytes_per_sec * CLIB_US_TIME_PERIOD;
+  pacer->idle_timeout_us = clib_max (rtt * TRANSPORT_PACER_IDLE_FACTOR,
+				     TRANSPORT_PACER_MIN_IDLE);
 }
 
 static inline u64
@@ -658,17 +660,19 @@ spacer_reset (spacer_t * pacer, clib_us_time_t time_now, u64 bucket)
 
 void
 transport_connection_tx_pacer_reset (transport_connection_t * tc,
-				     u64 rate_bytes_per_sec, u32 start_bucket)
+				     u64 rate_bytes_per_sec, u32 start_bucket,
+				     clib_us_time_t rtt)
 {
-  spacer_set_pace_rate (&tc->pacer, rate_bytes_per_sec);
+  spacer_set_pace_rate (&tc->pacer, rate_bytes_per_sec, rtt);
   spacer_reset (&tc->pacer, transport_us_time_now (tc->thread_index),
 		start_bucket);
 }
 
 void
-transport_connection_tx_pacer_reset_bucket (transport_connection_t * tc)
+transport_connection_tx_pacer_reset_bucket (transport_connection_t * tc,
+					    u32 bucket)
 {
-  spacer_reset (&tc->pacer, transport_us_time_now (tc->thread_index), 0);
+  spacer_reset (&tc->pacer, transport_us_time_now (tc->thread_index), bucket);
 }
 
 void
@@ -678,14 +682,14 @@ transport_connection_tx_pacer_init (transport_connection_t * tc,
 {
   tc->flags |= TRANSPORT_CONNECTION_F_IS_TX_PACED;
   transport_connection_tx_pacer_reset (tc, rate_bytes_per_sec,
-				       initial_bucket);
+				       initial_bucket, 1e6);
 }
 
 void
 transport_connection_tx_pacer_update (transport_connection_t * tc,
-				      u64 bytes_per_sec)
+				      u64 bytes_per_sec, clib_us_time_t rtt)
 {
-  spacer_set_pace_rate (&tc->pacer, bytes_per_sec);
+  spacer_set_pace_rate (&tc->pacer, bytes_per_sec, rtt);
 }
 
 u32
@@ -695,24 +699,6 @@ transport_connection_tx_pacer_burst (transport_connection_t * tc)
 			   transport_us_time_now (tc->thread_index));
 }
 
-u32
-transport_connection_snd_space (transport_connection_t * tc, u16 mss)
-{
-  u32 snd_space, max_paced_burst;
-
-  snd_space = tp_vfts[tc->proto].send_space (tc);
-  if (snd_space && transport_connection_is_tx_paced (tc))
-    {
-      clib_us_time_t now = transport_us_time_now (tc->thread_index);
-      max_paced_burst = spacer_max_burst (&tc->pacer, now);
-      max_paced_burst =
-	(max_paced_burst < TRANSPORT_PACER_MIN_BURST) ? 0 : max_paced_burst;
-      snd_space = clib_min (snd_space, max_paced_burst);
-      return snd_space >= mss ? snd_space - snd_space % mss : snd_space;
-    }
-  return snd_space;
-}
-
 u64
 transport_connection_tx_pacer_rate (transport_connection_t * tc)
 {
diff --git a/src/vnet/session/transport.h b/src/vnet/session/transport.h
index df246a9b305..adc695f5e5a 100644
--- a/src/vnet/session/transport.h
+++ b/src/vnet/session/transport.h
@@ -22,6 +22,8 @@
 #define TRANSPORT_PACER_MIN_MSS 	1460
 #define TRANSPORT_PACER_MIN_BURST 	TRANSPORT_PACER_MIN_MSS
 #define TRANSPORT_PACER_MAX_BURST	(43 * TRANSPORT_PACER_MIN_MSS)
+#define TRANSPORT_PACER_MIN_IDLE	100
+#define TRANSPORT_PACER_IDLE_FACTOR	0.05
 
 typedef struct _transport_options_t
 {
@@ -148,6 +150,17 @@ transport_app_rx_evt (transport_proto_t tp, u32 conn_index, u32 thread_index)
   return tp_vfts[tp].app_rx_evt (tc);
 }
 
+/**
+ * Get maximum tx burst allowed for transport connection
+ *
+ * @param tc		transport connection
+ */
+static inline u32
+transport_connection_snd_space (transport_connection_t * tc)
+{
+  return tp_vfts[tc->proto].send_space (tc);
+}
+
 void transport_register_protocol (transport_proto_t transport_proto,
 				  const transport_proto_vft_t * vft,
 				  fib_protocol_t fib_proto, u32 output_node);
@@ -174,7 +187,8 @@ transport_elog_track_index (transport_connection_t * tc)
 
 void transport_connection_tx_pacer_reset (transport_connection_t * tc,
 					  u64 rate_bytes_per_sec,
-					  u32 initial_bucket);
+					  u32 initial_bucket,
+					  clib_us_time_t rtt);
 /**
  * Initialize tx pacer for connection
  *
@@ -191,18 +205,13 @@ void transport_connection_tx_pacer_init (transport_connection_t * tc,
  *
  * @param tc			transport connection
  * @param bytes_per_sec		new pacing rate
+ * @param rtt			connection rtt that is used to compute
+ * 				inactivity time after which pacer bucket is
+ * 				reset to 1 mtu
  */
 void transport_connection_tx_pacer_update (transport_connection_t * tc,
-					   u64 bytes_per_sec);
-
-/**
- * Get maximum tx burst allowed for transport connection
- *
- * @param tc		transport connection
- * @param time_now	current cpu time as returned by @ref clib_cpu_time_now
- * @param mss		transport's mss
- */
-u32 transport_connection_snd_space (transport_connection_t * tc, u16 mss);
+					   u64 bytes_per_sec,
+					   clib_us_time_t rtt);
 
 /**
  * Get tx pacer max burst
@@ -225,9 +234,10 @@ u64 transport_connection_tx_pacer_rate (transport_connection_t * tc);
  * Reset tx pacer bucket
  *
  * @param tc		transport connection
- * @param time_now	current cpu time
+ * @param bucket	value the bucket will be reset to
  */
-void transport_connection_tx_pacer_reset_bucket (transport_connection_t * tc);
+void transport_connection_tx_pacer_reset_bucket (transport_connection_t * tc,
+						 u32 bucket);
 
 /**
  * Check if transport connection is paced
diff --git a/src/vnet/session/transport_types.h b/src/vnet/session/transport_types.h
index b39a07cd3de..214a6380d88 100644
--- a/src/vnet/session/transport_types.h
+++ b/src/vnet/session/transport_types.h
@@ -54,6 +54,7 @@ typedef struct _spacer
   u64 bucket;
   clib_us_time_t last_update;
   f32 tokens_per_period;
+  u32 idle_timeout_us;
 } spacer_t;
 
 #define TRANSPORT_CONN_ID_LEN	44