summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/vnet/session/session.c2
-rw-r--r--src/vnet/session/session.h15
-rw-r--r--src/vnet/session/session_node.c10
-rw-r--r--src/vnet/session/transport.c100
-rw-r--r--src/vnet/session/transport.h25
-rw-r--r--src/vnet/session/transport_types.h2
-rw-r--r--src/vnet/tcp/tcp.c6
-rwxr-xr-xsrc/vnet/tcp/tcp_input.c4
-rw-r--r--src/vnet/tcp/tcp_output.c14
-rw-r--r--src/vppinfra/time.h7
10 files changed, 81 insertions, 104 deletions
diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c
index d9a733d356e..5cd840bd834 100644
--- a/src/vnet/session/session.c
+++ b/src/vnet/session/session.c
@@ -1494,6 +1494,7 @@ session_manager_main_enable (vlib_main_t * vm)
wrk->old_head = clib_llist_make_head (wrk->event_elts, evt_list);
wrk->vm = vlib_mains[i];
wrk->last_vlib_time = vlib_time_now (vlib_mains[i]);
+ wrk->last_vlib_us_time = wrk->last_vlib_time * CLIB_US_TIME_FREQ;
if (num_threads > 1)
clib_rwlock_init (&smm->wrk[i].peekers_rw_locks);
@@ -1537,7 +1538,6 @@ session_manager_main_enable (vlib_main_t * vm)
/* Enable transports */
transport_enable_disable (vm, 1);
- transport_init_tx_pacers_period ();
return 0;
}
diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h
index 9fe35b19b25..830a34b72e8 100644
--- a/src/vnet/session/session.h
+++ b/src/vnet/session/session.h
@@ -84,7 +84,10 @@ typedef struct session_worker_
svm_msg_q_t *vpp_event_queue;
/** vlib_time_now last time around the track */
- f64 last_vlib_time;
+ clib_time_type_t last_vlib_time;
+
+ /** vlib_time_now rounded to us precision and as u64 */
+ clib_us_time_t last_vlib_us_time;
/** Convenience pointer to this worker's vlib_main */
vlib_main_t *vm;
@@ -124,7 +127,7 @@ typedef struct session_worker_
#if SESSION_DEBUG
/** last event poll time by thread */
- f64 last_event_poll;
+ clib_time_type_t last_event_poll;
#endif
} session_worker_t;
@@ -516,12 +519,18 @@ transport_rx_fifo_has_ooo_data (transport_connection_t * tc)
return svm_fifo_has_ooo_data (s->rx_fifo);
}
-always_inline f64
+always_inline clib_time_type_t
transport_time_now (u32 thread_index)
{
return session_main.wrk[thread_index].last_vlib_time;
}
+always_inline clib_us_time_t
+transport_us_time_now (u32 thread_index)
+{
+ return session_main.wrk[thread_index].last_vlib_us_time;
+}
+
always_inline void
transport_add_tx_event (transport_connection_t * tc)
{
diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c
index 3bfc80eca66..e5faf3c1663 100644
--- a/src/vnet/session/session_node.c
+++ b/src/vnet/session/session_node.c
@@ -871,10 +871,7 @@ session_tx_fifo_read_and_snd_i (session_worker_t * wrk,
return SESSION_TX_NO_DATA;
}
- ctx->snd_space = transport_connection_snd_space (ctx->tc,
- vm->clib_time.
- last_cpu_time,
- ctx->snd_mss);
+ ctx->snd_space = transport_connection_snd_space (ctx->tc, ctx->snd_mss);
/* This flow queue is "empty" so it should be re-evaluated before
* the ones that have data to send. */
@@ -892,9 +889,7 @@ session_tx_fifo_read_and_snd_i (session_worker_t * wrk,
if (PREDICT_FALSE (!ctx->max_len_to_snd))
{
- transport_connection_tx_pacer_reset_bucket (ctx->tc,
- vm->clib_time.
- last_cpu_time);
+ transport_connection_tx_pacer_reset_bucket (ctx->tc);
return SESSION_TX_NO_DATA;
}
@@ -1257,6 +1252,7 @@ session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
SESSION_EVT (SESSION_EVT_DISPATCH_START, wrk);
wrk->last_vlib_time = vlib_time_now (vm);
+ wrk->last_vlib_us_time = wrk->last_vlib_time * CLIB_US_TIME_FREQ;
/*
* Update transport time
diff --git a/src/vnet/session/transport.c b/src/vnet/session/transport.c
index 8f7e30c8602..3160a48d37d 100644
--- a/src/vnet/session/transport.c
+++ b/src/vnet/session/transport.c
@@ -42,11 +42,6 @@ static transport_endpoint_t *local_endpoints;
*/
static clib_spinlock_t local_endpoints_lock;
-/*
- * Period used by transport pacers. Initialized by session layer
- */
-static double transport_pacer_period;
-
u8 *
format_transport_proto (u8 * s, va_list * args)
{
@@ -107,7 +102,7 @@ format_transport_connection (u8 * s, va_list * args)
{
indent = format_get_indent (s) + 1;
s = format (s, "%Upacer: %U\n", format_white_space, indent,
- format_transport_pacer, &tc->pacer);
+ format_transport_pacer, &tc->pacer, tc->thread_index);
}
return s;
}
@@ -555,33 +550,41 @@ transport_alloc_local_endpoint (u8 proto, transport_endpoint_cfg_t * rmt_cfg,
return 0;
}
-#define SPACER_CPU_TICKS_PER_PERIOD_SHIFT 10
-#define SPACER_CPU_TICKS_PER_PERIOD (1 << SPACER_CPU_TICKS_PER_PERIOD_SHIFT)
+u8 *
+format_clib_us_time (u8 * s, va_list * args)
+{
+ clib_us_time_t t = va_arg (*args, clib_us_time_t);
+ if (t < 1e3)
+ s = format (s, "%u us", t);
+ else
+ s = format (s, "%.3f s", (f64) t * CLIB_US_TIME_PERIOD);
+ return s;
+}
u8 *
format_transport_pacer (u8 * s, va_list * args)
{
spacer_t *pacer = va_arg (*args, spacer_t *);
- vlib_main_t *vm = vlib_get_main ();
- u64 now, diff;
+ u32 thread_index = va_arg (*args, int);
+ clib_us_time_t now, diff;
- now = vm->clib_time.last_cpu_time;
- diff = now - (pacer->last_update << SPACER_CPU_TICKS_PER_PERIOD_SHIFT);
- s = format (s, "rate %u bucket %u t/p %.3f last_update %.3f",
+ now = transport_us_time_now (thread_index);
+ diff = now - pacer->last_update;
+ s = format (s, "rate %lu bucket %lu t/p %.3f last_update %U",
pacer->bytes_per_sec, pacer->bucket, pacer->tokens_per_period,
- diff * vm->clib_time.seconds_per_clock);
+ format_clib_us_time, diff);
return s;
}
static inline u32
-spacer_max_burst (spacer_t * pacer, u64 norm_time_now)
+spacer_max_burst (spacer_t * pacer, clib_us_time_t time_now)
{
- u64 n_periods = norm_time_now - pacer->last_update;
+ u64 n_periods = (time_now - pacer->last_update);
u64 inc;
- if (PREDICT_FALSE (n_periods > 5e5))
+ if (PREDICT_FALSE (n_periods > 5e4))
{
- pacer->last_update = norm_time_now;
+ pacer->last_update = time_now;
pacer->bucket = TRANSPORT_PACER_MIN_BURST;
return TRANSPORT_PACER_MIN_BURST;
}
@@ -589,7 +592,7 @@ spacer_max_burst (spacer_t * pacer, u64 norm_time_now)
if (n_periods > 0
&& (inc = (f32) n_periods * pacer->tokens_per_period) > 10)
{
- pacer->last_update = norm_time_now;
+ pacer->last_update = time_now;
pacer->bucket = clib_min (pacer->bucket + inc, pacer->bytes_per_sec);
}
@@ -608,7 +611,7 @@ spacer_set_pace_rate (spacer_t * pacer, u64 rate_bytes_per_sec)
{
ASSERT (rate_bytes_per_sec != 0);
pacer->bytes_per_sec = rate_bytes_per_sec;
- pacer->tokens_per_period = rate_bytes_per_sec / transport_pacer_period;
+ pacer->tokens_per_period = rate_bytes_per_sec * CLIB_US_TIME_PERIOD;
}
static inline u64
@@ -618,33 +621,35 @@ spacer_pace_rate (spacer_t * pacer)
}
static inline void
-spacer_reset_bucket (spacer_t * pacer, u64 norm_time_now)
+spacer_reset (spacer_t * pacer, clib_us_time_t time_now, u64 bucket)
{
- pacer->last_update = norm_time_now;
- pacer->bucket = 0;
+ pacer->last_update = time_now;
+ pacer->bucket = bucket;
}
void
transport_connection_tx_pacer_reset (transport_connection_t * tc,
- u32 rate_bytes_per_sec,
- u32 start_bucket, u64 time_now)
+ u64 rate_bytes_per_sec, u32 start_bucket)
{
- spacer_t *pacer = &tc->pacer;
spacer_set_pace_rate (&tc->pacer, rate_bytes_per_sec);
- pacer->last_update = time_now >> SPACER_CPU_TICKS_PER_PERIOD_SHIFT;
- pacer->bucket = start_bucket;
+ spacer_reset (&tc->pacer, transport_us_time_now (tc->thread_index),
+ start_bucket);
+}
+
+void
+transport_connection_tx_pacer_reset_bucket (transport_connection_t * tc)
+{
+ spacer_reset (&tc->pacer, transport_us_time_now (tc->thread_index), 0);
}
void
transport_connection_tx_pacer_init (transport_connection_t * tc,
- u32 rate_bytes_per_sec,
+ u64 rate_bytes_per_sec,
u32 initial_bucket)
{
- vlib_main_t *vm = vlib_get_main ();
tc->flags |= TRANSPORT_CONNECTION_F_IS_TX_PACED;
transport_connection_tx_pacer_reset (tc, rate_bytes_per_sec,
- initial_bucket,
- vm->clib_time.last_cpu_time);
+ initial_bucket);
}
void
@@ -655,32 +660,22 @@ transport_connection_tx_pacer_update (transport_connection_t * tc,
}
u32
-transport_connection_tx_pacer_burst (transport_connection_t * tc,
- u64 time_now)
-{
- time_now >>= SPACER_CPU_TICKS_PER_PERIOD_SHIFT;
- return spacer_max_burst (&tc->pacer, time_now);
-}
-
-void
-transport_connection_tx_pacer_reset_bucket (transport_connection_t * tc,
- u64 time_now)
+transport_connection_tx_pacer_burst (transport_connection_t * tc)
{
- time_now >>= SPACER_CPU_TICKS_PER_PERIOD_SHIFT;
- spacer_reset_bucket (&tc->pacer, time_now);
+ return spacer_max_burst (&tc->pacer,
+ transport_us_time_now (tc->thread_index));
}
u32
-transport_connection_snd_space (transport_connection_t * tc, u64 time_now,
- u16 mss)
+transport_connection_snd_space (transport_connection_t * tc, u16 mss)
{
u32 snd_space, max_paced_burst;
snd_space = tp_vfts[tc->proto].send_space (tc);
if (snd_space && transport_connection_is_tx_paced (tc))
{
- time_now >>= SPACER_CPU_TICKS_PER_PERIOD_SHIFT;
- max_paced_burst = spacer_max_burst (&tc->pacer, time_now);
+ clib_us_time_t now = transport_us_time_now (tc->thread_index);
+ max_paced_burst = spacer_max_burst (&tc->pacer, now);
max_paced_burst =
(max_paced_burst < TRANSPORT_PACER_MIN_BURST) ? 0 : max_paced_burst;
snd_space = clib_min (snd_space, max_paced_burst);
@@ -710,14 +705,7 @@ transport_connection_tx_pacer_update_bytes (transport_connection_t * tc,
}
void
-transport_init_tx_pacers_period (void)
-{
- f64 cpu_freq = os_cpu_clock_frequency ();
- transport_pacer_period = cpu_freq / SPACER_CPU_TICKS_PER_PERIOD;
-}
-
-void
-transport_update_time (f64 time_now, u8 thread_index)
+transport_update_time (clib_time_type_t time_now, u8 thread_index)
{
transport_proto_vft_t *vft;
vec_foreach (vft, tp_vfts)
diff --git a/src/vnet/session/transport.h b/src/vnet/session/transport.h
index 914991d5b31..df246a9b305 100644
--- a/src/vnet/session/transport.h
+++ b/src/vnet/session/transport.h
@@ -152,7 +152,7 @@ void transport_register_protocol (transport_proto_t transport_proto,
const transport_proto_vft_t * vft,
fib_protocol_t fib_proto, u32 output_node);
transport_proto_vft_t *transport_protocol_get_vft (transport_proto_t tp);
-void transport_update_time (f64 time_now, u8 thread_index);
+void transport_update_time (clib_time_type_t time_now, u8 thread_index);
int transport_alloc_local_port (u8 proto, ip46_address_t * ip);
int transport_alloc_local_endpoint (u8 proto, transport_endpoint_cfg_t * rmt,
@@ -173,8 +173,8 @@ transport_elog_track_index (transport_connection_t * tc)
}
void transport_connection_tx_pacer_reset (transport_connection_t * tc,
- u32 rate_bytes_per_sec,
- u32 initial_bucket, u64 time_now);
+ u64 rate_bytes_per_sec,
+ u32 initial_bucket);
/**
* Initialize tx pacer for connection
*
@@ -183,7 +183,7 @@ void transport_connection_tx_pacer_reset (transport_connection_t * tc,
* @param burst_bytes initial burst size in bytes
*/
void transport_connection_tx_pacer_init (transport_connection_t * tc,
- u32 rate_bytes_per_sec,
+ u64 rate_bytes_per_sec,
u32 initial_bucket);
/**
@@ -202,8 +202,7 @@ void transport_connection_tx_pacer_update (transport_connection_t * tc,
* @param time_now current cpu time as returned by @ref clib_cpu_time_now
* @param mss transport's mss
*/
-u32 transport_connection_snd_space (transport_connection_t * tc,
- u64 time_now, u16 mss);
+u32 transport_connection_snd_space (transport_connection_t * tc, u16 mss);
/**
* Get tx pacer max burst
@@ -212,8 +211,7 @@ u32 transport_connection_snd_space (transport_connection_t * tc,
* @param time_now current cpu time
* @return max burst for connection
*/
-u32 transport_connection_tx_pacer_burst (transport_connection_t * tc,
- u64 time_now);
+u32 transport_connection_tx_pacer_burst (transport_connection_t * tc);
/**
* Get tx pacer current rate
@@ -229,16 +227,7 @@ u64 transport_connection_tx_pacer_rate (transport_connection_t * tc);
* @param tc transport connection
* @param time_now current cpu time
*/
-void transport_connection_tx_pacer_reset_bucket (transport_connection_t * tc,
- u64 time_now);
-
-/**
- * Initialize period for tx pacers
- *
- * Defines a unit of time with respect to number of cpu cycles that is to
- * be used by all tx pacers.
- */
-void transport_init_tx_pacers_period (void);
+void transport_connection_tx_pacer_reset_bucket (transport_connection_t * tc);
/**
* Check if transport connection is paced
diff --git a/src/vnet/session/transport_types.h b/src/vnet/session/transport_types.h
index b6e01b54345..1e19798348f 100644
--- a/src/vnet/session/transport_types.h
+++ b/src/vnet/session/transport_types.h
@@ -51,7 +51,7 @@ typedef struct _spacer
{
u64 bytes_per_sec;
u64 bucket;
- u64 last_update;
+ clib_us_time_t last_update;
f32 tokens_per_period;
} spacer_t;
diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c
index 1d030c9ce71..d8ba65871d9 100644
--- a/src/vnet/tcp/tcp.c
+++ b/src/vnet/tcp/tcp.c
@@ -1399,11 +1399,9 @@ void
tcp_connection_tx_pacer_reset (tcp_connection_t * tc, u32 window,
u32 start_bucket)
{
- tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index);
f64 srtt = clib_min ((f64) tc->srtt * TCP_TICK, tc->mrtt_us);
- u64 last_time = wrk->vm->clib_time.last_cpu_time;
- transport_connection_tx_pacer_reset (&tc->connection, window / srtt,
- start_bucket, last_time);
+ u64 rate = (u64) window / srtt;
+ transport_connection_tx_pacer_reset (&tc->connection, rate, start_bucket);
}
static void
diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c
index b2dc2f7b1a8..69db57ced95 100755
--- a/src/vnet/tcp/tcp_input.c
+++ b/src/vnet/tcp/tcp_input.c
@@ -629,9 +629,7 @@ tcp_handle_postponed_dequeues (tcp_worker_ctx_t * wrk)
* we're in recovery and snd space constrained */
if (tc->data_segs_out == tc->prev_dsegs_out
|| tcp_recovery_no_snd_space (tc))
- transport_connection_tx_pacer_reset_bucket (&tc->connection,
- wrk->vm->clib_time.
- last_cpu_time);
+ transport_connection_tx_pacer_reset_bucket (&tc->connection);
tc->prev_dsegs_out = tc->data_segs_out;
}
_vec_len (wrk->pending_deq_acked) = 0;
diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index 45282168bd7..71e9d7bbdb0 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -1876,13 +1876,10 @@ tcp_retransmit_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
vlib_buffer_t *b = 0;
sack_scoreboard_t *sb;
int snd_space;
- u64 time_now;
ASSERT (tcp_in_cong_recovery (tc));
- time_now = wrk->vm->clib_time.last_cpu_time;
- burst_bytes = transport_connection_tx_pacer_burst (&tc->connection,
- time_now);
+ burst_bytes = transport_connection_tx_pacer_burst (&tc->connection);
burst_size = clib_min (burst_size, burst_bytes / tc->snd_mss);
if (!burst_size)
{
@@ -2017,9 +2014,7 @@ done:
if (reset_pacer)
{
- transport_connection_tx_pacer_reset_bucket (&tc->connection,
- vm->clib_time.
- last_cpu_time);
+ transport_connection_tx_pacer_reset_bucket (&tc->connection);
}
else
{
@@ -2044,14 +2039,11 @@ tcp_retransmit_no_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
int snd_space, n_segs = 0;
u8 cc_limited = 0;
vlib_buffer_t *b;
- u64 time_now;
ASSERT (tcp_in_fastrecovery (tc));
TCP_EVT (TCP_EVT_CC_EVT, tc, 0);
- time_now = wrk->vm->clib_time.last_cpu_time;
- burst_bytes = transport_connection_tx_pacer_burst (&tc->connection,
- time_now);
+ burst_bytes = transport_connection_tx_pacer_burst (&tc->connection);
burst_size = clib_min (burst_size, burst_bytes / tc->snd_mss);
if (!burst_size)
{
diff --git a/src/vppinfra/time.h b/src/vppinfra/time.h
index 0327ce0fd05..ae738db1307 100644
--- a/src/vppinfra/time.h
+++ b/src/vppinfra/time.h
@@ -193,6 +193,13 @@ clib_cpu_time_now (void)
void clib_time_verify_frequency (clib_time_t * c);
+/* Define it as the type returned by clib_time_now */
+typedef f64 clib_time_type_t;
+typedef u64 clib_us_time_t;
+
+#define CLIB_US_TIME_PERIOD (1e-6)
+#define CLIB_US_TIME_FREQ (1.0/CLIB_US_TIME_PERIOD)
+
always_inline f64
clib_time_now_internal (clib_time_t * c, u64 n)
{