From 2c4144367cb4fd9f44853e2bb974e78eccff3cc7 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Tue, 19 Jun 2018 09:58:04 -0700 Subject: tcp: add per worker ctx structure Change-Id: I28d3c31bdc4255a4ca223d80bcf44709fb39f4ed Signed-off-by: Florin Coras --- src/vnet/tcp/tcp.c | 19 +++++-------------- src/vnet/tcp/tcp.h | 46 +++++++++++++++++++++++++++------------------- src/vnet/tcp/tcp_output.c | 45 ++++++++++++++++++++++++--------------------- 3 files changed, 56 insertions(+), 54 deletions(-) (limited to 'src/vnet/tcp') diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c index 854577b5575..30173619e0e 100644 --- a/src/vnet/tcp/tcp.c +++ b/src/vnet/tcp/tcp.c @@ -1063,7 +1063,8 @@ void tcp_update_time (f64 now, u8 thread_index) { tcp_set_time_now (thread_index); - tw_timer_expire_timers_16t_2w_512sl (&tcp_main.timer_wheels[thread_index], + tw_timer_expire_timers_16t_2w_512sl (&tcp_main. + wrk_ctx[thread_index].timer_wheel, now); tcp_flush_frames_to_output (thread_index); } @@ -1203,7 +1204,7 @@ tcp_initialize_timer_wheels (tcp_main_t * tm) tw_timer_wheel_16t_2w_512sl_t *tw; /* *INDENT-OFF* */ foreach_vlib_main (({ - tw = &tm->timer_wheels[ii]; + tw = &tm->wrk_ctx[ii].timer_wheel; tw_timer_wheel_init_16t_2w_512sl (tw, tcp_expired_timers_dispatch, 100e-3 /* timer period 100ms */ , ~0); tw->last_run_time = vlib_time_now (this_vlib_main); @@ -1272,13 +1273,6 @@ tcp_main_enable (vlib_main_t * vm) pool_init_fixed (tm->half_open_connections, tm->preallocated_half_open_connections); - /* Initialize per worker thread tx buffers (used for control messages) */ - vec_validate (tm->tx_buffers, num_threads - 1); - - /* Initialize timer wheels */ - vec_validate (tm->timer_wheels, num_threads - 1); - tcp_initialize_timer_wheels (tm); - /* Initialize clocks per tick for TCP timestamp. Used to compute * monotonically increasing timestamps. */ tm->tstamp_ticks_per_clock = vm->clib_time.seconds_per_clock @@ -1289,15 +1283,12 @@ tcp_main_enable (vlib_main_t * vm) clib_spinlock_init (&tm->half_open_lock); } - vec_validate (tm->tx_frames[0], num_threads - 1); - vec_validate (tm->tx_frames[1], num_threads - 1); - vec_validate (tm->ip_lookup_tx_frames[0], num_threads - 1); - vec_validate (tm->ip_lookup_tx_frames[1], num_threads - 1); + vec_validate (tm->wrk_ctx, num_threads - 1); + tcp_initialize_timer_wheels (tm); tm->bytes_per_buffer = vlib_buffer_free_list_buffer_size (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); - vec_validate (tm->time_now, num_threads - 1); return error; } diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h index 5673c8cab81..ba57e3a1fef 100644 --- a/src/vnet/tcp/tcp.h +++ b/src/vnet/tcp/tcp.h @@ -358,6 +358,18 @@ typedef struct _tcp_lookup_dispatch u8 next, error; } tcp_lookup_dispatch_t; +typedef struct tcp_worker_ctx_ +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + u32 time_now; /**< worker time */ + tw_timer_wheel_16t_2w_512sl_t timer_wheel; /**< worker timer wheel */ + u32 *tx_buffers; /**< tx buffer free list */ + vlib_frame_t *tx_frames[2]; /**< tx frames for tcp 4/6 + output nodes */ + vlib_frame_t *ip_lookup_tx_frames[2]; /**< tx frames for ip 4/6 + lookup nodes */ +} tcp_worker_ctx_t; + typedef struct _tcp_main { /* Per-worker thread tcp connection pools */ @@ -371,17 +383,9 @@ typedef struct _tcp_main u8 log2_tstamp_clocks_per_tick; f64 tstamp_ticks_per_clock; - u32 *time_now; - - /** per-worker tx buffer free lists */ - u32 **tx_buffers; - /** per-worker tx frames to tcp 4/6 output nodes */ - vlib_frame_t **tx_frames[2]; - /** per-worker tx frames to ip 4/6 lookup nodes */ - vlib_frame_t **ip_lookup_tx_frames[2]; - /* Per worker-thread timer wheel for connections timers */ - tw_timer_wheel_16t_2w_512sl_t *timer_wheels; + /** per-worker context */ + tcp_worker_ctx_t *wrk_ctx; /* Pool of half-open connections on which we've sent a SYN */ tcp_connection_t *half_open_connections; @@ -657,15 +661,15 @@ u32 tcp_sack_list_bytes (tcp_connection_t * tc); always_inline u32 tcp_time_now (void) { - return tcp_main.time_now[vlib_get_thread_index ()]; + return tcp_main.wrk_ctx[vlib_get_thread_index ()].time_now; } always_inline u32 tcp_set_time_now (u32 thread_index) { - tcp_main.time_now[thread_index] = clib_cpu_time_now () + tcp_main.wrk_ctx[thread_index].time_now = clib_cpu_time_now () * tcp_main.tstamp_ticks_per_clock; - return tcp_main.time_now[thread_index]; + return tcp_main.wrk_ctx[thread_index].time_now; } u32 tcp_session_push_header (transport_connection_t * tconn, @@ -693,9 +697,10 @@ tcp_timer_set (tcp_connection_t * tc, u8 timer_id, u32 interval) { ASSERT (tc->c_thread_index == vlib_get_thread_index ()); ASSERT (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID); - tc->timers[timer_id] - = tw_timer_start_16t_2w_512sl (&tcp_main.timer_wheels[tc->c_thread_index], - tc->c_c_index, timer_id, interval); + tc->timers[timer_id] = + tw_timer_start_16t_2w_512sl (&tcp_main. + wrk_ctx[tc->c_thread_index].timer_wheel, + tc->c_c_index, timer_id, interval); } always_inline void @@ -705,7 +710,8 @@ tcp_timer_reset (tcp_connection_t * tc, u8 timer_id) if (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID) return; - tw_timer_stop_16t_2w_512sl (&tcp_main.timer_wheels[tc->c_thread_index], + tw_timer_stop_16t_2w_512sl (&tcp_main. + wrk_ctx[tc->c_thread_index].timer_wheel, tc->timers[timer_id]); tc->timers[timer_id] = TCP_TIMER_HANDLE_INVALID; } @@ -715,10 +721,12 @@ tcp_timer_update (tcp_connection_t * tc, u8 timer_id, u32 interval) { ASSERT (tc->c_thread_index == vlib_get_thread_index ()); if (tc->timers[timer_id] != TCP_TIMER_HANDLE_INVALID) - tw_timer_stop_16t_2w_512sl (&tcp_main.timer_wheels[tc->c_thread_index], + tw_timer_stop_16t_2w_512sl (&tcp_main. + wrk_ctx[tc->c_thread_index].timer_wheel, tc->timers[timer_id]); tc->timers[timer_id] = - tw_timer_start_16t_2w_512sl (&tcp_main.timer_wheels[tc->c_thread_index], + tw_timer_start_16t_2w_512sl (&tcp_main. + wrk_ctx[tc->c_thread_index].timer_wheel, tc->c_c_index, timer_id, interval); } diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index 641277b67fa..e29eb6d7f9a 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -448,16 +448,16 @@ always_inline int tcp_alloc_tx_buffers (tcp_main_t * tm, u8 thread_index, u16 * n_bufs, u32 wanted) { + tcp_worker_ctx_t *ctx = &tm->wrk_ctx[thread_index]; vlib_main_t *vm = vlib_get_main (); u32 n_alloc; ASSERT (wanted > *n_bufs); - vec_validate_aligned (tm->tx_buffers[thread_index], wanted - 1, - CLIB_CACHE_LINE_BYTES); - n_alloc = vlib_buffer_alloc (vm, &tm->tx_buffers[thread_index][*n_bufs], + vec_validate_aligned (ctx->tx_buffers, wanted - 1, CLIB_CACHE_LINE_BYTES); + n_alloc = vlib_buffer_alloc (vm, &ctx->tx_buffers[*n_bufs], wanted - *n_bufs); *n_bufs += n_alloc; - _vec_len (tm->tx_buffers[thread_index]) = *n_bufs; + _vec_len (ctx->tx_buffers) = *n_bufs; return n_alloc; } @@ -465,7 +465,8 @@ always_inline int tcp_get_free_buffer_index (tcp_main_t * tm, u32 * bidx) { u32 thread_index = vlib_get_thread_index (); - u16 n_bufs = vec_len (tm->tx_buffers[thread_index]); + tcp_worker_ctx_t *ctx = &tm->wrk_ctx[thread_index]; + u16 n_bufs = vec_len (ctx->tx_buffers); TCP_DBG_BUFFER_ALLOC_MAYBE_FAIL (thread_index); @@ -477,8 +478,8 @@ tcp_get_free_buffer_index (tcp_main_t * tm, u32 * bidx) return -1; } } - *bidx = tm->tx_buffers[thread_index][--n_bufs]; - _vec_len (tm->tx_buffers[thread_index]) = n_bufs; + *bidx = ctx->tx_buffers[--n_bufs]; + _vec_len (ctx->tx_buffers) = n_bufs; return 0; } @@ -647,12 +648,12 @@ tcp_enqueue_to_ip_lookup_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index; tcp_trajectory_add_start (b, 1); - f = tm->ip_lookup_tx_frames[!is_ip4][thread_index]; + f = tm->wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4]; if (!f) { f = vlib_get_frame_to_node (vm, next_index); ASSERT (f); - tm->ip_lookup_tx_frames[!is_ip4][thread_index] = f; + tm->wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4] = f; } to_next = vlib_frame_vector_args (f); @@ -661,7 +662,7 @@ tcp_enqueue_to_ip_lookup_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, if (flush || f->n_vectors == VLIB_FRAME_SIZE) { vlib_put_frame_to_node (vm, next_index, f); - tm->ip_lookup_tx_frames[!is_ip4][thread_index] = 0; + tm->wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4] = 0; } } @@ -698,12 +699,12 @@ tcp_enqueue_to_output_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, tcp_trajectory_add_start (b, 2); /* Get frame to v4/6 output node */ - f = tm->tx_frames[!is_ip4][thread_index]; + f = tm->wrk_ctx[thread_index].tx_frames[!is_ip4]; if (!f) { f = vlib_get_frame_to_node (vm, next_index); ASSERT (f); - tm->tx_frames[!is_ip4][thread_index] = f; + tm->wrk_ctx[thread_index].tx_frames[!is_ip4] = f; } to_next = vlib_frame_vector_args (f); to_next[f->n_vectors] = bi; @@ -711,7 +712,7 @@ tcp_enqueue_to_output_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, if (flush || f->n_vectors == VLIB_FRAME_SIZE) { vlib_put_frame_to_node (vm, next_index, f); - tm->tx_frames[!is_ip4][thread_index] = 0; + tm->wrk_ctx[thread_index].tx_frames[!is_ip4] = 0; } } @@ -1007,13 +1008,14 @@ tcp_send_syn (tcp_connection_t * tc) void tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4) { - if (tcp_main.tx_frames[!is_ip4][thread_index]) + if (tcp_main.wrk_ctx[thread_index].tx_frames[!is_ip4]) { u32 next_index; next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index; vlib_put_frame_to_node (vm, next_index, - tcp_main.tx_frames[!is_ip4][thread_index]); - tcp_main.tx_frames[!is_ip4][thread_index] = 0; + tcp_main. + wrk_ctx[thread_index].tx_frames[!is_ip4]); + tcp_main.wrk_ctx[thread_index].tx_frames[!is_ip4] = 0; } } @@ -1023,14 +1025,15 @@ tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4) always_inline void tcp_flush_frame_to_ip_lookup (vlib_main_t * vm, u8 thread_index, u8 is_ip4) { - if (tcp_main.ip_lookup_tx_frames[!is_ip4][thread_index]) + if (tcp_main.wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4]) { u32 next_index; next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index; vlib_put_frame_to_node (vm, next_index, - tcp_main.ip_lookup_tx_frames[!is_ip4] - [thread_index]); - tcp_main.ip_lookup_tx_frames[!is_ip4][thread_index] = 0; + tcp_main. + wrk_ctx[thread_index].ip_lookup_tx_frames + [!is_ip4]); + tcp_main.wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4] = 0; } } @@ -1264,7 +1267,7 @@ tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset, /* Make sure we have enough buffers */ n_bufs_per_seg = ceil ((double) seg_size / tm->bytes_per_buffer); - available_bufs = vec_len (tm->tx_buffers[thread_index]); + available_bufs = vec_len (tm->wrk_ctx[thread_index].tx_buffers); if (n_bufs_per_seg > available_bufs) { tcp_alloc_tx_buffers (tm, thread_index, &available_bufs, -- cgit 1.2.3-korg