diff options
| -rw-r--r-- | src/vnet/tcp/tcp.c | 106 | ||||
| -rw-r--r-- | src/vnet/tcp/tcp.h | 168 | ||||
| -rw-r--r-- | src/vnet/tcp/tcp_input.c | 112 | ||||
| -rw-r--r-- | src/vnet/tcp/tcp_output.c | 135 |
4 files changed, 258 insertions, 263 deletions
diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c index 30173619e0e..cca9f1c5c90 100644 --- a/src/vnet/tcp/tcp.c +++ b/src/vnet/tcp/tcp.c @@ -103,7 +103,7 @@ tcp_connection_bind (u32 session_index, transport_endpoint_t * lcl) return listener->c_c_index; } -u32 +static u32 tcp_session_bind (u32 session_index, transport_endpoint_t * tep) { return tcp_connection_bind (session_index, tep); @@ -126,14 +126,14 @@ tcp_connection_unbind (u32 listener_index) pool_put_index (tm->listener_pool, listener_index); } -u32 +static u32 tcp_session_unbind (u32 listener_index) { tcp_connection_unbind (listener_index); return 0; } -transport_connection_t * +static transport_connection_t * tcp_session_get_listener (u32 listener_index) { tcp_main_t *tm = vnet_get_tcp_main (); @@ -146,7 +146,7 @@ tcp_session_get_listener (u32 listener_index) * Cleanup half-open connection * */ -void +static void tcp_half_open_connection_del (tcp_connection_t * tc) { tcp_main_t *tm = vnet_get_tcp_main (); @@ -178,7 +178,7 @@ tcp_half_open_connection_cleanup (tcp_connection_t * tc) return 0; } -tcp_connection_t * +static tcp_connection_t * tcp_half_open_connection_new (void) { tcp_main_t *tm = vnet_get_tcp_main (); @@ -358,7 +358,7 @@ tcp_connection_close (tcp_connection_t * tc) tcp_connection_del (tc); } -void +static void tcp_session_close (u32 conn_index, u32 thread_index) { tcp_connection_t *tc; @@ -366,7 +366,7 @@ tcp_session_close (u32 conn_index, u32 thread_index) tcp_connection_close (tc); } -void +static void tcp_session_cleanup (u32 conn_index, u32 thread_index) { tcp_connection_t *tc; @@ -504,6 +504,31 @@ tcp_connection_fib_attach (tcp_connection_t * tc) } #endif /* 0 */ +static void +tcp_cc_init (tcp_connection_t * tc) +{ + tc->cc_algo = tcp_cc_algo_get (TCP_CC_NEWRENO); + tc->cc_algo->init (tc); +} + +void +tcp_cc_algo_register (tcp_cc_algorithm_type_e type, + const tcp_cc_algorithm_t * vft) +{ + tcp_main_t *tm = vnet_get_tcp_main (); + vec_validate (tm->cc_algos, type); + + tm->cc_algos[type] = *vft; +} + +tcp_cc_algorithm_t * +tcp_cc_algo_get (tcp_cc_algorithm_type_e type) +{ + tcp_main_t *tm = vnet_get_tcp_main (); + return &tm->cc_algos[type]; +} + + /** * Initialize connection send variables. */ @@ -577,7 +602,7 @@ tcp_alloc_custom_local_endpoint (tcp_main_t * tm, ip46_address_t * lcl_addr, return 0; } -int +static int tcp_connection_open (transport_endpoint_t * rmt) { tcp_main_t *tm = vnet_get_tcp_main (); @@ -624,7 +649,7 @@ tcp_connection_open (transport_endpoint_t * rmt) return tc->c_c_index; } -int +static int tcp_session_open (transport_endpoint_t * tep) { return tcp_connection_open (tep); @@ -660,7 +685,7 @@ const char *tcp_connection_flags_str[] = { #undef _ }; -u8 * +static u8 * format_tcp_connection_flags (u8 * s, va_list * args) { tcp_connection_t *tc = va_arg (*args, tcp_connection_t *); @@ -685,7 +710,7 @@ const char *tcp_conn_timers[] = { #undef _ }; -u8 * +static u8 * format_tcp_timers (u8 * s, va_list * args) { tcp_connection_t *tc = va_arg (*args, tcp_connection_t *); @@ -710,7 +735,7 @@ format_tcp_timers (u8 * s, va_list * args) return s; } -u8 * +static u8 * format_tcp_congestion_status (u8 * s, va_list * args) { tcp_connection_t *tc = va_arg (*args, tcp_connection_t *); @@ -723,7 +748,13 @@ format_tcp_congestion_status (u8 * s, va_list * args) return s; } -u8 * +static i32 +tcp_rcv_wnd_available (tcp_connection_t * tc) +{ + return (i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las); +} + +static u8 * format_tcp_vars (u8 * s, va_list * args) { tcp_connection_t *tc = va_arg (*args, tcp_connection_t *); @@ -763,7 +794,7 @@ format_tcp_vars (u8 * s, va_list * args) return s; } -u8 * +static u8 * format_tcp_connection_id (u8 * s, va_list * args) { tcp_connection_t *tc = va_arg (*args, tcp_connection_t *); @@ -806,7 +837,7 @@ format_tcp_connection (u8 * s, va_list * args) return s; } -u8 * +static u8 * format_tcp_session (u8 * s, va_list * args) { u32 tci = va_arg (*args, u32); @@ -822,7 +853,7 @@ format_tcp_session (u8 * s, va_list * args) return s; } -u8 * +static u8 * format_tcp_listener_session (u8 * s, va_list * args) { u32 tci = va_arg (*args, u32); @@ -830,7 +861,7 @@ format_tcp_listener_session (u8 * s, va_list * args) return format (s, "%U", format_tcp_connection_id, tc); } -u8 * +static u8 * format_tcp_half_open_session (u8 * s, va_list * args) { u32 tci = va_arg (*args, u32); @@ -886,7 +917,7 @@ format_tcp_rcv_sacks (u8 * s, va_list * args) return s; } -u8 * +static u8 * format_tcp_sack_hole (u8 * s, va_list * args) { sack_scoreboard_hole_t *hole = va_arg (*args, sack_scoreboard_hole_t *); @@ -924,14 +955,14 @@ format_tcp_scoreboard (u8 * s, va_list * args) return s; } -transport_connection_t * +static transport_connection_t * tcp_session_get_transport (u32 conn_index, u32 thread_index) { tcp_connection_t *tc = tcp_connection_get (conn_index, thread_index); return &tc->connection; } -transport_connection_t * +static transport_connection_t * tcp_half_open_session_get_transport (u32 conn_index) { tcp_connection_t *tc = tcp_half_open_connection_get (conn_index); @@ -945,7 +976,7 @@ tcp_half_open_session_get_transport (u32 conn_index) * the tcp options to be used in the next burst and subtracts their * length from the connection's snd_mss. */ -u16 +static u16 tcp_session_send_mss (transport_connection_t * trans_conn) { tcp_connection_t *tc = (tcp_connection_t *) trans_conn; @@ -985,7 +1016,7 @@ tcp_round_snd_space (tcp_connection_t * tc, u32 snd_space) * @param tc tcp connection * @return number of bytes session is allowed to write */ -u32 +static u32 tcp_snd_space (tcp_connection_t * tc) { int snd_space, snt_limited; @@ -1034,7 +1065,7 @@ tcp_snd_space (tcp_connection_t * tc) return 0; } -u32 +static u32 tcp_session_send_space (transport_connection_t * trans_conn) { tcp_connection_t *tc = (tcp_connection_t *) trans_conn; @@ -1042,13 +1073,7 @@ tcp_session_send_space (transport_connection_t * trans_conn) tc->snd_wnd - (tc->snd_nxt - tc->snd_una)); } -i32 -tcp_rcv_wnd_available (tcp_connection_t * tc) -{ - return (i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las); -} - -u32 +static u32 tcp_session_tx_fifo_offset (transport_connection_t * trans_conn) { tcp_connection_t *tc = (tcp_connection_t *) trans_conn; @@ -1059,7 +1084,7 @@ tcp_session_tx_fifo_offset (transport_connection_t * trans_conn) return (tc->snd_nxt - tc->snd_una); } -void +static void tcp_update_time (f64 now, u8 thread_index) { tcp_set_time_now (thread_index); @@ -1069,6 +1094,13 @@ tcp_update_time (f64 now, u8 thread_index) tcp_flush_frames_to_output (thread_index); } +static u32 +tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b) +{ + tcp_connection_t *tc = (tcp_connection_t *) tconn; + return tcp_push_header (tc, b); +} + /* *INDENT-OFF* */ const static transport_proto_vft_t tcp_proto = { .enable = vnet_tcp_enable_disable, @@ -1093,7 +1125,7 @@ const static transport_proto_vft_t tcp_proto = { }; /* *INDENT-ON* */ -void +static void tcp_timer_keep_handler (u32 conn_index) { u32 thread_index = vlib_get_thread_index (); @@ -1105,7 +1137,7 @@ tcp_timer_keep_handler (u32 conn_index) tcp_connection_close (tc); } -void +static void tcp_timer_establish_handler (u32 conn_index) { tcp_connection_t *tc; @@ -1133,7 +1165,7 @@ tcp_timer_establish_handler (u32 conn_index) tcp_connection_cleanup (tc); } -void +static void tcp_timer_waitclose_handler (u32 conn_index) { u32 thread_index = vlib_get_thread_index (); @@ -1198,7 +1230,7 @@ tcp_expired_timers_dispatch (u32 * expired_timers) } } -void +static void tcp_initialize_timer_wheels (tcp_main_t * tm) { tw_timer_wheel_16t_2w_512sl_t *tw; @@ -1212,7 +1244,7 @@ tcp_initialize_timer_wheels (tcp_main_t * tm) /* *INDENT-ON* */ } -clib_error_t * +static clib_error_t * tcp_main_enable (vlib_main_t * vm) { tcp_main_t *tm = vnet_get_tcp_main (); @@ -1320,7 +1352,7 @@ tcp_punt_unknown (vlib_main_t * vm, u8 is_ip4, u8 is_add) tm->punt_unknown6 = is_add; } -clib_error_t * +static clib_error_t * tcp_init (vlib_main_t * vm) { tcp_main_t *tm = vnet_get_tcp_main (); diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h index 1fd03c9e7ba..176de2c10ac 100644 --- a/src/vnet/tcp/tcp.h +++ b/src/vnet/tcp/tcp.h @@ -225,6 +225,24 @@ typedef struct _sack_scoreboard #define tcp_scoreboard_trace_add(_tc, _ack) #endif +sack_scoreboard_hole_t *scoreboard_next_rxt_hole (sack_scoreboard_t * sb, + sack_scoreboard_hole_t * + start, u8 have_sent_1_smss, + u8 * can_rescue, + u8 * snd_limited); +sack_scoreboard_hole_t *scoreboard_get_hole (sack_scoreboard_t * sb, + u32 index); + +sack_scoreboard_hole_t *scoreboard_next_hole (sack_scoreboard_t * sb, + sack_scoreboard_hole_t * hole); +sack_scoreboard_hole_t *scoreboard_prev_hole (sack_scoreboard_t * sb, + sack_scoreboard_hole_t * hole); +sack_scoreboard_hole_t *scoreboard_first_hole (sack_scoreboard_t * sb); +sack_scoreboard_hole_t *scoreboard_last_hole (sack_scoreboard_t * sb); +void scoreboard_clear (sack_scoreboard_t * sb); +void scoreboard_init (sack_scoreboard_t * sb); +u8 *format_tcp_scoreboard (u8 * s, va_list * args); + typedef enum _tcp_cc_algorithm_type { TCP_CC_NEWRENO, @@ -267,13 +285,13 @@ typedef struct _tcp_connection u32 irs; /**< initial remote sequence */ /* Options */ - tcp_options_t rcv_opts; /**< Rx options for connection */ - tcp_options_t snd_opts; /**< Tx options for connection */ u8 snd_opts_len; /**< Tx options len */ - u8 rcv_wscale; /**< Window scale to advertise to peer */ - u8 snd_wscale; /**< Window scale to use when sending */ - u32 tsval_recent; /**< Last timestamp received */ - u32 tsval_recent_age; /**< When last updated tstamp_recent*/ + u8 rcv_wscale; /**< Window scale to advertise to peer */ + u8 snd_wscale; /**< Window scale to use when sending */ + u32 tsval_recent; /**< Last timestamp received */ + u32 tsval_recent_age; /**< When last updated tstamp_recent*/ + tcp_options_t snd_opts; /**< Tx options for connection */ + tcp_options_t rcv_opts; /**< Rx options for connection */ sack_block_t *snd_sacks; /**< Vector of SACKs to send. XXX Fixed size? */ sack_scoreboard_t sack_sb; /**< SACK "scoreboard" that tracks holes */ @@ -338,13 +356,6 @@ tcp_cong_recovery_off (tcp_connection_t * tc) tcp_fastrecovery_1_smss_off (tc); } -typedef enum -{ - TCP_IP4, - TCP_IP6, - TCP_N_AF, -} tcp_af_t; - typedef enum _tcp_error { #define tcp_error(n,s) TCP_ERROR_##n, @@ -488,11 +499,7 @@ int tcp_configure_v6_source_address_range (vlib_main_t * vm, ip6_address_t * start, ip6_address_t * end, u32 table_id); void tcp_api_reference (void); -u8 *format_tcp_connection_id (u8 * s, va_list * args); u8 *format_tcp_connection (u8 * s, va_list * args); -u8 *format_tcp_scoreboard (u8 * s, va_list * args); - -u8 *tcp_scoreboard_replay (u8 * s, tcp_connection_t * tc, u8 verbose); always_inline tcp_connection_t * tcp_listener_get (u32 tli) @@ -640,16 +647,11 @@ tcp_is_lost_fin (tcp_connection_t * tc) return 0; } -i32 tcp_rcv_wnd_available (tcp_connection_t * tc); -u32 tcp_snd_space (tcp_connection_t * tc); -void tcp_update_rcv_wnd (tcp_connection_t * tc); - void tcp_retransmit_first_unacked (tcp_connection_t * tc); void tcp_fast_retransmit_no_sack (tcp_connection_t * tc); void tcp_fast_retransmit_sack (tcp_connection_t * tc); void tcp_fast_retransmit (tcp_connection_t * tc); void tcp_cc_init_congestion (tcp_connection_t * tc); -int tcp_cc_recover (tcp_connection_t * tc); void tcp_cc_fastrecovery_exit (tcp_connection_t * tc); fib_node_index_t tcp_lookup_rmt_in_fib (tcp_connection_t * tc); @@ -672,12 +674,7 @@ tcp_set_time_now (u32 thread_index) return tcp_main.wrk_ctx[thread_index].time_now; } -u32 tcp_session_push_header (transport_connection_t * tconn, - vlib_buffer_t * b); - -u32 -tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset, - u32 max_bytes, vlib_buffer_t ** b); +u32 tcp_push_header (tcp_connection_t * tconn, vlib_buffer_t * b); void tcp_connection_timers_init (tcp_connection_t * tc); void tcp_connection_timers_reset (tcp_connection_t * tc); @@ -799,120 +796,13 @@ tcp_timer_is_active (tcp_connection_t * tc, tcp_timers_e timer) ASSERT(_tc->state != TCP_STATE_ESTABLISHED \ || session_tx_fifo_max_dequeue (&_tc->connection) >= _a) -void -scoreboard_remove_hole (sack_scoreboard_t * sb, - sack_scoreboard_hole_t * hole); -sack_scoreboard_hole_t *scoreboard_insert_hole (sack_scoreboard_t * sb, - u32 prev_index, u32 start, - u32 end); -sack_scoreboard_hole_t *scoreboard_next_rxt_hole (sack_scoreboard_t * sb, - sack_scoreboard_hole_t * - start, u8 have_sent_1_smss, - u8 * can_rescue, - u8 * snd_limited); -void scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 seq); - -always_inline sack_scoreboard_hole_t * -scoreboard_get_hole (sack_scoreboard_t * sb, u32 index) -{ - if (index != TCP_INVALID_SACK_HOLE_INDEX) - return pool_elt_at_index (sb->holes, index); - return 0; -} - -always_inline sack_scoreboard_hole_t * -scoreboard_next_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole) -{ - if (hole->next != TCP_INVALID_SACK_HOLE_INDEX) - return pool_elt_at_index (sb->holes, hole->next); - return 0; -} - -always_inline sack_scoreboard_hole_t * -scoreboard_prev_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole) -{ - if (hole->prev != TCP_INVALID_SACK_HOLE_INDEX) - return pool_elt_at_index (sb->holes, hole->prev); - return 0; -} - -always_inline sack_scoreboard_hole_t * -scoreboard_first_hole (sack_scoreboard_t * sb) -{ - if (sb->head != TCP_INVALID_SACK_HOLE_INDEX) - return pool_elt_at_index (sb->holes, sb->head); - return 0; -} - -always_inline sack_scoreboard_hole_t * -scoreboard_last_hole (sack_scoreboard_t * sb) -{ - if (sb->tail != TCP_INVALID_SACK_HOLE_INDEX) - return pool_elt_at_index (sb->holes, sb->tail); - return 0; -} - -always_inline void -scoreboard_clear (sack_scoreboard_t * sb) -{ - sack_scoreboard_hole_t *hole; - while ((hole = scoreboard_first_hole (sb))) - { - scoreboard_remove_hole (sb, hole); - } - ASSERT (sb->head == sb->tail && sb->head == TCP_INVALID_SACK_HOLE_INDEX); - ASSERT (pool_elts (sb->holes) == 0); - sb->sacked_bytes = 0; - sb->last_sacked_bytes = 0; - sb->last_bytes_delivered = 0; - sb->snd_una_adv = 0; - sb->high_sacked = 0; - sb->high_rxt = 0; - sb->lost_bytes = 0; - sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX; -} - -always_inline u32 -scoreboard_hole_bytes (sack_scoreboard_hole_t * hole) -{ - return hole->end - hole->start; -} - -always_inline u32 -scoreboard_hole_index (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole) -{ - ASSERT (!pool_is_free_index (sb->holes, hole - sb->holes)); - return hole - sb->holes; -} - -always_inline void -scoreboard_init (sack_scoreboard_t * sb) -{ - sb->head = TCP_INVALID_SACK_HOLE_INDEX; - sb->tail = TCP_INVALID_SACK_HOLE_INDEX; - sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX; -} - void tcp_rcv_sacks (tcp_connection_t * tc, u32 ack); +u8 *tcp_scoreboard_replay (u8 * s, tcp_connection_t * tc, u8 verbose); -always_inline void -tcp_cc_algo_register (tcp_cc_algorithm_type_e type, - const tcp_cc_algorithm_t * vft) -{ - tcp_main_t *tm = vnet_get_tcp_main (); - vec_validate (tm->cc_algos, type); - - tm->cc_algos[type] = *vft; -} - -always_inline tcp_cc_algorithm_t * -tcp_cc_algo_get (tcp_cc_algorithm_type_e type) -{ - tcp_main_t *tm = vnet_get_tcp_main (); - return &tm->cc_algos[type]; -} +void tcp_cc_algo_register (tcp_cc_algorithm_type_e type, + const tcp_cc_algorithm_t * vft); -void tcp_cc_init (tcp_connection_t * tc); +tcp_cc_algorithm_t *tcp_cc_algo_get (tcp_cc_algorithm_type_e type); /** * Push TCP header to buffer diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c index 289a6d66e3e..c99d6cca6fe 100644 --- a/src/vnet/tcp/tcp_input.c +++ b/src/vnet/tcp/tcp_input.c @@ -123,7 +123,7 @@ tcp_segment_in_rcv_wnd (tcp_connection_t * tc, u32 seq, u32 end_seq) * @param to TCP options data structure to be populated * @return -1 if parsing failed */ -int +static int tcp_options_parse (tcp_header_t * th, tcp_options_t * to) { const u8 *data; @@ -534,7 +534,60 @@ tcp_ack_is_cc_event (tcp_connection_t * tc, vlib_buffer_t * b, return ((*is_dack || tcp_in_cong_recovery (tc)) && !tcp_is_lost_fin (tc)); } -void +static u32 +scoreboard_hole_index (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole) +{ + ASSERT (!pool_is_free_index (sb->holes, hole - sb->holes)); + return hole - sb->holes; +} + +static u32 +scoreboard_hole_bytes (sack_scoreboard_hole_t * hole) +{ + return hole->end - hole->start; +} + +sack_scoreboard_hole_t * +scoreboard_get_hole (sack_scoreboard_t * sb, u32 index) +{ + if (index != TCP_INVALID_SACK_HOLE_INDEX) + return pool_elt_at_index (sb->holes, index); + return 0; +} + +sack_scoreboard_hole_t * +scoreboard_next_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole) +{ + if (hole->next != TCP_INVALID_SACK_HOLE_INDEX) + return pool_elt_at_index (sb->holes, hole->next); + return 0; +} + +sack_scoreboard_hole_t * +scoreboard_prev_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole) +{ + if (hole->prev != TCP_INVALID_SACK_HOLE_INDEX) + return pool_elt_at_index (sb->holes, hole->prev); + return 0; +} + +sack_scoreboard_hole_t * +scoreboard_first_hole (sack_scoreboard_t * sb) +{ + if (sb->head != TCP_INVALID_SACK_HOLE_INDEX) + return pool_elt_at_index (sb->holes, sb->head); + return 0; +} + +sack_scoreboard_hole_t * +scoreboard_last_hole (sack_scoreboard_t * sb) +{ + if (sb->tail != TCP_INVALID_SACK_HOLE_INDEX) + return pool_elt_at_index (sb->holes, sb->tail); + return 0; +} + +static void scoreboard_remove_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole) { sack_scoreboard_hole_t *next, *prev; @@ -569,7 +622,7 @@ scoreboard_remove_hole (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole) pool_put (sb->holes, hole); } -sack_scoreboard_hole_t * +static sack_scoreboard_hole_t * scoreboard_insert_hole (sack_scoreboard_t * sb, u32 prev_index, u32 start, u32 end) { @@ -606,7 +659,7 @@ scoreboard_insert_hole (sack_scoreboard_t * sb, u32 prev_index, return hole; } -void +static void scoreboard_update_bytes (tcp_connection_t * tc, sack_scoreboard_t * sb) { sack_scoreboard_hole_t *hole, *prev; @@ -705,7 +758,7 @@ scoreboard_next_rxt_hole (sack_scoreboard_t * sb, return hole; } -void +static void scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 seq) { sack_scoreboard_hole_t *hole; @@ -718,13 +771,41 @@ scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 seq) sb->high_rxt = seq; } +void +scoreboard_init (sack_scoreboard_t * sb) +{ + sb->head = TCP_INVALID_SACK_HOLE_INDEX; + sb->tail = TCP_INVALID_SACK_HOLE_INDEX; + sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX; +} + +void +scoreboard_clear (sack_scoreboard_t * sb) +{ + sack_scoreboard_hole_t *hole; + while ((hole = scoreboard_first_hole (sb))) + { + scoreboard_remove_hole (sb, hole); + } + ASSERT (sb->head == sb->tail && sb->head == TCP_INVALID_SACK_HOLE_INDEX); + ASSERT (pool_elts (sb->holes) == 0); + sb->sacked_bytes = 0; + sb->last_sacked_bytes = 0; + sb->last_bytes_delivered = 0; + sb->snd_una_adv = 0; + sb->high_sacked = 0; + sb->high_rxt = 0; + sb->lost_bytes = 0; + sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX; +} + /** * Test that scoreboard is sane after recovery * * Returns 1 if scoreboard is empty or if first hole beyond * snd_una. */ -u8 +static u8 tcp_scoreboard_is_sane_post_recovery (tcp_connection_t * tc) { sack_scoreboard_hole_t *hole; @@ -1000,7 +1081,7 @@ tcp_cc_is_spurious_retransmit (tcp_connection_t * tc) && timestamp_lt (tc->rcv_opts.tsecr, tc->snd_rxt_ts)); } -int +static int tcp_cc_recover (tcp_connection_t * tc) { ASSERT (tcp_in_cong_recovery (tc)); @@ -1222,13 +1303,6 @@ partial_ack: tcp_fast_retransmit (tc); } -void -tcp_cc_init (tcp_connection_t * tc) -{ - tc->cc_algo = tcp_cc_algo_get (TCP_CC_NEWRENO); - tc->cc_algo->init (tc); -} - /** * Process incoming ACK */ @@ -1415,7 +1489,7 @@ tcp_sack_list_bytes (tcp_connection_t * tc) } /** Enqueue data for delivery to application */ -always_inline int +static int tcp_session_enqueue_data (tcp_connection_t * tc, vlib_buffer_t * b, u16 data_len) { @@ -1470,7 +1544,7 @@ tcp_session_enqueue_data (tcp_connection_t * tc, vlib_buffer_t * b, } /** Enqueue out-of-order data */ -always_inline int +static int tcp_session_enqueue_ooo (tcp_connection_t * tc, vlib_buffer_t * b, u16 data_len) { @@ -1648,7 +1722,7 @@ typedef struct tcp_connection_t tcp_connection; } tcp_rx_trace_t; -u8 * +static u8 * format_tcp_rx_trace (u8 * s, va_list * args) { CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); @@ -1664,7 +1738,7 @@ format_tcp_rx_trace (u8 * s, va_list * args) return s; } -u8 * +static u8 * format_tcp_rx_trace_short (u8 * s, va_list * args) { CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); @@ -3410,7 +3484,7 @@ do { \ #undef _ } -clib_error_t * +static clib_error_t * tcp_input_init (vlib_main_t * vm) { clib_error_t *error = 0; diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index e29eb6d7f9a..13eac11105c 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -131,30 +131,7 @@ tcp_initial_window_to_advertise (tcp_connection_t * tc) return clib_min (tc->rcv_wnd, TCP_WND_MAX); } -/** - * Compute and return window to advertise, scaled as per RFC1323 - */ -u32 -tcp_window_to_advertise (tcp_connection_t * tc, tcp_state_t state) -{ - if (state < TCP_STATE_ESTABLISHED) - return tcp_initial_window_to_advertise (tc); - - tcp_update_rcv_wnd (tc); - - if (tc->rcv_wnd == 0) - { - tc->flags |= TCP_CONN_SENT_RCV_WND0; - } - else - { - tc->flags &= ~TCP_CONN_SENT_RCV_WND0; - } - - return tc->rcv_wnd >> tc->rcv_wscale; -} - -void +static void tcp_update_rcv_wnd (tcp_connection_t * tc) { i32 observed_wnd; @@ -201,9 +178,32 @@ tcp_update_rcv_wnd (tcp_connection_t * tc) } /** + * Compute and return window to advertise, scaled as per RFC1323 + */ +static u32 +tcp_window_to_advertise (tcp_connection_t * tc, tcp_state_t state) +{ + if (state < TCP_STATE_ESTABLISHED) + return tcp_initial_window_to_advertise (tc); + + tcp_update_rcv_wnd (tc); + + if (tc->rcv_wnd == 0) + { + tc->flags |= TCP_CONN_SENT_RCV_WND0; + } + else + { + tc->flags &= ~TCP_CONN_SENT_RCV_WND0; + } + + return tc->rcv_wnd >> tc->rcv_wscale; +} + +/** * Write TCP options to segment. */ -u32 +static u32 tcp_options_write (u8 * data, tcp_options_t * opts) { u32 opts_len = 0; @@ -286,7 +286,7 @@ tcp_options_write (u8 * data, tcp_options_t * opts) return opts_len; } -always_inline int +static int tcp_make_syn_options (tcp_options_t * opts, u8 wnd_scale) { u8 len = 0; @@ -315,7 +315,7 @@ tcp_make_syn_options (tcp_options_t * opts, u8 wnd_scale) return len; } -always_inline int +static int tcp_make_synack_options (tcp_connection_t * tc, tcp_options_t * opts) { u8 len = 0; @@ -350,7 +350,7 @@ tcp_make_synack_options (tcp_connection_t * tc, tcp_options_t * opts) return len; } -always_inline int +static int tcp_make_established_options (tcp_connection_t * tc, tcp_options_t * opts) { u8 len = 0; @@ -444,7 +444,7 @@ tcp_init_mss (tcp_connection_t * tc) tc->snd_mss -= TCP_OPTION_LEN_TIMESTAMP; } -always_inline int +static int tcp_alloc_tx_buffers (tcp_main_t * tm, u8 thread_index, u16 * n_bufs, u32 wanted) { @@ -483,7 +483,7 @@ tcp_get_free_buffer_index (tcp_main_t * tm, u32 * bidx) return 0; } -always_inline void * +static void * tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b) { if (b->flags & VLIB_BUFFER_NEXT_PRESENT) @@ -499,7 +499,7 @@ tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b) return vlib_buffer_make_headroom (b, MAX_HDRS_LEN); } -always_inline void * +static void * tcp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b) { ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); @@ -516,7 +516,7 @@ tcp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b) /** * Prepare ACK */ -void +static void tcp_make_ack_i (tcp_connection_t * tc, vlib_buffer_t * b, tcp_state_t state, u8 flags) { @@ -666,14 +666,14 @@ tcp_enqueue_to_ip_lookup_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, } } -always_inline void +static void tcp_enqueue_to_ip_lookup_now (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, u8 is_ip4, u32 fib_index) { tcp_enqueue_to_ip_lookup_i (vm, b, bi, is_ip4, fib_index, 1); } -always_inline void +static void tcp_enqueue_to_ip_lookup (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, u8 is_ip4, u32 fib_index) { @@ -716,20 +716,20 @@ tcp_enqueue_to_output_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, } } -always_inline void +static void tcp_enqueue_to_output (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, u8 is_ip4) { tcp_enqueue_to_output_i (vm, b, bi, is_ip4, 0); } -always_inline void +static void tcp_enqueue_to_output_now (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, u8 is_ip4) { tcp_enqueue_to_output_i (vm, b, bi, is_ip4, 1); } -int +static int tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0, tcp_state_t state, u8 thread_index, u8 is_ip4) { @@ -938,7 +938,7 @@ tcp_send_reset (tcp_connection_t * tc) TCP_EVT_DBG (TCP_EVT_RST_SENT, tc); } -void +static void tcp_push_ip_hdr (tcp_main_t * tm, tcp_connection_t * tc, vlib_buffer_t * b) { tcp_header_t *th = vlib_buffer_get_current (b); @@ -1022,7 +1022,7 @@ tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4) /** * Flush ip lookup tx frames populated by timer pops */ -always_inline void +static void tcp_flush_frame_to_ip_lookup (vlib_main_t * vm, u8 thread_index, u8 is_ip4) { if (tcp_main.wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4]) @@ -1114,7 +1114,7 @@ tcp_make_state_flags (tcp_connection_t * tc, tcp_state_t next_state) /** * Push TCP header and update connection variables */ -static void +always_inline void tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b, tcp_state_t next_state, u8 compute_opts) { @@ -1153,6 +1153,28 @@ tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b, TCP_EVT_DBG (TCP_EVT_PKTIZE, tc); } +u32 +tcp_push_header (tcp_connection_t * tc, vlib_buffer_t * b) +{ + tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, 0); + tc->snd_una_max = tc->snd_nxt; + ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd)); + tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una); + /* If not tracking an ACK, start tracking */ + if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc)) + { + tc->rtt_ts = tcp_time_now (); + tc->rtt_seq = tc->snd_nxt; + } + if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT))) + { + tcp_retransmit_timer_set (tc); + tc->rto_boff = 0; + } + tcp_trajectory_add_start (b, 3); + return 0; +} + void tcp_send_ack (tcp_connection_t * tc) { @@ -1195,7 +1217,7 @@ tcp_timer_delack_handler (u32 index) * @return the number of bytes in the segment or 0 if there's nothing to * retransmit */ -u32 +static u32 tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset, u32 max_deq_bytes, vlib_buffer_t ** b) { @@ -1356,7 +1378,7 @@ tcp_rxt_timeout_cc (tcp_connection_t * tc) tcp_recovery_on (tc); } -static void +static inline void tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) { tcp_main_t *tm = vnet_get_tcp_main (); @@ -1755,7 +1777,7 @@ tcp_fast_retransmit (tcp_connection_t * tc) tcp_fast_retransmit_no_sack (tc); } -always_inline u32 +static u32 tcp_session_has_ooo_data (tcp_connection_t * tc) { stream_session_t *s = session_get (tc->c_s_index, tc->c_thread_index); @@ -1817,7 +1839,7 @@ tcp46_output_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node, } } -static inline void +always_inline void tcp_output_push_ip (vlib_main_t * vm, vlib_buffer_t * b0, tcp_connection_t * tc0, u8 is_ip4) { @@ -1845,7 +1867,7 @@ tcp_output_push_ip (vlib_main_t * vm, vlib_buffer_t * b0, } } -static inline void +always_inline void tcp_output_handle_packet (tcp_connection_t * tc0, vlib_buffer_t * b0, u32 * error0, u16 * next0, u8 is_ip4) { @@ -2022,29 +2044,6 @@ VLIB_REGISTER_NODE (tcp6_output_node) = VLIB_NODE_FUNCTION_MULTIARCH (tcp6_output_node, tcp6_output); -u32 -tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b) -{ - tcp_connection_t *tc = (tcp_connection_t *) tconn; - tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, 0); - tc->snd_una_max = tc->snd_nxt; - ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd)); - tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una); - /* If not tracking an ACK, start tracking */ - if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc)) - { - tc->rtt_ts = tcp_time_now (); - tc->rtt_seq = tc->snd_nxt; - } - if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT))) - { - tcp_retransmit_timer_set (tc); - tc->rto_boff = 0; - } - tcp_trajectory_add_start (b, 3); - return 0; -} - typedef enum _tcp_reset_next { TCP_RESET_NEXT_DROP, |