From 999840cf805f26a490e8e6b8acc1fe7a7c21a181 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Wed, 18 Mar 2020 20:31:34 +0000 Subject: tcp: move features to separate files Type: refactor Signed-off-by: Florin Coras Change-Id: Ia477b8dba9266f47907967e363c11048e5cd95ab --- src/vnet/tcp/tcp_inlines.h | 457 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 457 insertions(+) create mode 100644 src/vnet/tcp/tcp_inlines.h (limited to 'src/vnet/tcp/tcp_inlines.h') diff --git a/src/vnet/tcp/tcp_inlines.h b/src/vnet/tcp/tcp_inlines.h new file mode 100644 index 00000000000..2281cd3db7d --- /dev/null +++ b/src/vnet/tcp/tcp_inlines.h @@ -0,0 +1,457 @@ +/* + * Copyright (c) 2020 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_VNET_TCP_TCP_INLINES_H_ +#define SRC_VNET_TCP_TCP_INLINES_H_ + +#include + +always_inline tcp_header_t * +tcp_buffer_hdr (vlib_buffer_t * b) +{ + ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE); + return (tcp_header_t *) (b->data + b->current_data + + vnet_buffer (b)->tcp.hdr_offset); +} + +always_inline tcp_connection_t * +tcp_connection_get (u32 conn_index, u32 thread_index) +{ + tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index); + if (PREDICT_FALSE (pool_is_free_index (wrk->connections, conn_index))) + return 0; + return pool_elt_at_index (wrk->connections, conn_index); +} + +always_inline tcp_connection_t * +tcp_connection_get_if_valid (u32 conn_index, u32 thread_index) +{ + tcp_worker_ctx_t *wrk; + if (thread_index >= vec_len (tcp_main.wrk_ctx)) + return 0; + wrk = tcp_get_worker (thread_index); + if (pool_is_free_index (wrk->connections, conn_index)) + return 0; + return pool_elt_at_index (wrk->connections, conn_index); +} + +always_inline void +tcp_connection_set_state (tcp_connection_t * tc, tcp_state_t state) +{ + tc->state = state; + TCP_EVT (TCP_EVT_STATE_CHANGE, tc); +} + +always_inline tcp_connection_t * +tcp_listener_get (u32 tli) +{ + tcp_connection_t *tc = 0; + if (!pool_is_free_index (tcp_main.listener_pool, tli)) + tc = pool_elt_at_index (tcp_main.listener_pool, tli); + return tc; +} + +always_inline tcp_connection_t * +tcp_half_open_connection_get (u32 conn_index) +{ + tcp_connection_t *tc = 0; + clib_spinlock_lock_if_init (&tcp_main.half_open_lock); + if (!pool_is_free_index (tcp_main.half_open_connections, conn_index)) + tc = pool_elt_at_index (tcp_main.half_open_connections, conn_index); + clib_spinlock_unlock_if_init (&tcp_main.half_open_lock); + return tc; +} + +/** + * Our estimate of the number of bytes that have left the network + */ +always_inline u32 +tcp_bytes_out (const tcp_connection_t * tc) +{ + if (tcp_opts_sack_permitted (&tc->rcv_opts)) + return tc->sack_sb.sacked_bytes + tc->sack_sb.lost_bytes; + else + return clib_min (tc->rcv_dupacks * tc->snd_mss, + tc->snd_nxt - tc->snd_una); +} + +/** + * Our estimate of the number of bytes in flight (pipe size) + */ +always_inline u32 +tcp_flight_size (const tcp_connection_t * tc) +{ + int flight_size; + + flight_size = (int) (tc->snd_nxt - tc->snd_una) - tcp_bytes_out (tc) + + tc->snd_rxt_bytes - tc->rxt_delivered; + + ASSERT (flight_size >= 0); + + return flight_size; +} + +/** + * Initial cwnd as per RFC5681 + */ +always_inline u32 +tcp_initial_cwnd (const tcp_connection_t * tc) +{ + if (tcp_cfg.initial_cwnd_multiplier > 0) + return tcp_cfg.initial_cwnd_multiplier * tc->snd_mss; + + if (tc->snd_mss > 2190) + return 2 * tc->snd_mss; + else if (tc->snd_mss > 1095) + return 3 * tc->snd_mss; + else + return 4 * tc->snd_mss; +} + +/* + * Accumulate acked bytes for cwnd increase + * + * Once threshold bytes are accumulated, snd_mss bytes are added + * to the cwnd. + */ +always_inline void +tcp_cwnd_accumulate (tcp_connection_t * tc, u32 thresh, u32 bytes) +{ + tc->cwnd_acc_bytes += bytes; + if (tc->cwnd_acc_bytes >= thresh) + { + u32 inc = tc->cwnd_acc_bytes / thresh; + tc->cwnd_acc_bytes -= inc * thresh; + tc->cwnd += inc * tc->snd_mss; + tc->cwnd = clib_min (tc->cwnd, tc->tx_fifo_size); + } +} + +always_inline u32 +tcp_loss_wnd (const tcp_connection_t * tc) +{ + /* Whatever we have in flight + the packet we're about to send */ + return tcp_flight_size (tc) + tc->snd_mss; +} + +always_inline u32 +tcp_available_snd_wnd (const tcp_connection_t * tc) +{ + return clib_min (tc->cwnd, tc->snd_wnd); +} + +always_inline u32 +tcp_available_output_snd_space (const tcp_connection_t * tc) +{ + u32 available_wnd = tcp_available_snd_wnd (tc); + int flight_size = (int) (tc->snd_nxt - tc->snd_una); + + if (available_wnd <= flight_size) + return 0; + + return available_wnd - flight_size; +} + +/** + * Estimate of how many bytes we can still push into the network + */ +always_inline u32 +tcp_available_cc_snd_space (const tcp_connection_t * tc) +{ + u32 available_wnd = tcp_available_snd_wnd (tc); + u32 flight_size = tcp_flight_size (tc); + + if (available_wnd <= flight_size) + return 0; + + return available_wnd - flight_size; +} + +always_inline u8 +tcp_is_lost_fin (tcp_connection_t * tc) +{ + if ((tc->flags & TCP_CONN_FINSNT) && (tc->snd_una_max - tc->snd_una == 1)) + return 1; + return 0; +} + +always_inline u32 +tcp_time_now (void) +{ + return tcp_main.wrk_ctx[vlib_get_thread_index ()].time_now; +} + +always_inline u32 +tcp_time_now_w_thread (u32 thread_index) +{ + return tcp_main.wrk_ctx[thread_index].time_now; +} + +/** + * Generate timestamp for tcp connection + */ +always_inline u32 +tcp_tstamp (tcp_connection_t * tc) +{ + return (tcp_main.wrk_ctx[tc->c_thread_index].time_now - + tc->timestamp_delta); +} + +always_inline f64 +tcp_time_now_us (u32 thread_index) +{ + return transport_time_now (thread_index); +} + +always_inline u32 +tcp_set_time_now (tcp_worker_ctx_t * wrk) +{ + wrk->time_now = clib_cpu_time_now () * tcp_main.tstamp_ticks_per_clock; + return wrk->time_now; +} + +always_inline tcp_connection_t * +tcp_input_lookup_buffer (vlib_buffer_t * b, u8 thread_index, u32 * error, + u8 is_ip4, u8 is_nolookup) +{ + u32 fib_index = vnet_buffer (b)->ip.fib_index; + int n_advance_bytes, n_data_bytes; + transport_connection_t *tc; + tcp_header_t *tcp; + u8 result = 0; + + if (is_ip4) + { + ip4_header_t *ip4 = vlib_buffer_get_current (b); + int ip_hdr_bytes = ip4_header_bytes (ip4); + if (PREDICT_FALSE (b->current_length < ip_hdr_bytes + sizeof (*tcp))) + { + *error = TCP_ERROR_LENGTH; + return 0; + } + tcp = ip4_next_header (ip4); + vnet_buffer (b)->tcp.hdr_offset = (u8 *) tcp - (u8 *) ip4; + n_advance_bytes = (ip_hdr_bytes + tcp_header_bytes (tcp)); + n_data_bytes = clib_net_to_host_u16 (ip4->length) - n_advance_bytes; + + /* Length check. Checksum computed by ipx_local no need to compute again */ + if (PREDICT_FALSE (n_data_bytes < 0)) + { + *error = TCP_ERROR_LENGTH; + return 0; + } + + if (!is_nolookup) + tc = session_lookup_connection_wt4 (fib_index, &ip4->dst_address, + &ip4->src_address, tcp->dst_port, + tcp->src_port, + TRANSPORT_PROTO_TCP, thread_index, + &result); + } + else + { + ip6_header_t *ip6 = vlib_buffer_get_current (b); + if (PREDICT_FALSE (b->current_length < sizeof (*ip6) + sizeof (*tcp))) + { + *error = TCP_ERROR_LENGTH; + return 0; + } + tcp = ip6_next_header (ip6); + vnet_buffer (b)->tcp.hdr_offset = (u8 *) tcp - (u8 *) ip6; + n_advance_bytes = tcp_header_bytes (tcp); + n_data_bytes = clib_net_to_host_u16 (ip6->payload_length) + - n_advance_bytes; + n_advance_bytes += sizeof (ip6[0]); + + if (PREDICT_FALSE (n_data_bytes < 0)) + { + *error = TCP_ERROR_LENGTH; + return 0; + } + + if (!is_nolookup) + { + if (PREDICT_FALSE + (ip6_address_is_link_local_unicast (&ip6->dst_address))) + { + ip4_main_t *im = &ip4_main; + fib_index = vec_elt (im->fib_index_by_sw_if_index, + vnet_buffer (b)->sw_if_index[VLIB_RX]); + } + + tc = session_lookup_connection_wt6 (fib_index, &ip6->dst_address, + &ip6->src_address, + tcp->dst_port, tcp->src_port, + TRANSPORT_PROTO_TCP, + thread_index, &result); + } + } + + if (is_nolookup) + tc = + (transport_connection_t *) tcp_connection_get (vnet_buffer (b)-> + tcp.connection_index, + thread_index); + + vnet_buffer (b)->tcp.seq_number = clib_net_to_host_u32 (tcp->seq_number); + vnet_buffer (b)->tcp.ack_number = clib_net_to_host_u32 (tcp->ack_number); + vnet_buffer (b)->tcp.data_offset = n_advance_bytes; + vnet_buffer (b)->tcp.data_len = n_data_bytes; + vnet_buffer (b)->tcp.seq_end = vnet_buffer (b)->tcp.seq_number + + n_data_bytes; + vnet_buffer (b)->tcp.flags = 0; + + *error = result ? TCP_ERROR_NONE + result : *error; + + return tcp_get_connection_from_transport (tc); +} + +/** + * Initialize connection by gleaning network and rcv params from buffer + * + * @param tc connection to initialize + * @param b buffer whose current data is pointing at ip + * @param is_ip4 flag set to 1 if using ip4 + */ +always_inline void +tcp_init_w_buffer (tcp_connection_t * tc, vlib_buffer_t * b, u8 is_ip4) +{ + tcp_header_t *th = tcp_buffer_hdr (b); + + tc->c_lcl_port = th->dst_port; + tc->c_rmt_port = th->src_port; + tc->c_is_ip4 = is_ip4; + + if (is_ip4) + { + ip4_header_t *ip4 = vlib_buffer_get_current (b); + tc->c_lcl_ip4.as_u32 = ip4->dst_address.as_u32; + tc->c_rmt_ip4.as_u32 = ip4->src_address.as_u32; + } + else + { + ip6_header_t *ip6 = vlib_buffer_get_current (b); + clib_memcpy_fast (&tc->c_lcl_ip6, &ip6->dst_address, + sizeof (ip6_address_t)); + clib_memcpy_fast (&tc->c_rmt_ip6, &ip6->src_address, + sizeof (ip6_address_t)); + } + + tc->irs = vnet_buffer (b)->tcp.seq_number; + tc->rcv_nxt = vnet_buffer (b)->tcp.seq_number + 1; + tc->rcv_las = tc->rcv_nxt; + tc->sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX]; + tc->snd_wl1 = vnet_buffer (b)->tcp.seq_number; + tc->snd_wl2 = vnet_buffer (b)->tcp.ack_number; + + /* RFC1323: TSval timestamps sent on {SYN} and {SYN,ACK} + * segments are used to initialize PAWS. */ + if (tcp_opts_tstamp (&tc->rcv_opts)) + { + tc->tsval_recent = tc->rcv_opts.tsval; + tc->tsval_recent_age = tcp_time_now (); + } + + if (tcp_opts_wscale (&tc->rcv_opts)) + tc->snd_wscale = tc->rcv_opts.wscale; + + tc->snd_wnd = clib_net_to_host_u16 (th->window) << tc->snd_wscale; +} + +always_inline void +tcp_update_rto (tcp_connection_t * tc) +{ + tc->rto = clib_min (tc->srtt + (tc->rttvar << 2), TCP_RTO_MAX); + tc->rto = clib_max (tc->rto, TCP_RTO_MIN); +} + +always_inline u8 +tcp_is_descheduled (tcp_connection_t * tc) +{ + return (transport_connection_is_descheduled (&tc->connection) ? 1 : 0); +} + +/** + * Push TCP header to buffer + * + * @param vm - vlib_main + * @param b - buffer to write the header to + * @param sp_net - source port net order + * @param dp_net - destination port net order + * @param seq - sequence number net order + * @param ack - ack number net order + * @param tcp_hdr_opts_len - header and options length in bytes + * @param flags - header flags + * @param wnd - window size + * + * @return - pointer to start of TCP header + */ +always_inline void * +vlib_buffer_push_tcp_net_order (vlib_buffer_t * b, u16 sp, u16 dp, u32 seq, + u32 ack, u8 tcp_hdr_opts_len, u8 flags, + u16 wnd) +{ + tcp_header_t *th; + + th = vlib_buffer_push_uninit (b, tcp_hdr_opts_len); + + th->src_port = sp; + th->dst_port = dp; + th->seq_number = seq; + th->ack_number = ack; + th->data_offset_and_reserved = (tcp_hdr_opts_len >> 2) << 4; + th->flags = flags; + th->window = wnd; + th->checksum = 0; + th->urgent_pointer = 0; + vnet_buffer (b)->l4_hdr_offset = (u8 *) th - b->data; + b->flags |= VNET_BUFFER_F_L4_HDR_OFFSET_VALID; + return th; +} + +/** + * Push TCP header to buffer + * + * @param b - buffer to write the header to + * @param sp_net - source port net order + * @param dp_net - destination port net order + * @param seq - sequence number host order + * @param ack - ack number host order + * @param tcp_hdr_opts_len - header and options length in bytes + * @param flags - header flags + * @param wnd - window size + * + * @return - pointer to start of TCP header + */ +always_inline void * +vlib_buffer_push_tcp (vlib_buffer_t * b, u16 sp_net, u16 dp_net, u32 seq, + u32 ack, u8 tcp_hdr_opts_len, u8 flags, u16 wnd) +{ + return vlib_buffer_push_tcp_net_order (b, sp_net, dp_net, + clib_host_to_net_u32 (seq), + clib_host_to_net_u32 (ack), + tcp_hdr_opts_len, flags, + clib_host_to_net_u16 (wnd)); +} + +#endif /* SRC_VNET_TCP_TCP_INLINES_H_ */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ -- cgit 1.2.3-korg