diff options
Diffstat (limited to 'src/vnet/gso')
| -rw-r--r-- | src/vnet/gso/FEATURE.yaml | 2 | ||||
| -rw-r--r-- | src/vnet/gso/gro.h | 292 | ||||
| -rw-r--r-- | src/vnet/gso/gro_func.h | 593 |
3 files changed, 887 insertions, 0 deletions
diff --git a/src/vnet/gso/FEATURE.yaml b/src/vnet/gso/FEATURE.yaml index 79b506df69c..d3db0cc23e3 100644 --- a/src/vnet/gso/FEATURE.yaml +++ b/src/vnet/gso/FEATURE.yaml @@ -8,6 +8,8 @@ features: - GSO for IP-IP tunnel - GSO for IPSec tunnel - Provide inline function to get header offsets + - Basic GRO support + - Implements flow table support description: "Generic Segmentation Offload" missing: - Thorough Testing, GRE, Geneve diff --git a/src/vnet/gso/gro.h b/src/vnet/gso/gro.h new file mode 100644 index 00000000000..bfa592041e5 --- /dev/null +++ b/src/vnet/gso/gro.h @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2020 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef included_gro_h +#define included_gro_h + +#include <vlib/vlib.h> +#include <vppinfra/error.h> +#include <vnet/ip/ip46_address.h> + +#define GRO_FLOW_TABLE_MAX_SIZE 16 +#define GRO_FLOW_TABLE_FLUSH 1e-5 +#define GRO_FLOW_N_BUFFERS 64 +#define GRO_FLOW_TIMEOUT 1e-5 /* 10 micro-seconds */ +#define GRO_TO_VECTOR_SIZE(X) (X + GRO_FLOW_TABLE_MAX_SIZE) + +typedef union +{ + struct + { + u32 sw_if_index[VLIB_N_RX_TX]; + ip46_address_t src_address; + ip46_address_t dst_address; + u16 src_port; + u16 dst_port; + }; + + u64 flow_data[5]; + u32 flow_data_u32; +} gro_flow_key_t; + +typedef struct +{ + gro_flow_key_t flow_key; + f64 next_timeout_ts; + u32 last_ack_number; + u32 buffer_index; + u16 n_buffers; +} gro_flow_t; + +typedef struct +{ + f64 timeout_ts; + u64 total_vectors; + u32 n_vectors; + u32 node_index; + u8 is_enable; + u8 is_l2; + u8 flow_table_size; + gro_flow_t gro_flow[GRO_FLOW_TABLE_MAX_SIZE]; +} gro_flow_table_t; + +static_always_inline void +gro_flow_set_flow_key (gro_flow_t * to, gro_flow_key_t * from) +{ + to->flow_key.flow_data[0] = from->flow_data[0]; + to->flow_key.flow_data[1] = from->flow_data[1]; + to->flow_key.flow_data[2] = from->flow_data[2]; + to->flow_key.flow_data[3] = from->flow_data[3]; + to->flow_key.flow_data[4] = from->flow_data[4]; + to->flow_key.flow_data_u32 = from->flow_data_u32; +} + +static_always_inline u8 +gro_flow_is_equal (gro_flow_key_t * first, gro_flow_key_t * second) +{ + if (first->flow_data[0] == second->flow_data[0] && + first->flow_data[1] == second->flow_data[1] && + first->flow_data[2] == second->flow_data[2] && + first->flow_data[3] == second->flow_data[3] && + first->flow_data[4] == second->flow_data[4] && + first->flow_data_u32 == second->flow_data_u32) + return 1; + + return 0; +} + +/** + * timeout_expire is in between 3 to 10 microseconds + * 3e-6 1e-5 + */ +static_always_inline void +gro_flow_set_timeout (vlib_main_t * vm, gro_flow_t * gro_flow, + f64 timeout_expire) +{ + gro_flow->next_timeout_ts = vlib_time_now (vm) + timeout_expire; +} + +static_always_inline u8 +gro_flow_is_timeout (vlib_main_t * vm, gro_flow_t * gro_flow) +{ + if (gro_flow->next_timeout_ts < vlib_time_now (vm)) + return 1; + return 0; +} + +static_always_inline void +gro_flow_store_packet (gro_flow_t * gro_flow, u32 bi0) +{ + if (gro_flow->n_buffers == 0) + { + gro_flow->buffer_index = bi0; + } + gro_flow->n_buffers++; +} + +static_always_inline u32 +gro_flow_table_init (gro_flow_table_t ** flow_table, u8 is_l2, u32 node_index) +{ + if (*flow_table) + return 0; + + gro_flow_table_t *flow_table_temp = 0; + flow_table_temp = + (gro_flow_table_t *) clib_mem_alloc (sizeof (gro_flow_table_t)); + if (!flow_table_temp) + return 0; + clib_memset (flow_table_temp, 0, sizeof (gro_flow_table_t)); + flow_table_temp->node_index = node_index; + flow_table_temp->is_enable = 1; + flow_table_temp->is_l2 = is_l2; + *flow_table = flow_table_temp; + return 1; +} + +static_always_inline void +gro_flow_table_set_timeout (vlib_main_t * vm, gro_flow_table_t * flow_table, + f64 timeout_expire) +{ + if (flow_table) + flow_table->timeout_ts = vlib_time_now (vm) + timeout_expire; +} + +static_always_inline u8 +gro_flow_table_is_timeout (vlib_main_t * vm, gro_flow_table_t * flow_table) +{ + if (flow_table && (flow_table->timeout_ts < vlib_time_now (vm))) + return 1; + return 0; +} + +static_always_inline u8 +gro_flow_table_is_enable (gro_flow_table_t * flow_table) +{ + if (flow_table) + return flow_table->is_enable; + + return 0; +} + +static_always_inline void +gro_flow_table_set_is_enable (gro_flow_table_t * flow_table, u8 is_enable) +{ + if (flow_table) + { + if (is_enable) + { + flow_table->is_enable = 1; + } + else + { + flow_table->is_enable = 0; + } + } +} + +static_always_inline void +gro_flow_table_free (gro_flow_table_t * flow_table) +{ + if (flow_table) + clib_mem_free (flow_table); +} + +static_always_inline void +gro_flow_table_set_node_index (gro_flow_table_t * flow_table, u32 node_index) +{ + if (flow_table) + flow_table->node_index = node_index; +} + +static_always_inline gro_flow_t * +gro_flow_table_new_flow (gro_flow_table_t * flow_table) +{ + if (PREDICT_TRUE (flow_table->flow_table_size < GRO_FLOW_TABLE_MAX_SIZE)) + { + gro_flow_t *gro_flow; + u32 i = 0; + while (i < GRO_FLOW_TABLE_MAX_SIZE) + { + gro_flow = &flow_table->gro_flow[i]; + if (gro_flow->n_buffers == 0) + { + flow_table->flow_table_size++; + return gro_flow; + } + i++; + } + } + + return (0); +} + +static_always_inline gro_flow_t * +gro_flow_table_get_flow (gro_flow_table_t * flow_table, + gro_flow_key_t * flow_key) +{ + gro_flow_t *gro_flow = 0; + u32 i = 0; + while (i < GRO_FLOW_TABLE_MAX_SIZE) + { + gro_flow = &flow_table->gro_flow[i]; + if (gro_flow_is_equal (flow_key, &gro_flow->flow_key)) + return gro_flow; + i++; + } + return (0); +} + +static_always_inline gro_flow_t * +gro_flow_table_find_or_add_flow (gro_flow_table_t * flow_table, + gro_flow_key_t * flow_key) +{ + gro_flow_t *gro_flow = 0; + + gro_flow = gro_flow_table_get_flow (flow_table, flow_key); + if (gro_flow) + return gro_flow; + + gro_flow = gro_flow_table_new_flow (flow_table); + + if (gro_flow) + { + gro_flow_set_flow_key (gro_flow, flow_key); + return gro_flow; + } + + return (0); +} + +static_always_inline void +gro_flow_table_reset_flow (gro_flow_table_t * flow_table, + gro_flow_t * gro_flow) +{ + if (PREDICT_TRUE (flow_table->flow_table_size > 0)) + { + clib_memset (gro_flow, 0, sizeof (gro_flow_t)); + flow_table->flow_table_size--; + } +} + +static_always_inline u8 * +gro_flow_table_format (u8 * s, va_list * args) +{ + gro_flow_table_t *flow_table = va_arg (*args, gro_flow_table_t *); + + s = + format (s, + "flow-table: size %u gro-total-vectors %lu gro-n-vectors %u", + flow_table->flow_table_size, flow_table->total_vectors, + flow_table->n_vectors); + if (flow_table->n_vectors) + { + double average_rate = + (double) flow_table->total_vectors / (double) flow_table->n_vectors; + s = format (s, " gro-average-rate %.2f", average_rate); + } + else + s = format (s, " gro-average-rate 0.00"); + + return s; +} +#endif /* included_gro_h */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vnet/gso/gro_func.h b/src/vnet/gso/gro_func.h new file mode 100644 index 00000000000..a410a651933 --- /dev/null +++ b/src/vnet/gso/gro_func.h @@ -0,0 +1,593 @@ +/* + * Copyright (c) 2020 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef included_gro_func_h +#define included_gro_func_h + +#include <vnet/ethernet/ethernet.h> +#include <vnet/gso/gro.h> +#include <vnet/gso/hdr_offset_parser.h> +#include <vnet/udp/udp_packet.h> +#include <vnet/tcp/tcp.h> +#include <vnet/vnet.h> + +static_always_inline u8 +gro_is_bad_packet (vlib_buffer_t * b, u8 flags, i16 l234_sz) +{ + if (((b->current_length - l234_sz) <= 0) || ((flags &= ~TCP_FLAG_ACK) != 0)) + return 1; + return 0; +} + +static_always_inline void +gro_get_ip4_flow_from_packet (u32 * sw_if_index, + ip4_header_t * ip4, tcp_header_t * tcp, + gro_flow_key_t * flow_key, int is_l2) +{ + flow_key->sw_if_index[VLIB_RX] = sw_if_index[VLIB_RX]; + flow_key->sw_if_index[VLIB_TX] = sw_if_index[VLIB_TX]; + ip46_address_set_ip4 (&flow_key->src_address, &ip4->src_address); + ip46_address_set_ip4 (&flow_key->dst_address, &ip4->dst_address); + flow_key->src_port = tcp->src_port; + flow_key->dst_port = tcp->dst_port; +} + +static_always_inline void +gro_get_ip6_flow_from_packet (u32 * sw_if_index, + ip6_header_t * ip6, tcp_header_t * tcp, + gro_flow_key_t * flow_key, int is_l2) +{ + flow_key->sw_if_index[VLIB_RX] = sw_if_index[VLIB_RX]; + flow_key->sw_if_index[VLIB_TX] = sw_if_index[VLIB_TX]; + ip46_address_set_ip6 (&flow_key->src_address, &ip6->src_address); + ip46_address_set_ip6 (&flow_key->dst_address, &ip6->dst_address); + flow_key->src_port = tcp->src_port; + flow_key->dst_port = tcp->dst_port; +} + +static_always_inline u32 +gro_is_ip4_or_ip6_packet (vlib_buffer_t * b0, int is_l2) +{ + if (b0->flags & VNET_BUFFER_F_IS_IP4) + return VNET_BUFFER_F_IS_IP4; + if (b0->flags & VNET_BUFFER_F_IS_IP6) + return VNET_BUFFER_F_IS_IP6; + if (is_l2) + { + ethernet_header_t *eh = + (ethernet_header_t *) vlib_buffer_get_current (b0); + u16 ethertype = clib_net_to_host_u16 (eh->type); + + if (ethernet_frame_is_tagged (ethertype)) + { + ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1); + + ethertype = clib_net_to_host_u16 (vlan->type); + if (ethertype == ETHERNET_TYPE_VLAN) + { + vlan++; + ethertype = clib_net_to_host_u16 (vlan->type); + } + } + if (ethertype == ETHERNET_TYPE_IP4) + return VNET_BUFFER_F_IS_IP4; + if (ethertype == ETHERNET_TYPE_IP6) + return VNET_BUFFER_F_IS_IP6; + } + else + { + if ((((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0) == 0x40) + return VNET_BUFFER_F_IS_IP4; + if ((((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0) == 0x60) + return VNET_BUFFER_F_IS_IP6; + } + + return 0; +} + +typedef enum +{ + GRO_PACKET_ACTION_NONE = 0, + GRO_PACKET_ACTION_ENQUEUE = 1, + GRO_PACKET_ACTION_FLUSH = 2, +} gro_packet_action_t; + +static_always_inline gro_packet_action_t +gro_tcp_sequence_check (tcp_header_t * tcp0, tcp_header_t * tcp1, + u32 payload_len0) +{ + u32 next_tcp_seq0 = clib_net_to_host_u32 (tcp0->seq_number); + u32 next_tcp_seq1 = clib_net_to_host_u32 (tcp1->seq_number); + + /* next packet, enqueue */ + if (PREDICT_TRUE (next_tcp_seq0 + payload_len0 == next_tcp_seq1)) + return GRO_PACKET_ACTION_ENQUEUE; + /* flush all packets */ + else + return GRO_PACKET_ACTION_FLUSH; +} + +static_always_inline void +gro_merge_buffers (vlib_main_t * vm, vlib_buffer_t * b0, + vlib_buffer_t * b1, u32 bi1, u32 payload_len1, + u16 l234_sz1) +{ + vlib_buffer_t *pb = b0; + + if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0)) + b0->total_length_not_including_first_buffer = 0; + + while (pb->flags & VLIB_BUFFER_NEXT_PRESENT) + pb = vlib_get_buffer (vm, pb->next_buffer); + + vlib_buffer_advance (b1, l234_sz1); + pb->flags |= VLIB_BUFFER_NEXT_PRESENT; + pb->next_buffer = bi1; + b0->total_length_not_including_first_buffer += payload_len1; + b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; +} + +static_always_inline u32 +gro_get_packet_data (vlib_main_t * vm, vlib_buffer_t * b0, + generic_header_offset_t * gho0, + gro_flow_key_t * flow_key0, int is_l2) +{ + ip4_header_t *ip4_0 = 0; + ip6_header_t *ip6_0 = 0; + tcp_header_t *tcp0 = 0; + u32 pkt_len0 = 0; + u16 l234_sz0 = 0; + u32 sw_if_index0[VLIB_N_RX_TX] = { ~0 }; + + u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2); + + if (is_ip0 & VNET_BUFFER_F_IS_IP4) + vnet_generic_header_offset_parser (b0, gho0, is_l2, 1 /* is_ip4 */ , + 0 /* is_ip6 */ ); + else if (is_ip0 & VNET_BUFFER_F_IS_IP6) + vnet_generic_header_offset_parser (b0, gho0, is_l2, 0 /* is_ip4 */ , + 1 /* is_ip6 */ ); + else + return 0; + + if (PREDICT_FALSE ((gho0->gho_flags & GHO_F_TCP) == 0)) + return 0; + + ip4_0 = + (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0->l3_hdr_offset); + ip6_0 = + (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0->l3_hdr_offset); + tcp0 = + (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0->l4_hdr_offset); + + l234_sz0 = gho0->hdr_sz; + if (PREDICT_FALSE (gro_is_bad_packet (b0, tcp0->flags, l234_sz0))) + return 0; + + sw_if_index0[VLIB_RX] = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + sw_if_index0[VLIB_TX] = vnet_buffer (b0)->sw_if_index[VLIB_TX]; + + if (gho0->gho_flags & GHO_F_IP4) + { + gro_get_ip4_flow_from_packet (sw_if_index0, ip4_0, tcp0, flow_key0, + is_l2); + } + else if (gho0->gho_flags & GHO_F_IP6) + { + gro_get_ip6_flow_from_packet (sw_if_index0, ip6_0, tcp0, flow_key0, + is_l2); + } + else + return 0; + + pkt_len0 = vlib_buffer_length_in_chain (vm, b0); + if (PREDICT_FALSE (pkt_len0 >= TCP_MAX_GSO_SZ)) + return 0; + + return pkt_len0; +} + +static_always_inline u32 +gro_coalesce_buffers (vlib_main_t * vm, vlib_buffer_t * b0, + vlib_buffer_t * b1, u32 bi1, int is_l2) +{ + generic_header_offset_t gho0 = { 0 }; + generic_header_offset_t gho1 = { 0 }; + gro_flow_key_t flow_key0, flow_key1; + ip4_header_t *ip4_0, *ip4_1; + ip6_header_t *ip6_0, *ip6_1; + tcp_header_t *tcp0, *tcp1; + u16 l234_sz0, l234_sz1; + u32 pkt_len0, pkt_len1, payload_len0, payload_len1; + u32 sw_if_index0[VLIB_N_RX_TX] = { ~0 }; + u32 sw_if_index1[VLIB_N_RX_TX] = { ~0 }; + + u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2); + u32 is_ip1 = gro_is_ip4_or_ip6_packet (b1, is_l2); + + if (is_ip0 & VNET_BUFFER_F_IS_IP4) + vnet_generic_header_offset_parser (b0, &gho0, is_l2, 1 /* is_ip4 */ , + 0 /* is_ip6 */ ); + else if (is_ip0 & VNET_BUFFER_F_IS_IP6) + vnet_generic_header_offset_parser (b0, &gho0, is_l2, 0 /* is_ip4 */ , + 1 /* is_ip6 */ ); + else + return 0; + + if (is_ip1 & VNET_BUFFER_F_IS_IP4) + vnet_generic_header_offset_parser (b1, &gho1, is_l2, 1 /* is_ip4 */ , + 0 /* is_ip6 */ ); + else if (is_ip1 & VNET_BUFFER_F_IS_IP6) + vnet_generic_header_offset_parser (b1, &gho1, is_l2, 0 /* is_ip4 */ , + 1 /* is_ip6 */ ); + else + return 0; + + pkt_len0 = vlib_buffer_length_in_chain (vm, b0); + pkt_len1 = vlib_buffer_length_in_chain (vm, b1); + + if (((gho0.gho_flags & GHO_F_TCP) == 0) + || ((gho1.gho_flags & GHO_F_TCP) == 0)) + return 0; + + ip4_0 = + (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset); + ip4_1 = + (ip4_header_t *) (vlib_buffer_get_current (b1) + gho1.l3_hdr_offset); + ip6_0 = + (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset); + ip6_1 = + (ip6_header_t *) (vlib_buffer_get_current (b1) + gho1.l3_hdr_offset); + + tcp0 = (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset); + tcp1 = (tcp_header_t *) (vlib_buffer_get_current (b1) + gho1.l4_hdr_offset); + + l234_sz0 = gho0.hdr_sz; + l234_sz1 = gho1.hdr_sz; + + if (gro_is_bad_packet (b0, tcp0->flags, l234_sz0) + || gro_is_bad_packet (b1, tcp1->flags, l234_sz1)) + return 0; + + sw_if_index0[VLIB_RX] = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + sw_if_index0[VLIB_TX] = vnet_buffer (b0)->sw_if_index[VLIB_TX]; + + sw_if_index1[VLIB_RX] = vnet_buffer (b1)->sw_if_index[VLIB_RX]; + sw_if_index1[VLIB_TX] = vnet_buffer (b1)->sw_if_index[VLIB_TX]; + + if ((gho0.gho_flags & GHO_F_IP4) && (gho1.gho_flags & GHO_F_IP4)) + { + gro_get_ip4_flow_from_packet (sw_if_index0, ip4_0, tcp0, &flow_key0, + is_l2); + gro_get_ip4_flow_from_packet (sw_if_index1, ip4_1, tcp1, &flow_key1, + is_l2); + } + else if ((gho0.gho_flags & GHO_F_IP6) && (gho1.gho_flags & GHO_F_IP6)) + { + gro_get_ip6_flow_from_packet (sw_if_index0, ip6_0, tcp0, &flow_key0, + is_l2); + gro_get_ip6_flow_from_packet (sw_if_index1, ip6_1, tcp1, &flow_key1, + is_l2); + } + else + return 0; + + if (gro_flow_is_equal (&flow_key0, &flow_key1) == 0) + return 0; + + payload_len0 = pkt_len0 - l234_sz0; + payload_len1 = pkt_len1 - l234_sz1; + + if (pkt_len0 >= TCP_MAX_GSO_SZ || pkt_len1 >= TCP_MAX_GSO_SZ + || (pkt_len0 + payload_len1) >= TCP_MAX_GSO_SZ) + return 0; + + if (gro_tcp_sequence_check (tcp0, tcp1, payload_len0) == + GRO_PACKET_ACTION_ENQUEUE) + { + gro_merge_buffers (vm, b0, b1, bi1, payload_len1, l234_sz1); + return tcp1->ack_number; + } + + return 0; +} + +static_always_inline void +gro_fixup_header (vlib_main_t * vm, vlib_buffer_t * b0, u32 ack_number, + int is_l2) +{ + generic_header_offset_t gho0 = { 0 }; + + u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2); + + if (is_ip0 & VNET_BUFFER_F_IS_IP4) + vnet_generic_header_offset_parser (b0, &gho0, is_l2, 1 /* is_ip4 */ , + 0 /* is_ip6 */ ); + else if (is_ip0 & VNET_BUFFER_F_IS_IP6) + vnet_generic_header_offset_parser (b0, &gho0, is_l2, 0 /* is_ip4 */ , + 1 /* is_ip6 */ ); + + vnet_buffer2 (b0)->gso_size = b0->current_length - gho0.hdr_sz; + + if (gho0.gho_flags & GHO_F_IP4) + { + ip4_header_t *ip4 = + (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset); + ip4->length = + clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - + gho0.l3_hdr_offset); + b0->flags |= + (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4 | + VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_IP_CKSUM); + } + else if (gho0.gho_flags & GHO_F_IP6) + { + ip6_header_t *ip6 = + (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset); + ip6->payload_length = + clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - + gho0.l4_hdr_offset); + b0->flags |= + (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6 | + VNET_BUFFER_F_OFFLOAD_TCP_CKSUM); + } + + tcp_header_t *tcp0 = + (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset); + tcp0->ack_number = ack_number; + b0->flags &= ~VLIB_BUFFER_IS_TRACED; +} + +static_always_inline u32 +vnet_gro_flow_table_flush (vlib_main_t * vm, gro_flow_table_t * flow_table, + u32 * to) +{ + if (flow_table->flow_table_size > 0) + { + gro_flow_t *gro_flow; + u32 i = 0, j = 0; + while (i < GRO_FLOW_TABLE_MAX_SIZE) + { + gro_flow = &flow_table->gro_flow[i]; + if (gro_flow->n_buffers && gro_flow_is_timeout (vm, gro_flow)) + { + // flush the packet + vlib_buffer_t *b0 = + vlib_get_buffer (vm, gro_flow->buffer_index); + gro_fixup_header (vm, b0, gro_flow->last_ack_number, + flow_table->is_l2); + to[j] = gro_flow->buffer_index; + gro_flow_table_reset_flow (flow_table, gro_flow); + flow_table->n_vectors++; + j++; + } + i++; + } + + return j; + } + return 0; +} + +static_always_inline void +vnet_gro_flow_table_schedule_node_on_dispatcher (vlib_main_t * vm, + gro_flow_table_t * + flow_table) +{ + if (gro_flow_table_is_timeout (vm, flow_table)) + { + u32 to[GRO_FLOW_TABLE_MAX_SIZE] = { 0 }; + u32 n_to = vnet_gro_flow_table_flush (vm, flow_table, to); + + if (n_to > 0) + { + u32 node_index = flow_table->node_index; + vlib_frame_t *f = vlib_get_frame_to_node (vm, node_index); + u32 *f_to = vlib_frame_vector_args (f); + u32 i = 0; + + while (i < n_to) + { + f_to[f->n_vectors] = to[i]; + i++; + f->n_vectors++; + } + vlib_put_frame_to_node (vm, node_index, f); + } + gro_flow_table_set_timeout (vm, flow_table, GRO_FLOW_TABLE_FLUSH); + } +} + +static_always_inline u32 +vnet_gro_flow_table_inline (vlib_main_t * vm, gro_flow_table_t * flow_table, + u32 bi0, u32 * to) +{ + vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); + generic_header_offset_t gho0 = { 0 }; + gro_flow_t *gro_flow = 0; + gro_flow_key_t flow_key0 = { }; + tcp_header_t *tcp0 = 0; + u32 pkt_len0 = 0; + int is_l2 = flow_table->is_l2; + + if (!gro_flow_table_is_enable (flow_table)) + { + to[0] = bi0; + return 1; + } + + if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_GSO)) + { + to[0] = bi0; + return 1; + } + + pkt_len0 = gro_get_packet_data (vm, b0, &gho0, &flow_key0, is_l2); + if (pkt_len0 == 0) + { + to[0] = bi0; + return 1; + } + + gro_flow = gro_flow_table_find_or_add_flow (flow_table, &flow_key0); + if (!gro_flow) + { + to[0] = bi0; + return 1; + } + + if (PREDICT_FALSE (gro_flow->n_buffers == 0)) + { + flow_table->total_vectors++; + gro_flow_store_packet (gro_flow, bi0); + tcp0 = + (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset); + gro_flow->last_ack_number = tcp0->ack_number; + gro_flow_set_timeout (vm, gro_flow, GRO_FLOW_TIMEOUT); + return 0; + } + else + { + tcp0 = + (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset); + generic_header_offset_t gho_s = { 0 }; + tcp_header_t *tcp_s; + u16 l234_sz0, l234_sz_s; + u32 pkt_len_s, payload_len0, payload_len_s; + u32 bi_s = gro_flow->buffer_index; + + vlib_buffer_t *b_s = vlib_get_buffer (vm, bi_s); + u32 is_ip_s = gro_is_ip4_or_ip6_packet (b_s, is_l2); + if (is_ip_s & VNET_BUFFER_F_IS_IP4) + vnet_generic_header_offset_parser (b_s, &gho_s, is_l2, + 1 /* is_ip4 */ , 0 /* is_ip6 */ ); + else if (is_ip_s & VNET_BUFFER_F_IS_IP6) + vnet_generic_header_offset_parser (b_s, &gho_s, is_l2, + 0 /* is_ip4 */ , 1 /* is_ip6 */ ); + + tcp_s = + (tcp_header_t *) (vlib_buffer_get_current (b_s) + + gho_s.l4_hdr_offset); + pkt_len_s = vlib_buffer_length_in_chain (vm, b_s); + l234_sz0 = gho0.hdr_sz; + l234_sz_s = gho_s.hdr_sz; + payload_len0 = pkt_len0 - l234_sz0; + payload_len_s = pkt_len_s - l234_sz_s; + gro_packet_action_t action = + gro_tcp_sequence_check (tcp_s, tcp0, payload_len_s); + + if (PREDICT_TRUE (action == GRO_PACKET_ACTION_ENQUEUE)) + { + if (PREDICT_TRUE ((pkt_len_s + payload_len0) < TCP_MAX_GSO_SZ)) + { + flow_table->total_vectors++; + gro_merge_buffers (vm, b_s, b0, bi0, payload_len0, l234_sz0); + gro_flow_store_packet (gro_flow, bi0); + gro_flow->last_ack_number = tcp0->ack_number; + return 0; + } + else + { + // flush the stored GSO size packet and buffer the current packet + flow_table->n_vectors++; + flow_table->total_vectors++; + gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2); + gro_flow->n_buffers = 0; + gro_flow_store_packet (gro_flow, bi0); + gro_flow->last_ack_number = tcp0->ack_number; + gro_flow_set_timeout (vm, gro_flow, GRO_FLOW_TIMEOUT); + to[0] = bi_s; + return 1; + } + } + else + { + // flush the all (current and stored) packets + flow_table->n_vectors++; + flow_table->total_vectors++; + gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2); + gro_flow->n_buffers = 0; + gro_flow_table_reset_flow (flow_table, gro_flow); + to[0] = bi_s; + to[1] = bi0; + return 2; + } + } +} + +/** + * coalesce buffers with flow tables + */ +static_always_inline u32 +vnet_gro_inline (vlib_main_t * vm, gro_flow_table_t * flow_table, u32 * from, + u16 n_left_from, u32 * to) +{ + u16 count = 0, i = 0; + + for (i = 0; i < n_left_from; i++) + count += vnet_gro_flow_table_inline (vm, flow_table, from[i], &to[count]); + + return count; +} + +/** + * coalesce buffers in opportunistic way without flow tables + */ +static_always_inline u32 +vnet_gro_simple_inline (vlib_main_t * vm, u32 * from, u16 n_left_from, + int is_l2) +{ + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; + vlib_get_buffers (vm, from, b, n_left_from); + u32 bi = 1, ack_number = 0; + if (PREDICT_TRUE (((b[0]->flags & VNET_BUFFER_F_GSO) == 0))) + { + while (n_left_from > 1) + { + if (PREDICT_TRUE (((b[bi]->flags & VNET_BUFFER_F_GSO) == 0))) + { + u32 ret; + if ((ret = + gro_coalesce_buffers (vm, b[0], b[bi], from[bi], + is_l2)) != 0) + { + n_left_from -= 1; + bi += 1; + ack_number = ret; + continue; + } + else + break; + } + else + break; + } + + if (bi >= 2) + { + gro_fixup_header (vm, b[0], ack_number, is_l2); + } + } + return bi; +} +#endif /* included_gro_func_h */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ |