diff options
Diffstat (limited to 'src/vnet')
| -rw-r--r-- | src/vnet/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | src/vnet/gso/FEATURE.yaml | 2 | ||||
| -rw-r--r-- | src/vnet/gso/gro.h | 292 | ||||
| -rw-r--r-- | src/vnet/gso/gro_func.h | 593 | ||||
| -rw-r--r-- | src/vnet/pg/cli.c | 10 | ||||
| -rw-r--r-- | src/vnet/pg/input.c | 4 | ||||
| -rw-r--r-- | src/vnet/pg/output.c | 18 | ||||
| -rw-r--r-- | src/vnet/pg/pg.api | 15 | ||||
| -rw-r--r-- | src/vnet/pg/pg.h | 10 | ||||
| -rw-r--r-- | src/vnet/pg/pg_api.c | 40 | ||||
| -rw-r--r-- | src/vnet/pg/stream.c | 25 |
11 files changed, 1002 insertions, 9 deletions
diff --git a/src/vnet/CMakeLists.txt b/src/vnet/CMakeLists.txt index 7757301c73a..a7f97bea4ca 100644 --- a/src/vnet/CMakeLists.txt +++ b/src/vnet/CMakeLists.txt @@ -1001,6 +1001,8 @@ list(APPEND VNET_SOURCES ) list(APPEND VNET_HEADERS + gso/gro.h + gso/gro_func.h gso/hdr_offset_parser.h gso/gso.h ) diff --git a/src/vnet/gso/FEATURE.yaml b/src/vnet/gso/FEATURE.yaml index 79b506df69c..d3db0cc23e3 100644 --- a/src/vnet/gso/FEATURE.yaml +++ b/src/vnet/gso/FEATURE.yaml @@ -8,6 +8,8 @@ features: - GSO for IP-IP tunnel - GSO for IPSec tunnel - Provide inline function to get header offsets + - Basic GRO support + - Implements flow table support description: "Generic Segmentation Offload" missing: - Thorough Testing, GRE, Geneve diff --git a/src/vnet/gso/gro.h b/src/vnet/gso/gro.h new file mode 100644 index 00000000000..bfa592041e5 --- /dev/null +++ b/src/vnet/gso/gro.h @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2020 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef included_gro_h +#define included_gro_h + +#include <vlib/vlib.h> +#include <vppinfra/error.h> +#include <vnet/ip/ip46_address.h> + +#define GRO_FLOW_TABLE_MAX_SIZE 16 +#define GRO_FLOW_TABLE_FLUSH 1e-5 +#define GRO_FLOW_N_BUFFERS 64 +#define GRO_FLOW_TIMEOUT 1e-5 /* 10 micro-seconds */ +#define GRO_TO_VECTOR_SIZE(X) (X + GRO_FLOW_TABLE_MAX_SIZE) + +typedef union +{ + struct + { + u32 sw_if_index[VLIB_N_RX_TX]; + ip46_address_t src_address; + ip46_address_t dst_address; + u16 src_port; + u16 dst_port; + }; + + u64 flow_data[5]; + u32 flow_data_u32; +} gro_flow_key_t; + +typedef struct +{ + gro_flow_key_t flow_key; + f64 next_timeout_ts; + u32 last_ack_number; + u32 buffer_index; + u16 n_buffers; +} gro_flow_t; + +typedef struct +{ + f64 timeout_ts; + u64 total_vectors; + u32 n_vectors; + u32 node_index; + u8 is_enable; + u8 is_l2; + u8 flow_table_size; + gro_flow_t gro_flow[GRO_FLOW_TABLE_MAX_SIZE]; +} gro_flow_table_t; + +static_always_inline void +gro_flow_set_flow_key (gro_flow_t * to, gro_flow_key_t * from) +{ + to->flow_key.flow_data[0] = from->flow_data[0]; + to->flow_key.flow_data[1] = from->flow_data[1]; + to->flow_key.flow_data[2] = from->flow_data[2]; + to->flow_key.flow_data[3] = from->flow_data[3]; + to->flow_key.flow_data[4] = from->flow_data[4]; + to->flow_key.flow_data_u32 = from->flow_data_u32; +} + +static_always_inline u8 +gro_flow_is_equal (gro_flow_key_t * first, gro_flow_key_t * second) +{ + if (first->flow_data[0] == second->flow_data[0] && + first->flow_data[1] == second->flow_data[1] && + first->flow_data[2] == second->flow_data[2] && + first->flow_data[3] == second->flow_data[3] && + first->flow_data[4] == second->flow_data[4] && + first->flow_data_u32 == second->flow_data_u32) + return 1; + + return 0; +} + +/** + * timeout_expire is in between 3 to 10 microseconds + * 3e-6 1e-5 + */ +static_always_inline void +gro_flow_set_timeout (vlib_main_t * vm, gro_flow_t * gro_flow, + f64 timeout_expire) +{ + gro_flow->next_timeout_ts = vlib_time_now (vm) + timeout_expire; +} + +static_always_inline u8 +gro_flow_is_timeout (vlib_main_t * vm, gro_flow_t * gro_flow) +{ + if (gro_flow->next_timeout_ts < vlib_time_now (vm)) + return 1; + return 0; +} + +static_always_inline void +gro_flow_store_packet (gro_flow_t * gro_flow, u32 bi0) +{ + if (gro_flow->n_buffers == 0) + { + gro_flow->buffer_index = bi0; + } + gro_flow->n_buffers++; +} + +static_always_inline u32 +gro_flow_table_init (gro_flow_table_t ** flow_table, u8 is_l2, u32 node_index) +{ + if (*flow_table) + return 0; + + gro_flow_table_t *flow_table_temp = 0; + flow_table_temp = + (gro_flow_table_t *) clib_mem_alloc (sizeof (gro_flow_table_t)); + if (!flow_table_temp) + return 0; + clib_memset (flow_table_temp, 0, sizeof (gro_flow_table_t)); + flow_table_temp->node_index = node_index; + flow_table_temp->is_enable = 1; + flow_table_temp->is_l2 = is_l2; + *flow_table = flow_table_temp; + return 1; +} + +static_always_inline void +gro_flow_table_set_timeout (vlib_main_t * vm, gro_flow_table_t * flow_table, + f64 timeout_expire) +{ + if (flow_table) + flow_table->timeout_ts = vlib_time_now (vm) + timeout_expire; +} + +static_always_inline u8 +gro_flow_table_is_timeout (vlib_main_t * vm, gro_flow_table_t * flow_table) +{ + if (flow_table && (flow_table->timeout_ts < vlib_time_now (vm))) + return 1; + return 0; +} + +static_always_inline u8 +gro_flow_table_is_enable (gro_flow_table_t * flow_table) +{ + if (flow_table) + return flow_table->is_enable; + + return 0; +} + +static_always_inline void +gro_flow_table_set_is_enable (gro_flow_table_t * flow_table, u8 is_enable) +{ + if (flow_table) + { + if (is_enable) + { + flow_table->is_enable = 1; + } + else + { + flow_table->is_enable = 0; + } + } +} + +static_always_inline void +gro_flow_table_free (gro_flow_table_t * flow_table) +{ + if (flow_table) + clib_mem_free (flow_table); +} + +static_always_inline void +gro_flow_table_set_node_index (gro_flow_table_t * flow_table, u32 node_index) +{ + if (flow_table) + flow_table->node_index = node_index; +} + +static_always_inline gro_flow_t * +gro_flow_table_new_flow (gro_flow_table_t * flow_table) +{ + if (PREDICT_TRUE (flow_table->flow_table_size < GRO_FLOW_TABLE_MAX_SIZE)) + { + gro_flow_t *gro_flow; + u32 i = 0; + while (i < GRO_FLOW_TABLE_MAX_SIZE) + { + gro_flow = &flow_table->gro_flow[i]; + if (gro_flow->n_buffers == 0) + { + flow_table->flow_table_size++; + return gro_flow; + } + i++; + } + } + + return (0); +} + +static_always_inline gro_flow_t * +gro_flow_table_get_flow (gro_flow_table_t * flow_table, + gro_flow_key_t * flow_key) +{ + gro_flow_t *gro_flow = 0; + u32 i = 0; + while (i < GRO_FLOW_TABLE_MAX_SIZE) + { + gro_flow = &flow_table->gro_flow[i]; + if (gro_flow_is_equal (flow_key, &gro_flow->flow_key)) + return gro_flow; + i++; + } + return (0); +} + +static_always_inline gro_flow_t * +gro_flow_table_find_or_add_flow (gro_flow_table_t * flow_table, + gro_flow_key_t * flow_key) +{ + gro_flow_t *gro_flow = 0; + + gro_flow = gro_flow_table_get_flow (flow_table, flow_key); + if (gro_flow) + return gro_flow; + + gro_flow = gro_flow_table_new_flow (flow_table); + + if (gro_flow) + { + gro_flow_set_flow_key (gro_flow, flow_key); + return gro_flow; + } + + return (0); +} + +static_always_inline void +gro_flow_table_reset_flow (gro_flow_table_t * flow_table, + gro_flow_t * gro_flow) +{ + if (PREDICT_TRUE (flow_table->flow_table_size > 0)) + { + clib_memset (gro_flow, 0, sizeof (gro_flow_t)); + flow_table->flow_table_size--; + } +} + +static_always_inline u8 * +gro_flow_table_format (u8 * s, va_list * args) +{ + gro_flow_table_t *flow_table = va_arg (*args, gro_flow_table_t *); + + s = + format (s, + "flow-table: size %u gro-total-vectors %lu gro-n-vectors %u", + flow_table->flow_table_size, flow_table->total_vectors, + flow_table->n_vectors); + if (flow_table->n_vectors) + { + double average_rate = + (double) flow_table->total_vectors / (double) flow_table->n_vectors; + s = format (s, " gro-average-rate %.2f", average_rate); + } + else + s = format (s, " gro-average-rate 0.00"); + + return s; +} +#endif /* included_gro_h */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vnet/gso/gro_func.h b/src/vnet/gso/gro_func.h new file mode 100644 index 00000000000..a410a651933 --- /dev/null +++ b/src/vnet/gso/gro_func.h @@ -0,0 +1,593 @@ +/* + * Copyright (c) 2020 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef included_gro_func_h +#define included_gro_func_h + +#include <vnet/ethernet/ethernet.h> +#include <vnet/gso/gro.h> +#include <vnet/gso/hdr_offset_parser.h> +#include <vnet/udp/udp_packet.h> +#include <vnet/tcp/tcp.h> +#include <vnet/vnet.h> + +static_always_inline u8 +gro_is_bad_packet (vlib_buffer_t * b, u8 flags, i16 l234_sz) +{ + if (((b->current_length - l234_sz) <= 0) || ((flags &= ~TCP_FLAG_ACK) != 0)) + return 1; + return 0; +} + +static_always_inline void +gro_get_ip4_flow_from_packet (u32 * sw_if_index, + ip4_header_t * ip4, tcp_header_t * tcp, + gro_flow_key_t * flow_key, int is_l2) +{ + flow_key->sw_if_index[VLIB_RX] = sw_if_index[VLIB_RX]; + flow_key->sw_if_index[VLIB_TX] = sw_if_index[VLIB_TX]; + ip46_address_set_ip4 (&flow_key->src_address, &ip4->src_address); + ip46_address_set_ip4 (&flow_key->dst_address, &ip4->dst_address); + flow_key->src_port = tcp->src_port; + flow_key->dst_port = tcp->dst_port; +} + +static_always_inline void +gro_get_ip6_flow_from_packet (u32 * sw_if_index, + ip6_header_t * ip6, tcp_header_t * tcp, + gro_flow_key_t * flow_key, int is_l2) +{ + flow_key->sw_if_index[VLIB_RX] = sw_if_index[VLIB_RX]; + flow_key->sw_if_index[VLIB_TX] = sw_if_index[VLIB_TX]; + ip46_address_set_ip6 (&flow_key->src_address, &ip6->src_address); + ip46_address_set_ip6 (&flow_key->dst_address, &ip6->dst_address); + flow_key->src_port = tcp->src_port; + flow_key->dst_port = tcp->dst_port; +} + +static_always_inline u32 +gro_is_ip4_or_ip6_packet (vlib_buffer_t * b0, int is_l2) +{ + if (b0->flags & VNET_BUFFER_F_IS_IP4) + return VNET_BUFFER_F_IS_IP4; + if (b0->flags & VNET_BUFFER_F_IS_IP6) + return VNET_BUFFER_F_IS_IP6; + if (is_l2) + { + ethernet_header_t *eh = + (ethernet_header_t *) vlib_buffer_get_current (b0); + u16 ethertype = clib_net_to_host_u16 (eh->type); + + if (ethernet_frame_is_tagged (ethertype)) + { + ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eh + 1); + + ethertype = clib_net_to_host_u16 (vlan->type); + if (ethertype == ETHERNET_TYPE_VLAN) + { + vlan++; + ethertype = clib_net_to_host_u16 (vlan->type); + } + } + if (ethertype == ETHERNET_TYPE_IP4) + return VNET_BUFFER_F_IS_IP4; + if (ethertype == ETHERNET_TYPE_IP6) + return VNET_BUFFER_F_IS_IP6; + } + else + { + if ((((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0) == 0x40) + return VNET_BUFFER_F_IS_IP4; + if ((((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0) == 0x60) + return VNET_BUFFER_F_IS_IP6; + } + + return 0; +} + +typedef enum +{ + GRO_PACKET_ACTION_NONE = 0, + GRO_PACKET_ACTION_ENQUEUE = 1, + GRO_PACKET_ACTION_FLUSH = 2, +} gro_packet_action_t; + +static_always_inline gro_packet_action_t +gro_tcp_sequence_check (tcp_header_t * tcp0, tcp_header_t * tcp1, + u32 payload_len0) +{ + u32 next_tcp_seq0 = clib_net_to_host_u32 (tcp0->seq_number); + u32 next_tcp_seq1 = clib_net_to_host_u32 (tcp1->seq_number); + + /* next packet, enqueue */ + if (PREDICT_TRUE (next_tcp_seq0 + payload_len0 == next_tcp_seq1)) + return GRO_PACKET_ACTION_ENQUEUE; + /* flush all packets */ + else + return GRO_PACKET_ACTION_FLUSH; +} + +static_always_inline void +gro_merge_buffers (vlib_main_t * vm, vlib_buffer_t * b0, + vlib_buffer_t * b1, u32 bi1, u32 payload_len1, + u16 l234_sz1) +{ + vlib_buffer_t *pb = b0; + + if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_NEXT_PRESENT) == 0)) + b0->total_length_not_including_first_buffer = 0; + + while (pb->flags & VLIB_BUFFER_NEXT_PRESENT) + pb = vlib_get_buffer (vm, pb->next_buffer); + + vlib_buffer_advance (b1, l234_sz1); + pb->flags |= VLIB_BUFFER_NEXT_PRESENT; + pb->next_buffer = bi1; + b0->total_length_not_including_first_buffer += payload_len1; + b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; +} + +static_always_inline u32 +gro_get_packet_data (vlib_main_t * vm, vlib_buffer_t * b0, + generic_header_offset_t * gho0, + gro_flow_key_t * flow_key0, int is_l2) +{ + ip4_header_t *ip4_0 = 0; + ip6_header_t *ip6_0 = 0; + tcp_header_t *tcp0 = 0; + u32 pkt_len0 = 0; + u16 l234_sz0 = 0; + u32 sw_if_index0[VLIB_N_RX_TX] = { ~0 }; + + u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2); + + if (is_ip0 & VNET_BUFFER_F_IS_IP4) + vnet_generic_header_offset_parser (b0, gho0, is_l2, 1 /* is_ip4 */ , + 0 /* is_ip6 */ ); + else if (is_ip0 & VNET_BUFFER_F_IS_IP6) + vnet_generic_header_offset_parser (b0, gho0, is_l2, 0 /* is_ip4 */ , + 1 /* is_ip6 */ ); + else + return 0; + + if (PREDICT_FALSE ((gho0->gho_flags & GHO_F_TCP) == 0)) + return 0; + + ip4_0 = + (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0->l3_hdr_offset); + ip6_0 = + (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0->l3_hdr_offset); + tcp0 = + (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0->l4_hdr_offset); + + l234_sz0 = gho0->hdr_sz; + if (PREDICT_FALSE (gro_is_bad_packet (b0, tcp0->flags, l234_sz0))) + return 0; + + sw_if_index0[VLIB_RX] = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + sw_if_index0[VLIB_TX] = vnet_buffer (b0)->sw_if_index[VLIB_TX]; + + if (gho0->gho_flags & GHO_F_IP4) + { + gro_get_ip4_flow_from_packet (sw_if_index0, ip4_0, tcp0, flow_key0, + is_l2); + } + else if (gho0->gho_flags & GHO_F_IP6) + { + gro_get_ip6_flow_from_packet (sw_if_index0, ip6_0, tcp0, flow_key0, + is_l2); + } + else + return 0; + + pkt_len0 = vlib_buffer_length_in_chain (vm, b0); + if (PREDICT_FALSE (pkt_len0 >= TCP_MAX_GSO_SZ)) + return 0; + + return pkt_len0; +} + +static_always_inline u32 +gro_coalesce_buffers (vlib_main_t * vm, vlib_buffer_t * b0, + vlib_buffer_t * b1, u32 bi1, int is_l2) +{ + generic_header_offset_t gho0 = { 0 }; + generic_header_offset_t gho1 = { 0 }; + gro_flow_key_t flow_key0, flow_key1; + ip4_header_t *ip4_0, *ip4_1; + ip6_header_t *ip6_0, *ip6_1; + tcp_header_t *tcp0, *tcp1; + u16 l234_sz0, l234_sz1; + u32 pkt_len0, pkt_len1, payload_len0, payload_len1; + u32 sw_if_index0[VLIB_N_RX_TX] = { ~0 }; + u32 sw_if_index1[VLIB_N_RX_TX] = { ~0 }; + + u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2); + u32 is_ip1 = gro_is_ip4_or_ip6_packet (b1, is_l2); + + if (is_ip0 & VNET_BUFFER_F_IS_IP4) + vnet_generic_header_offset_parser (b0, &gho0, is_l2, 1 /* is_ip4 */ , + 0 /* is_ip6 */ ); + else if (is_ip0 & VNET_BUFFER_F_IS_IP6) + vnet_generic_header_offset_parser (b0, &gho0, is_l2, 0 /* is_ip4 */ , + 1 /* is_ip6 */ ); + else + return 0; + + if (is_ip1 & VNET_BUFFER_F_IS_IP4) + vnet_generic_header_offset_parser (b1, &gho1, is_l2, 1 /* is_ip4 */ , + 0 /* is_ip6 */ ); + else if (is_ip1 & VNET_BUFFER_F_IS_IP6) + vnet_generic_header_offset_parser (b1, &gho1, is_l2, 0 /* is_ip4 */ , + 1 /* is_ip6 */ ); + else + return 0; + + pkt_len0 = vlib_buffer_length_in_chain (vm, b0); + pkt_len1 = vlib_buffer_length_in_chain (vm, b1); + + if (((gho0.gho_flags & GHO_F_TCP) == 0) + || ((gho1.gho_flags & GHO_F_TCP) == 0)) + return 0; + + ip4_0 = + (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset); + ip4_1 = + (ip4_header_t *) (vlib_buffer_get_current (b1) + gho1.l3_hdr_offset); + ip6_0 = + (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset); + ip6_1 = + (ip6_header_t *) (vlib_buffer_get_current (b1) + gho1.l3_hdr_offset); + + tcp0 = (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset); + tcp1 = (tcp_header_t *) (vlib_buffer_get_current (b1) + gho1.l4_hdr_offset); + + l234_sz0 = gho0.hdr_sz; + l234_sz1 = gho1.hdr_sz; + + if (gro_is_bad_packet (b0, tcp0->flags, l234_sz0) + || gro_is_bad_packet (b1, tcp1->flags, l234_sz1)) + return 0; + + sw_if_index0[VLIB_RX] = vnet_buffer (b0)->sw_if_index[VLIB_RX]; + sw_if_index0[VLIB_TX] = vnet_buffer (b0)->sw_if_index[VLIB_TX]; + + sw_if_index1[VLIB_RX] = vnet_buffer (b1)->sw_if_index[VLIB_RX]; + sw_if_index1[VLIB_TX] = vnet_buffer (b1)->sw_if_index[VLIB_TX]; + + if ((gho0.gho_flags & GHO_F_IP4) && (gho1.gho_flags & GHO_F_IP4)) + { + gro_get_ip4_flow_from_packet (sw_if_index0, ip4_0, tcp0, &flow_key0, + is_l2); + gro_get_ip4_flow_from_packet (sw_if_index1, ip4_1, tcp1, &flow_key1, + is_l2); + } + else if ((gho0.gho_flags & GHO_F_IP6) && (gho1.gho_flags & GHO_F_IP6)) + { + gro_get_ip6_flow_from_packet (sw_if_index0, ip6_0, tcp0, &flow_key0, + is_l2); + gro_get_ip6_flow_from_packet (sw_if_index1, ip6_1, tcp1, &flow_key1, + is_l2); + } + else + return 0; + + if (gro_flow_is_equal (&flow_key0, &flow_key1) == 0) + return 0; + + payload_len0 = pkt_len0 - l234_sz0; + payload_len1 = pkt_len1 - l234_sz1; + + if (pkt_len0 >= TCP_MAX_GSO_SZ || pkt_len1 >= TCP_MAX_GSO_SZ + || (pkt_len0 + payload_len1) >= TCP_MAX_GSO_SZ) + return 0; + + if (gro_tcp_sequence_check (tcp0, tcp1, payload_len0) == + GRO_PACKET_ACTION_ENQUEUE) + { + gro_merge_buffers (vm, b0, b1, bi1, payload_len1, l234_sz1); + return tcp1->ack_number; + } + + return 0; +} + +static_always_inline void +gro_fixup_header (vlib_main_t * vm, vlib_buffer_t * b0, u32 ack_number, + int is_l2) +{ + generic_header_offset_t gho0 = { 0 }; + + u32 is_ip0 = gro_is_ip4_or_ip6_packet (b0, is_l2); + + if (is_ip0 & VNET_BUFFER_F_IS_IP4) + vnet_generic_header_offset_parser (b0, &gho0, is_l2, 1 /* is_ip4 */ , + 0 /* is_ip6 */ ); + else if (is_ip0 & VNET_BUFFER_F_IS_IP6) + vnet_generic_header_offset_parser (b0, &gho0, is_l2, 0 /* is_ip4 */ , + 1 /* is_ip6 */ ); + + vnet_buffer2 (b0)->gso_size = b0->current_length - gho0.hdr_sz; + + if (gho0.gho_flags & GHO_F_IP4) + { + ip4_header_t *ip4 = + (ip4_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset); + ip4->length = + clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - + gho0.l3_hdr_offset); + b0->flags |= + (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4 | + VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_IP_CKSUM); + } + else if (gho0.gho_flags & GHO_F_IP6) + { + ip6_header_t *ip6 = + (ip6_header_t *) (vlib_buffer_get_current (b0) + gho0.l3_hdr_offset); + ip6->payload_length = + clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - + gho0.l4_hdr_offset); + b0->flags |= + (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6 | + VNET_BUFFER_F_OFFLOAD_TCP_CKSUM); + } + + tcp_header_t *tcp0 = + (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset); + tcp0->ack_number = ack_number; + b0->flags &= ~VLIB_BUFFER_IS_TRACED; +} + +static_always_inline u32 +vnet_gro_flow_table_flush (vlib_main_t * vm, gro_flow_table_t * flow_table, + u32 * to) +{ + if (flow_table->flow_table_size > 0) + { + gro_flow_t *gro_flow; + u32 i = 0, j = 0; + while (i < GRO_FLOW_TABLE_MAX_SIZE) + { + gro_flow = &flow_table->gro_flow[i]; + if (gro_flow->n_buffers && gro_flow_is_timeout (vm, gro_flow)) + { + // flush the packet + vlib_buffer_t *b0 = + vlib_get_buffer (vm, gro_flow->buffer_index); + gro_fixup_header (vm, b0, gro_flow->last_ack_number, + flow_table->is_l2); + to[j] = gro_flow->buffer_index; + gro_flow_table_reset_flow (flow_table, gro_flow); + flow_table->n_vectors++; + j++; + } + i++; + } + + return j; + } + return 0; +} + +static_always_inline void +vnet_gro_flow_table_schedule_node_on_dispatcher (vlib_main_t * vm, + gro_flow_table_t * + flow_table) +{ + if (gro_flow_table_is_timeout (vm, flow_table)) + { + u32 to[GRO_FLOW_TABLE_MAX_SIZE] = { 0 }; + u32 n_to = vnet_gro_flow_table_flush (vm, flow_table, to); + + if (n_to > 0) + { + u32 node_index = flow_table->node_index; + vlib_frame_t *f = vlib_get_frame_to_node (vm, node_index); + u32 *f_to = vlib_frame_vector_args (f); + u32 i = 0; + + while (i < n_to) + { + f_to[f->n_vectors] = to[i]; + i++; + f->n_vectors++; + } + vlib_put_frame_to_node (vm, node_index, f); + } + gro_flow_table_set_timeout (vm, flow_table, GRO_FLOW_TABLE_FLUSH); + } +} + +static_always_inline u32 +vnet_gro_flow_table_inline (vlib_main_t * vm, gro_flow_table_t * flow_table, + u32 bi0, u32 * to) +{ + vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); + generic_header_offset_t gho0 = { 0 }; + gro_flow_t *gro_flow = 0; + gro_flow_key_t flow_key0 = { }; + tcp_header_t *tcp0 = 0; + u32 pkt_len0 = 0; + int is_l2 = flow_table->is_l2; + + if (!gro_flow_table_is_enable (flow_table)) + { + to[0] = bi0; + return 1; + } + + if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_GSO)) + { + to[0] = bi0; + return 1; + } + + pkt_len0 = gro_get_packet_data (vm, b0, &gho0, &flow_key0, is_l2); + if (pkt_len0 == 0) + { + to[0] = bi0; + return 1; + } + + gro_flow = gro_flow_table_find_or_add_flow (flow_table, &flow_key0); + if (!gro_flow) + { + to[0] = bi0; + return 1; + } + + if (PREDICT_FALSE (gro_flow->n_buffers == 0)) + { + flow_table->total_vectors++; + gro_flow_store_packet (gro_flow, bi0); + tcp0 = + (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset); + gro_flow->last_ack_number = tcp0->ack_number; + gro_flow_set_timeout (vm, gro_flow, GRO_FLOW_TIMEOUT); + return 0; + } + else + { + tcp0 = + (tcp_header_t *) (vlib_buffer_get_current (b0) + gho0.l4_hdr_offset); + generic_header_offset_t gho_s = { 0 }; + tcp_header_t *tcp_s; + u16 l234_sz0, l234_sz_s; + u32 pkt_len_s, payload_len0, payload_len_s; + u32 bi_s = gro_flow->buffer_index; + + vlib_buffer_t *b_s = vlib_get_buffer (vm, bi_s); + u32 is_ip_s = gro_is_ip4_or_ip6_packet (b_s, is_l2); + if (is_ip_s & VNET_BUFFER_F_IS_IP4) + vnet_generic_header_offset_parser (b_s, &gho_s, is_l2, + 1 /* is_ip4 */ , 0 /* is_ip6 */ ); + else if (is_ip_s & VNET_BUFFER_F_IS_IP6) + vnet_generic_header_offset_parser (b_s, &gho_s, is_l2, + 0 /* is_ip4 */ , 1 /* is_ip6 */ ); + + tcp_s = + (tcp_header_t *) (vlib_buffer_get_current (b_s) + + gho_s.l4_hdr_offset); + pkt_len_s = vlib_buffer_length_in_chain (vm, b_s); + l234_sz0 = gho0.hdr_sz; + l234_sz_s = gho_s.hdr_sz; + payload_len0 = pkt_len0 - l234_sz0; + payload_len_s = pkt_len_s - l234_sz_s; + gro_packet_action_t action = + gro_tcp_sequence_check (tcp_s, tcp0, payload_len_s); + + if (PREDICT_TRUE (action == GRO_PACKET_ACTION_ENQUEUE)) + { + if (PREDICT_TRUE ((pkt_len_s + payload_len0) < TCP_MAX_GSO_SZ)) + { + flow_table->total_vectors++; + gro_merge_buffers (vm, b_s, b0, bi0, payload_len0, l234_sz0); + gro_flow_store_packet (gro_flow, bi0); + gro_flow->last_ack_number = tcp0->ack_number; + return 0; + } + else + { + // flush the stored GSO size packet and buffer the current packet + flow_table->n_vectors++; + flow_table->total_vectors++; + gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2); + gro_flow->n_buffers = 0; + gro_flow_store_packet (gro_flow, bi0); + gro_flow->last_ack_number = tcp0->ack_number; + gro_flow_set_timeout (vm, gro_flow, GRO_FLOW_TIMEOUT); + to[0] = bi_s; + return 1; + } + } + else + { + // flush the all (current and stored) packets + flow_table->n_vectors++; + flow_table->total_vectors++; + gro_fixup_header (vm, b_s, gro_flow->last_ack_number, is_l2); + gro_flow->n_buffers = 0; + gro_flow_table_reset_flow (flow_table, gro_flow); + to[0] = bi_s; + to[1] = bi0; + return 2; + } + } +} + +/** + * coalesce buffers with flow tables + */ +static_always_inline u32 +vnet_gro_inline (vlib_main_t * vm, gro_flow_table_t * flow_table, u32 * from, + u16 n_left_from, u32 * to) +{ + u16 count = 0, i = 0; + + for (i = 0; i < n_left_from; i++) + count += vnet_gro_flow_table_inline (vm, flow_table, from[i], &to[count]); + + return count; +} + +/** + * coalesce buffers in opportunistic way without flow tables + */ +static_always_inline u32 +vnet_gro_simple_inline (vlib_main_t * vm, u32 * from, u16 n_left_from, + int is_l2) +{ + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; + vlib_get_buffers (vm, from, b, n_left_from); + u32 bi = 1, ack_number = 0; + if (PREDICT_TRUE (((b[0]->flags & VNET_BUFFER_F_GSO) == 0))) + { + while (n_left_from > 1) + { + if (PREDICT_TRUE (((b[bi]->flags & VNET_BUFFER_F_GSO) == 0))) + { + u32 ret; + if ((ret = + gro_coalesce_buffers (vm, b[0], b[bi], from[bi], + is_l2)) != 0) + { + n_left_from -= 1; + bi += 1; + ack_number = ret; + continue; + } + else + break; + } + else + break; + } + + if (bi >= 2) + { + gro_fixup_header (vm, b[0], ack_number, is_l2); + } + } + return bi; +} +#endif /* included_gro_func_h */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vnet/pg/cli.c b/src/vnet/pg/cli.c index b3aaddfda2b..cb8b5bbb287 100644 --- a/src/vnet/pg/cli.c +++ b/src/vnet/pg/cli.c @@ -661,7 +661,7 @@ create_pg_if_cmd_fn (vlib_main_t * vm, { pg_main_t *pg = &pg_main; unformat_input_t _line_input, *line_input = &_line_input; - u32 if_id, gso_enabled = 0, gso_size = 0; + u32 if_id, gso_enabled = 0, gso_size = 0, coalesce_enabled = 0; clib_error_t *error = NULL; if (!unformat_user (input, unformat_line_input, line_input)) @@ -681,6 +681,8 @@ create_pg_if_cmd_fn (vlib_main_t * vm, error = clib_error_create ("gso enabled but gso size missing"); goto done; } + if (unformat (line_input, "coalesce-enabled")) + coalesce_enabled = 1; } else { @@ -690,7 +692,8 @@ create_pg_if_cmd_fn (vlib_main_t * vm, } } - pg_interface_add_or_get (pg, if_id, gso_enabled, gso_size); + pg_interface_add_or_get (pg, if_id, gso_enabled, gso_size, + coalesce_enabled); done: unformat_free (line_input); @@ -701,7 +704,8 @@ done: /* *INDENT-OFF* */ VLIB_CLI_COMMAND (create_pg_if_cmd, static) = { .path = "create packet-generator", - .short_help = "create packet-generator interface <interface name> [gso-enabled gso-size <size>]", + .short_help = "create packet-generator interface <interface name>" + " [gso-enabled gso-size <size> [coalesce-enabled]]", .function = create_pg_if_cmd_fn, }; /* *INDENT-ON* */ diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c index 6968cce19e7..785592f3618 100644 --- a/src/vnet/pg/input.c +++ b/src/vnet/pg/input.c @@ -54,6 +54,7 @@ #include <vnet/ip/ip6_packet.h> #include <vnet/udp/udp_packet.h> #include <vnet/devices/devices.h> +#include <vnet/gso/gro_func.h> static int validate_buffer_data2 (vlib_buffer_t * b, pg_stream_t * s, @@ -1640,6 +1641,9 @@ pg_generate_packets (vlib_node_runtime_t * node, &next_index, 0); } + if (PREDICT_FALSE (pi->coalesce_enabled)) + vnet_gro_flow_table_schedule_node_on_dispatcher (vm, pi->flow_table); + while (n_packets_to_generate > 0) { u32 *head, *start, *end; diff --git a/src/vnet/pg/output.c b/src/vnet/pg/output.c index d8059fab186..042591a7709 100644 --- a/src/vnet/pg/output.c +++ b/src/vnet/pg/output.c @@ -42,6 +42,7 @@ #include <vnet/vnet.h> #include <vnet/pg/pg.h> #include <vnet/ethernet/ethernet.h> +#include <vnet/gso/gro_func.h> uword pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) @@ -50,6 +51,8 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) u32 *buffers = vlib_frame_vector_args (frame); uword n_buffers = frame->n_vectors; uword n_left = n_buffers; + u32 to[GRO_TO_VECTOR_SIZE (n_buffers)]; + uword n_to = 0; vnet_interface_output_runtime_t *rd = (void *) node->runtime_data; pg_interface_t *pif = pool_elt_at_index (pg->interfaces, rd->dev_instance); @@ -57,6 +60,13 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) while (clib_atomic_test_and_set (pif->lockp)) ; + if (PREDICT_FALSE (pif->coalesce_enabled)) + { + n_to = vnet_gro_inline (vm, pif->flow_table, buffers, n_left, to); + buffers = to; + n_left = n_to; + } + while (n_left > 0) { n_left--; @@ -84,7 +94,13 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) pif->pcap_main.n_packets_to_capture) pcap_close (&pif->pcap_main); - vlib_buffer_free (vm, vlib_frame_vector_args (frame), n_buffers); + if (PREDICT_FALSE (pif->coalesce_enabled)) + { + n_buffers = n_to; + vlib_buffer_free (vm, to, n_to); + } + else + vlib_buffer_free (vm, vlib_frame_vector_args (frame), n_buffers); if (PREDICT_FALSE (pif->lockp != 0)) clib_atomic_release (pif->lockp); diff --git a/src/vnet/pg/pg.api b/src/vnet/pg/pg.api index 86343d5c8df..3a44f1d87a7 100644 --- a/src/vnet/pg/pg.api +++ b/src/vnet/pg/pg.api @@ -49,6 +49,21 @@ define pg_create_interface_reply vl_api_interface_index_t sw_if_index; }; +/** \brief PacketGenerator interface enable/disable packet coalesce + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param interface_id - interface index + @param coalesce_enabled - enable/disable packet coalesce on this interface +*/ +autoreply define pg_interface_enable_disable_coalesce +{ + u32 client_index; + u32 context; + vl_api_interface_index_t sw_if_index; + bool coalesce_enabled; + option status="in_progress"; +}; + /** \brief PacketGenerator capture packets on given interface request @param client_index - opaque cookie to identify the sender @param context - sender context, to match reply w/ request diff --git a/src/vnet/pg/pg.h b/src/vnet/pg/pg.h index a6616d9ced7..06e61261b7d 100644 --- a/src/vnet/pg/pg.h +++ b/src/vnet/pg/pg.h @@ -45,6 +45,7 @@ #include <vppinfra/fifo.h> /* for buffer_fifo */ #include <vppinfra/pcap.h> #include <vnet/interface.h> +#include <vnet/gso/gro.h> extern vnet_device_class_t pg_dev_class; @@ -305,6 +306,8 @@ typedef struct /* Identifies stream for this interface. */ u32 id; + u8 coalesce_enabled; + gro_flow_table_t *flow_table; u8 gso_enabled; u32 gso_size; pcap_main_t pcap_main; @@ -358,9 +361,14 @@ void pg_stream_change (pg_main_t * pg, pg_stream_t * s); void pg_stream_enable_disable (pg_main_t * pg, pg_stream_t * s, int is_enable); +/* Enable/disable packet coalesce on given interface */ +void pg_interface_enable_disable_coalesce (pg_interface_t * pi, u8 enable, + u32 tx_node_index); + /* Find/create free packet-generator interface index. */ u32 pg_interface_add_or_get (pg_main_t * pg, uword stream_index, - u8 gso_enabled, u32 gso_size); + u8 gso_enabled, u32 gso_size, + u8 coalesce_enabled); always_inline pg_node_t * pg_get_node (uword node_index) diff --git a/src/vnet/pg/pg_api.c b/src/vnet/pg/pg_api.c index bb58a4f0cec..554e8ea31c1 100644 --- a/src/vnet/pg/pg_api.c +++ b/src/vnet/pg/pg_api.c @@ -44,7 +44,8 @@ #define foreach_pg_api_msg \ _(PG_CREATE_INTERFACE, pg_create_interface) \ _(PG_CAPTURE, pg_capture) \ -_(PG_ENABLE_DISABLE, pg_enable_disable) +_(PG_ENABLE_DISABLE, pg_enable_disable) \ +_(PG_INTERFACE_ENABLE_DISABLE_COALESCE, pg_interface_enable_disable_coalesce) static void vl_api_pg_create_interface_t_handler (vl_api_pg_create_interface_t * mp) @@ -55,7 +56,7 @@ vl_api_pg_create_interface_t_handler (vl_api_pg_create_interface_t * mp) pg_main_t *pg = &pg_main; u32 pg_if_id = pg_interface_add_or_get (pg, ntohl (mp->interface_id), mp->gso_enabled, - ntohl (mp->gso_size)); + ntohl (mp->gso_size), 0); pg_interface_t *pi = pool_elt_at_index (pg->interfaces, pg_if_id); /* *INDENT-OFF* */ @@ -67,6 +68,41 @@ vl_api_pg_create_interface_t_handler (vl_api_pg_create_interface_t * mp) } static void + vl_api_pg_interface_enable_disable_coalesce_t_handler + (vl_api_pg_interface_enable_disable_coalesce_t * mp) +{ + vl_api_pg_interface_enable_disable_coalesce_reply_t *rmp; + int rv = 0; + + VALIDATE_SW_IF_INDEX (mp); + + u32 sw_if_index = ntohl (mp->sw_if_index); + + pg_main_t *pg = &pg_main; + vnet_main_t *vnm = vnet_get_main (); + vnet_hw_interface_t *hw = + vnet_get_sup_hw_interface_api_visible_or_null (vnm, sw_if_index); + + if (hw) + { + pg_interface_t *pi = + pool_elt_at_index (pg->interfaces, hw->dev_instance); + if (pi->gso_enabled) + pg_interface_enable_disable_coalesce (pi, mp->coalesce_enabled, + hw->tx_node_index); + else + rv = VNET_API_ERROR_CANNOT_ENABLE_DISABLE_FEATURE; + } + else + { + rv = VNET_API_ERROR_NO_MATCHING_INTERFACE; + } + + BAD_SW_IF_INDEX_LABEL; + REPLY_MACRO (VL_API_PG_INTERFACE_ENABLE_DISABLE_COALESCE_REPLY); +} + +static void vl_api_pg_capture_t_handler (vl_api_pg_capture_t * mp) { vl_api_pg_capture_reply_t *rmp; diff --git a/src/vnet/pg/stream.c b/src/vnet/pg/stream.c index f09e9a44398..88c89371c6c 100644 --- a/src/vnet/pg/stream.c +++ b/src/vnet/pg/stream.c @@ -178,9 +178,26 @@ pg_eth_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hi, u32 flags) return 0; } +void +pg_interface_enable_disable_coalesce (pg_interface_t * pi, u8 enable, + u32 tx_node_index) +{ + if (enable) + { + gro_flow_table_init (&pi->flow_table, 1 /* is_l2 */ , + tx_node_index); + pi->coalesce_enabled = 1; + } + else + { + pi->coalesce_enabled = 0; + gro_flow_table_free (pi->flow_table); + } +} + u32 pg_interface_add_or_get (pg_main_t * pg, uword if_id, u8 gso_enabled, - u32 gso_size) + u32 gso_size, u8 coalesce_enabled) { vnet_main_t *vnm = vnet_get_main (); vlib_main_t *vm = vlib_get_main (); @@ -219,6 +236,10 @@ pg_interface_add_or_get (pg_main_t * pg, uword if_id, u8 gso_enabled, hi->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO; pi->gso_enabled = 1; pi->gso_size = gso_size; + if (coalesce_enabled) + { + pg_interface_enable_disable_coalesce (pi, 1, hi->tx_node_index); + } } pi->sw_if_index = hi->sw_if_index; @@ -454,7 +475,7 @@ pg_stream_add (pg_main_t * pg, pg_stream_t * s_init) /* Find an interface to use. */ s->pg_if_index = pg_interface_add_or_get (pg, s->if_id, 0 /* gso_enabled */ , - 0 /* gso_size */ ); + 0 /* gso_size */ , 0 /* coalesce_enabled */ ); if (s->sw_if_index[VLIB_RX] == ~0) { |