diff options
Diffstat (limited to 'src/vnet/gso/gso.h')
| -rw-r--r-- | src/vnet/gso/gso.h | 273 |
1 files changed, 273 insertions, 0 deletions
diff --git a/src/vnet/gso/gso.h b/src/vnet/gso/gso.h index 8e174dfd1f6..dee5da5c70b 100644 --- a/src/vnet/gso/gso.h +++ b/src/vnet/gso/gso.h @@ -17,6 +17,8 @@ #define included_gso_h #include <vnet/vnet.h> +#include <vnet/gso/hdr_offset_parser.h> +#include <vnet/ip/ip_psh_cksum.h> typedef struct { @@ -28,6 +30,277 @@ typedef struct extern gso_main_t gso_main; int vnet_sw_interface_gso_enable_disable (u32 sw_if_index, u8 enable); +u32 gso_segment_buffer (vlib_main_t *vm, vnet_interface_per_thread_data_t *ptd, + u32 bi, vlib_buffer_t *b, generic_header_offset_t *gho, + u32 n_bytes_b, u8 is_l2, u8 is_ip6); + +static_always_inline void +gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0, + u32 flags, u16 n_bufs, u16 hdr_sz) +{ + u32 i = n_bufs; + while (i >= 6) + { + /* prefetches */ + CLIB_PREFETCH (bufs[2], 2 * CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (bufs[3], 2 * CLIB_CACHE_LINE_BYTES, LOAD); + vlib_prefetch_buffer_data (bufs[4], LOAD); + vlib_prefetch_buffer_data (bufs[5], LOAD); + + /* copying objects from cacheline 0 */ + bufs[0]->current_data = 0; + bufs[1]->current_data = 0; + + bufs[0]->current_length = hdr_sz; + bufs[1]->current_length = hdr_sz; + + bufs[0]->flags = bufs[1]->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags; + bufs[0]->flow_id = bufs[1]->flow_id = b0->flow_id; + bufs[0]->error = bufs[1]->error = b0->error; + bufs[0]->current_config_index = bufs[1]->current_config_index = + b0->current_config_index; + + clib_memcpy_fast (&bufs[0]->opaque, &b0->opaque, sizeof (b0->opaque)); + clib_memcpy_fast (&bufs[1]->opaque, &b0->opaque, sizeof (b0->opaque)); + + /* copying objects from cacheline 1 */ + bufs[0]->trace_handle = b0->trace_handle; + bufs[1]->trace_handle = b0->trace_handle; + + bufs[0]->total_length_not_including_first_buffer = 0; + bufs[1]->total_length_not_including_first_buffer = 0; + + clib_memcpy_fast (&bufs[0]->opaque2, &b0->opaque2, sizeof (b0->opaque2)); + clib_memcpy_fast (&bufs[1]->opaque2, &b0->opaque2, sizeof (b0->opaque2)); + + /* copying data */ + clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz); + clib_memcpy_fast (bufs[1]->data, vlib_buffer_get_current (b0), hdr_sz); + + /* header offset fixup */ + vnet_buffer (bufs[0])->l2_hdr_offset -= b0->current_data; + vnet_buffer (bufs[0])->l3_hdr_offset -= b0->current_data; + vnet_buffer (bufs[0])->l4_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[0])->outer_l3_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[0])->outer_l4_hdr_offset -= b0->current_data; + + vnet_buffer (bufs[1])->l2_hdr_offset -= b0->current_data; + vnet_buffer (bufs[1])->l3_hdr_offset -= b0->current_data; + vnet_buffer (bufs[1])->l4_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[1])->outer_l3_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[1])->outer_l4_hdr_offset -= b0->current_data; + + bufs += 2; + i -= 2; + } + + while (i > 0) + { + /* copying objects from cacheline 0 */ + bufs[0]->current_data = 0; + bufs[0]->current_length = hdr_sz; + bufs[0]->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags; + bufs[0]->flow_id = b0->flow_id; + bufs[0]->error = b0->error; + bufs[0]->current_config_index = b0->current_config_index; + clib_memcpy_fast (&bufs[0]->opaque, &b0->opaque, sizeof (b0->opaque)); + + /* copying objects from cacheline 1 */ + bufs[0]->trace_handle = b0->trace_handle; + bufs[0]->total_length_not_including_first_buffer = 0; + clib_memcpy_fast (&bufs[0]->opaque2, &b0->opaque2, sizeof (b0->opaque2)); + + /* copying data */ + clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz); + + /* header offset fixup */ + vnet_buffer (bufs[0])->l2_hdr_offset -= b0->current_data; + vnet_buffer (bufs[0])->l3_hdr_offset -= b0->current_data; + vnet_buffer (bufs[0])->l4_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[0])->outer_l3_hdr_offset -= b0->current_data; + vnet_buffer2 (bufs[0])->outer_l4_hdr_offset -= b0->current_data; + + bufs++; + i--; + } +} + +static_always_inline void +gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq, + int is_l2, u8 oflags, u16 hdr_sz, u16 l4_hdr_sz, + clib_ip_csum_t *c, u8 tcp_flags, u8 is_prefetch, + vlib_buffer_t *b1) +{ + + i16 l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset; + i16 l4_hdr_offset = vnet_buffer (b0)->l4_hdr_offset; + + ip4_header_t *ip4 = (ip4_header_t *) (b0->data + l3_hdr_offset); + ip6_header_t *ip6 = (ip6_header_t *) (b0->data + l3_hdr_offset); + tcp_header_t *tcp = (tcp_header_t *) (b0->data + l4_hdr_offset); + + tcp->flags = tcp_flags; + tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq); + c->odd = 0; + + if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM) + { + ip4->length = + clib_host_to_net_u16 (b0->current_length - hdr_sz + + (l4_hdr_offset - l3_hdr_offset) + l4_hdr_sz); + ip4->checksum = 0; + ip4->checksum = ip4_header_checksum (ip4); + vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM | + VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)); + c->sum += clib_mem_unaligned (&ip4->src_address, u32); + c->sum += clib_mem_unaligned (&ip4->dst_address, u32); + c->sum += clib_host_to_net_u32 ( + (clib_net_to_host_u16 (ip4->length) - ip4_header_bytes (ip4)) + + (ip4->protocol << 16)); + } + else + { + ip6->payload_length = + clib_host_to_net_u16 (b0->current_length - hdr_sz + l4_hdr_sz); + vnet_buffer_offload_flags_clear (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM); + ip6_psh_t psh = { 0 }; + u32 *p = (u32 *) &psh; + psh.src = ip6->src_address; + psh.dst = ip6->dst_address; + psh.l4len = ip6->payload_length; + psh.proto = clib_host_to_net_u32 ((u32) ip6->protocol); + for (int i = 0; i < 10; i++) + c->sum += p[i]; + } + + if (is_prefetch) + CLIB_PREFETCH (vlib_buffer_get_current (b1) + hdr_sz, + CLIB_CACHE_LINE_BYTES, LOAD); + + clib_ip_csum_chunk (c, (u8 *) tcp, l4_hdr_sz); + tcp->checksum = clib_ip_csum_fold (c); + + if (!is_l2 && ((oflags & VNET_BUFFER_OFFLOAD_F_TNL_MASK) == 0)) + { + u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX]; + + ip_adjacency_t *adj0 = adj_get (adj_index0); + + if (adj0->lookup_next_index == IP_LOOKUP_NEXT_MIDCHAIN && + adj0->sub_type.midchain.fixup_func) + /* calls e.g. ipip44_fixup */ + adj0->sub_type.midchain.fixup_func ( + vm, adj0, b0, adj0->sub_type.midchain.fixup_data); + } +} + +static_always_inline u32 +gso_segment_buffer_inline (vlib_main_t *vm, + vnet_interface_per_thread_data_t *ptd, + vlib_buffer_t *b, int is_l2) +{ + vlib_buffer_t **bufs = 0; + u32 n_tx_bytes = 0; + + u8 oflags = vnet_buffer (b)->oflags; + i16 l4_hdr_offset = vnet_buffer (b)->l4_hdr_offset; + u16 gso_size = vnet_buffer2 (b)->gso_size; + u16 l4_hdr_sz = vnet_buffer2 (b)->gso_l4_hdr_sz; + + u8 tcp_flags = 0, tcp_flags_no_fin_psh = 0; + u32 default_bflags = + b->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT); + u16 hdr_sz = (l4_hdr_offset - b->current_data) + l4_hdr_sz; + u32 next_tcp_seq = 0, tcp_seq = 0; + u32 data_size = vlib_buffer_length_in_chain (vm, b) - hdr_sz; + u16 size = + clib_min (gso_size, vlib_buffer_get_default_data_size (vm) - hdr_sz); + u16 n_alloc = 0, n_bufs = ((data_size + size - 1) / size); + clib_ip_csum_t c = { .sum = 0, .odd = 0 }; + u8 *src_ptr, *dst_ptr; + u16 src_left, dst_left, bytes_to_copy; + u32 i = 0; + + vec_validate (ptd->split_buffers, n_bufs - 1); + n_alloc = vlib_buffer_alloc (vm, ptd->split_buffers, n_bufs); + if (n_alloc < n_bufs) + { + vlib_buffer_free (vm, ptd->split_buffers, n_alloc); + return 0; + } + + vec_validate (bufs, n_bufs - 1); + vlib_get_buffers (vm, ptd->split_buffers, bufs, n_bufs); + + tcp_header_t *tcp = (tcp_header_t *) (b->data + l4_hdr_offset); + + tcp_seq = next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number); + /* store original flags for last packet and reset FIN and PSH */ + tcp_flags = tcp->flags; + tcp_flags_no_fin_psh = tcp->flags & ~(TCP_FLAG_FIN | TCP_FLAG_PSH); + tcp->checksum = 0; + + gso_init_bufs_from_template_base (bufs, b, default_bflags, n_bufs, hdr_sz); + + src_ptr = vlib_buffer_get_current (b) + hdr_sz; + src_left = b->current_length - hdr_sz; + dst_ptr = vlib_buffer_get_current (bufs[i]) + hdr_sz; + dst_left = size; + + while (data_size) + { + bytes_to_copy = clib_min (src_left, dst_left); + clib_ip_csum_and_copy_chunk (&c, src_ptr, dst_ptr, bytes_to_copy); + + src_left -= bytes_to_copy; + src_ptr += bytes_to_copy; + data_size -= bytes_to_copy; + dst_left -= bytes_to_copy; + dst_ptr += bytes_to_copy; + next_tcp_seq += bytes_to_copy; + bufs[i]->current_length += bytes_to_copy; + + if (0 == src_left) + { + /* init src to the next buffer in chain */ + if (b->flags & VLIB_BUFFER_NEXT_PRESENT) + { + b = vlib_get_buffer (vm, b->next_buffer); + src_left = b->current_length; + src_ptr = vlib_buffer_get_current (b); + } + else + { + ASSERT (data_size == 0); + break; + } + } + if (0 == dst_left && data_size) + { + vlib_prefetch_buffer_header (bufs[i + 1], LOAD); + + n_tx_bytes += bufs[i]->current_length; + gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, oflags, hdr_sz, + l4_hdr_sz, &c, tcp_flags_no_fin_psh, 1, + bufs[i + 1]); + i++; + dst_left = size; + dst_ptr = vlib_buffer_get_current (bufs[i]) + hdr_sz; + tcp_seq = next_tcp_seq; + // reset clib_ip_csum_t + c.odd = 0; + c.sum = 0; + } + } + + ASSERT ((i + 1) == n_alloc); + n_tx_bytes += bufs[i]->current_length; + gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, oflags, hdr_sz, + l4_hdr_sz, &c, tcp_flags, 0, NULL); + + vec_free (bufs); + return n_tx_bytes; +} #endif /* included_gso_h */ |