summaryrefslogtreecommitdiffstats
path: root/src/vnet/gso
diff options
context:
space:
mode:
Diffstat (limited to 'src/vnet/gso')
-rw-r--r--src/vnet/gso/gso.h237
-rw-r--r--src/vnet/gso/node.c17
2 files changed, 239 insertions, 15 deletions
diff --git a/src/vnet/gso/gso.h b/src/vnet/gso/gso.h
index 926ce634fd0..041fab3bcc4 100644
--- a/src/vnet/gso/gso.h
+++ b/src/vnet/gso/gso.h
@@ -18,6 +18,7 @@
#include <vnet/vnet.h>
#include <vnet/gso/hdr_offset_parser.h>
+#include <vnet/ip/ip_psh_cksum.h>
typedef struct
{
@@ -33,6 +34,242 @@ u32 gso_segment_buffer (vlib_main_t *vm, vnet_interface_per_thread_data_t *ptd,
u32 bi, vlib_buffer_t *b, generic_header_offset_t *gho,
u32 n_bytes_b, u8 is_l2, u8 is_ip6);
+static_always_inline void
+gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0,
+ u32 flags, u16 n_bufs, u16 hdr_sz)
+{
+ u32 i = n_bufs;
+ while (i >= 4)
+ {
+ /* prefetches */
+ CLIB_PREFETCH (bufs[2], 2 * CLIB_CACHE_LINE_BYTES, LOAD);
+ CLIB_PREFETCH (bufs[3], 2 * CLIB_CACHE_LINE_BYTES, LOAD);
+ vlib_prefetch_buffer_data (bufs[2], LOAD);
+ vlib_prefetch_buffer_data (bufs[3], LOAD);
+
+ /* copying objects from cacheline 0 */
+ bufs[0]->current_data = 0;
+ bufs[1]->current_data = 0;
+
+ bufs[0]->current_length = hdr_sz;
+ bufs[1]->current_length = hdr_sz;
+
+ bufs[0]->flags = bufs[1]->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags;
+ bufs[0]->flow_id = bufs[1]->flow_id = b0->flow_id;
+ bufs[0]->error = bufs[1]->error = b0->error;
+ bufs[0]->current_config_index = bufs[1]->current_config_index =
+ b0->current_config_index;
+
+ clib_memcpy_fast (&bufs[0]->opaque, &b0->opaque, sizeof (b0->opaque));
+ clib_memcpy_fast (&bufs[1]->opaque, &b0->opaque, sizeof (b0->opaque));
+
+ /* copying objects from cacheline 1 */
+ bufs[0]->trace_handle = b0->trace_handle;
+ bufs[1]->trace_handle = b0->trace_handle;
+
+ bufs[0]->total_length_not_including_first_buffer = 0;
+ bufs[1]->total_length_not_including_first_buffer = 0;
+
+ /* copying data */
+ clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz);
+ clib_memcpy_fast (bufs[1]->data, vlib_buffer_get_current (b0), hdr_sz);
+
+ bufs += 2;
+ i -= 2;
+ }
+
+ while (i > 0)
+ {
+ /* copying objects from cacheline 0 */
+ bufs[0]->current_data = 0;
+ bufs[0]->current_length = hdr_sz;
+ bufs[0]->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags;
+ bufs[0]->flow_id = b0->flow_id;
+ bufs[0]->error = b0->error;
+ bufs[0]->current_config_index = b0->current_config_index;
+ clib_memcpy_fast (&bufs[0]->opaque, &b0->opaque, sizeof (b0->opaque));
+
+ /* copying objects from cacheline 1 */
+ bufs[0]->trace_handle = b0->trace_handle;
+ bufs[0]->total_length_not_including_first_buffer = 0;
+
+ /* copying data */
+ clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz);
+
+ bufs++;
+ i--;
+ }
+}
+
+static_always_inline void
+gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq,
+ int is_l2, int is_ip6, generic_header_offset_t *gho,
+ clib_ip_csum_t *c, u8 tcp_flags)
+{
+
+ ip4_header_t *ip4 =
+ (ip4_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset +
+ gho->outer_hdr_sz);
+ ip6_header_t *ip6 =
+ (ip6_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset +
+ gho->outer_hdr_sz);
+ tcp_header_t *tcp =
+ (tcp_header_t *) (vlib_buffer_get_current (b0) + gho->l4_hdr_offset +
+ gho->outer_hdr_sz);
+
+ tcp->flags = tcp_flags;
+ tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq);
+
+ if (is_ip6)
+ {
+ ip6->payload_length = clib_host_to_net_u16 (
+ b0->current_length - gho->l4_hdr_offset - gho->outer_hdr_sz);
+ vnet_buffer_offload_flags_clear (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
+ ip6_psh_t psh = { 0 };
+ u32 *p = (u32 *) &psh;
+ psh.src = ip6->src_address;
+ psh.dst = ip6->dst_address;
+ psh.l4len = ip6->payload_length;
+ psh.proto = clib_host_to_net_u32 ((u32) ip6->protocol);
+ for (int i = 0; i < 10; i++)
+ c->sum += p[i];
+ }
+ else
+ {
+ ip4->length = clib_host_to_net_u16 (
+ b0->current_length - gho->l3_hdr_offset - gho->outer_hdr_sz);
+ if (gho->gho_flags & GHO_F_IP4)
+ ip4->checksum = ip4_header_checksum (ip4);
+ vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
+ VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
+ c->sum += clib_mem_unaligned (&ip4->src_address, u32);
+ c->sum += clib_mem_unaligned (&ip4->dst_address, u32);
+ c->sum += clib_host_to_net_u32 (
+ (clib_net_to_host_u16 (ip4->length) - ip4_header_bytes (ip4)) +
+ (ip4->protocol << 16));
+ }
+ clib_ip_csum_chunk (c, (u8 *) tcp, gho->l4_hdr_sz);
+ tcp->checksum = clib_ip_csum_fold (c);
+
+ if (!is_l2 && ((gho->gho_flags & GHO_F_TUNNEL) == 0))
+ {
+ u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
+
+ ip_adjacency_t *adj0 = adj_get (adj_index0);
+
+ if (adj0->lookup_next_index == IP_LOOKUP_NEXT_MIDCHAIN &&
+ adj0->sub_type.midchain.fixup_func)
+ /* calls e.g. ipip44_fixup */
+ adj0->sub_type.midchain.fixup_func (
+ vm, adj0, b0, adj0->sub_type.midchain.fixup_data);
+ }
+}
+
+static_always_inline u32
+gso_segment_buffer_inline (vlib_main_t *vm,
+ vnet_interface_per_thread_data_t *ptd,
+ vlib_buffer_t *b, generic_header_offset_t *gho,
+ int is_l2, int is_ip6)
+{
+ vlib_buffer_t **bufs = 0;
+ u32 n_tx_bytes = 0;
+ u16 gso_size = vnet_buffer2 (b)->gso_size;
+ u8 tcp_flags = 0, tcp_flags_no_fin_psh = 0;
+ u32 default_bflags =
+ b->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT);
+ u16 hdr_sz = gho->hdr_sz + gho->outer_hdr_sz;
+ u32 next_tcp_seq = 0, tcp_seq = 0;
+ u32 data_size = vlib_buffer_length_in_chain (vm, b) - hdr_sz;
+ u16 size =
+ clib_min (gso_size, vlib_buffer_get_default_data_size (vm) - hdr_sz);
+ u16 n_alloc = 0, n_bufs = ((data_size + size - 1) / size);
+ clib_ip_csum_t c = { .sum = 0, .odd = 0 };
+ u8 *src_ptr, *dst_ptr;
+ u16 src_left, dst_left, bytes_to_copy;
+ u32 i = 0;
+
+ vec_validate (ptd->split_buffers, n_bufs - 1);
+ n_alloc = vlib_buffer_alloc (vm, ptd->split_buffers, n_bufs);
+ if (n_alloc < n_bufs)
+ {
+ vlib_buffer_free (vm, ptd->split_buffers, n_alloc);
+ return 0;
+ }
+
+ vec_validate (bufs, n_bufs - 1);
+ vlib_get_buffers (vm, ptd->split_buffers, bufs, n_bufs);
+
+ tcp_header_t *tcp =
+ (tcp_header_t *) (vlib_buffer_get_current (b) + gho->l4_hdr_offset +
+ gho->outer_hdr_sz);
+ tcp_seq = next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number);
+ /* store original flags for last packet and reset FIN and PSH */
+ tcp_flags = tcp->flags;
+ tcp_flags_no_fin_psh = tcp->flags & ~(TCP_FLAG_FIN | TCP_FLAG_PSH);
+ tcp->checksum = 0;
+
+ gso_init_bufs_from_template_base (bufs, b, default_bflags, n_bufs, hdr_sz);
+
+ src_ptr = vlib_buffer_get_current (b) + hdr_sz;
+ src_left = b->current_length - hdr_sz;
+ dst_ptr = vlib_buffer_get_current (bufs[i]) + hdr_sz;
+ dst_left = size;
+
+ while (data_size)
+ {
+ bytes_to_copy = clib_min (src_left, dst_left);
+ clib_ip_csum_and_copy_chunk (&c, src_ptr, dst_ptr, bytes_to_copy);
+
+ src_left -= bytes_to_copy;
+ src_ptr += bytes_to_copy;
+ data_size -= bytes_to_copy;
+ dst_left -= bytes_to_copy;
+ dst_ptr += bytes_to_copy;
+ next_tcp_seq += bytes_to_copy;
+ bufs[i]->current_length += bytes_to_copy;
+
+ if (0 == src_left)
+ {
+ /* init src to the next buffer in chain */
+ if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
+ {
+ b = vlib_get_buffer (vm, b->next_buffer);
+ src_left = b->current_length;
+ src_ptr = vlib_buffer_get_current (b);
+ }
+ else
+ {
+ ASSERT (data_size == 0);
+ break;
+ }
+ }
+ if (0 == dst_left && data_size)
+ {
+ vlib_prefetch_buffer_header (bufs[i + 1], LOAD);
+ vlib_prefetch_buffer_data (bufs[i + 1], LOAD);
+
+ n_tx_bytes += bufs[i]->current_length;
+ gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho,
+ &c, tcp_flags_no_fin_psh);
+ i++;
+ dst_left = size;
+ dst_ptr = vlib_buffer_get_current (bufs[i]) + hdr_sz;
+ tcp_seq = next_tcp_seq;
+ // reset clib_ip_csum_t
+ c.odd = 0;
+ c.sum = 0;
+ }
+ }
+
+ ASSERT ((i + 1) == n_alloc);
+ n_tx_bytes += bufs[i]->current_length;
+ gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho, &c,
+ tcp_flags);
+
+ vec_free (bufs);
+ return n_tx_bytes;
+}
+
#endif /* included_gso_h */
/*
diff --git a/src/vnet/gso/node.c b/src/vnet/gso/node.c
index c48d8fefe16..109d178c33c 100644
--- a/src/vnet/gso/node.c
+++ b/src/vnet/gso/node.c
@@ -92,9 +92,6 @@ tso_segment_ipip_tunnel_fixup (vlib_main_t * vm,
while (i < n_tx_bufs)
{
vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[i]);
- vnet_get_outer_header (b0, gho);
- clib_memcpy_fast (vlib_buffer_get_current (b0),
- vlib_buffer_get_current (sb0), gho->outer_hdr_sz);
ip4_header_t *ip4 =
(ip4_header_t *) (vlib_buffer_get_current (b0) +
@@ -184,9 +181,6 @@ tso_segment_vxlan_tunnel_fixup (vlib_main_t * vm,
while (i < n_tx_bufs)
{
vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[i]);
- vnet_get_outer_header (b0, gho);
- clib_memcpy_fast (vlib_buffer_get_current (b0),
- vlib_buffer_get_current (sb0), gho->outer_hdr_sz);
tso_segment_vxlan_tunnel_headers_fixup (vm, b0, gho);
n_tx_bytes += gho->outer_hdr_sz;
@@ -686,7 +680,6 @@ vnet_gso_node_inline (vlib_main_t * vm,
n_left_to_next += 1;
/* undo the counting. */
generic_header_offset_t gho = { 0 };
- u32 n_bytes_b0 = vlib_buffer_length_in_chain (vm, b[0]);
u32 n_tx_bytes = 0;
u32 inner_is_ip6 = is_ip6;
@@ -707,15 +700,11 @@ vnet_gso_node_inline (vlib_main_t * vm,
continue;
}
- vnet_get_inner_header (b[0], &gho);
-
- n_bytes_b0 -= gho.outer_hdr_sz;
inner_is_ip6 = (gho.gho_flags & GHO_F_IP6) != 0;
}
- n_tx_bytes =
- tso_segment_buffer (vm, ptd, bi0, b[0], &gho, n_bytes_b0,
- is_l2, inner_is_ip6);
+ n_tx_bytes = gso_segment_buffer_inline (vm, ptd, b[0], &gho,
+ is_l2, inner_is_ip6);
if (PREDICT_FALSE (n_tx_bytes == 0))
{
@@ -729,7 +718,6 @@ vnet_gso_node_inline (vlib_main_t * vm,
if (PREDICT_FALSE (gho.gho_flags & GHO_F_VXLAN_TUNNEL))
{
- vnet_get_outer_header (b[0], &gho);
n_tx_bytes +=
tso_segment_vxlan_tunnel_fixup (vm, ptd, b[0], &gho);
}
@@ -738,7 +726,6 @@ vnet_gso_node_inline (vlib_main_t * vm,
(gho.gho_flags & (GHO_F_IPIP_TUNNEL |
GHO_F_IPIP6_TUNNEL)))
{
- vnet_get_outer_header (b[0], &gho);
n_tx_bytes +=
tso_segment_ipip_tunnel_fixup (vm, ptd, b[0], &gho);
}