1 files changed, 237 insertions, 0 deletions

diff --git a/src/vnet/gso/gso.h b/src/vnet/gso/gso.h
index 926ce634fd0..041fab3bcc4 100644
--- a/src/vnet/gso/gso.h
+++ b/src/vnet/gso/gso.h
@@ -18,6 +18,7 @@
 
 #include <vnet/vnet.h>
 #include <vnet/gso/hdr_offset_parser.h>
+#include <vnet/ip/ip_psh_cksum.h>
 
 typedef struct
 {
@@ -33,6 +34,242 @@ u32 gso_segment_buffer (vlib_main_t *vm, vnet_interface_per_thread_data_t *ptd,
 			u32 bi, vlib_buffer_t *b, generic_header_offset_t *gho,
 			u32 n_bytes_b, u8 is_l2, u8 is_ip6);
 
+static_always_inline void
+gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0,
+				  u32 flags, u16 n_bufs, u16 hdr_sz)
+{
+  u32 i = n_bufs;
+  while (i >= 4)
+    {
+      /* prefetches */
+      CLIB_PREFETCH (bufs[2], 2 * CLIB_CACHE_LINE_BYTES, LOAD);
+      CLIB_PREFETCH (bufs[3], 2 * CLIB_CACHE_LINE_BYTES, LOAD);
+      vlib_prefetch_buffer_data (bufs[2], LOAD);
+      vlib_prefetch_buffer_data (bufs[3], LOAD);
+
+      /* copying objects from cacheline 0 */
+      bufs[0]->current_data = 0;
+      bufs[1]->current_data = 0;
+
+      bufs[0]->current_length = hdr_sz;
+      bufs[1]->current_length = hdr_sz;
+
+      bufs[0]->flags = bufs[1]->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags;
+      bufs[0]->flow_id = bufs[1]->flow_id = b0->flow_id;
+      bufs[0]->error = bufs[1]->error = b0->error;
+      bufs[0]->current_config_index = bufs[1]->current_config_index =
+	b0->current_config_index;
+
+      clib_memcpy_fast (&bufs[0]->opaque, &b0->opaque, sizeof (b0->opaque));
+      clib_memcpy_fast (&bufs[1]->opaque, &b0->opaque, sizeof (b0->opaque));
+
+      /* copying objects from cacheline 1 */
+      bufs[0]->trace_handle = b0->trace_handle;
+      bufs[1]->trace_handle = b0->trace_handle;
+
+      bufs[0]->total_length_not_including_first_buffer = 0;
+      bufs[1]->total_length_not_including_first_buffer = 0;
+
+      /* copying data */
+      clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz);
+      clib_memcpy_fast (bufs[1]->data, vlib_buffer_get_current (b0), hdr_sz);
+
+      bufs += 2;
+      i -= 2;
+    }
+
+  while (i > 0)
+    {
+      /* copying objects from cacheline 0 */
+      bufs[0]->current_data = 0;
+      bufs[0]->current_length = hdr_sz;
+      bufs[0]->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags;
+      bufs[0]->flow_id = b0->flow_id;
+      bufs[0]->error = b0->error;
+      bufs[0]->current_config_index = b0->current_config_index;
+      clib_memcpy_fast (&bufs[0]->opaque, &b0->opaque, sizeof (b0->opaque));
+
+      /* copying objects from cacheline 1 */
+      bufs[0]->trace_handle = b0->trace_handle;
+      bufs[0]->total_length_not_including_first_buffer = 0;
+
+      /* copying data */
+      clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz);
+
+      bufs++;
+      i--;
+    }
+}
+
+static_always_inline void
+gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq,
+			 int is_l2, int is_ip6, generic_header_offset_t *gho,
+			 clib_ip_csum_t *c, u8 tcp_flags)
+{
+
+  ip4_header_t *ip4 =
+    (ip4_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset +
+		      gho->outer_hdr_sz);
+  ip6_header_t *ip6 =
+    (ip6_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset +
+		      gho->outer_hdr_sz);
+  tcp_header_t *tcp =
+    (tcp_header_t *) (vlib_buffer_get_current (b0) + gho->l4_hdr_offset +
+		      gho->outer_hdr_sz);
+
+  tcp->flags = tcp_flags;
+  tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq);
+
+  if (is_ip6)
+    {
+      ip6->payload_length = clib_host_to_net_u16 (
+	b0->current_length - gho->l4_hdr_offset - gho->outer_hdr_sz);
+      vnet_buffer_offload_flags_clear (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
+      ip6_psh_t psh = { 0 };
+      u32 *p = (u32 *) &psh;
+      psh.src = ip6->src_address;
+      psh.dst = ip6->dst_address;
+      psh.l4len = ip6->payload_length;
+      psh.proto = clib_host_to_net_u32 ((u32) ip6->protocol);
+      for (int i = 0; i < 10; i++)
+	c->sum += p[i];
+    }
+  else
+    {
+      ip4->length = clib_host_to_net_u16 (
+	b0->current_length - gho->l3_hdr_offset - gho->outer_hdr_sz);
+      if (gho->gho_flags & GHO_F_IP4)
+	ip4->checksum = ip4_header_checksum (ip4);
+      vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
+					    VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
+      c->sum += clib_mem_unaligned (&ip4->src_address, u32);
+      c->sum += clib_mem_unaligned (&ip4->dst_address, u32);
+      c->sum += clib_host_to_net_u32 (
+	(clib_net_to_host_u16 (ip4->length) - ip4_header_bytes (ip4)) +
+	(ip4->protocol << 16));
+    }
+  clib_ip_csum_chunk (c, (u8 *) tcp, gho->l4_hdr_sz);
+  tcp->checksum = clib_ip_csum_fold (c);
+
+  if (!is_l2 && ((gho->gho_flags & GHO_F_TUNNEL) == 0))
+    {
+      u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
+
+      ip_adjacency_t *adj0 = adj_get (adj_index0);
+
+      if (adj0->lookup_next_index == IP_LOOKUP_NEXT_MIDCHAIN &&
+	  adj0->sub_type.midchain.fixup_func)
+	/* calls e.g. ipip44_fixup */
+	adj0->sub_type.midchain.fixup_func (
+	  vm, adj0, b0, adj0->sub_type.midchain.fixup_data);
+    }
+}
+
+static_always_inline u32
+gso_segment_buffer_inline (vlib_main_t *vm,
+			   vnet_interface_per_thread_data_t *ptd,
+			   vlib_buffer_t *b, generic_header_offset_t *gho,
+			   int is_l2, int is_ip6)
+{
+  vlib_buffer_t **bufs = 0;
+  u32 n_tx_bytes = 0;
+  u16 gso_size = vnet_buffer2 (b)->gso_size;
+  u8 tcp_flags = 0, tcp_flags_no_fin_psh = 0;
+  u32 default_bflags =
+    b->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT);
+  u16 hdr_sz = gho->hdr_sz + gho->outer_hdr_sz;
+  u32 next_tcp_seq = 0, tcp_seq = 0;
+  u32 data_size = vlib_buffer_length_in_chain (vm, b) - hdr_sz;
+  u16 size =
+    clib_min (gso_size, vlib_buffer_get_default_data_size (vm) - hdr_sz);
+  u16 n_alloc = 0, n_bufs = ((data_size + size - 1) / size);
+  clib_ip_csum_t c = { .sum = 0, .odd = 0 };
+  u8 *src_ptr, *dst_ptr;
+  u16 src_left, dst_left, bytes_to_copy;
+  u32 i = 0;
+
+  vec_validate (ptd->split_buffers, n_bufs - 1);
+  n_alloc = vlib_buffer_alloc (vm, ptd->split_buffers, n_bufs);
+  if (n_alloc < n_bufs)
+    {
+      vlib_buffer_free (vm, ptd->split_buffers, n_alloc);
+      return 0;
+    }
+
+  vec_validate (bufs, n_bufs - 1);
+  vlib_get_buffers (vm, ptd->split_buffers, bufs, n_bufs);
+
+  tcp_header_t *tcp =
+    (tcp_header_t *) (vlib_buffer_get_current (b) + gho->l4_hdr_offset +
+		      gho->outer_hdr_sz);
+  tcp_seq = next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number);
+  /* store original flags for last packet and reset FIN and PSH */
+  tcp_flags = tcp->flags;
+  tcp_flags_no_fin_psh = tcp->flags & ~(TCP_FLAG_FIN | TCP_FLAG_PSH);
+  tcp->checksum = 0;
+
+  gso_init_bufs_from_template_base (bufs, b, default_bflags, n_bufs, hdr_sz);
+
+  src_ptr = vlib_buffer_get_current (b) + hdr_sz;
+  src_left = b->current_length - hdr_sz;
+  dst_ptr = vlib_buffer_get_current (bufs[i]) + hdr_sz;
+  dst_left = size;
+
+  while (data_size)
+    {
+      bytes_to_copy = clib_min (src_left, dst_left);
+      clib_ip_csum_and_copy_chunk (&c, src_ptr, dst_ptr, bytes_to_copy);
+
+      src_left -= bytes_to_copy;
+      src_ptr += bytes_to_copy;
+      data_size -= bytes_to_copy;
+      dst_left -= bytes_to_copy;
+      dst_ptr += bytes_to_copy;
+      next_tcp_seq += bytes_to_copy;
+      bufs[i]->current_length += bytes_to_copy;
+
+      if (0 == src_left)
+	{
+	  /* init src to the next buffer in chain */
+	  if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
+	    {
+	      b = vlib_get_buffer (vm, b->next_buffer);
+	      src_left = b->current_length;
+	      src_ptr = vlib_buffer_get_current (b);
+	    }
+	  else
+	    {
+	      ASSERT (data_size == 0);
+	      break;
+	    }
+	}
+      if (0 == dst_left && data_size)
+	{
+	  vlib_prefetch_buffer_header (bufs[i + 1], LOAD);
+	  vlib_prefetch_buffer_data (bufs[i + 1], LOAD);
+
+	  n_tx_bytes += bufs[i]->current_length;
+	  gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho,
+				   &c, tcp_flags_no_fin_psh);
+	  i++;
+	  dst_left = size;
+	  dst_ptr = vlib_buffer_get_current (bufs[i]) + hdr_sz;
+	  tcp_seq = next_tcp_seq;
+	  // reset clib_ip_csum_t
+	  c.odd = 0;
+	  c.sum = 0;
+	}
+    }
+
+  ASSERT ((i + 1) == n_alloc);
+  n_tx_bytes += bufs[i]->current_length;
+  gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho, &c,
+			   tcp_flags);
+
+  vec_free (bufs);
+  return n_tx_bytes;
+}
+
 #endif /* included_gso_h */
 
 /*