1 files changed, 75 insertions, 44 deletions

diff --git a/src/vnet/gso/gso.h b/src/vnet/gso/gso.h
index 883a4941ee2..dee5da5c70b 100644
--- a/src/vnet/gso/gso.h
+++ b/src/vnet/gso/gso.h
@@ -39,13 +39,13 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0,
 				  u32 flags, u16 n_bufs, u16 hdr_sz)
 {
   u32 i = n_bufs;
-  while (i >= 4)
+  while (i >= 6)
     {
       /* prefetches */
       CLIB_PREFETCH (bufs[2], 2 * CLIB_CACHE_LINE_BYTES, LOAD);
       CLIB_PREFETCH (bufs[3], 2 * CLIB_CACHE_LINE_BYTES, LOAD);
-      vlib_prefetch_buffer_data (bufs[2], LOAD);
-      vlib_prefetch_buffer_data (bufs[3], LOAD);
+      vlib_prefetch_buffer_data (bufs[4], LOAD);
+      vlib_prefetch_buffer_data (bufs[5], LOAD);
 
       /* copying objects from cacheline 0 */
       bufs[0]->current_data = 0;
@@ -70,10 +70,26 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0,
       bufs[0]->total_length_not_including_first_buffer = 0;
       bufs[1]->total_length_not_including_first_buffer = 0;
 
+      clib_memcpy_fast (&bufs[0]->opaque2, &b0->opaque2, sizeof (b0->opaque2));
+      clib_memcpy_fast (&bufs[1]->opaque2, &b0->opaque2, sizeof (b0->opaque2));
+
       /* copying data */
       clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz);
       clib_memcpy_fast (bufs[1]->data, vlib_buffer_get_current (b0), hdr_sz);
 
+      /* header offset fixup */
+      vnet_buffer (bufs[0])->l2_hdr_offset -= b0->current_data;
+      vnet_buffer (bufs[0])->l3_hdr_offset -= b0->current_data;
+      vnet_buffer (bufs[0])->l4_hdr_offset -= b0->current_data;
+      vnet_buffer2 (bufs[0])->outer_l3_hdr_offset -= b0->current_data;
+      vnet_buffer2 (bufs[0])->outer_l4_hdr_offset -= b0->current_data;
+
+      vnet_buffer (bufs[1])->l2_hdr_offset -= b0->current_data;
+      vnet_buffer (bufs[1])->l3_hdr_offset -= b0->current_data;
+      vnet_buffer (bufs[1])->l4_hdr_offset -= b0->current_data;
+      vnet_buffer2 (bufs[1])->outer_l3_hdr_offset -= b0->current_data;
+      vnet_buffer2 (bufs[1])->outer_l4_hdr_offset -= b0->current_data;
+
       bufs += 2;
       i -= 2;
     }
@@ -92,10 +108,18 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0,
       /* copying objects from cacheline 1 */
       bufs[0]->trace_handle = b0->trace_handle;
       bufs[0]->total_length_not_including_first_buffer = 0;
+      clib_memcpy_fast (&bufs[0]->opaque2, &b0->opaque2, sizeof (b0->opaque2));
 
       /* copying data */
       clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz);
 
+      /* header offset fixup */
+      vnet_buffer (bufs[0])->l2_hdr_offset -= b0->current_data;
+      vnet_buffer (bufs[0])->l3_hdr_offset -= b0->current_data;
+      vnet_buffer (bufs[0])->l4_hdr_offset -= b0->current_data;
+      vnet_buffer2 (bufs[0])->outer_l3_hdr_offset -= b0->current_data;
+      vnet_buffer2 (bufs[0])->outer_l4_hdr_offset -= b0->current_data;
+
       bufs++;
       i--;
     }
@@ -103,28 +127,41 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0,
 
 static_always_inline void
 gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq,
-			 int is_l2, int is_ip6, generic_header_offset_t *gho,
-			 clib_ip_csum_t *c, u8 tcp_flags)
+			 int is_l2, u8 oflags, u16 hdr_sz, u16 l4_hdr_sz,
+			 clib_ip_csum_t *c, u8 tcp_flags, u8 is_prefetch,
+			 vlib_buffer_t *b1)
 {
 
-  ip4_header_t *ip4 =
-    (ip4_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset +
-		      gho->outer_hdr_sz);
-  ip6_header_t *ip6 =
-    (ip6_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset +
-		      gho->outer_hdr_sz);
-  tcp_header_t *tcp =
-    (tcp_header_t *) (vlib_buffer_get_current (b0) + gho->l4_hdr_offset +
-		      gho->outer_hdr_sz);
+  i16 l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset;
+  i16 l4_hdr_offset = vnet_buffer (b0)->l4_hdr_offset;
+
+  ip4_header_t *ip4 = (ip4_header_t *) (b0->data + l3_hdr_offset);
+  ip6_header_t *ip6 = (ip6_header_t *) (b0->data + l3_hdr_offset);
+  tcp_header_t *tcp = (tcp_header_t *) (b0->data + l4_hdr_offset);
 
   tcp->flags = tcp_flags;
   tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq);
   c->odd = 0;
 
-  if (is_ip6)
+  if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
+    {
+      ip4->length =
+	clib_host_to_net_u16 (b0->current_length - hdr_sz +
+			      (l4_hdr_offset - l3_hdr_offset) + l4_hdr_sz);
+      ip4->checksum = 0;
+      ip4->checksum = ip4_header_checksum (ip4);
+      vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
+					    VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
+      c->sum += clib_mem_unaligned (&ip4->src_address, u32);
+      c->sum += clib_mem_unaligned (&ip4->dst_address, u32);
+      c->sum += clib_host_to_net_u32 (
+	(clib_net_to_host_u16 (ip4->length) - ip4_header_bytes (ip4)) +
+	(ip4->protocol << 16));
+    }
+  else
     {
-      ip6->payload_length = clib_host_to_net_u16 (
-	b0->current_length - gho->l4_hdr_offset - gho->outer_hdr_sz);
+      ip6->payload_length =
+	clib_host_to_net_u16 (b0->current_length - hdr_sz + l4_hdr_sz);
       vnet_buffer_offload_flags_clear (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
       ip6_psh_t psh = { 0 };
       u32 *p = (u32 *) &psh;
@@ -135,24 +172,15 @@ gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq,
       for (int i = 0; i < 10; i++)
 	c->sum += p[i];
     }
-  else
-    {
-      ip4->length = clib_host_to_net_u16 (
-	b0->current_length - gho->l3_hdr_offset - gho->outer_hdr_sz);
-      if (gho->gho_flags & GHO_F_IP4)
-	ip4->checksum = ip4_header_checksum (ip4);
-      vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
-					    VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
-      c->sum += clib_mem_unaligned (&ip4->src_address, u32);
-      c->sum += clib_mem_unaligned (&ip4->dst_address, u32);
-      c->sum += clib_host_to_net_u32 (
-	(clib_net_to_host_u16 (ip4->length) - ip4_header_bytes (ip4)) +
-	(ip4->protocol << 16));
-    }
-  clib_ip_csum_chunk (c, (u8 *) tcp, gho->l4_hdr_sz);
+
+  if (is_prefetch)
+    CLIB_PREFETCH (vlib_buffer_get_current (b1) + hdr_sz,
+		   CLIB_CACHE_LINE_BYTES, LOAD);
+
+  clib_ip_csum_chunk (c, (u8 *) tcp, l4_hdr_sz);
   tcp->checksum = clib_ip_csum_fold (c);
 
-  if (!is_l2 && ((gho->gho_flags & GHO_F_TUNNEL) == 0))
+  if (!is_l2 && ((oflags & VNET_BUFFER_OFFLOAD_F_TNL_MASK) == 0))
     {
       u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
 
@@ -169,16 +197,20 @@ gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq,
 static_always_inline u32
 gso_segment_buffer_inline (vlib_main_t *vm,
 			   vnet_interface_per_thread_data_t *ptd,
-			   vlib_buffer_t *b, generic_header_offset_t *gho,
-			   int is_l2, int is_ip6)
+			   vlib_buffer_t *b, int is_l2)
 {
   vlib_buffer_t **bufs = 0;
   u32 n_tx_bytes = 0;
+
+  u8 oflags = vnet_buffer (b)->oflags;
+  i16 l4_hdr_offset = vnet_buffer (b)->l4_hdr_offset;
   u16 gso_size = vnet_buffer2 (b)->gso_size;
+  u16 l4_hdr_sz = vnet_buffer2 (b)->gso_l4_hdr_sz;
+
   u8 tcp_flags = 0, tcp_flags_no_fin_psh = 0;
   u32 default_bflags =
     b->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT);
-  u16 hdr_sz = gho->hdr_sz + gho->outer_hdr_sz;
+  u16 hdr_sz = (l4_hdr_offset - b->current_data) + l4_hdr_sz;
   u32 next_tcp_seq = 0, tcp_seq = 0;
   u32 data_size = vlib_buffer_length_in_chain (vm, b) - hdr_sz;
   u16 size =
@@ -200,9 +232,8 @@ gso_segment_buffer_inline (vlib_main_t *vm,
   vec_validate (bufs, n_bufs - 1);
   vlib_get_buffers (vm, ptd->split_buffers, bufs, n_bufs);
 
-  tcp_header_t *tcp =
-    (tcp_header_t *) (vlib_buffer_get_current (b) + gho->l4_hdr_offset +
-		      gho->outer_hdr_sz);
+  tcp_header_t *tcp = (tcp_header_t *) (b->data + l4_hdr_offset);
+
   tcp_seq = next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number);
   /* store original flags for last packet and reset FIN and PSH */
   tcp_flags = tcp->flags;
@@ -247,11 +278,11 @@ gso_segment_buffer_inline (vlib_main_t *vm,
       if (0 == dst_left && data_size)
 	{
 	  vlib_prefetch_buffer_header (bufs[i + 1], LOAD);
-	  vlib_prefetch_buffer_data (bufs[i + 1], LOAD);
 
 	  n_tx_bytes += bufs[i]->current_length;
-	  gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho,
-				   &c, tcp_flags_no_fin_psh);
+	  gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, oflags, hdr_sz,
+				   l4_hdr_sz, &c, tcp_flags_no_fin_psh, 1,
+				   bufs[i + 1]);
 	  i++;
 	  dst_left = size;
 	  dst_ptr = vlib_buffer_get_current (bufs[i]) + hdr_sz;
@@ -264,8 +295,8 @@ gso_segment_buffer_inline (vlib_main_t *vm,
 
   ASSERT ((i + 1) == n_alloc);
   n_tx_bytes += bufs[i]->current_length;
-  gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho, &c,
-			   tcp_flags);
+  gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, oflags, hdr_sz,
+			   l4_hdr_sz, &c, tcp_flags, 0, NULL);
 
   vec_free (bufs);
   return n_tx_bytes;