gso: Add gso feature arc

Type: refactor

Change-Id: Ib4a8e7e64093b2db1deda6663325080bc8337605
Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>

1 files changed, 14 insertions, 377 deletions

diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c
index 707b1167a48..5e702e3b96c 100644
--- a/src/vnet/interface_output.c
+++ b/src/vnet/interface_output.c
@@ -49,9 +49,7 @@ typedef struct
 {
   u32 sw_if_index;
   u32 flags;
-  u16 gso_size;
-  u8 gso_l4_hdr_sz;
-  u8 data[128 - 3 * sizeof (u32)];
+  u8 data[128 - 2 * sizeof (u32)];
 }
 interface_output_trace_t;
 
@@ -83,17 +81,6 @@ format_vnet_interface_output_trace (u8 * s, va_list * va)
 	    format (s, "%U ", format_vnet_sw_interface_name, vnm, si,
 		    t->flags);
 	}
-#define _(bit, name, v, x) \
-          if (v && (t->flags & VNET_BUFFER_F_##name)) \
-            s = format (s, "%s ", v);
-      foreach_vnet_buffer_flag
-#undef _
-	if (t->flags & VNET_BUFFER_F_GSO)
-	{
-	  s = format (s, "\n%Ugso_sz %d gso_l4_hdr_sz %d",
-		      format_white_space, indent + 2, t->gso_size,
-		      t->gso_l4_hdr_sz);
-	}
       s =
 	format (s, "\n%U%U", format_white_space, indent,
 		node->format_buffer ? node->format_buffer : format_hex_bytes,
@@ -133,8 +120,6 @@ vnet_interface_output_trace (vlib_main_t * vm,
 	  t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 	  t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
 	  t0->flags = b0->flags;
-	  t0->gso_size = vnet_buffer2 (b0)->gso_size;
-	  t0->gso_l4_hdr_sz = vnet_buffer2 (b0)->gso_l4_hdr_sz;
 	  clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0),
 			    sizeof (t0->data));
 	}
@@ -143,8 +128,6 @@ vnet_interface_output_trace (vlib_main_t * vm,
 	  t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
 	  t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_TX];
 	  t1->flags = b1->flags;
-	  t1->gso_size = vnet_buffer2 (b1)->gso_size;
-	  t1->gso_l4_hdr_sz = vnet_buffer2 (b1)->gso_l4_hdr_sz;
 	  clib_memcpy_fast (t1->data, vlib_buffer_get_current (b1),
 			    sizeof (t1->data));
 	}
@@ -167,8 +150,6 @@ vnet_interface_output_trace (vlib_main_t * vm,
 	  t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 	  t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
 	  t0->flags = b0->flags;
-	  t0->gso_size = vnet_buffer2 (b0)->gso_size;
-	  t0->gso_l4_hdr_sz = vnet_buffer2 (b0)->gso_l4_hdr_sz;
 	  clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0),
 			    sizeof (t0->data));
 	}
@@ -230,250 +211,13 @@ calc_checksums (vlib_main_t * vm, vlib_buffer_t * b)
   b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
 }
 
-static_always_inline u16
-tso_alloc_tx_bufs (vlib_main_t * vm,
-		   vnet_interface_per_thread_data_t * ptd,
-		   vlib_buffer_t * b0, u32 n_bytes_b0, u16 l234_sz,
-		   u16 gso_size)
-{
-  u16 size =
-    clib_min (gso_size, vlib_buffer_get_default_data_size (vm) - l234_sz);
-
-  /* rounded-up division */
-  u16 n_bufs = (n_bytes_b0 - l234_sz + (size - 1)) / size;
-  u16 n_alloc;
-
-  ASSERT (n_bufs > 0);
-  vec_validate (ptd->split_buffers, n_bufs - 1);
-
-  n_alloc = vlib_buffer_alloc (vm, ptd->split_buffers, n_bufs);
-  if (n_alloc < n_bufs)
-    {
-      vlib_buffer_free (vm, ptd->split_buffers, n_alloc);
-      return 0;
-    }
-  return n_alloc;
-}
-
-static_always_inline void
-tso_init_buf_from_template_base (vlib_buffer_t * nb0, vlib_buffer_t * b0,
-				 u32 flags, u16 length)
-{
-  nb0->current_data = b0->current_data;
-  nb0->total_length_not_including_first_buffer = 0;
-  nb0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID | flags;
-  clib_memcpy_fast (&nb0->opaque, &b0->opaque, sizeof (nb0->opaque));
-  clib_memcpy_fast (vlib_buffer_get_current (nb0),
-		    vlib_buffer_get_current (b0), length);
-  nb0->current_length = length;
-}
-
-static_always_inline void
-tso_init_buf_from_template (vlib_main_t * vm, vlib_buffer_t * nb0,
-			    vlib_buffer_t * b0, u16 template_data_sz,
-			    u16 gso_size, u8 ** p_dst_ptr, u16 * p_dst_left,
-			    u32 next_tcp_seq, u32 flags)
-{
-  tso_init_buf_from_template_base (nb0, b0, flags, template_data_sz);
-
-  *p_dst_left =
-    clib_min (gso_size,
-	      vlib_buffer_get_default_data_size (vm) - (template_data_sz +
-							nb0->current_data));
-  *p_dst_ptr = vlib_buffer_get_current (nb0) + template_data_sz;
-
-  tcp_header_t *tcp =
-    (tcp_header_t *) (nb0->data + vnet_buffer (nb0)->l4_hdr_offset);
-  tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq);
-}
-
-static_always_inline void
-tso_fixup_segmented_buf (vlib_buffer_t * b0, u8 tcp_flags, int is_ip6)
-{
-  u16 l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset;
-  u16 l4_hdr_offset = vnet_buffer (b0)->l4_hdr_offset;
-  ip4_header_t *ip4 = (ip4_header_t *) (b0->data + l3_hdr_offset);
-  ip6_header_t *ip6 = (ip6_header_t *) (b0->data + l3_hdr_offset);
-  tcp_header_t *tcp = (tcp_header_t *) (b0->data + l4_hdr_offset);
-
-  tcp->flags = tcp_flags;
-
-  if (is_ip6)
-    ip6->payload_length =
-      clib_host_to_net_u16 (b0->current_length -
-			    (l4_hdr_offset - b0->current_data));
-  else
-    ip4->length =
-      clib_host_to_net_u16 (b0->current_length -
-			    (l3_hdr_offset - b0->current_data));
-}
-
-/**
- * Allocate the necessary number of ptd->split_buffers,
- * and segment the possibly chained buffer(s) from b0 into
- * there.
- *
- * Return the cumulative number of bytes sent or zero
- * if allocation failed.
- */
-
-static_always_inline u32
-tso_segment_buffer (vlib_main_t * vm, vnet_interface_per_thread_data_t * ptd,
-		    int do_tx_offloads, u32 sbi0, vlib_buffer_t * sb0,
-		    u32 n_bytes_b0)
-{
-  u32 n_tx_bytes = 0;
-  int is_ip4 = sb0->flags & VNET_BUFFER_F_IS_IP4;
-  int is_ip6 = sb0->flags & VNET_BUFFER_F_IS_IP6;
-  ASSERT (is_ip4 || is_ip6);
-  ASSERT (sb0->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID);
-  ASSERT (sb0->flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID);
-  ASSERT (sb0->flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
-  u16 gso_size = vnet_buffer2 (sb0)->gso_size;
-
-  int l4_hdr_sz = vnet_buffer2 (sb0)->gso_l4_hdr_sz;
-  u8 save_tcp_flags = 0;
-  u8 tcp_flags_no_fin_psh = 0;
-  u32 next_tcp_seq = 0;
-
-  tcp_header_t *tcp =
-    (tcp_header_t *) (sb0->data + vnet_buffer (sb0)->l4_hdr_offset);
-  next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number);
-  /* store original flags for last packet and reset FIN and PSH */
-  save_tcp_flags = tcp->flags;
-  tcp_flags_no_fin_psh = tcp->flags & ~(TCP_FLAG_FIN | TCP_FLAG_PSH);
-  tcp->checksum = 0;
-
-  u32 default_bflags =
-    sb0->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT);
-  u16 l234_sz = vnet_buffer (sb0)->l4_hdr_offset + l4_hdr_sz
-    - sb0->current_data;
-  int first_data_size = clib_min (gso_size, sb0->current_length - l234_sz);
-  next_tcp_seq += first_data_size;
-
-  if (PREDICT_FALSE
-      (!tso_alloc_tx_bufs (vm, ptd, sb0, n_bytes_b0, l234_sz, gso_size)))
-    return 0;
-
-  vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[0]);
-  tso_init_buf_from_template_base (b0, sb0, default_bflags,
-				   l234_sz + first_data_size);
-
-  u32 total_src_left = n_bytes_b0 - l234_sz - first_data_size;
-  if (total_src_left)
-    {
-      /* Need to copy more segments */
-      u8 *src_ptr, *dst_ptr;
-      u16 src_left, dst_left;
-      /* current source buffer */
-      vlib_buffer_t *csb0 = sb0;
-      u32 csbi0 = sbi0;
-      /* current dest buffer */
-      vlib_buffer_t *cdb0;
-      u16 dbi = 1;		/* the buffer [0] is b0 */
-
-      src_ptr = vlib_buffer_get_current (sb0) + l234_sz + first_data_size;
-      src_left = sb0->current_length - l234_sz - first_data_size;
-
-      tso_fixup_segmented_buf (b0, tcp_flags_no_fin_psh, is_ip6);
-      if (do_tx_offloads)
-	calc_checksums (vm, b0);
-
-      /* grab a second buffer and prepare the loop */
-      ASSERT (dbi < vec_len (ptd->split_buffers));
-      cdb0 = vlib_get_buffer (vm, ptd->split_buffers[dbi++]);
-      tso_init_buf_from_template (vm, cdb0, b0, l234_sz, gso_size, &dst_ptr,
-				  &dst_left, next_tcp_seq, default_bflags);
-
-      /* an arbitrary large number to catch the runaway loops */
-      int nloops = 2000;
-      while (total_src_left)
-	{
-	  if (nloops-- <= 0)
-	    clib_panic ("infinite loop detected");
-	  u16 bytes_to_copy = clib_min (src_left, dst_left);
-
-	  clib_memcpy_fast (dst_ptr, src_ptr, bytes_to_copy);
-
-	  src_left -= bytes_to_copy;
-	  src_ptr += bytes_to_copy;
-	  total_src_left -= bytes_to_copy;
-	  dst_left -= bytes_to_copy;
-	  dst_ptr += bytes_to_copy;
-	  next_tcp_seq += bytes_to_copy;
-	  cdb0->current_length += bytes_to_copy;
-
-	  if (0 == src_left)
-	    {
-	      int has_next = (csb0->flags & VLIB_BUFFER_NEXT_PRESENT);
-	      u32 next_bi = csb0->next_buffer;
-
-	      /* init src to the next buffer in chain */
-	      if (has_next)
-		{
-		  csbi0 = next_bi;
-		  csb0 = vlib_get_buffer (vm, csbi0);
-		  src_left = csb0->current_length;
-		  src_ptr = vlib_buffer_get_current (csb0);
-		}
-	      else
-		{
-		  ASSERT (total_src_left == 0);
-		  break;
-		}
-	    }
-	  if (0 == dst_left && total_src_left)
-	    {
-	      if (do_tx_offloads)
-		calc_checksums (vm, cdb0);
-	      n_tx_bytes += cdb0->current_length;
-	      ASSERT (dbi < vec_len (ptd->split_buffers));
-	      cdb0 = vlib_get_buffer (vm, ptd->split_buffers[dbi++]);
-	      tso_init_buf_from_template (vm, cdb0, b0, l234_sz,
-					  gso_size, &dst_ptr, &dst_left,
-					  next_tcp_seq, default_bflags);
-	    }
-	}
-
-      tso_fixup_segmented_buf (cdb0, save_tcp_flags, is_ip6);
-      if (do_tx_offloads)
-	calc_checksums (vm, cdb0);
-
-      n_tx_bytes += cdb0->current_length;
-    }
-  n_tx_bytes += b0->current_length;
-  return n_tx_bytes;
-}
-
-static_always_inline void
-drop_one_buffer_and_count (vlib_main_t * vm, vnet_main_t * vnm,
-			   vlib_node_runtime_t * node, u32 * pbi0,
-			   u32 drop_error_code)
-{
-  u32 thread_index = vm->thread_index;
-  vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
-
-  vlib_simple_counter_main_t *cm;
-  cm =
-    vec_elt_at_index (vnm->interface_main.sw_if_counters,
-		      VNET_INTERFACE_COUNTER_TX_ERROR);
-  vlib_increment_simple_counter (cm, thread_index, rt->sw_if_index, 1);
-
-  vlib_error_drop_buffers (vm, node, pbi0,
-			   /* buffer stride */ 1,
-			   /* n_buffers */ 1,
-			   VNET_INTERFACE_OUTPUT_NEXT_DROP,
-			   node->node_index, drop_error_code);
-}
-
 static_always_inline uword
-vnet_interface_output_node_inline_gso (vlib_main_t * vm,
-				       vlib_node_runtime_t * node,
-				       vlib_frame_t * frame,
-				       vnet_main_t * vnm,
-				       vnet_hw_interface_t * hi,
-				       int do_tx_offloads,
-				       int do_segmentation)
+vnet_interface_output_node_inline (vlib_main_t * vm,
+				   vlib_node_runtime_t * node,
+				   vlib_frame_t * frame,
+				   vnet_main_t * vnm,
+				   vnet_hw_interface_t * hi,
+				   int do_tx_offloads)
 {
   vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
   vnet_sw_interface_t *si;
@@ -485,8 +229,6 @@ vnet_interface_output_node_inline_gso (vlib_main_t * vm,
   u32 next_index = VNET_INTERFACE_OUTPUT_NEXT_TX;
   u32 current_config_index = ~0;
   u8 arc = im->output_feature_arc_index;
-  vnet_interface_per_thread_data_t *ptd =
-    vec_elt_at_index (im->per_thread_data, thread_index);
   vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
 
   n_buffers = frame->n_vectors;
@@ -571,12 +313,6 @@ vnet_interface_output_node_inline_gso (vlib_main_t * vm,
 
 	  or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
 
-	  if (do_segmentation)
-	    {
-	      /* go to single loop if we need TSO segmentation */
-	      if (PREDICT_FALSE (or_flags & VNET_BUFFER_F_GSO))
-		break;
-	    }
 	  from += 4;
 	  to_tx += 4;
 	  n_left_to_tx -= 4;
@@ -691,84 +427,6 @@ vnet_interface_output_node_inline_gso (vlib_main_t * vm,
 	      b[0]->current_config_index = current_config_index;
 	    }
 
-	  if (do_segmentation)
-	    {
-	      if (PREDICT_FALSE (b[0]->flags & VNET_BUFFER_F_GSO))
-		{
-		  /*
-		   * Undo the enqueue of the b0 - it is not going anywhere,
-		   * and will be freed either after it's segmented or
-		   * when dropped, if there is no buffers to segment into.
-		   */
-		  to_tx -= 1;
-		  n_left_to_tx += 1;
-		  /* undo the counting. */
-		  n_bytes -= n_bytes_b0;
-		  n_packets -= 1;
-
-		  u32 n_tx_bytes = 0;
-
-		  n_tx_bytes =
-		    tso_segment_buffer (vm, ptd, do_tx_offloads, bi0, b[0],
-					n_bytes_b0);
-
-		  if (PREDICT_FALSE (n_tx_bytes == 0))
-		    {
-		      drop_one_buffer_and_count (vm, vnm, node, from - 1,
-						 VNET_INTERFACE_OUTPUT_ERROR_NO_BUFFERS_FOR_GSO);
-		      b += 1;
-		      continue;
-		    }
-
-		  u16 n_tx_bufs = vec_len (ptd->split_buffers);
-		  u32 *from_tx_seg = ptd->split_buffers;
-
-		  while (n_tx_bufs > 0)
-		    {
-		      if (n_tx_bufs >= n_left_to_tx)
-			{
-			  while (n_left_to_tx > 0)
-			    {
-			      to_tx[0] = from_tx_seg[0];
-			      to_tx += 1;
-			      from_tx_seg += 1;
-			      n_left_to_tx -= 1;
-			      n_tx_bufs -= 1;
-			      n_packets += 1;
-			    }
-			  vlib_put_next_frame (vm, node, next_index,
-					       n_left_to_tx);
-			  vlib_get_new_next_frame (vm, node, next_index,
-						   to_tx, n_left_to_tx);
-			}
-		      while (n_tx_bufs > 0)
-			{
-			  to_tx[0] = from_tx_seg[0];
-			  to_tx += 1;
-			  from_tx_seg += 1;
-			  n_left_to_tx -= 1;
-			  n_tx_bufs -= 1;
-			  n_packets += 1;
-			}
-		    }
-		  n_bytes += n_tx_bytes;
-		  if (PREDICT_FALSE (tx_swif0 != rt->sw_if_index))
-		    {
-
-		      vlib_increment_combined_counter
-			(im->combined_sw_if_counters +
-			 VNET_INTERFACE_COUNTER_TX, thread_index, tx_swif0,
-			 _vec_len (ptd->split_buffers), n_tx_bytes);
-		    }
-		  /* The buffers were enqueued. Reset the length */
-		  _vec_len (ptd->split_buffers) = 0;
-		  /* Free the now segmented buffer */
-		  vlib_buffer_free_one (vm, bi0);
-		  b += 1;
-		  continue;
-		}
-	    }
-
 	  if (PREDICT_FALSE (tx_swif0 != rt->sw_if_index))
 	    {
 
@@ -779,8 +437,13 @@ vnet_interface_output_node_inline_gso (vlib_main_t * vm,
 	    }
 
 	  if (do_tx_offloads)
-	    calc_checksums (vm, b[0]);
-
+	    {
+	      if (b[0]->flags &
+		  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
+		   VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
+		   VNET_BUFFER_F_OFFLOAD_IP_CKSUM))
+		calc_checksums (vm, b[0]);
+	    }
 	  b += 1;
 	}
 
@@ -845,32 +508,6 @@ static_always_inline void vnet_interface_pcap_tx_trace
 }
 
 #ifndef CLIB_MARCH_VARIANT
-static_always_inline uword
-vnet_interface_output_node_inline (vlib_main_t * vm,
-				   vlib_node_runtime_t * node,
-				   vlib_frame_t * frame, vnet_main_t * vnm,
-				   vnet_hw_interface_t * hi,
-				   int do_tx_offloads)
-{
-  /*
-   * The 3-headed "if" is here because we want to err on the side
-   * of not impacting the non-GSO performance - so for the more
-   * common case of no GSO interfaces we want to prevent the
-   * segmentation codepath from being there altogether.
-   */
-  if (PREDICT_TRUE (vnm->interface_main.gso_interface_count == 0))
-    return vnet_interface_output_node_inline_gso (vm, node, frame, vnm, hi,
-						  do_tx_offloads,
-						  /* do_segmentation */ 0);
-  else if (hi->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO)
-    return vnet_interface_output_node_inline_gso (vm, node, frame, vnm, hi,
-						  do_tx_offloads,
-						  /* do_segmentation */ 0);
-  else
-    return vnet_interface_output_node_inline_gso (vm, node, frame, vnm, hi,
-						  do_tx_offloads,
-						  /* do_segmentation */ 1);
-}
 
 uword
 vnet_interface_output_node (vlib_main_t * vm, vlib_node_runtime_t * node,