gso: use the header offsets from buffer metadata

Type: improvement

Change-Id: I955fbef0e0238cb69307e96cd1c677061737e5f3
Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>

2 files changed, 140 insertions, 141 deletions

diff --git a/src/vnet/gso/gso.h b/src/vnet/gso/gso.h
index 883a4941ee2..dee5da5c70b 100644
--- a/src/vnet/gso/gso.h
+++ b/src/vnet/gso/gso.h
@@ -39,13 +39,13 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0,
 				  u32 flags, u16 n_bufs, u16 hdr_sz)
 {
   u32 i = n_bufs;
-  while (i >= 4)
+  while (i >= 6)
     {
       /* prefetches */
       CLIB_PREFETCH (bufs[2], 2 * CLIB_CACHE_LINE_BYTES, LOAD);
       CLIB_PREFETCH (bufs[3], 2 * CLIB_CACHE_LINE_BYTES, LOAD);
-      vlib_prefetch_buffer_data (bufs[2], LOAD);
-      vlib_prefetch_buffer_data (bufs[3], LOAD);
+      vlib_prefetch_buffer_data (bufs[4], LOAD);
+      vlib_prefetch_buffer_data (bufs[5], LOAD);
 
       /* copying objects from cacheline 0 */
       bufs[0]->current_data = 0;
@@ -70,10 +70,26 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0,
       bufs[0]->total_length_not_including_first_buffer = 0;
       bufs[1]->total_length_not_including_first_buffer = 0;
 
+      clib_memcpy_fast (&bufs[0]->opaque2, &b0->opaque2, sizeof (b0->opaque2));
+      clib_memcpy_fast (&bufs[1]->opaque2, &b0->opaque2, sizeof (b0->opaque2));
+
       /* copying data */
       clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz);
       clib_memcpy_fast (bufs[1]->data, vlib_buffer_get_current (b0), hdr_sz);
 
+      /* header offset fixup */
+      vnet_buffer (bufs[0])->l2_hdr_offset -= b0->current_data;
+      vnet_buffer (bufs[0])->l3_hdr_offset -= b0->current_data;
+      vnet_buffer (bufs[0])->l4_hdr_offset -= b0->current_data;
+      vnet_buffer2 (bufs[0])->outer_l3_hdr_offset -= b0->current_data;
+      vnet_buffer2 (bufs[0])->outer_l4_hdr_offset -= b0->current_data;
+
+      vnet_buffer (bufs[1])->l2_hdr_offset -= b0->current_data;
+      vnet_buffer (bufs[1])->l3_hdr_offset -= b0->current_data;
+      vnet_buffer (bufs[1])->l4_hdr_offset -= b0->current_data;
+      vnet_buffer2 (bufs[1])->outer_l3_hdr_offset -= b0->current_data;
+      vnet_buffer2 (bufs[1])->outer_l4_hdr_offset -= b0->current_data;
+
       bufs += 2;
       i -= 2;
     }
@@ -92,10 +108,18 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0,
       /* copying objects from cacheline 1 */
       bufs[0]->trace_handle = b0->trace_handle;
       bufs[0]->total_length_not_including_first_buffer = 0;
+      clib_memcpy_fast (&bufs[0]->opaque2, &b0->opaque2, sizeof (b0->opaque2));
 
       /* copying data */
       clib_memcpy_fast (bufs[0]->data, vlib_buffer_get_current (b0), hdr_sz);
 
+      /* header offset fixup */
+      vnet_buffer (bufs[0])->l2_hdr_offset -= b0->current_data;
+      vnet_buffer (bufs[0])->l3_hdr_offset -= b0->current_data;
+      vnet_buffer (bufs[0])->l4_hdr_offset -= b0->current_data;
+      vnet_buffer2 (bufs[0])->outer_l3_hdr_offset -= b0->current_data;
+      vnet_buffer2 (bufs[0])->outer_l4_hdr_offset -= b0->current_data;
+
       bufs++;
       i--;
     }
@@ -103,28 +127,41 @@ gso_init_bufs_from_template_base (vlib_buffer_t **bufs, vlib_buffer_t *b0,
 
 static_always_inline void
 gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq,
-			 int is_l2, int is_ip6, generic_header_offset_t *gho,
-			 clib_ip_csum_t *c, u8 tcp_flags)
+			 int is_l2, u8 oflags, u16 hdr_sz, u16 l4_hdr_sz,
+			 clib_ip_csum_t *c, u8 tcp_flags, u8 is_prefetch,
+			 vlib_buffer_t *b1)
 {
 
-  ip4_header_t *ip4 =
-    (ip4_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset +
-		      gho->outer_hdr_sz);
-  ip6_header_t *ip6 =
-    (ip6_header_t *) (vlib_buffer_get_current (b0) + gho->l3_hdr_offset +
-		      gho->outer_hdr_sz);
-  tcp_header_t *tcp =
-    (tcp_header_t *) (vlib_buffer_get_current (b0) + gho->l4_hdr_offset +
-		      gho->outer_hdr_sz);
+  i16 l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset;
+  i16 l4_hdr_offset = vnet_buffer (b0)->l4_hdr_offset;
+
+  ip4_header_t *ip4 = (ip4_header_t *) (b0->data + l3_hdr_offset);
+  ip6_header_t *ip6 = (ip6_header_t *) (b0->data + l3_hdr_offset);
+  tcp_header_t *tcp = (tcp_header_t *) (b0->data + l4_hdr_offset);
 
   tcp->flags = tcp_flags;
   tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq);
   c->odd = 0;
 
-  if (is_ip6)
+  if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
+    {
+      ip4->length =
+	clib_host_to_net_u16 (b0->current_length - hdr_sz +
+			      (l4_hdr_offset - l3_hdr_offset) + l4_hdr_sz);
+      ip4->checksum = 0;
+      ip4->checksum = ip4_header_checksum (ip4);
+      vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
+					    VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
+      c->sum += clib_mem_unaligned (&ip4->src_address, u32);
+      c->sum += clib_mem_unaligned (&ip4->dst_address, u32);
+      c->sum += clib_host_to_net_u32 (
+	(clib_net_to_host_u16 (ip4->length) - ip4_header_bytes (ip4)) +
+	(ip4->protocol << 16));
+    }
+  else
     {
-      ip6->payload_length = clib_host_to_net_u16 (
-	b0->current_length - gho->l4_hdr_offset - gho->outer_hdr_sz);
+      ip6->payload_length =
+	clib_host_to_net_u16 (b0->current_length - hdr_sz + l4_hdr_sz);
       vnet_buffer_offload_flags_clear (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
       ip6_psh_t psh = { 0 };
       u32 *p = (u32 *) &psh;
@@ -135,24 +172,15 @@ gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq,
       for (int i = 0; i < 10; i++)
 	c->sum += p[i];
     }
-  else
-    {
-      ip4->length = clib_host_to_net_u16 (
-	b0->current_length - gho->l3_hdr_offset - gho->outer_hdr_sz);
-      if (gho->gho_flags & GHO_F_IP4)
-	ip4->checksum = ip4_header_checksum (ip4);
-      vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
-					    VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
-      c->sum += clib_mem_unaligned (&ip4->src_address, u32);
-      c->sum += clib_mem_unaligned (&ip4->dst_address, u32);
-      c->sum += clib_host_to_net_u32 (
-	(clib_net_to_host_u16 (ip4->length) - ip4_header_bytes (ip4)) +
-	(ip4->protocol << 16));
-    }
-  clib_ip_csum_chunk (c, (u8 *) tcp, gho->l4_hdr_sz);
+
+  if (is_prefetch)
+    CLIB_PREFETCH (vlib_buffer_get_current (b1) + hdr_sz,
+		   CLIB_CACHE_LINE_BYTES, LOAD);
+
+  clib_ip_csum_chunk (c, (u8 *) tcp, l4_hdr_sz);
   tcp->checksum = clib_ip_csum_fold (c);
 
-  if (!is_l2 && ((gho->gho_flags & GHO_F_TUNNEL) == 0))
+  if (!is_l2 && ((oflags & VNET_BUFFER_OFFLOAD_F_TNL_MASK) == 0))
     {
       u32 adj_index0 = vnet_buffer (b0)->ip.adj_index[VLIB_TX];
 
@@ -169,16 +197,20 @@ gso_fixup_segmented_buf (vlib_main_t *vm, vlib_buffer_t *b0, u32 next_tcp_seq,
 static_always_inline u32
 gso_segment_buffer_inline (vlib_main_t *vm,
 			   vnet_interface_per_thread_data_t *ptd,
-			   vlib_buffer_t *b, generic_header_offset_t *gho,
-			   int is_l2, int is_ip6)
+			   vlib_buffer_t *b, int is_l2)
 {
   vlib_buffer_t **bufs = 0;
   u32 n_tx_bytes = 0;
+
+  u8 oflags = vnet_buffer (b)->oflags;
+  i16 l4_hdr_offset = vnet_buffer (b)->l4_hdr_offset;
   u16 gso_size = vnet_buffer2 (b)->gso_size;
+  u16 l4_hdr_sz = vnet_buffer2 (b)->gso_l4_hdr_sz;
+
   u8 tcp_flags = 0, tcp_flags_no_fin_psh = 0;
   u32 default_bflags =
     b->flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT);
-  u16 hdr_sz = gho->hdr_sz + gho->outer_hdr_sz;
+  u16 hdr_sz = (l4_hdr_offset - b->current_data) + l4_hdr_sz;
   u32 next_tcp_seq = 0, tcp_seq = 0;
   u32 data_size = vlib_buffer_length_in_chain (vm, b) - hdr_sz;
   u16 size =
@@ -200,9 +232,8 @@ gso_segment_buffer_inline (vlib_main_t *vm,
   vec_validate (bufs, n_bufs - 1);
   vlib_get_buffers (vm, ptd->split_buffers, bufs, n_bufs);
 
-  tcp_header_t *tcp =
-    (tcp_header_t *) (vlib_buffer_get_current (b) + gho->l4_hdr_offset +
-		      gho->outer_hdr_sz);
+  tcp_header_t *tcp = (tcp_header_t *) (b->data + l4_hdr_offset);
+
   tcp_seq = next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number);
   /* store original flags for last packet and reset FIN and PSH */
   tcp_flags = tcp->flags;
@@ -247,11 +278,11 @@ gso_segment_buffer_inline (vlib_main_t *vm,
       if (0 == dst_left && data_size)
 	{
 	  vlib_prefetch_buffer_header (bufs[i + 1], LOAD);
-	  vlib_prefetch_buffer_data (bufs[i + 1], LOAD);
 
 	  n_tx_bytes += bufs[i]->current_length;
-	  gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho,
-				   &c, tcp_flags_no_fin_psh);
+	  gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, oflags, hdr_sz,
+				   l4_hdr_sz, &c, tcp_flags_no_fin_psh, 1,
+				   bufs[i + 1]);
 	  i++;
 	  dst_left = size;
 	  dst_ptr = vlib_buffer_get_current (bufs[i]) + hdr_sz;
@@ -264,8 +295,8 @@ gso_segment_buffer_inline (vlib_main_t *vm,
 
   ASSERT ((i + 1) == n_alloc);
   n_tx_bytes += bufs[i]->current_length;
-  gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, is_ip6, gho, &c,
-			   tcp_flags);
+  gso_fixup_segmented_buf (vm, bufs[i], tcp_seq, is_l2, oflags, hdr_sz,
+			   l4_hdr_sz, &c, tcp_flags, 0, NULL);
 
   vec_free (bufs);
   return n_tx_bytes;
diff --git a/src/vnet/gso/node.c b/src/vnet/gso/node.c
index 910f1585e7f..c1d4459476e 100644
--- a/src/vnet/gso/node.c
+++ b/src/vnet/gso/node.c
@@ -80,113 +80,108 @@ format_gso_trace (u8 * s, va_list * args)
   return s;
 }
 
-static_always_inline u16
-tso_segment_ipip_tunnel_fixup (vlib_main_t * vm,
-			       vnet_interface_per_thread_data_t * ptd,
-			       vlib_buffer_t * sb0,
-			       generic_header_offset_t * gho)
+static_always_inline void
+tso_segment_ipip_tunnel_fixup (vlib_main_t *vm,
+			       vnet_interface_per_thread_data_t *ptd,
+			       vlib_buffer_t *sb0)
 {
   u16 n_tx_bufs = vec_len (ptd->split_buffers);
-  u16 i = 0, n_tx_bytes = 0;
+  u16 i = 0;
 
   while (i < n_tx_bufs)
     {
       vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[i]);
+      i16 outer_l3_hdr_offset = vnet_buffer2 (b0)->outer_l3_hdr_offset;
+      i16 l3_hdr_offset = vnet_buffer (b0)->l3_hdr_offset;
 
-      ip4_header_t *ip4 =
-	(ip4_header_t *) (vlib_buffer_get_current (b0) +
-			  gho->outer_l3_hdr_offset);
-      ip6_header_t *ip6 =
-	(ip6_header_t *) (vlib_buffer_get_current (b0) +
-			  gho->outer_l3_hdr_offset);
+      ip4_header_t *ip4 = (ip4_header_t *) (b0->data + outer_l3_hdr_offset);
+      ip6_header_t *ip6 = (ip6_header_t *) (b0->data + outer_l3_hdr_offset);
 
-      if (gho->gho_flags & GHO_F_OUTER_IP4)
+      if (vnet_buffer (b0)->oflags & VNET_BUFFER_OFFLOAD_F_OUTER_IP_CKSUM)
 	{
-	  ip4->length =
-	    clib_host_to_net_u16 (b0->current_length -
-				  gho->outer_l3_hdr_offset);
+	  ip4->length = clib_host_to_net_u16 (
+	    b0->current_length - (outer_l3_hdr_offset - b0->current_data));
 	  ip4->checksum = ip4_header_checksum (ip4);
+	  vnet_buffer_offload_flags_clear (
+	    b0, VNET_BUFFER_OFFLOAD_F_OUTER_IP_CKSUM |
+		  VNET_BUFFER_OFFLOAD_F_TNL_IPIP);
 	}
-      else if (gho->gho_flags & GHO_F_OUTER_IP6)
+      else
 	{
-	  ip6->payload_length =
-	    clib_host_to_net_u16 (b0->current_length -
-				  gho->outer_l4_hdr_offset);
+	  ip6->payload_length = clib_host_to_net_u16 (
+	    b0->current_length - (l3_hdr_offset - b0->current_data));
+	  vnet_buffer_offload_flags_clear (b0, VNET_BUFFER_OFFLOAD_F_TNL_IPIP);
 	}
 
-      n_tx_bytes += gho->outer_hdr_sz;
       i++;
     }
-  return n_tx_bytes;
 }
 
 static_always_inline void
-tso_segment_vxlan_tunnel_headers_fixup (vlib_main_t * vm, vlib_buffer_t * b,
-					generic_header_offset_t * gho)
+tso_segment_vxlan_tunnel_headers_fixup (vlib_main_t *vm, vlib_buffer_t *b)
 {
-  u8 proto = 0;
   ip4_header_t *ip4 = 0;
   ip6_header_t *ip6 = 0;
   udp_header_t *udp = 0;
+  i16 outer_l3_hdr_offset = vnet_buffer2 (b)->outer_l3_hdr_offset;
+  i16 outer_l4_hdr_offset = vnet_buffer2 (b)->outer_l4_hdr_offset;
 
-  ip4 =
-    (ip4_header_t *) (vlib_buffer_get_current (b) + gho->outer_l3_hdr_offset);
-  ip6 =
-    (ip6_header_t *) (vlib_buffer_get_current (b) + gho->outer_l3_hdr_offset);
-  udp =
-    (udp_header_t *) (vlib_buffer_get_current (b) + gho->outer_l4_hdr_offset);
+  ip4 = (ip4_header_t *) (b->data + outer_l3_hdr_offset);
+  ip6 = (ip6_header_t *) (b->data + outer_l3_hdr_offset);
+  udp = (udp_header_t *) (b->data + outer_l4_hdr_offset);
 
-  if (gho->gho_flags & GHO_F_OUTER_IP4)
+  if (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_OUTER_IP_CKSUM)
     {
-      proto = ip4->protocol;
-      ip4->length =
-	clib_host_to_net_u16 (b->current_length - gho->outer_l3_hdr_offset);
+      ip4->length = clib_host_to_net_u16 (
+	b->current_length - (outer_l3_hdr_offset - b->current_data));
       ip4->checksum = ip4_header_checksum (ip4);
+      if (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_OUTER_UDP_CKSUM)
+	{
+	  udp->length = clib_host_to_net_u16 (
+	    b->current_length - (outer_l4_hdr_offset - b->current_data));
+	  // udp checksum is 0, in udp tunnel
+	  udp->checksum = 0;
+	}
+      vnet_buffer_offload_flags_clear (
+	b, VNET_BUFFER_OFFLOAD_F_OUTER_IP_CKSUM |
+	     VNET_BUFFER_OFFLOAD_F_OUTER_UDP_CKSUM |
+	     VNET_BUFFER_OFFLOAD_F_TNL_VXLAN);
     }
-  else if (gho->gho_flags & GHO_F_OUTER_IP6)
-    {
-      proto = ip6->protocol;
-      ip6->payload_length =
-	clib_host_to_net_u16 (b->current_length - gho->outer_l4_hdr_offset);
-    }
-  if (proto == IP_PROTOCOL_UDP)
+  else
     {
-      int bogus;
-      udp->length =
-	clib_host_to_net_u16 (b->current_length - gho->outer_l4_hdr_offset);
-      udp->checksum = 0;
-      if (gho->gho_flags & GHO_F_OUTER_IP6)
+      ip6->payload_length = clib_host_to_net_u16 (
+	b->current_length - (outer_l4_hdr_offset - b->current_data));
+
+      if (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_OUTER_UDP_CKSUM)
 	{
+	  int bogus;
+	  udp->length = ip6->payload_length;
+	  // udp checksum is 0, in udp tunnel
+	  udp->checksum = 0;
 	  udp->checksum =
 	    ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
+	  vnet_buffer_offload_flags_clear (
+	    b, VNET_BUFFER_OFFLOAD_F_OUTER_UDP_CKSUM |
+		 VNET_BUFFER_OFFLOAD_F_TNL_VXLAN);
 	}
-      else if (gho->gho_flags & GHO_F_OUTER_IP4)
-	{
-	  udp->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
-	}
-      /* FIXME: it should be OUTER_UDP_CKSUM */
-      vnet_buffer_offload_flags_clear (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
     }
 }
 
-static_always_inline u16
-tso_segment_vxlan_tunnel_fixup (vlib_main_t * vm,
-				vnet_interface_per_thread_data_t * ptd,
-				vlib_buffer_t * sb0,
-				generic_header_offset_t * gho)
+static_always_inline void
+tso_segment_vxlan_tunnel_fixup (vlib_main_t *vm,
+				vnet_interface_per_thread_data_t *ptd,
+				vlib_buffer_t *sb0)
 {
   u16 n_tx_bufs = vec_len (ptd->split_buffers);
-  u16 i = 0, n_tx_bytes = 0;
+  u16 i = 0;
 
   while (i < n_tx_bufs)
     {
       vlib_buffer_t *b0 = vlib_get_buffer (vm, ptd->split_buffers[i]);
 
-      tso_segment_vxlan_tunnel_headers_fixup (vm, b0, gho);
-      n_tx_bytes += gho->outer_hdr_sz;
+      tso_segment_vxlan_tunnel_headers_fixup (vm, b0);
       i++;
     }
-  return n_tx_bytes;
 }
 
 static_always_inline u16
@@ -682,32 +677,10 @@ vnet_gso_node_inline (vlib_main_t * vm,
 		  to_next -= 1;
 		  n_left_to_next += 1;
 		  /* undo the counting. */
-		  generic_header_offset_t gho = { 0 };
 		  u32 n_tx_bytes = 0;
-		  u32 inner_is_ip6 = is_ip6;
-
-		  vnet_generic_header_offset_parser (b[0], &gho, is_l2,
-						     is_ip4, is_ip6);
-
-		  if (PREDICT_FALSE (gho.gho_flags & GHO_F_TUNNEL))
-		    {
-		      if (PREDICT_FALSE
-			  (gho.gho_flags & (GHO_F_GRE_TUNNEL |
-					    GHO_F_GENEVE_TUNNEL)))
-			{
-			  /* not supported yet */
-			  drop_one_buffer_and_count (vm, vnm, node, from - 1,
-						     hi->sw_if_index,
-						     GSO_ERROR_UNHANDLED_TYPE);
-			  b += 1;
-			  continue;
-			}
 
-		      inner_is_ip6 = (gho.gho_flags & GHO_F_IP6) != 0;
-		    }
-
-		  n_tx_bytes = gso_segment_buffer_inline (vm, ptd, b[0], &gho,
-							  is_l2, inner_is_ip6);
+		  n_tx_bytes =
+		    gso_segment_buffer_inline (vm, ptd, b[0], is_l2);
 
 		  if (PREDICT_FALSE (n_tx_bytes == 0))
 		    {
@@ -718,19 +691,15 @@ vnet_gso_node_inline (vlib_main_t * vm,
 		      continue;
 		    }
 
-
-		  if (PREDICT_FALSE (gho.gho_flags & GHO_F_VXLAN_TUNNEL))
+		  if (PREDICT_FALSE (vnet_buffer (b[0])->oflags &
+				     VNET_BUFFER_OFFLOAD_F_TNL_VXLAN))
 		    {
-		      n_tx_bytes +=
-			tso_segment_vxlan_tunnel_fixup (vm, ptd, b[0], &gho);
+		      tso_segment_vxlan_tunnel_fixup (vm, ptd, b[0]);
 		    }
-		  else
-		    if (PREDICT_FALSE
-			(gho.gho_flags & (GHO_F_IPIP_TUNNEL |
-					  GHO_F_IPIP6_TUNNEL)))
+		  else if (PREDICT_FALSE (vnet_buffer (b[0])->oflags &
+					  VNET_BUFFER_OFFLOAD_F_TNL_IPIP))
 		    {
-		      n_tx_bytes +=
-			tso_segment_ipip_tunnel_fixup (vm, ptd, b[0], &gho);
+		      tso_segment_ipip_tunnel_fixup (vm, ptd, b[0]);
 		    }
 
 		  u16 n_tx_bufs = vec_len (ptd->split_buffers);
@@ -744,7 +713,6 @@ vnet_gso_node_inline (vlib_main_t * vm,
 			{
 			  sbi0 = to_next[0] = from_seg[0];
 			  sb0 = vlib_get_buffer (vm, sbi0);
-			  vnet_buffer_offload_flags_clear (sb0, 0x7F);
 			  ASSERT (sb0->current_length > 0);
 			  to_next += 1;
 			  from_seg += 1;