TCP/UDP checksum offload API

Change-Id: I2cb6ce4e29813f6602b14e6e61713fb381fbcef8
Signed-off-by: Dave Barach <dave@barachs.net>

9 files changed, 173 insertions, 13 deletions

diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index 8801bfd3..c755060d 100644
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -335,6 +335,37 @@ dpdk_buffer_recycle (vlib_main_t * vm, vlib_node_runtime_t * node,
   vec_add1 (dm->recycle[my_cpu], bi);
 }
 
+static_always_inline void
+dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b,
+			struct rte_mbuf *mb)
+{
+  u32 ip_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+  u32 tcp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+  u32 udp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+  int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
+  u64 ol_flags;
+
+  /* Is there any work for us? */
+  if (PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum) == 0))
+    return;
+
+  mb->l2_len = vnet_buffer (b)->l3_hdr_offset - b->current_data;
+  mb->l3_len = vnet_buffer (b)->l4_hdr_offset -
+    vnet_buffer (b)->l3_hdr_offset;
+  mb->outer_l3_len = 0;
+  mb->outer_l2_len = 0;
+  ol_flags = is_ip4 ? PKT_TX_IPV4 : PKT_TX_IPV6;
+  ol_flags |= ip_cksum ? PKT_TX_IP_CKSUM : 0;
+  ol_flags |= tcp_cksum ? PKT_TX_TCP_CKSUM : 0;
+  ol_flags |= udp_cksum ? PKT_TX_UDP_CKSUM : 0;
+  mb->ol_flags |= ol_flags;
+
+  /* we are trying to help compiler here by using local ol_flags with known
+     state of all flags */
+  if (xd->flags & DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM)
+    rte_net_intel_cksum_flags_prepare (mb, ol_flags);
+}
+
 /*
  * Transmits the packets on the frame to the interface associated with the
  * node. It first copies packets on the frame to a tx_vector containing the
@@ -455,6 +486,15 @@ dpdk_interface_tx (vlib_main_t * vm,
       mb2 = rte_mbuf_from_vlib_buffer (b2);
       mb3 = rte_mbuf_from_vlib_buffer (b3);
 
+      if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
+			 (or_flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)))
+	{
+	  dpdk_buffer_tx_offload (xd, b0, mb0);
+	  dpdk_buffer_tx_offload (xd, b1, mb1);
+	  dpdk_buffer_tx_offload (xd, b2, mb2);
+	  dpdk_buffer_tx_offload (xd, b3, mb3);
+	}
+
       if (PREDICT_FALSE (or_flags & VLIB_BUFFER_RECYCLE))
 	{
 	  dpdk_buffer_recycle (vm, node, b0, bi0, &mb0);
@@ -521,6 +561,7 @@ dpdk_interface_tx (vlib_main_t * vm,
       dpdk_validate_rte_mbuf (vm, b0, 1);
 
       mb0 = rte_mbuf_from_vlib_buffer (b0);
+      dpdk_buffer_tx_offload (xd, b0, mb0);
       dpdk_buffer_recycle (vm, node, b0, bi0, &mb0);
 
       if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h
index 55f63b37..29a2c760 100644
--- a/src/plugins/dpdk/device/dpdk.h
+++ b/src/plugins/dpdk/device/dpdk.h
@@ -38,6 +38,7 @@
 #include <rte_version.h>
 #include <rte_eth_bond.h>
 #include <rte_sched.h>
+#include <rte_net.h>
 
 #include <vnet/unix/pcap.h>
 #include <vnet/devices/devices.h>
@@ -176,6 +177,8 @@ typedef struct
 #define DPDK_DEVICE_FLAG_HQOS               (1 << 6)
 #define DPDK_DEVICE_FLAG_BOND_SLAVE         (1 << 7)
 #define DPDK_DEVICE_FLAG_BOND_SLAVE_UP      (1 << 8)
+#define DPDK_DEVICE_FLAG_TX_OFFLOAD         (1 << 9)
+#define DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM   (1 << 10)
 
   u16 nb_tx_desc;
     CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c
index 7ca3d358..8a708035 100755
--- a/src/plugins/dpdk/device/init.c
+++ b/src/plugins/dpdk/device/init.c
@@ -351,6 +351,11 @@ dpdk_lib_init (dpdk_main_t * dm)
 	    case VNET_DPDK_PMD_IGB:
 	    case VNET_DPDK_PMD_IXGBE:
 	    case VNET_DPDK_PMD_I40E:
+	      xd->port_type = port_type_from_speed_capa (&dev_info);
+	      xd->flags |= DPDK_DEVICE_FLAG_TX_OFFLOAD |
+		DPDK_DEVICE_FLAG_INTEL_PHDR_CKSUM;
+
+	      break;
 	    case VNET_DPDK_PMD_CXGBE:
 	    case VNET_DPDK_PMD_MLX4:
 	    case VNET_DPDK_PMD_MLX5:
@@ -575,6 +580,9 @@ dpdk_lib_init (dpdk_main_t * dm)
 
       hi = vnet_get_hw_interface (dm->vnet_main, xd->hw_if_index);
 
+      if (xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD)
+	hi->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD;
+
       dpdk_device_setup (xd);
 
       if (vec_len (xd->errors))
diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h
index 52dada30..56cb07ae 100644
--- a/src/vnet/buffer.h
+++ b/src/vnet/buffer.h
@@ -47,9 +47,14 @@
   _( 2, L4_CHECKSUM_CORRECT)				\
   _( 3, VLAN_2_DEEP)					\
   _( 4, VLAN_1_DEEP)					\
+  _( 8, SPAN_CLONE)					\
   _( 6, HANDOFF_NEXT_VALID)				\
   _( 7, LOCALLY_ORIGINATED)				\
-  _( 8, SPAN_CLONE)
+  _( 8, IS_IP4)						\
+  _( 9, IS_IP6)						\
+  _(10, OFFLOAD_IP_CKSUM)				\
+  _(11, OFFLOAD_TCP_CKSUM)				\
+  _(12, OFFLOAD_UDP_CKSUM)
 
 #define VNET_BUFFER_FLAGS_VLAN_BITS \
   (VNET_BUFFER_F_VLAN_1_DEEP | VNET_BUFFER_F_VLAN_2_DEEP)
diff --git a/src/vnet/interface.h b/src/vnet/interface.h
index 9d64fc28..fb75ff34 100644
--- a/src/vnet/interface.h
+++ b/src/vnet/interface.h
@@ -419,6 +419,9 @@ typedef struct vnet_hw_interface_t
   /* rx mode flags */
 #define VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE (1 << 10)
 
+  /* tx checksum offload */
+#define VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD (1 << 11)
+
   /* Hardware address as vector.  Zero (e.g. zero-length vector) if no
      address for this class (e.g. PPP). */
   u8 *hw_address;
diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c
index cdf18738..c95f08b1 100644
--- a/src/vnet/interface_output.c
+++ b/src/vnet/interface_output.c
@@ -38,6 +38,10 @@
  */
 
 #include <vnet/vnet.h>
+#include <vnet/ip/icmp46_packet.h>
+#include <vnet/ip/ip4.h>
+#include <vnet/ip/ip6.h>
+#include <vnet/udp/udp_packet.h>
 #include <vnet/feature/feature.h>
 
 typedef struct
@@ -153,14 +157,58 @@ vnet_interface_output_trace (vlib_main_t * vm,
     }
 }
 
-uword
-vnet_interface_output_node (vlib_main_t * vm,
-			    vlib_node_runtime_t * node, vlib_frame_t * frame)
+static_always_inline void
+calc_checksums (vlib_main_t * vm, vlib_buffer_t * b)
+{
+  ip4_header_t *ip4;
+  ip6_header_t *ip6;
+  tcp_header_t *th;
+  udp_header_t *uh;
+
+  int is_ip4 = (b->flags & VNET_BUFFER_F_IS_IP4) != 0;
+  int is_ip6 = (b->flags & VNET_BUFFER_F_IS_IP6) != 0;
+
+  ASSERT (!(is_ip4 && is_ip6));
+
+  ip4 = (ip4_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
+  ip6 = (ip6_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
+  th = (tcp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
+  uh = (udp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
+
+  if (is_ip4)
+    {
+      ip4 = (ip4_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
+      if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+	ip4->checksum = ip4_header_checksum (ip4);
+      if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+	th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
+      if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+	uh->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
+    }
+  if (is_ip6)
+    {
+      int bogus;
+      ASSERT (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
+      if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+	th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
+      if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+	uh->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
+    }
+
+  b->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+  b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+  b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+}
+
+static_always_inline uword
+vnet_interface_output_node_inline (vlib_main_t * vm,
+				   vlib_node_runtime_t * node,
+				   vlib_frame_t * frame, vnet_main_t * vnm,
+				   vnet_hw_interface_t * hi,
+				   int do_tx_offloads)
 {
-  vnet_main_t *vnm = vnet_get_main ();
   vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
   vnet_sw_interface_t *si;
-  vnet_hw_interface_t *hi;
   u32 n_left_to_tx, *from, *from_end, *to_tx;
   u32 n_bytes, n_buffers, n_packets;
   u32 n_bytes_b0, n_bytes_b1, n_bytes_b2, n_bytes_b3;
@@ -234,6 +282,7 @@ vnet_interface_output_node (vlib_main_t * vm,
 	  u32 bi0, bi1, bi2, bi3;
 	  vlib_buffer_t *b0, *b1, *b2, *b3;
 	  u32 tx_swif0, tx_swif1, tx_swif2, tx_swif3;
+	  u32 or_flags;
 
 	  /* Prefetch next iteration. */
 	  vlib_prefetch_buffer_with_index (vm, from[4], LOAD);
@@ -324,6 +373,22 @@ vnet_interface_output_node (vlib_main_t * vm,
 					       thread_index, tx_swif3, 1,
 					       n_bytes_b3);
 	    }
+
+	  or_flags = b0->flags | b1->flags | b2->flags | b3->flags;
+
+	  if (do_tx_offloads)
+	    {
+	      if (or_flags &
+		  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
+		   VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
+		   VNET_BUFFER_F_OFFLOAD_IP_CKSUM))
+		{
+		  calc_checksums (vm, b0);
+		  calc_checksums (vm, b1);
+		  calc_checksums (vm, b2);
+		  calc_checksums (vm, b3);
+		}
+	    }
 	}
 
       while (from + 1 <= from_end && n_left_to_tx >= 1)
@@ -363,6 +428,9 @@ vnet_interface_output_node (vlib_main_t * vm,
 					       thread_index, tx_swif0, 1,
 					       n_bytes_b0);
 	    }
+
+	  if (do_tx_offloads)
+	    calc_checksums (vm, b0);
 	}
 
       vlib_put_next_frame (vm, node, next_index, n_left_to_tx);
@@ -376,6 +444,23 @@ vnet_interface_output_node (vlib_main_t * vm,
   return n_buffers;
 }
 
+static_always_inline uword
+vnet_interface_output_node (vlib_main_t * vm, vlib_node_runtime_t * node,
+			    vlib_frame_t * frame)
+{
+  vnet_main_t *vnm = vnet_get_main ();
+  vnet_hw_interface_t *hi;
+  vnet_interface_output_runtime_t *rt = (void *) node->runtime_data;
+  hi = vnet_get_sup_hw_interface (vnm, rt->sw_if_index);
+
+  if (hi->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD)
+    return vnet_interface_output_node_inline (vm, node, frame, vnm, hi,
+					      /* do_tx_offloads */ 0);
+  else
+    return vnet_interface_output_node_inline (vm, node, frame, vnm, hi,
+					      /* do_tx_offloads */ 1);
+}
+
 VLIB_NODE_FUNCTION_MULTIARCH_CLONE (vnet_interface_output_node);
 CLIB_MULTIARCH_SELECT_FN (vnet_interface_output_node);
 
diff --git a/src/vnet/ip/ip4.h b/src/vnet/ip/ip4.h
index 71640def..19198bcb 100644
--- a/src/vnet/ip/ip4.h
+++ b/src/vnet/ip/ip4.h
@@ -42,6 +42,7 @@
 
 #include <vnet/ip/ip4_packet.h>
 #include <vnet/ip/lookup.h>
+#include <vnet/buffer.h>
 #include <vnet/feature/feature.h>
 
 typedef struct ip4_mfib_t
@@ -346,7 +347,8 @@ vlib_buffer_push_ip4 (vlib_main_t * vm, vlib_buffer_t * b,
   ih->src_address.as_u32 = src->as_u32;
   ih->dst_address.as_u32 = dst->as_u32;
 
-  ih->checksum = ip4_header_checksum (ih);
+  ih->checksum = 0;
+  b->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_IS_IP4;
   return ih;
 }
 #endif /* included_ip_ip4_h */
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index 8dd927d4..4a9e7919 100755
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -2457,7 +2457,8 @@ ip4_rewrite_inline (vlib_main_t * vm,
 		}
 
 	      /* Verify checksum. */
-	      ASSERT (ip0->checksum == ip4_header_checksum (ip0));
+	      ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
+		      (p0->flags | VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
 	    }
 	  else
 	    {
@@ -2492,7 +2493,8 @@ ip4_rewrite_inline (vlib_main_t * vm,
 		}
 
 	      /* Verify checksum. */
-	      ASSERT (ip1->checksum == ip4_header_checksum (ip1));
+	      ASSERT ((ip1->checksum == ip4_header_checksum (ip1)) ||
+		      (p1->flags | VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
 	    }
 	  else
 	    {
@@ -2630,7 +2632,8 @@ ip4_rewrite_inline (vlib_main_t * vm,
 
 	      ip0->ttl = ttl0;
 
-	      ASSERT (ip0->checksum == ip4_header_checksum (ip0));
+	      ASSERT ((ip0->checksum == ip4_header_checksum (ip0)) ||
+		      (p0->flags | VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
 
 	      if (PREDICT_FALSE (ttl0 <= 0))
 		{
diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index f34eb797..35f3eba1 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -1485,7 +1485,12 @@ tcp46_output_inline (vlib_main_t * vm,
 	      ip4_header_t *ih0;
 	      ih0 = vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4,
 					  &tc0->c_rmt_ip4, IP_PROTOCOL_TCP);
-	      th0->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ih0);
+	      b0->flags |= VNET_BUFFER_F_IS_IP4 |
+		VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
+		VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+	      vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data;
+	      vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
+	      th0->checksum = 0;
 	    }
 	  else
 	    {
@@ -1494,8 +1499,13 @@ tcp46_output_inline (vlib_main_t * vm,
 
 	      ih0 = vlib_buffer_push_ip6 (vm, b0, &tc0->c_lcl_ip6,
 					  &tc0->c_rmt_ip6, IP_PROTOCOL_TCP);
-	      th0->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b0, ih0,
-								 &bogus);
+
+	      b0->flags |= VNET_BUFFER_F_IS_IP6 |
+		VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
+		VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+	      vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data;
+	      vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
+	      th0->checksum = 0;
 	      ASSERT (!bogus);
 	    }