avf: add L4 checksum offload support

Change-Id: Ie90437cf597c5d53ab1cc41ea7db15b97614e2fc
Signed-off-by: Mohammed Hawari <mohammed@hawari.fr>
Type: improvement

3 files changed, 136 insertions, 6 deletions

diff --git a/src/plugins/avf/avf.h b/src/plugins/avf/avf.h
index 57e71263f9e..66a791ec9b2 100644
--- a/src/plugins/avf/avf.h
+++ b/src/plugins/avf/avf.h
@@ -46,10 +46,26 @@
 #define AVF_RXD_ERROR_L4E		(1ULL << (AVF_RXD_ERROR_SHIFT + 4))
 
 #define AVF_TXD_CMD(x)			(1 << (x + 4))
+#define AVF_TXD_CMD_EXT(x, val)         ((u64)val << (x + 4))
 #define AVF_TXD_CMD_EOP			AVF_TXD_CMD(0)
 #define AVF_TXD_CMD_RS			AVF_TXD_CMD(1)
 #define AVF_TXD_CMD_RSV			AVF_TXD_CMD(2)
 
+#define AVF_TXD_CMD_IIPT_NONE           AVF_TXD_CMD_EXT(5, 0)
+#define AVF_TXD_CMD_IIPT_IPV6           AVF_TXD_CMD_EXT(5, 1)
+#define AVF_TXD_CMD_IIPT_IPV4_NO_CSUM   AVF_TXD_CMD_EXT(5, 2)
+#define AVF_TXD_CMD_IIPT_IPV4           AVF_TXD_CMD_EXT(5, 3)
+
+#define AVF_TXD_CMD_L4T_UNKNOWN         AVF_TXD_CMD_EXT(8, 0)
+#define AVF_TXD_CMD_L4T_TCP             AVF_TXD_CMD_EXT(8, 1)
+#define AVF_TXD_CMD_L4T_SCTP            AVF_TXD_CMD_EXT(8, 2)
+#define AVF_TXD_CMD_L4T_UDP             AVF_TXD_CMD_EXT(8, 3)
+
+#define AVF_TXD_OFFSET(x,factor,val)    (((u64)val/(u64)factor) << (16 + x))
+#define AVF_TXD_OFFSET_MACLEN(val)      AVF_TXD_OFFSET( 0, 2, val)
+#define AVF_TXD_OFFSET_IPLEN(val)       AVF_TXD_OFFSET( 7, 4, val)
+#define AVF_TXD_OFFSET_L4LEN(val)       AVF_TXD_OFFSET(14, 4, val)
+
 extern vlib_log_class_registration_t avf_log;
 
 #define avf_log_err(dev, f, ...)                        \
diff --git a/src/plugins/avf/device.c b/src/plugins/avf/device.c
index 6f0df6dc1a9..bb2dcf74e07 100644
--- a/src/plugins/avf/device.c
+++ b/src/plugins/avf/device.c
@@ -1636,7 +1636,9 @@ avf_create_if (vlib_main_t * vm, avf_create_if_args_t * args)
   /* Indicate ability to support L3 DMAC filtering and
    * initialize interface to L3 non-promisc mode */
   vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, ad->hw_if_index);
-  hi->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_MAC_FILTER;
+  hi->flags |=
+    VNET_HW_INTERFACE_FLAG_SUPPORTS_MAC_FILTER |
+    VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD;
   ethernet_set_flags (vnm, ad->hw_if_index,
 		      ETHERNET_INTERFACE_FLAG_DEFAULT_L3);
 
diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c
index de645319e73..b1b5bd005ff 100644
--- a/src/plugins/avf/output.c
+++ b/src/plugins/avf/output.c
@@ -19,7 +19,13 @@
 #include <vlib/unix/unix.h>
 #include <vlib/pci/pci.h>
 #include <vppinfra/ring.h>
+
 #include <vnet/ethernet/ethernet.h>
+#include <vnet/ip/ip4_packet.h>
+#include <vnet/ip/ip6_packet.h>
+#include <vnet/udp/udp_packet.h>
+#include <vnet/tcp/tcp_packet.h>
+
 #include <vnet/devices/devices.h>
 
 #include <avf/avf.h>
@@ -30,12 +36,97 @@ avf_tx_desc_get_dtyp (avf_tx_desc_t * d)
   return d->qword[1] & 0x0f;
 }
 
+struct avf_ip4_psh
+{
+  u32 src;
+  u32 dst;
+  u8 zero;
+  u8 proto;
+  u16 l4len;
+};
+
+struct avf_ip6_psh
+{
+  u32 src[4];
+  u32 dst[4];
+  u32 l4len;
+  u32 proto;
+};
+
+static_always_inline u64
+avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso)
+{
+  u64 flags = 0;
+  if (!is_tso && !(b->flags & ((VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
+				VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
+				VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+    return 0;
+  u32 is_tcp = is_tso || b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+  u32 is_udp = !is_tso && b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+  u32 is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
+  u32 is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
+  ASSERT (!is_tcp || !is_udp);
+  ASSERT (is_ip4 || is_ip6);
+  i16 l2_hdr_offset = vnet_buffer (b)->l2_hdr_offset;
+  i16 l3_hdr_offset = vnet_buffer (b)->l3_hdr_offset;
+  i16 l4_hdr_offset = vnet_buffer (b)->l4_hdr_offset;
+  u16 l2_len = l3_hdr_offset - l2_hdr_offset;
+  u16 l3_len = l4_hdr_offset - l3_hdr_offset;
+  ip4_header_t *ip4 = (void *) (b->data + l3_hdr_offset);
+  ip6_header_t *ip6 = (void *) (b->data + l3_hdr_offset);
+  tcp_header_t *tcp = (void *) (b->data + l4_hdr_offset);
+  udp_header_t *udp = (void *) (b->data + l4_hdr_offset);
+  u16 l4_len =
+    is_tcp ? tcp_header_bytes (tcp) : is_udp ? sizeof (udp_header_t) : 0;
+  u16 sum = 0;
+
+  flags |= AVF_TXD_OFFSET_MACLEN (l2_len) |
+    AVF_TXD_OFFSET_IPLEN (l3_len) | AVF_TXD_OFFSET_L4LEN (l4_len);
+  flags |= is_ip4 ? AVF_TXD_CMD_IIPT_IPV4 : AVF_TXD_CMD_IIPT_IPV6;
+  flags |= is_tcp ? AVF_TXD_CMD_L4T_TCP : is_udp ? AVF_TXD_CMD_L4T_UDP : 0;
+
+  if (is_ip4)
+    ip4->checksum = 0;
+
+  if (is_tcp || is_udp)
+    {
+      if (is_ip4)
+	{
+	  struct avf_ip4_psh psh = { 0 };
+	  psh.src = ip4->src_address.as_u32;
+	  psh.dst = ip4->dst_address.as_u32;
+	  psh.proto = ip4->protocol;
+	  psh.l4len =
+	    is_tso ? 0 :
+	    clib_host_to_net_u16 (clib_net_to_host_u16 (ip4->length) -
+				  (l4_hdr_offset - l3_hdr_offset));
+	  sum = ~ip_csum (&psh, sizeof (psh));
+	}
+      else
+	{
+	  struct avf_ip6_psh psh = { 0 };
+	  clib_memcpy_fast (&psh.src, &ip6->src_address, 32);
+	  psh.proto = clib_host_to_net_u32 ((u32) ip6->protocol);
+	  psh.l4len = is_tso ? 0 : ip6->payload_length;
+	  sum = ~ip_csum (&psh, sizeof (psh));
+	}
+    }
+  /* ip_csum does a byte swap for some reason... */
+  sum = clib_net_to_host_u16 (sum);
+  if (is_tcp)
+    tcp->checksum = sum;
+  else if (is_udp)
+    udp->checksum = sum;
+  return flags;
+}
+
 static_always_inline u16
 avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
 		u32 * buffers, u32 n_packets, int use_va_dma)
 {
   u16 next = txq->next;
   u64 bits = AVF_TXD_CMD_EOP | AVF_TXD_CMD_RSV;
+  u64 one_by_one_offload_flags = 0;
   u16 n_desc = 0;
   u16 *slot, n_desc_left, n_packets_left = n_packets;
   u16 mask = txq->size - 1;
@@ -69,7 +160,9 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
 
       or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
 
-      if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
+      if (or_flags &
+	  (VLIB_BUFFER_NEXT_PRESENT | VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
+	   VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
 	goto one_by_one;
 
       vlib_buffer_copy_indices (txq->bufs + next, buffers, 4);
@@ -103,9 +196,16 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
       continue;
 
     one_by_one:
+      one_by_one_offload_flags = 0;
       txq->bufs[next] = buffers[0];
       b[0] = vlib_get_buffer (vm, buffers[0]);
 
+      if (PREDICT_FALSE (b[0]->flags & (VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
+					VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
+					VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)))
+	one_by_one_offload_flags |=
+	  avf_tx_prepare_cksum (b[0], 0 /* no TSO */ );
+
       /* Deal with chain buffer if present */
       if (b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
 	{
@@ -145,7 +245,7 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
 		d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);
 
 	      d[0].qword[1] = (((u64) b[0]->current_length) << 34) |
-		AVF_TXD_CMD_RSV;
+		AVF_TXD_CMD_RSV | one_by_one_offload_flags;
 
 	      next += 1;
 	      n_desc += 1;
@@ -162,7 +262,9 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
       else
 	d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);
 
-      d[0].qword[1] = (((u64) b[0]->current_length) << 34) | bits;
+      d[0].qword[1] =
+	(((u64) b[0]->current_length) << 34) | bits |
+	one_by_one_offload_flags;
 
       next += 1;
       n_desc += 1;
@@ -185,9 +287,17 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
 
       while (n_packets_left && n_desc_left)
 	{
+
 	  txq->bufs[next & mask] = buffers[0];
 	  b[0] = vlib_get_buffer (vm, buffers[0]);
 
+	  one_by_one_offload_flags = 0;
+	  if (PREDICT_FALSE (b[0]->flags & (VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
+					    VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
+					    VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)))
+	    one_by_one_offload_flags |=
+	      avf_tx_prepare_cksum (b[0], 0 /* no TSO */ );
+
 	  /* Deal with chain buffer if present */
 	  if (b[0]->flags & VLIB_BUFFER_NEXT_PRESENT)
 	    {
@@ -222,7 +332,7 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
 		    d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);
 
 		  d[0].qword[1] = (((u64) b[0]->current_length) << 34) |
-		    AVF_TXD_CMD_RSV;
+		    AVF_TXD_CMD_RSV | one_by_one_offload_flags;
 
 		  next += 1;
 		  n_desc += 1;
@@ -239,7 +349,9 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
 	  else
 	    d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);
 
-	  d[0].qword[1] = (((u64) b[0]->current_length) << 34) | bits;
+	  d[0].qword[1] =
+	    (((u64) b[0]->current_length) << 34) | bits |
+	    one_by_one_offload_flags;
 
 	  next += 1;
 	  n_desc += 1;