vlib: refactor checksum offload support

Type: refactor

This patch refactors the offload flags in vlib_buffer_t.
There are two main reasons behind this refactoring.
First, offload flags are insufficient to represent outer
and inner headers offloads. Second, room for these flags
in first cacheline of vlib_buffer_t is also limited.

This patch introduces a generic offload flag in first
cacheline. And detailed offload flags in 2nd cacheline
of the structure for performance optimization.

Change-Id: Icc363a142fb9208ec7113ab5bbfc8230181f6004
Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>

6 files changed, 39 insertions, 38 deletions

diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c
index c53fc42a850..5bcb68cc95c 100644
--- a/src/plugins/avf/output.c
+++ b/src/plugins/avf/output.c
@@ -57,12 +57,12 @@ static_always_inline u64
 avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso)
 {
   u64 flags = 0;
-  if (!is_tso && !(b->flags & ((VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
-				VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-				VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+  if (!is_tso && !(b->flags & VNET_BUFFER_F_OFFLOAD))
     return 0;
-  u32 is_tcp = is_tso || b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
-  u32 is_udp = !is_tso && b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+
+  u32 oflags = vnet_buffer2 (b)->oflags;
+  u32 is_tcp = is_tso || oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
+  u32 is_udp = !is_tso && oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
   u32 is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
   u32 is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
   ASSERT (!is_tcp || !is_udp);
@@ -173,9 +173,7 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
 {
   u16 next = txq->next;
   u64 bits = AVF_TXD_CMD_EOP | AVF_TXD_CMD_RSV;
-  const u32 offload_mask = VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
-    VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
-    VNET_BUFFER_F_GSO;
+  const u32 offload_mask = VNET_BUFFER_F_OFFLOAD | VNET_BUFFER_F_GSO;
   u64 one_by_one_offload_flags = 0;
   int is_tso;
   u16 n_desc = 0;
diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index ec33f6a4461..6466b443633 100644
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -218,17 +218,20 @@ static_always_inline void
 dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b,
 			struct rte_mbuf *mb)
 {
-  u32 ip_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
-  u32 tcp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
-  u32 udp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
   int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
   u32 tso = b->flags & VNET_BUFFER_F_GSO, max_pkt_len;
+  u32 oflags, ip_cksum, tcp_cksum, udp_cksum;
   u64 ol_flags;
 
   /* Is there any work for us? */
-  if (PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum | tso) == 0))
+  if (PREDICT_TRUE (((b->flags & VNET_BUFFER_F_OFFLOAD) | tso) == 0))
     return;
 
+  oflags = vnet_buffer2 (b)->oflags;
+  ip_cksum = oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
+  tcp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
+  udp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
+
   mb->l2_len = vnet_buffer (b)->l3_hdr_offset - b->current_data;
   mb->l3_len = vnet_buffer (b)->l4_hdr_offset -
     vnet_buffer (b)->l3_hdr_offset;
@@ -328,10 +331,7 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm,
 	}
 
       if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
-			 (or_flags &
-			  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
-			   | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
-			   | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+			 (or_flags & VNET_BUFFER_F_OFFLOAD)))
 	{
 	  dpdk_buffer_tx_offload (xd, b[0], mb[0]);
 	  dpdk_buffer_tx_offload (xd, b[1], mb[1]);
@@ -388,10 +388,7 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm,
 	}
 
       if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
-			 (or_flags &
-			  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
-			   | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
-			   | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+			 (or_flags & VNET_BUFFER_F_OFFLOAD)))
 	{
 	  dpdk_buffer_tx_offload (xd, b[0], mb[0]);
 	  dpdk_buffer_tx_offload (xd, b[1], mb[1]);
diff --git a/src/plugins/gtpu/gtpu_decap.c b/src/plugins/gtpu/gtpu_decap.c
index e3bc476ed6d..974ae8a8190 100644
--- a/src/plugins/gtpu/gtpu_decap.c
+++ b/src/plugins/gtpu/gtpu_decap.c
@@ -1257,13 +1257,15 @@ static char *gtpu_flow_error_strings[] = {
 
 };
 
-#define gtpu_local_need_csum_check(_b) 			\
-    (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED 	\
-	|| _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
-
-#define gtpu_local_csum_is_valid(_b)  \
-    ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT \
-	|| _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) != 0)
+#define gtpu_local_need_csum_check(_b)                                        \
+  (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED ||                        \
+     (_b->flags & VNET_BUFFER_F_OFFLOAD &&                                    \
+      vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
+
+#define gtpu_local_csum_is_valid(_b)                                          \
+  ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT ||                          \
+    (_b->flags & VNET_BUFFER_F_OFFLOAD &&                                     \
+     vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0)
 
 static_always_inline u8
 gtpu_validate_udp_csum (vlib_main_t * vm, vlib_buffer_t *b)
diff --git a/src/plugins/lisp/lisp-cp/packets.c b/src/plugins/lisp/lisp-cp/packets.c
index 8e2386e72bd..3f4292b4841 100644
--- a/src/plugins/lisp/lisp-cp/packets.c
+++ b/src/plugins/lisp/lisp-cp/packets.c
@@ -182,7 +182,7 @@ pkt_push_udp_and_ip (vlib_main_t * vm, vlib_buffer_t * b, u16 sp, u16 dp,
   if (csum_offload)
     {
       ih = pkt_push_ip (vm, b, sip, dip, IP_PROTOCOL_UDP, 1);
-      b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+      vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
       vnet_buffer (b)->l3_hdr_offset = (u8 *) ih - b->data;
       vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data;
       uh->checksum = 0;
diff --git a/src/plugins/vmxnet3/input.c b/src/plugins/vmxnet3/input.c
index affc3691af1..be528b26494 100644
--- a/src/plugins/vmxnet3/input.c
+++ b/src/plugins/vmxnet3/input.c
@@ -79,6 +79,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
 			u16 gso_size)
 {
   u8 l4_hdr_sz = 0;
+  u32 oflags = 0;
 
   if (rx_comp->flags & VMXNET3_RXCF_IP4)
     {
@@ -90,15 +91,15 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
       vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) +
 	ip4_header_bytes (ip4);
       hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
-	VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
-	VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4;
+		   VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+		   VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4;
 
       /* checksum offload */
       if (!(rx_comp->index & VMXNET3_RXCI_CNC))
 	{
 	  if (!(rx_comp->flags & VMXNET3_RXCF_IPC))
 	    {
-	      hb->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+	      oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
 	      ip4->checksum = 0;
 	    }
 	  if (!(rx_comp->flags & VMXNET3_RXCF_TUC))
@@ -108,7 +109,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
 		  tcp_header_t *tcp =
 		    (tcp_header_t *) (hb->data +
 				      vnet_buffer (hb)->l4_hdr_offset);
-		  hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+		  oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
 		  tcp->checksum = 0;
 		}
 	      else if (rx_comp->flags & VMXNET3_RXCF_UDP)
@@ -116,7 +117,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
 		  udp_header_t *udp =
 		    (udp_header_t *) (hb->data +
 				      vnet_buffer (hb)->l4_hdr_offset);
-		  hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+		  oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
 		  udp->checksum = 0;
 		}
 	    }
@@ -148,8 +149,8 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
       vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) +
 	sizeof (ip6_header_t);
       hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
-	VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
-	VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6;
+		   VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+		   VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6;
 
       /* checksum offload */
       if (!(rx_comp->index & VMXNET3_RXCI_CNC))
@@ -161,7 +162,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
 		  tcp_header_t *tcp =
 		    (tcp_header_t *) (hb->data +
 				      vnet_buffer (hb)->l4_hdr_offset);
-		  hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+		  oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
 		  tcp->checksum = 0;
 		}
 	      else if (rx_comp->flags & VMXNET3_RXCF_UDP)
@@ -169,7 +170,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
 		  udp_header_t *udp =
 		    (udp_header_t *) (hb->data +
 				      vnet_buffer (hb)->l4_hdr_offset);
-		  hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+		  oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
 		  udp->checksum = 0;
 		}
 	    }
@@ -194,6 +195,8 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
 	  hb->flags |= VNET_BUFFER_F_GSO;
 	}
     }
+  if (oflags)
+    vnet_buffer_offload_flags_set (hb, oflags);
 }
 
 static_always_inline uword
diff --git a/src/plugins/wireguard/wireguard_input.c b/src/plugins/wireguard/wireguard_input.c
index d9843d90c7d..2690cf5e097 100755
--- a/src/plugins/wireguard/wireguard_input.c
+++ b/src/plugins/wireguard/wireguard_input.c
@@ -368,7 +368,8 @@ VLIB_NODE_FN (wg_input_node) (vlib_main_t * vm,
 
 	  clib_memcpy (vlib_buffer_get_current (b[0]), decr_data, decr_len);
 	  b[0]->current_length = decr_len;
-	  b[0]->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+	  vnet_buffer_offload_flags_clear (b[0],
+					   VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
 
 	  wg_timers_any_authenticated_packet_received (peer);
 	  wg_timers_any_authenticated_packet_traversal (peer);