vlib: refactor checksum offload support

Type: refactor

This patch refactors the offload flags in vlib_buffer_t.
There are two main reasons behind this refactoring.
First, offload flags are insufficient to represent outer
and inner headers offloads. Second, room for these flags
in first cacheline of vlib_buffer_t is also limited.

This patch introduces a generic offload flag in first
cacheline. And detailed offload flags in 2nd cacheline
of the structure for performance optimization.

Change-Id: Icc363a142fb9208ec7113ab5bbfc8230181f6004
Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>

34 files changed, 377 insertions, 264 deletions

diff --git a/extras/pg/checksum_offload.pg b/extras/pg/checksum_offload.pg
index a2e3205a21e..bfbdad66891 100644
--- a/extras/pg/checksum_offload.pg
+++ b/extras/pg/checksum_offload.pg
@@ -15,7 +15,8 @@ packet-generator new {
     interface loop0
     tx-interface loop1
     node loop1-output
-    buffer-flags ip4 offload-ip-cksum offload-udp-cksum
+    buffer-flags ip4 offload
+    buffer-offload-flags offload-ip-cksum offload-udp-cksum
     data { IP4: 1.2.3 -> 4.5.6
            UDP: 11.22.33.44 -> 11.22.34.44
                 ttl 2 checksum 13
diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c
index c53fc42a850..5bcb68cc95c 100644
--- a/src/plugins/avf/output.c
+++ b/src/plugins/avf/output.c
@@ -57,12 +57,12 @@ static_always_inline u64
 avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso)
 {
   u64 flags = 0;
-  if (!is_tso && !(b->flags & ((VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
-				VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-				VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+  if (!is_tso && !(b->flags & VNET_BUFFER_F_OFFLOAD))
     return 0;
-  u32 is_tcp = is_tso || b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
-  u32 is_udp = !is_tso && b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+
+  u32 oflags = vnet_buffer2 (b)->oflags;
+  u32 is_tcp = is_tso || oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
+  u32 is_udp = !is_tso && oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
   u32 is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
   u32 is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
   ASSERT (!is_tcp || !is_udp);
@@ -173,9 +173,7 @@ avf_tx_enqueue (vlib_main_t * vm, vlib_node_runtime_t * node, avf_txq_t * txq,
 {
   u16 next = txq->next;
   u64 bits = AVF_TXD_CMD_EOP | AVF_TXD_CMD_RSV;
-  const u32 offload_mask = VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
-    VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
-    VNET_BUFFER_F_GSO;
+  const u32 offload_mask = VNET_BUFFER_F_OFFLOAD | VNET_BUFFER_F_GSO;
   u64 one_by_one_offload_flags = 0;
   int is_tso;
   u16 n_desc = 0;
diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index ec33f6a4461..6466b443633 100644
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -218,17 +218,20 @@ static_always_inline void
 dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b,
 			struct rte_mbuf *mb)
 {
-  u32 ip_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
-  u32 tcp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
-  u32 udp_cksum = b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
   int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
   u32 tso = b->flags & VNET_BUFFER_F_GSO, max_pkt_len;
+  u32 oflags, ip_cksum, tcp_cksum, udp_cksum;
   u64 ol_flags;
 
   /* Is there any work for us? */
-  if (PREDICT_TRUE ((ip_cksum | tcp_cksum | udp_cksum | tso) == 0))
+  if (PREDICT_TRUE (((b->flags & VNET_BUFFER_F_OFFLOAD) | tso) == 0))
     return;
 
+  oflags = vnet_buffer2 (b)->oflags;
+  ip_cksum = oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
+  tcp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
+  udp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
+
   mb->l2_len = vnet_buffer (b)->l3_hdr_offset - b->current_data;
   mb->l3_len = vnet_buffer (b)->l4_hdr_offset -
     vnet_buffer (b)->l3_hdr_offset;
@@ -328,10 +331,7 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm,
 	}
 
       if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
-			 (or_flags &
-			  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
-			   | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
-			   | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+			 (or_flags & VNET_BUFFER_F_OFFLOAD)))
 	{
 	  dpdk_buffer_tx_offload (xd, b[0], mb[0]);
 	  dpdk_buffer_tx_offload (xd, b[1], mb[1]);
@@ -388,10 +388,7 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm,
 	}
 
       if (PREDICT_FALSE ((xd->flags & DPDK_DEVICE_FLAG_TX_OFFLOAD) &&
-			 (or_flags &
-			  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM
-			   | VNET_BUFFER_F_OFFLOAD_IP_CKSUM
-			   | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))))
+			 (or_flags & VNET_BUFFER_F_OFFLOAD)))
 	{
 	  dpdk_buffer_tx_offload (xd, b[0], mb[0]);
 	  dpdk_buffer_tx_offload (xd, b[1], mb[1]);
diff --git a/src/plugins/gtpu/gtpu_decap.c b/src/plugins/gtpu/gtpu_decap.c
index e3bc476ed6d..974ae8a8190 100644
--- a/src/plugins/gtpu/gtpu_decap.c
+++ b/src/plugins/gtpu/gtpu_decap.c
@@ -1257,13 +1257,15 @@ static char *gtpu_flow_error_strings[] = {
 
 };
 
-#define gtpu_local_need_csum_check(_b) 			\
-    (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED 	\
-	|| _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
-
-#define gtpu_local_csum_is_valid(_b)  \
-    ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT \
-	|| _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) != 0)
+#define gtpu_local_need_csum_check(_b)                                        \
+  (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED ||                        \
+     (_b->flags & VNET_BUFFER_F_OFFLOAD &&                                    \
+      vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
+
+#define gtpu_local_csum_is_valid(_b)                                          \
+  ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT ||                          \
+    (_b->flags & VNET_BUFFER_F_OFFLOAD &&                                     \
+     vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0)
 
 static_always_inline u8
 gtpu_validate_udp_csum (vlib_main_t * vm, vlib_buffer_t *b)
diff --git a/src/plugins/lisp/lisp-cp/packets.c b/src/plugins/lisp/lisp-cp/packets.c
index 8e2386e72bd..3f4292b4841 100644
--- a/src/plugins/lisp/lisp-cp/packets.c
+++ b/src/plugins/lisp/lisp-cp/packets.c
@@ -182,7 +182,7 @@ pkt_push_udp_and_ip (vlib_main_t * vm, vlib_buffer_t * b, u16 sp, u16 dp,
   if (csum_offload)
     {
       ih = pkt_push_ip (vm, b, sip, dip, IP_PROTOCOL_UDP, 1);
-      b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+      vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
       vnet_buffer (b)->l3_hdr_offset = (u8 *) ih - b->data;
       vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data;
       uh->checksum = 0;
diff --git a/src/plugins/vmxnet3/input.c b/src/plugins/vmxnet3/input.c
index affc3691af1..be528b26494 100644
--- a/src/plugins/vmxnet3/input.c
+++ b/src/plugins/vmxnet3/input.c
@@ -79,6 +79,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
 			u16 gso_size)
 {
   u8 l4_hdr_sz = 0;
+  u32 oflags = 0;
 
   if (rx_comp->flags & VMXNET3_RXCF_IP4)
     {
@@ -90,15 +91,15 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
       vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) +
 	ip4_header_bytes (ip4);
       hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
-	VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
-	VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4;
+		   VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+		   VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP4;
 
       /* checksum offload */
       if (!(rx_comp->index & VMXNET3_RXCI_CNC))
 	{
 	  if (!(rx_comp->flags & VMXNET3_RXCF_IPC))
 	    {
-	      hb->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+	      oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
 	      ip4->checksum = 0;
 	    }
 	  if (!(rx_comp->flags & VMXNET3_RXCF_TUC))
@@ -108,7 +109,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
 		  tcp_header_t *tcp =
 		    (tcp_header_t *) (hb->data +
 				      vnet_buffer (hb)->l4_hdr_offset);
-		  hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+		  oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
 		  tcp->checksum = 0;
 		}
 	      else if (rx_comp->flags & VMXNET3_RXCF_UDP)
@@ -116,7 +117,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
 		  udp_header_t *udp =
 		    (udp_header_t *) (hb->data +
 				      vnet_buffer (hb)->l4_hdr_offset);
-		  hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+		  oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
 		  udp->checksum = 0;
 		}
 	    }
@@ -148,8 +149,8 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
       vnet_buffer (hb)->l4_hdr_offset = sizeof (ethernet_header_t) +
 	sizeof (ip6_header_t);
       hb->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
-	VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
-	VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6;
+		   VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+		   VNET_BUFFER_F_L4_HDR_OFFSET_VALID | VNET_BUFFER_F_IS_IP6;
 
       /* checksum offload */
       if (!(rx_comp->index & VMXNET3_RXCI_CNC))
@@ -161,7 +162,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
 		  tcp_header_t *tcp =
 		    (tcp_header_t *) (hb->data +
 				      vnet_buffer (hb)->l4_hdr_offset);
-		  hb->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+		  oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
 		  tcp->checksum = 0;
 		}
 	      else if (rx_comp->flags & VMXNET3_RXCF_UDP)
@@ -169,7 +170,7 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
 		  udp_header_t *udp =
 		    (udp_header_t *) (hb->data +
 				      vnet_buffer (hb)->l4_hdr_offset);
-		  hb->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+		  oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
 		  udp->checksum = 0;
 		}
 	    }
@@ -194,6 +195,8 @@ vmxnet3_handle_offload (vmxnet3_rx_comp * rx_comp, vlib_buffer_t * hb,
 	  hb->flags |= VNET_BUFFER_F_GSO;
 	}
     }
+  if (oflags)
+    vnet_buffer_offload_flags_set (hb, oflags);
 }
 
 static_always_inline uword
diff --git a/src/plugins/wireguard/wireguard_input.c b/src/plugins/wireguard/wireguard_input.c
index d9843d90c7d..2690cf5e097 100755
--- a/src/plugins/wireguard/wireguard_input.c
+++ b/src/plugins/wireguard/wireguard_input.c
@@ -368,7 +368,8 @@ VLIB_NODE_FN (wg_input_node) (vlib_main_t * vm,
 
 	  clib_memcpy (vlib_buffer_get_current (b[0]), decr_data, decr_len);
 	  b[0]->current_length = decr_len;
-	  b[0]->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+	  vnet_buffer_offload_flags_clear (b[0],
+					   VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
 
 	  wg_timers_any_authenticated_packet_received (peer);
 	  wg_timers_any_authenticated_packet_traversal (peer);
diff --git a/src/vnet/buffer.c b/src/vnet/buffer.c
index 3cb6b948d2b..b41036d8124 100644
--- a/src/vnet/buffer.c
+++ b/src/vnet/buffer.c
@@ -16,6 +16,18 @@
 #include <vlib/vlib.h>
 #include <vnet/buffer.h>
 
+u8 *
+format_vnet_buffer_offload (u8 *s, va_list *args)
+{
+  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
+
+#define _(bit, name, ss, v)                                                   \
+  if (v && (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name))         \
+    s = format (s, "%s ", ss);
+  foreach_vnet_buffer_offload_flag
+#undef _
+    return s;
+}
 
 u8 *
 format_vnet_buffer (u8 * s, va_list * args)
@@ -29,7 +41,10 @@ format_vnet_buffer (u8 * s, va_list * args)
     a = format (a, "%s ", ss);
   foreach_vnet_buffer_flag
 #undef _
-    if (b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID)
+    if (b->flags & VNET_BUFFER_F_OFFLOAD) a =
+      format (a, "%U ", format_vnet_buffer_offload, b);
+
+  if (b->flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID)
     a = format (a, "l2-hdr-offset %d ", vnet_buffer (b)->l2_hdr_offset);
 
   if (b->flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID)
@@ -38,6 +53,9 @@ format_vnet_buffer (u8 * s, va_list * args)
   if (b->flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID)
     a = format (a, "l4-hdr-offset %d ", vnet_buffer (b)->l4_hdr_offset);
 
+  if (b->flags & VNET_BUFFER_F_GSO)
+    a = format (a, "gso gso-size %d", vnet_buffer2 (b)->gso_size);
+
   if (b->flags & VNET_BUFFER_F_QOS_DATA_VALID)
     a = format (a, "qos %d.%d ",
 		vnet_buffer2 (b)->qos.bits, vnet_buffer2 (b)->qos.source);
diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h
index 27aeb3b42ea..fb734d5acde 100644
--- a/src/vnet/buffer.h
+++ b/src/vnet/buffer.h
@@ -46,34 +46,34 @@
  * Flags that are set in the high order bits of ((vlib_buffer*)b)->flags
  *
  */
-#define foreach_vnet_buffer_flag                        \
-  _( 1, L4_CHECKSUM_COMPUTED, "l4-cksum-computed", 1)	\
-  _( 2, L4_CHECKSUM_CORRECT, "l4-cksum-correct", 1)	\
-  _( 3, VLAN_2_DEEP, "vlan-2-deep", 1)			\
-  _( 4, VLAN_1_DEEP, "vlan-1-deep", 1)			\
-  _( 5, SPAN_CLONE, "span-clone", 1)                    \
-  _( 6, LOOP_COUNTER_VALID, "loop-counter-valid", 0)    \
-  _( 7, LOCALLY_ORIGINATED, "local", 1)                 \
-  _( 8, IS_IP4, "ip4", 1)                               \
-  _( 9, IS_IP6, "ip6", 1)                               \
-  _(10, OFFLOAD_IP_CKSUM, "offload-ip-cksum", 1)        \
-  _(11, OFFLOAD_TCP_CKSUM, "offload-tcp-cksum", 1)      \
-  _(12, OFFLOAD_UDP_CKSUM, "offload-udp-cksum", 1)      \
-  _(13, IS_NATED, "natted", 1)                          \
-  _(14, L2_HDR_OFFSET_VALID, "l2_hdr_offset_valid", 0)  \
-  _(15, L3_HDR_OFFSET_VALID, "l3_hdr_offset_valid", 0)  \
-  _(16, L4_HDR_OFFSET_VALID, "l4_hdr_offset_valid", 0)  \
-  _(17, FLOW_REPORT, "flow-report", 1)                  \
-  _(18, IS_DVR, "dvr", 1)                               \
-  _(19, QOS_DATA_VALID, "qos-data-valid", 0)            \
-  _(20, GSO, "gso", 0)                                  \
-  _(21, AVAIL1, "avail1", 1)                            \
-  _(22, AVAIL2, "avail2", 1)                            \
-  _(23, AVAIL3, "avail3", 1)                            \
-  _(24, AVAIL4, "avail4", 1)                            \
-  _(25, AVAIL5, "avail5", 1)                            \
-  _(26, AVAIL6, "avail6", 1)                            \
-  _(27, AVAIL7, "avail7", 1)
+#define foreach_vnet_buffer_flag                                              \
+  _ (1, L4_CHECKSUM_COMPUTED, "l4-cksum-computed", 1)                         \
+  _ (2, L4_CHECKSUM_CORRECT, "l4-cksum-correct", 1)                           \
+  _ (3, VLAN_2_DEEP, "vlan-2-deep", 1)                                        \
+  _ (4, VLAN_1_DEEP, "vlan-1-deep", 1)                                        \
+  _ (5, SPAN_CLONE, "span-clone", 1)                                          \
+  _ (6, LOOP_COUNTER_VALID, "loop-counter-valid", 0)                          \
+  _ (7, LOCALLY_ORIGINATED, "local", 1)                                       \
+  _ (8, IS_IP4, "ip4", 1)                                                     \
+  _ (9, IS_IP6, "ip6", 1)                                                     \
+  _ (10, OFFLOAD, "offload", 0)                                               \
+  _ (11, IS_NATED, "natted", 1)                                               \
+  _ (12, L2_HDR_OFFSET_VALID, "l2_hdr_offset_valid", 0)                       \
+  _ (13, L3_HDR_OFFSET_VALID, "l3_hdr_offset_valid", 0)                       \
+  _ (14, L4_HDR_OFFSET_VALID, "l4_hdr_offset_valid", 0)                       \
+  _ (15, FLOW_REPORT, "flow-report", 1)                                       \
+  _ (16, IS_DVR, "dvr", 1)                                                    \
+  _ (17, QOS_DATA_VALID, "qos-data-valid", 0)                                 \
+  _ (18, GSO, "gso", 0)                                                       \
+  _ (19, AVAIL1, "avail1", 1)                                                 \
+  _ (20, AVAIL2, "avail2", 1)                                                 \
+  _ (21, AVAIL3, "avail3", 1)                                                 \
+  _ (22, AVAIL4, "avail4", 1)                                                 \
+  _ (23, AVAIL5, "avail5", 1)                                                 \
+  _ (24, AVAIL6, "avail6", 1)                                                 \
+  _ (25, AVAIL7, "avail7", 1)                                                 \
+  _ (26, AVAIL8, "avail8", 1)                                                 \
+  _ (27, AVAIL9, "avail9", 1)
 
 /*
  * Please allocate the FIRST available bit, redefine
@@ -81,10 +81,10 @@
  * VNET_BUFFER_FLAGS_ALL_AVAIL definition.
  */
 
-#define VNET_BUFFER_FLAGS_ALL_AVAIL                                     \
-  (VNET_BUFFER_F_AVAIL1 | VNET_BUFFER_F_AVAIL2 | VNET_BUFFER_F_AVAIL3 | \
-   VNET_BUFFER_F_AVAIL4 | VNET_BUFFER_F_AVAIL5 | VNET_BUFFER_F_AVAIL6 | \
-   VNET_BUFFER_F_AVAIL7)
+#define VNET_BUFFER_FLAGS_ALL_AVAIL                                           \
+  (VNET_BUFFER_F_AVAIL1 | VNET_BUFFER_F_AVAIL2 | VNET_BUFFER_F_AVAIL3 |       \
+   VNET_BUFFER_F_AVAIL4 | VNET_BUFFER_F_AVAIL5 | VNET_BUFFER_F_AVAIL6 |       \
+   VNET_BUFFER_F_AVAIL7 | VNET_BUFFER_F_AVAIL8 | VNET_BUFFER_F_AVAIL9)
 
 #define VNET_BUFFER_FLAGS_VLAN_BITS \
   (VNET_BUFFER_F_VLAN_1_DEEP | VNET_BUFFER_F_VLAN_2_DEEP)
@@ -420,6 +420,21 @@ STATIC_ASSERT (sizeof (vnet_buffer_opaque_t) <=
 
 #define vnet_buffer(b) ((vnet_buffer_opaque_t *) (b)->opaque)
 
+#define foreach_vnet_buffer_offload_flag                                      \
+  _ (0, IP_CKSUM, "offload-ip-cksum", 1)                                      \
+  _ (1, TCP_CKSUM, "offload-tcp-cksum", 1)                                    \
+  _ (2, UDP_CKSUM, "offload-udp-cksum", 1)                                    \
+  _ (3, OUTER_IP_CKSUM, "offload-outer-ip-cksum", 1)                          \
+  _ (4, OUTER_TCP_CKSUM, "offload-outer-tcp-cksum", 1)                        \
+  _ (5, OUTER_UDP_CKSUM, "offload-outer-udp-cksum", 1)
+
+enum
+{
+#define _(bit, name, s, v) VNET_BUFFER_OFFLOAD_F_##name = (1 << bit),
+  foreach_vnet_buffer_offload_flag
+#undef _
+};
+
 /* Full cache line (64 bytes) of additional space */
 typedef struct
 {
@@ -452,12 +467,15 @@ typedef struct
    * in case the egress interface is not GSO-enabled - then we need to perform
    * the segmentation, and use this value to cut the payload appropriately.
    */
-  u16 gso_size;
-  /* size of L4 prototol header */
-  u16 gso_l4_hdr_sz;
+  struct
+  {
+    u16 gso_size;
+    /* size of L4 prototol header */
+    u16 gso_l4_hdr_sz;
 
-  /* The union below has a u64 alignment, so this space is unused */
-  u32 __unused2[1];
+    /* offload flags */
+    u32 oflags;
+  };
 
   struct
   {
@@ -501,6 +519,22 @@ STATIC_ASSERT (sizeof (vnet_buffer_opaque2_t) <=
 
 
 format_function_t format_vnet_buffer;
+format_function_t format_vnet_buffer_offload;
+
+static_always_inline void
+vnet_buffer_offload_flags_set (vlib_buffer_t *b, u32 oflags)
+{
+  vnet_buffer2 (b)->oflags |= oflags;
+  b->flags |= VNET_BUFFER_F_OFFLOAD;
+}
+
+static_always_inline void
+vnet_buffer_offload_flags_clear (vlib_buffer_t *b, u32 oflags)
+{
+  vnet_buffer2 (b)->oflags &= ~oflags;
+  if (0 == vnet_buffer2 (b)->oflags)
+    b->flags &= ~VNET_BUFFER_F_OFFLOAD;
+}
 
 #endif /* included_vnet_buffer_h */
 
diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c
index f4db6399a57..9f1782e2d13 100644
--- a/src/vnet/devices/af_packet/node.c
+++ b/src/vnet/devices/af_packet/node.c
@@ -120,6 +120,7 @@ static_always_inline void
 mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
 {
   ethernet_header_t *eth = vlib_buffer_get_current (b);
+  u32 oflags = 0;
   if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP4)
     {
       ip4_header_t *ip4 =
@@ -127,7 +128,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
       b->flags |= VNET_BUFFER_F_IS_IP4;
       if (ip4->protocol == IP_PROTOCOL_TCP)
 	{
-	  b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+	  oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
 	  tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b) +
 						sizeof (ethernet_header_t) +
 						ip4_header_bytes (ip4));
@@ -136,7 +137,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
 	}
       else if (ip4->protocol == IP_PROTOCOL_UDP)
 	{
-	  b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+	  oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
 	  udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b) +
 						sizeof (ethernet_header_t) +
 						ip4_header_bytes (ip4));
@@ -146,6 +147,8 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
       vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t);
       vnet_buffer (b)->l4_hdr_offset =
 	sizeof (ethernet_header_t) + ip4_header_bytes (ip4);
+      if (oflags)
+	vnet_buffer_offload_flags_set (b, oflags);
     }
   else if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP6)
     {
@@ -165,7 +168,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
 	}
       if (ip6->protocol == IP_PROTOCOL_TCP)
 	{
-	  b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+	  oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
 	  tcp_header_t *tcp =
 	    (tcp_header_t *) (vlib_buffer_get_current (b) +
 			      sizeof (ethernet_header_t) + ip6_hdr_len);
@@ -174,7 +177,7 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
 	}
       else if (ip6->protocol == IP_PROTOCOL_UDP)
 	{
-	  b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+	  oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
 	  udp_header_t *udp =
 	    (udp_header_t *) (vlib_buffer_get_current (b) +
 			      sizeof (ethernet_header_t) + ip6_hdr_len);
@@ -184,6 +187,8 @@ mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz)
       vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t);
       vnet_buffer (b)->l4_hdr_offset =
 	sizeof (ethernet_header_t) + ip6_hdr_len;
+      if (oflags)
+	vnet_buffer_offload_flags_set (b, oflags);
     }
 }
 
diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c
index 3482336b2ad..fb996d551fa 100644
--- a/src/vnet/devices/virtio/device.c
+++ b/src/vnet/devices/virtio/device.c
@@ -282,6 +282,8 @@ static_always_inline void
 set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
 		      const int is_l2)
 {
+  u32 oflags = vnet_buffer2 (b)->oflags;
+
   if (b->flags & VNET_BUFFER_F_IS_IP4)
     {
       ip4_header_t *ip4;
@@ -290,11 +292,11 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
 					 0 /* ip6 */ );
       hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
       hdr->csum_start = gho.l4_hdr_offset;	// 0x22;
-      if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+      if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
 	{
 	  hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
 	}
-      else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+      else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
 	{
 	  hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
 	}
@@ -305,7 +307,7 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
        */
       ip4 =
 	(ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
-      if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+      if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
 	ip4->checksum = ip4_header_checksum (ip4);
     }
   else if (b->flags & VNET_BUFFER_F_IS_IP6)
@@ -315,11 +317,11 @@ set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
 					 1 /* ip6 */ );
       hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
       hdr->csum_start = gho.l4_hdr_offset;	// 0x36;
-      if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+      if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
 	{
 	  hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
 	}
-      else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+      else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
 	{
 	  hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
 	}
@@ -330,6 +332,8 @@ static_always_inline void
 set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
 		 const int is_l2)
 {
+  u32 oflags = vnet_buffer2 (b)->oflags;
+
   if (b->flags & VNET_BUFFER_F_IS_IP4)
     {
       ip4_header_t *ip4;
@@ -348,7 +352,7 @@ set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr,
        * virtio devices do not support IP4 checksum offload. So driver takes care
        * of it while doing tx.
        */
-      if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+      if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
 	ip4->checksum = ip4_header_checksum (ip4);
     }
   else if (b->flags & VNET_BUFFER_F_IS_IP6)
@@ -392,8 +396,7 @@ add_buffer_to_slot (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  goto done;
 	}
     }
-  else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-		       VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+  else if (b->flags & VNET_BUFFER_F_OFFLOAD)
     {
       if (csum_offload)
 	set_checksum_offsets (b, hdr, is_l2);
@@ -584,8 +587,7 @@ add_buffer_to_slot_packed (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  goto done;
 	}
     }
-  else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-		       VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+  else if (b->flags & VNET_BUFFER_F_OFFLOAD)
     {
       if (csum_offload)
 	set_checksum_offsets (b, hdr, is_l2);
diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c
index be2405bb3b9..98df322ea15 100644
--- a/src/vnet/devices/virtio/node.c
+++ b/src/vnet/devices/virtio/node.c
@@ -235,6 +235,7 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
   if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
     {
       u16 ethertype = 0, l2hdr_sz = 0;
+      u32 oflags = 0;
 
       if (type == VIRTIO_IF_TYPE_TUN)
 	{
@@ -280,11 +281,10 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
 	    (ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz);
 	  vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
 	  *l4_proto = ip4->protocol;
+	  oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
 	  b0->flags |=
-	    (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
-	  b0->flags |=
-	    (VNET_BUFFER_F_L2_HDR_OFFSET_VALID
-	     | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+	    (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
+	     VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
 	     VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
 	}
       else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
@@ -301,7 +301,7 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
 	}
       if (*l4_proto == IP_PROTOCOL_TCP)
 	{
-	  b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+	  oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
 	  tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b0) +
 						vnet_buffer
 						(b0)->l4_hdr_offset);
@@ -309,12 +309,14 @@ virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr,
 	}
       else if (*l4_proto == IP_PROTOCOL_UDP)
 	{
-	  b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+	  oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
 	  udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b0) +
 						vnet_buffer
 						(b0)->l4_hdr_offset);
 	  *l4_hdr_sz = sizeof (*udp);
 	}
+      if (oflags)
+	vnet_buffer_offload_flags_set (b0, oflags);
     }
 }
 
diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c
index 62b59f69ba9..739125bbcfe 100644
--- a/src/vnet/devices/virtio/vhost_user_input.c
+++ b/src/vnet/devices/virtio/vhost_user_input.c
@@ -253,6 +253,7 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
   ethernet_header_t *eh = (ethernet_header_t *) b0_data;
   u16 ethertype = clib_net_to_host_u16 (eh->type);
   u16 l2hdr_sz = sizeof (ethernet_header_t);
+  u32 oflags = 0;
 
   if (ethernet_frame_is_tagged (ethertype))
     {
@@ -278,7 +279,8 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
     {
       ip4_header_t *ip4 = (ip4_header_t *) (b0_data + l2hdr_sz);
       l4_proto = ip4->protocol;
-      b0->flags |= VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+      b0->flags |= VNET_BUFFER_F_IS_IP4;
+      oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
     }
   else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
     {
@@ -292,12 +294,12 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
       tcp_header_t *tcp = (tcp_header_t *)
 	(b0_data + vnet_buffer (b0)->l4_hdr_offset);
       l4_hdr_sz = tcp_header_bytes (tcp);
-      b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+      oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
     }
   else if (l4_proto == IP_PROTOCOL_UDP)
     {
       l4_hdr_sz = sizeof (udp_header_t);
-      b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+      oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
     }
 
   if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP)
@@ -318,6 +320,9 @@ vhost_user_handle_rx_offload (vlib_buffer_t * b0, u8 * b0_data,
       vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz;
       b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
     }
+
+  if (oflags)
+    vnet_buffer_offload_flags_set (b0, oflags);
 }
 
 static_always_inline void
diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c
index 465c0ea0903..8dbddea46f5 100644
--- a/src/vnet/devices/virtio/vhost_user_output.c
+++ b/src/vnet/devices/virtio/vhost_user_output.c
@@ -225,10 +225,11 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
   generic_header_offset_t gho = { 0 };
   int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
   int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
+  u32 oflags = vnet_buffer2 (b)->oflags;
 
   ASSERT (!(is_ip4 && is_ip6));
   vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6);
-  if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+  if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
     {
       ip4_header_t *ip4;
 
@@ -238,13 +239,13 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
     }
 
   /* checksum offload */
-  if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+  if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
     {
       hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
       hdr->csum_start = gho.l4_hdr_offset;
       hdr->csum_offset = offsetof (udp_header_t, checksum);
     }
-  else if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+  else if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
     {
       hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
       hdr->csum_start = gho.l4_hdr_offset;
@@ -254,7 +255,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
   /* GSO offload */
   if (b->flags & VNET_BUFFER_F_GSO)
     {
-      if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+      if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
 	{
 	  if (is_ip4 &&
 	      (vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4)))
@@ -270,7 +271,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
 	    }
 	}
       else if ((vui->features & VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO)) &&
-	       (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+	       (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
 	{
 	  hdr->gso_size = vnet_buffer2 (b)->gso_size;
 	  hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
@@ -477,9 +478,7 @@ retry:
       hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
       hdr->num_buffers = 1;
 
-      or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
-	(b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
-	(b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+      or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
 
       /* Guest supports csum offload and buffer requires checksum offload? */
       if (or_flags &&
@@ -813,9 +812,7 @@ retry:
 	hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
 	hdr->num_buffers = 1;	//This is local, no need to check
 
-	or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM) ||
-	  (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM) ||
-	  (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+	or_flags = (b0->flags & VNET_BUFFER_F_OFFLOAD);
 
 	/* Guest supports csum offload and buffer requires checksum offload? */
 	if (or_flags
diff --git a/src/vnet/gso/gro_func.h b/src/vnet/gso/gro_func.h
index b821a034e71..239009d680b 100644
--- a/src/vnet/gso/gro_func.h
+++ b/src/vnet/gso/gro_func.h
@@ -147,7 +147,7 @@ gro_validate_checksum (vlib_main_t * vm, vlib_buffer_t * b0,
 {
   u32 flags = 0;
 
-  if (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+  if (b0->flags & VNET_BUFFER_F_OFFLOAD)
     return VNET_BUFFER_F_L4_CHECKSUM_CORRECT;
   vlib_buffer_advance (b0, gho0->l3_hdr_offset);
   if (is_ip4)
@@ -353,9 +353,9 @@ gro_fixup_header (vlib_main_t * vm, vlib_buffer_t * b0, u32 ack_number,
       ip4->length =
 	clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
 			      gho0.l3_hdr_offset);
-      b0->flags |=
-	(VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4 |
-	 VNET_BUFFER_F_OFFLOAD_TCP_CKSUM | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
+      b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4);
+      vnet_buffer_offload_flags_set (b0, (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+					  VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
     }
   else if (gho0.gho_flags & GHO_F_IP6)
     {
@@ -364,9 +364,8 @@ gro_fixup_header (vlib_main_t * vm, vlib_buffer_t * b0, u32 ack_number,
       ip6->payload_length =
 	clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) -
 			      gho0.l4_hdr_offset);
-      b0->flags |=
-	(VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6 |
-	 VNET_BUFFER_F_OFFLOAD_TCP_CKSUM);
+      b0->flags |= (VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6);
+      vnet_buffer_offload_flags_set (b0, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
     }
 
   tcp_header_t *tcp0 =
diff --git a/src/vnet/gso/node.c b/src/vnet/gso/node.c
index d0c5ca054e6..ecbdc188ec8 100644
--- a/src/vnet/gso/node.c
+++ b/src/vnet/gso/node.c
@@ -143,7 +143,8 @@ tso_segment_vxlan_tunnel_headers_fixup (vlib_main_t * vm, vlib_buffer_t * b,
 	{
 	  udp->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
 	}
-      b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+      /* FIXME: it should be OUTER_UDP_CKSUM */
+      vnet_buffer_offload_flags_clear (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
     }
 }
 
@@ -281,7 +282,8 @@ tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags,
 	  tcp->checksum = 0;
 	  tcp->checksum =
 	    ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6, &bogus);
-	  b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+	  vnet_buffer_offload_flags_clear (b0,
+					   VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
 	}
     }
   else
@@ -295,8 +297,8 @@ tso_fixup_segmented_buf (vlib_main_t * vm, vlib_buffer_t * b0, u8 tcp_flags,
 	  tcp->checksum = 0;
 	  tcp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip4);
 	}
-      b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
-      b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+      vnet_buffer_offload_flags_clear (b0, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
+					    VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
     }
 
   if (!is_l2 && ((gho->gho_flags & GHO_F_TUNNEL) == 0))
diff --git a/src/vnet/interface_format.c b/src/vnet/interface_format.c
index dc2edff7c67..036ca53174c 100644
--- a/src/vnet/interface_format.c
+++ b/src/vnet/interface_format.c
@@ -722,7 +722,7 @@ unformat_vnet_buffer_flags (unformat_input_t * input, va_list * args)
   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
     {
       /* Red herring, there is no such buffer flag */
-      if (unformat (input, "avail8"))
+      if (unformat (input, "avail10"))
 	return 0;
 #define _(bit,enum,str,verbose)                                 \
       else if (unformat (input, str))                           \
@@ -741,6 +741,32 @@ unformat_vnet_buffer_flags (unformat_input_t * input, va_list * args)
 }
 
 uword
+unformat_vnet_buffer_offload_flags (unformat_input_t *input, va_list *args)
+{
+  u32 *flagp = va_arg (*args, u32 *);
+  int rv = 0;
+  u32 oflags = 0;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (0)
+	;
+#define _(bit, enum, str, verbose)                                            \
+  else if (unformat (input, str))                                             \
+  {                                                                           \
+    oflags |= (1 << bit);                                                     \
+    rv = 1;                                                                   \
+  }
+      foreach_vnet_buffer_offload_flag
+#undef _
+	else break;
+    }
+  if (rv)
+    *flagp = oflags;
+  return rv;
+}
+
+uword
 unformat_vnet_hw_interface (unformat_input_t * input, va_list * args)
 {
   vnet_main_t *vnm = va_arg (*args, vnet_main_t *);
diff --git a/src/vnet/interface_funcs.h b/src/vnet/interface_funcs.h
index dbb0549314a..186e0327884 100644
--- a/src/vnet/interface_funcs.h
+++ b/src/vnet/interface_funcs.h
@@ -457,6 +457,7 @@ format_function_t format_vnet_sw_interface_flags;
 unformat_function_t unformat_vnet_sw_interface;
 unformat_function_t unformat_vnet_hw_interface;
 unformat_function_t unformat_vnet_buffer_flags;
+unformat_function_t unformat_vnet_buffer_offload_flags;
 
 /* Parses interface flags (up, down, enable, disable, etc.) */
 unformat_function_t unformat_vnet_hw_interface_flags;
diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c
index 3363f76b722..ff315f6abe0 100644
--- a/src/vnet/interface_output.c
+++ b/src/vnet/interface_output.c
@@ -337,28 +337,24 @@ vnet_interface_output_node_inline (vlib_main_t * vm,
 
 	  if (do_tx_offloads)
 	    {
-	      u32 vnet_buffer_offload_flags =
-		(VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-		 VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
-		 VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
-	      if (or_flags & vnet_buffer_offload_flags)
+	      if (or_flags & VNET_BUFFER_F_OFFLOAD)
 		{
-		  if (b[0]->flags & vnet_buffer_offload_flags)
+		  if (b[0]->flags & VNET_BUFFER_F_OFFLOAD)
 		    vnet_calc_checksums_inline
 		      (vm, b[0],
 		       b[0]->flags & VNET_BUFFER_F_IS_IP4,
 		       b[0]->flags & VNET_BUFFER_F_IS_IP6);
-		  if (b[1]->flags & vnet_buffer_offload_flags)
+		  if (b[1]->flags & VNET_BUFFER_F_OFFLOAD)
 		    vnet_calc_checksums_inline
 		      (vm, b[1],
 		       b[1]->flags & VNET_BUFFER_F_IS_IP4,
 		       b[1]->flags & VNET_BUFFER_F_IS_IP6);
-		  if (b[2]->flags & vnet_buffer_offload_flags)
+		  if (b[2]->flags & VNET_BUFFER_F_OFFLOAD)
 		    vnet_calc_checksums_inline
 		      (vm, b[2],
 		       b[2]->flags & VNET_BUFFER_F_IS_IP4,
 		       b[2]->flags & VNET_BUFFER_F_IS_IP6);
-		  if (b[3]->flags & vnet_buffer_offload_flags)
+		  if (b[3]->flags & VNET_BUFFER_F_OFFLOAD)
 		    vnet_calc_checksums_inline
 		      (vm, b[3],
 		       b[3]->flags & VNET_BUFFER_F_IS_IP4,
@@ -406,10 +402,7 @@ vnet_interface_output_node_inline (vlib_main_t * vm,
 
 	  if (do_tx_offloads)
 	    {
-	      if (b[0]->flags &
-		  (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-		   VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
-		   VNET_BUFFER_F_OFFLOAD_IP_CKSUM))
+	      if (b[0]->flags & VNET_BUFFER_F_OFFLOAD)
 		vnet_calc_checksums_inline
 		  (vm, b[0],
 		   b[0]->flags & VNET_BUFFER_F_IS_IP4,
diff --git a/src/vnet/interface_output.h b/src/vnet/interface_output.h
index 4b082b314ad..27aadc6c5fd 100644
--- a/src/vnet/interface_output.h
+++ b/src/vnet/interface_output.h
@@ -44,18 +44,17 @@
 #include <vnet/gso/hdr_offset_parser.h>
 
 static_always_inline void
-vnet_calc_ip4_checksums (vlib_main_t * vm, vlib_buffer_t * b,
-			 ip4_header_t * ip4, tcp_header_t * th,
-			 udp_header_t * uh)
+vnet_calc_ip4_checksums (vlib_main_t *vm, vlib_buffer_t *b, ip4_header_t *ip4,
+			 tcp_header_t *th, udp_header_t *uh, u32 oflags)
 {
-  if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
+  if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
     ip4->checksum = ip4_header_checksum (ip4);
-  if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+  if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
     {
       th->checksum = 0;
       th->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
     }
-  if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+  if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
     {
       uh->checksum = 0;
       uh->checksum = ip4_tcp_udp_compute_checksum (vm, b, ip4);
@@ -63,17 +62,16 @@ vnet_calc_ip4_checksums (vlib_main_t * vm, vlib_buffer_t * b,
 }
 
 static_always_inline void
-vnet_calc_ip6_checksums (vlib_main_t * vm, vlib_buffer_t * b,
-			 ip6_header_t * ip6, tcp_header_t * th,
-			 udp_header_t * uh)
+vnet_calc_ip6_checksums (vlib_main_t *vm, vlib_buffer_t *b, ip6_header_t *ip6,
+			 tcp_header_t *th, udp_header_t *uh, u32 oflags)
 {
   int bogus;
-  if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
+  if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
     {
       th->checksum = 0;
       th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
     }
-  if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
+  if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
     {
       uh->checksum = 0;
       uh->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ip6, &bogus);
@@ -88,6 +86,7 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b,
   ip6_header_t *ip6;
   tcp_header_t *th;
   udp_header_t *uh;
+  u32 oflags = vnet_buffer2 (b)->oflags;
 
   ASSERT (!(is_ip4 && is_ip6));
 
@@ -98,16 +97,16 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b,
 
   if (is_ip4)
     {
-      vnet_calc_ip4_checksums (vm, b, ip4, th, uh);
+      vnet_calc_ip4_checksums (vm, b, ip4, th, uh, oflags);
     }
   else if (is_ip6)
     {
-      vnet_calc_ip6_checksums (vm, b, ip6, th, uh);
+      vnet_calc_ip6_checksums (vm, b, ip6, th, uh, oflags);
     }
 
-  b->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
-  b->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
-  b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+  vnet_buffer_offload_flags_clear (b, (VNET_BUFFER_OFFLOAD_F_IP_CKSUM |
+				       VNET_BUFFER_OFFLOAD_F_UDP_CKSUM |
+				       VNET_BUFFER_OFFLOAD_F_TCP_CKSUM));
 }
 
 #endif
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index 332c483aa9d..9e8d164b51f 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -1417,9 +1417,10 @@ ip4_local_l4_csum_validate (vlib_main_t * vm, vlib_buffer_t * p,
     }
 }
 
-#define ip4_local_csum_is_offloaded(_b)					\
-    _b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM				\
-	|| _b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM
+#define ip4_local_csum_is_offloaded(_b)                                       \
+  ((_b->flags & VNET_BUFFER_F_OFFLOAD) &&                                     \
+   (vnet_buffer2 (_b)->oflags &                                               \
+    (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
 
 #define ip4_local_need_csum_check(is_tcp_udp, _b) 			\
     (is_tcp_udp && !(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED 	\
@@ -2066,7 +2067,7 @@ ip4_ttl_and_checksum_check (vlib_buffer_t * b, ip4_header_t * ip, u16 * next,
 
   /* Verify checksum. */
   ASSERT (ip4_header_checksum_is_valid (ip) ||
-	  (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM));
+	  (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
 }
 
 
diff --git a/src/vnet/ip/ip4_inlines.h b/src/vnet/ip/ip4_inlines.h
index 3075fbf42a1..00a47125b8a 100644
--- a/src/vnet/ip/ip4_inlines.h
+++ b/src/vnet/ip/ip4_inlines.h
@@ -125,7 +125,7 @@ vlib_buffer_push_ip4_custom (vlib_main_t * vm, vlib_buffer_t * b,
   if (csum_offload)
     {
       ih->checksum = 0;
-      b->flags |= VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+      vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_IP_CKSUM);
     }
   else
     ih->checksum = ip4_header_checksum (ih);
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index bf6aa7f3212..0c1bc3b33ec 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -1333,15 +1333,23 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  flags[0] = b[0]->flags;
 	  flags[1] = b[1]->flags;
 
+	  u32 oflags[2];
+	  oflags[0] = vnet_buffer2 (b[0])->oflags;
+	  oflags[1] = vnet_buffer2 (b[1])->oflags;
+
+	  u32 l4_offload[2];
+	  l4_offload[0] = (flags[0] & VNET_BUFFER_F_OFFLOAD) &&
+			  (oflags[0] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+					VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
+	  l4_offload[1] = (flags[1] & VNET_BUFFER_F_OFFLOAD) &&
+			  (oflags[1] & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+					VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
+
 	  u32 good_l4_csum[2];
 	  good_l4_csum[0] =
-	    flags[0] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
-			VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-			VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
+	    (flags[0] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[0];
 	  good_l4_csum[1] =
-	    flags[1] & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
-			VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-			VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
+	    (flags[1] & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload[1];
 
 	  u32 udp_offset[2] = { };
 	  u8 is_tcp_udp[2];
@@ -1515,11 +1523,15 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  u8 type = lm->builtin_protocol_by_ip_protocol[ip->protocol];
 
 	  u32 flags = b[0]->flags;
-	  u32 good_l4_csum =
-	    flags & (VNET_BUFFER_F_L4_CHECKSUM_CORRECT |
-		     VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-		     VNET_BUFFER_F_OFFLOAD_UDP_CKSUM);
 
+	  u32 oflags = vnet_buffer2 (b[0])->oflags;
+
+	  u32 l4_offload = (flags & VNET_BUFFER_F_OFFLOAD) &&
+			   (oflags & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
+				      VNET_BUFFER_OFFLOAD_F_UDP_CKSUM));
+
+	  u32 good_l4_csum =
+	    (flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) | l4_offload;
 	  u32 udp_offset;
 	  i16 len_diff = 0;
 	  u8 is_tcp_udp = ip6_next_proto_is_tcp_udp (b[0], ip, &udp_offset);
@@ -1539,8 +1551,9 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  good_l4_csum |= type == IP_BUILTIN_PROTOCOL_UNKNOWN;
 	  len_diff = type == IP_BUILTIN_PROTOCOL_UDP ? len_diff : 0;
 
-	  u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN && !good_l4_csum
-	    && !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
+	  u8 need_csum = type != IP_BUILTIN_PROTOCOL_UNKNOWN &&
+			 !good_l4_csum &&
+			 !(flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED);
 	  if (PREDICT_FALSE (need_csum))
 	    {
 	      flags = ip6_tcp_udp_icmp_validate_checksum (vm, b[0]);
@@ -1556,7 +1569,6 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
 
 	  error = len_diff < 0 ? IP6_ERROR_UDP_LENGTH : error;
-
 	  STATIC_ASSERT (IP6_ERROR_UDP_CHECKSUM + IP_BUILTIN_PROTOCOL_UDP ==
 			 IP6_ERROR_UDP_CHECKSUM,
 			 "Wrong IP6 errors constants");
@@ -1882,8 +1894,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
 	  if (p1->flags & VNET_BUFFER_F_GSO)
 	    ip1_len = gso_mtu_sz (p1);
 
-
-
 	  ip6_mtu_check (p0, ip0_len,
 			 adj0[0].rewrite_header.max_l3_packet_bytes,
 			 is_locally_originated0, &next0, is_midchain,
@@ -1892,18 +1902,15 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
 			 adj1[0].rewrite_header.max_l3_packet_bytes,
 			 is_locally_originated1, &next1, is_midchain,
 			 &error1);
-
 	  /* Don't adjust the buffer for hop count issue; icmp-error node
 	   * wants to see the IP header */
 	  if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
 	    {
 	      p0->current_data -= rw_len0;
 	      p0->current_length += rw_len0;
-
 	      tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
 	      vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
 	      next0 = adj0[0].rewrite_header.next_index;
-
 	      if (PREDICT_FALSE
 		  (adj0[0].rewrite_header.flags & VNET_REWRITE_HAS_FEATURES))
 		vnet_feature_arc_start_w_cfg_index
@@ -2070,7 +2077,6 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm,
 			 adj0[0].rewrite_header.max_l3_packet_bytes,
 			 is_locally_originated0, &next0, is_midchain,
 			 &error0);
-
 	  /* Don't adjust the buffer for hop count issue; icmp-error node
 	   * wants to see the IP header */
 	  if (PREDICT_TRUE (error0 == IP6_ERROR_NONE))
@@ -2289,7 +2295,6 @@ format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
   ip6_hop_by_hop_option_t *opt0, *limit0;
   ip6_hop_by_hop_main_t *hm = &ip6_hop_by_hop_main;
   u8 type0;
-
   s = format (s, "IP6_HOP_BY_HOP: next protocol %d len %d total %d",
 	      hbh0->protocol, (hbh0->length + 1) << 3, total_len);
 
@@ -2312,9 +2317,8 @@ format_ip6_hop_by_hop_ext_hdr (u8 * s, va_list * args)
 	    }
 	  else
 	    {
-	      s =
-		format (s, "\n    unrecognized option %d length %d", type0,
-			opt0->length);
+	      s = format (s, "\n    unrecognized option %d length %d", type0,
+			  opt0->length);
 	    }
 	  opt0 =
 	    (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
@@ -2362,9 +2366,8 @@ format_ip6_hop_by_hop_trace (u8 * s, va_list * args)
 	    }
 	  else
 	    {
-	      s =
-		format (s, "\n    unrecognized option %d length %d", type0,
-			opt0->length);
+	      s = format (s, "\n    unrecognized option %d length %d", type0,
+			  opt0->length);
 	    }
 	  opt0 =
 	    (ip6_hop_by_hop_option_t *) (((u8 *) opt0) + opt0->length +
@@ -3012,8 +3015,7 @@ VLIB_CLI_COMMAND (show_ip6_local, static) =
 
 #ifndef CLIB_MARCH_VARIANT
 int
-vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
-			     u32 table_index)
+vnet_set_ip6_classify_intfc (vlib_main_t *vm, u32 sw_if_index, u32 table_index)
 {
   vnet_main_t *vnm = vnet_get_main ();
   vnet_interface_main_t *im = &vnm->interface_main;
@@ -3044,17 +3046,13 @@ vnet_set_ip6_classify_intfc (vlib_main_t * vm, u32 sw_if_index,
 
       fib_index = fib_table_get_index_for_sw_if_index (FIB_PROTOCOL_IP4,
 						       sw_if_index);
-
-
       if (table_index != (u32) ~ 0)
 	{
 	  dpo_id_t dpo = DPO_INVALID;
-
 	  dpo_set (&dpo,
 		   DPO_CLASSIFY,
 		   DPO_PROTO_IP6,
 		   classify_dpo_create (DPO_PROTO_IP6, table_index));
-
 	  fib_table_entry_special_dpo_add (fib_index,
 					   &pfx,
 					   FIB_SOURCE_CLASSIFY,
diff --git a/src/vnet/ip/ip_frag.c b/src/vnet/ip/ip_frag.c
index 9aa8777514f..33fbb9b2a9d 100644
--- a/src/vnet/ip/ip_frag.c
+++ b/src/vnet/ip/ip_frag.c
@@ -226,7 +226,7 @@ ip4_frag_do_fragment (vlib_main_t * vm, u32 from_bi, u16 mtu,
       to_ip4->checksum = ip4_header_checksum (to_ip4);
 
       /* we've just done the IP checksum .. */
-      to_b->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
+      vnet_buffer_offload_flags_clear (to_b, VNET_BUFFER_OFFLOAD_F_IP_CKSUM);
 
       rem -= len;
       fo += len;
diff --git a/src/vnet/ipsec/ipsec_output.c b/src/vnet/ipsec/ipsec_output.c
index d09a027de29..e6cf837d796 100644
--- a/src/vnet/ipsec/ipsec_output.c
+++ b/src/vnet/ipsec/ipsec_output.c
@@ -309,45 +309,50 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 		next_node_index = im->ah4_encrypt_node_index;
 	      vnet_buffer (b0)->ipsec.sad_index = p0->sa_index;
 
-	      if (is_ipv6)
+	      if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_OFFLOAD))
 		{
-		  if (PREDICT_FALSE
-		      (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM))
-		    {
-		      tcp0->checksum =
-			ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0,
-							   &bogus);
-		      b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
-		    }
-		  if (PREDICT_FALSE
-		      (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
-		    {
-		      udp0->checksum =
-			ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0,
-							   &bogus);
-		      b0->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
-		    }
-		}
-	      else
-		{
-		  if (b0->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
-		    {
-		      ip0->checksum = ip4_header_checksum (ip0);
-		      b0->flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
-		    }
-		  if (PREDICT_FALSE
-		      (b0->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM))
+		  u32 oflags = vnet_buffer2 (b0)->oflags;
+
+		  /*
+		   * Clearing offload flags before checksum is computed
+		   * It guarantees the cache hit!
+		   */
+		  vnet_buffer_offload_flags_clear (b0, oflags);
+
+		  if (is_ipv6)
 		    {
-		      tcp0->checksum =
-			ip4_tcp_udp_compute_checksum (vm, b0, ip0);
-		      b0->flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+		      if (PREDICT_FALSE (oflags &
+					 VNET_BUFFER_OFFLOAD_F_TCP_CKSUM))
+			{
+			  tcp0->checksum = ip6_tcp_udp_icmp_compute_checksum (
+			    vm, b0, ip6_0, &bogus);
+			}
+		      if (PREDICT_FALSE (oflags &
+					 VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
+			{
+			  udp0->checksum = ip6_tcp_udp_icmp_compute_checksum (
+			    vm, b0, ip6_0, &bogus);
+			}
 		    }
-		  if (PREDICT_FALSE
-		      (b0->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
+		  else
 		    {
-		      udp0->checksum =
-			ip4_tcp_udp_compute_checksum (vm, b0, ip0);
-		      b0->flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+		      if (PREDICT_FALSE (oflags &
+					 VNET_BUFFER_OFFLOAD_F_IP_CKSUM))
+			{
+			  ip0->checksum = ip4_header_checksum (ip0);
+			}
+		      if (PREDICT_FALSE (oflags &
+					 VNET_BUFFER_OFFLOAD_F_TCP_CKSUM))
+			{
+			  tcp0->checksum =
+			    ip4_tcp_udp_compute_checksum (vm, b0, ip0);
+			}
+		      if (PREDICT_FALSE (oflags &
+					 VNET_BUFFER_OFFLOAD_F_UDP_CKSUM))
+			{
+			  udp0->checksum =
+			    ip4_tcp_udp_compute_checksum (vm, b0, ip0);
+			}
 		    }
 		}
 	      vlib_buffer_advance (b0, iph_offset);
diff --git a/src/vnet/pg/cli.c b/src/vnet/pg/cli.c
index 625198634f5..3ef7a959f20 100644
--- a/src/vnet/pg/cli.c
+++ b/src/vnet/pg/cli.c
@@ -379,7 +379,9 @@ new_stream (vlib_main_t * vm,
       else if (unformat (input, "buffer-flags %U",
 			 unformat_vnet_buffer_flags, &s.buffer_flags))
 	;
-
+      else if (unformat (input, "buffer-offload-flags %U",
+			 unformat_vnet_buffer_offload_flags, &s.buffer_oflags))
+	;
       else if (unformat (input, "node %U",
 			 unformat_vlib_node, vm, &s.node_index))
 	;
diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c
index 940bf124c56..d3501b3e3b6 100644
--- a/src/vnet/pg/input.c
+++ b/src/vnet/pg/input.c
@@ -1537,13 +1537,14 @@ pg_input_trace (pg_main_t * pg,
 }
 
 static_always_inline void
-fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
-			   int gso_enabled, u32 gso_size)
+fill_buffer_offload_flags (vlib_main_t *vm, u32 *buffers, u32 n_buffers,
+			   u32 buffer_oflags, int gso_enabled, u32 gso_size)
 {
   for (int i = 0; i < n_buffers; i++)
     {
       vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[i]);
       u8 l4_proto = 0;
+      u32 oflags = 0;
 
       ethernet_header_t *eh =
 	(ethernet_header_t *) vlib_buffer_get_current (b0);
@@ -1574,10 +1575,11 @@ fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
 	  vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4);
 	  l4_proto = ip4->protocol;
 	  b0->flags |=
-	    (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_IP_CKSUM);
-	  b0->flags |= (VNET_BUFFER_F_L2_HDR_OFFSET_VALID
-			| VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
-			VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
+	    (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
+	     VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+	     VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
+	  if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
+	    oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
 	}
       else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6))
 	{
@@ -1594,7 +1596,8 @@ fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
 
       if (l4_proto == IP_PROTOCOL_TCP)
 	{
-	  b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+	  if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
+	    oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
 
 	  /* only set GSO flag for chained buffers */
 	  if (gso_enabled && (b0->flags & VLIB_BUFFER_NEXT_PRESENT))
@@ -1609,8 +1612,12 @@ fill_buffer_offload_flags (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
 	}
       else if (l4_proto == IP_PROTOCOL_UDP)
 	{
-	  b0->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+	  if (buffer_oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
+	    oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
 	}
+
+      if (oflags)
+	vnet_buffer_offload_flags_set (b0, oflags);
     }
 }
 
@@ -1710,13 +1717,11 @@ pg_generate_packets (vlib_node_runtime_t * node,
 	    vnet_buffer (b)->feature_arc_index = feature_arc_index;
 	  }
 
-      if (pi->gso_enabled ||
-	  (s->buffer_flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
-			      VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
-			      VNET_BUFFER_F_OFFLOAD_IP_CKSUM)))
+      if (pi->gso_enabled || (s->buffer_flags & VNET_BUFFER_F_OFFLOAD))
 	{
 	  fill_buffer_offload_flags (vm, to_next, n_this_frame,
-				     pi->gso_enabled, pi->gso_size);
+				     s->buffer_oflags, pi->gso_enabled,
+				     pi->gso_size);
 	}
 
       n_trace = vlib_get_trace_count (vm, node);
diff --git a/src/vnet/pg/pg.h b/src/vnet/pg/pg.h
index da2c2b54347..da5af25b22e 100644
--- a/src/vnet/pg/pg.h
+++ b/src/vnet/pg/pg.h
@@ -125,9 +125,12 @@ typedef struct pg_stream_t
      for max_packet_bytes. */
   u32 buffer_bytes;
 
-  /* Buffer flags to set in each packet e.g. checksum offload flags */
+  /* Buffer flags to set in each packet e.g. l2 valid  flags */
   u32 buffer_flags;
 
+  /* Buffer offload flags to set in each packet e.g. checksum offload flags */
+  u32 buffer_oflags;
+
   /* Last packet length if packet size edit type is increment. */
   u32 last_increment_packet_size;
 
diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index e2c2b231985..235a59be0b3 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -430,7 +430,7 @@ tcp_compute_checksum (tcp_connection_t * tc, vlib_buffer_t * b)
     }
   else
     {
-      b->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+      vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_TCP_CKSUM);
     }
   return checksum;
 }
diff --git a/src/vnet/udp/udp_inlines.h b/src/vnet/udp/udp_inlines.h
index ec800f9a413..067289c9979 100644
--- a/src/vnet/udp/udp_inlines.h
+++ b/src/vnet/udp/udp_inlines.h
@@ -35,7 +35,7 @@ vlib_buffer_push_udp (vlib_buffer_t * b, u16 sp, u16 dp, u8 offload_csum)
   uh->checksum = 0;
   uh->length = clib_host_to_net_u16 (udp_len);
   if (offload_csum)
-    b->flags |= VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
+    vnet_buffer_offload_flags_set (b, VNET_BUFFER_OFFLOAD_F_UDP_CKSUM);
   vnet_buffer (b)->l4_hdr_offset = (u8 *) uh - b->data;
   b->flags |= VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
   return uh;
diff --git a/src/vnet/vxlan-gbp/encap.c b/src/vnet/vxlan-gbp/encap.c
index 4514fdae7a7..2a4e8a8e312 100644
--- a/src/vnet/vxlan-gbp/encap.c
+++ b/src/vnet/vxlan-gbp/encap.c
@@ -102,14 +102,14 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
   u8 const underlay_hdr_len = is_ip4 ?
     sizeof (ip4_vxlan_gbp_header_t) : sizeof (ip6_vxlan_gbp_header_t);
   u16 const l3_len = is_ip4 ? sizeof (ip4_header_t) : sizeof (ip6_header_t);
-  u32 const csum_flags = is_ip4 ? VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
-    VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
-    VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID :
-    VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
-    VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
-  u32 const inner_packet_csum_offload_flags =
-    VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
-    VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+  u32 const csum_flags =
+    is_ip4 ? VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+	       VNET_BUFFER_F_L4_HDR_OFFSET_VALID :
+	     VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+	       VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
+  u32 const outer_packet_csum_offload_flags =
+    is_ip4 ? VNET_BUFFER_OFFLOAD_F_IP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM :
+	     VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
   u32 const inner_packet_removed_flags =
     VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_IS_IP6 |
     VNET_BUFFER_F_L2_HDR_OFFSET_VALID | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
@@ -147,7 +147,7 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
 	  n_left_from -= 2;
 
 	  u32 or_flags = b[0]->flags | b[1]->flags;
-	  if (csum_offload && (or_flags & inner_packet_csum_offload_flags))
+	  if (csum_offload && (or_flags & VNET_BUFFER_F_OFFLOAD))
 	    {
 	      /* Only calculate the non-GSO packet csum offload */
 	      if ((b[0]->flags & VNET_BUFFER_F_GSO) == 0)
@@ -300,9 +300,13 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
 	      b[0]->flags |= csum_flags;
 	      vnet_buffer (b[0])->l3_hdr_offset = l3_0 - b[0]->data;
 	      vnet_buffer (b[0])->l4_hdr_offset = (u8 *) udp0 - b[0]->data;
+	      vnet_buffer_offload_flags_set (b[0],
+					     outer_packet_csum_offload_flags);
 	      b[1]->flags |= csum_flags;
 	      vnet_buffer (b[1])->l3_hdr_offset = l3_1 - b[1]->data;
 	      vnet_buffer (b[1])->l4_hdr_offset = (u8 *) udp1 - b[1]->data;
+	      vnet_buffer_offload_flags_set (b[1],
+					     outer_packet_csum_offload_flags);
 	    }
 	  /* IPv4 UDP checksum only if checksum offload is used */
 	  else if (is_ip4)
@@ -387,7 +391,7 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
 	  n_left_from -= 1;
 	  n_left_to_next -= 1;
 
-	  if (csum_offload && (b[0]->flags & inner_packet_csum_offload_flags))
+	  if (csum_offload && (b[0]->flags & VNET_BUFFER_F_OFFLOAD))
 	    {
 	      /* Only calculate the non-GSO packet csum offload */
 	      if ((b[0]->flags & VNET_BUFFER_F_GSO) == 0)
@@ -477,6 +481,8 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
 	      b[0]->flags |= csum_flags;
 	      vnet_buffer (b[0])->l3_hdr_offset = l3_0 - b[0]->data;
 	      vnet_buffer (b[0])->l4_hdr_offset = (u8 *) udp0 - b[0]->data;
+	      vnet_buffer_offload_flags_set (b[0],
+					     outer_packet_csum_offload_flags);
 	    }
 	  /* IPv4 UDP checksum only if checksum offload is used */
 	  else if (is_ip4)
diff --git a/src/vnet/vxlan/encap.c b/src/vnet/vxlan/encap.c
index 787758a94f8..476e0f2b3b7 100644
--- a/src/vnet/vxlan/encap.c
+++ b/src/vnet/vxlan/encap.c
@@ -98,14 +98,14 @@ vxlan_encap_inline (vlib_main_t * vm,
   u8 const underlay_hdr_len = is_ip4 ?
     sizeof (ip4_vxlan_header_t) : sizeof (ip6_vxlan_header_t);
   u16 const l3_len = is_ip4 ? sizeof (ip4_header_t) : sizeof (ip6_header_t);
-  u32 const csum_flags = is_ip4 ? VNET_BUFFER_F_OFFLOAD_IP_CKSUM |
-    VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
-    VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID :
-    VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
-    VNET_BUFFER_F_L3_HDR_OFFSET_VALID | VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
-  u32 const inner_packet_csum_offload_flags =
-    VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
-    VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+  u32 const csum_flags =
+    is_ip4 ? VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+	       VNET_BUFFER_F_L4_HDR_OFFSET_VALID :
+	     VNET_BUFFER_F_IS_IP6 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
+	       VNET_BUFFER_F_L4_HDR_OFFSET_VALID;
+  u32 const outer_packet_csum_offload_flags =
+    is_ip4 ? VNET_BUFFER_OFFLOAD_F_IP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM :
+	     VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
   u32 const inner_packet_removed_flags =
     VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_IS_IP6 |
     VNET_BUFFER_F_L2_HDR_OFFSET_VALID | VNET_BUFFER_F_L3_HDR_OFFSET_VALID |
@@ -144,7 +144,7 @@ vxlan_encap_inline (vlib_main_t * vm,
 	  b += 2;
 
 	  u32 or_flags = b0->flags | b1->flags;
-	  if (csum_offload && (or_flags & inner_packet_csum_offload_flags))
+	  if (csum_offload && (or_flags & VNET_BUFFER_F_OFFLOAD))
 	    {
 	      /* Only calculate the non-GSO packet csum offload */
 	      if ((b0->flags & VNET_BUFFER_F_GSO) == 0)
@@ -284,9 +284,13 @@ vxlan_encap_inline (vlib_main_t * vm,
 	      b0->flags |= csum_flags;
 	      vnet_buffer (b0)->l3_hdr_offset = l3_0 - b0->data;
 	      vnet_buffer (b0)->l4_hdr_offset = (u8 *) udp0 - b0->data;
+	      vnet_buffer_offload_flags_set (b0,
+					     outer_packet_csum_offload_flags);
 	      b1->flags |= csum_flags;
 	      vnet_buffer (b1)->l3_hdr_offset = l3_1 - b1->data;
 	      vnet_buffer (b1)->l4_hdr_offset = (u8 *) udp1 - b1->data;
+	      vnet_buffer_offload_flags_set (b1,
+					     outer_packet_csum_offload_flags);
 	    }
 	  /* IPv4 UDP checksum only if checksum offload is used */
 	  else if (is_ip4)
@@ -377,7 +381,7 @@ vxlan_encap_inline (vlib_main_t * vm,
 	  vlib_buffer_t *b0 = b[0];
 	  b += 1;
 
-	  if (csum_offload && (b0->flags & inner_packet_csum_offload_flags))
+	  if (csum_offload && (b0->flags & VNET_BUFFER_F_OFFLOAD))
 	    {
 	      /* Only calculate the non-GSO packet csum offload */
 	      if ((b0->flags & VNET_BUFFER_F_GSO) == 0)
@@ -459,6 +463,8 @@ vxlan_encap_inline (vlib_main_t * vm,
 	      b0->flags |= csum_flags;
 	      vnet_buffer (b0)->l3_hdr_offset = l3_0 - b0->data;
 	      vnet_buffer (b0)->l4_hdr_offset = (u8 *) udp0 - b0->data;
+	      vnet_buffer_offload_flags_set (b0,
+					     outer_packet_csum_offload_flags);
 	    }
 	  /* IPv4 UDP checksum only if checksum offload is used */
 	  else if (is_ip4)
diff --git a/test/test_offload.py b/test/test_offload.py
index 8b045bfe131..9f6aec65482 100644
--- a/test/test_offload.py
+++ b/test/test_offload.py
@@ -39,7 +39,8 @@ class TestOffload(VppTestCase):
                 "  interface loop0\n"
                 "  tx-interface loop1\n"
                 "  node loop1-output\n"
-                "  buffer-flags ip4 offload-ip-cksum offload-udp-cksum\n"
+                "  buffer-flags ip4 offload\n"
+                "  buffer-offload-flags offload-ip-cksum offload-udp-cksum\n"
                 "  data {\n"
                 "    IP4: 1.2.3 -> dead.0000.0001\n"
                 "    UDP: 11.22.33.44 -> 11.22.34.44\n"
diff --git a/test/test_pcap.py b/test/test_pcap.py
index 5c086ef21c8..c058c5467af 100644
--- a/test/test_pcap.py
+++ b/test/test_pcap.py
@@ -42,7 +42,8 @@ class TestPcap(VppTestCase):
                 "  interface loop0\n"
                 "  tx-interface loop1\n"
                 "  node loop1-output\n"
-                "  buffer-flags ip4 offload-ip-cksum offload-udp-cksum\n"
+                "  buffer-flags ip4 offload\n"
+                "  buffer-offload-flags offload-ip-cksum offload-udp-cksum\n"
                 "  data {\n"
                 "    IP4: 1.2.3 -> dead.0000.0001\n"
                 "    UDP: 11.22.33.44 -> 11.22.34.44\n"