vlib: move offload flags to 1st cacheline in vlib_buffer_t

Type: improvement

Some tests i.e. ipsec see performance regression when offload flags
are moved to 2nd cacheline. This patch moves them back to 1st cacheline.

Change-Id: I6ead45ff6d2c467b0d248f409e27c2ba31758741
Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>

11 files changed, 22 insertions, 23 deletions

diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c
index e16d79dfc6d..f35f7bb962b 100644
--- a/src/plugins/avf/output.c
+++ b/src/plugins/avf/output.c
@@ -60,7 +60,7 @@ avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso)
   if (!is_tso && !(b->flags & VNET_BUFFER_F_OFFLOAD))
     return 0;
 
-  u32 oflags = vnet_buffer2 (b)->oflags;
+  u32 oflags = vnet_buffer (b)->oflags;
   u32 is_tcp = is_tso || oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
   u32 is_udp = !is_tso && oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
   u32 is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index 94b3f6b289a..7e0f2d43994 100644
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -228,7 +228,7 @@ dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b,
   if (PREDICT_TRUE (((b->flags & VNET_BUFFER_F_OFFLOAD) | tso) == 0))
     return;
 
-  oflags = vnet_buffer2 (b)->oflags;
+  oflags = vnet_buffer (b)->oflags;
   ip_cksum = oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
   tcp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
   udp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
diff --git a/src/plugins/gtpu/gtpu_decap.c b/src/plugins/gtpu/gtpu_decap.c
index 5657c08031b..40243dbcc53 100644
--- a/src/plugins/gtpu/gtpu_decap.c
+++ b/src/plugins/gtpu/gtpu_decap.c
@@ -1256,12 +1256,12 @@ static char *gtpu_flow_error_strings[] = {
 #define gtpu_local_need_csum_check(_b)                                        \
   (!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED ||                        \
      (_b->flags & VNET_BUFFER_F_OFFLOAD &&                                    \
-      vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
+      vnet_buffer (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
 
 #define gtpu_local_csum_is_valid(_b)                                          \
   ((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT ||                          \
     (_b->flags & VNET_BUFFER_F_OFFLOAD &&                                     \
-     vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0)
+     vnet_buffer (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0)
 
 static_always_inline u8
 gtpu_validate_udp_csum (vlib_main_t * vm, vlib_buffer_t *b)
diff --git a/src/vnet/buffer.c b/src/vnet/buffer.c
index b41036d8124..545018a79e9 100644
--- a/src/vnet/buffer.c
+++ b/src/vnet/buffer.c
@@ -22,7 +22,7 @@ format_vnet_buffer_offload (u8 *s, va_list *args)
   vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
 
 #define _(bit, name, ss, v)                                                   \
-  if (v && (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name))         \
+  if (v && (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name))          \
     s = format (s, "%s ", ss);
   foreach_vnet_buffer_offload_flag
 #undef _
diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h
index 19585e22506..0816232aed7 100644
--- a/src/vnet/buffer.h
+++ b/src/vnet/buffer.h
@@ -138,7 +138,8 @@ typedef struct
   i16 l3_hdr_offset;
   i16 l4_hdr_offset;
   u8 feature_arc_index;
-  u8 dont_waste_me;
+  /* offload flags */
+  u8 oflags;
 
   union
   {
@@ -472,9 +473,6 @@ typedef struct
     u16 gso_size;
     /* size of L4 prototol header */
     u16 gso_l4_hdr_sz;
-
-    /* offload flags */
-    u32 oflags;
   };
 
   struct
@@ -526,12 +524,12 @@ vnet_buffer_offload_flags_set (vlib_buffer_t *b, u32 oflags)
   if (b->flags & VNET_BUFFER_F_OFFLOAD)
     {
       /* add a flag to existing offload */
-      vnet_buffer2 (b)->oflags |= oflags;
+      vnet_buffer (b)->oflags |= oflags;
     }
   else
     {
       /* no offload yet: reset offload flags to new value */
-      vnet_buffer2 (b)->oflags = oflags;
+      vnet_buffer (b)->oflags = oflags;
       b->flags |= VNET_BUFFER_F_OFFLOAD;
     }
 }
@@ -539,8 +537,8 @@ vnet_buffer_offload_flags_set (vlib_buffer_t *b, u32 oflags)
 static_always_inline void
 vnet_buffer_offload_flags_clear (vlib_buffer_t *b, u32 oflags)
 {
-  vnet_buffer2 (b)->oflags &= ~oflags;
-  if (0 == vnet_buffer2 (b)->oflags)
+  vnet_buffer (b)->oflags &= ~oflags;
+  if (0 == vnet_buffer (b)->oflags)
     b->flags &= ~VNET_BUFFER_F_OFFLOAD;
 }
 
diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c
index 237e3d91f1a..20264787292 100644
--- a/src/vnet/devices/virtio/device.c
+++ b/src/vnet/devices/virtio/device.c
@@ -285,7 +285,7 @@ static void
 set_checksum_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr,
 		      const int is_l2)
 {
-  u32 oflags = vnet_buffer2 (b)->oflags;
+  u32 oflags = vnet_buffer (b)->oflags;
 
   if (b->flags & VNET_BUFFER_F_IS_IP4)
     {
@@ -334,7 +334,7 @@ set_checksum_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr,
 static void
 set_gso_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr, const int is_l2)
 {
-  u32 oflags = vnet_buffer2 (b)->oflags;
+  u32 oflags = vnet_buffer (b)->oflags;
 
   if (b->flags & VNET_BUFFER_F_IS_IP4)
     {
diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c
index 8dbddea46f5..6406fe3f587 100644
--- a/src/vnet/devices/virtio/vhost_user_output.c
+++ b/src/vnet/devices/virtio/vhost_user_output.c
@@ -225,7 +225,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
   generic_header_offset_t gho = { 0 };
   int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
   int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
-  u32 oflags = vnet_buffer2 (b)->oflags;
+  u32 oflags = vnet_buffer (b)->oflags;
 
   ASSERT (!(is_ip4 && is_ip6));
   vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6);
diff --git a/src/vnet/interface_output.h b/src/vnet/interface_output.h
index bfd2066c2a8..bb27cdb616e 100644
--- a/src/vnet/interface_output.h
+++ b/src/vnet/interface_output.h
@@ -85,7 +85,7 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b,
   ip6_header_t *ip6;
   tcp_header_t *th;
   udp_header_t *uh;
-  u32 oflags = vnet_buffer2 (b)->oflags;
+  u32 oflags;
 
   if (!(b->flags & VNET_BUFFER_F_OFFLOAD))
     return;
@@ -96,6 +96,7 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b,
   ip6 = (ip6_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
   th = (tcp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
   uh = (udp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
+  oflags = vnet_buffer (b)->oflags;
 
   if (is_ip4)
     {
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index ddf97d48a15..da8522de775 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -1419,7 +1419,7 @@ ip4_local_l4_csum_validate (vlib_main_t * vm, vlib_buffer_t * p,
 
 #define ip4_local_csum_is_offloaded(_b)                                       \
   ((_b->flags & VNET_BUFFER_F_OFFLOAD) &&                                     \
-   (vnet_buffer2 (_b)->oflags &                                               \
+   (vnet_buffer (_b)->oflags &                                                \
     (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
 
 #define ip4_local_need_csum_check(is_tcp_udp, _b) 			\
@@ -2067,7 +2067,7 @@ ip4_ttl_and_checksum_check (vlib_buffer_t * b, ip4_header_t * ip, u16 * next,
 
   /* Verify checksum. */
   ASSERT (ip4_header_checksum_is_valid (ip) ||
-	  (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
+	  (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
 }
 
 always_inline uword
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index 0c1bc3b33ec..4f1b148c365 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -1334,8 +1334,8 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	  flags[1] = b[1]->flags;
 
 	  u32 oflags[2];
-	  oflags[0] = vnet_buffer2 (b[0])->oflags;
-	  oflags[1] = vnet_buffer2 (b[1])->oflags;
+	  oflags[0] = vnet_buffer (b[0])->oflags;
+	  oflags[1] = vnet_buffer (b[1])->oflags;
 
 	  u32 l4_offload[2];
 	  l4_offload[0] = (flags[0] & VNET_BUFFER_F_OFFLOAD) &&
@@ -1524,7 +1524,7 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
 	  u32 flags = b[0]->flags;
 
-	  u32 oflags = vnet_buffer2 (b[0])->oflags;
+	  u32 oflags = vnet_buffer (b[0])->oflags;
 
 	  u32 l4_offload = (flags & VNET_BUFFER_F_OFFLOAD) &&
 			   (oflags & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
diff --git a/src/vnet/ipsec/ipsec_output.c b/src/vnet/ipsec/ipsec_output.c
index 527aea5d11d..598195f2737 100644
--- a/src/vnet/ipsec/ipsec_output.c
+++ b/src/vnet/ipsec/ipsec_output.c
@@ -309,7 +309,7 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
 	      if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_OFFLOAD))
 		{
-		  u32 oflags = vnet_buffer2 (b0)->oflags;
+		  u32 oflags = vnet_buffer (b0)->oflags;
 
 		  /*
 		   * Clearing offload flags before checksum is computed