aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMohsin Kazmi <sykazmi@cisco.com>2021-04-23 15:16:50 +0200
committerDamjan Marion <dmarion@me.com>2021-05-05 11:10:51 +0000
commita7e830ed468f35590871988e68153222325870b8 (patch)
tree6a92f425056bf061e7df502e9c6235cfaf2ca62a
parentbaa18701b9d54d8924771c25b96bd5d99472b7ad (diff)
vlib: move offload flags to 1st cacheline in vlib_buffer_t
Type: improvement Some tests i.e. ipsec see performance regression when offload flags are moved to 2nd cacheline. This patch moves them back to 1st cacheline. Change-Id: I6ead45ff6d2c467b0d248f409e27c2ba31758741 Signed-off-by: Mohsin Kazmi <sykazmi@cisco.com>
-rw-r--r--src/plugins/avf/output.c2
-rw-r--r--src/plugins/dpdk/device/device.c2
-rw-r--r--src/plugins/gtpu/gtpu_decap.c4
-rw-r--r--src/vnet/buffer.c2
-rw-r--r--src/vnet/buffer.h14
-rw-r--r--src/vnet/devices/virtio/device.c4
-rw-r--r--src/vnet/devices/virtio/vhost_user_output.c2
-rw-r--r--src/vnet/interface_output.h3
-rw-r--r--src/vnet/ip/ip4_forward.c4
-rw-r--r--src/vnet/ip/ip6_forward.c6
-rw-r--r--src/vnet/ipsec/ipsec_output.c2
11 files changed, 22 insertions, 23 deletions
diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c
index e16d79dfc6d..f35f7bb962b 100644
--- a/src/plugins/avf/output.c
+++ b/src/plugins/avf/output.c
@@ -60,7 +60,7 @@ avf_tx_prepare_cksum (vlib_buffer_t * b, u8 is_tso)
if (!is_tso && !(b->flags & VNET_BUFFER_F_OFFLOAD))
return 0;
- u32 oflags = vnet_buffer2 (b)->oflags;
+ u32 oflags = vnet_buffer (b)->oflags;
u32 is_tcp = is_tso || oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
u32 is_udp = !is_tso && oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
u32 is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c
index 94b3f6b289a..7e0f2d43994 100644
--- a/src/plugins/dpdk/device/device.c
+++ b/src/plugins/dpdk/device/device.c
@@ -228,7 +228,7 @@ dpdk_buffer_tx_offload (dpdk_device_t * xd, vlib_buffer_t * b,
if (PREDICT_TRUE (((b->flags & VNET_BUFFER_F_OFFLOAD) | tso) == 0))
return;
- oflags = vnet_buffer2 (b)->oflags;
+ oflags = vnet_buffer (b)->oflags;
ip_cksum = oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM;
tcp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM;
udp_cksum = oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM;
diff --git a/src/plugins/gtpu/gtpu_decap.c b/src/plugins/gtpu/gtpu_decap.c
index 5657c08031b..40243dbcc53 100644
--- a/src/plugins/gtpu/gtpu_decap.c
+++ b/src/plugins/gtpu/gtpu_decap.c
@@ -1256,12 +1256,12 @@ static char *gtpu_flow_error_strings[] = {
#define gtpu_local_need_csum_check(_b) \
(!(_b->flags & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED || \
(_b->flags & VNET_BUFFER_F_OFFLOAD && \
- vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
+ vnet_buffer (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
#define gtpu_local_csum_is_valid(_b) \
((_b->flags & VNET_BUFFER_F_L4_CHECKSUM_CORRECT || \
(_b->flags & VNET_BUFFER_F_OFFLOAD && \
- vnet_buffer2 (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0)
+ vnet_buffer (_b)->oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)) != 0)
static_always_inline u8
gtpu_validate_udp_csum (vlib_main_t * vm, vlib_buffer_t *b)
diff --git a/src/vnet/buffer.c b/src/vnet/buffer.c
index b41036d8124..545018a79e9 100644
--- a/src/vnet/buffer.c
+++ b/src/vnet/buffer.c
@@ -22,7 +22,7 @@ format_vnet_buffer_offload (u8 *s, va_list *args)
vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
#define _(bit, name, ss, v) \
- if (v && (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name)) \
+ if (v && (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_##name)) \
s = format (s, "%s ", ss);
foreach_vnet_buffer_offload_flag
#undef _
diff --git a/src/vnet/buffer.h b/src/vnet/buffer.h
index 19585e22506..0816232aed7 100644
--- a/src/vnet/buffer.h
+++ b/src/vnet/buffer.h
@@ -138,7 +138,8 @@ typedef struct
i16 l3_hdr_offset;
i16 l4_hdr_offset;
u8 feature_arc_index;
- u8 dont_waste_me;
+ /* offload flags */
+ u8 oflags;
union
{
@@ -472,9 +473,6 @@ typedef struct
u16 gso_size;
/* size of L4 prototol header */
u16 gso_l4_hdr_sz;
-
- /* offload flags */
- u32 oflags;
};
struct
@@ -526,12 +524,12 @@ vnet_buffer_offload_flags_set (vlib_buffer_t *b, u32 oflags)
if (b->flags & VNET_BUFFER_F_OFFLOAD)
{
/* add a flag to existing offload */
- vnet_buffer2 (b)->oflags |= oflags;
+ vnet_buffer (b)->oflags |= oflags;
}
else
{
/* no offload yet: reset offload flags to new value */
- vnet_buffer2 (b)->oflags = oflags;
+ vnet_buffer (b)->oflags = oflags;
b->flags |= VNET_BUFFER_F_OFFLOAD;
}
}
@@ -539,8 +537,8 @@ vnet_buffer_offload_flags_set (vlib_buffer_t *b, u32 oflags)
static_always_inline void
vnet_buffer_offload_flags_clear (vlib_buffer_t *b, u32 oflags)
{
- vnet_buffer2 (b)->oflags &= ~oflags;
- if (0 == vnet_buffer2 (b)->oflags)
+ vnet_buffer (b)->oflags &= ~oflags;
+ if (0 == vnet_buffer (b)->oflags)
b->flags &= ~VNET_BUFFER_F_OFFLOAD;
}
diff --git a/src/vnet/devices/virtio/device.c b/src/vnet/devices/virtio/device.c
index 237e3d91f1a..20264787292 100644
--- a/src/vnet/devices/virtio/device.c
+++ b/src/vnet/devices/virtio/device.c
@@ -285,7 +285,7 @@ static void
set_checksum_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr,
const int is_l2)
{
- u32 oflags = vnet_buffer2 (b)->oflags;
+ u32 oflags = vnet_buffer (b)->oflags;
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
@@ -334,7 +334,7 @@ set_checksum_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr,
static void
set_gso_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr, const int is_l2)
{
- u32 oflags = vnet_buffer2 (b)->oflags;
+ u32 oflags = vnet_buffer (b)->oflags;
if (b->flags & VNET_BUFFER_F_IS_IP4)
{
diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c
index 8dbddea46f5..6406fe3f587 100644
--- a/src/vnet/devices/virtio/vhost_user_output.c
+++ b/src/vnet/devices/virtio/vhost_user_output.c
@@ -225,7 +225,7 @@ vhost_user_handle_tx_offload (vhost_user_intf_t * vui, vlib_buffer_t * b,
generic_header_offset_t gho = { 0 };
int is_ip4 = b->flags & VNET_BUFFER_F_IS_IP4;
int is_ip6 = b->flags & VNET_BUFFER_F_IS_IP6;
- u32 oflags = vnet_buffer2 (b)->oflags;
+ u32 oflags = vnet_buffer (b)->oflags;
ASSERT (!(is_ip4 && is_ip6));
vnet_generic_header_offset_parser (b, &gho, 1 /* l2 */ , is_ip4, is_ip6);
diff --git a/src/vnet/interface_output.h b/src/vnet/interface_output.h
index bfd2066c2a8..bb27cdb616e 100644
--- a/src/vnet/interface_output.h
+++ b/src/vnet/interface_output.h
@@ -85,7 +85,7 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b,
ip6_header_t *ip6;
tcp_header_t *th;
udp_header_t *uh;
- u32 oflags = vnet_buffer2 (b)->oflags;
+ u32 oflags;
if (!(b->flags & VNET_BUFFER_F_OFFLOAD))
return;
@@ -96,6 +96,7 @@ vnet_calc_checksums_inline (vlib_main_t * vm, vlib_buffer_t * b,
ip6 = (ip6_header_t *) (b->data + vnet_buffer (b)->l3_hdr_offset);
th = (tcp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
uh = (udp_header_t *) (b->data + vnet_buffer (b)->l4_hdr_offset);
+ oflags = vnet_buffer (b)->oflags;
if (is_ip4)
{
diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c
index ddf97d48a15..da8522de775 100644
--- a/src/vnet/ip/ip4_forward.c
+++ b/src/vnet/ip/ip4_forward.c
@@ -1419,7 +1419,7 @@ ip4_local_l4_csum_validate (vlib_main_t * vm, vlib_buffer_t * p,
#define ip4_local_csum_is_offloaded(_b) \
((_b->flags & VNET_BUFFER_F_OFFLOAD) && \
- (vnet_buffer2 (_b)->oflags & \
+ (vnet_buffer (_b)->oflags & \
(VNET_BUFFER_OFFLOAD_F_TCP_CKSUM | VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)))
#define ip4_local_need_csum_check(is_tcp_udp, _b) \
@@ -2067,7 +2067,7 @@ ip4_ttl_and_checksum_check (vlib_buffer_t * b, ip4_header_t * ip, u16 * next,
/* Verify checksum. */
ASSERT (ip4_header_checksum_is_valid (ip) ||
- (vnet_buffer2 (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
+ (vnet_buffer (b)->oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM));
}
always_inline uword
diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c
index 0c1bc3b33ec..4f1b148c365 100644
--- a/src/vnet/ip/ip6_forward.c
+++ b/src/vnet/ip/ip6_forward.c
@@ -1334,8 +1334,8 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
flags[1] = b[1]->flags;
u32 oflags[2];
- oflags[0] = vnet_buffer2 (b[0])->oflags;
- oflags[1] = vnet_buffer2 (b[1])->oflags;
+ oflags[0] = vnet_buffer (b[0])->oflags;
+ oflags[1] = vnet_buffer (b[1])->oflags;
u32 l4_offload[2];
l4_offload[0] = (flags[0] & VNET_BUFFER_F_OFFLOAD) &&
@@ -1524,7 +1524,7 @@ ip6_local_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
u32 flags = b[0]->flags;
- u32 oflags = vnet_buffer2 (b[0])->oflags;
+ u32 oflags = vnet_buffer (b[0])->oflags;
u32 l4_offload = (flags & VNET_BUFFER_F_OFFLOAD) &&
(oflags & (VNET_BUFFER_OFFLOAD_F_TCP_CKSUM |
diff --git a/src/vnet/ipsec/ipsec_output.c b/src/vnet/ipsec/ipsec_output.c
index 527aea5d11d..598195f2737 100644
--- a/src/vnet/ipsec/ipsec_output.c
+++ b/src/vnet/ipsec/ipsec_output.c
@@ -309,7 +309,7 @@ ipsec_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_OFFLOAD))
{
- u32 oflags = vnet_buffer2 (b0)->oflags;
+ u32 oflags = vnet_buffer (b0)->oflags;
/*
* Clearing offload flags before checksum is computed