summaryrefslogtreecommitdiffstats
path: root/src/vnet
diff options
context:
space:
mode:
Diffstat (limited to 'src/vnet')
-rw-r--r--src/vnet/adj/rewrite.c30
-rw-r--r--src/vnet/adj/rewrite.h138
-rw-r--r--src/vnet/vxlan-gbp/encap.c23
-rw-r--r--src/vnet/vxlan/encap.c14
4 files changed, 33 insertions, 172 deletions
diff --git a/src/vnet/adj/rewrite.c b/src/vnet/adj/rewrite.c
index 975dc4ab817..95671b06930 100644
--- a/src/vnet/adj/rewrite.c
+++ b/src/vnet/adj/rewrite.c
@@ -40,30 +40,6 @@
#include <vnet/vnet.h>
#include <vnet/ip/lookup.h>
-void
-vnet_rewrite_copy_slow_path (vnet_rewrite_data_t * p0,
- vnet_rewrite_data_t * rw0,
- word n_left, uword most_likely_size)
-{
- uword n_done =
- round_pow2 (most_likely_size, sizeof (rw0[0])) / sizeof (rw0[0]);
-
- p0 -= n_done;
- rw0 -= n_done;
-
- /* As we enter the cleanup loop, p0 and rw0 point to the last chunk written
- by the fast path. Hence, the constant 1, which the
- vnet_rewrite_copy_one macro renders as p0[-1] = rw0[-1]. */
-
- while (n_left > 0)
- {
- vnet_rewrite_copy_one (p0, rw0, 1);
- p0--;
- rw0--;
- n_left--;
- }
-}
-
u8 *
format_vnet_rewrite (u8 * s, va_list * args)
{
@@ -72,6 +48,8 @@ format_vnet_rewrite (u8 * s, va_list * args)
CLIB_UNUSED (u32 indent) = va_arg (*args, u32);
vnet_main_t *vnm = vnet_get_main ();
+ ASSERT (rw->data_bytes <= max_data_bytes);
+
if (rw->sw_if_index != ~0)
{
vnet_sw_interface_t *si;
@@ -86,9 +64,7 @@ format_vnet_rewrite (u8 * s, va_list * args)
/* Format rewrite string. */
if (rw->data_bytes > 0)
- s = format (s, " %U",
- format_hex_bytes,
- rw->data + max_data_bytes - rw->data_bytes, rw->data_bytes);
+ s = format (s, " %U", format_hex_bytes, rw->data, rw->data_bytes);
return s;
}
diff --git a/src/vnet/adj/rewrite.h b/src/vnet/adj/rewrite.h
index 32781134ecc..05a722fc764 100644
--- a/src/vnet/adj/rewrite.h
+++ b/src/vnet/adj/rewrite.h
@@ -131,8 +131,8 @@ vnet_rewrite_set_data_internal (vnet_rewrite_header_t * rw,
ASSERT ((data_bytes >= 0) && (data_bytes < max_size));
rw->data_bytes = data_bytes;
- clib_memcpy_fast (rw->data + max_size - data_bytes, data, data_bytes);
- clib_memset (rw->data, 0xfe, max_size - data_bytes);
+ clib_memcpy_fast (rw->data, data, data_bytes);
+ clib_memset (rw->data + data_bytes, 0xfe, max_size - data_bytes);
}
#define vnet_rewrite_set_data(rw,data,data_bytes) \
@@ -145,76 +145,28 @@ always_inline void *
vnet_rewrite_get_data_internal (vnet_rewrite_header_t * rw, int max_size)
{
ASSERT (rw->data_bytes <= max_size);
- return rw->data + max_size - rw->data_bytes;
+ return rw->data;
}
#define vnet_rewrite_get_data(rw) \
vnet_rewrite_get_data_internal (&((rw).rewrite_header), sizeof ((rw).rewrite_data))
always_inline void
-vnet_rewrite_copy_one (vnet_rewrite_data_t * p0, vnet_rewrite_data_t * rw0,
- int i)
-{
- p0[-i] = rw0[-i];
-}
-
-void vnet_rewrite_copy_slow_path (vnet_rewrite_data_t * p0,
- vnet_rewrite_data_t * rw0,
- word n_left, uword most_likely_size);
-
-/* *INDENT-OFF* */
-typedef CLIB_PACKED (struct {
- u64 a;
- u32 b;
- u16 c;
-}) eh_copy_t;
-/* *INDENT-ON* */
-
-always_inline void
_vnet_rewrite_one_header (vnet_rewrite_header_t * h0,
void *packet0, int max_size, int most_likely_size)
{
- vnet_rewrite_data_t *p0 = packet0;
- vnet_rewrite_data_t *rw0 = (vnet_rewrite_data_t *) (h0->data + max_size);
- word n_left0;
-
/* 0xfefe => poisoned adjacency => crash */
ASSERT (h0->data_bytes != 0xfefe);
-
- if (PREDICT_TRUE (h0->data_bytes == sizeof (eh_copy_t)))
+ if (PREDICT_TRUE (most_likely_size == h0->data_bytes))
{
- eh_copy_t *s, *d;
- s = (eh_copy_t *) (h0->data + max_size - sizeof (eh_copy_t));
- d = (eh_copy_t *) (((u8 *) packet0) - sizeof (eh_copy_t));
- clib_memcpy (d, s, sizeof (eh_copy_t));
- return;
+ clib_memcpy_fast ((u8 *) packet0 - most_likely_size,
+ h0->data, most_likely_size);
+ }
+ else
+ {
+ clib_memcpy_fast ((u8 *) packet0 - h0->data_bytes,
+ h0->data, h0->data_bytes);
}
- /*
- * Stop now if the data_bytes field is zero, to avoid the cache
- * miss consequences of spraying [functionally harmless] junk into
- * un-prefetched rewrite space.
- */
- if (PREDICT_FALSE (h0->data_bytes == 0))
- return;
-
-#define _(i) \
- do { \
- if (most_likely_size > ((i)-1)*sizeof (vnet_rewrite_data_t)) \
- vnet_rewrite_copy_one (p0, rw0, (i)); \
- } while (0)
-
- _(4);
- _(3);
- _(2);
- _(1);
-
-#undef _
-
- n_left0 = (int)
- (((int) h0->data_bytes - most_likely_size) + (sizeof (rw0[0]) - 1))
- / (int) sizeof (rw0[0]);
- if (PREDICT_FALSE (n_left0 > 0))
- vnet_rewrite_copy_slow_path (p0, rw0, n_left0, most_likely_size);
}
always_inline void
@@ -223,68 +175,24 @@ _vnet_rewrite_two_headers (vnet_rewrite_header_t * h0,
void *packet0,
void *packet1, int max_size, int most_likely_size)
{
- vnet_rewrite_data_t *p0 = packet0;
- vnet_rewrite_data_t *p1 = packet1;
- vnet_rewrite_data_t *rw0 = (vnet_rewrite_data_t *) (h0->data + max_size);
- vnet_rewrite_data_t *rw1 = (vnet_rewrite_data_t *) (h1->data + max_size);
- word n_left0, n_left1;
- int slow_path;
-
/* 0xfefe => poisoned adjacency => crash */
ASSERT (h0->data_bytes != 0xfefe);
ASSERT (h1->data_bytes != 0xfefe);
-
- /* Arithmetic calculation: bytes0 == bytes1 == 14 */
- slow_path = h0->data_bytes ^ h1->data_bytes;
- slow_path += h0->data_bytes ^ sizeof (eh_copy_t);
-
- if (PREDICT_TRUE (slow_path == 0))
+ if (PREDICT_TRUE
+ (most_likely_size == h0->data_bytes
+ && most_likely_size == h1->data_bytes))
{
- eh_copy_t *s0, *d0, *s1, *d1;
- s0 = (eh_copy_t *) (h0->data + max_size - sizeof (eh_copy_t));
- d0 = (eh_copy_t *) (((u8 *) packet0) - sizeof (eh_copy_t));
- clib_memcpy (d0, s0, sizeof (eh_copy_t));
- s1 = (eh_copy_t *) (h1->data + max_size - sizeof (eh_copy_t));
- d1 = (eh_copy_t *) (((u8 *) packet1) - sizeof (eh_copy_t));
- clib_memcpy (d1, s1, sizeof (eh_copy_t));
- return;
+ clib_memcpy_fast ((u8 *) packet0 - most_likely_size,
+ h0->data, most_likely_size);
+ clib_memcpy_fast ((u8 *) packet1 - most_likely_size,
+ h1->data, most_likely_size);
}
-
- /*
- * Stop now if both rewrite data_bytes fields are zero, to avoid the cache
- * miss consequences of spraying [functionally harmless] junk into
- * un-prefetched rewrite space.
- */
- if (PREDICT_FALSE (h0->data_bytes + h1->data_bytes == 0))
- return;
-
-#define _(i) \
- do { \
- if (most_likely_size > ((i)-1)*sizeof (vnet_rewrite_data_t)) \
- { \
- vnet_rewrite_copy_one (p0, rw0, (i)); \
- vnet_rewrite_copy_one (p1, rw1, (i)); \
- } \
- } while (0)
-
- _(4);
- _(3);
- _(2);
- _(1);
-
-#undef _
-
- n_left0 = (int)
- (((int) h0->data_bytes - most_likely_size) + (sizeof (rw0[0]) - 1))
- / (int) sizeof (rw0[0]);
- n_left1 = (int)
- (((int) h1->data_bytes - most_likely_size) + (sizeof (rw1[0]) - 1))
- / (int) sizeof (rw1[0]);
-
- if (PREDICT_FALSE (n_left0 > 0 || n_left1 > 0))
+ else
{
- vnet_rewrite_copy_slow_path (p0, rw0, n_left0, most_likely_size);
- vnet_rewrite_copy_slow_path (p1, rw1, n_left1, most_likely_size);
+ clib_memcpy_fast ((u8 *) packet0 - h0->data_bytes,
+ h0->data, h0->data_bytes);
+ clib_memcpy_fast ((u8 *) packet1 - h1->data_bytes,
+ h1->data, h1->data_bytes);
}
}
diff --git a/src/vnet/vxlan-gbp/encap.c b/src/vnet/vxlan-gbp/encap.c
index 2fe3fa8d437..b687cbf1cfe 100644
--- a/src/vnet/vxlan-gbp/encap.c
+++ b/src/vnet/vxlan-gbp/encap.c
@@ -97,7 +97,6 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
u8 const underlay_hdr_len = is_ip4 ?
sizeof (ip4_vxlan_gbp_header_t) : sizeof (ip6_vxlan_gbp_header_t);
- u8 const rw_hdr_offset = sizeof t0->rewrite_data - underlay_hdr_len;
u16 const l3_len = is_ip4 ? sizeof (ip4_header_t) : sizeof (ip6_header_t);
u32 const csum_flags = is_ip4 ?
VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_IS_IP4 |
@@ -176,6 +175,9 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
ASSERT (t0->rewrite_header.data_bytes == underlay_hdr_len);
ASSERT (t1->rewrite_header.data_bytes == underlay_hdr_len);
+ vnet_rewrite_two_headers (*t0, *t1, vlib_buffer_get_current (b0),
+ vlib_buffer_get_current (b1),
+ underlay_hdr_len);
vlib_buffer_advance (b0, -underlay_hdr_len);
vlib_buffer_advance (b1, -underlay_hdr_len);
@@ -188,16 +190,6 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
void *underlay0 = vlib_buffer_get_current (b0);
void *underlay1 = vlib_buffer_get_current (b1);
- /* vnet_rewrite_two_header writes only in (uword) 8 bytes chunks
- * and discards the first 4 bytes of the (36 bytes ip4 underlay) rewrite
- * use memcpy as a workaround */
- clib_memcpy_fast (underlay0,
- t0->rewrite_header.data + rw_hdr_offset,
- underlay_hdr_len);
- clib_memcpy_fast (underlay1,
- t1->rewrite_header.data + rw_hdr_offset,
- underlay_hdr_len);
-
ip4_header_t *ip4_0, *ip4_1;
qos_bits_t ip4_0_tos = 0, ip4_1_tos = 0;
ip6_header_t *ip6_0, *ip6_1;
@@ -370,17 +362,12 @@ vxlan_gbp_encap_inline (vlib_main_t * vm,
vnet_buffer (b0)->ip.adj_index[VLIB_TX] = dpoi_idx0;
ASSERT (t0->rewrite_header.data_bytes == underlay_hdr_len);
+ vnet_rewrite_one_header (*t0, vlib_buffer_get_current (b0),
+ underlay_hdr_len);
vlib_buffer_advance (b0, -underlay_hdr_len);
void *underlay0 = vlib_buffer_get_current (b0);
- /* vnet_rewrite_one_header writes only in (uword) 8 bytes chunks
- * and discards the first 4 bytes of the (36 bytes ip4 underlay) rewrite
- * use memcpy as a workaround */
- clib_memcpy_fast (underlay0,
- t0->rewrite_header.data + rw_hdr_offset,
- underlay_hdr_len);
-
u32 len0 = vlib_buffer_length_in_chain (vm, b0);
u16 payload_l0 = clib_host_to_net_u16 (len0 - l3_len);
diff --git a/src/vnet/vxlan/encap.c b/src/vnet/vxlan/encap.c
index fe2374c6c0d..b797624ae06 100644
--- a/src/vnet/vxlan/encap.c
+++ b/src/vnet/vxlan/encap.c
@@ -92,7 +92,6 @@ vxlan_encap_inline (vlib_main_t * vm,
u8 const underlay_hdr_len = is_ip4 ?
sizeof(ip4_vxlan_header_t) : sizeof(ip6_vxlan_header_t);
- u8 const rw_hdr_offset = sizeof t0->rewrite_data - underlay_hdr_len;
u16 const l3_len = is_ip4 ? sizeof(ip4_header_t) : sizeof(ip6_header_t);
u32 const csum_flags = is_ip4 ?
VNET_BUFFER_F_OFFLOAD_IP_CKSUM | VNET_BUFFER_F_IS_IP4 |
@@ -173,6 +172,7 @@ vxlan_encap_inline (vlib_main_t * vm,
ASSERT(t0->rewrite_header.data_bytes == underlay_hdr_len);
ASSERT(t1->rewrite_header.data_bytes == underlay_hdr_len);
+ vnet_rewrite_two_headers(*t0, *t1, vlib_buffer_get_current(b0), vlib_buffer_get_current(b1), underlay_hdr_len);
vlib_buffer_advance (b0, -underlay_hdr_len);
vlib_buffer_advance (b1, -underlay_hdr_len);
@@ -185,12 +185,6 @@ vxlan_encap_inline (vlib_main_t * vm,
void * underlay0 = vlib_buffer_get_current(b0);
void * underlay1 = vlib_buffer_get_current(b1);
- /* vnet_rewrite_two_header writes only in (uword) 8 bytes chunks
- * and discards the first 4 bytes of the (36 bytes ip4 underlay) rewrite
- * use memcpy as a workaround */
- clib_memcpy_fast(underlay0, t0->rewrite_header.data + rw_hdr_offset, underlay_hdr_len);
- clib_memcpy_fast(underlay1, t1->rewrite_header.data + rw_hdr_offset, underlay_hdr_len);
-
ip4_header_t * ip4_0, * ip4_1;
qos_bits_t ip4_0_tos = 0, ip4_1_tos = 0;
ip6_header_t * ip6_0, * ip6_1;
@@ -354,15 +348,11 @@ vxlan_encap_inline (vlib_main_t * vm,
vnet_buffer(b0)->ip.adj_index[VLIB_TX] = dpoi_idx0;
ASSERT(t0->rewrite_header.data_bytes == underlay_hdr_len);
+ vnet_rewrite_one_header(*t0, vlib_buffer_get_current(b0), underlay_hdr_len);
vlib_buffer_advance (b0, -underlay_hdr_len);
void * underlay0 = vlib_buffer_get_current(b0);
- /* vnet_rewrite_one_header writes only in (uword) 8 bytes chunks
- * and discards the first 4 bytes of the (36 bytes ip4 underlay) rewrite
- * use memcpy as a workaround */
- clib_memcpy_fast(underlay0, t0->rewrite_header.data + rw_hdr_offset, underlay_hdr_len);
-
u32 len0 = vlib_buffer_length_in_chain (vm, b0);
u16 payload_l0 = clib_host_to_net_u16 (len0 - l3_len);