summaryrefslogtreecommitdiffstats
path: root/src/vppinfra
diff options
context:
space:
mode:
Diffstat (limited to 'src/vppinfra')
-rw-r--r--src/vppinfra/bihash_template.c23
-rw-r--r--src/vppinfra/fifo.c6
-rw-r--r--src/vppinfra/fifo.h4
-rw-r--r--src/vppinfra/hash.c20
-rw-r--r--src/vppinfra/hash.h2
-rw-r--r--src/vppinfra/heap.h2
-rw-r--r--src/vppinfra/mem.h2
-rw-r--r--src/vppinfra/memcpy_avx2.h2
-rw-r--r--src/vppinfra/memcpy_avx512.h2
-rw-r--r--src/vppinfra/memcpy_sse3.h2
-rw-r--r--src/vppinfra/mhash.c4
-rw-r--r--src/vppinfra/serialize.c12
-rw-r--r--src/vppinfra/socket.c3
-rw-r--r--src/vppinfra/string.h24
-rw-r--r--src/vppinfra/vec.c2
-rw-r--r--src/vppinfra/vec.h18
16 files changed, 70 insertions, 58 deletions
diff --git a/src/vppinfra/bihash_template.c b/src/vppinfra/bihash_template.c
index e218f4b0dc6..698053867a4 100644
--- a/src/vppinfra/bihash_template.c
+++ b/src/vppinfra/bihash_template.c
@@ -295,7 +295,7 @@ BV (make_working_copy) (BVT (clib_bihash) * h, BVT (clib_bihash_bucket) * b)
v = BV (clib_bihash_get_value) (h, b->offset);
- _clib_memcpy (working_copy, v, sizeof (*v) * (1 << b->log2_pages));
+ clib_memcpy_fast (working_copy, v, sizeof (*v) * (1 << b->log2_pages));
working_bucket.as_u64 = b->as_u64;
working_bucket.offset = BV (clib_bihash_get_offset) (h, working_copy);
CLIB_MEMORY_BARRIER ();
@@ -338,8 +338,8 @@ BV (split_and_rehash)
/* Empty slot */
if (BV (clib_bihash_is_free) (&(new_v->kvp[j])))
{
- _clib_memcpy (&(new_v->kvp[j]), &(old_values->kvp[i]),
- sizeof (new_v->kvp[j]));
+ clib_memcpy_fast (&(new_v->kvp[j]), &(old_values->kvp[i]),
+ sizeof (new_v->kvp[j]));
goto doublebreak;
}
}
@@ -383,8 +383,8 @@ BV (split_and_rehash_linear)
if (BV (clib_bihash_is_free) (&(new_values->kvp[j])))
{
/* Copy the old value and move along */
- _clib_memcpy (&(new_values->kvp[j]), &(old_values->kvp[i]),
- sizeof (new_values->kvp[j]));
+ clib_memcpy_fast (&(new_values->kvp[j]), &(old_values->kvp[i]),
+ sizeof (new_values->kvp[j]));
j++;
goto doublebreak;
}
@@ -472,7 +472,7 @@ static inline int BV (clib_bihash_add_del_inline)
if (!memcmp (&(v->kvp[i]), &add_v->key, sizeof (add_v->key)))
{
CLIB_MEMORY_BARRIER (); /* Add a delay */
- _clib_memcpy (&(v->kvp[i]), add_v, sizeof (*add_v));
+ clib_memcpy_fast (&(v->kvp[i]), add_v, sizeof (*add_v));
BV (clib_bihash_unlock_bucket) (b);
return (0);
}
@@ -488,10 +488,11 @@ static inline int BV (clib_bihash_add_del_inline)
* Copy the value first, so that if a reader manages
* to match the new key, the value will be right...
*/
- _clib_memcpy (&(v->kvp[i].value),
- &add_v->value, sizeof (add_v->value));
+ clib_memcpy_fast (&(v->kvp[i].value),
+ &add_v->value, sizeof (add_v->value));
CLIB_MEMORY_BARRIER (); /* Make sure the value has settled */
- _clib_memcpy (&(v->kvp[i]), &add_v->key, sizeof (add_v->key));
+ clib_memcpy_fast (&(v->kvp[i]), &add_v->key,
+ sizeof (add_v->key));
b->refcnt++;
ASSERT (b->refcnt > 0);
BV (clib_bihash_unlock_bucket) (b);
@@ -506,7 +507,7 @@ static inline int BV (clib_bihash_add_del_inline)
if (is_stale_cb (&(v->kvp[i]), arg))
{
CLIB_MEMORY_BARRIER ();
- _clib_memcpy (&(v->kvp[i]), add_v, sizeof (*add_v));
+ clib_memcpy_fast (&(v->kvp[i]), add_v, sizeof (*add_v));
BV (clib_bihash_unlock_bucket) (b);
return (0);
}
@@ -602,7 +603,7 @@ static inline int BV (clib_bihash_add_del_inline)
{
if (BV (clib_bihash_is_free) (&(new_v->kvp[i])))
{
- _clib_memcpy (&(new_v->kvp[i]), add_v, sizeof (*add_v));
+ clib_memcpy_fast (&(new_v->kvp[i]), add_v, sizeof (*add_v));
goto expand_ok;
}
}
diff --git a/src/vppinfra/fifo.c b/src/vppinfra/fifo.c
index e3b7b415cb3..e97b2c3ee00 100644
--- a/src/vppinfra/fifo.c
+++ b/src/vppinfra/fifo.c
@@ -112,11 +112,11 @@ _clib_fifo_resize (void *v_old, uword n_new_elts, uword elt_bytes)
if (head + n_copy_bytes >= end)
{
uword n = end - head;
- clib_memcpy (v_new, head, n);
- clib_memcpy (v_new + n, v_old, n_copy_bytes - n);
+ clib_memcpy_fast (v_new, head, n);
+ clib_memcpy_fast (v_new + n, v_old, n_copy_bytes - n);
}
else
- clib_memcpy (v_new, head, n_copy_bytes);
+ clib_memcpy_fast (v_new, head, n_copy_bytes);
}
/* Zero empty space. */
diff --git a/src/vppinfra/fifo.h b/src/vppinfra/fifo.h
index b0b35e25af7..5dc1b4512cf 100644
--- a/src/vppinfra/fifo.h
+++ b/src/vppinfra/fifo.h
@@ -215,9 +215,9 @@ do { \
_n1 = _i + _n0 - _l; \
_n1 = _n1 < 0 ? 0 : _n1; \
_n0 -= _n1; \
- clib_memcpy ((f) + _i, (e), _n0 * sizeof ((f)[0])); \
+ clib_memcpy_fast ((f) + _i, (e), _n0 * sizeof ((f)[0])); \
if (_n1) \
- clib_memcpy ((f) + 0, (e) + _n0, _n1 * sizeof ((f)[0])); \
+ clib_memcpy_fast ((f) + 0, (e) + _n0, _n1 * sizeof ((f)[0])); \
} while (0)
/* Subtract element from fifo. */
diff --git a/src/vppinfra/hash.c b/src/vppinfra/hash.c
index 2ff8ebfd17b..eae79d48592 100644
--- a/src/vppinfra/hash.c
+++ b/src/vppinfra/hash.c
@@ -376,7 +376,7 @@ set_indirect_is_user (void *v, uword i, hash_pair_union_t * p, uword key)
log2_bytes = 1 + hash_pair_log2_bytes (h);
q = clib_mem_alloc (1ULL << log2_bytes);
}
- clib_memcpy (q, &p->direct, hash_pair_bytes (h));
+ clib_memcpy_fast (q, &p->direct, hash_pair_bytes (h));
pi->pairs = q;
if (h->log2_pair_size > 0)
@@ -457,8 +457,8 @@ unset_indirect (void *v, uword i, hash_pair_t * q)
if (len == 2)
{
- clib_memcpy (p, q == r ? hash_forward1 (h, r) : r,
- hash_pair_bytes (h));
+ clib_memcpy_fast (p, q == r ? hash_forward1 (h, r) : r,
+ hash_pair_bytes (h));
set_is_user (v, i, 1);
}
else
@@ -473,7 +473,7 @@ unset_indirect (void *v, uword i, hash_pair_t * q)
{
/* If deleting a pair we need to keep non-null pairs together. */
if (q < e)
- clib_memcpy (q, e, hash_pair_bytes (h));
+ clib_memcpy_fast (q, e, hash_pair_bytes (h));
else
zero_pair (h, q);
if (is_vec)
@@ -514,8 +514,8 @@ lookup (void *v, uword key, enum lookup_opcode op,
{
set_is_user (v, i, 0);
if (old_value)
- clib_memcpy (old_value, p->direct.value,
- hash_value_bytes (h));
+ clib_memcpy_fast (old_value, p->direct.value,
+ hash_value_bytes (h));
zero_pair (h, &p->direct);
}
}
@@ -548,8 +548,8 @@ lookup (void *v, uword key, enum lookup_opcode op,
if (found_key && op == UNSET)
{
if (old_value)
- clib_memcpy (old_value, &p->direct.value,
- hash_value_bytes (h));
+ clib_memcpy_fast (old_value, &p->direct.value,
+ hash_value_bytes (h));
unset_indirect (v, i, &p->direct);
@@ -564,8 +564,8 @@ lookup (void *v, uword key, enum lookup_opcode op,
{
/* Save away old value for caller. */
if (old_value && found_key)
- clib_memcpy (old_value, &p->direct.value, hash_value_bytes (h));
- clib_memcpy (&p->direct.value, new_value, hash_value_bytes (h));
+ clib_memcpy_fast (old_value, &p->direct.value, hash_value_bytes (h));
+ clib_memcpy_fast (&p->direct.value, new_value, hash_value_bytes (h));
}
if (op == SET)
diff --git a/src/vppinfra/hash.h b/src/vppinfra/hash.h
index 892b2ea916c..59dbf8cde76 100644
--- a/src/vppinfra/hash.h
+++ b/src/vppinfra/hash.h
@@ -280,7 +280,7 @@ hash_set_mem_alloc (uword ** h, void *key, uword v)
{
size_t ksz = hash_header (*h)->user;
void *copy = clib_mem_alloc (ksz);
- clib_memcpy (copy, key, ksz);
+ clib_memcpy_fast (copy, key, ksz);
hash_set_mem (*h, copy, v);
}
diff --git a/src/vppinfra/heap.h b/src/vppinfra/heap.h
index b6e9f022852..22fc335c072 100644
--- a/src/vppinfra/heap.h
+++ b/src/vppinfra/heap.h
@@ -204,7 +204,7 @@ _heap_dup (void *v_old, uword v_bytes)
HEAP_DATA_ALIGN);
h_new = heap_header (v_new);
heap_dup_header (h_old, h_new);
- clib_memcpy (v_new, v_old, v_bytes);
+ clib_memcpy_fast (v_new, v_old, v_bytes);
return v_new;
}
diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h
index a2c5474a46e..65b25f004a7 100644
--- a/src/vppinfra/mem.h
+++ b/src/vppinfra/mem.h
@@ -232,7 +232,7 @@ clib_mem_realloc (void *p, uword new_size, uword old_size)
copy_size = old_size;
else
copy_size = new_size;
- clib_memcpy (q, p, copy_size);
+ clib_memcpy_fast (q, p, copy_size);
clib_mem_free (p);
}
return q;
diff --git a/src/vppinfra/memcpy_avx2.h b/src/vppinfra/memcpy_avx2.h
index 64dff4e5ddb..caff4f6afde 100644
--- a/src/vppinfra/memcpy_avx2.h
+++ b/src/vppinfra/memcpy_avx2.h
@@ -109,7 +109,7 @@ clib_mov128blocks (u8 * dst, const u8 * src, size_t n)
}
static inline void *
-_clib_memcpy (void *dst, const void *src, size_t n)
+clib_memcpy_fast (void *dst, const void *src, size_t n)
{
uword dstu = (uword) dst;
uword srcu = (uword) src;
diff --git a/src/vppinfra/memcpy_avx512.h b/src/vppinfra/memcpy_avx512.h
index e5245268770..9032b5436a3 100644
--- a/src/vppinfra/memcpy_avx512.h
+++ b/src/vppinfra/memcpy_avx512.h
@@ -139,7 +139,7 @@ clib_mov512blocks (u8 * dst, const u8 * src, size_t n)
}
static inline void *
-_clib_memcpy (void *dst, const void *src, size_t n)
+clib_memcpy_fast (void *dst, const void *src, size_t n)
{
uword dstu = (uword) dst;
uword srcu = (uword) src;
diff --git a/src/vppinfra/memcpy_sse3.h b/src/vppinfra/memcpy_sse3.h
index 2dd9399d428..5346e913817 100644
--- a/src/vppinfra/memcpy_sse3.h
+++ b/src/vppinfra/memcpy_sse3.h
@@ -183,7 +183,7 @@ clib_mov256 (u8 * dst, const u8 * src)
})
static inline void *
-_clib_memcpy (void *dst, const void *src, size_t n)
+clib_memcpy_fast (void *dst, const void *src, size_t n)
{
__m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8;
uword dstu = (uword) dst;
diff --git a/src/vppinfra/mhash.c b/src/vppinfra/mhash.c
index e09d7193f68..d4d5457ea7a 100644
--- a/src/vppinfra/mhash.c
+++ b/src/vppinfra/mhash.c
@@ -289,7 +289,7 @@ mhash_set_mem (mhash_t * h, void *key, uword * new_value, uword * old_value)
sk = (void *) (h->key_vector_or_heap + i);
sk->heap_handle = handle;
sk->vec.len = n_key_bytes;
- clib_memcpy (sk->vec.vector_data, key, n_key_bytes);
+ clib_memcpy_fast (sk->vec.vector_data, key, n_key_bytes);
/* Advance key past vector header. */
i += sizeof (sk[0]);
@@ -311,7 +311,7 @@ mhash_set_mem (mhash_t * h, void *key, uword * new_value, uword * old_value)
}
n_key_bytes = h->n_key_bytes;
- clib_memcpy (k, key, n_key_bytes);
+ clib_memcpy_fast (k, key, n_key_bytes);
}
ikey = i;
diff --git a/src/vppinfra/serialize.c b/src/vppinfra/serialize.c
index ef9cf03f362..93e44f94e07 100644
--- a/src/vppinfra/serialize.c
+++ b/src/vppinfra/serialize.c
@@ -170,7 +170,7 @@ serialize_cstring (serialize_main_t * m, char *s)
if (len > 0)
{
p = serialize_get (m, len);
- clib_memcpy (p, s, len);
+ clib_memcpy_fast (p, s, len);
}
}
@@ -191,7 +191,7 @@ unserialize_cstring (serialize_main_t * m, char **s)
{
r = vec_new (char, len + 1);
p = unserialize_get (m, len);
- clib_memcpy (r, p, len);
+ clib_memcpy_fast (r, p, len);
/* Null terminate. */
r[len] = 0;
@@ -206,7 +206,7 @@ serialize_vec_8 (serialize_main_t * m, va_list * va)
u8 *s = va_arg (*va, u8 *);
u32 n = va_arg (*va, u32);
u8 *p = serialize_get (m, n * sizeof (u8));
- clib_memcpy (p, s, n * sizeof (u8));
+ clib_memcpy_fast (p, s, n * sizeof (u8));
}
void
@@ -215,7 +215,7 @@ unserialize_vec_8 (serialize_main_t * m, va_list * va)
u8 *s = va_arg (*va, u8 *);
u32 n = va_arg (*va, u32);
u8 *p = unserialize_get (m, n);
- clib_memcpy (s, p, n);
+ clib_memcpy_fast (s, p, n);
}
#define _(n_bits) \
@@ -627,7 +627,7 @@ serialize_magic (serialize_main_t * m, void *magic, u32 magic_bytes)
void *p;
serialize_integer (m, magic_bytes, sizeof (magic_bytes));
p = serialize_get (m, magic_bytes);
- clib_memcpy (p, magic, magic_bytes);
+ clib_memcpy_fast (p, magic, magic_bytes);
}
void
@@ -710,7 +710,7 @@ serialize_write_not_inline (serialize_main_header_t * m,
if (n_left_o > 0 && n_left_b > 0)
{
uword n = clib_min (n_left_b, n_left_o);
- clib_memcpy (s->buffer + cur_bi, s->overflow_buffer, n);
+ clib_memcpy_fast (s->buffer + cur_bi, s->overflow_buffer, n);
cur_bi += n;
n_left_b -= n;
n_left_o -= n;
diff --git a/src/vppinfra/socket.c b/src/vppinfra/socket.c
index 109cbad7bb8..bcee3dea9de 100644
--- a/src/vppinfra/socket.c
+++ b/src/vppinfra/socket.c
@@ -356,7 +356,8 @@ default_socket_recvmsg (clib_socket_t * s, void *msg, int msglen,
#endif
if (cmsg->cmsg_type == SCM_RIGHTS)
{
- clib_memcpy (fds, CMSG_DATA (cmsg), num_fds * sizeof (int));
+ clib_memcpy_fast (fds, CMSG_DATA (cmsg),
+ num_fds * sizeof (int));
}
}
cmsg = CMSG_NXTHDR (&mh, cmsg);
diff --git a/src/vppinfra/string.h b/src/vppinfra/string.h
index 5c1d8267742..b00c0cfbcc2 100644
--- a/src/vppinfra/string.h
+++ b/src/vppinfra/string.h
@@ -78,10 +78,10 @@ void clib_memswap (void *_a, void *_b, uword bytes);
#elif __SSSE3__
#include <vppinfra/memcpy_sse3.h>
#else
-#define _clib_memcpy(a,b,c) memcpy(a,b,c)
+#define clib_memcpy_fast(a,b,c) memcpy(a,b,c)
#endif
#else /* __COVERITY__ */
-#define _clib_memcpy(a,b,c) memcpy(a,b,c)
+#define clib_memcpy_fast(a,b,c) memcpy(a,b,c)
#endif
/* c-11 string manipulation variants */
@@ -108,6 +108,16 @@ memcpy_s_inline (void *__restrict__ dest, rsize_t dmax,
u8 bad;
/*
+ * Optimize constant-number-of-bytes calls without asking
+ * "too many questions for someone from New Jersey"
+ */
+ if (__builtin_constant_p (n))
+ {
+ clib_memcpy_fast (dest, src, n);
+ return EOK;
+ }
+
+ /*
* call bogus if: src or dst NULL, trying to copy
* more data than we have space in dst, or src == dst.
* n == 0 isn't really "bad", so check first in the
@@ -140,7 +150,7 @@ memcpy_s_inline (void *__restrict__ dest, rsize_t dmax,
return EINVAL;
}
- _clib_memcpy (dest, src, n);
+ clib_memcpy_fast (dest, src, n);
return EOK;
}
@@ -247,10 +257,10 @@ clib_memcpy64_x4 (void *d0, void *d1, void *d2, void *d3, void *s)
_mm_storeu_si128 ((__m128i *) (d3 + 3 * 16), r3);
#else
- clib_memcpy (d0, s, 64);
- clib_memcpy (d1, s, 64);
- clib_memcpy (d2, s, 64);
- clib_memcpy (d3, s, 64);
+ clib_memcpy_fast (d0, s, 64);
+ clib_memcpy_fast (d1, s, 64);
+ clib_memcpy_fast (d2, s, 64);
+ clib_memcpy_fast (d3, s, 64);
#endif
}
diff --git a/src/vppinfra/vec.c b/src/vppinfra/vec.c
index 1e0164a9e34..232eb4d4c24 100644
--- a/src/vppinfra/vec.c
+++ b/src/vppinfra/vec.c
@@ -92,7 +92,7 @@ vec_resize_allocate_memory (void *v,
("vec_resize fails, length increment %d, data bytes %d, alignment %d",
length_increment, data_bytes, data_align);
- clib_memcpy (new, old, old_alloc_bytes);
+ clib_memcpy_fast (new, old, old_alloc_bytes);
clib_mem_free (old);
/* Allocator may give a bit of extra room. */
diff --git a/src/vppinfra/vec.h b/src/vppinfra/vec.h
index dc7b908a3b5..470a4f190c2 100644
--- a/src/vppinfra/vec.h
+++ b/src/vppinfra/vec.h
@@ -360,7 +360,7 @@ do { \
if (_v(l) > 0) \
{ \
vec_resize_ha (_v(v), _v(l), (H), (A)); \
- clib_memcpy (_v(v), (V), _v(l) * sizeof ((V)[0]));\
+ clib_memcpy_fast (_v(v), (V), _v(l) * sizeof ((V)[0]));\
} \
_v(v); \
})
@@ -387,7 +387,7 @@ do { \
@param DST destination
@param SRC source
*/
-#define vec_copy(DST,SRC) clib_memcpy (DST, SRC, vec_len (DST) * \
+#define vec_copy(DST,SRC) clib_memcpy_fast (DST, SRC, vec_len (DST) * \
sizeof ((DST)[0]))
/** \brief Clone a vector. Make a new vector with the
@@ -587,7 +587,7 @@ do { \
word _v(n) = (N); \
word _v(l) = vec_len (V); \
V = _vec_resize ((V), _v(n), (_v(l) + _v(n)) * sizeof ((V)[0]), (H), (A)); \
- clib_memcpy ((V) + _v(l), (E), _v(n) * sizeof ((V)[0])); \
+ clib_memcpy_fast ((V) + _v(l), (E), _v(n) * sizeof ((V)[0])); \
} while (0)
/** \brief Add N elements to end of vector V (no header, unspecified alignment)
@@ -749,7 +749,7 @@ do { \
memmove ((V) + _v(m) + _v(n), \
(V) + _v(m), \
(_v(l) - _v(m)) * sizeof ((V)[0])); \
- clib_memcpy ((V) + _v(m), (E), \
+ clib_memcpy_fast ((V) + _v(m), (E), \
_v(n) * sizeof ((V)[0])); \
} while (0)
@@ -824,7 +824,7 @@ do { \
\
v1 = _vec_resize ((v1), _v(l2), \
(_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, 0); \
- clib_memcpy ((v1) + _v(l1), (v2), _v(l2) * sizeof ((v2)[0])); \
+ clib_memcpy_fast ((v1) + _v(l1), (v2), _v(l2) * sizeof ((v2)[0])); \
} while (0)
/** \brief Append v2 after v1. Result in v1. Specified alignment.
@@ -840,7 +840,7 @@ do { \
\
v1 = _vec_resize ((v1), _v(l2), \
(_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, align); \
- clib_memcpy ((v1) + _v(l1), (v2), _v(l2) * sizeof ((v2)[0])); \
+ clib_memcpy_fast ((v1) + _v(l1), (v2), _v(l2) * sizeof ((v2)[0])); \
} while (0)
/** \brief Prepend v2 before v1. Result in v1.
@@ -856,7 +856,7 @@ do { \
v1 = _vec_resize ((v1), _v(l2), \
(_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, 0); \
memmove ((v1) + _v(l2), (v1), _v(l1) * sizeof ((v1)[0])); \
- clib_memcpy ((v1), (v2), _v(l2) * sizeof ((v2)[0])); \
+ clib_memcpy_fast ((v1), (v2), _v(l2) * sizeof ((v2)[0])); \
} while (0)
/** \brief Prepend v2 before v1. Result in v1. Specified alignment
@@ -873,7 +873,7 @@ do { \
v1 = _vec_resize ((v1), _v(l2), \
(_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, align); \
memmove ((v1) + _v(l2), (v1), _v(l1) * sizeof ((v1)[0])); \
- clib_memcpy ((v1), (v2), _v(l2) * sizeof ((v2)[0])); \
+ clib_memcpy_fast ((v1), (v2), _v(l2) * sizeof ((v2)[0])); \
} while (0)
@@ -995,7 +995,7 @@ do { \
vec_reset_length (V); \
vec_validate ((V), (L)); \
if ((S) && (L)) \
- clib_memcpy ((V), (S), (L)); \
+ clib_memcpy_fast ((V), (S), (L)); \
(V)[(L)] = 0; \
} while (0)