aboutsummaryrefslogtreecommitdiffstats
path: root/src/vlib
diff options
context:
space:
mode:
Diffstat (limited to 'src/vlib')
-rw-r--r--src/vlib/buffer.c19
-rw-r--r--src/vlib/buffer_funcs.h44
-rw-r--r--src/vlib/buffer_node.h8
-rw-r--r--src/vlib/linux/pci.c2
-rw-r--r--src/vlib/main.c8
-rw-r--r--src/vlib/node_funcs.h3
-rw-r--r--src/vlib/threads.c38
-rw-r--r--src/vlib/trace.c12
8 files changed, 68 insertions, 66 deletions
diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index cc852358541..133d640d693 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -672,8 +672,8 @@ vlib_packet_template_buffer_init (vlib_main_t * vm,
{
vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]);
ASSERT (b->current_length == vec_len (t->packet_data));
- clib_memcpy (vlib_buffer_get_current (b), t->packet_data,
- b->current_length);
+ clib_memcpy_fast (vlib_buffer_get_current (b), t->packet_data,
+ b->current_length);
}
}
@@ -738,8 +738,8 @@ vlib_packet_template_get_packet (vlib_main_t * vm,
*bi_result = bi;
b = vlib_get_buffer (vm, bi);
- clib_memcpy (vlib_buffer_get_current (b),
- t->packet_data, vec_len (t->packet_data));
+ clib_memcpy_fast (vlib_buffer_get_current (b),
+ t->packet_data, vec_len (t->packet_data));
b->current_length = vec_len (t->packet_data);
return b->data;
@@ -796,7 +796,8 @@ vlib_buffer_add_data (vlib_main_t * vm,
n_left_this_buffer =
n_buffer_bytes - (b->current_data + b->current_length);
n = clib_min (n_left_this_buffer, n_left);
- clib_memcpy (vlib_buffer_get_current (b) + b->current_length, d, n);
+ clib_memcpy_fast (vlib_buffer_get_current (b) + b->current_length, d,
+ n);
b->current_length += n;
n_left -= n;
if (n_left == 0)
@@ -847,8 +848,8 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
}
u16 len = (data_len > max) ? max : data_len;
- clib_memcpy (vlib_buffer_get_current (l) + l->current_length,
- data + copied, len);
+ clib_memcpy_fast (vlib_buffer_get_current (l) + l->current_length,
+ data + copied, len);
vlib_buffer_chain_increase_length (first, l, len);
data_len -= len;
copied += len;
@@ -974,8 +975,8 @@ vlib_buffer_main_init (struct vlib_main_t * vm)
{
/* external plugin has registered own buffer callbacks
so we just copy them and quit */
- clib_memcpy (&bm->cb, vlib_buffer_callbacks,
- sizeof (vlib_buffer_callbacks_t));
+ clib_memcpy_fast (&bm->cb, vlib_buffer_callbacks,
+ sizeof (vlib_buffer_callbacks_t));
bm->callbacks_registered = 1;
return 0;
}
diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index ce62c8b83fa..d15ef5702f8 100644
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -304,7 +304,7 @@ vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
{
b = vlib_get_buffer (vm, buffer_index);
l = b->current_length;
- clib_memcpy (contents + content_len, b->data + b->current_data, l);
+ clib_memcpy_fast (contents + content_len, b->data + b->current_data, l);
content_len += l;
if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
break;
@@ -470,7 +470,7 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
/* following code is intentionaly duplicated to allow compiler
to optimize fast path when n_buffers is constant value */
src = fl->buffers + len - n_buffers;
- clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+ clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32));
_vec_len (fl->buffers) -= n_buffers;
/* Verify that buffers are known free. */
@@ -481,7 +481,7 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
}
src = fl->buffers + len - n_buffers;
- clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+ clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32));
_vec_len (fl->buffers) -= n_buffers;
/* Verify that buffers are known free. */
@@ -758,10 +758,10 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b)
d->flags = s->flags & flag_mask;
d->total_length_not_including_first_buffer =
s->total_length_not_including_first_buffer;
- clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
- clib_memcpy (d->opaque2, s->opaque2, sizeof (s->opaque2));
- clib_memcpy (vlib_buffer_get_current (d),
- vlib_buffer_get_current (s), s->current_length);
+ clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
+ clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (s), s->current_length);
/* next segments */
for (i = 1; i < n_buffers; i++)
@@ -773,8 +773,8 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b)
d = vlib_get_buffer (vm, new_buffers[i]);
d->current_data = s->current_data;
d->current_length = s->current_length;
- clib_memcpy (vlib_buffer_get_current (d),
- vlib_buffer_get_current (s), s->current_length);
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (s), s->current_length);
d->flags = s->flags & flag_mask;
}
@@ -846,10 +846,10 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
}
d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
- clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
- clib_memcpy (d->opaque2, s->opaque2, sizeof (s->opaque2));
- clib_memcpy (vlib_buffer_get_current (d), vlib_buffer_get_current (s),
- head_end_offset);
+ clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
+ clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
+ clib_memcpy_fast (vlib_buffer_get_current (d),
+ vlib_buffer_get_current (s), head_end_offset);
d->next_buffer = src_buffer;
}
vlib_buffer_advance (s, head_end_offset);
@@ -982,8 +982,8 @@ vlib_buffer_chain_append_data (vlib_main_t * vm,
u16 len = clib_min (data_len,
n_buffer_bytes - last->current_length -
last->current_data);
- clib_memcpy (vlib_buffer_get_current (last) + last->current_length, data,
- len);
+ clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length,
+ data, len);
vlib_buffer_chain_increase_length (first, last, len);
return len;
}
@@ -1088,10 +1088,10 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst,
/* Make sure buffer template is sane. */
ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
- clib_memcpy (STRUCT_MARK_PTR (dst, template_start),
- STRUCT_MARK_PTR (src, template_start),
- STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
- STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+ clib_memcpy_fast (STRUCT_MARK_PTR (dst, template_start),
+ STRUCT_MARK_PTR (src, template_start),
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
+ STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
/* Not in the first 16 octets. */
dst->n_add_refs = src->n_add_refs;
@@ -1240,9 +1240,9 @@ vlib_buffer_chain_compress (vlib_main_t * vm,
vlib_buffer_t *second = vlib_get_buffer (vm, first->next_buffer);
u32 need = want_first_size - first->current_length;
u32 amount_to_copy = clib_min (need, second->current_length);
- clib_memcpy (((u8 *) vlib_buffer_get_current (first)) +
- first->current_length,
- vlib_buffer_get_current (second), amount_to_copy);
+ clib_memcpy_fast (((u8 *) vlib_buffer_get_current (first)) +
+ first->current_length,
+ vlib_buffer_get_current (second), amount_to_copy);
first->current_length += amount_to_copy;
vlib_buffer_advance (second, amount_to_copy);
if (first->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)
diff --git a/src/vlib/buffer_node.h b/src/vlib/buffer_node.h
index 35e15a5d919..bff7a78cb94 100644
--- a/src/vlib/buffer_node.h
+++ b/src/vlib/buffer_node.h
@@ -383,7 +383,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
#ifdef CLIB_HAVE_VEC512
if (n_enqueued >= 32)
{
- clib_memcpy (to_next, buffers, 32 * sizeof (u32));
+ clib_memcpy_fast (to_next, buffers, 32 * sizeof (u32));
nexts += 32;
to_next += 32;
buffers += 32;
@@ -397,7 +397,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
#ifdef CLIB_HAVE_VEC256
if (n_enqueued >= 16)
{
- clib_memcpy (to_next, buffers, 16 * sizeof (u32));
+ clib_memcpy_fast (to_next, buffers, 16 * sizeof (u32));
nexts += 16;
to_next += 16;
buffers += 16;
@@ -411,7 +411,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
#ifdef CLIB_HAVE_VEC128
if (n_enqueued >= 8)
{
- clib_memcpy (to_next, buffers, 8 * sizeof (u32));
+ clib_memcpy_fast (to_next, buffers, 8 * sizeof (u32));
nexts += 8;
to_next += 8;
buffers += 8;
@@ -424,7 +424,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
if (n_enqueued >= 4)
{
- clib_memcpy (to_next, buffers, 4 * sizeof (u32));
+ clib_memcpy_fast (to_next, buffers, 4 * sizeof (u32));
nexts += 4;
to_next += 4;
buffers += 4;
diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c
index c6fa8a7da34..ed43580ffc0 100644
--- a/src/vlib/linux/pci.c
+++ b/src/vlib/linux/pci.c
@@ -584,7 +584,7 @@ vfio_set_irqs (vlib_main_t * vm, linux_pci_device_t * p, u32 index, u32 start,
if (efds)
{
flags |= VFIO_IRQ_SET_DATA_EVENTFD;
- clib_memcpy (&irq_set->data, efds, data_len);
+ clib_memcpy_fast (&irq_set->data, efds, data_len);
}
else
flags |= VFIO_IRQ_SET_DATA_NONE;
diff --git a/src/vlib/main.c b/src/vlib/main.c
index b46caf1bcb4..964feb06c65 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -1650,12 +1650,12 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
te->n_data_elts,
te->n_data_elt_bytes);
if (te->n_data_bytes < sizeof (te->inline_event_data))
- clib_memcpy (data, te->inline_event_data,
- te->n_data_bytes);
+ clib_memcpy_fast (data, te->inline_event_data,
+ te->n_data_bytes);
else
{
- clib_memcpy (data, te->event_data_as_vector,
- te->n_data_bytes);
+ clib_memcpy_fast (data, te->event_data_as_vector,
+ te->n_data_bytes);
vec_free (te->event_data_as_vector);
}
pool_put (nm->signal_timed_event_data_pool, te);
diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h
index 13614aa14cd..777746a0f78 100644
--- a/src/vlib/node_funcs.h
+++ b/src/vlib/node_funcs.h
@@ -135,7 +135,8 @@ vlib_node_set_runtime_data (vlib_main_t * vm, u32 node_index,
STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data));
if (vec_len (n->runtime_data) > 0)
- clib_memcpy (r->runtime_data, n->runtime_data, vec_len (n->runtime_data));
+ clib_memcpy_fast (r->runtime_data, n->runtime_data,
+ vec_len (n->runtime_data));
}
/** \brief Set node dispatch state.
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 7ecfa309378..fc2586c8967 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -1041,8 +1041,8 @@ vlib_worker_thread_node_refork (void)
u64 *old_counters = vm_clone->error_main.counters;
u64 *old_counters_all_clear = vm_clone->error_main.counters_last_clear;
- clib_memcpy (&vm_clone->error_main, &vm->error_main,
- sizeof (vm->error_main));
+ clib_memcpy_fast (&vm_clone->error_main, &vm->error_main,
+ sizeof (vm->error_main));
j = vec_len (vm->error_main.counters) - 1;
vec_validate_aligned (old_counters, j, CLIB_CACHE_LINE_BYTES);
vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
@@ -1083,7 +1083,7 @@ vlib_worker_thread_node_refork (void)
new_n = nm->nodes[j];
old_n_clone = old_nodes_clone[j];
- clib_memcpy (new_n_clone, new_n, sizeof (*new_n));
+ clib_memcpy_fast (new_n_clone, new_n, sizeof (*new_n));
/* none of the copied nodes have enqueue rights given out */
new_n_clone->owner_node_index = VLIB_INVALID_NODE_INDEX;
@@ -1098,12 +1098,12 @@ vlib_worker_thread_node_refork (void)
else
{
/* Copy stats if the old data is valid */
- clib_memcpy (&new_n_clone->stats_total,
- &old_n_clone->stats_total,
- sizeof (new_n_clone->stats_total));
- clib_memcpy (&new_n_clone->stats_last_clear,
- &old_n_clone->stats_last_clear,
- sizeof (new_n_clone->stats_last_clear));
+ clib_memcpy_fast (&new_n_clone->stats_total,
+ &old_n_clone->stats_total,
+ sizeof (new_n_clone->stats_total));
+ clib_memcpy_fast (&new_n_clone->stats_last_clear,
+ &old_n_clone->stats_last_clear,
+ sizeof (new_n_clone->stats_last_clear));
/* keep previous node state */
new_n_clone->state = old_n_clone->state;
@@ -1129,17 +1129,17 @@ vlib_worker_thread_node_refork (void)
rt->thread_index = vm_clone->thread_index;
/* copy runtime_data, will be overwritten later for existing rt */
if (n->runtime_data && n->runtime_data_bytes > 0)
- clib_memcpy (rt->runtime_data, n->runtime_data,
- clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
- n->runtime_data_bytes));
+ clib_memcpy_fast (rt->runtime_data, n->runtime_data,
+ clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
+ n->runtime_data_bytes));
}
for (j = 0; j < vec_len (old_rt); j++)
{
rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
rt->state = old_rt[j].state;
- clib_memcpy (rt->runtime_data, old_rt[j].runtime_data,
- VLIB_NODE_RUNTIME_DATA_SIZE);
+ clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
+ VLIB_NODE_RUNTIME_DATA_SIZE);
}
vec_free (old_rt);
@@ -1156,17 +1156,17 @@ vlib_worker_thread_node_refork (void)
rt->thread_index = vm_clone->thread_index;
/* copy runtime_data, will be overwritten later for existing rt */
if (n->runtime_data && n->runtime_data_bytes > 0)
- clib_memcpy (rt->runtime_data, n->runtime_data,
- clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
- n->runtime_data_bytes));
+ clib_memcpy_fast (rt->runtime_data, n->runtime_data,
+ clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
+ n->runtime_data_bytes));
}
for (j = 0; j < vec_len (old_rt); j++)
{
rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
rt->state = old_rt[j].state;
- clib_memcpy (rt->runtime_data, old_rt[j].runtime_data,
- VLIB_NODE_RUNTIME_DATA_SIZE);
+ clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
+ VLIB_NODE_RUNTIME_DATA_SIZE);
}
vec_free (old_rt);
diff --git a/src/vlib/trace.c b/src/vlib/trace.c
index cfc3bda2cd0..e3b289879da 100644
--- a/src/vlib/trace.c
+++ b/src/vlib/trace.c
@@ -75,14 +75,14 @@ vlib_trace_frame_buffers_only (vlib_main_t * vm,
if (b0->flags & VLIB_BUFFER_IS_TRACED)
{
t0 = vlib_add_trace (vm, node, b0, n_buffer_data_bytes_in_trace);
- clib_memcpy (t0, b0->data + b0->current_data,
- n_buffer_data_bytes_in_trace);
+ clib_memcpy_fast (t0, b0->data + b0->current_data,
+ n_buffer_data_bytes_in_trace);
}
if (b1->flags & VLIB_BUFFER_IS_TRACED)
{
t1 = vlib_add_trace (vm, node, b1, n_buffer_data_bytes_in_trace);
- clib_memcpy (t1, b1->data + b1->current_data,
- n_buffer_data_bytes_in_trace);
+ clib_memcpy_fast (t1, b1->data + b1->current_data,
+ n_buffer_data_bytes_in_trace);
}
from += 2;
n_left -= 2;
@@ -101,8 +101,8 @@ vlib_trace_frame_buffers_only (vlib_main_t * vm,
if (b0->flags & VLIB_BUFFER_IS_TRACED)
{
t0 = vlib_add_trace (vm, node, b0, n_buffer_data_bytes_in_trace);
- clib_memcpy (t0, b0->data + b0->current_data,
- n_buffer_data_bytes_in_trace);
+ clib_memcpy_fast (t0, b0->data + b0->current_data,
+ n_buffer_data_bytes_in_trace);
}
from += 1;
n_left -= 1;