From 178cf493d009995b28fdf220f04c98860ff79a9b Mon Sep 17 00:00:00 2001 From: Dave Barach Date: Tue, 13 Nov 2018 16:34:13 -0500 Subject: Remove c-11 memcpy checks from perf-critical code Change-Id: Id4f37f5d4a03160572954a416efa1ef9b3d79ad1 Signed-off-by: Dave Barach --- src/vlib/buffer.c | 19 ++++++++++--------- src/vlib/buffer_funcs.h | 44 ++++++++++++++++++++++---------------------- src/vlib/buffer_node.h | 8 ++++---- src/vlib/linux/pci.c | 2 +- src/vlib/main.c | 8 ++++---- src/vlib/node_funcs.h | 3 ++- src/vlib/threads.c | 38 +++++++++++++++++++------------------- src/vlib/trace.c | 12 ++++++------ 8 files changed, 68 insertions(+), 66 deletions(-) (limited to 'src/vlib') diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index cc852358541..133d640d693 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -672,8 +672,8 @@ vlib_packet_template_buffer_init (vlib_main_t * vm, { vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]); ASSERT (b->current_length == vec_len (t->packet_data)); - clib_memcpy (vlib_buffer_get_current (b), t->packet_data, - b->current_length); + clib_memcpy_fast (vlib_buffer_get_current (b), t->packet_data, + b->current_length); } } @@ -738,8 +738,8 @@ vlib_packet_template_get_packet (vlib_main_t * vm, *bi_result = bi; b = vlib_get_buffer (vm, bi); - clib_memcpy (vlib_buffer_get_current (b), - t->packet_data, vec_len (t->packet_data)); + clib_memcpy_fast (vlib_buffer_get_current (b), + t->packet_data, vec_len (t->packet_data)); b->current_length = vec_len (t->packet_data); return b->data; @@ -796,7 +796,8 @@ vlib_buffer_add_data (vlib_main_t * vm, n_left_this_buffer = n_buffer_bytes - (b->current_data + b->current_length); n = clib_min (n_left_this_buffer, n_left); - clib_memcpy (vlib_buffer_get_current (b) + b->current_length, d, n); + clib_memcpy_fast (vlib_buffer_get_current (b) + b->current_length, d, + n); b->current_length += n; n_left -= n; if (n_left == 0) @@ -847,8 +848,8 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, } u16 len = (data_len > max) ? max : data_len; - clib_memcpy (vlib_buffer_get_current (l) + l->current_length, - data + copied, len); + clib_memcpy_fast (vlib_buffer_get_current (l) + l->current_length, + data + copied, len); vlib_buffer_chain_increase_length (first, l, len); data_len -= len; copied += len; @@ -974,8 +975,8 @@ vlib_buffer_main_init (struct vlib_main_t * vm) { /* external plugin has registered own buffer callbacks so we just copy them and quit */ - clib_memcpy (&bm->cb, vlib_buffer_callbacks, - sizeof (vlib_buffer_callbacks_t)); + clib_memcpy_fast (&bm->cb, vlib_buffer_callbacks, + sizeof (vlib_buffer_callbacks_t)); bm->callbacks_registered = 1; return 0; } diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index ce62c8b83fa..d15ef5702f8 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -304,7 +304,7 @@ vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents) { b = vlib_get_buffer (vm, buffer_index); l = b->current_length; - clib_memcpy (contents + content_len, b->data + b->current_data, l); + clib_memcpy_fast (contents + content_len, b->data + b->current_data, l); content_len += l; if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT)) break; @@ -470,7 +470,7 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm, /* following code is intentionaly duplicated to allow compiler to optimize fast path when n_buffers is constant value */ src = fl->buffers + len - n_buffers; - clib_memcpy (buffers, src, n_buffers * sizeof (u32)); + clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32)); _vec_len (fl->buffers) -= n_buffers; /* Verify that buffers are known free. */ @@ -481,7 +481,7 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm, } src = fl->buffers + len - n_buffers; - clib_memcpy (buffers, src, n_buffers * sizeof (u32)); + clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32)); _vec_len (fl->buffers) -= n_buffers; /* Verify that buffers are known free. */ @@ -758,10 +758,10 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b) d->flags = s->flags & flag_mask; d->total_length_not_including_first_buffer = s->total_length_not_including_first_buffer; - clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque)); - clib_memcpy (d->opaque2, s->opaque2, sizeof (s->opaque2)); - clib_memcpy (vlib_buffer_get_current (d), - vlib_buffer_get_current (s), s->current_length); + clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque)); + clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2)); + clib_memcpy_fast (vlib_buffer_get_current (d), + vlib_buffer_get_current (s), s->current_length); /* next segments */ for (i = 1; i < n_buffers; i++) @@ -773,8 +773,8 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b) d = vlib_get_buffer (vm, new_buffers[i]); d->current_data = s->current_data; d->current_length = s->current_length; - clib_memcpy (vlib_buffer_get_current (d), - vlib_buffer_get_current (s), s->current_length); + clib_memcpy_fast (vlib_buffer_get_current (d), + vlib_buffer_get_current (s), s->current_length); d->flags = s->flags & flag_mask; } @@ -846,10 +846,10 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, } d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT; d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID; - clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque)); - clib_memcpy (d->opaque2, s->opaque2, sizeof (s->opaque2)); - clib_memcpy (vlib_buffer_get_current (d), vlib_buffer_get_current (s), - head_end_offset); + clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque)); + clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2)); + clib_memcpy_fast (vlib_buffer_get_current (d), + vlib_buffer_get_current (s), head_end_offset); d->next_buffer = src_buffer; } vlib_buffer_advance (s, head_end_offset); @@ -982,8 +982,8 @@ vlib_buffer_chain_append_data (vlib_main_t * vm, u16 len = clib_min (data_len, n_buffer_bytes - last->current_length - last->current_data); - clib_memcpy (vlib_buffer_get_current (last) + last->current_length, data, - len); + clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length, + data, len); vlib_buffer_chain_increase_length (first, last, len); return len; } @@ -1088,10 +1088,10 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst, /* Make sure buffer template is sane. */ ASSERT (fl->index == vlib_buffer_get_free_list_index (src)); - clib_memcpy (STRUCT_MARK_PTR (dst, template_start), - STRUCT_MARK_PTR (src, template_start), - STRUCT_OFFSET_OF (vlib_buffer_t, template_end) - - STRUCT_OFFSET_OF (vlib_buffer_t, template_start)); + clib_memcpy_fast (STRUCT_MARK_PTR (dst, template_start), + STRUCT_MARK_PTR (src, template_start), + STRUCT_OFFSET_OF (vlib_buffer_t, template_end) - + STRUCT_OFFSET_OF (vlib_buffer_t, template_start)); /* Not in the first 16 octets. */ dst->n_add_refs = src->n_add_refs; @@ -1240,9 +1240,9 @@ vlib_buffer_chain_compress (vlib_main_t * vm, vlib_buffer_t *second = vlib_get_buffer (vm, first->next_buffer); u32 need = want_first_size - first->current_length; u32 amount_to_copy = clib_min (need, second->current_length); - clib_memcpy (((u8 *) vlib_buffer_get_current (first)) + - first->current_length, - vlib_buffer_get_current (second), amount_to_copy); + clib_memcpy_fast (((u8 *) vlib_buffer_get_current (first)) + + first->current_length, + vlib_buffer_get_current (second), amount_to_copy); first->current_length += amount_to_copy; vlib_buffer_advance (second, amount_to_copy); if (first->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID) diff --git a/src/vlib/buffer_node.h b/src/vlib/buffer_node.h index 35e15a5d919..bff7a78cb94 100644 --- a/src/vlib/buffer_node.h +++ b/src/vlib/buffer_node.h @@ -383,7 +383,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node, #ifdef CLIB_HAVE_VEC512 if (n_enqueued >= 32) { - clib_memcpy (to_next, buffers, 32 * sizeof (u32)); + clib_memcpy_fast (to_next, buffers, 32 * sizeof (u32)); nexts += 32; to_next += 32; buffers += 32; @@ -397,7 +397,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node, #ifdef CLIB_HAVE_VEC256 if (n_enqueued >= 16) { - clib_memcpy (to_next, buffers, 16 * sizeof (u32)); + clib_memcpy_fast (to_next, buffers, 16 * sizeof (u32)); nexts += 16; to_next += 16; buffers += 16; @@ -411,7 +411,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node, #ifdef CLIB_HAVE_VEC128 if (n_enqueued >= 8) { - clib_memcpy (to_next, buffers, 8 * sizeof (u32)); + clib_memcpy_fast (to_next, buffers, 8 * sizeof (u32)); nexts += 8; to_next += 8; buffers += 8; @@ -424,7 +424,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node, if (n_enqueued >= 4) { - clib_memcpy (to_next, buffers, 4 * sizeof (u32)); + clib_memcpy_fast (to_next, buffers, 4 * sizeof (u32)); nexts += 4; to_next += 4; buffers += 4; diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c index c6fa8a7da34..ed43580ffc0 100644 --- a/src/vlib/linux/pci.c +++ b/src/vlib/linux/pci.c @@ -584,7 +584,7 @@ vfio_set_irqs (vlib_main_t * vm, linux_pci_device_t * p, u32 index, u32 start, if (efds) { flags |= VFIO_IRQ_SET_DATA_EVENTFD; - clib_memcpy (&irq_set->data, efds, data_len); + clib_memcpy_fast (&irq_set->data, efds, data_len); } else flags |= VFIO_IRQ_SET_DATA_NONE; diff --git a/src/vlib/main.c b/src/vlib/main.c index b46caf1bcb4..964feb06c65 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -1650,12 +1650,12 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) te->n_data_elts, te->n_data_elt_bytes); if (te->n_data_bytes < sizeof (te->inline_event_data)) - clib_memcpy (data, te->inline_event_data, - te->n_data_bytes); + clib_memcpy_fast (data, te->inline_event_data, + te->n_data_bytes); else { - clib_memcpy (data, te->event_data_as_vector, - te->n_data_bytes); + clib_memcpy_fast (data, te->event_data_as_vector, + te->n_data_bytes); vec_free (te->event_data_as_vector); } pool_put (nm->signal_timed_event_data_pool, te); diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h index 13614aa14cd..777746a0f78 100644 --- a/src/vlib/node_funcs.h +++ b/src/vlib/node_funcs.h @@ -135,7 +135,8 @@ vlib_node_set_runtime_data (vlib_main_t * vm, u32 node_index, STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data)); if (vec_len (n->runtime_data) > 0) - clib_memcpy (r->runtime_data, n->runtime_data, vec_len (n->runtime_data)); + clib_memcpy_fast (r->runtime_data, n->runtime_data, + vec_len (n->runtime_data)); } /** \brief Set node dispatch state. diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 7ecfa309378..fc2586c8967 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -1041,8 +1041,8 @@ vlib_worker_thread_node_refork (void) u64 *old_counters = vm_clone->error_main.counters; u64 *old_counters_all_clear = vm_clone->error_main.counters_last_clear; - clib_memcpy (&vm_clone->error_main, &vm->error_main, - sizeof (vm->error_main)); + clib_memcpy_fast (&vm_clone->error_main, &vm->error_main, + sizeof (vm->error_main)); j = vec_len (vm->error_main.counters) - 1; vec_validate_aligned (old_counters, j, CLIB_CACHE_LINE_BYTES); vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES); @@ -1083,7 +1083,7 @@ vlib_worker_thread_node_refork (void) new_n = nm->nodes[j]; old_n_clone = old_nodes_clone[j]; - clib_memcpy (new_n_clone, new_n, sizeof (*new_n)); + clib_memcpy_fast (new_n_clone, new_n, sizeof (*new_n)); /* none of the copied nodes have enqueue rights given out */ new_n_clone->owner_node_index = VLIB_INVALID_NODE_INDEX; @@ -1098,12 +1098,12 @@ vlib_worker_thread_node_refork (void) else { /* Copy stats if the old data is valid */ - clib_memcpy (&new_n_clone->stats_total, - &old_n_clone->stats_total, - sizeof (new_n_clone->stats_total)); - clib_memcpy (&new_n_clone->stats_last_clear, - &old_n_clone->stats_last_clear, - sizeof (new_n_clone->stats_last_clear)); + clib_memcpy_fast (&new_n_clone->stats_total, + &old_n_clone->stats_total, + sizeof (new_n_clone->stats_total)); + clib_memcpy_fast (&new_n_clone->stats_last_clear, + &old_n_clone->stats_last_clear, + sizeof (new_n_clone->stats_last_clear)); /* keep previous node state */ new_n_clone->state = old_n_clone->state; @@ -1129,17 +1129,17 @@ vlib_worker_thread_node_refork (void) rt->thread_index = vm_clone->thread_index; /* copy runtime_data, will be overwritten later for existing rt */ if (n->runtime_data && n->runtime_data_bytes > 0) - clib_memcpy (rt->runtime_data, n->runtime_data, - clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, - n->runtime_data_bytes)); + clib_memcpy_fast (rt->runtime_data, n->runtime_data, + clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, + n->runtime_data_bytes)); } for (j = 0; j < vec_len (old_rt); j++) { rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); rt->state = old_rt[j].state; - clib_memcpy (rt->runtime_data, old_rt[j].runtime_data, - VLIB_NODE_RUNTIME_DATA_SIZE); + clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); } vec_free (old_rt); @@ -1156,17 +1156,17 @@ vlib_worker_thread_node_refork (void) rt->thread_index = vm_clone->thread_index; /* copy runtime_data, will be overwritten later for existing rt */ if (n->runtime_data && n->runtime_data_bytes > 0) - clib_memcpy (rt->runtime_data, n->runtime_data, - clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, - n->runtime_data_bytes)); + clib_memcpy_fast (rt->runtime_data, n->runtime_data, + clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, + n->runtime_data_bytes)); } for (j = 0; j < vec_len (old_rt); j++) { rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); rt->state = old_rt[j].state; - clib_memcpy (rt->runtime_data, old_rt[j].runtime_data, - VLIB_NODE_RUNTIME_DATA_SIZE); + clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); } vec_free (old_rt); diff --git a/src/vlib/trace.c b/src/vlib/trace.c index cfc3bda2cd0..e3b289879da 100644 --- a/src/vlib/trace.c +++ b/src/vlib/trace.c @@ -75,14 +75,14 @@ vlib_trace_frame_buffers_only (vlib_main_t * vm, if (b0->flags & VLIB_BUFFER_IS_TRACED) { t0 = vlib_add_trace (vm, node, b0, n_buffer_data_bytes_in_trace); - clib_memcpy (t0, b0->data + b0->current_data, - n_buffer_data_bytes_in_trace); + clib_memcpy_fast (t0, b0->data + b0->current_data, + n_buffer_data_bytes_in_trace); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { t1 = vlib_add_trace (vm, node, b1, n_buffer_data_bytes_in_trace); - clib_memcpy (t1, b1->data + b1->current_data, - n_buffer_data_bytes_in_trace); + clib_memcpy_fast (t1, b1->data + b1->current_data, + n_buffer_data_bytes_in_trace); } from += 2; n_left -= 2; @@ -101,8 +101,8 @@ vlib_trace_frame_buffers_only (vlib_main_t * vm, if (b0->flags & VLIB_BUFFER_IS_TRACED) { t0 = vlib_add_trace (vm, node, b0, n_buffer_data_bytes_in_trace); - clib_memcpy (t0, b0->data + b0->current_data, - n_buffer_data_bytes_in_trace); + clib_memcpy_fast (t0, b0->data + b0->current_data, + n_buffer_data_bytes_in_trace); } from += 1; n_left -= 1; -- cgit 1.2.3-korg