Remove c-11 memcpy checks from perf-critical code

Change-Id: Id4f37f5d4a03160572954a416efa1ef9b3d79ad1
Signed-off-by: Dave Barach <dave@barachs.net>

8 files changed, 68 insertions, 66 deletions

diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index cc852358541..133d640d693 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -672,8 +672,8 @@ vlib_packet_template_buffer_init (vlib_main_t * vm,
     {
       vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]);
       ASSERT (b->current_length == vec_len (t->packet_data));
-      clib_memcpy (vlib_buffer_get_current (b), t->packet_data,
-		   b->current_length);
+      clib_memcpy_fast (vlib_buffer_get_current (b), t->packet_data,
+			b->current_length);
     }
 }
 
@@ -738,8 +738,8 @@ vlib_packet_template_get_packet (vlib_main_t * vm,
   *bi_result = bi;
 
   b = vlib_get_buffer (vm, bi);
-  clib_memcpy (vlib_buffer_get_current (b),
-	       t->packet_data, vec_len (t->packet_data));
+  clib_memcpy_fast (vlib_buffer_get_current (b),
+		    t->packet_data, vec_len (t->packet_data));
   b->current_length = vec_len (t->packet_data);
 
   return b->data;
@@ -796,7 +796,8 @@ vlib_buffer_add_data (vlib_main_t * vm,
       n_left_this_buffer =
 	n_buffer_bytes - (b->current_data + b->current_length);
       n = clib_min (n_left_this_buffer, n_left);
-      clib_memcpy (vlib_buffer_get_current (b) + b->current_length, d, n);
+      clib_memcpy_fast (vlib_buffer_get_current (b) + b->current_length, d,
+			n);
       b->current_length += n;
       n_left -= n;
       if (n_left == 0)
@@ -847,8 +848,8 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm,
 	}
 
       u16 len = (data_len > max) ? max : data_len;
-      clib_memcpy (vlib_buffer_get_current (l) + l->current_length,
-		   data + copied, len);
+      clib_memcpy_fast (vlib_buffer_get_current (l) + l->current_length,
+			data + copied, len);
       vlib_buffer_chain_increase_length (first, l, len);
       data_len -= len;
       copied += len;
@@ -974,8 +975,8 @@ vlib_buffer_main_init (struct vlib_main_t * vm)
     {
       /* external plugin has registered own buffer callbacks
          so we just copy them  and quit */
-      clib_memcpy (&bm->cb, vlib_buffer_callbacks,
-		   sizeof (vlib_buffer_callbacks_t));
+      clib_memcpy_fast (&bm->cb, vlib_buffer_callbacks,
+			sizeof (vlib_buffer_callbacks_t));
       bm->callbacks_registered = 1;
       return 0;
     }
diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h
index ce62c8b83fa..d15ef5702f8 100644
--- a/src/vlib/buffer_funcs.h
+++ b/src/vlib/buffer_funcs.h
@@ -304,7 +304,7 @@ vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
     {
       b = vlib_get_buffer (vm, buffer_index);
       l = b->current_length;
-      clib_memcpy (contents + content_len, b->data + b->current_data, l);
+      clib_memcpy_fast (contents + content_len, b->data + b->current_data, l);
       content_len += l;
       if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
 	break;
@@ -470,7 +470,7 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
       /* following code is intentionaly duplicated to allow compiler
          to optimize fast path when n_buffers is constant value */
       src = fl->buffers + len - n_buffers;
-      clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+      clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32));
       _vec_len (fl->buffers) -= n_buffers;
 
       /* Verify that buffers are known free. */
@@ -481,7 +481,7 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm,
     }
 
   src = fl->buffers + len - n_buffers;
-  clib_memcpy (buffers, src, n_buffers * sizeof (u32));
+  clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32));
   _vec_len (fl->buffers) -= n_buffers;
 
   /* Verify that buffers are known free. */
@@ -758,10 +758,10 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b)
   d->flags = s->flags & flag_mask;
   d->total_length_not_including_first_buffer =
     s->total_length_not_including_first_buffer;
-  clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
-  clib_memcpy (d->opaque2, s->opaque2, sizeof (s->opaque2));
-  clib_memcpy (vlib_buffer_get_current (d),
-	       vlib_buffer_get_current (s), s->current_length);
+  clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
+  clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
+  clib_memcpy_fast (vlib_buffer_get_current (d),
+		    vlib_buffer_get_current (s), s->current_length);
 
   /* next segments */
   for (i = 1; i < n_buffers; i++)
@@ -773,8 +773,8 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b)
       d = vlib_get_buffer (vm, new_buffers[i]);
       d->current_data = s->current_data;
       d->current_length = s->current_length;
-      clib_memcpy (vlib_buffer_get_current (d),
-		   vlib_buffer_get_current (s), s->current_length);
+      clib_memcpy_fast (vlib_buffer_get_current (d),
+			vlib_buffer_get_current (s), s->current_length);
       d->flags = s->flags & flag_mask;
     }
 
@@ -846,10 +846,10 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
 	}
       d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
       d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
-      clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque));
-      clib_memcpy (d->opaque2, s->opaque2, sizeof (s->opaque2));
-      clib_memcpy (vlib_buffer_get_current (d), vlib_buffer_get_current (s),
-		   head_end_offset);
+      clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
+      clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
+      clib_memcpy_fast (vlib_buffer_get_current (d),
+			vlib_buffer_get_current (s), head_end_offset);
       d->next_buffer = src_buffer;
     }
   vlib_buffer_advance (s, head_end_offset);
@@ -982,8 +982,8 @@ vlib_buffer_chain_append_data (vlib_main_t * vm,
   u16 len = clib_min (data_len,
 		      n_buffer_bytes - last->current_length -
 		      last->current_data);
-  clib_memcpy (vlib_buffer_get_current (last) + last->current_length, data,
-	       len);
+  clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length,
+		    data, len);
   vlib_buffer_chain_increase_length (first, last, len);
   return len;
 }
@@ -1088,10 +1088,10 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst,
   /* Make sure buffer template is sane. */
   ASSERT (fl->index == vlib_buffer_get_free_list_index (src));
 
-  clib_memcpy (STRUCT_MARK_PTR (dst, template_start),
-	       STRUCT_MARK_PTR (src, template_start),
-	       STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
-	       STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
+  clib_memcpy_fast (STRUCT_MARK_PTR (dst, template_start),
+		    STRUCT_MARK_PTR (src, template_start),
+		    STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
+		    STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
 
   /* Not in the first 16 octets. */
   dst->n_add_refs = src->n_add_refs;
@@ -1240,9 +1240,9 @@ vlib_buffer_chain_compress (vlib_main_t * vm,
       vlib_buffer_t *second = vlib_get_buffer (vm, first->next_buffer);
       u32 need = want_first_size - first->current_length;
       u32 amount_to_copy = clib_min (need, second->current_length);
-      clib_memcpy (((u8 *) vlib_buffer_get_current (first)) +
-		   first->current_length,
-		   vlib_buffer_get_current (second), amount_to_copy);
+      clib_memcpy_fast (((u8 *) vlib_buffer_get_current (first)) +
+			first->current_length,
+			vlib_buffer_get_current (second), amount_to_copy);
       first->current_length += amount_to_copy;
       vlib_buffer_advance (second, amount_to_copy);
       if (first->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)
diff --git a/src/vlib/buffer_node.h b/src/vlib/buffer_node.h
index 35e15a5d919..bff7a78cb94 100644
--- a/src/vlib/buffer_node.h
+++ b/src/vlib/buffer_node.h
@@ -383,7 +383,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
 #ifdef CLIB_HAVE_VEC512
       if (n_enqueued >= 32)
 	{
-	  clib_memcpy (to_next, buffers, 32 * sizeof (u32));
+	  clib_memcpy_fast (to_next, buffers, 32 * sizeof (u32));
 	  nexts += 32;
 	  to_next += 32;
 	  buffers += 32;
@@ -397,7 +397,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
 #ifdef CLIB_HAVE_VEC256
       if (n_enqueued >= 16)
 	{
-	  clib_memcpy (to_next, buffers, 16 * sizeof (u32));
+	  clib_memcpy_fast (to_next, buffers, 16 * sizeof (u32));
 	  nexts += 16;
 	  to_next += 16;
 	  buffers += 16;
@@ -411,7 +411,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
 #ifdef CLIB_HAVE_VEC128
       if (n_enqueued >= 8)
 	{
-	  clib_memcpy (to_next, buffers, 8 * sizeof (u32));
+	  clib_memcpy_fast (to_next, buffers, 8 * sizeof (u32));
 	  nexts += 8;
 	  to_next += 8;
 	  buffers += 8;
@@ -424,7 +424,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
 
       if (n_enqueued >= 4)
 	{
-	  clib_memcpy (to_next, buffers, 4 * sizeof (u32));
+	  clib_memcpy_fast (to_next, buffers, 4 * sizeof (u32));
 	  nexts += 4;
 	  to_next += 4;
 	  buffers += 4;
diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c
index c6fa8a7da34..ed43580ffc0 100644
--- a/src/vlib/linux/pci.c
+++ b/src/vlib/linux/pci.c
@@ -584,7 +584,7 @@ vfio_set_irqs (vlib_main_t * vm, linux_pci_device_t * p, u32 index, u32 start,
   if (efds)
     {
       flags |= VFIO_IRQ_SET_DATA_EVENTFD;
-      clib_memcpy (&irq_set->data, efds, data_len);
+      clib_memcpy_fast (&irq_set->data, efds, data_len);
     }
   else
     flags |= VFIO_IRQ_SET_DATA_NONE;
diff --git a/src/vlib/main.c b/src/vlib/main.c
index b46caf1bcb4..964feb06c65 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -1650,12 +1650,12 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
 							  te->n_data_elts,
 							  te->n_data_elt_bytes);
 		      if (te->n_data_bytes < sizeof (te->inline_event_data))
-			clib_memcpy (data, te->inline_event_data,
-				     te->n_data_bytes);
+			clib_memcpy_fast (data, te->inline_event_data,
+					  te->n_data_bytes);
 		      else
 			{
-			  clib_memcpy (data, te->event_data_as_vector,
-				       te->n_data_bytes);
+			  clib_memcpy_fast (data, te->event_data_as_vector,
+					    te->n_data_bytes);
 			  vec_free (te->event_data_as_vector);
 			}
 		      pool_put (nm->signal_timed_event_data_pool, te);
diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h
index 13614aa14cd..777746a0f78 100644
--- a/src/vlib/node_funcs.h
+++ b/src/vlib/node_funcs.h
@@ -135,7 +135,8 @@ vlib_node_set_runtime_data (vlib_main_t * vm, u32 node_index,
 	  STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data));
 
   if (vec_len (n->runtime_data) > 0)
-    clib_memcpy (r->runtime_data, n->runtime_data, vec_len (n->runtime_data));
+    clib_memcpy_fast (r->runtime_data, n->runtime_data,
+		      vec_len (n->runtime_data));
 }
 
 /** \brief Set node dispatch state.
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 7ecfa309378..fc2586c8967 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -1041,8 +1041,8 @@ vlib_worker_thread_node_refork (void)
   u64 *old_counters = vm_clone->error_main.counters;
   u64 *old_counters_all_clear = vm_clone->error_main.counters_last_clear;
 
-  clib_memcpy (&vm_clone->error_main, &vm->error_main,
-	       sizeof (vm->error_main));
+  clib_memcpy_fast (&vm_clone->error_main, &vm->error_main,
+		    sizeof (vm->error_main));
   j = vec_len (vm->error_main.counters) - 1;
   vec_validate_aligned (old_counters, j, CLIB_CACHE_LINE_BYTES);
   vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
@@ -1083,7 +1083,7 @@ vlib_worker_thread_node_refork (void)
       new_n = nm->nodes[j];
       old_n_clone = old_nodes_clone[j];
 
-      clib_memcpy (new_n_clone, new_n, sizeof (*new_n));
+      clib_memcpy_fast (new_n_clone, new_n, sizeof (*new_n));
       /* none of the copied nodes have enqueue rights given out */
       new_n_clone->owner_node_index = VLIB_INVALID_NODE_INDEX;
 
@@ -1098,12 +1098,12 @@ vlib_worker_thread_node_refork (void)
       else
 	{
 	  /* Copy stats if the old data is valid */
-	  clib_memcpy (&new_n_clone->stats_total,
-		       &old_n_clone->stats_total,
-		       sizeof (new_n_clone->stats_total));
-	  clib_memcpy (&new_n_clone->stats_last_clear,
-		       &old_n_clone->stats_last_clear,
-		       sizeof (new_n_clone->stats_last_clear));
+	  clib_memcpy_fast (&new_n_clone->stats_total,
+			    &old_n_clone->stats_total,
+			    sizeof (new_n_clone->stats_total));
+	  clib_memcpy_fast (&new_n_clone->stats_last_clear,
+			    &old_n_clone->stats_last_clear,
+			    sizeof (new_n_clone->stats_last_clear));
 
 	  /* keep previous node state */
 	  new_n_clone->state = old_n_clone->state;
@@ -1129,17 +1129,17 @@ vlib_worker_thread_node_refork (void)
     rt->thread_index = vm_clone->thread_index;
     /* copy runtime_data, will be overwritten later for existing rt */
     if (n->runtime_data && n->runtime_data_bytes > 0)
-      clib_memcpy (rt->runtime_data, n->runtime_data,
-		   clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
-			     n->runtime_data_bytes));
+      clib_memcpy_fast (rt->runtime_data, n->runtime_data,
+			clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
+				  n->runtime_data_bytes));
   }
 
   for (j = 0; j < vec_len (old_rt); j++)
     {
       rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
       rt->state = old_rt[j].state;
-      clib_memcpy (rt->runtime_data, old_rt[j].runtime_data,
-		   VLIB_NODE_RUNTIME_DATA_SIZE);
+      clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
+			VLIB_NODE_RUNTIME_DATA_SIZE);
     }
 
   vec_free (old_rt);
@@ -1156,17 +1156,17 @@ vlib_worker_thread_node_refork (void)
     rt->thread_index = vm_clone->thread_index;
     /* copy runtime_data, will be overwritten later for existing rt */
     if (n->runtime_data && n->runtime_data_bytes > 0)
-      clib_memcpy (rt->runtime_data, n->runtime_data,
-		   clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
-			     n->runtime_data_bytes));
+      clib_memcpy_fast (rt->runtime_data, n->runtime_data,
+			clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
+				  n->runtime_data_bytes));
   }
 
   for (j = 0; j < vec_len (old_rt); j++)
     {
       rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
       rt->state = old_rt[j].state;
-      clib_memcpy (rt->runtime_data, old_rt[j].runtime_data,
-		   VLIB_NODE_RUNTIME_DATA_SIZE);
+      clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
+			VLIB_NODE_RUNTIME_DATA_SIZE);
     }
 
   vec_free (old_rt);
diff --git a/src/vlib/trace.c b/src/vlib/trace.c
index cfc3bda2cd0..e3b289879da 100644
--- a/src/vlib/trace.c
+++ b/src/vlib/trace.c
@@ -75,14 +75,14 @@ vlib_trace_frame_buffers_only (vlib_main_t * vm,
       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 	{
 	  t0 = vlib_add_trace (vm, node, b0, n_buffer_data_bytes_in_trace);
-	  clib_memcpy (t0, b0->data + b0->current_data,
-		       n_buffer_data_bytes_in_trace);
+	  clib_memcpy_fast (t0, b0->data + b0->current_data,
+			    n_buffer_data_bytes_in_trace);
 	}
       if (b1->flags & VLIB_BUFFER_IS_TRACED)
 	{
 	  t1 = vlib_add_trace (vm, node, b1, n_buffer_data_bytes_in_trace);
-	  clib_memcpy (t1, b1->data + b1->current_data,
-		       n_buffer_data_bytes_in_trace);
+	  clib_memcpy_fast (t1, b1->data + b1->current_data,
+			    n_buffer_data_bytes_in_trace);
 	}
       from += 2;
       n_left -= 2;
@@ -101,8 +101,8 @@ vlib_trace_frame_buffers_only (vlib_main_t * vm,
       if (b0->flags & VLIB_BUFFER_IS_TRACED)
 	{
 	  t0 = vlib_add_trace (vm, node, b0, n_buffer_data_bytes_in_trace);
-	  clib_memcpy (t0, b0->data + b0->current_data,
-		       n_buffer_data_bytes_in_trace);
+	  clib_memcpy_fast (t0, b0->data + b0->current_data,
+			    n_buffer_data_bytes_in_trace);
 	}
       from += 1;
       n_left -= 1;