Remove c-11 memcpy checks from perf-critical code

Change-Id: Id4f37f5d4a03160572954a416efa1ef9b3d79ad1
Signed-off-by: Dave Barach <dave@barachs.net>

9 files changed, 27 insertions, 28 deletions

diff --git a/src/vnet/devices/af_packet/device.c b/src/vnet/devices/af_packet/device.c
index 811642ec7b9..9d08c620d94 100644
--- a/src/vnet/devices/af_packet/device.c
+++ b/src/vnet/devices/af_packet/device.c
@@ -119,9 +119,9 @@ af_packet_interface_tx (vlib_main_t * vm,
 	{
 	  b0 = vlib_get_buffer (vm, bi);
 	  len = b0->current_length;
-	  clib_memcpy ((u8 *) tph +
-		       TPACKET_ALIGN (sizeof (struct tpacket2_hdr)) + offset,
-		       vlib_buffer_get_current (b0), len);
+	  clib_memcpy_fast ((u8 *) tph +
+			    TPACKET_ALIGN (sizeof (struct tpacket2_hdr)) +
+			    offset, vlib_buffer_get_current (b0), len);
 	  offset += len;
 	}
       while ((bi =
diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c
index 8dbd8554f01..97aa170c0a6 100644
--- a/src/vnet/devices/af_packet/node.c
+++ b/src/vnet/devices/af_packet/node.c
@@ -248,9 +248,9 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 		{
 		  if (PREDICT_TRUE (offset == 0))
 		    {
-		      clib_memcpy (vlib_buffer_get_current (b0),
-				   (u8 *) tph + tph->tp_mac,
-				   sizeof (ethernet_header_t));
+		      clib_memcpy_fast (vlib_buffer_get_current (b0),
+					(u8 *) tph + tph->tp_mac,
+					sizeof (ethernet_header_t));
 		      ethernet_header_t *eth = vlib_buffer_get_current (b0);
 		      ethernet_vlan_header_t *vlan =
 			(ethernet_vlan_header_t *) (eth + 1);
@@ -262,10 +262,10 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 		      bytes_copied = sizeof (ethernet_header_t);
 		    }
 		}
-	      clib_memcpy (((u8 *) vlib_buffer_get_current (b0)) +
-			   bytes_copied + vlan_len,
-			   (u8 *) tph + tph->tp_mac + offset + bytes_copied,
-			   (bytes_to_copy - bytes_copied));
+	      clib_memcpy_fast (((u8 *) vlib_buffer_get_current (b0)) +
+				bytes_copied + vlan_len,
+				(u8 *) tph + tph->tp_mac + offset +
+				bytes_copied, (bytes_to_copy - bytes_copied));
 
 	      /* fill buffer header */
 	      b0->current_length = bytes_to_copy + vlan_len;
@@ -319,7 +319,7 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      tr = vlib_add_trace (vm, node, first_b0, sizeof (*tr));
 	      tr->next_index = next0;
 	      tr->hw_if_index = apif->hw_if_index;
-	      clib_memcpy (&tr->tph, tph, sizeof (struct tpacket2_hdr));
+	      clib_memcpy_fast (&tr->tph, tph, sizeof (struct tpacket2_hdr));
 	    }
 
 	  /* enque and take next packet */
diff --git a/src/vnet/devices/netmap/device.c b/src/vnet/devices/netmap/device.c
index 409a52f1005..b14bff43c06 100644
--- a/src/vnet/devices/netmap/device.c
+++ b/src/vnet/devices/netmap/device.c
@@ -143,8 +143,8 @@ netmap_interface_tx (vlib_main_t * vm,
 	      b0 = vlib_get_buffer (vm, bi);
 	      len = b0->current_length;
 	      /* memcpy */
-	      clib_memcpy ((u8 *) NETMAP_BUF (ring, slot->buf_idx) + offset,
-			   vlib_buffer_get_current (b0), len);
+	      clib_memcpy_fast ((u8 *) NETMAP_BUF (ring, slot->buf_idx) +
+				offset, vlib_buffer_get_current (b0), len);
 	      offset += len;
 	    }
 	  while ((bi = b0->next_buffer));
diff --git a/src/vnet/devices/netmap/node.c b/src/vnet/devices/netmap/node.c
index e541700f4ea..dde706f20c7 100644
--- a/src/vnet/devices/netmap/node.c
+++ b/src/vnet/devices/netmap/node.c
@@ -174,10 +174,9 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
 		  u32 bytes_to_copy =
 		    data_len > n_buffer_bytes ? n_buffer_bytes : data_len;
 		  b0->current_data = 0;
-		  clib_memcpy (vlib_buffer_get_current (b0),
-			       (u8 *) NETMAP_BUF (ring,
-						  slot->buf_idx) + offset,
-			       bytes_to_copy);
+		  clib_memcpy_fast (vlib_buffer_get_current (b0),
+				    (u8 *) NETMAP_BUF (ring, slot->buf_idx) +
+				    offset, bytes_to_copy);
 
 		  /* fill buffer header */
 		  b0->current_length = bytes_to_copy;
diff --git a/src/vnet/devices/pipe/pipe.c b/src/vnet/devices/pipe/pipe.c
index f45c08863a3..c6d3f659eae 100644
--- a/src/vnet/devices/pipe/pipe.c
+++ b/src/vnet/devices/pipe/pipe.c
@@ -148,7 +148,7 @@ pipe_tx (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
 
       n_copy = clib_min (n_left_from, n_left_to_next);
 
-      clib_memcpy (to_next, from, n_copy * sizeof (from[0]));
+      clib_memcpy_fast (to_next, from, n_copy * sizeof (from[0]));
       n_left_to_next -= n_copy;
       n_left_from -= n_copy;
       i = 0;
diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c
index 339c48c93f5..e0195139321 100644
--- a/src/vnet/devices/virtio/node.c
+++ b/src/vnet/devices/virtio/node.c
@@ -216,7 +216,7 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 	      tr->next_index = next0;
 	      tr->hw_if_index = vif->hw_if_index;
 	      tr->len = len;
-	      clib_memcpy (&tr->hdr, hdr, hdr_sz);
+	      clib_memcpy_fast (&tr->hdr, hdr, hdr_sz);
 	    }
 
 	  /* enqueue buffer */
diff --git a/src/vnet/devices/virtio/vhost_user.c b/src/vnet/devices/virtio/vhost_user.c
index 9725a91027d..d13ea3beab3 100644
--- a/src/vnet/devices/virtio/vhost_user.c
+++ b/src/vnet/devices/virtio/vhost_user.c
@@ -443,7 +443,7 @@ vhost_user_socket_read (clib_file_t * uf)
        VHOST_MEMORY_MAX_NREGIONS * sizeof (int)))
     {
       number_of_fds = (cmsg->cmsg_len - CMSG_LEN (0)) / sizeof (int);
-      clib_memcpy (fds, CMSG_DATA (cmsg), number_of_fds * sizeof (int));
+      clib_memcpy_fast (fds, CMSG_DATA (cmsg), number_of_fds * sizeof (int));
     }
 
   /* version 1, no reply bit set */
@@ -537,8 +537,8 @@ vhost_user_socket_read (clib_file_t * uf)
       unmap_all_mem_regions (vui);
       for (i = 0; i < msg.memory.nregions; i++)
 	{
-	  clib_memcpy (&(vui->regions[i]), &msg.memory.regions[i],
-		       sizeof (vhost_user_memory_region_t));
+	  clib_memcpy_fast (&(vui->regions[i]), &msg.memory.regions[i],
+			    sizeof (vhost_user_memory_region_t));
 
 	  long page_sz = get_huge_page_size (fds[i]);
 
diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c
index 8e88695957b..32b924f1362 100644
--- a/src/vnet/devices/virtio/vhost_user_input.c
+++ b/src/vnet/devices/virtio/vhost_user_input.c
@@ -162,8 +162,8 @@ vhost_user_input_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy,
 	  CLIB_PREFETCH (src2, 64, LOAD);
 	  CLIB_PREFETCH (src3, 64, LOAD);
 
-	  clib_memcpy ((void *) cpy[0].dst, src0, cpy[0].len);
-	  clib_memcpy ((void *) cpy[1].dst, src1, cpy[1].len);
+	  clib_memcpy_fast ((void *) cpy[0].dst, src0, cpy[0].len);
+	  clib_memcpy_fast ((void *) cpy[1].dst, src1, cpy[1].len);
 	  copy_len -= 2;
 	  cpy += 2;
 	}
@@ -172,7 +172,7 @@ vhost_user_input_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy,
     {
       if (PREDICT_FALSE (!(src0 = map_guest_mem (vui, cpy->src, map_hint))))
 	return 1;
-      clib_memcpy ((void *) cpy->dst, src0, cpy->len);
+      clib_memcpy_fast ((void *) cpy->dst, src0, cpy->len);
       copy_len -= 1;
       cpy += 1;
     }
diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c
index dff129e763e..58db1a9647b 100644
--- a/src/vnet/devices/virtio/vhost_user_output.c
+++ b/src/vnet/devices/virtio/vhost_user_output.c
@@ -205,8 +205,8 @@ vhost_user_tx_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy,
 	  CLIB_PREFETCH ((void *) cpy[2].src, 64, LOAD);
 	  CLIB_PREFETCH ((void *) cpy[3].src, 64, LOAD);
 
-	  clib_memcpy (dst0, (void *) cpy[0].src, cpy[0].len);
-	  clib_memcpy (dst1, (void *) cpy[1].src, cpy[1].len);
+	  clib_memcpy_fast (dst0, (void *) cpy[0].src, cpy[0].len);
+	  clib_memcpy_fast (dst1, (void *) cpy[1].src, cpy[1].len);
 
 	  vhost_user_log_dirty_pages_2 (vui, cpy[0].dst, cpy[0].len, 1);
 	  vhost_user_log_dirty_pages_2 (vui, cpy[1].dst, cpy[1].len, 1);
@@ -218,7 +218,7 @@ vhost_user_tx_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy,
     {
       if (PREDICT_FALSE (!(dst0 = map_guest_mem (vui, cpy->dst, map_hint))))
 	return 1;
-      clib_memcpy (dst0, (void *) cpy->src, cpy->len);
+      clib_memcpy_fast (dst0, (void *) cpy->src, cpy->len);
       vhost_user_log_dirty_pages_2 (vui, cpy->dst, cpy->len, 1);
       copy_len -= 1;
       cpy += 1;