diff options
| author |   Dave Barach <dave@barachs.net> | 2018-11-13 16:34:13 -0500 |
|---|---|---|
| committer |   Florin Coras <florin.coras@gmail.com> | 2018-11-14 15:54:01 +0000 |
| commit | 178cf493d009995b28fdf220f04c98860ff79a9b (patch) | |
| tree | 097c1be82b8f6fa9bc04b9b1e193158e2e4997eb /src/vnet | |
| parent | 6917b94f2146aa51195a6a2a1ccd8416a1d74bf3 (diff) | |
Remove c-11 memcpy checks from perf-critical code
Change-Id: Id4f37f5d4a03160572954a416efa1ef9b3d79ad1
Signed-off-by: Dave Barach <dave@barachs.net>
Diffstat (limited to 'src/vnet')
88 files changed, 717 insertions, 683 deletions
diff --git a/src/vnet/adj/rewrite.h b/src/vnet/adj/rewrite.h index 42b982e1bf4..58d47e0c737 100644 --- a/src/vnet/adj/rewrite.h +++ b/src/vnet/adj/rewrite.h @@ -131,7 +131,7 @@ vnet_rewrite_set_data_internal (vnet_rewrite_header_t * rw, ASSERT ((data_bytes >= 0) && (data_bytes < max_size)); rw->data_bytes = data_bytes; - clib_memcpy (rw->data + max_size - data_bytes, data, data_bytes); + clib_memcpy_fast (rw->data + max_size - data_bytes, data, data_bytes); clib_memset (rw->data, 0xfe, max_size - data_bytes); } diff --git a/src/vnet/bfd/bfd_api.c b/src/vnet/bfd/bfd_api.c index bb852256ad7..ea6730fb8b9 100644 --- a/src/vnet/bfd/bfd_api.c +++ b/src/vnet/bfd/bfd_api.c @@ -72,13 +72,13 @@ pub_sub_handler (bfd_events, BFD_EVENTS); clib_memset (&peer_addr, 0, sizeof (peer_addr)); \ if (mp->is_ipv6) \ { \ - clib_memcpy (&local_addr.ip6, mp->local_addr, sizeof (local_addr.ip6)); \ - clib_memcpy (&peer_addr.ip6, mp->peer_addr, sizeof (peer_addr.ip6)); \ + clib_memcpy_fast (&local_addr.ip6, mp->local_addr, sizeof (local_addr.ip6)); \ + clib_memcpy_fast (&peer_addr.ip6, mp->peer_addr, sizeof (peer_addr.ip6)); \ } \ else \ { \ - clib_memcpy (&local_addr.ip4, mp->local_addr, sizeof (local_addr.ip4)); \ - clib_memcpy (&peer_addr.ip4, mp->peer_addr, sizeof (peer_addr.ip4)); \ + clib_memcpy_fast (&local_addr.ip4, mp->local_addr, sizeof (local_addr.ip4)); \ + clib_memcpy_fast (&peer_addr.ip4, mp->peer_addr, sizeof (peer_addr.ip4)); \ } #define BFD_UDP_API_PARAM_FROM_MP(mp) \ @@ -176,16 +176,17 @@ send_bfd_udp_session_details (vl_api_registration_t * reg, u32 context, } if (mp->is_ipv6) { - clib_memcpy (mp->local_addr, &key->local_addr, - sizeof (key->local_addr)); - clib_memcpy (mp->peer_addr, &key->peer_addr, sizeof (key->peer_addr)); + clib_memcpy_fast (mp->local_addr, &key->local_addr, + sizeof (key->local_addr)); + clib_memcpy_fast (mp->peer_addr, &key->peer_addr, + sizeof (key->peer_addr)); } else { - clib_memcpy (mp->local_addr, key->local_addr.ip4.data, - sizeof (key->local_addr.ip4.data)); - clib_memcpy (mp->peer_addr, key->peer_addr.ip4.data, - sizeof (key->peer_addr.ip4.data)); + clib_memcpy_fast (mp->local_addr, key->local_addr.ip4.data, + sizeof (key->local_addr.ip4.data)); + clib_memcpy_fast (mp->peer_addr, key->peer_addr.ip4.data, + sizeof (key->peer_addr.ip4.data)); } mp->required_min_rx = @@ -387,7 +388,7 @@ vl_api_bfd_udp_get_echo_source_t_handler (vl_api_bfd_udp_get_echo_source_t * if (have_usable_ip4) { rmp->have_usable_ip4 = 1; - clib_memcpy (rmp->ip4_addr, &ip4, sizeof (ip4)); + clib_memcpy_fast (rmp->ip4_addr, &ip4, sizeof (ip4)); } else { @@ -396,7 +397,7 @@ vl_api_bfd_udp_get_echo_source_t_handler (vl_api_bfd_udp_get_echo_source_t * if (have_usable_ip6) { rmp->have_usable_ip6 = 1; - clib_memcpy (rmp->ip6_addr, &ip6, sizeof (ip6)); + clib_memcpy_fast (rmp->ip6_addr, &ip6, sizeof (ip6)); } else { diff --git a/src/vnet/bfd/bfd_udp.c b/src/vnet/bfd/bfd_udp.c index 0da19bdb630..d35c6dee5e1 100644 --- a/src/vnet/bfd/bfd_udp.c +++ b/src/vnet/bfd/bfd_udp.c @@ -330,17 +330,17 @@ bfd_add_udp6_transport (vlib_main_t * vm, u32 bi, const bfd_session_t * bs, { return rv; } - clib_memcpy (&headers->ip6.dst_address, &key->local_addr.ip6, - sizeof (headers->ip6.dst_address)); + clib_memcpy_fast (&headers->ip6.dst_address, &key->local_addr.ip6, + sizeof (headers->ip6.dst_address)); headers->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_bfd_echo6); } else { - clib_memcpy (&headers->ip6.src_address, &key->local_addr.ip6, - sizeof (headers->ip6.src_address)); - clib_memcpy (&headers->ip6.dst_address, &key->peer_addr.ip6, - sizeof (headers->ip6.dst_address)); + clib_memcpy_fast (&headers->ip6.src_address, &key->local_addr.ip6, + sizeof (headers->ip6.src_address)); + clib_memcpy_fast (&headers->ip6.dst_address, &key->peer_addr.ip6, + sizeof (headers->ip6.dst_address)); headers->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_bfd6); } @@ -1211,7 +1211,7 @@ bfd_udp_input (vlib_main_t * vm, vlib_node_runtime_t * rt, len = (b0->current_length < sizeof (t0->data)) ? b0->current_length : sizeof (t0->data); t0->len = len; - clib_memcpy (t0->data, vlib_buffer_get_current (b0), len); + clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0), len); } /* scan this bfd pkt. error0 is the counter index to bmp */ @@ -1369,7 +1369,7 @@ bfd_udp_echo_input (vlib_main_t * vm, vlib_node_runtime_t * rt, len = (b0->current_length < sizeof (t0->data)) ? b0->current_length : sizeof (t0->data); t0->len = len; - clib_memcpy (t0->data, vlib_buffer_get_current (b0), len); + clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0), len); } bfd_lock (bm); diff --git a/src/vnet/bier/bier_api.c b/src/vnet/bier/bier_api.c index b4bde3a97eb..ed1e35f5b24 100644 --- a/src/vnet/bier/bier_api.c +++ b/src/vnet/bier/bier_api.c @@ -505,13 +505,13 @@ vl_api_bier_disp_entry_add_del_t_handler (vl_api_bier_disp_entry_add_del_t * mp) if (0 == mp->bde_paths[ii].afi) { - clib_memcpy (&brp->frp_addr.ip4, + clib_memcpy_fast (&brp->frp_addr.ip4, mp->bde_paths[ii].next_hop, sizeof (brp->frp_addr.ip4)); } else { - clib_memcpy (&brp->frp_addr.ip6, + clib_memcpy_fast (&brp->frp_addr.ip6, mp->bde_paths[ii].next_hop, sizeof (brp->frp_addr.ip6)); } diff --git a/src/vnet/bier/bier_imp.c b/src/vnet/bier/bier_imp.c index 87a8f909a85..6150ea1c067 100644 --- a/src/vnet/bier/bier_imp.c +++ b/src/vnet/bier/bier_imp.c @@ -80,7 +80,7 @@ bier_imp_add_or_lock (const bier_table_id_t *bti, 0, // entropy sender); bier_hdr_hton(&bi->bi_hdr); - clib_memcpy(&bi->bi_bits, bs->bbs_buckets, bs->bbs_len); + clib_memcpy_fast(&bi->bi_bits, bs->bbs_buckets, bs->bbs_len); bier_imp_lock_i(bi); diff --git a/src/vnet/bier/bier_imp_node.c b/src/vnet/bier/bier_imp_node.c index 0f1820b4028..dcfcc1a1982 100644 --- a/src/vnet/bier/bier_imp_node.c +++ b/src/vnet/bier/bier_imp_node.c @@ -128,7 +128,7 @@ bier_imp_dpo_inline (vlib_main_t * vm, } else { - clib_memcpy(hdr0, &bimp0->bi_hdr, + clib_memcpy_fast(hdr0, &bimp0->bi_hdr, (sizeof(bier_hdr_t) + bier_hdr_len_id_to_num_bytes(bimp0->bi_tbl.bti_hdr_len))); /* diff --git a/src/vnet/bonding/device.c b/src/vnet/bonding/device.c index e7d6cdad407..2bbb90d20f9 100644 --- a/src/vnet/bonding/device.c +++ b/src/vnet/bonding/device.c @@ -745,8 +745,8 @@ done: f = vnet_get_frame_to_sw_interface (vnm, sw_if_index); f->n_vectors = ptd->per_port_queue[p].n_buffers; to_next = vlib_frame_vector_args (f); - clib_memcpy (to_next, ptd->per_port_queue[p].buffers, - f->n_vectors * sizeof (u32)); + clib_memcpy_fast (to_next, ptd->per_port_queue[p].buffers, + f->n_vectors * sizeof (u32)); vnet_put_frame_to_sw_interface (vnm, sw_if_index, f); ptd->per_port_queue[p].n_buffers = 0; } diff --git a/src/vnet/bonding/node.c b/src/vnet/bonding/node.c index 361509c549d..2bc2dc6f265 100644 --- a/src/vnet/bonding/node.c +++ b/src/vnet/bonding/node.c @@ -334,8 +334,8 @@ VLIB_NODE_FN (bond_input_node) (vlib_main_t * vm, { t0 = vlib_add_trace (vm, node, b[0], sizeof (*t0)); t0->sw_if_index = sw_if_index[0]; - clib_memcpy (&t0->ethernet, vlib_buffer_get_current (b[0]), - sizeof (ethernet_header_t)); + clib_memcpy_fast (&t0->ethernet, vlib_buffer_get_current (b[0]), + sizeof (ethernet_header_t)); t0->bond_sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX]; } /* next */ diff --git a/src/vnet/classify/vnet_classify.c b/src/vnet/classify/vnet_classify.c index 36f967a38ac..4e822226aaa 100644 --- a/src/vnet/classify/vnet_classify.c +++ b/src/vnet/classify/vnet_classify.c @@ -131,7 +131,7 @@ vnet_classify_new_table (vnet_classify_main_t * cm, clib_memset (t, 0, sizeof (*t)); vec_validate_aligned (t->mask, match_n_vectors - 1, sizeof (u32x4)); - clib_memcpy (t->mask, mask, match_n_vectors * sizeof (u32x4)); + clib_memcpy_fast (t->mask, mask, match_n_vectors * sizeof (u32x4)); t->next_table_index = ~0; t->nbuckets = nbuckets; @@ -273,7 +273,7 @@ static inline void make_working_copy v = vnet_classify_get_entry (t, b->offset); - clib_memcpy (working_copy, v, required_length); + clib_memcpy_fast (working_copy, v, required_length); working_bucket.as_u64 = b->as_u64; working_bucket.offset = vnet_classify_get_offset (t, working_copy); @@ -317,8 +317,8 @@ split_and_rehash (vnet_classify_table_t * t, if (vnet_classify_entry_is_free (new_v)) { - clib_memcpy (new_v, v, sizeof (vnet_classify_entry_t) - + (t->match_n_vectors * sizeof (u32x4))); + clib_memcpy_fast (new_v, v, sizeof (vnet_classify_entry_t) + + (t->match_n_vectors * sizeof (u32x4))); new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE); goto doublebreak; } @@ -361,8 +361,8 @@ split_and_rehash_linear (vnet_classify_table_t * t, clib_warning ("BUG: linear rehash new entry not free!"); continue; } - clib_memcpy (new_v, v, sizeof (vnet_classify_entry_t) - + (t->match_n_vectors * sizeof (u32x4))); + clib_memcpy_fast (new_v, v, sizeof (vnet_classify_entry_t) + + (t->match_n_vectors * sizeof (u32x4))); new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE); j++; goto doublebreak; @@ -457,8 +457,8 @@ vnet_classify_add_del (vnet_classify_table_t * t, } v = vnet_classify_entry_alloc (t, 0 /* new_log2_pages */ ); - clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) + - t->match_n_vectors * sizeof (u32x4)); + clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) + + t->match_n_vectors * sizeof (u32x4)); v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE); vnet_classify_entry_claim_resource (v); @@ -496,8 +496,8 @@ vnet_classify_add_del (vnet_classify_table_t * t, if (!memcmp (v->key, add_v->key, t->match_n_vectors * sizeof (u32x4))) { - clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) + - t->match_n_vectors * sizeof (u32x4)); + clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) + + t->match_n_vectors * sizeof (u32x4)); v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE); vnet_classify_entry_claim_resource (v); @@ -513,8 +513,8 @@ vnet_classify_add_del (vnet_classify_table_t * t, if (vnet_classify_entry_is_free (v)) { - clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) + - t->match_n_vectors * sizeof (u32x4)); + clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) + + t->match_n_vectors * sizeof (u32x4)); v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE); vnet_classify_entry_claim_resource (v); @@ -609,8 +609,8 @@ vnet_classify_add_del (vnet_classify_table_t * t, if (vnet_classify_entry_is_free (new_v)) { - clib_memcpy (new_v, add_v, sizeof (vnet_classify_entry_t) + - t->match_n_vectors * sizeof (u32x4)); + clib_memcpy_fast (new_v, add_v, sizeof (vnet_classify_entry_t) + + t->match_n_vectors * sizeof (u32x4)); new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE); vnet_classify_entry_claim_resource (new_v); @@ -1885,10 +1885,10 @@ unformat_ip6_match (unformat_input_t * input, va_list * args) ip = (ip6_header_t *) match; if (src) - clib_memcpy (&ip->src_address, &src_val, sizeof (ip->src_address)); + clib_memcpy_fast (&ip->src_address, &src_val, sizeof (ip->src_address)); if (dst) - clib_memcpy (&ip->dst_address, &dst_val, sizeof (ip->dst_address)); + clib_memcpy_fast (&ip->dst_address, &dst_val, sizeof (ip->dst_address)); if (proto) ip->protocol = proto_val; @@ -2012,10 +2012,10 @@ unformat_l2_match (unformat_input_t * input, va_list * args) vec_validate_aligned (match, len - 1, sizeof (u32x4)); if (dst) - clib_memcpy (match, dst_val, 6); + clib_memcpy_fast (match, dst_val, 6); if (src) - clib_memcpy (match + 6, src_val, 6); + clib_memcpy_fast (match + 6, src_val, 6); if (tag2) { @@ -2188,8 +2188,8 @@ vnet_classify_add_del_session (vnet_classify_main_t * cm, e->metadata = 0; /* Copy key data, honoring skip_n_vectors */ - clib_memcpy (&e->key, match + t->skip_n_vectors * sizeof (u32x4), - t->match_n_vectors * sizeof (u32x4)); + clib_memcpy_fast (&e->key, match + t->skip_n_vectors * sizeof (u32x4), + t->match_n_vectors * sizeof (u32x4)); /* Clear don't-care bits; likely when dynamically creating sessions */ for (i = 0; i < t->match_n_vectors; i++) diff --git a/src/vnet/devices/af_packet/device.c b/src/vnet/devices/af_packet/device.c index 811642ec7b9..9d08c620d94 100644 --- a/src/vnet/devices/af_packet/device.c +++ b/src/vnet/devices/af_packet/device.c @@ -119,9 +119,9 @@ af_packet_interface_tx (vlib_main_t * vm, { b0 = vlib_get_buffer (vm, bi); len = b0->current_length; - clib_memcpy ((u8 *) tph + - TPACKET_ALIGN (sizeof (struct tpacket2_hdr)) + offset, - vlib_buffer_get_current (b0), len); + clib_memcpy_fast ((u8 *) tph + + TPACKET_ALIGN (sizeof (struct tpacket2_hdr)) + + offset, vlib_buffer_get_current (b0), len); offset += len; } while ((bi = diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c index 8dbd8554f01..97aa170c0a6 100644 --- a/src/vnet/devices/af_packet/node.c +++ b/src/vnet/devices/af_packet/node.c @@ -248,9 +248,9 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, { if (PREDICT_TRUE (offset == 0)) { - clib_memcpy (vlib_buffer_get_current (b0), - (u8 *) tph + tph->tp_mac, - sizeof (ethernet_header_t)); + clib_memcpy_fast (vlib_buffer_get_current (b0), + (u8 *) tph + tph->tp_mac, + sizeof (ethernet_header_t)); ethernet_header_t *eth = vlib_buffer_get_current (b0); ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eth + 1); @@ -262,10 +262,10 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, bytes_copied = sizeof (ethernet_header_t); } } - clib_memcpy (((u8 *) vlib_buffer_get_current (b0)) + - bytes_copied + vlan_len, - (u8 *) tph + tph->tp_mac + offset + bytes_copied, - (bytes_to_copy - bytes_copied)); + clib_memcpy_fast (((u8 *) vlib_buffer_get_current (b0)) + + bytes_copied + vlan_len, + (u8 *) tph + tph->tp_mac + offset + + bytes_copied, (bytes_to_copy - bytes_copied)); /* fill buffer header */ b0->current_length = bytes_to_copy + vlan_len; @@ -319,7 +319,7 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, tr = vlib_add_trace (vm, node, first_b0, sizeof (*tr)); tr->next_index = next0; tr->hw_if_index = apif->hw_if_index; - clib_memcpy (&tr->tph, tph, sizeof (struct tpacket2_hdr)); + clib_memcpy_fast (&tr->tph, tph, sizeof (struct tpacket2_hdr)); } /* enque and take next packet */ diff --git a/src/vnet/devices/netmap/device.c b/src/vnet/devices/netmap/device.c index 409a52f1005..b14bff43c06 100644 --- a/src/vnet/devices/netmap/device.c +++ b/src/vnet/devices/netmap/device.c @@ -143,8 +143,8 @@ netmap_interface_tx (vlib_main_t * vm, b0 = vlib_get_buffer (vm, bi); len = b0->current_length; /* memcpy */ - clib_memcpy ((u8 *) NETMAP_BUF (ring, slot->buf_idx) + offset, - vlib_buffer_get_current (b0), len); + clib_memcpy_fast ((u8 *) NETMAP_BUF (ring, slot->buf_idx) + + offset, vlib_buffer_get_current (b0), len); offset += len; } while ((bi = b0->next_buffer)); diff --git a/src/vnet/devices/netmap/node.c b/src/vnet/devices/netmap/node.c index e541700f4ea..dde706f20c7 100644 --- a/src/vnet/devices/netmap/node.c +++ b/src/vnet/devices/netmap/node.c @@ -174,10 +174,9 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, u32 bytes_to_copy = data_len > n_buffer_bytes ? n_buffer_bytes : data_len; b0->current_data = 0; - clib_memcpy (vlib_buffer_get_current (b0), - (u8 *) NETMAP_BUF (ring, - slot->buf_idx) + offset, - bytes_to_copy); + clib_memcpy_fast (vlib_buffer_get_current (b0), + (u8 *) NETMAP_BUF (ring, slot->buf_idx) + + offset, bytes_to_copy); /* fill buffer header */ b0->current_length = bytes_to_copy; diff --git a/src/vnet/devices/pipe/pipe.c b/src/vnet/devices/pipe/pipe.c index f45c08863a3..c6d3f659eae 100644 --- a/src/vnet/devices/pipe/pipe.c +++ b/src/vnet/devices/pipe/pipe.c @@ -148,7 +148,7 @@ pipe_tx (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) n_copy = clib_min (n_left_from, n_left_to_next); - clib_memcpy (to_next, from, n_copy * sizeof (from[0])); + clib_memcpy_fast (to_next, from, n_copy * sizeof (from[0])); n_left_to_next -= n_copy; n_left_from -= n_copy; i = 0; diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c index 339c48c93f5..e0195139321 100644 --- a/src/vnet/devices/virtio/node.c +++ b/src/vnet/devices/virtio/node.c @@ -216,7 +216,7 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tr->next_index = next0; tr->hw_if_index = vif->hw_if_index; tr->len = len; - clib_memcpy (&tr->hdr, hdr, hdr_sz); + clib_memcpy_fast (&tr->hdr, hdr, hdr_sz); } /* enqueue buffer */ diff --git a/src/vnet/devices/virtio/vhost_user.c b/src/vnet/devices/virtio/vhost_user.c index 9725a91027d..d13ea3beab3 100644 --- a/src/vnet/devices/virtio/vhost_user.c +++ b/src/vnet/devices/virtio/vhost_user.c @@ -443,7 +443,7 @@ vhost_user_socket_read (clib_file_t * uf) VHOST_MEMORY_MAX_NREGIONS * sizeof (int))) { number_of_fds = (cmsg->cmsg_len - CMSG_LEN (0)) / sizeof (int); - clib_memcpy (fds, CMSG_DATA (cmsg), number_of_fds * sizeof (int)); + clib_memcpy_fast (fds, CMSG_DATA (cmsg), number_of_fds * sizeof (int)); } /* version 1, no reply bit set */ @@ -537,8 +537,8 @@ vhost_user_socket_read (clib_file_t * uf) unmap_all_mem_regions (vui); for (i = 0; i < msg.memory.nregions; i++) { - clib_memcpy (&(vui->regions[i]), &msg.memory.regions[i], - sizeof (vhost_user_memory_region_t)); + clib_memcpy_fast (&(vui->regions[i]), &msg.memory.regions[i], + sizeof (vhost_user_memory_region_t)); long page_sz = get_huge_page_size (fds[i]); diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c index 8e88695957b..32b924f1362 100644 --- a/src/vnet/devices/virtio/vhost_user_input.c +++ b/src/vnet/devices/virtio/vhost_user_input.c @@ -162,8 +162,8 @@ vhost_user_input_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, CLIB_PREFETCH (src2, 64, LOAD); CLIB_PREFETCH (src3, 64, LOAD); - clib_memcpy ((void *) cpy[0].dst, src0, cpy[0].len); - clib_memcpy ((void *) cpy[1].dst, src1, cpy[1].len); + clib_memcpy_fast ((void *) cpy[0].dst, src0, cpy[0].len); + clib_memcpy_fast ((void *) cpy[1].dst, src1, cpy[1].len); copy_len -= 2; cpy += 2; } @@ -172,7 +172,7 @@ vhost_user_input_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, { if (PREDICT_FALSE (!(src0 = map_guest_mem (vui, cpy->src, map_hint)))) return 1; - clib_memcpy ((void *) cpy->dst, src0, cpy->len); + clib_memcpy_fast ((void *) cpy->dst, src0, cpy->len); copy_len -= 1; cpy += 1; } diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c index dff129e763e..58db1a9647b 100644 --- a/src/vnet/devices/virtio/vhost_user_output.c +++ b/src/vnet/devices/virtio/vhost_user_output.c @@ -205,8 +205,8 @@ vhost_user_tx_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, CLIB_PREFETCH ((void *) cpy[2].src, 64, LOAD); CLIB_PREFETCH ((void *) cpy[3].src, 64, LOAD); - clib_memcpy (dst0, (void *) cpy[0].src, cpy[0].len); - clib_memcpy (dst1, (void *) cpy[1].src, cpy[1].len); + clib_memcpy_fast (dst0, (void *) cpy[0].src, cpy[0].len); + clib_memcpy_fast (dst1, (void *) cpy[1].src, cpy[1].len); vhost_user_log_dirty_pages_2 (vui, cpy[0].dst, cpy[0].len, 1); vhost_user_log_dirty_pages_2 (vui, cpy[1].dst, cpy[1].len, 1); @@ -218,7 +218,7 @@ vhost_user_tx_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, { if (PREDICT_FALSE (!(dst0 = map_guest_mem (vui, cpy->dst, map_hint)))) return 1; - clib_memcpy (dst0, (void *) cpy->src, cpy->len); + clib_memcpy_fast (dst0, (void *) cpy->src, cpy->len); vhost_user_log_dirty_pages_2 (vui, cpy->dst, cpy->len, 1); copy_len -= 1; cpy += 1; diff --git a/src/vnet/dns/reply_node.c b/src/vnet/dns/reply_node.c index 5681e11d8e2..81ce8f480d0 100644 --- a/src/vnet/dns/reply_node.c +++ b/src/vnet/dns/reply_node.c @@ -166,7 +166,7 @@ dns46_reply_node_fn (vlib_main_t * vm, /* Save the reply */ vec_validate (resp0, vlib_buffer_length_in_chain (vm, b0) - 1); - clib_memcpy (resp0, d0, vlib_buffer_length_in_chain (vm, b0)); + clib_memcpy_fast (resp0, d0, vlib_buffer_length_in_chain (vm, b0)); /* * Deal with everything in process ctx on the main thread diff --git a/src/vnet/dns/request_node.c b/src/vnet/dns/request_node.c index b91d2997dfb..f9ad7cc0ad8 100644 --- a/src/vnet/dns/request_node.c +++ b/src/vnet/dns/request_node.c @@ -236,11 +236,11 @@ dns46_request_inline (vlib_main_t * vm, t0->id = d0->id; t0->name = name0; if (is_ip6) - clib_memcpy (t0->dst_address, ip60->src_address.as_u8, - sizeof (ip6_address_t)); + clib_memcpy_fast (t0->dst_address, ip60->src_address.as_u8, + sizeof (ip6_address_t)); else - clib_memcpy (t0->dst_address, ip40->src_address.as_u8, - sizeof (ip4_address_t)); + clib_memcpy_fast (t0->dst_address, ip40->src_address.as_u8, + sizeof (ip4_address_t)); vnet_dns_resolve_name (dm, name0, t0, &ep0); diff --git a/src/vnet/dpo/load_balance.c b/src/vnet/dpo/load_balance.c index 1abfab1cb01..939e25c08fa 100644 --- a/src/vnet/dpo/load_balance.c +++ b/src/vnet/dpo/load_balance.c @@ -383,7 +383,7 @@ ip_multipath_normalize_next_hops (const load_balance_path_t * raw_next_hops, } else { - clib_memcpy (nhs, raw_next_hops, n_nhs * sizeof (raw_next_hops[0])); + clib_memcpy_fast (nhs, raw_next_hops, n_nhs * sizeof (raw_next_hops[0])); qsort (nhs, n_nhs, sizeof (nhs[0]), (void *) next_hop_sort_by_weight); } diff --git a/src/vnet/dpo/mpls_label_dpo.c b/src/vnet/dpo/mpls_label_dpo.c index c3710aefe28..ab8c06df4d5 100644 --- a/src/vnet/dpo/mpls_label_dpo.c +++ b/src/vnet/dpo/mpls_label_dpo.c @@ -287,7 +287,7 @@ mpls_label_paint (vlib_buffer_t * b0, } else { - clib_memcpy(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes); + clib_memcpy_fast(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes); hdr0 = hdr0 + (mld0->mld_n_labels - 1); } @@ -1218,7 +1218,7 @@ mpls_label_interpose (const dpo_id_t *original, mld = mpls_label_dpo_get(original->dpoi_index); mld_clone->mld_locks = 0; - clib_memcpy(&mld_clone->mld_hdr, + clib_memcpy_fast(&mld_clone->mld_hdr, &mld->mld_hdr, sizeof(mld_clone->mld_hdr)); mld_clone->mld_payload_proto = mld->mld_payload_proto; diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c index b6b7aba731d..549e05d889c 100644 --- a/src/vnet/ethernet/arp.c +++ b/src/vnet/ethernet/arp.c @@ -358,8 +358,9 @@ arp_nbr_probe (ip_adjacency_t * adj) hi = vnet_get_sup_hw_interface (vnm, adj->rewrite_header.sw_if_index); - clib_memcpy (h->ip4_over_ethernet[0].ethernet, - hi->hw_address, sizeof (h->ip4_over_ethernet[0].ethernet)); + clib_memcpy_fast (h->ip4_over_ethernet[0].ethernet, + hi->hw_address, + sizeof (h->ip4_over_ethernet[0].ethernet)); h->ip4_over_ethernet[0].ip4 = src[0]; h->ip4_over_ethernet[1].ip4 = adj->sub_type.nbr.next_hop.ip4; @@ -674,8 +675,8 @@ vnet_arp_set_ip4_over_ethernet_internal (vnet_main_t * vnm, e->sw_if_index = sw_if_index; e->ip4_address = a->ip4; e->fib_entry_index = FIB_NODE_INDEX_INVALID; - clib_memcpy (e->ethernet_address, - a->ethernet, sizeof (e->ethernet_address)); + clib_memcpy_fast (e->ethernet_address, + a->ethernet, sizeof (e->ethernet_address)); if (!is_no_fib_entry) { @@ -702,8 +703,8 @@ vnet_arp_set_ip4_over_ethernet_internal (vnet_main_t * vnm, } /* Update ethernet address. */ - clib_memcpy (e->ethernet_address, a->ethernet, - sizeof (e->ethernet_address)); + clib_memcpy_fast (e->ethernet_address, a->ethernet, + sizeof (e->ethernet_address)); } /* Update time stamp and flags. */ @@ -1238,8 +1239,8 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) arp0->ip4_over_ethernet[1] = arp0->ip4_over_ethernet[0]; - clib_memcpy (arp0->ip4_over_ethernet[0].ethernet, - hw_if0->hw_address, 6); + clib_memcpy_fast (arp0->ip4_over_ethernet[0].ethernet, + hw_if0->hw_address, 6); clib_mem_unaligned (&arp0->ip4_over_ethernet[0].ip4.data_u32, u32) = if_addr0->data_u32; @@ -1249,7 +1250,7 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) /* the rx nd tx ethernet headers wil overlap in the case * when we received a tagged VLAN=0 packet, but we are sending * back untagged */ - clib_memcpy (eth_tx, rewrite0, vec_len (rewrite0)); + clib_memcpy_fast (eth_tx, rewrite0, vec_len (rewrite0)); vec_free (rewrite0); if (NULL == pa) @@ -1557,7 +1558,7 @@ vnet_arp_unset_ip4_over_ethernet (vnet_main_t * vnm, args.sw_if_index = sw_if_index; args.flags = ETHERNET_ARP_ARGS_REMOVE; - clib_memcpy (&args.a, a, sizeof (*a)); + clib_memcpy_fast (&args.a, a, sizeof (*a)); vl_api_rpc_call_main_thread (set_ip4_over_ethernet_rpc_callback, (u8 *) & args, sizeof (args)); @@ -1706,7 +1707,7 @@ arp_add_del_interface_address (ip4_main_t * im, .sw_if_index = e->sw_if_index, .flags = ETHERNET_ARP_ARGS_FLUSH, }; - clib_memcpy (&delme.a.ethernet, e->ethernet_address, 6); + clib_memcpy_fast (&delme.a.ethernet, e->ethernet_address, 6); vnet_arp_flush_ip4_over_ethernet_internal (vnet_get_main (), &delme); @@ -1915,7 +1916,7 @@ ethernet_arp_sw_interface_up_down (vnet_main_t * vnm, .sw_if_index = e->sw_if_index, }; - clib_memcpy (&update_me.a.ethernet, e->ethernet_address, 6); + clib_memcpy_fast (&update_me.a.ethernet, e->ethernet_address, 6); if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) { @@ -1970,7 +1971,7 @@ vnet_arp_set_ip4_over_ethernet (vnet_main_t * vnm, args.is_static = is_static; args.is_no_fib_entry = is_no_fib_entry; args.flags = 0; - clib_memcpy (&args.a, a, sizeof (*a)); + clib_memcpy_fast (&args.a, a, sizeof (*a)); vl_api_rpc_call_main_thread (set_ip4_over_ethernet_rpc_callback, (u8 *) & args, sizeof (args)); @@ -2344,7 +2345,8 @@ arp_term_l2bd (vlib_main_t * vm, { u8 *t0 = vlib_add_trace (vm, node, p0, sizeof (ethernet_arp_input_trace_t)); - clib_memcpy (t0, l3h0, sizeof (ethernet_arp_input_trace_t)); + clib_memcpy_fast (t0, l3h0, + sizeof (ethernet_arp_input_trace_t)); } error0 = 0; @@ -2410,9 +2412,9 @@ arp_term_l2bd (vlib_main_t * vm, arp0->opcode = clib_host_to_net_u16 (ETHERNET_ARP_OPCODE_reply); arp0->ip4_over_ethernet[1] = arp0->ip4_over_ethernet[0]; arp0->ip4_over_ethernet[0].ip4.as_u32 = ip0; - clib_memcpy (arp0->ip4_over_ethernet[0].ethernet, macp0, 6); - clib_memcpy (eth0->dst_address, eth0->src_address, 6); - clib_memcpy (eth0->src_address, macp0, 6); + clib_memcpy_fast (arp0->ip4_over_ethernet[0].ethernet, macp0, 6); + clib_memcpy_fast (eth0->dst_address, eth0->src_address, 6); + clib_memcpy_fast (eth0->src_address, macp0, 6); n_replies_sent += 1; output_response: @@ -2577,10 +2579,10 @@ send_ip4_garp_w_addr (vlib_main_t * vm, if (!h) return; - clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address, - sizeof (h->ip4_over_ethernet[0].ethernet)); - clib_memcpy (h->ip4_over_ethernet[1].ethernet, hi->hw_address, - sizeof (h->ip4_over_ethernet[1].ethernet)); + clib_memcpy_fast (h->ip4_over_ethernet[0].ethernet, hi->hw_address, + sizeof (h->ip4_over_ethernet[0].ethernet)); + clib_memcpy_fast (h->ip4_over_ethernet[1].ethernet, hi->hw_address, + sizeof (h->ip4_over_ethernet[1].ethernet)); h->ip4_over_ethernet[0].ip4 = ip4_addr[0]; h->ip4_over_ethernet[1].ip4 = ip4_addr[0]; @@ -2592,7 +2594,7 @@ send_ip4_garp_w_addr (vlib_main_t * vm, rewrite_len = vec_len (rewrite); vlib_buffer_advance (b, -rewrite_len); ethernet_header_t *e = vlib_buffer_get_current (b); - clib_memcpy (e->dst_address, rewrite, rewrite_len); + clib_memcpy_fast (e->dst_address, rewrite, rewrite_len); vec_free (rewrite); /* Send GARP packet out the specified interface */ diff --git a/src/vnet/ethernet/ethernet_types_api.c b/src/vnet/ethernet/ethernet_types_api.c index 67e6231cfc8..4b84d386e4e 100644 --- a/src/vnet/ethernet/ethernet_types_api.c +++ b/src/vnet/ethernet/ethernet_types_api.c @@ -38,7 +38,7 @@ mac_address_decode (const vl_api_mac_address_t * in, mac_address_t * out) void mac_address_encode (const mac_address_t * in, vl_api_mac_address_t * out) { - clib_memcpy (out->bytes, in->bytes, 6); + clib_memcpy_fast (out->bytes, in->bytes, 6); } /* diff --git a/src/vnet/flow/flow.c b/src/vnet/flow/flow.c index 485a845e5ee..4d867fee365 100644 --- a/src/vnet/flow/flow.c +++ b/src/vnet/flow/flow.c @@ -47,7 +47,7 @@ vnet_flow_add (vnet_main_t * vnm, vnet_flow_t * flow, u32 * flow_index) pool_get (fm->global_flow_pool, f); *flow_index = f - fm->global_flow_pool; - clib_memcpy (f, flow, sizeof (vnet_flow_t)); + clib_memcpy_fast (f, flow, sizeof (vnet_flow_t)); f->private_data = 0; f->index = *flow_index; return 0; diff --git a/src/vnet/geneve/decap.c b/src/vnet/geneve/decap.c index 93f2a8548c1..ceb04447817 100644 --- a/src/vnet/geneve/decap.c +++ b/src/vnet/geneve/decap.c @@ -286,7 +286,7 @@ geneve_input (vlib_main_t * vm, next0 = GENEVE_INPUT_NEXT_DROP; goto trace0; } - clib_memcpy (&last_key6, &key6_0, sizeof (key6_0)); + clib_memcpy_fast (&last_key6, &key6_0, sizeof (key6_0)); tunnel_index0 = last_tunnel_index = p0[0]; } else @@ -455,7 +455,7 @@ geneve_input (vlib_main_t * vm, goto trace1; } - clib_memcpy (&last_key6, &key6_1, sizeof (key6_1)); + clib_memcpy_fast (&last_key6, &key6_1, sizeof (key6_1)); tunnel_index1 = last_tunnel_index = p1[0]; } else @@ -690,7 +690,7 @@ geneve_input (vlib_main_t * vm, next0 = GENEVE_INPUT_NEXT_DROP; goto trace00; } - clib_memcpy (&last_key6, &key6_0, sizeof (key6_0)); + clib_memcpy_fast (&last_key6, &key6_0, sizeof (key6_0)); tunnel_index0 = last_tunnel_index = p0[0]; } else diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c index be4ab7d291a..b09f7996e18 100644 --- a/src/vnet/interface_output.c +++ b/src/vnet/interface_output.c @@ -121,15 +121,15 @@ vnet_interface_output_trace (vlib_main_t * vm, { t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0])); t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX]; - clib_memcpy (t0->data, vlib_buffer_get_current (b0), - sizeof (t0->data)); + clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0), + sizeof (t0->data)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0])); t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_TX]; - clib_memcpy (t1->data, vlib_buffer_get_current (b1), - sizeof (t1->data)); + clib_memcpy_fast (t1->data, vlib_buffer_get_current (b1), + sizeof (t1->data)); } from += 2; n_left -= 2; @@ -149,8 +149,8 @@ vnet_interface_output_trace (vlib_main_t * vm, { t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0])); t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX]; - clib_memcpy (t0->data, vlib_buffer_get_current (b0), - sizeof (t0->data)); + clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0), + sizeof (t0->data)); } from += 1; n_left -= 1; diff --git a/src/vnet/ip/ip.c b/src/vnet/ip/ip.c index 144bbcc405d..cab0cd0315a 100644 --- a/src/vnet/ip/ip.c +++ b/src/vnet/ip/ip.c @@ -70,7 +70,7 @@ ip_is_local (u32 fib_index, ip46_address_t * ip46_address, u8 is_ip4) prefix.fp_proto = FIB_PROTOCOL_IP6; } - clib_memcpy (&prefix.fp_addr, ip46_address, sizeof (ip46_address_t)); + clib_memcpy_fast (&prefix.fp_addr, ip46_address, sizeof (ip46_address_t)); fei = fib_table_lookup (fib_index, &prefix); flags = fib_entry_get_flags (fei); @@ -83,7 +83,7 @@ ip_copy (ip46_address_t * dst, ip46_address_t * src, u8 is_ip4) if (is_ip4) dst->ip4.as_u32 = src->ip4.as_u32; else - clib_memcpy (&dst->ip6, &src->ip6, sizeof (ip6_address_t)); + clib_memcpy_fast (&dst->ip6, &src->ip6, sizeof (ip6_address_t)); } void @@ -92,7 +92,8 @@ ip_set (ip46_address_t * dst, void *src, u8 is_ip4) if (is_ip4) dst->ip4.as_u32 = ((ip4_address_t *) src)->as_u32; else - clib_memcpy (&dst->ip6, (ip6_address_t *) src, sizeof (ip6_address_t)); + clib_memcpy_fast (&dst->ip6, (ip6_address_t *) src, + sizeof (ip6_address_t)); } u8 diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c index 9911eb71b99..e011e5134d8 100644 --- a/src/vnet/ip/ip4_forward.c +++ b/src/vnet/ip/ip4_forward.c @@ -1072,9 +1072,9 @@ ip4_forward_next_trace (vlib_main_t * vm, vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b0)->sw_if_index[VLIB_RX]); - clib_memcpy (t0->packet_data, - vlib_buffer_get_current (b0), - sizeof (t0->packet_data)); + clib_memcpy_fast (t0->packet_data, + vlib_buffer_get_current (b0), + sizeof (t0->packet_data)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -1086,8 +1086,8 @@ ip4_forward_next_trace (vlib_main_t * vm, (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] : vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b1)->sw_if_index[VLIB_RX]); - clib_memcpy (t1->packet_data, vlib_buffer_get_current (b1), - sizeof (t1->packet_data)); + clib_memcpy_fast (t1->packet_data, vlib_buffer_get_current (b1), + sizeof (t1->packet_data)); } from += 2; n_left -= 2; @@ -1113,8 +1113,8 @@ ip4_forward_next_trace (vlib_main_t * vm, (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] : vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b0)->sw_if_index[VLIB_RX]); - clib_memcpy (t0->packet_data, vlib_buffer_get_current (b0), - sizeof (t0->packet_data)); + clib_memcpy_fast (t0->packet_data, vlib_buffer_get_current (b0), + sizeof (t0->packet_data)); } from += 1; n_left -= 1; @@ -1852,9 +1852,9 @@ ip4_arp_inline (vlib_main_t * vm, hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); /* Src ethernet address in ARP header. */ - clib_memcpy (h0->ip4_over_ethernet[0].ethernet, - hw_if0->hw_address, - sizeof (h0->ip4_over_ethernet[0].ethernet)); + clib_memcpy_fast (h0->ip4_over_ethernet[0].ethernet, + hw_if0->hw_address, + sizeof (h0->ip4_over_ethernet[0].ethernet)); if (is_glean) { /* The interface's source address is stashed in the Glean Adj */ @@ -2027,8 +2027,8 @@ ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index, sw_if_index); } - clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address, - sizeof (h->ip4_over_ethernet[0].ethernet)); + clib_memcpy_fast (h->ip4_over_ethernet[0].ethernet, hi->hw_address, + sizeof (h->ip4_over_ethernet[0].ethernet)); h->ip4_over_ethernet[0].ip4 = src[0]; h->ip4_over_ethernet[1].ip4 = dst[0]; diff --git a/src/vnet/ip/ip4_options.c b/src/vnet/ip/ip4_options.c index c008b9bde8c..1b5a7878512 100644 --- a/src/vnet/ip/ip4_options.c +++ b/src/vnet/ip/ip4_options.c @@ -94,7 +94,7 @@ VLIB_NODE_FN (ip4_options_node) (vlib_main_t * vm, ip4_options_trace_t *t = vlib_add_trace (vm, node, b, sizeof (*t)); - clib_memcpy (t->option, options, 4); + clib_memcpy_fast (t->option, options, 4); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi, next); diff --git a/src/vnet/ip/ip4_packet.h b/src/vnet/ip/ip4_packet.h index fa86229a073..2ce6763f8d7 100644 --- a/src/vnet/ip/ip4_packet.h +++ b/src/vnet/ip/ip4_packet.h @@ -67,7 +67,8 @@ always_inline void ip4_addr_fib_init (ip4_address_fib_t * addr_fib, const ip4_address_t * address, u32 fib_index) { - clib_memcpy (&addr_fib->ip4_addr, address, sizeof (addr_fib->ip4_addr)); + clib_memcpy_fast (&addr_fib->ip4_addr, address, + sizeof (addr_fib->ip4_addr)); addr_fib->fib_index = fib_index; } diff --git a/src/vnet/ip/ip4_reassembly.c b/src/vnet/ip/ip4_reassembly.c index 42231d0e30d..bd33026db85 100644 --- a/src/vnet/ip/ip4_reassembly.c +++ b/src/vnet/ip/ip4_reassembly.c @@ -1125,8 +1125,8 @@ ip4_reass_set (u32 timeout_ms, u32 max_reassemblies, else { clib_bihash_free_16_8 (&ip4_reass_main.hash); - clib_memcpy (&ip4_reass_main.hash, &new_hash, - sizeof (ip4_reass_main.hash)); + clib_memcpy_fast (&ip4_reass_main.hash, &new_hash, + sizeof (ip4_reass_main.hash)); } } return 0; diff --git a/src/vnet/ip/ip6.h b/src/vnet/ip/ip6.h index e807886cd2a..aef24454413 100644 --- a/src/vnet/ip/ip6.h +++ b/src/vnet/ip/ip6.h @@ -616,10 +616,10 @@ vlib_buffer_push_ip6 (vlib_main_t * vm, vlib_buffer_t * b, ip6h->hop_limit = 0xff; ip6h->protocol = proto; - clib_memcpy (ip6h->src_address.as_u8, src->as_u8, - sizeof (ip6h->src_address)); - clib_memcpy (ip6h->dst_address.as_u8, dst->as_u8, - sizeof (ip6h->src_address)); + clib_memcpy_fast (ip6h->src_address.as_u8, src->as_u8, + sizeof (ip6h->src_address)); + clib_memcpy_fast (ip6h->dst_address.as_u8, dst->as_u8, + sizeof (ip6h->src_address)); b->flags |= VNET_BUFFER_F_IS_IP6; return ip6h; diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index be0037e5edd..5a176435510 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -883,9 +883,9 @@ ip6_forward_next_trace (vlib_main_t * vm, vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b0)->sw_if_index[VLIB_RX]); - clib_memcpy (t0->packet_data, - vlib_buffer_get_current (b0), - sizeof (t0->packet_data)); + clib_memcpy_fast (t0->packet_data, + vlib_buffer_get_current (b0), + sizeof (t0->packet_data)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -898,9 +898,9 @@ ip6_forward_next_trace (vlib_main_t * vm, vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b1)->sw_if_index[VLIB_RX]); - clib_memcpy (t1->packet_data, - vlib_buffer_get_current (b1), - sizeof (t1->packet_data)); + clib_memcpy_fast (t1->packet_data, + vlib_buffer_get_current (b1), + sizeof (t1->packet_data)); } from += 2; n_left -= 2; @@ -927,9 +927,9 @@ ip6_forward_next_trace (vlib_main_t * vm, vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b0)->sw_if_index[VLIB_RX]); - clib_memcpy (t0->packet_data, - vlib_buffer_get_current (b0), - sizeof (t0->packet_data)); + clib_memcpy_fast (t0->packet_data, + vlib_buffer_get_current (b0), + sizeof (t0->packet_data)); } from += 1; n_left -= 1; @@ -1530,8 +1530,8 @@ ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index, sw_if_index); } - clib_memcpy (h->link_layer_option.ethernet_address, hi->hw_address, - vec_len (hi->hw_address)); + clib_memcpy_fast (h->link_layer_option.ethernet_address, hi->hw_address, + vec_len (hi->hw_address)); h->neighbor.icmp.checksum = ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length); @@ -2415,7 +2415,7 @@ ip6_hop_by_hop (vlib_main_t * vm, ARRAY_LEN (t->option_data) ? trace_len : ARRAY_LEN (t->option_data); t->trace_len = trace_len; - clib_memcpy (t->option_data, hbh0, trace_len); + clib_memcpy_fast (t->option_data, hbh0, trace_len); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -2429,7 +2429,7 @@ ip6_hop_by_hop (vlib_main_t * vm, ARRAY_LEN (t->option_data) ? trace_len : ARRAY_LEN (t->option_data); t->trace_len = trace_len; - clib_memcpy (t->option_data, hbh1, trace_len); + clib_memcpy_fast (t->option_data, hbh1, trace_len); } } @@ -2510,7 +2510,7 @@ ip6_hop_by_hop (vlib_main_t * vm, ARRAY_LEN (t->option_data) ? trace_len : ARRAY_LEN (t->option_data); t->trace_len = trace_len; - clib_memcpy (t->option_data, hbh0, trace_len); + clib_memcpy_fast (t->option_data, hbh0, trace_len); } b0->error = error_node->errors[error0]; diff --git a/src/vnet/ip/ip6_hop_by_hop.c b/src/vnet/ip/ip6_hop_by_hop.c index fbaf0007f19..38cdbfffeaa 100644 --- a/src/vnet/ip/ip6_hop_by_hop.c +++ b/src/vnet/ip/ip6_hop_by_hop.c @@ -321,8 +321,8 @@ ip6_add_hop_by_hop_node_fn (vlib_main_t * vm, hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1); hbh1 = (ip6_hop_by_hop_header_t *) (ip1 + 1); /* $$$ tune, rewrite_length is a multiple of 8 */ - clib_memcpy (hbh0, rewrite, rewrite_length); - clib_memcpy (hbh1, rewrite, rewrite_length); + clib_memcpy_fast (hbh0, rewrite, rewrite_length); + clib_memcpy_fast (hbh1, rewrite, rewrite_length); /* Patch the protocol chain, insert the h-b-h (type 0) header */ hbh0->protocol = ip0->protocol; hbh1->protocol = ip1->protocol; @@ -399,7 +399,7 @@ ip6_add_hop_by_hop_node_fn (vlib_main_t * vm, hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1); /* $$$ tune, rewrite_length is a multiple of 8 */ - clib_memcpy (hbh0, rewrite, rewrite_length); + clib_memcpy_fast (hbh0, rewrite, rewrite_length); /* Patch the protocol chain, insert the h-b-h (type 0) header */ hbh0->protocol = ip0->protocol; ip0->protocol = 0; diff --git a/src/vnet/ip/ip6_reassembly.c b/src/vnet/ip/ip6_reassembly.c index 7f6c5ca4af7..de9fbe52a75 100644 --- a/src/vnet/ip/ip6_reassembly.c +++ b/src/vnet/ip/ip6_reassembly.c @@ -1165,8 +1165,8 @@ ip6_reass_set (u32 timeout_ms, u32 max_reassemblies, else { clib_bihash_free_48_8 (&ip6_reass_main.hash); - clib_memcpy (&ip6_reass_main.hash, &new_hash, - sizeof (ip6_reass_main.hash)); + clib_memcpy_fast (&ip6_reass_main.hash, &new_hash, + sizeof (ip6_reass_main.hash)); } } return 0; diff --git a/src/vnet/ip/ip6_to_ip4.h b/src/vnet/ip/ip6_to_ip4.h index f25c9e408f0..d13a0c1f104 100644 --- a/src/vnet/ip/ip6_to_ip4.h +++ b/src/vnet/ip/ip6_to_ip4.h @@ -373,7 +373,7 @@ icmp6_to_icmp (vlib_buffer_t * p, ip6_to_ip4_set_fn_t fn, void *ctx, //Move up icmp header ip4 = (ip4_header_t *) u8_ptr_add (inner_l4, -2 * sizeof (*ip4) - 8); - clib_memcpy (u8_ptr_add (inner_l4, -sizeof (*ip4) - 8), icmp, 8); + clib_memcpy_fast (u8_ptr_add (inner_l4, -sizeof (*ip4) - 8), icmp, 8); icmp = (icmp46_header_t *) u8_ptr_add (inner_l4, -sizeof (*ip4) - 8); } else diff --git a/src/vnet/ip/ip_frag.c b/src/vnet/ip/ip_frag.c index 8de4dfc5d58..6efb8834c4b 100644 --- a/src/vnet/ip/ip_frag.c +++ b/src/vnet/ip/ip_frag.c @@ -165,7 +165,7 @@ ip4_frag_do_fragment (vlib_main_t * vm, u32 from_bi, u32 ** buffer, frag_set_sw_if_index (to_b, org_from_b); /* Copy ip4 header */ - clib_memcpy (to_b->data, org_from_packet, sizeof (ip4_header_t)); + clib_memcpy_fast (to_b->data, org_from_packet, sizeof (ip4_header_t)); to_ip4 = vlib_buffer_get_current (to_b); to_data = (void *) (to_ip4 + 1); @@ -178,7 +178,7 @@ ip4_frag_do_fragment (vlib_main_t * vm, u32 from_bi, u32 ** buffer, /* Figure out how many bytes we can safely copy */ bytes_to_copy = left_in_to_buffer <= left_in_from_buffer ? left_in_to_buffer : left_in_from_buffer; - clib_memcpy (to_data + to_ptr, from_data + ptr, bytes_to_copy); + clib_memcpy_fast (to_data + to_ptr, from_data + ptr, bytes_to_copy); left_in_to_buffer -= bytes_to_copy; ptr += bytes_to_copy; left_in_from_buffer -= bytes_to_copy; @@ -441,7 +441,7 @@ ip6_frag_do_fragment (vlib_main_t * vm, u32 from_bi, u32 ** buffer, frag_set_sw_if_index (to_b, org_from_b); /* Copy ip6 header */ - clib_memcpy (to_b->data, ip6, sizeof (ip6_header_t)); + clib_memcpy_fast (to_b->data, ip6, sizeof (ip6_header_t)); to_ip6 = vlib_buffer_get_current (to_b); to_frag_hdr = (ip6_frag_hdr_t *) (to_ip6 + 1); to_data = (void *) (to_frag_hdr + 1); @@ -455,7 +455,7 @@ ip6_frag_do_fragment (vlib_main_t * vm, u32 from_bi, u32 ** buffer, /* Figure out how many bytes we can safely copy */ bytes_to_copy = left_in_to_buffer <= left_in_from_buffer ? left_in_to_buffer : left_in_from_buffer; - clib_memcpy (to_data + to_ptr, from_data + ptr, bytes_to_copy); + clib_memcpy_fast (to_data + to_ptr, from_data + ptr, bytes_to_copy); left_in_to_buffer -= bytes_to_copy; ptr += bytes_to_copy; left_in_from_buffer -= bytes_to_copy; diff --git a/src/vnet/ip/punt.c b/src/vnet/ip/punt.c index a6d6c08818b..d8c7d81a23a 100644 --- a/src/vnet/ip/punt.c +++ b/src/vnet/ip/punt.c @@ -368,7 +368,7 @@ udp46_punt_socket_inline (vlib_main_t * vm, c = punt_client_get (is_ip4, port); udp_punt_trace_t *t; t = vlib_add_trace (vm, node, b, sizeof (t[0])); - clib_memcpy (&t->client, c, sizeof (t->client)); + clib_memcpy_fast (&t->client, c, sizeof (t->client)); } /* Re-set iovecs if present. */ @@ -401,7 +401,7 @@ udp46_punt_socket_inline (vlib_main_t * vm, } udp_punt_trace_t *t; t = vlib_add_trace (vm, node, b, sizeof (t[0])); - clib_memcpy (&t->client, c, sizeof (t->client)); + clib_memcpy_fast (&t->client, c, sizeof (t->client)); t->is_midchain = 1; } diff --git a/src/vnet/ip/rd_cp.c b/src/vnet/ip/rd_cp.c index 164b3045775..1a6122fc9d3 100644 --- a/src/vnet/ip/rd_cp.c +++ b/src/vnet/ip/rd_cp.c @@ -227,7 +227,7 @@ get_interface_mac_address (u32 sw_if_index, u8 mac[]) return 1; } - clib_memcpy (mac, eth_if->address, 6); + clib_memcpy_fast (mac, eth_if->address, 6); return 0; } diff --git a/src/vnet/ipfix-export/flow_report.c b/src/vnet/ipfix-export/flow_report.c index 78ad1f5023c..c9370a748cf 100644 --- a/src/vnet/ipfix-export/flow_report.c +++ b/src/vnet/ipfix-export/flow_report.c @@ -126,7 +126,7 @@ send_template_packet (flow_report_main_t * frm, ASSERT (vec_len (fr->rewrite) < VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES); - clib_memcpy (b0->data, fr->rewrite, vec_len (fr->rewrite)); + clib_memcpy_fast (b0->data, fr->rewrite, vec_len (fr->rewrite)); b0->current_data = 0; b0->current_length = vec_len (fr->rewrite); b0->flags |= (VLIB_BUFFER_TOTAL_LENGTH_VALID | VNET_BUFFER_F_FLOW_REPORT); diff --git a/src/vnet/ipfix-export/flow_report_classify.c b/src/vnet/ipfix-export/flow_report_classify.c index 449bf228ce9..6bddb1407fe 100644 --- a/src/vnet/ipfix-export/flow_report_classify.c +++ b/src/vnet/ipfix-export/flow_report_classify.c @@ -226,7 +226,7 @@ ipfix_classify_send_flows (flow_report_main_t * frm, u32 copy_len = sizeof (ip4_header_t) + sizeof (udp_header_t) + sizeof (ipfix_message_header_t); - clib_memcpy (b0->data, fr->rewrite, copy_len); + clib_memcpy_fast (b0->data, fr->rewrite, copy_len); b0->current_data = 0; b0->current_length = copy_len; b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; @@ -261,7 +261,7 @@ ipfix_classify_send_flows (flow_report_main_t * frm, #define _(field,mask,item,length) \ if (clib_bitmap_get (fr->fields_to_send, field_index)) \ { \ - clib_memcpy (b0->data + next_offset, &field, \ + clib_memcpy_fast (b0->data + next_offset, &field, \ length); \ next_offset += length; \ } \ @@ -272,8 +272,8 @@ ipfix_classify_send_flows (flow_report_main_t * frm, /* Add packetTotalCount manually */ { u64 packets = clib_host_to_net_u64 (v->hits); - clib_memcpy (b0->data + next_offset, &packets, - sizeof (packets)); + clib_memcpy_fast (b0->data + next_offset, &packets, + sizeof (packets)); next_offset += sizeof (packets); } records_this_buffer++; diff --git a/src/vnet/ipsec/ah_encrypt.c b/src/vnet/ipsec/ah_encrypt.c index 7a6703da2de..6529828f0e9 100644 --- a/src/vnet/ipsec/ah_encrypt.c +++ b/src/vnet/ipsec/ah_encrypt.c @@ -171,7 +171,7 @@ ah_encrypt_inline (vlib_main_t * vm, sizeof (ethernet_header_t)); ethernet_header_t *oeh0 = (ethernet_header_t *) ((u8 *) ieh0 + (adv - icv_size)); - clib_memcpy (oeh0, ieh0, sizeof (ethernet_header_t)); + clib_memcpy_fast (oeh0, ieh0, sizeof (ethernet_header_t)); } vlib_buffer_advance (i_b0, adv - icv_size); diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c index 8edd5653667..101c5efbfc8 100644 --- a/src/vnet/ipsec/esp_encrypt.c +++ b/src/vnet/ipsec/esp_encrypt.c @@ -316,7 +316,7 @@ esp_encrypt_inline (vlib_main_t * vm, vlib_buffer_get_current (i_b0) - sizeof (ethernet_header_t)); oeh0 = (ethernet_header_t *) o_b0->data; - clib_memcpy (oeh0, ieh0, sizeof (ethernet_header_t)); + clib_memcpy_fast (oeh0, ieh0, sizeof (ethernet_header_t)); next0 = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT; vnet_buffer (o_b0)->sw_if_index[VLIB_TX] = vnet_buffer (i_b0)->sw_if_index[VLIB_TX]; @@ -359,10 +359,10 @@ esp_encrypt_inline (vlib_main_t * vm, ipsec_proto_main_crypto_algs[sa0->crypto_alg].iv_size]; RAND_bytes (iv, sizeof (iv)); - clib_memcpy ((u8 *) vlib_buffer_get_current (o_b0) + - ip_udp_hdr_size + sizeof (esp_header_t), iv, - em->ipsec_proto_main_crypto_algs[sa0-> - crypto_alg].iv_size); + clib_memcpy_fast ((u8 *) vlib_buffer_get_current (o_b0) + + ip_udp_hdr_size + sizeof (esp_header_t), iv, + em->ipsec_proto_main_crypto_algs[sa0-> + crypto_alg].iv_size); esp_encrypt_cbc (vm, sa0->crypto_alg, (u8 *) vlib_buffer_get_current (i_b0), diff --git a/src/vnet/ipsec/ikev2.c b/src/vnet/ipsec/ikev2.c index ac20d4a9a10..80497f20807 100644 --- a/src/vnet/ipsec/ikev2.c +++ b/src/vnet/ipsec/ikev2.c @@ -161,7 +161,7 @@ ikev2_select_proposal (ikev2_sa_proposal_t * proposals, { bitmap |= 1 << transform->type; vec_add2 (rv->transforms, new_t, 1); - clib_memcpy (new_t, transform, sizeof (*new_t)); + clib_memcpy_fast (new_t, transform, sizeof (*new_t)); new_t->attrs = vec_dup (transform->attrs); } } @@ -469,37 +469,37 @@ ikev2_calc_keys (ikev2_sa_t * sa) /* SK_d */ sa->sk_d = vec_new (u8, tr_prf->key_trunc); - clib_memcpy (sa->sk_d, keymat + pos, tr_prf->key_trunc); + clib_memcpy_fast (sa->sk_d, keymat + pos, tr_prf->key_trunc); pos += tr_prf->key_trunc; /* SK_ai */ sa->sk_ai = vec_new (u8, tr_integ->key_len); - clib_memcpy (sa->sk_ai, keymat + pos, tr_integ->key_len); + clib_memcpy_fast (sa->sk_ai, keymat + pos, tr_integ->key_len); pos += tr_integ->key_len; /* SK_ar */ sa->sk_ar = vec_new (u8, tr_integ->key_len); - clib_memcpy (sa->sk_ar, keymat + pos, tr_integ->key_len); + clib_memcpy_fast (sa->sk_ar, keymat + pos, tr_integ->key_len); pos += tr_integ->key_len; /* SK_ei */ sa->sk_ei = vec_new (u8, tr_encr->key_len); - clib_memcpy (sa->sk_ei, keymat + pos, tr_encr->key_len); + clib_memcpy_fast (sa->sk_ei, keymat + pos, tr_encr->key_len); pos += tr_encr->key_len; /* SK_er */ sa->sk_er = vec_new (u8, tr_encr->key_len); - clib_memcpy (sa->sk_er, keymat + pos, tr_encr->key_len); + clib_memcpy_fast (sa->sk_er, keymat + pos, tr_encr->key_len); pos += tr_encr->key_len; /* SK_pi */ sa->sk_pi = vec_new (u8, tr_prf->key_len); - clib_memcpy (sa->sk_pi, keymat + pos, tr_prf->key_len); + clib_memcpy_fast (sa->sk_pi, keymat + pos, tr_prf->key_len); pos += tr_prf->key_len; /* SK_pr */ sa->sk_pr = vec_new (u8, tr_prf->key_len); - clib_memcpy (sa->sk_pr, keymat + pos, tr_prf->key_len); + clib_memcpy_fast (sa->sk_pr, keymat + pos, tr_prf->key_len); pos += tr_prf->key_len; vec_free (keymat); @@ -529,22 +529,22 @@ ikev2_calc_child_keys (ikev2_sa_t * sa, ikev2_child_sa_t * child) /* SK_ei */ child->sk_ei = vec_new (u8, ctr_encr->key_len); - clib_memcpy (child->sk_ei, keymat + pos, ctr_encr->key_len); + clib_memcpy_fast (child->sk_ei, keymat + pos, ctr_encr->key_len); pos += ctr_encr->key_len; /* SK_ai */ child->sk_ai = vec_new (u8, ctr_integ->key_len); - clib_memcpy (child->sk_ai, keymat + pos, ctr_integ->key_len); + clib_memcpy_fast (child->sk_ai, keymat + pos, ctr_integ->key_len); pos += ctr_integ->key_len; /* SK_er */ child->sk_er = vec_new (u8, ctr_encr->key_len); - clib_memcpy (child->sk_er, keymat + pos, ctr_encr->key_len); + clib_memcpy_fast (child->sk_er, keymat + pos, ctr_encr->key_len); pos += ctr_encr->key_len; /* SK_ar */ child->sk_ar = vec_new (u8, ctr_integ->key_len); - clib_memcpy (child->sk_ar, keymat + pos, ctr_integ->key_len); + clib_memcpy_fast (child->sk_ar, keymat + pos, ctr_integ->key_len); pos += ctr_integ->key_len; ASSERT (pos == len); @@ -1106,7 +1106,7 @@ ikev2_process_create_child_sa_req (vlib_main_t * vm, ikev2_sa_t * sa, } else if (payload == IKEV2_PAYLOAD_NONCE) { - clib_memcpy (nonce, ikep->payload, plen - sizeof (*ikep)); + clib_memcpy_fast (nonce, ikep->payload, plen - sizeof (*ikep)); } else if (payload == IKEV2_PAYLOAD_TSI) { @@ -1596,15 +1596,15 @@ ikev2_create_tunnel_interface (vnet_main_t * vnm, ikev2_sa_t * sa, a.integ_alg = integ_type; a.local_integ_key_len = vec_len (loc_ikey); - clib_memcpy (a.local_integ_key, loc_ikey, a.local_integ_key_len); + clib_memcpy_fast (a.local_integ_key, loc_ikey, a.local_integ_key_len); a.remote_integ_key_len = vec_len (rem_ikey); - clib_memcpy (a.remote_integ_key, rem_ikey, a.remote_integ_key_len); + clib_memcpy_fast (a.remote_integ_key, rem_ikey, a.remote_integ_key_len); a.crypto_alg = encr_type; a.local_crypto_key_len = vec_len (loc_ckey); - clib_memcpy (a.local_crypto_key, loc_ckey, a.local_crypto_key_len); + clib_memcpy_fast (a.local_crypto_key, loc_ckey, a.local_crypto_key_len); a.remote_crypto_key_len = vec_len (rem_ckey); - clib_memcpy (a.remote_crypto_key, rem_ckey, a.remote_crypto_key_len); + clib_memcpy_fast (a.remote_crypto_key, rem_ckey, a.remote_crypto_key_len); if (sa->profile && sa->profile->lifetime) { @@ -1898,7 +1898,7 @@ ikev2_generate_message (ikev2_sa_t * sa, ike_header_t * ike, void *user) tlen += vec_len (chain->data); ike->nextpayload = chain->first_payload_type; ike->length = clib_host_to_net_u32 (tlen); - clib_memcpy (ike->payload, chain->data, vec_len (chain->data)); + clib_memcpy_fast (ike->payload, chain->data, vec_len (chain->data)); /* store whole IKE payload - needed for PSK auth */ vec_free (sa->last_sa_init_res_packet_data); @@ -1930,8 +1930,8 @@ ikev2_generate_message (ikev2_sa_t * sa, ike_header_t * ike, void *user) ikev2_calc_integr (tr_integ, sa->is_initiator ? sa->sk_ai : sa->sk_ar, (u8 *) ike, tlen - tr_integ->key_trunc); - clib_memcpy (ike->payload + tlen - tr_integ->key_trunc - sizeof (*ike), - integ, tr_integ->key_trunc); + clib_memcpy_fast (ike->payload + tlen - tr_integ->key_trunc - + sizeof (*ike), integ, tr_integ->key_trunc); /* store whole IKE payload - needed for retransmit */ vec_free (sa->last_res_packet_data); @@ -1986,7 +1986,7 @@ ikev2_retransmit_sa_init (ike_header_t * ike, ike->flags = tmp->flags; ike->msgid = tmp->msgid; ike->length = tmp->length; - clib_memcpy(ike->payload, tmp->payload, + clib_memcpy_fast(ike->payload, tmp->payload, clib_net_to_host_u32(tmp->length) - sizeof(*ike)); clib_warning("IKE_SA_INIT retransmit from %U to %U", format_ip4_address, &raddr, @@ -2038,8 +2038,8 @@ ikev2_retransmit_resp (ikev2_sa_t * sa, ike_header_t * ike) ike->flags = tmp->flags; ike->msgid = tmp->msgid; ike->length = tmp->length; - clib_memcpy (ike->payload, tmp->payload, - clib_net_to_host_u32 (tmp->length) - sizeof (*ike)); + clib_memcpy_fast (ike->payload, tmp->payload, + clib_net_to_host_u32 (tmp->length) - sizeof (*ike)); clib_warning ("IKE msgid %u retransmit from %U to %U", msg_id, format_ip4_address, &sa->raddr, @@ -2165,7 +2165,7 @@ ikev2_node_fn (vlib_main_t * vm, /* add SA to the pool */ pool_get (km->per_thread_data[thread_index].sas, sa0); - clib_memcpy (sa0, &sa, sizeof (*sa0)); + clib_memcpy_fast (sa0, &sa, sizeof (*sa0)); hash_set (km-> per_thread_data[thread_index].sa_by_rspi, sa0->rspi, @@ -2202,7 +2202,7 @@ ikev2_node_fn (vlib_main_t * vm, { /* add SA to the pool */ pool_get (km->per_thread_data[thread_index].sas, sa0); - clib_memcpy (sa0, &sa, sizeof (*sa0)); + clib_memcpy_fast (sa0, &sa, sizeof (*sa0)); hash_set (km->per_thread_data[thread_index].sa_by_rspi, sa0->rspi, sa0 - km->per_thread_data[thread_index].sas); @@ -2993,20 +2993,20 @@ ikev2_initiate_sa_init (vlib_main_t * vm, u8 * name) u8 *nat_detection_sha1 = vec_new (u8, 20); u64 tmpspi = clib_host_to_net_u64 (sa.ispi); - clib_memcpy (&nat_detection_source[0], &tmpspi, sizeof (tmpspi)); + clib_memcpy_fast (&nat_detection_source[0], &tmpspi, sizeof (tmpspi)); tmpspi = clib_host_to_net_u64 (sa.rspi); - clib_memcpy (&nat_detection_source[8], &tmpspi, sizeof (tmpspi)); + clib_memcpy_fast (&nat_detection_source[8], &tmpspi, sizeof (tmpspi)); u16 tmpport = clib_host_to_net_u16 (500); - clib_memcpy (&nat_detection_source[8 + 8 + 4], &tmpport, - sizeof (tmpport)); + clib_memcpy_fast (&nat_detection_source[8 + 8 + 4], &tmpport, + sizeof (tmpport)); u32 tmpip = clib_host_to_net_u32 (if_ip->as_u32); - clib_memcpy (&nat_detection_source[8 + 8], &tmpip, sizeof (tmpip)); + clib_memcpy_fast (&nat_detection_source[8 + 8], &tmpip, sizeof (tmpip)); SHA1 (nat_detection_source, sizeof (nat_detection_source), nat_detection_sha1); ikev2_payload_add_notify (chain, IKEV2_NOTIFY_MSG_NAT_DETECTION_SOURCE_IP, nat_detection_sha1); tmpip = clib_host_to_net_u32 (p->responder.ip4.as_u32); - clib_memcpy (&nat_detection_source[8 + 8], &tmpip, sizeof (tmpip)); + clib_memcpy_fast (&nat_detection_source[8 + 8], &tmpip, sizeof (tmpip)); SHA1 (nat_detection_source, sizeof (nat_detection_source), nat_detection_sha1); ikev2_payload_add_notify (chain, @@ -3016,7 +3016,7 @@ ikev2_initiate_sa_init (vlib_main_t * vm, u8 * name) u8 *sig_hash_algo = vec_new (u8, 8); u64 tmpsig = clib_host_to_net_u64 (0x0001000200030004); - clib_memcpy (sig_hash_algo, &tmpsig, sizeof (tmpsig)); + clib_memcpy_fast (sig_hash_algo, &tmpsig, sizeof (tmpsig)); ikev2_payload_add_notify (chain, IKEV2_NOTIFY_MSG_SIGNATURE_HASH_ALGORITHMS, sig_hash_algo); @@ -3027,7 +3027,7 @@ ikev2_initiate_sa_init (vlib_main_t * vm, u8 * name) len += vec_len (chain->data); ike0->nextpayload = chain->first_payload_type; ike0->length = clib_host_to_net_u32 (len); - clib_memcpy (ike0->payload, chain->data, vec_len (chain->data)); + clib_memcpy_fast (ike0->payload, chain->data, vec_len (chain->data)); ikev2_payload_destroy_chain (chain); ike0->version = IKE_VERSION_2; @@ -3048,7 +3048,8 @@ ikev2_initiate_sa_init (vlib_main_t * vm, u8 * name) sa.i_auth.hex = p->auth.hex; sa.i_auth.data = vec_dup (p->auth.data); #if OPENSSL_VERSION_NUMBER >= 0x10100000L - clib_memcpy (sa.i_auth.key, p->auth.key, EVP_PKEY_size (p->auth.key)); + clib_memcpy_fast (sa.i_auth.key, p->auth.key, + EVP_PKEY_size (p->auth.key)); #else sa.i_auth.key = vec_dup (p->auth.key); #endif @@ -3058,7 +3059,7 @@ ikev2_initiate_sa_init (vlib_main_t * vm, u8 * name) /* add SA to the pool */ ikev2_sa_t *sa0 = 0; pool_get (km->sais, sa0); - clib_memcpy (sa0, &sa, sizeof (*sa0)); + clib_memcpy_fast (sa0, &sa, sizeof (*sa0)); hash_set (km->sa_by_ispi, sa0->ispi, sa0 - km->sais); ikev2_send_ike (vm, if_ip, &p->responder.ip4, bi0, len); diff --git a/src/vnet/ipsec/ikev2_payload.c b/src/vnet/ipsec/ikev2_payload.c index e5fa0149feb..3aae6d3a1f4 100644 --- a/src/vnet/ipsec/ikev2_payload.c +++ b/src/vnet/ipsec/ikev2_payload.c @@ -205,7 +205,7 @@ ikev2_payload_add_sa (ikev2_payload_chain_t * c, clib_host_to_net_u16 (sizeof (*tr) + vec_len (t->attrs)); if (vec_len (t->attrs) > 0) - clib_memcpy (tr->attributes, t->attrs, vec_len (t->attrs)); + clib_memcpy_fast (tr->attributes, t->attrs, vec_len (t->attrs)); DBG_PLD ("transform type %U transform_id %u last_or_more %u attr_size %u%s%U", diff --git a/src/vnet/l2/l2_efp_filter.c b/src/vnet/l2/l2_efp_filter.c index 59f9f515d47..f10c5eaf7f9 100644 --- a/src/vnet/l2/l2_efp_filter.c +++ b/src/vnet/l2/l2_efp_filter.c @@ -340,9 +340,9 @@ l2_efp_filter_node_fn (vlib_main_t * vm, l2_efp_filter_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); - clib_memcpy (t->raw, &h0->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->raw, &h0->type, sizeof (t->raw)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -350,9 +350,9 @@ l2_efp_filter_node_fn (vlib_main_t * vm, l2_efp_filter_trace_t *t = vlib_add_trace (vm, node, b1, sizeof (*t)); t->sw_if_index = sw_if_index1; - clib_memcpy (t->src, h1->src_address, 6); - clib_memcpy (t->dst, h1->dst_address, 6); - clib_memcpy (t->raw, &h1->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h1->src_address, 6); + clib_memcpy_fast (t->dst, h1->dst_address, 6); + clib_memcpy_fast (t->raw, &h1->type, sizeof (t->raw)); } } @@ -441,9 +441,9 @@ l2_efp_filter_node_fn (vlib_main_t * vm, l2_efp_filter_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); - clib_memcpy (t->raw, &h0->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->raw, &h0->type, sizeof (t->raw)); } /* verify speculative enqueue, maybe switch current next frame */ diff --git a/src/vnet/l2/l2_fib.c b/src/vnet/l2/l2_fib.c index 48be42e8ce9..fc518fe9dbf 100644 --- a/src/vnet/l2/l2_fib.c +++ b/src/vnet/l2/l2_fib.c @@ -83,7 +83,7 @@ incr_mac_address (u8 * mac) tmp += 1 << 16; /* skip unused (least significant) octets */ tmp = clib_host_to_net_u64 (tmp); - clib_memcpy (mac, &tmp, 6); + clib_memcpy_fast (mac, &tmp, 6); } /** Format sw_if_index. If the value is ~0, use the text "N/A" */ @@ -584,7 +584,7 @@ l2fib_test_command_fn (vlib_main_t * vm, return clib_error_return (0, "noop: pick at least one of (add,del,check)"); - clib_memcpy (save_mac, mac, 6); + clib_memcpy_fast (save_mac, mac, 6); if (is_add) { @@ -601,7 +601,7 @@ l2fib_test_command_fn (vlib_main_t * vm, BVT (clib_bihash_kv) kv; l2fib_main_t *mp = &l2fib_main; - clib_memcpy (mac, save_mac, 6); + clib_memcpy_fast (mac, save_mac, 6); for (i = 0; i < count; i++) { @@ -617,7 +617,7 @@ l2fib_test_command_fn (vlib_main_t * vm, if (is_del) { - clib_memcpy (mac, save_mac, 6); + clib_memcpy_fast (mac, save_mac, 6); for (i = 0; i < count; i++) { @@ -1071,8 +1071,8 @@ l2fib_scan (vlib_main_t * vm, f64 start_time, u8 event_only) if (l2fib_entry_result_is_set_LRN_EVT (&result)) { /* copy mac entry to event msg */ - clib_memcpy (mp->mac[evt_idx].mac_addr, key.fields.mac, - 6); + clib_memcpy_fast (mp->mac[evt_idx].mac_addr, + key.fields.mac, 6); mp->mac[evt_idx].action = l2fib_entry_result_is_set_LRN_MOV (&result) ? MAC_EVENT_ACTION_MOVE : MAC_EVENT_ACTION_ADD; @@ -1116,7 +1116,8 @@ l2fib_scan (vlib_main_t * vm, f64 start_time, u8 event_only) if (client) { /* copy mac entry to event msg */ - clib_memcpy (mp->mac[evt_idx].mac_addr, key.fields.mac, 6); + clib_memcpy_fast (mp->mac[evt_idx].mac_addr, key.fields.mac, + 6); mp->mac[evt_idx].action = MAC_EVENT_ACTION_DELETE; mp->mac[evt_idx].sw_if_index = htonl (result.fields.sw_if_index); diff --git a/src/vnet/l2/l2_flood.c b/src/vnet/l2/l2_flood.c index aeac8ff10d8..3198d323d60 100644 --- a/src/vnet/l2/l2_flood.c +++ b/src/vnet/l2/l2_flood.c @@ -256,8 +256,8 @@ l2flood_node_fn (vlib_main_t * vm, h0 = vlib_buffer_get_current (c0); t->sw_if_index = sw_if_index0; t->bd_index = vnet_buffer (c0)->l2.bd_index; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); } /* Do normal L2 forwarding */ @@ -307,8 +307,8 @@ l2flood_node_fn (vlib_main_t * vm, h0 = vlib_buffer_get_current (c0); t->sw_if_index = sw_if_index0; t->bd_index = vnet_buffer (c0)->l2.bd_index; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); } diff --git a/src/vnet/l2/l2_fwd.c b/src/vnet/l2/l2_fwd.c index 9fb7b7d355a..dfc1a74a38c 100644 --- a/src/vnet/l2/l2_fwd.c +++ b/src/vnet/l2/l2_fwd.c @@ -324,8 +324,8 @@ l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, l2fwd_trace_t *t = vlib_add_trace (vm, node, b[0], sizeof (*t)); t->sw_if_index = sw_if_index0; t->bd_index = vnet_buffer (b[0])->l2.bd_index; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); t->result = result0; } if (b[1]->flags & VLIB_BUFFER_IS_TRACED) @@ -333,8 +333,8 @@ l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, l2fwd_trace_t *t = vlib_add_trace (vm, node, b[1], sizeof (*t)); t->sw_if_index = sw_if_index1; t->bd_index = vnet_buffer (b[1])->l2.bd_index; - clib_memcpy (t->src, h1->src_address, 6); - clib_memcpy (t->dst, h1->dst_address, 6); + clib_memcpy_fast (t->src, h1->src_address, 6); + clib_memcpy_fast (t->dst, h1->dst_address, 6); t->result = result1; } if (b[2]->flags & VLIB_BUFFER_IS_TRACED) @@ -342,8 +342,8 @@ l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, l2fwd_trace_t *t = vlib_add_trace (vm, node, b[2], sizeof (*t)); t->sw_if_index = sw_if_index2; t->bd_index = vnet_buffer (b[2])->l2.bd_index; - clib_memcpy (t->src, h2->src_address, 6); - clib_memcpy (t->dst, h2->dst_address, 6); + clib_memcpy_fast (t->src, h2->src_address, 6); + clib_memcpy_fast (t->dst, h2->dst_address, 6); t->result = result2; } if (b[3]->flags & VLIB_BUFFER_IS_TRACED) @@ -351,8 +351,8 @@ l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, l2fwd_trace_t *t = vlib_add_trace (vm, node, b[3], sizeof (*t)); t->sw_if_index = sw_if_index3; t->bd_index = vnet_buffer (b[3])->l2.bd_index; - clib_memcpy (t->src, h3->src_address, 6); - clib_memcpy (t->dst, h3->dst_address, 6); + clib_memcpy_fast (t->src, h3->src_address, 6); + clib_memcpy_fast (t->dst, h3->dst_address, 6); t->result = result3; } } @@ -387,8 +387,8 @@ l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, l2fwd_trace_t *t = vlib_add_trace (vm, node, b[0], sizeof (*t)); t->sw_if_index = sw_if_index0; t->bd_index = vnet_buffer (b[0])->l2.bd_index; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); t->result = result0; } diff --git a/src/vnet/l2/l2_input.c b/src/vnet/l2/l2_input.c index 28b8697a7e4..b66a7446d66 100644 --- a/src/vnet/l2/l2_input.c +++ b/src/vnet/l2/l2_input.c @@ -366,8 +366,8 @@ l2input_node_inline (vlib_main_t * vm, l2input_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -375,8 +375,8 @@ l2input_node_inline (vlib_main_t * vm, l2input_trace_t *t = vlib_add_trace (vm, node, b1, sizeof (*t)); t->sw_if_index = sw_if_index1; - clib_memcpy (t->src, h1->src_address, 6); - clib_memcpy (t->dst, h1->dst_address, 6); + clib_memcpy_fast (t->src, h1->src_address, 6); + clib_memcpy_fast (t->dst, h1->dst_address, 6); } if (b2->flags & VLIB_BUFFER_IS_TRACED) { @@ -384,8 +384,8 @@ l2input_node_inline (vlib_main_t * vm, l2input_trace_t *t = vlib_add_trace (vm, node, b2, sizeof (*t)); t->sw_if_index = sw_if_index2; - clib_memcpy (t->src, h2->src_address, 6); - clib_memcpy (t->dst, h2->dst_address, 6); + clib_memcpy_fast (t->src, h2->src_address, 6); + clib_memcpy_fast (t->dst, h2->dst_address, 6); } if (b3->flags & VLIB_BUFFER_IS_TRACED) { @@ -393,8 +393,8 @@ l2input_node_inline (vlib_main_t * vm, l2input_trace_t *t = vlib_add_trace (vm, node, b3, sizeof (*t)); t->sw_if_index = sw_if_index3; - clib_memcpy (t->src, h3->src_address, 6); - clib_memcpy (t->dst, h3->dst_address, 6); + clib_memcpy_fast (t->src, h3->src_address, 6); + clib_memcpy_fast (t->dst, h3->dst_address, 6); } } @@ -434,8 +434,8 @@ l2input_node_inline (vlib_main_t * vm, l2input_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); } classify_and_dispatch (msm, b0, &next0); diff --git a/src/vnet/l2/l2_input_vtr.c b/src/vnet/l2/l2_input_vtr.c index 9470752f0e0..25b3bd03e65 100644 --- a/src/vnet/l2/l2_input_vtr.c +++ b/src/vnet/l2/l2_input_vtr.c @@ -222,9 +222,9 @@ l2_invtr_node_fn (vlib_main_t * vm, vlib_add_trace (vm, node, b0, sizeof (*t)); ethernet_header_t *h0 = vlib_buffer_get_current (b0); t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); - clib_memcpy (t->raw, &h0->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->raw, &h0->type, sizeof (t->raw)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -232,9 +232,9 @@ l2_invtr_node_fn (vlib_main_t * vm, vlib_add_trace (vm, node, b1, sizeof (*t)); ethernet_header_t *h1 = vlib_buffer_get_current (b1); t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h1->src_address, 6); - clib_memcpy (t->dst, h1->dst_address, 6); - clib_memcpy (t->raw, &h1->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h1->src_address, 6); + clib_memcpy_fast (t->dst, h1->dst_address, 6); + clib_memcpy_fast (t->raw, &h1->type, sizeof (t->raw)); } } @@ -301,9 +301,9 @@ l2_invtr_node_fn (vlib_main_t * vm, vlib_add_trace (vm, node, b0, sizeof (*t)); ethernet_header_t *h0 = vlib_buffer_get_current (b0); t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); - clib_memcpy (t->raw, &h0->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->raw, &h0->type, sizeof (t->raw)); } /* verify speculative enqueue, maybe switch current next frame */ diff --git a/src/vnet/l2/l2_learn.c b/src/vnet/l2/l2_learn.c index c6ecef33b0f..e47365751c7 100644 --- a/src/vnet/l2/l2_learn.c +++ b/src/vnet/l2/l2_learn.c @@ -306,8 +306,8 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_add_trace (vm, node, b[0], sizeof (*t)); t->sw_if_index = sw_if_index0; t->bd_index = vnet_buffer (b[0])->l2.bd_index; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); } if (b[1]->flags & VLIB_BUFFER_IS_TRACED) { @@ -315,8 +315,8 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_add_trace (vm, node, b[1], sizeof (*t)); t->sw_if_index = sw_if_index1; t->bd_index = vnet_buffer (b[1])->l2.bd_index; - clib_memcpy (t->src, h1->src_address, 6); - clib_memcpy (t->dst, h1->dst_address, 6); + clib_memcpy_fast (t->src, h1->src_address, 6); + clib_memcpy_fast (t->dst, h1->dst_address, 6); } if (b[2]->flags & VLIB_BUFFER_IS_TRACED) { @@ -324,8 +324,8 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_add_trace (vm, node, b[2], sizeof (*t)); t->sw_if_index = sw_if_index2; t->bd_index = vnet_buffer (b[2])->l2.bd_index; - clib_memcpy (t->src, h2->src_address, 6); - clib_memcpy (t->dst, h2->dst_address, 6); + clib_memcpy_fast (t->src, h2->src_address, 6); + clib_memcpy_fast (t->dst, h2->dst_address, 6); } if (b[3]->flags & VLIB_BUFFER_IS_TRACED) { @@ -333,8 +333,8 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_add_trace (vm, node, b[3], sizeof (*t)); t->sw_if_index = sw_if_index3; t->bd_index = vnet_buffer (b[3])->l2.bd_index; - clib_memcpy (t->src, h3->src_address, 6); - clib_memcpy (t->dst, h3->dst_address, 6); + clib_memcpy_fast (t->src, h3->src_address, 6); + clib_memcpy_fast (t->dst, h3->dst_address, 6); } } @@ -391,8 +391,8 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, l2learn_trace_t *t = vlib_add_trace (vm, node, b[0], sizeof (*t)); t->sw_if_index = sw_if_index0; t->bd_index = vnet_buffer (b[0])->l2.bd_index; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); } /* process 1 pkt */ diff --git a/src/vnet/l2/l2_output.c b/src/vnet/l2/l2_output.c index 27b838f4378..e15f0ac45b2 100644 --- a/src/vnet/l2/l2_output.c +++ b/src/vnet/l2/l2_output.c @@ -426,9 +426,9 @@ VLIB_NODE_FN (l2output_node) (vlib_main_t * vm, vlib_add_trace (vm, node, b[0], sizeof (*t)); t->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; h = vlib_buffer_get_current (b[0]); - clib_memcpy (t->src, h->src_address, 6); - clib_memcpy (t->dst, h->dst_address, 6); - clib_memcpy (t->raw, &h->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h->src_address, 6); + clib_memcpy_fast (t->dst, h->dst_address, 6); + clib_memcpy_fast (t->raw, &h->type, sizeof (t->raw)); } /* next */ n_left--; diff --git a/src/vnet/l2/l2_vtr.c b/src/vnet/l2/l2_vtr.c index 7096891381d..aa3d5c45bee 100644 --- a/src/vnet/l2/l2_vtr.c +++ b/src/vnet/l2/l2_vtr.c @@ -91,10 +91,10 @@ l2pbb_configure (vlib_main_t * vlib_main, } else if (vtr_op == L2_VTR_PUSH_2) { - clib_memcpy (in_config->macs_tags.b_dst_address, b_dmac, - sizeof (in_config->macs_tags.b_dst_address)); - clib_memcpy (in_config->macs_tags.b_src_address, b_smac, - sizeof (in_config->macs_tags.b_src_address)); + clib_memcpy_fast (in_config->macs_tags.b_dst_address, b_dmac, + sizeof (in_config->macs_tags.b_dst_address)); + clib_memcpy_fast (in_config->macs_tags.b_src_address, b_smac, + sizeof (in_config->macs_tags.b_src_address)); in_config->macs_tags.b_type = clib_net_to_host_u16 (ETHERNET_TYPE_DOT1AD); in_config->macs_tags.priority_dei_id = @@ -726,10 +726,12 @@ l2pbb_get (vlib_main_t * vlib_main, vnet_main_t * vnet_main, u32 sw_if_index, else if (in_config->push_bytes) *vtr_op = L2_VTR_PUSH_2; - clib_memcpy (ð_hdr->dst_address, in_config->macs_tags.b_dst_address, - sizeof (eth_hdr->dst_address)); - clib_memcpy (ð_hdr->src_address, in_config->macs_tags.b_src_address, - sizeof (eth_hdr->src_address)); + clib_memcpy_fast (ð_hdr->dst_address, + in_config->macs_tags.b_dst_address, + sizeof (eth_hdr->dst_address)); + clib_memcpy_fast (ð_hdr->src_address, + in_config->macs_tags.b_src_address, + sizeof (eth_hdr->src_address)); *b_vlanid = clib_host_to_net_u16 (in_config->macs_tags.priority_dei_id) & 0xFFF; diff --git a/src/vnet/lisp-gpe/lisp_gpe.c b/src/vnet/lisp-gpe/lisp_gpe.c index 367dbae4159..9a10836707c 100644 --- a/src/vnet/lisp-gpe/lisp_gpe.c +++ b/src/vnet/lisp-gpe/lisp_gpe.c @@ -669,7 +669,7 @@ lisp_gpe_test_send_nsh_packet (u8 * file_name) vnet_buffer (b)->sw_if_index[VLIB_TX] = hi->sw_if_index; u8 *p = vlib_buffer_put_uninit (b, vec_len (pm.packets_read[0])); - clib_memcpy (p, pm.packets_read[0], vec_len (pm.packets_read[0])); + clib_memcpy_fast (p, pm.packets_read[0], vec_len (pm.packets_read[0])); vlib_buffer_pull (b, sizeof (ethernet_header_t)); vlib_node_t *n = vlib_get_node_by_name (lgm->vlib_main, diff --git a/src/vnet/lldp/lldp_node.c b/src/vnet/lldp/lldp_node.c index 2d169b98924..043ca9f01b5 100644 --- a/src/vnet/lldp/lldp_node.c +++ b/src/vnet/lldp/lldp_node.c @@ -87,7 +87,7 @@ lldp_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, len = (b0->current_length < sizeof (t0->data)) ? b0->current_length : sizeof (t0->data); t0->len = len; - clib_memcpy (t0->data, vlib_buffer_get_current (b0), len); + clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0), len); } /* push this pkt to the next graph node, always error-drop */ vlib_set_next_frame_buffer (vm, node, next0, bi0); diff --git a/src/vnet/lldp/lldp_output.c b/src/vnet/lldp/lldp_output.c index 4ae25a45b7b..9c171c5da7b 100644 --- a/src/vnet/lldp/lldp_output.c +++ b/src/vnet/lldp/lldp_output.c @@ -27,7 +27,7 @@ lldp_build_mgmt_addr_tlv (u8 ** t0p, u8 subtype, u8 addr_len, u8 * addr, lldp_tlv_set_code (t, LLDP_TLV_NAME (mgmt_addr)); t->v[0] = addr_len + 1; /* address string length */ t->v[1] = subtype; /* address subtype */ - clib_memcpy (&(t->v[2]), addr, addr_len); /* address */ + clib_memcpy_fast (&(t->v[2]), addr, addr_len); /* address */ t->v[addr_len + 2] = 2; /* interface numbering subtype: ifIndex */ t->v[addr_len + 3] = (if_index >> 24) & 0xFF; /* interface number */ t->v[addr_len + 4] = (if_index >> 16) & 0xFF; @@ -36,7 +36,7 @@ lldp_build_mgmt_addr_tlv (u8 ** t0p, u8 subtype, u8 addr_len, u8 * addr, t->v[addr_len + 7] = oid_len; /* OID string length */ if (oid_len > 0) - clib_memcpy ((u8 *) & (t->v[addr_len + 8]), oid, oid_len); + clib_memcpy_fast ((u8 *) & (t->v[addr_len + 8]), oid, oid_len); lldp_tlv_set_length (t, addr_len + oid_len + 8); *t0p += STRUCT_SIZE_OF (lldp_tlv_t, head) + addr_len + oid_len + 8; @@ -51,7 +51,7 @@ lldp_add_chassis_id (const vnet_hw_interface_t * hw, u8 ** t0p) t->subtype = LLDP_CHASS_ID_SUBTYPE_NAME (mac_addr); const size_t addr_len = 6; - clib_memcpy (&t->id, hw->hw_address, addr_len); + clib_memcpy_fast (&t->id, hw->hw_address, addr_len); const size_t len = STRUCT_SIZE_OF (lldp_chassis_id_tlv_t, subtype) + addr_len; lldp_tlv_set_length ((lldp_tlv_t *) t, len); @@ -67,7 +67,7 @@ lldp_add_port_id (const vnet_hw_interface_t * hw, u8 ** t0p) t->subtype = LLDP_PORT_ID_SUBTYPE_NAME (intf_name); const size_t name_len = vec_len (hw->name); - clib_memcpy (&t->id, hw->name, name_len); + clib_memcpy_fast (&t->id, hw->name, name_len); const size_t len = STRUCT_SIZE_OF (lldp_port_id_tlv_t, subtype) + name_len; lldp_tlv_set_length ((lldp_tlv_t *) t, len); *t0p += STRUCT_SIZE_OF (lldp_tlv_t, head) + len; @@ -107,7 +107,7 @@ lldp_add_port_desc (const lldp_main_t * lm, lldp_intf_t * n, u8 ** t0p) lldp_tlv_t *t = (lldp_tlv_t *) * t0p; lldp_tlv_set_code (t, LLDP_TLV_NAME (port_desc)); lldp_tlv_set_length (t, len); - clib_memcpy (t->v, n->port_desc, len); + clib_memcpy_fast (t->v, n->port_desc, len); *t0p += STRUCT_SIZE_OF (lldp_tlv_t, head) + len; } } @@ -121,7 +121,7 @@ lldp_add_sys_name (const lldp_main_t * lm, u8 ** t0p) lldp_tlv_t *t = (lldp_tlv_t *) * t0p; lldp_tlv_set_code (t, LLDP_TLV_NAME (sys_name)); lldp_tlv_set_length (t, len); - clib_memcpy (t->v, lm->sys_name, len); + clib_memcpy_fast (t->v, lm->sys_name, len); *t0p += STRUCT_SIZE_OF (lldp_tlv_t, head) + len; } } @@ -220,7 +220,8 @@ lldp_send_ethernet (lldp_main_t * lm, lldp_intf_t * n, int shutdown) /* Add the interface's ethernet source address */ hw = vnet_get_hw_interface (vnm, n->hw_if_index); - clib_memcpy (h0->src_address, hw->hw_address, vec_len (hw->hw_address)); + clib_memcpy_fast (h0->src_address, hw->hw_address, + vec_len (hw->hw_address)); u8 *data = ((u8 *) h0) + sizeof (*h0); t0 = data; diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c index 82c6c81cf18..8b46688929f 100644 --- a/src/vnet/pg/input.c +++ b/src/vnet/pg/input.c @@ -1091,7 +1091,7 @@ init_replay_buffers_inline (vlib_main_t * vm, b0->current_length = n0; - clib_memcpy (b0->data, d0 + data_offset, n0); + clib_memcpy_fast (b0->data, d0 + data_offset, n0); i = i + 1 == l ? 0 : i + 1; } } @@ -1148,8 +1148,8 @@ init_buffers_inline (vlib_main_t * vm, if (set_data) { - clib_memcpy (b0->data, data, n_data); - clib_memcpy (b1->data, data, n_data); + clib_memcpy_fast (b0->data, data, n_data); + clib_memcpy_fast (b1->data, data, n_data); } else { @@ -1173,7 +1173,7 @@ init_buffers_inline (vlib_main_t * vm, vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; if (set_data) - clib_memcpy (b0->data, data, n_data); + clib_memcpy_fast (b0->data, data, n_data); else ASSERT (validate_buffer_data2 (b0, s, data_offset, n_data)); } @@ -1424,13 +1424,15 @@ pg_input_trace (pg_main_t * pg, t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX]; - clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data)); - clib_memcpy (&t1->buffer, b1, sizeof (b1[0]) - sizeof (b1->pre_data)); + clib_memcpy_fast (&t0->buffer, b0, + sizeof (b0[0]) - sizeof (b0->pre_data)); + clib_memcpy_fast (&t1->buffer, b1, + sizeof (b1[0]) - sizeof (b1->pre_data)); - clib_memcpy (t0->buffer.pre_data, b0->data, - sizeof (t0->buffer.pre_data)); - clib_memcpy (t1->buffer.pre_data, b1->data, - sizeof (t1->buffer.pre_data)); + clib_memcpy_fast (t0->buffer.pre_data, b0->data, + sizeof (t0->buffer.pre_data)); + clib_memcpy_fast (t1->buffer.pre_data, b1->data, + sizeof (t1->buffer.pre_data)); } while (n_left >= 1) @@ -1451,9 +1453,10 @@ pg_input_trace (pg_main_t * pg, t0->stream_index = stream_index; t0->packet_length = vlib_buffer_length_in_chain (vm, b0); t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data)); - clib_memcpy (t0->buffer.pre_data, b0->data, - sizeof (t0->buffer.pre_data)); + clib_memcpy_fast (&t0->buffer, b0, + sizeof (b0[0]) - sizeof (b0->pre_data)); + clib_memcpy_fast (t0->buffer.pre_data, b0->data, + sizeof (t0->buffer.pre_data)); } } diff --git a/src/vnet/pg/output.c b/src/vnet/pg/output.c index 410a433621c..a84f30301dc 100644 --- a/src/vnet/pg/output.c +++ b/src/vnet/pg/output.c @@ -68,9 +68,10 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { pg_output_trace_t *t = vlib_add_trace (vm, node, b, sizeof (*t)); t->buffer_index = bi0; - clib_memcpy (&t->buffer, b, sizeof (b[0]) - sizeof (b->pre_data)); - clib_memcpy (t->buffer.pre_data, b->data + b->current_data, - sizeof (t->buffer.pre_data)); + clib_memcpy_fast (&t->buffer, b, + sizeof (b[0]) - sizeof (b->pre_data)); + clib_memcpy_fast (t->buffer.pre_data, b->data + b->current_data, + sizeof (t->buffer.pre_data)); } if (pif->pcap_file_name != 0) diff --git a/src/vnet/pg/stream.c b/src/vnet/pg/stream.c index 2837d800421..bf0eac370aa 100644 --- a/src/vnet/pg/stream.c +++ b/src/vnet/pg/stream.c @@ -205,7 +205,7 @@ pg_interface_add_or_get (pg_main_t * pg, uword if_id) rnd = (u32) (now * 1e6); rnd = random_u32 (&rnd); - clib_memcpy (hw_addr + 2, &rnd, sizeof (rnd)); + clib_memcpy_fast (hw_addr + 2, &rnd, sizeof (rnd)); hw_addr[0] = 2; hw_addr[1] = 0xfe; @@ -333,10 +333,10 @@ pg_edit_group_get_fixed_packet_data (pg_stream_t * s, vec_foreach (e, g->edits) do_edit (s, g, e, /* want_commit */ 0); - clib_memcpy (packet_data, g->fixed_packet_data, - vec_len (g->fixed_packet_data)); - clib_memcpy (packet_data_mask, g->fixed_packet_data_mask, - vec_len (g->fixed_packet_data_mask)); + clib_memcpy_fast (packet_data, g->fixed_packet_data, + vec_len (g->fixed_packet_data)); + clib_memcpy_fast (packet_data_mask, g->fixed_packet_data_mask, + vec_len (g->fixed_packet_data_mask)); } static void diff --git a/src/vnet/sctp/sctp_input.c b/src/vnet/sctp/sctp_input.c index e548707d32a..c58da9ef9b5 100644 --- a/src/vnet/sctp/sctp_input.c +++ b/src/vnet/sctp/sctp_input.c @@ -245,15 +245,15 @@ sctp_set_rx_trace_data (sctp_rx_trace_t * rx_trace, { if (sctp_conn) { - clib_memcpy (&rx_trace->sctp_connection, sctp_conn, - sizeof (rx_trace->sctp_connection)); + clib_memcpy_fast (&rx_trace->sctp_connection, sctp_conn, + sizeof (rx_trace->sctp_connection)); } else { sctp_hdr = sctp_buffer_hdr (b0); } - clib_memcpy (&rx_trace->sctp_header, sctp_hdr, - sizeof (rx_trace->sctp_header)); + clib_memcpy_fast (&rx_trace->sctp_header, sctp_hdr, + sizeof (rx_trace->sctp_header)); } always_inline u16 @@ -398,8 +398,8 @@ sctp_handle_init (sctp_header_t * sctp_hdr, { sctp_ipv4_addr_param_t *ipv4 = (sctp_ipv4_addr_param_t *) opt_params_hdr; - clib_memcpy (&ip4_addr, &ipv4->address, - sizeof (ip4_address_t)); + clib_memcpy_fast (&ip4_addr, &ipv4->address, + sizeof (ip4_address_t)); if (sctp_sub_connection_add_ip4 (vlib_get_main (), &sctp_conn->sub_conn @@ -415,8 +415,8 @@ sctp_handle_init (sctp_header_t * sctp_hdr, { sctp_ipv6_addr_param_t *ipv6 = (sctp_ipv6_addr_param_t *) opt_params_hdr; - clib_memcpy (&ip6_addr, &ipv6->address, - sizeof (ip6_address_t)); + clib_memcpy_fast (&ip6_addr, &ipv6->address, + sizeof (ip6_address_t)); if (sctp_sub_connection_add_ip6 (vlib_get_main (), &sctp_conn->sub_conn @@ -440,8 +440,8 @@ sctp_handle_init (sctp_header_t * sctp_hdr, { sctp_hostname_param_t *hostname_addr = (sctp_hostname_param_t *) opt_params_hdr; - clib_memcpy (hostname, hostname_addr->hostname, - FQDN_MAX_LENGTH); + clib_memcpy_fast (hostname, hostname_addr->hostname, + FQDN_MAX_LENGTH); break; } case SCTP_SUPPORTED_ADDRESS_TYPES: @@ -568,8 +568,9 @@ sctp_handle_init_ack (sctp_header_t * sctp_hdr, sctp_state_cookie_param_t *state_cookie_param = (sctp_state_cookie_param_t *) opt_params_hdr; - clib_memcpy (&(sctp_conn->cookie_param), state_cookie_param, - sizeof (sctp_state_cookie_param_t)); + clib_memcpy_fast (&(sctp_conn->cookie_param), + state_cookie_param, + sizeof (sctp_state_cookie_param_t)); break; } @@ -577,8 +578,8 @@ sctp_handle_init_ack (sctp_header_t * sctp_hdr, { sctp_hostname_param_t *hostname_addr = (sctp_hostname_param_t *) opt_params_hdr; - clib_memcpy (hostname, hostname_addr->hostname, - FQDN_MAX_LENGTH); + clib_memcpy_fast (hostname, hostname_addr->hostname, + FQDN_MAX_LENGTH); break; } case SCTP_UNRECOGNIZED_TYPE: @@ -1035,8 +1036,8 @@ sctp46_rcv_phase_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (error0 == SCTP_ERROR_NONE) { pool_get (tm->connections[my_thread_index], new_sctp_conn); - clib_memcpy (new_sctp_conn, sctp_conn, - sizeof (*new_sctp_conn)); + clib_memcpy_fast (new_sctp_conn, sctp_conn, + sizeof (*new_sctp_conn)); new_sctp_conn->sub_conn[idx].c_c_index = new_sctp_conn - tm->connections[my_thread_index]; new_sctp_conn->sub_conn[idx].c_thread_index = @@ -1447,12 +1448,12 @@ sctp46_shutdown_phase_inline (vlib_main_t * vm, vlib_add_trace (vm, node, b0, sizeof (*sctp_trace)); if (sctp_hdr != NULL) - clib_memcpy (&sctp_trace->sctp_header, sctp_hdr, - sizeof (sctp_trace->sctp_header)); + clib_memcpy_fast (&sctp_trace->sctp_header, sctp_hdr, + sizeof (sctp_trace->sctp_header)); if (sctp_conn != NULL) - clib_memcpy (&sctp_trace->sctp_connection, sctp_conn, - sizeof (sctp_trace->sctp_connection)); + clib_memcpy_fast (&sctp_trace->sctp_connection, sctp_conn, + sizeof (sctp_trace->sctp_connection)); } b0->error = node->errors[error0]; @@ -1718,12 +1719,14 @@ sctp46_listen_process_inline (vlib_main_t * vm, } else { - clib_memcpy (&child_conn-> - sub_conn[SCTP_PRIMARY_PATH_IDX].c_lcl_ip6, - &ip6_hdr->dst_address, sizeof (ip6_address_t)); - clib_memcpy (&child_conn-> - sub_conn[SCTP_PRIMARY_PATH_IDX].c_rmt_ip6, - &ip6_hdr->src_address, sizeof (ip6_address_t)); + clib_memcpy_fast (&child_conn-> + sub_conn[SCTP_PRIMARY_PATH_IDX].c_lcl_ip6, + &ip6_hdr->dst_address, + sizeof (ip6_address_t)); + clib_memcpy_fast (&child_conn-> + sub_conn[SCTP_PRIMARY_PATH_IDX].c_rmt_ip6, + &ip6_hdr->src_address, + sizeof (ip6_address_t)); } sctp_full_hdr_t *full_hdr = (sctp_full_hdr_t *) sctp_hdr; @@ -1795,10 +1798,10 @@ sctp46_listen_process_inline (vlib_main_t * vm, { sctp_rx_trace_t *t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); - clib_memcpy (&t0->sctp_header, sctp_hdr, - sizeof (t0->sctp_header)); - clib_memcpy (&t0->sctp_connection, sctp_listener, - sizeof (t0->sctp_connection)); + clib_memcpy_fast (&t0->sctp_header, sctp_hdr, + sizeof (t0->sctp_header)); + clib_memcpy_fast (&t0->sctp_connection, sctp_listener, + sizeof (t0->sctp_connection)); } b0->error = node->errors[error0]; diff --git a/src/vnet/sctp/sctp_output.c b/src/vnet/sctp/sctp_output.c index aa0bb41e504..c0e1a6eb339 100644 --- a/src/vnet/sctp/sctp_output.c +++ b/src/vnet/sctp/sctp_output.c @@ -584,8 +584,8 @@ sctp_prepare_cookie_echo_chunk (sctp_connection_t * sctp_conn, u8 idx, cookie_echo_chunk->sctp_hdr.verification_tag = sctp_conn->remote_tag; vnet_sctp_set_chunk_type (&cookie_echo_chunk->chunk_hdr, COOKIE_ECHO); vnet_sctp_set_chunk_length (&cookie_echo_chunk->chunk_hdr, chunk_len); - clib_memcpy (&(cookie_echo_chunk->cookie), &sctp_conn->cookie_param, - sizeof (sctp_state_cookie_param_t)); + clib_memcpy_fast (&(cookie_echo_chunk->cookie), &sctp_conn->cookie_param, + sizeof (sctp_state_cookie_param_t)); vnet_buffer (b)->sctp.connection_index = sctp_conn->sub_conn[idx].connection.c_index; @@ -1850,15 +1850,15 @@ sctp46_output_inline (vlib_main_t * vm, t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); if (th0) { - clib_memcpy (&t0->sctp_header, th0, - sizeof (t0->sctp_header)); + clib_memcpy_fast (&t0->sctp_header, th0, + sizeof (t0->sctp_header)); } else { clib_memset (&t0->sctp_header, 0, sizeof (t0->sctp_header)); } - clib_memcpy (&t0->sctp_connection, sctp_conn, - sizeof (t0->sctp_connection)); + clib_memcpy_fast (&t0->sctp_connection, sctp_conn, + sizeof (t0->sctp_connection)); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, diff --git a/src/vnet/session-apps/echo_client.c b/src/vnet/session-apps/echo_client.c index 7935eb8b242..faf8521ce8c 100644 --- a/src/vnet/session-apps/echo_client.c +++ b/src/vnet/session-apps/echo_client.c @@ -86,10 +86,12 @@ send_data_chunk (echo_client_main_t * ecm, eclient_session_t * s) hdr.data_length = rv; hdr.data_offset = 0; - clib_memcpy (&hdr.rmt_ip, &at->rmt_ip, sizeof (ip46_address_t)); + clib_memcpy_fast (&hdr.rmt_ip, &at->rmt_ip, + sizeof (ip46_address_t)); hdr.is_ip4 = at->is_ip4; hdr.rmt_port = at->rmt_port; - clib_memcpy (&hdr.lcl_ip, &at->lcl_ip, sizeof (ip46_address_t)); + clib_memcpy_fast (&hdr.lcl_ip, &at->lcl_ip, + sizeof (ip46_address_t)); hdr.lcl_port = at->lcl_port; svm_fifo_enqueue_nowait (f, sizeof (hdr), (u8 *) & hdr); svm_fifo_enqueue_nocopy (f, rv); @@ -221,10 +223,10 @@ echo_client_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, ASSERT (nconnections_this_batch > 0); vec_validate (connections_this_batch, nconnections_this_batch - 1); - clib_memcpy (connections_this_batch, - connection_indices + vec_len (connection_indices) - - nconnections_this_batch, - nconnections_this_batch * sizeof (u32)); + clib_memcpy_fast (connections_this_batch, + connection_indices + vec_len (connection_indices) + - nconnections_this_batch, + nconnections_this_batch * sizeof (u32)); _vec_len (connection_indices) -= nconnections_this_batch; } @@ -402,8 +404,8 @@ echo_clients_session_connected_callback (u32 app_index, u32 api_context, { transport_connection_t *tc; tc = session_get_transport (s); - clib_memcpy (&session->data.transport, tc, - sizeof (session->data.transport)); + clib_memcpy_fast (&session->data.transport, tc, + sizeof (session->data.transport)); session->data.is_dgram = 1; } diff --git a/src/vnet/session-apps/echo_server.c b/src/vnet/session-apps/echo_server.c index 77710752a16..a4afc5d512f 100644 --- a/src/vnet/session-apps/echo_server.c +++ b/src/vnet/session-apps/echo_server.c @@ -330,13 +330,13 @@ echo_server_attach (u8 * appns_id, u64 appns_flags, u64 appns_secret) clib_memset (a_cert, 0, sizeof (*a_cert)); a_cert->app_index = a->app_index; vec_validate (a_cert->cert, test_srv_crt_rsa_len); - clib_memcpy (a_cert->cert, test_srv_crt_rsa, test_srv_crt_rsa_len); + clib_memcpy_fast (a_cert->cert, test_srv_crt_rsa, test_srv_crt_rsa_len); vnet_app_add_tls_cert (a_cert); clib_memset (a_key, 0, sizeof (*a_key)); a_key->app_index = a->app_index; vec_validate (a_key->key, test_srv_key_rsa_len); - clib_memcpy (a_key->key, test_srv_key_rsa, test_srv_key_rsa_len); + clib_memcpy_fast (a_key->key, test_srv_key_rsa, test_srv_key_rsa_len); vnet_app_add_tls_key (a_key); return 0; } diff --git a/src/vnet/session-apps/http_server.c b/src/vnet/session-apps/http_server.c index 8501cb4ee70..79c2d4a7bfc 100644 --- a/src/vnet/session-apps/http_server.c +++ b/src/vnet/session-apps/http_server.c @@ -132,7 +132,7 @@ http_cli_output (uword arg, u8 * buffer, uword buffer_bytes) offset = vec_len (output_vec); vec_validate (output_vec, offset + buffer_bytes - 1); - clib_memcpy (output_vec + offset, buffer, buffer_bytes); + clib_memcpy_fast (output_vec + offset, buffer, buffer_bytes); *output_vecp = output_vec; } @@ -525,13 +525,13 @@ server_attach () clib_memset (a_cert, 0, sizeof (*a_cert)); a_cert->app_index = a->app_index; vec_validate (a_cert->cert, test_srv_crt_rsa_len); - clib_memcpy (a_cert->cert, test_srv_crt_rsa, test_srv_crt_rsa_len); + clib_memcpy_fast (a_cert->cert, test_srv_crt_rsa, test_srv_crt_rsa_len); vnet_app_add_tls_cert (a_cert); clib_memset (a_key, 0, sizeof (*a_key)); a_key->app_index = a->app_index; vec_validate (a_key->key, test_srv_key_rsa_len); - clib_memcpy (a_key->key, test_srv_key_rsa, test_srv_key_rsa_len); + clib_memcpy_fast (a_key->key, test_srv_key_rsa, test_srv_key_rsa_len); vnet_app_add_tls_key (a_key); return 0; diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h index 2da69fcab92..dc4c46906c4 100644 --- a/src/vnet/session/application_interface.h +++ b/src/vnet/session/application_interface.h @@ -397,10 +397,10 @@ app_send_dgram_raw (svm_fifo_t * f, app_session_transport_t * at, actual_write = clib_min (len, max_enqueue); hdr.data_length = actual_write; hdr.data_offset = 0; - clib_memcpy (&hdr.rmt_ip, &at->rmt_ip, sizeof (ip46_address_t)); + clib_memcpy_fast (&hdr.rmt_ip, &at->rmt_ip, sizeof (ip46_address_t)); hdr.is_ip4 = at->is_ip4; hdr.rmt_port = at->rmt_port; - clib_memcpy (&hdr.lcl_ip, &at->lcl_ip, sizeof (ip46_address_t)); + clib_memcpy_fast (&hdr.lcl_ip, &at->lcl_ip, sizeof (ip46_address_t)); hdr.lcl_port = at->lcl_port; rv = svm_fifo_enqueue_nowait (f, sizeof (hdr), (u8 *) & hdr); ASSERT (rv == sizeof (hdr)); diff --git a/src/vnet/session/mma_template.c b/src/vnet/session/mma_template.c index 15fd7c7aeed..9dd7c76cb06 100644 --- a/src/vnet/session/mma_template.c +++ b/src/vnet/session/mma_template.c @@ -238,7 +238,7 @@ RT (mma_rules_table_del_rule) (RTT (mma_rules_table) * srt, if (i != 0) { vec_add2 (next_indices, new_elts, i); - clib_memcpy (new_elts, rp->next_indices, i * sizeof (u32)); + clib_memcpy_fast (new_elts, rp->next_indices, i * sizeof (u32)); } if (vec_len (child->next_indices)) vec_append (next_indices, child->next_indices); @@ -246,8 +246,8 @@ RT (mma_rules_table_del_rule) (RTT (mma_rules_table) * srt, if (left_to_add) { vec_add2 (next_indices, new_elts, left_to_add); - clib_memcpy (new_elts, &rp->next_indices[i + 1], - left_to_add * sizeof (u32)); + clib_memcpy_fast (new_elts, &rp->next_indices[i + 1], + left_to_add * sizeof (u32)); } RT (mma_rule_free) (srt, child); vec_free (rp->next_indices); diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h index 35c8fb4069f..ea21464ee54 100644 --- a/src/vnet/session/session.h +++ b/src/vnet/session/session.h @@ -563,7 +563,7 @@ session_clone_safe (u32 session_index, u32 thread_index) session_pool_add_peeker (thread_index); new_s = session_alloc (current_thread_index); old_s = session_manager_main.wrk[thread_index].sessions + session_index; - clib_memcpy (new_s, old_s, sizeof (*new_s)); + clib_memcpy_fast (new_s, old_s, sizeof (*new_s)); session_pool_remove_peeker (thread_index); new_s->thread_index = current_thread_index; new_s->session_index = session_get_index (new_s); diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c index eb33db03f33..8315c19a7ac 100755 --- a/src/vnet/session/session_api.c +++ b/src/vnet/session/session_api.c @@ -241,7 +241,7 @@ send_session_accept_callback (stream_session_t * s) tc = tp_vft->get_connection (s->connection_index, s->thread_index); mp->port = tc->rmt_port; mp->is_ip4 = tc->is_ip4; - clib_memcpy (&mp->ip, &tc->rmt_ip, sizeof (tc->rmt_ip)); + clib_memcpy_fast (&mp->ip, &tc->rmt_ip, sizeof (tc->rmt_ip)); } else { @@ -364,7 +364,7 @@ send_session_connected_callback (u32 app_wrk_index, u32 api_context, vpp_queue = session_manager_get_vpp_event_queue (s->thread_index); mp->handle = session_handle (s); mp->vpp_event_queue_address = pointer_to_uword (vpp_queue); - clib_memcpy (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); + clib_memcpy_fast (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); mp->is_ip4 = tc->is_ip4; mp->lcl_port = tc->lcl_port; mp->server_rx_fifo = pointer_to_uword (s->server_rx_fifo); @@ -459,7 +459,7 @@ mq_send_session_accepted_cb (stream_session_t * s) tc = tp_vft->get_connection (s->connection_index, s->thread_index); mp->port = tc->rmt_port; mp->is_ip4 = tc->is_ip4; - clib_memcpy (&mp->ip, &tc->rmt_ip, sizeof (tc->rmt_ip)); + clib_memcpy_fast (&mp->ip, &tc->rmt_ip, sizeof (tc->rmt_ip)); } else { @@ -616,7 +616,7 @@ mq_send_session_connected_cb (u32 app_wrk_index, u32 api_context, vpp_mq = session_manager_get_vpp_event_queue (s->thread_index); mp->handle = session_handle (s); mp->vpp_event_queue_address = pointer_to_uword (vpp_mq); - clib_memcpy (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); + clib_memcpy_fast (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); mp->is_ip4 = tc->is_ip4; mp->lcl_port = tc->lcl_port; mp->server_rx_fifo = pointer_to_uword (s->server_rx_fifo); @@ -692,7 +692,7 @@ mq_send_session_bound_cb (u32 app_wrk_index, u32 api_context, tc = listen_session_get_transport (ls); mp->lcl_port = tc->lcl_port; mp->lcl_is_ip4 = tc->is_ip4; - clib_memcpy (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); + clib_memcpy_fast (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); } else { @@ -779,7 +779,8 @@ vl_api_application_attach_t_handler (vl_api_application_attach_t * mp) if (mp->namespace_id_len) { vec_validate (a->namespace_id, mp->namespace_id_len - 1); - clib_memcpy (a->namespace_id, mp->namespace_id, mp->namespace_id_len); + clib_memcpy_fast (a->namespace_id, mp->namespace_id, + mp->namespace_id_len); } if ((error = vnet_application_attach (a))) @@ -906,7 +907,7 @@ done: tc = listen_session_get_transport (s); rmp->lcl_is_ip4 = tc->is_ip4; rmp->lcl_port = tc->lcl_port; - clib_memcpy (rmp->lcl_ip, &tc->lcl_ip, sizeof(tc->lcl_ip)); + clib_memcpy_fast (rmp->lcl_ip, &tc->lcl_ip, sizeof(tc->lcl_ip)); if (session_transport_service_type (s) == TRANSPORT_SERVICE_CL) { rmp->rx_fifo = pointer_to_uword (s->server_rx_fifo); @@ -1208,7 +1209,7 @@ done: { s = listen_session_get_from_handle (a->handle); tc = listen_session_get_transport (s); - clib_memcpy (rmp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); + clib_memcpy_fast (rmp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); if (session_transport_service_type (s) == TRANSPORT_SERVICE_CL) { rmp->rx_fifo = pointer_to_uword (s->server_rx_fifo); @@ -1295,7 +1296,8 @@ vl_api_connect_sock_t_handler (vl_api_connect_sock_t * mp) if (mp->hostname_len) { vec_validate (a->sep_ext.hostname, mp->hostname_len - 1); - clib_memcpy (a->sep_ext.hostname, mp->hostname, mp->hostname_len); + clib_memcpy_fast (a->sep_ext.hostname, mp->hostname, + mp->hostname_len); } a->api_context = mp->context; a->app_index = app->app_index; @@ -1429,7 +1431,7 @@ vl_api_app_namespace_add_del_t_handler (vl_api_app_namespace_add_del_t * mp) } vec_validate (ns_id, mp->namespace_id_len - 1); - clib_memcpy (ns_id, mp->namespace_id, mp->namespace_id_len); + clib_memcpy_fast (ns_id, mp->namespace_id, mp->namespace_id_len); vnet_app_namespace_add_del_args_t args = { .ns_id = ns_id, .secret = clib_net_to_host_u64 (mp->secret), @@ -1522,8 +1524,8 @@ send_session_rule_details4 (mma_rule_16_t * rule, u8 is_local, rmp->context = context; rmp->is_ip4 = 1; - clib_memcpy (rmp->lcl_ip, &match->lcl_ip, sizeof (match->lcl_ip)); - clib_memcpy (rmp->rmt_ip, &match->rmt_ip, sizeof (match->rmt_ip)); + clib_memcpy_fast (rmp->lcl_ip, &match->lcl_ip, sizeof (match->lcl_ip)); + clib_memcpy_fast (rmp->rmt_ip, &match->rmt_ip, sizeof (match->rmt_ip)); rmp->lcl_plen = ip4_mask_to_preflen (&mask->lcl_ip); rmp->rmt_plen = ip4_mask_to_preflen (&mask->rmt_ip); rmp->lcl_port = match->lcl_port; @@ -1535,7 +1537,7 @@ send_session_rule_details4 (mma_rule_16_t * rule, u8 is_local, rmp->appns_index = clib_host_to_net_u32 (appns_index); if (tag) { - clib_memcpy (rmp->tag, tag, vec_len (tag)); + clib_memcpy_fast (rmp->tag, tag, vec_len (tag)); rmp->tag[vec_len (tag)] = 0; } @@ -1559,8 +1561,8 @@ send_session_rule_details6 (mma_rule_40_t * rule, u8 is_local, rmp->context = context; rmp->is_ip4 = 0; - clib_memcpy (rmp->lcl_ip, &match->lcl_ip, sizeof (match->lcl_ip)); - clib_memcpy (rmp->rmt_ip, &match->rmt_ip, sizeof (match->rmt_ip)); + clib_memcpy_fast (rmp->lcl_ip, &match->lcl_ip, sizeof (match->lcl_ip)); + clib_memcpy_fast (rmp->rmt_ip, &match->rmt_ip, sizeof (match->rmt_ip)); rmp->lcl_plen = ip6_mask_to_preflen (&mask->lcl_ip); rmp->rmt_plen = ip6_mask_to_preflen (&mask->rmt_ip); rmp->lcl_port = match->lcl_port; @@ -1572,7 +1574,7 @@ send_session_rule_details6 (mma_rule_40_t * rule, u8 is_local, rmp->appns_index = clib_host_to_net_u32 (appns_index); if (tag) { - clib_memcpy (rmp->tag, tag, vec_len (tag)); + clib_memcpy_fast (rmp->tag, tag, vec_len (tag)); rmp->tag[vec_len (tag)] = 0; } @@ -1671,7 +1673,7 @@ vl_api_application_tls_cert_add_t_handler (vl_api_application_tls_cert_add_t * goto done; } vec_validate (a->cert, cert_len); - clib_memcpy (a->cert, mp->cert, cert_len); + clib_memcpy_fast (a->cert, mp->cert, cert_len); if ((error = vnet_app_add_tls_cert (a))) { rv = clib_error_get_code (error); @@ -1711,7 +1713,7 @@ vl_api_application_tls_key_add_t_handler (vl_api_application_tls_key_add_t * goto done; } vec_validate (a->key, key_len); - clib_memcpy (a->key, mp->key, key_len); + clib_memcpy_fast (a->key, mp->key, key_len); if ((error = vnet_app_add_tls_key (a))) { rv = clib_error_get_code (error); diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c index 22d8d3c45b0..cf2d5766fb0 100644 --- a/src/vnet/session/session_node.c +++ b/src/vnet/session/session_node.c @@ -800,7 +800,7 @@ session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, { vec_add2 (fifo_events, e, 1); svm_msg_q_sub_w_lock (mq, msg); - clib_memcpy (e, svm_msg_q_msg_data (mq, msg), sizeof (*e)); + clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), sizeof (*e)); svm_msg_q_free_msg (mq, msg); } svm_msg_q_unlock (mq); @@ -961,7 +961,7 @@ dump_thread_0_event_queue (void) { msg = (svm_msg_q_msg_t *) (&mq->q->data[0] + mq->q->elsize * index); ring = svm_msg_q_ring (mq, msg->ring_index); - clib_memcpy (e, svm_msg_q_msg_data (mq, msg), ring->elsize); + clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), ring->elsize); switch (e->event_type) { @@ -1053,7 +1053,7 @@ session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e) { msg = (svm_msg_q_msg_t *) (&mq->q->data[0] + mq->q->elsize * index); ring = svm_msg_q_ring (mq, msg->ring_index); - clib_memcpy (e, svm_msg_q_msg_data (mq, msg), ring->elsize); + clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), ring->elsize); found = session_node_cmp_event (e, f); if (found) return 1; @@ -1069,7 +1069,7 @@ session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e) found = session_node_cmp_event (evt, f); if (found) { - clib_memcpy (e, evt, sizeof (*evt)); + clib_memcpy_fast (e, evt, sizeof (*evt)); break; } } diff --git a/src/vnet/session/session_rules_table.c b/src/vnet/session/session_rules_table.c index 33f7de07944..6304606ca43 100644 --- a/src/vnet/session/session_rules_table.c +++ b/src/vnet/session/session_rules_table.c @@ -270,8 +270,10 @@ session_rules_table_init_rule_40 (mma_rule_40_t * rule, fib_pref_normalize (lcl); fib_pref_normalize (rmt); match = (session_mask_or_match_6_t *) & rule->match; - clib_memcpy (&match->lcl_ip, &lcl->fp_addr.ip6, sizeof (match->lcl_ip)); - clib_memcpy (&match->rmt_ip, &rmt->fp_addr.ip6, sizeof (match->rmt_ip)); + clib_memcpy_fast (&match->lcl_ip, &lcl->fp_addr.ip6, + sizeof (match->lcl_ip)); + clib_memcpy_fast (&match->rmt_ip, &rmt->fp_addr.ip6, + sizeof (match->rmt_ip)); match->lcl_port = lcl_port; match->rmt_port = rmt_port; mask = (session_mask_or_match_6_t *) & rule->mask; @@ -355,8 +357,8 @@ session_rules_table_lookup_rule6 (session_rules_table_t * srt, .lcl_port = lcl_port, .rmt_port = rmt_port, }; - clib_memcpy (&key.lcl_ip, lcl_ip, sizeof (*lcl_ip)); - clib_memcpy (&key.rmt_ip, rmt_ip, sizeof (*rmt_ip)); + clib_memcpy_fast (&key.lcl_ip, lcl_ip, sizeof (*lcl_ip)); + clib_memcpy_fast (&key.rmt_ip, rmt_ip, sizeof (*rmt_ip)); return mma_rules_table_lookup_rule_40 (srt6, (mma_mask_or_match_40_t *) & key, srt6->root_index); @@ -372,8 +374,8 @@ session_rules_table_lookup6 (session_rules_table_t * srt, .lcl_port = lcl_port, .rmt_port = rmt_port, }; - clib_memcpy (&key.lcl_ip, lcl_ip, sizeof (*lcl_ip)); - clib_memcpy (&key.rmt_ip, rmt_ip, sizeof (*rmt_ip)); + clib_memcpy_fast (&key.lcl_ip, lcl_ip, sizeof (*lcl_ip)); + clib_memcpy_fast (&key.rmt_ip, rmt_ip, sizeof (*rmt_ip)); return mma_rules_table_lookup_40 (srt6, (mma_mask_or_match_40_t *) & key, srt6->root_index); } @@ -577,8 +579,8 @@ session_rules_table_show_rule (vlib_main_t * vm, session_rules_table_t * srt, .lcl_port = lcl_port, .rmt_port = rmt_port, }; - clib_memcpy (&key.lcl_ip, &lcl_ip->ip6, sizeof (lcl_ip->ip6)); - clib_memcpy (&key.rmt_ip, &rmt_ip->ip6, sizeof (rmt_ip->ip6)); + clib_memcpy_fast (&key.lcl_ip, &lcl_ip->ip6, sizeof (lcl_ip->ip6)); + clib_memcpy_fast (&key.rmt_ip, &rmt_ip->ip6, sizeof (rmt_ip->ip6)); ri = mma_rules_table_lookup_rule_40 (srt6, (mma_mask_or_match_40_t *) & key, srt6->root_index); diff --git a/src/vnet/session/transport.c b/src/vnet/session/transport.c index 06c541a9e2b..91b80d7c11e 100644 --- a/src/vnet/session/transport.c +++ b/src/vnet/session/transport.c @@ -319,7 +319,7 @@ transport_endpoint_mark_used (u8 proto, ip46_address_t * ip, u16 port) transport_endpoint_t *tep; clib_spinlock_lock_if_init (&local_endpoints_lock); tep = transport_endpoint_new (); - clib_memcpy (&tep->ip, ip, sizeof (*ip)); + clib_memcpy_fast (&tep->ip, ip, sizeof (*ip)); tep->port = port; transport_endpoint_table_add (&local_endpoints_table, proto, tep, tep - local_endpoints); @@ -388,7 +388,7 @@ transport_get_interface_ip (u32 sw_if_index, u8 is_ip4, ip46_address_t * addr) return clib_error_return (0, "no routable ip6 addresses on %U", format_vnet_sw_if_index_name, vnet_get_main (), sw_if_index); - clib_memcpy (&addr->ip6, ip6, sizeof (*ip6)); + clib_memcpy_fast (&addr->ip6, ip6, sizeof (*ip6)); } return 0; } @@ -404,7 +404,7 @@ transport_find_local_ip_for_remote (u32 sw_if_index, if (sw_if_index == ENDPOINT_INVALID_INDEX) { /* Find a FIB path to the destination */ - clib_memcpy (&prefix.fp_addr, &rmt->ip, sizeof (rmt->ip)); + clib_memcpy_fast (&prefix.fp_addr, &rmt->ip, sizeof (rmt->ip)); prefix.fp_proto = rmt->is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6; prefix.fp_len = rmt->is_ip4 ? 32 : 128; @@ -449,7 +449,8 @@ transport_alloc_local_endpoint (u8 proto, transport_endpoint_cfg_t * rmt_cfg, else { /* Assume session layer vetted this address */ - clib_memcpy (lcl_addr, &rmt_cfg->peer.ip, sizeof (rmt_cfg->peer.ip)); + clib_memcpy_fast (lcl_addr, &rmt_cfg->peer.ip, + sizeof (rmt_cfg->peer.ip)); } /* diff --git a/src/vnet/srp/interface.c b/src/vnet/srp/interface.c index 28e519986f0..3c98f416781 100644 --- a/src/vnet/srp/interface.c +++ b/src/vnet/srp/interface.c @@ -289,7 +289,7 @@ simulated_srp_interface_tx (vlib_main_t * vm, n_copy = clib_min (n_left_from, n_left_to_next); - clib_memcpy (to_next, from, n_copy * sizeof (from[0])); + clib_memcpy_fast (to_next, from, n_copy * sizeof (from[0])); n_left_to_next -= n_copy; n_left_from -= n_copy; for (i = 0; i < n_copy; i++) diff --git a/src/vnet/srp/node.c b/src/vnet/srp/node.c index 7cbc71b39ae..cec014f987d 100644 --- a/src/vnet/srp/node.c +++ b/src/vnet/srp/node.c @@ -309,7 +309,7 @@ srp_topology_packet (vlib_main_t * vm, u32 sw_if_index, u8 ** contents) return SRP_ERROR_TOPOLOGY_BAD_LENGTH; /* Fill in our source MAC address. */ - clib_memcpy (t->ethernet.src_address, hi->hw_address, vec_len (hi->hw_address)); + clib_memcpy_fast (t->ethernet.src_address, hi->hw_address, vec_len (hi->hw_address)); /* Make space for our MAC binding. */ vec_resize (*contents, sizeof (srp_topology_mac_binding_t)); @@ -321,7 +321,7 @@ srp_topology_packet (vlib_main_t * vm, u32 sw_if_index, u8 ** contents) mb->flags = ((t->srp.is_inner_ring ? SRP_TOPOLOGY_MAC_BINDING_FLAG_IS_INNER_RING : 0) | (/* is wrapped FIXME */ 0)); - clib_memcpy (mb->address, hi->hw_address, vec_len (hi->hw_address)); + clib_memcpy_fast (mb->address, hi->hw_address, vec_len (hi->hw_address)); t->control.checksum = ~ip_csum_fold (ip_incremental_checksum (0, &t->control, @@ -588,7 +588,7 @@ static void init_ips_packet (srp_interface_t * si, i->srp.mode = SRP_MODE_control_locally_buffered_for_host; srp_header_compute_parity (&i->srp); - clib_memcpy (&i->ethernet.src_address, &si->my_address, sizeof (si->my_address)); + clib_memcpy_fast (&i->ethernet.src_address, &si->my_address, sizeof (si->my_address)); i->ethernet.type = clib_host_to_net_u16 (ETHERNET_TYPE_SRP_CONTROL); /* Checksum will be filled in later. */ @@ -596,7 +596,7 @@ static void init_ips_packet (srp_interface_t * si, i->control.type = SRP_CONTROL_PACKET_TYPE_ips; i->control.ttl = 255; - clib_memcpy (&i->originator_address, &si->my_address, sizeof (si->my_address)); + clib_memcpy_fast (&i->originator_address, &si->my_address, sizeof (si->my_address)); } static void tx_ips_packet (srp_interface_t * si, @@ -680,7 +680,7 @@ void srp_ips_rx_packet (u32 sw_if_index, srp_ips_header_t * h) ASSERT (0); } ir->rx_neighbor_address_valid = 1; - clib_memcpy (ir->rx_neighbor_address, h->originator_address, sizeof (ir->rx_neighbor_address)); + clib_memcpy_fast (ir->rx_neighbor_address, h->originator_address, sizeof (ir->rx_neighbor_address)); } switch (si->current_ips_state) diff --git a/src/vnet/srv6/sr.h b/src/vnet/srv6/sr.h index 1a30cf3e3e0..2353b284363 100755 --- a/src/vnet/srv6/sr.h +++ b/src/vnet/srv6/sr.h @@ -310,7 +310,8 @@ ip6_sr_compute_rewrite_string_insert (ip6_address_t * sl) addrp = srh->segments + vec_len (sl); vec_foreach (this_address, sl) { - clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t)); + clib_memcpy_fast (addrp->as_u8, this_address->as_u8, + sizeof (ip6_address_t)); addrp--; } return rs; diff --git a/src/vnet/srv6/sr_policy_rewrite.c b/src/vnet/srv6/sr_policy_rewrite.c index c80b9c4505a..e9c42211209 100755 --- a/src/vnet/srv6/sr_policy_rewrite.c +++ b/src/vnet/srv6/sr_policy_rewrite.c @@ -115,7 +115,7 @@ static ip6_address_t sr_pr_encaps_src; void sr_set_source (ip6_address_t * address) { - clib_memcpy (&sr_pr_encaps_src, address, sizeof (sr_pr_encaps_src)); + clib_memcpy_fast (&sr_pr_encaps_src, address, sizeof (sr_pr_encaps_src)); } static clib_error_t * @@ -192,8 +192,8 @@ compute_rewrite_encaps (ip6_address_t * sl) addrp = srh->segments + vec_len (sl) - 1; vec_foreach (this_address, sl) { - clib_memcpy (addrp->as_u8, this_address->as_u8, - sizeof (ip6_address_t)); + clib_memcpy_fast (addrp->as_u8, this_address->as_u8, + sizeof (ip6_address_t)); addrp--; } } @@ -234,7 +234,8 @@ compute_rewrite_insert (ip6_address_t * sl) addrp = srh->segments + vec_len (sl); vec_foreach (this_address, sl) { - clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t)); + clib_memcpy_fast (addrp->as_u8, this_address->as_u8, + sizeof (ip6_address_t)); addrp--; } return rs; @@ -272,7 +273,8 @@ compute_rewrite_bsid (ip6_address_t * sl) addrp = srh->segments + vec_len (sl) - 1; vec_foreach (this_address, sl) { - clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t)); + clib_memcpy_fast (addrp->as_u8, this_address->as_u8, + sizeof (ip6_address_t)); addrp--; } return rs; @@ -589,7 +591,7 @@ sr_policy_add (ip6_address_t * bsid, ip6_address_t * segments, /* Add an SR policy object */ pool_get (sm->sr_policies, sr_policy); clib_memset (sr_policy, 0, sizeof (*sr_policy)); - clib_memcpy (&sr_policy->bsid, bsid, sizeof (ip6_address_t)); + clib_memcpy_fast (&sr_policy->bsid, bsid, sizeof (ip6_address_t)); sr_policy->type = behavior; sr_policy->fib_table = (fib_table != (u32) ~ 0 ? fib_table : 0); //Is default FIB 0 ? sr_policy->is_encap = is_encap; @@ -852,8 +854,8 @@ sr_policy_command_fn (vlib_main_t * vm, unformat_input_t * input, if (unformat (input, "next %U", unformat_ip6_address, &next_address)) { vec_add2 (segments, this_seg, 1); - clib_memcpy (this_seg->as_u8, next_address.as_u8, - sizeof (*this_seg)); + clib_memcpy_fast (this_seg->as_u8, next_address.as_u8, + sizeof (*this_seg)); } else if (unformat (input, "add sl")) operation = 1; @@ -1159,14 +1161,14 @@ sr_policy_rewrite_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, ip2_encap = vlib_buffer_get_current (b2); ip3_encap = vlib_buffer_get_current (b3); - clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), - sl0->rewrite, vec_len (sl0->rewrite)); - clib_memcpy (((u8 *) ip1_encap) - vec_len (sl1->rewrite), - sl1->rewrite, vec_len (sl1->rewrite)); - clib_memcpy (((u8 *) ip2_encap) - vec_len (sl2->rewrite), - sl2->rewrite, vec_len (sl2->rewrite)); - clib_memcpy (((u8 *) ip3_encap) - vec_len (sl3->rewrite), - sl3->rewrite, vec_len (sl3->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) ip1_encap) - vec_len (sl1->rewrite), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy_fast (((u8 *) ip2_encap) - vec_len (sl2->rewrite), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy_fast (((u8 *) ip3_encap) - vec_len (sl3->rewrite), + sl3->rewrite, vec_len (sl3->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); @@ -1189,40 +1191,40 @@ sr_policy_rewrite_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b3, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } } @@ -1257,8 +1259,8 @@ sr_policy_rewrite_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, ip0_encap = vlib_buffer_get_current (b0); - clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), - sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); ip0 = vlib_buffer_get_current (b0); @@ -1270,10 +1272,10 @@ sr_policy_rewrite_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } encap_pkts++; @@ -1442,14 +1444,14 @@ sr_policy_rewrite_encaps_v4 (vlib_main_t * vm, vlib_node_runtime_t * node, ip2_encap = vlib_buffer_get_current (b2); ip3_encap = vlib_buffer_get_current (b3); - clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), - sl0->rewrite, vec_len (sl0->rewrite)); - clib_memcpy (((u8 *) ip1_encap) - vec_len (sl1->rewrite), - sl1->rewrite, vec_len (sl1->rewrite)); - clib_memcpy (((u8 *) ip2_encap) - vec_len (sl2->rewrite), - sl2->rewrite, vec_len (sl2->rewrite)); - clib_memcpy (((u8 *) ip3_encap) - vec_len (sl3->rewrite), - sl3->rewrite, vec_len (sl3->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) ip1_encap) - vec_len (sl1->rewrite), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy_fast (((u8 *) ip2_encap) - vec_len (sl2->rewrite), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy_fast (((u8 *) ip3_encap) - vec_len (sl3->rewrite), + sl3->rewrite, vec_len (sl3->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); @@ -1472,40 +1474,40 @@ sr_policy_rewrite_encaps_v4 (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b3, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } } @@ -1541,8 +1543,8 @@ sr_policy_rewrite_encaps_v4 (vlib_main_t * vm, vlib_node_runtime_t * node, ip0_encap = vlib_buffer_get_current (b0); - clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), - sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); ip0 = vlib_buffer_get_current (b0); @@ -1554,10 +1556,10 @@ sr_policy_rewrite_encaps_v4 (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } encap_pkts++; @@ -1799,14 +1801,14 @@ sr_policy_rewrite_encaps_l2 (vlib_main_t * vm, vlib_node_runtime_t * node, en2 = vlib_buffer_get_current (b2); en3 = vlib_buffer_get_current (b3); - clib_memcpy (((u8 *) en0) - vec_len (sl0->rewrite), sl0->rewrite, - vec_len (sl0->rewrite)); - clib_memcpy (((u8 *) en1) - vec_len (sl1->rewrite), sl1->rewrite, - vec_len (sl1->rewrite)); - clib_memcpy (((u8 *) en2) - vec_len (sl2->rewrite), sl2->rewrite, - vec_len (sl2->rewrite)); - clib_memcpy (((u8 *) en3) - vec_len (sl3->rewrite), sl3->rewrite, - vec_len (sl3->rewrite)); + clib_memcpy_fast (((u8 *) en0) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) en1) - vec_len (sl1->rewrite), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy_fast (((u8 *) en2) - vec_len (sl2->rewrite), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy_fast (((u8 *) en3) - vec_len (sl3->rewrite), + sl3->rewrite, vec_len (sl3->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); @@ -1868,40 +1870,40 @@ sr_policy_rewrite_encaps_l2 (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b3, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } } @@ -1955,8 +1957,8 @@ sr_policy_rewrite_encaps_l2 (vlib_main_t * vm, vlib_node_runtime_t * node, en0 = vlib_buffer_get_current (b0); - clib_memcpy (((u8 *) en0) - vec_len (sl0->rewrite), sl0->rewrite, - vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) en0) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); @@ -1978,10 +1980,10 @@ sr_policy_rewrite_encaps_l2 (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } encap_pkts++; @@ -2145,23 +2147,23 @@ sr_policy_rewrite_insert (vlib_main_t * vm, vlib_node_runtime_t * node, else sr3 = (ip6_sr_header_t *) (ip3 + 1); - clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite), (u8 *) ip0, - (void *) sr0 - (void *) ip0); - clib_memcpy ((u8 *) ip1 - vec_len (sl1->rewrite), (u8 *) ip1, - (void *) sr1 - (void *) ip1); - clib_memcpy ((u8 *) ip2 - vec_len (sl2->rewrite), (u8 *) ip2, - (void *) sr2 - (void *) ip2); - clib_memcpy ((u8 *) ip3 - vec_len (sl3->rewrite), (u8 *) ip3, - (void *) sr3 - (void *) ip3); - - clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite)), sl0->rewrite, - vec_len (sl0->rewrite)); - clib_memcpy (((u8 *) sr1 - vec_len (sl1->rewrite)), sl1->rewrite, - vec_len (sl1->rewrite)); - clib_memcpy (((u8 *) sr2 - vec_len (sl2->rewrite)), sl2->rewrite, - vec_len (sl2->rewrite)); - clib_memcpy (((u8 *) sr3 - vec_len (sl3->rewrite)), sl3->rewrite, - vec_len (sl3->rewrite)); + clib_memcpy_fast ((u8 *) ip0 - vec_len (sl0->rewrite), (u8 *) ip0, + (void *) sr0 - (void *) ip0); + clib_memcpy_fast ((u8 *) ip1 - vec_len (sl1->rewrite), (u8 *) ip1, + (void *) sr1 - (void *) ip1); + clib_memcpy_fast ((u8 *) ip2 - vec_len (sl2->rewrite), (u8 *) ip2, + (void *) sr2 - (void *) ip2); + clib_memcpy_fast ((u8 *) ip3 - vec_len (sl3->rewrite), (u8 *) ip3, + (void *) sr3 - (void *) ip3); + + clib_memcpy_fast (((u8 *) sr0 - vec_len (sl0->rewrite)), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) sr1 - vec_len (sl1->rewrite)), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy_fast (((u8 *) sr2 - vec_len (sl2->rewrite)), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy_fast (((u8 *) sr3 - vec_len (sl3->rewrite)), + sl3->rewrite, vec_len (sl3->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); @@ -2284,40 +2286,40 @@ sr_policy_rewrite_insert (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b3, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } } @@ -2360,10 +2362,10 @@ sr_policy_rewrite_insert (vlib_main_t * vm, vlib_node_runtime_t * node, else sr0 = (ip6_sr_header_t *) (ip0 + 1); - clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite), (u8 *) ip0, - (void *) sr0 - (void *) ip0); - clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite)), sl0->rewrite, - vec_len (sl0->rewrite)); + clib_memcpy_fast ((u8 *) ip0 - vec_len (sl0->rewrite), (u8 *) ip0, + (void *) sr0 - (void *) ip0); + clib_memcpy_fast (((u8 *) sr0 - vec_len (sl0->rewrite)), + sl0->rewrite, vec_len (sl0->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); @@ -2400,10 +2402,10 @@ sr_policy_rewrite_insert (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } insert_pkts++; @@ -2567,23 +2569,23 @@ sr_policy_rewrite_b_insert (vlib_main_t * vm, vlib_node_runtime_t * node, else sr3 = (ip6_sr_header_t *) (ip3 + 1); - clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite_bsid), (u8 *) ip0, - (void *) sr0 - (void *) ip0); - clib_memcpy ((u8 *) ip1 - vec_len (sl1->rewrite_bsid), (u8 *) ip1, - (void *) sr1 - (void *) ip1); - clib_memcpy ((u8 *) ip2 - vec_len (sl2->rewrite_bsid), (u8 *) ip2, - (void *) sr2 - (void *) ip2); - clib_memcpy ((u8 *) ip3 - vec_len (sl3->rewrite_bsid), (u8 *) ip3, - (void *) sr3 - (void *) ip3); - - clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite_bsid)), - sl0->rewrite_bsid, vec_len (sl0->rewrite_bsid)); - clib_memcpy (((u8 *) sr1 - vec_len (sl1->rewrite_bsid)), - sl1->rewrite_bsid, vec_len (sl1->rewrite_bsid)); - clib_memcpy (((u8 *) sr2 - vec_len (sl2->rewrite_bsid)), - sl2->rewrite_bsid, vec_len (sl2->rewrite_bsid)); - clib_memcpy (((u8 *) sr3 - vec_len (sl3->rewrite_bsid)), - sl3->rewrite_bsid, vec_len (sl3->rewrite_bsid)); + clib_memcpy_fast ((u8 *) ip0 - vec_len (sl0->rewrite_bsid), + (u8 *) ip0, (void *) sr0 - (void *) ip0); + clib_memcpy_fast ((u8 *) ip1 - vec_len (sl1->rewrite_bsid), + (u8 *) ip1, (void *) sr1 - (void *) ip1); + clib_memcpy_fast ((u8 *) ip2 - vec_len (sl2->rewrite_bsid), + (u8 *) ip2, (void *) sr2 - (void *) ip2); + clib_memcpy_fast ((u8 *) ip3 - vec_len (sl3->rewrite_bsid), + (u8 *) ip3, (void *) sr3 - (void *) ip3); + + clib_memcpy_fast (((u8 *) sr0 - vec_len (sl0->rewrite_bsid)), + sl0->rewrite_bsid, vec_len (sl0->rewrite_bsid)); + clib_memcpy_fast (((u8 *) sr1 - vec_len (sl1->rewrite_bsid)), + sl1->rewrite_bsid, vec_len (sl1->rewrite_bsid)); + clib_memcpy_fast (((u8 *) sr2 - vec_len (sl2->rewrite_bsid)), + sl2->rewrite_bsid, vec_len (sl2->rewrite_bsid)); + clib_memcpy_fast (((u8 *) sr3 - vec_len (sl3->rewrite_bsid)), + sl3->rewrite_bsid, vec_len (sl3->rewrite_bsid)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite_bsid)); vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite_bsid)); @@ -2697,40 +2699,40 @@ sr_policy_rewrite_b_insert (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b3, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } } @@ -2773,10 +2775,10 @@ sr_policy_rewrite_b_insert (vlib_main_t * vm, vlib_node_runtime_t * node, else sr0 = (ip6_sr_header_t *) (ip0 + 1); - clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite_bsid), (u8 *) ip0, - (void *) sr0 - (void *) ip0); - clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite_bsid)), - sl0->rewrite_bsid, vec_len (sl0->rewrite_bsid)); + clib_memcpy_fast ((u8 *) ip0 - vec_len (sl0->rewrite_bsid), + (u8 *) ip0, (void *) sr0 - (void *) ip0); + clib_memcpy_fast (((u8 *) sr0 - vec_len (sl0->rewrite_bsid)), + sl0->rewrite_bsid, vec_len (sl0->rewrite_bsid)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite_bsid)); @@ -2811,10 +2813,10 @@ sr_policy_rewrite_b_insert (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } insert_pkts++; @@ -2997,14 +2999,14 @@ sr_policy_rewrite_b_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, end_bsid_encaps_srh_processing (node, b2, ip2_encap, sr2, &next2); end_bsid_encaps_srh_processing (node, b3, ip3_encap, sr3, &next3); - clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), - sl0->rewrite, vec_len (sl0->rewrite)); - clib_memcpy (((u8 *) ip1_encap) - vec_len (sl1->rewrite), - sl1->rewrite, vec_len (sl1->rewrite)); - clib_memcpy (((u8 *) ip2_encap) - vec_len (sl2->rewrite), - sl2->rewrite, vec_len (sl2->rewrite)); - clib_memcpy (((u8 *) ip3_encap) - vec_len (sl3->rewrite), - sl3->rewrite, vec_len (sl3->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) ip1_encap) - vec_len (sl1->rewrite), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy_fast (((u8 *) ip2_encap) - vec_len (sl2->rewrite), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy_fast (((u8 *) ip3_encap) - vec_len (sl3->rewrite), + sl3->rewrite, vec_len (sl3->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); @@ -3027,40 +3029,40 @@ sr_policy_rewrite_b_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b3, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } } @@ -3100,8 +3102,8 @@ sr_policy_rewrite_b_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, IP_PROTOCOL_IPV6_ROUTE); end_bsid_encaps_srh_processing (node, b0, ip0_encap, sr0, &next0); - clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), - sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); ip0 = vlib_buffer_get_current (b0); @@ -3113,10 +3115,10 @@ sr_policy_rewrite_b_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } encap_pkts++; diff --git a/src/vnet/srv6/sr_steering.c b/src/vnet/srv6/sr_steering.c index d44bbae576f..66186e9fca8 100755 --- a/src/vnet/srv6/sr_steering.c +++ b/src/vnet/srv6/sr_steering.c @@ -249,8 +249,8 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index, if (traffic_type == SR_STEER_IPV4 || traffic_type == SR_STEER_IPV6) { - clib_memcpy (&steer_pl->classify.l3.prefix, prefix, - sizeof (ip46_address_t)); + clib_memcpy_fast (&steer_pl->classify.l3.prefix, prefix, + sizeof (ip46_address_t)); steer_pl->classify.l3.mask_width = mask_width; steer_pl->classify.l3.fib_table = (table_id != (u32) ~ 0 ? table_id : 0); diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c index 25c2615b101..04613cd6943 100644 --- a/src/vnet/tcp/tcp.c +++ b/src/vnet/tcp/tcp.c @@ -448,10 +448,10 @@ tcp_connection_select_lb_bucket (tcp_connection_t * tc, const dpo_id_t * dpo, ip6_tcp_hdr_t hdr; clib_memset (&hdr, 0, sizeof (hdr)); hdr.ip.protocol = IP_PROTOCOL_TCP; - clib_memcpy (&hdr.ip.src_address, &tc->c_lcl_ip.ip6, - sizeof (ip6_address_t)); - clib_memcpy (&hdr.ip.dst_address, &tc->c_rmt_ip.ip6, - sizeof (ip6_address_t)); + clib_memcpy_fast (&hdr.ip.src_address, &tc->c_lcl_ip.ip6, + sizeof (ip6_address_t)); + clib_memcpy_fast (&hdr.ip.dst_address, &tc->c_rmt_ip.ip6, + sizeof (ip6_address_t)); hdr.tcp.src_port = tc->c_lcl_port; hdr.tcp.dst_port = tc->c_rmt_port; hash = ip6_compute_flow_hash (&hdr.ip, lb->lb_hash_config); @@ -466,7 +466,7 @@ tcp_lookup_rmt_in_fib (tcp_connection_t * tc) fib_prefix_t prefix; u32 fib_index; - clib_memcpy (&prefix.fp_addr, &tc->c_rmt_ip, sizeof (prefix.fp_addr)); + clib_memcpy_fast (&prefix.fp_addr, &tc->c_rmt_ip, sizeof (prefix.fp_addr)); prefix.fp_proto = tc->c_is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6; prefix.fp_len = tc->c_is_ip4 ? 32 : 128; fib_index = fib_table_find (prefix.fp_proto, tc->c_fib_index); @@ -607,8 +607,8 @@ tcp_alloc_custom_local_endpoint (tcp_main_t * tm, ip46_address_t * lcl_addr, index = tm->last_v6_address_rotor++; if (tm->last_v6_address_rotor >= vec_len (tm->ip6_src_addresses)) tm->last_v6_address_rotor = 0; - clib_memcpy (&lcl_addr->ip6, &tm->ip6_src_addresses[index], - sizeof (ip6_address_t)); + clib_memcpy_fast (&lcl_addr->ip6, &tm->ip6_src_addresses[index], + sizeof (ip6_address_t)); } port = transport_alloc_local_port (TRANSPORT_PROTO_TCP, lcl_addr); if (port < 1) diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c index 0c8706567ae..6809a9173df 100644 --- a/src/vnet/tcp/tcp_input.c +++ b/src/vnet/tcp/tcp_input.c @@ -1990,13 +1990,14 @@ tcp_set_rx_trace_data (tcp_rx_trace_t * t0, tcp_connection_t * tc0, { if (tc0) { - clib_memcpy (&t0->tcp_connection, tc0, sizeof (t0->tcp_connection)); + clib_memcpy_fast (&t0->tcp_connection, tc0, + sizeof (t0->tcp_connection)); } else { th0 = tcp_buffer_hdr (b0); } - clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header)); + clib_memcpy_fast (&t0->tcp_header, th0, sizeof (t0->tcp_header)); } static void @@ -2443,7 +2444,7 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node, /* Valid SYN or SYN-ACK. Move connection from half-open pool to * current thread pool. */ pool_get (tm->connections[my_thread_index], new_tc0); - clib_memcpy (new_tc0, tc0, sizeof (*new_tc0)); + clib_memcpy_fast (new_tc0, tc0, sizeof (*new_tc0)); new_tc0->c_c_index = new_tc0 - tm->connections[my_thread_index]; new_tc0->c_thread_index = my_thread_index; new_tc0->rcv_nxt = vnet_buffer (b0)->tcp.seq_end; @@ -2544,8 +2545,9 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_IS_TRACED) && tcp0 != 0)) { t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); - clib_memcpy (&t0->tcp_header, tcp0, sizeof (t0->tcp_header)); - clib_memcpy (&t0->tcp_connection, tc0, sizeof (t0->tcp_connection)); + clib_memcpy_fast (&t0->tcp_header, tcp0, sizeof (t0->tcp_header)); + clib_memcpy_fast (&t0->tcp_connection, tc0, + sizeof (t0->tcp_connection)); } } @@ -3085,10 +3087,10 @@ tcp46_listen_inline (vlib_main_t * vm, vlib_node_runtime_t * node, } else { - clib_memcpy (&child0->c_lcl_ip6, &ip60->dst_address, - sizeof (ip6_address_t)); - clib_memcpy (&child0->c_rmt_ip6, &ip60->src_address, - sizeof (ip6_address_t)); + clib_memcpy_fast (&child0->c_lcl_ip6, &ip60->dst_address, + sizeof (ip6_address_t)); + clib_memcpy_fast (&child0->c_rmt_ip6, &ip60->src_address, + sizeof (ip6_address_t)); } if (tcp_options_parse (th0, &child0->rcv_opts)) @@ -3139,8 +3141,9 @@ tcp46_listen_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); - clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header)); - clib_memcpy (&t0->tcp_connection, lc0, sizeof (t0->tcp_connection)); + clib_memcpy_fast (&t0->tcp_header, th0, sizeof (t0->tcp_header)); + clib_memcpy_fast (&t0->tcp_connection, lc0, + sizeof (t0->tcp_connection)); } n_syns += (error0 == TCP_ERROR_NONE); diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index 49156368574..2068e435a2a 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -215,7 +215,7 @@ tcp_options_write (u8 * data, tcp_options_t * opts) *data++ = TCP_OPTION_MSS; *data++ = TCP_OPTION_LEN_MSS; buf = clib_host_to_net_u16 (opts->mss); - clib_memcpy (data, &buf, sizeof (opts->mss)); + clib_memcpy_fast (data, &buf, sizeof (opts->mss)); data += sizeof (opts->mss); opts_len += TCP_OPTION_LEN_MSS; } @@ -240,10 +240,10 @@ tcp_options_write (u8 * data, tcp_options_t * opts) *data++ = TCP_OPTION_TIMESTAMP; *data++ = TCP_OPTION_LEN_TIMESTAMP; buf = clib_host_to_net_u32 (opts->tsval); - clib_memcpy (data, &buf, sizeof (opts->tsval)); + clib_memcpy_fast (data, &buf, sizeof (opts->tsval)); data += sizeof (opts->tsval); buf = clib_host_to_net_u32 (opts->tsecr); - clib_memcpy (data, &buf, sizeof (opts->tsecr)); + clib_memcpy_fast (data, &buf, sizeof (opts->tsecr)); data += sizeof (opts->tsecr); opts_len += TCP_OPTION_LEN_TIMESTAMP; } @@ -261,10 +261,10 @@ tcp_options_write (u8 * data, tcp_options_t * opts) for (i = 0; i < n_sack_blocks; i++) { buf = clib_host_to_net_u32 (opts->sacks[i].start); - clib_memcpy (data, &buf, seq_len); + clib_memcpy_fast (data, &buf, seq_len); data += seq_len; buf = clib_host_to_net_u32 (opts->sacks[i].end); - clib_memcpy (data, &buf, seq_len); + clib_memcpy_fast (data, &buf, seq_len); data += seq_len; } opts_len += 2 + n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK; @@ -764,8 +764,8 @@ tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0, { ih6 = vlib_buffer_get_current (b0); ASSERT ((ih6->ip_version_traffic_class_and_flow_label & 0xF0) == 0x60); - clib_memcpy (&src_ip60, &ih6->src_address, sizeof (ip6_address_t)); - clib_memcpy (&dst_ip60, &ih6->dst_address, sizeof (ip6_address_t)); + clib_memcpy_fast (&src_ip60, &ih6->src_address, sizeof (ip6_address_t)); + clib_memcpy_fast (&dst_ip60, &ih6->dst_address, sizeof (ip6_address_t)); } src_port = th0->src_port; @@ -1175,9 +1175,9 @@ tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b, if (maybe_burst) { - clib_memcpy ((u8 *) (th + 1), - tm->wrk_ctx[tc->c_thread_index].cached_opts, - tc->snd_opts_len); + clib_memcpy_fast ((u8 *) (th + 1), + tm->wrk_ctx[tc->c_thread_index].cached_opts, + tc->snd_opts_len); } else { @@ -2058,8 +2058,8 @@ tcp46_output_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node, tc = tcp_connection_get (vnet_buffer (b)->tcp.connection_index, vm->thread_index); t = vlib_add_trace (vm, node, b, sizeof (*t)); - clib_memcpy (&t->tcp_header, th, sizeof (t->tcp_header)); - clib_memcpy (&t->tcp_connection, tc, sizeof (t->tcp_connection)); + clib_memcpy_fast (&t->tcp_header, th, sizeof (t->tcp_header)); + clib_memcpy_fast (&t->tcp_connection, tc, sizeof (t->tcp_connection)); } } @@ -2322,7 +2322,8 @@ tcp46_send_reset_inline (vlib_main_t * vm, vlib_node_runtime_t * node, else th0 = ip6_next_header ((ip6_header_t *) th0); t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); - clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header)); + clib_memcpy_fast (&t0->tcp_header, th0, + sizeof (t0->tcp_header)); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c index 109280b4be1..7378093bd04 100644 --- a/src/vnet/tls/tls.c +++ b/src/vnet/tls/tls.c @@ -479,7 +479,7 @@ tls_session_connected_callback (u32 tls_app_index, u32 ho_ctx_index, ctx_handle = tls_ctx_alloc (ho_ctx->tls_ctx_engine); ctx = tls_ctx_get (ctx_handle); - clib_memcpy (ctx, ho_ctx, sizeof (*ctx)); + clib_memcpy_fast (ctx, ho_ctx, sizeof (*ctx)); tls_ctx_half_open_reader_unlock (); tls_ctx_half_open_free (ho_ctx_index); @@ -548,7 +548,7 @@ tls_connect (transport_endpoint_cfg_t * tep) app_worker_alloc_connects_segment_manager (app_wrk); ctx->tls_ctx_engine = engine_type; - clib_memcpy (&cargs->sep, sep, sizeof (session_endpoint_t)); + clib_memcpy_fast (&cargs->sep, sep, sizeof (session_endpoint_t)); cargs->sep.transport_proto = TRANSPORT_PROTO_TCP; cargs->app_index = tm->app_index; cargs->api_context = ctx_index; diff --git a/src/vnet/udp/udp.h b/src/vnet/udp/udp.h index 3b6de6ca271..8b94a0088f6 100644 --- a/src/vnet/udp/udp.h +++ b/src/vnet/udp/udp.h @@ -228,7 +228,7 @@ udp_connection_clone_safe (u32 connection_index, u32 thread_index) */ udp_pool_add_peeker (thread_index); old_c = udp_main.connections[thread_index] + connection_index; - clib_memcpy (new_c, old_c, sizeof (*new_c)); + clib_memcpy_fast (new_c, old_c, sizeof (*new_c)); udp_pool_remove_peeker (thread_index); new_c->c_thread_index = current_thread_index; new_c->c_c_index = udp_connection_index (new_c); @@ -343,7 +343,7 @@ ip_udp_encap_one (vlib_main_t * vm, vlib_buffer_t * b0, u8 * ec0, word ec_len, ip0 = vlib_buffer_get_current (b0); /* Apply the encap string. */ - clib_memcpy (ip0, ec0, ec_len); + clib_memcpy_fast (ip0, ec0, ec_len); ip_udp_fixup_one (vm, b0, 1); } else @@ -353,7 +353,7 @@ ip_udp_encap_one (vlib_main_t * vm, vlib_buffer_t * b0, u8 * ec0, word ec_len, ip0 = vlib_buffer_get_current (b0); /* Apply the encap string. */ - clib_memcpy (ip0, ec0, ec_len); + clib_memcpy_fast (ip0, ec0, ec_len); ip_udp_fixup_one (vm, b0, 0); } } @@ -380,8 +380,8 @@ ip_udp_encap_two (vlib_main_t * vm, vlib_buffer_t * b0, vlib_buffer_t * b1, ip1 = vlib_buffer_get_current (b1); /* Apply the encap string */ - clib_memcpy (ip0, ec0, ec_len); - clib_memcpy (ip1, ec1, ec_len); + clib_memcpy_fast (ip0, ec0, ec_len); + clib_memcpy_fast (ip1, ec1, ec_len); /* fix the <bleep>ing outer-IP checksum */ sum0 = ip0->checksum; @@ -424,8 +424,8 @@ ip_udp_encap_two (vlib_main_t * vm, vlib_buffer_t * b0, vlib_buffer_t * b1, ip1 = vlib_buffer_get_current (b1); /* Apply the encap string. */ - clib_memcpy (ip0, ec0, ec_len); - clib_memcpy (ip1, ec1, ec_len); + clib_memcpy_fast (ip0, ec0, ec_len); + clib_memcpy_fast (ip1, ec1, ec_len); new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - sizeof (*ip0)); diff --git a/src/vnet/unix/pcap.h b/src/vnet/unix/pcap.h index 2c174fbedf9..0706aace496 100644 --- a/src/vnet/unix/pcap.h +++ b/src/vnet/unix/pcap.h @@ -221,7 +221,7 @@ pcap_add_buffer (pcap_main_t * pm, while (1) { u32 copy_length = clib_min ((u32) n_left, b->current_length); - clib_memcpy (d, b->data + b->current_data, copy_length); + clib_memcpy_fast (d, b->data + b->current_data, copy_length); n_left -= b->current_length; if (n_left <= 0) break; diff --git a/src/vnet/unix/tuntap.c b/src/vnet/unix/tuntap.c index d663d9698a0..e002a248ce4 100644 --- a/src/vnet/unix/tuntap.c +++ b/src/vnet/unix/tuntap.c @@ -166,7 +166,8 @@ tuntap_tx (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) if (tm->is_ether && (!tm->have_normal_interface)) { vlib_buffer_reset (b); - clib_memcpy (vlib_buffer_get_current (b), tm->ether_dst_mac, 6); + clib_memcpy_fast (vlib_buffer_get_current (b), tm->ether_dst_mac, + 6); } /* Re-set iovecs if present. */ @@ -626,7 +627,7 @@ tuntap_config (vlib_main_t * vm, unformat_input_t * input) goto done; } else - clib_memcpy (tm->ether_dst_mac, ifr.ifr_hwaddr.sa_data, 6); + clib_memcpy_fast (tm->ether_dst_mac, ifr.ifr_hwaddr.sa_data, 6); } if (have_normal_interface) @@ -721,7 +722,7 @@ tuntap_ip4_add_del_interface_address (ip4_main_t * im, /** See if we already know about this subif */ clib_memset (&subif_addr, 0, sizeof (subif_addr)); subif_addr.sw_if_index = sw_if_index; - clib_memcpy (&subif_addr.addr, address, sizeof (*address)); + clib_memcpy_fast (&subif_addr.addr, address, sizeof (*address)); p = mhash_get (&tm->subif_mhash, &subif_addr); @@ -753,7 +754,7 @@ tuntap_ip4_add_del_interface_address (ip4_main_t * im, /* Set ipv4 address, netmask. */ sin->sin_family = AF_INET; - clib_memcpy (&sin->sin_addr.s_addr, address, 4); + clib_memcpy_fast (&sin->sin_addr.s_addr, address, 4); if (ioctl (tm->dev_tap_fd, SIOCSIFADDR, &ifr) < 0) clib_unix_warning ("ioctl SIOCSIFADDR"); @@ -838,7 +839,7 @@ tuntap_ip6_add_del_interface_address (ip6_main_t * im, clib_memset (&subif_addr, 0, sizeof (subif_addr)); subif_addr.sw_if_index = sw_if_index; subif_addr.is_v6 = 1; - clib_memcpy (&subif_addr.addr, address, sizeof (*address)); + clib_memcpy_fast (&subif_addr.addr, address, sizeof (*address)); p = mhash_get (&tm->subif_mhash, &subif_addr); @@ -874,7 +875,7 @@ tuntap_ip6_add_del_interface_address (ip6_main_t * im, ifr6.ifr6_ifindex = ifr.ifr_ifindex; ifr6.ifr6_prefixlen = address_length; - clib_memcpy (&ifr6.ifr6_addr, address, 16); + clib_memcpy_fast (&ifr6.ifr6_addr, address, 16); if (ioctl (sockfd, SIOCSIFADDR, &ifr6) < 0) clib_unix_warning ("set address"); @@ -893,7 +894,7 @@ tuntap_ip6_add_del_interface_address (ip6_main_t * im, ifr6.ifr6_ifindex = ifr.ifr_ifindex; ifr6.ifr6_prefixlen = address_length; - clib_memcpy (&ifr6.ifr6_addr, address, 16); + clib_memcpy_fast (&ifr6.ifr6_addr, address, 16); if (ioctl (sockfd, SIOCDIFADDR, &ifr6) < 0) clib_unix_warning ("del address"); diff --git a/src/vnet/vxlan-gbp/encap.c b/src/vnet/vxlan-gbp/encap.c index 07142c8a521..d260ccc0deb 100644 --- a/src/vnet/vxlan-gbp/encap.c +++ b/src/vnet/vxlan-gbp/encap.c @@ -186,10 +186,12 @@ vxlan_gbp_encap_inline (vlib_main_t * vm, /* vnet_rewrite_two_header writes only in (uword) 8 bytes chunks * and discards the first 4 bytes of the (36 bytes ip4 underlay) rewrite * use memcpy as a workaround */ - clib_memcpy (underlay0, t0->rewrite_header.data + rw_hdr_offset, - underlay_hdr_len); - clib_memcpy (underlay1, t1->rewrite_header.data + rw_hdr_offset, - underlay_hdr_len); + clib_memcpy_fast (underlay0, + t0->rewrite_header.data + rw_hdr_offset, + underlay_hdr_len); + clib_memcpy_fast (underlay1, + t1->rewrite_header.data + rw_hdr_offset, + underlay_hdr_len); ip4_header_t *ip4_0, *ip4_1; qos_bits_t ip4_0_tos = 0, ip4_1_tos = 0; @@ -368,8 +370,9 @@ vxlan_gbp_encap_inline (vlib_main_t * vm, /* vnet_rewrite_one_header writes only in (uword) 8 bytes chunks * and discards the first 4 bytes of the (36 bytes ip4 underlay) rewrite * use memcpy as a workaround */ - clib_memcpy (underlay0, t0->rewrite_header.data + rw_hdr_offset, - underlay_hdr_len); + clib_memcpy_fast (underlay0, + t0->rewrite_header.data + rw_hdr_offset, + underlay_hdr_len); u32 len0 = vlib_buffer_length_in_chain (vm, b0); u16 payload_l0 = clib_host_to_net_u16 (len0 - l3_len); diff --git a/src/vnet/vxlan-gpe/vxlan_gpe.c b/src/vnet/vxlan-gpe/vxlan_gpe.c index c14cc734fb3..a7eaba6a49f 100644 --- a/src/vnet/vxlan-gpe/vxlan_gpe.c +++ b/src/vnet/vxlan-gpe/vxlan_gpe.c @@ -543,14 +543,14 @@ int vnet_vxlan_gpe_add_del_tunnel if (!is_ip6) { key4_copy = clib_mem_alloc (sizeof (*key4_copy)); - clib_memcpy (key4_copy, &key4, sizeof (*key4_copy)); + clib_memcpy_fast (key4_copy, &key4, sizeof (*key4_copy)); hash_set_mem (ngm->vxlan4_gpe_tunnel_by_key, key4_copy, t - ngm->tunnels); } else { key6_copy = clib_mem_alloc (sizeof (*key6_copy)); - clib_memcpy (key6_copy, &key6, sizeof (*key6_copy)); + clib_memcpy_fast (key6_copy, &key6, sizeof (*key6_copy)); hash_set_mem (ngm->vxlan6_gpe_tunnel_by_key, key6_copy, t - ngm->tunnels); } diff --git a/src/vnet/vxlan/encap.c b/src/vnet/vxlan/encap.c index 31320900d3e..33b6526f2fa 100644 --- a/src/vnet/vxlan/encap.c +++ b/src/vnet/vxlan/encap.c @@ -186,8 +186,8 @@ vxlan_encap_inline (vlib_main_t * vm, /* vnet_rewrite_two_header writes only in (uword) 8 bytes chunks * and discards the first 4 bytes of the (36 bytes ip4 underlay) rewrite * use memcpy as a workaround */ - clib_memcpy(underlay0, t0->rewrite_header.data + rw_hdr_offset, underlay_hdr_len); - clib_memcpy(underlay1, t1->rewrite_header.data + rw_hdr_offset, underlay_hdr_len); + clib_memcpy_fast(underlay0, t0->rewrite_header.data + rw_hdr_offset, underlay_hdr_len); + clib_memcpy_fast(underlay1, t1->rewrite_header.data + rw_hdr_offset, underlay_hdr_len); ip4_header_t * ip4_0, * ip4_1; qos_bits_t ip4_0_tos = 0, ip4_1_tos = 0; @@ -359,7 +359,7 @@ vxlan_encap_inline (vlib_main_t * vm, /* vnet_rewrite_one_header writes only in (uword) 8 bytes chunks * and discards the first 4 bytes of the (36 bytes ip4 underlay) rewrite * use memcpy as a workaround */ - clib_memcpy(underlay0, t0->rewrite_header.data + rw_hdr_offset, underlay_hdr_len); + clib_memcpy_fast(underlay0, t0->rewrite_header.data + rw_hdr_offset, underlay_hdr_len); u32 len0 = vlib_buffer_length_in_chain (vm, b0); u16 payload_l0 = clib_host_to_net_u16 (len0 - l3_len); |