diff options
| author |   Dave Barach <dave@barachs.net> | 2018-11-13 16:34:13 -0500 |
|---|---|---|
| committer |   Florin Coras <florin.coras@gmail.com> | 2018-11-14 15:54:01 +0000 |
| commit | 178cf493d009995b28fdf220f04c98860ff79a9b (patch) | |
| tree | 097c1be82b8f6fa9bc04b9b1e193158e2e4997eb | |
| parent | 6917b94f2146aa51195a6a2a1ccd8416a1d74bf3 (diff) | |
Remove c-11 memcpy checks from perf-critical code
Change-Id: Id4f37f5d4a03160572954a416efa1ef9b3d79ad1
Signed-off-by: Dave Barach <dave@barachs.net>
178 files changed, 1353 insertions, 1303 deletions
diff --git a/src/examples/sample-plugin/sample/node.c b/src/examples/sample-plugin/sample/node.c index 009968b6099..1f0a2e9775b 100644 --- a/src/examples/sample-plugin/sample/node.c +++ b/src/examples/sample-plugin/sample/node.c @@ -183,10 +183,10 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_add_trace (vm, node, b0, sizeof (*t)); t->sw_if_index = sw_if_index0; t->next_index = next0; - clib_memcpy (t->new_src_mac, en0->src_address, - sizeof (t->new_src_mac)); - clib_memcpy (t->new_dst_mac, en0->dst_address, - sizeof (t->new_dst_mac)); + clib_memcpy_fast (t->new_src_mac, en0->src_address, + sizeof (t->new_src_mac)); + clib_memcpy_fast (t->new_dst_mac, en0->dst_address, + sizeof (t->new_dst_mac)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) @@ -195,10 +195,10 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_add_trace (vm, node, b1, sizeof (*t)); t->sw_if_index = sw_if_index1; t->next_index = next1; - clib_memcpy (t->new_src_mac, en1->src_address, - sizeof (t->new_src_mac)); - clib_memcpy (t->new_dst_mac, en1->dst_address, - sizeof (t->new_dst_mac)); + clib_memcpy_fast (t->new_src_mac, en1->src_address, + sizeof (t->new_src_mac)); + clib_memcpy_fast (t->new_dst_mac, en1->dst_address, + sizeof (t->new_dst_mac)); } } @@ -256,10 +256,10 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node, sample_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); t->sw_if_index = sw_if_index0; t->next_index = next0; - clib_memcpy (t->new_src_mac, en0->src_address, - sizeof (t->new_src_mac)); - clib_memcpy (t->new_dst_mac, en0->dst_address, - sizeof (t->new_dst_mac)); + clib_memcpy_fast (t->new_src_mac, en0->src_address, + sizeof (t->new_src_mac)); + clib_memcpy_fast (t->new_dst_mac, en0->dst_address, + sizeof (t->new_dst_mac)); } pkts_swapped += 1; @@ -367,10 +367,10 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_add_trace (vm, node, b0, sizeof (*t)); t->sw_if_index = sw_if_index0; t->next_index = next0; - clib_memcpy (t->new_src_mac, en0->src_address, - sizeof (t->new_src_mac)); - clib_memcpy (t->new_dst_mac, en0->dst_address, - sizeof (t->new_dst_mac)); + clib_memcpy_fast (t->new_src_mac, en0->src_address, + sizeof (t->new_src_mac)); + clib_memcpy_fast (t->new_dst_mac, en0->dst_address, + sizeof (t->new_dst_mac)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) @@ -379,10 +379,10 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_add_trace (vm, node, b1, sizeof (*t)); t->sw_if_index = sw_if_index1; t->next_index = next1; - clib_memcpy (t->new_src_mac, en1->src_address, - sizeof (t->new_src_mac)); - clib_memcpy (t->new_dst_mac, en1->dst_address, - sizeof (t->new_dst_mac)); + clib_memcpy_fast (t->new_src_mac, en1->src_address, + sizeof (t->new_src_mac)); + clib_memcpy_fast (t->new_dst_mac, en1->dst_address, + sizeof (t->new_dst_mac)); } } @@ -432,10 +432,10 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node, sample_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); t->sw_if_index = sw_if_index0; t->next_index = next0; - clib_memcpy (t->new_src_mac, en0->src_address, - sizeof (t->new_src_mac)); - clib_memcpy (t->new_dst_mac, en0->dst_address, - sizeof (t->new_dst_mac)); + clib_memcpy_fast (t->new_src_mac, en0->src_address, + sizeof (t->new_src_mac)); + clib_memcpy_fast (t->new_dst_mac, en0->dst_address, + sizeof (t->new_dst_mac)); } pkts_swapped += 1; @@ -585,10 +585,10 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node, t->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; t->next_index = SAMPLE_NEXT_INTERFACE_OUTPUT; en = vlib_buffer_get_current (b[0]); - clib_memcpy (t->new_src_mac, en->src_address, - sizeof (t->new_src_mac)); - clib_memcpy (t->new_dst_mac, en->dst_address, - sizeof (t->new_dst_mac)); + clib_memcpy_fast (t->new_src_mac, en->src_address, + sizeof (t->new_src_mac)); + clib_memcpy_fast (t->new_dst_mac, en->dst_address, + sizeof (t->new_dst_mac)); b++; } else diff --git a/src/plugins/acl/hash_lookup.c b/src/plugins/acl/hash_lookup.c index 8c1004e9212..aeec004d77a 100644 --- a/src/plugins/acl/hash_lookup.c +++ b/src/plugins/acl/hash_lookup.c @@ -280,7 +280,7 @@ assign_mask_type_index(acl_main_t *am, fa_5tuple_t *mask) if(~0 == mask_type_index) { pool_get_aligned (am->ace_mask_type_pool, mte, CLIB_CACHE_LINE_BYTES); mask_type_index = mte - am->ace_mask_type_pool; - clib_memcpy(&mte->mask, mask, sizeof(mte->mask)); + clib_memcpy_fast(&mte->mask, mask, sizeof(mte->mask)); mte->refcount = 0; /* @@ -1500,8 +1500,8 @@ split_partition(acl_main_t *am, u32 first_index, if(pae->mask_type_index != coll_mask_type_index) continue; /* Computing min_mask and max_mask for colliding rules */ if(i==0){ - clib_memcpy(min_tuple, mask, sizeof(fa_5tuple_t)); - clib_memcpy(max_tuple, mask, sizeof(fa_5tuple_t)); + clib_memcpy_fast(min_tuple, mask, sizeof(fa_5tuple_t)); + clib_memcpy_fast(max_tuple, mask, sizeof(fa_5tuple_t)); }else{ int j; for(j=0; j<2; j++){ diff --git a/src/plugins/avf/device.c b/src/plugins/avf/device.c index 6bf1fe05f05..d95de2ec892 100644 --- a/src/plugins/avf/device.c +++ b/src/plugins/avf/device.c @@ -106,7 +106,7 @@ avf_aq_desc_enq (vlib_main_t * vm, avf_device_t * ad, avf_aq_desc_t * dt, int n_retry = 5; d = &ad->atq[ad->atq_next_slot]; - clib_memcpy (d, dt, sizeof (avf_aq_desc_t)); + clib_memcpy_fast (d, dt, sizeof (avf_aq_desc_t)); d->flags |= AVF_AQ_F_RD | AVF_AQ_F_SI; if (len) d->datalen = len; @@ -116,13 +116,13 @@ avf_aq_desc_enq (vlib_main_t * vm, avf_device_t * ad, avf_aq_desc_t * dt, pa = ad->atq_bufs_pa + ad->atq_next_slot * AVF_MBOX_BUF_SZ; d->addr_hi = (u32) (pa >> 32); d->addr_lo = (u32) pa; - clib_memcpy (ad->atq_bufs + ad->atq_next_slot * AVF_MBOX_BUF_SZ, data, - len); + clib_memcpy_fast (ad->atq_bufs + ad->atq_next_slot * AVF_MBOX_BUF_SZ, + data, len); d->flags |= AVF_AQ_F_BUF; } if (ad->flags & AVF_DEVICE_F_ELOG) - clib_memcpy (&dc, d, sizeof (avf_aq_desc_t)); + clib_memcpy_fast (&dc, d, sizeof (avf_aq_desc_t)); CLIB_MEMORY_BARRIER (); vlib_log_debug (am->log_class, "%U", format_hexdump, data, len); @@ -144,7 +144,7 @@ retry: goto retry; } - clib_memcpy (dt, d, sizeof (avf_aq_desc_t)); + clib_memcpy_fast (dt, d, sizeof (avf_aq_desc_t)); if (d->flags & AVF_AQ_F_ERR) return clib_error_return (0, "adminq enqueue error [opcode 0x%x, retval " "%d]", d->opcode, d->retval); @@ -398,7 +398,7 @@ retry: return clib_error_return (0, "event message error"); vec_add2 (ad->events, e, 1); - clib_memcpy (e, buf, sizeof (virtchnl_pf_event_t)); + clib_memcpy_fast (e, buf, sizeof (virtchnl_pf_event_t)); avf_arq_slot_init (ad, ad->arq_next_slot); ad->arq_next_slot++; n_retry = 5; @@ -425,7 +425,7 @@ retry: if (d->flags & AVF_AQ_F_BUF) { void *buf = ad->arq_bufs + ad->arq_next_slot * AVF_MBOX_BUF_SZ; - clib_memcpy (out, buf, out_len); + clib_memcpy_fast (out, buf, out_len); } avf_arq_slot_init (ad, ad->arq_next_slot); @@ -637,7 +637,7 @@ avf_op_add_eth_addr (vlib_main_t * vm, avf_device_t * ad, u8 count, u8 * macs) al->vsi_id = ad->vsi_id; al->num_elements = count; for (i = 0; i < count; i++) - clib_memcpy (&al->list[i].addr, macs + i * 6, 6); + clib_memcpy_fast (&al->list[i].addr, macs + i * 6, 6); return avf_send_to_pf (vm, ad, VIRTCHNL_OP_ADD_ETH_ADDR, msg, msg_len, 0, 0); } @@ -788,7 +788,7 @@ avf_device_init (vlib_main_t * vm, avf_main_t * am, avf_device_t * ad, ad->rss_key_size = res.rss_key_size; ad->rss_lut_size = res.rss_lut_size; - clib_memcpy (ad->hwaddr, res.vsi_res[0].default_mac_addr, 6); + clib_memcpy_fast (ad->hwaddr, res.vsi_res[0].default_mac_addr, 6); /* * Disable VLAN stripping diff --git a/src/plugins/avf/input.c b/src/plugins/avf/input.c index 6d39ed646b5..509b4970dad 100644 --- a/src/plugins/avf/input.c +++ b/src/plugins/avf/input.c @@ -258,14 +258,14 @@ avf_process_rx_burst (vlib_main_t * vm, vlib_node_runtime_t * node, vnet_buffer (bt)->feature_arc_index; } - clib_memcpy (vnet_buffer (b[0])->sw_if_index, - vnet_buffer (bt)->sw_if_index, 2 * sizeof (u32)); - clib_memcpy (vnet_buffer (b[1])->sw_if_index, - vnet_buffer (bt)->sw_if_index, 2 * sizeof (u32)); - clib_memcpy (vnet_buffer (b[2])->sw_if_index, - vnet_buffer (bt)->sw_if_index, 2 * sizeof (u32)); - clib_memcpy (vnet_buffer (b[3])->sw_if_index, - vnet_buffer (bt)->sw_if_index, 2 * sizeof (u32)); + clib_memcpy_fast (vnet_buffer (b[0])->sw_if_index, + vnet_buffer (bt)->sw_if_index, 2 * sizeof (u32)); + clib_memcpy_fast (vnet_buffer (b[1])->sw_if_index, + vnet_buffer (bt)->sw_if_index, 2 * sizeof (u32)); + clib_memcpy_fast (vnet_buffer (b[2])->sw_if_index, + vnet_buffer (bt)->sw_if_index, 2 * sizeof (u32)); + clib_memcpy_fast (vnet_buffer (b[3])->sw_if_index, + vnet_buffer (bt)->sw_if_index, 2 * sizeof (u32)); VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]); @@ -295,8 +295,8 @@ avf_process_rx_burst (vlib_main_t * vm, vlib_node_runtime_t * node, vnet_buffer (bt)->feature_arc_index; } - clib_memcpy (vnet_buffer (b[0])->sw_if_index, - vnet_buffer (bt)->sw_if_index, 2 * sizeof (u32)); + clib_memcpy_fast (vnet_buffer (b[0])->sw_if_index, + vnet_buffer (bt)->sw_if_index, 2 * sizeof (u32)); VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); @@ -392,7 +392,7 @@ avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u64x4_store_unaligned (v, ptd->rx_vector + n_rxv); maybe_error |= !u64x4_is_all_zero (err4); - clib_memcpy (bi, rxq->bufs + rxq->next, 4 * sizeof (u32)); + clib_memcpy_fast (bi, rxq->bufs + rxq->next, 4 * sizeof (u32)); /* next */ rxq->next = (rxq->next + 4) & mask; @@ -486,7 +486,7 @@ avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tr = vlib_add_trace (vm, node, b, sizeof (*tr)); tr->next_index = next[0]; tr->hw_if_index = ad->hw_if_index; - clib_memcpy (&tr->rxve, rxve, sizeof (avf_rx_vector_entry_t)); + clib_memcpy_fast (&tr->rxve, rxve, sizeof (avf_rx_vector_entry_t)); /* next */ n_trace--; diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index ea00df25a25..9b54f34a5b2 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -80,12 +80,13 @@ dpdk_tx_trace_buffer (dpdk_main_t * dm, vlib_node_runtime_t * node, t0->queue_index = queue_id; t0->device_index = xd->device_index; t0->buffer_index = vlib_get_buffer_index (vm, buffer); - clib_memcpy (&t0->mb, mb, sizeof (t0->mb)); - clib_memcpy (&t0->buffer, buffer, - sizeof (buffer[0]) - sizeof (buffer->pre_data)); - clib_memcpy (t0->buffer.pre_data, buffer->data + buffer->current_data, - sizeof (t0->buffer.pre_data)); - clib_memcpy (&t0->data, mb->buf_addr + mb->data_off, sizeof (t0->data)); + clib_memcpy_fast (&t0->mb, mb, sizeof (t0->mb)); + clib_memcpy_fast (&t0->buffer, buffer, + sizeof (buffer[0]) - sizeof (buffer->pre_data)); + clib_memcpy_fast (t0->buffer.pre_data, buffer->data + buffer->current_data, + sizeof (t0->buffer.pre_data)); + clib_memcpy_fast (&t0->data, mb->buf_addr + mb->data_off, + sizeof (t0->data)); } static_always_inline void @@ -430,10 +431,10 @@ dpdk_clear_hw_interface_counters (u32 instance) */ dpdk_update_counters (xd, vlib_time_now (dm->vlib_main)); - clib_memcpy (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats)); - clib_memcpy (xd->last_cleared_xstats, xd->xstats, - vec_len (xd->last_cleared_xstats) * - sizeof (xd->last_cleared_xstats[0])); + clib_memcpy_fast (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats)); + clib_memcpy_fast (xd->last_cleared_xstats, xd->xstats, + vec_len (xd->last_cleared_xstats) * + sizeof (xd->last_cleared_xstats[0])); } diff --git a/src/plugins/dpdk/device/dpdk_priv.h b/src/plugins/dpdk/device/dpdk_priv.h index e5db674ca6d..b44cd49142c 100644 --- a/src/plugins/dpdk/device/dpdk_priv.h +++ b/src/plugins/dpdk/device/dpdk_priv.h @@ -97,7 +97,7 @@ dpdk_update_counters (dpdk_device_t * xd, f64 now) return; xd->time_last_stats_update = now ? now : xd->time_last_stats_update; - clib_memcpy (&xd->last_stats, &xd->stats, sizeof (xd->last_stats)); + clib_memcpy_fast (&xd->last_stats, &xd->stats, sizeof (xd->last_stats)); rte_eth_stats_get (xd->port_id, &xd->stats); /* maybe bump interface rx no buffer counter */ diff --git a/src/plugins/dpdk/device/flow.c b/src/plugins/dpdk/device/flow.c index 6612ce4e1ad..63f04b8323d 100644 --- a/src/plugins/dpdk/device/flow.c +++ b/src/plugins/dpdk/device/flow.c @@ -84,10 +84,10 @@ dpdk_flow_add (dpdk_device_t * xd, vnet_flow_t * f, dpdk_flow_entry_t * fe) if (f->type == VNET_FLOW_TYPE_IP6_N_TUPLE) { vnet_flow_ip6_n_tuple_t *t6 = &f->ip6_n_tuple; - clib_memcpy (ip6[0].hdr.src_addr, &t6->src_addr.addr, 16); - clib_memcpy (ip6[1].hdr.src_addr, &t6->src_addr.mask, 16); - clib_memcpy (ip6[0].hdr.dst_addr, &t6->dst_addr.addr, 16); - clib_memcpy (ip6[1].hdr.dst_addr, &t6->dst_addr.mask, 16); + clib_memcpy_fast (ip6[0].hdr.src_addr, &t6->src_addr.addr, 16); + clib_memcpy_fast (ip6[1].hdr.src_addr, &t6->src_addr.mask, 16); + clib_memcpy_fast (ip6[0].hdr.dst_addr, &t6->dst_addr.addr, 16); + clib_memcpy_fast (ip6[1].hdr.dst_addr, &t6->dst_addr.mask, 16); item->type = RTE_FLOW_ITEM_TYPE_IPV6; item->spec = ip6; item->mask = ip6 + 1; @@ -183,9 +183,9 @@ dpdk_flow_add (dpdk_device_t * xd, vnet_flow_t * f, dpdk_flow_entry_t * fe) raw[0].item.relative = 1; raw[0].item.length = vxlan_hdr_sz; - clib_memcpy (raw[0].val + raw_sz, &spec_hdr, vxlan_hdr_sz); + clib_memcpy_fast (raw[0].val + raw_sz, &spec_hdr, vxlan_hdr_sz); raw[0].item.pattern = raw[0].val + raw_sz; - clib_memcpy (raw[1].val + raw_sz, &mask_hdr, vxlan_hdr_sz); + clib_memcpy_fast (raw[1].val + raw_sz, &mask_hdr, vxlan_hdr_sz); raw[1].item.pattern = raw[1].val + raw_sz; vec_add2 (items, item, 1); diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index e28ad7f7108..6a3c1c936fc 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -304,7 +304,7 @@ dpdk_process_rx_burst (vlib_main_t * vm, dpdk_per_thread_data_t * ptd, while (n_left) { b[0] = vlib_buffer_from_rte_mbuf (mb[0]); - clib_memcpy (b[0], &ptd->buffer_template, 64); + clib_memcpy_fast (b[0], &ptd->buffer_template, 64); or_flags |= dpdk_ol_flags_extract (mb, flags, 1); flags += 1; @@ -549,12 +549,13 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd, t0->device_index = xd->device_index; t0->buffer_index = vlib_get_buffer_index (vm, b0); - clib_memcpy (&t0->mb, mb[0], sizeof t0->mb); - clib_memcpy (&t0->buffer, b0, sizeof b0[0] - sizeof b0->pre_data); - clib_memcpy (t0->buffer.pre_data, b0->data, - sizeof t0->buffer.pre_data); - clib_memcpy (&t0->data, mb[0]->buf_addr + mb[0]->data_off, - sizeof t0->data); + clib_memcpy_fast (&t0->mb, mb[0], sizeof t0->mb); + clib_memcpy_fast (&t0->buffer, b0, + sizeof b0[0] - sizeof b0->pre_data); + clib_memcpy_fast (t0->buffer.pre_data, b0->data, + sizeof t0->buffer.pre_data); + clib_memcpy_fast (&t0->data, mb[0]->buf_addr + mb[0]->data_off, + sizeof t0->data); n_trace--; n_left--; buffers++; diff --git a/src/plugins/dpdk/ipsec/esp_decrypt.c b/src/plugins/dpdk/ipsec/esp_decrypt.c index 6d8d6bbeaf0..f0fa506debb 100644 --- a/src/plugins/dpdk/ipsec/esp_decrypt.c +++ b/src/plugins/dpdk/ipsec/esp_decrypt.c @@ -304,7 +304,7 @@ dpdk_esp_decrypt_inline (vlib_main_t * vm, mb0->buf_physaddr + digest - ((u8 *) mb0->buf_addr); if (!is_aead && cipher_alg->alg == RTE_CRYPTO_CIPHER_AES_CBC) - clib_memcpy(icb, iv, 16); + clib_memcpy_fast(icb, iv, 16); else /* CTR/GCM */ { u32 *_iv = (u32 *) iv; @@ -316,7 +316,7 @@ dpdk_esp_decrypt_inline (vlib_main_t * vm, { aad = priv->aad; u32 * _aad = (u32 *) aad; - clib_memcpy (aad, esp0, 8); + clib_memcpy_fast (aad, esp0, 8); /* _aad[3] should always be 0 */ if (PREDICT_FALSE (sa0->use_esn)) @@ -330,7 +330,7 @@ dpdk_esp_decrypt_inline (vlib_main_t * vm, if (sa0->use_esn) { - clib_memcpy (priv->icv, digest, trunc_size); + clib_memcpy_fast (priv->icv, digest, trunc_size); u32 *_digest = (u32 *) digest; _digest[0] = clib_host_to_net_u32 (sa0->seq_hi); auth_len += sizeof(sa0->seq_hi); @@ -349,7 +349,7 @@ trace: esp_decrypt_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->crypto_alg = sa0->crypto_alg; tr->integ_alg = sa0->integ_alg; - clib_memcpy (tr->packet_data, vlib_buffer_get_current (b0), + clib_memcpy_fast (tr->packet_data, vlib_buffer_get_current (b0), sizeof (esp_header_t)); } } @@ -647,7 +647,7 @@ dpdk_esp_decrypt_post_inline (vlib_main_t * vm, tr->crypto_alg = sa0->crypto_alg; tr->integ_alg = sa0->integ_alg; ih4 = vlib_buffer_get_current (b0); - clib_memcpy (tr->packet_data, ih4, sizeof (ip6_header_t)); + clib_memcpy_fast (tr->packet_data, ih4, sizeof (ip6_header_t)); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, diff --git a/src/plugins/dpdk/ipsec/esp_encrypt.c b/src/plugins/dpdk/ipsec/esp_encrypt.c index fb3f2a41b48..2259b1b7b81 100644 --- a/src/plugins/dpdk/ipsec/esp_encrypt.c +++ b/src/plugins/dpdk/ipsec/esp_encrypt.c @@ -444,7 +444,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm, /* The extra pad bytes would be overwritten by the digest */ if (pad_bytes) - clib_memcpy (padding, pad_data, 16); + clib_memcpy_fast (padding, pad_data, 16); f0 = (esp_footer_t *) (padding + pad_bytes); f0->pad_length = pad_bytes; @@ -538,7 +538,7 @@ dpdk_esp_encrypt_inline (vlib_main_t * vm, u8 *p = vlib_buffer_get_current (b0); if (!sa0->is_tunnel) p += vnet_buffer (b0)->ip.save_rewrite_length; - clib_memcpy (tr->packet_data, p, sizeof (tr->packet_data)); + clib_memcpy_fast (tr->packet_data, p, sizeof (tr->packet_data)); } } vlib_put_next_frame (vm, node, next_index, n_left_to_next); diff --git a/src/plugins/flowprobe/node.c b/src/plugins/flowprobe/node.c index 34f50aacbbc..79d37f8a75e 100644 --- a/src/plugins/flowprobe/node.c +++ b/src/plugins/flowprobe/node.c @@ -157,33 +157,33 @@ flowprobe_common_add (vlib_buffer_t * to_b, flowprobe_entry_t * e, u16 offset) /* Ingress interface */ u32 rx_if = clib_host_to_net_u32 (e->key.rx_sw_if_index); - clib_memcpy (to_b->data + offset, &rx_if, sizeof (rx_if)); + clib_memcpy_fast (to_b->data + offset, &rx_if, sizeof (rx_if)); offset += sizeof (rx_if); /* Egress interface */ u32 tx_if = clib_host_to_net_u32 (e->key.tx_sw_if_index); - clib_memcpy (to_b->data + offset, &tx_if, sizeof (tx_if)); + clib_memcpy_fast (to_b->data + offset, &tx_if, sizeof (tx_if)); offset += sizeof (tx_if); /* packet delta count */ u64 packetdelta = clib_host_to_net_u64 (e->packetcount); - clib_memcpy (to_b->data + offset, &packetdelta, sizeof (u64)); + clib_memcpy_fast (to_b->data + offset, &packetdelta, sizeof (u64)); offset += sizeof (u64); /* flowStartNanoseconds */ u32 t = clib_host_to_net_u32 (e->flow_start.sec + NTP_TIMESTAMP); - clib_memcpy (to_b->data + offset, &t, sizeof (u32)); + clib_memcpy_fast (to_b->data + offset, &t, sizeof (u32)); offset += sizeof (u32); t = clib_host_to_net_u32 (e->flow_start.nsec); - clib_memcpy (to_b->data + offset, &t, sizeof (u32)); + clib_memcpy_fast (to_b->data + offset, &t, sizeof (u32)); offset += sizeof (u32); /* flowEndNanoseconds */ t = clib_host_to_net_u32 (e->flow_end.sec + NTP_TIMESTAMP); - clib_memcpy (to_b->data + offset, &t, sizeof (u32)); + clib_memcpy_fast (to_b->data + offset, &t, sizeof (u32)); offset += sizeof (u32); t = clib_host_to_net_u32 (e->flow_end.nsec); - clib_memcpy (to_b->data + offset, &t, sizeof (u32)); + clib_memcpy_fast (to_b->data + offset, &t, sizeof (u32)); offset += sizeof (u32); return offset - start; @@ -195,15 +195,15 @@ flowprobe_l2_add (vlib_buffer_t * to_b, flowprobe_entry_t * e, u16 offset) u16 start = offset; /* src mac address */ - clib_memcpy (to_b->data + offset, &e->key.src_mac, 6); + clib_memcpy_fast (to_b->data + offset, &e->key.src_mac, 6); offset += 6; /* dst mac address */ - clib_memcpy (to_b->data + offset, &e->key.dst_mac, 6); + clib_memcpy_fast (to_b->data + offset, &e->key.dst_mac, 6); offset += 6; /* ethertype */ - clib_memcpy (to_b->data + offset, &e->key.ethertype, 2); + clib_memcpy_fast (to_b->data + offset, &e->key.ethertype, 2); offset += 2; return offset - start; @@ -215,13 +215,13 @@ flowprobe_l3_ip6_add (vlib_buffer_t * to_b, flowprobe_entry_t * e, u16 offset) u16 start = offset; /* ip6 src address */ - clib_memcpy (to_b->data + offset, &e->key.src_address, - sizeof (ip6_address_t)); + clib_memcpy_fast (to_b->data + offset, &e->key.src_address, + sizeof (ip6_address_t)); offset += sizeof (ip6_address_t); /* ip6 dst address */ - clib_memcpy (to_b->data + offset, &e->key.dst_address, - sizeof (ip6_address_t)); + clib_memcpy_fast (to_b->data + offset, &e->key.dst_address, + sizeof (ip6_address_t)); offset += sizeof (ip6_address_t); /* Protocol */ @@ -229,7 +229,7 @@ flowprobe_l3_ip6_add (vlib_buffer_t * to_b, flowprobe_entry_t * e, u16 offset) /* octetDeltaCount */ u64 octetdelta = clib_host_to_net_u64 (e->octetcount); - clib_memcpy (to_b->data + offset, &octetdelta, sizeof (u64)); + clib_memcpy_fast (to_b->data + offset, &octetdelta, sizeof (u64)); offset += sizeof (u64); return offset - start; @@ -241,13 +241,13 @@ flowprobe_l3_ip4_add (vlib_buffer_t * to_b, flowprobe_entry_t * e, u16 offset) u16 start = offset; /* ip4 src address */ - clib_memcpy (to_b->data + offset, &e->key.src_address.ip4, - sizeof (ip4_address_t)); + clib_memcpy_fast (to_b->data + offset, &e->key.src_address.ip4, + sizeof (ip4_address_t)); offset += sizeof (ip4_address_t); /* ip4 dst address */ - clib_memcpy (to_b->data + offset, &e->key.dst_address.ip4, - sizeof (ip4_address_t)); + clib_memcpy_fast (to_b->data + offset, &e->key.dst_address.ip4, + sizeof (ip4_address_t)); offset += sizeof (ip4_address_t); /* Protocol */ @@ -255,7 +255,7 @@ flowprobe_l3_ip4_add (vlib_buffer_t * to_b, flowprobe_entry_t * e, u16 offset) /* octetDeltaCount */ u64 octetdelta = clib_host_to_net_u64 (e->octetcount); - clib_memcpy (to_b->data + offset, &octetdelta, sizeof (u64)); + clib_memcpy_fast (to_b->data + offset, &octetdelta, sizeof (u64)); offset += sizeof (u64); return offset - start; @@ -267,16 +267,16 @@ flowprobe_l4_add (vlib_buffer_t * to_b, flowprobe_entry_t * e, u16 offset) u16 start = offset; /* src port */ - clib_memcpy (to_b->data + offset, &e->key.src_port, 2); + clib_memcpy_fast (to_b->data + offset, &e->key.src_port, 2); offset += 2; /* dst port */ - clib_memcpy (to_b->data + offset, &e->key.dst_port, 2); + clib_memcpy_fast (to_b->data + offset, &e->key.dst_port, 2); offset += 2; /* tcp control bits */ u16 control_bits = htons (e->prot.tcp.flags); - clib_memcpy (to_b->data + offset, &control_bits, 2); + clib_memcpy_fast (to_b->data + offset, &control_bits, 2); offset += 2; return offset - start; @@ -394,8 +394,8 @@ add_to_flow_record_state (vlib_main_t * vm, vlib_node_runtime_t * node, if (flags & FLOW_RECORD_L2) { - clib_memcpy (k.src_mac, eth->src_address, 6); - clib_memcpy (k.dst_mac, eth->dst_address, 6); + clib_memcpy_fast (k.src_mac, eth->src_address, 6); + clib_memcpy_fast (k.dst_mac, eth->dst_address, 6); k.ethertype = ethertype; } if (collect_ip6 && ethertype == ETHERNET_TYPE_IP6) @@ -450,8 +450,8 @@ add_to_flow_record_state (vlib_main_t * vm, vlib_node_runtime_t * node, { t->rx_sw_if_index = k.rx_sw_if_index; t->tx_sw_if_index = k.tx_sw_if_index; - clib_memcpy (t->src_mac, k.src_mac, 6); - clib_memcpy (t->dst_mac, k.dst_mac, 6); + clib_memcpy_fast (t->src_mac, k.src_mac, 6); + clib_memcpy_fast (t->dst_mac, k.dst_mac, 6); t->ethertype = k.ethertype; t->src_address.ip4.as_u32 = k.src_address.ip4.as_u32; t->dst_address.ip4.as_u32 = k.dst_address.ip4.as_u32; diff --git a/src/plugins/gbp/gbp_learn.c b/src/plugins/gbp/gbp_learn.c index 883f3d1c862..cd6a15d955d 100644 --- a/src/plugins/gbp/gbp_learn.c +++ b/src/plugins/gbp/gbp_learn.c @@ -338,7 +338,7 @@ gbp_learn_l2 (vlib_main_t * vm, { gbp_learn_l2_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); - clib_memcpy (t->mac.bytes, eh0->src_address, 6); + clib_memcpy_fast (t->mac.bytes, eh0->src_address, 6); t->new = (NULL == ge0); t->throttled = t0; t->sw_if_index = sw_if_index0; diff --git a/src/plugins/gtpu/gtpu_decap.c b/src/plugins/gtpu/gtpu_decap.c index c4e6c85418e..b2ac553da9c 100644 --- a/src/plugins/gtpu/gtpu_decap.c +++ b/src/plugins/gtpu/gtpu_decap.c @@ -256,7 +256,7 @@ gtpu_input (vlib_main_t * vm, next0 = GTPU_INPUT_NEXT_DROP; goto trace0; } - clib_memcpy (&last_key6, &key6_0, sizeof(key6_0)); + clib_memcpy_fast (&last_key6, &key6_0, sizeof(key6_0)); tunnel_index0 = last_tunnel_index = p0[0]; } else @@ -432,7 +432,7 @@ gtpu_input (vlib_main_t * vm, goto trace1; } - clib_memcpy (&last_key6, &key6_1, sizeof(key6_1)); + clib_memcpy_fast (&last_key6, &key6_1, sizeof(key6_1)); tunnel_index1 = last_tunnel_index = p1[0]; } else @@ -659,7 +659,7 @@ gtpu_input (vlib_main_t * vm, next0 = GTPU_INPUT_NEXT_DROP; goto trace00; } - clib_memcpy (&last_key6, &key6_0, sizeof(key6_0)); + clib_memcpy_fast (&last_key6, &key6_0, sizeof(key6_0)); tunnel_index0 = last_tunnel_index = p0[0]; } else diff --git a/src/plugins/igmp/igmp_input.c b/src/plugins/igmp/igmp_input.c index 5f54a0ba223..83e11ec6cc5 100644 --- a/src/plugins/igmp/igmp_input.c +++ b/src/plugins/igmp/igmp_input.c @@ -203,8 +203,8 @@ igmp_input (vlib_main_t * vm, vlib_node_runtime_t * node, tr = vlib_add_trace (vm, node, b, sizeof (*tr)); tr->next_index = next; tr->sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX]; - clib_memcpy (tr->packet_data, vlib_buffer_get_current (b), - sizeof (tr->packet_data)); + clib_memcpy_fast (tr->packet_data, vlib_buffer_get_current (b), + sizeof (tr->packet_data)); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, @@ -280,8 +280,8 @@ igmp_parse_query (vlib_main_t * vm, vlib_node_runtime_t * node, tr = vlib_add_trace (vm, node, b, sizeof (*tr)); tr->next_index = next; tr->sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX]; - clib_memcpy (tr->packet_data, vlib_buffer_get_current (b), - sizeof (tr->packet_data)); + clib_memcpy_fast (tr->packet_data, vlib_buffer_get_current (b), + sizeof (tr->packet_data)); } len = igmp_membership_query_v3_length (igmp); @@ -387,8 +387,8 @@ igmp_parse_report (vlib_main_t * vm, vlib_node_runtime_t * node, tr = vlib_add_trace (vm, node, b, sizeof (*tr)); tr->next_index = next; tr->sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX]; - clib_memcpy (tr->packet_data, vlib_buffer_get_current (b), - sizeof (tr->packet_data)); + clib_memcpy_fast (tr->packet_data, vlib_buffer_get_current (b), + sizeof (tr->packet_data)); } /* diff --git a/src/plugins/igmp/igmp_pkt.c b/src/plugins/igmp/igmp_pkt.c index c2a5d0911c6..81b8ecb18bc 100644 --- a/src/plugins/igmp/igmp_pkt.c +++ b/src/plugins/igmp/igmp_pkt.c @@ -92,8 +92,8 @@ igmp_pkt_build_ip_header (igmp_pkt_build_t * bk, break; case IGMP_MSG_QUERY: if (group != NULL) - clib_memcpy (&ip4->dst_address, &group->key->ip4, - sizeof (ip4_address_t)); + clib_memcpy_fast (&ip4->dst_address, &group->key->ip4, + sizeof (ip4_address_t)); else ip4->dst_address.as_u32 = IGMP_GENERAL_QUERY_ADDRESS; break; diff --git a/src/plugins/ioam/export-common/ioam_export.h b/src/plugins/ioam/export-common/ioam_export.h index 511cb88a73b..672576ebded 100644 --- a/src/plugins/ioam/export-common/ioam_export.h +++ b/src/plugins/ioam/export-common/ioam_export.h @@ -117,7 +117,7 @@ ioam_export_get_my_buffer (ioam_export_main_t * em, u32 thread_id) inline static int ioam_export_buffer_add_header (ioam_export_main_t * em, vlib_buffer_t * b0) { - clib_memcpy (b0->data, em->record_header, vec_len (em->record_header)); + clib_memcpy_fast (b0->data, em->record_header, vec_len (em->record_header)); b0->current_data = 0; b0->current_length = vec_len (em->record_header); b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; diff --git a/src/plugins/ioam/ip6/ioam_cache.h b/src/plugins/ioam/ip6/ioam_cache.h index 717aaf11b8f..e75deba73f7 100644 --- a/src/plugins/ioam/ip6/ioam_cache.h +++ b/src/plugins/ioam/ip6/ioam_cache.h @@ -374,10 +374,10 @@ ioam_cache_add (vlib_buffer_t * b0, clib_memset (entry, 0, sizeof (*entry)); pool_index = entry - cm->ioam_rewrite_pool; - clib_memcpy (entry->dst_address.as_u64, ip0->dst_address.as_u64, - sizeof (ip6_address_t)); - clib_memcpy (entry->src_address.as_u64, ip0->src_address.as_u64, - sizeof (ip6_address_t)); + clib_memcpy_fast (entry->dst_address.as_u64, ip0->dst_address.as_u64, + sizeof (ip6_address_t)); + clib_memcpy_fast (entry->src_address.as_u64, ip0->src_address.as_u64, + sizeof (ip6_address_t)); entry->src_port = src_port; entry->dst_port = dst_port; entry->seq_no = seq_no; @@ -395,7 +395,7 @@ ioam_cache_add (vlib_buffer_t * b0, } e2e_id_offset = (u8 *) e2e - (u8 *) hbh0; /* setup e2e id option to insert v6 address of the node caching it */ - clib_memcpy (entry->ioam_rewrite_string, hbh0, rewrite_len); + clib_memcpy_fast (entry->ioam_rewrite_string, hbh0, rewrite_len); hbh0 = (ip6_hop_by_hop_header_t *) entry->ioam_rewrite_string; /* suffix rewrite string with e2e ID option */ @@ -657,10 +657,10 @@ ioam_cache_ts_add (ip6_header_t * ip0, clib_memset (entry, 0, sizeof (*entry)); *pool_index = entry - cm->ioam_ts_pool[thread_id]; - clib_memcpy (entry->dst_address.as_u64, ip0->dst_address.as_u64, - sizeof (ip6_address_t)); - clib_memcpy (entry->src_address.as_u64, ip0->src_address.as_u64, - sizeof (ip6_address_t)); + clib_memcpy_fast (entry->dst_address.as_u64, ip0->dst_address.as_u64, + sizeof (ip6_address_t)); + clib_memcpy_fast (entry->src_address.as_u64, ip0->src_address.as_u64, + sizeof (ip6_address_t)); entry->src_port = src_port; entry->dst_port = dst_port; entry->seq_no = seq_no; diff --git a/src/plugins/ioam/ip6/ioam_cache_node.c b/src/plugins/ioam/ip6/ioam_cache_node.c index dd27e127a09..59bc1a64811 100644 --- a/src/plugins/ioam/ip6/ioam_cache_node.c +++ b/src/plugins/ioam/ip6/ioam_cache_node.c @@ -344,17 +344,17 @@ ip6_add_from_cache_hbh_node_fn (vlib_main_t * vm, hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1); srh0 = (ip6_sr_header_t *) ((u8 *) hbh0 + rewrite_len); /* $$$ tune, rewrite_len is a multiple of 8 */ - clib_memcpy (hbh0, rewrite, rewrite_len); - clib_memcpy (srh0, cm->sr_rewrite_template, sr_rewrite_len); + clib_memcpy_fast (hbh0, rewrite, rewrite_len); + clib_memcpy_fast (srh0, cm->sr_rewrite_template, sr_rewrite_len); /* Copy dst address into the DA slot in the segment list */ - clib_memcpy (srh0->segments, ip0->dst_address.as_u64, - sizeof (ip6_address_t)); + clib_memcpy_fast (srh0->segments, ip0->dst_address.as_u64, + sizeof (ip6_address_t)); /* Rewrite the ip6 dst address with the first hop */ - clib_memcpy (ip0->dst_address.as_u64, entry->next_hop.as_u64, - sizeof (ip6_address_t)); - clib_memcpy (&srh0->segments[1], - (u8 *) hbh0 + entry->my_address_offset, - sizeof (ip6_address_t)); + clib_memcpy_fast (ip0->dst_address.as_u64, entry->next_hop.as_u64, + sizeof (ip6_address_t)); + clib_memcpy_fast (&srh0->segments[1], + (u8 *) hbh0 + entry->my_address_offset, + sizeof (ip6_address_t)); ioam_cache_entry_free (entry); /* Patch the protocol chain, insert the h-b-h (type 0) header */ diff --git a/src/plugins/ioam/ip6/ioam_cache_tunnel_select_node.c b/src/plugins/ioam/ip6/ioam_cache_tunnel_select_node.c index 79ee58eccda..63bcaff14b2 100644 --- a/src/plugins/ioam/ip6/ioam_cache_tunnel_select_node.c +++ b/src/plugins/ioam/ip6/ioam_cache_tunnel_select_node.c @@ -415,7 +415,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1); /* $$$ tune, rewrite_length is a multiple of 8 */ - clib_memcpy (hbh0, rewrite, rewrite_length); + clib_memcpy_fast (hbh0, rewrite, rewrite_length); e2e = (ioam_e2e_cache_option_t *) ((u8 *) hbh0 + cm->rewrite_pool_index_offset); @@ -475,7 +475,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, hbh1 = (ip6_hop_by_hop_header_t *) (ip1 + 1); /* $$$ tune, rewrite_length is a multiple of 8 */ - clib_memcpy (hbh1, rewrite, rewrite_length); + clib_memcpy_fast (hbh1, rewrite, rewrite_length); e2e = (ioam_e2e_cache_option_t *) ((u8 *) hbh1 + cm->rewrite_pool_index_offset); @@ -581,7 +581,7 @@ ip6_reset_ts_hbh_node_fn (vlib_main_t * vm, hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1); /* $$$ tune, rewrite_length is a multiple of 8 */ - clib_memcpy (hbh0, rewrite, rewrite_length); + clib_memcpy_fast (hbh0, rewrite, rewrite_length); e2e = (ioam_e2e_cache_option_t *) ((u8 *) hbh0 + cm->rewrite_pool_index_offset); diff --git a/src/plugins/ioam/lib-vxlan-gpe/ioam_pop.c b/src/plugins/ioam/lib-vxlan-gpe/ioam_pop.c index 7a4580d804f..84900eb7e01 100644 --- a/src/plugins/ioam/lib-vxlan-gpe/ioam_pop.c +++ b/src/plugins/ioam/lib-vxlan-gpe/ioam_pop.c @@ -203,7 +203,7 @@ trace00: option_data) ? trace_len : ARRAY_LEN (t->fmt_trace. option_data); t->fmt_trace.trace_len = trace_len; - clib_memcpy (&(t->fmt_trace.option_data), gpe_ioam0, trace_len); + clib_memcpy_fast (&(t->fmt_trace.option_data), gpe_ioam0, trace_len); } /* Remove the iOAM header inside the VxLAN-GPE header */ diff --git a/src/plugins/ioam/udp-ping/udp_ping_util.c b/src/plugins/ioam/udp-ping/udp_ping_util.c index 55f48ea4999..d3612cd01e0 100644 --- a/src/plugins/ioam/udp-ping/udp_ping_util.c +++ b/src/plugins/ioam/udp-ping/udp_ping_util.c @@ -266,7 +266,7 @@ udp_ping_send_ip6_pak (vlib_main_t * vm, ip46_udp_ping_flow * flow) int bogus; b0 = vlib_get_buffer (vm, buffers[i]); stats = flow->udp_data.stats + i; - clib_memcpy (b0->data, stats->ping_rewrite, stats->rewrite_len); + clib_memcpy_fast (b0->data, stats->ping_rewrite, stats->rewrite_len); b0->current_data = 0; b0->current_length = stats->rewrite_len; b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; diff --git a/src/plugins/lacp/node.c b/src/plugins/lacp/node.c index 447e85fd373..d86972666dc 100644 --- a/src/plugins/lacp/node.c +++ b/src/plugins/lacp/node.c @@ -102,7 +102,7 @@ lacp_node_fn (vlib_main_t * vm, ? b0->current_length : sizeof (t0->pkt); t0->len = len; t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - clib_memcpy (&t0->pkt, vlib_buffer_get_current (b0), len); + clib_memcpy_fast (&t0->pkt, vlib_buffer_get_current (b0), len); } /* push this pkt to the next graph node, always error-drop */ vlib_set_next_frame_buffer (vm, node, next0, bi0); diff --git a/src/plugins/mactime/node.c b/src/plugins/mactime/node.c index 1d45a39e1ce..4d45dd54ef3 100644 --- a/src/plugins/mactime/node.c +++ b/src/plugins/mactime/node.c @@ -142,9 +142,9 @@ mactime_node_inline (vlib_main_t * vm, en0 = vlib_buffer_get_current (b0); kv.key = 0; if (is_tx) - clib_memcpy (&kv.key, en0->dst_address, 6); + clib_memcpy_fast (&kv.key, en0->dst_address, 6); else - clib_memcpy (&kv.key, en0->src_address, 6); + clib_memcpy_fast (&kv.key, en0->src_address, 6); /* Lookup the src/dst mac address */ if (clib_bihash_search_8_8 (lut, &kv, &kv) < 0) @@ -242,15 +242,16 @@ mactime_node_inline (vlib_main_t * vm, && (b0->flags & VLIB_BUFFER_IS_TRACED))) { mactime_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); - clib_memcpy (t->src_mac, en0->src_address, sizeof (t->src_mac)); + clib_memcpy_fast (t->src_mac, en0->src_address, + sizeof (t->src_mac)); t->next_index = next0; t->device_index = device_index0; if (dp) { - clib_memcpy (t->device_name, dp->device_name, - ARRAY_LEN (t->device_name)); + clib_memcpy_fast (t->device_name, dp->device_name, + ARRAY_LEN (t->device_name)); t->device_name[ARRAY_LEN (t->device_name) - 1] = 0; } } diff --git a/src/plugins/map/ip4_map.c b/src/plugins/map/ip4_map.c index 44392e820ba..fc47d0fee46 100644 --- a/src/plugins/map/ip4_map.c +++ b/src/plugins/map/ip4_map.c @@ -666,17 +666,16 @@ ip4_map_reass (vlib_main_t * vm, u32 len = vec_len (fragments_to_loopback); if (len <= VLIB_FRAME_SIZE) { - clib_memcpy (from, fragments_to_loopback, - sizeof (u32) * len); + clib_memcpy_fast (from, fragments_to_loopback, + sizeof (u32) * len); n_left_from = len; vec_reset_length (fragments_to_loopback); } else { - clib_memcpy (from, - fragments_to_loopback + (len - - VLIB_FRAME_SIZE), - sizeof (u32) * VLIB_FRAME_SIZE); + clib_memcpy_fast (from, fragments_to_loopback + + (len - VLIB_FRAME_SIZE), + sizeof (u32) * VLIB_FRAME_SIZE); n_left_from = VLIB_FRAME_SIZE; _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE; } diff --git a/src/plugins/map/ip6_map.c b/src/plugins/map/ip6_map.c index 5b80209ae0a..35b58110876 100644 --- a/src/plugins/map/ip6_map.c +++ b/src/plugins/map/ip6_map.c @@ -583,7 +583,7 @@ ip6_map_ip6_reass_prepare (vlib_main_t * vm, vlib_node_runtime_t * node, if (ip6_frag_hdr_offset (frag0)) { //Not first fragment, add the IPv4 header - clib_memcpy (ip40, &r->ip4_header, 20); + clib_memcpy_fast (ip40, &r->ip4_header, 20); } #ifdef MAP_IP6_REASS_COUNT_BYTES @@ -594,8 +594,8 @@ ip6_map_ip6_reass_prepare (vlib_main_t * vm, vlib_node_runtime_t * node, if (ip6_frag_hdr_more (frag0)) { //Not last fragment, we copy end of next - clib_memcpy (u8_ptr_add (ip60, p0->current_length), - r->fragments[i].next_data, 20); + clib_memcpy_fast (u8_ptr_add (ip60, p0->current_length), + r->fragments[i].next_data, 20); p0->current_length += 20; ip60->payload_length = u16_net_add (ip60->payload_length, 20); } @@ -976,17 +976,16 @@ ip6_map_ip4_reass (vlib_main_t * vm, u32 len = vec_len (fragments_to_loopback); if (len <= VLIB_FRAME_SIZE) { - clib_memcpy (from, fragments_to_loopback, - sizeof (u32) * len); + clib_memcpy_fast (from, fragments_to_loopback, + sizeof (u32) * len); n_left_from = len; vec_reset_length (fragments_to_loopback); } else { - clib_memcpy (from, - fragments_to_loopback + (len - - VLIB_FRAME_SIZE), - sizeof (u32) * VLIB_FRAME_SIZE); + clib_memcpy_fast (from, fragments_to_loopback + + (len - VLIB_FRAME_SIZE), + sizeof (u32) * VLIB_FRAME_SIZE); n_left_from = VLIB_FRAME_SIZE; _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE; } diff --git a/src/plugins/map/map.c b/src/plugins/map/map.c index 13caa368322..49796e47a8e 100644 --- a/src/plugins/map/map.c +++ b/src/plugins/map/map.c @@ -1807,7 +1807,7 @@ map_ip6_reass_add_fragment (map_ip6_reass_t * r, u32 pi, if (!prev_f) return -1; - clib_memcpy (prev_f->next_data, data_start, copied_len); + clib_memcpy_fast (prev_f->next_data, data_start, copied_len); prev_f->next_data_len = copied_len; prev_f->next_data_offset = data_offset; } @@ -1817,7 +1817,7 @@ map_ip6_reass_add_fragment (map_ip6_reass_t * r, u32 pi, return -1; if (r->ip4_header.ip_version_and_header_length == 0) - clib_memcpy (&r->ip4_header, data_start, sizeof (ip4_header_t)); + clib_memcpy_fast (&r->ip4_header, data_start, sizeof (ip4_header_t)); } if (data_len > 20) diff --git a/src/plugins/map/map.h b/src/plugins/map/map.h index 1e07b59f762..c4cc855d2e2 100644 --- a/src/plugins/map/map.h +++ b/src/plugins/map/map.h @@ -538,7 +538,7 @@ ip4_map_t_embedded_address (map_domain_t *d, u8 offset = d->ip6_src_len == 64 ? 9 : 12; ip6->as_u64[0] = d->ip6_src.as_u64[0]; ip6->as_u64[1] = d->ip6_src.as_u64[1]; - clib_memcpy(&ip6->as_u8[offset], ip4, 4); + clib_memcpy_fast(&ip6->as_u8[offset], ip4, 4); } static_always_inline u32 diff --git a/src/plugins/marvell/pp2/input.c b/src/plugins/marvell/pp2/input.c index c1a0c624ea9..50f09633272 100644 --- a/src/plugins/marvell/pp2/input.c +++ b/src/plugins/marvell/pp2/input.c @@ -65,7 +65,7 @@ mrvl_pp2_input_trace (vlib_main_t * vm, vlib_node_runtime_t * node, u32 next0, tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->next_index = next0; tr->hw_if_index = ppif->hw_if_index; - clib_memcpy (&tr->desc, d, sizeof (struct pp2_ppio_desc)); + clib_memcpy_fast (&tr->desc, d, sizeof (struct pp2_ppio_desc)); } static_always_inline u16 @@ -225,10 +225,10 @@ mrvl_pp2_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, next0 = next1 = ppif->per_interface_next_index; } - clib_memcpy (vnet_buffer (b0)->sw_if_index, sw_if_index, - sizeof (sw_if_index)); - clib_memcpy (vnet_buffer (b1)->sw_if_index, sw_if_index, - sizeof (sw_if_index)); + clib_memcpy_fast (vnet_buffer (b0)->sw_if_index, sw_if_index, + sizeof (sw_if_index)); + clib_memcpy_fast (vnet_buffer (b1)->sw_if_index, sw_if_index, + sizeof (sw_if_index)); VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1); @@ -271,8 +271,8 @@ mrvl_pp2_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, next0 = ppif->per_interface_next_index; } - clib_memcpy (vnet_buffer (b0)->sw_if_index, sw_if_index, - sizeof (sw_if_index)); + clib_memcpy_fast (vnet_buffer (b0)->sw_if_index, sw_if_index, + sizeof (sw_if_index)); VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); diff --git a/src/plugins/marvell/pp2/output.c b/src/plugins/marvell/pp2/output.c index 18969a458e2..dbd106a5930 100644 --- a/src/plugins/marvell/pp2/output.c +++ b/src/plugins/marvell/pp2/output.c @@ -101,10 +101,10 @@ mrvl_pp2_interface_tx (vlib_main_t * vm, buffers = vlib_frame_vector_args (frame); u16 n_copy = clib_min (outq->size - slot, n_sent); - clib_memcpy (outq->buffers + slot, buffers, n_copy * sizeof (u32)); + clib_memcpy_fast (outq->buffers + slot, buffers, n_copy * sizeof (u32)); if (PREDICT_FALSE (n_copy < n_sent)) - clib_memcpy (outq->buffers, buffers + n_copy, - (n_sent - n_copy) * sizeof (u32)); + clib_memcpy_fast (outq->buffers, buffers + n_copy, + (n_sent - n_copy) * sizeof (u32)); outq->head += n_sent; } diff --git a/src/plugins/memif/device.c b/src/plugins/memif/device.c index 9e4cd69b75f..28bf8259311 100644 --- a/src/plugins/memif/device.c +++ b/src/plugins/memif/device.c @@ -239,14 +239,14 @@ no_free_slots: b2 = vlib_get_buffer (vm, ptd->buffers[co[2].buffer_vec_index]); b3 = vlib_get_buffer (vm, ptd->buffers[co[3].buffer_vec_index]); - clib_memcpy (co[0].data, b0->data + co[0].buffer_offset, - co[0].data_len); - clib_memcpy (co[1].data, b1->data + co[1].buffer_offset, - co[1].data_len); - clib_memcpy (co[2].data, b2->data + co[2].buffer_offset, - co[2].data_len); - clib_memcpy (co[3].data, b3->data + co[3].buffer_offset, - co[3].data_len); + clib_memcpy_fast (co[0].data, b0->data + co[0].buffer_offset, + co[0].data_len); + clib_memcpy_fast (co[1].data, b1->data + co[1].buffer_offset, + co[1].data_len); + clib_memcpy_fast (co[2].data, b2->data + co[2].buffer_offset, + co[2].data_len); + clib_memcpy_fast (co[3].data, b3->data + co[3].buffer_offset, + co[3].data_len); co += 4; n_copy_op -= 4; @@ -254,8 +254,8 @@ no_free_slots: while (n_copy_op) { b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]); - clib_memcpy (co[0].data, b0->data + co[0].buffer_offset, - co[0].data_len); + clib_memcpy_fast (co[0].data, b0->data + co[0].buffer_offset, + co[0].data_len); co += 1; n_copy_op -= 1; } diff --git a/src/plugins/memif/node.c b/src/plugins/memif/node.c index 997ac8fcbad..8bd9fcddcb6 100644 --- a/src/plugins/memif/node.c +++ b/src/plugins/memif/node.c @@ -303,14 +303,14 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, b2 = vlib_get_buffer (vm, ptd->buffers[co[2].buffer_vec_index]); b3 = vlib_get_buffer (vm, ptd->buffers[co[3].buffer_vec_index]); - clib_memcpy (b0->data + co[0].buffer_offset, co[0].data, - co[0].data_len); - clib_memcpy (b1->data + co[1].buffer_offset, co[1].data, - co[1].data_len); - clib_memcpy (b2->data + co[2].buffer_offset, co[2].data, - co[2].data_len); - clib_memcpy (b3->data + co[3].buffer_offset, co[3].data, - co[3].data_len); + clib_memcpy_fast (b0->data + co[0].buffer_offset, co[0].data, + co[0].data_len); + clib_memcpy_fast (b1->data + co[1].buffer_offset, co[1].data, + co[1].data_len); + clib_memcpy_fast (b2->data + co[2].buffer_offset, co[2].data, + co[2].data_len); + clib_memcpy_fast (b3->data + co[3].buffer_offset, co[3].data, + co[3].data_len); co += 4; n_left -= 4; @@ -318,8 +318,8 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, while (n_left) { b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]); - clib_memcpy (b0->data + co[0].buffer_offset, co[0].data, - co[0].data_len); + clib_memcpy_fast (b0->data + co[0].buffer_offset, co[0].data, + co[0].data_len); co += 1; n_left -= 1; } @@ -424,7 +424,7 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, fbvi[0] = po[0].first_buffer_vec_index; bi[0] = ptd->buffers[fbvi[0]]; b0 = vlib_get_buffer (vm, bi[0]); - clib_memcpy (b0, bt, 64); + clib_memcpy_fast (b0, bt, 64); b0->current_length = po->packet_len; n_rx_bytes += b0->current_length; @@ -783,10 +783,10 @@ refill: d2 = &ring->desc[s2]; d3 = &ring->desc[s3]; - clib_memcpy (d0, dt, sizeof (memif_desc_t)); - clib_memcpy (d1, dt, sizeof (memif_desc_t)); - clib_memcpy (d2, dt, sizeof (memif_desc_t)); - clib_memcpy (d3, dt, sizeof (memif_desc_t)); + clib_memcpy_fast (d0, dt, sizeof (memif_desc_t)); + clib_memcpy_fast (d1, dt, sizeof (memif_desc_t)); + clib_memcpy_fast (d2, dt, sizeof (memif_desc_t)); + clib_memcpy_fast (d3, dt, sizeof (memif_desc_t)); b0 = vlib_get_buffer (vm, mq->buffers[s0]); b1 = vlib_get_buffer (vm, mq->buffers[s1]); @@ -813,7 +813,7 @@ refill: { s0 = head++ & mask; d0 = &ring->desc[s0]; - clib_memcpy (d0, dt, sizeof (memif_desc_t)); + clib_memcpy_fast (d0, dt, sizeof (memif_desc_t)); b0 = vlib_get_buffer (vm, mq->buffers[s0]); d0->region = b0->buffer_pool_index + 1; d0->offset = diff --git a/src/plugins/nat/in2out.c b/src/plugins/nat/in2out.c index cb169814eac..d4432d98016 100755 --- a/src/plugins/nat/in2out.c +++ b/src/plugins/nat/in2out.c @@ -1891,17 +1891,16 @@ nat44_in2out_reass_node_fn (vlib_main_t * vm, u32 len = vec_len (fragments_to_loopback); if (len <= VLIB_FRAME_SIZE) { - clib_memcpy (from, fragments_to_loopback, - sizeof (u32) * len); + clib_memcpy_fast (from, fragments_to_loopback, + sizeof (u32) * len); n_left_from = len; vec_reset_length (fragments_to_loopback); } else { - clib_memcpy (from, - fragments_to_loopback + (len - - VLIB_FRAME_SIZE), - sizeof (u32) * VLIB_FRAME_SIZE); + clib_memcpy_fast (from, fragments_to_loopback + + (len - VLIB_FRAME_SIZE), + sizeof (u32) * VLIB_FRAME_SIZE); n_left_from = VLIB_FRAME_SIZE; _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE; } diff --git a/src/plugins/nat/in2out_ed.c b/src/plugins/nat/in2out_ed.c index f9f8d776eb4..e32e1386d8d 100644 --- a/src/plugins/nat/in2out_ed.c +++ b/src/plugins/nat/in2out_ed.c @@ -2002,17 +2002,16 @@ nat44_ed_in2out_reass_node_fn_inline (vlib_main_t * vm, u32 len = vec_len (fragments_to_loopback); if (len <= VLIB_FRAME_SIZE) { - clib_memcpy (from, fragments_to_loopback, - sizeof (u32) * len); + clib_memcpy_fast (from, fragments_to_loopback, + sizeof (u32) * len); n_left_from = len; vec_reset_length (fragments_to_loopback); } else { - clib_memcpy (from, - fragments_to_loopback + (len - - VLIB_FRAME_SIZE), - sizeof (u32) * VLIB_FRAME_SIZE); + clib_memcpy_fast (from, fragments_to_loopback + + (len - VLIB_FRAME_SIZE), + sizeof (u32) * VLIB_FRAME_SIZE); n_left_from = VLIB_FRAME_SIZE; _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE; } diff --git a/src/plugins/nat/nat44_classify.c b/src/plugins/nat/nat44_classify.c index 0e9863c02ac..8608fc532e2 100644 --- a/src/plugins/nat/nat44_classify.c +++ b/src/plugins/nat/nat44_classify.c @@ -374,17 +374,16 @@ nat44_classify_node_fn_inline (vlib_main_t * vm, u32 len = vec_len (fragments_to_loopback); if (len <= VLIB_FRAME_SIZE) { - clib_memcpy (from, fragments_to_loopback, - sizeof (u32) * len); + clib_memcpy_fast (from, fragments_to_loopback, + sizeof (u32) * len); n_left_from = len; vec_reset_length (fragments_to_loopback); } else { - clib_memcpy (from, - fragments_to_loopback + (len - - VLIB_FRAME_SIZE), - sizeof (u32) * VLIB_FRAME_SIZE); + clib_memcpy_fast (from, fragments_to_loopback + + (len - VLIB_FRAME_SIZE), + sizeof (u32) * VLIB_FRAME_SIZE); n_left_from = VLIB_FRAME_SIZE; _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE; } diff --git a/src/plugins/nat/nat64.c b/src/plugins/nat/nat64.c index 86c65edfbee..8bcca2b33d3 100644 --- a/src/plugins/nat/nat64.c +++ b/src/plugins/nat/nat64.c @@ -1018,7 +1018,7 @@ nat64_compose_ip6 (ip6_address_t * ip6, ip4_address_t * ip4, u32 fib_index) if (prefix) { - clib_memcpy (ip6, &p->prefix, sizeof (ip6_address_t)); + clib_memcpy_fast (ip6, &p->prefix, sizeof (ip6_address_t)); switch (p->plen) { case 32: @@ -1058,7 +1058,7 @@ nat64_compose_ip6 (ip6_address_t * ip6, ip4_address_t * ip4, u32 fib_index) } else { - clib_memcpy (ip6, well_known_prefix, sizeof (ip6_address_t)); + clib_memcpy_fast (ip6, well_known_prefix, sizeof (ip6_address_t)); ip6->as_u32[3] = ip4->as_u32; } } diff --git a/src/plugins/nat/nat64_in2out.c b/src/plugins/nat/nat64_in2out.c index d19aab87983..0652f4481bc 100644 --- a/src/plugins/nat/nat64_in2out.c +++ b/src/plugins/nat/nat64_in2out.c @@ -1559,17 +1559,16 @@ nat64_in2out_reass_node_fn (vlib_main_t * vm, u32 len = vec_len (fragments_to_loopback); if (len <= VLIB_FRAME_SIZE) { - clib_memcpy (from, fragments_to_loopback, - sizeof (u32) * len); + clib_memcpy_fast (from, fragments_to_loopback, + sizeof (u32) * len); n_left_from = len; vec_reset_length (fragments_to_loopback); } else { - clib_memcpy (from, - fragments_to_loopback + (len - - VLIB_FRAME_SIZE), - sizeof (u32) * VLIB_FRAME_SIZE); + clib_memcpy_fast (from, fragments_to_loopback + + (len - VLIB_FRAME_SIZE), + sizeof (u32) * VLIB_FRAME_SIZE); n_left_from = VLIB_FRAME_SIZE; _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE; } diff --git a/src/plugins/nat/nat64_out2in.c b/src/plugins/nat/nat64_out2in.c index 79ff161b326..1c8df8a16db 100644 --- a/src/plugins/nat/nat64_out2in.c +++ b/src/plugins/nat/nat64_out2in.c @@ -842,17 +842,16 @@ nat64_out2in_reass_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, u32 len = vec_len (fragments_to_loopback); if (len <= VLIB_FRAME_SIZE) { - clib_memcpy (from, fragments_to_loopback, - sizeof (u32) * len); + clib_memcpy_fast (from, fragments_to_loopback, + sizeof (u32) * len); n_left_from = len; vec_reset_length (fragments_to_loopback); } else { - clib_memcpy (from, - fragments_to_loopback + (len - - VLIB_FRAME_SIZE), - sizeof (u32) * VLIB_FRAME_SIZE); + clib_memcpy_fast (from, fragments_to_loopback + + (len - VLIB_FRAME_SIZE), + sizeof (u32) * VLIB_FRAME_SIZE); n_left_from = VLIB_FRAME_SIZE; _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE; } diff --git a/src/plugins/nat/nat_inlines.h b/src/plugins/nat/nat_inlines.h index 4bdb2cb66d9..c0f16721173 100644 --- a/src/plugins/nat/nat_inlines.h +++ b/src/plugins/nat/nat_inlines.h @@ -371,7 +371,7 @@ mss_clamping (snat_main_t * sm, tcp_header_t * tcp, ip_csum_t * sum) *sum = ip_csum_update (*sum, mss, sm->mss_value_net, ip4_header_t, length); - clib_memcpy (data + 2, &sm->mss_value_net, 2); + clib_memcpy_fast (data + 2, &sm->mss_value_net, 2); } return; } diff --git a/src/plugins/nat/nat_ipfix_logging.c b/src/plugins/nat/nat_ipfix_logging.c index 8a3668e2759..042239fee85 100644 --- a/src/plugins/nat/nat_ipfix_logging.c +++ b/src/plugins/nat/nat_ipfix_logging.c @@ -674,28 +674,28 @@ snat_ipfix_logging_nat44_ses (u8 nat_event, u32 src_ip, u32 nat_src_ip, if (PREDICT_TRUE (do_flush == 0)) { u64 time_stamp = clib_host_to_net_u64 (now); - clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp)); + clib_memcpy_fast (b0->data + offset, &time_stamp, sizeof (time_stamp)); offset += sizeof (time_stamp); - clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event)); + clib_memcpy_fast (b0->data + offset, &nat_event, sizeof (nat_event)); offset += sizeof (nat_event); - clib_memcpy (b0->data + offset, &src_ip, sizeof (src_ip)); + clib_memcpy_fast (b0->data + offset, &src_ip, sizeof (src_ip)); offset += sizeof (src_ip); - clib_memcpy (b0->data + offset, &nat_src_ip, sizeof (nat_src_ip)); + clib_memcpy_fast (b0->data + offset, &nat_src_ip, sizeof (nat_src_ip)); offset += sizeof (nat_src_ip); - clib_memcpy (b0->data + offset, &proto, sizeof (proto)); + clib_memcpy_fast (b0->data + offset, &proto, sizeof (proto)); offset += sizeof (proto); - clib_memcpy (b0->data + offset, &src_port, sizeof (src_port)); + clib_memcpy_fast (b0->data + offset, &src_port, sizeof (src_port)); offset += sizeof (src_port); - clib_memcpy (b0->data + offset, &nat_src_port, sizeof (nat_src_port)); + clib_memcpy_fast (b0->data + offset, &nat_src_port, sizeof (nat_src_port)); offset += sizeof (nat_src_port); - clib_memcpy (b0->data + offset, &vrf_id, sizeof (vrf_id)); + clib_memcpy_fast (b0->data + offset, &vrf_id, sizeof (vrf_id)); offset += sizeof (vrf_id); b0->current_length += NAT44_SESSION_CREATE_LEN; @@ -775,13 +775,13 @@ snat_ipfix_logging_addr_exhausted (u32 pool_id, int do_flush) if (PREDICT_TRUE (do_flush == 0)) { u64 time_stamp = clib_host_to_net_u64 (now); - clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp)); + clib_memcpy_fast (b0->data + offset, &time_stamp, sizeof (time_stamp)); offset += sizeof (time_stamp); - clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event)); + clib_memcpy_fast (b0->data + offset, &nat_event, sizeof (nat_event)); offset += sizeof (nat_event); - clib_memcpy (b0->data + offset, &pool_id, sizeof (pool_id)); + clib_memcpy_fast (b0->data + offset, &pool_id, sizeof (pool_id)); offset += sizeof (pool_id); b0->current_length += NAT_ADDRESSES_EXHAUTED_LEN; @@ -862,19 +862,19 @@ snat_ipfix_logging_max_entries_per_usr (u32 limit, u32 src_ip, int do_flush) if (PREDICT_TRUE (do_flush == 0)) { u64 time_stamp = clib_host_to_net_u64 (now); - clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp)); + clib_memcpy_fast (b0->data + offset, &time_stamp, sizeof (time_stamp)); offset += sizeof (time_stamp); - clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event)); + clib_memcpy_fast (b0->data + offset, &nat_event, sizeof (nat_event)); offset += sizeof (nat_event); - clib_memcpy (b0->data + offset, "a_event, sizeof (quota_event)); + clib_memcpy_fast (b0->data + offset, "a_event, sizeof (quota_event)); offset += sizeof (quota_event); - clib_memcpy (b0->data + offset, &limit, sizeof (limit)); + clib_memcpy_fast (b0->data + offset, &limit, sizeof (limit)); offset += sizeof (limit); - clib_memcpy (b0->data + offset, &src_ip, sizeof (src_ip)); + clib_memcpy_fast (b0->data + offset, &src_ip, sizeof (src_ip)); offset += sizeof (src_ip); b0->current_length += MAX_ENTRIES_PER_USER_LEN; @@ -955,16 +955,16 @@ nat_ipfix_logging_max_ses (u32 limit, int do_flush) if (PREDICT_TRUE (do_flush == 0)) { u64 time_stamp = clib_host_to_net_u64 (now); - clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp)); + clib_memcpy_fast (b0->data + offset, &time_stamp, sizeof (time_stamp)); offset += sizeof (time_stamp); - clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event)); + clib_memcpy_fast (b0->data + offset, &nat_event, sizeof (nat_event)); offset += sizeof (nat_event); - clib_memcpy (b0->data + offset, "a_event, sizeof (quota_event)); + clib_memcpy_fast (b0->data + offset, "a_event, sizeof (quota_event)); offset += sizeof (quota_event); - clib_memcpy (b0->data + offset, &limit, sizeof (limit)); + clib_memcpy_fast (b0->data + offset, &limit, sizeof (limit)); offset += sizeof (limit); b0->current_length += MAX_SESSIONS_LEN; @@ -1045,16 +1045,16 @@ nat_ipfix_logging_max_bib (u32 limit, int do_flush) if (PREDICT_TRUE (do_flush == 0)) { u64 time_stamp = clib_host_to_net_u64 (now); - clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp)); + clib_memcpy_fast (b0->data + offset, &time_stamp, sizeof (time_stamp)); offset += sizeof (time_stamp); - clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event)); + clib_memcpy_fast (b0->data + offset, &nat_event, sizeof (nat_event)); offset += sizeof (nat_event); - clib_memcpy (b0->data + offset, "a_event, sizeof (quota_event)); + clib_memcpy_fast (b0->data + offset, "a_event, sizeof (quota_event)); offset += sizeof (quota_event); - clib_memcpy (b0->data + offset, &limit, sizeof (limit)); + clib_memcpy_fast (b0->data + offset, &limit, sizeof (limit)); offset += sizeof (limit); b0->current_length += MAX_BIBS_LEN; @@ -1135,19 +1135,19 @@ nat_ipfix_logging_max_frag_ip4 (u32 limit, u32 src, int do_flush) if (PREDICT_TRUE (do_flush == 0)) { u64 time_stamp = clib_host_to_net_u64 (now); - clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp)); + clib_memcpy_fast (b0->data + offset, &time_stamp, sizeof (time_stamp)); offset += sizeof (time_stamp); - clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event)); + clib_memcpy_fast (b0->data + offset, &nat_event, sizeof (nat_event)); offset += sizeof (nat_event); - clib_memcpy (b0->data + offset, "a_event, sizeof (quota_event)); + clib_memcpy_fast (b0->data + offset, "a_event, sizeof (quota_event)); offset += sizeof (quota_event); - clib_memcpy (b0->data + offset, &limit, sizeof (limit)); + clib_memcpy_fast (b0->data + offset, &limit, sizeof (limit)); offset += sizeof (limit); - clib_memcpy (b0->data + offset, &src, sizeof (src)); + clib_memcpy_fast (b0->data + offset, &src, sizeof (src)); offset += sizeof (src); b0->current_length += MAX_FRAGMENTS_IP4_LEN; @@ -1228,19 +1228,19 @@ nat_ipfix_logging_max_frag_ip6 (u32 limit, ip6_address_t * src, int do_flush) if (PREDICT_TRUE (do_flush == 0)) { u64 time_stamp = clib_host_to_net_u64 (now); - clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp)); + clib_memcpy_fast (b0->data + offset, &time_stamp, sizeof (time_stamp)); offset += sizeof (time_stamp); - clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event)); + clib_memcpy_fast (b0->data + offset, &nat_event, sizeof (nat_event)); offset += sizeof (nat_event); - clib_memcpy (b0->data + offset, "a_event, sizeof (quota_event)); + clib_memcpy_fast (b0->data + offset, "a_event, sizeof (quota_event)); offset += sizeof (quota_event); - clib_memcpy (b0->data + offset, &limit, sizeof (limit)); + clib_memcpy_fast (b0->data + offset, &limit, sizeof (limit)); offset += sizeof (limit); - clib_memcpy (b0->data + offset, src, sizeof (ip6_address_t)); + clib_memcpy_fast (b0->data + offset, src, sizeof (ip6_address_t)); offset += sizeof (ip6_address_t); b0->current_length += MAX_FRAGMENTS_IP6_LEN; @@ -1321,28 +1321,28 @@ nat_ipfix_logging_nat64_bibe (u8 nat_event, ip6_address_t * src_ip, if (PREDICT_TRUE (do_flush == 0)) { u64 time_stamp = clib_host_to_net_u64 (now); - clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp)); + clib_memcpy_fast (b0->data + offset, &time_stamp, sizeof (time_stamp)); offset += sizeof (time_stamp); - clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event)); + clib_memcpy_fast (b0->data + offset, &nat_event, sizeof (nat_event)); offset += sizeof (nat_event); - clib_memcpy (b0->data + offset, src_ip, sizeof (ip6_address_t)); + clib_memcpy_fast (b0->data + offset, src_ip, sizeof (ip6_address_t)); offset += sizeof (ip6_address_t); - clib_memcpy (b0->data + offset, &nat_src_ip, sizeof (nat_src_ip)); + clib_memcpy_fast (b0->data + offset, &nat_src_ip, sizeof (nat_src_ip)); offset += sizeof (nat_src_ip); - clib_memcpy (b0->data + offset, &proto, sizeof (proto)); + clib_memcpy_fast (b0->data + offset, &proto, sizeof (proto)); offset += sizeof (proto); - clib_memcpy (b0->data + offset, &src_port, sizeof (src_port)); + clib_memcpy_fast (b0->data + offset, &src_port, sizeof (src_port)); offset += sizeof (src_port); - clib_memcpy (b0->data + offset, &nat_src_port, sizeof (nat_src_port)); + clib_memcpy_fast (b0->data + offset, &nat_src_port, sizeof (nat_src_port)); offset += sizeof (nat_src_port); - clib_memcpy (b0->data + offset, &vrf_id, sizeof (vrf_id)); + clib_memcpy_fast (b0->data + offset, &vrf_id, sizeof (vrf_id)); offset += sizeof (vrf_id); b0->current_length += NAT64_BIB_LEN; @@ -1425,40 +1425,40 @@ nat_ipfix_logging_nat64_ses (u8 nat_event, ip6_address_t * src_ip, if (PREDICT_TRUE (do_flush == 0)) { u64 time_stamp = clib_host_to_net_u64 (now); - clib_memcpy (b0->data + offset, &time_stamp, sizeof (time_stamp)); + clib_memcpy_fast (b0->data + offset, &time_stamp, sizeof (time_stamp)); offset += sizeof (time_stamp); - clib_memcpy (b0->data + offset, &nat_event, sizeof (nat_event)); + clib_memcpy_fast (b0->data + offset, &nat_event, sizeof (nat_event)); offset += sizeof (nat_event); - clib_memcpy (b0->data + offset, src_ip, sizeof (ip6_address_t)); + clib_memcpy_fast (b0->data + offset, src_ip, sizeof (ip6_address_t)); offset += sizeof (ip6_address_t); - clib_memcpy (b0->data + offset, &nat_src_ip, sizeof (nat_src_ip)); + clib_memcpy_fast (b0->data + offset, &nat_src_ip, sizeof (nat_src_ip)); offset += sizeof (nat_src_ip); - clib_memcpy (b0->data + offset, &proto, sizeof (proto)); + clib_memcpy_fast (b0->data + offset, &proto, sizeof (proto)); offset += sizeof (proto); - clib_memcpy (b0->data + offset, &src_port, sizeof (src_port)); + clib_memcpy_fast (b0->data + offset, &src_port, sizeof (src_port)); offset += sizeof (src_port); - clib_memcpy (b0->data + offset, &nat_src_port, sizeof (nat_src_port)); + clib_memcpy_fast (b0->data + offset, &nat_src_port, sizeof (nat_src_port)); offset += sizeof (nat_src_port); - clib_memcpy (b0->data + offset, dst_ip, sizeof (ip6_address_t)); + clib_memcpy_fast (b0->data + offset, dst_ip, sizeof (ip6_address_t)); offset += sizeof (ip6_address_t); - clib_memcpy (b0->data + offset, &nat_dst_ip, sizeof (nat_dst_ip)); + clib_memcpy_fast (b0->data + offset, &nat_dst_ip, sizeof (nat_dst_ip)); offset += sizeof (nat_dst_ip); - clib_memcpy (b0->data + offset, &dst_port, sizeof (dst_port)); + clib_memcpy_fast (b0->data + offset, &dst_port, sizeof (dst_port)); offset += sizeof (dst_port); - clib_memcpy (b0->data + offset, &nat_dst_port, sizeof (nat_dst_port)); + clib_memcpy_fast (b0->data + offset, &nat_dst_port, sizeof (nat_dst_port)); offset += sizeof (nat_dst_port); - clib_memcpy (b0->data + offset, &vrf_id, sizeof (vrf_id)); + clib_memcpy_fast (b0->data + offset, &vrf_id, sizeof (vrf_id)); offset += sizeof (vrf_id); b0->current_length += NAT64_SES_LEN; diff --git a/src/plugins/nat/out2in.c b/src/plugins/nat/out2in.c index c4d1fbf6900..9cace08c565 100755 --- a/src/plugins/nat/out2in.c +++ b/src/plugins/nat/out2in.c @@ -1568,17 +1568,16 @@ nat44_out2in_reass_node_fn (vlib_main_t * vm, u32 len = vec_len (fragments_to_loopback); if (len <= VLIB_FRAME_SIZE) { - clib_memcpy (from, fragments_to_loopback, - sizeof (u32) * len); + clib_memcpy_fast (from, fragments_to_loopback, + sizeof (u32) * len); n_left_from = len; vec_reset_length (fragments_to_loopback); } else { - clib_memcpy (from, - fragments_to_loopback + (len - - VLIB_FRAME_SIZE), - sizeof (u32) * VLIB_FRAME_SIZE); + clib_memcpy_fast (from, fragments_to_loopback + + (len - VLIB_FRAME_SIZE), + sizeof (u32) * VLIB_FRAME_SIZE); n_left_from = VLIB_FRAME_SIZE; _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE; } diff --git a/src/plugins/nat/out2in_ed.c b/src/plugins/nat/out2in_ed.c index b2dbc513df6..55b34f5c2e8 100644 --- a/src/plugins/nat/out2in_ed.c +++ b/src/plugins/nat/out2in_ed.c @@ -1844,17 +1844,16 @@ nat44_ed_out2in_reass_node_fn (vlib_main_t * vm, u32 len = vec_len (fragments_to_loopback); if (len <= VLIB_FRAME_SIZE) { - clib_memcpy (from, fragments_to_loopback, - sizeof (u32) * len); + clib_memcpy_fast (from, fragments_to_loopback, + sizeof (u32) * len); n_left_from = len; vec_reset_length (fragments_to_loopback); } else { - clib_memcpy (from, - fragments_to_loopback + (len - - VLIB_FRAME_SIZE), - sizeof (u32) * VLIB_FRAME_SIZE); + clib_memcpy_fast (from, fragments_to_loopback + + (len - VLIB_FRAME_SIZE), + sizeof (u32) * VLIB_FRAME_SIZE); n_left_from = VLIB_FRAME_SIZE; _vec_len (fragments_to_loopback) = len - VLIB_FRAME_SIZE; } diff --git a/src/plugins/nsh/nsh-md2-ioam/nsh_md2_ioam.c b/src/plugins/nsh/nsh-md2-ioam/nsh_md2_ioam.c index 73b57f64e90..57e5e6cfa39 100644 --- a/src/plugins/nsh/nsh-md2-ioam/nsh_md2_ioam.c +++ b/src/plugins/nsh/nsh-md2-ioam/nsh_md2_ioam.c @@ -242,7 +242,7 @@ u32 fib_path_get_resolving_interface (fib_node_index_t path_index); t1->dst_addr.ip4.as_u32 = fib_prefix.fp_addr.ip4.as_u32; key4_copy = clib_mem_alloc (sizeof (*key4_copy)); clib_memset(key4_copy, 0, sizeof(*key4_copy)); - clib_memcpy (key4_copy, &key4, sizeof (*key4_copy)); + clib_memcpy_fast (key4_copy, &key4, sizeof (*key4_copy)); hash_set_mem (hm->dst_by_ip4, key4_copy, t1 - hm->dst_tunnels); /* * Attach to the FIB entry for the VxLAN-GPE destination diff --git a/src/plugins/nsh/nsh-md2-ioam/nsh_md2_ioam_trace.c b/src/plugins/nsh/nsh-md2-ioam/nsh_md2_ioam_trace.c index 495bb16aab2..bd56a6ab7b1 100644 --- a/src/plugins/nsh/nsh-md2-ioam/nsh_md2_ioam_trace.c +++ b/src/plugins/nsh/nsh-md2-ioam/nsh_md2_ioam_trace.c @@ -308,7 +308,8 @@ nsh_md2_ioam_trace_swap_handler (vlib_buffer_t * b, nsh_tlv_header_t * new_opt) { - clib_memcpy (new_opt, old_opt, new_opt->length + sizeof (nsh_tlv_header_t)); + clib_memcpy_fast (new_opt, old_opt, + new_opt->length + sizeof (nsh_tlv_header_t)); return nsh_md2_ioam_trace_data_list_handler (b, new_opt); } diff --git a/src/plugins/nsh/nsh-md2-ioam/nsh_md2_ioam_util.h b/src/plugins/nsh/nsh-md2-ioam/nsh_md2_ioam_util.h index 8fbb3b7aaf7..77cc8792c93 100644 --- a/src/plugins/nsh/nsh-md2-ioam/nsh_md2_ioam_util.h +++ b/src/plugins/nsh/nsh-md2-ioam/nsh_md2_ioam_util.h @@ -105,7 +105,7 @@ nsh_md2_ioam_encap_decap_ioam_v4_one_inline (vlib_main_t * vm, if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { nsh_transit_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (&(tr->trace_data), nsh_hdr, (nsh_hdr->length * 4)); + clib_memcpy_fast (&(tr->trace_data), nsh_hdr, (nsh_hdr->length * 4)); } return; } diff --git a/src/plugins/nsh/nsh.c b/src/plugins/nsh/nsh.c index d9b625cda04..b293b85db8a 100644 --- a/src/plugins/nsh/nsh.c +++ b/src/plugins/nsh/nsh.c @@ -1771,13 +1771,13 @@ nsh_input_map (vlib_main_t * vm, else if (node_type == NSH_AWARE_VNF_PROXY_TYPE) { /* Push dummy Eth header */ - clib_memcpy (dummy_eth0.dst_address, dummy_dst_address, 6); - clib_memcpy (dummy_eth0.src_address, dummy_src_address, 6); + clib_memcpy_fast (dummy_eth0.dst_address, dummy_dst_address, 6); + clib_memcpy_fast (dummy_eth0.src_address, dummy_src_address, 6); dummy_eth0.type = 0x0800; vlib_buffer_advance (b0, -(word) sizeof (ethernet_header_t)); hdr0 = vlib_buffer_get_current (b0); - clib_memcpy (hdr0, &dummy_eth0, - (word) sizeof (ethernet_header_t)); + clib_memcpy_fast (hdr0, &dummy_eth0, + (word) sizeof (ethernet_header_t)); sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX]; nsp_nsi0 = nm->tunnel_index_by_sw_if_index[sw_if_index0]; @@ -1880,7 +1880,7 @@ nsh_input_map (vlib_main_t * vm, /* Push new NSH header */ vlib_buffer_advance (b0, -(word) encap_hdr_len0); hdr0 = vlib_buffer_get_current (b0); - clib_memcpy (hdr0, encap_hdr0, (word) encap_hdr_len0); + clib_memcpy_fast (hdr0, encap_hdr0, (word) encap_hdr_len0); goto trace0; } @@ -1892,7 +1892,7 @@ nsh_input_map (vlib_main_t * vm, /* Push new NSH header */ vlib_buffer_advance (b0, -(word) encap_hdr_len0); hdr0 = vlib_buffer_get_current (b0); - clib_memcpy (hdr0, encap_hdr0, (word) encap_hdr_len0); + clib_memcpy_fast (hdr0, encap_hdr0, (word) encap_hdr_len0); /* Manipulate MD2 */ if (PREDICT_FALSE (nsh_entry0->nsh_base.md_type == 2)) @@ -1909,8 +1909,8 @@ nsh_input_map (vlib_main_t * vm, { nsh_input_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (&(tr->trace_data), hdr0, - ((hdr0->length & NSH_LEN_MASK) * 4)); + clib_memcpy_fast (&(tr->trace_data), hdr0, + ((hdr0->length & NSH_LEN_MASK) * 4)); } /* Process packet 1 */ @@ -1936,13 +1936,13 @@ nsh_input_map (vlib_main_t * vm, else if (node_type == NSH_AWARE_VNF_PROXY_TYPE) { /* Push dummy Eth header */ - clib_memcpy (dummy_eth1.dst_address, dummy_dst_address, 6); - clib_memcpy (dummy_eth1.src_address, dummy_src_address, 6); + clib_memcpy_fast (dummy_eth1.dst_address, dummy_dst_address, 6); + clib_memcpy_fast (dummy_eth1.src_address, dummy_src_address, 6); dummy_eth1.type = 0x0800; vlib_buffer_advance (b1, -(word) sizeof (ethernet_header_t)); hdr1 = vlib_buffer_get_current (b1); - clib_memcpy (hdr1, &dummy_eth1, - (word) sizeof (ethernet_header_t)); + clib_memcpy_fast (hdr1, &dummy_eth1, + (word) sizeof (ethernet_header_t)); sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_TX]; nsp_nsi1 = nm->tunnel_index_by_sw_if_index[sw_if_index1]; @@ -2045,7 +2045,7 @@ nsh_input_map (vlib_main_t * vm, /* Push new NSH header */ vlib_buffer_advance (b1, -(word) encap_hdr_len1); hdr1 = vlib_buffer_get_current (b1); - clib_memcpy (hdr1, encap_hdr1, (word) encap_hdr_len1); + clib_memcpy_fast (hdr1, encap_hdr1, (word) encap_hdr_len1); goto trace1; } @@ -2057,7 +2057,7 @@ nsh_input_map (vlib_main_t * vm, /* Push new NSH header */ vlib_buffer_advance (b1, -(word) encap_hdr_len1); hdr1 = vlib_buffer_get_current (b1); - clib_memcpy (hdr1, encap_hdr1, (word) encap_hdr_len1); + clib_memcpy_fast (hdr1, encap_hdr1, (word) encap_hdr_len1); /* Manipulate MD2 */ if (PREDICT_FALSE (nsh_entry1->nsh_base.md_type == 2)) @@ -2074,8 +2074,8 @@ nsh_input_map (vlib_main_t * vm, { nsh_input_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (&(tr->trace_data), hdr1, - ((hdr1->length & NSH_LEN_MASK) * 4)); + clib_memcpy_fast (&(tr->trace_data), hdr1, + ((hdr1->length & NSH_LEN_MASK) * 4)); } vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, @@ -2138,13 +2138,13 @@ nsh_input_map (vlib_main_t * vm, else if (node_type == NSH_AWARE_VNF_PROXY_TYPE) { /* Push dummy Eth header */ - clib_memcpy (dummy_eth0.dst_address, dummy_dst_address, 6); - clib_memcpy (dummy_eth0.src_address, dummy_src_address, 6); + clib_memcpy_fast (dummy_eth0.dst_address, dummy_dst_address, 6); + clib_memcpy_fast (dummy_eth0.src_address, dummy_src_address, 6); dummy_eth0.type = 0x0800; vlib_buffer_advance (b0, -(word) sizeof (ethernet_header_t)); hdr0 = vlib_buffer_get_current (b0); - clib_memcpy (hdr0, &dummy_eth0, - (word) sizeof (ethernet_header_t)); + clib_memcpy_fast (hdr0, &dummy_eth0, + (word) sizeof (ethernet_header_t)); sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX]; nsp_nsi0 = nm->tunnel_index_by_sw_if_index[sw_if_index0]; @@ -2250,7 +2250,7 @@ nsh_input_map (vlib_main_t * vm, /* Push new NSH header */ vlib_buffer_advance (b0, -(word) encap_hdr_len0); hdr0 = vlib_buffer_get_current (b0); - clib_memcpy (hdr0, encap_hdr0, (word) encap_hdr_len0); + clib_memcpy_fast (hdr0, encap_hdr0, (word) encap_hdr_len0); goto trace00; } @@ -2262,7 +2262,7 @@ nsh_input_map (vlib_main_t * vm, /* Push new NSH header */ vlib_buffer_advance (b0, -(word) encap_hdr_len0); hdr0 = vlib_buffer_get_current (b0); - clib_memcpy (hdr0, encap_hdr0, (word) encap_hdr_len0); + clib_memcpy_fast (hdr0, encap_hdr0, (word) encap_hdr_len0); /* Manipulate MD2 */ if (PREDICT_FALSE (nsh_entry0->nsh_base.md_type == 2)) { @@ -2277,8 +2277,8 @@ nsh_input_map (vlib_main_t * vm, { nsh_input_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (&(tr->trace_data[0]), hdr0, - ((hdr0->length & NSH_LEN_MASK) * 4)); + clib_memcpy_fast (&(tr->trace_data[0]), hdr0, + ((hdr0->length & NSH_LEN_MASK) * 4)); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, diff --git a/src/plugins/nsh/nsh_pop.c b/src/plugins/nsh/nsh_pop.c index cd25bb9e3f7..682a92d178a 100644 --- a/src/plugins/nsh/nsh_pop.c +++ b/src/plugins/nsh/nsh_pop.c @@ -162,7 +162,7 @@ nsh_pop_inline (vlib_main_t * vm, if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) { nsh_input_trace_t *tr = vlib_add_trace(vm, node, b0, sizeof(*tr)); - clib_memcpy ( &(tr->trace_data), hdr0, (hdr0->length*4) ); + clib_memcpy_fast ( &(tr->trace_data), hdr0, (hdr0->length*4) ); } /* Process packet 1 */ @@ -216,7 +216,7 @@ nsh_pop_inline (vlib_main_t * vm, if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED)) { nsh_input_trace_t *tr = vlib_add_trace(vm, node, b1, sizeof(*tr)); - clib_memcpy ( &(tr->trace_data), hdr1, (hdr1->length*4) ); + clib_memcpy_fast ( &(tr->trace_data), hdr1, (hdr1->length*4) ); } vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, @@ -301,7 +301,7 @@ nsh_pop_inline (vlib_main_t * vm, if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) { nsh_input_trace_t *tr = vlib_add_trace(vm, node, b0, sizeof(*tr)); - clib_memcpy ( &(tr->trace_data[0]), hdr0, (hdr0->length*4) ); + clib_memcpy_fast ( &(tr->trace_data[0]), hdr0, (hdr0->length*4) ); } vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, diff --git a/src/plugins/nsim/node.c b/src/plugins/nsim/node.c index dfd0bf747f1..d6413778bf3 100644 --- a/src/plugins/nsim/node.c +++ b/src/plugins/nsim/node.c @@ -121,8 +121,8 @@ nsim_inline (vlib_main_t * vm, ? nsm->sw_if_index1 : nsm->sw_if_index0; ep->current_length = vlib_buffer_length_in_chain (vm, b[0]); ASSERT (ep->current_length <= WHEEL_ENTRY_DATA_SIZE); - clib_memcpy (ep->data, vlib_buffer_get_current (b[0]), - ep->current_length); + clib_memcpy_fast (ep->data, vlib_buffer_get_current (b[0]), + ep->current_length); } else /* out of wheel space, drop pkt */ { diff --git a/src/plugins/nsim/nsim_input.c b/src/plugins/nsim/nsim_input.c index 38c3891fd95..66bd8d10954 100644 --- a/src/plugins/nsim/nsim_input.c +++ b/src/plugins/nsim/nsim_input.c @@ -154,7 +154,7 @@ nsim_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, b0->current_length = ep->current_length; /* Copy data from the ring */ - clib_memcpy (b0->data, ep->data, ep->current_length); + clib_memcpy_fast (b0->data, ep->data, ep->current_length); b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; vnet_buffer (b0)->sw_if_index[VLIB_TX] = ep->tx_sw_if_index; vnet_buffer (b0)->sw_if_index[VLIB_RX] = diff --git a/src/plugins/perfmon/perfmon_periodic.c b/src/plugins/perfmon/perfmon_periodic.c index 329a75d7305..4e7e2378320 100644 --- a/src/plugins/perfmon/perfmon_periodic.c +++ b/src/plugins/perfmon/perfmon_periodic.c @@ -271,7 +271,7 @@ scrape_and_clear_counters (perfmon_main_t * pm) { n = nm->nodes[i]; nodes[i] = clib_mem_alloc (sizeof (*n)); - clib_memcpy (nodes[i], n, sizeof (*n)); + clib_memcpy_fast (nodes[i], n, sizeof (*n)); n->stats_total.perf_counter_ticks = 0; n->stats_total.perf_counter_vectors = 0; n->stats_last_clear.perf_counter_ticks = 0; diff --git a/src/plugins/pppoe/pppoe_cp_node.c b/src/plugins/pppoe/pppoe_cp_node.c index c73666de744..884d207d655 100644 --- a/src/plugins/pppoe/pppoe_cp_node.c +++ b/src/plugins/pppoe/pppoe_cp_node.c @@ -157,7 +157,7 @@ pppoe_cp_dispatch (vlib_main_t * vm, /* set src mac address */ si = vnet_get_sw_interface(vnm, tx_sw_if_index0); hi = vnet_get_hw_interface (vnm, si->hw_if_index); - clib_memcpy (vlib_buffer_get_current (b0)+6, hi->hw_address, 6); + clib_memcpy_fast (vlib_buffer_get_current (b0)+6, hi->hw_address, 6); } else { diff --git a/src/plugins/srv6-ad/node.c b/src/plugins/srv6-ad/node.c index 776d283dbe7..8f32ae0e49e 100644 --- a/src/plugins/srv6-ad/node.c +++ b/src/plugins/srv6-ad/node.c @@ -166,7 +166,7 @@ end_ad_processing (vlib_buffer_t * b0, { vec_validate (ls0_mem->rewrite, total_size - 1); } - clib_memcpy (ls0_mem->rewrite, ip0, total_size); + clib_memcpy_fast (ls0_mem->rewrite, ip0, total_size); ls0_mem->rw_len = total_size; /* Remove IP header and extensions */ @@ -349,8 +349,8 @@ srv6_ad2_rewrite_fn (vlib_main_t * vm, ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= (ls0_mem->rw_len + b0->current_data)); - clib_memcpy (((u8 *) en0) - ls0_mem->rw_len, - ls0_mem->rewrite, ls0_mem->rw_len); + clib_memcpy_fast (((u8 *) en0) - ls0_mem->rw_len, + ls0_mem->rewrite, ls0_mem->rw_len); vlib_buffer_advance (b0, -(word) ls0_mem->rw_len); ip0 = vlib_buffer_get_current (b0); @@ -373,10 +373,10 @@ srv6_ad2_rewrite_fn (vlib_main_t * vm, } else { - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof tr->src.as_u8); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof tr->dst.as_u8); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof tr->src.as_u8); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof tr->dst.as_u8); } } @@ -484,8 +484,8 @@ srv6_ad4_rewrite_fn (vlib_main_t * vm, ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= (ls0_mem->rw_len + b0->current_data)); - clib_memcpy (((u8 *) ip0_encap) - ls0_mem->rw_len, - ls0_mem->rewrite, ls0_mem->rw_len); + clib_memcpy_fast (((u8 *) ip0_encap) - ls0_mem->rw_len, + ls0_mem->rewrite, ls0_mem->rw_len); vlib_buffer_advance (b0, -(word) ls0_mem->rw_len); ip0 = vlib_buffer_get_current (b0); @@ -516,10 +516,10 @@ srv6_ad4_rewrite_fn (vlib_main_t * vm, } else { - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof tr->src.as_u8); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof tr->dst.as_u8); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof tr->src.as_u8); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof tr->dst.as_u8); } } @@ -626,8 +626,8 @@ srv6_ad6_rewrite_fn (vlib_main_t * vm, ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= (ls0_mem->rw_len + b0->current_data)); - clib_memcpy (((u8 *) ip0_encap) - ls0_mem->rw_len, - ls0_mem->rewrite, ls0_mem->rw_len); + clib_memcpy_fast (((u8 *) ip0_encap) - ls0_mem->rw_len, + ls0_mem->rewrite, ls0_mem->rw_len); vlib_buffer_advance (b0, -(word) ls0_mem->rw_len); ip0 = vlib_buffer_get_current (b0); @@ -654,10 +654,10 @@ srv6_ad6_rewrite_fn (vlib_main_t * vm, } else { - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof tr->src.as_u8); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof tr->dst.as_u8); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof tr->src.as_u8); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof tr->dst.as_u8); } } diff --git a/src/plugins/srv6-am/node.c b/src/plugins/srv6-am/node.c index 84014c5ee8d..dd71e60d5a2 100644 --- a/src/plugins/srv6-am/node.c +++ b/src/plugins/srv6-am/node.c @@ -306,10 +306,10 @@ srv6_am_rewrite_fn (vlib_main_t * vm, { srv6_am_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof *tr); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof tr->src.as_u8); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof tr->dst.as_u8); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof tr->src.as_u8); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof tr->dst.as_u8); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, diff --git a/src/plugins/srv6-as/node.c b/src/plugins/srv6-as/node.c index d81a7e88722..6f424fcca12 100644 --- a/src/plugins/srv6-as/node.c +++ b/src/plugins/srv6-as/node.c @@ -316,8 +316,8 @@ srv6_as2_rewrite_fn (vlib_main_t * vm, ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= (vec_len (ls0_mem->rewrite) + b0->current_data)); - clib_memcpy (((u8 *) en0) - vec_len (ls0_mem->rewrite), - ls0_mem->rewrite, vec_len (ls0_mem->rewrite)); + clib_memcpy_fast (((u8 *) en0) - vec_len (ls0_mem->rewrite), + ls0_mem->rewrite, vec_len (ls0_mem->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (ls0_mem->rewrite)); ip0 = vlib_buffer_get_current (b0); @@ -340,10 +340,10 @@ srv6_as2_rewrite_fn (vlib_main_t * vm, } else { - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof tr->src.as_u8); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof tr->dst.as_u8); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof tr->src.as_u8); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof tr->dst.as_u8); } } @@ -451,8 +451,9 @@ srv6_as4_rewrite_fn (vlib_main_t * vm, ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= (vec_len (ls0_mem->rewrite) + b0->current_data)); - clib_memcpy (((u8 *) ip0_encap) - vec_len (ls0_mem->rewrite), - ls0_mem->rewrite, vec_len (ls0_mem->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - + vec_len (ls0_mem->rewrite), ls0_mem->rewrite, + vec_len (ls0_mem->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (ls0_mem->rewrite)); ip0 = vlib_buffer_get_current (b0); @@ -483,10 +484,10 @@ srv6_as4_rewrite_fn (vlib_main_t * vm, } else { - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof tr->src.as_u8); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof tr->dst.as_u8); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof tr->src.as_u8); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof tr->dst.as_u8); } } @@ -593,8 +594,9 @@ srv6_as6_rewrite_fn (vlib_main_t * vm, ASSERT (VLIB_BUFFER_PRE_DATA_SIZE >= (vec_len (ls0_mem->rewrite) + b0->current_data)); - clib_memcpy (((u8 *) ip0_encap) - vec_len (ls0_mem->rewrite), - ls0_mem->rewrite, vec_len (ls0_mem->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - + vec_len (ls0_mem->rewrite), ls0_mem->rewrite, + vec_len (ls0_mem->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (ls0_mem->rewrite)); ip0 = vlib_buffer_get_current (b0); @@ -621,10 +623,10 @@ srv6_as6_rewrite_fn (vlib_main_t * vm, } else { - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof tr->src.as_u8); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof tr->dst.as_u8); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof tr->src.as_u8); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof tr->dst.as_u8); } } diff --git a/src/plugins/stn/stn.c b/src/plugins/stn/stn.c index e3f56d38a15..6f5fbdb2866 100644 --- a/src/plugins/stn/stn.c +++ b/src/plugins/stn/stn.c @@ -141,9 +141,9 @@ stn_ip46_punt_fn (vlib_main_t * vm, vlib_buffer_advance(p0, -sizeof(*eth)); eth = (ethernet_header_t *) vlib_buffer_get_current(p0); if (is_ipv4) - clib_memcpy(eth, &stn_ip4_ethernet_header, sizeof(*eth)); + clib_memcpy_fast(eth, &stn_ip4_ethernet_header, sizeof(*eth)); else - clib_memcpy(eth, &stn_ip6_ethernet_header, sizeof(*eth)); + clib_memcpy_fast(eth, &stn_ip6_ethernet_header, sizeof(*eth)); } else { @@ -261,12 +261,12 @@ stn_init (vlib_main_t * vm) clib_bihash_init_16_8(&stn->rule_by_address_table, "stn addresses", 1024, 1<<20); - clib_memcpy(stn_ip4_ethernet_header.dst_address, stn_hw_addr_dst, 6); - clib_memcpy(stn_ip4_ethernet_header.src_address, stn_hw_addr_local, 6); + clib_memcpy_fast(stn_ip4_ethernet_header.dst_address, stn_hw_addr_dst, 6); + clib_memcpy_fast(stn_ip4_ethernet_header.src_address, stn_hw_addr_local, 6); stn_ip4_ethernet_header.type = clib_host_to_net_u16(ETHERNET_TYPE_IP4); - clib_memcpy(stn_ip6_ethernet_header.dst_address, stn_hw_addr_dst, 6); - clib_memcpy(stn_ip6_ethernet_header.src_address, stn_hw_addr_local, 6); + clib_memcpy_fast(stn_ip6_ethernet_header.dst_address, stn_hw_addr_dst, 6); + clib_memcpy_fast(stn_ip6_ethernet_header.src_address, stn_hw_addr_local, 6); stn_ip6_ethernet_header.type = clib_host_to_net_u16(ETHERNET_TYPE_IP6); return stn_api_init (vm, stn); diff --git a/src/plugins/unittest/tcp_test.c b/src/plugins/unittest/tcp_test.c index 0bc1153e6dd..bbb80404019 100644 --- a/src/plugins/unittest/tcp_test.c +++ b/src/plugins/unittest/tcp_test.c @@ -1662,7 +1662,7 @@ tcp_test_lookup (vlib_main_t * vm, unformat_input_t * input) tc->connection.rmt_port = 53764; tc->connection.proto = TRANSPORT_PROTO_TCP; tc->connection.is_ip4 = 1; - clib_memcpy (tc1, &tc->connection, sizeof (*tc1)); + clib_memcpy_fast (tc1, &tc->connection, sizeof (*tc1)); /* * Allocate fake session and connection 2 @@ -1683,7 +1683,7 @@ tcp_test_lookup (vlib_main_t * vm, unformat_input_t * input) tc->connection.rmt_port = 53764; tc->connection.proto = TRANSPORT_PROTO_TCP; tc->connection.is_ip4 = 1; - clib_memcpy (tc2, &tc->connection, sizeof (*tc2)); + clib_memcpy_fast (tc2, &tc->connection, sizeof (*tc2)); /* * Confirm that connection lookup works diff --git a/src/svm/queue.c b/src/svm/queue.c index 6972f14cd33..3577c783dba 100644 --- a/src/svm/queue.c +++ b/src/svm/queue.c @@ -201,7 +201,7 @@ svm_queue_add_nolock (svm_queue_t * q, u8 * elem) } tailp = (i8 *) (&q->data[0] + q->elsize * q->tail); - clib_memcpy (tailp, elem, q->elsize); + clib_memcpy_fast (tailp, elem, q->elsize); q->tail++; q->cursize++; @@ -222,7 +222,7 @@ svm_queue_add_raw (svm_queue_t * q, u8 * elem) i8 *tailp; tailp = (i8 *) (&q->data[0] + q->elsize * q->tail); - clib_memcpy (tailp, elem, q->elsize); + clib_memcpy_fast (tailp, elem, q->elsize); q->tail = (q->tail + 1) % q->maxsize; q->cursize++; @@ -264,7 +264,7 @@ svm_queue_add (svm_queue_t * q, u8 * elem, int nowait) } tailp = (i8 *) (&q->data[0] + q->elsize * q->tail); - clib_memcpy (tailp, elem, q->elsize); + clib_memcpy_fast (tailp, elem, q->elsize); q->tail++; q->cursize++; @@ -314,7 +314,7 @@ svm_queue_add2 (svm_queue_t * q, u8 * elem, u8 * elem2, int nowait) } tailp = (i8 *) (&q->data[0] + q->elsize * q->tail); - clib_memcpy (tailp, elem, q->elsize); + clib_memcpy_fast (tailp, elem, q->elsize); q->tail++; q->cursize++; @@ -325,7 +325,7 @@ svm_queue_add2 (svm_queue_t * q, u8 * elem, u8 * elem2, int nowait) need_broadcast = (q->cursize == 1); tailp = (i8 *) (&q->data[0] + q->elsize * q->tail); - clib_memcpy (tailp, elem2, q->elsize); + clib_memcpy_fast (tailp, elem2, q->elsize); q->tail++; q->cursize++; @@ -389,7 +389,7 @@ svm_queue_sub (svm_queue_t * q, u8 * elem, svm_q_conditional_wait_t cond, } headp = (i8 *) (&q->data[0] + q->elsize * q->head); - clib_memcpy (elem, headp, q->elsize); + clib_memcpy_fast (elem, headp, q->elsize); q->head++; /* $$$$ JFC shouldn't this be == 0? */ @@ -423,7 +423,7 @@ svm_queue_sub2 (svm_queue_t * q, u8 * elem) } headp = (i8 *) (&q->data[0] + q->elsize * q->head); - clib_memcpy (elem, headp, q->elsize); + clib_memcpy_fast (elem, headp, q->elsize); q->head++; need_broadcast = (q->cursize == q->maxsize / 2); @@ -451,7 +451,7 @@ svm_queue_sub_raw (svm_queue_t * q, u8 * elem) } headp = (i8 *) (&q->data[0] + q->elsize * q->head); - clib_memcpy (elem, headp, q->elsize); + clib_memcpy_fast (elem, headp, q->elsize); q->head = (q->head + 1) % q->maxsize; q->cursize--; diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index 817cb8d2639..fb942a63d43 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -485,7 +485,7 @@ CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes, first_copy_bytes = ((nitems - f->tail) < total_copy_bytes) ? (nitems - f->tail) : total_copy_bytes; - _clib_memcpy (&f->data[f->tail], copy_from_here, first_copy_bytes); + clib_memcpy_fast (&f->data[f->tail], copy_from_here, first_copy_bytes); f->tail += first_copy_bytes; f->tail = (f->tail == nitems) ? 0 : f->tail; @@ -493,8 +493,9 @@ CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes, second_copy_bytes = total_copy_bytes - first_copy_bytes; if (second_copy_bytes) { - _clib_memcpy (&f->data[f->tail], copy_from_here + first_copy_bytes, - second_copy_bytes); + clib_memcpy_fast (&f->data[f->tail], + copy_from_here + first_copy_bytes, + second_copy_bytes); f->tail += second_copy_bytes; f->tail = (f->tail == nitems) ? 0 : f->tail; } @@ -571,8 +572,8 @@ CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f, first_copy_bytes = ((nitems - normalized_offset) < total_copy_bytes) ? (nitems - normalized_offset) : total_copy_bytes; - _clib_memcpy (&f->data[normalized_offset], copy_from_here, - first_copy_bytes); + clib_memcpy_fast (&f->data[normalized_offset], copy_from_here, + first_copy_bytes); /* Number of bytes in second copy segment, if any */ second_copy_bytes = total_copy_bytes - first_copy_bytes; @@ -583,8 +584,8 @@ CLIB_MARCH_FN (svm_fifo_enqueue_with_offset, int, svm_fifo_t * f, ASSERT (normalized_offset == 0); - _clib_memcpy (&f->data[normalized_offset], - copy_from_here + first_copy_bytes, second_copy_bytes); + clib_memcpy_fast (&f->data[normalized_offset], + copy_from_here + first_copy_bytes, second_copy_bytes); } return (0); @@ -608,11 +609,11 @@ svm_fifo_overwrite_head (svm_fifo_t * f, u8 * data, u32 len) first_chunk = f->nitems - f->head; ASSERT (len <= f->nitems); if (len <= first_chunk) - _clib_memcpy (&f->data[f->head], data, len); + clib_memcpy_fast (&f->data[f->head], data, len); else { - _clib_memcpy (&f->data[f->head], data, first_chunk); - _clib_memcpy (&f->data[0], data + first_chunk, len - first_chunk); + clib_memcpy_fast (&f->data[f->head], data, first_chunk); + clib_memcpy_fast (&f->data[0], data + first_chunk, len - first_chunk); } } #endif @@ -638,7 +639,7 @@ CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes, /* Number of bytes in first copy segment */ first_copy_bytes = ((nitems - f->head) < total_copy_bytes) ? (nitems - f->head) : total_copy_bytes; - _clib_memcpy (copy_here, &f->data[f->head], first_copy_bytes); + clib_memcpy_fast (copy_here, &f->data[f->head], first_copy_bytes); f->head += first_copy_bytes; f->head = (f->head == nitems) ? 0 : f->head; @@ -646,8 +647,8 @@ CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes, second_copy_bytes = total_copy_bytes - first_copy_bytes; if (second_copy_bytes) { - _clib_memcpy (copy_here + first_copy_bytes, - &f->data[f->head], second_copy_bytes); + clib_memcpy_fast (copy_here + first_copy_bytes, + &f->data[f->head], second_copy_bytes); f->head += second_copy_bytes; f->head = (f->head == nitems) ? 0 : f->head; } @@ -705,14 +706,14 @@ CLIB_MARCH_FN (svm_fifo_peek, int, svm_fifo_t * f, u32 relative_offset, first_copy_bytes = ((nitems - real_head) < total_copy_bytes) ? (nitems - real_head) : total_copy_bytes; - _clib_memcpy (copy_here, &f->data[real_head], first_copy_bytes); + clib_memcpy_fast (copy_here, &f->data[real_head], first_copy_bytes); /* Number of bytes in second copy segment, if any */ second_copy_bytes = total_copy_bytes - first_copy_bytes; if (second_copy_bytes) { - _clib_memcpy (copy_here + first_copy_bytes, &f->data[0], - second_copy_bytes); + clib_memcpy_fast (copy_here + first_copy_bytes, &f->data[0], + second_copy_bytes); } } return total_copy_bytes; diff --git a/src/tests/vnet/session/tcp_echo.c b/src/tests/vnet/session/tcp_echo.c index 1a0df31feec..bde55ad37f4 100644 --- a/src/tests/vnet/session/tcp_echo.c +++ b/src/tests/vnet/session/tcp_echo.c @@ -306,7 +306,7 @@ application_send_attach (echo_main_t * em) cert_mp->client_index = em->my_client_index; cert_mp->context = ntohl (0xfeedface); cert_mp->cert_len = clib_host_to_net_u16 (test_srv_crt_rsa_len); - clib_memcpy (cert_mp->cert, test_srv_crt_rsa, test_srv_crt_rsa_len); + clib_memcpy_fast (cert_mp->cert, test_srv_crt_rsa, test_srv_crt_rsa_len); vl_msg_api_send_shmem (em->vl_input_queue, (u8 *) & cert_mp); key_mp = vl_msg_api_alloc (sizeof (*key_mp) + test_srv_key_rsa_len); @@ -315,7 +315,7 @@ application_send_attach (echo_main_t * em) key_mp->client_index = em->my_client_index; key_mp->context = ntohl (0xfeedface); key_mp->key_len = clib_host_to_net_u16 (test_srv_key_rsa_len); - clib_memcpy (key_mp->key, test_srv_key_rsa, test_srv_key_rsa_len); + clib_memcpy_fast (key_mp->key, test_srv_key_rsa, test_srv_key_rsa_len); vl_msg_api_send_shmem (em->vl_input_queue, (u8 *) & key_mp); } diff --git a/src/tests/vnet/session/udp_echo.c b/src/tests/vnet/session/udp_echo.c index 887150cca82..3c928965c02 100644 --- a/src/tests/vnet/session/udp_echo.c +++ b/src/tests/vnet/session/udp_echo.c @@ -539,8 +539,8 @@ session_accepted_handler (session_accepted_msg_t * mp) tx_fifo->client_session_index = session_index; session->rx_fifo = rx_fifo; session->tx_fifo = tx_fifo; - clib_memcpy (&session->transport.rmt_ip, mp->ip, - sizeof (ip46_address_t)); + clib_memcpy_fast (&session->transport.rmt_ip, mp->ip, + sizeof (ip46_address_t)); session->transport.is_ip4 = mp->is_ip4; session->transport.rmt_port = mp->port; } @@ -645,8 +645,8 @@ session_connected_handler (session_connected_msg_t * mp) session->rx_fifo->client_session_index = session->session_index; session->tx_fifo->client_session_index = session->session_index; - clib_memcpy (&session->transport.lcl_ip, mp->lcl_ip, - sizeof (ip46_address_t)); + clib_memcpy_fast (&session->transport.lcl_ip, mp->lcl_ip, + sizeof (ip46_address_t)); session->transport.is_ip4 = mp->is_ip4; session->transport.lcl_port = mp->lcl_port; @@ -659,8 +659,8 @@ session_connected_handler (session_connected_msg_t * mp) return; } unformat_free (input); - clib_memcpy (&session->transport.rmt_ip, &sep->ip, - sizeof (ip46_address_t)); + clib_memcpy_fast (&session->transport.rmt_ip, &sep->ip, + sizeof (ip46_address_t)); session->transport.rmt_port = sep->port; session->is_dgram = !utm->is_connected; } @@ -693,8 +693,8 @@ session_bound_handler (session_bound_msg_t * mp) tx_fifo->client_session_index = session_index; session->rx_fifo = rx_fifo; session->tx_fifo = tx_fifo; - clib_memcpy (&session->transport.lcl_ip, mp->lcl_ip, - sizeof (ip46_address_t)); + clib_memcpy_fast (&session->transport.lcl_ip, mp->lcl_ip, + sizeof (ip46_address_t)); session->transport.is_ip4 = mp->lcl_is_ip4; session->transport.lcl_port = mp->lcl_port; session->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *); @@ -900,8 +900,8 @@ vl_api_bind_uri_reply_t_handler (vl_api_bind_uri_reply_t * mp) tx_fifo->client_session_index = session_index; session->rx_fifo = rx_fifo; session->tx_fifo = tx_fifo; - clib_memcpy (&session->transport.lcl_ip, mp->lcl_ip, - sizeof (ip46_address_t)); + clib_memcpy_fast (&session->transport.lcl_ip, mp->lcl_ip, + sizeof (ip46_address_t)); session->transport.is_ip4 = mp->lcl_is_ip4; session->transport.lcl_port = mp->lcl_port; session->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *); diff --git a/src/vcl/ldp.c b/src/vcl/ldp.c index 96a7c36acef..e806dff7959 100644 --- a/src/vcl/ldp.c +++ b/src/vcl/ldp.c @@ -850,8 +850,8 @@ ldp_pselect (int nfds, fd_set * __restrict readfds, clib_bitmap_validate (ldp->sid_rd_bitmap, minbits); clib_bitmap_validate (ldp->libc_rd_bitmap, minbits); clib_bitmap_validate (ldp->rd_bitmap, minbits); - clib_memcpy (ldp->rd_bitmap, readfds, - vec_len (ldp->rd_bitmap) * sizeof (clib_bitmap_t)); + clib_memcpy_fast (ldp->rd_bitmap, readfds, + vec_len (ldp->rd_bitmap) * sizeof (clib_bitmap_t)); FD_ZERO (readfds); /* *INDENT-OFF* */ @@ -884,8 +884,8 @@ ldp_pselect (int nfds, fd_set * __restrict readfds, clib_bitmap_validate (ldp->sid_wr_bitmap, minbits); clib_bitmap_validate (ldp->libc_wr_bitmap, minbits); clib_bitmap_validate (ldp->wr_bitmap, minbits); - clib_memcpy (ldp->wr_bitmap, writefds, - vec_len (ldp->wr_bitmap) * sizeof (clib_bitmap_t)); + clib_memcpy_fast (ldp->wr_bitmap, writefds, + vec_len (ldp->wr_bitmap) * sizeof (clib_bitmap_t)); FD_ZERO (writefds); /* *INDENT-OFF* */ @@ -918,8 +918,8 @@ ldp_pselect (int nfds, fd_set * __restrict readfds, clib_bitmap_validate (ldp->sid_ex_bitmap, minbits); clib_bitmap_validate (ldp->libc_ex_bitmap, minbits); clib_bitmap_validate (ldp->ex_bitmap, minbits); - clib_memcpy (ldp->ex_bitmap, exceptfds, - vec_len (ldp->ex_bitmap) * sizeof (clib_bitmap_t)); + clib_memcpy_fast (ldp->ex_bitmap, exceptfds, + vec_len (ldp->ex_bitmap) * sizeof (clib_bitmap_t)); FD_ZERO (exceptfds); /* *INDENT-OFF* */ @@ -964,17 +964,17 @@ ldp_pselect (int nfds, fd_set * __restrict readfds, func_str = "vppcom_select"; if (readfds) - clib_memcpy (ldp->rd_bitmap, ldp->sid_rd_bitmap, - vec_len (ldp->rd_bitmap) * - sizeof (clib_bitmap_t)); + clib_memcpy_fast (ldp->rd_bitmap, ldp->sid_rd_bitmap, + vec_len (ldp->rd_bitmap) * + sizeof (clib_bitmap_t)); if (writefds) - clib_memcpy (ldp->wr_bitmap, ldp->sid_wr_bitmap, - vec_len (ldp->wr_bitmap) * - sizeof (clib_bitmap_t)); + clib_memcpy_fast (ldp->wr_bitmap, ldp->sid_wr_bitmap, + vec_len (ldp->wr_bitmap) * + sizeof (clib_bitmap_t)); if (exceptfds) - clib_memcpy (ldp->ex_bitmap, ldp->sid_ex_bitmap, - vec_len (ldp->ex_bitmap) * - sizeof (clib_bitmap_t)); + clib_memcpy_fast (ldp->ex_bitmap, ldp->sid_ex_bitmap, + vec_len (ldp->ex_bitmap) * + sizeof (clib_bitmap_t)); rv = vppcom_select (sid_bits, readfds ? ldp->rd_bitmap : NULL, @@ -1049,14 +1049,17 @@ ldp_pselect (int nfds, fd_set * __restrict readfds, func_str = "libc_pselect"; if (readfds) - clib_memcpy (readfds, ldp->libc_rd_bitmap, - vec_len (ldp->rd_bitmap) * sizeof (clib_bitmap_t)); + clib_memcpy_fast (readfds, ldp->libc_rd_bitmap, + vec_len (ldp->rd_bitmap) * + sizeof (clib_bitmap_t)); if (writefds) - clib_memcpy (writefds, ldp->libc_wr_bitmap, - vec_len (ldp->wr_bitmap) * sizeof (clib_bitmap_t)); + clib_memcpy_fast (writefds, ldp->libc_wr_bitmap, + vec_len (ldp->wr_bitmap) * + sizeof (clib_bitmap_t)); if (exceptfds) - clib_memcpy (exceptfds, ldp->libc_ex_bitmap, - vec_len (ldp->ex_bitmap) * sizeof (clib_bitmap_t)); + clib_memcpy_fast (exceptfds, ldp->libc_ex_bitmap, + vec_len (ldp->ex_bitmap) * + sizeof (clib_bitmap_t)); tspec.tv_sec = tspec.tv_nsec = 0; rv = libc_pselect (libc_bits, readfds ? readfds : NULL, diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c index 7d02c4fc793..ea8f2ce5651 100644 --- a/src/vcl/vcl_bapi.c +++ b/src/vcl/vcl_bapi.c @@ -401,7 +401,7 @@ vppcom_app_send_attach (void) if (nsid_len) { bmp->namespace_id_len = nsid_len; - clib_memcpy (bmp->namespace_id, vcm->cfg.namespace_id, nsid_len); + clib_memcpy_fast (bmp->namespace_id, vcm->cfg.namespace_id, nsid_len); bmp->options[APP_OPTIONS_NAMESPACE_SECRET] = vcm->cfg.namespace_secret; } vl_msg_api_send_shmem (vcm->vl_input_queue, (u8 *) & bmp); @@ -453,10 +453,10 @@ vppcom_send_connect_sock (vcl_session_t * session) cmp->context = session->session_index; cmp->wrk_index = vcl_get_worker_index (); cmp->is_ip4 = session->transport.is_ip4; - clib_memcpy (cmp->ip, &session->transport.rmt_ip, sizeof (cmp->ip)); + clib_memcpy_fast (cmp->ip, &session->transport.rmt_ip, sizeof (cmp->ip)); cmp->port = session->transport.rmt_port; cmp->proto = session->session_type; - clib_memcpy (cmp->options, session->options, sizeof (cmp->options)); + clib_memcpy_fast (cmp->options, session->options, sizeof (cmp->options)); vl_msg_api_send_shmem (vcm->vl_input_queue, (u8 *) & cmp); } @@ -490,10 +490,10 @@ vppcom_send_bind_sock (vcl_session_t * session) bmp->context = session->session_index; bmp->wrk_index = vcl_get_worker_index (); bmp->is_ip4 = session->transport.is_ip4; - clib_memcpy (bmp->ip, &session->transport.lcl_ip, sizeof (bmp->ip)); + clib_memcpy_fast (bmp->ip, &session->transport.lcl_ip, sizeof (bmp->ip)); bmp->port = session->transport.lcl_port; bmp->proto = session->session_type; - clib_memcpy (bmp->options, session->options, sizeof (bmp->options)); + clib_memcpy_fast (bmp->options, session->options, sizeof (bmp->options)); vl_msg_api_send_shmem (vcm->vl_input_queue, (u8 *) & bmp); } diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index 46b1c106d20..99547377523 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -311,7 +311,8 @@ vcl_session_accepted_handler (vcl_worker_t * wrk, session_accepted_msg_t * mp) session->session_state = STATE_ACCEPT; session->transport.rmt_port = mp->port; session->transport.is_ip4 = mp->is_ip4; - clib_memcpy (&session->transport.rmt_ip, mp->ip, sizeof (ip46_address_t)); + clib_memcpy_fast (&session->transport.rmt_ip, mp->ip, + sizeof (ip46_address_t)); vcl_session_table_add_vpp_handle (wrk, mp->handle, session->session_index); session->transport.lcl_port = listen_session->transport.lcl_port; @@ -389,8 +390,8 @@ vcl_session_connected_handler (vcl_worker_t * wrk, session->tx_fifo = tx_fifo; session->vpp_handle = mp->handle; session->transport.is_ip4 = mp->is_ip4; - clib_memcpy (&session->transport.lcl_ip, mp->lcl_ip, - sizeof (session->transport.lcl_ip)); + clib_memcpy_fast (&session->transport.lcl_ip, mp->lcl_ip, + sizeof (session->transport.lcl_ip)); session->transport.lcl_port = mp->lcl_port; session->session_state = STATE_CONNECT; @@ -454,8 +455,8 @@ vcl_session_bound_handler (vcl_worker_t * wrk, session_bound_msg_t * mp) session->vpp_handle = mp->handle; session->transport.is_ip4 = mp->lcl_is_ip4; - clib_memcpy (&session->transport.lcl_ip, mp->lcl_ip, - sizeof (ip46_address_t)); + clib_memcpy_fast (&session->transport.lcl_ip, mp->lcl_ip, + sizeof (ip46_address_t)); session->transport.lcl_port = mp->lcl_port; vcl_session_table_add_listener (wrk, mp->handle, sid); session->session_state = STATE_LISTEN; @@ -942,11 +943,11 @@ vppcom_session_bind (uint32_t session_handle, vppcom_endpt_t * ep) session->transport.is_ip4 = ep->is_ip4; if (ep->is_ip4) - clib_memcpy (&session->transport.lcl_ip.ip4, ep->ip, - sizeof (ip4_address_t)); + clib_memcpy_fast (&session->transport.lcl_ip.ip4, ep->ip, + sizeof (ip4_address_t)); else - clib_memcpy (&session->transport.lcl_ip.ip6, ep->ip, - sizeof (ip6_address_t)); + clib_memcpy_fast (&session->transport.lcl_ip.ip6, ep->ip, + sizeof (ip6_address_t)); session->transport.lcl_port = ep->port; VDBG (0, "VCL<%d>: sid %u: binding to local %s address %U port %u, " @@ -1091,7 +1092,7 @@ vppcom_session_accept (uint32_t listen_session_handle, vppcom_endpt_t * ep, svm_msg_q_free_msg (wrk->app_event_queue, &msg); continue; } - clib_memcpy (&accepted_msg, e->data, sizeof (accepted_msg)); + clib_memcpy_fast (&accepted_msg, e->data, sizeof (accepted_msg)); svm_msg_q_free_msg (wrk->app_event_queue, &msg); break; } @@ -1118,11 +1119,11 @@ handle: ep->is_ip4 = client_session->transport.is_ip4; ep->port = client_session->transport.rmt_port; if (client_session->transport.is_ip4) - clib_memcpy (ep->ip, &client_session->transport.rmt_ip.ip4, - sizeof (ip4_address_t)); + clib_memcpy_fast (ep->ip, &client_session->transport.rmt_ip.ip4, + sizeof (ip4_address_t)); else - clib_memcpy (ep->ip, &client_session->transport.rmt_ip.ip6, - sizeof (ip6_address_t)); + clib_memcpy_fast (ep->ip, &client_session->transport.rmt_ip.ip6, + sizeof (ip6_address_t)); } if (accepted_msg.server_event_queue_address) @@ -1191,11 +1192,11 @@ vppcom_session_connect (uint32_t session_handle, vppcom_endpt_t * server_ep) session->transport.is_ip4 = server_ep->is_ip4; if (session->transport.is_ip4) - clib_memcpy (&session->transport.rmt_ip.ip4, server_ep->ip, - sizeof (ip4_address_t)); + clib_memcpy_fast (&session->transport.rmt_ip.ip4, server_ep->ip, + sizeof (ip4_address_t)); else - clib_memcpy (&session->transport.rmt_ip.ip6, server_ep->ip, - sizeof (ip6_address_t)); + clib_memcpy_fast (&session->transport.rmt_ip.ip6, server_ep->ip, + sizeof (ip6_address_t)); session->transport.rmt_port = server_ep->port; VDBG (0, "VCL<%d>: vpp handle 0x%llx, sid %u: connecting to server %s %U " @@ -1481,11 +1482,11 @@ int vppcom_data_segment_copy (void *buf, vppcom_data_segments_t ds, u32 max_bytes) { u32 first_copy = clib_min (ds[0].len, max_bytes); - clib_memcpy (buf, ds[0].data, first_copy); + clib_memcpy_fast (buf, ds[0].data, first_copy); if (first_copy < max_bytes) { - clib_memcpy (buf + first_copy, ds[1].data, - clib_min (ds[1].len, max_bytes - first_copy)); + clib_memcpy_fast (buf + first_copy, ds[1].data, + clib_min (ds[1].len, max_bytes - first_copy)); } return 0; } @@ -1908,23 +1909,23 @@ vppcom_select (unsigned long n_bits, unsigned long *read_map, if (n_bits && read_map) { clib_bitmap_validate (wrk->rd_bitmap, minbits); - clib_memcpy (wrk->rd_bitmap, read_map, - vec_len (wrk->rd_bitmap) * sizeof (clib_bitmap_t)); + clib_memcpy_fast (wrk->rd_bitmap, read_map, + vec_len (wrk->rd_bitmap) * sizeof (clib_bitmap_t)); memset (read_map, 0, vec_len (wrk->rd_bitmap) * sizeof (clib_bitmap_t)); } if (n_bits && write_map) { clib_bitmap_validate (wrk->wr_bitmap, minbits); - clib_memcpy (wrk->wr_bitmap, write_map, - vec_len (wrk->wr_bitmap) * sizeof (clib_bitmap_t)); + clib_memcpy_fast (wrk->wr_bitmap, write_map, + vec_len (wrk->wr_bitmap) * sizeof (clib_bitmap_t)); memset (write_map, 0, vec_len (wrk->wr_bitmap) * sizeof (clib_bitmap_t)); } if (n_bits && except_map) { clib_bitmap_validate (wrk->ex_bitmap, minbits); - clib_memcpy (wrk->ex_bitmap, except_map, - vec_len (wrk->ex_bitmap) * sizeof (clib_bitmap_t)); + clib_memcpy_fast (wrk->ex_bitmap, except_map, + vec_len (wrk->ex_bitmap) * sizeof (clib_bitmap_t)); memset (except_map, 0, vec_len (wrk->ex_bitmap) * sizeof (clib_bitmap_t)); } @@ -2646,11 +2647,11 @@ vppcom_session_attr (uint32_t session_handle, uint32_t op, ep->is_ip4 = session->transport.is_ip4; ep->port = session->transport.rmt_port; if (session->transport.is_ip4) - clib_memcpy (ep->ip, &session->transport.rmt_ip.ip4, - sizeof (ip4_address_t)); + clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip4, + sizeof (ip4_address_t)); else - clib_memcpy (ep->ip, &session->transport.rmt_ip.ip6, - sizeof (ip6_address_t)); + clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip6, + sizeof (ip6_address_t)); *buflen = sizeof (*ep); VDBG (1, "VCL<%d>: VPPCOM_ATTR_GET_PEER_ADDR: sid %u, is_ip4 = %u, " "addr = %U, port %u", getpid (), @@ -2670,11 +2671,11 @@ vppcom_session_attr (uint32_t session_handle, uint32_t op, ep->is_ip4 = session->transport.is_ip4; ep->port = session->transport.lcl_port; if (session->transport.is_ip4) - clib_memcpy (ep->ip, &session->transport.lcl_ip.ip4, - sizeof (ip4_address_t)); + clib_memcpy_fast (ep->ip, &session->transport.lcl_ip.ip4, + sizeof (ip4_address_t)); else - clib_memcpy (ep->ip, &session->transport.lcl_ip.ip6, - sizeof (ip6_address_t)); + clib_memcpy_fast (ep->ip, &session->transport.lcl_ip.ip6, + sizeof (ip6_address_t)); *buflen = sizeof (*ep); VDBG (1, "VCL<%d>: VPPCOM_ATTR_GET_LCL_ADDR: sid %u, is_ip4 = %u," " addr = %U port %d", getpid (), @@ -3146,11 +3147,11 @@ vppcom_session_recvfrom (uint32_t session_handle, void *buffer, if (ep) { if (session->transport.is_ip4) - clib_memcpy (ep->ip, &session->transport.rmt_ip.ip4, - sizeof (ip4_address_t)); + clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip4, + sizeof (ip4_address_t)); else - clib_memcpy (ep->ip, &session->transport.rmt_ip.ip6, - sizeof (ip6_address_t)); + clib_memcpy_fast (ep->ip, &session->transport.rmt_ip.ip6, + sizeof (ip6_address_t)); } return rv; diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index cc852358541..133d640d693 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -672,8 +672,8 @@ vlib_packet_template_buffer_init (vlib_main_t * vm, { vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]); ASSERT (b->current_length == vec_len (t->packet_data)); - clib_memcpy (vlib_buffer_get_current (b), t->packet_data, - b->current_length); + clib_memcpy_fast (vlib_buffer_get_current (b), t->packet_data, + b->current_length); } } @@ -738,8 +738,8 @@ vlib_packet_template_get_packet (vlib_main_t * vm, *bi_result = bi; b = vlib_get_buffer (vm, bi); - clib_memcpy (vlib_buffer_get_current (b), - t->packet_data, vec_len (t->packet_data)); + clib_memcpy_fast (vlib_buffer_get_current (b), + t->packet_data, vec_len (t->packet_data)); b->current_length = vec_len (t->packet_data); return b->data; @@ -796,7 +796,8 @@ vlib_buffer_add_data (vlib_main_t * vm, n_left_this_buffer = n_buffer_bytes - (b->current_data + b->current_length); n = clib_min (n_left_this_buffer, n_left); - clib_memcpy (vlib_buffer_get_current (b) + b->current_length, d, n); + clib_memcpy_fast (vlib_buffer_get_current (b) + b->current_length, d, + n); b->current_length += n; n_left -= n; if (n_left == 0) @@ -847,8 +848,8 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, } u16 len = (data_len > max) ? max : data_len; - clib_memcpy (vlib_buffer_get_current (l) + l->current_length, - data + copied, len); + clib_memcpy_fast (vlib_buffer_get_current (l) + l->current_length, + data + copied, len); vlib_buffer_chain_increase_length (first, l, len); data_len -= len; copied += len; @@ -974,8 +975,8 @@ vlib_buffer_main_init (struct vlib_main_t * vm) { /* external plugin has registered own buffer callbacks so we just copy them and quit */ - clib_memcpy (&bm->cb, vlib_buffer_callbacks, - sizeof (vlib_buffer_callbacks_t)); + clib_memcpy_fast (&bm->cb, vlib_buffer_callbacks, + sizeof (vlib_buffer_callbacks_t)); bm->callbacks_registered = 1; return 0; } diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index ce62c8b83fa..d15ef5702f8 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -304,7 +304,7 @@ vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents) { b = vlib_get_buffer (vm, buffer_index); l = b->current_length; - clib_memcpy (contents + content_len, b->data + b->current_data, l); + clib_memcpy_fast (contents + content_len, b->data + b->current_data, l); content_len += l; if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT)) break; @@ -470,7 +470,7 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm, /* following code is intentionaly duplicated to allow compiler to optimize fast path when n_buffers is constant value */ src = fl->buffers + len - n_buffers; - clib_memcpy (buffers, src, n_buffers * sizeof (u32)); + clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32)); _vec_len (fl->buffers) -= n_buffers; /* Verify that buffers are known free. */ @@ -481,7 +481,7 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm, } src = fl->buffers + len - n_buffers; - clib_memcpy (buffers, src, n_buffers * sizeof (u32)); + clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32)); _vec_len (fl->buffers) -= n_buffers; /* Verify that buffers are known free. */ @@ -758,10 +758,10 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b) d->flags = s->flags & flag_mask; d->total_length_not_including_first_buffer = s->total_length_not_including_first_buffer; - clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque)); - clib_memcpy (d->opaque2, s->opaque2, sizeof (s->opaque2)); - clib_memcpy (vlib_buffer_get_current (d), - vlib_buffer_get_current (s), s->current_length); + clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque)); + clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2)); + clib_memcpy_fast (vlib_buffer_get_current (d), + vlib_buffer_get_current (s), s->current_length); /* next segments */ for (i = 1; i < n_buffers; i++) @@ -773,8 +773,8 @@ vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b) d = vlib_get_buffer (vm, new_buffers[i]); d->current_data = s->current_data; d->current_length = s->current_length; - clib_memcpy (vlib_buffer_get_current (d), - vlib_buffer_get_current (s), s->current_length); + clib_memcpy_fast (vlib_buffer_get_current (d), + vlib_buffer_get_current (s), s->current_length); d->flags = s->flags & flag_mask; } @@ -846,10 +846,10 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, } d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT; d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID; - clib_memcpy (d->opaque, s->opaque, sizeof (s->opaque)); - clib_memcpy (d->opaque2, s->opaque2, sizeof (s->opaque2)); - clib_memcpy (vlib_buffer_get_current (d), vlib_buffer_get_current (s), - head_end_offset); + clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque)); + clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2)); + clib_memcpy_fast (vlib_buffer_get_current (d), + vlib_buffer_get_current (s), head_end_offset); d->next_buffer = src_buffer; } vlib_buffer_advance (s, head_end_offset); @@ -982,8 +982,8 @@ vlib_buffer_chain_append_data (vlib_main_t * vm, u16 len = clib_min (data_len, n_buffer_bytes - last->current_length - last->current_data); - clib_memcpy (vlib_buffer_get_current (last) + last->current_length, data, - len); + clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length, + data, len); vlib_buffer_chain_increase_length (first, last, len); return len; } @@ -1088,10 +1088,10 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst, /* Make sure buffer template is sane. */ ASSERT (fl->index == vlib_buffer_get_free_list_index (src)); - clib_memcpy (STRUCT_MARK_PTR (dst, template_start), - STRUCT_MARK_PTR (src, template_start), - STRUCT_OFFSET_OF (vlib_buffer_t, template_end) - - STRUCT_OFFSET_OF (vlib_buffer_t, template_start)); + clib_memcpy_fast (STRUCT_MARK_PTR (dst, template_start), + STRUCT_MARK_PTR (src, template_start), + STRUCT_OFFSET_OF (vlib_buffer_t, template_end) - + STRUCT_OFFSET_OF (vlib_buffer_t, template_start)); /* Not in the first 16 octets. */ dst->n_add_refs = src->n_add_refs; @@ -1240,9 +1240,9 @@ vlib_buffer_chain_compress (vlib_main_t * vm, vlib_buffer_t *second = vlib_get_buffer (vm, first->next_buffer); u32 need = want_first_size - first->current_length; u32 amount_to_copy = clib_min (need, second->current_length); - clib_memcpy (((u8 *) vlib_buffer_get_current (first)) + - first->current_length, - vlib_buffer_get_current (second), amount_to_copy); + clib_memcpy_fast (((u8 *) vlib_buffer_get_current (first)) + + first->current_length, + vlib_buffer_get_current (second), amount_to_copy); first->current_length += amount_to_copy; vlib_buffer_advance (second, amount_to_copy); if (first->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID) diff --git a/src/vlib/buffer_node.h b/src/vlib/buffer_node.h index 35e15a5d919..bff7a78cb94 100644 --- a/src/vlib/buffer_node.h +++ b/src/vlib/buffer_node.h @@ -383,7 +383,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node, #ifdef CLIB_HAVE_VEC512 if (n_enqueued >= 32) { - clib_memcpy (to_next, buffers, 32 * sizeof (u32)); + clib_memcpy_fast (to_next, buffers, 32 * sizeof (u32)); nexts += 32; to_next += 32; buffers += 32; @@ -397,7 +397,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node, #ifdef CLIB_HAVE_VEC256 if (n_enqueued >= 16) { - clib_memcpy (to_next, buffers, 16 * sizeof (u32)); + clib_memcpy_fast (to_next, buffers, 16 * sizeof (u32)); nexts += 16; to_next += 16; buffers += 16; @@ -411,7 +411,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node, #ifdef CLIB_HAVE_VEC128 if (n_enqueued >= 8) { - clib_memcpy (to_next, buffers, 8 * sizeof (u32)); + clib_memcpy_fast (to_next, buffers, 8 * sizeof (u32)); nexts += 8; to_next += 8; buffers += 8; @@ -424,7 +424,7 @@ vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node, if (n_enqueued >= 4) { - clib_memcpy (to_next, buffers, 4 * sizeof (u32)); + clib_memcpy_fast (to_next, buffers, 4 * sizeof (u32)); nexts += 4; to_next += 4; buffers += 4; diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c index c6fa8a7da34..ed43580ffc0 100644 --- a/src/vlib/linux/pci.c +++ b/src/vlib/linux/pci.c @@ -584,7 +584,7 @@ vfio_set_irqs (vlib_main_t * vm, linux_pci_device_t * p, u32 index, u32 start, if (efds) { flags |= VFIO_IRQ_SET_DATA_EVENTFD; - clib_memcpy (&irq_set->data, efds, data_len); + clib_memcpy_fast (&irq_set->data, efds, data_len); } else flags |= VFIO_IRQ_SET_DATA_NONE; diff --git a/src/vlib/main.c b/src/vlib/main.c index b46caf1bcb4..964feb06c65 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -1650,12 +1650,12 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) te->n_data_elts, te->n_data_elt_bytes); if (te->n_data_bytes < sizeof (te->inline_event_data)) - clib_memcpy (data, te->inline_event_data, - te->n_data_bytes); + clib_memcpy_fast (data, te->inline_event_data, + te->n_data_bytes); else { - clib_memcpy (data, te->event_data_as_vector, - te->n_data_bytes); + clib_memcpy_fast (data, te->event_data_as_vector, + te->n_data_bytes); vec_free (te->event_data_as_vector); } pool_put (nm->signal_timed_event_data_pool, te); diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h index 13614aa14cd..777746a0f78 100644 --- a/src/vlib/node_funcs.h +++ b/src/vlib/node_funcs.h @@ -135,7 +135,8 @@ vlib_node_set_runtime_data (vlib_main_t * vm, u32 node_index, STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data)); if (vec_len (n->runtime_data) > 0) - clib_memcpy (r->runtime_data, n->runtime_data, vec_len (n->runtime_data)); + clib_memcpy_fast (r->runtime_data, n->runtime_data, + vec_len (n->runtime_data)); } /** \brief Set node dispatch state. diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 7ecfa309378..fc2586c8967 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -1041,8 +1041,8 @@ vlib_worker_thread_node_refork (void) u64 *old_counters = vm_clone->error_main.counters; u64 *old_counters_all_clear = vm_clone->error_main.counters_last_clear; - clib_memcpy (&vm_clone->error_main, &vm->error_main, - sizeof (vm->error_main)); + clib_memcpy_fast (&vm_clone->error_main, &vm->error_main, + sizeof (vm->error_main)); j = vec_len (vm->error_main.counters) - 1; vec_validate_aligned (old_counters, j, CLIB_CACHE_LINE_BYTES); vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES); @@ -1083,7 +1083,7 @@ vlib_worker_thread_node_refork (void) new_n = nm->nodes[j]; old_n_clone = old_nodes_clone[j]; - clib_memcpy (new_n_clone, new_n, sizeof (*new_n)); + clib_memcpy_fast (new_n_clone, new_n, sizeof (*new_n)); /* none of the copied nodes have enqueue rights given out */ new_n_clone->owner_node_index = VLIB_INVALID_NODE_INDEX; @@ -1098,12 +1098,12 @@ vlib_worker_thread_node_refork (void) else { /* Copy stats if the old data is valid */ - clib_memcpy (&new_n_clone->stats_total, - &old_n_clone->stats_total, - sizeof (new_n_clone->stats_total)); - clib_memcpy (&new_n_clone->stats_last_clear, - &old_n_clone->stats_last_clear, - sizeof (new_n_clone->stats_last_clear)); + clib_memcpy_fast (&new_n_clone->stats_total, + &old_n_clone->stats_total, + sizeof (new_n_clone->stats_total)); + clib_memcpy_fast (&new_n_clone->stats_last_clear, + &old_n_clone->stats_last_clear, + sizeof (new_n_clone->stats_last_clear)); /* keep previous node state */ new_n_clone->state = old_n_clone->state; @@ -1129,17 +1129,17 @@ vlib_worker_thread_node_refork (void) rt->thread_index = vm_clone->thread_index; /* copy runtime_data, will be overwritten later for existing rt */ if (n->runtime_data && n->runtime_data_bytes > 0) - clib_memcpy (rt->runtime_data, n->runtime_data, - clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, - n->runtime_data_bytes)); + clib_memcpy_fast (rt->runtime_data, n->runtime_data, + clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, + n->runtime_data_bytes)); } for (j = 0; j < vec_len (old_rt); j++) { rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); rt->state = old_rt[j].state; - clib_memcpy (rt->runtime_data, old_rt[j].runtime_data, - VLIB_NODE_RUNTIME_DATA_SIZE); + clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); } vec_free (old_rt); @@ -1156,17 +1156,17 @@ vlib_worker_thread_node_refork (void) rt->thread_index = vm_clone->thread_index; /* copy runtime_data, will be overwritten later for existing rt */ if (n->runtime_data && n->runtime_data_bytes > 0) - clib_memcpy (rt->runtime_data, n->runtime_data, - clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, - n->runtime_data_bytes)); + clib_memcpy_fast (rt->runtime_data, n->runtime_data, + clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, + n->runtime_data_bytes)); } for (j = 0; j < vec_len (old_rt); j++) { rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); rt->state = old_rt[j].state; - clib_memcpy (rt->runtime_data, old_rt[j].runtime_data, - VLIB_NODE_RUNTIME_DATA_SIZE); + clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data, + VLIB_NODE_RUNTIME_DATA_SIZE); } vec_free (old_rt); diff --git a/src/vlib/trace.c b/src/vlib/trace.c index cfc3bda2cd0..e3b289879da 100644 --- a/src/vlib/trace.c +++ b/src/vlib/trace.c @@ -75,14 +75,14 @@ vlib_trace_frame_buffers_only (vlib_main_t * vm, if (b0->flags & VLIB_BUFFER_IS_TRACED) { t0 = vlib_add_trace (vm, node, b0, n_buffer_data_bytes_in_trace); - clib_memcpy (t0, b0->data + b0->current_data, - n_buffer_data_bytes_in_trace); + clib_memcpy_fast (t0, b0->data + b0->current_data, + n_buffer_data_bytes_in_trace); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { t1 = vlib_add_trace (vm, node, b1, n_buffer_data_bytes_in_trace); - clib_memcpy (t1, b1->data + b1->current_data, - n_buffer_data_bytes_in_trace); + clib_memcpy_fast (t1, b1->data + b1->current_data, + n_buffer_data_bytes_in_trace); } from += 2; n_left -= 2; @@ -101,8 +101,8 @@ vlib_trace_frame_buffers_only (vlib_main_t * vm, if (b0->flags & VLIB_BUFFER_IS_TRACED) { t0 = vlib_add_trace (vm, node, b0, n_buffer_data_bytes_in_trace); - clib_memcpy (t0, b0->data + b0->current_data, - n_buffer_data_bytes_in_trace); + clib_memcpy_fast (t0, b0->data + b0->current_data, + n_buffer_data_bytes_in_trace); } from += 1; n_left -= 1; diff --git a/src/vlibapi/api_shared.c b/src/vlibapi/api_shared.c index 44b09ee4f73..07abe3e2759 100644 --- a/src/vlibapi/api_shared.c +++ b/src/vlibapi/api_shared.c @@ -104,7 +104,7 @@ vl_msg_api_trace (api_main_t * am, vl_api_trace_t * tp, void *msg) length = clib_net_to_host_u32 (header->data_len); vec_validate (msg_copy, length - 1); - clib_memcpy (msg_copy, msg, length); + clib_memcpy_fast (msg_copy, msg, length); *this_trace = msg_copy; } diff --git a/src/vlibmemory/socket_api.c b/src/vlibmemory/socket_api.c index 3cf2a1cfe74..0f8c0f046a3 100644 --- a/src/vlibmemory/socket_api.c +++ b/src/vlibmemory/socket_api.c @@ -282,8 +282,8 @@ vl_socket_read_ready (clib_file_t * uf) { ASSERT (vec_len (rp->unprocessed_input) == 0); vec_validate (rp->unprocessed_input, vec_len (msg_buffer) - 1); - clib_memcpy (rp->unprocessed_input, msg_buffer, - vec_len (msg_buffer)); + clib_memcpy_fast (rp->unprocessed_input, msg_buffer, + vec_len (msg_buffer)); _vec_len (rp->unprocessed_input) = vec_len (msg_buffer); } _vec_len (socket_main.input_buffer) = save_input_buffer_length; @@ -513,7 +513,7 @@ vl_sock_api_send_fd_msg (int socket_fd, int fds[], int n_fds) cmsg->cmsg_len = CMSG_LEN (sizeof (int) * n_fds); cmsg->cmsg_level = SOL_SOCKET; cmsg->cmsg_type = SCM_RIGHTS; - clib_memcpy (CMSG_DATA (cmsg), fds, sizeof (int) * n_fds); + clib_memcpy_fast (CMSG_DATA (cmsg), fds, sizeof (int) * n_fds); rv = sendmsg (socket_fd, &mh, 0); if (rv < 0) diff --git a/src/vlibmemory/socket_client.c b/src/vlibmemory/socket_client.c index febb737ac2d..43294a0ab6e 100644 --- a/src/vlibmemory/socket_client.c +++ b/src/vlibmemory/socket_client.c @@ -246,7 +246,7 @@ vl_sock_api_recv_fd_msg (int socket_fd, int fds[], int n_fds, u32 wait) } else if (cmsg->cmsg_type == SCM_RIGHTS) { - clib_memcpy (fds, CMSG_DATA (cmsg), sizeof (int) * n_fds); + clib_memcpy_fast (fds, CMSG_DATA (cmsg), sizeof (int) * n_fds); } } cmsg = CMSG_NXTHDR (&mh, cmsg); diff --git a/src/vlibmemory/vlib_api.c b/src/vlibmemory/vlib_api.c index 6f0cdf94ef0..7763a88becf 100644 --- a/src/vlibmemory/vlib_api.c +++ b/src/vlibmemory/vlib_api.c @@ -601,7 +601,7 @@ vl_api_rpc_call_main_thread_inline (void *fp, u8 * data, u32 data_length, mp = vl_msg_api_alloc_as_if_client (sizeof (*mp) + data_length); clib_memset (mp, 0, sizeof (*mp)); - clib_memcpy (mp->data, data, data_length); + clib_memcpy_fast (mp->data, data, data_length); mp->_vl_msg_id = ntohs (VL_API_RPC_CALL); mp->function = pointer_to_uword (fp); mp->need_barrier_sync = 1; diff --git a/src/vnet/adj/rewrite.h b/src/vnet/adj/rewrite.h index 42b982e1bf4..58d47e0c737 100644 --- a/src/vnet/adj/rewrite.h +++ b/src/vnet/adj/rewrite.h @@ -131,7 +131,7 @@ vnet_rewrite_set_data_internal (vnet_rewrite_header_t * rw, ASSERT ((data_bytes >= 0) && (data_bytes < max_size)); rw->data_bytes = data_bytes; - clib_memcpy (rw->data + max_size - data_bytes, data, data_bytes); + clib_memcpy_fast (rw->data + max_size - data_bytes, data, data_bytes); clib_memset (rw->data, 0xfe, max_size - data_bytes); } diff --git a/src/vnet/bfd/bfd_api.c b/src/vnet/bfd/bfd_api.c index bb852256ad7..ea6730fb8b9 100644 --- a/src/vnet/bfd/bfd_api.c +++ b/src/vnet/bfd/bfd_api.c @@ -72,13 +72,13 @@ pub_sub_handler (bfd_events, BFD_EVENTS); clib_memset (&peer_addr, 0, sizeof (peer_addr)); \ if (mp->is_ipv6) \ { \ - clib_memcpy (&local_addr.ip6, mp->local_addr, sizeof (local_addr.ip6)); \ - clib_memcpy (&peer_addr.ip6, mp->peer_addr, sizeof (peer_addr.ip6)); \ + clib_memcpy_fast (&local_addr.ip6, mp->local_addr, sizeof (local_addr.ip6)); \ + clib_memcpy_fast (&peer_addr.ip6, mp->peer_addr, sizeof (peer_addr.ip6)); \ } \ else \ { \ - clib_memcpy (&local_addr.ip4, mp->local_addr, sizeof (local_addr.ip4)); \ - clib_memcpy (&peer_addr.ip4, mp->peer_addr, sizeof (peer_addr.ip4)); \ + clib_memcpy_fast (&local_addr.ip4, mp->local_addr, sizeof (local_addr.ip4)); \ + clib_memcpy_fast (&peer_addr.ip4, mp->peer_addr, sizeof (peer_addr.ip4)); \ } #define BFD_UDP_API_PARAM_FROM_MP(mp) \ @@ -176,16 +176,17 @@ send_bfd_udp_session_details (vl_api_registration_t * reg, u32 context, } if (mp->is_ipv6) { - clib_memcpy (mp->local_addr, &key->local_addr, - sizeof (key->local_addr)); - clib_memcpy (mp->peer_addr, &key->peer_addr, sizeof (key->peer_addr)); + clib_memcpy_fast (mp->local_addr, &key->local_addr, + sizeof (key->local_addr)); + clib_memcpy_fast (mp->peer_addr, &key->peer_addr, + sizeof (key->peer_addr)); } else { - clib_memcpy (mp->local_addr, key->local_addr.ip4.data, - sizeof (key->local_addr.ip4.data)); - clib_memcpy (mp->peer_addr, key->peer_addr.ip4.data, - sizeof (key->peer_addr.ip4.data)); + clib_memcpy_fast (mp->local_addr, key->local_addr.ip4.data, + sizeof (key->local_addr.ip4.data)); + clib_memcpy_fast (mp->peer_addr, key->peer_addr.ip4.data, + sizeof (key->peer_addr.ip4.data)); } mp->required_min_rx = @@ -387,7 +388,7 @@ vl_api_bfd_udp_get_echo_source_t_handler (vl_api_bfd_udp_get_echo_source_t * if (have_usable_ip4) { rmp->have_usable_ip4 = 1; - clib_memcpy (rmp->ip4_addr, &ip4, sizeof (ip4)); + clib_memcpy_fast (rmp->ip4_addr, &ip4, sizeof (ip4)); } else { @@ -396,7 +397,7 @@ vl_api_bfd_udp_get_echo_source_t_handler (vl_api_bfd_udp_get_echo_source_t * if (have_usable_ip6) { rmp->have_usable_ip6 = 1; - clib_memcpy (rmp->ip6_addr, &ip6, sizeof (ip6)); + clib_memcpy_fast (rmp->ip6_addr, &ip6, sizeof (ip6)); } else { diff --git a/src/vnet/bfd/bfd_udp.c b/src/vnet/bfd/bfd_udp.c index 0da19bdb630..d35c6dee5e1 100644 --- a/src/vnet/bfd/bfd_udp.c +++ b/src/vnet/bfd/bfd_udp.c @@ -330,17 +330,17 @@ bfd_add_udp6_transport (vlib_main_t * vm, u32 bi, const bfd_session_t * bs, { return rv; } - clib_memcpy (&headers->ip6.dst_address, &key->local_addr.ip6, - sizeof (headers->ip6.dst_address)); + clib_memcpy_fast (&headers->ip6.dst_address, &key->local_addr.ip6, + sizeof (headers->ip6.dst_address)); headers->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_bfd_echo6); } else { - clib_memcpy (&headers->ip6.src_address, &key->local_addr.ip6, - sizeof (headers->ip6.src_address)); - clib_memcpy (&headers->ip6.dst_address, &key->peer_addr.ip6, - sizeof (headers->ip6.dst_address)); + clib_memcpy_fast (&headers->ip6.src_address, &key->local_addr.ip6, + sizeof (headers->ip6.src_address)); + clib_memcpy_fast (&headers->ip6.dst_address, &key->peer_addr.ip6, + sizeof (headers->ip6.dst_address)); headers->udp.dst_port = clib_host_to_net_u16 (UDP_DST_PORT_bfd6); } @@ -1211,7 +1211,7 @@ bfd_udp_input (vlib_main_t * vm, vlib_node_runtime_t * rt, len = (b0->current_length < sizeof (t0->data)) ? b0->current_length : sizeof (t0->data); t0->len = len; - clib_memcpy (t0->data, vlib_buffer_get_current (b0), len); + clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0), len); } /* scan this bfd pkt. error0 is the counter index to bmp */ @@ -1369,7 +1369,7 @@ bfd_udp_echo_input (vlib_main_t * vm, vlib_node_runtime_t * rt, len = (b0->current_length < sizeof (t0->data)) ? b0->current_length : sizeof (t0->data); t0->len = len; - clib_memcpy (t0->data, vlib_buffer_get_current (b0), len); + clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0), len); } bfd_lock (bm); diff --git a/src/vnet/bier/bier_api.c b/src/vnet/bier/bier_api.c index b4bde3a97eb..ed1e35f5b24 100644 --- a/src/vnet/bier/bier_api.c +++ b/src/vnet/bier/bier_api.c @@ -505,13 +505,13 @@ vl_api_bier_disp_entry_add_del_t_handler (vl_api_bier_disp_entry_add_del_t * mp) if (0 == mp->bde_paths[ii].afi) { - clib_memcpy (&brp->frp_addr.ip4, + clib_memcpy_fast (&brp->frp_addr.ip4, mp->bde_paths[ii].next_hop, sizeof (brp->frp_addr.ip4)); } else { - clib_memcpy (&brp->frp_addr.ip6, + clib_memcpy_fast (&brp->frp_addr.ip6, mp->bde_paths[ii].next_hop, sizeof (brp->frp_addr.ip6)); } diff --git a/src/vnet/bier/bier_imp.c b/src/vnet/bier/bier_imp.c index 87a8f909a85..6150ea1c067 100644 --- a/src/vnet/bier/bier_imp.c +++ b/src/vnet/bier/bier_imp.c @@ -80,7 +80,7 @@ bier_imp_add_or_lock (const bier_table_id_t *bti, 0, // entropy sender); bier_hdr_hton(&bi->bi_hdr); - clib_memcpy(&bi->bi_bits, bs->bbs_buckets, bs->bbs_len); + clib_memcpy_fast(&bi->bi_bits, bs->bbs_buckets, bs->bbs_len); bier_imp_lock_i(bi); diff --git a/src/vnet/bier/bier_imp_node.c b/src/vnet/bier/bier_imp_node.c index 0f1820b4028..dcfcc1a1982 100644 --- a/src/vnet/bier/bier_imp_node.c +++ b/src/vnet/bier/bier_imp_node.c @@ -128,7 +128,7 @@ bier_imp_dpo_inline (vlib_main_t * vm, } else { - clib_memcpy(hdr0, &bimp0->bi_hdr, + clib_memcpy_fast(hdr0, &bimp0->bi_hdr, (sizeof(bier_hdr_t) + bier_hdr_len_id_to_num_bytes(bimp0->bi_tbl.bti_hdr_len))); /* diff --git a/src/vnet/bonding/device.c b/src/vnet/bonding/device.c index e7d6cdad407..2bbb90d20f9 100644 --- a/src/vnet/bonding/device.c +++ b/src/vnet/bonding/device.c @@ -745,8 +745,8 @@ done: f = vnet_get_frame_to_sw_interface (vnm, sw_if_index); f->n_vectors = ptd->per_port_queue[p].n_buffers; to_next = vlib_frame_vector_args (f); - clib_memcpy (to_next, ptd->per_port_queue[p].buffers, - f->n_vectors * sizeof (u32)); + clib_memcpy_fast (to_next, ptd->per_port_queue[p].buffers, + f->n_vectors * sizeof (u32)); vnet_put_frame_to_sw_interface (vnm, sw_if_index, f); ptd->per_port_queue[p].n_buffers = 0; } diff --git a/src/vnet/bonding/node.c b/src/vnet/bonding/node.c index 361509c549d..2bc2dc6f265 100644 --- a/src/vnet/bonding/node.c +++ b/src/vnet/bonding/node.c @@ -334,8 +334,8 @@ VLIB_NODE_FN (bond_input_node) (vlib_main_t * vm, { t0 = vlib_add_trace (vm, node, b[0], sizeof (*t0)); t0->sw_if_index = sw_if_index[0]; - clib_memcpy (&t0->ethernet, vlib_buffer_get_current (b[0]), - sizeof (ethernet_header_t)); + clib_memcpy_fast (&t0->ethernet, vlib_buffer_get_current (b[0]), + sizeof (ethernet_header_t)); t0->bond_sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_RX]; } /* next */ diff --git a/src/vnet/classify/vnet_classify.c b/src/vnet/classify/vnet_classify.c index 36f967a38ac..4e822226aaa 100644 --- a/src/vnet/classify/vnet_classify.c +++ b/src/vnet/classify/vnet_classify.c @@ -131,7 +131,7 @@ vnet_classify_new_table (vnet_classify_main_t * cm, clib_memset (t, 0, sizeof (*t)); vec_validate_aligned (t->mask, match_n_vectors - 1, sizeof (u32x4)); - clib_memcpy (t->mask, mask, match_n_vectors * sizeof (u32x4)); + clib_memcpy_fast (t->mask, mask, match_n_vectors * sizeof (u32x4)); t->next_table_index = ~0; t->nbuckets = nbuckets; @@ -273,7 +273,7 @@ static inline void make_working_copy v = vnet_classify_get_entry (t, b->offset); - clib_memcpy (working_copy, v, required_length); + clib_memcpy_fast (working_copy, v, required_length); working_bucket.as_u64 = b->as_u64; working_bucket.offset = vnet_classify_get_offset (t, working_copy); @@ -317,8 +317,8 @@ split_and_rehash (vnet_classify_table_t * t, if (vnet_classify_entry_is_free (new_v)) { - clib_memcpy (new_v, v, sizeof (vnet_classify_entry_t) - + (t->match_n_vectors * sizeof (u32x4))); + clib_memcpy_fast (new_v, v, sizeof (vnet_classify_entry_t) + + (t->match_n_vectors * sizeof (u32x4))); new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE); goto doublebreak; } @@ -361,8 +361,8 @@ split_and_rehash_linear (vnet_classify_table_t * t, clib_warning ("BUG: linear rehash new entry not free!"); continue; } - clib_memcpy (new_v, v, sizeof (vnet_classify_entry_t) - + (t->match_n_vectors * sizeof (u32x4))); + clib_memcpy_fast (new_v, v, sizeof (vnet_classify_entry_t) + + (t->match_n_vectors * sizeof (u32x4))); new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE); j++; goto doublebreak; @@ -457,8 +457,8 @@ vnet_classify_add_del (vnet_classify_table_t * t, } v = vnet_classify_entry_alloc (t, 0 /* new_log2_pages */ ); - clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) + - t->match_n_vectors * sizeof (u32x4)); + clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) + + t->match_n_vectors * sizeof (u32x4)); v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE); vnet_classify_entry_claim_resource (v); @@ -496,8 +496,8 @@ vnet_classify_add_del (vnet_classify_table_t * t, if (!memcmp (v->key, add_v->key, t->match_n_vectors * sizeof (u32x4))) { - clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) + - t->match_n_vectors * sizeof (u32x4)); + clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) + + t->match_n_vectors * sizeof (u32x4)); v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE); vnet_classify_entry_claim_resource (v); @@ -513,8 +513,8 @@ vnet_classify_add_del (vnet_classify_table_t * t, if (vnet_classify_entry_is_free (v)) { - clib_memcpy (v, add_v, sizeof (vnet_classify_entry_t) + - t->match_n_vectors * sizeof (u32x4)); + clib_memcpy_fast (v, add_v, sizeof (vnet_classify_entry_t) + + t->match_n_vectors * sizeof (u32x4)); v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE); vnet_classify_entry_claim_resource (v); @@ -609,8 +609,8 @@ vnet_classify_add_del (vnet_classify_table_t * t, if (vnet_classify_entry_is_free (new_v)) { - clib_memcpy (new_v, add_v, sizeof (vnet_classify_entry_t) + - t->match_n_vectors * sizeof (u32x4)); + clib_memcpy_fast (new_v, add_v, sizeof (vnet_classify_entry_t) + + t->match_n_vectors * sizeof (u32x4)); new_v->flags &= ~(VNET_CLASSIFY_ENTRY_FREE); vnet_classify_entry_claim_resource (new_v); @@ -1885,10 +1885,10 @@ unformat_ip6_match (unformat_input_t * input, va_list * args) ip = (ip6_header_t *) match; if (src) - clib_memcpy (&ip->src_address, &src_val, sizeof (ip->src_address)); + clib_memcpy_fast (&ip->src_address, &src_val, sizeof (ip->src_address)); if (dst) - clib_memcpy (&ip->dst_address, &dst_val, sizeof (ip->dst_address)); + clib_memcpy_fast (&ip->dst_address, &dst_val, sizeof (ip->dst_address)); if (proto) ip->protocol = proto_val; @@ -2012,10 +2012,10 @@ unformat_l2_match (unformat_input_t * input, va_list * args) vec_validate_aligned (match, len - 1, sizeof (u32x4)); if (dst) - clib_memcpy (match, dst_val, 6); + clib_memcpy_fast (match, dst_val, 6); if (src) - clib_memcpy (match + 6, src_val, 6); + clib_memcpy_fast (match + 6, src_val, 6); if (tag2) { @@ -2188,8 +2188,8 @@ vnet_classify_add_del_session (vnet_classify_main_t * cm, e->metadata = 0; /* Copy key data, honoring skip_n_vectors */ - clib_memcpy (&e->key, match + t->skip_n_vectors * sizeof (u32x4), - t->match_n_vectors * sizeof (u32x4)); + clib_memcpy_fast (&e->key, match + t->skip_n_vectors * sizeof (u32x4), + t->match_n_vectors * sizeof (u32x4)); /* Clear don't-care bits; likely when dynamically creating sessions */ for (i = 0; i < t->match_n_vectors; i++) diff --git a/src/vnet/devices/af_packet/device.c b/src/vnet/devices/af_packet/device.c index 811642ec7b9..9d08c620d94 100644 --- a/src/vnet/devices/af_packet/device.c +++ b/src/vnet/devices/af_packet/device.c @@ -119,9 +119,9 @@ af_packet_interface_tx (vlib_main_t * vm, { b0 = vlib_get_buffer (vm, bi); len = b0->current_length; - clib_memcpy ((u8 *) tph + - TPACKET_ALIGN (sizeof (struct tpacket2_hdr)) + offset, - vlib_buffer_get_current (b0), len); + clib_memcpy_fast ((u8 *) tph + + TPACKET_ALIGN (sizeof (struct tpacket2_hdr)) + + offset, vlib_buffer_get_current (b0), len); offset += len; } while ((bi = diff --git a/src/vnet/devices/af_packet/node.c b/src/vnet/devices/af_packet/node.c index 8dbd8554f01..97aa170c0a6 100644 --- a/src/vnet/devices/af_packet/node.c +++ b/src/vnet/devices/af_packet/node.c @@ -248,9 +248,9 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, { if (PREDICT_TRUE (offset == 0)) { - clib_memcpy (vlib_buffer_get_current (b0), - (u8 *) tph + tph->tp_mac, - sizeof (ethernet_header_t)); + clib_memcpy_fast (vlib_buffer_get_current (b0), + (u8 *) tph + tph->tp_mac, + sizeof (ethernet_header_t)); ethernet_header_t *eth = vlib_buffer_get_current (b0); ethernet_vlan_header_t *vlan = (ethernet_vlan_header_t *) (eth + 1); @@ -262,10 +262,10 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, bytes_copied = sizeof (ethernet_header_t); } } - clib_memcpy (((u8 *) vlib_buffer_get_current (b0)) + - bytes_copied + vlan_len, - (u8 *) tph + tph->tp_mac + offset + bytes_copied, - (bytes_to_copy - bytes_copied)); + clib_memcpy_fast (((u8 *) vlib_buffer_get_current (b0)) + + bytes_copied + vlan_len, + (u8 *) tph + tph->tp_mac + offset + + bytes_copied, (bytes_to_copy - bytes_copied)); /* fill buffer header */ b0->current_length = bytes_to_copy + vlan_len; @@ -319,7 +319,7 @@ af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, tr = vlib_add_trace (vm, node, first_b0, sizeof (*tr)); tr->next_index = next0; tr->hw_if_index = apif->hw_if_index; - clib_memcpy (&tr->tph, tph, sizeof (struct tpacket2_hdr)); + clib_memcpy_fast (&tr->tph, tph, sizeof (struct tpacket2_hdr)); } /* enque and take next packet */ diff --git a/src/vnet/devices/netmap/device.c b/src/vnet/devices/netmap/device.c index 409a52f1005..b14bff43c06 100644 --- a/src/vnet/devices/netmap/device.c +++ b/src/vnet/devices/netmap/device.c @@ -143,8 +143,8 @@ netmap_interface_tx (vlib_main_t * vm, b0 = vlib_get_buffer (vm, bi); len = b0->current_length; /* memcpy */ - clib_memcpy ((u8 *) NETMAP_BUF (ring, slot->buf_idx) + offset, - vlib_buffer_get_current (b0), len); + clib_memcpy_fast ((u8 *) NETMAP_BUF (ring, slot->buf_idx) + + offset, vlib_buffer_get_current (b0), len); offset += len; } while ((bi = b0->next_buffer)); diff --git a/src/vnet/devices/netmap/node.c b/src/vnet/devices/netmap/node.c index e541700f4ea..dde706f20c7 100644 --- a/src/vnet/devices/netmap/node.c +++ b/src/vnet/devices/netmap/node.c @@ -174,10 +174,9 @@ netmap_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, u32 bytes_to_copy = data_len > n_buffer_bytes ? n_buffer_bytes : data_len; b0->current_data = 0; - clib_memcpy (vlib_buffer_get_current (b0), - (u8 *) NETMAP_BUF (ring, - slot->buf_idx) + offset, - bytes_to_copy); + clib_memcpy_fast (vlib_buffer_get_current (b0), + (u8 *) NETMAP_BUF (ring, slot->buf_idx) + + offset, bytes_to_copy); /* fill buffer header */ b0->current_length = bytes_to_copy; diff --git a/src/vnet/devices/pipe/pipe.c b/src/vnet/devices/pipe/pipe.c index f45c08863a3..c6d3f659eae 100644 --- a/src/vnet/devices/pipe/pipe.c +++ b/src/vnet/devices/pipe/pipe.c @@ -148,7 +148,7 @@ pipe_tx (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) n_copy = clib_min (n_left_from, n_left_to_next); - clib_memcpy (to_next, from, n_copy * sizeof (from[0])); + clib_memcpy_fast (to_next, from, n_copy * sizeof (from[0])); n_left_to_next -= n_copy; n_left_from -= n_copy; i = 0; diff --git a/src/vnet/devices/virtio/node.c b/src/vnet/devices/virtio/node.c index 339c48c93f5..e0195139321 100644 --- a/src/vnet/devices/virtio/node.c +++ b/src/vnet/devices/virtio/node.c @@ -216,7 +216,7 @@ virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tr->next_index = next0; tr->hw_if_index = vif->hw_if_index; tr->len = len; - clib_memcpy (&tr->hdr, hdr, hdr_sz); + clib_memcpy_fast (&tr->hdr, hdr, hdr_sz); } /* enqueue buffer */ diff --git a/src/vnet/devices/virtio/vhost_user.c b/src/vnet/devices/virtio/vhost_user.c index 9725a91027d..d13ea3beab3 100644 --- a/src/vnet/devices/virtio/vhost_user.c +++ b/src/vnet/devices/virtio/vhost_user.c @@ -443,7 +443,7 @@ vhost_user_socket_read (clib_file_t * uf) VHOST_MEMORY_MAX_NREGIONS * sizeof (int))) { number_of_fds = (cmsg->cmsg_len - CMSG_LEN (0)) / sizeof (int); - clib_memcpy (fds, CMSG_DATA (cmsg), number_of_fds * sizeof (int)); + clib_memcpy_fast (fds, CMSG_DATA (cmsg), number_of_fds * sizeof (int)); } /* version 1, no reply bit set */ @@ -537,8 +537,8 @@ vhost_user_socket_read (clib_file_t * uf) unmap_all_mem_regions (vui); for (i = 0; i < msg.memory.nregions; i++) { - clib_memcpy (&(vui->regions[i]), &msg.memory.regions[i], - sizeof (vhost_user_memory_region_t)); + clib_memcpy_fast (&(vui->regions[i]), &msg.memory.regions[i], + sizeof (vhost_user_memory_region_t)); long page_sz = get_huge_page_size (fds[i]); diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c index 8e88695957b..32b924f1362 100644 --- a/src/vnet/devices/virtio/vhost_user_input.c +++ b/src/vnet/devices/virtio/vhost_user_input.c @@ -162,8 +162,8 @@ vhost_user_input_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, CLIB_PREFETCH (src2, 64, LOAD); CLIB_PREFETCH (src3, 64, LOAD); - clib_memcpy ((void *) cpy[0].dst, src0, cpy[0].len); - clib_memcpy ((void *) cpy[1].dst, src1, cpy[1].len); + clib_memcpy_fast ((void *) cpy[0].dst, src0, cpy[0].len); + clib_memcpy_fast ((void *) cpy[1].dst, src1, cpy[1].len); copy_len -= 2; cpy += 2; } @@ -172,7 +172,7 @@ vhost_user_input_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, { if (PREDICT_FALSE (!(src0 = map_guest_mem (vui, cpy->src, map_hint)))) return 1; - clib_memcpy ((void *) cpy->dst, src0, cpy->len); + clib_memcpy_fast ((void *) cpy->dst, src0, cpy->len); copy_len -= 1; cpy += 1; } diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c index dff129e763e..58db1a9647b 100644 --- a/src/vnet/devices/virtio/vhost_user_output.c +++ b/src/vnet/devices/virtio/vhost_user_output.c @@ -205,8 +205,8 @@ vhost_user_tx_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, CLIB_PREFETCH ((void *) cpy[2].src, 64, LOAD); CLIB_PREFETCH ((void *) cpy[3].src, 64, LOAD); - clib_memcpy (dst0, (void *) cpy[0].src, cpy[0].len); - clib_memcpy (dst1, (void *) cpy[1].src, cpy[1].len); + clib_memcpy_fast (dst0, (void *) cpy[0].src, cpy[0].len); + clib_memcpy_fast (dst1, (void *) cpy[1].src, cpy[1].len); vhost_user_log_dirty_pages_2 (vui, cpy[0].dst, cpy[0].len, 1); vhost_user_log_dirty_pages_2 (vui, cpy[1].dst, cpy[1].len, 1); @@ -218,7 +218,7 @@ vhost_user_tx_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, { if (PREDICT_FALSE (!(dst0 = map_guest_mem (vui, cpy->dst, map_hint)))) return 1; - clib_memcpy (dst0, (void *) cpy->src, cpy->len); + clib_memcpy_fast (dst0, (void *) cpy->src, cpy->len); vhost_user_log_dirty_pages_2 (vui, cpy->dst, cpy->len, 1); copy_len -= 1; cpy += 1; diff --git a/src/vnet/dns/reply_node.c b/src/vnet/dns/reply_node.c index 5681e11d8e2..81ce8f480d0 100644 --- a/src/vnet/dns/reply_node.c +++ b/src/vnet/dns/reply_node.c @@ -166,7 +166,7 @@ dns46_reply_node_fn (vlib_main_t * vm, /* Save the reply */ vec_validate (resp0, vlib_buffer_length_in_chain (vm, b0) - 1); - clib_memcpy (resp0, d0, vlib_buffer_length_in_chain (vm, b0)); + clib_memcpy_fast (resp0, d0, vlib_buffer_length_in_chain (vm, b0)); /* * Deal with everything in process ctx on the main thread diff --git a/src/vnet/dns/request_node.c b/src/vnet/dns/request_node.c index b91d2997dfb..f9ad7cc0ad8 100644 --- a/src/vnet/dns/request_node.c +++ b/src/vnet/dns/request_node.c @@ -236,11 +236,11 @@ dns46_request_inline (vlib_main_t * vm, t0->id = d0->id; t0->name = name0; if (is_ip6) - clib_memcpy (t0->dst_address, ip60->src_address.as_u8, - sizeof (ip6_address_t)); + clib_memcpy_fast (t0->dst_address, ip60->src_address.as_u8, + sizeof (ip6_address_t)); else - clib_memcpy (t0->dst_address, ip40->src_address.as_u8, - sizeof (ip4_address_t)); + clib_memcpy_fast (t0->dst_address, ip40->src_address.as_u8, + sizeof (ip4_address_t)); vnet_dns_resolve_name (dm, name0, t0, &ep0); diff --git a/src/vnet/dpo/load_balance.c b/src/vnet/dpo/load_balance.c index 1abfab1cb01..939e25c08fa 100644 --- a/src/vnet/dpo/load_balance.c +++ b/src/vnet/dpo/load_balance.c @@ -383,7 +383,7 @@ ip_multipath_normalize_next_hops (const load_balance_path_t * raw_next_hops, } else { - clib_memcpy (nhs, raw_next_hops, n_nhs * sizeof (raw_next_hops[0])); + clib_memcpy_fast (nhs, raw_next_hops, n_nhs * sizeof (raw_next_hops[0])); qsort (nhs, n_nhs, sizeof (nhs[0]), (void *) next_hop_sort_by_weight); } diff --git a/src/vnet/dpo/mpls_label_dpo.c b/src/vnet/dpo/mpls_label_dpo.c index c3710aefe28..ab8c06df4d5 100644 --- a/src/vnet/dpo/mpls_label_dpo.c +++ b/src/vnet/dpo/mpls_label_dpo.c @@ -287,7 +287,7 @@ mpls_label_paint (vlib_buffer_t * b0, } else { - clib_memcpy(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes); + clib_memcpy_fast(hdr0, mld0->mld_hdr, mld0->mld_n_hdr_bytes); hdr0 = hdr0 + (mld0->mld_n_labels - 1); } @@ -1218,7 +1218,7 @@ mpls_label_interpose (const dpo_id_t *original, mld = mpls_label_dpo_get(original->dpoi_index); mld_clone->mld_locks = 0; - clib_memcpy(&mld_clone->mld_hdr, + clib_memcpy_fast(&mld_clone->mld_hdr, &mld->mld_hdr, sizeof(mld_clone->mld_hdr)); mld_clone->mld_payload_proto = mld->mld_payload_proto; diff --git a/src/vnet/ethernet/arp.c b/src/vnet/ethernet/arp.c index b6b7aba731d..549e05d889c 100644 --- a/src/vnet/ethernet/arp.c +++ b/src/vnet/ethernet/arp.c @@ -358,8 +358,9 @@ arp_nbr_probe (ip_adjacency_t * adj) hi = vnet_get_sup_hw_interface (vnm, adj->rewrite_header.sw_if_index); - clib_memcpy (h->ip4_over_ethernet[0].ethernet, - hi->hw_address, sizeof (h->ip4_over_ethernet[0].ethernet)); + clib_memcpy_fast (h->ip4_over_ethernet[0].ethernet, + hi->hw_address, + sizeof (h->ip4_over_ethernet[0].ethernet)); h->ip4_over_ethernet[0].ip4 = src[0]; h->ip4_over_ethernet[1].ip4 = adj->sub_type.nbr.next_hop.ip4; @@ -674,8 +675,8 @@ vnet_arp_set_ip4_over_ethernet_internal (vnet_main_t * vnm, e->sw_if_index = sw_if_index; e->ip4_address = a->ip4; e->fib_entry_index = FIB_NODE_INDEX_INVALID; - clib_memcpy (e->ethernet_address, - a->ethernet, sizeof (e->ethernet_address)); + clib_memcpy_fast (e->ethernet_address, + a->ethernet, sizeof (e->ethernet_address)); if (!is_no_fib_entry) { @@ -702,8 +703,8 @@ vnet_arp_set_ip4_over_ethernet_internal (vnet_main_t * vnm, } /* Update ethernet address. */ - clib_memcpy (e->ethernet_address, a->ethernet, - sizeof (e->ethernet_address)); + clib_memcpy_fast (e->ethernet_address, a->ethernet, + sizeof (e->ethernet_address)); } /* Update time stamp and flags. */ @@ -1238,8 +1239,8 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) arp0->ip4_over_ethernet[1] = arp0->ip4_over_ethernet[0]; - clib_memcpy (arp0->ip4_over_ethernet[0].ethernet, - hw_if0->hw_address, 6); + clib_memcpy_fast (arp0->ip4_over_ethernet[0].ethernet, + hw_if0->hw_address, 6); clib_mem_unaligned (&arp0->ip4_over_ethernet[0].ip4.data_u32, u32) = if_addr0->data_u32; @@ -1249,7 +1250,7 @@ arp_input (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) /* the rx nd tx ethernet headers wil overlap in the case * when we received a tagged VLAN=0 packet, but we are sending * back untagged */ - clib_memcpy (eth_tx, rewrite0, vec_len (rewrite0)); + clib_memcpy_fast (eth_tx, rewrite0, vec_len (rewrite0)); vec_free (rewrite0); if (NULL == pa) @@ -1557,7 +1558,7 @@ vnet_arp_unset_ip4_over_ethernet (vnet_main_t * vnm, args.sw_if_index = sw_if_index; args.flags = ETHERNET_ARP_ARGS_REMOVE; - clib_memcpy (&args.a, a, sizeof (*a)); + clib_memcpy_fast (&args.a, a, sizeof (*a)); vl_api_rpc_call_main_thread (set_ip4_over_ethernet_rpc_callback, (u8 *) & args, sizeof (args)); @@ -1706,7 +1707,7 @@ arp_add_del_interface_address (ip4_main_t * im, .sw_if_index = e->sw_if_index, .flags = ETHERNET_ARP_ARGS_FLUSH, }; - clib_memcpy (&delme.a.ethernet, e->ethernet_address, 6); + clib_memcpy_fast (&delme.a.ethernet, e->ethernet_address, 6); vnet_arp_flush_ip4_over_ethernet_internal (vnet_get_main (), &delme); @@ -1915,7 +1916,7 @@ ethernet_arp_sw_interface_up_down (vnet_main_t * vnm, .sw_if_index = e->sw_if_index, }; - clib_memcpy (&update_me.a.ethernet, e->ethernet_address, 6); + clib_memcpy_fast (&update_me.a.ethernet, e->ethernet_address, 6); if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) { @@ -1970,7 +1971,7 @@ vnet_arp_set_ip4_over_ethernet (vnet_main_t * vnm, args.is_static = is_static; args.is_no_fib_entry = is_no_fib_entry; args.flags = 0; - clib_memcpy (&args.a, a, sizeof (*a)); + clib_memcpy_fast (&args.a, a, sizeof (*a)); vl_api_rpc_call_main_thread (set_ip4_over_ethernet_rpc_callback, (u8 *) & args, sizeof (args)); @@ -2344,7 +2345,8 @@ arp_term_l2bd (vlib_main_t * vm, { u8 *t0 = vlib_add_trace (vm, node, p0, sizeof (ethernet_arp_input_trace_t)); - clib_memcpy (t0, l3h0, sizeof (ethernet_arp_input_trace_t)); + clib_memcpy_fast (t0, l3h0, + sizeof (ethernet_arp_input_trace_t)); } error0 = 0; @@ -2410,9 +2412,9 @@ arp_term_l2bd (vlib_main_t * vm, arp0->opcode = clib_host_to_net_u16 (ETHERNET_ARP_OPCODE_reply); arp0->ip4_over_ethernet[1] = arp0->ip4_over_ethernet[0]; arp0->ip4_over_ethernet[0].ip4.as_u32 = ip0; - clib_memcpy (arp0->ip4_over_ethernet[0].ethernet, macp0, 6); - clib_memcpy (eth0->dst_address, eth0->src_address, 6); - clib_memcpy (eth0->src_address, macp0, 6); + clib_memcpy_fast (arp0->ip4_over_ethernet[0].ethernet, macp0, 6); + clib_memcpy_fast (eth0->dst_address, eth0->src_address, 6); + clib_memcpy_fast (eth0->src_address, macp0, 6); n_replies_sent += 1; output_response: @@ -2577,10 +2579,10 @@ send_ip4_garp_w_addr (vlib_main_t * vm, if (!h) return; - clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address, - sizeof (h->ip4_over_ethernet[0].ethernet)); - clib_memcpy (h->ip4_over_ethernet[1].ethernet, hi->hw_address, - sizeof (h->ip4_over_ethernet[1].ethernet)); + clib_memcpy_fast (h->ip4_over_ethernet[0].ethernet, hi->hw_address, + sizeof (h->ip4_over_ethernet[0].ethernet)); + clib_memcpy_fast (h->ip4_over_ethernet[1].ethernet, hi->hw_address, + sizeof (h->ip4_over_ethernet[1].ethernet)); h->ip4_over_ethernet[0].ip4 = ip4_addr[0]; h->ip4_over_ethernet[1].ip4 = ip4_addr[0]; @@ -2592,7 +2594,7 @@ send_ip4_garp_w_addr (vlib_main_t * vm, rewrite_len = vec_len (rewrite); vlib_buffer_advance (b, -rewrite_len); ethernet_header_t *e = vlib_buffer_get_current (b); - clib_memcpy (e->dst_address, rewrite, rewrite_len); + clib_memcpy_fast (e->dst_address, rewrite, rewrite_len); vec_free (rewrite); /* Send GARP packet out the specified interface */ diff --git a/src/vnet/ethernet/ethernet_types_api.c b/src/vnet/ethernet/ethernet_types_api.c index 67e6231cfc8..4b84d386e4e 100644 --- a/src/vnet/ethernet/ethernet_types_api.c +++ b/src/vnet/ethernet/ethernet_types_api.c @@ -38,7 +38,7 @@ mac_address_decode (const vl_api_mac_address_t * in, mac_address_t * out) void mac_address_encode (const mac_address_t * in, vl_api_mac_address_t * out) { - clib_memcpy (out->bytes, in->bytes, 6); + clib_memcpy_fast (out->bytes, in->bytes, 6); } /* diff --git a/src/vnet/flow/flow.c b/src/vnet/flow/flow.c index 485a845e5ee..4d867fee365 100644 --- a/src/vnet/flow/flow.c +++ b/src/vnet/flow/flow.c @@ -47,7 +47,7 @@ vnet_flow_add (vnet_main_t * vnm, vnet_flow_t * flow, u32 * flow_index) pool_get (fm->global_flow_pool, f); *flow_index = f - fm->global_flow_pool; - clib_memcpy (f, flow, sizeof (vnet_flow_t)); + clib_memcpy_fast (f, flow, sizeof (vnet_flow_t)); f->private_data = 0; f->index = *flow_index; return 0; diff --git a/src/vnet/geneve/decap.c b/src/vnet/geneve/decap.c index 93f2a8548c1..ceb04447817 100644 --- a/src/vnet/geneve/decap.c +++ b/src/vnet/geneve/decap.c @@ -286,7 +286,7 @@ geneve_input (vlib_main_t * vm, next0 = GENEVE_INPUT_NEXT_DROP; goto trace0; } - clib_memcpy (&last_key6, &key6_0, sizeof (key6_0)); + clib_memcpy_fast (&last_key6, &key6_0, sizeof (key6_0)); tunnel_index0 = last_tunnel_index = p0[0]; } else @@ -455,7 +455,7 @@ geneve_input (vlib_main_t * vm, goto trace1; } - clib_memcpy (&last_key6, &key6_1, sizeof (key6_1)); + clib_memcpy_fast (&last_key6, &key6_1, sizeof (key6_1)); tunnel_index1 = last_tunnel_index = p1[0]; } else @@ -690,7 +690,7 @@ geneve_input (vlib_main_t * vm, next0 = GENEVE_INPUT_NEXT_DROP; goto trace00; } - clib_memcpy (&last_key6, &key6_0, sizeof (key6_0)); + clib_memcpy_fast (&last_key6, &key6_0, sizeof (key6_0)); tunnel_index0 = last_tunnel_index = p0[0]; } else diff --git a/src/vnet/interface_output.c b/src/vnet/interface_output.c index be4ab7d291a..b09f7996e18 100644 --- a/src/vnet/interface_output.c +++ b/src/vnet/interface_output.c @@ -121,15 +121,15 @@ vnet_interface_output_trace (vlib_main_t * vm, { t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0])); t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX]; - clib_memcpy (t0->data, vlib_buffer_get_current (b0), - sizeof (t0->data)); + clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0), + sizeof (t0->data)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0])); t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_TX]; - clib_memcpy (t1->data, vlib_buffer_get_current (b1), - sizeof (t1->data)); + clib_memcpy_fast (t1->data, vlib_buffer_get_current (b1), + sizeof (t1->data)); } from += 2; n_left -= 2; @@ -149,8 +149,8 @@ vnet_interface_output_trace (vlib_main_t * vm, { t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0])); t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX]; - clib_memcpy (t0->data, vlib_buffer_get_current (b0), - sizeof (t0->data)); + clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0), + sizeof (t0->data)); } from += 1; n_left -= 1; diff --git a/src/vnet/ip/ip.c b/src/vnet/ip/ip.c index 144bbcc405d..cab0cd0315a 100644 --- a/src/vnet/ip/ip.c +++ b/src/vnet/ip/ip.c @@ -70,7 +70,7 @@ ip_is_local (u32 fib_index, ip46_address_t * ip46_address, u8 is_ip4) prefix.fp_proto = FIB_PROTOCOL_IP6; } - clib_memcpy (&prefix.fp_addr, ip46_address, sizeof (ip46_address_t)); + clib_memcpy_fast (&prefix.fp_addr, ip46_address, sizeof (ip46_address_t)); fei = fib_table_lookup (fib_index, &prefix); flags = fib_entry_get_flags (fei); @@ -83,7 +83,7 @@ ip_copy (ip46_address_t * dst, ip46_address_t * src, u8 is_ip4) if (is_ip4) dst->ip4.as_u32 = src->ip4.as_u32; else - clib_memcpy (&dst->ip6, &src->ip6, sizeof (ip6_address_t)); + clib_memcpy_fast (&dst->ip6, &src->ip6, sizeof (ip6_address_t)); } void @@ -92,7 +92,8 @@ ip_set (ip46_address_t * dst, void *src, u8 is_ip4) if (is_ip4) dst->ip4.as_u32 = ((ip4_address_t *) src)->as_u32; else - clib_memcpy (&dst->ip6, (ip6_address_t *) src, sizeof (ip6_address_t)); + clib_memcpy_fast (&dst->ip6, (ip6_address_t *) src, + sizeof (ip6_address_t)); } u8 diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c index 9911eb71b99..e011e5134d8 100644 --- a/src/vnet/ip/ip4_forward.c +++ b/src/vnet/ip/ip4_forward.c @@ -1072,9 +1072,9 @@ ip4_forward_next_trace (vlib_main_t * vm, vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b0)->sw_if_index[VLIB_RX]); - clib_memcpy (t0->packet_data, - vlib_buffer_get_current (b0), - sizeof (t0->packet_data)); + clib_memcpy_fast (t0->packet_data, + vlib_buffer_get_current (b0), + sizeof (t0->packet_data)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -1086,8 +1086,8 @@ ip4_forward_next_trace (vlib_main_t * vm, (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] : vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b1)->sw_if_index[VLIB_RX]); - clib_memcpy (t1->packet_data, vlib_buffer_get_current (b1), - sizeof (t1->packet_data)); + clib_memcpy_fast (t1->packet_data, vlib_buffer_get_current (b1), + sizeof (t1->packet_data)); } from += 2; n_left -= 2; @@ -1113,8 +1113,8 @@ ip4_forward_next_trace (vlib_main_t * vm, (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] : vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b0)->sw_if_index[VLIB_RX]); - clib_memcpy (t0->packet_data, vlib_buffer_get_current (b0), - sizeof (t0->packet_data)); + clib_memcpy_fast (t0->packet_data, vlib_buffer_get_current (b0), + sizeof (t0->packet_data)); } from += 1; n_left -= 1; @@ -1852,9 +1852,9 @@ ip4_arp_inline (vlib_main_t * vm, hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); /* Src ethernet address in ARP header. */ - clib_memcpy (h0->ip4_over_ethernet[0].ethernet, - hw_if0->hw_address, - sizeof (h0->ip4_over_ethernet[0].ethernet)); + clib_memcpy_fast (h0->ip4_over_ethernet[0].ethernet, + hw_if0->hw_address, + sizeof (h0->ip4_over_ethernet[0].ethernet)); if (is_glean) { /* The interface's source address is stashed in the Glean Adj */ @@ -2027,8 +2027,8 @@ ip4_probe_neighbor (vlib_main_t * vm, ip4_address_t * dst, u32 sw_if_index, sw_if_index); } - clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address, - sizeof (h->ip4_over_ethernet[0].ethernet)); + clib_memcpy_fast (h->ip4_over_ethernet[0].ethernet, hi->hw_address, + sizeof (h->ip4_over_ethernet[0].ethernet)); h->ip4_over_ethernet[0].ip4 = src[0]; h->ip4_over_ethernet[1].ip4 = dst[0]; diff --git a/src/vnet/ip/ip4_options.c b/src/vnet/ip/ip4_options.c index c008b9bde8c..1b5a7878512 100644 --- a/src/vnet/ip/ip4_options.c +++ b/src/vnet/ip/ip4_options.c @@ -94,7 +94,7 @@ VLIB_NODE_FN (ip4_options_node) (vlib_main_t * vm, ip4_options_trace_t *t = vlib_add_trace (vm, node, b, sizeof (*t)); - clib_memcpy (t->option, options, 4); + clib_memcpy_fast (t->option, options, 4); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi, next); diff --git a/src/vnet/ip/ip4_packet.h b/src/vnet/ip/ip4_packet.h index fa86229a073..2ce6763f8d7 100644 --- a/src/vnet/ip/ip4_packet.h +++ b/src/vnet/ip/ip4_packet.h @@ -67,7 +67,8 @@ always_inline void ip4_addr_fib_init (ip4_address_fib_t * addr_fib, const ip4_address_t * address, u32 fib_index) { - clib_memcpy (&addr_fib->ip4_addr, address, sizeof (addr_fib->ip4_addr)); + clib_memcpy_fast (&addr_fib->ip4_addr, address, + sizeof (addr_fib->ip4_addr)); addr_fib->fib_index = fib_index; } diff --git a/src/vnet/ip/ip4_reassembly.c b/src/vnet/ip/ip4_reassembly.c index 42231d0e30d..bd33026db85 100644 --- a/src/vnet/ip/ip4_reassembly.c +++ b/src/vnet/ip/ip4_reassembly.c @@ -1125,8 +1125,8 @@ ip4_reass_set (u32 timeout_ms, u32 max_reassemblies, else { clib_bihash_free_16_8 (&ip4_reass_main.hash); - clib_memcpy (&ip4_reass_main.hash, &new_hash, - sizeof (ip4_reass_main.hash)); + clib_memcpy_fast (&ip4_reass_main.hash, &new_hash, + sizeof (ip4_reass_main.hash)); } } return 0; diff --git a/src/vnet/ip/ip6.h b/src/vnet/ip/ip6.h index e807886cd2a..aef24454413 100644 --- a/src/vnet/ip/ip6.h +++ b/src/vnet/ip/ip6.h @@ -616,10 +616,10 @@ vlib_buffer_push_ip6 (vlib_main_t * vm, vlib_buffer_t * b, ip6h->hop_limit = 0xff; ip6h->protocol = proto; - clib_memcpy (ip6h->src_address.as_u8, src->as_u8, - sizeof (ip6h->src_address)); - clib_memcpy (ip6h->dst_address.as_u8, dst->as_u8, - sizeof (ip6h->src_address)); + clib_memcpy_fast (ip6h->src_address.as_u8, src->as_u8, + sizeof (ip6h->src_address)); + clib_memcpy_fast (ip6h->dst_address.as_u8, dst->as_u8, + sizeof (ip6h->src_address)); b->flags |= VNET_BUFFER_F_IS_IP6; return ip6h; diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index be0037e5edd..5a176435510 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -883,9 +883,9 @@ ip6_forward_next_trace (vlib_main_t * vm, vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b0)->sw_if_index[VLIB_RX]); - clib_memcpy (t0->packet_data, - vlib_buffer_get_current (b0), - sizeof (t0->packet_data)); + clib_memcpy_fast (t0->packet_data, + vlib_buffer_get_current (b0), + sizeof (t0->packet_data)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -898,9 +898,9 @@ ip6_forward_next_trace (vlib_main_t * vm, vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b1)->sw_if_index[VLIB_RX]); - clib_memcpy (t1->packet_data, - vlib_buffer_get_current (b1), - sizeof (t1->packet_data)); + clib_memcpy_fast (t1->packet_data, + vlib_buffer_get_current (b1), + sizeof (t1->packet_data)); } from += 2; n_left -= 2; @@ -927,9 +927,9 @@ ip6_forward_next_trace (vlib_main_t * vm, vec_elt (im->fib_index_by_sw_if_index, vnet_buffer (b0)->sw_if_index[VLIB_RX]); - clib_memcpy (t0->packet_data, - vlib_buffer_get_current (b0), - sizeof (t0->packet_data)); + clib_memcpy_fast (t0->packet_data, + vlib_buffer_get_current (b0), + sizeof (t0->packet_data)); } from += 1; n_left -= 1; @@ -1530,8 +1530,8 @@ ip6_probe_neighbor (vlib_main_t * vm, ip6_address_t * dst, u32 sw_if_index, sw_if_index); } - clib_memcpy (h->link_layer_option.ethernet_address, hi->hw_address, - vec_len (hi->hw_address)); + clib_memcpy_fast (h->link_layer_option.ethernet_address, hi->hw_address, + vec_len (hi->hw_address)); h->neighbor.icmp.checksum = ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length); @@ -2415,7 +2415,7 @@ ip6_hop_by_hop (vlib_main_t * vm, ARRAY_LEN (t->option_data) ? trace_len : ARRAY_LEN (t->option_data); t->trace_len = trace_len; - clib_memcpy (t->option_data, hbh0, trace_len); + clib_memcpy_fast (t->option_data, hbh0, trace_len); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -2429,7 +2429,7 @@ ip6_hop_by_hop (vlib_main_t * vm, ARRAY_LEN (t->option_data) ? trace_len : ARRAY_LEN (t->option_data); t->trace_len = trace_len; - clib_memcpy (t->option_data, hbh1, trace_len); + clib_memcpy_fast (t->option_data, hbh1, trace_len); } } @@ -2510,7 +2510,7 @@ ip6_hop_by_hop (vlib_main_t * vm, ARRAY_LEN (t->option_data) ? trace_len : ARRAY_LEN (t->option_data); t->trace_len = trace_len; - clib_memcpy (t->option_data, hbh0, trace_len); + clib_memcpy_fast (t->option_data, hbh0, trace_len); } b0->error = error_node->errors[error0]; diff --git a/src/vnet/ip/ip6_hop_by_hop.c b/src/vnet/ip/ip6_hop_by_hop.c index fbaf0007f19..38cdbfffeaa 100644 --- a/src/vnet/ip/ip6_hop_by_hop.c +++ b/src/vnet/ip/ip6_hop_by_hop.c @@ -321,8 +321,8 @@ ip6_add_hop_by_hop_node_fn (vlib_main_t * vm, hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1); hbh1 = (ip6_hop_by_hop_header_t *) (ip1 + 1); /* $$$ tune, rewrite_length is a multiple of 8 */ - clib_memcpy (hbh0, rewrite, rewrite_length); - clib_memcpy (hbh1, rewrite, rewrite_length); + clib_memcpy_fast (hbh0, rewrite, rewrite_length); + clib_memcpy_fast (hbh1, rewrite, rewrite_length); /* Patch the protocol chain, insert the h-b-h (type 0) header */ hbh0->protocol = ip0->protocol; hbh1->protocol = ip1->protocol; @@ -399,7 +399,7 @@ ip6_add_hop_by_hop_node_fn (vlib_main_t * vm, hbh0 = (ip6_hop_by_hop_header_t *) (ip0 + 1); /* $$$ tune, rewrite_length is a multiple of 8 */ - clib_memcpy (hbh0, rewrite, rewrite_length); + clib_memcpy_fast (hbh0, rewrite, rewrite_length); /* Patch the protocol chain, insert the h-b-h (type 0) header */ hbh0->protocol = ip0->protocol; ip0->protocol = 0; diff --git a/src/vnet/ip/ip6_reassembly.c b/src/vnet/ip/ip6_reassembly.c index 7f6c5ca4af7..de9fbe52a75 100644 --- a/src/vnet/ip/ip6_reassembly.c +++ b/src/vnet/ip/ip6_reassembly.c @@ -1165,8 +1165,8 @@ ip6_reass_set (u32 timeout_ms, u32 max_reassemblies, else { clib_bihash_free_48_8 (&ip6_reass_main.hash); - clib_memcpy (&ip6_reass_main.hash, &new_hash, - sizeof (ip6_reass_main.hash)); + clib_memcpy_fast (&ip6_reass_main.hash, &new_hash, + sizeof (ip6_reass_main.hash)); } } return 0; diff --git a/src/vnet/ip/ip6_to_ip4.h b/src/vnet/ip/ip6_to_ip4.h index f25c9e408f0..d13a0c1f104 100644 --- a/src/vnet/ip/ip6_to_ip4.h +++ b/src/vnet/ip/ip6_to_ip4.h @@ -373,7 +373,7 @@ icmp6_to_icmp (vlib_buffer_t * p, ip6_to_ip4_set_fn_t fn, void *ctx, //Move up icmp header ip4 = (ip4_header_t *) u8_ptr_add (inner_l4, -2 * sizeof (*ip4) - 8); - clib_memcpy (u8_ptr_add (inner_l4, -sizeof (*ip4) - 8), icmp, 8); + clib_memcpy_fast (u8_ptr_add (inner_l4, -sizeof (*ip4) - 8), icmp, 8); icmp = (icmp46_header_t *) u8_ptr_add (inner_l4, -sizeof (*ip4) - 8); } else diff --git a/src/vnet/ip/ip_frag.c b/src/vnet/ip/ip_frag.c index 8de4dfc5d58..6efb8834c4b 100644 --- a/src/vnet/ip/ip_frag.c +++ b/src/vnet/ip/ip_frag.c @@ -165,7 +165,7 @@ ip4_frag_do_fragment (vlib_main_t * vm, u32 from_bi, u32 ** buffer, frag_set_sw_if_index (to_b, org_from_b); /* Copy ip4 header */ - clib_memcpy (to_b->data, org_from_packet, sizeof (ip4_header_t)); + clib_memcpy_fast (to_b->data, org_from_packet, sizeof (ip4_header_t)); to_ip4 = vlib_buffer_get_current (to_b); to_data = (void *) (to_ip4 + 1); @@ -178,7 +178,7 @@ ip4_frag_do_fragment (vlib_main_t * vm, u32 from_bi, u32 ** buffer, /* Figure out how many bytes we can safely copy */ bytes_to_copy = left_in_to_buffer <= left_in_from_buffer ? left_in_to_buffer : left_in_from_buffer; - clib_memcpy (to_data + to_ptr, from_data + ptr, bytes_to_copy); + clib_memcpy_fast (to_data + to_ptr, from_data + ptr, bytes_to_copy); left_in_to_buffer -= bytes_to_copy; ptr += bytes_to_copy; left_in_from_buffer -= bytes_to_copy; @@ -441,7 +441,7 @@ ip6_frag_do_fragment (vlib_main_t * vm, u32 from_bi, u32 ** buffer, frag_set_sw_if_index (to_b, org_from_b); /* Copy ip6 header */ - clib_memcpy (to_b->data, ip6, sizeof (ip6_header_t)); + clib_memcpy_fast (to_b->data, ip6, sizeof (ip6_header_t)); to_ip6 = vlib_buffer_get_current (to_b); to_frag_hdr = (ip6_frag_hdr_t *) (to_ip6 + 1); to_data = (void *) (to_frag_hdr + 1); @@ -455,7 +455,7 @@ ip6_frag_do_fragment (vlib_main_t * vm, u32 from_bi, u32 ** buffer, /* Figure out how many bytes we can safely copy */ bytes_to_copy = left_in_to_buffer <= left_in_from_buffer ? left_in_to_buffer : left_in_from_buffer; - clib_memcpy (to_data + to_ptr, from_data + ptr, bytes_to_copy); + clib_memcpy_fast (to_data + to_ptr, from_data + ptr, bytes_to_copy); left_in_to_buffer -= bytes_to_copy; ptr += bytes_to_copy; left_in_from_buffer -= bytes_to_copy; diff --git a/src/vnet/ip/punt.c b/src/vnet/ip/punt.c index a6d6c08818b..d8c7d81a23a 100644 --- a/src/vnet/ip/punt.c +++ b/src/vnet/ip/punt.c @@ -368,7 +368,7 @@ udp46_punt_socket_inline (vlib_main_t * vm, c = punt_client_get (is_ip4, port); udp_punt_trace_t *t; t = vlib_add_trace (vm, node, b, sizeof (t[0])); - clib_memcpy (&t->client, c, sizeof (t->client)); + clib_memcpy_fast (&t->client, c, sizeof (t->client)); } /* Re-set iovecs if present. */ @@ -401,7 +401,7 @@ udp46_punt_socket_inline (vlib_main_t * vm, } udp_punt_trace_t *t; t = vlib_add_trace (vm, node, b, sizeof (t[0])); - clib_memcpy (&t->client, c, sizeof (t->client)); + clib_memcpy_fast (&t->client, c, sizeof (t->client)); t->is_midchain = 1; } diff --git a/src/vnet/ip/rd_cp.c b/src/vnet/ip/rd_cp.c index 164b3045775..1a6122fc9d3 100644 --- a/src/vnet/ip/rd_cp.c +++ b/src/vnet/ip/rd_cp.c @@ -227,7 +227,7 @@ get_interface_mac_address (u32 sw_if_index, u8 mac[]) return 1; } - clib_memcpy (mac, eth_if->address, 6); + clib_memcpy_fast (mac, eth_if->address, 6); return 0; } diff --git a/src/vnet/ipfix-export/flow_report.c b/src/vnet/ipfix-export/flow_report.c index 78ad1f5023c..c9370a748cf 100644 --- a/src/vnet/ipfix-export/flow_report.c +++ b/src/vnet/ipfix-export/flow_report.c @@ -126,7 +126,7 @@ send_template_packet (flow_report_main_t * frm, ASSERT (vec_len (fr->rewrite) < VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES); - clib_memcpy (b0->data, fr->rewrite, vec_len (fr->rewrite)); + clib_memcpy_fast (b0->data, fr->rewrite, vec_len (fr->rewrite)); b0->current_data = 0; b0->current_length = vec_len (fr->rewrite); b0->flags |= (VLIB_BUFFER_TOTAL_LENGTH_VALID | VNET_BUFFER_F_FLOW_REPORT); diff --git a/src/vnet/ipfix-export/flow_report_classify.c b/src/vnet/ipfix-export/flow_report_classify.c index 449bf228ce9..6bddb1407fe 100644 --- a/src/vnet/ipfix-export/flow_report_classify.c +++ b/src/vnet/ipfix-export/flow_report_classify.c @@ -226,7 +226,7 @@ ipfix_classify_send_flows (flow_report_main_t * frm, u32 copy_len = sizeof (ip4_header_t) + sizeof (udp_header_t) + sizeof (ipfix_message_header_t); - clib_memcpy (b0->data, fr->rewrite, copy_len); + clib_memcpy_fast (b0->data, fr->rewrite, copy_len); b0->current_data = 0; b0->current_length = copy_len; b0->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; @@ -261,7 +261,7 @@ ipfix_classify_send_flows (flow_report_main_t * frm, #define _(field,mask,item,length) \ if (clib_bitmap_get (fr->fields_to_send, field_index)) \ { \ - clib_memcpy (b0->data + next_offset, &field, \ + clib_memcpy_fast (b0->data + next_offset, &field, \ length); \ next_offset += length; \ } \ @@ -272,8 +272,8 @@ ipfix_classify_send_flows (flow_report_main_t * frm, /* Add packetTotalCount manually */ { u64 packets = clib_host_to_net_u64 (v->hits); - clib_memcpy (b0->data + next_offset, &packets, - sizeof (packets)); + clib_memcpy_fast (b0->data + next_offset, &packets, + sizeof (packets)); next_offset += sizeof (packets); } records_this_buffer++; diff --git a/src/vnet/ipsec/ah_encrypt.c b/src/vnet/ipsec/ah_encrypt.c index 7a6703da2de..6529828f0e9 100644 --- a/src/vnet/ipsec/ah_encrypt.c +++ b/src/vnet/ipsec/ah_encrypt.c @@ -171,7 +171,7 @@ ah_encrypt_inline (vlib_main_t * vm, sizeof (ethernet_header_t)); ethernet_header_t *oeh0 = (ethernet_header_t *) ((u8 *) ieh0 + (adv - icv_size)); - clib_memcpy (oeh0, ieh0, sizeof (ethernet_header_t)); + clib_memcpy_fast (oeh0, ieh0, sizeof (ethernet_header_t)); } vlib_buffer_advance (i_b0, adv - icv_size); diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c index 8edd5653667..101c5efbfc8 100644 --- a/src/vnet/ipsec/esp_encrypt.c +++ b/src/vnet/ipsec/esp_encrypt.c @@ -316,7 +316,7 @@ esp_encrypt_inline (vlib_main_t * vm, vlib_buffer_get_current (i_b0) - sizeof (ethernet_header_t)); oeh0 = (ethernet_header_t *) o_b0->data; - clib_memcpy (oeh0, ieh0, sizeof (ethernet_header_t)); + clib_memcpy_fast (oeh0, ieh0, sizeof (ethernet_header_t)); next0 = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT; vnet_buffer (o_b0)->sw_if_index[VLIB_TX] = vnet_buffer (i_b0)->sw_if_index[VLIB_TX]; @@ -359,10 +359,10 @@ esp_encrypt_inline (vlib_main_t * vm, ipsec_proto_main_crypto_algs[sa0->crypto_alg].iv_size]; RAND_bytes (iv, sizeof (iv)); - clib_memcpy ((u8 *) vlib_buffer_get_current (o_b0) + - ip_udp_hdr_size + sizeof (esp_header_t), iv, - em->ipsec_proto_main_crypto_algs[sa0-> - crypto_alg].iv_size); + clib_memcpy_fast ((u8 *) vlib_buffer_get_current (o_b0) + + ip_udp_hdr_size + sizeof (esp_header_t), iv, + em->ipsec_proto_main_crypto_algs[sa0-> + crypto_alg].iv_size); esp_encrypt_cbc (vm, sa0->crypto_alg, (u8 *) vlib_buffer_get_current (i_b0), diff --git a/src/vnet/ipsec/ikev2.c b/src/vnet/ipsec/ikev2.c index ac20d4a9a10..80497f20807 100644 --- a/src/vnet/ipsec/ikev2.c +++ b/src/vnet/ipsec/ikev2.c @@ -161,7 +161,7 @@ ikev2_select_proposal (ikev2_sa_proposal_t * proposals, { bitmap |= 1 << transform->type; vec_add2 (rv->transforms, new_t, 1); - clib_memcpy (new_t, transform, sizeof (*new_t)); + clib_memcpy_fast (new_t, transform, sizeof (*new_t)); new_t->attrs = vec_dup (transform->attrs); } } @@ -469,37 +469,37 @@ ikev2_calc_keys (ikev2_sa_t * sa) /* SK_d */ sa->sk_d = vec_new (u8, tr_prf->key_trunc); - clib_memcpy (sa->sk_d, keymat + pos, tr_prf->key_trunc); + clib_memcpy_fast (sa->sk_d, keymat + pos, tr_prf->key_trunc); pos += tr_prf->key_trunc; /* SK_ai */ sa->sk_ai = vec_new (u8, tr_integ->key_len); - clib_memcpy (sa->sk_ai, keymat + pos, tr_integ->key_len); + clib_memcpy_fast (sa->sk_ai, keymat + pos, tr_integ->key_len); pos += tr_integ->key_len; /* SK_ar */ sa->sk_ar = vec_new (u8, tr_integ->key_len); - clib_memcpy (sa->sk_ar, keymat + pos, tr_integ->key_len); + clib_memcpy_fast (sa->sk_ar, keymat + pos, tr_integ->key_len); pos += tr_integ->key_len; /* SK_ei */ sa->sk_ei = vec_new (u8, tr_encr->key_len); - clib_memcpy (sa->sk_ei, keymat + pos, tr_encr->key_len); + clib_memcpy_fast (sa->sk_ei, keymat + pos, tr_encr->key_len); pos += tr_encr->key_len; /* SK_er */ sa->sk_er = vec_new (u8, tr_encr->key_len); - clib_memcpy (sa->sk_er, keymat + pos, tr_encr->key_len); + clib_memcpy_fast (sa->sk_er, keymat + pos, tr_encr->key_len); pos += tr_encr->key_len; /* SK_pi */ sa->sk_pi = vec_new (u8, tr_prf->key_len); - clib_memcpy (sa->sk_pi, keymat + pos, tr_prf->key_len); + clib_memcpy_fast (sa->sk_pi, keymat + pos, tr_prf->key_len); pos += tr_prf->key_len; /* SK_pr */ sa->sk_pr = vec_new (u8, tr_prf->key_len); - clib_memcpy (sa->sk_pr, keymat + pos, tr_prf->key_len); + clib_memcpy_fast (sa->sk_pr, keymat + pos, tr_prf->key_len); pos += tr_prf->key_len; vec_free (keymat); @@ -529,22 +529,22 @@ ikev2_calc_child_keys (ikev2_sa_t * sa, ikev2_child_sa_t * child) /* SK_ei */ child->sk_ei = vec_new (u8, ctr_encr->key_len); - clib_memcpy (child->sk_ei, keymat + pos, ctr_encr->key_len); + clib_memcpy_fast (child->sk_ei, keymat + pos, ctr_encr->key_len); pos += ctr_encr->key_len; /* SK_ai */ child->sk_ai = vec_new (u8, ctr_integ->key_len); - clib_memcpy (child->sk_ai, keymat + pos, ctr_integ->key_len); + clib_memcpy_fast (child->sk_ai, keymat + pos, ctr_integ->key_len); pos += ctr_integ->key_len; /* SK_er */ child->sk_er = vec_new (u8, ctr_encr->key_len); - clib_memcpy (child->sk_er, keymat + pos, ctr_encr->key_len); + clib_memcpy_fast (child->sk_er, keymat + pos, ctr_encr->key_len); pos += ctr_encr->key_len; /* SK_ar */ child->sk_ar = vec_new (u8, ctr_integ->key_len); - clib_memcpy (child->sk_ar, keymat + pos, ctr_integ->key_len); + clib_memcpy_fast (child->sk_ar, keymat + pos, ctr_integ->key_len); pos += ctr_integ->key_len; ASSERT (pos == len); @@ -1106,7 +1106,7 @@ ikev2_process_create_child_sa_req (vlib_main_t * vm, ikev2_sa_t * sa, } else if (payload == IKEV2_PAYLOAD_NONCE) { - clib_memcpy (nonce, ikep->payload, plen - sizeof (*ikep)); + clib_memcpy_fast (nonce, ikep->payload, plen - sizeof (*ikep)); } else if (payload == IKEV2_PAYLOAD_TSI) { @@ -1596,15 +1596,15 @@ ikev2_create_tunnel_interface (vnet_main_t * vnm, ikev2_sa_t * sa, a.integ_alg = integ_type; a.local_integ_key_len = vec_len (loc_ikey); - clib_memcpy (a.local_integ_key, loc_ikey, a.local_integ_key_len); + clib_memcpy_fast (a.local_integ_key, loc_ikey, a.local_integ_key_len); a.remote_integ_key_len = vec_len (rem_ikey); - clib_memcpy (a.remote_integ_key, rem_ikey, a.remote_integ_key_len); + clib_memcpy_fast (a.remote_integ_key, rem_ikey, a.remote_integ_key_len); a.crypto_alg = encr_type; a.local_crypto_key_len = vec_len (loc_ckey); - clib_memcpy (a.local_crypto_key, loc_ckey, a.local_crypto_key_len); + clib_memcpy_fast (a.local_crypto_key, loc_ckey, a.local_crypto_key_len); a.remote_crypto_key_len = vec_len (rem_ckey); - clib_memcpy (a.remote_crypto_key, rem_ckey, a.remote_crypto_key_len); + clib_memcpy_fast (a.remote_crypto_key, rem_ckey, a.remote_crypto_key_len); if (sa->profile && sa->profile->lifetime) { @@ -1898,7 +1898,7 @@ ikev2_generate_message (ikev2_sa_t * sa, ike_header_t * ike, void *user) tlen += vec_len (chain->data); ike->nextpayload = chain->first_payload_type; ike->length = clib_host_to_net_u32 (tlen); - clib_memcpy (ike->payload, chain->data, vec_len (chain->data)); + clib_memcpy_fast (ike->payload, chain->data, vec_len (chain->data)); /* store whole IKE payload - needed for PSK auth */ vec_free (sa->last_sa_init_res_packet_data); @@ -1930,8 +1930,8 @@ ikev2_generate_message (ikev2_sa_t * sa, ike_header_t * ike, void *user) ikev2_calc_integr (tr_integ, sa->is_initiator ? sa->sk_ai : sa->sk_ar, (u8 *) ike, tlen - tr_integ->key_trunc); - clib_memcpy (ike->payload + tlen - tr_integ->key_trunc - sizeof (*ike), - integ, tr_integ->key_trunc); + clib_memcpy_fast (ike->payload + tlen - tr_integ->key_trunc - + sizeof (*ike), integ, tr_integ->key_trunc); /* store whole IKE payload - needed for retransmit */ vec_free (sa->last_res_packet_data); @@ -1986,7 +1986,7 @@ ikev2_retransmit_sa_init (ike_header_t * ike, ike->flags = tmp->flags; ike->msgid = tmp->msgid; ike->length = tmp->length; - clib_memcpy(ike->payload, tmp->payload, + clib_memcpy_fast(ike->payload, tmp->payload, clib_net_to_host_u32(tmp->length) - sizeof(*ike)); clib_warning("IKE_SA_INIT retransmit from %U to %U", format_ip4_address, &raddr, @@ -2038,8 +2038,8 @@ ikev2_retransmit_resp (ikev2_sa_t * sa, ike_header_t * ike) ike->flags = tmp->flags; ike->msgid = tmp->msgid; ike->length = tmp->length; - clib_memcpy (ike->payload, tmp->payload, - clib_net_to_host_u32 (tmp->length) - sizeof (*ike)); + clib_memcpy_fast (ike->payload, tmp->payload, + clib_net_to_host_u32 (tmp->length) - sizeof (*ike)); clib_warning ("IKE msgid %u retransmit from %U to %U", msg_id, format_ip4_address, &sa->raddr, @@ -2165,7 +2165,7 @@ ikev2_node_fn (vlib_main_t * vm, /* add SA to the pool */ pool_get (km->per_thread_data[thread_index].sas, sa0); - clib_memcpy (sa0, &sa, sizeof (*sa0)); + clib_memcpy_fast (sa0, &sa, sizeof (*sa0)); hash_set (km-> per_thread_data[thread_index].sa_by_rspi, sa0->rspi, @@ -2202,7 +2202,7 @@ ikev2_node_fn (vlib_main_t * vm, { /* add SA to the pool */ pool_get (km->per_thread_data[thread_index].sas, sa0); - clib_memcpy (sa0, &sa, sizeof (*sa0)); + clib_memcpy_fast (sa0, &sa, sizeof (*sa0)); hash_set (km->per_thread_data[thread_index].sa_by_rspi, sa0->rspi, sa0 - km->per_thread_data[thread_index].sas); @@ -2993,20 +2993,20 @@ ikev2_initiate_sa_init (vlib_main_t * vm, u8 * name) u8 *nat_detection_sha1 = vec_new (u8, 20); u64 tmpspi = clib_host_to_net_u64 (sa.ispi); - clib_memcpy (&nat_detection_source[0], &tmpspi, sizeof (tmpspi)); + clib_memcpy_fast (&nat_detection_source[0], &tmpspi, sizeof (tmpspi)); tmpspi = clib_host_to_net_u64 (sa.rspi); - clib_memcpy (&nat_detection_source[8], &tmpspi, sizeof (tmpspi)); + clib_memcpy_fast (&nat_detection_source[8], &tmpspi, sizeof (tmpspi)); u16 tmpport = clib_host_to_net_u16 (500); - clib_memcpy (&nat_detection_source[8 + 8 + 4], &tmpport, - sizeof (tmpport)); + clib_memcpy_fast (&nat_detection_source[8 + 8 + 4], &tmpport, + sizeof (tmpport)); u32 tmpip = clib_host_to_net_u32 (if_ip->as_u32); - clib_memcpy (&nat_detection_source[8 + 8], &tmpip, sizeof (tmpip)); + clib_memcpy_fast (&nat_detection_source[8 + 8], &tmpip, sizeof (tmpip)); SHA1 (nat_detection_source, sizeof (nat_detection_source), nat_detection_sha1); ikev2_payload_add_notify (chain, IKEV2_NOTIFY_MSG_NAT_DETECTION_SOURCE_IP, nat_detection_sha1); tmpip = clib_host_to_net_u32 (p->responder.ip4.as_u32); - clib_memcpy (&nat_detection_source[8 + 8], &tmpip, sizeof (tmpip)); + clib_memcpy_fast (&nat_detection_source[8 + 8], &tmpip, sizeof (tmpip)); SHA1 (nat_detection_source, sizeof (nat_detection_source), nat_detection_sha1); ikev2_payload_add_notify (chain, @@ -3016,7 +3016,7 @@ ikev2_initiate_sa_init (vlib_main_t * vm, u8 * name) u8 *sig_hash_algo = vec_new (u8, 8); u64 tmpsig = clib_host_to_net_u64 (0x0001000200030004); - clib_memcpy (sig_hash_algo, &tmpsig, sizeof (tmpsig)); + clib_memcpy_fast (sig_hash_algo, &tmpsig, sizeof (tmpsig)); ikev2_payload_add_notify (chain, IKEV2_NOTIFY_MSG_SIGNATURE_HASH_ALGORITHMS, sig_hash_algo); @@ -3027,7 +3027,7 @@ ikev2_initiate_sa_init (vlib_main_t * vm, u8 * name) len += vec_len (chain->data); ike0->nextpayload = chain->first_payload_type; ike0->length = clib_host_to_net_u32 (len); - clib_memcpy (ike0->payload, chain->data, vec_len (chain->data)); + clib_memcpy_fast (ike0->payload, chain->data, vec_len (chain->data)); ikev2_payload_destroy_chain (chain); ike0->version = IKE_VERSION_2; @@ -3048,7 +3048,8 @@ ikev2_initiate_sa_init (vlib_main_t * vm, u8 * name) sa.i_auth.hex = p->auth.hex; sa.i_auth.data = vec_dup (p->auth.data); #if OPENSSL_VERSION_NUMBER >= 0x10100000L - clib_memcpy (sa.i_auth.key, p->auth.key, EVP_PKEY_size (p->auth.key)); + clib_memcpy_fast (sa.i_auth.key, p->auth.key, + EVP_PKEY_size (p->auth.key)); #else sa.i_auth.key = vec_dup (p->auth.key); #endif @@ -3058,7 +3059,7 @@ ikev2_initiate_sa_init (vlib_main_t * vm, u8 * name) /* add SA to the pool */ ikev2_sa_t *sa0 = 0; pool_get (km->sais, sa0); - clib_memcpy (sa0, &sa, sizeof (*sa0)); + clib_memcpy_fast (sa0, &sa, sizeof (*sa0)); hash_set (km->sa_by_ispi, sa0->ispi, sa0 - km->sais); ikev2_send_ike (vm, if_ip, &p->responder.ip4, bi0, len); diff --git a/src/vnet/ipsec/ikev2_payload.c b/src/vnet/ipsec/ikev2_payload.c index e5fa0149feb..3aae6d3a1f4 100644 --- a/src/vnet/ipsec/ikev2_payload.c +++ b/src/vnet/ipsec/ikev2_payload.c @@ -205,7 +205,7 @@ ikev2_payload_add_sa (ikev2_payload_chain_t * c, clib_host_to_net_u16 (sizeof (*tr) + vec_len (t->attrs)); if (vec_len (t->attrs) > 0) - clib_memcpy (tr->attributes, t->attrs, vec_len (t->attrs)); + clib_memcpy_fast (tr->attributes, t->attrs, vec_len (t->attrs)); DBG_PLD ("transform type %U transform_id %u last_or_more %u attr_size %u%s%U", diff --git a/src/vnet/l2/l2_efp_filter.c b/src/vnet/l2/l2_efp_filter.c index 59f9f515d47..f10c5eaf7f9 100644 --- a/src/vnet/l2/l2_efp_filter.c +++ b/src/vnet/l2/l2_efp_filter.c @@ -340,9 +340,9 @@ l2_efp_filter_node_fn (vlib_main_t * vm, l2_efp_filter_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); - clib_memcpy (t->raw, &h0->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->raw, &h0->type, sizeof (t->raw)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -350,9 +350,9 @@ l2_efp_filter_node_fn (vlib_main_t * vm, l2_efp_filter_trace_t *t = vlib_add_trace (vm, node, b1, sizeof (*t)); t->sw_if_index = sw_if_index1; - clib_memcpy (t->src, h1->src_address, 6); - clib_memcpy (t->dst, h1->dst_address, 6); - clib_memcpy (t->raw, &h1->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h1->src_address, 6); + clib_memcpy_fast (t->dst, h1->dst_address, 6); + clib_memcpy_fast (t->raw, &h1->type, sizeof (t->raw)); } } @@ -441,9 +441,9 @@ l2_efp_filter_node_fn (vlib_main_t * vm, l2_efp_filter_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); - clib_memcpy (t->raw, &h0->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->raw, &h0->type, sizeof (t->raw)); } /* verify speculative enqueue, maybe switch current next frame */ diff --git a/src/vnet/l2/l2_fib.c b/src/vnet/l2/l2_fib.c index 48be42e8ce9..fc518fe9dbf 100644 --- a/src/vnet/l2/l2_fib.c +++ b/src/vnet/l2/l2_fib.c @@ -83,7 +83,7 @@ incr_mac_address (u8 * mac) tmp += 1 << 16; /* skip unused (least significant) octets */ tmp = clib_host_to_net_u64 (tmp); - clib_memcpy (mac, &tmp, 6); + clib_memcpy_fast (mac, &tmp, 6); } /** Format sw_if_index. If the value is ~0, use the text "N/A" */ @@ -584,7 +584,7 @@ l2fib_test_command_fn (vlib_main_t * vm, return clib_error_return (0, "noop: pick at least one of (add,del,check)"); - clib_memcpy (save_mac, mac, 6); + clib_memcpy_fast (save_mac, mac, 6); if (is_add) { @@ -601,7 +601,7 @@ l2fib_test_command_fn (vlib_main_t * vm, BVT (clib_bihash_kv) kv; l2fib_main_t *mp = &l2fib_main; - clib_memcpy (mac, save_mac, 6); + clib_memcpy_fast (mac, save_mac, 6); for (i = 0; i < count; i++) { @@ -617,7 +617,7 @@ l2fib_test_command_fn (vlib_main_t * vm, if (is_del) { - clib_memcpy (mac, save_mac, 6); + clib_memcpy_fast (mac, save_mac, 6); for (i = 0; i < count; i++) { @@ -1071,8 +1071,8 @@ l2fib_scan (vlib_main_t * vm, f64 start_time, u8 event_only) if (l2fib_entry_result_is_set_LRN_EVT (&result)) { /* copy mac entry to event msg */ - clib_memcpy (mp->mac[evt_idx].mac_addr, key.fields.mac, - 6); + clib_memcpy_fast (mp->mac[evt_idx].mac_addr, + key.fields.mac, 6); mp->mac[evt_idx].action = l2fib_entry_result_is_set_LRN_MOV (&result) ? MAC_EVENT_ACTION_MOVE : MAC_EVENT_ACTION_ADD; @@ -1116,7 +1116,8 @@ l2fib_scan (vlib_main_t * vm, f64 start_time, u8 event_only) if (client) { /* copy mac entry to event msg */ - clib_memcpy (mp->mac[evt_idx].mac_addr, key.fields.mac, 6); + clib_memcpy_fast (mp->mac[evt_idx].mac_addr, key.fields.mac, + 6); mp->mac[evt_idx].action = MAC_EVENT_ACTION_DELETE; mp->mac[evt_idx].sw_if_index = htonl (result.fields.sw_if_index); diff --git a/src/vnet/l2/l2_flood.c b/src/vnet/l2/l2_flood.c index aeac8ff10d8..3198d323d60 100644 --- a/src/vnet/l2/l2_flood.c +++ b/src/vnet/l2/l2_flood.c @@ -256,8 +256,8 @@ l2flood_node_fn (vlib_main_t * vm, h0 = vlib_buffer_get_current (c0); t->sw_if_index = sw_if_index0; t->bd_index = vnet_buffer (c0)->l2.bd_index; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); } /* Do normal L2 forwarding */ @@ -307,8 +307,8 @@ l2flood_node_fn (vlib_main_t * vm, h0 = vlib_buffer_get_current (c0); t->sw_if_index = sw_if_index0; t->bd_index = vnet_buffer (c0)->l2.bd_index; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); } diff --git a/src/vnet/l2/l2_fwd.c b/src/vnet/l2/l2_fwd.c index 9fb7b7d355a..dfc1a74a38c 100644 --- a/src/vnet/l2/l2_fwd.c +++ b/src/vnet/l2/l2_fwd.c @@ -324,8 +324,8 @@ l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, l2fwd_trace_t *t = vlib_add_trace (vm, node, b[0], sizeof (*t)); t->sw_if_index = sw_if_index0; t->bd_index = vnet_buffer (b[0])->l2.bd_index; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); t->result = result0; } if (b[1]->flags & VLIB_BUFFER_IS_TRACED) @@ -333,8 +333,8 @@ l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, l2fwd_trace_t *t = vlib_add_trace (vm, node, b[1], sizeof (*t)); t->sw_if_index = sw_if_index1; t->bd_index = vnet_buffer (b[1])->l2.bd_index; - clib_memcpy (t->src, h1->src_address, 6); - clib_memcpy (t->dst, h1->dst_address, 6); + clib_memcpy_fast (t->src, h1->src_address, 6); + clib_memcpy_fast (t->dst, h1->dst_address, 6); t->result = result1; } if (b[2]->flags & VLIB_BUFFER_IS_TRACED) @@ -342,8 +342,8 @@ l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, l2fwd_trace_t *t = vlib_add_trace (vm, node, b[2], sizeof (*t)); t->sw_if_index = sw_if_index2; t->bd_index = vnet_buffer (b[2])->l2.bd_index; - clib_memcpy (t->src, h2->src_address, 6); - clib_memcpy (t->dst, h2->dst_address, 6); + clib_memcpy_fast (t->src, h2->src_address, 6); + clib_memcpy_fast (t->dst, h2->dst_address, 6); t->result = result2; } if (b[3]->flags & VLIB_BUFFER_IS_TRACED) @@ -351,8 +351,8 @@ l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, l2fwd_trace_t *t = vlib_add_trace (vm, node, b[3], sizeof (*t)); t->sw_if_index = sw_if_index3; t->bd_index = vnet_buffer (b[3])->l2.bd_index; - clib_memcpy (t->src, h3->src_address, 6); - clib_memcpy (t->dst, h3->dst_address, 6); + clib_memcpy_fast (t->src, h3->src_address, 6); + clib_memcpy_fast (t->dst, h3->dst_address, 6); t->result = result3; } } @@ -387,8 +387,8 @@ l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, l2fwd_trace_t *t = vlib_add_trace (vm, node, b[0], sizeof (*t)); t->sw_if_index = sw_if_index0; t->bd_index = vnet_buffer (b[0])->l2.bd_index; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); t->result = result0; } diff --git a/src/vnet/l2/l2_input.c b/src/vnet/l2/l2_input.c index 28b8697a7e4..b66a7446d66 100644 --- a/src/vnet/l2/l2_input.c +++ b/src/vnet/l2/l2_input.c @@ -366,8 +366,8 @@ l2input_node_inline (vlib_main_t * vm, l2input_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -375,8 +375,8 @@ l2input_node_inline (vlib_main_t * vm, l2input_trace_t *t = vlib_add_trace (vm, node, b1, sizeof (*t)); t->sw_if_index = sw_if_index1; - clib_memcpy (t->src, h1->src_address, 6); - clib_memcpy (t->dst, h1->dst_address, 6); + clib_memcpy_fast (t->src, h1->src_address, 6); + clib_memcpy_fast (t->dst, h1->dst_address, 6); } if (b2->flags & VLIB_BUFFER_IS_TRACED) { @@ -384,8 +384,8 @@ l2input_node_inline (vlib_main_t * vm, l2input_trace_t *t = vlib_add_trace (vm, node, b2, sizeof (*t)); t->sw_if_index = sw_if_index2; - clib_memcpy (t->src, h2->src_address, 6); - clib_memcpy (t->dst, h2->dst_address, 6); + clib_memcpy_fast (t->src, h2->src_address, 6); + clib_memcpy_fast (t->dst, h2->dst_address, 6); } if (b3->flags & VLIB_BUFFER_IS_TRACED) { @@ -393,8 +393,8 @@ l2input_node_inline (vlib_main_t * vm, l2input_trace_t *t = vlib_add_trace (vm, node, b3, sizeof (*t)); t->sw_if_index = sw_if_index3; - clib_memcpy (t->src, h3->src_address, 6); - clib_memcpy (t->dst, h3->dst_address, 6); + clib_memcpy_fast (t->src, h3->src_address, 6); + clib_memcpy_fast (t->dst, h3->dst_address, 6); } } @@ -434,8 +434,8 @@ l2input_node_inline (vlib_main_t * vm, l2input_trace_t *t = vlib_add_trace (vm, node, b0, sizeof (*t)); sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); } classify_and_dispatch (msm, b0, &next0); diff --git a/src/vnet/l2/l2_input_vtr.c b/src/vnet/l2/l2_input_vtr.c index 9470752f0e0..25b3bd03e65 100644 --- a/src/vnet/l2/l2_input_vtr.c +++ b/src/vnet/l2/l2_input_vtr.c @@ -222,9 +222,9 @@ l2_invtr_node_fn (vlib_main_t * vm, vlib_add_trace (vm, node, b0, sizeof (*t)); ethernet_header_t *h0 = vlib_buffer_get_current (b0); t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); - clib_memcpy (t->raw, &h0->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->raw, &h0->type, sizeof (t->raw)); } if (b1->flags & VLIB_BUFFER_IS_TRACED) { @@ -232,9 +232,9 @@ l2_invtr_node_fn (vlib_main_t * vm, vlib_add_trace (vm, node, b1, sizeof (*t)); ethernet_header_t *h1 = vlib_buffer_get_current (b1); t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h1->src_address, 6); - clib_memcpy (t->dst, h1->dst_address, 6); - clib_memcpy (t->raw, &h1->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h1->src_address, 6); + clib_memcpy_fast (t->dst, h1->dst_address, 6); + clib_memcpy_fast (t->raw, &h1->type, sizeof (t->raw)); } } @@ -301,9 +301,9 @@ l2_invtr_node_fn (vlib_main_t * vm, vlib_add_trace (vm, node, b0, sizeof (*t)); ethernet_header_t *h0 = vlib_buffer_get_current (b0); t->sw_if_index = sw_if_index0; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); - clib_memcpy (t->raw, &h0->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->raw, &h0->type, sizeof (t->raw)); } /* verify speculative enqueue, maybe switch current next frame */ diff --git a/src/vnet/l2/l2_learn.c b/src/vnet/l2/l2_learn.c index c6ecef33b0f..e47365751c7 100644 --- a/src/vnet/l2/l2_learn.c +++ b/src/vnet/l2/l2_learn.c @@ -306,8 +306,8 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_add_trace (vm, node, b[0], sizeof (*t)); t->sw_if_index = sw_if_index0; t->bd_index = vnet_buffer (b[0])->l2.bd_index; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); } if (b[1]->flags & VLIB_BUFFER_IS_TRACED) { @@ -315,8 +315,8 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_add_trace (vm, node, b[1], sizeof (*t)); t->sw_if_index = sw_if_index1; t->bd_index = vnet_buffer (b[1])->l2.bd_index; - clib_memcpy (t->src, h1->src_address, 6); - clib_memcpy (t->dst, h1->dst_address, 6); + clib_memcpy_fast (t->src, h1->src_address, 6); + clib_memcpy_fast (t->dst, h1->dst_address, 6); } if (b[2]->flags & VLIB_BUFFER_IS_TRACED) { @@ -324,8 +324,8 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_add_trace (vm, node, b[2], sizeof (*t)); t->sw_if_index = sw_if_index2; t->bd_index = vnet_buffer (b[2])->l2.bd_index; - clib_memcpy (t->src, h2->src_address, 6); - clib_memcpy (t->dst, h2->dst_address, 6); + clib_memcpy_fast (t->src, h2->src_address, 6); + clib_memcpy_fast (t->dst, h2->dst_address, 6); } if (b[3]->flags & VLIB_BUFFER_IS_TRACED) { @@ -333,8 +333,8 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_add_trace (vm, node, b[3], sizeof (*t)); t->sw_if_index = sw_if_index3; t->bd_index = vnet_buffer (b[3])->l2.bd_index; - clib_memcpy (t->src, h3->src_address, 6); - clib_memcpy (t->dst, h3->dst_address, 6); + clib_memcpy_fast (t->src, h3->src_address, 6); + clib_memcpy_fast (t->dst, h3->dst_address, 6); } } @@ -391,8 +391,8 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, l2learn_trace_t *t = vlib_add_trace (vm, node, b[0], sizeof (*t)); t->sw_if_index = sw_if_index0; t->bd_index = vnet_buffer (b[0])->l2.bd_index; - clib_memcpy (t->src, h0->src_address, 6); - clib_memcpy (t->dst, h0->dst_address, 6); + clib_memcpy_fast (t->src, h0->src_address, 6); + clib_memcpy_fast (t->dst, h0->dst_address, 6); } /* process 1 pkt */ diff --git a/src/vnet/l2/l2_output.c b/src/vnet/l2/l2_output.c index 27b838f4378..e15f0ac45b2 100644 --- a/src/vnet/l2/l2_output.c +++ b/src/vnet/l2/l2_output.c @@ -426,9 +426,9 @@ VLIB_NODE_FN (l2output_node) (vlib_main_t * vm, vlib_add_trace (vm, node, b[0], sizeof (*t)); t->sw_if_index = vnet_buffer (b[0])->sw_if_index[VLIB_TX]; h = vlib_buffer_get_current (b[0]); - clib_memcpy (t->src, h->src_address, 6); - clib_memcpy (t->dst, h->dst_address, 6); - clib_memcpy (t->raw, &h->type, sizeof (t->raw)); + clib_memcpy_fast (t->src, h->src_address, 6); + clib_memcpy_fast (t->dst, h->dst_address, 6); + clib_memcpy_fast (t->raw, &h->type, sizeof (t->raw)); } /* next */ n_left--; diff --git a/src/vnet/l2/l2_vtr.c b/src/vnet/l2/l2_vtr.c index 7096891381d..aa3d5c45bee 100644 --- a/src/vnet/l2/l2_vtr.c +++ b/src/vnet/l2/l2_vtr.c @@ -91,10 +91,10 @@ l2pbb_configure (vlib_main_t * vlib_main, } else if (vtr_op == L2_VTR_PUSH_2) { - clib_memcpy (in_config->macs_tags.b_dst_address, b_dmac, - sizeof (in_config->macs_tags.b_dst_address)); - clib_memcpy (in_config->macs_tags.b_src_address, b_smac, - sizeof (in_config->macs_tags.b_src_address)); + clib_memcpy_fast (in_config->macs_tags.b_dst_address, b_dmac, + sizeof (in_config->macs_tags.b_dst_address)); + clib_memcpy_fast (in_config->macs_tags.b_src_address, b_smac, + sizeof (in_config->macs_tags.b_src_address)); in_config->macs_tags.b_type = clib_net_to_host_u16 (ETHERNET_TYPE_DOT1AD); in_config->macs_tags.priority_dei_id = @@ -726,10 +726,12 @@ l2pbb_get (vlib_main_t * vlib_main, vnet_main_t * vnet_main, u32 sw_if_index, else if (in_config->push_bytes) *vtr_op = L2_VTR_PUSH_2; - clib_memcpy (ð_hdr->dst_address, in_config->macs_tags.b_dst_address, - sizeof (eth_hdr->dst_address)); - clib_memcpy (ð_hdr->src_address, in_config->macs_tags.b_src_address, - sizeof (eth_hdr->src_address)); + clib_memcpy_fast (ð_hdr->dst_address, + in_config->macs_tags.b_dst_address, + sizeof (eth_hdr->dst_address)); + clib_memcpy_fast (ð_hdr->src_address, + in_config->macs_tags.b_src_address, + sizeof (eth_hdr->src_address)); *b_vlanid = clib_host_to_net_u16 (in_config->macs_tags.priority_dei_id) & 0xFFF; diff --git a/src/vnet/lisp-gpe/lisp_gpe.c b/src/vnet/lisp-gpe/lisp_gpe.c index 367dbae4159..9a10836707c 100644 --- a/src/vnet/lisp-gpe/lisp_gpe.c +++ b/src/vnet/lisp-gpe/lisp_gpe.c @@ -669,7 +669,7 @@ lisp_gpe_test_send_nsh_packet (u8 * file_name) vnet_buffer (b)->sw_if_index[VLIB_TX] = hi->sw_if_index; u8 *p = vlib_buffer_put_uninit (b, vec_len (pm.packets_read[0])); - clib_memcpy (p, pm.packets_read[0], vec_len (pm.packets_read[0])); + clib_memcpy_fast (p, pm.packets_read[0], vec_len (pm.packets_read[0])); vlib_buffer_pull (b, sizeof (ethernet_header_t)); vlib_node_t *n = vlib_get_node_by_name (lgm->vlib_main, diff --git a/src/vnet/lldp/lldp_node.c b/src/vnet/lldp/lldp_node.c index 2d169b98924..043ca9f01b5 100644 --- a/src/vnet/lldp/lldp_node.c +++ b/src/vnet/lldp/lldp_node.c @@ -87,7 +87,7 @@ lldp_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, len = (b0->current_length < sizeof (t0->data)) ? b0->current_length : sizeof (t0->data); t0->len = len; - clib_memcpy (t0->data, vlib_buffer_get_current (b0), len); + clib_memcpy_fast (t0->data, vlib_buffer_get_current (b0), len); } /* push this pkt to the next graph node, always error-drop */ vlib_set_next_frame_buffer (vm, node, next0, bi0); diff --git a/src/vnet/lldp/lldp_output.c b/src/vnet/lldp/lldp_output.c index 4ae25a45b7b..9c171c5da7b 100644 --- a/src/vnet/lldp/lldp_output.c +++ b/src/vnet/lldp/lldp_output.c @@ -27,7 +27,7 @@ lldp_build_mgmt_addr_tlv (u8 ** t0p, u8 subtype, u8 addr_len, u8 * addr, lldp_tlv_set_code (t, LLDP_TLV_NAME (mgmt_addr)); t->v[0] = addr_len + 1; /* address string length */ t->v[1] = subtype; /* address subtype */ - clib_memcpy (&(t->v[2]), addr, addr_len); /* address */ + clib_memcpy_fast (&(t->v[2]), addr, addr_len); /* address */ t->v[addr_len + 2] = 2; /* interface numbering subtype: ifIndex */ t->v[addr_len + 3] = (if_index >> 24) & 0xFF; /* interface number */ t->v[addr_len + 4] = (if_index >> 16) & 0xFF; @@ -36,7 +36,7 @@ lldp_build_mgmt_addr_tlv (u8 ** t0p, u8 subtype, u8 addr_len, u8 * addr, t->v[addr_len + 7] = oid_len; /* OID string length */ if (oid_len > 0) - clib_memcpy ((u8 *) & (t->v[addr_len + 8]), oid, oid_len); + clib_memcpy_fast ((u8 *) & (t->v[addr_len + 8]), oid, oid_len); lldp_tlv_set_length (t, addr_len + oid_len + 8); *t0p += STRUCT_SIZE_OF (lldp_tlv_t, head) + addr_len + oid_len + 8; @@ -51,7 +51,7 @@ lldp_add_chassis_id (const vnet_hw_interface_t * hw, u8 ** t0p) t->subtype = LLDP_CHASS_ID_SUBTYPE_NAME (mac_addr); const size_t addr_len = 6; - clib_memcpy (&t->id, hw->hw_address, addr_len); + clib_memcpy_fast (&t->id, hw->hw_address, addr_len); const size_t len = STRUCT_SIZE_OF (lldp_chassis_id_tlv_t, subtype) + addr_len; lldp_tlv_set_length ((lldp_tlv_t *) t, len); @@ -67,7 +67,7 @@ lldp_add_port_id (const vnet_hw_interface_t * hw, u8 ** t0p) t->subtype = LLDP_PORT_ID_SUBTYPE_NAME (intf_name); const size_t name_len = vec_len (hw->name); - clib_memcpy (&t->id, hw->name, name_len); + clib_memcpy_fast (&t->id, hw->name, name_len); const size_t len = STRUCT_SIZE_OF (lldp_port_id_tlv_t, subtype) + name_len; lldp_tlv_set_length ((lldp_tlv_t *) t, len); *t0p += STRUCT_SIZE_OF (lldp_tlv_t, head) + len; @@ -107,7 +107,7 @@ lldp_add_port_desc (const lldp_main_t * lm, lldp_intf_t * n, u8 ** t0p) lldp_tlv_t *t = (lldp_tlv_t *) * t0p; lldp_tlv_set_code (t, LLDP_TLV_NAME (port_desc)); lldp_tlv_set_length (t, len); - clib_memcpy (t->v, n->port_desc, len); + clib_memcpy_fast (t->v, n->port_desc, len); *t0p += STRUCT_SIZE_OF (lldp_tlv_t, head) + len; } } @@ -121,7 +121,7 @@ lldp_add_sys_name (const lldp_main_t * lm, u8 ** t0p) lldp_tlv_t *t = (lldp_tlv_t *) * t0p; lldp_tlv_set_code (t, LLDP_TLV_NAME (sys_name)); lldp_tlv_set_length (t, len); - clib_memcpy (t->v, lm->sys_name, len); + clib_memcpy_fast (t->v, lm->sys_name, len); *t0p += STRUCT_SIZE_OF (lldp_tlv_t, head) + len; } } @@ -220,7 +220,8 @@ lldp_send_ethernet (lldp_main_t * lm, lldp_intf_t * n, int shutdown) /* Add the interface's ethernet source address */ hw = vnet_get_hw_interface (vnm, n->hw_if_index); - clib_memcpy (h0->src_address, hw->hw_address, vec_len (hw->hw_address)); + clib_memcpy_fast (h0->src_address, hw->hw_address, + vec_len (hw->hw_address)); u8 *data = ((u8 *) h0) + sizeof (*h0); t0 = data; diff --git a/src/vnet/pg/input.c b/src/vnet/pg/input.c index 82c6c81cf18..8b46688929f 100644 --- a/src/vnet/pg/input.c +++ b/src/vnet/pg/input.c @@ -1091,7 +1091,7 @@ init_replay_buffers_inline (vlib_main_t * vm, b0->current_length = n0; - clib_memcpy (b0->data, d0 + data_offset, n0); + clib_memcpy_fast (b0->data, d0 + data_offset, n0); i = i + 1 == l ? 0 : i + 1; } } @@ -1148,8 +1148,8 @@ init_buffers_inline (vlib_main_t * vm, if (set_data) { - clib_memcpy (b0->data, data, n_data); - clib_memcpy (b1->data, data, n_data); + clib_memcpy_fast (b0->data, data, n_data); + clib_memcpy_fast (b1->data, data, n_data); } else { @@ -1173,7 +1173,7 @@ init_buffers_inline (vlib_main_t * vm, vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; if (set_data) - clib_memcpy (b0->data, data, n_data); + clib_memcpy_fast (b0->data, data, n_data); else ASSERT (validate_buffer_data2 (b0, s, data_offset, n_data)); } @@ -1424,13 +1424,15 @@ pg_input_trace (pg_main_t * pg, t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; t1->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_RX]; - clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data)); - clib_memcpy (&t1->buffer, b1, sizeof (b1[0]) - sizeof (b1->pre_data)); + clib_memcpy_fast (&t0->buffer, b0, + sizeof (b0[0]) - sizeof (b0->pre_data)); + clib_memcpy_fast (&t1->buffer, b1, + sizeof (b1[0]) - sizeof (b1->pre_data)); - clib_memcpy (t0->buffer.pre_data, b0->data, - sizeof (t0->buffer.pre_data)); - clib_memcpy (t1->buffer.pre_data, b1->data, - sizeof (t1->buffer.pre_data)); + clib_memcpy_fast (t0->buffer.pre_data, b0->data, + sizeof (t0->buffer.pre_data)); + clib_memcpy_fast (t1->buffer.pre_data, b1->data, + sizeof (t1->buffer.pre_data)); } while (n_left >= 1) @@ -1451,9 +1453,10 @@ pg_input_trace (pg_main_t * pg, t0->stream_index = stream_index; t0->packet_length = vlib_buffer_length_in_chain (vm, b0); t0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data)); - clib_memcpy (t0->buffer.pre_data, b0->data, - sizeof (t0->buffer.pre_data)); + clib_memcpy_fast (&t0->buffer, b0, + sizeof (b0[0]) - sizeof (b0->pre_data)); + clib_memcpy_fast (t0->buffer.pre_data, b0->data, + sizeof (t0->buffer.pre_data)); } } diff --git a/src/vnet/pg/output.c b/src/vnet/pg/output.c index 410a433621c..a84f30301dc 100644 --- a/src/vnet/pg/output.c +++ b/src/vnet/pg/output.c @@ -68,9 +68,10 @@ pg_output (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) { pg_output_trace_t *t = vlib_add_trace (vm, node, b, sizeof (*t)); t->buffer_index = bi0; - clib_memcpy (&t->buffer, b, sizeof (b[0]) - sizeof (b->pre_data)); - clib_memcpy (t->buffer.pre_data, b->data + b->current_data, - sizeof (t->buffer.pre_data)); + clib_memcpy_fast (&t->buffer, b, + sizeof (b[0]) - sizeof (b->pre_data)); + clib_memcpy_fast (t->buffer.pre_data, b->data + b->current_data, + sizeof (t->buffer.pre_data)); } if (pif->pcap_file_name != 0) diff --git a/src/vnet/pg/stream.c b/src/vnet/pg/stream.c index 2837d800421..bf0eac370aa 100644 --- a/src/vnet/pg/stream.c +++ b/src/vnet/pg/stream.c @@ -205,7 +205,7 @@ pg_interface_add_or_get (pg_main_t * pg, uword if_id) rnd = (u32) (now * 1e6); rnd = random_u32 (&rnd); - clib_memcpy (hw_addr + 2, &rnd, sizeof (rnd)); + clib_memcpy_fast (hw_addr + 2, &rnd, sizeof (rnd)); hw_addr[0] = 2; hw_addr[1] = 0xfe; @@ -333,10 +333,10 @@ pg_edit_group_get_fixed_packet_data (pg_stream_t * s, vec_foreach (e, g->edits) do_edit (s, g, e, /* want_commit */ 0); - clib_memcpy (packet_data, g->fixed_packet_data, - vec_len (g->fixed_packet_data)); - clib_memcpy (packet_data_mask, g->fixed_packet_data_mask, - vec_len (g->fixed_packet_data_mask)); + clib_memcpy_fast (packet_data, g->fixed_packet_data, + vec_len (g->fixed_packet_data)); + clib_memcpy_fast (packet_data_mask, g->fixed_packet_data_mask, + vec_len (g->fixed_packet_data_mask)); } static void diff --git a/src/vnet/sctp/sctp_input.c b/src/vnet/sctp/sctp_input.c index e548707d32a..c58da9ef9b5 100644 --- a/src/vnet/sctp/sctp_input.c +++ b/src/vnet/sctp/sctp_input.c @@ -245,15 +245,15 @@ sctp_set_rx_trace_data (sctp_rx_trace_t * rx_trace, { if (sctp_conn) { - clib_memcpy (&rx_trace->sctp_connection, sctp_conn, - sizeof (rx_trace->sctp_connection)); + clib_memcpy_fast (&rx_trace->sctp_connection, sctp_conn, + sizeof (rx_trace->sctp_connection)); } else { sctp_hdr = sctp_buffer_hdr (b0); } - clib_memcpy (&rx_trace->sctp_header, sctp_hdr, - sizeof (rx_trace->sctp_header)); + clib_memcpy_fast (&rx_trace->sctp_header, sctp_hdr, + sizeof (rx_trace->sctp_header)); } always_inline u16 @@ -398,8 +398,8 @@ sctp_handle_init (sctp_header_t * sctp_hdr, { sctp_ipv4_addr_param_t *ipv4 = (sctp_ipv4_addr_param_t *) opt_params_hdr; - clib_memcpy (&ip4_addr, &ipv4->address, - sizeof (ip4_address_t)); + clib_memcpy_fast (&ip4_addr, &ipv4->address, + sizeof (ip4_address_t)); if (sctp_sub_connection_add_ip4 (vlib_get_main (), &sctp_conn->sub_conn @@ -415,8 +415,8 @@ sctp_handle_init (sctp_header_t * sctp_hdr, { sctp_ipv6_addr_param_t *ipv6 = (sctp_ipv6_addr_param_t *) opt_params_hdr; - clib_memcpy (&ip6_addr, &ipv6->address, - sizeof (ip6_address_t)); + clib_memcpy_fast (&ip6_addr, &ipv6->address, + sizeof (ip6_address_t)); if (sctp_sub_connection_add_ip6 (vlib_get_main (), &sctp_conn->sub_conn @@ -440,8 +440,8 @@ sctp_handle_init (sctp_header_t * sctp_hdr, { sctp_hostname_param_t *hostname_addr = (sctp_hostname_param_t *) opt_params_hdr; - clib_memcpy (hostname, hostname_addr->hostname, - FQDN_MAX_LENGTH); + clib_memcpy_fast (hostname, hostname_addr->hostname, + FQDN_MAX_LENGTH); break; } case SCTP_SUPPORTED_ADDRESS_TYPES: @@ -568,8 +568,9 @@ sctp_handle_init_ack (sctp_header_t * sctp_hdr, sctp_state_cookie_param_t *state_cookie_param = (sctp_state_cookie_param_t *) opt_params_hdr; - clib_memcpy (&(sctp_conn->cookie_param), state_cookie_param, - sizeof (sctp_state_cookie_param_t)); + clib_memcpy_fast (&(sctp_conn->cookie_param), + state_cookie_param, + sizeof (sctp_state_cookie_param_t)); break; } @@ -577,8 +578,8 @@ sctp_handle_init_ack (sctp_header_t * sctp_hdr, { sctp_hostname_param_t *hostname_addr = (sctp_hostname_param_t *) opt_params_hdr; - clib_memcpy (hostname, hostname_addr->hostname, - FQDN_MAX_LENGTH); + clib_memcpy_fast (hostname, hostname_addr->hostname, + FQDN_MAX_LENGTH); break; } case SCTP_UNRECOGNIZED_TYPE: @@ -1035,8 +1036,8 @@ sctp46_rcv_phase_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (error0 == SCTP_ERROR_NONE) { pool_get (tm->connections[my_thread_index], new_sctp_conn); - clib_memcpy (new_sctp_conn, sctp_conn, - sizeof (*new_sctp_conn)); + clib_memcpy_fast (new_sctp_conn, sctp_conn, + sizeof (*new_sctp_conn)); new_sctp_conn->sub_conn[idx].c_c_index = new_sctp_conn - tm->connections[my_thread_index]; new_sctp_conn->sub_conn[idx].c_thread_index = @@ -1447,12 +1448,12 @@ sctp46_shutdown_phase_inline (vlib_main_t * vm, vlib_add_trace (vm, node, b0, sizeof (*sctp_trace)); if (sctp_hdr != NULL) - clib_memcpy (&sctp_trace->sctp_header, sctp_hdr, - sizeof (sctp_trace->sctp_header)); + clib_memcpy_fast (&sctp_trace->sctp_header, sctp_hdr, + sizeof (sctp_trace->sctp_header)); if (sctp_conn != NULL) - clib_memcpy (&sctp_trace->sctp_connection, sctp_conn, - sizeof (sctp_trace->sctp_connection)); + clib_memcpy_fast (&sctp_trace->sctp_connection, sctp_conn, + sizeof (sctp_trace->sctp_connection)); } b0->error = node->errors[error0]; @@ -1718,12 +1719,14 @@ sctp46_listen_process_inline (vlib_main_t * vm, } else { - clib_memcpy (&child_conn-> - sub_conn[SCTP_PRIMARY_PATH_IDX].c_lcl_ip6, - &ip6_hdr->dst_address, sizeof (ip6_address_t)); - clib_memcpy (&child_conn-> - sub_conn[SCTP_PRIMARY_PATH_IDX].c_rmt_ip6, - &ip6_hdr->src_address, sizeof (ip6_address_t)); + clib_memcpy_fast (&child_conn-> + sub_conn[SCTP_PRIMARY_PATH_IDX].c_lcl_ip6, + &ip6_hdr->dst_address, + sizeof (ip6_address_t)); + clib_memcpy_fast (&child_conn-> + sub_conn[SCTP_PRIMARY_PATH_IDX].c_rmt_ip6, + &ip6_hdr->src_address, + sizeof (ip6_address_t)); } sctp_full_hdr_t *full_hdr = (sctp_full_hdr_t *) sctp_hdr; @@ -1795,10 +1798,10 @@ sctp46_listen_process_inline (vlib_main_t * vm, { sctp_rx_trace_t *t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); - clib_memcpy (&t0->sctp_header, sctp_hdr, - sizeof (t0->sctp_header)); - clib_memcpy (&t0->sctp_connection, sctp_listener, - sizeof (t0->sctp_connection)); + clib_memcpy_fast (&t0->sctp_header, sctp_hdr, + sizeof (t0->sctp_header)); + clib_memcpy_fast (&t0->sctp_connection, sctp_listener, + sizeof (t0->sctp_connection)); } b0->error = node->errors[error0]; diff --git a/src/vnet/sctp/sctp_output.c b/src/vnet/sctp/sctp_output.c index aa0bb41e504..c0e1a6eb339 100644 --- a/src/vnet/sctp/sctp_output.c +++ b/src/vnet/sctp/sctp_output.c @@ -584,8 +584,8 @@ sctp_prepare_cookie_echo_chunk (sctp_connection_t * sctp_conn, u8 idx, cookie_echo_chunk->sctp_hdr.verification_tag = sctp_conn->remote_tag; vnet_sctp_set_chunk_type (&cookie_echo_chunk->chunk_hdr, COOKIE_ECHO); vnet_sctp_set_chunk_length (&cookie_echo_chunk->chunk_hdr, chunk_len); - clib_memcpy (&(cookie_echo_chunk->cookie), &sctp_conn->cookie_param, - sizeof (sctp_state_cookie_param_t)); + clib_memcpy_fast (&(cookie_echo_chunk->cookie), &sctp_conn->cookie_param, + sizeof (sctp_state_cookie_param_t)); vnet_buffer (b)->sctp.connection_index = sctp_conn->sub_conn[idx].connection.c_index; @@ -1850,15 +1850,15 @@ sctp46_output_inline (vlib_main_t * vm, t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); if (th0) { - clib_memcpy (&t0->sctp_header, th0, - sizeof (t0->sctp_header)); + clib_memcpy_fast (&t0->sctp_header, th0, + sizeof (t0->sctp_header)); } else { clib_memset (&t0->sctp_header, 0, sizeof (t0->sctp_header)); } - clib_memcpy (&t0->sctp_connection, sctp_conn, - sizeof (t0->sctp_connection)); + clib_memcpy_fast (&t0->sctp_connection, sctp_conn, + sizeof (t0->sctp_connection)); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, diff --git a/src/vnet/session-apps/echo_client.c b/src/vnet/session-apps/echo_client.c index 7935eb8b242..faf8521ce8c 100644 --- a/src/vnet/session-apps/echo_client.c +++ b/src/vnet/session-apps/echo_client.c @@ -86,10 +86,12 @@ send_data_chunk (echo_client_main_t * ecm, eclient_session_t * s) hdr.data_length = rv; hdr.data_offset = 0; - clib_memcpy (&hdr.rmt_ip, &at->rmt_ip, sizeof (ip46_address_t)); + clib_memcpy_fast (&hdr.rmt_ip, &at->rmt_ip, + sizeof (ip46_address_t)); hdr.is_ip4 = at->is_ip4; hdr.rmt_port = at->rmt_port; - clib_memcpy (&hdr.lcl_ip, &at->lcl_ip, sizeof (ip46_address_t)); + clib_memcpy_fast (&hdr.lcl_ip, &at->lcl_ip, + sizeof (ip46_address_t)); hdr.lcl_port = at->lcl_port; svm_fifo_enqueue_nowait (f, sizeof (hdr), (u8 *) & hdr); svm_fifo_enqueue_nocopy (f, rv); @@ -221,10 +223,10 @@ echo_client_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, ASSERT (nconnections_this_batch > 0); vec_validate (connections_this_batch, nconnections_this_batch - 1); - clib_memcpy (connections_this_batch, - connection_indices + vec_len (connection_indices) - - nconnections_this_batch, - nconnections_this_batch * sizeof (u32)); + clib_memcpy_fast (connections_this_batch, + connection_indices + vec_len (connection_indices) + - nconnections_this_batch, + nconnections_this_batch * sizeof (u32)); _vec_len (connection_indices) -= nconnections_this_batch; } @@ -402,8 +404,8 @@ echo_clients_session_connected_callback (u32 app_index, u32 api_context, { transport_connection_t *tc; tc = session_get_transport (s); - clib_memcpy (&session->data.transport, tc, - sizeof (session->data.transport)); + clib_memcpy_fast (&session->data.transport, tc, + sizeof (session->data.transport)); session->data.is_dgram = 1; } diff --git a/src/vnet/session-apps/echo_server.c b/src/vnet/session-apps/echo_server.c index 77710752a16..a4afc5d512f 100644 --- a/src/vnet/session-apps/echo_server.c +++ b/src/vnet/session-apps/echo_server.c @@ -330,13 +330,13 @@ echo_server_attach (u8 * appns_id, u64 appns_flags, u64 appns_secret) clib_memset (a_cert, 0, sizeof (*a_cert)); a_cert->app_index = a->app_index; vec_validate (a_cert->cert, test_srv_crt_rsa_len); - clib_memcpy (a_cert->cert, test_srv_crt_rsa, test_srv_crt_rsa_len); + clib_memcpy_fast (a_cert->cert, test_srv_crt_rsa, test_srv_crt_rsa_len); vnet_app_add_tls_cert (a_cert); clib_memset (a_key, 0, sizeof (*a_key)); a_key->app_index = a->app_index; vec_validate (a_key->key, test_srv_key_rsa_len); - clib_memcpy (a_key->key, test_srv_key_rsa, test_srv_key_rsa_len); + clib_memcpy_fast (a_key->key, test_srv_key_rsa, test_srv_key_rsa_len); vnet_app_add_tls_key (a_key); return 0; } diff --git a/src/vnet/session-apps/http_server.c b/src/vnet/session-apps/http_server.c index 8501cb4ee70..79c2d4a7bfc 100644 --- a/src/vnet/session-apps/http_server.c +++ b/src/vnet/session-apps/http_server.c @@ -132,7 +132,7 @@ http_cli_output (uword arg, u8 * buffer, uword buffer_bytes) offset = vec_len (output_vec); vec_validate (output_vec, offset + buffer_bytes - 1); - clib_memcpy (output_vec + offset, buffer, buffer_bytes); + clib_memcpy_fast (output_vec + offset, buffer, buffer_bytes); *output_vecp = output_vec; } @@ -525,13 +525,13 @@ server_attach () clib_memset (a_cert, 0, sizeof (*a_cert)); a_cert->app_index = a->app_index; vec_validate (a_cert->cert, test_srv_crt_rsa_len); - clib_memcpy (a_cert->cert, test_srv_crt_rsa, test_srv_crt_rsa_len); + clib_memcpy_fast (a_cert->cert, test_srv_crt_rsa, test_srv_crt_rsa_len); vnet_app_add_tls_cert (a_cert); clib_memset (a_key, 0, sizeof (*a_key)); a_key->app_index = a->app_index; vec_validate (a_key->key, test_srv_key_rsa_len); - clib_memcpy (a_key->key, test_srv_key_rsa, test_srv_key_rsa_len); + clib_memcpy_fast (a_key->key, test_srv_key_rsa, test_srv_key_rsa_len); vnet_app_add_tls_key (a_key); return 0; diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h index 2da69fcab92..dc4c46906c4 100644 --- a/src/vnet/session/application_interface.h +++ b/src/vnet/session/application_interface.h @@ -397,10 +397,10 @@ app_send_dgram_raw (svm_fifo_t * f, app_session_transport_t * at, actual_write = clib_min (len, max_enqueue); hdr.data_length = actual_write; hdr.data_offset = 0; - clib_memcpy (&hdr.rmt_ip, &at->rmt_ip, sizeof (ip46_address_t)); + clib_memcpy_fast (&hdr.rmt_ip, &at->rmt_ip, sizeof (ip46_address_t)); hdr.is_ip4 = at->is_ip4; hdr.rmt_port = at->rmt_port; - clib_memcpy (&hdr.lcl_ip, &at->lcl_ip, sizeof (ip46_address_t)); + clib_memcpy_fast (&hdr.lcl_ip, &at->lcl_ip, sizeof (ip46_address_t)); hdr.lcl_port = at->lcl_port; rv = svm_fifo_enqueue_nowait (f, sizeof (hdr), (u8 *) & hdr); ASSERT (rv == sizeof (hdr)); diff --git a/src/vnet/session/mma_template.c b/src/vnet/session/mma_template.c index 15fd7c7aeed..9dd7c76cb06 100644 --- a/src/vnet/session/mma_template.c +++ b/src/vnet/session/mma_template.c @@ -238,7 +238,7 @@ RT (mma_rules_table_del_rule) (RTT (mma_rules_table) * srt, if (i != 0) { vec_add2 (next_indices, new_elts, i); - clib_memcpy (new_elts, rp->next_indices, i * sizeof (u32)); + clib_memcpy_fast (new_elts, rp->next_indices, i * sizeof (u32)); } if (vec_len (child->next_indices)) vec_append (next_indices, child->next_indices); @@ -246,8 +246,8 @@ RT (mma_rules_table_del_rule) (RTT (mma_rules_table) * srt, if (left_to_add) { vec_add2 (next_indices, new_elts, left_to_add); - clib_memcpy (new_elts, &rp->next_indices[i + 1], - left_to_add * sizeof (u32)); + clib_memcpy_fast (new_elts, &rp->next_indices[i + 1], + left_to_add * sizeof (u32)); } RT (mma_rule_free) (srt, child); vec_free (rp->next_indices); diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h index 35c8fb4069f..ea21464ee54 100644 --- a/src/vnet/session/session.h +++ b/src/vnet/session/session.h @@ -563,7 +563,7 @@ session_clone_safe (u32 session_index, u32 thread_index) session_pool_add_peeker (thread_index); new_s = session_alloc (current_thread_index); old_s = session_manager_main.wrk[thread_index].sessions + session_index; - clib_memcpy (new_s, old_s, sizeof (*new_s)); + clib_memcpy_fast (new_s, old_s, sizeof (*new_s)); session_pool_remove_peeker (thread_index); new_s->thread_index = current_thread_index; new_s->session_index = session_get_index (new_s); diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c index eb33db03f33..8315c19a7ac 100755 --- a/src/vnet/session/session_api.c +++ b/src/vnet/session/session_api.c @@ -241,7 +241,7 @@ send_session_accept_callback (stream_session_t * s) tc = tp_vft->get_connection (s->connection_index, s->thread_index); mp->port = tc->rmt_port; mp->is_ip4 = tc->is_ip4; - clib_memcpy (&mp->ip, &tc->rmt_ip, sizeof (tc->rmt_ip)); + clib_memcpy_fast (&mp->ip, &tc->rmt_ip, sizeof (tc->rmt_ip)); } else { @@ -364,7 +364,7 @@ send_session_connected_callback (u32 app_wrk_index, u32 api_context, vpp_queue = session_manager_get_vpp_event_queue (s->thread_index); mp->handle = session_handle (s); mp->vpp_event_queue_address = pointer_to_uword (vpp_queue); - clib_memcpy (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); + clib_memcpy_fast (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); mp->is_ip4 = tc->is_ip4; mp->lcl_port = tc->lcl_port; mp->server_rx_fifo = pointer_to_uword (s->server_rx_fifo); @@ -459,7 +459,7 @@ mq_send_session_accepted_cb (stream_session_t * s) tc = tp_vft->get_connection (s->connection_index, s->thread_index); mp->port = tc->rmt_port; mp->is_ip4 = tc->is_ip4; - clib_memcpy (&mp->ip, &tc->rmt_ip, sizeof (tc->rmt_ip)); + clib_memcpy_fast (&mp->ip, &tc->rmt_ip, sizeof (tc->rmt_ip)); } else { @@ -616,7 +616,7 @@ mq_send_session_connected_cb (u32 app_wrk_index, u32 api_context, vpp_mq = session_manager_get_vpp_event_queue (s->thread_index); mp->handle = session_handle (s); mp->vpp_event_queue_address = pointer_to_uword (vpp_mq); - clib_memcpy (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); + clib_memcpy_fast (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); mp->is_ip4 = tc->is_ip4; mp->lcl_port = tc->lcl_port; mp->server_rx_fifo = pointer_to_uword (s->server_rx_fifo); @@ -692,7 +692,7 @@ mq_send_session_bound_cb (u32 app_wrk_index, u32 api_context, tc = listen_session_get_transport (ls); mp->lcl_port = tc->lcl_port; mp->lcl_is_ip4 = tc->is_ip4; - clib_memcpy (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); + clib_memcpy_fast (mp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); } else { @@ -779,7 +779,8 @@ vl_api_application_attach_t_handler (vl_api_application_attach_t * mp) if (mp->namespace_id_len) { vec_validate (a->namespace_id, mp->namespace_id_len - 1); - clib_memcpy (a->namespace_id, mp->namespace_id, mp->namespace_id_len); + clib_memcpy_fast (a->namespace_id, mp->namespace_id, + mp->namespace_id_len); } if ((error = vnet_application_attach (a))) @@ -906,7 +907,7 @@ done: tc = listen_session_get_transport (s); rmp->lcl_is_ip4 = tc->is_ip4; rmp->lcl_port = tc->lcl_port; - clib_memcpy (rmp->lcl_ip, &tc->lcl_ip, sizeof(tc->lcl_ip)); + clib_memcpy_fast (rmp->lcl_ip, &tc->lcl_ip, sizeof(tc->lcl_ip)); if (session_transport_service_type (s) == TRANSPORT_SERVICE_CL) { rmp->rx_fifo = pointer_to_uword (s->server_rx_fifo); @@ -1208,7 +1209,7 @@ done: { s = listen_session_get_from_handle (a->handle); tc = listen_session_get_transport (s); - clib_memcpy (rmp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); + clib_memcpy_fast (rmp->lcl_ip, &tc->lcl_ip, sizeof (tc->lcl_ip)); if (session_transport_service_type (s) == TRANSPORT_SERVICE_CL) { rmp->rx_fifo = pointer_to_uword (s->server_rx_fifo); @@ -1295,7 +1296,8 @@ vl_api_connect_sock_t_handler (vl_api_connect_sock_t * mp) if (mp->hostname_len) { vec_validate (a->sep_ext.hostname, mp->hostname_len - 1); - clib_memcpy (a->sep_ext.hostname, mp->hostname, mp->hostname_len); + clib_memcpy_fast (a->sep_ext.hostname, mp->hostname, + mp->hostname_len); } a->api_context = mp->context; a->app_index = app->app_index; @@ -1429,7 +1431,7 @@ vl_api_app_namespace_add_del_t_handler (vl_api_app_namespace_add_del_t * mp) } vec_validate (ns_id, mp->namespace_id_len - 1); - clib_memcpy (ns_id, mp->namespace_id, mp->namespace_id_len); + clib_memcpy_fast (ns_id, mp->namespace_id, mp->namespace_id_len); vnet_app_namespace_add_del_args_t args = { .ns_id = ns_id, .secret = clib_net_to_host_u64 (mp->secret), @@ -1522,8 +1524,8 @@ send_session_rule_details4 (mma_rule_16_t * rule, u8 is_local, rmp->context = context; rmp->is_ip4 = 1; - clib_memcpy (rmp->lcl_ip, &match->lcl_ip, sizeof (match->lcl_ip)); - clib_memcpy (rmp->rmt_ip, &match->rmt_ip, sizeof (match->rmt_ip)); + clib_memcpy_fast (rmp->lcl_ip, &match->lcl_ip, sizeof (match->lcl_ip)); + clib_memcpy_fast (rmp->rmt_ip, &match->rmt_ip, sizeof (match->rmt_ip)); rmp->lcl_plen = ip4_mask_to_preflen (&mask->lcl_ip); rmp->rmt_plen = ip4_mask_to_preflen (&mask->rmt_ip); rmp->lcl_port = match->lcl_port; @@ -1535,7 +1537,7 @@ send_session_rule_details4 (mma_rule_16_t * rule, u8 is_local, rmp->appns_index = clib_host_to_net_u32 (appns_index); if (tag) { - clib_memcpy (rmp->tag, tag, vec_len (tag)); + clib_memcpy_fast (rmp->tag, tag, vec_len (tag)); rmp->tag[vec_len (tag)] = 0; } @@ -1559,8 +1561,8 @@ send_session_rule_details6 (mma_rule_40_t * rule, u8 is_local, rmp->context = context; rmp->is_ip4 = 0; - clib_memcpy (rmp->lcl_ip, &match->lcl_ip, sizeof (match->lcl_ip)); - clib_memcpy (rmp->rmt_ip, &match->rmt_ip, sizeof (match->rmt_ip)); + clib_memcpy_fast (rmp->lcl_ip, &match->lcl_ip, sizeof (match->lcl_ip)); + clib_memcpy_fast (rmp->rmt_ip, &match->rmt_ip, sizeof (match->rmt_ip)); rmp->lcl_plen = ip6_mask_to_preflen (&mask->lcl_ip); rmp->rmt_plen = ip6_mask_to_preflen (&mask->rmt_ip); rmp->lcl_port = match->lcl_port; @@ -1572,7 +1574,7 @@ send_session_rule_details6 (mma_rule_40_t * rule, u8 is_local, rmp->appns_index = clib_host_to_net_u32 (appns_index); if (tag) { - clib_memcpy (rmp->tag, tag, vec_len (tag)); + clib_memcpy_fast (rmp->tag, tag, vec_len (tag)); rmp->tag[vec_len (tag)] = 0; } @@ -1671,7 +1673,7 @@ vl_api_application_tls_cert_add_t_handler (vl_api_application_tls_cert_add_t * goto done; } vec_validate (a->cert, cert_len); - clib_memcpy (a->cert, mp->cert, cert_len); + clib_memcpy_fast (a->cert, mp->cert, cert_len); if ((error = vnet_app_add_tls_cert (a))) { rv = clib_error_get_code (error); @@ -1711,7 +1713,7 @@ vl_api_application_tls_key_add_t_handler (vl_api_application_tls_key_add_t * goto done; } vec_validate (a->key, key_len); - clib_memcpy (a->key, mp->key, key_len); + clib_memcpy_fast (a->key, mp->key, key_len); if ((error = vnet_app_add_tls_key (a))) { rv = clib_error_get_code (error); diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c index 22d8d3c45b0..cf2d5766fb0 100644 --- a/src/vnet/session/session_node.c +++ b/src/vnet/session/session_node.c @@ -800,7 +800,7 @@ session_queue_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node, { vec_add2 (fifo_events, e, 1); svm_msg_q_sub_w_lock (mq, msg); - clib_memcpy (e, svm_msg_q_msg_data (mq, msg), sizeof (*e)); + clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), sizeof (*e)); svm_msg_q_free_msg (mq, msg); } svm_msg_q_unlock (mq); @@ -961,7 +961,7 @@ dump_thread_0_event_queue (void) { msg = (svm_msg_q_msg_t *) (&mq->q->data[0] + mq->q->elsize * index); ring = svm_msg_q_ring (mq, msg->ring_index); - clib_memcpy (e, svm_msg_q_msg_data (mq, msg), ring->elsize); + clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), ring->elsize); switch (e->event_type) { @@ -1053,7 +1053,7 @@ session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e) { msg = (svm_msg_q_msg_t *) (&mq->q->data[0] + mq->q->elsize * index); ring = svm_msg_q_ring (mq, msg->ring_index); - clib_memcpy (e, svm_msg_q_msg_data (mq, msg), ring->elsize); + clib_memcpy_fast (e, svm_msg_q_msg_data (mq, msg), ring->elsize); found = session_node_cmp_event (e, f); if (found) return 1; @@ -1069,7 +1069,7 @@ session_node_lookup_fifo_event (svm_fifo_t * f, session_event_t * e) found = session_node_cmp_event (evt, f); if (found) { - clib_memcpy (e, evt, sizeof (*evt)); + clib_memcpy_fast (e, evt, sizeof (*evt)); break; } } diff --git a/src/vnet/session/session_rules_table.c b/src/vnet/session/session_rules_table.c index 33f7de07944..6304606ca43 100644 --- a/src/vnet/session/session_rules_table.c +++ b/src/vnet/session/session_rules_table.c @@ -270,8 +270,10 @@ session_rules_table_init_rule_40 (mma_rule_40_t * rule, fib_pref_normalize (lcl); fib_pref_normalize (rmt); match = (session_mask_or_match_6_t *) & rule->match; - clib_memcpy (&match->lcl_ip, &lcl->fp_addr.ip6, sizeof (match->lcl_ip)); - clib_memcpy (&match->rmt_ip, &rmt->fp_addr.ip6, sizeof (match->rmt_ip)); + clib_memcpy_fast (&match->lcl_ip, &lcl->fp_addr.ip6, + sizeof (match->lcl_ip)); + clib_memcpy_fast (&match->rmt_ip, &rmt->fp_addr.ip6, + sizeof (match->rmt_ip)); match->lcl_port = lcl_port; match->rmt_port = rmt_port; mask = (session_mask_or_match_6_t *) & rule->mask; @@ -355,8 +357,8 @@ session_rules_table_lookup_rule6 (session_rules_table_t * srt, .lcl_port = lcl_port, .rmt_port = rmt_port, }; - clib_memcpy (&key.lcl_ip, lcl_ip, sizeof (*lcl_ip)); - clib_memcpy (&key.rmt_ip, rmt_ip, sizeof (*rmt_ip)); + clib_memcpy_fast (&key.lcl_ip, lcl_ip, sizeof (*lcl_ip)); + clib_memcpy_fast (&key.rmt_ip, rmt_ip, sizeof (*rmt_ip)); return mma_rules_table_lookup_rule_40 (srt6, (mma_mask_or_match_40_t *) & key, srt6->root_index); @@ -372,8 +374,8 @@ session_rules_table_lookup6 (session_rules_table_t * srt, .lcl_port = lcl_port, .rmt_port = rmt_port, }; - clib_memcpy (&key.lcl_ip, lcl_ip, sizeof (*lcl_ip)); - clib_memcpy (&key.rmt_ip, rmt_ip, sizeof (*rmt_ip)); + clib_memcpy_fast (&key.lcl_ip, lcl_ip, sizeof (*lcl_ip)); + clib_memcpy_fast (&key.rmt_ip, rmt_ip, sizeof (*rmt_ip)); return mma_rules_table_lookup_40 (srt6, (mma_mask_or_match_40_t *) & key, srt6->root_index); } @@ -577,8 +579,8 @@ session_rules_table_show_rule (vlib_main_t * vm, session_rules_table_t * srt, .lcl_port = lcl_port, .rmt_port = rmt_port, }; - clib_memcpy (&key.lcl_ip, &lcl_ip->ip6, sizeof (lcl_ip->ip6)); - clib_memcpy (&key.rmt_ip, &rmt_ip->ip6, sizeof (rmt_ip->ip6)); + clib_memcpy_fast (&key.lcl_ip, &lcl_ip->ip6, sizeof (lcl_ip->ip6)); + clib_memcpy_fast (&key.rmt_ip, &rmt_ip->ip6, sizeof (rmt_ip->ip6)); ri = mma_rules_table_lookup_rule_40 (srt6, (mma_mask_or_match_40_t *) & key, srt6->root_index); diff --git a/src/vnet/session/transport.c b/src/vnet/session/transport.c index 06c541a9e2b..91b80d7c11e 100644 --- a/src/vnet/session/transport.c +++ b/src/vnet/session/transport.c @@ -319,7 +319,7 @@ transport_endpoint_mark_used (u8 proto, ip46_address_t * ip, u16 port) transport_endpoint_t *tep; clib_spinlock_lock_if_init (&local_endpoints_lock); tep = transport_endpoint_new (); - clib_memcpy (&tep->ip, ip, sizeof (*ip)); + clib_memcpy_fast (&tep->ip, ip, sizeof (*ip)); tep->port = port; transport_endpoint_table_add (&local_endpoints_table, proto, tep, tep - local_endpoints); @@ -388,7 +388,7 @@ transport_get_interface_ip (u32 sw_if_index, u8 is_ip4, ip46_address_t * addr) return clib_error_return (0, "no routable ip6 addresses on %U", format_vnet_sw_if_index_name, vnet_get_main (), sw_if_index); - clib_memcpy (&addr->ip6, ip6, sizeof (*ip6)); + clib_memcpy_fast (&addr->ip6, ip6, sizeof (*ip6)); } return 0; } @@ -404,7 +404,7 @@ transport_find_local_ip_for_remote (u32 sw_if_index, if (sw_if_index == ENDPOINT_INVALID_INDEX) { /* Find a FIB path to the destination */ - clib_memcpy (&prefix.fp_addr, &rmt->ip, sizeof (rmt->ip)); + clib_memcpy_fast (&prefix.fp_addr, &rmt->ip, sizeof (rmt->ip)); prefix.fp_proto = rmt->is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6; prefix.fp_len = rmt->is_ip4 ? 32 : 128; @@ -449,7 +449,8 @@ transport_alloc_local_endpoint (u8 proto, transport_endpoint_cfg_t * rmt_cfg, else { /* Assume session layer vetted this address */ - clib_memcpy (lcl_addr, &rmt_cfg->peer.ip, sizeof (rmt_cfg->peer.ip)); + clib_memcpy_fast (lcl_addr, &rmt_cfg->peer.ip, + sizeof (rmt_cfg->peer.ip)); } /* diff --git a/src/vnet/srp/interface.c b/src/vnet/srp/interface.c index 28e519986f0..3c98f416781 100644 --- a/src/vnet/srp/interface.c +++ b/src/vnet/srp/interface.c @@ -289,7 +289,7 @@ simulated_srp_interface_tx (vlib_main_t * vm, n_copy = clib_min (n_left_from, n_left_to_next); - clib_memcpy (to_next, from, n_copy * sizeof (from[0])); + clib_memcpy_fast (to_next, from, n_copy * sizeof (from[0])); n_left_to_next -= n_copy; n_left_from -= n_copy; for (i = 0; i < n_copy; i++) diff --git a/src/vnet/srp/node.c b/src/vnet/srp/node.c index 7cbc71b39ae..cec014f987d 100644 --- a/src/vnet/srp/node.c +++ b/src/vnet/srp/node.c @@ -309,7 +309,7 @@ srp_topology_packet (vlib_main_t * vm, u32 sw_if_index, u8 ** contents) return SRP_ERROR_TOPOLOGY_BAD_LENGTH; /* Fill in our source MAC address. */ - clib_memcpy (t->ethernet.src_address, hi->hw_address, vec_len (hi->hw_address)); + clib_memcpy_fast (t->ethernet.src_address, hi->hw_address, vec_len (hi->hw_address)); /* Make space for our MAC binding. */ vec_resize (*contents, sizeof (srp_topology_mac_binding_t)); @@ -321,7 +321,7 @@ srp_topology_packet (vlib_main_t * vm, u32 sw_if_index, u8 ** contents) mb->flags = ((t->srp.is_inner_ring ? SRP_TOPOLOGY_MAC_BINDING_FLAG_IS_INNER_RING : 0) | (/* is wrapped FIXME */ 0)); - clib_memcpy (mb->address, hi->hw_address, vec_len (hi->hw_address)); + clib_memcpy_fast (mb->address, hi->hw_address, vec_len (hi->hw_address)); t->control.checksum = ~ip_csum_fold (ip_incremental_checksum (0, &t->control, @@ -588,7 +588,7 @@ static void init_ips_packet (srp_interface_t * si, i->srp.mode = SRP_MODE_control_locally_buffered_for_host; srp_header_compute_parity (&i->srp); - clib_memcpy (&i->ethernet.src_address, &si->my_address, sizeof (si->my_address)); + clib_memcpy_fast (&i->ethernet.src_address, &si->my_address, sizeof (si->my_address)); i->ethernet.type = clib_host_to_net_u16 (ETHERNET_TYPE_SRP_CONTROL); /* Checksum will be filled in later. */ @@ -596,7 +596,7 @@ static void init_ips_packet (srp_interface_t * si, i->control.type = SRP_CONTROL_PACKET_TYPE_ips; i->control.ttl = 255; - clib_memcpy (&i->originator_address, &si->my_address, sizeof (si->my_address)); + clib_memcpy_fast (&i->originator_address, &si->my_address, sizeof (si->my_address)); } static void tx_ips_packet (srp_interface_t * si, @@ -680,7 +680,7 @@ void srp_ips_rx_packet (u32 sw_if_index, srp_ips_header_t * h) ASSERT (0); } ir->rx_neighbor_address_valid = 1; - clib_memcpy (ir->rx_neighbor_address, h->originator_address, sizeof (ir->rx_neighbor_address)); + clib_memcpy_fast (ir->rx_neighbor_address, h->originator_address, sizeof (ir->rx_neighbor_address)); } switch (si->current_ips_state) diff --git a/src/vnet/srv6/sr.h b/src/vnet/srv6/sr.h index 1a30cf3e3e0..2353b284363 100755 --- a/src/vnet/srv6/sr.h +++ b/src/vnet/srv6/sr.h @@ -310,7 +310,8 @@ ip6_sr_compute_rewrite_string_insert (ip6_address_t * sl) addrp = srh->segments + vec_len (sl); vec_foreach (this_address, sl) { - clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t)); + clib_memcpy_fast (addrp->as_u8, this_address->as_u8, + sizeof (ip6_address_t)); addrp--; } return rs; diff --git a/src/vnet/srv6/sr_policy_rewrite.c b/src/vnet/srv6/sr_policy_rewrite.c index c80b9c4505a..e9c42211209 100755 --- a/src/vnet/srv6/sr_policy_rewrite.c +++ b/src/vnet/srv6/sr_policy_rewrite.c @@ -115,7 +115,7 @@ static ip6_address_t sr_pr_encaps_src; void sr_set_source (ip6_address_t * address) { - clib_memcpy (&sr_pr_encaps_src, address, sizeof (sr_pr_encaps_src)); + clib_memcpy_fast (&sr_pr_encaps_src, address, sizeof (sr_pr_encaps_src)); } static clib_error_t * @@ -192,8 +192,8 @@ compute_rewrite_encaps (ip6_address_t * sl) addrp = srh->segments + vec_len (sl) - 1; vec_foreach (this_address, sl) { - clib_memcpy (addrp->as_u8, this_address->as_u8, - sizeof (ip6_address_t)); + clib_memcpy_fast (addrp->as_u8, this_address->as_u8, + sizeof (ip6_address_t)); addrp--; } } @@ -234,7 +234,8 @@ compute_rewrite_insert (ip6_address_t * sl) addrp = srh->segments + vec_len (sl); vec_foreach (this_address, sl) { - clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t)); + clib_memcpy_fast (addrp->as_u8, this_address->as_u8, + sizeof (ip6_address_t)); addrp--; } return rs; @@ -272,7 +273,8 @@ compute_rewrite_bsid (ip6_address_t * sl) addrp = srh->segments + vec_len (sl) - 1; vec_foreach (this_address, sl) { - clib_memcpy (addrp->as_u8, this_address->as_u8, sizeof (ip6_address_t)); + clib_memcpy_fast (addrp->as_u8, this_address->as_u8, + sizeof (ip6_address_t)); addrp--; } return rs; @@ -589,7 +591,7 @@ sr_policy_add (ip6_address_t * bsid, ip6_address_t * segments, /* Add an SR policy object */ pool_get (sm->sr_policies, sr_policy); clib_memset (sr_policy, 0, sizeof (*sr_policy)); - clib_memcpy (&sr_policy->bsid, bsid, sizeof (ip6_address_t)); + clib_memcpy_fast (&sr_policy->bsid, bsid, sizeof (ip6_address_t)); sr_policy->type = behavior; sr_policy->fib_table = (fib_table != (u32) ~ 0 ? fib_table : 0); //Is default FIB 0 ? sr_policy->is_encap = is_encap; @@ -852,8 +854,8 @@ sr_policy_command_fn (vlib_main_t * vm, unformat_input_t * input, if (unformat (input, "next %U", unformat_ip6_address, &next_address)) { vec_add2 (segments, this_seg, 1); - clib_memcpy (this_seg->as_u8, next_address.as_u8, - sizeof (*this_seg)); + clib_memcpy_fast (this_seg->as_u8, next_address.as_u8, + sizeof (*this_seg)); } else if (unformat (input, "add sl")) operation = 1; @@ -1159,14 +1161,14 @@ sr_policy_rewrite_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, ip2_encap = vlib_buffer_get_current (b2); ip3_encap = vlib_buffer_get_current (b3); - clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), - sl0->rewrite, vec_len (sl0->rewrite)); - clib_memcpy (((u8 *) ip1_encap) - vec_len (sl1->rewrite), - sl1->rewrite, vec_len (sl1->rewrite)); - clib_memcpy (((u8 *) ip2_encap) - vec_len (sl2->rewrite), - sl2->rewrite, vec_len (sl2->rewrite)); - clib_memcpy (((u8 *) ip3_encap) - vec_len (sl3->rewrite), - sl3->rewrite, vec_len (sl3->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) ip1_encap) - vec_len (sl1->rewrite), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy_fast (((u8 *) ip2_encap) - vec_len (sl2->rewrite), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy_fast (((u8 *) ip3_encap) - vec_len (sl3->rewrite), + sl3->rewrite, vec_len (sl3->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); @@ -1189,40 +1191,40 @@ sr_policy_rewrite_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b3, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } } @@ -1257,8 +1259,8 @@ sr_policy_rewrite_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, ip0_encap = vlib_buffer_get_current (b0); - clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), - sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); ip0 = vlib_buffer_get_current (b0); @@ -1270,10 +1272,10 @@ sr_policy_rewrite_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } encap_pkts++; @@ -1442,14 +1444,14 @@ sr_policy_rewrite_encaps_v4 (vlib_main_t * vm, vlib_node_runtime_t * node, ip2_encap = vlib_buffer_get_current (b2); ip3_encap = vlib_buffer_get_current (b3); - clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), - sl0->rewrite, vec_len (sl0->rewrite)); - clib_memcpy (((u8 *) ip1_encap) - vec_len (sl1->rewrite), - sl1->rewrite, vec_len (sl1->rewrite)); - clib_memcpy (((u8 *) ip2_encap) - vec_len (sl2->rewrite), - sl2->rewrite, vec_len (sl2->rewrite)); - clib_memcpy (((u8 *) ip3_encap) - vec_len (sl3->rewrite), - sl3->rewrite, vec_len (sl3->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) ip1_encap) - vec_len (sl1->rewrite), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy_fast (((u8 *) ip2_encap) - vec_len (sl2->rewrite), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy_fast (((u8 *) ip3_encap) - vec_len (sl3->rewrite), + sl3->rewrite, vec_len (sl3->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); @@ -1472,40 +1474,40 @@ sr_policy_rewrite_encaps_v4 (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b3, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } } @@ -1541,8 +1543,8 @@ sr_policy_rewrite_encaps_v4 (vlib_main_t * vm, vlib_node_runtime_t * node, ip0_encap = vlib_buffer_get_current (b0); - clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), - sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); ip0 = vlib_buffer_get_current (b0); @@ -1554,10 +1556,10 @@ sr_policy_rewrite_encaps_v4 (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } encap_pkts++; @@ -1799,14 +1801,14 @@ sr_policy_rewrite_encaps_l2 (vlib_main_t * vm, vlib_node_runtime_t * node, en2 = vlib_buffer_get_current (b2); en3 = vlib_buffer_get_current (b3); - clib_memcpy (((u8 *) en0) - vec_len (sl0->rewrite), sl0->rewrite, - vec_len (sl0->rewrite)); - clib_memcpy (((u8 *) en1) - vec_len (sl1->rewrite), sl1->rewrite, - vec_len (sl1->rewrite)); - clib_memcpy (((u8 *) en2) - vec_len (sl2->rewrite), sl2->rewrite, - vec_len (sl2->rewrite)); - clib_memcpy (((u8 *) en3) - vec_len (sl3->rewrite), sl3->rewrite, - vec_len (sl3->rewrite)); + clib_memcpy_fast (((u8 *) en0) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) en1) - vec_len (sl1->rewrite), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy_fast (((u8 *) en2) - vec_len (sl2->rewrite), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy_fast (((u8 *) en3) - vec_len (sl3->rewrite), + sl3->rewrite, vec_len (sl3->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); @@ -1868,40 +1870,40 @@ sr_policy_rewrite_encaps_l2 (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b3, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } } @@ -1955,8 +1957,8 @@ sr_policy_rewrite_encaps_l2 (vlib_main_t * vm, vlib_node_runtime_t * node, en0 = vlib_buffer_get_current (b0); - clib_memcpy (((u8 *) en0) - vec_len (sl0->rewrite), sl0->rewrite, - vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) en0) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); @@ -1978,10 +1980,10 @@ sr_policy_rewrite_encaps_l2 (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } encap_pkts++; @@ -2145,23 +2147,23 @@ sr_policy_rewrite_insert (vlib_main_t * vm, vlib_node_runtime_t * node, else sr3 = (ip6_sr_header_t *) (ip3 + 1); - clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite), (u8 *) ip0, - (void *) sr0 - (void *) ip0); - clib_memcpy ((u8 *) ip1 - vec_len (sl1->rewrite), (u8 *) ip1, - (void *) sr1 - (void *) ip1); - clib_memcpy ((u8 *) ip2 - vec_len (sl2->rewrite), (u8 *) ip2, - (void *) sr2 - (void *) ip2); - clib_memcpy ((u8 *) ip3 - vec_len (sl3->rewrite), (u8 *) ip3, - (void *) sr3 - (void *) ip3); - - clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite)), sl0->rewrite, - vec_len (sl0->rewrite)); - clib_memcpy (((u8 *) sr1 - vec_len (sl1->rewrite)), sl1->rewrite, - vec_len (sl1->rewrite)); - clib_memcpy (((u8 *) sr2 - vec_len (sl2->rewrite)), sl2->rewrite, - vec_len (sl2->rewrite)); - clib_memcpy (((u8 *) sr3 - vec_len (sl3->rewrite)), sl3->rewrite, - vec_len (sl3->rewrite)); + clib_memcpy_fast ((u8 *) ip0 - vec_len (sl0->rewrite), (u8 *) ip0, + (void *) sr0 - (void *) ip0); + clib_memcpy_fast ((u8 *) ip1 - vec_len (sl1->rewrite), (u8 *) ip1, + (void *) sr1 - (void *) ip1); + clib_memcpy_fast ((u8 *) ip2 - vec_len (sl2->rewrite), (u8 *) ip2, + (void *) sr2 - (void *) ip2); + clib_memcpy_fast ((u8 *) ip3 - vec_len (sl3->rewrite), (u8 *) ip3, + (void *) sr3 - (void *) ip3); + + clib_memcpy_fast (((u8 *) sr0 - vec_len (sl0->rewrite)), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) sr1 - vec_len (sl1->rewrite)), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy_fast (((u8 *) sr2 - vec_len (sl2->rewrite)), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy_fast (((u8 *) sr3 - vec_len (sl3->rewrite)), + sl3->rewrite, vec_len (sl3->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); @@ -2284,40 +2286,40 @@ sr_policy_rewrite_insert (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b3, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } } @@ -2360,10 +2362,10 @@ sr_policy_rewrite_insert (vlib_main_t * vm, vlib_node_runtime_t * node, else sr0 = (ip6_sr_header_t *) (ip0 + 1); - clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite), (u8 *) ip0, - (void *) sr0 - (void *) ip0); - clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite)), sl0->rewrite, - vec_len (sl0->rewrite)); + clib_memcpy_fast ((u8 *) ip0 - vec_len (sl0->rewrite), (u8 *) ip0, + (void *) sr0 - (void *) ip0); + clib_memcpy_fast (((u8 *) sr0 - vec_len (sl0->rewrite)), + sl0->rewrite, vec_len (sl0->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); @@ -2400,10 +2402,10 @@ sr_policy_rewrite_insert (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } insert_pkts++; @@ -2567,23 +2569,23 @@ sr_policy_rewrite_b_insert (vlib_main_t * vm, vlib_node_runtime_t * node, else sr3 = (ip6_sr_header_t *) (ip3 + 1); - clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite_bsid), (u8 *) ip0, - (void *) sr0 - (void *) ip0); - clib_memcpy ((u8 *) ip1 - vec_len (sl1->rewrite_bsid), (u8 *) ip1, - (void *) sr1 - (void *) ip1); - clib_memcpy ((u8 *) ip2 - vec_len (sl2->rewrite_bsid), (u8 *) ip2, - (void *) sr2 - (void *) ip2); - clib_memcpy ((u8 *) ip3 - vec_len (sl3->rewrite_bsid), (u8 *) ip3, - (void *) sr3 - (void *) ip3); - - clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite_bsid)), - sl0->rewrite_bsid, vec_len (sl0->rewrite_bsid)); - clib_memcpy (((u8 *) sr1 - vec_len (sl1->rewrite_bsid)), - sl1->rewrite_bsid, vec_len (sl1->rewrite_bsid)); - clib_memcpy (((u8 *) sr2 - vec_len (sl2->rewrite_bsid)), - sl2->rewrite_bsid, vec_len (sl2->rewrite_bsid)); - clib_memcpy (((u8 *) sr3 - vec_len (sl3->rewrite_bsid)), - sl3->rewrite_bsid, vec_len (sl3->rewrite_bsid)); + clib_memcpy_fast ((u8 *) ip0 - vec_len (sl0->rewrite_bsid), + (u8 *) ip0, (void *) sr0 - (void *) ip0); + clib_memcpy_fast ((u8 *) ip1 - vec_len (sl1->rewrite_bsid), + (u8 *) ip1, (void *) sr1 - (void *) ip1); + clib_memcpy_fast ((u8 *) ip2 - vec_len (sl2->rewrite_bsid), + (u8 *) ip2, (void *) sr2 - (void *) ip2); + clib_memcpy_fast ((u8 *) ip3 - vec_len (sl3->rewrite_bsid), + (u8 *) ip3, (void *) sr3 - (void *) ip3); + + clib_memcpy_fast (((u8 *) sr0 - vec_len (sl0->rewrite_bsid)), + sl0->rewrite_bsid, vec_len (sl0->rewrite_bsid)); + clib_memcpy_fast (((u8 *) sr1 - vec_len (sl1->rewrite_bsid)), + sl1->rewrite_bsid, vec_len (sl1->rewrite_bsid)); + clib_memcpy_fast (((u8 *) sr2 - vec_len (sl2->rewrite_bsid)), + sl2->rewrite_bsid, vec_len (sl2->rewrite_bsid)); + clib_memcpy_fast (((u8 *) sr3 - vec_len (sl3->rewrite_bsid)), + sl3->rewrite_bsid, vec_len (sl3->rewrite_bsid)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite_bsid)); vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite_bsid)); @@ -2697,40 +2699,40 @@ sr_policy_rewrite_b_insert (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b3, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } } @@ -2773,10 +2775,10 @@ sr_policy_rewrite_b_insert (vlib_main_t * vm, vlib_node_runtime_t * node, else sr0 = (ip6_sr_header_t *) (ip0 + 1); - clib_memcpy ((u8 *) ip0 - vec_len (sl0->rewrite_bsid), (u8 *) ip0, - (void *) sr0 - (void *) ip0); - clib_memcpy (((u8 *) sr0 - vec_len (sl0->rewrite_bsid)), - sl0->rewrite_bsid, vec_len (sl0->rewrite_bsid)); + clib_memcpy_fast ((u8 *) ip0 - vec_len (sl0->rewrite_bsid), + (u8 *) ip0, (void *) sr0 - (void *) ip0); + clib_memcpy_fast (((u8 *) sr0 - vec_len (sl0->rewrite_bsid)), + sl0->rewrite_bsid, vec_len (sl0->rewrite_bsid)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite_bsid)); @@ -2811,10 +2813,10 @@ sr_policy_rewrite_b_insert (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } insert_pkts++; @@ -2997,14 +2999,14 @@ sr_policy_rewrite_b_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, end_bsid_encaps_srh_processing (node, b2, ip2_encap, sr2, &next2); end_bsid_encaps_srh_processing (node, b3, ip3_encap, sr3, &next3); - clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), - sl0->rewrite, vec_len (sl0->rewrite)); - clib_memcpy (((u8 *) ip1_encap) - vec_len (sl1->rewrite), - sl1->rewrite, vec_len (sl1->rewrite)); - clib_memcpy (((u8 *) ip2_encap) - vec_len (sl2->rewrite), - sl2->rewrite, vec_len (sl2->rewrite)); - clib_memcpy (((u8 *) ip3_encap) - vec_len (sl3->rewrite), - sl3->rewrite, vec_len (sl3->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) ip1_encap) - vec_len (sl1->rewrite), + sl1->rewrite, vec_len (sl1->rewrite)); + clib_memcpy_fast (((u8 *) ip2_encap) - vec_len (sl2->rewrite), + sl2->rewrite, vec_len (sl2->rewrite)); + clib_memcpy_fast (((u8 *) ip3_encap) - vec_len (sl3->rewrite), + sl3->rewrite, vec_len (sl3->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); vlib_buffer_advance (b1, -(word) vec_len (sl1->rewrite)); @@ -3027,40 +3029,40 @@ sr_policy_rewrite_b_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip1->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip1->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip1->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip1->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b2->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b2, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip2->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip2->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip2->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip2->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } if (PREDICT_FALSE (b3->flags & VLIB_BUFFER_IS_TRACED)) { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b3, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip3->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip3->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip3->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip3->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } } @@ -3100,8 +3102,8 @@ sr_policy_rewrite_b_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, IP_PROTOCOL_IPV6_ROUTE); end_bsid_encaps_srh_processing (node, b0, ip0_encap, sr0, &next0); - clib_memcpy (((u8 *) ip0_encap) - vec_len (sl0->rewrite), - sl0->rewrite, vec_len (sl0->rewrite)); + clib_memcpy_fast (((u8 *) ip0_encap) - vec_len (sl0->rewrite), + sl0->rewrite, vec_len (sl0->rewrite)); vlib_buffer_advance (b0, -(word) vec_len (sl0->rewrite)); ip0 = vlib_buffer_get_current (b0); @@ -3113,10 +3115,10 @@ sr_policy_rewrite_b_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, { sr_policy_rewrite_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); - clib_memcpy (tr->src.as_u8, ip0->src_address.as_u8, - sizeof (tr->src.as_u8)); - clib_memcpy (tr->dst.as_u8, ip0->dst_address.as_u8, - sizeof (tr->dst.as_u8)); + clib_memcpy_fast (tr->src.as_u8, ip0->src_address.as_u8, + sizeof (tr->src.as_u8)); + clib_memcpy_fast (tr->dst.as_u8, ip0->dst_address.as_u8, + sizeof (tr->dst.as_u8)); } encap_pkts++; diff --git a/src/vnet/srv6/sr_steering.c b/src/vnet/srv6/sr_steering.c index d44bbae576f..66186e9fca8 100755 --- a/src/vnet/srv6/sr_steering.c +++ b/src/vnet/srv6/sr_steering.c @@ -249,8 +249,8 @@ sr_steering_policy (int is_del, ip6_address_t * bsid, u32 sr_policy_index, if (traffic_type == SR_STEER_IPV4 || traffic_type == SR_STEER_IPV6) { - clib_memcpy (&steer_pl->classify.l3.prefix, prefix, - sizeof (ip46_address_t)); + clib_memcpy_fast (&steer_pl->classify.l3.prefix, prefix, + sizeof (ip46_address_t)); steer_pl->classify.l3.mask_width = mask_width; steer_pl->classify.l3.fib_table = (table_id != (u32) ~ 0 ? table_id : 0); diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c index 25c2615b101..04613cd6943 100644 --- a/src/vnet/tcp/tcp.c +++ b/src/vnet/tcp/tcp.c @@ -448,10 +448,10 @@ tcp_connection_select_lb_bucket (tcp_connection_t * tc, const dpo_id_t * dpo, ip6_tcp_hdr_t hdr; clib_memset (&hdr, 0, sizeof (hdr)); hdr.ip.protocol = IP_PROTOCOL_TCP; - clib_memcpy (&hdr.ip.src_address, &tc->c_lcl_ip.ip6, - sizeof (ip6_address_t)); - clib_memcpy (&hdr.ip.dst_address, &tc->c_rmt_ip.ip6, - sizeof (ip6_address_t)); + clib_memcpy_fast (&hdr.ip.src_address, &tc->c_lcl_ip.ip6, + sizeof (ip6_address_t)); + clib_memcpy_fast (&hdr.ip.dst_address, &tc->c_rmt_ip.ip6, + sizeof (ip6_address_t)); hdr.tcp.src_port = tc->c_lcl_port; hdr.tcp.dst_port = tc->c_rmt_port; hash = ip6_compute_flow_hash (&hdr.ip, lb->lb_hash_config); @@ -466,7 +466,7 @@ tcp_lookup_rmt_in_fib (tcp_connection_t * tc) fib_prefix_t prefix; u32 fib_index; - clib_memcpy (&prefix.fp_addr, &tc->c_rmt_ip, sizeof (prefix.fp_addr)); + clib_memcpy_fast (&prefix.fp_addr, &tc->c_rmt_ip, sizeof (prefix.fp_addr)); prefix.fp_proto = tc->c_is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6; prefix.fp_len = tc->c_is_ip4 ? 32 : 128; fib_index = fib_table_find (prefix.fp_proto, tc->c_fib_index); @@ -607,8 +607,8 @@ tcp_alloc_custom_local_endpoint (tcp_main_t * tm, ip46_address_t * lcl_addr, index = tm->last_v6_address_rotor++; if (tm->last_v6_address_rotor >= vec_len (tm->ip6_src_addresses)) tm->last_v6_address_rotor = 0; - clib_memcpy (&lcl_addr->ip6, &tm->ip6_src_addresses[index], - sizeof (ip6_address_t)); + clib_memcpy_fast (&lcl_addr->ip6, &tm->ip6_src_addresses[index], + sizeof (ip6_address_t)); } port = transport_alloc_local_port (TRANSPORT_PROTO_TCP, lcl_addr); if (port < 1) diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c index 0c8706567ae..6809a9173df 100644 --- a/src/vnet/tcp/tcp_input.c +++ b/src/vnet/tcp/tcp_input.c @@ -1990,13 +1990,14 @@ tcp_set_rx_trace_data (tcp_rx_trace_t * t0, tcp_connection_t * tc0, { if (tc0) { - clib_memcpy (&t0->tcp_connection, tc0, sizeof (t0->tcp_connection)); + clib_memcpy_fast (&t0->tcp_connection, tc0, + sizeof (t0->tcp_connection)); } else { th0 = tcp_buffer_hdr (b0); } - clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header)); + clib_memcpy_fast (&t0->tcp_header, th0, sizeof (t0->tcp_header)); } static void @@ -2443,7 +2444,7 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node, /* Valid SYN or SYN-ACK. Move connection from half-open pool to * current thread pool. */ pool_get (tm->connections[my_thread_index], new_tc0); - clib_memcpy (new_tc0, tc0, sizeof (*new_tc0)); + clib_memcpy_fast (new_tc0, tc0, sizeof (*new_tc0)); new_tc0->c_c_index = new_tc0 - tm->connections[my_thread_index]; new_tc0->c_thread_index = my_thread_index; new_tc0->rcv_nxt = vnet_buffer (b0)->tcp.seq_end; @@ -2544,8 +2545,9 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (PREDICT_FALSE ((b0->flags & VLIB_BUFFER_IS_TRACED) && tcp0 != 0)) { t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); - clib_memcpy (&t0->tcp_header, tcp0, sizeof (t0->tcp_header)); - clib_memcpy (&t0->tcp_connection, tc0, sizeof (t0->tcp_connection)); + clib_memcpy_fast (&t0->tcp_header, tcp0, sizeof (t0->tcp_header)); + clib_memcpy_fast (&t0->tcp_connection, tc0, + sizeof (t0->tcp_connection)); } } @@ -3085,10 +3087,10 @@ tcp46_listen_inline (vlib_main_t * vm, vlib_node_runtime_t * node, } else { - clib_memcpy (&child0->c_lcl_ip6, &ip60->dst_address, - sizeof (ip6_address_t)); - clib_memcpy (&child0->c_rmt_ip6, &ip60->src_address, - sizeof (ip6_address_t)); + clib_memcpy_fast (&child0->c_lcl_ip6, &ip60->dst_address, + sizeof (ip6_address_t)); + clib_memcpy_fast (&child0->c_rmt_ip6, &ip60->src_address, + sizeof (ip6_address_t)); } if (tcp_options_parse (th0, &child0->rcv_opts)) @@ -3139,8 +3141,9 @@ tcp46_listen_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); - clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header)); - clib_memcpy (&t0->tcp_connection, lc0, sizeof (t0->tcp_connection)); + clib_memcpy_fast (&t0->tcp_header, th0, sizeof (t0->tcp_header)); + clib_memcpy_fast (&t0->tcp_connection, lc0, + sizeof (t0->tcp_connection)); } n_syns += (error0 == TCP_ERROR_NONE); diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index 49156368574..2068e435a2a 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -215,7 +215,7 @@ tcp_options_write (u8 * data, tcp_options_t * opts) *data++ = TCP_OPTION_MSS; *data++ = TCP_OPTION_LEN_MSS; buf = clib_host_to_net_u16 (opts->mss); - clib_memcpy (data, &buf, sizeof (opts->mss)); + clib_memcpy_fast (data, &buf, sizeof (opts->mss)); data += sizeof (opts->mss); opts_len += TCP_OPTION_LEN_MSS; } @@ -240,10 +240,10 @@ tcp_options_write (u8 * data, tcp_options_t * opts) *data++ = TCP_OPTION_TIMESTAMP; *data++ = TCP_OPTION_LEN_TIMESTAMP; buf = clib_host_to_net_u32 (opts->tsval); - clib_memcpy (data, &buf, sizeof (opts->tsval)); + clib_memcpy_fast (data, &buf, sizeof (opts->tsval)); data += sizeof (opts->tsval); buf = clib_host_to_net_u32 (opts->tsecr); - clib_memcpy (data, &buf, sizeof (opts->tsecr)); + clib_memcpy_fast (data, &buf, sizeof (opts->tsecr)); data += sizeof (opts->tsecr); opts_len += TCP_OPTION_LEN_TIMESTAMP; } @@ -261,10 +261,10 @@ tcp_options_write (u8 * data, tcp_options_t * opts) for (i = 0; i < n_sack_blocks; i++) { buf = clib_host_to_net_u32 (opts->sacks[i].start); - clib_memcpy (data, &buf, seq_len); + clib_memcpy_fast (data, &buf, seq_len); data += seq_len; buf = clib_host_to_net_u32 (opts->sacks[i].end); - clib_memcpy (data, &buf, seq_len); + clib_memcpy_fast (data, &buf, seq_len); data += seq_len; } opts_len += 2 + n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK; @@ -764,8 +764,8 @@ tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0, { ih6 = vlib_buffer_get_current (b0); ASSERT ((ih6->ip_version_traffic_class_and_flow_label & 0xF0) == 0x60); - clib_memcpy (&src_ip60, &ih6->src_address, sizeof (ip6_address_t)); - clib_memcpy (&dst_ip60, &ih6->dst_address, sizeof (ip6_address_t)); + clib_memcpy_fast (&src_ip60, &ih6->src_address, sizeof (ip6_address_t)); + clib_memcpy_fast (&dst_ip60, &ih6->dst_address, sizeof (ip6_address_t)); } src_port = th0->src_port; @@ -1175,9 +1175,9 @@ tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b, if (maybe_burst) { - clib_memcpy ((u8 *) (th + 1), - tm->wrk_ctx[tc->c_thread_index].cached_opts, - tc->snd_opts_len); + clib_memcpy_fast ((u8 *) (th + 1), + tm->wrk_ctx[tc->c_thread_index].cached_opts, + tc->snd_opts_len); } else { @@ -2058,8 +2058,8 @@ tcp46_output_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node, tc = tcp_connection_get (vnet_buffer (b)->tcp.connection_index, vm->thread_index); t = vlib_add_trace (vm, node, b, sizeof (*t)); - clib_memcpy (&t->tcp_header, th, sizeof (t->tcp_header)); - clib_memcpy (&t->tcp_connection, tc, sizeof (t->tcp_connection)); + clib_memcpy_fast (&t->tcp_header, th, sizeof (t->tcp_header)); + clib_memcpy_fast (&t->tcp_connection, tc, sizeof (t->tcp_connection)); } } @@ -2322,7 +2322,8 @@ tcp46_send_reset_inline (vlib_main_t * vm, vlib_node_runtime_t * node, else th0 = ip6_next_header ((ip6_header_t *) th0); t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); - clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header)); + clib_memcpy_fast (&t0->tcp_header, th0, + sizeof (t0->tcp_header)); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c index 109280b4be1..7378093bd04 100644 --- a/src/vnet/tls/tls.c +++ b/src/vnet/tls/tls.c @@ -479,7 +479,7 @@ tls_session_connected_callback (u32 tls_app_index, u32 ho_ctx_index, ctx_handle = tls_ctx_alloc (ho_ctx->tls_ctx_engine); ctx = tls_ctx_get (ctx_handle); - clib_memcpy (ctx, ho_ctx, sizeof (*ctx)); + clib_memcpy_fast (ctx, ho_ctx, sizeof (*ctx)); tls_ctx_half_open_reader_unlock (); tls_ctx_half_open_free (ho_ctx_index); @@ -548,7 +548,7 @@ tls_connect (transport_endpoint_cfg_t * tep) app_worker_alloc_connects_segment_manager (app_wrk); ctx->tls_ctx_engine = engine_type; - clib_memcpy (&cargs->sep, sep, sizeof (session_endpoint_t)); + clib_memcpy_fast (&cargs->sep, sep, sizeof (session_endpoint_t)); cargs->sep.transport_proto = TRANSPORT_PROTO_TCP; cargs->app_index = tm->app_index; cargs->api_context = ctx_index; diff --git a/src/vnet/udp/udp.h b/src/vnet/udp/udp.h index 3b6de6ca271..8b94a0088f6 100644 --- a/src/vnet/udp/udp.h +++ b/src/vnet/udp/udp.h @@ -228,7 +228,7 @@ udp_connection_clone_safe (u32 connection_index, u32 thread_index) */ udp_pool_add_peeker (thread_index); old_c = udp_main.connections[thread_index] + connection_index; - clib_memcpy (new_c, old_c, sizeof (*new_c)); + clib_memcpy_fast (new_c, old_c, sizeof (*new_c)); udp_pool_remove_peeker (thread_index); new_c->c_thread_index = current_thread_index; new_c->c_c_index = udp_connection_index (new_c); @@ -343,7 +343,7 @@ ip_udp_encap_one (vlib_main_t * vm, vlib_buffer_t * b0, u8 * ec0, word ec_len, ip0 = vlib_buffer_get_current (b0); /* Apply the encap string. */ - clib_memcpy (ip0, ec0, ec_len); + clib_memcpy_fast (ip0, ec0, ec_len); ip_udp_fixup_one (vm, b0, 1); } else @@ -353,7 +353,7 @@ ip_udp_encap_one (vlib_main_t * vm, vlib_buffer_t * b0, u8 * ec0, word ec_len, ip0 = vlib_buffer_get_current (b0); /* Apply the encap string. */ - clib_memcpy (ip0, ec0, ec_len); + clib_memcpy_fast (ip0, ec0, ec_len); ip_udp_fixup_one (vm, b0, 0); } } @@ -380,8 +380,8 @@ ip_udp_encap_two (vlib_main_t * vm, vlib_buffer_t * b0, vlib_buffer_t * b1, ip1 = vlib_buffer_get_current (b1); /* Apply the encap string */ - clib_memcpy (ip0, ec0, ec_len); - clib_memcpy (ip1, ec1, ec_len); + clib_memcpy_fast (ip0, ec0, ec_len); + clib_memcpy_fast (ip1, ec1, ec_len); /* fix the <bleep>ing outer-IP checksum */ sum0 = ip0->checksum; @@ -424,8 +424,8 @@ ip_udp_encap_two (vlib_main_t * vm, vlib_buffer_t * b0, vlib_buffer_t * b1, ip1 = vlib_buffer_get_current (b1); /* Apply the encap string. */ - clib_memcpy (ip0, ec0, ec_len); - clib_memcpy (ip1, ec1, ec_len); + clib_memcpy_fast (ip0, ec0, ec_len); + clib_memcpy_fast (ip1, ec1, ec_len); new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - sizeof (*ip0)); diff --git a/src/vnet/unix/pcap.h b/src/vnet/unix/pcap.h index 2c174fbedf9..0706aace496 100644 --- a/src/vnet/unix/pcap.h +++ b/src/vnet/unix/pcap.h @@ -221,7 +221,7 @@ pcap_add_buffer (pcap_main_t * pm, while (1) { u32 copy_length = clib_min ((u32) n_left, b->current_length); - clib_memcpy (d, b->data + b->current_data, copy_length); + clib_memcpy_fast (d, b->data + b->current_data, copy_length); n_left -= b->current_length; if (n_left <= 0) break; diff --git a/src/vnet/unix/tuntap.c b/src/vnet/unix/tuntap.c index d663d9698a0..e002a248ce4 100644 --- a/src/vnet/unix/tuntap.c +++ b/src/vnet/unix/tuntap.c @@ -166,7 +166,8 @@ tuntap_tx (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame) if (tm->is_ether && (!tm->have_normal_interface)) { vlib_buffer_reset (b); - clib_memcpy (vlib_buffer_get_current (b), tm->ether_dst_mac, 6); + clib_memcpy_fast (vlib_buffer_get_current (b), tm->ether_dst_mac, + 6); } /* Re-set iovecs if present. */ @@ -626,7 +627,7 @@ tuntap_config (vlib_main_t * vm, unformat_input_t * input) goto done; } else - clib_memcpy (tm->ether_dst_mac, ifr.ifr_hwaddr.sa_data, 6); + clib_memcpy_fast (tm->ether_dst_mac, ifr.ifr_hwaddr.sa_data, 6); } if (have_normal_interface) @@ -721,7 +722,7 @@ tuntap_ip4_add_del_interface_address (ip4_main_t * im, /** See if we already know about this subif */ clib_memset (&subif_addr, 0, sizeof (subif_addr)); subif_addr.sw_if_index = sw_if_index; - clib_memcpy (&subif_addr.addr, address, sizeof (*address)); + clib_memcpy_fast (&subif_addr.addr, address, sizeof (*address)); p = mhash_get (&tm->subif_mhash, &subif_addr); @@ -753,7 +754,7 @@ tuntap_ip4_add_del_interface_address (ip4_main_t * im, /* Set ipv4 address, netmask. */ sin->sin_family = AF_INET; - clib_memcpy (&sin->sin_addr.s_addr, address, 4); + clib_memcpy_fast (&sin->sin_addr.s_addr, address, 4); if (ioctl (tm->dev_tap_fd, SIOCSIFADDR, &ifr) < 0) clib_unix_warning ("ioctl SIOCSIFADDR"); @@ -838,7 +839,7 @@ tuntap_ip6_add_del_interface_address (ip6_main_t * im, clib_memset (&subif_addr, 0, sizeof (subif_addr)); subif_addr.sw_if_index = sw_if_index; subif_addr.is_v6 = 1; - clib_memcpy (&subif_addr.addr, address, sizeof (*address)); + clib_memcpy_fast (&subif_addr.addr, address, sizeof (*address)); p = mhash_get (&tm->subif_mhash, &subif_addr); @@ -874,7 +875,7 @@ tuntap_ip6_add_del_interface_address (ip6_main_t * im, ifr6.ifr6_ifindex = ifr.ifr_ifindex; ifr6.ifr6_prefixlen = address_length; - clib_memcpy (&ifr6.ifr6_addr, address, 16); + clib_memcpy_fast (&ifr6.ifr6_addr, address, 16); if (ioctl (sockfd, SIOCSIFADDR, &ifr6) < 0) clib_unix_warning ("set address"); @@ -893,7 +894,7 @@ tuntap_ip6_add_del_interface_address (ip6_main_t * im, ifr6.ifr6_ifindex = ifr.ifr_ifindex; ifr6.ifr6_prefixlen = address_length; - clib_memcpy (&ifr6.ifr6_addr, address, 16); + clib_memcpy_fast (&ifr6.ifr6_addr, address, 16); if (ioctl (sockfd, SIOCDIFADDR, &ifr6) < 0) clib_unix_warning ("del address"); diff --git a/src/vnet/vxlan-gbp/encap.c b/src/vnet/vxlan-gbp/encap.c index 07142c8a521..d260ccc0deb 100644 --- a/src/vnet/vxlan-gbp/encap.c +++ b/src/vnet/vxlan-gbp/encap.c @@ -186,10 +186,12 @@ vxlan_gbp_encap_inline (vlib_main_t * vm, /* vnet_rewrite_two_header writes only in (uword) 8 bytes chunks * and discards the first 4 bytes of the (36 bytes ip4 underlay) rewrite * use memcpy as a workaround */ - clib_memcpy (underlay0, t0->rewrite_header.data + rw_hdr_offset, - underlay_hdr_len); - clib_memcpy (underlay1, t1->rewrite_header.data + rw_hdr_offset, - underlay_hdr_len); + clib_memcpy_fast (underlay0, + t0->rewrite_header.data + rw_hdr_offset, + underlay_hdr_len); + clib_memcpy_fast (underlay1, + t1->rewrite_header.data + rw_hdr_offset, + underlay_hdr_len); ip4_header_t *ip4_0, *ip4_1; qos_bits_t ip4_0_tos = 0, ip4_1_tos = 0; @@ -368,8 +370,9 @@ vxlan_gbp_encap_inline (vlib_main_t * vm, /* vnet_rewrite_one_header writes only in (uword) 8 bytes chunks * and discards the first 4 bytes of the (36 bytes ip4 underlay) rewrite * use memcpy as a workaround */ - clib_memcpy (underlay0, t0->rewrite_header.data + rw_hdr_offset, - underlay_hdr_len); + clib_memcpy_fast (underlay0, + t0->rewrite_header.data + rw_hdr_offset, + underlay_hdr_len); u32 len0 = vlib_buffer_length_in_chain (vm, b0); u16 payload_l0 = clib_host_to_net_u16 (len0 - l3_len); diff --git a/src/vnet/vxlan-gpe/vxlan_gpe.c b/src/vnet/vxlan-gpe/vxlan_gpe.c index c14cc734fb3..a7eaba6a49f 100644 --- a/src/vnet/vxlan-gpe/vxlan_gpe.c +++ b/src/vnet/vxlan-gpe/vxlan_gpe.c @@ -543,14 +543,14 @@ int vnet_vxlan_gpe_add_del_tunnel if (!is_ip6) { key4_copy = clib_mem_alloc (sizeof (*key4_copy)); - clib_memcpy (key4_copy, &key4, sizeof (*key4_copy)); + clib_memcpy_fast (key4_copy, &key4, sizeof (*key4_copy)); hash_set_mem (ngm->vxlan4_gpe_tunnel_by_key, key4_copy, t - ngm->tunnels); } else { key6_copy = clib_mem_alloc (sizeof (*key6_copy)); - clib_memcpy (key6_copy, &key6, sizeof (*key6_copy)); + clib_memcpy_fast (key6_copy, &key6, sizeof (*key6_copy)); hash_set_mem (ngm->vxlan6_gpe_tunnel_by_key, key6_copy, t - ngm->tunnels); } diff --git a/src/vnet/vxlan/encap.c b/src/vnet/vxlan/encap.c index 31320900d3e..33b6526f2fa 100644 --- a/src/vnet/vxlan/encap.c +++ b/src/vnet/vxlan/encap.c @@ -186,8 +186,8 @@ vxlan_encap_inline (vlib_main_t * vm, /* vnet_rewrite_two_header writes only in (uword) 8 bytes chunks * and discards the first 4 bytes of the (36 bytes ip4 underlay) rewrite * use memcpy as a workaround */ - clib_memcpy(underlay0, t0->rewrite_header.data + rw_hdr_offset, underlay_hdr_len); - clib_memcpy(underlay1, t1->rewrite_header.data + rw_hdr_offset, underlay_hdr_len); + clib_memcpy_fast(underlay0, t0->rewrite_header.data + rw_hdr_offset, underlay_hdr_len); + clib_memcpy_fast(underlay1, t1->rewrite_header.data + rw_hdr_offset, underlay_hdr_len); ip4_header_t * ip4_0, * ip4_1; qos_bits_t ip4_0_tos = 0, ip4_1_tos = 0; @@ -359,7 +359,7 @@ vxlan_encap_inline (vlib_main_t * vm, /* vnet_rewrite_one_header writes only in (uword) 8 bytes chunks * and discards the first 4 bytes of the (36 bytes ip4 underlay) rewrite * use memcpy as a workaround */ - clib_memcpy(underlay0, t0->rewrite_header.data + rw_hdr_offset, underlay_hdr_len); + clib_memcpy_fast(underlay0, t0->rewrite_header.data + rw_hdr_offset, underlay_hdr_len); u32 len0 = vlib_buffer_length_in_chain (vm, b0); u16 payload_l0 = clib_host_to_net_u16 (len0 - l3_len); diff --git a/src/vppinfra/bihash_template.c b/src/vppinfra/bihash_template.c index e218f4b0dc6..698053867a4 100644 --- a/src/vppinfra/bihash_template.c +++ b/src/vppinfra/bihash_template.c @@ -295,7 +295,7 @@ BV (make_working_copy) (BVT (clib_bihash) * h, BVT (clib_bihash_bucket) * b) v = BV (clib_bihash_get_value) (h, b->offset); - _clib_memcpy (working_copy, v, sizeof (*v) * (1 << b->log2_pages)); + clib_memcpy_fast (working_copy, v, sizeof (*v) * (1 << b->log2_pages)); working_bucket.as_u64 = b->as_u64; working_bucket.offset = BV (clib_bihash_get_offset) (h, working_copy); CLIB_MEMORY_BARRIER (); @@ -338,8 +338,8 @@ BV (split_and_rehash) /* Empty slot */ if (BV (clib_bihash_is_free) (&(new_v->kvp[j]))) { - _clib_memcpy (&(new_v->kvp[j]), &(old_values->kvp[i]), - sizeof (new_v->kvp[j])); + clib_memcpy_fast (&(new_v->kvp[j]), &(old_values->kvp[i]), + sizeof (new_v->kvp[j])); goto doublebreak; } } @@ -383,8 +383,8 @@ BV (split_and_rehash_linear) if (BV (clib_bihash_is_free) (&(new_values->kvp[j]))) { /* Copy the old value and move along */ - _clib_memcpy (&(new_values->kvp[j]), &(old_values->kvp[i]), - sizeof (new_values->kvp[j])); + clib_memcpy_fast (&(new_values->kvp[j]), &(old_values->kvp[i]), + sizeof (new_values->kvp[j])); j++; goto doublebreak; } @@ -472,7 +472,7 @@ static inline int BV (clib_bihash_add_del_inline) if (!memcmp (&(v->kvp[i]), &add_v->key, sizeof (add_v->key))) { CLIB_MEMORY_BARRIER (); /* Add a delay */ - _clib_memcpy (&(v->kvp[i]), add_v, sizeof (*add_v)); + clib_memcpy_fast (&(v->kvp[i]), add_v, sizeof (*add_v)); BV (clib_bihash_unlock_bucket) (b); return (0); } @@ -488,10 +488,11 @@ static inline int BV (clib_bihash_add_del_inline) * Copy the value first, so that if a reader manages * to match the new key, the value will be right... */ - _clib_memcpy (&(v->kvp[i].value), - &add_v->value, sizeof (add_v->value)); + clib_memcpy_fast (&(v->kvp[i].value), + &add_v->value, sizeof (add_v->value)); CLIB_MEMORY_BARRIER (); /* Make sure the value has settled */ - _clib_memcpy (&(v->kvp[i]), &add_v->key, sizeof (add_v->key)); + clib_memcpy_fast (&(v->kvp[i]), &add_v->key, + sizeof (add_v->key)); b->refcnt++; ASSERT (b->refcnt > 0); BV (clib_bihash_unlock_bucket) (b); @@ -506,7 +507,7 @@ static inline int BV (clib_bihash_add_del_inline) if (is_stale_cb (&(v->kvp[i]), arg)) { CLIB_MEMORY_BARRIER (); - _clib_memcpy (&(v->kvp[i]), add_v, sizeof (*add_v)); + clib_memcpy_fast (&(v->kvp[i]), add_v, sizeof (*add_v)); BV (clib_bihash_unlock_bucket) (b); return (0); } @@ -602,7 +603,7 @@ static inline int BV (clib_bihash_add_del_inline) { if (BV (clib_bihash_is_free) (&(new_v->kvp[i]))) { - _clib_memcpy (&(new_v->kvp[i]), add_v, sizeof (*add_v)); + clib_memcpy_fast (&(new_v->kvp[i]), add_v, sizeof (*add_v)); goto expand_ok; } } diff --git a/src/vppinfra/fifo.c b/src/vppinfra/fifo.c index e3b7b415cb3..e97b2c3ee00 100644 --- a/src/vppinfra/fifo.c +++ b/src/vppinfra/fifo.c @@ -112,11 +112,11 @@ _clib_fifo_resize (void *v_old, uword n_new_elts, uword elt_bytes) if (head + n_copy_bytes >= end) { uword n = end - head; - clib_memcpy (v_new, head, n); - clib_memcpy (v_new + n, v_old, n_copy_bytes - n); + clib_memcpy_fast (v_new, head, n); + clib_memcpy_fast (v_new + n, v_old, n_copy_bytes - n); } else - clib_memcpy (v_new, head, n_copy_bytes); + clib_memcpy_fast (v_new, head, n_copy_bytes); } /* Zero empty space. */ diff --git a/src/vppinfra/fifo.h b/src/vppinfra/fifo.h index b0b35e25af7..5dc1b4512cf 100644 --- a/src/vppinfra/fifo.h +++ b/src/vppinfra/fifo.h @@ -215,9 +215,9 @@ do { \ _n1 = _i + _n0 - _l; \ _n1 = _n1 < 0 ? 0 : _n1; \ _n0 -= _n1; \ - clib_memcpy ((f) + _i, (e), _n0 * sizeof ((f)[0])); \ + clib_memcpy_fast ((f) + _i, (e), _n0 * sizeof ((f)[0])); \ if (_n1) \ - clib_memcpy ((f) + 0, (e) + _n0, _n1 * sizeof ((f)[0])); \ + clib_memcpy_fast ((f) + 0, (e) + _n0, _n1 * sizeof ((f)[0])); \ } while (0) /* Subtract element from fifo. */ diff --git a/src/vppinfra/hash.c b/src/vppinfra/hash.c index 2ff8ebfd17b..eae79d48592 100644 --- a/src/vppinfra/hash.c +++ b/src/vppinfra/hash.c @@ -376,7 +376,7 @@ set_indirect_is_user (void *v, uword i, hash_pair_union_t * p, uword key) log2_bytes = 1 + hash_pair_log2_bytes (h); q = clib_mem_alloc (1ULL << log2_bytes); } - clib_memcpy (q, &p->direct, hash_pair_bytes (h)); + clib_memcpy_fast (q, &p->direct, hash_pair_bytes (h)); pi->pairs = q; if (h->log2_pair_size > 0) @@ -457,8 +457,8 @@ unset_indirect (void *v, uword i, hash_pair_t * q) if (len == 2) { - clib_memcpy (p, q == r ? hash_forward1 (h, r) : r, - hash_pair_bytes (h)); + clib_memcpy_fast (p, q == r ? hash_forward1 (h, r) : r, + hash_pair_bytes (h)); set_is_user (v, i, 1); } else @@ -473,7 +473,7 @@ unset_indirect (void *v, uword i, hash_pair_t * q) { /* If deleting a pair we need to keep non-null pairs together. */ if (q < e) - clib_memcpy (q, e, hash_pair_bytes (h)); + clib_memcpy_fast (q, e, hash_pair_bytes (h)); else zero_pair (h, q); if (is_vec) @@ -514,8 +514,8 @@ lookup (void *v, uword key, enum lookup_opcode op, { set_is_user (v, i, 0); if (old_value) - clib_memcpy (old_value, p->direct.value, - hash_value_bytes (h)); + clib_memcpy_fast (old_value, p->direct.value, + hash_value_bytes (h)); zero_pair (h, &p->direct); } } @@ -548,8 +548,8 @@ lookup (void *v, uword key, enum lookup_opcode op, if (found_key && op == UNSET) { if (old_value) - clib_memcpy (old_value, &p->direct.value, - hash_value_bytes (h)); + clib_memcpy_fast (old_value, &p->direct.value, + hash_value_bytes (h)); unset_indirect (v, i, &p->direct); @@ -564,8 +564,8 @@ lookup (void *v, uword key, enum lookup_opcode op, { /* Save away old value for caller. */ if (old_value && found_key) - clib_memcpy (old_value, &p->direct.value, hash_value_bytes (h)); - clib_memcpy (&p->direct.value, new_value, hash_value_bytes (h)); + clib_memcpy_fast (old_value, &p->direct.value, hash_value_bytes (h)); + clib_memcpy_fast (&p->direct.value, new_value, hash_value_bytes (h)); } if (op == SET) diff --git a/src/vppinfra/hash.h b/src/vppinfra/hash.h index 892b2ea916c..59dbf8cde76 100644 --- a/src/vppinfra/hash.h +++ b/src/vppinfra/hash.h @@ -280,7 +280,7 @@ hash_set_mem_alloc (uword ** h, void *key, uword v) { size_t ksz = hash_header (*h)->user; void *copy = clib_mem_alloc (ksz); - clib_memcpy (copy, key, ksz); + clib_memcpy_fast (copy, key, ksz); hash_set_mem (*h, copy, v); } diff --git a/src/vppinfra/heap.h b/src/vppinfra/heap.h index b6e9f022852..22fc335c072 100644 --- a/src/vppinfra/heap.h +++ b/src/vppinfra/heap.h @@ -204,7 +204,7 @@ _heap_dup (void *v_old, uword v_bytes) HEAP_DATA_ALIGN); h_new = heap_header (v_new); heap_dup_header (h_old, h_new); - clib_memcpy (v_new, v_old, v_bytes); + clib_memcpy_fast (v_new, v_old, v_bytes); return v_new; } diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index a2c5474a46e..65b25f004a7 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -232,7 +232,7 @@ clib_mem_realloc (void *p, uword new_size, uword old_size) copy_size = old_size; else copy_size = new_size; - clib_memcpy (q, p, copy_size); + clib_memcpy_fast (q, p, copy_size); clib_mem_free (p); } return q; diff --git a/src/vppinfra/memcpy_avx2.h b/src/vppinfra/memcpy_avx2.h index 64dff4e5ddb..caff4f6afde 100644 --- a/src/vppinfra/memcpy_avx2.h +++ b/src/vppinfra/memcpy_avx2.h @@ -109,7 +109,7 @@ clib_mov128blocks (u8 * dst, const u8 * src, size_t n) } static inline void * -_clib_memcpy (void *dst, const void *src, size_t n) +clib_memcpy_fast (void *dst, const void *src, size_t n) { uword dstu = (uword) dst; uword srcu = (uword) src; diff --git a/src/vppinfra/memcpy_avx512.h b/src/vppinfra/memcpy_avx512.h index e5245268770..9032b5436a3 100644 --- a/src/vppinfra/memcpy_avx512.h +++ b/src/vppinfra/memcpy_avx512.h @@ -139,7 +139,7 @@ clib_mov512blocks (u8 * dst, const u8 * src, size_t n) } static inline void * -_clib_memcpy (void *dst, const void *src, size_t n) +clib_memcpy_fast (void *dst, const void *src, size_t n) { uword dstu = (uword) dst; uword srcu = (uword) src; diff --git a/src/vppinfra/memcpy_sse3.h b/src/vppinfra/memcpy_sse3.h index 2dd9399d428..5346e913817 100644 --- a/src/vppinfra/memcpy_sse3.h +++ b/src/vppinfra/memcpy_sse3.h @@ -183,7 +183,7 @@ clib_mov256 (u8 * dst, const u8 * src) }) static inline void * -_clib_memcpy (void *dst, const void *src, size_t n) +clib_memcpy_fast (void *dst, const void *src, size_t n) { __m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8; uword dstu = (uword) dst; diff --git a/src/vppinfra/mhash.c b/src/vppinfra/mhash.c index e09d7193f68..d4d5457ea7a 100644 --- a/src/vppinfra/mhash.c +++ b/src/vppinfra/mhash.c @@ -289,7 +289,7 @@ mhash_set_mem (mhash_t * h, void *key, uword * new_value, uword * old_value) sk = (void *) (h->key_vector_or_heap + i); sk->heap_handle = handle; sk->vec.len = n_key_bytes; - clib_memcpy (sk->vec.vector_data, key, n_key_bytes); + clib_memcpy_fast (sk->vec.vector_data, key, n_key_bytes); /* Advance key past vector header. */ i += sizeof (sk[0]); @@ -311,7 +311,7 @@ mhash_set_mem (mhash_t * h, void *key, uword * new_value, uword * old_value) } n_key_bytes = h->n_key_bytes; - clib_memcpy (k, key, n_key_bytes); + clib_memcpy_fast (k, key, n_key_bytes); } ikey = i; diff --git a/src/vppinfra/serialize.c b/src/vppinfra/serialize.c index ef9cf03f362..93e44f94e07 100644 --- a/src/vppinfra/serialize.c +++ b/src/vppinfra/serialize.c @@ -170,7 +170,7 @@ serialize_cstring (serialize_main_t * m, char *s) if (len > 0) { p = serialize_get (m, len); - clib_memcpy (p, s, len); + clib_memcpy_fast (p, s, len); } } @@ -191,7 +191,7 @@ unserialize_cstring (serialize_main_t * m, char **s) { r = vec_new (char, len + 1); p = unserialize_get (m, len); - clib_memcpy (r, p, len); + clib_memcpy_fast (r, p, len); /* Null terminate. */ r[len] = 0; @@ -206,7 +206,7 @@ serialize_vec_8 (serialize_main_t * m, va_list * va) u8 *s = va_arg (*va, u8 *); u32 n = va_arg (*va, u32); u8 *p = serialize_get (m, n * sizeof (u8)); - clib_memcpy (p, s, n * sizeof (u8)); + clib_memcpy_fast (p, s, n * sizeof (u8)); } void @@ -215,7 +215,7 @@ unserialize_vec_8 (serialize_main_t * m, va_list * va) u8 *s = va_arg (*va, u8 *); u32 n = va_arg (*va, u32); u8 *p = unserialize_get (m, n); - clib_memcpy (s, p, n); + clib_memcpy_fast (s, p, n); } #define _(n_bits) \ @@ -627,7 +627,7 @@ serialize_magic (serialize_main_t * m, void *magic, u32 magic_bytes) void *p; serialize_integer (m, magic_bytes, sizeof (magic_bytes)); p = serialize_get (m, magic_bytes); - clib_memcpy (p, magic, magic_bytes); + clib_memcpy_fast (p, magic, magic_bytes); } void @@ -710,7 +710,7 @@ serialize_write_not_inline (serialize_main_header_t * m, if (n_left_o > 0 && n_left_b > 0) { uword n = clib_min (n_left_b, n_left_o); - clib_memcpy (s->buffer + cur_bi, s->overflow_buffer, n); + clib_memcpy_fast (s->buffer + cur_bi, s->overflow_buffer, n); cur_bi += n; n_left_b -= n; n_left_o -= n; diff --git a/src/vppinfra/socket.c b/src/vppinfra/socket.c index 109cbad7bb8..bcee3dea9de 100644 --- a/src/vppinfra/socket.c +++ b/src/vppinfra/socket.c @@ -356,7 +356,8 @@ default_socket_recvmsg (clib_socket_t * s, void *msg, int msglen, #endif if (cmsg->cmsg_type == SCM_RIGHTS) { - clib_memcpy (fds, CMSG_DATA (cmsg), num_fds * sizeof (int)); + clib_memcpy_fast (fds, CMSG_DATA (cmsg), + num_fds * sizeof (int)); } } cmsg = CMSG_NXTHDR (&mh, cmsg); diff --git a/src/vppinfra/string.h b/src/vppinfra/string.h index 5c1d8267742..b00c0cfbcc2 100644 --- a/src/vppinfra/string.h +++ b/src/vppinfra/string.h @@ -78,10 +78,10 @@ void clib_memswap (void *_a, void *_b, uword bytes); #elif __SSSE3__ #include <vppinfra/memcpy_sse3.h> #else -#define _clib_memcpy(a,b,c) memcpy(a,b,c) +#define clib_memcpy_fast(a,b,c) memcpy(a,b,c) #endif #else /* __COVERITY__ */ -#define _clib_memcpy(a,b,c) memcpy(a,b,c) +#define clib_memcpy_fast(a,b,c) memcpy(a,b,c) #endif /* c-11 string manipulation variants */ @@ -108,6 +108,16 @@ memcpy_s_inline (void *__restrict__ dest, rsize_t dmax, u8 bad; /* + * Optimize constant-number-of-bytes calls without asking + * "too many questions for someone from New Jersey" + */ + if (__builtin_constant_p (n)) + { + clib_memcpy_fast (dest, src, n); + return EOK; + } + + /* * call bogus if: src or dst NULL, trying to copy * more data than we have space in dst, or src == dst. * n == 0 isn't really "bad", so check first in the @@ -140,7 +150,7 @@ memcpy_s_inline (void *__restrict__ dest, rsize_t dmax, return EINVAL; } - _clib_memcpy (dest, src, n); + clib_memcpy_fast (dest, src, n); return EOK; } @@ -247,10 +257,10 @@ clib_memcpy64_x4 (void *d0, void *d1, void *d2, void *d3, void *s) _mm_storeu_si128 ((__m128i *) (d3 + 3 * 16), r3); #else - clib_memcpy (d0, s, 64); - clib_memcpy (d1, s, 64); - clib_memcpy (d2, s, 64); - clib_memcpy (d3, s, 64); + clib_memcpy_fast (d0, s, 64); + clib_memcpy_fast (d1, s, 64); + clib_memcpy_fast (d2, s, 64); + clib_memcpy_fast (d3, s, 64); #endif } diff --git a/src/vppinfra/vec.c b/src/vppinfra/vec.c index 1e0164a9e34..232eb4d4c24 100644 --- a/src/vppinfra/vec.c +++ b/src/vppinfra/vec.c @@ -92,7 +92,7 @@ vec_resize_allocate_memory (void *v, ("vec_resize fails, length increment %d, data bytes %d, alignment %d", length_increment, data_bytes, data_align); - clib_memcpy (new, old, old_alloc_bytes); + clib_memcpy_fast (new, old, old_alloc_bytes); clib_mem_free (old); /* Allocator may give a bit of extra room. */ diff --git a/src/vppinfra/vec.h b/src/vppinfra/vec.h index dc7b908a3b5..470a4f190c2 100644 --- a/src/vppinfra/vec.h +++ b/src/vppinfra/vec.h @@ -360,7 +360,7 @@ do { \ if (_v(l) > 0) \ { \ vec_resize_ha (_v(v), _v(l), (H), (A)); \ - clib_memcpy (_v(v), (V), _v(l) * sizeof ((V)[0]));\ + clib_memcpy_fast (_v(v), (V), _v(l) * sizeof ((V)[0]));\ } \ _v(v); \ }) @@ -387,7 +387,7 @@ do { \ @param DST destination @param SRC source */ -#define vec_copy(DST,SRC) clib_memcpy (DST, SRC, vec_len (DST) * \ +#define vec_copy(DST,SRC) clib_memcpy_fast (DST, SRC, vec_len (DST) * \ sizeof ((DST)[0])) /** \brief Clone a vector. Make a new vector with the @@ -587,7 +587,7 @@ do { \ word _v(n) = (N); \ word _v(l) = vec_len (V); \ V = _vec_resize ((V), _v(n), (_v(l) + _v(n)) * sizeof ((V)[0]), (H), (A)); \ - clib_memcpy ((V) + _v(l), (E), _v(n) * sizeof ((V)[0])); \ + clib_memcpy_fast ((V) + _v(l), (E), _v(n) * sizeof ((V)[0])); \ } while (0) /** \brief Add N elements to end of vector V (no header, unspecified alignment) @@ -749,7 +749,7 @@ do { \ memmove ((V) + _v(m) + _v(n), \ (V) + _v(m), \ (_v(l) - _v(m)) * sizeof ((V)[0])); \ - clib_memcpy ((V) + _v(m), (E), \ + clib_memcpy_fast ((V) + _v(m), (E), \ _v(n) * sizeof ((V)[0])); \ } while (0) @@ -824,7 +824,7 @@ do { \ \ v1 = _vec_resize ((v1), _v(l2), \ (_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, 0); \ - clib_memcpy ((v1) + _v(l1), (v2), _v(l2) * sizeof ((v2)[0])); \ + clib_memcpy_fast ((v1) + _v(l1), (v2), _v(l2) * sizeof ((v2)[0])); \ } while (0) /** \brief Append v2 after v1. Result in v1. Specified alignment. @@ -840,7 +840,7 @@ do { \ \ v1 = _vec_resize ((v1), _v(l2), \ (_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, align); \ - clib_memcpy ((v1) + _v(l1), (v2), _v(l2) * sizeof ((v2)[0])); \ + clib_memcpy_fast ((v1) + _v(l1), (v2), _v(l2) * sizeof ((v2)[0])); \ } while (0) /** \brief Prepend v2 before v1. Result in v1. @@ -856,7 +856,7 @@ do { \ v1 = _vec_resize ((v1), _v(l2), \ (_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, 0); \ memmove ((v1) + _v(l2), (v1), _v(l1) * sizeof ((v1)[0])); \ - clib_memcpy ((v1), (v2), _v(l2) * sizeof ((v2)[0])); \ + clib_memcpy_fast ((v1), (v2), _v(l2) * sizeof ((v2)[0])); \ } while (0) /** \brief Prepend v2 before v1. Result in v1. Specified alignment @@ -873,7 +873,7 @@ do { \ v1 = _vec_resize ((v1), _v(l2), \ (_v(l1) + _v(l2)) * sizeof ((v1)[0]), 0, align); \ memmove ((v1) + _v(l2), (v1), _v(l1) * sizeof ((v1)[0])); \ - clib_memcpy ((v1), (v2), _v(l2) * sizeof ((v2)[0])); \ + clib_memcpy_fast ((v1), (v2), _v(l2) * sizeof ((v2)[0])); \ } while (0) @@ -995,7 +995,7 @@ do { \ vec_reset_length (V); \ vec_validate ((V), (L)); \ if ((S) && (L)) \ - clib_memcpy ((V), (S), (L)); \ + clib_memcpy_fast ((V), (S), (L)); \ (V)[(L)] = 0; \ } while (0) |