diff options
70 files changed, 282 insertions, 288 deletions
diff --git a/src/examples/sample-plugin/sample/node.c b/src/examples/sample-plugin/sample/node.c index 2c71b48b42e..a31c3e86e08 100644 --- a/src/examples/sample-plugin/sample/node.c +++ b/src/examples/sample-plugin/sample/node.c @@ -124,8 +124,8 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); } /* speculatively enqueue b0 and b1 to the current next frame */ @@ -322,8 +322,8 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); } /* speculatively enqueue b0 and b1 to the current next frame */ @@ -507,10 +507,10 @@ VLIB_NODE_FN (sample_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (b[5], STORE); vlib_prefetch_buffer_header (b[6], STORE); vlib_prefetch_buffer_header (b[7], STORE); - CLIB_PREFETCH (&b[4]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (&b[5]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (&b[6]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (&b[7]->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (&b[4]->data); + clib_prefetch_store (&b[5]->data); + clib_prefetch_store (&b[6]->data); + clib_prefetch_store (&b[7]->data); } src_dst0 = ((u8x16 *) vlib_buffer_get_current (b[0]))[0]; diff --git a/src/plugins/acl/dataplane_node.c b/src/plugins/acl/dataplane_node.c index 3a351a01a04..1a050f54b14 100644 --- a/src/plugins/acl/dataplane_node.c +++ b/src/plugins/acl/dataplane_node.c @@ -277,7 +277,7 @@ acl_fa_node_common_prepare_fn (vlib_main_t * vm, for (ii = ACL_PLUGIN_PREFETCH_GAP * vec_sz; ii < (ACL_PLUGIN_PREFETCH_GAP + 1) * vec_sz; ii++) { - CLIB_PREFETCH (b[ii], CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b[ii]); CLIB_PREFETCH (b[ii]->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); } } diff --git a/src/plugins/adl/ip4_allowlist.c b/src/plugins/adl/ip4_allowlist.c index ca7b4c115f0..316e2cb558b 100644 --- a/src/plugins/adl/ip4_allowlist.c +++ b/src/plugins/adl/ip4_allowlist.c @@ -98,30 +98,28 @@ VLIB_NODE_FN (ip4_adl_allowlist_node) (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); - } - - /* speculatively enqueue b0 and b1 to the current next frame */ - to_next[0] = bi0 = from[0]; - to_next[1] = bi1 = from[1]; - from += 2; - to_next += 2; - n_left_from -= 2; - n_left_to_next -= 2; - - b0 = vlib_get_buffer (vm, bi0); - sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX]; + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); + } + + /* speculatively enqueue b0 and b1 to the current next frame */ + to_next[0] = bi0 = from[0]; + to_next[1] = bi1 = from[1]; + from += 2; + to_next += 2; + n_left_from -= 2; + n_left_to_next -= 2; - ip0 = vlib_buffer_get_current (b0); + b0 = vlib_get_buffer (vm, bi0); + sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - ccm0 = cm->adl_config_mains + VNET_ADL_IP4; + ip0 = vlib_buffer_get_current (b0); - c0 = vnet_get_config_data - (&ccm0->config_main, - &adl_buffer (b0)->adl.current_config_index, - &next0, - sizeof (c0[0])); + ccm0 = cm->adl_config_mains + VNET_ADL_IP4; + + c0 = vnet_get_config_data ( + &ccm0->config_main, &adl_buffer (b0)->adl.current_config_index, + &next0, sizeof (c0[0])); mtrie0 = &ip4_fib_get (c0->fib_index)->mtrie; diff --git a/src/plugins/adl/ip6_allowlist.c b/src/plugins/adl/ip6_allowlist.c index d272f101b43..5f38484666b 100644 --- a/src/plugins/adl/ip6_allowlist.c +++ b/src/plugins/adl/ip6_allowlist.c @@ -96,8 +96,8 @@ VLIB_NODE_FN (ip6_adl_allowlist_node) (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); } /* speculatively enqueue b0 and b1 to the current next frame */ diff --git a/src/plugins/adl/node.c b/src/plugins/adl/node.c index 7dfaf3b2eb2..5578c13086a 100644 --- a/src/plugins/adl/node.c +++ b/src/plugins/adl/node.c @@ -88,8 +88,8 @@ VLIB_NODE_FN (adl_input_node) (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); } /* speculatively enqueue b0 and b1 to the current next frame */ diff --git a/src/plugins/avf/input.c b/src/plugins/avf/input.c index 4d037b44a28..1406d789e0b 100644 --- a/src/plugins/avf/input.c +++ b/src/plugins/avf/input.c @@ -309,14 +309,10 @@ avf_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node, if (next + 11 < size) { int stride = 8; - CLIB_PREFETCH ((void *) (fd + (next + stride)), - CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH ((void *) (fd + (next + stride + 1)), - CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH ((void *) (fd + (next + stride + 2)), - CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH ((void *) (fd + (next + stride + 3)), - CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load ((void *) (fd + (next + stride))); + clib_prefetch_load ((void *) (fd + (next + stride + 1))); + clib_prefetch_load ((void *) (fd + (next + stride + 2))); + clib_prefetch_load ((void *) (fd + (next + stride + 3))); } #ifdef CLIB_HAVE_VEC256 @@ -375,8 +371,7 @@ avf_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node, continue; one_by_one: #endif - CLIB_PREFETCH ((void *) (fd + ((next + 8) & mask)), - CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load ((void *) (fd + ((next + 8) & mask))); if (avf_rxd_is_not_dd (d)) break; diff --git a/src/plugins/cnat/cnat_session.c b/src/plugins/cnat/cnat_session.c index bb04de07c82..216d2575c37 100644 --- a/src/plugins/cnat/cnat_session.c +++ b/src/plugins/cnat/cnat_session.c @@ -192,13 +192,13 @@ cnat_session_scan (vlib_main_t * vm, f64 start_time, int i) { BVT (clib_bihash_bucket) * b = BV (clib_bihash_get_bucket) (h, i + 3); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b); b = BV (clib_bihash_get_bucket) (h, i + 1); if (!BV (clib_bihash_bucket_is_empty) (b)) { BVT (clib_bihash_value) * v = BV (clib_bihash_get_value) (h, b->offset); - CLIB_PREFETCH (v, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (v); } } diff --git a/src/plugins/crypto_sw_scheduler/main.c b/src/plugins/crypto_sw_scheduler/main.c index 2842b18d70b..b0548fa297a 100644 --- a/src/plugins/crypto_sw_scheduler/main.c +++ b/src/plugins/crypto_sw_scheduler/main.c @@ -370,7 +370,7 @@ crypto_sw_scheduler_dequeue_aead (vlib_main_t * vm, while (n_elts--) { if (n_elts > 1) - CLIB_PREFETCH (fe + 1, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (fe + 1); crypto_sw_scheduler_convert_aead (vm, ptd, fe, fe - f->elts, bi[0], sync_op_id, aad_len, tag_len); @@ -438,7 +438,7 @@ crypto_sw_scheduler_dequeue_link (vlib_main_t * vm, while (n_elts--) { if (n_elts > 1) - CLIB_PREFETCH (fe + 1, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (fe + 1); crypto_sw_scheduler_convert_link_crypto (vm, ptd, cm->keys + fe->key_index, diff --git a/src/plugins/ct6/ct6_in2out.c b/src/plugins/ct6/ct6_in2out.c index 39da7891c75..b8bda18370c 100644 --- a/src/plugins/ct6/ct6_in2out.c +++ b/src/plugins/ct6/ct6_in2out.c @@ -165,10 +165,10 @@ ct6_in2out_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (b[5], STORE); vlib_prefetch_buffer_header (b[6], STORE); vlib_prefetch_buffer_header (b[7], STORE); - CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (b[4]->data); + clib_prefetch_store (b[5]->data); + clib_prefetch_store (b[6]->data); + clib_prefetch_store (b[7]->data); } /* $$$$ process 4x pkts right here */ diff --git a/src/plugins/ct6/ct6_out2in.c b/src/plugins/ct6/ct6_out2in.c index bbe9c2e293a..ebb6da56134 100644 --- a/src/plugins/ct6/ct6_out2in.c +++ b/src/plugins/ct6/ct6_out2in.c @@ -102,10 +102,10 @@ ct6_out2in_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (b[5], STORE); vlib_prefetch_buffer_header (b[6], STORE); vlib_prefetch_buffer_header (b[7], STORE); - CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (b[4]->data); + clib_prefetch_store (b[5]->data); + clib_prefetch_store (b[6]->data); + clib_prefetch_store (b[7]->data); } /* $$$$ process 4x pkts right here */ diff --git a/src/plugins/dns/reply_node.c b/src/plugins/dns/reply_node.c index 5d7e735e885..b15943a6448 100644 --- a/src/plugins/dns/reply_node.c +++ b/src/plugins/dns/reply_node.c @@ -93,8 +93,8 @@ dns46_reply_node_fn (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); } /* speculatively enqueue b0 and b1 to the current next frame */ diff --git a/src/plugins/dns/request_node.c b/src/plugins/dns/request_node.c index a80876927e9..72a76d12e4f 100644 --- a/src/plugins/dns/request_node.c +++ b/src/plugins/dns/request_node.c @@ -95,8 +95,8 @@ dns46_request_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); } /* speculatively enqueue b0 and b1 to the current next frame */ diff --git a/src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c b/src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c index 86f8e041f9e..4545e24fc83 100644 --- a/src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c +++ b/src/plugins/dpdk/cryptodev/cryptodev_op_data_path.c @@ -171,8 +171,8 @@ cryptodev_frame_linked_algs_enqueue (vlib_main_t *vm, { CLIB_PREFETCH (cop[1], CLIB_CACHE_LINE_BYTES * 3, STORE); CLIB_PREFETCH (cop[2], CLIB_CACHE_LINE_BYTES * 3, STORE); - CLIB_PREFETCH (&fe[1], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (&fe[2], CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (&fe[1]); + clib_prefetch_load (&fe[2]); } if (last_key_index != fe->key_index) { @@ -288,8 +288,8 @@ cryptodev_frame_aead_enqueue (vlib_main_t *vm, { CLIB_PREFETCH (cop[1], CLIB_CACHE_LINE_BYTES * 3, STORE); CLIB_PREFETCH (cop[2], CLIB_CACHE_LINE_BYTES * 3, STORE); - CLIB_PREFETCH (&fe[1], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (&fe[2], CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (&fe[1]); + clib_prefetch_load (&fe[2]); } if (last_key_index != fe->key_index) { diff --git a/src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c b/src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c index c045d78ecb1..41a1e0c2a09 100644 --- a/src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c +++ b/src/plugins/dpdk/cryptodev/cryptodev_raw_data_path.c @@ -144,8 +144,8 @@ cryptodev_frame_linked_algs_enqueue (vlib_main_t *vm, if (n_elts > 2) { - CLIB_PREFETCH (&fe[1], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (&fe[2], CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (&fe[1]); + clib_prefetch_load (&fe[2]); vlib_prefetch_buffer_header (b[1], LOAD); vlib_prefetch_buffer_header (b[2], LOAD); } @@ -275,7 +275,7 @@ cryptodev_raw_aead_enqueue (vlib_main_t *vm, vnet_crypto_async_frame_t *frame, if (n_elts > 1) { - CLIB_PREFETCH (&fe[1], CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (&fe[1]); vlib_prefetch_buffer_header (b[1], LOAD); } @@ -449,8 +449,8 @@ cryptodev_raw_dequeue (vlib_main_t *vm, u32 *nb_elts_processed, vnet_crypto_async_frame_t *f1, *f2; GET_RING_OBJ (cet->cached_frame, i + 1, f1); GET_RING_OBJ (cet->cached_frame, i + 2, f2); - CLIB_PREFETCH (f1, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (f2, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (f1); + clib_prefetch_load (f2); } n_left = f->state & 0x7f; diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index 6d130a95e9c..7c083e1dcf4 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -212,7 +212,7 @@ dpdk_prefetch_buffer (vlib_main_t * vm, struct rte_mbuf *mb) { vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb); CLIB_PREFETCH (mb, sizeof (struct rte_mbuf), STORE); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b); } static_always_inline void @@ -357,12 +357,12 @@ VNET_DEVICE_CLASS_TX_FN (dpdk_device_class) (vlib_main_t * vm, vlib_buffer_t *b2, *b3; u32 or_flags; - CLIB_PREFETCH (mb[2], CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (mb[3], CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (mb[2]); + clib_prefetch_store (mb[3]); b2 = vlib_buffer_from_rte_mbuf (mb[2]); - CLIB_PREFETCH (b2, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b2); b3 = vlib_buffer_from_rte_mbuf (mb[3]); - CLIB_PREFETCH (b3, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b3); b[0] = vlib_buffer_from_rte_mbuf (mb[0]); b[1] = vlib_buffer_from_rte_mbuf (mb[1]); diff --git a/src/plugins/dpdk/device/node.c b/src/plugins/dpdk/device/node.c index 98cda4c452f..25222856912 100644 --- a/src/plugins/dpdk/device/node.c +++ b/src/plugins/dpdk/device/node.c @@ -86,10 +86,10 @@ dpdk_process_subseq_segs (vlib_main_t * vm, vlib_buffer_t * b, static_always_inline void dpdk_prefetch_mbuf_x4 (struct rte_mbuf *mb[]) { - CLIB_PREFETCH (mb[0], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (mb[1], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (mb[2], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (mb[3], CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (mb[0]); + clib_prefetch_load (mb[1]); + clib_prefetch_load (mb[2]); + clib_prefetch_load (mb[3]); } static_always_inline void @@ -97,13 +97,13 @@ dpdk_prefetch_buffer_x4 (struct rte_mbuf *mb[]) { vlib_buffer_t *b; b = vlib_buffer_from_rte_mbuf (mb[0]); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b); b = vlib_buffer_from_rte_mbuf (mb[1]); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b); b = vlib_buffer_from_rte_mbuf (mb[2]); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b); b = vlib_buffer_from_rte_mbuf (mb[3]); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b); } /** \brief Main DPDK input node diff --git a/src/plugins/flowprobe/node.c b/src/plugins/flowprobe/node.c index ab3640ca135..e7a39a7ed7e 100644 --- a/src/plugins/flowprobe/node.c +++ b/src/plugins/flowprobe/node.c @@ -755,8 +755,8 @@ flowprobe_node_fn (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); } /* speculatively enqueue b0 and b1 to the current next frame */ diff --git a/src/plugins/memif/device.c b/src/plugins/memif/device.c index fce4ab22004..fc66420a6ad 100644 --- a/src/plugins/memif/device.c +++ b/src/plugins/memif/device.c @@ -143,8 +143,7 @@ retry: u32 saved_ptd_buffers_len = _vec_len (ptd->buffers); u16 saved_slot = slot; - CLIB_PREFETCH (&ring->desc[(slot + 8) & mask], CLIB_CACHE_LINE_BYTES, - LOAD); + clib_prefetch_load (&ring->desc[(slot + 8) & mask]); d0 = &ring->desc[slot & mask]; if (PREDICT_FALSE (last_region != d0->region)) @@ -235,10 +234,10 @@ no_free_slots: co = ptd->copy_ops; while (n_copy_op >= 8) { - CLIB_PREFETCH (co[4].data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (co[5].data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (co[6].data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (co[7].data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (co[4].data); + clib_prefetch_load (co[5].data); + clib_prefetch_load (co[6].data); + clib_prefetch_load (co[7].data); b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]); b1 = vlib_get_buffer (vm, ptd->buffers[co[1].buffer_vec_index]); @@ -315,8 +314,7 @@ retry: memif_desc_t *d0; u32 bi0; - CLIB_PREFETCH (&ring->desc[(slot + 8) & mask], CLIB_CACHE_LINE_BYTES, - STORE); + clib_prefetch_store (&ring->desc[(slot + 8) & mask]); if (PREDICT_TRUE (n_left >= 4)) vlib_prefetch_buffer_header (vlib_get_buffer (vm, buffers[3]), LOAD); diff --git a/src/plugins/memif/node.c b/src/plugins/memif/node.c index a69ab7f7c2c..8bb3758c5fd 100644 --- a/src/plugins/memif/node.c +++ b/src/plugins/memif/node.c @@ -207,8 +207,7 @@ memif_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node, dst_off = start_offset; next_slot: - CLIB_PREFETCH (&ring->desc[(cur_slot + 8) & mask], - CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (&ring->desc[(cur_slot + 8) & mask]); s0 = cur_slot & mask; d0 = &ring->desc[s0]; n_bytes_left = d0->length; @@ -277,10 +276,10 @@ memif_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node, co = ptd->copy_ops; while (n_left >= 8) { - CLIB_PREFETCH (co[4].data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (co[5].data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (co[6].data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (co[7].data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (co[4].data); + clib_prefetch_load (co[5].data); + clib_prefetch_load (co[6].data); + clib_prefetch_load (co[7].data); b0 = vlib_get_buffer (vm, ptd->buffers[co[0].buffer_vec_index]); b1 = vlib_get_buffer (vm, ptd->buffers[co[1].buffer_vec_index]); @@ -574,8 +573,7 @@ memif_device_input_zc_inline (vlib_main_t *vm, vlib_node_runtime_t *node, bi0 = mq->buffers[s0]; ptd->buffers[n_rx_packets++] = bi0; - CLIB_PREFETCH (&ring->desc[(cur_slot + 8) & mask], - CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (&ring->desc[(cur_slot + 8) & mask]); d0 = &ring->desc[s0]; hb = b0 = vlib_get_buffer (vm, bi0); b0->current_data = start_offset; diff --git a/src/plugins/nat/det44/det44_in2out.c b/src/plugins/nat/det44/det44_in2out.c index 0857ca18886..5fe4a9a0658 100644 --- a/src/plugins/nat/det44/det44_in2out.c +++ b/src/plugins/nat/det44/det44_in2out.c @@ -451,8 +451,8 @@ VLIB_NODE_FN (det44_in2out_node) (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p2->data); + clib_prefetch_load (p3->data); } next0 = DET44_IN2OUT_NEXT_LOOKUP; diff --git a/src/plugins/nat/det44/det44_out2in.c b/src/plugins/nat/det44/det44_out2in.c index 48b816bca56..111bc61c476 100644 --- a/src/plugins/nat/det44/det44_out2in.c +++ b/src/plugins/nat/det44/det44_out2in.c @@ -415,8 +415,8 @@ VLIB_NODE_FN (det44_out2in_node) (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p2->data); + clib_prefetch_load (p3->data); } diff --git a/src/plugins/nat/nat44-ed/nat44_ed_handoff.c b/src/plugins/nat/nat44-ed/nat44_ed_handoff.c index 52d52b4e65c..918d7728c1d 100644 --- a/src/plugins/nat/nat44-ed/nat44_ed_handoff.c +++ b/src/plugins/nat/nat44-ed/nat44_ed_handoff.c @@ -112,10 +112,10 @@ nat44_worker_handoff_fn_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (b[5], LOAD); vlib_prefetch_buffer_header (b[6], LOAD); vlib_prefetch_buffer_header (b[7], LOAD); - CLIB_PREFETCH (&b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (&b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (&b[6]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (&b[7]->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (&b[4]->data); + clib_prefetch_load (&b[5]->data); + clib_prefetch_load (&b[6]->data); + clib_prefetch_load (&b[7]->data); } if (is_output) diff --git a/src/plugins/nat/nat44-ed/nat44_ed_in2out.c b/src/plugins/nat/nat44-ed/nat44_ed_in2out.c index 523d26257bf..bfabdbd71d0 100644 --- a/src/plugins/nat/nat44-ed/nat44_ed_in2out.c +++ b/src/plugins/nat/nat44-ed/nat44_ed_in2out.c @@ -1010,7 +1010,7 @@ nat44_ed_in2out_fast_path_node_fn_inline (vlib_main_t *vm, vlib_prefetch_buffer_header (p2, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p2->data); } if (is_output_feature) diff --git a/src/plugins/nat/nat44-ed/nat44_ed_inlines.h b/src/plugins/nat/nat44-ed/nat44_ed_inlines.h index 680bdef1f9f..0d75e736849 100644 --- a/src/plugins/nat/nat44-ed/nat44_ed_inlines.h +++ b/src/plugins/nat/nat44-ed/nat44_ed_inlines.h @@ -638,8 +638,8 @@ nat_pre_node_fn_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p2->data); + clib_prefetch_load (p3->data); } next0 = def_next; diff --git a/src/plugins/nat/nat44-ed/nat44_ed_out2in.c b/src/plugins/nat/nat44-ed/nat44_ed_out2in.c index 995260a2499..eaf89937e54 100644 --- a/src/plugins/nat/nat44-ed/nat44_ed_out2in.c +++ b/src/plugins/nat/nat44-ed/nat44_ed_out2in.c @@ -838,7 +838,7 @@ nat44_ed_out2in_fast_path_node_fn_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p2->data); } next[0] = vnet_buffer2 (b0)->nat.arc_next; diff --git a/src/plugins/nat/nat44-ei/nat44_ei_handoff.c b/src/plugins/nat/nat44-ei/nat44_ei_handoff.c index 567f53915d2..c7a1317026b 100644 --- a/src/plugins/nat/nat44-ei/nat44_ei_handoff.c +++ b/src/plugins/nat/nat44-ei/nat44_ei_handoff.c @@ -108,10 +108,10 @@ nat44_ei_worker_handoff_fn_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_prefetch_buffer_header (b[5], LOAD); vlib_prefetch_buffer_header (b[6], LOAD); vlib_prefetch_buffer_header (b[7], LOAD); - CLIB_PREFETCH (&b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (&b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (&b[6]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (&b[7]->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (&b[4]->data); + clib_prefetch_load (&b[5]->data); + clib_prefetch_load (&b[6]->data); + clib_prefetch_load (&b[7]->data); } if (is_output) diff --git a/src/plugins/nat/nat44-ei/nat44_ei_in2out.c b/src/plugins/nat/nat44-ei/nat44_ei_in2out.c index 42923aa0f77..7ac1a92a61b 100644 --- a/src/plugins/nat/nat44-ei/nat44_ei_in2out.c +++ b/src/plugins/nat/nat44-ei/nat44_ei_in2out.c @@ -964,8 +964,8 @@ nat44_ei_in2out_node_fn_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p2->data); + clib_prefetch_load (p3->data); } if (is_output_feature) diff --git a/src/plugins/nat/nat44-ei/nat44_ei_out2in.c b/src/plugins/nat/nat44-ei/nat44_ei_out2in.c index 1f04ba056b5..7796b11cfd7 100644 --- a/src/plugins/nat/nat44-ei/nat44_ei_out2in.c +++ b/src/plugins/nat/nat44-ei/nat44_ei_out2in.c @@ -758,8 +758,8 @@ VLIB_NODE_FN (nat44_ei_out2in_node) vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p2->data); + clib_prefetch_load (p3->data); } vnet_buffer (b0)->snat.flags = 0; diff --git a/src/plugins/nsim/nsim_input.c b/src/plugins/nsim/nsim_input.c index 1976d051df7..65a7ae11251 100644 --- a/src/plugins/nsim/nsim_input.c +++ b/src/plugins/nsim/nsim_input.c @@ -94,7 +94,7 @@ nsim_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { /* prefetch one line / 2 entries ahead */ if ((((uword) ep) & (CLIB_CACHE_LINE_BYTES - 1)) == 0) - CLIB_PREFETCH ((ep + 2), CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load ((ep + 2)); ep = wp->entries + wp->head; from[0] = ep->buffer_index; diff --git a/src/plugins/rdma/input.c b/src/plugins/rdma/input.c index df101e544bd..f1c508affa2 100644 --- a/src/plugins/rdma/input.c +++ b/src/plugins/rdma/input.c @@ -389,10 +389,10 @@ rdma_device_input_bufs (vlib_main_t * vm, const rdma_device_t * rd, { if (PREDICT_TRUE (n_left_from >= 8)) { - CLIB_PREFETCH (&wc[4 + 0], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (&wc[4 + 1], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (&wc[4 + 2], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (&wc[4 + 3], CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (&wc[4 + 0]); + clib_prefetch_load (&wc[4 + 1]); + clib_prefetch_load (&wc[4 + 2]); + clib_prefetch_load (&wc[4 + 3]); vlib_prefetch_buffer_header (b[4 + 0], STORE); vlib_prefetch_buffer_header (b[4 + 1], STORE); vlib_prefetch_buffer_header (b[4 + 2], STORE); diff --git a/src/plugins/unittest/mpcap_node.c b/src/plugins/unittest/mpcap_node.c index f3c24843d82..083c22e32df 100644 --- a/src/plugins/unittest/mpcap_node.c +++ b/src/plugins/unittest/mpcap_node.c @@ -116,10 +116,10 @@ mpcap_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (b[5], STORE); vlib_prefetch_buffer_header (b[6], STORE); vlib_prefetch_buffer_header (b[7], STORE); - CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (b[4]->data); + clib_prefetch_store (b[5]->data); + clib_prefetch_store (b[6]->data); + clib_prefetch_store (b[7]->data); } next[0] = 0; diff --git a/src/vlib/buffer_node.h b/src/vlib/buffer_node.h index 9fece67f31f..9ca43d425fc 100644 --- a/src/vlib/buffer_node.h +++ b/src/vlib/buffer_node.h @@ -287,8 +287,8 @@ generic_buffer_node_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, 64, LOAD); - CLIB_PREFETCH (p3->data, 64, LOAD); + clib_prefetch_load (p2->data); + clib_prefetch_load (p3->data); } pi0 = to_next[0] = from[0]; diff --git a/src/vlib/counter.h b/src/vlib/counter.h index 01aefd8ebc0..9f5654292b9 100644 --- a/src/vlib/counter.h +++ b/src/vlib/counter.h @@ -254,7 +254,7 @@ vlib_prefetch_combined_counter (const vlib_combined_counter_main_t * cm, * This CPU's index is assumed to already be in cache */ cpu_counters = cm->counters[thread_index]; - CLIB_PREFETCH (cpu_counters + index, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (cpu_counters + index); } diff --git a/src/vnet/bonding/device.c b/src/vnet/bonding/device.c index 8a0c1092251..9e949b87214 100644 --- a/src/vnet/bonding/device.c +++ b/src/vnet/bonding/device.c @@ -416,10 +416,10 @@ bond_tx_inline (vlib_main_t * vm, bond_if_t * bif, vlib_buffer_t ** b, vlib_prefetch_buffer_header (pb[2], LOAD); vlib_prefetch_buffer_header (pb[3], LOAD); - CLIB_PREFETCH (pb[0]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (pb[1]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (pb[2]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (pb[3]->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (pb[0]->data); + clib_prefetch_load (pb[1]->data); + clib_prefetch_load (pb[2]->data); + clib_prefetch_load (pb[3]->data); } if (lb_alg == BOND_LB_L2) diff --git a/src/vnet/classify/flow_classify_node.c b/src/vnet/classify/flow_classify_node.c index 9462d467b70..4989bf0a012 100644 --- a/src/vnet/classify/flow_classify_node.c +++ b/src/vnet/classify/flow_classify_node.c @@ -97,9 +97,9 @@ flow_classify_inline (vlib_main_t * vm, p2 = vlib_get_buffer (vm, from[2]); vlib_prefetch_buffer_header (p1, STORE); - CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p1->data); vlib_prefetch_buffer_header (p2, STORE); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); } bi0 = from[0]; diff --git a/src/vnet/classify/ip_classify.c b/src/vnet/classify/ip_classify.c index 5a5cfed6904..a5c044521bf 100644 --- a/src/vnet/classify/ip_classify.c +++ b/src/vnet/classify/ip_classify.c @@ -102,9 +102,9 @@ ip_classify_inline (vlib_main_t * vm, p2 = vlib_get_buffer (vm, from[2]); vlib_prefetch_buffer_header (p1, STORE); - CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p1->data); vlib_prefetch_buffer_header (p2, STORE); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); } bi0 = from[0]; diff --git a/src/vnet/classify/vnet_classify.h b/src/vnet/classify/vnet_classify.h index fbac77c45bc..06784e0541e 100644 --- a/src/vnet/classify/vnet_classify.h +++ b/src/vnet/classify/vnet_classify.h @@ -328,7 +328,7 @@ vnet_classify_prefetch_bucket (vnet_classify_table_t * t, u64 hash) bucket_index = hash & (t->nbuckets - 1); - CLIB_PREFETCH (&t->buckets[bucket_index], CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (&t->buckets[bucket_index]); } static inline vnet_classify_entry_t * @@ -389,7 +389,7 @@ vnet_classify_prefetch_entry (vnet_classify_table_t * t, u64 hash) e = vnet_classify_entry_at_index (t, e, value_index); - CLIB_PREFETCH (e, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (e); } vnet_classify_entry_t *vnet_classify_find_entry (vnet_classify_table_t * t, diff --git a/src/vnet/devices/pipe/pipe.c b/src/vnet/devices/pipe/pipe.c index ec50f63d9b5..eb92b3c788a 100644 --- a/src/vnet/devices/pipe/pipe.c +++ b/src/vnet/devices/pipe/pipe.c @@ -324,8 +324,8 @@ pipe_rx (vlib_main_t * vm, p3 = vlib_get_buffer (vm, from[3]); vlib_prefetch_buffer_header (p2, STORE); vlib_prefetch_buffer_header (p3, STORE); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p2->data); + clib_prefetch_load (p3->data); } bi0 = from[0]; diff --git a/src/vnet/devices/virtio/vhost_user_input.c b/src/vnet/devices/virtio/vhost_user_input.c index 6abc1fc6e94..bdb3d27245b 100644 --- a/src/vnet/devices/virtio/vhost_user_input.c +++ b/src/vnet/devices/virtio/vhost_user_input.c @@ -163,8 +163,8 @@ vhost_user_input_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, (!(src3 = map_guest_mem (vui, cpy[3].src, map_hint)))) return 1; - CLIB_PREFETCH (src2, 64, LOAD); - CLIB_PREFETCH (src3, 64, LOAD); + clib_prefetch_load (src2); + clib_prefetch_load (src3); clib_memcpy_fast ((void *) cpy[0].dst, src0, cpy[0].len); clib_memcpy_fast ((void *) cpy[1].dst, src1, cpy[1].len); @@ -872,10 +872,10 @@ vhost_user_input_copy_packed (vhost_user_intf_t * vui, vhost_copy_t * cpy, bad = (src4 == 0) + (src5 == 0) + (src6 == 0) + (src7 == 0); if (PREDICT_FALSE (bad)) goto one_by_one; - CLIB_PREFETCH (src4, 64, LOAD); - CLIB_PREFETCH (src5, 64, LOAD); - CLIB_PREFETCH (src6, 64, LOAD); - CLIB_PREFETCH (src7, 64, LOAD); + clib_prefetch_load (src4); + clib_prefetch_load (src5); + clib_prefetch_load (src6); + clib_prefetch_load (src7); while (PREDICT_TRUE (copy_len >= 8)) { @@ -892,10 +892,10 @@ vhost_user_input_copy_packed (vhost_user_intf_t * vui, vhost_copy_t * cpy, if (PREDICT_FALSE (bad)) break; - CLIB_PREFETCH (src4, 64, LOAD); - CLIB_PREFETCH (src5, 64, LOAD); - CLIB_PREFETCH (src6, 64, LOAD); - CLIB_PREFETCH (src7, 64, LOAD); + clib_prefetch_load (src4); + clib_prefetch_load (src5); + clib_prefetch_load (src6); + clib_prefetch_load (src7); clib_memcpy_fast ((void *) cpy[0].dst, src0, cpy[0].len); clib_memcpy_fast ((void *) cpy[1].dst, src1, cpy[1].len); diff --git a/src/vnet/devices/virtio/vhost_user_output.c b/src/vnet/devices/virtio/vhost_user_output.c index 59ea8104d5b..40faefadb42 100644 --- a/src/vnet/devices/virtio/vhost_user_output.c +++ b/src/vnet/devices/virtio/vhost_user_output.c @@ -194,8 +194,8 @@ vhost_user_tx_copy (vhost_user_intf_t * vui, vhost_copy_t * cpy, (!(dst3 = map_guest_mem (vui, cpy[3].dst, map_hint)))) return 1; - CLIB_PREFETCH ((void *) cpy[2].src, 64, LOAD); - CLIB_PREFETCH ((void *) cpy[3].src, 64, LOAD); + clib_prefetch_load ((void *) cpy[2].src); + clib_prefetch_load ((void *) cpy[3].src); clib_memcpy_fast (dst0, (void *) cpy[0].src, cpy[0].len); clib_memcpy_fast (dst1, (void *) cpy[1].src, cpy[1].len); @@ -601,7 +601,7 @@ retry: buffer_map_addr += cpy->len; desc_len += cpy->len; - CLIB_PREFETCH (&rxvq->packed_desc, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (&rxvq->packed_desc); /* Check if vlib buffer has more data. If not, get more or break */ if (PREDICT_TRUE (!bytes_left)) @@ -922,7 +922,7 @@ retry: buffer_map_addr += cpy->len; desc_len += cpy->len; - CLIB_PREFETCH (&rxvq->desc, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (&rxvq->desc); } // Check if vlib buffer has more data. If not, get more or break. diff --git a/src/vnet/dpo/lookup_dpo.c b/src/vnet/dpo/lookup_dpo.c index 23a1285be53..3f34cefe0c4 100644 --- a/src/vnet/dpo/lookup_dpo.c +++ b/src/vnet/dpo/lookup_dpo.c @@ -377,8 +377,8 @@ lookup_dpo_ip4_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); } bi0 = from[0]; @@ -734,8 +734,8 @@ lookup_dpo_ip6_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); } bi0 = from[0]; diff --git a/src/vnet/ethernet/node.c b/src/vnet/ethernet/node.c index f470c1c7b46..cc0a0c30b9c 100644 --- a/src/vnet/ethernet/node.c +++ b/src/vnet/ethernet/node.c @@ -1505,7 +1505,7 @@ ethernet_input_inline (vlib_main_t * vm, if (n_left_from > 1) { vlib_prefetch_buffer_header (b[1], STORE); - CLIB_PREFETCH (b[1]->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b[1]->data); } bi0 = from[0]; diff --git a/src/vnet/ip/icmp4.c b/src/vnet/ip/icmp4.c index 05262e6f8e4..0363092d5d5 100644 --- a/src/vnet/ip/icmp4.c +++ b/src/vnet/ip/icmp4.c @@ -176,7 +176,7 @@ ip4_icmp_input (vlib_main_t * vm, vlib_prefetch_buffer_with_index (vm, from[2], LOAD); p0 = vlib_get_buffer (vm, from[1]); ip0 = vlib_buffer_get_current (p0); - CLIB_PREFETCH (ip0, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (ip0); } bi0 = to_next[0] = from[0]; diff --git a/src/vnet/ip/ip4_forward.c b/src/vnet/ip/ip4_forward.c index 36f05a27820..750b75ffcff 100644 --- a/src/vnet/ip/ip4_forward.c +++ b/src/vnet/ip/ip4_forward.c @@ -1762,8 +1762,8 @@ ip4_local_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (b[4], LOAD); vlib_prefetch_buffer_header (b[5], LOAD); - CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b[4]->data); + clib_prefetch_load (b[5]->data); } error[0] = error[1] = IP4_ERROR_UNKNOWN_PROTOCOL; @@ -2148,12 +2148,12 @@ ip4_rewrite_inline (vlib_main_t *vm, vlib_node_runtime_t *node, vnet_buffer (b[1])->ip.save_rewrite_length = rw_len1; p = vlib_buffer_get_current (b[2]); - CLIB_PREFETCH (p - CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_store (p - CLIB_CACHE_LINE_BYTES); + clib_prefetch_load (p); p = vlib_buffer_get_current (b[3]); - CLIB_PREFETCH (p - CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_store (p - CLIB_CACHE_LINE_BYTES); + clib_prefetch_load (p); /* Check MTU of outgoing interface. */ u16 ip0_len = clib_net_to_host_u16 (ip0->length); diff --git a/src/vnet/ip/ip6_forward.c b/src/vnet/ip/ip6_forward.c index fd742d6b3e2..ba616ebc7c7 100644 --- a/src/vnet/ip/ip6_forward.c +++ b/src/vnet/ip/ip6_forward.c @@ -1780,8 +1780,8 @@ ip6_rewrite_inline_with_gso (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->pre_data, 32, STORE); - CLIB_PREFETCH (p3->pre_data, 32, STORE); + clib_prefetch_store (p2->pre_data); + clib_prefetch_store (p3->pre_data); CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE); CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE); diff --git a/src/vnet/ip/ip6_hop_by_hop.c b/src/vnet/ip/ip6_hop_by_hop.c index 9923b416b65..e66084c2c4d 100644 --- a/src/vnet/ip/ip6_hop_by_hop.c +++ b/src/vnet/ip/ip6_hop_by_hop.c @@ -620,8 +620,8 @@ VLIB_NODE_FN (ip6_pop_hop_by_hop_node) (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); } /* speculatively enqueue b0 and b1 to the current next frame */ @@ -879,10 +879,10 @@ ip6_local_hop_by_hop_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (b[5], STORE); vlib_prefetch_buffer_header (b[6], STORE); vlib_prefetch_buffer_header (b[7], STORE); - CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (b[4]->data); + clib_prefetch_store (b[5]->data); + clib_prefetch_store (b[6]->data); + clib_prefetch_store (b[7]->data); } /* diff --git a/src/vnet/ip/ip_in_out_acl.c b/src/vnet/ip/ip_in_out_acl.c index 2f73e1307dc..a5e652e1ee8 100644 --- a/src/vnet/ip/ip_in_out_acl.c +++ b/src/vnet/ip/ip_in_out_acl.c @@ -220,8 +220,8 @@ ip_in_out_acl_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (b[4], LOAD); vlib_prefetch_buffer_header (b[5], LOAD); - CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b[4]->data); + clib_prefetch_load (b[5]->data); } /* calculate hashes for b[2] & b[3] */ diff --git a/src/vnet/ip/reass/ip4_sv_reass.c b/src/vnet/ip/reass/ip4_sv_reass.c index 43390eae2ab..9b3f1b98558 100644 --- a/src/vnet/ip/reass/ip4_sv_reass.c +++ b/src/vnet/ip/reass/ip4_sv_reass.c @@ -468,8 +468,8 @@ ip4_sv_reass_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p2->data); + clib_prefetch_load (p3->data); } ip4_header_t *ip0 = diff --git a/src/vnet/ipsec/esp_decrypt.c b/src/vnet/ipsec/esp_decrypt.c index b700f2c9935..e30fc9effcb 100644 --- a/src/vnet/ipsec/esp_decrypt.c +++ b/src/vnet/ipsec/esp_decrypt.c @@ -1068,9 +1068,9 @@ esp_decrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, u8 *p; vlib_prefetch_buffer_header (b[2], LOAD); p = vlib_buffer_get_current (b[1]); - CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p); p -= CLIB_CACHE_LINE_BYTES; - CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p); } u32 n_bufs = vlib_buffer_chain_linearize (vm, b[0]); @@ -1095,7 +1095,7 @@ esp_decrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, sa0 = ipsec_sa_get (current_sa_index); /* fetch the second cacheline ASAP */ - CLIB_PREFETCH (sa0->cacheline1, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (sa0->cacheline1); cpd.icv_sz = sa0->integ_icv_size; cpd.iv_sz = sa0->crypto_iv_size; cpd.flags = sa0->flags; diff --git a/src/vnet/ipsec/esp_encrypt.c b/src/vnet/ipsec/esp_encrypt.c index da9c56a7e03..d102bd68f74 100644 --- a/src/vnet/ipsec/esp_encrypt.c +++ b/src/vnet/ipsec/esp_encrypt.c @@ -626,9 +626,9 @@ esp_encrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, u8 *p; vlib_prefetch_buffer_header (b[2], LOAD); p = vlib_buffer_get_current (b[1]); - CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p); p -= CLIB_CACHE_LINE_BYTES; - CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (p); /* speculate that the trailer goes in the first buffer */ CLIB_PREFETCH (vlib_buffer_get_tail (b[1]), CLIB_CACHE_LINE_BYTES, LOAD); @@ -656,7 +656,7 @@ esp_encrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, sa0 = ipsec_sa_get (sa_index0); /* fetch the second cacheline ASAP */ - CLIB_PREFETCH (sa0->cacheline1, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (sa0->cacheline1); current_sa_index = sa_index0; spi = clib_net_to_host_u32 (sa0->spi); diff --git a/src/vnet/l2/l2_efp_filter.c b/src/vnet/l2/l2_efp_filter.c index 3d152ad1f11..ad325b83df2 100644 --- a/src/vnet/l2/l2_efp_filter.c +++ b/src/vnet/l2/l2_efp_filter.c @@ -227,8 +227,8 @@ VLIB_NODE_FN (l2_efp_filter_node) (vlib_main_t * vm, vlib_prefetch_buffer_header (p4, LOAD); vlib_prefetch_buffer_header (p5, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); /* * Prefetch the input config for the N+1 loop iteration @@ -238,8 +238,10 @@ VLIB_NODE_FN (l2_efp_filter_node) (vlib_main_t * vm, sw_if_index3 = vnet_buffer (p3)->sw_if_index[VLIB_TX]; /* * $$$ TODO - * CLIB_PREFETCH (vec_elt_at_index(l2output_main.configs, sw_if_index2), CLIB_CACHE_LINE_BYTES, LOAD); - * CLIB_PREFETCH (vec_elt_at_index(l2output_main.configs, sw_if_index3), CLIB_CACHE_LINE_BYTES, LOAD); + * clib_prefetch_load (vec_elt_at_index(l2output_main.configs, + * sw_if_index2)); + * clib_prefetch_load (vec_elt_at_index(l2output_main.configs, + * sw_if_index3)); */ } diff --git a/src/vnet/l2/l2_fib.c b/src/vnet/l2/l2_fib.c index 478f6766701..d1ee82273b1 100644 --- a/src/vnet/l2/l2_fib.c +++ b/src/vnet/l2/l2_fib.c @@ -1131,13 +1131,13 @@ l2fib_scan (vlib_main_t * vm, f64 start_time, u8 event_only) { BVT (clib_bihash_bucket) * b = BV (clib_bihash_get_bucket) (h, i + 3); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b); b = BV (clib_bihash_get_bucket) (h, i + 1); if (!BV (clib_bihash_bucket_is_empty) (b)) { BVT (clib_bihash_value) * v = BV (clib_bihash_get_value) (h, b->offset); - CLIB_PREFETCH (v, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (v); } } diff --git a/src/vnet/l2/l2_fwd.c b/src/vnet/l2/l2_fwd.c index e7accc29878..3414f6c490e 100644 --- a/src/vnet/l2/l2_fwd.c +++ b/src/vnet/l2/l2_fwd.c @@ -269,10 +269,10 @@ l2fwd_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (b[6], LOAD); vlib_prefetch_buffer_header (b[7], LOAD); - CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b[4]->data); + clib_prefetch_load (b[5]->data); + clib_prefetch_load (b[6]->data); + clib_prefetch_load (b[7]->data); } /* RX interface handles */ diff --git a/src/vnet/l2/l2_in_out_acl.c b/src/vnet/l2/l2_in_out_acl.c index df233672ef0..f8293c1feee 100644 --- a/src/vnet/l2/l2_in_out_acl.c +++ b/src/vnet/l2/l2_in_out_acl.c @@ -172,9 +172,9 @@ l2_in_out_acl_node_fn (vlib_main_t * vm, p2 = vlib_get_buffer (vm, from[2]); vlib_prefetch_buffer_header (p1, STORE); - CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p1->data); vlib_prefetch_buffer_header (p2, STORE); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); } bi0 = from[0]; diff --git a/src/vnet/l2/l2_in_out_feat_arc.c b/src/vnet/l2/l2_in_out_feat_arc.c index 37efa0732a5..b3b4a8cbb73 100644 --- a/src/vnet/l2/l2_in_out_feat_arc.c +++ b/src/vnet/l2/l2_in_out_feat_arc.c @@ -131,7 +131,7 @@ buffer_prefetch_xN (int vector_sz, vlib_buffer_t ** b) { int ii; for (ii = 0; ii < vector_sz; ii++) - CLIB_PREFETCH (b[ii], CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (b[ii]); } static_always_inline void diff --git a/src/vnet/l2/l2_input_classify.c b/src/vnet/l2/l2_input_classify.c index ffd1078d469..53d46399daf 100644 --- a/src/vnet/l2/l2_input_classify.c +++ b/src/vnet/l2/l2_input_classify.c @@ -190,9 +190,9 @@ VLIB_NODE_FN (l2_input_classify_node) (vlib_main_t * vm, p3 = vlib_get_buffer (vm, from[3]); vlib_prefetch_buffer_header (p2, STORE); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); vlib_prefetch_buffer_header (p3, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p3->data); } bi0 = from[0]; diff --git a/src/vnet/l2/l2_input_node.c b/src/vnet/l2/l2_input_node.c index 59a437e1b3f..3638a8aa00d 100644 --- a/src/vnet/l2/l2_input_node.c +++ b/src/vnet/l2/l2_input_node.c @@ -258,10 +258,10 @@ l2input_node_inline (vlib_main_t * vm, vlib_prefetch_buffer_header (b[6], LOAD); vlib_prefetch_buffer_header (b[7], LOAD); - CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (b[4]->data); + clib_prefetch_store (b[5]->data); + clib_prefetch_store (b[6]->data); + clib_prefetch_store (b[7]->data); } classify_and_dispatch (msm, b[0], &next[0]); diff --git a/src/vnet/l2/l2_input_vtr.c b/src/vnet/l2/l2_input_vtr.c index be3e6ba85f4..3c1235bfa32 100644 --- a/src/vnet/l2/l2_input_vtr.c +++ b/src/vnet/l2/l2_input_vtr.c @@ -126,8 +126,8 @@ VLIB_NODE_FN (l2_invtr_node) (vlib_main_t * vm, vlib_prefetch_buffer_header (p4, LOAD); vlib_prefetch_buffer_header (p5, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); /* * Prefetch the input config for the N+1 loop iteration diff --git a/src/vnet/l2/l2_learn.c b/src/vnet/l2/l2_learn.c index 3b9b1dbcaa9..6d90cee62a7 100644 --- a/src/vnet/l2/l2_learn.c +++ b/src/vnet/l2/l2_learn.c @@ -292,10 +292,10 @@ l2learn_node_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (b[6], LOAD); vlib_prefetch_buffer_header (b[7], LOAD); - CLIB_PREFETCH (b[4]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b[5]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b[6]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b[7]->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b[4]->data); + clib_prefetch_load (b[5]->data); + clib_prefetch_load (b[6]->data); + clib_prefetch_load (b[7]->data); } /* RX interface handles */ diff --git a/src/vnet/l2/l2_output.c b/src/vnet/l2/l2_output.c index 2c472b5c702..ba40de316d1 100644 --- a/src/vnet/l2/l2_output.c +++ b/src/vnet/l2/l2_output.c @@ -147,10 +147,10 @@ l2output_process_batch_inline (vlib_main_t * vm, vlib_node_runtime_t * node, /* prefetch eth headers only if we need to touch them */ if (l2_vtr || l2_pbb || shg_set) { - CLIB_PREFETCH (b[4]->data + cdo[4], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b[5]->data + cdo[5], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b[6]->data + cdo[6], CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b[7]->data + cdo[7], CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b[4]->data + cdo[4]); + clib_prefetch_load (b[5]->data + cdo[5]); + clib_prefetch_load (b[6]->data + cdo[6]); + clib_prefetch_load (b[7]->data + cdo[7]); } if (update_feature_bitmap) diff --git a/src/vnet/l2/l2_output_classify.c b/src/vnet/l2/l2_output_classify.c index 1e55afcaa4d..96d0b14753a 100644 --- a/src/vnet/l2/l2_output_classify.c +++ b/src/vnet/l2/l2_output_classify.c @@ -183,9 +183,9 @@ VLIB_NODE_FN (l2_output_classify_node) (vlib_main_t * vm, p3 = vlib_get_buffer (vm, from[3]); vlib_prefetch_buffer_header (p2, STORE); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); vlib_prefetch_buffer_header (p3, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p3->data); } bi0 = from[0]; diff --git a/src/vnet/l2/l2_xcrw.c b/src/vnet/l2/l2_xcrw.c index 942cd118d3d..c2c325a796f 100644 --- a/src/vnet/l2/l2_xcrw.c +++ b/src/vnet/l2/l2_xcrw.c @@ -107,8 +107,8 @@ VLIB_NODE_FN (l2_xcrw_node) (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); } /* speculatively enqueue b0 and b1 to the current next frame */ diff --git a/src/vnet/lawful-intercept/node.c b/src/vnet/lawful-intercept/node.c index 03b507adbdd..c5328e672d0 100644 --- a/src/vnet/lawful-intercept/node.c +++ b/src/vnet/lawful-intercept/node.c @@ -119,8 +119,8 @@ VLIB_NODE_FN (li_hit_node) (vlib_main_t * vm, vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); + clib_prefetch_store (p3->data); } /* speculatively enqueue b0 and b1 to the current next frame */ diff --git a/src/vnet/pipeline.h b/src/vnet/pipeline.h index 2ac1a9ac27d..afb3c38ca57 100644 --- a/src/vnet/pipeline.h +++ b/src/vnet/pipeline.h @@ -75,7 +75,7 @@ generic_stage0 (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_buffer_t * b AUX_DATA_ARG) { vlib_prefetch_buffer_header (b, STORE); - CLIB_PREFETCH (b->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (b->data); } #if NSTAGES == 2 diff --git a/src/vnet/policer/node_funcs.c b/src/vnet/policer/node_funcs.c index c7b6e8c7585..21b9393a222 100644 --- a/src/vnet/policer/node_funcs.c +++ b/src/vnet/policer/node_funcs.c @@ -387,9 +387,9 @@ policer_classify_inline (vlib_main_t * vm, p2 = vlib_get_buffer (vm, from[2]); vlib_prefetch_buffer_header (p1, STORE); - CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p1->data); vlib_prefetch_buffer_header (p2, STORE); - CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p2->data); } bi0 = from[0]; diff --git a/src/vnet/srv6/sr_localsid.c b/src/vnet/srv6/sr_localsid.c index 93619071ca3..a055c923be9 100644 --- a/src/vnet/srv6/sr_localsid.c +++ b/src/vnet/srv6/sr_localsid.c @@ -1195,10 +1195,10 @@ sr_localsid_d_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p6, LOAD); vlib_prefetch_buffer_header (p7, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); + clib_prefetch_store (p6->data); + clib_prefetch_store (p7->data); } to_next[0] = bi0 = from[0]; @@ -1502,10 +1502,10 @@ sr_localsid_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p6, LOAD); vlib_prefetch_buffer_header (p7, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); + clib_prefetch_store (p6->data); + clib_prefetch_store (p7->data); } to_next[0] = bi0 = from[0]; @@ -1812,10 +1812,10 @@ sr_localsid_un_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p6, LOAD); vlib_prefetch_buffer_header (p7, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); + clib_prefetch_store (p6->data); + clib_prefetch_store (p7->data); } to_next[0] = bi0 = from[0]; @@ -2117,10 +2117,10 @@ sr_localsid_un_perf_fn (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p6, LOAD); vlib_prefetch_buffer_header (p7, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); + clib_prefetch_store (p6->data); + clib_prefetch_store (p7->data); } to_next[0] = bi0 = from[0]; diff --git a/src/vnet/srv6/sr_policy_rewrite.c b/src/vnet/srv6/sr_policy_rewrite.c index 98fae3bd4a1..79de7792061 100644 --- a/src/vnet/srv6/sr_policy_rewrite.c +++ b/src/vnet/srv6/sr_policy_rewrite.c @@ -1307,10 +1307,10 @@ sr_policy_rewrite_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p6, LOAD); vlib_prefetch_buffer_header (p7, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); + clib_prefetch_store (p6->data); + clib_prefetch_store (p7->data); } to_next[0] = bi0 = from[0]; @@ -1600,10 +1600,10 @@ sr_policy_rewrite_encaps_v4 (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p6, LOAD); vlib_prefetch_buffer_header (p7, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); + clib_prefetch_store (p6->data); + clib_prefetch_store (p7->data); } to_next[0] = bi0 = from[0]; @@ -1904,10 +1904,10 @@ sr_policy_rewrite_encaps_l2 (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p6, LOAD); vlib_prefetch_buffer_header (p7, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); + clib_prefetch_store (p6->data); + clib_prefetch_store (p7->data); } to_next[0] = bi0 = from[0]; @@ -2300,10 +2300,10 @@ sr_policy_rewrite_insert (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p6, LOAD); vlib_prefetch_buffer_header (p7, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); + clib_prefetch_store (p6->data); + clib_prefetch_store (p7->data); } to_next[0] = bi0 = from[0]; @@ -2722,10 +2722,10 @@ sr_policy_rewrite_b_insert (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p6, LOAD); vlib_prefetch_buffer_header (p7, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); + clib_prefetch_store (p6->data); + clib_prefetch_store (p7->data); } to_next[0] = bi0 = from[0]; @@ -3165,10 +3165,10 @@ sr_policy_rewrite_b_encaps (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_prefetch_buffer_header (p6, LOAD); vlib_prefetch_buffer_header (p7, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); + clib_prefetch_store (p6->data); + clib_prefetch_store (p7->data); } to_next[0] = bi0 = from[0]; diff --git a/src/vnet/tcp/tcp_syn_filter4.c b/src/vnet/tcp/tcp_syn_filter4.c index b107899cfb9..ef7a3280c03 100644 --- a/src/vnet/tcp/tcp_syn_filter4.c +++ b/src/vnet/tcp/tcp_syn_filter4.c @@ -135,10 +135,10 @@ VLIB_NODE_FN (syn_filter4_node) (vlib_main_t * vm, vlib_prefetch_buffer_header (p6, LOAD); vlib_prefetch_buffer_header (p7, LOAD); - CLIB_PREFETCH (p4->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p5->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p6->data, CLIB_CACHE_LINE_BYTES, STORE); - CLIB_PREFETCH (p7->data, CLIB_CACHE_LINE_BYTES, STORE); + clib_prefetch_store (p4->data); + clib_prefetch_store (p5->data); + clib_prefetch_store (p6->data); + clib_prefetch_store (p7->data); } /* speculatively enqueue b0 and b1 to the current next frame */ diff --git a/src/vnet/vxlan/decap.c b/src/vnet/vxlan/decap.c index 2ba24d881af..729293fb3e5 100644 --- a/src/vnet/vxlan/decap.c +++ b/src/vnet/vxlan/decap.c @@ -249,8 +249,8 @@ vxlan_input (vlib_main_t * vm, vxlan6_find_tunnel (vxm, &last6, fi1, ip6_1, vxlan1, &stats_if1); /* Prefetch next iteration. */ - CLIB_PREFETCH (b[2]->data, CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b[3]->data, CLIB_CACHE_LINE_BYTES, LOAD); + clib_prefetch_load (b[2]->data); + clib_prefetch_load (b[3]->data); u32 len0 = vlib_buffer_length_in_chain (vm, b[0]); u32 len1 = vlib_buffer_length_in_chain (vm, b[1]); diff --git a/src/vppinfra/callback_data.h b/src/vppinfra/callback_data.h index 9a1ad0a9778..23a7f900b99 100644 --- a/src/vppinfra/callback_data.h +++ b/src/vppinfra/callback_data.h @@ -273,16 +273,18 @@ do { \ * @param vec_ the callback vector * @param varargs additional callback parameters */ -#define clib_callback_data_call_vec(vec_, ...) \ -do { \ - u32 sz_ = vec_len (vec_); \ - u32 i_; \ - for (i_ = 0; i_ < sz_; i_++) \ - { \ - CLIB_PREFETCH (&vec_[i_+1], CLIB_CACHE_LINE_BYTES, STORE); \ - (vec_[i_].fp) (&vec_[i_], __VA_ARGS__); \ - } \ -} while (0) +#define clib_callback_data_call_vec(vec_, ...) \ + do \ + { \ + u32 sz_ = vec_len (vec_); \ + u32 i_; \ + for (i_ = 0; i_ < sz_; i_++) \ + { \ + clib_prefetch_store (&vec_[i_ + 1]); \ + (vec_[i_].fp) (&vec_[i_], __VA_ARGS__); \ + } \ + } \ + while (0) /** @brief Call the specified callback set * @param set_ the callback set @@ -297,12 +299,13 @@ do { \ /** @brief prefetch the callback set * @param set_ The callback set */ -#define clib_callback_data_prefetch(set_) \ -do { \ - if (PREDICT_FALSE ((set_)->curr)) \ - CLIB_PREFETCH ((set_)->curr, CLIB_CACHE_LINE_BYTES, STORE); \ -} while (0) - +#define clib_callback_data_prefetch(set_) \ + do \ + { \ + if (PREDICT_FALSE ((set_)->curr)) \ + clib_prefetch_store ((set_)->curr); \ + } \ + while (0) #endif /* included_callback_data_h */ |