From 01b2c23faecee4a60cccf8219bf37fa588009b8d Mon Sep 17 00:00:00 2001 From: Dave Barach Date: Thu, 3 May 2018 08:26:39 -0400 Subject: Harmonize vec/pool_get_aligned object sizes and alignment requests Object sizes must evenly divide alignment requests, or vice versa. Otherwise, only the first object will be aligned as requested. Three choices: add CLIB_CACHE_LINE_ALIGN_MARK(align_me) at the end of structures, manually pad to an even divisor or multiple of the alignment request, or use plain vectors/pools. static assert for enforcement. Change-Id: I41aa6ff1a58267301d32aaf4b9cd24678ac1c147 Signed-off-by: Dave Barach --- src/plugins/acl/acl.h | 6 ++++++ src/plugins/dpdk/device/dpdk.h | 5 +++++ src/plugins/dpdk/ipsec/ipsec.h | 2 ++ src/plugins/gtpu/gtpu.h | 3 +++ src/plugins/ioam/analyse/ioam_analyse.h | 2 ++ src/plugins/ioam/export-common/ioam_export.h | 2 ++ src/plugins/ioam/ip6/ioam_cache.h | 7 +++++-- src/plugins/ioam/udp-ping/udp_ping.h | 3 +++ src/plugins/memif/private.h | 2 ++ src/plugins/pppoe/pppoe.h | 3 +++ src/vlib/unix/input.c | 1 + src/vnet/bier/bier_disp_entry.h | 5 +++++ src/vnet/bier/bier_disp_table.h | 5 +++++ src/vnet/bier/bier_fmask.h | 5 +++++ src/vnet/bier/bier_imp.h | 5 +++++ src/vnet/bier/bier_table.h | 11 ++++++----- src/vnet/dpo/classify_dpo.h | 5 +++++ src/vnet/dpo/l3_proxy_dpo.h | 6 ++++++ src/vnet/dpo/load_balance.h | 6 ++++++ src/vnet/dpo/load_balance_map.h | 6 ++++++ src/vnet/dpo/lookup_dpo.h | 6 ++++++ src/vnet/dpo/mpls_disposition.h | 6 ++++++ src/vnet/dpo/mpls_label_dpo.h | 6 ++++++ src/vnet/dpo/receive_dpo.h | 6 ++++++ src/vnet/dpo/replicate_dpo.h | 6 ++++++ src/vnet/fib/ip4_fib.c | 2 +- src/vnet/fib/ip4_fib.h | 2 ++ src/vnet/fib/ip6_fib.c | 2 +- src/vnet/fib/mpls_fib.c | 2 +- src/vnet/fib/mpls_fib.h | 5 +++++ src/vnet/geneve/geneve.h | 3 +++ src/vnet/gre/gre.h | 5 +++++ src/vnet/ip/ip6.h | 3 +++ src/vnet/ip/ip_source_and_port_range_check.h | 5 +++++ src/vnet/ipip/ipip.h | 3 +++ src/vnet/ipsec-gre/ipsec_gre.h | 2 ++ src/vnet/ipsec/ipsec.h | 2 ++ src/vnet/map/map.h | 2 ++ src/vnet/mfib/mfib_itf.h | 7 ++++++- src/vnet/mfib/mfib_table.h | 5 +++++ src/vnet/qos/qos_egress_map.h | 13 +++++++++---- src/vnet/sctp/sctp.h | 3 +++ src/vnet/session/session_table.h | 2 ++ src/vnet/udp/udp.h | 2 ++ src/vnet/vxlan-gpe/vxlan_gpe.h | 3 +++ src/vnet/vxlan/vxlan.c | 2 +- src/vnet/vxlan/vxlan.h | 3 +++ src/vpp/oam/oam.c | 2 +- src/vppinfra/pool.h | 2 ++ src/vppinfra/vec.h | 2 ++ 50 files changed, 187 insertions(+), 17 deletions(-) diff --git a/src/plugins/acl/acl.h b/src/plugins/acl/acl.h index 8f3112dd258..0865b1ed720 100644 --- a/src/plugins/acl/acl.h +++ b/src/plugins/acl/acl.h @@ -101,6 +101,8 @@ typedef struct */ typedef struct { + /** Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); u8 tag[64]; u32 count; acl_rule_t *rules; @@ -108,6 +110,8 @@ typedef struct typedef struct { + /** Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); u8 tag[64]; u32 count; macip_acl_rule_t *rules; @@ -127,6 +131,8 @@ typedef struct */ typedef struct { + /** Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); fa_5tuple_t mask; u32 refcount; } ace_mask_type_entry_t; diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h index 1f43a484224..18f15f76767 100644 --- a/src/plugins/dpdk/device/dpdk.h +++ b/src/plugins/dpdk/device/dpdk.h @@ -125,6 +125,9 @@ typedef uint16_t dpdk_portid_t; typedef struct { + /* Required for vec_validate_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + struct rte_ring *swq; u64 hqos_field0_slabmask; @@ -141,6 +144,8 @@ typedef struct typedef struct { + /* Required for vec_validate_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); struct rte_ring **swq; struct rte_mbuf **pkts_enq; struct rte_mbuf **pkts_deq; diff --git a/src/plugins/dpdk/ipsec/ipsec.h b/src/plugins/dpdk/ipsec/ipsec.h index d1009ecdb33..18c015e8ac2 100644 --- a/src/plugins/dpdk/ipsec/ipsec.h +++ b/src/plugins/dpdk/ipsec/ipsec.h @@ -126,6 +126,8 @@ typedef struct typedef struct { + /* Required for vec_validate_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); struct rte_mempool *crypto_op; struct rte_mempool *session_h; struct rte_mempool **session_drv; diff --git a/src/plugins/gtpu/gtpu.h b/src/plugins/gtpu/gtpu.h index b1e677d4f29..6331aa31890 100644 --- a/src/plugins/gtpu/gtpu.h +++ b/src/plugins/gtpu/gtpu.h @@ -123,6 +123,9 @@ typedef CLIB_PACKED typedef struct { + /* Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /* Rewrite string */ u8 *rewrite; diff --git a/src/plugins/ioam/analyse/ioam_analyse.h b/src/plugins/ioam/analyse/ioam_analyse.h index ef2865daf8c..881b0451721 100644 --- a/src/plugins/ioam/analyse/ioam_analyse.h +++ b/src/plugins/ioam/analyse/ioam_analyse.h @@ -96,6 +96,8 @@ typedef struct */ typedef struct ioam_analyser_data_t_ { + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + u8 is_free; u8 pad[3]; diff --git a/src/plugins/ioam/export-common/ioam_export.h b/src/plugins/ioam/export-common/ioam_export.h index 9de0d13bd5e..74347683e10 100644 --- a/src/plugins/ioam/export-common/ioam_export.h +++ b/src/plugins/ioam/export-common/ioam_export.h @@ -33,6 +33,8 @@ typedef struct ioam_export_buffer { + /** Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); /* Allocated buffer */ u32 buffer_index; u64 touched_at; diff --git a/src/plugins/ioam/ip6/ioam_cache.h b/src/plugins/ioam/ip6/ioam_cache.h index 338ed9828ee..3a3cf6a620e 100644 --- a/src/plugins/ioam/ip6/ioam_cache.h +++ b/src/plugins/ioam/ip6/ioam_cache.h @@ -98,6 +98,8 @@ */ typedef struct { + /** Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); ip6_address_t src_address; ip6_address_t dst_address; u16 src_port; @@ -122,6 +124,8 @@ typedef struct */ typedef struct { + /** Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); u32 pool_id; u32 pool_index; ip6_address_t src_address; @@ -528,8 +532,7 @@ ioam_cache_ts_table_init (vlib_main_t * vm) CLIB_CACHE_LINE_BYTES); vec_validate_aligned (cm->ts_stats, no_of_threads - 1, CLIB_CACHE_LINE_BYTES); - vec_validate_aligned (cm->timer_wheels, no_of_threads - 1, - CLIB_CACHE_LINE_BYTES); + vec_validate (cm->timer_wheels, no_of_threads - 1); cm->lookup_table_nbuckets = IOAM_CACHE_TABLE_DEFAULT_HASH_NUM_BUCKETS; cm->lookup_table_nbuckets = 1 << max_log2 (cm->lookup_table_nbuckets); cm->lookup_table_size = IOAM_CACHE_TABLE_DEFAULT_HASH_MEMORY_SIZE; diff --git a/src/plugins/ioam/udp-ping/udp_ping.h b/src/plugins/ioam/udp-ping/udp_ping.h index 26c4201900e..2c0d214ee35 100644 --- a/src/plugins/ioam/udp-ping/udp_ping.h +++ b/src/plugins/ioam/udp-ping/udp_ping.h @@ -84,6 +84,9 @@ typedef struct */ typedef struct { + /** Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /** Local source IPv4/6 address to be used. */ ip46_address_t src; diff --git a/src/plugins/memif/private.h b/src/plugins/memif/private.h index ad6295e4bb8..5c8a919061a 100644 --- a/src/plugins/memif/private.h +++ b/src/plugins/memif/private.h @@ -80,6 +80,8 @@ typedef struct typedef struct { + /* Required for vec_validate_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); void *shm; memif_region_size_t region_size; int fd; diff --git a/src/plugins/pppoe/pppoe.h b/src/plugins/pppoe/pppoe.h index 77bc88fea60..f010750f420 100644 --- a/src/plugins/pppoe/pppoe.h +++ b/src/plugins/pppoe/pppoe.h @@ -48,6 +48,9 @@ typedef struct typedef struct { + /* Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /* pppoe session_id in HOST byte order */ u16 session_id; diff --git a/src/vlib/unix/input.c b/src/vlib/unix/input.c index 0c298446636..321e443dee9 100644 --- a/src/vlib/unix/input.c +++ b/src/vlib/unix/input.c @@ -52,6 +52,7 @@ typedef struct { + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); int epoll_fd; struct epoll_event *epoll_events; int n_epoll_fds; diff --git a/src/vnet/bier/bier_disp_entry.h b/src/vnet/bier/bier_disp_entry.h index 34ca5d4a09a..1647a7e166a 100644 --- a/src/vnet/bier/bier_disp_entry.h +++ b/src/vnet/bier/bier_disp_entry.h @@ -30,6 +30,11 @@ * The BIER dispositon object */ typedef struct bier_disp_entry_t_ { + /** + * Required for pool_get_aligned + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * The DPO contirubted from the per-payload protocol parents * on cachline 1. diff --git a/src/vnet/bier/bier_disp_table.h b/src/vnet/bier/bier_disp_table.h index 6f9380a507f..c5b211001e2 100644 --- a/src/vnet/bier/bier_disp_table.h +++ b/src/vnet/bier/bier_disp_table.h @@ -29,6 +29,11 @@ */ typedef struct bier_disp_table_t_ { + /** + * Required for pool_get_aligned + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * number of locks on the table */ diff --git a/src/vnet/bier/bier_fmask.h b/src/vnet/bier/bier_fmask.h index 81b3923f54c..56930f6cd44 100644 --- a/src/vnet/bier/bier_fmask.h +++ b/src/vnet/bier/bier_fmask.h @@ -97,6 +97,11 @@ typedef enum bier_fmask_flags_t_ * for the next lookup */ typedef struct bier_fmask_t_ { + /** + * Required for pool_get_aligned + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * The BIER fmask is a child of a FIB entry in the FIB graph. */ diff --git a/src/vnet/bier/bier_imp.h b/src/vnet/bier/bier_imp.h index 5b21b06b12d..cdc4dbb4fc1 100644 --- a/src/vnet/bier/bier_imp.h +++ b/src/vnet/bier/bier_imp.h @@ -32,6 +32,11 @@ * The BIER imposition object */ typedef struct bier_imp_t_ { + /** + * Required for pool_get_aligned + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * The DPO contirubted from the resolving BIER table. * One per-IP protocol. This allows us to share a BIER imposition diff --git a/src/vnet/bier/bier_table.h b/src/vnet/bier/bier_table.h index a22e2e335b2..5af275f104a 100644 --- a/src/vnet/bier/bier_table.h +++ b/src/vnet/bier/bier_table.h @@ -36,6 +36,12 @@ struct bier_route_update_t_; * bit-position. Since this is smal <4096, the table is a flat arry */ typedef struct bier_table_t_ { + /** + * required for pool_get_aligned. + * memebers used in the switch path come first! + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * Save the MPLS local label associated with the table */ @@ -64,11 +70,6 @@ typedef struct bier_table_t_ { */ index_t *bt_entries; - /** - * Everything before this declaration is unused in the switch path - */ - CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); - /** * The identity/key or the table. we need the hdr_len in the data-path */ diff --git a/src/vnet/dpo/classify_dpo.h b/src/vnet/dpo/classify_dpo.h index 48f4b2bf8a5..439727d351f 100644 --- a/src/vnet/dpo/classify_dpo.h +++ b/src/vnet/dpo/classify_dpo.h @@ -25,6 +25,11 @@ */ typedef struct classify_dpo_t { + /** + * required for pool_get_aligned. + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + dpo_proto_t cd_proto; u32 cd_table_index; diff --git a/src/vnet/dpo/l3_proxy_dpo.h b/src/vnet/dpo/l3_proxy_dpo.h index f17ace50876..fcc28b3bf97 100644 --- a/src/vnet/dpo/l3_proxy_dpo.h +++ b/src/vnet/dpo/l3_proxy_dpo.h @@ -26,6 +26,12 @@ typedef struct l3_proxy_dpo_t_ { + /** + * required for pool_get_aligned. + * memebers used in the switch path come first! + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * The Software interface index on which traffic is l3_proxyd */ diff --git a/src/vnet/dpo/load_balance.h b/src/vnet/dpo/load_balance.h index d37f07d579f..dd9589f9ee8 100644 --- a/src/vnet/dpo/load_balance.h +++ b/src/vnet/dpo/load_balance.h @@ -82,6 +82,12 @@ typedef struct load_balance_path_t_ { * - per-route counters */ typedef struct load_balance_t_ { + /** + * required for pool_get_aligned. + * memebers used in the switch path come first! + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * number of buckets in the load-balance. always a power of 2. */ diff --git a/src/vnet/dpo/load_balance_map.h b/src/vnet/dpo/load_balance_map.h index f9344c01c1e..82dd36b41b1 100644 --- a/src/vnet/dpo/load_balance_map.h +++ b/src/vnet/dpo/load_balance_map.h @@ -28,6 +28,12 @@ struct load_balance_map_path_t_; /** */ typedef struct load_balance_map_t_ { + /** + * required for pool_get_aligned. + * memebers used in the switch path come first! + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * The buckets of the map that provide the index to index translation. * In the first cacheline. diff --git a/src/vnet/dpo/lookup_dpo.h b/src/vnet/dpo/lookup_dpo.h index 4ebd6050fc6..21aecadc280 100644 --- a/src/vnet/dpo/lookup_dpo.h +++ b/src/vnet/dpo/lookup_dpo.h @@ -64,6 +64,12 @@ typedef enum lookup_cast_t_ { */ typedef struct lookup_dpo_t { + /** + * required for pool_get_aligned. + * memebers used in the switch path come first! + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * The FIB, or interface from which to get a FIB, in which to perform * the next lookup; diff --git a/src/vnet/dpo/mpls_disposition.h b/src/vnet/dpo/mpls_disposition.h index 9c3cc46ff30..86dd60ae290 100644 --- a/src/vnet/dpo/mpls_disposition.h +++ b/src/vnet/dpo/mpls_disposition.h @@ -26,6 +26,12 @@ */ typedef struct mpls_disp_dpo_t { + /** + * required for pool_get_aligned. + * memebers used in the switch path come first! + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * Next DPO in the graph */ diff --git a/src/vnet/dpo/mpls_label_dpo.h b/src/vnet/dpo/mpls_label_dpo.h index 98c88f7d812..4e032d33458 100644 --- a/src/vnet/dpo/mpls_label_dpo.h +++ b/src/vnet/dpo/mpls_label_dpo.h @@ -66,6 +66,12 @@ extern u8* format_mpls_label_dpo_flags(u8 *s, va_list *args); */ typedef struct mpls_label_dpo_t { + /** + * required for pool_get_aligned. + * memebers used in the switch path come first! + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * The MPLS label header to impose. Outer most label first. * Each DPO will occupy one cache line, stuff that many labels in. diff --git a/src/vnet/dpo/receive_dpo.h b/src/vnet/dpo/receive_dpo.h index 2420fd7843c..9ab86a824c6 100644 --- a/src/vnet/dpo/receive_dpo.h +++ b/src/vnet/dpo/receive_dpo.h @@ -25,6 +25,12 @@ typedef struct receive_dpo_t_ { + /** + * required for pool_get_aligned. + * memebers used in the switch path come first! + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * The Software interface index on which traffic is received */ diff --git a/src/vnet/dpo/replicate_dpo.h b/src/vnet/dpo/replicate_dpo.h index ccb25630e16..7b075394bc3 100644 --- a/src/vnet/dpo/replicate_dpo.h +++ b/src/vnet/dpo/replicate_dpo.h @@ -52,6 +52,12 @@ extern replicate_main_t replicate_main; * - per-route counters */ typedef struct replicate_t_ { + /** + * required for pool_get_aligned. + * memebers used in the switch path come first! + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * number of buckets in the replicate. */ diff --git a/src/vnet/fib/ip4_fib.c b/src/vnet/fib/ip4_fib.c index ef3324645c3..4ab6b1bcff6 100644 --- a/src/vnet/fib/ip4_fib.c +++ b/src/vnet/fib/ip4_fib.c @@ -108,7 +108,7 @@ ip4_create_fib_with_table_id (u32 table_id, ip4_fib_t *v4_fib; void *old_heap; - pool_get_aligned(ip4_main.fibs, fib_table, CLIB_CACHE_LINE_BYTES); + pool_get(ip4_main.fibs, fib_table); memset(fib_table, 0, sizeof(*fib_table)); old_heap = clib_mem_set_heap (ip4_main.mtrie_mheap); diff --git a/src/vnet/fib/ip4_fib.h b/src/vnet/fib/ip4_fib.h index a0b1143ba8c..ff0d6677e53 100644 --- a/src/vnet/fib/ip4_fib.h +++ b/src/vnet/fib/ip4_fib.h @@ -58,6 +58,8 @@ typedef struct ip4_fib_t_ u32 fwd_classify_table_index; u32 rev_classify_table_index; + /* Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); } ip4_fib_t; extern fib_node_index_t ip4_fib_table_lookup(const ip4_fib_t *fib, diff --git a/src/vnet/fib/ip6_fib.c b/src/vnet/fib/ip6_fib.c index fa9eb112986..da2f1ea3368 100644 --- a/src/vnet/fib/ip6_fib.c +++ b/src/vnet/fib/ip6_fib.c @@ -60,7 +60,7 @@ create_fib_with_table_id (u32 table_id, fib_table_t *fib_table; ip6_fib_t *v6_fib; - pool_get_aligned(ip6_main.fibs, fib_table, CLIB_CACHE_LINE_BYTES); + pool_get(ip6_main.fibs, fib_table); pool_get_aligned(ip6_main.v6_fibs, v6_fib, CLIB_CACHE_LINE_BYTES); memset(fib_table, 0, sizeof(*fib_table)); diff --git a/src/vnet/fib/mpls_fib.c b/src/vnet/fib/mpls_fib.c index 15931203122..223a0ddd753 100644 --- a/src/vnet/fib/mpls_fib.c +++ b/src/vnet/fib/mpls_fib.c @@ -92,7 +92,7 @@ mpls_fib_create_with_table_id (u32 table_id, mpls_fib_t *mf; int i; - pool_get_aligned(mpls_main.fibs, fib_table, CLIB_CACHE_LINE_BYTES); + pool_get(mpls_main.fibs, fib_table); pool_get_aligned(mpls_main.mpls_fibs, mf, CLIB_CACHE_LINE_BYTES); ASSERT((fib_table - mpls_main.fibs) == diff --git a/src/vnet/fib/mpls_fib.h b/src/vnet/fib/mpls_fib.h index 9e0c7d76ff8..8d18b008b0e 100644 --- a/src/vnet/fib/mpls_fib.h +++ b/src/vnet/fib/mpls_fib.h @@ -40,6 +40,11 @@ typedef struct mpls_fib_t_ { + /** + * Required for pool_get_aligned + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * A hash table of entries. 21 bit key * Hash table for reduced memory footprint diff --git a/src/vnet/geneve/geneve.h b/src/vnet/geneve/geneve.h index cce57c8873b..8f028df517f 100644 --- a/src/vnet/geneve/geneve.h +++ b/src/vnet/geneve/geneve.h @@ -75,6 +75,9 @@ typedef CLIB_PACKED (struct typedef struct { + /* Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /* Rewrite string. $$$$ embed vnet_rewrite header */ u8 *rewrite; diff --git a/src/vnet/gre/gre.h b/src/vnet/gre/gre.h index b3b0b545190..416ab300877 100644 --- a/src/vnet/gre/gre.h +++ b/src/vnet/gre/gre.h @@ -175,6 +175,11 @@ typedef struct */ typedef struct { + /** + * Required for pool_get_aligned + */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /** * Linkage into the FIB object graph */ diff --git a/src/vnet/ip/ip6.h b/src/vnet/ip/ip6.h index 959d72c0fad..b16359475ee 100644 --- a/src/vnet/ip/ip6.h +++ b/src/vnet/ip/ip6.h @@ -66,6 +66,9 @@ typedef struct typedef struct { + /* required for pool_get_aligned. */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /* Table ID (hash key) for this FIB. */ u32 table_id; diff --git a/src/vnet/ip/ip_source_and_port_range_check.h b/src/vnet/ip/ip_source_and_port_range_check.h index b04fe613d1c..ece2ec216e4 100644 --- a/src/vnet/ip/ip_source_and_port_range_check.h +++ b/src/vnet/ip/ip_source_and_port_range_check.h @@ -101,6 +101,11 @@ typedef struct */ typedef struct protocol_port_range_dpo_t_ { + /** + * Required for pool_get_aligned + */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /** * The number of blocks from the 'block' array below * that have rnages configured. We keep this count so that in the data-path diff --git a/src/vnet/ipip/ipip.h b/src/vnet/ipip/ipip.h index e9277d8a061..f52094f1f0c 100644 --- a/src/vnet/ipip/ipip.h +++ b/src/vnet/ipip/ipip.h @@ -68,6 +68,9 @@ typedef enum */ typedef struct { + /* Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + ipip_mode_t mode; ipip_transport_t transport; ipip_tunnel_key_t *key; diff --git a/src/vnet/ipsec-gre/ipsec_gre.h b/src/vnet/ipsec-gre/ipsec_gre.h index 5d56b4e1e1f..caf2ecf8b30 100644 --- a/src/vnet/ipsec-gre/ipsec_gre.h +++ b/src/vnet/ipsec-gre/ipsec_gre.h @@ -49,6 +49,8 @@ typedef enum */ typedef struct { + /* Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); ip4_address_t tunnel_src; /**< tunnel IPv4 src address */ ip4_address_t tunnel_dst; /**< tunnel IPv4 dst address */ u32 local_sa; /**< local IPSec SA index */ diff --git a/src/vnet/ipsec/ipsec.h b/src/vnet/ipsec/ipsec.h index 32bdee9c062..319f1f6750c 100644 --- a/src/vnet/ipsec/ipsec.h +++ b/src/vnet/ipsec/ipsec.h @@ -240,6 +240,8 @@ typedef struct typedef struct { + /* Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); u32 input_sa_index; u32 output_sa_index; u32 hw_if_index; diff --git a/src/vnet/map/map.h b/src/vnet/map/map.h index 5700cc1c191..c304a1ea38a 100644 --- a/src/vnet/map/map.h +++ b/src/vnet/map/map.h @@ -80,6 +80,8 @@ typedef enum */ typedef struct { + /* Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); ip6_address_t ip6_src; ip6_address_t ip6_prefix; ip6_address_t *rules; diff --git a/src/vnet/mfib/mfib_itf.h b/src/vnet/mfib/mfib_itf.h index 5f26a476525..7bf64febf85 100644 --- a/src/vnet/mfib/mfib_itf.h +++ b/src/vnet/mfib/mfib_itf.h @@ -25,7 +25,12 @@ typedef struct mfib_itf_t_ { /** - * @brief Falags on the entry + * Required for pool_get_aligned + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + + /** + * @brief Forwarding Flags on the entry - checked in the data-path */ mfib_itf_flags_t mfi_flags; diff --git a/src/vnet/mfib/mfib_table.h b/src/vnet/mfib/mfib_table.h index 63af25b0151..b8ade8b5cd9 100644 --- a/src/vnet/mfib/mfib_table.h +++ b/src/vnet/mfib/mfib_table.h @@ -34,6 +34,11 @@ */ typedef struct mfib_table_t_ { + /** + * Required for pool_get_aligned + */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /** * A union of the protocol specific FIBs that provide the * underlying LPM mechanism. diff --git a/src/vnet/qos/qos_egress_map.h b/src/vnet/qos/qos_egress_map.h index f4b282cb7ba..92baf6bf3b2 100644 --- a/src/vnet/qos/qos_egress_map.h +++ b/src/vnet/qos/qos_egress_map.h @@ -36,10 +36,15 @@ typedef u32 qos_egress_map_id_t; */ typedef struct qos_egress_map_t_ { - /** - * The array of output mapped values; - * output = eq_qos[input-source][input-value] - */ + /** + * Required for pool_get_aligned + */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + + /** + * The array of output mapped values; + * output = eq_qos[input-source][input-value] + */ qos_bits_t qem_output[QOS_N_SOURCES][256]; } qos_egress_map_t; diff --git a/src/vnet/sctp/sctp.h b/src/vnet/sctp/sctp.h index ffe3e1b195f..5f19566d240 100644 --- a/src/vnet/sctp/sctp.h +++ b/src/vnet/sctp/sctp.h @@ -192,6 +192,9 @@ typedef struct _sctp_user_configuration typedef struct _sctp_connection { + /** Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + sctp_sub_connection_t sub_conn[MAX_SCTP_CONNECTIONS]; /**< Common transport data. First! */ sctp_user_configuration_t conn_config; /**< Allows tuning of some SCTP behaviors */ diff --git a/src/vnet/session/session_table.h b/src/vnet/session/session_table.h index 31b1f647e23..f7f8c91b26c 100644 --- a/src/vnet/session/session_table.h +++ b/src/vnet/session/session_table.h @@ -49,6 +49,8 @@ typedef struct _session_lookup_table * byproduct of fib table ids not necessarily being the same for * identical fib idices of v4 and v6 fib protos */ u8 active_fib_proto; + /* Required for pool_get_aligned(...) */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); } session_table_t; #define SESSION_TABLE_INVALID_INDEX ((u32)~0) diff --git a/src/vnet/udp/udp.h b/src/vnet/udp/udp.h index af6c6b82c6a..d6a93d422c6 100644 --- a/src/vnet/udp/udp.h +++ b/src/vnet/udp/udp.h @@ -36,6 +36,8 @@ typedef enum typedef struct { + /** Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); transport_connection_t connection; /** must be first */ /** ersatz MTU to limit fifo pushes to test data size */ u32 mtu; diff --git a/src/vnet/vxlan-gpe/vxlan_gpe.h b/src/vnet/vxlan-gpe/vxlan_gpe.h index 8c0d37eb07a..794ed7ef706 100644 --- a/src/vnet/vxlan-gpe/vxlan_gpe.h +++ b/src/vnet/vxlan-gpe/vxlan_gpe.h @@ -101,6 +101,9 @@ typedef CLIB_PACKED(struct { */ typedef struct { + /* Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + /** Rewrite string. $$$$ embed vnet_rewrite header */ u8 *rewrite; diff --git a/src/vnet/vxlan/vxlan.c b/src/vnet/vxlan/vxlan.c index 9a68ae80154..356cd3d9aa4 100644 --- a/src/vnet/vxlan/vxlan.c +++ b/src/vnet/vxlan/vxlan.c @@ -374,7 +374,7 @@ int vnet_vxlan_add_del_tunnel return VNET_API_ERROR_INVALID_DECAP_NEXT; pool_get_aligned (vxm->tunnels, t, CLIB_CACHE_LINE_BYTES); - *t = (vxlan_tunnel_t){ 0 }; + memset(t, 0, sizeof(*t)); dev_instance = t - vxm->tunnels; /* copy from arg structure */ diff --git a/src/vnet/vxlan/vxlan.h b/src/vnet/vxlan/vxlan.h index e8bb272d759..74ab8e04a3e 100644 --- a/src/vnet/vxlan/vxlan.h +++ b/src/vnet/vxlan/vxlan.h @@ -66,6 +66,9 @@ typedef CLIB_PACKED(struct { }) vxlan6_tunnel_key_t; typedef struct { + /* Required for pool_get_aligned */ + CLIB_CACHE_LINE_ALIGN_MARK(cacheline0); + /* Rewrite string. $$$$ embed vnet_rewrite header */ u8 * rewrite; diff --git a/src/vpp/oam/oam.c b/src/vpp/oam/oam.c index ef061207544..1579f562a67 100644 --- a/src/vpp/oam/oam.c +++ b/src/vpp/oam/oam.c @@ -26,7 +26,7 @@ init_oam_packet_template (oam_main_t * om, oam_target_t * t) ip_csum_t sum; u16 csum; - vec_validate_aligned (t->template, 0, CLIB_CACHE_LINE_BYTES); + vec_validate (t->template, 0); h = t->template; memset (h, 0, sizeof (*h)); diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h index 14c6a75c13b..8b39a914eb5 100644 --- a/src/vppinfra/pool.h +++ b/src/vppinfra/pool.h @@ -190,6 +190,8 @@ do { \ pool_header_t * _pool_var (p) = pool_header (P); \ uword _pool_var (l); \ \ + STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) || ((sizeof(P[0]) % A) == 0), \ + "Pool aligned alloc of incorrectly sized object"); \ _pool_var (l) = 0; \ if (P) \ _pool_var (l) = vec_len (_pool_var (p)->free_indices); \ diff --git a/src/vppinfra/vec.h b/src/vppinfra/vec.h index e2cb24c5ce7..a029630559c 100644 --- a/src/vppinfra/vec.h +++ b/src/vppinfra/vec.h @@ -411,6 +411,8 @@ do { \ #define vec_validate_ha(V,I,H,A) \ do { \ + STATIC_ASSERT(A==0 || ((A % sizeof(V[0]))==0) || ((sizeof(V[0]) % A) == 0),\ + "vector validate aligned on incorrectly sized object"); \ word _v(i) = (I); \ word _v(l) = vec_len (V); \ if (_v(i) >= _v(l)) \ -- cgit 1.2.3-korg