diff options
Diffstat (limited to 'src/vppinfra/bihash_template.h')
| -rw-r--r-- | src/vppinfra/bihash_template.h | 96 |
1 files changed, 83 insertions, 13 deletions
diff --git a/src/vppinfra/bihash_template.h b/src/vppinfra/bihash_template.h index 4ff7e1bfb6c..ef7e83013ae 100644 --- a/src/vppinfra/bihash_template.h +++ b/src/vppinfra/bihash_template.h @@ -33,6 +33,17 @@ #error BIHASH_TYPE not defined #endif +#ifdef BIHASH_32_64_SVM +#undef HAVE_MEMFD_CREATE +#include <vppinfra/linux/syscall.h> +#include <fcntl.h> +#define F_LINUX_SPECIFIC_BASE 1024 +#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9) +#define F_SEAL_SHRINK (2) +/* Max page size 2**16 due to refcount width */ +#define BIHASH_FREELIST_LENGTH 17 +#endif + #define _bv(a,b) a##b #define __bv(a,b) _bv(a,b) #define BV(a) __bv(a,BIHASH_TYPE) @@ -41,12 +52,22 @@ #define __bvt(a,b) _bvt(a,b) #define BVT(a) __bvt(a,BIHASH_TYPE) +#if _LP64 == 0 +#define OVERFLOW_ASSERT(x) ASSERT(((x) & 0xFFFFFFFF00000000ULL) == 0) +#define u64_to_pointer(x) (void *)(u32)((x)) +#define pointer_to_u64(x) (u64)(u32)((x)) +#else +#define OVERFLOW_ASSERT(x) +#define u64_to_pointer(x) (void *)((x)) +#define pointer_to_u64(x) (u64)((x)) +#endif + typedef struct BV (clib_bihash_value) { union { BVT (clib_bihash_kv) kvp[BIHASH_KVP_PER_PAGE]; - struct BV (clib_bihash_value) * next_free; + u64 next_free_as_u64; }; } BVT (clib_bihash_value); @@ -62,7 +83,7 @@ typedef struct u64 lock:1; u64 linear_search:1; u64 log2_pages:8; - i64 refcnt:16; + u64 refcnt:16; }; u64 as_u64; }; @@ -70,9 +91,31 @@ typedef struct STATIC_ASSERT_SIZEOF (BVT (clib_bihash_bucket), sizeof (u64)); +/* *INDENT-OFF* */ +typedef CLIB_PACKED (struct { + /* + * Backing store allocation. Since bihash manages its own + * freelists, we simple dole out memory at alloc_arena_next. + */ + u64 alloc_arena_next; /* Next VA to allocate, definitely NOT a constant */ + u64 alloc_arena_size; /* Size of the arena */ + u64 alloc_arena; /* Base VA of the arena */ + /* Two SVM pointers stored as 8-byte integers */ + u64 alloc_lock_as_u64; + u64 buckets_as_u64; + /* freelist list-head arrays/vectors */ + u64 freelists_as_u64; + u32 nbuckets; /* Number of buckets */ + /* Set when header valid */ + volatile u32 ready; + u64 pad; +}) BVT (clib_bihash_shared_header); +/* *INDENT-ON* */ + +STATIC_ASSERT_SIZEOF (BVT (clib_bihash_shared_header), 8 * sizeof (u64)); + typedef struct { - BVT (clib_bihash_value) * values; BVT (clib_bihash_bucket) * buckets; volatile u32 *alloc_lock; @@ -84,15 +127,14 @@ typedef struct u32 log2_nbuckets; u8 *name; - BVT (clib_bihash_value) ** freelists; + u64 *freelists; - /* - * Backing store allocation. Since bihash manages its own - * freelists, we simple dole out memory at alloc_arena_next. - */ - uword alloc_arena; - uword alloc_arena_next; - uword alloc_arena_size; +#if BIHASH_32_64_SVM + BVT (clib_bihash_shared_header) * sh; + int memfd; +#else + BVT (clib_bihash_shared_header) sh; +#endif /** * A custom format function to print the Key and Value of bihash_key instead of default hexdump @@ -101,6 +143,26 @@ typedef struct } BVT (clib_bihash); +#if BIHASH_32_64_SVM +#undef alloc_arena_next +#undef alloc_arena_size +#undef alloc_arena +#undef CLIB_BIHASH_READY_MAGIC +#define alloc_arena_next(h) (((h)->sh)->alloc_arena_next) +#define alloc_arena_size(h) (((h)->sh)->alloc_arena_size) +#define alloc_arena(h) (((h)->sh)->alloc_arena) +#define CLIB_BIHASH_READY_MAGIC 0xFEEDFACE +#else +#undef alloc_arena_next +#undef alloc_arena_size +#undef alloc_arena +#undef CLIB_BIHASH_READY_MAGIC +#define alloc_arena_next(h) ((h)->sh.alloc_arena_next) +#define alloc_arena_size(h) ((h)->sh.alloc_arena_size) +#define alloc_arena(h) ((h)->sh.alloc_arena) +#define CLIB_BIHASH_READY_MAGIC 0 +#endif + static inline void BV (clib_bihash_alloc_lock) (BVT (clib_bihash) * h) { while (__atomic_test_and_set (h->alloc_lock, __ATOMIC_ACQUIRE)) @@ -139,7 +201,7 @@ static inline void BV (clib_bihash_unlock_bucket) static inline void *BV (clib_bihash_get_value) (BVT (clib_bihash) * h, uword offset) { - u8 *hp = (u8 *) h->alloc_arena; + u8 *hp = (u8 *) (uword) alloc_arena (h); u8 *vp = hp + offset; return (void *) vp; @@ -157,7 +219,7 @@ static inline uword BV (clib_bihash_get_offset) (BVT (clib_bihash) * h, { u8 *hp, *vp; - hp = (u8 *) h->alloc_arena; + hp = (u8 *) (uword) alloc_arena (h); vp = (u8 *) v; return vp - hp; @@ -166,6 +228,14 @@ static inline uword BV (clib_bihash_get_offset) (BVT (clib_bihash) * h, void BV (clib_bihash_init) (BVT (clib_bihash) * h, char *name, u32 nbuckets, uword memory_size); +#if BIHASH_32_64_SVM +void BV (clib_bihash_master_init_svm) + (BVT (clib_bihash) * h, char *name, u32 nbuckets, + u64 base_address, u64 memory_size); +void BV (clib_bihash_slave_init_svm) + (BVT (clib_bihash) * h, char *name, int fd); +#endif + void BV (clib_bihash_set_kvp_format_fn) (BVT (clib_bihash) * h, format_function_t * fmt_fn); |