32/64 shmem bihash interoperability

Move the binary api segment above 4gb

Change-Id: I40e8aa7a97722a32397f5a538b5ff8344c50d408
Signed-off-by: Dave Barach <dave@barachs.net>

1 files changed, 83 insertions, 13 deletions

diff --git a/src/vppinfra/bihash_template.h b/src/vppinfra/bihash_template.h
index 4ff7e1bfb6c..ef7e83013ae 100644
--- a/src/vppinfra/bihash_template.h
+++ b/src/vppinfra/bihash_template.h
@@ -33,6 +33,17 @@
 #error BIHASH_TYPE not defined
 #endif
 
+#ifdef BIHASH_32_64_SVM
+#undef HAVE_MEMFD_CREATE
+#include <vppinfra/linux/syscall.h>
+#include <fcntl.h>
+#define F_LINUX_SPECIFIC_BASE 1024
+#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9)
+#define F_SEAL_SHRINK (2)
+/* Max page size 2**16 due to refcount width  */
+#define BIHASH_FREELIST_LENGTH 17
+#endif
+
 #define _bv(a,b) a##b
 #define __bv(a,b) _bv(a,b)
 #define BV(a) __bv(a,BIHASH_TYPE)
@@ -41,12 +52,22 @@
 #define __bvt(a,b) _bvt(a,b)
 #define BVT(a) __bvt(a,BIHASH_TYPE)
 
+#if _LP64 == 0
+#define OVERFLOW_ASSERT(x) ASSERT(((x) & 0xFFFFFFFF00000000ULL) == 0)
+#define u64_to_pointer(x) (void *)(u32)((x))
+#define pointer_to_u64(x) (u64)(u32)((x))
+#else
+#define OVERFLOW_ASSERT(x)
+#define u64_to_pointer(x) (void *)((x))
+#define pointer_to_u64(x) (u64)((x))
+#endif
+
 typedef struct BV (clib_bihash_value)
 {
   union
   {
     BVT (clib_bihash_kv) kvp[BIHASH_KVP_PER_PAGE];
-    struct BV (clib_bihash_value) * next_free;
+    u64 next_free_as_u64;
   };
 } BVT (clib_bihash_value);
 
@@ -62,7 +83,7 @@ typedef struct
       u64 lock:1;
       u64 linear_search:1;
       u64 log2_pages:8;
-      i64 refcnt:16;
+      u64 refcnt:16;
     };
     u64 as_u64;
   };
@@ -70,9 +91,31 @@ typedef struct
 
 STATIC_ASSERT_SIZEOF (BVT (clib_bihash_bucket), sizeof (u64));
 
+/* *INDENT-OFF* */
+typedef CLIB_PACKED (struct {
+  /*
+   * Backing store allocation. Since bihash manages its own
+   * freelists, we simple dole out memory at alloc_arena_next.
+   */
+  u64 alloc_arena_next;	/* Next VA to allocate, definitely NOT a constant */
+  u64 alloc_arena_size;	/* Size of the arena */
+  u64 alloc_arena;	/* Base VA of the arena */
+  /* Two SVM pointers stored as 8-byte integers */
+  u64 alloc_lock_as_u64;
+  u64 buckets_as_u64;
+  /* freelist list-head arrays/vectors */
+  u64 freelists_as_u64;
+  u32 nbuckets;	/* Number of buckets */
+  /* Set when header valid */
+  volatile u32 ready;
+  u64 pad;
+}) BVT (clib_bihash_shared_header);
+/* *INDENT-ON* */
+
+STATIC_ASSERT_SIZEOF (BVT (clib_bihash_shared_header), 8 * sizeof (u64));
+
 typedef struct
 {
-  BVT (clib_bihash_value) * values;
   BVT (clib_bihash_bucket) * buckets;
   volatile u32 *alloc_lock;
 
@@ -84,15 +127,14 @@ typedef struct
   u32 log2_nbuckets;
   u8 *name;
 
-    BVT (clib_bihash_value) ** freelists;
+  u64 *freelists;
 
-  /*
-   * Backing store allocation. Since bihash manages its own
-   * freelists, we simple dole out memory at alloc_arena_next.
-   */
-  uword alloc_arena;
-  uword alloc_arena_next;
-  uword alloc_arena_size;
+#if BIHASH_32_64_SVM
+    BVT (clib_bihash_shared_header) * sh;
+  int memfd;
+#else
+    BVT (clib_bihash_shared_header) sh;
+#endif
 
   /**
     * A custom format function to print the Key and Value of bihash_key instead of default hexdump
@@ -101,6 +143,26 @@ typedef struct
 
 } BVT (clib_bihash);
 
+#if BIHASH_32_64_SVM
+#undef alloc_arena_next
+#undef alloc_arena_size
+#undef alloc_arena
+#undef CLIB_BIHASH_READY_MAGIC
+#define alloc_arena_next(h) (((h)->sh)->alloc_arena_next)
+#define alloc_arena_size(h) (((h)->sh)->alloc_arena_size)
+#define alloc_arena(h) (((h)->sh)->alloc_arena)
+#define CLIB_BIHASH_READY_MAGIC 0xFEEDFACE
+#else
+#undef alloc_arena_next
+#undef alloc_arena_size
+#undef alloc_arena
+#undef CLIB_BIHASH_READY_MAGIC
+#define alloc_arena_next(h) ((h)->sh.alloc_arena_next)
+#define alloc_arena_size(h) ((h)->sh.alloc_arena_size)
+#define alloc_arena(h) ((h)->sh.alloc_arena)
+#define CLIB_BIHASH_READY_MAGIC 0
+#endif
+
 static inline void BV (clib_bihash_alloc_lock) (BVT (clib_bihash) * h)
 {
   while (__atomic_test_and_set (h->alloc_lock, __ATOMIC_ACQUIRE))
@@ -139,7 +201,7 @@ static inline void BV (clib_bihash_unlock_bucket)
 static inline void *BV (clib_bihash_get_value) (BVT (clib_bihash) * h,
 						uword offset)
 {
-  u8 *hp = (u8 *) h->alloc_arena;
+  u8 *hp = (u8 *) (uword) alloc_arena (h);
   u8 *vp = hp + offset;
 
   return (void *) vp;
@@ -157,7 +219,7 @@ static inline uword BV (clib_bihash_get_offset) (BVT (clib_bihash) * h,
 {
   u8 *hp, *vp;
 
-  hp = (u8 *) h->alloc_arena;
+  hp = (u8 *) (uword) alloc_arena (h);
   vp = (u8 *) v;
 
   return vp - hp;
@@ -166,6 +228,14 @@ static inline uword BV (clib_bihash_get_offset) (BVT (clib_bihash) * h,
 void BV (clib_bihash_init)
   (BVT (clib_bihash) * h, char *name, u32 nbuckets, uword memory_size);
 
+#if BIHASH_32_64_SVM
+void BV (clib_bihash_master_init_svm)
+  (BVT (clib_bihash) * h, char *name, u32 nbuckets,
+   u64 base_address, u64 memory_size);
+void BV (clib_bihash_slave_init_svm)
+  (BVT (clib_bihash) * h, char *name, int fd);
+#endif
+
 void BV (clib_bihash_set_kvp_format_fn) (BVT (clib_bihash) * h,
 					 format_function_t * fmt_fn);