VPP-236 Support 64-bit vector lengths, shared memory segments >4 GB

Change-Id: I02aee33e96e7ae32094b9f82f6a667d30bb52f59
Signed-off-by: Dave Barach <dave@barachs.net>

13 files changed, 264 insertions, 76 deletions

diff --git a/svm/svm.c b/svm/svm.c
index 1ee5e95df29..851baea55ed 100644
--- a/svm/svm.c
+++ b/svm/svm.c
@@ -216,10 +216,10 @@ format_svm_region (u8 * s, va_list * args)
  * rnd_pagesize
  * Round to a pagesize multiple, presumably 4k works
  */
-static unsigned int
-rnd_pagesize (unsigned int size)
+static u64
+rnd_pagesize (u64 size)
 {
-  unsigned int rv;
+  u64 rv;
 
   rv = (size + (MMAP_PAGESIZE - 1)) & ~(MMAP_PAGESIZE - 1);
   return (rv);
@@ -235,7 +235,9 @@ svm_data_region_create (svm_map_region_args_t * a, svm_region_t * rp)
   u8 junk = 0;
   uword map_size;
 
-  map_size = rp->virtual_size - (MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE);
+  map_size = rp->virtual_size - (MMAP_PAGESIZE + 
+                                 (a->pvt_heap_size ? a->pvt_heap_size : 
+                                  SVM_PVT_MHEAP_SIZE));
 
   if (a->flags & SVM_FLAGS_FILE)
     {
@@ -316,7 +318,9 @@ svm_data_region_map (svm_map_region_args_t * a, svm_region_t * rp)
   uword map_size;
   struct stat statb;
 
-  map_size = rp->virtual_size - (MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE);
+  map_size = rp->virtual_size - 
+    (MMAP_PAGESIZE 
+     + (a->pvt_heap_size ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE));
 
   if (a->flags & SVM_FLAGS_FILE)
     {
@@ -364,7 +368,9 @@ svm_data_region_map (svm_map_region_args_t * a, svm_region_t * rp)
 	}
 
       ASSERT (map_size <= rp->virtual_size
-	      - (MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE));
+	      - (MMAP_PAGESIZE 
+                 + 
+                 (a->pvt_heap_size ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE)));
 
       if (mmap (rp->data_base, map_size, PROT_READ | PROT_WRITE,
 		MAP_SHARED | MAP_FIXED, fd, 0) == MAP_FAILED)
@@ -528,7 +534,9 @@ svm_map_region (svm_map_region_args_t * a)
 
       rp->region_heap =
 	mheap_alloc_with_flags ((void *) (a->baseva + MMAP_PAGESIZE),
-				SVM_PVT_MHEAP_SIZE, MHEAP_FLAG_DISABLE_VM);
+                                (a->pvt_heap_size != 0) ? 
+                                a->pvt_heap_size : SVM_PVT_MHEAP_SIZE, 
+                                MHEAP_FLAG_DISABLE_VM);
       oldheap = svm_push_pvt_heap (rp);
 
       rp->region_name = (char *) format (0, "%s%c", a->name, 0);
@@ -542,7 +550,7 @@ svm_map_region (svm_map_region_args_t * a)
       vec_validate (rp->bitmap, words - 1);
 
       overhead_space = MMAP_PAGESIZE /* header */  +
-	SVM_PVT_MHEAP_SIZE;
+        ((a->pvt_heap_size != 0)? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE);
 
       bit = 0;
       data_base = (uword) rp->virtual_base;
@@ -724,10 +732,9 @@ svm_mutex_cleanup (void)
 }
 
 static void
-svm_region_init_internal (char *root_path, int uid, int gid)
+svm_region_init_internal (svm_map_region_args_t * a)
 {
   svm_region_t *rp;
-  svm_map_region_args_t _a, *a = &_a;
   u64 ticks = clib_cpu_time_now ();
   uword randomize_baseva;
 
@@ -745,14 +752,7 @@ svm_region_init_internal (char *root_path, int uid, int gid)
   else
     randomize_baseva = (ticks & 3) * MMAP_PAGESIZE;
 
-  memset (a, 0, sizeof (*a));
-  a->root_path = root_path;
-  a->name = SVM_GLOBAL_REGION_NAME;
-  a->baseva = SVM_GLOBAL_REGION_BASEVA + randomize_baseva;
-  a->size = SVM_GLOBAL_REGION_SIZE;
-  a->flags = SVM_FLAGS_NODATA;
-  a->uid = uid;
-  a->gid = gid;
+  a->baseva += randomize_baseva;
 
   rp = svm_map_region (a);
   ASSERT (rp);
@@ -770,7 +770,7 @@ svm_region_init_internal (char *root_path, int uid, int gid)
       oldheap = svm_push_pvt_heap (rp);
       vec_validate (mp, 0);
       mp->name_hash = hash_create_string (0, sizeof (uword));
-      mp->root_path = root_path ? format (0, "%s%c", root_path, 0) : 0;
+      mp->root_path = a->root_path ? format (0, "%s%c", a->root_path, 0) : 0;
       rp->data_base = mp;
       svm_pop_heap (oldheap);
     }
@@ -781,19 +781,58 @@ svm_region_init_internal (char *root_path, int uid, int gid)
 void
 svm_region_init (void)
 {
-  svm_region_init_internal (0, 0 /* uid */ , 0 /* gid */ );
+  svm_map_region_args_t _a, *a = &_a;
+  
+  memset (a, 0, sizeof (*a));
+  a->root_path = 0;
+  a->name = SVM_GLOBAL_REGION_NAME;
+  a->baseva = SVM_GLOBAL_REGION_BASEVA;
+  a->size = SVM_GLOBAL_REGION_SIZE;
+  a->flags = SVM_FLAGS_NODATA;
+  a->uid = 0;
+  a->gid = 0;
+
+  svm_region_init_internal (a);
 }
 
 void
 svm_region_init_chroot (char *root_path)
 {
-  svm_region_init_internal (root_path, 0 /* uid */ , 0 /* gid */ );
+  svm_map_region_args_t _a, *a = &_a;
+  
+  memset (a, 0, sizeof (*a));
+  a->root_path = root_path;
+  a->name = SVM_GLOBAL_REGION_NAME;
+  a->baseva = SVM_GLOBAL_REGION_BASEVA;
+  a->size = SVM_GLOBAL_REGION_SIZE;
+  a->flags = SVM_FLAGS_NODATA;
+  a->uid = 0;
+  a->gid = 0;
+
+  svm_region_init_internal (a);
 }
 
 void
 svm_region_init_chroot_uid_gid (char *root_path, int uid, int gid)
 {
-  svm_region_init_internal (root_path, uid, gid);
+  svm_map_region_args_t _a, *a = &_a;
+  
+  memset (a, 0, sizeof (*a));
+  a->root_path = root_path;
+  a->name = SVM_GLOBAL_REGION_NAME;
+  a->baseva = SVM_GLOBAL_REGION_BASEVA;
+  a->size = SVM_GLOBAL_REGION_SIZE;
+  a->flags = SVM_FLAGS_NODATA;
+  a->uid = uid;
+  a->gid = gid;
+
+  svm_region_init_internal (a);
+}
+
+void
+svm_region_init_args (svm_map_region_args_t * a)
+{
+  svm_region_init_internal (a);
 }
 
 void *
@@ -810,7 +849,8 @@ svm_region_find_or_create (svm_map_region_args_t * a)
 
   ASSERT (root_rp);
 
-  a->size += MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE;
+  a->size += MMAP_PAGESIZE + 
+    ((a->pvt_heap_size != 0) ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE);
   a->size = rnd_pagesize (a->size);
 
   region_lock (root_rp, 4);
@@ -869,8 +909,8 @@ svm_region_find_or_create (svm_map_region_args_t * a)
   /* Completely out of VM? */
   if (index >= root_rp->bitmap_size)
     {
-      clib_warning ("region %s: not enough VM to allocate 0x%x",
-		    root_rp->region_name, a->size);
+      clib_warning ("region %s: not enough VM to allocate 0x%llx (%lld)",
+		    root_rp->region_name, a->size, a->size);
       svm_pop_heap (oldheap);
       region_unlock (root_rp);
       return 0;
@@ -892,7 +932,7 @@ svm_region_find_or_create (svm_map_region_args_t * a)
   a->baseva = root_rp->virtual_base + index * MMAP_PAGESIZE;
 
   rp = svm_map_region (a);
-
+  
   pool_get (mp->subregions, subp);
   name = format (0, "%s%c", a->name, 0);
   subp->subregion_name = name;
diff --git a/svm/svm.h b/svm/svm.h
index 1682b7ad2fa..c42d2b578df 100644
--- a/svm/svm.h
+++ b/svm/svm.h
@@ -71,8 +71,9 @@ typedef struct svm_map_region_args_
 {
   char *root_path;		/* NULL means use the truly global arena */
   char *name;
-  uword baseva;
-  uword size;
+  u64 baseva;
+  u64 size;
+  u64 pvt_heap_size;
   uword flags;
   char *backing_file;
   uword backing_mmap_size;
@@ -116,6 +117,7 @@ void *svm_region_find_or_create (svm_map_region_args_t * a);
 void svm_region_init (void);
 void svm_region_init_chroot (char *root_path);
 void svm_region_init_chroot_uid_gid (char *root_path, int uid, int gid);
+void svm_region_init_args (svm_map_region_args_t *a);
 void svm_region_exit (void);
 void svm_region_unmap (void *rp_arg);
 void svm_client_scan (char *root_path);
diff --git a/svm/svmtool.c b/svm/svmtool.c
index 545f071e47e..2e55a0873e3 100644
--- a/svm/svmtool.c
+++ b/svm/svmtool.c
@@ -206,10 +206,10 @@ svm_map_region_nolock (svm_map_region_args_t * a)
  * rnd_pagesize
  * Round to a pagesize multiple, presumably 4k works
  */
-static unsigned int
-rnd_pagesize (unsigned int size)
+static u64
+rnd_pagesize (u64 size)
 {
-  unsigned int rv;
+  u64 rv;
 
   rv = (size + (MMAP_PAGESIZE - 1)) & ~(MMAP_PAGESIZE - 1);
   return (rv);
@@ -247,7 +247,8 @@ svm_existing_region_map_nolock (void *root_arg, svm_map_region_args_t * a)
   void *oldheap;
   uword *p;
 
-  a->size += MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE;
+  a->size += MMAP_PAGESIZE + 
+    (a->pvt_heap_size ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE);
   a->size = rnd_pagesize (a->size);
 
   region_lock (root_rp, 4);
diff --git a/vlib-api/vlibapi/api.h b/vlib-api/vlibapi/api.h
index 9d885c17930..a932bf42177 100644
--- a/vlib-api/vlibapi/api.h
+++ b/vlib-api/vlibapi/api.h
@@ -141,9 +141,25 @@ typedef struct
   /* vector of message ranges */
   vl_api_msg_range_t *msg_ranges;
 
+  /* uid for the api shared memory region */
+  int api_uid;
   /* gid for the api shared memory region */
   int api_gid;
-  int api_uid;
+
+  /* base virtual address for global VM region */
+  u64 global_baseva;
+
+  /* size of the global VM region */
+  u64 global_size;
+
+  /* size of the API region */
+  u64 api_size;
+
+  /* size of the global VM private mheap */
+  u64 global_pvt_heap_size;
+
+  /* size of the api private mheap */
+  u64 api_pvt_heap_size;
 
   /* Client-only data structures */
   unix_shared_memory_queue_t *vl_input_queue;
diff --git a/vlib-api/vlibmemory/api.h b/vlib-api/vlibmemory/api.h
index e3a95843df6..bc4cfbf49b3 100644
--- a/vlib-api/vlibmemory/api.h
+++ b/vlib-api/vlibmemory/api.h
@@ -142,6 +142,11 @@ void vl_set_memory_region_name (char *name);
 void vl_set_memory_root_path (char *root_path);
 void vl_set_memory_uid (int uid);
 void vl_set_memory_gid (int gid);
+void vl_set_global_memory_baseva (u64 baseva);
+void vl_set_global_memory_size (u64 size);
+void vl_set_api_memory_size (u64 size);
+void vl_set_global_pvt_heap_size (u64 size);
+void vl_set_api_pvt_heap_size (u64 size);
 void vl_enable_disable_memory_api (vlib_main_t * vm, int yesno);
 void vl_client_disconnect_from_vlib (void);
 int vl_client_connect_to_vlib (char *svm_name, char *client_name,
diff --git a/vlib-api/vlibmemory/memory_shared.c b/vlib-api/vlibmemory/memory_shared.c
index 53976fbedf9..a2f22771596 100644
--- a/vlib-api/vlibmemory/memory_shared.c
+++ b/vlib-api/vlibmemory/memory_shared.c
@@ -228,6 +228,46 @@ vl_set_memory_gid (int gid)
   am->api_gid = gid;
 }
 
+void
+vl_set_global_memory_baseva (u64 baseva)
+{
+  api_main_t *am = &api_main;
+
+  am->global_baseva = baseva;
+}
+
+void
+vl_set_global_memory_size (u64 size)
+{
+  api_main_t *am = &api_main;
+
+  am->global_size = size;
+}
+
+void
+vl_set_api_memory_size (u64 size)
+{
+  api_main_t *am = &api_main;
+
+  am->api_size = size;
+}
+
+void
+vl_set_global_pvt_heap_size (u64 size)
+{
+  api_main_t *am = &api_main;
+
+  am->global_pvt_heap_size = size;
+}
+
+void
+vl_set_api_pvt_heap_size (u64 size)
+{
+  api_main_t *am = &api_main;
+
+  am->api_pvt_heap_size = size;
+}
+
 int
 vl_map_shmem (char *region_name, int is_vlib)
 {
@@ -245,10 +285,11 @@ vl_map_shmem (char *region_name, int is_vlib)
   memset (a, 0, sizeof (*a));
 
   a->name = region_name;
-  a->size = 16 << 20;
+  a->size = am->api_size ? am->api_size: (16 << 20);
   a->flags = SVM_FLAGS_MHEAP;
   a->uid = am->api_uid;
   a->gid = am->api_gid;
+  a->pvt_heap_size = am->api_pvt_heap_size;
 
   vlib_rp = svm_region_find_or_create (a);
 
diff --git a/vlib-api/vlibmemory/memory_vlib.c b/vlib-api/vlibmemory/memory_vlib.c
index 68752745d66..2212b46bce4 100644
--- a/vlib-api/vlibmemory/memory_vlib.c
+++ b/vlib-api/vlibmemory/memory_vlib.c
@@ -1176,8 +1176,21 @@ clib_error_t *
 vlibmemory_init (vlib_main_t * vm)
 {
   api_main_t *am = &api_main;
-  /* Normally NULL / 0, set by cmd line "api-segment" */
-  svm_region_init_chroot_uid_gid (am->root_path, am->api_uid, am->api_gid);
+  svm_map_region_args_t _a, *a = &_a;
+  
+  memset (a, 0, sizeof (*a));
+  a->root_path = am->root_path;
+  a->name = SVM_GLOBAL_REGION_NAME;
+  a->baseva = (am->global_baseva != 0) ? 
+    am->global_baseva : SVM_GLOBAL_REGION_BASEVA;
+  a->size = (am->global_size != 0) ? am->global_size : SVM_GLOBAL_REGION_SIZE;
+  a->flags = SVM_FLAGS_NODATA;
+  a->uid = am->api_uid;
+  a->gid = am->api_gid;
+  a->pvt_heap_size = (am->global_pvt_heap_size != 0) ? am->global_pvt_heap_size 
+    : SVM_PVT_MHEAP_SIZE;
+
+  svm_region_init_args (a);
   return 0;
 }
 
diff --git a/vpp/vpp-api/api.c b/vpp/vpp-api/api.c
index 8a341597a0a..ee3ced4379d 100644
--- a/vpp/vpp-api/api.c
+++ b/vpp/vpp-api/api.c
@@ -8111,7 +8111,8 @@ VLIB_INIT_FUNCTION (vpe_api_init);
 static clib_error_t *
 api_segment_config (vlib_main_t * vm, unformat_input_t * input)
 {
-  u8 *chroot_path;
+  u8 * chroot_path;
+  u64 baseva, size, pvt_heap_size;
   int uid, gid, rv;
   const int max_buf_size = 4096;
   char *s, *buf;
@@ -8129,7 +8130,31 @@ api_segment_config (vlib_main_t * vm, unformat_input_t * input)
       else if (unformat (input, "uid %d", &uid))
 	vl_set_memory_uid (uid);
       else if (unformat (input, "gid %d", &gid))
-	vl_set_memory_gid (gid);
+        vl_set_memory_gid (gid);
+      else if (unformat (input, "baseva %llx", &baseva))
+        vl_set_global_memory_baseva (baseva);
+      else if (unformat (input, "global-size %lldM", &size))
+        vl_set_global_memory_size (size * (1ULL<<20));
+      else if (unformat (input, "global-size %lldG", &size))
+        vl_set_global_memory_size (size * (1ULL<<30));
+      else if (unformat (input, "global-size %lld", &size))
+        vl_set_global_memory_size (size);
+      else if (unformat (input, "global-pvt-heap-size %lldM", &pvt_heap_size))
+        vl_set_global_pvt_heap_size (pvt_heap_size * (1ULL<<20));
+      else if (unformat (input, "global-pvt-heap-size size %lld", 
+                         &pvt_heap_size))
+        vl_set_global_pvt_heap_size (pvt_heap_size);
+      else if (unformat (input, "api-pvt-heap-size %lldM", &pvt_heap_size))
+        vl_set_api_pvt_heap_size (pvt_heap_size * (1ULL<<20));
+      else if (unformat (input, "api-pvt-heap-size size %lld", 
+                         &pvt_heap_size))
+        vl_set_api_pvt_heap_size (pvt_heap_size);
+      else if (unformat (input, "api-size %lldM", &size))
+        vl_set_api_memory_size (size * (1ULL<<20));
+      else if (unformat (input, "api-size %lldG", &size))
+        vl_set_api_memory_size (size * (1ULL<<30));
+      else if (unformat (input, "api-size %lld", &size))
+        vl_set_api_memory_size (size);
       else if (unformat (input, "uid %s", &s))
 	{
 	  /* lookup the username */
diff --git a/vpp/vpp-api/gmon.c b/vpp/vpp-api/gmon.c
index ea4a25efc5b..330dc9fe5d3 100644
--- a/vpp/vpp-api/gmon.c
+++ b/vpp/vpp-api/gmon.c
@@ -180,8 +180,8 @@ gmon_init (vlib_main_t * vm)
   if ((error = vlib_call_init_function (vm, vpe_api_init)))
     return (error);
 
-  /* Make sure that /global-vm is owned as directed */
-  svm_region_init_chroot_uid_gid (am->root_path, am->api_uid, am->api_gid);
+  if ((error = vlib_call_init_function(vm, vlibmemory_init)))
+    return(error);
 
   gm->vlib_main = vm;
 
diff --git a/vppinfra/Makefile.am b/vppinfra/Makefile.am
index f9855e34f79..48a347e009f 100644
--- a/vppinfra/Makefile.am
+++ b/vppinfra/Makefile.am
@@ -12,7 +12,7 @@
 # limitations under the License.
 AUTOMAKE_OPTIONS = foreign subdir-objects
 
-AM_CPPFLAGS = -Wall @VEC64@ # -Werror
+AM_CPPFLAGS = -Wall -Werror
 
 if WITH_UNIX
   lib_LTLIBRARIES = libvppinfra.la
@@ -20,7 +20,7 @@ endif
 
 lib_LIBRARIES =
 
-TESTS = 
+TESTS =
 
 if ENABLE_TESTS
 TESTS  +=  test_bihash_template \
@@ -267,5 +267,5 @@ libvppinfra_la_SOURCES =			\
 bin_PROGRAMS = elftool
 
 elftool_SOURCES = tools/elftool.c
-elftool_CPPFLAGS = -Wall
+elftool_CPPFLAGS = $(AM_CPPFLAGS)
 elftool_LDADD = libvppinfra.la -lpthread -lrt -lm
diff --git a/vppinfra/configure.ac b/vppinfra/configure.ac
index 88720f717a1..88939383057 100644
--- a/vppinfra/configure.ac
+++ b/vppinfra/configure.ac
@@ -48,14 +48,5 @@ AC_ARG_WITH(unix,
 
 AM_CONDITIONAL(WITH_UNIX, test "$with_unix" = "yes")
 
-
-## Enable 64-bit vector lengths
-AC_ARG_WITH(vec64,
-	    AC_HELP_STRING([--with-vec64],[Enable 64-bit vector lengths]),
-            [with_vec64=1],
-            [with_vec64=0])
-
-AC_SUBST(VEC64,[-DCLIB_VEC64=${with_vec64}])
-
 AC_CONFIG_FILES([Makefile])
 AC_OUTPUT
diff --git a/vppinfra/vppinfra/mheap.c b/vppinfra/vppinfra/mheap.c
index 6163d0b1341..3bb2d037d0f 100644
--- a/vppinfra/vppinfra/mheap.c
+++ b/vppinfra/vppinfra/mheap.c
@@ -133,6 +133,7 @@ mheap_elt_set_size (void * v,
   e = mheap_elt_at_uoffset (v, uoffset);
 
   ASSERT (n_user_data_bytes % MHEAP_USER_DATA_WORD_BYTES == 0);
+
   e->n_user_data = n_user_data_bytes / MHEAP_USER_DATA_WORD_BYTES;
   e->is_free = is_free;
   ASSERT (e->prev_n_user_data * sizeof (e->user_data[0]) >= MHEAP_MIN_USER_DATA_BYTES);
@@ -152,7 +153,7 @@ always_inline void set_first_free_elt_offset (mheap_t * h, uword bin, uword uoff
   i1 = (uword) 1 << (uword) (bin % BITS (h->non_empty_free_elt_heads[0]));
 
   ASSERT (i0 < ARRAY_LEN (h->non_empty_free_elt_heads));
-  if (h->first_free_elt_uoffset_by_bin[bin] == ~0)
+  if (h->first_free_elt_uoffset_by_bin[bin] == MHEAP_GROUNDED)
     h->non_empty_free_elt_heads[i0] &= ~i1;
   else
     h->non_empty_free_elt_heads[i0] |= i1;
@@ -169,11 +170,11 @@ set_free_elt (void * v, uword uoffset, uword n_user_data_bytes)
   ASSERT (n->prev_is_free);
   ASSERT (e->is_free);
 
-  e->free_elt.prev_uoffset = ~0;
+  e->free_elt.prev_uoffset = MHEAP_GROUNDED;
   e->free_elt.next_uoffset = h->first_free_elt_uoffset_by_bin[bin];
 
   /* Fill in next free elt's previous pointer. */
-  if (e->free_elt.next_uoffset != ~0)
+  if (e->free_elt.next_uoffset != MHEAP_GROUNDED)
     {
       mheap_elt_t * nf = mheap_elt_at_uoffset (v, e->free_elt.next_uoffset);
       ASSERT (nf->is_free);
@@ -195,12 +196,17 @@ remove_free_elt (void * v, mheap_elt_t * e, uword bin)
 {
   mheap_t * h = mheap_header (v);
   mheap_elt_t * p, * n;
+#if CLIB_VEC64 > 0
+  u64 no, po;
+#else
   u32 no, po;
+#endif
 
   no = e->free_elt.next_uoffset;
-  n = no != ~0 ? mheap_elt_at_uoffset (v, no) : 0;
+
+  n = no != MHEAP_GROUNDED ? mheap_elt_at_uoffset (v, no) : 0;
   po = e->free_elt.prev_uoffset;
-  p = po != ~0 ? mheap_elt_at_uoffset (v, po) : 0;
+  p = po != MHEAP_GROUNDED ? mheap_elt_at_uoffset (v, po) : 0;
 
   if (! p)
     set_first_free_elt_offset (h, bin, no);
@@ -309,13 +315,13 @@ mheap_get_small_object (mheap_t * h, uword bin)
 {
   mheap_small_object_cache_t * c = &h->small_object_cache;
   uword mask = mheap_small_object_cache_mask (c, bin + 1);
-  uword offset = ~0;
+  uword offset = MHEAP_GROUNDED;
 
   if (mask)
     {
       uword i = min_log2 (mask);
       uword o = c->offsets[i];
-      ASSERT (o != ~0);
+      ASSERT (o != MHEAP_GROUNDED);
       c->bins.as_u8[i] = 0;
       offset = o;
     }
@@ -369,7 +375,7 @@ mheap_get_search_free_bin (void * v,
   word o0, o1, f0, f1, search_n_user_data_bytes;
   word lo_free_usize, hi_free_usize;
 
-  ASSERT (h->first_free_elt_uoffset_by_bin[bin] != ~0);
+  ASSERT (h->first_free_elt_uoffset_by_bin[bin] != MHEAP_GROUNDED);
   e = mheap_elt_at_uoffset (v, h->first_free_elt_uoffset_by_bin[bin]);
 
   search_n_user_data_bytes = *n_user_data_bytes_arg;
@@ -420,8 +426,8 @@ mheap_get_search_free_bin (void * v,
 
     next:
       /* Reached end of free list without finding large enough object. */
-      if (e->free_elt.next_uoffset == ~0)
-	return ~0;
+      if (e->free_elt.next_uoffset == MHEAP_GROUNDED)
+	return MHEAP_GROUNDED;
 
       /* Otherwise keep searching for large enough object. */
       e = mheap_elt_at_uoffset (v, e->free_elt.next_uoffset);
@@ -517,7 +523,7 @@ mheap_get_search_free_list (void * v,
     {
       uword r = mheap_get_small_object (h, bin);
       h->stats.n_small_object_cache_attempts += 1;
-      if (r != ~0)
+      if (r != MHEAP_GROUNDED)
 	{
 	  h->stats.n_small_object_cache_hits += 1;
 	  return r;
@@ -535,12 +541,12 @@ mheap_get_search_free_list (void * v,
       /* Search each occupied free bin which is large enough. */
       foreach_set_bit (bi, non_empty_bin_mask, ({
 	uword r = mheap_get_search_free_bin (v, bi + i * BITS (uword), n_user_bytes_arg, align, align_offset);
-	if (r != ~0)
+	if (r != MHEAP_GROUNDED)
 	  return r;
       }));
     }
 
-  return ~0;
+  return MHEAP_GROUNDED;
 }
 
 static never_inline void *
@@ -586,7 +592,7 @@ mheap_get_extend_vector (void * v,
   /* Make sure we have space for object plus overhead. */
   if (f1 > h->max_size)
     {
-      *offset_return = ~0;
+      *offset_return = MHEAP_GROUNDED;
       return v;
     }
 
@@ -639,7 +645,7 @@ void * mheap_get_aligned (void * v,
   /* Align offset must be multiple of minimum object size. */
   if (align_offset % STRUCT_SIZE_OF (mheap_elt_t, user_data[0]) != 0)
     {
-      *offset_return = ~0;
+      *offset_return = MHEAP_GROUNDED;
       return v;
     }
 
@@ -663,15 +669,15 @@ void * mheap_get_aligned (void * v,
   h = mheap_header (v);
 
   /* If that fails allocate object at end of heap by extending vector. */
-  if (offset == ~0 && _vec_len (v) < h->max_size)
+  if (offset == MHEAP_GROUNDED && _vec_len (v) < h->max_size)
     {
       v = mheap_get_extend_vector (v, n_user_data_bytes, align, align_offset, &offset);
       h = mheap_header (v);
-      h->stats.n_vector_expands += offset != ~0;
+      h->stats.n_vector_expands += offset != MHEAP_GROUNDED;
     }
 
   *offset_return = offset;
-  if (offset != ~0)
+  if (offset != MHEAP_GROUNDED)
     {
       h->n_elts += 1;
 
@@ -909,7 +915,8 @@ void * mheap_alloc_with_flags (void * memory, uword memory_size, uword flags)
 	      (clib_address_t) v, h->max_size);
 
   /* Initialize free list heads to empty. */
-  memset (h->first_free_elt_uoffset_by_bin, ~0, sizeof (h->first_free_elt_uoffset_by_bin));
+  memset (h->first_free_elt_uoffset_by_bin, 0xFF, 
+          sizeof (h->first_free_elt_uoffset_by_bin));
 
   return v;
 }
@@ -1310,10 +1317,10 @@ void mheap_validate (void * v)
       mheap_elt_t * e, * n;
       uword is_first;
 
-      CHECK ((h->first_free_elt_uoffset_by_bin[i] != ~0)
+      CHECK ((h->first_free_elt_uoffset_by_bin[i] != MHEAP_GROUNDED)
 	     == ((h->non_empty_free_elt_heads[i / BITS (uword)] & ((uword) 1 << (uword) (i % BITS (uword)))) != 0));
 
-      if (h->first_free_elt_uoffset_by_bin[i] == ~0)
+      if (h->first_free_elt_uoffset_by_bin[i] == MHEAP_GROUNDED)
 	continue;
 
       e = mheap_elt_at_uoffset (v, h->first_free_elt_uoffset_by_bin[i]);
@@ -1331,7 +1338,7 @@ void mheap_validate (void * v)
 	  CHECK (n->prev_is_free);
 
 	  if (is_first)
-	    CHECK (e->free_elt.prev_uoffset == ~0);
+	    CHECK (e->free_elt.prev_uoffset == MHEAP_GROUNDED);
 	  is_first = 0;
 
 	  s = mheap_elt_data_bytes (e);
@@ -1340,7 +1347,7 @@ void mheap_validate (void * v)
 	  free_count_from_free_lists += 1;
 	  free_size_from_free_lists += s;
 
-	  if (e->free_elt.next_uoffset == ~0)
+	  if (e->free_elt.next_uoffset == MHEAP_GROUNDED)
 	    break;
 
 	  n = mheap_elt_at_uoffset (v, e->free_elt.next_uoffset);
diff --git a/vppinfra/vppinfra/mheap_bootstrap.h b/vppinfra/vppinfra/mheap_bootstrap.h
index 8a2c991bdea..c6cff8e7b7d 100644
--- a/vppinfra/vppinfra/mheap_bootstrap.h
+++ b/vppinfra/vppinfra/mheap_bootstrap.h
@@ -50,10 +50,29 @@
 typedef struct {
   /* Number of mheap_size_t words of user data in previous object.
      Used to find mheap_elt_t for previous object. */
+#if CLIB_VEC64 > 0
+  u64 prev_n_user_data : 63;
+
+  /* Used to mark end/start of of doubly-linked list of mheap_elt_t's. */
+#define MHEAP_N_USER_DATA_INVALID (0x7fffffffffffffffULL)
+#define MHEAP_GROUNDED (~0ULL)
+
+  /* Set if previous object is free. */
+  u64 prev_is_free : 1;
+
+  /* Number of mheap_size_t words of user data that follow this object. */
+  u64 n_user_data : 63;
+
+  /* Set if this object is on free list (and therefore following free_elt
+     is valid). */
+  u64 is_free : 1;
+
+#else
   u32 prev_n_user_data : 31;
 
   /* Used to mark end/start of of doubly-linked list of mheap_elt_t's. */
 #define MHEAP_N_USER_DATA_INVALID (0x7fffffff)
+#define MHEAP_GROUNDED (~0)
 
   /* Set if previous object is free. */
   u32 prev_is_free : 1;
@@ -64,8 +83,22 @@ typedef struct {
   /* Set if this object is on free list (and therefore following free_elt
      is valid). */
   u32 is_free : 1;
-
+#endif
+    
   union {
+#if CLIB_VEC64 > 0
+    /* For allocated objects: user data follows.
+       User data is allocated in units of typeof (user_data[0]). */
+    u64 user_data[0];
+
+    /* For free objects, offsets of next and previous free objects of this size;
+       ~0 means end of doubly-linked list.
+       This is stored in user data (guaranteed to be at least 8 bytes)
+       but only for *free* objects. */
+    struct {
+      u64 next_uoffset, prev_uoffset;
+    } free_elt;
+#else
     /* For allocated objects: user data follows.
        User data is allocated in units of typeof (user_data[0]). */
     u32 user_data[0];
@@ -77,6 +110,7 @@ typedef struct {
     struct {
       u32 next_uoffset, prev_uoffset;
     } free_elt;
+#endif
   };
 } mheap_elt_t;
 
@@ -94,7 +128,11 @@ typedef struct {
   uword callers[12];
 
   /* Count of allocations with this traceback. */
+#if CLIB_VEC64 > 0
+  u64 n_allocations;
+#else
   u32 n_allocations;
+#endif
 
   /* Count of bytes allocated with this traceback. */
   u32 n_bytes;
@@ -149,6 +187,11 @@ typedef struct {
 #define MHEAP_HAVE_SMALL_OBJECT_CACHE 0
 #endif
 
+#if CLIB_VEC64 > 0
+#undef MHEAP_HAVE_SMALL_OBJECT_CACHE
+#define MHEAP_HAVE_SMALL_OBJECT_CACHE 0
+#endif
+
 /* For objects with align == 4 and align_offset == 0 (e.g. vector strings). */
 typedef struct {
   union {
@@ -168,7 +211,11 @@ typedef struct {
 /* Vec header for heaps. */
 typedef struct {
   /* User offsets for head of doubly-linked list of free objects of this size. */
+#if CLIB_VEC64 > 0
+  u64 first_free_elt_uoffset_by_bin[MHEAP_N_BINS];
+#else
   u32 first_free_elt_uoffset_by_bin[MHEAP_N_BINS];
+#endif
 
   /* Bitmap of non-empty free list bins. */
   uword non_empty_free_elt_heads[(MHEAP_N_BINS + BITS (uword) - 1) / BITS (uword)];