From b3d93dacfde8ab21bbce171fff2971b2ed7bce6a Mon Sep 17 00:00:00 2001 From: Dave Barach Date: Wed, 3 Aug 2016 14:34:38 -0400 Subject: VPP-236 Support 64-bit vector lengths, shared memory segments >4 GB Change-Id: I02aee33e96e7ae32094b9f82f6a667d30bb52f59 Signed-off-by: Dave Barach --- svm/svm.c | 92 ++++++++++++++++++++++++++----------- svm/svm.h | 6 ++- svm/svmtool.c | 9 ++-- vlib-api/vlibapi/api.h | 18 +++++++- vlib-api/vlibmemory/api.h | 5 ++ vlib-api/vlibmemory/memory_shared.c | 43 ++++++++++++++++- vlib-api/vlibmemory/memory_vlib.c | 17 ++++++- vpp/vpp-api/api.c | 29 +++++++++++- vpp/vpp-api/gmon.c | 4 +- vppinfra/Makefile.am | 6 +-- vppinfra/configure.ac | 9 ---- vppinfra/vppinfra/mheap.c | 53 +++++++++++---------- vppinfra/vppinfra/mheap_bootstrap.h | 49 +++++++++++++++++++- 13 files changed, 264 insertions(+), 76 deletions(-) diff --git a/svm/svm.c b/svm/svm.c index 1ee5e95d..851baea5 100644 --- a/svm/svm.c +++ b/svm/svm.c @@ -216,10 +216,10 @@ format_svm_region (u8 * s, va_list * args) * rnd_pagesize * Round to a pagesize multiple, presumably 4k works */ -static unsigned int -rnd_pagesize (unsigned int size) +static u64 +rnd_pagesize (u64 size) { - unsigned int rv; + u64 rv; rv = (size + (MMAP_PAGESIZE - 1)) & ~(MMAP_PAGESIZE - 1); return (rv); @@ -235,7 +235,9 @@ svm_data_region_create (svm_map_region_args_t * a, svm_region_t * rp) u8 junk = 0; uword map_size; - map_size = rp->virtual_size - (MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE); + map_size = rp->virtual_size - (MMAP_PAGESIZE + + (a->pvt_heap_size ? a->pvt_heap_size : + SVM_PVT_MHEAP_SIZE)); if (a->flags & SVM_FLAGS_FILE) { @@ -316,7 +318,9 @@ svm_data_region_map (svm_map_region_args_t * a, svm_region_t * rp) uword map_size; struct stat statb; - map_size = rp->virtual_size - (MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE); + map_size = rp->virtual_size - + (MMAP_PAGESIZE + + (a->pvt_heap_size ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE)); if (a->flags & SVM_FLAGS_FILE) { @@ -364,7 +368,9 @@ svm_data_region_map (svm_map_region_args_t * a, svm_region_t * rp) } ASSERT (map_size <= rp->virtual_size - - (MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE)); + - (MMAP_PAGESIZE + + + (a->pvt_heap_size ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE))); if (mmap (rp->data_base, map_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, 0) == MAP_FAILED) @@ -528,7 +534,9 @@ svm_map_region (svm_map_region_args_t * a) rp->region_heap = mheap_alloc_with_flags ((void *) (a->baseva + MMAP_PAGESIZE), - SVM_PVT_MHEAP_SIZE, MHEAP_FLAG_DISABLE_VM); + (a->pvt_heap_size != 0) ? + a->pvt_heap_size : SVM_PVT_MHEAP_SIZE, + MHEAP_FLAG_DISABLE_VM); oldheap = svm_push_pvt_heap (rp); rp->region_name = (char *) format (0, "%s%c", a->name, 0); @@ -542,7 +550,7 @@ svm_map_region (svm_map_region_args_t * a) vec_validate (rp->bitmap, words - 1); overhead_space = MMAP_PAGESIZE /* header */ + - SVM_PVT_MHEAP_SIZE; + ((a->pvt_heap_size != 0)? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE); bit = 0; data_base = (uword) rp->virtual_base; @@ -724,10 +732,9 @@ svm_mutex_cleanup (void) } static void -svm_region_init_internal (char *root_path, int uid, int gid) +svm_region_init_internal (svm_map_region_args_t * a) { svm_region_t *rp; - svm_map_region_args_t _a, *a = &_a; u64 ticks = clib_cpu_time_now (); uword randomize_baseva; @@ -745,14 +752,7 @@ svm_region_init_internal (char *root_path, int uid, int gid) else randomize_baseva = (ticks & 3) * MMAP_PAGESIZE; - memset (a, 0, sizeof (*a)); - a->root_path = root_path; - a->name = SVM_GLOBAL_REGION_NAME; - a->baseva = SVM_GLOBAL_REGION_BASEVA + randomize_baseva; - a->size = SVM_GLOBAL_REGION_SIZE; - a->flags = SVM_FLAGS_NODATA; - a->uid = uid; - a->gid = gid; + a->baseva += randomize_baseva; rp = svm_map_region (a); ASSERT (rp); @@ -770,7 +770,7 @@ svm_region_init_internal (char *root_path, int uid, int gid) oldheap = svm_push_pvt_heap (rp); vec_validate (mp, 0); mp->name_hash = hash_create_string (0, sizeof (uword)); - mp->root_path = root_path ? format (0, "%s%c", root_path, 0) : 0; + mp->root_path = a->root_path ? format (0, "%s%c", a->root_path, 0) : 0; rp->data_base = mp; svm_pop_heap (oldheap); } @@ -781,19 +781,58 @@ svm_region_init_internal (char *root_path, int uid, int gid) void svm_region_init (void) { - svm_region_init_internal (0, 0 /* uid */ , 0 /* gid */ ); + svm_map_region_args_t _a, *a = &_a; + + memset (a, 0, sizeof (*a)); + a->root_path = 0; + a->name = SVM_GLOBAL_REGION_NAME; + a->baseva = SVM_GLOBAL_REGION_BASEVA; + a->size = SVM_GLOBAL_REGION_SIZE; + a->flags = SVM_FLAGS_NODATA; + a->uid = 0; + a->gid = 0; + + svm_region_init_internal (a); } void svm_region_init_chroot (char *root_path) { - svm_region_init_internal (root_path, 0 /* uid */ , 0 /* gid */ ); + svm_map_region_args_t _a, *a = &_a; + + memset (a, 0, sizeof (*a)); + a->root_path = root_path; + a->name = SVM_GLOBAL_REGION_NAME; + a->baseva = SVM_GLOBAL_REGION_BASEVA; + a->size = SVM_GLOBAL_REGION_SIZE; + a->flags = SVM_FLAGS_NODATA; + a->uid = 0; + a->gid = 0; + + svm_region_init_internal (a); } void svm_region_init_chroot_uid_gid (char *root_path, int uid, int gid) { - svm_region_init_internal (root_path, uid, gid); + svm_map_region_args_t _a, *a = &_a; + + memset (a, 0, sizeof (*a)); + a->root_path = root_path; + a->name = SVM_GLOBAL_REGION_NAME; + a->baseva = SVM_GLOBAL_REGION_BASEVA; + a->size = SVM_GLOBAL_REGION_SIZE; + a->flags = SVM_FLAGS_NODATA; + a->uid = uid; + a->gid = gid; + + svm_region_init_internal (a); +} + +void +svm_region_init_args (svm_map_region_args_t * a) +{ + svm_region_init_internal (a); } void * @@ -810,7 +849,8 @@ svm_region_find_or_create (svm_map_region_args_t * a) ASSERT (root_rp); - a->size += MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE; + a->size += MMAP_PAGESIZE + + ((a->pvt_heap_size != 0) ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE); a->size = rnd_pagesize (a->size); region_lock (root_rp, 4); @@ -869,8 +909,8 @@ svm_region_find_or_create (svm_map_region_args_t * a) /* Completely out of VM? */ if (index >= root_rp->bitmap_size) { - clib_warning ("region %s: not enough VM to allocate 0x%x", - root_rp->region_name, a->size); + clib_warning ("region %s: not enough VM to allocate 0x%llx (%lld)", + root_rp->region_name, a->size, a->size); svm_pop_heap (oldheap); region_unlock (root_rp); return 0; @@ -892,7 +932,7 @@ svm_region_find_or_create (svm_map_region_args_t * a) a->baseva = root_rp->virtual_base + index * MMAP_PAGESIZE; rp = svm_map_region (a); - + pool_get (mp->subregions, subp); name = format (0, "%s%c", a->name, 0); subp->subregion_name = name; diff --git a/svm/svm.h b/svm/svm.h index 1682b7ad..c42d2b57 100644 --- a/svm/svm.h +++ b/svm/svm.h @@ -71,8 +71,9 @@ typedef struct svm_map_region_args_ { char *root_path; /* NULL means use the truly global arena */ char *name; - uword baseva; - uword size; + u64 baseva; + u64 size; + u64 pvt_heap_size; uword flags; char *backing_file; uword backing_mmap_size; @@ -116,6 +117,7 @@ void *svm_region_find_or_create (svm_map_region_args_t * a); void svm_region_init (void); void svm_region_init_chroot (char *root_path); void svm_region_init_chroot_uid_gid (char *root_path, int uid, int gid); +void svm_region_init_args (svm_map_region_args_t *a); void svm_region_exit (void); void svm_region_unmap (void *rp_arg); void svm_client_scan (char *root_path); diff --git a/svm/svmtool.c b/svm/svmtool.c index 545f071e..2e55a087 100644 --- a/svm/svmtool.c +++ b/svm/svmtool.c @@ -206,10 +206,10 @@ svm_map_region_nolock (svm_map_region_args_t * a) * rnd_pagesize * Round to a pagesize multiple, presumably 4k works */ -static unsigned int -rnd_pagesize (unsigned int size) +static u64 +rnd_pagesize (u64 size) { - unsigned int rv; + u64 rv; rv = (size + (MMAP_PAGESIZE - 1)) & ~(MMAP_PAGESIZE - 1); return (rv); @@ -247,7 +247,8 @@ svm_existing_region_map_nolock (void *root_arg, svm_map_region_args_t * a) void *oldheap; uword *p; - a->size += MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE; + a->size += MMAP_PAGESIZE + + (a->pvt_heap_size ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE); a->size = rnd_pagesize (a->size); region_lock (root_rp, 4); diff --git a/vlib-api/vlibapi/api.h b/vlib-api/vlibapi/api.h index 9d885c17..a932bf42 100644 --- a/vlib-api/vlibapi/api.h +++ b/vlib-api/vlibapi/api.h @@ -141,9 +141,25 @@ typedef struct /* vector of message ranges */ vl_api_msg_range_t *msg_ranges; + /* uid for the api shared memory region */ + int api_uid; /* gid for the api shared memory region */ int api_gid; - int api_uid; + + /* base virtual address for global VM region */ + u64 global_baseva; + + /* size of the global VM region */ + u64 global_size; + + /* size of the API region */ + u64 api_size; + + /* size of the global VM private mheap */ + u64 global_pvt_heap_size; + + /* size of the api private mheap */ + u64 api_pvt_heap_size; /* Client-only data structures */ unix_shared_memory_queue_t *vl_input_queue; diff --git a/vlib-api/vlibmemory/api.h b/vlib-api/vlibmemory/api.h index e3a95843..bc4cfbf4 100644 --- a/vlib-api/vlibmemory/api.h +++ b/vlib-api/vlibmemory/api.h @@ -142,6 +142,11 @@ void vl_set_memory_region_name (char *name); void vl_set_memory_root_path (char *root_path); void vl_set_memory_uid (int uid); void vl_set_memory_gid (int gid); +void vl_set_global_memory_baseva (u64 baseva); +void vl_set_global_memory_size (u64 size); +void vl_set_api_memory_size (u64 size); +void vl_set_global_pvt_heap_size (u64 size); +void vl_set_api_pvt_heap_size (u64 size); void vl_enable_disable_memory_api (vlib_main_t * vm, int yesno); void vl_client_disconnect_from_vlib (void); int vl_client_connect_to_vlib (char *svm_name, char *client_name, diff --git a/vlib-api/vlibmemory/memory_shared.c b/vlib-api/vlibmemory/memory_shared.c index 53976fbe..a2f22771 100644 --- a/vlib-api/vlibmemory/memory_shared.c +++ b/vlib-api/vlibmemory/memory_shared.c @@ -228,6 +228,46 @@ vl_set_memory_gid (int gid) am->api_gid = gid; } +void +vl_set_global_memory_baseva (u64 baseva) +{ + api_main_t *am = &api_main; + + am->global_baseva = baseva; +} + +void +vl_set_global_memory_size (u64 size) +{ + api_main_t *am = &api_main; + + am->global_size = size; +} + +void +vl_set_api_memory_size (u64 size) +{ + api_main_t *am = &api_main; + + am->api_size = size; +} + +void +vl_set_global_pvt_heap_size (u64 size) +{ + api_main_t *am = &api_main; + + am->global_pvt_heap_size = size; +} + +void +vl_set_api_pvt_heap_size (u64 size) +{ + api_main_t *am = &api_main; + + am->api_pvt_heap_size = size; +} + int vl_map_shmem (char *region_name, int is_vlib) { @@ -245,10 +285,11 @@ vl_map_shmem (char *region_name, int is_vlib) memset (a, 0, sizeof (*a)); a->name = region_name; - a->size = 16 << 20; + a->size = am->api_size ? am->api_size: (16 << 20); a->flags = SVM_FLAGS_MHEAP; a->uid = am->api_uid; a->gid = am->api_gid; + a->pvt_heap_size = am->api_pvt_heap_size; vlib_rp = svm_region_find_or_create (a); diff --git a/vlib-api/vlibmemory/memory_vlib.c b/vlib-api/vlibmemory/memory_vlib.c index 68752745..2212b46b 100644 --- a/vlib-api/vlibmemory/memory_vlib.c +++ b/vlib-api/vlibmemory/memory_vlib.c @@ -1176,8 +1176,21 @@ clib_error_t * vlibmemory_init (vlib_main_t * vm) { api_main_t *am = &api_main; - /* Normally NULL / 0, set by cmd line "api-segment" */ - svm_region_init_chroot_uid_gid (am->root_path, am->api_uid, am->api_gid); + svm_map_region_args_t _a, *a = &_a; + + memset (a, 0, sizeof (*a)); + a->root_path = am->root_path; + a->name = SVM_GLOBAL_REGION_NAME; + a->baseva = (am->global_baseva != 0) ? + am->global_baseva : SVM_GLOBAL_REGION_BASEVA; + a->size = (am->global_size != 0) ? am->global_size : SVM_GLOBAL_REGION_SIZE; + a->flags = SVM_FLAGS_NODATA; + a->uid = am->api_uid; + a->gid = am->api_gid; + a->pvt_heap_size = (am->global_pvt_heap_size != 0) ? am->global_pvt_heap_size + : SVM_PVT_MHEAP_SIZE; + + svm_region_init_args (a); return 0; } diff --git a/vpp/vpp-api/api.c b/vpp/vpp-api/api.c index 8a341597..ee3ced43 100644 --- a/vpp/vpp-api/api.c +++ b/vpp/vpp-api/api.c @@ -8111,7 +8111,8 @@ VLIB_INIT_FUNCTION (vpe_api_init); static clib_error_t * api_segment_config (vlib_main_t * vm, unformat_input_t * input) { - u8 *chroot_path; + u8 * chroot_path; + u64 baseva, size, pvt_heap_size; int uid, gid, rv; const int max_buf_size = 4096; char *s, *buf; @@ -8129,7 +8130,31 @@ api_segment_config (vlib_main_t * vm, unformat_input_t * input) else if (unformat (input, "uid %d", &uid)) vl_set_memory_uid (uid); else if (unformat (input, "gid %d", &gid)) - vl_set_memory_gid (gid); + vl_set_memory_gid (gid); + else if (unformat (input, "baseva %llx", &baseva)) + vl_set_global_memory_baseva (baseva); + else if (unformat (input, "global-size %lldM", &size)) + vl_set_global_memory_size (size * (1ULL<<20)); + else if (unformat (input, "global-size %lldG", &size)) + vl_set_global_memory_size (size * (1ULL<<30)); + else if (unformat (input, "global-size %lld", &size)) + vl_set_global_memory_size (size); + else if (unformat (input, "global-pvt-heap-size %lldM", &pvt_heap_size)) + vl_set_global_pvt_heap_size (pvt_heap_size * (1ULL<<20)); + else if (unformat (input, "global-pvt-heap-size size %lld", + &pvt_heap_size)) + vl_set_global_pvt_heap_size (pvt_heap_size); + else if (unformat (input, "api-pvt-heap-size %lldM", &pvt_heap_size)) + vl_set_api_pvt_heap_size (pvt_heap_size * (1ULL<<20)); + else if (unformat (input, "api-pvt-heap-size size %lld", + &pvt_heap_size)) + vl_set_api_pvt_heap_size (pvt_heap_size); + else if (unformat (input, "api-size %lldM", &size)) + vl_set_api_memory_size (size * (1ULL<<20)); + else if (unformat (input, "api-size %lldG", &size)) + vl_set_api_memory_size (size * (1ULL<<30)); + else if (unformat (input, "api-size %lld", &size)) + vl_set_api_memory_size (size); else if (unformat (input, "uid %s", &s)) { /* lookup the username */ diff --git a/vpp/vpp-api/gmon.c b/vpp/vpp-api/gmon.c index ea4a25ef..330dc9fe 100644 --- a/vpp/vpp-api/gmon.c +++ b/vpp/vpp-api/gmon.c @@ -180,8 +180,8 @@ gmon_init (vlib_main_t * vm) if ((error = vlib_call_init_function (vm, vpe_api_init))) return (error); - /* Make sure that /global-vm is owned as directed */ - svm_region_init_chroot_uid_gid (am->root_path, am->api_uid, am->api_gid); + if ((error = vlib_call_init_function(vm, vlibmemory_init))) + return(error); gm->vlib_main = vm; diff --git a/vppinfra/Makefile.am b/vppinfra/Makefile.am index f9855e34..48a347e0 100644 --- a/vppinfra/Makefile.am +++ b/vppinfra/Makefile.am @@ -12,7 +12,7 @@ # limitations under the License. AUTOMAKE_OPTIONS = foreign subdir-objects -AM_CPPFLAGS = -Wall @VEC64@ # -Werror +AM_CPPFLAGS = -Wall -Werror if WITH_UNIX lib_LTLIBRARIES = libvppinfra.la @@ -20,7 +20,7 @@ endif lib_LIBRARIES = -TESTS = +TESTS = if ENABLE_TESTS TESTS += test_bihash_template \ @@ -267,5 +267,5 @@ libvppinfra_la_SOURCES = \ bin_PROGRAMS = elftool elftool_SOURCES = tools/elftool.c -elftool_CPPFLAGS = -Wall +elftool_CPPFLAGS = $(AM_CPPFLAGS) elftool_LDADD = libvppinfra.la -lpthread -lrt -lm diff --git a/vppinfra/configure.ac b/vppinfra/configure.ac index 88720f71..88939383 100644 --- a/vppinfra/configure.ac +++ b/vppinfra/configure.ac @@ -48,14 +48,5 @@ AC_ARG_WITH(unix, AM_CONDITIONAL(WITH_UNIX, test "$with_unix" = "yes") - -## Enable 64-bit vector lengths -AC_ARG_WITH(vec64, - AC_HELP_STRING([--with-vec64],[Enable 64-bit vector lengths]), - [with_vec64=1], - [with_vec64=0]) - -AC_SUBST(VEC64,[-DCLIB_VEC64=${with_vec64}]) - AC_CONFIG_FILES([Makefile]) AC_OUTPUT diff --git a/vppinfra/vppinfra/mheap.c b/vppinfra/vppinfra/mheap.c index 6163d0b1..3bb2d037 100644 --- a/vppinfra/vppinfra/mheap.c +++ b/vppinfra/vppinfra/mheap.c @@ -133,6 +133,7 @@ mheap_elt_set_size (void * v, e = mheap_elt_at_uoffset (v, uoffset); ASSERT (n_user_data_bytes % MHEAP_USER_DATA_WORD_BYTES == 0); + e->n_user_data = n_user_data_bytes / MHEAP_USER_DATA_WORD_BYTES; e->is_free = is_free; ASSERT (e->prev_n_user_data * sizeof (e->user_data[0]) >= MHEAP_MIN_USER_DATA_BYTES); @@ -152,7 +153,7 @@ always_inline void set_first_free_elt_offset (mheap_t * h, uword bin, uword uoff i1 = (uword) 1 << (uword) (bin % BITS (h->non_empty_free_elt_heads[0])); ASSERT (i0 < ARRAY_LEN (h->non_empty_free_elt_heads)); - if (h->first_free_elt_uoffset_by_bin[bin] == ~0) + if (h->first_free_elt_uoffset_by_bin[bin] == MHEAP_GROUNDED) h->non_empty_free_elt_heads[i0] &= ~i1; else h->non_empty_free_elt_heads[i0] |= i1; @@ -169,11 +170,11 @@ set_free_elt (void * v, uword uoffset, uword n_user_data_bytes) ASSERT (n->prev_is_free); ASSERT (e->is_free); - e->free_elt.prev_uoffset = ~0; + e->free_elt.prev_uoffset = MHEAP_GROUNDED; e->free_elt.next_uoffset = h->first_free_elt_uoffset_by_bin[bin]; /* Fill in next free elt's previous pointer. */ - if (e->free_elt.next_uoffset != ~0) + if (e->free_elt.next_uoffset != MHEAP_GROUNDED) { mheap_elt_t * nf = mheap_elt_at_uoffset (v, e->free_elt.next_uoffset); ASSERT (nf->is_free); @@ -195,12 +196,17 @@ remove_free_elt (void * v, mheap_elt_t * e, uword bin) { mheap_t * h = mheap_header (v); mheap_elt_t * p, * n; +#if CLIB_VEC64 > 0 + u64 no, po; +#else u32 no, po; +#endif no = e->free_elt.next_uoffset; - n = no != ~0 ? mheap_elt_at_uoffset (v, no) : 0; + + n = no != MHEAP_GROUNDED ? mheap_elt_at_uoffset (v, no) : 0; po = e->free_elt.prev_uoffset; - p = po != ~0 ? mheap_elt_at_uoffset (v, po) : 0; + p = po != MHEAP_GROUNDED ? mheap_elt_at_uoffset (v, po) : 0; if (! p) set_first_free_elt_offset (h, bin, no); @@ -309,13 +315,13 @@ mheap_get_small_object (mheap_t * h, uword bin) { mheap_small_object_cache_t * c = &h->small_object_cache; uword mask = mheap_small_object_cache_mask (c, bin + 1); - uword offset = ~0; + uword offset = MHEAP_GROUNDED; if (mask) { uword i = min_log2 (mask); uword o = c->offsets[i]; - ASSERT (o != ~0); + ASSERT (o != MHEAP_GROUNDED); c->bins.as_u8[i] = 0; offset = o; } @@ -369,7 +375,7 @@ mheap_get_search_free_bin (void * v, word o0, o1, f0, f1, search_n_user_data_bytes; word lo_free_usize, hi_free_usize; - ASSERT (h->first_free_elt_uoffset_by_bin[bin] != ~0); + ASSERT (h->first_free_elt_uoffset_by_bin[bin] != MHEAP_GROUNDED); e = mheap_elt_at_uoffset (v, h->first_free_elt_uoffset_by_bin[bin]); search_n_user_data_bytes = *n_user_data_bytes_arg; @@ -420,8 +426,8 @@ mheap_get_search_free_bin (void * v, next: /* Reached end of free list without finding large enough object. */ - if (e->free_elt.next_uoffset == ~0) - return ~0; + if (e->free_elt.next_uoffset == MHEAP_GROUNDED) + return MHEAP_GROUNDED; /* Otherwise keep searching for large enough object. */ e = mheap_elt_at_uoffset (v, e->free_elt.next_uoffset); @@ -517,7 +523,7 @@ mheap_get_search_free_list (void * v, { uword r = mheap_get_small_object (h, bin); h->stats.n_small_object_cache_attempts += 1; - if (r != ~0) + if (r != MHEAP_GROUNDED) { h->stats.n_small_object_cache_hits += 1; return r; @@ -535,12 +541,12 @@ mheap_get_search_free_list (void * v, /* Search each occupied free bin which is large enough. */ foreach_set_bit (bi, non_empty_bin_mask, ({ uword r = mheap_get_search_free_bin (v, bi + i * BITS (uword), n_user_bytes_arg, align, align_offset); - if (r != ~0) + if (r != MHEAP_GROUNDED) return r; })); } - return ~0; + return MHEAP_GROUNDED; } static never_inline void * @@ -586,7 +592,7 @@ mheap_get_extend_vector (void * v, /* Make sure we have space for object plus overhead. */ if (f1 > h->max_size) { - *offset_return = ~0; + *offset_return = MHEAP_GROUNDED; return v; } @@ -639,7 +645,7 @@ void * mheap_get_aligned (void * v, /* Align offset must be multiple of minimum object size. */ if (align_offset % STRUCT_SIZE_OF (mheap_elt_t, user_data[0]) != 0) { - *offset_return = ~0; + *offset_return = MHEAP_GROUNDED; return v; } @@ -663,15 +669,15 @@ void * mheap_get_aligned (void * v, h = mheap_header (v); /* If that fails allocate object at end of heap by extending vector. */ - if (offset == ~0 && _vec_len (v) < h->max_size) + if (offset == MHEAP_GROUNDED && _vec_len (v) < h->max_size) { v = mheap_get_extend_vector (v, n_user_data_bytes, align, align_offset, &offset); h = mheap_header (v); - h->stats.n_vector_expands += offset != ~0; + h->stats.n_vector_expands += offset != MHEAP_GROUNDED; } *offset_return = offset; - if (offset != ~0) + if (offset != MHEAP_GROUNDED) { h->n_elts += 1; @@ -909,7 +915,8 @@ void * mheap_alloc_with_flags (void * memory, uword memory_size, uword flags) (clib_address_t) v, h->max_size); /* Initialize free list heads to empty. */ - memset (h->first_free_elt_uoffset_by_bin, ~0, sizeof (h->first_free_elt_uoffset_by_bin)); + memset (h->first_free_elt_uoffset_by_bin, 0xFF, + sizeof (h->first_free_elt_uoffset_by_bin)); return v; } @@ -1310,10 +1317,10 @@ void mheap_validate (void * v) mheap_elt_t * e, * n; uword is_first; - CHECK ((h->first_free_elt_uoffset_by_bin[i] != ~0) + CHECK ((h->first_free_elt_uoffset_by_bin[i] != MHEAP_GROUNDED) == ((h->non_empty_free_elt_heads[i / BITS (uword)] & ((uword) 1 << (uword) (i % BITS (uword)))) != 0)); - if (h->first_free_elt_uoffset_by_bin[i] == ~0) + if (h->first_free_elt_uoffset_by_bin[i] == MHEAP_GROUNDED) continue; e = mheap_elt_at_uoffset (v, h->first_free_elt_uoffset_by_bin[i]); @@ -1331,7 +1338,7 @@ void mheap_validate (void * v) CHECK (n->prev_is_free); if (is_first) - CHECK (e->free_elt.prev_uoffset == ~0); + CHECK (e->free_elt.prev_uoffset == MHEAP_GROUNDED); is_first = 0; s = mheap_elt_data_bytes (e); @@ -1340,7 +1347,7 @@ void mheap_validate (void * v) free_count_from_free_lists += 1; free_size_from_free_lists += s; - if (e->free_elt.next_uoffset == ~0) + if (e->free_elt.next_uoffset == MHEAP_GROUNDED) break; n = mheap_elt_at_uoffset (v, e->free_elt.next_uoffset); diff --git a/vppinfra/vppinfra/mheap_bootstrap.h b/vppinfra/vppinfra/mheap_bootstrap.h index 8a2c991b..c6cff8e7 100644 --- a/vppinfra/vppinfra/mheap_bootstrap.h +++ b/vppinfra/vppinfra/mheap_bootstrap.h @@ -50,10 +50,29 @@ typedef struct { /* Number of mheap_size_t words of user data in previous object. Used to find mheap_elt_t for previous object. */ +#if CLIB_VEC64 > 0 + u64 prev_n_user_data : 63; + + /* Used to mark end/start of of doubly-linked list of mheap_elt_t's. */ +#define MHEAP_N_USER_DATA_INVALID (0x7fffffffffffffffULL) +#define MHEAP_GROUNDED (~0ULL) + + /* Set if previous object is free. */ + u64 prev_is_free : 1; + + /* Number of mheap_size_t words of user data that follow this object. */ + u64 n_user_data : 63; + + /* Set if this object is on free list (and therefore following free_elt + is valid). */ + u64 is_free : 1; + +#else u32 prev_n_user_data : 31; /* Used to mark end/start of of doubly-linked list of mheap_elt_t's. */ #define MHEAP_N_USER_DATA_INVALID (0x7fffffff) +#define MHEAP_GROUNDED (~0) /* Set if previous object is free. */ u32 prev_is_free : 1; @@ -64,8 +83,22 @@ typedef struct { /* Set if this object is on free list (and therefore following free_elt is valid). */ u32 is_free : 1; - +#endif + union { +#if CLIB_VEC64 > 0 + /* For allocated objects: user data follows. + User data is allocated in units of typeof (user_data[0]). */ + u64 user_data[0]; + + /* For free objects, offsets of next and previous free objects of this size; + ~0 means end of doubly-linked list. + This is stored in user data (guaranteed to be at least 8 bytes) + but only for *free* objects. */ + struct { + u64 next_uoffset, prev_uoffset; + } free_elt; +#else /* For allocated objects: user data follows. User data is allocated in units of typeof (user_data[0]). */ u32 user_data[0]; @@ -77,6 +110,7 @@ typedef struct { struct { u32 next_uoffset, prev_uoffset; } free_elt; +#endif }; } mheap_elt_t; @@ -94,7 +128,11 @@ typedef struct { uword callers[12]; /* Count of allocations with this traceback. */ +#if CLIB_VEC64 > 0 + u64 n_allocations; +#else u32 n_allocations; +#endif /* Count of bytes allocated with this traceback. */ u32 n_bytes; @@ -149,6 +187,11 @@ typedef struct { #define MHEAP_HAVE_SMALL_OBJECT_CACHE 0 #endif +#if CLIB_VEC64 > 0 +#undef MHEAP_HAVE_SMALL_OBJECT_CACHE +#define MHEAP_HAVE_SMALL_OBJECT_CACHE 0 +#endif + /* For objects with align == 4 and align_offset == 0 (e.g. vector strings). */ typedef struct { union { @@ -168,7 +211,11 @@ typedef struct { /* Vec header for heaps. */ typedef struct { /* User offsets for head of doubly-linked list of free objects of this size. */ +#if CLIB_VEC64 > 0 + u64 first_free_elt_uoffset_by_bin[MHEAP_N_BINS]; +#else u32 first_free_elt_uoffset_by_bin[MHEAP_N_BINS]; +#endif /* Bitmap of non-empty free list bins. */ uword non_empty_free_elt_heads[(MHEAP_N_BINS + BITS (uword) - 1) / BITS (uword)]; -- cgit 1.2.3-korg