diff options
Diffstat (limited to 'src/vppinfra')
| -rw-r--r-- | src/vppinfra/bihash_all_vector.c | 4 | ||||
| -rw-r--r-- | src/vppinfra/format.h | 1 | ||||
| -rw-r--r-- | src/vppinfra/linux/mem.c | 2 | ||||
| -rw-r--r-- | src/vppinfra/mem.c | 24 | ||||
| -rw-r--r-- | src/vppinfra/mem.h | 96 | ||||
| -rw-r--r-- | src/vppinfra/mem_dlmalloc.c | 219 | ||||
| -rw-r--r-- | src/vppinfra/unix-formats.c | 62 |
7 files changed, 220 insertions, 188 deletions
diff --git a/src/vppinfra/bihash_all_vector.c b/src/vppinfra/bihash_all_vector.c index b6c64a30e0b..35355b54062 100644 --- a/src/vppinfra/bihash_all_vector.c +++ b/src/vppinfra/bihash_all_vector.c @@ -17,9 +17,9 @@ /* Vector of all bihashes */ void **clib_all_bihashes; -static void **clib_all_bihash_heap; +static clib_mem_heap_t *clib_all_bihash_heap; -void * +clib_mem_heap_t * clib_all_bihash_set_heap (void) { if (PREDICT_FALSE (clib_all_bihash_heap == 0)) diff --git a/src/vppinfra/format.h b/src/vppinfra/format.h index e71c7b347e8..470070d8861 100644 --- a/src/vppinfra/format.h +++ b/src/vppinfra/format.h @@ -117,7 +117,6 @@ _(format_timeval); _(format_time_float); _(format_signal); _(format_ucontext_pc); -_(format_page_map); #endif #undef _ diff --git a/src/vppinfra/linux/mem.c b/src/vppinfra/linux/mem.c index 06e18cd400d..22268eadb8e 100644 --- a/src/vppinfra/linux/mem.c +++ b/src/vppinfra/linux/mem.c @@ -542,6 +542,8 @@ clib_mem_get_page_stats (void *start, clib_mem_page_sz_t log2_page_size, ptr[i] = start + (i << log2_page_size); clib_memset (stats, 0, sizeof (clib_mem_page_stats_t)); + stats->total = n_pages; + stats->log2_page_sz = log2_page_size; if (move_pages (0, n_pages, ptr, 0, status, 0) != 0) { diff --git a/src/vppinfra/mem.c b/src/vppinfra/mem.c index b417b8503ad..1f8b6bfe618 100644 --- a/src/vppinfra/mem.c +++ b/src/vppinfra/mem.c @@ -71,6 +71,30 @@ clib_mem_vm_map_shared (void *base, uword size, int fd, uword offset, return rv; } +u8 * +format_clib_mem_page_stats (u8 * s, va_list * va) +{ + clib_mem_page_stats_t *stats = va_arg (*va, clib_mem_page_stats_t *); + u32 indent = format_get_indent (s) + 2; + + s = format (s, "page stats: page-size %U, total %lu, mapped %lu, " + "not-mapped %lu", format_log2_page_size, stats->log2_page_sz, + stats->total, stats->mapped, stats->not_mapped); + + if (stats->unknown) + s = format (s, ", unknown %lu", stats->unknown); + + for (int i = 0; i < CLIB_MAX_NUMAS; i++) + if (stats->per_numa[i]) + s = format (s, "\n%Unuma %u: %lu pages, %U bytes", + format_white_space, indent, i, + stats->per_numa[i], + format_memory_size, + stats->per_numa[i] << stats->log2_page_sz); + + return s; +} + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index 34283f5aab1..a94b66ae187 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -93,6 +93,38 @@ typedef struct _clib_mem_vm_map_hdr struct _clib_mem_vm_map_hdr *prev, *next; } clib_mem_vm_map_hdr_t; +#define foreach_clib_mem_heap_flag \ + _(0, LOCKED, "locked") \ + _(1, UNMAP_ON_DESTROY, "unmap-on-destroy") + +typedef enum +{ +#define _(i, v, s) CLIB_MEM_HEAP_F_##v = (1 << i), + foreach_clib_mem_heap_flag +#undef _ +} clib_mem_heap_flag_t; + +typedef struct +{ + /* base address */ + void *base; + + /* dlmalloc mspace */ + void *mspace; + + /* heap size */ + uword size; + + /* page size (log2) */ + clib_mem_page_sz_t log2_page_sz:8; + + /* flags */ + clib_mem_heap_flag_t flags:8; + + /* name - _MUST_ be last */ + char name[0]; +} clib_mem_heap_t; + typedef struct { /* log2 system page size */ @@ -122,7 +154,7 @@ extern clib_mem_main_t clib_mem_main; /* Unspecified NUMA socket */ #define VEC_NUMA_UNSPECIFIED (0xFF) -always_inline void * +always_inline clib_mem_heap_t * clib_mem_get_per_cpu_heap (void) { int cpu = os_get_thread_index (); @@ -130,7 +162,7 @@ clib_mem_get_per_cpu_heap (void) } always_inline void * -clib_mem_set_per_cpu_heap (u8 * new_heap) +clib_mem_set_per_cpu_heap (void *new_heap) { int cpu = os_get_thread_index (); void *old = clib_mem_main.per_cpu_mheaps[cpu]; @@ -146,7 +178,7 @@ clib_mem_get_per_numa_heap (u32 numa_id) } always_inline void * -clib_mem_set_per_numa_heap (u8 * new_heap) +clib_mem_set_per_numa_heap (void *new_heap) { int numa = os_get_numa_index (); void *old = clib_mem_main.per_numa_mheaps[numa]; @@ -187,10 +219,10 @@ always_inline void * clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset, int os_out_of_memory_on_failure) { - void *heap, *p; - uword cpu; void *mspace_get_aligned (void *msp, unsigned long n_user_data_bytes, unsigned long align, unsigned long align_offset); + clib_mem_heap_t *h = clib_mem_get_per_cpu_heap (); + void *p; if (align_offset > align) { @@ -200,10 +232,7 @@ clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset, align_offset = align; } - cpu = os_get_thread_index (); - heap = clib_mem_main.per_cpu_mheaps[cpu]; - - p = mspace_get_aligned (heap, size, align, align_offset); + p = mspace_get_aligned (h->mspace, size, align, align_offset); if (PREDICT_FALSE (0 == p)) { @@ -270,24 +299,23 @@ clib_mem_alloc_aligned_or_null (uword size, uword align) always_inline uword clib_mem_is_heap_object (void *p) { - void *heap = clib_mem_get_per_cpu_heap (); int mspace_is_heap_object (void *msp, void *p); - - return mspace_is_heap_object (heap, p); + clib_mem_heap_t *h = clib_mem_get_per_cpu_heap (); + return mspace_is_heap_object (h->mspace, p); } always_inline void clib_mem_free (void *p) { - u8 *heap = clib_mem_get_per_cpu_heap (); - void mspace_put (void *msp, void *p_arg); + clib_mem_heap_t *h = clib_mem_get_per_cpu_heap (); + /* Make sure object is in the correct heap. */ ASSERT (clib_mem_is_heap_object (p)); CLIB_MEM_POISON (p, clib_mem_size_nocheck (p)); - mspace_put (heap, p); + mspace_put (h->mspace, p); } always_inline void * @@ -324,24 +352,24 @@ clib_mem_free_s (void *p) clib_mem_free (p); } -always_inline void * +always_inline clib_mem_heap_t * clib_mem_get_heap (void) { return clib_mem_get_per_cpu_heap (); } -always_inline void * -clib_mem_set_heap (void *heap) +always_inline clib_mem_heap_t * +clib_mem_set_heap (clib_mem_heap_t * heap) { return clib_mem_set_per_cpu_heap (heap); } -void clib_mem_destroy_heap (void *heap); -void *clib_mem_create_heap (void *base, uword size, int is_locked, char *fmt, - ...); +void clib_mem_destroy_heap (clib_mem_heap_t * heap); +clib_mem_heap_t *clib_mem_create_heap (void *base, uword size, int is_locked, + char *fmt, ...); void clib_mem_main_init (); -void *clib_mem_init (void *heap, uword size); +void *clib_mem_init (void *base, uword size); void *clib_mem_init_with_page_size (uword memory_size, clib_mem_page_sz_t log2_page_sz); void *clib_mem_init_thread_safe (void *memory, uword memory_size); @@ -377,14 +405,16 @@ typedef struct uword bytes_max; } clib_mem_usage_t; -void clib_mem_get_heap_usage (void *heap, clib_mem_usage_t * usage); +void clib_mem_get_heap_usage (clib_mem_heap_t * heap, + clib_mem_usage_t * usage); -void *clib_mem_get_heap_base (void *heap); -uword clib_mem_get_heap_size (void *heap); -uword clib_mem_get_heap_free_space (void *heap); +void *clib_mem_get_heap_base (clib_mem_heap_t * heap); +uword clib_mem_get_heap_size (clib_mem_heap_t * heap); +uword clib_mem_get_heap_free_space (clib_mem_heap_t * heap); u8 *format_clib_mem_usage (u8 * s, va_list * args); u8 *format_clib_mem_heap (u8 * s, va_list * va); +u8 *format_clib_mem_page_stats (u8 * s, va_list * va); /* Allocate virtual address space. */ always_inline void * @@ -454,19 +484,9 @@ u64 *clib_mem_vm_get_paddr (void *mem, clib_mem_page_sz_t log2_page_size, void clib_mem_destroy (void); int clib_mem_set_numa_affinity (u8 numa_node, int force); int clib_mem_set_default_numa_affinity (); - -typedef struct -{ - uword size; /**< Map size */ - int fd; /**< File descriptor to be mapped */ - uword requested_va; /**< Request fixed position mapping */ - void *addr; /**< Pointer to mapped memory, if successful */ - u8 numa_node; -} clib_mem_vm_map_t; - void clib_mem_vm_randomize_va (uword * requested_va, clib_mem_page_sz_t log2_page_size); -void mheap_trace (void *v, int enable); +void mheap_trace (clib_mem_heap_t * v, int enable); uword clib_mem_trace_enable_disable (uword enable); void clib_mem_trace (int enable); @@ -485,6 +505,8 @@ clib_mem_round_to_page_size (uword size, clib_mem_page_sz_t log2_page_size) typedef struct { + clib_mem_page_sz_t log2_page_sz; + uword total; uword mapped; uword not_mapped; uword per_numa[CLIB_MAX_NUMAS]; diff --git a/src/vppinfra/mem_dlmalloc.c b/src/vppinfra/mem_dlmalloc.c index 1c57373e650..a0bd1d09da2 100644 --- a/src/vppinfra/mem_dlmalloc.c +++ b/src/vppinfra/mem_dlmalloc.c @@ -194,87 +194,104 @@ mheap_trace_main_free (mheap_trace_main_t * tm) hash_free (tm->trace_index_by_offset); } -/* Initialize CLIB heap based on memory/size given by user. - Set memory to 0 and CLIB will try to allocate its own heap. */ -static void * -clib_mem_init_internal (void *memory, uword memory_size, - clib_mem_page_sz_t log2_page_sz, int set_heap) +static clib_mem_heap_t * +clib_mem_create_heap_internal (void *base, uword size, + clib_mem_page_sz_t log2_page_sz, int is_locked, + char *name) { - u8 *heap; - - clib_mem_main_init (); + clib_mem_heap_t *h; + u8 flags = 0; + int sz = sizeof (clib_mem_heap_t); - if (memory) - { - heap = create_mspace_with_base (memory, memory_size, 1 /* locked */ ); - mspace_disable_expand (heap); - } - else + if (base == 0) { - memory_size = round_pow2 (memory_size, - clib_mem_page_bytes (log2_page_sz)); - memory = clib_mem_vm_map_internal (0, log2_page_sz, memory_size, -1, 0, - "main heap"); + log2_page_sz = clib_mem_log2_page_size_validate (log2_page_sz); + size = round_pow2 (size, clib_mem_page_bytes (log2_page_sz)); + base = clib_mem_vm_map_internal (0, log2_page_sz, size, -1, 0, + "main heap"); - if (memory == CLIB_MEM_VM_MAP_FAILED) + if (base == CLIB_MEM_VM_MAP_FAILED) return 0; - heap = create_mspace_with_base (memory, memory_size, 1 /* locked */ ); - mspace_disable_expand (heap); + flags = CLIB_MEM_HEAP_F_UNMAP_ON_DESTROY; } + else + log2_page_sz = CLIB_MEM_PAGE_SZ_UNKNOWN; + + if (is_locked) + flags |= CLIB_MEM_HEAP_F_LOCKED; + + h = base; + h->base = base; + h->size = size; + h->log2_page_sz = log2_page_sz; + h->flags = flags; + sz = strlen (name); + strcpy (h->name, name); + sz = round_pow2 (sz + sizeof (clib_mem_heap_t), 16); + h->mspace = create_mspace_with_base (base + sz, size - sz, is_locked); - CLIB_MEM_POISON (mspace_least_addr (heap), mspace_footprint (heap)); + mspace_disable_expand (h->mspace); - if (set_heap) - clib_mem_set_heap (heap); + CLIB_MEM_POISON (mspace_least_addr (h->mspace), + mspace_footprint (h->mspace)); + + return h; +} + +/* Initialize CLIB heap based on memory/size given by user. + Set memory to 0 and CLIB will try to allocate its own heap. */ +static void * +clib_mem_init_internal (void *base, uword size, + clib_mem_page_sz_t log2_page_sz) +{ + clib_mem_heap_t *h; + + clib_mem_main_init (); + + h = clib_mem_create_heap_internal (base, size, log2_page_sz, + 1 /*is_locked */ , "main heap"); + + clib_mem_set_heap (h); if (mheap_trace_main.lock == 0) clib_spinlock_init (&mheap_trace_main.lock); - return heap; + return h; } void * clib_mem_init (void *memory, uword memory_size) { return clib_mem_init_internal (memory, memory_size, - CLIB_MEM_PAGE_SZ_DEFAULT, - 1 /* do clib_mem_set_heap */ ); + CLIB_MEM_PAGE_SZ_DEFAULT); } void * clib_mem_init_with_page_size (uword memory_size, clib_mem_page_sz_t log2_page_sz) { - return clib_mem_init_internal (0, memory_size, log2_page_sz, - 1 /* do clib_mem_set_heap */ ); + return clib_mem_init_internal (0, memory_size, log2_page_sz); } void * clib_mem_init_thread_safe (void *memory, uword memory_size) { return clib_mem_init_internal (memory, memory_size, - CLIB_MEM_PAGE_SZ_DEFAULT, - 1 /* do clib_mem_set_heap */ ); + CLIB_MEM_PAGE_SZ_DEFAULT); } void -clib_mem_destroy_mspace (void *mspace) +clib_mem_destroy (void) { mheap_trace_main_t *tm = &mheap_trace_main; + clib_mem_heap_t *heap = clib_mem_get_heap (); + void *base = mspace_least_addr (heap->mspace); - if (tm->enabled && mspace == tm->current_traced_mheap) + if (tm->enabled && heap->mspace == tm->current_traced_mheap) tm->enabled = 0; - destroy_mspace (mspace); -} - -void -clib_mem_destroy (void) -{ - void *heap = clib_mem_get_heap (); - void *base = mspace_least_addr (heap); - clib_mem_destroy_mspace (clib_mem_get_heap ()); + destroy_mspace (heap->mspace); clib_mem_vm_unmap (base); } @@ -399,47 +416,61 @@ format_mheap_trace (u8 * s, va_list * va) return s; } - u8 * format_clib_mem_heap (u8 * s, va_list * va) { - void *heap = va_arg (*va, u8 *); + clib_mem_heap_t *heap = va_arg (*va, clib_mem_heap_t *); int verbose = va_arg (*va, int); struct dlmallinfo mi; mheap_trace_main_t *tm = &mheap_trace_main; + u32 indent = format_get_indent (s) + 2; if (heap == 0) heap = clib_mem_get_heap (); - mi = mspace_mallinfo (heap); + mi = mspace_mallinfo (heap->mspace); + + s = format (s, "base %p, size %U", + heap->base, format_memory_size, heap->size); + +#define _(i,v,str) \ + if (heap->flags & CLIB_MEM_HEAP_F_##v) s = format (s, ", %s", str); + foreach_clib_mem_heap_flag; +#undef _ + + s = format (s, ", name '%s'", heap->name); + + if (heap->log2_page_sz != CLIB_MEM_PAGE_SZ_UNKNOWN) + { + clib_mem_page_stats_t stats; + clib_mem_get_page_stats (heap->base, heap->log2_page_sz, + heap->size >> heap->log2_page_sz, &stats); + s = format (s, "\n%U%U", format_white_space, indent, + format_clib_mem_page_stats, &stats); + } - s = format (s, "total: %U, used: %U, free: %U, trimmable: %U", + s = format (s, "\n%Utotal: %U, used: %U, free: %U, trimmable: %U", + format_white_space, indent, format_msize, mi.arena, format_msize, mi.uordblks, format_msize, mi.fordblks, format_msize, mi.keepcost); if (verbose > 0) { - s = format (s, "\n free chunks %llu free fastbin blks %llu", - mi.ordblks, mi.smblks); - s = - format (s, "\n max total allocated %U", format_msize, mi.usmblks); + s = format (s, "\n%Ufree chunks %llu free fastbin blks %llu", + format_white_space, indent + 2, mi.ordblks, mi.smblks); + s = format (s, "\n%Umax total allocated %U", + format_white_space, indent + 2, format_msize, mi.usmblks); } - if (mspace_is_traced (heap)) + if (mspace_is_traced (heap->mspace)) s = format (s, "\n%U", format_mheap_trace, tm, verbose); return s; } void -clib_mem_usage (clib_mem_usage_t * u) +clib_mem_get_heap_usage (clib_mem_heap_t * heap, clib_mem_usage_t * usage) { - clib_warning ("unimp"); -} - -void -clib_mem_get_heap_usage (void *heap, clib_mem_usage_t * usage) -{ - struct dlmallinfo mi = mspace_mallinfo (heap); + struct dlmallinfo mi = mspace_mallinfo (heap->mspace); /* TODO: Fill in some more values */ usage->object_count = 0; @@ -455,15 +486,9 @@ clib_mem_get_heap_usage (void *heap, clib_mem_usage_t * usage) uword clib_mem_validate_serial = 0; void -clib_mem_validate (void) -{ - clib_warning ("unimp"); -} - -void -mheap_trace (void *v, int enable) +mheap_trace (clib_mem_heap_t * h, int enable) { - (void) mspace_enable_disable_trace (v, enable); + (void) mspace_enable_disable_trace (h->mspace, enable); if (enable == 0) mheap_trace_main_free (&mheap_trace_main); @@ -487,7 +512,8 @@ clib_mem_trace (int enable) int clib_mem_is_traced (void) { - return mspace_is_traced (clib_mem_get_heap ()); + clib_mem_heap_t *h = clib_mem_get_heap (); + return mspace_is_traced (h->mspace); } uword @@ -501,45 +527,66 @@ clib_mem_trace_enable_disable (uword enable) return rv; } -void * +clib_mem_heap_t * clib_mem_create_heap (void *base, uword size, int is_locked, char *fmt, ...) { - void *rv; - if (base == 0) - rv = create_mspace (size, is_locked); + clib_mem_page_sz_t log2_page_sz = clib_mem_get_log2_page_size (); + clib_mem_heap_t *h; + char *name; + u8 *s = 0; + + if (fmt == 0) + { + name = ""; + } + else if (strchr (fmt, '%')) + { + va_list va; + va_start (va, fmt); + s = va_format (0, fmt, &va); + vec_add1 (s, 0); + va_end (va); + name = (char *) s; + } else - rv = create_mspace_with_base (base, size, is_locked); + name = fmt; - if (rv) - mspace_disable_expand (rv); - return rv; + h = clib_mem_create_heap_internal (base, size, log2_page_sz, is_locked, + name); + vec_free (s); + return h; } void -clib_mem_destroy_heap (void *heap) +clib_mem_destroy_heap (clib_mem_heap_t * h) { - destroy_mspace (heap); + mheap_trace_main_t *tm = &mheap_trace_main; + + if (tm->enabled && h->mspace == tm->current_traced_mheap) + tm->enabled = 0; + + destroy_mspace (h->mspace); + if (h->flags & CLIB_MEM_HEAP_F_UNMAP_ON_DESTROY) + clib_mem_vm_unmap (h->base); } uword -clib_mem_get_heap_free_space (void *heap) +clib_mem_get_heap_free_space (clib_mem_heap_t * h) { - struct dlmallinfo dlminfo = mspace_mallinfo (heap); + struct dlmallinfo dlminfo = mspace_mallinfo (h->mspace); return dlminfo.fordblks; } void * -clib_mem_get_heap_base (void *heap) +clib_mem_get_heap_base (clib_mem_heap_t * h) { - return mspace_least_addr (heap); + return h->base; } uword -clib_mem_get_heap_size (void *heap) +clib_mem_get_heap_size (clib_mem_heap_t * heap) { - struct dlmallinfo mi; - mi = mspace_mallinfo (heap); - return mi.arena; + return heap->size; } /* diff --git a/src/vppinfra/unix-formats.c b/src/vppinfra/unix-formats.c index 7059686555b..4e29bb86ddb 100644 --- a/src/vppinfra/unix-formats.c +++ b/src/vppinfra/unix-formats.c @@ -964,66 +964,4 @@ unformat_unix_gid (unformat_input_t * input, va_list * args) return 0; } -#define MAX_NUMNODES 16 -u8 * -format_page_map (u8 * s, va_list * args) -{ - uword va = va_arg (*args, uword); - uword size = va_arg (*args, uword); - uword page_size = clib_mem_get_page_size (); - u32 indent = format_get_indent (s); - uword n_pages = size / page_size; - uword pages_per_numa[MAX_NUMNODES] = { 0 }; - uword pages_not_mapped = 0; - uword pages_unknown = 0; - int *status = 0; - void **ptr = 0; - int i; - - s = format (s, "virtual memory start 0x%llx, size %lluk, %u pages, " - "page size %uk", va, size / 1024, n_pages, page_size / 1024); - - vec_validate (status, n_pages - 1); - vec_validate (ptr, n_pages - 1); - - for (i = 0; i < n_pages; i++) - ptr[i] = uword_to_pointer (va + i * page_size, void *); - - if (move_pages (0, n_pages, ptr, 0, status, 0) != 0) - { - s = format (s, "\n%Upage information not available (errno %u)", - format_white_space, indent + 2, errno); - goto done; - } - - for (i = 0; i < n_pages; i++) - { - if (status[i] >= 0 && status[i] < MAX_NUMNODES) - pages_per_numa[status[i]]++; - else if (status[i] == -EFAULT) - pages_not_mapped++; - else - pages_unknown++; - } - - for (i = 0; i < MAX_NUMNODES; i++) - if (pages_per_numa[i]) - s = format (s, "\n%Unuma %u: %d pages, %luk", format_white_space, - indent + 2, i, pages_per_numa[i], pages_per_numa[i] * - page_size / 1024); - - s = format (s, "\n%Unot mapped: %u pages, %luk", format_white_space, - indent + 2, pages_not_mapped, pages_not_mapped * - page_size / 1024); - - if (pages_unknown) - s = format (s, "\n%Uunknown: %u pages, %luk", format_white_space, - indent + 2, pages_unknown, pages_unknown * page_size / 1024); - -done: - vec_free (status); - vec_free (ptr); - return s; -} - #endif /* __KERNEL__ */ |