diff options
| -rw-r--r-- | src/vppinfra/clib.h | 1 | ||||
| -rw-r--r-- | src/vppinfra/mem.h | 21 | ||||
| -rw-r--r-- | src/vppinfra/mem_dlmalloc.c | 132 | ||||
| -rw-r--r-- | src/vppinfra/pool.h | 13 | ||||
| -rw-r--r-- | src/vppinfra/vec.c | 28 | ||||
| -rw-r--r-- | src/vppinfra/vec.h | 89 | ||||
| -rw-r--r-- | src/vppinfra/vec_bootstrap.h | 4 |
7 files changed, 213 insertions, 75 deletions
diff --git a/src/vppinfra/clib.h b/src/vppinfra/clib.h index 76dd0ebf2ad..8d1674945dd 100644 --- a/src/vppinfra/clib.h +++ b/src/vppinfra/clib.h @@ -98,6 +98,7 @@ #define __clib_unused __attribute__ ((unused)) #define __clib_weak __attribute__ ((weak)) #define __clib_packed __attribute__ ((packed)) +#define __clib_flatten __attribute__ ((flatten)) #define __clib_constructor __attribute__ ((constructor)) #define __clib_noinline __attribute__ ((noinline)) #ifdef __clang__ diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index 1a813be68b6..e33ab37e877 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -223,6 +223,22 @@ void *clib_mem_alloc_or_null (uword size); void *clib_mem_alloc_aligned_or_null (uword size, uword align); void *clib_mem_realloc (void *p, uword new_size); void *clib_mem_realloc_aligned (void *p, uword new_size, uword align); +uword clib_mem_is_heap_object (void *p); +void clib_mem_free (void *p); + +void *clib_mem_heap_alloc (void *heap, uword size); +void *clib_mem_heap_alloc_aligned (void *heap, uword size, uword align); +void *clib_mem_heap_alloc_or_null (void *heap, uword size); +void *clib_mem_heap_alloc_aligned_or_null (void *heap, uword size, + uword align); +void *clib_mem_heap_realloc (void *heap, void *p, uword new_size); +void *clib_mem_heap_realloc_aligned (void *heap, void *p, uword new_size, + uword align); +uword clib_mem_heap_is_heap_object (void *heap, void *p); +void clib_mem_heap_free (void *heap, void *p); + +uword clib_mem_size (void *p); +void clib_mem_free_s (void *p); /* Memory allocator which panics when it fails. Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */ @@ -241,11 +257,6 @@ void *clib_mem_realloc_aligned (void *p, uword new_size, uword align); /* Alias to stack allocator for naming consistency. */ #define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes) -uword clib_mem_is_heap_object (void *p); -void clib_mem_free (void *p); -uword clib_mem_size (void *p); -void clib_mem_free_s (void *p); - always_inline clib_mem_heap_t * clib_mem_get_heap (void) { diff --git a/src/vppinfra/mem_dlmalloc.c b/src/vppinfra/mem_dlmalloc.c index 4d6d11f3489..3829e7c7740 100644 --- a/src/vppinfra/mem_dlmalloc.c +++ b/src/vppinfra/mem_dlmalloc.c @@ -469,8 +469,8 @@ format_clib_mem_heap (u8 * s, va_list * va) return s; } -__clib_export void -clib_mem_get_heap_usage (clib_mem_heap_t * heap, clib_mem_usage_t * usage) +__clib_export __clib_flatten void +clib_mem_get_heap_usage (clib_mem_heap_t *heap, clib_mem_usage_t *usage) { struct dlmallinfo mi = mspace_mallinfo (heap->mspace); @@ -578,31 +578,31 @@ clib_mem_destroy_heap (clib_mem_heap_t * h) clib_mem_vm_unmap (h->base); } -__clib_export uword -clib_mem_get_heap_free_space (clib_mem_heap_t * h) +__clib_export __clib_flatten uword +clib_mem_get_heap_free_space (clib_mem_heap_t *h) { struct dlmallinfo dlminfo = mspace_mallinfo (h->mspace); return dlminfo.fordblks; } -__clib_export void * -clib_mem_get_heap_base (clib_mem_heap_t * h) +__clib_export __clib_flatten void * +clib_mem_get_heap_base (clib_mem_heap_t *h) { return h->base; } -__clib_export uword -clib_mem_get_heap_size (clib_mem_heap_t * heap) +__clib_export __clib_flatten uword +clib_mem_get_heap_size (clib_mem_heap_t *heap) { return heap->size; } /* Memory allocator which may call os_out_of_memory() if it fails */ -static void * -clib_mem_alloc_inline (uword size, uword align, - int os_out_of_memory_on_failure) +static inline void * +clib_mem_heap_alloc_inline (void *heap, uword size, uword align, + int os_out_of_memory_on_failure) { - clib_mem_heap_t *h = clib_mem_get_per_cpu_heap (); + clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap (); void *p; align = clib_max (CLIB_MEM_MIN_ALIGN, align); @@ -624,40 +624,69 @@ clib_mem_alloc_inline (uword size, uword align, } /* Memory allocator which calls os_out_of_memory() when it fails */ -__clib_export void * +__clib_export __clib_flatten void * clib_mem_alloc (uword size) { - return clib_mem_alloc_inline (size, CLIB_MEM_MIN_ALIGN, - /* os_out_of_memory */ 1); + return clib_mem_heap_alloc_inline (0, size, CLIB_MEM_MIN_ALIGN, + /* os_out_of_memory */ 1); } -__clib_export void * +__clib_export __clib_flatten void * clib_mem_alloc_aligned (uword size, uword align) { - return clib_mem_alloc_inline (size, align, - /* os_out_of_memory */ 1); + return clib_mem_heap_alloc_inline (0, size, align, + /* os_out_of_memory */ 1); } /* Memory allocator which calls os_out_of_memory() when it fails */ -__clib_export void * +__clib_export __clib_flatten void * clib_mem_alloc_or_null (uword size) { - return clib_mem_alloc_inline (size, CLIB_MEM_MIN_ALIGN, - /* os_out_of_memory */ 0); + return clib_mem_heap_alloc_inline (0, size, CLIB_MEM_MIN_ALIGN, + /* os_out_of_memory */ 0); } -__clib_export void * +__clib_export __clib_flatten void * clib_mem_alloc_aligned_or_null (uword size, uword align) { - return clib_mem_alloc_inline (size, align, - /* os_out_of_memory */ 0); + return clib_mem_heap_alloc_inline (0, size, align, + /* os_out_of_memory */ 0); } -__clib_export void * -clib_mem_realloc_aligned (void *p, uword new_size, uword align) +__clib_export __clib_flatten void * +clib_mem_heap_alloc (void *heap, uword size) +{ + return clib_mem_heap_alloc_inline (heap, size, CLIB_MEM_MIN_ALIGN, + /* os_out_of_memory */ 1); +} + +__clib_export __clib_flatten void * +clib_mem_heap_alloc_aligned (void *heap, uword size, uword align) +{ + return clib_mem_heap_alloc_inline (heap, size, align, + /* os_out_of_memory */ 1); +} + +__clib_export __clib_flatten void * +clib_mem_heap_alloc_or_null (void *heap, uword size) +{ + return clib_mem_heap_alloc_inline (heap, size, CLIB_MEM_MIN_ALIGN, + /* os_out_of_memory */ 0); +} + +__clib_export __clib_flatten void * +clib_mem_heap_alloc_aligned_or_null (void *heap, uword size, uword align) +{ + return clib_mem_heap_alloc_inline (heap, size, align, + /* os_out_of_memory */ 0); +} + +__clib_export __clib_flatten void * +clib_mem_heap_realloc_aligned (void *heap, void *p, uword new_size, + uword align) { uword old_alloc_size; - clib_mem_heap_t *h = clib_mem_get_per_cpu_heap (); + clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap (); void *new; ASSERT (count_set_bits (align) == 1); @@ -674,14 +703,14 @@ clib_mem_realloc_aligned (void *p, uword new_size, uword align) } else { - new = clib_mem_alloc_inline (new_size, align, 1); + new = clib_mem_heap_alloc_inline (h, new_size, align, 1); CLIB_MEM_UNPOISON (new, new_size); if (old_alloc_size) { CLIB_MEM_UNPOISON (p, old_alloc_size); clib_memcpy_fast (new, p, clib_min (new_size, old_alloc_size)); - clib_mem_free (p); + clib_mem_heap_free (h, p); } p = new; } @@ -689,28 +718,45 @@ clib_mem_realloc_aligned (void *p, uword new_size, uword align) return p; } -__clib_export void * +__clib_export __clib_flatten void * +clib_mem_heap_realloc (void *heap, void *p, uword new_size) +{ + return clib_mem_heap_realloc_aligned (heap, p, new_size, CLIB_MEM_MIN_ALIGN); +} + +__clib_export __clib_flatten void * +clib_mem_realloc_aligned (void *p, uword new_size, uword align) +{ + return clib_mem_heap_realloc_aligned (0, p, new_size, align); +} + +__clib_export __clib_flatten void * clib_mem_realloc (void *p, uword new_size) { - return clib_mem_realloc_aligned (p, new_size, CLIB_MEM_MIN_ALIGN); + return clib_mem_heap_realloc_aligned (0, p, new_size, CLIB_MEM_MIN_ALIGN); } -__clib_export uword -clib_mem_is_heap_object (void *p) +__clib_export __clib_flatten uword +clib_mem_heap_is_heap_object (void *heap, void *p) { - int mspace_is_heap_object (void *msp, void *p); - clib_mem_heap_t *h = clib_mem_get_per_cpu_heap (); + clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap (); return mspace_is_heap_object (h->mspace, p); } -__clib_export void -clib_mem_free (void *p) +__clib_export __clib_flatten uword +clib_mem_is_heap_object (void *p) +{ + return clib_mem_heap_is_heap_object (0, p); +} + +__clib_export __clib_flatten void +clib_mem_heap_free (void *heap, void *p) { - clib_mem_heap_t *h = clib_mem_get_per_cpu_heap (); + clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap (); uword size = clib_mem_size (p); /* Make sure object is in the correct heap. */ - ASSERT (clib_mem_is_heap_object (p)); + ASSERT (clib_mem_heap_is_heap_object (h, p)); if (PREDICT_FALSE (h->flags & CLIB_MEM_HEAP_F_TRACED)) mheap_put_trace (pointer_to_uword (p), size); @@ -719,7 +765,13 @@ clib_mem_free (void *p) mspace_free (h->mspace, p); } -__clib_export uword +__clib_export __clib_flatten void +clib_mem_free (void *p) +{ + clib_mem_heap_free (0, p); +} + +__clib_export __clib_flatten uword clib_mem_size (void *p) { return mspace_usable_size (p); diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h index 32360daeffc..4db1f7b6bdf 100644 --- a/src/vppinfra/pool.h +++ b/src/vppinfra/pool.h @@ -308,7 +308,7 @@ _pool_put_index (void *p, uword index, uword elt_sz) /** Allocate N more free elements to pool (general version). */ static_always_inline void -_pool_alloc (void **pp, uword n_elts, uword align, uword elt_sz) +_pool_alloc (void **pp, uword n_elts, uword align, void *heap, uword elt_sz) { pool_header_t *ph = pool_header (pp[0]); uword len = vec_len (pp[0]); @@ -320,7 +320,7 @@ _pool_alloc (void **pp, uword n_elts, uword align, uword elt_sz) } pp[0] = _vec_realloc_inline (pp[0], len + n_elts, elt_sz, - sizeof (pool_header_t), align, 0); + sizeof (pool_header_t), align, heap); _vec_len (pp[0]) = len; CLIB_MEM_POISON (pp[0] + len * elt_sz, n_elts * elt_sz); @@ -330,11 +330,12 @@ _pool_alloc (void **pp, uword n_elts, uword align, uword elt_sz) clib_bitmap_vec_validate (ph->free_bitmap, len + n_elts - 1); } -#define pool_alloc_aligned(P, N, A) \ - _pool_alloc ((void **) &(P), N, _vec_align (P, A), _vec_elt_sz (P)) +#define pool_alloc_aligned_heap(P, N, A, H) \ + _pool_alloc ((void **) &(P), N, _vec_align (P, A), H, _vec_elt_sz (P)) -/** Allocate N more free elements to pool (unspecified alignment). */ -#define pool_alloc(P,N) pool_alloc_aligned(P,N,0) +#define pool_alloc_heap(P, N, H) pool_alloc_aligned_heap (P, N, 0, H) +#define pool_alloc_aligned(P, N, A) pool_alloc_aligned_heap (P, N, A, 0) +#define pool_alloc(P, N) pool_alloc_aligned_heap (P, N, 0, 0) static_always_inline void * _pool_dup (void *p, uword align, uword elt_sz) diff --git a/src/vppinfra/vec.c b/src/vppinfra/vec.c index 2fbab2f7e45..805fdc02b7e 100644 --- a/src/vppinfra/vec.c +++ b/src/vppinfra/vec.c @@ -19,26 +19,22 @@ __clib_export void * _vec_realloc (void *v, uword n_elts, uword elt_sz, uword hdr_sz, uword align, void *heap) { - uword n_data_bytes, data_offset, new_data_size, alloc_size; + uword n_data_bytes, alloc_size, new_data_size; void *p; /* alignment must be power of 2 */ align = clib_max (align, VEC_MIN_ALIGN); ASSERT (count_set_bits (align) == 1); - /* number of bytes needed to store both vector header and optional user - * header */ - data_offset = round_pow2 (hdr_sz + sizeof (vec_header_t), align); - /* mumber of bytes needed to store vector data */ n_data_bytes = n_elts * elt_sz; - /* minimal allocation needed to store data and headers */ - new_data_size = data_offset + n_data_bytes; - if (v) { + uword data_offset = vec_get_header_size (v); uword old_data_size = data_offset + _vec_len (v) * elt_sz; + new_data_size = data_offset + n_data_bytes; + heap = _vec_find (v)->default_heap ? 0 : _vec_heap (v); p = vec_header (v); alloc_size = clib_mem_size (p); @@ -54,7 +50,7 @@ _vec_realloc (void *v, uword n_elts, uword elt_sz, uword hdr_sz, uword align, else alloc_size = (n_data_bytes * 3) / 2 + data_offset; - p = clib_mem_realloc_aligned (p, alloc_size, align); + p = clib_mem_heap_realloc_aligned (heap, p, alloc_size, align); alloc_size = clib_mem_size (p); v = p + data_offset; } @@ -65,13 +61,25 @@ _vec_realloc (void *v, uword n_elts, uword elt_sz, uword hdr_sz, uword align, else { /* new allocation */ - p = clib_mem_alloc_aligned (new_data_size, align); + uword data_offset = hdr_sz + sizeof (vec_header_t); + data_offset += heap ? sizeof (void *) : 0; + data_offset = round_pow2 (data_offset, align); + + new_data_size = data_offset + n_data_bytes; + p = clib_mem_heap_alloc_aligned (heap, new_data_size, align); alloc_size = clib_mem_size (p); CLIB_MEM_UNPOISON (p, alloc_size); clib_memset_u8 (p, 0, alloc_size); v = p + data_offset; _vec_find (v)->hdr_size = data_offset / VEC_MIN_ALIGN; _vec_find (v)->log2_align = min_log2 (align); + if (heap) + { + _vec_find (v)->default_heap = 0; + _vec_heap (v) = heap; + } + else + _vec_find (v)->default_heap = 1; } CLIB_MEM_POISON (p + new_data_size, alloc_size - new_data_size); diff --git a/src/vppinfra/vec.h b/src/vppinfra/vec.h index e90c27c1bc1..1dc300ad1ea 100644 --- a/src/vppinfra/vec.h +++ b/src/vppinfra/vec.h @@ -54,6 +54,7 @@ ~~~~~~~~ user header (start of memory allocation) padding + heap pointer (optional, only if default_heap == 0) vector header: number of elements, header size user's pointer-> vector element #0 vector element #1 @@ -130,14 +131,22 @@ _vec_update_pointer (void **vp, void *v) vp[0] = v; } -always_inline void * +static_always_inline void * +vec_get_heap (void *v) +{ + if (v == 0 || _vec_find (v)->default_heap == 1) + return 0; + return _vec_heap (v); +} + +static_always_inline void * _vec_realloc_inline (void *v, uword n_elts, uword elt_sz, uword hdr_sz, uword align, void *heap) { if (PREDICT_TRUE (v != 0)) { /* Vector header must start heap object. */ - ASSERT (clib_mem_is_heap_object (vec_header (v))); + ASSERT (clib_mem_heap_is_heap_object (vec_get_heap (v), vec_header (v))); /* Typically we'll not need to resize. */ if ((n_elts * elt_sz) <= vec_max_bytes (v)) @@ -151,6 +160,49 @@ _vec_realloc_inline (void *v, uword n_elts, uword elt_sz, uword hdr_sz, return _vec_realloc (v, n_elts, elt_sz, hdr_sz, align, heap); } +static_always_inline void +_vec_prealloc (void **vp, uword n_elts, uword hdr_sz, uword align, void *heap, + uword elt_sz) +{ + void *v; + + ASSERT (vp[0] == 0); + + v = _vec_realloc (0, n_elts, elt_sz, hdr_sz, align, heap); + _vec_set_len (v, 0, elt_sz); + _vec_update_pointer (vp, v); +} + +/** \brief Pre-allocate a vector (generic version) + + @param V pointer to a vector + @param N number of elements to pre-allocate + @param H header size in bytes (may be zero) + @param A alignment (zero means default alignment of the data structure) + @param P heap (zero means default heap) + @return V (value-result macro parameter) +*/ + +#define vec_prealloc_hap(V, N, H, A, P) \ + _vec_prealloc ((void **) &(V), N, H, _vec_align (V, A), P, _vec_elt_sz (V)) + +/** \brief Pre-allocate a vector (simple version) + + @param V pointer to a vector + @param N number of elements to pre-allocate + @return V (value-result macro parameter) +*/ +#define vec_prealloc(V, N) vec_prealloc_hap (V, N, 0, 0, 0) + +/** \brief Pre-allocate a vector (heap version) + + @param V pointer to a vector + @param N number of elements to pre-allocate + @param P heap (zero means default heap) + @return V (value-result macro parameter) +*/ +#define vec_prealloc_heap(V, N, P) vec_prealloc_hap (V, N, 0, 0, P) + always_inline int _vec_resize_will_expand (void *v, uword n_elts, uword elt_sz) { @@ -158,7 +210,7 @@ _vec_resize_will_expand (void *v, uword n_elts, uword elt_sz) return 1; /* Vector header must start heap object. */ - ASSERT (clib_mem_is_heap_object (vec_header (v))); + ASSERT (clib_mem_heap_is_heap_object (vec_get_heap (v), vec_header (v))); n_elts += _vec_len (v); if ((n_elts * elt_sz) <= vec_max_bytes (v)) @@ -398,22 +450,22 @@ _vec_zero_elts (void *v, uword first, uword count, uword elt_sz) } #define vec_zero_elts(V, F, C) _vec_zero_elts (V, F, C, sizeof ((V)[0])) -static_always_inline void * -_vec_validate_ha (void *v, uword index, uword header_size, uword align, - uword elt_sz) +static_always_inline void +_vec_validate (void **vp, uword index, uword header_size, uword align, + void *heap, uword elt_sz) { + void *v = vp[0]; uword vl = vec_len (v); if (index >= vl) { - v = _vec_realloc_inline (v, index + 1, elt_sz, header_size, align, 0); + v = _vec_realloc_inline (v, index + 1, elt_sz, header_size, align, heap); _vec_zero_elts (v, vl, index - vl + 1, elt_sz); + _vec_update_pointer (vp, v); } - return v; } -#define vec_validate_ha(V, I, H, A) \ - (V) = \ - _vec_validate_ha ((void *) (V), I, H, _vec_align (V, A), sizeof ((V)[0])) +#define vec_validate_hap(V, I, H, A, P) \ + _vec_validate ((void **) &(V), I, H, _vec_align (V, A), 0, sizeof ((V)[0])) /** \brief Make sure vector is long enough for given index (no header, unspecified alignment) @@ -422,7 +474,7 @@ _vec_validate_ha (void *v, uword index, uword header_size, uword align, @param I vector index which will be valid upon return @return V (value-result macro parameter) */ -#define vec_validate(V,I) vec_validate_ha(V,I,0,0) +#define vec_validate(V, I) vec_validate_hap (V, I, 0, 0, 0) /** \brief Make sure vector is long enough for given index (no header, specified alignment) @@ -433,7 +485,18 @@ _vec_validate_ha (void *v, uword index, uword header_size, uword align, @return V (value-result macro parameter) */ -#define vec_validate_aligned(V,I,A) vec_validate_ha(V,I,0,A) +#define vec_validate_aligned(V, I, A) vec_validate_hap (V, I, 0, A, 0) + +/** \brief Make sure vector is long enough for given index + (no header, specified heap) + + @param V (possibly NULL) pointer to a vector. + @param I vector index which will be valid upon return + @param H heap (may be zero) + @return V (value-result macro parameter) +*/ + +#define vec_validate_heap(V, I, P) vec_validate_hap (V, I, 0, 0, P) /** \brief Make sure vector is long enough for given index and initialize empty space (general version) diff --git a/src/vppinfra/vec_bootstrap.h b/src/vppinfra/vec_bootstrap.h index 304ea2dee1a..d6451f3d172 100644 --- a/src/vppinfra/vec_bootstrap.h +++ b/src/vppinfra/vec_bootstrap.h @@ -56,7 +56,8 @@ typedef struct { u32 len; /**< Number of elements in vector (NOT its allocated length). */ u8 hdr_size; /**< header size divided by VEC_MIN_ALIGN */ - u8 log2_align; /**< data alignment */ + u8 log2_align : 7; /**< data alignment */ + u8 default_heap : 1; /**< vector uses default heap */ u8 vpad[2]; /**< pad to 8 bytes */ u8 vector_data[0]; /**< Vector data . */ } vec_header_t; @@ -72,6 +73,7 @@ typedef struct @return pointer to the vector's vector_header_t */ #define _vec_find(v) ((vec_header_t *) (v) - 1) +#define _vec_heap(v) (((void **) (_vec_find (v)))[-1]) always_inline uword __vec_align (uword data_align, uword configuered_align); always_inline uword __vec_elt_sz (uword elt_sz, int is_void); |