summaryrefslogtreecommitdiffstats
path: root/src/vppinfra/mem.h
diff options
context:
space:
mode:
authorDamjan Marion <damarion@cisco.com>2022-03-19 00:07:52 +0100
committerFlorin Coras <florin.coras@gmail.com>2022-03-30 18:27:13 +0000
commit299571aca34d36e637e43cfbba6275662d0d7795 (patch)
treea48be21950d082afb7dd93562f76f0ba554e8919 /src/vppinfra/mem.h
parent9539647b895c456ca53892a9259e3127c6b92d35 (diff)
vppinfra: vector allocator rework
- support of in-place growth of vectors (if there is available space next to existing alloc) - drops the need for alloc_aligned_at_offset from memory allocator, which allows easier swap to different memory allocator and reduces malloc overhead - rework of pool and vec macros to inline functions to improve debuggability - fix alignment - in many cases macros were not using native alignment of the particular datatype. Explicitly setting alignment with XXX_aligned() versions of the macro is not needed anymore in > 99% of cases - fix ASAN usage - avoid use of vector of voids, this was root cause of several bugs found in vec_* and pool_* function where sizeof() was used on voids instead of real vector data type - introduce minimal alignment which is currently 8 bytes, vectors will be always aligned at least to that value (underlay allocator actually always provide 16-byte aligned allocs) Type: improvement Change-Id: I20f4b081bb13bbf7bc0ace85cc4e301787f12fdf Signed-off-by: Damjan Marion <damarion@cisco.com>
Diffstat (limited to 'src/vppinfra/mem.h')
-rw-r--r--src/vppinfra/mem.h144
1 files changed, 16 insertions, 128 deletions
diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h
index dfe8de93626..1a813be68b6 100644
--- a/src/vppinfra/mem.h
+++ b/src/vppinfra/mem.h
@@ -53,6 +53,8 @@
#define CLIB_MAX_NUMAS 16
#define CLIB_MEM_VM_MAP_FAILED ((void *) ~0)
#define CLIB_MEM_ERROR (-1)
+#define CLIB_MEM_LOG2_MIN_ALIGN (3)
+#define CLIB_MEM_MIN_ALIGN (1 << CLIB_MEM_LOG2_MIN_ALIGN)
typedef enum
{
@@ -93,9 +95,10 @@ typedef struct _clib_mem_vm_map_hdr
struct _clib_mem_vm_map_hdr *prev, *next;
} clib_mem_vm_map_hdr_t;
-#define foreach_clib_mem_heap_flag \
- _(0, LOCKED, "locked") \
- _(1, UNMAP_ON_DESTROY, "unmap-on-destroy")
+#define foreach_clib_mem_heap_flag \
+ _ (0, LOCKED, "locked") \
+ _ (1, UNMAP_ON_DESTROY, "unmap-on-destroy") \
+ _ (2, TRACED, "traced")
typedef enum
{
@@ -213,77 +216,13 @@ clib_mem_set_thread_index (void)
ASSERT (__os_thread_index > 0);
}
-always_inline uword
-clib_mem_size_nocheck (void *p)
-{
- size_t mspace_usable_size_with_delta (const void *p);
- return mspace_usable_size_with_delta (p);
-}
-
-/* Memory allocator which may call os_out_of_memory() if it fails */
-always_inline void *
-clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset,
- int os_out_of_memory_on_failure)
-{
- void *mspace_get_aligned (void *msp, unsigned long n_user_data_bytes,
- unsigned long align, unsigned long align_offset);
- clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
- void *p;
-
- if (align_offset > align)
- {
- if (align > 0)
- align_offset %= align;
- else
- align_offset = align;
- }
-
- p = mspace_get_aligned (h->mspace, size, align, align_offset);
-
- if (PREDICT_FALSE (0 == p))
- {
- if (os_out_of_memory_on_failure)
- os_out_of_memory ();
- return 0;
- }
-
- CLIB_MEM_UNPOISON (p, size);
- return p;
-}
-
-/* Memory allocator which calls os_out_of_memory() when it fails */
-always_inline void *
-clib_mem_alloc (uword size)
-{
- return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
- /* align_offset */ 0,
- /* os_out_of_memory */ 1);
-}
-
-always_inline void *
-clib_mem_alloc_aligned (uword size, uword align)
-{
- return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
- /* os_out_of_memory */ 1);
-}
-
/* Memory allocator which calls os_out_of_memory() when it fails */
-always_inline void *
-clib_mem_alloc_or_null (uword size)
-{
- return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
- /* align_offset */ 0,
- /* os_out_of_memory */ 0);
-}
-
-always_inline void *
-clib_mem_alloc_aligned_or_null (uword size, uword align)
-{
- return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
- /* os_out_of_memory */ 0);
-}
-
-
+void *clib_mem_alloc (uword size);
+void *clib_mem_alloc_aligned (uword size, uword align);
+void *clib_mem_alloc_or_null (uword size);
+void *clib_mem_alloc_aligned_or_null (uword size, uword align);
+void *clib_mem_realloc (void *p, uword new_size);
+void *clib_mem_realloc_aligned (void *p, uword new_size, uword align);
/* Memory allocator which panics when it fails.
Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */
@@ -302,61 +241,10 @@ clib_mem_alloc_aligned_or_null (uword size, uword align)
/* Alias to stack allocator for naming consistency. */
#define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes)
-always_inline uword
-clib_mem_is_heap_object (void *p)
-{
- int mspace_is_heap_object (void *msp, void *p);
- clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
- return mspace_is_heap_object (h->mspace, p);
-}
-
-always_inline void
-clib_mem_free (void *p)
-{
- void mspace_put (void *msp, void *p_arg);
- clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
-
- /* Make sure object is in the correct heap. */
- ASSERT (clib_mem_is_heap_object (p));
-
- CLIB_MEM_POISON (p, clib_mem_size_nocheck (p));
-
- mspace_put (h->mspace, p);
-}
-
-always_inline void *
-clib_mem_realloc (void *p, uword new_size, uword old_size)
-{
- /* By default use alloc, copy and free to emulate realloc. */
- void *q = clib_mem_alloc (new_size);
- if (q)
- {
- uword copy_size;
- if (old_size < new_size)
- copy_size = old_size;
- else
- copy_size = new_size;
- clib_memcpy_fast (q, p, copy_size);
- clib_mem_free (p);
- }
- return q;
-}
-
-always_inline uword
-clib_mem_size (void *p)
-{
- ASSERT (clib_mem_is_heap_object (p));
- return clib_mem_size_nocheck (p);
-}
-
-always_inline void
-clib_mem_free_s (void *p)
-{
- uword size = clib_mem_size (p);
- CLIB_MEM_UNPOISON (p, size);
- memset_s_inline (p, size, 0, size);
- clib_mem_free (p);
-}
+uword clib_mem_is_heap_object (void *p);
+void clib_mem_free (void *p);
+uword clib_mem_size (void *p);
+void clib_mem_free_s (void *p);
always_inline clib_mem_heap_t *
clib_mem_get_heap (void)