vppinfra: vector allocator rework

- support of in-place growth of vectors (if there is available space next to
  existing alloc)
- drops the need for alloc_aligned_at_offset from memory allocator,
  which allows easier swap to different memory allocator and reduces
  malloc overhead
- rework of pool and vec macros to inline functions to improve debuggability
- fix alignment - in many cases macros were not using native alignment
  of the particular datatype. Explicitly setting alignment with XXX_aligned()
  versions of the macro is not needed anymore in > 99% of cases
- fix ASAN usage
- avoid use of vector of voids, this was root cause of several bugs
  found in vec_* and pool_* function where sizeof() was used on voids
  instead of real vector data type
- introduce minimal alignment which is currently 8 bytes, vectors will
  be always aligned at least to that value (underlay allocator actually always
  provide 16-byte aligned allocs)

Type: improvement
Change-Id: I20f4b081bb13bbf7bc0ace85cc4e301787f12fdf
Signed-off-by: Damjan Marion <damarion@cisco.com>

1 files changed, 16 insertions, 128 deletions

diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h
index dfe8de93626..1a813be68b6 100644
--- a/src/vppinfra/mem.h
+++ b/src/vppinfra/mem.h
@@ -53,6 +53,8 @@
 #define CLIB_MAX_NUMAS 16
 #define CLIB_MEM_VM_MAP_FAILED ((void *) ~0)
 #define CLIB_MEM_ERROR (-1)
+#define CLIB_MEM_LOG2_MIN_ALIGN (3)
+#define CLIB_MEM_MIN_ALIGN	(1 << CLIB_MEM_LOG2_MIN_ALIGN)
 
 typedef enum
 {
@@ -93,9 +95,10 @@ typedef struct _clib_mem_vm_map_hdr
   struct _clib_mem_vm_map_hdr *prev, *next;
 } clib_mem_vm_map_hdr_t;
 
-#define foreach_clib_mem_heap_flag \
-  _(0, LOCKED, "locked") \
-  _(1, UNMAP_ON_DESTROY, "unmap-on-destroy")
+#define foreach_clib_mem_heap_flag                                            \
+  _ (0, LOCKED, "locked")                                                     \
+  _ (1, UNMAP_ON_DESTROY, "unmap-on-destroy")                                 \
+  _ (2, TRACED, "traced")
 
 typedef enum
 {
@@ -213,77 +216,13 @@ clib_mem_set_thread_index (void)
   ASSERT (__os_thread_index > 0);
 }
 
-always_inline uword
-clib_mem_size_nocheck (void *p)
-{
-  size_t mspace_usable_size_with_delta (const void *p);
-  return mspace_usable_size_with_delta (p);
-}
-
-/* Memory allocator which may call os_out_of_memory() if it fails */
-always_inline void *
-clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset,
-				  int os_out_of_memory_on_failure)
-{
-  void *mspace_get_aligned (void *msp, unsigned long n_user_data_bytes,
-			    unsigned long align, unsigned long align_offset);
-  clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
-  void *p;
-
-  if (align_offset > align)
-    {
-      if (align > 0)
-	align_offset %= align;
-      else
-	align_offset = align;
-    }
-
-  p = mspace_get_aligned (h->mspace, size, align, align_offset);
-
-  if (PREDICT_FALSE (0 == p))
-    {
-      if (os_out_of_memory_on_failure)
-	os_out_of_memory ();
-      return 0;
-    }
-
-  CLIB_MEM_UNPOISON (p, size);
-  return p;
-}
-
-/* Memory allocator which calls os_out_of_memory() when it fails */
-always_inline void *
-clib_mem_alloc (uword size)
-{
-  return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
-					   /* align_offset */ 0,
-					   /* os_out_of_memory */ 1);
-}
-
-always_inline void *
-clib_mem_alloc_aligned (uword size, uword align)
-{
-  return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
-					   /* os_out_of_memory */ 1);
-}
-
 /* Memory allocator which calls os_out_of_memory() when it fails */
-always_inline void *
-clib_mem_alloc_or_null (uword size)
-{
-  return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
-					   /* align_offset */ 0,
-					   /* os_out_of_memory */ 0);
-}
-
-always_inline void *
-clib_mem_alloc_aligned_or_null (uword size, uword align)
-{
-  return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
-					   /* os_out_of_memory */ 0);
-}
-
-
+void *clib_mem_alloc (uword size);
+void *clib_mem_alloc_aligned (uword size, uword align);
+void *clib_mem_alloc_or_null (uword size);
+void *clib_mem_alloc_aligned_or_null (uword size, uword align);
+void *clib_mem_realloc (void *p, uword new_size);
+void *clib_mem_realloc_aligned (void *p, uword new_size, uword align);
 
 /* Memory allocator which panics when it fails.
    Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */
@@ -302,61 +241,10 @@ clib_mem_alloc_aligned_or_null (uword size, uword align)
 /* Alias to stack allocator for naming consistency. */
 #define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes)
 
-always_inline uword
-clib_mem_is_heap_object (void *p)
-{
-  int mspace_is_heap_object (void *msp, void *p);
-  clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
-  return mspace_is_heap_object (h->mspace, p);
-}
-
-always_inline void
-clib_mem_free (void *p)
-{
-  void mspace_put (void *msp, void *p_arg);
-  clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
-
-  /* Make sure object is in the correct heap. */
-  ASSERT (clib_mem_is_heap_object (p));
-
-  CLIB_MEM_POISON (p, clib_mem_size_nocheck (p));
-
-  mspace_put (h->mspace, p);
-}
-
-always_inline void *
-clib_mem_realloc (void *p, uword new_size, uword old_size)
-{
-  /* By default use alloc, copy and free to emulate realloc. */
-  void *q = clib_mem_alloc (new_size);
-  if (q)
-    {
-      uword copy_size;
-      if (old_size < new_size)
-	copy_size = old_size;
-      else
-	copy_size = new_size;
-      clib_memcpy_fast (q, p, copy_size);
-      clib_mem_free (p);
-    }
-  return q;
-}
-
-always_inline uword
-clib_mem_size (void *p)
-{
-  ASSERT (clib_mem_is_heap_object (p));
-  return clib_mem_size_nocheck (p);
-}
-
-always_inline void
-clib_mem_free_s (void *p)
-{
-  uword size = clib_mem_size (p);
-  CLIB_MEM_UNPOISON (p, size);
-  memset_s_inline (p, size, 0, size);
-  clib_mem_free (p);
-}
+uword clib_mem_is_heap_object (void *p);
+void clib_mem_free (void *p);
+uword clib_mem_size (void *p);
+void clib_mem_free_s (void *p);
 
 always_inline clib_mem_heap_t *
 clib_mem_get_heap (void)