vppinfra: vectors with non-default heap

Type: improvement
Change-Id: Ic675ad4edbf27b7230fc2a77f00c90c46d6350c3
Signed-off-by: Damjan Marion <damarion@cisco.com>

7 files changed, 213 insertions, 75 deletions

diff --git a/src/vppinfra/clib.h b/src/vppinfra/clib.h
index 76dd0ebf2ad..8d1674945dd 100644
--- a/src/vppinfra/clib.h
+++ b/src/vppinfra/clib.h
@@ -98,6 +98,7 @@
 #define __clib_unused __attribute__ ((unused))
 #define __clib_weak __attribute__ ((weak))
 #define __clib_packed __attribute__ ((packed))
+#define __clib_flatten	   __attribute__ ((flatten))
 #define __clib_constructor __attribute__ ((constructor))
 #define __clib_noinline __attribute__ ((noinline))
 #ifdef __clang__
diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h
index 1a813be68b6..e33ab37e877 100644
--- a/src/vppinfra/mem.h
+++ b/src/vppinfra/mem.h
@@ -223,6 +223,22 @@ void *clib_mem_alloc_or_null (uword size);
 void *clib_mem_alloc_aligned_or_null (uword size, uword align);
 void *clib_mem_realloc (void *p, uword new_size);
 void *clib_mem_realloc_aligned (void *p, uword new_size, uword align);
+uword clib_mem_is_heap_object (void *p);
+void clib_mem_free (void *p);
+
+void *clib_mem_heap_alloc (void *heap, uword size);
+void *clib_mem_heap_alloc_aligned (void *heap, uword size, uword align);
+void *clib_mem_heap_alloc_or_null (void *heap, uword size);
+void *clib_mem_heap_alloc_aligned_or_null (void *heap, uword size,
+					   uword align);
+void *clib_mem_heap_realloc (void *heap, void *p, uword new_size);
+void *clib_mem_heap_realloc_aligned (void *heap, void *p, uword new_size,
+				     uword align);
+uword clib_mem_heap_is_heap_object (void *heap, void *p);
+void clib_mem_heap_free (void *heap, void *p);
+
+uword clib_mem_size (void *p);
+void clib_mem_free_s (void *p);
 
 /* Memory allocator which panics when it fails.
    Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */
@@ -241,11 +257,6 @@ void *clib_mem_realloc_aligned (void *p, uword new_size, uword align);
 /* Alias to stack allocator for naming consistency. */
 #define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes)
 
-uword clib_mem_is_heap_object (void *p);
-void clib_mem_free (void *p);
-uword clib_mem_size (void *p);
-void clib_mem_free_s (void *p);
-
 always_inline clib_mem_heap_t *
 clib_mem_get_heap (void)
 {
diff --git a/src/vppinfra/mem_dlmalloc.c b/src/vppinfra/mem_dlmalloc.c
index 4d6d11f3489..3829e7c7740 100644
--- a/src/vppinfra/mem_dlmalloc.c
+++ b/src/vppinfra/mem_dlmalloc.c
@@ -469,8 +469,8 @@ format_clib_mem_heap (u8 * s, va_list * va)
   return s;
 }
 
-__clib_export void
-clib_mem_get_heap_usage (clib_mem_heap_t * heap, clib_mem_usage_t * usage)
+__clib_export __clib_flatten void
+clib_mem_get_heap_usage (clib_mem_heap_t *heap, clib_mem_usage_t *usage)
 {
   struct dlmallinfo mi = mspace_mallinfo (heap->mspace);
 
@@ -578,31 +578,31 @@ clib_mem_destroy_heap (clib_mem_heap_t * h)
     clib_mem_vm_unmap (h->base);
 }
 
-__clib_export uword
-clib_mem_get_heap_free_space (clib_mem_heap_t * h)
+__clib_export __clib_flatten uword
+clib_mem_get_heap_free_space (clib_mem_heap_t *h)
 {
   struct dlmallinfo dlminfo = mspace_mallinfo (h->mspace);
   return dlminfo.fordblks;
 }
 
-__clib_export void *
-clib_mem_get_heap_base (clib_mem_heap_t * h)
+__clib_export __clib_flatten void *
+clib_mem_get_heap_base (clib_mem_heap_t *h)
 {
   return h->base;
 }
 
-__clib_export uword
-clib_mem_get_heap_size (clib_mem_heap_t * heap)
+__clib_export __clib_flatten uword
+clib_mem_get_heap_size (clib_mem_heap_t *heap)
 {
   return heap->size;
 }
 
 /* Memory allocator which may call os_out_of_memory() if it fails */
-static void *
-clib_mem_alloc_inline (uword size, uword align,
-		       int os_out_of_memory_on_failure)
+static inline void *
+clib_mem_heap_alloc_inline (void *heap, uword size, uword align,
+			    int os_out_of_memory_on_failure)
 {
-  clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+  clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap ();
   void *p;
 
   align = clib_max (CLIB_MEM_MIN_ALIGN, align);
@@ -624,40 +624,69 @@ clib_mem_alloc_inline (uword size, uword align,
 }
 
 /* Memory allocator which calls os_out_of_memory() when it fails */
-__clib_export void *
+__clib_export __clib_flatten void *
 clib_mem_alloc (uword size)
 {
-  return clib_mem_alloc_inline (size, CLIB_MEM_MIN_ALIGN,
-				/* os_out_of_memory */ 1);
+  return clib_mem_heap_alloc_inline (0, size, CLIB_MEM_MIN_ALIGN,
+				     /* os_out_of_memory */ 1);
 }
 
-__clib_export void *
+__clib_export __clib_flatten void *
 clib_mem_alloc_aligned (uword size, uword align)
 {
-  return clib_mem_alloc_inline (size, align,
-				/* os_out_of_memory */ 1);
+  return clib_mem_heap_alloc_inline (0, size, align,
+				     /* os_out_of_memory */ 1);
 }
 
 /* Memory allocator which calls os_out_of_memory() when it fails */
-__clib_export void *
+__clib_export __clib_flatten void *
 clib_mem_alloc_or_null (uword size)
 {
-  return clib_mem_alloc_inline (size, CLIB_MEM_MIN_ALIGN,
-				/* os_out_of_memory */ 0);
+  return clib_mem_heap_alloc_inline (0, size, CLIB_MEM_MIN_ALIGN,
+				     /* os_out_of_memory */ 0);
 }
 
-__clib_export void *
+__clib_export __clib_flatten void *
 clib_mem_alloc_aligned_or_null (uword size, uword align)
 {
-  return clib_mem_alloc_inline (size, align,
-				/* os_out_of_memory */ 0);
+  return clib_mem_heap_alloc_inline (0, size, align,
+				     /* os_out_of_memory */ 0);
 }
 
-__clib_export void *
-clib_mem_realloc_aligned (void *p, uword new_size, uword align)
+__clib_export __clib_flatten void *
+clib_mem_heap_alloc (void *heap, uword size)
+{
+  return clib_mem_heap_alloc_inline (heap, size, CLIB_MEM_MIN_ALIGN,
+				     /* os_out_of_memory */ 1);
+}
+
+__clib_export __clib_flatten void *
+clib_mem_heap_alloc_aligned (void *heap, uword size, uword align)
+{
+  return clib_mem_heap_alloc_inline (heap, size, align,
+				     /* os_out_of_memory */ 1);
+}
+
+__clib_export __clib_flatten void *
+clib_mem_heap_alloc_or_null (void *heap, uword size)
+{
+  return clib_mem_heap_alloc_inline (heap, size, CLIB_MEM_MIN_ALIGN,
+				     /* os_out_of_memory */ 0);
+}
+
+__clib_export __clib_flatten void *
+clib_mem_heap_alloc_aligned_or_null (void *heap, uword size, uword align)
+{
+  return clib_mem_heap_alloc_inline (heap, size, align,
+				     /* os_out_of_memory */ 0);
+}
+
+__clib_export __clib_flatten void *
+clib_mem_heap_realloc_aligned (void *heap, void *p, uword new_size,
+			       uword align)
 {
   uword old_alloc_size;
-  clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+  clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap ();
   void *new;
 
   ASSERT (count_set_bits (align) == 1);
@@ -674,14 +703,14 @@ clib_mem_realloc_aligned (void *p, uword new_size, uword align)
     }
   else
     {
-      new = clib_mem_alloc_inline (new_size, align, 1);
+      new = clib_mem_heap_alloc_inline (h, new_size, align, 1);
 
       CLIB_MEM_UNPOISON (new, new_size);
       if (old_alloc_size)
 	{
 	  CLIB_MEM_UNPOISON (p, old_alloc_size);
 	  clib_memcpy_fast (new, p, clib_min (new_size, old_alloc_size));
-	  clib_mem_free (p);
+	  clib_mem_heap_free (h, p);
 	}
       p = new;
     }
@@ -689,28 +718,45 @@ clib_mem_realloc_aligned (void *p, uword new_size, uword align)
   return p;
 }
 
-__clib_export void *
+__clib_export __clib_flatten void *
+clib_mem_heap_realloc (void *heap, void *p, uword new_size)
+{
+  return clib_mem_heap_realloc_aligned (heap, p, new_size, CLIB_MEM_MIN_ALIGN);
+}
+
+__clib_export __clib_flatten void *
+clib_mem_realloc_aligned (void *p, uword new_size, uword align)
+{
+  return clib_mem_heap_realloc_aligned (0, p, new_size, align);
+}
+
+__clib_export __clib_flatten void *
 clib_mem_realloc (void *p, uword new_size)
 {
-  return clib_mem_realloc_aligned (p, new_size, CLIB_MEM_MIN_ALIGN);
+  return clib_mem_heap_realloc_aligned (0, p, new_size, CLIB_MEM_MIN_ALIGN);
 }
 
-__clib_export uword
-clib_mem_is_heap_object (void *p)
+__clib_export __clib_flatten uword
+clib_mem_heap_is_heap_object (void *heap, void *p)
 {
-  int mspace_is_heap_object (void *msp, void *p);
-  clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+  clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap ();
   return mspace_is_heap_object (h->mspace, p);
 }
 
-__clib_export void
-clib_mem_free (void *p)
+__clib_export __clib_flatten uword
+clib_mem_is_heap_object (void *p)
+{
+  return clib_mem_heap_is_heap_object (0, p);
+}
+
+__clib_export __clib_flatten void
+clib_mem_heap_free (void *heap, void *p)
 {
-  clib_mem_heap_t *h = clib_mem_get_per_cpu_heap ();
+  clib_mem_heap_t *h = heap ? heap : clib_mem_get_per_cpu_heap ();
   uword size = clib_mem_size (p);
 
   /* Make sure object is in the correct heap. */
-  ASSERT (clib_mem_is_heap_object (p));
+  ASSERT (clib_mem_heap_is_heap_object (h, p));
 
   if (PREDICT_FALSE (h->flags & CLIB_MEM_HEAP_F_TRACED))
     mheap_put_trace (pointer_to_uword (p), size);
@@ -719,7 +765,13 @@ clib_mem_free (void *p)
   mspace_free (h->mspace, p);
 }
 
-__clib_export uword
+__clib_export __clib_flatten void
+clib_mem_free (void *p)
+{
+  clib_mem_heap_free (0, p);
+}
+
+__clib_export __clib_flatten uword
 clib_mem_size (void *p)
 {
   return mspace_usable_size (p);
diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h
index 32360daeffc..4db1f7b6bdf 100644
--- a/src/vppinfra/pool.h
+++ b/src/vppinfra/pool.h
@@ -308,7 +308,7 @@ _pool_put_index (void *p, uword index, uword elt_sz)
 /** Allocate N more free elements to pool (general version). */
 
 static_always_inline void
-_pool_alloc (void **pp, uword n_elts, uword align, uword elt_sz)
+_pool_alloc (void **pp, uword n_elts, uword align, void *heap, uword elt_sz)
 {
   pool_header_t *ph = pool_header (pp[0]);
   uword len = vec_len (pp[0]);
@@ -320,7 +320,7 @@ _pool_alloc (void **pp, uword n_elts, uword align, uword elt_sz)
     }
 
   pp[0] = _vec_realloc_inline (pp[0], len + n_elts, elt_sz,
-			       sizeof (pool_header_t), align, 0);
+			       sizeof (pool_header_t), align, heap);
   _vec_len (pp[0]) = len;
   CLIB_MEM_POISON (pp[0] + len * elt_sz, n_elts * elt_sz);
 
@@ -330,11 +330,12 @@ _pool_alloc (void **pp, uword n_elts, uword align, uword elt_sz)
   clib_bitmap_vec_validate (ph->free_bitmap, len + n_elts - 1);
 }
 
-#define pool_alloc_aligned(P, N, A)                                           \
-  _pool_alloc ((void **) &(P), N, _vec_align (P, A), _vec_elt_sz (P))
+#define pool_alloc_aligned_heap(P, N, A, H)                                   \
+  _pool_alloc ((void **) &(P), N, _vec_align (P, A), H, _vec_elt_sz (P))
 
-/** Allocate N more free elements to pool (unspecified alignment). */
-#define pool_alloc(P,N) pool_alloc_aligned(P,N,0)
+#define pool_alloc_heap(P, N, H)    pool_alloc_aligned_heap (P, N, 0, H)
+#define pool_alloc_aligned(P, N, A) pool_alloc_aligned_heap (P, N, A, 0)
+#define pool_alloc(P, N)	    pool_alloc_aligned_heap (P, N, 0, 0)
 
 static_always_inline void *
 _pool_dup (void *p, uword align, uword elt_sz)
diff --git a/src/vppinfra/vec.c b/src/vppinfra/vec.c
index 2fbab2f7e45..805fdc02b7e 100644
--- a/src/vppinfra/vec.c
+++ b/src/vppinfra/vec.c
@@ -19,26 +19,22 @@ __clib_export void *
 _vec_realloc (void *v, uword n_elts, uword elt_sz, uword hdr_sz, uword align,
 	      void *heap)
 {
-  uword n_data_bytes, data_offset, new_data_size, alloc_size;
+  uword n_data_bytes, alloc_size, new_data_size;
   void *p;
 
   /* alignment must be power of 2 */
   align = clib_max (align, VEC_MIN_ALIGN);
   ASSERT (count_set_bits (align) == 1);
 
-  /* number of bytes needed to store both vector header and optional user
-   * header */
-  data_offset = round_pow2 (hdr_sz + sizeof (vec_header_t), align);
-
   /* mumber of bytes needed to store vector data */
   n_data_bytes = n_elts * elt_sz;
 
-  /* minimal allocation needed to store data and headers */
-  new_data_size = data_offset + n_data_bytes;
-
   if (v)
     {
+      uword data_offset = vec_get_header_size (v);
       uword old_data_size = data_offset + _vec_len (v) * elt_sz;
+      new_data_size = data_offset + n_data_bytes;
+      heap = _vec_find (v)->default_heap ? 0 : _vec_heap (v);
       p = vec_header (v);
       alloc_size = clib_mem_size (p);
 
@@ -54,7 +50,7 @@ _vec_realloc (void *v, uword n_elts, uword elt_sz, uword hdr_sz, uword align,
 	  else
 	    alloc_size = (n_data_bytes * 3) / 2 + data_offset;
 
-	  p = clib_mem_realloc_aligned (p, alloc_size, align);
+	  p = clib_mem_heap_realloc_aligned (heap, p, alloc_size, align);
 	  alloc_size = clib_mem_size (p);
 	  v = p + data_offset;
 	}
@@ -65,13 +61,25 @@ _vec_realloc (void *v, uword n_elts, uword elt_sz, uword hdr_sz, uword align,
   else
     {
       /* new allocation */
-      p = clib_mem_alloc_aligned (new_data_size, align);
+      uword data_offset = hdr_sz + sizeof (vec_header_t);
+      data_offset += heap ? sizeof (void *) : 0;
+      data_offset = round_pow2 (data_offset, align);
+
+      new_data_size = data_offset + n_data_bytes;
+      p = clib_mem_heap_alloc_aligned (heap, new_data_size, align);
       alloc_size = clib_mem_size (p);
       CLIB_MEM_UNPOISON (p, alloc_size);
       clib_memset_u8 (p, 0, alloc_size);
       v = p + data_offset;
       _vec_find (v)->hdr_size = data_offset / VEC_MIN_ALIGN;
       _vec_find (v)->log2_align = min_log2 (align);
+      if (heap)
+	{
+	  _vec_find (v)->default_heap = 0;
+	  _vec_heap (v) = heap;
+	}
+      else
+	_vec_find (v)->default_heap = 1;
     }
 
   CLIB_MEM_POISON (p + new_data_size, alloc_size - new_data_size);
diff --git a/src/vppinfra/vec.h b/src/vppinfra/vec.h
index e90c27c1bc1..1dc300ad1ea 100644
--- a/src/vppinfra/vec.h
+++ b/src/vppinfra/vec.h
@@ -54,6 +54,7 @@
 ~~~~~~~~
 		    user header (start of memory allocation)
 		    padding
+		    heap pointer (optional, only if default_heap == 0)
 		    vector header: number of elements, header size
    user's pointer-> vector element #0
 		    vector element #1
@@ -130,14 +131,22 @@ _vec_update_pointer (void **vp, void *v)
     vp[0] = v;
 }
 
-always_inline void *
+static_always_inline void *
+vec_get_heap (void *v)
+{
+  if (v == 0 || _vec_find (v)->default_heap == 1)
+    return 0;
+  return _vec_heap (v);
+}
+
+static_always_inline void *
 _vec_realloc_inline (void *v, uword n_elts, uword elt_sz, uword hdr_sz,
 		     uword align, void *heap)
 {
   if (PREDICT_TRUE (v != 0))
     {
       /* Vector header must start heap object. */
-      ASSERT (clib_mem_is_heap_object (vec_header (v)));
+      ASSERT (clib_mem_heap_is_heap_object (vec_get_heap (v), vec_header (v)));
 
       /* Typically we'll not need to resize. */
       if ((n_elts * elt_sz) <= vec_max_bytes (v))
@@ -151,6 +160,49 @@ _vec_realloc_inline (void *v, uword n_elts, uword elt_sz, uword hdr_sz,
   return _vec_realloc (v, n_elts, elt_sz, hdr_sz, align, heap);
 }
 
+static_always_inline void
+_vec_prealloc (void **vp, uword n_elts, uword hdr_sz, uword align, void *heap,
+	       uword elt_sz)
+{
+  void *v;
+
+  ASSERT (vp[0] == 0);
+
+  v = _vec_realloc (0, n_elts, elt_sz, hdr_sz, align, heap);
+  _vec_set_len (v, 0, elt_sz);
+  _vec_update_pointer (vp, v);
+}
+
+/** \brief Pre-allocate a vector (generic version)
+
+    @param V pointer to a vector
+    @param N number of elements to pre-allocate
+    @param H header size in bytes (may be zero)
+    @param A alignment (zero means default alignment of the data structure)
+    @param P heap (zero means default heap)
+    @return V (value-result macro parameter)
+*/
+
+#define vec_prealloc_hap(V, N, H, A, P)                                       \
+  _vec_prealloc ((void **) &(V), N, H, _vec_align (V, A), P, _vec_elt_sz (V))
+
+/** \brief Pre-allocate a vector (simple version)
+
+    @param V pointer to a vector
+    @param N number of elements to pre-allocate
+    @return V (value-result macro parameter)
+*/
+#define vec_prealloc(V, N) vec_prealloc_hap (V, N, 0, 0, 0)
+
+/** \brief Pre-allocate a vector (heap version)
+
+    @param V pointer to a vector
+    @param N number of elements to pre-allocate
+    @param P heap (zero means default heap)
+    @return V (value-result macro parameter)
+*/
+#define vec_prealloc_heap(V, N, P) vec_prealloc_hap (V, N, 0, 0, P)
+
 always_inline int
 _vec_resize_will_expand (void *v, uword n_elts, uword elt_sz)
 {
@@ -158,7 +210,7 @@ _vec_resize_will_expand (void *v, uword n_elts, uword elt_sz)
     return 1;
 
   /* Vector header must start heap object. */
-  ASSERT (clib_mem_is_heap_object (vec_header (v)));
+  ASSERT (clib_mem_heap_is_heap_object (vec_get_heap (v), vec_header (v)));
 
   n_elts += _vec_len (v);
   if ((n_elts * elt_sz) <= vec_max_bytes (v))
@@ -398,22 +450,22 @@ _vec_zero_elts (void *v, uword first, uword count, uword elt_sz)
 }
 #define vec_zero_elts(V, F, C) _vec_zero_elts (V, F, C, sizeof ((V)[0]))
 
-static_always_inline void *
-_vec_validate_ha (void *v, uword index, uword header_size, uword align,
-		  uword elt_sz)
+static_always_inline void
+_vec_validate (void **vp, uword index, uword header_size, uword align,
+	       void *heap, uword elt_sz)
 {
+  void *v = vp[0];
   uword vl = vec_len (v);
   if (index >= vl)
     {
-      v = _vec_realloc_inline (v, index + 1, elt_sz, header_size, align, 0);
+      v = _vec_realloc_inline (v, index + 1, elt_sz, header_size, align, heap);
       _vec_zero_elts (v, vl, index - vl + 1, elt_sz);
+      _vec_update_pointer (vp, v);
     }
-  return v;
 }
 
-#define vec_validate_ha(V, I, H, A)                                           \
-  (V) =                                                                       \
-    _vec_validate_ha ((void *) (V), I, H, _vec_align (V, A), sizeof ((V)[0]))
+#define vec_validate_hap(V, I, H, A, P)                                       \
+  _vec_validate ((void **) &(V), I, H, _vec_align (V, A), 0, sizeof ((V)[0]))
 
 /** \brief Make sure vector is long enough for given index
     (no header, unspecified alignment)
@@ -422,7 +474,7 @@ _vec_validate_ha (void *v, uword index, uword header_size, uword align,
     @param I vector index which will be valid upon return
     @return V (value-result macro parameter)
 */
-#define vec_validate(V,I)           vec_validate_ha(V,I,0,0)
+#define vec_validate(V, I) vec_validate_hap (V, I, 0, 0, 0)
 
 /** \brief Make sure vector is long enough for given index
     (no header, specified alignment)
@@ -433,7 +485,18 @@ _vec_validate_ha (void *v, uword index, uword header_size, uword align,
     @return V (value-result macro parameter)
 */
 
-#define vec_validate_aligned(V,I,A) vec_validate_ha(V,I,0,A)
+#define vec_validate_aligned(V, I, A) vec_validate_hap (V, I, 0, A, 0)
+
+/** \brief Make sure vector is long enough for given index
+    (no header, specified heap)
+
+    @param V (possibly NULL) pointer to a vector.
+    @param I vector index which will be valid upon return
+    @param H heap (may be zero)
+    @return V (value-result macro parameter)
+*/
+
+#define vec_validate_heap(V, I, P) vec_validate_hap (V, I, 0, 0, P)
 
 /** \brief Make sure vector is long enough for given index
     and initialize empty space (general version)
diff --git a/src/vppinfra/vec_bootstrap.h b/src/vppinfra/vec_bootstrap.h
index 304ea2dee1a..d6451f3d172 100644
--- a/src/vppinfra/vec_bootstrap.h
+++ b/src/vppinfra/vec_bootstrap.h
@@ -56,7 +56,8 @@ typedef struct
 {
   u32 len; /**< Number of elements in vector (NOT its allocated length). */
   u8 hdr_size;	      /**< header size divided by VEC_MIN_ALIGN */
-  u8 log2_align;      /**< data alignment */
+  u8 log2_align : 7;  /**< data alignment */
+  u8 default_heap : 1; /**< vector uses default heap */
   u8 vpad[2];	      /**< pad to 8 bytes */
   u8 vector_data[0];  /**< Vector data . */
 } vec_header_t;
@@ -72,6 +73,7 @@ typedef struct
     @return pointer to the vector's vector_header_t
 */
 #define _vec_find(v)	((vec_header_t *) (v) - 1)
+#define _vec_heap(v)	(((void **) (_vec_find (v)))[-1])
 
 always_inline uword __vec_align (uword data_align, uword configuered_align);
 always_inline uword __vec_elt_sz (uword elt_sz, int is_void);