diff options
Diffstat (limited to 'src/vppinfra')
| -rw-r--r-- | src/vppinfra/CMakeLists.txt | 9 | ||||
| -rw-r--r-- | src/vppinfra/bihash_template.c | 1 | ||||
| -rw-r--r-- | src/vppinfra/config.h.in | 1 | ||||
| -rw-r--r-- | src/vppinfra/mem.h | 54 | ||||
| -rw-r--r-- | src/vppinfra/mem_dlmalloc.c | 64 | ||||
| -rw-r--r-- | src/vppinfra/os.h | 13 | ||||
| -rw-r--r-- | src/vppinfra/pool.h | 35 | ||||
| -rw-r--r-- | src/vppinfra/unix-misc.c | 1 | ||||
| -rw-r--r-- | src/vppinfra/vec.c | 19 | ||||
| -rw-r--r-- | src/vppinfra/vec.h | 113 | ||||
| -rw-r--r-- | src/vppinfra/vec_bootstrap.h | 32 |
11 files changed, 282 insertions, 60 deletions
diff --git a/src/vppinfra/CMakeLists.txt b/src/vppinfra/CMakeLists.txt index 1c234cce234..60e6eeff9fe 100644 --- a/src/vppinfra/CMakeLists.txt +++ b/src/vppinfra/CMakeLists.txt @@ -24,6 +24,13 @@ else(VPP_USE_DLMALLOC) set(DLMALLOC 0) endif(VPP_USE_DLMALLOC) +find_library(NUMA numa) +if (NUMA) + set(NUMA_LIBRARY_FOUND 1) +else(NUMA) + set(NUMA_LIBRARY_FOUND 0) +endif() + configure_file( ${CMAKE_SOURCE_DIR}/vppinfra/config.h.in ${CMAKE_BINARY_DIR}/vppinfra/config.h @@ -210,7 +217,7 @@ endif(VPP_USE_DLMALLOC) add_vpp_library(vppinfra SOURCES ${VPPINFRA_SRCS} - LINK_LIBRARIES m + LINK_LIBRARIES m ${NUMA} INSTALL_HEADERS ${VPPINFRA_HEADERS} COMPONENT libvppinfra ) diff --git a/src/vppinfra/bihash_template.c b/src/vppinfra/bihash_template.c index dda35969058..2b378427ce8 100644 --- a/src/vppinfra/bihash_template.c +++ b/src/vppinfra/bihash_template.c @@ -179,7 +179,6 @@ void BV (clib_bihash_master_init_svm) sizeof (vec_header_t) + BIHASH_FREELIST_LENGTH * sizeof (u64)); freelist_vh->len = BIHASH_FREELIST_LENGTH; - freelist_vh->dlmalloc_header_offset = 0xDEADBEEF; h->sh->freelists_as_u64 = (u64) BV (clib_bihash_get_offset) (h, freelist_vh->vector_data); h->freelists = (void *) (freelist_vh->vector_data); diff --git a/src/vppinfra/config.h.in b/src/vppinfra/config.h.in index a7a22a6a992..b2366630447 100644 --- a/src/vppinfra/config.h.in +++ b/src/vppinfra/config.h.in @@ -21,6 +21,7 @@ #endif #define USE_DLMALLOC @DLMALLOC@ +#define HAVE_NUMA_LIBRARY @NUMA_LIBRARY_FOUND@ #define CLIB_TARGET_TRIPLET "@CMAKE_C_COMPILER_TARGET@" #endif diff --git a/src/vppinfra/mem.h b/src/vppinfra/mem.h index d4819b7f989..5492e106d91 100644 --- a/src/vppinfra/mem.h +++ b/src/vppinfra/mem.h @@ -56,9 +56,46 @@ #include <vppinfra/sanitizer.h> #define CLIB_MAX_MHEAPS 256 +#define CLIB_MAX_NUMAS 8 + +/* Unspecified NUMA socket */ +#define VEC_NUMA_UNSPECIFIED (0xFF) /* Per CPU heaps. */ extern void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]; +extern void *clib_per_numa_mheaps[CLIB_MAX_NUMAS]; + +always_inline void * +clib_mem_get_per_cpu_heap (void) +{ + int cpu = os_get_thread_index (); + return clib_per_cpu_mheaps[cpu]; +} + +always_inline void * +clib_mem_set_per_cpu_heap (u8 * new_heap) +{ + int cpu = os_get_thread_index (); + void *old = clib_per_cpu_mheaps[cpu]; + clib_per_cpu_mheaps[cpu] = new_heap; + return old; +} + +always_inline void * +clib_mem_get_per_numa_heap (u32 numa_id) +{ + ASSERT (numa_id >= 0 && numa_id < ARRAY_LEN (clib_per_numa_mheaps)); + return clib_per_numa_mheaps[numa_id]; +} + +always_inline void * +clib_mem_set_per_numa_heap (u8 * new_heap) +{ + int numa = os_get_numa_index (); + void *old = clib_per_numa_mheaps[numa]; + clib_per_numa_mheaps[numa] = new_heap; + return old; +} always_inline void clib_mem_set_thread_index (void) @@ -81,22 +118,6 @@ clib_mem_set_thread_index (void) ASSERT (__os_thread_index > 0); } -always_inline void * -clib_mem_get_per_cpu_heap (void) -{ - int cpu = os_get_thread_index (); - return clib_per_cpu_mheaps[cpu]; -} - -always_inline void * -clib_mem_set_per_cpu_heap (u8 * new_heap) -{ - int cpu = os_get_thread_index (); - void *old = clib_per_cpu_mheaps[cpu]; - clib_per_cpu_mheaps[cpu] = new_heap; - return old; -} - always_inline uword clib_mem_size_nocheck (void *p) { @@ -287,6 +308,7 @@ clib_mem_set_heap (void *heap) void *clib_mem_init (void *heap, uword size); void *clib_mem_init_thread_safe (void *memory, uword memory_size); +void *clib_mem_init_thread_safe_numa (void *memory, uword memory_size); void clib_mem_exit (void); diff --git a/src/vppinfra/mem_dlmalloc.c b/src/vppinfra/mem_dlmalloc.c index 68901a5530a..38226e26f8f 100644 --- a/src/vppinfra/mem_dlmalloc.c +++ b/src/vppinfra/mem_dlmalloc.c @@ -20,8 +20,10 @@ #include <vppinfra/hash.h> #include <vppinfra/elf_clib.h> #include <vppinfra/sanitizer.h> +#include <numaif.h> void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]; +void *clib_per_numa_mheaps[CLIB_MAX_NUMAS]; typedef struct { @@ -202,8 +204,8 @@ mheap_trace_main_free (mheap_trace_main_t * tm) /* Initialize CLIB heap based on memory/size given by user. Set memory to 0 and CLIB will try to allocate its own heap. */ -void * -clib_mem_init (void *memory, uword memory_size) +static void * +clib_mem_init_internal (void *memory, uword memory_size, int set_heap) { u8 *heap; @@ -217,7 +219,8 @@ clib_mem_init (void *memory, uword memory_size) CLIB_MEM_POISON (mspace_least_addr (heap), mspace_footprint (heap)); - clib_mem_set_heap (heap); + if (set_heap) + clib_mem_set_heap (heap); if (mheap_trace_main.lock == 0) clib_spinlock_init (&mheap_trace_main.lock); @@ -226,9 +229,62 @@ clib_mem_init (void *memory, uword memory_size) } void * +clib_mem_init (void *memory, uword memory_size) +{ + return clib_mem_init_internal (memory, memory_size, + 1 /* do clib_mem_set_heap */ ); +} + +void * clib_mem_init_thread_safe (void *memory, uword memory_size) { - return clib_mem_init (memory, memory_size); + return clib_mem_init_internal (memory, memory_size, + 1 /* do clib_mem_set_heap */ ); +} + +void * +clib_mem_init_thread_safe_numa (void *memory, uword memory_size) +{ + void *heap; + unsigned long this_numa; + + heap = + clib_mem_init_internal (memory, memory_size, + 0 /* do NOT clib_mem_set_heap */ ); + + ASSERT (heap); + + this_numa = os_get_numa_index (); + +#if HAVE_NUMA_LIBRARY > 0 + unsigned long nodemask = 1 << this_numa; + void *page_base; + unsigned long page_mask; + long rv; + + /* + * Bind the heap to the current thread's NUMA node. + * heap is not naturally page-aligned, so fix it. + */ + + page_mask = ~(clib_mem_get_page_size () - 1); + page_base = (void *) (((unsigned long) heap) & page_mask); + + clib_warning ("Bind heap at %llx size %llx to NUMA numa %d", + page_base, memory_size, this_numa); + + rv = mbind (page_base, memory_size, MPOL_BIND /* mode */ , + &nodemask /* nodemask */ , + BITS (nodemask) /* max node number */ , + MPOL_MF_MOVE /* flags */ ); + + if (rv < 0) + clib_unix_warning ("mbind"); +#else + clib_warning ("mbind unavailable, can't bind to numa %d", this_numa); +#endif + + return heap; } u8 * diff --git a/src/vppinfra/os.h b/src/vppinfra/os.h index 50a4ad97c93..cd3b4289da6 100644 --- a/src/vppinfra/os.h +++ b/src/vppinfra/os.h @@ -57,6 +57,7 @@ void os_out_of_memory (void); f64 os_cpu_clock_frequency (void); extern __thread uword __os_thread_index; +extern __thread uword __os_numa_index; static_always_inline uword os_get_thread_index (void) @@ -71,6 +72,18 @@ os_set_thread_index (uword thread_index) } static_always_inline uword +os_get_numa_index (void) +{ + return __os_numa_index; +} + +static_always_inline void +os_set_numa_index (uword numa_index) +{ + __os_numa_index = numa_index; +} + +static_always_inline uword os_get_cpu_number (void) __attribute__ ((deprecated)); static_always_inline uword diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h index e6ffe1e874a..db950d27d18 100644 --- a/src/vppinfra/pool.h +++ b/src/vppinfra/pool.h @@ -185,12 +185,13 @@ pool_free_elts (void *v) First search free list. If nothing is free extend vector of objects. */ -#define _pool_get_aligned_internal(P,E,A,Z) \ +#define _pool_get_aligned_internal_numa(P,E,A,Z,N) \ do { \ pool_header_t * _pool_var (p) = pool_header (P); \ uword _pool_var (l); \ \ - STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) || ((sizeof(P[0]) % A) == 0), \ + STATIC_ASSERT(A==0 || ((A % sizeof(P[0]))==0) \ + || ((sizeof(P[0]) % A) == 0), \ "Pool aligned alloc of incorrectly sized object"); \ _pool_var (l) = 0; \ if (P) \ @@ -199,11 +200,12 @@ do { \ if (_pool_var (l) > 0) \ { \ /* Return free element from free list. */ \ - uword _pool_var (i) = _pool_var (p)->free_indices[_pool_var (l) - 1]; \ + uword _pool_var (i) = \ + _pool_var (p)->free_indices[_pool_var (l) - 1]; \ (E) = (P) + _pool_var (i); \ - _pool_var (p)->free_bitmap = \ - clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap, \ - _pool_var (i)); \ + _pool_var (p)->free_bitmap = \ + clib_bitmap_andnoti_notrim (_pool_var (p)->free_bitmap, \ + _pool_var (i)); \ _vec_len (_pool_var (p)->free_indices) = _pool_var (l) - 1; \ CLIB_MEM_UNPOISON((E), sizeof((E)[0])); \ } \ @@ -216,17 +218,30 @@ do { \ os_out_of_memory(); \ } \ /* Nothing on free list, make a new element and return it. */ \ - P = _vec_resize (P, \ + P = _vec_resize_numa (P, \ /* length_increment */ 1, \ /* new size */ (vec_len (P) + 1) * sizeof (P[0]), \ pool_aligned_header_bytes, \ - /* align */ (A)); \ + /* align */ (A), \ + /* numa */ (N)); \ E = vec_end (P) - 1; \ - } \ + } \ if (Z) \ - memset(E, 0, sizeof(*E)); \ + memset(E, 0, sizeof(*E)); \ } while (0) +#define pool_get_aligned_zero_numa(P,E,A,Z,S) \ + _pool_get_aligned_internal_numa(P,E,A,Z,S) + +#define pool_get_aligned_numa(P,E,A,S) \ + _pool_get_aligned_internal_numa(P,E,A,0/*zero*/,S) + +#define pool_get_numa(P,E,S) \ + _pool_get_aligned_internal_numa(P,E,0/*align*/,0/*zero*/,S) + +#define _pool_get_aligned_internal(P,E,A,Z) \ + _pool_get_aligned_internal_numa(P,E,A,Z,VEC_NUMA_UNSPECIFIED) + /** Allocate an object E from a pool P with alignment A */ #define pool_get_aligned(P,E,A) _pool_get_aligned_internal(P,E,A,0) diff --git a/src/vppinfra/unix-misc.c b/src/vppinfra/unix-misc.c index f693102c65a..54016ed74f4 100644 --- a/src/vppinfra/unix-misc.c +++ b/src/vppinfra/unix-misc.c @@ -46,6 +46,7 @@ #include <stdio.h> /* for sprintf */ __thread uword __os_thread_index = 0; +__thread uword __os_numa_index = 0; clib_error_t * clib_file_n_bytes (char *file, uword * result) diff --git a/src/vppinfra/vec.c b/src/vppinfra/vec.c index 16372e9ef22..2ee78952d19 100644 --- a/src/vppinfra/vec.c +++ b/src/vppinfra/vec.c @@ -44,16 +44,24 @@ void * vec_resize_allocate_memory (void *v, word length_increment, uword data_bytes, - uword header_bytes, uword data_align) + uword header_bytes, uword data_align, + uword numa_id) { vec_header_t *vh = _vec_find (v); uword old_alloc_bytes, new_alloc_bytes; void *old, *new; + void *oldheap; header_bytes = vec_header_bytes (header_bytes); data_bytes += header_bytes; + if (PREDICT_FALSE (numa_id != VEC_NUMA_UNSPECIFIED)) + { + oldheap = clib_mem_get_per_cpu_heap (); + clib_mem_set_per_cpu_heap (clib_mem_get_per_numa_heap (numa_id)); + } + if (!v) { new = clib_mem_alloc_aligned_at_offset (data_bytes, data_align, header_bytes, 1 /* yes, call os_out_of_memory */ @@ -64,6 +72,9 @@ vec_resize_allocate_memory (void *v, CLIB_MEM_POISON (new + data_bytes, new_alloc_bytes - data_bytes); v = new + header_bytes; _vec_len (v) = length_increment; + _vec_numa (v) = numa_id; + if (PREDICT_FALSE (numa_id != VEC_NUMA_UNSPECIFIED)) + clib_mem_set_per_cpu_heap (oldheap); return v; } @@ -79,6 +90,8 @@ vec_resize_allocate_memory (void *v, if (data_bytes <= old_alloc_bytes) { CLIB_MEM_UNPOISON (v, data_bytes); + if (PREDICT_FALSE (numa_id != VEC_NUMA_UNSPECIFIED)) + clib_mem_set_per_cpu_heap (oldheap); return v; } @@ -110,6 +123,10 @@ vec_resize_allocate_memory (void *v, memset (v + old_alloc_bytes, 0, new_alloc_bytes - old_alloc_bytes); CLIB_MEM_POISON (new + data_bytes, new_alloc_bytes - data_bytes); + _vec_numa ((v + header_bytes)) = numa_id; + if (PREDICT_FALSE (numa_id != VEC_NUMA_UNSPECIFIED)) + clib_mem_set_per_cpu_heap (oldheap); + return v + header_bytes; } diff --git a/src/vppinfra/vec.h b/src/vppinfra/vec.h index 021b2295964..9054eaa5e57 100644 --- a/src/vppinfra/vec.h +++ b/src/vppinfra/vec.h @@ -96,12 +96,14 @@ @param data_bytes requested size in bytes @param header_bytes header size in bytes (may be zero) @param data_align alignment (may be zero) + @param numa_id numa id (may be zero) @return v_prime pointer to resized vector, may or may not equal v */ void *vec_resize_allocate_memory (void *v, word length_increment, uword data_bytes, - uword header_bytes, uword data_align); + uword header_bytes, uword data_align, + uword numa_id); /** \brief Low-level vector resize function, usually not called directly @@ -110,19 +112,25 @@ void *vec_resize_allocate_memory (void *v, @param data_bytes requested size in bytes @param header_bytes header size in bytes (may be zero) @param data_align alignment (may be zero) + @param numa_id (may be ~0) @return v_prime pointer to resized vector, may or may not equal v */ -#define _vec_resize(V,L,DB,HB,A) \ - _vec_resize_inline(V,L,DB,HB,clib_max((__alignof__((V)[0])),(A))) +#define _vec_resize_numa(V,L,DB,HB,A,S) \ + _vec_resize_inline(V,L,DB,HB,clib_max((__alignof__((V)[0])),(A)),(S)) + +#define _vec_resize(V,L,DB,HB,A) \ + _vec_resize_numa(V,L,DB,HB,A,VEC_NUMA_UNSPECIFIED) always_inline void * _vec_resize_inline (void *v, word length_increment, - uword data_bytes, uword header_bytes, uword data_align) + uword data_bytes, uword header_bytes, uword data_align, + uword numa_id) { vec_header_t *vh = _vec_find (v); uword new_data_bytes, aligned_header_bytes; + void *oldheap; aligned_header_bytes = vec_header_bytes (header_bytes); @@ -132,6 +140,12 @@ _vec_resize_inline (void *v, { void *p = v - aligned_header_bytes; + if (PREDICT_FALSE (numa_id != VEC_NUMA_UNSPECIFIED)) + { + oldheap = clib_mem_get_per_cpu_heap (); + clib_mem_set_per_cpu_heap (clib_mem_get_per_numa_heap (numa_id)); + } + /* Vector header must start heap object. */ ASSERT (clib_mem_is_heap_object (p)); @@ -140,15 +154,19 @@ _vec_resize_inline (void *v, { CLIB_MEM_UNPOISON (v, data_bytes); vh->len += length_increment; + if (PREDICT_FALSE (numa_id != VEC_NUMA_UNSPECIFIED)) + clib_mem_set_per_cpu_heap (oldheap); return v; } + if (PREDICT_FALSE (numa_id != VEC_NUMA_UNSPECIFIED)) + clib_mem_set_per_cpu_heap (oldheap); } /* Slow path: call helper function. */ return vec_resize_allocate_memory (v, length_increment, data_bytes, header_bytes, clib_max (sizeof (vec_header_t), - data_align)); + data_align), numa_id); } /** \brief Determine if vector will resize with next allocation @@ -221,16 +239,32 @@ clib_mem_is_vec (void *v) @param N number of elements to add @param H header size in bytes (may be zero) @param A alignment (may be zero) + @param S numa_id (may be zero) @return V (value-result macro parameter) */ -#define vec_resize_ha(V,N,H,A) \ -do { \ - word _v(n) = (N); \ - word _v(l) = vec_len (V); \ - V = _vec_resize ((V), _v(n), (_v(l) + _v(n)) * sizeof ((V)[0]), (H), (A)); \ +#define vec_resize_has(V,N,H,A,S) \ +do { \ + word _v(n) = (N); \ + word _v(l) = vec_len (V); \ + V = _vec_resize_numa ((V), _v(n), \ + (_v(l) + _v(n)) * sizeof ((V)[0]), \ + (H), (A),(S)); \ } while (0) +/** \brief Resize a vector (less general version). + Add N elements to end of given vector V, return pointer to start of vector. + Vector will have room for H header bytes and will have user's data aligned + at alignment A (rounded to next power of 2). + + @param V pointer to a vector + @param N number of elements to add + @param H header size in bytes (may be zero) + @param A alignment (may be zero) + @return V (value-result macro parameter) +*/ +#define vec_resize_ha(V,N,H,A) vec_resize_has(V,N,H,A,VEC_NUMA_UNSPECIFIED) + /** \brief Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V, return pointer to start of vector. Vector will have room for H header bytes and will have user's data aligned @@ -352,22 +386,35 @@ do { \ @param V pointer to a vector @param H size of header in bytes @param A alignment (may be zero) + @param S numa (may be VEC_NUMA_UNSPECIFIED) @return Vdup copy of vector */ -#define vec_dup_ha(V,H,A) \ +#define vec_dup_ha_numa(V,H,A,S) \ ({ \ __typeof__ ((V)[0]) * _v(v) = 0; \ uword _v(l) = vec_len (V); \ if (_v(l) > 0) \ { \ - vec_resize_ha (_v(v), _v(l), (H), (A)); \ + vec_resize_has (_v(v), _v(l), (H), (A), (S)); \ clib_memcpy_fast (_v(v), (V), _v(l) * sizeof ((V)[0]));\ } \ _v(v); \ }) +/** \brief Return copy of vector (VEC_NUMA_UNSPECIFIED). + + @param V pointer to a vector + @param H size of header in bytes + @param A alignment (may be zero) + + @return Vdup copy of vector +*/ +#define vec_dup_ha(V,H,A) \ + vec_dup_ha_numa(V,H,A,VEC_NUMA_UNSPECIFIED) + + /** \brief Return copy of vector (no header, no alignment) @param V pointer to a vector @@ -412,24 +459,40 @@ do { \ @param I vector index which will be valid upon return @param H header size in bytes (may be zero) @param A alignment (may be zero) + @param N numa_id (may be zero) @return V (value-result macro parameter) */ -#define vec_validate_ha(V,I,H,A) \ -do { \ - STATIC_ASSERT(A==0 || ((A % sizeof(V[0]))==0) || ((sizeof(V[0]) % A) == 0),\ - "vector validate aligned on incorrectly sized object"); \ - word _v(i) = (I); \ - word _v(l) = vec_len (V); \ - if (_v(i) >= _v(l)) \ - { \ - vec_resize_ha ((V), 1 + (_v(i) - _v(l)), (H), (A)); \ - /* Must zero new space since user may have previously \ - used e.g. _vec_len (v) -= 10 */ \ - clib_memset ((V) + _v(l), 0, (1 + (_v(i) - _v(l))) * sizeof ((V)[0])); \ - } \ +#define vec_validate_han(V,I,H,A,N) \ +do { \ + void *oldheap; \ + STATIC_ASSERT(A==0 || ((A % sizeof(V[0]))==0) \ + || ((sizeof(V[0]) % A) == 0), \ + "vector validate aligned on incorrectly sized object"); \ + word _v(i) = (I); \ + word _v(l) = vec_len (V); \ + if (_v(i) >= _v(l)) \ + { \ + /* switch to the per-numa heap if directed */ \ + if (PREDICT_FALSE(N != VEC_NUMA_UNSPECIFIED)) \ + { \ + oldheap = clib_mem_get_per_cpu_heap(); \ + clib_mem_set_per_cpu_heap (clib_mem_get_per_numa_heap(N)); \ + } \ + \ + vec_resize_ha ((V), 1 + (_v(i) - _v(l)), (H), (A)); \ + /* Must zero new space since user may have previously \ + used e.g. _vec_len (v) -= 10 */ \ + clib_memset ((V) + _v(l), 0, \ + (1 + (_v(i) - _v(l))) * sizeof ((V)[0])); \ + /* Switch back to the global heap */ \ + if (PREDICT_FALSE (N != VEC_NUMA_UNSPECIFIED)) \ + clib_mem_set_per_cpu_heap (oldheap); \ + } \ } while (0) +#define vec_validate_ha(V,I,H,A) vec_validate_han(V,I,H,A,VEC_NUMA_UNSPECIFIED) + /** \brief Make sure vector is long enough for given index (no header, unspecified alignment) diff --git a/src/vppinfra/vec_bootstrap.h b/src/vppinfra/vec_bootstrap.h index fbb01b685ca..879703c8f6a 100644 --- a/src/vppinfra/vec_bootstrap.h +++ b/src/vppinfra/vec_bootstrap.h @@ -58,11 +58,14 @@ typedef struct u64 len; #else u32 len; /**< Number of elements in vector (NOT its allocated length). */ - u32 dlmalloc_header_offset; /**< offset to memory allocator offset */ + u8 numa_id; /**< NUMA id */ + u8 vpad[3]; /**< pad to 8 bytes */ #endif u8 vector_data[0]; /**< Vector data . */ } vec_header_t; +#define VEC_NUMA_UNSPECIFIED (0xFF) + /** \brief Find the vector header Given the user's pointer to a vector, find the corresponding @@ -128,7 +131,7 @@ vec_aligned_header_end (void *v, uword header_bytes, uword align) /** \brief Number of elements in vector (lvalue-capable) - _vec_len (v) does not check for null, but can be used as a lvalue + _vec_len (v) does not check for null, but can be used as an lvalue (e.g. _vec_len (v) = 99). */ @@ -142,6 +145,20 @@ vec_aligned_header_end (void *v, uword header_bytes, uword align) #define vec_len(v) ((v) ? _vec_len(v) : 0) +/** \brief Vector's NUMA id (lvalue-capable) + + _vec_numa(v) does not check for null, but can be used as an lvalue + (e.g. _vec_numa(v) = 1). +*/ + +#define _vec_numa(v) (_vec_find(v)->numa_id) + +/** \brief Return vector's NUMA ID (rvalue-only, NULL tolerant) + vec_numa(v) checks for NULL, but cannot be used as an lvalue. +*/ +#define vec_numa(v) ((v) ? _vec_numa(v) : 0) + + /** \brief Number of data bytes in vector. */ #define vec_bytes(v) (vec_len (v) * sizeof (v[0])) @@ -208,6 +225,17 @@ for (var = vec_end (vec) - 1; var >= (vec); var--) #define vec_foreach_index_backwards(var,v) \ for ((var) = vec_len((v)) - 1; (var) >= 0; (var)--) +/** \brief return the NUMA index for a vector */ +always_inline uword +vec_get_numa (void *v) +{ + vec_header_t *vh; + if (v == 0) + return 0; + vh = _vec_find (v); + return vh->numa_id; +} + #endif /* included_clib_vec_bootstrap_h */ /* |