diff options
| -rw-r--r-- | src/plugins/dpdk/buffer.c | 4 | ||||
| -rw-r--r-- | src/plugins/memif/memif.c | 2 | ||||
| -rw-r--r-- | src/vlib/buffer.c | 33 | ||||
| -rw-r--r-- | src/vlib/buffer.h | 11 | ||||
| -rw-r--r-- | src/vlib/buffer_funcs.h | 36 | ||||
| -rw-r--r-- | src/vlib/linux/pci.c | 2 | ||||
| -rw-r--r-- | src/vlib/main.c | 2 | ||||
| -rw-r--r-- | src/vlib/main.h | 3 |
8 files changed, 48 insertions, 45 deletions
diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c index 768d66153e5..37966ba9759 100644 --- a/src/plugins/dpdk/buffer.c +++ b/src/plugins/dpdk/buffer.c @@ -286,9 +286,9 @@ static_always_inline void vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, u32 follow_buffer_next) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_t *bufp[n_buffers], **b = bufp; - u32 thread_index = vlib_get_thread_index (); + u32 thread_index = vm->thread_index; int i = 0; u32 simple_mask = VLIB_BUFFER_NEXT_PRESENT; u32 n_left, *bi; diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c index 3171ba22f7d..6e45b573f2c 100644 --- a/src/plugins/memif/memif.c +++ b/src/plugins/memif/memif.c @@ -338,7 +338,7 @@ memif_init_regions_and_queues (memif_if_t * mif) { vlib_buffer_pool_t *bp; /* *INDENT-OFF* */ - vec_foreach (bp, buffer_main.buffer_pools) + vec_foreach (bp, vm->buffer_main->buffer_pools) { vlib_physmem_map_t *pm; pm = vlib_physmem_get_map (vm, bp->physmem_map_index); diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index 0251f52bd87..547c26445af 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -53,8 +53,6 @@ vlib_buffer_callbacks_t *vlib_buffer_callbacks = 0; */ static u32 vlib_buffer_physmem_sz = 14 << 20; -vlib_buffer_main_t buffer_main; - /* logging */ static vlib_log_class_t buffer_log_default; @@ -197,7 +195,7 @@ vlib_validate_buffer_helper (vlib_main_t * vm, vlib_buffer_known_state_t k; u8 *msg, *result; - k = vlib_buffer_is_known (b->next_buffer); + k = vlib_buffer_is_known (vm, b->next_buffer); if (k != VLIB_BUFFER_KNOWN_ALLOCATED) return format (0, "next 0x%x: %U", b->next_buffer, format_vlib_buffer_known_state, k); @@ -255,7 +253,7 @@ vlib_validate_buffers (vlib_main_t * vm, goto done; } - k = vlib_buffer_is_known (bi); + k = vlib_buffer_is_known (vm, bi); if (k != known_state) { msg = format (0, "is %U; expected %U", @@ -329,7 +327,7 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm, bi = b[0]; b += 1; - known = vlib_buffer_is_known (bi); + known = vlib_buffer_is_known (vm, bi); if (known != expected_state) { ASSERT (0); @@ -339,8 +337,8 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm, format_vlib_buffer_known_state, known, bi); } - vlib_buffer_set_known_state - (bi, is_free ? VLIB_BUFFER_KNOWN_FREE : VLIB_BUFFER_KNOWN_ALLOCATED); + vlib_buffer_set_known_state (vm, bi, is_free ? VLIB_BUFFER_KNOWN_FREE : + VLIB_BUFFER_KNOWN_ALLOCATED); } } @@ -350,7 +348,7 @@ vlib_buffer_create_free_list_helper (vlib_main_t * vm, u32 n_data_bytes, u32 is_public, u32 is_default, u8 * name) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_free_list_t *f; int i; @@ -435,7 +433,7 @@ vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes, static void del_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f) { - vlib_buffer_pool_t *bp = vlib_buffer_pool_get (f->buffer_pool_index); + vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, f->buffer_pool_index); vec_add_aligned (bp->buffers, f->buffers, vec_len (f->buffers), CLIB_CACHE_LINE_BYTES); @@ -487,7 +485,7 @@ vlib_buffer_fill_free_list_internal (vlib_main_t * vm, uword min_free_buffers) { vlib_buffer_t *b; - vlib_buffer_pool_t *bp = vlib_buffer_pool_get (fl->buffer_pool_index); + vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, fl->buffer_pool_index); int n; u32 *bi; u32 n_alloc = 0; @@ -530,7 +528,7 @@ vlib_buffer_fill_free_list_internal (vlib_main_t * vm, bi[0] = vlib_get_buffer_index (vm, b); if (CLIB_DEBUG > 0) - vlib_buffer_set_known_state (bi[0], VLIB_BUFFER_KNOWN_FREE); + vlib_buffer_set_known_state (vm, bi[0], VLIB_BUFFER_KNOWN_FREE); clib_memset (b, 0, sizeof (vlib_buffer_t)); vlib_buffer_init_for_free_list (b, fl); @@ -548,7 +546,7 @@ done: void * vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; void *rv = bm->buffer_free_callback; bm->buffer_free_callback = fp; @@ -587,7 +585,7 @@ static_always_inline void vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, u32 follow_buffer_next) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_t *p, *b0, *b1, *b2, *b3; int i = 0; u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers, @@ -792,7 +790,7 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, u8 vlib_buffer_register_physmem_map (vlib_main_t * vm, u32 physmem_map_index) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_pool_t *p; vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index); uword start = pointer_to_uword (m->base); @@ -895,7 +893,7 @@ VLIB_CLI_COMMAND (show_buffers_command, static) = { clib_error_t * vlib_buffer_main_init (struct vlib_main_t * vm) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm; clib_error_t *error; u32 physmem_map_index; u8 pool_index; @@ -903,6 +901,9 @@ vlib_buffer_main_init (struct vlib_main_t * vm) buffer_log_default = vlib_log_register_class ("buffer", 0); + bm = vm->buffer_main = clib_mem_alloc (sizeof (bm[0])); + clib_memset (bm, 0, sizeof (bm[0])); + if (vlib_buffer_callbacks) { /* external plugin has registered own buffer callbacks @@ -941,7 +942,7 @@ retry: return error; pool_index = vlib_buffer_register_physmem_map (vm, physmem_map_index); - vlib_buffer_pool_t *bp = vlib_buffer_pool_get (pool_index); + vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, pool_index); clib_spinlock_init (&bp->lock); bp->buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES + sizeof (vlib_buffer_t); diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h index 56d855101d5..672131b935d 100644 --- a/src/vlib/buffer.h +++ b/src/vlib/buffer.h @@ -458,16 +458,7 @@ typedef struct int callbacks_registered; } vlib_buffer_main_t; -extern vlib_buffer_main_t buffer_main; - -static_always_inline vlib_buffer_pool_t * -vlib_buffer_pool_get (u8 buffer_pool_index) -{ - vlib_buffer_main_t *bm = &buffer_main; - return vec_elt_at_index (bm->buffer_pools, buffer_pool_index); -} - -u8 vlib_buffer_register_physmem_map (struct vlib_main_t * vm, +u8 vlib_buffer_register_physmem_map (struct vlib_main_t *vm, u32 physmem_map_index); clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm); diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index 41b5164e0d8..9095bd3387f 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -57,7 +57,7 @@ always_inline vlib_buffer_t * vlib_get_buffer (vlib_main_t * vm, u32 buffer_index) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES; ASSERT (offset < bm->buffer_mem_size); @@ -88,8 +88,9 @@ static_always_inline void vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count, i32 offset) { + uword buffer_mem_start = vm->buffer_main->buffer_mem_start; #ifdef CLIB_HAVE_VEC256 - u64x4 off = u64x4_splat (buffer_main.buffer_mem_start + offset); + u64x4 off = u64x4_splat (buffer_mem_start + offset); /* if count is not const, compiler will not unroll while loop se we maintain two-in-parallel variant */ while (count >= 8) @@ -111,7 +112,7 @@ vlib_get_buffers_with_offset (vlib_main_t * vm, u32 * bi, void **b, int count, /* shift and add to get vlib_buffer_t pointer */ u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b); #elif defined (CLIB_HAVE_VEC128) - u64x2 off = u64x2_splat (buffer_main.buffer_mem_start + offset); + u64x2 off = u64x2_splat (buffer_mem_start + offset); u32x4 bi4 = u32x4_load_unaligned (bi); u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4); #if defined (__aarch64__) @@ -165,7 +166,7 @@ vlib_get_buffers (vlib_main_t * vm, u32 * bi, vlib_buffer_t ** b, int count) always_inline u32 vlib_get_buffer_index (vlib_main_t * vm, void *p) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; uword offset = pointer_to_uword (p) - bm->buffer_mem_start; ASSERT (pointer_to_uword (p) >= bm->buffer_mem_start); ASSERT (offset < bm->buffer_mem_size); @@ -187,7 +188,7 @@ vlib_get_buffer_indices_with_offset (vlib_main_t * vm, void **b, u32 * bi, { #ifdef CLIB_HAVE_VEC256 u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 }; - u64x4 off4 = u64x4_splat (buffer_main.buffer_mem_start - offset); + u64x4 off4 = u64x4_splat (vm->buffer_main->buffer_mem_start - offset); while (count >= 8) { @@ -368,9 +369,9 @@ void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers, expected_state); always_inline vlib_buffer_known_state_t -vlib_buffer_is_known (u32 buffer_index) +vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; clib_spinlock_lock (&bm->buffer_known_hash_lockp); uword *p = hash_get (bm->buffer_known_hash, buffer_index); @@ -379,10 +380,10 @@ vlib_buffer_is_known (u32 buffer_index) } always_inline void -vlib_buffer_set_known_state (u32 buffer_index, +vlib_buffer_set_known_state (vlib_main_t * vm, u32 buffer_index, vlib_buffer_known_state_t state) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; clib_spinlock_lock (&bm->buffer_known_hash_lockp); hash_set (bm->buffer_known_hash, buffer_index, state); @@ -414,8 +415,8 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm, u32 n_buffers, vlib_buffer_free_list_index_t index) { - vlib_buffer_main_t *bm = &buffer_main; vlib_buffer_free_list_t *fl; + vlib_buffer_main_t *bm = vm->buffer_main; u32 *src; uword len; @@ -518,7 +519,7 @@ vlib_buffer_free (vlib_main_t * vm, /* number of buffers to free */ u32 n_buffers) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; ASSERT (bm->cb.vlib_buffer_free_cb); @@ -539,7 +540,7 @@ vlib_buffer_free_no_next (vlib_main_t * vm, /* number of buffers to free */ u32 n_buffers) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; ASSERT (bm->cb.vlib_buffer_free_no_next_cb); @@ -616,7 +617,7 @@ always_inline void vlib_buffer_delete_free_list (vlib_main_t * vm, vlib_buffer_free_list_index_t free_list_index) { - vlib_buffer_main_t *bm = &buffer_main; + vlib_buffer_main_t *bm = vm->buffer_main; ASSERT (bm->cb.vlib_buffer_delete_free_list_cb); @@ -1003,12 +1004,19 @@ vlib_buffer_init_for_free_list (vlib_buffer_t * dst, ASSERT (dst->n_add_refs == 0); } +static_always_inline vlib_buffer_pool_t * +vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + return vec_elt_at_index (bm->buffer_pools, buffer_pool_index); +} + always_inline void vlib_buffer_add_to_free_list (vlib_main_t * vm, vlib_buffer_free_list_t * f, u32 buffer_index, u8 do_init) { - vlib_buffer_pool_t *bp = vlib_buffer_pool_get (f->buffer_pool_index); + vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, f->buffer_pool_index); vlib_buffer_t *b; b = vlib_get_buffer (vm, buffer_index); if (PREDICT_TRUE (do_init)) diff --git a/src/vlib/linux/pci.c b/src/vlib/linux/pci.c index d31b3c94d62..28a5140f3eb 100644 --- a/src/vlib/linux/pci.c +++ b/src/vlib/linux/pci.c @@ -967,7 +967,7 @@ add_device_vfio (vlib_main_t * vm, linux_pci_device_t * p, { vlib_buffer_pool_t *bp; /* *INDENT-OFF* */ - vec_foreach (bp, buffer_main.buffer_pools) + vec_foreach (bp, vm->buffer_main->buffer_pools) { u32 i; vlib_physmem_map_t *pm; diff --git a/src/vlib/main.c b/src/vlib/main.c index 650fa281746..160050486c5 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -463,7 +463,7 @@ vlib_put_next_frame (vlib_main_t * vm, vlib_frame_t *f; u32 n_vectors_in_frame; - if (buffer_main.callbacks_registered == 0 && CLIB_DEBUG > 0) + if (vm->buffer_main->callbacks_registered == 0 && CLIB_DEBUG > 0) vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left); nf = vlib_node_runtime_get_next_frame (vm, r, next_index); diff --git a/src/vlib/main.h b/src/vlib/main.h index 8a25ce8b4cc..91661fdaaad 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -119,6 +119,9 @@ typedef struct vlib_main_t /* Pool of buffer free lists. */ vlib_buffer_free_list_t *buffer_free_list_pool; + /* buffer main structure. */ + vlib_buffer_main_t *buffer_main; + /* physical memory main structure. */ vlib_physmem_main_t physmem_main; |