From 910d3694e8b22c9d14e5f2913d14ae149e184620 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 21 Jan 2019 11:48:34 +0100 Subject: buffers: major cleanup and improvements This patch introduces following changes: - deprecated free lists which are not used and not compatible with external buffer managers (i.e. DPDK) - introduces native support for per-numa buffer pools - significantly improves performance of buffer alloc and free Change-Id: I4a8e723ae47056717afd6cac0efe87cb731b5be7 Signed-off-by: Damjan Marion --- src/plugins/avf/avf.h | 1 + src/plugins/avf/device.c | 6 +- src/plugins/avf/input.c | 5 +- src/plugins/dpdk/buffer.c | 750 +++++++++++++----------------------- src/plugins/dpdk/buffer.h | 6 +- src/plugins/dpdk/device/cli.c | 91 +---- src/plugins/dpdk/device/common.c | 20 +- src/plugins/dpdk/device/device.c | 8 +- src/plugins/dpdk/device/dpdk.h | 7 +- src/plugins/dpdk/device/dpdk_priv.h | 8 - src/plugins/dpdk/device/init.c | 94 +---- src/plugins/dpdk/ipsec/ipsec.c | 12 +- src/plugins/memif/memif.c | 5 + src/plugins/memif/node.c | 9 +- src/plugins/memif/private.h | 1 + src/vlib/buffer.c | 599 +++++++++++----------------- src/vlib/buffer.h | 122 ++---- src/vlib/buffer_funcs.h | 516 ++++++++++++++++--------- src/vlib/log.h | 2 - src/vlib/main.c | 9 +- src/vlib/main.h | 3 - src/vlib/threads.c | 22 -- src/vlib/vlib.h | 1 + src/vpp/conf/startup.conf | 12 +- src/vppinfra/cpu.h | 56 +++ src/vppinfra/error_bootstrap.h | 4 + 26 files changed, 997 insertions(+), 1372 deletions(-) diff --git a/src/plugins/avf/avf.h b/src/plugins/avf/avf.h index 518c7d8329a..b79a22b073f 100644 --- a/src/plugins/avf/avf.h +++ b/src/plugins/avf/avf.h @@ -102,6 +102,7 @@ typedef struct u32 *bufs; u16 n_enqueued; u8 int_mode; + u8 buffer_pool_index; } avf_rxq_t; typedef struct diff --git a/src/plugins/avf/device.c b/src/plugins/avf/device.c index f73e5d372d2..039a8203edb 100644 --- a/src/plugins/avf/device.c +++ b/src/plugins/avf/device.c @@ -229,6 +229,9 @@ avf_rxq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 rxq_size) 2 * CLIB_CACHE_LINE_BYTES, ad->numa_node); + rxq->buffer_pool_index = + vlib_buffer_pool_get_default_for_numa (vm, ad->numa_node); + if (rxq->descs == 0) return vlib_physmem_last_error (vm); @@ -239,7 +242,8 @@ avf_rxq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 rxq_size) vec_validate_aligned (rxq->bufs, rxq->size, CLIB_CACHE_LINE_BYTES); rxq->qrx_tail = ad->bar0 + AVF_QRX_TAIL (qid); - n_alloc = vlib_buffer_alloc (vm, rxq->bufs, rxq->size - 8); + n_alloc = vlib_buffer_alloc_from_pool (vm, rxq->bufs, rxq->size - 8, + rxq->buffer_pool_index); if (n_alloc == 0) return clib_error_return (0, "buffer allocation error"); diff --git a/src/plugins/avf/input.c b/src/plugins/avf/input.c index fc884ce39f6..beb5c052423 100644 --- a/src/plugins/avf/input.c +++ b/src/plugins/avf/input.c @@ -72,7 +72,9 @@ avf_rxq_refill (vlib_main_t * vm, vlib_node_runtime_t * node, avf_rxq_t * rxq, slot = (rxq->next - n_refill - 1) & mask; n_refill &= ~7; /* round to 8 */ - n_alloc = vlib_buffer_alloc_to_ring (vm, rxq->bufs, slot, size, n_refill); + n_alloc = + vlib_buffer_alloc_to_ring_from_pool (vm, rxq->bufs, slot, size, n_refill, + rxq->buffer_pool_index); if (PREDICT_FALSE (n_alloc != n_refill)) { @@ -368,6 +370,7 @@ no_more_desc: vnet_buffer (bt)->sw_if_index[VLIB_RX] = ad->sw_if_index; vnet_buffer (bt)->sw_if_index[VLIB_TX] = ~0; + bt->buffer_pool_index = rxq->buffer_pool_index; if (n_tail_desc) n_rx_bytes = avf_process_rx_burst (vm, node, ptd, n_rx_packets, 1); diff --git a/src/plugins/dpdk/buffer.c b/src/plugins/dpdk/buffer.c index aed194074cc..73310eff95a 100644 --- a/src/plugins/dpdk/buffer.c +++ b/src/plugins/dpdk/buffer.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Cisco and/or its affiliates. + * Copyright (c) 2017-2019 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -12,586 +12,378 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * buffer.c: allocate/free network buffers. - * - * Copyright (c) 2008 Eliot Dresselhaus - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -/** - * @file - * - * Allocate/free network buffers. - */ #include +#include #include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include +#include +#include #include -#include -#include -#include -#include #include STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM, "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM"); -typedef struct -{ - CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); - struct rte_mbuf **mbuf_alloc_list; -} dpdk_buffer_per_thread_data; typedef struct { - int vfio_container_fd; - dpdk_buffer_per_thread_data *ptd; -} dpdk_buffer_main_t; - -dpdk_buffer_main_t dpdk_buffer_main; - -static_always_inline void -dpdk_rte_pktmbuf_free (vlib_main_t * vm, u32 thread_index, vlib_buffer_t * b, - int maybe_next) -{ - struct rte_mbuf *mb; - u32 next, flags; + /* must be first */ + struct rte_pktmbuf_pool_private mbp_priv; + u8 buffer_pool_index; +} dpdk_mempool_private_t; -next: - flags = b->flags; - next = b->next_buffer; - mb = rte_mbuf_from_vlib_buffer (b); - - if (PREDICT_FALSE (b->n_add_refs)) - { - rte_mbuf_refcnt_update (mb, b->n_add_refs); - b->n_add_refs = 0; - } - - if ((mb = rte_pktmbuf_prefree_seg (mb))) - rte_mempool_put (mb->pool, mb); - - if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT)) - { - b = vlib_get_buffer (vm, next); - goto next; - } -} +#ifndef CLIB_MARCH_VARIANT +struct rte_mempool **dpdk_mempool_by_buffer_pool_index = 0; +struct rte_mempool **dpdk_no_cache_mempool_by_buffer_pool_index = 0; -/* Make sure free list has at least given number of free buffers. */ -uword -CLIB_MULTIARCH_FN (dpdk_buffer_fill_free_list) (vlib_main_t * vm, - vlib_buffer_free_list_t * fl, - uword min_free_buffers) +clib_error_t * +dpdk_buffer_pool_init (vlib_main_t * vm, vlib_buffer_pool_t * bp) { - dpdk_main_t *dm = &dpdk_main; - dpdk_buffer_main_t *dbm = &dpdk_buffer_main; - struct rte_mbuf **mb; - uword n_left, first; - word n_alloc; - unsigned socket_id = rte_socket_id (); - u32 thread_index = vlib_get_thread_index (); - dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index); - struct rte_mempool *rmp = dm->pktmbuf_pools[socket_id]; - dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp); - vlib_buffer_t bt; + struct rte_mempool *mp, *nmp; + dpdk_mempool_private_t priv; + enum rte_iova_mode iova_mode; u32 *bi; + u8 *name = 0; - /* Too early? */ - if (PREDICT_FALSE (rmp == 0)) - return 0; - - /* Already have enough free buffers on free list? */ - n_alloc = min_free_buffers - vec_len (fl->buffers); - if (n_alloc <= 0) - return min_free_buffers; - - /* Always allocate round number of buffers. */ - n_alloc = round_pow2 (n_alloc, CLIB_CACHE_LINE_BYTES / sizeof (u32)); + u32 elt_size = + sizeof (struct rte_mbuf) + sizeof (vlib_buffer_t) + bp->data_size; - /* Always allocate new buffers in reasonably large sized chunks. */ - n_alloc = clib_max (n_alloc, fl->min_n_buffers_each_alloc); - - vec_validate_aligned (d->mbuf_alloc_list, n_alloc - 1, + /* create empty mempools */ + vec_validate_aligned (dpdk_mempool_by_buffer_pool_index, bp->index, + CLIB_CACHE_LINE_BYTES); + vec_validate_aligned (dpdk_no_cache_mempool_by_buffer_pool_index, bp->index, CLIB_CACHE_LINE_BYTES); - if (rte_mempool_get_bulk (rmp, (void *) d->mbuf_alloc_list, n_alloc) < 0) - return 0; + /* normal mempool */ + name = format (name, "vpp pool %u%c", bp->index, 0); + mp = rte_mempool_create_empty ((char *) name, vec_len (bp->buffers), + elt_size, 512, sizeof (priv), + bp->numa_node, 0); + vec_reset_length (name); - clib_memset (&bt, 0, sizeof (vlib_buffer_t)); - bt.buffer_pool_index = privp->buffer_pool_index; + /* non-cached mempool */ + name = format (name, "vpp pool %u (no cache)%c", bp->index, 0); + nmp = rte_mempool_create_empty ((char *) name, vec_len (bp->buffers), + elt_size, 0, sizeof (priv), + bp->numa_node, 0); + vec_free (name); - _vec_len (d->mbuf_alloc_list) = n_alloc; + dpdk_mempool_by_buffer_pool_index[bp->index] = mp; + dpdk_no_cache_mempool_by_buffer_pool_index[bp->index] = nmp; - first = vec_len (fl->buffers); - vec_resize_aligned (fl->buffers, n_alloc, CLIB_CACHE_LINE_BYTES); + rte_mempool_set_ops_byname (mp, "vpp", NULL); + rte_mempool_set_ops_byname (nmp, "vpp-no-cache", NULL); - n_left = n_alloc; - mb = d->mbuf_alloc_list; - bi = fl->buffers + first; + /* Call the mempool priv initializer */ + priv.mbp_priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE + + VLIB_BUFFER_DATA_SIZE; + priv.mbp_priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE; + priv.buffer_pool_index = bp->index; + rte_pktmbuf_pool_init (mp, &priv); + rte_pktmbuf_pool_init (nmp, &priv); - ASSERT (n_left % 8 == 0); + iova_mode = rte_eal_iova_mode (); - while (n_left >= 8) + /* populate mempool object buffer header */ + vec_foreach (bi, bp->buffers) + { + struct rte_mempool_objhdr *hdr; + vlib_buffer_t *b = vlib_get_buffer (vm, *bi); + struct rte_mbuf *mb = rte_mbuf_from_vlib_buffer (b); + hdr = (struct rte_mempool_objhdr *) RTE_PTR_SUB (mb, sizeof (*hdr)); + hdr->mp = mp; + hdr->iova = (iova_mode == RTE_IOVA_VA) ? + pointer_to_uword (mb) : vlib_physmem_get_pa (vm, mb); + STAILQ_INSERT_TAIL (&mp->elt_list, hdr, next); + STAILQ_INSERT_TAIL (&nmp->elt_list, hdr, next); + mp->populated_size++; + nmp->populated_size++; + } + + /* call the object initializers */ + rte_mempool_obj_iter (mp, rte_pktmbuf_init, 0); + + /* map DMA pages if at least one physical device exists */ + if (rte_eth_dev_count_avail ()) { - if (PREDICT_FALSE (n_left < 24)) - goto no_prefetch; - - vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[16]), STORE); - vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[17]), STORE); - vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[18]), STORE); - vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[19]), STORE); - vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[20]), STORE); - vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[21]), STORE); - vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[22]), STORE); - vlib_prefetch_buffer_header (vlib_buffer_from_rte_mbuf (mb[23]), STORE); - - no_prefetch: - vlib_get_buffer_indices_with_offset (vm, (void **) mb, bi, 8, - sizeof (struct rte_mbuf)); + uword i; + size_t page_sz; + vlib_physmem_map_t *pm; - vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[0]), &bt); - vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[1]), &bt); - vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[2]), &bt); - vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[3]), &bt); - vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[4]), &bt); - vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[5]), &bt); - vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[6]), &bt); - vlib_buffer_copy_template (vlib_buffer_from_rte_mbuf (mb[7]), &bt); - - n_left -= 8; - mb += 8; - bi += 8; - } + pm = vlib_physmem_get_map (vm, bp->physmem_map_index); + page_sz = 1ULL << pm->log2_page_size; - if (fl->buffer_init_function) - fl->buffer_init_function (vm, fl, fl->buffers + first, n_alloc); + for (i = 0; i < pm->n_pages; i++) + { + char *va = ((char *) pm->base) + i * page_sz; + uword pa = (iova_mode == RTE_IOVA_VA) ? + pointer_to_uword (va) : pm->page_table[i]; - fl->n_alloc += n_alloc; + if (rte_vfio_dma_map (pointer_to_uword (va), pa, page_sz)) + break; + } + } - return n_alloc; + return 0; } -static_always_inline void -dpdk_prefetch_buffer (vlib_buffer_t * b) +static int +dpdk_ops_vpp_alloc (struct rte_mempool *mp) { - struct rte_mbuf *mb; - mb = rte_mbuf_from_vlib_buffer (b); - CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (b, CLIB_CACHE_LINE_BYTES, LOAD); + clib_warning (""); + return 0; } -static_always_inline void -recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi, - vlib_buffer_t * b) +static void +dpdk_ops_vpp_free (struct rte_mempool *mp) { - u32 thread_index = vlib_get_thread_index (); - - dpdk_rte_pktmbuf_free (vm, thread_index, b, 1); + clib_warning (""); } +#endif + static_always_inline void -vlib_buffer_free_inline (vlib_main_t * vm, - u32 * buffers, u32 n_buffers, u32 follow_buffer_next) +dpdk_ops_vpp_enqueue_one (vlib_buffer_t * bt, void *obj) { - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_t *bufp[n_buffers], **b = bufp; - u32 thread_index = vm->thread_index; - int i = 0; - u32 simple_mask = VLIB_BUFFER_NEXT_PRESENT; - u32 n_left, *bi; - u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers, - u32 follow_buffer_next); - - cb = bm->buffer_free_callback; + /* Only non-replicated packets (b->ref_count == 1) expected */ - if (PREDICT_FALSE (cb != 0)) - n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next); + struct rte_mbuf *mb = obj; + vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb); + ASSERT (b->ref_count == 1); + ASSERT (b->buffer_pool_index == bt->buffer_pool_index); + vlib_buffer_copy_template (b, bt); +} - if (!n_buffers) - return; +int +CLIB_MULTIARCH_FN (dpdk_ops_vpp_enqueue) (struct rte_mempool * mp, + void *const *obj_table, unsigned n) +{ + const int batch_size = 32; + vlib_main_t *vm = vlib_get_main (); + vlib_buffer_t bt; + dpdk_mempool_private_t *privp = rte_mempool_get_priv (mp); + u8 buffer_pool_index = privp->buffer_pool_index; + vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); + u32 bufs[batch_size]; + u32 n_left = n; + void *const *obj = obj_table; - n_left = n_buffers; - bi = buffers; - b = bufp; - vlib_get_buffers (vm, bi, b, n_buffers); + vlib_buffer_copy_template (&bt, &bp->buffer_template); while (n_left >= 4) { - u32 or_flags; - vlib_buffer_t **p; - - if (n_left < 16) - goto no_prefetch; - - p = b + 12; - dpdk_prefetch_buffer (p[0]); - dpdk_prefetch_buffer (p[1]); - dpdk_prefetch_buffer (p[2]); - dpdk_prefetch_buffer (p[3]); - no_prefetch: - - for (i = 0; i < 4; i++) - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[i]); - - or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags; - - if (or_flags & simple_mask) - { - recycle_or_free (vm, bm, bi[0], b[0]); - recycle_or_free (vm, bm, bi[1], b[1]); - recycle_or_free (vm, bm, bi[2], b[2]); - recycle_or_free (vm, bm, bi[3], b[3]); - } - else - { - dpdk_rte_pktmbuf_free (vm, thread_index, b[0], 0); - dpdk_rte_pktmbuf_free (vm, thread_index, b[1], 0); - dpdk_rte_pktmbuf_free (vm, thread_index, b[2], 0); - dpdk_rte_pktmbuf_free (vm, thread_index, b[3], 0); - } - bi += 4; - b += 4; + dpdk_ops_vpp_enqueue_one (&bt, obj[0]); + dpdk_ops_vpp_enqueue_one (&bt, obj[1]); + dpdk_ops_vpp_enqueue_one (&bt, obj[2]); + dpdk_ops_vpp_enqueue_one (&bt, obj[3]); + obj += 4; n_left -= 4; } + while (n_left) { - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); - recycle_or_free (vm, bm, bi[0], b[0]); - bi += 1; - b += 1; + dpdk_ops_vpp_enqueue_one (&bt, obj[0]); + obj += 1; n_left -= 1; } -} - -void -CLIB_MULTIARCH_FN (dpdk_buffer_free) (vlib_main_t * vm, u32 * buffers, - u32 n_buffers) -{ - vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ - 1); -} - -void -CLIB_MULTIARCH_FN (dpdk_buffer_free_no_next) (vlib_main_t * vm, u32 * buffers, - u32 n_buffers) -{ - vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ - 0); -} -#ifndef CLIB_MARCH_VARIANT -clib_error_t * -dpdk_pool_create (vlib_main_t * vm, u8 * pool_name, u32 elt_size, - u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa, - struct rte_mempool **_mp, u32 * map_index) -{ - struct rte_mempool *mp; - enum rte_iova_mode iova_mode; - dpdk_mempool_private_t priv; - vlib_physmem_map_t *pm; - clib_error_t *error = 0; - size_t min_chunk_size, align; - int map_dma = 1; - u32 size; - i32 ret; - uword i; - - mp = rte_mempool_create_empty ((char *) pool_name, num_elts, elt_size, - 512, pool_priv_size, numa, 0); - if (!mp) - return clib_error_return (0, "failed to create %s", pool_name); - - rte_mempool_set_ops_byname (mp, RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL); - - size = rte_mempool_op_calc_mem_size_default (mp, num_elts, 21, - &min_chunk_size, &align); - - if ((error = vlib_physmem_shared_map_create (vm, (char *) pool_name, size, - 0, numa, map_index))) + while (n >= batch_size) { - rte_mempool_free (mp); - return error; + vlib_get_buffer_indices_with_offset (vm, (void **) obj_table, bufs, + batch_size, + sizeof (struct rte_mbuf)); + vlib_buffer_pool_put (vm, buffer_pool_index, bufs, batch_size); + n -= batch_size; + obj_table += batch_size; } - pm = vlib_physmem_get_map (vm, *map_index); - - /* Call the mempool priv initializer */ - priv.mbp_priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE + - VLIB_BUFFER_DATA_SIZE; - priv.mbp_priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE; - rte_pktmbuf_pool_init (mp, &priv); - if (rte_eth_dev_count_avail () == 0) - map_dma = 0; - - iova_mode = rte_eal_iova_mode (); - for (i = 0; i < pm->n_pages; i++) + if (n) { - size_t page_sz = 1ULL << pm->log2_page_size; - char *va = ((char *) pm->base) + i * page_sz; - uword pa = iova_mode == RTE_IOVA_VA ? - pointer_to_uword (va) : pm->page_table[i]; - ret = rte_mempool_populate_iova (mp, va, pa, page_sz, 0, 0); - if (ret < 0) - { - rte_mempool_free (mp); - return clib_error_return (0, "failed to populate %s", pool_name); - } - /* -1 likely means there is no PCI devices assigned to vfio - container or noiommu mode is used so we stop trying */ - if (map_dma && rte_vfio_dma_map (pointer_to_uword (va), pa, page_sz)) - map_dma = 0; + vlib_get_buffer_indices_with_offset (vm, (void **) obj_table, bufs, + n, sizeof (struct rte_mbuf)); + vlib_buffer_pool_put (vm, buffer_pool_index, bufs, batch_size); } - _mp[0] = mp; - return 0; } -clib_error_t * -dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, - unsigned socket_id) -{ - dpdk_main_t *dm = &dpdk_main; - struct rte_mempool *rmp; - clib_error_t *error = 0; - u8 *pool_name; - u32 elt_size, i; - u32 map_index; +CLIB_MARCH_FN_REGISTRATION (dpdk_ops_vpp_enqueue); - vec_validate_aligned (dm->pktmbuf_pools, socket_id, CLIB_CACHE_LINE_BYTES); - - /* pool already exists, nothing to do */ - if (dm->pktmbuf_pools[socket_id]) - return 0; - - pool_name = format (0, "dpdk_mbuf_pool_socket%u%c", socket_id, 0); - - elt_size = sizeof (struct rte_mbuf) + - VLIB_BUFFER_HDR_SIZE /* priv size */ + - VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE; /*data room size */ - - error = dpdk_pool_create (vm, pool_name, elt_size, num_mbufs, - sizeof (dpdk_mempool_private_t), 512, socket_id, - &rmp, &map_index); - - vec_free (pool_name); +static_always_inline void +dpdk_ops_vpp_enqueue_no_cache_one (vlib_main_t * vm, struct rte_mempool *old, + struct rte_mempool *new, void *obj, + vlib_buffer_t * bt) +{ + struct rte_mbuf *mb = obj; + vlib_buffer_t *b = vlib_buffer_from_rte_mbuf (mb); - if (!error) + if (clib_atomic_sub_fetch (&b->ref_count, 1) == 0) { - /* call the object initializers */ - rte_mempool_obj_iter (rmp, rte_pktmbuf_init, 0); - - dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp); - privp->buffer_pool_index = - vlib_buffer_register_physmem_map (vm, map_index); + u32 bi = vlib_get_buffer_index (vm, b); + mb->pool = new; + vlib_buffer_copy_template (b, bt); + vlib_buffer_pool_put (vm, bt->buffer_pool_index, &bi, 1); + return; + } +} - dm->pktmbuf_pools[socket_id] = rmp; +int +CLIB_MULTIARCH_FN (dpdk_ops_vpp_enqueue_no_cache) (struct rte_mempool * cmp, + void *const *obj_table, + unsigned n) +{ + vlib_main_t *vm = vlib_get_main (); + vlib_buffer_t bt; + dpdk_mempool_private_t *privp = rte_mempool_get_priv (cmp); + struct rte_mempool *mp; + mp = dpdk_mempool_by_buffer_pool_index[privp->buffer_pool_index]; + u8 buffer_pool_index = privp->buffer_pool_index; + vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); + vlib_buffer_copy_template (&bt, &bp->buffer_template); - return 0; + while (n >= 4) + { + dpdk_ops_vpp_enqueue_no_cache_one (vm, cmp, mp, obj_table[0], &bt); + dpdk_ops_vpp_enqueue_no_cache_one (vm, cmp, mp, obj_table[1], &bt); + dpdk_ops_vpp_enqueue_no_cache_one (vm, cmp, mp, obj_table[2], &bt); + dpdk_ops_vpp_enqueue_no_cache_one (vm, cmp, mp, obj_table[3], &bt); + obj_table += 4; + n -= 4; } - clib_error_report (error); - - /* no usable pool for this socket, try to use pool from another one */ - for (i = 0; i < vec_len (dm->pktmbuf_pools); i++) + while (n) { - if (dm->pktmbuf_pools[i]) - { - clib_warning ("WARNING: Failed to allocate mempool for CPU socket " - "%u. Threads running on socket %u will use socket %u " - "mempool.", socket_id, socket_id, i); - dm->pktmbuf_pools[socket_id] = dm->pktmbuf_pools[i]; - return 0; - } + dpdk_ops_vpp_enqueue_no_cache_one (vm, cmp, mp, obj_table[0], &bt); + obj_table += 1; + n -= 1; } - return clib_error_return (0, "failed to allocate mempool on socket %u", - socket_id); + return 0; } -#if CLIB_DEBUG > 0 +CLIB_MARCH_FN_REGISTRATION (dpdk_ops_vpp_enqueue_no_cache); -u32 *vlib_buffer_state_validation_lock; -uword *vlib_buffer_state_validation_hash; -void *vlib_buffer_state_heap; - -static clib_error_t * -buffer_state_validation_init (vlib_main_t * vm) +int +CLIB_MULTIARCH_FN (dpdk_ops_vpp_dequeue) (struct rte_mempool * mp, + void **obj_table, unsigned n) { - void *oldheap; - - vlib_buffer_state_heap = - mheap_alloc_with_lock (0, 10 << 20, 0 /* locked */ ); - oldheap = clib_mem_set_heap (vlib_buffer_state_heap); + const int batch_size = 32; + vlib_main_t *vm = vlib_get_main (); + u32 bufs[batch_size], total = 0, n_alloc = 0; + dpdk_mempool_private_t *privp = rte_mempool_get_priv (mp); + u8 buffer_pool_index = privp->buffer_pool_index; + void **obj = obj_table; + + while (n >= batch_size) + { + n_alloc = vlib_buffer_alloc_from_pool (vm, bufs, batch_size, + buffer_pool_index); + if (n_alloc != batch_size) + goto alloc_fail; + + vlib_get_buffers_with_offset (vm, bufs, obj, batch_size, + -(i32) sizeof (struct rte_mbuf)); + total += batch_size; + obj += batch_size; + n -= batch_size; + } - vlib_buffer_state_validation_hash = hash_create (0, sizeof (uword)); - vec_validate_aligned (vlib_buffer_state_validation_lock, 0, - CLIB_CACHE_LINE_BYTES); - clib_mem_set_heap (oldheap); - return 0; -} + if (n) + { + n_alloc = vlib_buffer_alloc_from_pool (vm, bufs, n, buffer_pool_index); -VLIB_INIT_FUNCTION (buffer_state_validation_init); -#endif + if (n_alloc != n) + goto alloc_fail; -#if CLI_DEBUG -struct dpdk_validate_buf_result -{ - u32 invalid; - u32 uninitialized; -}; + vlib_get_buffers_with_offset (vm, bufs, obj, n, + -(i32) sizeof (struct rte_mbuf)); + } -#define DPDK_TRAJECTORY_POISON 31 + return 0; -static void -dpdk_buffer_validate_trajectory (struct rte_mempool *mp, void *opaque, - void *obj, unsigned obj_idx) -{ - vlib_buffer_t *b; - struct dpdk_validate_buf_result *counter = opaque; - b = vlib_buffer_from_rte_mbuf ((struct rte_mbuf *) obj); - if (b->pre_data[0] != 0) +alloc_fail: + /* dpdk doesn't support partial alloc, so we need to return what we + already got */ + if (n_alloc) + vlib_buffer_pool_put (vm, buffer_pool_index, bufs, n_alloc); + obj = obj_table; + while (total) { - if (b->pre_data[0] == DPDK_TRAJECTORY_POISON) - counter->uninitialized++; - else - counter->invalid++; + vlib_get_buffer_indices_with_offset (vm, obj, bufs, batch_size, + sizeof (struct rte_mbuf)); + vlib_buffer_pool_put (vm, buffer_pool_index, bufs, batch_size); + + obj += batch_size; + total -= batch_size; } + return -ENOENT; } -int -dpdk_buffer_validate_trajectory_all (u32 * uninitialized) +CLIB_MARCH_FN_REGISTRATION (dpdk_ops_vpp_dequeue); + +#ifndef CLIB_MARCH_VARIANT + +static int +dpdk_ops_vpp_dequeue_no_cache (struct rte_mempool *mp, void **obj_table, + unsigned n) { - dpdk_main_t *dm = &dpdk_main; - struct dpdk_validate_buf_result counter = { 0 }; - int i; - - for (i = 0; i < vec_len (dm->pktmbuf_pools); i++) - rte_mempool_obj_iter (dm->pktmbuf_pools[i], - dpdk_buffer_validate_trajectory, &counter); - if (uninitialized) - *uninitialized = counter.uninitialized; - return counter.invalid; + clib_error ("bug"); + return 0; } -static void -dpdk_buffer_poison_trajectory (struct rte_mempool *mp, void *opaque, - void *obj, unsigned obj_idx) +static unsigned +dpdk_ops_vpp_get_count (const struct rte_mempool *mp) { - vlib_buffer_t *b; - b = vlib_buffer_from_rte_mbuf ((struct rte_mbuf *) obj); - b->pre_data[0] = DPDK_TRAJECTORY_POISON; + clib_warning (""); + return 0; } -void -dpdk_buffer_poison_trajectory_all (void) +static unsigned +dpdk_ops_vpp_get_count_no_cache (const struct rte_mempool *mp) { - dpdk_main_t *dm = &dpdk_main; - int i; - - for (i = 0; i < vec_len (dm->pktmbuf_pools); i++) - rte_mempool_obj_iter (dm->pktmbuf_pools[i], dpdk_buffer_poison_trajectory, - 0); + dpdk_mempool_private_t *privp; + struct rte_mempool *cmp; + privp = rte_mempool_get_priv ((struct rte_mempool *) mp); + cmp = dpdk_no_cache_mempool_by_buffer_pool_index[privp->buffer_pool_index]; + return dpdk_ops_vpp_get_count (cmp); } -#endif -static clib_error_t * -dpdk_buffer_init (vlib_main_t * vm) +clib_error_t * +dpdk_buffer_pools_create (vlib_main_t * vm) { - dpdk_buffer_main_t *dbm = &dpdk_buffer_main; - vlib_thread_main_t *tm = vlib_get_thread_main (); - - vec_validate_aligned (dbm->ptd, tm->n_vlib_mains - 1, - CLIB_CACHE_LINE_BYTES); - - dbm->vfio_container_fd = -1; - + clib_error_t *err; + vlib_buffer_pool_t *bp; + + struct rte_mempool_ops ops = { }; + + strncpy (ops.name, "vpp", 4); + ops.alloc = dpdk_ops_vpp_alloc; + ops.free = dpdk_ops_vpp_free; + ops.get_count = dpdk_ops_vpp_get_count; + ops.enqueue = CLIB_MARCH_FN_POINTER (dpdk_ops_vpp_enqueue); + ops.dequeue = CLIB_MARCH_FN_POINTER (dpdk_ops_vpp_dequeue); + rte_mempool_register_ops (&ops); + + strncpy (ops.name, "vpp-no-cache", 13); + ops.get_count = dpdk_ops_vpp_get_count_no_cache; + ops.enqueue = CLIB_MARCH_FN_POINTER (dpdk_ops_vpp_enqueue_no_cache); + ops.dequeue = dpdk_ops_vpp_dequeue_no_cache; + rte_mempool_register_ops (&ops); + + /* *INDENT-OFF* */ + vec_foreach (bp, vm->buffer_main->buffer_pools) + if (bp->start && (err = dpdk_buffer_pool_init (vm, bp))) + return err; + /* *INDENT-ON* */ return 0; } -VLIB_INIT_FUNCTION (dpdk_buffer_init); - -/* *INDENT-OFF* */ -VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = { - .vlib_buffer_fill_free_list_cb = &dpdk_buffer_fill_free_list, - .vlib_buffer_free_cb = &dpdk_buffer_free, - .vlib_buffer_free_no_next_cb = &dpdk_buffer_free_no_next, -}; -/* *INDENT-ON* */ - -#if __x86_64__ -vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx512; -vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx2; -vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx512; -vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx2; -vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx512; -vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx2; - -static void __clib_constructor -dpdk_input_multiarch_select (void) -{ - vlib_buffer_callbacks_t *cb = &__dpdk_buffer_callbacks; - if (dpdk_buffer_fill_free_list_avx512 && clib_cpu_supports_avx512f ()) - { - cb->vlib_buffer_fill_free_list_cb = dpdk_buffer_fill_free_list_avx512; - cb->vlib_buffer_free_cb = dpdk_buffer_free_avx512; - cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_avx512; - } - else if (dpdk_buffer_fill_free_list_avx2 && clib_cpu_supports_avx2 ()) - { - cb->vlib_buffer_fill_free_list_cb = dpdk_buffer_fill_free_list_avx2; - cb->vlib_buffer_free_cb = dpdk_buffer_free_avx2; - cb->vlib_buffer_free_no_next_cb = dpdk_buffer_free_no_next_avx2; - } -} -#endif +VLIB_BUFFER_SET_EXT_HDR_SIZE (sizeof (struct rte_mempool_objhdr) + + sizeof (struct rte_mbuf)); + #endif /** @endcond */ diff --git a/src/plugins/dpdk/buffer.h b/src/plugins/dpdk/buffer.h index 56bf4fff3e6..2c2579e9803 100644 --- a/src/plugins/dpdk/buffer.h +++ b/src/plugins/dpdk/buffer.h @@ -19,8 +19,10 @@ #define rte_mbuf_from_vlib_buffer(x) (((struct rte_mbuf *)x) - 1) #define vlib_buffer_from_rte_mbuf(x) ((vlib_buffer_t *)(x+1)) -clib_error_t *dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs, - unsigned socket_id); +extern struct rte_mempool **dpdk_mempool_by_buffer_pool_index; +extern struct rte_mempool **dpdk_no_cache_mempool_by_buffer_pool_index; + +clib_error_t *dpdk_buffer_pools_create (vlib_main_t * vm); #endif /* include_dpdk_buffer_h */ diff --git a/src/plugins/dpdk/device/cli.c b/src/plugins/dpdk/device/cli.c index bbe367f7ec5..209f4ed51ed 100644 --- a/src/plugins/dpdk/device/cli.c +++ b/src/plugins/dpdk/device/cli.c @@ -382,27 +382,27 @@ static clib_error_t * show_dpdk_buffer (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - struct rte_mempool *rmp; - int i; + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_pool_t *bp; - for (i = 0; i < vec_len (dpdk_main.pktmbuf_pools); i++) - { - rmp = dpdk_main.pktmbuf_pools[i]; - if (rmp) - { - unsigned count = rte_mempool_avail_count (rmp); - unsigned free_count = rte_mempool_in_use_count (rmp); + vec_foreach (bp, bm->buffer_pools) + { + struct rte_mempool *rmp = dpdk_mempool_by_buffer_pool_index[bp->index]; + if (rmp) + { + unsigned count = rte_mempool_avail_count (rmp); + unsigned free_count = rte_mempool_in_use_count (rmp); - vlib_cli_output (vm, - "name=\"%s\" available = %7d allocated = %7d total = %7d\n", - rmp->name, (u32) count, (u32) free_count, - (u32) (count + free_count)); - } - else - { - vlib_cli_output (vm, "rte_mempool is NULL (!)\n"); - } - } + vlib_cli_output (vm, + "name=\"%s\" available = %7d allocated = %7d total = %7d\n", + rmp->name, (u32) count, (u32) free_count, + (u32) (count + free_count)); + } + else + { + vlib_cli_output (vm, "rte_mempool is NULL (!)\n"); + } + } return 0; } @@ -2018,59 +2018,6 @@ VLIB_CLI_COMMAND (show_vpe_version_command, static) = { }; /* *INDENT-ON* */ -#if CLI_DEBUG - -static clib_error_t * -dpdk_validate_buffers_fn (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd_arg) -{ - u32 n_invalid_bufs = 0, uninitialized = 0; - u32 is_poison = 0, is_test = 0; - while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) - { - if (unformat (input, "poison")) - is_poison = 1; - else if (unformat (input, "trajectory")) - is_test = 1; - else - return clib_error_return (0, "unknown input `%U'", - format_unformat_error, input); - } - - if (VLIB_BUFFER_TRACE_TRAJECTORY == 0) - { - vlib_cli_output (vm, "Trajectory not enabled. Recompile with " - "VLIB_BUFFER_TRACE_TRAJECTORY 1"); - return 0; - } - if (is_poison) - { - dpdk_buffer_poison_trajectory_all (); - } - if (is_test) - { - n_invalid_bufs = dpdk_buffer_validate_trajectory_all (&uninitialized); - if (!n_invalid_bufs) - vlib_cli_output (vm, "All buffers are valid %d uninitialized", - uninitialized); - else - vlib_cli_output (vm, "Found %d invalid buffers and %d uninitialized", - n_invalid_bufs, uninitialized); - } - return 0; -} - -/* *INDENT-OFF* */ -VLIB_CLI_COMMAND (test_dpdk_buffers_command, static) = -{ - .path = "test dpdk buffers", - .short_help = "test dpdk buffers [poison] [trajectory]", - .function = dpdk_validate_buffers_fn, -}; -/* *INDENT-ON* */ - -#endif - clib_error_t * dpdk_cli_init (vlib_main_t * vm) { diff --git a/src/plugins/dpdk/device/common.c b/src/plugins/dpdk/device/common.c index 25d5802899f..b239616b7d9 100644 --- a/src/plugins/dpdk/device/common.c +++ b/src/plugins/dpdk/device/common.c @@ -40,6 +40,7 @@ void dpdk_device_setup (dpdk_device_t * xd) { dpdk_main_t *dm = &dpdk_main; + vlib_main_t *vm = vlib_get_main (); vnet_main_t *vnm = vnet_get_main (); vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, xd->sw_if_index); vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, xd->hw_if_index); @@ -116,26 +117,23 @@ dpdk_device_setup (dpdk_device_t * xd) CLIB_CACHE_LINE_BYTES); for (j = 0; j < xd->rx_q_used; j++) { - dpdk_mempool_private_t *privp; uword tidx = vnet_get_device_input_thread_index (dm->vnet_main, xd->hw_if_index, j); unsigned lcore = vlib_worker_threads[tidx].cpu_id; u16 socket_id = rte_lcore_to_socket_id (lcore); + u8 bpidx = vlib_buffer_pool_get_default_for_numa (vm, socket_id); + vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, bpidx); + struct rte_mempool *mp = dpdk_mempool_by_buffer_pool_index[bpidx]; - rv = - rte_eth_rx_queue_setup (xd->port_id, j, xd->nb_rx_desc, - xd->cpu_socket, 0, - dm->pktmbuf_pools[socket_id]); + rv = rte_eth_rx_queue_setup (xd->port_id, j, xd->nb_rx_desc, + xd->cpu_socket, 0, mp); /* retry with any other CPU socket */ if (rv < 0) - rv = - rte_eth_rx_queue_setup (xd->port_id, j, - xd->nb_rx_desc, SOCKET_ID_ANY, 0, - dm->pktmbuf_pools[socket_id]); + rv = rte_eth_rx_queue_setup (xd->port_id, j, xd->nb_rx_desc, + SOCKET_ID_ANY, 0, mp); - privp = rte_mempool_get_priv (dm->pktmbuf_pools[socket_id]); - xd->buffer_pool_for_queue[j] = privp->buffer_pool_index; + xd->buffer_pool_for_queue[j] = bp->index; if (rv < 0) dpdk_device_error (xd, "rte_eth_rx_queue_setup", rv); diff --git a/src/plugins/dpdk/device/device.c b/src/plugins/dpdk/device/device.c index 86b9a50f41b..e3520fc720e 100644 --- a/src/plugins/dpdk/device/device.c +++ b/src/plugins/dpdk/device/device.c @@ -127,11 +127,9 @@ dpdk_validate_rte_mbuf (vlib_main_t * vm, vlib_buffer_t * b, mb->pkt_len = b->current_length; mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data; first_mb->nb_segs++; - if (PREDICT_FALSE (b->n_add_refs)) - { - rte_mbuf_refcnt_update (mb, b->n_add_refs); - b->n_add_refs = 0; - } + if (PREDICT_FALSE (b->ref_count > 1)) + mb->pool = + dpdk_no_cache_mempool_by_buffer_pool_index[b->buffer_pool_index]; } } diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h index 425cf265873..14630967ea5 100644 --- a/src/plugins/dpdk/device/dpdk.h +++ b/src/plugins/dpdk/device/dpdk.h @@ -56,8 +56,6 @@ #include #include -#define NB_MBUF (16<<10) - extern vnet_device_class_t dpdk_device_class; extern vlib_node_registration_t dpdk_input_node; extern vlib_node_registration_t admin_up_down_process_node; @@ -364,7 +362,7 @@ typedef struct u8 nchannels_set_manually; u32 coremask; u32 nchannels; - u32 num_mbufs; + u32 num_crypto_mbufs; /* * format interface names ala xxxEthernet%d/%d/%d instead of @@ -443,9 +441,6 @@ typedef struct vnet_main_t *vnet_main; dpdk_config_main_t *conf; - /* mempool */ - struct rte_mempool **pktmbuf_pools; - /* API message ID base */ u16 msg_id_base; diff --git a/src/plugins/dpdk/device/dpdk_priv.h b/src/plugins/dpdk/device/dpdk_priv.h index 1956cca19c3..e0e06826935 100644 --- a/src/plugins/dpdk/device/dpdk_priv.h +++ b/src/plugins/dpdk/device/dpdk_priv.h @@ -49,14 +49,6 @@ _(file-prefix) \ _(vdev) \ _(log-level) -typedef struct -{ - /* must be first */ - struct rte_pktmbuf_pool_private mbp_priv; - u8 buffer_pool_index; -} dpdk_mempool_private_t; - - static inline void dpdk_get_xstats (dpdk_device_t * xd) { diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index dcff0e52293..fa3b691fca4 100644 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -151,60 +151,6 @@ dpdk_device_lock_init (dpdk_device_t * xd) } } -static struct rte_mempool_ops * -get_ops_by_name (char *ops_name) -{ - u32 i; - - for (i = 0; i < rte_mempool_ops_table.num_ops; i++) - { - if (!strcmp (ops_name, rte_mempool_ops_table.ops[i].name)) - return &rte_mempool_ops_table.ops[i]; - } - - return 0; -} - -static int -dpdk_ring_alloc (struct rte_mempool *mp) -{ - u32 rg_flags = 0, count; - i32 ret; - char rg_name[RTE_RING_NAMESIZE]; - struct rte_ring *r; - - ret = snprintf (rg_name, sizeof (rg_name), RTE_MEMPOOL_MZ_FORMAT, mp->name); - if (ret < 0 || ret >= (i32) sizeof (rg_name)) - return -ENAMETOOLONG; - - /* ring flags */ - if (mp->flags & MEMPOOL_F_SP_PUT) - rg_flags |= RING_F_SP_ENQ; - if (mp->flags & MEMPOOL_F_SC_GET) - rg_flags |= RING_F_SC_DEQ; - - count = rte_align32pow2 (mp->size + 1); - /* - * Allocate the ring that will be used to store objects. - * Ring functions will return appropriate errors if we are - * running as a secondary process etc., so no checks made - * in this function for that condition. - */ - /* XXX can we get memory from the right socket? */ - r = clib_mem_alloc_aligned (rte_ring_get_memsize (count), - CLIB_CACHE_LINE_BYTES); - - /* XXX rte_ring_lookup will not work */ - - ret = rte_ring_init (r, rg_name, count, rg_flags); - if (ret) - return ret; - - mp->pool_data = r; - - return 0; -} - static int dpdk_port_crc_strip_enabled (dpdk_device_t * xd) { @@ -220,7 +166,6 @@ dpdk_lib_init (dpdk_main_t * dm) { u32 nports; u32 mtu, max_rx_frame; - u32 nb_desc = 0; int i; clib_error_t *error; vlib_main_t *vm = vlib_get_main (); @@ -631,9 +576,6 @@ dpdk_lib_init (dpdk_main_t * dm) dq->queue_id = 0; } - /* count the number of descriptors used for this device */ - nb_desc += xd->nb_rx_desc + xd->nb_tx_desc * xd->tx_q_used; - error = ethernet_register_interface (dm->vnet_main, dpdk_device_class.index, xd->device_index, /* ethernet address */ addr, @@ -811,10 +753,6 @@ dpdk_lib_init (dpdk_main_t * dm) } /* *INDENT-ON* */ - if (nb_desc > dm->conf->num_mbufs) - dpdk_log_err ("%d mbufs allocated but total rx/tx ring size is %d\n", - dm->conf->num_mbufs, nb_desc); - return 0; } @@ -1209,7 +1147,8 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) } else if (unformat (input, "num-mem-channels %d", &conf->nchannels)) conf->nchannels_set_manually = 0; - else if (unformat (input, "num-mbufs %d", &conf->num_mbufs)) + else if (unformat (input, "num-crypto-mbufs %d", + &conf->num_crypto_mbufs)) ; else if (unformat (input, "uio-driver %s", &conf->uio_driver_name)) ; @@ -1452,36 +1391,10 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) if (ret < 0) return clib_error_return (0, "rte_eal_init returned %d", ret); - /* set custom ring memory allocator */ - { - struct rte_mempool_ops *ops = NULL; - - ops = get_ops_by_name ("ring_sp_sc"); - ops->alloc = dpdk_ring_alloc; - - ops = get_ops_by_name ("ring_mp_sc"); - ops->alloc = dpdk_ring_alloc; - - ops = get_ops_by_name ("ring_sp_mc"); - ops->alloc = dpdk_ring_alloc; - - ops = get_ops_by_name ("ring_mp_mc"); - ops->alloc = dpdk_ring_alloc; - } - /* main thread 1st */ - error = dpdk_buffer_pool_create (vm, conf->num_mbufs, rte_socket_id ()); - if (error) + if ((error = dpdk_buffer_pools_create (vm))) return error; - for (i = 0; i < RTE_MAX_LCORE; i++) - { - error = dpdk_buffer_pool_create (vm, conf->num_mbufs, - rte_lcore_to_socket_id (i)); - if (error) - return error; - } - done: return error; } @@ -1768,7 +1681,6 @@ dpdk_init (vlib_main_t * vm) dm->conf = &dpdk_config_main; dm->conf->nchannels = 4; - dm->conf->num_mbufs = dm->conf->num_mbufs ? dm->conf->num_mbufs : NB_MBUF; vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet"); vec_add1 (dm->conf->eal_init_args, (u8 *) "--in-memory"); diff --git a/src/plugins/dpdk/ipsec/ipsec.c b/src/plugins/dpdk/ipsec/ipsec.c index ade454f22cc..dd64f6db568 100644 --- a/src/plugins/dpdk/ipsec/ipsec.c +++ b/src/plugins/dpdk/ipsec/ipsec.c @@ -24,6 +24,7 @@ dpdk_crypto_main_t dpdk_crypto_main; #define EMPTY_STRUCT {0} +#define NUM_CRYPTO_MBUFS 16384 static void algos_init (u32 n_mains) @@ -835,11 +836,12 @@ crypto_create_crypto_op_pool (vlib_main_t * vm, u8 numa) pool_name = format (0, "crypto_pool_numa%u%c", numa, 0); - mp = - rte_mempool_create ((char *) pool_name, - conf->num_mbufs, - crypto_op_len (), 512, pool_priv_size, NULL, NULL, - crypto_op_init, NULL, numa, 0); + if (conf->num_crypto_mbufs == 0) + conf->num_crypto_mbufs = NUM_CRYPTO_MBUFS; + + mp = rte_mempool_create ((char *) pool_name, conf->num_crypto_mbufs, + crypto_op_len (), 512, pool_priv_size, NULL, NULL, + crypto_op_init, NULL, numa, 0); vec_free (pool_name); diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c index 6e45b573f2c..7f29f4d7f87 100644 --- a/src/plugins/memif/memif.c +++ b/src/plugins/memif/memif.c @@ -185,6 +185,7 @@ memif_int_fd_read_ready (clib_file_t * uf) clib_error_t * memif_connect (memif_if_t * mif) { + vlib_main_t *vm = vlib_get_main (); vnet_main_t *vnm = vnet_get_main (); clib_file_t template = { 0 }; memif_region_t *mr; @@ -235,6 +236,7 @@ memif_connect (memif_if_t * mif) vec_foreach_index (i, mif->rx_queues) { memif_queue_t *mq = vec_elt_at_index (mif->rx_queues, i); + u32 ti; int rv; mq->ring = mif->regions[mq->region].shm + mq->offset; @@ -254,6 +256,9 @@ memif_connect (memif_if_t * mif) memif_file_add (&mq->int_clib_file_index, &template); } vnet_hw_interface_assign_rx_thread (vnm, mif->hw_if_index, i, ~0); + ti = vnet_get_device_input_thread_index (vnm, mif->hw_if_index, i); + mq->buffer_pool_index = + vlib_buffer_pool_get_default_for_numa (vm, vlib_mains[ti]->numa_node); rv = vnet_hw_interface_set_rx_mode (vnm, mif->hw_if_index, i, VNET_HW_INTERFACE_RX_MODE_DEFAULT); if (rv) diff --git a/src/plugins/memif/node.c b/src/plugins/memif/node.c index 490c60356db..347112892be 100644 --- a/src/plugins/memif/node.c +++ b/src/plugins/memif/node.c @@ -280,7 +280,8 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, /* allocate free buffers */ vec_validate_aligned (ptd->buffers, n_buffers - 1, CLIB_CACHE_LINE_BYTES); - n_alloc = vlib_buffer_alloc (vm, ptd->buffers, n_buffers); + n_alloc = vlib_buffer_alloc_from_pool (vm, ptd->buffers, n_buffers, + mq->buffer_pool_index); if (PREDICT_FALSE (n_alloc != n_buffers)) { if (n_alloc) @@ -343,6 +344,7 @@ memif_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vnet_buffer (&ptd->buffer_template)->feature_arc_index = 0; ptd->buffer_template.current_data = start_offset; ptd->buffer_template.current_config_index = 0; + ptd->buffer_template.buffer_pool_index = mq->buffer_pool_index; if (mode == MEMIF_INTERFACE_MODE_ETHERNET) { @@ -783,8 +785,9 @@ refill: clib_memset (dt, 0, sizeof (memif_desc_t)); dt->length = buffer_length; - n_alloc = vlib_buffer_alloc_to_ring (vm, mq->buffers, head & mask, - ring_size, n_slots); + n_alloc = vlib_buffer_alloc_to_ring_from_pool (vm, mq->buffers, head & mask, + ring_size, n_slots, + mq->buffer_pool_index); if (PREDICT_FALSE (n_alloc != n_slots)) { diff --git a/src/plugins/memif/private.h b/src/plugins/memif/private.h index a938b85e7da..2aa1f5a0cdc 100644 --- a/src/plugins/memif/private.h +++ b/src/plugins/memif/private.h @@ -123,6 +123,7 @@ typedef struct u16 last_head; u16 last_tail; u32 *buffers; + u8 buffer_pool_index; /* interrupts */ int int_fd; diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c index 643f2c50342..3e194113675 100644 --- a/src/vlib/buffer.c +++ b/src/vlib/buffer.c @@ -43,18 +43,24 @@ * Allocate/free network buffers. */ +#include #include #include -vlib_buffer_callbacks_t *vlib_buffer_callbacks = 0; +#define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA 16384 +#define VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV 8192 -/* when running unpriviledged we are limited by RLIMIT_MEMLOCK which is - typically set to 16MB so setting default size for buffer memory to 14MB - */ -static u32 vlib_buffer_physmem_sz = 14 << 20; +#ifdef CLIB_HAVE_VEC128 +/* Assumptions by vlib_buffer_free_inline: */ +STATIC_ASSERT_FITS_IN (vlib_buffer_t, flags, 16); +STATIC_ASSERT_FITS_IN (vlib_buffer_t, ref_count, 16); +STATIC_ASSERT_FITS_IN (vlib_buffer_t, buffer_pool_index, 16); +#endif -/* logging */ -static vlib_log_class_t buffer_log_default; +/* Make sure that buffer template size is not accidentally changed */ +STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64); + +u16 __vlib_buffer_external_hdr_size = 0; uword vlib_buffer_length_in_chain_slow_path (vlib_main_t * vm, @@ -86,8 +92,8 @@ format_vlib_buffer (u8 * s, va_list * args) foreach_vlib_buffer_flag #undef _ s = format (s, "current data %d, length %d, buffer-pool %d, " - "clone-count %u", b->current_data, b->current_length, - b->buffer_pool_index, b->n_add_refs); + "ref-count %u", b->current_data, b->current_length, + b->buffer_pool_index, b->ref_count); if (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID) s = format (s, ", totlen-nifb %d", @@ -107,9 +113,9 @@ format_vlib_buffer (u8 * s, va_list * args) b = vlib_get_buffer (vm, next_buffer); s = - format (s, "\n%Unext-buffer 0x%x, segment length %d, clone-count %u", + format (s, "\n%Unext-buffer 0x%x, segment length %d, ref-count %u", format_white_space, indent, next_buffer, b->current_length, - b->n_add_refs); + b->ref_count); } return s; @@ -177,14 +183,18 @@ vlib_validate_buffer_helper (vlib_main_t * vm, u32 bi, uword follow_buffer_next, uword ** unique_hash) { + vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_t *b = vlib_get_buffer (vm, bi); + if (vec_len (bm->buffer_pools) <= b->buffer_pool_index) + return format (0, "unknown buffer pool 0x%x", b->buffer_pool_index); + if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE) return format (0, "current data %d before pre-data", b->current_data); if (b->current_data + b->current_length > VLIB_BUFFER_DATA_SIZE) - return format (0, "%d-%d beyond end of buffer %d", - b->current_data, b->current_length, VLIB_BUFFER_DATA_SIZE); + return format (0, "%d-%d beyond end of buffer %d", b->current_data, + b->current_length, VLIB_BUFFER_DATA_SIZE); if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT)) { @@ -306,15 +316,13 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm, uword n_buffers, vlib_buffer_known_state_t expected_state) { + vlib_buffer_main_t *bm = vm->buffer_main; u32 *b; uword i, bi, is_free; if (CLIB_DEBUG == 0) return; - if (vlib_buffer_callbacks) - return; - is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED; b = buffers; for (i = 0; i < n_buffers; i++) @@ -324,287 +332,26 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm, bi = b[0]; b += 1; known = vlib_buffer_is_known (vm, bi); + + if (known == VLIB_BUFFER_UNKNOWN && + expected_state == VLIB_BUFFER_KNOWN_FREE) + known = VLIB_BUFFER_KNOWN_FREE; + if (known != expected_state) { - ASSERT (0); vlib_panic_with_msg (vm, "%s %U buffer 0x%x", is_free ? "freeing" : "allocating", format_vlib_buffer_known_state, known, bi); } - vlib_buffer_set_known_state (vm, bi, is_free ? VLIB_BUFFER_KNOWN_FREE : - VLIB_BUFFER_KNOWN_ALLOCATED); - } -} - -/* Add buffer free list. */ -static vlib_buffer_free_list_index_t -vlib_buffer_create_free_list_helper (vlib_main_t * vm, - u32 n_data_bytes, - u32 is_public, u32 is_default, u8 * name) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_free_list_t *f; - int i; - - ASSERT (vlib_get_thread_index () == 0); - - if (!is_default && pool_elts (vm->buffer_free_list_pool) == 0) - { - vlib_buffer_free_list_index_t default_free_free_list_index; - - /* *INDENT-OFF* */ - default_free_free_list_index = - vlib_buffer_create_free_list_helper - (vm, - /* default buffer size */ VLIB_BUFFER_DATA_SIZE, - /* is_public */ 1, - /* is_default */ 1, - (u8 *) "default"); - /* *INDENT-ON* */ - ASSERT (default_free_free_list_index == - VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); - - if (n_data_bytes == VLIB_BUFFER_DATA_SIZE && is_public) - return default_free_free_list_index; - } - - pool_get_aligned (vm->buffer_free_list_pool, f, CLIB_CACHE_LINE_BYTES); - - clib_memset (f, 0, sizeof (f[0])); - f->index = f - vm->buffer_free_list_pool; - vec_validate (f->buffers, 0); - vec_reset_length (f->buffers); - f->min_n_buffers_each_alloc = VLIB_FRAME_SIZE; - f->buffer_pool_index = 0; - f->name = clib_mem_is_vec (name) ? name : format (0, "%s", name); - - /* Setup free buffer template. */ - f->buffer_init_template.n_add_refs = 0; - - if (is_public) - { - uword *p = hash_get (bm->free_list_by_size, VLIB_BUFFER_DATA_SIZE); - if (!p) - hash_set (bm->free_list_by_size, VLIB_BUFFER_DATA_SIZE, f->index); - } - - for (i = 1; i < vec_len (vlib_mains); i++) - { - vlib_main_t *wvm = vlib_mains[i]; - vlib_buffer_free_list_t *wf; - pool_get_aligned (wvm->buffer_free_list_pool, - wf, CLIB_CACHE_LINE_BYTES); - ASSERT (f - vm->buffer_free_list_pool == - wf - wvm->buffer_free_list_pool); - wf[0] = f[0]; - wf->buffers = 0; - vec_validate (wf->buffers, 0); - vec_reset_length (wf->buffers); - wf->n_alloc = 0; - } - - return f->index; -} - -vlib_buffer_free_list_index_t -vlib_buffer_create_free_list (vlib_main_t * vm, u32 n_data_bytes, - char *fmt, ...) -{ - va_list va; - u8 *name; - - va_start (va, fmt); - name = va_format (0, fmt, &va); - va_end (va); - - return vlib_buffer_create_free_list_helper (vm, n_data_bytes, - /* is_public */ 0, - /* is_default */ 0, - name); -} - -static_always_inline void * -vlib_buffer_pool_get_buffer (vlib_main_t * vm, vlib_buffer_pool_t * bp) -{ - return vlib_physmem_alloc_from_map (vm, bp->physmem_map_index, - bp->buffer_size, CLIB_CACHE_LINE_BYTES); -} - -/* Make sure free list has at least given number of free buffers. */ -static uword -vlib_buffer_fill_free_list_internal (vlib_main_t * vm, - vlib_buffer_free_list_t * fl, - uword min_free_buffers) -{ - vlib_buffer_t *b; - vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, fl->buffer_pool_index); - int n; - u32 *bi; - u32 n_alloc = 0; - - /* Already have enough free buffers on free list? */ - n = min_free_buffers - vec_len (fl->buffers); - if (n <= 0) - return min_free_buffers; - - if (vec_len (bp->buffers) > 0) - { - int n_copy, n_left; - clib_spinlock_lock (&bp->lock); - n_copy = clib_min (vec_len (bp->buffers), n); - n_left = vec_len (bp->buffers) - n_copy; - vec_add_aligned (fl->buffers, bp->buffers + n_left, n_copy, - CLIB_CACHE_LINE_BYTES); - _vec_len (bp->buffers) = n_left; - clib_spinlock_unlock (&bp->lock); - n = min_free_buffers - vec_len (fl->buffers); - if (n <= 0) - return min_free_buffers; - } - - /* Always allocate round number of buffers. */ - n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32)); - - /* Always allocate new buffers in reasonably large sized chunks. */ - n = clib_max (n, fl->min_n_buffers_each_alloc); - - clib_spinlock_lock (&bp->lock); - while (n_alloc < n) - { - if ((b = vlib_buffer_pool_get_buffer (vm, bp)) == 0) - goto done; - - n_alloc += 1; - - vec_add2_aligned (fl->buffers, bi, 1, CLIB_CACHE_LINE_BYTES); - bi[0] = vlib_get_buffer_index (vm, b); - - if (CLIB_DEBUG > 0) - vlib_buffer_set_known_state (vm, bi[0], VLIB_BUFFER_KNOWN_FREE); - - clib_memset (b, 0, sizeof (vlib_buffer_t)); - - if (fl->buffer_init_function) - fl->buffer_init_function (vm, fl, bi, 1); - } - -done: - clib_spinlock_unlock (&bp->lock); - fl->n_alloc += n_alloc; - return n_alloc; -} - -void * -vlib_set_buffer_free_callback (vlib_main_t * vm, void *fp) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - void *rv = bm->buffer_free_callback; - - bm->buffer_free_callback = fp; - return rv; -} - -static_always_inline void -recycle_or_free (vlib_main_t * vm, vlib_buffer_main_t * bm, u32 bi, - vlib_buffer_t * b, u32 follow_buffer_next) -{ - vlib_buffer_free_list_t *fl; - u32 flags, next; - - fl = pool_elt_at_index (vm->buffer_free_list_pool, - VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); - - do - { - vlib_buffer_t *nb = vlib_get_buffer (vm, bi); - flags = nb->flags; - next = nb->next_buffer; - if (nb->n_add_refs) - nb->n_add_refs--; - else - { - vlib_buffer_validate_alloc_free (vm, &bi, 1, - VLIB_BUFFER_KNOWN_ALLOCATED); - vlib_buffer_add_to_free_list (vm, fl, bi, 1); - } - bi = next; - } - while (follow_buffer_next && (flags & VLIB_BUFFER_NEXT_PRESENT)); -} - -static_always_inline void -vlib_buffer_free_inline (vlib_main_t * vm, - u32 * buffers, u32 n_buffers, u32 follow_buffer_next) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_t *p, *b0, *b1, *b2, *b3; - int i = 0; - u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers, - u32 follow_buffer_next); - - cb = bm->buffer_free_callback; - - if (PREDICT_FALSE (cb != 0)) - n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next); - - if (!n_buffers) - return; - - while (i + 11 < n_buffers) - { - p = vlib_get_buffer (vm, buffers[i + 8]); - vlib_prefetch_buffer_header (p, LOAD); - p = vlib_get_buffer (vm, buffers[i + 9]); - vlib_prefetch_buffer_header (p, LOAD); - p = vlib_get_buffer (vm, buffers[i + 10]); - vlib_prefetch_buffer_header (p, LOAD); - p = vlib_get_buffer (vm, buffers[i + 11]); - vlib_prefetch_buffer_header (p, LOAD); - - b0 = vlib_get_buffer (vm, buffers[i]); - b1 = vlib_get_buffer (vm, buffers[i + 1]); - b2 = vlib_get_buffer (vm, buffers[i + 2]); - b3 = vlib_get_buffer (vm, buffers[i + 3]); - - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b1); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b2); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b3); - - recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next); - recycle_or_free (vm, bm, buffers[i + 1], b1, follow_buffer_next); - recycle_or_free (vm, bm, buffers[i + 2], b2, follow_buffer_next); - recycle_or_free (vm, bm, buffers[i + 3], b3, follow_buffer_next); - - i += 4; - } - - while (i < n_buffers) - { - b0 = vlib_get_buffer (vm, buffers[i]); - VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b0); - recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next); - i++; + clib_spinlock_lock (&bm->buffer_known_hash_lockp); + hash_set (bm->buffer_known_hash, bi, is_free ? VLIB_BUFFER_KNOWN_FREE : + VLIB_BUFFER_KNOWN_ALLOCATED); + clib_spinlock_unlock (&bm->buffer_known_hash_lockp); } } -static void -vlib_buffer_free_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers) -{ - vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ - 1); -} - -static void -vlib_buffer_free_no_next_internal (vlib_main_t * vm, u32 * buffers, - u32 n_buffers) -{ - vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */ - 0); -} - void vlib_packet_template_init (vlib_main_t * vm, vlib_packet_template_t * t, @@ -624,7 +371,6 @@ vlib_packet_template_init (vlib_main_t * vm, vec_add (t->packet_data, packet_data, n_packet_data_bytes); t->min_n_buffers_each_alloc = min_n_buffers_each_alloc; - vlib_worker_thread_barrier_release (vm); } @@ -706,8 +452,6 @@ out_of_buffers: u16 vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, - vlib_buffer_free_list_index_t - free_list_index, vlib_buffer_t * first, vlib_buffer_t ** last, void *data, u16 data_len) @@ -721,9 +465,8 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, u16 max = n_buffer_bytes - l->current_length - l->current_data; if (max == 0) { - if (1 != - vlib_buffer_alloc_from_free_list (vm, &l->next_buffer, 1, - free_list_index)) + if (1 != vlib_buffer_alloc_from_pool (vm, &l->next_buffer, 1, + first->buffer_pool_index)) return copied; *last = l = vlib_buffer_chain_buffer (vm, l, l->next_buffer); max = n_buffer_bytes - l->current_length - l->current_data; @@ -739,14 +482,26 @@ vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, return copied; } -u8 -vlib_buffer_register_physmem_map (vlib_main_t * vm, u32 physmem_map_index) +clib_error_t * +vlib_buffer_pool_create (vlib_main_t * vm, u8 index, char *name, + u32 data_size, u32 physmem_map_index) { vlib_buffer_main_t *bm = vm->buffer_main; - vlib_buffer_pool_t *p; + vlib_buffer_pool_t *bp; vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index); uword start = pointer_to_uword (m->base); uword size = (uword) m->n_pages << m->log2_page_size; + u32 alloc_size, n_alloc_per_page;; + + vec_validate_aligned (bm->buffer_pools, index, CLIB_CACHE_LINE_BYTES); + bp = vec_elt_at_index (bm->buffer_pools, index); + + if (bp->start) + return clib_error_return (0, "buffer with index %u already exists", + index); + + if (index >= 255) + return clib_error_return (0, "buffer index must be < 255", index); if (bm->buffer_mem_size == 0) { @@ -773,36 +528,77 @@ vlib_buffer_register_physmem_map (vlib_main_t * vm, u32 physmem_map_index) clib_panic ("buffer memory size out of range!"); } - vec_add2 (bm->buffer_pools, p, 1); - p->start = start; - p->size = size; - p->physmem_map_index = physmem_map_index; + bp->start = start; + bp->size = size; + bp->index = bp - bm->buffer_pools; + bp->buffer_template.buffer_pool_index = bp->index; + bp->buffer_template.ref_count = 1; + bp->physmem_map_index = physmem_map_index; + bp->name = format (0, "%s%c", name, 0); + bp->data_size = data_size; + bp->numa_node = m->numa_node; - ASSERT (p - bm->buffer_pools < 256); - return p - bm->buffer_pools; -} + vec_validate_aligned (bp->threads, vec_len (vlib_mains) - 1, + CLIB_CACHE_LINE_BYTES); -static u8 * -format_vlib_buffer_free_list (u8 * s, va_list * va) -{ - vlib_buffer_free_list_t *f = va_arg (*va, vlib_buffer_free_list_t *); - u32 threadnum = va_arg (*va, u32); - uword bytes_alloc, bytes_free, n_free, size; + alloc_size = data_size + sizeof (vlib_buffer_t) + bm->ext_hdr_size; + n_alloc_per_page = (1ULL << m->log2_page_size) / alloc_size; + + /* preallocate buffer indices memory */ + vec_validate_aligned (bp->buffers, m->n_pages * n_alloc_per_page, + CLIB_CACHE_LINE_BYTES); + vec_reset_length (bp->buffers); + + clib_spinlock_init (&bp->lock); + + while (1) + { + u8 *p; + u32 bi; + + p = vlib_physmem_alloc_from_map (vm, bp->physmem_map_index, alloc_size, + CLIB_CACHE_LINE_BYTES); + + if (p == 0) + break; + + p += bm->ext_hdr_size; + + vlib_buffer_copy_template ((vlib_buffer_t *) p, &bp->buffer_template); - if (!f) - return format (s, "%=7s%=30s%=12s%=12s%=12s%=12s%=12s%=12s", - "Thread", "Name", "Index", "Size", "Alloc", "Free", - "#Alloc", "#Free"); + bi = vlib_get_buffer_index (vm, (vlib_buffer_t *) p); - size = sizeof (vlib_buffer_t) + VLIB_BUFFER_DATA_SIZE; - n_free = vec_len (f->buffers); - bytes_alloc = size * f->n_alloc; - bytes_free = size * n_free; + vec_add1_aligned (bp->buffers, bi, CLIB_CACHE_LINE_BYTES); + + bp->n_buffers += 1; + } + + return 0; +} - s = format (s, "%7d%30v%12d%12d%=12U%=12U%=12d%=12d", threadnum, - f->name, f->index, VLIB_BUFFER_DATA_SIZE, - format_memory_size, bytes_alloc, - format_memory_size, bytes_free, f->n_alloc, n_free); +static u8 * +format_vlib_buffer_pool (u8 * s, va_list * va) +{ + vlib_main_t *vm = va_arg (*va, vlib_main_t *); + vlib_buffer_pool_t *bp = va_arg (*va, vlib_buffer_pool_t *); + vlib_buffer_pool_thread_t *bpt; + u32 cached = 0; + + if (!bp) + return format (s, "%-20s%=6s%=6s%=6s%=11s%=6s%=8s%=8s%=8s", + "Pool Name", "Index", "NUMA", "Size", "Data Size", + "Total", "Avail", "Cached", "Used"); + + /* *INDENT-OFF* */ + vec_foreach (bpt, bp->threads) + cached += vec_len (bpt->cached_buffers); + /* *INDENT-ON* */ + + s = format (s, "%-20s%=6d%=6d%=6u%=11u%=6u%=8u%=8u%=8u", + bp->name, bp->index, bp->numa_node, bp->data_size + + sizeof (vlib_buffer_t) + vm->buffer_main->ext_hdr_size, + bp->data_size, bp->n_buffers, vec_len (bp->buffers), cached, + bp->n_buffers - vec_len (bp->buffers) - cached); return s; } @@ -811,25 +607,15 @@ static clib_error_t * show_buffers (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { - vlib_buffer_free_list_t *f; - vlib_main_t *curr_vm; - u32 vm_index = 0; - - vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, 0, 0); - - do - { - curr_vm = vlib_mains[vm_index]; + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_pool_t *bp; - /* *INDENT-OFF* */ - pool_foreach (f, curr_vm->buffer_free_list_pool, ({ - vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, f, vm_index); - })); - /* *INDENT-ON* */ + vlib_cli_output (vm, "%U", format_vlib_buffer_pool, vm, 0); - vm_index++; - } - while (vm_index < vec_len (vlib_mains)); + /* *INDENT-OFF* */ + vec_foreach (bp, bm->buffer_pools) + vlib_cli_output (vm, "%U", format_vlib_buffer_pool, vm, bp); + /* *INDENT-ON* */ return 0; } @@ -843,71 +629,134 @@ VLIB_CLI_COMMAND (show_buffers_command, static) = { /* *INDENT-ON* */ clib_error_t * -vlib_buffer_main_init (struct vlib_main_t * vm) +vlib_buffer_worker_init (vlib_main_t * vm) { - vlib_buffer_main_t *bm; - clib_error_t *error; - u32 physmem_map_index; - u8 pool_index; - int log2_page_size = 0; - - buffer_log_default = vlib_log_register_class ("buffer", 0); - - bm = vm->buffer_main = clib_mem_alloc (sizeof (bm[0])); - clib_memset (bm, 0, sizeof (bm[0])); + vlib_buffer_main_t *bm = vm->buffer_main; + vlib_buffer_pool_t *bp; - if (vlib_buffer_callbacks) + /* *INDENT-OFF* */ + vec_foreach (bp, bm->buffer_pools) { - /* external plugin has registered own buffer callbacks - so we just copy them and quit */ - clib_memcpy_fast (&bm->cb, vlib_buffer_callbacks, - sizeof (vlib_buffer_callbacks_t)); - bm->callbacks_registered = 1; - return 0; + clib_spinlock_lock (&bp->lock); + vec_validate_aligned (bp->threads, vec_len (vlib_mains) - 1, + CLIB_CACHE_LINE_BYTES); + clib_spinlock_unlock (&bp->lock); } + /* *INDENT-ON* */ - bm->cb.vlib_buffer_fill_free_list_cb = &vlib_buffer_fill_free_list_internal; - bm->cb.vlib_buffer_free_cb = &vlib_buffer_free_internal; - bm->cb.vlib_buffer_free_no_next_cb = &vlib_buffer_free_no_next_internal; - clib_spinlock_init (&bm->buffer_known_hash_lockp); + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (vlib_buffer_worker_init); + +static clib_error_t * +vlib_buffer_main_init_numa_node (struct vlib_main_t *vm, u32 numa_node) +{ + vlib_buffer_main_t *bm = vm->buffer_main; + clib_error_t *error; + u32 physmem_map_index; + uword n_pages, pagesize; + u32 buffers_per_numa; + u32 buffer_size = CLIB_CACHE_LINE_ROUND (bm->ext_hdr_size + + sizeof (vlib_buffer_t) + + VLIB_BUFFER_DATA_SIZE); + u8 *name; + + pagesize = clib_mem_get_default_hugepage_size (); + name = format (0, "buffers-numa-%d%c", numa_node, 0); + + buffers_per_numa = bm->buffers_per_numa ? bm->buffers_per_numa : + VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA; retry: - error = vlib_physmem_shared_map_create (vm, "buffers", - vlib_buffer_physmem_sz, - log2_page_size, - CLIB_PMALLOC_NUMA_LOCAL, + n_pages = (buffers_per_numa - 1) / (pagesize / buffer_size) + 1; + error = vlib_physmem_shared_map_create (vm, (char *) name, + n_pages * pagesize, + min_log2 (pagesize), numa_node, &physmem_map_index); - if (error && log2_page_size == 0) + if (error && pagesize != clib_mem_get_page_size ()) { - vlib_log_warn (buffer_log_default, "%U", format_clib_error, error); + vlib_log_warn (bm->log_default, "%U", format_clib_error, error); clib_error_free (error); - vlib_log_warn (buffer_log_default, "falling back to non-hugepage " + vlib_log_warn (bm->log_default, "falling back to non-hugepage " "backed buffer pool"); - log2_page_size = min_log2 (clib_mem_get_page_size ()); + pagesize = clib_mem_get_page_size (); + buffers_per_numa = bm->buffers_per_numa ? bm->buffers_per_numa : + VLIB_BUFFER_DEFAULT_BUFFERS_PER_NUMA_UNPRIV; goto retry; } if (error) return error; - pool_index = vlib_buffer_register_physmem_map (vm, physmem_map_index); - vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, pool_index); - clib_spinlock_init (&bp->lock); - bp->buffer_size = VLIB_BUFFER_DATA_SIZE + sizeof (vlib_buffer_t); + vec_reset_length (name); + name = format (name, "default-numa-%d%c", numa_node, 0); - return 0; + return vlib_buffer_pool_create (vm, numa_node, (char *) name, + VLIB_BUFFER_DATA_SIZE, physmem_map_index); +} + +clib_error_t * +vlib_buffer_main_init (struct vlib_main_t *vm) +{ + vlib_buffer_main_t *bm; + clib_error_t *err; + clib_bitmap_t *bmp = 0; + u32 numa_node; + + if (vm->buffer_main == 0) + { + vm->buffer_main = clib_mem_alloc (sizeof (bm[0])); + clib_memset (vm->buffer_main, 0, sizeof (bm[0])); + } + + bm = vm->buffer_main; + bm->log_default = vlib_log_register_class ("buffer", 0); + bm->ext_hdr_size = __vlib_buffer_external_hdr_size; + + clib_spinlock_init (&bm->buffer_known_hash_lockp); + + err = clib_sysfs_read ("/sys/devices/system/node/possible", "%U", + unformat_bitmap_list, &bmp); + if (err) + { + /* no info from sysfs, assuming that only numa 0 exists */ + clib_error_free (err); + bmp = clib_bitmap_set (bmp, 0, 1); + } + + /* *INDENT-OFF* */ + clib_bitmap_foreach (numa_node, bmp, { + if ((err = vlib_buffer_main_init_numa_node(vm, numa_node))) + goto done; + }); + /* *INDENT-ON* */ + + bm->n_numa_nodes = clib_bitmap_last_set (bmp) + 1; + +done: + vec_free (bmp); + return err; } static clib_error_t * vlib_buffers_configure (vlib_main_t * vm, unformat_input_t * input) { - u32 size_in_mb; + vlib_buffer_main_t *bm; + + if (vm->buffer_main == 0) + { + vm->buffer_main = clib_mem_alloc (sizeof (bm[0])); + clib_memset (vm->buffer_main, 0, sizeof (bm[0])); + } + + bm = vm->buffer_main; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { - if (unformat (input, "memory-size-in-mb %d", &size_in_mb)) - vlib_buffer_physmem_sz = size_in_mb << 20; + if (unformat (input, "buffers-per-numa %u", &bm->buffers_per_numa)) + ; else return unformat_parse_error (input); } diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h index f7fdf208a83..1adde73d4ee 100644 --- a/src/vlib/buffer.h +++ b/src/vlib/buffer.h @@ -59,8 +59,6 @@ /* Amount of head buffer data copied to each replica head buffer */ #define VLIB_BUFFER_CLONE_HEAD_SIZE (256) -typedef u8 vlib_buffer_free_list_index_t; - /** \file vlib buffer structure definition and a few select access methods. This structure and the buffer allocation @@ -127,8 +125,8 @@ typedef union /** Generic flow identifier */ u32 flow_id; - /** Number of additional references to this buffer. */ - u8 n_add_refs; + /** Reference count for this buffer. */ + volatile u8 ref_count; /** index of buffer pool this buffer belongs. */ u8 buffer_pool_index; @@ -367,66 +365,32 @@ vlib_buffer_pull (vlib_buffer_t * b, u8 size) /* Forward declaration. */ struct vlib_main_t; -typedef struct vlib_buffer_free_list_t -{ - /* Template buffer used to initialize first 16 bytes of buffers - allocated on this free list. */ - vlib_buffer_t buffer_init_template; - - /* Our index into vlib_main_t's buffer_free_list_pool. */ - vlib_buffer_free_list_index_t index; - - /* Number of buffers to allocate when we need to allocate new buffers */ - u32 min_n_buffers_each_alloc; - - /* Total number of buffers allocated from this free list. */ - u32 n_alloc; - - /* Vector of free buffers. Each element is a byte offset into I/O heap. */ - u32 *buffers; - - /* index of buffer pool used to get / put buffers */ - u8 buffer_pool_index; - - /* Free list name. */ - u8 *name; - - /* Callback functions to initialize newly allocated buffers. - If null buffers are zeroed. */ - void (*buffer_init_function) (struct vlib_main_t * vm, - struct vlib_buffer_free_list_t * fl, - u32 * buffers, u32 n_buffers); - - uword buffer_init_function_opaque; -} __attribute__ ((aligned (16))) vlib_buffer_free_list_t; - -typedef uword (vlib_buffer_fill_free_list_cb_t) (struct vlib_main_t * vm, - vlib_buffer_free_list_t * fl, - uword min_free_buffers); -typedef void (vlib_buffer_free_cb_t) (struct vlib_main_t * vm, u32 * buffers, - u32 n_buffers); -typedef void (vlib_buffer_free_no_next_cb_t) (struct vlib_main_t * vm, - u32 * buffers, u32 n_buffers); - typedef struct { - vlib_buffer_fill_free_list_cb_t *vlib_buffer_fill_free_list_cb; - vlib_buffer_free_cb_t *vlib_buffer_free_cb; - vlib_buffer_free_no_next_cb_t *vlib_buffer_free_no_next_cb; -} vlib_buffer_callbacks_t; - -extern vlib_buffer_callbacks_t *vlib_buffer_callbacks; - + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + u32 *cached_buffers; + u32 n_alloc; +} vlib_buffer_pool_thread_t; typedef struct { CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); uword start; uword size; uword log2_page_size; + u8 index; + u32 numa_node; u32 physmem_map_index; - u32 buffer_size; + u32 data_size; + u32 n_buffers; u32 *buffers; + u8 *name; clib_spinlock_t lock; + + /* per-thread data */ + vlib_buffer_pool_thread_t *threads; + + /* buffer metadata template */ + vlib_buffer_t buffer_template; } vlib_buffer_pool_t; typedef struct @@ -438,36 +402,24 @@ typedef struct uword buffer_mem_size; vlib_buffer_pool_t *buffer_pools; - /* Buffer free callback, for subversive activities */ - u32 (*buffer_free_callback) (struct vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, u32 follow_buffer_next); -#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX (0) - - /* Hash table mapping buffer size (rounded to next unit of - sizeof (vlib_buffer_t)) to free list index. */ - uword *free_list_by_size; - /* Hash table mapping buffer index into number 0 => allocated but free, 1 => allocated and not-free. If buffer index is not in hash table then this buffer has never been allocated. */ uword *buffer_known_hash; clib_spinlock_t buffer_known_hash_lockp; + u32 n_numa_nodes; - /* Callbacks */ - vlib_buffer_callbacks_t cb; - int callbacks_registered; -} vlib_buffer_main_t; + /* config */ + u32 buffers_per_numa; + u16 ext_hdr_size; -u8 vlib_buffer_register_physmem_map (struct vlib_main_t *vm, - u32 physmem_map_index); + /* logging */ + vlib_log_class_t log_default; +} vlib_buffer_main_t; clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm); - -void *vlib_set_buffer_free_callback (struct vlib_main_t *vm, void *fp); - /* */ @@ -488,23 +440,17 @@ extern void vlib_buffer_trace_trajectory_init (vlib_buffer_t * b); #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */ -#endif /* included_vlib_buffer_h */ +extern u16 __vlib_buffer_external_hdr_size; +#define VLIB_BUFFER_SET_EXT_HDR_SIZE(x) \ +static void __clib_constructor \ +vnet_buffer_set_ext_hdr_size() \ +{ \ + if (__vlib_buffer_external_hdr_size) \ + clib_error ("buffer external header space already set"); \ + __vlib_buffer_external_hdr_size = CLIB_CACHE_LINE_ROUND (x); \ +} -#define VLIB_BUFFER_REGISTER_CALLBACKS(x,...) \ - __VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks; \ -static void __vlib_add_buffer_callbacks_t_##x (void) \ - __attribute__((__constructor__)) ; \ -static void __vlib_add_buffer_callbacks_t_##x (void) \ -{ \ - if (vlib_buffer_callbacks) \ - clib_panic ("vlib buffer callbacks already registered"); \ - vlib_buffer_callbacks = &__##x##_buffer_callbacks; \ -} \ -static void __vlib_rm_buffer_callbacks_t_##x (void) \ - __attribute__((__destructor__)) ; \ -static void __vlib_rm_buffer_callbacks_t_##x (void) \ -{ vlib_buffer_callbacks = 0; } \ -__VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks +#endif /* included_vlib_buffer_h */ /* * fd.io coding-style-patch-verification: ON diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index cedd7d02646..ce8d1ef9078 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -89,6 +89,13 @@ vlib_buffer_copy_template (vlib_buffer_t * b, vlib_buffer_t * bt) #endif } +always_inline u8 +vlib_buffer_pool_get_default_for_numa (vlib_main_t * vm, u32 numa_node) +{ + ASSERT (numa_node < vm->buffer_main->n_numa_nodes); + return numa_node; +} + /** \brief Translate array of buffer indices into buffer pointers with offset @param vm - (vlib_main_t *) vlib main data structure pointer @@ -394,29 +401,48 @@ vlib_buffer_is_known (vlib_main_t * vm, u32 buffer_index) return p ? p[0] : VLIB_BUFFER_UNKNOWN; } -always_inline void -vlib_buffer_set_known_state (vlib_main_t * vm, u32 buffer_index, - vlib_buffer_known_state_t state) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - - clib_spinlock_lock (&bm->buffer_known_hash_lockp); - hash_set (bm->buffer_known_hash, buffer_index, state); - clib_spinlock_unlock (&bm->buffer_known_hash_lockp); -} - /* Validates sanity of a single buffer. Returns format'ed vector with error message if any. */ u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index, uword follow_chain); -always_inline u32 -vlib_buffer_round_size (u32 size) +static_always_inline vlib_buffer_pool_t * +vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index) { - return round_pow2 (size, sizeof (vlib_buffer_t)); + vlib_buffer_main_t *bm = vm->buffer_main; + return vec_elt_at_index (bm->buffer_pools, buffer_pool_index); } -/** \brief Allocate buffers from specific freelist into supplied array +static_always_inline uword +vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers, + u32 n_buffers) +{ + vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); + u32 len; + + ASSERT (bp->buffers); + + clib_spinlock_lock (&bp->lock); + len = vec_len (bp->buffers); + if (PREDICT_TRUE (n_buffers < len)) + { + len -= n_buffers; + vlib_buffer_copy_indices (buffers, bp->buffers + len, n_buffers); + _vec_len (bp->buffers) = len; + clib_spinlock_unlock (&bp->lock); + return n_buffers; + } + else + { + vlib_buffer_copy_indices (buffers, bp->buffers, len); + _vec_len (bp->buffers) = 0; + clib_spinlock_unlock (&bp->lock); + return len; + } +} + + +/** \brief Allocate buffers from specific pool into supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index array @@ -424,57 +450,87 @@ vlib_buffer_round_size (u32 size) @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ + always_inline u32 -vlib_buffer_alloc_from_free_list (vlib_main_t * vm, - u32 * buffers, - u32 n_buffers, - vlib_buffer_free_list_index_t index) +vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers, + u8 buffer_pool_index) { - vlib_buffer_free_list_t *fl; vlib_buffer_main_t *bm = vm->buffer_main; - u32 *src; - uword len; - - ASSERT (bm->cb.vlib_buffer_fill_free_list_cb); + vlib_buffer_pool_t *bp; + vlib_buffer_pool_thread_t *bpt; + u32 *src, *dst, len, n_left; - fl = pool_elt_at_index (vm->buffer_free_list_pool, index); + bp = vec_elt_at_index (bm->buffer_pools, buffer_pool_index); + bpt = vec_elt_at_index (bp->threads, vm->thread_index); - len = vec_len (fl->buffers); + dst = buffers; + n_left = n_buffers; + len = vec_len (bpt->cached_buffers); - if (PREDICT_FALSE (len < n_buffers)) + /* per-thread cache contains enough buffers */ + if (len >= n_buffers) { - bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers); - if (PREDICT_FALSE ((len = vec_len (fl->buffers)) == 0)) - return 0; + src = bpt->cached_buffers + len - n_buffers; + vlib_buffer_copy_indices (dst, src, n_buffers); + _vec_len (bpt->cached_buffers) -= n_buffers; - /* even if fill free list didn't manage to refill free list - we should give what we have */ - n_buffers = clib_min (len, n_buffers); + if (CLIB_DEBUG > 0) + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); + return n_buffers; + } - /* following code is intentionaly duplicated to allow compiler - to optimize fast path when n_buffers is constant value */ - src = fl->buffers + len - n_buffers; - vlib_buffer_copy_indices (buffers, src, n_buffers); - _vec_len (fl->buffers) -= n_buffers; + /* take everything available in the cache */ + if (len) + { + vlib_buffer_copy_indices (dst, bpt->cached_buffers, len); + _vec_len (bpt->cached_buffers) = 0; + dst += len; + n_left -= len; + } - /* Verify that buffers are known free. */ - vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, - VLIB_BUFFER_KNOWN_FREE); + len = round_pow2 (n_left, 32); + vec_validate_aligned (bpt->cached_buffers, len - 1, CLIB_CACHE_LINE_BYTES); + len = vlib_buffer_pool_get (vm, buffer_pool_index, bpt->cached_buffers, + len); + _vec_len (bpt->cached_buffers) = len; - return n_buffers; + if (len) + { + u32 n_copy = clib_min (len, n_left); + src = bpt->cached_buffers + len - n_copy; + vlib_buffer_copy_indices (dst, src, n_copy); + _vec_len (bpt->cached_buffers) -= n_copy; + n_left -= n_copy; } - src = fl->buffers + len - n_buffers; - vlib_buffer_copy_indices (buffers, src, n_buffers); - _vec_len (fl->buffers) -= n_buffers; + n_buffers -= n_left; /* Verify that buffers are known free. */ - vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, - VLIB_BUFFER_KNOWN_FREE); + if (CLIB_DEBUG > 0) + vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, + VLIB_BUFFER_KNOWN_FREE); return n_buffers; } +/** \brief Allocate buffers from specific numa node into supplied array + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param buffers - (u32 * ) buffer index array + @param n_buffers - (u32) number of buffers requested + @param numa_node - (u32) numa node + @return - (u32) number of buffers actually allocated, may be + less than the number requested or zero +*/ +always_inline u32 +vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers, + u32 numa_node) +{ + u8 index = vlib_buffer_pool_get_default_for_numa (vm, numa_node); + return vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, index); +} + /** \brief Allocate buffers into supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @@ -483,11 +539,11 @@ vlib_buffer_alloc_from_free_list (vlib_main_t * vm, @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ + always_inline u32 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) { - return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers, - VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); + return vlib_buffer_alloc_on_numa (vm, buffers, n_buffers, vm->numa_node); } /** \brief Allocate buffers into ring @@ -519,6 +575,215 @@ vlib_buffer_alloc_to_ring (vlib_main_t * vm, u32 * ring, u32 start, return n_alloc; } +/** \brief Allocate buffers into ring from specific buffer pool + + @param vm - (vlib_main_t *) vlib main data structure pointer + @param buffers - (u32 * ) buffer index ring + @param start - (u32) first slot in the ring + @param ring_size - (u32) ring size + @param n_buffers - (u32) number of buffers requested + @return - (u32) number of buffers actually allocated, may be + less than the number requested or zero +*/ +always_inline u32 +vlib_buffer_alloc_to_ring_from_pool (vlib_main_t * vm, u32 * ring, u32 start, + u32 ring_size, u32 n_buffers, + u8 buffer_pool_index) +{ + u32 n_alloc; + + ASSERT (n_buffers <= ring_size); + + if (PREDICT_TRUE (start + n_buffers <= ring_size)) + return vlib_buffer_alloc_from_pool (vm, ring + start, n_buffers, + buffer_pool_index); + + n_alloc = vlib_buffer_alloc_from_pool (vm, ring + start, ring_size - start, + buffer_pool_index); + + if (PREDICT_TRUE (n_alloc == ring_size - start)) + n_alloc += vlib_buffer_alloc_from_pool (vm, ring, n_buffers - n_alloc, + buffer_pool_index); + + return n_alloc; +} + +static void +vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index, + u32 * buffers, u32 n_buffers) +{ + vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); + vlib_buffer_pool_thread_t *bpt = + vec_elt_at_index (bp->threads, vm->thread_index); + + vec_add_aligned (bpt->cached_buffers, buffers, n_buffers, + CLIB_CACHE_LINE_BYTES); + + if (vec_len (bpt->cached_buffers) > 4 * VLIB_FRAME_SIZE) + { + clib_spinlock_lock (&bp->lock); + /* keep last stored buffers, as they are more likely hot in the cache */ + vec_add_aligned (bp->buffers, bpt->cached_buffers, VLIB_FRAME_SIZE, + CLIB_CACHE_LINE_BYTES); + vec_delete (bpt->cached_buffers, VLIB_FRAME_SIZE, 0); + bpt->n_alloc -= VLIB_FRAME_SIZE; + clib_spinlock_unlock (&bp->lock); + } +} + +static_always_inline void +vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, + int maybe_next) +{ + const int queue_size = 128; + vlib_buffer_pool_t *bp = 0; + u8 buffer_pool_index = ~0; + u32 n_queue = 0, queue[queue_size + 4]; + vlib_buffer_t bt = { }; +#if defined(CLIB_HAVE_VEC128) && !__aarch64__ + vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 }; + vlib_buffer_t bpi_vec = {.buffer_pool_index = ~0 }; + vlib_buffer_t flags_refs_mask = { + .flags = VLIB_BUFFER_NEXT_PRESENT, + .ref_count = ~0 + }; +#endif + + while (n_buffers) + { + vlib_buffer_t *b[8]; + u32 bi, sum = 0, flags, next; + + if (n_buffers < 12) + goto one_by_one; + + vlib_get_buffers (vm, buffers, b, 4); + vlib_get_buffers (vm, buffers + 8, b + 4, 4); + + vlib_prefetch_buffer_header (b[4], LOAD); + vlib_prefetch_buffer_header (b[5], LOAD); + vlib_prefetch_buffer_header (b[6], LOAD); + vlib_prefetch_buffer_header (b[7], LOAD); + +#if defined(CLIB_HAVE_VEC128) && !__aarch64__ + u8x16 p0, p1, p2, p3, r; + p0 = u8x16_load_unaligned (b[0]); + p1 = u8x16_load_unaligned (b[1]); + p2 = u8x16_load_unaligned (b[2]); + p3 = u8x16_load_unaligned (b[3]); + + r = p0 ^ bpi_vec.as_u8x16[0]; + r |= p1 ^ bpi_vec.as_u8x16[0]; + r |= p2 ^ bpi_vec.as_u8x16[0]; + r |= p3 ^ bpi_vec.as_u8x16[0]; + r &= bpi_mask.as_u8x16[0]; + r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0]; + + sum = !u8x16_is_all_zero (r); +#else + sum |= b[0]->flags; + sum |= b[1]->flags; + sum |= b[2]->flags; + sum |= b[3]->flags; + sum &= VLIB_BUFFER_NEXT_PRESENT; + sum += b[0]->ref_count - 1; + sum += b[1]->ref_count - 1; + sum += b[2]->ref_count - 1; + sum += b[3]->ref_count - 1; + sum |= b[0]->buffer_pool_index ^ buffer_pool_index; + sum |= b[1]->buffer_pool_index ^ buffer_pool_index; + sum |= b[2]->buffer_pool_index ^ buffer_pool_index; + sum |= b[3]->buffer_pool_index ^ buffer_pool_index; +#endif + + if (sum) + goto one_by_one; + + vlib_buffer_copy_indices (queue + n_queue, buffers, 4); + vlib_buffer_copy_template (b[0], &bt); + vlib_buffer_copy_template (b[1], &bt); + vlib_buffer_copy_template (b[2], &bt); + vlib_buffer_copy_template (b[3], &bt); + n_queue += 4; + + if (CLIB_DEBUG > 0) + vlib_buffer_validate_alloc_free (vm, buffers, 4, + VLIB_BUFFER_KNOWN_ALLOCATED); + + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]); + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]); + + if (n_queue >= queue_size) + { + vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue); + n_queue = 0; + } + buffers += 4; + n_buffers -= 4; + continue; + + one_by_one: + bi = buffers[0]; + + next_in_chain: + b[0] = vlib_get_buffer (vm, bi); + flags = b[0]->flags; + next = b[0]->next_buffer; + + if (PREDICT_FALSE (buffer_pool_index != b[0]->buffer_pool_index)) + { + buffer_pool_index = b[0]->buffer_pool_index; +#if defined(CLIB_HAVE_VEC128) && !__aarch64__ + bpi_vec.buffer_pool_index = buffer_pool_index; +#endif + bp = vlib_get_buffer_pool (vm, buffer_pool_index); + vlib_buffer_copy_template (&bt, &bp->buffer_template); + + if (n_queue) + { + vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue); + n_queue = 0; + } + } + + ASSERT (pointer_to_uword (b[0]) >= bp->start && + pointer_to_uword (b[0]) < + bp->start + bp->size - (bp->data_size + sizeof (*b[0]))); + + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); + + if (clib_atomic_sub_fetch (&b[0]->ref_count, 1) == 0) + { + if (CLIB_DEBUG > 0) + vlib_buffer_validate_alloc_free (vm, &bi, 1, + VLIB_BUFFER_KNOWN_ALLOCATED); + vlib_buffer_copy_template (b[0], &bt); + queue[n_queue++] = bi; + } + + if (n_queue == queue_size) + { + vlib_buffer_pool_put (vm, buffer_pool_index, queue, queue_size); + n_queue = 0; + } + + if (flags & VLIB_BUFFER_NEXT_PRESENT) + { + bi = next; + goto next_in_chain; + } + + buffers++; + n_buffers--; + } + + if (n_queue) + vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue); +} + + /** \brief Free buffers Frees the entire buffer chain for each buffer @@ -534,11 +799,7 @@ vlib_buffer_free (vlib_main_t * vm, /* number of buffers to free */ u32 n_buffers) { - vlib_buffer_main_t *bm = vm->buffer_main; - - ASSERT (bm->cb.vlib_buffer_free_cb); - - return bm->cb.vlib_buffer_free_cb (vm, buffers, n_buffers); + vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 1); } /** \brief Free buffers, does not free the buffer chain for each buffer @@ -555,11 +816,7 @@ vlib_buffer_free_no_next (vlib_main_t * vm, /* number of buffers to free */ u32 n_buffers) { - vlib_buffer_main_t *bm = vm->buffer_main; - - ASSERT (bm->cb.vlib_buffer_free_no_next_cb); - - return bm->cb.vlib_buffer_free_no_next_cb (vm, buffers, n_buffers); + vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 0); } /** \brief Free one buffer @@ -571,7 +828,7 @@ vlib_buffer_free_no_next (vlib_main_t * vm, always_inline void vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index) { - vlib_buffer_free (vm, &buffer_index, /* n_buffers */ 1); + vlib_buffer_free_inline (vm, &buffer_index, 1, /* maybe next */ 1); } /** \brief Free buffers from ring @@ -624,11 +881,6 @@ vlib_buffer_free_from_ring_no_next (vlib_main_t * vm, u32 * ring, u32 start, } } -/* Add/delete buffer free lists. */ -vlib_buffer_free_list_index_t vlib_buffer_create_free_list (vlib_main_t * vm, - u32 n_data_bytes, - char *fmt, ...); - /* Append given data to end of buffer, possibly allocating new buffers. */ int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data, u32 n_data_bytes); @@ -710,7 +962,7 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, u16 i; vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer); - ASSERT (s->n_add_refs == 0); + ASSERT (s->ref_count == 1); ASSERT (n_buffers); ASSERT (n_buffers <= 256); @@ -735,13 +987,16 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, return 1; } - n_buffers = vlib_buffer_alloc (vm, buffers, n_buffers); + n_buffers = vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, + s->buffer_pool_index); for (i = 0; i < n_buffers; i++) { vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]); d->current_data = s->current_data; d->current_length = head_end_offset; + ASSERT (d->buffer_pool_index == s->buffer_pool_index); + d->total_length_not_including_first_buffer = s->current_length - head_end_offset; if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT)) @@ -758,11 +1013,11 @@ vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, d->next_buffer = src_buffer; } vlib_buffer_advance (s, head_end_offset); - s->n_add_refs = n_buffers - 1; + s->ref_count = n_buffers; while (s->flags & VLIB_BUFFER_NEXT_PRESENT) { s = vlib_get_buffer (vm, s->next_buffer); - s->n_add_refs = n_buffers - 1; + s->ref_count = n_buffers; } return n_buffers; @@ -816,6 +1071,7 @@ vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head, vlib_buffer_t * tail) { ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); + ASSERT (head->buffer_pool_index == tail->buffer_pool_index); head->flags |= VLIB_BUFFER_NEXT_PRESENT; head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; @@ -826,7 +1082,7 @@ vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head, tail->total_length_not_including_first_buffer; next_segment: - clib_atomic_add_fetch (&tail->n_add_refs, 1); + clib_atomic_add_fetch (&tail->ref_count, 1); if (tail->flags & VLIB_BUFFER_NEXT_PRESENT) { @@ -875,7 +1131,6 @@ vlib_buffer_chain_increase_length (vlib_buffer_t * first, * Returns the number of copied bytes. */ always_inline u16 vlib_buffer_chain_append_data (vlib_main_t * vm, - vlib_buffer_free_list_index_t free_list_index, vlib_buffer_t * first, vlib_buffer_t * last, void *data, u16 data_len) { @@ -897,8 +1152,6 @@ vlib_buffer_chain_append_data (vlib_main_t * vm, * chained and points to the last buffer in the chain. */ u16 vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, - vlib_buffer_free_list_index_t - free_list_index, vlib_buffer_t * first, vlib_buffer_t ** last, void *data, u16 data_len); @@ -915,17 +1168,9 @@ typedef struct /* Number of buffers to allocate in each call to allocator. */ u32 min_n_buffers_each_alloc; - /* Buffer free list for this template. */ - vlib_buffer_free_list_index_t free_list_index; - - u32 *free_buffers; - u8 *name; } vlib_packet_template_t; -void vlib_packet_template_get_packet_helper (vlib_main_t * vm, - vlib_packet_template_t * t); - void vlib_packet_template_init (vlib_main_t * vm, vlib_packet_template_t * t, void *packet_data, @@ -943,112 +1188,9 @@ vlib_packet_template_free (vlib_main_t * vm, vlib_packet_template_t * t) vec_free (t->packet_data); } -static_always_inline vlib_buffer_pool_t * -vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index) -{ - vlib_buffer_main_t *bm = vm->buffer_main; - return vec_elt_at_index (bm->buffer_pools, buffer_pool_index); -} - -always_inline void -vlib_buffer_add_to_free_list (vlib_main_t * vm, - vlib_buffer_free_list_t * f, - u32 buffer_index, u8 do_init) -{ - vlib_buffer_pool_t *bp = vlib_buffer_pool_get (vm, f->buffer_pool_index); - vlib_buffer_t *b; - b = vlib_get_buffer (vm, buffer_index); - if (PREDICT_TRUE (do_init)) - clib_memset (b, 0, STRUCT_OFFSET_OF (vlib_buffer_t, template_end)); - vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES); - - if (vec_len (f->buffers) > 4 * VLIB_FRAME_SIZE) - { - clib_spinlock_lock (&bp->lock); - /* keep last stored buffers, as they are more likely hot in the cache */ - vec_add_aligned (bp->buffers, f->buffers, VLIB_FRAME_SIZE, - CLIB_CACHE_LINE_BYTES); - vec_delete (f->buffers, VLIB_FRAME_SIZE, 0); - f->n_alloc -= VLIB_FRAME_SIZE; - clib_spinlock_unlock (&bp->lock); - } -} - -#if CLIB_DEBUG > 0 -extern u32 *vlib_buffer_state_validation_lock; -extern uword *vlib_buffer_state_validation_hash; -extern void *vlib_buffer_state_heap; -#endif - -static inline void -vlib_validate_buffer_in_use (vlib_buffer_t * b, u32 expected) -{ -#if CLIB_DEBUG > 0 - uword *p; - void *oldheap; - - oldheap = clib_mem_set_heap (vlib_buffer_state_heap); - - while (clib_atomic_test_and_set (vlib_buffer_state_validation_lock)) - ; - - p = hash_get (vlib_buffer_state_validation_hash, b); - - /* If we don't know about b, declare it to be in the expected state */ - if (!p) - { - hash_set (vlib_buffer_state_validation_hash, b, expected); - goto out; - } - - if (p[0] != expected) - { - void cj_stop (void); - u32 bi; - vlib_main_t *vm = &vlib_global_main; - - cj_stop (); - - bi = vlib_get_buffer_index (vm, b); - - clib_mem_set_heap (oldheap); - clib_warning ("%.6f buffer %llx (%d): %s, not %s", - vlib_time_now (vm), bi, - p[0] ? "busy" : "free", expected ? "busy" : "free"); - os_panic (); - } -out: - CLIB_MEMORY_BARRIER (); - *vlib_buffer_state_validation_lock = 0; - clib_mem_set_heap (oldheap); -#endif -} - -static inline void -vlib_validate_buffer_set_in_use (vlib_buffer_t * b, u32 expected) -{ -#if CLIB_DEBUG > 0 - void *oldheap; - - oldheap = clib_mem_set_heap (vlib_buffer_state_heap); - - while (clib_atomic_test_and_set (vlib_buffer_state_validation_lock)) - ; - - hash_set (vlib_buffer_state_validation_hash, b, expected); - - CLIB_MEMORY_BARRIER (); - *vlib_buffer_state_validation_lock = 0; - clib_mem_set_heap (oldheap); -#endif -} - -/** minimum data size of first buffer in a buffer chain */ -#define VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SIZE (256) - /** * @brief compress buffer chain in a way where the first buffer is at least - * VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SIZE long + * VLIB_BUFFER_CLONE_HEAD_SIZE long * * @param[in] vm - vlib_main * @param[in,out] first - first buffer in chain @@ -1059,14 +1201,14 @@ always_inline void vlib_buffer_chain_compress (vlib_main_t * vm, vlib_buffer_t * first, u32 ** discard_vector) { - if (first->current_length >= VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SIZE || + if (first->current_length >= VLIB_BUFFER_CLONE_HEAD_SIZE || !(first->flags & VLIB_BUFFER_NEXT_PRESENT)) { /* this is already big enough or not a chain */ return; } - u32 want_first_size = clib_min (VLIB_BUFFER_CHAIN_MIN_FIRST_DATA_SIZE, + u32 want_first_size = clib_min (VLIB_BUFFER_CLONE_HEAD_SIZE, VLIB_BUFFER_DATA_SIZE - first->current_data); do diff --git a/src/vlib/log.h b/src/vlib/log.h index d21bca9f195..9b285429b9a 100644 --- a/src/vlib/log.h +++ b/src/vlib/log.h @@ -18,8 +18,6 @@ #include -typedef u32 vlib_log_class_t; - #define foreach_vlib_log_level \ _(0, EMERG, emerg) \ _(1, ALERT, alert) \ diff --git a/src/vlib/main.c b/src/vlib/main.c index a64a991d438..c0ab8e129b9 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -463,7 +463,7 @@ vlib_put_next_frame (vlib_main_t * vm, vlib_frame_t *f; u32 n_vectors_in_frame; - if (vm->buffer_main->callbacks_registered == 0 && CLIB_DEBUG > 0) + if (CLIB_DEBUG > 0) vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left); nf = vlib_node_runtime_get_next_frame (vm, r, next_index); @@ -987,8 +987,8 @@ format_buffer_metadata (u8 * s, va_list * args) (i32) (b->current_data), (i32) (b->current_length)); s = format (s, "current_config_index: %d, flow_id: %x, next_buffer: %x\n", b->current_config_index, b->flow_id, b->next_buffer); - s = format (s, "error: %d, n_add_refs: %d, buffer_pool_index: %d\n", - (u32) (b->error), (u32) (b->n_add_refs), + s = format (s, "error: %d, ref_count: %d, buffer_pool_index: %d\n", + (u32) (b->error), (u32) (b->ref_count), (u32) (b->buffer_pool_index)); s = format (s, "trace_index: %d, len_not_first_buf: %d\n", @@ -1993,9 +1993,6 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input) if ((error = vlib_call_all_init_functions (vm))) goto done; - /* Create default buffer free list. */ - vlib_buffer_create_free_list (vm, VLIB_BUFFER_DATA_SIZE, "default"); - nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)), CLIB_CACHE_LINE_BYTES); diff --git a/src/vlib/main.h b/src/vlib/main.h index c6786d8f788..920e5f3f95a 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -115,9 +115,6 @@ typedef struct vlib_main_t /* Size of the heap */ uword heap_size; - /* Pool of buffer free lists. */ - vlib_buffer_free_list_t *buffer_free_list_pool; - /* buffer main structure. */ vlib_buffer_main_t *buffer_main; diff --git a/src/vlib/threads.c b/src/vlib/threads.c index ed1ba61a0cc..cdb4eb082b5 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -719,8 +719,6 @@ start_workers (vlib_main_t * vm) for (i = 0; i < vec_len (tm->registrations); i++) { vlib_node_main_t *nm, *nm_clone; - vlib_buffer_free_list_t *fl_clone, *fl_orig; - vlib_buffer_free_list_t *orig_freelist_pool; int k; tr = tm->registrations[i]; @@ -883,26 +881,6 @@ start_workers (vlib_main_t * vm) (vlib_mains[0]->error_main.counters_last_clear, CLIB_CACHE_LINE_BYTES); - /* Fork the vlib_buffer_main_t free lists, etc. */ - orig_freelist_pool = vm_clone->buffer_free_list_pool; - vm_clone->buffer_free_list_pool = 0; - - /* *INDENT-OFF* */ - pool_foreach (fl_orig, orig_freelist_pool, - ({ - pool_get_aligned (vm_clone->buffer_free_list_pool, - fl_clone, CLIB_CACHE_LINE_BYTES); - ASSERT (fl_orig - orig_freelist_pool - == fl_clone - vm_clone->buffer_free_list_pool); - - fl_clone[0] = fl_orig[0]; - fl_clone->buffers = 0; - vec_validate(fl_clone->buffers, 0); - vec_reset_length(fl_clone->buffers); - fl_clone->n_alloc = 0; - })); -/* *INDENT-ON* */ - worker_thread_index++; } } diff --git a/src/vlib/vlib.h b/src/vlib/vlib.h index 49bb51bfbe3..2149c4763ab 100644 --- a/src/vlib/vlib.h +++ b/src/vlib/vlib.h @@ -48,6 +48,7 @@ /* Forward declarations of structs to avoid circular dependencies. */ struct vlib_main_t; +typedef u32 vlib_log_class_t; /* All includes in alphabetical order. */ #include diff --git a/src/vpp/conf/startup.conf b/src/vpp/conf/startup.conf index 7b4f69d51c7..73e9d2cfc1b 100644 --- a/src/vpp/conf/startup.conf +++ b/src/vpp/conf/startup.conf @@ -69,6 +69,13 @@ cpu { # scheduler-priority 50 } +# buffers { + ## Increase number of buffers allocated, needed only in scenarios with + ## large number of interfaces and worker threads. Value is per numa node. + ## Default is 16384 (8192 if running unpriviledged) + # buffers-per-numa 128000 +# } + # dpdk { ## Change default settings for all interfaces # dev default { @@ -127,11 +134,6 @@ cpu { ## disables Jumbo MTU support # no-multi-seg - ## Increase number of buffers allocated, needed only in scenarios with - ## large number of interfaces and worker threads. Value is per CPU socket. - ## Default is 16384 - # num-mbufs 128000 - ## Change hugepages allocation per-socket, needed only if there is need for ## larger number of mbufs. Default is 256M on each detected CPU socket # socket-mem 2048,2048 diff --git a/src/vppinfra/cpu.h b/src/vppinfra/cpu.h index 97e0144be60..7f0d6ad2e34 100644 --- a/src/vppinfra/cpu.h +++ b/src/vppinfra/cpu.h @@ -62,6 +62,62 @@ #define CLIB_MARCH_SFX CLIB_MULTIARCH_FN +typedef struct _clib_march_fn_registration +{ + void *function; + int priority; + struct _clib_march_fn_registration *next; + char *name; +} clib_march_fn_registration; + +static_always_inline void * +clib_march_select_fn_ptr (clib_march_fn_registration * r) +{ + void *rv = 0; + int last_prio = -1; + + while (r) + { + if (last_prio < r->priority) + { + last_prio = r->priority; + rv = r->function; + } + r = r->next; + } + return rv; +} + +#define CLIB_MARCH_FN_POINTER(fn) \ + clib_march_select_fn_ptr (fn##_march_fn_registrations); + +#define _CLIB_MARCH_FN_REGISTRATION(fn) \ +static clib_march_fn_registration \ +CLIB_MARCH_SFX(fn##_march_fn_registration) = \ +{ \ + .name = CLIB_MARCH_VARIANT_STR \ +}; \ +\ +static void __clib_constructor \ +fn##_march_register () \ +{ \ + clib_march_fn_registration *r; \ + r = & CLIB_MARCH_SFX (fn##_march_fn_registration); \ + r->priority = CLIB_MARCH_FN_PRIORITY(); \ + r->next = fn##_march_fn_registrations; \ + r->function = CLIB_MARCH_SFX (fn); \ + fn##_march_fn_registrations = r; \ +} + +#ifdef CLIB_MARCH_VARIANT +#define CLIB_MARCH_FN_REGISTRATION(fn) \ +extern clib_march_fn_registration *fn##_march_fn_registrations; \ +_CLIB_MARCH_FN_REGISTRATION(fn) +#else +#define CLIB_MARCH_FN_REGISTRATION(fn) \ +clib_march_fn_registration *fn##_march_fn_registrations = 0; \ +_CLIB_MARCH_FN_REGISTRATION(fn) +#endif #define foreach_x86_64_flags \ _ (sse3, 1, ecx, 0) \ _ (ssse3, 1, ecx, 9) \ diff --git a/src/vppinfra/error_bootstrap.h b/src/vppinfra/error_bootstrap.h index 248d389d0f2..18d2176812d 100644 --- a/src/vppinfra/error_bootstrap.h +++ b/src/vppinfra/error_bootstrap.h @@ -91,6 +91,10 @@ do { \ #define STATIC_ASSERT_OFFSET_OF(s, e, o) \ STATIC_ASSERT (STRUCT_OFFSET_OF(s,e) == o, "Offset of " #s "." #e " must be " # o) +#define STATIC_ASSERT_FITS_IN(s, e, o) \ + STATIC_ASSERT (STRUCT_OFFSET_OF(s,e) <= (o - sizeof(((s *)0)->e)), \ + #s "." #e " does not fit into " # o " bytes") + /* Assert without allocating memory. */ #define ASSERT_AND_PANIC(truth) \ do { \ -- cgit 1.2.3-korg