diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/plugins/af_xdp/device.c | 13 | ||||
| -rw-r--r-- | src/plugins/af_xdp/input.c | 54 |
2 files changed, 26 insertions, 41 deletions
diff --git a/src/plugins/af_xdp/device.c b/src/plugins/af_xdp/device.c index 03a3053e449..7a10bce4290 100644 --- a/src/plugins/af_xdp/device.c +++ b/src/plugins/af_xdp/device.c @@ -215,18 +215,7 @@ af_xdp_create_queue (vlib_main_t *vm, af_xdp_create_if_args_t *args, umem_config.comp_size = args->txq_size; umem_config.frame_size = sizeof (vlib_buffer_t) + vlib_buffer_get_default_data_size (vm); - /* - * Note about headroom: for some reasons, there seem to be a discrepency - * between 0-copy and copy mode: - * - 0-copy: XDP_PACKET_HEADROOM will be added to the user headroom - * - copy: nothing is added to the user headroom - * We privileged 0-copy and set headroom so that frame_headroom + - * XDP_PACKET_HEADROOM == sizeof(vlib_buffer_t), ie data will correctly - * point to vlib_buffer_t->data for 0-copy. In copy mode, we have to add - * XDP_PACKET_HEADROOM to desc offset during refill. - */ - STATIC_ASSERT (sizeof (vlib_buffer_t) >= XDP_PACKET_HEADROOM, "wrong size"); - umem_config.frame_headroom = sizeof (vlib_buffer_t) - XDP_PACKET_HEADROOM; + umem_config.frame_headroom = sizeof (vlib_buffer_t); umem_config.flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG; if (xsk_umem__create (umem, uword_to_pointer (vm->buffer_main->buffer_mem_start, void *), diff --git a/src/plugins/af_xdp/input.c b/src/plugins/af_xdp/input.c index 98d841b77a0..4f3ac5725a4 100644 --- a/src/plugins/af_xdp/input.c +++ b/src/plugins/af_xdp/input.c @@ -106,8 +106,7 @@ af_xdp_device_input_refill_db (vlib_main_t * vm, static_always_inline void af_xdp_device_input_refill_inline (vlib_main_t *vm, const vlib_node_runtime_t *node, - af_xdp_device_t *ad, af_xdp_rxq_t *rxq, - const int copy) + af_xdp_device_t *ad, af_xdp_rxq_t *rxq) { __u64 *fill; const u32 size = rxq->fq.size; @@ -133,13 +132,7 @@ af_xdp_device_input_refill_inline (vlib_main_t *vm, n = clib_min (n_alloc, size - (idx & mask)); n_wrap = n_alloc - n; - /* - * Note about headroom: for some reasons, there seem to be a discrepency - * between 0-copy and copy mode. See - * src/plugins/af_xdp/device.c:af_xdp_create_queue() - */ -#define bi2addr(bi) \ - (((bi) << CLIB_LOG2_CACHE_LINE_BYTES) + (copy ? XDP_PACKET_HEADROOM : 0)) +#define bi2addr(bi) ((bi) << CLIB_LOG2_CACHE_LINE_BYTES) wrap_around: @@ -208,27 +201,30 @@ af_xdp_device_input_ethernet (vlib_main_t * vm, vlib_node_runtime_t * node, } static_always_inline u32 -af_xdp_device_input_bufs (vlib_main_t * vm, const af_xdp_device_t * ad, - af_xdp_rxq_t * rxq, u32 * bis, const u32 n_rx, - vlib_buffer_t * bt, u32 idx, const int copy) +af_xdp_device_input_bufs (vlib_main_t *vm, const af_xdp_device_t *ad, + af_xdp_rxq_t *rxq, u32 *bis, const u32 n_rx, + vlib_buffer_t *bt, u32 idx) { vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; + u16 offs[VLIB_FRAME_SIZE], *off = offs; u16 lens[VLIB_FRAME_SIZE], *len = lens; const u32 mask = rxq->rx.mask; u32 n = n_rx, *bi = bis, bytes = 0; -#define addr2bi(addr) \ - (((addr) - (copy ? XDP_PACKET_HEADROOM : 0)) >> CLIB_LOG2_CACHE_LINE_BYTES) +#define addr2bi(addr) ((addr) >> CLIB_LOG2_CACHE_LINE_BYTES) while (n >= 1) { const struct xdp_desc *desc = xsk_ring_cons__rx_desc (&rxq->rx, idx); - bi[0] = addr2bi (xsk_umem__extract_addr (desc->addr)); + const u64 addr = desc->addr; + bi[0] = addr2bi (xsk_umem__extract_addr (addr)); ASSERT (vlib_buffer_is_known (vm, bi[0]) == VLIB_BUFFER_KNOWN_ALLOCATED); + off[0] = xsk_umem__extract_offset (addr) - sizeof (vlib_buffer_t); len[0] = desc->len; idx = (idx + 1) & mask; bi += 1; + off += 1; len += 1; n -= 1; } @@ -236,27 +232,33 @@ af_xdp_device_input_bufs (vlib_main_t * vm, const af_xdp_device_t * ad, vlib_get_buffers (vm, bis, bufs, n_rx); n = n_rx; + off = offs; len = lens; while (n >= 8) { vlib_prefetch_buffer_header (b[4], LOAD); vlib_buffer_copy_template (b[0], bt); + b[0]->current_data = off[0]; bytes += b[0]->current_length = len[0]; vlib_prefetch_buffer_header (b[5], LOAD); vlib_buffer_copy_template (b[1], bt); + b[1]->current_data = off[1]; bytes += b[1]->current_length = len[1]; vlib_prefetch_buffer_header (b[6], LOAD); vlib_buffer_copy_template (b[2], bt); + b[2]->current_data = off[2]; bytes += b[2]->current_length = len[2]; vlib_prefetch_buffer_header (b[7], LOAD); vlib_buffer_copy_template (b[3], bt); + b[3]->current_data = off[3]; bytes += b[3]->current_length = len[3]; b += 4; + off += 4; len += 4; n -= 4; } @@ -264,8 +266,10 @@ af_xdp_device_input_bufs (vlib_main_t * vm, const af_xdp_device_t * ad, while (n >= 1) { vlib_buffer_copy_template (b[0], bt); + b[0]->current_data = off[0]; bytes += b[0]->current_length = len[0]; b += 1; + off += 1; len += 1; n -= 1; } @@ -275,9 +279,8 @@ af_xdp_device_input_bufs (vlib_main_t * vm, const af_xdp_device_t * ad, } static_always_inline uword -af_xdp_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * frame, af_xdp_device_t * ad, - u16 qid, const int copy) +af_xdp_device_input_inline (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, af_xdp_device_t *ad, u16 qid) { vnet_main_t *vnm = vnet_get_main (); af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, qid); @@ -299,8 +302,7 @@ af_xdp_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_get_new_next_frame (vm, node, next_index, to_next, n_left_to_next); n_rx_bytes = - af_xdp_device_input_bufs (vm, ad, rxq, to_next, n_rx_packets, &bt, idx, - copy); + af_xdp_device_input_bufs (vm, ad, rxq, to_next, n_rx_packets, &bt, idx); af_xdp_device_input_ethernet (vm, node, next_index, ad->sw_if_index, ad->hw_if_index); @@ -315,7 +317,7 @@ af_xdp_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, ad->hw_if_index, n_rx_packets, n_rx_bytes); refill: - af_xdp_device_input_refill_inline (vm, node, ad, rxq, copy); + af_xdp_device_input_refill_inline (vm, node, ad, rxq); return n_rx_packets; } @@ -334,12 +336,7 @@ VLIB_NODE_FN (af_xdp_input_node) (vlib_main_t * vm, af_xdp_device_t *ad = vec_elt_at_index (am->devices, p->dev_instance); if ((ad->flags & AF_XDP_DEVICE_F_ADMIN_UP) == 0) continue; - if (PREDICT_TRUE (ad->flags & AF_XDP_DEVICE_F_ZEROCOPY)) - n_rx += af_xdp_device_input_inline (vm, node, frame, ad, p->queue_id, - /* copy */ 0); - else - n_rx += af_xdp_device_input_inline (vm, node, frame, ad, p->queue_id, - /* copy */ 1); + n_rx += af_xdp_device_input_inline (vm, node, frame, ad, p->queue_id); } return n_rx; @@ -352,8 +349,7 @@ af_xdp_device_input_refill (af_xdp_device_t *ad) vlib_main_t *vm = vlib_get_main (); af_xdp_rxq_t *rxq; vec_foreach (rxq, ad->rxqs) - af_xdp_device_input_refill_inline ( - vm, 0, ad, rxq, 0 == (ad->flags & AF_XDP_DEVICE_F_ZEROCOPY)); + af_xdp_device_input_refill_inline (vm, 0, ad, rxq); } #endif /* CLIB_MARCH_VARIANT */ |