From 8f49936b20508d57968f01dcb18f8d31b1a58f3f Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Mon, 22 Oct 2018 13:07:02 +0200 Subject: vlib: introduce vlib_buffer_get_{pa,va,current_va,current_pa} inlines Change-Id: I0b42ac6b05bc9910904a97924ea4bebc84507d4d Signed-off-by: Damjan Marion --- src/plugins/avf/device.c | 9 +++------ src/plugins/avf/input.c | 40 +++++++++++++++++----------------------- src/plugins/avf/output.c | 36 ++++++++++++++++++++---------------- src/plugins/ixge/ixge.c | 29 +++++++++++++---------------- src/plugins/marvell/pp2/input.c | 3 ++- src/plugins/marvell/pp2/output.c | 2 +- src/plugins/vmxnet3/output.c | 4 +--- src/plugins/vmxnet3/vmxnet3.h | 8 ++++---- src/vlib/buffer.h | 12 ++++++++++++ src/vlib/buffer_funcs.h | 13 ++++++++----- 10 files changed, 81 insertions(+), 75 deletions(-) diff --git a/src/plugins/avf/device.c b/src/plugins/avf/device.c index 71105e88677..c23746be1b2 100644 --- a/src/plugins/avf/device.c +++ b/src/plugins/avf/device.c @@ -240,14 +240,11 @@ avf_rxq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 rxq_size) avf_rx_desc_t *d = rxq->descs; for (i = 0; i < n_alloc; i++) { + vlib_buffer_t *b = vlib_get_buffer (vm, rxq->bufs[i]); if (ad->flags & AVF_DEVICE_F_IOVA) - { - vlib_buffer_t *b = vlib_get_buffer (vm, rxq->bufs[i]); - d->qword[0] = pointer_to_uword (b->data); - } + d->qword[0] = vlib_buffer_get_va (b); else - d->qword[0] = - vlib_get_buffer_data_physical_address (vm, rxq->bufs[i]); + d->qword[0] = vlib_buffer_get_pa (vm, b); d++; } diff --git a/src/plugins/avf/input.c b/src/plugins/avf/input.c index 0b1ac744a1f..efe3b0e0251 100644 --- a/src/plugins/avf/input.c +++ b/src/plugins/avf/input.c @@ -52,6 +52,7 @@ avf_rxq_refill (vlib_main_t * vm, vlib_node_runtime_t * node, avf_rxq_t * rxq, { u16 n_refill, mask, n_alloc, slot; u32 s0, s1, s2, s3; + vlib_buffer_t *b[4]; avf_rx_desc_t *d[4]; n_refill = rxq->size - 1 - rxq->n_enqueued; @@ -97,28 +98,24 @@ avf_rxq_refill (vlib_main_t * vm, vlib_node_runtime_t * node, avf_rxq_t * rxq, d[1] = ((avf_rx_desc_t *) rxq->descs) + s1; d[2] = ((avf_rx_desc_t *) rxq->descs) + s2; d[3] = ((avf_rx_desc_t *) rxq->descs) + s3; + b[0] = vlib_get_buffer (vm, rxq->bufs[s0]); + b[1] = vlib_get_buffer (vm, rxq->bufs[s1]); + b[2] = vlib_get_buffer (vm, rxq->bufs[s2]); + b[3] = vlib_get_buffer (vm, rxq->bufs[s3]); + if (use_iova) { - vlib_buffer_t *b; - b = vlib_get_buffer (vm, rxq->bufs[s0]); - d[0]->qword[0] = pointer_to_uword (b->data); - b = vlib_get_buffer (vm, rxq->bufs[s1]); - d[1]->qword[0] = pointer_to_uword (b->data); - b = vlib_get_buffer (vm, rxq->bufs[s2]); - d[2]->qword[0] = pointer_to_uword (b->data); - b = vlib_get_buffer (vm, rxq->bufs[s3]); - d[3]->qword[0] = pointer_to_uword (b->data); + d[0]->qword[0] = vlib_buffer_get_va (b[0]); + d[1]->qword[0] = vlib_buffer_get_va (b[1]); + d[2]->qword[0] = vlib_buffer_get_va (b[2]); + d[3]->qword[0] = vlib_buffer_get_va (b[3]); } else { - d[0]->qword[0] = - vlib_get_buffer_data_physical_address (vm, rxq->bufs[s0]); - d[1]->qword[0] = - vlib_get_buffer_data_physical_address (vm, rxq->bufs[s1]); - d[2]->qword[0] = - vlib_get_buffer_data_physical_address (vm, rxq->bufs[s2]); - d[3]->qword[0] = - vlib_get_buffer_data_physical_address (vm, rxq->bufs[s3]); + d[0]->qword[0] = vlib_buffer_get_pa (vm, b[0]); + d[1]->qword[0] = vlib_buffer_get_pa (vm, b[1]); + d[2]->qword[0] = vlib_buffer_get_pa (vm, b[2]); + d[3]->qword[0] = vlib_buffer_get_pa (vm, b[3]); } d[0]->qword[1] = 0; @@ -134,14 +131,11 @@ avf_rxq_refill (vlib_main_t * vm, vlib_node_runtime_t * node, avf_rxq_t * rxq, { s0 = slot; d[0] = ((avf_rx_desc_t *) rxq->descs) + s0; + b[0] = vlib_get_buffer (vm, rxq->bufs[s0]); if (use_iova) - { - vlib_buffer_t *b = vlib_get_buffer (vm, rxq->bufs[s0]); - d[0]->qword[0] = pointer_to_uword (b->data); - } + d[0]->qword[0] = vlib_buffer_get_va (b[0]); else - d[0]->qword[0] = - vlib_get_buffer_data_physical_address (vm, rxq->bufs[s0]); + d[0]->qword[0] = vlib_buffer_get_pa (vm, b[0]); d[0]->qword[1] = 0; /* next */ diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c index 8f8e82d146c..c2c4b01d022 100644 --- a/src/plugins/avf/output.c +++ b/src/plugins/avf/output.c @@ -118,16 +118,21 @@ retry: b2 = vlib_get_buffer (vm, bi2); b3 = vlib_get_buffer (vm, bi3); -#if 0 - d->qword[0] = vlib_get_buffer_data_physical_address (vm, bi0) + - b0->current_data; -#else - d0->qword[0] = pointer_to_uword (b0->data) + b0->current_data; - d1->qword[0] = pointer_to_uword (b1->data) + b1->current_data; - d2->qword[0] = pointer_to_uword (b2->data) + b2->current_data; - d3->qword[0] = pointer_to_uword (b3->data) + b3->current_data; - -#endif + if (ad->flags & AVF_DEVICE_F_IOVA) + { + d0->qword[0] = vlib_buffer_get_current_va (b0); + d1->qword[0] = vlib_buffer_get_current_va (b1); + d2->qword[0] = vlib_buffer_get_current_va (b2); + d3->qword[0] = vlib_buffer_get_current_va (b3); + } + else + { + d0->qword[0] = vlib_buffer_get_current_pa (vm, b0); + d1->qword[0] = vlib_buffer_get_current_pa (vm, b1); + d2->qword[0] = vlib_buffer_get_current_pa (vm, b2); + d3->qword[0] = vlib_buffer_get_current_pa (vm, b3); + } + d0->qword[1] = ((u64) b0->current_length) << 34 | bits; d1->qword[1] = ((u64) b1->current_length) << 34 | bits; d2->qword[1] = ((u64) b2->current_length) << 34 | bits; @@ -146,12 +151,11 @@ retry: txq->bufs[next] = bi0; b0 = vlib_get_buffer (vm, bi0); -#if 0 - d->qword[0] = vlib_get_buffer_data_physical_address (vm, bi0) + - b0->current_data; -#else - d0->qword[0] = pointer_to_uword (b0->data) + b0->current_data; -#endif + if (ad->flags & AVF_DEVICE_F_IOVA) + d0->qword[0] = vlib_buffer_get_current_va (b0); + else + d0->qword[0] = vlib_buffer_get_current_pa (vm, b0); + d0->qword[1] = (((u64) b0->current_length) << 34) | bits; next = (next + 1) & mask; diff --git a/src/plugins/ixge/ixge.c b/src/plugins/ixge/ixge.c index aa14eac7fd7..5cf83cfb670 100644 --- a/src/plugins/ixge/ixge.c +++ b/src/plugins/ixge/ixge.c @@ -1097,10 +1097,8 @@ ixge_tx_no_wrap (ixge_main_t * xm, ASSERT (ixge_tx_descriptor_matches_template (xm, d + 0)); ASSERT (ixge_tx_descriptor_matches_template (xm, d + 1)); - d[0].buffer_address = - vlib_get_buffer_data_physical_address (vm, bi0) + b0->current_data; - d[1].buffer_address = - vlib_get_buffer_data_physical_address (vm, bi1) + b1->current_data; + d[0].buffer_address = vlib_buffer_get_pa (vm, b0); + d[1].buffer_address = vlib_buffer_get_pa (vm, b1); d[0].n_bytes_this_buffer = len0; d[1].n_bytes_this_buffer = len1; @@ -1153,9 +1151,7 @@ ixge_tx_no_wrap (ixge_main_t * xm, ASSERT (ixge_tx_descriptor_matches_template (xm, d + 0)); - d[0].buffer_address = - vlib_get_buffer_data_physical_address (vm, bi0) + b0->current_data; - + d[0].buffer_address = vlib_buffer_get_pa (vm, b0); d[0].n_bytes_this_buffer = len0; d[0].status0 = @@ -1425,6 +1421,7 @@ ixge_rx_queue_no_wrap (ixge_main_t * xm, while (n_descriptors_left >= 4 && n_left_to_next >= 2) { vlib_buffer_t *b0, *b1; + vlib_buffer_t *f0, *f1; u32 bi0, fi0, len0, l3_offset0, s20, s00, flags0; u32 bi1, fi1, len1, l3_offset1, s21, s01, flags1; u8 is_eop0, error0, next0; @@ -1510,10 +1507,10 @@ ixge_rx_queue_no_wrap (ixge_main_t * xm, n_packets += is_eop0 + is_eop1; /* Give new buffers to hardware. */ - d0.rx_to_hw.tail_address = - vlib_get_buffer_data_physical_address (vm, fi0); - d1.rx_to_hw.tail_address = - vlib_get_buffer_data_physical_address (vm, fi1); + f0 = vlib_get_buffer (vm, fi0); + f1 = vlib_get_buffer (vm, fi1); + d0.rx_to_hw.tail_address = vlib_buffer_get_pa (vm, f0); + d1.rx_to_hw.tail_address = vlib_buffer_get_pa (vm, f1); d0.rx_to_hw.head_address = d[0].rx_to_hw.tail_address; d1.rx_to_hw.head_address = d[1].rx_to_hw.tail_address; d[0].as_u32x4 = d0.as_u32x4; @@ -1655,6 +1652,7 @@ ixge_rx_queue_no_wrap (ixge_main_t * xm, while (n_descriptors_left > 0 && n_left_to_next > 0) { vlib_buffer_t *b0; + vlib_buffer_t *f0; u32 bi0, fi0, len0, l3_offset0, s20, s00, flags0; u8 is_eop0, error0, next0; ixge_descriptor_t d0; @@ -1708,8 +1706,8 @@ ixge_rx_queue_no_wrap (ixge_main_t * xm, n_packets += is_eop0; /* Give new buffer to hardware. */ - d0.rx_to_hw.tail_address = - vlib_get_buffer_data_physical_address (vm, fi0); + f0 = vlib_get_buffer (vm, fi0); + d0.rx_to_hw.tail_address = vlib_buffer_get_pa (vm, f0); d0.rx_to_hw.head_address = d0.rx_to_hw.tail_address; d[0].as_u32x4 = d0.as_u32x4; @@ -2510,9 +2508,8 @@ ixge_dma_init (ixge_device_t * xd, vlib_rx_or_tx_t rt, u32 queue_index) for (i = 0; i < n_alloc; i++) { dq->descriptors[i].rx_to_hw.tail_address = - vlib_get_buffer_data_physical_address (vm, - dq->descriptor_buffer_indices - [i]); + vlib_buffer_get_pa + (vm, vlib_get_buffer (vm, dq->descriptor_buffer_indices[i])); } } else diff --git a/src/plugins/marvell/pp2/input.c b/src/plugins/marvell/pp2/input.c index 72c1fe02f28..c1a0c624ea9 100644 --- a/src/plugins/marvell/pp2/input.c +++ b/src/plugins/marvell/pp2/input.c @@ -324,7 +324,8 @@ mrvl_pp2_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, while (i--) { u32 bi = buffers[0]; - e->buff.addr = vlib_get_buffer_data_physical_address (vm, bi) - 64; + vlib_buffer_t *b = vlib_get_buffer (vm, bi); + e->buff.addr = vlib_buffer_get_pa (vm, b) - 64; e->buff.cookie = bi; e->bpool = inq->bpool; e++; diff --git a/src/plugins/marvell/pp2/output.c b/src/plugins/marvell/pp2/output.c index 16efae5eecc..108bbef72f2 100644 --- a/src/plugins/marvell/pp2/output.c +++ b/src/plugins/marvell/pp2/output.c @@ -67,7 +67,7 @@ mrvl_pp2_interface_tx (vlib_main_t * vm, { u32 bi0 = buffers[0]; vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); - u64 paddr = vlib_get_buffer_data_physical_address (vm, bi0); + u64 paddr = vlib_buffer_get_va (vm, b0); pp2_ppio_outq_desc_reset (d); pp2_ppio_outq_desc_set_phys_addr (d, paddr + b0->current_data); diff --git a/src/plugins/vmxnet3/output.c b/src/plugins/vmxnet3/output.c index 2a8494ed447..ce810f049e2 100644 --- a/src/plugins/vmxnet3/output.c +++ b/src/plugins/vmxnet3/output.c @@ -178,9 +178,7 @@ VNET_DEVICE_CLASS_TX_FN (vmxnet3_device_class) (vlib_main_t * vm, txq->tx_ring.bufs[desc_idx] = bi0; txd = &txq->tx_desc[desc_idx]; - txd->address = - vlib_get_buffer_data_physical_address (vm, - bi0) + b0->current_data; + txd->address = vlib_buffer_get_current_pa (vm, b0); txd->flags[0] = generation | b0->current_length; diff --git a/src/plugins/vmxnet3/vmxnet3.h b/src/plugins/vmxnet3/vmxnet3.h index 491b8c10246..391ddc17113 100644 --- a/src/plugins/vmxnet3/vmxnet3.h +++ b/src/plugins/vmxnet3/vmxnet3.h @@ -590,9 +590,9 @@ vmxnet3_rxq_refill_ring0 (vlib_main_t * vm, vmxnet3_device_t * vd, while (n_alloc) { + vlib_buffer_t *b = vlib_get_buffer (vm, ring->bufs[ring->produce]); rxd = &rxq->rx_desc[0][ring->produce]; - rxd->address = - vlib_get_buffer_data_physical_address (vm, ring->bufs[ring->produce]); + rxd->address = vlib_buffer_get_pa (vm, b); rxd->flags = ring->gen | VLIB_BUFFER_DATA_SIZE; vmxnet3_rx_ring_advance_produce (rxq, ring); @@ -632,9 +632,9 @@ vmxnet3_rxq_refill_ring1 (vlib_main_t * vm, vmxnet3_device_t * vd, while (n_alloc) { + vlib_buffer_t *b = vlib_get_buffer (vm, ring->bufs[ring->produce]); rxd = &rxq->rx_desc[1][ring->produce]; - rxd->address = - vlib_get_buffer_data_physical_address (vm, ring->bufs[ring->produce]); + rxd->address = vlib_buffer_get_pa (vm, b); rxd->flags = ring->gen | VLIB_BUFFER_DATA_SIZE | VMXNET3_RXF_BTYPE; vmxnet3_rx_ring_advance_produce (rxq, ring); diff --git a/src/vlib/buffer.h b/src/vlib/buffer.h index 651e7f0dc8f..0d24779e52b 100644 --- a/src/vlib/buffer.h +++ b/src/vlib/buffer.h @@ -187,6 +187,12 @@ vlib_buffer_struct_is_sane (vlib_buffer_t * b) ASSERT (b->pre_data + VLIB_BUFFER_PRE_DATA_SIZE == b->data); } +always_inline uword +vlib_buffer_get_va (vlib_buffer_t * b) +{ + return pointer_to_uword (b->data); +} + /** \brief Get pointer to current data to process @param b - (vlib_buffer_t *) pointer to the buffer @@ -201,6 +207,12 @@ vlib_buffer_get_current (vlib_buffer_t * b) return b->data + b->current_data; } +always_inline uword +vlib_buffer_get_current_va (vlib_buffer_t * b) +{ + return vlib_buffer_get_va (b) + b->current_data; +} + /** \brief Advance current data pointer by the supplied (signed!) amount @param b - (vlib_buffer_t *) pointer to the buffer diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index 438bf7e5ee4..1110c206e52 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -314,18 +314,21 @@ vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents) return content_len; } -/* Return physical address of buffer->data start. */ -always_inline u64 -vlib_get_buffer_data_physical_address (vlib_main_t * vm, u32 buffer_index) +always_inline uword +vlib_buffer_get_pa (vlib_main_t * vm, vlib_buffer_t * b) { vlib_buffer_main_t *bm = &buffer_main; - vlib_buffer_t *b = vlib_get_buffer (vm, buffer_index); vlib_buffer_pool_t *pool = vec_elt_at_index (bm->buffer_pools, b->buffer_pool_index); - return vlib_physmem_virtual_to_physical (vm, pool->physmem_region, b->data); } +always_inline uword +vlib_buffer_get_current_pa (vlib_main_t * vm, vlib_buffer_t * b) +{ + return vlib_buffer_get_pa (vm, b) + b->current_data; +} + /** \brief Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header information -- cgit 1.2.3-korg