From 9a02c54e4f3c53cbaceee4cbb30835113f0641ce Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Wed, 11 Jul 2018 13:29:17 +0200 Subject: avf: descriptor should be volatile Change-Id: I2cb4cf2167b6e958d2e57b461848a4a189e3fda0 Signed-off-by: Damjan Marion --- src/plugins/avf/avf.h | 30 +++++++++++++++++++++++++----- src/plugins/avf/device.c | 32 ++++++++++++++++---------------- src/plugins/avf/input.c | 26 +++++++++++++------------- src/plugins/avf/output.c | 12 ++++++------ 4 files changed, 60 insertions(+), 40 deletions(-) diff --git a/src/plugins/avf/avf.h b/src/plugins/avf/avf.h index c8074ebf6c7..32bbb46fde4 100644 --- a/src/plugins/avf/avf.h +++ b/src/plugins/avf/avf.h @@ -38,19 +38,39 @@ enum #undef _ }; -typedef struct +typedef volatile struct { - u64 qword[4]; + union + { + struct + { + u64 mirr:13; + u64 rsv1:3; + u64 l2tag1:16; + u64 filter_status:32; + u64 status:19; + u64 error:8; + u64 rsv2:3; + u64 ptype:8; + u64 length:26; + }; + u64 qword[4]; +#ifdef CLIB_HAVE_VEC256 + u64x4 as_u64x4; +#endif + }; } avf_rx_desc_t; STATIC_ASSERT_SIZEOF (avf_rx_desc_t, 32); -typedef struct +typedef volatile struct { union { u64 qword[2]; +#ifdef CLIB_HAVE_VEC128 u64x2 as_u64x2; +#endif }; } avf_tx_desc_t; @@ -64,7 +84,7 @@ typedef struct u16 size; avf_rx_desc_t *descs; u32 *bufs; - u16 n_bufs; + u16 n_enqueued; } avf_rxq_t; typedef struct @@ -76,7 +96,7 @@ typedef struct clib_spinlock_t lock; avf_tx_desc_t *descs; u32 *bufs; - u16 n_bufs; + u16 n_enqueued; } avf_txq_t; typedef struct diff --git a/src/plugins/avf/device.c b/src/plugins/avf/device.c index 0a7de09060e..1d214e18ce7 100644 --- a/src/plugins/avf/device.c +++ b/src/plugins/avf/device.c @@ -227,7 +227,7 @@ avf_rxq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 rxq_size) rxq->descs = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error, rxq->size * sizeof (avf_rx_desc_t), 2 * CLIB_CACHE_LINE_BYTES); - memset (rxq->descs, 0, rxq->size * sizeof (avf_rx_desc_t)); + memset ((void *) rxq->descs, 0, rxq->size * sizeof (avf_rx_desc_t)); vec_validate_aligned (rxq->bufs, rxq->size, CLIB_CACHE_LINE_BYTES); rxq->qrx_tail = ad->bar0 + AVF_QRX_TAIL (qid); @@ -236,7 +236,7 @@ avf_rxq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 rxq_size) if (n_alloc == 0) return clib_error_return (0, "buffer allocation error"); - rxq->n_bufs = n_alloc; + rxq->n_enqueued = n_alloc; avf_rx_desc_t *d = rxq->descs; for (i = 0; i < n_alloc; i++) { @@ -323,7 +323,7 @@ avf_adminq_init (vlib_main_t * vm, avf_device_t * ad) pa = avf_dma_addr (vm, ad, ad->atq); avf_reg_write (ad, AVF_ATQT, 0); /* Tail */ avf_reg_write (ad, AVF_ATQH, 0); /* Head */ - avf_reg_write (ad, AVF_ATQLEN, AVF_MBOX_LEN | (1 << 31)); /* len & ena */ + avf_reg_write (ad, AVF_ATQLEN, AVF_MBOX_LEN | (1ULL << 31)); /* len & ena */ avf_reg_write (ad, AVF_ATQBAL, (u32) pa); /* Base Address Low */ avf_reg_write (ad, AVF_ATQBAH, (u32) (pa >> 32)); /* Base Address High */ @@ -338,7 +338,7 @@ avf_adminq_init (vlib_main_t * vm, avf_device_t * ad) avf_reg_write (ad, AVF_ARQH, 0); /* Head */ avf_reg_write (ad, AVF_ARQT, 0); /* Head */ - avf_reg_write (ad, AVF_ARQLEN, AVF_MBOX_LEN | (1 << 31)); /* len & ena */ + avf_reg_write (ad, AVF_ARQLEN, AVF_MBOX_LEN | (1ULL << 31)); /* len & ena */ avf_reg_write (ad, AVF_ARQBAL, (u32) pa); /* Base Address Low */ avf_reg_write (ad, AVF_ARQBAH, (u32) (pa >> 32)); /* Base Address High */ avf_reg_write (ad, AVF_ARQT, AVF_MBOX_LEN - 1); /* Tail */ @@ -552,7 +552,7 @@ avf_op_config_vsi_queues (vlib_main_t * vm, avf_device_t * ad) avf_rxq_t *q = vec_elt_at_index (ad->rxqs, i); rxq->ring_len = q->size; rxq->databuffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES; - rxq->dma_ring_addr = avf_dma_addr (vm, ad, q->descs); + rxq->dma_ring_addr = avf_dma_addr (vm, ad, (void *) q->descs); avf_reg_write (ad, AVF_QRX_TAIL (i), q->size - 1); } @@ -562,7 +562,7 @@ avf_op_config_vsi_queues (vlib_main_t * vm, avf_device_t * ad) { txq->queue_id = i; txq->ring_len = q->size; - txq->dma_ring_addr = avf_dma_addr (vm, ad, q->descs); + txq->dma_ring_addr = avf_dma_addr (vm, ad, (void *) q->descs); } } @@ -618,7 +618,7 @@ avf_op_enable_queues (vlib_main_t * vm, avf_device_t * ad, u32 rx, u32 tx) qs.rx_queues = rx; qs.tx_queues = tx; avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, 0); - avf_reg_write (ad, AVF_QRX_TAIL (0), rxq->n_bufs); + avf_reg_write (ad, AVF_QRX_TAIL (0), rxq->n_enqueued); return avf_send_to_pf (vm, ad, VIRTCHNL_OP_ENABLE_QUEUES, &qs, sizeof (virtchnl_queue_select_t), 0, 0); } @@ -769,14 +769,14 @@ avf_process_one_device (vlib_main_t * vm, avf_device_t * ad, int is_irq) ASSERT (ad->error == 0); r = avf_get_u32 (ad->bar0, AVF_ARQLEN); - if ((r & 0xf0000000) != (1 << 31)) + if ((r & 0xf0000000) != (1ULL << 31)) { ad->error = clib_error_return (0, "arq not enabled, arqlen = 0x%x", r); goto error; } r = avf_get_u32 (ad->bar0, AVF_ATQLEN); - if ((r & 0xf0000000) != (1 << 31)) + if ((r & 0xf0000000) != (1ULL << 31)) { ad->error = clib_error_return (0, "atq not enabled, atqlen = 0x%x", r); goto error; @@ -1032,10 +1032,10 @@ avf_delete_if (vlib_main_t * vm, avf_device_t * ad) vec_foreach_index (i, ad->rxqs) { avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i); - vlib_physmem_free (vm, am->physmem_region, rxq->descs); - if (rxq->n_bufs) + vlib_physmem_free (vm, am->physmem_region, (void *) rxq->descs); + if (rxq->n_enqueued) vlib_buffer_free_from_ring (vm, rxq->bufs, rxq->next, rxq->size, - rxq->n_bufs); + rxq->n_enqueued); vec_free (rxq->bufs); } /* *INDENT-ON* */ @@ -1045,12 +1045,12 @@ avf_delete_if (vlib_main_t * vm, avf_device_t * ad) vec_foreach_index (i, ad->txqs) { avf_txq_t *txq = vec_elt_at_index (ad->txqs, i); - vlib_physmem_free (vm, am->physmem_region, txq->descs); - if (txq->n_bufs) + vlib_physmem_free (vm, am->physmem_region, (void *) txq->descs); + if (txq->n_enqueued) { - u16 first = (txq->next - txq->n_bufs) & (txq->size -1); + u16 first = (txq->next - txq->n_enqueued) & (txq->size -1); vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size, - txq->n_bufs); + txq->n_enqueued); } vec_free (txq->bufs); } diff --git a/src/plugins/avf/input.c b/src/plugins/avf/input.c index 475b207ed74..efec7234e2d 100644 --- a/src/plugins/avf/input.c +++ b/src/plugins/avf/input.c @@ -54,7 +54,7 @@ avf_rxq_refill (vlib_main_t * vm, vlib_node_runtime_t * node, avf_rxq_t * rxq, u32 s0, s1, s2, s3; avf_rx_desc_t *d[4]; - n_refill = rxq->size - 1 - rxq->n_bufs; + n_refill = rxq->size - 1 - rxq->n_enqueued; if (PREDICT_TRUE (n_refill <= AVF_INPUT_REFILL_TRESHOLD)) return; @@ -74,7 +74,7 @@ avf_rxq_refill (vlib_main_t * vm, vlib_node_runtime_t * node, avf_rxq_t * rxq, return; } - rxq->n_bufs += n_alloc; + rxq->n_enqueued += n_alloc; while (n_alloc >= 4) { @@ -355,13 +355,13 @@ avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, if (rxq->next + 11 < rxq->size) { int stride = 8; - CLIB_PREFETCH (rxq->descs + (rxq->next + stride), + CLIB_PREFETCH ((void *) (rxq->descs + (rxq->next + stride)), CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (rxq->descs + (rxq->next + stride + 1), + CLIB_PREFETCH ((void *) (rxq->descs + (rxq->next + stride + 1)), CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (rxq->descs + (rxq->next + stride + 2), + CLIB_PREFETCH ((void *) (rxq->descs + (rxq->next + stride + 2)), CLIB_CACHE_LINE_BYTES, LOAD); - CLIB_PREFETCH (rxq->descs + (rxq->next + stride + 3), + CLIB_PREFETCH ((void *) (rxq->descs + (rxq->next + stride + 3)), CLIB_CACHE_LINE_BYTES, LOAD); } @@ -404,21 +404,21 @@ avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, rxq->next = (rxq->next + 4) & mask; d = rxq->descs + rxq->next; n_rxv += 4; - rxq->n_bufs -= 4; + rxq->n_enqueued -= 4; bi += 4; continue; one_by_one: #endif - CLIB_PREFETCH (rxq->descs + ((rxq->next + 8) & mask), + CLIB_PREFETCH ((void *) (rxq->descs + ((rxq->next + 8) & mask)), CLIB_CACHE_LINE_BYTES, LOAD); if ((d->qword[1] & AVF_RX_DESC_STATUS_DD) == 0) break; rxve = ptd->rx_vector + n_rxv; bi[0] = rxq->bufs[rxq->next]; - rxve->status = avf_get_u64_bits (d, 8, 18, 0); - rxve->error = avf_get_u64_bits (d, 8, 26, 19); - rxve->ptype = avf_get_u64_bits (d, 8, 37, 30); - rxve->length = avf_get_u64_bits (d, 8, 63, 38); + rxve->status = avf_get_u64_bits ((void *) d, 8, 18, 0); + rxve->error = avf_get_u64_bits ((void *) d, 8, 26, 19); + rxve->ptype = avf_get_u64_bits ((void *) d, 8, 37, 30); + rxve->length = avf_get_u64_bits ((void *) d, 8, 63, 38); maybe_error |= rxve->error; /* deal with chained buffers */ @@ -431,7 +431,7 @@ avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, rxq->next = (rxq->next + 1) & mask; d = rxq->descs + rxq->next; n_rxv++; - rxq->n_bufs--; + rxq->n_enqueued--; bi++; } diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c index d5e71d528ab..2722c99f31c 100644 --- a/src/plugins/avf/output.c +++ b/src/plugins/avf/output.c @@ -57,12 +57,12 @@ CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm, clib_spinlock_lock_if_init (&txq->lock); /* release cosumed bufs */ - if (txq->n_bufs) + if (txq->n_enqueued) { u16 first, slot, n_free = 0; - first = slot = (txq->next - txq->n_bufs) & mask; + first = slot = (txq->next - txq->n_enqueued) & mask; d0 = txq->descs + slot; - while (n_free < txq->n_bufs && avf_tx_desc_get_dtyp (d0) == 0x0F) + while (n_free < txq->n_enqueued && avf_tx_desc_get_dtyp (d0) == 0x0F) { n_free++; slot = (slot + 1) & mask; @@ -71,7 +71,7 @@ CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm, if (n_free) { - txq->n_bufs -= n_free;; + txq->n_enqueued -= n_free; vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size, n_free); } @@ -126,7 +126,7 @@ CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm, d3->qword[1] = ((u64) b3->current_length) << 34 | bits; txq->next = (txq->next + 4) & mask; - txq->n_bufs += 4; + txq->n_enqueued += 4; buffers += 4; n_left -= 4; } @@ -147,7 +147,7 @@ CLIB_MULTIARCH_FN (avf_interface_tx) (vlib_main_t * vm, d0->qword[1] = (((u64) b0->current_length) << 34) | bits; txq->next = (txq->next + 1) & mask; - txq->n_bufs++; + txq->n_enqueued++; buffers++; n_left--; } -- cgit 1.2.3-korg