From 68b4da67deb2e8ca224bb5abaeb9dbc7ae8e378c Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Sun, 30 Sep 2018 18:26:20 +0200 Subject: Numa-aware, growable physical memory allocator (pmalloc) Change-Id: Ic4c46bc733afae8bf0d8146623ed15633928de30 Signed-off-by: Damjan Marion --- src/plugins/avf/avf.h | 4 +- src/plugins/avf/device.c | 103 +++++++++++++++++++++++++++-------------------- src/plugins/avf/input.c | 12 +++--- src/plugins/avf/output.c | 4 +- 4 files changed, 68 insertions(+), 55 deletions(-) (limited to 'src/plugins/avf') diff --git a/src/plugins/avf/avf.h b/src/plugins/avf/avf.h index 187e5c2cd69..4fa19b11b73 100644 --- a/src/plugins/avf/avf.h +++ b/src/plugins/avf/avf.h @@ -26,7 +26,7 @@ _(0, INITIALIZED, "initialized") \ _(1, ERROR, "error") \ _(2, ADMIN_UP, "admin-up") \ - _(3, IOVA, "iova") \ + _(3, VA_DMA, "vaddr-dma") \ _(4, LINK_UP, "link-up") \ _(5, SHARED_TXQ_LOCK, "shared-txq-lock") \ _(6, ELOG, "elog") @@ -188,8 +188,6 @@ typedef struct avf_device_t *devices; avf_per_thread_data_t *per_thread_data; - vlib_physmem_region_index_t physmem_region; - int physmem_region_alloc; vlib_log_class_t log_class; diff --git a/src/plugins/avf/device.c b/src/plugins/avf/device.c index 914c32cab4b..713953fe4c6 100644 --- a/src/plugins/avf/device.c +++ b/src/plugins/avf/device.c @@ -215,18 +215,23 @@ avf_cmd_rx_ctl_reg_write (vlib_main_t * vm, avf_device_t * ad, u32 reg, clib_error_t * avf_rxq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 rxq_size) { - avf_main_t *am = &avf_main; + clib_error_t *err; avf_rxq_t *rxq; - clib_error_t *error = 0; u32 n_alloc, i; vec_validate_aligned (ad->rxqs, qid, CLIB_CACHE_LINE_BYTES); rxq = vec_elt_at_index (ad->rxqs, qid); rxq->size = rxq_size; rxq->next = 0; - rxq->descs = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error, - rxq->size * sizeof (avf_rx_desc_t), + rxq->descs = vlib_physmem_alloc_aligned (vm, rxq->size * + sizeof (avf_rx_desc_t), 2 * CLIB_CACHE_LINE_BYTES); + if (rxq->descs == 0) + return vlib_physmem_last_error (vm); + + if ((err = vlib_pci_map_dma (vm, ad->pci_dev_handle, (void *) rxq->descs))) + return err; + clib_memset ((void *) rxq->descs, 0, rxq->size * sizeof (avf_rx_desc_t)); vec_validate_aligned (rxq->bufs, rxq->size, CLIB_CACHE_LINE_BYTES); rxq->qrx_tail = ad->bar0 + AVF_QRX_TAIL (qid); @@ -241,7 +246,7 @@ avf_rxq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 rxq_size) for (i = 0; i < n_alloc; i++) { vlib_buffer_t *b = vlib_get_buffer (vm, rxq->bufs[i]); - if (ad->flags & AVF_DEVICE_F_IOVA) + if (ad->flags & AVF_DEVICE_F_VA_DMA) d->qword[0] = vlib_buffer_get_va (b); else d->qword[0] = vlib_buffer_get_pa (vm, b); @@ -255,9 +260,8 @@ avf_rxq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 rxq_size) clib_error_t * avf_txq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 txq_size) { - avf_main_t *am = &avf_main; + clib_error_t *err; avf_txq_t *txq; - clib_error_t *error = 0; if (qid >= ad->num_queue_pairs) { @@ -273,9 +277,15 @@ avf_txq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 txq_size) txq = vec_elt_at_index (ad->txqs, qid); txq->size = txq_size; txq->next = 0; - txq->descs = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error, - txq->size * sizeof (avf_tx_desc_t), + txq->descs = vlib_physmem_alloc_aligned (vm, txq->size * + sizeof (avf_tx_desc_t), 2 * CLIB_CACHE_LINE_BYTES); + if (txq->descs == 0) + return vlib_physmem_last_error (vm); + + if ((err = vlib_pci_map_dma (vm, ad->pci_dev_handle, (void *) txq->descs))) + return err; + vec_validate_aligned (txq->bufs, txq->size, CLIB_CACHE_LINE_BYTES); txq->qtx_tail = ad->bar0 + AVF_QTX_TAIL (qid); @@ -305,10 +315,8 @@ avf_arq_slot_init (avf_device_t * ad, u16 slot) static inline uword avf_dma_addr (vlib_main_t * vm, avf_device_t * ad, void *p) { - avf_main_t *am = &avf_main; - return (ad->flags & AVF_DEVICE_F_IOVA) ? - pointer_to_uword (p) : - vlib_physmem_virtual_to_physical (vm, am->physmem_region, p); + return (ad->flags & AVF_DEVICE_F_VA_DMA) ? + pointer_to_uword (p) : vlib_physmem_get_pa (vm, p); } static void @@ -1126,16 +1134,16 @@ avf_delete_if (vlib_main_t * vm, avf_device_t * ad) vlib_pci_device_close (vm, ad->pci_dev_handle); - vlib_physmem_free (vm, am->physmem_region, ad->atq); - vlib_physmem_free (vm, am->physmem_region, ad->arq); - vlib_physmem_free (vm, am->physmem_region, ad->atq_bufs); - vlib_physmem_free (vm, am->physmem_region, ad->arq_bufs); + vlib_physmem_free (vm, ad->atq); + vlib_physmem_free (vm, ad->arq); + vlib_physmem_free (vm, ad->atq_bufs); + vlib_physmem_free (vm, ad->arq_bufs); /* *INDENT-OFF* */ vec_foreach_index (i, ad->rxqs) { avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i); - vlib_physmem_free (vm, am->physmem_region, (void *) rxq->descs); + vlib_physmem_free (vm, (void *) rxq->descs); if (rxq->n_enqueued) vlib_buffer_free_from_ring (vm, rxq->bufs, rxq->next, rxq->size, rxq->n_enqueued); @@ -1148,7 +1156,7 @@ avf_delete_if (vlib_main_t * vm, avf_device_t * ad) vec_foreach_index (i, ad->txqs) { avf_txq_t *txq = vec_elt_at_index (ad->txqs, i); - vlib_physmem_free (vm, am->physmem_region, (void *) txq->descs); + vlib_physmem_free (vm, (void *) txq->descs); if (txq->n_enqueued) { u16 first = (txq->next - txq->n_enqueued) & (txq->size -1); @@ -1226,44 +1234,51 @@ avf_create_if (vlib_main_t * vm, avf_create_if_args_t * args) if ((error = vlib_pci_enable_msix_irq (vm, h, 0, 2))) goto error; - if (am->physmem_region_alloc == 0) + if (!(ad->atq = vlib_physmem_alloc (vm, sizeof (avf_aq_desc_t) * + AVF_MBOX_LEN))) { - u32 flags = VLIB_PHYSMEM_F_INIT_MHEAP | VLIB_PHYSMEM_F_HUGETLB; - error = vlib_physmem_region_alloc (vm, "avf descriptors", 4 << 20, 0, - flags, &am->physmem_region); - if (error) - goto error; - am->physmem_region_alloc = 1; + error = vlib_physmem_last_error (vm); + goto error; } - ad->atq = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error, - sizeof (avf_aq_desc_t) * AVF_MBOX_LEN, - 64); - if (error) + + if ((error = vlib_pci_map_dma (vm, h, ad->atq))) goto error; - ad->arq = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error, - sizeof (avf_aq_desc_t) * AVF_MBOX_LEN, - 64); - if (error) + if (!(ad->arq = vlib_physmem_alloc (vm, sizeof (avf_aq_desc_t) * + AVF_MBOX_LEN))) + { + error = vlib_physmem_last_error (vm); + goto error; + } + + if ((error = vlib_pci_map_dma (vm, h, ad->arq))) goto error; - ad->atq_bufs = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error, - AVF_MBOX_BUF_SZ * AVF_MBOX_LEN, - 64); - if (error) + if (!(ad->atq_bufs = vlib_physmem_alloc (vm, AVF_MBOX_BUF_SZ * + AVF_MBOX_LEN))) + { + error = vlib_physmem_last_error (vm); + goto error; + } + + if ((error = vlib_pci_map_dma (vm, h, ad->atq_bufs))) goto error; - ad->arq_bufs = vlib_physmem_alloc_aligned (vm, am->physmem_region, &error, - AVF_MBOX_BUF_SZ * AVF_MBOX_LEN, - 64); - if (error) + if (!(ad->arq_bufs = vlib_physmem_alloc (vm, AVF_MBOX_BUF_SZ * + AVF_MBOX_LEN))) + { + error = vlib_physmem_last_error (vm); + goto error; + } + + if ((error = vlib_pci_map_dma (vm, h, ad->arq_bufs))) goto error; if ((error = vlib_pci_intr_enable (vm, h))) goto error; - /* FIXME detect */ - ad->flags |= AVF_DEVICE_F_IOVA; + if (vlib_pci_supports_virtual_addr_dma (vm, h)) + ad->flags |= AVF_DEVICE_F_VA_DMA; if ((error = avf_device_init (vm, am, ad, args))) goto error; diff --git a/src/plugins/avf/input.c b/src/plugins/avf/input.c index efe3b0e0251..6d39ed646b5 100644 --- a/src/plugins/avf/input.c +++ b/src/plugins/avf/input.c @@ -48,7 +48,7 @@ static __clib_unused char *avf_input_error_strings[] = { #define AVF_INPUT_REFILL_TRESHOLD 32 static_always_inline void avf_rxq_refill (vlib_main_t * vm, vlib_node_runtime_t * node, avf_rxq_t * rxq, - int use_iova) + int use_va_dma) { u16 n_refill, mask, n_alloc, slot; u32 s0, s1, s2, s3; @@ -103,7 +103,7 @@ avf_rxq_refill (vlib_main_t * vm, vlib_node_runtime_t * node, avf_rxq_t * rxq, b[2] = vlib_get_buffer (vm, rxq->bufs[s2]); b[3] = vlib_get_buffer (vm, rxq->bufs[s3]); - if (use_iova) + if (use_va_dma) { d[0]->qword[0] = vlib_buffer_get_va (b[0]); d[1]->qword[0] = vlib_buffer_get_va (b[1]); @@ -132,7 +132,7 @@ avf_rxq_refill (vlib_main_t * vm, vlib_node_runtime_t * node, avf_rxq_t * rxq, s0 = slot; d[0] = ((avf_rx_desc_t *) rxq->descs) + s0; b[0] = vlib_get_buffer (vm, rxq->bufs[s0]); - if (use_iova) + if (use_va_dma) d[0]->qword[0] = vlib_buffer_get_va (b[0]); else d[0]->qword[0] = vlib_buffer_get_pa (vm, b[0]); @@ -433,10 +433,10 @@ avf_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, goto done; /* refill rx ring */ - if (ad->flags & AVF_DEVICE_F_IOVA) - avf_rxq_refill (vm, node, rxq, 1 /* use_iova */ ); + if (ad->flags & AVF_DEVICE_F_VA_DMA) + avf_rxq_refill (vm, node, rxq, 1 /* use_va_dma */ ); else - avf_rxq_refill (vm, node, rxq, 0 /* use_iova */ ); + avf_rxq_refill (vm, node, rxq, 0 /* use_va_dma */ ); vlib_get_buffers (vm, buffer_indices, bufs, n_rxv); n_rx_packets = n_rxv; diff --git a/src/plugins/avf/output.c b/src/plugins/avf/output.c index c2c4b01d022..72892d16dff 100644 --- a/src/plugins/avf/output.c +++ b/src/plugins/avf/output.c @@ -118,7 +118,7 @@ retry: b2 = vlib_get_buffer (vm, bi2); b3 = vlib_get_buffer (vm, bi3); - if (ad->flags & AVF_DEVICE_F_IOVA) + if (ad->flags & AVF_DEVICE_F_VA_DMA) { d0->qword[0] = vlib_buffer_get_current_va (b0); d1->qword[0] = vlib_buffer_get_current_va (b1); @@ -151,7 +151,7 @@ retry: txq->bufs[next] = bi0; b0 = vlib_get_buffer (vm, bi0); - if (ad->flags & AVF_DEVICE_F_IOVA) + if (ad->flags & AVF_DEVICE_F_VA_DMA) d0->qword[0] = vlib_buffer_get_current_va (b0); else d0->qword[0] = vlib_buffer_get_current_pa (vm, b0); -- cgit 1.2.3-korg