diff options
Diffstat (limited to 'drivers/net/virtio')
| -rw-r--r-- | drivers/net/virtio/virtio_ethdev.c | 32 | ||||
| -rw-r--r-- | drivers/net/virtio/virtio_rxtx.c | 40 | ||||
| -rw-r--r-- | drivers/net/virtio/virtio_rxtx.h | 3 | ||||
| -rw-r--r-- | drivers/net/virtio/virtio_rxtx_simple.c | 30 | ||||
| -rw-r--r-- | drivers/net/virtio/virtio_rxtx_simple.h | 2 | ||||
| -rw-r--r-- | drivers/net/virtio/virtio_user/virtio_user_dev.c | 4 | ||||
| -rw-r--r-- | drivers/net/virtio/virtqueue.c | 61 | ||||
| -rw-r--r-- | drivers/net/virtio/virtqueue.h | 15 |
8 files changed, 107 insertions, 80 deletions
diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c index 8592485d..5a27ab77 100644 --- a/drivers/net/virtio/virtio_ethdev.c +++ b/drivers/net/virtio/virtio_ethdev.c @@ -288,17 +288,6 @@ virtio_dev_queue_release(void *queue __rte_unused) /* do nothing */ } -static int -virtio_get_queue_type(struct virtio_hw *hw, uint16_t vtpci_queue_idx) -{ - if (vtpci_queue_idx == hw->max_queue_pairs * 2) - return VTNET_CQ; - else if (vtpci_queue_idx % 2 == 0) - return VTNET_RQ; - else - return VTNET_TQ; -} - static uint16_t virtio_get_nr_vq(struct virtio_hw *hw) { @@ -847,7 +836,7 @@ static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev, /* Note: limit checked in rte_eth_xstats_names() */ for (i = 0; i < dev->data->nb_rx_queues; i++) { - struct virtqueue *rxvq = dev->data->rx_queues[i]; + struct virtnet_rx *rxvq = dev->data->rx_queues[i]; if (rxvq == NULL) continue; for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) { @@ -860,7 +849,7 @@ static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev, } for (i = 0; i < dev->data->nb_tx_queues; i++) { - struct virtqueue *txvq = dev->data->tx_queues[i]; + struct virtnet_tx *txvq = dev->data->tx_queues[i]; if (txvq == NULL) continue; for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) { @@ -1205,6 +1194,11 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) /* Reset the device although not necessary at startup */ vtpci_reset(hw); + if (hw->vqs) { + virtio_dev_free_mbufs(eth_dev); + virtio_free_queues(hw); + } + /* Tell the host we've noticed this device. */ vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); @@ -1565,7 +1559,7 @@ virtio_dev_start(struct rte_eth_dev *dev) for (i = 0; i < dev->data->nb_rx_queues; i++) { rxvq = dev->data->rx_queues[i]; /* Flush the old packets */ - virtqueue_flush(rxvq->vq); + virtqueue_rxvq_flush(rxvq->vq); virtqueue_notify(rxvq->vq); } @@ -1597,12 +1591,15 @@ static void virtio_dev_free_mbufs(struct rte_eth_dev *dev) for (i = 0; i < dev->data->nb_rx_queues; i++) { struct virtnet_rx *rxvq = dev->data->rx_queues[i]; + if (rxvq == NULL || rxvq->vq == NULL) + continue; + PMD_INIT_LOG(DEBUG, "Before freeing rxq[%d] used and unused buf", i); VIRTQUEUE_DUMP(rxvq->vq); PMD_INIT_LOG(DEBUG, "rx_queues[%d]=%p", i, rxvq); - while ((buf = virtqueue_detatch_unused(rxvq->vq)) != NULL) { + while ((buf = virtqueue_detach_unused(rxvq->vq)) != NULL) { rte_pktmbuf_free(buf); mbuf_num++; } @@ -1616,13 +1613,16 @@ static void virtio_dev_free_mbufs(struct rte_eth_dev *dev) for (i = 0; i < dev->data->nb_tx_queues; i++) { struct virtnet_tx *txvq = dev->data->tx_queues[i]; + if (txvq == NULL || txvq->vq == NULL) + continue; + PMD_INIT_LOG(DEBUG, "Before freeing txq[%d] used and unused bufs", i); VIRTQUEUE_DUMP(txvq->vq); mbuf_num = 0; - while ((buf = virtqueue_detatch_unused(txvq->vq)) != NULL) { + while ((buf = virtqueue_detach_unused(txvq->vq)) != NULL) { rte_pktmbuf_free(buf); mbuf_num++; } diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c index 43fac63b..c2fe9eb5 100644 --- a/drivers/net/virtio/virtio_rxtx.c +++ b/drivers/net/virtio/virtio_rxtx.c @@ -61,6 +61,7 @@ #include "virtio_pci.h" #include "virtqueue.h" #include "virtio_rxtx.h" +#include "virtio_rxtx_simple.h" #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len) @@ -458,6 +459,8 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) vq->vq_ring.desc[desc_idx].flags = VRING_DESC_F_WRITE; } + + virtio_rxq_vec_setup(rxvq); } memset(&rxvq->fake_mbuf, 0, sizeof(rxvq->fake_mbuf)); @@ -467,30 +470,31 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) &rxvq->fake_mbuf; } - while (!virtqueue_full(vq)) { - m = rte_mbuf_raw_alloc(rxvq->mpool); - if (m == NULL) - break; + if (hw->use_simple_rxtx) { + while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { + virtio_rxq_rearm_vec(rxvq); + nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH; + } + } else { + while (!virtqueue_full(vq)) { + m = rte_mbuf_raw_alloc(rxvq->mpool); + if (m == NULL) + break; - /* Enqueue allocated buffers */ - if (hw->use_simple_rxtx) - error = virtqueue_enqueue_recv_refill_simple(vq, m); - else + /* Enqueue allocated buffers */ error = virtqueue_enqueue_recv_refill(vq, m); - - if (error) { - rte_pktmbuf_free(m); - break; + if (error) { + rte_pktmbuf_free(m); + break; + } + nbufs++; } - nbufs++; - } - vq_update_avail_idx(vq); + vq_update_avail_idx(vq); + } PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); - virtio_rxq_vec_setup(rxvq); - VIRTQUEUE_DUMP(vq); return 0; @@ -506,7 +510,7 @@ virtio_update_rxtx_handler(struct rte_eth_dev *dev, #if defined RTE_ARCH_X86 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE3)) use_simple_rxtx = 1; -#elif defined RTE_ARCH_ARM64 || defined CONFIG_RTE_ARCH_ARM +#elif defined RTE_ARCH_ARM64 || defined RTE_ARCH_ARM if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) use_simple_rxtx = 1; #endif diff --git a/drivers/net/virtio/virtio_rxtx.h b/drivers/net/virtio/virtio_rxtx.h index 28f82d6a..e2db823d 100644 --- a/drivers/net/virtio/virtio_rxtx.h +++ b/drivers/net/virtio/virtio_rxtx.h @@ -88,7 +88,4 @@ struct virtnet_ctl { int virtio_rxq_vec_setup(struct virtnet_rx *rxvq); -int virtqueue_enqueue_recv_refill_simple(struct virtqueue *vq, - struct rte_mbuf *m); - #endif /* _VIRTIO_RXTX_H_ */ diff --git a/drivers/net/virtio/virtio_rxtx_simple.c b/drivers/net/virtio/virtio_rxtx_simple.c index a6c0b34e..5285c2b1 100644 --- a/drivers/net/virtio/virtio_rxtx_simple.c +++ b/drivers/net/virtio/virtio_rxtx_simple.c @@ -57,34 +57,6 @@ #pragma GCC diagnostic ignored "-Wcast-qual" #endif -int __attribute__((cold)) -virtqueue_enqueue_recv_refill_simple(struct virtqueue *vq, - struct rte_mbuf *cookie) -{ - struct vq_desc_extra *dxp; - struct vring_desc *start_dp; - uint16_t desc_idx; - - cookie->port = vq->rxq.port_id; - - desc_idx = vq->vq_avail_idx & (vq->vq_nentries - 1); - dxp = &vq->vq_descx[desc_idx]; - dxp->cookie = (void *)cookie; - vq->sw_ring[desc_idx] = cookie; - - start_dp = vq->vq_ring.desc; - start_dp[desc_idx].addr = - VIRTIO_MBUF_ADDR(cookie, vq) + - RTE_PKTMBUF_HEADROOM - vq->hw->vtnet_hdr_size; - start_dp[desc_idx].len = cookie->buf_len - - RTE_PKTMBUF_HEADROOM + vq->hw->vtnet_hdr_size; - - vq->vq_free_cnt--; - vq->vq_avail_idx++; - - return 0; -} - uint16_t virtio_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) @@ -102,7 +74,7 @@ virtio_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, rte_compiler_barrier(); if (nb_used >= VIRTIO_TX_FREE_THRESH) - virtio_xmit_cleanup(vq); + virtio_xmit_cleanup_simple(vq); nb_commit = nb_pkts = RTE_MIN((vq->vq_free_cnt >> 1), nb_pkts); desc_idx = (uint16_t)(vq->vq_avail_idx & desc_idx_max); diff --git a/drivers/net/virtio/virtio_rxtx_simple.h b/drivers/net/virtio/virtio_rxtx_simple.h index b08f8594..bca677c0 100644 --- a/drivers/net/virtio/virtio_rxtx_simple.h +++ b/drivers/net/virtio/virtio_rxtx_simple.h @@ -89,7 +89,7 @@ virtio_rxq_rearm_vec(struct virtnet_rx *rxvq) #define VIRTIO_TX_FREE_NR 32 /* TODO: vq->tx_free_cnt could mean num of free slots so we could avoid shift */ static inline void -virtio_xmit_cleanup(struct virtqueue *vq) +virtio_xmit_cleanup_simple(struct virtqueue *vq) { uint16_t i, desc_idx; uint32_t nb_free = 0; diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c b/drivers/net/virtio/virtio_user/virtio_user_dev.c index 91f6a59a..8bb155d8 100644 --- a/drivers/net/virtio/virtio_user/virtio_user_dev.c +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c @@ -142,6 +142,9 @@ virtio_user_start_device(struct virtio_user_dev *dev) uint64_t features; int ret; + /* Do not check return as already done in init, or reset in stop */ + vhost_user_sock(dev->vhostfd, VHOST_USER_SET_OWNER, NULL); + /* Step 0: tell vhost to create queues */ if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0) goto error; @@ -240,6 +243,7 @@ virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues, PMD_INIT_LOG(ERR, "backend set up fails"); return -1; } + if (vhost_user_sock(dev->vhostfd, VHOST_USER_SET_OWNER, NULL) < 0) { PMD_INIT_LOG(ERR, "set_owner fails: %s", strerror(errno)); return -1; diff --git a/drivers/net/virtio/virtqueue.c b/drivers/net/virtio/virtqueue.c index 4f8707ae..0e9e03c7 100644 --- a/drivers/net/virtio/virtqueue.c +++ b/drivers/net/virtio/virtqueue.c @@ -37,6 +37,7 @@ #include "virtqueue.h" #include "virtio_logs.h" #include "virtio_pci.h" +#include "virtio_rxtx_simple.h" void virtqueue_disable_intr(struct virtqueue *vq) @@ -55,26 +56,50 @@ virtqueue_disable_intr(struct virtqueue *vq) * 2) mbuf that hasn't been consued by backend. */ struct rte_mbuf * -virtqueue_detatch_unused(struct virtqueue *vq) +virtqueue_detach_unused(struct virtqueue *vq) { struct rte_mbuf *cookie; - int idx; + struct virtio_hw *hw; + uint16_t start, end; + int type, idx; - if (vq != NULL) - for (idx = 0; idx < vq->vq_nentries; idx++) { + if (vq == NULL) + return NULL; + + hw = vq->hw; + type = virtio_get_queue_type(hw, vq->vq_queue_index); + start = vq->vq_avail_idx & (vq->vq_nentries - 1); + end = (vq->vq_avail_idx + vq->vq_free_cnt) & (vq->vq_nentries - 1); + + for (idx = 0; idx < vq->vq_nentries; idx++) { + if (hw->use_simple_rxtx && type == VTNET_RQ) { + if (start <= end && idx >= start && idx < end) + continue; + if (start > end && (idx >= start || idx < end)) + continue; + cookie = vq->sw_ring[idx]; + if (cookie != NULL) { + vq->sw_ring[idx] = NULL; + return cookie; + } + } else { cookie = vq->vq_descx[idx].cookie; if (cookie != NULL) { vq->vq_descx[idx].cookie = NULL; return cookie; } } + } + return NULL; } /* Flush the elements in the used ring. */ void -virtqueue_flush(struct virtqueue *vq) +virtqueue_rxvq_flush(struct virtqueue *vq) { + struct virtnet_rx *rxq = &vq->rxq; + struct virtio_hw *hw = vq->hw; struct vring_used_elem *uep; struct vq_desc_extra *dxp; uint16_t used_idx, desc_idx; @@ -85,13 +110,27 @@ virtqueue_flush(struct virtqueue *vq) for (i = 0; i < nb_used; i++) { used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1); uep = &vq->vq_ring.used->ring[used_idx]; - desc_idx = (uint16_t)uep->id; - dxp = &vq->vq_descx[desc_idx]; - if (dxp->cookie != NULL) { - rte_pktmbuf_free(dxp->cookie); - dxp->cookie = NULL; + if (hw->use_simple_rxtx) { + desc_idx = used_idx; + rte_pktmbuf_free(vq->sw_ring[desc_idx]); + vq->vq_free_cnt++; + } else { + desc_idx = (uint16_t)uep->id; + dxp = &vq->vq_descx[desc_idx]; + if (dxp->cookie != NULL) { + rte_pktmbuf_free(dxp->cookie); + dxp->cookie = NULL; + } + vq_ring_free_chain(vq, desc_idx); } vq->vq_used_cons_idx++; - vq_ring_free_chain(vq, desc_idx); + } + + if (hw->use_simple_rxtx) { + while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { + virtio_rxq_rearm_vec(rxq); + if (virtqueue_kick_prepare(vq)) + virtqueue_notify(vq); + } } } diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h index ec967a5b..3748f606 100644 --- a/drivers/net/virtio/virtqueue.h +++ b/drivers/net/virtio/virtqueue.h @@ -288,10 +288,10 @@ void virtqueue_dump(struct virtqueue *vq); /** * Get all mbufs to be freed. */ -struct rte_mbuf *virtqueue_detatch_unused(struct virtqueue *vq); +struct rte_mbuf *virtqueue_detach_unused(struct virtqueue *vq); /* Flush the elements in the used ring. */ -void virtqueue_flush(struct virtqueue *vq); +void virtqueue_rxvq_flush(struct virtqueue *vq); static inline int virtqueue_full(const struct virtqueue *vq) @@ -299,6 +299,17 @@ virtqueue_full(const struct virtqueue *vq) return vq->vq_free_cnt == 0; } +static inline int +virtio_get_queue_type(struct virtio_hw *hw, uint16_t vtpci_queue_idx) +{ + if (vtpci_queue_idx == hw->max_queue_pairs * 2) + return VTNET_CQ; + else if (vtpci_queue_idx % 2 == 0) + return VTNET_RQ; + else + return VTNET_TQ; +} + #define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx)) void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx); |