1 files changed, 75 insertions, 24 deletions

diff --git a/drivers/net/thunderx/nicvf_ethdev.c b/drivers/net/thunderx/nicvf_ethdev.c
index 466e49ce..e4910c9b 100644
--- a/drivers/net/thunderx/nicvf_ethdev.c
+++ b/drivers/net/thunderx/nicvf_ethdev.c
@@ -41,7 +41,6 @@
 #include <inttypes.h>
 #include <netinet/in.h>
 #include <sys/queue.h>
-#include <sys/timerfd.h>
 
 #include <rte_alarm.h>
 #include <rte_atomic.h>
@@ -54,6 +53,7 @@
 #include <rte_eal.h>
 #include <rte_ether.h>
 #include <rte_ethdev.h>
+#include <rte_ethdev_pci.h>
 #include <rte_interrupts.h>
 #include <rte_log.h>
 #include <rte_memory.h>
@@ -145,16 +145,29 @@ nicvf_periodic_alarm_stop(void (fn)(void *), void *arg)
  * Return 0 means link status changed, -1 means not changed
  */
 static int
-nicvf_dev_link_update(struct rte_eth_dev *dev,
-		      int wait_to_complete __rte_unused)
+nicvf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 {
+#define CHECK_INTERVAL 100  /* 100ms */
+#define MAX_CHECK_TIME 90   /* 9s (90 * 100ms) in total */
 	struct rte_eth_link link;
 	struct nicvf *nic = nicvf_pmd_priv(dev);
+	int i;
 
 	PMD_INIT_FUNC_TRACE();
 
-	memset(&link, 0, sizeof(link));
-	nicvf_set_eth_link_status(nic, &link);
+	if (wait_to_complete) {
+		/* rte_eth_link_get() might need to wait up to 9 seconds */
+		for (i = 0; i < MAX_CHECK_TIME; i++) {
+			memset(&link, 0, sizeof(link));
+			nicvf_set_eth_link_status(nic, &link);
+			if (link.link_status)
+				break;
+			rte_delay_ms(CHECK_INTERVAL);
+		}
+	} else {
+		memset(&link, 0, sizeof(link));
+		nicvf_set_eth_link_status(nic, &link);
+	}
 	return nicvf_atomic_write_link_status(dev, &link);
 }
 
@@ -245,7 +258,7 @@ nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 
 	/* Reading per RX ring stats */
 	for (qidx = rx_start; qidx <= rx_end; qidx++) {
-		if (qidx == RTE_ETHDEV_QUEUE_STAT_CNTRS)
+		if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
 			break;
 
 		nicvf_hw_get_rx_qstats(nic, &rx_qstats, qidx);
@@ -258,7 +271,7 @@ nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 
 	/* Reading per TX ring stats */
 	for (qidx = tx_start; qidx <= tx_end; qidx++) {
-		if (qidx == RTE_ETHDEV_QUEUE_STAT_CNTRS)
+		if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
 			break;
 
 		nicvf_hw_get_tx_qstats(nic, &tx_qstats, qidx);
@@ -277,7 +290,7 @@ nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 
 		/* Reading per RX ring stats */
 		for (qidx = rx_start; qidx <= rx_end; qidx++) {
-			if (qidx == RTE_ETHDEV_QUEUE_STAT_CNTRS)
+			if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
 				break;
 
 			nicvf_hw_get_rx_qstats(snic, &rx_qstats,
@@ -290,7 +303,7 @@ nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 		nicvf_tx_range(dev, snic, &tx_start, &tx_end);
 		/* Reading per TX ring stats */
 		for (qidx = tx_start; qidx <= tx_end; qidx++) {
-			if (qidx == RTE_ETHDEV_QUEUE_STAT_CNTRS)
+			if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
 				break;
 
 			nicvf_hw_get_tx_qstats(snic, &tx_qstats,
@@ -1219,6 +1232,23 @@ nicvf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
 	return nicvf_vf_stop_tx_queue(dev, nic, qidx);
 }
 
+static inline void
+nicvf_rxq_mbuf_setup(struct nicvf_rxq *rxq)
+{
+	uintptr_t p;
+	struct rte_mbuf mb_def;
+
+	RTE_BUILD_BUG_ON(sizeof(union mbuf_initializer) != 8);
+	mb_def.nb_segs = 1;
+	mb_def.data_off = RTE_PKTMBUF_HEADROOM;
+	mb_def.port = rxq->port_id;
+	rte_mbuf_refcnt_set(&mb_def, 1);
+
+	/* Prevent compiler reordering: rearm_data covers previous fields */
+	rte_compiler_barrier();
+	p = (uintptr_t)&mb_def.rearm_data;
+	rxq->mbuf_initializer.value = *(uint64_t *)p;
+}
 
 static int
 nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
@@ -1311,6 +1341,7 @@ nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
 	else
 		rxq->rbptr_offset = NICVF_CQE_RBPTR_WORD;
 
+	nicvf_rxq_mbuf_setup(rxq);
 
 	/* Alloc completion queue */
 	if (nicvf_qset_cq_alloc(dev, nic, rxq, rxq->queue_id, nb_desc)) {
@@ -1335,9 +1366,12 @@ static void
 nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 {
 	struct nicvf *nic = nicvf_pmd_priv(dev);
+	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
 
 	PMD_INIT_FUNC_TRACE();
 
+	dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device);
+
 	dev_info->min_rx_bufsize = ETHER_MIN_MTU;
 	dev_info->max_rx_pktlen = NIC_HW_MAX_FRS;
 	dev_info->max_rx_queues =
@@ -1345,7 +1379,7 @@ nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	dev_info->max_tx_queues =
 			(uint16_t)MAX_SND_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1);
 	dev_info->max_mac_addrs = 1;
-	dev_info->max_vfs = dev->pci_dev->max_vfs;
+	dev_info->max_vfs = pci_dev->max_vfs;
 
 	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
 	dev_info->tx_offload_capa =
@@ -1407,7 +1441,7 @@ static int
 nicvf_vf_start(struct rte_eth_dev *dev, struct nicvf *nic, uint32_t rbdrsz)
 {
 	int ret;
-	uint16_t qidx;
+	uint16_t qidx, data_off;
 	uint32_t total_rxq_desc, nb_rbdr_desc, exp_buffs;
 	uint64_t mbuf_phys_off = 0;
 	struct nicvf_rxq *rxq;
@@ -1448,10 +1482,18 @@ nicvf_vf_start(struct rte_eth_dev *dev, struct nicvf *nic, uint32_t rbdrsz)
 				     nic->vf_id, qidx, rxq->pool->name);
 			return -ENOMEM;
 		}
-		rxq->mbuf_phys_off -= nicvf_mbuff_meta_length(mbuf);
-		rxq->mbuf_phys_off -= RTE_PKTMBUF_HEADROOM;
+		data_off = nicvf_mbuff_meta_length(mbuf);
+		data_off += RTE_PKTMBUF_HEADROOM;
 		rte_pktmbuf_free(mbuf);
 
+		if (data_off % RTE_CACHE_LINE_SIZE) {
+			PMD_INIT_LOG(ERR, "%s: unaligned data_off=%d delta=%d",
+				rxq->pool->name, data_off,
+				data_off % RTE_CACHE_LINE_SIZE);
+			return -EINVAL;
+		}
+		rxq->mbuf_phys_off -= data_off;
+
 		if (mbuf_phys_off == 0)
 			mbuf_phys_off = rxq->mbuf_phys_off;
 		if (mbuf_phys_off != rxq->mbuf_phys_off) {
@@ -1975,7 +2017,7 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
 		}
 	}
 
-	pci_dev = eth_dev->pci_dev;
+	pci_dev = RTE_DEV_TO_PCI(eth_dev->device);
 	rte_eth_copy_pci_info(eth_dev, pci_dev);
 
 	nic->device_id = pci_dev->id.device_id;
@@ -2108,16 +2150,25 @@ static const struct rte_pci_id pci_id_nicvf_map[] = {
 	},
 };
 
-static struct eth_driver rte_nicvf_pmd = {
-	.pci_drv = {
-		.id_table = pci_id_nicvf_map,
-		.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
-		.probe = rte_eth_dev_pci_probe,
-		.remove = rte_eth_dev_pci_remove,
-	},
-	.eth_dev_init = nicvf_eth_dev_init,
-	.dev_private_size = sizeof(struct nicvf),
+static int nicvf_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+	struct rte_pci_device *pci_dev)
+{
+	return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct nicvf),
+		nicvf_eth_dev_init);
+}
+
+static int nicvf_eth_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return rte_eth_dev_pci_generic_remove(pci_dev, NULL);
+}
+
+static struct rte_pci_driver rte_nicvf_pmd = {
+	.id_table = pci_id_nicvf_map,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
+	.probe = nicvf_eth_pci_probe,
+	.remove = nicvf_eth_pci_remove,
 };
 
-RTE_PMD_REGISTER_PCI(net_thunderx, rte_nicvf_pmd.pci_drv);
+RTE_PMD_REGISTER_PCI(net_thunderx, rte_nicvf_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_thunderx, pci_id_nicvf_map);
+RTE_PMD_REGISTER_KMOD_DEP(net_thunderx, "* igb_uio | uio_pci_generic | vfio");