From 23650e6cada50246ba211cec168ddd6738a26f67 Mon Sep 17 00:00:00 2001 From: John Lo Date: Tue, 29 Mar 2016 16:14:35 -0400 Subject: Update ENIC driver patches for DPDK 2.2.0 This includes all patches in ENIC driver which are up-streamed to DPDK to improve RX performance, fix buffer/error handling and interoperation with link bonding PMD library. Change-Id: Id4c71a350d5234834951f9261c69db5476ba396b Signed-off-by: John Lo --- ...nding-PMD-slave-status-polling-path-used-.patch | 34 - .../0011-enic-improve-Rx-performance.patch | 1336 +++++++++++++++++++ ...D-problem-with-not-sending-the-first-pack.patch | 31 - .../0012-enic-fix-last-packet-not-being-sent.patch | 34 + ...-of-ENIC-PMD-receive-path-to-improve-perf.patch | 1347 -------------------- ...c-add-missing-newline-to-print-statements.patch | 42 + ...fix-crash-when-allocating-too-many-queues.patch | 46 + .../0016-enic-fix-mbuf-flags-on-Rx.patch | 38 + .../0017-enic-fix-error-packets-handling.patch | 112 ++ ...0018-enic-remove-packet-error-conditional.patch | 53 + .../0019-enic-update-maintainers.patch | 36 + .../0020-enic-fix-Rx-descriptor-limit.patch | 61 + ...hang-when-number-of-packets-gt-queue-size.patch | 82 ++ ...ix-bond-link-detect-in-non-interrupt-mode.patch | 71 ++ 14 files changed, 1911 insertions(+), 1412 deletions(-) delete mode 100644 dpdk/dpdk-2.2.0_patches/0011-Fix-link-bonding-PMD-slave-status-polling-path-used-.patch create mode 100644 dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch delete mode 100644 dpdk/dpdk-2.2.0_patches/0012-Fix-ENIC-PMD-problem-with-not-sending-the-first-pack.patch create mode 100644 dpdk/dpdk-2.2.0_patches/0012-enic-fix-last-packet-not-being-sent.patch delete mode 100644 dpdk/dpdk-2.2.0_patches/0013-Replacement-of-ENIC-PMD-receive-path-to-improve-perf.patch create mode 100644 dpdk/dpdk-2.2.0_patches/0013-enic-add-missing-newline-to-print-statements.patch create mode 100644 dpdk/dpdk-2.2.0_patches/0015-enic-fix-crash-when-allocating-too-many-queues.patch create mode 100644 dpdk/dpdk-2.2.0_patches/0016-enic-fix-mbuf-flags-on-Rx.patch create mode 100644 dpdk/dpdk-2.2.0_patches/0017-enic-fix-error-packets-handling.patch create mode 100644 dpdk/dpdk-2.2.0_patches/0018-enic-remove-packet-error-conditional.patch create mode 100644 dpdk/dpdk-2.2.0_patches/0019-enic-update-maintainers.patch create mode 100644 dpdk/dpdk-2.2.0_patches/0020-enic-fix-Rx-descriptor-limit.patch create mode 100644 dpdk/dpdk-2.2.0_patches/0021-enic-fix-TX-hang-when-number-of-packets-gt-queue-size.patch create mode 100644 dpdk/dpdk-2.2.0_patches/0022-bonding-fix-bond-link-detect-in-non-interrupt-mode.patch diff --git a/dpdk/dpdk-2.2.0_patches/0011-Fix-link-bonding-PMD-slave-status-polling-path-used-.patch b/dpdk/dpdk-2.2.0_patches/0011-Fix-link-bonding-PMD-slave-status-polling-path-used-.patch deleted file mode 100644 index 210ee458..00000000 --- a/dpdk/dpdk-2.2.0_patches/0011-Fix-link-bonding-PMD-slave-status-polling-path-used-.patch +++ /dev/null @@ -1,34 +0,0 @@ -From 39272e0a1c227b1ab9360e6fbcbc497eaaed4cb0 Mon Sep 17 00:00:00 2001 -From: John Lo -Date: Fri, 26 Feb 2016 12:45:55 -0500 -Subject: [PATCH 1/2] Fix link bonding PMD slave status polling path used by - ENIC driver, - ---- - drivers/net/bonding/rte_eth_bond_pmd.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/drivers/net/bonding/rte_eth_bond_pmd.c b/drivers/net/bonding/rte_eth_bond_pmd.c -index b1373c6..9efd73c 100644 ---- a/drivers/net/bonding/rte_eth_bond_pmd.c -+++ b/drivers/net/bonding/rte_eth_bond_pmd.c -@@ -1734,7 +1734,7 @@ bond_ethdev_slave_link_status_change_monitor(void *cb_arg) - - if (!bonded_ethdev->data->dev_started || - !internals->link_status_polling_enabled) -- return; -+ goto rearm_and_exit; - - /* If device is currently being configured then don't check slaves link - * status, wait until next period */ -@@ -1768,6 +1768,7 @@ bond_ethdev_slave_link_status_change_monitor(void *cb_arg) - rte_spinlock_unlock(&internals->lock); - } - -+ rearm_and_exit: - if (polling_slave_found) - /* Set alarm to continue monitoring link status of slave ethdev's */ - rte_eal_alarm_set(internals->link_status_polling_interval_ms * 1000, --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch b/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch new file mode 100644 index 00000000..98824c57 --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch @@ -0,0 +1,1336 @@ +commit 947d860c821f4248dcf2fc01e98671524973eeea +Author: John Daley +Date: Fri Mar 4 13:09:00 2016 -0800 + + enic: improve Rx performance + + This is a wholesale replacement of the Enic PMD receive path in order + to improve performance and code clarity. The changes are: + - Simplify and reduce code path length of receive function. + - Put most of the fast-path receive functions in one file. + - Reduce the number of posted_index updates (pay attention to + rx_free_thresh) + - Remove the unneeded container structure around the RQ mbuf ring + - Prefetch next Mbuf and descriptors while processing the current one + - Use a lookup table for converting CQ flags to mbuf flags. + + Signed-off-by: John Daley + +diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile +index f0ee093..f316274 100644 +--- a/drivers/net/enic/Makefile ++++ b/drivers/net/enic/Makefile +@@ -53,6 +53,7 @@ VPATH += $(SRCDIR)/src + # + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c ++SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c +diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c +index 1441604..cb62c5e 100644 +--- a/drivers/net/enic/base/vnic_rq.c ++++ b/drivers/net/enic/base/vnic_rq.c +@@ -35,77 +35,21 @@ + #include "vnic_dev.h" + #include "vnic_rq.h" + +-static int vnic_rq_alloc_bufs(struct vnic_rq *rq) +-{ +- struct vnic_rq_buf *buf; +- unsigned int i, j, count = rq->ring.desc_count; +- unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count); +- +- for (i = 0; i < blks; i++) { +- rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ(count), GFP_ATOMIC); +- if (!rq->bufs[i]) +- return -ENOMEM; +- } +- +- for (i = 0; i < blks; i++) { +- buf = rq->bufs[i]; +- for (j = 0; j < VNIC_RQ_BUF_BLK_ENTRIES(count); j++) { +- buf->index = i * VNIC_RQ_BUF_BLK_ENTRIES(count) + j; +- buf->desc = (u8 *)rq->ring.descs + +- rq->ring.desc_size * buf->index; +- if (buf->index + 1 == count) { +- buf->next = rq->bufs[0]; +- break; +- } else if (j + 1 == VNIC_RQ_BUF_BLK_ENTRIES(count)) { +- buf->next = rq->bufs[i + 1]; +- } else { +- buf->next = buf + 1; +- buf++; +- } +- } +- } +- +- rq->to_use = rq->to_clean = rq->bufs[0]; +- +- return 0; +-} +- +-int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count, +- unsigned int desc_size) +-{ +- int mem_size = 0; +- +- mem_size += vnic_dev_desc_ring_size(&rq->ring, desc_count, desc_size); +- +- mem_size += VNIC_RQ_BUF_BLKS_NEEDED(rq->ring.desc_count) * +- VNIC_RQ_BUF_BLK_SZ(rq->ring.desc_count); +- +- return mem_size; +-} +- + void vnic_rq_free(struct vnic_rq *rq) + { + struct vnic_dev *vdev; +- unsigned int i; + + vdev = rq->vdev; + + vnic_dev_free_desc_ring(vdev, &rq->ring); + +- for (i = 0; i < VNIC_RQ_BUF_BLKS_MAX; i++) { +- if (rq->bufs[i]) { +- kfree(rq->bufs[i]); +- rq->bufs[i] = NULL; +- } +- } +- + rq->ctrl = NULL; + } + + int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, + unsigned int desc_count, unsigned int desc_size) + { +- int err; ++ int rc; + char res_name[NAME_MAX]; + static int instance; + +@@ -121,18 +65,9 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, + vnic_rq_disable(rq); + + snprintf(res_name, sizeof(res_name), "%d-rq-%d", instance++, index); +- err = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, ++ rc = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, + rq->socket_id, res_name); +- if (err) +- return err; +- +- err = vnic_rq_alloc_bufs(rq); +- if (err) { +- vnic_rq_free(rq); +- return err; +- } +- +- return 0; ++ return rc; + } + + void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, +@@ -154,9 +89,6 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, + iowrite32(fetch_index, &rq->ctrl->fetch_index); + iowrite32(posted_index, &rq->ctrl->posted_index); + +- rq->to_use = rq->to_clean = +- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] +- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; + } + + void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, +@@ -176,6 +108,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, + fetch_index, fetch_index, + error_interrupt_enable, + error_interrupt_offset); ++ rq->rxst_idx = 0; ++ rq->tot_pkts = 0; + } + + void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error) +@@ -212,21 +146,20 @@ int vnic_rq_disable(struct vnic_rq *rq) + } + + void vnic_rq_clean(struct vnic_rq *rq, +- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)) ++ void (*buf_clean)(struct rte_mbuf **buf)) + { +- struct vnic_rq_buf *buf; +- u32 fetch_index; ++ struct rte_mbuf **buf; ++ u32 fetch_index, i; + unsigned int count = rq->ring.desc_count; + +- buf = rq->to_clean; +- +- while (vnic_rq_desc_used(rq) > 0) { ++ buf = &rq->mbuf_ring[0]; + +- (*buf_clean)(rq, buf); +- +- buf = rq->to_clean = buf->next; +- rq->ring.desc_avail++; ++ for (i = 0; i < count; i++) { ++ (*buf_clean)(buf); ++ buf++; + } ++ rq->ring.desc_avail = count - 1; ++ rq->rx_nb_hold = 0; + + /* Use current fetch_index as the ring starting point */ + fetch_index = ioread32(&rq->ctrl->fetch_index); +@@ -235,9 +168,7 @@ void vnic_rq_clean(struct vnic_rq *rq, + /* Hardware surprise removal: reset fetch_index */ + fetch_index = 0; + } +- rq->to_use = rq->to_clean = +- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] +- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; ++ + iowrite32(fetch_index, &rq->ctrl->posted_index); + + vnic_dev_clear_desc_ring(&rq->ring); +diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h +index 0f5c3c1..e083ccc 100644 +--- a/drivers/net/enic/base/vnic_rq.h ++++ b/drivers/net/enic/base/vnic_rq.h +@@ -66,42 +66,22 @@ struct vnic_rq_ctrl { + u32 pad10; + }; + +-/* Break the vnic_rq_buf allocations into blocks of 32/64 entries */ +-#define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32 +-#define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64 +-#define VNIC_RQ_BUF_BLK_ENTRIES(entries) \ +- ((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \ +- VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES)) +-#define VNIC_RQ_BUF_BLK_SZ(entries) \ +- (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf)) +-#define VNIC_RQ_BUF_BLKS_NEEDED(entries) \ +- DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries)) +-#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096) +- +-struct vnic_rq_buf { +- struct vnic_rq_buf *next; +- dma_addr_t dma_addr; +- void *os_buf; +- unsigned int os_buf_index; +- unsigned int len; +- unsigned int index; +- void *desc; +- uint64_t wr_id; +-}; +- + struct vnic_rq { + unsigned int index; ++ unsigned int posted_index; + struct vnic_dev *vdev; +- struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ ++ struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ + struct vnic_dev_ring ring; +- struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX]; +- struct vnic_rq_buf *to_use; +- struct vnic_rq_buf *to_clean; ++ struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */ ++ unsigned int mbuf_next_idx; /* next mb to consume */ + void *os_buf_head; + unsigned int pkts_outstanding; +- ++ uint16_t rx_nb_hold; ++ uint16_t rx_free_thresh; + unsigned int socket_id; + struct rte_mempool *mp; ++ uint16_t rxst_idx; ++ uint32_t tot_pkts; + }; + + static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq) +@@ -116,119 +96,13 @@ static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq) + return rq->ring.desc_count - rq->ring.desc_avail - 1; + } + +-static inline void *vnic_rq_next_desc(struct vnic_rq *rq) +-{ +- return rq->to_use->desc; +-} +- +-static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq) +-{ +- return rq->to_use->index; +-} +- +-static inline void vnic_rq_post(struct vnic_rq *rq, +- void *os_buf, unsigned int os_buf_index, +- dma_addr_t dma_addr, unsigned int len, +- uint64_t wrid) +-{ +- struct vnic_rq_buf *buf = rq->to_use; +- +- buf->os_buf = os_buf; +- buf->os_buf_index = os_buf_index; +- buf->dma_addr = dma_addr; +- buf->len = len; +- buf->wr_id = wrid; +- +- buf = buf->next; +- rq->to_use = buf; +- rq->ring.desc_avail--; +- +- /* Move the posted_index every nth descriptor +- */ +- +-#ifndef VNIC_RQ_RETURN_RATE +-#define VNIC_RQ_RETURN_RATE 0xf /* keep 2^n - 1 */ +-#endif +- +- if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) { +- /* Adding write memory barrier prevents compiler and/or CPU +- * reordering, thus avoiding descriptor posting before +- * descriptor is initialized. Otherwise, hardware can read +- * stale descriptor fields. +- */ +- wmb(); +- iowrite32(buf->index, &rq->ctrl->posted_index); +- } +-} +- +-static inline void vnic_rq_post_commit(struct vnic_rq *rq, +- void *os_buf, unsigned int os_buf_index, +- dma_addr_t dma_addr, unsigned int len) +-{ +- struct vnic_rq_buf *buf = rq->to_use; +- +- buf->os_buf = os_buf; +- buf->os_buf_index = os_buf_index; +- buf->dma_addr = dma_addr; +- buf->len = len; +- +- buf = buf->next; +- rq->to_use = buf; +- rq->ring.desc_avail--; +- +- /* Move the posted_index every descriptor +- */ +- +- /* Adding write memory barrier prevents compiler and/or CPU +- * reordering, thus avoiding descriptor posting before +- * descriptor is initialized. Otherwise, hardware can read +- * stale descriptor fields. +- */ +- wmb(); +- iowrite32(buf->index, &rq->ctrl->posted_index); +-} + +-static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count) +-{ +- rq->ring.desc_avail += count; +-} + + enum desc_return_options { + VNIC_RQ_RETURN_DESC, + VNIC_RQ_DEFER_RETURN_DESC, + }; + +-static inline int vnic_rq_service(struct vnic_rq *rq, +- struct cq_desc *cq_desc, u16 completed_index, +- int desc_return, int (*buf_service)(struct vnic_rq *rq, +- struct cq_desc *cq_desc, struct vnic_rq_buf *buf, +- int skipped, void *opaque), void *opaque) +-{ +- struct vnic_rq_buf *buf; +- int skipped; +- int eop = 0; +- +- buf = rq->to_clean; +- while (1) { +- +- skipped = (buf->index != completed_index); +- +- if ((*buf_service)(rq, cq_desc, buf, skipped, opaque)) +- eop++; +- +- if (desc_return == VNIC_RQ_RETURN_DESC) +- rq->ring.desc_avail++; +- +- rq->to_clean = buf->next; +- +- if (!skipped) +- break; +- +- buf = rq->to_clean; +- } +- return eop; +-} +- + static inline int vnic_rq_fill(struct vnic_rq *rq, + int (*buf_fill)(struct vnic_rq *rq)) + { +@@ -274,8 +148,5 @@ unsigned int vnic_rq_error_status(struct vnic_rq *rq); + void vnic_rq_enable(struct vnic_rq *rq); + int vnic_rq_disable(struct vnic_rq *rq); + void vnic_rq_clean(struct vnic_rq *rq, +- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)); +-int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count, +- unsigned int desc_size); +- ++ void (*buf_clean)(struct rte_mbuf **buf)); + #endif /* _VNIC_RQ_H_ */ +diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h +index 9e78305..8c914f5 100644 +--- a/drivers/net/enic/enic.h ++++ b/drivers/net/enic/enic.h +@@ -45,6 +45,7 @@ + #include "vnic_nic.h" + #include "vnic_rss.h" + #include "enic_res.h" ++#include "cq_enet_desc.h" + + #define DRV_NAME "enic_pmd" + #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Poll-mode Driver" +@@ -154,6 +155,16 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) + return (struct enic *)eth_dev->data->dev_private; + } + ++#define RTE_LIBRTE_ENIC_ASSERT_ENABLE ++#ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE ++#define ASSERT(x) do { \ ++ if (!(x)) \ ++ rte_panic("ENIC: x"); \ ++} while (0) ++#else ++#define ASSERT(x) ++#endif ++ + extern void enic_fdir_stats_get(struct enic *enic, + struct rte_eth_fdir_stats *stats); + extern int enic_fdir_add_fltr(struct enic *enic, +@@ -193,9 +204,10 @@ extern void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, + uint16_t ol_flags, uint16_t vlan_tag); + + extern void enic_post_wq_index(struct vnic_wq *wq); +-extern int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts, +- unsigned int budget, unsigned int *work_done); + extern int enic_probe(struct enic *enic); + extern int enic_clsf_init(struct enic *enic); + extern void enic_clsf_destroy(struct enic *enic); ++uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, ++ uint16_t nb_pkts); ++ + #endif /* _ENIC_H_ */ +diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c +index 2a88043..6f2ada5 100644 +--- a/drivers/net/enic/enic_ethdev.c ++++ b/drivers/net/enic/enic_ethdev.c +@@ -255,7 +255,7 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, + uint16_t queue_idx, + uint16_t nb_desc, + unsigned int socket_id, +- __rte_unused const struct rte_eth_rxconf *rx_conf, ++ const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mp) + { + int ret; +@@ -270,6 +270,10 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, + return ret; + } + ++ enic->rq[queue_idx].rx_free_thresh = rx_conf->rx_free_thresh; ++ dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx, ++ enic->rq[queue_idx].rx_free_thresh); ++ + return enicpmd_dev_setup_intr(enic); + } + +@@ -429,6 +433,9 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev, + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM; ++ device_info->default_rxconf = (struct rte_eth_rxconf) { ++ .rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH ++ }; + } + + static void enicpmd_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) +@@ -538,18 +545,6 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + return index; + } + +-static uint16_t enicpmd_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, +- uint16_t nb_pkts) +-{ +- struct vnic_rq *rq = (struct vnic_rq *)rx_queue; +- unsigned int work_done; +- +- if (enic_poll(rq, rx_pkts, (unsigned int)nb_pkts, &work_done)) +- dev_err(enic, "error in enicpmd poll\n"); +- +- return work_done; +-} +- + static const struct eth_dev_ops enicpmd_eth_dev_ops = { + .dev_configure = enicpmd_dev_configure, + .dev_start = enicpmd_dev_start, +@@ -606,7 +601,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) + enic->port_id = eth_dev->data->port_id; + enic->rte_dev = eth_dev; + eth_dev->dev_ops = &enicpmd_eth_dev_ops; +- eth_dev->rx_pkt_burst = &enicpmd_recv_pkts; ++ eth_dev->rx_pkt_burst = &enic_recv_pkts; + eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts; + + pdev = eth_dev->pci_dev; +@@ -635,8 +630,8 @@ static struct eth_driver rte_enic_pmd = { + * Register as the [Poll Mode] Driver of Cisco ENIC device. + */ + static int +-rte_enic_pmd_init(const char *name __rte_unused, +- const char *params __rte_unused) ++rte_enic_pmd_init(__rte_unused const char *name, ++ __rte_unused const char *params) + { + ENICPMD_FUNC_TRACE(); + +diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c +index f818c32..9fff020 100644 +--- a/drivers/net/enic/enic_main.c ++++ b/drivers/net/enic/enic_main.c +@@ -60,6 +60,17 @@ + #include "vnic_nic.h" + #include "enic_vnic_wq.h" + ++static inline struct rte_mbuf * ++rte_rxmbuf_alloc(struct rte_mempool *mp) ++{ ++ struct rte_mbuf *m; ++ ++ m = __rte_mbuf_raw_alloc(mp); ++ __rte_mbuf_sanity_check_raw(m, 0); ++ return m; ++} ++ ++ + static inline int enic_is_sriov_vf(struct enic *enic) + { + return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF; +@@ -80,16 +91,25 @@ static int is_eth_addr_valid(uint8_t *addr) + return !is_mcast_addr(addr) && !is_zero_addr(addr); + } + +-static inline struct rte_mbuf * +-enic_rxmbuf_alloc(struct rte_mempool *mp) ++static void ++enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) + { +- struct rte_mbuf *m; ++ uint16_t i; + +- m = __rte_mbuf_raw_alloc(mp); +- __rte_mbuf_sanity_check_raw(m, 0); +- return m; ++ if (!rq || !rq->mbuf_ring) { ++ dev_debug(enic, "Pointer to rq or mbuf_ring is NULL"); ++ return; ++ } ++ ++ for (i = 0; i < enic->config.rq_desc_count; i++) { ++ if (rq->mbuf_ring[i]) { ++ rte_pktmbuf_free_seg(rq->mbuf_ring[i]); ++ rq->mbuf_ring[i] = NULL; ++ } ++ } + } + ++ + void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) + { + vnic_set_hdr_split_size(enic->vdev, split_hdr_size); +@@ -262,13 +282,13 @@ void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr) + } + + static void +-enic_free_rq_buf(__rte_unused struct vnic_rq *rq, struct vnic_rq_buf *buf) ++enic_free_rq_buf(struct rte_mbuf **mbuf) + { +- if (!buf->os_buf) ++ if (*mbuf == NULL) + return; + +- rte_pktmbuf_free((struct rte_mbuf *)buf->os_buf); +- buf->os_buf = NULL; ++ rte_pktmbuf_free(*mbuf); ++ mbuf = NULL; + } + + void enic_init_vnic_resources(struct enic *enic) +@@ -314,221 +334,47 @@ void enic_init_vnic_resources(struct enic *enic) + } + + +-static int enic_rq_alloc_buf(struct vnic_rq *rq) ++static int ++enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) + { +- struct enic *enic = vnic_dev_priv(rq->vdev); ++ struct rte_mbuf *mb; ++ struct rq_enet_desc *rqd = rq->ring.descs; ++ unsigned i; + dma_addr_t dma_addr; +- struct rq_enet_desc *desc = vnic_rq_next_desc(rq); +- uint8_t type = RQ_ENET_TYPE_ONLY_SOP; +- u16 split_hdr_size = vnic_get_hdr_split_size(enic->vdev); +- struct rte_mbuf *mbuf = enic_rxmbuf_alloc(rq->mp); +- struct rte_mbuf *hdr_mbuf = NULL; +- +- if (!mbuf) { +- dev_err(enic, "mbuf alloc in enic_rq_alloc_buf failed\n"); +- return -1; +- } +- +- if (unlikely(split_hdr_size)) { +- if (vnic_rq_desc_avail(rq) < 2) { +- rte_mempool_put(mbuf->pool, mbuf); +- return -1; +- } +- hdr_mbuf = enic_rxmbuf_alloc(rq->mp); +- if (!hdr_mbuf) { +- rte_mempool_put(mbuf->pool, mbuf); +- dev_err(enic, +- "hdr_mbuf alloc in enic_rq_alloc_buf failed\n"); +- return -1; +- } +- +- hdr_mbuf->data_off = RTE_PKTMBUF_HEADROOM; +- +- hdr_mbuf->nb_segs = 2; +- hdr_mbuf->port = enic->port_id; +- hdr_mbuf->next = mbuf; +- +- dma_addr = (dma_addr_t) +- (hdr_mbuf->buf_physaddr + hdr_mbuf->data_off); +- +- rq_enet_desc_enc(desc, dma_addr, type, split_hdr_size); + +- vnic_rq_post(rq, (void *)hdr_mbuf, 0 /*os_buf_index*/, dma_addr, +- (unsigned int)split_hdr_size, 0 /*wrid*/); ++ dev_debug(enic, "queue %u, allocating %u rx queue mbufs", rq->index, ++ rq->ring.desc_count); + +- desc = vnic_rq_next_desc(rq); +- type = RQ_ENET_TYPE_NOT_SOP; +- } else { +- mbuf->nb_segs = 1; +- mbuf->port = enic->port_id; +- } +- +- mbuf->data_off = RTE_PKTMBUF_HEADROOM; +- mbuf->next = NULL; +- +- dma_addr = (dma_addr_t) +- (mbuf->buf_physaddr + mbuf->data_off); +- +- rq_enet_desc_enc(desc, dma_addr, type, mbuf->buf_len); +- +- vnic_rq_post(rq, (void *)mbuf, 0 /*os_buf_index*/, dma_addr, +- (unsigned int)mbuf->buf_len, 0 /*wrid*/); +- +- return 0; +-} +- +-static int enic_rq_indicate_buf(struct vnic_rq *rq, +- struct cq_desc *cq_desc, struct vnic_rq_buf *buf, +- int skipped, void *opaque) +-{ +- struct enic *enic = vnic_dev_priv(rq->vdev); +- struct rte_mbuf **rx_pkt_bucket = (struct rte_mbuf **)opaque; +- struct rte_mbuf *rx_pkt = NULL; +- struct rte_mbuf *hdr_rx_pkt = NULL; +- +- u8 type, color, eop, sop, ingress_port, vlan_stripped; +- u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof; +- u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok; +- u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc; +- u8 packet_error; +- u16 q_number, completed_index, bytes_written, vlan_tci, checksum; +- u32 rss_hash; +- +- cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, +- &type, &color, &q_number, &completed_index, +- &ingress_port, &fcoe, &eop, &sop, &rss_type, +- &csum_not_calc, &rss_hash, &bytes_written, +- &packet_error, &vlan_stripped, &vlan_tci, &checksum, +- &fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error, +- &fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp, +- &ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment, +- &fcs_ok); +- +- rx_pkt = (struct rte_mbuf *)buf->os_buf; +- buf->os_buf = NULL; +- +- if (unlikely(packet_error)) { +- dev_err(enic, "packet error\n"); +- rx_pkt->data_len = 0; +- return 0; +- } +- +- if (unlikely(skipped)) { +- rx_pkt->data_len = 0; +- return 0; +- } +- +- if (likely(!vnic_get_hdr_split_size(enic->vdev))) { +- /* No header split configured */ +- *rx_pkt_bucket = rx_pkt; +- rx_pkt->pkt_len = bytes_written; +- +- if (ipv4) { +- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4; +- if (!csum_not_calc) { +- if (unlikely(!ipv4_csum_ok)) +- rx_pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD; +- +- if ((tcp || udp) && (!tcp_udp_csum_ok)) +- rx_pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD; +- } +- } else if (ipv6) +- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6; +- } else { +- /* Header split */ +- if (sop && !eop) { +- /* This piece is header */ +- *rx_pkt_bucket = rx_pkt; +- rx_pkt->pkt_len = bytes_written; +- } else { +- if (sop && eop) { +- /* The packet is smaller than split_hdr_size */ +- *rx_pkt_bucket = rx_pkt; +- rx_pkt->pkt_len = bytes_written; +- if (ipv4) { +- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4; +- if (!csum_not_calc) { +- if (unlikely(!ipv4_csum_ok)) +- rx_pkt->ol_flags |= +- PKT_RX_IP_CKSUM_BAD; +- +- if ((tcp || udp) && +- (!tcp_udp_csum_ok)) +- rx_pkt->ol_flags |= +- PKT_RX_L4_CKSUM_BAD; +- } +- } else if (ipv6) +- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6; +- } else { +- /* Payload */ +- hdr_rx_pkt = *rx_pkt_bucket; +- hdr_rx_pkt->pkt_len += bytes_written; +- if (ipv4) { +- hdr_rx_pkt->packet_type = +- RTE_PTYPE_L3_IPV4; +- if (!csum_not_calc) { +- if (unlikely(!ipv4_csum_ok)) +- hdr_rx_pkt->ol_flags |= +- PKT_RX_IP_CKSUM_BAD; +- +- if ((tcp || udp) && +- (!tcp_udp_csum_ok)) +- hdr_rx_pkt->ol_flags |= +- PKT_RX_L4_CKSUM_BAD; +- } +- } else if (ipv6) +- hdr_rx_pkt->packet_type = +- RTE_PTYPE_L3_IPV6; +- } ++ for (i = 0; i < rq->ring.desc_count; i++, rqd++) { ++ mb = rte_rxmbuf_alloc(rq->mp); ++ if (mb == NULL) { ++ dev_err(enic, "RX mbuf alloc failed queue_id=%u", ++ (unsigned)rq->index); ++ return -ENOMEM; + } +- } + +- rx_pkt->data_len = bytes_written; ++ dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off); + +- if (rss_hash) { +- rx_pkt->ol_flags |= PKT_RX_RSS_HASH; +- rx_pkt->hash.rss = rss_hash; ++ rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, ++ mb->buf_len); ++ rq->mbuf_ring[i] = mb; + } + +- if (vlan_tci) { +- rx_pkt->ol_flags |= PKT_RX_VLAN_PKT; +- rx_pkt->vlan_tci = vlan_tci; +- } ++ /* make sure all prior writes are complete before doing the PIO write */ ++ rte_rmb(); + +- return eop; +-} ++ /* Post all but the last 2 cache lines' worth of descriptors */ ++ rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE ++ / sizeof(struct rq_enet_desc)); ++ rq->rx_nb_hold = 0; + +-static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, +- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque) +-{ +- struct enic *enic = vnic_dev_priv(vdev); +- +- return vnic_rq_service(&enic->rq[q_number], cq_desc, +- completed_index, VNIC_RQ_RETURN_DESC, +- enic_rq_indicate_buf, opaque); +- +-} ++ dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n", ++ enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold); ++ iowrite32(rq->posted_index, &rq->ctrl->posted_index); ++ rte_rmb(); + +-int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts, +- unsigned int budget, unsigned int *work_done) +-{ +- struct enic *enic = vnic_dev_priv(rq->vdev); +- unsigned int cq = enic_cq_rq(enic, rq->index); +- int err = 0; +- +- *work_done = vnic_cq_service(&enic->cq[cq], +- budget, enic_rq_service, (void *)rx_pkts); +- +- if (*work_done) { +- vnic_rq_fill(rq, enic_rq_alloc_buf); ++ return 0; + +- /* Need at least one buffer on ring to get going */ +- if (vnic_rq_desc_used(rq) == 0) { +- dev_err(enic, "Unable to alloc receive buffers\n"); +- err = -1; +- } +- } +- return err; + } + + static void * +@@ -576,6 +422,7 @@ enic_intr_handler(__rte_unused struct rte_intr_handle *handle, + int enic_enable(struct enic *enic) + { + unsigned int index; ++ int err; + struct rte_eth_dev *eth_dev = enic->rte_dev; + + eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev); +@@ -586,15 +433,11 @@ int enic_enable(struct enic *enic) + dev_warning(enic, "Init of hash table for clsf failed."\ + "Flow director feature will not work\n"); + +- /* Fill RQ bufs */ + for (index = 0; index < enic->rq_count; index++) { +- vnic_rq_fill(&enic->rq[index], enic_rq_alloc_buf); +- +- /* Need at least one buffer on ring to get going +- */ +- if (vnic_rq_desc_used(&enic->rq[index]) == 0) { +- dev_err(enic, "Unable to alloc receive buffers\n"); +- return -1; ++ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]); ++ if (err) { ++ dev_err(enic, "Failed to alloc RX queue mbufs\n"); ++ return err; + } + } + +@@ -636,6 +479,9 @@ void enic_free_rq(void *rxq) + struct vnic_rq *rq = (struct vnic_rq *)rxq; + struct enic *enic = vnic_dev_priv(rq->vdev); + ++ enic_rxmbuf_queue_release(enic, rq); ++ rte_free(rq->mbuf_ring); ++ rq->mbuf_ring = NULL; + vnic_rq_free(rq); + vnic_cq_free(&enic->cq[rq->index]); + } +@@ -664,7 +510,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, + unsigned int socket_id, struct rte_mempool *mp, + uint16_t nb_desc) + { +- int err; ++ int rc; + struct vnic_rq *rq = &enic->rq[queue_idx]; + + rq->socket_id = socket_id; +@@ -687,23 +533,35 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, + } + + /* Allocate queue resources */ +- err = vnic_rq_alloc(enic->vdev, &enic->rq[queue_idx], queue_idx, +- enic->config.rq_desc_count, +- sizeof(struct rq_enet_desc)); +- if (err) { ++ rc = vnic_rq_alloc(enic->vdev, rq, queue_idx, ++ enic->config.rq_desc_count, sizeof(struct rq_enet_desc)); ++ if (rc) { + dev_err(enic, "error in allocation of rq\n"); +- return err; ++ goto err_exit; + } + +- err = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, ++ rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, + socket_id, enic->config.rq_desc_count, + sizeof(struct cq_enet_rq_desc)); +- if (err) { +- vnic_rq_free(rq); ++ if (rc) { + dev_err(enic, "error in allocation of cq for rq\n"); ++ goto err_free_rq_exit; + } + +- return err; ++ /* Allocate the mbuf ring */ ++ rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", ++ sizeof(struct rte_mbuf *) * enic->config.rq_desc_count, ++ RTE_CACHE_LINE_SIZE, rq->socket_id); ++ ++ if (rq->mbuf_ring != NULL) ++ return 0; ++ ++ /* cleanup on error */ ++ vnic_cq_free(&enic->cq[queue_idx]); ++err_free_rq_exit: ++ vnic_rq_free(rq); ++err_exit: ++ return -ENOMEM; + } + + void enic_free_wq(void *txq) +@@ -790,6 +648,7 @@ int enic_disable(struct enic *enic) + + for (i = 0; i < enic->wq_count; i++) + vnic_wq_clean(&enic->wq[i], enic_free_wq_buf); ++ + for (i = 0; i < enic->rq_count; i++) + vnic_rq_clean(&enic->rq[i], enic_free_rq_buf); + for (i = 0; i < enic->cq_count; i++) +@@ -1074,7 +933,7 @@ int enic_probe(struct enic *enic) + + /* Set ingress vlan rewrite mode before vnic initialization */ + err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, +- IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN); ++ IG_VLAN_REWRITE_MODE_PASS_THRU); + if (err) { + dev_err(enic, + "Failed to set ingress vlan rewrite mode, aborting.\n"); +diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h +index 49f7e22..33f2e84 100644 +--- a/drivers/net/enic/enic_res.h ++++ b/drivers/net/enic/enic_res.h +@@ -52,6 +52,7 @@ + #define ENIC_UNICAST_PERFECT_FILTERS 32 + + #define ENIC_NON_TSO_MAX_DESC 16 ++#define ENIC_DEFAULT_RX_FREE_THRESH 32 + + #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) + +@@ -133,21 +134,6 @@ static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq, + WQ_ENET_OFFLOAD_MODE_TSO, + eop, 1 /* SOP */, eop, loopback); + } +-static inline void enic_queue_rq_desc(struct vnic_rq *rq, +- void *os_buf, unsigned int os_buf_index, +- dma_addr_t dma_addr, unsigned int len) +-{ +- struct rq_enet_desc *desc = vnic_rq_next_desc(rq); +- u64 wrid = 0; +- u8 type = os_buf_index ? +- RQ_ENET_TYPE_NOT_SOP : RQ_ENET_TYPE_ONLY_SOP; +- +- rq_enet_desc_enc(desc, +- (u64)dma_addr | VNIC_PADDR_TARGET, +- type, (u16)len); +- +- vnic_rq_post(rq, os_buf, os_buf_index, dma_addr, len, wrid); +-} + + struct enic; + +diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c +new file mode 100644 +index 0000000..945a60f +--- /dev/null ++++ b/drivers/net/enic/enic_rx.c +@@ -0,0 +1,370 @@ ++/* ++ * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved. ++ * Copyright 2007 Nuova Systems, Inc. All rights reserved. ++ * ++ * Copyright (c) 2014, Cisco Systems, Inc. ++ * All rights reserved. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions ++ * are met: ++ * ++ * 1. Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * 2. Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in ++ * the documentation and/or other materials provided with the ++ * distribution. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS ++ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ++ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, ++ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, ++ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER ++ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT ++ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ++ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE ++ * POSSIBILITY OF SUCH DAMAGE. ++ * ++ */ ++ ++#include ++#include ++#include ++ ++#include "enic_compat.h" ++#include "rq_enet_desc.h" ++#include "enic.h" ++ ++#define RTE_PMD_USE_PREFETCH ++ ++#ifdef RTE_PMD_USE_PREFETCH ++/* ++ * Prefetch a cache line into all cache levels. ++ */ ++#define rte_enic_prefetch(p) rte_prefetch0(p) ++#else ++#define rte_enic_prefetch(p) do {} while (0) ++#endif ++ ++#ifdef RTE_PMD_PACKET_PREFETCH ++#define rte_packet_prefetch(p) rte_prefetch1(p) ++#else ++#define rte_packet_prefetch(p) do {} while (0) ++#endif ++ ++static inline struct rte_mbuf * ++rte_rxmbuf_alloc(struct rte_mempool *mp) ++{ ++ struct rte_mbuf *m; ++ ++ m = __rte_mbuf_raw_alloc(mp); ++ __rte_mbuf_sanity_check_raw(m, 0); ++ return m; ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) ++{ ++ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) ++{ ++ return(le16_to_cpu(crd->bytes_written_flags) & ++ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_packet_error(uint16_t bwflags) ++{ ++ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == ++ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_eop(uint16_t ciflags) ++{ ++ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) ++ == CQ_ENET_RQ_DESC_FLAGS_EOP; ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) ++{ ++ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & ++ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == ++ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) ++{ ++ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == ++ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) ++{ ++ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == ++ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) ++{ ++ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> ++ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); ++} ++ ++static inline uint32_t ++enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) ++{ ++ return le32_to_cpu(cqrd->rss_hash); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_fcs_ok(struct cq_enet_rq_desc *cqrd) ++{ ++ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) == ++ CQ_ENET_RQ_DESC_FLAGS_FCS_OK); ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) ++{ ++ return le16_to_cpu(cqrd->vlan); ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ return le16_to_cpu(cqrd->bytes_written_flags) & ++ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; ++} ++ ++static inline uint64_t ++enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ uint16_t bwflags; ++ uint64_t pkt_err_flags = 0; ++ ++ bwflags = enic_cq_rx_desc_bwflags(cqrd); ++ ++ /* Check for packet error. Can't be more specific than MAC error */ ++ if (enic_cq_rx_desc_packet_error(bwflags)) { ++ pkt_err_flags |= PKT_RX_MAC_ERR; ++ } ++ ++ /* Check for bad FCS. MAC error isn't quite, but no other choice */ ++ if (!enic_cq_rx_desc_fcs_ok(cqrd)) { ++ pkt_err_flags |= PKT_RX_MAC_ERR; ++ } ++ return pkt_err_flags; ++} ++ ++/* ++ * Lookup table to translate RX CQ flags to mbuf flags. ++ */ ++static inline uint32_t ++enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ uint8_t cqrd_flags = cqrd->flags; ++ static const uint32_t cq_type_table[128] __rte_cache_aligned = { ++ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, ++ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 ++ | RTE_PTYPE_L4_UDP, ++ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 ++ | RTE_PTYPE_L4_TCP, ++ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 ++ | RTE_PTYPE_L4_FRAG, ++ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, ++ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 ++ | RTE_PTYPE_L4_UDP, ++ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 ++ | RTE_PTYPE_L4_TCP, ++ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 ++ | RTE_PTYPE_L4_FRAG, ++ /* All others reserved */ ++ }; ++ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT ++ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 ++ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; ++ return cq_type_table[cqrd_flags]; ++} ++ ++static inline void ++enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ uint16_t ciflags, bwflags, pkt_flags = 0; ++ ciflags = enic_cq_rx_desc_ciflags(cqrd); ++ bwflags = enic_cq_rx_desc_bwflags(cqrd); ++ ++ ASSERT(mbuf->ol_flags == 0); ++ ++ /* flags are meaningless if !EOP */ ++ if (unlikely(!enic_cq_rx_desc_eop(ciflags))) ++ goto mbuf_flags_done; ++ ++ /* VLAN stripping */ ++ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { ++ pkt_flags |= PKT_RX_VLAN_PKT; ++ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); ++ } else { ++ mbuf->vlan_tci = 0; ++ } ++ ++ /* RSS flag */ ++ if (enic_cq_rx_desc_rss_type(cqrd)) { ++ pkt_flags |= PKT_RX_RSS_HASH; ++ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); ++ } ++ ++ /* checksum flags */ ++ if (!enic_cq_rx_desc_csum_not_calc(cqrd) && ++ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { ++ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) ++ pkt_flags |= PKT_RX_IP_CKSUM_BAD; ++ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { ++ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) ++ pkt_flags |= PKT_RX_L4_CKSUM_BAD; ++ } ++ } ++ ++ mbuf_flags_done: ++ mbuf->ol_flags = pkt_flags; ++} ++ ++static inline uint32_t ++enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) ++{ ++ uint32_t d = i0 + i1; ++ ASSERT(i0 < n_descriptors); ++ ASSERT(i1 < n_descriptors); ++ d -= (d >= n_descriptors) ? n_descriptors : 0; ++ return d; ++} ++ ++ ++uint16_t ++enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, ++ uint16_t nb_pkts) ++{ ++ struct vnic_rq *rq = rx_queue; ++ struct enic *enic = vnic_dev_priv(rq->vdev); ++ unsigned int rx_id; ++ struct rte_mbuf *nmb, *rxmb; ++ uint16_t nb_rx = 0; ++ uint16_t nb_hold; ++ struct vnic_cq *cq; ++ volatile struct cq_desc *cqd_ptr; ++ uint8_t color; ++ ++ cq = &enic->cq[enic_cq_rq(enic, rq->index)]; ++ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ ++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; ++ ++ nb_hold = rq->rx_nb_hold; /* mbufs held by software */ ++ ++ while (nb_rx < nb_pkts) { ++ uint16_t rx_pkt_len; ++ volatile struct rq_enet_desc *rqd_ptr; ++ dma_addr_t dma_addr; ++ struct cq_desc cqd; ++ uint64_t ol_err_flags; ++ ++ /* Check for pkts available */ ++ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) ++ & CQ_DESC_COLOR_MASK; ++ if (color == cq->last_color) ++ break; ++ ++ /* Get the cq descriptor and rq pointer */ ++ cqd = *cqd_ptr; ++ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; ++ ++ /* allocate a new mbuf */ ++ nmb = rte_rxmbuf_alloc(rq->mp); ++ if (nmb == NULL) { ++ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", ++ enic->port_id, (unsigned)rq->index); ++ rte_eth_devices[enic->port_id]. ++ data->rx_mbuf_alloc_failed++; ++ break; ++ } ++ ++ /* Check for FCS or packet errors */ ++ ol_err_flags = enic_cq_rx_to_pkt_err_flags(&cqd); ++ if (ol_err_flags == 0) ++ rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd); ++ else ++ rx_pkt_len = 0; ++ ++ /* Get the mbuf to return and replace with one just allocated */ ++ rxmb = rq->mbuf_ring[rx_id]; ++ rq->mbuf_ring[rx_id] = nmb; ++ ++ /* Increment cqd, rqd, mbuf_table index */ ++ rx_id++; ++ if (unlikely(rx_id == rq->ring.desc_count)) { ++ rx_id = 0; ++ cq->last_color = cq->last_color ? 0 : 1; ++ } ++ ++ /* Prefetch next mbuf & desc while processing current one */ ++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; ++ rte_enic_prefetch(cqd_ptr); ++ rte_enic_prefetch(rq->mbuf_ring[rx_id]); ++ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) ++ + rx_id); ++ ++ /* Push descriptor for newly allocated mbuf */ ++ dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off); ++ rqd_ptr->address = rte_cpu_to_le_64(dma_addr); ++ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len); ++ ++ /* Fill in the rest of the mbuf */ ++ rxmb->data_off = RTE_PKTMBUF_HEADROOM; ++ rxmb->nb_segs = 1; ++ rxmb->next = NULL; ++ rxmb->pkt_len = rx_pkt_len; ++ rxmb->data_len = rx_pkt_len; ++ rxmb->port = enic->port_id; ++ rxmb->ol_flags = ol_err_flags; ++ if (!ol_err_flags) ++ enic_cq_rx_to_pkt_flags(&cqd, rxmb); ++ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); ++ ++ /* prefetch mbuf data for caller */ ++ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, ++ RTE_PKTMBUF_HEADROOM)); ++ ++ /* store the mbuf address into the next entry of the array */ ++ rx_pkts[nb_rx++] = rxmb; ++ } ++ ++ nb_hold += nb_rx; ++ cq->to_clean = rx_id; ++ ++ if (nb_hold > rq->rx_free_thresh) { ++ rq->posted_index = enic_ring_add(rq->ring.desc_count, ++ rq->posted_index, nb_hold); ++ nb_hold = 0; ++ rte_mb(); ++ iowrite32(rq->posted_index, &rq->ctrl->posted_index); ++ } ++ ++ rq->rx_nb_hold = nb_hold; ++ ++ return nb_rx; ++} diff --git a/dpdk/dpdk-2.2.0_patches/0012-Fix-ENIC-PMD-problem-with-not-sending-the-first-pack.patch b/dpdk/dpdk-2.2.0_patches/0012-Fix-ENIC-PMD-problem-with-not-sending-the-first-pack.patch deleted file mode 100644 index 9730adc4..00000000 --- a/dpdk/dpdk-2.2.0_patches/0012-Fix-ENIC-PMD-problem-with-not-sending-the-first-pack.patch +++ /dev/null @@ -1,31 +0,0 @@ -From 70725980c4d8f1e01cfa352a95341893aab00e7e Mon Sep 17 00:00:00 2001 -From: John Lo -Date: Mon, 7 Mar 2016 14:07:19 -0500 -Subject: [PATCH 1/2] Fix-ENIC-PMD-problem-with-not-sending-the-first-pack. - ---- - drivers/net/enic/base/enic_vnic_wq.h | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - -diff --git a/drivers/net/enic/base/enic_vnic_wq.h b/drivers/net/enic/base/enic_vnic_wq.h -index e3ea574..b019109 100644 ---- a/drivers/net/enic/base/enic_vnic_wq.h -+++ b/drivers/net/enic/base/enic_vnic_wq.h -@@ -69,11 +69,11 @@ static inline void enic_vnic_post_wq(struct vnic_wq *wq, - buf->wr_id = wrid; - - buf = buf->next; -- if (cq_entry) -- enic_vnic_post_wq_index(wq); -+ wq->ring.desc_avail -= desc_skip_cnt; - wq->to_use = buf; - -- wq->ring.desc_avail -= desc_skip_cnt; -+ if (cq_entry) -+ enic_vnic_post_wq_index(wq); - } - - #endif /* _ENIC_VNIC_WQ_H_ */ --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0012-enic-fix-last-packet-not-being-sent.patch b/dpdk/dpdk-2.2.0_patches/0012-enic-fix-last-packet-not-being-sent.patch new file mode 100644 index 00000000..181ca912 --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0012-enic-fix-last-packet-not-being-sent.patch @@ -0,0 +1,34 @@ +commit aba31298526865f5db99eaa54b63eb39dc95b74f +Author: John Daley +Date: Tue Mar 8 10:49:07 2016 -0800 + + enic: fix last packet not being sent + + The last packet of the tx burst function array was not being + emitted until the subsequent call. The nic descriptor index + was being set to the current tx descriptor instead of one past + the descriptor as required by the nic. + + Fixes: d739ba4c6abf ("enic: improve Tx packet rate") + + Signed-off-by: John Daley + +diff --git a/drivers/net/enic/base/enic_vnic_wq.h b/drivers/net/enic/base/enic_vnic_wq.h +index e3ea574..b019109 100644 +--- a/drivers/net/enic/base/enic_vnic_wq.h ++++ b/drivers/net/enic/base/enic_vnic_wq.h +@@ -69,11 +69,11 @@ static inline void enic_vnic_post_wq(struct vnic_wq *wq, + buf->wr_id = wrid; + + buf = buf->next; +- if (cq_entry) +- enic_vnic_post_wq_index(wq); ++ wq->ring.desc_avail -= desc_skip_cnt; + wq->to_use = buf; + +- wq->ring.desc_avail -= desc_skip_cnt; ++ if (cq_entry) ++ enic_vnic_post_wq_index(wq); + } + + #endif /* _ENIC_VNIC_WQ_H_ */ diff --git a/dpdk/dpdk-2.2.0_patches/0013-Replacement-of-ENIC-PMD-receive-path-to-improve-perf.patch b/dpdk/dpdk-2.2.0_patches/0013-Replacement-of-ENIC-PMD-receive-path-to-improve-perf.patch deleted file mode 100644 index 92bd1e89..00000000 --- a/dpdk/dpdk-2.2.0_patches/0013-Replacement-of-ENIC-PMD-receive-path-to-improve-perf.patch +++ /dev/null @@ -1,1347 +0,0 @@ -From 0963caf48db910ca6f93c5cbff8d5d24b2be64dd Mon Sep 17 00:00:00 2001 -From: John Lo -Date: Mon, 7 Mar 2016 14:12:29 -0500 -Subject: [PATCH 2/2] Replacement of ENIC PMD receive path to improve - performance and code clarifty. - ---- - drivers/net/enic/Makefile | 1 + - drivers/net/enic/base/vnic_rq.c | 99 ++--------- - drivers/net/enic/base/vnic_rq.h | 147 +--------------- - drivers/net/enic/enic.h | 16 +- - drivers/net/enic/enic_ethdev.c | 27 ++- - drivers/net/enic/enic_main.c | 321 ++++++++++------------------------ - drivers/net/enic/enic_res.h | 16 +- - drivers/net/enic/enic_rx.c | 378 ++++++++++++++++++++++++++++++++++++++++ - 8 files changed, 519 insertions(+), 486 deletions(-) - create mode 100644 drivers/net/enic/enic_rx.c - -diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile -index f0ee093..f316274 100644 ---- a/drivers/net/enic/Makefile -+++ b/drivers/net/enic/Makefile -@@ -53,6 +53,7 @@ VPATH += $(SRCDIR)/src - # - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c -+SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c -diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c -index 1441604..cb62c5e 100644 ---- a/drivers/net/enic/base/vnic_rq.c -+++ b/drivers/net/enic/base/vnic_rq.c -@@ -35,77 +35,21 @@ - #include "vnic_dev.h" - #include "vnic_rq.h" - --static int vnic_rq_alloc_bufs(struct vnic_rq *rq) --{ -- struct vnic_rq_buf *buf; -- unsigned int i, j, count = rq->ring.desc_count; -- unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count); -- -- for (i = 0; i < blks; i++) { -- rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ(count), GFP_ATOMIC); -- if (!rq->bufs[i]) -- return -ENOMEM; -- } -- -- for (i = 0; i < blks; i++) { -- buf = rq->bufs[i]; -- for (j = 0; j < VNIC_RQ_BUF_BLK_ENTRIES(count); j++) { -- buf->index = i * VNIC_RQ_BUF_BLK_ENTRIES(count) + j; -- buf->desc = (u8 *)rq->ring.descs + -- rq->ring.desc_size * buf->index; -- if (buf->index + 1 == count) { -- buf->next = rq->bufs[0]; -- break; -- } else if (j + 1 == VNIC_RQ_BUF_BLK_ENTRIES(count)) { -- buf->next = rq->bufs[i + 1]; -- } else { -- buf->next = buf + 1; -- buf++; -- } -- } -- } -- -- rq->to_use = rq->to_clean = rq->bufs[0]; -- -- return 0; --} -- --int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count, -- unsigned int desc_size) --{ -- int mem_size = 0; -- -- mem_size += vnic_dev_desc_ring_size(&rq->ring, desc_count, desc_size); -- -- mem_size += VNIC_RQ_BUF_BLKS_NEEDED(rq->ring.desc_count) * -- VNIC_RQ_BUF_BLK_SZ(rq->ring.desc_count); -- -- return mem_size; --} -- - void vnic_rq_free(struct vnic_rq *rq) - { - struct vnic_dev *vdev; -- unsigned int i; - - vdev = rq->vdev; - - vnic_dev_free_desc_ring(vdev, &rq->ring); - -- for (i = 0; i < VNIC_RQ_BUF_BLKS_MAX; i++) { -- if (rq->bufs[i]) { -- kfree(rq->bufs[i]); -- rq->bufs[i] = NULL; -- } -- } -- - rq->ctrl = NULL; - } - - int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, - unsigned int desc_count, unsigned int desc_size) - { -- int err; -+ int rc; - char res_name[NAME_MAX]; - static int instance; - -@@ -121,18 +65,9 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, - vnic_rq_disable(rq); - - snprintf(res_name, sizeof(res_name), "%d-rq-%d", instance++, index); -- err = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, -+ rc = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, - rq->socket_id, res_name); -- if (err) -- return err; -- -- err = vnic_rq_alloc_bufs(rq); -- if (err) { -- vnic_rq_free(rq); -- return err; -- } -- -- return 0; -+ return rc; - } - - void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, -@@ -154,9 +89,6 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, - iowrite32(fetch_index, &rq->ctrl->fetch_index); - iowrite32(posted_index, &rq->ctrl->posted_index); - -- rq->to_use = rq->to_clean = -- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] -- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; - } - - void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, -@@ -176,6 +108,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, - fetch_index, fetch_index, - error_interrupt_enable, - error_interrupt_offset); -+ rq->rxst_idx = 0; -+ rq->tot_pkts = 0; - } - - void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error) -@@ -212,21 +146,20 @@ int vnic_rq_disable(struct vnic_rq *rq) - } - - void vnic_rq_clean(struct vnic_rq *rq, -- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)) -+ void (*buf_clean)(struct rte_mbuf **buf)) - { -- struct vnic_rq_buf *buf; -- u32 fetch_index; -+ struct rte_mbuf **buf; -+ u32 fetch_index, i; - unsigned int count = rq->ring.desc_count; - -- buf = rq->to_clean; -- -- while (vnic_rq_desc_used(rq) > 0) { -+ buf = &rq->mbuf_ring[0]; - -- (*buf_clean)(rq, buf); -- -- buf = rq->to_clean = buf->next; -- rq->ring.desc_avail++; -+ for (i = 0; i < count; i++) { -+ (*buf_clean)(buf); -+ buf++; - } -+ rq->ring.desc_avail = count - 1; -+ rq->rx_nb_hold = 0; - - /* Use current fetch_index as the ring starting point */ - fetch_index = ioread32(&rq->ctrl->fetch_index); -@@ -235,9 +168,7 @@ void vnic_rq_clean(struct vnic_rq *rq, - /* Hardware surprise removal: reset fetch_index */ - fetch_index = 0; - } -- rq->to_use = rq->to_clean = -- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] -- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; -+ - iowrite32(fetch_index, &rq->ctrl->posted_index); - - vnic_dev_clear_desc_ring(&rq->ring); -diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h -index 0f5c3c1..e083ccc 100644 ---- a/drivers/net/enic/base/vnic_rq.h -+++ b/drivers/net/enic/base/vnic_rq.h -@@ -66,42 +66,22 @@ struct vnic_rq_ctrl { - u32 pad10; - }; - --/* Break the vnic_rq_buf allocations into blocks of 32/64 entries */ --#define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32 --#define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64 --#define VNIC_RQ_BUF_BLK_ENTRIES(entries) \ -- ((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \ -- VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES)) --#define VNIC_RQ_BUF_BLK_SZ(entries) \ -- (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf)) --#define VNIC_RQ_BUF_BLKS_NEEDED(entries) \ -- DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries)) --#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096) -- --struct vnic_rq_buf { -- struct vnic_rq_buf *next; -- dma_addr_t dma_addr; -- void *os_buf; -- unsigned int os_buf_index; -- unsigned int len; -- unsigned int index; -- void *desc; -- uint64_t wr_id; --}; -- - struct vnic_rq { - unsigned int index; -+ unsigned int posted_index; - struct vnic_dev *vdev; -- struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ -+ struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ - struct vnic_dev_ring ring; -- struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX]; -- struct vnic_rq_buf *to_use; -- struct vnic_rq_buf *to_clean; -+ struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */ -+ unsigned int mbuf_next_idx; /* next mb to consume */ - void *os_buf_head; - unsigned int pkts_outstanding; -- -+ uint16_t rx_nb_hold; -+ uint16_t rx_free_thresh; - unsigned int socket_id; - struct rte_mempool *mp; -+ uint16_t rxst_idx; -+ uint32_t tot_pkts; - }; - - static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq) -@@ -116,119 +96,13 @@ static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq) - return rq->ring.desc_count - rq->ring.desc_avail - 1; - } - --static inline void *vnic_rq_next_desc(struct vnic_rq *rq) --{ -- return rq->to_use->desc; --} -- --static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq) --{ -- return rq->to_use->index; --} -- --static inline void vnic_rq_post(struct vnic_rq *rq, -- void *os_buf, unsigned int os_buf_index, -- dma_addr_t dma_addr, unsigned int len, -- uint64_t wrid) --{ -- struct vnic_rq_buf *buf = rq->to_use; -- -- buf->os_buf = os_buf; -- buf->os_buf_index = os_buf_index; -- buf->dma_addr = dma_addr; -- buf->len = len; -- buf->wr_id = wrid; -- -- buf = buf->next; -- rq->to_use = buf; -- rq->ring.desc_avail--; -- -- /* Move the posted_index every nth descriptor -- */ -- --#ifndef VNIC_RQ_RETURN_RATE --#define VNIC_RQ_RETURN_RATE 0xf /* keep 2^n - 1 */ --#endif -- -- if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) { -- /* Adding write memory barrier prevents compiler and/or CPU -- * reordering, thus avoiding descriptor posting before -- * descriptor is initialized. Otherwise, hardware can read -- * stale descriptor fields. -- */ -- wmb(); -- iowrite32(buf->index, &rq->ctrl->posted_index); -- } --} -- --static inline void vnic_rq_post_commit(struct vnic_rq *rq, -- void *os_buf, unsigned int os_buf_index, -- dma_addr_t dma_addr, unsigned int len) --{ -- struct vnic_rq_buf *buf = rq->to_use; -- -- buf->os_buf = os_buf; -- buf->os_buf_index = os_buf_index; -- buf->dma_addr = dma_addr; -- buf->len = len; -- -- buf = buf->next; -- rq->to_use = buf; -- rq->ring.desc_avail--; -- -- /* Move the posted_index every descriptor -- */ -- -- /* Adding write memory barrier prevents compiler and/or CPU -- * reordering, thus avoiding descriptor posting before -- * descriptor is initialized. Otherwise, hardware can read -- * stale descriptor fields. -- */ -- wmb(); -- iowrite32(buf->index, &rq->ctrl->posted_index); --} - --static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count) --{ -- rq->ring.desc_avail += count; --} - - enum desc_return_options { - VNIC_RQ_RETURN_DESC, - VNIC_RQ_DEFER_RETURN_DESC, - }; - --static inline int vnic_rq_service(struct vnic_rq *rq, -- struct cq_desc *cq_desc, u16 completed_index, -- int desc_return, int (*buf_service)(struct vnic_rq *rq, -- struct cq_desc *cq_desc, struct vnic_rq_buf *buf, -- int skipped, void *opaque), void *opaque) --{ -- struct vnic_rq_buf *buf; -- int skipped; -- int eop = 0; -- -- buf = rq->to_clean; -- while (1) { -- -- skipped = (buf->index != completed_index); -- -- if ((*buf_service)(rq, cq_desc, buf, skipped, opaque)) -- eop++; -- -- if (desc_return == VNIC_RQ_RETURN_DESC) -- rq->ring.desc_avail++; -- -- rq->to_clean = buf->next; -- -- if (!skipped) -- break; -- -- buf = rq->to_clean; -- } -- return eop; --} -- - static inline int vnic_rq_fill(struct vnic_rq *rq, - int (*buf_fill)(struct vnic_rq *rq)) - { -@@ -274,8 +148,5 @@ unsigned int vnic_rq_error_status(struct vnic_rq *rq); - void vnic_rq_enable(struct vnic_rq *rq); - int vnic_rq_disable(struct vnic_rq *rq); - void vnic_rq_clean(struct vnic_rq *rq, -- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)); --int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count, -- unsigned int desc_size); -- -+ void (*buf_clean)(struct rte_mbuf **buf)); - #endif /* _VNIC_RQ_H_ */ -diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h -index 9e78305..8c914f5 100644 ---- a/drivers/net/enic/enic.h -+++ b/drivers/net/enic/enic.h -@@ -45,6 +45,7 @@ - #include "vnic_nic.h" - #include "vnic_rss.h" - #include "enic_res.h" -+#include "cq_enet_desc.h" - - #define DRV_NAME "enic_pmd" - #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Poll-mode Driver" -@@ -154,6 +155,16 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) - return (struct enic *)eth_dev->data->dev_private; - } - -+#define RTE_LIBRTE_ENIC_ASSERT_ENABLE -+#ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE -+#define ASSERT(x) do { \ -+ if (!(x)) \ -+ rte_panic("ENIC: x"); \ -+} while (0) -+#else -+#define ASSERT(x) -+#endif -+ - extern void enic_fdir_stats_get(struct enic *enic, - struct rte_eth_fdir_stats *stats); - extern int enic_fdir_add_fltr(struct enic *enic, -@@ -193,9 +204,10 @@ extern void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, - uint16_t ol_flags, uint16_t vlan_tag); - - extern void enic_post_wq_index(struct vnic_wq *wq); --extern int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts, -- unsigned int budget, unsigned int *work_done); - extern int enic_probe(struct enic *enic); - extern int enic_clsf_init(struct enic *enic); - extern void enic_clsf_destroy(struct enic *enic); -+uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -+ uint16_t nb_pkts); -+ - #endif /* _ENIC_H_ */ -diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c -index 2a88043..6f2ada5 100644 ---- a/drivers/net/enic/enic_ethdev.c -+++ b/drivers/net/enic/enic_ethdev.c -@@ -255,7 +255,7 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, - uint16_t queue_idx, - uint16_t nb_desc, - unsigned int socket_id, -- __rte_unused const struct rte_eth_rxconf *rx_conf, -+ const struct rte_eth_rxconf *rx_conf, - struct rte_mempool *mp) - { - int ret; -@@ -270,6 +270,10 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, - return ret; - } - -+ enic->rq[queue_idx].rx_free_thresh = rx_conf->rx_free_thresh; -+ dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx, -+ enic->rq[queue_idx].rx_free_thresh); -+ - return enicpmd_dev_setup_intr(enic); - } - -@@ -429,6 +433,9 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev, - DEV_TX_OFFLOAD_IPV4_CKSUM | - DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_CKSUM; -+ device_info->default_rxconf = (struct rte_eth_rxconf) { -+ .rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH -+ }; - } - - static void enicpmd_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) -@@ -538,18 +545,6 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - return index; - } - --static uint16_t enicpmd_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -- uint16_t nb_pkts) --{ -- struct vnic_rq *rq = (struct vnic_rq *)rx_queue; -- unsigned int work_done; -- -- if (enic_poll(rq, rx_pkts, (unsigned int)nb_pkts, &work_done)) -- dev_err(enic, "error in enicpmd poll\n"); -- -- return work_done; --} -- - static const struct eth_dev_ops enicpmd_eth_dev_ops = { - .dev_configure = enicpmd_dev_configure, - .dev_start = enicpmd_dev_start, -@@ -606,7 +601,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) - enic->port_id = eth_dev->data->port_id; - enic->rte_dev = eth_dev; - eth_dev->dev_ops = &enicpmd_eth_dev_ops; -- eth_dev->rx_pkt_burst = &enicpmd_recv_pkts; -+ eth_dev->rx_pkt_burst = &enic_recv_pkts; - eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts; - - pdev = eth_dev->pci_dev; -@@ -635,8 +630,8 @@ static struct eth_driver rte_enic_pmd = { - * Register as the [Poll Mode] Driver of Cisco ENIC device. - */ - static int --rte_enic_pmd_init(const char *name __rte_unused, -- const char *params __rte_unused) -+rte_enic_pmd_init(__rte_unused const char *name, -+ __rte_unused const char *params) - { - ENICPMD_FUNC_TRACE(); - -diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index 07a9810..d0c9bff 100644 ---- a/drivers/net/enic/enic_main.c -+++ b/drivers/net/enic/enic_main.c -@@ -60,6 +60,17 @@ - #include "vnic_nic.h" - #include "enic_vnic_wq.h" - -+static inline struct rte_mbuf * -+rte_rxmbuf_alloc(struct rte_mempool *mp) -+{ -+ struct rte_mbuf *m; -+ -+ m = __rte_mbuf_raw_alloc(mp); -+ __rte_mbuf_sanity_check_raw(m, 0); -+ return m; -+} -+ -+ - static inline int enic_is_sriov_vf(struct enic *enic) - { - return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF; -@@ -80,16 +91,25 @@ static int is_eth_addr_valid(uint8_t *addr) - return !is_mcast_addr(addr) && !is_zero_addr(addr); - } - --static inline struct rte_mbuf * --enic_rxmbuf_alloc(struct rte_mempool *mp) -+static void -+enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) - { -- struct rte_mbuf *m; -+ uint16_t i; - -- m = __rte_mbuf_raw_alloc(mp); -- __rte_mbuf_sanity_check_raw(m, 0); -- return m; -+ if (!rq || !rq->mbuf_ring) { -+ dev_debug(enic, "Pointer to rq or mbuf_ring is NULL"); -+ return; -+ } -+ -+ for (i = 0; i < enic->config.rq_desc_count; i++) { -+ if (rq->mbuf_ring[i]) { -+ rte_pktmbuf_free_seg(rq->mbuf_ring[i]); -+ rq->mbuf_ring[i] = NULL; -+ } -+ } - } - -+ - void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) - { - vnic_set_hdr_split_size(enic->vdev, split_hdr_size); -@@ -262,13 +282,13 @@ void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr) - } - - static void --enic_free_rq_buf(__rte_unused struct vnic_rq *rq, struct vnic_rq_buf *buf) -+enic_free_rq_buf(struct rte_mbuf **mbuf) - { -- if (!buf->os_buf) -+ if (*mbuf == NULL) - return; - -- rte_pktmbuf_free((struct rte_mbuf *)buf->os_buf); -- buf->os_buf = NULL; -+ rte_pktmbuf_free(*mbuf); -+ mbuf = NULL; - } - - void enic_init_vnic_resources(struct enic *enic) -@@ -314,221 +334,47 @@ void enic_init_vnic_resources(struct enic *enic) - } - - --static int enic_rq_alloc_buf(struct vnic_rq *rq) -+static int -+enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) - { -- struct enic *enic = vnic_dev_priv(rq->vdev); -+ struct rte_mbuf *mb; -+ struct rq_enet_desc *rqd = rq->ring.descs; -+ unsigned i; - dma_addr_t dma_addr; -- struct rq_enet_desc *desc = vnic_rq_next_desc(rq); -- uint8_t type = RQ_ENET_TYPE_ONLY_SOP; -- u16 split_hdr_size = vnic_get_hdr_split_size(enic->vdev); -- struct rte_mbuf *mbuf = enic_rxmbuf_alloc(rq->mp); -- struct rte_mbuf *hdr_mbuf = NULL; -- -- if (!mbuf) { -- dev_err(enic, "mbuf alloc in enic_rq_alloc_buf failed\n"); -- return -1; -- } -- -- if (unlikely(split_hdr_size)) { -- if (vnic_rq_desc_avail(rq) < 2) { -- rte_mempool_put(mbuf->pool, mbuf); -- return -1; -- } -- hdr_mbuf = enic_rxmbuf_alloc(rq->mp); -- if (!hdr_mbuf) { -- rte_mempool_put(mbuf->pool, mbuf); -- dev_err(enic, -- "hdr_mbuf alloc in enic_rq_alloc_buf failed\n"); -- return -1; -- } -- -- hdr_mbuf->data_off = RTE_PKTMBUF_HEADROOM; -- -- hdr_mbuf->nb_segs = 2; -- hdr_mbuf->port = enic->port_id; -- hdr_mbuf->next = mbuf; -- -- dma_addr = (dma_addr_t) -- (hdr_mbuf->buf_physaddr + hdr_mbuf->data_off); -- -- rq_enet_desc_enc(desc, dma_addr, type, split_hdr_size); - -- vnic_rq_post(rq, (void *)hdr_mbuf, 0 /*os_buf_index*/, dma_addr, -- (unsigned int)split_hdr_size, 0 /*wrid*/); -+ dev_debug(enic, "queue %u, allocating %u rx queue mbufs", rq->index, -+ rq->ring.desc_count); - -- desc = vnic_rq_next_desc(rq); -- type = RQ_ENET_TYPE_NOT_SOP; -- } else { -- mbuf->nb_segs = 1; -- mbuf->port = enic->port_id; -- } -- -- mbuf->data_off = RTE_PKTMBUF_HEADROOM; -- mbuf->next = NULL; -- -- dma_addr = (dma_addr_t) -- (mbuf->buf_physaddr + mbuf->data_off); -- -- rq_enet_desc_enc(desc, dma_addr, type, mbuf->buf_len); -- -- vnic_rq_post(rq, (void *)mbuf, 0 /*os_buf_index*/, dma_addr, -- (unsigned int)mbuf->buf_len, 0 /*wrid*/); -- -- return 0; --} -- --static int enic_rq_indicate_buf(struct vnic_rq *rq, -- struct cq_desc *cq_desc, struct vnic_rq_buf *buf, -- int skipped, void *opaque) --{ -- struct enic *enic = vnic_dev_priv(rq->vdev); -- struct rte_mbuf **rx_pkt_bucket = (struct rte_mbuf **)opaque; -- struct rte_mbuf *rx_pkt = NULL; -- struct rte_mbuf *hdr_rx_pkt = NULL; -- -- u8 type, color, eop, sop, ingress_port, vlan_stripped; -- u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof; -- u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok; -- u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc; -- u8 packet_error; -- u16 q_number, completed_index, bytes_written, vlan_tci, checksum; -- u32 rss_hash; -- -- cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, -- &type, &color, &q_number, &completed_index, -- &ingress_port, &fcoe, &eop, &sop, &rss_type, -- &csum_not_calc, &rss_hash, &bytes_written, -- &packet_error, &vlan_stripped, &vlan_tci, &checksum, -- &fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error, -- &fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp, -- &ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment, -- &fcs_ok); -- -- rx_pkt = (struct rte_mbuf *)buf->os_buf; -- buf->os_buf = NULL; -- -- if (unlikely(packet_error)) { -- dev_err(enic, "packet error\n"); -- rx_pkt->data_len = 0; -- return 0; -- } -- -- if (unlikely(skipped)) { -- rx_pkt->data_len = 0; -- return 0; -- } -- -- if (likely(!vnic_get_hdr_split_size(enic->vdev))) { -- /* No header split configured */ -- *rx_pkt_bucket = rx_pkt; -- rx_pkt->pkt_len = bytes_written; -- -- if (ipv4) { -- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4; -- if (!csum_not_calc) { -- if (unlikely(!ipv4_csum_ok)) -- rx_pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD; -- -- if ((tcp || udp) && (!tcp_udp_csum_ok)) -- rx_pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD; -- } -- } else if (ipv6) -- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6; -- } else { -- /* Header split */ -- if (sop && !eop) { -- /* This piece is header */ -- *rx_pkt_bucket = rx_pkt; -- rx_pkt->pkt_len = bytes_written; -- } else { -- if (sop && eop) { -- /* The packet is smaller than split_hdr_size */ -- *rx_pkt_bucket = rx_pkt; -- rx_pkt->pkt_len = bytes_written; -- if (ipv4) { -- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4; -- if (!csum_not_calc) { -- if (unlikely(!ipv4_csum_ok)) -- rx_pkt->ol_flags |= -- PKT_RX_IP_CKSUM_BAD; -- -- if ((tcp || udp) && -- (!tcp_udp_csum_ok)) -- rx_pkt->ol_flags |= -- PKT_RX_L4_CKSUM_BAD; -- } -- } else if (ipv6) -- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6; -- } else { -- /* Payload */ -- hdr_rx_pkt = *rx_pkt_bucket; -- hdr_rx_pkt->pkt_len += bytes_written; -- if (ipv4) { -- hdr_rx_pkt->packet_type = -- RTE_PTYPE_L3_IPV4; -- if (!csum_not_calc) { -- if (unlikely(!ipv4_csum_ok)) -- hdr_rx_pkt->ol_flags |= -- PKT_RX_IP_CKSUM_BAD; -- -- if ((tcp || udp) && -- (!tcp_udp_csum_ok)) -- hdr_rx_pkt->ol_flags |= -- PKT_RX_L4_CKSUM_BAD; -- } -- } else if (ipv6) -- hdr_rx_pkt->packet_type = -- RTE_PTYPE_L3_IPV6; -- } -+ for (i = 0; i < rq->ring.desc_count; i++, rqd++) { -+ mb = rte_rxmbuf_alloc(rq->mp); -+ if (mb == NULL) { -+ dev_err(enic, "RX mbuf alloc failed queue_id=%u", -+ (unsigned)rq->index); -+ return -ENOMEM; - } -- } - -- rx_pkt->data_len = bytes_written; -+ dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off); - -- if (rss_hash) { -- rx_pkt->ol_flags |= PKT_RX_RSS_HASH; -- rx_pkt->hash.rss = rss_hash; -+ rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, -+ mb->buf_len); -+ rq->mbuf_ring[i] = mb; - } - -- if (vlan_tci) { -- rx_pkt->ol_flags |= PKT_RX_VLAN_PKT; -- rx_pkt->vlan_tci = vlan_tci; -- } -+ /* make sure all prior writes are complete before doing the PIO write */ -+ rte_rmb(); - -- return eop; --} -+ /* Post all but the last 2 cache lines' worth of descriptors */ -+ rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE -+ / sizeof(struct rq_enet_desc)); -+ rq->rx_nb_hold = 0; - --static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, -- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque) --{ -- struct enic *enic = vnic_dev_priv(vdev); -- -- return vnic_rq_service(&enic->rq[q_number], cq_desc, -- completed_index, VNIC_RQ_RETURN_DESC, -- enic_rq_indicate_buf, opaque); -- --} -+ dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n", -+ enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold); -+ iowrite32(rq->posted_index, &rq->ctrl->posted_index); -+ rte_rmb(); - --int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts, -- unsigned int budget, unsigned int *work_done) --{ -- struct enic *enic = vnic_dev_priv(rq->vdev); -- unsigned int cq = enic_cq_rq(enic, rq->index); -- int err = 0; -- -- *work_done = vnic_cq_service(&enic->cq[cq], -- budget, enic_rq_service, (void *)rx_pkts); -- -- if (*work_done) { -- vnic_rq_fill(rq, enic_rq_alloc_buf); -+ return 0; - -- /* Need at least one buffer on ring to get going */ -- if (vnic_rq_desc_used(rq) == 0) { -- dev_err(enic, "Unable to alloc receive buffers\n"); -- err = -1; -- } -- } -- return err; - } - - static void * -@@ -576,6 +422,7 @@ enic_intr_handler(__rte_unused struct rte_intr_handle *handle, - int enic_enable(struct enic *enic) - { - unsigned int index; -+ int err; - struct rte_eth_dev *eth_dev = enic->rte_dev; - - eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev); -@@ -586,15 +433,11 @@ int enic_enable(struct enic *enic) - dev_warning(enic, "Init of hash table for clsf failed."\ - "Flow director feature will not work\n"); - -- /* Fill RQ bufs */ - for (index = 0; index < enic->rq_count; index++) { -- vnic_rq_fill(&enic->rq[index], enic_rq_alloc_buf); -- -- /* Need at least one buffer on ring to get going -- */ -- if (vnic_rq_desc_used(&enic->rq[index]) == 0) { -- dev_err(enic, "Unable to alloc receive buffers\n"); -- return -1; -+ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]); -+ if (err) { -+ dev_err(enic, "Failed to alloc RX queue mbufs\n"); -+ return err; - } - } - -@@ -636,6 +479,9 @@ void enic_free_rq(void *rxq) - struct vnic_rq *rq = (struct vnic_rq *)rxq; - struct enic *enic = vnic_dev_priv(rq->vdev); - -+ enic_rxmbuf_queue_release(enic, rq); -+ rte_free(rq->mbuf_ring); -+ rq->mbuf_ring = NULL; - vnic_rq_free(rq); - vnic_cq_free(&enic->cq[rq->index]); - } -@@ -664,7 +510,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, - unsigned int socket_id, struct rte_mempool *mp, - uint16_t nb_desc) - { -- int err; -+ int rc; - struct vnic_rq *rq = &enic->rq[queue_idx]; - - rq->socket_id = socket_id; -@@ -687,23 +533,35 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, - } - - /* Allocate queue resources */ -- err = vnic_rq_alloc(enic->vdev, &enic->rq[queue_idx], queue_idx, -- enic->config.rq_desc_count, -- sizeof(struct rq_enet_desc)); -- if (err) { -+ rc = vnic_rq_alloc(enic->vdev, rq, queue_idx, -+ enic->config.rq_desc_count, sizeof(struct rq_enet_desc)); -+ if (rc) { - dev_err(enic, "error in allocation of rq\n"); -- return err; -+ goto err_exit; - } - -- err = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, -+ rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, - socket_id, enic->config.rq_desc_count, - sizeof(struct cq_enet_rq_desc)); -- if (err) { -- vnic_rq_free(rq); -+ if (rc) { - dev_err(enic, "error in allocation of cq for rq\n"); -+ goto err_free_rq_exit; - } - -- return err; -+ /* Allocate the mbuf ring */ -+ rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", -+ sizeof(struct rte_mbuf *) * enic->config.rq_desc_count, -+ RTE_CACHE_LINE_SIZE, rq->socket_id); -+ -+ if (rq->mbuf_ring != NULL) -+ return 0; -+ -+ /* cleanup on error */ -+ vnic_cq_free(&enic->cq[queue_idx]); -+err_free_rq_exit: -+ vnic_rq_free(rq); -+err_exit: -+ return -ENOMEM; - } - - void enic_free_wq(void *txq) -@@ -790,6 +648,7 @@ int enic_disable(struct enic *enic) - - for (i = 0; i < enic->wq_count; i++) - vnic_wq_clean(&enic->wq[i], enic_free_wq_buf); -+ - for (i = 0; i < enic->rq_count; i++) - vnic_rq_clean(&enic->rq[i], enic_free_rq_buf); - for (i = 0; i < enic->cq_count; i++) -@@ -1074,7 +933,7 @@ int enic_probe(struct enic *enic) - - /* Set ingress vlan rewrite mode before vnic initialization */ - err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, -- IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN); -+ IG_VLAN_REWRITE_MODE_PASS_THRU); - if (err) { - dev_err(enic, - "Failed to set ingress vlan rewrite mode, aborting.\n"); -diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h -index 49f7e22..33f2e84 100644 ---- a/drivers/net/enic/enic_res.h -+++ b/drivers/net/enic/enic_res.h -@@ -52,6 +52,7 @@ - #define ENIC_UNICAST_PERFECT_FILTERS 32 - - #define ENIC_NON_TSO_MAX_DESC 16 -+#define ENIC_DEFAULT_RX_FREE_THRESH 32 - - #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) - -@@ -133,21 +134,6 @@ static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq, - WQ_ENET_OFFLOAD_MODE_TSO, - eop, 1 /* SOP */, eop, loopback); - } --static inline void enic_queue_rq_desc(struct vnic_rq *rq, -- void *os_buf, unsigned int os_buf_index, -- dma_addr_t dma_addr, unsigned int len) --{ -- struct rq_enet_desc *desc = vnic_rq_next_desc(rq); -- u64 wrid = 0; -- u8 type = os_buf_index ? -- RQ_ENET_TYPE_NOT_SOP : RQ_ENET_TYPE_ONLY_SOP; -- -- rq_enet_desc_enc(desc, -- (u64)dma_addr | VNIC_PADDR_TARGET, -- type, (u16)len); -- -- vnic_rq_post(rq, os_buf, os_buf_index, dma_addr, len, wrid); --} - - struct enic; - -diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c -new file mode 100644 -index 0000000..12b1f62 ---- /dev/null -+++ b/drivers/net/enic/enic_rx.c -@@ -0,0 +1,378 @@ -+/* -+ * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved. -+ * Copyright 2007 Nuova Systems, Inc. All rights reserved. -+ * -+ * Copyright (c) 2014, Cisco Systems, Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions -+ * are met: -+ * -+ * 1. Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * 2. Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in -+ * the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ * -+ */ -+ -+#include -+#include -+#include -+ -+#include "enic_compat.h" -+#include "rq_enet_desc.h" -+#include "enic.h" -+ -+#define RTE_PMD_USE_PREFETCH -+ -+#ifdef RTE_PMD_USE_PREFETCH -+/* -+ * Prefetch a cache line into all cache levels. -+ */ -+#define rte_enic_prefetch(p) rte_prefetch0(p) -+#else -+#define rte_enic_prefetch(p) do {} while (0) -+#endif -+ -+#ifdef RTE_PMD_PACKET_PREFETCH -+#define rte_packet_prefetch(p) rte_prefetch1(p) -+#else -+#define rte_packet_prefetch(p) do {} while (0) -+#endif -+ -+static inline struct rte_mbuf * -+rte_rxmbuf_alloc(struct rte_mempool *mp) -+{ -+ struct rte_mbuf *m; -+ -+ m = __rte_mbuf_raw_alloc(mp); -+ __rte_mbuf_sanity_check_raw(m, 0); -+ return m; -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) -+{ -+ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) -+{ -+ return(le16_to_cpu(crd->bytes_written_flags) & -+ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_packet_error(uint16_t bwflags) -+{ -+ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == -+ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_eop(uint16_t ciflags) -+{ -+ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) -+ == CQ_ENET_RQ_DESC_FLAGS_EOP; -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & -+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == -+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) -+{ -+ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> -+ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); -+} -+ -+static inline uint32_t -+enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) -+{ -+ return le32_to_cpu(cqrd->rss_hash); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_fcs_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_FCS_OK); -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) -+{ -+ return le16_to_cpu(cqrd->vlan); -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ return le16_to_cpu(cqrd->bytes_written_flags) & -+ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; -+} -+ -+static inline uint64_t -+enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint16_t bwflags; -+ uint64_t pkt_err_flags = 0; -+ -+ bwflags = enic_cq_rx_desc_bwflags(cqrd); -+ -+ /* Check for packet error. Can't be more specific than MAC error */ -+ if (enic_cq_rx_desc_packet_error(bwflags)) { -+#ifdef RTE_NEXT_ABI -+ pkt_err_flags |= PKT_RX_MAC_ERR; -+#else -+ pkt_err_flags |= PKT_EXT_RX_PKT_ERROR; -+#endif -+ } -+ -+ /* Check for bad FCS. MAC error isn't quite, but no other choice */ -+ if (!enic_cq_rx_desc_fcs_ok(cqrd)) { -+#ifdef RTE_NEXT_ABI -+ pkt_err_flags |= PKT_RX_MAC_ERR; -+#else -+ pkt_err_flags |= PKT_EXT_RX_BAD_FCS; -+#endif -+ } -+ return pkt_err_flags; -+} -+ -+/* -+ * Lookup table to translate RX CQ flags to mbuf flags. -+ */ -+static inline uint32_t -+enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint8_t cqrd_flags = cqrd->flags; -+ static const uint32_t cq_type_table[128] __rte_cache_aligned = { -+ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, -+ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_UDP, -+ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_TCP, -+ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_FRAG, -+ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, -+ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_UDP, -+ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_TCP, -+ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_FRAG, -+ /* All others reserved */ -+ }; -+ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT -+ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 -+ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; -+ return cq_type_table[cqrd_flags]; -+} -+ -+static inline void -+enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint16_t ciflags, bwflags, pkt_flags = 0; -+ ciflags = enic_cq_rx_desc_ciflags(cqrd); -+ bwflags = enic_cq_rx_desc_bwflags(cqrd); -+ -+ ASSERT(mbuf->ol_flags == 0); -+ -+ /* flags are meaningless if !EOP */ -+ if (unlikely(!enic_cq_rx_desc_eop(ciflags))) -+ goto mbuf_flags_done; -+ -+ /* VLAN stripping */ -+ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { -+ pkt_flags |= PKT_RX_VLAN_PKT; -+ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); -+ } else { -+ mbuf->vlan_tci = 0; -+ } -+ -+ /* RSS flag */ -+ if (enic_cq_rx_desc_rss_type(cqrd)) { -+ pkt_flags |= PKT_RX_RSS_HASH; -+ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); -+ } -+ -+ /* checksum flags */ -+ if (!enic_cq_rx_desc_csum_not_calc(cqrd) && -+ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { -+ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) -+ pkt_flags |= PKT_RX_IP_CKSUM_BAD; -+ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { -+ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) -+ pkt_flags |= PKT_RX_L4_CKSUM_BAD; -+ } -+ } -+ -+ mbuf_flags_done: -+ mbuf->ol_flags = pkt_flags; -+} -+ -+static inline uint32_t -+enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) -+{ -+ uint32_t d = i0 + i1; -+ ASSERT(i0 < n_descriptors); -+ ASSERT(i1 < n_descriptors); -+ d -= (d >= n_descriptors) ? n_descriptors : 0; -+ return d; -+} -+ -+ -+uint16_t -+enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -+ uint16_t nb_pkts) -+{ -+ struct vnic_rq *rq = rx_queue; -+ struct enic *enic = vnic_dev_priv(rq->vdev); -+ unsigned int rx_id; -+ struct rte_mbuf *nmb, *rxmb; -+ uint16_t nb_rx = 0; -+ uint16_t nb_hold; -+ struct vnic_cq *cq; -+ volatile struct cq_desc *cqd_ptr; -+ uint8_t color; -+ -+ cq = &enic->cq[enic_cq_rq(enic, rq->index)]; -+ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ -+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -+ -+ nb_hold = rq->rx_nb_hold; /* mbufs held by software */ -+ -+ while (nb_rx < nb_pkts) { -+ uint16_t rx_pkt_len; -+ volatile struct rq_enet_desc *rqd_ptr; -+ dma_addr_t dma_addr; -+ struct cq_desc cqd; -+ uint64_t ol_err_flags; -+ -+ /* Check for pkts available */ -+ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) -+ & CQ_DESC_COLOR_MASK; -+ if (color == cq->last_color) -+ break; -+ -+ /* Get the cq descriptor and rq pointer */ -+ cqd = *cqd_ptr; -+ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; -+ -+ /* allocate a new mbuf */ -+ nmb = rte_rxmbuf_alloc(rq->mp); -+ if (nmb == NULL) { -+ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", -+ enic->port_id, (unsigned)rq->index); -+ rte_eth_devices[enic->port_id]. -+ data->rx_mbuf_alloc_failed++; -+ break; -+ } -+ -+ /* Check for FCS or packet errors */ -+ ol_err_flags = enic_cq_rx_to_pkt_err_flags(&cqd); -+ if (ol_err_flags == 0) -+ rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd); -+ else -+ rx_pkt_len = 0; -+ -+ /* Get the mbuf to return and replace with one just allocated */ -+ rxmb = rq->mbuf_ring[rx_id]; -+ rq->mbuf_ring[rx_id] = nmb; -+ -+ /* Increment cqd, rqd, mbuf_table index */ -+ rx_id++; -+ if (unlikely(rx_id == rq->ring.desc_count)) { -+ rx_id = 0; -+ cq->last_color = cq->last_color ? 0 : 1; -+ } -+ -+ /* Prefetch next mbuf & desc while processing current one */ -+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -+ rte_enic_prefetch(cqd_ptr); -+ rte_enic_prefetch(rq->mbuf_ring[rx_id]); -+ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) -+ + rx_id); -+ -+ /* Push descriptor for newly allocated mbuf */ -+ dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off); -+ rqd_ptr->address = rte_cpu_to_le_64(dma_addr); -+ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len); -+ -+ /* Fill in the rest of the mbuf */ -+ rxmb->data_off = RTE_PKTMBUF_HEADROOM; -+ rxmb->nb_segs = 1; -+ rxmb->next = NULL; -+ rxmb->pkt_len = rx_pkt_len; -+ rxmb->data_len = rx_pkt_len; -+ rxmb->port = enic->port_id; -+ rxmb->ol_flags = ol_err_flags; -+ if (!ol_err_flags) -+ enic_cq_rx_to_pkt_flags(&cqd, rxmb); -+ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); -+ -+ /* prefetch mbuf data for caller */ -+ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, -+ RTE_PKTMBUF_HEADROOM)); -+ -+ /* store the mbuf address into the next entry of the array */ -+ rx_pkts[nb_rx++] = rxmb; -+ } -+ -+ nb_hold += nb_rx; -+ cq->to_clean = rx_id; -+ -+ if (nb_hold > rq->rx_free_thresh) { -+ rq->posted_index = enic_ring_add(rq->ring.desc_count, -+ rq->posted_index, nb_hold); -+ nb_hold = 0; -+ rte_mb(); -+ iowrite32(rq->posted_index, &rq->ctrl->posted_index); -+ } -+ -+ rq->rx_nb_hold = nb_hold; -+ -+ return nb_rx; -+} --- -1.9.1 - diff --git a/dpdk/dpdk-2.2.0_patches/0013-enic-add-missing-newline-to-print-statements.patch b/dpdk/dpdk-2.2.0_patches/0013-enic-add-missing-newline-to-print-statements.patch new file mode 100644 index 00000000..e95327d2 --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0013-enic-add-missing-newline-to-print-statements.patch @@ -0,0 +1,42 @@ +commit bba57df3861c644e98c5e8f79e62f6ca5074cb40 +Author: Nelson Escobar +Date: Thu Mar 17 15:48:13 2016 -0700 + + enic: add missing newline to print statements + + Add the missing '\n' character to the end of a few print statements. + + Fixes: fefed3d1e62c ("enic: new driver") + + Signed-off-by: Nelson Escobar + Acked-by: John Daley + +diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c +index 9fff020..e30672c 100644 +--- a/drivers/net/enic/enic_main.c ++++ b/drivers/net/enic/enic_main.c +@@ -342,13 +342,13 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) + unsigned i; + dma_addr_t dma_addr; + +- dev_debug(enic, "queue %u, allocating %u rx queue mbufs", rq->index, ++ dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index, + rq->ring.desc_count); + + for (i = 0; i < rq->ring.desc_count; i++, rqd++) { + mb = rte_rxmbuf_alloc(rq->mp); + if (mb == NULL) { +- dev_err(enic, "RX mbuf alloc failed queue_id=%u", ++ dev_err(enic, "RX mbuf alloc failed queue_id=%u\n", + (unsigned)rq->index); + return -ENOMEM; + } +@@ -388,7 +388,7 @@ enic_alloc_consistent(__rte_unused void *priv, size_t size, + rz = rte_memzone_reserve_aligned((const char *)name, + size, SOCKET_ID_ANY, 0, ENIC_ALIGN); + if (!rz) { +- pr_err("%s : Failed to allocate memory requested for %s", ++ pr_err("%s : Failed to allocate memory requested for %s\n", + __func__, name); + return NULL; + } diff --git a/dpdk/dpdk-2.2.0_patches/0015-enic-fix-crash-when-allocating-too-many-queues.patch b/dpdk/dpdk-2.2.0_patches/0015-enic-fix-crash-when-allocating-too-many-queues.patch new file mode 100644 index 00000000..f8e661fc --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0015-enic-fix-crash-when-allocating-too-many-queues.patch @@ -0,0 +1,46 @@ +commit ddf2da3ecec97a316838f40fe13e217afafc6252 +Author: Nelson Escobar +Date: Thu Mar 17 15:49:58 2016 -0700 + + enic: fix crash when allocating too many queues + + Add checks to make sure we don't try to allocate more tx or rx queues + than we support. + + Fixes: fefed3d1e62c ("enic: new driver") + + Signed-off-by: Nelson Escobar + Reviewed-by: John Daley + +diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c +index bab0f7d..4969476 100644 +--- a/drivers/net/enic/enic_ethdev.c ++++ b/drivers/net/enic/enic_ethdev.c +@@ -174,6 +174,13 @@ static int enicpmd_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, + struct enic *enic = pmd_priv(eth_dev); + + ENICPMD_FUNC_TRACE(); ++ if (queue_idx >= ENIC_WQ_MAX) { ++ dev_err(enic, ++ "Max number of TX queues exceeded. Max is %d\n", ++ ENIC_WQ_MAX); ++ return -EINVAL; ++ } ++ + eth_dev->data->tx_queues[queue_idx] = (void *)&enic->wq[queue_idx]; + + ret = enic_alloc_wq(enic, queue_idx, socket_id, nb_desc); +@@ -262,6 +269,13 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, + struct enic *enic = pmd_priv(eth_dev); + + ENICPMD_FUNC_TRACE(); ++ if (queue_idx >= ENIC_RQ_MAX) { ++ dev_err(enic, ++ "Max number of RX queues exceeded. Max is %d\n", ++ ENIC_RQ_MAX); ++ return -EINVAL; ++ } ++ + eth_dev->data->rx_queues[queue_idx] = (void *)&enic->rq[queue_idx]; + + ret = enic_alloc_rq(enic, queue_idx, socket_id, mp, nb_desc); diff --git a/dpdk/dpdk-2.2.0_patches/0016-enic-fix-mbuf-flags-on-Rx.patch b/dpdk/dpdk-2.2.0_patches/0016-enic-fix-mbuf-flags-on-Rx.patch new file mode 100644 index 00000000..e8f74db5 --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0016-enic-fix-mbuf-flags-on-Rx.patch @@ -0,0 +1,38 @@ +commit 3253bbc79c8a1eddf791d9ec11bcea4a004d258e +Author: John Daley +Date: Thu Mar 17 15:57:05 2016 -0700 + + enic: fix mbuf flags on Rx + + In the receive path, the function to set mbuf ol_flags used the + mbuf packet_type before it was set. + + Fixes: 947d860c821f ("enic: improve Rx performance") + + Signed-off-by: John Daley + +diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c +index 945a60f..59ebaa4 100644 +--- a/drivers/net/enic/enic_rx.c ++++ b/drivers/net/enic/enic_rx.c +@@ -210,7 +210,7 @@ enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) + ciflags = enic_cq_rx_desc_ciflags(cqrd); + bwflags = enic_cq_rx_desc_bwflags(cqrd); + +- ASSERT(mbuf->ol_flags == 0); ++ mbuf->ol_flags = 0; + + /* flags are meaningless if !EOP */ + if (unlikely(!enic_cq_rx_desc_eop(ciflags))) +@@ -340,10 +340,10 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + rxmb->pkt_len = rx_pkt_len; + rxmb->data_len = rx_pkt_len; + rxmb->port = enic->port_id; ++ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); + rxmb->ol_flags = ol_err_flags; + if (!ol_err_flags) + enic_cq_rx_to_pkt_flags(&cqd, rxmb); +- rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); + + /* prefetch mbuf data for caller */ + rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, diff --git a/dpdk/dpdk-2.2.0_patches/0017-enic-fix-error-packets-handling.patch b/dpdk/dpdk-2.2.0_patches/0017-enic-fix-error-packets-handling.patch new file mode 100644 index 00000000..058d5eb4 --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0017-enic-fix-error-packets-handling.patch @@ -0,0 +1,112 @@ +commit 5776c30293bbbdb3e332c868fbccf99b2026fba0 +Author: John Daley +Date: Thu Mar 17 15:57:06 2016 -0700 + + enic: fix error packets handling + + If the packet_error bit in the completion descriptor is set, the + remainder of the descriptor and data are invalid. PKT_RX_MAC_ERR + was set in the mbuf->ol_flags if packet_error was set and used + later to indicate an error packet. But since PKT_RX_MAC_ERR is + defined as 0, mbuf flags and packet types and length were being + misinterpreted. + + Make the function enic_cq_rx_to_pkt_err_flags() return true for error + packets and use the return value instead of mbuf->ol_flags to indicate + error packets. Also remove warning for error packets and rely on + rx_error stats. + + Fixes: 947d860c821f ("enic: improve Rx performance") + + Signed-off-by: John Daley + +diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c +index 59ebaa4..817a891 100644 +--- a/drivers/net/enic/enic_rx.c ++++ b/drivers/net/enic/enic_rx.c +@@ -129,13 +129,6 @@ enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) + return le32_to_cpu(cqrd->rss_hash); + } + +-static inline uint8_t +-enic_cq_rx_desc_fcs_ok(struct cq_enet_rq_desc *cqrd) +-{ +- return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) == +- CQ_ENET_RQ_DESC_FLAGS_FCS_OK); +-} +- + static inline uint16_t + enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) + { +@@ -150,25 +143,21 @@ enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) + CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; + } + +-static inline uint64_t +-enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd) ++static inline uint8_t ++enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out) + { + struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; + uint16_t bwflags; ++ int ret = 0; + uint64_t pkt_err_flags = 0; + + bwflags = enic_cq_rx_desc_bwflags(cqrd); +- +- /* Check for packet error. Can't be more specific than MAC error */ +- if (enic_cq_rx_desc_packet_error(bwflags)) { +- pkt_err_flags |= PKT_RX_MAC_ERR; +- } +- +- /* Check for bad FCS. MAC error isn't quite, but no other choice */ +- if (!enic_cq_rx_desc_fcs_ok(cqrd)) { +- pkt_err_flags |= PKT_RX_MAC_ERR; ++ if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) { ++ pkt_err_flags = PKT_RX_MAC_ERR; ++ ret = 1; + } +- return pkt_err_flags; ++ *pkt_err_flags_out = pkt_err_flags; ++ return ret; + } + + /* +@@ -282,6 +271,7 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + dma_addr_t dma_addr; + struct cq_desc cqd; + uint64_t ol_err_flags; ++ uint8_t packet_error; + + /* Check for pkts available */ + color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) +@@ -303,9 +293,9 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + break; + } + +- /* Check for FCS or packet errors */ +- ol_err_flags = enic_cq_rx_to_pkt_err_flags(&cqd); +- if (ol_err_flags == 0) ++ /* A packet error means descriptor and data are untrusted */ ++ packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); ++ if (!packet_error) + rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd); + else + rx_pkt_len = 0; +@@ -340,10 +330,13 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + rxmb->pkt_len = rx_pkt_len; + rxmb->data_len = rx_pkt_len; + rxmb->port = enic->port_id; +- rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); +- rxmb->ol_flags = ol_err_flags; +- if (!ol_err_flags) ++ if (!packet_error) { ++ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); + enic_cq_rx_to_pkt_flags(&cqd, rxmb); ++ } else { ++ rxmb->packet_type = 0; ++ rxmb->ol_flags = 0; ++ } + + /* prefetch mbuf data for caller */ + rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, diff --git a/dpdk/dpdk-2.2.0_patches/0018-enic-remove-packet-error-conditional.patch b/dpdk/dpdk-2.2.0_patches/0018-enic-remove-packet-error-conditional.patch new file mode 100644 index 00000000..d3084c84 --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0018-enic-remove-packet-error-conditional.patch @@ -0,0 +1,53 @@ +commit 50765c820e98a4434efbc0a58df4b9d78afb7a5f +Author: John Daley +Date: Thu Mar 17 15:57:07 2016 -0700 + + enic: remove packet error conditional + + small cleanup to remove conditional. + + Signed-off-by: John Daley + +diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c +index 817a891..232987a 100644 +--- a/drivers/net/enic/enic_rx.c ++++ b/drivers/net/enic/enic_rx.c +@@ -266,7 +266,6 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + nb_hold = rq->rx_nb_hold; /* mbufs held by software */ + + while (nb_rx < nb_pkts) { +- uint16_t rx_pkt_len; + volatile struct rq_enet_desc *rqd_ptr; + dma_addr_t dma_addr; + struct cq_desc cqd; +@@ -295,10 +294,6 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + + /* A packet error means descriptor and data are untrusted */ + packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); +- if (!packet_error) +- rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd); +- else +- rx_pkt_len = 0; + + /* Get the mbuf to return and replace with one just allocated */ + rxmb = rq->mbuf_ring[rx_id]; +@@ -327,16 +322,17 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + rxmb->data_off = RTE_PKTMBUF_HEADROOM; + rxmb->nb_segs = 1; + rxmb->next = NULL; +- rxmb->pkt_len = rx_pkt_len; +- rxmb->data_len = rx_pkt_len; + rxmb->port = enic->port_id; + if (!packet_error) { ++ rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); + rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); + enic_cq_rx_to_pkt_flags(&cqd, rxmb); + } else { ++ rxmb->pkt_len = 0; + rxmb->packet_type = 0; + rxmb->ol_flags = 0; + } ++ rxmb->data_len = rxmb->pkt_len; + + /* prefetch mbuf data for caller */ + rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, diff --git a/dpdk/dpdk-2.2.0_patches/0019-enic-update-maintainers.patch b/dpdk/dpdk-2.2.0_patches/0019-enic-update-maintainers.patch new file mode 100644 index 00000000..8beae3b2 --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0019-enic-update-maintainers.patch @@ -0,0 +1,36 @@ +commit 57524648749fb7dc1daf3af3213dab472ee432de +Author: John Daley +Date: Fri Mar 18 11:27:07 2016 -0700 + + enic: update maintainers + + Change maintainers for ENIC PMD and fix pointer to enic + documentation in MAINTAINERS. + + Signed-off-by: John Daley + +diff --git a/MAINTAINERS b/MAINTAINERS +index 6ed54dd..e848ffa 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -276,8 +276,9 @@ F: doc/guides/nics/cxgbe.rst + + Cisco enic + M: John Daley +-M: Sujith Sankar ++M: Nelson Escobar + F: drivers/net/enic/ ++F: doc/guides/nics/enic.rst + + Combo szedata2 + M: Matej Vido +diff --git a/doc/guides/nics/enic.rst b/doc/guides/nics/enic.rst +index 2a228fd..e67c3db 100644 +--- a/doc/guides/nics/enic.rst ++++ b/doc/guides/nics/enic.rst +@@ -218,4 +218,4 @@ Any questions or bugs should be reported to DPDK community and to the ENIC PMD + maintainers: + + - John Daley +-- Sujith Sankar ++- Nelson Escobar diff --git a/dpdk/dpdk-2.2.0_patches/0020-enic-fix-Rx-descriptor-limit.patch b/dpdk/dpdk-2.2.0_patches/0020-enic-fix-Rx-descriptor-limit.patch new file mode 100644 index 00000000..f2fb1245 --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0020-enic-fix-Rx-descriptor-limit.patch @@ -0,0 +1,61 @@ +commit 65ca78fdf9a684743bfca278cf1fcfea4603931d +Author: Nelson Escobar +Date: Fri Mar 18 11:33:34 2016 -0700 + + enic: fix Rx descriptor limit + + On initialization, the rq descriptor count was set to the limit + of the vic. When the requested number of rx descriptors was + less than this count, enic_alloc_rq() was incorrectly setting + the count to the lower value. This results in later calls to + enic_alloc_rq() incorrectly using the lower value as the adapter + limit. + + Fixes: fefed3d1e62c ("enic: new driver") + + Signed-off-by: Nelson Escobar + Reviewed-by: John Daley + +diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c +index e30672c..2f79cf0 100644 +--- a/drivers/net/enic/enic_main.c ++++ b/drivers/net/enic/enic_main.c +@@ -524,24 +524,22 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, + "policy. Applying the value in the adapter "\ + "policy (%d).\n", + queue_idx, nb_desc, enic->config.rq_desc_count); +- } else if (nb_desc != enic->config.rq_desc_count) { +- enic->config.rq_desc_count = nb_desc; +- dev_info(enic, +- "RX Queues - effective number of descs:%d\n", +- nb_desc); ++ nb_desc = enic->config.rq_desc_count; + } ++ dev_info(enic, "RX Queues - effective number of descs:%d\n", ++ nb_desc); + } + + /* Allocate queue resources */ + rc = vnic_rq_alloc(enic->vdev, rq, queue_idx, +- enic->config.rq_desc_count, sizeof(struct rq_enet_desc)); ++ nb_desc, sizeof(struct rq_enet_desc)); + if (rc) { + dev_err(enic, "error in allocation of rq\n"); + goto err_exit; + } + + rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, +- socket_id, enic->config.rq_desc_count, ++ socket_id, nb_desc, + sizeof(struct cq_enet_rq_desc)); + if (rc) { + dev_err(enic, "error in allocation of cq for rq\n"); +@@ -550,7 +548,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, + + /* Allocate the mbuf ring */ + rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", +- sizeof(struct rte_mbuf *) * enic->config.rq_desc_count, ++ sizeof(struct rte_mbuf *) * nb_desc, + RTE_CACHE_LINE_SIZE, rq->socket_id); + + if (rq->mbuf_ring != NULL) diff --git a/dpdk/dpdk-2.2.0_patches/0021-enic-fix-TX-hang-when-number-of-packets-gt-queue-size.patch b/dpdk/dpdk-2.2.0_patches/0021-enic-fix-TX-hang-when-number-of-packets-gt-queue-size.patch new file mode 100644 index 00000000..11e54e13 --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0021-enic-fix-TX-hang-when-number-of-packets-gt-queue-size.patch @@ -0,0 +1,82 @@ +commit 67c4432ec364ce21f5059ba0696a9d0f3393356c +Author: John Daley +Date: Thu Mar 24 14:00:39 2016 -0700 + + enic: fix TX hang when number of packets > queue size + + If the nb_pkts parameter to rte_eth_tx_burst() was greater than + the TX descriptor count, a completion was not being requested + from the NIC, so descriptors would not be released back to the + host causing a lock-up. + + Introduce a limit of how many TX descriptors can be used in a single + call to the enic PMD burst TX function before requesting a completion. + + Fixes: d739ba4c6abf ("enic: improve Tx packet rate") + + Signed-off-by: John Daley + +diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c +index 4969476..6bea940 100644 +--- a/drivers/net/enic/enic_ethdev.c ++++ b/drivers/net/enic/enic_ethdev.c +@@ -523,7 +523,7 @@ static void enicpmd_remove_mac_addr(struct rte_eth_dev *eth_dev, __rte_unused ui + static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) + { +- unsigned int index; ++ uint16_t index; + unsigned int frags; + unsigned int pkt_len; + unsigned int seg_len; +@@ -535,6 +535,7 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + unsigned short vlan_id; + unsigned short ol_flags; + uint8_t last_seg, eop; ++ unsigned int host_tx_descs = 0; + + for (index = 0; index < nb_pkts; index++) { + tx_pkt = *tx_pkts++; +@@ -550,6 +551,7 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + return index; + } + } ++ + pkt_len = tx_pkt->pkt_len; + vlan_id = tx_pkt->vlan_tci; + ol_flags = tx_pkt->ol_flags; +@@ -559,9 +561,19 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + next_tx_pkt = tx_pkt->next; + seg_len = tx_pkt->data_len; + inc_len += seg_len; +- eop = (pkt_len == inc_len) || (!next_tx_pkt); +- last_seg = eop && +- (index == ((unsigned int)nb_pkts - 1)); ++ ++ host_tx_descs++; ++ last_seg = 0; ++ eop = 0; ++ if ((pkt_len == inc_len) || !next_tx_pkt) { ++ eop = 1; ++ /* post if last packet in batch or > thresh */ ++ if ((index == (nb_pkts - 1)) || ++ (host_tx_descs > ENIC_TX_POST_THRESH)) { ++ last_seg = 1; ++ host_tx_descs = 0; ++ } ++ } + enic_send_pkt(enic, wq, tx_pkt, (unsigned short)seg_len, + !frags, eop, last_seg, ol_flags, vlan_id); + tx_pkt = next_tx_pkt; +diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h +index 33f2e84..00fa71d 100644 +--- a/drivers/net/enic/enic_res.h ++++ b/drivers/net/enic/enic_res.h +@@ -53,6 +53,7 @@ + + #define ENIC_NON_TSO_MAX_DESC 16 + #define ENIC_DEFAULT_RX_FREE_THRESH 32 ++#define ENIC_TX_POST_THRESH (ENIC_MIN_WQ_DESCS / 2) + + #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) + diff --git a/dpdk/dpdk-2.2.0_patches/0022-bonding-fix-bond-link-detect-in-non-interrupt-mode.patch b/dpdk/dpdk-2.2.0_patches/0022-bonding-fix-bond-link-detect-in-non-interrupt-mode.patch new file mode 100644 index 00000000..9cb56d2d --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0022-bonding-fix-bond-link-detect-in-non-interrupt-mode.patch @@ -0,0 +1,71 @@ +commit 65bcf215aae2e0b9935557e237af054ad0860bad +Author: Nelson Escobar +Date: Tue Mar 22 13:42:08 2016 -0700 + + bonding: fix bond link detect in non-interrupt mode + + Stopping then re-starting a bond interface containing slaves that + used polling for link detection caused the bond to think all slave + links were down and inactive. + + Move the start of the polling for link from slave_add() to + bond_ethdev_start() and in bond_ethdev_stop() make sure we clear + the last_link_status of the slaves. + + Signed-off-by: Nelson Escobar + Signed-off-by: John Daley + +diff --git a/drivers/net/bonding/rte_eth_bond_pmd.c b/drivers/net/bonding/rte_eth_bond_pmd.c +index fb26d35..f0960c6 100644 +--- a/drivers/net/bonding/rte_eth_bond_pmd.c ++++ b/drivers/net/bonding/rte_eth_bond_pmd.c +@@ -1454,18 +1454,11 @@ slave_add(struct bond_dev_private *internals, + slave_details->port_id = slave_eth_dev->data->port_id; + slave_details->last_link_status = 0; + +- /* If slave device doesn't support interrupts then we need to enabled +- * polling to monitor link status */ ++ /* Mark slave devices that don't support interrupts so we can ++ * compensate when we start the bond ++ */ + if (!(slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) { + slave_details->link_status_poll_enabled = 1; +- +- if (!internals->link_status_polling_enabled) { +- internals->link_status_polling_enabled = 1; +- +- rte_eal_alarm_set(internals->link_status_polling_interval_ms * 1000, +- bond_ethdev_slave_link_status_change_monitor, +- (void *)&rte_eth_devices[internals->port_id]); +- } + } + + slave_details->link_status_wait_to_complete = 0; +@@ -1550,6 +1543,18 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) + eth_dev->data->port_id, internals->slaves[i].port_id); + return -1; + } ++ /* We will need to poll for link status if any slave doesn't ++ * support interrupts ++ */ ++ if (internals->slaves[i].link_status_poll_enabled) ++ internals->link_status_polling_enabled = 1; ++ } ++ /* start polling if needed */ ++ if (internals->link_status_polling_enabled) { ++ rte_eal_alarm_set( ++ internals->link_status_polling_interval_ms * 1000, ++ bond_ethdev_slave_link_status_change_monitor, ++ (void *)&rte_eth_devices[internals->port_id]); + } + + if (internals->user_defined_primary_port) +@@ -1622,6 +1627,8 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev) + + internals->active_slave_count = 0; + internals->link_status_polling_enabled = 0; ++ for (i = 0; i < internals->slave_count; i++) ++ internals->slaves[i].last_link_status = 0; + + eth_dev->data->dev_link.link_status = 0; + eth_dev->data->dev_started = 0; -- cgit 1.2.3-korg