aboutsummaryrefslogtreecommitdiffstats
path: root/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch
diff options
context:
space:
mode:
Diffstat (limited to 'dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch')
-rw-r--r--dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch1349
1 files changed, 0 insertions, 1349 deletions
diff --git a/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch b/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch
deleted file mode 100644
index 2aa4840dbc7..00000000000
--- a/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch
+++ /dev/null
@@ -1,1349 +0,0 @@
-From 057358356e7d05f07ab2df37c12b1cce37a3cca9 Mon Sep 17 00:00:00 2001
-From: John Daley <johndale@cisco.com>
-Date: Fri, 4 Mar 2016 13:09:00 -0800
-Subject: [PATCH 11/22] enic: improve Rx performance
-
- This is a wholesale replacement of the Enic PMD receive path in order
- to improve performance and code clarity. The changes are:
- - Simplify and reduce code path length of receive function.
- - Put most of the fast-path receive functions in one file.
- - Reduce the number of posted_index updates (pay attention to
- rx_free_thresh)
- - Remove the unneeded container structure around the RQ mbuf ring
- - Prefetch next Mbuf and descriptors while processing the current one
- - Use a lookup table for converting CQ flags to mbuf flags.
-
- Signed-off-by: John Daley <johndale@cisco.com>
----
- drivers/net/enic/Makefile | 1 +
- drivers/net/enic/base/vnic_rq.c | 99 ++---------
- drivers/net/enic/base/vnic_rq.h | 147 +---------------
- drivers/net/enic/enic.h | 16 +-
- drivers/net/enic/enic_ethdev.c | 27 ++-
- drivers/net/enic/enic_main.c | 321 ++++++++++------------------------
- drivers/net/enic/enic_res.h | 16 +-
- drivers/net/enic/enic_rx.c | 370 ++++++++++++++++++++++++++++++++++++++++
- 8 files changed, 511 insertions(+), 486 deletions(-)
- create mode 100644 drivers/net/enic/enic_rx.c
-
-diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile
-index f0ee093..f316274 100644
---- a/drivers/net/enic/Makefile
-+++ b/drivers/net/enic/Makefile
-@@ -53,6 +53,7 @@ VPATH += $(SRCDIR)/src
- #
- SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c
- SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c
-+SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c
- SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c
- SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c
- SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c
-diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c
-index 1441604..cb62c5e 100644
---- a/drivers/net/enic/base/vnic_rq.c
-+++ b/drivers/net/enic/base/vnic_rq.c
-@@ -35,77 +35,21 @@
- #include "vnic_dev.h"
- #include "vnic_rq.h"
-
--static int vnic_rq_alloc_bufs(struct vnic_rq *rq)
--{
-- struct vnic_rq_buf *buf;
-- unsigned int i, j, count = rq->ring.desc_count;
-- unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count);
--
-- for (i = 0; i < blks; i++) {
-- rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ(count), GFP_ATOMIC);
-- if (!rq->bufs[i])
-- return -ENOMEM;
-- }
--
-- for (i = 0; i < blks; i++) {
-- buf = rq->bufs[i];
-- for (j = 0; j < VNIC_RQ_BUF_BLK_ENTRIES(count); j++) {
-- buf->index = i * VNIC_RQ_BUF_BLK_ENTRIES(count) + j;
-- buf->desc = (u8 *)rq->ring.descs +
-- rq->ring.desc_size * buf->index;
-- if (buf->index + 1 == count) {
-- buf->next = rq->bufs[0];
-- break;
-- } else if (j + 1 == VNIC_RQ_BUF_BLK_ENTRIES(count)) {
-- buf->next = rq->bufs[i + 1];
-- } else {
-- buf->next = buf + 1;
-- buf++;
-- }
-- }
-- }
--
-- rq->to_use = rq->to_clean = rq->bufs[0];
--
-- return 0;
--}
--
--int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count,
-- unsigned int desc_size)
--{
-- int mem_size = 0;
--
-- mem_size += vnic_dev_desc_ring_size(&rq->ring, desc_count, desc_size);
--
-- mem_size += VNIC_RQ_BUF_BLKS_NEEDED(rq->ring.desc_count) *
-- VNIC_RQ_BUF_BLK_SZ(rq->ring.desc_count);
--
-- return mem_size;
--}
--
- void vnic_rq_free(struct vnic_rq *rq)
- {
- struct vnic_dev *vdev;
-- unsigned int i;
-
- vdev = rq->vdev;
-
- vnic_dev_free_desc_ring(vdev, &rq->ring);
-
-- for (i = 0; i < VNIC_RQ_BUF_BLKS_MAX; i++) {
-- if (rq->bufs[i]) {
-- kfree(rq->bufs[i]);
-- rq->bufs[i] = NULL;
-- }
-- }
--
- rq->ctrl = NULL;
- }
-
- int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
- unsigned int desc_count, unsigned int desc_size)
- {
-- int err;
-+ int rc;
- char res_name[NAME_MAX];
- static int instance;
-
-@@ -121,18 +65,9 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
- vnic_rq_disable(rq);
-
- snprintf(res_name, sizeof(res_name), "%d-rq-%d", instance++, index);
-- err = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size,
-+ rc = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size,
- rq->socket_id, res_name);
-- if (err)
-- return err;
--
-- err = vnic_rq_alloc_bufs(rq);
-- if (err) {
-- vnic_rq_free(rq);
-- return err;
-- }
--
-- return 0;
-+ return rc;
- }
-
- void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index,
-@@ -154,9 +89,6 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index,
- iowrite32(fetch_index, &rq->ctrl->fetch_index);
- iowrite32(posted_index, &rq->ctrl->posted_index);
-
-- rq->to_use = rq->to_clean =
-- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)]
-- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)];
- }
-
- void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
-@@ -176,6 +108,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
- fetch_index, fetch_index,
- error_interrupt_enable,
- error_interrupt_offset);
-+ rq->rxst_idx = 0;
-+ rq->tot_pkts = 0;
- }
-
- void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error)
-@@ -212,21 +146,20 @@ int vnic_rq_disable(struct vnic_rq *rq)
- }
-
- void vnic_rq_clean(struct vnic_rq *rq,
-- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf))
-+ void (*buf_clean)(struct rte_mbuf **buf))
- {
-- struct vnic_rq_buf *buf;
-- u32 fetch_index;
-+ struct rte_mbuf **buf;
-+ u32 fetch_index, i;
- unsigned int count = rq->ring.desc_count;
-
-- buf = rq->to_clean;
--
-- while (vnic_rq_desc_used(rq) > 0) {
-+ buf = &rq->mbuf_ring[0];
-
-- (*buf_clean)(rq, buf);
--
-- buf = rq->to_clean = buf->next;
-- rq->ring.desc_avail++;
-+ for (i = 0; i < count; i++) {
-+ (*buf_clean)(buf);
-+ buf++;
- }
-+ rq->ring.desc_avail = count - 1;
-+ rq->rx_nb_hold = 0;
-
- /* Use current fetch_index as the ring starting point */
- fetch_index = ioread32(&rq->ctrl->fetch_index);
-@@ -235,9 +168,7 @@ void vnic_rq_clean(struct vnic_rq *rq,
- /* Hardware surprise removal: reset fetch_index */
- fetch_index = 0;
- }
-- rq->to_use = rq->to_clean =
-- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)]
-- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)];
-+
- iowrite32(fetch_index, &rq->ctrl->posted_index);
-
- vnic_dev_clear_desc_ring(&rq->ring);
-diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h
-index 0f5c3c1..e083ccc 100644
---- a/drivers/net/enic/base/vnic_rq.h
-+++ b/drivers/net/enic/base/vnic_rq.h
-@@ -66,42 +66,22 @@ struct vnic_rq_ctrl {
- u32 pad10;
- };
-
--/* Break the vnic_rq_buf allocations into blocks of 32/64 entries */
--#define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32
--#define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64
--#define VNIC_RQ_BUF_BLK_ENTRIES(entries) \
-- ((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \
-- VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES))
--#define VNIC_RQ_BUF_BLK_SZ(entries) \
-- (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf))
--#define VNIC_RQ_BUF_BLKS_NEEDED(entries) \
-- DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries))
--#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096)
--
--struct vnic_rq_buf {
-- struct vnic_rq_buf *next;
-- dma_addr_t dma_addr;
-- void *os_buf;
-- unsigned int os_buf_index;
-- unsigned int len;
-- unsigned int index;
-- void *desc;
-- uint64_t wr_id;
--};
--
- struct vnic_rq {
- unsigned int index;
-+ unsigned int posted_index;
- struct vnic_dev *vdev;
-- struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */
-+ struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */
- struct vnic_dev_ring ring;
-- struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX];
-- struct vnic_rq_buf *to_use;
-- struct vnic_rq_buf *to_clean;
-+ struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */
-+ unsigned int mbuf_next_idx; /* next mb to consume */
- void *os_buf_head;
- unsigned int pkts_outstanding;
--
-+ uint16_t rx_nb_hold;
-+ uint16_t rx_free_thresh;
- unsigned int socket_id;
- struct rte_mempool *mp;
-+ uint16_t rxst_idx;
-+ uint32_t tot_pkts;
- };
-
- static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq)
-@@ -116,119 +96,13 @@ static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq)
- return rq->ring.desc_count - rq->ring.desc_avail - 1;
- }
-
--static inline void *vnic_rq_next_desc(struct vnic_rq *rq)
--{
-- return rq->to_use->desc;
--}
--
--static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq)
--{
-- return rq->to_use->index;
--}
--
--static inline void vnic_rq_post(struct vnic_rq *rq,
-- void *os_buf, unsigned int os_buf_index,
-- dma_addr_t dma_addr, unsigned int len,
-- uint64_t wrid)
--{
-- struct vnic_rq_buf *buf = rq->to_use;
--
-- buf->os_buf = os_buf;
-- buf->os_buf_index = os_buf_index;
-- buf->dma_addr = dma_addr;
-- buf->len = len;
-- buf->wr_id = wrid;
--
-- buf = buf->next;
-- rq->to_use = buf;
-- rq->ring.desc_avail--;
--
-- /* Move the posted_index every nth descriptor
-- */
--
--#ifndef VNIC_RQ_RETURN_RATE
--#define VNIC_RQ_RETURN_RATE 0xf /* keep 2^n - 1 */
--#endif
--
-- if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) {
-- /* Adding write memory barrier prevents compiler and/or CPU
-- * reordering, thus avoiding descriptor posting before
-- * descriptor is initialized. Otherwise, hardware can read
-- * stale descriptor fields.
-- */
-- wmb();
-- iowrite32(buf->index, &rq->ctrl->posted_index);
-- }
--}
--
--static inline void vnic_rq_post_commit(struct vnic_rq *rq,
-- void *os_buf, unsigned int os_buf_index,
-- dma_addr_t dma_addr, unsigned int len)
--{
-- struct vnic_rq_buf *buf = rq->to_use;
--
-- buf->os_buf = os_buf;
-- buf->os_buf_index = os_buf_index;
-- buf->dma_addr = dma_addr;
-- buf->len = len;
--
-- buf = buf->next;
-- rq->to_use = buf;
-- rq->ring.desc_avail--;
--
-- /* Move the posted_index every descriptor
-- */
--
-- /* Adding write memory barrier prevents compiler and/or CPU
-- * reordering, thus avoiding descriptor posting before
-- * descriptor is initialized. Otherwise, hardware can read
-- * stale descriptor fields.
-- */
-- wmb();
-- iowrite32(buf->index, &rq->ctrl->posted_index);
--}
-
--static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count)
--{
-- rq->ring.desc_avail += count;
--}
-
- enum desc_return_options {
- VNIC_RQ_RETURN_DESC,
- VNIC_RQ_DEFER_RETURN_DESC,
- };
-
--static inline int vnic_rq_service(struct vnic_rq *rq,
-- struct cq_desc *cq_desc, u16 completed_index,
-- int desc_return, int (*buf_service)(struct vnic_rq *rq,
-- struct cq_desc *cq_desc, struct vnic_rq_buf *buf,
-- int skipped, void *opaque), void *opaque)
--{
-- struct vnic_rq_buf *buf;
-- int skipped;
-- int eop = 0;
--
-- buf = rq->to_clean;
-- while (1) {
--
-- skipped = (buf->index != completed_index);
--
-- if ((*buf_service)(rq, cq_desc, buf, skipped, opaque))
-- eop++;
--
-- if (desc_return == VNIC_RQ_RETURN_DESC)
-- rq->ring.desc_avail++;
--
-- rq->to_clean = buf->next;
--
-- if (!skipped)
-- break;
--
-- buf = rq->to_clean;
-- }
-- return eop;
--}
--
- static inline int vnic_rq_fill(struct vnic_rq *rq,
- int (*buf_fill)(struct vnic_rq *rq))
- {
-@@ -274,8 +148,5 @@ unsigned int vnic_rq_error_status(struct vnic_rq *rq);
- void vnic_rq_enable(struct vnic_rq *rq);
- int vnic_rq_disable(struct vnic_rq *rq);
- void vnic_rq_clean(struct vnic_rq *rq,
-- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf));
--int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count,
-- unsigned int desc_size);
--
-+ void (*buf_clean)(struct rte_mbuf **buf));
- #endif /* _VNIC_RQ_H_ */
-diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
-index 9e78305..8c914f5 100644
---- a/drivers/net/enic/enic.h
-+++ b/drivers/net/enic/enic.h
-@@ -45,6 +45,7 @@
- #include "vnic_nic.h"
- #include "vnic_rss.h"
- #include "enic_res.h"
-+#include "cq_enet_desc.h"
-
- #define DRV_NAME "enic_pmd"
- #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Poll-mode Driver"
-@@ -154,6 +155,16 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev)
- return (struct enic *)eth_dev->data->dev_private;
- }
-
-+#define RTE_LIBRTE_ENIC_ASSERT_ENABLE
-+#ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE
-+#define ASSERT(x) do { \
-+ if (!(x)) \
-+ rte_panic("ENIC: x"); \
-+} while (0)
-+#else
-+#define ASSERT(x)
-+#endif
-+
- extern void enic_fdir_stats_get(struct enic *enic,
- struct rte_eth_fdir_stats *stats);
- extern int enic_fdir_add_fltr(struct enic *enic,
-@@ -193,9 +204,10 @@ extern void enic_send_pkt(struct enic *enic, struct vnic_wq *wq,
- uint16_t ol_flags, uint16_t vlan_tag);
-
- extern void enic_post_wq_index(struct vnic_wq *wq);
--extern int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts,
-- unsigned int budget, unsigned int *work_done);
- extern int enic_probe(struct enic *enic);
- extern int enic_clsf_init(struct enic *enic);
- extern void enic_clsf_destroy(struct enic *enic);
-+uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
-+ uint16_t nb_pkts);
-+
- #endif /* _ENIC_H_ */
-diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c
-index 2a88043..6f2ada5 100644
---- a/drivers/net/enic/enic_ethdev.c
-+++ b/drivers/net/enic/enic_ethdev.c
-@@ -255,7 +255,7 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
- uint16_t queue_idx,
- uint16_t nb_desc,
- unsigned int socket_id,
-- __rte_unused const struct rte_eth_rxconf *rx_conf,
-+ const struct rte_eth_rxconf *rx_conf,
- struct rte_mempool *mp)
- {
- int ret;
-@@ -270,6 +270,10 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
- return ret;
- }
-
-+ enic->rq[queue_idx].rx_free_thresh = rx_conf->rx_free_thresh;
-+ dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
-+ enic->rq[queue_idx].rx_free_thresh);
-+
- return enicpmd_dev_setup_intr(enic);
- }
-
-@@ -429,6 +433,9 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev,
- DEV_TX_OFFLOAD_IPV4_CKSUM |
- DEV_TX_OFFLOAD_UDP_CKSUM |
- DEV_TX_OFFLOAD_TCP_CKSUM;
-+ device_info->default_rxconf = (struct rte_eth_rxconf) {
-+ .rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH
-+ };
- }
-
- static void enicpmd_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
-@@ -538,18 +545,6 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
- return index;
- }
-
--static uint16_t enicpmd_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
-- uint16_t nb_pkts)
--{
-- struct vnic_rq *rq = (struct vnic_rq *)rx_queue;
-- unsigned int work_done;
--
-- if (enic_poll(rq, rx_pkts, (unsigned int)nb_pkts, &work_done))
-- dev_err(enic, "error in enicpmd poll\n");
--
-- return work_done;
--}
--
- static const struct eth_dev_ops enicpmd_eth_dev_ops = {
- .dev_configure = enicpmd_dev_configure,
- .dev_start = enicpmd_dev_start,
-@@ -606,7 +601,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev)
- enic->port_id = eth_dev->data->port_id;
- enic->rte_dev = eth_dev;
- eth_dev->dev_ops = &enicpmd_eth_dev_ops;
-- eth_dev->rx_pkt_burst = &enicpmd_recv_pkts;
-+ eth_dev->rx_pkt_burst = &enic_recv_pkts;
- eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts;
-
- pdev = eth_dev->pci_dev;
-@@ -635,8 +630,8 @@ static struct eth_driver rte_enic_pmd = {
- * Register as the [Poll Mode] Driver of Cisco ENIC device.
- */
- static int
--rte_enic_pmd_init(const char *name __rte_unused,
-- const char *params __rte_unused)
-+rte_enic_pmd_init(__rte_unused const char *name,
-+ __rte_unused const char *params)
- {
- ENICPMD_FUNC_TRACE();
-
-diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
-index f818c32..9fff020 100644
---- a/drivers/net/enic/enic_main.c
-+++ b/drivers/net/enic/enic_main.c
-@@ -60,6 +60,17 @@
- #include "vnic_nic.h"
- #include "enic_vnic_wq.h"
-
-+static inline struct rte_mbuf *
-+rte_rxmbuf_alloc(struct rte_mempool *mp)
-+{
-+ struct rte_mbuf *m;
-+
-+ m = __rte_mbuf_raw_alloc(mp);
-+ __rte_mbuf_sanity_check_raw(m, 0);
-+ return m;
-+}
-+
-+
- static inline int enic_is_sriov_vf(struct enic *enic)
- {
- return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
-@@ -80,16 +91,25 @@ static int is_eth_addr_valid(uint8_t *addr)
- return !is_mcast_addr(addr) && !is_zero_addr(addr);
- }
-
--static inline struct rte_mbuf *
--enic_rxmbuf_alloc(struct rte_mempool *mp)
-+static void
-+enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq)
- {
-- struct rte_mbuf *m;
-+ uint16_t i;
-
-- m = __rte_mbuf_raw_alloc(mp);
-- __rte_mbuf_sanity_check_raw(m, 0);
-- return m;
-+ if (!rq || !rq->mbuf_ring) {
-+ dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
-+ return;
-+ }
-+
-+ for (i = 0; i < enic->config.rq_desc_count; i++) {
-+ if (rq->mbuf_ring[i]) {
-+ rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
-+ rq->mbuf_ring[i] = NULL;
-+ }
-+ }
- }
-
-+
- void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size)
- {
- vnic_set_hdr_split_size(enic->vdev, split_hdr_size);
-@@ -262,13 +282,13 @@ void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
- }
-
- static void
--enic_free_rq_buf(__rte_unused struct vnic_rq *rq, struct vnic_rq_buf *buf)
-+enic_free_rq_buf(struct rte_mbuf **mbuf)
- {
-- if (!buf->os_buf)
-+ if (*mbuf == NULL)
- return;
-
-- rte_pktmbuf_free((struct rte_mbuf *)buf->os_buf);
-- buf->os_buf = NULL;
-+ rte_pktmbuf_free(*mbuf);
-+ mbuf = NULL;
- }
-
- void enic_init_vnic_resources(struct enic *enic)
-@@ -314,221 +334,47 @@ void enic_init_vnic_resources(struct enic *enic)
- }
-
-
--static int enic_rq_alloc_buf(struct vnic_rq *rq)
-+static int
-+enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
- {
-- struct enic *enic = vnic_dev_priv(rq->vdev);
-+ struct rte_mbuf *mb;
-+ struct rq_enet_desc *rqd = rq->ring.descs;
-+ unsigned i;
- dma_addr_t dma_addr;
-- struct rq_enet_desc *desc = vnic_rq_next_desc(rq);
-- uint8_t type = RQ_ENET_TYPE_ONLY_SOP;
-- u16 split_hdr_size = vnic_get_hdr_split_size(enic->vdev);
-- struct rte_mbuf *mbuf = enic_rxmbuf_alloc(rq->mp);
-- struct rte_mbuf *hdr_mbuf = NULL;
--
-- if (!mbuf) {
-- dev_err(enic, "mbuf alloc in enic_rq_alloc_buf failed\n");
-- return -1;
-- }
--
-- if (unlikely(split_hdr_size)) {
-- if (vnic_rq_desc_avail(rq) < 2) {
-- rte_mempool_put(mbuf->pool, mbuf);
-- return -1;
-- }
-- hdr_mbuf = enic_rxmbuf_alloc(rq->mp);
-- if (!hdr_mbuf) {
-- rte_mempool_put(mbuf->pool, mbuf);
-- dev_err(enic,
-- "hdr_mbuf alloc in enic_rq_alloc_buf failed\n");
-- return -1;
-- }
--
-- hdr_mbuf->data_off = RTE_PKTMBUF_HEADROOM;
--
-- hdr_mbuf->nb_segs = 2;
-- hdr_mbuf->port = enic->port_id;
-- hdr_mbuf->next = mbuf;
--
-- dma_addr = (dma_addr_t)
-- (hdr_mbuf->buf_physaddr + hdr_mbuf->data_off);
--
-- rq_enet_desc_enc(desc, dma_addr, type, split_hdr_size);
-
-- vnic_rq_post(rq, (void *)hdr_mbuf, 0 /*os_buf_index*/, dma_addr,
-- (unsigned int)split_hdr_size, 0 /*wrid*/);
-+ dev_debug(enic, "queue %u, allocating %u rx queue mbufs", rq->index,
-+ rq->ring.desc_count);
-
-- desc = vnic_rq_next_desc(rq);
-- type = RQ_ENET_TYPE_NOT_SOP;
-- } else {
-- mbuf->nb_segs = 1;
-- mbuf->port = enic->port_id;
-- }
--
-- mbuf->data_off = RTE_PKTMBUF_HEADROOM;
-- mbuf->next = NULL;
--
-- dma_addr = (dma_addr_t)
-- (mbuf->buf_physaddr + mbuf->data_off);
--
-- rq_enet_desc_enc(desc, dma_addr, type, mbuf->buf_len);
--
-- vnic_rq_post(rq, (void *)mbuf, 0 /*os_buf_index*/, dma_addr,
-- (unsigned int)mbuf->buf_len, 0 /*wrid*/);
--
-- return 0;
--}
--
--static int enic_rq_indicate_buf(struct vnic_rq *rq,
-- struct cq_desc *cq_desc, struct vnic_rq_buf *buf,
-- int skipped, void *opaque)
--{
-- struct enic *enic = vnic_dev_priv(rq->vdev);
-- struct rte_mbuf **rx_pkt_bucket = (struct rte_mbuf **)opaque;
-- struct rte_mbuf *rx_pkt = NULL;
-- struct rte_mbuf *hdr_rx_pkt = NULL;
--
-- u8 type, color, eop, sop, ingress_port, vlan_stripped;
-- u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof;
-- u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok;
-- u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc;
-- u8 packet_error;
-- u16 q_number, completed_index, bytes_written, vlan_tci, checksum;
-- u32 rss_hash;
--
-- cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc,
-- &type, &color, &q_number, &completed_index,
-- &ingress_port, &fcoe, &eop, &sop, &rss_type,
-- &csum_not_calc, &rss_hash, &bytes_written,
-- &packet_error, &vlan_stripped, &vlan_tci, &checksum,
-- &fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error,
-- &fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp,
-- &ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment,
-- &fcs_ok);
--
-- rx_pkt = (struct rte_mbuf *)buf->os_buf;
-- buf->os_buf = NULL;
--
-- if (unlikely(packet_error)) {
-- dev_err(enic, "packet error\n");
-- rx_pkt->data_len = 0;
-- return 0;
-- }
--
-- if (unlikely(skipped)) {
-- rx_pkt->data_len = 0;
-- return 0;
-- }
--
-- if (likely(!vnic_get_hdr_split_size(enic->vdev))) {
-- /* No header split configured */
-- *rx_pkt_bucket = rx_pkt;
-- rx_pkt->pkt_len = bytes_written;
--
-- if (ipv4) {
-- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4;
-- if (!csum_not_calc) {
-- if (unlikely(!ipv4_csum_ok))
-- rx_pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD;
--
-- if ((tcp || udp) && (!tcp_udp_csum_ok))
-- rx_pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD;
-- }
-- } else if (ipv6)
-- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6;
-- } else {
-- /* Header split */
-- if (sop && !eop) {
-- /* This piece is header */
-- *rx_pkt_bucket = rx_pkt;
-- rx_pkt->pkt_len = bytes_written;
-- } else {
-- if (sop && eop) {
-- /* The packet is smaller than split_hdr_size */
-- *rx_pkt_bucket = rx_pkt;
-- rx_pkt->pkt_len = bytes_written;
-- if (ipv4) {
-- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4;
-- if (!csum_not_calc) {
-- if (unlikely(!ipv4_csum_ok))
-- rx_pkt->ol_flags |=
-- PKT_RX_IP_CKSUM_BAD;
--
-- if ((tcp || udp) &&
-- (!tcp_udp_csum_ok))
-- rx_pkt->ol_flags |=
-- PKT_RX_L4_CKSUM_BAD;
-- }
-- } else if (ipv6)
-- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6;
-- } else {
-- /* Payload */
-- hdr_rx_pkt = *rx_pkt_bucket;
-- hdr_rx_pkt->pkt_len += bytes_written;
-- if (ipv4) {
-- hdr_rx_pkt->packet_type =
-- RTE_PTYPE_L3_IPV4;
-- if (!csum_not_calc) {
-- if (unlikely(!ipv4_csum_ok))
-- hdr_rx_pkt->ol_flags |=
-- PKT_RX_IP_CKSUM_BAD;
--
-- if ((tcp || udp) &&
-- (!tcp_udp_csum_ok))
-- hdr_rx_pkt->ol_flags |=
-- PKT_RX_L4_CKSUM_BAD;
-- }
-- } else if (ipv6)
-- hdr_rx_pkt->packet_type =
-- RTE_PTYPE_L3_IPV6;
-- }
-+ for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
-+ mb = rte_rxmbuf_alloc(rq->mp);
-+ if (mb == NULL) {
-+ dev_err(enic, "RX mbuf alloc failed queue_id=%u",
-+ (unsigned)rq->index);
-+ return -ENOMEM;
- }
-- }
-
-- rx_pkt->data_len = bytes_written;
-+ dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off);
-
-- if (rss_hash) {
-- rx_pkt->ol_flags |= PKT_RX_RSS_HASH;
-- rx_pkt->hash.rss = rss_hash;
-+ rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP,
-+ mb->buf_len);
-+ rq->mbuf_ring[i] = mb;
- }
-
-- if (vlan_tci) {
-- rx_pkt->ol_flags |= PKT_RX_VLAN_PKT;
-- rx_pkt->vlan_tci = vlan_tci;
-- }
-+ /* make sure all prior writes are complete before doing the PIO write */
-+ rte_rmb();
-
-- return eop;
--}
-+ /* Post all but the last 2 cache lines' worth of descriptors */
-+ rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE
-+ / sizeof(struct rq_enet_desc));
-+ rq->rx_nb_hold = 0;
-
--static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
-- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque)
--{
-- struct enic *enic = vnic_dev_priv(vdev);
--
-- return vnic_rq_service(&enic->rq[q_number], cq_desc,
-- completed_index, VNIC_RQ_RETURN_DESC,
-- enic_rq_indicate_buf, opaque);
--
--}
-+ dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
-+ enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
-+ iowrite32(rq->posted_index, &rq->ctrl->posted_index);
-+ rte_rmb();
-
--int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts,
-- unsigned int budget, unsigned int *work_done)
--{
-- struct enic *enic = vnic_dev_priv(rq->vdev);
-- unsigned int cq = enic_cq_rq(enic, rq->index);
-- int err = 0;
--
-- *work_done = vnic_cq_service(&enic->cq[cq],
-- budget, enic_rq_service, (void *)rx_pkts);
--
-- if (*work_done) {
-- vnic_rq_fill(rq, enic_rq_alloc_buf);
-+ return 0;
-
-- /* Need at least one buffer on ring to get going */
-- if (vnic_rq_desc_used(rq) == 0) {
-- dev_err(enic, "Unable to alloc receive buffers\n");
-- err = -1;
-- }
-- }
-- return err;
- }
-
- static void *
-@@ -576,6 +422,7 @@ enic_intr_handler(__rte_unused struct rte_intr_handle *handle,
- int enic_enable(struct enic *enic)
- {
- unsigned int index;
-+ int err;
- struct rte_eth_dev *eth_dev = enic->rte_dev;
-
- eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
-@@ -586,15 +433,11 @@ int enic_enable(struct enic *enic)
- dev_warning(enic, "Init of hash table for clsf failed."\
- "Flow director feature will not work\n");
-
-- /* Fill RQ bufs */
- for (index = 0; index < enic->rq_count; index++) {
-- vnic_rq_fill(&enic->rq[index], enic_rq_alloc_buf);
--
-- /* Need at least one buffer on ring to get going
-- */
-- if (vnic_rq_desc_used(&enic->rq[index]) == 0) {
-- dev_err(enic, "Unable to alloc receive buffers\n");
-- return -1;
-+ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]);
-+ if (err) {
-+ dev_err(enic, "Failed to alloc RX queue mbufs\n");
-+ return err;
- }
- }
-
-@@ -636,6 +479,9 @@ void enic_free_rq(void *rxq)
- struct vnic_rq *rq = (struct vnic_rq *)rxq;
- struct enic *enic = vnic_dev_priv(rq->vdev);
-
-+ enic_rxmbuf_queue_release(enic, rq);
-+ rte_free(rq->mbuf_ring);
-+ rq->mbuf_ring = NULL;
- vnic_rq_free(rq);
- vnic_cq_free(&enic->cq[rq->index]);
- }
-@@ -664,7 +510,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
- unsigned int socket_id, struct rte_mempool *mp,
- uint16_t nb_desc)
- {
-- int err;
-+ int rc;
- struct vnic_rq *rq = &enic->rq[queue_idx];
-
- rq->socket_id = socket_id;
-@@ -687,23 +533,35 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
- }
-
- /* Allocate queue resources */
-- err = vnic_rq_alloc(enic->vdev, &enic->rq[queue_idx], queue_idx,
-- enic->config.rq_desc_count,
-- sizeof(struct rq_enet_desc));
-- if (err) {
-+ rc = vnic_rq_alloc(enic->vdev, rq, queue_idx,
-+ enic->config.rq_desc_count, sizeof(struct rq_enet_desc));
-+ if (rc) {
- dev_err(enic, "error in allocation of rq\n");
-- return err;
-+ goto err_exit;
- }
-
-- err = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
-+ rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
- socket_id, enic->config.rq_desc_count,
- sizeof(struct cq_enet_rq_desc));
-- if (err) {
-- vnic_rq_free(rq);
-+ if (rc) {
- dev_err(enic, "error in allocation of cq for rq\n");
-+ goto err_free_rq_exit;
- }
-
-- return err;
-+ /* Allocate the mbuf ring */
-+ rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring",
-+ sizeof(struct rte_mbuf *) * enic->config.rq_desc_count,
-+ RTE_CACHE_LINE_SIZE, rq->socket_id);
-+
-+ if (rq->mbuf_ring != NULL)
-+ return 0;
-+
-+ /* cleanup on error */
-+ vnic_cq_free(&enic->cq[queue_idx]);
-+err_free_rq_exit:
-+ vnic_rq_free(rq);
-+err_exit:
-+ return -ENOMEM;
- }
-
- void enic_free_wq(void *txq)
-@@ -790,6 +648,7 @@ int enic_disable(struct enic *enic)
-
- for (i = 0; i < enic->wq_count; i++)
- vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
-+
- for (i = 0; i < enic->rq_count; i++)
- vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
- for (i = 0; i < enic->cq_count; i++)
-@@ -1074,7 +933,7 @@ int enic_probe(struct enic *enic)
-
- /* Set ingress vlan rewrite mode before vnic initialization */
- err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
-- IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN);
-+ IG_VLAN_REWRITE_MODE_PASS_THRU);
- if (err) {
- dev_err(enic,
- "Failed to set ingress vlan rewrite mode, aborting.\n");
-diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h
-index 49f7e22..33f2e84 100644
---- a/drivers/net/enic/enic_res.h
-+++ b/drivers/net/enic/enic_res.h
-@@ -52,6 +52,7 @@
- #define ENIC_UNICAST_PERFECT_FILTERS 32
-
- #define ENIC_NON_TSO_MAX_DESC 16
-+#define ENIC_DEFAULT_RX_FREE_THRESH 32
-
- #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0)
-
-@@ -133,21 +134,6 @@ static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq,
- WQ_ENET_OFFLOAD_MODE_TSO,
- eop, 1 /* SOP */, eop, loopback);
- }
--static inline void enic_queue_rq_desc(struct vnic_rq *rq,
-- void *os_buf, unsigned int os_buf_index,
-- dma_addr_t dma_addr, unsigned int len)
--{
-- struct rq_enet_desc *desc = vnic_rq_next_desc(rq);
-- u64 wrid = 0;
-- u8 type = os_buf_index ?
-- RQ_ENET_TYPE_NOT_SOP : RQ_ENET_TYPE_ONLY_SOP;
--
-- rq_enet_desc_enc(desc,
-- (u64)dma_addr | VNIC_PADDR_TARGET,
-- type, (u16)len);
--
-- vnic_rq_post(rq, os_buf, os_buf_index, dma_addr, len, wrid);
--}
-
- struct enic;
-
-diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c
-new file mode 100644
-index 0000000..945a60f
---- /dev/null
-+++ b/drivers/net/enic/enic_rx.c
-@@ -0,0 +1,370 @@
-+/*
-+ * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved.
-+ * Copyright 2007 Nuova Systems, Inc. All rights reserved.
-+ *
-+ * Copyright (c) 2014, Cisco Systems, Inc.
-+ * All rights reserved.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions
-+ * are met:
-+ *
-+ * 1. Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ *
-+ * 2. Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in
-+ * the documentation and/or other materials provided with the
-+ * distribution.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-+ * POSSIBILITY OF SUCH DAMAGE.
-+ *
-+ */
-+
-+#include <rte_mbuf.h>
-+#include <rte_ethdev.h>
-+#include <rte_prefetch.h>
-+
-+#include "enic_compat.h"
-+#include "rq_enet_desc.h"
-+#include "enic.h"
-+
-+#define RTE_PMD_USE_PREFETCH
-+
-+#ifdef RTE_PMD_USE_PREFETCH
-+/*
-+ * Prefetch a cache line into all cache levels.
-+ */
-+#define rte_enic_prefetch(p) rte_prefetch0(p)
-+#else
-+#define rte_enic_prefetch(p) do {} while (0)
-+#endif
-+
-+#ifdef RTE_PMD_PACKET_PREFETCH
-+#define rte_packet_prefetch(p) rte_prefetch1(p)
-+#else
-+#define rte_packet_prefetch(p) do {} while (0)
-+#endif
-+
-+static inline struct rte_mbuf *
-+rte_rxmbuf_alloc(struct rte_mempool *mp)
-+{
-+ struct rte_mbuf *m;
-+
-+ m = __rte_mbuf_raw_alloc(mp);
-+ __rte_mbuf_sanity_check_raw(m, 0);
-+ return m;
-+}
-+
-+static inline uint16_t
-+enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
-+{
-+ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
-+}
-+
-+static inline uint16_t
-+enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
-+{
-+ return(le16_to_cpu(crd->bytes_written_flags) &
-+ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK);
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_packet_error(uint16_t bwflags)
-+{
-+ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
-+ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED);
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_eop(uint16_t ciflags)
-+{
-+ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
-+ == CQ_ENET_RQ_DESC_FLAGS_EOP;
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
-+{
-+ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) &
-+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
-+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC);
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
-+{
-+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
-+ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK);
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
-+{
-+ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
-+ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK);
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
-+{
-+ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
-+ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
-+}
-+
-+static inline uint32_t
-+enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
-+{
-+ return le32_to_cpu(cqrd->rss_hash);
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_fcs_ok(struct cq_enet_rq_desc *cqrd)
-+{
-+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) ==
-+ CQ_ENET_RQ_DESC_FLAGS_FCS_OK);
-+}
-+
-+static inline uint16_t
-+enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
-+{
-+ return le16_to_cpu(cqrd->vlan);
-+}
-+
-+static inline uint16_t
-+enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
-+{
-+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
-+ return le16_to_cpu(cqrd->bytes_written_flags) &
-+ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
-+}
-+
-+static inline uint64_t
-+enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd)
-+{
-+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
-+ uint16_t bwflags;
-+ uint64_t pkt_err_flags = 0;
-+
-+ bwflags = enic_cq_rx_desc_bwflags(cqrd);
-+
-+ /* Check for packet error. Can't be more specific than MAC error */
-+ if (enic_cq_rx_desc_packet_error(bwflags)) {
-+ pkt_err_flags |= PKT_RX_MAC_ERR;
-+ }
-+
-+ /* Check for bad FCS. MAC error isn't quite, but no other choice */
-+ if (!enic_cq_rx_desc_fcs_ok(cqrd)) {
-+ pkt_err_flags |= PKT_RX_MAC_ERR;
-+ }
-+ return pkt_err_flags;
-+}
-+
-+/*
-+ * Lookup table to translate RX CQ flags to mbuf flags.
-+ */
-+static inline uint32_t
-+enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd)
-+{
-+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
-+ uint8_t cqrd_flags = cqrd->flags;
-+ static const uint32_t cq_type_table[128] __rte_cache_aligned = {
-+ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
-+ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
-+ | RTE_PTYPE_L4_UDP,
-+ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
-+ | RTE_PTYPE_L4_TCP,
-+ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
-+ | RTE_PTYPE_L4_FRAG,
-+ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6,
-+ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
-+ | RTE_PTYPE_L4_UDP,
-+ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
-+ | RTE_PTYPE_L4_TCP,
-+ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
-+ | RTE_PTYPE_L4_FRAG,
-+ /* All others reserved */
-+ };
-+ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
-+ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
-+ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
-+ return cq_type_table[cqrd_flags];
-+}
-+
-+static inline void
-+enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
-+{
-+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
-+ uint16_t ciflags, bwflags, pkt_flags = 0;
-+ ciflags = enic_cq_rx_desc_ciflags(cqrd);
-+ bwflags = enic_cq_rx_desc_bwflags(cqrd);
-+
-+ ASSERT(mbuf->ol_flags == 0);
-+
-+ /* flags are meaningless if !EOP */
-+ if (unlikely(!enic_cq_rx_desc_eop(ciflags)))
-+ goto mbuf_flags_done;
-+
-+ /* VLAN stripping */
-+ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
-+ pkt_flags |= PKT_RX_VLAN_PKT;
-+ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd);
-+ } else {
-+ mbuf->vlan_tci = 0;
-+ }
-+
-+ /* RSS flag */
-+ if (enic_cq_rx_desc_rss_type(cqrd)) {
-+ pkt_flags |= PKT_RX_RSS_HASH;
-+ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
-+ }
-+
-+ /* checksum flags */
-+ if (!enic_cq_rx_desc_csum_not_calc(cqrd) &&
-+ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) {
-+ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd)))
-+ pkt_flags |= PKT_RX_IP_CKSUM_BAD;
-+ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) {
-+ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)))
-+ pkt_flags |= PKT_RX_L4_CKSUM_BAD;
-+ }
-+ }
-+
-+ mbuf_flags_done:
-+ mbuf->ol_flags = pkt_flags;
-+}
-+
-+static inline uint32_t
-+enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
-+{
-+ uint32_t d = i0 + i1;
-+ ASSERT(i0 < n_descriptors);
-+ ASSERT(i1 < n_descriptors);
-+ d -= (d >= n_descriptors) ? n_descriptors : 0;
-+ return d;
-+}
-+
-+
-+uint16_t
-+enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
-+ uint16_t nb_pkts)
-+{
-+ struct vnic_rq *rq = rx_queue;
-+ struct enic *enic = vnic_dev_priv(rq->vdev);
-+ unsigned int rx_id;
-+ struct rte_mbuf *nmb, *rxmb;
-+ uint16_t nb_rx = 0;
-+ uint16_t nb_hold;
-+ struct vnic_cq *cq;
-+ volatile struct cq_desc *cqd_ptr;
-+ uint8_t color;
-+
-+ cq = &enic->cq[enic_cq_rq(enic, rq->index)];
-+ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */
-+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
-+
-+ nb_hold = rq->rx_nb_hold; /* mbufs held by software */
-+
-+ while (nb_rx < nb_pkts) {
-+ uint16_t rx_pkt_len;
-+ volatile struct rq_enet_desc *rqd_ptr;
-+ dma_addr_t dma_addr;
-+ struct cq_desc cqd;
-+ uint64_t ol_err_flags;
-+
-+ /* Check for pkts available */
-+ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
-+ & CQ_DESC_COLOR_MASK;
-+ if (color == cq->last_color)
-+ break;
-+
-+ /* Get the cq descriptor and rq pointer */
-+ cqd = *cqd_ptr;
-+ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id;
-+
-+ /* allocate a new mbuf */
-+ nmb = rte_rxmbuf_alloc(rq->mp);
-+ if (nmb == NULL) {
-+ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u",
-+ enic->port_id, (unsigned)rq->index);
-+ rte_eth_devices[enic->port_id].
-+ data->rx_mbuf_alloc_failed++;
-+ break;
-+ }
-+
-+ /* Check for FCS or packet errors */
-+ ol_err_flags = enic_cq_rx_to_pkt_err_flags(&cqd);
-+ if (ol_err_flags == 0)
-+ rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd);
-+ else
-+ rx_pkt_len = 0;
-+
-+ /* Get the mbuf to return and replace with one just allocated */
-+ rxmb = rq->mbuf_ring[rx_id];
-+ rq->mbuf_ring[rx_id] = nmb;
-+
-+ /* Increment cqd, rqd, mbuf_table index */
-+ rx_id++;
-+ if (unlikely(rx_id == rq->ring.desc_count)) {
-+ rx_id = 0;
-+ cq->last_color = cq->last_color ? 0 : 1;
-+ }
-+
-+ /* Prefetch next mbuf & desc while processing current one */
-+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
-+ rte_enic_prefetch(cqd_ptr);
-+ rte_enic_prefetch(rq->mbuf_ring[rx_id]);
-+ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs)
-+ + rx_id);
-+
-+ /* Push descriptor for newly allocated mbuf */
-+ dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off);
-+ rqd_ptr->address = rte_cpu_to_le_64(dma_addr);
-+ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len);
-+
-+ /* Fill in the rest of the mbuf */
-+ rxmb->data_off = RTE_PKTMBUF_HEADROOM;
-+ rxmb->nb_segs = 1;
-+ rxmb->next = NULL;
-+ rxmb->pkt_len = rx_pkt_len;
-+ rxmb->data_len = rx_pkt_len;
-+ rxmb->port = enic->port_id;
-+ rxmb->ol_flags = ol_err_flags;
-+ if (!ol_err_flags)
-+ enic_cq_rx_to_pkt_flags(&cqd, rxmb);
-+ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd);
-+
-+ /* prefetch mbuf data for caller */
-+ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr,
-+ RTE_PKTMBUF_HEADROOM));
-+
-+ /* store the mbuf address into the next entry of the array */
-+ rx_pkts[nb_rx++] = rxmb;
-+ }
-+
-+ nb_hold += nb_rx;
-+ cq->to_clean = rx_id;
-+
-+ if (nb_hold > rq->rx_free_thresh) {
-+ rq->posted_index = enic_ring_add(rq->ring.desc_count,
-+ rq->posted_index, nb_hold);
-+ nb_hold = 0;
-+ rte_mb();
-+ iowrite32(rq->posted_index, &rq->ctrl->posted_index);
-+ }
-+
-+ rq->rx_nb_hold = nb_hold;
-+
-+ return nb_rx;
-+}
---
-1.9.1
-