Update ENIC driver patches for DPDK 2.2.0

This includes all patches in ENIC driver which are up-streamed to DPDK
to improve RX performance, fix buffer/error handling and interoperation
with link bonding PMD library.

Change-Id: Id4c71a350d5234834951f9261c69db5476ba396b
Signed-off-by: John Lo <loj@cisco.com>

1 files changed, 1336 insertions, 0 deletions

diff --git a/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch b/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch
new file mode 100644
index 00000000000..98824c57f9c
--- /dev/null
+++ b/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch
@@ -0,0 +1,1336 @@
+commit 947d860c821f4248dcf2fc01e98671524973eeea
+Author: John Daley <johndale@cisco.com>
+Date:   Fri Mar 4 13:09:00 2016 -0800
+
+    enic: improve Rx performance
+    
+    This is a wholesale replacement of the Enic PMD receive path in order
+    to improve performance and code clarity. The changes are:
+    - Simplify and reduce code path length of receive function.
+    - Put most of the fast-path receive functions in one file.
+    - Reduce the number of posted_index updates (pay attention to
+      rx_free_thresh)
+    - Remove the unneeded container structure around the RQ mbuf ring
+    - Prefetch next Mbuf and descriptors while processing the current one
+    - Use a lookup table for converting CQ flags to mbuf flags.
+    
+    Signed-off-by: John Daley <johndale@cisco.com>
+
+diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile
+index f0ee093..f316274 100644
+--- a/drivers/net/enic/Makefile
++++ b/drivers/net/enic/Makefile
+@@ -53,6 +53,7 @@ VPATH += $(SRCDIR)/src
+ #
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c
++SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c
+diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c
+index 1441604..cb62c5e 100644
+--- a/drivers/net/enic/base/vnic_rq.c
++++ b/drivers/net/enic/base/vnic_rq.c
+@@ -35,77 +35,21 @@
+ #include "vnic_dev.h"
+ #include "vnic_rq.h"
+ 
+-static int vnic_rq_alloc_bufs(struct vnic_rq *rq)
+-{
+-	struct vnic_rq_buf *buf;
+-	unsigned int i, j, count = rq->ring.desc_count;
+-	unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count);
+-
+-	for (i = 0; i < blks; i++) {
+-		rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ(count), GFP_ATOMIC);
+-		if (!rq->bufs[i])
+-			return -ENOMEM;
+-	}
+-
+-	for (i = 0; i < blks; i++) {
+-		buf = rq->bufs[i];
+-		for (j = 0; j < VNIC_RQ_BUF_BLK_ENTRIES(count); j++) {
+-			buf->index = i * VNIC_RQ_BUF_BLK_ENTRIES(count) + j;
+-			buf->desc = (u8 *)rq->ring.descs +
+-				rq->ring.desc_size * buf->index;
+-			if (buf->index + 1 == count) {
+-				buf->next = rq->bufs[0];
+-				break;
+-			} else if (j + 1 == VNIC_RQ_BUF_BLK_ENTRIES(count)) {
+-				buf->next = rq->bufs[i + 1];
+-			} else {
+-				buf->next = buf + 1;
+-				buf++;
+-			}
+-		}
+-	}
+-
+-	rq->to_use = rq->to_clean = rq->bufs[0];
+-
+-	return 0;
+-}
+-
+-int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count,
+-	unsigned int desc_size)
+-{
+-	int mem_size = 0;
+-
+-	mem_size += vnic_dev_desc_ring_size(&rq->ring, desc_count, desc_size);
+-
+-	mem_size += VNIC_RQ_BUF_BLKS_NEEDED(rq->ring.desc_count) *
+-		VNIC_RQ_BUF_BLK_SZ(rq->ring.desc_count);
+-
+-	return mem_size;
+-}
+-
+ void vnic_rq_free(struct vnic_rq *rq)
+ {
+ 	struct vnic_dev *vdev;
+-	unsigned int i;
+ 
+ 	vdev = rq->vdev;
+ 
+ 	vnic_dev_free_desc_ring(vdev, &rq->ring);
+ 
+-	for (i = 0; i < VNIC_RQ_BUF_BLKS_MAX; i++) {
+-		if (rq->bufs[i]) {
+-			kfree(rq->bufs[i]);
+-			rq->bufs[i] = NULL;
+-		}
+-	}
+-
+ 	rq->ctrl = NULL;
+ }
+ 
+ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
+ 	unsigned int desc_count, unsigned int desc_size)
+ {
+-	int err;
++	int rc;
+ 	char res_name[NAME_MAX];
+ 	static int instance;
+ 
+@@ -121,18 +65,9 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
+ 	vnic_rq_disable(rq);
+ 
+ 	snprintf(res_name, sizeof(res_name), "%d-rq-%d", instance++, index);
+-	err = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size,
++	rc = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size,
+ 		rq->socket_id, res_name);
+-	if (err)
+-		return err;
+-
+-	err = vnic_rq_alloc_bufs(rq);
+-	if (err) {
+-		vnic_rq_free(rq);
+-		return err;
+-	}
+-
+-	return 0;
++	return rc;
+ }
+ 
+ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index,
+@@ -154,9 +89,6 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index,
+ 	iowrite32(fetch_index, &rq->ctrl->fetch_index);
+ 	iowrite32(posted_index, &rq->ctrl->posted_index);
+ 
+-	rq->to_use = rq->to_clean =
+-		&rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)]
+-			[fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)];
+ }
+ 
+ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
+@@ -176,6 +108,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
+ 		fetch_index, fetch_index,
+ 		error_interrupt_enable,
+ 		error_interrupt_offset);
++	rq->rxst_idx = 0;
++	rq->tot_pkts = 0;
+ }
+ 
+ void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error)
+@@ -212,21 +146,20 @@ int vnic_rq_disable(struct vnic_rq *rq)
+ }
+ 
+ void vnic_rq_clean(struct vnic_rq *rq,
+-	void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf))
++	void (*buf_clean)(struct rte_mbuf **buf))
+ {
+-	struct vnic_rq_buf *buf;
+-	u32 fetch_index;
++	struct rte_mbuf **buf;
++	u32 fetch_index, i;
+ 	unsigned int count = rq->ring.desc_count;
+ 
+-	buf = rq->to_clean;
+-
+-	while (vnic_rq_desc_used(rq) > 0) {
++	buf = &rq->mbuf_ring[0];
+ 
+-		(*buf_clean)(rq, buf);
+-
+-		buf = rq->to_clean = buf->next;
+-		rq->ring.desc_avail++;
++	for (i = 0; i < count; i++) {
++		(*buf_clean)(buf);
++		buf++;
+ 	}
++	rq->ring.desc_avail = count - 1;
++	rq->rx_nb_hold = 0;
+ 
+ 	/* Use current fetch_index as the ring starting point */
+ 	fetch_index = ioread32(&rq->ctrl->fetch_index);
+@@ -235,9 +168,7 @@ void vnic_rq_clean(struct vnic_rq *rq,
+ 		/* Hardware surprise removal: reset fetch_index */
+ 		fetch_index = 0;
+ 	}
+-	rq->to_use = rq->to_clean =
+-		&rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)]
+-			[fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)];
++
+ 	iowrite32(fetch_index, &rq->ctrl->posted_index);
+ 
+ 	vnic_dev_clear_desc_ring(&rq->ring);
+diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h
+index 0f5c3c1..e083ccc 100644
+--- a/drivers/net/enic/base/vnic_rq.h
++++ b/drivers/net/enic/base/vnic_rq.h
+@@ -66,42 +66,22 @@ struct vnic_rq_ctrl {
+ 	u32 pad10;
+ };
+ 
+-/* Break the vnic_rq_buf allocations into blocks of 32/64 entries */
+-#define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32
+-#define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64
+-#define VNIC_RQ_BUF_BLK_ENTRIES(entries) \
+-	((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \
+-	VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES))
+-#define VNIC_RQ_BUF_BLK_SZ(entries) \
+-	(VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf))
+-#define VNIC_RQ_BUF_BLKS_NEEDED(entries) \
+-	DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries))
+-#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096)
+-
+-struct vnic_rq_buf {
+-	struct vnic_rq_buf *next;
+-	dma_addr_t dma_addr;
+-	void *os_buf;
+-	unsigned int os_buf_index;
+-	unsigned int len;
+-	unsigned int index;
+-	void *desc;
+-	uint64_t wr_id;
+-};
+-
+ struct vnic_rq {
+ 	unsigned int index;
++	unsigned int posted_index;
+ 	struct vnic_dev *vdev;
+-	struct vnic_rq_ctrl __iomem *ctrl;              /* memory-mapped */
++	struct vnic_rq_ctrl __iomem *ctrl;	/* memory-mapped */
+ 	struct vnic_dev_ring ring;
+-	struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX];
+-	struct vnic_rq_buf *to_use;
+-	struct vnic_rq_buf *to_clean;
++	struct rte_mbuf **mbuf_ring;		/* array of allocated mbufs */
++	unsigned int mbuf_next_idx;		/* next mb to consume */
+ 	void *os_buf_head;
+ 	unsigned int pkts_outstanding;
+-
++	uint16_t rx_nb_hold;
++	uint16_t rx_free_thresh;
+ 	unsigned int socket_id;
+ 	struct rte_mempool *mp;
++	uint16_t rxst_idx;
++	uint32_t tot_pkts;
+ };
+ 
+ static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq)
+@@ -116,119 +96,13 @@ static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq)
+ 	return rq->ring.desc_count - rq->ring.desc_avail - 1;
+ }
+ 
+-static inline void *vnic_rq_next_desc(struct vnic_rq *rq)
+-{
+-	return rq->to_use->desc;
+-}
+-
+-static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq)
+-{
+-	return rq->to_use->index;
+-}
+-
+-static inline void vnic_rq_post(struct vnic_rq *rq,
+-	void *os_buf, unsigned int os_buf_index,
+-	dma_addr_t dma_addr, unsigned int len,
+-	uint64_t wrid)
+-{
+-	struct vnic_rq_buf *buf = rq->to_use;
+-
+-	buf->os_buf = os_buf;
+-	buf->os_buf_index = os_buf_index;
+-	buf->dma_addr = dma_addr;
+-	buf->len = len;
+-	buf->wr_id = wrid;
+-
+-	buf = buf->next;
+-	rq->to_use = buf;
+-	rq->ring.desc_avail--;
+-
+-	/* Move the posted_index every nth descriptor
+-	 */
+-
+-#ifndef VNIC_RQ_RETURN_RATE
+-#define VNIC_RQ_RETURN_RATE		0xf	/* keep 2^n - 1 */
+-#endif
+-
+-	if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) {
+-		/* Adding write memory barrier prevents compiler and/or CPU
+-		 * reordering, thus avoiding descriptor posting before
+-		 * descriptor is initialized. Otherwise, hardware can read
+-		 * stale descriptor fields.
+-		 */
+-		wmb();
+-		iowrite32(buf->index, &rq->ctrl->posted_index);
+-	}
+-}
+-
+-static inline void vnic_rq_post_commit(struct vnic_rq *rq,
+-	void *os_buf, unsigned int os_buf_index,
+-	dma_addr_t dma_addr, unsigned int len)
+-{
+-	struct vnic_rq_buf *buf = rq->to_use;
+-
+-	buf->os_buf = os_buf;
+-	buf->os_buf_index = os_buf_index;
+-	buf->dma_addr = dma_addr;
+-	buf->len = len;
+-
+-	buf = buf->next;
+-	rq->to_use = buf;
+-	rq->ring.desc_avail--;
+-
+-	/* Move the posted_index every descriptor
+-	 */
+-
+-	/* Adding write memory barrier prevents compiler and/or CPU
+-	 * reordering, thus avoiding descriptor posting before
+-	 * descriptor is initialized. Otherwise, hardware can read
+-	 * stale descriptor fields.
+-	 */
+-	wmb();
+-	iowrite32(buf->index, &rq->ctrl->posted_index);
+-}
+ 
+-static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count)
+-{
+-	rq->ring.desc_avail += count;
+-}
+ 
+ enum desc_return_options {
+ 	VNIC_RQ_RETURN_DESC,
+ 	VNIC_RQ_DEFER_RETURN_DESC,
+ };
+ 
+-static inline int vnic_rq_service(struct vnic_rq *rq,
+-	struct cq_desc *cq_desc, u16 completed_index,
+-	int desc_return, int (*buf_service)(struct vnic_rq *rq,
+-	struct cq_desc *cq_desc, struct vnic_rq_buf *buf,
+-	int skipped, void *opaque), void *opaque)
+-{
+-	struct vnic_rq_buf *buf;
+-	int skipped;
+-	int eop = 0;
+-
+-	buf = rq->to_clean;
+-	while (1) {
+-
+-		skipped = (buf->index != completed_index);
+-
+-		if ((*buf_service)(rq, cq_desc, buf, skipped, opaque))
+-			eop++;
+-
+-		if (desc_return == VNIC_RQ_RETURN_DESC)
+-			rq->ring.desc_avail++;
+-
+-		rq->to_clean = buf->next;
+-
+-		if (!skipped)
+-			break;
+-
+-		buf = rq->to_clean;
+-	}
+-	return eop;
+-}
+-
+ static inline int vnic_rq_fill(struct vnic_rq *rq,
+ 	int (*buf_fill)(struct vnic_rq *rq))
+ {
+@@ -274,8 +148,5 @@ unsigned int vnic_rq_error_status(struct vnic_rq *rq);
+ void vnic_rq_enable(struct vnic_rq *rq);
+ int vnic_rq_disable(struct vnic_rq *rq);
+ void vnic_rq_clean(struct vnic_rq *rq,
+-	void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf));
+-int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count,
+-	unsigned int desc_size);
+-
++	void (*buf_clean)(struct rte_mbuf **buf));
+ #endif /* _VNIC_RQ_H_ */
+diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
+index 9e78305..8c914f5 100644
+--- a/drivers/net/enic/enic.h
++++ b/drivers/net/enic/enic.h
+@@ -45,6 +45,7 @@
+ #include "vnic_nic.h"
+ #include "vnic_rss.h"
+ #include "enic_res.h"
++#include "cq_enet_desc.h"
+ 
+ #define DRV_NAME		"enic_pmd"
+ #define DRV_DESCRIPTION		"Cisco VIC Ethernet NIC Poll-mode Driver"
+@@ -154,6 +155,16 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev)
+ 	return (struct enic *)eth_dev->data->dev_private;
+ }
+ 
++#define RTE_LIBRTE_ENIC_ASSERT_ENABLE
++#ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE
++#define ASSERT(x) do {			\
++	if (!(x))			\
++		rte_panic("ENIC: x");	\
++} while (0)
++#else
++#define ASSERT(x)
++#endif
++
+ extern void enic_fdir_stats_get(struct enic *enic,
+ 	struct rte_eth_fdir_stats *stats);
+ extern int enic_fdir_add_fltr(struct enic *enic,
+@@ -193,9 +204,10 @@ extern void enic_send_pkt(struct enic *enic, struct vnic_wq *wq,
+ 			  uint16_t ol_flags, uint16_t vlan_tag);
+ 
+ extern void enic_post_wq_index(struct vnic_wq *wq);
+-extern int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts,
+-	unsigned int budget, unsigned int *work_done);
+ extern int enic_probe(struct enic *enic);
+ extern int enic_clsf_init(struct enic *enic);
+ extern void enic_clsf_destroy(struct enic *enic);
++uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
++			uint16_t nb_pkts);
++
+ #endif /* _ENIC_H_ */
+diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c
+index 2a88043..6f2ada5 100644
+--- a/drivers/net/enic/enic_ethdev.c
++++ b/drivers/net/enic/enic_ethdev.c
+@@ -255,7 +255,7 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
+ 	uint16_t queue_idx,
+ 	uint16_t nb_desc,
+ 	unsigned int socket_id,
+-	__rte_unused const struct rte_eth_rxconf *rx_conf,
++	const struct rte_eth_rxconf *rx_conf,
+ 	struct rte_mempool *mp)
+ {
+ 	int ret;
+@@ -270,6 +270,10 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
+ 		return ret;
+ 	}
+ 
++	enic->rq[queue_idx].rx_free_thresh = rx_conf->rx_free_thresh;
++	dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
++			enic->rq[queue_idx].rx_free_thresh);
++
+ 	return enicpmd_dev_setup_intr(enic);
+ }
+ 
+@@ -429,6 +433,9 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev,
+ 		DEV_TX_OFFLOAD_IPV4_CKSUM  |
+ 		DEV_TX_OFFLOAD_UDP_CKSUM   |
+ 		DEV_TX_OFFLOAD_TCP_CKSUM;
++	device_info->default_rxconf = (struct rte_eth_rxconf) {
++		.rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH
++	};
+ }
+ 
+ static void enicpmd_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
+@@ -538,18 +545,6 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ 	return index;
+ }
+ 
+-static uint16_t enicpmd_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+-	uint16_t nb_pkts)
+-{
+-	struct vnic_rq *rq = (struct vnic_rq *)rx_queue;
+-	unsigned int work_done;
+-
+-	if (enic_poll(rq, rx_pkts, (unsigned int)nb_pkts, &work_done))
+-		dev_err(enic, "error in enicpmd poll\n");
+-
+-	return work_done;
+-}
+-
+ static const struct eth_dev_ops enicpmd_eth_dev_ops = {
+ 	.dev_configure        = enicpmd_dev_configure,
+ 	.dev_start            = enicpmd_dev_start,
+@@ -606,7 +601,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev)
+ 	enic->port_id = eth_dev->data->port_id;
+ 	enic->rte_dev = eth_dev;
+ 	eth_dev->dev_ops = &enicpmd_eth_dev_ops;
+-	eth_dev->rx_pkt_burst = &enicpmd_recv_pkts;
++	eth_dev->rx_pkt_burst = &enic_recv_pkts;
+ 	eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts;
+ 
+ 	pdev = eth_dev->pci_dev;
+@@ -635,8 +630,8 @@ static struct eth_driver rte_enic_pmd = {
+  * Register as the [Poll Mode] Driver of Cisco ENIC device.
+  */
+ static int
+-rte_enic_pmd_init(const char *name __rte_unused,
+-	const char *params __rte_unused)
++rte_enic_pmd_init(__rte_unused const char *name,
++	 __rte_unused const char *params)
+ {
+ 	ENICPMD_FUNC_TRACE();
+ 
+diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
+index f818c32..9fff020 100644
+--- a/drivers/net/enic/enic_main.c
++++ b/drivers/net/enic/enic_main.c
+@@ -60,6 +60,17 @@
+ #include "vnic_nic.h"
+ #include "enic_vnic_wq.h"
+ 
++static inline struct rte_mbuf *
++rte_rxmbuf_alloc(struct rte_mempool *mp)
++{
++	struct rte_mbuf *m;
++
++	m = __rte_mbuf_raw_alloc(mp);
++	__rte_mbuf_sanity_check_raw(m, 0);
++	return m;
++}
++
++
+ static inline int enic_is_sriov_vf(struct enic *enic)
+ {
+ 	return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
+@@ -80,16 +91,25 @@ static int is_eth_addr_valid(uint8_t *addr)
+ 	return !is_mcast_addr(addr) && !is_zero_addr(addr);
+ }
+ 
+-static inline struct rte_mbuf *
+-enic_rxmbuf_alloc(struct rte_mempool *mp)
++static void
++enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq)
+ {
+-	struct rte_mbuf *m;
++	uint16_t i;
+ 
+-	m = __rte_mbuf_raw_alloc(mp);
+-	__rte_mbuf_sanity_check_raw(m, 0);
+-	return m;
++	if (!rq || !rq->mbuf_ring) {
++		dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
++		return;
++	}
++
++	for (i = 0; i < enic->config.rq_desc_count; i++) {
++		if (rq->mbuf_ring[i]) {
++			rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
++			rq->mbuf_ring[i] = NULL;
++		}
++	}
+ }
+ 
++
+ void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size)
+ {
+ 	vnic_set_hdr_split_size(enic->vdev, split_hdr_size);
+@@ -262,13 +282,13 @@ void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
+ }
+ 
+ static void
+-enic_free_rq_buf(__rte_unused struct vnic_rq *rq, struct vnic_rq_buf *buf)
++enic_free_rq_buf(struct rte_mbuf **mbuf)
+ {
+-	if (!buf->os_buf)
++	if (*mbuf == NULL)
+ 		return;
+ 
+-	rte_pktmbuf_free((struct rte_mbuf *)buf->os_buf);
+-	buf->os_buf = NULL;
++	rte_pktmbuf_free(*mbuf);
++	mbuf = NULL;
+ }
+ 
+ void enic_init_vnic_resources(struct enic *enic)
+@@ -314,221 +334,47 @@ void enic_init_vnic_resources(struct enic *enic)
+ }
+ 
+ 
+-static int enic_rq_alloc_buf(struct vnic_rq *rq)
++static int
++enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
+ {
+-	struct enic *enic = vnic_dev_priv(rq->vdev);
++	struct rte_mbuf *mb;
++	struct rq_enet_desc *rqd = rq->ring.descs;
++	unsigned i;
+ 	dma_addr_t dma_addr;
+-	struct rq_enet_desc *desc = vnic_rq_next_desc(rq);
+-	uint8_t type = RQ_ENET_TYPE_ONLY_SOP;
+-	u16 split_hdr_size = vnic_get_hdr_split_size(enic->vdev);
+-	struct rte_mbuf *mbuf = enic_rxmbuf_alloc(rq->mp);
+-	struct rte_mbuf *hdr_mbuf = NULL;
+-
+-	if (!mbuf) {
+-		dev_err(enic, "mbuf alloc in enic_rq_alloc_buf failed\n");
+-		return -1;
+-	}
+-
+-	if (unlikely(split_hdr_size)) {
+-		if (vnic_rq_desc_avail(rq) < 2) {
+-			rte_mempool_put(mbuf->pool, mbuf);
+-			return -1;
+-		}
+-		hdr_mbuf = enic_rxmbuf_alloc(rq->mp);
+-		if (!hdr_mbuf) {
+-			rte_mempool_put(mbuf->pool, mbuf);
+-			dev_err(enic,
+-				"hdr_mbuf alloc in enic_rq_alloc_buf failed\n");
+-			return -1;
+-		}
+-
+-		hdr_mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+-
+-		hdr_mbuf->nb_segs = 2;
+-		hdr_mbuf->port = enic->port_id;
+-		hdr_mbuf->next = mbuf;
+-
+-		dma_addr = (dma_addr_t)
+-		    (hdr_mbuf->buf_physaddr + hdr_mbuf->data_off);
+-
+-		rq_enet_desc_enc(desc, dma_addr, type, split_hdr_size);
+ 
+-		vnic_rq_post(rq, (void *)hdr_mbuf, 0 /*os_buf_index*/, dma_addr,
+-			(unsigned int)split_hdr_size, 0 /*wrid*/);
++	dev_debug(enic, "queue %u, allocating %u rx queue mbufs", rq->index,
++		  rq->ring.desc_count);
+ 
+-		desc = vnic_rq_next_desc(rq);
+-		type = RQ_ENET_TYPE_NOT_SOP;
+-	} else {
+-		mbuf->nb_segs = 1;
+-		mbuf->port = enic->port_id;
+-	}
+-
+-	mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+-	mbuf->next = NULL;
+-
+-	dma_addr = (dma_addr_t)
+-	    (mbuf->buf_physaddr + mbuf->data_off);
+-
+-	rq_enet_desc_enc(desc, dma_addr, type, mbuf->buf_len);
+-
+-	vnic_rq_post(rq, (void *)mbuf, 0 /*os_buf_index*/, dma_addr,
+-		(unsigned int)mbuf->buf_len, 0 /*wrid*/);
+-
+-	return 0;
+-}
+-
+-static int enic_rq_indicate_buf(struct vnic_rq *rq,
+-	struct cq_desc *cq_desc, struct vnic_rq_buf *buf,
+-	int skipped, void *opaque)
+-{
+-	struct enic *enic = vnic_dev_priv(rq->vdev);
+-	struct rte_mbuf **rx_pkt_bucket = (struct rte_mbuf **)opaque;
+-	struct rte_mbuf *rx_pkt = NULL;
+-	struct rte_mbuf *hdr_rx_pkt = NULL;
+-
+-	u8 type, color, eop, sop, ingress_port, vlan_stripped;
+-	u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof;
+-	u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok;
+-	u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc;
+-	u8 packet_error;
+-	u16 q_number, completed_index, bytes_written, vlan_tci, checksum;
+-	u32 rss_hash;
+-
+-	cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc,
+-		&type, &color, &q_number, &completed_index,
+-		&ingress_port, &fcoe, &eop, &sop, &rss_type,
+-		&csum_not_calc, &rss_hash, &bytes_written,
+-		&packet_error, &vlan_stripped, &vlan_tci, &checksum,
+-		&fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error,
+-		&fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp,
+-		&ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment,
+-		&fcs_ok);
+-
+-	rx_pkt = (struct rte_mbuf *)buf->os_buf;
+-	buf->os_buf = NULL;
+-
+-	if (unlikely(packet_error)) {
+-		dev_err(enic, "packet error\n");
+-		rx_pkt->data_len = 0;
+-		return 0;
+-	}
+-
+-	if (unlikely(skipped)) {
+-		rx_pkt->data_len = 0;
+-		return 0;
+-	}
+-
+-	if (likely(!vnic_get_hdr_split_size(enic->vdev))) {
+-		/* No header split configured */
+-		*rx_pkt_bucket = rx_pkt;
+-		rx_pkt->pkt_len = bytes_written;
+-
+-		if (ipv4) {
+-			rx_pkt->packet_type = RTE_PTYPE_L3_IPV4;
+-			if (!csum_not_calc) {
+-				if (unlikely(!ipv4_csum_ok))
+-					rx_pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD;
+-
+-				if ((tcp || udp) && (!tcp_udp_csum_ok))
+-					rx_pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD;
+-			}
+-		} else if (ipv6)
+-			rx_pkt->packet_type = RTE_PTYPE_L3_IPV6;
+-	} else {
+-		/* Header split */
+-		if (sop && !eop) {
+-			/* This piece is header */
+-			*rx_pkt_bucket = rx_pkt;
+-			rx_pkt->pkt_len = bytes_written;
+-		} else {
+-			if (sop && eop) {
+-				/* The packet is smaller than split_hdr_size */
+-				*rx_pkt_bucket = rx_pkt;
+-				rx_pkt->pkt_len = bytes_written;
+-				if (ipv4) {
+-					rx_pkt->packet_type = RTE_PTYPE_L3_IPV4;
+-					if (!csum_not_calc) {
+-						if (unlikely(!ipv4_csum_ok))
+-							rx_pkt->ol_flags |=
+-							    PKT_RX_IP_CKSUM_BAD;
+-
+-						if ((tcp || udp) &&
+-						    (!tcp_udp_csum_ok))
+-							rx_pkt->ol_flags |=
+-							    PKT_RX_L4_CKSUM_BAD;
+-					}
+-				} else if (ipv6)
+-					rx_pkt->packet_type = RTE_PTYPE_L3_IPV6;
+-			} else {
+-				/* Payload */
+-				hdr_rx_pkt = *rx_pkt_bucket;
+-				hdr_rx_pkt->pkt_len += bytes_written;
+-				if (ipv4) {
+-					hdr_rx_pkt->packet_type =
+-						RTE_PTYPE_L3_IPV4;
+-					if (!csum_not_calc) {
+-						if (unlikely(!ipv4_csum_ok))
+-							hdr_rx_pkt->ol_flags |=
+-							    PKT_RX_IP_CKSUM_BAD;
+-
+-						if ((tcp || udp) &&
+-						    (!tcp_udp_csum_ok))
+-							hdr_rx_pkt->ol_flags |=
+-							    PKT_RX_L4_CKSUM_BAD;
+-					}
+-				} else if (ipv6)
+-					hdr_rx_pkt->packet_type =
+-						RTE_PTYPE_L3_IPV6;
+-			}
++	for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
++		mb = rte_rxmbuf_alloc(rq->mp);
++		if (mb == NULL) {
++			dev_err(enic, "RX mbuf alloc failed queue_id=%u",
++			(unsigned)rq->index);
++			return -ENOMEM;
+ 		}
+-	}
+ 
+-	rx_pkt->data_len = bytes_written;
++		dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off);
+ 
+-	if (rss_hash) {
+-		rx_pkt->ol_flags |= PKT_RX_RSS_HASH;
+-		rx_pkt->hash.rss = rss_hash;
++		rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP,
++				 mb->buf_len);
++		rq->mbuf_ring[i] = mb;
+ 	}
+ 
+-	if (vlan_tci) {
+-		rx_pkt->ol_flags |= PKT_RX_VLAN_PKT;
+-		rx_pkt->vlan_tci = vlan_tci;
+-	}
++	/* make sure all prior writes are complete before doing the PIO write */
++	rte_rmb();
+ 
+-	return eop;
+-}
++	/* Post all but the last 2 cache lines' worth of descriptors */
++	rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE
++			/ sizeof(struct rq_enet_desc));
++	rq->rx_nb_hold = 0;
+ 
+-static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
+-	__rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque)
+-{
+-	struct enic *enic = vnic_dev_priv(vdev);
+-
+-	return vnic_rq_service(&enic->rq[q_number], cq_desc,
+-		completed_index, VNIC_RQ_RETURN_DESC,
+-		enic_rq_indicate_buf, opaque);
+-
+-}
++	dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
++		enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
++	iowrite32(rq->posted_index, &rq->ctrl->posted_index);
++	rte_rmb();
+ 
+-int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts,
+-	unsigned int budget, unsigned int *work_done)
+-{
+-	struct enic *enic = vnic_dev_priv(rq->vdev);
+-	unsigned int cq = enic_cq_rq(enic, rq->index);
+-	int err = 0;
+-
+-	*work_done = vnic_cq_service(&enic->cq[cq],
+-		budget, enic_rq_service, (void *)rx_pkts);
+-
+-	if (*work_done) {
+-		vnic_rq_fill(rq, enic_rq_alloc_buf);
++	return 0;
+ 
+-		/* Need at least one buffer on ring to get going */
+-		if (vnic_rq_desc_used(rq) == 0) {
+-			dev_err(enic, "Unable to alloc receive buffers\n");
+-			err = -1;
+-		}
+-	}
+-	return err;
+ }
+ 
+ static void *
+@@ -576,6 +422,7 @@ enic_intr_handler(__rte_unused struct rte_intr_handle *handle,
+ int enic_enable(struct enic *enic)
+ {
+ 	unsigned int index;
++	int err;
+ 	struct rte_eth_dev *eth_dev = enic->rte_dev;
+ 
+ 	eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
+@@ -586,15 +433,11 @@ int enic_enable(struct enic *enic)
+ 		dev_warning(enic, "Init of hash table for clsf failed."\
+ 			"Flow director feature will not work\n");
+ 
+-	/* Fill RQ bufs */
+ 	for (index = 0; index < enic->rq_count; index++) {
+-		vnic_rq_fill(&enic->rq[index], enic_rq_alloc_buf);
+-
+-		/* Need at least one buffer on ring to get going
+-		*/
+-		if (vnic_rq_desc_used(&enic->rq[index]) == 0) {
+-			dev_err(enic, "Unable to alloc receive buffers\n");
+-			return -1;
++		err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]);
++		if (err) {
++			dev_err(enic, "Failed to alloc RX queue mbufs\n");
++			return err;
+ 		}
+ 	}
+ 
+@@ -636,6 +479,9 @@ void enic_free_rq(void *rxq)
+ 	struct vnic_rq *rq = (struct vnic_rq *)rxq;
+ 	struct enic *enic = vnic_dev_priv(rq->vdev);
+ 
++	enic_rxmbuf_queue_release(enic, rq);
++	rte_free(rq->mbuf_ring);
++	rq->mbuf_ring = NULL;
+ 	vnic_rq_free(rq);
+ 	vnic_cq_free(&enic->cq[rq->index]);
+ }
+@@ -664,7 +510,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
+ 	unsigned int socket_id, struct rte_mempool *mp,
+ 	uint16_t nb_desc)
+ {
+-	int err;
++	int rc;
+ 	struct vnic_rq *rq = &enic->rq[queue_idx];
+ 
+ 	rq->socket_id = socket_id;
+@@ -687,23 +533,35 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
+ 	}
+ 
+ 	/* Allocate queue resources */
+-	err = vnic_rq_alloc(enic->vdev, &enic->rq[queue_idx], queue_idx,
+-		enic->config.rq_desc_count,
+-		sizeof(struct rq_enet_desc));
+-	if (err) {
++	rc = vnic_rq_alloc(enic->vdev, rq, queue_idx,
++		enic->config.rq_desc_count, sizeof(struct rq_enet_desc));
++	if (rc) {
+ 		dev_err(enic, "error in allocation of rq\n");
+-		return err;
++		goto err_exit;
+ 	}
+ 
+-	err = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
++	rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
+ 		socket_id, enic->config.rq_desc_count,
+ 		sizeof(struct cq_enet_rq_desc));
+-	if (err) {
+-		vnic_rq_free(rq);
++	if (rc) {
+ 		dev_err(enic, "error in allocation of cq for rq\n");
++		goto err_free_rq_exit;
+ 	}
+ 
+-	return err;
++	/* Allocate the mbuf ring */
++	rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring",
++			sizeof(struct rte_mbuf *) * enic->config.rq_desc_count,
++			RTE_CACHE_LINE_SIZE, rq->socket_id);
++
++	if (rq->mbuf_ring != NULL)
++		return 0;
++
++	/* cleanup on error */
++	vnic_cq_free(&enic->cq[queue_idx]);
++err_free_rq_exit:
++	vnic_rq_free(rq);
++err_exit:
++	return -ENOMEM;
+ }
+ 
+ void enic_free_wq(void *txq)
+@@ -790,6 +648,7 @@ int enic_disable(struct enic *enic)
+ 
+ 	for (i = 0; i < enic->wq_count; i++)
+ 		vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
++
+ 	for (i = 0; i < enic->rq_count; i++)
+ 		vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
+ 	for (i = 0; i < enic->cq_count; i++)
+@@ -1074,7 +933,7 @@ int enic_probe(struct enic *enic)
+ 
+ 	/* Set ingress vlan rewrite mode before vnic initialization */
+ 	err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
+-		IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN);
++		IG_VLAN_REWRITE_MODE_PASS_THRU);
+ 	if (err) {
+ 		dev_err(enic,
+ 			"Failed to set ingress vlan rewrite mode, aborting.\n");
+diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h
+index 49f7e22..33f2e84 100644
+--- a/drivers/net/enic/enic_res.h
++++ b/drivers/net/enic/enic_res.h
+@@ -52,6 +52,7 @@
+ #define ENIC_UNICAST_PERFECT_FILTERS	32
+ 
+ #define ENIC_NON_TSO_MAX_DESC		16
++#define ENIC_DEFAULT_RX_FREE_THRESH	32
+ 
+ #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0)
+ 
+@@ -133,21 +134,6 @@ static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq,
+ 		WQ_ENET_OFFLOAD_MODE_TSO,
+ 		eop, 1 /* SOP */, eop, loopback);
+ }
+-static inline void enic_queue_rq_desc(struct vnic_rq *rq,
+-	void *os_buf, unsigned int os_buf_index,
+-	dma_addr_t dma_addr, unsigned int len)
+-{
+-	struct rq_enet_desc *desc = vnic_rq_next_desc(rq);
+-	u64 wrid = 0;
+-	u8 type = os_buf_index ?
+-		RQ_ENET_TYPE_NOT_SOP : RQ_ENET_TYPE_ONLY_SOP;
+-
+-	rq_enet_desc_enc(desc,
+-		(u64)dma_addr | VNIC_PADDR_TARGET,
+-		type, (u16)len);
+-
+-	vnic_rq_post(rq, os_buf, os_buf_index, dma_addr, len, wrid);
+-}
+ 
+ struct enic;
+ 
+diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c
+new file mode 100644
+index 0000000..945a60f
+--- /dev/null
++++ b/drivers/net/enic/enic_rx.c
+@@ -0,0 +1,370 @@
++/*
++ * Copyright 2008-2014 Cisco Systems, Inc.  All rights reserved.
++ * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
++ *
++ * Copyright (c) 2014, Cisco Systems, Inc.
++ * All rights reserved.
++ *
++ * Redistribution and use in source and binary forms, with or without
++ * modification, are permitted provided that the following conditions
++ * are met:
++ *
++ * 1. Redistributions of source code must retain the above copyright
++ * notice, this list of conditions and the following disclaimer.
++ *
++ * 2. Redistributions in binary form must reproduce the above copyright
++ * notice, this list of conditions and the following disclaimer in
++ * the documentation and/or other materials provided with the
++ * distribution.
++ *
++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
++ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
++ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
++ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
++ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
++ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
++ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
++ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
++ * POSSIBILITY OF SUCH DAMAGE.
++ *
++ */
++
++#include <rte_mbuf.h>
++#include <rte_ethdev.h>
++#include <rte_prefetch.h>
++
++#include "enic_compat.h"
++#include "rq_enet_desc.h"
++#include "enic.h"
++
++#define RTE_PMD_USE_PREFETCH
++
++#ifdef RTE_PMD_USE_PREFETCH
++/*
++ * Prefetch a cache line into all cache levels.
++ */
++#define rte_enic_prefetch(p) rte_prefetch0(p)
++#else
++#define rte_enic_prefetch(p) do {} while (0)
++#endif
++
++#ifdef RTE_PMD_PACKET_PREFETCH
++#define rte_packet_prefetch(p) rte_prefetch1(p)
++#else
++#define rte_packet_prefetch(p) do {} while (0)
++#endif
++
++static inline struct rte_mbuf *
++rte_rxmbuf_alloc(struct rte_mempool *mp)
++{
++	struct rte_mbuf *m;
++
++	m = __rte_mbuf_raw_alloc(mp);
++	__rte_mbuf_sanity_check_raw(m, 0);
++	return m;
++}
++
++static inline uint16_t
++enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
++{
++	return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
++}
++
++static inline uint16_t
++enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
++{
++	return(le16_to_cpu(crd->bytes_written_flags) &
++		~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_packet_error(uint16_t bwflags)
++{
++	return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
++		CQ_ENET_RQ_DESC_FLAGS_TRUNCATED);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_eop(uint16_t ciflags)
++{
++	return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
++		== CQ_ENET_RQ_DESC_FLAGS_EOP;
++}
++
++static inline uint8_t
++enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
++{
++	return ((le16_to_cpu(cqrd->q_number_rss_type_flags) &
++		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
++		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
++{
++	return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
++		CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
++{
++	return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
++		CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
++{
++	return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
++		CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
++}
++
++static inline uint32_t
++enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
++{
++	return le32_to_cpu(cqrd->rss_hash);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_fcs_ok(struct cq_enet_rq_desc *cqrd)
++{
++	return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) ==
++		CQ_ENET_RQ_DESC_FLAGS_FCS_OK);
++}
++
++static inline uint16_t
++enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
++{
++	return le16_to_cpu(cqrd->vlan);
++}
++
++static inline uint16_t
++enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
++{
++	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
++	return le16_to_cpu(cqrd->bytes_written_flags) &
++		CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
++}
++
++static inline uint64_t
++enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd)
++{
++	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
++	uint16_t bwflags;
++	uint64_t pkt_err_flags = 0;
++
++	bwflags = enic_cq_rx_desc_bwflags(cqrd);
++
++	/* Check for packet error. Can't be more specific than MAC error */
++	if (enic_cq_rx_desc_packet_error(bwflags)) {
++		pkt_err_flags |= PKT_RX_MAC_ERR;
++	}
++
++	/* Check for bad FCS. MAC error isn't quite, but no other choice */
++	if (!enic_cq_rx_desc_fcs_ok(cqrd)) {
++		pkt_err_flags |= PKT_RX_MAC_ERR;
++	}
++	return pkt_err_flags;
++}
++
++/*
++ * Lookup table to translate RX CQ flags to mbuf flags.
++ */
++static inline uint32_t
++enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd)
++{
++	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
++	uint8_t cqrd_flags = cqrd->flags;
++	static const uint32_t cq_type_table[128] __rte_cache_aligned = {
++		[32] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
++		[34] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
++			| RTE_PTYPE_L4_UDP,
++		[36] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
++			| RTE_PTYPE_L4_TCP,
++		[96] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
++			| RTE_PTYPE_L4_FRAG,
++		[16] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6,
++		[18] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
++			| RTE_PTYPE_L4_UDP,
++		[20] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
++			| RTE_PTYPE_L4_TCP,
++		[80] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
++			| RTE_PTYPE_L4_FRAG,
++		/* All others reserved */
++	};
++	cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
++		| CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
++		| CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
++	return cq_type_table[cqrd_flags];
++}
++
++static inline void
++enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
++{
++	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
++	uint16_t ciflags, bwflags, pkt_flags = 0;
++	ciflags = enic_cq_rx_desc_ciflags(cqrd);
++	bwflags = enic_cq_rx_desc_bwflags(cqrd);
++
++	ASSERT(mbuf->ol_flags == 0);
++
++	/* flags are meaningless if !EOP */
++	if (unlikely(!enic_cq_rx_desc_eop(ciflags)))
++		goto mbuf_flags_done;
++
++	/* VLAN stripping */
++	if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
++		pkt_flags |= PKT_RX_VLAN_PKT;
++		mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd);
++	} else {
++		mbuf->vlan_tci = 0;
++	}
++
++	/* RSS flag */
++	if (enic_cq_rx_desc_rss_type(cqrd)) {
++		pkt_flags |= PKT_RX_RSS_HASH;
++		mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
++	}
++
++	/* checksum flags */
++	if (!enic_cq_rx_desc_csum_not_calc(cqrd) &&
++		(mbuf->packet_type & RTE_PTYPE_L3_IPV4)) {
++		if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd)))
++			pkt_flags |= PKT_RX_IP_CKSUM_BAD;
++		if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) {
++			if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)))
++				pkt_flags |= PKT_RX_L4_CKSUM_BAD;
++		}
++	}
++
++ mbuf_flags_done:
++	mbuf->ol_flags = pkt_flags;
++}
++
++static inline uint32_t
++enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
++{
++	uint32_t d = i0 + i1;
++	ASSERT(i0 < n_descriptors);
++	ASSERT(i1 < n_descriptors);
++	d -= (d >= n_descriptors) ? n_descriptors : 0;
++	return d;
++}
++
++
++uint16_t
++enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
++	       uint16_t nb_pkts)
++{
++	struct vnic_rq *rq = rx_queue;
++	struct enic *enic = vnic_dev_priv(rq->vdev);
++	unsigned int rx_id;
++	struct rte_mbuf *nmb, *rxmb;
++	uint16_t nb_rx = 0;
++	uint16_t nb_hold;
++	struct vnic_cq *cq;
++	volatile struct cq_desc *cqd_ptr;
++	uint8_t color;
++
++	cq = &enic->cq[enic_cq_rq(enic, rq->index)];
++	rx_id = cq->to_clean;		/* index of cqd, rqd, mbuf_table */
++	cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
++
++	nb_hold = rq->rx_nb_hold;	/* mbufs held by software */
++
++	while (nb_rx < nb_pkts) {
++		uint16_t rx_pkt_len;
++		volatile struct rq_enet_desc *rqd_ptr;
++		dma_addr_t dma_addr;
++		struct cq_desc cqd;
++		uint64_t ol_err_flags;
++
++		/* Check for pkts available */
++		color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
++			& CQ_DESC_COLOR_MASK;
++		if (color == cq->last_color)
++			break;
++
++		/* Get the cq descriptor and rq pointer */
++		cqd = *cqd_ptr;
++		rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id;
++
++		/* allocate a new mbuf */
++		nmb = rte_rxmbuf_alloc(rq->mp);
++		if (nmb == NULL) {
++			dev_err(enic, "RX mbuf alloc failed port=%u qid=%u",
++			enic->port_id, (unsigned)rq->index);
++			rte_eth_devices[enic->port_id].
++					data->rx_mbuf_alloc_failed++;
++			break;
++		}
++
++		/* Check for FCS or packet errors */
++		ol_err_flags = enic_cq_rx_to_pkt_err_flags(&cqd);
++		if (ol_err_flags == 0)
++			rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd);
++		else
++			rx_pkt_len = 0;
++
++		/* Get the mbuf to return and replace with one just allocated */
++		rxmb = rq->mbuf_ring[rx_id];
++		rq->mbuf_ring[rx_id] = nmb;
++
++		/* Increment cqd, rqd, mbuf_table index */
++		rx_id++;
++		if (unlikely(rx_id == rq->ring.desc_count)) {
++			rx_id = 0;
++			cq->last_color = cq->last_color ? 0 : 1;
++		}
++
++		/* Prefetch next mbuf & desc while processing current one */
++		cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
++		rte_enic_prefetch(cqd_ptr);
++		rte_enic_prefetch(rq->mbuf_ring[rx_id]);
++		rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs)
++				 + rx_id);
++
++		/* Push descriptor for newly allocated mbuf */
++		dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off);
++		rqd_ptr->address = rte_cpu_to_le_64(dma_addr);
++		rqd_ptr->length_type = cpu_to_le16(nmb->buf_len);
++
++		/* Fill in the rest of the mbuf */
++		rxmb->data_off = RTE_PKTMBUF_HEADROOM;
++		rxmb->nb_segs = 1;
++		rxmb->next = NULL;
++		rxmb->pkt_len = rx_pkt_len;
++		rxmb->data_len = rx_pkt_len;
++		rxmb->port = enic->port_id;
++		rxmb->ol_flags = ol_err_flags;
++		if (!ol_err_flags)
++			enic_cq_rx_to_pkt_flags(&cqd, rxmb);
++		rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd);
++
++		/* prefetch mbuf data for caller */
++		rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr,
++				    RTE_PKTMBUF_HEADROOM));
++
++		/* store the mbuf address into the next entry of the array */
++		rx_pkts[nb_rx++] = rxmb;
++	}
++
++	nb_hold += nb_rx;
++	cq->to_clean = rx_id;
++
++	if (nb_hold > rq->rx_free_thresh) {
++		rq->posted_index = enic_ring_add(rq->ring.desc_count,
++				rq->posted_index, nb_hold);
++		nb_hold = 0;
++		rte_mb();
++		iowrite32(rq->posted_index, &rq->ctrl->posted_index);
++	}
++
++	rq->rx_nb_hold = nb_hold;
++
++	return nb_rx;
++}