dpdk: remove old patches

Change-Id: I31244207ca5420558c6ff00b2021126ff5628e08
Signed-off-by: Damjan Marion <damarion@cisco.com>

1 files changed, 0 insertions, 1844 deletions

diff --git a/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch b/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch
deleted file mode 100644
index 4858b8f5d39..00000000000
--- a/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch
+++ /dev/null
@@ -1,1844 +0,0 @@
-From ca6bbb723880e91d006de6cc485259da988859aa Mon Sep 17 00:00:00 2001
-From: John Daley <johndale@cisco.com>
-Date: Tue, 5 Apr 2016 15:19:06 -0700
-Subject: [PATCH 3/3] enic: Optimization of Tx path to reduce Host CPU
- overhead, cleanup
-
-Optimizations and cleanup:
-- flatten packet send path
-- flatten mbuf free path
-- disable CQ entry writing and use CQ messages instead
-- use rte_mempool_put_bulk() to bulk return freed mbufs
-- remove unnecessary fields vnic_bufs struct, use contiguous array of cache
-  aligned divisible elements. No next pointers.
-- use local variables inside per packet loop instead of fields in structs.
-- factor book keeping out of the per packet tx loop where possible
-  (removed several conditionals)
-- put Tx and Rx code in 1 file (enic_rxtx.c)
-
-Reviewed-by: Nelson Escobar <neescoba@cisco.com>
-Signed-off-by: John Daley <johndale@cisco.com>
----
- drivers/net/enic/Makefile            |   2 +-
- drivers/net/enic/base/enic_vnic_wq.h |  79 ------
- drivers/net/enic/base/vnic_cq.h      |  37 +--
- drivers/net/enic/base/vnic_rq.h      |   2 +-
- drivers/net/enic/base/vnic_wq.c      |  89 +++---
- drivers/net/enic/base/vnic_wq.h      | 113 +-------
- drivers/net/enic/enic.h              |  27 +-
- drivers/net/enic/enic_ethdev.c       |  67 +----
- drivers/net/enic/enic_main.c         | 132 +++------
- drivers/net/enic/enic_res.h          |  81 +-----
- drivers/net/enic/enic_rx.c           | 361 -------------------------
- drivers/net/enic/enic_rxtx.c         | 505 +++++++++++++++++++++++++++++++++++
- 12 files changed, 635 insertions(+), 860 deletions(-)
- delete mode 100644 drivers/net/enic/base/enic_vnic_wq.h
- delete mode 100644 drivers/net/enic/enic_rx.c
- create mode 100644 drivers/net/enic/enic_rxtx.c
-
-diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile
-index f316274..3926b79 100644
---- a/drivers/net/enic/Makefile
-+++ b/drivers/net/enic/Makefile
-@@ -53,7 +53,7 @@ VPATH += $(SRCDIR)/src
- #
- SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c
- SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c
--SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c
-+SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rxtx.c
- SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c
- SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c
- SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c
-diff --git a/drivers/net/enic/base/enic_vnic_wq.h b/drivers/net/enic/base/enic_vnic_wq.h
-deleted file mode 100644
-index b019109..0000000
---- a/drivers/net/enic/base/enic_vnic_wq.h
-+++ /dev/null
-@@ -1,79 +0,0 @@
--/*
-- * Copyright 2008-2015 Cisco Systems, Inc.  All rights reserved.
-- * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
-- *
-- * Copyright (c) 2015, Cisco Systems, Inc.
-- * All rights reserved.
-- *
-- * Redistribution and use in source and binary forms, with or without
-- * modification, are permitted provided that the following conditions
-- * are met:
-- *
-- * 1. Redistributions of source code must retain the above copyright
-- * notice, this list of conditions and the following disclaimer.
-- *
-- * 2. Redistributions in binary form must reproduce the above copyright
-- * notice, this list of conditions and the following disclaimer in
-- * the documentation and/or other materials provided with the
-- * distribution.
-- *
-- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-- * POSSIBILITY OF SUCH DAMAGE.
-- *
-- */
--
--#ifndef _ENIC_VNIC_WQ_H_
--#define _ENIC_VNIC_WQ_H_
--
--#include "vnic_dev.h"
--#include "vnic_cq.h"
--
--static inline void enic_vnic_post_wq_index(struct vnic_wq *wq)
--{
--	struct vnic_wq_buf *buf = wq->to_use;
--
--	/* Adding write memory barrier prevents compiler and/or CPU
--	 * reordering, thus avoiding descriptor posting before
--	 * descriptor is initialized. Otherwise, hardware can read
--	 * stale descriptor fields.
--	*/
--	wmb();
--	iowrite32(buf->index, &wq->ctrl->posted_index);
--}
--
--static inline void enic_vnic_post_wq(struct vnic_wq *wq,
--				     void *os_buf, dma_addr_t dma_addr,
--				     unsigned int len, int sop,
--				     uint8_t desc_skip_cnt, uint8_t cq_entry,
--				     uint8_t compressed_send, uint64_t wrid)
--{
--	struct vnic_wq_buf *buf = wq->to_use;
--
--	buf->sop = sop;
--	buf->cq_entry = cq_entry;
--	buf->compressed_send = compressed_send;
--	buf->desc_skip_cnt = desc_skip_cnt;
--	buf->os_buf = os_buf;
--	buf->dma_addr = dma_addr;
--	buf->len = len;
--	buf->wr_id = wrid;
--
--	buf = buf->next;
--	wq->ring.desc_avail -= desc_skip_cnt;
--	wq->to_use = buf;
--
--	if (cq_entry)
--		enic_vnic_post_wq_index(wq);
--}
--
--#endif /* _ENIC_VNIC_WQ_H_ */
-diff --git a/drivers/net/enic/base/vnic_cq.h b/drivers/net/enic/base/vnic_cq.h
-index 922391b..ffc1aaa 100644
---- a/drivers/net/enic/base/vnic_cq.h
-+++ b/drivers/net/enic/base/vnic_cq.h
-@@ -96,41 +96,46 @@ static inline unsigned int vnic_cq_service(struct vnic_cq *cq,
- 	u8 type, u16 q_number, u16 completed_index, void *opaque),
- 	void *opaque)
- {
--	struct cq_desc *cq_desc;
-+	struct cq_desc *cq_desc, *cq_desc_last;
- 	unsigned int work_done = 0;
- 	u16 q_number, completed_index;
--	u8 type, color;
--	struct rte_mbuf **rx_pkts = opaque;
--	unsigned int ret;
-+	u8 type, color, type_color;
- 
- 	cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs +
- 		cq->ring.desc_size * cq->to_clean);
--	cq_desc_dec(cq_desc, &type, &color,
--		&q_number, &completed_index);
-+
-+	type_color = cq_desc->type_color;
-+	color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK;
-+	if (color == cq->last_color)
-+		return 0;
- 
- 	while (color != cq->last_color) {
--		if (opaque)
--			opaque = (void *)&(rx_pkts[work_done]);
-+		cq_desc_last = cq_desc;
- 
--		ret = (*q_service)(cq->vdev, cq_desc, type,
--			q_number, completed_index, opaque);
- 		cq->to_clean++;
- 		if (cq->to_clean == cq->ring.desc_count) {
- 			cq->to_clean = 0;
- 			cq->last_color = cq->last_color ? 0 : 1;
- 		}
- 
-+		work_done++;
-+		if (work_done >= work_to_do)
-+			break;
-+
- 		cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs +
- 			cq->ring.desc_size * cq->to_clean);
--		cq_desc_dec(cq_desc, &type, &color,
--			&q_number, &completed_index);
- 
--		if (ret)
--			work_done++;
--		if (work_done >= work_to_do)
--			break;
-+		type_color = cq_desc->type_color;
-+		color = (type_color >> CQ_DESC_COLOR_SHIFT)
-+			& CQ_DESC_COLOR_MASK;
-+
- 	}
- 
-+	cq_desc_dec(cq_desc_last, &type, &color,
-+		&q_number, &completed_index);
-+
-+	(*q_service)(cq->vdev, cq_desc, type,
-+		q_number, completed_index, opaque);
- 	return work_done;
- }
- 
-diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h
-index e083ccc..424415c 100644
---- a/drivers/net/enic/base/vnic_rq.h
-+++ b/drivers/net/enic/base/vnic_rq.h
-@@ -74,7 +74,7 @@ struct vnic_rq {
- 	struct vnic_dev_ring ring;
- 	struct rte_mbuf **mbuf_ring;		/* array of allocated mbufs */
- 	unsigned int mbuf_next_idx;		/* next mb to consume */
--	void *os_buf_head;
-+	void *mb_head;
- 	unsigned int pkts_outstanding;
- 	uint16_t rx_nb_hold;
- 	uint16_t rx_free_thresh;
-diff --git a/drivers/net/enic/base/vnic_wq.c b/drivers/net/enic/base/vnic_wq.c
-index a3ef417..ccbbd61 100644
---- a/drivers/net/enic/base/vnic_wq.c
-+++ b/drivers/net/enic/base/vnic_wq.c
-@@ -59,71 +59,30 @@ int vnic_wq_alloc_ring(struct vnic_dev *vdev, struct vnic_wq *wq,
- 
- static int vnic_wq_alloc_bufs(struct vnic_wq *wq)
- {
--	struct vnic_wq_buf *buf;
--	unsigned int i, j, count = wq->ring.desc_count;
--	unsigned int blks = VNIC_WQ_BUF_BLKS_NEEDED(count);
--
--	for (i = 0; i < blks; i++) {
--		wq->bufs[i] = kzalloc(VNIC_WQ_BUF_BLK_SZ(count), GFP_ATOMIC);
--		if (!wq->bufs[i])
--			return -ENOMEM;
--	}
--
--	for (i = 0; i < blks; i++) {
--		buf = wq->bufs[i];
--		for (j = 0; j < VNIC_WQ_BUF_BLK_ENTRIES(count); j++) {
--			buf->index = i * VNIC_WQ_BUF_BLK_ENTRIES(count) + j;
--			buf->desc = (u8 *)wq->ring.descs +
--				wq->ring.desc_size * buf->index;
--			if (buf->index + 1 == count) {
--				buf->next = wq->bufs[0];
--				break;
--			} else if (j + 1 == VNIC_WQ_BUF_BLK_ENTRIES(count)) {
--				buf->next = wq->bufs[i + 1];
--			} else {
--				buf->next = buf + 1;
--				buf++;
--			}
--		}
--	}
--
--	wq->to_use = wq->to_clean = wq->bufs[0];
--
-+	unsigned int count = wq->ring.desc_count;
-+       /* Allocate the mbuf ring */
-+	wq->bufs = (struct vnic_wq_buf *)rte_zmalloc_socket("wq->bufs",
-+		    sizeof(struct vnic_wq_buf) * count,
-+		    RTE_CACHE_LINE_SIZE, wq->socket_id);
-+	wq->head_idx = 0;
-+	wq->tail_idx = 0;
-+	if (wq->bufs == NULL)
-+		return -ENOMEM;
- 	return 0;
- }
- 
- void vnic_wq_free(struct vnic_wq *wq)
- {
- 	struct vnic_dev *vdev;
--	unsigned int i;
- 
- 	vdev = wq->vdev;
- 
- 	vnic_dev_free_desc_ring(vdev, &wq->ring);
- 
--	for (i = 0; i < VNIC_WQ_BUF_BLKS_MAX; i++) {
--		if (wq->bufs[i]) {
--			kfree(wq->bufs[i]);
--			wq->bufs[i] = NULL;
--		}
--	}
--
-+	rte_free(wq->bufs);
- 	wq->ctrl = NULL;
- }
- 
--int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count,
--	unsigned int desc_size)
--{
--	int mem_size = 0;
--
--	mem_size += vnic_dev_desc_ring_size(&wq->ring, desc_count, desc_size);
--
--	mem_size += VNIC_WQ_BUF_BLKS_NEEDED(wq->ring.desc_count) *
--		VNIC_WQ_BUF_BLK_SZ(wq->ring.desc_count);
--
--	return mem_size;
--}
--
- 
- int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index,
- 	unsigned int desc_count, unsigned int desc_size)
-@@ -172,9 +131,8 @@ void vnic_wq_init_start(struct vnic_wq *wq, unsigned int cq_index,
- 	iowrite32(error_interrupt_offset, &wq->ctrl->error_interrupt_offset);
- 	iowrite32(0, &wq->ctrl->error_status);
- 
--	wq->to_use = wq->to_clean =
--		&wq->bufs[fetch_index / VNIC_WQ_BUF_BLK_ENTRIES(count)]
--			[fetch_index % VNIC_WQ_BUF_BLK_ENTRIES(count)];
-+	wq->head_idx = fetch_index;
-+	wq->tail_idx = wq->head_idx;
- }
- 
- void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
-@@ -184,6 +142,7 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
- 	vnic_wq_init_start(wq, cq_index, 0, 0,
- 		error_interrupt_enable,
- 		error_interrupt_offset);
-+	wq->last_completed_index = 0;
- }
- 
- void vnic_wq_error_out(struct vnic_wq *wq, unsigned int error)
-@@ -219,22 +178,34 @@ int vnic_wq_disable(struct vnic_wq *wq)
- 	return -ETIMEDOUT;
- }
- 
-+static inline uint32_t
-+buf_idx_incr(uint32_t n_descriptors, uint32_t idx)
-+{
-+	idx++;
-+	if (unlikely(idx == n_descriptors))
-+		idx = 0;
-+	return idx;
-+}
-+
- void vnic_wq_clean(struct vnic_wq *wq,
--	void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf))
-+		   void (*buf_clean)(struct vnic_wq_buf *buf))
- {
- 	struct vnic_wq_buf *buf;
-+	unsigned int  to_clean = wq->tail_idx;
- 
--	buf = wq->to_clean;
-+	buf = &wq->bufs[to_clean];
- 
- 	while (vnic_wq_desc_used(wq) > 0) {
- 
--		(*buf_clean)(wq, buf);
-+		(*buf_clean)(buf);
-+		to_clean = buf_idx_incr(wq->ring.desc_count, to_clean);
- 
--		buf = wq->to_clean = buf->next;
-+		buf = &wq->bufs[to_clean];
- 		wq->ring.desc_avail++;
- 	}
- 
--	wq->to_use = wq->to_clean = wq->bufs[0];
-+	wq->head_idx = 0;
-+	wq->tail_idx = 0;
- 
- 	iowrite32(0, &wq->ctrl->fetch_index);
- 	iowrite32(0, &wq->ctrl->posted_index);
-diff --git a/drivers/net/enic/base/vnic_wq.h b/drivers/net/enic/base/vnic_wq.h
-index c23de62..37c3ff9 100644
---- a/drivers/net/enic/base/vnic_wq.h
-+++ b/drivers/net/enic/base/vnic_wq.h
-@@ -64,42 +64,23 @@ struct vnic_wq_ctrl {
- 	u32 pad9;
- };
- 
-+/* 16 bytes */
- struct vnic_wq_buf {
--	struct vnic_wq_buf *next;
--	dma_addr_t dma_addr;
--	void *os_buf;
--	unsigned int len;
--	unsigned int index;
--	int sop;
--	void *desc;
--	uint64_t wr_id; /* Cookie */
--	uint8_t cq_entry; /* Gets completion event from hw */
--	uint8_t desc_skip_cnt; /* Num descs to occupy */
--	uint8_t compressed_send; /* Both hdr and payload in one desc */
-+	struct rte_mempool *pool;
-+	void *mb;
- };
- 
--/* Break the vnic_wq_buf allocations into blocks of 32/64 entries */
--#define VNIC_WQ_BUF_MIN_BLK_ENTRIES 32
--#define VNIC_WQ_BUF_DFLT_BLK_ENTRIES 64
--#define VNIC_WQ_BUF_BLK_ENTRIES(entries) \
--	((unsigned int)((entries < VNIC_WQ_BUF_DFLT_BLK_ENTRIES) ? \
--	VNIC_WQ_BUF_MIN_BLK_ENTRIES : VNIC_WQ_BUF_DFLT_BLK_ENTRIES))
--#define VNIC_WQ_BUF_BLK_SZ(entries) \
--	(VNIC_WQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_wq_buf))
--#define VNIC_WQ_BUF_BLKS_NEEDED(entries) \
--	DIV_ROUND_UP(entries, VNIC_WQ_BUF_BLK_ENTRIES(entries))
--#define VNIC_WQ_BUF_BLKS_MAX VNIC_WQ_BUF_BLKS_NEEDED(4096)
--
- struct vnic_wq {
- 	unsigned int index;
- 	struct vnic_dev *vdev;
- 	struct vnic_wq_ctrl __iomem *ctrl;              /* memory-mapped */
- 	struct vnic_dev_ring ring;
--	struct vnic_wq_buf *bufs[VNIC_WQ_BUF_BLKS_MAX];
--	struct vnic_wq_buf *to_use;
--	struct vnic_wq_buf *to_clean;
--	unsigned int pkts_outstanding;
-+	struct vnic_wq_buf *bufs;
-+	unsigned int head_idx;
-+	unsigned int tail_idx;
- 	unsigned int socket_id;
-+	const struct rte_memzone *cqmsg_rz;
-+	uint16_t last_completed_index;
- };
- 
- static inline unsigned int vnic_wq_desc_avail(struct vnic_wq *wq)
-@@ -114,11 +95,6 @@ static inline unsigned int vnic_wq_desc_used(struct vnic_wq *wq)
- 	return wq->ring.desc_count - wq->ring.desc_avail - 1;
- }
- 
--static inline void *vnic_wq_next_desc(struct vnic_wq *wq)
--{
--	return wq->to_use->desc;
--}
--
- #define PI_LOG2_CACHE_LINE_SIZE        5
- #define PI_INDEX_BITS            12
- #define PI_INDEX_MASK ((1U << PI_INDEX_BITS) - 1)
-@@ -191,75 +167,6 @@ static inline u64 vnic_cached_posted_index(dma_addr_t addr, unsigned int len,
- 	PI_PREFETCH_ADDR_MASK) << PI_PREFETCH_ADDR_OFF);
- }
- 
--static inline void vnic_wq_post(struct vnic_wq *wq,
--	void *os_buf, dma_addr_t dma_addr,
--	unsigned int len, int sop, int eop,
--	uint8_t desc_skip_cnt, uint8_t cq_entry,
--	uint8_t compressed_send, uint64_t wrid)
--{
--	struct vnic_wq_buf *buf = wq->to_use;
--
--	buf->sop = sop;
--	buf->cq_entry = cq_entry;
--	buf->compressed_send = compressed_send;
--	buf->desc_skip_cnt = desc_skip_cnt;
--	buf->os_buf = os_buf;
--	buf->dma_addr = dma_addr;
--	buf->len = len;
--	buf->wr_id = wrid;
--
--	buf = buf->next;
--	if (eop) {
--#ifdef DO_PREFETCH
--		uint64_t wr = vnic_cached_posted_index(dma_addr, len,
--							buf->index);
--#endif
--		/* Adding write memory barrier prevents compiler and/or CPU
--		 * reordering, thus avoiding descriptor posting before
--		 * descriptor is initialized. Otherwise, hardware can read
--		 * stale descriptor fields.
--		 */
--		wmb();
--#ifdef DO_PREFETCH
--		/* Intel chipsets seem to limit the rate of PIOs that we can
--		 * push on the bus.  Thus, it is very important to do a single
--		 * 64 bit write here.  With two 32-bit writes, my maximum
--		 * pkt/sec rate was cut almost in half. -AJF
--		 */
--		iowrite64((uint64_t)wr, &wq->ctrl->posted_index);
--#else
--		iowrite32(buf->index, &wq->ctrl->posted_index);
--#endif
--	}
--	wq->to_use = buf;
--
--	wq->ring.desc_avail -= desc_skip_cnt;
--}
--
--static inline void vnic_wq_service(struct vnic_wq *wq,
--	struct cq_desc *cq_desc, u16 completed_index,
--	void (*buf_service)(struct vnic_wq *wq,
--	struct cq_desc *cq_desc, struct vnic_wq_buf *buf, void *opaque),
--	void *opaque)
--{
--	struct vnic_wq_buf *buf;
--
--	buf = wq->to_clean;
--	while (1) {
--
--		(*buf_service)(wq, cq_desc, buf, opaque);
--
--		wq->ring.desc_avail++;
--
--		wq->to_clean = buf->next;
--
--		if (buf->index == completed_index)
--			break;
--
--		buf = wq->to_clean;
--	}
--}
--
- void vnic_wq_free(struct vnic_wq *wq);
- int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index,
- 	unsigned int desc_count, unsigned int desc_size);
-@@ -275,8 +182,6 @@ unsigned int vnic_wq_error_status(struct vnic_wq *wq);
- void vnic_wq_enable(struct vnic_wq *wq);
- int vnic_wq_disable(struct vnic_wq *wq);
- void vnic_wq_clean(struct vnic_wq *wq,
--	void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf));
--int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count,
--	unsigned int desc_size);
-+	void (*buf_clean)(struct vnic_wq_buf *buf));
- 
- #endif /* _VNIC_WQ_H_ */
-diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
-index 8c914f5..43b82a6 100644
---- a/drivers/net/enic/enic.h
-+++ b/drivers/net/enic/enic.h
-@@ -155,6 +155,30 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev)
- 	return (struct enic *)eth_dev->data->dev_private;
- }
- 
-+static inline uint32_t
-+enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
-+{
-+	uint32_t d = i0 + i1;
-+	d -= (d >= n_descriptors) ? n_descriptors : 0;
-+	return d;
-+}
-+
-+static inline uint32_t
-+enic_ring_sub(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
-+{
-+	int32_t d = i1 - i0;
-+	return (uint32_t)((d < 0) ? ((int32_t)n_descriptors + d) : d);
-+}
-+
-+static inline uint32_t
-+enic_ring_incr(uint32_t n_descriptors, uint32_t idx)
-+{
-+	idx++;
-+	if (unlikely(idx == n_descriptors))
-+		idx = 0;
-+	return idx;
-+}
-+
- #define RTE_LIBRTE_ENIC_ASSERT_ENABLE
- #ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE
- #define ASSERT(x) do {			\
-@@ -209,5 +233,6 @@ extern int enic_clsf_init(struct enic *enic);
- extern void enic_clsf_destroy(struct enic *enic);
- uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
- 			uint16_t nb_pkts);
--
-+uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
-+			uint16_t nb_pkts);
- #endif /* _ENIC_H_ */
-diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c
-index 6bea940..697ff82 100644
---- a/drivers/net/enic/enic_ethdev.c
-+++ b/drivers/net/enic/enic_ethdev.c
-@@ -519,71 +519,6 @@ static void enicpmd_remove_mac_addr(struct rte_eth_dev *eth_dev, __rte_unused ui
- 	enic_del_mac_address(enic);
- }
- 
--
--static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
--	uint16_t nb_pkts)
--{
--	uint16_t index;
--	unsigned int frags;
--	unsigned int pkt_len;
--	unsigned int seg_len;
--	unsigned int inc_len;
--	unsigned int nb_segs;
--	struct rte_mbuf *tx_pkt, *next_tx_pkt;
--	struct vnic_wq *wq = (struct vnic_wq *)tx_queue;
--	struct enic *enic = vnic_dev_priv(wq->vdev);
--	unsigned short vlan_id;
--	unsigned short ol_flags;
--	uint8_t last_seg, eop;
--	unsigned int host_tx_descs = 0;
--
--	for (index = 0; index < nb_pkts; index++) {
--		tx_pkt = *tx_pkts++;
--		inc_len = 0;
--		nb_segs = tx_pkt->nb_segs;
--		if (nb_segs > vnic_wq_desc_avail(wq)) {
--			if (index > 0)
--				enic_post_wq_index(wq);
--
--			/* wq cleanup and try again */
--			if (!enic_cleanup_wq(enic, wq) ||
--				(nb_segs > vnic_wq_desc_avail(wq))) {
--				return index;
--			}
--		}
--
--		pkt_len = tx_pkt->pkt_len;
--		vlan_id = tx_pkt->vlan_tci;
--		ol_flags = tx_pkt->ol_flags;
--		for (frags = 0; inc_len < pkt_len; frags++) {
--			if (!tx_pkt)
--				break;
--			next_tx_pkt = tx_pkt->next;
--			seg_len = tx_pkt->data_len;
--			inc_len += seg_len;
--
--			host_tx_descs++;
--			last_seg = 0;
--			eop = 0;
--			if ((pkt_len == inc_len) || !next_tx_pkt) {
--				eop = 1;
--				/* post if last packet in batch or > thresh */
--				if ((index == (nb_pkts - 1)) ||
--				   (host_tx_descs > ENIC_TX_POST_THRESH)) {
--					last_seg = 1;
--					host_tx_descs = 0;
--				}
--			}
--			enic_send_pkt(enic, wq, tx_pkt, (unsigned short)seg_len,
--				      !frags, eop, last_seg, ol_flags, vlan_id);
--			tx_pkt = next_tx_pkt;
--		}
--	}
--
--	enic_cleanup_wq(enic, wq);
--	return index;
--}
--
- static const struct eth_dev_ops enicpmd_eth_dev_ops = {
- 	.dev_configure        = enicpmd_dev_configure,
- 	.dev_start            = enicpmd_dev_start,
-@@ -642,7 +577,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev)
- 	enic->rte_dev = eth_dev;
- 	eth_dev->dev_ops = &enicpmd_eth_dev_ops;
- 	eth_dev->rx_pkt_burst = &enic_recv_pkts;
--	eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts;
-+	eth_dev->tx_pkt_burst = &enic_xmit_pkts;
- 
- 	pdev = eth_dev->pci_dev;
- 	rte_eth_copy_pci_info(eth_dev, pdev);
-diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
-index 646d87f..ba73604 100644
---- a/drivers/net/enic/enic_main.c
-+++ b/drivers/net/enic/enic_main.c
-@@ -40,11 +40,11 @@
- #include <libgen.h>
- 
- #include <rte_pci.h>
--#include <rte_memzone.h>
- #include <rte_malloc.h>
- #include <rte_mbuf.h>
- #include <rte_string_fns.h>
- #include <rte_ethdev.h>
-+#include <rte_memzone.h>
- 
- #include "enic_compat.h"
- #include "enic.h"
-@@ -58,7 +58,6 @@
- #include "vnic_cq.h"
- #include "vnic_intr.h"
- #include "vnic_nic.h"
--#include "enic_vnic_wq.h"
- 
- static inline struct rte_mbuf *
- rte_rxmbuf_alloc(struct rte_mempool *mp)
-@@ -109,38 +108,17 @@ enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq)
- 	}
- }
- 
--
- void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size)
- {
- 	vnic_set_hdr_split_size(enic->vdev, split_hdr_size);
- }
- 
--static void enic_free_wq_buf(__rte_unused struct vnic_wq *wq, struct vnic_wq_buf *buf)
-+static void enic_free_wq_buf(struct vnic_wq_buf *buf)
- {
--	struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->os_buf;
-+	struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb;
- 
- 	rte_mempool_put(mbuf->pool, mbuf);
--	buf->os_buf = NULL;
--}
--
--static void enic_wq_free_buf(struct vnic_wq *wq,
--	__rte_unused struct cq_desc *cq_desc,
--	struct vnic_wq_buf *buf,
--	__rte_unused void *opaque)
--{
--	enic_free_wq_buf(wq, buf);
--}
--
--static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
--	__rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque)
--{
--	struct enic *enic = vnic_dev_priv(vdev);
--
--	vnic_wq_service(&enic->wq[q_number], cq_desc,
--		completed_index, enic_wq_free_buf,
--		opaque);
--
--	return 0;
-+	buf->mb = NULL;
- }
- 
- static void enic_log_q_error(struct enic *enic)
-@@ -163,64 +141,6 @@ static void enic_log_q_error(struct enic *enic)
- 	}
- }
- 
--unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq)
--{
--	unsigned int cq = enic_cq_wq(enic, wq->index);
--
--	/* Return the work done */
--	return vnic_cq_service(&enic->cq[cq],
--		-1 /*wq_work_to_do*/, enic_wq_service, NULL);
--}
--
--void enic_post_wq_index(struct vnic_wq *wq)
--{
--	enic_vnic_post_wq_index(wq);
--}
--
--void enic_send_pkt(struct enic *enic, struct vnic_wq *wq,
--		   struct rte_mbuf *tx_pkt, unsigned short len,
--		   uint8_t sop, uint8_t eop, uint8_t cq_entry,
--		   uint16_t ol_flags, uint16_t vlan_tag)
--{
--	struct wq_enet_desc *desc = vnic_wq_next_desc(wq);
--	uint16_t mss = 0;
--	uint8_t vlan_tag_insert = 0;
--	uint64_t bus_addr = (dma_addr_t)
--	    (tx_pkt->buf_physaddr + tx_pkt->data_off);
--
--	if (sop) {
--		if (ol_flags & PKT_TX_VLAN_PKT)
--			vlan_tag_insert = 1;
--
--		if (enic->hw_ip_checksum) {
--			if (ol_flags & PKT_TX_IP_CKSUM)
--				mss |= ENIC_CALC_IP_CKSUM;
--
--			if (ol_flags & PKT_TX_TCP_UDP_CKSUM)
--				mss |= ENIC_CALC_TCP_UDP_CKSUM;
--		}
--	}
--
--	wq_enet_desc_enc(desc,
--		bus_addr,
--		len,
--		mss,
--		0 /* header_length */,
--		0 /* offload_mode WQ_ENET_OFFLOAD_MODE_CSUM */,
--		eop,
--		cq_entry,
--		0 /* fcoe_encap */,
--		vlan_tag_insert,
--		vlan_tag,
--		0 /* loopback */);
--
--	enic_vnic_post_wq(wq, (void *)tx_pkt, bus_addr, len,
--			  sop,
--			  1 /*desc_skip_cnt*/,
--			  cq_entry,
--			  0 /*compressed send*/,
--			  0 /*wrid*/);
--}
- 
- void enic_dev_stats_clear(struct enic *enic)
- {
-@@ -297,12 +217,28 @@ void enic_init_vnic_resources(struct enic *enic)
- 	unsigned int error_interrupt_enable = 1;
- 	unsigned int error_interrupt_offset = 0;
- 	unsigned int index = 0;
-+	unsigned int cq_idx;
-+
-+	vnic_dev_stats_clear(enic->vdev);
- 
- 	for (index = 0; index < enic->rq_count; index++) {
- 		vnic_rq_init(&enic->rq[index],
- 			enic_cq_rq(enic, index),
- 			error_interrupt_enable,
- 			error_interrupt_offset);
-+
-+		cq_idx = enic_cq_rq(enic, index);
-+		vnic_cq_init(&enic->cq[cq_idx],
-+			0 /* flow_control_enable */,
-+			1 /* color_enable */,
-+			0 /* cq_head */,
-+			0 /* cq_tail */,
-+			1 /* cq_tail_color */,
-+			0 /* interrupt_enable */,
-+			1 /* cq_entry_enable */,
-+			0 /* cq_message_enable */,
-+			0 /* interrupt offset */,
-+			0 /* cq_message_addr */);
- 	}
- 
- 	for (index = 0; index < enic->wq_count; index++) {
-@@ -310,22 +246,19 @@ void enic_init_vnic_resources(struct enic *enic)
- 			enic_cq_wq(enic, index),
- 			error_interrupt_enable,
- 			error_interrupt_offset);
--	}
--
--	vnic_dev_stats_clear(enic->vdev);
- 
--	for (index = 0; index < enic->cq_count; index++) {
--		vnic_cq_init(&enic->cq[index],
-+		cq_idx = enic_cq_wq(enic, index);
-+		vnic_cq_init(&enic->cq[cq_idx],
- 			0 /* flow_control_enable */,
- 			1 /* color_enable */,
- 			0 /* cq_head */,
- 			0 /* cq_tail */,
- 			1 /* cq_tail_color */,
- 			0 /* interrupt_enable */,
--			1 /* cq_entry_enable */,
--			0 /* cq_message_enable */,
-+			0 /* cq_entry_enable */,
-+			1 /* cq_message_enable */,
- 			0 /* interrupt offset */,
--			0 /* cq_message_addr */);
-+			(u64)enic->wq[index].cqmsg_rz->phys_addr);
- 	}
- 
- 	vnic_intr_init(&enic->intr,
-@@ -569,6 +502,7 @@ void enic_free_wq(void *txq)
- 	struct vnic_wq *wq = (struct vnic_wq *)txq;
- 	struct enic *enic = vnic_dev_priv(wq->vdev);
- 
-+	rte_memzone_free(wq->cqmsg_rz);
- 	vnic_wq_free(wq);
- 	vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
- }
-@@ -579,6 +513,8 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
- 	int err;
- 	struct vnic_wq *wq = &enic->wq[queue_idx];
- 	unsigned int cq_index = enic_cq_wq(enic, queue_idx);
-+	char name[NAME_MAX];
-+	static int instance;
- 
- 	wq->socket_id = socket_id;
- 	if (nb_desc) {
-@@ -614,6 +550,18 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
- 		dev_err(enic, "error in allocation of cq for wq\n");
- 	}
- 
-+	/* setup up CQ message */
-+	snprintf((char *)name, sizeof(name),
-+		 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
-+		instance++);
-+
-+	wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
-+						   sizeof(uint32_t),
-+						   SOCKET_ID_ANY, 0,
-+						   ENIC_ALIGN);
-+	if (!wq->cqmsg_rz)
-+		return -ENOMEM;
-+
- 	return err;
- }
- 
-diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h
-index 00fa71d..3e1bdf5 100644
---- a/drivers/net/enic/enic_res.h
-+++ b/drivers/net/enic/enic_res.h
-@@ -53,89 +53,10 @@
- 
- #define ENIC_NON_TSO_MAX_DESC		16
- #define ENIC_DEFAULT_RX_FREE_THRESH	32
--#define ENIC_TX_POST_THRESH		(ENIC_MIN_WQ_DESCS / 2)
-+#define ENIC_TX_XMIT_MAX		64
- 
- #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0)
- 
--static inline void enic_queue_wq_desc_ex(struct vnic_wq *wq,
--	void *os_buf, dma_addr_t dma_addr, unsigned int len,
--	unsigned int mss_or_csum_offset, unsigned int hdr_len,
--	int vlan_tag_insert, unsigned int vlan_tag,
--	int offload_mode, int cq_entry, int sop, int eop, int loopback)
--{
--	struct wq_enet_desc *desc = vnic_wq_next_desc(wq);
--	u8 desc_skip_cnt = 1;
--	u8 compressed_send = 0;
--	u64 wrid = 0;
--
--	wq_enet_desc_enc(desc,
--		(u64)dma_addr | VNIC_PADDR_TARGET,
--		(u16)len,
--		(u16)mss_or_csum_offset,
--		(u16)hdr_len, (u8)offload_mode,
--		(u8)eop, (u8)cq_entry,
--		0, /* fcoe_encap */
--		(u8)vlan_tag_insert,
--		(u16)vlan_tag,
--		(u8)loopback);
--
--	vnic_wq_post(wq, os_buf, dma_addr, len, sop, eop, desc_skip_cnt,
--			(u8)cq_entry, compressed_send, wrid);
--}
--
--static inline void enic_queue_wq_desc_cont(struct vnic_wq *wq,
--	void *os_buf, dma_addr_t dma_addr, unsigned int len,
--	int eop, int loopback)
--{
--	enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
--		0, 0, 0, 0, 0,
--		eop, 0 /* !SOP */, eop, loopback);
--}
--
--static inline void enic_queue_wq_desc(struct vnic_wq *wq, void *os_buf,
--	dma_addr_t dma_addr, unsigned int len, int vlan_tag_insert,
--	unsigned int vlan_tag, int eop, int loopback)
--{
--	enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
--		0, 0, vlan_tag_insert, vlan_tag,
--		WQ_ENET_OFFLOAD_MODE_CSUM,
--		eop, 1 /* SOP */, eop, loopback);
--}
--
--static inline void enic_queue_wq_desc_csum(struct vnic_wq *wq,
--	void *os_buf, dma_addr_t dma_addr, unsigned int len,
--	int ip_csum, int tcpudp_csum, int vlan_tag_insert,
--	unsigned int vlan_tag, int eop, int loopback)
--{
--	enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
--		(ip_csum ? 1 : 0) + (tcpudp_csum ? 2 : 0),
--		0, vlan_tag_insert, vlan_tag,
--		WQ_ENET_OFFLOAD_MODE_CSUM,
--		eop, 1 /* SOP */, eop, loopback);
--}
--
--static inline void enic_queue_wq_desc_csum_l4(struct vnic_wq *wq,
--	void *os_buf, dma_addr_t dma_addr, unsigned int len,
--	unsigned int csum_offset, unsigned int hdr_len,
--	int vlan_tag_insert, unsigned int vlan_tag, int eop, int loopback)
--{
--	enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
--		csum_offset, hdr_len, vlan_tag_insert, vlan_tag,
--		WQ_ENET_OFFLOAD_MODE_CSUM_L4,
--		eop, 1 /* SOP */, eop, loopback);
--}
--
--static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq,
--	void *os_buf, dma_addr_t dma_addr, unsigned int len,
--	unsigned int mss, unsigned int hdr_len, int vlan_tag_insert,
--	unsigned int vlan_tag, int eop, int loopback)
--{
--	enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
--		mss, hdr_len, vlan_tag_insert, vlan_tag,
--		WQ_ENET_OFFLOAD_MODE_TSO,
--		eop, 1 /* SOP */, eop, loopback);
--}
--
- struct enic;
- 
- int enic_get_vnic_config(struct enic *);
-diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c
-deleted file mode 100644
-index 39bb55c..0000000
---- a/drivers/net/enic/enic_rx.c
-+++ /dev/null
-@@ -1,361 +0,0 @@
--/*
-- * Copyright 2008-2014 Cisco Systems, Inc.  All rights reserved.
-- * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
-- *
-- * Copyright (c) 2014, Cisco Systems, Inc.
-- * All rights reserved.
-- *
-- * Redistribution and use in source and binary forms, with or without
-- * modification, are permitted provided that the following conditions
-- * are met:
-- *
-- * 1. Redistributions of source code must retain the above copyright
-- * notice, this list of conditions and the following disclaimer.
-- *
-- * 2. Redistributions in binary form must reproduce the above copyright
-- * notice, this list of conditions and the following disclaimer in
-- * the documentation and/or other materials provided with the
-- * distribution.
-- *
-- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-- * POSSIBILITY OF SUCH DAMAGE.
-- *
-- */
--
--#include <rte_mbuf.h>
--#include <rte_ethdev.h>
--#include <rte_prefetch.h>
--
--#include "enic_compat.h"
--#include "rq_enet_desc.h"
--#include "enic.h"
--
--#define RTE_PMD_USE_PREFETCH
--
--#ifdef RTE_PMD_USE_PREFETCH
--/*
-- * Prefetch a cache line into all cache levels.
-- */
--#define rte_enic_prefetch(p) rte_prefetch0(p)
--#else
--#define rte_enic_prefetch(p) do {} while (0)
--#endif
--
--#ifdef RTE_PMD_PACKET_PREFETCH
--#define rte_packet_prefetch(p) rte_prefetch1(p)
--#else
--#define rte_packet_prefetch(p) do {} while (0)
--#endif
--
--static inline struct rte_mbuf *
--rte_rxmbuf_alloc(struct rte_mempool *mp)
--{
--	struct rte_mbuf *m;
--
--	m = __rte_mbuf_raw_alloc(mp);
--	__rte_mbuf_sanity_check_raw(m, 0);
--	return m;
--}
--
--static inline uint16_t
--enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
--{
--	return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
--}
--
--static inline uint16_t
--enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
--{
--	return(le16_to_cpu(crd->bytes_written_flags) &
--		~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK);
--}
--
--static inline uint8_t
--enic_cq_rx_desc_packet_error(uint16_t bwflags)
--{
--	return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
--		CQ_ENET_RQ_DESC_FLAGS_TRUNCATED);
--}
--
--static inline uint8_t
--enic_cq_rx_desc_eop(uint16_t ciflags)
--{
--	return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
--		== CQ_ENET_RQ_DESC_FLAGS_EOP;
--}
--
--static inline uint8_t
--enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
--{
--	return ((le16_to_cpu(cqrd->q_number_rss_type_flags) &
--		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
--		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC);
--}
--
--static inline uint8_t
--enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
--{
--	return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
--		CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK);
--}
--
--static inline uint8_t
--enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
--{
--	return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
--		CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK);
--}
--
--static inline uint8_t
--enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
--{
--	return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
--		CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
--}
--
--static inline uint32_t
--enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
--{
--	return le32_to_cpu(cqrd->rss_hash);
--}
--
--static inline uint16_t
--enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
--{
--	return le16_to_cpu(cqrd->vlan);
--}
--
--static inline uint16_t
--enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
--{
--	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
--	return le16_to_cpu(cqrd->bytes_written_flags) &
--		CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
--}
--
--static inline uint8_t
--enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out)
--{
--	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
--	uint16_t bwflags;
--	int ret = 0;
--	uint64_t pkt_err_flags = 0;
--
--	bwflags = enic_cq_rx_desc_bwflags(cqrd);
--	if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) {
--		pkt_err_flags = PKT_RX_MAC_ERR;
--		ret = 1;
--	}
--	*pkt_err_flags_out = pkt_err_flags;
--	return ret;
--}
--
--/*
-- * Lookup table to translate RX CQ flags to mbuf flags.
-- */
--static inline uint32_t
--enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd)
--{
--	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
--	uint8_t cqrd_flags = cqrd->flags;
--	static const uint32_t cq_type_table[128] __rte_cache_aligned = {
--		[32] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
--		[34] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
--			| RTE_PTYPE_L4_UDP,
--		[36] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
--			| RTE_PTYPE_L4_TCP,
--		[96] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
--			| RTE_PTYPE_L4_FRAG,
--		[16] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6,
--		[18] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
--			| RTE_PTYPE_L4_UDP,
--		[20] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
--			| RTE_PTYPE_L4_TCP,
--		[80] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
--			| RTE_PTYPE_L4_FRAG,
--		/* All others reserved */
--	};
--	cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
--		| CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
--		| CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
--	return cq_type_table[cqrd_flags];
--}
--
--static inline void
--enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
--{
--	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
--	uint16_t ciflags, bwflags, pkt_flags = 0;
--	ciflags = enic_cq_rx_desc_ciflags(cqrd);
--	bwflags = enic_cq_rx_desc_bwflags(cqrd);
--
--	mbuf->ol_flags = 0;
--
--	/* flags are meaningless if !EOP */
--	if (unlikely(!enic_cq_rx_desc_eop(ciflags)))
--		goto mbuf_flags_done;
--
--	/* VLAN stripping */
--	if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
--		pkt_flags |= PKT_RX_VLAN_PKT;
--		mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd);
--	} else {
--		mbuf->vlan_tci = 0;
--	}
--
--	/* RSS flag */
--	if (enic_cq_rx_desc_rss_type(cqrd)) {
--		pkt_flags |= PKT_RX_RSS_HASH;
--		mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
--	}
--
--	/* checksum flags */
--	if (!enic_cq_rx_desc_csum_not_calc(cqrd) &&
--		(mbuf->packet_type & RTE_PTYPE_L3_IPV4)) {
--		if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd)))
--			pkt_flags |= PKT_RX_IP_CKSUM_BAD;
--		if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) {
--			if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)))
--				pkt_flags |= PKT_RX_L4_CKSUM_BAD;
--		}
--	}
--
-- mbuf_flags_done:
--	mbuf->ol_flags = pkt_flags;
--}
--
--static inline uint32_t
--enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
--{
--	uint32_t d = i0 + i1;
--	ASSERT(i0 < n_descriptors);
--	ASSERT(i1 < n_descriptors);
--	d -= (d >= n_descriptors) ? n_descriptors : 0;
--	return d;
--}
--
--
--uint16_t
--enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
--	       uint16_t nb_pkts)
--{
--	struct vnic_rq *rq = rx_queue;
--	struct enic *enic = vnic_dev_priv(rq->vdev);
--	unsigned int rx_id;
--	struct rte_mbuf *nmb, *rxmb;
--	uint16_t nb_rx = 0;
--	uint16_t nb_hold;
--	struct vnic_cq *cq;
--	volatile struct cq_desc *cqd_ptr;
--	uint8_t color;
--
--	cq = &enic->cq[enic_cq_rq(enic, rq->index)];
--	rx_id = cq->to_clean;		/* index of cqd, rqd, mbuf_table */
--	cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
--
--	nb_hold = rq->rx_nb_hold;	/* mbufs held by software */
--
--	while (nb_rx < nb_pkts) {
--		volatile struct rq_enet_desc *rqd_ptr;
--		dma_addr_t dma_addr;
--		struct cq_desc cqd;
--		uint64_t ol_err_flags;
--		uint8_t packet_error;
--
--		/* Check for pkts available */
--		color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
--			& CQ_DESC_COLOR_MASK;
--		if (color == cq->last_color)
--			break;
--
--		/* Get the cq descriptor and rq pointer */
--		cqd = *cqd_ptr;
--		rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id;
--
--		/* allocate a new mbuf */
--		nmb = rte_rxmbuf_alloc(rq->mp);
--		if (nmb == NULL) {
--			dev_err(enic, "RX mbuf alloc failed port=%u qid=%u",
--			enic->port_id, (unsigned)rq->index);
--			rte_eth_devices[enic->port_id].
--					data->rx_mbuf_alloc_failed++;
--			break;
--		}
--
--		/* A packet error means descriptor and data are untrusted */
--		packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags);
--
--		/* Get the mbuf to return and replace with one just allocated */
--		rxmb = rq->mbuf_ring[rx_id];
--		rq->mbuf_ring[rx_id] = nmb;
--
--		/* Increment cqd, rqd, mbuf_table index */
--		rx_id++;
--		if (unlikely(rx_id == rq->ring.desc_count)) {
--			rx_id = 0;
--			cq->last_color = cq->last_color ? 0 : 1;
--		}
--
--		/* Prefetch next mbuf & desc while processing current one */
--		cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
--		rte_enic_prefetch(cqd_ptr);
--		rte_enic_prefetch(rq->mbuf_ring[rx_id]);
--		rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs)
--				 + rx_id);
--
--		/* Push descriptor for newly allocated mbuf */
--		dma_addr = (dma_addr_t)(nmb->buf_physaddr
--			   + RTE_PKTMBUF_HEADROOM);
--		rqd_ptr->address = rte_cpu_to_le_64(dma_addr);
--		rqd_ptr->length_type = cpu_to_le16(nmb->buf_len
--				       - RTE_PKTMBUF_HEADROOM);
--
--		/* Fill in the rest of the mbuf */
--		rxmb->data_off = RTE_PKTMBUF_HEADROOM;
--		rxmb->nb_segs = 1;
--		rxmb->next = NULL;
--		rxmb->port = enic->port_id;
--		if (!packet_error) {
--			rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd);
--			rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd);
--			enic_cq_rx_to_pkt_flags(&cqd, rxmb);
--		} else {
--			rxmb->pkt_len = 0;
--			rxmb->packet_type = 0;
--			rxmb->ol_flags = 0;
--		}
--		rxmb->data_len = rxmb->pkt_len;
--
--		/* prefetch mbuf data for caller */
--		rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr,
--				    RTE_PKTMBUF_HEADROOM));
--
--		/* store the mbuf address into the next entry of the array */
--		rx_pkts[nb_rx++] = rxmb;
--	}
--
--	nb_hold += nb_rx;
--	cq->to_clean = rx_id;
--
--	if (nb_hold > rq->rx_free_thresh) {
--		rq->posted_index = enic_ring_add(rq->ring.desc_count,
--				rq->posted_index, nb_hold);
--		nb_hold = 0;
--		rte_mb();
--		iowrite32(rq->posted_index, &rq->ctrl->posted_index);
--	}
--
--	rq->rx_nb_hold = nb_hold;
--
--	return nb_rx;
--}
-diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c
-new file mode 100644
-index 0000000..71ca34e
---- /dev/null
-+++ b/drivers/net/enic/enic_rxtx.c
-@@ -0,0 +1,505 @@
-+/*
-+ * Copyright 2008-2016 Cisco Systems, Inc.  All rights reserved.
-+ * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
-+ *
-+ * Copyright (c) 2016, Cisco Systems, Inc.
-+ * All rights reserved.
-+ *
-+ * Redistribution and use in source and binary forms, with or without
-+ * modification, are permitted provided that the following conditions
-+ * are met:
-+ *
-+ * 1. Redistributions of source code must retain the above copyright
-+ * notice, this list of conditions and the following disclaimer.
-+ *
-+ * 2. Redistributions in binary form must reproduce the above copyright
-+ * notice, this list of conditions and the following disclaimer in
-+ * the documentation and/or other materials provided with the
-+ * distribution.
-+ *
-+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-+ * POSSIBILITY OF SUCH DAMAGE.
-+ *
-+ */
-+
-+#include <rte_mbuf.h>
-+#include <rte_ethdev.h>
-+#include <rte_prefetch.h>
-+#include <rte_memzone.h>
-+
-+#include "enic_compat.h"
-+#include "rq_enet_desc.h"
-+#include "enic.h"
-+
-+#define RTE_PMD_USE_PREFETCH
-+
-+#ifdef RTE_PMD_USE_PREFETCH
-+/*
-+ * Prefetch a cache line into all cache levels.
-+ */
-+#define rte_enic_prefetch(p) rte_prefetch0(p)
-+#else
-+#define rte_enic_prefetch(p) do {} while (0)
-+#endif
-+
-+#ifdef RTE_PMD_PACKET_PREFETCH
-+#define rte_packet_prefetch(p) rte_prefetch1(p)
-+#else
-+#define rte_packet_prefetch(p) do {} while (0)
-+#endif
-+
-+static inline struct rte_mbuf *
-+rte_rxmbuf_alloc(struct rte_mempool *mp)
-+{
-+	struct rte_mbuf *m;
-+
-+	m = __rte_mbuf_raw_alloc(mp);
-+	__rte_mbuf_sanity_check_raw(m, 0);
-+	return m;
-+}
-+
-+static inline uint16_t
-+enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
-+{
-+	return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
-+}
-+
-+static inline uint16_t
-+enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
-+{
-+	return(le16_to_cpu(crd->bytes_written_flags) &
-+		~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK);
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_packet_error(uint16_t bwflags)
-+{
-+	return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
-+		CQ_ENET_RQ_DESC_FLAGS_TRUNCATED);
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_eop(uint16_t ciflags)
-+{
-+	return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
-+		== CQ_ENET_RQ_DESC_FLAGS_EOP;
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
-+{
-+	return ((le16_to_cpu(cqrd->q_number_rss_type_flags) &
-+		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
-+		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC);
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
-+{
-+	return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
-+		CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK);
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
-+{
-+	return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
-+		CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK);
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
-+{
-+	return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
-+		CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
-+}
-+
-+static inline uint32_t
-+enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
-+{
-+	return le32_to_cpu(cqrd->rss_hash);
-+}
-+
-+static inline uint16_t
-+enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
-+{
-+	return le16_to_cpu(cqrd->vlan);
-+}
-+
-+static inline uint16_t
-+enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
-+{
-+	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
-+	return le16_to_cpu(cqrd->bytes_written_flags) &
-+		CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
-+}
-+
-+static inline uint8_t
-+enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out)
-+{
-+	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
-+	uint16_t bwflags;
-+	int ret = 0;
-+	uint64_t pkt_err_flags = 0;
-+
-+	bwflags = enic_cq_rx_desc_bwflags(cqrd);
-+	if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) {
-+		pkt_err_flags = PKT_RX_MAC_ERR;
-+		ret = 1;
-+	}
-+	*pkt_err_flags_out = pkt_err_flags;
-+	return ret;
-+}
-+
-+/*
-+ * Lookup table to translate RX CQ flags to mbuf flags.
-+ */
-+static inline uint32_t
-+enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd)
-+{
-+	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
-+	uint8_t cqrd_flags = cqrd->flags;
-+	static const uint32_t cq_type_table[128] __rte_cache_aligned = {
-+		[32] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
-+		[34] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
-+			| RTE_PTYPE_L4_UDP,
-+		[36] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
-+			| RTE_PTYPE_L4_TCP,
-+		[96] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
-+			| RTE_PTYPE_L4_FRAG,
-+		[16] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6,
-+		[18] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
-+			| RTE_PTYPE_L4_UDP,
-+		[20] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
-+			| RTE_PTYPE_L4_TCP,
-+		[80] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
-+			| RTE_PTYPE_L4_FRAG,
-+		/* All others reserved */
-+	};
-+	cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
-+		| CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
-+		| CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
-+	return cq_type_table[cqrd_flags];
-+}
-+
-+static inline void
-+enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
-+{
-+	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
-+	uint16_t ciflags, bwflags, pkt_flags = 0;
-+	ciflags = enic_cq_rx_desc_ciflags(cqrd);
-+	bwflags = enic_cq_rx_desc_bwflags(cqrd);
-+
-+	mbuf->ol_flags = 0;
-+
-+	/* flags are meaningless if !EOP */
-+	if (unlikely(!enic_cq_rx_desc_eop(ciflags)))
-+		goto mbuf_flags_done;
-+
-+	/* VLAN stripping */
-+	if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
-+		pkt_flags |= PKT_RX_VLAN_PKT;
-+		mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd);
-+	} else {
-+		mbuf->vlan_tci = 0;
-+	}
-+
-+	/* RSS flag */
-+	if (enic_cq_rx_desc_rss_type(cqrd)) {
-+		pkt_flags |= PKT_RX_RSS_HASH;
-+		mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
-+	}
-+
-+	/* checksum flags */
-+	if (!enic_cq_rx_desc_csum_not_calc(cqrd) &&
-+		(mbuf->packet_type & RTE_PTYPE_L3_IPV4)) {
-+		if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd)))
-+			pkt_flags |= PKT_RX_IP_CKSUM_BAD;
-+		if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) {
-+			if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)))
-+				pkt_flags |= PKT_RX_L4_CKSUM_BAD;
-+		}
-+	}
-+
-+ mbuf_flags_done:
-+	mbuf->ol_flags = pkt_flags;
-+}
-+
-+uint16_t
-+enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
-+	       uint16_t nb_pkts)
-+{
-+	struct vnic_rq *rq = rx_queue;
-+	struct enic *enic = vnic_dev_priv(rq->vdev);
-+	unsigned int rx_id;
-+	struct rte_mbuf *nmb, *rxmb;
-+	uint16_t nb_rx = 0;
-+	uint16_t nb_hold;
-+	struct vnic_cq *cq;
-+	volatile struct cq_desc *cqd_ptr;
-+	uint8_t color;
-+
-+	cq = &enic->cq[enic_cq_rq(enic, rq->index)];
-+	rx_id = cq->to_clean;		/* index of cqd, rqd, mbuf_table */
-+	cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
-+
-+	nb_hold = rq->rx_nb_hold;	/* mbufs held by software */
-+
-+	while (nb_rx < nb_pkts) {
-+		volatile struct rq_enet_desc *rqd_ptr;
-+		dma_addr_t dma_addr;
-+		struct cq_desc cqd;
-+		uint64_t ol_err_flags;
-+		uint8_t packet_error;
-+
-+		/* Check for pkts available */
-+		color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
-+			& CQ_DESC_COLOR_MASK;
-+		if (color == cq->last_color)
-+			break;
-+
-+		/* Get the cq descriptor and rq pointer */
-+		cqd = *cqd_ptr;
-+		rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id;
-+
-+		/* allocate a new mbuf */
-+		nmb = rte_rxmbuf_alloc(rq->mp);
-+		if (nmb == NULL) {
-+			dev_err(enic, "RX mbuf alloc failed port=%u qid=%u",
-+			enic->port_id, (unsigned)rq->index);
-+			rte_eth_devices[enic->port_id].
-+					data->rx_mbuf_alloc_failed++;
-+			break;
-+		}
-+
-+		/* A packet error means descriptor and data are untrusted */
-+		packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags);
-+
-+		/* Get the mbuf to return and replace with one just allocated */
-+		rxmb = rq->mbuf_ring[rx_id];
-+		rq->mbuf_ring[rx_id] = nmb;
-+
-+		/* Increment cqd, rqd, mbuf_table index */
-+		rx_id++;
-+		if (unlikely(rx_id == rq->ring.desc_count)) {
-+			rx_id = 0;
-+			cq->last_color = cq->last_color ? 0 : 1;
-+		}
-+
-+		/* Prefetch next mbuf & desc while processing current one */
-+		cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
-+		rte_enic_prefetch(cqd_ptr);
-+		rte_enic_prefetch(rq->mbuf_ring[rx_id]);
-+		rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs)
-+				 + rx_id);
-+
-+		/* Push descriptor for newly allocated mbuf */
-+		dma_addr = (dma_addr_t)(nmb->buf_physaddr
-+			   + RTE_PKTMBUF_HEADROOM);
-+		rqd_ptr->address = rte_cpu_to_le_64(dma_addr);
-+		rqd_ptr->length_type = cpu_to_le16(nmb->buf_len
-+				       - RTE_PKTMBUF_HEADROOM);
-+
-+		/* Fill in the rest of the mbuf */
-+		rxmb->data_off = RTE_PKTMBUF_HEADROOM;
-+		rxmb->nb_segs = 1;
-+		rxmb->next = NULL;
-+		rxmb->port = enic->port_id;
-+		if (!packet_error) {
-+			rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd);
-+			rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd);
-+			enic_cq_rx_to_pkt_flags(&cqd, rxmb);
-+		} else {
-+			rxmb->pkt_len = 0;
-+			rxmb->packet_type = 0;
-+			rxmb->ol_flags = 0;
-+		}
-+		rxmb->data_len = rxmb->pkt_len;
-+
-+		/* prefetch mbuf data for caller */
-+		rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr,
-+				    RTE_PKTMBUF_HEADROOM));
-+
-+		/* store the mbuf address into the next entry of the array */
-+		rx_pkts[nb_rx++] = rxmb;
-+	}
-+
-+	nb_hold += nb_rx;
-+	cq->to_clean = rx_id;
-+
-+	if (nb_hold > rq->rx_free_thresh) {
-+		rq->posted_index = enic_ring_add(rq->ring.desc_count,
-+				rq->posted_index, nb_hold);
-+		nb_hold = 0;
-+		rte_mb();
-+		iowrite32(rq->posted_index, &rq->ctrl->posted_index);
-+	}
-+
-+	rq->rx_nb_hold = nb_hold;
-+
-+	return nb_rx;
-+}
-+
-+static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index)
-+{
-+	struct vnic_wq_buf *buf;
-+	struct rte_mbuf *m, *free[ENIC_MAX_WQ_DESCS];
-+	unsigned int nb_to_free, nb_free = 0, i;
-+	struct rte_mempool *pool;
-+	unsigned int tail_idx;
-+	unsigned int desc_count = wq->ring.desc_count;
-+
-+	nb_to_free = enic_ring_sub(desc_count, wq->tail_idx, completed_index)
-+				   + 1;
-+	tail_idx = wq->tail_idx;
-+	buf = &wq->bufs[tail_idx];
-+	pool = ((struct rte_mbuf *)buf->mb)->pool;
-+	for (i = 0; i < nb_to_free; i++) {
-+		buf = &wq->bufs[tail_idx];
-+		m = (struct rte_mbuf *)(buf->mb);
-+		if (likely(m->pool == pool)) {
-+			ASSERT(nb_free < ENIC_MAX_WQ_DESCS);
-+			free[nb_free++] = m;
-+		} else {
-+			rte_mempool_put_bulk(pool, (void *)free, nb_free);
-+			free[0] = m;
-+			nb_free = 1;
-+			pool = m->pool;
-+		}
-+		tail_idx = enic_ring_incr(desc_count, tail_idx);
-+		buf->mb = NULL;
-+	}
-+
-+	rte_mempool_put_bulk(pool, (void **)free, nb_free);
-+
-+	wq->tail_idx = tail_idx;
-+	wq->ring.desc_avail += nb_to_free;
-+}
-+
-+unsigned int enic_cleanup_wq(__rte_unused struct enic *enic, struct vnic_wq *wq)
-+{
-+	u16 completed_index;
-+
-+	completed_index = *((uint32_t *)wq->cqmsg_rz->addr) & 0xffff;
-+
-+	if (wq->last_completed_index != completed_index) {
-+		enic_free_wq_bufs(wq, completed_index);
-+		wq->last_completed_index = completed_index;
-+	}
-+	return 0;
-+}
-+
-+uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
-+	uint16_t nb_pkts)
-+{
-+	uint16_t index;
-+	unsigned int pkt_len, data_len;
-+	unsigned int nb_segs;
-+	struct rte_mbuf *tx_pkt;
-+	struct vnic_wq *wq = (struct vnic_wq *)tx_queue;
-+	struct enic *enic = vnic_dev_priv(wq->vdev);
-+	unsigned short vlan_id;
-+	unsigned short ol_flags;
-+	unsigned int wq_desc_avail;
-+	int head_idx;
-+	struct vnic_wq_buf *buf;
-+	unsigned int hw_ip_cksum_enabled;
-+	unsigned int desc_count;
-+	struct wq_enet_desc *descs, *desc_p, desc_tmp;
-+	uint16_t mss;
-+	uint8_t vlan_tag_insert;
-+	uint8_t eop;
-+	uint64_t bus_addr;
-+
-+	enic_cleanup_wq(enic, wq);
-+	wq_desc_avail = vnic_wq_desc_avail(wq);
-+	head_idx = wq->head_idx;
-+	desc_count = wq->ring.desc_count;
-+
-+	nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX);
-+
-+	hw_ip_cksum_enabled = enic->hw_ip_checksum;
-+	for (index = 0; index < nb_pkts; index++) {
-+		tx_pkt = *tx_pkts++;
-+		nb_segs = tx_pkt->nb_segs;
-+		if (nb_segs > wq_desc_avail) {
-+			if (index > 0)
-+				goto post;
-+			goto done;
-+		}
-+
-+		pkt_len = tx_pkt->pkt_len;
-+		data_len = tx_pkt->data_len;
-+		vlan_id = tx_pkt->vlan_tci;
-+		ol_flags = tx_pkt->ol_flags;
-+
-+		mss = 0;
-+		vlan_tag_insert = 0;
-+		bus_addr = (dma_addr_t)
-+			   (tx_pkt->buf_physaddr + tx_pkt->data_off);
-+
-+		descs = (struct wq_enet_desc *)wq->ring.descs;
-+		desc_p = descs + head_idx;
-+
-+		eop = (data_len == pkt_len);
-+
-+		if (ol_flags & PKT_TX_VLAN_PKT)
-+			vlan_tag_insert = 1;
-+
-+		if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_IP_CKSUM))
-+			mss |= ENIC_CALC_IP_CKSUM;
-+
-+		if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_TCP_UDP_CKSUM))
-+			mss |= ENIC_CALC_TCP_UDP_CKSUM;
-+
-+		wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, 0, 0, eop,
-+				 eop, 0, vlan_tag_insert, vlan_id, 0);
-+
-+		*desc_p = desc_tmp;
-+		buf = &wq->bufs[head_idx];
-+		buf->mb = (void *)tx_pkt;
-+		head_idx = enic_ring_incr(desc_count, head_idx);
-+		wq_desc_avail--;
-+
-+		if (!eop) {
-+			for (tx_pkt = tx_pkt->next; tx_pkt; tx_pkt =
-+			    tx_pkt->next) {
-+				data_len = tx_pkt->data_len;
-+
-+				if (tx_pkt->next == NULL)
-+					eop = 1;
-+				desc_p = descs + head_idx;
-+				bus_addr = (dma_addr_t)(tx_pkt->buf_physaddr
-+					   + tx_pkt->data_off);
-+				wq_enet_desc_enc((struct wq_enet_desc *)
-+						 &desc_tmp, bus_addr, data_len,
-+						 mss, 0, 0, eop, eop, 0,
-+						 vlan_tag_insert, vlan_id, 0);
-+
-+				*desc_p = desc_tmp;
-+				buf = &wq->bufs[head_idx];
-+				buf->mb = (void *)tx_pkt;
-+				head_idx = enic_ring_incr(desc_count, head_idx);
-+				wq_desc_avail--;
-+			}
-+		}
-+	}
-+ post:
-+	rte_wmb();
-+	iowrite32(head_idx, &wq->ctrl->posted_index);
-+ done:
-+	wq->ring.desc_avail = wq_desc_avail;
-+	wq->head_idx = head_idx;
-+
-+	return index;
-+}
--- 
-2.7.0
-