diff options
Diffstat (limited to 'dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch')
| -rw-r--r-- | dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch | 1349 |
1 files changed, 0 insertions, 1349 deletions
diff --git a/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch b/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch deleted file mode 100644 index 2aa4840d..00000000 --- a/dpdk/dpdk-2.2.0_patches/0011-enic-improve-Rx-performance.patch +++ /dev/null @@ -1,1349 +0,0 @@ -From 057358356e7d05f07ab2df37c12b1cce37a3cca9 Mon Sep 17 00:00:00 2001 -From: John Daley <johndale@cisco.com> -Date: Fri, 4 Mar 2016 13:09:00 -0800 -Subject: [PATCH 11/22] enic: improve Rx performance - - This is a wholesale replacement of the Enic PMD receive path in order - to improve performance and code clarity. The changes are: - - Simplify and reduce code path length of receive function. - - Put most of the fast-path receive functions in one file. - - Reduce the number of posted_index updates (pay attention to - rx_free_thresh) - - Remove the unneeded container structure around the RQ mbuf ring - - Prefetch next Mbuf and descriptors while processing the current one - - Use a lookup table for converting CQ flags to mbuf flags. - - Signed-off-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/Makefile | 1 + - drivers/net/enic/base/vnic_rq.c | 99 ++--------- - drivers/net/enic/base/vnic_rq.h | 147 +--------------- - drivers/net/enic/enic.h | 16 +- - drivers/net/enic/enic_ethdev.c | 27 ++- - drivers/net/enic/enic_main.c | 321 ++++++++++------------------------ - drivers/net/enic/enic_res.h | 16 +- - drivers/net/enic/enic_rx.c | 370 ++++++++++++++++++++++++++++++++++++++++ - 8 files changed, 511 insertions(+), 486 deletions(-) - create mode 100644 drivers/net/enic/enic_rx.c - -diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile -index f0ee093..f316274 100644 ---- a/drivers/net/enic/Makefile -+++ b/drivers/net/enic/Makefile -@@ -53,6 +53,7 @@ VPATH += $(SRCDIR)/src - # - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c -+SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c -diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c -index 1441604..cb62c5e 100644 ---- a/drivers/net/enic/base/vnic_rq.c -+++ b/drivers/net/enic/base/vnic_rq.c -@@ -35,77 +35,21 @@ - #include "vnic_dev.h" - #include "vnic_rq.h" - --static int vnic_rq_alloc_bufs(struct vnic_rq *rq) --{ -- struct vnic_rq_buf *buf; -- unsigned int i, j, count = rq->ring.desc_count; -- unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count); -- -- for (i = 0; i < blks; i++) { -- rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ(count), GFP_ATOMIC); -- if (!rq->bufs[i]) -- return -ENOMEM; -- } -- -- for (i = 0; i < blks; i++) { -- buf = rq->bufs[i]; -- for (j = 0; j < VNIC_RQ_BUF_BLK_ENTRIES(count); j++) { -- buf->index = i * VNIC_RQ_BUF_BLK_ENTRIES(count) + j; -- buf->desc = (u8 *)rq->ring.descs + -- rq->ring.desc_size * buf->index; -- if (buf->index + 1 == count) { -- buf->next = rq->bufs[0]; -- break; -- } else if (j + 1 == VNIC_RQ_BUF_BLK_ENTRIES(count)) { -- buf->next = rq->bufs[i + 1]; -- } else { -- buf->next = buf + 1; -- buf++; -- } -- } -- } -- -- rq->to_use = rq->to_clean = rq->bufs[0]; -- -- return 0; --} -- --int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count, -- unsigned int desc_size) --{ -- int mem_size = 0; -- -- mem_size += vnic_dev_desc_ring_size(&rq->ring, desc_count, desc_size); -- -- mem_size += VNIC_RQ_BUF_BLKS_NEEDED(rq->ring.desc_count) * -- VNIC_RQ_BUF_BLK_SZ(rq->ring.desc_count); -- -- return mem_size; --} -- - void vnic_rq_free(struct vnic_rq *rq) - { - struct vnic_dev *vdev; -- unsigned int i; - - vdev = rq->vdev; - - vnic_dev_free_desc_ring(vdev, &rq->ring); - -- for (i = 0; i < VNIC_RQ_BUF_BLKS_MAX; i++) { -- if (rq->bufs[i]) { -- kfree(rq->bufs[i]); -- rq->bufs[i] = NULL; -- } -- } -- - rq->ctrl = NULL; - } - - int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, - unsigned int desc_count, unsigned int desc_size) - { -- int err; -+ int rc; - char res_name[NAME_MAX]; - static int instance; - -@@ -121,18 +65,9 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, - vnic_rq_disable(rq); - - snprintf(res_name, sizeof(res_name), "%d-rq-%d", instance++, index); -- err = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, -+ rc = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, - rq->socket_id, res_name); -- if (err) -- return err; -- -- err = vnic_rq_alloc_bufs(rq); -- if (err) { -- vnic_rq_free(rq); -- return err; -- } -- -- return 0; -+ return rc; - } - - void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, -@@ -154,9 +89,6 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, - iowrite32(fetch_index, &rq->ctrl->fetch_index); - iowrite32(posted_index, &rq->ctrl->posted_index); - -- rq->to_use = rq->to_clean = -- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] -- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; - } - - void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, -@@ -176,6 +108,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, - fetch_index, fetch_index, - error_interrupt_enable, - error_interrupt_offset); -+ rq->rxst_idx = 0; -+ rq->tot_pkts = 0; - } - - void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error) -@@ -212,21 +146,20 @@ int vnic_rq_disable(struct vnic_rq *rq) - } - - void vnic_rq_clean(struct vnic_rq *rq, -- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)) -+ void (*buf_clean)(struct rte_mbuf **buf)) - { -- struct vnic_rq_buf *buf; -- u32 fetch_index; -+ struct rte_mbuf **buf; -+ u32 fetch_index, i; - unsigned int count = rq->ring.desc_count; - -- buf = rq->to_clean; -- -- while (vnic_rq_desc_used(rq) > 0) { -+ buf = &rq->mbuf_ring[0]; - -- (*buf_clean)(rq, buf); -- -- buf = rq->to_clean = buf->next; -- rq->ring.desc_avail++; -+ for (i = 0; i < count; i++) { -+ (*buf_clean)(buf); -+ buf++; - } -+ rq->ring.desc_avail = count - 1; -+ rq->rx_nb_hold = 0; - - /* Use current fetch_index as the ring starting point */ - fetch_index = ioread32(&rq->ctrl->fetch_index); -@@ -235,9 +168,7 @@ void vnic_rq_clean(struct vnic_rq *rq, - /* Hardware surprise removal: reset fetch_index */ - fetch_index = 0; - } -- rq->to_use = rq->to_clean = -- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] -- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; -+ - iowrite32(fetch_index, &rq->ctrl->posted_index); - - vnic_dev_clear_desc_ring(&rq->ring); -diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h -index 0f5c3c1..e083ccc 100644 ---- a/drivers/net/enic/base/vnic_rq.h -+++ b/drivers/net/enic/base/vnic_rq.h -@@ -66,42 +66,22 @@ struct vnic_rq_ctrl { - u32 pad10; - }; - --/* Break the vnic_rq_buf allocations into blocks of 32/64 entries */ --#define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32 --#define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64 --#define VNIC_RQ_BUF_BLK_ENTRIES(entries) \ -- ((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \ -- VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES)) --#define VNIC_RQ_BUF_BLK_SZ(entries) \ -- (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf)) --#define VNIC_RQ_BUF_BLKS_NEEDED(entries) \ -- DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries)) --#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096) -- --struct vnic_rq_buf { -- struct vnic_rq_buf *next; -- dma_addr_t dma_addr; -- void *os_buf; -- unsigned int os_buf_index; -- unsigned int len; -- unsigned int index; -- void *desc; -- uint64_t wr_id; --}; -- - struct vnic_rq { - unsigned int index; -+ unsigned int posted_index; - struct vnic_dev *vdev; -- struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ -+ struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ - struct vnic_dev_ring ring; -- struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX]; -- struct vnic_rq_buf *to_use; -- struct vnic_rq_buf *to_clean; -+ struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */ -+ unsigned int mbuf_next_idx; /* next mb to consume */ - void *os_buf_head; - unsigned int pkts_outstanding; -- -+ uint16_t rx_nb_hold; -+ uint16_t rx_free_thresh; - unsigned int socket_id; - struct rte_mempool *mp; -+ uint16_t rxst_idx; -+ uint32_t tot_pkts; - }; - - static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq) -@@ -116,119 +96,13 @@ static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq) - return rq->ring.desc_count - rq->ring.desc_avail - 1; - } - --static inline void *vnic_rq_next_desc(struct vnic_rq *rq) --{ -- return rq->to_use->desc; --} -- --static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq) --{ -- return rq->to_use->index; --} -- --static inline void vnic_rq_post(struct vnic_rq *rq, -- void *os_buf, unsigned int os_buf_index, -- dma_addr_t dma_addr, unsigned int len, -- uint64_t wrid) --{ -- struct vnic_rq_buf *buf = rq->to_use; -- -- buf->os_buf = os_buf; -- buf->os_buf_index = os_buf_index; -- buf->dma_addr = dma_addr; -- buf->len = len; -- buf->wr_id = wrid; -- -- buf = buf->next; -- rq->to_use = buf; -- rq->ring.desc_avail--; -- -- /* Move the posted_index every nth descriptor -- */ -- --#ifndef VNIC_RQ_RETURN_RATE --#define VNIC_RQ_RETURN_RATE 0xf /* keep 2^n - 1 */ --#endif -- -- if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) { -- /* Adding write memory barrier prevents compiler and/or CPU -- * reordering, thus avoiding descriptor posting before -- * descriptor is initialized. Otherwise, hardware can read -- * stale descriptor fields. -- */ -- wmb(); -- iowrite32(buf->index, &rq->ctrl->posted_index); -- } --} -- --static inline void vnic_rq_post_commit(struct vnic_rq *rq, -- void *os_buf, unsigned int os_buf_index, -- dma_addr_t dma_addr, unsigned int len) --{ -- struct vnic_rq_buf *buf = rq->to_use; -- -- buf->os_buf = os_buf; -- buf->os_buf_index = os_buf_index; -- buf->dma_addr = dma_addr; -- buf->len = len; -- -- buf = buf->next; -- rq->to_use = buf; -- rq->ring.desc_avail--; -- -- /* Move the posted_index every descriptor -- */ -- -- /* Adding write memory barrier prevents compiler and/or CPU -- * reordering, thus avoiding descriptor posting before -- * descriptor is initialized. Otherwise, hardware can read -- * stale descriptor fields. -- */ -- wmb(); -- iowrite32(buf->index, &rq->ctrl->posted_index); --} - --static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count) --{ -- rq->ring.desc_avail += count; --} - - enum desc_return_options { - VNIC_RQ_RETURN_DESC, - VNIC_RQ_DEFER_RETURN_DESC, - }; - --static inline int vnic_rq_service(struct vnic_rq *rq, -- struct cq_desc *cq_desc, u16 completed_index, -- int desc_return, int (*buf_service)(struct vnic_rq *rq, -- struct cq_desc *cq_desc, struct vnic_rq_buf *buf, -- int skipped, void *opaque), void *opaque) --{ -- struct vnic_rq_buf *buf; -- int skipped; -- int eop = 0; -- -- buf = rq->to_clean; -- while (1) { -- -- skipped = (buf->index != completed_index); -- -- if ((*buf_service)(rq, cq_desc, buf, skipped, opaque)) -- eop++; -- -- if (desc_return == VNIC_RQ_RETURN_DESC) -- rq->ring.desc_avail++; -- -- rq->to_clean = buf->next; -- -- if (!skipped) -- break; -- -- buf = rq->to_clean; -- } -- return eop; --} -- - static inline int vnic_rq_fill(struct vnic_rq *rq, - int (*buf_fill)(struct vnic_rq *rq)) - { -@@ -274,8 +148,5 @@ unsigned int vnic_rq_error_status(struct vnic_rq *rq); - void vnic_rq_enable(struct vnic_rq *rq); - int vnic_rq_disable(struct vnic_rq *rq); - void vnic_rq_clean(struct vnic_rq *rq, -- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)); --int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count, -- unsigned int desc_size); -- -+ void (*buf_clean)(struct rte_mbuf **buf)); - #endif /* _VNIC_RQ_H_ */ -diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h -index 9e78305..8c914f5 100644 ---- a/drivers/net/enic/enic.h -+++ b/drivers/net/enic/enic.h -@@ -45,6 +45,7 @@ - #include "vnic_nic.h" - #include "vnic_rss.h" - #include "enic_res.h" -+#include "cq_enet_desc.h" - - #define DRV_NAME "enic_pmd" - #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Poll-mode Driver" -@@ -154,6 +155,16 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) - return (struct enic *)eth_dev->data->dev_private; - } - -+#define RTE_LIBRTE_ENIC_ASSERT_ENABLE -+#ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE -+#define ASSERT(x) do { \ -+ if (!(x)) \ -+ rte_panic("ENIC: x"); \ -+} while (0) -+#else -+#define ASSERT(x) -+#endif -+ - extern void enic_fdir_stats_get(struct enic *enic, - struct rte_eth_fdir_stats *stats); - extern int enic_fdir_add_fltr(struct enic *enic, -@@ -193,9 +204,10 @@ extern void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, - uint16_t ol_flags, uint16_t vlan_tag); - - extern void enic_post_wq_index(struct vnic_wq *wq); --extern int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts, -- unsigned int budget, unsigned int *work_done); - extern int enic_probe(struct enic *enic); - extern int enic_clsf_init(struct enic *enic); - extern void enic_clsf_destroy(struct enic *enic); -+uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -+ uint16_t nb_pkts); -+ - #endif /* _ENIC_H_ */ -diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c -index 2a88043..6f2ada5 100644 ---- a/drivers/net/enic/enic_ethdev.c -+++ b/drivers/net/enic/enic_ethdev.c -@@ -255,7 +255,7 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, - uint16_t queue_idx, - uint16_t nb_desc, - unsigned int socket_id, -- __rte_unused const struct rte_eth_rxconf *rx_conf, -+ const struct rte_eth_rxconf *rx_conf, - struct rte_mempool *mp) - { - int ret; -@@ -270,6 +270,10 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, - return ret; - } - -+ enic->rq[queue_idx].rx_free_thresh = rx_conf->rx_free_thresh; -+ dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx, -+ enic->rq[queue_idx].rx_free_thresh); -+ - return enicpmd_dev_setup_intr(enic); - } - -@@ -429,6 +433,9 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev, - DEV_TX_OFFLOAD_IPV4_CKSUM | - DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_CKSUM; -+ device_info->default_rxconf = (struct rte_eth_rxconf) { -+ .rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH -+ }; - } - - static void enicpmd_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) -@@ -538,18 +545,6 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - return index; - } - --static uint16_t enicpmd_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -- uint16_t nb_pkts) --{ -- struct vnic_rq *rq = (struct vnic_rq *)rx_queue; -- unsigned int work_done; -- -- if (enic_poll(rq, rx_pkts, (unsigned int)nb_pkts, &work_done)) -- dev_err(enic, "error in enicpmd poll\n"); -- -- return work_done; --} -- - static const struct eth_dev_ops enicpmd_eth_dev_ops = { - .dev_configure = enicpmd_dev_configure, - .dev_start = enicpmd_dev_start, -@@ -606,7 +601,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) - enic->port_id = eth_dev->data->port_id; - enic->rte_dev = eth_dev; - eth_dev->dev_ops = &enicpmd_eth_dev_ops; -- eth_dev->rx_pkt_burst = &enicpmd_recv_pkts; -+ eth_dev->rx_pkt_burst = &enic_recv_pkts; - eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts; - - pdev = eth_dev->pci_dev; -@@ -635,8 +630,8 @@ static struct eth_driver rte_enic_pmd = { - * Register as the [Poll Mode] Driver of Cisco ENIC device. - */ - static int --rte_enic_pmd_init(const char *name __rte_unused, -- const char *params __rte_unused) -+rte_enic_pmd_init(__rte_unused const char *name, -+ __rte_unused const char *params) - { - ENICPMD_FUNC_TRACE(); - -diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index f818c32..9fff020 100644 ---- a/drivers/net/enic/enic_main.c -+++ b/drivers/net/enic/enic_main.c -@@ -60,6 +60,17 @@ - #include "vnic_nic.h" - #include "enic_vnic_wq.h" - -+static inline struct rte_mbuf * -+rte_rxmbuf_alloc(struct rte_mempool *mp) -+{ -+ struct rte_mbuf *m; -+ -+ m = __rte_mbuf_raw_alloc(mp); -+ __rte_mbuf_sanity_check_raw(m, 0); -+ return m; -+} -+ -+ - static inline int enic_is_sriov_vf(struct enic *enic) - { - return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF; -@@ -80,16 +91,25 @@ static int is_eth_addr_valid(uint8_t *addr) - return !is_mcast_addr(addr) && !is_zero_addr(addr); - } - --static inline struct rte_mbuf * --enic_rxmbuf_alloc(struct rte_mempool *mp) -+static void -+enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) - { -- struct rte_mbuf *m; -+ uint16_t i; - -- m = __rte_mbuf_raw_alloc(mp); -- __rte_mbuf_sanity_check_raw(m, 0); -- return m; -+ if (!rq || !rq->mbuf_ring) { -+ dev_debug(enic, "Pointer to rq or mbuf_ring is NULL"); -+ return; -+ } -+ -+ for (i = 0; i < enic->config.rq_desc_count; i++) { -+ if (rq->mbuf_ring[i]) { -+ rte_pktmbuf_free_seg(rq->mbuf_ring[i]); -+ rq->mbuf_ring[i] = NULL; -+ } -+ } - } - -+ - void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) - { - vnic_set_hdr_split_size(enic->vdev, split_hdr_size); -@@ -262,13 +282,13 @@ void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr) - } - - static void --enic_free_rq_buf(__rte_unused struct vnic_rq *rq, struct vnic_rq_buf *buf) -+enic_free_rq_buf(struct rte_mbuf **mbuf) - { -- if (!buf->os_buf) -+ if (*mbuf == NULL) - return; - -- rte_pktmbuf_free((struct rte_mbuf *)buf->os_buf); -- buf->os_buf = NULL; -+ rte_pktmbuf_free(*mbuf); -+ mbuf = NULL; - } - - void enic_init_vnic_resources(struct enic *enic) -@@ -314,221 +334,47 @@ void enic_init_vnic_resources(struct enic *enic) - } - - --static int enic_rq_alloc_buf(struct vnic_rq *rq) -+static int -+enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) - { -- struct enic *enic = vnic_dev_priv(rq->vdev); -+ struct rte_mbuf *mb; -+ struct rq_enet_desc *rqd = rq->ring.descs; -+ unsigned i; - dma_addr_t dma_addr; -- struct rq_enet_desc *desc = vnic_rq_next_desc(rq); -- uint8_t type = RQ_ENET_TYPE_ONLY_SOP; -- u16 split_hdr_size = vnic_get_hdr_split_size(enic->vdev); -- struct rte_mbuf *mbuf = enic_rxmbuf_alloc(rq->mp); -- struct rte_mbuf *hdr_mbuf = NULL; -- -- if (!mbuf) { -- dev_err(enic, "mbuf alloc in enic_rq_alloc_buf failed\n"); -- return -1; -- } -- -- if (unlikely(split_hdr_size)) { -- if (vnic_rq_desc_avail(rq) < 2) { -- rte_mempool_put(mbuf->pool, mbuf); -- return -1; -- } -- hdr_mbuf = enic_rxmbuf_alloc(rq->mp); -- if (!hdr_mbuf) { -- rte_mempool_put(mbuf->pool, mbuf); -- dev_err(enic, -- "hdr_mbuf alloc in enic_rq_alloc_buf failed\n"); -- return -1; -- } -- -- hdr_mbuf->data_off = RTE_PKTMBUF_HEADROOM; -- -- hdr_mbuf->nb_segs = 2; -- hdr_mbuf->port = enic->port_id; -- hdr_mbuf->next = mbuf; -- -- dma_addr = (dma_addr_t) -- (hdr_mbuf->buf_physaddr + hdr_mbuf->data_off); -- -- rq_enet_desc_enc(desc, dma_addr, type, split_hdr_size); - -- vnic_rq_post(rq, (void *)hdr_mbuf, 0 /*os_buf_index*/, dma_addr, -- (unsigned int)split_hdr_size, 0 /*wrid*/); -+ dev_debug(enic, "queue %u, allocating %u rx queue mbufs", rq->index, -+ rq->ring.desc_count); - -- desc = vnic_rq_next_desc(rq); -- type = RQ_ENET_TYPE_NOT_SOP; -- } else { -- mbuf->nb_segs = 1; -- mbuf->port = enic->port_id; -- } -- -- mbuf->data_off = RTE_PKTMBUF_HEADROOM; -- mbuf->next = NULL; -- -- dma_addr = (dma_addr_t) -- (mbuf->buf_physaddr + mbuf->data_off); -- -- rq_enet_desc_enc(desc, dma_addr, type, mbuf->buf_len); -- -- vnic_rq_post(rq, (void *)mbuf, 0 /*os_buf_index*/, dma_addr, -- (unsigned int)mbuf->buf_len, 0 /*wrid*/); -- -- return 0; --} -- --static int enic_rq_indicate_buf(struct vnic_rq *rq, -- struct cq_desc *cq_desc, struct vnic_rq_buf *buf, -- int skipped, void *opaque) --{ -- struct enic *enic = vnic_dev_priv(rq->vdev); -- struct rte_mbuf **rx_pkt_bucket = (struct rte_mbuf **)opaque; -- struct rte_mbuf *rx_pkt = NULL; -- struct rte_mbuf *hdr_rx_pkt = NULL; -- -- u8 type, color, eop, sop, ingress_port, vlan_stripped; -- u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof; -- u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok; -- u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc; -- u8 packet_error; -- u16 q_number, completed_index, bytes_written, vlan_tci, checksum; -- u32 rss_hash; -- -- cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, -- &type, &color, &q_number, &completed_index, -- &ingress_port, &fcoe, &eop, &sop, &rss_type, -- &csum_not_calc, &rss_hash, &bytes_written, -- &packet_error, &vlan_stripped, &vlan_tci, &checksum, -- &fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error, -- &fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp, -- &ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment, -- &fcs_ok); -- -- rx_pkt = (struct rte_mbuf *)buf->os_buf; -- buf->os_buf = NULL; -- -- if (unlikely(packet_error)) { -- dev_err(enic, "packet error\n"); -- rx_pkt->data_len = 0; -- return 0; -- } -- -- if (unlikely(skipped)) { -- rx_pkt->data_len = 0; -- return 0; -- } -- -- if (likely(!vnic_get_hdr_split_size(enic->vdev))) { -- /* No header split configured */ -- *rx_pkt_bucket = rx_pkt; -- rx_pkt->pkt_len = bytes_written; -- -- if (ipv4) { -- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4; -- if (!csum_not_calc) { -- if (unlikely(!ipv4_csum_ok)) -- rx_pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD; -- -- if ((tcp || udp) && (!tcp_udp_csum_ok)) -- rx_pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD; -- } -- } else if (ipv6) -- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6; -- } else { -- /* Header split */ -- if (sop && !eop) { -- /* This piece is header */ -- *rx_pkt_bucket = rx_pkt; -- rx_pkt->pkt_len = bytes_written; -- } else { -- if (sop && eop) { -- /* The packet is smaller than split_hdr_size */ -- *rx_pkt_bucket = rx_pkt; -- rx_pkt->pkt_len = bytes_written; -- if (ipv4) { -- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4; -- if (!csum_not_calc) { -- if (unlikely(!ipv4_csum_ok)) -- rx_pkt->ol_flags |= -- PKT_RX_IP_CKSUM_BAD; -- -- if ((tcp || udp) && -- (!tcp_udp_csum_ok)) -- rx_pkt->ol_flags |= -- PKT_RX_L4_CKSUM_BAD; -- } -- } else if (ipv6) -- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6; -- } else { -- /* Payload */ -- hdr_rx_pkt = *rx_pkt_bucket; -- hdr_rx_pkt->pkt_len += bytes_written; -- if (ipv4) { -- hdr_rx_pkt->packet_type = -- RTE_PTYPE_L3_IPV4; -- if (!csum_not_calc) { -- if (unlikely(!ipv4_csum_ok)) -- hdr_rx_pkt->ol_flags |= -- PKT_RX_IP_CKSUM_BAD; -- -- if ((tcp || udp) && -- (!tcp_udp_csum_ok)) -- hdr_rx_pkt->ol_flags |= -- PKT_RX_L4_CKSUM_BAD; -- } -- } else if (ipv6) -- hdr_rx_pkt->packet_type = -- RTE_PTYPE_L3_IPV6; -- } -+ for (i = 0; i < rq->ring.desc_count; i++, rqd++) { -+ mb = rte_rxmbuf_alloc(rq->mp); -+ if (mb == NULL) { -+ dev_err(enic, "RX mbuf alloc failed queue_id=%u", -+ (unsigned)rq->index); -+ return -ENOMEM; - } -- } - -- rx_pkt->data_len = bytes_written; -+ dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off); - -- if (rss_hash) { -- rx_pkt->ol_flags |= PKT_RX_RSS_HASH; -- rx_pkt->hash.rss = rss_hash; -+ rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, -+ mb->buf_len); -+ rq->mbuf_ring[i] = mb; - } - -- if (vlan_tci) { -- rx_pkt->ol_flags |= PKT_RX_VLAN_PKT; -- rx_pkt->vlan_tci = vlan_tci; -- } -+ /* make sure all prior writes are complete before doing the PIO write */ -+ rte_rmb(); - -- return eop; --} -+ /* Post all but the last 2 cache lines' worth of descriptors */ -+ rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE -+ / sizeof(struct rq_enet_desc)); -+ rq->rx_nb_hold = 0; - --static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, -- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque) --{ -- struct enic *enic = vnic_dev_priv(vdev); -- -- return vnic_rq_service(&enic->rq[q_number], cq_desc, -- completed_index, VNIC_RQ_RETURN_DESC, -- enic_rq_indicate_buf, opaque); -- --} -+ dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n", -+ enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold); -+ iowrite32(rq->posted_index, &rq->ctrl->posted_index); -+ rte_rmb(); - --int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts, -- unsigned int budget, unsigned int *work_done) --{ -- struct enic *enic = vnic_dev_priv(rq->vdev); -- unsigned int cq = enic_cq_rq(enic, rq->index); -- int err = 0; -- -- *work_done = vnic_cq_service(&enic->cq[cq], -- budget, enic_rq_service, (void *)rx_pkts); -- -- if (*work_done) { -- vnic_rq_fill(rq, enic_rq_alloc_buf); -+ return 0; - -- /* Need at least one buffer on ring to get going */ -- if (vnic_rq_desc_used(rq) == 0) { -- dev_err(enic, "Unable to alloc receive buffers\n"); -- err = -1; -- } -- } -- return err; - } - - static void * -@@ -576,6 +422,7 @@ enic_intr_handler(__rte_unused struct rte_intr_handle *handle, - int enic_enable(struct enic *enic) - { - unsigned int index; -+ int err; - struct rte_eth_dev *eth_dev = enic->rte_dev; - - eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev); -@@ -586,15 +433,11 @@ int enic_enable(struct enic *enic) - dev_warning(enic, "Init of hash table for clsf failed."\ - "Flow director feature will not work\n"); - -- /* Fill RQ bufs */ - for (index = 0; index < enic->rq_count; index++) { -- vnic_rq_fill(&enic->rq[index], enic_rq_alloc_buf); -- -- /* Need at least one buffer on ring to get going -- */ -- if (vnic_rq_desc_used(&enic->rq[index]) == 0) { -- dev_err(enic, "Unable to alloc receive buffers\n"); -- return -1; -+ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]); -+ if (err) { -+ dev_err(enic, "Failed to alloc RX queue mbufs\n"); -+ return err; - } - } - -@@ -636,6 +479,9 @@ void enic_free_rq(void *rxq) - struct vnic_rq *rq = (struct vnic_rq *)rxq; - struct enic *enic = vnic_dev_priv(rq->vdev); - -+ enic_rxmbuf_queue_release(enic, rq); -+ rte_free(rq->mbuf_ring); -+ rq->mbuf_ring = NULL; - vnic_rq_free(rq); - vnic_cq_free(&enic->cq[rq->index]); - } -@@ -664,7 +510,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, - unsigned int socket_id, struct rte_mempool *mp, - uint16_t nb_desc) - { -- int err; -+ int rc; - struct vnic_rq *rq = &enic->rq[queue_idx]; - - rq->socket_id = socket_id; -@@ -687,23 +533,35 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, - } - - /* Allocate queue resources */ -- err = vnic_rq_alloc(enic->vdev, &enic->rq[queue_idx], queue_idx, -- enic->config.rq_desc_count, -- sizeof(struct rq_enet_desc)); -- if (err) { -+ rc = vnic_rq_alloc(enic->vdev, rq, queue_idx, -+ enic->config.rq_desc_count, sizeof(struct rq_enet_desc)); -+ if (rc) { - dev_err(enic, "error in allocation of rq\n"); -- return err; -+ goto err_exit; - } - -- err = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, -+ rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, - socket_id, enic->config.rq_desc_count, - sizeof(struct cq_enet_rq_desc)); -- if (err) { -- vnic_rq_free(rq); -+ if (rc) { - dev_err(enic, "error in allocation of cq for rq\n"); -+ goto err_free_rq_exit; - } - -- return err; -+ /* Allocate the mbuf ring */ -+ rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", -+ sizeof(struct rte_mbuf *) * enic->config.rq_desc_count, -+ RTE_CACHE_LINE_SIZE, rq->socket_id); -+ -+ if (rq->mbuf_ring != NULL) -+ return 0; -+ -+ /* cleanup on error */ -+ vnic_cq_free(&enic->cq[queue_idx]); -+err_free_rq_exit: -+ vnic_rq_free(rq); -+err_exit: -+ return -ENOMEM; - } - - void enic_free_wq(void *txq) -@@ -790,6 +648,7 @@ int enic_disable(struct enic *enic) - - for (i = 0; i < enic->wq_count; i++) - vnic_wq_clean(&enic->wq[i], enic_free_wq_buf); -+ - for (i = 0; i < enic->rq_count; i++) - vnic_rq_clean(&enic->rq[i], enic_free_rq_buf); - for (i = 0; i < enic->cq_count; i++) -@@ -1074,7 +933,7 @@ int enic_probe(struct enic *enic) - - /* Set ingress vlan rewrite mode before vnic initialization */ - err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, -- IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN); -+ IG_VLAN_REWRITE_MODE_PASS_THRU); - if (err) { - dev_err(enic, - "Failed to set ingress vlan rewrite mode, aborting.\n"); -diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h -index 49f7e22..33f2e84 100644 ---- a/drivers/net/enic/enic_res.h -+++ b/drivers/net/enic/enic_res.h -@@ -52,6 +52,7 @@ - #define ENIC_UNICAST_PERFECT_FILTERS 32 - - #define ENIC_NON_TSO_MAX_DESC 16 -+#define ENIC_DEFAULT_RX_FREE_THRESH 32 - - #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) - -@@ -133,21 +134,6 @@ static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq, - WQ_ENET_OFFLOAD_MODE_TSO, - eop, 1 /* SOP */, eop, loopback); - } --static inline void enic_queue_rq_desc(struct vnic_rq *rq, -- void *os_buf, unsigned int os_buf_index, -- dma_addr_t dma_addr, unsigned int len) --{ -- struct rq_enet_desc *desc = vnic_rq_next_desc(rq); -- u64 wrid = 0; -- u8 type = os_buf_index ? -- RQ_ENET_TYPE_NOT_SOP : RQ_ENET_TYPE_ONLY_SOP; -- -- rq_enet_desc_enc(desc, -- (u64)dma_addr | VNIC_PADDR_TARGET, -- type, (u16)len); -- -- vnic_rq_post(rq, os_buf, os_buf_index, dma_addr, len, wrid); --} - - struct enic; - -diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c -new file mode 100644 -index 0000000..945a60f ---- /dev/null -+++ b/drivers/net/enic/enic_rx.c -@@ -0,0 +1,370 @@ -+/* -+ * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved. -+ * Copyright 2007 Nuova Systems, Inc. All rights reserved. -+ * -+ * Copyright (c) 2014, Cisco Systems, Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions -+ * are met: -+ * -+ * 1. Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * 2. Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in -+ * the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ * -+ */ -+ -+#include <rte_mbuf.h> -+#include <rte_ethdev.h> -+#include <rte_prefetch.h> -+ -+#include "enic_compat.h" -+#include "rq_enet_desc.h" -+#include "enic.h" -+ -+#define RTE_PMD_USE_PREFETCH -+ -+#ifdef RTE_PMD_USE_PREFETCH -+/* -+ * Prefetch a cache line into all cache levels. -+ */ -+#define rte_enic_prefetch(p) rte_prefetch0(p) -+#else -+#define rte_enic_prefetch(p) do {} while (0) -+#endif -+ -+#ifdef RTE_PMD_PACKET_PREFETCH -+#define rte_packet_prefetch(p) rte_prefetch1(p) -+#else -+#define rte_packet_prefetch(p) do {} while (0) -+#endif -+ -+static inline struct rte_mbuf * -+rte_rxmbuf_alloc(struct rte_mempool *mp) -+{ -+ struct rte_mbuf *m; -+ -+ m = __rte_mbuf_raw_alloc(mp); -+ __rte_mbuf_sanity_check_raw(m, 0); -+ return m; -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) -+{ -+ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) -+{ -+ return(le16_to_cpu(crd->bytes_written_flags) & -+ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_packet_error(uint16_t bwflags) -+{ -+ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == -+ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_eop(uint16_t ciflags) -+{ -+ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) -+ == CQ_ENET_RQ_DESC_FLAGS_EOP; -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & -+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == -+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) -+{ -+ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> -+ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); -+} -+ -+static inline uint32_t -+enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) -+{ -+ return le32_to_cpu(cqrd->rss_hash); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_fcs_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_FCS_OK); -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) -+{ -+ return le16_to_cpu(cqrd->vlan); -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ return le16_to_cpu(cqrd->bytes_written_flags) & -+ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; -+} -+ -+static inline uint64_t -+enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint16_t bwflags; -+ uint64_t pkt_err_flags = 0; -+ -+ bwflags = enic_cq_rx_desc_bwflags(cqrd); -+ -+ /* Check for packet error. Can't be more specific than MAC error */ -+ if (enic_cq_rx_desc_packet_error(bwflags)) { -+ pkt_err_flags |= PKT_RX_MAC_ERR; -+ } -+ -+ /* Check for bad FCS. MAC error isn't quite, but no other choice */ -+ if (!enic_cq_rx_desc_fcs_ok(cqrd)) { -+ pkt_err_flags |= PKT_RX_MAC_ERR; -+ } -+ return pkt_err_flags; -+} -+ -+/* -+ * Lookup table to translate RX CQ flags to mbuf flags. -+ */ -+static inline uint32_t -+enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint8_t cqrd_flags = cqrd->flags; -+ static const uint32_t cq_type_table[128] __rte_cache_aligned = { -+ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, -+ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_UDP, -+ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_TCP, -+ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_FRAG, -+ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, -+ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_UDP, -+ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_TCP, -+ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_FRAG, -+ /* All others reserved */ -+ }; -+ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT -+ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 -+ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; -+ return cq_type_table[cqrd_flags]; -+} -+ -+static inline void -+enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint16_t ciflags, bwflags, pkt_flags = 0; -+ ciflags = enic_cq_rx_desc_ciflags(cqrd); -+ bwflags = enic_cq_rx_desc_bwflags(cqrd); -+ -+ ASSERT(mbuf->ol_flags == 0); -+ -+ /* flags are meaningless if !EOP */ -+ if (unlikely(!enic_cq_rx_desc_eop(ciflags))) -+ goto mbuf_flags_done; -+ -+ /* VLAN stripping */ -+ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { -+ pkt_flags |= PKT_RX_VLAN_PKT; -+ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); -+ } else { -+ mbuf->vlan_tci = 0; -+ } -+ -+ /* RSS flag */ -+ if (enic_cq_rx_desc_rss_type(cqrd)) { -+ pkt_flags |= PKT_RX_RSS_HASH; -+ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); -+ } -+ -+ /* checksum flags */ -+ if (!enic_cq_rx_desc_csum_not_calc(cqrd) && -+ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { -+ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) -+ pkt_flags |= PKT_RX_IP_CKSUM_BAD; -+ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { -+ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) -+ pkt_flags |= PKT_RX_L4_CKSUM_BAD; -+ } -+ } -+ -+ mbuf_flags_done: -+ mbuf->ol_flags = pkt_flags; -+} -+ -+static inline uint32_t -+enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) -+{ -+ uint32_t d = i0 + i1; -+ ASSERT(i0 < n_descriptors); -+ ASSERT(i1 < n_descriptors); -+ d -= (d >= n_descriptors) ? n_descriptors : 0; -+ return d; -+} -+ -+ -+uint16_t -+enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -+ uint16_t nb_pkts) -+{ -+ struct vnic_rq *rq = rx_queue; -+ struct enic *enic = vnic_dev_priv(rq->vdev); -+ unsigned int rx_id; -+ struct rte_mbuf *nmb, *rxmb; -+ uint16_t nb_rx = 0; -+ uint16_t nb_hold; -+ struct vnic_cq *cq; -+ volatile struct cq_desc *cqd_ptr; -+ uint8_t color; -+ -+ cq = &enic->cq[enic_cq_rq(enic, rq->index)]; -+ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ -+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -+ -+ nb_hold = rq->rx_nb_hold; /* mbufs held by software */ -+ -+ while (nb_rx < nb_pkts) { -+ uint16_t rx_pkt_len; -+ volatile struct rq_enet_desc *rqd_ptr; -+ dma_addr_t dma_addr; -+ struct cq_desc cqd; -+ uint64_t ol_err_flags; -+ -+ /* Check for pkts available */ -+ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) -+ & CQ_DESC_COLOR_MASK; -+ if (color == cq->last_color) -+ break; -+ -+ /* Get the cq descriptor and rq pointer */ -+ cqd = *cqd_ptr; -+ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; -+ -+ /* allocate a new mbuf */ -+ nmb = rte_rxmbuf_alloc(rq->mp); -+ if (nmb == NULL) { -+ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", -+ enic->port_id, (unsigned)rq->index); -+ rte_eth_devices[enic->port_id]. -+ data->rx_mbuf_alloc_failed++; -+ break; -+ } -+ -+ /* Check for FCS or packet errors */ -+ ol_err_flags = enic_cq_rx_to_pkt_err_flags(&cqd); -+ if (ol_err_flags == 0) -+ rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd); -+ else -+ rx_pkt_len = 0; -+ -+ /* Get the mbuf to return and replace with one just allocated */ -+ rxmb = rq->mbuf_ring[rx_id]; -+ rq->mbuf_ring[rx_id] = nmb; -+ -+ /* Increment cqd, rqd, mbuf_table index */ -+ rx_id++; -+ if (unlikely(rx_id == rq->ring.desc_count)) { -+ rx_id = 0; -+ cq->last_color = cq->last_color ? 0 : 1; -+ } -+ -+ /* Prefetch next mbuf & desc while processing current one */ -+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -+ rte_enic_prefetch(cqd_ptr); -+ rte_enic_prefetch(rq->mbuf_ring[rx_id]); -+ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) -+ + rx_id); -+ -+ /* Push descriptor for newly allocated mbuf */ -+ dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off); -+ rqd_ptr->address = rte_cpu_to_le_64(dma_addr); -+ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len); -+ -+ /* Fill in the rest of the mbuf */ -+ rxmb->data_off = RTE_PKTMBUF_HEADROOM; -+ rxmb->nb_segs = 1; -+ rxmb->next = NULL; -+ rxmb->pkt_len = rx_pkt_len; -+ rxmb->data_len = rx_pkt_len; -+ rxmb->port = enic->port_id; -+ rxmb->ol_flags = ol_err_flags; -+ if (!ol_err_flags) -+ enic_cq_rx_to_pkt_flags(&cqd, rxmb); -+ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); -+ -+ /* prefetch mbuf data for caller */ -+ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, -+ RTE_PKTMBUF_HEADROOM)); -+ -+ /* store the mbuf address into the next entry of the array */ -+ rx_pkts[nb_rx++] = rxmb; -+ } -+ -+ nb_hold += nb_rx; -+ cq->to_clean = rx_id; -+ -+ if (nb_hold > rq->rx_free_thresh) { -+ rq->posted_index = enic_ring_add(rq->ring.desc_count, -+ rq->posted_index, nb_hold); -+ nb_hold = 0; -+ rte_mb(); -+ iowrite32(rq->posted_index, &rq->ctrl->posted_index); -+ } -+ -+ rq->rx_nb_hold = nb_hold; -+ -+ return nb_rx; -+} --- -1.9.1 - |