diff options
Diffstat (limited to 'dpdk/dpdk-2.2.0_patches/0013-Replacement-of-ENIC-PMD-receive-path-to-improve-perf.patch')
| -rw-r--r-- | dpdk/dpdk-2.2.0_patches/0013-Replacement-of-ENIC-PMD-receive-path-to-improve-perf.patch | 1347 |
1 files changed, 1347 insertions, 0 deletions
diff --git a/dpdk/dpdk-2.2.0_patches/0013-Replacement-of-ENIC-PMD-receive-path-to-improve-perf.patch b/dpdk/dpdk-2.2.0_patches/0013-Replacement-of-ENIC-PMD-receive-path-to-improve-perf.patch new file mode 100644 index 00000000000..92bd1e89c7e --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0013-Replacement-of-ENIC-PMD-receive-path-to-improve-perf.patch @@ -0,0 +1,1347 @@ +From 0963caf48db910ca6f93c5cbff8d5d24b2be64dd Mon Sep 17 00:00:00 2001 +From: John Lo <loj@cisco.com> +Date: Mon, 7 Mar 2016 14:12:29 -0500 +Subject: [PATCH 2/2] Replacement of ENIC PMD receive path to improve + performance and code clarifty. + +--- + drivers/net/enic/Makefile | 1 + + drivers/net/enic/base/vnic_rq.c | 99 ++--------- + drivers/net/enic/base/vnic_rq.h | 147 +--------------- + drivers/net/enic/enic.h | 16 +- + drivers/net/enic/enic_ethdev.c | 27 ++- + drivers/net/enic/enic_main.c | 321 ++++++++++------------------------ + drivers/net/enic/enic_res.h | 16 +- + drivers/net/enic/enic_rx.c | 378 ++++++++++++++++++++++++++++++++++++++++ + 8 files changed, 519 insertions(+), 486 deletions(-) + create mode 100644 drivers/net/enic/enic_rx.c + +diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile +index f0ee093..f316274 100644 +--- a/drivers/net/enic/Makefile ++++ b/drivers/net/enic/Makefile +@@ -53,6 +53,7 @@ VPATH += $(SRCDIR)/src + # + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c ++SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c +diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c +index 1441604..cb62c5e 100644 +--- a/drivers/net/enic/base/vnic_rq.c ++++ b/drivers/net/enic/base/vnic_rq.c +@@ -35,77 +35,21 @@ + #include "vnic_dev.h" + #include "vnic_rq.h" + +-static int vnic_rq_alloc_bufs(struct vnic_rq *rq) +-{ +- struct vnic_rq_buf *buf; +- unsigned int i, j, count = rq->ring.desc_count; +- unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count); +- +- for (i = 0; i < blks; i++) { +- rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ(count), GFP_ATOMIC); +- if (!rq->bufs[i]) +- return -ENOMEM; +- } +- +- for (i = 0; i < blks; i++) { +- buf = rq->bufs[i]; +- for (j = 0; j < VNIC_RQ_BUF_BLK_ENTRIES(count); j++) { +- buf->index = i * VNIC_RQ_BUF_BLK_ENTRIES(count) + j; +- buf->desc = (u8 *)rq->ring.descs + +- rq->ring.desc_size * buf->index; +- if (buf->index + 1 == count) { +- buf->next = rq->bufs[0]; +- break; +- } else if (j + 1 == VNIC_RQ_BUF_BLK_ENTRIES(count)) { +- buf->next = rq->bufs[i + 1]; +- } else { +- buf->next = buf + 1; +- buf++; +- } +- } +- } +- +- rq->to_use = rq->to_clean = rq->bufs[0]; +- +- return 0; +-} +- +-int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count, +- unsigned int desc_size) +-{ +- int mem_size = 0; +- +- mem_size += vnic_dev_desc_ring_size(&rq->ring, desc_count, desc_size); +- +- mem_size += VNIC_RQ_BUF_BLKS_NEEDED(rq->ring.desc_count) * +- VNIC_RQ_BUF_BLK_SZ(rq->ring.desc_count); +- +- return mem_size; +-} +- + void vnic_rq_free(struct vnic_rq *rq) + { + struct vnic_dev *vdev; +- unsigned int i; + + vdev = rq->vdev; + + vnic_dev_free_desc_ring(vdev, &rq->ring); + +- for (i = 0; i < VNIC_RQ_BUF_BLKS_MAX; i++) { +- if (rq->bufs[i]) { +- kfree(rq->bufs[i]); +- rq->bufs[i] = NULL; +- } +- } +- + rq->ctrl = NULL; + } + + int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, + unsigned int desc_count, unsigned int desc_size) + { +- int err; ++ int rc; + char res_name[NAME_MAX]; + static int instance; + +@@ -121,18 +65,9 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, + vnic_rq_disable(rq); + + snprintf(res_name, sizeof(res_name), "%d-rq-%d", instance++, index); +- err = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, ++ rc = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, + rq->socket_id, res_name); +- if (err) +- return err; +- +- err = vnic_rq_alloc_bufs(rq); +- if (err) { +- vnic_rq_free(rq); +- return err; +- } +- +- return 0; ++ return rc; + } + + void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, +@@ -154,9 +89,6 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, + iowrite32(fetch_index, &rq->ctrl->fetch_index); + iowrite32(posted_index, &rq->ctrl->posted_index); + +- rq->to_use = rq->to_clean = +- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] +- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; + } + + void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, +@@ -176,6 +108,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, + fetch_index, fetch_index, + error_interrupt_enable, + error_interrupt_offset); ++ rq->rxst_idx = 0; ++ rq->tot_pkts = 0; + } + + void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error) +@@ -212,21 +146,20 @@ int vnic_rq_disable(struct vnic_rq *rq) + } + + void vnic_rq_clean(struct vnic_rq *rq, +- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)) ++ void (*buf_clean)(struct rte_mbuf **buf)) + { +- struct vnic_rq_buf *buf; +- u32 fetch_index; ++ struct rte_mbuf **buf; ++ u32 fetch_index, i; + unsigned int count = rq->ring.desc_count; + +- buf = rq->to_clean; +- +- while (vnic_rq_desc_used(rq) > 0) { ++ buf = &rq->mbuf_ring[0]; + +- (*buf_clean)(rq, buf); +- +- buf = rq->to_clean = buf->next; +- rq->ring.desc_avail++; ++ for (i = 0; i < count; i++) { ++ (*buf_clean)(buf); ++ buf++; + } ++ rq->ring.desc_avail = count - 1; ++ rq->rx_nb_hold = 0; + + /* Use current fetch_index as the ring starting point */ + fetch_index = ioread32(&rq->ctrl->fetch_index); +@@ -235,9 +168,7 @@ void vnic_rq_clean(struct vnic_rq *rq, + /* Hardware surprise removal: reset fetch_index */ + fetch_index = 0; + } +- rq->to_use = rq->to_clean = +- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] +- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; ++ + iowrite32(fetch_index, &rq->ctrl->posted_index); + + vnic_dev_clear_desc_ring(&rq->ring); +diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h +index 0f5c3c1..e083ccc 100644 +--- a/drivers/net/enic/base/vnic_rq.h ++++ b/drivers/net/enic/base/vnic_rq.h +@@ -66,42 +66,22 @@ struct vnic_rq_ctrl { + u32 pad10; + }; + +-/* Break the vnic_rq_buf allocations into blocks of 32/64 entries */ +-#define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32 +-#define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64 +-#define VNIC_RQ_BUF_BLK_ENTRIES(entries) \ +- ((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \ +- VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES)) +-#define VNIC_RQ_BUF_BLK_SZ(entries) \ +- (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf)) +-#define VNIC_RQ_BUF_BLKS_NEEDED(entries) \ +- DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries)) +-#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096) +- +-struct vnic_rq_buf { +- struct vnic_rq_buf *next; +- dma_addr_t dma_addr; +- void *os_buf; +- unsigned int os_buf_index; +- unsigned int len; +- unsigned int index; +- void *desc; +- uint64_t wr_id; +-}; +- + struct vnic_rq { + unsigned int index; ++ unsigned int posted_index; + struct vnic_dev *vdev; +- struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ ++ struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ + struct vnic_dev_ring ring; +- struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX]; +- struct vnic_rq_buf *to_use; +- struct vnic_rq_buf *to_clean; ++ struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */ ++ unsigned int mbuf_next_idx; /* next mb to consume */ + void *os_buf_head; + unsigned int pkts_outstanding; +- ++ uint16_t rx_nb_hold; ++ uint16_t rx_free_thresh; + unsigned int socket_id; + struct rte_mempool *mp; ++ uint16_t rxst_idx; ++ uint32_t tot_pkts; + }; + + static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq) +@@ -116,119 +96,13 @@ static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq) + return rq->ring.desc_count - rq->ring.desc_avail - 1; + } + +-static inline void *vnic_rq_next_desc(struct vnic_rq *rq) +-{ +- return rq->to_use->desc; +-} +- +-static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq) +-{ +- return rq->to_use->index; +-} +- +-static inline void vnic_rq_post(struct vnic_rq *rq, +- void *os_buf, unsigned int os_buf_index, +- dma_addr_t dma_addr, unsigned int len, +- uint64_t wrid) +-{ +- struct vnic_rq_buf *buf = rq->to_use; +- +- buf->os_buf = os_buf; +- buf->os_buf_index = os_buf_index; +- buf->dma_addr = dma_addr; +- buf->len = len; +- buf->wr_id = wrid; +- +- buf = buf->next; +- rq->to_use = buf; +- rq->ring.desc_avail--; +- +- /* Move the posted_index every nth descriptor +- */ +- +-#ifndef VNIC_RQ_RETURN_RATE +-#define VNIC_RQ_RETURN_RATE 0xf /* keep 2^n - 1 */ +-#endif +- +- if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) { +- /* Adding write memory barrier prevents compiler and/or CPU +- * reordering, thus avoiding descriptor posting before +- * descriptor is initialized. Otherwise, hardware can read +- * stale descriptor fields. +- */ +- wmb(); +- iowrite32(buf->index, &rq->ctrl->posted_index); +- } +-} +- +-static inline void vnic_rq_post_commit(struct vnic_rq *rq, +- void *os_buf, unsigned int os_buf_index, +- dma_addr_t dma_addr, unsigned int len) +-{ +- struct vnic_rq_buf *buf = rq->to_use; +- +- buf->os_buf = os_buf; +- buf->os_buf_index = os_buf_index; +- buf->dma_addr = dma_addr; +- buf->len = len; +- +- buf = buf->next; +- rq->to_use = buf; +- rq->ring.desc_avail--; +- +- /* Move the posted_index every descriptor +- */ +- +- /* Adding write memory barrier prevents compiler and/or CPU +- * reordering, thus avoiding descriptor posting before +- * descriptor is initialized. Otherwise, hardware can read +- * stale descriptor fields. +- */ +- wmb(); +- iowrite32(buf->index, &rq->ctrl->posted_index); +-} + +-static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count) +-{ +- rq->ring.desc_avail += count; +-} + + enum desc_return_options { + VNIC_RQ_RETURN_DESC, + VNIC_RQ_DEFER_RETURN_DESC, + }; + +-static inline int vnic_rq_service(struct vnic_rq *rq, +- struct cq_desc *cq_desc, u16 completed_index, +- int desc_return, int (*buf_service)(struct vnic_rq *rq, +- struct cq_desc *cq_desc, struct vnic_rq_buf *buf, +- int skipped, void *opaque), void *opaque) +-{ +- struct vnic_rq_buf *buf; +- int skipped; +- int eop = 0; +- +- buf = rq->to_clean; +- while (1) { +- +- skipped = (buf->index != completed_index); +- +- if ((*buf_service)(rq, cq_desc, buf, skipped, opaque)) +- eop++; +- +- if (desc_return == VNIC_RQ_RETURN_DESC) +- rq->ring.desc_avail++; +- +- rq->to_clean = buf->next; +- +- if (!skipped) +- break; +- +- buf = rq->to_clean; +- } +- return eop; +-} +- + static inline int vnic_rq_fill(struct vnic_rq *rq, + int (*buf_fill)(struct vnic_rq *rq)) + { +@@ -274,8 +148,5 @@ unsigned int vnic_rq_error_status(struct vnic_rq *rq); + void vnic_rq_enable(struct vnic_rq *rq); + int vnic_rq_disable(struct vnic_rq *rq); + void vnic_rq_clean(struct vnic_rq *rq, +- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)); +-int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count, +- unsigned int desc_size); +- ++ void (*buf_clean)(struct rte_mbuf **buf)); + #endif /* _VNIC_RQ_H_ */ +diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h +index 9e78305..8c914f5 100644 +--- a/drivers/net/enic/enic.h ++++ b/drivers/net/enic/enic.h +@@ -45,6 +45,7 @@ + #include "vnic_nic.h" + #include "vnic_rss.h" + #include "enic_res.h" ++#include "cq_enet_desc.h" + + #define DRV_NAME "enic_pmd" + #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Poll-mode Driver" +@@ -154,6 +155,16 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) + return (struct enic *)eth_dev->data->dev_private; + } + ++#define RTE_LIBRTE_ENIC_ASSERT_ENABLE ++#ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE ++#define ASSERT(x) do { \ ++ if (!(x)) \ ++ rte_panic("ENIC: x"); \ ++} while (0) ++#else ++#define ASSERT(x) ++#endif ++ + extern void enic_fdir_stats_get(struct enic *enic, + struct rte_eth_fdir_stats *stats); + extern int enic_fdir_add_fltr(struct enic *enic, +@@ -193,9 +204,10 @@ extern void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, + uint16_t ol_flags, uint16_t vlan_tag); + + extern void enic_post_wq_index(struct vnic_wq *wq); +-extern int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts, +- unsigned int budget, unsigned int *work_done); + extern int enic_probe(struct enic *enic); + extern int enic_clsf_init(struct enic *enic); + extern void enic_clsf_destroy(struct enic *enic); ++uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, ++ uint16_t nb_pkts); ++ + #endif /* _ENIC_H_ */ +diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c +index 2a88043..6f2ada5 100644 +--- a/drivers/net/enic/enic_ethdev.c ++++ b/drivers/net/enic/enic_ethdev.c +@@ -255,7 +255,7 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, + uint16_t queue_idx, + uint16_t nb_desc, + unsigned int socket_id, +- __rte_unused const struct rte_eth_rxconf *rx_conf, ++ const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mp) + { + int ret; +@@ -270,6 +270,10 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, + return ret; + } + ++ enic->rq[queue_idx].rx_free_thresh = rx_conf->rx_free_thresh; ++ dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx, ++ enic->rq[queue_idx].rx_free_thresh); ++ + return enicpmd_dev_setup_intr(enic); + } + +@@ -429,6 +433,9 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev, + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM; ++ device_info->default_rxconf = (struct rte_eth_rxconf) { ++ .rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH ++ }; + } + + static void enicpmd_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) +@@ -538,18 +545,6 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + return index; + } + +-static uint16_t enicpmd_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, +- uint16_t nb_pkts) +-{ +- struct vnic_rq *rq = (struct vnic_rq *)rx_queue; +- unsigned int work_done; +- +- if (enic_poll(rq, rx_pkts, (unsigned int)nb_pkts, &work_done)) +- dev_err(enic, "error in enicpmd poll\n"); +- +- return work_done; +-} +- + static const struct eth_dev_ops enicpmd_eth_dev_ops = { + .dev_configure = enicpmd_dev_configure, + .dev_start = enicpmd_dev_start, +@@ -606,7 +601,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) + enic->port_id = eth_dev->data->port_id; + enic->rte_dev = eth_dev; + eth_dev->dev_ops = &enicpmd_eth_dev_ops; +- eth_dev->rx_pkt_burst = &enicpmd_recv_pkts; ++ eth_dev->rx_pkt_burst = &enic_recv_pkts; + eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts; + + pdev = eth_dev->pci_dev; +@@ -635,8 +630,8 @@ static struct eth_driver rte_enic_pmd = { + * Register as the [Poll Mode] Driver of Cisco ENIC device. + */ + static int +-rte_enic_pmd_init(const char *name __rte_unused, +- const char *params __rte_unused) ++rte_enic_pmd_init(__rte_unused const char *name, ++ __rte_unused const char *params) + { + ENICPMD_FUNC_TRACE(); + +diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c +index 07a9810..d0c9bff 100644 +--- a/drivers/net/enic/enic_main.c ++++ b/drivers/net/enic/enic_main.c +@@ -60,6 +60,17 @@ + #include "vnic_nic.h" + #include "enic_vnic_wq.h" + ++static inline struct rte_mbuf * ++rte_rxmbuf_alloc(struct rte_mempool *mp) ++{ ++ struct rte_mbuf *m; ++ ++ m = __rte_mbuf_raw_alloc(mp); ++ __rte_mbuf_sanity_check_raw(m, 0); ++ return m; ++} ++ ++ + static inline int enic_is_sriov_vf(struct enic *enic) + { + return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF; +@@ -80,16 +91,25 @@ static int is_eth_addr_valid(uint8_t *addr) + return !is_mcast_addr(addr) && !is_zero_addr(addr); + } + +-static inline struct rte_mbuf * +-enic_rxmbuf_alloc(struct rte_mempool *mp) ++static void ++enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) + { +- struct rte_mbuf *m; ++ uint16_t i; + +- m = __rte_mbuf_raw_alloc(mp); +- __rte_mbuf_sanity_check_raw(m, 0); +- return m; ++ if (!rq || !rq->mbuf_ring) { ++ dev_debug(enic, "Pointer to rq or mbuf_ring is NULL"); ++ return; ++ } ++ ++ for (i = 0; i < enic->config.rq_desc_count; i++) { ++ if (rq->mbuf_ring[i]) { ++ rte_pktmbuf_free_seg(rq->mbuf_ring[i]); ++ rq->mbuf_ring[i] = NULL; ++ } ++ } + } + ++ + void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) + { + vnic_set_hdr_split_size(enic->vdev, split_hdr_size); +@@ -262,13 +282,13 @@ void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr) + } + + static void +-enic_free_rq_buf(__rte_unused struct vnic_rq *rq, struct vnic_rq_buf *buf) ++enic_free_rq_buf(struct rte_mbuf **mbuf) + { +- if (!buf->os_buf) ++ if (*mbuf == NULL) + return; + +- rte_pktmbuf_free((struct rte_mbuf *)buf->os_buf); +- buf->os_buf = NULL; ++ rte_pktmbuf_free(*mbuf); ++ mbuf = NULL; + } + + void enic_init_vnic_resources(struct enic *enic) +@@ -314,221 +334,47 @@ void enic_init_vnic_resources(struct enic *enic) + } + + +-static int enic_rq_alloc_buf(struct vnic_rq *rq) ++static int ++enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) + { +- struct enic *enic = vnic_dev_priv(rq->vdev); ++ struct rte_mbuf *mb; ++ struct rq_enet_desc *rqd = rq->ring.descs; ++ unsigned i; + dma_addr_t dma_addr; +- struct rq_enet_desc *desc = vnic_rq_next_desc(rq); +- uint8_t type = RQ_ENET_TYPE_ONLY_SOP; +- u16 split_hdr_size = vnic_get_hdr_split_size(enic->vdev); +- struct rte_mbuf *mbuf = enic_rxmbuf_alloc(rq->mp); +- struct rte_mbuf *hdr_mbuf = NULL; +- +- if (!mbuf) { +- dev_err(enic, "mbuf alloc in enic_rq_alloc_buf failed\n"); +- return -1; +- } +- +- if (unlikely(split_hdr_size)) { +- if (vnic_rq_desc_avail(rq) < 2) { +- rte_mempool_put(mbuf->pool, mbuf); +- return -1; +- } +- hdr_mbuf = enic_rxmbuf_alloc(rq->mp); +- if (!hdr_mbuf) { +- rte_mempool_put(mbuf->pool, mbuf); +- dev_err(enic, +- "hdr_mbuf alloc in enic_rq_alloc_buf failed\n"); +- return -1; +- } +- +- hdr_mbuf->data_off = RTE_PKTMBUF_HEADROOM; +- +- hdr_mbuf->nb_segs = 2; +- hdr_mbuf->port = enic->port_id; +- hdr_mbuf->next = mbuf; +- +- dma_addr = (dma_addr_t) +- (hdr_mbuf->buf_physaddr + hdr_mbuf->data_off); +- +- rq_enet_desc_enc(desc, dma_addr, type, split_hdr_size); + +- vnic_rq_post(rq, (void *)hdr_mbuf, 0 /*os_buf_index*/, dma_addr, +- (unsigned int)split_hdr_size, 0 /*wrid*/); ++ dev_debug(enic, "queue %u, allocating %u rx queue mbufs", rq->index, ++ rq->ring.desc_count); + +- desc = vnic_rq_next_desc(rq); +- type = RQ_ENET_TYPE_NOT_SOP; +- } else { +- mbuf->nb_segs = 1; +- mbuf->port = enic->port_id; +- } +- +- mbuf->data_off = RTE_PKTMBUF_HEADROOM; +- mbuf->next = NULL; +- +- dma_addr = (dma_addr_t) +- (mbuf->buf_physaddr + mbuf->data_off); +- +- rq_enet_desc_enc(desc, dma_addr, type, mbuf->buf_len); +- +- vnic_rq_post(rq, (void *)mbuf, 0 /*os_buf_index*/, dma_addr, +- (unsigned int)mbuf->buf_len, 0 /*wrid*/); +- +- return 0; +-} +- +-static int enic_rq_indicate_buf(struct vnic_rq *rq, +- struct cq_desc *cq_desc, struct vnic_rq_buf *buf, +- int skipped, void *opaque) +-{ +- struct enic *enic = vnic_dev_priv(rq->vdev); +- struct rte_mbuf **rx_pkt_bucket = (struct rte_mbuf **)opaque; +- struct rte_mbuf *rx_pkt = NULL; +- struct rte_mbuf *hdr_rx_pkt = NULL; +- +- u8 type, color, eop, sop, ingress_port, vlan_stripped; +- u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof; +- u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok; +- u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc; +- u8 packet_error; +- u16 q_number, completed_index, bytes_written, vlan_tci, checksum; +- u32 rss_hash; +- +- cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, +- &type, &color, &q_number, &completed_index, +- &ingress_port, &fcoe, &eop, &sop, &rss_type, +- &csum_not_calc, &rss_hash, &bytes_written, +- &packet_error, &vlan_stripped, &vlan_tci, &checksum, +- &fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error, +- &fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp, +- &ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment, +- &fcs_ok); +- +- rx_pkt = (struct rte_mbuf *)buf->os_buf; +- buf->os_buf = NULL; +- +- if (unlikely(packet_error)) { +- dev_err(enic, "packet error\n"); +- rx_pkt->data_len = 0; +- return 0; +- } +- +- if (unlikely(skipped)) { +- rx_pkt->data_len = 0; +- return 0; +- } +- +- if (likely(!vnic_get_hdr_split_size(enic->vdev))) { +- /* No header split configured */ +- *rx_pkt_bucket = rx_pkt; +- rx_pkt->pkt_len = bytes_written; +- +- if (ipv4) { +- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4; +- if (!csum_not_calc) { +- if (unlikely(!ipv4_csum_ok)) +- rx_pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD; +- +- if ((tcp || udp) && (!tcp_udp_csum_ok)) +- rx_pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD; +- } +- } else if (ipv6) +- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6; +- } else { +- /* Header split */ +- if (sop && !eop) { +- /* This piece is header */ +- *rx_pkt_bucket = rx_pkt; +- rx_pkt->pkt_len = bytes_written; +- } else { +- if (sop && eop) { +- /* The packet is smaller than split_hdr_size */ +- *rx_pkt_bucket = rx_pkt; +- rx_pkt->pkt_len = bytes_written; +- if (ipv4) { +- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4; +- if (!csum_not_calc) { +- if (unlikely(!ipv4_csum_ok)) +- rx_pkt->ol_flags |= +- PKT_RX_IP_CKSUM_BAD; +- +- if ((tcp || udp) && +- (!tcp_udp_csum_ok)) +- rx_pkt->ol_flags |= +- PKT_RX_L4_CKSUM_BAD; +- } +- } else if (ipv6) +- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6; +- } else { +- /* Payload */ +- hdr_rx_pkt = *rx_pkt_bucket; +- hdr_rx_pkt->pkt_len += bytes_written; +- if (ipv4) { +- hdr_rx_pkt->packet_type = +- RTE_PTYPE_L3_IPV4; +- if (!csum_not_calc) { +- if (unlikely(!ipv4_csum_ok)) +- hdr_rx_pkt->ol_flags |= +- PKT_RX_IP_CKSUM_BAD; +- +- if ((tcp || udp) && +- (!tcp_udp_csum_ok)) +- hdr_rx_pkt->ol_flags |= +- PKT_RX_L4_CKSUM_BAD; +- } +- } else if (ipv6) +- hdr_rx_pkt->packet_type = +- RTE_PTYPE_L3_IPV6; +- } ++ for (i = 0; i < rq->ring.desc_count; i++, rqd++) { ++ mb = rte_rxmbuf_alloc(rq->mp); ++ if (mb == NULL) { ++ dev_err(enic, "RX mbuf alloc failed queue_id=%u", ++ (unsigned)rq->index); ++ return -ENOMEM; + } +- } + +- rx_pkt->data_len = bytes_written; ++ dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off); + +- if (rss_hash) { +- rx_pkt->ol_flags |= PKT_RX_RSS_HASH; +- rx_pkt->hash.rss = rss_hash; ++ rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, ++ mb->buf_len); ++ rq->mbuf_ring[i] = mb; + } + +- if (vlan_tci) { +- rx_pkt->ol_flags |= PKT_RX_VLAN_PKT; +- rx_pkt->vlan_tci = vlan_tci; +- } ++ /* make sure all prior writes are complete before doing the PIO write */ ++ rte_rmb(); + +- return eop; +-} ++ /* Post all but the last 2 cache lines' worth of descriptors */ ++ rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE ++ / sizeof(struct rq_enet_desc)); ++ rq->rx_nb_hold = 0; + +-static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, +- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque) +-{ +- struct enic *enic = vnic_dev_priv(vdev); +- +- return vnic_rq_service(&enic->rq[q_number], cq_desc, +- completed_index, VNIC_RQ_RETURN_DESC, +- enic_rq_indicate_buf, opaque); +- +-} ++ dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n", ++ enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold); ++ iowrite32(rq->posted_index, &rq->ctrl->posted_index); ++ rte_rmb(); + +-int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts, +- unsigned int budget, unsigned int *work_done) +-{ +- struct enic *enic = vnic_dev_priv(rq->vdev); +- unsigned int cq = enic_cq_rq(enic, rq->index); +- int err = 0; +- +- *work_done = vnic_cq_service(&enic->cq[cq], +- budget, enic_rq_service, (void *)rx_pkts); +- +- if (*work_done) { +- vnic_rq_fill(rq, enic_rq_alloc_buf); ++ return 0; + +- /* Need at least one buffer on ring to get going */ +- if (vnic_rq_desc_used(rq) == 0) { +- dev_err(enic, "Unable to alloc receive buffers\n"); +- err = -1; +- } +- } +- return err; + } + + static void * +@@ -576,6 +422,7 @@ enic_intr_handler(__rte_unused struct rte_intr_handle *handle, + int enic_enable(struct enic *enic) + { + unsigned int index; ++ int err; + struct rte_eth_dev *eth_dev = enic->rte_dev; + + eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev); +@@ -586,15 +433,11 @@ int enic_enable(struct enic *enic) + dev_warning(enic, "Init of hash table for clsf failed."\ + "Flow director feature will not work\n"); + +- /* Fill RQ bufs */ + for (index = 0; index < enic->rq_count; index++) { +- vnic_rq_fill(&enic->rq[index], enic_rq_alloc_buf); +- +- /* Need at least one buffer on ring to get going +- */ +- if (vnic_rq_desc_used(&enic->rq[index]) == 0) { +- dev_err(enic, "Unable to alloc receive buffers\n"); +- return -1; ++ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]); ++ if (err) { ++ dev_err(enic, "Failed to alloc RX queue mbufs\n"); ++ return err; + } + } + +@@ -636,6 +479,9 @@ void enic_free_rq(void *rxq) + struct vnic_rq *rq = (struct vnic_rq *)rxq; + struct enic *enic = vnic_dev_priv(rq->vdev); + ++ enic_rxmbuf_queue_release(enic, rq); ++ rte_free(rq->mbuf_ring); ++ rq->mbuf_ring = NULL; + vnic_rq_free(rq); + vnic_cq_free(&enic->cq[rq->index]); + } +@@ -664,7 +510,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, + unsigned int socket_id, struct rte_mempool *mp, + uint16_t nb_desc) + { +- int err; ++ int rc; + struct vnic_rq *rq = &enic->rq[queue_idx]; + + rq->socket_id = socket_id; +@@ -687,23 +533,35 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, + } + + /* Allocate queue resources */ +- err = vnic_rq_alloc(enic->vdev, &enic->rq[queue_idx], queue_idx, +- enic->config.rq_desc_count, +- sizeof(struct rq_enet_desc)); +- if (err) { ++ rc = vnic_rq_alloc(enic->vdev, rq, queue_idx, ++ enic->config.rq_desc_count, sizeof(struct rq_enet_desc)); ++ if (rc) { + dev_err(enic, "error in allocation of rq\n"); +- return err; ++ goto err_exit; + } + +- err = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, ++ rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, + socket_id, enic->config.rq_desc_count, + sizeof(struct cq_enet_rq_desc)); +- if (err) { +- vnic_rq_free(rq); ++ if (rc) { + dev_err(enic, "error in allocation of cq for rq\n"); ++ goto err_free_rq_exit; + } + +- return err; ++ /* Allocate the mbuf ring */ ++ rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", ++ sizeof(struct rte_mbuf *) * enic->config.rq_desc_count, ++ RTE_CACHE_LINE_SIZE, rq->socket_id); ++ ++ if (rq->mbuf_ring != NULL) ++ return 0; ++ ++ /* cleanup on error */ ++ vnic_cq_free(&enic->cq[queue_idx]); ++err_free_rq_exit: ++ vnic_rq_free(rq); ++err_exit: ++ return -ENOMEM; + } + + void enic_free_wq(void *txq) +@@ -790,6 +648,7 @@ int enic_disable(struct enic *enic) + + for (i = 0; i < enic->wq_count; i++) + vnic_wq_clean(&enic->wq[i], enic_free_wq_buf); ++ + for (i = 0; i < enic->rq_count; i++) + vnic_rq_clean(&enic->rq[i], enic_free_rq_buf); + for (i = 0; i < enic->cq_count; i++) +@@ -1074,7 +933,7 @@ int enic_probe(struct enic *enic) + + /* Set ingress vlan rewrite mode before vnic initialization */ + err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, +- IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN); ++ IG_VLAN_REWRITE_MODE_PASS_THRU); + if (err) { + dev_err(enic, + "Failed to set ingress vlan rewrite mode, aborting.\n"); +diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h +index 49f7e22..33f2e84 100644 +--- a/drivers/net/enic/enic_res.h ++++ b/drivers/net/enic/enic_res.h +@@ -52,6 +52,7 @@ + #define ENIC_UNICAST_PERFECT_FILTERS 32 + + #define ENIC_NON_TSO_MAX_DESC 16 ++#define ENIC_DEFAULT_RX_FREE_THRESH 32 + + #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) + +@@ -133,21 +134,6 @@ static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq, + WQ_ENET_OFFLOAD_MODE_TSO, + eop, 1 /* SOP */, eop, loopback); + } +-static inline void enic_queue_rq_desc(struct vnic_rq *rq, +- void *os_buf, unsigned int os_buf_index, +- dma_addr_t dma_addr, unsigned int len) +-{ +- struct rq_enet_desc *desc = vnic_rq_next_desc(rq); +- u64 wrid = 0; +- u8 type = os_buf_index ? +- RQ_ENET_TYPE_NOT_SOP : RQ_ENET_TYPE_ONLY_SOP; +- +- rq_enet_desc_enc(desc, +- (u64)dma_addr | VNIC_PADDR_TARGET, +- type, (u16)len); +- +- vnic_rq_post(rq, os_buf, os_buf_index, dma_addr, len, wrid); +-} + + struct enic; + +diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c +new file mode 100644 +index 0000000..12b1f62 +--- /dev/null ++++ b/drivers/net/enic/enic_rx.c +@@ -0,0 +1,378 @@ ++/* ++ * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved. ++ * Copyright 2007 Nuova Systems, Inc. All rights reserved. ++ * ++ * Copyright (c) 2014, Cisco Systems, Inc. ++ * All rights reserved. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions ++ * are met: ++ * ++ * 1. Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * 2. Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in ++ * the documentation and/or other materials provided with the ++ * distribution. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS ++ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ++ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, ++ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, ++ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER ++ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT ++ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ++ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE ++ * POSSIBILITY OF SUCH DAMAGE. ++ * ++ */ ++ ++#include <rte_mbuf.h> ++#include <rte_ethdev.h> ++#include <rte_prefetch.h> ++ ++#include "enic_compat.h" ++#include "rq_enet_desc.h" ++#include "enic.h" ++ ++#define RTE_PMD_USE_PREFETCH ++ ++#ifdef RTE_PMD_USE_PREFETCH ++/* ++ * Prefetch a cache line into all cache levels. ++ */ ++#define rte_enic_prefetch(p) rte_prefetch0(p) ++#else ++#define rte_enic_prefetch(p) do {} while (0) ++#endif ++ ++#ifdef RTE_PMD_PACKET_PREFETCH ++#define rte_packet_prefetch(p) rte_prefetch1(p) ++#else ++#define rte_packet_prefetch(p) do {} while (0) ++#endif ++ ++static inline struct rte_mbuf * ++rte_rxmbuf_alloc(struct rte_mempool *mp) ++{ ++ struct rte_mbuf *m; ++ ++ m = __rte_mbuf_raw_alloc(mp); ++ __rte_mbuf_sanity_check_raw(m, 0); ++ return m; ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) ++{ ++ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) ++{ ++ return(le16_to_cpu(crd->bytes_written_flags) & ++ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_packet_error(uint16_t bwflags) ++{ ++ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == ++ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_eop(uint16_t ciflags) ++{ ++ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) ++ == CQ_ENET_RQ_DESC_FLAGS_EOP; ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) ++{ ++ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & ++ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == ++ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) ++{ ++ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == ++ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) ++{ ++ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == ++ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) ++{ ++ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> ++ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); ++} ++ ++static inline uint32_t ++enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) ++{ ++ return le32_to_cpu(cqrd->rss_hash); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_fcs_ok(struct cq_enet_rq_desc *cqrd) ++{ ++ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) == ++ CQ_ENET_RQ_DESC_FLAGS_FCS_OK); ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) ++{ ++ return le16_to_cpu(cqrd->vlan); ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ return le16_to_cpu(cqrd->bytes_written_flags) & ++ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; ++} ++ ++static inline uint64_t ++enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ uint16_t bwflags; ++ uint64_t pkt_err_flags = 0; ++ ++ bwflags = enic_cq_rx_desc_bwflags(cqrd); ++ ++ /* Check for packet error. Can't be more specific than MAC error */ ++ if (enic_cq_rx_desc_packet_error(bwflags)) { ++#ifdef RTE_NEXT_ABI ++ pkt_err_flags |= PKT_RX_MAC_ERR; ++#else ++ pkt_err_flags |= PKT_EXT_RX_PKT_ERROR; ++#endif ++ } ++ ++ /* Check for bad FCS. MAC error isn't quite, but no other choice */ ++ if (!enic_cq_rx_desc_fcs_ok(cqrd)) { ++#ifdef RTE_NEXT_ABI ++ pkt_err_flags |= PKT_RX_MAC_ERR; ++#else ++ pkt_err_flags |= PKT_EXT_RX_BAD_FCS; ++#endif ++ } ++ return pkt_err_flags; ++} ++ ++/* ++ * Lookup table to translate RX CQ flags to mbuf flags. ++ */ ++static inline uint32_t ++enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ uint8_t cqrd_flags = cqrd->flags; ++ static const uint32_t cq_type_table[128] __rte_cache_aligned = { ++ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, ++ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 ++ | RTE_PTYPE_L4_UDP, ++ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 ++ | RTE_PTYPE_L4_TCP, ++ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 ++ | RTE_PTYPE_L4_FRAG, ++ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, ++ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 ++ | RTE_PTYPE_L4_UDP, ++ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 ++ | RTE_PTYPE_L4_TCP, ++ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 ++ | RTE_PTYPE_L4_FRAG, ++ /* All others reserved */ ++ }; ++ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT ++ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 ++ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; ++ return cq_type_table[cqrd_flags]; ++} ++ ++static inline void ++enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ uint16_t ciflags, bwflags, pkt_flags = 0; ++ ciflags = enic_cq_rx_desc_ciflags(cqrd); ++ bwflags = enic_cq_rx_desc_bwflags(cqrd); ++ ++ ASSERT(mbuf->ol_flags == 0); ++ ++ /* flags are meaningless if !EOP */ ++ if (unlikely(!enic_cq_rx_desc_eop(ciflags))) ++ goto mbuf_flags_done; ++ ++ /* VLAN stripping */ ++ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { ++ pkt_flags |= PKT_RX_VLAN_PKT; ++ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); ++ } else { ++ mbuf->vlan_tci = 0; ++ } ++ ++ /* RSS flag */ ++ if (enic_cq_rx_desc_rss_type(cqrd)) { ++ pkt_flags |= PKT_RX_RSS_HASH; ++ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); ++ } ++ ++ /* checksum flags */ ++ if (!enic_cq_rx_desc_csum_not_calc(cqrd) && ++ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { ++ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) ++ pkt_flags |= PKT_RX_IP_CKSUM_BAD; ++ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { ++ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) ++ pkt_flags |= PKT_RX_L4_CKSUM_BAD; ++ } ++ } ++ ++ mbuf_flags_done: ++ mbuf->ol_flags = pkt_flags; ++} ++ ++static inline uint32_t ++enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) ++{ ++ uint32_t d = i0 + i1; ++ ASSERT(i0 < n_descriptors); ++ ASSERT(i1 < n_descriptors); ++ d -= (d >= n_descriptors) ? n_descriptors : 0; ++ return d; ++} ++ ++ ++uint16_t ++enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, ++ uint16_t nb_pkts) ++{ ++ struct vnic_rq *rq = rx_queue; ++ struct enic *enic = vnic_dev_priv(rq->vdev); ++ unsigned int rx_id; ++ struct rte_mbuf *nmb, *rxmb; ++ uint16_t nb_rx = 0; ++ uint16_t nb_hold; ++ struct vnic_cq *cq; ++ volatile struct cq_desc *cqd_ptr; ++ uint8_t color; ++ ++ cq = &enic->cq[enic_cq_rq(enic, rq->index)]; ++ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ ++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; ++ ++ nb_hold = rq->rx_nb_hold; /* mbufs held by software */ ++ ++ while (nb_rx < nb_pkts) { ++ uint16_t rx_pkt_len; ++ volatile struct rq_enet_desc *rqd_ptr; ++ dma_addr_t dma_addr; ++ struct cq_desc cqd; ++ uint64_t ol_err_flags; ++ ++ /* Check for pkts available */ ++ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) ++ & CQ_DESC_COLOR_MASK; ++ if (color == cq->last_color) ++ break; ++ ++ /* Get the cq descriptor and rq pointer */ ++ cqd = *cqd_ptr; ++ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; ++ ++ /* allocate a new mbuf */ ++ nmb = rte_rxmbuf_alloc(rq->mp); ++ if (nmb == NULL) { ++ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", ++ enic->port_id, (unsigned)rq->index); ++ rte_eth_devices[enic->port_id]. ++ data->rx_mbuf_alloc_failed++; ++ break; ++ } ++ ++ /* Check for FCS or packet errors */ ++ ol_err_flags = enic_cq_rx_to_pkt_err_flags(&cqd); ++ if (ol_err_flags == 0) ++ rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd); ++ else ++ rx_pkt_len = 0; ++ ++ /* Get the mbuf to return and replace with one just allocated */ ++ rxmb = rq->mbuf_ring[rx_id]; ++ rq->mbuf_ring[rx_id] = nmb; ++ ++ /* Increment cqd, rqd, mbuf_table index */ ++ rx_id++; ++ if (unlikely(rx_id == rq->ring.desc_count)) { ++ rx_id = 0; ++ cq->last_color = cq->last_color ? 0 : 1; ++ } ++ ++ /* Prefetch next mbuf & desc while processing current one */ ++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; ++ rte_enic_prefetch(cqd_ptr); ++ rte_enic_prefetch(rq->mbuf_ring[rx_id]); ++ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) ++ + rx_id); ++ ++ /* Push descriptor for newly allocated mbuf */ ++ dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off); ++ rqd_ptr->address = rte_cpu_to_le_64(dma_addr); ++ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len); ++ ++ /* Fill in the rest of the mbuf */ ++ rxmb->data_off = RTE_PKTMBUF_HEADROOM; ++ rxmb->nb_segs = 1; ++ rxmb->next = NULL; ++ rxmb->pkt_len = rx_pkt_len; ++ rxmb->data_len = rx_pkt_len; ++ rxmb->port = enic->port_id; ++ rxmb->ol_flags = ol_err_flags; ++ if (!ol_err_flags) ++ enic_cq_rx_to_pkt_flags(&cqd, rxmb); ++ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); ++ ++ /* prefetch mbuf data for caller */ ++ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, ++ RTE_PKTMBUF_HEADROOM)); ++ ++ /* store the mbuf address into the next entry of the array */ ++ rx_pkts[nb_rx++] = rxmb; ++ } ++ ++ nb_hold += nb_rx; ++ cq->to_clean = rx_id; ++ ++ if (nb_hold > rq->rx_free_thresh) { ++ rq->posted_index = enic_ring_add(rq->ring.desc_count, ++ rq->posted_index, nb_hold); ++ nb_hold = 0; ++ rte_mb(); ++ iowrite32(rq->posted_index, &rq->ctrl->posted_index); ++ } ++ ++ rq->rx_nb_hold = nb_hold; ++ ++ return nb_rx; ++} +-- +1.9.1 + |