diff options
6 files changed, 3814 insertions, 12 deletions
diff --git a/dpdk/dpdk-16.04_patches/0009-enic-fix-imissed-rx-counter.patch b/dpdk/dpdk-16.04_patches/0009-enic-fix-imissed-rx-counter.patch index db86c9b057a..81e7bf3dbf6 100644 --- a/dpdk/dpdk-16.04_patches/0009-enic-fix-imissed-rx-counter.patch +++ b/dpdk/dpdk-16.04_patches/0009-enic-fix-imissed-rx-counter.patch @@ -1,18 +1,20 @@ -From c6c5a3f41528b28764ee4aa96a72d89c94d695df Mon Sep 17 00:00:00 2001 +From 3433c7828ec909fccb768636ee21867030da14c9 Mon Sep 17 00:00:00 2001 From: John Daley <johndale@cisco.com> -Date: Fri, 15 Apr 2016 14:43:11 -0700 -Subject: [PATCH] enic: fix imissed rx counter +Date: Tue, 26 Apr 2016 13:30:50 -0700 +Subject: [PATCH 1/3] enic: fix 'imissed' to count drops due to lack of RX + buffers +Fixes: 7182d3e7d177 ("enic: expose Rx missed packets counter") Signed-off-by: John Daley <johndale@cisco.com> --- drivers/net/enic/enic_main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index c61433f..8d6bbeb 100644 +index 60fe765..be4e9e5 100644 --- a/drivers/net/enic/enic_main.c +++ b/drivers/net/enic/enic_main.c -@@ -165,10 +165,10 @@ void enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats) +@@ -243,10 +243,10 @@ void enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats) r_stats->ibytes = stats->rx.rx_bytes_ok; r_stats->obytes = stats->tx.tx_bytes_ok; @@ -23,8 +25,8 @@ index c61433f..8d6bbeb 100644 - r_stats->imissed = stats->rx.rx_drop; + r_stats->imissed = stats->rx.rx_no_bufs; - r_stats->imcasts = stats->rx.rx_multicast_frames_ok; r_stats->rx_nombuf = stats->rx.rx_no_bufs; + } -- 2.7.0 diff --git a/dpdk/dpdk-16.04_patches/0011-enic-fix-misalignment-of-Rx-mbuf-data.patch b/dpdk/dpdk-16.04_patches/0011-enic-fix-misalignment-of-Rx-mbuf-data.patch new file mode 100644 index 00000000000..69ca3f31087 --- /dev/null +++ b/dpdk/dpdk-16.04_patches/0011-enic-fix-misalignment-of-Rx-mbuf-data.patch @@ -0,0 +1,55 @@ +From 454eb71eca1912e32a509c738a99a340cc2488cf Mon Sep 17 00:00:00 2001 +From: John Daley <johndale@cisco.com> +Date: Mon, 25 Apr 2016 16:24:53 -0700 +Subject: [PATCH 2/3] enic: fix misalignment of Rx mbuf data + +Data DMA used m->data_off of uninitialized mbufs instead of +RTE_PKTMBUF_HEADROOM, potentially causing Rx data to be +placed at the wrong alignment in the mbuf. + +Fixes: 947d860c821f ("enic: improve Rx performance") +Signed-off-by: John Daley <johndale@cisco.com> +--- + drivers/net/enic/enic_main.c | 5 +++-- + drivers/net/enic/enic_rx.c | 6 ++++-- + 2 files changed, 7 insertions(+), 4 deletions(-) + +diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c +index be4e9e5..646d87f 100644 +--- a/drivers/net/enic/enic_main.c ++++ b/drivers/net/enic/enic_main.c +@@ -354,10 +354,11 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) + return -ENOMEM; + } + +- dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off); ++ dma_addr = (dma_addr_t)(mb->buf_physaddr ++ + RTE_PKTMBUF_HEADROOM); + + rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, +- mb->buf_len); ++ mb->buf_len - RTE_PKTMBUF_HEADROOM); + rq->mbuf_ring[i] = mb; + } + +diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c +index 232987a..39bb55c 100644 +--- a/drivers/net/enic/enic_rx.c ++++ b/drivers/net/enic/enic_rx.c +@@ -314,9 +314,11 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + + rx_id); + + /* Push descriptor for newly allocated mbuf */ +- dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off); ++ dma_addr = (dma_addr_t)(nmb->buf_physaddr ++ + RTE_PKTMBUF_HEADROOM); + rqd_ptr->address = rte_cpu_to_le_64(dma_addr); +- rqd_ptr->length_type = cpu_to_le16(nmb->buf_len); ++ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len ++ - RTE_PKTMBUF_HEADROOM); + + /* Fill in the rest of the mbuf */ + rxmb->data_off = RTE_PKTMBUF_HEADROOM; +-- +2.7.0 + diff --git a/dpdk/dpdk-16.04_patches/0012-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch b/dpdk/dpdk-16.04_patches/0012-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch new file mode 100644 index 00000000000..4858b8f5d39 --- /dev/null +++ b/dpdk/dpdk-16.04_patches/0012-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch @@ -0,0 +1,1844 @@ +From ca6bbb723880e91d006de6cc485259da988859aa Mon Sep 17 00:00:00 2001 +From: John Daley <johndale@cisco.com> +Date: Tue, 5 Apr 2016 15:19:06 -0700 +Subject: [PATCH 3/3] enic: Optimization of Tx path to reduce Host CPU + overhead, cleanup + +Optimizations and cleanup: +- flatten packet send path +- flatten mbuf free path +- disable CQ entry writing and use CQ messages instead +- use rte_mempool_put_bulk() to bulk return freed mbufs +- remove unnecessary fields vnic_bufs struct, use contiguous array of cache + aligned divisible elements. No next pointers. +- use local variables inside per packet loop instead of fields in structs. +- factor book keeping out of the per packet tx loop where possible + (removed several conditionals) +- put Tx and Rx code in 1 file (enic_rxtx.c) + +Reviewed-by: Nelson Escobar <neescoba@cisco.com> +Signed-off-by: John Daley <johndale@cisco.com> +--- + drivers/net/enic/Makefile | 2 +- + drivers/net/enic/base/enic_vnic_wq.h | 79 ------ + drivers/net/enic/base/vnic_cq.h | 37 +-- + drivers/net/enic/base/vnic_rq.h | 2 +- + drivers/net/enic/base/vnic_wq.c | 89 +++--- + drivers/net/enic/base/vnic_wq.h | 113 +------- + drivers/net/enic/enic.h | 27 +- + drivers/net/enic/enic_ethdev.c | 67 +---- + drivers/net/enic/enic_main.c | 132 +++------ + drivers/net/enic/enic_res.h | 81 +----- + drivers/net/enic/enic_rx.c | 361 ------------------------- + drivers/net/enic/enic_rxtx.c | 505 +++++++++++++++++++++++++++++++++++ + 12 files changed, 635 insertions(+), 860 deletions(-) + delete mode 100644 drivers/net/enic/base/enic_vnic_wq.h + delete mode 100644 drivers/net/enic/enic_rx.c + create mode 100644 drivers/net/enic/enic_rxtx.c + +diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile +index f316274..3926b79 100644 +--- a/drivers/net/enic/Makefile ++++ b/drivers/net/enic/Makefile +@@ -53,7 +53,7 @@ VPATH += $(SRCDIR)/src + # + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c +-SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c ++SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rxtx.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c +diff --git a/drivers/net/enic/base/enic_vnic_wq.h b/drivers/net/enic/base/enic_vnic_wq.h +deleted file mode 100644 +index b019109..0000000 +--- a/drivers/net/enic/base/enic_vnic_wq.h ++++ /dev/null +@@ -1,79 +0,0 @@ +-/* +- * Copyright 2008-2015 Cisco Systems, Inc. All rights reserved. +- * Copyright 2007 Nuova Systems, Inc. All rights reserved. +- * +- * Copyright (c) 2015, Cisco Systems, Inc. +- * All rights reserved. +- * +- * Redistribution and use in source and binary forms, with or without +- * modification, are permitted provided that the following conditions +- * are met: +- * +- * 1. Redistributions of source code must retain the above copyright +- * notice, this list of conditions and the following disclaimer. +- * +- * 2. Redistributions in binary form must reproduce the above copyright +- * notice, this list of conditions and the following disclaimer in +- * the documentation and/or other materials provided with the +- * distribution. +- * +- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +- * POSSIBILITY OF SUCH DAMAGE. +- * +- */ +- +-#ifndef _ENIC_VNIC_WQ_H_ +-#define _ENIC_VNIC_WQ_H_ +- +-#include "vnic_dev.h" +-#include "vnic_cq.h" +- +-static inline void enic_vnic_post_wq_index(struct vnic_wq *wq) +-{ +- struct vnic_wq_buf *buf = wq->to_use; +- +- /* Adding write memory barrier prevents compiler and/or CPU +- * reordering, thus avoiding descriptor posting before +- * descriptor is initialized. Otherwise, hardware can read +- * stale descriptor fields. +- */ +- wmb(); +- iowrite32(buf->index, &wq->ctrl->posted_index); +-} +- +-static inline void enic_vnic_post_wq(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, +- unsigned int len, int sop, +- uint8_t desc_skip_cnt, uint8_t cq_entry, +- uint8_t compressed_send, uint64_t wrid) +-{ +- struct vnic_wq_buf *buf = wq->to_use; +- +- buf->sop = sop; +- buf->cq_entry = cq_entry; +- buf->compressed_send = compressed_send; +- buf->desc_skip_cnt = desc_skip_cnt; +- buf->os_buf = os_buf; +- buf->dma_addr = dma_addr; +- buf->len = len; +- buf->wr_id = wrid; +- +- buf = buf->next; +- wq->ring.desc_avail -= desc_skip_cnt; +- wq->to_use = buf; +- +- if (cq_entry) +- enic_vnic_post_wq_index(wq); +-} +- +-#endif /* _ENIC_VNIC_WQ_H_ */ +diff --git a/drivers/net/enic/base/vnic_cq.h b/drivers/net/enic/base/vnic_cq.h +index 922391b..ffc1aaa 100644 +--- a/drivers/net/enic/base/vnic_cq.h ++++ b/drivers/net/enic/base/vnic_cq.h +@@ -96,41 +96,46 @@ static inline unsigned int vnic_cq_service(struct vnic_cq *cq, + u8 type, u16 q_number, u16 completed_index, void *opaque), + void *opaque) + { +- struct cq_desc *cq_desc; ++ struct cq_desc *cq_desc, *cq_desc_last; + unsigned int work_done = 0; + u16 q_number, completed_index; +- u8 type, color; +- struct rte_mbuf **rx_pkts = opaque; +- unsigned int ret; ++ u8 type, color, type_color; + + cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs + + cq->ring.desc_size * cq->to_clean); +- cq_desc_dec(cq_desc, &type, &color, +- &q_number, &completed_index); ++ ++ type_color = cq_desc->type_color; ++ color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK; ++ if (color == cq->last_color) ++ return 0; + + while (color != cq->last_color) { +- if (opaque) +- opaque = (void *)&(rx_pkts[work_done]); ++ cq_desc_last = cq_desc; + +- ret = (*q_service)(cq->vdev, cq_desc, type, +- q_number, completed_index, opaque); + cq->to_clean++; + if (cq->to_clean == cq->ring.desc_count) { + cq->to_clean = 0; + cq->last_color = cq->last_color ? 0 : 1; + } + ++ work_done++; ++ if (work_done >= work_to_do) ++ break; ++ + cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs + + cq->ring.desc_size * cq->to_clean); +- cq_desc_dec(cq_desc, &type, &color, +- &q_number, &completed_index); + +- if (ret) +- work_done++; +- if (work_done >= work_to_do) +- break; ++ type_color = cq_desc->type_color; ++ color = (type_color >> CQ_DESC_COLOR_SHIFT) ++ & CQ_DESC_COLOR_MASK; ++ + } + ++ cq_desc_dec(cq_desc_last, &type, &color, ++ &q_number, &completed_index); ++ ++ (*q_service)(cq->vdev, cq_desc, type, ++ q_number, completed_index, opaque); + return work_done; + } + +diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h +index e083ccc..424415c 100644 +--- a/drivers/net/enic/base/vnic_rq.h ++++ b/drivers/net/enic/base/vnic_rq.h +@@ -74,7 +74,7 @@ struct vnic_rq { + struct vnic_dev_ring ring; + struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */ + unsigned int mbuf_next_idx; /* next mb to consume */ +- void *os_buf_head; ++ void *mb_head; + unsigned int pkts_outstanding; + uint16_t rx_nb_hold; + uint16_t rx_free_thresh; +diff --git a/drivers/net/enic/base/vnic_wq.c b/drivers/net/enic/base/vnic_wq.c +index a3ef417..ccbbd61 100644 +--- a/drivers/net/enic/base/vnic_wq.c ++++ b/drivers/net/enic/base/vnic_wq.c +@@ -59,71 +59,30 @@ int vnic_wq_alloc_ring(struct vnic_dev *vdev, struct vnic_wq *wq, + + static int vnic_wq_alloc_bufs(struct vnic_wq *wq) + { +- struct vnic_wq_buf *buf; +- unsigned int i, j, count = wq->ring.desc_count; +- unsigned int blks = VNIC_WQ_BUF_BLKS_NEEDED(count); +- +- for (i = 0; i < blks; i++) { +- wq->bufs[i] = kzalloc(VNIC_WQ_BUF_BLK_SZ(count), GFP_ATOMIC); +- if (!wq->bufs[i]) +- return -ENOMEM; +- } +- +- for (i = 0; i < blks; i++) { +- buf = wq->bufs[i]; +- for (j = 0; j < VNIC_WQ_BUF_BLK_ENTRIES(count); j++) { +- buf->index = i * VNIC_WQ_BUF_BLK_ENTRIES(count) + j; +- buf->desc = (u8 *)wq->ring.descs + +- wq->ring.desc_size * buf->index; +- if (buf->index + 1 == count) { +- buf->next = wq->bufs[0]; +- break; +- } else if (j + 1 == VNIC_WQ_BUF_BLK_ENTRIES(count)) { +- buf->next = wq->bufs[i + 1]; +- } else { +- buf->next = buf + 1; +- buf++; +- } +- } +- } +- +- wq->to_use = wq->to_clean = wq->bufs[0]; +- ++ unsigned int count = wq->ring.desc_count; ++ /* Allocate the mbuf ring */ ++ wq->bufs = (struct vnic_wq_buf *)rte_zmalloc_socket("wq->bufs", ++ sizeof(struct vnic_wq_buf) * count, ++ RTE_CACHE_LINE_SIZE, wq->socket_id); ++ wq->head_idx = 0; ++ wq->tail_idx = 0; ++ if (wq->bufs == NULL) ++ return -ENOMEM; + return 0; + } + + void vnic_wq_free(struct vnic_wq *wq) + { + struct vnic_dev *vdev; +- unsigned int i; + + vdev = wq->vdev; + + vnic_dev_free_desc_ring(vdev, &wq->ring); + +- for (i = 0; i < VNIC_WQ_BUF_BLKS_MAX; i++) { +- if (wq->bufs[i]) { +- kfree(wq->bufs[i]); +- wq->bufs[i] = NULL; +- } +- } +- ++ rte_free(wq->bufs); + wq->ctrl = NULL; + } + +-int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count, +- unsigned int desc_size) +-{ +- int mem_size = 0; +- +- mem_size += vnic_dev_desc_ring_size(&wq->ring, desc_count, desc_size); +- +- mem_size += VNIC_WQ_BUF_BLKS_NEEDED(wq->ring.desc_count) * +- VNIC_WQ_BUF_BLK_SZ(wq->ring.desc_count); +- +- return mem_size; +-} +- + + int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, + unsigned int desc_count, unsigned int desc_size) +@@ -172,9 +131,8 @@ void vnic_wq_init_start(struct vnic_wq *wq, unsigned int cq_index, + iowrite32(error_interrupt_offset, &wq->ctrl->error_interrupt_offset); + iowrite32(0, &wq->ctrl->error_status); + +- wq->to_use = wq->to_clean = +- &wq->bufs[fetch_index / VNIC_WQ_BUF_BLK_ENTRIES(count)] +- [fetch_index % VNIC_WQ_BUF_BLK_ENTRIES(count)]; ++ wq->head_idx = fetch_index; ++ wq->tail_idx = wq->head_idx; + } + + void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, +@@ -184,6 +142,7 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, + vnic_wq_init_start(wq, cq_index, 0, 0, + error_interrupt_enable, + error_interrupt_offset); ++ wq->last_completed_index = 0; + } + + void vnic_wq_error_out(struct vnic_wq *wq, unsigned int error) +@@ -219,22 +178,34 @@ int vnic_wq_disable(struct vnic_wq *wq) + return -ETIMEDOUT; + } + ++static inline uint32_t ++buf_idx_incr(uint32_t n_descriptors, uint32_t idx) ++{ ++ idx++; ++ if (unlikely(idx == n_descriptors)) ++ idx = 0; ++ return idx; ++} ++ + void vnic_wq_clean(struct vnic_wq *wq, +- void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf)) ++ void (*buf_clean)(struct vnic_wq_buf *buf)) + { + struct vnic_wq_buf *buf; ++ unsigned int to_clean = wq->tail_idx; + +- buf = wq->to_clean; ++ buf = &wq->bufs[to_clean]; + + while (vnic_wq_desc_used(wq) > 0) { + +- (*buf_clean)(wq, buf); ++ (*buf_clean)(buf); ++ to_clean = buf_idx_incr(wq->ring.desc_count, to_clean); + +- buf = wq->to_clean = buf->next; ++ buf = &wq->bufs[to_clean]; + wq->ring.desc_avail++; + } + +- wq->to_use = wq->to_clean = wq->bufs[0]; ++ wq->head_idx = 0; ++ wq->tail_idx = 0; + + iowrite32(0, &wq->ctrl->fetch_index); + iowrite32(0, &wq->ctrl->posted_index); +diff --git a/drivers/net/enic/base/vnic_wq.h b/drivers/net/enic/base/vnic_wq.h +index c23de62..37c3ff9 100644 +--- a/drivers/net/enic/base/vnic_wq.h ++++ b/drivers/net/enic/base/vnic_wq.h +@@ -64,42 +64,23 @@ struct vnic_wq_ctrl { + u32 pad9; + }; + ++/* 16 bytes */ + struct vnic_wq_buf { +- struct vnic_wq_buf *next; +- dma_addr_t dma_addr; +- void *os_buf; +- unsigned int len; +- unsigned int index; +- int sop; +- void *desc; +- uint64_t wr_id; /* Cookie */ +- uint8_t cq_entry; /* Gets completion event from hw */ +- uint8_t desc_skip_cnt; /* Num descs to occupy */ +- uint8_t compressed_send; /* Both hdr and payload in one desc */ ++ struct rte_mempool *pool; ++ void *mb; + }; + +-/* Break the vnic_wq_buf allocations into blocks of 32/64 entries */ +-#define VNIC_WQ_BUF_MIN_BLK_ENTRIES 32 +-#define VNIC_WQ_BUF_DFLT_BLK_ENTRIES 64 +-#define VNIC_WQ_BUF_BLK_ENTRIES(entries) \ +- ((unsigned int)((entries < VNIC_WQ_BUF_DFLT_BLK_ENTRIES) ? \ +- VNIC_WQ_BUF_MIN_BLK_ENTRIES : VNIC_WQ_BUF_DFLT_BLK_ENTRIES)) +-#define VNIC_WQ_BUF_BLK_SZ(entries) \ +- (VNIC_WQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_wq_buf)) +-#define VNIC_WQ_BUF_BLKS_NEEDED(entries) \ +- DIV_ROUND_UP(entries, VNIC_WQ_BUF_BLK_ENTRIES(entries)) +-#define VNIC_WQ_BUF_BLKS_MAX VNIC_WQ_BUF_BLKS_NEEDED(4096) +- + struct vnic_wq { + unsigned int index; + struct vnic_dev *vdev; + struct vnic_wq_ctrl __iomem *ctrl; /* memory-mapped */ + struct vnic_dev_ring ring; +- struct vnic_wq_buf *bufs[VNIC_WQ_BUF_BLKS_MAX]; +- struct vnic_wq_buf *to_use; +- struct vnic_wq_buf *to_clean; +- unsigned int pkts_outstanding; ++ struct vnic_wq_buf *bufs; ++ unsigned int head_idx; ++ unsigned int tail_idx; + unsigned int socket_id; ++ const struct rte_memzone *cqmsg_rz; ++ uint16_t last_completed_index; + }; + + static inline unsigned int vnic_wq_desc_avail(struct vnic_wq *wq) +@@ -114,11 +95,6 @@ static inline unsigned int vnic_wq_desc_used(struct vnic_wq *wq) + return wq->ring.desc_count - wq->ring.desc_avail - 1; + } + +-static inline void *vnic_wq_next_desc(struct vnic_wq *wq) +-{ +- return wq->to_use->desc; +-} +- + #define PI_LOG2_CACHE_LINE_SIZE 5 + #define PI_INDEX_BITS 12 + #define PI_INDEX_MASK ((1U << PI_INDEX_BITS) - 1) +@@ -191,75 +167,6 @@ static inline u64 vnic_cached_posted_index(dma_addr_t addr, unsigned int len, + PI_PREFETCH_ADDR_MASK) << PI_PREFETCH_ADDR_OFF); + } + +-static inline void vnic_wq_post(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, +- unsigned int len, int sop, int eop, +- uint8_t desc_skip_cnt, uint8_t cq_entry, +- uint8_t compressed_send, uint64_t wrid) +-{ +- struct vnic_wq_buf *buf = wq->to_use; +- +- buf->sop = sop; +- buf->cq_entry = cq_entry; +- buf->compressed_send = compressed_send; +- buf->desc_skip_cnt = desc_skip_cnt; +- buf->os_buf = os_buf; +- buf->dma_addr = dma_addr; +- buf->len = len; +- buf->wr_id = wrid; +- +- buf = buf->next; +- if (eop) { +-#ifdef DO_PREFETCH +- uint64_t wr = vnic_cached_posted_index(dma_addr, len, +- buf->index); +-#endif +- /* Adding write memory barrier prevents compiler and/or CPU +- * reordering, thus avoiding descriptor posting before +- * descriptor is initialized. Otherwise, hardware can read +- * stale descriptor fields. +- */ +- wmb(); +-#ifdef DO_PREFETCH +- /* Intel chipsets seem to limit the rate of PIOs that we can +- * push on the bus. Thus, it is very important to do a single +- * 64 bit write here. With two 32-bit writes, my maximum +- * pkt/sec rate was cut almost in half. -AJF +- */ +- iowrite64((uint64_t)wr, &wq->ctrl->posted_index); +-#else +- iowrite32(buf->index, &wq->ctrl->posted_index); +-#endif +- } +- wq->to_use = buf; +- +- wq->ring.desc_avail -= desc_skip_cnt; +-} +- +-static inline void vnic_wq_service(struct vnic_wq *wq, +- struct cq_desc *cq_desc, u16 completed_index, +- void (*buf_service)(struct vnic_wq *wq, +- struct cq_desc *cq_desc, struct vnic_wq_buf *buf, void *opaque), +- void *opaque) +-{ +- struct vnic_wq_buf *buf; +- +- buf = wq->to_clean; +- while (1) { +- +- (*buf_service)(wq, cq_desc, buf, opaque); +- +- wq->ring.desc_avail++; +- +- wq->to_clean = buf->next; +- +- if (buf->index == completed_index) +- break; +- +- buf = wq->to_clean; +- } +-} +- + void vnic_wq_free(struct vnic_wq *wq); + int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, + unsigned int desc_count, unsigned int desc_size); +@@ -275,8 +182,6 @@ unsigned int vnic_wq_error_status(struct vnic_wq *wq); + void vnic_wq_enable(struct vnic_wq *wq); + int vnic_wq_disable(struct vnic_wq *wq); + void vnic_wq_clean(struct vnic_wq *wq, +- void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf)); +-int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count, +- unsigned int desc_size); ++ void (*buf_clean)(struct vnic_wq_buf *buf)); + + #endif /* _VNIC_WQ_H_ */ +diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h +index 8c914f5..43b82a6 100644 +--- a/drivers/net/enic/enic.h ++++ b/drivers/net/enic/enic.h +@@ -155,6 +155,30 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) + return (struct enic *)eth_dev->data->dev_private; + } + ++static inline uint32_t ++enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) ++{ ++ uint32_t d = i0 + i1; ++ d -= (d >= n_descriptors) ? n_descriptors : 0; ++ return d; ++} ++ ++static inline uint32_t ++enic_ring_sub(uint32_t n_descriptors, uint32_t i0, uint32_t i1) ++{ ++ int32_t d = i1 - i0; ++ return (uint32_t)((d < 0) ? ((int32_t)n_descriptors + d) : d); ++} ++ ++static inline uint32_t ++enic_ring_incr(uint32_t n_descriptors, uint32_t idx) ++{ ++ idx++; ++ if (unlikely(idx == n_descriptors)) ++ idx = 0; ++ return idx; ++} ++ + #define RTE_LIBRTE_ENIC_ASSERT_ENABLE + #ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE + #define ASSERT(x) do { \ +@@ -209,5 +233,6 @@ extern int enic_clsf_init(struct enic *enic); + extern void enic_clsf_destroy(struct enic *enic); + uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +- ++uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, ++ uint16_t nb_pkts); + #endif /* _ENIC_H_ */ +diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c +index 6bea940..697ff82 100644 +--- a/drivers/net/enic/enic_ethdev.c ++++ b/drivers/net/enic/enic_ethdev.c +@@ -519,71 +519,6 @@ static void enicpmd_remove_mac_addr(struct rte_eth_dev *eth_dev, __rte_unused ui + enic_del_mac_address(enic); + } + +- +-static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, +- uint16_t nb_pkts) +-{ +- uint16_t index; +- unsigned int frags; +- unsigned int pkt_len; +- unsigned int seg_len; +- unsigned int inc_len; +- unsigned int nb_segs; +- struct rte_mbuf *tx_pkt, *next_tx_pkt; +- struct vnic_wq *wq = (struct vnic_wq *)tx_queue; +- struct enic *enic = vnic_dev_priv(wq->vdev); +- unsigned short vlan_id; +- unsigned short ol_flags; +- uint8_t last_seg, eop; +- unsigned int host_tx_descs = 0; +- +- for (index = 0; index < nb_pkts; index++) { +- tx_pkt = *tx_pkts++; +- inc_len = 0; +- nb_segs = tx_pkt->nb_segs; +- if (nb_segs > vnic_wq_desc_avail(wq)) { +- if (index > 0) +- enic_post_wq_index(wq); +- +- /* wq cleanup and try again */ +- if (!enic_cleanup_wq(enic, wq) || +- (nb_segs > vnic_wq_desc_avail(wq))) { +- return index; +- } +- } +- +- pkt_len = tx_pkt->pkt_len; +- vlan_id = tx_pkt->vlan_tci; +- ol_flags = tx_pkt->ol_flags; +- for (frags = 0; inc_len < pkt_len; frags++) { +- if (!tx_pkt) +- break; +- next_tx_pkt = tx_pkt->next; +- seg_len = tx_pkt->data_len; +- inc_len += seg_len; +- +- host_tx_descs++; +- last_seg = 0; +- eop = 0; +- if ((pkt_len == inc_len) || !next_tx_pkt) { +- eop = 1; +- /* post if last packet in batch or > thresh */ +- if ((index == (nb_pkts - 1)) || +- (host_tx_descs > ENIC_TX_POST_THRESH)) { +- last_seg = 1; +- host_tx_descs = 0; +- } +- } +- enic_send_pkt(enic, wq, tx_pkt, (unsigned short)seg_len, +- !frags, eop, last_seg, ol_flags, vlan_id); +- tx_pkt = next_tx_pkt; +- } +- } +- +- enic_cleanup_wq(enic, wq); +- return index; +-} +- + static const struct eth_dev_ops enicpmd_eth_dev_ops = { + .dev_configure = enicpmd_dev_configure, + .dev_start = enicpmd_dev_start, +@@ -642,7 +577,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) + enic->rte_dev = eth_dev; + eth_dev->dev_ops = &enicpmd_eth_dev_ops; + eth_dev->rx_pkt_burst = &enic_recv_pkts; +- eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts; ++ eth_dev->tx_pkt_burst = &enic_xmit_pkts; + + pdev = eth_dev->pci_dev; + rte_eth_copy_pci_info(eth_dev, pdev); +diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c +index 646d87f..ba73604 100644 +--- a/drivers/net/enic/enic_main.c ++++ b/drivers/net/enic/enic_main.c +@@ -40,11 +40,11 @@ + #include <libgen.h> + + #include <rte_pci.h> +-#include <rte_memzone.h> + #include <rte_malloc.h> + #include <rte_mbuf.h> + #include <rte_string_fns.h> + #include <rte_ethdev.h> ++#include <rte_memzone.h> + + #include "enic_compat.h" + #include "enic.h" +@@ -58,7 +58,6 @@ + #include "vnic_cq.h" + #include "vnic_intr.h" + #include "vnic_nic.h" +-#include "enic_vnic_wq.h" + + static inline struct rte_mbuf * + rte_rxmbuf_alloc(struct rte_mempool *mp) +@@ -109,38 +108,17 @@ enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) + } + } + +- + void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) + { + vnic_set_hdr_split_size(enic->vdev, split_hdr_size); + } + +-static void enic_free_wq_buf(__rte_unused struct vnic_wq *wq, struct vnic_wq_buf *buf) ++static void enic_free_wq_buf(struct vnic_wq_buf *buf) + { +- struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->os_buf; ++ struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb; + + rte_mempool_put(mbuf->pool, mbuf); +- buf->os_buf = NULL; +-} +- +-static void enic_wq_free_buf(struct vnic_wq *wq, +- __rte_unused struct cq_desc *cq_desc, +- struct vnic_wq_buf *buf, +- __rte_unused void *opaque) +-{ +- enic_free_wq_buf(wq, buf); +-} +- +-static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, +- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque) +-{ +- struct enic *enic = vnic_dev_priv(vdev); +- +- vnic_wq_service(&enic->wq[q_number], cq_desc, +- completed_index, enic_wq_free_buf, +- opaque); +- +- return 0; ++ buf->mb = NULL; + } + + static void enic_log_q_error(struct enic *enic) +@@ -163,64 +141,6 @@ static void enic_log_q_error(struct enic *enic) + } + } + +-unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq) +-{ +- unsigned int cq = enic_cq_wq(enic, wq->index); +- +- /* Return the work done */ +- return vnic_cq_service(&enic->cq[cq], +- -1 /*wq_work_to_do*/, enic_wq_service, NULL); +-} +- +-void enic_post_wq_index(struct vnic_wq *wq) +-{ +- enic_vnic_post_wq_index(wq); +-} +- +-void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, +- struct rte_mbuf *tx_pkt, unsigned short len, +- uint8_t sop, uint8_t eop, uint8_t cq_entry, +- uint16_t ol_flags, uint16_t vlan_tag) +-{ +- struct wq_enet_desc *desc = vnic_wq_next_desc(wq); +- uint16_t mss = 0; +- uint8_t vlan_tag_insert = 0; +- uint64_t bus_addr = (dma_addr_t) +- (tx_pkt->buf_physaddr + tx_pkt->data_off); +- +- if (sop) { +- if (ol_flags & PKT_TX_VLAN_PKT) +- vlan_tag_insert = 1; +- +- if (enic->hw_ip_checksum) { +- if (ol_flags & PKT_TX_IP_CKSUM) +- mss |= ENIC_CALC_IP_CKSUM; +- +- if (ol_flags & PKT_TX_TCP_UDP_CKSUM) +- mss |= ENIC_CALC_TCP_UDP_CKSUM; +- } +- } +- +- wq_enet_desc_enc(desc, +- bus_addr, +- len, +- mss, +- 0 /* header_length */, +- 0 /* offload_mode WQ_ENET_OFFLOAD_MODE_CSUM */, +- eop, +- cq_entry, +- 0 /* fcoe_encap */, +- vlan_tag_insert, +- vlan_tag, +- 0 /* loopback */); +- +- enic_vnic_post_wq(wq, (void *)tx_pkt, bus_addr, len, +- sop, +- 1 /*desc_skip_cnt*/, +- cq_entry, +- 0 /*compressed send*/, +- 0 /*wrid*/); +-} + + void enic_dev_stats_clear(struct enic *enic) + { +@@ -297,12 +217,28 @@ void enic_init_vnic_resources(struct enic *enic) + unsigned int error_interrupt_enable = 1; + unsigned int error_interrupt_offset = 0; + unsigned int index = 0; ++ unsigned int cq_idx; ++ ++ vnic_dev_stats_clear(enic->vdev); + + for (index = 0; index < enic->rq_count; index++) { + vnic_rq_init(&enic->rq[index], + enic_cq_rq(enic, index), + error_interrupt_enable, + error_interrupt_offset); ++ ++ cq_idx = enic_cq_rq(enic, index); ++ vnic_cq_init(&enic->cq[cq_idx], ++ 0 /* flow_control_enable */, ++ 1 /* color_enable */, ++ 0 /* cq_head */, ++ 0 /* cq_tail */, ++ 1 /* cq_tail_color */, ++ 0 /* interrupt_enable */, ++ 1 /* cq_entry_enable */, ++ 0 /* cq_message_enable */, ++ 0 /* interrupt offset */, ++ 0 /* cq_message_addr */); + } + + for (index = 0; index < enic->wq_count; index++) { +@@ -310,22 +246,19 @@ void enic_init_vnic_resources(struct enic *enic) + enic_cq_wq(enic, index), + error_interrupt_enable, + error_interrupt_offset); +- } +- +- vnic_dev_stats_clear(enic->vdev); + +- for (index = 0; index < enic->cq_count; index++) { +- vnic_cq_init(&enic->cq[index], ++ cq_idx = enic_cq_wq(enic, index); ++ vnic_cq_init(&enic->cq[cq_idx], + 0 /* flow_control_enable */, + 1 /* color_enable */, + 0 /* cq_head */, + 0 /* cq_tail */, + 1 /* cq_tail_color */, + 0 /* interrupt_enable */, +- 1 /* cq_entry_enable */, +- 0 /* cq_message_enable */, ++ 0 /* cq_entry_enable */, ++ 1 /* cq_message_enable */, + 0 /* interrupt offset */, +- 0 /* cq_message_addr */); ++ (u64)enic->wq[index].cqmsg_rz->phys_addr); + } + + vnic_intr_init(&enic->intr, +@@ -569,6 +502,7 @@ void enic_free_wq(void *txq) + struct vnic_wq *wq = (struct vnic_wq *)txq; + struct enic *enic = vnic_dev_priv(wq->vdev); + ++ rte_memzone_free(wq->cqmsg_rz); + vnic_wq_free(wq); + vnic_cq_free(&enic->cq[enic->rq_count + wq->index]); + } +@@ -579,6 +513,8 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, + int err; + struct vnic_wq *wq = &enic->wq[queue_idx]; + unsigned int cq_index = enic_cq_wq(enic, queue_idx); ++ char name[NAME_MAX]; ++ static int instance; + + wq->socket_id = socket_id; + if (nb_desc) { +@@ -614,6 +550,18 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, + dev_err(enic, "error in allocation of cq for wq\n"); + } + ++ /* setup up CQ message */ ++ snprintf((char *)name, sizeof(name), ++ "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx, ++ instance++); ++ ++ wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name, ++ sizeof(uint32_t), ++ SOCKET_ID_ANY, 0, ++ ENIC_ALIGN); ++ if (!wq->cqmsg_rz) ++ return -ENOMEM; ++ + return err; + } + +diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h +index 00fa71d..3e1bdf5 100644 +--- a/drivers/net/enic/enic_res.h ++++ b/drivers/net/enic/enic_res.h +@@ -53,89 +53,10 @@ + + #define ENIC_NON_TSO_MAX_DESC 16 + #define ENIC_DEFAULT_RX_FREE_THRESH 32 +-#define ENIC_TX_POST_THRESH (ENIC_MIN_WQ_DESCS / 2) ++#define ENIC_TX_XMIT_MAX 64 + + #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) + +-static inline void enic_queue_wq_desc_ex(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, unsigned int len, +- unsigned int mss_or_csum_offset, unsigned int hdr_len, +- int vlan_tag_insert, unsigned int vlan_tag, +- int offload_mode, int cq_entry, int sop, int eop, int loopback) +-{ +- struct wq_enet_desc *desc = vnic_wq_next_desc(wq); +- u8 desc_skip_cnt = 1; +- u8 compressed_send = 0; +- u64 wrid = 0; +- +- wq_enet_desc_enc(desc, +- (u64)dma_addr | VNIC_PADDR_TARGET, +- (u16)len, +- (u16)mss_or_csum_offset, +- (u16)hdr_len, (u8)offload_mode, +- (u8)eop, (u8)cq_entry, +- 0, /* fcoe_encap */ +- (u8)vlan_tag_insert, +- (u16)vlan_tag, +- (u8)loopback); +- +- vnic_wq_post(wq, os_buf, dma_addr, len, sop, eop, desc_skip_cnt, +- (u8)cq_entry, compressed_send, wrid); +-} +- +-static inline void enic_queue_wq_desc_cont(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, unsigned int len, +- int eop, int loopback) +-{ +- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, +- 0, 0, 0, 0, 0, +- eop, 0 /* !SOP */, eop, loopback); +-} +- +-static inline void enic_queue_wq_desc(struct vnic_wq *wq, void *os_buf, +- dma_addr_t dma_addr, unsigned int len, int vlan_tag_insert, +- unsigned int vlan_tag, int eop, int loopback) +-{ +- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, +- 0, 0, vlan_tag_insert, vlan_tag, +- WQ_ENET_OFFLOAD_MODE_CSUM, +- eop, 1 /* SOP */, eop, loopback); +-} +- +-static inline void enic_queue_wq_desc_csum(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, unsigned int len, +- int ip_csum, int tcpudp_csum, int vlan_tag_insert, +- unsigned int vlan_tag, int eop, int loopback) +-{ +- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, +- (ip_csum ? 1 : 0) + (tcpudp_csum ? 2 : 0), +- 0, vlan_tag_insert, vlan_tag, +- WQ_ENET_OFFLOAD_MODE_CSUM, +- eop, 1 /* SOP */, eop, loopback); +-} +- +-static inline void enic_queue_wq_desc_csum_l4(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, unsigned int len, +- unsigned int csum_offset, unsigned int hdr_len, +- int vlan_tag_insert, unsigned int vlan_tag, int eop, int loopback) +-{ +- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, +- csum_offset, hdr_len, vlan_tag_insert, vlan_tag, +- WQ_ENET_OFFLOAD_MODE_CSUM_L4, +- eop, 1 /* SOP */, eop, loopback); +-} +- +-static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, unsigned int len, +- unsigned int mss, unsigned int hdr_len, int vlan_tag_insert, +- unsigned int vlan_tag, int eop, int loopback) +-{ +- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, +- mss, hdr_len, vlan_tag_insert, vlan_tag, +- WQ_ENET_OFFLOAD_MODE_TSO, +- eop, 1 /* SOP */, eop, loopback); +-} +- + struct enic; + + int enic_get_vnic_config(struct enic *); +diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c +deleted file mode 100644 +index 39bb55c..0000000 +--- a/drivers/net/enic/enic_rx.c ++++ /dev/null +@@ -1,361 +0,0 @@ +-/* +- * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved. +- * Copyright 2007 Nuova Systems, Inc. All rights reserved. +- * +- * Copyright (c) 2014, Cisco Systems, Inc. +- * All rights reserved. +- * +- * Redistribution and use in source and binary forms, with or without +- * modification, are permitted provided that the following conditions +- * are met: +- * +- * 1. Redistributions of source code must retain the above copyright +- * notice, this list of conditions and the following disclaimer. +- * +- * 2. Redistributions in binary form must reproduce the above copyright +- * notice, this list of conditions and the following disclaimer in +- * the documentation and/or other materials provided with the +- * distribution. +- * +- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +- * POSSIBILITY OF SUCH DAMAGE. +- * +- */ +- +-#include <rte_mbuf.h> +-#include <rte_ethdev.h> +-#include <rte_prefetch.h> +- +-#include "enic_compat.h" +-#include "rq_enet_desc.h" +-#include "enic.h" +- +-#define RTE_PMD_USE_PREFETCH +- +-#ifdef RTE_PMD_USE_PREFETCH +-/* +- * Prefetch a cache line into all cache levels. +- */ +-#define rte_enic_prefetch(p) rte_prefetch0(p) +-#else +-#define rte_enic_prefetch(p) do {} while (0) +-#endif +- +-#ifdef RTE_PMD_PACKET_PREFETCH +-#define rte_packet_prefetch(p) rte_prefetch1(p) +-#else +-#define rte_packet_prefetch(p) do {} while (0) +-#endif +- +-static inline struct rte_mbuf * +-rte_rxmbuf_alloc(struct rte_mempool *mp) +-{ +- struct rte_mbuf *m; +- +- m = __rte_mbuf_raw_alloc(mp); +- __rte_mbuf_sanity_check_raw(m, 0); +- return m; +-} +- +-static inline uint16_t +-enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) +-{ +- return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; +-} +- +-static inline uint16_t +-enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) +-{ +- return(le16_to_cpu(crd->bytes_written_flags) & +- ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); +-} +- +-static inline uint8_t +-enic_cq_rx_desc_packet_error(uint16_t bwflags) +-{ +- return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == +- CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); +-} +- +-static inline uint8_t +-enic_cq_rx_desc_eop(uint16_t ciflags) +-{ +- return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) +- == CQ_ENET_RQ_DESC_FLAGS_EOP; +-} +- +-static inline uint8_t +-enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) +-{ +- return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & +- CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == +- CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); +-} +- +-static inline uint8_t +-enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) +-{ +- return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == +- CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); +-} +- +-static inline uint8_t +-enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) +-{ +- return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == +- CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); +-} +- +-static inline uint8_t +-enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) +-{ +- return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> +- CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); +-} +- +-static inline uint32_t +-enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) +-{ +- return le32_to_cpu(cqrd->rss_hash); +-} +- +-static inline uint16_t +-enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) +-{ +- return le16_to_cpu(cqrd->vlan); +-} +- +-static inline uint16_t +-enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) +-{ +- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; +- return le16_to_cpu(cqrd->bytes_written_flags) & +- CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; +-} +- +-static inline uint8_t +-enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out) +-{ +- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; +- uint16_t bwflags; +- int ret = 0; +- uint64_t pkt_err_flags = 0; +- +- bwflags = enic_cq_rx_desc_bwflags(cqrd); +- if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) { +- pkt_err_flags = PKT_RX_MAC_ERR; +- ret = 1; +- } +- *pkt_err_flags_out = pkt_err_flags; +- return ret; +-} +- +-/* +- * Lookup table to translate RX CQ flags to mbuf flags. +- */ +-static inline uint32_t +-enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) +-{ +- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; +- uint8_t cqrd_flags = cqrd->flags; +- static const uint32_t cq_type_table[128] __rte_cache_aligned = { +- [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, +- [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 +- | RTE_PTYPE_L4_UDP, +- [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 +- | RTE_PTYPE_L4_TCP, +- [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 +- | RTE_PTYPE_L4_FRAG, +- [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, +- [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 +- | RTE_PTYPE_L4_UDP, +- [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 +- | RTE_PTYPE_L4_TCP, +- [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 +- | RTE_PTYPE_L4_FRAG, +- /* All others reserved */ +- }; +- cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT +- | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 +- | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; +- return cq_type_table[cqrd_flags]; +-} +- +-static inline void +-enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) +-{ +- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; +- uint16_t ciflags, bwflags, pkt_flags = 0; +- ciflags = enic_cq_rx_desc_ciflags(cqrd); +- bwflags = enic_cq_rx_desc_bwflags(cqrd); +- +- mbuf->ol_flags = 0; +- +- /* flags are meaningless if !EOP */ +- if (unlikely(!enic_cq_rx_desc_eop(ciflags))) +- goto mbuf_flags_done; +- +- /* VLAN stripping */ +- if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { +- pkt_flags |= PKT_RX_VLAN_PKT; +- mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); +- } else { +- mbuf->vlan_tci = 0; +- } +- +- /* RSS flag */ +- if (enic_cq_rx_desc_rss_type(cqrd)) { +- pkt_flags |= PKT_RX_RSS_HASH; +- mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); +- } +- +- /* checksum flags */ +- if (!enic_cq_rx_desc_csum_not_calc(cqrd) && +- (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { +- if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) +- pkt_flags |= PKT_RX_IP_CKSUM_BAD; +- if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { +- if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) +- pkt_flags |= PKT_RX_L4_CKSUM_BAD; +- } +- } +- +- mbuf_flags_done: +- mbuf->ol_flags = pkt_flags; +-} +- +-static inline uint32_t +-enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) +-{ +- uint32_t d = i0 + i1; +- ASSERT(i0 < n_descriptors); +- ASSERT(i1 < n_descriptors); +- d -= (d >= n_descriptors) ? n_descriptors : 0; +- return d; +-} +- +- +-uint16_t +-enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, +- uint16_t nb_pkts) +-{ +- struct vnic_rq *rq = rx_queue; +- struct enic *enic = vnic_dev_priv(rq->vdev); +- unsigned int rx_id; +- struct rte_mbuf *nmb, *rxmb; +- uint16_t nb_rx = 0; +- uint16_t nb_hold; +- struct vnic_cq *cq; +- volatile struct cq_desc *cqd_ptr; +- uint8_t color; +- +- cq = &enic->cq[enic_cq_rq(enic, rq->index)]; +- rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ +- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; +- +- nb_hold = rq->rx_nb_hold; /* mbufs held by software */ +- +- while (nb_rx < nb_pkts) { +- volatile struct rq_enet_desc *rqd_ptr; +- dma_addr_t dma_addr; +- struct cq_desc cqd; +- uint64_t ol_err_flags; +- uint8_t packet_error; +- +- /* Check for pkts available */ +- color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) +- & CQ_DESC_COLOR_MASK; +- if (color == cq->last_color) +- break; +- +- /* Get the cq descriptor and rq pointer */ +- cqd = *cqd_ptr; +- rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; +- +- /* allocate a new mbuf */ +- nmb = rte_rxmbuf_alloc(rq->mp); +- if (nmb == NULL) { +- dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", +- enic->port_id, (unsigned)rq->index); +- rte_eth_devices[enic->port_id]. +- data->rx_mbuf_alloc_failed++; +- break; +- } +- +- /* A packet error means descriptor and data are untrusted */ +- packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); +- +- /* Get the mbuf to return and replace with one just allocated */ +- rxmb = rq->mbuf_ring[rx_id]; +- rq->mbuf_ring[rx_id] = nmb; +- +- /* Increment cqd, rqd, mbuf_table index */ +- rx_id++; +- if (unlikely(rx_id == rq->ring.desc_count)) { +- rx_id = 0; +- cq->last_color = cq->last_color ? 0 : 1; +- } +- +- /* Prefetch next mbuf & desc while processing current one */ +- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; +- rte_enic_prefetch(cqd_ptr); +- rte_enic_prefetch(rq->mbuf_ring[rx_id]); +- rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) +- + rx_id); +- +- /* Push descriptor for newly allocated mbuf */ +- dma_addr = (dma_addr_t)(nmb->buf_physaddr +- + RTE_PKTMBUF_HEADROOM); +- rqd_ptr->address = rte_cpu_to_le_64(dma_addr); +- rqd_ptr->length_type = cpu_to_le16(nmb->buf_len +- - RTE_PKTMBUF_HEADROOM); +- +- /* Fill in the rest of the mbuf */ +- rxmb->data_off = RTE_PKTMBUF_HEADROOM; +- rxmb->nb_segs = 1; +- rxmb->next = NULL; +- rxmb->port = enic->port_id; +- if (!packet_error) { +- rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); +- rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); +- enic_cq_rx_to_pkt_flags(&cqd, rxmb); +- } else { +- rxmb->pkt_len = 0; +- rxmb->packet_type = 0; +- rxmb->ol_flags = 0; +- } +- rxmb->data_len = rxmb->pkt_len; +- +- /* prefetch mbuf data for caller */ +- rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, +- RTE_PKTMBUF_HEADROOM)); +- +- /* store the mbuf address into the next entry of the array */ +- rx_pkts[nb_rx++] = rxmb; +- } +- +- nb_hold += nb_rx; +- cq->to_clean = rx_id; +- +- if (nb_hold > rq->rx_free_thresh) { +- rq->posted_index = enic_ring_add(rq->ring.desc_count, +- rq->posted_index, nb_hold); +- nb_hold = 0; +- rte_mb(); +- iowrite32(rq->posted_index, &rq->ctrl->posted_index); +- } +- +- rq->rx_nb_hold = nb_hold; +- +- return nb_rx; +-} +diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c +new file mode 100644 +index 0000000..71ca34e +--- /dev/null ++++ b/drivers/net/enic/enic_rxtx.c +@@ -0,0 +1,505 @@ ++/* ++ * Copyright 2008-2016 Cisco Systems, Inc. All rights reserved. ++ * Copyright 2007 Nuova Systems, Inc. All rights reserved. ++ * ++ * Copyright (c) 2016, Cisco Systems, Inc. ++ * All rights reserved. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions ++ * are met: ++ * ++ * 1. Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * 2. Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in ++ * the documentation and/or other materials provided with the ++ * distribution. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS ++ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ++ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, ++ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, ++ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER ++ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT ++ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ++ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE ++ * POSSIBILITY OF SUCH DAMAGE. ++ * ++ */ ++ ++#include <rte_mbuf.h> ++#include <rte_ethdev.h> ++#include <rte_prefetch.h> ++#include <rte_memzone.h> ++ ++#include "enic_compat.h" ++#include "rq_enet_desc.h" ++#include "enic.h" ++ ++#define RTE_PMD_USE_PREFETCH ++ ++#ifdef RTE_PMD_USE_PREFETCH ++/* ++ * Prefetch a cache line into all cache levels. ++ */ ++#define rte_enic_prefetch(p) rte_prefetch0(p) ++#else ++#define rte_enic_prefetch(p) do {} while (0) ++#endif ++ ++#ifdef RTE_PMD_PACKET_PREFETCH ++#define rte_packet_prefetch(p) rte_prefetch1(p) ++#else ++#define rte_packet_prefetch(p) do {} while (0) ++#endif ++ ++static inline struct rte_mbuf * ++rte_rxmbuf_alloc(struct rte_mempool *mp) ++{ ++ struct rte_mbuf *m; ++ ++ m = __rte_mbuf_raw_alloc(mp); ++ __rte_mbuf_sanity_check_raw(m, 0); ++ return m; ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) ++{ ++ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) ++{ ++ return(le16_to_cpu(crd->bytes_written_flags) & ++ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_packet_error(uint16_t bwflags) ++{ ++ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == ++ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_eop(uint16_t ciflags) ++{ ++ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) ++ == CQ_ENET_RQ_DESC_FLAGS_EOP; ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) ++{ ++ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & ++ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == ++ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) ++{ ++ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == ++ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) ++{ ++ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == ++ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) ++{ ++ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> ++ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); ++} ++ ++static inline uint32_t ++enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) ++{ ++ return le32_to_cpu(cqrd->rss_hash); ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) ++{ ++ return le16_to_cpu(cqrd->vlan); ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ return le16_to_cpu(cqrd->bytes_written_flags) & ++ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; ++} ++ ++static inline uint8_t ++enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ uint16_t bwflags; ++ int ret = 0; ++ uint64_t pkt_err_flags = 0; ++ ++ bwflags = enic_cq_rx_desc_bwflags(cqrd); ++ if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) { ++ pkt_err_flags = PKT_RX_MAC_ERR; ++ ret = 1; ++ } ++ *pkt_err_flags_out = pkt_err_flags; ++ return ret; ++} ++ ++/* ++ * Lookup table to translate RX CQ flags to mbuf flags. ++ */ ++static inline uint32_t ++enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ uint8_t cqrd_flags = cqrd->flags; ++ static const uint32_t cq_type_table[128] __rte_cache_aligned = { ++ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, ++ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 ++ | RTE_PTYPE_L4_UDP, ++ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 ++ | RTE_PTYPE_L4_TCP, ++ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 ++ | RTE_PTYPE_L4_FRAG, ++ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, ++ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 ++ | RTE_PTYPE_L4_UDP, ++ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 ++ | RTE_PTYPE_L4_TCP, ++ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 ++ | RTE_PTYPE_L4_FRAG, ++ /* All others reserved */ ++ }; ++ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT ++ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 ++ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; ++ return cq_type_table[cqrd_flags]; ++} ++ ++static inline void ++enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ uint16_t ciflags, bwflags, pkt_flags = 0; ++ ciflags = enic_cq_rx_desc_ciflags(cqrd); ++ bwflags = enic_cq_rx_desc_bwflags(cqrd); ++ ++ mbuf->ol_flags = 0; ++ ++ /* flags are meaningless if !EOP */ ++ if (unlikely(!enic_cq_rx_desc_eop(ciflags))) ++ goto mbuf_flags_done; ++ ++ /* VLAN stripping */ ++ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { ++ pkt_flags |= PKT_RX_VLAN_PKT; ++ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); ++ } else { ++ mbuf->vlan_tci = 0; ++ } ++ ++ /* RSS flag */ ++ if (enic_cq_rx_desc_rss_type(cqrd)) { ++ pkt_flags |= PKT_RX_RSS_HASH; ++ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); ++ } ++ ++ /* checksum flags */ ++ if (!enic_cq_rx_desc_csum_not_calc(cqrd) && ++ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { ++ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) ++ pkt_flags |= PKT_RX_IP_CKSUM_BAD; ++ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { ++ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) ++ pkt_flags |= PKT_RX_L4_CKSUM_BAD; ++ } ++ } ++ ++ mbuf_flags_done: ++ mbuf->ol_flags = pkt_flags; ++} ++ ++uint16_t ++enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, ++ uint16_t nb_pkts) ++{ ++ struct vnic_rq *rq = rx_queue; ++ struct enic *enic = vnic_dev_priv(rq->vdev); ++ unsigned int rx_id; ++ struct rte_mbuf *nmb, *rxmb; ++ uint16_t nb_rx = 0; ++ uint16_t nb_hold; ++ struct vnic_cq *cq; ++ volatile struct cq_desc *cqd_ptr; ++ uint8_t color; ++ ++ cq = &enic->cq[enic_cq_rq(enic, rq->index)]; ++ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ ++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; ++ ++ nb_hold = rq->rx_nb_hold; /* mbufs held by software */ ++ ++ while (nb_rx < nb_pkts) { ++ volatile struct rq_enet_desc *rqd_ptr; ++ dma_addr_t dma_addr; ++ struct cq_desc cqd; ++ uint64_t ol_err_flags; ++ uint8_t packet_error; ++ ++ /* Check for pkts available */ ++ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) ++ & CQ_DESC_COLOR_MASK; ++ if (color == cq->last_color) ++ break; ++ ++ /* Get the cq descriptor and rq pointer */ ++ cqd = *cqd_ptr; ++ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; ++ ++ /* allocate a new mbuf */ ++ nmb = rte_rxmbuf_alloc(rq->mp); ++ if (nmb == NULL) { ++ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", ++ enic->port_id, (unsigned)rq->index); ++ rte_eth_devices[enic->port_id]. ++ data->rx_mbuf_alloc_failed++; ++ break; ++ } ++ ++ /* A packet error means descriptor and data are untrusted */ ++ packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); ++ ++ /* Get the mbuf to return and replace with one just allocated */ ++ rxmb = rq->mbuf_ring[rx_id]; ++ rq->mbuf_ring[rx_id] = nmb; ++ ++ /* Increment cqd, rqd, mbuf_table index */ ++ rx_id++; ++ if (unlikely(rx_id == rq->ring.desc_count)) { ++ rx_id = 0; ++ cq->last_color = cq->last_color ? 0 : 1; ++ } ++ ++ /* Prefetch next mbuf & desc while processing current one */ ++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; ++ rte_enic_prefetch(cqd_ptr); ++ rte_enic_prefetch(rq->mbuf_ring[rx_id]); ++ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) ++ + rx_id); ++ ++ /* Push descriptor for newly allocated mbuf */ ++ dma_addr = (dma_addr_t)(nmb->buf_physaddr ++ + RTE_PKTMBUF_HEADROOM); ++ rqd_ptr->address = rte_cpu_to_le_64(dma_addr); ++ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len ++ - RTE_PKTMBUF_HEADROOM); ++ ++ /* Fill in the rest of the mbuf */ ++ rxmb->data_off = RTE_PKTMBUF_HEADROOM; ++ rxmb->nb_segs = 1; ++ rxmb->next = NULL; ++ rxmb->port = enic->port_id; ++ if (!packet_error) { ++ rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); ++ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); ++ enic_cq_rx_to_pkt_flags(&cqd, rxmb); ++ } else { ++ rxmb->pkt_len = 0; ++ rxmb->packet_type = 0; ++ rxmb->ol_flags = 0; ++ } ++ rxmb->data_len = rxmb->pkt_len; ++ ++ /* prefetch mbuf data for caller */ ++ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, ++ RTE_PKTMBUF_HEADROOM)); ++ ++ /* store the mbuf address into the next entry of the array */ ++ rx_pkts[nb_rx++] = rxmb; ++ } ++ ++ nb_hold += nb_rx; ++ cq->to_clean = rx_id; ++ ++ if (nb_hold > rq->rx_free_thresh) { ++ rq->posted_index = enic_ring_add(rq->ring.desc_count, ++ rq->posted_index, nb_hold); ++ nb_hold = 0; ++ rte_mb(); ++ iowrite32(rq->posted_index, &rq->ctrl->posted_index); ++ } ++ ++ rq->rx_nb_hold = nb_hold; ++ ++ return nb_rx; ++} ++ ++static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index) ++{ ++ struct vnic_wq_buf *buf; ++ struct rte_mbuf *m, *free[ENIC_MAX_WQ_DESCS]; ++ unsigned int nb_to_free, nb_free = 0, i; ++ struct rte_mempool *pool; ++ unsigned int tail_idx; ++ unsigned int desc_count = wq->ring.desc_count; ++ ++ nb_to_free = enic_ring_sub(desc_count, wq->tail_idx, completed_index) ++ + 1; ++ tail_idx = wq->tail_idx; ++ buf = &wq->bufs[tail_idx]; ++ pool = ((struct rte_mbuf *)buf->mb)->pool; ++ for (i = 0; i < nb_to_free; i++) { ++ buf = &wq->bufs[tail_idx]; ++ m = (struct rte_mbuf *)(buf->mb); ++ if (likely(m->pool == pool)) { ++ ASSERT(nb_free < ENIC_MAX_WQ_DESCS); ++ free[nb_free++] = m; ++ } else { ++ rte_mempool_put_bulk(pool, (void *)free, nb_free); ++ free[0] = m; ++ nb_free = 1; ++ pool = m->pool; ++ } ++ tail_idx = enic_ring_incr(desc_count, tail_idx); ++ buf->mb = NULL; ++ } ++ ++ rte_mempool_put_bulk(pool, (void **)free, nb_free); ++ ++ wq->tail_idx = tail_idx; ++ wq->ring.desc_avail += nb_to_free; ++} ++ ++unsigned int enic_cleanup_wq(__rte_unused struct enic *enic, struct vnic_wq *wq) ++{ ++ u16 completed_index; ++ ++ completed_index = *((uint32_t *)wq->cqmsg_rz->addr) & 0xffff; ++ ++ if (wq->last_completed_index != completed_index) { ++ enic_free_wq_bufs(wq, completed_index); ++ wq->last_completed_index = completed_index; ++ } ++ return 0; ++} ++ ++uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, ++ uint16_t nb_pkts) ++{ ++ uint16_t index; ++ unsigned int pkt_len, data_len; ++ unsigned int nb_segs; ++ struct rte_mbuf *tx_pkt; ++ struct vnic_wq *wq = (struct vnic_wq *)tx_queue; ++ struct enic *enic = vnic_dev_priv(wq->vdev); ++ unsigned short vlan_id; ++ unsigned short ol_flags; ++ unsigned int wq_desc_avail; ++ int head_idx; ++ struct vnic_wq_buf *buf; ++ unsigned int hw_ip_cksum_enabled; ++ unsigned int desc_count; ++ struct wq_enet_desc *descs, *desc_p, desc_tmp; ++ uint16_t mss; ++ uint8_t vlan_tag_insert; ++ uint8_t eop; ++ uint64_t bus_addr; ++ ++ enic_cleanup_wq(enic, wq); ++ wq_desc_avail = vnic_wq_desc_avail(wq); ++ head_idx = wq->head_idx; ++ desc_count = wq->ring.desc_count; ++ ++ nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX); ++ ++ hw_ip_cksum_enabled = enic->hw_ip_checksum; ++ for (index = 0; index < nb_pkts; index++) { ++ tx_pkt = *tx_pkts++; ++ nb_segs = tx_pkt->nb_segs; ++ if (nb_segs > wq_desc_avail) { ++ if (index > 0) ++ goto post; ++ goto done; ++ } ++ ++ pkt_len = tx_pkt->pkt_len; ++ data_len = tx_pkt->data_len; ++ vlan_id = tx_pkt->vlan_tci; ++ ol_flags = tx_pkt->ol_flags; ++ ++ mss = 0; ++ vlan_tag_insert = 0; ++ bus_addr = (dma_addr_t) ++ (tx_pkt->buf_physaddr + tx_pkt->data_off); ++ ++ descs = (struct wq_enet_desc *)wq->ring.descs; ++ desc_p = descs + head_idx; ++ ++ eop = (data_len == pkt_len); ++ ++ if (ol_flags & PKT_TX_VLAN_PKT) ++ vlan_tag_insert = 1; ++ ++ if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_IP_CKSUM)) ++ mss |= ENIC_CALC_IP_CKSUM; ++ ++ if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_TCP_UDP_CKSUM)) ++ mss |= ENIC_CALC_TCP_UDP_CKSUM; ++ ++ wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, 0, 0, eop, ++ eop, 0, vlan_tag_insert, vlan_id, 0); ++ ++ *desc_p = desc_tmp; ++ buf = &wq->bufs[head_idx]; ++ buf->mb = (void *)tx_pkt; ++ head_idx = enic_ring_incr(desc_count, head_idx); ++ wq_desc_avail--; ++ ++ if (!eop) { ++ for (tx_pkt = tx_pkt->next; tx_pkt; tx_pkt = ++ tx_pkt->next) { ++ data_len = tx_pkt->data_len; ++ ++ if (tx_pkt->next == NULL) ++ eop = 1; ++ desc_p = descs + head_idx; ++ bus_addr = (dma_addr_t)(tx_pkt->buf_physaddr ++ + tx_pkt->data_off); ++ wq_enet_desc_enc((struct wq_enet_desc *) ++ &desc_tmp, bus_addr, data_len, ++ mss, 0, 0, eop, eop, 0, ++ vlan_tag_insert, vlan_id, 0); ++ ++ *desc_p = desc_tmp; ++ buf = &wq->bufs[head_idx]; ++ buf->mb = (void *)tx_pkt; ++ head_idx = enic_ring_incr(desc_count, head_idx); ++ wq_desc_avail--; ++ } ++ } ++ } ++ post: ++ rte_wmb(); ++ iowrite32(head_idx, &wq->ctrl->posted_index); ++ done: ++ wq->ring.desc_avail = wq_desc_avail; ++ wq->head_idx = head_idx; ++ ++ return index; ++} +-- +2.7.0 + diff --git a/dpdk/dpdk-2.2.0_patches/0024-enic-fix-imissed-rx-counter.patch b/dpdk/dpdk-2.2.0_patches/0024-enic-fix-imissed-rx-counter.patch index db86c9b057a..81e7bf3dbf6 100644 --- a/dpdk/dpdk-2.2.0_patches/0024-enic-fix-imissed-rx-counter.patch +++ b/dpdk/dpdk-2.2.0_patches/0024-enic-fix-imissed-rx-counter.patch @@ -1,18 +1,20 @@ -From c6c5a3f41528b28764ee4aa96a72d89c94d695df Mon Sep 17 00:00:00 2001 +From 3433c7828ec909fccb768636ee21867030da14c9 Mon Sep 17 00:00:00 2001 From: John Daley <johndale@cisco.com> -Date: Fri, 15 Apr 2016 14:43:11 -0700 -Subject: [PATCH] enic: fix imissed rx counter +Date: Tue, 26 Apr 2016 13:30:50 -0700 +Subject: [PATCH 1/3] enic: fix 'imissed' to count drops due to lack of RX + buffers +Fixes: 7182d3e7d177 ("enic: expose Rx missed packets counter") Signed-off-by: John Daley <johndale@cisco.com> --- drivers/net/enic/enic_main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index c61433f..8d6bbeb 100644 +index 60fe765..be4e9e5 100644 --- a/drivers/net/enic/enic_main.c +++ b/drivers/net/enic/enic_main.c -@@ -165,10 +165,10 @@ void enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats) +@@ -243,10 +243,10 @@ void enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats) r_stats->ibytes = stats->rx.rx_bytes_ok; r_stats->obytes = stats->tx.tx_bytes_ok; @@ -23,8 +25,8 @@ index c61433f..8d6bbeb 100644 - r_stats->imissed = stats->rx.rx_drop; + r_stats->imissed = stats->rx.rx_no_bufs; - r_stats->imcasts = stats->rx.rx_multicast_frames_ok; r_stats->rx_nombuf = stats->rx.rx_no_bufs; + } -- 2.7.0 diff --git a/dpdk/dpdk-2.2.0_patches/0025-enic-fix-misalignment-of-Rx-mbuf-data.patch b/dpdk/dpdk-2.2.0_patches/0025-enic-fix-misalignment-of-Rx-mbuf-data.patch new file mode 100644 index 00000000000..69ca3f31087 --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0025-enic-fix-misalignment-of-Rx-mbuf-data.patch @@ -0,0 +1,55 @@ +From 454eb71eca1912e32a509c738a99a340cc2488cf Mon Sep 17 00:00:00 2001 +From: John Daley <johndale@cisco.com> +Date: Mon, 25 Apr 2016 16:24:53 -0700 +Subject: [PATCH 2/3] enic: fix misalignment of Rx mbuf data + +Data DMA used m->data_off of uninitialized mbufs instead of +RTE_PKTMBUF_HEADROOM, potentially causing Rx data to be +placed at the wrong alignment in the mbuf. + +Fixes: 947d860c821f ("enic: improve Rx performance") +Signed-off-by: John Daley <johndale@cisco.com> +--- + drivers/net/enic/enic_main.c | 5 +++-- + drivers/net/enic/enic_rx.c | 6 ++++-- + 2 files changed, 7 insertions(+), 4 deletions(-) + +diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c +index be4e9e5..646d87f 100644 +--- a/drivers/net/enic/enic_main.c ++++ b/drivers/net/enic/enic_main.c +@@ -354,10 +354,11 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) + return -ENOMEM; + } + +- dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off); ++ dma_addr = (dma_addr_t)(mb->buf_physaddr ++ + RTE_PKTMBUF_HEADROOM); + + rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, +- mb->buf_len); ++ mb->buf_len - RTE_PKTMBUF_HEADROOM); + rq->mbuf_ring[i] = mb; + } + +diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c +index 232987a..39bb55c 100644 +--- a/drivers/net/enic/enic_rx.c ++++ b/drivers/net/enic/enic_rx.c +@@ -314,9 +314,11 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + + rx_id); + + /* Push descriptor for newly allocated mbuf */ +- dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off); ++ dma_addr = (dma_addr_t)(nmb->buf_physaddr ++ + RTE_PKTMBUF_HEADROOM); + rqd_ptr->address = rte_cpu_to_le_64(dma_addr); +- rqd_ptr->length_type = cpu_to_le16(nmb->buf_len); ++ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len ++ - RTE_PKTMBUF_HEADROOM); + + /* Fill in the rest of the mbuf */ + rxmb->data_off = RTE_PKTMBUF_HEADROOM; +-- +2.7.0 + diff --git a/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch b/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch new file mode 100644 index 00000000000..4858b8f5d39 --- /dev/null +++ b/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch @@ -0,0 +1,1844 @@ +From ca6bbb723880e91d006de6cc485259da988859aa Mon Sep 17 00:00:00 2001 +From: John Daley <johndale@cisco.com> +Date: Tue, 5 Apr 2016 15:19:06 -0700 +Subject: [PATCH 3/3] enic: Optimization of Tx path to reduce Host CPU + overhead, cleanup + +Optimizations and cleanup: +- flatten packet send path +- flatten mbuf free path +- disable CQ entry writing and use CQ messages instead +- use rte_mempool_put_bulk() to bulk return freed mbufs +- remove unnecessary fields vnic_bufs struct, use contiguous array of cache + aligned divisible elements. No next pointers. +- use local variables inside per packet loop instead of fields in structs. +- factor book keeping out of the per packet tx loop where possible + (removed several conditionals) +- put Tx and Rx code in 1 file (enic_rxtx.c) + +Reviewed-by: Nelson Escobar <neescoba@cisco.com> +Signed-off-by: John Daley <johndale@cisco.com> +--- + drivers/net/enic/Makefile | 2 +- + drivers/net/enic/base/enic_vnic_wq.h | 79 ------ + drivers/net/enic/base/vnic_cq.h | 37 +-- + drivers/net/enic/base/vnic_rq.h | 2 +- + drivers/net/enic/base/vnic_wq.c | 89 +++--- + drivers/net/enic/base/vnic_wq.h | 113 +------- + drivers/net/enic/enic.h | 27 +- + drivers/net/enic/enic_ethdev.c | 67 +---- + drivers/net/enic/enic_main.c | 132 +++------ + drivers/net/enic/enic_res.h | 81 +----- + drivers/net/enic/enic_rx.c | 361 ------------------------- + drivers/net/enic/enic_rxtx.c | 505 +++++++++++++++++++++++++++++++++++ + 12 files changed, 635 insertions(+), 860 deletions(-) + delete mode 100644 drivers/net/enic/base/enic_vnic_wq.h + delete mode 100644 drivers/net/enic/enic_rx.c + create mode 100644 drivers/net/enic/enic_rxtx.c + +diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile +index f316274..3926b79 100644 +--- a/drivers/net/enic/Makefile ++++ b/drivers/net/enic/Makefile +@@ -53,7 +53,7 @@ VPATH += $(SRCDIR)/src + # + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c +-SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c ++SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rxtx.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c + SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c +diff --git a/drivers/net/enic/base/enic_vnic_wq.h b/drivers/net/enic/base/enic_vnic_wq.h +deleted file mode 100644 +index b019109..0000000 +--- a/drivers/net/enic/base/enic_vnic_wq.h ++++ /dev/null +@@ -1,79 +0,0 @@ +-/* +- * Copyright 2008-2015 Cisco Systems, Inc. All rights reserved. +- * Copyright 2007 Nuova Systems, Inc. All rights reserved. +- * +- * Copyright (c) 2015, Cisco Systems, Inc. +- * All rights reserved. +- * +- * Redistribution and use in source and binary forms, with or without +- * modification, are permitted provided that the following conditions +- * are met: +- * +- * 1. Redistributions of source code must retain the above copyright +- * notice, this list of conditions and the following disclaimer. +- * +- * 2. Redistributions in binary form must reproduce the above copyright +- * notice, this list of conditions and the following disclaimer in +- * the documentation and/or other materials provided with the +- * distribution. +- * +- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +- * POSSIBILITY OF SUCH DAMAGE. +- * +- */ +- +-#ifndef _ENIC_VNIC_WQ_H_ +-#define _ENIC_VNIC_WQ_H_ +- +-#include "vnic_dev.h" +-#include "vnic_cq.h" +- +-static inline void enic_vnic_post_wq_index(struct vnic_wq *wq) +-{ +- struct vnic_wq_buf *buf = wq->to_use; +- +- /* Adding write memory barrier prevents compiler and/or CPU +- * reordering, thus avoiding descriptor posting before +- * descriptor is initialized. Otherwise, hardware can read +- * stale descriptor fields. +- */ +- wmb(); +- iowrite32(buf->index, &wq->ctrl->posted_index); +-} +- +-static inline void enic_vnic_post_wq(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, +- unsigned int len, int sop, +- uint8_t desc_skip_cnt, uint8_t cq_entry, +- uint8_t compressed_send, uint64_t wrid) +-{ +- struct vnic_wq_buf *buf = wq->to_use; +- +- buf->sop = sop; +- buf->cq_entry = cq_entry; +- buf->compressed_send = compressed_send; +- buf->desc_skip_cnt = desc_skip_cnt; +- buf->os_buf = os_buf; +- buf->dma_addr = dma_addr; +- buf->len = len; +- buf->wr_id = wrid; +- +- buf = buf->next; +- wq->ring.desc_avail -= desc_skip_cnt; +- wq->to_use = buf; +- +- if (cq_entry) +- enic_vnic_post_wq_index(wq); +-} +- +-#endif /* _ENIC_VNIC_WQ_H_ */ +diff --git a/drivers/net/enic/base/vnic_cq.h b/drivers/net/enic/base/vnic_cq.h +index 922391b..ffc1aaa 100644 +--- a/drivers/net/enic/base/vnic_cq.h ++++ b/drivers/net/enic/base/vnic_cq.h +@@ -96,41 +96,46 @@ static inline unsigned int vnic_cq_service(struct vnic_cq *cq, + u8 type, u16 q_number, u16 completed_index, void *opaque), + void *opaque) + { +- struct cq_desc *cq_desc; ++ struct cq_desc *cq_desc, *cq_desc_last; + unsigned int work_done = 0; + u16 q_number, completed_index; +- u8 type, color; +- struct rte_mbuf **rx_pkts = opaque; +- unsigned int ret; ++ u8 type, color, type_color; + + cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs + + cq->ring.desc_size * cq->to_clean); +- cq_desc_dec(cq_desc, &type, &color, +- &q_number, &completed_index); ++ ++ type_color = cq_desc->type_color; ++ color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK; ++ if (color == cq->last_color) ++ return 0; + + while (color != cq->last_color) { +- if (opaque) +- opaque = (void *)&(rx_pkts[work_done]); ++ cq_desc_last = cq_desc; + +- ret = (*q_service)(cq->vdev, cq_desc, type, +- q_number, completed_index, opaque); + cq->to_clean++; + if (cq->to_clean == cq->ring.desc_count) { + cq->to_clean = 0; + cq->last_color = cq->last_color ? 0 : 1; + } + ++ work_done++; ++ if (work_done >= work_to_do) ++ break; ++ + cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs + + cq->ring.desc_size * cq->to_clean); +- cq_desc_dec(cq_desc, &type, &color, +- &q_number, &completed_index); + +- if (ret) +- work_done++; +- if (work_done >= work_to_do) +- break; ++ type_color = cq_desc->type_color; ++ color = (type_color >> CQ_DESC_COLOR_SHIFT) ++ & CQ_DESC_COLOR_MASK; ++ + } + ++ cq_desc_dec(cq_desc_last, &type, &color, ++ &q_number, &completed_index); ++ ++ (*q_service)(cq->vdev, cq_desc, type, ++ q_number, completed_index, opaque); + return work_done; + } + +diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h +index e083ccc..424415c 100644 +--- a/drivers/net/enic/base/vnic_rq.h ++++ b/drivers/net/enic/base/vnic_rq.h +@@ -74,7 +74,7 @@ struct vnic_rq { + struct vnic_dev_ring ring; + struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */ + unsigned int mbuf_next_idx; /* next mb to consume */ +- void *os_buf_head; ++ void *mb_head; + unsigned int pkts_outstanding; + uint16_t rx_nb_hold; + uint16_t rx_free_thresh; +diff --git a/drivers/net/enic/base/vnic_wq.c b/drivers/net/enic/base/vnic_wq.c +index a3ef417..ccbbd61 100644 +--- a/drivers/net/enic/base/vnic_wq.c ++++ b/drivers/net/enic/base/vnic_wq.c +@@ -59,71 +59,30 @@ int vnic_wq_alloc_ring(struct vnic_dev *vdev, struct vnic_wq *wq, + + static int vnic_wq_alloc_bufs(struct vnic_wq *wq) + { +- struct vnic_wq_buf *buf; +- unsigned int i, j, count = wq->ring.desc_count; +- unsigned int blks = VNIC_WQ_BUF_BLKS_NEEDED(count); +- +- for (i = 0; i < blks; i++) { +- wq->bufs[i] = kzalloc(VNIC_WQ_BUF_BLK_SZ(count), GFP_ATOMIC); +- if (!wq->bufs[i]) +- return -ENOMEM; +- } +- +- for (i = 0; i < blks; i++) { +- buf = wq->bufs[i]; +- for (j = 0; j < VNIC_WQ_BUF_BLK_ENTRIES(count); j++) { +- buf->index = i * VNIC_WQ_BUF_BLK_ENTRIES(count) + j; +- buf->desc = (u8 *)wq->ring.descs + +- wq->ring.desc_size * buf->index; +- if (buf->index + 1 == count) { +- buf->next = wq->bufs[0]; +- break; +- } else if (j + 1 == VNIC_WQ_BUF_BLK_ENTRIES(count)) { +- buf->next = wq->bufs[i + 1]; +- } else { +- buf->next = buf + 1; +- buf++; +- } +- } +- } +- +- wq->to_use = wq->to_clean = wq->bufs[0]; +- ++ unsigned int count = wq->ring.desc_count; ++ /* Allocate the mbuf ring */ ++ wq->bufs = (struct vnic_wq_buf *)rte_zmalloc_socket("wq->bufs", ++ sizeof(struct vnic_wq_buf) * count, ++ RTE_CACHE_LINE_SIZE, wq->socket_id); ++ wq->head_idx = 0; ++ wq->tail_idx = 0; ++ if (wq->bufs == NULL) ++ return -ENOMEM; + return 0; + } + + void vnic_wq_free(struct vnic_wq *wq) + { + struct vnic_dev *vdev; +- unsigned int i; + + vdev = wq->vdev; + + vnic_dev_free_desc_ring(vdev, &wq->ring); + +- for (i = 0; i < VNIC_WQ_BUF_BLKS_MAX; i++) { +- if (wq->bufs[i]) { +- kfree(wq->bufs[i]); +- wq->bufs[i] = NULL; +- } +- } +- ++ rte_free(wq->bufs); + wq->ctrl = NULL; + } + +-int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count, +- unsigned int desc_size) +-{ +- int mem_size = 0; +- +- mem_size += vnic_dev_desc_ring_size(&wq->ring, desc_count, desc_size); +- +- mem_size += VNIC_WQ_BUF_BLKS_NEEDED(wq->ring.desc_count) * +- VNIC_WQ_BUF_BLK_SZ(wq->ring.desc_count); +- +- return mem_size; +-} +- + + int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, + unsigned int desc_count, unsigned int desc_size) +@@ -172,9 +131,8 @@ void vnic_wq_init_start(struct vnic_wq *wq, unsigned int cq_index, + iowrite32(error_interrupt_offset, &wq->ctrl->error_interrupt_offset); + iowrite32(0, &wq->ctrl->error_status); + +- wq->to_use = wq->to_clean = +- &wq->bufs[fetch_index / VNIC_WQ_BUF_BLK_ENTRIES(count)] +- [fetch_index % VNIC_WQ_BUF_BLK_ENTRIES(count)]; ++ wq->head_idx = fetch_index; ++ wq->tail_idx = wq->head_idx; + } + + void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, +@@ -184,6 +142,7 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, + vnic_wq_init_start(wq, cq_index, 0, 0, + error_interrupt_enable, + error_interrupt_offset); ++ wq->last_completed_index = 0; + } + + void vnic_wq_error_out(struct vnic_wq *wq, unsigned int error) +@@ -219,22 +178,34 @@ int vnic_wq_disable(struct vnic_wq *wq) + return -ETIMEDOUT; + } + ++static inline uint32_t ++buf_idx_incr(uint32_t n_descriptors, uint32_t idx) ++{ ++ idx++; ++ if (unlikely(idx == n_descriptors)) ++ idx = 0; ++ return idx; ++} ++ + void vnic_wq_clean(struct vnic_wq *wq, +- void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf)) ++ void (*buf_clean)(struct vnic_wq_buf *buf)) + { + struct vnic_wq_buf *buf; ++ unsigned int to_clean = wq->tail_idx; + +- buf = wq->to_clean; ++ buf = &wq->bufs[to_clean]; + + while (vnic_wq_desc_used(wq) > 0) { + +- (*buf_clean)(wq, buf); ++ (*buf_clean)(buf); ++ to_clean = buf_idx_incr(wq->ring.desc_count, to_clean); + +- buf = wq->to_clean = buf->next; ++ buf = &wq->bufs[to_clean]; + wq->ring.desc_avail++; + } + +- wq->to_use = wq->to_clean = wq->bufs[0]; ++ wq->head_idx = 0; ++ wq->tail_idx = 0; + + iowrite32(0, &wq->ctrl->fetch_index); + iowrite32(0, &wq->ctrl->posted_index); +diff --git a/drivers/net/enic/base/vnic_wq.h b/drivers/net/enic/base/vnic_wq.h +index c23de62..37c3ff9 100644 +--- a/drivers/net/enic/base/vnic_wq.h ++++ b/drivers/net/enic/base/vnic_wq.h +@@ -64,42 +64,23 @@ struct vnic_wq_ctrl { + u32 pad9; + }; + ++/* 16 bytes */ + struct vnic_wq_buf { +- struct vnic_wq_buf *next; +- dma_addr_t dma_addr; +- void *os_buf; +- unsigned int len; +- unsigned int index; +- int sop; +- void *desc; +- uint64_t wr_id; /* Cookie */ +- uint8_t cq_entry; /* Gets completion event from hw */ +- uint8_t desc_skip_cnt; /* Num descs to occupy */ +- uint8_t compressed_send; /* Both hdr and payload in one desc */ ++ struct rte_mempool *pool; ++ void *mb; + }; + +-/* Break the vnic_wq_buf allocations into blocks of 32/64 entries */ +-#define VNIC_WQ_BUF_MIN_BLK_ENTRIES 32 +-#define VNIC_WQ_BUF_DFLT_BLK_ENTRIES 64 +-#define VNIC_WQ_BUF_BLK_ENTRIES(entries) \ +- ((unsigned int)((entries < VNIC_WQ_BUF_DFLT_BLK_ENTRIES) ? \ +- VNIC_WQ_BUF_MIN_BLK_ENTRIES : VNIC_WQ_BUF_DFLT_BLK_ENTRIES)) +-#define VNIC_WQ_BUF_BLK_SZ(entries) \ +- (VNIC_WQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_wq_buf)) +-#define VNIC_WQ_BUF_BLKS_NEEDED(entries) \ +- DIV_ROUND_UP(entries, VNIC_WQ_BUF_BLK_ENTRIES(entries)) +-#define VNIC_WQ_BUF_BLKS_MAX VNIC_WQ_BUF_BLKS_NEEDED(4096) +- + struct vnic_wq { + unsigned int index; + struct vnic_dev *vdev; + struct vnic_wq_ctrl __iomem *ctrl; /* memory-mapped */ + struct vnic_dev_ring ring; +- struct vnic_wq_buf *bufs[VNIC_WQ_BUF_BLKS_MAX]; +- struct vnic_wq_buf *to_use; +- struct vnic_wq_buf *to_clean; +- unsigned int pkts_outstanding; ++ struct vnic_wq_buf *bufs; ++ unsigned int head_idx; ++ unsigned int tail_idx; + unsigned int socket_id; ++ const struct rte_memzone *cqmsg_rz; ++ uint16_t last_completed_index; + }; + + static inline unsigned int vnic_wq_desc_avail(struct vnic_wq *wq) +@@ -114,11 +95,6 @@ static inline unsigned int vnic_wq_desc_used(struct vnic_wq *wq) + return wq->ring.desc_count - wq->ring.desc_avail - 1; + } + +-static inline void *vnic_wq_next_desc(struct vnic_wq *wq) +-{ +- return wq->to_use->desc; +-} +- + #define PI_LOG2_CACHE_LINE_SIZE 5 + #define PI_INDEX_BITS 12 + #define PI_INDEX_MASK ((1U << PI_INDEX_BITS) - 1) +@@ -191,75 +167,6 @@ static inline u64 vnic_cached_posted_index(dma_addr_t addr, unsigned int len, + PI_PREFETCH_ADDR_MASK) << PI_PREFETCH_ADDR_OFF); + } + +-static inline void vnic_wq_post(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, +- unsigned int len, int sop, int eop, +- uint8_t desc_skip_cnt, uint8_t cq_entry, +- uint8_t compressed_send, uint64_t wrid) +-{ +- struct vnic_wq_buf *buf = wq->to_use; +- +- buf->sop = sop; +- buf->cq_entry = cq_entry; +- buf->compressed_send = compressed_send; +- buf->desc_skip_cnt = desc_skip_cnt; +- buf->os_buf = os_buf; +- buf->dma_addr = dma_addr; +- buf->len = len; +- buf->wr_id = wrid; +- +- buf = buf->next; +- if (eop) { +-#ifdef DO_PREFETCH +- uint64_t wr = vnic_cached_posted_index(dma_addr, len, +- buf->index); +-#endif +- /* Adding write memory barrier prevents compiler and/or CPU +- * reordering, thus avoiding descriptor posting before +- * descriptor is initialized. Otherwise, hardware can read +- * stale descriptor fields. +- */ +- wmb(); +-#ifdef DO_PREFETCH +- /* Intel chipsets seem to limit the rate of PIOs that we can +- * push on the bus. Thus, it is very important to do a single +- * 64 bit write here. With two 32-bit writes, my maximum +- * pkt/sec rate was cut almost in half. -AJF +- */ +- iowrite64((uint64_t)wr, &wq->ctrl->posted_index); +-#else +- iowrite32(buf->index, &wq->ctrl->posted_index); +-#endif +- } +- wq->to_use = buf; +- +- wq->ring.desc_avail -= desc_skip_cnt; +-} +- +-static inline void vnic_wq_service(struct vnic_wq *wq, +- struct cq_desc *cq_desc, u16 completed_index, +- void (*buf_service)(struct vnic_wq *wq, +- struct cq_desc *cq_desc, struct vnic_wq_buf *buf, void *opaque), +- void *opaque) +-{ +- struct vnic_wq_buf *buf; +- +- buf = wq->to_clean; +- while (1) { +- +- (*buf_service)(wq, cq_desc, buf, opaque); +- +- wq->ring.desc_avail++; +- +- wq->to_clean = buf->next; +- +- if (buf->index == completed_index) +- break; +- +- buf = wq->to_clean; +- } +-} +- + void vnic_wq_free(struct vnic_wq *wq); + int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, + unsigned int desc_count, unsigned int desc_size); +@@ -275,8 +182,6 @@ unsigned int vnic_wq_error_status(struct vnic_wq *wq); + void vnic_wq_enable(struct vnic_wq *wq); + int vnic_wq_disable(struct vnic_wq *wq); + void vnic_wq_clean(struct vnic_wq *wq, +- void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf)); +-int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count, +- unsigned int desc_size); ++ void (*buf_clean)(struct vnic_wq_buf *buf)); + + #endif /* _VNIC_WQ_H_ */ +diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h +index 8c914f5..43b82a6 100644 +--- a/drivers/net/enic/enic.h ++++ b/drivers/net/enic/enic.h +@@ -155,6 +155,30 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) + return (struct enic *)eth_dev->data->dev_private; + } + ++static inline uint32_t ++enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) ++{ ++ uint32_t d = i0 + i1; ++ d -= (d >= n_descriptors) ? n_descriptors : 0; ++ return d; ++} ++ ++static inline uint32_t ++enic_ring_sub(uint32_t n_descriptors, uint32_t i0, uint32_t i1) ++{ ++ int32_t d = i1 - i0; ++ return (uint32_t)((d < 0) ? ((int32_t)n_descriptors + d) : d); ++} ++ ++static inline uint32_t ++enic_ring_incr(uint32_t n_descriptors, uint32_t idx) ++{ ++ idx++; ++ if (unlikely(idx == n_descriptors)) ++ idx = 0; ++ return idx; ++} ++ + #define RTE_LIBRTE_ENIC_ASSERT_ENABLE + #ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE + #define ASSERT(x) do { \ +@@ -209,5 +233,6 @@ extern int enic_clsf_init(struct enic *enic); + extern void enic_clsf_destroy(struct enic *enic); + uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +- ++uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, ++ uint16_t nb_pkts); + #endif /* _ENIC_H_ */ +diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c +index 6bea940..697ff82 100644 +--- a/drivers/net/enic/enic_ethdev.c ++++ b/drivers/net/enic/enic_ethdev.c +@@ -519,71 +519,6 @@ static void enicpmd_remove_mac_addr(struct rte_eth_dev *eth_dev, __rte_unused ui + enic_del_mac_address(enic); + } + +- +-static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, +- uint16_t nb_pkts) +-{ +- uint16_t index; +- unsigned int frags; +- unsigned int pkt_len; +- unsigned int seg_len; +- unsigned int inc_len; +- unsigned int nb_segs; +- struct rte_mbuf *tx_pkt, *next_tx_pkt; +- struct vnic_wq *wq = (struct vnic_wq *)tx_queue; +- struct enic *enic = vnic_dev_priv(wq->vdev); +- unsigned short vlan_id; +- unsigned short ol_flags; +- uint8_t last_seg, eop; +- unsigned int host_tx_descs = 0; +- +- for (index = 0; index < nb_pkts; index++) { +- tx_pkt = *tx_pkts++; +- inc_len = 0; +- nb_segs = tx_pkt->nb_segs; +- if (nb_segs > vnic_wq_desc_avail(wq)) { +- if (index > 0) +- enic_post_wq_index(wq); +- +- /* wq cleanup and try again */ +- if (!enic_cleanup_wq(enic, wq) || +- (nb_segs > vnic_wq_desc_avail(wq))) { +- return index; +- } +- } +- +- pkt_len = tx_pkt->pkt_len; +- vlan_id = tx_pkt->vlan_tci; +- ol_flags = tx_pkt->ol_flags; +- for (frags = 0; inc_len < pkt_len; frags++) { +- if (!tx_pkt) +- break; +- next_tx_pkt = tx_pkt->next; +- seg_len = tx_pkt->data_len; +- inc_len += seg_len; +- +- host_tx_descs++; +- last_seg = 0; +- eop = 0; +- if ((pkt_len == inc_len) || !next_tx_pkt) { +- eop = 1; +- /* post if last packet in batch or > thresh */ +- if ((index == (nb_pkts - 1)) || +- (host_tx_descs > ENIC_TX_POST_THRESH)) { +- last_seg = 1; +- host_tx_descs = 0; +- } +- } +- enic_send_pkt(enic, wq, tx_pkt, (unsigned short)seg_len, +- !frags, eop, last_seg, ol_flags, vlan_id); +- tx_pkt = next_tx_pkt; +- } +- } +- +- enic_cleanup_wq(enic, wq); +- return index; +-} +- + static const struct eth_dev_ops enicpmd_eth_dev_ops = { + .dev_configure = enicpmd_dev_configure, + .dev_start = enicpmd_dev_start, +@@ -642,7 +577,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) + enic->rte_dev = eth_dev; + eth_dev->dev_ops = &enicpmd_eth_dev_ops; + eth_dev->rx_pkt_burst = &enic_recv_pkts; +- eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts; ++ eth_dev->tx_pkt_burst = &enic_xmit_pkts; + + pdev = eth_dev->pci_dev; + rte_eth_copy_pci_info(eth_dev, pdev); +diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c +index 646d87f..ba73604 100644 +--- a/drivers/net/enic/enic_main.c ++++ b/drivers/net/enic/enic_main.c +@@ -40,11 +40,11 @@ + #include <libgen.h> + + #include <rte_pci.h> +-#include <rte_memzone.h> + #include <rte_malloc.h> + #include <rte_mbuf.h> + #include <rte_string_fns.h> + #include <rte_ethdev.h> ++#include <rte_memzone.h> + + #include "enic_compat.h" + #include "enic.h" +@@ -58,7 +58,6 @@ + #include "vnic_cq.h" + #include "vnic_intr.h" + #include "vnic_nic.h" +-#include "enic_vnic_wq.h" + + static inline struct rte_mbuf * + rte_rxmbuf_alloc(struct rte_mempool *mp) +@@ -109,38 +108,17 @@ enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) + } + } + +- + void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) + { + vnic_set_hdr_split_size(enic->vdev, split_hdr_size); + } + +-static void enic_free_wq_buf(__rte_unused struct vnic_wq *wq, struct vnic_wq_buf *buf) ++static void enic_free_wq_buf(struct vnic_wq_buf *buf) + { +- struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->os_buf; ++ struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb; + + rte_mempool_put(mbuf->pool, mbuf); +- buf->os_buf = NULL; +-} +- +-static void enic_wq_free_buf(struct vnic_wq *wq, +- __rte_unused struct cq_desc *cq_desc, +- struct vnic_wq_buf *buf, +- __rte_unused void *opaque) +-{ +- enic_free_wq_buf(wq, buf); +-} +- +-static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, +- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque) +-{ +- struct enic *enic = vnic_dev_priv(vdev); +- +- vnic_wq_service(&enic->wq[q_number], cq_desc, +- completed_index, enic_wq_free_buf, +- opaque); +- +- return 0; ++ buf->mb = NULL; + } + + static void enic_log_q_error(struct enic *enic) +@@ -163,64 +141,6 @@ static void enic_log_q_error(struct enic *enic) + } + } + +-unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq) +-{ +- unsigned int cq = enic_cq_wq(enic, wq->index); +- +- /* Return the work done */ +- return vnic_cq_service(&enic->cq[cq], +- -1 /*wq_work_to_do*/, enic_wq_service, NULL); +-} +- +-void enic_post_wq_index(struct vnic_wq *wq) +-{ +- enic_vnic_post_wq_index(wq); +-} +- +-void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, +- struct rte_mbuf *tx_pkt, unsigned short len, +- uint8_t sop, uint8_t eop, uint8_t cq_entry, +- uint16_t ol_flags, uint16_t vlan_tag) +-{ +- struct wq_enet_desc *desc = vnic_wq_next_desc(wq); +- uint16_t mss = 0; +- uint8_t vlan_tag_insert = 0; +- uint64_t bus_addr = (dma_addr_t) +- (tx_pkt->buf_physaddr + tx_pkt->data_off); +- +- if (sop) { +- if (ol_flags & PKT_TX_VLAN_PKT) +- vlan_tag_insert = 1; +- +- if (enic->hw_ip_checksum) { +- if (ol_flags & PKT_TX_IP_CKSUM) +- mss |= ENIC_CALC_IP_CKSUM; +- +- if (ol_flags & PKT_TX_TCP_UDP_CKSUM) +- mss |= ENIC_CALC_TCP_UDP_CKSUM; +- } +- } +- +- wq_enet_desc_enc(desc, +- bus_addr, +- len, +- mss, +- 0 /* header_length */, +- 0 /* offload_mode WQ_ENET_OFFLOAD_MODE_CSUM */, +- eop, +- cq_entry, +- 0 /* fcoe_encap */, +- vlan_tag_insert, +- vlan_tag, +- 0 /* loopback */); +- +- enic_vnic_post_wq(wq, (void *)tx_pkt, bus_addr, len, +- sop, +- 1 /*desc_skip_cnt*/, +- cq_entry, +- 0 /*compressed send*/, +- 0 /*wrid*/); +-} + + void enic_dev_stats_clear(struct enic *enic) + { +@@ -297,12 +217,28 @@ void enic_init_vnic_resources(struct enic *enic) + unsigned int error_interrupt_enable = 1; + unsigned int error_interrupt_offset = 0; + unsigned int index = 0; ++ unsigned int cq_idx; ++ ++ vnic_dev_stats_clear(enic->vdev); + + for (index = 0; index < enic->rq_count; index++) { + vnic_rq_init(&enic->rq[index], + enic_cq_rq(enic, index), + error_interrupt_enable, + error_interrupt_offset); ++ ++ cq_idx = enic_cq_rq(enic, index); ++ vnic_cq_init(&enic->cq[cq_idx], ++ 0 /* flow_control_enable */, ++ 1 /* color_enable */, ++ 0 /* cq_head */, ++ 0 /* cq_tail */, ++ 1 /* cq_tail_color */, ++ 0 /* interrupt_enable */, ++ 1 /* cq_entry_enable */, ++ 0 /* cq_message_enable */, ++ 0 /* interrupt offset */, ++ 0 /* cq_message_addr */); + } + + for (index = 0; index < enic->wq_count; index++) { +@@ -310,22 +246,19 @@ void enic_init_vnic_resources(struct enic *enic) + enic_cq_wq(enic, index), + error_interrupt_enable, + error_interrupt_offset); +- } +- +- vnic_dev_stats_clear(enic->vdev); + +- for (index = 0; index < enic->cq_count; index++) { +- vnic_cq_init(&enic->cq[index], ++ cq_idx = enic_cq_wq(enic, index); ++ vnic_cq_init(&enic->cq[cq_idx], + 0 /* flow_control_enable */, + 1 /* color_enable */, + 0 /* cq_head */, + 0 /* cq_tail */, + 1 /* cq_tail_color */, + 0 /* interrupt_enable */, +- 1 /* cq_entry_enable */, +- 0 /* cq_message_enable */, ++ 0 /* cq_entry_enable */, ++ 1 /* cq_message_enable */, + 0 /* interrupt offset */, +- 0 /* cq_message_addr */); ++ (u64)enic->wq[index].cqmsg_rz->phys_addr); + } + + vnic_intr_init(&enic->intr, +@@ -569,6 +502,7 @@ void enic_free_wq(void *txq) + struct vnic_wq *wq = (struct vnic_wq *)txq; + struct enic *enic = vnic_dev_priv(wq->vdev); + ++ rte_memzone_free(wq->cqmsg_rz); + vnic_wq_free(wq); + vnic_cq_free(&enic->cq[enic->rq_count + wq->index]); + } +@@ -579,6 +513,8 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, + int err; + struct vnic_wq *wq = &enic->wq[queue_idx]; + unsigned int cq_index = enic_cq_wq(enic, queue_idx); ++ char name[NAME_MAX]; ++ static int instance; + + wq->socket_id = socket_id; + if (nb_desc) { +@@ -614,6 +550,18 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, + dev_err(enic, "error in allocation of cq for wq\n"); + } + ++ /* setup up CQ message */ ++ snprintf((char *)name, sizeof(name), ++ "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx, ++ instance++); ++ ++ wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name, ++ sizeof(uint32_t), ++ SOCKET_ID_ANY, 0, ++ ENIC_ALIGN); ++ if (!wq->cqmsg_rz) ++ return -ENOMEM; ++ + return err; + } + +diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h +index 00fa71d..3e1bdf5 100644 +--- a/drivers/net/enic/enic_res.h ++++ b/drivers/net/enic/enic_res.h +@@ -53,89 +53,10 @@ + + #define ENIC_NON_TSO_MAX_DESC 16 + #define ENIC_DEFAULT_RX_FREE_THRESH 32 +-#define ENIC_TX_POST_THRESH (ENIC_MIN_WQ_DESCS / 2) ++#define ENIC_TX_XMIT_MAX 64 + + #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) + +-static inline void enic_queue_wq_desc_ex(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, unsigned int len, +- unsigned int mss_or_csum_offset, unsigned int hdr_len, +- int vlan_tag_insert, unsigned int vlan_tag, +- int offload_mode, int cq_entry, int sop, int eop, int loopback) +-{ +- struct wq_enet_desc *desc = vnic_wq_next_desc(wq); +- u8 desc_skip_cnt = 1; +- u8 compressed_send = 0; +- u64 wrid = 0; +- +- wq_enet_desc_enc(desc, +- (u64)dma_addr | VNIC_PADDR_TARGET, +- (u16)len, +- (u16)mss_or_csum_offset, +- (u16)hdr_len, (u8)offload_mode, +- (u8)eop, (u8)cq_entry, +- 0, /* fcoe_encap */ +- (u8)vlan_tag_insert, +- (u16)vlan_tag, +- (u8)loopback); +- +- vnic_wq_post(wq, os_buf, dma_addr, len, sop, eop, desc_skip_cnt, +- (u8)cq_entry, compressed_send, wrid); +-} +- +-static inline void enic_queue_wq_desc_cont(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, unsigned int len, +- int eop, int loopback) +-{ +- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, +- 0, 0, 0, 0, 0, +- eop, 0 /* !SOP */, eop, loopback); +-} +- +-static inline void enic_queue_wq_desc(struct vnic_wq *wq, void *os_buf, +- dma_addr_t dma_addr, unsigned int len, int vlan_tag_insert, +- unsigned int vlan_tag, int eop, int loopback) +-{ +- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, +- 0, 0, vlan_tag_insert, vlan_tag, +- WQ_ENET_OFFLOAD_MODE_CSUM, +- eop, 1 /* SOP */, eop, loopback); +-} +- +-static inline void enic_queue_wq_desc_csum(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, unsigned int len, +- int ip_csum, int tcpudp_csum, int vlan_tag_insert, +- unsigned int vlan_tag, int eop, int loopback) +-{ +- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, +- (ip_csum ? 1 : 0) + (tcpudp_csum ? 2 : 0), +- 0, vlan_tag_insert, vlan_tag, +- WQ_ENET_OFFLOAD_MODE_CSUM, +- eop, 1 /* SOP */, eop, loopback); +-} +- +-static inline void enic_queue_wq_desc_csum_l4(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, unsigned int len, +- unsigned int csum_offset, unsigned int hdr_len, +- int vlan_tag_insert, unsigned int vlan_tag, int eop, int loopback) +-{ +- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, +- csum_offset, hdr_len, vlan_tag_insert, vlan_tag, +- WQ_ENET_OFFLOAD_MODE_CSUM_L4, +- eop, 1 /* SOP */, eop, loopback); +-} +- +-static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq, +- void *os_buf, dma_addr_t dma_addr, unsigned int len, +- unsigned int mss, unsigned int hdr_len, int vlan_tag_insert, +- unsigned int vlan_tag, int eop, int loopback) +-{ +- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, +- mss, hdr_len, vlan_tag_insert, vlan_tag, +- WQ_ENET_OFFLOAD_MODE_TSO, +- eop, 1 /* SOP */, eop, loopback); +-} +- + struct enic; + + int enic_get_vnic_config(struct enic *); +diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c +deleted file mode 100644 +index 39bb55c..0000000 +--- a/drivers/net/enic/enic_rx.c ++++ /dev/null +@@ -1,361 +0,0 @@ +-/* +- * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved. +- * Copyright 2007 Nuova Systems, Inc. All rights reserved. +- * +- * Copyright (c) 2014, Cisco Systems, Inc. +- * All rights reserved. +- * +- * Redistribution and use in source and binary forms, with or without +- * modification, are permitted provided that the following conditions +- * are met: +- * +- * 1. Redistributions of source code must retain the above copyright +- * notice, this list of conditions and the following disclaimer. +- * +- * 2. Redistributions in binary form must reproduce the above copyright +- * notice, this list of conditions and the following disclaimer in +- * the documentation and/or other materials provided with the +- * distribution. +- * +- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +- * POSSIBILITY OF SUCH DAMAGE. +- * +- */ +- +-#include <rte_mbuf.h> +-#include <rte_ethdev.h> +-#include <rte_prefetch.h> +- +-#include "enic_compat.h" +-#include "rq_enet_desc.h" +-#include "enic.h" +- +-#define RTE_PMD_USE_PREFETCH +- +-#ifdef RTE_PMD_USE_PREFETCH +-/* +- * Prefetch a cache line into all cache levels. +- */ +-#define rte_enic_prefetch(p) rte_prefetch0(p) +-#else +-#define rte_enic_prefetch(p) do {} while (0) +-#endif +- +-#ifdef RTE_PMD_PACKET_PREFETCH +-#define rte_packet_prefetch(p) rte_prefetch1(p) +-#else +-#define rte_packet_prefetch(p) do {} while (0) +-#endif +- +-static inline struct rte_mbuf * +-rte_rxmbuf_alloc(struct rte_mempool *mp) +-{ +- struct rte_mbuf *m; +- +- m = __rte_mbuf_raw_alloc(mp); +- __rte_mbuf_sanity_check_raw(m, 0); +- return m; +-} +- +-static inline uint16_t +-enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) +-{ +- return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; +-} +- +-static inline uint16_t +-enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) +-{ +- return(le16_to_cpu(crd->bytes_written_flags) & +- ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); +-} +- +-static inline uint8_t +-enic_cq_rx_desc_packet_error(uint16_t bwflags) +-{ +- return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == +- CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); +-} +- +-static inline uint8_t +-enic_cq_rx_desc_eop(uint16_t ciflags) +-{ +- return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) +- == CQ_ENET_RQ_DESC_FLAGS_EOP; +-} +- +-static inline uint8_t +-enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) +-{ +- return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & +- CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == +- CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); +-} +- +-static inline uint8_t +-enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) +-{ +- return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == +- CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); +-} +- +-static inline uint8_t +-enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) +-{ +- return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == +- CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); +-} +- +-static inline uint8_t +-enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) +-{ +- return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> +- CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); +-} +- +-static inline uint32_t +-enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) +-{ +- return le32_to_cpu(cqrd->rss_hash); +-} +- +-static inline uint16_t +-enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) +-{ +- return le16_to_cpu(cqrd->vlan); +-} +- +-static inline uint16_t +-enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) +-{ +- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; +- return le16_to_cpu(cqrd->bytes_written_flags) & +- CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; +-} +- +-static inline uint8_t +-enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out) +-{ +- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; +- uint16_t bwflags; +- int ret = 0; +- uint64_t pkt_err_flags = 0; +- +- bwflags = enic_cq_rx_desc_bwflags(cqrd); +- if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) { +- pkt_err_flags = PKT_RX_MAC_ERR; +- ret = 1; +- } +- *pkt_err_flags_out = pkt_err_flags; +- return ret; +-} +- +-/* +- * Lookup table to translate RX CQ flags to mbuf flags. +- */ +-static inline uint32_t +-enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) +-{ +- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; +- uint8_t cqrd_flags = cqrd->flags; +- static const uint32_t cq_type_table[128] __rte_cache_aligned = { +- [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, +- [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 +- | RTE_PTYPE_L4_UDP, +- [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 +- | RTE_PTYPE_L4_TCP, +- [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 +- | RTE_PTYPE_L4_FRAG, +- [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, +- [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 +- | RTE_PTYPE_L4_UDP, +- [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 +- | RTE_PTYPE_L4_TCP, +- [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 +- | RTE_PTYPE_L4_FRAG, +- /* All others reserved */ +- }; +- cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT +- | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 +- | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; +- return cq_type_table[cqrd_flags]; +-} +- +-static inline void +-enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) +-{ +- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; +- uint16_t ciflags, bwflags, pkt_flags = 0; +- ciflags = enic_cq_rx_desc_ciflags(cqrd); +- bwflags = enic_cq_rx_desc_bwflags(cqrd); +- +- mbuf->ol_flags = 0; +- +- /* flags are meaningless if !EOP */ +- if (unlikely(!enic_cq_rx_desc_eop(ciflags))) +- goto mbuf_flags_done; +- +- /* VLAN stripping */ +- if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { +- pkt_flags |= PKT_RX_VLAN_PKT; +- mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); +- } else { +- mbuf->vlan_tci = 0; +- } +- +- /* RSS flag */ +- if (enic_cq_rx_desc_rss_type(cqrd)) { +- pkt_flags |= PKT_RX_RSS_HASH; +- mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); +- } +- +- /* checksum flags */ +- if (!enic_cq_rx_desc_csum_not_calc(cqrd) && +- (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { +- if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) +- pkt_flags |= PKT_RX_IP_CKSUM_BAD; +- if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { +- if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) +- pkt_flags |= PKT_RX_L4_CKSUM_BAD; +- } +- } +- +- mbuf_flags_done: +- mbuf->ol_flags = pkt_flags; +-} +- +-static inline uint32_t +-enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) +-{ +- uint32_t d = i0 + i1; +- ASSERT(i0 < n_descriptors); +- ASSERT(i1 < n_descriptors); +- d -= (d >= n_descriptors) ? n_descriptors : 0; +- return d; +-} +- +- +-uint16_t +-enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, +- uint16_t nb_pkts) +-{ +- struct vnic_rq *rq = rx_queue; +- struct enic *enic = vnic_dev_priv(rq->vdev); +- unsigned int rx_id; +- struct rte_mbuf *nmb, *rxmb; +- uint16_t nb_rx = 0; +- uint16_t nb_hold; +- struct vnic_cq *cq; +- volatile struct cq_desc *cqd_ptr; +- uint8_t color; +- +- cq = &enic->cq[enic_cq_rq(enic, rq->index)]; +- rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ +- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; +- +- nb_hold = rq->rx_nb_hold; /* mbufs held by software */ +- +- while (nb_rx < nb_pkts) { +- volatile struct rq_enet_desc *rqd_ptr; +- dma_addr_t dma_addr; +- struct cq_desc cqd; +- uint64_t ol_err_flags; +- uint8_t packet_error; +- +- /* Check for pkts available */ +- color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) +- & CQ_DESC_COLOR_MASK; +- if (color == cq->last_color) +- break; +- +- /* Get the cq descriptor and rq pointer */ +- cqd = *cqd_ptr; +- rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; +- +- /* allocate a new mbuf */ +- nmb = rte_rxmbuf_alloc(rq->mp); +- if (nmb == NULL) { +- dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", +- enic->port_id, (unsigned)rq->index); +- rte_eth_devices[enic->port_id]. +- data->rx_mbuf_alloc_failed++; +- break; +- } +- +- /* A packet error means descriptor and data are untrusted */ +- packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); +- +- /* Get the mbuf to return and replace with one just allocated */ +- rxmb = rq->mbuf_ring[rx_id]; +- rq->mbuf_ring[rx_id] = nmb; +- +- /* Increment cqd, rqd, mbuf_table index */ +- rx_id++; +- if (unlikely(rx_id == rq->ring.desc_count)) { +- rx_id = 0; +- cq->last_color = cq->last_color ? 0 : 1; +- } +- +- /* Prefetch next mbuf & desc while processing current one */ +- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; +- rte_enic_prefetch(cqd_ptr); +- rte_enic_prefetch(rq->mbuf_ring[rx_id]); +- rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) +- + rx_id); +- +- /* Push descriptor for newly allocated mbuf */ +- dma_addr = (dma_addr_t)(nmb->buf_physaddr +- + RTE_PKTMBUF_HEADROOM); +- rqd_ptr->address = rte_cpu_to_le_64(dma_addr); +- rqd_ptr->length_type = cpu_to_le16(nmb->buf_len +- - RTE_PKTMBUF_HEADROOM); +- +- /* Fill in the rest of the mbuf */ +- rxmb->data_off = RTE_PKTMBUF_HEADROOM; +- rxmb->nb_segs = 1; +- rxmb->next = NULL; +- rxmb->port = enic->port_id; +- if (!packet_error) { +- rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); +- rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); +- enic_cq_rx_to_pkt_flags(&cqd, rxmb); +- } else { +- rxmb->pkt_len = 0; +- rxmb->packet_type = 0; +- rxmb->ol_flags = 0; +- } +- rxmb->data_len = rxmb->pkt_len; +- +- /* prefetch mbuf data for caller */ +- rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, +- RTE_PKTMBUF_HEADROOM)); +- +- /* store the mbuf address into the next entry of the array */ +- rx_pkts[nb_rx++] = rxmb; +- } +- +- nb_hold += nb_rx; +- cq->to_clean = rx_id; +- +- if (nb_hold > rq->rx_free_thresh) { +- rq->posted_index = enic_ring_add(rq->ring.desc_count, +- rq->posted_index, nb_hold); +- nb_hold = 0; +- rte_mb(); +- iowrite32(rq->posted_index, &rq->ctrl->posted_index); +- } +- +- rq->rx_nb_hold = nb_hold; +- +- return nb_rx; +-} +diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c +new file mode 100644 +index 0000000..71ca34e +--- /dev/null ++++ b/drivers/net/enic/enic_rxtx.c +@@ -0,0 +1,505 @@ ++/* ++ * Copyright 2008-2016 Cisco Systems, Inc. All rights reserved. ++ * Copyright 2007 Nuova Systems, Inc. All rights reserved. ++ * ++ * Copyright (c) 2016, Cisco Systems, Inc. ++ * All rights reserved. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions ++ * are met: ++ * ++ * 1. Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * ++ * 2. Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in ++ * the documentation and/or other materials provided with the ++ * distribution. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS ++ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ++ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, ++ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, ++ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER ++ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT ++ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ++ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE ++ * POSSIBILITY OF SUCH DAMAGE. ++ * ++ */ ++ ++#include <rte_mbuf.h> ++#include <rte_ethdev.h> ++#include <rte_prefetch.h> ++#include <rte_memzone.h> ++ ++#include "enic_compat.h" ++#include "rq_enet_desc.h" ++#include "enic.h" ++ ++#define RTE_PMD_USE_PREFETCH ++ ++#ifdef RTE_PMD_USE_PREFETCH ++/* ++ * Prefetch a cache line into all cache levels. ++ */ ++#define rte_enic_prefetch(p) rte_prefetch0(p) ++#else ++#define rte_enic_prefetch(p) do {} while (0) ++#endif ++ ++#ifdef RTE_PMD_PACKET_PREFETCH ++#define rte_packet_prefetch(p) rte_prefetch1(p) ++#else ++#define rte_packet_prefetch(p) do {} while (0) ++#endif ++ ++static inline struct rte_mbuf * ++rte_rxmbuf_alloc(struct rte_mempool *mp) ++{ ++ struct rte_mbuf *m; ++ ++ m = __rte_mbuf_raw_alloc(mp); ++ __rte_mbuf_sanity_check_raw(m, 0); ++ return m; ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) ++{ ++ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) ++{ ++ return(le16_to_cpu(crd->bytes_written_flags) & ++ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_packet_error(uint16_t bwflags) ++{ ++ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == ++ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_eop(uint16_t ciflags) ++{ ++ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) ++ == CQ_ENET_RQ_DESC_FLAGS_EOP; ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) ++{ ++ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & ++ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == ++ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) ++{ ++ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == ++ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) ++{ ++ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == ++ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); ++} ++ ++static inline uint8_t ++enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) ++{ ++ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> ++ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); ++} ++ ++static inline uint32_t ++enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) ++{ ++ return le32_to_cpu(cqrd->rss_hash); ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) ++{ ++ return le16_to_cpu(cqrd->vlan); ++} ++ ++static inline uint16_t ++enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ return le16_to_cpu(cqrd->bytes_written_flags) & ++ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; ++} ++ ++static inline uint8_t ++enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ uint16_t bwflags; ++ int ret = 0; ++ uint64_t pkt_err_flags = 0; ++ ++ bwflags = enic_cq_rx_desc_bwflags(cqrd); ++ if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) { ++ pkt_err_flags = PKT_RX_MAC_ERR; ++ ret = 1; ++ } ++ *pkt_err_flags_out = pkt_err_flags; ++ return ret; ++} ++ ++/* ++ * Lookup table to translate RX CQ flags to mbuf flags. ++ */ ++static inline uint32_t ++enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ uint8_t cqrd_flags = cqrd->flags; ++ static const uint32_t cq_type_table[128] __rte_cache_aligned = { ++ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, ++ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 ++ | RTE_PTYPE_L4_UDP, ++ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 ++ | RTE_PTYPE_L4_TCP, ++ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 ++ | RTE_PTYPE_L4_FRAG, ++ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, ++ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 ++ | RTE_PTYPE_L4_UDP, ++ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 ++ | RTE_PTYPE_L4_TCP, ++ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 ++ | RTE_PTYPE_L4_FRAG, ++ /* All others reserved */ ++ }; ++ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT ++ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 ++ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; ++ return cq_type_table[cqrd_flags]; ++} ++ ++static inline void ++enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) ++{ ++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; ++ uint16_t ciflags, bwflags, pkt_flags = 0; ++ ciflags = enic_cq_rx_desc_ciflags(cqrd); ++ bwflags = enic_cq_rx_desc_bwflags(cqrd); ++ ++ mbuf->ol_flags = 0; ++ ++ /* flags are meaningless if !EOP */ ++ if (unlikely(!enic_cq_rx_desc_eop(ciflags))) ++ goto mbuf_flags_done; ++ ++ /* VLAN stripping */ ++ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { ++ pkt_flags |= PKT_RX_VLAN_PKT; ++ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); ++ } else { ++ mbuf->vlan_tci = 0; ++ } ++ ++ /* RSS flag */ ++ if (enic_cq_rx_desc_rss_type(cqrd)) { ++ pkt_flags |= PKT_RX_RSS_HASH; ++ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); ++ } ++ ++ /* checksum flags */ ++ if (!enic_cq_rx_desc_csum_not_calc(cqrd) && ++ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { ++ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) ++ pkt_flags |= PKT_RX_IP_CKSUM_BAD; ++ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { ++ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) ++ pkt_flags |= PKT_RX_L4_CKSUM_BAD; ++ } ++ } ++ ++ mbuf_flags_done: ++ mbuf->ol_flags = pkt_flags; ++} ++ ++uint16_t ++enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, ++ uint16_t nb_pkts) ++{ ++ struct vnic_rq *rq = rx_queue; ++ struct enic *enic = vnic_dev_priv(rq->vdev); ++ unsigned int rx_id; ++ struct rte_mbuf *nmb, *rxmb; ++ uint16_t nb_rx = 0; ++ uint16_t nb_hold; ++ struct vnic_cq *cq; ++ volatile struct cq_desc *cqd_ptr; ++ uint8_t color; ++ ++ cq = &enic->cq[enic_cq_rq(enic, rq->index)]; ++ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ ++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; ++ ++ nb_hold = rq->rx_nb_hold; /* mbufs held by software */ ++ ++ while (nb_rx < nb_pkts) { ++ volatile struct rq_enet_desc *rqd_ptr; ++ dma_addr_t dma_addr; ++ struct cq_desc cqd; ++ uint64_t ol_err_flags; ++ uint8_t packet_error; ++ ++ /* Check for pkts available */ ++ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) ++ & CQ_DESC_COLOR_MASK; ++ if (color == cq->last_color) ++ break; ++ ++ /* Get the cq descriptor and rq pointer */ ++ cqd = *cqd_ptr; ++ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; ++ ++ /* allocate a new mbuf */ ++ nmb = rte_rxmbuf_alloc(rq->mp); ++ if (nmb == NULL) { ++ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", ++ enic->port_id, (unsigned)rq->index); ++ rte_eth_devices[enic->port_id]. ++ data->rx_mbuf_alloc_failed++; ++ break; ++ } ++ ++ /* A packet error means descriptor and data are untrusted */ ++ packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); ++ ++ /* Get the mbuf to return and replace with one just allocated */ ++ rxmb = rq->mbuf_ring[rx_id]; ++ rq->mbuf_ring[rx_id] = nmb; ++ ++ /* Increment cqd, rqd, mbuf_table index */ ++ rx_id++; ++ if (unlikely(rx_id == rq->ring.desc_count)) { ++ rx_id = 0; ++ cq->last_color = cq->last_color ? 0 : 1; ++ } ++ ++ /* Prefetch next mbuf & desc while processing current one */ ++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; ++ rte_enic_prefetch(cqd_ptr); ++ rte_enic_prefetch(rq->mbuf_ring[rx_id]); ++ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) ++ + rx_id); ++ ++ /* Push descriptor for newly allocated mbuf */ ++ dma_addr = (dma_addr_t)(nmb->buf_physaddr ++ + RTE_PKTMBUF_HEADROOM); ++ rqd_ptr->address = rte_cpu_to_le_64(dma_addr); ++ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len ++ - RTE_PKTMBUF_HEADROOM); ++ ++ /* Fill in the rest of the mbuf */ ++ rxmb->data_off = RTE_PKTMBUF_HEADROOM; ++ rxmb->nb_segs = 1; ++ rxmb->next = NULL; ++ rxmb->port = enic->port_id; ++ if (!packet_error) { ++ rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); ++ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); ++ enic_cq_rx_to_pkt_flags(&cqd, rxmb); ++ } else { ++ rxmb->pkt_len = 0; ++ rxmb->packet_type = 0; ++ rxmb->ol_flags = 0; ++ } ++ rxmb->data_len = rxmb->pkt_len; ++ ++ /* prefetch mbuf data for caller */ ++ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, ++ RTE_PKTMBUF_HEADROOM)); ++ ++ /* store the mbuf address into the next entry of the array */ ++ rx_pkts[nb_rx++] = rxmb; ++ } ++ ++ nb_hold += nb_rx; ++ cq->to_clean = rx_id; ++ ++ if (nb_hold > rq->rx_free_thresh) { ++ rq->posted_index = enic_ring_add(rq->ring.desc_count, ++ rq->posted_index, nb_hold); ++ nb_hold = 0; ++ rte_mb(); ++ iowrite32(rq->posted_index, &rq->ctrl->posted_index); ++ } ++ ++ rq->rx_nb_hold = nb_hold; ++ ++ return nb_rx; ++} ++ ++static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index) ++{ ++ struct vnic_wq_buf *buf; ++ struct rte_mbuf *m, *free[ENIC_MAX_WQ_DESCS]; ++ unsigned int nb_to_free, nb_free = 0, i; ++ struct rte_mempool *pool; ++ unsigned int tail_idx; ++ unsigned int desc_count = wq->ring.desc_count; ++ ++ nb_to_free = enic_ring_sub(desc_count, wq->tail_idx, completed_index) ++ + 1; ++ tail_idx = wq->tail_idx; ++ buf = &wq->bufs[tail_idx]; ++ pool = ((struct rte_mbuf *)buf->mb)->pool; ++ for (i = 0; i < nb_to_free; i++) { ++ buf = &wq->bufs[tail_idx]; ++ m = (struct rte_mbuf *)(buf->mb); ++ if (likely(m->pool == pool)) { ++ ASSERT(nb_free < ENIC_MAX_WQ_DESCS); ++ free[nb_free++] = m; ++ } else { ++ rte_mempool_put_bulk(pool, (void *)free, nb_free); ++ free[0] = m; ++ nb_free = 1; ++ pool = m->pool; ++ } ++ tail_idx = enic_ring_incr(desc_count, tail_idx); ++ buf->mb = NULL; ++ } ++ ++ rte_mempool_put_bulk(pool, (void **)free, nb_free); ++ ++ wq->tail_idx = tail_idx; ++ wq->ring.desc_avail += nb_to_free; ++} ++ ++unsigned int enic_cleanup_wq(__rte_unused struct enic *enic, struct vnic_wq *wq) ++{ ++ u16 completed_index; ++ ++ completed_index = *((uint32_t *)wq->cqmsg_rz->addr) & 0xffff; ++ ++ if (wq->last_completed_index != completed_index) { ++ enic_free_wq_bufs(wq, completed_index); ++ wq->last_completed_index = completed_index; ++ } ++ return 0; ++} ++ ++uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, ++ uint16_t nb_pkts) ++{ ++ uint16_t index; ++ unsigned int pkt_len, data_len; ++ unsigned int nb_segs; ++ struct rte_mbuf *tx_pkt; ++ struct vnic_wq *wq = (struct vnic_wq *)tx_queue; ++ struct enic *enic = vnic_dev_priv(wq->vdev); ++ unsigned short vlan_id; ++ unsigned short ol_flags; ++ unsigned int wq_desc_avail; ++ int head_idx; ++ struct vnic_wq_buf *buf; ++ unsigned int hw_ip_cksum_enabled; ++ unsigned int desc_count; ++ struct wq_enet_desc *descs, *desc_p, desc_tmp; ++ uint16_t mss; ++ uint8_t vlan_tag_insert; ++ uint8_t eop; ++ uint64_t bus_addr; ++ ++ enic_cleanup_wq(enic, wq); ++ wq_desc_avail = vnic_wq_desc_avail(wq); ++ head_idx = wq->head_idx; ++ desc_count = wq->ring.desc_count; ++ ++ nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX); ++ ++ hw_ip_cksum_enabled = enic->hw_ip_checksum; ++ for (index = 0; index < nb_pkts; index++) { ++ tx_pkt = *tx_pkts++; ++ nb_segs = tx_pkt->nb_segs; ++ if (nb_segs > wq_desc_avail) { ++ if (index > 0) ++ goto post; ++ goto done; ++ } ++ ++ pkt_len = tx_pkt->pkt_len; ++ data_len = tx_pkt->data_len; ++ vlan_id = tx_pkt->vlan_tci; ++ ol_flags = tx_pkt->ol_flags; ++ ++ mss = 0; ++ vlan_tag_insert = 0; ++ bus_addr = (dma_addr_t) ++ (tx_pkt->buf_physaddr + tx_pkt->data_off); ++ ++ descs = (struct wq_enet_desc *)wq->ring.descs; ++ desc_p = descs + head_idx; ++ ++ eop = (data_len == pkt_len); ++ ++ if (ol_flags & PKT_TX_VLAN_PKT) ++ vlan_tag_insert = 1; ++ ++ if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_IP_CKSUM)) ++ mss |= ENIC_CALC_IP_CKSUM; ++ ++ if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_TCP_UDP_CKSUM)) ++ mss |= ENIC_CALC_TCP_UDP_CKSUM; ++ ++ wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, 0, 0, eop, ++ eop, 0, vlan_tag_insert, vlan_id, 0); ++ ++ *desc_p = desc_tmp; ++ buf = &wq->bufs[head_idx]; ++ buf->mb = (void *)tx_pkt; ++ head_idx = enic_ring_incr(desc_count, head_idx); ++ wq_desc_avail--; ++ ++ if (!eop) { ++ for (tx_pkt = tx_pkt->next; tx_pkt; tx_pkt = ++ tx_pkt->next) { ++ data_len = tx_pkt->data_len; ++ ++ if (tx_pkt->next == NULL) ++ eop = 1; ++ desc_p = descs + head_idx; ++ bus_addr = (dma_addr_t)(tx_pkt->buf_physaddr ++ + tx_pkt->data_off); ++ wq_enet_desc_enc((struct wq_enet_desc *) ++ &desc_tmp, bus_addr, data_len, ++ mss, 0, 0, eop, eop, 0, ++ vlan_tag_insert, vlan_id, 0); ++ ++ *desc_p = desc_tmp; ++ buf = &wq->bufs[head_idx]; ++ buf->mb = (void *)tx_pkt; ++ head_idx = enic_ring_incr(desc_count, head_idx); ++ wq_desc_avail--; ++ } ++ } ++ } ++ post: ++ rte_wmb(); ++ iowrite32(head_idx, &wq->ctrl->posted_index); ++ done: ++ wq->ring.desc_avail = wq_desc_avail; ++ wq->head_idx = head_idx; ++ ++ return index; ++} +-- +2.7.0 + |