diff options
Diffstat (limited to 'dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch')
| -rw-r--r-- | dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch | 1844 |
1 files changed, 0 insertions, 1844 deletions
diff --git a/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch b/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch deleted file mode 100644 index 4858b8f5d39..00000000000 --- a/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch +++ /dev/null @@ -1,1844 +0,0 @@ -From ca6bbb723880e91d006de6cc485259da988859aa Mon Sep 17 00:00:00 2001 -From: John Daley <johndale@cisco.com> -Date: Tue, 5 Apr 2016 15:19:06 -0700 -Subject: [PATCH 3/3] enic: Optimization of Tx path to reduce Host CPU - overhead, cleanup - -Optimizations and cleanup: -- flatten packet send path -- flatten mbuf free path -- disable CQ entry writing and use CQ messages instead -- use rte_mempool_put_bulk() to bulk return freed mbufs -- remove unnecessary fields vnic_bufs struct, use contiguous array of cache - aligned divisible elements. No next pointers. -- use local variables inside per packet loop instead of fields in structs. -- factor book keeping out of the per packet tx loop where possible - (removed several conditionals) -- put Tx and Rx code in 1 file (enic_rxtx.c) - -Reviewed-by: Nelson Escobar <neescoba@cisco.com> -Signed-off-by: John Daley <johndale@cisco.com> ---- - drivers/net/enic/Makefile | 2 +- - drivers/net/enic/base/enic_vnic_wq.h | 79 ------ - drivers/net/enic/base/vnic_cq.h | 37 +-- - drivers/net/enic/base/vnic_rq.h | 2 +- - drivers/net/enic/base/vnic_wq.c | 89 +++--- - drivers/net/enic/base/vnic_wq.h | 113 +------- - drivers/net/enic/enic.h | 27 +- - drivers/net/enic/enic_ethdev.c | 67 +---- - drivers/net/enic/enic_main.c | 132 +++------ - drivers/net/enic/enic_res.h | 81 +----- - drivers/net/enic/enic_rx.c | 361 ------------------------- - drivers/net/enic/enic_rxtx.c | 505 +++++++++++++++++++++++++++++++++++ - 12 files changed, 635 insertions(+), 860 deletions(-) - delete mode 100644 drivers/net/enic/base/enic_vnic_wq.h - delete mode 100644 drivers/net/enic/enic_rx.c - create mode 100644 drivers/net/enic/enic_rxtx.c - -diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile -index f316274..3926b79 100644 ---- a/drivers/net/enic/Makefile -+++ b/drivers/net/enic/Makefile -@@ -53,7 +53,7 @@ VPATH += $(SRCDIR)/src - # - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c --SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c -+SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rxtx.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c - SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c -diff --git a/drivers/net/enic/base/enic_vnic_wq.h b/drivers/net/enic/base/enic_vnic_wq.h -deleted file mode 100644 -index b019109..0000000 ---- a/drivers/net/enic/base/enic_vnic_wq.h -+++ /dev/null -@@ -1,79 +0,0 @@ --/* -- * Copyright 2008-2015 Cisco Systems, Inc. All rights reserved. -- * Copyright 2007 Nuova Systems, Inc. All rights reserved. -- * -- * Copyright (c) 2015, Cisco Systems, Inc. -- * All rights reserved. -- * -- * Redistribution and use in source and binary forms, with or without -- * modification, are permitted provided that the following conditions -- * are met: -- * -- * 1. Redistributions of source code must retain the above copyright -- * notice, this list of conditions and the following disclaimer. -- * -- * 2. Redistributions in binary form must reproduce the above copyright -- * notice, this list of conditions and the following disclaimer in -- * the documentation and/or other materials provided with the -- * distribution. -- * -- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -- * POSSIBILITY OF SUCH DAMAGE. -- * -- */ -- --#ifndef _ENIC_VNIC_WQ_H_ --#define _ENIC_VNIC_WQ_H_ -- --#include "vnic_dev.h" --#include "vnic_cq.h" -- --static inline void enic_vnic_post_wq_index(struct vnic_wq *wq) --{ -- struct vnic_wq_buf *buf = wq->to_use; -- -- /* Adding write memory barrier prevents compiler and/or CPU -- * reordering, thus avoiding descriptor posting before -- * descriptor is initialized. Otherwise, hardware can read -- * stale descriptor fields. -- */ -- wmb(); -- iowrite32(buf->index, &wq->ctrl->posted_index); --} -- --static inline void enic_vnic_post_wq(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, -- unsigned int len, int sop, -- uint8_t desc_skip_cnt, uint8_t cq_entry, -- uint8_t compressed_send, uint64_t wrid) --{ -- struct vnic_wq_buf *buf = wq->to_use; -- -- buf->sop = sop; -- buf->cq_entry = cq_entry; -- buf->compressed_send = compressed_send; -- buf->desc_skip_cnt = desc_skip_cnt; -- buf->os_buf = os_buf; -- buf->dma_addr = dma_addr; -- buf->len = len; -- buf->wr_id = wrid; -- -- buf = buf->next; -- wq->ring.desc_avail -= desc_skip_cnt; -- wq->to_use = buf; -- -- if (cq_entry) -- enic_vnic_post_wq_index(wq); --} -- --#endif /* _ENIC_VNIC_WQ_H_ */ -diff --git a/drivers/net/enic/base/vnic_cq.h b/drivers/net/enic/base/vnic_cq.h -index 922391b..ffc1aaa 100644 ---- a/drivers/net/enic/base/vnic_cq.h -+++ b/drivers/net/enic/base/vnic_cq.h -@@ -96,41 +96,46 @@ static inline unsigned int vnic_cq_service(struct vnic_cq *cq, - u8 type, u16 q_number, u16 completed_index, void *opaque), - void *opaque) - { -- struct cq_desc *cq_desc; -+ struct cq_desc *cq_desc, *cq_desc_last; - unsigned int work_done = 0; - u16 q_number, completed_index; -- u8 type, color; -- struct rte_mbuf **rx_pkts = opaque; -- unsigned int ret; -+ u8 type, color, type_color; - - cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs + - cq->ring.desc_size * cq->to_clean); -- cq_desc_dec(cq_desc, &type, &color, -- &q_number, &completed_index); -+ -+ type_color = cq_desc->type_color; -+ color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK; -+ if (color == cq->last_color) -+ return 0; - - while (color != cq->last_color) { -- if (opaque) -- opaque = (void *)&(rx_pkts[work_done]); -+ cq_desc_last = cq_desc; - -- ret = (*q_service)(cq->vdev, cq_desc, type, -- q_number, completed_index, opaque); - cq->to_clean++; - if (cq->to_clean == cq->ring.desc_count) { - cq->to_clean = 0; - cq->last_color = cq->last_color ? 0 : 1; - } - -+ work_done++; -+ if (work_done >= work_to_do) -+ break; -+ - cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs + - cq->ring.desc_size * cq->to_clean); -- cq_desc_dec(cq_desc, &type, &color, -- &q_number, &completed_index); - -- if (ret) -- work_done++; -- if (work_done >= work_to_do) -- break; -+ type_color = cq_desc->type_color; -+ color = (type_color >> CQ_DESC_COLOR_SHIFT) -+ & CQ_DESC_COLOR_MASK; -+ - } - -+ cq_desc_dec(cq_desc_last, &type, &color, -+ &q_number, &completed_index); -+ -+ (*q_service)(cq->vdev, cq_desc, type, -+ q_number, completed_index, opaque); - return work_done; - } - -diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h -index e083ccc..424415c 100644 ---- a/drivers/net/enic/base/vnic_rq.h -+++ b/drivers/net/enic/base/vnic_rq.h -@@ -74,7 +74,7 @@ struct vnic_rq { - struct vnic_dev_ring ring; - struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */ - unsigned int mbuf_next_idx; /* next mb to consume */ -- void *os_buf_head; -+ void *mb_head; - unsigned int pkts_outstanding; - uint16_t rx_nb_hold; - uint16_t rx_free_thresh; -diff --git a/drivers/net/enic/base/vnic_wq.c b/drivers/net/enic/base/vnic_wq.c -index a3ef417..ccbbd61 100644 ---- a/drivers/net/enic/base/vnic_wq.c -+++ b/drivers/net/enic/base/vnic_wq.c -@@ -59,71 +59,30 @@ int vnic_wq_alloc_ring(struct vnic_dev *vdev, struct vnic_wq *wq, - - static int vnic_wq_alloc_bufs(struct vnic_wq *wq) - { -- struct vnic_wq_buf *buf; -- unsigned int i, j, count = wq->ring.desc_count; -- unsigned int blks = VNIC_WQ_BUF_BLKS_NEEDED(count); -- -- for (i = 0; i < blks; i++) { -- wq->bufs[i] = kzalloc(VNIC_WQ_BUF_BLK_SZ(count), GFP_ATOMIC); -- if (!wq->bufs[i]) -- return -ENOMEM; -- } -- -- for (i = 0; i < blks; i++) { -- buf = wq->bufs[i]; -- for (j = 0; j < VNIC_WQ_BUF_BLK_ENTRIES(count); j++) { -- buf->index = i * VNIC_WQ_BUF_BLK_ENTRIES(count) + j; -- buf->desc = (u8 *)wq->ring.descs + -- wq->ring.desc_size * buf->index; -- if (buf->index + 1 == count) { -- buf->next = wq->bufs[0]; -- break; -- } else if (j + 1 == VNIC_WQ_BUF_BLK_ENTRIES(count)) { -- buf->next = wq->bufs[i + 1]; -- } else { -- buf->next = buf + 1; -- buf++; -- } -- } -- } -- -- wq->to_use = wq->to_clean = wq->bufs[0]; -- -+ unsigned int count = wq->ring.desc_count; -+ /* Allocate the mbuf ring */ -+ wq->bufs = (struct vnic_wq_buf *)rte_zmalloc_socket("wq->bufs", -+ sizeof(struct vnic_wq_buf) * count, -+ RTE_CACHE_LINE_SIZE, wq->socket_id); -+ wq->head_idx = 0; -+ wq->tail_idx = 0; -+ if (wq->bufs == NULL) -+ return -ENOMEM; - return 0; - } - - void vnic_wq_free(struct vnic_wq *wq) - { - struct vnic_dev *vdev; -- unsigned int i; - - vdev = wq->vdev; - - vnic_dev_free_desc_ring(vdev, &wq->ring); - -- for (i = 0; i < VNIC_WQ_BUF_BLKS_MAX; i++) { -- if (wq->bufs[i]) { -- kfree(wq->bufs[i]); -- wq->bufs[i] = NULL; -- } -- } -- -+ rte_free(wq->bufs); - wq->ctrl = NULL; - } - --int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count, -- unsigned int desc_size) --{ -- int mem_size = 0; -- -- mem_size += vnic_dev_desc_ring_size(&wq->ring, desc_count, desc_size); -- -- mem_size += VNIC_WQ_BUF_BLKS_NEEDED(wq->ring.desc_count) * -- VNIC_WQ_BUF_BLK_SZ(wq->ring.desc_count); -- -- return mem_size; --} -- - - int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, - unsigned int desc_count, unsigned int desc_size) -@@ -172,9 +131,8 @@ void vnic_wq_init_start(struct vnic_wq *wq, unsigned int cq_index, - iowrite32(error_interrupt_offset, &wq->ctrl->error_interrupt_offset); - iowrite32(0, &wq->ctrl->error_status); - -- wq->to_use = wq->to_clean = -- &wq->bufs[fetch_index / VNIC_WQ_BUF_BLK_ENTRIES(count)] -- [fetch_index % VNIC_WQ_BUF_BLK_ENTRIES(count)]; -+ wq->head_idx = fetch_index; -+ wq->tail_idx = wq->head_idx; - } - - void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, -@@ -184,6 +142,7 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, - vnic_wq_init_start(wq, cq_index, 0, 0, - error_interrupt_enable, - error_interrupt_offset); -+ wq->last_completed_index = 0; - } - - void vnic_wq_error_out(struct vnic_wq *wq, unsigned int error) -@@ -219,22 +178,34 @@ int vnic_wq_disable(struct vnic_wq *wq) - return -ETIMEDOUT; - } - -+static inline uint32_t -+buf_idx_incr(uint32_t n_descriptors, uint32_t idx) -+{ -+ idx++; -+ if (unlikely(idx == n_descriptors)) -+ idx = 0; -+ return idx; -+} -+ - void vnic_wq_clean(struct vnic_wq *wq, -- void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf)) -+ void (*buf_clean)(struct vnic_wq_buf *buf)) - { - struct vnic_wq_buf *buf; -+ unsigned int to_clean = wq->tail_idx; - -- buf = wq->to_clean; -+ buf = &wq->bufs[to_clean]; - - while (vnic_wq_desc_used(wq) > 0) { - -- (*buf_clean)(wq, buf); -+ (*buf_clean)(buf); -+ to_clean = buf_idx_incr(wq->ring.desc_count, to_clean); - -- buf = wq->to_clean = buf->next; -+ buf = &wq->bufs[to_clean]; - wq->ring.desc_avail++; - } - -- wq->to_use = wq->to_clean = wq->bufs[0]; -+ wq->head_idx = 0; -+ wq->tail_idx = 0; - - iowrite32(0, &wq->ctrl->fetch_index); - iowrite32(0, &wq->ctrl->posted_index); -diff --git a/drivers/net/enic/base/vnic_wq.h b/drivers/net/enic/base/vnic_wq.h -index c23de62..37c3ff9 100644 ---- a/drivers/net/enic/base/vnic_wq.h -+++ b/drivers/net/enic/base/vnic_wq.h -@@ -64,42 +64,23 @@ struct vnic_wq_ctrl { - u32 pad9; - }; - -+/* 16 bytes */ - struct vnic_wq_buf { -- struct vnic_wq_buf *next; -- dma_addr_t dma_addr; -- void *os_buf; -- unsigned int len; -- unsigned int index; -- int sop; -- void *desc; -- uint64_t wr_id; /* Cookie */ -- uint8_t cq_entry; /* Gets completion event from hw */ -- uint8_t desc_skip_cnt; /* Num descs to occupy */ -- uint8_t compressed_send; /* Both hdr and payload in one desc */ -+ struct rte_mempool *pool; -+ void *mb; - }; - --/* Break the vnic_wq_buf allocations into blocks of 32/64 entries */ --#define VNIC_WQ_BUF_MIN_BLK_ENTRIES 32 --#define VNIC_WQ_BUF_DFLT_BLK_ENTRIES 64 --#define VNIC_WQ_BUF_BLK_ENTRIES(entries) \ -- ((unsigned int)((entries < VNIC_WQ_BUF_DFLT_BLK_ENTRIES) ? \ -- VNIC_WQ_BUF_MIN_BLK_ENTRIES : VNIC_WQ_BUF_DFLT_BLK_ENTRIES)) --#define VNIC_WQ_BUF_BLK_SZ(entries) \ -- (VNIC_WQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_wq_buf)) --#define VNIC_WQ_BUF_BLKS_NEEDED(entries) \ -- DIV_ROUND_UP(entries, VNIC_WQ_BUF_BLK_ENTRIES(entries)) --#define VNIC_WQ_BUF_BLKS_MAX VNIC_WQ_BUF_BLKS_NEEDED(4096) -- - struct vnic_wq { - unsigned int index; - struct vnic_dev *vdev; - struct vnic_wq_ctrl __iomem *ctrl; /* memory-mapped */ - struct vnic_dev_ring ring; -- struct vnic_wq_buf *bufs[VNIC_WQ_BUF_BLKS_MAX]; -- struct vnic_wq_buf *to_use; -- struct vnic_wq_buf *to_clean; -- unsigned int pkts_outstanding; -+ struct vnic_wq_buf *bufs; -+ unsigned int head_idx; -+ unsigned int tail_idx; - unsigned int socket_id; -+ const struct rte_memzone *cqmsg_rz; -+ uint16_t last_completed_index; - }; - - static inline unsigned int vnic_wq_desc_avail(struct vnic_wq *wq) -@@ -114,11 +95,6 @@ static inline unsigned int vnic_wq_desc_used(struct vnic_wq *wq) - return wq->ring.desc_count - wq->ring.desc_avail - 1; - } - --static inline void *vnic_wq_next_desc(struct vnic_wq *wq) --{ -- return wq->to_use->desc; --} -- - #define PI_LOG2_CACHE_LINE_SIZE 5 - #define PI_INDEX_BITS 12 - #define PI_INDEX_MASK ((1U << PI_INDEX_BITS) - 1) -@@ -191,75 +167,6 @@ static inline u64 vnic_cached_posted_index(dma_addr_t addr, unsigned int len, - PI_PREFETCH_ADDR_MASK) << PI_PREFETCH_ADDR_OFF); - } - --static inline void vnic_wq_post(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, -- unsigned int len, int sop, int eop, -- uint8_t desc_skip_cnt, uint8_t cq_entry, -- uint8_t compressed_send, uint64_t wrid) --{ -- struct vnic_wq_buf *buf = wq->to_use; -- -- buf->sop = sop; -- buf->cq_entry = cq_entry; -- buf->compressed_send = compressed_send; -- buf->desc_skip_cnt = desc_skip_cnt; -- buf->os_buf = os_buf; -- buf->dma_addr = dma_addr; -- buf->len = len; -- buf->wr_id = wrid; -- -- buf = buf->next; -- if (eop) { --#ifdef DO_PREFETCH -- uint64_t wr = vnic_cached_posted_index(dma_addr, len, -- buf->index); --#endif -- /* Adding write memory barrier prevents compiler and/or CPU -- * reordering, thus avoiding descriptor posting before -- * descriptor is initialized. Otherwise, hardware can read -- * stale descriptor fields. -- */ -- wmb(); --#ifdef DO_PREFETCH -- /* Intel chipsets seem to limit the rate of PIOs that we can -- * push on the bus. Thus, it is very important to do a single -- * 64 bit write here. With two 32-bit writes, my maximum -- * pkt/sec rate was cut almost in half. -AJF -- */ -- iowrite64((uint64_t)wr, &wq->ctrl->posted_index); --#else -- iowrite32(buf->index, &wq->ctrl->posted_index); --#endif -- } -- wq->to_use = buf; -- -- wq->ring.desc_avail -= desc_skip_cnt; --} -- --static inline void vnic_wq_service(struct vnic_wq *wq, -- struct cq_desc *cq_desc, u16 completed_index, -- void (*buf_service)(struct vnic_wq *wq, -- struct cq_desc *cq_desc, struct vnic_wq_buf *buf, void *opaque), -- void *opaque) --{ -- struct vnic_wq_buf *buf; -- -- buf = wq->to_clean; -- while (1) { -- -- (*buf_service)(wq, cq_desc, buf, opaque); -- -- wq->ring.desc_avail++; -- -- wq->to_clean = buf->next; -- -- if (buf->index == completed_index) -- break; -- -- buf = wq->to_clean; -- } --} -- - void vnic_wq_free(struct vnic_wq *wq); - int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, - unsigned int desc_count, unsigned int desc_size); -@@ -275,8 +182,6 @@ unsigned int vnic_wq_error_status(struct vnic_wq *wq); - void vnic_wq_enable(struct vnic_wq *wq); - int vnic_wq_disable(struct vnic_wq *wq); - void vnic_wq_clean(struct vnic_wq *wq, -- void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf)); --int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count, -- unsigned int desc_size); -+ void (*buf_clean)(struct vnic_wq_buf *buf)); - - #endif /* _VNIC_WQ_H_ */ -diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h -index 8c914f5..43b82a6 100644 ---- a/drivers/net/enic/enic.h -+++ b/drivers/net/enic/enic.h -@@ -155,6 +155,30 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) - return (struct enic *)eth_dev->data->dev_private; - } - -+static inline uint32_t -+enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) -+{ -+ uint32_t d = i0 + i1; -+ d -= (d >= n_descriptors) ? n_descriptors : 0; -+ return d; -+} -+ -+static inline uint32_t -+enic_ring_sub(uint32_t n_descriptors, uint32_t i0, uint32_t i1) -+{ -+ int32_t d = i1 - i0; -+ return (uint32_t)((d < 0) ? ((int32_t)n_descriptors + d) : d); -+} -+ -+static inline uint32_t -+enic_ring_incr(uint32_t n_descriptors, uint32_t idx) -+{ -+ idx++; -+ if (unlikely(idx == n_descriptors)) -+ idx = 0; -+ return idx; -+} -+ - #define RTE_LIBRTE_ENIC_ASSERT_ENABLE - #ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE - #define ASSERT(x) do { \ -@@ -209,5 +233,6 @@ extern int enic_clsf_init(struct enic *enic); - extern void enic_clsf_destroy(struct enic *enic); - uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts); -- -+uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, -+ uint16_t nb_pkts); - #endif /* _ENIC_H_ */ -diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c -index 6bea940..697ff82 100644 ---- a/drivers/net/enic/enic_ethdev.c -+++ b/drivers/net/enic/enic_ethdev.c -@@ -519,71 +519,6 @@ static void enicpmd_remove_mac_addr(struct rte_eth_dev *eth_dev, __rte_unused ui - enic_del_mac_address(enic); - } - -- --static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, -- uint16_t nb_pkts) --{ -- uint16_t index; -- unsigned int frags; -- unsigned int pkt_len; -- unsigned int seg_len; -- unsigned int inc_len; -- unsigned int nb_segs; -- struct rte_mbuf *tx_pkt, *next_tx_pkt; -- struct vnic_wq *wq = (struct vnic_wq *)tx_queue; -- struct enic *enic = vnic_dev_priv(wq->vdev); -- unsigned short vlan_id; -- unsigned short ol_flags; -- uint8_t last_seg, eop; -- unsigned int host_tx_descs = 0; -- -- for (index = 0; index < nb_pkts; index++) { -- tx_pkt = *tx_pkts++; -- inc_len = 0; -- nb_segs = tx_pkt->nb_segs; -- if (nb_segs > vnic_wq_desc_avail(wq)) { -- if (index > 0) -- enic_post_wq_index(wq); -- -- /* wq cleanup and try again */ -- if (!enic_cleanup_wq(enic, wq) || -- (nb_segs > vnic_wq_desc_avail(wq))) { -- return index; -- } -- } -- -- pkt_len = tx_pkt->pkt_len; -- vlan_id = tx_pkt->vlan_tci; -- ol_flags = tx_pkt->ol_flags; -- for (frags = 0; inc_len < pkt_len; frags++) { -- if (!tx_pkt) -- break; -- next_tx_pkt = tx_pkt->next; -- seg_len = tx_pkt->data_len; -- inc_len += seg_len; -- -- host_tx_descs++; -- last_seg = 0; -- eop = 0; -- if ((pkt_len == inc_len) || !next_tx_pkt) { -- eop = 1; -- /* post if last packet in batch or > thresh */ -- if ((index == (nb_pkts - 1)) || -- (host_tx_descs > ENIC_TX_POST_THRESH)) { -- last_seg = 1; -- host_tx_descs = 0; -- } -- } -- enic_send_pkt(enic, wq, tx_pkt, (unsigned short)seg_len, -- !frags, eop, last_seg, ol_flags, vlan_id); -- tx_pkt = next_tx_pkt; -- } -- } -- -- enic_cleanup_wq(enic, wq); -- return index; --} -- - static const struct eth_dev_ops enicpmd_eth_dev_ops = { - .dev_configure = enicpmd_dev_configure, - .dev_start = enicpmd_dev_start, -@@ -642,7 +577,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) - enic->rte_dev = eth_dev; - eth_dev->dev_ops = &enicpmd_eth_dev_ops; - eth_dev->rx_pkt_burst = &enic_recv_pkts; -- eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts; -+ eth_dev->tx_pkt_burst = &enic_xmit_pkts; - - pdev = eth_dev->pci_dev; - rte_eth_copy_pci_info(eth_dev, pdev); -diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index 646d87f..ba73604 100644 ---- a/drivers/net/enic/enic_main.c -+++ b/drivers/net/enic/enic_main.c -@@ -40,11 +40,11 @@ - #include <libgen.h> - - #include <rte_pci.h> --#include <rte_memzone.h> - #include <rte_malloc.h> - #include <rte_mbuf.h> - #include <rte_string_fns.h> - #include <rte_ethdev.h> -+#include <rte_memzone.h> - - #include "enic_compat.h" - #include "enic.h" -@@ -58,7 +58,6 @@ - #include "vnic_cq.h" - #include "vnic_intr.h" - #include "vnic_nic.h" --#include "enic_vnic_wq.h" - - static inline struct rte_mbuf * - rte_rxmbuf_alloc(struct rte_mempool *mp) -@@ -109,38 +108,17 @@ enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) - } - } - -- - void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) - { - vnic_set_hdr_split_size(enic->vdev, split_hdr_size); - } - --static void enic_free_wq_buf(__rte_unused struct vnic_wq *wq, struct vnic_wq_buf *buf) -+static void enic_free_wq_buf(struct vnic_wq_buf *buf) - { -- struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->os_buf; -+ struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb; - - rte_mempool_put(mbuf->pool, mbuf); -- buf->os_buf = NULL; --} -- --static void enic_wq_free_buf(struct vnic_wq *wq, -- __rte_unused struct cq_desc *cq_desc, -- struct vnic_wq_buf *buf, -- __rte_unused void *opaque) --{ -- enic_free_wq_buf(wq, buf); --} -- --static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, -- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque) --{ -- struct enic *enic = vnic_dev_priv(vdev); -- -- vnic_wq_service(&enic->wq[q_number], cq_desc, -- completed_index, enic_wq_free_buf, -- opaque); -- -- return 0; -+ buf->mb = NULL; - } - - static void enic_log_q_error(struct enic *enic) -@@ -163,64 +141,6 @@ static void enic_log_q_error(struct enic *enic) - } - } - --unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq) --{ -- unsigned int cq = enic_cq_wq(enic, wq->index); -- -- /* Return the work done */ -- return vnic_cq_service(&enic->cq[cq], -- -1 /*wq_work_to_do*/, enic_wq_service, NULL); --} -- --void enic_post_wq_index(struct vnic_wq *wq) --{ -- enic_vnic_post_wq_index(wq); --} -- --void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, -- struct rte_mbuf *tx_pkt, unsigned short len, -- uint8_t sop, uint8_t eop, uint8_t cq_entry, -- uint16_t ol_flags, uint16_t vlan_tag) --{ -- struct wq_enet_desc *desc = vnic_wq_next_desc(wq); -- uint16_t mss = 0; -- uint8_t vlan_tag_insert = 0; -- uint64_t bus_addr = (dma_addr_t) -- (tx_pkt->buf_physaddr + tx_pkt->data_off); -- -- if (sop) { -- if (ol_flags & PKT_TX_VLAN_PKT) -- vlan_tag_insert = 1; -- -- if (enic->hw_ip_checksum) { -- if (ol_flags & PKT_TX_IP_CKSUM) -- mss |= ENIC_CALC_IP_CKSUM; -- -- if (ol_flags & PKT_TX_TCP_UDP_CKSUM) -- mss |= ENIC_CALC_TCP_UDP_CKSUM; -- } -- } -- -- wq_enet_desc_enc(desc, -- bus_addr, -- len, -- mss, -- 0 /* header_length */, -- 0 /* offload_mode WQ_ENET_OFFLOAD_MODE_CSUM */, -- eop, -- cq_entry, -- 0 /* fcoe_encap */, -- vlan_tag_insert, -- vlan_tag, -- 0 /* loopback */); -- -- enic_vnic_post_wq(wq, (void *)tx_pkt, bus_addr, len, -- sop, -- 1 /*desc_skip_cnt*/, -- cq_entry, -- 0 /*compressed send*/, -- 0 /*wrid*/); --} - - void enic_dev_stats_clear(struct enic *enic) - { -@@ -297,12 +217,28 @@ void enic_init_vnic_resources(struct enic *enic) - unsigned int error_interrupt_enable = 1; - unsigned int error_interrupt_offset = 0; - unsigned int index = 0; -+ unsigned int cq_idx; -+ -+ vnic_dev_stats_clear(enic->vdev); - - for (index = 0; index < enic->rq_count; index++) { - vnic_rq_init(&enic->rq[index], - enic_cq_rq(enic, index), - error_interrupt_enable, - error_interrupt_offset); -+ -+ cq_idx = enic_cq_rq(enic, index); -+ vnic_cq_init(&enic->cq[cq_idx], -+ 0 /* flow_control_enable */, -+ 1 /* color_enable */, -+ 0 /* cq_head */, -+ 0 /* cq_tail */, -+ 1 /* cq_tail_color */, -+ 0 /* interrupt_enable */, -+ 1 /* cq_entry_enable */, -+ 0 /* cq_message_enable */, -+ 0 /* interrupt offset */, -+ 0 /* cq_message_addr */); - } - - for (index = 0; index < enic->wq_count; index++) { -@@ -310,22 +246,19 @@ void enic_init_vnic_resources(struct enic *enic) - enic_cq_wq(enic, index), - error_interrupt_enable, - error_interrupt_offset); -- } -- -- vnic_dev_stats_clear(enic->vdev); - -- for (index = 0; index < enic->cq_count; index++) { -- vnic_cq_init(&enic->cq[index], -+ cq_idx = enic_cq_wq(enic, index); -+ vnic_cq_init(&enic->cq[cq_idx], - 0 /* flow_control_enable */, - 1 /* color_enable */, - 0 /* cq_head */, - 0 /* cq_tail */, - 1 /* cq_tail_color */, - 0 /* interrupt_enable */, -- 1 /* cq_entry_enable */, -- 0 /* cq_message_enable */, -+ 0 /* cq_entry_enable */, -+ 1 /* cq_message_enable */, - 0 /* interrupt offset */, -- 0 /* cq_message_addr */); -+ (u64)enic->wq[index].cqmsg_rz->phys_addr); - } - - vnic_intr_init(&enic->intr, -@@ -569,6 +502,7 @@ void enic_free_wq(void *txq) - struct vnic_wq *wq = (struct vnic_wq *)txq; - struct enic *enic = vnic_dev_priv(wq->vdev); - -+ rte_memzone_free(wq->cqmsg_rz); - vnic_wq_free(wq); - vnic_cq_free(&enic->cq[enic->rq_count + wq->index]); - } -@@ -579,6 +513,8 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, - int err; - struct vnic_wq *wq = &enic->wq[queue_idx]; - unsigned int cq_index = enic_cq_wq(enic, queue_idx); -+ char name[NAME_MAX]; -+ static int instance; - - wq->socket_id = socket_id; - if (nb_desc) { -@@ -614,6 +550,18 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, - dev_err(enic, "error in allocation of cq for wq\n"); - } - -+ /* setup up CQ message */ -+ snprintf((char *)name, sizeof(name), -+ "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx, -+ instance++); -+ -+ wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name, -+ sizeof(uint32_t), -+ SOCKET_ID_ANY, 0, -+ ENIC_ALIGN); -+ if (!wq->cqmsg_rz) -+ return -ENOMEM; -+ - return err; - } - -diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h -index 00fa71d..3e1bdf5 100644 ---- a/drivers/net/enic/enic_res.h -+++ b/drivers/net/enic/enic_res.h -@@ -53,89 +53,10 @@ - - #define ENIC_NON_TSO_MAX_DESC 16 - #define ENIC_DEFAULT_RX_FREE_THRESH 32 --#define ENIC_TX_POST_THRESH (ENIC_MIN_WQ_DESCS / 2) -+#define ENIC_TX_XMIT_MAX 64 - - #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) - --static inline void enic_queue_wq_desc_ex(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, unsigned int len, -- unsigned int mss_or_csum_offset, unsigned int hdr_len, -- int vlan_tag_insert, unsigned int vlan_tag, -- int offload_mode, int cq_entry, int sop, int eop, int loopback) --{ -- struct wq_enet_desc *desc = vnic_wq_next_desc(wq); -- u8 desc_skip_cnt = 1; -- u8 compressed_send = 0; -- u64 wrid = 0; -- -- wq_enet_desc_enc(desc, -- (u64)dma_addr | VNIC_PADDR_TARGET, -- (u16)len, -- (u16)mss_or_csum_offset, -- (u16)hdr_len, (u8)offload_mode, -- (u8)eop, (u8)cq_entry, -- 0, /* fcoe_encap */ -- (u8)vlan_tag_insert, -- (u16)vlan_tag, -- (u8)loopback); -- -- vnic_wq_post(wq, os_buf, dma_addr, len, sop, eop, desc_skip_cnt, -- (u8)cq_entry, compressed_send, wrid); --} -- --static inline void enic_queue_wq_desc_cont(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, unsigned int len, -- int eop, int loopback) --{ -- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, -- 0, 0, 0, 0, 0, -- eop, 0 /* !SOP */, eop, loopback); --} -- --static inline void enic_queue_wq_desc(struct vnic_wq *wq, void *os_buf, -- dma_addr_t dma_addr, unsigned int len, int vlan_tag_insert, -- unsigned int vlan_tag, int eop, int loopback) --{ -- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, -- 0, 0, vlan_tag_insert, vlan_tag, -- WQ_ENET_OFFLOAD_MODE_CSUM, -- eop, 1 /* SOP */, eop, loopback); --} -- --static inline void enic_queue_wq_desc_csum(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, unsigned int len, -- int ip_csum, int tcpudp_csum, int vlan_tag_insert, -- unsigned int vlan_tag, int eop, int loopback) --{ -- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, -- (ip_csum ? 1 : 0) + (tcpudp_csum ? 2 : 0), -- 0, vlan_tag_insert, vlan_tag, -- WQ_ENET_OFFLOAD_MODE_CSUM, -- eop, 1 /* SOP */, eop, loopback); --} -- --static inline void enic_queue_wq_desc_csum_l4(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, unsigned int len, -- unsigned int csum_offset, unsigned int hdr_len, -- int vlan_tag_insert, unsigned int vlan_tag, int eop, int loopback) --{ -- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, -- csum_offset, hdr_len, vlan_tag_insert, vlan_tag, -- WQ_ENET_OFFLOAD_MODE_CSUM_L4, -- eop, 1 /* SOP */, eop, loopback); --} -- --static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq, -- void *os_buf, dma_addr_t dma_addr, unsigned int len, -- unsigned int mss, unsigned int hdr_len, int vlan_tag_insert, -- unsigned int vlan_tag, int eop, int loopback) --{ -- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, -- mss, hdr_len, vlan_tag_insert, vlan_tag, -- WQ_ENET_OFFLOAD_MODE_TSO, -- eop, 1 /* SOP */, eop, loopback); --} -- - struct enic; - - int enic_get_vnic_config(struct enic *); -diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c -deleted file mode 100644 -index 39bb55c..0000000 ---- a/drivers/net/enic/enic_rx.c -+++ /dev/null -@@ -1,361 +0,0 @@ --/* -- * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved. -- * Copyright 2007 Nuova Systems, Inc. All rights reserved. -- * -- * Copyright (c) 2014, Cisco Systems, Inc. -- * All rights reserved. -- * -- * Redistribution and use in source and binary forms, with or without -- * modification, are permitted provided that the following conditions -- * are met: -- * -- * 1. Redistributions of source code must retain the above copyright -- * notice, this list of conditions and the following disclaimer. -- * -- * 2. Redistributions in binary form must reproduce the above copyright -- * notice, this list of conditions and the following disclaimer in -- * the documentation and/or other materials provided with the -- * distribution. -- * -- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -- * POSSIBILITY OF SUCH DAMAGE. -- * -- */ -- --#include <rte_mbuf.h> --#include <rte_ethdev.h> --#include <rte_prefetch.h> -- --#include "enic_compat.h" --#include "rq_enet_desc.h" --#include "enic.h" -- --#define RTE_PMD_USE_PREFETCH -- --#ifdef RTE_PMD_USE_PREFETCH --/* -- * Prefetch a cache line into all cache levels. -- */ --#define rte_enic_prefetch(p) rte_prefetch0(p) --#else --#define rte_enic_prefetch(p) do {} while (0) --#endif -- --#ifdef RTE_PMD_PACKET_PREFETCH --#define rte_packet_prefetch(p) rte_prefetch1(p) --#else --#define rte_packet_prefetch(p) do {} while (0) --#endif -- --static inline struct rte_mbuf * --rte_rxmbuf_alloc(struct rte_mempool *mp) --{ -- struct rte_mbuf *m; -- -- m = __rte_mbuf_raw_alloc(mp); -- __rte_mbuf_sanity_check_raw(m, 0); -- return m; --} -- --static inline uint16_t --enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) --{ -- return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; --} -- --static inline uint16_t --enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) --{ -- return(le16_to_cpu(crd->bytes_written_flags) & -- ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); --} -- --static inline uint8_t --enic_cq_rx_desc_packet_error(uint16_t bwflags) --{ -- return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == -- CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); --} -- --static inline uint8_t --enic_cq_rx_desc_eop(uint16_t ciflags) --{ -- return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) -- == CQ_ENET_RQ_DESC_FLAGS_EOP; --} -- --static inline uint8_t --enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) --{ -- return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & -- CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == -- CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); --} -- --static inline uint8_t --enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) --{ -- return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == -- CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); --} -- --static inline uint8_t --enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) --{ -- return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == -- CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); --} -- --static inline uint8_t --enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) --{ -- return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> -- CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); --} -- --static inline uint32_t --enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) --{ -- return le32_to_cpu(cqrd->rss_hash); --} -- --static inline uint16_t --enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) --{ -- return le16_to_cpu(cqrd->vlan); --} -- --static inline uint16_t --enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) --{ -- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -- return le16_to_cpu(cqrd->bytes_written_flags) & -- CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; --} -- --static inline uint8_t --enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out) --{ -- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -- uint16_t bwflags; -- int ret = 0; -- uint64_t pkt_err_flags = 0; -- -- bwflags = enic_cq_rx_desc_bwflags(cqrd); -- if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) { -- pkt_err_flags = PKT_RX_MAC_ERR; -- ret = 1; -- } -- *pkt_err_flags_out = pkt_err_flags; -- return ret; --} -- --/* -- * Lookup table to translate RX CQ flags to mbuf flags. -- */ --static inline uint32_t --enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) --{ -- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -- uint8_t cqrd_flags = cqrd->flags; -- static const uint32_t cq_type_table[128] __rte_cache_aligned = { -- [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, -- [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -- | RTE_PTYPE_L4_UDP, -- [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -- | RTE_PTYPE_L4_TCP, -- [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -- | RTE_PTYPE_L4_FRAG, -- [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, -- [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -- | RTE_PTYPE_L4_UDP, -- [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -- | RTE_PTYPE_L4_TCP, -- [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -- | RTE_PTYPE_L4_FRAG, -- /* All others reserved */ -- }; -- cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT -- | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 -- | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; -- return cq_type_table[cqrd_flags]; --} -- --static inline void --enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) --{ -- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -- uint16_t ciflags, bwflags, pkt_flags = 0; -- ciflags = enic_cq_rx_desc_ciflags(cqrd); -- bwflags = enic_cq_rx_desc_bwflags(cqrd); -- -- mbuf->ol_flags = 0; -- -- /* flags are meaningless if !EOP */ -- if (unlikely(!enic_cq_rx_desc_eop(ciflags))) -- goto mbuf_flags_done; -- -- /* VLAN stripping */ -- if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { -- pkt_flags |= PKT_RX_VLAN_PKT; -- mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); -- } else { -- mbuf->vlan_tci = 0; -- } -- -- /* RSS flag */ -- if (enic_cq_rx_desc_rss_type(cqrd)) { -- pkt_flags |= PKT_RX_RSS_HASH; -- mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); -- } -- -- /* checksum flags */ -- if (!enic_cq_rx_desc_csum_not_calc(cqrd) && -- (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { -- if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) -- pkt_flags |= PKT_RX_IP_CKSUM_BAD; -- if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { -- if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) -- pkt_flags |= PKT_RX_L4_CKSUM_BAD; -- } -- } -- -- mbuf_flags_done: -- mbuf->ol_flags = pkt_flags; --} -- --static inline uint32_t --enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) --{ -- uint32_t d = i0 + i1; -- ASSERT(i0 < n_descriptors); -- ASSERT(i1 < n_descriptors); -- d -= (d >= n_descriptors) ? n_descriptors : 0; -- return d; --} -- -- --uint16_t --enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -- uint16_t nb_pkts) --{ -- struct vnic_rq *rq = rx_queue; -- struct enic *enic = vnic_dev_priv(rq->vdev); -- unsigned int rx_id; -- struct rte_mbuf *nmb, *rxmb; -- uint16_t nb_rx = 0; -- uint16_t nb_hold; -- struct vnic_cq *cq; -- volatile struct cq_desc *cqd_ptr; -- uint8_t color; -- -- cq = &enic->cq[enic_cq_rq(enic, rq->index)]; -- rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ -- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -- -- nb_hold = rq->rx_nb_hold; /* mbufs held by software */ -- -- while (nb_rx < nb_pkts) { -- volatile struct rq_enet_desc *rqd_ptr; -- dma_addr_t dma_addr; -- struct cq_desc cqd; -- uint64_t ol_err_flags; -- uint8_t packet_error; -- -- /* Check for pkts available */ -- color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) -- & CQ_DESC_COLOR_MASK; -- if (color == cq->last_color) -- break; -- -- /* Get the cq descriptor and rq pointer */ -- cqd = *cqd_ptr; -- rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; -- -- /* allocate a new mbuf */ -- nmb = rte_rxmbuf_alloc(rq->mp); -- if (nmb == NULL) { -- dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", -- enic->port_id, (unsigned)rq->index); -- rte_eth_devices[enic->port_id]. -- data->rx_mbuf_alloc_failed++; -- break; -- } -- -- /* A packet error means descriptor and data are untrusted */ -- packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); -- -- /* Get the mbuf to return and replace with one just allocated */ -- rxmb = rq->mbuf_ring[rx_id]; -- rq->mbuf_ring[rx_id] = nmb; -- -- /* Increment cqd, rqd, mbuf_table index */ -- rx_id++; -- if (unlikely(rx_id == rq->ring.desc_count)) { -- rx_id = 0; -- cq->last_color = cq->last_color ? 0 : 1; -- } -- -- /* Prefetch next mbuf & desc while processing current one */ -- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -- rte_enic_prefetch(cqd_ptr); -- rte_enic_prefetch(rq->mbuf_ring[rx_id]); -- rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) -- + rx_id); -- -- /* Push descriptor for newly allocated mbuf */ -- dma_addr = (dma_addr_t)(nmb->buf_physaddr -- + RTE_PKTMBUF_HEADROOM); -- rqd_ptr->address = rte_cpu_to_le_64(dma_addr); -- rqd_ptr->length_type = cpu_to_le16(nmb->buf_len -- - RTE_PKTMBUF_HEADROOM); -- -- /* Fill in the rest of the mbuf */ -- rxmb->data_off = RTE_PKTMBUF_HEADROOM; -- rxmb->nb_segs = 1; -- rxmb->next = NULL; -- rxmb->port = enic->port_id; -- if (!packet_error) { -- rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); -- rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); -- enic_cq_rx_to_pkt_flags(&cqd, rxmb); -- } else { -- rxmb->pkt_len = 0; -- rxmb->packet_type = 0; -- rxmb->ol_flags = 0; -- } -- rxmb->data_len = rxmb->pkt_len; -- -- /* prefetch mbuf data for caller */ -- rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, -- RTE_PKTMBUF_HEADROOM)); -- -- /* store the mbuf address into the next entry of the array */ -- rx_pkts[nb_rx++] = rxmb; -- } -- -- nb_hold += nb_rx; -- cq->to_clean = rx_id; -- -- if (nb_hold > rq->rx_free_thresh) { -- rq->posted_index = enic_ring_add(rq->ring.desc_count, -- rq->posted_index, nb_hold); -- nb_hold = 0; -- rte_mb(); -- iowrite32(rq->posted_index, &rq->ctrl->posted_index); -- } -- -- rq->rx_nb_hold = nb_hold; -- -- return nb_rx; --} -diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c -new file mode 100644 -index 0000000..71ca34e ---- /dev/null -+++ b/drivers/net/enic/enic_rxtx.c -@@ -0,0 +1,505 @@ -+/* -+ * Copyright 2008-2016 Cisco Systems, Inc. All rights reserved. -+ * Copyright 2007 Nuova Systems, Inc. All rights reserved. -+ * -+ * Copyright (c) 2016, Cisco Systems, Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions -+ * are met: -+ * -+ * 1. Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * 2. Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in -+ * the documentation and/or other materials provided with the -+ * distribution. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ * -+ */ -+ -+#include <rte_mbuf.h> -+#include <rte_ethdev.h> -+#include <rte_prefetch.h> -+#include <rte_memzone.h> -+ -+#include "enic_compat.h" -+#include "rq_enet_desc.h" -+#include "enic.h" -+ -+#define RTE_PMD_USE_PREFETCH -+ -+#ifdef RTE_PMD_USE_PREFETCH -+/* -+ * Prefetch a cache line into all cache levels. -+ */ -+#define rte_enic_prefetch(p) rte_prefetch0(p) -+#else -+#define rte_enic_prefetch(p) do {} while (0) -+#endif -+ -+#ifdef RTE_PMD_PACKET_PREFETCH -+#define rte_packet_prefetch(p) rte_prefetch1(p) -+#else -+#define rte_packet_prefetch(p) do {} while (0) -+#endif -+ -+static inline struct rte_mbuf * -+rte_rxmbuf_alloc(struct rte_mempool *mp) -+{ -+ struct rte_mbuf *m; -+ -+ m = __rte_mbuf_raw_alloc(mp); -+ __rte_mbuf_sanity_check_raw(m, 0); -+ return m; -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) -+{ -+ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) -+{ -+ return(le16_to_cpu(crd->bytes_written_flags) & -+ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_packet_error(uint16_t bwflags) -+{ -+ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == -+ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_eop(uint16_t ciflags) -+{ -+ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) -+ == CQ_ENET_RQ_DESC_FLAGS_EOP; -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & -+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == -+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) -+{ -+ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == -+ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); -+} -+ -+static inline uint8_t -+enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) -+{ -+ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> -+ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); -+} -+ -+static inline uint32_t -+enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) -+{ -+ return le32_to_cpu(cqrd->rss_hash); -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) -+{ -+ return le16_to_cpu(cqrd->vlan); -+} -+ -+static inline uint16_t -+enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ return le16_to_cpu(cqrd->bytes_written_flags) & -+ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; -+} -+ -+static inline uint8_t -+enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint16_t bwflags; -+ int ret = 0; -+ uint64_t pkt_err_flags = 0; -+ -+ bwflags = enic_cq_rx_desc_bwflags(cqrd); -+ if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) { -+ pkt_err_flags = PKT_RX_MAC_ERR; -+ ret = 1; -+ } -+ *pkt_err_flags_out = pkt_err_flags; -+ return ret; -+} -+ -+/* -+ * Lookup table to translate RX CQ flags to mbuf flags. -+ */ -+static inline uint32_t -+enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint8_t cqrd_flags = cqrd->flags; -+ static const uint32_t cq_type_table[128] __rte_cache_aligned = { -+ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, -+ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_UDP, -+ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_TCP, -+ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 -+ | RTE_PTYPE_L4_FRAG, -+ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, -+ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_UDP, -+ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_TCP, -+ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 -+ | RTE_PTYPE_L4_FRAG, -+ /* All others reserved */ -+ }; -+ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT -+ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 -+ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; -+ return cq_type_table[cqrd_flags]; -+} -+ -+static inline void -+enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) -+{ -+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; -+ uint16_t ciflags, bwflags, pkt_flags = 0; -+ ciflags = enic_cq_rx_desc_ciflags(cqrd); -+ bwflags = enic_cq_rx_desc_bwflags(cqrd); -+ -+ mbuf->ol_flags = 0; -+ -+ /* flags are meaningless if !EOP */ -+ if (unlikely(!enic_cq_rx_desc_eop(ciflags))) -+ goto mbuf_flags_done; -+ -+ /* VLAN stripping */ -+ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { -+ pkt_flags |= PKT_RX_VLAN_PKT; -+ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); -+ } else { -+ mbuf->vlan_tci = 0; -+ } -+ -+ /* RSS flag */ -+ if (enic_cq_rx_desc_rss_type(cqrd)) { -+ pkt_flags |= PKT_RX_RSS_HASH; -+ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); -+ } -+ -+ /* checksum flags */ -+ if (!enic_cq_rx_desc_csum_not_calc(cqrd) && -+ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { -+ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) -+ pkt_flags |= PKT_RX_IP_CKSUM_BAD; -+ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { -+ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) -+ pkt_flags |= PKT_RX_L4_CKSUM_BAD; -+ } -+ } -+ -+ mbuf_flags_done: -+ mbuf->ol_flags = pkt_flags; -+} -+ -+uint16_t -+enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, -+ uint16_t nb_pkts) -+{ -+ struct vnic_rq *rq = rx_queue; -+ struct enic *enic = vnic_dev_priv(rq->vdev); -+ unsigned int rx_id; -+ struct rte_mbuf *nmb, *rxmb; -+ uint16_t nb_rx = 0; -+ uint16_t nb_hold; -+ struct vnic_cq *cq; -+ volatile struct cq_desc *cqd_ptr; -+ uint8_t color; -+ -+ cq = &enic->cq[enic_cq_rq(enic, rq->index)]; -+ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ -+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -+ -+ nb_hold = rq->rx_nb_hold; /* mbufs held by software */ -+ -+ while (nb_rx < nb_pkts) { -+ volatile struct rq_enet_desc *rqd_ptr; -+ dma_addr_t dma_addr; -+ struct cq_desc cqd; -+ uint64_t ol_err_flags; -+ uint8_t packet_error; -+ -+ /* Check for pkts available */ -+ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) -+ & CQ_DESC_COLOR_MASK; -+ if (color == cq->last_color) -+ break; -+ -+ /* Get the cq descriptor and rq pointer */ -+ cqd = *cqd_ptr; -+ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; -+ -+ /* allocate a new mbuf */ -+ nmb = rte_rxmbuf_alloc(rq->mp); -+ if (nmb == NULL) { -+ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", -+ enic->port_id, (unsigned)rq->index); -+ rte_eth_devices[enic->port_id]. -+ data->rx_mbuf_alloc_failed++; -+ break; -+ } -+ -+ /* A packet error means descriptor and data are untrusted */ -+ packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); -+ -+ /* Get the mbuf to return and replace with one just allocated */ -+ rxmb = rq->mbuf_ring[rx_id]; -+ rq->mbuf_ring[rx_id] = nmb; -+ -+ /* Increment cqd, rqd, mbuf_table index */ -+ rx_id++; -+ if (unlikely(rx_id == rq->ring.desc_count)) { -+ rx_id = 0; -+ cq->last_color = cq->last_color ? 0 : 1; -+ } -+ -+ /* Prefetch next mbuf & desc while processing current one */ -+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -+ rte_enic_prefetch(cqd_ptr); -+ rte_enic_prefetch(rq->mbuf_ring[rx_id]); -+ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) -+ + rx_id); -+ -+ /* Push descriptor for newly allocated mbuf */ -+ dma_addr = (dma_addr_t)(nmb->buf_physaddr -+ + RTE_PKTMBUF_HEADROOM); -+ rqd_ptr->address = rte_cpu_to_le_64(dma_addr); -+ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len -+ - RTE_PKTMBUF_HEADROOM); -+ -+ /* Fill in the rest of the mbuf */ -+ rxmb->data_off = RTE_PKTMBUF_HEADROOM; -+ rxmb->nb_segs = 1; -+ rxmb->next = NULL; -+ rxmb->port = enic->port_id; -+ if (!packet_error) { -+ rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); -+ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); -+ enic_cq_rx_to_pkt_flags(&cqd, rxmb); -+ } else { -+ rxmb->pkt_len = 0; -+ rxmb->packet_type = 0; -+ rxmb->ol_flags = 0; -+ } -+ rxmb->data_len = rxmb->pkt_len; -+ -+ /* prefetch mbuf data for caller */ -+ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, -+ RTE_PKTMBUF_HEADROOM)); -+ -+ /* store the mbuf address into the next entry of the array */ -+ rx_pkts[nb_rx++] = rxmb; -+ } -+ -+ nb_hold += nb_rx; -+ cq->to_clean = rx_id; -+ -+ if (nb_hold > rq->rx_free_thresh) { -+ rq->posted_index = enic_ring_add(rq->ring.desc_count, -+ rq->posted_index, nb_hold); -+ nb_hold = 0; -+ rte_mb(); -+ iowrite32(rq->posted_index, &rq->ctrl->posted_index); -+ } -+ -+ rq->rx_nb_hold = nb_hold; -+ -+ return nb_rx; -+} -+ -+static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index) -+{ -+ struct vnic_wq_buf *buf; -+ struct rte_mbuf *m, *free[ENIC_MAX_WQ_DESCS]; -+ unsigned int nb_to_free, nb_free = 0, i; -+ struct rte_mempool *pool; -+ unsigned int tail_idx; -+ unsigned int desc_count = wq->ring.desc_count; -+ -+ nb_to_free = enic_ring_sub(desc_count, wq->tail_idx, completed_index) -+ + 1; -+ tail_idx = wq->tail_idx; -+ buf = &wq->bufs[tail_idx]; -+ pool = ((struct rte_mbuf *)buf->mb)->pool; -+ for (i = 0; i < nb_to_free; i++) { -+ buf = &wq->bufs[tail_idx]; -+ m = (struct rte_mbuf *)(buf->mb); -+ if (likely(m->pool == pool)) { -+ ASSERT(nb_free < ENIC_MAX_WQ_DESCS); -+ free[nb_free++] = m; -+ } else { -+ rte_mempool_put_bulk(pool, (void *)free, nb_free); -+ free[0] = m; -+ nb_free = 1; -+ pool = m->pool; -+ } -+ tail_idx = enic_ring_incr(desc_count, tail_idx); -+ buf->mb = NULL; -+ } -+ -+ rte_mempool_put_bulk(pool, (void **)free, nb_free); -+ -+ wq->tail_idx = tail_idx; -+ wq->ring.desc_avail += nb_to_free; -+} -+ -+unsigned int enic_cleanup_wq(__rte_unused struct enic *enic, struct vnic_wq *wq) -+{ -+ u16 completed_index; -+ -+ completed_index = *((uint32_t *)wq->cqmsg_rz->addr) & 0xffff; -+ -+ if (wq->last_completed_index != completed_index) { -+ enic_free_wq_bufs(wq, completed_index); -+ wq->last_completed_index = completed_index; -+ } -+ return 0; -+} -+ -+uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, -+ uint16_t nb_pkts) -+{ -+ uint16_t index; -+ unsigned int pkt_len, data_len; -+ unsigned int nb_segs; -+ struct rte_mbuf *tx_pkt; -+ struct vnic_wq *wq = (struct vnic_wq *)tx_queue; -+ struct enic *enic = vnic_dev_priv(wq->vdev); -+ unsigned short vlan_id; -+ unsigned short ol_flags; -+ unsigned int wq_desc_avail; -+ int head_idx; -+ struct vnic_wq_buf *buf; -+ unsigned int hw_ip_cksum_enabled; -+ unsigned int desc_count; -+ struct wq_enet_desc *descs, *desc_p, desc_tmp; -+ uint16_t mss; -+ uint8_t vlan_tag_insert; -+ uint8_t eop; -+ uint64_t bus_addr; -+ -+ enic_cleanup_wq(enic, wq); -+ wq_desc_avail = vnic_wq_desc_avail(wq); -+ head_idx = wq->head_idx; -+ desc_count = wq->ring.desc_count; -+ -+ nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX); -+ -+ hw_ip_cksum_enabled = enic->hw_ip_checksum; -+ for (index = 0; index < nb_pkts; index++) { -+ tx_pkt = *tx_pkts++; -+ nb_segs = tx_pkt->nb_segs; -+ if (nb_segs > wq_desc_avail) { -+ if (index > 0) -+ goto post; -+ goto done; -+ } -+ -+ pkt_len = tx_pkt->pkt_len; -+ data_len = tx_pkt->data_len; -+ vlan_id = tx_pkt->vlan_tci; -+ ol_flags = tx_pkt->ol_flags; -+ -+ mss = 0; -+ vlan_tag_insert = 0; -+ bus_addr = (dma_addr_t) -+ (tx_pkt->buf_physaddr + tx_pkt->data_off); -+ -+ descs = (struct wq_enet_desc *)wq->ring.descs; -+ desc_p = descs + head_idx; -+ -+ eop = (data_len == pkt_len); -+ -+ if (ol_flags & PKT_TX_VLAN_PKT) -+ vlan_tag_insert = 1; -+ -+ if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_IP_CKSUM)) -+ mss |= ENIC_CALC_IP_CKSUM; -+ -+ if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_TCP_UDP_CKSUM)) -+ mss |= ENIC_CALC_TCP_UDP_CKSUM; -+ -+ wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, 0, 0, eop, -+ eop, 0, vlan_tag_insert, vlan_id, 0); -+ -+ *desc_p = desc_tmp; -+ buf = &wq->bufs[head_idx]; -+ buf->mb = (void *)tx_pkt; -+ head_idx = enic_ring_incr(desc_count, head_idx); -+ wq_desc_avail--; -+ -+ if (!eop) { -+ for (tx_pkt = tx_pkt->next; tx_pkt; tx_pkt = -+ tx_pkt->next) { -+ data_len = tx_pkt->data_len; -+ -+ if (tx_pkt->next == NULL) -+ eop = 1; -+ desc_p = descs + head_idx; -+ bus_addr = (dma_addr_t)(tx_pkt->buf_physaddr -+ + tx_pkt->data_off); -+ wq_enet_desc_enc((struct wq_enet_desc *) -+ &desc_tmp, bus_addr, data_len, -+ mss, 0, 0, eop, eop, 0, -+ vlan_tag_insert, vlan_id, 0); -+ -+ *desc_p = desc_tmp; -+ buf = &wq->bufs[head_idx]; -+ buf->mb = (void *)tx_pkt; -+ head_idx = enic_ring_incr(desc_count, head_idx); -+ wq_desc_avail--; -+ } -+ } -+ } -+ post: -+ rte_wmb(); -+ iowrite32(head_idx, &wq->ctrl->posted_index); -+ done: -+ wq->ring.desc_avail = wq_desc_avail; -+ wq->head_idx = head_idx; -+ -+ return index; -+} --- -2.7.0 - |