From ca6bbb723880e91d006de6cc485259da988859aa Mon Sep 17 00:00:00 2001 From: John Daley <johndale@cisco.com> Date: Tue, 5 Apr 2016 15:19:06 -0700 Subject: [PATCH 3/3] enic: Optimization of Tx path to reduce Host CPU overhead, cleanup Optimizations and cleanup: - flatten packet send path - flatten mbuf free path - disable CQ entry writing and use CQ messages instead - use rte_mempool_put_bulk() to bulk return freed mbufs - remove unnecessary fields vnic_bufs struct, use contiguous array of cache aligned divisible elements. No next pointers. - use local variables inside per packet loop instead of fields in structs. - factor book keeping out of the per packet tx loop where possible (removed several conditionals) - put Tx and Rx code in 1 file (enic_rxtx.c) Reviewed-by: Nelson Escobar <neescoba@cisco.com> Signed-off-by: John Daley <johndale@cisco.com> --- drivers/net/enic/Makefile | 2 +- drivers/net/enic/base/enic_vnic_wq.h | 79 ------ drivers/net/enic/base/vnic_cq.h | 37 +-- drivers/net/enic/base/vnic_rq.h | 2 +- drivers/net/enic/base/vnic_wq.c | 89 +++--- drivers/net/enic/base/vnic_wq.h | 113 +------- drivers/net/enic/enic.h | 27 +- drivers/net/enic/enic_ethdev.c | 67 +---- drivers/net/enic/enic_main.c | 132 +++------ drivers/net/enic/enic_res.h | 81 +----- drivers/net/enic/enic_rx.c | 361 ------------------------- drivers/net/enic/enic_rxtx.c | 505 +++++++++++++++++++++++++++++++++++ 12 files changed, 635 insertions(+), 860 deletions(-) delete mode 100644 drivers/net/enic/base/enic_vnic_wq.h delete mode 100644 drivers/net/enic/enic_rx.c create mode 100644 drivers/net/enic/enic_rxtx.c diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile index f316274..3926b79 100644 --- a/drivers/net/enic/Makefile +++ b/drivers/net/enic/Makefile @@ -53,7 +53,7 @@ VPATH += $(SRCDIR)/src # SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c -SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c +SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rxtx.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c diff --git a/drivers/net/enic/base/enic_vnic_wq.h b/drivers/net/enic/base/enic_vnic_wq.h deleted file mode 100644 index b019109..0000000 --- a/drivers/net/enic/base/enic_vnic_wq.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2008-2015 Cisco Systems, Inc. All rights reserved. - * Copyright 2007 Nuova Systems, Inc. All rights reserved. - * - * Copyright (c) 2015, Cisco Systems, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef _ENIC_VNIC_WQ_H_ -#define _ENIC_VNIC_WQ_H_ - -#include "vnic_dev.h" -#include "vnic_cq.h" - -static inline void enic_vnic_post_wq_index(struct vnic_wq *wq) -{ - struct vnic_wq_buf *buf = wq->to_use; - - /* Adding write memory barrier prevents compiler and/or CPU - * reordering, thus avoiding descriptor posting before - * descriptor is initialized. Otherwise, hardware can read - * stale descriptor fields. - */ - wmb(); - iowrite32(buf->index, &wq->ctrl->posted_index); -} - -static inline void enic_vnic_post_wq(struct vnic_wq *wq, - void *os_buf, dma_addr_t dma_addr, - unsigned int len, int sop, - uint8_t desc_skip_cnt, uint8_t cq_entry, - uint8_t compressed_send, uint64_t wrid) -{ - struct vnic_wq_buf *buf = wq->to_use; - - buf->sop = sop; - buf->cq_entry = cq_entry; - buf->compressed_send = compressed_send; - buf->desc_skip_cnt = desc_skip_cnt; - buf->os_buf = os_buf; - buf->dma_addr = dma_addr; - buf->len = len; - buf->wr_id = wrid; - - buf = buf->next; - wq->ring.desc_avail -= desc_skip_cnt; - wq->to_use = buf; - - if (cq_entry) - enic_vnic_post_wq_index(wq); -} - -#endif /* _ENIC_VNIC_WQ_H_ */ diff --git a/drivers/net/enic/base/vnic_cq.h b/drivers/net/enic/base/vnic_cq.h index 922391b..ffc1aaa 100644 --- a/drivers/net/enic/base/vnic_cq.h +++ b/drivers/net/enic/base/vnic_cq.h @@ -96,41 +96,46 @@ static inline unsigned int vnic_cq_service(struct vnic_cq *cq, u8 type, u16 q_number, u16 completed_index, void *opaque), void *opaque) { - struct cq_desc *cq_desc; + struct cq_desc *cq_desc, *cq_desc_last; unsigned int work_done = 0; u16 q_number, completed_index; - u8 type, color; - struct rte_mbuf **rx_pkts = opaque; - unsigned int ret; + u8 type, color, type_color; cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs + cq->ring.desc_size * cq->to_clean); - cq_desc_dec(cq_desc, &type, &color, - &q_number, &completed_index); + + type_color = cq_desc->type_color; + color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK; + if (color == cq->last_color) + return 0; while (color != cq->last_color) { - if (opaque) - opaque = (void *)&(rx_pkts[work_done]); + cq_desc_last = cq_desc; - ret = (*q_service)(cq->vdev, cq_desc, type, - q_number, completed_index, opaque); cq->to_clean++; if (cq->to_clean == cq->ring.desc_count) { cq->to_clean = 0; cq->last_color = cq->last_color ? 0 : 1; } + work_done++; + if (work_done >= work_to_do) + break; + cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs + cq->ring.desc_size * cq->to_clean); - cq_desc_dec(cq_desc, &type, &color, - &q_number, &completed_index); - if (ret) - work_done++; - if (work_done >= work_to_do) - break; + type_color = cq_desc->type_color; + color = (type_color >> CQ_DESC_COLOR_SHIFT) + & CQ_DESC_COLOR_MASK; + } + cq_desc_dec(cq_desc_last, &type, &color, + &q_number, &completed_index); + + (*q_service)(cq->vdev, cq_desc, type, + q_number, completed_index, opaque); return work_done; } diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h index e083ccc..424415c 100644 --- a/drivers/net/enic/base/vnic_rq.h +++ b/drivers/net/enic/base/vnic_rq.h @@ -74,7 +74,7 @@ struct vnic_rq { struct vnic_dev_ring ring; struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */ unsigned int mbuf_next_idx; /* next mb to consume */ - void *os_buf_head; + void *mb_head; unsigned int pkts_outstanding; uint16_t rx_nb_hold; uint16_t rx_free_thresh; diff --git a/drivers/net/enic/base/vnic_wq.c b/drivers/net/enic/base/vnic_wq.c index a3ef417..ccbbd61 100644 --- a/drivers/net/enic/base/vnic_wq.c +++ b/drivers/net/enic/base/vnic_wq.c @@ -59,71 +59,30 @@ int vnic_wq_alloc_ring(struct vnic_dev *vdev, struct vnic_wq *wq, static int vnic_wq_alloc_bufs(struct vnic_wq *wq) { - struct vnic_wq_buf *buf; - unsigned int i, j, count = wq->ring.desc_count; - unsigned int blks = VNIC_WQ_BUF_BLKS_NEEDED(count); - - for (i = 0; i < blks; i++) { - wq->bufs[i] = kzalloc(VNIC_WQ_BUF_BLK_SZ(count), GFP_ATOMIC); - if (!wq->bufs[i]) - return -ENOMEM; - } - - for (i = 0; i < blks; i++) { - buf = wq->bufs[i]; - for (j = 0; j < VNIC_WQ_BUF_BLK_ENTRIES(count); j++) { - buf->index = i * VNIC_WQ_BUF_BLK_ENTRIES(count) + j; - buf->desc = (u8 *)wq->ring.descs + - wq->ring.desc_size * buf->index; - if (buf->index + 1 == count) { - buf->next = wq->bufs[0]; - break; - } else if (j + 1 == VNIC_WQ_BUF_BLK_ENTRIES(count)) { - buf->next = wq->bufs[i + 1]; - } else { - buf->next = buf + 1; - buf++; - } - } - } - - wq->to_use = wq->to_clean = wq->bufs[0]; - + unsigned int count = wq->ring.desc_count; + /* Allocate the mbuf ring */ + wq->bufs = (struct vnic_wq_buf *)rte_zmalloc_socket("wq->bufs", + sizeof(struct vnic_wq_buf) * count, + RTE_CACHE_LINE_SIZE, wq->socket_id); + wq->head_idx = 0; + wq->tail_idx = 0; + if (wq->bufs == NULL) + return -ENOMEM; return 0; } void vnic_wq_free(struct vnic_wq *wq) { struct vnic_dev *vdev; - unsigned int i; vdev = wq->vdev; vnic_dev_free_desc_ring(vdev, &wq->ring); - for (i = 0; i < VNIC_WQ_BUF_BLKS_MAX; i++) { - if (wq->bufs[i]) { - kfree(wq->bufs[i]); - wq->bufs[i] = NULL; - } - } - + rte_free(wq->bufs); wq->ctrl = NULL; } -int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count, - unsigned int desc_size) -{ - int mem_size = 0; - - mem_size += vnic_dev_desc_ring_size(&wq->ring, desc_count, desc_size); - - mem_size += VNIC_WQ_BUF_BLKS_NEEDED(wq->ring.desc_count) * - VNIC_WQ_BUF_BLK_SZ(wq->ring.desc_count); - - return mem_size; -} - int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, unsigned int desc_count, unsigned int desc_size) @@ -172,9 +131,8 @@ void vnic_wq_init_start(struct vnic_wq *wq, unsigned int cq_index, iowrite32(error_interrupt_offset, &wq->ctrl->error_interrupt_offset); iowrite32(0, &wq->ctrl->error_status); - wq->to_use = wq->to_clean = - &wq->bufs[fetch_index / VNIC_WQ_BUF_BLK_ENTRIES(count)] - [fetch_index % VNIC_WQ_BUF_BLK_ENTRIES(count)]; + wq->head_idx = fetch_index; + wq->tail_idx = wq->head_idx; } void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, @@ -184,6 +142,7 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, vnic_wq_init_start(wq, cq_index, 0, 0, error_interrupt_enable, error_interrupt_offset); + wq->last_completed_index = 0; } void vnic_wq_error_out(struct vnic_wq *wq, unsigned int error) @@ -219,22 +178,34 @@ int vnic_wq_disable(struct vnic_wq *wq) return -ETIMEDOUT; } +static inline uint32_t +buf_idx_incr(uint32_t n_descriptors, uint32_t idx) +{ + idx++; + if (unlikely(idx == n_descriptors)) + idx = 0; + return idx; +} + void vnic_wq_clean(struct vnic_wq *wq, - void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf)) + void (*buf_clean)(struct vnic_wq_buf *buf)) { struct vnic_wq_buf *buf; + unsigned int to_clean = wq->tail_idx; - buf = wq->to_clean; + buf = &wq->bufs[to_clean]; while (vnic_wq_desc_used(wq) > 0) { - (*buf_clean)(wq, buf); + (*buf_clean)(buf); + to_clean = buf_idx_incr(wq->ring.desc_count, to_clean); - buf = wq->to_clean = buf->next; + buf = &wq->bufs[to_clean]; wq->ring.desc_avail++; } - wq->to_use = wq->to_clean = wq->bufs[0]; + wq->head_idx = 0; + wq->tail_idx = 0; iowrite32(0, &wq->ctrl->fetch_index); iowrite32(0, &wq->ctrl->posted_index); diff --git a/drivers/net/enic/base/vnic_wq.h b/drivers/net/enic/base/vnic_wq.h index c23de62..37c3ff9 100644 --- a/drivers/net/enic/base/vnic_wq.h +++ b/drivers/net/enic/base/vnic_wq.h @@ -64,42 +64,23 @@ struct vnic_wq_ctrl { u32 pad9; }; +/* 16 bytes */ struct vnic_wq_buf { - struct vnic_wq_buf *next; - dma_addr_t dma_addr; - void *os_buf; - unsigned int len; - unsigned int index; - int sop; - void *desc; - uint64_t wr_id; /* Cookie */ - uint8_t cq_entry; /* Gets completion event from hw */ - uint8_t desc_skip_cnt; /* Num descs to occupy */ - uint8_t compressed_send; /* Both hdr and payload in one desc */ + struct rte_mempool *pool; + void *mb; }; -/* Break the vnic_wq_buf allocations into blocks of 32/64 entries */ -#define VNIC_WQ_BUF_MIN_BLK_ENTRIES 32 -#define VNIC_WQ_BUF_DFLT_BLK_ENTRIES 64 -#define VNIC_WQ_BUF_BLK_ENTRIES(entries) \ - ((unsigned int)((entries < VNIC_WQ_BUF_DFLT_BLK_ENTRIES) ? \ - VNIC_WQ_BUF_MIN_BLK_ENTRIES : VNIC_WQ_BUF_DFLT_BLK_ENTRIES)) -#define VNIC_WQ_BUF_BLK_SZ(entries) \ - (VNIC_WQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_wq_buf)) -#define VNIC_WQ_BUF_BLKS_NEEDED(entries) \ - DIV_ROUND_UP(entries, VNIC_WQ_BUF_BLK_ENTRIES(entries)) -#define VNIC_WQ_BUF_BLKS_MAX VNIC_WQ_BUF_BLKS_NEEDED(4096) - struct vnic_wq { unsigned int index; struct vnic_dev *vdev; struct vnic_wq_ctrl __iomem *ctrl; /* memory-mapped */ struct vnic_dev_ring ring; - struct vnic_wq_buf *bufs[VNIC_WQ_BUF_BLKS_MAX]; - struct vnic_wq_buf *to_use; - struct vnic_wq_buf *to_clean; - unsigned int pkts_outstanding; + struct vnic_wq_buf *bufs; + unsigned int head_idx; + unsigned int tail_idx; unsigned int socket_id; + const struct rte_memzone *cqmsg_rz; + uint16_t last_completed_index; }; static inline unsigned int vnic_wq_desc_avail(struct vnic_wq *wq) @@ -114,11 +95,6 @@ static inline unsigned int vnic_wq_desc_used(struct vnic_wq *wq) return wq->ring.desc_count - wq->ring.desc_avail - 1; } -static inline void *vnic_wq_next_desc(struct vnic_wq *wq) -{ - return wq->to_use->desc; -} - #define PI_LOG2_CACHE_LINE_SIZE 5 #define PI_INDEX_BITS 12 #define PI_INDEX_MASK ((1U << PI_INDEX_BITS) - 1) @@ -191,75 +167,6 @@ static inline u64 vnic_cached_posted_index(dma_addr_t addr, unsigned int len, PI_PREFETCH_ADDR_MASK) << PI_PREFETCH_ADDR_OFF); } -static inline void vnic_wq_post(struct vnic_wq *wq, - void *os_buf, dma_addr_t dma_addr, - unsigned int len, int sop, int eop, - uint8_t desc_skip_cnt, uint8_t cq_entry, - uint8_t compressed_send, uint64_t wrid) -{ - struct vnic_wq_buf *buf = wq->to_use; - - buf->sop = sop; - buf->cq_entry = cq_entry; - buf->compressed_send = compressed_send; - buf->desc_skip_cnt = desc_skip_cnt; - buf->os_buf = os_buf; - buf->dma_addr = dma_addr; - buf->len = len; - buf->wr_id = wrid; - - buf = buf->next; - if (eop) { -#ifdef DO_PREFETCH - uint64_t wr = vnic_cached_posted_index(dma_addr, len, - buf->index); -#endif - /* Adding write memory barrier prevents compiler and/or CPU - * reordering, thus avoiding descriptor posting before - * descriptor is initialized. Otherwise, hardware can read - * stale descriptor fields. - */ - wmb(); -#ifdef DO_PREFETCH - /* Intel chipsets seem to limit the rate of PIOs that we can - * push on the bus. Thus, it is very important to do a single - * 64 bit write here. With two 32-bit writes, my maximum - * pkt/sec rate was cut almost in half. -AJF - */ - iowrite64((uint64_t)wr, &wq->ctrl->posted_index); -#else - iowrite32(buf->index, &wq->ctrl->posted_index); -#endif - } - wq->to_use = buf; - - wq->ring.desc_avail -= desc_skip_cnt; -} - -static inline void vnic_wq_service(struct vnic_wq *wq, - struct cq_desc *cq_desc, u16 completed_index, - void (*buf_service)(struct vnic_wq *wq, - struct cq_desc *cq_desc, struct vnic_wq_buf *buf, void *opaque), - void *opaque) -{ - struct vnic_wq_buf *buf; - - buf = wq->to_clean; - while (1) { - - (*buf_service)(wq, cq_desc, buf, opaque); - - wq->ring.desc_avail++; - - wq->to_clean = buf->next; - - if (buf->index == completed_index) - break; - - buf = wq->to_clean; - } -} - void vnic_wq_free(struct vnic_wq *wq); int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, unsigned int desc_count, unsigned int desc_size); @@ -275,8 +182,6 @@ unsigned int vnic_wq_error_status(struct vnic_wq *wq); void vnic_wq_enable(struct vnic_wq *wq); int vnic_wq_disable(struct vnic_wq *wq); void vnic_wq_clean(struct vnic_wq *wq, - void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf)); -int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count, - unsigned int desc_size); + void (*buf_clean)(struct vnic_wq_buf *buf)); #endif /* _VNIC_WQ_H_ */ diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h index 8c914f5..43b82a6 100644 --- a/drivers/net/enic/enic.h +++ b/drivers/net/enic/enic.h @@ -155,6 +155,30 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) return (struct enic *)eth_dev->data->dev_private; } +static inline uint32_t +enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) +{ + uint32_t d = i0 + i1; + d -= (d >= n_descriptors) ? n_descriptors : 0; + return d; +} + +static inline uint32_t +enic_ring_sub(uint32_t n_descriptors, uint32_t i0, uint32_t i1) +{ + int32_t d = i1 - i0; + return (uint32_t)((d < 0) ? ((int32_t)n_descriptors + d) : d); +} + +static inline uint32_t +enic_ring_incr(uint32_t n_descriptors, uint32_t idx) +{ + idx++; + if (unlikely(idx == n_descriptors)) + idx = 0; + return idx; +} + #define RTE_LIBRTE_ENIC_ASSERT_ENABLE #ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE #define ASSERT(x) do { \ @@ -209,5 +233,6 @@ extern int enic_clsf_init(struct enic *enic); extern void enic_clsf_destroy(struct enic *enic); uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); - +uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); #endif /* _ENIC_H_ */ diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c index 6bea940..697ff82 100644 --- a/drivers/net/enic/enic_ethdev.c +++ b/drivers/net/enic/enic_ethdev.c @@ -519,71 +519,6 @@ static void enicpmd_remove_mac_addr(struct rte_eth_dev *eth_dev, __rte_unused ui enic_del_mac_address(enic); } - -static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts) -{ - uint16_t index; - unsigned int frags; - unsigned int pkt_len; - unsigned int seg_len; - unsigned int inc_len; - unsigned int nb_segs; - struct rte_mbuf *tx_pkt, *next_tx_pkt; - struct vnic_wq *wq = (struct vnic_wq *)tx_queue; - struct enic *enic = vnic_dev_priv(wq->vdev); - unsigned short vlan_id; - unsigned short ol_flags; - uint8_t last_seg, eop; - unsigned int host_tx_descs = 0; - - for (index = 0; index < nb_pkts; index++) { - tx_pkt = *tx_pkts++; - inc_len = 0; - nb_segs = tx_pkt->nb_segs; - if (nb_segs > vnic_wq_desc_avail(wq)) { - if (index > 0) - enic_post_wq_index(wq); - - /* wq cleanup and try again */ - if (!enic_cleanup_wq(enic, wq) || - (nb_segs > vnic_wq_desc_avail(wq))) { - return index; - } - } - - pkt_len = tx_pkt->pkt_len; - vlan_id = tx_pkt->vlan_tci; - ol_flags = tx_pkt->ol_flags; - for (frags = 0; inc_len < pkt_len; frags++) { - if (!tx_pkt) - break; - next_tx_pkt = tx_pkt->next; - seg_len = tx_pkt->data_len; - inc_len += seg_len; - - host_tx_descs++; - last_seg = 0; - eop = 0; - if ((pkt_len == inc_len) || !next_tx_pkt) { - eop = 1; - /* post if last packet in batch or > thresh */ - if ((index == (nb_pkts - 1)) || - (host_tx_descs > ENIC_TX_POST_THRESH)) { - last_seg = 1; - host_tx_descs = 0; - } - } - enic_send_pkt(enic, wq, tx_pkt, (unsigned short)seg_len, - !frags, eop, last_seg, ol_flags, vlan_id); - tx_pkt = next_tx_pkt; - } - } - - enic_cleanup_wq(enic, wq); - return index; -} - static const struct eth_dev_ops enicpmd_eth_dev_ops = { .dev_configure = enicpmd_dev_configure, .dev_start = enicpmd_dev_start, @@ -642,7 +577,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) enic->rte_dev = eth_dev; eth_dev->dev_ops = &enicpmd_eth_dev_ops; eth_dev->rx_pkt_burst = &enic_recv_pkts; - eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts; + eth_dev->tx_pkt_burst = &enic_xmit_pkts; pdev = eth_dev->pci_dev; rte_eth_copy_pci_info(eth_dev, pdev); diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c index 646d87f..ba73604 100644 --- a/drivers/net/enic/enic_main.c +++ b/drivers/net/enic/enic_main.c @@ -40,11 +40,11 @@ #include <libgen.h> #include <rte_pci.h> -#include <rte_memzone.h> #include <rte_malloc.h> #include <rte_mbuf.h> #include <rte_string_fns.h> #include <rte_ethdev.h> +#include <rte_memzone.h> #include "enic_compat.h" #include "enic.h" @@ -58,7 +58,6 @@ #include "vnic_cq.h" #include "vnic_intr.h" #include "vnic_nic.h" -#include "enic_vnic_wq.h" static inline struct rte_mbuf * rte_rxmbuf_alloc(struct rte_mempool *mp) @@ -109,38 +108,17 @@ enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) } } - void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) { vnic_set_hdr_split_size(enic->vdev, split_hdr_size); } -static void enic_free_wq_buf(__rte_unused struct vnic_wq *wq, struct vnic_wq_buf *buf) +static void enic_free_wq_buf(struct vnic_wq_buf *buf) { - struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->os_buf; + struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb; rte_mempool_put(mbuf->pool, mbuf); - buf->os_buf = NULL; -} - -static void enic_wq_free_buf(struct vnic_wq *wq, - __rte_unused struct cq_desc *cq_desc, - struct vnic_wq_buf *buf, - __rte_unused void *opaque) -{ - enic_free_wq_buf(wq, buf); -} - -static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, - __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque) -{ - struct enic *enic = vnic_dev_priv(vdev); - - vnic_wq_service(&enic->wq[q_number], cq_desc, - completed_index, enic_wq_free_buf, - opaque); - - return 0; + buf->mb = NULL; } static void enic_log_q_error(struct enic *enic) @@ -163,64 +141,6 @@ static void enic_log_q_error(struct enic *enic) } } -unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq) -{ - unsigned int cq = enic_cq_wq(enic, wq->index); - - /* Return the work done */ - return vnic_cq_service(&enic->cq[cq], - -1 /*wq_work_to_do*/, enic_wq_service, NULL); -} - -void enic_post_wq_index(struct vnic_wq *wq) -{ - enic_vnic_post_wq_index(wq); -} - -void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, - struct rte_mbuf *tx_pkt, unsigned short len, - uint8_t sop, uint8_t eop, uint8_t cq_entry, - uint16_t ol_flags, uint16_t vlan_tag) -{ - struct wq_enet_desc *desc = vnic_wq_next_desc(wq); - uint16_t mss = 0; - uint8_t vlan_tag_insert = 0; - uint64_t bus_addr = (dma_addr_t) - (tx_pkt->buf_physaddr + tx_pkt->data_off); - - if (sop) { - if (ol_flags & PKT_TX_VLAN_PKT) - vlan_tag_insert = 1; - - if (enic->hw_ip_checksum) { - if (ol_flags & PKT_TX_IP_CKSUM) - mss |= ENIC_CALC_IP_CKSUM; - - if (ol_flags & PKT_TX_TCP_UDP_CKSUM) - mss |= ENIC_CALC_TCP_UDP_CKSUM; - } - } - - wq_enet_desc_enc(desc, - bus_addr, - len, - mss, - 0 /* header_length */, - 0 /* offload_mode WQ_ENET_OFFLOAD_MODE_CSUM */, - eop, - cq_entry, - 0 /* fcoe_encap */, - vlan_tag_insert, - vlan_tag, - 0 /* loopback */); - - enic_vnic_post_wq(wq, (void *)tx_pkt, bus_addr, len, - sop, - 1 /*desc_skip_cnt*/, - cq_entry, - 0 /*compressed send*/, - 0 /*wrid*/); -} void enic_dev_stats_clear(struct enic *enic) { @@ -297,12 +217,28 @@ void enic_init_vnic_resources(struct enic *enic) unsigned int error_interrupt_enable = 1; unsigned int error_interrupt_offset = 0; unsigned int index = 0; + unsigned int cq_idx; + + vnic_dev_stats_clear(enic->vdev); for (index = 0; index < enic->rq_count; index++) { vnic_rq_init(&enic->rq[index], enic_cq_rq(enic, index), error_interrupt_enable, error_interrupt_offset); + + cq_idx = enic_cq_rq(enic, index); + vnic_cq_init(&enic->cq[cq_idx], + 0 /* flow_control_enable */, + 1 /* color_enable */, + 0 /* cq_head */, + 0 /* cq_tail */, + 1 /* cq_tail_color */, + 0 /* interrupt_enable */, + 1 /* cq_entry_enable */, + 0 /* cq_message_enable */, + 0 /* interrupt offset */, + 0 /* cq_message_addr */); } for (index = 0; index < enic->wq_count; index++) { @@ -310,22 +246,19 @@ void enic_init_vnic_resources(struct enic *enic) enic_cq_wq(enic, index), error_interrupt_enable, error_interrupt_offset); - } - - vnic_dev_stats_clear(enic->vdev); - for (index = 0; index < enic->cq_count; index++) { - vnic_cq_init(&enic->cq[index], + cq_idx = enic_cq_wq(enic, index); + vnic_cq_init(&enic->cq[cq_idx], 0 /* flow_control_enable */, 1 /* color_enable */, 0 /* cq_head */, 0 /* cq_tail */, 1 /* cq_tail_color */, 0 /* interrupt_enable */, - 1 /* cq_entry_enable */, - 0 /* cq_message_enable */, + 0 /* cq_entry_enable */, + 1 /* cq_message_enable */, 0 /* interrupt offset */, - 0 /* cq_message_addr */); + (u64)enic->wq[index].cqmsg_rz->phys_addr); } vnic_intr_init(&enic->intr, @@ -569,6 +502,7 @@ void enic_free_wq(void *txq) struct vnic_wq *wq = (struct vnic_wq *)txq; struct enic *enic = vnic_dev_priv(wq->vdev); + rte_memzone_free(wq->cqmsg_rz); vnic_wq_free(wq); vnic_cq_free(&enic->cq[enic->rq_count + wq->index]); } @@ -579,6 +513,8 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, int err; struct vnic_wq *wq = &enic->wq[queue_idx]; unsigned int cq_index = enic_cq_wq(enic, queue_idx); + char name[NAME_MAX]; + static int instance; wq->socket_id = socket_id; if (nb_desc) { @@ -614,6 +550,18 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, dev_err(enic, "error in allocation of cq for wq\n"); } + /* setup up CQ message */ + snprintf((char *)name, sizeof(name), + "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx, + instance++); + + wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name, + sizeof(uint32_t), + SOCKET_ID_ANY, 0, + ENIC_ALIGN); + if (!wq->cqmsg_rz) + return -ENOMEM; + return err; } diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h index 00fa71d..3e1bdf5 100644 --- a/drivers/net/enic/enic_res.h +++ b/drivers/net/enic/enic_res.h @@ -53,89 +53,10 @@ #define ENIC_NON_TSO_MAX_DESC 16 #define ENIC_DEFAULT_RX_FREE_THRESH 32 -#define ENIC_TX_POST_THRESH (ENIC_MIN_WQ_DESCS / 2) +#define ENIC_TX_XMIT_MAX 64 #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) -static inline void enic_queue_wq_desc_ex(struct vnic_wq *wq, - void *os_buf, dma_addr_t dma_addr, unsigned int len, - unsigned int mss_or_csum_offset, unsigned int hdr_len, - int vlan_tag_insert, unsigned int vlan_tag, - int offload_mode, int cq_entry, int sop, int eop, int loopback) -{ - struct wq_enet_desc *desc = vnic_wq_next_desc(wq); - u8 desc_skip_cnt = 1; - u8 compressed_send = 0; - u64 wrid = 0; - - wq_enet_desc_enc(desc, - (u64)dma_addr | VNIC_PADDR_TARGET, - (u16)len, - (u16)mss_or_csum_offset, - (u16)hdr_len, (u8)offload_mode, - (u8)eop, (u8)cq_entry, - 0, /* fcoe_encap */ - (u8)vlan_tag_insert, - (u16)vlan_tag, - (u8)loopback); - - vnic_wq_post(wq, os_buf, dma_addr, len, sop, eop, desc_skip_cnt, - (u8)cq_entry, compressed_send, wrid); -} - -static inline void enic_queue_wq_desc_cont(struct vnic_wq *wq, - void *os_buf, dma_addr_t dma_addr, unsigned int len, - int eop, int loopback) -{ - enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, - 0, 0, 0, 0, 0, - eop, 0 /* !SOP */, eop, loopback); -} - -static inline void enic_queue_wq_desc(struct vnic_wq *wq, void *os_buf, - dma_addr_t dma_addr, unsigned int len, int vlan_tag_insert, - unsigned int vlan_tag, int eop, int loopback) -{ - enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, - 0, 0, vlan_tag_insert, vlan_tag, - WQ_ENET_OFFLOAD_MODE_CSUM, - eop, 1 /* SOP */, eop, loopback); -} - -static inline void enic_queue_wq_desc_csum(struct vnic_wq *wq, - void *os_buf, dma_addr_t dma_addr, unsigned int len, - int ip_csum, int tcpudp_csum, int vlan_tag_insert, - unsigned int vlan_tag, int eop, int loopback) -{ - enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, - (ip_csum ? 1 : 0) + (tcpudp_csum ? 2 : 0), - 0, vlan_tag_insert, vlan_tag, - WQ_ENET_OFFLOAD_MODE_CSUM, - eop, 1 /* SOP */, eop, loopback); -} - -static inline void enic_queue_wq_desc_csum_l4(struct vnic_wq *wq, - void *os_buf, dma_addr_t dma_addr, unsigned int len, - unsigned int csum_offset, unsigned int hdr_len, - int vlan_tag_insert, unsigned int vlan_tag, int eop, int loopback) -{ - enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, - csum_offset, hdr_len, vlan_tag_insert, vlan_tag, - WQ_ENET_OFFLOAD_MODE_CSUM_L4, - eop, 1 /* SOP */, eop, loopback); -} - -static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq, - void *os_buf, dma_addr_t dma_addr, unsigned int len, - unsigned int mss, unsigned int hdr_len, int vlan_tag_insert, - unsigned int vlan_tag, int eop, int loopback) -{ - enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len, - mss, hdr_len, vlan_tag_insert, vlan_tag, - WQ_ENET_OFFLOAD_MODE_TSO, - eop, 1 /* SOP */, eop, loopback); -} - struct enic; int enic_get_vnic_config(struct enic *); diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c deleted file mode 100644 index 39bb55c..0000000 --- a/drivers/net/enic/enic_rx.c +++ /dev/null @@ -1,361 +0,0 @@ -/* - * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved. - * Copyright 2007 Nuova Systems, Inc. All rights reserved. - * - * Copyright (c) 2014, Cisco Systems, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include <rte_mbuf.h> -#include <rte_ethdev.h> -#include <rte_prefetch.h> - -#include "enic_compat.h" -#include "rq_enet_desc.h" -#include "enic.h" - -#define RTE_PMD_USE_PREFETCH - -#ifdef RTE_PMD_USE_PREFETCH -/* - * Prefetch a cache line into all cache levels. - */ -#define rte_enic_prefetch(p) rte_prefetch0(p) -#else -#define rte_enic_prefetch(p) do {} while (0) -#endif - -#ifdef RTE_PMD_PACKET_PREFETCH -#define rte_packet_prefetch(p) rte_prefetch1(p) -#else -#define rte_packet_prefetch(p) do {} while (0) -#endif - -static inline struct rte_mbuf * -rte_rxmbuf_alloc(struct rte_mempool *mp) -{ - struct rte_mbuf *m; - - m = __rte_mbuf_raw_alloc(mp); - __rte_mbuf_sanity_check_raw(m, 0); - return m; -} - -static inline uint16_t -enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) -{ - return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; -} - -static inline uint16_t -enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) -{ - return(le16_to_cpu(crd->bytes_written_flags) & - ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); -} - -static inline uint8_t -enic_cq_rx_desc_packet_error(uint16_t bwflags) -{ - return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == - CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); -} - -static inline uint8_t -enic_cq_rx_desc_eop(uint16_t ciflags) -{ - return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) - == CQ_ENET_RQ_DESC_FLAGS_EOP; -} - -static inline uint8_t -enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) -{ - return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & - CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == - CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); -} - -static inline uint8_t -enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) -{ - return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == - CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); -} - -static inline uint8_t -enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) -{ - return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == - CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); -} - -static inline uint8_t -enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) -{ - return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> - CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); -} - -static inline uint32_t -enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) -{ - return le32_to_cpu(cqrd->rss_hash); -} - -static inline uint16_t -enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) -{ - return le16_to_cpu(cqrd->vlan); -} - -static inline uint16_t -enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) -{ - struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; - return le16_to_cpu(cqrd->bytes_written_flags) & - CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; -} - -static inline uint8_t -enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out) -{ - struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; - uint16_t bwflags; - int ret = 0; - uint64_t pkt_err_flags = 0; - - bwflags = enic_cq_rx_desc_bwflags(cqrd); - if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) { - pkt_err_flags = PKT_RX_MAC_ERR; - ret = 1; - } - *pkt_err_flags_out = pkt_err_flags; - return ret; -} - -/* - * Lookup table to translate RX CQ flags to mbuf flags. - */ -static inline uint32_t -enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) -{ - struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; - uint8_t cqrd_flags = cqrd->flags; - static const uint32_t cq_type_table[128] __rte_cache_aligned = { - [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, - [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 - | RTE_PTYPE_L4_UDP, - [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 - | RTE_PTYPE_L4_TCP, - [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 - | RTE_PTYPE_L4_FRAG, - [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, - [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 - | RTE_PTYPE_L4_UDP, - [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 - | RTE_PTYPE_L4_TCP, - [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 - | RTE_PTYPE_L4_FRAG, - /* All others reserved */ - }; - cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT - | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 - | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; - return cq_type_table[cqrd_flags]; -} - -static inline void -enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) -{ - struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; - uint16_t ciflags, bwflags, pkt_flags = 0; - ciflags = enic_cq_rx_desc_ciflags(cqrd); - bwflags = enic_cq_rx_desc_bwflags(cqrd); - - mbuf->ol_flags = 0; - - /* flags are meaningless if !EOP */ - if (unlikely(!enic_cq_rx_desc_eop(ciflags))) - goto mbuf_flags_done; - - /* VLAN stripping */ - if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { - pkt_flags |= PKT_RX_VLAN_PKT; - mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); - } else { - mbuf->vlan_tci = 0; - } - - /* RSS flag */ - if (enic_cq_rx_desc_rss_type(cqrd)) { - pkt_flags |= PKT_RX_RSS_HASH; - mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); - } - - /* checksum flags */ - if (!enic_cq_rx_desc_csum_not_calc(cqrd) && - (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { - if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) - pkt_flags |= PKT_RX_IP_CKSUM_BAD; - if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { - if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) - pkt_flags |= PKT_RX_L4_CKSUM_BAD; - } - } - - mbuf_flags_done: - mbuf->ol_flags = pkt_flags; -} - -static inline uint32_t -enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) -{ - uint32_t d = i0 + i1; - ASSERT(i0 < n_descriptors); - ASSERT(i1 < n_descriptors); - d -= (d >= n_descriptors) ? n_descriptors : 0; - return d; -} - - -uint16_t -enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts) -{ - struct vnic_rq *rq = rx_queue; - struct enic *enic = vnic_dev_priv(rq->vdev); - unsigned int rx_id; - struct rte_mbuf *nmb, *rxmb; - uint16_t nb_rx = 0; - uint16_t nb_hold; - struct vnic_cq *cq; - volatile struct cq_desc *cqd_ptr; - uint8_t color; - - cq = &enic->cq[enic_cq_rq(enic, rq->index)]; - rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ - cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; - - nb_hold = rq->rx_nb_hold; /* mbufs held by software */ - - while (nb_rx < nb_pkts) { - volatile struct rq_enet_desc *rqd_ptr; - dma_addr_t dma_addr; - struct cq_desc cqd; - uint64_t ol_err_flags; - uint8_t packet_error; - - /* Check for pkts available */ - color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) - & CQ_DESC_COLOR_MASK; - if (color == cq->last_color) - break; - - /* Get the cq descriptor and rq pointer */ - cqd = *cqd_ptr; - rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; - - /* allocate a new mbuf */ - nmb = rte_rxmbuf_alloc(rq->mp); - if (nmb == NULL) { - dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", - enic->port_id, (unsigned)rq->index); - rte_eth_devices[enic->port_id]. - data->rx_mbuf_alloc_failed++; - break; - } - - /* A packet error means descriptor and data are untrusted */ - packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); - - /* Get the mbuf to return and replace with one just allocated */ - rxmb = rq->mbuf_ring[rx_id]; - rq->mbuf_ring[rx_id] = nmb; - - /* Increment cqd, rqd, mbuf_table index */ - rx_id++; - if (unlikely(rx_id == rq->ring.desc_count)) { - rx_id = 0; - cq->last_color = cq->last_color ? 0 : 1; - } - - /* Prefetch next mbuf & desc while processing current one */ - cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; - rte_enic_prefetch(cqd_ptr); - rte_enic_prefetch(rq->mbuf_ring[rx_id]); - rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) - + rx_id); - - /* Push descriptor for newly allocated mbuf */ - dma_addr = (dma_addr_t)(nmb->buf_physaddr - + RTE_PKTMBUF_HEADROOM); - rqd_ptr->address = rte_cpu_to_le_64(dma_addr); - rqd_ptr->length_type = cpu_to_le16(nmb->buf_len - - RTE_PKTMBUF_HEADROOM); - - /* Fill in the rest of the mbuf */ - rxmb->data_off = RTE_PKTMBUF_HEADROOM; - rxmb->nb_segs = 1; - rxmb->next = NULL; - rxmb->port = enic->port_id; - if (!packet_error) { - rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); - rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); - enic_cq_rx_to_pkt_flags(&cqd, rxmb); - } else { - rxmb->pkt_len = 0; - rxmb->packet_type = 0; - rxmb->ol_flags = 0; - } - rxmb->data_len = rxmb->pkt_len; - - /* prefetch mbuf data for caller */ - rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, - RTE_PKTMBUF_HEADROOM)); - - /* store the mbuf address into the next entry of the array */ - rx_pkts[nb_rx++] = rxmb; - } - - nb_hold += nb_rx; - cq->to_clean = rx_id; - - if (nb_hold > rq->rx_free_thresh) { - rq->posted_index = enic_ring_add(rq->ring.desc_count, - rq->posted_index, nb_hold); - nb_hold = 0; - rte_mb(); - iowrite32(rq->posted_index, &rq->ctrl->posted_index); - } - - rq->rx_nb_hold = nb_hold; - - return nb_rx; -} diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c new file mode 100644 index 0000000..71ca34e --- /dev/null +++ b/drivers/net/enic/enic_rxtx.c @@ -0,0 +1,505 @@ +/* + * Copyright 2008-2016 Cisco Systems, Inc. All rights reserved. + * Copyright 2007 Nuova Systems, Inc. All rights reserved. + * + * Copyright (c) 2016, Cisco Systems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <rte_mbuf.h> +#include <rte_ethdev.h> +#include <rte_prefetch.h> +#include <rte_memzone.h> + +#include "enic_compat.h" +#include "rq_enet_desc.h" +#include "enic.h" + +#define RTE_PMD_USE_PREFETCH + +#ifdef RTE_PMD_USE_PREFETCH +/* + * Prefetch a cache line into all cache levels. + */ +#define rte_enic_prefetch(p) rte_prefetch0(p) +#else +#define rte_enic_prefetch(p) do {} while (0) +#endif + +#ifdef RTE_PMD_PACKET_PREFETCH +#define rte_packet_prefetch(p) rte_prefetch1(p) +#else +#define rte_packet_prefetch(p) do {} while (0) +#endif + +static inline struct rte_mbuf * +rte_rxmbuf_alloc(struct rte_mempool *mp) +{ + struct rte_mbuf *m; + + m = __rte_mbuf_raw_alloc(mp); + __rte_mbuf_sanity_check_raw(m, 0); + return m; +} + +static inline uint16_t +enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) +{ + return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; +} + +static inline uint16_t +enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) +{ + return(le16_to_cpu(crd->bytes_written_flags) & + ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); +} + +static inline uint8_t +enic_cq_rx_desc_packet_error(uint16_t bwflags) +{ + return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == + CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); +} + +static inline uint8_t +enic_cq_rx_desc_eop(uint16_t ciflags) +{ + return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) + == CQ_ENET_RQ_DESC_FLAGS_EOP; +} + +static inline uint8_t +enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) +{ + return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & + CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == + CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); +} + +static inline uint8_t +enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) +{ + return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == + CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); +} + +static inline uint8_t +enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) +{ + return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == + CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); +} + +static inline uint8_t +enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) +{ + return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> + CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); +} + +static inline uint32_t +enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) +{ + return le32_to_cpu(cqrd->rss_hash); +} + +static inline uint16_t +enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) +{ + return le16_to_cpu(cqrd->vlan); +} + +static inline uint16_t +enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) +{ + struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; + return le16_to_cpu(cqrd->bytes_written_flags) & + CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; +} + +static inline uint8_t +enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out) +{ + struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; + uint16_t bwflags; + int ret = 0; + uint64_t pkt_err_flags = 0; + + bwflags = enic_cq_rx_desc_bwflags(cqrd); + if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) { + pkt_err_flags = PKT_RX_MAC_ERR; + ret = 1; + } + *pkt_err_flags_out = pkt_err_flags; + return ret; +} + +/* + * Lookup table to translate RX CQ flags to mbuf flags. + */ +static inline uint32_t +enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) +{ + struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; + uint8_t cqrd_flags = cqrd->flags; + static const uint32_t cq_type_table[128] __rte_cache_aligned = { + [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, + [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 + | RTE_PTYPE_L4_UDP, + [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 + | RTE_PTYPE_L4_TCP, + [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 + | RTE_PTYPE_L4_FRAG, + [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, + [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 + | RTE_PTYPE_L4_UDP, + [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 + | RTE_PTYPE_L4_TCP, + [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 + | RTE_PTYPE_L4_FRAG, + /* All others reserved */ + }; + cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT + | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 + | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; + return cq_type_table[cqrd_flags]; +} + +static inline void +enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) +{ + struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; + uint16_t ciflags, bwflags, pkt_flags = 0; + ciflags = enic_cq_rx_desc_ciflags(cqrd); + bwflags = enic_cq_rx_desc_bwflags(cqrd); + + mbuf->ol_flags = 0; + + /* flags are meaningless if !EOP */ + if (unlikely(!enic_cq_rx_desc_eop(ciflags))) + goto mbuf_flags_done; + + /* VLAN stripping */ + if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { + pkt_flags |= PKT_RX_VLAN_PKT; + mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); + } else { + mbuf->vlan_tci = 0; + } + + /* RSS flag */ + if (enic_cq_rx_desc_rss_type(cqrd)) { + pkt_flags |= PKT_RX_RSS_HASH; + mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); + } + + /* checksum flags */ + if (!enic_cq_rx_desc_csum_not_calc(cqrd) && + (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { + if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) + pkt_flags |= PKT_RX_IP_CKSUM_BAD; + if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { + if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) + pkt_flags |= PKT_RX_L4_CKSUM_BAD; + } + } + + mbuf_flags_done: + mbuf->ol_flags = pkt_flags; +} + +uint16_t +enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct vnic_rq *rq = rx_queue; + struct enic *enic = vnic_dev_priv(rq->vdev); + unsigned int rx_id; + struct rte_mbuf *nmb, *rxmb; + uint16_t nb_rx = 0; + uint16_t nb_hold; + struct vnic_cq *cq; + volatile struct cq_desc *cqd_ptr; + uint8_t color; + + cq = &enic->cq[enic_cq_rq(enic, rq->index)]; + rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ + cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; + + nb_hold = rq->rx_nb_hold; /* mbufs held by software */ + + while (nb_rx < nb_pkts) { + volatile struct rq_enet_desc *rqd_ptr; + dma_addr_t dma_addr; + struct cq_desc cqd; + uint64_t ol_err_flags; + uint8_t packet_error; + + /* Check for pkts available */ + color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) + & CQ_DESC_COLOR_MASK; + if (color == cq->last_color) + break; + + /* Get the cq descriptor and rq pointer */ + cqd = *cqd_ptr; + rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; + + /* allocate a new mbuf */ + nmb = rte_rxmbuf_alloc(rq->mp); + if (nmb == NULL) { + dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", + enic->port_id, (unsigned)rq->index); + rte_eth_devices[enic->port_id]. + data->rx_mbuf_alloc_failed++; + break; + } + + /* A packet error means descriptor and data are untrusted */ + packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); + + /* Get the mbuf to return and replace with one just allocated */ + rxmb = rq->mbuf_ring[rx_id]; + rq->mbuf_ring[rx_id] = nmb; + + /* Increment cqd, rqd, mbuf_table index */ + rx_id++; + if (unlikely(rx_id == rq->ring.desc_count)) { + rx_id = 0; + cq->last_color = cq->last_color ? 0 : 1; + } + + /* Prefetch next mbuf & desc while processing current one */ + cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; + rte_enic_prefetch(cqd_ptr); + rte_enic_prefetch(rq->mbuf_ring[rx_id]); + rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) + + rx_id); + + /* Push descriptor for newly allocated mbuf */ + dma_addr = (dma_addr_t)(nmb->buf_physaddr + + RTE_PKTMBUF_HEADROOM); + rqd_ptr->address = rte_cpu_to_le_64(dma_addr); + rqd_ptr->length_type = cpu_to_le16(nmb->buf_len + - RTE_PKTMBUF_HEADROOM); + + /* Fill in the rest of the mbuf */ + rxmb->data_off = RTE_PKTMBUF_HEADROOM; + rxmb->nb_segs = 1; + rxmb->next = NULL; + rxmb->port = enic->port_id; + if (!packet_error) { + rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); + rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); + enic_cq_rx_to_pkt_flags(&cqd, rxmb); + } else { + rxmb->pkt_len = 0; + rxmb->packet_type = 0; + rxmb->ol_flags = 0; + } + rxmb->data_len = rxmb->pkt_len; + + /* prefetch mbuf data for caller */ + rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, + RTE_PKTMBUF_HEADROOM)); + + /* store the mbuf address into the next entry of the array */ + rx_pkts[nb_rx++] = rxmb; + } + + nb_hold += nb_rx; + cq->to_clean = rx_id; + + if (nb_hold > rq->rx_free_thresh) { + rq->posted_index = enic_ring_add(rq->ring.desc_count, + rq->posted_index, nb_hold); + nb_hold = 0; + rte_mb(); + iowrite32(rq->posted_index, &rq->ctrl->posted_index); + } + + rq->rx_nb_hold = nb_hold; + + return nb_rx; +} + +static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index) +{ + struct vnic_wq_buf *buf; + struct rte_mbuf *m, *free[ENIC_MAX_WQ_DESCS]; + unsigned int nb_to_free, nb_free = 0, i; + struct rte_mempool *pool; + unsigned int tail_idx; + unsigned int desc_count = wq->ring.desc_count; + + nb_to_free = enic_ring_sub(desc_count, wq->tail_idx, completed_index) + + 1; + tail_idx = wq->tail_idx; + buf = &wq->bufs[tail_idx]; + pool = ((struct rte_mbuf *)buf->mb)->pool; + for (i = 0; i < nb_to_free; i++) { + buf = &wq->bufs[tail_idx]; + m = (struct rte_mbuf *)(buf->mb); + if (likely(m->pool == pool)) { + ASSERT(nb_free < ENIC_MAX_WQ_DESCS); + free[nb_free++] = m; + } else { + rte_mempool_put_bulk(pool, (void *)free, nb_free); + free[0] = m; + nb_free = 1; + pool = m->pool; + } + tail_idx = enic_ring_incr(desc_count, tail_idx); + buf->mb = NULL; + } + + rte_mempool_put_bulk(pool, (void **)free, nb_free); + + wq->tail_idx = tail_idx; + wq->ring.desc_avail += nb_to_free; +} + +unsigned int enic_cleanup_wq(__rte_unused struct enic *enic, struct vnic_wq *wq) +{ + u16 completed_index; + + completed_index = *((uint32_t *)wq->cqmsg_rz->addr) & 0xffff; + + if (wq->last_completed_index != completed_index) { + enic_free_wq_bufs(wq, completed_index); + wq->last_completed_index = completed_index; + } + return 0; +} + +uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + uint16_t index; + unsigned int pkt_len, data_len; + unsigned int nb_segs; + struct rte_mbuf *tx_pkt; + struct vnic_wq *wq = (struct vnic_wq *)tx_queue; + struct enic *enic = vnic_dev_priv(wq->vdev); + unsigned short vlan_id; + unsigned short ol_flags; + unsigned int wq_desc_avail; + int head_idx; + struct vnic_wq_buf *buf; + unsigned int hw_ip_cksum_enabled; + unsigned int desc_count; + struct wq_enet_desc *descs, *desc_p, desc_tmp; + uint16_t mss; + uint8_t vlan_tag_insert; + uint8_t eop; + uint64_t bus_addr; + + enic_cleanup_wq(enic, wq); + wq_desc_avail = vnic_wq_desc_avail(wq); + head_idx = wq->head_idx; + desc_count = wq->ring.desc_count; + + nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX); + + hw_ip_cksum_enabled = enic->hw_ip_checksum; + for (index = 0; index < nb_pkts; index++) { + tx_pkt = *tx_pkts++; + nb_segs = tx_pkt->nb_segs; + if (nb_segs > wq_desc_avail) { + if (index > 0) + goto post; + goto done; + } + + pkt_len = tx_pkt->pkt_len; + data_len = tx_pkt->data_len; + vlan_id = tx_pkt->vlan_tci; + ol_flags = tx_pkt->ol_flags; + + mss = 0; + vlan_tag_insert = 0; + bus_addr = (dma_addr_t) + (tx_pkt->buf_physaddr + tx_pkt->data_off); + + descs = (struct wq_enet_desc *)wq->ring.descs; + desc_p = descs + head_idx; + + eop = (data_len == pkt_len); + + if (ol_flags & PKT_TX_VLAN_PKT) + vlan_tag_insert = 1; + + if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_IP_CKSUM)) + mss |= ENIC_CALC_IP_CKSUM; + + if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_TCP_UDP_CKSUM)) + mss |= ENIC_CALC_TCP_UDP_CKSUM; + + wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, 0, 0, eop, + eop, 0, vlan_tag_insert, vlan_id, 0); + + *desc_p = desc_tmp; + buf = &wq->bufs[head_idx]; + buf->mb = (void *)tx_pkt; + head_idx = enic_ring_incr(desc_count, head_idx); + wq_desc_avail--; + + if (!eop) { + for (tx_pkt = tx_pkt->next; tx_pkt; tx_pkt = + tx_pkt->next) { + data_len = tx_pkt->data_len; + + if (tx_pkt->next == NULL) + eop = 1; + desc_p = descs + head_idx; + bus_addr = (dma_addr_t)(tx_pkt->buf_physaddr + + tx_pkt->data_off); + wq_enet_desc_enc((struct wq_enet_desc *) + &desc_tmp, bus_addr, data_len, + mss, 0, 0, eop, eop, 0, + vlan_tag_insert, vlan_id, 0); + + *desc_p = desc_tmp; + buf = &wq->bufs[head_idx]; + buf->mb = (void *)tx_pkt; + head_idx = enic_ring_incr(desc_count, head_idx); + wq_desc_avail--; + } + } + } + post: + rte_wmb(); + iowrite32(head_idx, &wq->ctrl->posted_index); + done: + wq->ring.desc_avail = wq_desc_avail; + wq->head_idx = head_idx; + + return index; +} -- 2.7.0