diff options
Diffstat (limited to 'dpdk/dpdk-16.04_patches/0016-ENIC-scatter-RX.patch')
| -rw-r--r-- | dpdk/dpdk-16.04_patches/0016-ENIC-scatter-RX.patch | 672 |
1 files changed, 0 insertions, 672 deletions
diff --git a/dpdk/dpdk-16.04_patches/0016-ENIC-scatter-RX.patch b/dpdk/dpdk-16.04_patches/0016-ENIC-scatter-RX.patch deleted file mode 100644 index e0daab060c7..00000000000 --- a/dpdk/dpdk-16.04_patches/0016-ENIC-scatter-RX.patch +++ /dev/null @@ -1,672 +0,0 @@ -From f03d5a02fc2b3cc24bf059a273ea1473cdb9993b Mon Sep 17 00:00:00 2001 -From: John Lo <loj@cisco.com> -Date: Tue, 7 Jun 2016 12:40:07 +0200 -Subject: [PATCH 16/17] ENIC scatter RX - ---- - drivers/net/enic/base/rq_enet_desc.h | 2 +- - drivers/net/enic/base/vnic_rq.c | 12 +- - drivers/net/enic/base/vnic_rq.h | 18 ++- - drivers/net/enic/enic.h | 10 ++ - drivers/net/enic/enic_main.c | 236 +++++++++++++++++++++++++++-------- - drivers/net/enic/enic_rxtx.c | 139 ++++++++++++++------- - 6 files changed, 313 insertions(+), 104 deletions(-) - -diff --git a/drivers/net/enic/base/rq_enet_desc.h b/drivers/net/enic/base/rq_enet_desc.h -index 7292d9d..13e24b4 100644 ---- a/drivers/net/enic/base/rq_enet_desc.h -+++ b/drivers/net/enic/base/rq_enet_desc.h -@@ -55,7 +55,7 @@ enum rq_enet_type_types { - #define RQ_ENET_TYPE_BITS 2 - #define RQ_ENET_TYPE_MASK ((1 << RQ_ENET_TYPE_BITS) - 1) - --static inline void rq_enet_desc_enc(struct rq_enet_desc *desc, -+static inline void rq_enet_desc_enc(volatile struct rq_enet_desc *desc, - u64 address, u8 type, u16 length) - { - desc->address = cpu_to_le64(address); -diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c -index cb62c5e..d97f93e 100644 ---- a/drivers/net/enic/base/vnic_rq.c -+++ b/drivers/net/enic/base/vnic_rq.c -@@ -84,11 +84,16 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, - iowrite32(cq_index, &rq->ctrl->cq_index); - iowrite32(error_interrupt_enable, &rq->ctrl->error_interrupt_enable); - iowrite32(error_interrupt_offset, &rq->ctrl->error_interrupt_offset); -- iowrite32(0, &rq->ctrl->dropped_packet_count); - iowrite32(0, &rq->ctrl->error_status); - iowrite32(fetch_index, &rq->ctrl->fetch_index); - iowrite32(posted_index, &rq->ctrl->posted_index); -- -+ if (rq->is_sop) { -+// printf("Writing 0x%x to %s rq\n", -+// ((rq->is_sop << 10) | rq->data_queue_idx), -+// rq->is_sop ? "sop":"data"); -+ iowrite32(((rq->is_sop << 10) | rq->data_queue_idx), -+ &rq->ctrl->data_ring); -+ } - } - - void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, -@@ -96,6 +101,7 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, - unsigned int error_interrupt_offset) - { - u32 fetch_index = 0; -+ - /* Use current fetch_index as the ring starting point */ - fetch_index = ioread32(&rq->ctrl->fetch_index); - -@@ -110,6 +116,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, - error_interrupt_offset); - rq->rxst_idx = 0; - rq->tot_pkts = 0; -+ rq->pkt_first_seg = NULL; -+ rq->pkt_last_seg = NULL; - } - - void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error) -diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h -index 424415c..d1e2f52 100644 ---- a/drivers/net/enic/base/vnic_rq.h -+++ b/drivers/net/enic/base/vnic_rq.h -@@ -60,10 +60,18 @@ struct vnic_rq_ctrl { - u32 pad7; - u32 error_status; /* 0x48 */ - u32 pad8; -- u32 dropped_packet_count; /* 0x50 */ -+ u32 tcp_sn; /* 0x50 */ - u32 pad9; -- u32 dropped_packet_count_rc; /* 0x58 */ -+ u32 unused; /* 0x58 */ - u32 pad10; -+ u32 dca_select; /* 0x60 */ -+ u32 pad11; -+ u32 dca_value; /* 0x68 */ -+ u32 pad12; -+ u32 data_ring; /* 0x70 */ -+ u32 pad13; -+ u32 header_split; /* 0x78 */ -+ u32 pad14; - }; - - struct vnic_rq { -@@ -82,6 +90,12 @@ struct vnic_rq { - struct rte_mempool *mp; - uint16_t rxst_idx; - uint32_t tot_pkts; -+ uint16_t data_queue_idx; -+ uint8_t is_sop; -+ uint8_t in_use; -+ struct rte_mbuf *pkt_first_seg; -+ struct rte_mbuf *pkt_last_seg; -+ unsigned int max_mbufs_per_pkt; - }; - - static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq) -diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h -index 7c1b5c9..d2de6ee 100644 ---- a/drivers/net/enic/enic.h -+++ b/drivers/net/enic/enic.h -@@ -142,6 +142,16 @@ struct enic { - struct enic_soft_stats soft_stats; - }; - -+static inline unsigned int enic_sop_rq(__rte_unused struct enic *enic, unsigned int rq) -+{ -+ return rq * 2; -+} -+ -+static inline unsigned int enic_data_rq(__rte_unused struct enic *enic, unsigned int rq) -+{ -+ return rq * 2 + 1; -+} -+ - static inline unsigned int enic_cq_rq(__rte_unused struct enic *enic, unsigned int rq) - { - return rq; -diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c -index a00565a..be17707 100644 ---- a/drivers/net/enic/enic_main.c -+++ b/drivers/net/enic/enic_main.c -@@ -247,15 +247,23 @@ void enic_init_vnic_resources(struct enic *enic) - unsigned int error_interrupt_offset = 0; - unsigned int index = 0; - unsigned int cq_idx; -+ struct vnic_rq *data_rq; - - vnic_dev_stats_clear(enic->vdev); - - for (index = 0; index < enic->rq_count; index++) { -- vnic_rq_init(&enic->rq[index], -+ vnic_rq_init(&enic->rq[enic_sop_rq(enic, index)], - enic_cq_rq(enic, index), - error_interrupt_enable, - error_interrupt_offset); - -+ data_rq = &enic->rq[enic_data_rq(enic, index)]; -+ if (data_rq->in_use) -+ vnic_rq_init(data_rq, -+ enic_cq_rq(enic, index), -+ error_interrupt_enable, -+ error_interrupt_offset); -+ - cq_idx = enic_cq_rq(enic, index); - vnic_cq_init(&enic->cq[cq_idx], - 0 /* flow_control_enable */, -@@ -305,6 +313,9 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) - unsigned i; - dma_addr_t dma_addr; - -+ if (!rq->in_use) -+ return 0; -+ - dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index, - rq->ring.desc_count); - -@@ -316,20 +327,20 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) - return -ENOMEM; - } - -- dma_addr = (dma_addr_t)(mb->buf_physaddr -- + RTE_PKTMBUF_HEADROOM); -- -- rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, -- mb->buf_len - RTE_PKTMBUF_HEADROOM); -+ dma_addr = (dma_addr_t)(mb->buf_physaddr + RTE_PKTMBUF_HEADROOM); -+ rq_enet_desc_enc(rqd, dma_addr, -+ (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP -+ : RQ_ENET_TYPE_NOT_SOP), -+ mb->buf_len - RTE_PKTMBUF_HEADROOM); - rq->mbuf_ring[i] = mb; - } - - /* make sure all prior writes are complete before doing the PIO write */ - rte_rmb(); - -- /* Post all but the last 2 cache lines' worth of descriptors */ -- rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE -- / sizeof(struct rq_enet_desc)); -+ /* Post all but the last buffer to VIC. */ -+ rq->posted_index = rq->ring.desc_count - 1; -+ - rq->rx_nb_hold = 0; - - dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n", -@@ -337,6 +348,8 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) - iowrite32(rq->posted_index, &rq->ctrl->posted_index); - rte_rmb(); - -+// printf("posted %d buffers to %s rq\n", rq->ring.desc_count, -+// rq->is_sop ? "sop" : "data"); - return 0; - - } -@@ -398,17 +411,25 @@ int enic_enable(struct enic *enic) - "Flow director feature will not work\n"); - - for (index = 0; index < enic->rq_count; index++) { -- err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]); -+ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[enic_sop_rq(enic, index)]); - if (err) { -- dev_err(enic, "Failed to alloc RX queue mbufs\n"); -+ dev_err(enic, "Failed to alloc sop RX queue mbufs\n"); -+ return err; -+ } -+ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[enic_data_rq(enic, index)]); -+ if (err) { -+ /* release the previously allocated mbufs for the sop rq */ -+ enic_rxmbuf_queue_release(enic, &enic->rq[enic_sop_rq(enic, index)]); -+ -+ dev_err(enic, "Failed to alloc data RX queue mbufs\n"); - return err; - } - } - - for (index = 0; index < enic->wq_count; index++) -- vnic_wq_enable(&enic->wq[index]); -+ enic_start_wq(enic, index); - for (index = 0; index < enic->rq_count; index++) -- vnic_rq_enable(&enic->rq[index]); -+ enic_start_rq(enic, index); - - vnic_dev_enable_wait(enic->vdev); - -@@ -440,14 +461,26 @@ int enic_alloc_intr_resources(struct enic *enic) - - void enic_free_rq(void *rxq) - { -- struct vnic_rq *rq = (struct vnic_rq *)rxq; -- struct enic *enic = vnic_dev_priv(rq->vdev); -+ struct vnic_rq *rq_sop = (struct vnic_rq *)rxq; -+ struct enic *enic = vnic_dev_priv(rq_sop->vdev); -+ struct vnic_rq *rq_data = &enic->rq[rq_sop->data_queue_idx]; - -- enic_rxmbuf_queue_release(enic, rq); -- rte_free(rq->mbuf_ring); -- rq->mbuf_ring = NULL; -- vnic_rq_free(rq); -- vnic_cq_free(&enic->cq[rq->index]); -+ enic_rxmbuf_queue_release(enic, rq_sop); -+ if (rq_data->in_use) -+ enic_rxmbuf_queue_release(enic, rq_data); -+ -+ rte_free(rq_sop->mbuf_ring); -+ if (rq_data->in_use) -+ rte_free(rq_data->mbuf_ring); -+ -+ rq_sop->mbuf_ring = NULL; -+ rq_data->mbuf_ring = NULL; -+ -+ vnic_rq_free(rq_sop); -+ if (rq_data->in_use) -+ vnic_rq_free(rq_data); -+ -+ vnic_cq_free(&enic->cq[rq_sop->index]); - } - - void enic_start_wq(struct enic *enic, uint16_t queue_idx) -@@ -462,12 +495,32 @@ int enic_stop_wq(struct enic *enic, uint16_t queue_idx) - - void enic_start_rq(struct enic *enic, uint16_t queue_idx) - { -- vnic_rq_enable(&enic->rq[queue_idx]); -+ struct vnic_rq *rq_sop = &enic->rq[enic_sop_rq(enic, queue_idx)]; -+ struct vnic_rq *rq_data = &enic->rq[rq_sop->data_queue_idx]; -+ -+ if (rq_data->in_use) -+ vnic_rq_enable(rq_data); -+ rte_mb(); -+ vnic_rq_enable(rq_sop); -+ - } - - int enic_stop_rq(struct enic *enic, uint16_t queue_idx) - { -- return vnic_rq_disable(&enic->rq[queue_idx]); -+ int ret1 = 0, ret2 = 0; -+ -+ struct vnic_rq *rq_sop = &enic->rq[enic_sop_rq(enic, queue_idx)]; -+ struct vnic_rq *rq_data = &enic->rq[rq_sop->data_queue_idx]; -+ -+ ret2 = vnic_rq_disable(rq_sop); -+ rte_mb(); -+ if (rq_data->in_use) -+ ret1 = vnic_rq_disable(rq_data); -+ -+ if (ret2) -+ return ret2; -+ else -+ return ret1; - } - - int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, -@@ -475,53 +528,128 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, - uint16_t nb_desc) - { - int rc; -- struct vnic_rq *rq = &enic->rq[queue_idx]; -- -- rq->socket_id = socket_id; -- rq->mp = mp; -+ uint16_t sop_queue_idx = enic_sop_rq(enic, queue_idx); -+ uint16_t data_queue_idx = enic_data_rq(enic, queue_idx); -+ struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx]; -+ struct vnic_rq *rq_data = &enic->rq[data_queue_idx]; -+ unsigned int mbuf_size, mbufs_per_pkt; -+ unsigned int nb_sop_desc, nb_data_desc; -+ uint16_t min_sop, max_sop, min_data, max_data; -+ -+ rq_sop->is_sop = 1; -+ rq_sop->data_queue_idx = data_queue_idx; -+ rq_data->is_sop = 0; -+ rq_data->data_queue_idx = 0; -+ rq_sop->socket_id = socket_id; -+ rq_sop->mp = mp; -+ rq_data->socket_id = socket_id; -+ rq_data->mp = mp; -+ rq_sop->in_use = 1; -+ -+ mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM); -+ -+ /* ceil(mtu/mbuf_size) */ -+ mbufs_per_pkt = (enic->config.mtu + (mbuf_size - 1)) / mbuf_size; -+ -+ if (mbufs_per_pkt > 1) -+ rq_data->in_use = 1; -+ else -+ rq_data->in_use = 0; -+ -+ /* number of descriptors have to be a multiple of 32 */ -+ nb_sop_desc = (nb_desc / mbufs_per_pkt) & ~0x1F; -+ nb_data_desc = (nb_desc - nb_sop_desc) & ~0x1F; -+ -+ rq_sop->max_mbufs_per_pkt = mbufs_per_pkt; -+ rq_data->max_mbufs_per_pkt = mbufs_per_pkt; -+ -+ //printf("mtu = %u, mbuf_size = %u, mbuf_per_pkt = %u\n", -+ // enic->config.mtu, mbuf_size, mbufs_per_pkt); -+ -+ if (mbufs_per_pkt > 1) { -+ min_sop = 64; -+ max_sop = ((enic->config.rq_desc_count / (mbufs_per_pkt - 1)) & ~0x1F); -+ min_data = min_sop * (mbufs_per_pkt - 1); -+ max_data = enic->config.rq_desc_count; -+ } else { -+ min_sop = 64; -+ max_sop = enic->config.rq_desc_count; -+ min_data = 0; -+ max_data = 0; -+ } - -- if (nb_desc) { -- if (nb_desc > enic->config.rq_desc_count) { -- dev_warning(enic, -- "RQ %d - number of rx desc in cmd line (%d)"\ -- "is greater than that in the UCSM/CIMC adapter"\ -- "policy. Applying the value in the adapter "\ -- "policy (%d).\n", -- queue_idx, nb_desc, enic->config.rq_desc_count); -- nb_desc = enic->config.rq_desc_count; -- } -- dev_info(enic, "RX Queues - effective number of descs:%d\n", -- nb_desc); -+ if (nb_desc < (min_sop + min_data)) { -+ dev_warning(enic, -+ "Number of rx descs too low, adjusting to minimum\n"); -+ nb_sop_desc = min_sop; -+ nb_data_desc = min_data; -+ } else if (nb_desc > (max_sop + max_data)){ -+ dev_warning(enic, -+ "Number of rx_descs too high, adjusting to maximum\n"); -+ nb_sop_desc = max_sop; -+ nb_data_desc = max_data; - } -+ dev_info(enic, "For mtu %d and mbuf size %d valid rx descriptor range is %d to %d\n", -+ enic->config.mtu, mbuf_size, min_sop + min_data, max_sop + max_data); - -- /* Allocate queue resources */ -- rc = vnic_rq_alloc(enic->vdev, rq, queue_idx, -- nb_desc, sizeof(struct rq_enet_desc)); -+ dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n", -+ nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc); -+ -+ /* Allocate sop queue resources */ -+ rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx, -+ nb_sop_desc, sizeof(struct rq_enet_desc)); - if (rc) { -- dev_err(enic, "error in allocation of rq\n"); -+ dev_err(enic, "error in allocation of sop rq\n"); - goto err_exit; - } -- -+ nb_sop_desc = rq_sop->ring.desc_count; -+ -+ if (rq_data->in_use) { -+ /* Allocate data queue resources */ -+ rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx, -+ nb_data_desc, -+ sizeof(struct rq_enet_desc)); -+ if (rc) { -+ dev_err(enic, "error in allocation of data rq\n"); -+ goto err_free_rq_sop; -+ } -+ nb_data_desc = rq_data->ring.desc_count; -+ } - rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, -- socket_id, nb_desc, -- sizeof(struct cq_enet_rq_desc)); -+ socket_id, nb_sop_desc + nb_data_desc, -+ sizeof(struct cq_enet_rq_desc)); - if (rc) { - dev_err(enic, "error in allocation of cq for rq\n"); -- goto err_free_rq_exit; -+ goto err_free_rq_data; - } - -- /* Allocate the mbuf ring */ -- rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", -- sizeof(struct rte_mbuf *) * nb_desc, -- RTE_CACHE_LINE_SIZE, rq->socket_id); -+ /* Allocate the mbuf rings */ -+ rq_sop->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", -+ sizeof(struct rte_mbuf *) * nb_sop_desc, -+ RTE_CACHE_LINE_SIZE, rq_sop->socket_id); -+ if (rq_sop->mbuf_ring == NULL) -+ goto err_free_cq; -+ -+ if (rq_data->in_use) { -+ rq_data->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", -+ sizeof(struct rte_mbuf *) * nb_data_desc, -+ RTE_CACHE_LINE_SIZE, rq_sop->socket_id); -+ if (rq_data->mbuf_ring == NULL) -+ goto err_free_sop_mbuf; -+ } - -- if (rq->mbuf_ring != NULL) -- return 0; -+ return 0; - -+err_free_sop_mbuf: -+ rte_free(rq_sop->mbuf_ring); -+err_free_cq: - /* cleanup on error */ - vnic_cq_free(&enic->cq[queue_idx]); --err_free_rq_exit: -- vnic_rq_free(rq); -+err_free_rq_data: -+ if (rq_data->in_use) -+ vnic_rq_free(rq_data); -+err_free_rq_sop: -+ vnic_rq_free(rq_sop); - err_exit: - return -ENOMEM; - } -diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c -index 174486b..463b954 100644 ---- a/drivers/net/enic/enic_rxtx.c -+++ b/drivers/net/enic/enic_rxtx.c -@@ -242,22 +242,27 @@ uint16_t - enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts) - { -- struct vnic_rq *rq = rx_queue; -- struct enic *enic = vnic_dev_priv(rq->vdev); -- unsigned int rx_id; -+ struct vnic_rq *sop_rq = rx_queue; -+ struct vnic_rq *data_rq; -+ struct vnic_rq *rq; -+ struct enic *enic = vnic_dev_priv(sop_rq->vdev); -+ uint16_t cq_idx; -+ uint16_t rq_idx; -+ uint16_t rq_num; - struct rte_mbuf *nmb, *rxmb; - uint16_t nb_rx = 0; -- uint16_t nb_hold; - struct vnic_cq *cq; - volatile struct cq_desc *cqd_ptr; - uint8_t color; -- uint16_t nb_err = 0; -+ uint16_t seg_length; -+ struct rte_mbuf *first_seg = sop_rq->pkt_first_seg; -+ struct rte_mbuf *last_seg = sop_rq->pkt_last_seg; - -- cq = &enic->cq[enic_cq_rq(enic, rq->index)]; -- rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ -- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -+ cq = &enic->cq[enic_cq_rq(enic, sop_rq->index)]; -+ cq_idx = cq->to_clean; /* index of cqd, rqd, mbuf_table */ -+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + cq_idx; - -- nb_hold = rq->rx_nb_hold; /* mbufs held by software */ -+ data_rq = &enic->rq[sop_rq->data_queue_idx]; - - while (nb_rx < nb_pkts) { - volatile struct rq_enet_desc *rqd_ptr; -@@ -265,6 +270,7 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - struct cq_desc cqd; - uint64_t ol_err_flags; - uint8_t packet_error; -+ uint16_t ciflags; - - /* Check for pkts available */ - color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) -@@ -272,9 +278,13 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - if (color == cq->last_color) - break; - -- /* Get the cq descriptor and rq pointer */ -+ /* Get the cq descriptor and extract rq info from it */ - cqd = *cqd_ptr; -- rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; -+ rq_num = cqd.q_number & CQ_DESC_Q_NUM_MASK; -+ rq_idx = cqd.completed_index & CQ_DESC_COMP_NDX_MASK; -+ -+ rq = &enic->rq[rq_num]; -+ rqd_ptr = ((struct rq_enet_desc *)rq->ring.descs) + rq_idx; - - /* allocate a new mbuf */ - nmb = rte_rxmbuf_alloc(rq->mp); -@@ -287,67 +297,106 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, - packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); - - /* Get the mbuf to return and replace with one just allocated */ -- rxmb = rq->mbuf_ring[rx_id]; -- rq->mbuf_ring[rx_id] = nmb; -+ rxmb = rq->mbuf_ring[rq_idx]; -+ rq->mbuf_ring[rq_idx] = nmb; - - /* Increment cqd, rqd, mbuf_table index */ -- rx_id++; -- if (unlikely(rx_id == rq->ring.desc_count)) { -- rx_id = 0; -+ cq_idx++; -+ if (unlikely(cq_idx == cq->ring.desc_count)) { -+ cq_idx = 0; - cq->last_color = cq->last_color ? 0 : 1; - } - - /* Prefetch next mbuf & desc while processing current one */ -- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; -+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + cq_idx; - rte_enic_prefetch(cqd_ptr); -- rte_enic_prefetch(rq->mbuf_ring[rx_id]); -- rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) -- + rx_id); -+// rte_enic_prefetch(rq->mbuf_ring[rx_id]); -+// rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) -+// + rx_id); -+ -+ ciflags = enic_cq_rx_desc_ciflags((struct cq_enet_rq_desc *) &cqd); - - /* Push descriptor for newly allocated mbuf */ -- dma_addr = (dma_addr_t)(nmb->buf_physaddr -- + RTE_PKTMBUF_HEADROOM); -- rqd_ptr->address = rte_cpu_to_le_64(dma_addr); -- rqd_ptr->length_type = cpu_to_le16(nmb->buf_len -- - RTE_PKTMBUF_HEADROOM); -+ -+ dma_addr = (dma_addr_t)(nmb->buf_physaddr + RTE_PKTMBUF_HEADROOM); -+ rq_enet_desc_enc(rqd_ptr, dma_addr, -+ (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP -+ : RQ_ENET_TYPE_NOT_SOP), -+ nmb->buf_len - RTE_PKTMBUF_HEADROOM); - - /* Fill in the rest of the mbuf */ -- rxmb->data_off = RTE_PKTMBUF_HEADROOM; -- rxmb->nb_segs = 1; -+ seg_length = enic_cq_rx_desc_n_bytes(&cqd); -+ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); -+ enic_cq_rx_to_pkt_flags(&cqd, rxmb); -+ if (rq->is_sop) { -+ first_seg = rxmb; -+ first_seg->nb_segs = 1; -+ first_seg->pkt_len = seg_length; -+ } else { -+ first_seg->pkt_len = (uint16_t)(first_seg->pkt_len -+ + seg_length); -+ first_seg->nb_segs++; -+ last_seg->next = rxmb; -+ } -+ - rxmb->next = NULL; - rxmb->port = enic->port_id; -- if (!packet_error) { -- rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); -- rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); -- enic_cq_rx_to_pkt_flags(&cqd, rxmb); -- } else { -- rte_pktmbuf_free(rxmb); -+ rxmb->data_len = seg_length; -+ -+ rq->rx_nb_hold++; -+ -+ if (!(enic_cq_rx_desc_eop(ciflags))) { -+ last_seg = rxmb; -+ continue; -+ } -+ -+ if (unlikely(packet_error)) { -+ rte_pktmbuf_free(first_seg); - rte_atomic64_inc(&enic->soft_stats.rx_packet_errors); -- nb_err++; -+ - continue; - } -- rxmb->data_len = rxmb->pkt_len; -+ -+ -+// printf("EOP: final packet length is %d\n", first_seg->pkt_len); -+// rte_pktmbuf_dump(stdout, first_seg, 64); - - /* prefetch mbuf data for caller */ -- rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, -+ rte_packet_prefetch(RTE_PTR_ADD(first_seg->buf_addr, - RTE_PKTMBUF_HEADROOM)); - - /* store the mbuf address into the next entry of the array */ -- rx_pkts[nb_rx++] = rxmb; -+ rx_pkts[nb_rx++] = first_seg; - } - -- nb_hold += nb_rx + nb_err; -- cq->to_clean = rx_id; -+ sop_rq->pkt_first_seg = first_seg; -+ sop_rq->pkt_last_seg = last_seg; -+ -+ cq->to_clean = cq_idx; -+ -+ if ((sop_rq->rx_nb_hold + data_rq->rx_nb_hold) > sop_rq->rx_free_thresh) { -+ if (data_rq->in_use) { -+ data_rq->posted_index = enic_ring_add(data_rq->ring.desc_count, -+ data_rq->posted_index, -+ data_rq->rx_nb_hold); -+ //printf("Processed %d data descs. Posted index now %d\n", -+ // data_rq->rx_nb_hold, data_rq->posted_index); -+ data_rq->rx_nb_hold = 0; -+ } -+ sop_rq->posted_index = enic_ring_add(sop_rq->ring.desc_count, -+ sop_rq->posted_index, -+ sop_rq->rx_nb_hold); -+ //printf("Processed %d sop descs. Posted index now %d\n", -+ // sop_rq->rx_nb_hold, sop_rq->posted_index); -+ sop_rq->rx_nb_hold = 0; - -- if (nb_hold > rq->rx_free_thresh) { -- rq->posted_index = enic_ring_add(rq->ring.desc_count, -- rq->posted_index, nb_hold); -- nb_hold = 0; - rte_mb(); -- iowrite32(rq->posted_index, &rq->ctrl->posted_index); -+ if (data_rq->in_use) -+ iowrite32(data_rq->posted_index, &data_rq->ctrl->posted_index); -+ rte_compiler_barrier(); -+ iowrite32(sop_rq->posted_index, &sop_rq->ctrl->posted_index); - } - -- rq->rx_nb_hold = nb_hold; - - return nb_rx; - } --- -2.7.4 - |