diff options
Diffstat (limited to 'dpdk/dpdk-16.04_patches/0016-ENIC-scatter-RX.patch')
| -rw-r--r-- | dpdk/dpdk-16.04_patches/0016-ENIC-scatter-RX.patch | 672 |
1 files changed, 672 insertions, 0 deletions
diff --git a/dpdk/dpdk-16.04_patches/0016-ENIC-scatter-RX.patch b/dpdk/dpdk-16.04_patches/0016-ENIC-scatter-RX.patch new file mode 100644 index 00000000000..e0daab060c7 --- /dev/null +++ b/dpdk/dpdk-16.04_patches/0016-ENIC-scatter-RX.patch @@ -0,0 +1,672 @@ +From f03d5a02fc2b3cc24bf059a273ea1473cdb9993b Mon Sep 17 00:00:00 2001 +From: John Lo <loj@cisco.com> +Date: Tue, 7 Jun 2016 12:40:07 +0200 +Subject: [PATCH 16/17] ENIC scatter RX + +--- + drivers/net/enic/base/rq_enet_desc.h | 2 +- + drivers/net/enic/base/vnic_rq.c | 12 +- + drivers/net/enic/base/vnic_rq.h | 18 ++- + drivers/net/enic/enic.h | 10 ++ + drivers/net/enic/enic_main.c | 236 +++++++++++++++++++++++++++-------- + drivers/net/enic/enic_rxtx.c | 139 ++++++++++++++------- + 6 files changed, 313 insertions(+), 104 deletions(-) + +diff --git a/drivers/net/enic/base/rq_enet_desc.h b/drivers/net/enic/base/rq_enet_desc.h +index 7292d9d..13e24b4 100644 +--- a/drivers/net/enic/base/rq_enet_desc.h ++++ b/drivers/net/enic/base/rq_enet_desc.h +@@ -55,7 +55,7 @@ enum rq_enet_type_types { + #define RQ_ENET_TYPE_BITS 2 + #define RQ_ENET_TYPE_MASK ((1 << RQ_ENET_TYPE_BITS) - 1) + +-static inline void rq_enet_desc_enc(struct rq_enet_desc *desc, ++static inline void rq_enet_desc_enc(volatile struct rq_enet_desc *desc, + u64 address, u8 type, u16 length) + { + desc->address = cpu_to_le64(address); +diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c +index cb62c5e..d97f93e 100644 +--- a/drivers/net/enic/base/vnic_rq.c ++++ b/drivers/net/enic/base/vnic_rq.c +@@ -84,11 +84,16 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, + iowrite32(cq_index, &rq->ctrl->cq_index); + iowrite32(error_interrupt_enable, &rq->ctrl->error_interrupt_enable); + iowrite32(error_interrupt_offset, &rq->ctrl->error_interrupt_offset); +- iowrite32(0, &rq->ctrl->dropped_packet_count); + iowrite32(0, &rq->ctrl->error_status); + iowrite32(fetch_index, &rq->ctrl->fetch_index); + iowrite32(posted_index, &rq->ctrl->posted_index); +- ++ if (rq->is_sop) { ++// printf("Writing 0x%x to %s rq\n", ++// ((rq->is_sop << 10) | rq->data_queue_idx), ++// rq->is_sop ? "sop":"data"); ++ iowrite32(((rq->is_sop << 10) | rq->data_queue_idx), ++ &rq->ctrl->data_ring); ++ } + } + + void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, +@@ -96,6 +101,7 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, + unsigned int error_interrupt_offset) + { + u32 fetch_index = 0; ++ + /* Use current fetch_index as the ring starting point */ + fetch_index = ioread32(&rq->ctrl->fetch_index); + +@@ -110,6 +116,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, + error_interrupt_offset); + rq->rxst_idx = 0; + rq->tot_pkts = 0; ++ rq->pkt_first_seg = NULL; ++ rq->pkt_last_seg = NULL; + } + + void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error) +diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h +index 424415c..d1e2f52 100644 +--- a/drivers/net/enic/base/vnic_rq.h ++++ b/drivers/net/enic/base/vnic_rq.h +@@ -60,10 +60,18 @@ struct vnic_rq_ctrl { + u32 pad7; + u32 error_status; /* 0x48 */ + u32 pad8; +- u32 dropped_packet_count; /* 0x50 */ ++ u32 tcp_sn; /* 0x50 */ + u32 pad9; +- u32 dropped_packet_count_rc; /* 0x58 */ ++ u32 unused; /* 0x58 */ + u32 pad10; ++ u32 dca_select; /* 0x60 */ ++ u32 pad11; ++ u32 dca_value; /* 0x68 */ ++ u32 pad12; ++ u32 data_ring; /* 0x70 */ ++ u32 pad13; ++ u32 header_split; /* 0x78 */ ++ u32 pad14; + }; + + struct vnic_rq { +@@ -82,6 +90,12 @@ struct vnic_rq { + struct rte_mempool *mp; + uint16_t rxst_idx; + uint32_t tot_pkts; ++ uint16_t data_queue_idx; ++ uint8_t is_sop; ++ uint8_t in_use; ++ struct rte_mbuf *pkt_first_seg; ++ struct rte_mbuf *pkt_last_seg; ++ unsigned int max_mbufs_per_pkt; + }; + + static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq) +diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h +index 7c1b5c9..d2de6ee 100644 +--- a/drivers/net/enic/enic.h ++++ b/drivers/net/enic/enic.h +@@ -142,6 +142,16 @@ struct enic { + struct enic_soft_stats soft_stats; + }; + ++static inline unsigned int enic_sop_rq(__rte_unused struct enic *enic, unsigned int rq) ++{ ++ return rq * 2; ++} ++ ++static inline unsigned int enic_data_rq(__rte_unused struct enic *enic, unsigned int rq) ++{ ++ return rq * 2 + 1; ++} ++ + static inline unsigned int enic_cq_rq(__rte_unused struct enic *enic, unsigned int rq) + { + return rq; +diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c +index a00565a..be17707 100644 +--- a/drivers/net/enic/enic_main.c ++++ b/drivers/net/enic/enic_main.c +@@ -247,15 +247,23 @@ void enic_init_vnic_resources(struct enic *enic) + unsigned int error_interrupt_offset = 0; + unsigned int index = 0; + unsigned int cq_idx; ++ struct vnic_rq *data_rq; + + vnic_dev_stats_clear(enic->vdev); + + for (index = 0; index < enic->rq_count; index++) { +- vnic_rq_init(&enic->rq[index], ++ vnic_rq_init(&enic->rq[enic_sop_rq(enic, index)], + enic_cq_rq(enic, index), + error_interrupt_enable, + error_interrupt_offset); + ++ data_rq = &enic->rq[enic_data_rq(enic, index)]; ++ if (data_rq->in_use) ++ vnic_rq_init(data_rq, ++ enic_cq_rq(enic, index), ++ error_interrupt_enable, ++ error_interrupt_offset); ++ + cq_idx = enic_cq_rq(enic, index); + vnic_cq_init(&enic->cq[cq_idx], + 0 /* flow_control_enable */, +@@ -305,6 +313,9 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) + unsigned i; + dma_addr_t dma_addr; + ++ if (!rq->in_use) ++ return 0; ++ + dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index, + rq->ring.desc_count); + +@@ -316,20 +327,20 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) + return -ENOMEM; + } + +- dma_addr = (dma_addr_t)(mb->buf_physaddr +- + RTE_PKTMBUF_HEADROOM); +- +- rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, +- mb->buf_len - RTE_PKTMBUF_HEADROOM); ++ dma_addr = (dma_addr_t)(mb->buf_physaddr + RTE_PKTMBUF_HEADROOM); ++ rq_enet_desc_enc(rqd, dma_addr, ++ (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP ++ : RQ_ENET_TYPE_NOT_SOP), ++ mb->buf_len - RTE_PKTMBUF_HEADROOM); + rq->mbuf_ring[i] = mb; + } + + /* make sure all prior writes are complete before doing the PIO write */ + rte_rmb(); + +- /* Post all but the last 2 cache lines' worth of descriptors */ +- rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE +- / sizeof(struct rq_enet_desc)); ++ /* Post all but the last buffer to VIC. */ ++ rq->posted_index = rq->ring.desc_count - 1; ++ + rq->rx_nb_hold = 0; + + dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n", +@@ -337,6 +348,8 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) + iowrite32(rq->posted_index, &rq->ctrl->posted_index); + rte_rmb(); + ++// printf("posted %d buffers to %s rq\n", rq->ring.desc_count, ++// rq->is_sop ? "sop" : "data"); + return 0; + + } +@@ -398,17 +411,25 @@ int enic_enable(struct enic *enic) + "Flow director feature will not work\n"); + + for (index = 0; index < enic->rq_count; index++) { +- err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]); ++ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[enic_sop_rq(enic, index)]); + if (err) { +- dev_err(enic, "Failed to alloc RX queue mbufs\n"); ++ dev_err(enic, "Failed to alloc sop RX queue mbufs\n"); ++ return err; ++ } ++ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[enic_data_rq(enic, index)]); ++ if (err) { ++ /* release the previously allocated mbufs for the sop rq */ ++ enic_rxmbuf_queue_release(enic, &enic->rq[enic_sop_rq(enic, index)]); ++ ++ dev_err(enic, "Failed to alloc data RX queue mbufs\n"); + return err; + } + } + + for (index = 0; index < enic->wq_count; index++) +- vnic_wq_enable(&enic->wq[index]); ++ enic_start_wq(enic, index); + for (index = 0; index < enic->rq_count; index++) +- vnic_rq_enable(&enic->rq[index]); ++ enic_start_rq(enic, index); + + vnic_dev_enable_wait(enic->vdev); + +@@ -440,14 +461,26 @@ int enic_alloc_intr_resources(struct enic *enic) + + void enic_free_rq(void *rxq) + { +- struct vnic_rq *rq = (struct vnic_rq *)rxq; +- struct enic *enic = vnic_dev_priv(rq->vdev); ++ struct vnic_rq *rq_sop = (struct vnic_rq *)rxq; ++ struct enic *enic = vnic_dev_priv(rq_sop->vdev); ++ struct vnic_rq *rq_data = &enic->rq[rq_sop->data_queue_idx]; + +- enic_rxmbuf_queue_release(enic, rq); +- rte_free(rq->mbuf_ring); +- rq->mbuf_ring = NULL; +- vnic_rq_free(rq); +- vnic_cq_free(&enic->cq[rq->index]); ++ enic_rxmbuf_queue_release(enic, rq_sop); ++ if (rq_data->in_use) ++ enic_rxmbuf_queue_release(enic, rq_data); ++ ++ rte_free(rq_sop->mbuf_ring); ++ if (rq_data->in_use) ++ rte_free(rq_data->mbuf_ring); ++ ++ rq_sop->mbuf_ring = NULL; ++ rq_data->mbuf_ring = NULL; ++ ++ vnic_rq_free(rq_sop); ++ if (rq_data->in_use) ++ vnic_rq_free(rq_data); ++ ++ vnic_cq_free(&enic->cq[rq_sop->index]); + } + + void enic_start_wq(struct enic *enic, uint16_t queue_idx) +@@ -462,12 +495,32 @@ int enic_stop_wq(struct enic *enic, uint16_t queue_idx) + + void enic_start_rq(struct enic *enic, uint16_t queue_idx) + { +- vnic_rq_enable(&enic->rq[queue_idx]); ++ struct vnic_rq *rq_sop = &enic->rq[enic_sop_rq(enic, queue_idx)]; ++ struct vnic_rq *rq_data = &enic->rq[rq_sop->data_queue_idx]; ++ ++ if (rq_data->in_use) ++ vnic_rq_enable(rq_data); ++ rte_mb(); ++ vnic_rq_enable(rq_sop); ++ + } + + int enic_stop_rq(struct enic *enic, uint16_t queue_idx) + { +- return vnic_rq_disable(&enic->rq[queue_idx]); ++ int ret1 = 0, ret2 = 0; ++ ++ struct vnic_rq *rq_sop = &enic->rq[enic_sop_rq(enic, queue_idx)]; ++ struct vnic_rq *rq_data = &enic->rq[rq_sop->data_queue_idx]; ++ ++ ret2 = vnic_rq_disable(rq_sop); ++ rte_mb(); ++ if (rq_data->in_use) ++ ret1 = vnic_rq_disable(rq_data); ++ ++ if (ret2) ++ return ret2; ++ else ++ return ret1; + } + + int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, +@@ -475,53 +528,128 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, + uint16_t nb_desc) + { + int rc; +- struct vnic_rq *rq = &enic->rq[queue_idx]; +- +- rq->socket_id = socket_id; +- rq->mp = mp; ++ uint16_t sop_queue_idx = enic_sop_rq(enic, queue_idx); ++ uint16_t data_queue_idx = enic_data_rq(enic, queue_idx); ++ struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx]; ++ struct vnic_rq *rq_data = &enic->rq[data_queue_idx]; ++ unsigned int mbuf_size, mbufs_per_pkt; ++ unsigned int nb_sop_desc, nb_data_desc; ++ uint16_t min_sop, max_sop, min_data, max_data; ++ ++ rq_sop->is_sop = 1; ++ rq_sop->data_queue_idx = data_queue_idx; ++ rq_data->is_sop = 0; ++ rq_data->data_queue_idx = 0; ++ rq_sop->socket_id = socket_id; ++ rq_sop->mp = mp; ++ rq_data->socket_id = socket_id; ++ rq_data->mp = mp; ++ rq_sop->in_use = 1; ++ ++ mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM); ++ ++ /* ceil(mtu/mbuf_size) */ ++ mbufs_per_pkt = (enic->config.mtu + (mbuf_size - 1)) / mbuf_size; ++ ++ if (mbufs_per_pkt > 1) ++ rq_data->in_use = 1; ++ else ++ rq_data->in_use = 0; ++ ++ /* number of descriptors have to be a multiple of 32 */ ++ nb_sop_desc = (nb_desc / mbufs_per_pkt) & ~0x1F; ++ nb_data_desc = (nb_desc - nb_sop_desc) & ~0x1F; ++ ++ rq_sop->max_mbufs_per_pkt = mbufs_per_pkt; ++ rq_data->max_mbufs_per_pkt = mbufs_per_pkt; ++ ++ //printf("mtu = %u, mbuf_size = %u, mbuf_per_pkt = %u\n", ++ // enic->config.mtu, mbuf_size, mbufs_per_pkt); ++ ++ if (mbufs_per_pkt > 1) { ++ min_sop = 64; ++ max_sop = ((enic->config.rq_desc_count / (mbufs_per_pkt - 1)) & ~0x1F); ++ min_data = min_sop * (mbufs_per_pkt - 1); ++ max_data = enic->config.rq_desc_count; ++ } else { ++ min_sop = 64; ++ max_sop = enic->config.rq_desc_count; ++ min_data = 0; ++ max_data = 0; ++ } + +- if (nb_desc) { +- if (nb_desc > enic->config.rq_desc_count) { +- dev_warning(enic, +- "RQ %d - number of rx desc in cmd line (%d)"\ +- "is greater than that in the UCSM/CIMC adapter"\ +- "policy. Applying the value in the adapter "\ +- "policy (%d).\n", +- queue_idx, nb_desc, enic->config.rq_desc_count); +- nb_desc = enic->config.rq_desc_count; +- } +- dev_info(enic, "RX Queues - effective number of descs:%d\n", +- nb_desc); ++ if (nb_desc < (min_sop + min_data)) { ++ dev_warning(enic, ++ "Number of rx descs too low, adjusting to minimum\n"); ++ nb_sop_desc = min_sop; ++ nb_data_desc = min_data; ++ } else if (nb_desc > (max_sop + max_data)){ ++ dev_warning(enic, ++ "Number of rx_descs too high, adjusting to maximum\n"); ++ nb_sop_desc = max_sop; ++ nb_data_desc = max_data; + } ++ dev_info(enic, "For mtu %d and mbuf size %d valid rx descriptor range is %d to %d\n", ++ enic->config.mtu, mbuf_size, min_sop + min_data, max_sop + max_data); + +- /* Allocate queue resources */ +- rc = vnic_rq_alloc(enic->vdev, rq, queue_idx, +- nb_desc, sizeof(struct rq_enet_desc)); ++ dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n", ++ nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc); ++ ++ /* Allocate sop queue resources */ ++ rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx, ++ nb_sop_desc, sizeof(struct rq_enet_desc)); + if (rc) { +- dev_err(enic, "error in allocation of rq\n"); ++ dev_err(enic, "error in allocation of sop rq\n"); + goto err_exit; + } +- ++ nb_sop_desc = rq_sop->ring.desc_count; ++ ++ if (rq_data->in_use) { ++ /* Allocate data queue resources */ ++ rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx, ++ nb_data_desc, ++ sizeof(struct rq_enet_desc)); ++ if (rc) { ++ dev_err(enic, "error in allocation of data rq\n"); ++ goto err_free_rq_sop; ++ } ++ nb_data_desc = rq_data->ring.desc_count; ++ } + rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, +- socket_id, nb_desc, +- sizeof(struct cq_enet_rq_desc)); ++ socket_id, nb_sop_desc + nb_data_desc, ++ sizeof(struct cq_enet_rq_desc)); + if (rc) { + dev_err(enic, "error in allocation of cq for rq\n"); +- goto err_free_rq_exit; ++ goto err_free_rq_data; + } + +- /* Allocate the mbuf ring */ +- rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", +- sizeof(struct rte_mbuf *) * nb_desc, +- RTE_CACHE_LINE_SIZE, rq->socket_id); ++ /* Allocate the mbuf rings */ ++ rq_sop->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", ++ sizeof(struct rte_mbuf *) * nb_sop_desc, ++ RTE_CACHE_LINE_SIZE, rq_sop->socket_id); ++ if (rq_sop->mbuf_ring == NULL) ++ goto err_free_cq; ++ ++ if (rq_data->in_use) { ++ rq_data->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", ++ sizeof(struct rte_mbuf *) * nb_data_desc, ++ RTE_CACHE_LINE_SIZE, rq_sop->socket_id); ++ if (rq_data->mbuf_ring == NULL) ++ goto err_free_sop_mbuf; ++ } + +- if (rq->mbuf_ring != NULL) +- return 0; ++ return 0; + ++err_free_sop_mbuf: ++ rte_free(rq_sop->mbuf_ring); ++err_free_cq: + /* cleanup on error */ + vnic_cq_free(&enic->cq[queue_idx]); +-err_free_rq_exit: +- vnic_rq_free(rq); ++err_free_rq_data: ++ if (rq_data->in_use) ++ vnic_rq_free(rq_data); ++err_free_rq_sop: ++ vnic_rq_free(rq_sop); + err_exit: + return -ENOMEM; + } +diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c +index 174486b..463b954 100644 +--- a/drivers/net/enic/enic_rxtx.c ++++ b/drivers/net/enic/enic_rxtx.c +@@ -242,22 +242,27 @@ uint16_t + enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) + { +- struct vnic_rq *rq = rx_queue; +- struct enic *enic = vnic_dev_priv(rq->vdev); +- unsigned int rx_id; ++ struct vnic_rq *sop_rq = rx_queue; ++ struct vnic_rq *data_rq; ++ struct vnic_rq *rq; ++ struct enic *enic = vnic_dev_priv(sop_rq->vdev); ++ uint16_t cq_idx; ++ uint16_t rq_idx; ++ uint16_t rq_num; + struct rte_mbuf *nmb, *rxmb; + uint16_t nb_rx = 0; +- uint16_t nb_hold; + struct vnic_cq *cq; + volatile struct cq_desc *cqd_ptr; + uint8_t color; +- uint16_t nb_err = 0; ++ uint16_t seg_length; ++ struct rte_mbuf *first_seg = sop_rq->pkt_first_seg; ++ struct rte_mbuf *last_seg = sop_rq->pkt_last_seg; + +- cq = &enic->cq[enic_cq_rq(enic, rq->index)]; +- rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ +- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; ++ cq = &enic->cq[enic_cq_rq(enic, sop_rq->index)]; ++ cq_idx = cq->to_clean; /* index of cqd, rqd, mbuf_table */ ++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + cq_idx; + +- nb_hold = rq->rx_nb_hold; /* mbufs held by software */ ++ data_rq = &enic->rq[sop_rq->data_queue_idx]; + + while (nb_rx < nb_pkts) { + volatile struct rq_enet_desc *rqd_ptr; +@@ -265,6 +270,7 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + struct cq_desc cqd; + uint64_t ol_err_flags; + uint8_t packet_error; ++ uint16_t ciflags; + + /* Check for pkts available */ + color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) +@@ -272,9 +278,13 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + if (color == cq->last_color) + break; + +- /* Get the cq descriptor and rq pointer */ ++ /* Get the cq descriptor and extract rq info from it */ + cqd = *cqd_ptr; +- rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; ++ rq_num = cqd.q_number & CQ_DESC_Q_NUM_MASK; ++ rq_idx = cqd.completed_index & CQ_DESC_COMP_NDX_MASK; ++ ++ rq = &enic->rq[rq_num]; ++ rqd_ptr = ((struct rq_enet_desc *)rq->ring.descs) + rq_idx; + + /* allocate a new mbuf */ + nmb = rte_rxmbuf_alloc(rq->mp); +@@ -287,67 +297,106 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); + + /* Get the mbuf to return and replace with one just allocated */ +- rxmb = rq->mbuf_ring[rx_id]; +- rq->mbuf_ring[rx_id] = nmb; ++ rxmb = rq->mbuf_ring[rq_idx]; ++ rq->mbuf_ring[rq_idx] = nmb; + + /* Increment cqd, rqd, mbuf_table index */ +- rx_id++; +- if (unlikely(rx_id == rq->ring.desc_count)) { +- rx_id = 0; ++ cq_idx++; ++ if (unlikely(cq_idx == cq->ring.desc_count)) { ++ cq_idx = 0; + cq->last_color = cq->last_color ? 0 : 1; + } + + /* Prefetch next mbuf & desc while processing current one */ +- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; ++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + cq_idx; + rte_enic_prefetch(cqd_ptr); +- rte_enic_prefetch(rq->mbuf_ring[rx_id]); +- rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) +- + rx_id); ++// rte_enic_prefetch(rq->mbuf_ring[rx_id]); ++// rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) ++// + rx_id); ++ ++ ciflags = enic_cq_rx_desc_ciflags((struct cq_enet_rq_desc *) &cqd); + + /* Push descriptor for newly allocated mbuf */ +- dma_addr = (dma_addr_t)(nmb->buf_physaddr +- + RTE_PKTMBUF_HEADROOM); +- rqd_ptr->address = rte_cpu_to_le_64(dma_addr); +- rqd_ptr->length_type = cpu_to_le16(nmb->buf_len +- - RTE_PKTMBUF_HEADROOM); ++ ++ dma_addr = (dma_addr_t)(nmb->buf_physaddr + RTE_PKTMBUF_HEADROOM); ++ rq_enet_desc_enc(rqd_ptr, dma_addr, ++ (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP ++ : RQ_ENET_TYPE_NOT_SOP), ++ nmb->buf_len - RTE_PKTMBUF_HEADROOM); + + /* Fill in the rest of the mbuf */ +- rxmb->data_off = RTE_PKTMBUF_HEADROOM; +- rxmb->nb_segs = 1; ++ seg_length = enic_cq_rx_desc_n_bytes(&cqd); ++ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); ++ enic_cq_rx_to_pkt_flags(&cqd, rxmb); ++ if (rq->is_sop) { ++ first_seg = rxmb; ++ first_seg->nb_segs = 1; ++ first_seg->pkt_len = seg_length; ++ } else { ++ first_seg->pkt_len = (uint16_t)(first_seg->pkt_len ++ + seg_length); ++ first_seg->nb_segs++; ++ last_seg->next = rxmb; ++ } ++ + rxmb->next = NULL; + rxmb->port = enic->port_id; +- if (!packet_error) { +- rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); +- rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); +- enic_cq_rx_to_pkt_flags(&cqd, rxmb); +- } else { +- rte_pktmbuf_free(rxmb); ++ rxmb->data_len = seg_length; ++ ++ rq->rx_nb_hold++; ++ ++ if (!(enic_cq_rx_desc_eop(ciflags))) { ++ last_seg = rxmb; ++ continue; ++ } ++ ++ if (unlikely(packet_error)) { ++ rte_pktmbuf_free(first_seg); + rte_atomic64_inc(&enic->soft_stats.rx_packet_errors); +- nb_err++; ++ + continue; + } +- rxmb->data_len = rxmb->pkt_len; ++ ++ ++// printf("EOP: final packet length is %d\n", first_seg->pkt_len); ++// rte_pktmbuf_dump(stdout, first_seg, 64); + + /* prefetch mbuf data for caller */ +- rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, ++ rte_packet_prefetch(RTE_PTR_ADD(first_seg->buf_addr, + RTE_PKTMBUF_HEADROOM)); + + /* store the mbuf address into the next entry of the array */ +- rx_pkts[nb_rx++] = rxmb; ++ rx_pkts[nb_rx++] = first_seg; + } + +- nb_hold += nb_rx + nb_err; +- cq->to_clean = rx_id; ++ sop_rq->pkt_first_seg = first_seg; ++ sop_rq->pkt_last_seg = last_seg; ++ ++ cq->to_clean = cq_idx; ++ ++ if ((sop_rq->rx_nb_hold + data_rq->rx_nb_hold) > sop_rq->rx_free_thresh) { ++ if (data_rq->in_use) { ++ data_rq->posted_index = enic_ring_add(data_rq->ring.desc_count, ++ data_rq->posted_index, ++ data_rq->rx_nb_hold); ++ //printf("Processed %d data descs. Posted index now %d\n", ++ // data_rq->rx_nb_hold, data_rq->posted_index); ++ data_rq->rx_nb_hold = 0; ++ } ++ sop_rq->posted_index = enic_ring_add(sop_rq->ring.desc_count, ++ sop_rq->posted_index, ++ sop_rq->rx_nb_hold); ++ //printf("Processed %d sop descs. Posted index now %d\n", ++ // sop_rq->rx_nb_hold, sop_rq->posted_index); ++ sop_rq->rx_nb_hold = 0; + +- if (nb_hold > rq->rx_free_thresh) { +- rq->posted_index = enic_ring_add(rq->ring.desc_count, +- rq->posted_index, nb_hold); +- nb_hold = 0; + rte_mb(); +- iowrite32(rq->posted_index, &rq->ctrl->posted_index); ++ if (data_rq->in_use) ++ iowrite32(data_rq->posted_index, &data_rq->ctrl->posted_index); ++ rte_compiler_barrier(); ++ iowrite32(sop_rq->posted_index, &sop_rq->ctrl->posted_index); + } + +- rq->rx_nb_hold = nb_hold; + + return nb_rx; + } +-- +2.7.4 + |