From f03d5a02fc2b3cc24bf059a273ea1473cdb9993b Mon Sep 17 00:00:00 2001 From: John Lo Date: Tue, 7 Jun 2016 12:40:07 +0200 Subject: [PATCH 16/17] ENIC scatter RX --- drivers/net/enic/base/rq_enet_desc.h | 2 +- drivers/net/enic/base/vnic_rq.c | 12 +- drivers/net/enic/base/vnic_rq.h | 18 ++- drivers/net/enic/enic.h | 10 ++ drivers/net/enic/enic_main.c | 236 +++++++++++++++++++++++++++-------- drivers/net/enic/enic_rxtx.c | 139 ++++++++++++++------- 6 files changed, 313 insertions(+), 104 deletions(-) diff --git a/drivers/net/enic/base/rq_enet_desc.h b/drivers/net/enic/base/rq_enet_desc.h index 7292d9d..13e24b4 100644 --- a/drivers/net/enic/base/rq_enet_desc.h +++ b/drivers/net/enic/base/rq_enet_desc.h @@ -55,7 +55,7 @@ enum rq_enet_type_types { #define RQ_ENET_TYPE_BITS 2 #define RQ_ENET_TYPE_MASK ((1 << RQ_ENET_TYPE_BITS) - 1) -static inline void rq_enet_desc_enc(struct rq_enet_desc *desc, +static inline void rq_enet_desc_enc(volatile struct rq_enet_desc *desc, u64 address, u8 type, u16 length) { desc->address = cpu_to_le64(address); diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c index cb62c5e..d97f93e 100644 --- a/drivers/net/enic/base/vnic_rq.c +++ b/drivers/net/enic/base/vnic_rq.c @@ -84,11 +84,16 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, iowrite32(cq_index, &rq->ctrl->cq_index); iowrite32(error_interrupt_enable, &rq->ctrl->error_interrupt_enable); iowrite32(error_interrupt_offset, &rq->ctrl->error_interrupt_offset); - iowrite32(0, &rq->ctrl->dropped_packet_count); iowrite32(0, &rq->ctrl->error_status); iowrite32(fetch_index, &rq->ctrl->fetch_index); iowrite32(posted_index, &rq->ctrl->posted_index); - + if (rq->is_sop) { +// printf("Writing 0x%x to %s rq\n", +// ((rq->is_sop << 10) | rq->data_queue_idx), +// rq->is_sop ? "sop":"data"); + iowrite32(((rq->is_sop << 10) | rq->data_queue_idx), + &rq->ctrl->data_ring); + } } void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, @@ -96,6 +101,7 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, unsigned int error_interrupt_offset) { u32 fetch_index = 0; + /* Use current fetch_index as the ring starting point */ fetch_index = ioread32(&rq->ctrl->fetch_index); @@ -110,6 +116,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, error_interrupt_offset); rq->rxst_idx = 0; rq->tot_pkts = 0; + rq->pkt_first_seg = NULL; + rq->pkt_last_seg = NULL; } void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error) diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h index 424415c..d1e2f52 100644 --- a/drivers/net/enic/base/vnic_rq.h +++ b/drivers/net/enic/base/vnic_rq.h @@ -60,10 +60,18 @@ struct vnic_rq_ctrl { u32 pad7; u32 error_status; /* 0x48 */ u32 pad8; - u32 dropped_packet_count; /* 0x50 */ + u32 tcp_sn; /* 0x50 */ u32 pad9; - u32 dropped_packet_count_rc; /* 0x58 */ + u32 unused; /* 0x58 */ u32 pad10; + u32 dca_select; /* 0x60 */ + u32 pad11; + u32 dca_value; /* 0x68 */ + u32 pad12; + u32 data_ring; /* 0x70 */ + u32 pad13; + u32 header_split; /* 0x78 */ + u32 pad14; }; struct vnic_rq { @@ -82,6 +90,12 @@ struct vnic_rq { struct rte_mempool *mp; uint16_t rxst_idx; uint32_t tot_pkts; + uint16_t data_queue_idx; + uint8_t is_sop; + uint8_t in_use; + struct rte_mbuf *pkt_first_seg; + struct rte_mbuf *pkt_last_seg; + unsigned int max_mbufs_per_pkt; }; static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq) diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h index 7c1b5c9..d2de6ee 100644 --- a/drivers/net/enic/enic.h +++ b/drivers/net/enic/enic.h @@ -142,6 +142,16 @@ struct enic { struct enic_soft_stats soft_stats; }; +static inline unsigned int enic_sop_rq(__rte_unused struct enic *enic, unsigned int rq) +{ + return rq * 2; +} + +static inline unsigned int enic_data_rq(__rte_unused struct enic *enic, unsigned int rq) +{ + return rq * 2 + 1; +} + static inline unsigned int enic_cq_rq(__rte_unused struct enic *enic, unsigned int rq) { return rq; diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c index a00565a..be17707 100644 --- a/drivers/net/enic/enic_main.c +++ b/drivers/net/enic/enic_main.c @@ -247,15 +247,23 @@ void enic_init_vnic_resources(struct enic *enic) unsigned int error_interrupt_offset = 0; unsigned int index = 0; unsigned int cq_idx; + struct vnic_rq *data_rq; vnic_dev_stats_clear(enic->vdev); for (index = 0; index < enic->rq_count; index++) { - vnic_rq_init(&enic->rq[index], + vnic_rq_init(&enic->rq[enic_sop_rq(enic, index)], enic_cq_rq(enic, index), error_interrupt_enable, error_interrupt_offset); + data_rq = &enic->rq[enic_data_rq(enic, index)]; + if (data_rq->in_use) + vnic_rq_init(data_rq, + enic_cq_rq(enic, index), + error_interrupt_enable, + error_interrupt_offset); + cq_idx = enic_cq_rq(enic, index); vnic_cq_init(&enic->cq[cq_idx], 0 /* flow_control_enable */, @@ -305,6 +313,9 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) unsigned i; dma_addr_t dma_addr; + if (!rq->in_use) + return 0; + dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index, rq->ring.desc_count); @@ -316,20 +327,20 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) return -ENOMEM; } - dma_addr = (dma_addr_t)(mb->buf_physaddr - + RTE_PKTMBUF_HEADROOM); - - rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, - mb->buf_len - RTE_PKTMBUF_HEADROOM); + dma_addr = (dma_addr_t)(mb->buf_physaddr + RTE_PKTMBUF_HEADROOM); + rq_enet_desc_enc(rqd, dma_addr, + (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP + : RQ_ENET_TYPE_NOT_SOP), + mb->buf_len - RTE_PKTMBUF_HEADROOM); rq->mbuf_ring[i] = mb; } /* make sure all prior writes are complete before doing the PIO write */ rte_rmb(); - /* Post all but the last 2 cache lines' worth of descriptors */ - rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE - / sizeof(struct rq_enet_desc)); + /* Post all but the last buffer to VIC. */ + rq->posted_index = rq->ring.desc_count - 1; + rq->rx_nb_hold = 0; dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n", @@ -337,6 +348,8 @@ enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) iowrite32(rq->posted_index, &rq->ctrl->posted_index); rte_rmb(); +// printf("posted %d buffers to %s rq\n", rq->ring.desc_count, +// rq->is_sop ? "sop" : "data"); return 0; } @@ -398,17 +411,25 @@ int enic_enable(struct enic *enic) "Flow director feature will not work\n"); for (index = 0; index < enic->rq_count; index++) { - err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]); + err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[enic_sop_rq(enic, index)]); if (err) { - dev_err(enic, "Failed to alloc RX queue mbufs\n"); + dev_err(enic, "Failed to alloc sop RX queue mbufs\n"); + return err; + } + err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[enic_data_rq(enic, index)]); + if (err) { + /* release the previously allocated mbufs for the sop rq */ + enic_rxmbuf_queue_release(enic, &enic->rq[enic_sop_rq(enic, index)]); + + dev_err(enic, "Failed to alloc data RX queue mbufs\n"); return err; } } for (index = 0; index < enic->wq_count; index++) - vnic_wq_enable(&enic->wq[index]); + enic_start_wq(enic, index); for (index = 0; index < enic->rq_count; index++) - vnic_rq_enable(&enic->rq[index]); + enic_start_rq(enic, index); vnic_dev_enable_wait(enic->vdev); @@ -440,14 +461,26 @@ int enic_alloc_intr_resources(struct enic *enic) void enic_free_rq(void *rxq) { - struct vnic_rq *rq = (struct vnic_rq *)rxq; - struct enic *enic = vnic_dev_priv(rq->vdev); + struct vnic_rq *rq_sop = (struct vnic_rq *)rxq; + struct enic *enic = vnic_dev_priv(rq_sop->vdev); + struct vnic_rq *rq_data = &enic->rq[rq_sop->data_queue_idx]; - enic_rxmbuf_queue_release(enic, rq); - rte_free(rq->mbuf_ring); - rq->mbuf_ring = NULL; - vnic_rq_free(rq); - vnic_cq_free(&enic->cq[rq->index]); + enic_rxmbuf_queue_release(enic, rq_sop); + if (rq_data->in_use) + enic_rxmbuf_queue_release(enic, rq_data); + + rte_free(rq_sop->mbuf_ring); + if (rq_data->in_use) + rte_free(rq_data->mbuf_ring); + + rq_sop->mbuf_ring = NULL; + rq_data->mbuf_ring = NULL; + + vnic_rq_free(rq_sop); + if (rq_data->in_use) + vnic_rq_free(rq_data); + + vnic_cq_free(&enic->cq[rq_sop->index]); } void enic_start_wq(struct enic *enic, uint16_t queue_idx) @@ -462,12 +495,32 @@ int enic_stop_wq(struct enic *enic, uint16_t queue_idx) void enic_start_rq(struct enic *enic, uint16_t queue_idx) { - vnic_rq_enable(&enic->rq[queue_idx]); + struct vnic_rq *rq_sop = &enic->rq[enic_sop_rq(enic, queue_idx)]; + struct vnic_rq *rq_data = &enic->rq[rq_sop->data_queue_idx]; + + if (rq_data->in_use) + vnic_rq_enable(rq_data); + rte_mb(); + vnic_rq_enable(rq_sop); + } int enic_stop_rq(struct enic *enic, uint16_t queue_idx) { - return vnic_rq_disable(&enic->rq[queue_idx]); + int ret1 = 0, ret2 = 0; + + struct vnic_rq *rq_sop = &enic->rq[enic_sop_rq(enic, queue_idx)]; + struct vnic_rq *rq_data = &enic->rq[rq_sop->data_queue_idx]; + + ret2 = vnic_rq_disable(rq_sop); + rte_mb(); + if (rq_data->in_use) + ret1 = vnic_rq_disable(rq_data); + + if (ret2) + return ret2; + else + return ret1; } int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, @@ -475,53 +528,128 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, uint16_t nb_desc) { int rc; - struct vnic_rq *rq = &enic->rq[queue_idx]; - - rq->socket_id = socket_id; - rq->mp = mp; + uint16_t sop_queue_idx = enic_sop_rq(enic, queue_idx); + uint16_t data_queue_idx = enic_data_rq(enic, queue_idx); + struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx]; + struct vnic_rq *rq_data = &enic->rq[data_queue_idx]; + unsigned int mbuf_size, mbufs_per_pkt; + unsigned int nb_sop_desc, nb_data_desc; + uint16_t min_sop, max_sop, min_data, max_data; + + rq_sop->is_sop = 1; + rq_sop->data_queue_idx = data_queue_idx; + rq_data->is_sop = 0; + rq_data->data_queue_idx = 0; + rq_sop->socket_id = socket_id; + rq_sop->mp = mp; + rq_data->socket_id = socket_id; + rq_data->mp = mp; + rq_sop->in_use = 1; + + mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM); + + /* ceil(mtu/mbuf_size) */ + mbufs_per_pkt = (enic->config.mtu + (mbuf_size - 1)) / mbuf_size; + + if (mbufs_per_pkt > 1) + rq_data->in_use = 1; + else + rq_data->in_use = 0; + + /* number of descriptors have to be a multiple of 32 */ + nb_sop_desc = (nb_desc / mbufs_per_pkt) & ~0x1F; + nb_data_desc = (nb_desc - nb_sop_desc) & ~0x1F; + + rq_sop->max_mbufs_per_pkt = mbufs_per_pkt; + rq_data->max_mbufs_per_pkt = mbufs_per_pkt; + + //printf("mtu = %u, mbuf_size = %u, mbuf_per_pkt = %u\n", + // enic->config.mtu, mbuf_size, mbufs_per_pkt); + + if (mbufs_per_pkt > 1) { + min_sop = 64; + max_sop = ((enic->config.rq_desc_count / (mbufs_per_pkt - 1)) & ~0x1F); + min_data = min_sop * (mbufs_per_pkt - 1); + max_data = enic->config.rq_desc_count; + } else { + min_sop = 64; + max_sop = enic->config.rq_desc_count; + min_data = 0; + max_data = 0; + } - if (nb_desc) { - if (nb_desc > enic->config.rq_desc_count) { - dev_warning(enic, - "RQ %d - number of rx desc in cmd line (%d)"\ - "is greater than that in the UCSM/CIMC adapter"\ - "policy. Applying the value in the adapter "\ - "policy (%d).\n", - queue_idx, nb_desc, enic->config.rq_desc_count); - nb_desc = enic->config.rq_desc_count; - } - dev_info(enic, "RX Queues - effective number of descs:%d\n", - nb_desc); + if (nb_desc < (min_sop + min_data)) { + dev_warning(enic, + "Number of rx descs too low, adjusting to minimum\n"); + nb_sop_desc = min_sop; + nb_data_desc = min_data; + } else if (nb_desc > (max_sop + max_data)){ + dev_warning(enic, + "Number of rx_descs too high, adjusting to maximum\n"); + nb_sop_desc = max_sop; + nb_data_desc = max_data; } + dev_info(enic, "For mtu %d and mbuf size %d valid rx descriptor range is %d to %d\n", + enic->config.mtu, mbuf_size, min_sop + min_data, max_sop + max_data); - /* Allocate queue resources */ - rc = vnic_rq_alloc(enic->vdev, rq, queue_idx, - nb_desc, sizeof(struct rq_enet_desc)); + dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n", + nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc); + + /* Allocate sop queue resources */ + rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx, + nb_sop_desc, sizeof(struct rq_enet_desc)); if (rc) { - dev_err(enic, "error in allocation of rq\n"); + dev_err(enic, "error in allocation of sop rq\n"); goto err_exit; } - + nb_sop_desc = rq_sop->ring.desc_count; + + if (rq_data->in_use) { + /* Allocate data queue resources */ + rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx, + nb_data_desc, + sizeof(struct rq_enet_desc)); + if (rc) { + dev_err(enic, "error in allocation of data rq\n"); + goto err_free_rq_sop; + } + nb_data_desc = rq_data->ring.desc_count; + } rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, - socket_id, nb_desc, - sizeof(struct cq_enet_rq_desc)); + socket_id, nb_sop_desc + nb_data_desc, + sizeof(struct cq_enet_rq_desc)); if (rc) { dev_err(enic, "error in allocation of cq for rq\n"); - goto err_free_rq_exit; + goto err_free_rq_data; } - /* Allocate the mbuf ring */ - rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", - sizeof(struct rte_mbuf *) * nb_desc, - RTE_CACHE_LINE_SIZE, rq->socket_id); + /* Allocate the mbuf rings */ + rq_sop->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", + sizeof(struct rte_mbuf *) * nb_sop_desc, + RTE_CACHE_LINE_SIZE, rq_sop->socket_id); + if (rq_sop->mbuf_ring == NULL) + goto err_free_cq; + + if (rq_data->in_use) { + rq_data->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", + sizeof(struct rte_mbuf *) * nb_data_desc, + RTE_CACHE_LINE_SIZE, rq_sop->socket_id); + if (rq_data->mbuf_ring == NULL) + goto err_free_sop_mbuf; + } - if (rq->mbuf_ring != NULL) - return 0; + return 0; +err_free_sop_mbuf: + rte_free(rq_sop->mbuf_ring); +err_free_cq: /* cleanup on error */ vnic_cq_free(&enic->cq[queue_idx]); -err_free_rq_exit: - vnic_rq_free(rq); +err_free_rq_data: + if (rq_data->in_use) + vnic_rq_free(rq_data); +err_free_rq_sop: + vnic_rq_free(rq_sop); err_exit: return -ENOMEM; } diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c index 174486b..463b954 100644 --- a/drivers/net/enic/enic_rxtx.c +++ b/drivers/net/enic/enic_rxtx.c @@ -242,22 +242,27 @@ uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { - struct vnic_rq *rq = rx_queue; - struct enic *enic = vnic_dev_priv(rq->vdev); - unsigned int rx_id; + struct vnic_rq *sop_rq = rx_queue; + struct vnic_rq *data_rq; + struct vnic_rq *rq; + struct enic *enic = vnic_dev_priv(sop_rq->vdev); + uint16_t cq_idx; + uint16_t rq_idx; + uint16_t rq_num; struct rte_mbuf *nmb, *rxmb; uint16_t nb_rx = 0; - uint16_t nb_hold; struct vnic_cq *cq; volatile struct cq_desc *cqd_ptr; uint8_t color; - uint16_t nb_err = 0; + uint16_t seg_length; + struct rte_mbuf *first_seg = sop_rq->pkt_first_seg; + struct rte_mbuf *last_seg = sop_rq->pkt_last_seg; - cq = &enic->cq[enic_cq_rq(enic, rq->index)]; - rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ - cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; + cq = &enic->cq[enic_cq_rq(enic, sop_rq->index)]; + cq_idx = cq->to_clean; /* index of cqd, rqd, mbuf_table */ + cqd_ptr = (struct cq_desc *)(cq->ring.descs) + cq_idx; - nb_hold = rq->rx_nb_hold; /* mbufs held by software */ + data_rq = &enic->rq[sop_rq->data_queue_idx]; while (nb_rx < nb_pkts) { volatile struct rq_enet_desc *rqd_ptr; @@ -265,6 +270,7 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, struct cq_desc cqd; uint64_t ol_err_flags; uint8_t packet_error; + uint16_t ciflags; /* Check for pkts available */ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) @@ -272,9 +278,13 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, if (color == cq->last_color) break; - /* Get the cq descriptor and rq pointer */ + /* Get the cq descriptor and extract rq info from it */ cqd = *cqd_ptr; - rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; + rq_num = cqd.q_number & CQ_DESC_Q_NUM_MASK; + rq_idx = cqd.completed_index & CQ_DESC_COMP_NDX_MASK; + + rq = &enic->rq[rq_num]; + rqd_ptr = ((struct rq_enet_desc *)rq->ring.descs) + rq_idx; /* allocate a new mbuf */ nmb = rte_rxmbuf_alloc(rq->mp); @@ -287,67 +297,106 @@ enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags); /* Get the mbuf to return and replace with one just allocated */ - rxmb = rq->mbuf_ring[rx_id]; - rq->mbuf_ring[rx_id] = nmb; + rxmb = rq->mbuf_ring[rq_idx]; + rq->mbuf_ring[rq_idx] = nmb; /* Increment cqd, rqd, mbuf_table index */ - rx_id++; - if (unlikely(rx_id == rq->ring.desc_count)) { - rx_id = 0; + cq_idx++; + if (unlikely(cq_idx == cq->ring.desc_count)) { + cq_idx = 0; cq->last_color = cq->last_color ? 0 : 1; } /* Prefetch next mbuf & desc while processing current one */ - cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; + cqd_ptr = (struct cq_desc *)(cq->ring.descs) + cq_idx; rte_enic_prefetch(cqd_ptr); - rte_enic_prefetch(rq->mbuf_ring[rx_id]); - rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) - + rx_id); +// rte_enic_prefetch(rq->mbuf_ring[rx_id]); +// rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) +// + rx_id); + + ciflags = enic_cq_rx_desc_ciflags((struct cq_enet_rq_desc *) &cqd); /* Push descriptor for newly allocated mbuf */ - dma_addr = (dma_addr_t)(nmb->buf_physaddr - + RTE_PKTMBUF_HEADROOM); - rqd_ptr->address = rte_cpu_to_le_64(dma_addr); - rqd_ptr->length_type = cpu_to_le16(nmb->buf_len - - RTE_PKTMBUF_HEADROOM); + + dma_addr = (dma_addr_t)(nmb->buf_physaddr + RTE_PKTMBUF_HEADROOM); + rq_enet_desc_enc(rqd_ptr, dma_addr, + (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP + : RQ_ENET_TYPE_NOT_SOP), + nmb->buf_len - RTE_PKTMBUF_HEADROOM); /* Fill in the rest of the mbuf */ - rxmb->data_off = RTE_PKTMBUF_HEADROOM; - rxmb->nb_segs = 1; + seg_length = enic_cq_rx_desc_n_bytes(&cqd); + rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); + enic_cq_rx_to_pkt_flags(&cqd, rxmb); + if (rq->is_sop) { + first_seg = rxmb; + first_seg->nb_segs = 1; + first_seg->pkt_len = seg_length; + } else { + first_seg->pkt_len = (uint16_t)(first_seg->pkt_len + + seg_length); + first_seg->nb_segs++; + last_seg->next = rxmb; + } + rxmb->next = NULL; rxmb->port = enic->port_id; - if (!packet_error) { - rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd); - rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); - enic_cq_rx_to_pkt_flags(&cqd, rxmb); - } else { - rte_pktmbuf_free(rxmb); + rxmb->data_len = seg_length; + + rq->rx_nb_hold++; + + if (!(enic_cq_rx_desc_eop(ciflags))) { + last_seg = rxmb; + continue; + } + + if (unlikely(packet_error)) { + rte_pktmbuf_free(first_seg); rte_atomic64_inc(&enic->soft_stats.rx_packet_errors); - nb_err++; + continue; } - rxmb->data_len = rxmb->pkt_len; + + +// printf("EOP: final packet length is %d\n", first_seg->pkt_len); +// rte_pktmbuf_dump(stdout, first_seg, 64); /* prefetch mbuf data for caller */ - rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, + rte_packet_prefetch(RTE_PTR_ADD(first_seg->buf_addr, RTE_PKTMBUF_HEADROOM)); /* store the mbuf address into the next entry of the array */ - rx_pkts[nb_rx++] = rxmb; + rx_pkts[nb_rx++] = first_seg; } - nb_hold += nb_rx + nb_err; - cq->to_clean = rx_id; + sop_rq->pkt_first_seg = first_seg; + sop_rq->pkt_last_seg = last_seg; + + cq->to_clean = cq_idx; + + if ((sop_rq->rx_nb_hold + data_rq->rx_nb_hold) > sop_rq->rx_free_thresh) { + if (data_rq->in_use) { + data_rq->posted_index = enic_ring_add(data_rq->ring.desc_count, + data_rq->posted_index, + data_rq->rx_nb_hold); + //printf("Processed %d data descs. Posted index now %d\n", + // data_rq->rx_nb_hold, data_rq->posted_index); + data_rq->rx_nb_hold = 0; + } + sop_rq->posted_index = enic_ring_add(sop_rq->ring.desc_count, + sop_rq->posted_index, + sop_rq->rx_nb_hold); + //printf("Processed %d sop descs. Posted index now %d\n", + // sop_rq->rx_nb_hold, sop_rq->posted_index); + sop_rq->rx_nb_hold = 0; - if (nb_hold > rq->rx_free_thresh) { - rq->posted_index = enic_ring_add(rq->ring.desc_count, - rq->posted_index, nb_hold); - nb_hold = 0; rte_mb(); - iowrite32(rq->posted_index, &rq->ctrl->posted_index); + if (data_rq->in_use) + iowrite32(data_rq->posted_index, &data_rq->ctrl->posted_index); + rte_compiler_barrier(); + iowrite32(sop_rq->posted_index, &sop_rq->ctrl->posted_index); } - rq->rx_nb_hold = nb_hold; return nb_rx; } -- 2.7.4