diff options
Diffstat (limited to 'drivers/net/liquidio/lio_rxtx.c')
| -rw-r--r-- | drivers/net/liquidio/lio_rxtx.c | 1885 |
1 files changed, 1885 insertions, 0 deletions
diff --git a/drivers/net/liquidio/lio_rxtx.c b/drivers/net/liquidio/lio_rxtx.c new file mode 100644 index 00000000..9533015c --- /dev/null +++ b/drivers/net/liquidio/lio_rxtx.c @@ -0,0 +1,1885 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_ethdev.h> +#include <rte_cycles.h> +#include <rte_malloc.h> + +#include "lio_logs.h" +#include "lio_struct.h" +#include "lio_ethdev.h" +#include "lio_rxtx.h" + +#define LIO_MAX_SG 12 +/* Flush iq if available tx_desc fall below LIO_FLUSH_WM */ +#define LIO_FLUSH_WM(_iq) ((_iq)->max_count / 2) +#define LIO_PKT_IN_DONE_CNT_MASK 0x00000000FFFFFFFFULL + +static void +lio_droq_compute_max_packet_bufs(struct lio_droq *droq) +{ + uint32_t count = 0; + + do { + count += droq->buffer_size; + } while (count < LIO_MAX_RX_PKTLEN); +} + +static void +lio_droq_reset_indices(struct lio_droq *droq) +{ + droq->read_idx = 0; + droq->write_idx = 0; + droq->refill_idx = 0; + droq->refill_count = 0; + rte_atomic64_set(&droq->pkts_pending, 0); +} + +static void +lio_droq_destroy_ring_buffers(struct lio_droq *droq) +{ + uint32_t i; + + for (i = 0; i < droq->max_count; i++) { + if (droq->recv_buf_list[i].buffer) { + rte_pktmbuf_free((struct rte_mbuf *) + droq->recv_buf_list[i].buffer); + droq->recv_buf_list[i].buffer = NULL; + } + } + + lio_droq_reset_indices(droq); +} + +static void * +lio_recv_buffer_alloc(struct lio_device *lio_dev, int q_no) +{ + struct lio_droq *droq = lio_dev->droq[q_no]; + struct rte_mempool *mpool = droq->mpool; + struct rte_mbuf *m; + + m = rte_pktmbuf_alloc(mpool); + if (m == NULL) { + lio_dev_err(lio_dev, "Cannot allocate\n"); + return NULL; + } + + rte_mbuf_refcnt_set(m, 1); + m->next = NULL; + m->data_off = RTE_PKTMBUF_HEADROOM; + m->nb_segs = 1; + m->pool = mpool; + + return m; +} + +static int +lio_droq_setup_ring_buffers(struct lio_device *lio_dev, + struct lio_droq *droq) +{ + struct lio_droq_desc *desc_ring = droq->desc_ring; + uint32_t i; + void *buf; + + for (i = 0; i < droq->max_count; i++) { + buf = lio_recv_buffer_alloc(lio_dev, droq->q_no); + if (buf == NULL) { + lio_dev_err(lio_dev, "buffer alloc failed\n"); + droq->stats.rx_alloc_failure++; + lio_droq_destroy_ring_buffers(droq); + return -ENOMEM; + } + + droq->recv_buf_list[i].buffer = buf; + droq->info_list[i].length = 0; + + /* map ring buffers into memory */ + desc_ring[i].info_ptr = lio_map_ring_info(droq, i); + desc_ring[i].buffer_ptr = + lio_map_ring(droq->recv_buf_list[i].buffer); + } + + lio_droq_reset_indices(droq); + + lio_droq_compute_max_packet_bufs(droq); + + return 0; +} + +static void +lio_dma_zone_free(struct lio_device *lio_dev, const struct rte_memzone *mz) +{ + const struct rte_memzone *mz_tmp; + int ret = 0; + + if (mz == NULL) { + lio_dev_err(lio_dev, "Memzone NULL\n"); + return; + } + + mz_tmp = rte_memzone_lookup(mz->name); + if (mz_tmp == NULL) { + lio_dev_err(lio_dev, "Memzone %s Not Found\n", mz->name); + return; + } + + ret = rte_memzone_free(mz); + if (ret) + lio_dev_err(lio_dev, "Memzone free Failed ret %d\n", ret); +} + +/** + * Frees the space for descriptor ring for the droq. + * + * @param lio_dev - pointer to the lio device structure + * @param q_no - droq no. + */ +static void +lio_delete_droq(struct lio_device *lio_dev, uint32_t q_no) +{ + struct lio_droq *droq = lio_dev->droq[q_no]; + + lio_dev_dbg(lio_dev, "OQ[%d]\n", q_no); + + lio_droq_destroy_ring_buffers(droq); + rte_free(droq->recv_buf_list); + droq->recv_buf_list = NULL; + lio_dma_zone_free(lio_dev, droq->info_mz); + lio_dma_zone_free(lio_dev, droq->desc_ring_mz); + + memset(droq, 0, LIO_DROQ_SIZE); +} + +static void * +lio_alloc_info_buffer(struct lio_device *lio_dev, + struct lio_droq *droq, unsigned int socket_id) +{ + droq->info_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev, + "info_list", droq->q_no, + (droq->max_count * + LIO_DROQ_INFO_SIZE), + RTE_CACHE_LINE_SIZE, + socket_id); + + if (droq->info_mz == NULL) + return NULL; + + droq->info_list_dma = droq->info_mz->phys_addr; + droq->info_alloc_size = droq->info_mz->len; + droq->info_base_addr = (size_t)droq->info_mz->addr; + + return droq->info_mz->addr; +} + +/** + * Allocates space for the descriptor ring for the droq and + * sets the base addr, num desc etc in Octeon registers. + * + * @param lio_dev - pointer to the lio device structure + * @param q_no - droq no. + * @param app_ctx - pointer to application context + * @return Success: 0 Failure: -1 + */ +static int +lio_init_droq(struct lio_device *lio_dev, uint32_t q_no, + uint32_t num_descs, uint32_t desc_size, + struct rte_mempool *mpool, unsigned int socket_id) +{ + uint32_t c_refill_threshold; + uint32_t desc_ring_size; + struct lio_droq *droq; + + lio_dev_dbg(lio_dev, "OQ[%d]\n", q_no); + + droq = lio_dev->droq[q_no]; + droq->lio_dev = lio_dev; + droq->q_no = q_no; + droq->mpool = mpool; + + c_refill_threshold = LIO_OQ_REFILL_THRESHOLD_CFG(lio_dev); + + droq->max_count = num_descs; + droq->buffer_size = desc_size; + + desc_ring_size = droq->max_count * LIO_DROQ_DESC_SIZE; + droq->desc_ring_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev, + "droq", q_no, + desc_ring_size, + RTE_CACHE_LINE_SIZE, + socket_id); + + if (droq->desc_ring_mz == NULL) { + lio_dev_err(lio_dev, + "Output queue %d ring alloc failed\n", q_no); + return -1; + } + + droq->desc_ring_dma = droq->desc_ring_mz->phys_addr; + droq->desc_ring = (struct lio_droq_desc *)droq->desc_ring_mz->addr; + + lio_dev_dbg(lio_dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", + q_no, droq->desc_ring, (unsigned long)droq->desc_ring_dma); + lio_dev_dbg(lio_dev, "droq[%d]: num_desc: %d\n", q_no, + droq->max_count); + + droq->info_list = lio_alloc_info_buffer(lio_dev, droq, socket_id); + if (droq->info_list == NULL) { + lio_dev_err(lio_dev, "Cannot allocate memory for info list.\n"); + goto init_droq_fail; + } + + droq->recv_buf_list = rte_zmalloc_socket("recv_buf_list", + (droq->max_count * + LIO_DROQ_RECVBUF_SIZE), + RTE_CACHE_LINE_SIZE, + socket_id); + if (droq->recv_buf_list == NULL) { + lio_dev_err(lio_dev, + "Output queue recv buf list alloc failed\n"); + goto init_droq_fail; + } + + if (lio_droq_setup_ring_buffers(lio_dev, droq)) + goto init_droq_fail; + + droq->refill_threshold = c_refill_threshold; + + rte_spinlock_init(&droq->lock); + + lio_dev->fn_list.setup_oq_regs(lio_dev, q_no); + + lio_dev->io_qmask.oq |= (1ULL << q_no); + + return 0; + +init_droq_fail: + lio_delete_droq(lio_dev, q_no); + + return -1; +} + +int +lio_setup_droq(struct lio_device *lio_dev, int oq_no, int num_descs, + int desc_size, struct rte_mempool *mpool, unsigned int socket_id) +{ + struct lio_droq *droq; + + PMD_INIT_FUNC_TRACE(); + + if (lio_dev->droq[oq_no]) { + lio_dev_dbg(lio_dev, "Droq %d in use\n", oq_no); + return 0; + } + + /* Allocate the DS for the new droq. */ + droq = rte_zmalloc_socket("ethdev RX queue", sizeof(*droq), + RTE_CACHE_LINE_SIZE, socket_id); + if (droq == NULL) + return -ENOMEM; + + lio_dev->droq[oq_no] = droq; + + /* Initialize the Droq */ + if (lio_init_droq(lio_dev, oq_no, num_descs, desc_size, mpool, + socket_id)) { + lio_dev_err(lio_dev, "Droq[%u] Initialization Failed\n", oq_no); + rte_free(lio_dev->droq[oq_no]); + lio_dev->droq[oq_no] = NULL; + return -ENOMEM; + } + + lio_dev->num_oqs++; + + lio_dev_dbg(lio_dev, "Total number of OQ: %d\n", lio_dev->num_oqs); + + /* Send credit for octeon output queues. credits are always + * sent after the output queue is enabled. + */ + rte_write32(lio_dev->droq[oq_no]->max_count, + lio_dev->droq[oq_no]->pkts_credit_reg); + rte_wmb(); + + return 0; +} + +static inline uint32_t +lio_droq_get_bufcount(uint32_t buf_size, uint32_t total_len) +{ + uint32_t buf_cnt = 0; + + while (total_len > (buf_size * buf_cnt)) + buf_cnt++; + + return buf_cnt; +} + +/* If we were not able to refill all buffers, try to move around + * the buffers that were not dispatched. + */ +static inline uint32_t +lio_droq_refill_pullup_descs(struct lio_droq *droq, + struct lio_droq_desc *desc_ring) +{ + uint32_t refill_index = droq->refill_idx; + uint32_t desc_refilled = 0; + + while (refill_index != droq->read_idx) { + if (droq->recv_buf_list[refill_index].buffer) { + droq->recv_buf_list[droq->refill_idx].buffer = + droq->recv_buf_list[refill_index].buffer; + desc_ring[droq->refill_idx].buffer_ptr = + desc_ring[refill_index].buffer_ptr; + droq->recv_buf_list[refill_index].buffer = NULL; + desc_ring[refill_index].buffer_ptr = 0; + do { + droq->refill_idx = lio_incr_index( + droq->refill_idx, 1, + droq->max_count); + desc_refilled++; + droq->refill_count--; + } while (droq->recv_buf_list[droq->refill_idx].buffer); + } + refill_index = lio_incr_index(refill_index, 1, + droq->max_count); + } /* while */ + + return desc_refilled; +} + +/* lio_droq_refill + * + * @param lio_dev - pointer to the lio device structure + * @param droq - droq in which descriptors require new buffers. + * + * Description: + * Called during normal DROQ processing in interrupt mode or by the poll + * thread to refill the descriptors from which buffers were dispatched + * to upper layers. Attempts to allocate new buffers. If that fails, moves + * up buffers (that were not dispatched) to form a contiguous ring. + * + * Returns: + * No of descriptors refilled. + * + * Locks: + * This routine is called with droq->lock held. + */ +static uint32_t +lio_droq_refill(struct lio_device *lio_dev, struct lio_droq *droq) +{ + struct lio_droq_desc *desc_ring; + uint32_t desc_refilled = 0; + void *buf = NULL; + + desc_ring = droq->desc_ring; + + while (droq->refill_count && (desc_refilled < droq->max_count)) { + /* If a valid buffer exists (happens if there is no dispatch), + * reuse the buffer, else allocate. + */ + if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) { + buf = lio_recv_buffer_alloc(lio_dev, droq->q_no); + /* If a buffer could not be allocated, no point in + * continuing + */ + if (buf == NULL) { + droq->stats.rx_alloc_failure++; + break; + } + + droq->recv_buf_list[droq->refill_idx].buffer = buf; + } + + desc_ring[droq->refill_idx].buffer_ptr = + lio_map_ring(droq->recv_buf_list[droq->refill_idx].buffer); + /* Reset any previous values in the length field. */ + droq->info_list[droq->refill_idx].length = 0; + + droq->refill_idx = lio_incr_index(droq->refill_idx, 1, + droq->max_count); + desc_refilled++; + droq->refill_count--; + } + + if (droq->refill_count) + desc_refilled += lio_droq_refill_pullup_descs(droq, desc_ring); + + /* if droq->refill_count + * The refill count would not change in pass two. We only moved buffers + * to close the gap in the ring, but we would still have the same no. of + * buffers to refill. + */ + return desc_refilled; +} + +static int +lio_droq_fast_process_packet(struct lio_device *lio_dev, + struct lio_droq *droq, + struct rte_mbuf **rx_pkts) +{ + struct rte_mbuf *nicbuf = NULL; + struct lio_droq_info *info; + uint32_t total_len = 0; + int data_total_len = 0; + uint32_t pkt_len = 0; + union octeon_rh *rh; + int data_pkts = 0; + + info = &droq->info_list[droq->read_idx]; + lio_swap_8B_data((uint64_t *)info, 2); + + if (!info->length) + return -1; + + /* Len of resp hdr in included in the received data len. */ + info->length -= OCTEON_RH_SIZE; + rh = &info->rh; + + total_len += (uint32_t)info->length; + + if (lio_opcode_slow_path(rh)) { + uint32_t buf_cnt; + + buf_cnt = lio_droq_get_bufcount(droq->buffer_size, + (uint32_t)info->length); + droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt, + droq->max_count); + droq->refill_count += buf_cnt; + } else { + if (info->length <= droq->buffer_size) { + if (rh->r_dh.has_hash) + pkt_len = (uint32_t)(info->length - 8); + else + pkt_len = (uint32_t)info->length; + + nicbuf = droq->recv_buf_list[droq->read_idx].buffer; + droq->recv_buf_list[droq->read_idx].buffer = NULL; + droq->read_idx = lio_incr_index( + droq->read_idx, 1, + droq->max_count); + droq->refill_count++; + + if (likely(nicbuf != NULL)) { + nicbuf->data_off = RTE_PKTMBUF_HEADROOM; + nicbuf->nb_segs = 1; + nicbuf->next = NULL; + /* We don't have a way to pass flags yet */ + nicbuf->ol_flags = 0; + if (rh->r_dh.has_hash) { + uint64_t *hash_ptr; + + nicbuf->ol_flags |= PKT_RX_RSS_HASH; + hash_ptr = rte_pktmbuf_mtod(nicbuf, + uint64_t *); + lio_swap_8B_data(hash_ptr, 1); + nicbuf->hash.rss = (uint32_t)*hash_ptr; + nicbuf->data_off += 8; + } + + nicbuf->pkt_len = pkt_len; + nicbuf->data_len = pkt_len; + nicbuf->port = lio_dev->port_id; + /* Store the mbuf */ + rx_pkts[data_pkts++] = nicbuf; + data_total_len += pkt_len; + } + + /* Prefetch buffer pointers when on a cache line + * boundary + */ + if ((droq->read_idx & 3) == 0) { + rte_prefetch0( + &droq->recv_buf_list[droq->read_idx]); + rte_prefetch0( + &droq->info_list[droq->read_idx]); + } + } else { + struct rte_mbuf *first_buf = NULL; + struct rte_mbuf *last_buf = NULL; + + while (pkt_len < info->length) { + int cpy_len = 0; + + cpy_len = ((pkt_len + droq->buffer_size) > + info->length) + ? ((uint32_t)info->length - + pkt_len) + : droq->buffer_size; + + nicbuf = + droq->recv_buf_list[droq->read_idx].buffer; + droq->recv_buf_list[droq->read_idx].buffer = + NULL; + + if (likely(nicbuf != NULL)) { + /* Note the first seg */ + if (!pkt_len) + first_buf = nicbuf; + + nicbuf->data_off = RTE_PKTMBUF_HEADROOM; + nicbuf->nb_segs = 1; + nicbuf->next = NULL; + nicbuf->port = lio_dev->port_id; + /* We don't have a way to pass + * flags yet + */ + nicbuf->ol_flags = 0; + if ((!pkt_len) && (rh->r_dh.has_hash)) { + uint64_t *hash_ptr; + + nicbuf->ol_flags |= + PKT_RX_RSS_HASH; + hash_ptr = rte_pktmbuf_mtod( + nicbuf, uint64_t *); + lio_swap_8B_data(hash_ptr, 1); + nicbuf->hash.rss = + (uint32_t)*hash_ptr; + nicbuf->data_off += 8; + nicbuf->pkt_len = cpy_len - 8; + nicbuf->data_len = cpy_len - 8; + } else { + nicbuf->pkt_len = cpy_len; + nicbuf->data_len = cpy_len; + } + + if (pkt_len) + first_buf->nb_segs++; + + if (last_buf) + last_buf->next = nicbuf; + + last_buf = nicbuf; + } else { + PMD_RX_LOG(lio_dev, ERR, "no buf\n"); + } + + pkt_len += cpy_len; + droq->read_idx = lio_incr_index( + droq->read_idx, + 1, droq->max_count); + droq->refill_count++; + + /* Prefetch buffer pointers when on a + * cache line boundary + */ + if ((droq->read_idx & 3) == 0) { + rte_prefetch0(&droq->recv_buf_list + [droq->read_idx]); + + rte_prefetch0( + &droq->info_list[droq->read_idx]); + } + } + rx_pkts[data_pkts++] = first_buf; + if (rh->r_dh.has_hash) + data_total_len += (pkt_len - 8); + else + data_total_len += pkt_len; + } + + /* Inform upper layer about packet checksum verification */ + struct rte_mbuf *m = rx_pkts[data_pkts - 1]; + + if (rh->r_dh.csum_verified & LIO_IP_CSUM_VERIFIED) + m->ol_flags |= PKT_RX_IP_CKSUM_GOOD; + + if (rh->r_dh.csum_verified & LIO_L4_CSUM_VERIFIED) + m->ol_flags |= PKT_RX_L4_CKSUM_GOOD; + } + + if (droq->refill_count >= droq->refill_threshold) { + int desc_refilled = lio_droq_refill(lio_dev, droq); + + /* Flush the droq descriptor data to memory to be sure + * that when we update the credits the data in memory is + * accurate. + */ + rte_wmb(); + rte_write32(desc_refilled, droq->pkts_credit_reg); + /* make sure mmio write completes */ + rte_wmb(); + } + + info->length = 0; + info->rh.rh64 = 0; + + droq->stats.pkts_received++; + droq->stats.rx_pkts_received += data_pkts; + droq->stats.rx_bytes_received += data_total_len; + droq->stats.bytes_received += total_len; + + return data_pkts; +} + +static uint32_t +lio_droq_fast_process_packets(struct lio_device *lio_dev, + struct lio_droq *droq, + struct rte_mbuf **rx_pkts, + uint32_t pkts_to_process) +{ + int ret, data_pkts = 0; + uint32_t pkt; + + for (pkt = 0; pkt < pkts_to_process; pkt++) { + ret = lio_droq_fast_process_packet(lio_dev, droq, + &rx_pkts[data_pkts]); + if (ret < 0) { + lio_dev_err(lio_dev, "Port[%d] DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", + lio_dev->port_id, droq->q_no, + droq->read_idx, pkts_to_process); + break; + } + data_pkts += ret; + } + + rte_atomic64_sub(&droq->pkts_pending, pkt); + + return data_pkts; +} + +static inline uint32_t +lio_droq_check_hw_for_pkts(struct lio_droq *droq) +{ + uint32_t last_count; + uint32_t pkt_count; + + pkt_count = rte_read32(droq->pkts_sent_reg); + + last_count = pkt_count - droq->pkt_count; + droq->pkt_count = pkt_count; + + if (last_count) + rte_atomic64_add(&droq->pkts_pending, last_count); + + return last_count; +} + +uint16_t +lio_dev_recv_pkts(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t budget) +{ + struct lio_droq *droq = rx_queue; + struct lio_device *lio_dev = droq->lio_dev; + uint32_t pkts_processed = 0; + uint32_t pkt_count = 0; + + lio_droq_check_hw_for_pkts(droq); + + pkt_count = rte_atomic64_read(&droq->pkts_pending); + if (!pkt_count) + return 0; + + if (pkt_count > budget) + pkt_count = budget; + + /* Grab the lock */ + rte_spinlock_lock(&droq->lock); + pkts_processed = lio_droq_fast_process_packets(lio_dev, + droq, rx_pkts, + pkt_count); + + if (droq->pkt_count) { + rte_write32(droq->pkt_count, droq->pkts_sent_reg); + droq->pkt_count = 0; + } + + /* Release the spin lock */ + rte_spinlock_unlock(&droq->lock); + + return pkts_processed; +} + +void +lio_delete_droq_queue(struct lio_device *lio_dev, + int oq_no) +{ + lio_delete_droq(lio_dev, oq_no); + lio_dev->num_oqs--; + rte_free(lio_dev->droq[oq_no]); + lio_dev->droq[oq_no] = NULL; +} + +/** + * lio_init_instr_queue() + * @param lio_dev - pointer to the lio device structure. + * @param txpciq - queue to be initialized. + * + * Called at driver init time for each input queue. iq_conf has the + * configuration parameters for the queue. + * + * @return Success: 0 Failure: -1 + */ +static int +lio_init_instr_queue(struct lio_device *lio_dev, + union octeon_txpciq txpciq, + uint32_t num_descs, unsigned int socket_id) +{ + uint32_t iq_no = (uint32_t)txpciq.s.q_no; + struct lio_instr_queue *iq; + uint32_t instr_type; + uint32_t q_size; + + instr_type = LIO_IQ_INSTR_TYPE(lio_dev); + + q_size = instr_type * num_descs; + iq = lio_dev->instr_queue[iq_no]; + iq->iq_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev, + "instr_queue", iq_no, q_size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (iq->iq_mz == NULL) { + lio_dev_err(lio_dev, "Cannot allocate memory for instr queue %d\n", + iq_no); + return -1; + } + + iq->base_addr_dma = iq->iq_mz->phys_addr; + iq->base_addr = (uint8_t *)iq->iq_mz->addr; + + iq->max_count = num_descs; + + /* Initialize a list to holds requests that have been posted to Octeon + * but has yet to be fetched by octeon + */ + iq->request_list = rte_zmalloc_socket("request_list", + sizeof(*iq->request_list) * + num_descs, + RTE_CACHE_LINE_SIZE, + socket_id); + if (iq->request_list == NULL) { + lio_dev_err(lio_dev, "Alloc failed for IQ[%d] nr free list\n", + iq_no); + lio_dma_zone_free(lio_dev, iq->iq_mz); + return -1; + } + + lio_dev_dbg(lio_dev, "IQ[%d]: base: %p basedma: %lx count: %d\n", + iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma, + iq->max_count); + + iq->lio_dev = lio_dev; + iq->txpciq.txpciq64 = txpciq.txpciq64; + iq->fill_cnt = 0; + iq->host_write_index = 0; + iq->lio_read_index = 0; + iq->flush_index = 0; + + rte_atomic64_set(&iq->instr_pending, 0); + + /* Initialize the spinlock for this instruction queue */ + rte_spinlock_init(&iq->lock); + rte_spinlock_init(&iq->post_lock); + + rte_atomic64_clear(&iq->iq_flush_running); + + lio_dev->io_qmask.iq |= (1ULL << iq_no); + + /* Set the 32B/64B mode for each input queue */ + lio_dev->io_qmask.iq64B |= ((instr_type == 64) << iq_no); + iq->iqcmd_64B = (instr_type == 64); + + lio_dev->fn_list.setup_iq_regs(lio_dev, iq_no); + + return 0; +} + +int +lio_setup_instr_queue0(struct lio_device *lio_dev) +{ + union octeon_txpciq txpciq; + uint32_t num_descs = 0; + uint32_t iq_no = 0; + + num_descs = LIO_NUM_DEF_TX_DESCS_CFG(lio_dev); + + lio_dev->num_iqs = 0; + + lio_dev->instr_queue[0] = rte_zmalloc(NULL, + sizeof(struct lio_instr_queue), 0); + if (lio_dev->instr_queue[0] == NULL) + return -ENOMEM; + + lio_dev->instr_queue[0]->q_index = 0; + lio_dev->instr_queue[0]->app_ctx = (void *)(size_t)0; + txpciq.txpciq64 = 0; + txpciq.s.q_no = iq_no; + txpciq.s.pkind = lio_dev->pfvf_hsword.pkind; + txpciq.s.use_qpg = 0; + txpciq.s.qpg = 0; + if (lio_init_instr_queue(lio_dev, txpciq, num_descs, SOCKET_ID_ANY)) { + rte_free(lio_dev->instr_queue[0]); + lio_dev->instr_queue[0] = NULL; + return -1; + } + + lio_dev->num_iqs++; + + return 0; +} + +/** + * lio_delete_instr_queue() + * @param lio_dev - pointer to the lio device structure. + * @param iq_no - queue to be deleted. + * + * Called at driver unload time for each input queue. Deletes all + * allocated resources for the input queue. + */ +static void +lio_delete_instr_queue(struct lio_device *lio_dev, uint32_t iq_no) +{ + struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; + + rte_free(iq->request_list); + iq->request_list = NULL; + lio_dma_zone_free(lio_dev, iq->iq_mz); +} + +void +lio_free_instr_queue0(struct lio_device *lio_dev) +{ + lio_delete_instr_queue(lio_dev, 0); + rte_free(lio_dev->instr_queue[0]); + lio_dev->instr_queue[0] = NULL; + lio_dev->num_iqs--; +} + +/* Return 0 on success, -1 on failure */ +int +lio_setup_iq(struct lio_device *lio_dev, int q_index, + union octeon_txpciq txpciq, uint32_t num_descs, void *app_ctx, + unsigned int socket_id) +{ + uint32_t iq_no = (uint32_t)txpciq.s.q_no; + + if (lio_dev->instr_queue[iq_no]) { + lio_dev_dbg(lio_dev, "IQ is in use. Cannot create the IQ: %d again\n", + iq_no); + lio_dev->instr_queue[iq_no]->txpciq.txpciq64 = txpciq.txpciq64; + lio_dev->instr_queue[iq_no]->app_ctx = app_ctx; + return 0; + } + + lio_dev->instr_queue[iq_no] = rte_zmalloc_socket("ethdev TX queue", + sizeof(struct lio_instr_queue), + RTE_CACHE_LINE_SIZE, socket_id); + if (lio_dev->instr_queue[iq_no] == NULL) + return -1; + + lio_dev->instr_queue[iq_no]->q_index = q_index; + lio_dev->instr_queue[iq_no]->app_ctx = app_ctx; + + if (lio_init_instr_queue(lio_dev, txpciq, num_descs, socket_id)) + goto release_lio_iq; + + lio_dev->num_iqs++; + if (lio_dev->fn_list.enable_io_queues(lio_dev)) + goto delete_lio_iq; + + return 0; + +delete_lio_iq: + lio_delete_instr_queue(lio_dev, iq_no); + lio_dev->num_iqs--; +release_lio_iq: + rte_free(lio_dev->instr_queue[iq_no]); + lio_dev->instr_queue[iq_no] = NULL; + + return -1; +} + +int +lio_wait_for_instr_fetch(struct lio_device *lio_dev) +{ + int pending, instr_cnt; + int i, retry = 1000; + + do { + instr_cnt = 0; + + for (i = 0; i < LIO_MAX_INSTR_QUEUES(lio_dev); i++) { + if (!(lio_dev->io_qmask.iq & (1ULL << i))) + continue; + + if (lio_dev->instr_queue[i] == NULL) + break; + + pending = rte_atomic64_read( + &lio_dev->instr_queue[i]->instr_pending); + if (pending) + lio_flush_iq(lio_dev, lio_dev->instr_queue[i]); + + instr_cnt += pending; + } + + if (instr_cnt == 0) + break; + + rte_delay_ms(1); + + } while (retry-- && instr_cnt); + + return instr_cnt; +} + +static inline void +lio_ring_doorbell(struct lio_device *lio_dev, + struct lio_instr_queue *iq) +{ + if (rte_atomic64_read(&lio_dev->status) == LIO_DEV_RUNNING) { + rte_write32(iq->fill_cnt, iq->doorbell_reg); + /* make sure doorbell write goes through */ + rte_wmb(); + iq->fill_cnt = 0; + } +} + +static inline void +copy_cmd_into_iq(struct lio_instr_queue *iq, uint8_t *cmd) +{ + uint8_t *iqptr, cmdsize; + + cmdsize = ((iq->iqcmd_64B) ? 64 : 32); + iqptr = iq->base_addr + (cmdsize * iq->host_write_index); + + rte_memcpy(iqptr, cmd, cmdsize); +} + +static inline struct lio_iq_post_status +post_command2(struct lio_instr_queue *iq, uint8_t *cmd) +{ + struct lio_iq_post_status st; + + st.status = LIO_IQ_SEND_OK; + + /* This ensures that the read index does not wrap around to the same + * position if queue gets full before Octeon could fetch any instr. + */ + if (rte_atomic64_read(&iq->instr_pending) >= + (int32_t)(iq->max_count - 1)) { + st.status = LIO_IQ_SEND_FAILED; + st.index = -1; + return st; + } + + if (rte_atomic64_read(&iq->instr_pending) >= + (int32_t)(iq->max_count - 2)) + st.status = LIO_IQ_SEND_STOP; + + copy_cmd_into_iq(iq, cmd); + + /* "index" is returned, host_write_index is modified. */ + st.index = iq->host_write_index; + iq->host_write_index = lio_incr_index(iq->host_write_index, 1, + iq->max_count); + iq->fill_cnt++; + + /* Flush the command into memory. We need to be sure the data is in + * memory before indicating that the instruction is pending. + */ + rte_wmb(); + + rte_atomic64_inc(&iq->instr_pending); + + return st; +} + +static inline void +lio_add_to_request_list(struct lio_instr_queue *iq, + int idx, void *buf, int reqtype) +{ + iq->request_list[idx].buf = buf; + iq->request_list[idx].reqtype = reqtype; +} + +static inline void +lio_free_netsgbuf(void *buf) +{ + struct lio_buf_free_info *finfo = buf; + struct lio_device *lio_dev = finfo->lio_dev; + struct rte_mbuf *m = finfo->mbuf; + struct lio_gather *g = finfo->g; + uint8_t iq = finfo->iq_no; + + /* This will take care of multiple segments also */ + rte_pktmbuf_free(m); + + rte_spinlock_lock(&lio_dev->glist_lock[iq]); + STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq], &g->list, entries); + rte_spinlock_unlock(&lio_dev->glist_lock[iq]); + rte_free(finfo); +} + +/* Can only run in process context */ +static int +lio_process_iq_request_list(struct lio_device *lio_dev, + struct lio_instr_queue *iq) +{ + struct octeon_instr_irh *irh = NULL; + uint32_t old = iq->flush_index; + struct lio_soft_command *sc; + uint32_t inst_count = 0; + int reqtype; + void *buf; + + while (old != iq->lio_read_index) { + reqtype = iq->request_list[old].reqtype; + buf = iq->request_list[old].buf; + + if (reqtype == LIO_REQTYPE_NONE) + goto skip_this; + + switch (reqtype) { + case LIO_REQTYPE_NORESP_NET: + rte_pktmbuf_free((struct rte_mbuf *)buf); + break; + case LIO_REQTYPE_NORESP_NET_SG: + lio_free_netsgbuf(buf); + break; + case LIO_REQTYPE_SOFT_COMMAND: + sc = buf; + irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; + if (irh->rflag) { + /* We're expecting a response from Octeon. + * It's up to lio_process_ordered_list() to + * process sc. Add sc to the ordered soft + * command response list because we expect + * a response from Octeon. + */ + rte_spinlock_lock(&lio_dev->response_list.lock); + rte_atomic64_inc( + &lio_dev->response_list.pending_req_count); + STAILQ_INSERT_TAIL( + &lio_dev->response_list.head, + &sc->node, entries); + rte_spinlock_unlock( + &lio_dev->response_list.lock); + } else { + if (sc->callback) { + /* This callback must not sleep */ + sc->callback(LIO_REQUEST_DONE, + sc->callback_arg); + } + } + break; + default: + lio_dev_err(lio_dev, + "Unknown reqtype: %d buf: %p at idx %d\n", + reqtype, buf, old); + } + + iq->request_list[old].buf = NULL; + iq->request_list[old].reqtype = 0; + +skip_this: + inst_count++; + old = lio_incr_index(old, 1, iq->max_count); + } + + iq->flush_index = old; + + return inst_count; +} + +static void +lio_update_read_index(struct lio_instr_queue *iq) +{ + uint32_t pkt_in_done = rte_read32(iq->inst_cnt_reg); + uint32_t last_done; + + last_done = pkt_in_done - iq->pkt_in_done; + iq->pkt_in_done = pkt_in_done; + + /* Add last_done and modulo with the IQ size to get new index */ + iq->lio_read_index = (iq->lio_read_index + + (uint32_t)(last_done & LIO_PKT_IN_DONE_CNT_MASK)) % + iq->max_count; +} + +int +lio_flush_iq(struct lio_device *lio_dev, struct lio_instr_queue *iq) +{ + uint32_t tot_inst_processed = 0; + uint32_t inst_processed = 0; + int tx_done = 1; + + if (rte_atomic64_test_and_set(&iq->iq_flush_running) == 0) + return tx_done; + + rte_spinlock_lock(&iq->lock); + + lio_update_read_index(iq); + + do { + /* Process any outstanding IQ packets. */ + if (iq->flush_index == iq->lio_read_index) + break; + + inst_processed = lio_process_iq_request_list(lio_dev, iq); + + if (inst_processed) { + rte_atomic64_sub(&iq->instr_pending, inst_processed); + iq->stats.instr_processed += inst_processed; + } + + tot_inst_processed += inst_processed; + inst_processed = 0; + + } while (1); + + rte_spinlock_unlock(&iq->lock); + + rte_atomic64_clear(&iq->iq_flush_running); + + return tx_done; +} + +static int +lio_send_command(struct lio_device *lio_dev, uint32_t iq_no, void *cmd, + void *buf, uint32_t datasize, uint32_t reqtype) +{ + struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; + struct lio_iq_post_status st; + + rte_spinlock_lock(&iq->post_lock); + + st = post_command2(iq, cmd); + + if (st.status != LIO_IQ_SEND_FAILED) { + lio_add_to_request_list(iq, st.index, buf, reqtype); + LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, bytes_sent, + datasize); + LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_posted, 1); + + lio_ring_doorbell(lio_dev, iq); + } else { + LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_dropped, 1); + } + + rte_spinlock_unlock(&iq->post_lock); + + return st.status; +} + +void +lio_prepare_soft_command(struct lio_device *lio_dev, + struct lio_soft_command *sc, uint8_t opcode, + uint8_t subcode, uint32_t irh_ossp, uint64_t ossp0, + uint64_t ossp1) +{ + struct octeon_instr_pki_ih3 *pki_ih3; + struct octeon_instr_ih3 *ih3; + struct octeon_instr_irh *irh; + struct octeon_instr_rdp *rdp; + + RTE_ASSERT(opcode <= 15); + RTE_ASSERT(subcode <= 127); + + ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; + + ih3->pkind = lio_dev->instr_queue[sc->iq_no]->txpciq.s.pkind; + + pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3; + + pki_ih3->w = 1; + pki_ih3->raw = 1; + pki_ih3->utag = 1; + pki_ih3->uqpg = lio_dev->instr_queue[sc->iq_no]->txpciq.s.use_qpg; + pki_ih3->utt = 1; + + pki_ih3->tag = LIO_CONTROL; + pki_ih3->tagtype = OCTEON_ATOMIC_TAG; + pki_ih3->qpg = lio_dev->instr_queue[sc->iq_no]->txpciq.s.qpg; + pki_ih3->pm = 0x7; + pki_ih3->sl = 8; + + if (sc->datasize) + ih3->dlengsz = sc->datasize; + + irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; + irh->opcode = opcode; + irh->subcode = subcode; + + /* opcode/subcode specific parameters (ossp) */ + irh->ossp = irh_ossp; + sc->cmd.cmd3.ossp[0] = ossp0; + sc->cmd.cmd3.ossp[1] = ossp1; + + if (sc->rdatasize) { + rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp; + rdp->pcie_port = lio_dev->pcie_port; + rdp->rlen = sc->rdatasize; + irh->rflag = 1; + /* PKI IH3 */ + ih3->fsz = OCTEON_SOFT_CMD_RESP_IH3; + } else { + irh->rflag = 0; + /* PKI IH3 */ + ih3->fsz = OCTEON_PCI_CMD_O3; + } +} + +int +lio_send_soft_command(struct lio_device *lio_dev, + struct lio_soft_command *sc) +{ + struct octeon_instr_ih3 *ih3; + struct octeon_instr_irh *irh; + uint32_t len = 0; + + ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; + if (ih3->dlengsz) { + RTE_ASSERT(sc->dmadptr); + sc->cmd.cmd3.dptr = sc->dmadptr; + } + + irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; + if (irh->rflag) { + RTE_ASSERT(sc->dmarptr); + RTE_ASSERT(sc->status_word != NULL); + *sc->status_word = LIO_COMPLETION_WORD_INIT; + sc->cmd.cmd3.rptr = sc->dmarptr; + } + + len = (uint32_t)ih3->dlengsz; + + if (sc->wait_time) + sc->timeout = lio_uptime + sc->wait_time; + + return lio_send_command(lio_dev, sc->iq_no, &sc->cmd, sc, len, + LIO_REQTYPE_SOFT_COMMAND); +} + +int +lio_setup_sc_buffer_pool(struct lio_device *lio_dev) +{ + char sc_pool_name[RTE_MEMPOOL_NAMESIZE]; + uint16_t buf_size; + + buf_size = LIO_SOFT_COMMAND_BUFFER_SIZE + RTE_PKTMBUF_HEADROOM; + snprintf(sc_pool_name, sizeof(sc_pool_name), + "lio_sc_pool_%u", lio_dev->port_id); + lio_dev->sc_buf_pool = rte_pktmbuf_pool_create(sc_pool_name, + LIO_MAX_SOFT_COMMAND_BUFFERS, + 0, 0, buf_size, SOCKET_ID_ANY); + return 0; +} + +void +lio_free_sc_buffer_pool(struct lio_device *lio_dev) +{ + rte_mempool_free(lio_dev->sc_buf_pool); +} + +struct lio_soft_command * +lio_alloc_soft_command(struct lio_device *lio_dev, uint32_t datasize, + uint32_t rdatasize, uint32_t ctxsize) +{ + uint32_t offset = sizeof(struct lio_soft_command); + struct lio_soft_command *sc; + struct rte_mbuf *m; + uint64_t dma_addr; + + RTE_ASSERT((offset + datasize + rdatasize + ctxsize) <= + LIO_SOFT_COMMAND_BUFFER_SIZE); + + m = rte_pktmbuf_alloc(lio_dev->sc_buf_pool); + if (m == NULL) { + lio_dev_err(lio_dev, "Cannot allocate mbuf for sc\n"); + return NULL; + } + + /* set rte_mbuf data size and there is only 1 segment */ + m->pkt_len = LIO_SOFT_COMMAND_BUFFER_SIZE; + m->data_len = LIO_SOFT_COMMAND_BUFFER_SIZE; + + /* use rte_mbuf buffer for soft command */ + sc = rte_pktmbuf_mtod(m, struct lio_soft_command *); + memset(sc, 0, LIO_SOFT_COMMAND_BUFFER_SIZE); + sc->size = LIO_SOFT_COMMAND_BUFFER_SIZE; + sc->dma_addr = rte_mbuf_data_dma_addr(m); + sc->mbuf = m; + + dma_addr = sc->dma_addr; + + if (ctxsize) { + sc->ctxptr = (uint8_t *)sc + offset; + sc->ctxsize = ctxsize; + } + + /* Start data at 128 byte boundary */ + offset = (offset + ctxsize + 127) & 0xffffff80; + + if (datasize) { + sc->virtdptr = (uint8_t *)sc + offset; + sc->dmadptr = dma_addr + offset; + sc->datasize = datasize; + } + + /* Start rdata at 128 byte boundary */ + offset = (offset + datasize + 127) & 0xffffff80; + + if (rdatasize) { + RTE_ASSERT(rdatasize >= 16); + sc->virtrptr = (uint8_t *)sc + offset; + sc->dmarptr = dma_addr + offset; + sc->rdatasize = rdatasize; + sc->status_word = (uint64_t *)((uint8_t *)(sc->virtrptr) + + rdatasize - 8); + } + + return sc; +} + +void +lio_free_soft_command(struct lio_soft_command *sc) +{ + rte_pktmbuf_free(sc->mbuf); +} + +void +lio_setup_response_list(struct lio_device *lio_dev) +{ + STAILQ_INIT(&lio_dev->response_list.head); + rte_spinlock_init(&lio_dev->response_list.lock); + rte_atomic64_set(&lio_dev->response_list.pending_req_count, 0); +} + +int +lio_process_ordered_list(struct lio_device *lio_dev) +{ + int resp_to_process = LIO_MAX_ORD_REQS_TO_PROCESS; + struct lio_response_list *ordered_sc_list; + struct lio_soft_command *sc; + int request_complete = 0; + uint64_t status64; + uint32_t status; + + ordered_sc_list = &lio_dev->response_list; + + do { + rte_spinlock_lock(&ordered_sc_list->lock); + + if (STAILQ_EMPTY(&ordered_sc_list->head)) { + /* ordered_sc_list is empty; there is + * nothing to process + */ + rte_spinlock_unlock(&ordered_sc_list->lock); + return -1; + } + + sc = LIO_STQUEUE_FIRST_ENTRY(&ordered_sc_list->head, + struct lio_soft_command, node); + + status = LIO_REQUEST_PENDING; + + /* check if octeon has finished DMA'ing a response + * to where rptr is pointing to + */ + status64 = *sc->status_word; + + if (status64 != LIO_COMPLETION_WORD_INIT) { + /* This logic ensures that all 64b have been written. + * 1. check byte 0 for non-FF + * 2. if non-FF, then swap result from BE to host order + * 3. check byte 7 (swapped to 0) for non-FF + * 4. if non-FF, use the low 32-bit status code + * 5. if either byte 0 or byte 7 is FF, don't use status + */ + if ((status64 & 0xff) != 0xff) { + lio_swap_8B_data(&status64, 1); + if (((status64 & 0xff) != 0xff)) { + /* retrieve 16-bit firmware status */ + status = (uint32_t)(status64 & + 0xffffULL); + if (status) { + status = + LIO_FIRMWARE_STATUS_CODE( + status); + } else { + /* i.e. no error */ + status = LIO_REQUEST_DONE; + } + } + } + } else if ((sc->timeout && lio_check_timeout(lio_uptime, + sc->timeout))) { + lio_dev_err(lio_dev, + "cmd failed, timeout (%ld, %ld)\n", + (long)lio_uptime, (long)sc->timeout); + status = LIO_REQUEST_TIMEOUT; + } + + if (status != LIO_REQUEST_PENDING) { + /* we have received a response or we have timed out. + * remove node from linked list + */ + STAILQ_REMOVE(&ordered_sc_list->head, + &sc->node, lio_stailq_node, entries); + rte_atomic64_dec( + &lio_dev->response_list.pending_req_count); + rte_spinlock_unlock(&ordered_sc_list->lock); + + if (sc->callback) + sc->callback(status, sc->callback_arg); + + request_complete++; + } else { + /* no response yet */ + request_complete = 0; + rte_spinlock_unlock(&ordered_sc_list->lock); + } + + /* If we hit the Max Ordered requests to process every loop, + * we quit and let this function be invoked the next time + * the poll thread runs to process the remaining requests. + * This function can take up the entire CPU if there is + * no upper limit to the requests processed. + */ + if (request_complete >= resp_to_process) + break; + } while (request_complete); + + return 0; +} + +static inline struct lio_stailq_node * +list_delete_first_node(struct lio_stailq_head *head) +{ + struct lio_stailq_node *node; + + if (STAILQ_EMPTY(head)) + node = NULL; + else + node = STAILQ_FIRST(head); + + if (node) + STAILQ_REMOVE(head, node, lio_stailq_node, entries); + + return node; +} + +void +lio_delete_sglist(struct lio_instr_queue *txq) +{ + struct lio_device *lio_dev = txq->lio_dev; + int iq_no = txq->q_index; + struct lio_gather *g; + + if (lio_dev->glist_head == NULL) + return; + + do { + g = (struct lio_gather *)list_delete_first_node( + &lio_dev->glist_head[iq_no]); + if (g) { + if (g->sg) + rte_free( + (void *)((unsigned long)g->sg - g->adjust)); + rte_free(g); + } + } while (g); +} + +/** + * \brief Setup gather lists + * @param lio per-network private data + */ +int +lio_setup_sglists(struct lio_device *lio_dev, int iq_no, + int fw_mapped_iq, int num_descs, unsigned int socket_id) +{ + struct lio_gather *g; + int i; + + rte_spinlock_init(&lio_dev->glist_lock[iq_no]); + + STAILQ_INIT(&lio_dev->glist_head[iq_no]); + + for (i = 0; i < num_descs; i++) { + g = rte_zmalloc_socket(NULL, sizeof(*g), RTE_CACHE_LINE_SIZE, + socket_id); + if (g == NULL) { + lio_dev_err(lio_dev, + "lio_gather memory allocation failed for qno %d\n", + iq_no); + break; + } + + g->sg_size = + ((ROUNDUP4(LIO_MAX_SG) >> 2) * LIO_SG_ENTRY_SIZE); + + g->sg = rte_zmalloc_socket(NULL, g->sg_size + 8, + RTE_CACHE_LINE_SIZE, socket_id); + if (g->sg == NULL) { + lio_dev_err(lio_dev, + "sg list memory allocation failed for qno %d\n", + iq_no); + rte_free(g); + break; + } + + /* The gather component should be aligned on 64-bit boundary */ + if (((unsigned long)g->sg) & 7) { + g->adjust = 8 - (((unsigned long)g->sg) & 7); + g->sg = + (struct lio_sg_entry *)((unsigned long)g->sg + + g->adjust); + } + + STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq_no], &g->list, + entries); + } + + if (i != num_descs) { + lio_delete_sglist(lio_dev->instr_queue[fw_mapped_iq]); + return -ENOMEM; + } + + return 0; +} + +void +lio_delete_instruction_queue(struct lio_device *lio_dev, int iq_no) +{ + lio_delete_instr_queue(lio_dev, iq_no); + rte_free(lio_dev->instr_queue[iq_no]); + lio_dev->instr_queue[iq_no] = NULL; + lio_dev->num_iqs--; +} + +static inline uint32_t +lio_iq_get_available(struct lio_device *lio_dev, uint32_t q_no) +{ + return ((lio_dev->instr_queue[q_no]->max_count - 1) - + (uint32_t)rte_atomic64_read( + &lio_dev->instr_queue[q_no]->instr_pending)); +} + +static inline int +lio_iq_is_full(struct lio_device *lio_dev, uint32_t q_no) +{ + return ((uint32_t)rte_atomic64_read( + &lio_dev->instr_queue[q_no]->instr_pending) >= + (lio_dev->instr_queue[q_no]->max_count - 2)); +} + +static int +lio_dev_cleanup_iq(struct lio_device *lio_dev, int iq_no) +{ + struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; + uint32_t count = 10000; + + while ((lio_iq_get_available(lio_dev, iq_no) < LIO_FLUSH_WM(iq)) && + --count) + lio_flush_iq(lio_dev, iq); + + return count ? 0 : 1; +} + +static void +lio_ctrl_cmd_callback(uint32_t status __rte_unused, void *sc_ptr) +{ + struct lio_soft_command *sc = sc_ptr; + struct lio_dev_ctrl_cmd *ctrl_cmd; + struct lio_ctrl_pkt *ctrl_pkt; + + ctrl_pkt = (struct lio_ctrl_pkt *)sc->ctxptr; + ctrl_cmd = ctrl_pkt->ctrl_cmd; + ctrl_cmd->cond = 1; + + lio_free_soft_command(sc); +} + +static inline struct lio_soft_command * +lio_alloc_ctrl_pkt_sc(struct lio_device *lio_dev, + struct lio_ctrl_pkt *ctrl_pkt) +{ + struct lio_soft_command *sc = NULL; + uint32_t uddsize, datasize; + uint32_t rdatasize; + uint8_t *data; + + uddsize = (uint32_t)(ctrl_pkt->ncmd.s.more * 8); + + datasize = OCTEON_CMD_SIZE + uddsize; + rdatasize = (ctrl_pkt->wait_time) ? 16 : 0; + + sc = lio_alloc_soft_command(lio_dev, datasize, + rdatasize, sizeof(struct lio_ctrl_pkt)); + if (sc == NULL) + return NULL; + + rte_memcpy(sc->ctxptr, ctrl_pkt, sizeof(struct lio_ctrl_pkt)); + + data = (uint8_t *)sc->virtdptr; + + rte_memcpy(data, &ctrl_pkt->ncmd, OCTEON_CMD_SIZE); + + lio_swap_8B_data((uint64_t *)data, OCTEON_CMD_SIZE >> 3); + + if (uddsize) { + /* Endian-Swap for UDD should have been done by caller. */ + rte_memcpy(data + OCTEON_CMD_SIZE, ctrl_pkt->udd, uddsize); + } + + sc->iq_no = (uint32_t)ctrl_pkt->iq_no; + + lio_prepare_soft_command(lio_dev, sc, + LIO_OPCODE, LIO_OPCODE_CMD, + 0, 0, 0); + + sc->callback = lio_ctrl_cmd_callback; + sc->callback_arg = sc; + sc->wait_time = ctrl_pkt->wait_time; + + return sc; +} + +int +lio_send_ctrl_pkt(struct lio_device *lio_dev, struct lio_ctrl_pkt *ctrl_pkt) +{ + struct lio_soft_command *sc = NULL; + int retval; + + sc = lio_alloc_ctrl_pkt_sc(lio_dev, ctrl_pkt); + if (sc == NULL) { + lio_dev_err(lio_dev, "soft command allocation failed\n"); + return -1; + } + + retval = lio_send_soft_command(lio_dev, sc); + if (retval == LIO_IQ_SEND_FAILED) { + lio_free_soft_command(sc); + lio_dev_err(lio_dev, "Port: %d soft command: %d send failed status: %x\n", + lio_dev->port_id, ctrl_pkt->ncmd.s.cmd, retval); + return -1; + } + + return retval; +} + +/** Send data packet to the device + * @param lio_dev - lio device pointer + * @param ndata - control structure with queueing, and buffer information + * + * @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if it the + * queue should be stopped, and LIO_IQ_SEND_OK if it sent okay. + */ +static inline int +lio_send_data_pkt(struct lio_device *lio_dev, struct lio_data_pkt *ndata) +{ + return lio_send_command(lio_dev, ndata->q_no, &ndata->cmd, + ndata->buf, ndata->datasize, ndata->reqtype); +} + +uint16_t +lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts) +{ + struct lio_instr_queue *txq = tx_queue; + union lio_cmd_setup cmdsetup; + struct lio_device *lio_dev; + struct lio_iq_stats *stats; + struct lio_data_pkt ndata; + int i, processed = 0; + struct rte_mbuf *m; + uint32_t tag = 0; + int status = 0; + int iq_no; + + lio_dev = txq->lio_dev; + iq_no = txq->txpciq.s.q_no; + stats = &lio_dev->instr_queue[iq_no]->stats; + + if (!lio_dev->intf_open || !lio_dev->linfo.link.s.link_up) { + PMD_TX_LOG(lio_dev, ERR, "Transmit failed link_status : %d\n", + lio_dev->linfo.link.s.link_up); + goto xmit_failed; + } + + lio_dev_cleanup_iq(lio_dev, iq_no); + + for (i = 0; i < nb_pkts; i++) { + uint32_t pkt_len = 0; + + m = pkts[i]; + + /* Prepare the attributes for the data to be passed to BASE. */ + memset(&ndata, 0, sizeof(struct lio_data_pkt)); + + ndata.buf = m; + + ndata.q_no = iq_no; + if (lio_iq_is_full(lio_dev, ndata.q_no)) { + stats->tx_iq_busy++; + if (lio_dev_cleanup_iq(lio_dev, iq_no)) { + PMD_TX_LOG(lio_dev, ERR, + "Transmit failed iq:%d full\n", + ndata.q_no); + break; + } + } + + cmdsetup.cmd_setup64 = 0; + cmdsetup.s.iq_no = iq_no; + + /* check checksum offload flags to form cmd */ + if (m->ol_flags & PKT_TX_IP_CKSUM) + cmdsetup.s.ip_csum = 1; + + if (m->ol_flags & PKT_TX_OUTER_IP_CKSUM) + cmdsetup.s.tnl_csum = 1; + else if ((m->ol_flags & PKT_TX_TCP_CKSUM) || + (m->ol_flags & PKT_TX_UDP_CKSUM)) + cmdsetup.s.transport_csum = 1; + + if (m->nb_segs == 1) { + pkt_len = rte_pktmbuf_data_len(m); + cmdsetup.s.u.datasize = pkt_len; + lio_prepare_pci_cmd(lio_dev, &ndata.cmd, + &cmdsetup, tag); + ndata.cmd.cmd3.dptr = rte_mbuf_data_dma_addr(m); + ndata.reqtype = LIO_REQTYPE_NORESP_NET; + } else { + struct lio_buf_free_info *finfo; + struct lio_gather *g; + phys_addr_t phyaddr; + int i, frags; + + finfo = (struct lio_buf_free_info *)rte_malloc(NULL, + sizeof(*finfo), 0); + if (finfo == NULL) { + PMD_TX_LOG(lio_dev, ERR, + "free buffer alloc failed\n"); + goto xmit_failed; + } + + rte_spinlock_lock(&lio_dev->glist_lock[iq_no]); + g = (struct lio_gather *)list_delete_first_node( + &lio_dev->glist_head[iq_no]); + rte_spinlock_unlock(&lio_dev->glist_lock[iq_no]); + if (g == NULL) { + PMD_TX_LOG(lio_dev, ERR, + "Transmit scatter gather: glist null!\n"); + goto xmit_failed; + } + + cmdsetup.s.gather = 1; + cmdsetup.s.u.gatherptrs = m->nb_segs; + lio_prepare_pci_cmd(lio_dev, &ndata.cmd, + &cmdsetup, tag); + + memset(g->sg, 0, g->sg_size); + g->sg[0].ptr[0] = rte_mbuf_data_dma_addr(m); + lio_add_sg_size(&g->sg[0], m->data_len, 0); + pkt_len = m->data_len; + finfo->mbuf = m; + + /* First seg taken care above */ + frags = m->nb_segs - 1; + i = 1; + m = m->next; + while (frags--) { + g->sg[(i >> 2)].ptr[(i & 3)] = + rte_mbuf_data_dma_addr(m); + lio_add_sg_size(&g->sg[(i >> 2)], + m->data_len, (i & 3)); + pkt_len += m->data_len; + i++; + m = m->next; + } + + phyaddr = rte_mem_virt2phy(g->sg); + if (phyaddr == RTE_BAD_PHYS_ADDR) { + PMD_TX_LOG(lio_dev, ERR, "bad phys addr\n"); + goto xmit_failed; + } + + ndata.cmd.cmd3.dptr = phyaddr; + ndata.reqtype = LIO_REQTYPE_NORESP_NET_SG; + + finfo->g = g; + finfo->lio_dev = lio_dev; + finfo->iq_no = (uint64_t)iq_no; + ndata.buf = finfo; + } + + ndata.datasize = pkt_len; + + status = lio_send_data_pkt(lio_dev, &ndata); + + if (unlikely(status == LIO_IQ_SEND_FAILED)) { + PMD_TX_LOG(lio_dev, ERR, "send failed\n"); + break; + } + + if (unlikely(status == LIO_IQ_SEND_STOP)) { + PMD_TX_LOG(lio_dev, DEBUG, "iq full\n"); + /* create space as iq is full */ + lio_dev_cleanup_iq(lio_dev, iq_no); + } + + stats->tx_done++; + stats->tx_tot_bytes += pkt_len; + processed++; + } + +xmit_failed: + stats->tx_dropped += (nb_pkts - processed); + + return processed; +} + +void +lio_dev_clear_queues(struct rte_eth_dev *eth_dev) +{ + struct lio_instr_queue *txq; + struct lio_droq *rxq; + uint16_t i; + + for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { + txq = eth_dev->data->tx_queues[i]; + if (txq != NULL) { + lio_dev_tx_queue_release(txq); + eth_dev->data->tx_queues[i] = NULL; + } + } + + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + rxq = eth_dev->data->rx_queues[i]; + if (rxq != NULL) { + lio_dev_rx_queue_release(rxq); + eth_dev->data->rx_queues[i] = NULL; + } + } +} |