diff options
Diffstat (limited to 'drivers/net/liquidio')
| -rw-r--r-- | drivers/net/liquidio/Makefile | 58 | ||||
| -rw-r--r-- | drivers/net/liquidio/base/lio_23xx_reg.h | 194 | ||||
| -rw-r--r-- | drivers/net/liquidio/base/lio_23xx_vf.c | 588 | ||||
| -rw-r--r-- | drivers/net/liquidio/base/lio_23xx_vf.h | 97 | ||||
| -rw-r--r-- | drivers/net/liquidio/base/lio_hw_defs.h | 249 | ||||
| -rw-r--r-- | drivers/net/liquidio/base/lio_mbox.c | 275 | ||||
| -rw-r--r-- | drivers/net/liquidio/base/lio_mbox.h | 131 | ||||
| -rw-r--r-- | drivers/net/liquidio/lio_ethdev.c | 2058 | ||||
| -rw-r--r-- | drivers/net/liquidio/lio_ethdev.h | 205 | ||||
| -rw-r--r-- | drivers/net/liquidio/lio_logs.h | 91 | ||||
| -rw-r--r-- | drivers/net/liquidio/lio_rxtx.c | 1885 | ||||
| -rw-r--r-- | drivers/net/liquidio/lio_rxtx.h | 769 | ||||
| -rw-r--r-- | drivers/net/liquidio/lio_struct.h | 689 | ||||
| -rw-r--r-- | drivers/net/liquidio/rte_pmd_lio_version.map | 4 |
14 files changed, 7293 insertions, 0 deletions
diff --git a/drivers/net/liquidio/Makefile b/drivers/net/liquidio/Makefile new file mode 100644 index 00000000..32c06f5b --- /dev/null +++ b/drivers/net/liquidio/Makefile @@ -0,0 +1,58 @@ +# +# BSD LICENSE +# +# Copyright(c) 2017 Cavium, Inc.. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Cavium, Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_lio.a + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR)/base -I$(SRCDIR) + +EXPORT_MAP := rte_pmd_lio_version.map + +LIBABIVER := 1 + +VPATH += $(RTE_SDK)/drivers/net/liquidio/base + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += lio_ethdev.c +SRCS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += lio_rxtx.c +SRCS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += lio_23xx_vf.c +SRCS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += lio_mbox.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/net/liquidio/base/lio_23xx_reg.h b/drivers/net/liquidio/base/lio_23xx_reg.h new file mode 100644 index 00000000..794bc2ca --- /dev/null +++ b/drivers/net/liquidio/base/lio_23xx_reg.h @@ -0,0 +1,194 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_23XX_REG_H_ +#define _LIO_23XX_REG_H_ + +/* ###################### REQUEST QUEUE ######################### */ + +/* 64 registers for Input Queues Start Addr - SLI_PKT(0..63)_INSTR_BADDR */ +#define CN23XX_SLI_PKT_INSTR_BADDR_START64 0x10010 + +/* 64 registers for Input Doorbell - SLI_PKT(0..63)_INSTR_BAOFF_DBELL */ +#define CN23XX_SLI_PKT_INSTR_BADDR_DBELL_START 0x10020 + +/* 64 registers for Input Queue size - SLI_PKT(0..63)_INSTR_FIFO_RSIZE */ +#define CN23XX_SLI_PKT_INSTR_FIFO_RSIZE_START 0x10030 + +/* 64 registers for Input Queue Instr Count - SLI_PKT_IN_DONE(0..63)_CNTS */ +#define CN23XX_SLI_PKT_IN_DONE_CNTS_START64 0x10040 + +/* 64 registers (64-bit) - ES, RO, NS, Arbitration for Input Queue Data & + * gather list fetches. SLI_PKT(0..63)_INPUT_CONTROL. + */ +#define CN23XX_SLI_PKT_INPUT_CONTROL_START64 0x10000 + +/* ------- Request Queue Macros --------- */ + +/* Each Input Queue register is at a 16-byte Offset in BAR0 */ +#define CN23XX_IQ_OFFSET 0x20000 + +#define CN23XX_SLI_IQ_PKT_CONTROL64(iq) \ + (CN23XX_SLI_PKT_INPUT_CONTROL_START64 + ((iq) * CN23XX_IQ_OFFSET)) + +#define CN23XX_SLI_IQ_BASE_ADDR64(iq) \ + (CN23XX_SLI_PKT_INSTR_BADDR_START64 + ((iq) * CN23XX_IQ_OFFSET)) + +#define CN23XX_SLI_IQ_SIZE(iq) \ + (CN23XX_SLI_PKT_INSTR_FIFO_RSIZE_START + ((iq) * CN23XX_IQ_OFFSET)) + +#define CN23XX_SLI_IQ_DOORBELL(iq) \ + (CN23XX_SLI_PKT_INSTR_BADDR_DBELL_START + ((iq) * CN23XX_IQ_OFFSET)) + +#define CN23XX_SLI_IQ_INSTR_COUNT64(iq) \ + (CN23XX_SLI_PKT_IN_DONE_CNTS_START64 + ((iq) * CN23XX_IQ_OFFSET)) + +/* Number of instructions to be read in one MAC read request. + * setting to Max value(4) + */ +#define CN23XX_PKT_INPUT_CTL_RDSIZE (3 << 25) +#define CN23XX_PKT_INPUT_CTL_IS_64B (1 << 24) +#define CN23XX_PKT_INPUT_CTL_RST (1 << 23) +#define CN23XX_PKT_INPUT_CTL_QUIET (1 << 28) +#define CN23XX_PKT_INPUT_CTL_RING_ENB (1 << 22) +#define CN23XX_PKT_INPUT_CTL_DATA_ES_64B_SWAP (1 << 6) +#define CN23XX_PKT_INPUT_CTL_USE_CSR (1 << 4) +#define CN23XX_PKT_INPUT_CTL_GATHER_ES_64B_SWAP (2) + +/* These bits[47:44] select the Physical function number within the MAC */ +#define CN23XX_PKT_INPUT_CTL_PF_NUM_POS 45 +/* These bits[43:32] select the function number within the PF */ +#define CN23XX_PKT_INPUT_CTL_VF_NUM_POS 32 + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN +#define CN23XX_PKT_INPUT_CTL_MASK \ + (CN23XX_PKT_INPUT_CTL_RDSIZE | \ + CN23XX_PKT_INPUT_CTL_DATA_ES_64B_SWAP | \ + CN23XX_PKT_INPUT_CTL_USE_CSR) +#elif RTE_BYTE_ORDER == RTE_BIG_ENDIAN +#define CN23XX_PKT_INPUT_CTL_MASK \ + (CN23XX_PKT_INPUT_CTL_RDSIZE | \ + CN23XX_PKT_INPUT_CTL_DATA_ES_64B_SWAP | \ + CN23XX_PKT_INPUT_CTL_USE_CSR | \ + CN23XX_PKT_INPUT_CTL_GATHER_ES_64B_SWAP) +#endif + +/* ############################ OUTPUT QUEUE ######################### */ + +/* 64 registers for Output queue control - SLI_PKT(0..63)_OUTPUT_CONTROL */ +#define CN23XX_SLI_PKT_OUTPUT_CONTROL_START 0x10050 + +/* 64 registers for Output queue buffer and info size + * SLI_PKT(0..63)_OUT_SIZE + */ +#define CN23XX_SLI_PKT_OUT_SIZE 0x10060 + +/* 64 registers for Output Queue Start Addr - SLI_PKT(0..63)_SLIST_BADDR */ +#define CN23XX_SLI_SLIST_BADDR_START64 0x10070 + +/* 64 registers for Output Queue Packet Credits + * SLI_PKT(0..63)_SLIST_BAOFF_DBELL + */ +#define CN23XX_SLI_PKT_SLIST_BAOFF_DBELL_START 0x10080 + +/* 64 registers for Output Queue size - SLI_PKT(0..63)_SLIST_FIFO_RSIZE */ +#define CN23XX_SLI_PKT_SLIST_FIFO_RSIZE_START 0x10090 + +/* 64 registers for Output Queue Packet Count - SLI_PKT(0..63)_CNTS */ +#define CN23XX_SLI_PKT_CNTS_START 0x100B0 + +/* Each Output Queue register is at a 16-byte Offset in BAR0 */ +#define CN23XX_OQ_OFFSET 0x20000 + +/* ------- Output Queue Macros --------- */ + +#define CN23XX_SLI_OQ_PKT_CONTROL(oq) \ + (CN23XX_SLI_PKT_OUTPUT_CONTROL_START + ((oq) * CN23XX_OQ_OFFSET)) + +#define CN23XX_SLI_OQ_BASE_ADDR64(oq) \ + (CN23XX_SLI_SLIST_BADDR_START64 + ((oq) * CN23XX_OQ_OFFSET)) + +#define CN23XX_SLI_OQ_SIZE(oq) \ + (CN23XX_SLI_PKT_SLIST_FIFO_RSIZE_START + ((oq) * CN23XX_OQ_OFFSET)) + +#define CN23XX_SLI_OQ_BUFF_INFO_SIZE(oq) \ + (CN23XX_SLI_PKT_OUT_SIZE + ((oq) * CN23XX_OQ_OFFSET)) + +#define CN23XX_SLI_OQ_PKTS_SENT(oq) \ + (CN23XX_SLI_PKT_CNTS_START + ((oq) * CN23XX_OQ_OFFSET)) + +#define CN23XX_SLI_OQ_PKTS_CREDIT(oq) \ + (CN23XX_SLI_PKT_SLIST_BAOFF_DBELL_START + ((oq) * CN23XX_OQ_OFFSET)) + +/* ------------------ Masks ---------------- */ +#define CN23XX_PKT_OUTPUT_CTL_IPTR (1 << 11) +#define CN23XX_PKT_OUTPUT_CTL_ES (1 << 9) +#define CN23XX_PKT_OUTPUT_CTL_NSR (1 << 8) +#define CN23XX_PKT_OUTPUT_CTL_ROR (1 << 7) +#define CN23XX_PKT_OUTPUT_CTL_DPTR (1 << 6) +#define CN23XX_PKT_OUTPUT_CTL_BMODE (1 << 5) +#define CN23XX_PKT_OUTPUT_CTL_ES_P (1 << 3) +#define CN23XX_PKT_OUTPUT_CTL_NSR_P (1 << 2) +#define CN23XX_PKT_OUTPUT_CTL_ROR_P (1 << 1) +#define CN23XX_PKT_OUTPUT_CTL_RING_ENB (1 << 0) + +/* Rings per Virtual Function [RO] */ +#define CN23XX_PKT_INPUT_CTL_RPVF_MASK 0x3F +#define CN23XX_PKT_INPUT_CTL_RPVF_POS 48 + +/* These bits[47:44][RO] give the Physical function + * number info within the MAC + */ +#define CN23XX_PKT_INPUT_CTL_PF_NUM_MASK 0x7 + +/* These bits[43:32][RO] give the virtual function + * number info within the PF + */ +#define CN23XX_PKT_INPUT_CTL_VF_NUM_MASK 0x1FFF + +/* ######################### Mailbox Reg Macros ######################## */ +#define CN23XX_SLI_PKT_PF_VF_MBOX_SIG_START 0x10200 +#define CN23XX_VF_SLI_PKT_MBOX_INT_START 0x10210 + +#define CN23XX_SLI_MBOX_OFFSET 0x20000 +#define CN23XX_SLI_MBOX_SIG_IDX_OFFSET 0x8 + +#define CN23XX_SLI_PKT_PF_VF_MBOX_SIG(q, idx) \ + (CN23XX_SLI_PKT_PF_VF_MBOX_SIG_START + \ + ((q) * CN23XX_SLI_MBOX_OFFSET + \ + (idx) * CN23XX_SLI_MBOX_SIG_IDX_OFFSET)) + +#define CN23XX_VF_SLI_PKT_MBOX_INT(q) \ + (CN23XX_VF_SLI_PKT_MBOX_INT_START + ((q) * CN23XX_SLI_MBOX_OFFSET)) + +#endif /* _LIO_23XX_REG_H_ */ diff --git a/drivers/net/liquidio/base/lio_23xx_vf.c b/drivers/net/liquidio/base/lio_23xx_vf.c new file mode 100644 index 00000000..e30c20dc --- /dev/null +++ b/drivers/net/liquidio/base/lio_23xx_vf.c @@ -0,0 +1,588 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> + +#include <rte_ethdev.h> +#include <rte_cycles.h> +#include <rte_malloc.h> + +#include "lio_logs.h" +#include "lio_23xx_vf.h" +#include "lio_23xx_reg.h" +#include "lio_mbox.h" + +static int +cn23xx_vf_reset_io_queues(struct lio_device *lio_dev, uint32_t num_queues) +{ + uint32_t loop = CN23XX_VF_BUSY_READING_REG_LOOP_COUNT; + uint64_t d64, q_no; + int ret_val = 0; + + PMD_INIT_FUNC_TRACE(); + + for (q_no = 0; q_no < num_queues; q_no++) { + /* set RST bit to 1. This bit applies to both IQ and OQ */ + d64 = lio_read_csr64(lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + d64 = d64 | CN23XX_PKT_INPUT_CTL_RST; + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), + d64); + } + + /* wait until the RST bit is clear or the RST and QUIET bits are set */ + for (q_no = 0; q_no < num_queues; q_no++) { + volatile uint64_t reg_val; + + reg_val = lio_read_csr64(lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + while ((reg_val & CN23XX_PKT_INPUT_CTL_RST) && + !(reg_val & CN23XX_PKT_INPUT_CTL_QUIET) && + loop) { + reg_val = lio_read_csr64( + lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + loop = loop - 1; + } + + if (loop == 0) { + lio_dev_err(lio_dev, + "clearing the reset reg failed or setting the quiet reg failed for qno: %lu\n", + (unsigned long)q_no); + return -1; + } + + reg_val = reg_val & ~CN23XX_PKT_INPUT_CTL_RST; + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), + reg_val); + + reg_val = lio_read_csr64( + lio_dev, CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + if (reg_val & CN23XX_PKT_INPUT_CTL_RST) { + lio_dev_err(lio_dev, + "clearing the reset failed for qno: %lu\n", + (unsigned long)q_no); + ret_val = -1; + } + } + + return ret_val; +} + +static int +cn23xx_vf_setup_global_input_regs(struct lio_device *lio_dev) +{ + uint64_t q_no; + uint64_t d64; + + PMD_INIT_FUNC_TRACE(); + + if (cn23xx_vf_reset_io_queues(lio_dev, + lio_dev->sriov_info.rings_per_vf)) + return -1; + + for (q_no = 0; q_no < (lio_dev->sriov_info.rings_per_vf); q_no++) { + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_DOORBELL(q_no), + 0xFFFFFFFF); + + d64 = lio_read_csr64(lio_dev, + CN23XX_SLI_IQ_INSTR_COUNT64(q_no)); + + d64 &= 0xEFFFFFFFFFFFFFFFL; + + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_INSTR_COUNT64(q_no), + d64); + + /* Select ES, RO, NS, RDSIZE,DPTR Fomat#0 for + * the Input Queues + */ + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), + CN23XX_PKT_INPUT_CTL_MASK); + } + + return 0; +} + +static void +cn23xx_vf_setup_global_output_regs(struct lio_device *lio_dev) +{ + uint32_t reg_val; + uint32_t q_no; + + PMD_INIT_FUNC_TRACE(); + + for (q_no = 0; q_no < lio_dev->sriov_info.rings_per_vf; q_no++) { + lio_write_csr(lio_dev, CN23XX_SLI_OQ_PKTS_CREDIT(q_no), + 0xFFFFFFFF); + + reg_val = + lio_read_csr(lio_dev, CN23XX_SLI_OQ_PKTS_SENT(q_no)); + + reg_val &= 0xEFFFFFFFFFFFFFFFL; + + reg_val = + lio_read_csr(lio_dev, CN23XX_SLI_OQ_PKT_CONTROL(q_no)); + + /* set IPTR & DPTR */ + reg_val |= + (CN23XX_PKT_OUTPUT_CTL_IPTR | CN23XX_PKT_OUTPUT_CTL_DPTR); + + /* reset BMODE */ + reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_BMODE); + + /* No Relaxed Ordering, No Snoop, 64-bit Byte swap + * for Output Queue Scatter List + * reset ROR_P, NSR_P + */ + reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_ROR_P); + reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_NSR_P); + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_ES_P); +#elif RTE_BYTE_ORDER == RTE_BIG_ENDIAN + reg_val |= (CN23XX_PKT_OUTPUT_CTL_ES_P); +#endif + /* No Relaxed Ordering, No Snoop, 64-bit Byte swap + * for Output Queue Data + * reset ROR, NSR + */ + reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_ROR); + reg_val &= ~(CN23XX_PKT_OUTPUT_CTL_NSR); + /* set the ES bit */ + reg_val |= (CN23XX_PKT_OUTPUT_CTL_ES); + + /* write all the selected settings */ + lio_write_csr(lio_dev, CN23XX_SLI_OQ_PKT_CONTROL(q_no), + reg_val); + } +} + +static int +cn23xx_vf_setup_device_regs(struct lio_device *lio_dev) +{ + PMD_INIT_FUNC_TRACE(); + + if (cn23xx_vf_setup_global_input_regs(lio_dev)) + return -1; + + cn23xx_vf_setup_global_output_regs(lio_dev); + + return 0; +} + +static void +cn23xx_vf_setup_iq_regs(struct lio_device *lio_dev, uint32_t iq_no) +{ + struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; + uint64_t pkt_in_done = 0; + + PMD_INIT_FUNC_TRACE(); + + /* Write the start of the input queue's ring and its size */ + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_BASE_ADDR64(iq_no), + iq->base_addr_dma); + lio_write_csr(lio_dev, CN23XX_SLI_IQ_SIZE(iq_no), iq->max_count); + + /* Remember the doorbell & instruction count register addr + * for this queue + */ + iq->doorbell_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_SLI_IQ_DOORBELL(iq_no); + iq->inst_cnt_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_SLI_IQ_INSTR_COUNT64(iq_no); + lio_dev_dbg(lio_dev, "InstQ[%d]:dbell reg @ 0x%p instcnt_reg @ 0x%p\n", + iq_no, iq->doorbell_reg, iq->inst_cnt_reg); + + /* Store the current instruction counter (used in flush_iq + * calculation) + */ + pkt_in_done = rte_read64(iq->inst_cnt_reg); + + /* Clear the count by writing back what we read, but don't + * enable data traffic here + */ + rte_write64(pkt_in_done, iq->inst_cnt_reg); +} + +static void +cn23xx_vf_setup_oq_regs(struct lio_device *lio_dev, uint32_t oq_no) +{ + struct lio_droq *droq = lio_dev->droq[oq_no]; + + PMD_INIT_FUNC_TRACE(); + + lio_write_csr64(lio_dev, CN23XX_SLI_OQ_BASE_ADDR64(oq_no), + droq->desc_ring_dma); + lio_write_csr(lio_dev, CN23XX_SLI_OQ_SIZE(oq_no), droq->max_count); + + lio_write_csr(lio_dev, CN23XX_SLI_OQ_BUFF_INFO_SIZE(oq_no), + (droq->buffer_size | (OCTEON_RH_SIZE << 16))); + + /* Get the mapped address of the pkt_sent and pkts_credit regs */ + droq->pkts_sent_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_SLI_OQ_PKTS_SENT(oq_no); + droq->pkts_credit_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_SLI_OQ_PKTS_CREDIT(oq_no); +} + +static void +cn23xx_vf_free_mbox(struct lio_device *lio_dev) +{ + PMD_INIT_FUNC_TRACE(); + + rte_free(lio_dev->mbox[0]); + lio_dev->mbox[0] = NULL; + + rte_free(lio_dev->mbox); + lio_dev->mbox = NULL; +} + +static int +cn23xx_vf_setup_mbox(struct lio_device *lio_dev) +{ + struct lio_mbox *mbox; + + PMD_INIT_FUNC_TRACE(); + + if (lio_dev->mbox == NULL) { + lio_dev->mbox = rte_zmalloc(NULL, sizeof(void *), 0); + if (lio_dev->mbox == NULL) + return -ENOMEM; + } + + mbox = rte_zmalloc(NULL, sizeof(struct lio_mbox), 0); + if (mbox == NULL) { + rte_free(lio_dev->mbox); + lio_dev->mbox = NULL; + return -ENOMEM; + } + + rte_spinlock_init(&mbox->lock); + + mbox->lio_dev = lio_dev; + + mbox->q_no = 0; + + mbox->state = LIO_MBOX_STATE_IDLE; + + /* VF mbox interrupt reg */ + mbox->mbox_int_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_VF_SLI_PKT_MBOX_INT(0); + /* VF reads from SIG0 reg */ + mbox->mbox_read_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_SLI_PKT_PF_VF_MBOX_SIG(0, 0); + /* VF writes into SIG1 reg */ + mbox->mbox_write_reg = (uint8_t *)lio_dev->hw_addr + + CN23XX_SLI_PKT_PF_VF_MBOX_SIG(0, 1); + + lio_dev->mbox[0] = mbox; + + rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg); + + return 0; +} + +static int +cn23xx_vf_enable_io_queues(struct lio_device *lio_dev) +{ + uint32_t q_no; + + PMD_INIT_FUNC_TRACE(); + + for (q_no = 0; q_no < lio_dev->num_iqs; q_no++) { + uint64_t reg_val; + + /* set the corresponding IQ IS_64B bit */ + if (lio_dev->io_qmask.iq64B & (1ULL << q_no)) { + reg_val = lio_read_csr64( + lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + reg_val = reg_val | CN23XX_PKT_INPUT_CTL_IS_64B; + lio_write_csr64(lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no), + reg_val); + } + + /* set the corresponding IQ ENB bit */ + if (lio_dev->io_qmask.iq & (1ULL << q_no)) { + reg_val = lio_read_csr64( + lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + reg_val = reg_val | CN23XX_PKT_INPUT_CTL_RING_ENB; + lio_write_csr64(lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no), + reg_val); + } + } + for (q_no = 0; q_no < lio_dev->num_oqs; q_no++) { + uint32_t reg_val; + + /* set the corresponding OQ ENB bit */ + if (lio_dev->io_qmask.oq & (1ULL << q_no)) { + reg_val = lio_read_csr( + lio_dev, + CN23XX_SLI_OQ_PKT_CONTROL(q_no)); + reg_val = reg_val | CN23XX_PKT_OUTPUT_CTL_RING_ENB; + lio_write_csr(lio_dev, + CN23XX_SLI_OQ_PKT_CONTROL(q_no), + reg_val); + } + } + + return 0; +} + +static void +cn23xx_vf_disable_io_queues(struct lio_device *lio_dev) +{ + uint32_t num_queues; + + PMD_INIT_FUNC_TRACE(); + + /* per HRM, rings can only be disabled via reset operation, + * NOT via SLI_PKT()_INPUT/OUTPUT_CONTROL[ENB] + */ + num_queues = lio_dev->num_iqs; + if (num_queues < lio_dev->num_oqs) + num_queues = lio_dev->num_oqs; + + cn23xx_vf_reset_io_queues(lio_dev, num_queues); +} + +void +cn23xx_vf_ask_pf_to_do_flr(struct lio_device *lio_dev) +{ + struct lio_mbox_cmd mbox_cmd; + + memset(&mbox_cmd, 0, sizeof(struct lio_mbox_cmd)); + mbox_cmd.msg.s.type = LIO_MBOX_REQUEST; + mbox_cmd.msg.s.resp_needed = 0; + mbox_cmd.msg.s.cmd = LIO_VF_FLR_REQUEST; + mbox_cmd.msg.s.len = 1; + mbox_cmd.q_no = 0; + mbox_cmd.recv_len = 0; + mbox_cmd.recv_status = 0; + mbox_cmd.fn = NULL; + mbox_cmd.fn_arg = 0; + + lio_mbox_write(lio_dev, &mbox_cmd); +} + +static void +cn23xx_pfvf_hs_callback(struct lio_device *lio_dev, + struct lio_mbox_cmd *cmd, void *arg) +{ + uint32_t major = 0; + + PMD_INIT_FUNC_TRACE(); + + rte_memcpy((uint8_t *)&lio_dev->pfvf_hsword, cmd->msg.s.params, 6); + if (cmd->recv_len > 1) { + struct lio_version *lio_ver = (struct lio_version *)cmd->data; + + major = lio_ver->major; + major = major << 16; + } + + rte_atomic64_set((rte_atomic64_t *)arg, major | 1); +} + +int +cn23xx_pfvf_handshake(struct lio_device *lio_dev) +{ + struct lio_mbox_cmd mbox_cmd; + struct lio_version *lio_ver = (struct lio_version *)&mbox_cmd.data[0]; + uint32_t q_no, count = 0; + rte_atomic64_t status; + uint32_t pfmajor; + uint32_t vfmajor; + uint32_t ret; + + PMD_INIT_FUNC_TRACE(); + + /* Sending VF_ACTIVE indication to the PF driver */ + lio_dev_dbg(lio_dev, "requesting info from PF\n"); + + mbox_cmd.msg.mbox_msg64 = 0; + mbox_cmd.msg.s.type = LIO_MBOX_REQUEST; + mbox_cmd.msg.s.resp_needed = 1; + mbox_cmd.msg.s.cmd = LIO_VF_ACTIVE; + mbox_cmd.msg.s.len = 2; + mbox_cmd.data[0] = 0; + lio_ver->major = LIO_BASE_MAJOR_VERSION; + lio_ver->minor = LIO_BASE_MINOR_VERSION; + lio_ver->micro = LIO_BASE_MICRO_VERSION; + mbox_cmd.q_no = 0; + mbox_cmd.recv_len = 0; + mbox_cmd.recv_status = 0; + mbox_cmd.fn = (lio_mbox_callback)cn23xx_pfvf_hs_callback; + mbox_cmd.fn_arg = (void *)&status; + + if (lio_mbox_write(lio_dev, &mbox_cmd)) { + lio_dev_err(lio_dev, "Write to mailbox failed\n"); + return -1; + } + + rte_atomic64_set(&status, 0); + + do { + rte_delay_ms(1); + } while ((rte_atomic64_read(&status) == 0) && (count++ < 10000)); + + ret = rte_atomic64_read(&status); + if (ret == 0) { + lio_dev_err(lio_dev, "cn23xx_pfvf_handshake timeout\n"); + return -1; + } + + for (q_no = 0; q_no < lio_dev->num_iqs; q_no++) + lio_dev->instr_queue[q_no]->txpciq.s.pkind = + lio_dev->pfvf_hsword.pkind; + + vfmajor = LIO_BASE_MAJOR_VERSION; + pfmajor = ret >> 16; + if (pfmajor != vfmajor) { + lio_dev_err(lio_dev, + "VF LiquidIO driver (major version %d) is not compatible with LiquidIO PF driver (major version %d)\n", + vfmajor, pfmajor); + ret = -EPERM; + } else { + lio_dev_dbg(lio_dev, + "VF LiquidIO driver (major version %d), LiquidIO PF driver (major version %d)\n", + vfmajor, pfmajor); + ret = 0; + } + + lio_dev_dbg(lio_dev, "got data from PF pkind is %d\n", + lio_dev->pfvf_hsword.pkind); + + return ret; +} + +void +cn23xx_vf_handle_mbox(struct lio_device *lio_dev) +{ + uint64_t mbox_int_val; + + /* read and clear by writing 1 */ + mbox_int_val = rte_read64(lio_dev->mbox[0]->mbox_int_reg); + rte_write64(mbox_int_val, lio_dev->mbox[0]->mbox_int_reg); + if (lio_mbox_read(lio_dev->mbox[0])) + lio_mbox_process_message(lio_dev->mbox[0]); +} + +int +cn23xx_vf_setup_device(struct lio_device *lio_dev) +{ + uint64_t reg_val; + + PMD_INIT_FUNC_TRACE(); + + /* INPUT_CONTROL[RPVF] gives the VF IOq count */ + reg_val = lio_read_csr64(lio_dev, CN23XX_SLI_IQ_PKT_CONTROL64(0)); + + lio_dev->pf_num = (reg_val >> CN23XX_PKT_INPUT_CTL_PF_NUM_POS) & + CN23XX_PKT_INPUT_CTL_PF_NUM_MASK; + lio_dev->vf_num = (reg_val >> CN23XX_PKT_INPUT_CTL_VF_NUM_POS) & + CN23XX_PKT_INPUT_CTL_VF_NUM_MASK; + + reg_val = reg_val >> CN23XX_PKT_INPUT_CTL_RPVF_POS; + + lio_dev->sriov_info.rings_per_vf = + reg_val & CN23XX_PKT_INPUT_CTL_RPVF_MASK; + + lio_dev->default_config = lio_get_conf(lio_dev); + if (lio_dev->default_config == NULL) + return -1; + + lio_dev->fn_list.setup_iq_regs = cn23xx_vf_setup_iq_regs; + lio_dev->fn_list.setup_oq_regs = cn23xx_vf_setup_oq_regs; + lio_dev->fn_list.setup_mbox = cn23xx_vf_setup_mbox; + lio_dev->fn_list.free_mbox = cn23xx_vf_free_mbox; + + lio_dev->fn_list.setup_device_regs = cn23xx_vf_setup_device_regs; + + lio_dev->fn_list.enable_io_queues = cn23xx_vf_enable_io_queues; + lio_dev->fn_list.disable_io_queues = cn23xx_vf_disable_io_queues; + + return 0; +} + +int +cn23xx_vf_set_io_queues_off(struct lio_device *lio_dev) +{ + uint32_t loop = CN23XX_VF_BUSY_READING_REG_LOOP_COUNT; + uint64_t q_no; + + /* Disable the i/p and o/p queues for this Octeon. + * IOQs will already be in reset. + * If RST bit is set, wait for Quiet bit to be set + * Once Quiet bit is set, clear the RST bit + */ + PMD_INIT_FUNC_TRACE(); + + for (q_no = 0; q_no < lio_dev->sriov_info.rings_per_vf; q_no++) { + volatile uint64_t reg_val; + + reg_val = lio_read_csr64(lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + while ((reg_val & CN23XX_PKT_INPUT_CTL_RST) && !(reg_val & + CN23XX_PKT_INPUT_CTL_QUIET) && loop) { + reg_val = lio_read_csr64( + lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + loop = loop - 1; + } + + if (loop == 0) { + lio_dev_err(lio_dev, + "clearing the reset reg failed or setting the quiet reg failed for qno %lu\n", + (unsigned long)q_no); + return -1; + } + + reg_val = reg_val & ~CN23XX_PKT_INPUT_CTL_RST; + lio_write_csr64(lio_dev, CN23XX_SLI_IQ_PKT_CONTROL64(q_no), + reg_val); + + reg_val = lio_read_csr64(lio_dev, + CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); + if (reg_val & CN23XX_PKT_INPUT_CTL_RST) { + lio_dev_err(lio_dev, "unable to reset qno %lu\n", + (unsigned long)q_no); + return -1; + } + } + + return 0; +} diff --git a/drivers/net/liquidio/base/lio_23xx_vf.h b/drivers/net/liquidio/base/lio_23xx_vf.h new file mode 100644 index 00000000..ad8db0df --- /dev/null +++ b/drivers/net/liquidio/base/lio_23xx_vf.h @@ -0,0 +1,97 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_23XX_VF_H_ +#define _LIO_23XX_VF_H_ + +#include <stdio.h> + +#include "lio_struct.h" + +static const struct lio_config default_cn23xx_conf = { + .card_type = LIO_23XX, + .card_name = LIO_23XX_NAME, + /** IQ attributes */ + .iq = { + .max_iqs = CN23XX_CFG_IO_QUEUES, + .pending_list_size = + (CN23XX_MAX_IQ_DESCRIPTORS * CN23XX_CFG_IO_QUEUES), + .instr_type = OCTEON_64BYTE_INSTR, + }, + + /** OQ attributes */ + .oq = { + .max_oqs = CN23XX_CFG_IO_QUEUES, + .info_ptr = OCTEON_OQ_INFOPTR_MODE, + .refill_threshold = CN23XX_OQ_REFIL_THRESHOLD, + }, + + .num_nic_ports = CN23XX_DEFAULT_NUM_PORTS, + .num_def_rx_descs = CN23XX_MAX_OQ_DESCRIPTORS, + .num_def_tx_descs = CN23XX_MAX_IQ_DESCRIPTORS, + .def_rx_buf_size = CN23XX_OQ_BUF_SIZE, +}; + +static inline const struct lio_config * +lio_get_conf(struct lio_device *lio_dev) +{ + const struct lio_config *default_lio_conf = NULL; + + /* check the LIO Device model & return the corresponding lio + * configuration + */ + default_lio_conf = &default_cn23xx_conf; + + if (default_lio_conf == NULL) { + lio_dev_err(lio_dev, "Configuration verification failed\n"); + return NULL; + } + + return default_lio_conf; +} + +/** Turns off the input and output queues for the device + * @param lio_dev which device io queues to disable + */ +int cn23xx_vf_set_io_queues_off(struct lio_device *lio_dev); + +#define CN23XX_VF_BUSY_READING_REG_LOOP_COUNT 100000 + +void cn23xx_vf_ask_pf_to_do_flr(struct lio_device *lio_dev); + +int cn23xx_pfvf_handshake(struct lio_device *lio_dev); + +int cn23xx_vf_setup_device(struct lio_device *lio_dev); + +void cn23xx_vf_handle_mbox(struct lio_device *lio_dev); +#endif /* _LIO_23XX_VF_H_ */ diff --git a/drivers/net/liquidio/base/lio_hw_defs.h b/drivers/net/liquidio/base/lio_hw_defs.h new file mode 100644 index 00000000..67eaa452 --- /dev/null +++ b/drivers/net/liquidio/base/lio_hw_defs.h @@ -0,0 +1,249 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_HW_DEFS_H_ +#define _LIO_HW_DEFS_H_ + +#include <rte_io.h> + +#ifndef PCI_VENDOR_ID_CAVIUM +#define PCI_VENDOR_ID_CAVIUM 0x177D +#endif + +#define LIO_CN23XX_VF_VID 0x9712 + +/* --------------------------CONFIG VALUES------------------------ */ + +/* CN23xx IQ configuration macros */ +#define CN23XX_MAX_RINGS_PER_PF 64 +#define CN23XX_MAX_RINGS_PER_VF 8 + +#define CN23XX_MAX_INPUT_QUEUES CN23XX_MAX_RINGS_PER_PF +#define CN23XX_MAX_IQ_DESCRIPTORS 512 +#define CN23XX_MIN_IQ_DESCRIPTORS 128 + +#define CN23XX_MAX_OUTPUT_QUEUES CN23XX_MAX_RINGS_PER_PF +#define CN23XX_MAX_OQ_DESCRIPTORS 512 +#define CN23XX_MIN_OQ_DESCRIPTORS 128 +#define CN23XX_OQ_BUF_SIZE 1536 + +#define CN23XX_OQ_REFIL_THRESHOLD 16 + +#define CN23XX_DEFAULT_NUM_PORTS 1 + +#define CN23XX_CFG_IO_QUEUES CN23XX_MAX_RINGS_PER_PF + +/* common OCTEON configuration macros */ +#define OCTEON_64BYTE_INSTR 64 +#define OCTEON_OQ_INFOPTR_MODE 1 + +/* Max IOQs per LIO Link */ +#define LIO_MAX_IOQS_PER_IF 64 + +enum lio_card_type { + LIO_23XX /* 23xx */ +}; + +#define LIO_23XX_NAME "23xx" + +#define LIO_DEV_RUNNING 0xc + +#define LIO_OQ_REFILL_THRESHOLD_CFG(cfg) \ + ((cfg)->default_config->oq.refill_threshold) +#define LIO_NUM_DEF_TX_DESCS_CFG(cfg) \ + ((cfg)->default_config->num_def_tx_descs) + +#define LIO_IQ_INSTR_TYPE(cfg) ((cfg)->default_config->iq.instr_type) + +/* The following config values are fixed and should not be modified. */ + +/* Maximum number of Instruction queues */ +#define LIO_MAX_INSTR_QUEUES(lio_dev) CN23XX_MAX_RINGS_PER_VF + +#define LIO_MAX_POSSIBLE_INSTR_QUEUES CN23XX_MAX_INPUT_QUEUES +#define LIO_MAX_POSSIBLE_OUTPUT_QUEUES CN23XX_MAX_OUTPUT_QUEUES + +#define LIO_DEVICE_NAME_LEN 32 +#define LIO_BASE_MAJOR_VERSION 1 +#define LIO_BASE_MINOR_VERSION 5 +#define LIO_BASE_MICRO_VERSION 1 + +#define LIO_FW_VERSION_LENGTH 32 + +/** Tag types used by Octeon cores in its work. */ +enum octeon_tag_type { + OCTEON_ORDERED_TAG = 0, + OCTEON_ATOMIC_TAG = 1, +}; + +/* pre-defined host->NIC tag values */ +#define LIO_CONTROL (0x11111110) +#define LIO_DATA(i) (0x11111111 + (i)) + +/* used for NIC operations */ +#define LIO_OPCODE 1 + +/* Subcodes are used by host driver/apps to identify the sub-operation + * for the core. They only need to by unique for a given subsystem. + */ +#define LIO_OPCODE_SUBCODE(op, sub) \ + ((((op) & 0x0f) << 8) | ((sub) & 0x7f)) + +/** LIO_OPCODE subcodes */ +/* This subcode is sent by core PCI driver to indicate cores are ready. */ +#define LIO_OPCODE_NW_DATA 0x02 /* network packet data */ +#define LIO_OPCODE_CMD 0x03 +#define LIO_OPCODE_INFO 0x04 +#define LIO_OPCODE_PORT_STATS 0x05 +#define LIO_OPCODE_IF_CFG 0x09 + +#define LIO_MIN_RX_BUF_SIZE 64 +#define LIO_MAX_RX_PKTLEN (64 * 1024) + +/* NIC Command types */ +#define LIO_CMD_CHANGE_DEVFLAGS 0x3 +#define LIO_CMD_RX_CTL 0x4 +#define LIO_CMD_CLEAR_STATS 0x6 +#define LIO_CMD_SET_RSS 0xD +#define LIO_CMD_TNL_RX_CSUM_CTL 0x10 +#define LIO_CMD_TNL_TX_CSUM_CTL 0x11 +#define LIO_CMD_ADD_VLAN_FILTER 0x17 +#define LIO_CMD_DEL_VLAN_FILTER 0x18 +#define LIO_CMD_VXLAN_PORT_CONFIG 0x19 + +#define LIO_CMD_VXLAN_PORT_ADD 0x0 +#define LIO_CMD_VXLAN_PORT_DEL 0x1 +#define LIO_CMD_RXCSUM_ENABLE 0x0 +#define LIO_CMD_TXCSUM_ENABLE 0x0 + +/* RX(packets coming from wire) Checksum verification flags */ +/* TCP/UDP csum */ +#define LIO_L4_CSUM_VERIFIED 0x1 +#define LIO_IP_CSUM_VERIFIED 0x2 + +/* RSS */ +#define LIO_RSS_PARAM_DISABLE_RSS 0x10 +#define LIO_RSS_PARAM_HASH_KEY_UNCHANGED 0x08 +#define LIO_RSS_PARAM_ITABLE_UNCHANGED 0x04 +#define LIO_RSS_PARAM_HASH_INFO_UNCHANGED 0x02 + +#define LIO_RSS_HASH_IPV4 0x100 +#define LIO_RSS_HASH_TCP_IPV4 0x200 +#define LIO_RSS_HASH_IPV6 0x400 +#define LIO_RSS_HASH_TCP_IPV6 0x1000 +#define LIO_RSS_HASH_IPV6_EX 0x800 +#define LIO_RSS_HASH_TCP_IPV6_EX 0x2000 + +#define LIO_RSS_OFFLOAD_ALL ( \ + LIO_RSS_HASH_IPV4 | \ + LIO_RSS_HASH_TCP_IPV4 | \ + LIO_RSS_HASH_IPV6 | \ + LIO_RSS_HASH_TCP_IPV6 | \ + LIO_RSS_HASH_IPV6_EX | \ + LIO_RSS_HASH_TCP_IPV6_EX) + +#define LIO_RSS_MAX_TABLE_SZ 128 +#define LIO_RSS_MAX_KEY_SZ 40 +#define LIO_RSS_PARAM_SIZE 16 + +/* Interface flags communicated between host driver and core app. */ +enum lio_ifflags { + LIO_IFFLAG_ALLMULTI = 0x02, + LIO_IFFLAG_UNICAST = 0x10 +}; + +/* Routines for reading and writing CSRs */ +#ifdef RTE_LIBRTE_LIO_DEBUG_REGS +#define lio_write_csr(lio_dev, reg_off, value) \ + do { \ + typeof(lio_dev) _dev = lio_dev; \ + typeof(reg_off) _reg_off = reg_off; \ + typeof(value) _value = value; \ + PMD_REGS_LOG(_dev, \ + "Write32: Reg: 0x%08lx Val: 0x%08lx\n", \ + (unsigned long)_reg_off, \ + (unsigned long)_value); \ + rte_write32(_value, _dev->hw_addr + _reg_off); \ + } while (0) + +#define lio_write_csr64(lio_dev, reg_off, val64) \ + do { \ + typeof(lio_dev) _dev = lio_dev; \ + typeof(reg_off) _reg_off = reg_off; \ + typeof(val64) _val64 = val64; \ + PMD_REGS_LOG( \ + _dev, \ + "Write64: Reg: 0x%08lx Val: 0x%016llx\n", \ + (unsigned long)_reg_off, \ + (unsigned long long)_val64); \ + rte_write64(_val64, _dev->hw_addr + _reg_off); \ + } while (0) + +#define lio_read_csr(lio_dev, reg_off) \ + ({ \ + typeof(lio_dev) _dev = lio_dev; \ + typeof(reg_off) _reg_off = reg_off; \ + uint32_t val = rte_read32(_dev->hw_addr + _reg_off); \ + PMD_REGS_LOG(_dev, \ + "Read32: Reg: 0x%08lx Val: 0x%08lx\n", \ + (unsigned long)_reg_off, \ + (unsigned long)val); \ + val; \ + }) + +#define lio_read_csr64(lio_dev, reg_off) \ + ({ \ + typeof(lio_dev) _dev = lio_dev; \ + typeof(reg_off) _reg_off = reg_off; \ + uint64_t val64 = rte_read64(_dev->hw_addr + _reg_off); \ + PMD_REGS_LOG( \ + _dev, \ + "Read64: Reg: 0x%08lx Val: 0x%016llx\n", \ + (unsigned long)_reg_off, \ + (unsigned long long)val64); \ + val64; \ + }) +#else +#define lio_write_csr(lio_dev, reg_off, value) \ + rte_write32(value, (lio_dev)->hw_addr + (reg_off)) + +#define lio_write_csr64(lio_dev, reg_off, val64) \ + rte_write64(val64, (lio_dev)->hw_addr + (reg_off)) + +#define lio_read_csr(lio_dev, reg_off) \ + rte_read32((lio_dev)->hw_addr + (reg_off)) + +#define lio_read_csr64(lio_dev, reg_off) \ + rte_read64((lio_dev)->hw_addr + (reg_off)) +#endif +#endif /* _LIO_HW_DEFS_H_ */ diff --git a/drivers/net/liquidio/base/lio_mbox.c b/drivers/net/liquidio/base/lio_mbox.c new file mode 100644 index 00000000..b4abc623 --- /dev/null +++ b/drivers/net/liquidio/base/lio_mbox.c @@ -0,0 +1,275 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_ethdev.h> +#include <rte_cycles.h> + +#include "lio_logs.h" +#include "lio_struct.h" +#include "lio_mbox.h" + +/** + * lio_mbox_read: + * @mbox: Pointer mailbox + * + * Reads the 8-bytes of data from the mbox register + * Writes back the acknowledgment indicating completion of read + */ +int +lio_mbox_read(struct lio_mbox *mbox) +{ + union lio_mbox_message msg; + int ret = 0; + + msg.mbox_msg64 = rte_read64(mbox->mbox_read_reg); + + if ((msg.mbox_msg64 == LIO_PFVFACK) || (msg.mbox_msg64 == LIO_PFVFSIG)) + return 0; + + if (mbox->state & LIO_MBOX_STATE_REQ_RECEIVING) { + mbox->mbox_req.data[mbox->mbox_req.recv_len - 1] = + msg.mbox_msg64; + mbox->mbox_req.recv_len++; + } else { + if (mbox->state & LIO_MBOX_STATE_RES_RECEIVING) { + mbox->mbox_resp.data[mbox->mbox_resp.recv_len - 1] = + msg.mbox_msg64; + mbox->mbox_resp.recv_len++; + } else { + if ((mbox->state & LIO_MBOX_STATE_IDLE) && + (msg.s.type == LIO_MBOX_REQUEST)) { + mbox->state &= ~LIO_MBOX_STATE_IDLE; + mbox->state |= LIO_MBOX_STATE_REQ_RECEIVING; + mbox->mbox_req.msg.mbox_msg64 = msg.mbox_msg64; + mbox->mbox_req.q_no = mbox->q_no; + mbox->mbox_req.recv_len = 1; + } else { + if ((mbox->state & + LIO_MBOX_STATE_RES_PENDING) && + (msg.s.type == LIO_MBOX_RESPONSE)) { + mbox->state &= + ~LIO_MBOX_STATE_RES_PENDING; + mbox->state |= + LIO_MBOX_STATE_RES_RECEIVING; + mbox->mbox_resp.msg.mbox_msg64 = + msg.mbox_msg64; + mbox->mbox_resp.q_no = mbox->q_no; + mbox->mbox_resp.recv_len = 1; + } else { + rte_write64(LIO_PFVFERR, + mbox->mbox_read_reg); + mbox->state |= LIO_MBOX_STATE_ERROR; + return -1; + } + } + } + } + + if (mbox->state & LIO_MBOX_STATE_REQ_RECEIVING) { + if (mbox->mbox_req.recv_len < msg.s.len) { + ret = 0; + } else { + mbox->state &= ~LIO_MBOX_STATE_REQ_RECEIVING; + mbox->state |= LIO_MBOX_STATE_REQ_RECEIVED; + ret = 1; + } + } else { + if (mbox->state & LIO_MBOX_STATE_RES_RECEIVING) { + if (mbox->mbox_resp.recv_len < msg.s.len) { + ret = 0; + } else { + mbox->state &= ~LIO_MBOX_STATE_RES_RECEIVING; + mbox->state |= LIO_MBOX_STATE_RES_RECEIVED; + ret = 1; + } + } else { + RTE_ASSERT(0); + } + } + + rte_write64(LIO_PFVFACK, mbox->mbox_read_reg); + + return ret; +} + +/** + * lio_mbox_write: + * @lio_dev: Pointer lio device + * @mbox_cmd: Cmd to send to mailbox. + * + * Populates the queue specific mbox structure + * with cmd information. + * Write the cmd to mbox register + */ +int +lio_mbox_write(struct lio_device *lio_dev, + struct lio_mbox_cmd *mbox_cmd) +{ + struct lio_mbox *mbox = lio_dev->mbox[mbox_cmd->q_no]; + uint32_t count, i, ret = LIO_MBOX_STATUS_SUCCESS; + + if ((mbox_cmd->msg.s.type == LIO_MBOX_RESPONSE) && + !(mbox->state & LIO_MBOX_STATE_REQ_RECEIVED)) + return LIO_MBOX_STATUS_FAILED; + + if ((mbox_cmd->msg.s.type == LIO_MBOX_REQUEST) && + !(mbox->state & LIO_MBOX_STATE_IDLE)) + return LIO_MBOX_STATUS_BUSY; + + if (mbox_cmd->msg.s.type == LIO_MBOX_REQUEST) { + rte_memcpy(&mbox->mbox_resp, mbox_cmd, + sizeof(struct lio_mbox_cmd)); + mbox->state = LIO_MBOX_STATE_RES_PENDING; + } + + count = 0; + + while (rte_read64(mbox->mbox_write_reg) != LIO_PFVFSIG) { + rte_delay_ms(1); + if (count++ == 1000) { + ret = LIO_MBOX_STATUS_FAILED; + break; + } + } + + if (ret == LIO_MBOX_STATUS_SUCCESS) { + rte_write64(mbox_cmd->msg.mbox_msg64, mbox->mbox_write_reg); + for (i = 0; i < (uint32_t)(mbox_cmd->msg.s.len - 1); i++) { + count = 0; + while (rte_read64(mbox->mbox_write_reg) != + LIO_PFVFACK) { + rte_delay_ms(1); + if (count++ == 1000) { + ret = LIO_MBOX_STATUS_FAILED; + break; + } + } + rte_write64(mbox_cmd->data[i], mbox->mbox_write_reg); + } + } + + if (mbox_cmd->msg.s.type == LIO_MBOX_RESPONSE) { + mbox->state = LIO_MBOX_STATE_IDLE; + rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg); + } else { + if ((!mbox_cmd->msg.s.resp_needed) || + (ret == LIO_MBOX_STATUS_FAILED)) { + mbox->state &= ~LIO_MBOX_STATE_RES_PENDING; + if (!(mbox->state & (LIO_MBOX_STATE_REQ_RECEIVING | + LIO_MBOX_STATE_REQ_RECEIVED))) + mbox->state = LIO_MBOX_STATE_IDLE; + } + } + + return ret; +} + +/** + * lio_mbox_process_cmd: + * @mbox: Pointer mailbox + * @mbox_cmd: Pointer to command received + * + * Process the cmd received in mbox + */ +static int +lio_mbox_process_cmd(struct lio_mbox *mbox, + struct lio_mbox_cmd *mbox_cmd) +{ + struct lio_device *lio_dev = mbox->lio_dev; + + if (mbox_cmd->msg.s.cmd == LIO_CORES_CRASHED) + lio_dev_err(lio_dev, "Octeon core(s) crashed or got stuck!\n"); + + return 0; +} + +/** + * Process the received mbox message. + */ +int +lio_mbox_process_message(struct lio_mbox *mbox) +{ + struct lio_mbox_cmd mbox_cmd; + + if (mbox->state & LIO_MBOX_STATE_ERROR) { + if (mbox->state & (LIO_MBOX_STATE_RES_PENDING | + LIO_MBOX_STATE_RES_RECEIVING)) { + rte_memcpy(&mbox_cmd, &mbox->mbox_resp, + sizeof(struct lio_mbox_cmd)); + mbox->state = LIO_MBOX_STATE_IDLE; + rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg); + mbox_cmd.recv_status = 1; + if (mbox_cmd.fn) + mbox_cmd.fn(mbox->lio_dev, &mbox_cmd, + mbox_cmd.fn_arg); + + return 0; + } + + mbox->state = LIO_MBOX_STATE_IDLE; + rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg); + + return 0; + } + + if (mbox->state & LIO_MBOX_STATE_RES_RECEIVED) { + rte_memcpy(&mbox_cmd, &mbox->mbox_resp, + sizeof(struct lio_mbox_cmd)); + mbox->state = LIO_MBOX_STATE_IDLE; + rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg); + mbox_cmd.recv_status = 0; + if (mbox_cmd.fn) + mbox_cmd.fn(mbox->lio_dev, &mbox_cmd, mbox_cmd.fn_arg); + + return 0; + } + + if (mbox->state & LIO_MBOX_STATE_REQ_RECEIVED) { + rte_memcpy(&mbox_cmd, &mbox->mbox_req, + sizeof(struct lio_mbox_cmd)); + if (!mbox_cmd.msg.s.resp_needed) { + mbox->state &= ~LIO_MBOX_STATE_REQ_RECEIVED; + if (!(mbox->state & LIO_MBOX_STATE_RES_PENDING)) + mbox->state = LIO_MBOX_STATE_IDLE; + rte_write64(LIO_PFVFSIG, mbox->mbox_read_reg); + } + + lio_mbox_process_cmd(mbox, &mbox_cmd); + + return 0; + } + + RTE_ASSERT(0); + + return 0; +} diff --git a/drivers/net/liquidio/base/lio_mbox.h b/drivers/net/liquidio/base/lio_mbox.h new file mode 100644 index 00000000..b0875d64 --- /dev/null +++ b/drivers/net/liquidio/base/lio_mbox.h @@ -0,0 +1,131 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_MBOX_H_ +#define _LIO_MBOX_H_ + +#include <stdint.h> + +#include <rte_spinlock.h> + +/* Macros for Mail Box Communication */ + +#define LIO_MBOX_DATA_MAX 32 + +#define LIO_VF_ACTIVE 0x1 +#define LIO_VF_FLR_REQUEST 0x2 +#define LIO_CORES_CRASHED 0x3 + +/* Macro for Read acknowledgment */ +#define LIO_PFVFACK 0xffffffffffffffff +#define LIO_PFVFSIG 0x1122334455667788 +#define LIO_PFVFERR 0xDEADDEADDEADDEAD + +enum lio_mbox_cmd_status { + LIO_MBOX_STATUS_SUCCESS = 0, + LIO_MBOX_STATUS_FAILED = 1, + LIO_MBOX_STATUS_BUSY = 2 +}; + +enum lio_mbox_message_type { + LIO_MBOX_REQUEST = 0, + LIO_MBOX_RESPONSE = 1 +}; + +union lio_mbox_message { + uint64_t mbox_msg64; + struct { + uint16_t type : 1; + uint16_t resp_needed : 1; + uint16_t cmd : 6; + uint16_t len : 8; + uint8_t params[6]; + } s; +}; + +typedef void (*lio_mbox_callback)(void *, void *, void *); + +struct lio_mbox_cmd { + union lio_mbox_message msg; + uint64_t data[LIO_MBOX_DATA_MAX]; + uint32_t q_no; + uint32_t recv_len; + uint32_t recv_status; + lio_mbox_callback fn; + void *fn_arg; +}; + +enum lio_mbox_state { + LIO_MBOX_STATE_IDLE = 1, + LIO_MBOX_STATE_REQ_RECEIVING = 2, + LIO_MBOX_STATE_REQ_RECEIVED = 4, + LIO_MBOX_STATE_RES_PENDING = 8, + LIO_MBOX_STATE_RES_RECEIVING = 16, + LIO_MBOX_STATE_RES_RECEIVED = 16, + LIO_MBOX_STATE_ERROR = 32 +}; + +struct lio_mbox { + /* A spinlock to protect access to this q_mbox. */ + rte_spinlock_t lock; + + struct lio_device *lio_dev; + + uint32_t q_no; + + enum lio_mbox_state state; + + /* SLI_MAC_PF_MBOX_INT for PF, SLI_PKT_MBOX_INT for VF. */ + void *mbox_int_reg; + + /* SLI_PKT_PF_VF_MBOX_SIG(0) for PF, + * SLI_PKT_PF_VF_MBOX_SIG(1) for VF. + */ + void *mbox_write_reg; + + /* SLI_PKT_PF_VF_MBOX_SIG(1) for PF, + * SLI_PKT_PF_VF_MBOX_SIG(0) for VF. + */ + void *mbox_read_reg; + + struct lio_mbox_cmd mbox_req; + + struct lio_mbox_cmd mbox_resp; + +}; + +int lio_mbox_read(struct lio_mbox *mbox); +int lio_mbox_write(struct lio_device *lio_dev, + struct lio_mbox_cmd *mbox_cmd); +int lio_mbox_process_message(struct lio_mbox *mbox); +#endif /* _LIO_MBOX_H_ */ diff --git a/drivers/net/liquidio/lio_ethdev.c b/drivers/net/liquidio/lio_ethdev.c new file mode 100644 index 00000000..436d25b0 --- /dev/null +++ b/drivers/net/liquidio/lio_ethdev.c @@ -0,0 +1,2058 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_ethdev.h> +#include <rte_ethdev_pci.h> +#include <rte_cycles.h> +#include <rte_malloc.h> +#include <rte_alarm.h> + +#include "lio_logs.h" +#include "lio_23xx_vf.h" +#include "lio_ethdev.h" +#include "lio_rxtx.h" + +/* Default RSS key in use */ +static uint8_t lio_rss_key[40] = { + 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, + 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, + 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, + 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, + 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, +}; + +static const struct rte_eth_desc_lim lio_rx_desc_lim = { + .nb_max = CN23XX_MAX_OQ_DESCRIPTORS, + .nb_min = CN23XX_MIN_OQ_DESCRIPTORS, + .nb_align = 1, +}; + +static const struct rte_eth_desc_lim lio_tx_desc_lim = { + .nb_max = CN23XX_MAX_IQ_DESCRIPTORS, + .nb_min = CN23XX_MIN_IQ_DESCRIPTORS, + .nb_align = 1, +}; + +/* Wait for control command to reach nic. */ +static uint16_t +lio_wait_for_ctrl_cmd(struct lio_device *lio_dev, + struct lio_dev_ctrl_cmd *ctrl_cmd) +{ + uint16_t timeout = LIO_MAX_CMD_TIMEOUT; + + while ((ctrl_cmd->cond == 0) && --timeout) { + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + rte_delay_ms(1); + } + + return !timeout; +} + +/** + * \brief Send Rx control command + * @param eth_dev Pointer to the structure rte_eth_dev + * @param start_stop whether to start or stop + */ +static int +lio_send_rx_ctrl_cmd(struct rte_eth_dev *eth_dev, int start_stop) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_RX_CTL; + ctrl_pkt.ncmd.s.param1 = start_stop; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send RX Control message\n"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "RX Control command timed out\n"); + return -1; + } + + return 0; +} + +/* store statistics names and its offset in stats structure */ +struct rte_lio_xstats_name_off { + char name[RTE_ETH_XSTATS_NAME_SIZE]; + unsigned int offset; +}; + +static const struct rte_lio_xstats_name_off rte_lio_stats_strings[] = { + {"rx_pkts", offsetof(struct octeon_rx_stats, total_rcvd)}, + {"rx_bytes", offsetof(struct octeon_rx_stats, bytes_rcvd)}, + {"rx_broadcast_pkts", offsetof(struct octeon_rx_stats, total_bcst)}, + {"rx_multicast_pkts", offsetof(struct octeon_rx_stats, total_mcst)}, + {"rx_flow_ctrl_pkts", offsetof(struct octeon_rx_stats, ctl_rcvd)}, + {"rx_fifo_err", offsetof(struct octeon_rx_stats, fifo_err)}, + {"rx_dmac_drop", offsetof(struct octeon_rx_stats, dmac_drop)}, + {"rx_fcs_err", offsetof(struct octeon_rx_stats, fcs_err)}, + {"rx_jabber_err", offsetof(struct octeon_rx_stats, jabber_err)}, + {"rx_l2_err", offsetof(struct octeon_rx_stats, l2_err)}, + {"rx_vxlan_pkts", offsetof(struct octeon_rx_stats, fw_rx_vxlan)}, + {"rx_vxlan_err", offsetof(struct octeon_rx_stats, fw_rx_vxlan_err)}, + {"rx_lro_pkts", offsetof(struct octeon_rx_stats, fw_lro_pkts)}, + {"tx_pkts", (offsetof(struct octeon_tx_stats, total_pkts_sent)) + + sizeof(struct octeon_rx_stats)}, + {"tx_bytes", (offsetof(struct octeon_tx_stats, total_bytes_sent)) + + sizeof(struct octeon_rx_stats)}, + {"tx_broadcast_pkts", + (offsetof(struct octeon_tx_stats, bcast_pkts_sent)) + + sizeof(struct octeon_rx_stats)}, + {"tx_multicast_pkts", + (offsetof(struct octeon_tx_stats, mcast_pkts_sent)) + + sizeof(struct octeon_rx_stats)}, + {"tx_flow_ctrl_pkts", (offsetof(struct octeon_tx_stats, ctl_sent)) + + sizeof(struct octeon_rx_stats)}, + {"tx_fifo_err", (offsetof(struct octeon_tx_stats, fifo_err)) + + sizeof(struct octeon_rx_stats)}, + {"tx_total_collisions", (offsetof(struct octeon_tx_stats, + total_collisions)) + + sizeof(struct octeon_rx_stats)}, + {"tx_tso", (offsetof(struct octeon_tx_stats, fw_tso)) + + sizeof(struct octeon_rx_stats)}, + {"tx_vxlan_pkts", (offsetof(struct octeon_tx_stats, fw_tx_vxlan)) + + sizeof(struct octeon_rx_stats)}, +}; + +#define LIO_NB_XSTATS RTE_DIM(rte_lio_stats_strings) + +/* Get hw stats of the port */ +static int +lio_dev_xstats_get(struct rte_eth_dev *eth_dev, struct rte_eth_xstat *xstats, + unsigned int n) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + uint16_t timeout = LIO_MAX_CMD_TIMEOUT; + struct octeon_link_stats *hw_stats; + struct lio_link_stats_resp *resp; + struct lio_soft_command *sc; + uint32_t resp_size; + unsigned int i; + int retval; + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down\n", + lio_dev->port_id); + return -EINVAL; + } + + if (n < LIO_NB_XSTATS) + return LIO_NB_XSTATS; + + resp_size = sizeof(struct lio_link_stats_resp); + sc = lio_alloc_soft_command(lio_dev, 0, resp_size, 0); + if (sc == NULL) + return -ENOMEM; + + resp = (struct lio_link_stats_resp *)sc->virtrptr; + lio_prepare_soft_command(lio_dev, sc, LIO_OPCODE, + LIO_OPCODE_PORT_STATS, 0, 0, 0); + + /* Setting wait time in seconds */ + sc->wait_time = LIO_MAX_CMD_TIMEOUT / 1000; + + retval = lio_send_soft_command(lio_dev, sc); + if (retval == LIO_IQ_SEND_FAILED) { + lio_dev_err(lio_dev, "failed to get port stats from firmware. status: %x\n", + retval); + goto get_stats_fail; + } + + while ((*sc->status_word == LIO_COMPLETION_WORD_INIT) && --timeout) { + lio_flush_iq(lio_dev, lio_dev->instr_queue[sc->iq_no]); + lio_process_ordered_list(lio_dev); + rte_delay_ms(1); + } + + retval = resp->status; + if (retval) { + lio_dev_err(lio_dev, "failed to get port stats from firmware\n"); + goto get_stats_fail; + } + + lio_swap_8B_data((uint64_t *)(&resp->link_stats), + sizeof(struct octeon_link_stats) >> 3); + + hw_stats = &resp->link_stats; + + for (i = 0; i < LIO_NB_XSTATS; i++) { + xstats[i].id = i; + xstats[i].value = + *(uint64_t *)(((char *)hw_stats) + + rte_lio_stats_strings[i].offset); + } + + lio_free_soft_command(sc); + + return LIO_NB_XSTATS; + +get_stats_fail: + lio_free_soft_command(sc); + + return -1; +} + +static int +lio_dev_xstats_get_names(struct rte_eth_dev *eth_dev, + struct rte_eth_xstat_name *xstats_names, + unsigned limit __rte_unused) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + unsigned int i; + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down\n", + lio_dev->port_id); + return -EINVAL; + } + + if (xstats_names == NULL) + return LIO_NB_XSTATS; + + /* Note: limit checked in rte_eth_xstats_names() */ + + for (i = 0; i < LIO_NB_XSTATS; i++) { + snprintf(xstats_names[i].name, sizeof(xstats_names[i].name), + "%s", rte_lio_stats_strings[i].name); + } + + return LIO_NB_XSTATS; +} + +/* Reset hw stats for the port */ +static void +lio_dev_xstats_reset(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down\n", + lio_dev->port_id); + return; + } + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_CLEAR_STATS; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send clear stats command\n"); + return; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "Clear stats command timed out\n"); + return; + } + + /* clear stored per queue stats */ + RTE_FUNC_PTR_OR_RET(*eth_dev->dev_ops->stats_reset); + (*eth_dev->dev_ops->stats_reset)(eth_dev); +} + +/* Retrieve the device statistics (# packets in/out, # bytes in/out, etc */ +static void +lio_dev_stats_get(struct rte_eth_dev *eth_dev, + struct rte_eth_stats *stats) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_droq_stats *oq_stats; + struct lio_iq_stats *iq_stats; + struct lio_instr_queue *txq; + struct lio_droq *droq; + int i, iq_no, oq_no; + uint64_t bytes = 0; + uint64_t pkts = 0; + uint64_t drop = 0; + + for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { + iq_no = lio_dev->linfo.txpciq[i].s.q_no; + txq = lio_dev->instr_queue[iq_no]; + if (txq != NULL) { + iq_stats = &txq->stats; + pkts += iq_stats->tx_done; + drop += iq_stats->tx_dropped; + bytes += iq_stats->tx_tot_bytes; + } + } + + stats->opackets = pkts; + stats->obytes = bytes; + stats->oerrors = drop; + + pkts = 0; + drop = 0; + bytes = 0; + + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + oq_no = lio_dev->linfo.rxpciq[i].s.q_no; + droq = lio_dev->droq[oq_no]; + if (droq != NULL) { + oq_stats = &droq->stats; + pkts += oq_stats->rx_pkts_received; + drop += (oq_stats->rx_dropped + + oq_stats->dropped_toomany + + oq_stats->dropped_nomem); + bytes += oq_stats->rx_bytes_received; + } + } + stats->ibytes = bytes; + stats->ipackets = pkts; + stats->ierrors = drop; +} + +static void +lio_dev_stats_reset(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_droq_stats *oq_stats; + struct lio_iq_stats *iq_stats; + struct lio_instr_queue *txq; + struct lio_droq *droq; + int i, iq_no, oq_no; + + for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { + iq_no = lio_dev->linfo.txpciq[i].s.q_no; + txq = lio_dev->instr_queue[iq_no]; + if (txq != NULL) { + iq_stats = &txq->stats; + memset(iq_stats, 0, sizeof(struct lio_iq_stats)); + } + } + + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + oq_no = lio_dev->linfo.rxpciq[i].s.q_no; + droq = lio_dev->droq[oq_no]; + if (droq != NULL) { + oq_stats = &droq->stats; + memset(oq_stats, 0, sizeof(struct lio_droq_stats)); + } + } +} + +static void +lio_dev_info_get(struct rte_eth_dev *eth_dev, + struct rte_eth_dev_info *devinfo) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + devinfo->max_rx_queues = lio_dev->max_rx_queues; + devinfo->max_tx_queues = lio_dev->max_tx_queues; + + devinfo->min_rx_bufsize = LIO_MIN_RX_BUF_SIZE; + devinfo->max_rx_pktlen = LIO_MAX_RX_PKTLEN; + + devinfo->max_mac_addrs = 1; + + devinfo->rx_offload_capa = (DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM | + DEV_RX_OFFLOAD_VLAN_STRIP); + devinfo->tx_offload_capa = (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM | + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM); + + devinfo->rx_desc_lim = lio_rx_desc_lim; + devinfo->tx_desc_lim = lio_tx_desc_lim; + + devinfo->reta_size = LIO_RSS_MAX_TABLE_SZ; + devinfo->hash_key_size = LIO_RSS_MAX_KEY_SZ; + devinfo->flow_type_rss_offloads = (ETH_RSS_IPV4 | + ETH_RSS_NONFRAG_IPV4_TCP | + ETH_RSS_IPV6 | + ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_IPV6_EX | + ETH_RSS_IPV6_TCP_EX); +} + +static int +lio_dev_validate_vf_mtu(struct rte_eth_dev *eth_dev, uint16_t new_mtu) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + PMD_INIT_FUNC_TRACE(); + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down, can't check MTU\n", + lio_dev->port_id); + return -EINVAL; + } + + /* Limit the MTU to make sure the ethernet packets are between + * ETHER_MIN_MTU bytes and PF's MTU + */ + if ((new_mtu < ETHER_MIN_MTU) || + (new_mtu > lio_dev->linfo.link.s.mtu)) { + lio_dev_err(lio_dev, "Invalid MTU: %d\n", new_mtu); + lio_dev_err(lio_dev, "Valid range %d and %d\n", + ETHER_MIN_MTU, lio_dev->linfo.link.s.mtu); + return -EINVAL; + } + + return 0; +} + +static int +lio_dev_rss_reta_update(struct rte_eth_dev *eth_dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_rss_ctx *rss_state = &lio_dev->rss_state; + struct lio_rss_set *rss_param; + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + int i, j, index; + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down, can't update reta\n", + lio_dev->port_id); + return -EINVAL; + } + + if (reta_size != LIO_RSS_MAX_TABLE_SZ) { + lio_dev_err(lio_dev, + "The size of hash lookup table configured (%d) doesn't match the number hardware can supported (%d)\n", + reta_size, LIO_RSS_MAX_TABLE_SZ); + return -EINVAL; + } + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + rss_param = (struct lio_rss_set *)&ctrl_pkt.udd[0]; + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_SET_RSS; + ctrl_pkt.ncmd.s.more = sizeof(struct lio_rss_set) >> 3; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + rss_param->param.flags = 0xF; + rss_param->param.flags &= ~LIO_RSS_PARAM_ITABLE_UNCHANGED; + rss_param->param.itablesize = LIO_RSS_MAX_TABLE_SZ; + + for (i = 0; i < (reta_size / RTE_RETA_GROUP_SIZE); i++) { + for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) { + if ((reta_conf[i].mask) & ((uint64_t)1 << j)) { + index = (i * RTE_RETA_GROUP_SIZE) + j; + rss_state->itable[index] = reta_conf[i].reta[j]; + } + } + } + + rss_state->itable_size = LIO_RSS_MAX_TABLE_SZ; + memcpy(rss_param->itable, rss_state->itable, rss_state->itable_size); + + lio_swap_8B_data((uint64_t *)rss_param, LIO_RSS_PARAM_SIZE >> 3); + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to set rss hash\n"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "Set rss hash timed out\n"); + return -1; + } + + return 0; +} + +static int +lio_dev_rss_reta_query(struct rte_eth_dev *eth_dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_rss_ctx *rss_state = &lio_dev->rss_state; + int i, num; + + if (reta_size != LIO_RSS_MAX_TABLE_SZ) { + lio_dev_err(lio_dev, + "The size of hash lookup table configured (%d) doesn't match the number hardware can supported (%d)\n", + reta_size, LIO_RSS_MAX_TABLE_SZ); + return -EINVAL; + } + + num = reta_size / RTE_RETA_GROUP_SIZE; + + for (i = 0; i < num; i++) { + memcpy(reta_conf->reta, + &rss_state->itable[i * RTE_RETA_GROUP_SIZE], + RTE_RETA_GROUP_SIZE); + reta_conf++; + } + + return 0; +} + +static int +lio_dev_rss_hash_conf_get(struct rte_eth_dev *eth_dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_rss_ctx *rss_state = &lio_dev->rss_state; + uint8_t *hash_key = NULL; + uint64_t rss_hf = 0; + + if (rss_state->hash_disable) { + lio_dev_info(lio_dev, "RSS disabled in nic\n"); + rss_conf->rss_hf = 0; + return 0; + } + + /* Get key value */ + hash_key = rss_conf->rss_key; + if (hash_key != NULL) + memcpy(hash_key, rss_state->hash_key, rss_state->hash_key_size); + + if (rss_state->ip) + rss_hf |= ETH_RSS_IPV4; + if (rss_state->tcp_hash) + rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; + if (rss_state->ipv6) + rss_hf |= ETH_RSS_IPV6; + if (rss_state->ipv6_tcp_hash) + rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP; + if (rss_state->ipv6_ex) + rss_hf |= ETH_RSS_IPV6_EX; + if (rss_state->ipv6_tcp_ex_hash) + rss_hf |= ETH_RSS_IPV6_TCP_EX; + + rss_conf->rss_hf = rss_hf; + + return 0; +} + +static int +lio_dev_rss_hash_update(struct rte_eth_dev *eth_dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_rss_ctx *rss_state = &lio_dev->rss_state; + struct lio_rss_set *rss_param; + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down, can't update hash\n", + lio_dev->port_id); + return -EINVAL; + } + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + rss_param = (struct lio_rss_set *)&ctrl_pkt.udd[0]; + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_SET_RSS; + ctrl_pkt.ncmd.s.more = sizeof(struct lio_rss_set) >> 3; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + rss_param->param.flags = 0xF; + + if (rss_conf->rss_key) { + rss_param->param.flags &= ~LIO_RSS_PARAM_HASH_KEY_UNCHANGED; + rss_state->hash_key_size = LIO_RSS_MAX_KEY_SZ; + rss_param->param.hashkeysize = LIO_RSS_MAX_KEY_SZ; + memcpy(rss_state->hash_key, rss_conf->rss_key, + rss_state->hash_key_size); + memcpy(rss_param->key, rss_state->hash_key, + rss_state->hash_key_size); + } + + if ((rss_conf->rss_hf & LIO_RSS_OFFLOAD_ALL) == 0) { + /* Can't disable rss through hash flags, + * if it is enabled by default during init + */ + if (!rss_state->hash_disable) + return -EINVAL; + + /* This is for --disable-rss during testpmd launch */ + rss_param->param.flags |= LIO_RSS_PARAM_DISABLE_RSS; + } else { + uint32_t hashinfo = 0; + + /* Can't enable rss if disabled by default during init */ + if (rss_state->hash_disable) + return -EINVAL; + + if (rss_conf->rss_hf & ETH_RSS_IPV4) { + hashinfo |= LIO_RSS_HASH_IPV4; + rss_state->ip = 1; + } else { + rss_state->ip = 0; + } + + if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) { + hashinfo |= LIO_RSS_HASH_TCP_IPV4; + rss_state->tcp_hash = 1; + } else { + rss_state->tcp_hash = 0; + } + + if (rss_conf->rss_hf & ETH_RSS_IPV6) { + hashinfo |= LIO_RSS_HASH_IPV6; + rss_state->ipv6 = 1; + } else { + rss_state->ipv6 = 0; + } + + if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) { + hashinfo |= LIO_RSS_HASH_TCP_IPV6; + rss_state->ipv6_tcp_hash = 1; + } else { + rss_state->ipv6_tcp_hash = 0; + } + + if (rss_conf->rss_hf & ETH_RSS_IPV6_EX) { + hashinfo |= LIO_RSS_HASH_IPV6_EX; + rss_state->ipv6_ex = 1; + } else { + rss_state->ipv6_ex = 0; + } + + if (rss_conf->rss_hf & ETH_RSS_IPV6_TCP_EX) { + hashinfo |= LIO_RSS_HASH_TCP_IPV6_EX; + rss_state->ipv6_tcp_ex_hash = 1; + } else { + rss_state->ipv6_tcp_ex_hash = 0; + } + + rss_param->param.flags &= ~LIO_RSS_PARAM_HASH_INFO_UNCHANGED; + rss_param->param.hashinfo = hashinfo; + } + + lio_swap_8B_data((uint64_t *)rss_param, LIO_RSS_PARAM_SIZE >> 3); + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to set rss hash\n"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "Set rss hash timed out\n"); + return -1; + } + + return 0; +} + +/** + * Add vxlan dest udp port for an interface. + * + * @param eth_dev + * Pointer to the structure rte_eth_dev + * @param udp_tnl + * udp tunnel conf + * + * @return + * On success return 0 + * On failure return -1 + */ +static int +lio_dev_udp_tunnel_add(struct rte_eth_dev *eth_dev, + struct rte_eth_udp_tunnel *udp_tnl) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + if (udp_tnl == NULL) + return -EINVAL; + + if (udp_tnl->prot_type != RTE_TUNNEL_TYPE_VXLAN) { + lio_dev_err(lio_dev, "Unsupported tunnel type\n"); + return -1; + } + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_VXLAN_PORT_CONFIG; + ctrl_pkt.ncmd.s.param1 = udp_tnl->udp_port; + ctrl_pkt.ncmd.s.more = LIO_CMD_VXLAN_PORT_ADD; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send VXLAN_PORT_ADD command\n"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "VXLAN_PORT_ADD command timed out\n"); + return -1; + } + + return 0; +} + +/** + * Remove vxlan dest udp port for an interface. + * + * @param eth_dev + * Pointer to the structure rte_eth_dev + * @param udp_tnl + * udp tunnel conf + * + * @return + * On success return 0 + * On failure return -1 + */ +static int +lio_dev_udp_tunnel_del(struct rte_eth_dev *eth_dev, + struct rte_eth_udp_tunnel *udp_tnl) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + if (udp_tnl == NULL) + return -EINVAL; + + if (udp_tnl->prot_type != RTE_TUNNEL_TYPE_VXLAN) { + lio_dev_err(lio_dev, "Unsupported tunnel type\n"); + return -1; + } + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_VXLAN_PORT_CONFIG; + ctrl_pkt.ncmd.s.param1 = udp_tnl->udp_port; + ctrl_pkt.ncmd.s.more = LIO_CMD_VXLAN_PORT_DEL; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send VXLAN_PORT_DEL command\n"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "VXLAN_PORT_DEL command timed out\n"); + return -1; + } + + return 0; +} + +static int +lio_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id, int on) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + if (lio_dev->linfo.vlan_is_admin_assigned) + return -EPERM; + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = on ? + LIO_CMD_ADD_VLAN_FILTER : LIO_CMD_DEL_VLAN_FILTER; + ctrl_pkt.ncmd.s.param1 = vlan_id; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to %s VLAN port\n", + on ? "add" : "remove"); + return -1; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { + lio_dev_err(lio_dev, "Command to %s VLAN port timed out\n", + on ? "add" : "remove"); + return -1; + } + + return 0; +} + +/** + * Atomically writes the link status information into global + * structure rte_eth_dev. + * + * @param eth_dev + * - Pointer to the structure rte_eth_dev to read from. + * - Pointer to the buffer to be saved with the link status. + * + * @return + * - On success, zero. + * - On failure, negative value. + */ +static inline int +lio_dev_atomic_write_link_status(struct rte_eth_dev *eth_dev, + struct rte_eth_link *link) +{ + struct rte_eth_link *dst = ð_dev->data->dev_link; + struct rte_eth_link *src = link; + + if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, + *(uint64_t *)src) == 0) + return -1; + + return 0; +} + +static uint64_t +lio_hweight64(uint64_t w) +{ + uint64_t res = w - ((w >> 1) & 0x5555555555555555ul); + + res = + (res & 0x3333333333333333ul) + ((res >> 2) & 0x3333333333333333ul); + res = (res + (res >> 4)) & 0x0F0F0F0F0F0F0F0Ful; + res = res + (res >> 8); + res = res + (res >> 16); + + return (res + (res >> 32)) & 0x00000000000000FFul; +} + +static int +lio_dev_link_update(struct rte_eth_dev *eth_dev, + int wait_to_complete __rte_unused) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct rte_eth_link link, old; + + /* Initialize */ + link.link_status = ETH_LINK_DOWN; + link.link_speed = ETH_SPEED_NUM_NONE; + link.link_duplex = ETH_LINK_HALF_DUPLEX; + memset(&old, 0, sizeof(old)); + + /* Return what we found */ + if (lio_dev->linfo.link.s.link_up == 0) { + /* Interface is down */ + if (lio_dev_atomic_write_link_status(eth_dev, &link)) + return -1; + if (link.link_status == old.link_status) + return -1; + return 0; + } + + link.link_status = ETH_LINK_UP; /* Interface is up */ + link.link_duplex = ETH_LINK_FULL_DUPLEX; + switch (lio_dev->linfo.link.s.speed) { + case LIO_LINK_SPEED_10000: + link.link_speed = ETH_SPEED_NUM_10G; + break; + case LIO_LINK_SPEED_25000: + link.link_speed = ETH_SPEED_NUM_25G; + break; + default: + link.link_speed = ETH_SPEED_NUM_NONE; + link.link_duplex = ETH_LINK_HALF_DUPLEX; + } + + if (lio_dev_atomic_write_link_status(eth_dev, &link)) + return -1; + + if (link.link_status == old.link_status) + return -1; + + return 0; +} + +/** + * \brief Net device enable, disable allmulticast + * @param eth_dev Pointer to the structure rte_eth_dev + */ +static void +lio_change_dev_flag(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + /* Create a ctrl pkt command to be sent to core app. */ + ctrl_pkt.ncmd.s.cmd = LIO_CMD_CHANGE_DEVFLAGS; + ctrl_pkt.ncmd.s.param1 = lio_dev->ifflags; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send change flag message\n"); + return; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) + lio_dev_err(lio_dev, "Change dev flag command timed out\n"); +} + +static void +lio_dev_allmulticast_enable(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down, can't enable multicast\n", + lio_dev->port_id); + return; + } + + lio_dev->ifflags |= LIO_IFFLAG_ALLMULTI; + lio_change_dev_flag(eth_dev); +} + +static void +lio_dev_allmulticast_disable(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + if (!lio_dev->intf_open) { + lio_dev_err(lio_dev, "Port %d down, can't disable multicast\n", + lio_dev->port_id); + return; + } + + lio_dev->ifflags &= ~LIO_IFFLAG_ALLMULTI; + lio_change_dev_flag(eth_dev); +} + +static void +lio_dev_rss_configure(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_rss_ctx *rss_state = &lio_dev->rss_state; + struct rte_eth_rss_reta_entry64 reta_conf[8]; + struct rte_eth_rss_conf rss_conf; + uint16_t i; + + /* Configure the RSS key and the RSS protocols used to compute + * the RSS hash of input packets. + */ + rss_conf = eth_dev->data->dev_conf.rx_adv_conf.rss_conf; + if ((rss_conf.rss_hf & LIO_RSS_OFFLOAD_ALL) == 0) { + rss_state->hash_disable = 1; + lio_dev_rss_hash_update(eth_dev, &rss_conf); + return; + } + + if (rss_conf.rss_key == NULL) + rss_conf.rss_key = lio_rss_key; /* Default hash key */ + + lio_dev_rss_hash_update(eth_dev, &rss_conf); + + memset(reta_conf, 0, sizeof(reta_conf)); + for (i = 0; i < LIO_RSS_MAX_TABLE_SZ; i++) { + uint8_t q_idx, conf_idx, reta_idx; + + q_idx = (uint8_t)((eth_dev->data->nb_rx_queues > 1) ? + i % eth_dev->data->nb_rx_queues : 0); + conf_idx = i / RTE_RETA_GROUP_SIZE; + reta_idx = i % RTE_RETA_GROUP_SIZE; + reta_conf[conf_idx].reta[reta_idx] = q_idx; + reta_conf[conf_idx].mask |= ((uint64_t)1 << reta_idx); + } + + lio_dev_rss_reta_update(eth_dev, reta_conf, LIO_RSS_MAX_TABLE_SZ); +} + +static void +lio_dev_mq_rx_configure(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_rss_ctx *rss_state = &lio_dev->rss_state; + struct rte_eth_rss_conf rss_conf; + + switch (eth_dev->data->dev_conf.rxmode.mq_mode) { + case ETH_MQ_RX_RSS: + lio_dev_rss_configure(eth_dev); + break; + case ETH_MQ_RX_NONE: + /* if mq_mode is none, disable rss mode. */ + default: + memset(&rss_conf, 0, sizeof(rss_conf)); + rss_state->hash_disable = 1; + lio_dev_rss_hash_update(eth_dev, &rss_conf); + } +} + +/** + * Setup our receive queue/ringbuffer. This is the + * queue the Octeon uses to send us packets and + * responses. We are given a memory pool for our + * packet buffers that are used to populate the receive + * queue. + * + * @param eth_dev + * Pointer to the structure rte_eth_dev + * @param q_no + * Queue number + * @param num_rx_descs + * Number of entries in the queue + * @param socket_id + * Where to allocate memory + * @param rx_conf + * Pointer to the struction rte_eth_rxconf + * @param mp + * Pointer to the packet pool + * + * @return + * - On success, return 0 + * - On failure, return -1 + */ +static int +lio_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t q_no, + uint16_t num_rx_descs, unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf __rte_unused, + struct rte_mempool *mp) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct rte_pktmbuf_pool_private *mbp_priv; + uint32_t fw_mapped_oq; + uint16_t buf_size; + + if (q_no >= lio_dev->nb_rx_queues) { + lio_dev_err(lio_dev, "Invalid rx queue number %u\n", q_no); + return -EINVAL; + } + + lio_dev_dbg(lio_dev, "setting up rx queue %u\n", q_no); + + fw_mapped_oq = lio_dev->linfo.rxpciq[q_no].s.q_no; + + if ((lio_dev->droq[fw_mapped_oq]) && + (num_rx_descs != lio_dev->droq[fw_mapped_oq]->max_count)) { + lio_dev_err(lio_dev, + "Reconfiguring Rx descs not supported. Configure descs to same value %u or restart application\n", + lio_dev->droq[fw_mapped_oq]->max_count); + return -ENOTSUP; + } + + mbp_priv = rte_mempool_get_priv(mp); + buf_size = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM; + + if (lio_setup_droq(lio_dev, fw_mapped_oq, num_rx_descs, buf_size, mp, + socket_id)) { + lio_dev_err(lio_dev, "droq allocation failed\n"); + return -1; + } + + eth_dev->data->rx_queues[q_no] = lio_dev->droq[fw_mapped_oq]; + + return 0; +} + +/** + * Release the receive queue/ringbuffer. Called by + * the upper layers. + * + * @param rxq + * Opaque pointer to the receive queue to release + * + * @return + * - nothing + */ +void +lio_dev_rx_queue_release(void *rxq) +{ + struct lio_droq *droq = rxq; + int oq_no; + + if (droq) { + /* Run time queue deletion not supported */ + if (droq->lio_dev->port_configured) + return; + + oq_no = droq->q_no; + lio_delete_droq_queue(droq->lio_dev, oq_no); + } +} + +/** + * Allocate and initialize SW ring. Initialize associated HW registers. + * + * @param eth_dev + * Pointer to structure rte_eth_dev + * + * @param q_no + * Queue number + * + * @param num_tx_descs + * Number of ringbuffer descriptors + * + * @param socket_id + * NUMA socket id, used for memory allocations + * + * @param tx_conf + * Pointer to the structure rte_eth_txconf + * + * @return + * - On success, return 0 + * - On failure, return -errno value + */ +static int +lio_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t q_no, + uint16_t num_tx_descs, unsigned int socket_id, + const struct rte_eth_txconf *tx_conf __rte_unused) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + int fw_mapped_iq = lio_dev->linfo.txpciq[q_no].s.q_no; + int retval; + + if (q_no >= lio_dev->nb_tx_queues) { + lio_dev_err(lio_dev, "Invalid tx queue number %u\n", q_no); + return -EINVAL; + } + + lio_dev_dbg(lio_dev, "setting up tx queue %u\n", q_no); + + if ((lio_dev->instr_queue[fw_mapped_iq] != NULL) && + (num_tx_descs != lio_dev->instr_queue[fw_mapped_iq]->max_count)) { + lio_dev_err(lio_dev, + "Reconfiguring Tx descs not supported. Configure descs to same value %u or restart application\n", + lio_dev->instr_queue[fw_mapped_iq]->max_count); + return -ENOTSUP; + } + + retval = lio_setup_iq(lio_dev, q_no, lio_dev->linfo.txpciq[q_no], + num_tx_descs, lio_dev, socket_id); + + if (retval) { + lio_dev_err(lio_dev, "Runtime IQ(TxQ) creation failed.\n"); + return retval; + } + + retval = lio_setup_sglists(lio_dev, q_no, fw_mapped_iq, + lio_dev->instr_queue[fw_mapped_iq]->max_count, + socket_id); + + if (retval) { + lio_delete_instruction_queue(lio_dev, fw_mapped_iq); + return retval; + } + + eth_dev->data->tx_queues[q_no] = lio_dev->instr_queue[fw_mapped_iq]; + + return 0; +} + +/** + * Release the transmit queue/ringbuffer. Called by + * the upper layers. + * + * @param txq + * Opaque pointer to the transmit queue to release + * + * @return + * - nothing + */ +void +lio_dev_tx_queue_release(void *txq) +{ + struct lio_instr_queue *tq = txq; + uint32_t fw_mapped_iq_no; + + + if (tq) { + /* Run time queue deletion not supported */ + if (tq->lio_dev->port_configured) + return; + + /* Free sg_list */ + lio_delete_sglist(tq); + + fw_mapped_iq_no = tq->txpciq.s.q_no; + lio_delete_instruction_queue(tq->lio_dev, fw_mapped_iq_no); + } +} + +/** + * Api to check link state. + */ +static void +lio_dev_get_link_status(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + uint16_t timeout = LIO_MAX_CMD_TIMEOUT; + struct lio_link_status_resp *resp; + union octeon_link_status *ls; + struct lio_soft_command *sc; + uint32_t resp_size; + + if (!lio_dev->intf_open) + return; + + resp_size = sizeof(struct lio_link_status_resp); + sc = lio_alloc_soft_command(lio_dev, 0, resp_size, 0); + if (sc == NULL) + return; + + resp = (struct lio_link_status_resp *)sc->virtrptr; + lio_prepare_soft_command(lio_dev, sc, LIO_OPCODE, + LIO_OPCODE_INFO, 0, 0, 0); + + /* Setting wait time in seconds */ + sc->wait_time = LIO_MAX_CMD_TIMEOUT / 1000; + + if (lio_send_soft_command(lio_dev, sc) == LIO_IQ_SEND_FAILED) + goto get_status_fail; + + while ((*sc->status_word == LIO_COMPLETION_WORD_INIT) && --timeout) { + lio_flush_iq(lio_dev, lio_dev->instr_queue[sc->iq_no]); + rte_delay_ms(1); + } + + if (resp->status) + goto get_status_fail; + + ls = &resp->link_info.link; + + lio_swap_8B_data((uint64_t *)ls, sizeof(union octeon_link_status) >> 3); + + if (lio_dev->linfo.link.link_status64 != ls->link_status64) { + lio_dev->linfo.link.link_status64 = ls->link_status64; + lio_dev_link_update(eth_dev, 0); + } + + lio_free_soft_command(sc); + + return; + +get_status_fail: + lio_free_soft_command(sc); +} + +/* This function will be invoked every LSC_TIMEOUT ns (100ms) + * and will update link state if it changes. + */ +static void +lio_sync_link_state_check(void *eth_dev) +{ + struct lio_device *lio_dev = + (((struct rte_eth_dev *)eth_dev)->data->dev_private); + + if (lio_dev->port_configured) + lio_dev_get_link_status(eth_dev); + + /* Schedule periodic link status check. + * Stop check if interface is close and start again while opening. + */ + if (lio_dev->intf_open) + rte_eal_alarm_set(LIO_LSC_TIMEOUT, lio_sync_link_state_check, + eth_dev); +} + +static int +lio_dev_start(struct rte_eth_dev *eth_dev) +{ + uint16_t mtu = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; + struct lio_device *lio_dev = LIO_DEV(eth_dev); + uint16_t timeout = LIO_MAX_CMD_TIMEOUT; + int ret = 0; + + lio_dev_info(lio_dev, "Starting port %d\n", eth_dev->data->port_id); + + if (lio_dev->fn_list.enable_io_queues(lio_dev)) + return -1; + + if (lio_send_rx_ctrl_cmd(eth_dev, 1)) + return -1; + + /* Ready for link status updates */ + lio_dev->intf_open = 1; + rte_mb(); + + /* Configure RSS if device configured with multiple RX queues. */ + lio_dev_mq_rx_configure(eth_dev); + + /* start polling for lsc */ + ret = rte_eal_alarm_set(LIO_LSC_TIMEOUT, + lio_sync_link_state_check, + eth_dev); + if (ret) { + lio_dev_err(lio_dev, + "link state check handler creation failed\n"); + goto dev_lsc_handle_error; + } + + while ((lio_dev->linfo.link.link_status64 == 0) && (--timeout)) + rte_delay_ms(1); + + if (lio_dev->linfo.link.link_status64 == 0) { + ret = -1; + goto dev_mtu_check_error; + } + + if (lio_dev->linfo.link.s.mtu != mtu) { + ret = lio_dev_validate_vf_mtu(eth_dev, mtu); + if (ret) + goto dev_mtu_check_error; + } + + return 0; + +dev_mtu_check_error: + rte_eal_alarm_cancel(lio_sync_link_state_check, eth_dev); + +dev_lsc_handle_error: + lio_dev->intf_open = 0; + lio_send_rx_ctrl_cmd(eth_dev, 0); + + return ret; +} + +/* Stop device and disable input/output functions */ +static void +lio_dev_stop(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + lio_dev_info(lio_dev, "Stopping port %d\n", eth_dev->data->port_id); + lio_dev->intf_open = 0; + rte_mb(); + + /* Cancel callback if still running. */ + rte_eal_alarm_cancel(lio_sync_link_state_check, eth_dev); + + lio_send_rx_ctrl_cmd(eth_dev, 0); + + /* Clear recorded link status */ + lio_dev->linfo.link.link_status64 = 0; +} + +static int +lio_dev_set_link_up(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + if (!lio_dev->intf_open) { + lio_dev_info(lio_dev, "Port is stopped, Start the port first\n"); + return 0; + } + + if (lio_dev->linfo.link.s.link_up) { + lio_dev_info(lio_dev, "Link is already UP\n"); + return 0; + } + + if (lio_send_rx_ctrl_cmd(eth_dev, 1)) { + lio_dev_err(lio_dev, "Unable to set Link UP\n"); + return -1; + } + + lio_dev->linfo.link.s.link_up = 1; + eth_dev->data->dev_link.link_status = ETH_LINK_UP; + + return 0; +} + +static int +lio_dev_set_link_down(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + if (!lio_dev->intf_open) { + lio_dev_info(lio_dev, "Port is stopped, Start the port first\n"); + return 0; + } + + if (!lio_dev->linfo.link.s.link_up) { + lio_dev_info(lio_dev, "Link is already DOWN\n"); + return 0; + } + + lio_dev->linfo.link.s.link_up = 0; + eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; + + if (lio_send_rx_ctrl_cmd(eth_dev, 0)) { + lio_dev->linfo.link.s.link_up = 1; + eth_dev->data->dev_link.link_status = ETH_LINK_UP; + lio_dev_err(lio_dev, "Unable to set Link Down\n"); + return -1; + } + + return 0; +} + +/** + * Reset and stop the device. This occurs on the first + * call to this routine. Subsequent calls will simply + * return. NB: This will require the NIC to be rebooted. + * + * @param eth_dev + * Pointer to the structure rte_eth_dev + * + * @return + * - nothing + */ +static void +lio_dev_close(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + uint32_t i; + + lio_dev_info(lio_dev, "closing port %d\n", eth_dev->data->port_id); + + if (lio_dev->intf_open) + lio_dev_stop(eth_dev); + + lio_wait_for_instr_fetch(lio_dev); + + lio_dev->fn_list.disable_io_queues(lio_dev); + + cn23xx_vf_set_io_queues_off(lio_dev); + + /* Reset iq regs (IQ_DBELL). + * Clear sli_pktx_cnts (OQ_PKTS_SENT). + */ + for (i = 0; i < lio_dev->nb_rx_queues; i++) { + struct lio_droq *droq = lio_dev->droq[i]; + + if (droq == NULL) + break; + + uint32_t pkt_count = rte_read32(droq->pkts_sent_reg); + + lio_dev_dbg(lio_dev, + "pending oq count %u\n", pkt_count); + rte_write32(pkt_count, droq->pkts_sent_reg); + } + + /* Do FLR for the VF */ + cn23xx_vf_ask_pf_to_do_flr(lio_dev); + + /* lio_free_mbox */ + lio_dev->fn_list.free_mbox(lio_dev); + + /* Free glist resources */ + rte_free(lio_dev->glist_head); + rte_free(lio_dev->glist_lock); + lio_dev->glist_head = NULL; + lio_dev->glist_lock = NULL; + + lio_dev->port_configured = 0; + + /* Delete all queues */ + lio_dev_clear_queues(eth_dev); +} + +/** + * Enable tunnel rx checksum verification from firmware. + */ +static void +lio_enable_hw_tunnel_rx_checksum(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_TNL_RX_CSUM_CTL; + ctrl_pkt.ncmd.s.param1 = LIO_CMD_RXCSUM_ENABLE; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send TNL_RX_CSUM command\n"); + return; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) + lio_dev_err(lio_dev, "TNL_RX_CSUM command timed out\n"); +} + +/** + * Enable checksum calculation for inner packet in a tunnel. + */ +static void +lio_enable_hw_tunnel_tx_checksum(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + struct lio_dev_ctrl_cmd ctrl_cmd; + struct lio_ctrl_pkt ctrl_pkt; + + /* flush added to prevent cmd failure + * incase the queue is full + */ + lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); + + memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); + memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); + + ctrl_cmd.eth_dev = eth_dev; + ctrl_cmd.cond = 0; + + ctrl_pkt.ncmd.s.cmd = LIO_CMD_TNL_TX_CSUM_CTL; + ctrl_pkt.ncmd.s.param1 = LIO_CMD_TXCSUM_ENABLE; + ctrl_pkt.ctrl_cmd = &ctrl_cmd; + + if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { + lio_dev_err(lio_dev, "Failed to send TNL_TX_CSUM command\n"); + return; + } + + if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) + lio_dev_err(lio_dev, "TNL_TX_CSUM command timed out\n"); +} + +static int lio_dev_configure(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + uint16_t timeout = LIO_MAX_CMD_TIMEOUT; + int retval, num_iqueues, num_oqueues; + uint8_t mac[ETHER_ADDR_LEN], i; + struct lio_if_cfg_resp *resp; + struct lio_soft_command *sc; + union lio_if_cfg if_cfg; + uint32_t resp_size; + + PMD_INIT_FUNC_TRACE(); + + /* Re-configuring firmware not supported. + * Can't change tx/rx queues per port from initial value. + */ + if (lio_dev->port_configured) { + if ((lio_dev->nb_rx_queues != eth_dev->data->nb_rx_queues) || + (lio_dev->nb_tx_queues != eth_dev->data->nb_tx_queues)) { + lio_dev_err(lio_dev, + "rxq/txq re-conf not supported. Restart application with new value.\n"); + return -ENOTSUP; + } + return 0; + } + + lio_dev->nb_rx_queues = eth_dev->data->nb_rx_queues; + lio_dev->nb_tx_queues = eth_dev->data->nb_tx_queues; + + resp_size = sizeof(struct lio_if_cfg_resp); + sc = lio_alloc_soft_command(lio_dev, 0, resp_size, 0); + if (sc == NULL) + return -ENOMEM; + + resp = (struct lio_if_cfg_resp *)sc->virtrptr; + + /* Firmware doesn't have capability to reconfigure the queues, + * Claim all queues, and use as many required + */ + if_cfg.if_cfg64 = 0; + if_cfg.s.num_iqueues = lio_dev->nb_tx_queues; + if_cfg.s.num_oqueues = lio_dev->nb_rx_queues; + if_cfg.s.base_queue = 0; + + if_cfg.s.gmx_port_id = lio_dev->pf_num; + + lio_prepare_soft_command(lio_dev, sc, LIO_OPCODE, + LIO_OPCODE_IF_CFG, 0, + if_cfg.if_cfg64, 0); + + /* Setting wait time in seconds */ + sc->wait_time = LIO_MAX_CMD_TIMEOUT / 1000; + + retval = lio_send_soft_command(lio_dev, sc); + if (retval == LIO_IQ_SEND_FAILED) { + lio_dev_err(lio_dev, "iq/oq config failed status: %x\n", + retval); + /* Soft instr is freed by driver in case of failure. */ + goto nic_config_fail; + } + + /* Sleep on a wait queue till the cond flag indicates that the + * response arrived or timed-out. + */ + while ((*sc->status_word == LIO_COMPLETION_WORD_INIT) && --timeout) { + lio_flush_iq(lio_dev, lio_dev->instr_queue[sc->iq_no]); + lio_process_ordered_list(lio_dev); + rte_delay_ms(1); + } + + retval = resp->status; + if (retval) { + lio_dev_err(lio_dev, "iq/oq config failed\n"); + goto nic_config_fail; + } + + lio_swap_8B_data((uint64_t *)(&resp->cfg_info), + sizeof(struct octeon_if_cfg_info) >> 3); + + num_iqueues = lio_hweight64(resp->cfg_info.iqmask); + num_oqueues = lio_hweight64(resp->cfg_info.oqmask); + + if (!(num_iqueues) || !(num_oqueues)) { + lio_dev_err(lio_dev, + "Got bad iqueues (%016lx) or oqueues (%016lx) from firmware.\n", + (unsigned long)resp->cfg_info.iqmask, + (unsigned long)resp->cfg_info.oqmask); + goto nic_config_fail; + } + + lio_dev_dbg(lio_dev, + "interface %d, iqmask %016lx, oqmask %016lx, numiqueues %d, numoqueues %d\n", + eth_dev->data->port_id, + (unsigned long)resp->cfg_info.iqmask, + (unsigned long)resp->cfg_info.oqmask, + num_iqueues, num_oqueues); + + lio_dev->linfo.num_rxpciq = num_oqueues; + lio_dev->linfo.num_txpciq = num_iqueues; + + for (i = 0; i < num_oqueues; i++) { + lio_dev->linfo.rxpciq[i].rxpciq64 = + resp->cfg_info.linfo.rxpciq[i].rxpciq64; + lio_dev_dbg(lio_dev, "index %d OQ %d\n", + i, lio_dev->linfo.rxpciq[i].s.q_no); + } + + for (i = 0; i < num_iqueues; i++) { + lio_dev->linfo.txpciq[i].txpciq64 = + resp->cfg_info.linfo.txpciq[i].txpciq64; + lio_dev_dbg(lio_dev, "index %d IQ %d\n", + i, lio_dev->linfo.txpciq[i].s.q_no); + } + + lio_dev->linfo.hw_addr = resp->cfg_info.linfo.hw_addr; + lio_dev->linfo.gmxport = resp->cfg_info.linfo.gmxport; + lio_dev->linfo.link.link_status64 = + resp->cfg_info.linfo.link.link_status64; + + /* 64-bit swap required on LE machines */ + lio_swap_8B_data(&lio_dev->linfo.hw_addr, 1); + for (i = 0; i < ETHER_ADDR_LEN; i++) + mac[i] = *((uint8_t *)(((uint8_t *)&lio_dev->linfo.hw_addr) + + 2 + i)); + + /* Copy the permanent MAC address */ + ether_addr_copy((struct ether_addr *)mac, ð_dev->data->mac_addrs[0]); + + /* enable firmware checksum support for tunnel packets */ + lio_enable_hw_tunnel_rx_checksum(eth_dev); + lio_enable_hw_tunnel_tx_checksum(eth_dev); + + lio_dev->glist_lock = + rte_zmalloc(NULL, sizeof(*lio_dev->glist_lock) * num_iqueues, 0); + if (lio_dev->glist_lock == NULL) + return -ENOMEM; + + lio_dev->glist_head = + rte_zmalloc(NULL, sizeof(*lio_dev->glist_head) * num_iqueues, + 0); + if (lio_dev->glist_head == NULL) { + rte_free(lio_dev->glist_lock); + lio_dev->glist_lock = NULL; + return -ENOMEM; + } + + lio_dev_link_update(eth_dev, 0); + + lio_dev->port_configured = 1; + + lio_free_soft_command(sc); + + /* Disable iq_0 for reconf */ + lio_dev->fn_list.disable_io_queues(lio_dev); + + /* Reset ioq regs */ + lio_dev->fn_list.setup_device_regs(lio_dev); + + /* Free iq_0 used during init */ + lio_free_instr_queue0(lio_dev); + + return 0; + +nic_config_fail: + lio_dev_err(lio_dev, "Failed retval %d\n", retval); + lio_free_soft_command(sc); + lio_free_instr_queue0(lio_dev); + + return -ENODEV; +} + +/* Define our ethernet definitions */ +static const struct eth_dev_ops liovf_eth_dev_ops = { + .dev_configure = lio_dev_configure, + .dev_start = lio_dev_start, + .dev_stop = lio_dev_stop, + .dev_set_link_up = lio_dev_set_link_up, + .dev_set_link_down = lio_dev_set_link_down, + .dev_close = lio_dev_close, + .allmulticast_enable = lio_dev_allmulticast_enable, + .allmulticast_disable = lio_dev_allmulticast_disable, + .link_update = lio_dev_link_update, + .stats_get = lio_dev_stats_get, + .xstats_get = lio_dev_xstats_get, + .xstats_get_names = lio_dev_xstats_get_names, + .stats_reset = lio_dev_stats_reset, + .xstats_reset = lio_dev_xstats_reset, + .dev_infos_get = lio_dev_info_get, + .vlan_filter_set = lio_dev_vlan_filter_set, + .rx_queue_setup = lio_dev_rx_queue_setup, + .rx_queue_release = lio_dev_rx_queue_release, + .tx_queue_setup = lio_dev_tx_queue_setup, + .tx_queue_release = lio_dev_tx_queue_release, + .reta_update = lio_dev_rss_reta_update, + .reta_query = lio_dev_rss_reta_query, + .rss_hash_conf_get = lio_dev_rss_hash_conf_get, + .rss_hash_update = lio_dev_rss_hash_update, + .udp_tunnel_port_add = lio_dev_udp_tunnel_add, + .udp_tunnel_port_del = lio_dev_udp_tunnel_del, +}; + +static void +lio_check_pf_hs_response(void *lio_dev) +{ + struct lio_device *dev = lio_dev; + + /* check till response arrives */ + if (dev->pfvf_hsword.coproc_tics_per_us) + return; + + cn23xx_vf_handle_mbox(dev); + + rte_eal_alarm_set(1, lio_check_pf_hs_response, lio_dev); +} + +/** + * \brief Identify the LIO device and to map the BAR address space + * @param lio_dev lio device + */ +static int +lio_chip_specific_setup(struct lio_device *lio_dev) +{ + struct rte_pci_device *pdev = lio_dev->pci_dev; + uint32_t dev_id = pdev->id.device_id; + const char *s; + int ret = 1; + + switch (dev_id) { + case LIO_CN23XX_VF_VID: + lio_dev->chip_id = LIO_CN23XX_VF_VID; + ret = cn23xx_vf_setup_device(lio_dev); + s = "CN23XX VF"; + break; + default: + s = "?"; + lio_dev_err(lio_dev, "Unsupported Chip\n"); + } + + if (!ret) + lio_dev_info(lio_dev, "DEVICE : %s\n", s); + + return ret; +} + +static int +lio_first_time_init(struct lio_device *lio_dev, + struct rte_pci_device *pdev) +{ + int dpdk_queues; + + PMD_INIT_FUNC_TRACE(); + + /* set dpdk specific pci device pointer */ + lio_dev->pci_dev = pdev; + + /* Identify the LIO type and set device ops */ + if (lio_chip_specific_setup(lio_dev)) { + lio_dev_err(lio_dev, "Chip specific setup failed\n"); + return -1; + } + + /* Initialize soft command buffer pool */ + if (lio_setup_sc_buffer_pool(lio_dev)) { + lio_dev_err(lio_dev, "sc buffer pool allocation failed\n"); + return -1; + } + + /* Initialize lists to manage the requests of different types that + * arrive from applications for this lio device. + */ + lio_setup_response_list(lio_dev); + + if (lio_dev->fn_list.setup_mbox(lio_dev)) { + lio_dev_err(lio_dev, "Mailbox setup failed\n"); + goto error; + } + + /* Check PF response */ + lio_check_pf_hs_response((void *)lio_dev); + + /* Do handshake and exit if incompatible PF driver */ + if (cn23xx_pfvf_handshake(lio_dev)) + goto error; + + /* Initial reset */ + cn23xx_vf_ask_pf_to_do_flr(lio_dev); + /* Wait for FLR for 100ms per SRIOV specification */ + rte_delay_ms(100); + + if (cn23xx_vf_set_io_queues_off(lio_dev)) { + lio_dev_err(lio_dev, "Setting io queues off failed\n"); + goto error; + } + + if (lio_dev->fn_list.setup_device_regs(lio_dev)) { + lio_dev_err(lio_dev, "Failed to configure device registers\n"); + goto error; + } + + if (lio_setup_instr_queue0(lio_dev)) { + lio_dev_err(lio_dev, "Failed to setup instruction queue 0\n"); + goto error; + } + + dpdk_queues = (int)lio_dev->sriov_info.rings_per_vf; + + lio_dev->max_tx_queues = dpdk_queues; + lio_dev->max_rx_queues = dpdk_queues; + + /* Enable input and output queues for this device */ + if (lio_dev->fn_list.enable_io_queues(lio_dev)) + goto error; + + return 0; + +error: + lio_free_sc_buffer_pool(lio_dev); + if (lio_dev->mbox[0]) + lio_dev->fn_list.free_mbox(lio_dev); + if (lio_dev->instr_queue[0]) + lio_free_instr_queue0(lio_dev); + + return -1; +} + +static int +lio_eth_dev_uninit(struct rte_eth_dev *eth_dev) +{ + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + PMD_INIT_FUNC_TRACE(); + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return -EPERM; + + /* lio_free_sc_buffer_pool */ + lio_free_sc_buffer_pool(lio_dev); + + rte_free(eth_dev->data->mac_addrs); + eth_dev->data->mac_addrs = NULL; + + eth_dev->dev_ops = NULL; + eth_dev->rx_pkt_burst = NULL; + eth_dev->tx_pkt_burst = NULL; + + return 0; +} + +static int +lio_eth_dev_init(struct rte_eth_dev *eth_dev) +{ + struct rte_pci_device *pdev = RTE_DEV_TO_PCI(eth_dev->device); + struct lio_device *lio_dev = LIO_DEV(eth_dev); + + PMD_INIT_FUNC_TRACE(); + + eth_dev->rx_pkt_burst = &lio_dev_recv_pkts; + eth_dev->tx_pkt_burst = &lio_dev_xmit_pkts; + + /* Primary does the initialization. */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + rte_eth_copy_pci_info(eth_dev, pdev); + eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE; + + if (pdev->mem_resource[0].addr) { + lio_dev->hw_addr = pdev->mem_resource[0].addr; + } else { + PMD_INIT_LOG(ERR, "ERROR: Failed to map BAR0\n"); + return -ENODEV; + } + + lio_dev->eth_dev = eth_dev; + /* set lio device print string */ + snprintf(lio_dev->dev_string, sizeof(lio_dev->dev_string), + "%s[%02x:%02x.%x]", pdev->driver->driver.name, + pdev->addr.bus, pdev->addr.devid, pdev->addr.function); + + lio_dev->port_id = eth_dev->data->port_id; + + if (lio_first_time_init(lio_dev, pdev)) { + lio_dev_err(lio_dev, "Device init failed\n"); + return -EINVAL; + } + + eth_dev->dev_ops = &liovf_eth_dev_ops; + eth_dev->data->mac_addrs = rte_zmalloc("lio", ETHER_ADDR_LEN, 0); + if (eth_dev->data->mac_addrs == NULL) { + lio_dev_err(lio_dev, + "MAC addresses memory allocation failed\n"); + eth_dev->dev_ops = NULL; + eth_dev->rx_pkt_burst = NULL; + eth_dev->tx_pkt_burst = NULL; + return -ENOMEM; + } + + rte_atomic64_set(&lio_dev->status, LIO_DEV_RUNNING); + rte_wmb(); + + lio_dev->port_configured = 0; + /* Always allow unicast packets */ + lio_dev->ifflags |= LIO_IFFLAG_UNICAST; + + return 0; +} + +static int +lio_eth_dev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + struct rte_eth_dev *eth_dev; + int ret; + + eth_dev = rte_eth_dev_pci_allocate(pci_dev, + sizeof(struct lio_device)); + if (eth_dev == NULL) + return -ENOMEM; + + ret = lio_eth_dev_init(eth_dev); + if (ret) + rte_eth_dev_pci_release(eth_dev); + + return ret; +} + +static int +lio_eth_dev_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, + lio_eth_dev_uninit); +} + +/* Set of PCI devices this driver supports */ +static const struct rte_pci_id pci_id_liovf_map[] = { + { RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_VF_VID) }, + { .vendor_id = 0, /* sentinel */ } +}; + +static struct rte_pci_driver rte_liovf_pmd = { + .id_table = pci_id_liovf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = lio_eth_dev_pci_probe, + .remove = lio_eth_dev_pci_remove, +}; + +RTE_PMD_REGISTER_PCI(net_liovf, rte_liovf_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_liovf, pci_id_liovf_map); +RTE_PMD_REGISTER_KMOD_DEP(net_liovf, "* igb_uio | vfio"); diff --git a/drivers/net/liquidio/lio_ethdev.h b/drivers/net/liquidio/lio_ethdev.h new file mode 100644 index 00000000..655c2011 --- /dev/null +++ b/drivers/net/liquidio/lio_ethdev.h @@ -0,0 +1,205 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_ETHDEV_H_ +#define _LIO_ETHDEV_H_ + +#include <stdint.h> + +#include "lio_struct.h" + +/* timeout to check link state updates from firmware in us */ +#define LIO_LSC_TIMEOUT 100000 /* 100000us (100ms) */ +#define LIO_MAX_CMD_TIMEOUT 10000 /* 10000ms (10s) */ + +#define LIO_DEV(_eth_dev) ((_eth_dev)->data->dev_private) + +/* LIO Response condition variable */ +struct lio_dev_ctrl_cmd { + struct rte_eth_dev *eth_dev; + uint64_t cond; +}; + +enum lio_bus_speed { + LIO_LINK_SPEED_UNKNOWN = 0, + LIO_LINK_SPEED_10000 = 10000, + LIO_LINK_SPEED_25000 = 25000 +}; + +struct octeon_if_cfg_info { + uint64_t iqmask; /** mask for IQs enabled for the port */ + uint64_t oqmask; /** mask for OQs enabled for the port */ + struct octeon_link_info linfo; /** initial link information */ + char lio_firmware_version[LIO_FW_VERSION_LENGTH]; +}; + +/** Stats for each NIC port in RX direction. */ +struct octeon_rx_stats { + /* link-level stats */ + uint64_t total_rcvd; + uint64_t bytes_rcvd; + uint64_t total_bcst; + uint64_t total_mcst; + uint64_t runts; + uint64_t ctl_rcvd; + uint64_t fifo_err; /* Accounts for over/under-run of buffers */ + uint64_t dmac_drop; + uint64_t fcs_err; + uint64_t jabber_err; + uint64_t l2_err; + uint64_t frame_err; + + /* firmware stats */ + uint64_t fw_total_rcvd; + uint64_t fw_total_fwd; + uint64_t fw_total_fwd_bytes; + uint64_t fw_err_pko; + uint64_t fw_err_link; + uint64_t fw_err_drop; + uint64_t fw_rx_vxlan; + uint64_t fw_rx_vxlan_err; + + /* LRO */ + uint64_t fw_lro_pkts; /* Number of packets that are LROed */ + uint64_t fw_lro_octs; /* Number of octets that are LROed */ + uint64_t fw_total_lro; /* Number of LRO packets formed */ + uint64_t fw_lro_aborts; /* Number of times lRO of packet aborted */ + uint64_t fw_lro_aborts_port; + uint64_t fw_lro_aborts_seq; + uint64_t fw_lro_aborts_tsval; + uint64_t fw_lro_aborts_timer; + /* intrmod: packet forward rate */ + uint64_t fwd_rate; +}; + +/** Stats for each NIC port in RX direction. */ +struct octeon_tx_stats { + /* link-level stats */ + uint64_t total_pkts_sent; + uint64_t total_bytes_sent; + uint64_t mcast_pkts_sent; + uint64_t bcast_pkts_sent; + uint64_t ctl_sent; + uint64_t one_collision_sent; /* Packets sent after one collision */ + /* Packets sent after multiple collision */ + uint64_t multi_collision_sent; + /* Packets not sent due to max collisions */ + uint64_t max_collision_fail; + /* Packets not sent due to max deferrals */ + uint64_t max_deferral_fail; + /* Accounts for over/under-run of buffers */ + uint64_t fifo_err; + uint64_t runts; + uint64_t total_collisions; /* Total number of collisions detected */ + + /* firmware stats */ + uint64_t fw_total_sent; + uint64_t fw_total_fwd; + uint64_t fw_total_fwd_bytes; + uint64_t fw_err_pko; + uint64_t fw_err_link; + uint64_t fw_err_drop; + uint64_t fw_err_tso; + uint64_t fw_tso; /* number of tso requests */ + uint64_t fw_tso_fwd; /* number of packets segmented in tso */ + uint64_t fw_tx_vxlan; +}; + +struct octeon_link_stats { + struct octeon_rx_stats fromwire; + struct octeon_tx_stats fromhost; +}; + +union lio_if_cfg { + uint64_t if_cfg64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t base_queue : 16; + uint64_t num_iqueues : 16; + uint64_t num_oqueues : 16; + uint64_t gmx_port_id : 8; + uint64_t vf_id : 8; +#else + uint64_t vf_id : 8; + uint64_t gmx_port_id : 8; + uint64_t num_oqueues : 16; + uint64_t num_iqueues : 16; + uint64_t base_queue : 16; +#endif + } s; +}; + +struct lio_if_cfg_resp { + uint64_t rh; + struct octeon_if_cfg_info cfg_info; + uint64_t status; +}; + +struct lio_link_stats_resp { + uint64_t rh; + struct octeon_link_stats link_stats; + uint64_t status; +}; + +struct lio_link_status_resp { + uint64_t rh; + struct octeon_link_info link_info; + uint64_t status; +}; + +struct lio_rss_set { + struct param { +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + uint64_t flags : 16; + uint64_t hashinfo : 32; + uint64_t itablesize : 16; + uint64_t hashkeysize : 16; + uint64_t reserved : 48; +#elif RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t itablesize : 16; + uint64_t hashinfo : 32; + uint64_t flags : 16; + uint64_t reserved : 48; + uint64_t hashkeysize : 16; +#endif + } param; + + uint8_t itable[LIO_RSS_MAX_TABLE_SZ]; + uint8_t key[LIO_RSS_MAX_KEY_SZ]; +}; + +void lio_dev_rx_queue_release(void *rxq); + +void lio_dev_tx_queue_release(void *txq); + +#endif /* _LIO_ETHDEV_H_ */ diff --git a/drivers/net/liquidio/lio_logs.h b/drivers/net/liquidio/lio_logs.h new file mode 100644 index 00000000..a4c9ca4d --- /dev/null +++ b/drivers/net/liquidio/lio_logs.h @@ -0,0 +1,91 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_LOGS_H_ +#define _LIO_LOGS_H_ + +#define lio_dev_printf(lio_dev, level, fmt, args...) \ + RTE_LOG(level, PMD, "%s" fmt, (lio_dev)->dev_string, ##args) + +#define lio_dev_info(lio_dev, fmt, args...) \ + lio_dev_printf(lio_dev, INFO, "INFO: " fmt, ##args) + +#define lio_dev_err(lio_dev, fmt, args...) \ + lio_dev_printf(lio_dev, ERR, "ERROR: %s() " fmt, __func__, ##args) + +#define PMD_INIT_LOG(level, fmt, args...) RTE_LOG(level, PMD, fmt, ## args) + +/* Enable these through config options */ + +#ifdef RTE_LIBRTE_LIO_DEBUG_INIT +#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, "%s() >>\n", __func__) +#else /* !RTE_LIBRTE_LIO_DEBUG_INIT */ +#define PMD_INIT_FUNC_TRACE() do { } while (0) +#endif /* RTE_LIBRTE_LIO_DEBUG_INIT */ + +#ifdef RTE_LIBRTE_LIO_DEBUG_DRIVER +#define lio_dev_dbg(lio_dev, fmt, args...) \ + lio_dev_printf(lio_dev, DEBUG, "DEBUG: %s() " fmt, __func__, ##args) +#else /* !RTE_LIBRTE_LIO_DEBUG_DRIVER */ +#define lio_dev_dbg(lio_dev, fmt, args...) do { } while (0) +#endif /* RTE_LIBRTE_LIO_DEBUG_DRIVER */ + +#ifdef RTE_LIBRTE_LIO_DEBUG_RX +#define PMD_RX_LOG(lio_dev, level, fmt, args...) \ + lio_dev_printf(lio_dev, level, "RX: %s() " fmt, __func__, ##args) +#else /* !RTE_LIBRTE_LIO_DEBUG_RX */ +#define PMD_RX_LOG(lio_dev, level, fmt, args...) do { } while (0) +#endif /* RTE_LIBRTE_LIO_DEBUG_RX */ + +#ifdef RTE_LIBRTE_LIO_DEBUG_TX +#define PMD_TX_LOG(lio_dev, level, fmt, args...) \ + lio_dev_printf(lio_dev, level, "TX: %s() " fmt, __func__, ##args) +#else /* !RTE_LIBRTE_LIO_DEBUG_TX */ +#define PMD_TX_LOG(lio_dev, level, fmt, args...) do { } while (0) +#endif /* RTE_LIBRTE_LIO_DEBUG_TX */ + +#ifdef RTE_LIBRTE_LIO_DEBUG_MBOX +#define PMD_MBOX_LOG(lio_dev, level, fmt, args...) \ + lio_dev_printf(lio_dev, level, "MBOX: %s() " fmt, __func__, ##args) +#else /* !RTE_LIBRTE_LIO_DEBUG_MBOX */ +#define PMD_MBOX_LOG(level, fmt, args...) do { } while (0) +#endif /* RTE_LIBRTE_LIO_DEBUG_MBOX */ + +#ifdef RTE_LIBRTE_LIO_DEBUG_REGS +#define PMD_REGS_LOG(lio_dev, fmt, args...) \ + lio_dev_printf(lio_dev, DEBUG, "REGS: " fmt, ##args) +#else /* !RTE_LIBRTE_LIO_DEBUG_REGS */ +#define PMD_REGS_LOG(level, fmt, args...) do { } while (0) +#endif /* RTE_LIBRTE_LIO_DEBUG_REGS */ + +#endif /* _LIO_LOGS_H_ */ diff --git a/drivers/net/liquidio/lio_rxtx.c b/drivers/net/liquidio/lio_rxtx.c new file mode 100644 index 00000000..9533015c --- /dev/null +++ b/drivers/net/liquidio/lio_rxtx.c @@ -0,0 +1,1885 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_ethdev.h> +#include <rte_cycles.h> +#include <rte_malloc.h> + +#include "lio_logs.h" +#include "lio_struct.h" +#include "lio_ethdev.h" +#include "lio_rxtx.h" + +#define LIO_MAX_SG 12 +/* Flush iq if available tx_desc fall below LIO_FLUSH_WM */ +#define LIO_FLUSH_WM(_iq) ((_iq)->max_count / 2) +#define LIO_PKT_IN_DONE_CNT_MASK 0x00000000FFFFFFFFULL + +static void +lio_droq_compute_max_packet_bufs(struct lio_droq *droq) +{ + uint32_t count = 0; + + do { + count += droq->buffer_size; + } while (count < LIO_MAX_RX_PKTLEN); +} + +static void +lio_droq_reset_indices(struct lio_droq *droq) +{ + droq->read_idx = 0; + droq->write_idx = 0; + droq->refill_idx = 0; + droq->refill_count = 0; + rte_atomic64_set(&droq->pkts_pending, 0); +} + +static void +lio_droq_destroy_ring_buffers(struct lio_droq *droq) +{ + uint32_t i; + + for (i = 0; i < droq->max_count; i++) { + if (droq->recv_buf_list[i].buffer) { + rte_pktmbuf_free((struct rte_mbuf *) + droq->recv_buf_list[i].buffer); + droq->recv_buf_list[i].buffer = NULL; + } + } + + lio_droq_reset_indices(droq); +} + +static void * +lio_recv_buffer_alloc(struct lio_device *lio_dev, int q_no) +{ + struct lio_droq *droq = lio_dev->droq[q_no]; + struct rte_mempool *mpool = droq->mpool; + struct rte_mbuf *m; + + m = rte_pktmbuf_alloc(mpool); + if (m == NULL) { + lio_dev_err(lio_dev, "Cannot allocate\n"); + return NULL; + } + + rte_mbuf_refcnt_set(m, 1); + m->next = NULL; + m->data_off = RTE_PKTMBUF_HEADROOM; + m->nb_segs = 1; + m->pool = mpool; + + return m; +} + +static int +lio_droq_setup_ring_buffers(struct lio_device *lio_dev, + struct lio_droq *droq) +{ + struct lio_droq_desc *desc_ring = droq->desc_ring; + uint32_t i; + void *buf; + + for (i = 0; i < droq->max_count; i++) { + buf = lio_recv_buffer_alloc(lio_dev, droq->q_no); + if (buf == NULL) { + lio_dev_err(lio_dev, "buffer alloc failed\n"); + droq->stats.rx_alloc_failure++; + lio_droq_destroy_ring_buffers(droq); + return -ENOMEM; + } + + droq->recv_buf_list[i].buffer = buf; + droq->info_list[i].length = 0; + + /* map ring buffers into memory */ + desc_ring[i].info_ptr = lio_map_ring_info(droq, i); + desc_ring[i].buffer_ptr = + lio_map_ring(droq->recv_buf_list[i].buffer); + } + + lio_droq_reset_indices(droq); + + lio_droq_compute_max_packet_bufs(droq); + + return 0; +} + +static void +lio_dma_zone_free(struct lio_device *lio_dev, const struct rte_memzone *mz) +{ + const struct rte_memzone *mz_tmp; + int ret = 0; + + if (mz == NULL) { + lio_dev_err(lio_dev, "Memzone NULL\n"); + return; + } + + mz_tmp = rte_memzone_lookup(mz->name); + if (mz_tmp == NULL) { + lio_dev_err(lio_dev, "Memzone %s Not Found\n", mz->name); + return; + } + + ret = rte_memzone_free(mz); + if (ret) + lio_dev_err(lio_dev, "Memzone free Failed ret %d\n", ret); +} + +/** + * Frees the space for descriptor ring for the droq. + * + * @param lio_dev - pointer to the lio device structure + * @param q_no - droq no. + */ +static void +lio_delete_droq(struct lio_device *lio_dev, uint32_t q_no) +{ + struct lio_droq *droq = lio_dev->droq[q_no]; + + lio_dev_dbg(lio_dev, "OQ[%d]\n", q_no); + + lio_droq_destroy_ring_buffers(droq); + rte_free(droq->recv_buf_list); + droq->recv_buf_list = NULL; + lio_dma_zone_free(lio_dev, droq->info_mz); + lio_dma_zone_free(lio_dev, droq->desc_ring_mz); + + memset(droq, 0, LIO_DROQ_SIZE); +} + +static void * +lio_alloc_info_buffer(struct lio_device *lio_dev, + struct lio_droq *droq, unsigned int socket_id) +{ + droq->info_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev, + "info_list", droq->q_no, + (droq->max_count * + LIO_DROQ_INFO_SIZE), + RTE_CACHE_LINE_SIZE, + socket_id); + + if (droq->info_mz == NULL) + return NULL; + + droq->info_list_dma = droq->info_mz->phys_addr; + droq->info_alloc_size = droq->info_mz->len; + droq->info_base_addr = (size_t)droq->info_mz->addr; + + return droq->info_mz->addr; +} + +/** + * Allocates space for the descriptor ring for the droq and + * sets the base addr, num desc etc in Octeon registers. + * + * @param lio_dev - pointer to the lio device structure + * @param q_no - droq no. + * @param app_ctx - pointer to application context + * @return Success: 0 Failure: -1 + */ +static int +lio_init_droq(struct lio_device *lio_dev, uint32_t q_no, + uint32_t num_descs, uint32_t desc_size, + struct rte_mempool *mpool, unsigned int socket_id) +{ + uint32_t c_refill_threshold; + uint32_t desc_ring_size; + struct lio_droq *droq; + + lio_dev_dbg(lio_dev, "OQ[%d]\n", q_no); + + droq = lio_dev->droq[q_no]; + droq->lio_dev = lio_dev; + droq->q_no = q_no; + droq->mpool = mpool; + + c_refill_threshold = LIO_OQ_REFILL_THRESHOLD_CFG(lio_dev); + + droq->max_count = num_descs; + droq->buffer_size = desc_size; + + desc_ring_size = droq->max_count * LIO_DROQ_DESC_SIZE; + droq->desc_ring_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev, + "droq", q_no, + desc_ring_size, + RTE_CACHE_LINE_SIZE, + socket_id); + + if (droq->desc_ring_mz == NULL) { + lio_dev_err(lio_dev, + "Output queue %d ring alloc failed\n", q_no); + return -1; + } + + droq->desc_ring_dma = droq->desc_ring_mz->phys_addr; + droq->desc_ring = (struct lio_droq_desc *)droq->desc_ring_mz->addr; + + lio_dev_dbg(lio_dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", + q_no, droq->desc_ring, (unsigned long)droq->desc_ring_dma); + lio_dev_dbg(lio_dev, "droq[%d]: num_desc: %d\n", q_no, + droq->max_count); + + droq->info_list = lio_alloc_info_buffer(lio_dev, droq, socket_id); + if (droq->info_list == NULL) { + lio_dev_err(lio_dev, "Cannot allocate memory for info list.\n"); + goto init_droq_fail; + } + + droq->recv_buf_list = rte_zmalloc_socket("recv_buf_list", + (droq->max_count * + LIO_DROQ_RECVBUF_SIZE), + RTE_CACHE_LINE_SIZE, + socket_id); + if (droq->recv_buf_list == NULL) { + lio_dev_err(lio_dev, + "Output queue recv buf list alloc failed\n"); + goto init_droq_fail; + } + + if (lio_droq_setup_ring_buffers(lio_dev, droq)) + goto init_droq_fail; + + droq->refill_threshold = c_refill_threshold; + + rte_spinlock_init(&droq->lock); + + lio_dev->fn_list.setup_oq_regs(lio_dev, q_no); + + lio_dev->io_qmask.oq |= (1ULL << q_no); + + return 0; + +init_droq_fail: + lio_delete_droq(lio_dev, q_no); + + return -1; +} + +int +lio_setup_droq(struct lio_device *lio_dev, int oq_no, int num_descs, + int desc_size, struct rte_mempool *mpool, unsigned int socket_id) +{ + struct lio_droq *droq; + + PMD_INIT_FUNC_TRACE(); + + if (lio_dev->droq[oq_no]) { + lio_dev_dbg(lio_dev, "Droq %d in use\n", oq_no); + return 0; + } + + /* Allocate the DS for the new droq. */ + droq = rte_zmalloc_socket("ethdev RX queue", sizeof(*droq), + RTE_CACHE_LINE_SIZE, socket_id); + if (droq == NULL) + return -ENOMEM; + + lio_dev->droq[oq_no] = droq; + + /* Initialize the Droq */ + if (lio_init_droq(lio_dev, oq_no, num_descs, desc_size, mpool, + socket_id)) { + lio_dev_err(lio_dev, "Droq[%u] Initialization Failed\n", oq_no); + rte_free(lio_dev->droq[oq_no]); + lio_dev->droq[oq_no] = NULL; + return -ENOMEM; + } + + lio_dev->num_oqs++; + + lio_dev_dbg(lio_dev, "Total number of OQ: %d\n", lio_dev->num_oqs); + + /* Send credit for octeon output queues. credits are always + * sent after the output queue is enabled. + */ + rte_write32(lio_dev->droq[oq_no]->max_count, + lio_dev->droq[oq_no]->pkts_credit_reg); + rte_wmb(); + + return 0; +} + +static inline uint32_t +lio_droq_get_bufcount(uint32_t buf_size, uint32_t total_len) +{ + uint32_t buf_cnt = 0; + + while (total_len > (buf_size * buf_cnt)) + buf_cnt++; + + return buf_cnt; +} + +/* If we were not able to refill all buffers, try to move around + * the buffers that were not dispatched. + */ +static inline uint32_t +lio_droq_refill_pullup_descs(struct lio_droq *droq, + struct lio_droq_desc *desc_ring) +{ + uint32_t refill_index = droq->refill_idx; + uint32_t desc_refilled = 0; + + while (refill_index != droq->read_idx) { + if (droq->recv_buf_list[refill_index].buffer) { + droq->recv_buf_list[droq->refill_idx].buffer = + droq->recv_buf_list[refill_index].buffer; + desc_ring[droq->refill_idx].buffer_ptr = + desc_ring[refill_index].buffer_ptr; + droq->recv_buf_list[refill_index].buffer = NULL; + desc_ring[refill_index].buffer_ptr = 0; + do { + droq->refill_idx = lio_incr_index( + droq->refill_idx, 1, + droq->max_count); + desc_refilled++; + droq->refill_count--; + } while (droq->recv_buf_list[droq->refill_idx].buffer); + } + refill_index = lio_incr_index(refill_index, 1, + droq->max_count); + } /* while */ + + return desc_refilled; +} + +/* lio_droq_refill + * + * @param lio_dev - pointer to the lio device structure + * @param droq - droq in which descriptors require new buffers. + * + * Description: + * Called during normal DROQ processing in interrupt mode or by the poll + * thread to refill the descriptors from which buffers were dispatched + * to upper layers. Attempts to allocate new buffers. If that fails, moves + * up buffers (that were not dispatched) to form a contiguous ring. + * + * Returns: + * No of descriptors refilled. + * + * Locks: + * This routine is called with droq->lock held. + */ +static uint32_t +lio_droq_refill(struct lio_device *lio_dev, struct lio_droq *droq) +{ + struct lio_droq_desc *desc_ring; + uint32_t desc_refilled = 0; + void *buf = NULL; + + desc_ring = droq->desc_ring; + + while (droq->refill_count && (desc_refilled < droq->max_count)) { + /* If a valid buffer exists (happens if there is no dispatch), + * reuse the buffer, else allocate. + */ + if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) { + buf = lio_recv_buffer_alloc(lio_dev, droq->q_no); + /* If a buffer could not be allocated, no point in + * continuing + */ + if (buf == NULL) { + droq->stats.rx_alloc_failure++; + break; + } + + droq->recv_buf_list[droq->refill_idx].buffer = buf; + } + + desc_ring[droq->refill_idx].buffer_ptr = + lio_map_ring(droq->recv_buf_list[droq->refill_idx].buffer); + /* Reset any previous values in the length field. */ + droq->info_list[droq->refill_idx].length = 0; + + droq->refill_idx = lio_incr_index(droq->refill_idx, 1, + droq->max_count); + desc_refilled++; + droq->refill_count--; + } + + if (droq->refill_count) + desc_refilled += lio_droq_refill_pullup_descs(droq, desc_ring); + + /* if droq->refill_count + * The refill count would not change in pass two. We only moved buffers + * to close the gap in the ring, but we would still have the same no. of + * buffers to refill. + */ + return desc_refilled; +} + +static int +lio_droq_fast_process_packet(struct lio_device *lio_dev, + struct lio_droq *droq, + struct rte_mbuf **rx_pkts) +{ + struct rte_mbuf *nicbuf = NULL; + struct lio_droq_info *info; + uint32_t total_len = 0; + int data_total_len = 0; + uint32_t pkt_len = 0; + union octeon_rh *rh; + int data_pkts = 0; + + info = &droq->info_list[droq->read_idx]; + lio_swap_8B_data((uint64_t *)info, 2); + + if (!info->length) + return -1; + + /* Len of resp hdr in included in the received data len. */ + info->length -= OCTEON_RH_SIZE; + rh = &info->rh; + + total_len += (uint32_t)info->length; + + if (lio_opcode_slow_path(rh)) { + uint32_t buf_cnt; + + buf_cnt = lio_droq_get_bufcount(droq->buffer_size, + (uint32_t)info->length); + droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt, + droq->max_count); + droq->refill_count += buf_cnt; + } else { + if (info->length <= droq->buffer_size) { + if (rh->r_dh.has_hash) + pkt_len = (uint32_t)(info->length - 8); + else + pkt_len = (uint32_t)info->length; + + nicbuf = droq->recv_buf_list[droq->read_idx].buffer; + droq->recv_buf_list[droq->read_idx].buffer = NULL; + droq->read_idx = lio_incr_index( + droq->read_idx, 1, + droq->max_count); + droq->refill_count++; + + if (likely(nicbuf != NULL)) { + nicbuf->data_off = RTE_PKTMBUF_HEADROOM; + nicbuf->nb_segs = 1; + nicbuf->next = NULL; + /* We don't have a way to pass flags yet */ + nicbuf->ol_flags = 0; + if (rh->r_dh.has_hash) { + uint64_t *hash_ptr; + + nicbuf->ol_flags |= PKT_RX_RSS_HASH; + hash_ptr = rte_pktmbuf_mtod(nicbuf, + uint64_t *); + lio_swap_8B_data(hash_ptr, 1); + nicbuf->hash.rss = (uint32_t)*hash_ptr; + nicbuf->data_off += 8; + } + + nicbuf->pkt_len = pkt_len; + nicbuf->data_len = pkt_len; + nicbuf->port = lio_dev->port_id; + /* Store the mbuf */ + rx_pkts[data_pkts++] = nicbuf; + data_total_len += pkt_len; + } + + /* Prefetch buffer pointers when on a cache line + * boundary + */ + if ((droq->read_idx & 3) == 0) { + rte_prefetch0( + &droq->recv_buf_list[droq->read_idx]); + rte_prefetch0( + &droq->info_list[droq->read_idx]); + } + } else { + struct rte_mbuf *first_buf = NULL; + struct rte_mbuf *last_buf = NULL; + + while (pkt_len < info->length) { + int cpy_len = 0; + + cpy_len = ((pkt_len + droq->buffer_size) > + info->length) + ? ((uint32_t)info->length - + pkt_len) + : droq->buffer_size; + + nicbuf = + droq->recv_buf_list[droq->read_idx].buffer; + droq->recv_buf_list[droq->read_idx].buffer = + NULL; + + if (likely(nicbuf != NULL)) { + /* Note the first seg */ + if (!pkt_len) + first_buf = nicbuf; + + nicbuf->data_off = RTE_PKTMBUF_HEADROOM; + nicbuf->nb_segs = 1; + nicbuf->next = NULL; + nicbuf->port = lio_dev->port_id; + /* We don't have a way to pass + * flags yet + */ + nicbuf->ol_flags = 0; + if ((!pkt_len) && (rh->r_dh.has_hash)) { + uint64_t *hash_ptr; + + nicbuf->ol_flags |= + PKT_RX_RSS_HASH; + hash_ptr = rte_pktmbuf_mtod( + nicbuf, uint64_t *); + lio_swap_8B_data(hash_ptr, 1); + nicbuf->hash.rss = + (uint32_t)*hash_ptr; + nicbuf->data_off += 8; + nicbuf->pkt_len = cpy_len - 8; + nicbuf->data_len = cpy_len - 8; + } else { + nicbuf->pkt_len = cpy_len; + nicbuf->data_len = cpy_len; + } + + if (pkt_len) + first_buf->nb_segs++; + + if (last_buf) + last_buf->next = nicbuf; + + last_buf = nicbuf; + } else { + PMD_RX_LOG(lio_dev, ERR, "no buf\n"); + } + + pkt_len += cpy_len; + droq->read_idx = lio_incr_index( + droq->read_idx, + 1, droq->max_count); + droq->refill_count++; + + /* Prefetch buffer pointers when on a + * cache line boundary + */ + if ((droq->read_idx & 3) == 0) { + rte_prefetch0(&droq->recv_buf_list + [droq->read_idx]); + + rte_prefetch0( + &droq->info_list[droq->read_idx]); + } + } + rx_pkts[data_pkts++] = first_buf; + if (rh->r_dh.has_hash) + data_total_len += (pkt_len - 8); + else + data_total_len += pkt_len; + } + + /* Inform upper layer about packet checksum verification */ + struct rte_mbuf *m = rx_pkts[data_pkts - 1]; + + if (rh->r_dh.csum_verified & LIO_IP_CSUM_VERIFIED) + m->ol_flags |= PKT_RX_IP_CKSUM_GOOD; + + if (rh->r_dh.csum_verified & LIO_L4_CSUM_VERIFIED) + m->ol_flags |= PKT_RX_L4_CKSUM_GOOD; + } + + if (droq->refill_count >= droq->refill_threshold) { + int desc_refilled = lio_droq_refill(lio_dev, droq); + + /* Flush the droq descriptor data to memory to be sure + * that when we update the credits the data in memory is + * accurate. + */ + rte_wmb(); + rte_write32(desc_refilled, droq->pkts_credit_reg); + /* make sure mmio write completes */ + rte_wmb(); + } + + info->length = 0; + info->rh.rh64 = 0; + + droq->stats.pkts_received++; + droq->stats.rx_pkts_received += data_pkts; + droq->stats.rx_bytes_received += data_total_len; + droq->stats.bytes_received += total_len; + + return data_pkts; +} + +static uint32_t +lio_droq_fast_process_packets(struct lio_device *lio_dev, + struct lio_droq *droq, + struct rte_mbuf **rx_pkts, + uint32_t pkts_to_process) +{ + int ret, data_pkts = 0; + uint32_t pkt; + + for (pkt = 0; pkt < pkts_to_process; pkt++) { + ret = lio_droq_fast_process_packet(lio_dev, droq, + &rx_pkts[data_pkts]); + if (ret < 0) { + lio_dev_err(lio_dev, "Port[%d] DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", + lio_dev->port_id, droq->q_no, + droq->read_idx, pkts_to_process); + break; + } + data_pkts += ret; + } + + rte_atomic64_sub(&droq->pkts_pending, pkt); + + return data_pkts; +} + +static inline uint32_t +lio_droq_check_hw_for_pkts(struct lio_droq *droq) +{ + uint32_t last_count; + uint32_t pkt_count; + + pkt_count = rte_read32(droq->pkts_sent_reg); + + last_count = pkt_count - droq->pkt_count; + droq->pkt_count = pkt_count; + + if (last_count) + rte_atomic64_add(&droq->pkts_pending, last_count); + + return last_count; +} + +uint16_t +lio_dev_recv_pkts(void *rx_queue, + struct rte_mbuf **rx_pkts, + uint16_t budget) +{ + struct lio_droq *droq = rx_queue; + struct lio_device *lio_dev = droq->lio_dev; + uint32_t pkts_processed = 0; + uint32_t pkt_count = 0; + + lio_droq_check_hw_for_pkts(droq); + + pkt_count = rte_atomic64_read(&droq->pkts_pending); + if (!pkt_count) + return 0; + + if (pkt_count > budget) + pkt_count = budget; + + /* Grab the lock */ + rte_spinlock_lock(&droq->lock); + pkts_processed = lio_droq_fast_process_packets(lio_dev, + droq, rx_pkts, + pkt_count); + + if (droq->pkt_count) { + rte_write32(droq->pkt_count, droq->pkts_sent_reg); + droq->pkt_count = 0; + } + + /* Release the spin lock */ + rte_spinlock_unlock(&droq->lock); + + return pkts_processed; +} + +void +lio_delete_droq_queue(struct lio_device *lio_dev, + int oq_no) +{ + lio_delete_droq(lio_dev, oq_no); + lio_dev->num_oqs--; + rte_free(lio_dev->droq[oq_no]); + lio_dev->droq[oq_no] = NULL; +} + +/** + * lio_init_instr_queue() + * @param lio_dev - pointer to the lio device structure. + * @param txpciq - queue to be initialized. + * + * Called at driver init time for each input queue. iq_conf has the + * configuration parameters for the queue. + * + * @return Success: 0 Failure: -1 + */ +static int +lio_init_instr_queue(struct lio_device *lio_dev, + union octeon_txpciq txpciq, + uint32_t num_descs, unsigned int socket_id) +{ + uint32_t iq_no = (uint32_t)txpciq.s.q_no; + struct lio_instr_queue *iq; + uint32_t instr_type; + uint32_t q_size; + + instr_type = LIO_IQ_INSTR_TYPE(lio_dev); + + q_size = instr_type * num_descs; + iq = lio_dev->instr_queue[iq_no]; + iq->iq_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev, + "instr_queue", iq_no, q_size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (iq->iq_mz == NULL) { + lio_dev_err(lio_dev, "Cannot allocate memory for instr queue %d\n", + iq_no); + return -1; + } + + iq->base_addr_dma = iq->iq_mz->phys_addr; + iq->base_addr = (uint8_t *)iq->iq_mz->addr; + + iq->max_count = num_descs; + + /* Initialize a list to holds requests that have been posted to Octeon + * but has yet to be fetched by octeon + */ + iq->request_list = rte_zmalloc_socket("request_list", + sizeof(*iq->request_list) * + num_descs, + RTE_CACHE_LINE_SIZE, + socket_id); + if (iq->request_list == NULL) { + lio_dev_err(lio_dev, "Alloc failed for IQ[%d] nr free list\n", + iq_no); + lio_dma_zone_free(lio_dev, iq->iq_mz); + return -1; + } + + lio_dev_dbg(lio_dev, "IQ[%d]: base: %p basedma: %lx count: %d\n", + iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma, + iq->max_count); + + iq->lio_dev = lio_dev; + iq->txpciq.txpciq64 = txpciq.txpciq64; + iq->fill_cnt = 0; + iq->host_write_index = 0; + iq->lio_read_index = 0; + iq->flush_index = 0; + + rte_atomic64_set(&iq->instr_pending, 0); + + /* Initialize the spinlock for this instruction queue */ + rte_spinlock_init(&iq->lock); + rte_spinlock_init(&iq->post_lock); + + rte_atomic64_clear(&iq->iq_flush_running); + + lio_dev->io_qmask.iq |= (1ULL << iq_no); + + /* Set the 32B/64B mode for each input queue */ + lio_dev->io_qmask.iq64B |= ((instr_type == 64) << iq_no); + iq->iqcmd_64B = (instr_type == 64); + + lio_dev->fn_list.setup_iq_regs(lio_dev, iq_no); + + return 0; +} + +int +lio_setup_instr_queue0(struct lio_device *lio_dev) +{ + union octeon_txpciq txpciq; + uint32_t num_descs = 0; + uint32_t iq_no = 0; + + num_descs = LIO_NUM_DEF_TX_DESCS_CFG(lio_dev); + + lio_dev->num_iqs = 0; + + lio_dev->instr_queue[0] = rte_zmalloc(NULL, + sizeof(struct lio_instr_queue), 0); + if (lio_dev->instr_queue[0] == NULL) + return -ENOMEM; + + lio_dev->instr_queue[0]->q_index = 0; + lio_dev->instr_queue[0]->app_ctx = (void *)(size_t)0; + txpciq.txpciq64 = 0; + txpciq.s.q_no = iq_no; + txpciq.s.pkind = lio_dev->pfvf_hsword.pkind; + txpciq.s.use_qpg = 0; + txpciq.s.qpg = 0; + if (lio_init_instr_queue(lio_dev, txpciq, num_descs, SOCKET_ID_ANY)) { + rte_free(lio_dev->instr_queue[0]); + lio_dev->instr_queue[0] = NULL; + return -1; + } + + lio_dev->num_iqs++; + + return 0; +} + +/** + * lio_delete_instr_queue() + * @param lio_dev - pointer to the lio device structure. + * @param iq_no - queue to be deleted. + * + * Called at driver unload time for each input queue. Deletes all + * allocated resources for the input queue. + */ +static void +lio_delete_instr_queue(struct lio_device *lio_dev, uint32_t iq_no) +{ + struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; + + rte_free(iq->request_list); + iq->request_list = NULL; + lio_dma_zone_free(lio_dev, iq->iq_mz); +} + +void +lio_free_instr_queue0(struct lio_device *lio_dev) +{ + lio_delete_instr_queue(lio_dev, 0); + rte_free(lio_dev->instr_queue[0]); + lio_dev->instr_queue[0] = NULL; + lio_dev->num_iqs--; +} + +/* Return 0 on success, -1 on failure */ +int +lio_setup_iq(struct lio_device *lio_dev, int q_index, + union octeon_txpciq txpciq, uint32_t num_descs, void *app_ctx, + unsigned int socket_id) +{ + uint32_t iq_no = (uint32_t)txpciq.s.q_no; + + if (lio_dev->instr_queue[iq_no]) { + lio_dev_dbg(lio_dev, "IQ is in use. Cannot create the IQ: %d again\n", + iq_no); + lio_dev->instr_queue[iq_no]->txpciq.txpciq64 = txpciq.txpciq64; + lio_dev->instr_queue[iq_no]->app_ctx = app_ctx; + return 0; + } + + lio_dev->instr_queue[iq_no] = rte_zmalloc_socket("ethdev TX queue", + sizeof(struct lio_instr_queue), + RTE_CACHE_LINE_SIZE, socket_id); + if (lio_dev->instr_queue[iq_no] == NULL) + return -1; + + lio_dev->instr_queue[iq_no]->q_index = q_index; + lio_dev->instr_queue[iq_no]->app_ctx = app_ctx; + + if (lio_init_instr_queue(lio_dev, txpciq, num_descs, socket_id)) + goto release_lio_iq; + + lio_dev->num_iqs++; + if (lio_dev->fn_list.enable_io_queues(lio_dev)) + goto delete_lio_iq; + + return 0; + +delete_lio_iq: + lio_delete_instr_queue(lio_dev, iq_no); + lio_dev->num_iqs--; +release_lio_iq: + rte_free(lio_dev->instr_queue[iq_no]); + lio_dev->instr_queue[iq_no] = NULL; + + return -1; +} + +int +lio_wait_for_instr_fetch(struct lio_device *lio_dev) +{ + int pending, instr_cnt; + int i, retry = 1000; + + do { + instr_cnt = 0; + + for (i = 0; i < LIO_MAX_INSTR_QUEUES(lio_dev); i++) { + if (!(lio_dev->io_qmask.iq & (1ULL << i))) + continue; + + if (lio_dev->instr_queue[i] == NULL) + break; + + pending = rte_atomic64_read( + &lio_dev->instr_queue[i]->instr_pending); + if (pending) + lio_flush_iq(lio_dev, lio_dev->instr_queue[i]); + + instr_cnt += pending; + } + + if (instr_cnt == 0) + break; + + rte_delay_ms(1); + + } while (retry-- && instr_cnt); + + return instr_cnt; +} + +static inline void +lio_ring_doorbell(struct lio_device *lio_dev, + struct lio_instr_queue *iq) +{ + if (rte_atomic64_read(&lio_dev->status) == LIO_DEV_RUNNING) { + rte_write32(iq->fill_cnt, iq->doorbell_reg); + /* make sure doorbell write goes through */ + rte_wmb(); + iq->fill_cnt = 0; + } +} + +static inline void +copy_cmd_into_iq(struct lio_instr_queue *iq, uint8_t *cmd) +{ + uint8_t *iqptr, cmdsize; + + cmdsize = ((iq->iqcmd_64B) ? 64 : 32); + iqptr = iq->base_addr + (cmdsize * iq->host_write_index); + + rte_memcpy(iqptr, cmd, cmdsize); +} + +static inline struct lio_iq_post_status +post_command2(struct lio_instr_queue *iq, uint8_t *cmd) +{ + struct lio_iq_post_status st; + + st.status = LIO_IQ_SEND_OK; + + /* This ensures that the read index does not wrap around to the same + * position if queue gets full before Octeon could fetch any instr. + */ + if (rte_atomic64_read(&iq->instr_pending) >= + (int32_t)(iq->max_count - 1)) { + st.status = LIO_IQ_SEND_FAILED; + st.index = -1; + return st; + } + + if (rte_atomic64_read(&iq->instr_pending) >= + (int32_t)(iq->max_count - 2)) + st.status = LIO_IQ_SEND_STOP; + + copy_cmd_into_iq(iq, cmd); + + /* "index" is returned, host_write_index is modified. */ + st.index = iq->host_write_index; + iq->host_write_index = lio_incr_index(iq->host_write_index, 1, + iq->max_count); + iq->fill_cnt++; + + /* Flush the command into memory. We need to be sure the data is in + * memory before indicating that the instruction is pending. + */ + rte_wmb(); + + rte_atomic64_inc(&iq->instr_pending); + + return st; +} + +static inline void +lio_add_to_request_list(struct lio_instr_queue *iq, + int idx, void *buf, int reqtype) +{ + iq->request_list[idx].buf = buf; + iq->request_list[idx].reqtype = reqtype; +} + +static inline void +lio_free_netsgbuf(void *buf) +{ + struct lio_buf_free_info *finfo = buf; + struct lio_device *lio_dev = finfo->lio_dev; + struct rte_mbuf *m = finfo->mbuf; + struct lio_gather *g = finfo->g; + uint8_t iq = finfo->iq_no; + + /* This will take care of multiple segments also */ + rte_pktmbuf_free(m); + + rte_spinlock_lock(&lio_dev->glist_lock[iq]); + STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq], &g->list, entries); + rte_spinlock_unlock(&lio_dev->glist_lock[iq]); + rte_free(finfo); +} + +/* Can only run in process context */ +static int +lio_process_iq_request_list(struct lio_device *lio_dev, + struct lio_instr_queue *iq) +{ + struct octeon_instr_irh *irh = NULL; + uint32_t old = iq->flush_index; + struct lio_soft_command *sc; + uint32_t inst_count = 0; + int reqtype; + void *buf; + + while (old != iq->lio_read_index) { + reqtype = iq->request_list[old].reqtype; + buf = iq->request_list[old].buf; + + if (reqtype == LIO_REQTYPE_NONE) + goto skip_this; + + switch (reqtype) { + case LIO_REQTYPE_NORESP_NET: + rte_pktmbuf_free((struct rte_mbuf *)buf); + break; + case LIO_REQTYPE_NORESP_NET_SG: + lio_free_netsgbuf(buf); + break; + case LIO_REQTYPE_SOFT_COMMAND: + sc = buf; + irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; + if (irh->rflag) { + /* We're expecting a response from Octeon. + * It's up to lio_process_ordered_list() to + * process sc. Add sc to the ordered soft + * command response list because we expect + * a response from Octeon. + */ + rte_spinlock_lock(&lio_dev->response_list.lock); + rte_atomic64_inc( + &lio_dev->response_list.pending_req_count); + STAILQ_INSERT_TAIL( + &lio_dev->response_list.head, + &sc->node, entries); + rte_spinlock_unlock( + &lio_dev->response_list.lock); + } else { + if (sc->callback) { + /* This callback must not sleep */ + sc->callback(LIO_REQUEST_DONE, + sc->callback_arg); + } + } + break; + default: + lio_dev_err(lio_dev, + "Unknown reqtype: %d buf: %p at idx %d\n", + reqtype, buf, old); + } + + iq->request_list[old].buf = NULL; + iq->request_list[old].reqtype = 0; + +skip_this: + inst_count++; + old = lio_incr_index(old, 1, iq->max_count); + } + + iq->flush_index = old; + + return inst_count; +} + +static void +lio_update_read_index(struct lio_instr_queue *iq) +{ + uint32_t pkt_in_done = rte_read32(iq->inst_cnt_reg); + uint32_t last_done; + + last_done = pkt_in_done - iq->pkt_in_done; + iq->pkt_in_done = pkt_in_done; + + /* Add last_done and modulo with the IQ size to get new index */ + iq->lio_read_index = (iq->lio_read_index + + (uint32_t)(last_done & LIO_PKT_IN_DONE_CNT_MASK)) % + iq->max_count; +} + +int +lio_flush_iq(struct lio_device *lio_dev, struct lio_instr_queue *iq) +{ + uint32_t tot_inst_processed = 0; + uint32_t inst_processed = 0; + int tx_done = 1; + + if (rte_atomic64_test_and_set(&iq->iq_flush_running) == 0) + return tx_done; + + rte_spinlock_lock(&iq->lock); + + lio_update_read_index(iq); + + do { + /* Process any outstanding IQ packets. */ + if (iq->flush_index == iq->lio_read_index) + break; + + inst_processed = lio_process_iq_request_list(lio_dev, iq); + + if (inst_processed) { + rte_atomic64_sub(&iq->instr_pending, inst_processed); + iq->stats.instr_processed += inst_processed; + } + + tot_inst_processed += inst_processed; + inst_processed = 0; + + } while (1); + + rte_spinlock_unlock(&iq->lock); + + rte_atomic64_clear(&iq->iq_flush_running); + + return tx_done; +} + +static int +lio_send_command(struct lio_device *lio_dev, uint32_t iq_no, void *cmd, + void *buf, uint32_t datasize, uint32_t reqtype) +{ + struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; + struct lio_iq_post_status st; + + rte_spinlock_lock(&iq->post_lock); + + st = post_command2(iq, cmd); + + if (st.status != LIO_IQ_SEND_FAILED) { + lio_add_to_request_list(iq, st.index, buf, reqtype); + LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, bytes_sent, + datasize); + LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_posted, 1); + + lio_ring_doorbell(lio_dev, iq); + } else { + LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_dropped, 1); + } + + rte_spinlock_unlock(&iq->post_lock); + + return st.status; +} + +void +lio_prepare_soft_command(struct lio_device *lio_dev, + struct lio_soft_command *sc, uint8_t opcode, + uint8_t subcode, uint32_t irh_ossp, uint64_t ossp0, + uint64_t ossp1) +{ + struct octeon_instr_pki_ih3 *pki_ih3; + struct octeon_instr_ih3 *ih3; + struct octeon_instr_irh *irh; + struct octeon_instr_rdp *rdp; + + RTE_ASSERT(opcode <= 15); + RTE_ASSERT(subcode <= 127); + + ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; + + ih3->pkind = lio_dev->instr_queue[sc->iq_no]->txpciq.s.pkind; + + pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3; + + pki_ih3->w = 1; + pki_ih3->raw = 1; + pki_ih3->utag = 1; + pki_ih3->uqpg = lio_dev->instr_queue[sc->iq_no]->txpciq.s.use_qpg; + pki_ih3->utt = 1; + + pki_ih3->tag = LIO_CONTROL; + pki_ih3->tagtype = OCTEON_ATOMIC_TAG; + pki_ih3->qpg = lio_dev->instr_queue[sc->iq_no]->txpciq.s.qpg; + pki_ih3->pm = 0x7; + pki_ih3->sl = 8; + + if (sc->datasize) + ih3->dlengsz = sc->datasize; + + irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; + irh->opcode = opcode; + irh->subcode = subcode; + + /* opcode/subcode specific parameters (ossp) */ + irh->ossp = irh_ossp; + sc->cmd.cmd3.ossp[0] = ossp0; + sc->cmd.cmd3.ossp[1] = ossp1; + + if (sc->rdatasize) { + rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp; + rdp->pcie_port = lio_dev->pcie_port; + rdp->rlen = sc->rdatasize; + irh->rflag = 1; + /* PKI IH3 */ + ih3->fsz = OCTEON_SOFT_CMD_RESP_IH3; + } else { + irh->rflag = 0; + /* PKI IH3 */ + ih3->fsz = OCTEON_PCI_CMD_O3; + } +} + +int +lio_send_soft_command(struct lio_device *lio_dev, + struct lio_soft_command *sc) +{ + struct octeon_instr_ih3 *ih3; + struct octeon_instr_irh *irh; + uint32_t len = 0; + + ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; + if (ih3->dlengsz) { + RTE_ASSERT(sc->dmadptr); + sc->cmd.cmd3.dptr = sc->dmadptr; + } + + irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; + if (irh->rflag) { + RTE_ASSERT(sc->dmarptr); + RTE_ASSERT(sc->status_word != NULL); + *sc->status_word = LIO_COMPLETION_WORD_INIT; + sc->cmd.cmd3.rptr = sc->dmarptr; + } + + len = (uint32_t)ih3->dlengsz; + + if (sc->wait_time) + sc->timeout = lio_uptime + sc->wait_time; + + return lio_send_command(lio_dev, sc->iq_no, &sc->cmd, sc, len, + LIO_REQTYPE_SOFT_COMMAND); +} + +int +lio_setup_sc_buffer_pool(struct lio_device *lio_dev) +{ + char sc_pool_name[RTE_MEMPOOL_NAMESIZE]; + uint16_t buf_size; + + buf_size = LIO_SOFT_COMMAND_BUFFER_SIZE + RTE_PKTMBUF_HEADROOM; + snprintf(sc_pool_name, sizeof(sc_pool_name), + "lio_sc_pool_%u", lio_dev->port_id); + lio_dev->sc_buf_pool = rte_pktmbuf_pool_create(sc_pool_name, + LIO_MAX_SOFT_COMMAND_BUFFERS, + 0, 0, buf_size, SOCKET_ID_ANY); + return 0; +} + +void +lio_free_sc_buffer_pool(struct lio_device *lio_dev) +{ + rte_mempool_free(lio_dev->sc_buf_pool); +} + +struct lio_soft_command * +lio_alloc_soft_command(struct lio_device *lio_dev, uint32_t datasize, + uint32_t rdatasize, uint32_t ctxsize) +{ + uint32_t offset = sizeof(struct lio_soft_command); + struct lio_soft_command *sc; + struct rte_mbuf *m; + uint64_t dma_addr; + + RTE_ASSERT((offset + datasize + rdatasize + ctxsize) <= + LIO_SOFT_COMMAND_BUFFER_SIZE); + + m = rte_pktmbuf_alloc(lio_dev->sc_buf_pool); + if (m == NULL) { + lio_dev_err(lio_dev, "Cannot allocate mbuf for sc\n"); + return NULL; + } + + /* set rte_mbuf data size and there is only 1 segment */ + m->pkt_len = LIO_SOFT_COMMAND_BUFFER_SIZE; + m->data_len = LIO_SOFT_COMMAND_BUFFER_SIZE; + + /* use rte_mbuf buffer for soft command */ + sc = rte_pktmbuf_mtod(m, struct lio_soft_command *); + memset(sc, 0, LIO_SOFT_COMMAND_BUFFER_SIZE); + sc->size = LIO_SOFT_COMMAND_BUFFER_SIZE; + sc->dma_addr = rte_mbuf_data_dma_addr(m); + sc->mbuf = m; + + dma_addr = sc->dma_addr; + + if (ctxsize) { + sc->ctxptr = (uint8_t *)sc + offset; + sc->ctxsize = ctxsize; + } + + /* Start data at 128 byte boundary */ + offset = (offset + ctxsize + 127) & 0xffffff80; + + if (datasize) { + sc->virtdptr = (uint8_t *)sc + offset; + sc->dmadptr = dma_addr + offset; + sc->datasize = datasize; + } + + /* Start rdata at 128 byte boundary */ + offset = (offset + datasize + 127) & 0xffffff80; + + if (rdatasize) { + RTE_ASSERT(rdatasize >= 16); + sc->virtrptr = (uint8_t *)sc + offset; + sc->dmarptr = dma_addr + offset; + sc->rdatasize = rdatasize; + sc->status_word = (uint64_t *)((uint8_t *)(sc->virtrptr) + + rdatasize - 8); + } + + return sc; +} + +void +lio_free_soft_command(struct lio_soft_command *sc) +{ + rte_pktmbuf_free(sc->mbuf); +} + +void +lio_setup_response_list(struct lio_device *lio_dev) +{ + STAILQ_INIT(&lio_dev->response_list.head); + rte_spinlock_init(&lio_dev->response_list.lock); + rte_atomic64_set(&lio_dev->response_list.pending_req_count, 0); +} + +int +lio_process_ordered_list(struct lio_device *lio_dev) +{ + int resp_to_process = LIO_MAX_ORD_REQS_TO_PROCESS; + struct lio_response_list *ordered_sc_list; + struct lio_soft_command *sc; + int request_complete = 0; + uint64_t status64; + uint32_t status; + + ordered_sc_list = &lio_dev->response_list; + + do { + rte_spinlock_lock(&ordered_sc_list->lock); + + if (STAILQ_EMPTY(&ordered_sc_list->head)) { + /* ordered_sc_list is empty; there is + * nothing to process + */ + rte_spinlock_unlock(&ordered_sc_list->lock); + return -1; + } + + sc = LIO_STQUEUE_FIRST_ENTRY(&ordered_sc_list->head, + struct lio_soft_command, node); + + status = LIO_REQUEST_PENDING; + + /* check if octeon has finished DMA'ing a response + * to where rptr is pointing to + */ + status64 = *sc->status_word; + + if (status64 != LIO_COMPLETION_WORD_INIT) { + /* This logic ensures that all 64b have been written. + * 1. check byte 0 for non-FF + * 2. if non-FF, then swap result from BE to host order + * 3. check byte 7 (swapped to 0) for non-FF + * 4. if non-FF, use the low 32-bit status code + * 5. if either byte 0 or byte 7 is FF, don't use status + */ + if ((status64 & 0xff) != 0xff) { + lio_swap_8B_data(&status64, 1); + if (((status64 & 0xff) != 0xff)) { + /* retrieve 16-bit firmware status */ + status = (uint32_t)(status64 & + 0xffffULL); + if (status) { + status = + LIO_FIRMWARE_STATUS_CODE( + status); + } else { + /* i.e. no error */ + status = LIO_REQUEST_DONE; + } + } + } + } else if ((sc->timeout && lio_check_timeout(lio_uptime, + sc->timeout))) { + lio_dev_err(lio_dev, + "cmd failed, timeout (%ld, %ld)\n", + (long)lio_uptime, (long)sc->timeout); + status = LIO_REQUEST_TIMEOUT; + } + + if (status != LIO_REQUEST_PENDING) { + /* we have received a response or we have timed out. + * remove node from linked list + */ + STAILQ_REMOVE(&ordered_sc_list->head, + &sc->node, lio_stailq_node, entries); + rte_atomic64_dec( + &lio_dev->response_list.pending_req_count); + rte_spinlock_unlock(&ordered_sc_list->lock); + + if (sc->callback) + sc->callback(status, sc->callback_arg); + + request_complete++; + } else { + /* no response yet */ + request_complete = 0; + rte_spinlock_unlock(&ordered_sc_list->lock); + } + + /* If we hit the Max Ordered requests to process every loop, + * we quit and let this function be invoked the next time + * the poll thread runs to process the remaining requests. + * This function can take up the entire CPU if there is + * no upper limit to the requests processed. + */ + if (request_complete >= resp_to_process) + break; + } while (request_complete); + + return 0; +} + +static inline struct lio_stailq_node * +list_delete_first_node(struct lio_stailq_head *head) +{ + struct lio_stailq_node *node; + + if (STAILQ_EMPTY(head)) + node = NULL; + else + node = STAILQ_FIRST(head); + + if (node) + STAILQ_REMOVE(head, node, lio_stailq_node, entries); + + return node; +} + +void +lio_delete_sglist(struct lio_instr_queue *txq) +{ + struct lio_device *lio_dev = txq->lio_dev; + int iq_no = txq->q_index; + struct lio_gather *g; + + if (lio_dev->glist_head == NULL) + return; + + do { + g = (struct lio_gather *)list_delete_first_node( + &lio_dev->glist_head[iq_no]); + if (g) { + if (g->sg) + rte_free( + (void *)((unsigned long)g->sg - g->adjust)); + rte_free(g); + } + } while (g); +} + +/** + * \brief Setup gather lists + * @param lio per-network private data + */ +int +lio_setup_sglists(struct lio_device *lio_dev, int iq_no, + int fw_mapped_iq, int num_descs, unsigned int socket_id) +{ + struct lio_gather *g; + int i; + + rte_spinlock_init(&lio_dev->glist_lock[iq_no]); + + STAILQ_INIT(&lio_dev->glist_head[iq_no]); + + for (i = 0; i < num_descs; i++) { + g = rte_zmalloc_socket(NULL, sizeof(*g), RTE_CACHE_LINE_SIZE, + socket_id); + if (g == NULL) { + lio_dev_err(lio_dev, + "lio_gather memory allocation failed for qno %d\n", + iq_no); + break; + } + + g->sg_size = + ((ROUNDUP4(LIO_MAX_SG) >> 2) * LIO_SG_ENTRY_SIZE); + + g->sg = rte_zmalloc_socket(NULL, g->sg_size + 8, + RTE_CACHE_LINE_SIZE, socket_id); + if (g->sg == NULL) { + lio_dev_err(lio_dev, + "sg list memory allocation failed for qno %d\n", + iq_no); + rte_free(g); + break; + } + + /* The gather component should be aligned on 64-bit boundary */ + if (((unsigned long)g->sg) & 7) { + g->adjust = 8 - (((unsigned long)g->sg) & 7); + g->sg = + (struct lio_sg_entry *)((unsigned long)g->sg + + g->adjust); + } + + STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq_no], &g->list, + entries); + } + + if (i != num_descs) { + lio_delete_sglist(lio_dev->instr_queue[fw_mapped_iq]); + return -ENOMEM; + } + + return 0; +} + +void +lio_delete_instruction_queue(struct lio_device *lio_dev, int iq_no) +{ + lio_delete_instr_queue(lio_dev, iq_no); + rte_free(lio_dev->instr_queue[iq_no]); + lio_dev->instr_queue[iq_no] = NULL; + lio_dev->num_iqs--; +} + +static inline uint32_t +lio_iq_get_available(struct lio_device *lio_dev, uint32_t q_no) +{ + return ((lio_dev->instr_queue[q_no]->max_count - 1) - + (uint32_t)rte_atomic64_read( + &lio_dev->instr_queue[q_no]->instr_pending)); +} + +static inline int +lio_iq_is_full(struct lio_device *lio_dev, uint32_t q_no) +{ + return ((uint32_t)rte_atomic64_read( + &lio_dev->instr_queue[q_no]->instr_pending) >= + (lio_dev->instr_queue[q_no]->max_count - 2)); +} + +static int +lio_dev_cleanup_iq(struct lio_device *lio_dev, int iq_no) +{ + struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; + uint32_t count = 10000; + + while ((lio_iq_get_available(lio_dev, iq_no) < LIO_FLUSH_WM(iq)) && + --count) + lio_flush_iq(lio_dev, iq); + + return count ? 0 : 1; +} + +static void +lio_ctrl_cmd_callback(uint32_t status __rte_unused, void *sc_ptr) +{ + struct lio_soft_command *sc = sc_ptr; + struct lio_dev_ctrl_cmd *ctrl_cmd; + struct lio_ctrl_pkt *ctrl_pkt; + + ctrl_pkt = (struct lio_ctrl_pkt *)sc->ctxptr; + ctrl_cmd = ctrl_pkt->ctrl_cmd; + ctrl_cmd->cond = 1; + + lio_free_soft_command(sc); +} + +static inline struct lio_soft_command * +lio_alloc_ctrl_pkt_sc(struct lio_device *lio_dev, + struct lio_ctrl_pkt *ctrl_pkt) +{ + struct lio_soft_command *sc = NULL; + uint32_t uddsize, datasize; + uint32_t rdatasize; + uint8_t *data; + + uddsize = (uint32_t)(ctrl_pkt->ncmd.s.more * 8); + + datasize = OCTEON_CMD_SIZE + uddsize; + rdatasize = (ctrl_pkt->wait_time) ? 16 : 0; + + sc = lio_alloc_soft_command(lio_dev, datasize, + rdatasize, sizeof(struct lio_ctrl_pkt)); + if (sc == NULL) + return NULL; + + rte_memcpy(sc->ctxptr, ctrl_pkt, sizeof(struct lio_ctrl_pkt)); + + data = (uint8_t *)sc->virtdptr; + + rte_memcpy(data, &ctrl_pkt->ncmd, OCTEON_CMD_SIZE); + + lio_swap_8B_data((uint64_t *)data, OCTEON_CMD_SIZE >> 3); + + if (uddsize) { + /* Endian-Swap for UDD should have been done by caller. */ + rte_memcpy(data + OCTEON_CMD_SIZE, ctrl_pkt->udd, uddsize); + } + + sc->iq_no = (uint32_t)ctrl_pkt->iq_no; + + lio_prepare_soft_command(lio_dev, sc, + LIO_OPCODE, LIO_OPCODE_CMD, + 0, 0, 0); + + sc->callback = lio_ctrl_cmd_callback; + sc->callback_arg = sc; + sc->wait_time = ctrl_pkt->wait_time; + + return sc; +} + +int +lio_send_ctrl_pkt(struct lio_device *lio_dev, struct lio_ctrl_pkt *ctrl_pkt) +{ + struct lio_soft_command *sc = NULL; + int retval; + + sc = lio_alloc_ctrl_pkt_sc(lio_dev, ctrl_pkt); + if (sc == NULL) { + lio_dev_err(lio_dev, "soft command allocation failed\n"); + return -1; + } + + retval = lio_send_soft_command(lio_dev, sc); + if (retval == LIO_IQ_SEND_FAILED) { + lio_free_soft_command(sc); + lio_dev_err(lio_dev, "Port: %d soft command: %d send failed status: %x\n", + lio_dev->port_id, ctrl_pkt->ncmd.s.cmd, retval); + return -1; + } + + return retval; +} + +/** Send data packet to the device + * @param lio_dev - lio device pointer + * @param ndata - control structure with queueing, and buffer information + * + * @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if it the + * queue should be stopped, and LIO_IQ_SEND_OK if it sent okay. + */ +static inline int +lio_send_data_pkt(struct lio_device *lio_dev, struct lio_data_pkt *ndata) +{ + return lio_send_command(lio_dev, ndata->q_no, &ndata->cmd, + ndata->buf, ndata->datasize, ndata->reqtype); +} + +uint16_t +lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts) +{ + struct lio_instr_queue *txq = tx_queue; + union lio_cmd_setup cmdsetup; + struct lio_device *lio_dev; + struct lio_iq_stats *stats; + struct lio_data_pkt ndata; + int i, processed = 0; + struct rte_mbuf *m; + uint32_t tag = 0; + int status = 0; + int iq_no; + + lio_dev = txq->lio_dev; + iq_no = txq->txpciq.s.q_no; + stats = &lio_dev->instr_queue[iq_no]->stats; + + if (!lio_dev->intf_open || !lio_dev->linfo.link.s.link_up) { + PMD_TX_LOG(lio_dev, ERR, "Transmit failed link_status : %d\n", + lio_dev->linfo.link.s.link_up); + goto xmit_failed; + } + + lio_dev_cleanup_iq(lio_dev, iq_no); + + for (i = 0; i < nb_pkts; i++) { + uint32_t pkt_len = 0; + + m = pkts[i]; + + /* Prepare the attributes for the data to be passed to BASE. */ + memset(&ndata, 0, sizeof(struct lio_data_pkt)); + + ndata.buf = m; + + ndata.q_no = iq_no; + if (lio_iq_is_full(lio_dev, ndata.q_no)) { + stats->tx_iq_busy++; + if (lio_dev_cleanup_iq(lio_dev, iq_no)) { + PMD_TX_LOG(lio_dev, ERR, + "Transmit failed iq:%d full\n", + ndata.q_no); + break; + } + } + + cmdsetup.cmd_setup64 = 0; + cmdsetup.s.iq_no = iq_no; + + /* check checksum offload flags to form cmd */ + if (m->ol_flags & PKT_TX_IP_CKSUM) + cmdsetup.s.ip_csum = 1; + + if (m->ol_flags & PKT_TX_OUTER_IP_CKSUM) + cmdsetup.s.tnl_csum = 1; + else if ((m->ol_flags & PKT_TX_TCP_CKSUM) || + (m->ol_flags & PKT_TX_UDP_CKSUM)) + cmdsetup.s.transport_csum = 1; + + if (m->nb_segs == 1) { + pkt_len = rte_pktmbuf_data_len(m); + cmdsetup.s.u.datasize = pkt_len; + lio_prepare_pci_cmd(lio_dev, &ndata.cmd, + &cmdsetup, tag); + ndata.cmd.cmd3.dptr = rte_mbuf_data_dma_addr(m); + ndata.reqtype = LIO_REQTYPE_NORESP_NET; + } else { + struct lio_buf_free_info *finfo; + struct lio_gather *g; + phys_addr_t phyaddr; + int i, frags; + + finfo = (struct lio_buf_free_info *)rte_malloc(NULL, + sizeof(*finfo), 0); + if (finfo == NULL) { + PMD_TX_LOG(lio_dev, ERR, + "free buffer alloc failed\n"); + goto xmit_failed; + } + + rte_spinlock_lock(&lio_dev->glist_lock[iq_no]); + g = (struct lio_gather *)list_delete_first_node( + &lio_dev->glist_head[iq_no]); + rte_spinlock_unlock(&lio_dev->glist_lock[iq_no]); + if (g == NULL) { + PMD_TX_LOG(lio_dev, ERR, + "Transmit scatter gather: glist null!\n"); + goto xmit_failed; + } + + cmdsetup.s.gather = 1; + cmdsetup.s.u.gatherptrs = m->nb_segs; + lio_prepare_pci_cmd(lio_dev, &ndata.cmd, + &cmdsetup, tag); + + memset(g->sg, 0, g->sg_size); + g->sg[0].ptr[0] = rte_mbuf_data_dma_addr(m); + lio_add_sg_size(&g->sg[0], m->data_len, 0); + pkt_len = m->data_len; + finfo->mbuf = m; + + /* First seg taken care above */ + frags = m->nb_segs - 1; + i = 1; + m = m->next; + while (frags--) { + g->sg[(i >> 2)].ptr[(i & 3)] = + rte_mbuf_data_dma_addr(m); + lio_add_sg_size(&g->sg[(i >> 2)], + m->data_len, (i & 3)); + pkt_len += m->data_len; + i++; + m = m->next; + } + + phyaddr = rte_mem_virt2phy(g->sg); + if (phyaddr == RTE_BAD_PHYS_ADDR) { + PMD_TX_LOG(lio_dev, ERR, "bad phys addr\n"); + goto xmit_failed; + } + + ndata.cmd.cmd3.dptr = phyaddr; + ndata.reqtype = LIO_REQTYPE_NORESP_NET_SG; + + finfo->g = g; + finfo->lio_dev = lio_dev; + finfo->iq_no = (uint64_t)iq_no; + ndata.buf = finfo; + } + + ndata.datasize = pkt_len; + + status = lio_send_data_pkt(lio_dev, &ndata); + + if (unlikely(status == LIO_IQ_SEND_FAILED)) { + PMD_TX_LOG(lio_dev, ERR, "send failed\n"); + break; + } + + if (unlikely(status == LIO_IQ_SEND_STOP)) { + PMD_TX_LOG(lio_dev, DEBUG, "iq full\n"); + /* create space as iq is full */ + lio_dev_cleanup_iq(lio_dev, iq_no); + } + + stats->tx_done++; + stats->tx_tot_bytes += pkt_len; + processed++; + } + +xmit_failed: + stats->tx_dropped += (nb_pkts - processed); + + return processed; +} + +void +lio_dev_clear_queues(struct rte_eth_dev *eth_dev) +{ + struct lio_instr_queue *txq; + struct lio_droq *rxq; + uint16_t i; + + for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { + txq = eth_dev->data->tx_queues[i]; + if (txq != NULL) { + lio_dev_tx_queue_release(txq); + eth_dev->data->tx_queues[i] = NULL; + } + } + + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + rxq = eth_dev->data->rx_queues[i]; + if (rxq != NULL) { + lio_dev_rx_queue_release(rxq); + eth_dev->data->rx_queues[i] = NULL; + } + } +} diff --git a/drivers/net/liquidio/lio_rxtx.h b/drivers/net/liquidio/lio_rxtx.h new file mode 100644 index 00000000..85685dc7 --- /dev/null +++ b/drivers/net/liquidio/lio_rxtx.h @@ -0,0 +1,769 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_RXTX_H_ +#define _LIO_RXTX_H_ + +#include <stdio.h> +#include <stdint.h> + +#include <rte_spinlock.h> +#include <rte_memory.h> + +#include "lio_struct.h" + +#ifndef ROUNDUP4 +#define ROUNDUP4(val) (((val) + 3) & 0xfffffffc) +#endif + +#define LIO_STQUEUE_FIRST_ENTRY(ptr, type, elem) \ + (type *)((char *)((ptr)->stqh_first) - offsetof(type, elem)) + +#define lio_check_timeout(cur_time, chk_time) ((cur_time) > (chk_time)) + +#define lio_uptime \ + (size_t)(rte_get_timer_cycles() / rte_get_timer_hz()) + +/** Descriptor format. + * The descriptor ring is made of descriptors which have 2 64-bit values: + * -# Physical (bus) address of the data buffer. + * -# Physical (bus) address of a lio_droq_info structure. + * The device DMA's incoming packets and its information at the address + * given by these descriptor fields. + */ +struct lio_droq_desc { + /** The buffer pointer */ + uint64_t buffer_ptr; + + /** The Info pointer */ + uint64_t info_ptr; +}; + +#define LIO_DROQ_DESC_SIZE (sizeof(struct lio_droq_desc)) + +/** Information about packet DMA'ed by Octeon. + * The format of the information available at Info Pointer after Octeon + * has posted a packet. Not all descriptors have valid information. Only + * the Info field of the first descriptor for a packet has information + * about the packet. + */ +struct lio_droq_info { + /** The Output Receive Header. */ + union octeon_rh rh; + + /** The Length of the packet. */ + uint64_t length; +}; + +#define LIO_DROQ_INFO_SIZE (sizeof(struct lio_droq_info)) + +/** Pointer to data buffer. + * Driver keeps a pointer to the data buffer that it made available to + * the Octeon device. Since the descriptor ring keeps physical (bus) + * addresses, this field is required for the driver to keep track of + * the virtual address pointers. + */ +struct lio_recv_buffer { + /** Packet buffer, including meta data. */ + void *buffer; + + /** Data in the packet buffer. */ + uint8_t *data; + +}; + +#define LIO_DROQ_RECVBUF_SIZE (sizeof(struct lio_recv_buffer)) + +#define LIO_DROQ_SIZE (sizeof(struct lio_droq)) + +#define LIO_IQ_SEND_OK 0 +#define LIO_IQ_SEND_STOP 1 +#define LIO_IQ_SEND_FAILED -1 + +/* conditions */ +#define LIO_REQTYPE_NONE 0 +#define LIO_REQTYPE_NORESP_NET 1 +#define LIO_REQTYPE_NORESP_NET_SG 2 +#define LIO_REQTYPE_SOFT_COMMAND 3 + +struct lio_request_list { + uint32_t reqtype; + void *buf; +}; + +/*---------------------- INSTRUCTION FORMAT ----------------------------*/ + +struct lio_instr3_64B { + /** Pointer where the input data is available. */ + uint64_t dptr; + + /** Instruction Header. */ + uint64_t ih3; + + /** Instruction Header. */ + uint64_t pki_ih3; + + /** Input Request Header. */ + uint64_t irh; + + /** opcode/subcode specific parameters */ + uint64_t ossp[2]; + + /** Return Data Parameters */ + uint64_t rdp; + + /** Pointer where the response for a RAW mode packet will be written + * by Octeon. + */ + uint64_t rptr; + +}; + +union lio_instr_64B { + struct lio_instr3_64B cmd3; +}; + +/** The size of each buffer in soft command buffer pool */ +#define LIO_SOFT_COMMAND_BUFFER_SIZE 1536 + +/** Maximum number of buffers to allocate into soft command buffer pool */ +#define LIO_MAX_SOFT_COMMAND_BUFFERS 255 + +struct lio_soft_command { + /** Soft command buffer info. */ + struct lio_stailq_node node; + uint64_t dma_addr; + uint32_t size; + + /** Command and return status */ + union lio_instr_64B cmd; + +#define LIO_COMPLETION_WORD_INIT 0xffffffffffffffffULL + uint64_t *status_word; + + /** Data buffer info */ + void *virtdptr; + uint64_t dmadptr; + uint32_t datasize; + + /** Return buffer info */ + void *virtrptr; + uint64_t dmarptr; + uint32_t rdatasize; + + /** Context buffer info */ + void *ctxptr; + uint32_t ctxsize; + + /** Time out and callback */ + size_t wait_time; + size_t timeout; + uint32_t iq_no; + void (*callback)(uint32_t, void *); + void *callback_arg; + struct rte_mbuf *mbuf; +}; + +struct lio_iq_post_status { + int status; + int index; +}; + +/* wqe + * --------------- 0 + * | wqe word0-3 | + * --------------- 32 + * | PCI IH | + * --------------- 40 + * | RPTR | + * --------------- 48 + * | PCI IRH | + * --------------- 56 + * | OCTEON_CMD | + * --------------- 64 + * | Addtl 8-BData | + * | | + * --------------- + */ + +union octeon_cmd { + uint64_t cmd64; + + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t cmd : 5; + + uint64_t more : 6; /* How many udd words follow the command */ + + uint64_t reserved : 29; + + uint64_t param1 : 16; + + uint64_t param2 : 8; + +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + + uint64_t param2 : 8; + + uint64_t param1 : 16; + + uint64_t reserved : 29; + + uint64_t more : 6; + + uint64_t cmd : 5; + +#endif + } s; +}; + +#define OCTEON_CMD_SIZE (sizeof(union octeon_cmd)) + +/* Maximum number of 8-byte words can be + * sent in a NIC control message. + */ +#define LIO_MAX_NCTRL_UDD 32 + +/* Structure of control information passed by driver to the BASE + * layer when sending control commands to Octeon device software. + */ +struct lio_ctrl_pkt { + /** Command to be passed to the Octeon device software. */ + union octeon_cmd ncmd; + + /** Send buffer */ + void *data; + uint64_t dmadata; + + /** Response buffer */ + void *rdata; + uint64_t dmardata; + + /** Additional data that may be needed by some commands. */ + uint64_t udd[LIO_MAX_NCTRL_UDD]; + + /** Input queue to use to send this command. */ + uint64_t iq_no; + + /** Time to wait for Octeon software to respond to this control command. + * If wait_time is 0, BASE assumes no response is expected. + */ + size_t wait_time; + + struct lio_dev_ctrl_cmd *ctrl_cmd; +}; + +/** Structure of data information passed by driver to the BASE + * layer when forwarding data to Octeon device software. + */ +struct lio_data_pkt { + /** Pointer to information maintained by NIC module for this packet. The + * BASE layer passes this as-is to the driver. + */ + void *buf; + + /** Type of buffer passed in "buf" above. */ + uint32_t reqtype; + + /** Total data bytes to be transferred in this command. */ + uint32_t datasize; + + /** Command to be passed to the Octeon device software. */ + union lio_instr_64B cmd; + + /** Input queue to use to send this command. */ + uint32_t q_no; +}; + +/** Structure passed by driver to BASE layer to prepare a command to send + * network data to Octeon. + */ +union lio_cmd_setup { + struct { + uint32_t iq_no : 8; + uint32_t gather : 1; + uint32_t timestamp : 1; + uint32_t ip_csum : 1; + uint32_t transport_csum : 1; + uint32_t tnl_csum : 1; + uint32_t rsvd : 19; + + union { + uint32_t datasize; + uint32_t gatherptrs; + } u; + } s; + + uint64_t cmd_setup64; +}; + +/* Instruction Header */ +struct octeon_instr_ih3 { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + + /** Reserved3 */ + uint64_t reserved3 : 1; + + /** Gather indicator 1=gather*/ + uint64_t gather : 1; + + /** Data length OR no. of entries in gather list */ + uint64_t dlengsz : 14; + + /** Front Data size */ + uint64_t fsz : 6; + + /** Reserved2 */ + uint64_t reserved2 : 4; + + /** PKI port kind - PKIND */ + uint64_t pkind : 6; + + /** Reserved1 */ + uint64_t reserved1 : 32; + +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + /** Reserved1 */ + uint64_t reserved1 : 32; + + /** PKI port kind - PKIND */ + uint64_t pkind : 6; + + /** Reserved2 */ + uint64_t reserved2 : 4; + + /** Front Data size */ + uint64_t fsz : 6; + + /** Data length OR no. of entries in gather list */ + uint64_t dlengsz : 14; + + /** Gather indicator 1=gather*/ + uint64_t gather : 1; + + /** Reserved3 */ + uint64_t reserved3 : 1; + +#endif +}; + +/* PKI Instruction Header(PKI IH) */ +struct octeon_instr_pki_ih3 { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + + /** Wider bit */ + uint64_t w : 1; + + /** Raw mode indicator 1 = RAW */ + uint64_t raw : 1; + + /** Use Tag */ + uint64_t utag : 1; + + /** Use QPG */ + uint64_t uqpg : 1; + + /** Reserved2 */ + uint64_t reserved2 : 1; + + /** Parse Mode */ + uint64_t pm : 3; + + /** Skip Length */ + uint64_t sl : 8; + + /** Use Tag Type */ + uint64_t utt : 1; + + /** Tag type */ + uint64_t tagtype : 2; + + /** Reserved1 */ + uint64_t reserved1 : 2; + + /** QPG Value */ + uint64_t qpg : 11; + + /** Tag Value */ + uint64_t tag : 32; + +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + + /** Tag Value */ + uint64_t tag : 32; + + /** QPG Value */ + uint64_t qpg : 11; + + /** Reserved1 */ + uint64_t reserved1 : 2; + + /** Tag type */ + uint64_t tagtype : 2; + + /** Use Tag Type */ + uint64_t utt : 1; + + /** Skip Length */ + uint64_t sl : 8; + + /** Parse Mode */ + uint64_t pm : 3; + + /** Reserved2 */ + uint64_t reserved2 : 1; + + /** Use QPG */ + uint64_t uqpg : 1; + + /** Use Tag */ + uint64_t utag : 1; + + /** Raw mode indicator 1 = RAW */ + uint64_t raw : 1; + + /** Wider bit */ + uint64_t w : 1; +#endif +}; + +/** Input Request Header */ +struct octeon_instr_irh { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t opcode : 4; + uint64_t rflag : 1; + uint64_t subcode : 7; + uint64_t vlan : 12; + uint64_t priority : 3; + uint64_t reserved : 5; + uint64_t ossp : 32; /* opcode/subcode specific parameters */ +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + uint64_t ossp : 32; /* opcode/subcode specific parameters */ + uint64_t reserved : 5; + uint64_t priority : 3; + uint64_t vlan : 12; + uint64_t subcode : 7; + uint64_t rflag : 1; + uint64_t opcode : 4; +#endif +}; + +/* pkiih3 + irh + ossp[0] + ossp[1] + rdp + rptr = 40 bytes */ +#define OCTEON_SOFT_CMD_RESP_IH3 (40 + 8) +/* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */ +#define OCTEON_PCI_CMD_O3 (24 + 8) + +/** Return Data Parameters */ +struct octeon_instr_rdp { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t reserved : 49; + uint64_t pcie_port : 3; + uint64_t rlen : 12; +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + uint64_t rlen : 12; + uint64_t pcie_port : 3; + uint64_t reserved : 49; +#endif +}; + +union octeon_packet_params { + uint32_t pkt_params32; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint32_t reserved : 24; + uint32_t ip_csum : 1; /* Perform IP header checksum(s) */ + /* Perform Outer transport header checksum */ + uint32_t transport_csum : 1; + /* Find tunnel, and perform transport csum. */ + uint32_t tnl_csum : 1; + uint32_t tsflag : 1; /* Timestamp this packet */ + uint32_t ipsec_ops : 4; /* IPsec operation */ +#else + uint32_t ipsec_ops : 4; + uint32_t tsflag : 1; + uint32_t tnl_csum : 1; + uint32_t transport_csum : 1; + uint32_t ip_csum : 1; + uint32_t reserved : 7; +#endif + } s; +}; + +/** Utility function to prepare a 64B NIC instruction based on a setup command + * @param cmd - pointer to instruction to be filled in. + * @param setup - pointer to the setup structure + * @param q_no - which queue for back pressure + * + * Assumes the cmd instruction is pre-allocated, but no fields are filled in. + */ +static inline void +lio_prepare_pci_cmd(struct lio_device *lio_dev, + union lio_instr_64B *cmd, + union lio_cmd_setup *setup, + uint32_t tag) +{ + union octeon_packet_params packet_params; + struct octeon_instr_pki_ih3 *pki_ih3; + struct octeon_instr_irh *irh; + struct octeon_instr_ih3 *ih3; + int port; + + memset(cmd, 0, sizeof(union lio_instr_64B)); + + ih3 = (struct octeon_instr_ih3 *)&cmd->cmd3.ih3; + pki_ih3 = (struct octeon_instr_pki_ih3 *)&cmd->cmd3.pki_ih3; + + /* assume that rflag is cleared so therefore front data will only have + * irh and ossp[1] and ossp[2] for a total of 24 bytes + */ + ih3->pkind = lio_dev->instr_queue[setup->s.iq_no]->txpciq.s.pkind; + /* PKI IH */ + ih3->fsz = OCTEON_PCI_CMD_O3; + + if (!setup->s.gather) { + ih3->dlengsz = setup->s.u.datasize; + } else { + ih3->gather = 1; + ih3->dlengsz = setup->s.u.gatherptrs; + } + + pki_ih3->w = 1; + pki_ih3->raw = 0; + pki_ih3->utag = 0; + pki_ih3->utt = 1; + pki_ih3->uqpg = lio_dev->instr_queue[setup->s.iq_no]->txpciq.s.use_qpg; + + port = (int)lio_dev->instr_queue[setup->s.iq_no]->txpciq.s.port; + + if (tag) + pki_ih3->tag = tag; + else + pki_ih3->tag = LIO_DATA(port); + + pki_ih3->tagtype = OCTEON_ORDERED_TAG; + pki_ih3->qpg = lio_dev->instr_queue[setup->s.iq_no]->txpciq.s.qpg; + pki_ih3->pm = 0x0; /* parse from L2 */ + pki_ih3->sl = 32; /* sl will be sizeof(pki_ih3) + irh + ossp0 + ossp1*/ + + irh = (struct octeon_instr_irh *)&cmd->cmd3.irh; + + irh->opcode = LIO_OPCODE; + irh->subcode = LIO_OPCODE_NW_DATA; + + packet_params.pkt_params32 = 0; + packet_params.s.ip_csum = setup->s.ip_csum; + packet_params.s.transport_csum = setup->s.transport_csum; + packet_params.s.tnl_csum = setup->s.tnl_csum; + packet_params.s.tsflag = setup->s.timestamp; + + irh->ossp = packet_params.pkt_params32; +} + +int lio_setup_sc_buffer_pool(struct lio_device *lio_dev); +void lio_free_sc_buffer_pool(struct lio_device *lio_dev); + +struct lio_soft_command * +lio_alloc_soft_command(struct lio_device *lio_dev, + uint32_t datasize, uint32_t rdatasize, + uint32_t ctxsize); +void lio_prepare_soft_command(struct lio_device *lio_dev, + struct lio_soft_command *sc, + uint8_t opcode, uint8_t subcode, + uint32_t irh_ossp, uint64_t ossp0, + uint64_t ossp1); +int lio_send_soft_command(struct lio_device *lio_dev, + struct lio_soft_command *sc); +void lio_free_soft_command(struct lio_soft_command *sc); + +/** Send control packet to the device + * @param lio_dev - lio device pointer + * @param nctrl - control structure with command, timeout, and callback info + * + * @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if it the + * queue should be stopped, and LIO_IQ_SEND_OK if it sent okay. + */ +int lio_send_ctrl_pkt(struct lio_device *lio_dev, + struct lio_ctrl_pkt *ctrl_pkt); + +/** Maximum ordered requests to process in every invocation of + * lio_process_ordered_list(). The function will continue to process requests + * as long as it can find one that has finished processing. If it keeps + * finding requests that have completed, the function can run for ever. The + * value defined here sets an upper limit on the number of requests it can + * process before it returns control to the poll thread. + */ +#define LIO_MAX_ORD_REQS_TO_PROCESS 4096 + +/** Error codes used in Octeon Host-Core communication. + * + * 31 16 15 0 + * ---------------------------- + * | | | + * ---------------------------- + * Error codes are 32-bit wide. The upper 16-bits, called Major Error Number, + * are reserved to identify the group to which the error code belongs. The + * lower 16-bits, called Minor Error Number, carry the actual code. + * + * So error codes are (MAJOR NUMBER << 16)| MINOR_NUMBER. + */ +/** Status for a request. + * If the request is successfully queued, the driver will return + * a LIO_REQUEST_PENDING status. LIO_REQUEST_TIMEOUT is only returned by + * the driver if the response for request failed to arrive before a + * time-out period or if the request processing * got interrupted due to + * a signal respectively. + */ +enum { + /** A value of 0x00000000 indicates no error i.e. success */ + LIO_REQUEST_DONE = 0x00000000, + /** (Major number: 0x0000; Minor Number: 0x0001) */ + LIO_REQUEST_PENDING = 0x00000001, + LIO_REQUEST_TIMEOUT = 0x00000003, + +}; + +/*------ Error codes used by firmware (bits 15..0 set by firmware */ +#define LIO_FIRMWARE_MAJOR_ERROR_CODE 0x0001 +#define LIO_FIRMWARE_STATUS_CODE(status) \ + ((LIO_FIRMWARE_MAJOR_ERROR_CODE << 16) | (status)) + +/** Initialize the response lists. The number of response lists to create is + * given by count. + * @param lio_dev - the lio device structure. + */ +void lio_setup_response_list(struct lio_device *lio_dev); + +/** Check the status of first entry in the ordered list. If the instruction at + * that entry finished processing or has timed-out, the entry is cleaned. + * @param lio_dev - the lio device structure. + * @return 1 if the ordered list is empty, 0 otherwise. + */ +int lio_process_ordered_list(struct lio_device *lio_dev); + +#define LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, field, count) \ + (((lio_dev)->instr_queue[iq_no]->stats.field) += count) + +static inline void +lio_swap_8B_data(uint64_t *data, uint32_t blocks) +{ + while (blocks) { + *data = rte_cpu_to_be_64(*data); + blocks--; + data++; + } +} + +static inline uint64_t +lio_map_ring(void *buf) +{ + phys_addr_t dma_addr; + + dma_addr = rte_mbuf_data_dma_addr_default(((struct rte_mbuf *)buf)); + + return (uint64_t)dma_addr; +} + +static inline uint64_t +lio_map_ring_info(struct lio_droq *droq, uint32_t i) +{ + phys_addr_t dma_addr; + + dma_addr = droq->info_list_dma + (i * LIO_DROQ_INFO_SIZE); + + return (uint64_t)dma_addr; +} + +static inline int +lio_opcode_slow_path(union octeon_rh *rh) +{ + uint16_t subcode1, subcode2; + + subcode1 = LIO_OPCODE_SUBCODE(rh->r.opcode, rh->r.subcode); + subcode2 = LIO_OPCODE_SUBCODE(LIO_OPCODE, LIO_OPCODE_NW_DATA); + + return subcode2 != subcode1; +} + +static inline void +lio_add_sg_size(struct lio_sg_entry *sg_entry, + uint16_t size, uint32_t pos) +{ +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + sg_entry->u.size[pos] = size; +#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + sg_entry->u.size[3 - pos] = size; +#endif +} + +/* Macro to increment index. + * Index is incremented by count; if the sum exceeds + * max, index is wrapped-around to the start. + */ +static inline uint32_t +lio_incr_index(uint32_t index, uint32_t count, uint32_t max) +{ + if ((index + count) >= max) + index = index + count - max; + else + index += count; + + return index; +} + +int lio_setup_droq(struct lio_device *lio_dev, int q_no, int num_descs, + int desc_size, struct rte_mempool *mpool, + unsigned int socket_id); +uint16_t lio_dev_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t budget); +void lio_delete_droq_queue(struct lio_device *lio_dev, int oq_no); + +void lio_delete_sglist(struct lio_instr_queue *txq); +int lio_setup_sglists(struct lio_device *lio_dev, int iq_no, + int fw_mapped_iq, int num_descs, unsigned int socket_id); +uint16_t lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, + uint16_t nb_pkts); +int lio_wait_for_instr_fetch(struct lio_device *lio_dev); +int lio_setup_iq(struct lio_device *lio_dev, int q_index, + union octeon_txpciq iq_no, uint32_t num_descs, void *app_ctx, + unsigned int socket_id); +int lio_flush_iq(struct lio_device *lio_dev, struct lio_instr_queue *iq); +void lio_delete_instruction_queue(struct lio_device *lio_dev, int iq_no); +/** Setup instruction queue zero for the device + * @param lio_dev which lio device to setup + * + * @return 0 if success. -1 if fails + */ +int lio_setup_instr_queue0(struct lio_device *lio_dev); +void lio_free_instr_queue0(struct lio_device *lio_dev); +void lio_dev_clear_queues(struct rte_eth_dev *eth_dev); +#endif /* _LIO_RXTX_H_ */ diff --git a/drivers/net/liquidio/lio_struct.h b/drivers/net/liquidio/lio_struct.h new file mode 100644 index 00000000..26f803f9 --- /dev/null +++ b/drivers/net/liquidio/lio_struct.h @@ -0,0 +1,689 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2017 Cavium, Inc.. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Cavium, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _LIO_STRUCT_H_ +#define _LIO_STRUCT_H_ + +#include <stdio.h> +#include <stdint.h> +#include <sys/queue.h> + +#include <rte_spinlock.h> +#include <rte_atomic.h> + +#include "lio_hw_defs.h" + +struct lio_stailq_node { + STAILQ_ENTRY(lio_stailq_node) entries; +}; + +STAILQ_HEAD(lio_stailq_head, lio_stailq_node); + +struct lio_version { + uint16_t major; + uint16_t minor; + uint16_t micro; + uint16_t reserved; +}; + +/** Input Queue statistics. Each input queue has four stats fields. */ +struct lio_iq_stats { + uint64_t instr_posted; /**< Instructions posted to this queue. */ + uint64_t instr_processed; /**< Instructions processed in this queue. */ + uint64_t instr_dropped; /**< Instructions that could not be processed */ + uint64_t bytes_sent; /**< Bytes sent through this queue. */ + uint64_t tx_done; /**< Num of packets sent to network. */ + uint64_t tx_iq_busy; /**< Num of times this iq was found to be full. */ + uint64_t tx_dropped; /**< Num of pkts dropped due to xmitpath errors. */ + uint64_t tx_tot_bytes; /**< Total count of bytes sent to network. */ +}; + +/** Output Queue statistics. Each output queue has four stats fields. */ +struct lio_droq_stats { + /** Number of packets received in this queue. */ + uint64_t pkts_received; + + /** Bytes received by this queue. */ + uint64_t bytes_received; + + /** Packets dropped due to no memory available. */ + uint64_t dropped_nomem; + + /** Packets dropped due to large number of pkts to process. */ + uint64_t dropped_toomany; + + /** Number of packets sent to stack from this queue. */ + uint64_t rx_pkts_received; + + /** Number of Bytes sent to stack from this queue. */ + uint64_t rx_bytes_received; + + /** Num of Packets dropped due to receive path failures. */ + uint64_t rx_dropped; + + /** Num of vxlan packets received; */ + uint64_t rx_vxlan; + + /** Num of failures of lio_recv_buffer_alloc() */ + uint64_t rx_alloc_failure; + +}; + +/** The Descriptor Ring Output Queue structure. + * This structure has all the information required to implement a + * DROQ. + */ +struct lio_droq { + /** A spinlock to protect access to this ring. */ + rte_spinlock_t lock; + + uint32_t q_no; + + uint32_t pkt_count; + + struct lio_device *lio_dev; + + /** The 8B aligned descriptor ring starts at this address. */ + struct lio_droq_desc *desc_ring; + + /** Index in the ring where the driver should read the next packet */ + uint32_t read_idx; + + /** Index in the ring where Octeon will write the next packet */ + uint32_t write_idx; + + /** Index in the ring where the driver will refill the descriptor's + * buffer + */ + uint32_t refill_idx; + + /** Packets pending to be processed */ + rte_atomic64_t pkts_pending; + + /** Number of descriptors in this ring. */ + uint32_t max_count; + + /** The number of descriptors pending refill. */ + uint32_t refill_count; + + uint32_t refill_threshold; + + /** The 8B aligned info ptrs begin from this address. */ + struct lio_droq_info *info_list; + + /** The receive buffer list. This list has the virtual addresses of the + * buffers. + */ + struct lio_recv_buffer *recv_buf_list; + + /** The size of each buffer pointed by the buffer pointer. */ + uint32_t buffer_size; + + /** Pointer to the mapped packet credit register. + * Host writes number of info/buffer ptrs available to this register + */ + void *pkts_credit_reg; + + /** Pointer to the mapped packet sent register. + * Octeon writes the number of packets DMA'ed to host memory + * in this register. + */ + void *pkts_sent_reg; + + /** Statistics for this DROQ. */ + struct lio_droq_stats stats; + + /** DMA mapped address of the DROQ descriptor ring. */ + size_t desc_ring_dma; + + /** Info ptr list are allocated at this virtual address. */ + size_t info_base_addr; + + /** DMA mapped address of the info list */ + size_t info_list_dma; + + /** Allocated size of info list. */ + uint32_t info_alloc_size; + + /** Memory zone **/ + const struct rte_memzone *desc_ring_mz; + const struct rte_memzone *info_mz; + struct rte_mempool *mpool; +}; + +/** Receive Header */ +union octeon_rh { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t rh64; + struct { + uint64_t opcode : 4; + uint64_t subcode : 8; + uint64_t len : 3; /** additional 64-bit words */ + uint64_t reserved : 17; + uint64_t ossp : 32; /** opcode/subcode specific parameters */ + } r; + struct { + uint64_t opcode : 4; + uint64_t subcode : 8; + uint64_t len : 3; /** additional 64-bit words */ + uint64_t extra : 28; + uint64_t vlan : 12; + uint64_t priority : 3; + uint64_t csum_verified : 3; /** checksum verified. */ + uint64_t has_hwtstamp : 1; /** Has hardware timestamp.1 = yes.*/ + uint64_t encap_on : 1; + uint64_t has_hash : 1; /** Has hash (rth or rss). 1 = yes. */ + } r_dh; + struct { + uint64_t opcode : 4; + uint64_t subcode : 8; + uint64_t len : 3; /** additional 64-bit words */ + uint64_t reserved : 8; + uint64_t extra : 25; + uint64_t gmxport : 16; + } r_nic_info; +#else + uint64_t rh64; + struct { + uint64_t ossp : 32; /** opcode/subcode specific parameters */ + uint64_t reserved : 17; + uint64_t len : 3; /** additional 64-bit words */ + uint64_t subcode : 8; + uint64_t opcode : 4; + } r; + struct { + uint64_t has_hash : 1; /** Has hash (rth or rss). 1 = yes. */ + uint64_t encap_on : 1; + uint64_t has_hwtstamp : 1; /** 1 = has hwtstamp */ + uint64_t csum_verified : 3; /** checksum verified. */ + uint64_t priority : 3; + uint64_t vlan : 12; + uint64_t extra : 28; + uint64_t len : 3; /** additional 64-bit words */ + uint64_t subcode : 8; + uint64_t opcode : 4; + } r_dh; + struct { + uint64_t gmxport : 16; + uint64_t extra : 25; + uint64_t reserved : 8; + uint64_t len : 3; /** additional 64-bit words */ + uint64_t subcode : 8; + uint64_t opcode : 4; + } r_nic_info; +#endif +}; + +#define OCTEON_RH_SIZE (sizeof(union octeon_rh)) + +/** The txpciq info passed to host from the firmware */ +union octeon_txpciq { + uint64_t txpciq64; + + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t q_no : 8; + uint64_t port : 8; + uint64_t pkind : 6; + uint64_t use_qpg : 1; + uint64_t qpg : 11; + uint64_t aura_num : 10; + uint64_t reserved : 20; +#else + uint64_t reserved : 20; + uint64_t aura_num : 10; + uint64_t qpg : 11; + uint64_t use_qpg : 1; + uint64_t pkind : 6; + uint64_t port : 8; + uint64_t q_no : 8; +#endif + } s; +}; + +/** The instruction (input) queue. + * The input queue is used to post raw (instruction) mode data or packet + * data to Octeon device from the host. Each input queue for + * a LIO device has one such structure to represent it. + */ +struct lio_instr_queue { + /** A spinlock to protect access to the input ring. */ + rte_spinlock_t lock; + + rte_spinlock_t post_lock; + + struct lio_device *lio_dev; + + uint32_t pkt_in_done; + + rte_atomic64_t iq_flush_running; + + /** Flag that indicates if the queue uses 64 byte commands. */ + uint32_t iqcmd_64B:1; + + /** Queue info. */ + union octeon_txpciq txpciq; + + uint32_t rsvd:17; + + uint32_t status:8; + + /** Maximum no. of instructions in this queue. */ + uint32_t max_count; + + /** Index in input ring where the driver should write the next packet */ + uint32_t host_write_index; + + /** Index in input ring where Octeon is expected to read the next + * packet. + */ + uint32_t lio_read_index; + + /** This index aids in finding the window in the queue where Octeon + * has read the commands. + */ + uint32_t flush_index; + + /** This field keeps track of the instructions pending in this queue. */ + rte_atomic64_t instr_pending; + + /** Pointer to the Virtual Base addr of the input ring. */ + uint8_t *base_addr; + + struct lio_request_list *request_list; + + /** Octeon doorbell register for the ring. */ + void *doorbell_reg; + + /** Octeon instruction count register for this ring. */ + void *inst_cnt_reg; + + /** Number of instructions pending to be posted to Octeon. */ + uint32_t fill_cnt; + + /** Statistics for this input queue. */ + struct lio_iq_stats stats; + + /** DMA mapped base address of the input descriptor ring. */ + uint64_t base_addr_dma; + + /** Application context */ + void *app_ctx; + + /* network stack queue index */ + int q_index; + + /* Memory zone */ + const struct rte_memzone *iq_mz; +}; + +/** This structure is used by driver to store information required + * to free the mbuff when the packet has been fetched by Octeon. + * Bytes offset below assume worst-case of a 64-bit system. + */ +struct lio_buf_free_info { + /** Bytes 1-8. Pointer to network device private structure. */ + struct lio_device *lio_dev; + + /** Bytes 9-16. Pointer to mbuff. */ + struct rte_mbuf *mbuf; + + /** Bytes 17-24. Pointer to gather list. */ + struct lio_gather *g; + + /** Bytes 25-32. Physical address of mbuf->data or gather list. */ + uint64_t dptr; + + /** Bytes 33-47. Piggybacked soft command, if any */ + struct lio_soft_command *sc; + + /** Bytes 48-63. iq no */ + uint64_t iq_no; +}; + +/* The Scatter-Gather List Entry. The scatter or gather component used with + * input instruction has this format. + */ +struct lio_sg_entry { + /** The first 64 bit gives the size of data in each dptr. */ + union { + uint16_t size[4]; + uint64_t size64; + } u; + + /** The 4 dptr pointers for this entry. */ + uint64_t ptr[4]; +}; + +#define LIO_SG_ENTRY_SIZE (sizeof(struct lio_sg_entry)) + +/** Structure of a node in list of gather components maintained by + * driver for each network device. + */ +struct lio_gather { + /** List manipulation. Next and prev pointers. */ + struct lio_stailq_node list; + + /** Size of the gather component at sg in bytes. */ + int sg_size; + + /** Number of bytes that sg was adjusted to make it 8B-aligned. */ + int adjust; + + /** Gather component that can accommodate max sized fragment list + * received from the IP layer. + */ + struct lio_sg_entry *sg; +}; + +struct lio_rss_ctx { + uint16_t hash_key_size; + uint8_t hash_key[LIO_RSS_MAX_KEY_SZ]; + /* Ideally a factor of number of queues */ + uint8_t itable[LIO_RSS_MAX_TABLE_SZ]; + uint8_t itable_size; + uint8_t ip; + uint8_t tcp_hash; + uint8_t ipv6; + uint8_t ipv6_tcp_hash; + uint8_t ipv6_ex; + uint8_t ipv6_tcp_ex_hash; + uint8_t hash_disable; +}; + +struct lio_io_enable { + uint64_t iq; + uint64_t oq; + uint64_t iq64B; +}; + +struct lio_fn_list { + void (*setup_iq_regs)(struct lio_device *, uint32_t); + void (*setup_oq_regs)(struct lio_device *, uint32_t); + + int (*setup_mbox)(struct lio_device *); + void (*free_mbox)(struct lio_device *); + + int (*setup_device_regs)(struct lio_device *); + int (*enable_io_queues)(struct lio_device *); + void (*disable_io_queues)(struct lio_device *); +}; + +struct lio_pf_vf_hs_word { +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + /** PKIND value assigned for the DPI interface */ + uint64_t pkind : 8; + + /** OCTEON core clock multiplier */ + uint64_t core_tics_per_us : 16; + + /** OCTEON coprocessor clock multiplier */ + uint64_t coproc_tics_per_us : 16; + + /** app that currently running on OCTEON */ + uint64_t app_mode : 8; + + /** RESERVED */ + uint64_t reserved : 16; + +#elif RTE_BYTE_ORDER == RTE_BIG_ENDIAN + + /** RESERVED */ + uint64_t reserved : 16; + + /** app that currently running on OCTEON */ + uint64_t app_mode : 8; + + /** OCTEON coprocessor clock multiplier */ + uint64_t coproc_tics_per_us : 16; + + /** OCTEON core clock multiplier */ + uint64_t core_tics_per_us : 16; + + /** PKIND value assigned for the DPI interface */ + uint64_t pkind : 8; +#endif +}; + +struct lio_sriov_info { + /** Number of rings assigned to VF */ + uint32_t rings_per_vf; + + /** Number of VF devices enabled */ + uint32_t num_vfs; +}; + +/* Head of a response list */ +struct lio_response_list { + /** List structure to add delete pending entries to */ + struct lio_stailq_head head; + + /** A lock for this response list */ + rte_spinlock_t lock; + + rte_atomic64_t pending_req_count; +}; + +/* Structure to define the configuration attributes for each Input queue. */ +struct lio_iq_config { + /* Max number of IQs available */ + uint8_t max_iqs; + + /** Pending list size (usually set to the sum of the size of all Input + * queues) + */ + uint32_t pending_list_size; + + /** Command size - 32 or 64 bytes */ + uint32_t instr_type; +}; + +/* Structure to define the configuration attributes for each Output queue. */ +struct lio_oq_config { + /* Max number of OQs available */ + uint8_t max_oqs; + + /** If set, the Output queue uses info-pointer mode. (Default: 1 ) */ + uint32_t info_ptr; + + /** The number of buffers that were consumed during packet processing by + * the driver on this Output queue before the driver attempts to + * replenish the descriptor ring with new buffers. + */ + uint32_t refill_threshold; +}; + +/* Structure to define the configuration. */ +struct lio_config { + uint16_t card_type; + const char *card_name; + + /** Input Queue attributes. */ + struct lio_iq_config iq; + + /** Output Queue attributes. */ + struct lio_oq_config oq; + + int num_nic_ports; + + int num_def_tx_descs; + + /* Num of desc for rx rings */ + int num_def_rx_descs; + + int def_rx_buf_size; +}; + +/** Status of a RGMII Link on Octeon as seen by core driver. */ +union octeon_link_status { + uint64_t link_status64; + + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t duplex : 8; + uint64_t mtu : 16; + uint64_t speed : 16; + uint64_t link_up : 1; + uint64_t autoneg : 1; + uint64_t if_mode : 5; + uint64_t pause : 1; + uint64_t flashing : 1; + uint64_t reserved : 15; +#else + uint64_t reserved : 15; + uint64_t flashing : 1; + uint64_t pause : 1; + uint64_t if_mode : 5; + uint64_t autoneg : 1; + uint64_t link_up : 1; + uint64_t speed : 16; + uint64_t mtu : 16; + uint64_t duplex : 8; +#endif + } s; +}; + +/** The rxpciq info passed to host from the firmware */ +union octeon_rxpciq { + uint64_t rxpciq64; + + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t q_no : 8; + uint64_t reserved : 56; +#else + uint64_t reserved : 56; + uint64_t q_no : 8; +#endif + } s; +}; + +/** Information for a OCTEON ethernet interface shared between core & host. */ +struct octeon_link_info { + union octeon_link_status link; + uint64_t hw_addr; + +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t gmxport : 16; + uint64_t macaddr_is_admin_assigned : 1; + uint64_t vlan_is_admin_assigned : 1; + uint64_t rsvd : 30; + uint64_t num_txpciq : 8; + uint64_t num_rxpciq : 8; +#else + uint64_t num_rxpciq : 8; + uint64_t num_txpciq : 8; + uint64_t rsvd : 30; + uint64_t vlan_is_admin_assigned : 1; + uint64_t macaddr_is_admin_assigned : 1; + uint64_t gmxport : 16; +#endif + + union octeon_txpciq txpciq[LIO_MAX_IOQS_PER_IF]; + union octeon_rxpciq rxpciq[LIO_MAX_IOQS_PER_IF]; +}; + +/* ----------------------- THE LIO DEVICE --------------------------- */ +/** The lio device. + * Each lio device has this structure to represent all its + * components. + */ +struct lio_device { + /** PCI device pointer */ + struct rte_pci_device *pci_dev; + + /** Octeon Chip type */ + uint16_t chip_id; + uint16_t pf_num; + uint16_t vf_num; + + /** This device's PCIe port used for traffic. */ + uint16_t pcie_port; + + /** The state of this device */ + rte_atomic64_t status; + + uint8_t intf_open; + + struct octeon_link_info linfo; + + uint8_t *hw_addr; + + struct lio_fn_list fn_list; + + uint32_t num_iqs; + + /** Guards each glist */ + rte_spinlock_t *glist_lock; + /** Array of gather component linked lists */ + struct lio_stailq_head *glist_head; + + /* The pool containing pre allocated buffers used for soft commands */ + struct rte_mempool *sc_buf_pool; + + /** The input instruction queues */ + struct lio_instr_queue *instr_queue[LIO_MAX_POSSIBLE_INSTR_QUEUES]; + + /** The singly-linked tail queues of instruction response */ + struct lio_response_list response_list; + + uint32_t num_oqs; + + /** The DROQ output queues */ + struct lio_droq *droq[LIO_MAX_POSSIBLE_OUTPUT_QUEUES]; + + struct lio_io_enable io_qmask; + + struct lio_sriov_info sriov_info; + + struct lio_pf_vf_hs_word pfvf_hsword; + + /** Mail Box details of each lio queue. */ + struct lio_mbox **mbox; + + char dev_string[LIO_DEVICE_NAME_LEN]; /* Device print string */ + + const struct lio_config *default_config; + + struct rte_eth_dev *eth_dev; + + uint64_t ifflags; + uint8_t max_rx_queues; + uint8_t max_tx_queues; + uint8_t nb_rx_queues; + uint8_t nb_tx_queues; + uint8_t port_configured; + struct lio_rss_ctx rss_state; + uint8_t port_id; +}; +#endif /* _LIO_STRUCT_H_ */ diff --git a/drivers/net/liquidio/rte_pmd_lio_version.map b/drivers/net/liquidio/rte_pmd_lio_version.map new file mode 100644 index 00000000..8591cc0b --- /dev/null +++ b/drivers/net/liquidio/rte_pmd_lio_version.map @@ -0,0 +1,4 @@ +DPDK_17.05 { + + local: *; +}; |