/*- * BSD LICENSE * * Copyright(c) 2013-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include "fm10k.h" #include "base/fm10k_api.h" /* Default delay to acquire mailbox lock */ #define FM10K_MBXLOCK_DELAY_US 20 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL #define MAIN_VSI_POOL_NUMBER 0 /* Max try times to acquire switch status */ #define MAX_QUERY_SWITCH_STATE_TIMES 10 /* Wait interval to get switch status */ #define WAIT_SWITCH_MSG_US 100000 /* A period of quiescence for switch */ #define FM10K_SWITCH_QUIESCE_US 100000 /* Number of chars per uint32 type */ #define CHARS_PER_UINT32 (sizeof(uint32_t)) #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1) /* default 1:1 map from queue ID to interrupt vector ID */ #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id]) /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */ #define MAX_LPORT_NUM 128 #define GLORT_FD_Q_BASE 0x40 #define GLORT_PF_MASK 0xFFC0 #define GLORT_FD_MASK GLORT_PF_MASK #define GLORT_FD_INDEX GLORT_FD_Q_BASE static void fm10k_close_mbx_service(struct fm10k_hw *hw); static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev); static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev); static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev); static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev); static inline int fm10k_glort_valid(struct fm10k_hw *hw); static int fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on); static void fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add, uint32_t pool); static void fm10k_tx_queue_release(void *queue); static void fm10k_rx_queue_release(void *queue); static void fm10k_set_rx_function(struct rte_eth_dev *dev); static void fm10k_set_tx_function(struct rte_eth_dev *dev); static int fm10k_check_ftag(struct rte_devargs *devargs); static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete); struct fm10k_xstats_name_off { char name[RTE_ETH_XSTATS_NAME_SIZE]; unsigned offset; }; struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = { {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)}, {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)}, {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)}, {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)}, {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)}, {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)}, {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)}, {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats, nodesc_drop)}, }; #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \ sizeof(fm10k_hw_stats_strings[0])) struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = { {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)}, {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)}, {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)}, }; #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \ sizeof(fm10k_hw_stats_rx_q_strings[0])) struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = { {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)}, {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)}, }; #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \ sizeof(fm10k_hw_stats_tx_q_strings[0])) #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \ (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS)) static int fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev); static void fm10k_mbx_initlock(struct fm10k_hw *hw) { rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)); } static void fm10k_mbx_lock(struct fm10k_hw *hw) { while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back))) rte_delay_us(FM10K_MBXLOCK_DELAY_US); } static void fm10k_mbx_unlock(struct fm10k_hw *hw) { rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)); } /* Stubs needed for linkage when vPMD is disabled */ int __attribute__((weak)) fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev) { return -1; } uint16_t __attribute__((weak)) fm10k_recv_pkts_vec( __rte_unused void *rx_queue, __rte_unused struct rte_mbuf **rx_pkts, __rte_unused uint16_t nb_pkts) { return 0; } uint16_t __attribute__((weak)) fm10k_recv_scattered_pkts_vec( __rte_unused void *rx_queue, __rte_unused struct rte_mbuf **rx_pkts, __rte_unused uint16_t nb_pkts) { return 0; } int __attribute__((weak)) fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq) { return -1; } void __attribute__((weak)) fm10k_rx_queue_release_mbufs_vec( __rte_unused struct fm10k_rx_queue *rxq) { return; } void __attribute__((weak)) fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq) { return; } int __attribute__((weak)) fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq) { return -1; } uint16_t __attribute__((weak)) fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue, __rte_unused struct rte_mbuf **tx_pkts, __rte_unused uint16_t nb_pkts) { return 0; } /* * reset queue to initial state, allocate software buffers used when starting * device. * return 0 on success * return -ENOMEM if buffers cannot be allocated * return -EINVAL if buffers do not satisfy alignment condition */ static inline int rx_queue_reset(struct fm10k_rx_queue *q) { static const union fm10k_rx_desc zero = {{0} }; uint64_t dma_addr; int i, diag; PMD_INIT_FUNC_TRACE(); diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc); if (diag != 0) return -ENOMEM; for (i = 0; i < q->nb_desc; ++i) { fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id); if (!fm10k_addr_alignment_valid(q->sw_ring[i])) { rte_mempool_put_bulk(q->mp, (void **)q->sw_ring, q->nb_desc); return -EINVAL; } dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]); q->hw_ring[i].q.pkt_addr = dma_addr; q->hw_ring[i].q.hdr_addr = dma_addr; } /* initialize extra software ring entries. Space for these extra * entries is always allocated. */ memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf)); for (i = 0; i < q->nb_fake_desc; ++i) { q->sw_ring[q->nb_desc + i] = &q->fake_mbuf; q->hw_ring[q->nb_desc + i] = zero; } q->next_dd = 0; q->next_alloc = 0; q->next_trigger = q->alloc_thresh - 1; FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1); q->rxrearm_start = 0; q->rxrearm_nb = 0; return 0; } /* * clean queue, descriptor rings, free software buffers used when stopping * device. */ static inline void rx_queue_clean(struct fm10k_rx_queue *q) { union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} }; uint32_t i; PMD_INIT_FUNC_TRACE(); /* zero descriptor rings */ for (i = 0; i < q->nb_desc; ++i) q->hw_ring[i] = zero; /* zero faked descriptors */ for (i = 0; i < q->nb_fake_desc; ++i) q->hw_ring[q->nb_desc + i] = zero; /* vPMD driver has a different way of releasing mbufs. */ if (q->rx_using_sse) { fm10k_rx_queue_release_mbufs_vec(q); return; } /* free software buffers */ for (i = 0; i < q->nb_desc; ++i) { if (q->sw_ring[i]) { rte_pktmbuf_free_seg(q->sw_ring[i]); q->sw_ring[i] = NULL; } } } /* * free all queue memory used when releasing the queue (i.e. configure) */ static inline void rx_queue_free(struct fm10k_rx_queue *q) { PMD_INIT_FUNC_TRACE(); if (q) { PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q); rx_queue_clean(q); if (q->sw_ring) { rte_free(q->sw_ring); q->sw_ring = NULL; } rte_free(q); q = NULL; } } /* * disable RX queue, wait unitl HW finished necessary flush operation */ static inline int rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum) { uint32_t reg, i; reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum)); FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum), reg & ~FM10K_RXQCTL_ENABLE); /* Wait 100us at most */ for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) { rte_delay_us(1); reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum)); if (!(reg & FM10K_RXQCTL_ENABLE)) break; } if (i == FM10K_QUEUE_DISABLE_TIMEOUT) return -1; return 0; } /* * reset queue to initial state, allocate software buffers used when starting * device */ static inline void tx_queue_reset(struct fm10k_tx_queue *q) { PMD_INIT_FUNC_TRACE(); q->last_free = 0; q->next_free = 0; q->nb_used = 0; q->nb_free = q->nb_desc - 1; fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh); FM10K_PCI_REG_WRITE(q->tail_ptr, 0); } /* * clean queue, descriptor rings, free software buffers used when stopping * device */ static inline void tx_queue_clean(struct fm10k_tx_queue *q) { struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0}; uint32_t i; PMD_INIT_FUNC_TRACE(); /* zero descriptor rings */ for (i = 0; i < q->nb_desc; ++i) q->hw_ring[i] = zero; /* free software buffers */ for (i = 0; i < q->nb_desc; ++i) { if (q->sw_ring[i]) { rte_pktmbuf_free_seg(q->sw_ring[i]); q->sw_ring[i] = NULL; } } } /* * free all queue memory used when releasing the queue (i.e. configure) */ static inline void tx_queue_free(struct fm10k_tx_queue *q) { PMD_INIT_FUNC_TRACE(); if (q) { PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q); tx_queue_clean(q); if (q->rs_tracker.list) { rte_free(q->rs_tracker.list); q->rs_tracker.list = NULL; } if (q->sw_ring) { rte_free(q->sw_ring); q->sw_ring = NULL; } rte_free(q); q = NULL; } } /* * disable TX queue, wait unitl HW finished necessary flush operation */ static inline int tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum) { uint32_t reg, i; reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum)); FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum), reg & ~FM10K_TXDCTL_ENABLE); /* Wait 100us at most */ for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) { rte_delay_us(1); reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum)); if (!(reg & FM10K_TXDCTL_ENABLE)) break; } if (i == FM10K_QUEUE_DISABLE_TIMEOUT) return -1; return 0; } static int fm10k_check_mq_mode(struct rte_eth_dev *dev) { enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_eth_vmdq_rx_conf *vmdq_conf; uint16_t nb_rx_q = dev->data->nb_rx_queues; vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) { PMD_INIT_LOG(ERR, "DCB mode is not supported."); return -EINVAL; } if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG)) return 0; if (hw->mac.type == fm10k_mac_vf) { PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF."); return -EINVAL; } /* Check VMDQ queue pool number */ if (vmdq_conf->nb_queue_pools > sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT || vmdq_conf->nb_queue_pools > nb_rx_q) { PMD_INIT_LOG(ERR, "Too many of queue pools: %d", vmdq_conf->nb_queue_pools); return -EINVAL; } return 0; } static const struct fm10k_txq_ops def_txq_ops = { .reset = tx_queue_reset, }; static int fm10k_dev_configure(struct rte_eth_dev *dev) { int ret; PMD_INIT_FUNC_TRACE(); if (dev->data->dev_conf.rxmode.hw_strip_crc == 0) PMD_INIT_LOG(WARNING, "fm10k always strip CRC"); /* multipe queue mode checking */ ret = fm10k_check_mq_mode(dev); if (ret != 0) { PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.", ret); return ret; } return 0; } static void fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_eth_vmdq_rx_conf *vmdq_conf; uint32_t i; vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; for (i = 0; i < vmdq_conf->nb_pool_maps; i++) { if (!vmdq_conf->pool_map[i].pools) continue; fm10k_mbx_lock(hw); fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true); fm10k_mbx_unlock(hw); } } static void fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); /* Add default mac address */ fm10k_MAC_filter_set(dev, hw->mac.addr, true, MAIN_VSI_POOL_NUMBER); } static void fm10k_dev_rss_configure(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_eth_conf *dev_conf = &dev->data->dev_conf; uint32_t mrqc, *key, i, reta, j; uint64_t hf; #define RSS_KEY_SIZE 40 static uint8_t rss_intel_key[RSS_KEY_SIZE] = { 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, }; if (dev->data->nb_rx_queues == 1 || dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS || dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) { FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0); return; } /* random key is rss_intel_key (default) or user provided (rss_key) */ if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL) key = (uint32_t *)rss_intel_key; else key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key; /* Now fill our hash function seeds, 4 bytes at a time */ for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i) FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]); /* * Fill in redirection table * The byte-swap is needed because NIC registers are in * little-endian order. */ reta = 0; for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) { if (j == dev->data->nb_rx_queues) j = 0; reta = (reta << CHAR_BIT) | j; if ((i & 3) == 3) FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), rte_bswap32(reta)); } /* * Generate RSS hash based on packet types, TCP/UDP * port numbers and/or IPv4/v6 src and dst addresses */ hf = dev_conf->rx_adv_conf.rss_conf.rss_hf; mrqc = 0; mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0; mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0; mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0; mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0; mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0; mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0; mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0; mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0; mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0; if (mrqc == 0) { PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not" "supported", hf); return; } FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc); } static void fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t i; for (i = 0; i < nb_lport_new; i++) { /* Set unicast mode by default. App can change * to other mode in other API func. */ fm10k_mbx_lock(hw); hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i, FM10K_XCAST_MODE_NONE); fm10k_mbx_unlock(hw); } } static void fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_eth_vmdq_rx_conf *vmdq_conf; struct rte_eth_conf *dev_conf = &dev->data->dev_conf; struct fm10k_macvlan_filter_info *macvlan; uint16_t nb_queue_pools = 0; /* pool number in configuration */ uint16_t nb_lport_new; macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; fm10k_dev_rss_configure(dev); /* only PF supports VMDQ */ if (hw->mac.type != fm10k_mac_pf) return; if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG) nb_queue_pools = vmdq_conf->nb_queue_pools; /* no pool number change, no need to update logic port and VLAN/MAC */ if (macvlan->nb_queue_pools == nb_queue_pools) return; nb_lport_new = nb_queue_pools ? nb_queue_pools : 1; fm10k_dev_logic_port_update(dev, nb_lport_new); /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */ memset(dev->data->mac_addrs, 0, ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM); ether_addr_copy((const struct ether_addr *)hw->mac.addr, &dev->data->mac_addrs[0]); memset(macvlan, 0, sizeof(*macvlan)); macvlan->nb_queue_pools = nb_queue_pools; if (nb_queue_pools) fm10k_dev_vmdq_rx_configure(dev); else fm10k_dev_pf_main_vsi_reset(dev); } static int fm10k_dev_tx_init(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); int i, ret; struct fm10k_tx_queue *txq; uint64_t base_addr; uint32_t size; /* Disable TXINT to avoid possible interrupt */ for (i = 0; i < hw->mac.max_queues; i++) FM10K_WRITE_REG(hw, FM10K_TXINT(i), 3 << FM10K_TXINT_TIMER_SHIFT); /* Setup TX queue */ for (i = 0; i < dev->data->nb_tx_queues; ++i) { txq = dev->data->tx_queues[i]; base_addr = txq->hw_ring_phys_addr; size = txq->nb_desc * sizeof(struct fm10k_tx_desc); /* disable queue to avoid issues while updating state */ ret = tx_queue_disable(hw, i); if (ret) { PMD_INIT_LOG(ERR, "failed to disable queue %d", i); return -1; } /* Enable use of FTAG bit in TX descriptor, PFVTCTL * register is read-only for VF. */ if (fm10k_check_ftag(dev->device->devargs)) { if (hw->mac.type == fm10k_mac_pf) { FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i), FM10K_PFVTCTL_FTAG_DESC_ENABLE); PMD_INIT_LOG(DEBUG, "FTAG mode is enabled"); } else { PMD_INIT_LOG(ERR, "VF FTAG is not supported."); return -ENOTSUP; } } /* set location and size for descriptor ring */ FM10K_WRITE_REG(hw, FM10K_TDBAL(i), base_addr & UINT64_LOWER_32BITS_MASK); FM10K_WRITE_REG(hw, FM10K_TDBAH(i), base_addr >> (CHAR_BIT * sizeof(uint32_t))); FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size); /* assign default SGLORT for each TX queue by PF */ if (hw->mac.type == fm10k_mac_pf) FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map); } /* set up vector or scalar TX function as appropriate */ fm10k_set_tx_function(dev); return 0; } static int fm10k_dev_rx_init(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct fm10k_macvlan_filter_info *macvlan; struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pdev->intr_handle; int i, ret; struct fm10k_rx_queue *rxq; uint64_t base_addr; uint32_t size; uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; uint32_t logic_port = hw->mac.dglort_map; uint16_t buf_size; uint16_t queue_stride = 0; /* enable RXINT for interrupt mode */ i = 0; if (rte_intr_dp_is_en(intr_handle)) { for (; i < dev->data->nb_rx_queues; i++) { FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i)); if (hw->mac.type == fm10k_mac_pf) FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); else FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); } } /* Disable other RXINT to avoid possible interrupt */ for (; i < hw->mac.max_queues; i++) FM10K_WRITE_REG(hw, FM10K_RXINT(i), 3 << FM10K_RXINT_TIMER_SHIFT); /* Setup RX queues */ for (i = 0; i < dev->data->nb_rx_queues; ++i) { rxq = dev->data->rx_queues[i]; base_addr = rxq->hw_ring_phys_addr; size = rxq->nb_desc * sizeof(union fm10k_rx_desc); /* disable queue to avoid issues while updating state */ ret = rx_queue_disable(hw, i); if (ret) { PMD_INIT_LOG(ERR, "failed to disable queue %d", i); return -1; } /* Setup the Base and Length of the Rx Descriptor Ring */ FM10K_WRITE_REG(hw, FM10K_RDBAL(i), base_addr & UINT64_LOWER_32BITS_MASK); FM10K_WRITE_REG(hw, FM10K_RDBAH(i), base_addr >> (CHAR_BIT * sizeof(uint32_t))); FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size); /* Configure the Rx buffer size for one buff without split */ buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM); /* As RX buffer is aligned to 512B within mbuf, some bytes are * reserved for this purpose, and the worst case could be 511B. * But SRR reg assumes all buffers have the same size. In order * to fill the gap, we'll have to consider the worst case and * assume 512B is reserved. If we don't do so, it's possible * for HW to overwrite data to next mbuf. */ buf_size -= FM10K_RX_DATABUF_ALIGN; FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) | FM10K_SRRCTL_LOOPBACK_SUPPRESS); /* It adds dual VLAN length for supporting dual VLAN */ if ((dev->data->dev_conf.rxmode.max_rx_pkt_len + 2 * FM10K_VLAN_TAG_SIZE) > buf_size || dev->data->dev_conf.rxmode.enable_scatter) { uint32_t reg; dev->data->scattered_rx = 1; reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i)); reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN; FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg); } /* Enable drop on empty, it's RO for VF */ if (hw->mac.type == fm10k_mac_pf && rxq->drop_en) rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY; FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl); FM10K_WRITE_FLUSH(hw); } /* Configure VMDQ/RSS if applicable */ fm10k_dev_mq_rx_configure(dev); /* Decide the best RX function */ fm10k_set_rx_function(dev); /* update RX_SGLORT for loopback suppress*/ if (hw->mac.type != fm10k_mac_pf) return 0; macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); if (macvlan->nb_queue_pools) queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools; for (i = 0; i < dev->data->nb_rx_queues; ++i) { if (i && queue_stride && !(i % queue_stride)) logic_port++; FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port); } return 0; } static int fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); int err = -1; uint32_t reg; struct fm10k_rx_queue *rxq; PMD_INIT_FUNC_TRACE(); if (rx_queue_id < dev->data->nb_rx_queues) { rxq = dev->data->rx_queues[rx_queue_id]; err = rx_queue_reset(rxq); if (err == -ENOMEM) { PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err); return err; } else if (err == -EINVAL) { PMD_INIT_LOG(ERR, "Invalid buffer address alignment :" " %d", err); return err; } /* Setup the HW Rx Head and Tail Descriptor Pointers * Note: this must be done AFTER the queue is enabled on real * hardware, but BEFORE the queue is enabled when using the * emulation platform. Do it in both places for now and remove * this comment and the following two register writes when the * emulation platform is no longer being used. */ FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0); FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1); /* Set PF ownership flag for PF devices */ reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id)); if (hw->mac.type == fm10k_mac_pf) reg |= FM10K_RXQCTL_PF; reg |= FM10K_RXQCTL_ENABLE; /* enable RX queue */ FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg); FM10K_WRITE_FLUSH(hw); /* Setup the HW Rx Head and Tail Descriptor Pointers * Note: this must be done AFTER the queue is enabled */ FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0); FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1); dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; } return err; } static int fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); if (rx_queue_id < dev->data->nb_rx_queues) { /* Disable RX queue */ rx_queue_disable(hw, rx_queue_id); /* Free mbuf and clean HW ring */ rx_queue_clean(dev->data->rx_queues[rx_queue_id]); dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; } return 0; } static int fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); /** @todo - this should be defined in the shared code */ #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY 0x00010000 uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY; int err = 0; PMD_INIT_FUNC_TRACE(); if (tx_queue_id < dev->data->nb_tx_queues) { struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id]; q->ops->reset(q); /* reset head and tail pointers */ FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0); FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0); /* enable TX queue */ FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id), FM10K_TXDCTL_ENABLE | txdctl); FM10K_WRITE_FLUSH(hw); dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; } else err = -1; return err; } static int fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); if (tx_queue_id < dev->data->nb_tx_queues) { tx_queue_disable(hw, tx_queue_id); tx_queue_clean(dev->data->tx_queues[tx_queue_id]); dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; } return 0; } static inline int fm10k_glort_valid(struct fm10k_hw *hw) { return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE) != FM10K_DGLORTMAP_NONE); } static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); int status; PMD_INIT_FUNC_TRACE(); /* Return if it didn't acquire valid glort range */ if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw)) return; fm10k_mbx_lock(hw); status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map, FM10K_XCAST_MODE_PROMISC); fm10k_mbx_unlock(hw); if (status != FM10K_SUCCESS) PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode"); } static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint8_t mode; int status; PMD_INIT_FUNC_TRACE(); /* Return if it didn't acquire valid glort range */ if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw)) return; if (dev->data->all_multicast == 1) mode = FM10K_XCAST_MODE_ALLMULTI; else mode = FM10K_XCAST_MODE_NONE; fm10k_mbx_lock(hw); status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map, mode); fm10k_mbx_unlock(hw); if (status != FM10K_SUCCESS) PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode"); } static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); int status; PMD_INIT_FUNC_TRACE(); /* Return if it didn't acquire valid glort range */ if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw)) return; /* If promiscuous mode is enabled, it doesn't make sense to enable * allmulticast and disable promiscuous since fm10k only can select * one of the modes. */ if (dev->data->promiscuous) { PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\ "needn't enable allmulticast"); return; } fm10k_mbx_lock(hw); status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map, FM10K_XCAST_MODE_ALLMULTI); fm10k_mbx_unlock(hw); if (status != FM10K_SUCCESS) PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode"); } static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); int status; PMD_INIT_FUNC_TRACE(); /* Return if it didn't acquire valid glort range */ if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw)) return; if (dev->data->promiscuous) { PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\ "since promisc mode is enabled"); return; } fm10k_mbx_lock(hw); /* Change mode to unicast mode */ status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map, FM10K_XCAST_MODE_NONE); fm10k_mbx_unlock(hw); if (status != FM10K_SUCCESS) PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode"); } static void fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t dglortdec, pool_len, rss_len, i, dglortmask; uint16_t nb_queue_pools; struct fm10k_macvlan_filter_info *macvlan; macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); nb_queue_pools = macvlan->nb_queue_pools; pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0; rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len; /* GLORT 0x0-0x3F are used by PF and VMDQ, 0x40-0x7F used by FD */ dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len; dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) | hw->mac.dglort_map; FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask); /* Configure VMDQ/RSS DGlort Decoder */ FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec); /* Flow Director configurations, only queue number is valid. */ dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1); dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) | (hw->mac.dglort_map + GLORT_FD_Q_BASE); FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask); FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec); /* Invalidate all other GLORT entries */ for (i = 2; i < FM10K_DGLORT_COUNT; i++) FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i), FM10K_DGLORTMAP_NONE); } #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1) static int fm10k_dev_start(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); int i, diag; PMD_INIT_FUNC_TRACE(); /* stop, init, then start the hw */ diag = fm10k_stop_hw(hw); if (diag != FM10K_SUCCESS) { PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag); return -EIO; } diag = fm10k_init_hw(hw); if (diag != FM10K_SUCCESS) { PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag); return -EIO; } diag = fm10k_start_hw(hw); if (diag != FM10K_SUCCESS) { PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag); return -EIO; } diag = fm10k_dev_tx_init(dev); if (diag) { PMD_INIT_LOG(ERR, "TX init failed: %d", diag); return diag; } if (fm10k_dev_rxq_interrupt_setup(dev)) return -EIO; diag = fm10k_dev_rx_init(dev); if (diag) { PMD_INIT_LOG(ERR, "RX init failed: %d", diag); return diag; } if (hw->mac.type == fm10k_mac_pf) fm10k_dev_dglort_map_configure(dev); for (i = 0; i < dev->data->nb_rx_queues; i++) { struct fm10k_rx_queue *rxq; rxq = dev->data->rx_queues[i]; if (rxq->rx_deferred_start) continue; diag = fm10k_dev_rx_queue_start(dev, i); if (diag != 0) { int j; for (j = 0; j < i; ++j) rx_queue_clean(dev->data->rx_queues[j]); return diag; } } for (i = 0; i < dev->data->nb_tx_queues; i++) { struct fm10k_tx_queue *txq; txq = dev->data->tx_queues[i]; if (txq->tx_deferred_start) continue; diag = fm10k_dev_tx_queue_start(dev, i); if (diag != 0) { int j; for (j = 0; j < i; ++j) tx_queue_clean(dev->data->tx_queues[j]); for (j = 0; j < dev->data->nb_rx_queues; ++j) rx_queue_clean(dev->data->rx_queues[j]); return diag; } } /* Update default vlan when not in VMDQ mode */ if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)) fm10k_vlan_filter_set(dev, hw->mac.default_vid, true); fm10k_link_update(dev, 0); return 0; } static void fm10k_dev_stop(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pdev->intr_handle; int i; PMD_INIT_FUNC_TRACE(); if (dev->data->tx_queues) for (i = 0; i < dev->data->nb_tx_queues; i++) fm10k_dev_tx_queue_stop(dev, i); if (dev->data->rx_queues) for (i = 0; i < dev->data->nb_rx_queues; i++) fm10k_dev_rx_queue_stop(dev, i); /* Disable datapath event */ if (rte_intr_dp_is_en(intr_handle)) { for (i = 0; i < dev->data->nb_rx_queues; i++) { FM10K_WRITE_REG(hw, FM10K_RXINT(i), 3 << FM10K_RXINT_TIMER_SHIFT); if (hw->mac.type == fm10k_mac_pf) FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)), FM10K_ITR_MASK_SET); else FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)), FM10K_ITR_MASK_SET); } } /* Clean datapath event and queue/vec mapping */ rte_intr_efd_disable(intr_handle); rte_free(intr_handle->intr_vec); intr_handle->intr_vec = NULL; } static void fm10k_dev_queue_release(struct rte_eth_dev *dev) { int i; PMD_INIT_FUNC_TRACE(); if (dev->data->tx_queues) { for (i = 0; i < dev->data->nb_tx_queues; i++) { struct fm10k_tx_queue *txq = dev->data->tx_queues[i]; tx_queue_free(txq); } } if (dev->data->rx_queues) { for (i = 0; i < dev->data->nb_rx_queues; i++) fm10k_rx_queue_release(dev->data->rx_queues[i]); } } static void fm10k_dev_close(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); fm10k_mbx_lock(hw); hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, MAX_LPORT_NUM, false); fm10k_mbx_unlock(hw); /* allow 100ms for device to quiesce */ rte_delay_us(FM10K_SWITCH_QUIESCE_US); /* Stop mailbox service first */ fm10k_close_mbx_service(hw); fm10k_dev_stop(dev); fm10k_dev_queue_release(dev); fm10k_stop_hw(hw); } static int fm10k_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) { struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); /* The speed is ~50Gbps per Gen3 x8 PCIe interface. For now, we * leave the speed undefined since there is no 50Gbps Ethernet. */ dev->data->dev_link.link_speed = 0; dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; dev->data->dev_link.link_status = dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP; return 0; } static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev, struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit) { unsigned i, q; unsigned count = 0; if (xstats_names != NULL) { /* Note: limit checked in rte_eth_xstats_names() */ /* Global stats */ for (i = 0; i < FM10K_NB_HW_XSTATS; i++) { snprintf(xstats_names[count].name, sizeof(xstats_names[count].name), "%s", fm10k_hw_stats_strings[count].name); count++; } /* PF queue stats */ for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) { for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) { snprintf(xstats_names[count].name, sizeof(xstats_names[count].name), "rx_q%u_%s", q, fm10k_hw_stats_rx_q_strings[i].name); count++; } for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) { snprintf(xstats_names[count].name, sizeof(xstats_names[count].name), "tx_q%u_%s", q, fm10k_hw_stats_tx_q_strings[i].name); count++; } } } return FM10K_NB_XSTATS; } static int fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, unsigned n) { struct fm10k_hw_stats *hw_stats = FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private); unsigned i, q, count = 0; if (n < FM10K_NB_XSTATS) return FM10K_NB_XSTATS; /* Global stats */ for (i = 0; i < FM10K_NB_HW_XSTATS; i++) { xstats[count].value = *(uint64_t *)(((char *)hw_stats) + fm10k_hw_stats_strings[count].offset); xstats[count].id = count; count++; } /* PF queue stats */ for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) { for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) { xstats[count].value = *(uint64_t *)(((char *)&hw_stats->q[q]) + fm10k_hw_stats_rx_q_strings[i].offset); xstats[count].id = count; count++; } for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) { xstats[count].value = *(uint64_t *)(((char *)&hw_stats->q[q]) + fm10k_hw_stats_tx_q_strings[i].offset); xstats[count].id = count; count++; } } return FM10K_NB_XSTATS; } static int fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { uint64_t ipackets, opackets, ibytes, obytes; struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct fm10k_hw_stats *hw_stats = FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private); int i; PMD_INIT_FUNC_TRACE(); fm10k_update_hw_stats(hw, hw_stats); ipackets = opackets = ibytes = obytes = 0; for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) && (i < hw->mac.max_queues); ++i) { stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count; stats->q_opackets[i] = hw_stats->q[i].tx_packets.count; stats->q_ibytes[i] = hw_stats->q[i].rx_bytes.count; stats->q_obytes[i] = hw_stats->q[i].tx_bytes.count; ipackets += stats->q_ipackets[i]; opackets += stats->q_opackets[i]; ibytes += stats->q_ibytes[i]; obytes += stats->q_obytes[i]; } stats->ipackets = ipackets; stats->opackets = opackets; stats->ibytes = ibytes; stats->obytes = obytes; return 0; } static void fm10k_stats_reset(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct fm10k_hw_stats *hw_stats = FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); memset(hw_stats, 0, sizeof(*hw_stats)); fm10k_rebind_hw_stats(hw, hw_stats); } static void fm10k_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); PMD_INIT_FUNC_TRACE(); dev_info->pci_dev = pdev; dev_info->min_rx_bufsize = FM10K_MIN_RX_BUF_SIZE; dev_info->max_rx_pktlen = FM10K_MAX_PKT_SIZE; dev_info->max_rx_queues = hw->mac.max_queues; dev_info->max_tx_queues = hw->mac.max_queues; dev_info->max_mac_addrs = FM10K_MAX_MACADDR_NUM; dev_info->max_hash_mac_addrs = 0; dev_info->max_vfs = pdev->max_vfs; dev_info->vmdq_pool_base = 0; dev_info->vmdq_queue_base = 0; dev_info->max_vmdq_pools = ETH_32_POOLS; dev_info->vmdq_queue_num = FM10K_MAX_QUEUES_PF; dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | DEV_RX_OFFLOAD_IPV4_CKSUM | DEV_RX_OFFLOAD_UDP_CKSUM | DEV_RX_OFFLOAD_TCP_CKSUM; dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM | DEV_TX_OFFLOAD_TCP_CKSUM | DEV_TX_OFFLOAD_TCP_TSO; dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t); dev_info->reta_size = FM10K_MAX_RSS_INDICES; dev_info->default_rxconf = (struct rte_eth_rxconf) { .rx_thresh = { .pthresh = FM10K_DEFAULT_RX_PTHRESH, .hthresh = FM10K_DEFAULT_RX_HTHRESH, .wthresh = FM10K_DEFAULT_RX_WTHRESH, }, .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0), .rx_drop_en = 0, }; dev_info->default_txconf = (struct rte_eth_txconf) { .tx_thresh = { .pthresh = FM10K_DEFAULT_TX_PTHRESH, .hthresh = FM10K_DEFAULT_TX_HTHRESH, .wthresh = FM10K_DEFAULT_TX_WTHRESH, }, .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0), .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0), .txq_flags = FM10K_SIMPLE_TX_FLAG, }; dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { .nb_max = FM10K_MAX_RX_DESC, .nb_min = FM10K_MIN_RX_DESC, .nb_align = FM10K_MULT_RX_DESC, }; dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { .nb_max = FM10K_MAX_TX_DESC, .nb_min = FM10K_MIN_TX_DESC, .nb_align = FM10K_MULT_TX_DESC, .nb_seg_max = FM10K_TX_MAX_SEG, .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG, }; dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G | ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G | ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G; } #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE static const uint32_t * fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev) { if (dev->rx_pkt_burst == fm10k_recv_pkts || dev->rx_pkt_burst == fm10k_recv_scattered_pkts) { static uint32_t ptypes[] = { /* refers to rx_desc_to_ol_flags() */ RTE_PTYPE_L2_ETHER, RTE_PTYPE_L3_IPV4, RTE_PTYPE_L3_IPV4_EXT, RTE_PTYPE_L3_IPV6, RTE_PTYPE_L3_IPV6_EXT, RTE_PTYPE_L4_TCP, RTE_PTYPE_L4_UDP, RTE_PTYPE_UNKNOWN }; return ptypes; } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec || dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) { static uint32_t ptypes_vec[] = { /* refers to fm10k_desc_to_pktype_v() */ RTE_PTYPE_L3_IPV4, RTE_PTYPE_L3_IPV4_EXT, RTE_PTYPE_L3_IPV6, RTE_PTYPE_L3_IPV6_EXT, RTE_PTYPE_L4_TCP, RTE_PTYPE_L4_UDP, RTE_PTYPE_TUNNEL_GENEVE, RTE_PTYPE_TUNNEL_NVGRE, RTE_PTYPE_TUNNEL_VXLAN, RTE_PTYPE_TUNNEL_GRE, RTE_PTYPE_UNKNOWN }; return ptypes_vec; } return NULL; } #else static const uint32_t * fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) { return NULL; } #endif static int fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) { s32 result; uint16_t mac_num = 0; uint32_t vid_idx, vid_bit, mac_index; struct fm10k_hw *hw; struct fm10k_macvlan_filter_info *macvlan; struct rte_eth_dev_data *data = dev->data; hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */ PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode"); return -EINVAL; } if (vlan_id > ETH_VLAN_ID_MAX) { PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096"); return -EINVAL; } vid_idx = FM10K_VFTA_IDX(vlan_id); vid_bit = FM10K_VFTA_BIT(vlan_id); /* this VLAN ID is already in the VLAN filter table, return SUCCESS */ if (on && (macvlan->vfta[vid_idx] & vid_bit)) return 0; /* this VLAN ID is NOT in the VLAN filter table, cannot remove */ if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) { PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing " "in the VLAN filter table"); return -EINVAL; } fm10k_mbx_lock(hw); result = fm10k_update_vlan(hw, vlan_id, 0, on); fm10k_mbx_unlock(hw); if (result != FM10K_SUCCESS) { PMD_INIT_LOG(ERR, "VLAN update failed: %d", result); return -EIO; } for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) && (result == FM10K_SUCCESS); mac_index++) { if (is_zero_ether_addr(&data->mac_addrs[mac_index])) continue; if (mac_num > macvlan->mac_num - 1) { PMD_INIT_LOG(ERR, "MAC address number " "not match"); break; } fm10k_mbx_lock(hw); result = fm10k_update_uc_addr(hw, hw->mac.dglort_map, data->mac_addrs[mac_index].addr_bytes, vlan_id, on, 0); fm10k_mbx_unlock(hw); mac_num++; } if (result != FM10K_SUCCESS) { PMD_INIT_LOG(ERR, "MAC address update failed: %d", result); return -EIO; } if (on) { macvlan->vlan_num++; macvlan->vfta[vid_idx] |= vid_bit; } else { macvlan->vlan_num--; macvlan->vfta[vid_idx] &= ~vid_bit; } return 0; } static int fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask) { if (mask & ETH_VLAN_STRIP_MASK) { if (!dev->data->dev_conf.rxmode.hw_vlan_strip) PMD_INIT_LOG(ERR, "VLAN stripping is " "always on in fm10k"); } if (mask & ETH_VLAN_EXTEND_MASK) { if (dev->data->dev_conf.rxmode.hw_vlan_extend) PMD_INIT_LOG(ERR, "VLAN QinQ is not " "supported in fm10k"); } if (mask & ETH_VLAN_FILTER_MASK) { if (!dev->data->dev_conf.rxmode.hw_vlan_filter) PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k"); } return 0; } /* Add/Remove a MAC address, and update filters to main VSI */ static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev, const u8 *mac, bool add, uint32_t pool) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct fm10k_macvlan_filter_info *macvlan; uint32_t i, j, k; macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); if (pool != MAIN_VSI_POOL_NUMBER) { PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set " "mac to pool %u", pool); return; } for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) { if (!macvlan->vfta[j]) continue; for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) { if (!(macvlan->vfta[j] & (1 << k))) continue; if (i + 1 > macvlan->vlan_num) { PMD_INIT_LOG(ERR, "vlan number not match"); return; } fm10k_mbx_lock(hw); fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac, j * FM10K_UINT32_BIT_SIZE + k, add, 0); fm10k_mbx_unlock(hw); i++; } } } /* Add/Remove a MAC address, and update filters to VMDQ */ static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev, const u8 *mac, bool add, uint32_t pool) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct fm10k_macvlan_filter_info *macvlan; struct rte_eth_vmdq_rx_conf *vmdq_conf; uint32_t i; macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; if (pool > macvlan->nb_queue_pools) { PMD_DRV_LOG(ERR, "Pool number %u invalid." " Max pool is %u", pool, macvlan->nb_queue_pools); return; } for (i = 0; i < vmdq_conf->nb_pool_maps; i++) { if (!(vmdq_conf->pool_map[i].pools & (1UL << pool))) continue; fm10k_mbx_lock(hw); fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac, vmdq_conf->pool_map[i].vlan_id, add, 0); fm10k_mbx_unlock(hw); } } /* Add/Remove a MAC address, and update filters */ static void fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add, uint32_t pool) { struct fm10k_macvlan_filter_info *macvlan; macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); if (macvlan->nb_queue_pools > 0) /* VMDQ mode */ fm10k_MAC_filter_set_vmdq(dev, mac, add, pool); else fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool); if (add) macvlan->mac_num++; else macvlan->mac_num--; } /* Add a MAC address, and update filters */ static int fm10k_macaddr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, uint32_t index, uint32_t pool) { struct fm10k_macvlan_filter_info *macvlan; macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool); macvlan->mac_vmdq_id[index] = pool; return 0; } /* Remove a MAC address, and update filters */ static void fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index) { struct rte_eth_dev_data *data = dev->data; struct fm10k_macvlan_filter_info *macvlan; macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes, FALSE, macvlan->mac_vmdq_id[index]); macvlan->mac_vmdq_id[index] = 0; } static inline int check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request) { if ((request < min) || (request > max) || ((request % mult) != 0)) return -1; else return 0; } static inline int check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request) { if ((request < min) || (request > max) || ((div % request) != 0)) return -1; else return 0; } static inline int handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf) { uint16_t rx_free_thresh; if (conf->rx_free_thresh == 0) rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q); else rx_free_thresh = conf->rx_free_thresh; /* make sure the requested threshold satisfies the constraints */ if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q), FM10K_RX_FREE_THRESH_MAX(q), FM10K_RX_FREE_THRESH_DIV(q), rx_free_thresh)) { PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be " "less than or equal to %u, " "greater than or equal to %u, " "and a divisor of %u", rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q), FM10K_RX_FREE_THRESH_MIN(q), FM10K_RX_FREE_THRESH_DIV(q)); return -EINVAL; } q->alloc_thresh = rx_free_thresh; q->drop_en = conf->rx_drop_en; q->rx_deferred_start = conf->rx_deferred_start; return 0; } /* * Hardware requires specific alignment for Rx packet buffers. At * least one of the following two conditions must be satisfied. * 1. Address is 512B aligned * 2. Address is 8B aligned and buffer does not cross 4K boundary. * * As such, the driver may need to adjust the DMA address within the * buffer by up to 512B. * * return 1 if the element size is valid, otherwise return 0. */ static int mempool_element_size_valid(struct rte_mempool *mp) { uint32_t min_size; /* elt_size includes mbuf header and headroom */ min_size = mp->elt_size - sizeof(struct rte_mbuf) - RTE_PKTMBUF_HEADROOM; /* account for up to 512B of alignment */ min_size -= FM10K_RX_DATABUF_ALIGN; /* sanity check for overflow */ if (min_size > mp->elt_size) return 0; /* size is valid */ return 1; } static int fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc, unsigned int socket_id, const struct rte_eth_rxconf *conf, struct rte_mempool *mp) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); struct fm10k_rx_queue *q; const struct rte_memzone *mz; PMD_INIT_FUNC_TRACE(); /* make sure the mempool element size can account for alignment. */ if (!mempool_element_size_valid(mp)) { PMD_INIT_LOG(ERR, "Error : Mempool element size is too small"); return -EINVAL; } /* make sure a valid number of descriptors have been requested */ if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC, FM10K_MULT_RX_DESC, nb_desc)) { PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be " "less than or equal to %"PRIu32", " "greater than or equal to %u, " "and a multiple of %u", nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC, FM10K_MULT_RX_DESC); return -EINVAL; } /* * if this queue existed already, free the associated memory. The * queue cannot be reused in case we need to allocate memory on * different socket than was previously used. */ if (dev->data->rx_queues[queue_id] != NULL) { rx_queue_free(dev->data->rx_queues[queue_id]); dev->data->rx_queues[queue_id] = NULL; } /* allocate memory for the queue structure */ q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE, socket_id); if (q == NULL) { PMD_INIT_LOG(ERR, "Cannot allocate queue structure"); return -ENOMEM; } /* setup queue */ q->mp = mp; q->nb_desc = nb_desc; q->nb_fake_desc = FM10K_MULT_RX_DESC; q->port_id = dev->data->port_id; q->queue_id = queue_id; q->tail_ptr = (volatile uint32_t *) &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)]; if (handle_rxconf(q, conf)) return -EINVAL; /* allocate memory for the software ring */ q->sw_ring = rte_zmalloc_socket("fm10k sw ring", (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *), RTE_CACHE_LINE_SIZE, socket_id); if (q->sw_ring == NULL) { PMD_INIT_LOG(ERR, "Cannot allocate software ring"); rte_free(q); return -ENOMEM; } /* * allocate memory for the hardware descriptor ring. A memzone large * enough to hold the maximum ring size is requested to allow for * resizing in later calls to the queue setup function. */ mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id, FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC, socket_id); if (mz == NULL) { PMD_INIT_LOG(ERR, "Cannot allocate hardware ring"); rte_free(q->sw_ring); rte_free(q); return -ENOMEM; } q->hw_ring = mz->addr; q->hw_ring_phys_addr = mz->iova; /* Check if number of descs satisfied Vector requirement */ if (!rte_is_power_of_2(nb_desc)) { PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx " "preconditions - canceling the feature for " "the whole port[%d]", q->queue_id, q->port_id); dev_info->rx_vec_allowed = false; } else fm10k_rxq_vec_setup(q); dev->data->rx_queues[queue_id] = q; return 0; } static void fm10k_rx_queue_release(void *queue) { PMD_INIT_FUNC_TRACE(); rx_queue_free(queue); } static inline int handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf) { uint16_t tx_free_thresh; uint16_t tx_rs_thresh; /* constraint MACROs require that tx_free_thresh is configured * before tx_rs_thresh */ if (conf->tx_free_thresh == 0) tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q); else tx_free_thresh = conf->tx_free_thresh; /* make sure the requested threshold satisfies the constraints */ if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q), FM10K_TX_FREE_THRESH_MAX(q), FM10K_TX_FREE_THRESH_DIV(q), tx_free_thresh)) { PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be " "less than or equal to %u, " "greater than or equal to %u, " "and a divisor of %u", tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q), FM10K_TX_FREE_THRESH_MIN(q), FM10K_TX_FREE_THRESH_DIV(q)); return -EINVAL; } q->free_thresh = tx_free_thresh; if (conf->tx_rs_thresh == 0) tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q); else tx_rs_thresh = conf->tx_rs_thresh; q->tx_deferred_start = conf->tx_deferred_start; /* make sure the requested threshold satisfies the constraints */ if (check_thresh(FM10K_TX_RS_THRESH_MIN(q), FM10K_TX_RS_THRESH_MAX(q), FM10K_TX_RS_THRESH_DIV(q), tx_rs_thresh)) { PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be " "less than or equal to %u, " "greater than or equal to %u, " "and a divisor of %u", tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q), FM10K_TX_RS_THRESH_MIN(q), FM10K_TX_RS_THRESH_DIV(q)); return -EINVAL; } q->rs_thresh = tx_rs_thresh; return 0; } static int fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc, unsigned int socket_id, const struct rte_eth_txconf *conf) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct fm10k_tx_queue *q; const struct rte_memzone *mz; PMD_INIT_FUNC_TRACE(); /* make sure a valid number of descriptors have been requested */ if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC, FM10K_MULT_TX_DESC, nb_desc)) { PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be " "less than or equal to %"PRIu32", " "greater than or equal to %u, " "and a multiple of %u", nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC, FM10K_MULT_TX_DESC); return -EINVAL; } /* * if this queue existed already, free the associated memory. The * queue cannot be reused in case we need to allocate memory on * different socket than was previously used. */ if (dev->data->tx_queues[queue_id] != NULL) { struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id]; tx_queue_free(txq); dev->data->tx_queues[queue_id] = NULL; } /* allocate memory for the queue structure */ q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE, socket_id); if (q == NULL) { PMD_INIT_LOG(ERR, "Cannot allocate queue structure"); return -ENOMEM; } /* setup queue */ q->nb_desc = nb_desc; q->port_id = dev->data->port_id; q->queue_id = queue_id; q->txq_flags = conf->txq_flags; q->ops = &def_txq_ops; q->tail_ptr = (volatile uint32_t *) &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)]; if (handle_txconf(q, conf)) return -EINVAL; /* allocate memory for the software ring */ q->sw_ring = rte_zmalloc_socket("fm10k sw ring", nb_desc * sizeof(struct rte_mbuf *), RTE_CACHE_LINE_SIZE, socket_id); if (q->sw_ring == NULL) { PMD_INIT_LOG(ERR, "Cannot allocate software ring"); rte_free(q); return -ENOMEM; } /* * allocate memory for the hardware descriptor ring. A memzone large * enough to hold the maximum ring size is requested to allow for * resizing in later calls to the queue setup function. */ mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id, FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC, socket_id); if (mz == NULL) { PMD_INIT_LOG(ERR, "Cannot allocate hardware ring"); rte_free(q->sw_ring); rte_free(q); return -ENOMEM; } q->hw_ring = mz->addr; q->hw_ring_phys_addr = mz->iova; /* * allocate memory for the RS bit tracker. Enough slots to hold the * descriptor index for each RS bit needing to be set are required. */ q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker", ((nb_desc + 1) / q->rs_thresh) * sizeof(uint16_t), RTE_CACHE_LINE_SIZE, socket_id); if (q->rs_tracker.list == NULL) { PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker"); rte_free(q->sw_ring); rte_free(q); return -ENOMEM; } dev->data->tx_queues[queue_id] = q; return 0; } static void fm10k_tx_queue_release(void *queue) { struct fm10k_tx_queue *q = queue; PMD_INIT_FUNC_TRACE(); tx_queue_free(q); } static int fm10k_reta_update(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint16_t i, j, idx, shift; uint8_t mask; uint32_t reta; PMD_INIT_FUNC_TRACE(); if (reta_size > FM10K_MAX_RSS_INDICES) { PMD_INIT_LOG(ERR, "The size of hash lookup table configured " "(%d) doesn't match the number hardware can supported " "(%d)", reta_size, FM10K_MAX_RSS_INDICES); return -EINVAL; } /* * Update Redirection Table RETA[n], n=0..31. The redirection table has * 128-entries in 32 registers */ for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) { idx = i / RTE_RETA_GROUP_SIZE; shift = i % RTE_RETA_GROUP_SIZE; mask = (uint8_t)((reta_conf[idx].mask >> shift) & BIT_MASK_PER_UINT32); if (mask == 0) continue; reta = 0; if (mask != BIT_MASK_PER_UINT32) reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2)); for (j = 0; j < CHARS_PER_UINT32; j++) { if (mask & (0x1 << j)) { if (mask != 0xF) reta &= ~(UINT8_MAX << CHAR_BIT * j); reta |= reta_conf[idx].reta[shift + j] << (CHAR_BIT * j); } } FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta); } return 0; } static int fm10k_reta_query(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint16_t i, j, idx, shift; uint8_t mask; uint32_t reta; PMD_INIT_FUNC_TRACE(); if (reta_size < FM10K_MAX_RSS_INDICES) { PMD_INIT_LOG(ERR, "The size of hash lookup table configured " "(%d) doesn't match the number hardware can supported " "(%d)", reta_size, FM10K_MAX_RSS_INDICES); return -EINVAL; } /* * Read Redirection Table RETA[n], n=0..31. The redirection table has * 128-entries in 32 registers */ for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) { idx = i / RTE_RETA_GROUP_SIZE; shift = i % RTE_RETA_GROUP_SIZE; mask = (uint8_t)((reta_conf[idx].mask >> shift) & BIT_MASK_PER_UINT32); if (mask == 0) continue; reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2)); for (j = 0; j < CHARS_PER_UINT32; j++) { if (mask & (0x1 << j)) reta_conf[idx].reta[shift + j] = ((reta >> CHAR_BIT * j) & UINT8_MAX); } } return 0; } static int fm10k_rss_hash_update(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t *key = (uint32_t *)rss_conf->rss_key; uint32_t mrqc; uint64_t hf = rss_conf->rss_hf; int i; PMD_INIT_FUNC_TRACE(); if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE * FM10K_RSSRK_ENTRIES_PER_REG)) return -EINVAL; if (hf == 0) return -EINVAL; mrqc = 0; mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0; mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0; mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0; mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0; mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0; mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0; mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0; mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0; mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0; /* If the mapping doesn't fit any supported, return */ if (mrqc == 0) return -EINVAL; if (key != NULL) for (i = 0; i < FM10K_RSSRK_SIZE; ++i) FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]); FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc); return 0; } static int fm10k_rss_hash_conf_get(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t *key = (uint32_t *)rss_conf->rss_key; uint32_t mrqc; uint64_t hf; int i; PMD_INIT_FUNC_TRACE(); if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE * FM10K_RSSRK_ENTRIES_PER_REG)) return -EINVAL; if (key != NULL) for (i = 0; i < FM10K_RSSRK_SIZE; ++i) key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i)); mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0)); hf = 0; hf |= (mrqc & FM10K_MRQC_IPV4) ? ETH_RSS_IPV4 : 0; hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6 : 0; hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6_EX : 0; hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP : 0; hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP : 0; hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX : 0; hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP : 0; hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP : 0; hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX : 0; rss_conf->rss_hf = hf; return 0; } static void fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t int_map = FM10K_INT_MAP_IMMEDIATE; /* Bind all local non-queue interrupt to vector 0 */ int_map |= FM10K_MISC_VEC_ID; FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map); FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map); FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map); FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map); FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map); FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map); /* Enable misc causes */ FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) | FM10K_EIMR_ENABLE(THI_FAULT) | FM10K_EIMR_ENABLE(FUM_FAULT) | FM10K_EIMR_ENABLE(MAILBOX) | FM10K_EIMR_ENABLE(SWITCHREADY) | FM10K_EIMR_ENABLE(SWITCHNOTREADY) | FM10K_EIMR_ENABLE(SRAMERROR) | FM10K_EIMR_ENABLE(VFLR)); /* Enable ITR 0 */ FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); FM10K_WRITE_FLUSH(hw); } static void fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t int_map = FM10K_INT_MAP_DISABLE; int_map |= FM10K_MISC_VEC_ID; FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map); FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map); FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map); FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map); FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map); FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map); /* Disable misc causes */ FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) | FM10K_EIMR_DISABLE(THI_FAULT) | FM10K_EIMR_DISABLE(FUM_FAULT) | FM10K_EIMR_DISABLE(MAILBOX) | FM10K_EIMR_DISABLE(SWITCHREADY) | FM10K_EIMR_DISABLE(SWITCHNOTREADY) | FM10K_EIMR_DISABLE(SRAMERROR) | FM10K_EIMR_DISABLE(VFLR)); /* Disable ITR 0 */ FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET); FM10K_WRITE_FLUSH(hw); } static void fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t int_map = FM10K_INT_MAP_IMMEDIATE; /* Bind all local non-queue interrupt to vector 0 */ int_map |= FM10K_MISC_VEC_ID; /* Only INT 0 available, other 15 are reserved. */ FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map); /* Enable ITR 0 */ FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); FM10K_WRITE_FLUSH(hw); } static void fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t int_map = FM10K_INT_MAP_DISABLE; int_map |= FM10K_MISC_VEC_ID; /* Only INT 0 available, other 15 are reserved. */ FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map); /* Disable ITR 0 */ FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET); FM10K_WRITE_FLUSH(hw); } static int fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); /* Enable ITR */ if (hw->mac.type == fm10k_mac_pf) FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); else FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); rte_intr_enable(&pdev->intr_handle); return 0; } static int fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); /* Disable ITR */ if (hw->mac.type == fm10k_mac_pf) FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)), FM10K_ITR_MASK_SET); else FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)), FM10K_ITR_MASK_SET); return 0; } static int fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pdev->intr_handle; uint32_t intr_vector, vec; uint16_t queue_id; int result = 0; /* fm10k needs one separate interrupt for mailbox, * so only drivers which support multiple interrupt vectors * e.g. vfio-pci can work for fm10k interrupt mode */ if (!rte_intr_cap_multiple(intr_handle) || dev->data->dev_conf.intr_conf.rxq == 0) return result; intr_vector = dev->data->nb_rx_queues; /* disable interrupt first */ rte_intr_disable(intr_handle); if (hw->mac.type == fm10k_mac_pf) fm10k_dev_disable_intr_pf(dev); else fm10k_dev_disable_intr_vf(dev); if (rte_intr_efd_enable(intr_handle, intr_vector)) { PMD_INIT_LOG(ERR, "Failed to init event fd"); result = -EIO; } if (rte_intr_dp_is_en(intr_handle) && !result) { intr_handle->intr_vec = rte_zmalloc("intr_vec", dev->data->nb_rx_queues * sizeof(int), 0); if (intr_handle->intr_vec) { for (queue_id = 0, vec = FM10K_RX_VEC_START; queue_id < dev->data->nb_rx_queues; queue_id++) { intr_handle->intr_vec[queue_id] = vec; if (vec < intr_handle->nb_efd - 1 + FM10K_RX_VEC_START) vec++; } } else { PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" " intr_vec", dev->data->nb_rx_queues); rte_intr_efd_disable(intr_handle); result = -ENOMEM; } } if (hw->mac.type == fm10k_mac_pf) fm10k_dev_enable_intr_pf(dev); else fm10k_dev_enable_intr_vf(dev); rte_intr_enable(intr_handle); hw->mac.ops.update_int_moderator(hw); return result; } static int fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr) { struct fm10k_fault fault; int err; const char *estr = "Unknown error"; /* Process PCA fault */ if (eicr & FM10K_EICR_PCA_FAULT) { err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault); if (err) goto error; switch (fault.type) { case PCA_NO_FAULT: estr = "PCA_NO_FAULT"; break; case PCA_UNMAPPED_ADDR: estr = "PCA_UNMAPPED_ADDR"; break; case PCA_BAD_QACCESS_PF: estr = "PCA_BAD_QACCESS_PF"; break; case PCA_BAD_QACCESS_VF: estr = "PCA_BAD_QACCESS_VF"; break; case PCA_MALICIOUS_REQ: estr = "PCA_MALICIOUS_REQ"; break; case PCA_POISONED_TLP: estr = "PCA_POISONED_TLP"; break; case PCA_TLP_ABORT: estr = "PCA_TLP_ABORT"; break; default: goto error; } PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x", estr, fault.func ? "VF" : "PF", fault.func, fault.address, fault.specinfo); } /* Process THI fault */ if (eicr & FM10K_EICR_THI_FAULT) { err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault); if (err) goto error; switch (fault.type) { case THI_NO_FAULT: estr = "THI_NO_FAULT"; break; case THI_MAL_DIS_Q_FAULT: estr = "THI_MAL_DIS_Q_FAULT"; break; default: goto error; } PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x", estr, fault.func ? "VF" : "PF", fault.func, fault.address, fault.specinfo); } /* Process FUM fault */ if (eicr & FM10K_EICR_FUM_FAULT) { err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault); if (err) goto error; switch (fault.type) { case FUM_NO_FAULT: estr = "FUM_NO_FAULT"; break; case FUM_UNMAPPED_ADDR: estr = "FUM_UNMAPPED_ADDR"; break; case FUM_POISONED_TLP: estr = "FUM_POISONED_TLP"; break; case FUM_BAD_VF_QACCESS: estr = "FUM_BAD_VF_QACCESS"; break; case FUM_ADD_DECODE_ERR: estr = "FUM_ADD_DECODE_ERR"; break; case FUM_RO_ERROR: estr = "FUM_RO_ERROR"; break; case FUM_QPRC_CRC_ERROR: estr = "FUM_QPRC_CRC_ERROR"; break; case FUM_CSR_TIMEOUT: estr = "FUM_CSR_TIMEOUT"; break; case FUM_INVALID_TYPE: estr = "FUM_INVALID_TYPE"; break; case FUM_INVALID_LENGTH: estr = "FUM_INVALID_LENGTH"; break; case FUM_INVALID_BE: estr = "FUM_INVALID_BE"; break; case FUM_INVALID_ALIGN: estr = "FUM_INVALID_ALIGN"; break; default: goto error; } PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x", estr, fault.func ? "VF" : "PF", fault.func, fault.address, fault.specinfo); } return 0; error: PMD_INIT_LOG(ERR, "Failed to handle fault event."); return err; } /** * PF interrupt handler triggered by NIC for handling specific interrupt. * * @param handle * Pointer to interrupt handle. * @param param * The address of parameter (struct rte_eth_dev *) regsitered before. * * @return * void */ static void fm10k_dev_interrupt_handler_pf(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t cause, status; struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); int status_mbx; s32 err; if (hw->mac.type != fm10k_mac_pf) return; cause = FM10K_READ_REG(hw, FM10K_EICR); /* Handle PCI fault cases */ if (cause & FM10K_EICR_FAULT_MASK) { PMD_INIT_LOG(ERR, "INT: find fault!"); fm10k_dev_handle_fault(hw, cause); } /* Handle switch up/down */ if (cause & FM10K_EICR_SWITCHNOTREADY) PMD_INIT_LOG(ERR, "INT: Switch is not ready"); if (cause & FM10K_EICR_SWITCHREADY) { PMD_INIT_LOG(INFO, "INT: Switch is ready"); if (dev_info->sm_down == 1) { fm10k_mbx_lock(hw); /* For recreating logical ports */ status_mbx = hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, MAX_LPORT_NUM, 1); if (status_mbx == FM10K_SUCCESS) PMD_INIT_LOG(INFO, "INT: Recreated Logical port"); else PMD_INIT_LOG(INFO, "INT: Logical ports weren't recreated"); status_mbx = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map, FM10K_XCAST_MODE_NONE); if (status_mbx != FM10K_SUCCESS) PMD_INIT_LOG(ERR, "Failed to set XCAST mode"); fm10k_mbx_unlock(hw); /* first clear the internal SW recording structure */ if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)) fm10k_vlan_filter_set(dev, hw->mac.default_vid, false); fm10k_MAC_filter_set(dev, hw->mac.addr, false, MAIN_VSI_POOL_NUMBER); /* * Add default mac address and vlan for the logical * ports that have been created, leave to the * application to fully recover Rx filtering. */ fm10k_MAC_filter_set(dev, hw->mac.addr, true, MAIN_VSI_POOL_NUMBER); if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)) fm10k_vlan_filter_set(dev, hw->mac.default_vid, true); dev_info->sm_down = 0; _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, NULL); } } /* Handle mailbox message */ fm10k_mbx_lock(hw); err = hw->mbx.ops.process(hw, &hw->mbx); fm10k_mbx_unlock(hw); if (err == FM10K_ERR_RESET_REQUESTED) { PMD_INIT_LOG(INFO, "INT: Switch is down"); dev_info->sm_down = 1; _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, NULL); } /* Handle SRAM error */ if (cause & FM10K_EICR_SRAMERROR) { PMD_INIT_LOG(ERR, "INT: SRAM error on PEP"); status = FM10K_READ_REG(hw, FM10K_SRAM_IP); /* Write to clear pending bits */ FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status); /* Todo: print out error message after shared code updates */ } /* Clear these 3 events if having any */ cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX | FM10K_EICR_SWITCHREADY; if (cause) FM10K_WRITE_REG(hw, FM10K_EICR, cause); /* Re-enable interrupt from device side */ FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); /* Re-enable interrupt from host side */ rte_intr_enable(dev->intr_handle); } /** * VF interrupt handler triggered by NIC for handling specific interrupt. * * @param handle * Pointer to interrupt handle. * @param param * The address of parameter (struct rte_eth_dev *) regsitered before. * * @return * void */ static void fm10k_dev_interrupt_handler_vf(void *param) { struct rte_eth_dev *dev = (struct rte_eth_dev *)param; struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct fm10k_mbx_info *mbx = &hw->mbx; struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); const enum fm10k_mbx_state state = mbx->state; int status_mbx; if (hw->mac.type != fm10k_mac_vf) return; /* Handle mailbox message if lock is acquired */ fm10k_mbx_lock(hw); hw->mbx.ops.process(hw, &hw->mbx); fm10k_mbx_unlock(hw); if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) { PMD_INIT_LOG(INFO, "INT: Switch has gone down"); fm10k_mbx_lock(hw); hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, MAX_LPORT_NUM, 1); fm10k_mbx_unlock(hw); /* Setting reset flag */ dev_info->sm_down = 1; _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, NULL); } if (dev_info->sm_down == 1 && hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) { PMD_INIT_LOG(INFO, "INT: Switch has gone up"); fm10k_mbx_lock(hw); status_mbx = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map, FM10K_XCAST_MODE_NONE); if (status_mbx != FM10K_SUCCESS) PMD_INIT_LOG(ERR, "Failed to set XCAST mode"); fm10k_mbx_unlock(hw); /* first clear the internal SW recording structure */ fm10k_vlan_filter_set(dev, hw->mac.default_vid, false); fm10k_MAC_filter_set(dev, hw->mac.addr, false, MAIN_VSI_POOL_NUMBER); /* * Add default mac address and vlan for the logical ports that * have been created, leave to the application to fully recover * Rx filtering. */ fm10k_MAC_filter_set(dev, hw->mac.addr, true, MAIN_VSI_POOL_NUMBER); fm10k_vlan_filter_set(dev, hw->mac.default_vid, true); dev_info->sm_down = 0; _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, NULL); } /* Re-enable interrupt from device side */ FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); /* Re-enable interrupt from host side */ rte_intr_enable(dev->intr_handle); } /* Mailbox message handler in VF */ static const struct fm10k_msg_data fm10k_msgdata_vf[] = { FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test), FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf), FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf), FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error), }; static int fm10k_setup_mbx_service(struct fm10k_hw *hw) { int err = 0; /* Initialize mailbox lock */ fm10k_mbx_initlock(hw); /* Replace default message handler with new ones */ if (hw->mac.type == fm10k_mac_vf) err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf); if (err) { PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d", err); return err; } /* Connect to SM for PF device or PF for VF device */ return hw->mbx.ops.connect(hw, &hw->mbx); } static void fm10k_close_mbx_service(struct fm10k_hw *hw) { /* Disconnect from SM for PF device or PF for VF device */ hw->mbx.ops.disconnect(hw, &hw->mbx); } static const struct eth_dev_ops fm10k_eth_dev_ops = { .dev_configure = fm10k_dev_configure, .dev_start = fm10k_dev_start, .dev_stop = fm10k_dev_stop, .dev_close = fm10k_dev_close, .promiscuous_enable = fm10k_dev_promiscuous_enable, .promiscuous_disable = fm10k_dev_promiscuous_disable, .allmulticast_enable = fm10k_dev_allmulticast_enable, .allmulticast_disable = fm10k_dev_allmulticast_disable, .stats_get = fm10k_stats_get, .xstats_get = fm10k_xstats_get, .xstats_get_names = fm10k_xstats_get_names, .stats_reset = fm10k_stats_reset, .xstats_reset = fm10k_stats_reset, .link_update = fm10k_link_update, .dev_infos_get = fm10k_dev_infos_get, .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get, .vlan_filter_set = fm10k_vlan_filter_set, .vlan_offload_set = fm10k_vlan_offload_set, .mac_addr_add = fm10k_macaddr_add, .mac_addr_remove = fm10k_macaddr_remove, .rx_queue_start = fm10k_dev_rx_queue_start, .rx_queue_stop = fm10k_dev_rx_queue_stop, .tx_queue_start = fm10k_dev_tx_queue_start, .tx_queue_stop = fm10k_dev_tx_queue_stop, .rx_queue_setup = fm10k_rx_queue_setup, .rx_queue_release = fm10k_rx_queue_release, .tx_queue_setup = fm10k_tx_queue_setup, .tx_queue_release = fm10k_tx_queue_release, .rx_descriptor_done = fm10k_dev_rx_descriptor_done, .rx_queue_intr_enable = fm10k_dev_rx_queue_intr_enable, .rx_queue_intr_disable = fm10k_dev_rx_queue_intr_disable, .reta_update = fm10k_reta_update, .reta_query = fm10k_reta_query, .rss_hash_update = fm10k_rss_hash_update, .rss_hash_conf_get = fm10k_rss_hash_conf_get, }; static int ftag_check_handler(__rte_unused const char *key, const char *value, __rte_unused void *opaque) { if (strcmp(value, "1")) return -1; return 0; } static int fm10k_check_ftag(struct rte_devargs *devargs) { struct rte_kvargs *kvlist; const char *ftag_key = "enable_ftag"; if (devargs == NULL) return 0; kvlist = rte_kvargs_parse(devargs->args, NULL); if (kvlist == NULL) return 0; if (!rte_kvargs_count(kvlist, ftag_key)) { rte_kvargs_free(kvlist); return 0; } /* FTAG is enabled when there's key-value pair: enable_ftag=1 */ if (rte_kvargs_process(kvlist, ftag_key, ftag_check_handler, NULL) < 0) { rte_kvargs_free(kvlist); return 0; } rte_kvargs_free(kvlist); return 1; } static uint16_t fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { uint16_t nb_tx = 0; struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue; while (nb_pkts) { uint16_t ret, num; num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh); ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], num); nb_tx += ret; nb_pkts -= ret; if (ret < num) break; } return nb_tx; } static void __attribute__((cold)) fm10k_set_tx_function(struct rte_eth_dev *dev) { struct fm10k_tx_queue *txq; int i; int use_sse = 1; uint16_t tx_ftag_en = 0; if (rte_eal_process_type() != RTE_PROC_PRIMARY) { /* primary process has set the ftag flag and txq_flags */ txq = dev->data->tx_queues[0]; if (fm10k_tx_vec_condition_check(txq)) { dev->tx_pkt_burst = fm10k_xmit_pkts; dev->tx_pkt_prepare = fm10k_prep_pkts; PMD_INIT_LOG(DEBUG, "Use regular Tx func"); } else { PMD_INIT_LOG(DEBUG, "Use vector Tx func"); dev->tx_pkt_burst = fm10k_xmit_pkts_vec; dev->tx_pkt_prepare = NULL; } return; } if (fm10k_check_ftag(dev->device->devargs)) tx_ftag_en = 1; for (i = 0; i < dev->data->nb_tx_queues; i++) { txq = dev->data->tx_queues[i]; txq->tx_ftag_en = tx_ftag_en; /* Check if Vector Tx is satisfied */ if (fm10k_tx_vec_condition_check(txq)) use_sse = 0; } if (use_sse) { PMD_INIT_LOG(DEBUG, "Use vector Tx func"); for (i = 0; i < dev->data->nb_tx_queues; i++) { txq = dev->data->tx_queues[i]; fm10k_txq_vec_setup(txq); } dev->tx_pkt_burst = fm10k_xmit_pkts_vec; dev->tx_pkt_prepare = NULL; } else { dev->tx_pkt_burst = fm10k_xmit_pkts; dev->tx_pkt_prepare = fm10k_prep_pkts; PMD_INIT_LOG(DEBUG, "Use regular Tx func"); } } static void __attribute__((cold)) fm10k_set_rx_function(struct rte_eth_dev *dev) { struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); uint16_t i, rx_using_sse; uint16_t rx_ftag_en = 0; if (fm10k_check_ftag(dev->device->devargs)) rx_ftag_en = 1; /* In order to allow Vector Rx there are a few configuration * conditions to be met. */ if (!fm10k_rx_vec_condition_check(dev) && dev_info->rx_vec_allowed && !rx_ftag_en) { if (dev->data->scattered_rx) dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec; else dev->rx_pkt_burst = fm10k_recv_pkts_vec; } else if (dev->data->scattered_rx) dev->rx_pkt_burst = fm10k_recv_scattered_pkts; else dev->rx_pkt_burst = fm10k_recv_pkts; rx_using_sse = (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec || dev->rx_pkt_burst == fm10k_recv_pkts_vec); if (rx_using_sse) PMD_INIT_LOG(DEBUG, "Use vector Rx func"); else PMD_INIT_LOG(DEBUG, "Use regular Rx func"); if (rte_eal_process_type() != RTE_PROC_PRIMARY) return; for (i = 0; i < dev->data->nb_rx_queues; i++) { struct fm10k_rx_queue *rxq = dev->data->rx_queues[i]; rxq->rx_using_sse = rx_using_sse; rxq->rx_ftag_en = rx_ftag_en; } } static void fm10k_params_init(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but * there is no way to get link status without reading BAR4. Until this * works, assume we have maximum bandwidth. * @todo - fix bus info */ hw->bus_caps.speed = fm10k_bus_speed_8000; hw->bus_caps.width = fm10k_bus_width_pcie_x8; hw->bus_caps.payload = fm10k_bus_payload_512; hw->bus.speed = fm10k_bus_speed_8000; hw->bus.width = fm10k_bus_width_pcie_x8; hw->bus.payload = fm10k_bus_payload_256; info->rx_vec_allowed = true; } static int eth_fm10k_dev_init(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pdev->intr_handle; int diag, i; struct fm10k_macvlan_filter_info *macvlan; PMD_INIT_FUNC_TRACE(); dev->dev_ops = &fm10k_eth_dev_ops; dev->rx_pkt_burst = &fm10k_recv_pkts; dev->tx_pkt_burst = &fm10k_xmit_pkts; dev->tx_pkt_prepare = &fm10k_prep_pkts; /* * Primary process does the whole initialization, for secondary * processes, we just select the same Rx and Tx function as primary. */ if (rte_eal_process_type() != RTE_PROC_PRIMARY) { fm10k_set_rx_function(dev); fm10k_set_tx_function(dev); return 0; } rte_eth_copy_pci_info(dev, pdev); macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); memset(macvlan, 0, sizeof(*macvlan)); /* Vendor and Device ID need to be set before init of shared code */ memset(hw, 0, sizeof(*hw)); hw->device_id = pdev->id.device_id; hw->vendor_id = pdev->id.vendor_id; hw->subsystem_device_id = pdev->id.subsystem_device_id; hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id; hw->revision_id = 0; hw->hw_addr = (void *)pdev->mem_resource[0].addr; if (hw->hw_addr == NULL) { PMD_INIT_LOG(ERR, "Bad mem resource." " Try to blacklist unused devices."); return -EIO; } /* Store fm10k_adapter pointer */ hw->back = dev->data->dev_private; /* Initialize the shared code */ diag = fm10k_init_shared_code(hw); if (diag != FM10K_SUCCESS) { PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag); return -EIO; } /* Initialize parameters */ fm10k_params_init(dev); /* Initialize the hw */ diag = fm10k_init_hw(hw); if (diag != FM10K_SUCCESS) { PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag); return -EIO; } /* Initialize MAC address(es) */ dev->data->mac_addrs = rte_zmalloc("fm10k", ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0); if (dev->data->mac_addrs == NULL) { PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses"); return -ENOMEM; } diag = fm10k_read_mac_addr(hw); ether_addr_copy((const struct ether_addr *)hw->mac.addr, &dev->data->mac_addrs[0]); if (diag != FM10K_SUCCESS || !is_valid_assigned_ether_addr(dev->data->mac_addrs)) { /* Generate a random addr */ eth_random_addr(hw->mac.addr); memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN); ether_addr_copy((const struct ether_addr *)hw->mac.addr, &dev->data->mac_addrs[0]); } /* Reset the hw statistics */ fm10k_stats_reset(dev); /* Reset the hw */ diag = fm10k_reset_hw(hw); if (diag != FM10K_SUCCESS) { PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag); return -EIO; } /* Setup mailbox service */ diag = fm10k_setup_mbx_service(hw); if (diag != FM10K_SUCCESS) { PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag); return -EIO; } /*PF/VF has different interrupt handling mechanism */ if (hw->mac.type == fm10k_mac_pf) { /* register callback func to eal lib */ rte_intr_callback_register(intr_handle, fm10k_dev_interrupt_handler_pf, (void *)dev); /* enable MISC interrupt */ fm10k_dev_enable_intr_pf(dev); } else { /* VF */ rte_intr_callback_register(intr_handle, fm10k_dev_interrupt_handler_vf, (void *)dev); fm10k_dev_enable_intr_vf(dev); } /* Enable intr after callback registered */ rte_intr_enable(intr_handle); hw->mac.ops.update_int_moderator(hw); /* Make sure Switch Manager is ready before going forward. */ if (hw->mac.type == fm10k_mac_pf) { int switch_ready = 0; for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) { fm10k_mbx_lock(hw); hw->mac.ops.get_host_state(hw, &switch_ready); fm10k_mbx_unlock(hw); if (switch_ready) break; /* Delay some time to acquire async LPORT_MAP info. */ rte_delay_us(WAIT_SWITCH_MSG_US); } if (switch_ready == 0) { PMD_INIT_LOG(ERR, "switch is not ready"); return -1; } } /* * Below function will trigger operations on mailbox, acquire lock to * avoid race condition from interrupt handler. Operations on mailbox * FIFO will trigger interrupt to PF/SM, in which interrupt handler * will handle and generate an interrupt to our side. Then, FIFO in * mailbox will be touched. */ fm10k_mbx_lock(hw); /* Enable port first */ hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, MAX_LPORT_NUM, 1); /* Set unicast mode by default. App can change to other mode in other * API func. */ hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map, FM10K_XCAST_MODE_NONE); fm10k_mbx_unlock(hw); /* Make sure default VID is ready before going forward. */ if (hw->mac.type == fm10k_mac_pf) { for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) { if (hw->mac.default_vid) break; /* Delay some time to acquire async port VLAN info. */ rte_delay_us(WAIT_SWITCH_MSG_US); } if (!hw->mac.default_vid) { PMD_INIT_LOG(ERR, "default VID is not ready"); return -1; } } /* Add default mac address */ fm10k_MAC_filter_set(dev, hw->mac.addr, true, MAIN_VSI_POOL_NUMBER); return 0; } static int eth_fm10k_dev_uninit(struct rte_eth_dev *dev) { struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pdev->intr_handle; PMD_INIT_FUNC_TRACE(); /* only uninitialize in the primary process */ if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; /* safe to close dev here */ fm10k_dev_close(dev); dev->dev_ops = NULL; dev->rx_pkt_burst = NULL; dev->tx_pkt_burst = NULL; /* disable uio/vfio intr */ rte_intr_disable(intr_handle); /*PF/VF has different interrupt handling mechanism */ if (hw->mac.type == fm10k_mac_pf) { /* disable interrupt */ fm10k_dev_disable_intr_pf(dev); /* unregister callback func to eal lib */ rte_intr_callback_unregister(intr_handle, fm10k_dev_interrupt_handler_pf, (void *)dev); } else { /* disable interrupt */ fm10k_dev_disable_intr_vf(dev); rte_intr_callback_unregister(intr_handle, fm10k_dev_interrupt_handler_vf, (void *)dev); } /* free mac memory */ if (dev->data->mac_addrs) { rte_free(dev->data->mac_addrs); dev->data->mac_addrs = NULL; } memset(hw, 0, sizeof(*hw)); return 0; } static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_device *pci_dev) { return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct fm10k_adapter), eth_fm10k_dev_init); } static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev) { return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit); } /* * The set of PCI devices this driver supports. This driver will enable both PF * and SRIOV-VF devices. */ static const struct rte_pci_id pci_id_fm10k_map[] = { { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) }, { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) }, { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) }, { .vendor_id = 0, /* sentinel */ }, }; static struct rte_pci_driver rte_pmd_fm10k = { .id_table = pci_id_fm10k_map, .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_IOVA_AS_VA, .probe = eth_fm10k_pci_probe, .remove = eth_fm10k_pci_remove, }; RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k); RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map); RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");