diff options
Diffstat (limited to 'drivers/net')
65 files changed, 1199 insertions, 1055 deletions
diff --git a/drivers/net/bnx2x/bnx2x.c b/drivers/net/bnx2x/bnx2x.c index a49a07fb..8970334e 100644 --- a/drivers/net/bnx2x/bnx2x.c +++ b/drivers/net/bnx2x/bnx2x.c @@ -178,7 +178,7 @@ bnx2x_dma_alloc(struct bnx2x_softc *sc, size_t size, struct bnx2x_dma *dma, /* Caller must take care that strlen(mz_name) < RTE_MEMZONE_NAMESIZE */ z = rte_memzone_reserve_aligned(mz_name, (uint64_t) (size), - rte_lcore_to_socket_id(rte_lcore_id()), + SOCKET_ID_ANY, 0, align); if (z == NULL) { PMD_DRV_LOG(ERR, "DMA alloc failed for %s", msg); @@ -9556,8 +9556,8 @@ static void bnx2x_init_rte(struct bnx2x_softc *sc) sc->max_rx_queues = min(BNX2X_VF_MAX_QUEUES_PER_VF, sc->igu_sb_cnt); } else { - sc->max_tx_queues = 128; - sc->max_rx_queues = 128; + sc->max_rx_queues = BNX2X_MAX_RSS_COUNT(sc); + sc->max_tx_queues = sc->max_rx_queues; } } diff --git a/drivers/net/bnx2x/elink.c b/drivers/net/bnx2x/elink.c index 149cc975..d9a72f0a 100644 --- a/drivers/net/bnx2x/elink.c +++ b/drivers/net/bnx2x/elink.c @@ -6645,7 +6645,7 @@ static elink_status_t elink_8073_8727_external_rom_boot(struct bnx2x_softc *sc, uint8_t port) { uint32_t count = 0; - uint16_t fw_ver1, fw_msgout; + uint16_t fw_ver1 = 0, fw_msgout; elink_status_t rc = ELINK_STATUS_OK; /* Boot port from external ROM */ diff --git a/drivers/net/bnxt/bnxt.h b/drivers/net/bnxt/bnxt.h index df1f7718..0e21aceb 100644 --- a/drivers/net/bnxt/bnxt.h +++ b/drivers/net/bnxt/bnxt.h @@ -171,6 +171,7 @@ struct bnxt { struct bnxt_pf_info pf; struct bnxt_vf_info vf; + uint8_t dev_stopped; }; #endif diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c index 3795facd..7052ecf3 100644 --- a/drivers/net/bnxt/bnxt_ethdev.c +++ b/drivers/net/bnxt/bnxt_ethdev.c @@ -384,6 +384,7 @@ static int bnxt_dev_start_op(struct rte_eth_dev *eth_dev) struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; int rc; + bp->dev_stopped = 0; rc = bnxt_hwrm_func_reset(bp); if (rc) { RTE_LOG(ERR, PMD, "hwrm chip reset failure rc: %x\n", rc); @@ -427,16 +428,6 @@ static int bnxt_dev_set_link_down_op(struct rte_eth_dev *eth_dev) return 0; } -static void bnxt_dev_close_op(struct rte_eth_dev *eth_dev) -{ - struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; - - bnxt_free_tx_mbufs(bp); - bnxt_free_rx_mbufs(bp); - bnxt_free_mem(bp); - rte_free(eth_dev->data->mac_addrs); -} - /* Unload the driver, release resources */ static void bnxt_dev_stop_op(struct rte_eth_dev *eth_dev) { @@ -449,6 +440,19 @@ static void bnxt_dev_stop_op(struct rte_eth_dev *eth_dev) bnxt_shutdown_nic(bp); } +static void bnxt_dev_close_op(struct rte_eth_dev *eth_dev) +{ + struct bnxt *bp = (struct bnxt *)eth_dev->data->dev_private; + + if (bp->dev_stopped == 0) + bnxt_dev_stop_op(eth_dev); + + bnxt_free_tx_mbufs(bp); + bnxt_free_rx_mbufs(bp); + bnxt_free_mem(bp); + rte_free(eth_dev->data->mac_addrs); +} + static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev, uint32_t index) { @@ -463,7 +467,7 @@ static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev, * remove the corresponding MAC addr filter */ for (i = 0; i < MAX_FF_POOLS; i++) { - if (!(pool_mask & (1 << i))) + if (!(pool_mask & (1ULL << i))) continue; STAILQ_FOREACH(vnic, &bp->ff_pool[i], next) { @@ -1021,6 +1025,8 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev) eth_dev->pci_dev->mem_resource[0].phys_addr, eth_dev->pci_dev->mem_resource[0].addr); + bp->dev_stopped = 0; + return 0; error_free: @@ -1040,6 +1046,8 @@ bnxt_dev_uninit(struct rte_eth_dev *eth_dev) { rte_free(bp->grp_info); rc = bnxt_hwrm_func_driver_unregister(bp, 0); bnxt_free_hwrm_resources(bp); + if (bp->dev_stopped == 0) + bnxt_dev_close_op(eth_dev); return rc; } diff --git a/drivers/net/bnxt/bnxt_vnic.c b/drivers/net/bnxt/bnxt_vnic.c index c04c4c74..1b5f54c4 100644 --- a/drivers/net/bnxt/bnxt_vnic.c +++ b/drivers/net/bnxt/bnxt_vnic.c @@ -175,7 +175,7 @@ int bnxt_alloc_vnic_attributes(struct bnxt *bp) struct rte_pci_device *pdev = bp->pdev; const struct rte_memzone *mz; char mz_name[RTE_MEMZONE_NAMESIZE]; - int entry_length = RTE_CACHE_LINE_ROUNDUP( + uint16_t entry_length = RTE_CACHE_LINE_ROUNDUP( HW_HASH_INDEX_SIZE * sizeof(*vnic->rss_table) + HW_HASH_KEY_SIZE); uint16_t max_vnics; diff --git a/drivers/net/bonding/rte_eth_bond_api.c b/drivers/net/bonding/rte_eth_bond_api.c index 203ebe9e..3c169730 100644 --- a/drivers/net/bonding/rte_eth_bond_api.c +++ b/drivers/net/bonding/rte_eth_bond_api.c @@ -373,21 +373,6 @@ __eth_bond_slave_add_lock_free(uint8_t bonded_port_id, uint8_t slave_port_id) internals->candidate_max_rx_pktlen = dev_info.max_rx_pktlen; } else { - /* Check slave link properties are supported if props are set, - * all slaves must be the same */ - if (internals->link_props_set) { - if (link_properties_valid(&(bonded_eth_dev->data->dev_link), - &(slave_eth_dev->data->dev_link))) { - slave_eth_dev->data->dev_flags &= (~RTE_ETH_DEV_BONDED_SLAVE); - RTE_BOND_LOG(ERR, - "Slave port %d link speed/duplex not supported", - slave_port_id); - return -1; - } - } else { - link_properties_set(bonded_eth_dev, - &(slave_eth_dev->data->dev_link)); - } internals->rx_offload_capa &= dev_info.rx_offload_capa; internals->tx_offload_capa &= dev_info.tx_offload_capa; internals->flow_type_rss_offloads &= dev_info.flow_type_rss_offloads; diff --git a/drivers/net/bonding/rte_eth_bond_pmd.c b/drivers/net/bonding/rte_eth_bond_pmd.c index b20a2729..25fe00a6 100644 --- a/drivers/net/bonding/rte_eth_bond_pmd.c +++ b/drivers/net/bonding/rte_eth_bond_pmd.c @@ -1985,6 +1985,16 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, /* Inherit eth dev link properties from first active slave */ link_properties_set(bonded_eth_dev, &(slave_eth_dev->data->dev_link)); + } else { + if (link_properties_valid( + &bonded_eth_dev->data->dev_link, &link) != 0) { + slave_eth_dev->data->dev_flags &= + (~RTE_ETH_DEV_BONDED_SLAVE); + RTE_LOG(ERR, PMD, + "port %u invalid speed/duplex\n", + port_id); + return; + } } activate_slave(bonded_eth_dev, port_id); diff --git a/drivers/net/e1000/igb_rxtx.c b/drivers/net/e1000/igb_rxtx.c index 9d80a0b3..c5db727d 100644 --- a/drivers/net/e1000/igb_rxtx.c +++ b/drivers/net/e1000/igb_rxtx.c @@ -1528,7 +1528,7 @@ eth_igb_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) desc - rxq->nb_rx_desc]); } - return 0; + return desc; } int diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c index ac0803d6..30581610 100644 --- a/drivers/net/ena/ena_ethdev.c +++ b/drivers/net/ena/ena_ethdev.c @@ -342,11 +342,13 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev) host_info->os_type = ENA_ADMIN_OS_DPDK; host_info->kernel_ver = RTE_VERSION; - strncpy((char *)host_info->kernel_ver_str, rte_version(), - strlen(rte_version())); + snprintf((char *)host_info->kernel_ver_str, + sizeof(host_info->kernel_ver_str), + "%s", rte_version()); host_info->os_dist = RTE_VERSION; - strncpy((char *)host_info->os_dist_str, rte_version(), - strlen(rte_version())); + snprintf((char *)host_info->os_dist_str, + sizeof(host_info->os_dist_str), + "%s", rte_version()); host_info->driver_version = (DRV_MODULE_VER_MAJOR) | (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | diff --git a/drivers/net/enic/base/vnic_dev.c b/drivers/net/enic/base/vnic_dev.c index fc2e4cc3..4db21a4d 100644 --- a/drivers/net/enic/base/vnic_dev.c +++ b/drivers/net/enic/base/vnic_dev.c @@ -266,32 +266,35 @@ void vnic_dev_clear_desc_ring(struct vnic_dev_ring *ring) memset(ring->descs, 0, ring->size); } -int vnic_dev_alloc_desc_ring(__attribute__((unused)) struct vnic_dev *vdev, +int vnic_dev_alloc_desc_ring(struct vnic_dev *vdev, struct vnic_dev_ring *ring, - unsigned int desc_count, unsigned int desc_size, unsigned int socket_id, + unsigned int desc_count, unsigned int desc_size, + __attribute__((unused)) unsigned int socket_id, char *z_name) { - const struct rte_memzone *rz; + void *alloc_addr = NULL; + dma_addr_t alloc_pa = 0; vnic_dev_desc_ring_size(ring, desc_count, desc_size); - - rz = rte_memzone_reserve_aligned(z_name, - ring->size_unaligned, socket_id, - 0, ENIC_ALIGN); - if (!rz) { + alloc_addr = vdev->alloc_consistent(vdev->priv, + ring->size_unaligned, + &alloc_pa, (u8 *)z_name); + if (!alloc_addr) { pr_err("Failed to allocate ring (size=%d), aborting\n", (int)ring->size); return -ENOMEM; } - - ring->descs_unaligned = rz->addr; - if (!ring->descs_unaligned) { + ring->descs_unaligned = alloc_addr; + if (!alloc_pa) { pr_err("Failed to map allocated ring (size=%d), aborting\n", (int)ring->size); + vdev->free_consistent(vdev->priv, + ring->size_unaligned, + alloc_addr, + alloc_pa); return -ENOMEM; } - - ring->base_addr_unaligned = (dma_addr_t)rz->phys_addr; + ring->base_addr_unaligned = alloc_pa; ring->base_addr = VNIC_ALIGN(ring->base_addr_unaligned, ring->base_align); @@ -308,8 +311,13 @@ int vnic_dev_alloc_desc_ring(__attribute__((unused)) struct vnic_dev *vdev, void vnic_dev_free_desc_ring(__attribute__((unused)) struct vnic_dev *vdev, struct vnic_dev_ring *ring) { - if (ring->descs) + if (ring->descs) { + vdev->free_consistent(vdev->priv, + ring->size_unaligned, + ring->descs_unaligned, + ring->base_addr_unaligned); ring->descs = NULL; + } } static int _vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c index 0e700a12..10a40c1b 100644 --- a/drivers/net/enic/base/vnic_rq.c +++ b/drivers/net/enic/base/vnic_rq.c @@ -87,9 +87,11 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, iowrite32(0, &rq->ctrl->error_status); iowrite32(fetch_index, &rq->ctrl->fetch_index); iowrite32(posted_index, &rq->ctrl->posted_index); - if (rq->is_sop) - iowrite32(((rq->is_sop << 10) | rq->data_queue_idx), + if (rq->data_queue_enable) + iowrite32(((1 << 10) | rq->data_queue_idx), &rq->ctrl->data_ring); + else + iowrite32(0, &rq->ctrl->data_ring); } void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h index fd9e1704..2d9104c4 100644 --- a/drivers/net/enic/base/vnic_rq.h +++ b/drivers/net/enic/base/vnic_rq.h @@ -91,6 +91,7 @@ struct vnic_rq { uint16_t rxst_idx; uint32_t tot_pkts; uint16_t data_queue_idx; + uint8_t data_queue_enable; uint8_t is_sop; uint8_t in_use; struct rte_mbuf *pkt_first_seg; diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h index 4c16ef17..8f12b435 100644 --- a/drivers/net/enic/enic.h +++ b/drivers/net/enic/enic.h @@ -163,6 +163,12 @@ struct enic { }; +/* Get the CQ index from a Start of Packet(SOP) RQ index */ +static inline unsigned int enic_sop_rq_idx_to_cq_idx(unsigned int sop_idx) +{ + return sop_idx / 2; +} + static inline unsigned int enic_sop_rq(unsigned int rq) { return rq * 2; @@ -244,7 +250,7 @@ extern int enic_stop_rq(struct enic *enic, uint16_t queue_idx); extern void enic_free_rq(void *rxq); extern int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, unsigned int socket_id, struct rte_mempool *mp, - uint16_t nb_desc); + uint16_t nb_desc, uint16_t free_thresh); extern int enic_set_rss_nic_cfg(struct enic *enic); extern int enic_set_vnic_res(struct enic *enic); extern void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size); diff --git a/drivers/net/enic/enic_clsf.c b/drivers/net/enic/enic_clsf.c index e6f57bea..111b1942 100644 --- a/drivers/net/enic/enic_clsf.c +++ b/drivers/net/enic/enic_clsf.c @@ -120,7 +120,9 @@ int enic_fdir_add_fltr(struct enic *enic, struct rte_eth_fdir_filter *params) return -ENOTSUP; } - queue = params->action.rx_queue; + /* Get the enicpmd RQ from the DPDK Rx queue */ + queue = enic_sop_rq(params->action.rx_queue); + /* See if the key is already there in the table */ pos = rte_hash_del_key(enic->fdir.hash, params); switch (pos) { @@ -238,6 +240,7 @@ void enic_clsf_destroy(struct enic *enic) vnic_dev_classifier(enic->vdev, CLSF_DEL, &key->fltr_id, NULL); rte_free(key); + enic->fdir.nodes[index] = NULL; } } diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c index 47b07c92..04e7ba8d 100644 --- a/drivers/net/enic/enic_ethdev.c +++ b/drivers/net/enic/enic_ethdev.c @@ -284,16 +284,13 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, eth_dev->data->rx_queues[queue_idx] = (void *)&enic->rq[enic_sop_rq(queue_idx)]; - ret = enic_alloc_rq(enic, queue_idx, socket_id, mp, nb_desc); + ret = enic_alloc_rq(enic, queue_idx, socket_id, mp, nb_desc, + rx_conf->rx_free_thresh); if (ret) { dev_err(enic, "error in allocating rq\n"); return ret; } - enic->rq[queue_idx].rx_free_thresh = rx_conf->rx_free_thresh; - dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx, - enic->rq[queue_idx].rx_free_thresh); - return enicpmd_dev_setup_intr(enic); } @@ -443,8 +440,7 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev, device_info->max_rx_queues = enic->conf_rq_count / 2; device_info->max_tx_queues = enic->conf_wq_count; device_info->min_rx_bufsize = ENIC_MIN_MTU; - device_info->max_rx_pktlen = enic->rte_dev->data->mtu - + ETHER_HDR_LEN + 4; + device_info->max_rx_pktlen = enic->max_mtu + ETHER_HDR_LEN + 4; device_info->max_mac_addrs = 1; device_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c index b4ca3710..7549c12e 100644 --- a/drivers/net/enic/enic_main.c +++ b/drivers/net/enic/enic_main.c @@ -174,8 +174,7 @@ void enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats) * which can make ibytes be slightly higher than it should be. */ rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors); - rx_truncated = rx_packet_errors - stats->rx.rx_errors - - stats->rx.rx_no_bufs; + rx_truncated = rx_packet_errors - stats->rx.rx_errors; r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated; r_stats->opackets = stats->tx.tx_frames_ok; @@ -517,7 +516,7 @@ void enic_free_rq(void *rxq) if (rq_data->in_use) vnic_rq_free(rq_data); - vnic_cq_free(&enic->cq[rq_sop->index]); + vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]); } void enic_start_wq(struct enic *enic, uint16_t queue_idx) @@ -576,7 +575,7 @@ int enic_stop_rq(struct enic *enic, uint16_t queue_idx) int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, unsigned int socket_id, struct rte_mempool *mp, - uint16_t nb_desc) + uint16_t nb_desc, uint16_t free_thresh) { int rc; uint16_t sop_queue_idx = enic_sop_rq(queue_idx); @@ -596,6 +595,10 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, rq_data->socket_id = socket_id; rq_data->mp = mp; rq_sop->in_use = 1; + rq_sop->rx_free_thresh = free_thresh; + rq_data->rx_free_thresh = free_thresh; + dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx, + free_thresh); mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM); @@ -611,10 +614,13 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, } if (mbufs_per_pkt > 1) { - dev_info(enic, "Scatter rx mode in use\n"); + dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx); + rq_sop->data_queue_enable = 1; rq_data->in_use = 1; } else { - dev_info(enic, "Scatter rx mode not being used\n"); + dev_info(enic, "Rq %u Scatter rx mode not being used\n", + queue_idx); + rq_sop->data_queue_enable = 0; rq_data->in_use = 0; } @@ -1122,6 +1128,15 @@ static int enic_dev_init(struct enic *enic) return err; } + /* Get available resource counts */ + enic_get_res_counts(enic); + if (enic->conf_rq_count == 1) { + dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n"); + dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n"); + dev_err(enic, "See the ENIC PMD guide for more information.\n"); + return -EINVAL; + } + eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN, 0); if (!eth_dev->data->mac_addrs) { dev_err(enic, "mac addr storage alloc failed, aborting.\n"); @@ -1130,11 +1145,6 @@ static int enic_dev_init(struct enic *enic) ether_addr_copy((struct ether_addr *) enic->mac_addr, ð_dev->data->mac_addrs[0]); - - /* Get available resource counts - */ - enic_get_res_counts(enic); - vnic_dev_set_reset_flag(enic->vdev, 0); return 0; diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c index 50f0b287..ad596136 100644 --- a/drivers/net/enic/enic_rxtx.c +++ b/drivers/net/enic/enic_rxtx.c @@ -212,9 +212,12 @@ enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) /* checksum flags */ if (!enic_cq_rx_desc_csum_not_calc(cqrd) && (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { + uint32_t l4_flags = mbuf->packet_type & RTE_PTYPE_L4_MASK; + if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) pkt_flags |= PKT_RX_IP_CKSUM_BAD; - if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { + if (l4_flags == RTE_PTYPE_L4_UDP || + l4_flags == RTE_PTYPE_L4_TCP) { if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) pkt_flags |= PKT_RX_L4_CKSUM_BAD; } diff --git a/drivers/net/fm10k/fm10k_ethdev.c b/drivers/net/fm10k/fm10k_ethdev.c index 01f4a72c..35cbe086 100644 --- a/drivers/net/fm10k/fm10k_ethdev.c +++ b/drivers/net/fm10k/fm10k_ethdev.c @@ -52,6 +52,8 @@ #define MAX_QUERY_SWITCH_STATE_TIMES 10 /* Wait interval to get switch status */ #define WAIT_SWITCH_MSG_US 100000 +/* A period of quiescence for switch */ +#define FM10K_SWITCH_QUIESCE_US 10000 /* Number of chars per uint32 type */ #define CHARS_PER_UINT32 (sizeof(uint32_t)) #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1) @@ -693,8 +695,9 @@ fm10k_dev_tx_init(struct rte_eth_dev *dev) base_addr >> (CHAR_BIT * sizeof(uint32_t))); FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size); - /* assign default SGLORT for each TX queue */ - FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map); + /* assign default SGLORT for each TX queue by PF */ + if (hw->mac.type == fm10k_mac_pf) + FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map); } /* set up vector or scalar TX function as appropriate */ @@ -1233,6 +1236,9 @@ fm10k_dev_close(struct rte_eth_dev *dev) MAX_LPORT_NUM, false); fm10k_mbx_unlock(hw); + /* allow 10ms for device to quiesce */ + rte_delay_us(FM10K_SWITCH_QUIESCE_US); + /* Stop mailbox service first */ fm10k_close_mbx_service(hw); fm10k_dev_stop(dev); diff --git a/drivers/net/fm10k/fm10k_rxtx.c b/drivers/net/fm10k/fm10k_rxtx.c index 5b2d04bf..bf5888b0 100644 --- a/drivers/net/fm10k/fm10k_rxtx.c +++ b/drivers/net/fm10k/fm10k_rxtx.c @@ -96,6 +96,16 @@ rx_desc_to_ol_flags(struct rte_mbuf *m, const union fm10k_rx_desc *d) if (d->w.pkt_info & FM10K_RXD_RSSTYPE_MASK) m->ol_flags |= PKT_RX_RSS_HASH; + + if (unlikely((d->d.staterr & + (FM10K_RXD_STATUS_IPCS | FM10K_RXD_STATUS_IPE)) == + (FM10K_RXD_STATUS_IPCS | FM10K_RXD_STATUS_IPE))) + m->ol_flags |= PKT_RX_IP_CKSUM_BAD; + + if (unlikely((d->d.staterr & + (FM10K_RXD_STATUS_L4CS | FM10K_RXD_STATUS_L4E)) == + (FM10K_RXD_STATUS_L4CS | FM10K_RXD_STATUS_L4E))) + m->ol_flags |= PKT_RX_L4_CKSUM_BAD; } uint16_t diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c index 9ea747e1..c9a49e36 100644 --- a/drivers/net/fm10k/fm10k_rxtx_vec.c +++ b/drivers/net/fm10k/fm10k_rxtx_vec.c @@ -468,6 +468,7 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, /* Read desc statuses backwards to avoid race condition */ /* A.1 load 4 pkts desc */ descs0[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); + rte_compiler_barrier(); /* B.2 copy 2 mbuf point into rx_pkts */ _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1); @@ -476,8 +477,10 @@ fm10k_recv_raw_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, mbp2 = _mm_loadu_si128((__m128i *)&mbufp[pos+2]); descs0[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); + rte_compiler_barrier(); /* B.1 load 2 mbuf point */ descs0[1] = _mm_loadu_si128((__m128i *)(rxdp + 1)); + rte_compiler_barrier(); descs0[0] = _mm_loadu_si128((__m128i *)(rxdp)); /* B.2 copy 2 mbuf point into rx_pkts */ diff --git a/drivers/net/i40e/base/i40e_common.c b/drivers/net/i40e/base/i40e_common.c index 98ed4b68..4407f2d3 100644 --- a/drivers/net/i40e/base/i40e_common.c +++ b/drivers/net/i40e/base/i40e_common.c @@ -773,7 +773,7 @@ struct i40e_rx_ptype_decoded i40e_ptype_lookup[] = { /* Non Tunneled IPv6 */ I40E_PTT(88, IP, IPV6, FRG, NONE, NONE, NOF, NONE, PAY3), I40E_PTT(89, IP, IPV6, NOF, NONE, NONE, NOF, NONE, PAY3), - I40E_PTT(90, IP, IPV6, NOF, NONE, NONE, NOF, UDP, PAY3), + I40E_PTT(90, IP, IPV6, NOF, NONE, NONE, NOF, UDP, PAY4), I40E_PTT_UNUSED_ENTRY(91), I40E_PTT(92, IP, IPV6, NOF, NONE, NONE, NOF, TCP, PAY4), I40E_PTT(93, IP, IPV6, NOF, NONE, NONE, NOF, SCTP, PAY4), diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c index d0aeb703..13068cc4 100644 --- a/drivers/net/i40e/i40e_ethdev.c +++ b/drivers/net/i40e/i40e_ethdev.c @@ -108,7 +108,6 @@ I40E_PFINT_ICR0_ENA_GRST_MASK | \ I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | \ I40E_PFINT_ICR0_ENA_STORM_DETECT_MASK | \ - I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK | \ I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | \ I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK | \ I40E_PFINT_ICR0_ENA_VFLR_MASK | \ @@ -202,7 +201,7 @@ /* Source MAC address */ #define I40E_REG_INSET_L2_SMAC 0x1C00000000000000ULL /* Outer (S-Tag) VLAN tag in the outer L2 header */ -#define I40E_REG_INSET_L2_OUTER_VLAN 0x0200000000000000ULL +#define I40E_REG_INSET_L2_OUTER_VLAN 0x0000000004000000ULL /* Inner (C-Tag) or single VLAN tag in the outer L2 header */ #define I40E_REG_INSET_L2_INNER_VLAN 0x0080000000000000ULL /* Single VLAN tag in the inner L2 header */ @@ -211,6 +210,14 @@ #define I40E_REG_INSET_L3_SRC_IP4 0x0001800000000000ULL /* Destination IPv4 address */ #define I40E_REG_INSET_L3_DST_IP4 0x0000001800000000ULL +/* Source IPv4 address for X722 */ +#define I40E_X722_REG_INSET_L3_SRC_IP4 0x0006000000000000ULL +/* Destination IPv4 address for X722 */ +#define I40E_X722_REG_INSET_L3_DST_IP4 0x0000060000000000ULL +/* IPv4 Protocol for X722 */ +#define I40E_X722_REG_INSET_L3_IP4_PROTO 0x0010000000000000ULL +/* IPv4 Time to Live for X722 */ +#define I40E_X722_REG_INSET_L3_IP4_TTL 0x0010000000000000ULL /* IPv4 Type of Service (TOS) */ #define I40E_REG_INSET_L3_IP4_TOS 0x0040000000000000ULL /* IPv4 Protocol */ @@ -724,10 +731,6 @@ static struct rte_driver rte_i40e_driver = { PMD_REGISTER_DRIVER(rte_i40e_driver, i40e); DRIVER_REGISTER_PCI_TABLE(i40e, pci_id_i40e_map); -/* - * Initialize registers for flexible payload, which should be set by NVM. - * This should be removed from code once it is fixed in NVM. - */ #ifndef I40E_GLQF_ORT #define I40E_GLQF_ORT(_i) (0x00268900 + ((_i) * 4)) #endif @@ -735,8 +738,12 @@ DRIVER_REGISTER_PCI_TABLE(i40e, pci_id_i40e_map); #define I40E_GLQF_PIT(_i) (0x00268C80 + ((_i) * 4)) #endif -static inline void i40e_flex_payload_reg_init(struct i40e_hw *hw) +static inline void i40e_GLQF_reg_init(struct i40e_hw *hw) { + /* + * Initialize registers for flexible payload, which should be set by NVM. + * This should be removed from code once it is fixed in NVM. + */ I40E_WRITE_REG(hw, I40E_GLQF_ORT(18), 0x00000030); I40E_WRITE_REG(hw, I40E_GLQF_ORT(19), 0x00000030); I40E_WRITE_REG(hw, I40E_GLQF_ORT(26), 0x0000002B); @@ -747,10 +754,12 @@ static inline void i40e_flex_payload_reg_init(struct i40e_hw *hw) I40E_WRITE_REG(hw, I40E_GLQF_ORT(20), 0x00000031); I40E_WRITE_REG(hw, I40E_GLQF_ORT(23), 0x00000031); I40E_WRITE_REG(hw, I40E_GLQF_ORT(63), 0x0000002D); - - /* GLQF_PIT Registers */ I40E_WRITE_REG(hw, I40E_GLQF_PIT(16), 0x00007480); I40E_WRITE_REG(hw, I40E_GLQF_PIT(17), 0x00007440); + + /* Initialize registers for parsing packet type of QinQ */ + I40E_WRITE_REG(hw, I40E_GLQF_ORT(40), 0x00000029); + I40E_WRITE_REG(hw, I40E_GLQF_PIT(9), 0x00009420); } #define I40E_FLOW_CONTROL_ETHERTYPE 0x8808 @@ -932,6 +941,9 @@ config_floating_veb(struct rte_eth_dev *dev) } } +#define I40E_L2_TAGS_S_TAG_SHIFT 1 +#define I40E_L2_TAGS_S_TAG_MASK I40E_MASK(0x1, I40E_L2_TAGS_S_TAG_SHIFT) + static int eth_i40e_dev_init(struct rte_eth_dev *dev) { @@ -1002,11 +1014,12 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) } /* - * To work around the NVM issue,initialize registers - * for flexible payload by software. - * It should be removed once issues are fixed in NVM. + * To work around the NVM issue, initialize registers + * for flexible payload and packet type of QinQ by + * software. It should be removed once issues are fixed + * in NVM. */ - i40e_flex_payload_reg_init(hw); + i40e_GLQF_reg_init(hw); /* Initialize the input set for filters (hash and fd) to default value */ i40e_filter_input_set_init(pf); @@ -1120,6 +1133,15 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) /* Disable double vlan by default */ i40e_vsi_config_double_vlan(vsi, FALSE); + /* Disable S-TAG identification when floating_veb is disabled */ + if (!pf->floating_veb) { + ret = I40E_READ_REG(hw, I40E_PRT_L2TAGSEN); + if (ret & I40E_L2_TAGS_S_TAG_MASK) { + ret &= ~I40E_L2_TAGS_S_TAG_MASK; + I40E_WRITE_REG(hw, I40E_PRT_L2TAGSEN, ret); + } + } + if (!vsi->max_macaddrs) len = ETHER_ADDR_LEN; else @@ -1214,11 +1236,6 @@ eth_i40e_dev_uninit(struct rte_eth_dev *dev) dev->rx_pkt_burst = NULL; dev->tx_pkt_burst = NULL; - /* Disable LLDP */ - ret = i40e_aq_stop_lldp(hw, true, NULL); - if (ret != I40E_SUCCESS) /* Its failure can be ignored */ - PMD_INIT_LOG(INFO, "Failed to stop lldp"); - /* Clear PXE mode */ i40e_clear_pxe_mode(hw); @@ -1768,6 +1785,16 @@ i40e_dev_start(struct rte_eth_dev *dev) if (dev->data->dev_conf.intr_conf.lsc != 0) PMD_INIT_LOG(INFO, "lsc won't enable because of" " no intr multiplex\n"); + } else if (dev->data->dev_conf.intr_conf.lsc != 0) { + ret = i40e_aq_set_phy_int_mask(hw, + ~(I40E_AQ_EVENT_LINK_UPDOWN | + I40E_AQ_EVENT_MODULE_QUAL_FAIL | + I40E_AQ_EVENT_MEDIA_NA), NULL); + if (ret != I40E_SUCCESS) + PMD_DRV_LOG(WARNING, "Fail to set phy mask"); + + /* Call get_link_info aq commond to enable LSE */ + i40e_dev_link_update(dev, 0); } /* enable uio intr after callback register */ @@ -1984,6 +2011,7 @@ i40e_dev_link_update(struct rte_eth_dev *dev, struct rte_eth_link link, old; int status; unsigned rep_cnt = MAX_REPEAT_TIME; + bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false; memset(&link, 0, sizeof(link)); memset(&old, 0, sizeof(old)); @@ -1992,7 +2020,8 @@ i40e_dev_link_update(struct rte_eth_dev *dev, do { /* Get link status information from hardware */ - status = i40e_aq_get_link_info(hw, false, &link_status, NULL); + status = i40e_aq_get_link_info(hw, enable_lse, + &link_status, NULL); if (status != I40E_SUCCESS) { link.link_speed = ETH_SPEED_NUM_100M; link.link_duplex = ETH_LINK_FULL_DUPLEX; @@ -4097,11 +4126,13 @@ i40e_vsi_release(struct i40e_vsi *vsi) void *temp; int ret; struct i40e_mac_filter *f; - uint16_t user_param = vsi->user_param; + uint16_t user_param; if (!vsi) return I40E_SUCCESS; + user_param = vsi->user_param; + pf = I40E_VSI_TO_PF(vsi); hw = I40E_VSI_TO_HW(vsi); @@ -5407,6 +5438,24 @@ i40e_dev_handle_vfr_event(struct rte_eth_dev *dev) } static void +i40e_notify_all_vfs_link_status(struct rte_eth_dev *dev) +{ + struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); + struct i40e_virtchnl_pf_event event; + int i; + + event.event = I40E_VIRTCHNL_EVENT_LINK_CHANGE; + event.event_data.link_event.link_status = + dev->data->dev_link.link_status; + event.event_data.link_event.link_speed = + (enum i40e_aq_link_speed)dev->data->dev_link.link_speed; + + for (i = 0; i < pf->vf_num; i++) + i40e_pf_host_send_msg_to_vf(&pf->vfs[i], I40E_VIRTCHNL_OP_EVENT, + I40E_SUCCESS, (uint8_t *)&event, sizeof(event)); +} + +static void i40e_dev_handle_aq_msg(struct rte_eth_dev *dev) { struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); @@ -5442,6 +5491,14 @@ i40e_dev_handle_aq_msg(struct rte_eth_dev *dev) info.msg_buf, info.msg_len); break; + case i40e_aqc_opc_get_link_status: + ret = i40e_dev_link_update(dev, 0); + if (!ret) { + i40e_notify_all_vfs_link_status(dev); + _rte_eth_dev_callback_process(dev, + RTE_ETH_EVENT_INTR_LSC); + } + break; default: PMD_DRV_LOG(ERR, "Request %u is not supported yet", opcode); @@ -5451,57 +5508,6 @@ i40e_dev_handle_aq_msg(struct rte_eth_dev *dev) rte_free(info.msg_buf); } -/* - * Interrupt handler is registered as the alarm callback for handling LSC - * interrupt in a definite of time, in order to wait the NIC into a stable - * state. Currently it waits 1 sec in i40e for the link up interrupt, and - * no need for link down interrupt. - */ -static void -i40e_dev_interrupt_delayed_handler(void *param) -{ - struct rte_eth_dev *dev = (struct rte_eth_dev *)param; - struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - uint32_t icr0; - - /* read interrupt causes again */ - icr0 = I40E_READ_REG(hw, I40E_PFINT_ICR0); - -#ifdef RTE_LIBRTE_I40E_DEBUG_DRIVER - if (icr0 & I40E_PFINT_ICR0_ECC_ERR_MASK) - PMD_DRV_LOG(ERR, "ICR0: unrecoverable ECC error\n"); - if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) - PMD_DRV_LOG(ERR, "ICR0: malicious programming detected\n"); - if (icr0 & I40E_PFINT_ICR0_GRST_MASK) - PMD_DRV_LOG(INFO, "ICR0: global reset requested\n"); - if (icr0 & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) - PMD_DRV_LOG(INFO, "ICR0: PCI exception\n activated\n"); - if (icr0 & I40E_PFINT_ICR0_STORM_DETECT_MASK) - PMD_DRV_LOG(INFO, "ICR0: a change in the storm control " - "state\n"); - if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) - PMD_DRV_LOG(ERR, "ICR0: HMC error\n"); - if (icr0 & I40E_PFINT_ICR0_PE_CRITERR_MASK) - PMD_DRV_LOG(ERR, "ICR0: protocol engine critical error\n"); -#endif /* RTE_LIBRTE_I40E_DEBUG_DRIVER */ - - if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { - PMD_DRV_LOG(INFO, "INT:VF reset detected\n"); - i40e_dev_handle_vfr_event(dev); - } - if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { - PMD_DRV_LOG(INFO, "INT:ADMINQ event\n"); - i40e_dev_handle_aq_msg(dev); - } - - /* handle the link up interrupt in an alarm callback */ - i40e_dev_link_update(dev, 0); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); - - i40e_pf_enable_irq0(hw); - rte_intr_enable(&(dev->pci_dev->intr_handle)); -} - /** * Interrupt handler triggered by NIC for handling * specific interrupt. @@ -5558,31 +5564,6 @@ i40e_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle, PMD_DRV_LOG(INFO, "ICR0: adminq event"); i40e_dev_handle_aq_msg(dev); } - - /* Link Status Change interrupt */ - if (icr0 & I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK) { -#define I40E_US_PER_SECOND 1000000 - struct rte_eth_link link; - - PMD_DRV_LOG(INFO, "ICR0: link status changed\n"); - memset(&link, 0, sizeof(link)); - rte_i40e_dev_atomic_read_link_status(dev, &link); - i40e_dev_link_update(dev, 0); - - /* - * For link up interrupt, it needs to wait 1 second to let the - * hardware be a stable state. Otherwise several consecutive - * interrupts can be observed. - * For link down interrupt, no need to wait. - */ - if (!link.link_status && rte_eal_alarm_set(I40E_US_PER_SECOND, - i40e_dev_interrupt_delayed_handler, (void *)dev) >= 0) - return; - else - _rte_eth_dev_callback_process(dev, - RTE_ETH_EVENT_INTR_LSC); - } - done: /* Enable interrupt */ i40e_pf_enable_irq0(hw); @@ -7372,25 +7353,23 @@ i40e_parse_input_set(uint64_t *inset, * and vice versa */ static uint64_t -i40e_translate_input_set_reg(uint64_t input) +i40e_translate_input_set_reg(enum i40e_mac_type type, uint64_t input) { uint64_t val = 0; uint16_t i; - static const struct { + struct inset_map { uint64_t inset; uint64_t inset_reg; - } inset_map[] = { + }; + + static const struct inset_map inset_map_common[] = { {I40E_INSET_DMAC, I40E_REG_INSET_L2_DMAC}, {I40E_INSET_SMAC, I40E_REG_INSET_L2_SMAC}, {I40E_INSET_VLAN_OUTER, I40E_REG_INSET_L2_OUTER_VLAN}, {I40E_INSET_VLAN_INNER, I40E_REG_INSET_L2_INNER_VLAN}, {I40E_INSET_LAST_ETHER_TYPE, I40E_REG_INSET_LAST_ETHER_TYPE}, - {I40E_INSET_IPV4_SRC, I40E_REG_INSET_L3_SRC_IP4}, - {I40E_INSET_IPV4_DST, I40E_REG_INSET_L3_DST_IP4}, {I40E_INSET_IPV4_TOS, I40E_REG_INSET_L3_IP4_TOS}, - {I40E_INSET_IPV4_PROTO, I40E_REG_INSET_L3_IP4_PROTO}, - {I40E_INSET_IPV4_TTL, I40E_REG_INSET_L3_IP4_TTL}, {I40E_INSET_IPV6_SRC, I40E_REG_INSET_L3_SRC_IP6}, {I40E_INSET_IPV6_DST, I40E_REG_INSET_L3_DST_IP6}, {I40E_INSET_IPV6_TC, I40E_REG_INSET_L3_IP6_TC}, @@ -7419,13 +7398,40 @@ i40e_translate_input_set_reg(uint64_t input) {I40E_INSET_FLEX_PAYLOAD_W8, I40E_REG_INSET_FLEX_PAYLOAD_WORD8}, }; + /* some different registers map in x722*/ + static const struct inset_map inset_map_diff_x722[] = { + {I40E_INSET_IPV4_SRC, I40E_X722_REG_INSET_L3_SRC_IP4}, + {I40E_INSET_IPV4_DST, I40E_X722_REG_INSET_L3_DST_IP4}, + {I40E_INSET_IPV4_PROTO, I40E_X722_REG_INSET_L3_IP4_PROTO}, + {I40E_INSET_IPV4_TTL, I40E_X722_REG_INSET_L3_IP4_TTL}, + }; + + static const struct inset_map inset_map_diff_not_x722[] = { + {I40E_INSET_IPV4_SRC, I40E_REG_INSET_L3_SRC_IP4}, + {I40E_INSET_IPV4_DST, I40E_REG_INSET_L3_DST_IP4}, + {I40E_INSET_IPV4_PROTO, I40E_REG_INSET_L3_IP4_PROTO}, + {I40E_INSET_IPV4_TTL, I40E_REG_INSET_L3_IP4_TTL}, + }; + if (input == 0) return val; /* Translate input set to register aware inset */ - for (i = 0; i < RTE_DIM(inset_map); i++) { - if (input & inset_map[i].inset) - val |= inset_map[i].inset_reg; + if (type == I40E_MAC_X722) { + for (i = 0; i < RTE_DIM(inset_map_diff_x722); i++) { + if (input & inset_map_diff_x722[i].inset) + val |= inset_map_diff_x722[i].inset_reg; + } + } else { + for (i = 0; i < RTE_DIM(inset_map_diff_not_x722); i++) { + if (input & inset_map_diff_not_x722[i].inset) + val |= inset_map_diff_not_x722[i].inset_reg; + } + } + + for (i = 0; i < RTE_DIM(inset_map_common); i++) { + if (input & inset_map_common[i].inset) + val |= inset_map_common[i].inset_reg; } return val; @@ -7510,7 +7516,8 @@ i40e_filter_input_set_init(struct i40e_pf *pf) I40E_INSET_MASK_NUM_REG); if (num < 0) return; - inset_reg = i40e_translate_input_set_reg(input_set); + inset_reg = i40e_translate_input_set_reg(hw->mac.type, + input_set); i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0), (uint32_t)(inset_reg & UINT32_MAX)); @@ -7592,7 +7599,7 @@ i40e_hash_filter_inset_select(struct i40e_hw *hw, if (num < 0) return -EINVAL; - inset_reg |= i40e_translate_input_set_reg(input_set); + inset_reg |= i40e_translate_input_set_reg(hw->mac.type, input_set); i40e_check_write_reg(hw, I40E_GLQF_HASH_INSET(0, pctype), (uint32_t)(inset_reg & UINT32_MAX)); @@ -7668,7 +7675,7 @@ i40e_fdir_filter_inset_select(struct i40e_pf *pf, if (num < 0) return -EINVAL; - inset_reg |= i40e_translate_input_set_reg(input_set); + inset_reg |= i40e_translate_input_set_reg(hw->mac.type, input_set); i40e_check_write_reg(hw, I40E_PRTQF_FD_INSET(pctype, 0), (uint32_t)(inset_reg & UINT32_MAX)); @@ -9148,17 +9155,13 @@ i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb) * LLDP MIB change event. */ if (sw_dcb == TRUE) { - ret = i40e_aq_stop_lldp(hw, TRUE, NULL); - if (ret != I40E_SUCCESS) - PMD_INIT_LOG(DEBUG, "Failed to stop lldp"); - ret = i40e_init_dcb(hw); - /* if sw_dcb, lldp agent is stopped, the return from + /* If lldp agent is stopped, the return value from * i40e_init_dcb we expect is failure with I40E_AQ_RC_EPERM - * adminq status. + * adminq status. Otherwise, it should return success. */ - if (ret != I40E_SUCCESS && - hw->aq.asq_last_status == I40E_AQ_RC_EPERM) { + if ((ret == I40E_SUCCESS) || (ret != I40E_SUCCESS && + hw->aq.asq_last_status == I40E_AQ_RC_EPERM)) { memset(&hw->local_dcbx_config, 0, sizeof(struct i40e_dcbx_config)); /* set dcb default configuration */ @@ -9187,8 +9190,8 @@ i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb) return -ENOSYS; } } else { - PMD_INIT_LOG(ERR, "DCBX configuration failed, err = %d," - " aq_err = %d.", ret, + PMD_INIT_LOG(ERR, "DCB initialization in FW fails," + " err = %d, aq_err = %d.", ret, hw->aq.asq_last_status); return -ENOTSUP; } diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h index 92c8fad0..61dfa932 100644 --- a/drivers/net/i40e/i40e_ethdev.h +++ b/drivers/net/i40e/i40e_ethdev.h @@ -599,7 +599,9 @@ int i40e_hash_filter_inset_select(struct i40e_hw *hw, struct rte_eth_input_set_conf *conf); int i40e_fdir_filter_inset_select(struct i40e_pf *pf, struct rte_eth_input_set_conf *conf); - +int i40e_pf_host_send_msg_to_vf(struct i40e_pf_vf *vf, uint32_t opcode, + uint32_t retval, uint8_t *msg, + uint16_t msglen); void i40e_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_rxq_info *qinfo); void i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c index a616ae0b..ba63a7f1 100644 --- a/drivers/net/i40e/i40e_ethdev_vf.c +++ b/drivers/net/i40e/i40e_ethdev_vf.c @@ -126,8 +126,6 @@ static void i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev); static void i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev); static void i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev); static void i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev); -static int i40evf_get_link_status(struct rte_eth_dev *dev, - struct rte_eth_link *link); static int i40evf_init_vlan(struct rte_eth_dev *dev); static int i40evf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id); @@ -1084,31 +1082,6 @@ i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid) return err; } -static int -i40evf_get_link_status(struct rte_eth_dev *dev, struct rte_eth_link *link) -{ - struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); - int err; - struct vf_cmd_info args; - struct rte_eth_link *new_link; - - args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_GET_LINK_STAT; - args.in_args = NULL; - args.in_args_size = 0; - args.out_buffer = vf->aq_resp; - args.out_size = I40E_AQ_BUF_SZ; - err = i40evf_execute_vf_cmd(dev, &args); - if (err) { - PMD_DRV_LOG(ERR, "fail to execute command OP_GET_LINK_STAT"); - return err; - } - - new_link = (struct rte_eth_link *)args.out_buffer; - (void)rte_memcpy(link, new_link, sizeof(*link)); - - return 0; -} - static const struct rte_pci_id pci_id_i40evf_map[] = { { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF) }, { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF_HV) }, @@ -2166,35 +2139,33 @@ i40evf_dev_link_update(struct rte_eth_dev *dev, * DPDK pf host provide interfacet to acquire link status * while Linux driver does not */ - if (vf->version_major == I40E_DPDK_VERSION_MAJOR) - i40evf_get_link_status(dev, &new_link); - else { - /* Linux driver PF host */ - switch (vf->link_speed) { - case I40E_LINK_SPEED_100MB: - new_link.link_speed = ETH_SPEED_NUM_100M; - break; - case I40E_LINK_SPEED_1GB: - new_link.link_speed = ETH_SPEED_NUM_1G; - break; - case I40E_LINK_SPEED_10GB: - new_link.link_speed = ETH_SPEED_NUM_10G; - break; - case I40E_LINK_SPEED_20GB: - new_link.link_speed = ETH_SPEED_NUM_20G; - break; - case I40E_LINK_SPEED_40GB: - new_link.link_speed = ETH_SPEED_NUM_40G; - break; - default: - new_link.link_speed = ETH_SPEED_NUM_100M; - break; - } - /* full duplex only */ - new_link.link_duplex = ETH_LINK_FULL_DUPLEX; - new_link.link_status = vf->link_up ? ETH_LINK_UP : - ETH_LINK_DOWN; + + /* Linux driver PF host */ + switch (vf->link_speed) { + case I40E_LINK_SPEED_100MB: + new_link.link_speed = ETH_SPEED_NUM_100M; + break; + case I40E_LINK_SPEED_1GB: + new_link.link_speed = ETH_SPEED_NUM_1G; + break; + case I40E_LINK_SPEED_10GB: + new_link.link_speed = ETH_SPEED_NUM_10G; + break; + case I40E_LINK_SPEED_20GB: + new_link.link_speed = ETH_SPEED_NUM_20G; + break; + case I40E_LINK_SPEED_40GB: + new_link.link_speed = ETH_SPEED_NUM_40G; + break; + default: + new_link.link_speed = ETH_SPEED_NUM_100M; + break; } + /* full duplex only */ + new_link.link_duplex = ETH_LINK_FULL_DUPLEX; + new_link.link_status = vf->link_up ? ETH_LINK_UP : + ETH_LINK_DOWN; + i40evf_dev_atomic_write_link_status(dev, &new_link); return 0; diff --git a/drivers/net/i40e/i40e_pf.c b/drivers/net/i40e/i40e_pf.c index d5b2d450..4e2f6b63 100644 --- a/drivers/net/i40e/i40e_pf.c +++ b/drivers/net/i40e/i40e_pf.c @@ -250,7 +250,7 @@ i40e_pf_host_vf_reset(struct i40e_pf_vf *vf, bool do_hw_reset) return ret; } -static int +int i40e_pf_host_send_msg_to_vf(struct i40e_pf_vf *vf, uint32_t opcode, uint32_t retval, @@ -847,18 +847,6 @@ i40e_pf_host_process_cmd_get_stats(struct i40e_pf_vf *vf) return I40E_SUCCESS; } -static void -i40e_pf_host_process_cmd_get_link_status(struct i40e_pf_vf *vf) -{ - struct rte_eth_dev *dev = I40E_VSI_TO_ETH_DEV(vf->pf->main_vsi); - - /* Update link status first to acquire latest link change */ - i40e_dev_link_update(dev, 1); - i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_GET_LINK_STAT, - I40E_SUCCESS, (uint8_t *)&dev->data->dev_link, - sizeof(struct rte_eth_link)); -} - static int i40e_pf_host_process_cmd_cfg_vlan_offload( struct i40e_pf_vf *vf, @@ -909,6 +897,20 @@ send_msg: return ret; } +static void +i40e_notify_vf_link_status(struct rte_eth_dev *dev, struct i40e_pf_vf *vf) +{ + struct i40e_virtchnl_pf_event event; + + event.event = I40E_VIRTCHNL_EVENT_LINK_CHANGE; + event.event_data.link_event.link_status = + dev->data->dev_link.link_status; + event.event_data.link_event.link_speed = + (enum i40e_aq_link_speed)dev->data->dev_link.link_speed; + i40e_pf_host_send_msg_to_vf(vf, I40E_VIRTCHNL_OP_EVENT, + I40E_SUCCESS, (uint8_t *)&event, sizeof(event)); +} + void i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, uint16_t abs_vf_id, uint32_t opcode, @@ -964,6 +966,7 @@ i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, case I40E_VIRTCHNL_OP_ENABLE_QUEUES: PMD_DRV_LOG(INFO, "OP_ENABLE_QUEUES received"); i40e_pf_host_process_cmd_enable_queues(vf, msg, msglen); + i40e_notify_vf_link_status(dev, vf); break; case I40E_VIRTCHNL_OP_DISABLE_QUEUES: PMD_DRV_LOG(INFO, "OP_DISABLE_QUEUE received"); @@ -993,10 +996,6 @@ i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, PMD_DRV_LOG(INFO, "OP_GET_STATS received"); i40e_pf_host_process_cmd_get_stats(vf); break; - case I40E_VIRTCHNL_OP_GET_LINK_STAT: - PMD_DRV_LOG(INFO, "OP_GET_LINK_STAT received"); - i40e_pf_host_process_cmd_get_link_status(vf); - break; case I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD: PMD_DRV_LOG(INFO, "OP_CFG_VLAN_OFFLOAD received"); i40e_pf_host_process_cmd_cfg_vlan_offload(vf, msg, msglen); diff --git a/drivers/net/i40e/i40e_pf.h b/drivers/net/i40e/i40e_pf.h index 9c01829a..cddc45cf 100644 --- a/drivers/net/i40e/i40e_pf.h +++ b/drivers/net/i40e/i40e_pf.h @@ -59,9 +59,8 @@ enum i40e_virtchnl_ops_dpdk { * Keep some gap between Linux PF commands and * DPDK PF extended commands. */ - I40E_VIRTCHNL_OP_GET_LINK_STAT = I40E_VIRTCHNL_OP_VERSION + + I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD = I40E_VIRTCHNL_OP_VERSION + I40E_DPDK_OFFSET, - I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD, I40E_VIRTCHNL_OP_CFG_VLAN_PVID, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT, }; diff --git a/drivers/net/i40e/i40e_rxtx.c b/drivers/net/i40e/i40e_rxtx.c index 554d1679..0556a4d4 100644 --- a/drivers/net/i40e/i40e_rxtx.c +++ b/drivers/net/i40e/i40e_rxtx.c @@ -2948,11 +2948,15 @@ i40e_dev_clear_queues(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); for (i = 0; i < dev->data->nb_tx_queues; i++) { + if (!dev->data->tx_queues[i]) + continue; i40e_tx_queue_release_mbufs(dev->data->tx_queues[i]); i40e_reset_tx_queue(dev->data->tx_queues[i]); } for (i = 0; i < dev->data->nb_rx_queues; i++) { + if (!dev->data->rx_queues[i]) + continue; i40e_rx_queue_release_mbufs(dev->data->rx_queues[i]); i40e_reset_rx_queue(dev->data->rx_queues[i]); } @@ -2966,12 +2970,16 @@ i40e_dev_free_queues(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); for (i = 0; i < dev->data->nb_rx_queues; i++) { + if (!dev->data->rx_queues[i]) + continue; i40e_dev_rx_queue_release(dev->data->rx_queues[i]); dev->data->rx_queues[i] = NULL; } dev->data->nb_rx_queues = 0; for (i = 0; i < dev->data->nb_tx_queues; i++) { + if (!dev->data->tx_queues[i]) + continue; i40e_dev_tx_queue_release(dev->data->tx_queues[i]); dev->data->tx_queues[i] = NULL; } @@ -3154,7 +3162,7 @@ i40e_set_rx_function(struct rte_eth_dev *dev) struct i40e_rx_queue *rxq = dev->data->rx_queues[i]; - if (i40e_rxq_vec_setup(rxq)) { + if (rxq && i40e_rxq_vec_setup(rxq)) { ad->rx_vec_allowed = false; break; } @@ -3216,7 +3224,8 @@ i40e_set_rx_function(struct rte_eth_dev *dev) for (i = 0; i < dev->data->nb_rx_queues; i++) { struct i40e_rx_queue *rxq = dev->data->rx_queues[i]; - rxq->rx_using_sse = rx_using_sse; + if (rxq) + rxq->rx_using_sse = rx_using_sse; } } } @@ -3255,7 +3264,7 @@ i40e_set_tx_function(struct rte_eth_dev *dev) struct i40e_tx_queue *txq = dev->data->tx_queues[i]; - if (i40e_txq_vec_setup(txq)) { + if (txq && i40e_txq_vec_setup(txq)) { ad->tx_vec_allowed = false; break; } diff --git a/drivers/net/i40e/i40e_rxtx_vec.c b/drivers/net/i40e/i40e_rxtx_vec.c index 51fb282a..a9649d35 100644 --- a/drivers/net/i40e/i40e_rxtx_vec.c +++ b/drivers/net/i40e/i40e_rxtx_vec.c @@ -282,7 +282,7 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts, /* Read desc statuses backwards to avoid race condition */ /* A.1 load 4 pkts desc */ descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); - + rte_compiler_barrier(); /* B.2 copy 2 mbuf point into rx_pkts */ _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1); @@ -290,8 +290,10 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts, mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]); descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); + rte_compiler_barrier(); /* B.1 load 2 mbuf point */ descs[1] = _mm_loadu_si128((__m128i *)(rxdp + 1)); + rte_compiler_barrier(); descs[0] = _mm_loadu_si128((__m128i *)(rxdp)); /* B.2 copy 2 mbuf point into rx_pkts */ @@ -692,8 +694,20 @@ i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq) return; /* free all mbufs that are valid in the ring */ - for (i = rxq->rx_tail; i != rxq->rxrearm_start; i = (i + 1) & mask) - rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); + if (rxq->rxrearm_nb == 0) { + for (i = 0; i < rxq->nb_rx_desc; i++) { + if (rxq->sw_ring[i].mbuf != NULL) + rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); + } + } else { + for (i = rxq->rx_tail; + i != rxq->rxrearm_start; + i = (i + 1) & mask) { + if (rxq->sw_ring[i].mbuf != NULL) + rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); + } + } + rxq->rxrearm_nb = rxq->nb_rx_desc; /* set all entries to NULL */ diff --git a/drivers/net/ixgbe/base/ixgbe_common.c b/drivers/net/ixgbe/base/ixgbe_common.c index 811875a4..1c5cb913 100644 --- a/drivers/net/ixgbe/base/ixgbe_common.c +++ b/drivers/net/ixgbe/base/ixgbe_common.c @@ -3967,7 +3967,7 @@ s32 ixgbe_set_vlvf_generic(struct ixgbe_hw *hw, u32 vlan, u32 vind, * we run the risk of stray packets leaking into * the PF via the default pool */ - if (vfta_delta) + if (*vfta_delta) IXGBE_WRITE_REG(hw, IXGBE_VFTA(vlan / 32), vfta); /* disable VLVF and clear remaining bit from pool */ @@ -4318,43 +4318,31 @@ u8 ixgbe_calculate_checksum(u8 *buffer, u32 length) } /** - * ixgbe_host_interface_command - Issue command to manageability block + * ixgbe_hic_unlocked - Issue command to manageability block unlocked * @hw: pointer to the HW structure - * @buffer: contains the command to write and where the return status will - * be placed + * @buffer: command to write and where the return status will be placed * @length: length of buffer, must be multiple of 4 bytes * @timeout: time in ms to wait for command completion - * @return_data: read and return data from the buffer (true) or not (false) - * Needed because FW structures are big endian and decoding of - * these fields can be 8 bit or 16 bit based on command. Decoding - * is not easily understood without making a table of commands. - * So we will leave this up to the caller to read back the data - * in these cases. * * Communicates with the manageability block. On success return IXGBE_SUCCESS * else returns semaphore error when encountering an error acquiring * semaphore or IXGBE_ERR_HOST_INTERFACE_COMMAND when command fails. + * + * This function assumes that the IXGBE_GSSR_SW_MNG_SM semaphore is held + * by the caller. **/ -s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, u32 *buffer, - u32 length, u32 timeout, bool return_data) +s32 ixgbe_hic_unlocked(struct ixgbe_hw *hw, u32 *buffer, u32 length, + u32 timeout) { - u32 hicr, i, bi, fwsts; - u32 hdr_size = sizeof(struct ixgbe_hic_hdr); - u16 buf_len; + u32 hicr, i, fwsts; u16 dword_len; - s32 status; - DEBUGFUNC("ixgbe_host_interface_command"); + DEBUGFUNC("ixgbe_hic_unlocked"); - if (length == 0 || length > IXGBE_HI_MAX_BLOCK_BYTE_LENGTH) { + if (!length || length > IXGBE_HI_MAX_BLOCK_BYTE_LENGTH) { DEBUGOUT1("Buffer length failure buffersize=%d.\n", length); return IXGBE_ERR_HOST_INTERFACE_COMMAND; } - /* Take management host interface semaphore */ - status = hw->mac.ops.acquire_swfw_sync(hw, IXGBE_GSSR_SW_MNG_SM); - - if (status) - return status; /* Set bit 9 of FWSTS clearing FW reset indication */ fwsts = IXGBE_READ_REG(hw, IXGBE_FWSTS); @@ -4362,17 +4350,15 @@ s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, u32 *buffer, /* Check that the host interface is enabled. */ hicr = IXGBE_READ_REG(hw, IXGBE_HICR); - if ((hicr & IXGBE_HICR_EN) == 0) { + if (!(hicr & IXGBE_HICR_EN)) { DEBUGOUT("IXGBE_HOST_EN bit disabled.\n"); - status = IXGBE_ERR_HOST_INTERFACE_COMMAND; - goto rel_out; + return IXGBE_ERR_HOST_INTERFACE_COMMAND; } /* Calculate length in DWORDs. We must be DWORD aligned */ - if ((length % (sizeof(u32))) != 0) { + if (length % sizeof(u32)) { DEBUGOUT("Buffer length failure, not aligned to dword"); - status = IXGBE_ERR_INVALID_ARGUMENT; - goto rel_out; + return IXGBE_ERR_INVALID_ARGUMENT; } dword_len = length >> 2; @@ -4395,14 +4381,59 @@ s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, u32 *buffer, } /* Check command completion */ - if ((timeout != 0 && i == timeout) || + if ((timeout && i == timeout) || !(IXGBE_READ_REG(hw, IXGBE_HICR) & IXGBE_HICR_SV)) { ERROR_REPORT1(IXGBE_ERROR_CAUTION, "Command has failed with no status valid.\n"); - status = IXGBE_ERR_HOST_INTERFACE_COMMAND; - goto rel_out; + return IXGBE_ERR_HOST_INTERFACE_COMMAND; } + return IXGBE_SUCCESS; +} + +/** + * ixgbe_host_interface_command - Issue command to manageability block + * @hw: pointer to the HW structure + * @buffer: contains the command to write and where the return status will + * be placed + * @length: length of buffer, must be multiple of 4 bytes + * @timeout: time in ms to wait for command completion + * @return_data: read and return data from the buffer (true) or not (false) + * Needed because FW structures are big endian and decoding of + * these fields can be 8 bit or 16 bit based on command. Decoding + * is not easily understood without making a table of commands. + * So we will leave this up to the caller to read back the data + * in these cases. + * + * Communicates with the manageability block. On success return IXGBE_SUCCESS + * else returns semaphore error when encountering an error acquiring + * semaphore or IXGBE_ERR_HOST_INTERFACE_COMMAND when command fails. + **/ +s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, u32 *buffer, + u32 length, u32 timeout, bool return_data) +{ + u32 hdr_size = sizeof(struct ixgbe_hic_hdr); + u16 dword_len; + u16 buf_len; + s32 status; + u32 bi; + + DEBUGFUNC("ixgbe_host_interface_command"); + + if (length == 0 || length > IXGBE_HI_MAX_BLOCK_BYTE_LENGTH) { + DEBUGOUT1("Buffer length failure buffersize=%d.\n", length); + return IXGBE_ERR_HOST_INTERFACE_COMMAND; + } + + /* Take management host interface semaphore */ + status = hw->mac.ops.acquire_swfw_sync(hw, IXGBE_GSSR_SW_MNG_SM); + if (status) + return status; + + status = ixgbe_hic_unlocked(hw, buffer, length, timeout); + if (status) + goto rel_out; + if (!return_data) goto rel_out; @@ -4417,7 +4448,7 @@ s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, u32 *buffer, /* If there is any thing in data position pull it in */ buf_len = ((struct ixgbe_hic_hdr *)buffer)->buf_len; - if (buf_len == 0) + if (!buf_len) goto rel_out; if (length < buf_len + hdr_size) { @@ -4923,14 +4954,6 @@ s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw, speedcnt++; highest_link_speed = IXGBE_LINK_SPEED_10GB_FULL; - /* If we already have link at this speed, just jump out */ - status = ixgbe_check_link(hw, &link_speed, &link_up, false); - if (status != IXGBE_SUCCESS) - return status; - - if ((link_speed == IXGBE_LINK_SPEED_10GB_FULL) && link_up) - goto out; - /* Set the module link speed */ switch (hw->phy.media_type) { case ixgbe_media_type_fiber: @@ -4981,14 +5004,6 @@ s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw, if (highest_link_speed == IXGBE_LINK_SPEED_UNKNOWN) highest_link_speed = IXGBE_LINK_SPEED_1GB_FULL; - /* If we already have link at this speed, just jump out */ - status = ixgbe_check_link(hw, &link_speed, &link_up, false); - if (status != IXGBE_SUCCESS) - return status; - - if ((link_speed == IXGBE_LINK_SPEED_1GB_FULL) && link_up) - goto out; - /* Set the module link speed */ switch (hw->phy.media_type) { case ixgbe_media_type_fiber: diff --git a/drivers/net/ixgbe/base/ixgbe_common.h b/drivers/net/ixgbe/base/ixgbe_common.h index 0545f85c..cd042375 100644 --- a/drivers/net/ixgbe/base/ixgbe_common.h +++ b/drivers/net/ixgbe/base/ixgbe_common.h @@ -159,6 +159,7 @@ s32 ixgbe_set_fw_drv_ver_generic(struct ixgbe_hw *hw, u8 maj, u8 min, u8 ixgbe_calculate_checksum(u8 *buffer, u32 length); s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, u32 *buffer, u32 length, u32 timeout, bool return_data); +s32 ixgbe_hic_unlocked(struct ixgbe_hw *, u32 *buffer, u32 length, u32 timeout); void ixgbe_clear_tx_pending(struct ixgbe_hw *hw); diff --git a/drivers/net/ixgbe/base/ixgbe_vf.c b/drivers/net/ixgbe/base/ixgbe_vf.c index a75074a5..26c0d81c 100644 --- a/drivers/net/ixgbe/base/ixgbe_vf.c +++ b/drivers/net/ixgbe/base/ixgbe_vf.c @@ -490,7 +490,7 @@ s32 ixgbe_get_mac_addr_vf(struct ixgbe_hw *hw, u8 *mac_addr) s32 ixgbevf_set_uc_addr_vf(struct ixgbe_hw *hw, u32 index, u8 *addr) { struct ixgbe_mbx_info *mbx = &hw->mbx; - u32 msgbuf[3]; + u32 msgbuf[3], msgbuf_chk; u8 *msg_addr = (u8 *)(&msgbuf[1]); s32 ret_val; @@ -503,18 +503,19 @@ s32 ixgbevf_set_uc_addr_vf(struct ixgbe_hw *hw, u32 index, u8 *addr) */ msgbuf[0] |= index << IXGBE_VT_MSGINFO_SHIFT; msgbuf[0] |= IXGBE_VF_SET_MACVLAN; + msgbuf_chk = msgbuf[0]; if (addr) memcpy(msg_addr, addr, 6); ret_val = mbx->ops.write_posted(hw, msgbuf, 3, 0); if (!ret_val) ret_val = mbx->ops.read_posted(hw, msgbuf, 3, 0); + if (!ret_val) { + msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS; - msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS; - - if (!ret_val) - if (msgbuf[0] == (IXGBE_VF_SET_MACVLAN | IXGBE_VT_MSGTYPE_NACK)) - ret_val = IXGBE_ERR_OUT_OF_MEM; + if (msgbuf[0] == (msgbuf_chk | IXGBE_VT_MSGTYPE_NACK)) + return IXGBE_ERR_OUT_OF_MEM; + } return ret_val; } diff --git a/drivers/net/ixgbe/base/ixgbe_x550.c b/drivers/net/ixgbe/base/ixgbe_x550.c index aa6e859f..e78c9c2c 100644 --- a/drivers/net/ixgbe/base/ixgbe_x550.c +++ b/drivers/net/ixgbe/base/ixgbe_x550.c @@ -2910,13 +2910,13 @@ s32 ixgbe_setup_phy_loopback_x550em(struct ixgbe_hw *hw) * * Reads a 16 bit word from the EEPROM using the hostif. **/ -s32 ixgbe_read_ee_hostif_data_X550(struct ixgbe_hw *hw, u16 offset, - u16 *data) +s32 ixgbe_read_ee_hostif_X550(struct ixgbe_hw *hw, u16 offset, u16 *data) { - s32 status; + const u32 mask = IXGBE_GSSR_SW_MNG_SM | IXGBE_GSSR_EEP_SM; struct ixgbe_hic_read_shadow_ram buffer; + s32 status; - DEBUGFUNC("ixgbe_read_ee_hostif_data_X550"); + DEBUGFUNC("ixgbe_read_ee_hostif_X550"); buffer.hdr.req.cmd = FW_READ_SHADOW_RAM_CMD; buffer.hdr.req.buf_lenh = 0; buffer.hdr.req.buf_lenl = FW_READ_SHADOW_RAM_LEN; @@ -2927,42 +2927,18 @@ s32 ixgbe_read_ee_hostif_data_X550(struct ixgbe_hw *hw, u16 offset, /* one word */ buffer.length = IXGBE_CPU_TO_BE16(sizeof(u16)); - status = ixgbe_host_interface_command(hw, (u32 *)&buffer, - sizeof(buffer), - IXGBE_HI_COMMAND_TIMEOUT, false); - + status = hw->mac.ops.acquire_swfw_sync(hw, mask); if (status) return status; - *data = (u16)IXGBE_READ_REG_ARRAY(hw, IXGBE_FLEX_MNG, - FW_NVM_DATA_OFFSET); - - return 0; -} - -/** - * ixgbe_read_ee_hostif_X550 - Read EEPROM word using a host interface command - * @hw: pointer to hardware structure - * @offset: offset of word in the EEPROM to read - * @data: word read from the EEPROM - * - * Reads a 16 bit word from the EEPROM using the hostif. - **/ -s32 ixgbe_read_ee_hostif_X550(struct ixgbe_hw *hw, u16 offset, - u16 *data) -{ - s32 status = IXGBE_SUCCESS; - - DEBUGFUNC("ixgbe_read_ee_hostif_X550"); - - if (hw->mac.ops.acquire_swfw_sync(hw, IXGBE_GSSR_EEP_SM) == - IXGBE_SUCCESS) { - status = ixgbe_read_ee_hostif_data_X550(hw, offset, data); - hw->mac.ops.release_swfw_sync(hw, IXGBE_GSSR_EEP_SM); - } else { - status = IXGBE_ERR_SWFW_SYNC; + status = ixgbe_hic_unlocked(hw, (u32 *)&buffer, sizeof(buffer), + IXGBE_HI_COMMAND_TIMEOUT); + if (!status) { + *data = (u16)IXGBE_READ_REG_ARRAY(hw, IXGBE_FLEX_MNG, + FW_NVM_DATA_OFFSET); } + hw->mac.ops.release_swfw_sync(hw, mask); return status; } @@ -2978,6 +2954,7 @@ s32 ixgbe_read_ee_hostif_X550(struct ixgbe_hw *hw, u16 offset, s32 ixgbe_read_ee_hostif_buffer_X550(struct ixgbe_hw *hw, u16 offset, u16 words, u16 *data) { + const u32 mask = IXGBE_GSSR_SW_MNG_SM | IXGBE_GSSR_EEP_SM; struct ixgbe_hic_read_shadow_ram buffer; u32 current_word = 0; u16 words_to_read; @@ -2987,7 +2964,7 @@ s32 ixgbe_read_ee_hostif_buffer_X550(struct ixgbe_hw *hw, DEBUGFUNC("ixgbe_read_ee_hostif_buffer_X550"); /* Take semaphore for the entire operation. */ - status = hw->mac.ops.acquire_swfw_sync(hw, IXGBE_GSSR_EEP_SM); + status = hw->mac.ops.acquire_swfw_sync(hw, mask); if (status) { DEBUGOUT("EEPROM read buffer - semaphore failed\n"); return status; @@ -3007,10 +2984,8 @@ s32 ixgbe_read_ee_hostif_buffer_X550(struct ixgbe_hw *hw, buffer.address = IXGBE_CPU_TO_BE32((offset + current_word) * 2); buffer.length = IXGBE_CPU_TO_BE16(words_to_read * 2); - status = ixgbe_host_interface_command(hw, (u32 *)&buffer, - sizeof(buffer), - IXGBE_HI_COMMAND_TIMEOUT, - false); + status = ixgbe_hic_unlocked(hw, (u32 *)&buffer, sizeof(buffer), + IXGBE_HI_COMMAND_TIMEOUT); if (status) { DEBUGOUT("Host interface command failed\n"); @@ -3035,7 +3010,7 @@ s32 ixgbe_read_ee_hostif_buffer_X550(struct ixgbe_hw *hw, } out: - hw->mac.ops.release_swfw_sync(hw, IXGBE_GSSR_EEP_SM); + hw->mac.ops.release_swfw_sync(hw, mask); return status; } diff --git a/drivers/net/ixgbe/base/ixgbe_x550.h b/drivers/net/ixgbe/base/ixgbe_x550.h index 27d5d02f..1d4b290c 100644 --- a/drivers/net/ixgbe/base/ixgbe_x550.h +++ b/drivers/net/ixgbe/base/ixgbe_x550.h @@ -98,8 +98,6 @@ s32 ixgbe_read_ee_hostif_buffer_X550(struct ixgbe_hw *hw, u16 offset, u16 words, u16 *data); s32 ixgbe_read_ee_hostif_X550(struct ixgbe_hw *hw, u16 offset, u16 *data); -s32 ixgbe_read_ee_hostif_data_X550(struct ixgbe_hw *hw, u16 offset, - u16 *data); s32 ixgbe_write_ee_hostif_data_X550(struct ixgbe_hw *hw, u16 offset, u16 data); s32 ixgbe_set_eee_X550(struct ixgbe_hw *hw, bool enable_eee); diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c b/drivers/net/ixgbe/ixgbe_ethdev.c index d478a159..e1029301 100644 --- a/drivers/net/ixgbe/ixgbe_ethdev.c +++ b/drivers/net/ixgbe/ixgbe_ethdev.c @@ -1768,6 +1768,7 @@ ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev) IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t ctrl; uint16_t i; + struct ixgbe_rx_queue *rxq; PMD_INIT_FUNC_TRACE(); @@ -1778,9 +1779,10 @@ ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev) } else { /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */ for (i = 0; i < dev->data->nb_rx_queues; i++) { - ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); + rxq = dev->data->rx_queues[i]; + ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); ctrl &= ~IXGBE_RXDCTL_VME; - IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); + IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), ctrl); /* record those setting for HW strip per queue */ ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0); @@ -1795,6 +1797,7 @@ ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev) IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); uint32_t ctrl; uint16_t i; + struct ixgbe_rx_queue *rxq; PMD_INIT_FUNC_TRACE(); @@ -1805,9 +1808,10 @@ ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev) } else { /* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */ for (i = 0; i < dev->data->nb_rx_queues; i++) { - ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); + rxq = dev->data->rx_queues[i]; + ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); ctrl |= IXGBE_RXDCTL_VME; - IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); + IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), ctrl); /* record those setting for HW strip per queue */ ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1); diff --git a/drivers/net/ixgbe/ixgbe_fdir.c b/drivers/net/ixgbe/ixgbe_fdir.c index 861c7cbe..4b81ee37 100644 --- a/drivers/net/ixgbe/ixgbe_fdir.c +++ b/drivers/net/ixgbe/ixgbe_fdir.c @@ -432,12 +432,12 @@ fdir_set_input_mask_x550(struct rte_eth_dev *dev, fdiripv6m |= IXGBE_FDIRIP6M_TUNNEL_TYPE | IXGBE_FDIRIP6M_TNI_VNI; - mac_mask = input_mask->mac_addr_byte_mask; - fdiripv6m |= (mac_mask << IXGBE_FDIRIP6M_INNER_MAC_SHIFT) - & IXGBE_FDIRIP6M_INNER_MAC; - info->mask.mac_addr_byte_mask = input_mask->mac_addr_byte_mask; - if (mode == RTE_FDIR_MODE_PERFECT_TUNNEL) { + mac_mask = input_mask->mac_addr_byte_mask; + fdiripv6m |= (mac_mask << IXGBE_FDIRIP6M_INNER_MAC_SHIFT) + & IXGBE_FDIRIP6M_INNER_MAC; + info->mask.mac_addr_byte_mask = input_mask->mac_addr_byte_mask; + switch (input_mask->tunnel_type_mask) { case 0: /* Mask turnnel type */ diff --git a/drivers/net/ixgbe/ixgbe_regs.h b/drivers/net/ixgbe/ixgbe_regs.h index c7457a6f..773e1693 100644 --- a/drivers/net/ixgbe/ixgbe_regs.h +++ b/drivers/net/ixgbe/ixgbe_regs.h @@ -56,10 +56,10 @@ static const struct reg_info ixgbe_regs_general[] = { }; static const struct reg_info ixgbevf_regs_general[] = { - {IXGBE_CTRL, 1, 1, "IXGBE_CTRL"}, - {IXGBE_STATUS, 1, 1, "IXGBE_STATUS"}, + {IXGBE_VFCTRL, 1, 1, "IXGBE_VFCTRL"}, + {IXGBE_VFSTATUS, 1, 1, "IXGBE_VFSTATUS"}, {IXGBE_VFLINKS, 1, 1, "IXGBE_VFLINKS"}, - {IXGBE_FRTIMER, 1, 1, "IXGBE_FRTIMER"}, + {IXGBE_VFFRTIMER, 1, 1, "IXGBE_VFFRTIMER"}, {IXGBE_VFMAILBOX, 1, 1, "IXGBE_VFMAILBOX"}, {IXGBE_VFMBMEM, 16, 4, "IXGBE_VFMBMEM"}, {IXGBE_VFRXMEMWRAP, 1, 1, "IXGBE_VFRXMEMWRAP"}, @@ -145,17 +145,17 @@ static const struct reg_info ixgbe_regs_rxdma[] = { }; static const struct reg_info ixgbevf_regs_rxdma[] = { - {IXGBE_RDBAL(0), 8, 0x40, "IXGBE_RDBAL"}, - {IXGBE_RDBAH(0), 8, 0x40, "IXGBE_RDBAH"}, - {IXGBE_RDLEN(0), 8, 0x40, "IXGBE_RDLEN"}, - {IXGBE_RDH(0), 8, 0x40, "IXGBE_RDH"}, - {IXGBE_RDT(0), 8, 0x40, "IXGBE_RDT"}, - {IXGBE_RXDCTL(0), 8, 0x40, "IXGBE_RXDCTL"}, - {IXGBE_SRRCTL(0), 8, 0x40, "IXGBE_SRRCTL"}, + {IXGBE_VFRDBAL(0), 8, 0x40, "IXGBE_VFRDBAL"}, + {IXGBE_VFRDBAH(0), 8, 0x40, "IXGBE_VFRDBAH"}, + {IXGBE_VFRDLEN(0), 8, 0x40, "IXGBE_VFRDLEN"}, + {IXGBE_VFRDH(0), 8, 0x40, "IXGBE_VFRDH"}, + {IXGBE_VFRDT(0), 8, 0x40, "IXGBE_VFRDT"}, + {IXGBE_VFRXDCTL(0), 8, 0x40, "IXGBE_VFRXDCTL"}, + {IXGBE_VFSRRCTL(0), 8, 0x40, "IXGBE_VFSRRCTL"}, {IXGBE_VFPSRTYPE, 1, 1, "IXGBE_VFPSRTYPE"}, {IXGBE_VFRSCCTL(0), 8, 0x40, "IXGBE_VFRSCCTL"}, - {IXGBE_PVFDCA_RXCTRL(0), 8, 0x40, "IXGBE_PVFDCA_RXCTRL"}, - {IXGBE_PVFDCA_TXCTRL(0), 8, 0x40, "IXGBE_PVFDCA_TXCTRL"}, + {IXGBE_VFDCA_RXCTRL(0), 8, 0x40, "IXGBE_VFDCA_RXCTRL"}, + {IXGBE_VFDCA_TXCTRL(0), 8, 0x40, "IXGBE_VFDCA_TXCTRL"}, {0, 0, 0, ""} }; @@ -193,14 +193,14 @@ static struct reg_info ixgbe_regs_tx[] = { }; static const struct reg_info ixgbevf_regs_tx[] = { - {IXGBE_TDBAL(0), 4, 0x40, "IXGBE_TDBAL"}, - {IXGBE_TDBAH(0), 4, 0x40, "IXGBE_TDBAH"}, - {IXGBE_TDLEN(0), 4, 0x40, "IXGBE_TDLEN"}, - {IXGBE_TDH(0), 4, 0x40, "IXGBE_TDH"}, - {IXGBE_TDT(0), 4, 0x40, "IXGBE_TDT"}, - {IXGBE_TXDCTL(0), 4, 0x40, "IXGBE_TXDCTL"}, - {IXGBE_TDWBAL(0), 4, 0x40, "IXGBE_TDWBAL"}, - {IXGBE_TDWBAH(0), 4, 0x40, "IXGBE_TDWBAH"}, + {IXGBE_VFTDBAL(0), 4, 0x40, "IXGBE_VFTDBAL"}, + {IXGBE_VFTDBAH(0), 4, 0x40, "IXGBE_VFTDBAH"}, + {IXGBE_VFTDLEN(0), 4, 0x40, "IXGBE_VFTDLEN"}, + {IXGBE_VFTDH(0), 4, 0x40, "IXGBE_VFTDH"}, + {IXGBE_VFTDT(0), 4, 0x40, "IXGBE_VFTDT"}, + {IXGBE_VFTXDCTL(0), 4, 0x40, "IXGBE_VFTXDCTL"}, + {IXGBE_VFTDWBAL(0), 4, 0x40, "IXGBE_VFTDWBAL"}, + {IXGBE_VFTDWBAH(0), 4, 0x40, "IXGBE_VFTDWBAH"}, {0, 0, 0, ""} }; diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h b/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h index 62b82013..3c3c0095 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h @@ -204,8 +204,20 @@ _ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq) return; /* free all mbufs that are valid in the ring */ - for (i = rxq->rx_tail; i != rxq->rxrearm_start; i = (i + 1) & mask) - rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); + if (rxq->rxrearm_nb == 0) { + for (i = 0; i < rxq->nb_rx_desc; i++) { + if (rxq->sw_ring[i].mbuf != NULL) + rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); + } + } else { + for (i = rxq->rx_tail; + i != rxq->rxrearm_start; + i = (i + 1) & mask) { + if (rxq->sw_ring[i].mbuf != NULL) + rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); + } + } + rxq->rxrearm_nb = rxq->nb_rx_desc; /* set all entries to NULL */ diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c index 1c4fd7c1..7fb155a4 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c +++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c @@ -305,6 +305,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, /* Read desc statuses backwards to avoid race condition */ /* A.1 load 4 pkts desc */ descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3)); + rte_compiler_barrier(); /* B.2 copy 2 mbuf point into rx_pkts */ _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1); @@ -313,8 +314,10 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]); descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2)); + rte_compiler_barrier(); /* B.1 load 2 mbuf point */ descs[1] = _mm_loadu_si128((__m128i *)(rxdp + 1)); + rte_compiler_barrier(); descs[0] = _mm_loadu_si128((__m128i *)(rxdp)); /* B.2 copy 2 mbuf point into rx_pkts */ diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c index 304c8461..9f276192 100644 --- a/drivers/net/mlx4/mlx4.c +++ b/drivers/net/mlx4/mlx4.c @@ -61,16 +61,16 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ether.h> #include <rte_ethdev.h> @@ -87,7 +87,7 @@ #include <rte_alarm.h> #include <rte_memory.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* Generated configuration header. */ diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile index f6d39388..cf87f0b1 100644 --- a/drivers/net/mlx5/Makefile +++ b/drivers/net/mlx5/Makefile @@ -116,6 +116,26 @@ mlx5_autoconf.h.new: $(RTE_SDK)/scripts/auto-config-h.sh infiniband/mlx5_hw.h \ enum MLX5_ETH_VLAN_INLINE_HEADER_SIZE \ $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_VERBS_MLX5_OPCODE_TSO \ + infiniband/mlx5_hw.h \ + enum MLX5_OPCODE_TSO \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_ETHTOOL_LINK_MODE_25G \ + /usr/include/linux/ethtool.h \ + enum ETHTOOL_LINK_MODE_25000baseCR_Full_BIT \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_ETHTOOL_LINK_MODE_50G \ + /usr/include/linux/ethtool.h \ + enum ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_ETHTOOL_LINK_MODE_100G \ + /usr/include/linux/ethtool.h \ + enum ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT \ + $(AUTOCONF_OUTPUT) # Create mlx5_autoconf.h or update it in case it differs from the new one. diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index d96a9aff..9448374e 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -43,16 +43,16 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_malloc.h> #include <rte_ethdev.h> @@ -60,7 +60,7 @@ #include <rte_common.h> #include <rte_kvargs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" @@ -668,6 +668,7 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) /* Bring Ethernet device up. */ DEBUG("forcing Ethernet interface up"); priv_set_flags(priv, ~IFF_UP, IFF_UP); + mlx5_link_update_unlocked(priv->dev, 1); continue; port_error: diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 3a866098..79b7a600 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -43,16 +43,16 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ether.h> #include <rte_ethdev.h> @@ -60,7 +60,7 @@ #include <rte_interrupts.h> #include <rte_errno.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5_utils.h" @@ -134,6 +134,8 @@ struct priv { unsigned int (*reta_idx)[]; /* RETA index table. */ unsigned int reta_idx_n; /* RETA index size. */ struct fdir_filter_list *fdir_filter_list; /* Flow director rules. */ + struct fdir_queue *fdir_drop_queue; /* Flow director drop queue. */ + uint32_t link_speed_capa; /* Link speed capabilities. */ rte_spinlock_t lock; /* Lock for control functions. */ }; @@ -186,6 +188,7 @@ int priv_set_flags(struct priv *, unsigned int, unsigned int); int mlx5_dev_configure(struct rte_eth_dev *); void mlx5_dev_infos_get(struct rte_eth_dev *, struct rte_eth_dev_info *); const uint32_t *mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev); +int mlx5_link_update_unlocked(struct rte_eth_dev *, int); int mlx5_link_update(struct rte_eth_dev *, int); int mlx5_dev_set_mtu(struct rte_eth_dev *, uint16_t); int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *, struct rte_eth_fc_conf *); @@ -257,6 +260,7 @@ void mlx5_dev_stop(struct rte_eth_dev *); /* mlx5_fdir.c */ +void priv_fdir_queue_destroy(struct priv *, struct fdir_queue *); int fdir_init_filters_list(struct priv *); void priv_fdir_delete_filters_list(struct priv *); void priv_fdir_disable(struct priv *); diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index 130e15d5..ba1ec2a6 100644 --- a/drivers/net/mlx5/mlx5_ethdev.c +++ b/drivers/net/mlx5/mlx5_ethdev.c @@ -50,7 +50,7 @@ /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_atomic.h> #include <rte_ethdev.h> @@ -60,7 +60,7 @@ #include <rte_alarm.h> #include <rte_malloc.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" @@ -583,7 +583,8 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) (DEV_RX_OFFLOAD_IPV4_CKSUM | DEV_RX_OFFLOAD_UDP_CKSUM | DEV_RX_OFFLOAD_TCP_CKSUM) : - 0); + 0) | + (priv->hw_vlan_strip ? DEV_RX_OFFLOAD_VLAN_STRIP : 0); if (!priv->mps) info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; if (priv->hw_csum) @@ -599,15 +600,10 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) * size if it is not fixed. * The API should be updated to solve this problem. */ info->reta_size = priv->ind_table_max_size; - info->speed_capa = - ETH_LINK_SPEED_1G | - ETH_LINK_SPEED_10G | - ETH_LINK_SPEED_20G | - ETH_LINK_SPEED_25G | - ETH_LINK_SPEED_40G | - ETH_LINK_SPEED_50G | - ETH_LINK_SPEED_56G | - ETH_LINK_SPEED_100G; + info->hash_key_size = ((*priv->rss_conf) ? + (*priv->rss_conf)[0]->rss_key_len : + 0); + info->speed_capa = priv->link_speed_capa; priv_unlock(priv); } @@ -630,7 +626,7 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) } /** - * DPDK callback to retrieve physical link information (unlocked version). + * Retrieve physical link information (unlocked version using legacy ioctl). * * @param dev * Pointer to Ethernet device structure. @@ -638,11 +634,11 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) * Wait for request completion (ignored). */ static int -mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) +mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) { struct priv *priv = mlx5_get_priv(dev); struct ethtool_cmd edata = { - .cmd = ETHTOOL_GSET + .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */ }; struct ifreq ifr; struct rte_eth_link dev_link; @@ -667,6 +663,19 @@ mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) dev_link.link_speed = 0; else dev_link.link_speed = link_speed; + priv->link_speed_capa = 0; + if (edata.supported & SUPPORTED_Autoneg) + priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; + if (edata.supported & (SUPPORTED_1000baseT_Full | + SUPPORTED_1000baseKX_Full)) + priv->link_speed_capa |= ETH_LINK_SPEED_1G; + if (edata.supported & SUPPORTED_10000baseKR_Full) + priv->link_speed_capa |= ETH_LINK_SPEED_10G; + if (edata.supported & (SUPPORTED_40000baseKR4_Full | + SUPPORTED_40000baseCR4_Full | + SUPPORTED_40000baseSR4_Full | + SUPPORTED_40000baseLR4_Full)) + priv->link_speed_capa |= ETH_LINK_SPEED_40G; dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & @@ -681,6 +690,123 @@ mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) } /** + * Retrieve physical link information (unlocked version using new ioctl from + * Linux 4.5). + * + * @param dev + * Pointer to Ethernet device structure. + * @param wait_to_complete + * Wait for request completion (ignored). + */ +static int +mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) +{ +#ifdef ETHTOOL_GLINKSETTINGS + struct priv *priv = mlx5_get_priv(dev); + struct ethtool_link_settings edata = { + .cmd = ETHTOOL_GLINKSETTINGS, + }; + struct ifreq ifr; + struct rte_eth_link dev_link; + uint64_t sc; + + (void)wait_to_complete; + if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { + WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); + return -1; + } + memset(&dev_link, 0, sizeof(dev_link)); + dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && + (ifr.ifr_flags & IFF_RUNNING)); + ifr.ifr_data = (void *)&edata; + if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", + strerror(errno)); + return -1; + } + dev_link.link_speed = edata.speed; + sc = edata.link_mode_masks[0] | + ((uint64_t)edata.link_mode_masks[1] << 32); + priv->link_speed_capa = 0; + /* Link speeds available in kernel v4.5. */ + if (sc & ETHTOOL_LINK_MODE_Autoneg_BIT) + priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; + if (sc & (ETHTOOL_LINK_MODE_1000baseT_Full_BIT | + ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_1G; + if (sc & (ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT | + ETHTOOL_LINK_MODE_10000baseKR_Full_BIT | + ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_10G; + if (sc & (ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT | + ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_20G; + if (sc & (ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT | + ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT | + ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT | + ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_40G; + if (sc & (ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT | + ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT | + ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT | + ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_56G; + /* Link speeds available in kernel v4.6. */ +#ifdef HAVE_ETHTOOL_LINK_MODE_25G + if (sc & (ETHTOOL_LINK_MODE_25000baseCR_Full_BIT | + ETHTOOL_LINK_MODE_25000baseKR_Full_BIT | + ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_25G; +#endif +#ifdef HAVE_ETHTOOL_LINK_MODE_50G + if (sc & (ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT | + ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_50G; +#endif +#ifdef HAVE_ETHTOOL_LINK_MODE_100G + if (sc & (ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT | + ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT | + ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT | + ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_100G; +#endif + dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? + ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); + dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & + ETH_LINK_SPEED_FIXED); + if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { + /* Link status changed. */ + dev->data->dev_link = dev_link; + return 0; + } +#else + (void)dev; + (void)wait_to_complete; +#endif + /* Link status is still the same. */ + return -1; +} + +/** + * DPDK callback to retrieve physical link information (unlocked version). + * + * @param dev + * Pointer to Ethernet device structure. + * @param wait_to_complete + * Wait for request completion (ignored). + */ +int +mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) +{ + int ret; + + ret = mlx5_link_update_unlocked_gs(dev, wait_to_complete); + if (ret < 0) + ret = mlx5_link_update_unlocked_gset(dev, wait_to_complete); + return ret; +} + +/** * DPDK callback to retrieve physical link information. * * @param dev @@ -1308,11 +1434,13 @@ mlx5_secondary_data_setup(struct priv *priv) continue; primary_txq_ctrl = container_of(primary_txq, struct txq_ctrl, txq); - txq_ctrl = rte_calloc_socket("TXQ", 1, sizeof(*txq_ctrl), 0, + txq_ctrl = rte_calloc_socket("TXQ", 1, sizeof(*txq_ctrl) + + (1 << primary_txq->elts_n) * + sizeof(struct rte_mbuf *), 0, primary_txq_ctrl->socket); if (txq_ctrl != NULL) { if (txq_ctrl_setup(priv->dev, - primary_txq_ctrl, + txq_ctrl, primary_txq->elts_n, primary_txq_ctrl->socket, NULL) == 0) { @@ -1397,10 +1525,6 @@ priv_select_tx_function(struct priv *priv) } else if ((priv->sriov == 0) && priv->mps) { priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw; DEBUG("selected MPW TX function"); - } else if (priv->txq_inline && (priv->txqs_n >= priv->txqs_inline)) { - priv->dev->tx_pkt_burst = mlx5_tx_burst_inline; - DEBUG("selected inline TX function (%u >= %u queues)", - priv->txqs_n, priv->txqs_inline); } } diff --git a/drivers/net/mlx5/mlx5_fdir.c b/drivers/net/mlx5/mlx5_fdir.c index 73eb00ec..1acf6826 100644 --- a/drivers/net/mlx5/mlx5_fdir.c +++ b/drivers/net/mlx5/mlx5_fdir.c @@ -40,23 +40,23 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ether.h> #include <rte_malloc.h> #include <rte_ethdev.h> #include <rte_common.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" @@ -75,6 +75,7 @@ struct fdir_flow_desc { struct mlx5_fdir_filter { LIST_ENTRY(mlx5_fdir_filter) next; uint16_t queue; /* Queue assigned to if FDIR match. */ + enum rte_eth_fdir_behavior behavior; struct fdir_flow_desc desc; struct ibv_exp_flow *flow; }; @@ -400,6 +401,145 @@ create_flow: } /** + * Destroy a flow director queue. + * + * @param fdir_queue + * Flow director queue to be destroyed. + */ +void +priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue) +{ + struct mlx5_fdir_filter *fdir_filter; + + /* Disable filter flows still applying to this queue. */ + LIST_FOREACH(fdir_filter, priv->fdir_filter_list, next) { + unsigned int idx = fdir_filter->queue; + struct rxq_ctrl *rxq_ctrl = + container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq); + + assert(idx < priv->rxqs_n); + if (fdir_queue == rxq_ctrl->fdir_queue && + fdir_filter->flow != NULL) { + claim_zero(ibv_exp_destroy_flow(fdir_filter->flow)); + fdir_filter->flow = NULL; + } + } + assert(fdir_queue->qp); + claim_zero(ibv_destroy_qp(fdir_queue->qp)); + assert(fdir_queue->ind_table); + claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table)); + if (fdir_queue->wq) + claim_zero(ibv_exp_destroy_wq(fdir_queue->wq)); + if (fdir_queue->cq) + claim_zero(ibv_destroy_cq(fdir_queue->cq)); +#ifndef NDEBUG + memset(fdir_queue, 0x2a, sizeof(*fdir_queue)); +#endif + rte_free(fdir_queue); +} + +/** + * Create a flow director queue. + * + * @param priv + * Private structure. + * @param wq + * Work queue to route matched packets to, NULL if one needs to + * be created. + * + * @return + * Related flow director queue on success, NULL otherwise. + */ +static struct fdir_queue * +priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq, + unsigned int socket) +{ + struct fdir_queue *fdir_queue; + + fdir_queue = rte_calloc_socket(__func__, 1, sizeof(*fdir_queue), + 0, socket); + if (!fdir_queue) { + ERROR("cannot allocate flow director queue"); + return NULL; + } + assert(priv->pd); + assert(priv->ctx); + if (!wq) { + fdir_queue->cq = ibv_exp_create_cq( + priv->ctx, 1, NULL, NULL, 0, + &(struct ibv_exp_cq_init_attr){ + .comp_mask = 0, + }); + if (!fdir_queue->cq) { + ERROR("cannot create flow director CQ"); + goto error; + } + fdir_queue->wq = ibv_exp_create_wq( + priv->ctx, + &(struct ibv_exp_wq_init_attr){ + .wq_type = IBV_EXP_WQT_RQ, + .max_recv_wr = 1, + .max_recv_sge = 1, + .pd = priv->pd, + .cq = fdir_queue->cq, + }); + if (!fdir_queue->wq) { + ERROR("cannot create flow director WQ"); + goto error; + } + wq = fdir_queue->wq; + } + fdir_queue->ind_table = ibv_exp_create_rwq_ind_table( + priv->ctx, + &(struct ibv_exp_rwq_ind_table_init_attr){ + .pd = priv->pd, + .log_ind_tbl_size = 0, + .ind_tbl = &wq, + .comp_mask = 0, + }); + if (!fdir_queue->ind_table) { + ERROR("cannot create flow director indirection table"); + goto error; + } + fdir_queue->qp = ibv_exp_create_qp( + priv->ctx, + &(struct ibv_exp_qp_init_attr){ + .qp_type = IBV_QPT_RAW_PACKET, + .comp_mask = + IBV_EXP_QP_INIT_ATTR_PD | + IBV_EXP_QP_INIT_ATTR_PORT | + IBV_EXP_QP_INIT_ATTR_RX_HASH, + .pd = priv->pd, + .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){ + .rx_hash_function = + IBV_EXP_RX_HASH_FUNC_TOEPLITZ, + .rx_hash_key_len = rss_hash_default_key_len, + .rx_hash_key = rss_hash_default_key, + .rx_hash_fields_mask = 0, + .rwq_ind_tbl = fdir_queue->ind_table, + }, + .port_num = priv->port, + }); + if (!fdir_queue->qp) { + ERROR("cannot create flow director hash RX QP"); + goto error; + } + return fdir_queue; +error: + assert(fdir_queue); + assert(!fdir_queue->qp); + if (fdir_queue->ind_table) + claim_zero(ibv_exp_destroy_rwq_ind_table + (fdir_queue->ind_table)); + if (fdir_queue->wq) + claim_zero(ibv_exp_destroy_wq(fdir_queue->wq)); + if (fdir_queue->cq) + claim_zero(ibv_destroy_cq(fdir_queue->cq)); + rte_free(fdir_queue); + return NULL; +} + +/** * Get flow director queue for a specific RX queue, create it in case * it does not exist. * @@ -416,74 +556,42 @@ priv_get_fdir_queue(struct priv *priv, uint16_t idx) { struct rxq_ctrl *rxq_ctrl = container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq); - struct fdir_queue *fdir_queue = &rxq_ctrl->fdir_queue; - struct ibv_exp_rwq_ind_table *ind_table = NULL; - struct ibv_qp *qp = NULL; - struct ibv_exp_rwq_ind_table_init_attr ind_init_attr; - struct ibv_exp_rx_hash_conf hash_conf; - struct ibv_exp_qp_init_attr qp_init_attr; - int err = 0; - - /* Return immediately if it has already been created. */ - if (fdir_queue->qp != NULL) - return fdir_queue; - - ind_init_attr = (struct ibv_exp_rwq_ind_table_init_attr){ - .pd = priv->pd, - .log_ind_tbl_size = 0, - .ind_tbl = &rxq_ctrl->wq, - .comp_mask = 0, - }; - - errno = 0; - ind_table = ibv_exp_create_rwq_ind_table(priv->ctx, - &ind_init_attr); - if (ind_table == NULL) { - /* Not clear whether errno is set. */ - err = (errno ? errno : EINVAL); - ERROR("RX indirection table creation failed with error %d: %s", - err, strerror(err)); - goto error; - } + struct fdir_queue *fdir_queue = rxq_ctrl->fdir_queue; - /* Create fdir_queue qp. */ - hash_conf = (struct ibv_exp_rx_hash_conf){ - .rx_hash_function = IBV_EXP_RX_HASH_FUNC_TOEPLITZ, - .rx_hash_key_len = rss_hash_default_key_len, - .rx_hash_key = rss_hash_default_key, - .rx_hash_fields_mask = 0, - .rwq_ind_tbl = ind_table, - }; - qp_init_attr = (struct ibv_exp_qp_init_attr){ - .max_inl_recv = 0, /* Currently not supported. */ - .qp_type = IBV_QPT_RAW_PACKET, - .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | - IBV_EXP_QP_INIT_ATTR_RX_HASH), - .pd = priv->pd, - .rx_hash_conf = &hash_conf, - .port_num = priv->port, - }; - - qp = ibv_exp_create_qp(priv->ctx, &qp_init_attr); - if (qp == NULL) { - err = (errno ? errno : EINVAL); - ERROR("hash RX QP creation failure: %s", strerror(err)); - goto error; + assert(rxq_ctrl->wq); + if (fdir_queue == NULL) { + fdir_queue = priv_fdir_queue_create(priv, rxq_ctrl->wq, + rxq_ctrl->socket); + rxq_ctrl->fdir_queue = fdir_queue; } - - fdir_queue->ind_table = ind_table; - fdir_queue->qp = qp; - return fdir_queue; +} -error: - if (qp != NULL) - claim_zero(ibv_destroy_qp(qp)); - - if (ind_table != NULL) - claim_zero(ibv_exp_destroy_rwq_ind_table(ind_table)); +/** + * Get or flow director drop queue. Create it if it does not exist. + * + * @param priv + * Private structure. + * + * @return + * Flow director drop queue on success, NULL otherwise. + */ +static struct fdir_queue * +priv_get_fdir_drop_queue(struct priv *priv) +{ + struct fdir_queue *fdir_queue = priv->fdir_drop_queue; - return NULL; + if (fdir_queue == NULL) { + unsigned int socket = SOCKET_ID_ANY; + + /* Select a known NUMA socket if possible. */ + if (priv->rxqs_n && (*priv->rxqs)[0]) + socket = container_of((*priv->rxqs)[0], + struct rxq_ctrl, rxq)->socket; + fdir_queue = priv_fdir_queue_create(priv, NULL, socket); + priv->fdir_drop_queue = fdir_queue; + } + return fdir_queue; } /** @@ -508,7 +616,11 @@ priv_fdir_filter_enable(struct priv *priv, return 0; /* Get fdir_queue for specific queue. */ - fdir_queue = priv_get_fdir_queue(priv, mlx5_fdir_filter->queue); + if (mlx5_fdir_filter->behavior == RTE_ETH_FDIR_REJECT) + fdir_queue = priv_get_fdir_drop_queue(priv); + else + fdir_queue = priv_get_fdir_queue(priv, + mlx5_fdir_filter->queue); if (fdir_queue == NULL) { ERROR("failed to create flow director rxq for queue %d", @@ -601,7 +713,6 @@ priv_fdir_disable(struct priv *priv) { unsigned int i; struct mlx5_fdir_filter *mlx5_fdir_filter; - struct fdir_queue *fdir_queue; /* Run on every flow director filter and destroy flow handle. */ LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) { @@ -618,23 +729,19 @@ priv_fdir_disable(struct priv *priv) } } - /* Run on every RX queue to destroy related flow director QP and - * indirection table. */ + /* Destroy flow director context in each RX queue. */ for (i = 0; (i != priv->rxqs_n); i++) { struct rxq_ctrl *rxq_ctrl = container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq); - fdir_queue = &rxq_ctrl->fdir_queue; - if (fdir_queue->qp != NULL) { - claim_zero(ibv_destroy_qp(fdir_queue->qp)); - fdir_queue->qp = NULL; - } - - if (fdir_queue->ind_table != NULL) { - claim_zero(ibv_exp_destroy_rwq_ind_table - (fdir_queue->ind_table)); - fdir_queue->ind_table = NULL; - } + if (!rxq_ctrl->fdir_queue) + continue; + priv_fdir_queue_destroy(priv, rxq_ctrl->fdir_queue); + rxq_ctrl->fdir_queue = NULL; + } + if (priv->fdir_drop_queue) { + priv_fdir_queue_destroy(priv, priv->fdir_drop_queue); + priv->fdir_drop_queue = NULL; } } @@ -736,8 +843,9 @@ priv_fdir_filter_add(struct priv *priv, return err; } - /* Set queue. */ + /* Set action parameters. */ mlx5_fdir_filter->queue = fdir_filter->action.rx_queue; + mlx5_fdir_filter->behavior = fdir_filter->action.behavior; /* Convert to mlx5 filter descriptor. */ fdir_filter_to_flow_desc(fdir_filter, @@ -955,7 +1063,7 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg) { - int ret = -EINVAL; + int ret = EINVAL; struct priv *priv = dev->data->dev_private; switch (filter_type) { @@ -970,5 +1078,5 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, break; } - return ret; + return -ret; } diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c index f6b27bb8..4fcfd3b8 100644 --- a/drivers/net/mlx5/mlx5_mac.c +++ b/drivers/net/mlx5/mlx5_mac.c @@ -44,22 +44,22 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ether.h> #include <rte_ethdev.h> #include <rte_common.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c index 67dfefa8..0a363846 100644 --- a/drivers/net/mlx5/mlx5_mr.c +++ b/drivers/net/mlx5/mlx5_mr.c @@ -34,20 +34,20 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_mempool.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h index 5db219b3..1c369cac 100644 --- a/drivers/net/mlx5/mlx5_prm.h +++ b/drivers/net/mlx5/mlx5_prm.h @@ -37,13 +37,15 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/mlx5_hw.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif +#include "mlx5_autoconf.h" + /* Get CQE owner bit. */ #define MLX5_CQE_OWNER(op_own) ((op_own) & MLX5_CQE_OWNER_MASK) @@ -71,6 +73,31 @@ /* Room for inline data in multi-packet WQE. */ #define MLX5_MWQE64_INL_DATA 28 +#ifndef HAVE_VERBS_MLX5_OPCODE_TSO +#define MLX5_OPCODE_TSO MLX5_OPCODE_LSO_MPW /* Compat with OFED 3.3. */ +#endif + +/* IPv4 packet. */ +#define MLX5_CQE_RX_IPV4_PACKET (1u << 2) + +/* IPv6 packet. */ +#define MLX5_CQE_RX_IPV6_PACKET (1u << 3) + +/* Outer IPv4 packet. */ +#define MLX5_CQE_RX_OUTER_IPV4_PACKET (1u << 7) + +/* Outer IPv6 packet. */ +#define MLX5_CQE_RX_OUTER_IPV6_PACKET (1u << 8) + +/* Tunnel packet bit in the CQE. */ +#define MLX5_CQE_RX_TUNNEL_PACKET (1u << 4) + +/* Outer IP checksum OK. */ +#define MLX5_CQE_RX_OUTER_IP_CSUM_OK (1u << 5) + +/* Outer UDP header and checksum OK. */ +#define MLX5_CQE_RX_OUTER_TCP_UDP_CSUM_OK (1u << 6) + /* Subset of struct mlx5_wqe_eth_seg. */ struct mlx5_wqe_eth_seg_small { uint32_t rsvd0; diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c index 639e935b..0bed74ee 100644 --- a/drivers/net/mlx5/mlx5_rss.c +++ b/drivers/net/mlx5/mlx5_rss.c @@ -40,21 +40,21 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_malloc.h> #include <rte_ethdev.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c index 8b585554..173e6e84 100644 --- a/drivers/net/mlx5/mlx5_rxmode.c +++ b/drivers/net/mlx5/mlx5_rxmode.c @@ -38,20 +38,20 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ethdev.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index 29c137cd..7dbe8dd2 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -40,25 +40,25 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #include <infiniband/arch.h> #include <infiniband/mlx5_hw.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_mbuf.h> #include <rte_malloc.h> #include <rte_ethdev.h> #include <rte_common.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" @@ -745,6 +745,8 @@ rxq_cleanup(struct rxq_ctrl *rxq_ctrl) DEBUG("cleaning up %p", (void *)rxq_ctrl); rxq_free_elts(rxq_ctrl); + if (rxq_ctrl->fdir_queue != NULL) + priv_fdir_queue_destroy(rxq_ctrl->priv, rxq_ctrl->fdir_queue); if (rxq_ctrl->if_wq != NULL) { assert(rxq_ctrl->priv != NULL); assert(rxq_ctrl->priv->ctx != NULL); @@ -943,6 +945,11 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, (void)conf; /* Thresholds configuration (ignored). */ /* Enable scattered packets support for this queue if necessary. */ assert(mb_len >= RTE_PKTMBUF_HEADROOM); + /* If smaller than MRU, multi-segment support must be enabled. */ + if (mb_len < (priv->mtu > dev->data->dev_conf.rxmode.max_rx_pkt_len ? + dev->data->dev_conf.rxmode.max_rx_pkt_len : + priv->mtu)) + dev->data->dev_conf.rxmode.jumbo_frame = 1; if ((dev->data->dev_conf.rxmode.jumbo_frame) && (dev->data->dev_conf.rxmode.max_rx_pkt_len > (mb_len - RTE_PKTMBUF_HEADROOM))) { @@ -1259,7 +1266,7 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, (void *)dev, (void *)rxq_ctrl); (*priv->rxqs)[idx] = &rxq_ctrl->rxq; /* Update receive callback. */ - dev->rx_pkt_burst = mlx5_rx_burst; + priv_select_rx_function(priv); } priv_unlock(priv); return -ret; diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index fce3381a..79f7fa99 100644 --- a/drivers/net/mlx5/mlx5_rxtx.c +++ b/drivers/net/mlx5/mlx5_rxtx.c @@ -39,18 +39,18 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #include <infiniband/mlx5_hw.h> #include <infiniband/arch.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_mbuf.h> #include <rte_mempool.h> @@ -59,7 +59,7 @@ #include <rte_branch_prediction.h> #include <rte_ether.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" @@ -290,226 +290,103 @@ txq_mp2mr(struct txq *txq, struct rte_mempool *mp) * Pointer to TX queue structure. * @param wqe * Pointer to the WQE to fill. - * @param addr - * Buffer data address. + * @param buf + * Buffer. * @param length * Packet length. - * @param lkey - * Memory region lkey. - */ -static inline void -mlx5_wqe_write(struct txq *txq, volatile union mlx5_wqe *wqe, - uintptr_t addr, uint32_t length, uint32_t lkey) -{ - wqe->wqe.ctrl.data[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND); - wqe->wqe.ctrl.data[1] = htonl((txq->qp_num_8s) | 4); - wqe->wqe.ctrl.data[2] = 0; - wqe->wqe.ctrl.data[3] = 0; - wqe->inl.eseg.rsvd0 = 0; - wqe->inl.eseg.rsvd1 = 0; - wqe->inl.eseg.mss = 0; - wqe->inl.eseg.rsvd2 = 0; - wqe->wqe.eseg.inline_hdr_sz = htons(MLX5_ETH_INLINE_HEADER_SIZE); - /* Copy the first 16 bytes into inline header. */ - rte_memcpy((uint8_t *)(uintptr_t)wqe->wqe.eseg.inline_hdr_start, - (uint8_t *)(uintptr_t)addr, - MLX5_ETH_INLINE_HEADER_SIZE); - addr += MLX5_ETH_INLINE_HEADER_SIZE; - length -= MLX5_ETH_INLINE_HEADER_SIZE; - /* Store remaining data in data segment. */ - wqe->wqe.dseg.byte_count = htonl(length); - wqe->wqe.dseg.lkey = lkey; - wqe->wqe.dseg.addr = htonll(addr); - /* Increment consumer index. */ - ++txq->wqe_ci; -} - -/** - * Write a regular WQE with VLAN. * - * @param txq - * Pointer to TX queue structure. - * @param wqe - * Pointer to the WQE to fill. - * @param addr - * Buffer data address. - * @param length - * Packet length. - * @param lkey - * Memory region lkey. - * @param vlan_tci - * VLAN field to insert in packet. + * @return ds + * Number of DS elements consumed. */ -static inline void -mlx5_wqe_write_vlan(struct txq *txq, volatile union mlx5_wqe *wqe, - uintptr_t addr, uint32_t length, uint32_t lkey, - uint16_t vlan_tci) +static inline unsigned int +mlx5_wqe_write(struct txq *txq, volatile union mlx5_wqe *wqe, + struct rte_mbuf *buf, uint32_t length) { - uint32_t vlan = htonl(0x81000000 | vlan_tci); - + uintptr_t raw = (uintptr_t)&wqe->wqe.eseg.inline_hdr_start; + uint16_t ds; + uint16_t pkt_inline_sz = 16; + uintptr_t addr = rte_pktmbuf_mtod(buf, uintptr_t); + struct mlx5_wqe_data_seg *dseg = NULL; + + assert(length >= 16); + /* Start the know and common part of the WQE structure. */ wqe->wqe.ctrl.data[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND); - wqe->wqe.ctrl.data[1] = htonl((txq->qp_num_8s) | 4); wqe->wqe.ctrl.data[2] = 0; wqe->wqe.ctrl.data[3] = 0; - wqe->inl.eseg.rsvd0 = 0; - wqe->inl.eseg.rsvd1 = 0; - wqe->inl.eseg.mss = 0; - wqe->inl.eseg.rsvd2 = 0; - wqe->wqe.eseg.inline_hdr_sz = htons(MLX5_ETH_VLAN_INLINE_HEADER_SIZE); - /* - * Copy 12 bytes of source & destination MAC address. - * Copy 4 bytes of VLAN. - * Copy 2 bytes of Ether type. - */ - rte_memcpy((uint8_t *)(uintptr_t)wqe->wqe.eseg.inline_hdr_start, - (uint8_t *)(uintptr_t)addr, 12); - rte_memcpy((uint8_t *)((uintptr_t)wqe->wqe.eseg.inline_hdr_start + 12), - &vlan, sizeof(vlan)); - rte_memcpy((uint8_t *)((uintptr_t)wqe->wqe.eseg.inline_hdr_start + 16), - (uint8_t *)((uintptr_t)addr + 12), 2); - addr += MLX5_ETH_VLAN_INLINE_HEADER_SIZE - sizeof(vlan); - length -= MLX5_ETH_VLAN_INLINE_HEADER_SIZE - sizeof(vlan); - /* Store remaining data in data segment. */ - wqe->wqe.dseg.byte_count = htonl(length); - wqe->wqe.dseg.lkey = lkey; - wqe->wqe.dseg.addr = htonll(addr); - /* Increment consumer index. */ - ++txq->wqe_ci; -} - -/** - * Write a inline WQE. - * - * @param txq - * Pointer to TX queue structure. - * @param wqe - * Pointer to the WQE to fill. - * @param addr - * Buffer data address. - * @param length - * Packet length. - * @param lkey - * Memory region lkey. - */ -static inline void -mlx5_wqe_write_inline(struct txq *txq, volatile union mlx5_wqe *wqe, - uintptr_t addr, uint32_t length) -{ - uint32_t size; - uint16_t wqe_cnt = txq->wqe_n - 1; - uint16_t wqe_ci = txq->wqe_ci + 1; - - /* Copy the first 16 bytes into inline header. */ - rte_memcpy((void *)(uintptr_t)wqe->inl.eseg.inline_hdr_start, - (void *)(uintptr_t)addr, - MLX5_ETH_INLINE_HEADER_SIZE); - addr += MLX5_ETH_INLINE_HEADER_SIZE; - length -= MLX5_ETH_INLINE_HEADER_SIZE; - size = 3 + ((4 + length + 15) / 16); - wqe->inl.byte_cnt = htonl(length | MLX5_INLINE_SEG); - rte_memcpy((void *)(uintptr_t)&wqe->inl.data[0], - (void *)addr, MLX5_WQE64_INL_DATA); - addr += MLX5_WQE64_INL_DATA; - length -= MLX5_WQE64_INL_DATA; - while (length) { - volatile union mlx5_wqe *wqe_next = - &(*txq->wqes)[wqe_ci & wqe_cnt]; - uint32_t copy_bytes = (length > sizeof(*wqe)) ? - sizeof(*wqe) : - length; - - rte_mov64((uint8_t *)(uintptr_t)&wqe_next->data[0], - (uint8_t *)addr); - addr += copy_bytes; - length -= copy_bytes; - ++wqe_ci; + wqe->wqe.eseg.rsvd0 = 0; + wqe->wqe.eseg.rsvd1 = 0; + wqe->wqe.eseg.mss = 0; + wqe->wqe.eseg.rsvd2 = 0; + /* Start by copying the Ethernet Header. */ + rte_mov16((uint8_t *)raw, (uint8_t *)addr); + length -= 16; + addr += 16; + /* Replace the Ethernet type by the VLAN if necessary. */ + if (buf->ol_flags & PKT_TX_VLAN_PKT) { + uint32_t vlan = htonl(0x81000000 | buf->vlan_tci); + + memcpy((uint8_t *)(raw + 16 - sizeof(vlan)), + &vlan, sizeof(vlan)); + addr -= sizeof(vlan); + length += sizeof(vlan); } - assert(size < 64); - wqe->inl.ctrl.data[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND); - wqe->inl.ctrl.data[1] = htonl(txq->qp_num_8s | size); - wqe->inl.ctrl.data[2] = 0; - wqe->inl.ctrl.data[3] = 0; - wqe->inl.eseg.rsvd0 = 0; - wqe->inl.eseg.rsvd1 = 0; - wqe->inl.eseg.mss = 0; - wqe->inl.eseg.rsvd2 = 0; - wqe->inl.eseg.inline_hdr_sz = htons(MLX5_ETH_INLINE_HEADER_SIZE); - /* Increment consumer index. */ - txq->wqe_ci = wqe_ci; -} - -/** - * Write a inline WQE with VLAN. - * - * @param txq - * Pointer to TX queue structure. - * @param wqe - * Pointer to the WQE to fill. - * @param addr - * Buffer data address. - * @param length - * Packet length. - * @param lkey - * Memory region lkey. - * @param vlan_tci - * VLAN field to insert in packet. - */ -static inline void -mlx5_wqe_write_inline_vlan(struct txq *txq, volatile union mlx5_wqe *wqe, - uintptr_t addr, uint32_t length, uint16_t vlan_tci) -{ - uint32_t size; - uint32_t wqe_cnt = txq->wqe_n - 1; - uint16_t wqe_ci = txq->wqe_ci + 1; - uint32_t vlan = htonl(0x81000000 | vlan_tci); - - /* - * Copy 12 bytes of source & destination MAC address. - * Copy 4 bytes of VLAN. - * Copy 2 bytes of Ether type. - */ - rte_memcpy((uint8_t *)(uintptr_t)wqe->inl.eseg.inline_hdr_start, - (uint8_t *)addr, 12); - rte_memcpy((uint8_t *)(uintptr_t)wqe->inl.eseg.inline_hdr_start + 12, - &vlan, sizeof(vlan)); - rte_memcpy((uint8_t *)((uintptr_t)wqe->inl.eseg.inline_hdr_start + 16), - (uint8_t *)(addr + 12), 2); - addr += MLX5_ETH_VLAN_INLINE_HEADER_SIZE - sizeof(vlan); - length -= MLX5_ETH_VLAN_INLINE_HEADER_SIZE - sizeof(vlan); - size = (sizeof(wqe->inl.ctrl.ctrl) + - sizeof(wqe->inl.eseg) + - sizeof(wqe->inl.byte_cnt) + - length + 15) / 16; - wqe->inl.byte_cnt = htonl(length | MLX5_INLINE_SEG); - rte_memcpy((void *)(uintptr_t)&wqe->inl.data[0], - (void *)addr, MLX5_WQE64_INL_DATA); - addr += MLX5_WQE64_INL_DATA; - length -= MLX5_WQE64_INL_DATA; - while (length) { - volatile union mlx5_wqe *wqe_next = - &(*txq->wqes)[wqe_ci & wqe_cnt]; - uint32_t copy_bytes = (length > sizeof(*wqe)) ? - sizeof(*wqe) : - length; - - rte_mov64((uint8_t *)(uintptr_t)&wqe_next->data[0], - (uint8_t *)addr); - addr += copy_bytes; - length -= copy_bytes; - ++wqe_ci; + /* Inline if enough room. */ + if (txq->max_inline != 0) { + uintptr_t end = (uintptr_t)&(*txq->wqes)[txq->wqe_n]; + uint16_t max_inline = txq->max_inline * RTE_CACHE_LINE_SIZE; + uint16_t room; + + raw += 16; + room = end - (uintptr_t)raw; + if (room > max_inline) { + uintptr_t addr_end = (addr + max_inline) & + ~(RTE_CACHE_LINE_SIZE - 1); + uint16_t copy_b = ((addr_end - addr) > length) ? + length : + (addr_end - addr); + + rte_memcpy((void *)raw, (void *)addr, copy_b); + addr += copy_b; + length -= copy_b; + pkt_inline_sz += copy_b; + /* Sanity check. */ + assert(addr <= addr_end); + } + /* Store the inlined packet size in the WQE. */ + wqe->wqe.eseg.inline_hdr_sz = htons(pkt_inline_sz); + /* + * 2 DWORDs consumed by the WQE header + 1 DSEG + + * the size of the inline part of the packet. + */ + ds = 2 + ((pkt_inline_sz - 2 + 15) / 16); + if (length > 0) { + dseg = (struct mlx5_wqe_data_seg *) + ((uintptr_t)wqe + (ds * 16)); + if ((uintptr_t)dseg >= end) + dseg = (struct mlx5_wqe_data_seg *) + ((uintptr_t)&(*txq->wqes)[0]); + goto use_dseg; + } + } else { + /* Add the remaining packet as a simple ds. */ + ds = 3; + /* + * No inline has been done in the packet, only the Ethernet + * Header as been stored. + */ + wqe->wqe.eseg.inline_hdr_sz = htons(16); + dseg = (struct mlx5_wqe_data_seg *) + ((uintptr_t)wqe + (ds * 16)); +use_dseg: + *dseg = (struct mlx5_wqe_data_seg) { + .addr = htonll(addr), + .byte_count = htonl(length), + .lkey = txq_mp2mr(txq, txq_mb2mp(buf)), + }; + ++ds; } - assert(size < 64); - wqe->inl.ctrl.data[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND); - wqe->inl.ctrl.data[1] = htonl(txq->qp_num_8s | size); - wqe->inl.ctrl.data[2] = 0; - wqe->inl.ctrl.data[3] = 0; - wqe->inl.eseg.rsvd0 = 0; - wqe->inl.eseg.rsvd1 = 0; - wqe->inl.eseg.mss = 0; - wqe->inl.eseg.rsvd2 = 0; - wqe->inl.eseg.inline_hdr_sz = htons(MLX5_ETH_VLAN_INLINE_HEADER_SIZE); - /* Increment consumer index. */ - txq->wqe_ci = wqe_ci; + wqe->wqe.ctrl.data[1] = htonl(txq->qp_num_8s | ds); + return ds; } /** @@ -609,9 +486,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) do { struct rte_mbuf *buf = *(pkts++); unsigned int elts_head_next; - uintptr_t addr; uint32_t length; - uint32_t lkey; unsigned int segs_n = buf->nb_segs; volatile struct mlx5_wqe_data_seg *dseg; unsigned int ds = sizeof(*wqe) / 16; @@ -627,12 +502,10 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) --pkts_n; elts_head_next = (elts_head + 1) & (elts_n - 1); wqe = &(*txq->wqes)[txq->wqe_ci & (txq->wqe_n - 1)]; - dseg = &wqe->wqe.dseg; - rte_prefetch0(wqe); + tx_prefetch_wqe(txq, txq->wqe_ci); + tx_prefetch_wqe(txq, txq->wqe_ci + 1); if (pkts_n) rte_prefetch0(*pkts); - /* Retrieve buffer information. */ - addr = rte_pktmbuf_mtod(buf, uintptr_t); length = DATA_LEN(buf); /* Update element. */ (*txq->elts)[elts_head] = buf; @@ -640,13 +513,6 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) if (pkts_n) rte_prefetch0(rte_pktmbuf_mtod(*pkts, volatile void *)); - /* Retrieve Memory Region key for this memory pool. */ - lkey = txq_mp2mr(txq, txq_mb2mp(buf)); - if (buf->ol_flags & PKT_TX_VLAN_PKT) - mlx5_wqe_write_vlan(txq, wqe, addr, length, lkey, - buf->vlan_tci); - else - mlx5_wqe_write(txq, wqe, addr, length, lkey); /* Should we enable HW CKSUM offload */ if (buf->ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { @@ -656,6 +522,11 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) } else { wqe->wqe.eseg.cs_flags = 0; } + ds = mlx5_wqe_write(txq, wqe, buf, length); + if (segs_n == 1) + goto skip_segs; + dseg = (volatile struct mlx5_wqe_data_seg *) + (((uintptr_t)wqe) + ds * 16); while (--segs_n) { /* * Spill on next WQE when the current one does not have @@ -686,11 +557,13 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) /* Update DS field in WQE. */ wqe->wqe.ctrl.data[1] &= htonl(0xffffffc0); wqe->wqe.ctrl.data[1] |= htonl(ds & 0x3f); - elts_head = elts_head_next; +skip_segs: #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment sent bytes counter. */ txq->stats.obytes += length; #endif + /* Increment consumer index. */ + txq->wqe_ci += (ds + 3) / 4; elts_head = elts_head_next; ++i; } while (pkts_n); @@ -719,166 +592,6 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) } /** - * DPDK callback for TX with inline support. - * - * @param dpdk_txq - * Generic pointer to TX queue structure. - * @param[in] pkts - * Packets to transmit. - * @param pkts_n - * Number of packets in array. - * - * @return - * Number of packets successfully transmitted (<= pkts_n). - */ -uint16_t -mlx5_tx_burst_inline(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) -{ - struct txq *txq = (struct txq *)dpdk_txq; - uint16_t elts_head = txq->elts_head; - const unsigned int elts_n = txq->elts_n; - unsigned int i = 0; - unsigned int j = 0; - unsigned int max; - unsigned int comp; - volatile union mlx5_wqe *wqe = NULL; - unsigned int max_inline = txq->max_inline; - - if (unlikely(!pkts_n)) - return 0; - /* Prefetch first packet cacheline. */ - tx_prefetch_cqe(txq, txq->cq_ci); - tx_prefetch_cqe(txq, txq->cq_ci + 1); - rte_prefetch0(*pkts); - /* Start processing. */ - txq_complete(txq); - max = (elts_n - (elts_head - txq->elts_tail)); - if (max > elts_n) - max -= elts_n; - do { - struct rte_mbuf *buf = *(pkts++); - unsigned int elts_head_next; - uintptr_t addr; - uint32_t length; - uint32_t lkey; - unsigned int segs_n = buf->nb_segs; - volatile struct mlx5_wqe_data_seg *dseg; - unsigned int ds = sizeof(*wqe) / 16; - - /* - * Make sure there is enough room to store this packet and - * that one ring entry remains unused. - */ - assert(segs_n); - if (max < segs_n + 1) - break; - max -= segs_n; - --pkts_n; - elts_head_next = (elts_head + 1) & (elts_n - 1); - wqe = &(*txq->wqes)[txq->wqe_ci & (txq->wqe_n - 1)]; - dseg = &wqe->wqe.dseg; - tx_prefetch_wqe(txq, txq->wqe_ci); - tx_prefetch_wqe(txq, txq->wqe_ci + 1); - if (pkts_n) - rte_prefetch0(*pkts); - /* Should we enable HW CKSUM offload */ - if (buf->ol_flags & - (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { - wqe->inl.eseg.cs_flags = - MLX5_ETH_WQE_L3_CSUM | - MLX5_ETH_WQE_L4_CSUM; - } else { - wqe->inl.eseg.cs_flags = 0; - } - /* Retrieve buffer information. */ - addr = rte_pktmbuf_mtod(buf, uintptr_t); - length = DATA_LEN(buf); - /* Update element. */ - (*txq->elts)[elts_head] = buf; - /* Prefetch next buffer data. */ - if (pkts_n) - rte_prefetch0(rte_pktmbuf_mtod(*pkts, - volatile void *)); - if ((length <= max_inline) && (segs_n == 1)) { - if (buf->ol_flags & PKT_TX_VLAN_PKT) - mlx5_wqe_write_inline_vlan(txq, wqe, - addr, length, - buf->vlan_tci); - else - mlx5_wqe_write_inline(txq, wqe, addr, length); - goto skip_segs; - } else { - /* Retrieve Memory Region key for this memory pool. */ - lkey = txq_mp2mr(txq, txq_mb2mp(buf)); - if (buf->ol_flags & PKT_TX_VLAN_PKT) - mlx5_wqe_write_vlan(txq, wqe, addr, length, - lkey, buf->vlan_tci); - else - mlx5_wqe_write(txq, wqe, addr, length, lkey); - } - while (--segs_n) { - /* - * Spill on next WQE when the current one does not have - * enough room left. Size of WQE must a be a multiple - * of data segment size. - */ - assert(!(sizeof(*wqe) % sizeof(*dseg))); - if (!(ds % (sizeof(*wqe) / 16))) - dseg = (volatile void *) - &(*txq->wqes)[txq->wqe_ci++ & - (txq->wqe_n - 1)]; - else - ++dseg; - ++ds; - buf = buf->next; - assert(buf); - /* Store segment information. */ - dseg->byte_count = htonl(DATA_LEN(buf)); - dseg->lkey = txq_mp2mr(txq, txq_mb2mp(buf)); - dseg->addr = htonll(rte_pktmbuf_mtod(buf, uintptr_t)); - (*txq->elts)[elts_head_next] = buf; - elts_head_next = (elts_head_next + 1) & (elts_n - 1); -#ifdef MLX5_PMD_SOFT_COUNTERS - length += DATA_LEN(buf); -#endif - ++j; - } - /* Update DS field in WQE. */ - wqe->inl.ctrl.data[1] &= htonl(0xffffffc0); - wqe->inl.ctrl.data[1] |= htonl(ds & 0x3f); -skip_segs: - elts_head = elts_head_next; -#ifdef MLX5_PMD_SOFT_COUNTERS - /* Increment sent bytes counter. */ - txq->stats.obytes += length; -#endif - ++i; - } while (pkts_n); - /* Take a shortcut if nothing must be sent. */ - if (unlikely(i == 0)) - return 0; - /* Check whether completion threshold has been reached. */ - comp = txq->elts_comp + i + j; - if (comp >= MLX5_TX_COMP_THRESH) { - /* Request completion on last WQE. */ - wqe->inl.ctrl.data[2] = htonl(8); - /* Save elts_head in unused "immediate" field of WQE. */ - wqe->inl.ctrl.data[3] = elts_head; - txq->elts_comp = 0; - } else { - txq->elts_comp = comp; - } -#ifdef MLX5_PMD_SOFT_COUNTERS - /* Increment sent packets counter. */ - txq->stats.opackets += i; -#endif - /* Ring QP doorbell. */ - mlx5_tx_dbrec(txq); - txq->elts_head = elts_head; - return i; -} - -/** * Open a MPW session. * * @param txq @@ -908,7 +621,7 @@ mlx5_mpw_new(struct txq *txq, struct mlx5_mpw *mpw, uint32_t length) mpw->wqe->mpw.eseg.rsvd2 = 0; mpw->wqe->mpw.ctrl.data[0] = htonl((MLX5_OPC_MOD_MPW << 24) | (txq->wqe_ci << 8) | - MLX5_OPCODE_LSO_MPW); + MLX5_OPCODE_TSO); mpw->wqe->mpw.ctrl.data[2] = 0; mpw->wqe->mpw.ctrl.data[3] = 0; mpw->data.dseg[0] = &mpw->wqe->mpw.dseg[0]; @@ -1107,7 +820,7 @@ mlx5_mpw_inline_new(struct txq *txq, struct mlx5_mpw *mpw, uint32_t length) mpw->wqe = &(*txq->wqes)[idx]; mpw->wqe->mpw_inl.ctrl.data[0] = htonl((MLX5_OPC_MOD_MPW << 24) | (txq->wqe_ci << 8) | - MLX5_OPCODE_LSO_MPW); + MLX5_OPCODE_TSO); mpw->wqe->mpw_inl.ctrl.data[2] = 0; mpw->wqe->mpw_inl.ctrl.data[3] = 0; mpw->wqe->mpw_inl.eseg.mss = htons(length); @@ -1168,7 +881,7 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, unsigned int j = 0; unsigned int max; unsigned int comp; - unsigned int inline_room = txq->max_inline; + unsigned int inline_room = txq->max_inline * RTE_CACHE_LINE_SIZE; struct mlx5_mpw mpw = { .state = MLX5_MPW_STATE_CLOSED, }; @@ -1222,7 +935,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, (length > inline_room) || (mpw.wqe->mpw_inl.eseg.cs_flags != cs_flags)) { mlx5_mpw_inline_close(txq, &mpw); - inline_room = txq->max_inline; + inline_room = + txq->max_inline * RTE_CACHE_LINE_SIZE; } } if (mpw.state == MLX5_MPW_STATE_CLOSED) { @@ -1238,7 +952,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, /* Multi-segment packets must be alone in their MPW. */ assert((segs_n == 1) || (mpw.pkts_n == 0)); if (mpw.state == MLX5_MPW_STATE_OPENED) { - assert(inline_room == txq->max_inline); + assert(inline_room == + txq->max_inline * RTE_CACHE_LINE_SIZE); #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG) length = 0; #endif @@ -1303,7 +1018,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, ++j; if (mpw.pkts_n == MLX5_MPW_DSEG_MAX) { mlx5_mpw_inline_close(txq, &mpw); - inline_room = txq->max_inline; + inline_room = + txq->max_inline * RTE_CACHE_LINE_SIZE; } else { inline_room -= length; } @@ -1365,19 +1081,19 @@ rxq_cq_to_pkt_type(volatile struct mlx5_cqe64 *cqe) uint8_t flags = cqe->l4_hdr_type_etc; uint8_t info = cqe->rsvd0[0]; - if (info & IBV_EXP_CQ_RX_TUNNEL_PACKET) + if (info & MLX5_CQE_RX_TUNNEL_PACKET) pkt_type = TRANSPOSE(flags, - IBV_EXP_CQ_RX_OUTER_IPV4_PACKET, + MLX5_CQE_RX_OUTER_IPV4_PACKET, RTE_PTYPE_L3_IPV4) | TRANSPOSE(flags, - IBV_EXP_CQ_RX_OUTER_IPV6_PACKET, + MLX5_CQE_RX_OUTER_IPV6_PACKET, RTE_PTYPE_L3_IPV6) | TRANSPOSE(flags, - IBV_EXP_CQ_RX_IPV4_PACKET, + MLX5_CQE_RX_IPV4_PACKET, RTE_PTYPE_INNER_L3_IPV4) | TRANSPOSE(flags, - IBV_EXP_CQ_RX_IPV6_PACKET, + MLX5_CQE_RX_IPV6_PACKET, RTE_PTYPE_INNER_L3_IPV6); else pkt_type = @@ -1520,13 +1236,13 @@ rxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe64 *cqe) * of PKT_RX_EIP_CKSUM_BAD because the latter is not functional * (its value is 0). */ - if ((info & IBV_EXP_CQ_RX_TUNNEL_PACKET) && (rxq->csum_l2tun)) + if ((info & MLX5_CQE_RX_TUNNEL_PACKET) && (rxq->csum_l2tun)) ol_flags |= TRANSPOSE(~cqe->l4_hdr_type_etc, - IBV_EXP_CQ_RX_OUTER_IP_CSUM_OK, + MLX5_CQE_RX_OUTER_IP_CSUM_OK, PKT_RX_IP_CKSUM_BAD) | TRANSPOSE(~cqe->l4_hdr_type_etc, - IBV_EXP_CQ_RX_OUTER_TCP_UDP_CSUM_OK, + MLX5_CQE_RX_OUTER_TCP_UDP_CSUM_OK, PKT_RX_L4_CKSUM_BAD); return ol_flags; } @@ -1572,6 +1288,14 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) rte_prefetch0(wqe); rep = rte_mbuf_raw_alloc(rxq->mp); if (unlikely(rep == NULL)) { + ++rxq->stats.rx_nombuf; + if (!pkt) { + /* + * no buffers before we even started, + * bail out silently. + */ + break; + } while (pkt != seg) { assert(pkt != (*rxq->elts)[idx]); seg = NEXT(pkt); @@ -1579,7 +1303,6 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) __rte_mbuf_raw_free(pkt); pkt = seg; } - ++rxq->stats.rx_nombuf; break; } if (!pkt) { diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index f6e2cbac..05779ef5 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -40,22 +40,22 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #include <infiniband/mlx5_hw.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_mbuf.h> #include <rte_mempool.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5_utils.h" @@ -87,6 +87,8 @@ struct mlx5_txq_stats { struct fdir_queue { struct ibv_qp *qp; /* Associated RX QP. */ struct ibv_exp_rwq_ind_table *ind_table; /* Indirection table. */ + struct ibv_exp_wq *wq; /* Work queue. */ + struct ibv_cq *cq; /* Completion queue. */ }; struct priv; @@ -128,7 +130,7 @@ struct rxq_ctrl { struct ibv_cq *cq; /* Completion Queue. */ struct ibv_exp_wq *wq; /* Work Queue. */ struct ibv_exp_res_domain *rd; /* Resource Domain. */ - struct fdir_queue fdir_queue; /* Flow director queue. */ + struct fdir_queue *fdir_queue; /* Flow director queue. */ struct ibv_mr *mr; /* Memory Region (for mp). */ struct ibv_exp_wq_family *if_wq; /* WQ burst interface. */ struct ibv_exp_cq_family_v1 *if_cq; /* CQ interface. */ @@ -247,7 +249,7 @@ struct txq { uint16_t wqe_n; /* Number of WQ elements. */ uint16_t bf_offset; /* Blueflame offset. */ uint16_t bf_buf_size; /* Blueflame size. */ - uint16_t max_inline; /* Maximum size to inline in a WQE. */ + uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */ uint32_t qp_num_8s; /* QP number shifted by 8. */ volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */ volatile union mlx5_wqe (*wqes)[]; /* Work queue. */ @@ -312,7 +314,6 @@ uint16_t mlx5_tx_burst_secondary_setup(void *, struct rte_mbuf **, uint16_t); /* mlx5_rxtx.c */ uint16_t mlx5_tx_burst(void *, struct rte_mbuf **, uint16_t); -uint16_t mlx5_tx_burst_inline(void *, struct rte_mbuf **, uint16_t); uint16_t mlx5_tx_burst_mpw(void *, struct rte_mbuf **, uint16_t); uint16_t mlx5_tx_burst_mpw_inline(void *, struct rte_mbuf **, uint16_t); uint16_t mlx5_rx_burst(void *, struct rte_mbuf **, uint16_t); diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c index 2d3cb519..f2b5781a 100644 --- a/drivers/net/mlx5/mlx5_stats.c +++ b/drivers/net/mlx5/mlx5_stats.c @@ -33,11 +33,11 @@ /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ethdev.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c index e9b9a293..d4dccd88 100644 --- a/drivers/net/mlx5/mlx5_trigger.c +++ b/drivers/net/mlx5/mlx5_trigger.c @@ -33,14 +33,14 @@ /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ether.h> #include <rte_ethdev.h> #include <rte_interrupts.h> #include <rte_alarm.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index 6fe61c4a..e4510efe 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -40,23 +40,23 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_mbuf.h> #include <rte_malloc.h> #include <rte_ethdev.h> #include <rte_common.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5_utils.h" @@ -338,9 +338,12 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), }; - if (priv->txq_inline && priv->txqs_n >= priv->txqs_inline) { - tmpl.txq.max_inline = priv->txq_inline; - attr.init.cap.max_inline_data = tmpl.txq.max_inline; + if (priv->txq_inline && (priv->txqs_n >= priv->txqs_inline)) { + tmpl.txq.max_inline = + ((priv->txq_inline + (RTE_CACHE_LINE_SIZE - 1)) / + RTE_CACHE_LINE_SIZE); + attr.init.cap.max_inline_data = + tmpl.txq.max_inline * RTE_CACHE_LINE_SIZE; } tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init); if (tmpl.qp == NULL) { diff --git a/drivers/net/mlx5/mlx5_vlan.c b/drivers/net/mlx5/mlx5_vlan.c index 4719e697..1b0fa40a 100644 --- a/drivers/net/mlx5/mlx5_vlan.c +++ b/drivers/net/mlx5/mlx5_vlan.c @@ -38,12 +38,12 @@ /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ethdev.h> #include <rte_common.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5_utils.h" @@ -87,7 +87,8 @@ vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) --priv->vlan_filter_n; memmove(&priv->vlan_filter[i], &priv->vlan_filter[i + 1], - priv->vlan_filter_n - i); + sizeof(priv->vlan_filter[i]) * + (priv->vlan_filter_n - i)); priv->vlan_filter[priv->vlan_filter_n] = 0; } else { assert(i == priv->vlan_filter_n); diff --git a/drivers/net/nfp/nfp_net.c b/drivers/net/nfp/nfp_net.c index 82e3e4e1..815296cb 100644 --- a/drivers/net/nfp/nfp_net.c +++ b/drivers/net/nfp/nfp_net.c @@ -2417,8 +2417,8 @@ nfp_net_init(struct rte_eth_dev *eth_dev) eth_random_addr(&hw->mac_addr[0]); /* Copying mac address to DPDK eth_dev struct */ - ether_addr_copy(ð_dev->data->mac_addrs[0], - (struct ether_addr *)hw->mac_addr); + ether_addr_copy((struct ether_addr *)hw->mac_addr, + ð_dev->data->mac_addrs[0]); PMD_INIT_LOG(INFO, "port %d VendorID=0x%x DeviceID=0x%x " "mac=%02x:%02x:%02x:%02x:%02x:%02x", diff --git a/drivers/net/pcap/rte_eth_pcap.c b/drivers/net/pcap/rte_eth_pcap.c index 7e213ebb..7b7126bf 100644 --- a/drivers/net/pcap/rte_eth_pcap.c +++ b/drivers/net/pcap/rte_eth_pcap.c @@ -229,8 +229,10 @@ eth_pcap_rx(void *queue, if (unlikely(eth_pcap_rx_jumbo(pcap_q->mb_pool, mbuf, packet, - header.caplen) == -1)) + header.caplen) == -1)) { + rte_pktmbuf_free(mbuf); break; + } } mbuf->pkt_len = (uint16_t)header.caplen; diff --git a/drivers/net/qede/Makefile b/drivers/net/qede/Makefile index fe449aa9..7965a831 100644 --- a/drivers/net/qede/Makefile +++ b/drivers/net/qede/Makefile @@ -48,9 +48,13 @@ endif endif ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +ifeq ($(shell gcc -Wno-unused-but-set-variable -Werror -E - < /dev/null > /dev/null 2>&1; echo $$?),0) CFLAGS_BASE_DRIVER += -Wno-unused-but-set-variable +endif CFLAGS_BASE_DRIVER += -Wno-missing-declarations +ifeq ($(shell gcc -Wno-maybe-uninitialized -Werror -E - < /dev/null > /dev/null 2>&1; echo $$?),0) CFLAGS_BASE_DRIVER += -Wno-maybe-uninitialized +endif CFLAGS_BASE_DRIVER += -Wno-strict-prototypes ifeq ($(shell test $(GCC_VERSION) -ge 60 && echo 1), 1) CFLAGS_BASE_DRIVER += -Wno-shift-negative-value diff --git a/drivers/net/ring/rte_eth_ring.c b/drivers/net/ring/rte_eth_ring.c index a7048c77..5551e187 100644 --- a/drivers/net/ring/rte_eth_ring.c +++ b/drivers/net/ring/rte_eth_ring.c @@ -557,7 +557,7 @@ rte_pmd_ring_devinit(const char *name, const char *params) goto out_free; for (info->count = 0; info->count < info->total; info->count++) { - ret = eth_dev_ring_create(name, + ret = eth_dev_ring_create(info->list[info->count].name, info->list[info->count].node, info->list[info->count].action); if ((ret == -1) && diff --git a/drivers/net/thunderx/nicvf_rxtx.c b/drivers/net/thunderx/nicvf_rxtx.c index eb51a72c..e15c7303 100644 --- a/drivers/net/thunderx/nicvf_rxtx.c +++ b/drivers/net/thunderx/nicvf_rxtx.c @@ -70,19 +70,20 @@ fill_sq_desc_header(union sq_entry_t *entry, struct rte_mbuf *pkt) ol_flags = pkt->ol_flags & NICVF_TX_OFFLOAD_MASK; if (unlikely(ol_flags)) { /* L4 cksum */ - if (ol_flags & PKT_TX_TCP_CKSUM) + uint64_t l4_flags = ol_flags & PKT_TX_L4_MASK; + if (l4_flags == PKT_TX_TCP_CKSUM) sqe.hdr.csum_l4 = SEND_L4_CSUM_TCP; - else if (ol_flags & PKT_TX_UDP_CKSUM) + else if (l4_flags == PKT_TX_UDP_CKSUM) sqe.hdr.csum_l4 = SEND_L4_CSUM_UDP; else sqe.hdr.csum_l4 = SEND_L4_CSUM_DISABLE; + + sqe.hdr.l3_offset = pkt->l2_len; sqe.hdr.l4_offset = pkt->l3_len + pkt->l2_len; /* L3 cksum */ - if (ol_flags & PKT_TX_IP_CKSUM) { + if (ol_flags & PKT_TX_IP_CKSUM) sqe.hdr.csum_l3 = 1; - sqe.hdr.l3_offset = pkt->l2_len; - } } entry->buff[0] = sqe.buff[0]; diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c index 07d64497..86cf8a38 100644 --- a/drivers/net/virtio/virtio_ethdev.c +++ b/drivers/net/virtio/virtio_ethdev.c @@ -125,8 +125,8 @@ static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings[] = { {"size_128_255_packets", offsetof(struct virtnet_rx, stats.size_bins[3])}, {"size_256_511_packets", offsetof(struct virtnet_rx, stats.size_bins[4])}, {"size_512_1023_packets", offsetof(struct virtnet_rx, stats.size_bins[5])}, - {"size_1024_1517_packets", offsetof(struct virtnet_rx, stats.size_bins[6])}, - {"size_1518_max_packets", offsetof(struct virtnet_rx, stats.size_bins[7])}, + {"size_1024_1518_packets", offsetof(struct virtnet_rx, stats.size_bins[6])}, + {"size_1519_max_packets", offsetof(struct virtnet_rx, stats.size_bins[7])}, }; /* [rt]x_qX_ is prepended to the name string here */ @@ -142,8 +142,8 @@ static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = { {"size_128_255_packets", offsetof(struct virtnet_tx, stats.size_bins[3])}, {"size_256_511_packets", offsetof(struct virtnet_tx, stats.size_bins[4])}, {"size_512_1023_packets", offsetof(struct virtnet_tx, stats.size_bins[5])}, - {"size_1024_1517_packets", offsetof(struct virtnet_tx, stats.size_bins[6])}, - {"size_1518_max_packets", offsetof(struct virtnet_tx, stats.size_bins[7])}, + {"size_1024_1518_packets", offsetof(struct virtnet_tx, stats.size_bins[6])}, + {"size_1519_max_packets", offsetof(struct virtnet_tx, stats.size_bins[7])}, }; #define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \ @@ -549,13 +549,11 @@ virtio_dev_close(struct rte_eth_dev *dev) PMD_INIT_LOG(DEBUG, "virtio_dev_close"); - if (hw->started == 1) - virtio_dev_stop(dev); - /* reset the NIC */ if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) vtpci_irq_config(hw, VIRTIO_MSI_NO_VECTOR); vtpci_reset(hw); + hw->started = 0; virtio_dev_free_mbufs(dev); virtio_free_queues(dev); } @@ -1275,9 +1273,10 @@ eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev) if (rte_eal_process_type() == RTE_PROC_SECONDARY) return -EPERM; - /* Close it anyway since there's no way to know if closed */ - virtio_dev_close(eth_dev); - + if (hw->started == 1) { + virtio_dev_stop(eth_dev); + virtio_dev_close(eth_dev); + } pci_dev = eth_dev->pci_dev; eth_dev->dev_ops = NULL; @@ -1486,12 +1485,9 @@ static void virtio_dev_stop(struct rte_eth_dev *dev) { struct rte_eth_link link; - struct virtio_hw *hw = dev->data->dev_private; PMD_INIT_LOG(DEBUG, "stop"); - hw->started = 0; - if (dev->data->dev_conf.intr_conf.lsc) rte_intr_disable(&dev->pci_dev->intr_handle); diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c b/drivers/net/virtio/virtio_user/virtio_user_dev.c index 376c9cf5..e239e0eb 100644 --- a/drivers/net/virtio/virtio_user/virtio_user_dev.c +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c @@ -45,20 +45,14 @@ #include "../virtio_ethdev.h" static int -virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) +virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel) { - int callfd, kickfd; + /* Of all per virtqueue MSGs, make sure VHOST_SET_VRING_CALL come + * firstly because vhost depends on this msg to allocate virtqueue + * pair. + */ + int callfd; struct vhost_vring_file file; - struct vhost_vring_state state; - struct vring *vring = &dev->vrings[queue_sel]; - struct vhost_vring_addr addr = { - .index = queue_sel, - .desc_user_addr = (uint64_t)(uintptr_t)vring->desc, - .avail_user_addr = (uint64_t)(uintptr_t)vring->avail, - .used_user_addr = (uint64_t)(uintptr_t)vring->used, - .log_guest_addr = 0, - .flags = 0, /* disable log */ - }; /* May use invalid flag, but some backend leverages kickfd and callfd as * criteria to judge if dev is alive. so finally we use real event_fd. @@ -68,22 +62,30 @@ virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) PMD_DRV_LOG(ERR, "callfd error, %s\n", strerror(errno)); return -1; } - kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); - if (kickfd < 0) { - close(callfd); - PMD_DRV_LOG(ERR, "kickfd error, %s\n", strerror(errno)); - return -1; - } - - /* Of all per virtqueue MSGs, make sure VHOST_SET_VRING_CALL come - * firstly because vhost depends on this msg to allocate virtqueue - * pair. - */ file.index = queue_sel; file.fd = callfd; vhost_user_sock(dev->vhostfd, VHOST_USER_SET_VRING_CALL, &file); dev->callfds[queue_sel] = callfd; + return 0; +} + +static int +virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) +{ + int kickfd; + struct vhost_vring_file file; + struct vhost_vring_state state; + struct vring *vring = &dev->vrings[queue_sel]; + struct vhost_vring_addr addr = { + .index = queue_sel, + .desc_user_addr = (uint64_t)(uintptr_t)vring->desc, + .avail_user_addr = (uint64_t)(uintptr_t)vring->avail, + .used_user_addr = (uint64_t)(uintptr_t)vring->used, + .log_guest_addr = 0, + .flags = 0, /* disable log */ + }; + state.index = queue_sel; state.num = vring->num; vhost_user_sock(dev->vhostfd, VHOST_USER_SET_VRING_NUM, &state); @@ -97,6 +99,12 @@ virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) * lastly because vhost depends on this msg to judge if * virtio is ready. */ + kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + if (kickfd < 0) { + PMD_DRV_LOG(ERR, "kickfd error, %s\n", strerror(errno)); + return -1; + } + file.index = queue_sel; file.fd = kickfd; vhost_user_sock(dev->vhostfd, VHOST_USER_SET_VRING_KICK, &file); dev->kickfds[queue_sel] = kickfd; @@ -104,37 +112,43 @@ virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) return 0; } -int -virtio_user_start_device(struct virtio_user_dev *dev) +static int +virtio_user_queue_setup(struct virtio_user_dev *dev, + int (*fn)(struct virtio_user_dev *, uint32_t)) { - uint64_t features; uint32_t i, queue_sel; - int ret; - - /* construct memory region inside each implementation */ - ret = vhost_user_sock(dev->vhostfd, VHOST_USER_SET_MEM_TABLE, NULL); - if (ret < 0) - goto error; for (i = 0; i < dev->max_queue_pairs; ++i) { queue_sel = 2 * i + VTNET_SQ_RQ_QUEUE_IDX; - if (virtio_user_kick_queue(dev, queue_sel) < 0) { - PMD_DRV_LOG(INFO, "kick rx vq fails: %u", i); - goto error; + if (fn(dev, queue_sel) < 0) { + PMD_DRV_LOG(INFO, "setup rx vq fails: %u", i); + return -1; } } for (i = 0; i < dev->max_queue_pairs; ++i) { queue_sel = 2 * i + VTNET_SQ_TQ_QUEUE_IDX; - if (virtio_user_kick_queue(dev, queue_sel) < 0) { - PMD_DRV_LOG(INFO, "kick tx vq fails: %u", i); - goto error; + if (fn(dev, queue_sel) < 0) { + PMD_DRV_LOG(INFO, "setup tx vq fails: %u", i); + return -1; } } - /* After setup all virtqueues, we need to set_features so that these - * features can be set into each virtqueue in vhost side. And before - * that, make sure VHOST_USER_F_PROTOCOL_FEATURES is added if mq is - * enabled, and VIRTIO_NET_F_MAC is stripped. + return 0; +} + +int +virtio_user_start_device(struct virtio_user_dev *dev) +{ + uint64_t features; + int ret; + + /* Step 0: tell vhost to create queues */ + if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0) + goto error; + + /* Step 1: set features + * Make sure VHOST_USER_F_PROTOCOL_FEATURES is added if mq is enabled, + * and VIRTIO_NET_F_MAC is stripped. */ features = dev->features; if (dev->max_queue_pairs > 1) @@ -145,6 +159,20 @@ virtio_user_start_device(struct virtio_user_dev *dev) goto error; PMD_DRV_LOG(INFO, "set features: %" PRIx64, features); + /* Step 2: share memory regions */ + ret = vhost_user_sock(dev->vhostfd, VHOST_USER_SET_MEM_TABLE, NULL); + if (ret < 0) + goto error; + + /* Step 3: kick queues */ + if (virtio_user_queue_setup(dev, virtio_user_kick_queue) < 0) + goto error; + + /* Step 4: enable queues + * we enable the 1st queue pair by default. + */ + vhost_user_enable_queue_pair(dev->vhostfd, 0, 1); + return 0; error: /* TODO: free resource here or caller to check */ diff --git a/drivers/net/virtio/virtio_user_ethdev.c b/drivers/net/virtio/virtio_user_ethdev.c index daef09bd..bba74028 100644 --- a/drivers/net/virtio/virtio_user_ethdev.c +++ b/drivers/net/virtio/virtio_user_ethdev.c @@ -313,6 +313,17 @@ virtio_user_eth_dev_alloc(const char *name) return eth_dev; } +static void +virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev) +{ + struct rte_eth_dev_data *data = eth_dev->data; + struct virtio_hw *hw = data->dev_private; + + rte_free(hw->virtio_user_dev); + rte_free(hw); + rte_eth_dev_release_port(eth_dev); +} + /* Dev initialization routine. Invoked once for each virtio vdev at * EAL init time, see rte_eal_dev_init(). * Returns 0 on success. @@ -343,9 +354,8 @@ virtio_user_pmd_devinit(const char *name, const char *params) } if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PATH) == 1) { - ret = rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PATH, - &get_string_arg, &path); - if (ret < 0) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PATH, + &get_string_arg, &path) < 0) { PMD_INIT_LOG(ERR, "error to parse %s", VIRTIO_USER_ARG_PATH); goto end; @@ -357,9 +367,8 @@ virtio_user_pmd_devinit(const char *name, const char *params) } if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MAC) == 1) { - ret = rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MAC, - &get_string_arg, &mac_addr); - if (ret < 0) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MAC, + &get_string_arg, &mac_addr) < 0) { PMD_INIT_LOG(ERR, "error to parse %s", VIRTIO_USER_ARG_MAC); goto end; @@ -367,9 +376,8 @@ virtio_user_pmd_devinit(const char *name, const char *params) } if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE) == 1) { - ret = rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE, - &get_integer_arg, &queue_size); - if (ret < 0) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE, + &get_integer_arg, &queue_size) < 0) { PMD_INIT_LOG(ERR, "error to parse %s", VIRTIO_USER_ARG_QUEUE_SIZE); goto end; @@ -377,9 +385,8 @@ virtio_user_pmd_devinit(const char *name, const char *params) } if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUES_NUM) == 1) { - ret = rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUES_NUM, - &get_integer_arg, &queues); - if (ret < 0) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUES_NUM, + &get_integer_arg, &queues) < 0) { PMD_INIT_LOG(ERR, "error to parse %s", VIRTIO_USER_ARG_QUEUES_NUM); goto end; @@ -387,9 +394,8 @@ virtio_user_pmd_devinit(const char *name, const char *params) } if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) { - ret = rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM, - &get_integer_arg, &cq); - if (ret < 0) { + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM, + &get_integer_arg, &cq) < 0) { PMD_INIT_LOG(ERR, "error to parse %s", VIRTIO_USER_ARG_CQ_NUM); goto end; @@ -411,12 +417,16 @@ virtio_user_pmd_devinit(const char *name, const char *params) hw = eth_dev->data->dev_private; if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq, - queue_size, mac_addr) < 0) + queue_size, mac_addr) < 0) { + PMD_INIT_LOG(ERR, "virtio_user_dev_init fails"); + virtio_user_eth_dev_free(eth_dev); goto end; + } /* previously called by rte_eal_pci_probe() for physical dev */ if (eth_virtio_dev_init(eth_dev) < 0) { PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails"); + virtio_user_eth_dev_free(eth_dev); goto end; } ret = 0; diff --git a/drivers/net/vmxnet3/vmxnet3_rxtx.c b/drivers/net/vmxnet3/vmxnet3_rxtx.c index 9deeb3ff..88df576c 100644 --- a/drivers/net/vmxnet3/vmxnet3_rxtx.c +++ b/drivers/net/vmxnet3/vmxnet3_rxtx.c @@ -141,10 +141,10 @@ vmxnet3_txq_dump(struct vmxnet3_tx_queue *txq) #endif static void -vmxnet3_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring) +vmxnet3_tx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring) { while (ring->next2comp != ring->next2fill) { - /* No need to worry about tx desc ownership, device is quiesced by now. */ + /* No need to worry about desc ownership, device is quiesced by now. */ vmxnet3_buf_info_t *buf_info = ring->buf_info + ring->next2comp; if (buf_info->m) { @@ -158,9 +158,27 @@ vmxnet3_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring) } static void +vmxnet3_rx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring) +{ + uint32_t i; + + for (i = 0; i < ring->size; i++) { + /* No need to worry about desc ownership, device is quiesced by now. */ + vmxnet3_buf_info_t *buf_info = &ring->buf_info[i]; + + if (buf_info->m) { + rte_pktmbuf_free_seg(buf_info->m); + buf_info->m = NULL; + buf_info->bufPA = 0; + buf_info->len = 0; + } + vmxnet3_cmd_ring_adv_next2comp(ring); + } +} + +static void vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring) { - vmxnet3_cmd_ring_release_mbufs(ring); rte_free(ring->buf_info); ring->buf_info = NULL; } @@ -172,6 +190,8 @@ vmxnet3_dev_tx_queue_release(void *txq) vmxnet3_tx_queue_t *tq = txq; if (tq != NULL) { + /* Release mbufs */ + vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring); /* Release the cmd_ring */ vmxnet3_cmd_ring_release(&tq->cmd_ring); } @@ -184,6 +204,10 @@ vmxnet3_dev_rx_queue_release(void *rxq) vmxnet3_rx_queue_t *rq = rxq; if (rq != NULL) { + /* Release mbufs */ + for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) + vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]); + /* Release both the cmd_rings */ for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) vmxnet3_cmd_ring_release(&rq->cmd_ring[i]); @@ -201,7 +225,7 @@ vmxnet3_dev_tx_queue_reset(void *txq) if (tq != NULL) { /* Release the cmd_ring mbufs */ - vmxnet3_cmd_ring_release_mbufs(&tq->cmd_ring); + vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring); } /* Tx vmxnet rings structure initialization*/ @@ -230,7 +254,7 @@ vmxnet3_dev_rx_queue_reset(void *rxq) if (rq != NULL) { /* Release both the cmd_rings mbufs */ for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) - vmxnet3_cmd_ring_release_mbufs(&rq->cmd_ring[i]); + vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]); } ring0 = &rq->cmd_ring[0]; |