diff options
Diffstat (limited to 'drivers/net/mlx5')
| -rw-r--r-- | drivers/net/mlx5/Makefile | 20 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5.c | 9 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5.h | 12 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_ethdev.c | 166 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_fdir.c | 278 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_mac.c | 8 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_mr.c | 8 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_prm.h | 31 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_rss.c | 8 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_rxmode.c | 8 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_rxq.c | 17 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_rxtx.c | 529 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_rxtx.h | 15 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_stats.c | 4 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_trigger.c | 4 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_txq.c | 17 | ||||
| -rw-r--r-- | drivers/net/mlx5/mlx5_vlan.c | 7 |
17 files changed, 580 insertions, 561 deletions
diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile index f6d39388..cf87f0b1 100644 --- a/drivers/net/mlx5/Makefile +++ b/drivers/net/mlx5/Makefile @@ -116,6 +116,26 @@ mlx5_autoconf.h.new: $(RTE_SDK)/scripts/auto-config-h.sh infiniband/mlx5_hw.h \ enum MLX5_ETH_VLAN_INLINE_HEADER_SIZE \ $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_VERBS_MLX5_OPCODE_TSO \ + infiniband/mlx5_hw.h \ + enum MLX5_OPCODE_TSO \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_ETHTOOL_LINK_MODE_25G \ + /usr/include/linux/ethtool.h \ + enum ETHTOOL_LINK_MODE_25000baseCR_Full_BIT \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_ETHTOOL_LINK_MODE_50G \ + /usr/include/linux/ethtool.h \ + enum ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_ETHTOOL_LINK_MODE_100G \ + /usr/include/linux/ethtool.h \ + enum ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT \ + $(AUTOCONF_OUTPUT) # Create mlx5_autoconf.h or update it in case it differs from the new one. diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index d96a9aff..9448374e 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -43,16 +43,16 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_malloc.h> #include <rte_ethdev.h> @@ -60,7 +60,7 @@ #include <rte_common.h> #include <rte_kvargs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" @@ -668,6 +668,7 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) /* Bring Ethernet device up. */ DEBUG("forcing Ethernet interface up"); priv_set_flags(priv, ~IFF_UP, IFF_UP); + mlx5_link_update_unlocked(priv->dev, 1); continue; port_error: diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 3a866098..79b7a600 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -43,16 +43,16 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ether.h> #include <rte_ethdev.h> @@ -60,7 +60,7 @@ #include <rte_interrupts.h> #include <rte_errno.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5_utils.h" @@ -134,6 +134,8 @@ struct priv { unsigned int (*reta_idx)[]; /* RETA index table. */ unsigned int reta_idx_n; /* RETA index size. */ struct fdir_filter_list *fdir_filter_list; /* Flow director rules. */ + struct fdir_queue *fdir_drop_queue; /* Flow director drop queue. */ + uint32_t link_speed_capa; /* Link speed capabilities. */ rte_spinlock_t lock; /* Lock for control functions. */ }; @@ -186,6 +188,7 @@ int priv_set_flags(struct priv *, unsigned int, unsigned int); int mlx5_dev_configure(struct rte_eth_dev *); void mlx5_dev_infos_get(struct rte_eth_dev *, struct rte_eth_dev_info *); const uint32_t *mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev); +int mlx5_link_update_unlocked(struct rte_eth_dev *, int); int mlx5_link_update(struct rte_eth_dev *, int); int mlx5_dev_set_mtu(struct rte_eth_dev *, uint16_t); int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *, struct rte_eth_fc_conf *); @@ -257,6 +260,7 @@ void mlx5_dev_stop(struct rte_eth_dev *); /* mlx5_fdir.c */ +void priv_fdir_queue_destroy(struct priv *, struct fdir_queue *); int fdir_init_filters_list(struct priv *); void priv_fdir_delete_filters_list(struct priv *); void priv_fdir_disable(struct priv *); diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index 130e15d5..ba1ec2a6 100644 --- a/drivers/net/mlx5/mlx5_ethdev.c +++ b/drivers/net/mlx5/mlx5_ethdev.c @@ -50,7 +50,7 @@ /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_atomic.h> #include <rte_ethdev.h> @@ -60,7 +60,7 @@ #include <rte_alarm.h> #include <rte_malloc.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" @@ -583,7 +583,8 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) (DEV_RX_OFFLOAD_IPV4_CKSUM | DEV_RX_OFFLOAD_UDP_CKSUM | DEV_RX_OFFLOAD_TCP_CKSUM) : - 0); + 0) | + (priv->hw_vlan_strip ? DEV_RX_OFFLOAD_VLAN_STRIP : 0); if (!priv->mps) info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; if (priv->hw_csum) @@ -599,15 +600,10 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) * size if it is not fixed. * The API should be updated to solve this problem. */ info->reta_size = priv->ind_table_max_size; - info->speed_capa = - ETH_LINK_SPEED_1G | - ETH_LINK_SPEED_10G | - ETH_LINK_SPEED_20G | - ETH_LINK_SPEED_25G | - ETH_LINK_SPEED_40G | - ETH_LINK_SPEED_50G | - ETH_LINK_SPEED_56G | - ETH_LINK_SPEED_100G; + info->hash_key_size = ((*priv->rss_conf) ? + (*priv->rss_conf)[0]->rss_key_len : + 0); + info->speed_capa = priv->link_speed_capa; priv_unlock(priv); } @@ -630,7 +626,7 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) } /** - * DPDK callback to retrieve physical link information (unlocked version). + * Retrieve physical link information (unlocked version using legacy ioctl). * * @param dev * Pointer to Ethernet device structure. @@ -638,11 +634,11 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) * Wait for request completion (ignored). */ static int -mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) +mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) { struct priv *priv = mlx5_get_priv(dev); struct ethtool_cmd edata = { - .cmd = ETHTOOL_GSET + .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */ }; struct ifreq ifr; struct rte_eth_link dev_link; @@ -667,6 +663,19 @@ mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) dev_link.link_speed = 0; else dev_link.link_speed = link_speed; + priv->link_speed_capa = 0; + if (edata.supported & SUPPORTED_Autoneg) + priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; + if (edata.supported & (SUPPORTED_1000baseT_Full | + SUPPORTED_1000baseKX_Full)) + priv->link_speed_capa |= ETH_LINK_SPEED_1G; + if (edata.supported & SUPPORTED_10000baseKR_Full) + priv->link_speed_capa |= ETH_LINK_SPEED_10G; + if (edata.supported & (SUPPORTED_40000baseKR4_Full | + SUPPORTED_40000baseCR4_Full | + SUPPORTED_40000baseSR4_Full | + SUPPORTED_40000baseLR4_Full)) + priv->link_speed_capa |= ETH_LINK_SPEED_40G; dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & @@ -681,6 +690,123 @@ mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) } /** + * Retrieve physical link information (unlocked version using new ioctl from + * Linux 4.5). + * + * @param dev + * Pointer to Ethernet device structure. + * @param wait_to_complete + * Wait for request completion (ignored). + */ +static int +mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) +{ +#ifdef ETHTOOL_GLINKSETTINGS + struct priv *priv = mlx5_get_priv(dev); + struct ethtool_link_settings edata = { + .cmd = ETHTOOL_GLINKSETTINGS, + }; + struct ifreq ifr; + struct rte_eth_link dev_link; + uint64_t sc; + + (void)wait_to_complete; + if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { + WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); + return -1; + } + memset(&dev_link, 0, sizeof(dev_link)); + dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && + (ifr.ifr_flags & IFF_RUNNING)); + ifr.ifr_data = (void *)&edata; + if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", + strerror(errno)); + return -1; + } + dev_link.link_speed = edata.speed; + sc = edata.link_mode_masks[0] | + ((uint64_t)edata.link_mode_masks[1] << 32); + priv->link_speed_capa = 0; + /* Link speeds available in kernel v4.5. */ + if (sc & ETHTOOL_LINK_MODE_Autoneg_BIT) + priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; + if (sc & (ETHTOOL_LINK_MODE_1000baseT_Full_BIT | + ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_1G; + if (sc & (ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT | + ETHTOOL_LINK_MODE_10000baseKR_Full_BIT | + ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_10G; + if (sc & (ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT | + ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_20G; + if (sc & (ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT | + ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT | + ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT | + ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_40G; + if (sc & (ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT | + ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT | + ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT | + ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_56G; + /* Link speeds available in kernel v4.6. */ +#ifdef HAVE_ETHTOOL_LINK_MODE_25G + if (sc & (ETHTOOL_LINK_MODE_25000baseCR_Full_BIT | + ETHTOOL_LINK_MODE_25000baseKR_Full_BIT | + ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_25G; +#endif +#ifdef HAVE_ETHTOOL_LINK_MODE_50G + if (sc & (ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT | + ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_50G; +#endif +#ifdef HAVE_ETHTOOL_LINK_MODE_100G + if (sc & (ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT | + ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT | + ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT | + ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)) + priv->link_speed_capa |= ETH_LINK_SPEED_100G; +#endif + dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? + ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); + dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & + ETH_LINK_SPEED_FIXED); + if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { + /* Link status changed. */ + dev->data->dev_link = dev_link; + return 0; + } +#else + (void)dev; + (void)wait_to_complete; +#endif + /* Link status is still the same. */ + return -1; +} + +/** + * DPDK callback to retrieve physical link information (unlocked version). + * + * @param dev + * Pointer to Ethernet device structure. + * @param wait_to_complete + * Wait for request completion (ignored). + */ +int +mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) +{ + int ret; + + ret = mlx5_link_update_unlocked_gs(dev, wait_to_complete); + if (ret < 0) + ret = mlx5_link_update_unlocked_gset(dev, wait_to_complete); + return ret; +} + +/** * DPDK callback to retrieve physical link information. * * @param dev @@ -1308,11 +1434,13 @@ mlx5_secondary_data_setup(struct priv *priv) continue; primary_txq_ctrl = container_of(primary_txq, struct txq_ctrl, txq); - txq_ctrl = rte_calloc_socket("TXQ", 1, sizeof(*txq_ctrl), 0, + txq_ctrl = rte_calloc_socket("TXQ", 1, sizeof(*txq_ctrl) + + (1 << primary_txq->elts_n) * + sizeof(struct rte_mbuf *), 0, primary_txq_ctrl->socket); if (txq_ctrl != NULL) { if (txq_ctrl_setup(priv->dev, - primary_txq_ctrl, + txq_ctrl, primary_txq->elts_n, primary_txq_ctrl->socket, NULL) == 0) { @@ -1397,10 +1525,6 @@ priv_select_tx_function(struct priv *priv) } else if ((priv->sriov == 0) && priv->mps) { priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw; DEBUG("selected MPW TX function"); - } else if (priv->txq_inline && (priv->txqs_n >= priv->txqs_inline)) { - priv->dev->tx_pkt_burst = mlx5_tx_burst_inline; - DEBUG("selected inline TX function (%u >= %u queues)", - priv->txqs_n, priv->txqs_inline); } } diff --git a/drivers/net/mlx5/mlx5_fdir.c b/drivers/net/mlx5/mlx5_fdir.c index 73eb00ec..1acf6826 100644 --- a/drivers/net/mlx5/mlx5_fdir.c +++ b/drivers/net/mlx5/mlx5_fdir.c @@ -40,23 +40,23 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ether.h> #include <rte_malloc.h> #include <rte_ethdev.h> #include <rte_common.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" @@ -75,6 +75,7 @@ struct fdir_flow_desc { struct mlx5_fdir_filter { LIST_ENTRY(mlx5_fdir_filter) next; uint16_t queue; /* Queue assigned to if FDIR match. */ + enum rte_eth_fdir_behavior behavior; struct fdir_flow_desc desc; struct ibv_exp_flow *flow; }; @@ -400,6 +401,145 @@ create_flow: } /** + * Destroy a flow director queue. + * + * @param fdir_queue + * Flow director queue to be destroyed. + */ +void +priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue) +{ + struct mlx5_fdir_filter *fdir_filter; + + /* Disable filter flows still applying to this queue. */ + LIST_FOREACH(fdir_filter, priv->fdir_filter_list, next) { + unsigned int idx = fdir_filter->queue; + struct rxq_ctrl *rxq_ctrl = + container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq); + + assert(idx < priv->rxqs_n); + if (fdir_queue == rxq_ctrl->fdir_queue && + fdir_filter->flow != NULL) { + claim_zero(ibv_exp_destroy_flow(fdir_filter->flow)); + fdir_filter->flow = NULL; + } + } + assert(fdir_queue->qp); + claim_zero(ibv_destroy_qp(fdir_queue->qp)); + assert(fdir_queue->ind_table); + claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table)); + if (fdir_queue->wq) + claim_zero(ibv_exp_destroy_wq(fdir_queue->wq)); + if (fdir_queue->cq) + claim_zero(ibv_destroy_cq(fdir_queue->cq)); +#ifndef NDEBUG + memset(fdir_queue, 0x2a, sizeof(*fdir_queue)); +#endif + rte_free(fdir_queue); +} + +/** + * Create a flow director queue. + * + * @param priv + * Private structure. + * @param wq + * Work queue to route matched packets to, NULL if one needs to + * be created. + * + * @return + * Related flow director queue on success, NULL otherwise. + */ +static struct fdir_queue * +priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq, + unsigned int socket) +{ + struct fdir_queue *fdir_queue; + + fdir_queue = rte_calloc_socket(__func__, 1, sizeof(*fdir_queue), + 0, socket); + if (!fdir_queue) { + ERROR("cannot allocate flow director queue"); + return NULL; + } + assert(priv->pd); + assert(priv->ctx); + if (!wq) { + fdir_queue->cq = ibv_exp_create_cq( + priv->ctx, 1, NULL, NULL, 0, + &(struct ibv_exp_cq_init_attr){ + .comp_mask = 0, + }); + if (!fdir_queue->cq) { + ERROR("cannot create flow director CQ"); + goto error; + } + fdir_queue->wq = ibv_exp_create_wq( + priv->ctx, + &(struct ibv_exp_wq_init_attr){ + .wq_type = IBV_EXP_WQT_RQ, + .max_recv_wr = 1, + .max_recv_sge = 1, + .pd = priv->pd, + .cq = fdir_queue->cq, + }); + if (!fdir_queue->wq) { + ERROR("cannot create flow director WQ"); + goto error; + } + wq = fdir_queue->wq; + } + fdir_queue->ind_table = ibv_exp_create_rwq_ind_table( + priv->ctx, + &(struct ibv_exp_rwq_ind_table_init_attr){ + .pd = priv->pd, + .log_ind_tbl_size = 0, + .ind_tbl = &wq, + .comp_mask = 0, + }); + if (!fdir_queue->ind_table) { + ERROR("cannot create flow director indirection table"); + goto error; + } + fdir_queue->qp = ibv_exp_create_qp( + priv->ctx, + &(struct ibv_exp_qp_init_attr){ + .qp_type = IBV_QPT_RAW_PACKET, + .comp_mask = + IBV_EXP_QP_INIT_ATTR_PD | + IBV_EXP_QP_INIT_ATTR_PORT | + IBV_EXP_QP_INIT_ATTR_RX_HASH, + .pd = priv->pd, + .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){ + .rx_hash_function = + IBV_EXP_RX_HASH_FUNC_TOEPLITZ, + .rx_hash_key_len = rss_hash_default_key_len, + .rx_hash_key = rss_hash_default_key, + .rx_hash_fields_mask = 0, + .rwq_ind_tbl = fdir_queue->ind_table, + }, + .port_num = priv->port, + }); + if (!fdir_queue->qp) { + ERROR("cannot create flow director hash RX QP"); + goto error; + } + return fdir_queue; +error: + assert(fdir_queue); + assert(!fdir_queue->qp); + if (fdir_queue->ind_table) + claim_zero(ibv_exp_destroy_rwq_ind_table + (fdir_queue->ind_table)); + if (fdir_queue->wq) + claim_zero(ibv_exp_destroy_wq(fdir_queue->wq)); + if (fdir_queue->cq) + claim_zero(ibv_destroy_cq(fdir_queue->cq)); + rte_free(fdir_queue); + return NULL; +} + +/** * Get flow director queue for a specific RX queue, create it in case * it does not exist. * @@ -416,74 +556,42 @@ priv_get_fdir_queue(struct priv *priv, uint16_t idx) { struct rxq_ctrl *rxq_ctrl = container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq); - struct fdir_queue *fdir_queue = &rxq_ctrl->fdir_queue; - struct ibv_exp_rwq_ind_table *ind_table = NULL; - struct ibv_qp *qp = NULL; - struct ibv_exp_rwq_ind_table_init_attr ind_init_attr; - struct ibv_exp_rx_hash_conf hash_conf; - struct ibv_exp_qp_init_attr qp_init_attr; - int err = 0; - - /* Return immediately if it has already been created. */ - if (fdir_queue->qp != NULL) - return fdir_queue; - - ind_init_attr = (struct ibv_exp_rwq_ind_table_init_attr){ - .pd = priv->pd, - .log_ind_tbl_size = 0, - .ind_tbl = &rxq_ctrl->wq, - .comp_mask = 0, - }; - - errno = 0; - ind_table = ibv_exp_create_rwq_ind_table(priv->ctx, - &ind_init_attr); - if (ind_table == NULL) { - /* Not clear whether errno is set. */ - err = (errno ? errno : EINVAL); - ERROR("RX indirection table creation failed with error %d: %s", - err, strerror(err)); - goto error; - } + struct fdir_queue *fdir_queue = rxq_ctrl->fdir_queue; - /* Create fdir_queue qp. */ - hash_conf = (struct ibv_exp_rx_hash_conf){ - .rx_hash_function = IBV_EXP_RX_HASH_FUNC_TOEPLITZ, - .rx_hash_key_len = rss_hash_default_key_len, - .rx_hash_key = rss_hash_default_key, - .rx_hash_fields_mask = 0, - .rwq_ind_tbl = ind_table, - }; - qp_init_attr = (struct ibv_exp_qp_init_attr){ - .max_inl_recv = 0, /* Currently not supported. */ - .qp_type = IBV_QPT_RAW_PACKET, - .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | - IBV_EXP_QP_INIT_ATTR_RX_HASH), - .pd = priv->pd, - .rx_hash_conf = &hash_conf, - .port_num = priv->port, - }; - - qp = ibv_exp_create_qp(priv->ctx, &qp_init_attr); - if (qp == NULL) { - err = (errno ? errno : EINVAL); - ERROR("hash RX QP creation failure: %s", strerror(err)); - goto error; + assert(rxq_ctrl->wq); + if (fdir_queue == NULL) { + fdir_queue = priv_fdir_queue_create(priv, rxq_ctrl->wq, + rxq_ctrl->socket); + rxq_ctrl->fdir_queue = fdir_queue; } - - fdir_queue->ind_table = ind_table; - fdir_queue->qp = qp; - return fdir_queue; +} -error: - if (qp != NULL) - claim_zero(ibv_destroy_qp(qp)); - - if (ind_table != NULL) - claim_zero(ibv_exp_destroy_rwq_ind_table(ind_table)); +/** + * Get or flow director drop queue. Create it if it does not exist. + * + * @param priv + * Private structure. + * + * @return + * Flow director drop queue on success, NULL otherwise. + */ +static struct fdir_queue * +priv_get_fdir_drop_queue(struct priv *priv) +{ + struct fdir_queue *fdir_queue = priv->fdir_drop_queue; - return NULL; + if (fdir_queue == NULL) { + unsigned int socket = SOCKET_ID_ANY; + + /* Select a known NUMA socket if possible. */ + if (priv->rxqs_n && (*priv->rxqs)[0]) + socket = container_of((*priv->rxqs)[0], + struct rxq_ctrl, rxq)->socket; + fdir_queue = priv_fdir_queue_create(priv, NULL, socket); + priv->fdir_drop_queue = fdir_queue; + } + return fdir_queue; } /** @@ -508,7 +616,11 @@ priv_fdir_filter_enable(struct priv *priv, return 0; /* Get fdir_queue for specific queue. */ - fdir_queue = priv_get_fdir_queue(priv, mlx5_fdir_filter->queue); + if (mlx5_fdir_filter->behavior == RTE_ETH_FDIR_REJECT) + fdir_queue = priv_get_fdir_drop_queue(priv); + else + fdir_queue = priv_get_fdir_queue(priv, + mlx5_fdir_filter->queue); if (fdir_queue == NULL) { ERROR("failed to create flow director rxq for queue %d", @@ -601,7 +713,6 @@ priv_fdir_disable(struct priv *priv) { unsigned int i; struct mlx5_fdir_filter *mlx5_fdir_filter; - struct fdir_queue *fdir_queue; /* Run on every flow director filter and destroy flow handle. */ LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) { @@ -618,23 +729,19 @@ priv_fdir_disable(struct priv *priv) } } - /* Run on every RX queue to destroy related flow director QP and - * indirection table. */ + /* Destroy flow director context in each RX queue. */ for (i = 0; (i != priv->rxqs_n); i++) { struct rxq_ctrl *rxq_ctrl = container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq); - fdir_queue = &rxq_ctrl->fdir_queue; - if (fdir_queue->qp != NULL) { - claim_zero(ibv_destroy_qp(fdir_queue->qp)); - fdir_queue->qp = NULL; - } - - if (fdir_queue->ind_table != NULL) { - claim_zero(ibv_exp_destroy_rwq_ind_table - (fdir_queue->ind_table)); - fdir_queue->ind_table = NULL; - } + if (!rxq_ctrl->fdir_queue) + continue; + priv_fdir_queue_destroy(priv, rxq_ctrl->fdir_queue); + rxq_ctrl->fdir_queue = NULL; + } + if (priv->fdir_drop_queue) { + priv_fdir_queue_destroy(priv, priv->fdir_drop_queue); + priv->fdir_drop_queue = NULL; } } @@ -736,8 +843,9 @@ priv_fdir_filter_add(struct priv *priv, return err; } - /* Set queue. */ + /* Set action parameters. */ mlx5_fdir_filter->queue = fdir_filter->action.rx_queue; + mlx5_fdir_filter->behavior = fdir_filter->action.behavior; /* Convert to mlx5 filter descriptor. */ fdir_filter_to_flow_desc(fdir_filter, @@ -955,7 +1063,7 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg) { - int ret = -EINVAL; + int ret = EINVAL; struct priv *priv = dev->data->dev_private; switch (filter_type) { @@ -970,5 +1078,5 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, break; } - return ret; + return -ret; } diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c index f6b27bb8..4fcfd3b8 100644 --- a/drivers/net/mlx5/mlx5_mac.c +++ b/drivers/net/mlx5/mlx5_mac.c @@ -44,22 +44,22 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ether.h> #include <rte_ethdev.h> #include <rte_common.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c index 67dfefa8..0a363846 100644 --- a/drivers/net/mlx5/mlx5_mr.c +++ b/drivers/net/mlx5/mlx5_mr.c @@ -34,20 +34,20 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_mempool.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" diff --git a/drivers/net/mlx5/mlx5_prm.h b/drivers/net/mlx5/mlx5_prm.h index 5db219b3..1c369cac 100644 --- a/drivers/net/mlx5/mlx5_prm.h +++ b/drivers/net/mlx5/mlx5_prm.h @@ -37,13 +37,15 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/mlx5_hw.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif +#include "mlx5_autoconf.h" + /* Get CQE owner bit. */ #define MLX5_CQE_OWNER(op_own) ((op_own) & MLX5_CQE_OWNER_MASK) @@ -71,6 +73,31 @@ /* Room for inline data in multi-packet WQE. */ #define MLX5_MWQE64_INL_DATA 28 +#ifndef HAVE_VERBS_MLX5_OPCODE_TSO +#define MLX5_OPCODE_TSO MLX5_OPCODE_LSO_MPW /* Compat with OFED 3.3. */ +#endif + +/* IPv4 packet. */ +#define MLX5_CQE_RX_IPV4_PACKET (1u << 2) + +/* IPv6 packet. */ +#define MLX5_CQE_RX_IPV6_PACKET (1u << 3) + +/* Outer IPv4 packet. */ +#define MLX5_CQE_RX_OUTER_IPV4_PACKET (1u << 7) + +/* Outer IPv6 packet. */ +#define MLX5_CQE_RX_OUTER_IPV6_PACKET (1u << 8) + +/* Tunnel packet bit in the CQE. */ +#define MLX5_CQE_RX_TUNNEL_PACKET (1u << 4) + +/* Outer IP checksum OK. */ +#define MLX5_CQE_RX_OUTER_IP_CSUM_OK (1u << 5) + +/* Outer UDP header and checksum OK. */ +#define MLX5_CQE_RX_OUTER_TCP_UDP_CSUM_OK (1u << 6) + /* Subset of struct mlx5_wqe_eth_seg. */ struct mlx5_wqe_eth_seg_small { uint32_t rsvd0; diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c index 639e935b..0bed74ee 100644 --- a/drivers/net/mlx5/mlx5_rss.c +++ b/drivers/net/mlx5/mlx5_rss.c @@ -40,21 +40,21 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_malloc.h> #include <rte_ethdev.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c index 8b585554..173e6e84 100644 --- a/drivers/net/mlx5/mlx5_rxmode.c +++ b/drivers/net/mlx5/mlx5_rxmode.c @@ -38,20 +38,20 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ethdev.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index 29c137cd..7dbe8dd2 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -40,25 +40,25 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #include <infiniband/arch.h> #include <infiniband/mlx5_hw.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_mbuf.h> #include <rte_malloc.h> #include <rte_ethdev.h> #include <rte_common.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" @@ -745,6 +745,8 @@ rxq_cleanup(struct rxq_ctrl *rxq_ctrl) DEBUG("cleaning up %p", (void *)rxq_ctrl); rxq_free_elts(rxq_ctrl); + if (rxq_ctrl->fdir_queue != NULL) + priv_fdir_queue_destroy(rxq_ctrl->priv, rxq_ctrl->fdir_queue); if (rxq_ctrl->if_wq != NULL) { assert(rxq_ctrl->priv != NULL); assert(rxq_ctrl->priv->ctx != NULL); @@ -943,6 +945,11 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, (void)conf; /* Thresholds configuration (ignored). */ /* Enable scattered packets support for this queue if necessary. */ assert(mb_len >= RTE_PKTMBUF_HEADROOM); + /* If smaller than MRU, multi-segment support must be enabled. */ + if (mb_len < (priv->mtu > dev->data->dev_conf.rxmode.max_rx_pkt_len ? + dev->data->dev_conf.rxmode.max_rx_pkt_len : + priv->mtu)) + dev->data->dev_conf.rxmode.jumbo_frame = 1; if ((dev->data->dev_conf.rxmode.jumbo_frame) && (dev->data->dev_conf.rxmode.max_rx_pkt_len > (mb_len - RTE_PKTMBUF_HEADROOM))) { @@ -1259,7 +1266,7 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, (void *)dev, (void *)rxq_ctrl); (*priv->rxqs)[idx] = &rxq_ctrl->rxq; /* Update receive callback. */ - dev->rx_pkt_burst = mlx5_rx_burst; + priv_select_rx_function(priv); } priv_unlock(priv); return -ret; diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index fce3381a..79f7fa99 100644 --- a/drivers/net/mlx5/mlx5_rxtx.c +++ b/drivers/net/mlx5/mlx5_rxtx.c @@ -39,18 +39,18 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #include <infiniband/mlx5_hw.h> #include <infiniband/arch.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_mbuf.h> #include <rte_mempool.h> @@ -59,7 +59,7 @@ #include <rte_branch_prediction.h> #include <rte_ether.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" @@ -290,226 +290,103 @@ txq_mp2mr(struct txq *txq, struct rte_mempool *mp) * Pointer to TX queue structure. * @param wqe * Pointer to the WQE to fill. - * @param addr - * Buffer data address. + * @param buf + * Buffer. * @param length * Packet length. - * @param lkey - * Memory region lkey. - */ -static inline void -mlx5_wqe_write(struct txq *txq, volatile union mlx5_wqe *wqe, - uintptr_t addr, uint32_t length, uint32_t lkey) -{ - wqe->wqe.ctrl.data[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND); - wqe->wqe.ctrl.data[1] = htonl((txq->qp_num_8s) | 4); - wqe->wqe.ctrl.data[2] = 0; - wqe->wqe.ctrl.data[3] = 0; - wqe->inl.eseg.rsvd0 = 0; - wqe->inl.eseg.rsvd1 = 0; - wqe->inl.eseg.mss = 0; - wqe->inl.eseg.rsvd2 = 0; - wqe->wqe.eseg.inline_hdr_sz = htons(MLX5_ETH_INLINE_HEADER_SIZE); - /* Copy the first 16 bytes into inline header. */ - rte_memcpy((uint8_t *)(uintptr_t)wqe->wqe.eseg.inline_hdr_start, - (uint8_t *)(uintptr_t)addr, - MLX5_ETH_INLINE_HEADER_SIZE); - addr += MLX5_ETH_INLINE_HEADER_SIZE; - length -= MLX5_ETH_INLINE_HEADER_SIZE; - /* Store remaining data in data segment. */ - wqe->wqe.dseg.byte_count = htonl(length); - wqe->wqe.dseg.lkey = lkey; - wqe->wqe.dseg.addr = htonll(addr); - /* Increment consumer index. */ - ++txq->wqe_ci; -} - -/** - * Write a regular WQE with VLAN. * - * @param txq - * Pointer to TX queue structure. - * @param wqe - * Pointer to the WQE to fill. - * @param addr - * Buffer data address. - * @param length - * Packet length. - * @param lkey - * Memory region lkey. - * @param vlan_tci - * VLAN field to insert in packet. + * @return ds + * Number of DS elements consumed. */ -static inline void -mlx5_wqe_write_vlan(struct txq *txq, volatile union mlx5_wqe *wqe, - uintptr_t addr, uint32_t length, uint32_t lkey, - uint16_t vlan_tci) +static inline unsigned int +mlx5_wqe_write(struct txq *txq, volatile union mlx5_wqe *wqe, + struct rte_mbuf *buf, uint32_t length) { - uint32_t vlan = htonl(0x81000000 | vlan_tci); - + uintptr_t raw = (uintptr_t)&wqe->wqe.eseg.inline_hdr_start; + uint16_t ds; + uint16_t pkt_inline_sz = 16; + uintptr_t addr = rte_pktmbuf_mtod(buf, uintptr_t); + struct mlx5_wqe_data_seg *dseg = NULL; + + assert(length >= 16); + /* Start the know and common part of the WQE structure. */ wqe->wqe.ctrl.data[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND); - wqe->wqe.ctrl.data[1] = htonl((txq->qp_num_8s) | 4); wqe->wqe.ctrl.data[2] = 0; wqe->wqe.ctrl.data[3] = 0; - wqe->inl.eseg.rsvd0 = 0; - wqe->inl.eseg.rsvd1 = 0; - wqe->inl.eseg.mss = 0; - wqe->inl.eseg.rsvd2 = 0; - wqe->wqe.eseg.inline_hdr_sz = htons(MLX5_ETH_VLAN_INLINE_HEADER_SIZE); - /* - * Copy 12 bytes of source & destination MAC address. - * Copy 4 bytes of VLAN. - * Copy 2 bytes of Ether type. - */ - rte_memcpy((uint8_t *)(uintptr_t)wqe->wqe.eseg.inline_hdr_start, - (uint8_t *)(uintptr_t)addr, 12); - rte_memcpy((uint8_t *)((uintptr_t)wqe->wqe.eseg.inline_hdr_start + 12), - &vlan, sizeof(vlan)); - rte_memcpy((uint8_t *)((uintptr_t)wqe->wqe.eseg.inline_hdr_start + 16), - (uint8_t *)((uintptr_t)addr + 12), 2); - addr += MLX5_ETH_VLAN_INLINE_HEADER_SIZE - sizeof(vlan); - length -= MLX5_ETH_VLAN_INLINE_HEADER_SIZE - sizeof(vlan); - /* Store remaining data in data segment. */ - wqe->wqe.dseg.byte_count = htonl(length); - wqe->wqe.dseg.lkey = lkey; - wqe->wqe.dseg.addr = htonll(addr); - /* Increment consumer index. */ - ++txq->wqe_ci; -} - -/** - * Write a inline WQE. - * - * @param txq - * Pointer to TX queue structure. - * @param wqe - * Pointer to the WQE to fill. - * @param addr - * Buffer data address. - * @param length - * Packet length. - * @param lkey - * Memory region lkey. - */ -static inline void -mlx5_wqe_write_inline(struct txq *txq, volatile union mlx5_wqe *wqe, - uintptr_t addr, uint32_t length) -{ - uint32_t size; - uint16_t wqe_cnt = txq->wqe_n - 1; - uint16_t wqe_ci = txq->wqe_ci + 1; - - /* Copy the first 16 bytes into inline header. */ - rte_memcpy((void *)(uintptr_t)wqe->inl.eseg.inline_hdr_start, - (void *)(uintptr_t)addr, - MLX5_ETH_INLINE_HEADER_SIZE); - addr += MLX5_ETH_INLINE_HEADER_SIZE; - length -= MLX5_ETH_INLINE_HEADER_SIZE; - size = 3 + ((4 + length + 15) / 16); - wqe->inl.byte_cnt = htonl(length | MLX5_INLINE_SEG); - rte_memcpy((void *)(uintptr_t)&wqe->inl.data[0], - (void *)addr, MLX5_WQE64_INL_DATA); - addr += MLX5_WQE64_INL_DATA; - length -= MLX5_WQE64_INL_DATA; - while (length) { - volatile union mlx5_wqe *wqe_next = - &(*txq->wqes)[wqe_ci & wqe_cnt]; - uint32_t copy_bytes = (length > sizeof(*wqe)) ? - sizeof(*wqe) : - length; - - rte_mov64((uint8_t *)(uintptr_t)&wqe_next->data[0], - (uint8_t *)addr); - addr += copy_bytes; - length -= copy_bytes; - ++wqe_ci; + wqe->wqe.eseg.rsvd0 = 0; + wqe->wqe.eseg.rsvd1 = 0; + wqe->wqe.eseg.mss = 0; + wqe->wqe.eseg.rsvd2 = 0; + /* Start by copying the Ethernet Header. */ + rte_mov16((uint8_t *)raw, (uint8_t *)addr); + length -= 16; + addr += 16; + /* Replace the Ethernet type by the VLAN if necessary. */ + if (buf->ol_flags & PKT_TX_VLAN_PKT) { + uint32_t vlan = htonl(0x81000000 | buf->vlan_tci); + + memcpy((uint8_t *)(raw + 16 - sizeof(vlan)), + &vlan, sizeof(vlan)); + addr -= sizeof(vlan); + length += sizeof(vlan); } - assert(size < 64); - wqe->inl.ctrl.data[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND); - wqe->inl.ctrl.data[1] = htonl(txq->qp_num_8s | size); - wqe->inl.ctrl.data[2] = 0; - wqe->inl.ctrl.data[3] = 0; - wqe->inl.eseg.rsvd0 = 0; - wqe->inl.eseg.rsvd1 = 0; - wqe->inl.eseg.mss = 0; - wqe->inl.eseg.rsvd2 = 0; - wqe->inl.eseg.inline_hdr_sz = htons(MLX5_ETH_INLINE_HEADER_SIZE); - /* Increment consumer index. */ - txq->wqe_ci = wqe_ci; -} - -/** - * Write a inline WQE with VLAN. - * - * @param txq - * Pointer to TX queue structure. - * @param wqe - * Pointer to the WQE to fill. - * @param addr - * Buffer data address. - * @param length - * Packet length. - * @param lkey - * Memory region lkey. - * @param vlan_tci - * VLAN field to insert in packet. - */ -static inline void -mlx5_wqe_write_inline_vlan(struct txq *txq, volatile union mlx5_wqe *wqe, - uintptr_t addr, uint32_t length, uint16_t vlan_tci) -{ - uint32_t size; - uint32_t wqe_cnt = txq->wqe_n - 1; - uint16_t wqe_ci = txq->wqe_ci + 1; - uint32_t vlan = htonl(0x81000000 | vlan_tci); - - /* - * Copy 12 bytes of source & destination MAC address. - * Copy 4 bytes of VLAN. - * Copy 2 bytes of Ether type. - */ - rte_memcpy((uint8_t *)(uintptr_t)wqe->inl.eseg.inline_hdr_start, - (uint8_t *)addr, 12); - rte_memcpy((uint8_t *)(uintptr_t)wqe->inl.eseg.inline_hdr_start + 12, - &vlan, sizeof(vlan)); - rte_memcpy((uint8_t *)((uintptr_t)wqe->inl.eseg.inline_hdr_start + 16), - (uint8_t *)(addr + 12), 2); - addr += MLX5_ETH_VLAN_INLINE_HEADER_SIZE - sizeof(vlan); - length -= MLX5_ETH_VLAN_INLINE_HEADER_SIZE - sizeof(vlan); - size = (sizeof(wqe->inl.ctrl.ctrl) + - sizeof(wqe->inl.eseg) + - sizeof(wqe->inl.byte_cnt) + - length + 15) / 16; - wqe->inl.byte_cnt = htonl(length | MLX5_INLINE_SEG); - rte_memcpy((void *)(uintptr_t)&wqe->inl.data[0], - (void *)addr, MLX5_WQE64_INL_DATA); - addr += MLX5_WQE64_INL_DATA; - length -= MLX5_WQE64_INL_DATA; - while (length) { - volatile union mlx5_wqe *wqe_next = - &(*txq->wqes)[wqe_ci & wqe_cnt]; - uint32_t copy_bytes = (length > sizeof(*wqe)) ? - sizeof(*wqe) : - length; - - rte_mov64((uint8_t *)(uintptr_t)&wqe_next->data[0], - (uint8_t *)addr); - addr += copy_bytes; - length -= copy_bytes; - ++wqe_ci; + /* Inline if enough room. */ + if (txq->max_inline != 0) { + uintptr_t end = (uintptr_t)&(*txq->wqes)[txq->wqe_n]; + uint16_t max_inline = txq->max_inline * RTE_CACHE_LINE_SIZE; + uint16_t room; + + raw += 16; + room = end - (uintptr_t)raw; + if (room > max_inline) { + uintptr_t addr_end = (addr + max_inline) & + ~(RTE_CACHE_LINE_SIZE - 1); + uint16_t copy_b = ((addr_end - addr) > length) ? + length : + (addr_end - addr); + + rte_memcpy((void *)raw, (void *)addr, copy_b); + addr += copy_b; + length -= copy_b; + pkt_inline_sz += copy_b; + /* Sanity check. */ + assert(addr <= addr_end); + } + /* Store the inlined packet size in the WQE. */ + wqe->wqe.eseg.inline_hdr_sz = htons(pkt_inline_sz); + /* + * 2 DWORDs consumed by the WQE header + 1 DSEG + + * the size of the inline part of the packet. + */ + ds = 2 + ((pkt_inline_sz - 2 + 15) / 16); + if (length > 0) { + dseg = (struct mlx5_wqe_data_seg *) + ((uintptr_t)wqe + (ds * 16)); + if ((uintptr_t)dseg >= end) + dseg = (struct mlx5_wqe_data_seg *) + ((uintptr_t)&(*txq->wqes)[0]); + goto use_dseg; + } + } else { + /* Add the remaining packet as a simple ds. */ + ds = 3; + /* + * No inline has been done in the packet, only the Ethernet + * Header as been stored. + */ + wqe->wqe.eseg.inline_hdr_sz = htons(16); + dseg = (struct mlx5_wqe_data_seg *) + ((uintptr_t)wqe + (ds * 16)); +use_dseg: + *dseg = (struct mlx5_wqe_data_seg) { + .addr = htonll(addr), + .byte_count = htonl(length), + .lkey = txq_mp2mr(txq, txq_mb2mp(buf)), + }; + ++ds; } - assert(size < 64); - wqe->inl.ctrl.data[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND); - wqe->inl.ctrl.data[1] = htonl(txq->qp_num_8s | size); - wqe->inl.ctrl.data[2] = 0; - wqe->inl.ctrl.data[3] = 0; - wqe->inl.eseg.rsvd0 = 0; - wqe->inl.eseg.rsvd1 = 0; - wqe->inl.eseg.mss = 0; - wqe->inl.eseg.rsvd2 = 0; - wqe->inl.eseg.inline_hdr_sz = htons(MLX5_ETH_VLAN_INLINE_HEADER_SIZE); - /* Increment consumer index. */ - txq->wqe_ci = wqe_ci; + wqe->wqe.ctrl.data[1] = htonl(txq->qp_num_8s | ds); + return ds; } /** @@ -609,9 +486,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) do { struct rte_mbuf *buf = *(pkts++); unsigned int elts_head_next; - uintptr_t addr; uint32_t length; - uint32_t lkey; unsigned int segs_n = buf->nb_segs; volatile struct mlx5_wqe_data_seg *dseg; unsigned int ds = sizeof(*wqe) / 16; @@ -627,12 +502,10 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) --pkts_n; elts_head_next = (elts_head + 1) & (elts_n - 1); wqe = &(*txq->wqes)[txq->wqe_ci & (txq->wqe_n - 1)]; - dseg = &wqe->wqe.dseg; - rte_prefetch0(wqe); + tx_prefetch_wqe(txq, txq->wqe_ci); + tx_prefetch_wqe(txq, txq->wqe_ci + 1); if (pkts_n) rte_prefetch0(*pkts); - /* Retrieve buffer information. */ - addr = rte_pktmbuf_mtod(buf, uintptr_t); length = DATA_LEN(buf); /* Update element. */ (*txq->elts)[elts_head] = buf; @@ -640,13 +513,6 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) if (pkts_n) rte_prefetch0(rte_pktmbuf_mtod(*pkts, volatile void *)); - /* Retrieve Memory Region key for this memory pool. */ - lkey = txq_mp2mr(txq, txq_mb2mp(buf)); - if (buf->ol_flags & PKT_TX_VLAN_PKT) - mlx5_wqe_write_vlan(txq, wqe, addr, length, lkey, - buf->vlan_tci); - else - mlx5_wqe_write(txq, wqe, addr, length, lkey); /* Should we enable HW CKSUM offload */ if (buf->ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { @@ -656,6 +522,11 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) } else { wqe->wqe.eseg.cs_flags = 0; } + ds = mlx5_wqe_write(txq, wqe, buf, length); + if (segs_n == 1) + goto skip_segs; + dseg = (volatile struct mlx5_wqe_data_seg *) + (((uintptr_t)wqe) + ds * 16); while (--segs_n) { /* * Spill on next WQE when the current one does not have @@ -686,11 +557,13 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) /* Update DS field in WQE. */ wqe->wqe.ctrl.data[1] &= htonl(0xffffffc0); wqe->wqe.ctrl.data[1] |= htonl(ds & 0x3f); - elts_head = elts_head_next; +skip_segs: #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment sent bytes counter. */ txq->stats.obytes += length; #endif + /* Increment consumer index. */ + txq->wqe_ci += (ds + 3) / 4; elts_head = elts_head_next; ++i; } while (pkts_n); @@ -719,166 +592,6 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) } /** - * DPDK callback for TX with inline support. - * - * @param dpdk_txq - * Generic pointer to TX queue structure. - * @param[in] pkts - * Packets to transmit. - * @param pkts_n - * Number of packets in array. - * - * @return - * Number of packets successfully transmitted (<= pkts_n). - */ -uint16_t -mlx5_tx_burst_inline(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) -{ - struct txq *txq = (struct txq *)dpdk_txq; - uint16_t elts_head = txq->elts_head; - const unsigned int elts_n = txq->elts_n; - unsigned int i = 0; - unsigned int j = 0; - unsigned int max; - unsigned int comp; - volatile union mlx5_wqe *wqe = NULL; - unsigned int max_inline = txq->max_inline; - - if (unlikely(!pkts_n)) - return 0; - /* Prefetch first packet cacheline. */ - tx_prefetch_cqe(txq, txq->cq_ci); - tx_prefetch_cqe(txq, txq->cq_ci + 1); - rte_prefetch0(*pkts); - /* Start processing. */ - txq_complete(txq); - max = (elts_n - (elts_head - txq->elts_tail)); - if (max > elts_n) - max -= elts_n; - do { - struct rte_mbuf *buf = *(pkts++); - unsigned int elts_head_next; - uintptr_t addr; - uint32_t length; - uint32_t lkey; - unsigned int segs_n = buf->nb_segs; - volatile struct mlx5_wqe_data_seg *dseg; - unsigned int ds = sizeof(*wqe) / 16; - - /* - * Make sure there is enough room to store this packet and - * that one ring entry remains unused. - */ - assert(segs_n); - if (max < segs_n + 1) - break; - max -= segs_n; - --pkts_n; - elts_head_next = (elts_head + 1) & (elts_n - 1); - wqe = &(*txq->wqes)[txq->wqe_ci & (txq->wqe_n - 1)]; - dseg = &wqe->wqe.dseg; - tx_prefetch_wqe(txq, txq->wqe_ci); - tx_prefetch_wqe(txq, txq->wqe_ci + 1); - if (pkts_n) - rte_prefetch0(*pkts); - /* Should we enable HW CKSUM offload */ - if (buf->ol_flags & - (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { - wqe->inl.eseg.cs_flags = - MLX5_ETH_WQE_L3_CSUM | - MLX5_ETH_WQE_L4_CSUM; - } else { - wqe->inl.eseg.cs_flags = 0; - } - /* Retrieve buffer information. */ - addr = rte_pktmbuf_mtod(buf, uintptr_t); - length = DATA_LEN(buf); - /* Update element. */ - (*txq->elts)[elts_head] = buf; - /* Prefetch next buffer data. */ - if (pkts_n) - rte_prefetch0(rte_pktmbuf_mtod(*pkts, - volatile void *)); - if ((length <= max_inline) && (segs_n == 1)) { - if (buf->ol_flags & PKT_TX_VLAN_PKT) - mlx5_wqe_write_inline_vlan(txq, wqe, - addr, length, - buf->vlan_tci); - else - mlx5_wqe_write_inline(txq, wqe, addr, length); - goto skip_segs; - } else { - /* Retrieve Memory Region key for this memory pool. */ - lkey = txq_mp2mr(txq, txq_mb2mp(buf)); - if (buf->ol_flags & PKT_TX_VLAN_PKT) - mlx5_wqe_write_vlan(txq, wqe, addr, length, - lkey, buf->vlan_tci); - else - mlx5_wqe_write(txq, wqe, addr, length, lkey); - } - while (--segs_n) { - /* - * Spill on next WQE when the current one does not have - * enough room left. Size of WQE must a be a multiple - * of data segment size. - */ - assert(!(sizeof(*wqe) % sizeof(*dseg))); - if (!(ds % (sizeof(*wqe) / 16))) - dseg = (volatile void *) - &(*txq->wqes)[txq->wqe_ci++ & - (txq->wqe_n - 1)]; - else - ++dseg; - ++ds; - buf = buf->next; - assert(buf); - /* Store segment information. */ - dseg->byte_count = htonl(DATA_LEN(buf)); - dseg->lkey = txq_mp2mr(txq, txq_mb2mp(buf)); - dseg->addr = htonll(rte_pktmbuf_mtod(buf, uintptr_t)); - (*txq->elts)[elts_head_next] = buf; - elts_head_next = (elts_head_next + 1) & (elts_n - 1); -#ifdef MLX5_PMD_SOFT_COUNTERS - length += DATA_LEN(buf); -#endif - ++j; - } - /* Update DS field in WQE. */ - wqe->inl.ctrl.data[1] &= htonl(0xffffffc0); - wqe->inl.ctrl.data[1] |= htonl(ds & 0x3f); -skip_segs: - elts_head = elts_head_next; -#ifdef MLX5_PMD_SOFT_COUNTERS - /* Increment sent bytes counter. */ - txq->stats.obytes += length; -#endif - ++i; - } while (pkts_n); - /* Take a shortcut if nothing must be sent. */ - if (unlikely(i == 0)) - return 0; - /* Check whether completion threshold has been reached. */ - comp = txq->elts_comp + i + j; - if (comp >= MLX5_TX_COMP_THRESH) { - /* Request completion on last WQE. */ - wqe->inl.ctrl.data[2] = htonl(8); - /* Save elts_head in unused "immediate" field of WQE. */ - wqe->inl.ctrl.data[3] = elts_head; - txq->elts_comp = 0; - } else { - txq->elts_comp = comp; - } -#ifdef MLX5_PMD_SOFT_COUNTERS - /* Increment sent packets counter. */ - txq->stats.opackets += i; -#endif - /* Ring QP doorbell. */ - mlx5_tx_dbrec(txq); - txq->elts_head = elts_head; - return i; -} - -/** * Open a MPW session. * * @param txq @@ -908,7 +621,7 @@ mlx5_mpw_new(struct txq *txq, struct mlx5_mpw *mpw, uint32_t length) mpw->wqe->mpw.eseg.rsvd2 = 0; mpw->wqe->mpw.ctrl.data[0] = htonl((MLX5_OPC_MOD_MPW << 24) | (txq->wqe_ci << 8) | - MLX5_OPCODE_LSO_MPW); + MLX5_OPCODE_TSO); mpw->wqe->mpw.ctrl.data[2] = 0; mpw->wqe->mpw.ctrl.data[3] = 0; mpw->data.dseg[0] = &mpw->wqe->mpw.dseg[0]; @@ -1107,7 +820,7 @@ mlx5_mpw_inline_new(struct txq *txq, struct mlx5_mpw *mpw, uint32_t length) mpw->wqe = &(*txq->wqes)[idx]; mpw->wqe->mpw_inl.ctrl.data[0] = htonl((MLX5_OPC_MOD_MPW << 24) | (txq->wqe_ci << 8) | - MLX5_OPCODE_LSO_MPW); + MLX5_OPCODE_TSO); mpw->wqe->mpw_inl.ctrl.data[2] = 0; mpw->wqe->mpw_inl.ctrl.data[3] = 0; mpw->wqe->mpw_inl.eseg.mss = htons(length); @@ -1168,7 +881,7 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, unsigned int j = 0; unsigned int max; unsigned int comp; - unsigned int inline_room = txq->max_inline; + unsigned int inline_room = txq->max_inline * RTE_CACHE_LINE_SIZE; struct mlx5_mpw mpw = { .state = MLX5_MPW_STATE_CLOSED, }; @@ -1222,7 +935,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, (length > inline_room) || (mpw.wqe->mpw_inl.eseg.cs_flags != cs_flags)) { mlx5_mpw_inline_close(txq, &mpw); - inline_room = txq->max_inline; + inline_room = + txq->max_inline * RTE_CACHE_LINE_SIZE; } } if (mpw.state == MLX5_MPW_STATE_CLOSED) { @@ -1238,7 +952,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, /* Multi-segment packets must be alone in their MPW. */ assert((segs_n == 1) || (mpw.pkts_n == 0)); if (mpw.state == MLX5_MPW_STATE_OPENED) { - assert(inline_room == txq->max_inline); + assert(inline_room == + txq->max_inline * RTE_CACHE_LINE_SIZE); #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG) length = 0; #endif @@ -1303,7 +1018,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, ++j; if (mpw.pkts_n == MLX5_MPW_DSEG_MAX) { mlx5_mpw_inline_close(txq, &mpw); - inline_room = txq->max_inline; + inline_room = + txq->max_inline * RTE_CACHE_LINE_SIZE; } else { inline_room -= length; } @@ -1365,19 +1081,19 @@ rxq_cq_to_pkt_type(volatile struct mlx5_cqe64 *cqe) uint8_t flags = cqe->l4_hdr_type_etc; uint8_t info = cqe->rsvd0[0]; - if (info & IBV_EXP_CQ_RX_TUNNEL_PACKET) + if (info & MLX5_CQE_RX_TUNNEL_PACKET) pkt_type = TRANSPOSE(flags, - IBV_EXP_CQ_RX_OUTER_IPV4_PACKET, + MLX5_CQE_RX_OUTER_IPV4_PACKET, RTE_PTYPE_L3_IPV4) | TRANSPOSE(flags, - IBV_EXP_CQ_RX_OUTER_IPV6_PACKET, + MLX5_CQE_RX_OUTER_IPV6_PACKET, RTE_PTYPE_L3_IPV6) | TRANSPOSE(flags, - IBV_EXP_CQ_RX_IPV4_PACKET, + MLX5_CQE_RX_IPV4_PACKET, RTE_PTYPE_INNER_L3_IPV4) | TRANSPOSE(flags, - IBV_EXP_CQ_RX_IPV6_PACKET, + MLX5_CQE_RX_IPV6_PACKET, RTE_PTYPE_INNER_L3_IPV6); else pkt_type = @@ -1520,13 +1236,13 @@ rxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe64 *cqe) * of PKT_RX_EIP_CKSUM_BAD because the latter is not functional * (its value is 0). */ - if ((info & IBV_EXP_CQ_RX_TUNNEL_PACKET) && (rxq->csum_l2tun)) + if ((info & MLX5_CQE_RX_TUNNEL_PACKET) && (rxq->csum_l2tun)) ol_flags |= TRANSPOSE(~cqe->l4_hdr_type_etc, - IBV_EXP_CQ_RX_OUTER_IP_CSUM_OK, + MLX5_CQE_RX_OUTER_IP_CSUM_OK, PKT_RX_IP_CKSUM_BAD) | TRANSPOSE(~cqe->l4_hdr_type_etc, - IBV_EXP_CQ_RX_OUTER_TCP_UDP_CSUM_OK, + MLX5_CQE_RX_OUTER_TCP_UDP_CSUM_OK, PKT_RX_L4_CKSUM_BAD); return ol_flags; } @@ -1572,6 +1288,14 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) rte_prefetch0(wqe); rep = rte_mbuf_raw_alloc(rxq->mp); if (unlikely(rep == NULL)) { + ++rxq->stats.rx_nombuf; + if (!pkt) { + /* + * no buffers before we even started, + * bail out silently. + */ + break; + } while (pkt != seg) { assert(pkt != (*rxq->elts)[idx]); seg = NEXT(pkt); @@ -1579,7 +1303,6 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) __rte_mbuf_raw_free(pkt); pkt = seg; } - ++rxq->stats.rx_nombuf; break; } if (!pkt) { diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index f6e2cbac..05779ef5 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -40,22 +40,22 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #include <infiniband/mlx5_hw.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_mbuf.h> #include <rte_mempool.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5_utils.h" @@ -87,6 +87,8 @@ struct mlx5_txq_stats { struct fdir_queue { struct ibv_qp *qp; /* Associated RX QP. */ struct ibv_exp_rwq_ind_table *ind_table; /* Indirection table. */ + struct ibv_exp_wq *wq; /* Work queue. */ + struct ibv_cq *cq; /* Completion queue. */ }; struct priv; @@ -128,7 +130,7 @@ struct rxq_ctrl { struct ibv_cq *cq; /* Completion Queue. */ struct ibv_exp_wq *wq; /* Work Queue. */ struct ibv_exp_res_domain *rd; /* Resource Domain. */ - struct fdir_queue fdir_queue; /* Flow director queue. */ + struct fdir_queue *fdir_queue; /* Flow director queue. */ struct ibv_mr *mr; /* Memory Region (for mp). */ struct ibv_exp_wq_family *if_wq; /* WQ burst interface. */ struct ibv_exp_cq_family_v1 *if_cq; /* CQ interface. */ @@ -247,7 +249,7 @@ struct txq { uint16_t wqe_n; /* Number of WQ elements. */ uint16_t bf_offset; /* Blueflame offset. */ uint16_t bf_buf_size; /* Blueflame size. */ - uint16_t max_inline; /* Maximum size to inline in a WQE. */ + uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */ uint32_t qp_num_8s; /* QP number shifted by 8. */ volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */ volatile union mlx5_wqe (*wqes)[]; /* Work queue. */ @@ -312,7 +314,6 @@ uint16_t mlx5_tx_burst_secondary_setup(void *, struct rte_mbuf **, uint16_t); /* mlx5_rxtx.c */ uint16_t mlx5_tx_burst(void *, struct rte_mbuf **, uint16_t); -uint16_t mlx5_tx_burst_inline(void *, struct rte_mbuf **, uint16_t); uint16_t mlx5_tx_burst_mpw(void *, struct rte_mbuf **, uint16_t); uint16_t mlx5_tx_burst_mpw_inline(void *, struct rte_mbuf **, uint16_t); uint16_t mlx5_rx_burst(void *, struct rte_mbuf **, uint16_t); diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c index 2d3cb519..f2b5781a 100644 --- a/drivers/net/mlx5/mlx5_stats.c +++ b/drivers/net/mlx5/mlx5_stats.c @@ -33,11 +33,11 @@ /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ethdev.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c index e9b9a293..d4dccd88 100644 --- a/drivers/net/mlx5/mlx5_trigger.c +++ b/drivers/net/mlx5/mlx5_trigger.c @@ -33,14 +33,14 @@ /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ether.h> #include <rte_ethdev.h> #include <rte_interrupts.h> #include <rte_alarm.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index 6fe61c4a..e4510efe 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -40,23 +40,23 @@ /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <infiniband/verbs.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_mbuf.h> #include <rte_malloc.h> #include <rte_ethdev.h> #include <rte_common.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5_utils.h" @@ -338,9 +338,12 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), }; - if (priv->txq_inline && priv->txqs_n >= priv->txqs_inline) { - tmpl.txq.max_inline = priv->txq_inline; - attr.init.cap.max_inline_data = tmpl.txq.max_inline; + if (priv->txq_inline && (priv->txqs_n >= priv->txqs_inline)) { + tmpl.txq.max_inline = + ((priv->txq_inline + (RTE_CACHE_LINE_SIZE - 1)) / + RTE_CACHE_LINE_SIZE); + attr.init.cap.max_inline_data = + tmpl.txq.max_inline * RTE_CACHE_LINE_SIZE; } tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init); if (tmpl.qp == NULL) { diff --git a/drivers/net/mlx5/mlx5_vlan.c b/drivers/net/mlx5/mlx5_vlan.c index 4719e697..1b0fa40a 100644 --- a/drivers/net/mlx5/mlx5_vlan.c +++ b/drivers/net/mlx5/mlx5_vlan.c @@ -38,12 +38,12 @@ /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include <rte_ethdev.h> #include <rte_common.h> #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5_utils.h" @@ -87,7 +87,8 @@ vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) --priv->vlan_filter_n; memmove(&priv->vlan_filter[i], &priv->vlan_filter[i + 1], - priv->vlan_filter_n - i); + sizeof(priv->vlan_filter[i]) * + (priv->vlan_filter_n - i)); priv->vlan_filter[priv->vlan_filter_n] = 0; } else { assert(i == priv->vlan_filter_n); |