diff options
Diffstat (limited to 'drivers/net/mlx5')
26 files changed, 291 insertions, 212 deletions
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index e7668bd5..d91d55b5 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -157,9 +157,10 @@ mlx5_prepare_shared_data(void) if (rte_eal_process_type() == RTE_PROC_PRIMARY) { LIST_INIT(&mlx5_shared_data->mem_event_cb_list); rte_rwlock_init(&mlx5_shared_data->mem_event_rwlock); + rte_mem_event_callback_register("MLX5_MEM_EVENT_CB", + mlx5_mr_mem_event_cb, + NULL); } - rte_mem_event_callback_register("MLX5_MEM_EVENT_CB", - mlx5_mr_mem_event_cb, NULL); } rte_spinlock_unlock(&mlx5_shared_data_lock); } @@ -200,7 +201,7 @@ mlx5_getenv_int(const char *name) static void * mlx5_alloc_verbs_buf(size_t size, void *data) { - struct priv *priv = data; + struct mlx5_priv *priv = data; void *ret; size_t alignment = sysconf(_SC_PAGESIZE); unsigned int socket = SOCKET_ID_ANY; @@ -248,7 +249,7 @@ mlx5_free_verbs_buf(void *ptr, void *data __rte_unused) static void mlx5_dev_close(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; int ret; @@ -335,7 +336,7 @@ mlx5_dev_close(struct rte_eth_dev *dev) i = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, i), i); while (i--) { - struct priv *opriv = + struct mlx5_priv *opriv = rte_eth_devices[port_id[i]].data->dev_private; if (!opriv || @@ -630,7 +631,7 @@ find_lower_va_bound(const struct rte_memseg_list *msl, static int mlx5_uar_init_primary(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; void *addr = (void *)0; if (uar_base) { /* UAR address space mapped. */ @@ -676,7 +677,7 @@ mlx5_uar_init_primary(struct rte_eth_dev *dev) static int mlx5_uar_init_secondary(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; void *addr; assert(priv->uar_base); @@ -739,7 +740,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, struct ibv_pd *pd = NULL; struct mlx5dv_context dv_attr = { .comp_mask = 0 }; struct rte_eth_dev *eth_dev = NULL; - struct priv *priv = NULL; + struct mlx5_priv *priv = NULL; int err = 0; unsigned int hw_padding = 0; unsigned int mps; @@ -1001,7 +1002,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, i = RTE_MIN(mlx5_dev_to_port_id(dpdk_dev, port_id, i), i); while (i--) { - const struct priv *opriv = + const struct mlx5_priv *opriv = rte_eth_devices[port_id[i]].data->dev_private; if (!opriv || @@ -1233,8 +1234,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, priv->config = config; /* Supported Verbs flow priority number detection. */ err = mlx5_flow_discover_priorities(eth_dev); - if (err < 0) + if (err < 0) { + err = -err; goto error; + } priv->config.flow_prio = err; /* * Once the device is added to the list of memory event diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index bc500b2b..91efd21b 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -62,7 +62,7 @@ struct mlx5_switch_info { uint64_t switch_id; /**< Switch identifier. */ }; -LIST_HEAD(mlx5_dev_list, priv); +LIST_HEAD(mlx5_dev_list, mlx5_priv); /* Shared memory between primary and secondary processes. */ struct mlx5_shared_data { @@ -172,8 +172,9 @@ struct mlx5_drop { struct mlx5_flow_tcf_context; -struct priv { - LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */ +struct mlx5_priv { + LIST_ENTRY(mlx5_priv) mem_event_cb; + /**< Called by memory event callback. */ struct rte_eth_dev_data *dev_data; /* Pointer to device data. */ struct ibv_context *ctx; /* Verbs context. */ struct ibv_device_attr_ex device_attr; /* Device properties. */ diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h index bfe66558..480b33c8 100644 --- a/drivers/net/mlx5/mlx5_defs.h +++ b/drivers/net/mlx5/mlx5_defs.h @@ -63,10 +63,11 @@ /* Default maximum number of Tx queues for vectorized Tx. */ #if defined(RTE_ARCH_ARM64) #define MLX5_VPMD_MAX_TXQS 8 +#define MLX5_VPMD_MAX_TXQS_BLUEFIELD 16 #else #define MLX5_VPMD_MAX_TXQS 4 +#define MLX5_VPMD_MAX_TXQS_BLUEFIELD MLX5_VPMD_MAX_TXQS #endif -#define MLX5_VPMD_MAX_TXQS_BLUEFIELD 16 /* Threshold of buffer replenishment for vectorized Rx. */ #define MLX5_VPMD_RXQ_RPLNSH_THRESH(n) \ diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index d178ed6a..fb8e313a 100644 --- a/drivers/net/mlx5/mlx5_ethdev.c +++ b/drivers/net/mlx5/mlx5_ethdev.c @@ -131,7 +131,7 @@ static int mlx5_get_master_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; DIR *dir; struct dirent *dent; unsigned int dev_type = 0; @@ -219,7 +219,7 @@ try_dev_id: int mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int ifindex = priv->nl_socket_rdma >= 0 ? mlx5_nl_ifindex(priv->nl_socket_rdma, priv->ibdev_name) : 0; @@ -377,7 +377,7 @@ mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags) int mlx5_dev_configure(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int rxqs_n = dev->data->nb_rx_queues; unsigned int txqs_n = dev->data->nb_tx_queues; unsigned int i; @@ -460,7 +460,7 @@ mlx5_dev_configure(struct rte_eth_dev *dev) static void mlx5_set_default_params(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; /* Minimum CPU utilization. */ info->default_rxportconf.ring_size = 256; @@ -499,7 +499,7 @@ mlx5_set_default_params(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) void mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_config *config = &priv->config; unsigned int max; char ifname[IF_NAMESIZE]; @@ -540,7 +540,7 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) i = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, i), i); while (i--) { - struct priv *opriv = + struct mlx5_priv *opriv = rte_eth_devices[port_id[i]].data->dev_private; if (!opriv || @@ -609,7 +609,7 @@ static int mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, struct rte_eth_link *link) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct ethtool_cmd edata = { .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */ }; @@ -685,7 +685,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, struct rte_eth_link *link) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS }; struct ifreq ifr; struct rte_eth_link dev_link; @@ -840,7 +840,7 @@ mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; uint16_t kern_mtu = 0; int ret; @@ -1015,7 +1015,7 @@ mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, static uint32_t mlx5_dev_status_handler(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct ibv_async_event event; uint32_t ret = 0; @@ -1087,7 +1087,7 @@ mlx5_dev_handler_socket(void *cb_arg) void mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; if (dev->data->dev_conf.intr_conf.lsc || dev->data->dev_conf.intr_conf.rmv) @@ -1111,7 +1111,7 @@ mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev) void mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; int ret; int flags; @@ -1187,7 +1187,7 @@ mlx5_set_link_up(struct rte_eth_dev *dev) eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; eth_tx_burst_t tx_pkt_burst = mlx5_tx_burst; struct mlx5_dev_config *config = &priv->config; uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads; @@ -1271,7 +1271,7 @@ int mlx5_is_removed(struct rte_eth_dev *dev) { struct ibv_device_attr device_attr; - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; if (mlx5_glue->query_device(priv->ctx, &device_attr) == EIO) return 1; diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index ee129b97..222cd81d 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -350,6 +350,7 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev) claim_zero(mlx5_glue->destroy_flow(flow)); priority = vprio[i]; } + mlx5_hrxq_drop_release(dev); switch (priority) { case 8: priority = RTE_DIM(priority_map_3); @@ -361,10 +362,9 @@ mlx5_flow_discover_priorities(struct rte_eth_dev *dev) rte_errno = ENOTSUP; DRV_LOG(ERR, "port %u verbs maximum priority: %d expected 8/16", - dev->data->port_id, vprio[i]); + dev->data->port_id, priority); return -rte_errno; } - mlx5_hrxq_drop_release(dev); DRV_LOG(INFO, "port %u flow maximum priority: %d", dev->data->port_id, priority); return priority; @@ -387,7 +387,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, uint32_t subpriority) { uint32_t res = 0; - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; switch (priv->config.flow_prio) { case RTE_DIM(priority_map_3): @@ -536,7 +536,7 @@ flow_rxq_tunnel_ptype_update(struct mlx5_rxq_ctrl *rxq_ctrl) static void flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow *flow = dev_flow->flow; const int mark = !!(flow->actions & (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); @@ -599,7 +599,7 @@ flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow) static void flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow *flow = dev_flow->flow; const int mark = !!(flow->actions & (MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK)); @@ -661,7 +661,7 @@ flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow) static void flow_rxq_flags_clear(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; for (i = 0; i != priv->rxqs_n; ++i) { @@ -786,7 +786,7 @@ mlx5_flow_validate_action_mark(const struct rte_flow_action *action, * Pointer to error structure. * * @return - * 0 on success, a negative errno value otherwise and rte_ernno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_flow_validate_action_drop(uint64_t action_flags, @@ -829,7 +829,7 @@ mlx5_flow_validate_action_drop(uint64_t action_flags, * Pointer to error structure. * * @return - * 0 on success, a negative errno value otherwise and rte_ernno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_flow_validate_action_queue(const struct rte_flow_action *action, @@ -838,7 +838,7 @@ mlx5_flow_validate_action_queue(const struct rte_flow_action *action, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_action_queue *queue = action->conf; if (action_flags & MLX5_FLOW_FATE_ACTIONS) @@ -875,21 +875,25 @@ mlx5_flow_validate_action_queue(const struct rte_flow_action *action, * Pointer to the Ethernet device structure. * @param[in] attr * Attributes of flow that includes this action. + * @param[in] item_flags + * Items that were detected. * @param[out] error * Pointer to error structure. * * @return - * 0 on success, a negative errno value otherwise and rte_ernno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_flow_validate_action_rss(const struct rte_flow_action *action, uint64_t action_flags, struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + uint64_t item_flags, struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_action_rss *rss = action->conf; + int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); unsigned int i; if (action_flags & MLX5_FLOW_FATE_ACTIONS) @@ -950,6 +954,11 @@ mlx5_flow_validate_action_rss(const struct rte_flow_action *action, RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL, "rss action not supported for " "egress"); + if (rss->level > 1 && !tunnel) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION_CONF, NULL, + "inner RSS is not supported for " + "non-tunnel flows"); return 0; } @@ -964,7 +973,7 @@ mlx5_flow_validate_action_rss(const struct rte_flow_action *action, * Pointer to error structure. * * @return - * 0 on success, a negative errno value otherwise and rte_ernno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused, @@ -998,7 +1007,7 @@ mlx5_flow_validate_attributes(struct rte_eth_dev *dev, const struct rte_flow_attr *attributes, struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; uint32_t priority_max = priv->config.flow_prio - 1; if (attributes->group) @@ -1462,7 +1471,7 @@ mlx5_flow_validate_item_vxlan_gpe(const struct rte_flow_item *item, struct rte_eth_dev *dev, struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_item_vxlan_gpe *spec = item->spec; const struct rte_flow_item_vxlan_gpe *mask = item->mask; int ret; @@ -1616,7 +1625,7 @@ mlx5_flow_validate_item_mpls(struct rte_eth_dev *dev __rte_unused, { #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT const struct rte_flow_item_mpls *mask = item->mask; - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; int ret; if (!priv->config.mpls_en) @@ -1747,7 +1756,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_null_drv_ops = { static enum mlx5_flow_drv_type flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; enum mlx5_flow_drv_type type = MLX5_FLOW_TYPE_MAX; if (attr->transfer) @@ -1776,7 +1785,7 @@ flow_get_drv_type(struct rte_eth_dev *dev, const struct rte_flow_attr *attr) * Pointer to the error structure. * * @return - * 0 on success, a negative errno value otherwise and rte_ernno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static inline int flow_drv_validate(struct rte_eth_dev *dev, @@ -1815,7 +1824,7 @@ flow_drv_validate(struct rte_eth_dev *dev, * Pointer to the error structure. * * @return - * Pointer to device flow on success, otherwise NULL and rte_ernno is set. + * Pointer to device flow on success, otherwise NULL and rte_errno is set. */ static inline struct mlx5_flow * flow_drv_prepare(const struct rte_flow *flow, @@ -1859,7 +1868,7 @@ flow_drv_prepare(const struct rte_flow *flow, * Pointer to the error structure. * * @return - * 0 on success, a negative errno value otherwise and rte_ernno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static inline int flow_drv_translate(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, @@ -2121,8 +2130,9 @@ mlx5_flow_create(struct rte_eth_dev *dev, const struct rte_flow_action actions[], struct rte_flow_error *error) { - return flow_list_create(dev, - &((struct priv *)dev->data->dev_private)->flows, + struct mlx5_priv *priv = (struct mlx5_priv *)dev->data->dev_private; + + return flow_list_create(dev, &priv->flows, attr, items, actions, error); } @@ -2232,7 +2242,7 @@ error: int mlx5_flow_verify(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow *flow; int ret = 0; @@ -2268,7 +2278,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, struct rte_flow_item_vlan *vlan_spec, struct rte_flow_item_vlan *vlan_mask) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_attr attr = { .ingress = 1, .priority = MLX5_FLOW_PRIO_RSVD, @@ -2359,7 +2369,7 @@ mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, struct rte_flow_error *error __rte_unused) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; flow_list_destroy(dev, &priv->flows, flow); return 0; @@ -2375,7 +2385,7 @@ int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error __rte_unused) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; mlx5_flow_list_flush(dev, &priv->flows); return 0; @@ -2392,7 +2402,7 @@ mlx5_flow_isolate(struct rte_eth_dev *dev, int enable, struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; if (dev->data->dev_started) { rte_flow_error_set(error, EBUSY, @@ -2470,7 +2480,7 @@ flow_fdir_filter_convert(struct rte_eth_dev *dev, const struct rte_eth_fdir_filter *fdir_filter, struct mlx5_fdir *attributes) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; const struct rte_eth_fdir_input *input = &fdir_filter->input; const struct rte_eth_fdir_masks *mask = &dev->data->dev_conf.fdir_conf.mask; @@ -2687,7 +2697,7 @@ flow_fdir_cmp(const struct mlx5_fdir *f1, const struct mlx5_fdir *f2) static struct rte_flow * flow_fdir_filter_lookup(struct rte_eth_dev *dev, struct mlx5_fdir *fdir_flow) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow *flow = NULL; assert(fdir_flow); @@ -2716,7 +2726,7 @@ static int flow_fdir_filter_add(struct rte_eth_dev *dev, const struct rte_eth_fdir_filter *fdir_filter) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_fdir *fdir_flow; struct rte_flow *flow; int ret; @@ -2763,7 +2773,7 @@ static int flow_fdir_filter_delete(struct rte_eth_dev *dev, const struct rte_eth_fdir_filter *fdir_filter) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow *flow; struct mlx5_fdir fdir_flow = { .attr.group = 0, @@ -2816,7 +2826,7 @@ flow_fdir_filter_update(struct rte_eth_dev *dev, static void flow_fdir_filter_flush(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; mlx5_flow_list_flush(dev, &priv->flows); } diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 4a7c0529..e1424c78 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -363,6 +363,7 @@ int mlx5_flow_validate_action_rss(const struct rte_flow_action *action, uint64_t action_flags, struct rte_eth_dev *dev, const struct rte_flow_attr *attr, + uint64_t item_flags, struct rte_flow_error *error); int mlx5_flow_validate_attributes(struct rte_eth_dev *dev, const struct rte_flow_attr *attributes, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 1f318748..207edcbc 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -293,7 +293,7 @@ flow_dv_encap_decap_resource_register struct mlx5_flow *dev_flow, struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_encap_decap_resource *cache_resource; /* Lookup a matching resource from cache. */ @@ -722,7 +722,7 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, const struct rte_flow_attr *attributes, struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; uint32_t priority_max = priv->config.flow_prio - 1; if (attributes->group) @@ -764,7 +764,7 @@ flow_dv_validate_attributes(struct rte_eth_dev *dev, * Pointer to the error structure. * * @return - * 0 on success, a negative errno value otherwise and rte_ernno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, @@ -776,7 +776,6 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, uint64_t action_flags = 0; uint64_t item_flags = 0; uint64_t last_item = 0; - int tunnel = 0; uint8_t next_protocol = 0xff; int actions_n = 0; @@ -786,7 +785,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, if (ret < 0) return ret; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { - tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); + int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); switch (items->type) { case RTE_FLOW_ITEM_TYPE_VOID: break; @@ -958,7 +957,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, case RTE_FLOW_ACTION_TYPE_RSS: ret = mlx5_flow_validate_action_rss(actions, action_flags, dev, - attr, error); + attr, item_flags, + error); if (ret < 0) return ret; action_flags |= MLX5_FLOW_ACTION_RSS; @@ -1043,7 +1043,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, * * @return * Pointer to mlx5_flow object on success, - * otherwise NULL and rte_ernno is set. + * otherwise NULL and rte_errno is set. */ static struct mlx5_flow * flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused, @@ -1800,7 +1800,7 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow, struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_matcher *cache_matcher; struct mlx5dv_flow_matcher_attr dv_attr = { .type = IBV_FLOW_ATTR_NORMAL, @@ -1873,7 +1873,7 @@ flow_dv_matcher_register(struct rte_eth_dev *dev, * Pointer to the error structure. * * @return - * 0 on success, a negative errno value otherwise and rte_ernno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int flow_dv_translate(struct rte_eth_dev *dev, @@ -1883,7 +1883,7 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action actions[], struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow *flow = dev_flow->flow; uint64_t item_flags = 0; uint64_t last_item = 0; diff --git a/drivers/net/mlx5/mlx5_flow_tcf.c b/drivers/net/mlx5/mlx5_flow_tcf.c index 96b9dd72..92f984f9 100644 --- a/drivers/net/mlx5/mlx5_flow_tcf.c +++ b/drivers/net/mlx5/mlx5_flow_tcf.c @@ -2344,7 +2344,7 @@ flow_tcf_validate(struct rte_eth_dev *dev, */ if ((action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) && (action_flags & MLX5_FLOW_ACTION_PORT_ID) && - ((struct priv *)port_id_dev->data->dev_private)->representor) + ((struct mlx5_priv *)port_id_dev->data->dev_private)->representor) return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, actions, "vlan push can only be applied" @@ -5321,7 +5321,7 @@ flow_tcf_check_inhw(struct mlx5_flow_tcf_context *tcf, static void flow_tcf_remove(struct rte_eth_dev *dev, struct rte_flow *flow) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_tcf_context *ctx = priv->tcf_context; struct mlx5_flow *dev_flow; struct nlmsghdr *nlh; @@ -5410,7 +5410,7 @@ static int flow_tcf_apply(struct rte_eth_dev *dev, struct rte_flow *flow, struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_tcf_context *ctx = priv->tcf_context; struct mlx5_flow *dev_flow; struct nlmsghdr *nlh; @@ -5894,7 +5894,7 @@ flow_tcf_query_count(struct rte_eth_dev *dev, { struct flow_tcf_stats_basic sb_data; struct rte_flow_query_count *qc = data; - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_tcf_context *ctx = priv->tcf_context; struct mnl_socket *nl = ctx->nl; struct mlx5_flow *dev_flow; diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c index 409e1cd0..1fdbca3d 100644 --- a/drivers/net/mlx5/mlx5_flow_verbs.c +++ b/drivers/net/mlx5/mlx5_flow_verbs.c @@ -55,7 +55,7 @@ flow_verbs_counter_create(struct rte_eth_dev *dev, struct mlx5_flow_counter *counter) { #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct ibv_counter_set_init_attr init = { .counter_set_id = counter->id}; @@ -66,7 +66,7 @@ flow_verbs_counter_create(struct rte_eth_dev *dev, } return 0; #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct ibv_counters_init_attr init = {0}; struct ibv_counter_attach_attr attach; int ret; @@ -117,7 +117,7 @@ flow_verbs_counter_create(struct rte_eth_dev *dev, static struct mlx5_flow_counter * flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_counter *cnt; int ret; @@ -1191,7 +1191,7 @@ flow_verbs_validate(struct rte_eth_dev *dev, case RTE_FLOW_ACTION_TYPE_RSS: ret = mlx5_flow_validate_action_rss(actions, action_flags, dev, - attr, + attr, item_flags, error); if (ret < 0) return ret; @@ -1383,7 +1383,7 @@ flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused, * Pointer to the error structure. * * @return - * 0 on success, else a negative errno value otherwise and rte_ernno is set. + * 0 on success, else a negative errno value otherwise and rte_errno is set. */ static int flow_verbs_translate(struct rte_eth_dev *dev, @@ -1398,7 +1398,7 @@ flow_verbs_translate(struct rte_eth_dev *dev, uint64_t action_flags = 0; uint64_t priority = attr->priority; uint32_t subpriority = 0; - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; if (priority == MLX5_FLOW_PRIO_RSVD) priority = priv->config.flow_prio - 1; diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c index 672a4761..bce026f9 100644 --- a/drivers/net/mlx5/mlx5_mac.c +++ b/drivers/net/mlx5/mlx5_mac.c @@ -67,7 +67,7 @@ mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN]) static void mlx5_internal_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; const int vf = priv->config.vf; assert(index < MLX5_MAX_MAC_ADDRESSES); @@ -96,7 +96,7 @@ static int mlx5_internal_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac, uint32_t index) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; const int vf = priv->config.vf; unsigned int i; diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c index 442b2d23..01bf5910 100644 --- a/drivers/net/mlx5/mlx5_mr.c +++ b/drivers/net/mlx5/mlx5_mr.c @@ -336,7 +336,7 @@ mr_find_next_chunk(struct mlx5_mr *mr, struct mlx5_mr_cache *entry, static int mr_insert_dev_cache(struct rte_eth_dev *dev, struct mlx5_mr *mr) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int n; DRV_LOG(DEBUG, "port %u inserting MR(%p) to global cache", @@ -377,7 +377,7 @@ static struct mlx5_mr * mr_lookup_dev_list(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry, uintptr_t addr) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_mr *mr; /* Iterate all the existing MRs. */ @@ -418,7 +418,7 @@ static uint32_t mr_lookup_dev(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry, uintptr_t addr) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; uint16_t idx; uint32_t lkey = UINT32_MAX; struct mlx5_mr *mr; @@ -465,7 +465,7 @@ mr_free(struct mlx5_mr *mr) } /** - * Releass resources of detached MR having no online entry. + * Release resources of detached MR having no online entry. * * @param dev * Pointer to Ethernet device. @@ -473,7 +473,7 @@ mr_free(struct mlx5_mr *mr) static void mlx5_mr_garbage_collect(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_mr *mr_next; struct mlx5_mr_list free_list = LIST_HEAD_INITIALIZER(free_list); @@ -515,7 +515,7 @@ mr_find_contig_memsegs_cb(const struct rte_memseg_list *msl, } /** - * Create a new global Memroy Region (MR) for a missing virtual address. + * Create a new global Memory Region (MR) for a missing virtual address. * Register entire virtually contiguous memory chunk around the address. * * @param dev @@ -533,7 +533,7 @@ static uint32_t mlx5_mr_create(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry, uintptr_t addr) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; const struct rte_memseg_list *msl; const struct rte_memseg *ms; @@ -623,7 +623,7 @@ alloc_resources: bmp_mem = RTE_PTR_ALIGN_CEIL(mr + 1, RTE_CACHE_LINE_SIZE); mr->ms_bmp = rte_bitmap_init(ms_n, bmp_mem, bmp_size); if (mr->ms_bmp == NULL) { - DEBUG("port %u unable to initialize bitamp for a new MR of" + DEBUG("port %u unable to initialize bitmap for a new MR of" " address (%p).", dev->data->port_id, (void *)addr); rte_errno = EINVAL; @@ -769,7 +769,7 @@ err_nolock: static void mr_rebuild_dev_cache(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_mr *mr; DRV_LOG(DEBUG, "port %u rebuild dev cache[]", dev->data->port_id); @@ -803,7 +803,7 @@ mr_rebuild_dev_cache(struct rte_eth_dev *dev) static void mlx5_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; const struct rte_memseg_list *msl; struct mlx5_mr *mr; int ms_n; @@ -888,9 +888,11 @@ void mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr, size_t len, void *arg __rte_unused) { - struct priv *priv; + struct mlx5_priv *priv; struct mlx5_dev_list *dev_list = &mlx5_shared_data->mem_event_cb_list; + /* Must be called from the primary process. */ + assert(rte_eal_process_type() == RTE_PROC_PRIMARY); switch (event_type) { case RTE_MEM_EVENT_FREE: rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock); @@ -926,7 +928,7 @@ static uint32_t mlx5_mr_lookup_dev(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl, struct mlx5_mr_cache *entry, uintptr_t addr) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_mr_btree *bt = &mr_ctrl->cache_bh; uint16_t idx; uint32_t lkey; @@ -1026,11 +1028,8 @@ mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr) struct mlx5_rxq_ctrl *rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl; - struct priv *priv = rxq_ctrl->priv; + struct mlx5_priv *priv = rxq_ctrl->priv; - DRV_LOG(DEBUG, - "Rx queue %u: miss on top-half, mru=%u, head=%u, addr=%p", - rxq_ctrl->idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr); return mlx5_mr_addr2mr_bh(ETH_DEV(priv), mr_ctrl, addr); } @@ -1051,11 +1050,8 @@ mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr) struct mlx5_txq_ctrl *txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl; - struct priv *priv = txq_ctrl->priv; + struct mlx5_priv *priv = txq_ctrl->priv; - DRV_LOG(DEBUG, - "Tx queue %u: miss on top-half, mru=%u, head=%u, addr=%p", - txq_ctrl->idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr); return mlx5_mr_addr2mr_bh(ETH_DEV(priv), mr_ctrl, addr); } @@ -1128,7 +1124,7 @@ mlx5_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque, { struct mr_update_mp_data *data = opaque; struct rte_eth_dev *dev = data->dev; - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_mr_ctrl *mr_ctrl = data->mr_ctrl; struct mlx5_mr *mr = NULL; uintptr_t addr = (uintptr_t)memhdr->addr; @@ -1136,6 +1132,7 @@ mlx5_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque, struct mlx5_mr_cache entry; uint32_t lkey; + assert(rte_eal_process_type() == RTE_PROC_PRIMARY); /* If already registered, it should return. */ rte_rwlock_read_lock(&priv->mr.rwlock); lkey = mr_lookup_dev(dev, &entry, addr); @@ -1235,8 +1232,17 @@ mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr, struct mlx5_txq_ctrl *txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl; - struct priv *priv = txq_ctrl->priv; + struct mlx5_priv *priv = txq_ctrl->priv; + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + DRV_LOG(WARNING, + "port %u using address (%p) from unregistered mempool" + " having externally allocated memory" + " in secondary process, please create mempool" + " prior to rte_eth_dev_start()", + PORT_ID(priv), (void *)addr); + return UINT32_MAX; + } mlx5_mr_update_ext_mp(ETH_DEV(priv), mr_ctrl, mp); return mlx5_tx_addr2mr_bh(txq, addr); } @@ -1301,7 +1307,7 @@ void mlx5_mr_dump_dev(struct rte_eth_dev *dev __rte_unused) { #ifndef NDEBUG - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_mr *mr; int mr_n = 0; int chunk_n = 0; @@ -1343,8 +1349,8 @@ mlx5_mr_dump_dev(struct rte_eth_dev *dev __rte_unused) void mlx5_mr_release(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - struct mlx5_mr *mr_next = LIST_FIRST(&priv->mr.mr_list); + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_mr *mr_next; /* Remove from memory callback device list. */ rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock); @@ -1354,6 +1360,7 @@ mlx5_mr_release(struct rte_eth_dev *dev) mlx5_mr_dump_dev(dev); rte_rwlock_write_lock(&priv->mr.rwlock); /* Detach from MR list and move to free list. */ + mr_next = LIST_FIRST(&priv->mr.mr_list); while (mr_next != NULL) { struct mlx5_mr *mr = mr_next; diff --git a/drivers/net/mlx5/mlx5_nl.c b/drivers/net/mlx5/mlx5_nl.c index d61826ae..fe5a2746 100644 --- a/drivers/net/mlx5/mlx5_nl.c +++ b/drivers/net/mlx5/mlx5_nl.c @@ -361,7 +361,7 @@ static int mlx5_nl_mac_addr_list(struct rte_eth_dev *dev, struct ether_addr (*mac)[], int *mac_n) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int iface_idx = mlx5_ifindex(dev); struct { struct nlmsghdr hdr; @@ -420,7 +420,7 @@ static int mlx5_nl_mac_addr_modify(struct rte_eth_dev *dev, struct ether_addr *mac, int add) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int iface_idx = mlx5_ifindex(dev); struct { struct nlmsghdr hdr; @@ -492,7 +492,7 @@ int mlx5_nl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac, uint32_t index) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; int ret; ret = mlx5_nl_mac_addr_modify(dev, mac, 1); @@ -520,7 +520,7 @@ int mlx5_nl_mac_addr_remove(struct rte_eth_dev *dev, struct ether_addr *mac, uint32_t index) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; BITFIELD_RESET(priv->mac_own, index); return mlx5_nl_mac_addr_modify(dev, mac, 0); @@ -572,7 +572,7 @@ mlx5_nl_mac_addr_sync(struct rte_eth_dev *dev) void mlx5_nl_mac_addr_flush(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; int i; for (i = MLX5_MAX_MAC_ADDRESSES - 1; i >= 0; --i) { @@ -599,7 +599,7 @@ mlx5_nl_mac_addr_flush(struct rte_eth_dev *dev) static int mlx5_nl_device_flags(struct rte_eth_dev *dev, uint32_t flags, int enable) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int iface_idx = mlx5_ifindex(dev); struct { struct nlmsghdr hdr; diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c index b95778a8..891d764b 100644 --- a/drivers/net/mlx5/mlx5_rss.c +++ b/drivers/net/mlx5/mlx5_rss.c @@ -41,7 +41,7 @@ int mlx5_rss_hash_update(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; unsigned int idx; @@ -95,7 +95,7 @@ int mlx5_rss_hash_conf_get(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; if (!rss_conf) { rte_errno = EINVAL; @@ -125,7 +125,7 @@ mlx5_rss_hash_conf_get(struct rte_eth_dev *dev, int mlx5_rss_reta_index_resize(struct rte_eth_dev *dev, unsigned int reta_size) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; void *mem; unsigned int old_size = priv->reta_idx_n; @@ -165,7 +165,7 @@ mlx5_dev_rss_reta_query(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int idx; unsigned int i; @@ -201,7 +201,7 @@ mlx5_dev_rss_reta_update(struct rte_eth_dev *dev, uint16_t reta_size) { int ret; - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int idx; unsigned int i; unsigned int pos; diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c index e74fdef8..d5077db0 100644 --- a/drivers/net/mlx5/mlx5_rxmode.c +++ b/drivers/net/mlx5/mlx5_rxmode.c @@ -32,7 +32,7 @@ void mlx5_promiscuous_enable(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; int ret; dev->data->promiscuous = 1; @@ -60,7 +60,7 @@ mlx5_promiscuous_enable(struct rte_eth_dev *dev) void mlx5_promiscuous_disable(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; int ret; dev->data->promiscuous = 0; @@ -81,7 +81,7 @@ mlx5_promiscuous_disable(struct rte_eth_dev *dev) void mlx5_allmulticast_enable(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; int ret; dev->data->all_multicast = 1; @@ -109,7 +109,7 @@ mlx5_allmulticast_enable(struct rte_eth_dev *dev) void mlx5_allmulticast_disable(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; int ret; dev->data->all_multicast = 0; diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index 10b6ce0c..f1ce3170 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -68,7 +68,7 @@ static_assert(MLX5_RSS_HASH_KEY_LEN == inline int mlx5_check_mprq_support(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; if (priv->config.mprq.enabled && priv->rxqs_n >= priv->config.mprq.min_rxqs_num) @@ -103,7 +103,7 @@ mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq) inline int mlx5_mprq_enabled(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; uint16_t i; uint16_t n = 0; @@ -382,7 +382,7 @@ mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *rxq_ctrl) uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_config *config = &priv->config; uint64_t offloads = (DEV_RX_OFFLOAD_SCATTER | DEV_RX_OFFLOAD_TIMESTAMP | @@ -438,7 +438,7 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, unsigned int socket, const struct rte_eth_rxconf *conf, struct rte_mempool *mp) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx]; struct mlx5_rxq_ctrl *rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); @@ -489,7 +489,7 @@ mlx5_rx_queue_release(void *dpdk_rxq) { struct mlx5_rxq_data *rxq = (struct mlx5_rxq_data *)dpdk_rxq; struct mlx5_rxq_ctrl *rxq_ctrl; - struct priv *priv; + struct mlx5_priv *priv; if (rxq == NULL) return; @@ -514,7 +514,7 @@ mlx5_rx_queue_release(void *dpdk_rxq) int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; unsigned int rxqs_n = priv->rxqs_n; unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); @@ -592,7 +592,7 @@ mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev) void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct rte_intr_handle *intr_handle = dev->intr_handle; unsigned int i; unsigned int rxqs_n = priv->rxqs_n; @@ -664,7 +664,7 @@ mlx5_arm_cq(struct mlx5_rxq_data *rxq, int sq_n_rxq) int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq_data; struct mlx5_rxq_ctrl *rxq_ctrl; @@ -702,7 +702,7 @@ mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id) int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq_data; struct mlx5_rxq_ctrl *rxq_ctrl; struct mlx5_rxq_ibv *rxq_ibv = NULL; @@ -730,6 +730,7 @@ mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) } rxq_data->cq_arm_sn++; mlx5_glue->ack_cq_events(rxq_ibv->cq, 1); + mlx5_rxq_ibv_release(rxq_ibv); return 0; exit: ret = rte_errno; /* Save rte_errno before cleanup. */ @@ -755,7 +756,7 @@ exit: struct mlx5_rxq_ibv * mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; struct mlx5_rxq_ctrl *rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); @@ -1042,7 +1043,7 @@ error: struct mlx5_rxq_ibv * mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; struct mlx5_rxq_ctrl *rxq_ctrl; @@ -1098,7 +1099,7 @@ mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv) int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; int ret = 0; struct mlx5_rxq_ibv *rxq_ibv; @@ -1149,7 +1150,7 @@ mlx5_mprq_buf_init(struct rte_mempool *mp, void *opaque_arg __rte_unused, int mlx5_mprq_free_mp(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct rte_mempool *mp = priv->mprq_mp; unsigned int i; @@ -1200,7 +1201,7 @@ mlx5_mprq_free_mp(struct rte_eth_dev *dev) int mlx5_mprq_alloc_mp(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct rte_mempool *mp = priv->mprq_mp; char name[RTE_MEMPOOL_NAMESIZE]; unsigned int desc = 0; @@ -1272,7 +1273,7 @@ mlx5_mprq_alloc_mp(struct rte_eth_dev *dev) return -rte_errno; } } - snprintf(name, sizeof(name), "%s-mprq", dev->device->name); + snprintf(name, sizeof(name), "port-%u-mprq", dev->data->port_id); mp = rte_mempool_create(name, obj_num, obj_size, MLX5_MPRQ_MP_CACHE_SZ, 0, NULL, NULL, mlx5_mprq_buf_init, NULL, dev->device->numa_node, 0); @@ -1319,7 +1320,7 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, unsigned int socket, const struct rte_eth_rxconf *conf, struct rte_mempool *mp) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *tmpl; unsigned int mb_len = rte_pktmbuf_data_room_size(mp); unsigned int mprq_stride_size; @@ -1494,7 +1495,7 @@ error: * @param dev * Pointer to Ethernet device. * @param idx - * TX queue index. + * RX queue index. * * @return * A pointer to the queue if it exists, NULL otherwise. @@ -1502,7 +1503,7 @@ error: struct mlx5_rxq_ctrl * mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *rxq_ctrl = NULL; if ((*priv->rxqs)[idx]) { @@ -1521,7 +1522,7 @@ mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx) * @param dev * Pointer to Ethernet device. * @param idx - * TX queue index. + * RX queue index. * * @return * 1 while a reference on it exists, 0 when freed. @@ -1529,7 +1530,7 @@ mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx) int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *rxq_ctrl; if (!(*priv->rxqs)[idx]) @@ -1554,7 +1555,7 @@ mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx) * @param dev * Pointer to Ethernet device. * @param idx - * TX queue index. + * RX queue index. * * @return * 1 if the queue can be released, negative errno otherwise and rte_errno is @@ -1563,7 +1564,7 @@ mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx) int mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *rxq_ctrl; if (!(*priv->rxqs)[idx]) { @@ -1586,7 +1587,7 @@ mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx) int mlx5_rxq_verify(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *rxq_ctrl; int ret = 0; @@ -1615,7 +1616,7 @@ struct mlx5_ind_table_ibv * mlx5_ind_table_ibv_new(struct rte_eth_dev *dev, const uint16_t *queues, uint32_t queues_n) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ind_table_ibv *ind_tbl; const unsigned int wq_n = rte_is_power_of_2(queues_n) ? log2above(queues_n) : @@ -1679,7 +1680,7 @@ struct mlx5_ind_table_ibv * mlx5_ind_table_ibv_get(struct rte_eth_dev *dev, const uint16_t *queues, uint32_t queues_n) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ind_table_ibv *ind_tbl; LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) { @@ -1741,7 +1742,7 @@ mlx5_ind_table_ibv_release(struct rte_eth_dev *dev, int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ind_table_ibv *ind_tbl; int ret = 0; @@ -1783,7 +1784,7 @@ mlx5_hrxq_new(struct rte_eth_dev *dev, const uint16_t *queues, uint32_t queues_n, int tunnel __rte_unused) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq; struct mlx5_ind_table_ibv *ind_tbl; struct ibv_qp *qp; @@ -1899,7 +1900,7 @@ mlx5_hrxq_get(struct rte_eth_dev *dev, uint64_t hash_fields, const uint16_t *queues, uint32_t queues_n) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq; queues_n = hash_fields ? queues_n : 1; @@ -1962,7 +1963,7 @@ mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq) int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq; int ret = 0; @@ -1987,7 +1988,7 @@ mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev) struct mlx5_rxq_ibv * mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct ibv_cq *cq; struct ibv_wq *wq = NULL; struct mlx5_rxq_ibv *rxq; @@ -2046,7 +2047,7 @@ error: void mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_ibv *rxq = priv->drop_queue.rxq; if (rxq->wq) @@ -2069,7 +2070,7 @@ mlx5_rxq_ibv_drop_release(struct rte_eth_dev *dev) struct mlx5_ind_table_ibv * mlx5_ind_table_ibv_drop_new(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ind_table_ibv *ind_tbl; struct mlx5_rxq_ibv *rxq; struct mlx5_ind_table_ibv tmpl; @@ -2112,7 +2113,7 @@ error: void mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ind_table_ibv *ind_tbl = priv->drop_queue.hrxq->ind_table; claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl->ind_table)); @@ -2133,7 +2134,7 @@ mlx5_ind_table_ibv_drop_release(struct rte_eth_dev *dev) struct mlx5_hrxq * mlx5_hrxq_drop_new(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ind_table_ibv *ind_tbl; struct ibv_qp *qp; struct mlx5_hrxq *hrxq; @@ -2196,7 +2197,7 @@ error: void mlx5_hrxq_drop_release(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq; if (rte_atomic32_dec_and_test(&hrxq->refcnt)) { diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index 6eceea5f..38ce0e29 100644 --- a/drivers/net/mlx5/mlx5_rxtx.c +++ b/drivers/net/mlx5/mlx5_rxtx.c @@ -509,7 +509,7 @@ mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset) uint32_t mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq; if (dev->rx_pkt_burst != mlx5_rx_burst) { @@ -693,7 +693,8 @@ pkt_inline: RTE_CACHE_LINE_SIZE); copy_b = (addr_end > addr) ? RTE_MIN((addr_end - addr), length) : 0; - if (copy_b && ((end - (uintptr_t)raw) > copy_b)) { + if (copy_b && ((end - (uintptr_t)raw) > + (copy_b + sizeof(inl)))) { /* * One Dseg remains in the current WQE. To * keep the computation positive, it is diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index 75194a3f..820675b8 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -58,7 +58,7 @@ struct mlx5_txq_stats { uint64_t oerrors; /**< Total number of failed transmitted packets. */ }; -struct priv; +struct mlx5_priv; /* Compressed CQE context. */ struct rxq_zip { @@ -143,7 +143,7 @@ struct mlx5_rxq_ctrl { LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */ rte_atomic32_t refcnt; /* Reference counter. */ struct mlx5_rxq_ibv *ibv; /* Verbs elements. */ - struct priv *priv; /* Back pointer to private data. */ + struct mlx5_priv *priv; /* Back pointer to private data. */ struct mlx5_rxq_data rxq; /* Data path structure. */ unsigned int socket; /* CPU socket ID for allocations. */ unsigned int irq:1; /* Whether IRQ is enabled. */ @@ -228,7 +228,7 @@ struct mlx5_txq_ctrl { unsigned int max_inline_data; /* Max inline data. */ unsigned int max_tso_header; /* Max TSO header size. */ struct mlx5_txq_ibv *ibv; /* Verbs queue object. */ - struct priv *priv; /* Back pointer to private data. */ + struct mlx5_priv *priv; /* Back pointer to private data. */ struct mlx5_txq_data txq; /* Data path structure. */ off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */ volatile void *bf_reg_orig; /* Blueflame register from verbs. */ @@ -491,7 +491,7 @@ check_cqe(volatile struct mlx5_cqe *cqe, op_code, op_code, syndrome); rte_hexdump(stderr, "MLX5 Error CQE:", (const void *)((uintptr_t)err_cqe), - sizeof(*err_cqe)); + sizeof(*cqe)); } return 1; } else if ((op_code != MLX5_CQE_RESP_SEND) && @@ -568,6 +568,7 @@ mlx5_tx_complete(struct mlx5_txq_data *txq) } #endif /* NDEBUG */ ++cq_ci; + rte_cio_rmb(); txq->wqe_pi = rte_be_to_cpu_16(cqe->wqe_counter); ctrl = (volatile struct mlx5_wqe_ctrl *) tx_mlx5_wqe(txq, txq->wqe_pi); diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c index 340292ad..9a3a5ae4 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec.c +++ b/drivers/net/mlx5/mlx5_rxtx_vec.c @@ -273,7 +273,7 @@ mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev) int __attribute__((cold)) mlx5_check_vec_tx_support(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; uint64_t offloads = dev->data->dev_conf.txmode.offloads; if (!priv->config.tx_vec_en || @@ -318,7 +318,7 @@ mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) int __attribute__((cold)) mlx5_check_vec_rx_support(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; uint16_t i; if (!priv->config.rx_vec_en) diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h index fda7004e..86735044 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec.h @@ -102,7 +102,22 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n) return; } for (i = 0; i < n; ++i) { - wq[i].addr = rte_cpu_to_be_64((uintptr_t)elts[i]->buf_addr + + void *buf_addr; + + /* + * Load the virtual address for Rx WQE. non-x86 processors + * (mostly RISC such as ARM and Power) are more vulnerable to + * load stall. For x86, reducing the number of instructions + * seems to matter most. + */ +#ifdef RTE_ARCH_X86_64 + buf_addr = elts[i]->buf_addr; +#else + buf_addr = (char *)elts[i] + sizeof(struct rte_mbuf) + + rte_pktmbuf_priv_size(rxq->mp); + assert(buf_addr == elts[i]->buf_addr); +#endif + wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr + RTE_PKTMBUF_HEADROOM); /* If there's only one MR, no need to replace LKey in WQE. */ if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1)) diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h index 883fe1bf..38e915c5 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h @@ -104,6 +104,8 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE; unsigned int n; volatile struct mlx5_wqe *wqe = NULL; + bool metadata_ol = + txq->offloads & DEV_TX_OFFLOAD_MATCH_METADATA ? true : false; assert(elts_n > pkts_n); mlx5_tx_complete(txq); @@ -127,6 +129,9 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint8x16_t *t_wqe; uint8_t *dseg; uint8x16_t ctrl; + rte_be32_t metadata = + metadata_ol && (buf->ol_flags & PKT_TX_METADATA) ? + buf->tx_metadata : 0; assert(segs_n); max_elts = elts_n - (elts_head - txq->elts_tail); @@ -164,9 +169,10 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, ctrl = vqtbl1q_u8(ctrl, ctrl_shuf_m); vst1q_u8((void *)t_wqe, ctrl); /* Fill ESEG in the header. */ - vst1q_u16((void *)(t_wqe + 1), - ((uint16x8_t) { 0, 0, cs_flags, rte_cpu_to_be_16(len), - 0, 0, 0, 0 })); + vst1q_u32((void *)(t_wqe + 1), + ((uint32x4_t){ 0, + cs_flags << 16 | rte_cpu_to_be_16(len), + metadata, 0 })); txq->wqe_ci = wqe_ci; } if (!n) diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h index 14117c4b..fb384efd 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h @@ -104,6 +104,8 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE; unsigned int n; volatile struct mlx5_wqe *wqe = NULL; + bool metadata_ol = + txq->offloads & DEV_TX_OFFLOAD_MATCH_METADATA ? true : false; assert(elts_n > pkts_n); mlx5_tx_complete(txq); @@ -125,6 +127,9 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t max_wqe; __m128i *t_wqe, *dseg; __m128i ctrl; + rte_be32_t metadata = + metadata_ol && (buf->ol_flags & PKT_TX_METADATA) ? + buf->tx_metadata : 0; assert(segs_n); max_elts = elts_n - (elts_head - txq->elts_tail); @@ -165,9 +170,9 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, _mm_store_si128(t_wqe, ctrl); /* Fill ESEG in the header. */ _mm_store_si128(t_wqe + 1, - _mm_set_epi16(0, 0, 0, 0, - rte_cpu_to_be_16(len), cs_flags, - 0, 0)); + _mm_set_epi32(0, metadata, + (rte_cpu_to_be_16(len) << 16) | + cs_flags, 0)); txq->wqe_ci = wqe_ci; } if (!n) diff --git a/drivers/net/mlx5/mlx5_socket.c b/drivers/net/mlx5/mlx5_socket.c index 00106171..41cac3c6 100644 --- a/drivers/net/mlx5/mlx5_socket.c +++ b/drivers/net/mlx5/mlx5_socket.c @@ -26,7 +26,7 @@ int mlx5_socket_init(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct sockaddr_un sun = { .sun_family = AF_UNIX, }; @@ -98,7 +98,7 @@ error: void mlx5_socket_uninit(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; MKSTR(path, "/var/tmp/%s_%d", MLX5_DRIVER_NAME, priv->primary_socket); claim_zero(close(priv->primary_socket)); @@ -115,7 +115,7 @@ mlx5_socket_uninit(struct rte_eth_dev *dev) void mlx5_socket_handle(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; int conn_sock; int ret = 0; struct cmsghdr *cmsg = NULL; @@ -208,7 +208,7 @@ error: int mlx5_socket_connect(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct sockaddr_un sun = { .sun_family = AF_UNIX, }; diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c index fccb9af0..132bf5b4 100644 --- a/drivers/net/mlx5/mlx5_stats.c +++ b/drivers/net/mlx5/mlx5_stats.c @@ -143,7 +143,7 @@ static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init); static int mlx5_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; unsigned int i; struct ifreq ifr; @@ -221,7 +221,7 @@ mlx5_ethtool_get_stats_n(struct rte_eth_dev *dev) { void mlx5_xstats_init(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; unsigned int i; unsigned int j; @@ -312,7 +312,7 @@ int mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats, unsigned int n) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; uint64_t counters[n]; struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; @@ -353,7 +353,7 @@ mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats, int mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct rte_eth_stats tmp; unsigned int i; unsigned int idx; @@ -416,7 +416,7 @@ mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) void mlx5_stats_reset(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; unsigned int idx; @@ -448,7 +448,7 @@ mlx5_stats_reset(struct rte_eth_dev *dev) void mlx5_xstats_reset(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; int stats_n; unsigned int i; @@ -492,7 +492,7 @@ mlx5_xstats_get_names(struct rte_eth_dev *dev __rte_unused, struct rte_eth_xstat_name *xstats_names, unsigned int n) { unsigned int i; - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; unsigned int mlx5_xstats_n = xstats_ctrl->mlx5_stats_n; diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c index e2a9bb70..f874657c 100644 --- a/drivers/net/mlx5/mlx5_trigger.c +++ b/drivers/net/mlx5/mlx5_trigger.c @@ -23,7 +23,7 @@ static void mlx5_txq_stop(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; for (i = 0; i != priv->txqs_n; ++i) @@ -42,7 +42,7 @@ mlx5_txq_stop(struct rte_eth_dev *dev) static int mlx5_txq_start(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; int ret; @@ -83,7 +83,7 @@ error: static void mlx5_rxq_stop(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; for (i = 0; i != priv->rxqs_n; ++i) @@ -102,7 +102,7 @@ mlx5_rxq_stop(struct rte_eth_dev *dev) static int mlx5_rxq_start(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; int ret = 0; @@ -157,7 +157,7 @@ error: int mlx5_dev_start(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; int ret; DRV_LOG(DEBUG, "port %u starting device", dev->data->port_id); @@ -221,7 +221,7 @@ error: void mlx5_dev_stop(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; dev->data->dev_started = 0; /* Prevent crashes when queues are still in use. */ @@ -252,7 +252,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev) int mlx5_traffic_enable(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_item_eth bcast = { .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", }; @@ -379,7 +379,7 @@ error: void mlx5_traffic_disable(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; mlx5_flow_list_flush(dev, &priv->ctrl_flows); } diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index b01bd675..c5a3d1b4 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -102,7 +102,7 @@ txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl) uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; uint64_t offloads = (DEV_TX_OFFLOAD_MULTI_SEGS | DEV_TX_OFFLOAD_VLAN_INSERT); struct mlx5_dev_config *config = &priv->config; @@ -155,7 +155,7 @@ int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, unsigned int socket, const struct rte_eth_txconf *conf) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_txq_data *txq = (*priv->txqs)[idx]; struct mlx5_txq_ctrl *txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); @@ -213,7 +213,7 @@ mlx5_tx_queue_release(void *dpdk_txq) { struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; struct mlx5_txq_ctrl *txq_ctrl; - struct priv *priv; + struct mlx5_priv *priv; unsigned int i; if (txq == NULL) @@ -246,7 +246,7 @@ mlx5_tx_queue_release(void *dpdk_txq) int mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i, j; uintptr_t pages[priv->txqs_n]; unsigned int pages_n = 0; @@ -346,7 +346,7 @@ is_empw_burst_func(eth_tx_burst_t tx_pkt_burst) * @param dev * Pointer to Ethernet device. * @param idx - * Queue index in DPDK Rx queue array + * Queue index in DPDK Tx queue array. * * @return * The Verbs object initialised, NULL otherwise and rte_errno is set. @@ -354,7 +354,7 @@ is_empw_burst_func(eth_tx_burst_t tx_pkt_burst) struct mlx5_txq_ibv * mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; struct mlx5_txq_ctrl *txq_ctrl = container_of(txq_data, struct mlx5_txq_ctrl, txq); @@ -554,7 +554,7 @@ error: * @param dev * Pointer to Ethernet device. * @param idx - * Queue index in DPDK Rx queue array + * Queue index in DPDK Tx queue array. * * @return * The Verbs object if it exists. @@ -562,7 +562,7 @@ error: struct mlx5_txq_ibv * mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *txq_ctrl; if (idx >= priv->txqs_n) @@ -623,7 +623,7 @@ mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv) int mlx5_txq_ibv_verify(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; int ret = 0; struct mlx5_txq_ibv *txq_ibv; @@ -636,6 +636,27 @@ mlx5_txq_ibv_verify(struct rte_eth_dev *dev) } /** + * Calcuate the total number of WQEBB for Tx queue. + * + * Simplified version of calc_sq_size() in rdma-core. + * + * @param txq_ctrl + * Pointer to Tx queue control structure. + * + * @return + * The number of WQEBB. + */ +static int +txq_calc_wqebb_cnt(struct mlx5_txq_ctrl *txq_ctrl) +{ + unsigned int wqe_size; + const unsigned int desc = 1 << txq_ctrl->txq.elts_n; + + wqe_size = MLX5_WQE_SIZE + txq_ctrl->max_inline_data; + return rte_align32pow2(wqe_size * desc) / MLX5_WQE_SIZE; +} + +/** * Set Tx queue parameters from device configuration. * * @param txq_ctrl @@ -644,7 +665,7 @@ mlx5_txq_ibv_verify(struct rte_eth_dev *dev) static void txq_set_params(struct mlx5_txq_ctrl *txq_ctrl) { - struct priv *priv = txq_ctrl->priv; + struct mlx5_priv *priv = txq_ctrl->priv; struct mlx5_dev_config *config = &priv->config; const unsigned int max_tso_inline = ((MLX5_MAX_TSO_HEADER + (RTE_CACHE_LINE_SIZE - 1)) / @@ -754,7 +775,7 @@ struct mlx5_txq_ctrl * mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, unsigned int socket, const struct rte_eth_txconf *conf) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *tmpl; tmpl = rte_calloc_socket("TXQ", 1, @@ -780,10 +801,16 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, tmpl->txq.elts_n = log2above(desc); tmpl->idx = idx; txq_set_params(tmpl); - DRV_LOG(DEBUG, "port %u priv->device_attr.max_qp_wr is %d", - dev->data->port_id, priv->device_attr.orig_attr.max_qp_wr); - DRV_LOG(DEBUG, "port %u priv->device_attr.max_sge is %d", - dev->data->port_id, priv->device_attr.orig_attr.max_sge); + if (txq_calc_wqebb_cnt(tmpl) > + priv->device_attr.orig_attr.max_qp_wr) { + DRV_LOG(ERR, + "port %u Tx WQEBB count (%d) exceeds the limit (%d)," + " try smaller queue size", + dev->data->port_id, txq_calc_wqebb_cnt(tmpl), + priv->device_attr.orig_attr.max_qp_wr); + rte_errno = ENOMEM; + goto error; + } tmpl->txq.elts = (struct rte_mbuf *(*)[1 << tmpl->txq.elts_n])(tmpl + 1); tmpl->txq.stats.idx = idx; @@ -809,7 +836,7 @@ error: struct mlx5_txq_ctrl * mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *ctrl = NULL; if ((*priv->txqs)[idx]) { @@ -835,7 +862,7 @@ mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx) int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *txq; size_t page_size = sysconf(_SC_PAGESIZE); @@ -872,7 +899,7 @@ mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx) int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *txq; if (!(*priv->txqs)[idx]) @@ -893,7 +920,7 @@ mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx) int mlx5_txq_verify(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *txq; int ret = 0; diff --git a/drivers/net/mlx5/mlx5_vlan.c b/drivers/net/mlx5/mlx5_vlan.c index c91d08be..6568a3a4 100644 --- a/drivers/net/mlx5/mlx5_vlan.c +++ b/drivers/net/mlx5/mlx5_vlan.c @@ -48,7 +48,7 @@ int mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; DRV_LOG(DEBUG, "port %u %s VLAN filter ID %" PRIu16, @@ -102,7 +102,7 @@ out: void mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq = (*priv->rxqs)[queue]; struct mlx5_rxq_ctrl *rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); @@ -160,7 +160,7 @@ mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) int mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask) { - struct priv *priv = dev->data->dev_private; + struct mlx5_priv *priv = dev->data->dev_private; unsigned int i; if (mask & ETH_VLAN_STRIP_MASK) { |