/*-
* BSD LICENSE
*
* Copyright 2015 6WIND S.A.
* Copyright 2015 Mellanox.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of 6WIND S.A. nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stddef.h>
#include <assert.h>
#include <stdint.h>
#include <string.h>
#include <errno.h>
#define TREX_PATCH
/* Verbs header. */
/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
#include <infiniband/verbs_exp.h>
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
/* DPDK headers don't like -pedantic. */
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
#include <rte_ether.h>
#include <rte_malloc.h>
#include <rte_ethdev.h>
#include <rte_common.h>
#include <rte_flow.h>
#include <rte_flow_driver.h>
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
#include "mlx5.h"
#include "mlx5_rxtx.h"
struct fdir_flow_desc {
uint16_t dst_port;
uint16_t src_port;
uint32_t src_ip[4];
uint32_t dst_ip[4];
uint8_t tos;
uint8_t ip_id;
uint8_t proto;
uint8_t mac[6];
uint16_t vlan_tag;
enum hash_rxq_type type;
};
struct mlx5_fdir_filter {
LIST_ENTRY(mlx5_fdir_filter) next;
uint16_t queue; /* Queue assigned to if FDIR match. */
enum rte_eth_fdir_behavior behavior;
struct fdir_flow_desc desc;
struct ibv_exp_flow *flow;
};
LIST_HEAD(fdir_filter_list, mlx5_fdir_filter);
/**
* Convert struct rte_eth_fdir_filter to mlx5 filter descriptor.
*
* @param[in] fdir_filter
* DPDK filter structure to convert.
* @param[out] desc
* Resulting mlx5 filter descriptor.
* @param mode
* Flow director mode.
*/
static void
fdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter,
struct fdir_flow_desc *desc, enum rte_fdir_mode mode)
{
/* Initialize descriptor. */
memset(desc, 0, sizeof(*desc));
/* Set VLAN ID. */
desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci;
#ifndef TREX_PATCH
/* Set MAC address. */
if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
rte_memcpy(desc->mac,
fdir_filter->input.flow.mac_vlan_flow.mac_addr.
addr_bytes,
sizeof(desc->mac));
desc->type = HASH_RXQ_ETH;
return;
}
#else
if (fdir_filter->input.flow.ip4_flow.ip_id == 2) {
desc->type = HASH_RXQ_ETH;
desc->ip_id = fdir_filter->input.flow.ip4_flow.ip_id;
return;
}
#endif
/* Set mode */
switch (fdir_filter->input.flow_type) {
case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
desc->type = HASH_RXQ_UDPV4;
break;
case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
desc->type = HASH_RXQ_TCPV4;
break;
case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
desc->type = HASH_RXQ_IPV4;
break;
case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
desc->type = HASH_RXQ_UDPV6;
break;
case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
desc->type = HASH_RXQ_TCPV6;
break;
case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
desc->type = HASH_RXQ_IPV6;
break;
default:
break;
}
/* Set flow values */
switch (fdir_filter->input.flow_type) {
case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
desc->src_port = fdir_filter->input.flow.udp4_flow.src_port;
desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port;
case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip;
desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip;
desc->tos = fdir_filter->input.flow.ip4_flow.ttl; /* TTL is mapped to TOS TREX_PATCH */
desc->ip_id = fdir_filter->input.flow.ip4_flow.ip_id;
desc->proto = fdir_filter->input.flow.ip4_flow.proto;
break;
case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
desc->src_port = fdir_filter->input.flow.udp6_flow.src_port;
desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port;
/* Fall through. */
case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
rte_memcpy(desc->src_ip,
fdir_filter->input.flow.ipv6_flow.src_ip,
sizeof(desc->src_ip));
rte_memcpy(desc->dst_ip,
fdir_filter->input.flow.ipv6_flow.dst_ip,
sizeof(desc->dst_ip));
desc->tos = (uint8_t)fdir_filter->input.flow.ipv6_flow.hop_limits; /* TTL is mapped to TOS - TREX_PATCH */
desc->ip_id = (uint8_t)fdir_filter->input.flow.ipv6_flow.flow_label;
desc->proto = fdir_filter->input.flow.ipv6_flow.proto;
break;
default:
break;
}
}
/**
* Check if two flow descriptors overlap according to configured mask.
*
* @param priv
* Private structure that provides flow director mask.
* @param desc1
* First flow descriptor to compare.
* @param desc2
* Second flow descriptor to compare.
*
* @return
* Nonzero if descriptors overlap.
*/
static int
priv_fdir_overlap(const struct priv *priv,
const struct fdir_flow_desc *desc1,
const struct fdir_flow_desc *desc2)
{
const struct rte_eth_fdir_masks *mask =
&priv->dev->data->dev_conf.fdir_conf.mask;
unsigned int i;
if (desc1->type != desc2->type)
return 0;
/* Ignore non masked bits. */
for (i = 0; i != RTE_DIM(desc1->mac); ++i)
if ((desc1->mac[i] & mask->mac_addr_byte_mask) !=
(desc2->mac[i] & mask->mac_addr_byte_mask))
return 0;
if (((desc1->src_port & mask->src_port_mask) !=
(desc2->src_port & mask->src_port_mask)) ||
((desc1->dst_port & mask->dst_port_mask) !=
(desc2->dst_port & mask->dst_port_mask)))
return 0;
if ( (desc1->tos != desc2->tos) ||
(desc1->ip_id != desc2->ip_id) ||
(desc1->proto != desc2->proto) )
return 0;
switch (desc1->type) {
case HASH_RXQ_IPV4:
case HASH_RXQ_UDPV4:
case HASH_RXQ_TCPV4:
if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) !=
(desc2->src_ip[0] & mask->ipv4_mask.src_ip)) ||
((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) !=
(desc2->dst_ip[0] & mask->ipv4_mask.dst_ip)))
return 0;
break;
case HASH_RXQ_IPV6:
case HASH_RXQ_UDPV6:
case HASH_RXQ_TCPV6:
for (i = 0; i != RTE_DIM(desc1->src_ip); ++i)
if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) !=
(desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) ||
((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) !=
(desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i])))
return 0;
break;
default:
break;
}
return 1;
}
/**
* Create flow director steering rule for a specific filter.
*
* @param priv
* Private structure.
* @param mlx5_fdir_filter
* Filter to create a steering rule for.
* @param fdir_queue
* Flow director queue for matching packets.
*
* @return
* 0 on success, errno value on failure.
*/
static int
priv_fdir_flow_add(struct priv *priv,
struct mlx5_fdir_filter *mlx5_fdir_filter,
struct fdir_queue *fdir_queue)
{
struct ibv_exp_flow *flow;
struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc;
enum rte_fdir_mode fdir_mode =
priv->dev->data->dev_conf.fdir_conf.mode;
struct rte_eth_fdir_masks *mask =
&priv->dev->data->dev_conf.fdir_conf.mask;
FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type));
struct ibv_exp_flow_attr *attr = &data->attr;
uintptr_t spec_offset = (uintptr_t)&data->spec;
struct ibv_exp_flow_spec_eth *spec_eth;
struct ibv_exp_flow_spec_ipv4_ext *spec_ipv4;
struct ibv_exp_flow_spec_ipv6_ext *spec_ipv6;
struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp;
struct mlx5_fdir_filter *iter_fdir_filter;
unsigned int i;
/* Abort if an existing flow overlaps this one to avoid packet
* duplication, even if it targets another queue. */
LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next)
if ((iter_fdir_filter != mlx5_fdir_filter) &&
(iter_fdir_filter->flow != NULL) &&
(priv_fdir_overlap(priv,
&mlx5_fdir_filter->desc,
&iter_fdir_filter->desc))){
ERROR("overlap rules, please check your rules");
return EEXIST;
}
/*
* No padding must be inserted by the compiler between attr and spec.
* This layout is expected by libibverbs.
*/
assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset);
priv_flow_attr(priv, attr, sizeof(data), desc->type);
/* Set Ethernet spec */
spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset;
/* The first specification must be Ethernet. */
assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH);
assert(spec_eth->size == sizeof(*spec_eth));
/* VLAN ID */
spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask;
spec_eth->mask.vlan_tag = mask->vlan_tci_mask;
/* Update priority */
attr->priority = 2;
#ifndef TREX_PATCH
if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
/* MAC Address */
for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) {
spec_eth->val.dst_mac[i] =
desc->mac[i] & mask->mac_addr_byte_mask;
spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask;
}
goto create_flow;
}
#else
// empty mask means "match everything". This rule will match all packets, no matter what is the ether type
if (desc->ip_id == 2) {
spec_eth->val.ether_type = 0x0000;
spec_eth->mask.ether_type = 0x0000;
goto create_flow;
}
#endif
switch (desc->type) {
case HASH_RXQ_IPV4:
case HASH_RXQ_UDPV4:
case HASH_RXQ_TCPV4:
spec_offset += spec_eth->size;
/* Set IP spec */
spec_ipv4 = (struct ibv_exp_flow_spec_ipv4_ext *)spec_offset;
/* The second specification must be IP. */
assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4_EXT);
assert(spec_ipv4->size == sizeof(*spec_ipv4));
spec_ipv4->val.src_ip =
desc->src_ip[0] & mask->ipv4_mask.src_ip;
spec_ipv4->val.dst_ip =
desc->dst_ip[0] & mask->ipv4_mask.dst_ip;
spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip;
spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip;
/* Update priority */
attr->priority = 1;
spec_ipv4->val.proto = desc->proto & mask->ipv4_mask.proto;
spec_ipv4->mask.proto = mask->ipv4_mask.proto;
#ifdef TREX_PATCH
/* TOS */
if (desc->ip_id == 1) {
spec_ipv4->mask.tos = 0x1;
spec_ipv4->val.tos = 0x1;
} else {
spec_ipv4->mask.tos = 0x0;
spec_ipv4->val.tos = 0x0;
}
#endif
if (desc->type == HASH_RXQ_IPV4)
goto create_flow;
spec_offset += spec_ipv4->size;
break;
case HASH_RXQ_IPV6:
case HASH_RXQ_UDPV6:
case HASH_RXQ_TCPV6:
spec_offset += spec_eth->size;
/* Set IP spec */
spec_ipv6 = (struct ibv_exp_flow_spec_ipv6_ext *)spec_offset;
/* The second specification must be IP. */
assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6_EXT);
assert(spec_ipv6->size == sizeof(*spec_ipv6));
for (i = 0; i != RTE_DIM(desc->src_ip); ++i) {
((uint32_t *)spec_ipv6->val.src_ip)[i] =
desc->src_ip[i] & mask->ipv6_mask.src_ip[i];
((uint32_t *)spec_ipv6->val.dst_ip)[i] =
desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i];
}
rte_memcpy(spec_ipv6->mask.src_ip,
mask->ipv6_mask.src_ip,
sizeof(spec_ipv6->mask.src_ip));
rte_memcpy(spec_ipv6->mask.dst_ip,
mask->ipv6_mask.dst_ip,
sizeof(spec_ipv6->mask.dst_ip));
spec_ipv6->val.next_hdr = desc->proto & mask->ipv6_mask.proto;
spec_ipv6->mask.next_hdr = mask->ipv6_mask.proto;
#ifdef TREX_PATCH
/* TOS */
if (desc->ip_id == 1) {
spec_ipv6->mask.traffic_class = 0x1;
spec_ipv6->val.traffic_class = 0x1;
} else {
spec_ipv6->mask.traffic_class = 0;
spec_ipv6->val.traffic_class = 0;
}
#endif
/* Update priority */
attr->priority = 1;
if (desc->type == HASH_RXQ_IPV6)
goto create_flow;
spec_offset += spec_ipv6->size;
break;
default:
ERROR("invalid flow attribute type");
return EINVAL;
}
/* Set TCP/UDP flow specification. */
spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset;
/* The third specification must be TCP/UDP. */
assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP ||
spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP);
assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp));
spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask;
spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask;
spec_tcp_udp->mask.src_port = mask->src_port_mask;
spec_tcp_udp->mask.dst_port = mask->dst_port_mask;
/* Update priority */
attr->priority = 0;
create_flow:
errno = 0;
flow = ibv_exp_create_flow(fdir_queue->qp, attr);
if (flow == NULL) {
/* It's not clear whether errno is always set in this case. */
ERROR("%p: flow director configuration failed, errno=%d: %s",
(void *)priv, errno,
(errno ? strerror(errno) : "Unknown error"));
if (errno)
return errno;
return EINVAL;
}
DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow);
mlx5_fdir_filter->flow = flow;
return 0;
}
/**
* Destroy a flow director queue.
*
* @param fdir_queue
* Flow director queue to be destroyed.
*/
void
priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue)
{
struct mlx5_fdir_filter *fdir_filter;
/* Disable filter flows still applying to this queue. */
LIST_FOREACH(fdir_filter, priv->fdir_filter_list, next) {
unsigned int idx = fdir_filter->queue;
struct rxq_ctrl *rxq_ctrl =
container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
assert(idx < priv->rxqs_n);
if (fdir_queue == rxq_ctrl->fdir_queue &&
fdir_filter->flow != NULL) {
claim_zero(ibv_exp_destroy_flow(fdir_filter->flow));
fdir_filter->flow = NULL;
}
}
assert(fdir_queue->qp);
claim_zero(ibv_destroy_qp(fdir_queue->qp));
assert(fdir_queue->ind_table);
claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table));
if (fdir_queue->wq)
claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
if (fdir_queue->cq)
claim_zero(ibv_destroy_cq(fdir_queue->cq));
#ifndef NDEBUG
memset(fdir_queue, 0x2a, sizeof(*fdir_queue));
#endif
rte_free(fdir_queue);
}
/**
* Create a flow director queue.
*
* @param priv
* Private structure.
* @param wq
* Work queue to route matched packets to, NULL if one needs to
* be created.
*
* @return
* Related flow director queue on success, NULL otherwise.
*/
static struct fdir_queue *
priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq,
unsigned int socket)
{
struct fdir_queue *fdir_queue;
fdir_queue = rte_calloc_socket(__func__, 1, sizeof(*fdir_queue),
0, socket);
if (!fdir_queue) {
ERROR("cannot allocate flow director queue");
return NULL;
}
assert(priv->pd);
assert(priv->ctx);
if (!wq) {
fdir_queue->cq = ibv_exp_create_cq(
priv->ctx, 1, NULL, NULL, 0,
&(struct ibv_exp_cq_init_attr){
.comp_mask = 0,
});
if (!fdir_queue->cq) {
ERROR("cannot create flow director CQ");
goto error;
}
fdir_queue->wq = ibv_exp_create_wq(
priv->ctx,
&(struct ibv_exp_wq_init_attr){
.wq_type = IBV_EXP_WQT_RQ,
.max_recv_wr = 1,
.max_recv_sge = 1,
.pd = priv->pd,
.cq = fdir_queue->cq,
});
if (!fdir_queue->wq) {
ERROR("cannot create flow director WQ");
goto error;
}
wq = fdir_queue->wq;
}
fdir_queue->ind_table = ibv_exp_create_rwq_ind_table(
priv->ctx,
&(struct ibv_exp_rwq_ind_table_init_attr){
.pd = priv->pd,
.log_ind_tbl_size = 0,
.ind_tbl = &wq,
.comp_mask = 0,
});
if (!fdir_queue->ind_table) {
ERROR("cannot create flow director indirection table");
goto error;
}
fdir_queue->qp = ibv_exp_create_qp(
priv->ctx,
&(struct ibv_exp_qp_init_attr){
.qp_type = IBV_QPT_RAW_PACKET,
.comp_mask =
IBV_EXP_QP_INIT_ATTR_PD |
IBV_EXP_QP_INIT_ATTR_PORT |
IBV_EXP_QP_INIT_ATTR_RX_HASH,
.pd = priv->pd,
.rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
.rx_hash_function =
IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
.rx_hash_key_len = rss_hash_default_key_len,
.rx_hash_key = rss_hash_default_key,
.rx_hash_fields_mask = 0,
.rwq_ind_tbl = fdir_queue->ind_table,
},
.port_num = priv->port,
});
if (!fdir_queue->qp) {
ERROR("cannot create flow director hash RX QP");
goto error;
}
return fdir_queue;
error:
assert(fdir_queue);
assert(!fdir_queue->qp);
if (fdir_queue->ind_table)
claim_zero(ibv_exp_destroy_rwq_ind_table
(fdir_queue->ind_table));
if (fdir_queue->wq)
claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
if (fdir_queue->cq)
claim_zero(ibv_destroy_cq(fdir_queue->cq));
rte_free(fdir_queue);
return NULL;
}
/**
* Get flow director queue for a specific RX queue, create it in case
* it does not exist.
*
* @param priv
* Private structure.
* @param idx
* RX queue index.
*
* @return
* Related flow director queue on success, NULL otherwise.
*/
static struct fdir_queue *
priv_get_fdir_queue(struct priv *priv, uint16_t idx)
{
struct rxq_ctrl *rxq_ctrl =
container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
struct fdir_queue *fdir_queue = rxq_ctrl->fdir_queue;
assert(rxq_ctrl->wq);
if (fdir_queue == NULL) {
fdir_queue = priv_fdir_queue_create(priv, rxq_ctrl->wq,
rxq_ctrl->socket);
rxq_ctrl->fdir_queue = fdir_queue;
}
return fdir_queue;
}
/**
* Get or flow director drop queue. Create it if it does not exist.
*
* @param priv
* Private structure.
*
* @return
* Flow director drop queue on success, NULL otherwise.
*/
static struct fdir_queue *
priv_get_fdir_drop_queue(struct priv *priv)
{
struct fdir_queue *fdir_queue = priv->fdir_drop_queue;
if (fdir_queue == NULL) {
unsigned int socket = SOCKET_ID_ANY;
/* Select a known NUMA socket if possible. */
if (priv->rxqs_n && (*priv->rxqs)[0])
socket = container_of((*priv->rxqs)[0],
struct rxq_ctrl, rxq)->socket;
fdir_queue = priv_fdir_queue_create(priv, NULL, socket);
priv->fdir_drop_queue = fdir_queue;
}
return fdir_queue;
}
/**
* Enable flow director filter and create steering rules.
*
* @param priv
* Private structure.
* @param mlx5_fdir_filter
* Filter to create steering rule for.
*
* @return
* 0 on success, errno value on failure.
*/
static int
priv_fdir_filter_enable(struct priv *priv,
struct mlx5_fdir_filter *mlx5_fdir_filter)
{
struct fdir_queue *fdir_queue;
/* Check if flow already exists. */
if (mlx5_fdir_filter->flow != NULL)
return 0;
/* Get fdir_queue for specific queue. */
if (mlx5_fdir_filter->behavior == RTE_ETH_FDIR_REJECT)
fdir_queue = priv_get_fdir_drop_queue(priv);
else
fdir_queue = priv_get_fdir_queue(priv,
mlx5_fdir_filter->queue);
if (fdir_queue == NULL) {
ERROR("failed to create flow director rxq for queue %d",
mlx5_fdir_filter->queue);
return EINVAL;
}
/* Create flow */
return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue);
}
/**
* Initialize flow director filters list.
*
* @param priv
* Private structure.
*
* @return
* 0 on success, errno value on failure.
*/
int
fdir_init_filters_list(struct priv *priv)
{
/* Filter list initialization should be done only once. */
if (priv->fdir_filter_list)
return 0;
/* Create filters list. */
priv->fdir_filter_list =
rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0);
if (priv->fdir_filter_list == NULL) {
int err = ENOMEM;
ERROR("cannot allocate flow director filter list: %s",
strerror(err));
return err;
}
LIST_INIT(priv->fdir_filter_list);
return 0;
}
/**
* Flush all filters.
*
* @param priv
* Private structure.
*/
static void
priv_fdir_filter_flush(struct priv *priv)
{
struct mlx5_fdir_filter *mlx5_fdir_filter;
while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) {
struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
DEBUG("%p: flushing flow director filter %p",
(void *)priv, (void *)mlx5_fdir_filter);
LIST_REMOVE(mlx5_fdir_filter, next);
if (flow != NULL)
claim_zero(ibv_exp_destroy_flow(flow));
rte_free(mlx5_fdir_filter);
}
}
/**
* Remove all flow director filters and delete list.
*
* @param priv
* Private structure.
*/
void
priv_fdir_delete_filters_list(struct priv *priv)
{
priv_fdir_filter_flush(priv);
rte_free(priv->fdir_filter_list);
priv->fdir_filter_list = NULL;
}
/**
* Disable flow director, remove all steering rules.
*
* @param priv
* Private structure.
*/
void
priv_fdir_disable(struct priv *priv)
{
unsigned int i;
struct mlx5_fdir_filter *mlx5_fdir_filter;
/* Run on every flow director filter and destroy flow handle. */
LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
struct ibv_exp_flow *flow;
/* Only valid elements should be in the list */
assert(mlx5_fdir_filter != NULL);
flow = mlx5_fdir_filter->flow;
/* Destroy flow handle */
if (flow != NULL) {
claim_zero(ibv_exp_destroy_flow(flow));
mlx5_fdir_filter->flow = NULL;
}
}
/* Destroy flow director context in each RX queue. */
for (i = 0; (i != priv->rxqs_n); i++) {
struct rxq_ctrl *rxq_ctrl =
container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq);
if (!rxq_ctrl->fdir_queue)
continue;
priv_fdir_queue_destroy(priv, rxq_ctrl->fdir_queue);
rxq_ctrl->fdir_queue = NULL;
}
if (priv->fdir_drop_queue) {
priv_fdir_queue_destroy(priv, priv->fdir_drop_queue);
priv->fdir_drop_queue = NULL;
}
}
/**
* Enable flow director, create steering rules.
*
* @param priv
* Private structure.
*/
void
priv_fdir_enable(struct priv *priv)
{
struct mlx5_fdir_filter *mlx5_fdir_filter;
/* Run on every fdir filter and create flow handle */
LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
/* Only valid elements should be in the list */
assert(mlx5_fdir_filter != NULL);
priv_fdir_filter_enable(priv, mlx5_fdir_filter);
}
}
/**
* Find specific filter in list.
*
* @param priv
* Private structure.
* @param fdir_filter
* Flow director filter to find.
*
* @return
* Filter element if found, otherwise NULL.
*/
static struct mlx5_fdir_filter *
priv_find_filter_in_list(struct priv *priv,
const struct rte_eth_fdir_filter *fdir_filter)
{
struct fdir_flow_desc desc;
struct mlx5_fdir_filter *mlx5_fdir_filter;
enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
/* Get flow director filter to look for. */
fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode);
/* Look for the requested element. */
LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
/* Only valid elements should be in the list. */
assert(mlx5_fdir_filter != NULL);
/* Return matching filter. */
if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc)))
return mlx5_fdir_filter;
}
/* Filter not found */
return NULL;
}
/**
* Add new flow director filter and store it in list.
*
* @param priv
* Private structure.
* @param fdir_filter
* Flow director filter to add.
*
* @return
* 0 on success, errno value on failure.
*/
static int
priv_fdir_filter_add(struct priv *priv,
const struct rte_eth_fdir_filter *fdir_filter)
{
struct mlx5_fdir_filter *mlx5_fdir_filter;
enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
int err = 0;
/* Validate queue number. */
if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
return EINVAL;
}
/* Duplicate filters are currently unsupported. */
mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
if (mlx5_fdir_filter != NULL) {
#ifndef TREX_PATCH
ERROR("filter already exists");
#endif
return EEXIST;
}
/* Create new flow director filter. */
mlx5_fdir_filter =
rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0);
if (mlx5_fdir_filter == NULL) {
err = ENOMEM;
ERROR("cannot allocate flow director filter: %s",
strerror(err));
return err;
}
/* Set action parameters. */
mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
mlx5_fdir_filter->behavior = fdir_filter->action.behavior;
/* Convert to mlx5 filter descriptor. */
fdir_filter_to_flow_desc(fdir_filter,
&mlx5_fdir_filter->desc, fdir_mode);
/* Insert new filter into list. */
LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next);
DEBUG("%p: flow director filter %p added",
(void *)priv, (void *)mlx5_fdir_filter);
/* Enable filter immediately if device is started. */
if (priv->started)
err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
return err;
}
/**
* Update queue for specific filter.
*
* @param priv
* Private structure.
* @param fdir_filter
* Filter to be updated.
*
* @return
* 0 on success, errno value on failure.
*/
static int
priv_fdir_filter_update(struct priv *priv,
const struct rte_eth_fdir_filter *fdir_filter)
{
struct mlx5_fdir_filter *mlx5_fdir_filter;
/* Validate queue number. */
if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
return EINVAL;
}
mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
if (mlx5_fdir_filter != NULL) {
struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
int err = 0;
/* Update queue number. */
mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
/* Destroy flow handle. */
if (flow != NULL) {
claim_zero(ibv_exp_destroy_flow(flow));
mlx5_fdir_filter->flow = NULL;
}
DEBUG("%p: flow director filter %p updated",
(void *)priv, (void *)mlx5_fdir_filter);
/* Enable filter if device is started. */
if (priv->started)
err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
return err;
}
/* Filter not found, create it. */
DEBUG("%p: filter not found for update, creating new filter",
(void *)priv);
return priv_fdir_filter_add(priv, fdir_filter);
}
/**
* Delete specific filter.
*
* @param priv
* Private structure.
* @param fdir_filter
* Filter to be deleted.
*
* @return
* 0 on success, errno value on failure.
*/
static int
priv_fdir_filter_delete(struct priv *priv,
const struct rte_eth_fdir_filter *fdir_filter)
{
struct mlx5_fdir_filter *mlx5_fdir_filter;
mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
if (mlx5_fdir_filter != NULL) {
struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
/* Remove element from list. */
LIST_REMOVE(mlx5_fdir_filter, next);
/* Destroy flow handle. */
if (flow != NULL) {
claim_zero(ibv_exp_destroy_flow(flow));
mlx5_fdir_filter->flow = NULL;
}
DEBUG("%p: flow director filter %p deleted",
(void *)priv, (void *)mlx5_fdir_filter);
/* Delete filter. */
rte_free(mlx5_fdir_filter);
return 0;
}
#ifndef TREX_PATCH
ERROR("%p: flow director delete failed, cannot find filter",
(void *)priv);
#endif
return ENOENT;
}
/**
* Get flow director information.
*
* @param priv
* Private structure.
* @param[out] fdir_info
* Resulting flow director information.
*/
static void
priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
{
struct rte_eth_fdir_masks *mask =
&priv->dev->data->dev_conf.fdir_conf.mask;
fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
fdir_info->guarant_spc = 0;
rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
fdir_info->max_flexpayload = 0;
fdir_info->flow_types_mask[0] = 0;
fdir_info->flex_payload_unit = 0;
fdir_info->max_flex_payload_segment_num = 0;
fdir_info->flex_payload_limit = 0;
memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
}
/**
* Deal with flow director operations.
*
* @param priv
* Pointer to private structure.
* @param filter_op
* Operation to perform.
* @param arg
* Pointer to operation-specific structure.
*
* @return
* 0 on success, errno value on failure.
*/
static int
priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
{
enum rte_fdir_mode fdir_mode =
priv->dev->data->dev_conf.fdir_conf.mode;
int ret = 0;
if (filter_op == RTE_ETH_FILTER_NOP)
return 0;
if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
ERROR("%p: flow director mode %d not supported",
(void *)priv, fdir_mode);
return EINVAL;
}
switch (filter_op) {
case RTE_ETH_FILTER_ADD:
ret = priv_fdir_filter_add(priv, arg);
break;
case RTE_ETH_FILTER_UPDATE:
ret = priv_fdir_filter_update(priv, arg);
break;
case RTE_ETH_FILTER_DELETE:
ret = priv_fdir_filter_delete(priv, arg);
break;
case RTE_ETH_FILTER_FLUSH:
priv_fdir_filter_flush(priv);
break;
case RTE_ETH_FILTER_INFO:
priv_fdir_info_get(priv, arg);
break;
default:
DEBUG("%p: unknown operation %u", (void *)priv, filter_op);
ret = EINVAL;
break;
}
return ret;
}
static const struct rte_flow_ops mlx5_flow_ops = {
.validate = mlx5_flow_validate,
.create = mlx5_flow_create,
.destroy = mlx5_flow_destroy,
.flush = mlx5_flow_flush,
.query = NULL,
};
/**
* Manage filter operations.
*
* @param dev
* Pointer to Ethernet device structure.
* @param filter_type
* Filter type.
* @param filter_op
* Operation to perform.
* @param arg
* Pointer to operation-specific structure.
*
* @return
* 0 on success, negative errno value on failure.
*/
int
mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
enum rte_filter_type filter_type,
enum rte_filter_op filter_op,
void *arg)
{
int ret = EINVAL;
struct priv *priv = dev->data->dev_private;
switch (filter_type) {
case RTE_ETH_FILTER_GENERIC:
if (filter_op != RTE_ETH_FILTER_GET)
return -EINVAL;
*(const void **)arg = &mlx5_flow_ops;
return 0;
case RTE_ETH_FILTER_FDIR:
priv_lock(priv);
ret = priv_fdir_ctrl_func(priv, filter_op, arg);
priv_unlock(priv);
break;
default:
ERROR("%p: filter type (%d) not supported",
(void *)dev, filter_type);
break;
}
return -ret;
}