summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNathan Brown <nathan.brown@arm.com>2022-12-30 20:04:39 +0000
committerBeno�t Ganne <bganne@cisco.com>2023-02-13 15:36:28 +0000
commit844a0e8b07010ccdd5af211b91d022aa8d43c934 (patch)
tree6d53b86703a57e9f42a471d0c44a5cd0f6d23c17
parent1d998b7c27e48f71a5f3edfaf182abbcc88eca07 (diff)
rdma: always use 64 byte CQEs for MLX5
When DPDK MLX PMDs are built, and the DPDK plugin is loaded, DPDK may set the MLX5_CQE_SIZE environment variable to 128. This causes the RDMA plugin to be unable to create completion queues. Since the RDMA plugin expects the CQEs to be 64 bytes, set the cqe_size explicitly when creating the CQ. This avoids any issues with different values for the MLX5_CQE_SIZE environment variable. Type: improvement Signed-off-by: Nathan Brown <nathan.brown@arm.com> Change-Id: Idfd078d3045a4dcb674325ef36f85a89df6fbebc
-rw-r--r--src/plugins/rdma/device.c28
1 files changed, 23 insertions, 5 deletions
diff --git a/src/plugins/rdma/device.c b/src/plugins/rdma/device.c
index 3fdc399a4e3..554304c1daf 100644
--- a/src/plugins/rdma/device.c
+++ b/src/plugins/rdma/device.c
@@ -445,9 +445,10 @@ rdma_rxq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc,
if (is_mlx5dv)
{
struct mlx5dv_cq_init_attr dvcq = { };
- dvcq.comp_mask = MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;
+ dvcq.comp_mask = MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE |
+ MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE;
dvcq.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH;
-
+ dvcq.cqe_size = 64;
if ((cqex = mlx5dv_create_cq (rd->ctx, &cqa, &dvcq)) == 0)
return clib_error_return_unix (0, "Create mlx5dv rx CQ Failed");
}
@@ -717,15 +718,32 @@ rdma_txq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc)
struct ibv_qp_init_attr qpia;
struct ibv_qp_attr qpa;
int qp_flags;
+ int is_mlx5dv = !!(rd->flags & RDMA_DEVICE_F_MLX5DV);
vec_validate_aligned (rd->txqs, qid, CLIB_CACHE_LINE_BYTES);
txq = vec_elt_at_index (rd->txqs, qid);
ASSERT (is_pow2 (n_desc));
txq->bufs_log2sz = min_log2 (n_desc);
vec_validate_aligned (txq->bufs, n_desc - 1, CLIB_CACHE_LINE_BYTES);
-
- if ((txq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0)
- return clib_error_return_unix (0, "Create CQ Failed");
+ if (is_mlx5dv)
+ {
+ struct ibv_cq_init_attr_ex cqa = {};
+ struct ibv_cq_ex *cqex;
+ struct mlx5dv_cq_init_attr dvcq = {};
+ dvcq.comp_mask = MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE |
+ MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE;
+ dvcq.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH;
+ dvcq.cqe_size = 64;
+ cqa.cqe = n_desc;
+ if ((cqex = mlx5dv_create_cq (rd->ctx, &cqa, &dvcq)) == 0)
+ return clib_error_return_unix (0, "Create mlx5dv tx CQ Failed");
+ txq->cq = ibv_cq_ex_to_cq (cqex);
+ }
+ else
+ {
+ if ((txq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0)
+ return clib_error_return_unix (0, "Create CQ Failed");
+ }
memset (&qpia, 0, sizeof (qpia));
qpia.send_cq = txq->cq;