aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBenoît Ganne <bganne@cisco.com>2021-05-03 17:02:39 +0200
committerDamjan Marion <dmarion@me.com>2021-05-04 07:22:34 +0000
commitee60ac638c9abcd0763c57fce0e3e646723ea0e0 (patch)
treef53c3438897063209366ede7bab5efec2712ec13
parent3a533cd561eb75deab489d5ab39a7ec4e1bf7952 (diff)
rdma: support non-power-of-2 number of rxq
Type: improvement Change-Id: I7b78a8470aa3fd271120587d08e82afb23cb195b Signed-off-by: Benoît Ganne <bganne@cisco.com>
-rw-r--r--src/plugins/rdma/device.c32
1 files changed, 19 insertions, 13 deletions
diff --git a/src/plugins/rdma/device.c b/src/plugins/rdma/device.c
index c218bf93c7f..14262f597b7 100644
--- a/src/plugins/rdma/device.c
+++ b/src/plugins/rdma/device.c
@@ -617,15 +617,29 @@ rdma_rxq_finalize (vlib_main_t * vm, rdma_device_t * rd)
struct ibv_rwq_ind_table_init_attr rwqia;
struct ibv_qp_init_attr_ex qpia;
struct ibv_wq **ind_tbl;
+ const u32 rxq_sz = vec_len (rd->rxqs);
+ u32 ind_tbl_sz = rxq_sz;
u32 i;
- ASSERT (is_pow2 (vec_len (rd->rxqs))
- && "rxq number should be a power of 2");
+ if (!is_pow2 (ind_tbl_sz))
+ {
+ /* in case we do not have a power-of-2 number of rxq, we try to use the
+ * maximum supported to minimize the imbalance */
+ struct ibv_device_attr_ex attr;
+ if (ibv_query_device_ex (rd->ctx, 0, &attr))
+ return clib_error_return_unix (0, "device query failed");
+ ind_tbl_sz = attr.rss_caps.max_rwq_indirection_table_size;
+ if (ind_tbl_sz < rxq_sz)
+ return clib_error_create ("too many rxqs requested (%d) compared to "
+ "max indirection table size (%d)",
+ rxq_sz, ind_tbl_sz);
+ }
- ind_tbl = vec_new (struct ibv_wq *, vec_len (rd->rxqs));
- vec_foreach_index (i, rd->rxqs)
- ind_tbl[i] = vec_elt_at_index (rd->rxqs, i)->wq;
+ ind_tbl = vec_new (struct ibv_wq *, ind_tbl_sz);
+ vec_foreach_index (i, ind_tbl)
+ vec_elt (ind_tbl, i) = vec_elt (rd->rxqs, i % rxq_sz).wq;
memset (&rwqia, 0, sizeof (rwqia));
+ ASSERT (is_pow2 (vec_len (ind_tbl)));
rwqia.log_ind_tbl_size = min_log2 (vec_len (ind_tbl));
rwqia.ind_tbl = ind_tbl;
if ((rd->rx_rwq_ind_tbl = ibv_create_rwq_ind_table (rd->ctx, &rwqia)) == 0)
@@ -843,14 +857,6 @@ rdma_create_if (vlib_main_t * vm, rdma_create_if_args_t * args)
args->txq_size = args->txq_size ? args->txq_size : 1024;
args->rxq_num = args->rxq_num ? args->rxq_num : 2;
- if (!is_pow2 (args->rxq_num))
- {
- args->rv = VNET_API_ERROR_INVALID_VALUE;
- args->error =
- clib_error_return (0, "rx queue number must be a power of two");
- goto err0;
- }
-
if (args->rxq_size < VLIB_FRAME_SIZE || args->txq_size < VLIB_FRAME_SIZE ||
args->rxq_size > 65535 || args->txq_size > 65535 ||
!is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size))