diff options
| author |   Benoît Ganne <bganne@cisco.com> | 2021-05-03 17:02:39 +0200 |
|---|---|---|
| committer |   Damjan Marion <dmarion@me.com> | 2021-05-04 07:22:34 +0000 |
| commit | ee60ac638c9abcd0763c57fce0e3e646723ea0e0 (patch) | |
| tree | f53c3438897063209366ede7bab5efec2712ec13 | |
| parent | 3a533cd561eb75deab489d5ab39a7ec4e1bf7952 (diff) | |
rdma: support non-power-of-2 number of rxq
Type: improvement
Change-Id: I7b78a8470aa3fd271120587d08e82afb23cb195b
Signed-off-by: Benoît Ganne <bganne@cisco.com>
| -rw-r--r-- | src/plugins/rdma/device.c | 32 |
1 files changed, 19 insertions, 13 deletions
diff --git a/src/plugins/rdma/device.c b/src/plugins/rdma/device.c index c218bf93c7f..14262f597b7 100644 --- a/src/plugins/rdma/device.c +++ b/src/plugins/rdma/device.c @@ -617,15 +617,29 @@ rdma_rxq_finalize (vlib_main_t * vm, rdma_device_t * rd) struct ibv_rwq_ind_table_init_attr rwqia; struct ibv_qp_init_attr_ex qpia; struct ibv_wq **ind_tbl; + const u32 rxq_sz = vec_len (rd->rxqs); + u32 ind_tbl_sz = rxq_sz; u32 i; - ASSERT (is_pow2 (vec_len (rd->rxqs)) - && "rxq number should be a power of 2"); + if (!is_pow2 (ind_tbl_sz)) + { + /* in case we do not have a power-of-2 number of rxq, we try to use the + * maximum supported to minimize the imbalance */ + struct ibv_device_attr_ex attr; + if (ibv_query_device_ex (rd->ctx, 0, &attr)) + return clib_error_return_unix (0, "device query failed"); + ind_tbl_sz = attr.rss_caps.max_rwq_indirection_table_size; + if (ind_tbl_sz < rxq_sz) + return clib_error_create ("too many rxqs requested (%d) compared to " + "max indirection table size (%d)", + rxq_sz, ind_tbl_sz); + } - ind_tbl = vec_new (struct ibv_wq *, vec_len (rd->rxqs)); - vec_foreach_index (i, rd->rxqs) - ind_tbl[i] = vec_elt_at_index (rd->rxqs, i)->wq; + ind_tbl = vec_new (struct ibv_wq *, ind_tbl_sz); + vec_foreach_index (i, ind_tbl) + vec_elt (ind_tbl, i) = vec_elt (rd->rxqs, i % rxq_sz).wq; memset (&rwqia, 0, sizeof (rwqia)); + ASSERT (is_pow2 (vec_len (ind_tbl))); rwqia.log_ind_tbl_size = min_log2 (vec_len (ind_tbl)); rwqia.ind_tbl = ind_tbl; if ((rd->rx_rwq_ind_tbl = ibv_create_rwq_ind_table (rd->ctx, &rwqia)) == 0) @@ -843,14 +857,6 @@ rdma_create_if (vlib_main_t * vm, rdma_create_if_args_t * args) args->txq_size = args->txq_size ? args->txq_size : 1024; args->rxq_num = args->rxq_num ? args->rxq_num : 2; - if (!is_pow2 (args->rxq_num)) - { - args->rv = VNET_API_ERROR_INVALID_VALUE; - args->error = - clib_error_return (0, "rx queue number must be a power of two"); - goto err0; - } - if (args->rxq_size < VLIB_FRAME_SIZE || args->txq_size < VLIB_FRAME_SIZE || args->rxq_size > 65535 || args->txq_size > 65535 || !is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size)) |