rdma: add multiqueue support

Add support for multiqueue for RDMA devices.

Change-Id: I78a2481cec6747494c670cc776475828be3af55b
Signed-off-by: Benoît Ganne <bganne@cisco.com>

4 files changed, 170 insertions, 76 deletions

diff --git a/src/plugins/rdma/cli.c b/src/plugins/rdma/cli.c
index e1ccbb66c0b..1f377a738fc 100644
--- a/src/plugins/rdma/cli.c
+++ b/src/plugins/rdma/cli.c
@@ -33,18 +33,24 @@ rdma_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
   unformat_input_t _line_input, *line_input = &_line_input;
   rdma_create_if_args_t args;
 
-  clib_memset (&args, 0, sizeof (rdma_create_if_args_t));
-
   /* Get a line of input. */
   if (!unformat_user (input, unformat_line_input, line_input))
     return 0;
 
+  clib_memset (&args, 0, sizeof (rdma_create_if_args_t));
+
   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
     {
       if (unformat (line_input, "host-if %s", &args.ifname))
 	;
       else if (unformat (line_input, "name %s", &args.name))
 	;
+      else if (unformat (line_input, "rx-queue-size %u", &args.rxq_size))
+	;
+      else if (unformat (line_input, "tx-queue-size %u", &args.txq_size))
+	;
+      else if (unformat (line_input, "num-rx-queues %u", &args.rxq_num))
+	;
       else
 	return clib_error_return (0, "unknown input `%U'",
 				  format_unformat_error, input);
@@ -62,7 +68,9 @@ rdma_create_command_fn (vlib_main_t * vm, unformat_input_t * input,
 /* *INDENT-OFF* */
 VLIB_CLI_COMMAND (rdma_create_command, static) = {
   .path = "create interface rdma",
-  .short_help = "create interface rdma <host-if ifname> [name <name>]",
+  .short_help = "create interface rdma <host-if ifname> [name <name>]"
+    " [rx-queue-size <size>] [tx-queue-size <size>]"
+    " [num-rx-queues <size>]",
   .function = rdma_create_command_fn,
 };
 /* *INDENT-ON* */
@@ -101,7 +109,7 @@ rdma_delete_command_fn (vlib_main_t * vm, unformat_input_t * input,
 
   hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
   if (hw == NULL || rdma_device_class.index != hw->dev_class_index)
-    return clib_error_return (0, "not an AVF interface");
+    return clib_error_return (0, "not a RDMA interface");
 
   rd = pool_elt_at_index (rm->devices, hw->dev_instance);
 
diff --git a/src/plugins/rdma/device.c b/src/plugins/rdma/device.c
index 09816644e72..b6609ca4bdc 100644
--- a/src/plugins/rdma/device.c
+++ b/src/plugins/rdma/device.c
@@ -29,6 +29,20 @@
 
 #include <rdma/rdma.h>
 
+/* Default RSS hash key (from DPDK MLX driver) */
+static u8 rdma_rss_hash_key[] = {
+  0x2c, 0xc6, 0x81, 0xd1,
+  0x5b, 0xdb, 0xf4, 0xf7,
+  0xfc, 0xa2, 0x83, 0x19,
+  0xdb, 0x1a, 0x3e, 0x94,
+  0x6b, 0x9e, 0x38, 0xd9,
+  0x2c, 0x9c, 0x03, 0xd1,
+  0xad, 0x99, 0x44, 0xa7,
+  0xd9, 0x56, 0x3d, 0x59,
+  0x06, 0x3c, 0x25, 0xf3,
+  0xfc, 0x1f, 0xdc, 0x2a,
+};
+
 rdma_main_t rdma_main;
 
 #define rdma_log_debug(dev, f, ...) \
@@ -201,7 +215,7 @@ static clib_error_t *
 rdma_register_interface (vnet_main_t * vnm, rdma_device_t * rd)
 {
   return ethernet_register_interface (vnm, rdma_device_class.index,
-				      rd->dev_instance, rd->hwaddr,
+				      rd->dev_instance, rd->hwaddr.bytes,
 				      &rd->hw_if_index, rdma_flag_change);
 }
 
@@ -237,9 +251,11 @@ rdma_dev_cleanup (rdma_device_t * rd)
   }
   vec_foreach (rxq, rd->rxqs)
   {
-    _(ibv_destroy_qp, rxq->qp);
+    _(ibv_destroy_wq, rxq->wq);
     _(ibv_destroy_cq, rxq->cq);
   }
+  _(ibv_destroy_rwq_ind_table, rd->rx_rwq_ind_tbl);
+  _(ibv_destroy_qp, rd->rx_qp);
   _(ibv_dealloc_pd, rd->pd);
   _(ibv_close_device, rd->ctx);
 #undef _
@@ -253,12 +269,38 @@ rdma_dev_cleanup (rdma_device_t * rd)
 }
 
 static clib_error_t *
+rdma_rxq_init_flow (struct ibv_flow **flow, struct ibv_qp *qp,
+		    const mac_address_t * mac, const mac_address_t * mask,
+		    u32 flags)
+{
+  struct raw_eth_flow_attr
+  {
+    struct ibv_flow_attr attr;
+    struct ibv_flow_spec_eth spec_eth;
+  } __attribute__ ((packed)) fa;
+
+  memset (&fa, 0, sizeof (fa));
+  fa.attr.num_of_specs = 1;
+  fa.attr.port = 1;
+  fa.attr.flags = flags;
+  fa.spec_eth.type = IBV_FLOW_SPEC_ETH;
+  fa.spec_eth.size = sizeof (struct ibv_flow_spec_eth);
+
+  memcpy (fa.spec_eth.val.dst_mac, mac, sizeof (fa.spec_eth.val.dst_mac));
+  memcpy (fa.spec_eth.mask.dst_mac, mask, sizeof (fa.spec_eth.mask.dst_mac));
+
+  if ((*flow = ibv_create_flow (qp, &fa.attr)) == 0)
+    return clib_error_return_unix (0, "create Flow Failed");
+
+  return 0;
+}
+
+static clib_error_t *
 rdma_rxq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc)
 {
   rdma_rxq_t *rxq;
-  struct ibv_qp_init_attr qpia;
-  struct ibv_qp_attr qpa;
-  int qp_flags;
+  struct ibv_wq_init_attr wqia;
+  struct ibv_wq_attr wqa;
 
   vec_validate_aligned (rd->rxqs, qid, CLIB_CACHE_LINE_BYTES);
   rxq = vec_elt_at_index (rd->rxqs, qid);
@@ -267,30 +309,75 @@ rdma_rxq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc)
   if ((rxq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0)
     return clib_error_return_unix (0, "Create CQ Failed");
 
-  memset (&qpia, 0, sizeof (qpia));
-  qpia.send_cq = rxq->cq;
-  qpia.recv_cq = rxq->cq;
-  qpia.cap.max_recv_wr = n_desc;
-  qpia.cap.max_recv_sge = 1;
-  qpia.qp_type = IBV_QPT_RAW_PACKET;
+  memset (&wqia, 0, sizeof (wqia));
+  wqia.wq_type = IBV_WQT_RQ;
+  wqia.max_wr = n_desc;
+  wqia.max_sge = 1;
+  wqia.pd = rd->pd;
+  wqia.cq = rxq->cq;
+  if ((rxq->wq = ibv_create_wq (rd->ctx, &wqia)) == 0)
+    return clib_error_return_unix (0, "Create WQ Failed");
+
+  memset (&wqa, 0, sizeof (wqa));
+  wqa.attr_mask = IBV_WQ_ATTR_STATE;
+  wqa.wq_state = IBV_WQS_RDY;
+  if (ibv_modify_wq (rxq->wq, &wqa) != 0)
+    return clib_error_return_unix (0, "Modify WQ (RDY) Failed");
 
-  if ((rxq->qp = ibv_create_qp (rd->pd, &qpia)) == 0)
-    return clib_error_return_unix (0, "Queue Pair create failed");
+  return 0;
+}
 
-  memset (&qpa, 0, sizeof (qpa));
-  qp_flags = IBV_QP_STATE | IBV_QP_PORT;
-  qpa.qp_state = IBV_QPS_INIT;
-  qpa.port_num = 1;
-  if (ibv_modify_qp (rxq->qp, &qpa, qp_flags) != 0)
-    return clib_error_return_unix (0, "Modify QP (init) Failed");
+static clib_error_t *
+rdma_rxq_finalize (vlib_main_t * vm, rdma_device_t * rd)
+{
+  struct ibv_rwq_ind_table_init_attr rwqia;
+  struct ibv_qp_init_attr_ex qpia;
+  const mac_address_t ucast = {.bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
+  };
+  const mac_address_t mcast = {.bytes = {0x1, 0x0, 0x0, 0x0, 0x0, 0x0} };
+  struct ibv_wq **ind_tbl;
+  clib_error_t *err;
+  u32 i;
 
-  memset (&qpa, 0, sizeof (qpa));
-  qp_flags = IBV_QP_STATE;
-  qpa.qp_state = IBV_QPS_RTR;
-  if (ibv_modify_qp (rxq->qp, &qpa, qp_flags) != 0)
-    return clib_error_return_unix (0, "Modify QP (receive) Failed");
+  ASSERT (is_pow2 (vec_len (rd->rxqs))
+	  && "rxq number should be a power of 2");
 
-  return 0;
+  ind_tbl = vec_new (struct ibv_wq *, vec_len (rd->rxqs));
+  vec_foreach_index (i, rd->rxqs)
+    ind_tbl[i] = vec_elt_at_index (rd->rxqs, i)->wq;
+  memset (&rwqia, 0, sizeof (rwqia));
+  rwqia.log_ind_tbl_size = min_log2 (vec_len (ind_tbl));
+  rwqia.ind_tbl = ind_tbl;
+  if ((rd->rx_rwq_ind_tbl = ibv_create_rwq_ind_table (rd->ctx, &rwqia)) == 0)
+    return clib_error_return_unix (0, "RWQ indirection table create failed");
+  vec_free (ind_tbl);
+
+  memset (&qpia, 0, sizeof (qpia));
+  qpia.qp_type = IBV_QPT_RAW_PACKET;
+  qpia.comp_mask =
+    IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_IND_TABLE |
+    IBV_QP_INIT_ATTR_RX_HASH;
+  qpia.pd = rd->pd;
+  qpia.rwq_ind_tbl = rd->rx_rwq_ind_tbl;
+  STATIC_ASSERT_SIZEOF (rdma_rss_hash_key, 40);
+  qpia.rx_hash_conf.rx_hash_key_len = sizeof (rdma_rss_hash_key);
+  qpia.rx_hash_conf.rx_hash_key = rdma_rss_hash_key;
+  qpia.rx_hash_conf.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ;
+  qpia.rx_hash_conf.rx_hash_fields_mask =
+    IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4;
+  if ((rd->rx_qp = ibv_create_qp_ex (rd->ctx, &qpia)) == 0)
+    return clib_error_return_unix (0, "Queue Pair create failed");
+
+  /* receive only packets with src = our MAC */
+  if ((err =
+       rdma_rxq_init_flow (&rd->flow_ucast, rd->rx_qp, &rd->hwaddr, &ucast,
+			   0)) != 0)
+    return err;
+  /* receive multicast packets */
+  return rdma_rxq_init_flow (&rd->flow_mcast, rd->rx_qp, &mcast, &mcast,
+			     IBV_FLOW_ATTR_FLAGS_DONT_TRAP
+			     /* let others receive mcast packet too (eg. Linux) */
+    );
 }
 
 static clib_error_t *
@@ -341,12 +428,13 @@ rdma_txq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc)
 }
 
 static clib_error_t *
-rdma_dev_init (vlib_main_t * vm, rdma_device_t * rd)
+rdma_dev_init (vlib_main_t * vm, rdma_device_t * rd, u32 rxq_size,
+	       u32 txq_size, u32 rxq_num)
 {
   clib_error_t *err;
   vlib_buffer_main_t *bm = vm->buffer_main;
   vlib_thread_main_t *tm = vlib_get_thread_main ();
-  u16 i;
+  u32 i;
 
   if (rd->ctx == 0)
     return clib_error_return_unix (0, "Device Open Failed");
@@ -354,11 +442,16 @@ rdma_dev_init (vlib_main_t * vm, rdma_device_t * rd)
   if ((rd->pd = ibv_alloc_pd (rd->ctx)) == 0)
     return clib_error_return_unix (0, "PD Alloc Failed");
 
-  if ((err = rdma_rxq_init (vm, rd, 0, 512)))
+  ethernet_mac_address_generate (rd->hwaddr.bytes);
+
+  for (i = 0; i < rxq_num; i++)
+    if ((err = rdma_rxq_init (vm, rd, i, rxq_size)))
+      return err;
+  if ((err = rdma_rxq_finalize (vm, rd)))
     return err;
 
   for (i = 0; i < tm->n_vlib_mains; i++)
-    if ((err = rdma_txq_init (vm, rd, i, 512)))
+    if ((err = rdma_txq_init (vm, rd, i, txq_size)))
       return err;
 
   if ((rd->mr = ibv_reg_mr (rd->pd, (void *) bm->buffer_mem_start,
@@ -366,41 +459,6 @@ rdma_dev_init (vlib_main_t * vm, rdma_device_t * rd)
 			    IBV_ACCESS_LOCAL_WRITE)) == 0)
     return clib_error_return_unix (0, "Register MR Failed");
 
-  ethernet_mac_address_generate (rd->hwaddr);
-
-  /*
-   * restrict packets steering to our MAC
-   * allows to share a single HW NIC with multiple RDMA ifaces
-   * and/or Linux
-   */
-  struct raw_eth_flow_attr
-  {
-    struct ibv_flow_attr attr;
-    struct ibv_flow_spec_eth spec_eth;
-  } __attribute__ ((packed)) fa;
-  memset (&fa, 0, sizeof (fa));
-  fa.attr.num_of_specs = 1;
-  fa.attr.port = 1;
-  fa.spec_eth.type = IBV_FLOW_SPEC_ETH;
-  fa.spec_eth.size = sizeof (struct ibv_flow_spec_eth);
-  memcpy (fa.spec_eth.val.dst_mac, rd->hwaddr,
-	  sizeof (fa.spec_eth.val.dst_mac));
-  memset (fa.spec_eth.mask.dst_mac, 0xff, sizeof (fa.spec_eth.mask.dst_mac));
-  if ((rd->flow_ucast = ibv_create_flow (rd->rxqs[0].qp, &fa.attr)) == 0)
-    return clib_error_return_unix (0, "create Flow Failed");
-
-  /* receive multicast packets too */
-  memset (&fa, 0, sizeof (fa));
-  fa.attr.num_of_specs = 1;
-  fa.attr.port = 1;
-  fa.attr.flags = IBV_FLOW_ATTR_FLAGS_DONT_TRAP;	/* let others receive them too */
-  fa.spec_eth.type = IBV_FLOW_SPEC_ETH;
-  fa.spec_eth.size = sizeof (struct ibv_flow_spec_eth);
-  fa.spec_eth.val.dst_mac[0] = 1;
-  fa.spec_eth.mask.dst_mac[0] = 1;
-  if ((rd->flow_mcast = ibv_create_flow (rd->rxqs[0].qp, &fa.attr)) == 0)
-    return clib_error_return_unix (0, "create Flow Failed");
-
   return 0;
 }
 
@@ -428,6 +486,27 @@ rdma_create_if (vlib_main_t * vm, rdma_create_if_args_t * args)
   struct ibv_device **dev_list = 0;
   int n_devs;
   u8 *s = 0, *s2 = 0;
+  u16 qid;
+
+  args->rxq_size = args->rxq_size ? args->rxq_size : 2 * VLIB_FRAME_SIZE;
+  args->txq_size = args->txq_size ? args->txq_size : 2 * VLIB_FRAME_SIZE;
+  args->rxq_num = args->rxq_num ? args->rxq_num : 1;
+
+  if (!is_pow2 (args->rxq_num))
+    {
+      args->rv = VNET_API_ERROR_INVALID_VALUE;
+      args->error =
+	clib_error_return (0, "rx queue number must be a power of two");
+      return;
+    }
+
+  if (!is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size))
+    {
+      args->rv = VNET_API_ERROR_INVALID_VALUE;
+      args->error =
+	clib_error_return (0, "queue size must be a power of two");
+      return;
+    }
 
   pool_get_zero (rm->devices, rd);
   rd->dev_instance = rd - rm->devices;
@@ -460,8 +539,8 @@ rdma_create_if (vlib_main_t * vm, rdma_create_if_args_t * args)
     {
       args->error =
 	clib_error_return_unix (0,
-				"no RDMA devices available, errno = %d. Is the ib_uverbs module loaded?",
-				errno);
+				"no RDMA devices available, errno = %d. "
+				"Is the ib_uverbs module loaded?", errno);
       goto err1;
     }
 
@@ -482,7 +561,8 @@ rdma_create_if (vlib_main_t * vm, rdma_create_if_args_t * args)
 	break;
     }
 
-  if ((args->error = rdma_dev_init (vm, rd)))
+  if ((args->error =
+       rdma_dev_init (vm, rd, args->rxq_size, args->txq_size, args->rxq_num)))
     goto err2;
 
   if ((args->error = rdma_register_interface (vnm, rd)))
@@ -502,7 +582,8 @@ rdma_create_if (vlib_main_t * vm, rdma_create_if_args_t * args)
    */
   vnet_hw_interface_set_input_node (vnm, rd->hw_if_index,
 				    rdma_input_node.index);
-  vnet_hw_interface_assign_rx_thread (vnm, rd->hw_if_index, 0, ~0);
+  vec_foreach_index (qid, rd->rxqs)
+    vnet_hw_interface_assign_rx_thread (vnm, rd->hw_if_index, qid, ~0);
   return;
 
 err3:
diff --git a/src/plugins/rdma/input.c b/src/plugins/rdma/input.c
index 2aa98f07e8c..02ac9bf27c6 100644
--- a/src/plugins/rdma/input.c
+++ b/src/plugins/rdma/input.c
@@ -129,7 +129,7 @@ rdma_device_input_refill (vlib_main_t * vm, rdma_device_t * rd,
   w[-1].next = 0;		/* fix next pointer in WR linked-list last item */
 
   w = wr;
-  ibv_post_recv (rxq->qp, wr, &w);
+  ibv_post_wq_recv (rxq->wq, wr, &w);
   n = wr == w ? n_alloc : (uintptr_t) (w - wr);
 
   if (PREDICT_FALSE (n != n_alloc))
diff --git a/src/plugins/rdma/rdma.h b/src/plugins/rdma/rdma.h
index 12901536897..82b62b34ed5 100644
--- a/src/plugins/rdma/rdma.h
+++ b/src/plugins/rdma/rdma.h
@@ -43,7 +43,7 @@ typedef struct
   u32 size;
   u32 n_enq;
   struct ibv_cq *cq;
-  struct ibv_qp *qp;
+  struct ibv_wq *wq;
 } rdma_rxq_t;
 
 typedef struct
@@ -71,13 +71,15 @@ typedef struct
   rdma_rxq_t *rxqs;
   rdma_txq_t *txqs;
 
-  u8 hwaddr[6];
-  vlib_pci_addr_t pci_addr;
   u8 *name;
+  mac_address_t hwaddr;
+  vlib_pci_addr_t pci_addr;
 
   struct ibv_context *ctx;
   struct ibv_pd *pd;
   struct ibv_mr *mr;
+  struct ibv_qp *rx_qp;
+  struct ibv_rwq_ind_table *rx_rwq_ind_tbl;
   struct ibv_flow *flow_ucast;
   struct ibv_flow *flow_mcast;
 
@@ -97,6 +99,9 @@ typedef struct
 {
   u8 *ifname;
   u8 *name;
+  u32 rxq_size;
+  u32 txq_size;
+  u32 rxq_num;
 
   /* return */
   int rv;