/* *------------------------------------------------------------------ * Copyright (c) 2018 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. *------------------------------------------------------------------ */ #include #include #include #include #include #include #include #include #include #include #include #include /* Default RSS hash key (from DPDK MLX driver) */ static u8 rdma_rss_hash_key[] = { 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7, 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94, 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1, 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59, 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a, }; rdma_main_t rdma_main; /* (dev) is of type (rdma_device_t *) */ #define rdma_log__(lvl, dev, f, ...) \ do \ { \ vlib_log ((lvl), rdma_main.log_class, "%s: " f, (dev)->name, \ ##__VA_ARGS__); \ } \ while (0) #define rdma_log(lvl, dev, f, ...) \ rdma_log__((lvl), (dev), "%s (%d): " f, strerror(errno), errno, ##__VA_ARGS__) static struct ibv_flow * rdma_rxq_init_flow (const rdma_device_t * rd, struct ibv_qp *qp, const mac_address_t * mac, const mac_address_t * mask, u16 ether_type, u32 flags) { struct ibv_flow *flow; struct raw_eth_flow_attr { struct ibv_flow_attr attr; struct ibv_flow_spec_eth spec_eth; } __attribute__ ((packed)) fa; memset (&fa, 0, sizeof (fa)); fa.attr.num_of_specs = 1; fa.attr.port = 1; fa.attr.flags = flags; fa.spec_eth.type = IBV_FLOW_SPEC_ETH; fa.spec_eth.size = sizeof (struct ibv_flow_spec_eth); memcpy (fa.spec_eth.val.dst_mac, mac, sizeof (fa.spec_eth.val.dst_mac)); memcpy (fa.spec_eth.mask.dst_mac, mask, sizeof (fa.spec_eth.mask.dst_mac)); if (ether_type) { fa.spec_eth.val.ether_type = ether_type; fa.spec_eth.mask.ether_type = 0xffff; } flow = ibv_create_flow (qp, &fa.attr); if (!flow) rdma_log (VLIB_LOG_LEVEL_ERR, rd, "ibv_create_flow() failed"); return flow; } static u32 rdma_rxq_destroy_flow (const rdma_device_t * rd, struct ibv_flow **flow) { if (!*flow) return 0; if (ibv_destroy_flow (*flow)) { rdma_log (VLIB_LOG_LEVEL_ERR, rd, "ibv_destroy_flow() failed"); return ~0; } *flow = 0; return 0; } static u32 rdma_dev_set_promisc (rdma_device_t * rd) { const mac_address_t all = {.bytes = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0} }; int err; err = rdma_rxq_destroy_flow (rd, &rd->flow_mcast6); err |= rdma_rxq_destroy_flow (rd, &rd->flow_ucast6); err |= rdma_rxq_destroy_flow (rd, &rd->flow_mcast4); err |= rdma_rxq_destroy_flow (rd, &rd->flow_ucast4); if (err) return ~0; rd->flow_ucast6 = rdma_rxq_init_flow (rd, rd->rx_qp6, &all, &all, ntohs (ETH_P_IPV6), 0); rd->flow_ucast4 = rdma_rxq_init_flow (rd, rd->rx_qp4, &all, &all, 0, 0); if (!rd->flow_ucast6 || !rd->flow_ucast4) return ~0; rd->flags |= RDMA_DEVICE_F_PROMISC; return 0; } static u32 rdma_dev_set_ucast (rdma_device_t * rd) { const mac_address_t ucast = {.bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff} }; const mac_address_t mcast = {.bytes = {0x1, 0x0, 0x0, 0x0, 0x0, 0x0} }; int err; err = rdma_rxq_destroy_flow (rd, &rd->flow_mcast6); err |= rdma_rxq_destroy_flow (rd, &rd->flow_ucast6); err |= rdma_rxq_destroy_flow (rd, &rd->flow_mcast4); err |= rdma_rxq_destroy_flow (rd, &rd->flow_ucast4); if (err) return ~0; rd->flow_ucast6 = rdma_rxq_init_flow (rd, rd->rx_qp6, &rd->hwaddr, &ucast, ntohs (ETH_P_IPV6), 0); rd->flow_mcast6 = rdma_rxq_init_flow (rd, rd->rx_qp6, &mcast, &mcast, ntohs (ETH_P_IPV6), IBV_FLOW_ATTR_FLAGS_DONT_TRAP /* let others receive mcast packet too (eg. Linux) */ ); rd->flow_ucast4 = rdma_rxq_init_flow (rd, rd->rx_qp4, &rd->hwaddr, &ucast, 0, 0); rd->flow_mcast4 = rdma_rxq_init_flow (rd, rd->rx_qp4, &mcast, &mcast, 0, IBV_FLOW_ATTR_FLAGS_DONT_TRAP /* let others receive mcast packet too (eg. Linux) */ ); if (!rd->flow_ucast6 || !rd->flow_mcast6 || !rd->flow_ucast4 || !rd->flow_mcast4) return ~0; rd->flags &= ~RDMA_DEVICE_F_PROMISC; return 0; } static clib_error_t * rdma_mac_change (vnet_hw_interface_t * hw, const u8 * old, const u8 * new) { rdma_main_t *rm = &rdma_main; rdma_device_t *rd = vec_elt_at_index (rm->devices, hw->dev_instance); mac_address_from_bytes (&rd->hwaddr, new); if (!(rd->flags & RDMA_DEVICE_F_PROMISC) && rdma_dev_set_ucast (rd)) { mac_address_from_bytes (&rd->hwaddr, old); return clib_error_return_unix (0, "MAC update failed"); } return 0; } static clib_error_t * rdma_set_max_frame_size (vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 frame_size) { return vnet_error (VNET_ERR_UNSUPPORTED, 0); } static u32 rdma_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, u32 flags) { rdma_main_t *rm = &rdma_main; rdma_device_t *rd = vec_elt_at_index (rm->devices, hw->dev_instance); switch (flags) { case ETHERNET_INTERFACE_FLAG_DEFAULT_L3: return rdma_dev_set_ucast (rd); case ETHERNET_INTERFACE_FLAG_ACCEPT_ALL: return rdma_dev_set_promisc (rd); } rdma_log__ (VLIB_LOG_LEVEL_ERR, rd, "unknown flag %x requested", flags); return ~0; } static void rdma_update_state (vnet_main_t * vnm, rdma_device_t * rd, int port) { struct ibv_port_attr attr; u32 width = 0; u32 speed = 0; if (ibv_query_port (rd->ctx, port, &attr)) { vnet_hw_interface_set_link_speed (vnm, rd->hw_if_index, 0); vnet_hw_interface_set_flags (vnm, rd->hw_if_index, 0); return; } /* update state */ switch (attr.state) { case IBV_PORT_ACTIVE: /* fallthrough */ case IBV_PORT_ACTIVE_DEFER: rd->flags |= RDMA_DEVICE_F_LINK_UP; vnet_hw_interface_set_flags (vnm, rd->hw_if_index, VNET_HW_INTERFACE_FLAG_LINK_UP); break; default: rd->flags &= ~RDMA_DEVICE_F_LINK_UP; vnet_hw_interface_set_flags (vnm, rd->hw_if_index, 0); break; } /* update speed */ switch (attr.active_width) { case 1: width = 1; break; case 2: width = 4; break; case 4: width = 8; break; case 8: width = 12; break; } switch (attr.active_speed) { case 1: speed = 2500000; break; case 2: speed = 5000000; break; case 4: /* fallthrough */ case 8: speed = 10000000; break; case 16: speed = 14000000; break; case 32: speed = 25000000; break; } vnet_hw_interface_set_link_speed (vnm, rd->hw_if_index, width * speed); } static clib_error_t * rdma_async_event_error_ready (clib_file_t * f) { rdma_main_t *rm = &rdma_main; rdma_device_t *rd = vec_elt_at_index (rm->devices, f->private_data); return clib_error_return (0, "RDMA: %s: async event error", rd->name); } static clib_error_t * rdma_async_event_read_ready (clib_file_t * f) { vnet_main_t *vnm = vnet_get_main (); rdma_main_t *rm = &rdma_main; rdma_device_t *rd = vec_elt_at_index (rm->devices, f->private_data); int ret; struct ibv_async_event event; ret = ibv_get_async_event (rd->ctx, &event); if (ret < 0) return clib_error_return_unix (0, "ibv_get_async_event() failed"); switch (event.event_type) { case IBV_EVENT_PORT_ACTIVE: rdma_update_state (vnm, rd, event.element.port_num); break; case IBV_EVENT_PORT_ERR: rdma_update_state (vnm, rd, event.element.port_num); break; case IBV_EVENT_DEVICE_FATAL: rd->flags &= ~RDMA_DEVICE_F_LINK_UP; vnet_hw_interface_set_flags (vnm, rd->hw_if_index, 0); vlib_log_emerg (rm->log_class, "%s: fatal error", rd->name); break; default: rdma_log__ (VLIB_LOG_LEVEL_ERR, rd, "unhandeld RDMA async event %d", event.event_type); break; } ibv_ack_async_event (&event); return 0; } static clib_error_t * rdma_async_event_init (rdma_device_t * rd) { clib_file_t t = { 0 }; int ret; /* make RDMA async event fd non-blocking */ ret = fcntl (rd->ctx->async_fd, F_GETFL); if (ret < 0) return clib_error_return_unix (0, "fcntl(F_GETFL) failed"); ret = fcntl (rd->ctx->async_fd, F_SETFL, ret | O_NONBLOCK); if (ret < 0) return clib_error_return_unix (0, "fcntl(F_SETFL, O_NONBLOCK) failed"); /* register RDMA async event fd */ t.read_function = rdma_async_event_read_ready; t.file_descriptor = rd->ctx->async_fd; t.error_function = rdma_async_event_error_ready; t.private_data = rd->dev_instance; t.description = format (0, "%v async event", rd->name); rd->async_event_clib_file_index = clib_file_add (&file_main, &t); return 0; } static void rdma_async_event_cleanup (rdma_device_t * rd) { clib_file_del_by_index (&file_main, rd->async_event_clib_file_index); } static clib_error_t * rdma_register_interface (vnet_main_t * vnm, rdma_device_t * rd) { vnet_eth_interface_registration_t eir = {}; eir.dev_class_index = rdma_device_class.index; eir.dev_instance = rd->dev_instance; eir.address = rd->hwaddr.bytes; eir.cb.flag_change = rdma_flag_change; eir.cb.set_max_frame_size = rdma_set_max_frame_size; rd->hw_if_index = vnet_eth_register_interface (vnm, &eir); /* Indicate ability to support L3 DMAC filtering and * initialize interface to L3 non-promisc mode */ vnet_hw_if_set_caps (vnm, rd->hw_if_index, VNET_HW_IF_CAP_MAC_FILTER); ethernet_set_flags (vnm, rd->hw_if_index, ETHERNET_INTERFACE_FLAG_DEFAULT_L3); return 0; } static void rdma_unregister_interface (vnet_main_t * vnm, rdma_device_t * rd) { vnet_hw_interface_set_flags (vnm, rd->hw_if_index, 0); ethernet_delete_interface (vnm, rd->hw_if_index); } static void rdma_dev_cleanup (rdma_device_t * rd) { rdma_main_t *rm = &rdma_main; rdma_rxq_t *rxq; rdma_txq_t *txq; #define _(fn, arg) if (arg) \ { \ int rv; \ if ((rv = fn (arg))) \ rdma_log (VLIB_LOG_LEVEL_DEBUG, rd, #fn "() failed (rv = %d)", rv); \ } _(ibv_destroy_flow, rd->flow_mcast6); _(ibv_destroy_flow, rd->flow_ucast6); _(ibv_destroy_flow, rd->flow_mcast4); _(ibv_destroy_flow, rd->flow_ucast4); _(ibv_dereg_mr, rd->mr); vec_foreach (txq, rd->txqs) { _(ibv_destroy_qp, txq->qp); _(ibv_destroy_cq, txq->cq); } vec_foreach (rxq, rd->rxqs) { _(ibv_destroy_wq, rxq->wq); _(ibv_destroy_cq, rxq->cq); } _(ibv_destroy_rwq_ind_table, rd->rx_rwq_ind_tbl); _(ibv_destroy_qp, rd->rx_qp6); _(ibv_destroy_qp, rd->rx_qp4); _(ibv_dealloc_pd, rd->pd); _(ibv_close_device, rd->ctx); #undef _ clib_error_free (rd->error); vec_free (rd->rxqs); vec_free (rd->txqs); vec_free (rd->name); vlib_pci_free_device_info (rd->pci); pool_put (rm->devices, rd); } static clib_error_t * rdma_rxq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc, u8 no_multi_seg, u16 max_pktlen) { rdma_rxq_t *rxq; struct ibv_wq_init_attr wqia; struct ibv_cq_init_attr_ex cqa = { }; struct ibv_wq_attr wqa; struct ibv_cq_ex *cqex; struct mlx5dv_wq_init_attr dv_wqia = { }; int is_mlx5dv = ! !(rd->flags & RDMA_DEVICE_F_MLX5DV); int is_striding = ! !(rd->flags & RDMA_DEVICE_F_STRIDING_RQ); vec_validate_aligned (rd->rxqs, qid, CLIB_CACHE_LINE_BYTES); rxq = vec_elt_at_index (rd->rxqs, qid); rxq->size = n_desc; rxq->log_wqe_sz = 0; rxq->buf_sz = vlib_buffer_get_default_data_size (vm); vec_validate_aligned (rxq->bufs, n_desc - 1, CLIB_CACHE_LINE_BYTES); cqa.cqe = n_desc; if (is_mlx5dv) { struct mlx5dv_cq_init_attr dvcq = { }; dvcq.comp_mask = MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE | MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE; dvcq.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH; dvcq.cqe_size = 64; if ((cqex = mlx5dv_create_cq (rd->ctx, &cqa, &dvcq)) == 0) return clib_error_return_unix (0, "Create mlx5dv rx CQ Failed"); } else { if ((cqex = ibv_create_cq_ex (rd->ctx, &cqa)) == 0) return clib_error_return_unix (0, "Create CQ Failed"); } rxq->cq = ibv_cq_ex_to_cq (cqex); memset (&wqia, 0, sizeof (wqia)); wqia.wq_type = IBV_WQT_RQ; wqia.max_wr = n_desc; wqia.max_sge = 1; wqia.pd = rd->pd; wqia.cq = rxq->cq; if (is_mlx5dv) { if (is_striding) { /* In STRIDING_RQ mode, map a descriptor to a stride, not a full WQE buffer */ uword data_seg_log2_sz = min_log2 (vlib_buffer_get_default_data_size (vm)); rxq->buf_sz = 1 << data_seg_log2_sz; /* The trick is also to map a descriptor to a data segment in the WQE SG list The number of strides per WQE and the size of a WQE (in 16-bytes words) both must be powers of two. Moreover, in striding RQ mode, WQEs must include the SRQ header, which occupies one 16-bytes word. That is why WQEs have 2*RDMA_RXQ_MAX_CHAIN_SZ 16-bytes words: - One for the SRQ Header - RDMA_RXQ_MAX_CHAIN_SZ for the different data segments (each mapped to a stride, and a vlib_buffer) - RDMA_RXQ_MAX_CHAIN_SZ-1 null data segments */ int max_chain_log_sz = max_pktlen ? max_log2 ((max_pktlen / (rxq->buf_sz)) + 1) : RDMA_RXQ_MAX_CHAIN_LOG_SZ; max_chain_log_sz = clib_max (max_chain_log_sz, 3); wqia.max_sge = 1 << max_chain_log_sz; dv_wqia.comp_mask = MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ; dv_wqia.striding_rq_attrs.two_byte_shift_en = 0; dv_wqia.striding_rq_attrs.single_wqe_log_num_of_strides = max_chain_log_sz; dv_wqia.striding_rq_attrs.single_stride_log_num_of_bytes = data_seg_log2_sz; wqia.max_wr >>= max_chain_log_sz; rxq->log_wqe_sz = max_chain_log_sz + 1; rxq->log_stride_per_wqe = max_chain_log_sz; } else { /* In non STRIDING_RQ mode and if multiseg is not disabled, each WQE is a SG list of data segments, each pointing to a vlib_buffer. */ if (no_multi_seg) { wqia.max_sge = 1; rxq->log_wqe_sz = 0; rxq->n_ds_per_wqe = 1; } else { int max_chain_sz = max_pktlen ? (max_pktlen / (rxq->buf_sz)) + 1 : RDMA_RXQ_LEGACY_MODE_MAX_CHAIN_SZ; int max_chain_log_sz = max_log2 (max_chain_sz); wqia.max_sge = 1 << max_chain_log_sz; rxq->log_wqe_sz = max_chain_log_sz; rxq->n_ds_per_wqe = max_chain_sz; } } if ((rxq->wq = mlx5dv_create_wq (rd->ctx, &wqia, &dv_wqia))) { rxq->wq->events_completed = 0; pthread_mutex_init (&rxq->wq->mutex, NULL); pthread_cond_init (&rxq->wq->cond, NULL); } else return clib_error_return_unix (0, "Create WQ Failed"); } else if ((rxq->wq = ibv_create_wq (rd->ctx, &wqia)) == 0) return clib_error_return_unix (0, "Create WQ Failed"); memset (&wqa, 0, sizeof (wqa)); wqa.attr_mask = IBV_WQ_ATTR_STATE; wqa.wq_state = IBV_WQS_RDY; if (ibv_modify_wq (rxq->wq, &wqa) != 0) return clib_error_return_unix (0, "Modify WQ (RDY) Failed"); if (is_mlx5dv) { struct mlx5dv_obj obj = { }; struct mlx5dv_cq dv_cq; struct mlx5dv_rwq dv_rwq; u64 qw0; u64 qw0_nullseg; u32 wqe_sz_mask = (1 << rxq->log_wqe_sz) - 1; obj.cq.in = rxq->cq; obj.cq.out = &dv_cq; obj.rwq.in = rxq->wq; obj.rwq.out = &dv_rwq; if ((mlx5dv_init_obj (&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_RWQ))) return clib_error_return_unix (0, "mlx5dv: failed to init rx obj"); if (dv_cq.cqe_size != sizeof (mlx5dv_cqe_t)) return clib_error_return_unix (0, "mlx5dv: incompatible rx CQE size"); rxq->log2_cq_size = max_log2 (dv_cq.cqe_cnt); rxq->cqes = (mlx5dv_cqe_t *) dv_cq.buf; rxq->cq_db = (volatile u32 *) dv_cq.dbrec; rxq->cqn = dv_cq.cqn; rxq->wqes = (mlx5dv_wqe_ds_t *) dv_rwq.buf; rxq->wq_db = (volatile u32 *) dv_rwq.dbrec; rxq->wq_stride = dv_rwq.stride; rxq->wqe_cnt = dv_rwq.wqe_cnt; qw0 = clib_host_to_net_u32 (rxq->buf_sz); qw0_nullseg = 0; qw0 |= (u64) clib_host_to_net_u32 (rd->lkey) << 32; qw0_nullseg |= (u64) clib_host_to_net_u32 (rd->lkey) << 32; /* Prefill the different 16 bytes words of the WQ. - If not in striding RQ mode, for each WQE, init with qw0 the first RDMA_RXQ_LEGACY_MODE_MAX_CHAIN_SZ, and init the rest of the WQE with null segments. - If in striding RQ mode, for each WQE, the RDMA_RXQ_MAX_CHAIN_SZ + 1 first 16-bytes words are initialised with qw0, the rest are null segments */ for (int i = 0; i < rxq->wqe_cnt << rxq->log_wqe_sz; i++) if ((!is_striding && ((i & wqe_sz_mask) < rxq->n_ds_per_wqe)) || (is_striding && ((i == 0) || !(((i - 1) >> rxq->log_stride_per_wqe) & 0x1)))) rxq->wqes[i].dsz_and_lkey = qw0; else rxq->wqes[i].dsz_and_lkey = qw0_nullseg; for (int i = 0; i < (1 << rxq->log2_cq_size); i++) rxq->cqes[i].opcode_cqefmt_se_owner = 0xff; if (!is_striding) { vec_validate_aligned (rxq->second_bufs, n_desc - 1, CLIB_CACHE_LINE_BYTES); vec_validate_aligned (rxq->n_used_per_chain, n_desc - 1, CLIB_CACHE_LINE_BYTES); rxq->n_total_additional_segs = n_desc * (rxq->n_ds_per_wqe - 1); for (int i = 0; i < n_desc; i++) rxq->n_used_per_chain[i] = rxq->n_ds_per_wqe - 1; } } return 0; } static uint64_t rdma_rss42ibv (const rdma_rss4_t rss4) { switch (rss4) { case RDMA_RSS4_IP: return IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4; case RDMA_RSS4_IP_UDP: return IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4 | IBV_RX_HASH_SRC_PORT_UDP | IBV_RX_HASH_DST_PORT_UDP; case RDMA_RSS4_AUTO: /* fallthrough */ case RDMA_RSS4_IP_TCP: return IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4 | IBV_RX_HASH_SRC_PORT_TCP | IBV_RX_HASH_DST_PORT_TCP; } ASSERT (0); return 0; } static uint64_t rdma_rss62ibv (const rdma_rss6_t rss6) { switch (rss6) { case RDMA_RSS6_IP: return IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6; case RDMA_RSS6_IP_UDP: return IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6 | IBV_RX_HASH_SRC_PORT_UDP | IBV_RX_HASH_DST_PORT_UDP; case RDMA_RSS6_AUTO: /* fallthrough */ case RDMA_RSS6_IP_TCP: return IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6 | IBV_RX_HASH_SRC_PORT_TCP | IBV_RX_HASH_DST_PORT_TCP; } ASSERT (0); return 0; } static clib_error_t * rdma_rxq_finalize (vlib_main_t *vm, rdma_device_t *rd) { struct ibv_rwq_ind_table_init_attr rwqia; struct ibv_qp_init_attr_ex qpia; struct ibv_wq **ind_tbl; const u32 rxq_sz = vec_len (rd->rxqs); u32 ind_tbl_sz = rxq_sz; u32 i; if (!is_pow2 (ind_tbl_sz)) { /* in case we do not have a power-of-2 number of rxq, we try to use the * maximum supported to minimize the imbalance */ struct ibv_device_attr_ex attr; if (ibv_query_device_ex (rd->ctx, 0, &attr)) return clib_error_return_unix (0, "device query failed"); ind_tbl_sz = attr.rss_caps.max_rwq_indirection_table_size; if (ind_tbl_sz < rxq_sz) return clib_error_create ("too many rxqs requested (%d) compared to " "max indirection table size (%d)", rxq_sz, ind_tbl_sz); } ind_tbl = vec_new (struct ibv_wq *, ind_tbl_sz); vec_foreach_index (i, ind_tbl) vec_elt (ind_tbl, i) = vec_elt (rd->rxqs, i % rxq_sz).wq; memset (&rwqia, 0, sizeof (rwqia)); ASSERT (is_pow2 (vec_len (ind_tbl))); rwqia.log_ind_tbl_size = min_log2 (vec_len (ind_tbl)); rwqia.ind_tbl = ind_tbl; if ((rd->rx_rwq_ind_tbl = ibv_create_rwq_ind_table (rd->ctx, &rwqia)) == 0) return clib_error_return_unix (0, "RWQ indirection table create failed"); vec_free (ind_tbl); memset (&qpia, 0, sizeof (qpia)); qpia.qp_type = IBV_QPT_RAW_PACKET; qpia.comp_mask = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_IND_TABLE | IBV_QP_INIT_ATTR_RX_HASH; qpia.pd = rd->pd; qpia.rwq_ind_tbl = rd->rx_rwq_ind_tbl; STATIC_ASSERT_SIZEOF (rdma_rss_hash_key, 40); qpia.rx_hash_conf.rx_hash_key_len = sizeof (rdma_rss_hash_key); qpia.rx_hash_conf.rx_hash_key = rdma_rss_hash_key; qpia.rx_hash_conf.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ; qpia.rx_hash_conf.rx_hash_fields_mask = rdma_rss42ibv (rd->rss4); if ((rd->rx_qp4 = ibv_create_qp_ex (rd->ctx, &qpia)) == 0) return clib_error_return_unix (0, "IPv4 Queue Pair create failed"); qpia.rx_hash_conf.rx_hash_fields_mask = rdma_rss62ibv (rd->rss6); if ((rd->rx_qp6 = ibv_create_qp_ex (rd->ctx, &qpia)) == 0) return clib_error_return_unix (0, "IPv6 Queue Pair create failed"); if (rdma_dev_set_ucast (rd)) return clib_error_return_unix (0, "Set unicast mode failed"); return 0; } static clib_error_t * rdma_txq_init (vlib_main_t * vm, rdma_device_t * rd, u16 qid, u32 n_desc) { rdma_txq_t *txq; struct ibv_qp_init_attr qpia; struct ibv_qp_attr qpa; int qp_flags; int is_mlx5dv = !!(rd->flags & RDMA_DEVICE_F_MLX5DV); vec_validate_aligned (rd->txqs, qid, CLIB_CACHE_LINE_BYTES); txq = vec_elt_at_index (rd->txqs, qid); ASSERT (is_pow2 (n_desc)); txq->bufs_log2sz = min_log2 (n_desc); vec_validate_aligned (txq->bufs, n_desc - 1, CLIB_CACHE_LINE_BYTES); if (is_mlx5dv) { struct ibv_cq_init_attr_ex cqa = {}; struct ibv_cq_ex *cqex; struct mlx5dv_cq_init_attr dvcq = {}; dvcq.comp_mask = MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE; dvcq.cqe_size = 64; cqa.cqe = n_desc; if ((cqex = mlx5dv_create_cq (rd->ctx, &cqa, &dvcq)) == 0) return clib_error_return_unix (0, "Create mlx5dv tx CQ Failed"); txq->cq = ibv_cq_ex_to_cq (cqex); } else { if ((txq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0) return clib_error_return_unix (0, "Create CQ Failed"); } memset (&qpia, 0, sizeof (qpia)); qpia.send_cq = txq->cq; qpia.recv_cq = txq->cq; qpia.cap.max_send_wr = n_desc; qpia.cap.max_send_sge = 1; qpia.qp_type = IBV_QPT_RAW_PACKET; if ((txq->qp = ibv_create_qp (rd->pd, &qpia)) == 0) return clib_error_return_unix (0, "Queue Pair create failed"); memset (&qpa, 0, sizeof (qpa)); qp_flags = IBV_QP_STATE | IBV_QP_PORT; qpa.qp_state = IBV_QPS_INIT; qpa.port_num = 1; if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0) return clib_error_return_unix (0, "Modify QP (init) Failed"); memset (&qpa, 0, sizeof (qpa)); qp_flags = IBV_QP_STATE; qpa.qp_state = IBV_QPS_RTR; if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0) return clib_error_return_unix (0, "Modify QP (receive) Failed"); memset (&qpa, 0, sizeof (qpa)); qp_flags = IBV_QP_STATE; qpa.qp_state = IBV_QPS_RTS; if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0) return clib_error_return_unix (0, "Modify QP (send) Failed"); txq->ibv_cq = txq->cq; txq->ibv_qp = txq->qp; if (rd->flags & RDMA_DEVICE_F_MLX5DV) { rdma_mlx5_wqe_t *tmpl = (void *) txq->dv_wqe_tmpl; struct mlx5dv_cq dv_cq; struct mlx5dv_qp dv_qp; struct mlx5dv_obj obj = { }; obj.cq.in = txq->cq; obj.cq.out = &dv_cq; obj.qp.in = txq->qp; obj.qp.out = &dv_qp; if (mlx5dv_init_obj (&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_QP)) return clib_error_return_unix (0, "DV init obj failed"); if (RDMA_TXQ_BUF_SZ (txq) > dv_qp.sq.wqe_cnt || !is_pow2 (dv_qp.sq.wqe_cnt) || sizeof (rdma_mlx5_wqe_t) != dv_qp.sq.stride || (uword) dv_qp.sq.buf % sizeof (rdma_mlx5_wqe_t)) return clib_error_return (0, "Unsupported DV SQ parameters"); if (RDMA_TXQ_BUF_SZ (txq) > dv_cq.cqe_cnt || !is_pow2 (dv_cq.cqe_cnt) || sizeof (struct mlx5_cqe64) != dv_cq.cqe_size || (uword) dv_cq.buf % sizeof (struct mlx5_cqe64)) return clib_error_return (0, "Unsupported DV CQ parameters"); /* get SQ and doorbell addresses */ txq->dv_sq_wqes = dv_qp.sq.buf; txq->dv_sq_dbrec = dv_qp.dbrec; txq->dv_sq_db = dv_qp.bf.reg; txq->dv_sq_log2sz = min_log2 (dv_qp.sq.wqe_cnt); /* get CQ and doorbell addresses */ txq->dv_cq_cqes = dv_cq.buf; txq->dv_cq_dbrec = dv_cq.dbrec; txq->dv_cq_log2sz = min_log2 (dv_cq.cqe_cnt); /* init tx desc template */ STATIC_ASSERT_SIZEOF (txq->dv_wqe_tmpl, sizeof (*tmpl)); mlx5dv_set_ctrl_seg (&tmpl->ctrl, 0, MLX5_OPCODE_SEND, 0, txq->qp->qp_num, 0, RDMA_MLX5_WQE_DS, 0, RDMA_TXQ_DV_INVALID_ID); tmpl->eseg.inline_hdr_sz = htobe16 (MLX5_ETH_L2_INLINE_HEADER_SIZE); mlx5dv_set_data_seg (&tmpl->dseg, 0, rd->lkey, 0); } return 0; } static clib_error_t * rdma_dev_init (vlib_main_t * vm, rdma_device_t * rd, rdma_create_if_args_t * args) { clib_error_t *err; vlib_buffer_main_t *bm = vm->buffer_main; vlib_thread_main_t *tm = vlib_get_thread_main (); u32 rxq_num = args->rxq_num; u32 rxq_size = args->rxq_size; u32 txq_size = args->txq_size; u32 i; if (rd->ctx == 0) return clib_error_return_unix (0, "Device Open Failed"); if ((rd->pd = ibv_alloc_pd (rd->ctx)) == 0) return clib_error_return_unix (0, "PD Alloc Failed"); if ((rd->mr = ibv_reg_mr (rd->pd, (void *) bm->buffer_mem_start, bm->buffer_mem_size, IBV_ACCESS_LOCAL_WRITE)) == 0) return clib_error_return_unix (0, "Register MR Failed"); rd->lkey = rd->mr->lkey; /* avoid indirection in datapath */ ethernet_mac_address_generate (rd->hwaddr.bytes); rd->rss4 = args->rss4; rd->rss6 = args->rss6; /* * /!\ WARNING /!\ creation order is important * We *must* create TX queues *before* RX queues, otherwise we will receive * the broacast packets we sent */ for (i = 0; i < tm->n_vlib_mains; i++) if ((err = rdma_txq_init (vm, rd, i, txq_size))) return err; for (i = 0; i < rxq_num; i++) if ((err = rdma_rxq_init (vm, rd, i, rxq_size, args->no_multi_seg, args->max_pktlen))) return err; if ((err = rdma_rxq_finalize (vm, rd))) return err; return 0; } static uword sysfs_path_to_pci_addr (char *path, vlib_pci_addr_t * addr) { uword rv; unformat_input_t in; u8 *s; s = clib_file_get_resolved_basename (path); if (!s) return 0; unformat_init_string (&in, (char *) s, strlen ((char *) s)); rv = unformat (&in, "%U", unformat_vlib_pci_addr, addr); unformat_free (&in); vec_free (s); return rv; } void rdma_create_if (vlib_main_t * vm, rdma_create_if_args_t * args) { vnet_main_t *vnm = vnet_get_main (); rdma_main_t *rm = &rdma_main; rdma_device_t *rd; vlib_pci_addr_t pci_addr; struct ibv_device **dev_list; int n_devs; u8 *s; u16 qid; int i; args->rxq_size = args->rxq_size ? args->rxq_size : 1024; args->txq_size = args->txq_size ? args->txq_size : 1024; args->rxq_num = args->rxq_num ? args->rxq_num : 2; if (args->rxq_size < VLIB_FRAME_SIZE || args->txq_size < VLIB_FRAME_SIZE || args->rxq_size > 65535 || args->txq_size > 65535 || !is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size)) { args->rv = VNET_API_ERROR_INVALID_VALUE; args->error = clib_error_return (0, "queue size must be a power of two " "between %d and 65535", VLIB_FRAME_SIZE); goto err0; } dev_list = ibv_get_device_list (&n_devs); if (n_devs == 0) { args->error = clib_error_return_unix (0, "no RDMA devices available. Is the ib_uverbs module loaded?"); goto err0; } /* get PCI address */ s = format (0, "/sys/class/net/%s/device%c", args->ifname, 0); if (sysfs_path_to_pci_addr ((char *) s, &pci_addr) == 0) { args->error = clib_error_return (0, "cannot find PCI address for device "); goto err1; } pool_get_zero (rm->devices, rd); rd->dev_instance = rd - rm->devices; rd->per_interface_next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; rd->linux_ifname = format (0, "%s", args->ifname); if (!args->name || 0 == args->name[0]) rd->name = format (0, "%s/%d", args->ifname, rd->dev_instance); else rd->name = format (0, "%s", args->name); rd->pci = vlib_pci_get_device_info (vm, &pci_addr, &args->error); if (!rd->pci) goto err2; /* if we failed to parse NUMA node, default to 0 */ if (-1 == rd->pci->numa_node) rd->pci->numa_node = 0; rd->pool = vlib_buffer_pool_get_default_for_numa (vm, rd->pci->numa_node); if (strncmp ((char *) rd->pci->driver_name, "mlx5_core", 9)) { args->error = clib_error_return (0, "invalid interface (only mlx5 supported for now)"); goto err2; } for (i = 0; i < n_devs; i++) { vlib_pci_addr_t addr; vec_reset_length (s); s = format (s, "%s/device%c", dev_list[i]->dev_path, 0); if (sysfs_path_to_pci_addr ((char *) s, &addr) == 0) continue; if (addr.as_u32 != rd->pci->addr.as_u32) continue; if ((rd->ctx = ibv_open_device (dev_list[i]))) break; } if (args->mode != RDMA_MODE_IBV) { struct mlx5dv_context mlx5dv_attrs = { }; mlx5dv_attrs.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ; if (mlx5dv_query_device (rd->ctx, &mlx5dv_attrs) == 0) { uword data_seg_log2_sz = min_log2 (vlib_buffer_get_default_data_size (vm)); if ((mlx5dv_attrs.flags & MLX5DV_CONTEXT_FLAGS_CQE_V1)) rd->flags |= RDMA_DEVICE_F_MLX5DV; /* Enable striding RQ if neither multiseg nor striding rq are explicitly disabled, and if the interface supports it.*/ if (!args->no_multi_seg && !args->disable_striding_rq && data_seg_log2_sz <= mlx5dv_attrs.striding_rq_caps.max_single_stride_log_num_of_bytes && data_seg_log2_sz >= mlx5dv_attrs.striding_rq_caps.min_single_stride_log_num_of_bytes && RDMA_RXQ_MAX_CHAIN_LOG_SZ >= mlx5dv_attrs.striding_rq_caps.min_single_wqe_log_num_of_strides && RDMA_RXQ_MAX_CHAIN_LOG_SZ <= mlx5dv_attrs.striding_rq_caps.max_single_wqe_log_num_of_strides) rd->flags |= RDMA_DEVICE_F_STRIDING_RQ; } else { if (args->mode == RDMA_MODE_DV) { args->error = clib_error_return (0, "Direct Verbs mode not " "supported on this interface"); goto err2; } } } if ((args->error = rdma_dev_init (vm, rd, args))) goto err2; if ((args->error = rdma_register_interface (vnm, rd))) goto err2; if ((args->error = rdma_async_event_init (rd))) goto err3; rdma_update_state (vnm, rd, 1); vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, rd->hw_if_index); args->sw_if_index = rd->sw_if_index = sw->sw_if_index; /* * FIXME: add support for interrupt mode * vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, rd->hw_if_index); * hw->caps |= VNET_HW_IF_CAP_INT_MODE; */ vnet_hw_if_set_input_node (vnm, rd->hw_if_index, rdma_input_node.index); vec_foreach_index (qid, rd->rxqs) { u32 queue_index = vnet_hw_if_register_rx_queue ( vnm, rd->hw_if_index, qid, VNET_HW_IF_RXQ_THREAD_ANY); rd->rxqs[qid].queue_index = queue_index; } vnet_hw_if_update_runtime_data (vnm, rd->hw_if_index); vec_free (s); return; err3: rdma_unregister_interface (vnm, rd); err2: rdma_dev_cleanup (rd); err1: ibv_free_device_list (dev_list); vec_free (s); args->rv = VNET_API_ERROR_INVALID_INTERFACE; err0: vlib_log_err (rm->log_class, "%U", format_clib_error, args->error); } void rdma_delete_if (vlib_main_t * vm, rdma_device_t * rd) { rdma_async_event_cleanup (rd); rdma_unregister_interface (vnet_get_main (), rd); rdma_dev_cleanup (rd); } static clib_error_t * rdma_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags) { vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index); rdma_main_t *rm = &rdma_main; rdma_device_t *rd = vec_elt_at_index (rm->devices, hi->dev_instance); uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0; if (rd->flags & RDMA_DEVICE_F_ERROR) return clib_error_return (0, "device is in error state"); if (is_up) { vnet_hw_interface_set_flags (vnm, rd->hw_if_index, VNET_HW_INTERFACE_FLAG_LINK_UP); rd->flags |= RDMA_DEVICE_F_ADMIN_UP; } else { vnet_hw_interface_set_flags (vnm, rd->hw_if_index, 0); rd->flags &= ~RDMA_DEVICE_F_ADMIN_UP; } return 0; } static void rdma_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index, u32 node_index) { rdma_main_t *rm = &rdma_main; vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); rdma_device_t *rd = pool_elt_at_index (rm->devices, hw->dev_instance); rd->per_interface_next_index = ~0 == node_index ? VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT : vlib_node_add_next (vlib_get_main (), rdma_input_node.index, node_index); } static char *rdma_tx_func_error_strings[] = { #define _(n,s) s, foreach_rdma_tx_func_error #undef _ }; VNET_DEVICE_CLASS (rdma_device_class) = { .name = "RDMA interface", .format_device = format_rdma_device, .format_device_name = format_rdma_device_name, .admin_up_down_function = rdma_interface_admin_up_down, .rx_redirect_to_node = rdma_set_interface_next_node, .tx_function_n_errors = RDMA_TX_N_ERROR, .tx_function_error_strings = rdma_tx_func_error_strings, .mac_addr_change_function = rdma_mac_change, }; clib_error_t * rdma_init (vlib_main_t * vm) { rdma_main_t *rm = &rdma_main; vlib_thread_main_t *tm = vlib_get_thread_main (); rm->log_class = vlib_log_register_class ("rdma", 0); /* vlib_buffer_t template */ vec_validate_aligned (rm->per_thread_data, tm->n_vlib_mains - 1, CLIB_CACHE_LINE_BYTES); for (int i = 0; i < tm->n_vlib_mains; i++) { rdma_per_thread_data_t *ptd = vec_elt_at_index (rm->per_thread_data, i); clib_memset (&ptd->buffer_template, 0, sizeof (vlib_buffer_t)); ptd->buffer_template.flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; ptd->buffer_template.ref_count = 1; vnet_buffer (&ptd->buffer_template)->sw_if_index[VLIB_TX] = (u32) ~ 0; } return 0; } VLIB_INIT_FUNCTION (rdma_init);