From 7d8d95fbce7b101c51bae6f468b7942dcdaf1032 Mon Sep 17 00:00:00 2001 From: Hanoh Haim Date: Tue, 8 Nov 2016 13:10:15 +0200 Subject: mlx5 support build WIP Signed-off-by: Hanoh Haim --- external_libs/ibverbs/include/infiniband/arch.h | 160 + external_libs/ibverbs/include/infiniband/driver.h | 236 ++ .../ibverbs/include/infiniband/driver_exp.h | 151 + .../ibverbs/include/infiniband/kern-abi.h | 1144 +++++++ .../ibverbs/include/infiniband/kern-abi_exp.h | 722 ++++ .../ibverbs/include/infiniband/marshall.h | 65 + external_libs/ibverbs/include/infiniband/mlx5_hw.h | 797 +++++ .../ibverbs/include/infiniband/ofa_verbs.h | 210 ++ external_libs/ibverbs/include/infiniband/opcode.h | 147 + .../ibverbs/include/infiniband/peer_ops.h | 370 ++ .../ibverbs/include/infiniband/sa-kern-abi.h | 65 + external_libs/ibverbs/include/infiniband/sa.h | 135 + external_libs/ibverbs/include/infiniband/verbs.h | 1642 +++++++++ .../ibverbs/include/infiniband/verbs_exp.h | 3585 ++++++++++++++++++++ external_libs/ibverbs/libibverbs.a | Bin 0 -> 937626 bytes external_libs/ibverbs/libibverbs.la | 41 + external_libs/ibverbs/libibverbs.lai | 41 + external_libs/ibverbs/libibverbs.so.1 | Bin 0 -> 494224 bytes external_libs/ibverbs/libibverbs.so.1.0.0 | Bin 0 -> 494224 bytes external_libs/ibverbs/readme.txt | 30 + 20 files changed, 9541 insertions(+) create mode 100644 external_libs/ibverbs/include/infiniband/arch.h create mode 100644 external_libs/ibverbs/include/infiniband/driver.h create mode 100644 external_libs/ibverbs/include/infiniband/driver_exp.h create mode 100644 external_libs/ibverbs/include/infiniband/kern-abi.h create mode 100644 external_libs/ibverbs/include/infiniband/kern-abi_exp.h create mode 100644 external_libs/ibverbs/include/infiniband/marshall.h create mode 100644 external_libs/ibverbs/include/infiniband/mlx5_hw.h create mode 100644 external_libs/ibverbs/include/infiniband/ofa_verbs.h create mode 100644 external_libs/ibverbs/include/infiniband/opcode.h create mode 100644 external_libs/ibverbs/include/infiniband/peer_ops.h create mode 100644 external_libs/ibverbs/include/infiniband/sa-kern-abi.h create mode 100644 external_libs/ibverbs/include/infiniband/sa.h create mode 100644 external_libs/ibverbs/include/infiniband/verbs.h create mode 100644 external_libs/ibverbs/include/infiniband/verbs_exp.h create mode 100644 external_libs/ibverbs/libibverbs.a create mode 100644 external_libs/ibverbs/libibverbs.la create mode 100644 external_libs/ibverbs/libibverbs.lai create mode 100644 external_libs/ibverbs/libibverbs.so.1 create mode 100644 external_libs/ibverbs/libibverbs.so.1.0.0 create mode 100644 external_libs/ibverbs/readme.txt (limited to 'external_libs') diff --git a/external_libs/ibverbs/include/infiniband/arch.h b/external_libs/ibverbs/include/infiniband/arch.h new file mode 100644 index 00000000..6f419c53 --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/arch.h @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2005 Topspin Communications. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef INFINIBAND_ARCH_H +#define INFINIBAND_ARCH_H + +#include +#include +#include + +#ifdef htonll +#undef htonll +#endif + +#ifdef ntohll +#undef ntohll +#endif + +#if __BYTE_ORDER == __LITTLE_ENDIAN +static inline uint64_t htonll(uint64_t x) { return bswap_64(x); } +static inline uint64_t ntohll(uint64_t x) { return bswap_64(x); } +#elif __BYTE_ORDER == __BIG_ENDIAN +static inline uint64_t htonll(uint64_t x) { return x; } +static inline uint64_t ntohll(uint64_t x) { return x; } +#else +#error __BYTE_ORDER is neither __LITTLE_ENDIAN nor __BIG_ENDIAN +#endif + +/* + * Architecture-specific defines. Currently, an architecture is + * required to implement the following operations: + * + * mb() - memory barrier. No loads or stores may be reordered across + * this macro by either the compiler or the CPU. + * rmb() - read memory barrier. No loads may be reordered across this + * macro by either the compiler or the CPU. + * wmb() - write memory barrier. No stores may be reordered across + * this macro by either the compiler or the CPU. + * wc_wmb() - flush write combine buffers. No write-combined writes + * will be reordered across this macro by either the compiler or + * the CPU. + */ + +#if defined(__i386__) + +#define mb() asm volatile("lock; addl $0,0(%%esp) " ::: "memory") +#define rmb() mb() +#define wmb() asm volatile("" ::: "memory") +#define wc_wmb() mb() +#define nc_wmb() wmb() + +#elif defined(__x86_64__) + +#define mb() asm volatile("" ::: "memory") +#define rmb() mb() +#define wmb() asm volatile("" ::: "memory") +#define wc_wmb() asm volatile("sfence" ::: "memory") +#define nc_wmb() wmb() +#define WC_AUTO_EVICT_SIZE 64 + +#elif defined(__PPC64__) + +#define mb() asm volatile("sync" ::: "memory") +#define rmb() asm volatile("lwsync" ::: "memory") +#define wmb() rmb() +#define wc_wmb() mb() +#define nc_wmb() mb() +#define WC_AUTO_EVICT_SIZE 64 + +#elif defined(__ia64__) + +#define mb() asm volatile("mf" ::: "memory") +#define rmb() mb() +#define wmb() mb() +#define wc_wmb() asm volatile("fwb" ::: "memory") +#define nc_wmb() wmb() + +#elif defined(__PPC__) + +#define mb() asm volatile("sync" ::: "memory") +#define rmb() mb() +#define wmb() mb() +#define wc_wmb() wmb() +#define nc_wmb() wmb() + +#elif defined(__sparc_v9__) + +#define mb() asm volatile("membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad" ::: "memory") +#define rmb() asm volatile("membar #LoadLoad" ::: "memory") +#define wmb() asm volatile("membar #StoreStore" ::: "memory") +#define wc_wmb() wmb() +#define nc_wmb() wmb() + +#elif defined(__sparc__) + +#define mb() asm volatile("" ::: "memory") +#define rmb() mb() +#define wmb() mb() +#define wc_wmb() wmb() +#define nc_wmb() wmb() + +#elif defined(__aarch64__) + +/* Perhaps dmb would be sufficient? Let us be conservative for now. */ +#define mb() asm volatile("dsb sy" ::: "memory") +#define rmb() asm volatile("dsb ld" ::: "memory") +#define wmb() asm volatile("dsb st" ::: "memory") +#define wc_wmb() wmb() +#define nc_wmb() wmb() + +#elif defined(__s390x__) + +#define mb() asm volatile("" ::: "memory") +#define rmb() mb() +#define wmb() mb() +#define wc_wmb() wmb() +#define nc_wmb() wmb() + +#else + +#error No architecture specific memory barrier defines found! + +#endif + +#ifdef WC_AUTO_EVICT_SIZE +static inline int wc_auto_evict_size(void) { return WC_AUTO_EVICT_SIZE; }; +#else +static inline int wc_auto_evict_size(void) { return 0; }; +#endif + +#endif /* INFINIBAND_ARCH_H */ diff --git a/external_libs/ibverbs/include/infiniband/driver.h b/external_libs/ibverbs/include/infiniband/driver.h new file mode 100644 index 00000000..b59bc452 --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/driver.h @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2005 PathScale, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef INFINIBAND_DRIVER_H +#define INFINIBAND_DRIVER_H + +#include +#include + +#ifdef __cplusplus +# define BEGIN_C_DECLS extern "C" { +# define END_C_DECLS } +#else /* !__cplusplus */ +# define BEGIN_C_DECLS +# define END_C_DECLS +#endif /* __cplusplus */ + +/* + * Extension that low-level drivers should add to their .so filename + * (probably via libtool "-release" option). For example a low-level + * driver named "libfoo" should build a plug-in named "libfoo-rdmav2.so". + */ +#define IBV_DEVICE_LIBRARY_EXTENSION rdmav2 + +enum verbs_xrcd_mask { + VERBS_XRCD_HANDLE = 1 << 0, + VERBS_XRCD_RESERVED = 1 << 1 +}; + +struct verbs_xrcd { + struct ibv_xrcd xrcd; + uint32_t comp_mask; + uint32_t handle; +}; + +enum verbs_srq_mask { + VERBS_SRQ_TYPE = 1 << 0, + VERBS_SRQ_XRCD = 1 << 1, + VERBS_SRQ_CQ = 1 << 2, + VERBS_SRQ_NUM = 1 << 3, + VERBS_SRQ_RESERVED = 1 << 4 +}; + +struct verbs_srq { + struct ibv_srq srq; + uint32_t comp_mask; + enum ibv_srq_type srq_type; + struct verbs_xrcd *xrcd; + struct ibv_cq *cq; + uint32_t srq_num; +}; + +enum verbs_qp_mask { + VERBS_QP_XRCD = 1 << 0, + VERBS_QP_RESERVED = 1 << 1 +}; + +struct verbs_mw { + struct ibv_mw mw; + uint32_t handle; + enum ibv_mw_type type; +}; + +struct verbs_qp { + struct ibv_qp qp; + uint32_t comp_mask; + struct verbs_xrcd *xrcd; +}; +typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path, + int abi_version); +typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path, + int abi_version); + +void ibv_register_driver(const char *name, ibv_driver_init_func init_func); +void verbs_register_driver(const char *name, verbs_driver_init_func init_func); +int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd, + size_t cmd_size, struct ibv_get_context_resp *resp, + size_t resp_size); +int ibv_cmd_query_device(struct ibv_context *context, + struct ibv_device_attr *device_attr, + uint64_t *raw_fw_ver, + struct ibv_query_device *cmd, size_t cmd_size); +int ibv_cmd_query_port(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr *port_attr, + struct ibv_query_port *cmd, size_t cmd_size); +int ibv_cmd_query_gid(struct ibv_context *context, uint8_t port_num, + int index, union ibv_gid *gid); +int ibv_cmd_query_pkey(struct ibv_context *context, uint8_t port_num, + int index, uint16_t *pkey); +int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd, + struct ibv_alloc_pd *cmd, size_t cmd_size, + struct ibv_alloc_pd_resp *resp, size_t resp_size); +int ibv_cmd_dealloc_pd(struct ibv_pd *pd); +int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd, + int vxrcd_size, + struct ibv_xrcd_init_attr *attr, + struct ibv_open_xrcd *cmd, size_t cmd_size, + struct ibv_open_xrcd_resp *resp, + size_t resp_size); +int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd); +#define IBV_CMD_REG_MR_HAS_RESP_PARAMS +int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length, + uint64_t hca_va, int access, + struct ibv_mr *mr, struct ibv_reg_mr *cmd, + size_t cmd_size, + struct ibv_reg_mr_resp *resp, size_t resp_size); +int ibv_cmd_dereg_mr(struct ibv_mr *mr); +int ibv_cmd_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type, + struct verbs_mw *mw, struct ibv_alloc_mw *cmd, + size_t cmd_size, + struct ibv_alloc_mw_resp *resp, size_t resp_size); +int ibv_cmd_dealloc_mw(struct verbs_mw *mw, + struct ibv_dealloc_mw *cmd, size_t cmd_size); +int ibv_cmd_create_cq(struct ibv_context *context, int cqe, + struct ibv_comp_channel *channel, + int comp_vector, struct ibv_cq *cq, + struct ibv_create_cq *cmd, size_t cmd_size, + struct ibv_create_cq_resp *resp, size_t resp_size); +int ibv_cmd_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc); +int ibv_cmd_req_notify_cq(struct ibv_cq *cq, int solicited_only); +#define IBV_CMD_RESIZE_CQ_HAS_RESP_PARAMS +int ibv_cmd_resize_cq(struct ibv_cq *cq, int cqe, + struct ibv_resize_cq *cmd, size_t cmd_size, + struct ibv_resize_cq_resp *resp, size_t resp_size); +int ibv_cmd_destroy_cq(struct ibv_cq *cq); + +int ibv_cmd_create_srq(struct ibv_pd *pd, + struct ibv_srq *srq, struct ibv_srq_init_attr *attr, + struct ibv_create_srq *cmd, size_t cmd_size, + struct ibv_create_srq_resp *resp, size_t resp_size); +int ibv_cmd_create_srq_ex(struct ibv_context *context, + struct verbs_srq *srq, int vsrq_sz, + struct ibv_srq_init_attr_ex *attr_ex, + struct ibv_create_xsrq *cmd, size_t cmd_size, + struct ibv_create_srq_resp *resp, size_t resp_size); +int ibv_cmd_modify_srq(struct ibv_srq *srq, + struct ibv_srq_attr *srq_attr, + int srq_attr_mask, + struct ibv_modify_srq *cmd, size_t cmd_size); +int ibv_cmd_query_srq(struct ibv_srq *srq, + struct ibv_srq_attr *srq_attr, + struct ibv_query_srq *cmd, size_t cmd_size); +int ibv_cmd_destroy_srq(struct ibv_srq *srq); + +int ibv_cmd_create_qp(struct ibv_pd *pd, + struct ibv_qp *qp, struct ibv_qp_init_attr *attr, + struct ibv_create_qp *cmd, size_t cmd_size, + struct ibv_create_qp_resp *resp, size_t resp_size); +int ibv_cmd_open_qp(struct ibv_context *context, + struct verbs_qp *qp, int vqp_sz, + struct ibv_qp_open_attr *attr, + struct ibv_open_qp *cmd, size_t cmd_size, + struct ibv_create_qp_resp *resp, size_t resp_size); +int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr, + int attr_mask, + struct ibv_qp_init_attr *qp_init_attr, + struct ibv_query_qp *cmd, size_t cmd_size); +int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask, + struct ibv_modify_qp *cmd, size_t cmd_size); +int ibv_cmd_destroy_qp(struct ibv_qp *qp); +int ibv_cmd_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, + struct ibv_send_wr **bad_wr); +int ibv_cmd_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, + struct ibv_recv_wr **bad_wr); +int ibv_cmd_post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *wr, + struct ibv_recv_wr **bad_wr); +int ibv_cmd_create_ah(struct ibv_pd *pd, struct ibv_ah *ah, + struct ibv_ah_attr *attr); +int ibv_cmd_destroy_ah(struct ibv_ah *ah); +int ibv_cmd_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); +int ibv_cmd_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); + +struct ibv_exp_flow *ibv_exp_cmd_create_flow(struct ibv_qp *qp, + struct ibv_exp_flow_attr *flow_attr); +int ibv_exp_cmd_destroy_flow(struct ibv_exp_flow *flow_id); +struct ibv_flow *ibv_cmd_create_flow(struct ibv_qp *qp, + struct ibv_flow_attr *flow_attr); +int ibv_cmd_destroy_flow(struct ibv_flow *flow_id); + +int ibv_dontfork_range(void *base, size_t size); +int ibv_dofork_range(void *base, size_t size); +void ibv_cmd_query_device_assign(struct ibv_device_attr *device_attr, + uint64_t *raw_fw_ver, + struct ibv_query_device_resp *resp); + +/* + * sysfs helper functions + */ +const char *ibv_get_sysfs_path(void); + +int ibv_read_sysfs_file(const char *dir, const char *file, + char *buf, size_t size); + +static inline int verbs_get_srq_num(struct ibv_srq *srq, uint32_t *srq_num) +{ + struct verbs_srq *vsrq = container_of(srq, struct verbs_srq, srq); + if (vsrq->comp_mask & VERBS_SRQ_NUM) { + *srq_num = vsrq->srq_num; + return 0; + } + return ENOSYS; +} + +#endif /* INFINIBAND_DRIVER_H */ diff --git a/external_libs/ibverbs/include/infiniband/driver_exp.h b/external_libs/ibverbs/include/infiniband/driver_exp.h new file mode 100644 index 00000000..31e4a5a8 --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/driver_exp.h @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2005 PathScale, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef INFINIBAND_DRIVER_EXP_H +#define INFINIBAND_DRIVER_EXP_H + +#include +#include +#include + +int ibv_exp_cmd_query_device(struct ibv_context *context, + struct ibv_exp_device_attr *device_attr, + uint64_t *raw_fw_ver, + struct ibv_exp_query_device *cmd, size_t cmd_size); +int ibv_exp_cmd_create_qp(struct ibv_context *context, + struct verbs_qp *qp, int vqp_sz, + struct ibv_exp_qp_init_attr *attr_exp, + void *cmd_buf, size_t lib_cmd_size, size_t drv_cmd_size, + void *resp_buf, size_t lib_resp_size, size_t drv_resp_size, + int force_exp); +int ibv_exp_cmd_create_dct(struct ibv_context *context, + struct ibv_exp_dct *dct, + struct ibv_exp_dct_init_attr *attr, + struct ibv_exp_create_dct *cmd, + size_t lib_cmd_sz, size_t drv_cmd_sz, + struct ibv_exp_create_dct_resp *resp, + size_t lib_resp_sz, size_t drv_resp_sz); +int ibv_exp_cmd_destroy_dct(struct ibv_context *context, + struct ibv_exp_dct *dct, + struct ibv_exp_destroy_dct *cmd, + size_t lib_cmd_sz, size_t drv_cmd_sz, + struct ibv_exp_destroy_dct_resp *resp, + size_t lib_resp_sz, size_t drv_resp_sz); +int ibv_exp_cmd_query_dct(struct ibv_context *context, + struct ibv_exp_query_dct *cmd, + size_t lib_cmd_sz, size_t drv_cmd_sz, + struct ibv_exp_query_dct_resp *resp, + size_t lib_resp_sz, size_t drv_resp_sz, + struct ibv_exp_dct_attr *attr); +int ibv_exp_cmd_arm_dct(struct ibv_context *context, + struct ibv_exp_arm_attr *attr, + struct ibv_exp_arm_dct *cmd, + size_t lib_cmd_sz, size_t drv_cmd_sz, + struct ibv_exp_arm_dct_resp *resp, + size_t lib_resp_sz, size_t drv_resp_sz); +int ibv_exp_cmd_modify_cq(struct ibv_cq *cq, + struct ibv_exp_cq_attr *attr, + int attr_mask, + struct ibv_exp_modify_cq *cmd, size_t cmd_size); +int ibv_exp_cmd_create_cq(struct ibv_context *context, int cqe, + struct ibv_comp_channel *channel, + int comp_vector, struct ibv_cq *cq, + struct ibv_exp_create_cq *cmd, size_t lib_cmd_sz, size_t drv_cmd_sz, + struct ibv_create_cq_resp *resp, size_t lib_resp_sz, size_t drv_resp_sz, + struct ibv_exp_cq_init_attr *attr); +int ibv_exp_cmd_modify_qp(struct ibv_qp *qp, struct ibv_exp_qp_attr *attr, + uint64_t attr_mask, struct ibv_exp_modify_qp *cmd, + size_t cmd_size); +int ibv_exp_cmd_create_mr(struct ibv_exp_create_mr_in *in, struct ibv_mr *mr, + struct ibv_exp_create_mr *cmd, size_t lib_cmd_sz, size_t drv_cmd_sz, + struct ibv_exp_create_mr_resp *resp, size_t lib_resp_sz, size_t drv_resp_sz); +int ibv_exp_cmd_query_mkey(struct ibv_context *context, + struct ibv_mr *mr, + struct ibv_exp_mkey_attr *mkey_attr, + struct ibv_exp_query_mkey *cmd, size_t lib_cmd_sz, size_t drv_cmd_sz, + struct ibv_exp_query_mkey_resp *resp, size_t lib_resp_sz, size_t drv_resp_sz); +int ibv_cmd_exp_reg_mr(const struct ibv_exp_reg_mr_in *mr_init_attr, + uint64_t hca_va, struct ibv_mr *mr, + struct ibv_exp_reg_mr *cmd, + size_t cmd_size, + struct ibv_exp_reg_mr_resp *resp, + size_t resp_size); +int ibv_cmd_exp_prefetch_mr(struct ibv_mr *mr, + struct ibv_exp_prefetch_attr *attr); +int ibv_exp_cmd_create_wq(struct ibv_context *context, + struct ibv_exp_wq_init_attr *wq_init_attr, + struct ibv_exp_wq *wq, + struct ibv_exp_create_wq *cmd, + size_t cmd_core_size, + size_t cmd_size, + struct ibv_exp_create_wq_resp *resp, + size_t resp_core_size, + size_t resp_size); +int ibv_exp_cmd_destroy_wq(struct ibv_exp_wq *wq); +int ibv_exp_cmd_modify_wq(struct ibv_exp_wq *wq, struct ibv_exp_wq_attr *attr, + struct ib_exp_modify_wq *cmd, size_t cmd_size); +int ibv_exp_cmd_create_rwq_ind_table(struct ibv_context *context, + struct ibv_exp_rwq_ind_table_init_attr *init_attr, + struct ibv_exp_rwq_ind_table *rwq_ind_table, + struct ibv_exp_create_rwq_ind_table *cmd, + size_t cmd_core_size, + size_t cmd_size, + struct ibv_exp_create_rwq_ind_table_resp *resp, + size_t resp_core_size, + size_t resp_size); +int ibv_exp_cmd_destroy_rwq_ind_table(struct ibv_exp_rwq_ind_table *rwq_ind_table); +int ibv_exp_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr, + size_t length, uint64_t hca_va, int access, + struct ibv_pd *pd, struct ibv_exp_rereg_mr_attr *attr, + struct ibv_exp_rereg_mr *cmd, + size_t lib_cmd_sz, size_t drv_cmd_sz, + struct ibv_exp_rereg_mr_resp *resp, + size_t lib_resp_sz, size_t drv_resp_sz); +/* + * ibv_exp_cmd_getenv + * + * @context: context to the device + * @name: the name of the variable to read + * @value: pointer where the value of the variable will be written + * @n: number of bytes pointed to by val + * + * return: 0 success + * < 0 varaible was not found + > 0 variable found but not enuogh space provided. requied space is the value returned. + */ +int ibv_exp_cmd_getenv(struct ibv_context *context, + const char *name, char *value, size_t n); + + +#endif /* INFINIBAND_DRIVER_EXP_H */ diff --git a/external_libs/ibverbs/include/infiniband/kern-abi.h b/external_libs/ibverbs/include/infiniband/kern-abi.h new file mode 100644 index 00000000..a6964bb7 --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/kern-abi.h @@ -0,0 +1,1144 @@ +/* + * Copyright (c) 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. + * Copyright (c) 2005 PathScale, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef KERN_ABI_H +#define KERN_ABI_H + +#include + +/* + * This file must be kept in sync with the kernel's version of + * drivers/infiniband/include/ib_user_verbs.h + */ + +/* + * The minimum and maximum kernel ABI that we can handle. + */ +#define IB_USER_VERBS_MIN_ABI_VERSION 3 +#define IB_USER_VERBS_MAX_ABI_VERSION 6 + +#define IB_USER_VERBS_CMD_THRESHOLD 50 + +enum { + IB_USER_VERBS_CMD_GET_CONTEXT, + IB_USER_VERBS_CMD_QUERY_DEVICE, + IB_USER_VERBS_CMD_QUERY_PORT, + IB_USER_VERBS_CMD_ALLOC_PD, + IB_USER_VERBS_CMD_DEALLOC_PD, + IB_USER_VERBS_CMD_CREATE_AH, + IB_USER_VERBS_CMD_MODIFY_AH, + IB_USER_VERBS_CMD_QUERY_AH, + IB_USER_VERBS_CMD_DESTROY_AH, + IB_USER_VERBS_CMD_REG_MR, + IB_USER_VERBS_CMD_REG_SMR, + IB_USER_VERBS_CMD_REREG_MR, + IB_USER_VERBS_CMD_QUERY_MR, + IB_USER_VERBS_CMD_DEREG_MR, + IB_USER_VERBS_CMD_ALLOC_MW, + IB_USER_VERBS_CMD_BIND_MW, + IB_USER_VERBS_CMD_DEALLOC_MW, + IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL, + IB_USER_VERBS_CMD_CREATE_CQ, + IB_USER_VERBS_CMD_RESIZE_CQ, + IB_USER_VERBS_CMD_DESTROY_CQ, + IB_USER_VERBS_CMD_POLL_CQ, + IB_USER_VERBS_CMD_PEEK_CQ, + IB_USER_VERBS_CMD_REQ_NOTIFY_CQ, + IB_USER_VERBS_CMD_CREATE_QP, + IB_USER_VERBS_CMD_QUERY_QP, + IB_USER_VERBS_CMD_MODIFY_QP, + IB_USER_VERBS_CMD_DESTROY_QP, + IB_USER_VERBS_CMD_POST_SEND, + IB_USER_VERBS_CMD_POST_RECV, + IB_USER_VERBS_CMD_ATTACH_MCAST, + IB_USER_VERBS_CMD_DETACH_MCAST, + IB_USER_VERBS_CMD_CREATE_SRQ, + IB_USER_VERBS_CMD_MODIFY_SRQ, + IB_USER_VERBS_CMD_QUERY_SRQ, + IB_USER_VERBS_CMD_DESTROY_SRQ, + IB_USER_VERBS_CMD_POST_SRQ_RECV, + IB_USER_VERBS_CMD_OPEN_XRCD, + IB_USER_VERBS_CMD_CLOSE_XRCD, + IB_USER_VERBS_CMD_CREATE_XSRQ, + IB_USER_VERBS_CMD_OPEN_QP, +}; + + +#define IB_USER_VERBS_CMD_COMMAND_MASK 0xff +#define IB_USER_VERBS_CMD_FLAGS_MASK 0xff000000u +#define IB_USER_VERBS_CMD_FLAGS_SHIFT 24 + + +#define IB_USER_VERBS_CMD_FLAG_EXTENDED 0x80ul + + +enum { + IB_USER_VERBS_CMD_CREATE_FLOW = (IB_USER_VERBS_CMD_FLAG_EXTENDED << + IB_USER_VERBS_CMD_FLAGS_SHIFT) + + IB_USER_VERBS_CMD_THRESHOLD, + IB_USER_VERBS_CMD_DESTROY_FLOW +}; + +/* + * Make sure that all structs defined in this file remain laid out so + * that they pack the same way on 32-bit and 64-bit architectures (to + * avoid incompatibility between 32-bit userspace and 64-bit kernels). + * Specifically: + * - Do not use pointer types -- pass pointers in __u64 instead. + * - Make sure that any structure larger than 4 bytes is padded to a + * multiple of 8 bytes. Otherwise the structure size will be + * different between 32-bit and 64-bit architectures. + */ + +struct hdr { + __u32 command; + __u16 in_words; + __u16 out_words; +}; + +struct response_hdr { + __u64 response; +}; + +struct ex_hdr { + struct { + __u32 command; + __u16 in_words; + __u16 out_words; + }; + struct { + __u64 response; + }; + struct { + __u16 provider_in_words; + __u16 provider_out_words; + __u32 reserved; + }; +}; + +enum ibv_event_rsc_type { + IBV_EVENT_RSC_CQ, + IBV_EVENT_RSC_QP, + IBV_EVENT_RSC_DCT, + IBV_EVENT_RSC_SRQ, + IBV_EVENT_RSC_DEVICE, +}; + +struct ibv_kern_async_event { + __u64 element; + __u32 event_type; + __u32 rsc_type; +}; + +struct ibv_comp_event { + __u64 cq_handle; +}; + +/* + * All commands from userspace should start with a __u32 command field + * followed by __u16 in_words and out_words fields (which give the + * length of the command block and response buffer if any in 32-bit + * words). The kernel driver will read these fields first and read + * the rest of the command struct based on these value. + */ + +#define IBV_RESP_TO_VERBS_RESP_EX_RAW(ex_ptr, ex_type, ibv_type, field) \ + ((ibv_type *)((void *)(ex_ptr) + offsetof(ex_type, \ + field) + sizeof((ex_ptr)->field))) + +#define IBV_RESP_TO_VERBS_RESP_EX(ex_ptr, ex_type, ibv_type) \ + IBV_RESP_TO_VERBS_RESP_EX_RAW(ex_ptr, ex_type, ibv_type, comp_mask) + +struct ibv_query_params { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; +}; + +struct ibv_query_params_resp { + __u32 num_cq_events; +}; + +struct ibv_get_context { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u64 driver_data[0]; +}; + +struct ibv_get_context_resp { + __u32 async_fd; + __u32 num_comp_vectors; +}; + +struct ibv_query_device { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u64 driver_data[0]; +}; + +struct ibv_query_device_resp { + __u64 fw_ver; + __u64 node_guid; + __u64 sys_image_guid; + __u64 max_mr_size; + __u64 page_size_cap; + __u32 vendor_id; + __u32 vendor_part_id; + __u32 hw_ver; + __u32 max_qp; + __u32 max_qp_wr; + __u32 device_cap_flags; + __u32 max_sge; + __u32 max_sge_rd; + __u32 max_cq; + __u32 max_cqe; + __u32 max_mr; + __u32 max_pd; + __u32 max_qp_rd_atom; + __u32 max_ee_rd_atom; + __u32 max_res_rd_atom; + __u32 max_qp_init_rd_atom; + __u32 max_ee_init_rd_atom; + __u32 atomic_cap; + __u32 max_ee; + __u32 max_rdd; + __u32 max_mw; + __u32 max_raw_ipv6_qp; + __u32 max_raw_ethy_qp; + __u32 max_mcast_grp; + __u32 max_mcast_qp_attach; + __u32 max_total_mcast_qp_attach; + __u32 max_ah; + __u32 max_fmr; + __u32 max_map_per_fmr; + __u32 max_srq; + __u32 max_srq_wr; + __u32 max_srq_sge; + __u16 max_pkeys; + __u8 local_ca_ack_delay; + __u8 phys_port_cnt; + __u8 reserved[4]; +}; + +struct ibv_query_port { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u8 port_num; + __u8 reserved[7]; + __u64 driver_data[0]; +}; + +struct ibv_query_port_resp { + __u32 port_cap_flags; + __u32 max_msg_sz; + __u32 bad_pkey_cntr; + __u32 qkey_viol_cntr; + __u32 gid_tbl_len; + __u16 pkey_tbl_len; + __u16 lid; + __u16 sm_lid; + __u8 state; + __u8 max_mtu; + __u8 active_mtu; + __u8 lmc; + __u8 max_vl_num; + __u8 sm_sl; + __u8 subnet_timeout; + __u8 init_type_reply; + __u8 active_width; + __u8 active_speed; + __u8 phys_state; + __u8 link_layer; + __u8 reserved[2]; +}; + +struct ibv_alloc_pd { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u64 driver_data[0]; +}; + +struct ibv_alloc_pd_resp { + __u32 pd_handle; +}; + +struct ibv_dealloc_pd { + __u32 command; + __u16 in_words; + __u16 out_words; + __u32 pd_handle; +}; + +struct ibv_open_xrcd { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u32 fd; + __u32 oflags; + __u64 driver_data[0]; +}; + +struct ibv_open_xrcd_resp { + __u32 xrcd_handle; +}; + +struct ibv_close_xrcd { + __u32 command; + __u16 in_words; + __u16 out_words; + __u32 xrcd_handle; +}; + +struct ibv_reg_mr { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u64 start; + __u64 length; + __u64 hca_va; + __u32 pd_handle; + __u32 access_flags; + __u64 driver_data[0]; +}; + +struct ibv_reg_mr_resp { + __u32 mr_handle; + __u32 lkey; + __u32 rkey; +}; + +struct ibv_dereg_mr { + __u32 command; + __u16 in_words; + __u16 out_words; + __u32 mr_handle; +}; + +struct ibv_alloc_mw { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u32 pd_handle; + __u8 mw_type; + __u8 reserved[3]; +}; + +struct ibv_alloc_mw_resp { + __u32 mw_handle; + __u32 rkey; +}; + +struct ibv_dealloc_mw { + __u32 command; + __u16 in_words; + __u16 out_words; + __u32 mw_handle; +}; + +struct ibv_create_comp_channel { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; +}; + +struct ibv_create_comp_channel_resp { + __u32 fd; +}; + +struct ibv_create_cq { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u64 user_handle; + __u32 cqe; + __u32 comp_vector; + __s32 comp_channel; + __u32 reserved; + __u64 driver_data[0]; +}; + +struct ibv_create_cq_resp { + __u32 cq_handle; + __u32 cqe; +}; + +struct ibv_kern_wc { + __u64 wr_id; + __u32 status; + __u32 opcode; + __u32 vendor_err; + __u32 byte_len; + __u32 imm_data; + __u32 qp_num; + __u32 src_qp; + __u32 wc_flags; + __u16 pkey_index; + __u16 slid; + __u8 sl; + __u8 dlid_path_bits; + __u8 port_num; + __u8 reserved; +}; + +struct ibv_poll_cq { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u32 cq_handle; + __u32 ne; +}; + +struct ibv_poll_cq_resp { + __u32 count; + __u32 reserved; + struct ibv_kern_wc wc[0]; +}; + +struct ibv_req_notify_cq { + __u32 command; + __u16 in_words; + __u16 out_words; + __u32 cq_handle; + __u32 solicited; +}; + +struct ibv_resize_cq { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u32 cq_handle; + __u32 cqe; + __u64 driver_data[0]; +}; + +struct ibv_resize_cq_resp { + __u32 cqe; + __u32 reserved; + __u64 driver_data[0]; +}; + +struct ibv_destroy_cq { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u32 cq_handle; + __u32 reserved; +}; + +struct ibv_destroy_cq_resp { + __u32 comp_events_reported; + __u32 async_events_reported; +}; + +struct ibv_kern_global_route { + __u8 dgid[16]; + __u32 flow_label; + __u8 sgid_index; + __u8 hop_limit; + __u8 traffic_class; + __u8 reserved; +}; + +struct ibv_kern_ah_attr { + struct ibv_kern_global_route grh; + __u16 dlid; + __u8 sl; + __u8 src_path_bits; + __u8 static_rate; + __u8 is_global; + __u8 port_num; + __u8 reserved; +}; + +struct ibv_kern_qp_attr { + __u32 qp_attr_mask; + __u32 qp_state; + __u32 cur_qp_state; + __u32 path_mtu; + __u32 path_mig_state; + __u32 qkey; + __u32 rq_psn; + __u32 sq_psn; + __u32 dest_qp_num; + __u32 qp_access_flags; + + struct ibv_kern_ah_attr ah_attr; + struct ibv_kern_ah_attr alt_ah_attr; + + /* ib_qp_cap */ + __u32 max_send_wr; + __u32 max_recv_wr; + __u32 max_send_sge; + __u32 max_recv_sge; + __u32 max_inline_data; + + __u16 pkey_index; + __u16 alt_pkey_index; + __u8 en_sqd_async_notify; + __u8 sq_draining; + __u8 max_rd_atomic; + __u8 max_dest_rd_atomic; + __u8 min_rnr_timer; + __u8 port_num; + __u8 timeout; + __u8 retry_cnt; + __u8 rnr_retry; + __u8 alt_port_num; + __u8 alt_timeout; + __u8 reserved[5]; +}; + +struct ibv_create_qp { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u64 user_handle; + __u32 pd_handle; + __u32 send_cq_handle; + __u32 recv_cq_handle; + __u32 srq_handle; + __u32 max_send_wr; + __u32 max_recv_wr; + __u32 max_send_sge; + __u32 max_recv_sge; + __u32 max_inline_data; + __u8 sq_sig_all; + __u8 qp_type; + __u8 is_srq; + __u8 reserved; + __u64 driver_data[0]; +}; + +struct ibv_open_qp { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u64 user_handle; + __u32 pd_handle; + __u32 qpn; + __u8 qp_type; + __u8 reserved[7]; + __u64 driver_data[0]; +}; + +/* also used for open response */ +struct ibv_create_qp_resp { + __u32 qp_handle; + __u32 qpn; + __u32 max_send_wr; + __u32 max_recv_wr; + __u32 max_send_sge; + __u32 max_recv_sge; + __u32 max_inline_data; + __u32 reserved; +}; + +enum ibv_create_qp_ex_comp_mask { + IBV_CREATE_QP_EX_CAP_FLAGS = (1ULL << 0) +}; + +struct ibv_create_qp_ex { + __u32 command; + __u16 in_words; + __u16 out_words; + __u16 provider_in_words; + __u16 provider_out_words; + __u32 cmd_hdr_reserved; + __u64 comp_mask; + __u64 response; + __u64 user_handle; + __u32 pd_handle; + __u32 send_cq_handle; + __u32 recv_cq_handle; + __u32 srq_handle; + __u32 max_send_wr; + __u32 max_recv_wr; + __u32 max_send_sge; + __u32 max_recv_sge; + __u32 max_inline_data; + __u8 sq_sig_all; + __u8 qp_type; + __u8 is_srq; + __u8 reserved; + __u64 qp_cap_flags; + __u64 driver_data[0]; +}; + +struct ibv_create_qp_resp_ex { + __u64 comp_mask; + __u32 qp_handle; + __u32 qpn; + __u32 max_send_wr; + __u32 max_recv_wr; + __u32 max_send_sge; + __u32 max_recv_sge; + __u32 max_inline_data; + __u32 reserved; +}; + +struct ibv_qp_dest { + __u8 dgid[16]; + __u32 flow_label; + __u16 dlid; + __u16 reserved; + __u8 sgid_index; + __u8 hop_limit; + __u8 traffic_class; + __u8 sl; + __u8 src_path_bits; + __u8 static_rate; + __u8 is_global; + __u8 port_num; +}; + +struct ibv_query_qp { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u32 qp_handle; + __u32 attr_mask; + __u64 driver_data[0]; +}; + +struct ibv_query_qp_resp { + struct ibv_qp_dest dest; + struct ibv_qp_dest alt_dest; + __u32 max_send_wr; + __u32 max_recv_wr; + __u32 max_send_sge; + __u32 max_recv_sge; + __u32 max_inline_data; + __u32 qkey; + __u32 rq_psn; + __u32 sq_psn; + __u32 dest_qp_num; + __u32 qp_access_flags; + __u16 pkey_index; + __u16 alt_pkey_index; + __u8 qp_state; + __u8 cur_qp_state; + __u8 path_mtu; + __u8 path_mig_state; + __u8 sq_draining; + __u8 max_rd_atomic; + __u8 max_dest_rd_atomic; + __u8 min_rnr_timer; + __u8 port_num; + __u8 timeout; + __u8 retry_cnt; + __u8 rnr_retry; + __u8 alt_port_num; + __u8 alt_timeout; + __u8 sq_sig_all; + __u8 reserved[5]; + __u64 driver_data[0]; +}; + +struct ibv_modify_qp { + __u32 command; + __u16 in_words; + __u16 out_words; + struct ibv_qp_dest dest; + struct ibv_qp_dest alt_dest; + __u32 qp_handle; + __u32 attr_mask; + __u32 qkey; + __u32 rq_psn; + __u32 sq_psn; + __u32 dest_qp_num; + __u32 qp_access_flags; + __u16 pkey_index; + __u16 alt_pkey_index; + __u8 qp_state; + __u8 cur_qp_state; + __u8 path_mtu; + __u8 path_mig_state; + __u8 en_sqd_async_notify; + __u8 max_rd_atomic; + __u8 max_dest_rd_atomic; + __u8 min_rnr_timer; + __u8 port_num; + __u8 timeout; + __u8 retry_cnt; + __u8 rnr_retry; + __u8 alt_port_num; + __u8 alt_timeout; + __u8 reserved[2]; + __u64 driver_data[0]; +}; + +struct ibv_destroy_qp { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u32 qp_handle; + __u32 reserved; +}; + +struct ibv_destroy_qp_resp { + __u32 events_reported; +}; + +struct ibv_kern_send_wr { + __u64 wr_id; + __u32 num_sge; + __u32 opcode; + __u32 send_flags; + __u32 imm_data; + union { + struct { + __u64 remote_addr; + __u32 rkey; + __u32 reserved; + } rdma; + struct { + __u64 remote_addr; + __u64 compare_add; + __u64 swap; + __u32 rkey; + __u32 reserved; + } atomic; + struct { + __u32 ah; + __u32 remote_qpn; + __u32 remote_qkey; + __u32 reserved; + } ud; + } wr; + union { + struct { + __u32 remote_srqn; + } xrc; + } qp_type; +}; + +struct ibv_kern_eth_filter { + __u8 dst_mac[6]; + __u8 src_mac[6]; + __u16 ether_type; + __u16 vlan_tag; +}; + +struct ibv_kern_spec_eth { + __u32 type; + __u16 size; + __u16 reserved; + struct ibv_kern_eth_filter val; + struct ibv_kern_eth_filter mask; +}; + +struct ibv_kern_ib_filter { + __u32 qpn; + __u8 dst_gid[16]; +}; + +struct ibv_kern_spec_ib { + __u32 type; + __u16 size; + __u16 reserved; + struct ibv_kern_ib_filter val; + struct ibv_kern_ib_filter mask; +}; + +struct ibv_kern_ipv4_filter { + __u32 src_ip; + __u32 dst_ip; +}; + +struct ibv_kern_spec_ipv4 { + __u32 type; + __u16 size; + __u16 reserved; + struct ibv_kern_ipv4_filter val; + struct ibv_kern_ipv4_filter mask; +}; + +struct ibv_kern_tcp_udp_filter { + __u16 dst_port; + __u16 src_port; +}; + +struct ibv_kern_spec_tcp_udp { + __u32 type; + __u16 size; + __u16 reserved; + struct ibv_kern_tcp_udp_filter val; + struct ibv_kern_tcp_udp_filter mask; +}; + + +struct ibv_kern_spec { + union { + struct { + __u32 type; + __u16 size; + __u16 reserved; + } hdr; + struct ibv_kern_spec_ib ib; + struct ibv_kern_spec_eth eth; + struct ibv_kern_spec_ipv4 ipv4; + struct ibv_kern_spec_tcp_udp tcp_udp; + }; + +}; + +struct ibv_kern_flow_attr { + __u32 type; + __u16 size; + __u16 priority; + __u8 num_of_specs; + __u8 reserved[2]; + __u8 port; + __u32 flags; + /* Following are the optional layers according to user request + * struct ibv_kern_flow_spec_xxx + * struct ibv_kern_flow_spec_yyy + */ +}; + +struct ibv_post_send { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u32 qp_handle; + __u32 wr_count; + __u32 sge_count; + __u32 wqe_size; + struct ibv_kern_send_wr send_wr[0]; +}; + +struct ibv_post_send_resp { + __u32 bad_wr; +}; + +struct ibv_kern_recv_wr { + __u64 wr_id; + __u32 num_sge; + __u32 reserved; +}; + +struct ibv_post_recv { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u32 qp_handle; + __u32 wr_count; + __u32 sge_count; + __u32 wqe_size; + struct ibv_kern_recv_wr recv_wr[0]; +}; + +struct ibv_post_recv_resp { + __u32 bad_wr; +}; + +struct ibv_post_srq_recv { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u32 srq_handle; + __u32 wr_count; + __u32 sge_count; + __u32 wqe_size; + struct ibv_kern_recv_wr recv_wr[0]; +}; + +struct ibv_post_srq_recv_resp { + __u32 bad_wr; +}; + +struct ibv_create_ah { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u64 user_handle; + __u32 pd_handle; + __u32 reserved; + struct ibv_kern_ah_attr attr; +}; + +struct ibv_create_ah_resp { + __u32 handle; +}; + +struct ibv_destroy_ah { + __u32 command; + __u16 in_words; + __u16 out_words; + __u32 ah_handle; +}; + +struct ibv_attach_mcast { + __u32 command; + __u16 in_words; + __u16 out_words; + __u8 gid[16]; + __u32 qp_handle; + __u16 mlid; + __u16 reserved; + __u64 driver_data[0]; +}; + +struct ibv_detach_mcast { + __u32 command; + __u16 in_words; + __u16 out_words; + __u8 gid[16]; + __u32 qp_handle; + __u16 mlid; + __u16 reserved; + __u64 driver_data[0]; +}; + +struct ibv_create_flow { + struct ex_hdr hdr; + __u32 comp_mask; + __u32 qp_handle; + struct ibv_kern_flow_attr flow_attr; +}; + +struct ibv_create_flow_resp { + __u32 comp_mask; + __u32 flow_handle; +}; + +struct ibv_destroy_flow { + struct ex_hdr hdr; + __u32 comp_mask; + __u32 flow_handle; +}; + +struct ibv_create_srq { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u64 user_handle; + __u32 pd_handle; + __u32 max_wr; + __u32 max_sge; + __u32 srq_limit; + __u64 driver_data[0]; +}; + +struct ibv_create_xsrq { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u64 user_handle; + __u32 srq_type; + __u32 pd_handle; + __u32 max_wr; + __u32 max_sge; + __u32 srq_limit; + __u32 reserved; + __u32 xrcd_handle; + __u32 cq_handle; + __u64 driver_data[0]; +}; + +struct ibv_create_srq_resp { + __u32 srq_handle; + __u32 max_wr; + __u32 max_sge; + __u32 srqn; +}; + +struct ibv_modify_srq { + __u32 command; + __u16 in_words; + __u16 out_words; + __u32 srq_handle; + __u32 attr_mask; + __u32 max_wr; + __u32 srq_limit; + __u64 driver_data[0]; +}; + +struct ibv_query_srq { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u32 srq_handle; + __u32 reserved; + __u64 driver_data[0]; +}; + +struct ibv_query_srq_resp { + __u32 max_wr; + __u32 max_sge; + __u32 srq_limit; + __u32 reserved; +}; + +struct ibv_destroy_srq { + __u32 command; + __u16 in_words; + __u16 out_words; + __u64 response; + __u32 srq_handle; + __u32 reserved; +}; + +struct ibv_destroy_srq_resp { + __u32 events_reported; +}; + +/* + * Compatibility with older ABI versions + */ + +enum { + IB_USER_VERBS_CMD_QUERY_PARAMS_V2, + IB_USER_VERBS_CMD_GET_CONTEXT_V2, + IB_USER_VERBS_CMD_QUERY_DEVICE_V2, + IB_USER_VERBS_CMD_QUERY_PORT_V2, + IB_USER_VERBS_CMD_QUERY_GID_V2, + IB_USER_VERBS_CMD_QUERY_PKEY_V2, + IB_USER_VERBS_CMD_ALLOC_PD_V2, + IB_USER_VERBS_CMD_DEALLOC_PD_V2, + IB_USER_VERBS_CMD_CREATE_AH_V2, + IB_USER_VERBS_CMD_MODIFY_AH_V2, + IB_USER_VERBS_CMD_QUERY_AH_V2, + IB_USER_VERBS_CMD_DESTROY_AH_V2, + IB_USER_VERBS_CMD_REG_MR_V2, + IB_USER_VERBS_CMD_REG_SMR_V2, + IB_USER_VERBS_CMD_REREG_MR_V2, + IB_USER_VERBS_CMD_QUERY_MR_V2, + IB_USER_VERBS_CMD_DEREG_MR_V2, + IB_USER_VERBS_CMD_ALLOC_MW_V2, + IB_USER_VERBS_CMD_BIND_MW_V2, + IB_USER_VERBS_CMD_DEALLOC_MW_V2, + IB_USER_VERBS_CMD_CREATE_CQ_V2, + IB_USER_VERBS_CMD_RESIZE_CQ_V2, + IB_USER_VERBS_CMD_DESTROY_CQ_V2, + IB_USER_VERBS_CMD_POLL_CQ_V2, + IB_USER_VERBS_CMD_PEEK_CQ_V2, + IB_USER_VERBS_CMD_REQ_NOTIFY_CQ_V2, + IB_USER_VERBS_CMD_CREATE_QP_V2, + IB_USER_VERBS_CMD_QUERY_QP_V2, + IB_USER_VERBS_CMD_MODIFY_QP_V2, + IB_USER_VERBS_CMD_DESTROY_QP_V2, + IB_USER_VERBS_CMD_POST_SEND_V2, + IB_USER_VERBS_CMD_POST_RECV_V2, + IB_USER_VERBS_CMD_ATTACH_MCAST_V2, + IB_USER_VERBS_CMD_DETACH_MCAST_V2, + IB_USER_VERBS_CMD_CREATE_SRQ_V2, + IB_USER_VERBS_CMD_MODIFY_SRQ_V2, + IB_USER_VERBS_CMD_QUERY_SRQ_V2, + IB_USER_VERBS_CMD_DESTROY_SRQ_V2, + IB_USER_VERBS_CMD_POST_SRQ_RECV_V2, + /* + * Set commands that didn't exist to -1 so our compile-time + * trick opcodes in IBV_INIT_CMD() doesn't break. + */ + IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1, + IB_USER_VERBS_CMD_CREATE_QP_EX_V2 = -1, + IB_USER_VERBS_CMD_MODIFY_CQ_EX_V2 = -1, + IB_USER_VERBS_CMD_CREATE_FLOW_V2 = -1, + IB_USER_VERBS_CMD_DESTROY_FLOW_V2 = -1, + IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1, + IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1, + IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1, + IB_USER_VERBS_CMD_OPEN_QP_V2 = -1, + IB_USER_VERBS_CMD_MODIFY_QP_EX_V2 = -1, + IB_USER_VERBS_CMD_CREATE_CQ_EX_V2 = -1, + IB_USER_VERBS_CMD_QUERY_DEVICE_EX_V2 = -1, + IB_USER_VERBS_CMD_CREATE_DCT_V2 = -1, + IB_USER_VERBS_CMD_DESTROY_DCT_V2 = -1, + IB_USER_VERBS_CMD_QUERY_DCT_V2 = -1, + IB_USER_VERBS_CMD_EXP_REG_MR_V2 = -1, + IB_USER_VERBS_CMD_EXP_PREFETCH_MR_V2 = -1, +}; + +struct ibv_modify_srq_v3 { + __u32 command; + __u16 in_words; + __u16 out_words; + __u32 srq_handle; + __u32 attr_mask; + __u32 max_wr; + __u32 max_sge; + __u32 srq_limit; + __u32 reserved; + __u64 driver_data[0]; +}; + +struct ibv_create_qp_resp_v3 { + __u32 qp_handle; + __u32 qpn; +}; + +struct ibv_create_qp_resp_v4 { + __u32 qp_handle; + __u32 qpn; + __u32 max_send_wr; + __u32 max_recv_wr; + __u32 max_send_sge; + __u32 max_recv_sge; + __u32 max_inline_data; +}; + +struct ibv_create_srq_resp_v5 { + __u32 srq_handle; +}; + +#endif /* KERN_ABI_H */ diff --git a/external_libs/ibverbs/include/infiniband/kern-abi_exp.h b/external_libs/ibverbs/include/infiniband/kern-abi_exp.h new file mode 100644 index 00000000..b03f19fb --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/kern-abi_exp.h @@ -0,0 +1,722 @@ +/* + * Copyright (c) 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. + * Copyright (c) 2005 PathScale, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef KERN_ABI_EXP_H +#define KERN_ABI_EXP_H + +#include + +/* + * This file must be kept in sync with the kernel's version of + * drivers/infiniband/include/ib_user_verbs_exp.h + */ + +enum { + IB_USER_VERBS_EXP_CMD_FIRST = 64 +}; + +enum { + IB_USER_VERBS_EXP_CMD_CREATE_QP, + IB_USER_VERBS_EXP_CMD_MODIFY_CQ, + IB_USER_VERBS_EXP_CMD_MODIFY_QP, + IB_USER_VERBS_EXP_CMD_CREATE_CQ, + IB_USER_VERBS_EXP_CMD_QUERY_DEVICE, + IB_USER_VERBS_EXP_CMD_CREATE_DCT, + IB_USER_VERBS_EXP_CMD_DESTROY_DCT, + IB_USER_VERBS_EXP_CMD_QUERY_DCT, + IB_USER_VERBS_EXP_CMD_ARM_DCT, + IB_USER_VERBS_EXP_CMD_CREATE_MR, + IB_USER_VERBS_EXP_CMD_QUERY_MKEY, + IB_USER_VERBS_EXP_CMD_REG_MR, + IB_USER_VERBS_EXP_CMD_PREFETCH_MR, + IB_USER_VERBS_EXP_CMD_REREG_MR, + IB_USER_VERBS_EXP_CMD_CREATE_WQ, + IB_USER_VERBS_EXP_CMD_MODIFY_WQ, + IB_USER_VERBS_EXP_CMD_DESTROY_WQ, + IB_USER_VERBS_EXP_CMD_CREATE_RWQ_IND_TBL, + IB_USER_VERBS_EXP_CMD_DESTROY_RWQ_IND_TBL, + IB_USER_VERBS_EXP_CMD_CREATE_FLOW, +}; + +enum { + IB_USER_VERBS_CMD_EXP_CREATE_WQ = + IB_USER_VERBS_EXP_CMD_CREATE_WQ + + IB_USER_VERBS_EXP_CMD_FIRST, + IB_USER_VERBS_CMD_EXP_MODIFY_WQ = + IB_USER_VERBS_EXP_CMD_MODIFY_WQ + + IB_USER_VERBS_EXP_CMD_FIRST, + IB_USER_VERBS_CMD_EXP_DESTROY_WQ = + IB_USER_VERBS_EXP_CMD_DESTROY_WQ + + IB_USER_VERBS_EXP_CMD_FIRST, + IB_USER_VERBS_CMD_EXP_CREATE_RWQ_IND_TBL = + IB_USER_VERBS_EXP_CMD_CREATE_RWQ_IND_TBL + + IB_USER_VERBS_EXP_CMD_FIRST, + IB_USER_VERBS_CMD_EXP_DESTROY_RWQ_IND_TBL = + IB_USER_VERBS_EXP_CMD_DESTROY_RWQ_IND_TBL + + IB_USER_VERBS_EXP_CMD_FIRST, + /* + * Set commands that didn't exist to -1 so our compile-time + * trick opcodes in IBV_INIT_CMD() doesn't break. + */ + IB_USER_VERBS_CMD_EXP_CREATE_WQ_V2 = -1, + IB_USER_VERBS_CMD_EXP_MODIFY_WQ_V2 = -1, + IB_USER_VERBS_CMD_EXP_DESTROY_WQ_V2 = -1, + IB_USER_VERBS_CMD_EXP_CREATE_RWQ_IND_TBL_V2 = -1, + IB_USER_VERBS_CMD_EXP_DESTROY_RWQ_IND_TBL_V2 = -1, +}; + +enum ibv_exp_create_qp_comp_mask { + IBV_EXP_CREATE_QP_CAP_FLAGS = (1ULL << 0), + IBV_EXP_CREATE_QP_INL_RECV = (1ULL << 1), + IBV_EXP_CREATE_QP_QPG = (1ULL << 2), + IBV_EXP_CREATE_QP_MAX_INL_KLMS = (1ULL << 3) +}; + +struct ibv_create_qpg_init_attrib { + __u32 tss_child_count; + __u32 rss_child_count; +}; + +struct ibv_create_qpg { + __u32 qpg_type; + union { + struct { + __u32 parent_handle; + __u32 reserved; + }; + struct ibv_create_qpg_init_attrib parent_attrib; + }; + __u32 reserved2; +}; + +enum ibv_exp_create_qp_kernel_flags { + IBV_EXP_CREATE_QP_KERNEL_FLAGS = IBV_EXP_QP_CREATE_CROSS_CHANNEL | + IBV_EXP_QP_CREATE_MANAGED_SEND | + IBV_EXP_QP_CREATE_MANAGED_RECV | + IBV_EXP_QP_CREATE_ATOMIC_BE_REPLY | + IBV_EXP_QP_CREATE_RX_END_PADDING | + IBV_EXP_QP_CREATE_SCATTER_FCS +}; + +struct ibv_exp_create_qp { + struct ex_hdr hdr; + __u64 comp_mask; + __u64 user_handle; + __u32 pd_handle; + __u32 send_cq_handle; + __u32 recv_cq_handle; + __u32 srq_handle; + __u32 max_send_wr; + __u32 max_recv_wr; + __u32 max_send_sge; + __u32 max_recv_sge; + __u32 max_inline_data; + __u8 sq_sig_all; + __u8 qp_type; + __u8 is_srq; + __u8 reserved; + __u64 qp_cap_flags; + __u32 max_inl_recv; + __u32 reserved1; + struct ibv_create_qpg qpg; + __u64 max_inl_send_klms; + struct { + __u64 rx_hash_fields_mask; + __u32 rwq_ind_tbl_handle; + __u8 rx_hash_function; + __u8 rx_hash_key_len; + __u8 rx_hash_key[128]; + __u16 reserved; + } rx_hash_info; + __u8 port_num; + __u8 reserved_2[7]; + __u64 driver_data[0]; +}; + +enum ibv_exp_create_qp_resp_comp_mask { + IBV_EXP_CREATE_QP_RESP_INL_RECV = (1ULL << 0), +}; + +struct ibv_exp_create_qp_resp { + __u64 comp_mask; + __u32 qp_handle; + __u32 qpn; + __u32 max_send_wr; + __u32 max_recv_wr; + __u32 max_send_sge; + __u32 max_recv_sge; + __u32 max_inline_data; + __u32 max_inl_recv; +}; + +struct ibv_exp_umr_caps_resp { + __u32 max_klm_list_size; + __u32 max_send_wqe_inline_klms; + __u32 max_umr_recursion_depth; + __u32 max_umr_stride_dimension; +}; + +struct ibv_exp_odp_caps_resp { + __u64 general_odp_caps; + struct { + __u32 rc_odp_caps; + __u32 uc_odp_caps; + __u32 ud_odp_caps; + __u32 dc_odp_caps; + __u32 xrc_odp_caps; + __u32 raw_eth_odp_caps; + } per_transport_caps; +}; + +struct ibv_exp_query_device { + struct ex_hdr hdr; + __u64 comp_mask; + __u64 driver_data[0]; +}; + +struct ibv_exp_rx_hash_caps_resp { + __u32 max_rwq_indirection_tables; + __u32 max_rwq_indirection_table_size; + __u64 supported_packet_fields; + __u32 supported_qps; + __u8 supported_hash_functions; + __u8 reserved[3]; +}; + +struct ibv_exp_mp_rq_caps_resp { + __u32 supported_qps; /* use ibv_exp_supported_qp_types */ + __u32 allowed_shifts; /* use ibv_exp_mp_rq_shifts */ + __u8 min_single_wqe_log_num_of_strides; + __u8 max_single_wqe_log_num_of_strides; + __u8 min_single_stride_log_num_of_bytes; + __u8 max_single_stride_log_num_of_bytes; + __u32 reserved; +}; + +struct ibv_exp_ec_caps_resp { + __u32 max_ec_data_vector_count; + __u32 max_ec_calc_inflight_calcs; +}; + +struct ibv_exp_masked_atomic_caps { + __u32 max_fa_bit_boundary; + __u32 log_max_atomic_inline; + __u64 masked_log_atomic_arg_sizes; + __u64 masked_log_atomic_arg_sizes_network_endianness; +}; + +struct ibv_exp_lso_caps_resp { + __u32 max_tso; + __u32 supported_qpts; +}; + +struct ibv_exp_packet_pacing_caps_resp { + __u32 qp_rate_limit_min; + __u32 qp_rate_limit_max; /* In kbps */ + __u32 supported_qpts; + __u32 reserved; +}; + +struct ibv_exp_query_device_resp { + __u64 comp_mask; + __u64 fw_ver; + __u64 node_guid; + __u64 sys_image_guid; + __u64 max_mr_size; + __u64 page_size_cap; + __u32 vendor_id; + __u32 vendor_part_id; + __u32 hw_ver; + __u32 max_qp; + __u32 max_qp_wr; + __u32 device_cap_flags; + __u32 max_sge; + __u32 max_sge_rd; + __u32 max_cq; + __u32 max_cqe; + __u32 max_mr; + __u32 max_pd; + __u32 max_qp_rd_atom; + __u32 max_ee_rd_atom; + __u32 max_res_rd_atom; + __u32 max_qp_init_rd_atom; + __u32 max_ee_init_rd_atom; + __u32 exp_atomic_cap; + __u32 max_ee; + __u32 max_rdd; + __u32 max_mw; + __u32 max_raw_ipv6_qp; + __u32 max_raw_ethy_qp; + __u32 max_mcast_grp; + __u32 max_mcast_qp_attach; + __u32 max_total_mcast_qp_attach; + __u32 max_ah; + __u32 max_fmr; + __u32 max_map_per_fmr; + __u32 max_srq; + __u32 max_srq_wr; + __u32 max_srq_sge; + __u16 max_pkeys; + __u8 local_ca_ack_delay; + __u8 phys_port_cnt; + __u8 reserved[4]; + __u64 timestamp_mask; + __u64 hca_core_clock; + __u64 device_cap_flags2; + __u32 dc_rd_req; + __u32 dc_rd_res; + __u32 inline_recv_sz; + __u32 max_rss_tbl_sz; + __u64 log_atomic_arg_sizes; + __u32 max_fa_bit_boundary; + __u32 log_max_atomic_inline; + struct ibv_exp_umr_caps_resp umr_caps; + struct ibv_exp_odp_caps_resp odp_caps; + __u32 max_dct; + __u32 max_ctx_res_domain; + struct ibv_exp_rx_hash_caps_resp rx_hash; + __u32 max_wq_type_rq; + __u32 max_device_ctx; + struct ibv_exp_mp_rq_caps_resp mp_rq_caps; + __u16 wq_vlan_offloads_cap; + __u8 reserved1[6]; + struct ibv_exp_ec_caps_resp ec_caps; + struct ibv_exp_masked_atomic_caps masked_atomic_caps; + __u16 rx_pad_end_addr_align; + __u8 reserved2[6]; + struct ibv_exp_lso_caps_resp tso_caps; + struct ibv_exp_packet_pacing_caps_resp packet_pacing_caps; +}; + +struct ibv_exp_create_dct { + struct ex_hdr hdr; + __u64 comp_mask; + __u64 user_handle; + __u32 pd_handle; + __u32 cq_handle; + __u32 srq_handle; + __u32 access_flags; + __u64 dc_key; + __u32 flow_label; + __u8 min_rnr_timer; + __u8 tclass; + __u8 port; + __u8 pkey_index; + __u8 gid_index; + __u8 hop_limit; + __u8 mtu; + __u8 rsvd0; + __u32 create_flags; + __u32 inline_size; + __u32 rsvd1; + __u64 driver_data[0]; +}; + +struct ibv_exp_create_dct_resp { + __u32 dct_handle; + __u32 dct_num; + __u32 inline_size; + __u32 rsvd; +}; + +struct ibv_exp_destroy_dct { + struct ex_hdr hdr; + __u64 comp_mask; + __u32 dct_handle; + __u32 rsvd; + __u64 driver_data[0]; +}; + +struct ibv_exp_destroy_dct_resp { + __u32 events_reported; + __u32 reserved; +}; + +struct ibv_exp_query_dct { + struct ex_hdr hdr; + __u64 comp_mask; + __u32 dct_handle; + __u32 reserved; + __u64 driver_data[0]; +}; + +struct ibv_exp_query_dct_resp { + __u64 dc_key; + __u32 access_flags; + __u32 flow_label; + __u32 key_violations; + __u8 port; + __u8 min_rnr_timer; + __u8 tclass; + __u8 mtu; + __u8 pkey_index; + __u8 gid_index; + __u8 hop_limit; + __u8 state; + __u32 rsvd; + __u64 driver_data[0]; +}; + +struct ibv_exp_arm_dct { + struct ex_hdr hdr; + __u64 comp_mask; + __u32 dct_handle; + __u32 reserved; + __u64 driver_data[0]; +}; + +struct ibv_exp_arm_dct_resp { + __u64 reserved; +}; + +struct ibv_exp_modify_cq { + struct ex_hdr hdr; + __u32 cq_handle; + __u32 attr_mask; + __u16 cq_count; + __u16 cq_period; + __u32 cq_cap_flags; + __u32 comp_mask; + __u32 rsvd; +}; + +struct ibv_exp_modify_qp { + struct ex_hdr hdr; + __u32 comp_mask; + struct ibv_qp_dest dest; + struct ibv_qp_dest alt_dest; + __u32 qp_handle; + __u32 attr_mask; + __u32 qkey; + __u32 rq_psn; + __u32 sq_psn; + __u32 dest_qp_num; + __u32 qp_access_flags; + __u16 pkey_index; + __u16 alt_pkey_index; + __u8 qp_state; + __u8 cur_qp_state; + __u8 path_mtu; + __u8 path_mig_state; + __u8 en_sqd_async_notify; + __u8 max_rd_atomic; + __u8 max_dest_rd_atomic; + __u8 min_rnr_timer; + __u8 port_num; + __u8 timeout; + __u8 retry_cnt; + __u8 rnr_retry; + __u8 alt_port_num; + __u8 alt_timeout; + __u8 reserved[6]; + __u64 dct_key; + __u32 exp_attr_mask; + __u32 flow_entropy; + __u64 driver_data[0]; + __u32 rate_limit; + __u32 reserved1; +}; + +enum ibv_exp_create_cq_comp_mask { + IBV_EXP_CREATE_CQ_CAP_FLAGS = (uint64_t)1 << 0, +}; + +struct ibv_exp_create_cq { + struct ex_hdr hdr; + __u64 comp_mask; + __u64 user_handle; + __u32 cqe; + __u32 comp_vector; + __s32 comp_channel; + __u32 reserved; + __u64 create_flags; + __u64 driver_data[0]; +}; + +struct ibv_exp_create_mr { + struct ex_hdr hdr; + __u64 comp_mask; + __u32 pd_handle; + __u32 max_klm_list_size; + __u64 exp_access_flags; + __u32 create_flags; + __u32 reserved; + __u64 driver_data[0]; +}; + +struct ibv_exp_create_mr_resp { + __u64 comp_mask; + __u32 handle; + __u32 lkey; + __u32 rkey; + __u32 reserved; + __u64 driver_data[0]; +}; + +struct ibv_exp_query_mkey { + struct ex_hdr hdr; + __u64 comp_mask; + __u32 handle; + __u32 lkey; + __u32 rkey; + __u32 reserved; + __u64 driver_data[0]; +}; + +struct ibv_exp_query_mkey_resp { + __u64 comp_mask; + __u32 max_klm_list_size; + __u32 reserved; + __u64 driver_data[0]; +}; + +enum ibv_exp_reg_mr_comp_mask { + IBV_EXP_REG_MR_EXP_ACCESS_FLAGS = 1ULL << 0, +}; + +struct ibv_exp_reg_mr { + struct ex_hdr hdr; + __u64 start; + __u64 length; + __u64 hca_va; + __u32 pd_handle; + __u32 reserved; + __u64 exp_access_flags; + __u64 comp_mask; +}; + +struct ibv_exp_prefetch_mr { + struct ex_hdr hdr; + __u64 comp_mask; + __u32 mr_handle; + __u32 flags; + __u64 start; + __u64 length; +}; + +struct ibv_exp_reg_mr_resp { + __u32 mr_handle; + __u32 lkey; + __u32 rkey; + __u32 reserved; + __u64 comp_mask; +}; + +struct ibv_exp_rereg_mr { + struct ex_hdr hdr; + __u32 comp_mask; + __u32 mr_handle; + __u32 flags; + __u32 reserved; + __u64 start; + __u64 length; + __u64 hca_va; + __u32 pd_handle; + __u32 access_flags; +}; + +struct ibv_exp_rereg_mr_resp { + __u32 comp_mask; + __u32 lkey; + __u32 rkey; + __u32 reserved; +}; + +struct ibv_exp_cmd_wq_mp_rq { + __u32 use_shift; /* use ibv_exp_mp_rq_shifts */ + __u8 single_wqe_log_num_of_strides; + __u8 single_stride_log_num_of_bytes; + __u16 reserved; +}; + +enum ibv_exp_cmd_create_wq_comp_mask { + IBV_EXP_CMD_CREATE_WQ_MP_RQ = 1 << 0, + IBV_EXP_CMD_CREATE_WQ_VLAN_OFFLOADS = 1 << 1, + IBV_EXP_CMD_CREATE_WQ_FLAGS = 1 << 2, +}; + +struct ibv_exp_create_wq { + struct ex_hdr hdr; + __u32 comp_mask; /* enum ibv_exp_cmd_create_wq_comp_mask */ + __u32 wq_type; /* enum ibv_exp_wq_type */ + __u64 user_handle; + __u32 pd_handle; + __u32 cq_handle; + __u32 srq_handle; + __u32 max_recv_wr; + __u32 max_recv_sge; + __u32 reserved; + struct ibv_exp_cmd_wq_mp_rq mp_rq; + __u16 wq_vlan_offloads; + __u8 reserved1[6]; + __u64 flags; +}; + +struct ibv_exp_create_wq_resp { + __u32 comp_mask; + __u32 response_length; + __u32 wq_handle; + __u32 max_recv_wr; + __u32 max_recv_sge; + __u32 wqn; +}; + +struct ib_exp_destroy_wq { + struct ex_hdr hdr; + __u32 comp_mask; + __u32 wq_handle; +}; + +struct ib_exp_modify_wq { + struct ex_hdr hdr; + __u32 comp_mask; + __u32 wq_handle; + __u32 wq_state; + __u32 curr_wq_state; + __u16 wq_vlan_offloads; + __u8 reserved[6]; +}; + +struct ibv_exp_create_rwq_ind_table { + struct ex_hdr hdr; + __u32 comp_mask; + __u32 pd_handle; + __u32 log_ind_tbl_size; + __u32 reserved; + /* Following are wq handles based on log_ind_tbl_size, must be 64 bytes aligned. + * __u32 wq_handle1 + * __u32 wq_handle2 + */ +}; + +struct ibv_exp_create_rwq_ind_table_resp { + __u32 comp_mask; + __u32 response_length; + __u32 ind_tbl_handle; + __u32 ind_tbl_num; +}; + +struct ibv_exp_destroy_rwq_ind_table { + struct ex_hdr hdr; + __u32 comp_mask; + __u32 ind_tbl_handle; +}; + +struct ibv_exp_kern_ipv6_filter { + __u8 src_ip[16]; + __u8 dst_ip[16]; +}; + +struct ibv_exp_kern_spec_ipv6 { + __u32 type; + __u16 size; + __u16 reserved; + struct ibv_exp_kern_ipv6_filter val; + struct ibv_exp_kern_ipv6_filter mask; +}; + +struct ibv_exp_kern_ipv6_ext_filter { + __u8 src_ip[16]; + __u8 dst_ip[16]; + __u32 flow_label; + __u8 next_hdr; + __u8 traffic_class; + __u8 hop_limit; + __u8 reserved; +}; + +struct ibv_exp_kern_spec_ipv6_ext { + __u32 type; + __u16 size; + __u16 reserved; + struct ibv_exp_kern_ipv6_ext_filter val; + struct ibv_exp_kern_ipv6_ext_filter mask; +}; + +struct ibv_exp_kern_ipv4_ext_filter { + __u32 src_ip; + __u32 dst_ip; + __u8 proto; + __u8 tos; + __u8 ttl; + __u8 flags; +}; + +struct ibv_exp_kern_spec_ipv4_ext { + __u32 type; + __u16 size; + __u16 reserved; + struct ibv_exp_kern_ipv4_ext_filter val; + struct ibv_exp_kern_ipv4_ext_filter mask; +}; + +struct ibv_exp_kern_tunnel_filter { + __u32 tunnel_id; +}; + +struct ibv_exp_kern_spec_tunnel { + __u32 type; + __u16 size; + __u16 reserved; + struct ibv_exp_kern_tunnel_filter val; + struct ibv_exp_kern_tunnel_filter mask; +}; + +struct ibv_exp_kern_spec_action_tag { + __u32 type; + __u16 size; + __u16 reserved; + __u32 tag_id; + __u32 reserved1; +}; + +struct ibv_exp_kern_spec { + union { + struct { + __u32 type; + __u16 size; + __u16 reserved; + } hdr; + struct ibv_kern_spec_ib ib; + struct ibv_kern_spec_eth eth; + struct ibv_kern_spec_ipv4 ipv4; + struct ibv_exp_kern_spec_ipv4_ext ipv4_ext; + struct ibv_kern_spec_tcp_udp tcp_udp; + struct ibv_exp_kern_spec_ipv6 ipv6; + struct ibv_exp_kern_spec_ipv6_ext ipv6_ext; + struct ibv_exp_kern_spec_tunnel tunnel; + struct ibv_exp_kern_spec_action_tag flow_tag; + }; +}; +#endif /* KERN_ABI_EXP_H */ diff --git a/external_libs/ibverbs/include/infiniband/marshall.h b/external_libs/ibverbs/include/infiniband/marshall.h new file mode 100644 index 00000000..8be76c54 --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/marshall.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2005 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef INFINIBAND_MARSHALL_H +#define INFINIBAND_MARSHALL_H + +#include +#include +#include +#include + +#ifdef __cplusplus +# define BEGIN_C_DECLS extern "C" { +# define END_C_DECLS } +#else /* !__cplusplus */ +# define BEGIN_C_DECLS +# define END_C_DECLS +#endif /* __cplusplus */ + +BEGIN_C_DECLS + +void ibv_copy_qp_attr_from_kern(struct ibv_qp_attr *dst, + struct ibv_kern_qp_attr *src); + +void ibv_copy_ah_attr_from_kern(struct ibv_ah_attr *dst, + struct ibv_kern_ah_attr *src); + +void ibv_copy_path_rec_from_kern(struct ibv_sa_path_rec *dst, + struct ibv_kern_path_rec *src); + +void ibv_copy_path_rec_to_kern(struct ibv_kern_path_rec *dst, + struct ibv_sa_path_rec *src); + +END_C_DECLS + +#endif /* INFINIBAND_MARSHALL_H */ diff --git a/external_libs/ibverbs/include/infiniband/mlx5_hw.h b/external_libs/ibverbs/include/infiniband/mlx5_hw.h new file mode 100644 index 00000000..c772f339 --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/mlx5_hw.h @@ -0,0 +1,797 @@ +/** + * Copyright (C) Mellanox Technologies Ltd. 2001-2014. ALL RIGHTS RESERVED. + * This software product is a proprietary product of Mellanox Technologies Ltd. + * (the "Company") and all right, title, and interest and to the software product, + * including all associated intellectual property rights, are and shall + * remain exclusively with the Company. + * + * This software product is governed by the End User License Agreement + * provided with the software product. + */ + +#ifndef MLX_HW_H_ +#define MLX_HW_H_ + +#include +#include +#include +#include +#include + +#define MLX5_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#if MLX5_GCC_VERSION >= 403 +# define __MLX5_ALGN_F__ __attribute__((noinline, aligned(64))) +# define __MLX5_ALGN_D__ __attribute__((aligned(64))) +#else +# define __MLX5_ALGN_F__ +# define __MLX5_ALGN_D__ +#endif + +#define MLX5_CQ_DB_REQ_NOT_SOL (1 << 24) +#define MLX5_CQ_DB_REQ_NOT (0 << 24) +#define MLX5E_CQE_FORMAT_MASK 0xc + + +enum mlx5_alloc_type { MXM_MLX5_ALLOC_TYPE_DUMMY }; +enum mlx5_rsc_type { MXM_MLX5_RSC_TYPE_DUMMY }; +enum mlx5_db_method { MXM_MLX5_DB_TYPE_DUMMY }; +enum mlx5_lock_type { MXM_MLX5_LOCK_TYPE_DUMMY }; +enum mlx5_lock_state { MXM_MLX5_LOCK_STATE_TYPE_DUMMY }; +enum { + MLX5_RCV_DBR = 0, + MLX5_SND_DBR = 1, + MLX5_SEND_WQE_BB = 64, + MLX5_SEND_WQE_SHIFT = 6, + MLX5_INLINE_SCATTER_32 = 0x4, + MLX5_INLINE_SCATTER_64 = 0x8, + MLX5_OPCODE_NOP = 0x00, + MLX5_OPCODE_SEND_INVAL = 0x01, + MLX5_OPCODE_RDMA_WRITE = 0x08, + MLX5_OPCODE_RDMA_WRITE_IMM = 0x09, + MLX5_OPCODE_SEND = 0x0a, + MLX5_OPCODE_SEND_IMM = 0x0b, + MLX5_OPCODE_TSO = 0x0e, + MLX5_OPC_MOD_MPW = 0x01, + MLX5_OPCODE_LSO_MPW = 0x0e, + MLX5_OPCODE_RDMA_READ = 0x10, + MLX5_OPCODE_ATOMIC_CS = 0x11, + MLX5_OPCODE_ATOMIC_FA = 0x12, + MLX5_OPCODE_ATOMIC_MASKED_CS = 0x14, + MLX5_OPCODE_ATOMIC_MASKED_FA = 0x15, + MLX5_OPCODE_BIND_MW = 0x18, + MLX5_OPCODE_FMR = 0x19, + MLX5_OPCODE_LOCAL_INVAL = 0x1b, + MLX5_OPCODE_CONFIG_CMD = 0x1f, + MLX5_OPCODE_SEND_ENABLE = 0x17, + MLX5_OPCODE_RECV_ENABLE = 0x16, + MLX5_OPCODE_CQE_WAIT = 0x0f, + MLX5_RECV_OPCODE_RDMA_WRITE_IMM = 0x00, + MLX5_RECV_OPCODE_SEND = 0x01, + MLX5_RECV_OPCODE_SEND_IMM = 0x02, + MLX5_RECV_OPCODE_SEND_INVAL = 0x03, + MLX5_CQE_OPCODE_ERROR = 0x1e, + MLX5_CQE_OPCODE_RESIZE = 0x16, + MLX5_SRQ_FLAG_SIGNATURE = 1 << 0, + MLX5_INLINE_SEG = 0x80000000, + MLX5_CALC_UINT64_ADD = 0x01, + MLX5_CALC_FLOAT64_ADD = 0x02, + MLX5_CALC_UINT64_MAXLOC = 0x03, + MLX5_CALC_UINT64_AND = 0x04, + MLX5_CALC_UINT64_OR = 0x05, + MLX5_CALC_UINT64_XOR = 0x06, + MLX5_CQ_DOORBELL = 0x20, + MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR = 0x01, + MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR = 0x02, + MLX5_CQE_SYNDROME_LOCAL_PROT_ERR = 0x04, + MLX5_CQE_SYNDROME_WR_FLUSH_ERR = 0x05, + MLX5_CQE_SYNDROME_MW_BIND_ERR = 0x06, + MLX5_CQE_SYNDROME_BAD_RESP_ERR = 0x10, + MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR = 0x11, + MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12, + MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR = 0x13, + MLX5_CQE_SYNDROME_REMOTE_OP_ERR = 0x14, + MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR = 0x15, + MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR = 0x16, + MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR = 0x22, + MLX5_CQE_OWNER_MASK = 1, + MLX5_CQE_REQ = 0, + MLX5_CQE_RESP_WR_IMM = 1, + MLX5_CQE_RESP_SEND = 2, + MLX5_CQE_RESP_SEND_IMM = 3, + MLX5_CQE_RESP_SEND_INV = 4, + MLX5_CQE_RESIZE_CQ = 5, + MLX5_CQE_SIG_ERR = 12, + MLX5_CQE_REQ_ERR = 13, + MLX5_CQE_RESP_ERR = 14, + MLX5_CQE_INVALID = 15, + MLX5_WQE_CTRL_CQ_UPDATE = 2 << 2, + MLX5_WQE_CTRL_SOLICITED = 1 << 1, + MLX5_WQE_CTRL_FENCE = 4 << 5, + MLX5_INVALID_LKEY = 0x100, + MLX5_EXTENDED_UD_AV = 0x80000000, + MLX5_NO_INLINE_DATA = 0x0, + MLX5_INLINE_DATA32_SEG = 0x1, + MLX5_INLINE_DATA64_SEG = 0x2, + MLX5_COMPRESSED = 0x3, + MLX5_MINI_ARR_SIZE = 8, + MLX5_ETH_WQE_L3_CSUM = (1 << 6), + MLX5_ETH_WQE_L4_CSUM = (1 << 7), + MLX5_ETH_INLINE_HEADER_SIZE = 18, + MLX5_ETH_VLAN_INLINE_HEADER_SIZE = 18, + MLX5_CQE_L2_OK = 1 << 0, + MLX5_CQE_L3_OK = 1 << 1, + MLX5_CQE_L4_OK = 1 << 2, + MLX5_CQE_L3_HDR_TYPE_NONE = 0x0, + MLX5_CQE_L3_HDR_TYPE_IPV6 = 0x4, + MLX5_CQE_L3_HDR_TYPE_IPV4 = 0x8, + MLX5_CQE_L4_HDR_TYPE_TCP = 0x10, + MLX5_CQE_L4_HDR_TYPE_UDP = 0x20, + MLX5_CQE_L4_HDR_TYPE_TCP_EMP_ACK = 0x30, + MLX5_CQE_L4_HDR_TYPE_TCP_ACK = 0x40, + MLX5_CQE_L3_HDR_TYPE_MASK = 0xC, + MLX5_CQE_L4_HDR_TYPE_MASK = 0x70, + MLX5_QP_PEER_VA_ID_MAX = 2, +}; + +struct mlx5_qp; + +struct mlx5_resource { + enum mlx5_rsc_type type; + uint32_t rsn; +}; + + +struct mlx5_wqe_srq_next_seg { + uint8_t rsvd0[2]; + uint16_t next_wqe_index; + uint8_t signature; + uint8_t rsvd1[11]; +}; + + +struct mlx5_wqe_data_seg { + uint32_t byte_count; + uint32_t lkey; + uint64_t addr; +}; + + +struct mlx5_eqe_comp { + uint32_t reserved[6]; + uint32_t cqn; +}; + + +struct mlx5_eqe_qp_srq { + uint32_t reserved[6]; + uint32_t qp_srq_n; +}; + + +struct mlx5_wqe_ctrl_seg { + uint32_t opmod_idx_opcode; + uint32_t qpn_ds; + uint8_t signature; + uint8_t rsvd[2]; + uint8_t fm_ce_se; + uint32_t imm; +}; + + +struct mlx5_wqe_xrc_seg { + uint32_t xrc_srqn; + uint8_t rsvd[12]; +}; + + +struct mlx5_wqe_masked_atomic_seg { + uint64_t swap_add; + uint64_t compare; + uint64_t swap_add_mask; + uint64_t compare_mask; +}; + + +struct mlx5_base_av { + union { + struct { + uint32_t qkey; + uint32_t reserved; + } qkey; + uint64_t dc_key; + } key; + uint32_t dqp_dct; + uint8_t stat_rate_sl; + uint8_t fl_mlid; + uint16_t rlid; +}; + + +struct mlx5_grh_av { + uint8_t reserved0[4]; + uint8_t rmac[6]; + uint8_t tclass; + uint8_t hop_limit; + uint32_t grh_gid_fl; + uint8_t rgid[16]; +}; + + +struct mlx5_wqe_av { + struct mlx5_base_av base; + struct mlx5_grh_av grh_sec; +}; + + +struct mlx5_wqe_datagram_seg { + struct mlx5_wqe_av av; +}; + + +struct mlx5_wqe_raddr_seg { + uint64_t raddr; + uint32_t rkey; + uint32_t reserved; +}; + + +struct mlx5_wqe_atomic_seg { + uint64_t swap_add; + uint64_t compare; +}; + + +struct mlx5_wqe_inl_data_seg { + uint32_t byte_count; +}; + + +struct mlx5_wqe_umr_ctrl_seg { + uint8_t flags; + uint8_t rsvd0[3]; + uint16_t klm_octowords; + uint16_t bsf_octowords; + uint64_t mkey_mask; + uint8_t rsvd1[32]; +}; + + +struct mlx5_seg_set_psv { + uint8_t rsvd[4]; + uint16_t syndrome; + uint16_t status; + uint16_t block_guard; + uint16_t app_tag; + uint32_t ref_tag; + uint32_t mkey; + uint64_t va; +}; + + +struct mlx5_seg_get_psv { + uint8_t rsvd[19]; + uint8_t num_psv; + uint32_t l_key; + uint64_t va; + uint32_t psv_index[4]; +}; + + +struct mlx5_seg_check_psv { + uint8_t rsvd0[2]; + uint16_t err_coalescing_op; + uint8_t rsvd1[2]; + uint16_t xport_err_op; + uint8_t rsvd2[2]; + uint16_t xport_err_mask; + uint8_t rsvd3[7]; + uint8_t num_psv; + uint32_t l_key; + uint64_t va; + uint32_t psv_index[4]; +}; + + +struct mlx5_rwqe_sig { + uint8_t rsvd0[4]; + uint8_t signature; + uint8_t rsvd1[11]; +}; + + +struct mlx5_wqe_signature_seg { + uint8_t rsvd0[4]; + uint8_t signature; + uint8_t rsvd1[11]; +}; + + +struct mlx5_wqe_inline_seg { + uint32_t byte_count; +}; + + +struct mlx5_wqe_wait_en_seg { + uint8_t rsvd0[8]; + uint32_t pi; + uint32_t obj_num; +}; + + +struct mlx5_err_cqe { + uint8_t rsvd0[32]; + uint32_t srqn; + uint8_t rsvd1[16]; + uint8_t hw_err_synd; + uint8_t hw_synd_type; + uint8_t vendor_err_synd; + uint8_t syndrome; + uint32_t s_wqe_opcode_qpn; + uint16_t wqe_counter; + uint8_t signature; + uint8_t op_own; +}; + + +struct mlx5_cqe64 { + uint8_t rsvd0[2]; + /* + * wqe_id is valid only for Striding RQ (Multi-Packet RQ). + * It provides the WQE index inside the RQ. + */ + uint16_t wqe_id; + uint8_t rsvd4[8]; + uint32_t rx_hash_res; + uint8_t rx_hash_type; + uint8_t ml_path; + uint8_t rsvd20[2]; + uint16_t checksum; + uint16_t slid; + uint32_t flags_rqpn; + uint8_t hds_ip_ext; + uint8_t l4_hdr_type_etc; + __be16 vlan_info; + uint32_t srqn_uidx; + uint32_t imm_inval_pkey; + uint8_t rsvd40[4]; + uint32_t byte_cnt; + __be64 timestamp; + union { + uint32_t sop_drop_qpn; + struct { + uint8_t sop; + uint8_t qpn[3]; + } sop_qpn; + }; + /* + * In Striding RQ (Multi-Packet RQ) wqe_counter provides + * the WQE stride index (to calc pointer to start of the message) + */ + uint16_t wqe_counter; + uint8_t signature; + uint8_t op_own; +}; + + +struct mlx5_spinlock { + pthread_spinlock_t lock; + enum mlx5_lock_state state; +}; + + +struct mlx5_lock { + pthread_mutex_t mutex; + pthread_spinlock_t slock; + enum mlx5_lock_state state; + enum mlx5_lock_type type; +}; + + +struct mlx5_numa_req { + int valid; + int numa_id; +}; + + +struct mlx5_peer_direct_mem { + uint32_t dir; + uint64_t va_id; + struct ibv_exp_peer_buf *pb; + struct ibv_exp_peer_direct_attr *ctx; +}; + + +struct mlx5_buf { + void *buf; + size_t length; + int base; + struct mlx5_hugetlb_mem *hmem; + struct mlx5_peer_direct_mem peer; + enum mlx5_alloc_type type; + struct mlx5_numa_req numa_req; + int numa_alloc; +}; + + +struct general_data_hot { + /* post_send hot data */ + unsigned *wqe_head; + int (*post_send_one)(struct ibv_exp_send_wr *wr, + struct mlx5_qp *qp, + uint64_t exp_send_flags, + void *seg, int *total_size); + void *sqstart; + void *sqend; + volatile uint32_t *db; + struct mlx5_bf *bf; + uint32_t scur_post; + /* Used for burst_family interface, keeps the last posted wqe */ + uint32_t last_post; + uint16_t create_flags; + uint8_t fm_cache; + uint8_t model_flags; /* use mlx5_qp_model_flags */ +}; + + +struct data_seg_data { + uint32_t max_inline_data; +}; + + +struct ctrl_seg_data { + uint32_t qp_num; + uint8_t fm_ce_se_tbl[8]; + uint8_t fm_ce_se_acc[32]; + uint8_t wq_sig; +}; + + +struct mpw_data { + uint8_t state; /* use mpw_states */ + uint8_t size; + uint8_t num_sge; + uint32_t len; + uint32_t total_len; + uint32_t flags; + uint32_t scur_post; + union { + struct mlx5_wqe_data_seg *last_dseg; + uint8_t *inl_data; + }; + uint32_t *ctrl_update; +}; + + +struct general_data_warm { + uint32_t pattern; + uint8_t qp_type; +}; + + +struct odp_data { + struct mlx5_pd *pd; +}; + + +struct mlx5_wq_recv_send_enable { + unsigned head_en_index; + unsigned head_en_count; +}; + + +struct mlx5_mini_cqe8 { + union { + uint32_t rx_hash_result; + uint32_t checksum; + struct { + uint16_t wqe_counter; + uint8_t s_wqe_opcode; + uint8_t reserved; + } s_wqe_info; + }; + uint32_t byte_cnt; +}; + + +struct mlx5_cq { + struct ibv_cq ibv_cq; + uint32_t creation_flags; + uint32_t pattern; + struct mlx5_buf buf_a; + struct mlx5_buf buf_b; + struct mlx5_buf *active_buf; + struct mlx5_buf *resize_buf; + int resize_cqes; + int active_cqes; + struct mlx5_lock lock; + uint32_t cqn; + uint32_t cons_index; + uint32_t wait_index; + uint32_t wait_count; + volatile uint32_t *dbrec; + int arm_sn; + int cqe_sz; + int resize_cqe_sz; + int stall_next_poll; + int stall_enable; + uint64_t stall_last_count; + int stall_adaptive_enable; + int stall_cycles; + uint8_t model_flags; /* use mlx5_cq_model_flags */ + uint16_t cqe_comp_max_num; + uint8_t cq_log_size; + /* Compressed CQE data */ + struct mlx5_cqe64 next_decomp_cqe64; + struct mlx5_resource *compressed_rsc; + uint16_t compressed_left; + uint16_t compressed_wqe_cnt; + uint8_t compressed_req; + uint8_t compressed_mp_rq; + uint8_t mini_arr_idx; + struct mlx5_mini_cqe8 mini_array[MLX5_MINI_ARR_SIZE]; + /* peer-direct data */ + int peer_enabled; + struct ibv_exp_peer_direct_attr *peer_ctx; + struct mlx5_buf peer_buf; + struct mlx5_peek_entry **peer_peek_table; + struct mlx5_peek_entry *peer_peek_free; +}; + + +struct mlx5_srq { + struct mlx5_resource rsc; /* This struct must be first */ + struct verbs_srq vsrq; + struct mlx5_buf buf; + struct mlx5_spinlock lock; + uint64_t *wrid; + uint32_t srqn; + int max; + int max_gs; + int wqe_shift; + int head; + int tail; + volatile uint32_t *db; + uint16_t counter; + int wq_sig; + struct ibv_srq_legacy *ibv_srq_legacy; + int is_xsrq; +}; + + +struct mlx5_wq { + /* common hot data */ + uint64_t *wrid; + unsigned wqe_cnt; + unsigned head; + unsigned tail; + unsigned max_post; + int max_gs; + struct mlx5_lock lock; + /* post_recv hot data */ + void *buff; + volatile uint32_t *db; + int wqe_shift; + int offset; +}; + + +struct mlx5_bf { + void *reg; + int need_lock; + /* + * Protect usage of BF address field including data written to the BF + * and the BF buffer toggling. + */ + struct mlx5_lock lock; + unsigned offset; + unsigned buf_size; + unsigned uuarn; + enum mlx5_db_method db_method; +}; + + +struct mlx5_qp { + struct mlx5_resource rsc; + struct verbs_qp verbs_qp; + struct mlx5_buf buf; + int buf_size; + /* For Raw Ethernet QP, use different Buffer for the SQ and RQ */ + struct mlx5_buf sq_buf; + int sq_buf_size; + uint8_t sq_signal_bits; + int umr_en; + + /* hot data used on data path */ + struct mlx5_wq rq __MLX5_ALGN_D__; + struct mlx5_wq sq __MLX5_ALGN_D__; + + struct general_data_hot gen_data; + struct mpw_data mpw; + struct data_seg_data data_seg; + struct ctrl_seg_data ctrl_seg; + + /* RAW_PACKET hot data */ + uint8_t link_layer; + + /* used on data-path but not so hot */ + struct general_data_warm gen_data_warm; + /* atomic hot data */ + int enable_atomics; + /* odp hot data */ + struct odp_data odp_data; + /* ext atomic hot data */ + uint32_t max_atomic_arg; + /* umr hot data */ + uint32_t max_inl_send_klms; + /* recv-send enable hot data */ + struct mlx5_wq_recv_send_enable rq_enable; + struct mlx5_wq_recv_send_enable sq_enable; + int rx_qp; + /* peer-direct data */ + int peer_enabled; + struct ibv_exp_peer_direct_attr *peer_ctx; + void *peer_ctrl_seg; + uint32_t peer_scur_post; + uint64_t peer_va_ids[MLX5_QP_PEER_VA_ID_MAX]; + struct ibv_exp_peer_buf *peer_db_buf; + uint32_t max_tso_header; +}; + + +struct mlx5_ah { + struct ibv_ah ibv_ah; + struct mlx5_wqe_av av; +}; + + +struct mlx5_rwq { + struct mlx5_resource rsc; + uint32_t pattern; + struct ibv_exp_wq wq; + struct mlx5_buf buf; + int buf_size; + /* hot data used on data path */ + struct mlx5_wq rq __MLX5_ALGN_D__; + volatile uint32_t *db; + /* Multi-Packet RQ hot data */ + /* Table to hold the consumed strides on each WQE */ + uint32_t *consumed_strides_counter; + uint16_t mp_rq_stride_size; + uint32_t mp_rq_strides_in_wqe; + uint8_t mp_rq_packet_padding; + /* recv-send enable hot data */ + struct mlx5_wq_recv_send_enable rq_enable; + int wq_sig; + uint8_t model_flags; /* use mlx5_wq_model_flags */ +}; + + +struct mlx5_wqe_eth_seg { + uint32_t rsvd0; + uint8_t cs_flags; + uint8_t rsvd1; + uint16_t mss; + uint32_t rsvd2; + uint16_t inline_hdr_sz; + uint8_t inline_hdr_start[2]; + uint8_t inline_hdr[16]; +}; + + +#define to_mxxx(xxx, type)\ + ((struct mlx5_##type *)\ + ((void *) ((uintptr_t)ib##xxx - offsetof(struct mlx5_##type, ibv_##xxx)))) + +static inline struct mlx5_qp *to_mqp(struct ibv_qp *ibqp) +{ + struct verbs_qp *vqp = (struct verbs_qp *)ibqp; + return container_of(vqp, struct mlx5_qp, verbs_qp); +} + +static inline struct mlx5_cq *to_mcq(struct ibv_cq *ibcq) +{ + return to_mxxx(cq, cq); +} + +struct ibv_mlx5_qp_info { + uint32_t qpn; + volatile uint32_t *dbrec; + struct { + void *buf; + unsigned wqe_cnt; + unsigned stride; + } sq, rq; + struct { + void *reg; + unsigned size; + int need_lock; + } bf; +}; + +static inline int ibv_mlx5_exp_get_qp_info(struct ibv_qp *qp, struct ibv_mlx5_qp_info *qp_info) +{ + struct mlx5_qp *mqp = to_mqp(qp); + + if ((mqp->gen_data.scur_post != 0) || (mqp->rq.head != 0)) + return -1; + + qp_info->qpn = mqp->ctrl_seg.qp_num; + qp_info->dbrec = mqp->gen_data.db; + qp_info->sq.buf = (void *)((uintptr_t)mqp->buf.buf + mqp->sq.offset); + qp_info->sq.wqe_cnt = mqp->sq.wqe_cnt; + qp_info->sq.stride = 1 << mqp->sq.wqe_shift; + qp_info->rq.buf = (void *)((uintptr_t)mqp->buf.buf + mqp->rq.offset); + qp_info->rq.wqe_cnt = mqp->rq.wqe_cnt; + qp_info->rq.stride = 1 << mqp->rq.wqe_shift; + qp_info->bf.reg = mqp->gen_data.bf->reg; + qp_info->bf.need_lock = mqp->gen_data.bf->need_lock; + + if (mqp->gen_data.bf->uuarn > 0) + qp_info->bf.size = mqp->gen_data.bf->buf_size; + else + qp_info->bf.size = 0; + + return 0; +} + +struct ibv_mlx5_cq_info { + uint32_t cqn; + unsigned cqe_cnt; + void *buf; + volatile uint32_t *dbrec; + unsigned cqe_size; +}; + +static inline int ibv_mlx5_exp_get_cq_info(struct ibv_cq *cq, struct ibv_mlx5_cq_info *cq_info) +{ + struct mlx5_cq *mcq = to_mcq(cq); + + if (mcq->cons_index != 0) + return -1; + + cq_info->cqn = mcq->cqn; + cq_info->cqe_cnt = mcq->ibv_cq.cqe + 1; + cq_info->cqe_size = mcq->cqe_sz; + cq_info->buf = mcq->active_buf->buf; + cq_info->dbrec = mcq->dbrec; + + return 0; +} + +struct ibv_mlx5_srq_info { + void *buf; + volatile uint32_t *dbrec; + unsigned stride; + unsigned head; + unsigned tail; +}; + +static inline int ibv_mlx5_exp_get_srq_info(struct ibv_srq *srq, struct ibv_mlx5_srq_info *srq_info) +{ + struct mlx5_srq *msrq; + + if (srq->handle == LEGACY_XRC_SRQ_HANDLE) + srq = (struct ibv_srq *)(((struct ibv_srq_legacy *)srq)->ibv_srq); + + msrq = container_of(srq, struct mlx5_srq, vsrq.srq); + + if (msrq->counter != 0) + return -1; + + srq_info->buf = msrq->buf.buf; + srq_info->dbrec = msrq->db; + srq_info->stride = 1 << msrq->wqe_shift; + srq_info->head = msrq->head; + srq_info->tail = msrq->tail; + + return 0; +} + +static inline void ibv_mlx5_exp_update_cq_ci(struct ibv_cq *cq, unsigned cq_ci) +{ + struct mlx5_cq *mcq = to_mcq(cq); + + mcq->cons_index = cq_ci; +} + +#endif diff --git a/external_libs/ibverbs/include/infiniband/ofa_verbs.h b/external_libs/ibverbs/include/infiniband/ofa_verbs.h new file mode 100644 index 00000000..cb0ad62d --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/ofa_verbs.h @@ -0,0 +1,210 @@ +#ifndef INFINIBAND_OFA_VERBS_H +#define INFINIBAND_OFA_VERBS_H + +struct ibv_srq_init_attr; +struct ibv_cq; +struct ibv_pd; +struct ibv_qp_init_attr; +struct ibv_qp_attr; + + +#ifdef __GNUC__ +#define DEPRECATED __attribute__((deprecated)) +#else +#define DEPRECATED +#endif + +/* XRC compatability layer */ +#define LEGACY_XRC_SRQ_HANDLE 0xffffffff + +struct ibv_xrc_domain { + struct ibv_context *context; + uint32_t handle; +}; + +struct ibv_srq_legacy { + struct ibv_context *context; + void *srq_context; + struct ibv_pd *pd; + uint32_t handle; + + uint32_t events_completed; + + uint32_t xrc_srq_num_bin_compat; + struct ibv_xrc_domain *xrc_domain_bin_compat; + struct ibv_cq *xrc_cq_bin_compat; + + pthread_mutex_t mutex; + pthread_cond_t cond; + /* Here we hook the new one from OFED 2.0 */ + void *ibv_srq; + /* Below 3 fields are for legacy source compatibility, reside + * on same offset as of those fields in struct ibv_srq. + */ + uint32_t xrc_srq_num; + struct ibv_xrc_domain *xrc_domain; + struct ibv_cq *xrc_cq; +}; + +/** + * ibv_open_xrc_domain - open an XRC domain + * Returns a reference to an XRC domain. + * + * @context: Device context + * @fd: descriptor for inode associated with the domain + * If fd == -1, no inode is associated with the domain; in this ca= se, + * the only legal value for oflag is O_CREAT + * + * @oflag: oflag values are constructed by OR-ing flags from the following list + * + * O_CREAT + * If a domain belonging to device named by context is already associated + * with the inode, this flag has no effect, except as noted under O_EXCL + * below. Otherwise, a new XRC domain is created and is associated with + * inode specified by fd. + * + * O_EXCL + * If O_EXCL and O_CREAT are set, open will fail if a domain associated with + * the inode exists. The check for the existence of the domain and creation + * of the domain if it does not exist is atomic with respect to other + * processes executing open with fd naming the same inode. + */ +struct ibv_xrc_domain *ibv_open_xrc_domain(struct ibv_context *context, + int fd, int oflag) DEPRECATED; + +/** + * ibv_create_xrc_srq - Creates a SRQ associated with the specified protection + * domain and xrc domain. + * @pd: The protection domain associated with the SRQ. + * @xrc_domain: The XRC domain associated with the SRQ. + * @xrc_cq: CQ to report completions for XRC packets on. + * + * @srq_init_attr: A list of initial attributes required to create the SRQ. + * + * srq_attr->max_wr and srq_attr->max_sge are read the determine the + * requested size of the SRQ, and set to the actual values allocated + * on return. If ibv_create_srq() succeeds, then max_wr and max_sge + * will always be at least as large as the requested values. + */ +struct ibv_srq *ibv_create_xrc_srq(struct ibv_pd *pd, + struct ibv_xrc_domain *xrc_domain, + struct ibv_cq *xrc_cq, + struct ibv_srq_init_attr *srq_init_attr) DEPRECATED; + +/** + * ibv_close_xrc_domain - close an XRC domain + * If this is the last reference, destroys the domain. + * + * @d: reference to XRC domain to close + * + * close is implicitly performed at process exit. + */ +int ibv_close_xrc_domain(struct ibv_xrc_domain *d) DEPRECATED; + +/** + * ibv_create_xrc_rcv_qp - creates an XRC QP for serving as a receive-side-only QP, + * + * This QP is created in kernel space, and persists until the last process + * registered for the QP calls ibv_unreg_xrc_rcv_qp() (at which time the QP + * is destroyed). + * + * @init_attr: init attributes to use for QP. xrc domain MUST be included here. + * All other fields are ignored. + * + * @xrc_rcv_qpn: qp_num of created QP (if success). To be passed to the + * remote node (sender). The remote node will use xrc_rcv_qpn + * in ibv_post_send when sending to XRC SRQ's on this host + * in the same xrc domain. + * + * RETURNS: success (0), or a (negative) error value. + * + * NOTE: this verb also registers the calling user-process with the QP at its + * creation time (implicit call to ibv_reg_xrc_rcv_qp), to avoid race + * conditions. The creating process will need to call ibv_unreg_xrc_qp() + * for the QP to release it from this process. + */ +int ibv_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr, + uint32_t *xrc_rcv_qpn) DEPRECATED; + +/** + * ibv_modify_xrc_rcv_qp - modifies an xrc_rcv qp. + * + * @xrc_domain: xrc domain the QP belongs to (for verification). + * @xrc_qp_num: The (24 bit) number of the XRC QP. + * @attr: modify-qp attributes. The following fields must be specified: + * for RESET_2_INIT: qp_state, pkey_index , port, qp_access_flags + * for INIT_2_RTR: qp_state, path_mtu, dest_qp_num, rq_psn, + * max_dest_rd_atomic, min_rnr_timer, ah_attr + * The QP need not be brought to RTS for the QP to operate as a + * receive-only QP. + * @attr_mask: bitmap indicating which attributes are provided in the attr + * struct. Used for validity checking. + * The following bits must be set: + * for RESET_2_INIT: IBV_QP_PKEY_INDEX, IBV_QP_PORT, + * IBV_QP_ACCESS_FLAGS, IBV_QP_STATE + * for INIT_2_RTR: IBV_QP_AV, IBV_QP_PATH_MTU, IBV_QP_DEST_QPN, + * IBV_QP_RQ_PSN, IBV_QP_MAX_DEST_RD_ATOMIC, + * IBV_QP_MIN_RNR_TIMER, IBV_QP_STATE + * + * RETURNS: success (0), or a (positive) error value. + * + */ +int ibv_modify_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, + uint32_t xrc_qp_num, + struct ibv_qp_attr *attr, int attr_mask) DEPRECATED; + +/** + * ibv_query_xrc_rcv_qp - queries an xrc_rcv qp. + * + * @xrc_domain: xrc domain the QP belongs to (for verification). + * @xrc_qp_num: The (24 bit) number of the XRC QP. + * @attr: for returning qp attributes. + * @attr_mask: bitmap indicating which attributes to return. + * @init_attr: for returning the init attributes + * + * RETURNS: success (0), or a (positive) error value. + * + */ +int ibv_query_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num, + struct ibv_qp_attr *attr, int attr_mask, + struct ibv_qp_init_attr *init_attr) DEPRECATED; + +/** + * ibv_reg_xrc_rcv_qp: registers a user process with an XRC QP which serves as + * a receive-side only QP. + * + * @xrc_domain: xrc domain the QP belongs to (for verification). + * @xrc_qp_num: The (24 bit) number of the XRC QP. + * + * RETURNS: success (0), + * or error (EINVAL), if: + * 1. There is no such QP_num allocated. + * 2. The QP is allocated, but is not an receive XRC QP + * 3. The XRC QP does not belong to the given domain. + */ +int ibv_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, + uint32_t xrc_qp_num) DEPRECATED; + +/** + * ibv_unreg_xrc_rcv_qp: detaches a user process from an XRC QP serving as + * a receive-side only QP. If as a result, there are no remaining + * userspace processes registered for this XRC QP, it is destroyed. + * + * @xrc_domain: xrc domain the QP belongs to (for verification). + * @xrc_qp_num: The (24 bit) number of the XRC QP. + * + * RETURNS: success (0), + * or error (EINVAL), if: + * 1. There is no such QP_num allocated. + * 2. The QP is allocated, but is not an XRC QP + * 3. The XRC QP does not belong to the given domain. + * NOTE: There is no reason to return a special code if the QP is destroyed. + * The unregister simply succeeds. + */ +int ibv_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, + uint32_t xrc_qp_num) DEPRECATED; + + +#endif + + diff --git a/external_libs/ibverbs/include/infiniband/opcode.h b/external_libs/ibverbs/include/infiniband/opcode.h new file mode 100644 index 00000000..fd4bc96a --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/opcode.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2005 Topspin Communications. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef INFINIBAND_OPCODE_H +#define INFINIBAND_OPCODE_H + +/* + * This macro cleans up the definitions of constants for BTH opcodes. + * It is used to define constants such as IBV_OPCODE_UD_SEND_ONLY, + * which becomes IBV_OPCODE_UD + IBV_OPCODE_SEND_ONLY, and this gives + * the correct value. + * + * In short, user code should use the constants defined using the + * macro rather than worrying about adding together other constants. +*/ +#define IBV_OPCODE(transport, op) \ + IBV_OPCODE_ ## transport ## _ ## op = \ + IBV_OPCODE_ ## transport + IBV_OPCODE_ ## op + +enum { + /* transport types -- just used to define real constants */ + IBV_OPCODE_RC = 0x00, + IBV_OPCODE_UC = 0x20, + IBV_OPCODE_RD = 0x40, + IBV_OPCODE_UD = 0x60, + + /* operations -- just used to define real constants */ + IBV_OPCODE_SEND_FIRST = 0x00, + IBV_OPCODE_SEND_MIDDLE = 0x01, + IBV_OPCODE_SEND_LAST = 0x02, + IBV_OPCODE_SEND_LAST_WITH_IMMEDIATE = 0x03, + IBV_OPCODE_SEND_ONLY = 0x04, + IBV_OPCODE_SEND_ONLY_WITH_IMMEDIATE = 0x05, + IBV_OPCODE_RDMA_WRITE_FIRST = 0x06, + IBV_OPCODE_RDMA_WRITE_MIDDLE = 0x07, + IBV_OPCODE_RDMA_WRITE_LAST = 0x08, + IBV_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE = 0x09, + IBV_OPCODE_RDMA_WRITE_ONLY = 0x0a, + IBV_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE = 0x0b, + IBV_OPCODE_RDMA_READ_REQUEST = 0x0c, + IBV_OPCODE_RDMA_READ_RESPONSE_FIRST = 0x0d, + IBV_OPCODE_RDMA_READ_RESPONSE_MIDDLE = 0x0e, + IBV_OPCODE_RDMA_READ_RESPONSE_LAST = 0x0f, + IBV_OPCODE_RDMA_READ_RESPONSE_ONLY = 0x10, + IBV_OPCODE_ACKNOWLEDGE = 0x11, + IBV_OPCODE_ATOMIC_ACKNOWLEDGE = 0x12, + IBV_OPCODE_COMPARE_SWAP = 0x13, + IBV_OPCODE_FETCH_ADD = 0x14, + + /* real constants follow -- see comment about above IBV_OPCODE() + macro for more details */ + + /* RC */ + IBV_OPCODE(RC, SEND_FIRST), + IBV_OPCODE(RC, SEND_MIDDLE), + IBV_OPCODE(RC, SEND_LAST), + IBV_OPCODE(RC, SEND_LAST_WITH_IMMEDIATE), + IBV_OPCODE(RC, SEND_ONLY), + IBV_OPCODE(RC, SEND_ONLY_WITH_IMMEDIATE), + IBV_OPCODE(RC, RDMA_WRITE_FIRST), + IBV_OPCODE(RC, RDMA_WRITE_MIDDLE), + IBV_OPCODE(RC, RDMA_WRITE_LAST), + IBV_OPCODE(RC, RDMA_WRITE_LAST_WITH_IMMEDIATE), + IBV_OPCODE(RC, RDMA_WRITE_ONLY), + IBV_OPCODE(RC, RDMA_WRITE_ONLY_WITH_IMMEDIATE), + IBV_OPCODE(RC, RDMA_READ_REQUEST), + IBV_OPCODE(RC, RDMA_READ_RESPONSE_FIRST), + IBV_OPCODE(RC, RDMA_READ_RESPONSE_MIDDLE), + IBV_OPCODE(RC, RDMA_READ_RESPONSE_LAST), + IBV_OPCODE(RC, RDMA_READ_RESPONSE_ONLY), + IBV_OPCODE(RC, ACKNOWLEDGE), + IBV_OPCODE(RC, ATOMIC_ACKNOWLEDGE), + IBV_OPCODE(RC, COMPARE_SWAP), + IBV_OPCODE(RC, FETCH_ADD), + + /* UC */ + IBV_OPCODE(UC, SEND_FIRST), + IBV_OPCODE(UC, SEND_MIDDLE), + IBV_OPCODE(UC, SEND_LAST), + IBV_OPCODE(UC, SEND_LAST_WITH_IMMEDIATE), + IBV_OPCODE(UC, SEND_ONLY), + IBV_OPCODE(UC, SEND_ONLY_WITH_IMMEDIATE), + IBV_OPCODE(UC, RDMA_WRITE_FIRST), + IBV_OPCODE(UC, RDMA_WRITE_MIDDLE), + IBV_OPCODE(UC, RDMA_WRITE_LAST), + IBV_OPCODE(UC, RDMA_WRITE_LAST_WITH_IMMEDIATE), + IBV_OPCODE(UC, RDMA_WRITE_ONLY), + IBV_OPCODE(UC, RDMA_WRITE_ONLY_WITH_IMMEDIATE), + + /* RD */ + IBV_OPCODE(RD, SEND_FIRST), + IBV_OPCODE(RD, SEND_MIDDLE), + IBV_OPCODE(RD, SEND_LAST), + IBV_OPCODE(RD, SEND_LAST_WITH_IMMEDIATE), + IBV_OPCODE(RD, SEND_ONLY), + IBV_OPCODE(RD, SEND_ONLY_WITH_IMMEDIATE), + IBV_OPCODE(RD, RDMA_WRITE_FIRST), + IBV_OPCODE(RD, RDMA_WRITE_MIDDLE), + IBV_OPCODE(RD, RDMA_WRITE_LAST), + IBV_OPCODE(RD, RDMA_WRITE_LAST_WITH_IMMEDIATE), + IBV_OPCODE(RD, RDMA_WRITE_ONLY), + IBV_OPCODE(RD, RDMA_WRITE_ONLY_WITH_IMMEDIATE), + IBV_OPCODE(RD, RDMA_READ_REQUEST), + IBV_OPCODE(RD, RDMA_READ_RESPONSE_FIRST), + IBV_OPCODE(RD, RDMA_READ_RESPONSE_MIDDLE), + IBV_OPCODE(RD, RDMA_READ_RESPONSE_LAST), + IBV_OPCODE(RD, RDMA_READ_RESPONSE_ONLY), + IBV_OPCODE(RD, ACKNOWLEDGE), + IBV_OPCODE(RD, ATOMIC_ACKNOWLEDGE), + IBV_OPCODE(RD, COMPARE_SWAP), + IBV_OPCODE(RD, FETCH_ADD), + + /* UD */ + IBV_OPCODE(UD, SEND_ONLY), + IBV_OPCODE(UD, SEND_ONLY_WITH_IMMEDIATE) +}; + +#endif /* INFINIBAND_OPCODE_H */ diff --git a/external_libs/ibverbs/include/infiniband/peer_ops.h b/external_libs/ibverbs/include/infiniband/peer_ops.h new file mode 100644 index 00000000..d2fd265d --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/peer_ops.h @@ -0,0 +1,370 @@ +/* + * Copyright (c) 2016 Mellanox Technologies Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef PEER_OPS_H +#define PEER_OPS_H + +#include +#include +#include +#include + +BEGIN_C_DECLS + +enum ibv_exp_peer_op { + IBV_EXP_PEER_OP_RESERVED1 = 1, + + IBV_EXP_PEER_OP_FENCE = 0, + + IBV_EXP_PEER_OP_STORE_DWORD = 4, + IBV_EXP_PEER_OP_STORE_QWORD = 2, + IBV_EXP_PEER_OP_COPY_BLOCK = 3, + + IBV_EXP_PEER_OP_POLL_AND_DWORD = 12, + IBV_EXP_PEER_OP_POLL_NOR_DWORD = 13, + IBV_EXP_PEER_OP_POLL_GEQ_DWORD = 14, +}; + +enum ibv_exp_peer_op_caps { + IBV_EXP_PEER_OP_FENCE_CAP = (1 << IBV_EXP_PEER_OP_FENCE), + IBV_EXP_PEER_OP_STORE_DWORD_CAP = (1 << IBV_EXP_PEER_OP_STORE_DWORD), + IBV_EXP_PEER_OP_STORE_QWORD_CAP = (1 << IBV_EXP_PEER_OP_STORE_QWORD), + IBV_EXP_PEER_OP_COPY_BLOCK_CAP = (1 << IBV_EXP_PEER_OP_COPY_BLOCK), + IBV_EXP_PEER_OP_POLL_AND_DWORD_CAP + = (1 << IBV_EXP_PEER_OP_POLL_AND_DWORD), + IBV_EXP_PEER_OP_POLL_NOR_DWORD_CAP + = (1 << IBV_EXP_PEER_OP_POLL_NOR_DWORD), + IBV_EXP_PEER_OP_POLL_GEQ_DWORD_CAP + = (1 << IBV_EXP_PEER_OP_POLL_GEQ_DWORD), +}; + +enum ibv_exp_peer_fence { + IBV_EXP_PEER_FENCE_OP_READ = (1 << 0), + IBV_EXP_PEER_FENCE_OP_WRITE = (1 << 1), + IBV_EXP_PEER_FENCE_FROM_CPU = (1 << 2), + IBV_EXP_PEER_FENCE_FROM_HCA = (1 << 3), + IBV_EXP_PEER_FENCE_MEM_SYS = (1 << 4), + IBV_EXP_PEER_FENCE_MEM_PEER = (1 << 5), +}; + +/* Indicate HW entities supposed to access memory buffer: + * IBV_EXP_PEER_DIRECTION_FROM_X means X writes to the buffer + * IBV_EXP_PEER_DIRECTION_TO_Y means Y read from the buffer + */ +enum ibv_exp_peer_direction { + IBV_EXP_PEER_DIRECTION_FROM_CPU = (1 << 0), + IBV_EXP_PEER_DIRECTION_FROM_HCA = (1 << 1), + IBV_EXP_PEER_DIRECTION_FROM_PEER = (1 << 2), + IBV_EXP_PEER_DIRECTION_TO_CPU = (1 << 3), + IBV_EXP_PEER_DIRECTION_TO_HCA = (1 << 4), + IBV_EXP_PEER_DIRECTION_TO_PEER = (1 << 5), +}; + +struct ibv_exp_peer_buf_alloc_attr { + size_t length; + /* Bitmask from enum ibv_exp_peer_direction */ + uint32_t dir; + /* The ID of the peer device which will be + * accessing the allocated buffer + */ + uint64_t peer_id; + /* Data alignment */ + uint32_t alignment; + /* Reserved for future extensions, must be 0 */ + uint32_t comp_mask; +}; + +struct ibv_exp_peer_buf { + void *addr; + size_t length; + /* Reserved for future extensions, must be 0 */ + uint32_t comp_mask; +}; + +enum ibv_exp_peer_direct_attr_mask { + IBV_EXP_PEER_DIRECT_VERSION = (1 << 0) /* Must be set */ +}; + +#define IBV_EXP_PEER_IOMEMORY ((struct ibv_exp_peer_buf *)-1UL) + +struct ibv_exp_peer_direct_attr { + /* Unique ID per peer device. + * Used to identify specific HW devices where relevant. + */ + uint64_t peer_id; + /* buf_alloc callback should return struct ibv_exp_peer_buf with buffer + * of at least attr->length. + * @attr: description of desired buffer + * + * Buffer should be mapped in the application address space + * for read/write (depends on attr->dir value). + * attr->dir value is supposed to indicate the expected directions + * of access to the buffer, to allow optimization by the peer driver. + * If NULL returned then buffer will be allocated in system memory + * by ibverbs driver. + */ + struct ibv_exp_peer_buf *(*buf_alloc)(struct ibv_exp_peer_buf_alloc_attr *attr); + /* If buffer was allocated by buf_alloc then buf_release will be + * called to release it. + * @pb: struct returned by buf_alloc + * + * buf_release is responsible to release everything allocated by + * buf_alloc. + * Return 0 on succes. + */ + int (*buf_release)(struct ibv_exp_peer_buf *pb); + /* register_va callback should register virtual address from the + * application as an area the peer is allowed to access. + * @start: pointer to beginning of region in virtual space + * @length: length of region + * @peer_id: the ID of the peer device which will be accessing + * the region. + * @pb: if registering a buffer that was returned from buf_alloc(), + * pb is the struct that was returned. If registering io memory area, + * pb is IBV_EXP_PEER_IOMEMORY. Otherwise - NULL + * + * Return id of registered address on success, 0 on failure. + */ + uint64_t (*register_va)(void *start, size_t length, uint64_t peer_id, + struct ibv_exp_peer_buf *pb); + /* If virtual address was registered with register_va then + * unregister_va will be called to unregister it. + * @target_id: id returned by register_va + * @peer_id: the ID of the peer device passed to register_va + * + * Return 0 on success. + */ + int (*unregister_va)(uint64_t target_id, uint64_t peer_id); + /* Bitmask from ibv_exp_peer_op_caps */ + uint64_t caps; + /* Maximal length of DMA operation the peer can do in copy-block */ + size_t peer_dma_op_map_len; + /* From ibv_exp_peer_direct_attr_mask */ + uint32_t comp_mask; + /* Feature version, must be 1 */ + uint32_t version; +}; + +/* QP API - CPU posts send work-requests without exposing them to the HW. + * Later, the peer device exposes the relevant work requests to the HCA + * for execution. + */ + +struct peer_op_wr { + struct peer_op_wr *next; + enum ibv_exp_peer_op type; + union { + struct { + uint64_t fence_flags; /* from ibv_exp_peer_fence */ + } fence; + + struct { + uint32_t data; + uint64_t target_id; + size_t offset; + } dword_va; /* Use for all operations targeting dword */ + + struct { + uint64_t data; + uint64_t target_id; + size_t offset; + } qword_va; /* Use for all operations targeting qword */ + + struct { + void *src; + uint64_t target_id; + size_t offset; + size_t len; + } copy_op; + } wr; + uint32_t comp_mask; /* Reserved for future expensions, must be 0 */ +}; + +struct ibv_exp_peer_commit { + /* IN/OUT - linked list of empty/filled descriptors */ + struct peer_op_wr *storage; + /* IN/OUT - number of allocated/filled descriptors */ + uint32_t entries; + /* OUT - identifier used in ibv_exp_rollback_qp to rollback WQEs set */ + uint64_t rollback_id; + uint32_t comp_mask; /* Reserved for future expensions, must be 0 */ +}; + +/** + * ibv_exp_peer_commit_qp - request descriptors for committing all WQEs + * currently posted to the send work queue + * @qp: the QP being requested + * @peer: context with list of &struct peer_op_wr describing actions + * necessary to commit WQEs + * + * Function + * - fill peer->storage with descriptors + * - put number of filled descriptors to peer->entries; + * - put data necessary for rollback to peer->rollback_id + * If number of entries is not sufficient - return -ENOSPC + * + * Note: caller is responsible to ensure that the peer fences any data store + * before executing the commit + */ +static inline int ibv_exp_peer_commit_qp(struct ibv_qp *qp, + struct ibv_exp_peer_commit *peer) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(qp->context, exp_peer_commit_qp); + if (!vctx) + return ENOSYS; + + return vctx->exp_peer_commit_qp(qp, peer); +} + +enum ibv_exp_rollback_flags { + /* Abort all WQEs which were not committed to HW yet. + * rollback_id is ignored. **/ + IBV_EXP_ROLLBACK_ABORT_UNCOMMITED = (1 << 0), + /* Abort the request even if there are following requests + * being aborted as well. **/ + IBV_EXP_ROLLBACK_ABORT_LATE = (1 << 1), +}; + +struct ibv_exp_rollback_ctx { + uint64_t rollback_id; /* from ibv_exp_peer_commit call */ + uint32_t flags; /* from ibv_exp_rollback_flags */ + uint32_t comp_mask; /* Reserved for future expensions, must be 0 */ +}; + +/** + * ibv_exp_rollback_qp - indicate that the commit attempt failed + * @qp: the QP being rolled back + * @rollback: context with rollback_id returned by + * earlier ibv_exp_peer_commit_qp and flags + */ +static inline int ibv_exp_rollback_qp(struct ibv_qp *qp, + struct ibv_exp_rollback_ctx *rollback) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(qp->context, exp_rollback_send); + if (!vctx) + return ENOSYS; + + return vctx->exp_rollback_send(qp, rollback); +} + +/* CQ interface - peek into a CQ and describe how to check if + * there is a CQ entry available. + */ + +enum { + IBV_EXP_PEER_PEEK_ABSOLUTE, + IBV_EXP_PEER_PEEK_RELATIVE +}; + +struct ibv_exp_peer_peek { + /* IN/OUT - linked list of empty/filled descriptors */ + struct peer_op_wr *storage; + /* IN/OUT - number of allocated/filled descriptors */ + uint32_t entries; + /* IN - Which CQ entry does the peer want to peek for + * completion. According to "whence" directive entry + * chosen as follows: + * IBV_EXP_PEER_PEEK_ABSOLUTE - + * "offset" is absolute index of entry wrapped to 32-bit + * IBV_EXP_PEER_PEEK_RELATIVE - + * "offset" is relative to current poll_cq location. + */ + uint32_t whence; + uint32_t offset; + /* OUT - identifier used in ibv_exp_peer_ack_peek_cq to advance CQ */ + uint64_t peek_id; + uint32_t comp_mask; /* Reserved for future expensions, must be 0 */ +}; + +/** + * ibv_exp_peer_peek_cq - request descriptors for peeking CQ in specific + * offset from currently expected CQ entry. + * @cq: the CQ being requested + * @peer_ctx: context with list of &struct peer_op_wr describing actions + * necessary to wait for desired CQ entry is delivered and report + * this to ibverbs. + * + * A peek CQ request places a "block" on the relevant CQ entry. + * Poll CQ requests to poll the CQ entry will fail with an error. + * The block will be removed by executing the descriptors. + * If the peer will not be able to execute the descriptors, + * it should call ibv_exp_peer_abort_peek_cq to remove the block. + * + * Function + * - fill peek_ctx->storage with descriptors. + * - put number of filled descriptors to peek_ctx->entries. + * - put data necessary to abort peek. + * If number of entries is not sufficient - return -ENOSPC. + */ +static inline int ibv_exp_peer_peek_cq(struct ibv_cq *cq, + struct ibv_exp_peer_peek *peek_ctx) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(cq->context, exp_peer_peek_cq); + if (!vctx) + return ENOSYS; + + return vctx->exp_peer_peek_cq(cq, peek_ctx); +} + +struct ibv_exp_peer_abort_peek { + uint64_t peek_id; /* From the peer_peek_cq call */ + uint32_t comp_mask; /* Reserved for future expensions, must be 0 */ +}; + +/** + * ibv_exp_peer_abort_peek_cq - indicate that peek is aborted + * @cq: the CQ being rolled back + * @abort_ctx: context with peek_id returned by earlier ibv_exp_peer_peek_cq + * + * Note: This should be done only if the peek descriptors were not executed + */ +static inline int ibv_exp_peer_abort_peek_cq(struct ibv_cq *cq, + struct ibv_exp_peer_abort_peek *abort_ctx) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(cq->context, exp_peer_abort_peek_cq); + if (!vctx) + return ENOSYS; + + return vctx->exp_peer_abort_peek_cq(cq, abort_ctx); +} + +END_C_DECLS + +#endif /* PEER_OPS_H */ diff --git a/external_libs/ibverbs/include/infiniband/sa-kern-abi.h b/external_libs/ibverbs/include/infiniband/sa-kern-abi.h new file mode 100644 index 00000000..4927d114 --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/sa-kern-abi.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2005 Intel Corporation. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef INFINIBAND_SA_KERN_ABI_H +#define INFINIBAND_SA_KERN_ABI_H + +#include + +/* + * Obsolete, deprecated names. Will be removed in libibverbs 1.1. + */ +#define ib_kern_path_rec ibv_kern_path_rec + +struct ibv_kern_path_rec { + __u8 dgid[16]; + __u8 sgid[16]; + __u16 dlid; + __u16 slid; + __u32 raw_traffic; + __u32 flow_label; + __u32 reversible; + __u32 mtu; + __u16 pkey; + __u8 hop_limit; + __u8 traffic_class; + __u8 numb_path; + __u8 sl; + __u8 mtu_selector; + __u8 rate_selector; + __u8 rate; + __u8 packet_life_time_selector; + __u8 packet_life_time; + __u8 preference; +}; + +#endif /* INFINIBAND_SA_KERN_ABI_H */ diff --git a/external_libs/ibverbs/include/infiniband/sa.h b/external_libs/ibverbs/include/infiniband/sa.h new file mode 100644 index 00000000..1153e943 --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/sa.h @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2004 Topspin Communications. All rights reserved. + * Copyright (c) 2005 Voltaire, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef INFINIBAND_SA_H +#define INFINIBAND_SA_H + +#include + +struct ibv_sa_path_rec { + /* reserved */ + /* reserved */ + union ibv_gid dgid; + union ibv_gid sgid; + uint16_t dlid; + uint16_t slid; + int raw_traffic; + /* reserved */ + uint32_t flow_label; + uint8_t hop_limit; + uint8_t traffic_class; + int reversible; + uint8_t numb_path; + uint16_t pkey; + /* reserved */ + uint8_t sl; + uint8_t mtu_selector; + uint8_t mtu; + uint8_t rate_selector; + uint8_t rate; + uint8_t packet_life_time_selector; + uint8_t packet_life_time; + uint8_t preference; +}; + +struct ibv_sa_mcmember_rec { + union ibv_gid mgid; + union ibv_gid port_gid; + uint32_t qkey; + uint16_t mlid; + uint8_t mtu_selector; + uint8_t mtu; + uint8_t traffic_class; + uint16_t pkey; + uint8_t rate_selector; + uint8_t rate; + uint8_t packet_life_time_selector; + uint8_t packet_life_time; + uint8_t sl; + uint32_t flow_label; + uint8_t hop_limit; + uint8_t scope; + uint8_t join_state; + int proxy_join; +}; + +struct ibv_sa_service_rec { + uint64_t id; + union ibv_gid gid; + uint16_t pkey; + /* uint16_t resv; */ + uint32_t lease; + uint8_t key[16]; + uint8_t name[64]; + uint8_t data8[16]; + uint16_t data16[8]; + uint32_t data32[4]; + uint64_t data64[2]; +}; + +#define IBV_PATH_RECORD_REVERSIBLE 0x80 + +struct ibv_path_record { + uint64_t service_id; + union ibv_gid dgid; + union ibv_gid sgid; + uint16_t dlid; + uint16_t slid; + uint32_t flowlabel_hoplimit; /* resv-31:28 flow label-27:8 hop limit-7:0*/ + uint8_t tclass; + uint8_t reversible_numpath; /* reversible-7:7 num path-6:0 */ + uint16_t pkey; + uint16_t qosclass_sl; /* qos class-15:4 sl-3:0 */ + uint8_t mtu; /* mtu selector-7:6 mtu-5:0 */ + uint8_t rate; /* rate selector-7:6 rate-5:0 */ + uint8_t packetlifetime; /* lifetime selector-7:6 lifetime-5:0 */ + uint8_t preference; + uint8_t reserved[6]; +}; + +#define IBV_PATH_FLAG_GMP (1<<0) +#define IBV_PATH_FLAG_PRIMARY (1<<1) +#define IBV_PATH_FLAG_ALTERNATE (1<<2) +#define IBV_PATH_FLAG_OUTBOUND (1<<3) +#define IBV_PATH_FLAG_INBOUND (1<<4) +#define IBV_PATH_FLAG_INBOUND_REVERSE (1<<5) +#define IBV_PATH_FLAG_BIDIRECTIONAL (IBV_PATH_FLAG_OUTBOUND | \ + IBV_PATH_FLAG_INBOUND_REVERSE) + +struct ibv_path_data { + uint32_t flags; + uint32_t reserved; + struct ibv_path_record path; +}; + +#endif /* INFINIBAND_SA_H */ diff --git a/external_libs/ibverbs/include/infiniband/verbs.h b/external_libs/ibverbs/include/infiniband/verbs.h new file mode 100644 index 00000000..6a8d7cc1 --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/verbs.h @@ -0,0 +1,1642 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2004, 2011-2012 Intel Corporation. All rights reserved. + * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2005 PathScale, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef INFINIBAND_VERBS_H +#define INFINIBAND_VERBS_H + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +# define BEGIN_C_DECLS extern "C" { +# define END_C_DECLS } +#else /* !__cplusplus */ +# define BEGIN_C_DECLS +# define END_C_DECLS +#endif /* __cplusplus */ + +#if __GNUC__ >= 3 +# define __attribute_const __attribute__((const)) +#else +# define __attribute_const +#endif + +BEGIN_C_DECLS + +union ibv_gid { + uint8_t raw[16]; + struct { + uint64_t subnet_prefix; + uint64_t interface_id; + } global; +}; + +#ifndef container_of +/** + * container_of - cast a member of a structure out to the containing structure + * @ptr: the pointer to the member. + * @type: the type of the container struct this is embedded in. + * @member: the name of the member within the struct. + * + */ +#define container_of(ptr, type, member) \ + ((type *) ((uint8_t *)(ptr) - offsetof(type, member))) +#endif + +#define vext_field_avail(type, fld, sz) (offsetof(type, fld) < (sz)) + +static void *__VERBS_ABI_IS_EXTENDED = ((uint8_t *)NULL) - 1; + +enum ibv_node_type { + IBV_NODE_UNKNOWN = -1, + IBV_NODE_CA = 1, + IBV_NODE_SWITCH, + IBV_NODE_ROUTER, + IBV_NODE_RNIC, + + /* Leave a gap for future node types before starting with + * experimental node types. + */ + IBV_EXP_NODE_TYPE_START = 32, + IBV_EXP_NODE_MIC = IBV_EXP_NODE_TYPE_START +}; + +enum ibv_transport_type { + IBV_TRANSPORT_UNKNOWN = -1, + IBV_TRANSPORT_IB = 0, + IBV_TRANSPORT_IWARP, + + /* Leave a gap for future transport types before starting with + * experimental transport types. + */ + IBV_EXP_TRANSPORT_TYPE_START = 32, + IBV_EXP_TRANSPORT_SCIF = IBV_EXP_TRANSPORT_TYPE_START +}; + +enum ibv_device_cap_flags { + IBV_DEVICE_RESIZE_MAX_WR = 1, + IBV_DEVICE_BAD_PKEY_CNTR = 1 << 1, + IBV_DEVICE_BAD_QKEY_CNTR = 1 << 2, + IBV_DEVICE_RAW_MULTI = 1 << 3, + IBV_DEVICE_AUTO_PATH_MIG = 1 << 4, + IBV_DEVICE_CHANGE_PHY_PORT = 1 << 5, + IBV_DEVICE_UD_AV_PORT_ENFORCE = 1 << 6, + IBV_DEVICE_CURR_QP_STATE_MOD = 1 << 7, + IBV_DEVICE_SHUTDOWN_PORT = 1 << 8, + IBV_DEVICE_INIT_TYPE = 1 << 9, + IBV_DEVICE_PORT_ACTIVE_EVENT = 1 << 10, + IBV_DEVICE_SYS_IMAGE_GUID = 1 << 11, + IBV_DEVICE_RC_RNR_NAK_GEN = 1 << 12, + IBV_DEVICE_SRQ_RESIZE = 1 << 13, + IBV_DEVICE_N_NOTIFY_CQ = 1 << 14, + IBV_DEVICE_XRC = 1 << 20, + IBV_DEVICE_MANAGED_FLOW_STEERING = 1 << 29 +}; + +enum ibv_atomic_cap { + IBV_ATOMIC_NONE, + IBV_ATOMIC_HCA, + IBV_ATOMIC_GLOB +}; + +struct ibv_device_attr { + char fw_ver[64]; + uint64_t node_guid; + uint64_t sys_image_guid; + uint64_t max_mr_size; + uint64_t page_size_cap; + uint32_t vendor_id; + uint32_t vendor_part_id; + uint32_t hw_ver; + int max_qp; + int max_qp_wr; + int device_cap_flags; + int max_sge; + int max_sge_rd; + int max_cq; + int max_cqe; + int max_mr; + int max_pd; + int max_qp_rd_atom; + int max_ee_rd_atom; + int max_res_rd_atom; + int max_qp_init_rd_atom; + int max_ee_init_rd_atom; + enum ibv_atomic_cap atomic_cap; + int max_ee; + int max_rdd; + int max_mw; + int max_raw_ipv6_qp; + int max_raw_ethy_qp; + int max_mcast_grp; + int max_mcast_qp_attach; + int max_total_mcast_qp_attach; + int max_ah; + int max_fmr; + int max_map_per_fmr; + int max_srq; + int max_srq_wr; + int max_srq_sge; + uint16_t max_pkeys; + uint8_t local_ca_ack_delay; + uint8_t phys_port_cnt; +}; + +enum ibv_mtu { + IBV_MTU_256 = 1, + IBV_MTU_512 = 2, + IBV_MTU_1024 = 3, + IBV_MTU_2048 = 4, + IBV_MTU_4096 = 5 +}; + +enum ibv_port_state { + IBV_PORT_NOP = 0, + IBV_PORT_DOWN = 1, + IBV_PORT_INIT = 2, + IBV_PORT_ARMED = 3, + IBV_PORT_ACTIVE = 4, + IBV_PORT_ACTIVE_DEFER = 5 +}; + +enum { + IBV_LINK_LAYER_UNSPECIFIED, + IBV_LINK_LAYER_INFINIBAND, + IBV_LINK_LAYER_ETHERNET, + + /* Leave a gap for future link layer types before starting with + * experimental link layer. + */ + IBV_EXP_LINK_LAYER_START = 32, + IBV_EXP_LINK_LAYER_SCIF = IBV_EXP_LINK_LAYER_START +}; + +enum ibv_port_cap_flags { + IBV_PORT_SM = 1 << 1, + IBV_PORT_NOTICE_SUP = 1 << 2, + IBV_PORT_TRAP_SUP = 1 << 3, + IBV_PORT_OPT_IPD_SUP = 1 << 4, + IBV_PORT_AUTO_MIGR_SUP = 1 << 5, + IBV_PORT_SL_MAP_SUP = 1 << 6, + IBV_PORT_MKEY_NVRAM = 1 << 7, + IBV_PORT_PKEY_NVRAM = 1 << 8, + IBV_PORT_LED_INFO_SUP = 1 << 9, + IBV_PORT_SYS_IMAGE_GUID_SUP = 1 << 11, + IBV_PORT_PKEY_SW_EXT_PORT_TRAP_SUP = 1 << 12, + IBV_PORT_EXTENDED_SPEEDS_SUP = 1 << 14, + IBV_PORT_CM_SUP = 1 << 16, + IBV_PORT_SNMP_TUNNEL_SUP = 1 << 17, + IBV_PORT_REINIT_SUP = 1 << 18, + IBV_PORT_DEVICE_MGMT_SUP = 1 << 19, + IBV_PORT_VENDOR_CLASS = 1 << 24, + IBV_PORT_CLIENT_REG_SUP = 1 << 25, + IBV_PORT_IP_BASED_GIDS = 1 << 26, +}; + +struct ibv_port_attr { + enum ibv_port_state state; + enum ibv_mtu max_mtu; + enum ibv_mtu active_mtu; + int gid_tbl_len; + uint32_t port_cap_flags; + uint32_t max_msg_sz; + uint32_t bad_pkey_cntr; + uint32_t qkey_viol_cntr; + uint16_t pkey_tbl_len; + uint16_t lid; + uint16_t sm_lid; + uint8_t lmc; + uint8_t max_vl_num; + uint8_t sm_sl; + uint8_t subnet_timeout; + uint8_t init_type_reply; + uint8_t active_width; + uint8_t active_speed; + uint8_t phys_state; + uint8_t link_layer; + uint8_t reserved; +}; + +enum ibv_event_type { + IBV_EVENT_CQ_ERR, + IBV_EVENT_QP_FATAL, + IBV_EVENT_QP_REQ_ERR, + IBV_EVENT_QP_ACCESS_ERR, + IBV_EVENT_COMM_EST, + IBV_EVENT_SQ_DRAINED, + IBV_EVENT_PATH_MIG, + IBV_EVENT_PATH_MIG_ERR, + IBV_EVENT_DEVICE_FATAL, + IBV_EVENT_PORT_ACTIVE, + IBV_EVENT_PORT_ERR, + IBV_EVENT_LID_CHANGE, + IBV_EVENT_PKEY_CHANGE, + IBV_EVENT_SM_CHANGE, + IBV_EVENT_SRQ_ERR, + IBV_EVENT_SRQ_LIMIT_REACHED, + IBV_EVENT_QP_LAST_WQE_REACHED, + IBV_EVENT_CLIENT_REREGISTER, + IBV_EVENT_GID_CHANGE, + + /* new experimental events start here leaving enough + * room for 14 events which should be enough + */ + IBV_EXP_EVENT_DCT_KEY_VIOLATION = 32, + IBV_EXP_EVENT_DCT_ACCESS_ERR, + IBV_EXP_EVENT_DCT_REQ_ERR, +}; + +struct ibv_async_event { + union { + struct ibv_cq *cq; + struct ibv_qp *qp; + struct ibv_srq *srq; + struct ibv_exp_dct *dct; + int port_num; + /* For source compatible with Legacy API */ + uint32_t xrc_qp_num; + } element; + enum ibv_event_type event_type; +}; + +enum ibv_wc_status { + IBV_WC_SUCCESS, + IBV_WC_LOC_LEN_ERR, + IBV_WC_LOC_QP_OP_ERR, + IBV_WC_LOC_EEC_OP_ERR, + IBV_WC_LOC_PROT_ERR, + IBV_WC_WR_FLUSH_ERR, + IBV_WC_MW_BIND_ERR, + IBV_WC_BAD_RESP_ERR, + IBV_WC_LOC_ACCESS_ERR, + IBV_WC_REM_INV_REQ_ERR, + IBV_WC_REM_ACCESS_ERR, + IBV_WC_REM_OP_ERR, + IBV_WC_RETRY_EXC_ERR, + IBV_WC_RNR_RETRY_EXC_ERR, + IBV_WC_LOC_RDD_VIOL_ERR, + IBV_WC_REM_INV_RD_REQ_ERR, + IBV_WC_REM_ABORT_ERR, + IBV_WC_INV_EECN_ERR, + IBV_WC_INV_EEC_STATE_ERR, + IBV_WC_FATAL_ERR, + IBV_WC_RESP_TIMEOUT_ERR, + IBV_WC_GENERAL_ERR +}; +const char *ibv_wc_status_str(enum ibv_wc_status status); + +enum ibv_wc_opcode { + IBV_WC_SEND, + IBV_WC_RDMA_WRITE, + IBV_WC_RDMA_READ, + IBV_WC_COMP_SWAP, + IBV_WC_FETCH_ADD, + IBV_WC_BIND_MW, +/* + * Set value of IBV_WC_RECV so consumers can test if a completion is a + * receive by testing (opcode & IBV_WC_RECV). + */ + IBV_WC_RECV = 1 << 7, + IBV_WC_RECV_RDMA_WITH_IMM +}; + +enum ibv_wc_flags { + IBV_WC_GRH = 1 << 0, + IBV_WC_WITH_IMM = 1 << 1 +}; + +struct ibv_wc { + uint64_t wr_id; + enum ibv_wc_status status; + enum ibv_wc_opcode opcode; + uint32_t vendor_err; + uint32_t byte_len; + uint32_t imm_data; /* in network byte order */ + uint32_t qp_num; + uint32_t src_qp; + int wc_flags; + uint16_t pkey_index; + uint16_t slid; + uint8_t sl; + uint8_t dlid_path_bits; +}; + +enum ibv_access_flags { + IBV_ACCESS_LOCAL_WRITE = 1, + IBV_ACCESS_REMOTE_WRITE = (1<<1), + IBV_ACCESS_REMOTE_READ = (1<<2), + IBV_ACCESS_REMOTE_ATOMIC = (1<<3), + IBV_ACCESS_MW_BIND = (1<<4) +}; + +struct ibv_pd { + struct ibv_context *context; + uint32_t handle; +}; + +enum ibv_xrcd_init_attr_mask { + IBV_XRCD_INIT_ATTR_FD = 1 << 0, + IBV_XRCD_INIT_ATTR_OFLAGS = 1 << 1, + IBV_XRCD_INIT_ATTR_RESERVED = 1 << 2 +}; + +struct ibv_xrcd_init_attr { + uint32_t comp_mask; + int fd; + int oflags; +}; + +struct ibv_xrcd { + struct ibv_context *context; +}; + +enum ibv_rereg_mr_flags { + IBV_REREG_MR_CHANGE_TRANSLATION = (1 << 0), + IBV_REREG_MR_CHANGE_PD = (1 << 1), + IBV_REREG_MR_CHANGE_ACCESS = (1 << 2), + IBV_REREG_MR_KEEP_VALID = (1 << 3) +}; + +struct ibv_mr { + struct ibv_context *context; + struct ibv_pd *pd; + void *addr; + size_t length; + uint32_t handle; + uint32_t lkey; + uint32_t rkey; +}; + +enum ibv_mw_type { + IBV_MW_TYPE_1 = 1, + IBV_MW_TYPE_2 = 2 +}; + +struct ibv_mw { + struct ibv_context *context; + struct ibv_pd *pd; + uint32_t rkey; +}; + +struct ibv_global_route { + union ibv_gid dgid; + uint32_t flow_label; + uint8_t sgid_index; + uint8_t hop_limit; + uint8_t traffic_class; +}; + +struct ibv_grh { + uint32_t version_tclass_flow; + uint16_t paylen; + uint8_t next_hdr; + uint8_t hop_limit; + union ibv_gid sgid; + union ibv_gid dgid; +}; + +enum ibv_rate { + IBV_RATE_MAX = 0, + IBV_RATE_2_5_GBPS = 2, + IBV_RATE_5_GBPS = 5, + IBV_RATE_10_GBPS = 3, + IBV_RATE_20_GBPS = 6, + IBV_RATE_30_GBPS = 4, + IBV_RATE_40_GBPS = 7, + IBV_RATE_60_GBPS = 8, + IBV_RATE_80_GBPS = 9, + IBV_RATE_120_GBPS = 10, + IBV_RATE_14_GBPS = 11, + IBV_RATE_56_GBPS = 12, + IBV_RATE_112_GBPS = 13, + IBV_RATE_168_GBPS = 14, + IBV_RATE_25_GBPS = 15, + IBV_RATE_100_GBPS = 16, + IBV_RATE_200_GBPS = 17, + IBV_RATE_300_GBPS = 18 +}; + +/** + * ibv_rate_to_mult - Convert the IB rate enum to a multiple of the + * base rate of 2.5 Gbit/sec. For example, IBV_RATE_5_GBPS will be + * converted to 2, since 5 Gbit/sec is 2 * 2.5 Gbit/sec. + * @rate: rate to convert. + */ +int ibv_rate_to_mult(enum ibv_rate rate) __attribute_const; + +/** + * mult_to_ibv_rate - Convert a multiple of 2.5 Gbit/sec to an IB rate enum. + * @mult: multiple to convert. + */ +enum ibv_rate mult_to_ibv_rate(int mult) __attribute_const; + +/** + * ibv_rate_to_mbps - Convert the IB rate enum to Mbit/sec. + * For example, IBV_RATE_5_GBPS will return the value 5000. + * @rate: rate to convert. + */ +int ibv_rate_to_mbps(enum ibv_rate rate) __attribute_const; + +/** + * mbps_to_ibv_rate - Convert a Mbit/sec value to an IB rate enum. + * @mbps: value to convert. + */ +enum ibv_rate mbps_to_ibv_rate(int mbps) __attribute_const; + +struct ibv_ah_attr { + struct ibv_global_route grh; + uint16_t dlid; + uint8_t sl; + uint8_t src_path_bits; + uint8_t static_rate; + uint8_t is_global; + uint8_t port_num; +}; + +enum ibv_srq_attr_mask { + IBV_SRQ_MAX_WR = 1 << 0, + IBV_SRQ_LIMIT = 1 << 1 +}; + +struct ibv_srq_attr { + uint32_t max_wr; + uint32_t max_sge; + uint32_t srq_limit; +}; + +struct ibv_srq_init_attr { + void *srq_context; + struct ibv_srq_attr attr; +}; + +enum ibv_srq_type { + IBV_SRQT_BASIC, + IBV_SRQT_XRC +}; + +enum ibv_srq_init_attr_mask { + IBV_SRQ_INIT_ATTR_TYPE = 1 << 0, + IBV_SRQ_INIT_ATTR_PD = 1 << 1, + IBV_SRQ_INIT_ATTR_XRCD = 1 << 2, + IBV_SRQ_INIT_ATTR_CQ = 1 << 3, + IBV_SRQ_INIT_ATTR_RESERVED = 1 << 4 +}; + +struct ibv_srq_init_attr_ex { + void *srq_context; + struct ibv_srq_attr attr; + + uint32_t comp_mask; + enum ibv_srq_type srq_type; + struct ibv_pd *pd; + struct ibv_xrcd *xrcd; + struct ibv_cq *cq; +}; + +enum ibv_qp_type { + IBV_QPT_RC = 2, + IBV_QPT_UC, + IBV_QPT_UD, + /* XRC compatible code */ + IBV_QPT_XRC, + IBV_QPT_RAW_PACKET = 8, + IBV_QPT_RAW_ETH = 8, + IBV_QPT_XRC_SEND = 9, + IBV_QPT_XRC_RECV, + + /* Leave a gap for future qp types before starting with + * experimental qp types. + */ + IBV_EXP_QP_TYPE_START = 32, + IBV_EXP_QPT_DC_INI = IBV_EXP_QP_TYPE_START +}; + +struct ibv_qp_cap { + uint32_t max_send_wr; + uint32_t max_recv_wr; + uint32_t max_send_sge; + uint32_t max_recv_sge; + uint32_t max_inline_data; +}; + +struct ibv_qp_init_attr { + void *qp_context; + struct ibv_cq *send_cq; + struct ibv_cq *recv_cq; + struct ibv_srq *srq; + struct ibv_qp_cap cap; + enum ibv_qp_type qp_type; + int sq_sig_all; + /* Below is needed for backwards compatabile */ + struct ibv_xrc_domain *xrc_domain; +}; + +enum ibv_qp_init_attr_mask { + IBV_QP_INIT_ATTR_PD = 1 << 0, + IBV_QP_INIT_ATTR_XRCD = 1 << 1, + IBV_QP_INIT_ATTR_RESERVED = 1 << 2 +}; + +struct ibv_qp_init_attr_ex { + void *qp_context; + struct ibv_cq *send_cq; + struct ibv_cq *recv_cq; + struct ibv_srq *srq; + struct ibv_qp_cap cap; + enum ibv_qp_type qp_type; + int sq_sig_all; + + uint32_t comp_mask; + struct ibv_pd *pd; + struct ibv_xrcd *xrcd; +}; + +enum ibv_qp_open_attr_mask { + IBV_QP_OPEN_ATTR_NUM = 1 << 0, + IBV_QP_OPEN_ATTR_XRCD = 1 << 1, + IBV_QP_OPEN_ATTR_CONTEXT = 1 << 2, + IBV_QP_OPEN_ATTR_TYPE = 1 << 3, + IBV_QP_OPEN_ATTR_RESERVED = 1 << 4 +}; + +struct ibv_qp_open_attr { + uint32_t comp_mask; + uint32_t qp_num; + struct ibv_xrcd *xrcd; + void *qp_context; + enum ibv_qp_type qp_type; +}; + +enum ibv_qp_attr_mask { + IBV_QP_STATE = 1 << 0, + IBV_QP_CUR_STATE = 1 << 1, + IBV_QP_EN_SQD_ASYNC_NOTIFY = 1 << 2, + IBV_QP_ACCESS_FLAGS = 1 << 3, + IBV_QP_PKEY_INDEX = 1 << 4, + IBV_QP_PORT = 1 << 5, + IBV_QP_QKEY = 1 << 6, + IBV_QP_AV = 1 << 7, + IBV_QP_PATH_MTU = 1 << 8, + IBV_QP_TIMEOUT = 1 << 9, + IBV_QP_RETRY_CNT = 1 << 10, + IBV_QP_RNR_RETRY = 1 << 11, + IBV_QP_RQ_PSN = 1 << 12, + IBV_QP_MAX_QP_RD_ATOMIC = 1 << 13, + IBV_QP_ALT_PATH = 1 << 14, + IBV_QP_MIN_RNR_TIMER = 1 << 15, + IBV_QP_SQ_PSN = 1 << 16, + IBV_QP_MAX_DEST_RD_ATOMIC = 1 << 17, + IBV_QP_PATH_MIG_STATE = 1 << 18, + IBV_QP_CAP = 1 << 19, + IBV_QP_DEST_QPN = 1 << 20 +}; + +enum ibv_qp_state { + IBV_QPS_RESET, + IBV_QPS_INIT, + IBV_QPS_RTR, + IBV_QPS_RTS, + IBV_QPS_SQD, + IBV_QPS_SQE, + IBV_QPS_ERR, + IBV_QPS_UNKNOWN +}; + +enum ibv_mig_state { + IBV_MIG_MIGRATED, + IBV_MIG_REARM, + IBV_MIG_ARMED +}; + +struct ibv_qp_attr { + enum ibv_qp_state qp_state; + enum ibv_qp_state cur_qp_state; + enum ibv_mtu path_mtu; + enum ibv_mig_state path_mig_state; + uint32_t qkey; + uint32_t rq_psn; + uint32_t sq_psn; + uint32_t dest_qp_num; + int qp_access_flags; + struct ibv_qp_cap cap; + struct ibv_ah_attr ah_attr; + struct ibv_ah_attr alt_ah_attr; + uint16_t pkey_index; + uint16_t alt_pkey_index; + uint8_t en_sqd_async_notify; + uint8_t sq_draining; + uint8_t max_rd_atomic; + uint8_t max_dest_rd_atomic; + uint8_t min_rnr_timer; + uint8_t port_num; + uint8_t timeout; + uint8_t retry_cnt; + uint8_t rnr_retry; + uint8_t alt_port_num; + uint8_t alt_timeout; +}; + +enum ibv_wr_opcode { + IBV_WR_RDMA_WRITE, + IBV_WR_RDMA_WRITE_WITH_IMM, + IBV_WR_SEND, + IBV_WR_SEND_WITH_IMM, + IBV_WR_RDMA_READ, + IBV_WR_ATOMIC_CMP_AND_SWP, + IBV_WR_ATOMIC_FETCH_AND_ADD +}; + +enum ibv_send_flags { + IBV_SEND_FENCE = 1 << 0, + IBV_SEND_SIGNALED = 1 << 1, + IBV_SEND_SOLICITED = 1 << 2, + IBV_SEND_INLINE = 1 << 3 +}; + +struct ibv_sge { + uint64_t addr; + uint32_t length; + uint32_t lkey; +}; + +struct ibv_send_wr { + uint64_t wr_id; + struct ibv_send_wr *next; + struct ibv_sge *sg_list; + int num_sge; + enum ibv_wr_opcode opcode; + int send_flags; + uint32_t imm_data; /* in network byte order */ + union { + struct { + uint64_t remote_addr; + uint32_t rkey; + } rdma; + struct { + uint64_t remote_addr; + uint64_t compare_add; + uint64_t swap; + uint32_t rkey; + } atomic; + struct { + struct ibv_ah *ah; + uint32_t remote_qpn; + uint32_t remote_qkey; + } ud; + } wr; + union { + union { + struct { + uint32_t remote_srqn; + } xrc; + } qp_type; + + uint32_t xrc_remote_srq_num; + }; +}; + +struct ibv_recv_wr { + uint64_t wr_id; + struct ibv_recv_wr *next; + struct ibv_sge *sg_list; + int num_sge; +}; + +struct ibv_mw_bind { + uint64_t wr_id; + struct ibv_mr *mr; + void *addr; + size_t length; + int send_flags; + int mw_access_flags; +}; + +struct ibv_srq { + struct ibv_context *context; + void *srq_context; + struct ibv_pd *pd; + uint32_t handle; + + pthread_mutex_t mutex; + pthread_cond_t cond; + uint32_t events_completed; + + /* below are for source compatabilty with legacy XRC, + * padding based on ibv_srq_legacy. + */ + uint32_t xrc_srq_num_bin_compat_padding; + struct ibv_xrc_domain *xrc_domain_bin_compat_padding; + struct ibv_cq *xrc_cq_bin_compat_padding; + void *ibv_srq_padding; + + /* legacy fields */ + uint32_t xrc_srq_num; + struct ibv_xrc_domain *xrc_domain; + struct ibv_cq *xrc_cq; +}; + +/* Not in use in new API, needed for compilation as part of source compat layer */ +enum ibv_event_flags { + IBV_XRC_QP_EVENT_FLAG = 0x80000000, +}; + + + +struct ibv_qp { + struct ibv_context *context; + void *qp_context; + struct ibv_pd *pd; + struct ibv_cq *send_cq; + struct ibv_cq *recv_cq; + struct ibv_srq *srq; + uint32_t handle; + uint32_t qp_num; + enum ibv_qp_state state; + enum ibv_qp_type qp_type; + + pthread_mutex_t mutex; + pthread_cond_t cond; + uint32_t events_completed; +}; + +struct ibv_comp_channel { + struct ibv_context *context; + int fd; + int refcnt; +}; + +struct ibv_cq { + struct ibv_context *context; + struct ibv_comp_channel *channel; + void *cq_context; + uint32_t handle; + int cqe; + + pthread_mutex_t mutex; + pthread_cond_t cond; + uint32_t comp_events_completed; + uint32_t async_events_completed; +}; + +struct ibv_ah { + struct ibv_context *context; + struct ibv_pd *pd; + uint32_t handle; +}; + +enum ibv_flow_flags { + IBV_FLOW_ATTR_FLAGS_ALLOW_LOOP_BACK = 1, + IBV_FLOW_ATTR_FLAGS_DONT_TRAP = 1 << 1, +}; + +enum ibv_flow_attr_type { + /* steering according to rule specifications */ + IBV_FLOW_ATTR_NORMAL = 0x0, + /* default unicast and multicast rule - + * receive all Eth traffic which isn't steered to any QP + */ + IBV_FLOW_ATTR_ALL_DEFAULT = 0x1, + /* default multicast rule - + * receive all Eth multicast traffic which isn't steered to any QP + */ + IBV_FLOW_ATTR_MC_DEFAULT = 0x2, +}; + +enum ibv_flow_spec_type { + IBV_FLOW_SPEC_ETH = 0x20, + IBV_FLOW_SPEC_IPV4 = 0x30, + IBV_FLOW_SPEC_TCP = 0x40, + IBV_FLOW_SPEC_UDP = 0x41, +}; + +struct ibv_flow_eth_filter { + uint8_t dst_mac[6]; + uint8_t src_mac[6]; + uint16_t ether_type; + /* + * same layout as 802.1q: prio 3, cfi 1, vlan id 12 + */ + uint16_t vlan_tag; +}; + +struct ibv_flow_spec_eth { + enum ibv_flow_spec_type type; + uint16_t size; + struct ibv_flow_eth_filter val; + struct ibv_flow_eth_filter mask; +}; + +struct ibv_flow_ipv4_filter { + uint32_t src_ip; + uint32_t dst_ip; +}; + +struct ibv_flow_spec_ipv4 { + enum ibv_flow_spec_type type; + uint16_t size; + struct ibv_flow_ipv4_filter val; + struct ibv_flow_ipv4_filter mask; +}; + +struct ibv_flow_tcp_udp_filter { + uint16_t dst_port; + uint16_t src_port; +}; + +struct ibv_flow_spec_tcp_udp { + enum ibv_flow_spec_type type; + uint16_t size; + struct ibv_flow_tcp_udp_filter val; + struct ibv_flow_tcp_udp_filter mask; +}; + +struct ibv_flow_spec { + union { + struct { + enum ibv_flow_spec_type type; + uint16_t size; + } hdr; + struct ibv_flow_spec_eth eth; + struct ibv_flow_spec_ipv4 ipv4; + struct ibv_flow_spec_tcp_udp tcp_udp; + }; +}; + +struct ibv_flow_attr { + uint32_t comp_mask; + enum ibv_flow_attr_type type; + uint16_t size; + uint16_t priority; + uint8_t num_of_specs; + uint8_t port; + uint32_t flags; + /* Following are the optional layers according to user request + * struct ibv_flow_spec_xxx [L2] + * struct ibv_flow_spec_yyy [L3/L4] + */ +}; + +struct ibv_flow { + uint32_t comp_mask; + struct ibv_context *context; + uint32_t handle; +}; + +struct ibv_device; +struct ibv_context; + +struct ibv_device_ops { + struct ibv_context * (*alloc_context)(struct ibv_device *device, int cmd_fd); + void (*free_context)(struct ibv_context *context); +}; + +enum { + IBV_SYSFS_NAME_MAX = 64, + IBV_SYSFS_PATH_MAX = 256 +}; + +struct ibv_device { + struct ibv_device_ops ops; + enum ibv_node_type node_type; + enum ibv_transport_type transport_type; + /* Name of underlying kernel IB device, eg "mthca0" */ + char name[IBV_SYSFS_NAME_MAX]; + /* Name of uverbs device, eg "uverbs0" */ + char dev_name[IBV_SYSFS_NAME_MAX]; + /* Path to infiniband_verbs class device in sysfs */ + char dev_path[IBV_SYSFS_PATH_MAX]; + /* Path to infiniband class device in sysfs */ + char ibdev_path[IBV_SYSFS_PATH_MAX]; +}; + +struct verbs_device { + struct ibv_device device; /* Must be first */ + size_t sz; + size_t size_of_context; + int (*init_context)(struct verbs_device *device, + struct ibv_context *ctx, int cmd_fd); + void (*uninit_context)(struct verbs_device *device, + struct ibv_context *ctx); + /* future fields added here */ +}; + +struct ibv_context_ops { + int (*query_device)(struct ibv_context *context, + struct ibv_device_attr *device_attr); + int (*query_port)(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr *port_attr); + struct ibv_pd * (*alloc_pd)(struct ibv_context *context); + int (*dealloc_pd)(struct ibv_pd *pd); + struct ibv_mr * (*reg_mr)(struct ibv_pd *pd, void *addr, size_t length, + int access); + struct ibv_mr * (*rereg_mr)(struct ibv_mr *mr, + int flags, + struct ibv_pd *pd, void *addr, + size_t length, + int access); + int (*dereg_mr)(struct ibv_mr *mr); + struct ibv_mw * (*alloc_mw)(struct ibv_pd *pd, enum ibv_mw_type type); + int (*bind_mw)(struct ibv_qp *qp, struct ibv_mw *mw, + struct ibv_mw_bind *mw_bind); + int (*dealloc_mw)(struct ibv_mw *mw); + struct ibv_cq * (*create_cq)(struct ibv_context *context, int cqe, + struct ibv_comp_channel *channel, + int comp_vector); + int (*poll_cq)(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc); + int (*req_notify_cq)(struct ibv_cq *cq, int solicited_only); + void (*cq_event)(struct ibv_cq *cq); + int (*resize_cq)(struct ibv_cq *cq, int cqe); + int (*destroy_cq)(struct ibv_cq *cq); + struct ibv_srq * (*create_srq)(struct ibv_pd *pd, + struct ibv_srq_init_attr *srq_init_attr); + int (*modify_srq)(struct ibv_srq *srq, + struct ibv_srq_attr *srq_attr, + int srq_attr_mask); + int (*query_srq)(struct ibv_srq *srq, + struct ibv_srq_attr *srq_attr); + int (*destroy_srq)(struct ibv_srq *srq); + int (*post_srq_recv)(struct ibv_srq *srq, + struct ibv_recv_wr *recv_wr, + struct ibv_recv_wr **bad_recv_wr); + struct ibv_qp * (*create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); + int (*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask, + struct ibv_qp_init_attr *init_attr); + int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask); + int (*destroy_qp)(struct ibv_qp *qp); + int (*post_send)(struct ibv_qp *qp, struct ibv_send_wr *wr, + struct ibv_send_wr **bad_wr); + int (*post_recv)(struct ibv_qp *qp, struct ibv_recv_wr *wr, + struct ibv_recv_wr **bad_wr); + struct ibv_ah * (*create_ah)(struct ibv_pd *pd, struct ibv_ah_attr *attr); + int (*destroy_ah)(struct ibv_ah *ah); + int (*attach_mcast)(struct ibv_qp *qp, const union ibv_gid *gid, + uint16_t lid); + int (*detach_mcast)(struct ibv_qp *qp, const union ibv_gid *gid, + uint16_t lid); + void (*async_event)(struct ibv_async_event *event); +}; + +struct ibv_context { + struct ibv_device *device; + struct ibv_context_ops ops; + int cmd_fd; + int async_fd; + int num_comp_vectors; + pthread_mutex_t mutex; + void *abi_compat; +}; + +enum verbs_context_mask { + VERBS_CONTEXT_XRCD = (uint64_t)1 << 0, + VERBS_CONTEXT_SRQ = (uint64_t)1 << 1, + VERBS_CONTEXT_QP = (uint64_t)1 << 2, + VERBS_CONTEXT_RESERVED = (uint64_t)1 << 3, + VERBS_CONTEXT_EXP = (uint64_t)1 << 62 +}; + +struct verbs_context { + /* "grows up" - new fields go here */ + int (*_reserved_2) (void); + int (*destroy_flow) (struct ibv_flow *flow); + int (*_reserved_1) (void); + struct ibv_flow * (*create_flow) (struct ibv_qp *qp, + struct ibv_flow_attr *flow_attr); + struct ibv_qp * (*open_qp)(struct ibv_context *context, + struct ibv_qp_open_attr *attr); + struct ibv_qp * (*create_qp_ex)(struct ibv_context *context, + struct ibv_qp_init_attr_ex *qp_init_attr_ex); + int (*get_srq_num)(struct ibv_srq *srq, uint32_t *srq_num); + struct ibv_srq * (*create_srq_ex)(struct ibv_context *context, + struct ibv_srq_init_attr_ex *srq_init_attr_ex); + struct ibv_xrcd * (*open_xrcd)(struct ibv_context *context, + struct ibv_xrcd_init_attr *xrcd_init_attr); + int (*close_xrcd)(struct ibv_xrcd *xrcd); + uint64_t has_comp_mask; + size_t sz; /* Must be immediately before struct ibv_context */ + struct ibv_context context;/* Must be last field in the struct */ +}; + +static inline struct verbs_context *verbs_get_ctx(struct ibv_context *ctx) +{ + return (!ctx || (ctx->abi_compat != __VERBS_ABI_IS_EXTENDED)) ? + NULL : container_of(ctx, struct verbs_context, context); +} + +#define verbs_get_ctx_op(ctx, op) ({ \ + struct verbs_context *_vctx = verbs_get_ctx(ctx); \ + (!_vctx || (_vctx->sz < sizeof(*_vctx) - offsetof(struct verbs_context, op)) || \ + !_vctx->op) ? NULL : _vctx; }) + +#define verbs_set_ctx_op(_vctx, op, ptr) ({ \ + struct verbs_context *vctx = _vctx; \ + if (vctx && (vctx->sz >= sizeof(*vctx) - offsetof(struct verbs_context, op))) \ + vctx->op = ptr; }) + +static inline struct verbs_device *verbs_get_device(struct ibv_device *dev) +{ + return (dev->ops.alloc_context) ? + NULL : container_of(dev, struct verbs_device, device); +} + +/** + * ibv_get_device_list - Get list of IB devices currently available + * @num_devices: optional. if non-NULL, set to the number of devices + * returned in the array. + * + * Return a NULL-terminated array of IB devices. The array can be + * released with ibv_free_device_list(). + */ +struct ibv_device **ibv_get_device_list(int *num_devices); + +/** + * ibv_free_device_list - Free list from ibv_get_device_list() + * + * Free an array of devices returned from ibv_get_device_list(). Once + * the array is freed, pointers to devices that were not opened with + * ibv_open_device() are no longer valid. Client code must open all + * devices it intends to use before calling ibv_free_device_list(). + */ +void ibv_free_device_list(struct ibv_device **list); + +/** + * ibv_get_device_name - Return kernel device name + */ +const char *ibv_get_device_name(struct ibv_device *device); + +/** + * ibv_get_device_guid - Return device's node GUID + */ +uint64_t ibv_get_device_guid(struct ibv_device *device); + +/** + * ibv_open_device - Initialize device for use + */ +struct ibv_context *ibv_open_device(struct ibv_device *device); + +/** + * ibv_close_device - Release device + */ +int ibv_close_device(struct ibv_context *context); + +/** + * ibv_get_async_event - Get next async event + * @event: Pointer to use to return async event + * + * All async events returned by ibv_get_async_event() must eventually + * be acknowledged with ibv_ack_async_event(). + */ +int ibv_get_async_event(struct ibv_context *context, + struct ibv_async_event *event); + +/** + * ibv_ack_async_event - Acknowledge an async event + * @event: Event to be acknowledged. + * + * All async events which are returned by ibv_get_async_event() must + * be acknowledged. To avoid races, destroying an object (CQ, SRQ or + * QP) will wait for all affiliated events to be acknowledged, so + * there should be a one-to-one correspondence between acks and + * successful gets. + */ +void ibv_ack_async_event(struct ibv_async_event *event); + +/** + * ibv_query_device - Get device properties + */ +int ibv_query_device(struct ibv_context *context, + struct ibv_device_attr *device_attr); + +/** + * ibv_query_port - Get port properties + */ +int ibv_query_port(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr *port_attr); + +static inline int ___ibv_query_port(struct ibv_context *context, + uint8_t port_num, + struct ibv_port_attr *port_attr) +{ + /* For compatibility when running with old libibverbs */ + port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED; + port_attr->reserved = 0; + + return ibv_query_port(context, port_num, port_attr); +} + +#define ibv_query_port(context, port_num, port_attr) \ + ___ibv_query_port(context, port_num, port_attr) + +/** + * ibv_query_gid - Get a GID table entry + */ +int ibv_query_gid(struct ibv_context *context, uint8_t port_num, + int index, union ibv_gid *gid); + +/** + * ibv_query_pkey - Get a P_Key table entry + */ +int ibv_query_pkey(struct ibv_context *context, uint8_t port_num, + int index, uint16_t *pkey); + +/** + * ibv_alloc_pd - Allocate a protection domain + */ +struct ibv_pd *ibv_alloc_pd(struct ibv_context *context); + +/** + * ibv_dealloc_pd - Free a protection domain + */ +int ibv_dealloc_pd(struct ibv_pd *pd); + +static inline struct ibv_flow *ibv_create_flow(struct ibv_qp *qp, + struct ibv_flow_attr *flow) +{ + struct verbs_context *vctx = verbs_get_ctx_op(qp->context, + create_flow); + if (!vctx) + return NULL; + + return vctx->create_flow(qp, flow); +} + +static inline int ibv_destroy_flow(struct ibv_flow *flow_id) +{ + struct verbs_context *vctx = verbs_get_ctx_op(flow_id->context, + destroy_flow); + if (!vctx) + return -ENOSYS; + return vctx->destroy_flow(flow_id); +} + +/** + * ibv_open_xrcd - Open an extended connection domain + */ +static inline struct ibv_xrcd * +ibv_open_xrcd(struct ibv_context *context, struct ibv_xrcd_init_attr *xrcd_init_attr) +{ + struct verbs_context *vctx = verbs_get_ctx_op(context, open_xrcd); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + return vctx->open_xrcd(context, xrcd_init_attr); +} + +/** + * ibv_close_xrcd - Close an extended connection domain + */ +static inline int ibv_close_xrcd(struct ibv_xrcd *xrcd) +{ + struct verbs_context *vctx = verbs_get_ctx(xrcd->context); + return vctx->close_xrcd(xrcd); +} + +/** + * ibv_reg_mr - Register a memory region + */ +struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr, + size_t length, int access); + +/** + * ibv_dereg_mr - Deregister a memory region + */ +int ibv_dereg_mr(struct ibv_mr *mr); + +/** + * ibv_alloc_mw - Allocate a memory window + */ +static inline struct ibv_mw *ibv_alloc_mw(struct ibv_pd *pd, + enum ibv_mw_type type) +{ + if (!pd->context->ops.alloc_mw) { + errno = ENOSYS; + return NULL; + } + + struct ibv_mw *mw = pd->context->ops.alloc_mw(pd, type); + if (mw) { + mw->context = pd->context; + mw->pd = pd; + } + return mw; +} + +/** + * ibv_dealloc_mw - Free a memory window + */ +static inline int ibv_dealloc_mw(struct ibv_mw *mw) +{ + return mw->context->ops.dealloc_mw(mw); +} + +/** + * ibv_inc_rkey - increase the 8 lsb in the given rkey + */ +static inline uint32_t ibv_inc_rkey(uint32_t rkey) +{ + const uint32_t mask = 0x000000ff; + uint8_t newtag = (uint8_t) ((rkey + 1) & mask); + return (rkey & ~mask) | newtag; +} + +/** + * ibv_create_comp_channel - Create a completion event channel + */ +struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context); + +/** + * ibv_destroy_comp_channel - Destroy a completion event channel + */ +int ibv_destroy_comp_channel(struct ibv_comp_channel *channel); + +/** + * ibv_create_cq - Create a completion queue + * @context - Context CQ will be attached to + * @cqe - Minimum number of entries required for CQ + * @cq_context - Consumer-supplied context returned for completion events + * @channel - Completion channel where completion events will be queued. + * May be NULL if completion events will not be used. + * @comp_vector - Completion vector used to signal completion events. + * Must be >= 0 and < context->num_comp_vectors. + */ +struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe, + void *cq_context, + struct ibv_comp_channel *channel, + int comp_vector); + +/** + * ibv_resize_cq - Modifies the capacity of the CQ. + * @cq: The CQ to resize. + * @cqe: The minimum size of the CQ. + * + * Users can examine the cq structure to determine the actual CQ size. + */ +int ibv_resize_cq(struct ibv_cq *cq, int cqe); + +/** + * ibv_destroy_cq - Destroy a completion queue + */ +int ibv_destroy_cq(struct ibv_cq *cq); + +/** + * ibv_get_cq_event - Read next CQ event + * @channel: Channel to get next event from. + * @cq: Used to return pointer to CQ. + * @cq_context: Used to return consumer-supplied CQ context. + * + * All completion events returned by ibv_get_cq_event() must + * eventually be acknowledged with ibv_ack_cq_events(). + */ +int ibv_get_cq_event(struct ibv_comp_channel *channel, + struct ibv_cq **cq, void **cq_context); + +/** + * ibv_ack_cq_events - Acknowledge CQ completion events + * @cq: CQ to acknowledge events for + * @nevents: Number of events to acknowledge. + * + * All completion events which are returned by ibv_get_cq_event() must + * be acknowledged. To avoid races, ibv_destroy_cq() will wait for + * all completion events to be acknowledged, so there should be a + * one-to-one correspondence between acks and successful gets. An + * application may accumulate multiple completion events and + * acknowledge them in a single call to ibv_ack_cq_events() by passing + * the number of events to ack in @nevents. + */ +void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents); + +/** + * ibv_poll_cq - Poll a CQ for work completions + * @cq:the CQ being polled + * @num_entries:maximum number of completions to return + * @wc:array of at least @num_entries of &struct ibv_wc where completions + * will be returned + * + * Poll a CQ for (possibly multiple) completions. If the return value + * is < 0, an error occurred. If the return value is >= 0, it is the + * number of completions returned. If the return value is + * non-negative and strictly less than num_entries, then the CQ was + * emptied. + */ +static inline int ibv_poll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc) +{ + return cq->context->ops.poll_cq(cq, num_entries, wc); +} + +/** + * ibv_req_notify_cq - Request completion notification on a CQ. An + * event will be added to the completion channel associated with the + * CQ when an entry is added to the CQ. + * @cq: The completion queue to request notification for. + * @solicited_only: If non-zero, an event will be generated only for + * the next solicited CQ entry. If zero, any CQ entry, solicited or + * not, will generate an event. + */ +static inline int ibv_req_notify_cq(struct ibv_cq *cq, int solicited_only) +{ + return cq->context->ops.req_notify_cq(cq, solicited_only); +} + +/** + * ibv_create_srq - Creates a SRQ associated with the specified protection + * domain. + * @pd: The protection domain associated with the SRQ. + * @srq_init_attr: A list of initial attributes required to create the SRQ. + * + * srq_attr->max_wr and srq_attr->max_sge are read the determine the + * requested size of the SRQ, and set to the actual values allocated + * on return. If ibv_create_srq() succeeds, then max_wr and max_sge + * will always be at least as large as the requested values. + */ +struct ibv_srq *ibv_create_srq(struct ibv_pd *pd, + struct ibv_srq_init_attr *srq_init_attr); + +static inline struct ibv_srq * +ibv_create_srq_ex(struct ibv_context *context, + struct ibv_srq_init_attr_ex *srq_init_attr_ex) +{ + struct verbs_context *vctx; + uint32_t mask = srq_init_attr_ex->comp_mask; + + if (!(mask & ~(IBV_SRQ_INIT_ATTR_PD | IBV_SRQ_INIT_ATTR_TYPE)) && + (mask & IBV_SRQ_INIT_ATTR_PD) && + (!(mask & IBV_SRQ_INIT_ATTR_TYPE) || + (srq_init_attr_ex->srq_type == IBV_SRQT_BASIC))) + return ibv_create_srq(srq_init_attr_ex->pd, + (struct ibv_srq_init_attr *) srq_init_attr_ex); + + vctx = verbs_get_ctx_op(context, create_srq_ex); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + return vctx->create_srq_ex(context, srq_init_attr_ex); +} + +/** + * ibv_modify_srq - Modifies the attributes for the specified SRQ. + * @srq: The SRQ to modify. + * @srq_attr: On input, specifies the SRQ attributes to modify. On output, + * the current values of selected SRQ attributes are returned. + * @srq_attr_mask: A bit-mask used to specify which attributes of the SRQ + * are being modified. + * + * The mask may contain IBV_SRQ_MAX_WR to resize the SRQ and/or + * IBV_SRQ_LIMIT to set the SRQ's limit and request notification when + * the number of receives queued drops below the limit. + */ +int ibv_modify_srq(struct ibv_srq *srq, + struct ibv_srq_attr *srq_attr, + int srq_attr_mask); + +/** + * ibv_query_srq - Returns the attribute list and current values for the + * specified SRQ. + * @srq: The SRQ to query. + * @srq_attr: The attributes of the specified SRQ. + */ +int ibv_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr); + +static inline int ibv_get_srq_num(struct ibv_srq *srq, uint32_t *srq_num) +{ + struct verbs_context *vctx = verbs_get_ctx_op(srq->context, get_srq_num); + + if (!vctx) + return ENOSYS; + + return vctx->get_srq_num(srq, srq_num); +} + +/** + * ibv_destroy_srq - Destroys the specified SRQ. + * @srq: The SRQ to destroy. + */ +int ibv_destroy_srq(struct ibv_srq *srq); + +/** + * ibv_post_srq_recv - Posts a list of work requests to the specified SRQ. + * @srq: The SRQ to post the work request on. + * @recv_wr: A list of work requests to post on the receive queue. + * @bad_recv_wr: On an immediate failure, this parameter will reference + * the work request that failed to be posted on the QP. + */ +static inline int ibv_post_srq_recv(struct ibv_srq *srq, + struct ibv_recv_wr *recv_wr, + struct ibv_recv_wr **bad_recv_wr) +{ + return srq->context->ops.post_srq_recv(srq, recv_wr, bad_recv_wr); +} + +/** + * ibv_create_qp - Create a queue pair. + */ +struct ibv_qp *ibv_create_qp(struct ibv_pd *pd, + struct ibv_qp_init_attr *qp_init_attr); + +static inline struct ibv_qp * +ibv_create_qp_ex(struct ibv_context *context, struct ibv_qp_init_attr_ex *qp_init_attr_ex) +{ + struct verbs_context *vctx; + uint32_t mask = qp_init_attr_ex->comp_mask; + + if (mask == IBV_QP_INIT_ATTR_PD) + return ibv_create_qp(qp_init_attr_ex->pd, + (struct ibv_qp_init_attr *) qp_init_attr_ex); + + vctx = verbs_get_ctx_op(context, create_qp_ex); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + return vctx->create_qp_ex(context, qp_init_attr_ex); +} + +/** + * ibv_open_qp - Open a shareable queue pair. + */ +static inline struct ibv_qp * +ibv_open_qp(struct ibv_context *context, struct ibv_qp_open_attr *qp_open_attr) +{ + struct verbs_context *vctx = verbs_get_ctx_op(context, open_qp); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + return vctx->open_qp(context, qp_open_attr); +} + +/** + * ibv_modify_qp - Modify a queue pair. + */ +int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask); + +/** + * ibv_query_qp - Returns the attribute list and current values for the + * specified QP. + * @qp: The QP to query. + * @attr: The attributes of the specified QP. + * @attr_mask: A bit-mask used to select specific attributes to query. + * @init_attr: Additional attributes of the selected QP. + * + * The qp_attr_mask may be used to limit the query to gathering only the + * selected attributes. + */ +int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask, + struct ibv_qp_init_attr *init_attr); + +/** + * ibv_destroy_qp - Destroy a queue pair. + */ +int ibv_destroy_qp(struct ibv_qp *qp); + +/** + * ibv_post_send - Post a list of work requests to a send queue. + * + * If IBV_SEND_INLINE flag is set, the data buffers can be reused + * immediately after the call returns. + */ +static inline int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr, + struct ibv_send_wr **bad_wr) +{ + return qp->context->ops.post_send(qp, wr, bad_wr); +} + +/** + * ibv_post_recv - Post a list of work requests to a receive queue. + */ +static inline int ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr, + struct ibv_recv_wr **bad_wr) +{ + return qp->context->ops.post_recv(qp, wr, bad_wr); +} + +/** + * ibv_create_ah - Create an address handle. + */ +struct ibv_ah *ibv_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr); + +/** + * ibv_init_ah_from_wc - Initializes address handle attributes from a + * work completion. + * @context: Device context on which the received message arrived. + * @port_num: Port on which the received message arrived. + * @wc: Work completion associated with the received message. + * @grh: References the received global route header. This parameter is + * ignored unless the work completion indicates that the GRH is valid. + * @ah_attr: Returned attributes that can be used when creating an address + * handle for replying to the message. + */ +int ibv_init_ah_from_wc(struct ibv_context *context, uint8_t port_num, + struct ibv_wc *wc, struct ibv_grh *grh, + struct ibv_ah_attr *ah_attr); + +/** + * ibv_create_ah_from_wc - Creates an address handle associated with the + * sender of the specified work completion. + * @pd: The protection domain associated with the address handle. + * @wc: Work completion information associated with a received message. + * @grh: References the received global route header. This parameter is + * ignored unless the work completion indicates that the GRH is valid. + * @port_num: The outbound port number to associate with the address. + * + * The address handle is used to reference a local or global destination + * in all UD QP post sends. + */ +struct ibv_ah *ibv_create_ah_from_wc(struct ibv_pd *pd, struct ibv_wc *wc, + struct ibv_grh *grh, uint8_t port_num); + +/** + * ibv_destroy_ah - Destroy an address handle. + */ +int ibv_destroy_ah(struct ibv_ah *ah); + +/** + * ibv_attach_mcast - Attaches the specified QP to a multicast group. + * @qp: QP to attach to the multicast group. The QP must be a UD QP. + * @gid: Multicast group GID. + * @lid: Multicast group LID in host byte order. + * + * In order to route multicast packets correctly, subnet + * administration must have created the multicast group and configured + * the fabric appropriately. The port associated with the specified + * QP must also be a member of the multicast group. + */ +int ibv_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); + +/** + * ibv_detach_mcast - Detaches the specified QP from a multicast group. + * @qp: QP to detach from the multicast group. + * @gid: Multicast group GID. + * @lid: Multicast group LID in host byte order. + */ +int ibv_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); + +/** + * ibv_fork_init - Prepare data structures so that fork() may be used + * safely. If this function is not called or returns a non-zero + * status, then libibverbs data structures are not fork()-safe and the + * effect of an application calling fork() is undefined. + */ +int ibv_fork_init(void); + +/** + * ibv_node_type_str - Return string describing node_type enum value + */ +const char *ibv_node_type_str(enum ibv_node_type node_type); + +/** + * ibv_port_state_str - Return string describing port_state enum value + */ +const char *ibv_port_state_str(enum ibv_port_state port_state); + +/** + * ibv_event_type_str - Return string describing event_type enum value + */ +const char *ibv_event_type_str(enum ibv_event_type event); + +END_C_DECLS + +# undef __attribute_const + +#include + +#endif /* INFINIBAND_VERBS_H */ diff --git a/external_libs/ibverbs/include/infiniband/verbs_exp.h b/external_libs/ibverbs/include/infiniband/verbs_exp.h new file mode 100644 index 00000000..ae94deb8 --- /dev/null +++ b/external_libs/ibverbs/include/infiniband/verbs_exp.h @@ -0,0 +1,3585 @@ +/* + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. + * Copyright (c) 2004, 2011-2012 Intel Corporation. All rights reserved. + * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2005 PathScale, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef INFINIBAND_VERBS_EXP_H +#define INFINIBAND_VERBS_EXP_H + +#include +#include +#include + +#if __GNUC__ >= 3 +# define __attribute_const __attribute__((const)) +#else +# define __attribute_const +#endif + +BEGIN_C_DECLS + +#define IBV_EXP_RET_ON_INVALID_COMP_MASK(val, valid_mask, ret) \ + if ((val) > (valid_mask)) { \ + fprintf(stderr, "%s: invalid comp_mask !!! (comp_mask = 0x%x valid_mask = 0x%x)\n", \ + __FUNCTION__, val, valid_mask); \ + errno = EINVAL; \ + return ret; \ + } + +#define IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(val, valid_mask) \ + IBV_EXP_RET_ON_INVALID_COMP_MASK(val, valid_mask, NULL) + +#define IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(val, valid_mask) \ + IBV_EXP_RET_ON_INVALID_COMP_MASK(val, valid_mask, EINVAL) + + +#define IBV_EXP_IMPLICIT_MR_SIZE (~((size_t)0)) + +enum ibv_exp_func_name { + IBV_EXP_POST_SEND_FUNC, + IBV_EXP_POLL_CQ_FUNC, + IBV_POST_SEND_FUNC, + IBV_POLL_CQ_FUNC, + IBV_POST_RECV_FUNC +}; + +enum ibv_exp_start_values { + IBV_EXP_START_ENUM = 0x40, + IBV_EXP_START_FLAG_LOC = 0x20, + IBV_EXP_START_FLAG = (1ULL << IBV_EXP_START_FLAG_LOC), +}; + +/* + * Capabilities for exp_atomic_cap field in ibv_exp_device_attr struct + */ +enum ibv_exp_atomic_cap { + IBV_EXP_ATOMIC_NONE = IBV_ATOMIC_NONE, + IBV_EXP_ATOMIC_HCA = IBV_ATOMIC_HCA, + IBV_EXP_ATOMIC_GLOB = IBV_ATOMIC_GLOB, + + IBV_EXP_ATOMIC_HCA_REPLY_BE = IBV_EXP_START_ENUM /* HOST is LE and atomic reply is BE */ +}; + +/* + * Flags for exp_device_cap_flags field in ibv_exp_device_attr struct + */ +enum ibv_exp_device_cap_flags { + IBV_EXP_DEVICE_RESIZE_MAX_WR = IBV_DEVICE_RESIZE_MAX_WR, + IBV_EXP_DEVICE_BAD_PKEY_CNTR = IBV_DEVICE_BAD_PKEY_CNTR, + IBV_EXP_DEVICE_BAD_QKEY_CNTR = IBV_DEVICE_BAD_QKEY_CNTR, + IBV_EXP_DEVICE_RAW_MULTI = IBV_DEVICE_RAW_MULTI, + IBV_EXP_DEVICE_AUTO_PATH_MIG = IBV_DEVICE_AUTO_PATH_MIG, + IBV_EXP_DEVICE_CHANGE_PHY_PORT = IBV_DEVICE_CHANGE_PHY_PORT, + IBV_EXP_DEVICE_UD_AV_PORT_ENFORCE = IBV_DEVICE_UD_AV_PORT_ENFORCE, + IBV_EXP_DEVICE_CURR_QP_STATE_MOD = IBV_DEVICE_CURR_QP_STATE_MOD, + IBV_EXP_DEVICE_SHUTDOWN_PORT = IBV_DEVICE_SHUTDOWN_PORT, + IBV_EXP_DEVICE_INIT_TYPE = IBV_DEVICE_INIT_TYPE, + IBV_EXP_DEVICE_PORT_ACTIVE_EVENT = IBV_DEVICE_PORT_ACTIVE_EVENT, + IBV_EXP_DEVICE_SYS_IMAGE_GUID = IBV_DEVICE_SYS_IMAGE_GUID, + IBV_EXP_DEVICE_RC_RNR_NAK_GEN = IBV_DEVICE_RC_RNR_NAK_GEN, + IBV_EXP_DEVICE_SRQ_RESIZE = IBV_DEVICE_SRQ_RESIZE, + IBV_EXP_DEVICE_N_NOTIFY_CQ = IBV_DEVICE_N_NOTIFY_CQ, + IBV_EXP_DEVICE_XRC = IBV_DEVICE_XRC, + + IBV_EXP_DEVICE_DC_TRANSPORT = (IBV_EXP_START_FLAG << 0), + IBV_EXP_DEVICE_QPG = (IBV_EXP_START_FLAG << 1), + IBV_EXP_DEVICE_UD_RSS = (IBV_EXP_START_FLAG << 2), + IBV_EXP_DEVICE_UD_TSS = (IBV_EXP_START_FLAG << 3), + IBV_EXP_DEVICE_EXT_ATOMICS = (IBV_EXP_START_FLAG << 4), + IBV_EXP_DEVICE_NOP = (IBV_EXP_START_FLAG << 5), + IBV_EXP_DEVICE_UMR = (IBV_EXP_START_FLAG << 6), + IBV_EXP_DEVICE_ODP = (IBV_EXP_START_FLAG << 7), + IBV_EXP_DEVICE_VXLAN_SUPPORT = (IBV_EXP_START_FLAG << 10), + IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT = (IBV_EXP_START_FLAG << 11), + IBV_EXP_DEVICE_RX_CSUM_IP_PKT = (IBV_EXP_START_FLAG << 12), + IBV_EXP_DEVICE_EC_OFFLOAD = (IBV_EXP_START_FLAG << 13), + IBV_EXP_DEVICE_EXT_MASKED_ATOMICS = (IBV_EXP_START_FLAG << 14), + IBV_EXP_DEVICE_RX_TCP_UDP_PKT_TYPE = (IBV_EXP_START_FLAG << 15), + IBV_EXP_DEVICE_SCATTER_FCS = (IBV_EXP_START_FLAG << 16), + IBV_EXP_DEVICE_MEM_WINDOW = (IBV_EXP_START_FLAG << 17), + IBV_EXP_DEVICE_MEM_MGT_EXTENSIONS = (IBV_EXP_START_FLAG << 21), + IBV_EXP_DEVICE_DC_INFO = (IBV_EXP_START_FLAG << 22), + /* Jumping to 23 as of next capability in include/rdma/ib_verbs.h */ + IBV_EXP_DEVICE_MW_TYPE_2A = (IBV_EXP_START_FLAG << 23), + IBV_EXP_DEVICE_MW_TYPE_2B = (IBV_EXP_START_FLAG << 24), + IBV_EXP_DEVICE_CROSS_CHANNEL = (IBV_EXP_START_FLAG << 28), + IBV_EXP_DEVICE_MANAGED_FLOW_STEERING = (IBV_EXP_START_FLAG << 29), + IBV_EXP_DEVICE_MR_ALLOCATE = (IBV_EXP_START_FLAG << 30), + IBV_EXP_DEVICE_SHARED_MR = (IBV_EXP_START_FLAG << 31), +}; + +/* + * Flags for ibv_exp_device_attr struct comp_mask. + */ +enum ibv_exp_device_attr_comp_mask { + IBV_EXP_DEVICE_ATTR_CALC_CAP = (1 << 0), + IBV_EXP_DEVICE_ATTR_WITH_TIMESTAMP_MASK = (1 << 1), + IBV_EXP_DEVICE_ATTR_WITH_HCA_CORE_CLOCK = (1 << 2), + IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS = (1 << 3), + IBV_EXP_DEVICE_DC_RD_REQ = (1 << 4), + IBV_EXP_DEVICE_DC_RD_RES = (1 << 5), + IBV_EXP_DEVICE_ATTR_INLINE_RECV_SZ = (1 << 6), + IBV_EXP_DEVICE_ATTR_RSS_TBL_SZ = (1 << 7), + IBV_EXP_DEVICE_ATTR_EXT_ATOMIC_ARGS = (1 << 8), + IBV_EXP_DEVICE_ATTR_UMR = (1 << 9), + IBV_EXP_DEVICE_ATTR_ODP = (1 << 10), + IBV_EXP_DEVICE_ATTR_MAX_DCT = (1 << 11), + IBV_EXP_DEVICE_ATTR_MAX_CTX_RES_DOMAIN = (1 << 12), + IBV_EXP_DEVICE_ATTR_RX_HASH = (1 << 13), + IBV_EXP_DEVICE_ATTR_MAX_WQ_TYPE_RQ = (1 << 14), + IBV_EXP_DEVICE_ATTR_MAX_DEVICE_CTX = (1 << 15), + IBV_EXP_DEVICE_ATTR_MP_RQ = (1 << 16), + IBV_EXP_DEVICE_ATTR_VLAN_OFFLOADS = (1 << 17), + IBV_EXP_DEVICE_ATTR_EC_CAPS = (1 << 18), + IBV_EXP_DEVICE_ATTR_MASKED_ATOMICS = (1 << 19), + IBV_EXP_DEVICE_ATTR_RX_PAD_END_ALIGN = (1 << 20), + IBV_EXP_DEVICE_ATTR_TSO_CAPS = (1 << 21), + IBV_EXP_DEVICE_ATTR_PACKET_PACING_CAPS = (1 << 22), + /* set supported bits for validity check */ + IBV_EXP_DEVICE_ATTR_RESERVED = (1 << 23), +}; + +struct ibv_exp_device_calc_cap { + uint64_t data_types; + uint64_t data_sizes; + uint64_t int_ops; + uint64_t uint_ops; + uint64_t fp_ops; +}; + +struct ibv_exp_ext_atomics_params { + /* defines which masked operation sizes are supported with same + * endianness as stated in atomic_cap field + */ + uint64_t log_atomic_arg_sizes; /* bit-mask of supported sizes */ + uint32_t max_fa_bit_boundary; + uint32_t log_max_atomic_inline; +}; + +struct ibv_exp_masked_atomic_params { + uint32_t max_fa_bit_boundary; + uint32_t log_max_atomic_inline; + uint64_t masked_log_atomic_arg_sizes; + uint64_t masked_log_atomic_arg_sizes_network_endianness; +}; + +enum ibv_odp_general_cap_bits { + IBV_EXP_ODP_SUPPORT = 1 << 0, +}; + +enum ibv_odp_transport_cap_bits { + IBV_EXP_ODP_SUPPORT_SEND = 1 << 0, + IBV_EXP_ODP_SUPPORT_RECV = 1 << 1, + IBV_EXP_ODP_SUPPORT_WRITE = 1 << 2, + IBV_EXP_ODP_SUPPORT_READ = 1 << 3, + IBV_EXP_ODP_SUPPORT_ATOMIC = 1 << 4, + IBV_EXP_ODP_SUPPORT_SRQ_RECV = 1 << 5, +}; + +struct ibv_exp_umr_caps { + uint32_t max_klm_list_size; + uint32_t max_send_wqe_inline_klms; + uint32_t max_umr_recursion_depth; + uint32_t max_umr_stride_dimension; +}; + +struct ibv_exp_odp_caps { + uint64_t general_odp_caps; + struct { + uint32_t rc_odp_caps; + uint32_t uc_odp_caps; + uint32_t ud_odp_caps; + uint32_t dc_odp_caps; + uint32_t xrc_odp_caps; + uint32_t raw_eth_odp_caps; + } per_transport_caps; +}; + +enum ibv_exp_supported_qp_types { + IBV_EXP_QPT_RC = 1ULL << 0, + IBV_EXP_QPT_UC = 1ULL << 1, + IBV_EXP_QPT_UD = 1ULL << 2, + IBV_EXP_QPT_XRC_INIT = 1ULL << 3, + IBV_EXP_QPT_XRC_TGT = 1ULL << 4, + IBV_EXP_QPT_RAW_PACKET = 1ULL << 5, + IBV_EXP_QPT_RESERVED = 1ULL << 6 +}; + +struct ibv_exp_rx_hash_caps { + uint32_t max_rwq_indirection_tables; + uint32_t max_rwq_indirection_table_size; + uint8_t supported_hash_functions; /* from ibv_exp_rx_hash_function_flags */ + uint64_t supported_packet_fields; /* from ibv_exp_rx_hash_fields */ + uint32_t supported_qps; /* from ibv_exp_supported_qp_types */ +}; + +enum ibv_exp_mp_rq_shifts { + IBV_EXP_MP_RQ_NO_SHIFT = 0, + IBV_EXP_MP_RQ_2BYTES_SHIFT = 1 << 0 +}; + +struct ibv_exp_mp_rq_caps { + uint32_t supported_qps; /* use ibv_exp_supported_qp_types */ + uint32_t allowed_shifts; /* use ibv_exp_mp_rq_shifts */ + uint8_t min_single_wqe_log_num_of_strides; + uint8_t max_single_wqe_log_num_of_strides; + uint8_t min_single_stride_log_num_of_bytes; + uint8_t max_single_stride_log_num_of_bytes; +}; + +struct ibv_exp_ec_caps { + uint32_t max_ec_data_vector_count; + uint32_t max_ec_calc_inflight_calcs; +}; + +#define ibv_is_qpt_supported(caps, qpt) ((caps) & (1 << (qpt))) + +struct ibv_exp_tso_caps { + uint32_t max_tso; + uint32_t supported_qpts; +}; + +struct ibv_exp_packet_pacing_caps { + uint32_t qp_rate_limit_min; + uint32_t qp_rate_limit_max; /* In kbps */ + uint32_t supported_qpts; + uint32_t reserved; +}; + +struct ibv_exp_device_attr { + char fw_ver[64]; + uint64_t node_guid; + uint64_t sys_image_guid; + uint64_t max_mr_size; + uint64_t page_size_cap; + uint32_t vendor_id; + uint32_t vendor_part_id; + uint32_t hw_ver; + int max_qp; + int max_qp_wr; + int reserved; /* place holder to align with ibv_device_attr */ + int max_sge; + int max_sge_rd; + int max_cq; + int max_cqe; + int max_mr; + int max_pd; + int max_qp_rd_atom; + int max_ee_rd_atom; + int max_res_rd_atom; + int max_qp_init_rd_atom; + int max_ee_init_rd_atom; + enum ibv_exp_atomic_cap exp_atomic_cap; + int max_ee; + int max_rdd; + int max_mw; + int max_raw_ipv6_qp; + int max_raw_ethy_qp; + int max_mcast_grp; + int max_mcast_qp_attach; + int max_total_mcast_qp_attach; + int max_ah; + int max_fmr; + int max_map_per_fmr; + int max_srq; + int max_srq_wr; + int max_srq_sge; + uint16_t max_pkeys; + uint8_t local_ca_ack_delay; + uint8_t phys_port_cnt; + uint32_t comp_mask; + struct ibv_exp_device_calc_cap calc_cap; + uint64_t timestamp_mask; + uint64_t hca_core_clock; + uint64_t exp_device_cap_flags; /* use ibv_exp_device_cap_flags */ + int max_dc_req_rd_atom; + int max_dc_res_rd_atom; + int inline_recv_sz; + uint32_t max_rss_tbl_sz; + struct ibv_exp_ext_atomics_params ext_atom; + struct ibv_exp_umr_caps umr_caps; + struct ibv_exp_odp_caps odp_caps; + int max_dct; + int max_ctx_res_domain; + struct ibv_exp_rx_hash_caps rx_hash_caps; + uint32_t max_wq_type_rq; + int max_device_ctx; + struct ibv_exp_mp_rq_caps mp_rq_caps; + uint16_t wq_vlan_offloads_cap; /* use ibv_exp_vlan_offloads enum */ + struct ibv_exp_ec_caps ec_caps; + struct ibv_exp_masked_atomic_params masked_atomic; + /* + * The alignment of the padding end address. + * When RX end of packet padding is enabled the device will pad the end + * of RX packet up until the next address which is aligned to the + * rx_pad_end_addr_align size. + * Expected size for this field is according to system cache line size, + * for example 64 or 128. When field is 0 padding is not supported. + */ + int rx_pad_end_addr_align; + struct ibv_exp_tso_caps tso_caps; + struct ibv_exp_packet_pacing_caps packet_pacing_caps; +}; + +enum ibv_exp_access_flags { + IBV_EXP_ACCESS_LOCAL_WRITE = IBV_ACCESS_LOCAL_WRITE, + IBV_EXP_ACCESS_REMOTE_WRITE = IBV_ACCESS_REMOTE_WRITE, + IBV_EXP_ACCESS_REMOTE_READ = IBV_ACCESS_REMOTE_READ, + IBV_EXP_ACCESS_REMOTE_ATOMIC = IBV_ACCESS_REMOTE_ATOMIC, + IBV_EXP_ACCESS_MW_BIND = IBV_ACCESS_MW_BIND, + + IBV_EXP_ACCESS_ALLOCATE_MR = (IBV_EXP_START_FLAG << 5), + IBV_EXP_ACCESS_SHARED_MR_USER_READ = (IBV_EXP_START_FLAG << 6), + IBV_EXP_ACCESS_SHARED_MR_USER_WRITE = (IBV_EXP_START_FLAG << 7), + IBV_EXP_ACCESS_SHARED_MR_GROUP_READ = (IBV_EXP_START_FLAG << 8), + IBV_EXP_ACCESS_SHARED_MR_GROUP_WRITE = (IBV_EXP_START_FLAG << 9), + IBV_EXP_ACCESS_SHARED_MR_OTHER_READ = (IBV_EXP_START_FLAG << 10), + IBV_EXP_ACCESS_SHARED_MR_OTHER_WRITE = (IBV_EXP_START_FLAG << 11), + IBV_EXP_ACCESS_NO_RDMA = (IBV_EXP_START_FLAG << 12), + IBV_EXP_ACCESS_MW_ZERO_BASED = (IBV_EXP_START_FLAG << 13), + IBV_EXP_ACCESS_ON_DEMAND = (IBV_EXP_START_FLAG << 14), + IBV_EXP_ACCESS_RELAXED = (IBV_EXP_START_FLAG << 15), + IBV_EXP_ACCESS_PHYSICAL_ADDR = (IBV_EXP_START_FLAG << 16), + /* set supported bits for validity check */ + IBV_EXP_ACCESS_RESERVED = (IBV_EXP_START_FLAG << 17) +}; + +/* memory window information struct that is common to types 1 and 2 */ +struct ibv_exp_mw_bind_info { + struct ibv_mr *mr; + uint64_t addr; + uint64_t length; + uint64_t exp_mw_access_flags; /* use ibv_exp_access_flags */ +}; + +/* + * Flags for ibv_exp_mw_bind struct comp_mask + */ +enum ibv_exp_bind_mw_comp_mask { + IBV_EXP_BIND_MW_RESERVED = (1 << 0) +}; + +/* type 1 specific info */ +struct ibv_exp_mw_bind { + struct ibv_qp *qp; + struct ibv_mw *mw; + uint64_t wr_id; + uint64_t exp_send_flags; /* use ibv_exp_send_flags */ + struct ibv_exp_mw_bind_info bind_info; + uint32_t comp_mask; /* reserved for future growth (must be 0) */ +}; + +enum ibv_exp_calc_op { + IBV_EXP_CALC_OP_ADD = 0, + IBV_EXP_CALC_OP_MAXLOC, + IBV_EXP_CALC_OP_BAND, + IBV_EXP_CALC_OP_BXOR, + IBV_EXP_CALC_OP_BOR, + IBV_EXP_CALC_OP_NUMBER +}; + +enum ibv_exp_calc_data_type { + IBV_EXP_CALC_DATA_TYPE_INT = 0, + IBV_EXP_CALC_DATA_TYPE_UINT, + IBV_EXP_CALC_DATA_TYPE_FLOAT, + IBV_EXP_CALC_DATA_TYPE_NUMBER +}; + +enum ibv_exp_calc_data_size { + IBV_EXP_CALC_DATA_SIZE_64_BIT = 0, + IBV_EXP_CALC_DATA_SIZE_NUMBER +}; + +enum ibv_exp_wr_opcode { + IBV_EXP_WR_RDMA_WRITE = IBV_WR_RDMA_WRITE, + IBV_EXP_WR_RDMA_WRITE_WITH_IMM = IBV_WR_RDMA_WRITE_WITH_IMM, + IBV_EXP_WR_SEND = IBV_WR_SEND, + IBV_EXP_WR_SEND_WITH_IMM = IBV_WR_SEND_WITH_IMM, + IBV_EXP_WR_RDMA_READ = IBV_WR_RDMA_READ, + IBV_EXP_WR_ATOMIC_CMP_AND_SWP = IBV_WR_ATOMIC_CMP_AND_SWP, + IBV_EXP_WR_ATOMIC_FETCH_AND_ADD = IBV_WR_ATOMIC_FETCH_AND_ADD, + + IBV_EXP_WR_SEND_WITH_INV = 8 + IBV_EXP_START_ENUM, + IBV_EXP_WR_LOCAL_INV = 10 + IBV_EXP_START_ENUM, + IBV_EXP_WR_BIND_MW = 14 + IBV_EXP_START_ENUM, + IBV_EXP_WR_TSO = 15 + IBV_EXP_START_ENUM, + IBV_EXP_WR_SEND_ENABLE = 0x20 + IBV_EXP_START_ENUM, + IBV_EXP_WR_RECV_ENABLE, + IBV_EXP_WR_CQE_WAIT, + IBV_EXP_WR_EXT_MASKED_ATOMIC_CMP_AND_SWP, + IBV_EXP_WR_EXT_MASKED_ATOMIC_FETCH_AND_ADD, + IBV_EXP_WR_NOP, + IBV_EXP_WR_UMR_FILL, + IBV_EXP_WR_UMR_INVALIDATE, +}; + +enum ibv_exp_send_flags { + IBV_EXP_SEND_FENCE = IBV_SEND_FENCE, + IBV_EXP_SEND_SIGNALED = IBV_SEND_SIGNALED, + IBV_EXP_SEND_SOLICITED = IBV_SEND_SOLICITED, + IBV_EXP_SEND_INLINE = IBV_SEND_INLINE, + + IBV_EXP_SEND_IP_CSUM = (IBV_EXP_START_FLAG << 0), + IBV_EXP_SEND_WITH_CALC = (IBV_EXP_START_FLAG << 1), + IBV_EXP_SEND_WAIT_EN_LAST = (IBV_EXP_START_FLAG << 2), + IBV_EXP_SEND_EXT_ATOMIC_INLINE = (IBV_EXP_START_FLAG << 3), +}; + +struct ibv_exp_cmp_swap { + uint64_t compare_mask; + uint64_t compare_val; + uint64_t swap_val; + uint64_t swap_mask; +}; + +struct ibv_exp_fetch_add { + uint64_t add_val; + uint64_t field_boundary; +}; + +/* + * Flags for ibv_exp_send_wr struct comp_mask + */ +enum ibv_exp_send_wr_comp_mask { + IBV_EXP_SEND_WR_ATTR_RESERVED = 1 << 0 +}; + +struct ibv_exp_mem_region { + uint64_t base_addr; + struct ibv_mr *mr; + size_t length; +}; + +struct ibv_exp_mem_repeat_block { + uint64_t base_addr; /* array, size corresponds to ndim */ + struct ibv_mr *mr; + size_t *byte_count; /* array, size corresponds to ndim */ + size_t *stride; /* array, size corresponds to ndim */ +}; + +enum ibv_exp_umr_wr_type { + IBV_EXP_UMR_MR_LIST, + IBV_EXP_UMR_REPEAT +}; + +struct ibv_exp_send_wr { + uint64_t wr_id; + struct ibv_exp_send_wr *next; + struct ibv_sge *sg_list; + int num_sge; + enum ibv_exp_wr_opcode exp_opcode; /* use ibv_exp_wr_opcode */ + int reserved; /* place holder to align with ibv_send_wr */ + union { + uint32_t imm_data; /* in network byte order */ + uint32_t invalidate_rkey; + } ex; + union { + struct { + uint64_t remote_addr; + uint32_t rkey; + } rdma; + struct { + uint64_t remote_addr; + uint64_t compare_add; + uint64_t swap; + uint32_t rkey; + } atomic; + struct { + struct ibv_ah *ah; + uint32_t remote_qpn; + uint32_t remote_qkey; + } ud; + } wr; + union { + union { + struct { + uint32_t remote_srqn; + } xrc; + } qp_type; + + uint32_t xrc_remote_srq_num; + }; + union { + struct { + uint64_t remote_addr; + uint32_t rkey; + } rdma; + struct { + uint64_t remote_addr; + uint64_t compare_add; + uint64_t swap; + uint32_t rkey; + } atomic; + struct { + struct ibv_cq *cq; + int32_t cq_count; + } cqe_wait; + struct { + struct ibv_qp *qp; + int32_t wqe_count; + } wqe_enable; + } task; + union { + struct { + enum ibv_exp_calc_op calc_op; + enum ibv_exp_calc_data_type data_type; + enum ibv_exp_calc_data_size data_size; + } calc; + } op; + struct { + struct ibv_ah *ah; + uint64_t dct_access_key; + uint32_t dct_number; + } dc; + union { + struct { + struct ibv_mw *mw; + uint32_t rkey; + struct ibv_exp_mw_bind_info bind_info; + } bind_mw; + struct { + void *hdr; + uint16_t hdr_sz; + uint16_t mss; + } tso; + }; + uint64_t exp_send_flags; /* use ibv_exp_send_flags */ + uint32_t comp_mask; /* reserved for future growth (must be 0) */ + union { + struct { + uint32_t umr_type; /* use ibv_exp_umr_wr_type */ + struct ibv_exp_mkey_list_container *memory_objects; /* used when IBV_EXP_SEND_INLINE is not set */ + uint64_t exp_access; /* use ibv_exp_access_flags */ + struct ibv_mr *modified_mr; + uint64_t base_addr; + uint32_t num_mrs; /* array size of mem_repeat_block_list or mem_reg_list */ + union { + struct ibv_exp_mem_region *mem_reg_list; /* array, size corresponds to num_mrs */ + struct { + struct ibv_exp_mem_repeat_block *mem_repeat_block_list; /* array, size corresponds to num_mr */ + size_t *repeat_count; /* array size corresponds to stride_dim */ + uint32_t stride_dim; + } rb; + } mem_list; + } umr; + struct { + uint32_t log_arg_sz; + uint64_t remote_addr; + uint32_t rkey; + union { + struct { + /* For the next four fields: + * If operand_size <= 8 then inline data is immediate + * from the corresponding field; for small opernands, + * ls bits are used. + * Else the fields are pointers in the process's address space + * where arguments are stored + */ + union { + struct ibv_exp_cmp_swap cmp_swap; + struct ibv_exp_fetch_add fetch_add; + } op; + } inline_data; /* IBV_EXP_SEND_EXT_ATOMIC_INLINE is set */ + /* in the future add support for non-inline argument provisioning */ + } wr_data; + } masked_atomics; + } ext_op; +}; + +/* + * Flags for ibv_exp_values struct comp_mask + */ +enum ibv_exp_values_comp_mask { + IBV_EXP_VALUES_HW_CLOCK_NS = 1 << 0, + IBV_EXP_VALUES_HW_CLOCK = 1 << 1, + IBV_EXP_VALUES_RESERVED = 1 << 2 +}; + +struct ibv_exp_values { + uint32_t comp_mask; + uint64_t hwclock_ns; + uint64_t hwclock; +}; + +/* + * Flags for flags field in the ibv_exp_cq_init_attr struct + */ +enum ibv_exp_cq_create_flags { + IBV_EXP_CQ_CREATE_CROSS_CHANNEL = 1 << 0, + IBV_EXP_CQ_TIMESTAMP = 1 << 1, + IBV_EXP_CQ_TIMESTAMP_TO_SYS_TIME = 1 << 2, + IBV_EXP_CQ_COMPRESSED_CQE = 1 << 3, + /* + * note: update IBV_EXP_CQ_CREATE_FLAGS_MASK when adding new fields + */ +}; + +enum { + IBV_EXP_CQ_CREATE_FLAGS_MASK = IBV_EXP_CQ_CREATE_CROSS_CHANNEL | + IBV_EXP_CQ_TIMESTAMP | + IBV_EXP_CQ_TIMESTAMP_TO_SYS_TIME | + IBV_EXP_CQ_COMPRESSED_CQE, +}; + +/* + * Flags for ibv_exp_cq_init_attr struct comp_mask + * Set flags only when relevant field is valid + */ +enum ibv_exp_cq_init_attr_mask { + IBV_EXP_CQ_INIT_ATTR_FLAGS = 1 << 0, + IBV_EXP_CQ_INIT_ATTR_RESERVED = 1 << 1, /* This field is kept for backward compatibility + * of application which use the following to set comp_mask: + * cq_init_attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_RESERVED - 1 + * This kind of setting is no longer accepted and application + * may set only valid known fields, for example: + * cq_init_attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_FLAGS | + * IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN + */ + IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN = 1 << 1, + IBV_EXP_CQ_INIT_ATTR_PEER_DIRECT = 1 << 2, + IBV_EXP_CQ_INIT_ATTR_RESERVED1 = 1 << 3, +}; + +struct ibv_exp_res_domain { + struct ibv_context *context; +}; + +struct ibv_exp_cq_init_attr { + uint32_t comp_mask; + uint32_t flags; + struct ibv_exp_res_domain *res_domain; + struct ibv_exp_peer_direct_attr *peer_direct_attrs; +}; + +/* + * Flags for ibv_exp_ah_attr struct comp_mask + */ +enum ibv_exp_ah_attr_attr_comp_mask { + IBV_EXP_AH_ATTR_LL = 1 << 0, + IBV_EXP_AH_ATTR_VID = 1 << 1, + IBV_EXP_AH_ATTR_RESERVED = 1 << 2 +}; + +enum ll_address_type { + LL_ADDRESS_UNKNOWN, + LL_ADDRESS_IB, + LL_ADDRESS_ETH, + LL_ADDRESS_SIZE +}; + +struct ibv_exp_ah_attr { + struct ibv_global_route grh; + uint16_t dlid; + uint8_t sl; + uint8_t src_path_bits; + uint8_t static_rate; + uint8_t is_global; + uint8_t port_num; + uint32_t comp_mask; + struct { + enum ll_address_type type; + uint32_t len; + char *address; + } ll_address; + uint16_t vid; +}; + +/* + * Flags for exp_attr_mask argument of ibv_exp_modify_qp + */ +enum ibv_exp_qp_attr_mask { + IBV_EXP_QP_STATE = IBV_QP_STATE, + IBV_EXP_QP_CUR_STATE = IBV_QP_CUR_STATE, + IBV_EXP_QP_EN_SQD_ASYNC_NOTIFY = IBV_QP_EN_SQD_ASYNC_NOTIFY, + IBV_EXP_QP_ACCESS_FLAGS = IBV_QP_ACCESS_FLAGS, + IBV_EXP_QP_PKEY_INDEX = IBV_QP_PKEY_INDEX, + IBV_EXP_QP_PORT = IBV_QP_PORT, + IBV_EXP_QP_QKEY = IBV_QP_QKEY, + IBV_EXP_QP_AV = IBV_QP_AV, + IBV_EXP_QP_PATH_MTU = IBV_QP_PATH_MTU, + IBV_EXP_QP_TIMEOUT = IBV_QP_TIMEOUT, + IBV_EXP_QP_RETRY_CNT = IBV_QP_RETRY_CNT, + IBV_EXP_QP_RNR_RETRY = IBV_QP_RNR_RETRY, + IBV_EXP_QP_RQ_PSN = IBV_QP_RQ_PSN, + IBV_EXP_QP_MAX_QP_RD_ATOMIC = IBV_QP_MAX_QP_RD_ATOMIC, + IBV_EXP_QP_ALT_PATH = IBV_QP_ALT_PATH, + IBV_EXP_QP_MIN_RNR_TIMER = IBV_QP_MIN_RNR_TIMER, + IBV_EXP_QP_SQ_PSN = IBV_QP_SQ_PSN, + IBV_EXP_QP_MAX_DEST_RD_ATOMIC = IBV_QP_MAX_DEST_RD_ATOMIC, + IBV_EXP_QP_PATH_MIG_STATE = IBV_QP_PATH_MIG_STATE, + IBV_EXP_QP_CAP = IBV_QP_CAP, + IBV_EXP_QP_DEST_QPN = IBV_QP_DEST_QPN, + + IBV_EXP_QP_GROUP_RSS = IBV_EXP_START_FLAG << 21, + IBV_EXP_QP_DC_KEY = IBV_EXP_START_FLAG << 22, + IBV_EXP_QP_FLOW_ENTROPY = IBV_EXP_START_FLAG << 23, + IBV_EXP_QP_RATE_LIMIT = IBV_EXP_START_FLAG << 25, +}; + +/* + * Flags for ibv_exp_qp_attr struct comp_mask + * Set flags only when relevant field is valid + */ +enum ibv_exp_qp_attr_comp_mask { + IBV_EXP_QP_ATTR_FLOW_ENTROPY = 1UL << 0, + IBV_EXP_QP_ATTR_RESERVED = 1UL << 1 +}; + +struct ibv_exp_qp_attr { + enum ibv_qp_state qp_state; + enum ibv_qp_state cur_qp_state; + enum ibv_mtu path_mtu; + enum ibv_mig_state path_mig_state; + uint32_t qkey; + uint32_t rq_psn; + uint32_t sq_psn; + uint32_t dest_qp_num; + int qp_access_flags; /* use ibv_access_flags form verbs.h */ + struct ibv_qp_cap cap; + struct ibv_ah_attr ah_attr; + struct ibv_ah_attr alt_ah_attr; + uint16_t pkey_index; + uint16_t alt_pkey_index; + uint8_t en_sqd_async_notify; + uint8_t sq_draining; + uint8_t max_rd_atomic; + uint8_t max_dest_rd_atomic; + uint8_t min_rnr_timer; + uint8_t port_num; + uint8_t timeout; + uint8_t retry_cnt; + uint8_t rnr_retry; + uint8_t alt_port_num; + uint8_t alt_timeout; + uint64_t dct_key; + uint32_t comp_mask; /* reserved for future growth (must be 0) */ + uint32_t flow_entropy; + uint32_t rate_limit; +}; + +/* + * Flags for ibv_exp_qp_init_attr struct comp_mask + * Set flags only when relevant field is valid + */ +enum ibv_exp_qp_init_attr_comp_mask { + IBV_EXP_QP_INIT_ATTR_PD = 1 << 0, + IBV_EXP_QP_INIT_ATTR_XRCD = 1 << 1, + IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS = 1 << 2, + IBV_EXP_QP_INIT_ATTR_INL_RECV = 1 << 3, + IBV_EXP_QP_INIT_ATTR_QPG = 1 << 4, + IBV_EXP_QP_INIT_ATTR_ATOMICS_ARG = 1 << 5, + IBV_EXP_QP_INIT_ATTR_MAX_INL_KLMS = 1 << 6, + IBV_EXP_QP_INIT_ATTR_RESERVED = 1 << 7, /* This field is kept for backward compatibility + * of application which use the following to set comp_mask: + * qp_init_attr.comp_mask = IBV_EXP_QP_INIT_ATTR_RESERVED - 1 + * This kind of setting is no longer accepted and application + * may set only valid known fields, for example: + * qp_init_attr.comp_mask = IBV_EXP_QP_INIT_ATTR_PD | + * IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS + */ + IBV_EXP_QP_INIT_ATTR_RES_DOMAIN = 1 << 7, + IBV_EXP_QP_INIT_ATTR_RX_HASH = 1 << 8, + IBV_EXP_QP_INIT_ATTR_PORT = 1 << 9, + IBV_EXP_QP_INIT_ATTR_PEER_DIRECT = 1 << 10, + IBV_EXP_QP_INIT_ATTR_MAX_TSO_HEADER = 1 << 11, + IBV_EXP_QP_INIT_ATTR_RESERVED1 = 1 << 12, +}; + +enum ibv_exp_qpg_type { + IBV_EXP_QPG_NONE = 0, + IBV_EXP_QPG_PARENT = (1<<0), + IBV_EXP_QPG_CHILD_RX = (1<<1), + IBV_EXP_QPG_CHILD_TX = (1<<2) +}; + +struct ibv_exp_qpg_init_attrib { + uint32_t tss_child_count; + uint32_t rss_child_count; +}; + +struct ibv_exp_qpg { + uint32_t qpg_type; + union { + struct ibv_qp *qpg_parent; /* see qpg_type */ + struct ibv_exp_qpg_init_attrib parent_attrib; + }; +}; + +/* + * RX Hash Function flags. +*/ +enum ibv_exp_rx_hash_function_flags { + IBV_EXP_RX_HASH_FUNC_TOEPLITZ = 1 << 0, + IBV_EXP_RX_HASH_FUNC_XOR = 1 << 1 +}; + +/* + * RX Hash flags, these flags allows to set which incoming packet field should + * participates in RX Hash. Each flag represent certain packet's field, + * when the flag is set the field that is represented by the flag will + * participate in RX Hash calculation. + * Notice: *IPV4 and *IPV6 flags can't be enabled together on the same QP + * and *TCP and *UDP flags can't be enabled together on the same QP. +*/ +enum ibv_exp_rx_hash_fields { + IBV_EXP_RX_HASH_SRC_IPV4 = 1 << 0, + IBV_EXP_RX_HASH_DST_IPV4 = 1 << 1, + IBV_EXP_RX_HASH_SRC_IPV6 = 1 << 2, + IBV_EXP_RX_HASH_DST_IPV6 = 1 << 3, + IBV_EXP_RX_HASH_SRC_PORT_TCP = 1 << 4, + IBV_EXP_RX_HASH_DST_PORT_TCP = 1 << 5, + IBV_EXP_RX_HASH_SRC_PORT_UDP = 1 << 6, + IBV_EXP_RX_HASH_DST_PORT_UDP = 1 << 7 +}; + +/* + * RX Hash QP configuration. Sets hash function, hash types and + * Indirection table for QPs with enabled IBV_QP_INIT_ATTR_RX_HASH flag. +*/ +struct ibv_exp_rx_hash_conf { + /* enum ibv_exp_rx_hash_function_flags */ + uint8_t rx_hash_function; + /* valid only for Toeplitz */ + uint8_t rx_hash_key_len; + uint8_t *rx_hash_key; + /* enum ibv_exp_rx_hash_fields */ + uint64_t rx_hash_fields_mask; + struct ibv_exp_rwq_ind_table *rwq_ind_tbl; +}; + +/* + * Flags for exp_create_flags field in ibv_exp_qp_init_attr struct + */ +enum ibv_exp_qp_create_flags { + IBV_EXP_QP_CREATE_CROSS_CHANNEL = (1 << 2), + IBV_EXP_QP_CREATE_MANAGED_SEND = (1 << 3), + IBV_EXP_QP_CREATE_MANAGED_RECV = (1 << 4), + IBV_EXP_QP_CREATE_IGNORE_SQ_OVERFLOW = (1 << 6), + IBV_EXP_QP_CREATE_IGNORE_RQ_OVERFLOW = (1 << 7), + IBV_EXP_QP_CREATE_ATOMIC_BE_REPLY = (1 << 8), + IBV_EXP_QP_CREATE_UMR = (1 << 9), + IBV_EXP_QP_CREATE_EC_PARITY_EN = (1 << 10), + IBV_EXP_QP_CREATE_RX_END_PADDING = (1 << 11), + IBV_EXP_QP_CREATE_SCATTER_FCS = (1 << 12), + IBV_EXP_QP_CREATE_INTERNAL_USE = (1 << 15), + /* set supported bits for validity check */ + IBV_EXP_QP_CREATE_MASK = (0x00001FDC) +}; + +struct ibv_exp_qp_init_attr { + void *qp_context; + struct ibv_cq *send_cq; + struct ibv_cq *recv_cq; + struct ibv_srq *srq; + struct ibv_qp_cap cap; + enum ibv_qp_type qp_type; + int sq_sig_all; + + uint32_t comp_mask; /* use ibv_exp_qp_init_attr_comp_mask */ + struct ibv_pd *pd; + struct ibv_xrcd *xrcd; + uint32_t exp_create_flags; /* use ibv_exp_qp_create_flags */ + + uint32_t max_inl_recv; + struct ibv_exp_qpg qpg; + uint32_t max_atomic_arg; + uint32_t max_inl_send_klms; + struct ibv_exp_res_domain *res_domain; + struct ibv_exp_rx_hash_conf *rx_hash_conf; + uint8_t port_num; + struct ibv_exp_peer_direct_attr *peer_direct_attrs; + uint16_t max_tso_header; +}; + +/* + * Flags for ibv_exp_dct_init_attr struct comp_mask + */ +enum ibv_exp_dct_init_attr_comp_mask { + IBV_EXP_DCT_INIT_ATTR_RESERVED = 1 << 0 +}; + +enum { + IBV_EXP_DCT_CREATE_FLAGS_MASK = (1 << 0) - 1, +}; + +struct ibv_exp_dct_init_attr { + struct ibv_pd *pd; + struct ibv_cq *cq; + struct ibv_srq *srq; + uint64_t dc_key; + uint8_t port; + uint32_t access_flags; /* use ibv_access_flags form verbs.h */ + uint8_t min_rnr_timer; + uint8_t tclass; + uint32_t flow_label; + enum ibv_mtu mtu; + uint8_t pkey_index; + uint8_t gid_index; + uint8_t hop_limit; + uint32_t inline_size; + uint32_t create_flags; + uint32_t comp_mask; /* reserved for future growth (must be 0) */ +}; + +enum { + IBV_EXP_DCT_STATE_ACTIVE = 0, + IBV_EXP_DCT_STATE_DRAINING = 1, + IBV_EXP_DCT_STATE_DRAINED = 2 +}; + +/* + * Flags for ibv_exp_dct_attr struct comp_mask + */ +enum ibv_exp_dct_attr_comp_mask { + IBV_EXP_DCT_ATTR_RESERVED = 1 << 0 +}; + +struct ibv_exp_dct_attr { + uint64_t dc_key; + uint8_t port; + uint32_t access_flags; /* use ibv_access_flags form verbs.h */ + uint8_t min_rnr_timer; + uint8_t tclass; + uint32_t flow_label; + enum ibv_mtu mtu; + uint8_t pkey_index; + uint8_t gid_index; + uint8_t hop_limit; + uint32_t key_violations; + uint8_t state; + struct ibv_srq *srq; + struct ibv_cq *cq; + struct ibv_pd *pd; + uint32_t comp_mask; /* reserved for future growth (must be 0) */ +}; + +enum { + IBV_EXP_QUERY_PORT_STATE = 1 << 0, + IBV_EXP_QUERY_PORT_MAX_MTU = 1 << 1, + IBV_EXP_QUERY_PORT_ACTIVE_MTU = 1 << 2, + IBV_EXP_QUERY_PORT_GID_TBL_LEN = 1 << 3, + IBV_EXP_QUERY_PORT_CAP_FLAGS = 1 << 4, + IBV_EXP_QUERY_PORT_MAX_MSG_SZ = 1 << 5, + IBV_EXP_QUERY_PORT_BAD_PKEY_CNTR = 1 << 6, + IBV_EXP_QUERY_PORT_QKEY_VIOL_CNTR = 1 << 7, + IBV_EXP_QUERY_PORT_PKEY_TBL_LEN = 1 << 8, + IBV_EXP_QUERY_PORT_LID = 1 << 9, + IBV_EXP_QUERY_PORT_SM_LID = 1 << 10, + IBV_EXP_QUERY_PORT_LMC = 1 << 11, + IBV_EXP_QUERY_PORT_MAX_VL_NUM = 1 << 12, + IBV_EXP_QUERY_PORT_SM_SL = 1 << 13, + IBV_EXP_QUERY_PORT_SUBNET_TIMEOUT = 1 << 14, + IBV_EXP_QUERY_PORT_INIT_TYPE_REPLY = 1 << 15, + IBV_EXP_QUERY_PORT_ACTIVE_WIDTH = 1 << 16, + IBV_EXP_QUERY_PORT_ACTIVE_SPEED = 1 << 17, + IBV_EXP_QUERY_PORT_PHYS_STATE = 1 << 18, + IBV_EXP_QUERY_PORT_LINK_LAYER = 1 << 19, + /* mask of the fields that exists in the standard query_port_command */ + IBV_EXP_QUERY_PORT_STD_MASK = (1 << 20) - 1, + /* mask of all supported fields */ + IBV_EXP_QUERY_PORT_MASK = IBV_EXP_QUERY_PORT_STD_MASK, +}; + +/* + * Flags for ibv_exp_port_attr struct comp_mask + * Set flags only when relevant field is valid + */ +enum ibv_exp_query_port_attr_comp_mask { + IBV_EXP_QUERY_PORT_ATTR_MASK1 = 1 << 0, + IBV_EXP_QUERY_PORT_ATTR_RESERVED = 1 << 1, + + IBV_EXP_QUERY_PORT_ATTR_MASKS = IBV_EXP_QUERY_PORT_ATTR_RESERVED - 1 +}; + +struct ibv_exp_port_attr { + union { + struct { + enum ibv_port_state state; + enum ibv_mtu max_mtu; + enum ibv_mtu active_mtu; + int gid_tbl_len; + uint32_t port_cap_flags; + uint32_t max_msg_sz; + uint32_t bad_pkey_cntr; + uint32_t qkey_viol_cntr; + uint16_t pkey_tbl_len; + uint16_t lid; + uint16_t sm_lid; + uint8_t lmc; + uint8_t max_vl_num; + uint8_t sm_sl; + uint8_t subnet_timeout; + uint8_t init_type_reply; + uint8_t active_width; + uint8_t active_speed; + uint8_t phys_state; + uint8_t link_layer; + uint8_t reserved; + }; + struct ibv_port_attr port_attr; + }; + uint32_t comp_mask; + uint32_t mask1; +}; + +enum ibv_exp_cq_attr_mask { + IBV_EXP_CQ_MODERATION = 1 << 0, + IBV_EXP_CQ_CAP_FLAGS = 1 << 1 +}; + +enum ibv_exp_cq_cap_flags { + IBV_EXP_CQ_IGNORE_OVERRUN = (1 << 0), + /* set supported bits for validity check */ + IBV_EXP_CQ_CAP_MASK = (0x00000001) +}; + +/* + * Flags for ibv_exp_cq_attr struct comp_mask + * Set flags only when relevant field is valid + */ +enum ibv_exp_cq_attr_comp_mask { + IBV_EXP_CQ_ATTR_MODERATION = (1 << 0), + IBV_EXP_CQ_ATTR_CQ_CAP_FLAGS = (1 << 1), + /* set supported bits for validity check */ + IBV_EXP_CQ_ATTR_RESERVED = (1 << 2) +}; + +struct ibv_exp_cq_attr { + uint32_t comp_mask; + struct { + uint16_t cq_count; + uint16_t cq_period; + } moderation; + uint32_t cq_cap_flags; +}; + +enum ibv_exp_rereg_mr_flags { + IBV_EXP_REREG_MR_CHANGE_TRANSLATION = IBV_REREG_MR_CHANGE_TRANSLATION, + IBV_EXP_REREG_MR_CHANGE_PD = IBV_REREG_MR_CHANGE_PD, + IBV_EXP_REREG_MR_CHANGE_ACCESS = IBV_REREG_MR_CHANGE_ACCESS, + IBV_EXP_REREG_MR_KEEP_VALID = IBV_REREG_MR_KEEP_VALID, + IBV_EXP_REREG_MR_FLAGS_SUPPORTED = ((IBV_EXP_REREG_MR_KEEP_VALID << 1) - 1) +}; + +enum ibv_exp_rereg_mr_attr_mask { + IBV_EXP_REREG_MR_ATTR_RESERVED = (1 << 0) +}; + +struct ibv_exp_rereg_mr_attr { + uint32_t comp_mask; /* use ibv_exp_rereg_mr_attr_mask */ +}; + +/* + * Flags for ibv_exp_reg_shared_mr_in struct comp_mask + */ +enum ibv_exp_reg_shared_mr_comp_mask { + IBV_EXP_REG_SHARED_MR_RESERVED = (1 << 0) +}; + +struct ibv_exp_reg_shared_mr_in { + uint32_t mr_handle; + struct ibv_pd *pd; + void *addr; + uint64_t exp_access; /* use ibv_exp_access_flags */ + uint32_t comp_mask; /* reserved for future growth (must be 0) */ +}; + +enum ibv_exp_flow_flags { + IBV_EXP_FLOW_ATTR_FLAGS_ALLOW_LOOP_BACK = 1, +}; + +enum ibv_exp_flow_attr_type { + /* steering according to rule specifications */ + IBV_EXP_FLOW_ATTR_NORMAL = 0x0, + /* default unicast and multicast rule - + * receive all Eth traffic which isn't steered to any QP + */ + IBV_EXP_FLOW_ATTR_ALL_DEFAULT = 0x1, + /* default multicast rule - + * receive all Eth multicast traffic which isn't steered to any QP + */ + IBV_EXP_FLOW_ATTR_MC_DEFAULT = 0x2, + /* sniffer rule - receive all port traffic */ + IBV_EXP_FLOW_ATTR_SNIFFER = 0x3, +}; + +enum ibv_exp_flow_spec_type { + IBV_EXP_FLOW_SPEC_ETH = 0x20, + IBV_EXP_FLOW_SPEC_IB = 0x21, + IBV_EXP_FLOW_SPEC_IPV4 = 0x30, + IBV_EXP_FLOW_SPEC_IPV6 = 0x31, + IBV_EXP_FLOW_SPEC_IPV4_EXT = 0x32, + IBV_EXP_FLOW_SPEC_IPV6_EXT = 0x33, + IBV_EXP_FLOW_SPEC_TCP = 0x40, + IBV_EXP_FLOW_SPEC_UDP = 0x41, + IBV_EXP_FLOW_SPEC_VXLAN_TUNNEL = 0x50, + IBV_EXP_FLOW_SPEC_INNER = 0x100, + IBV_EXP_FLOW_SPEC_ACTION_TAG = 0x1000, +}; + +struct ibv_exp_flow_eth_filter { + uint8_t dst_mac[6]; + uint8_t src_mac[6]; + uint16_t ether_type; + /* + * same layout as 802.1q: prio 3, cfi 1, vlan id 12 + */ + uint16_t vlan_tag; +}; + +struct ibv_exp_flow_spec_eth { + enum ibv_exp_flow_spec_type type; + uint16_t size; + struct ibv_exp_flow_eth_filter val; + struct ibv_exp_flow_eth_filter mask; +}; + +struct ibv_exp_flow_ib_filter { + uint32_t qpn; + uint8_t dst_gid[16]; +}; + +struct ibv_exp_flow_spec_ib { + enum ibv_exp_flow_spec_type type; + uint16_t size; + struct ibv_exp_flow_ib_filter val; + struct ibv_exp_flow_ib_filter mask; +}; + +struct ibv_exp_flow_ipv4_filter { + uint32_t src_ip; + uint32_t dst_ip; +}; + +struct ibv_exp_flow_spec_ipv4 { + enum ibv_exp_flow_spec_type type; + uint16_t size; + struct ibv_exp_flow_ipv4_filter val; + struct ibv_exp_flow_ipv4_filter mask; +}; + +struct ibv_exp_flow_ipv6_filter { + uint8_t src_ip[16]; + uint8_t dst_ip[16]; +}; + +struct ibv_exp_flow_spec_ipv6 { + enum ibv_exp_flow_spec_type type; + uint16_t size; + struct ibv_exp_flow_ipv6_filter val; + struct ibv_exp_flow_ipv6_filter mask; +}; + +struct ibv_exp_flow_spec_action_tag { + enum ibv_exp_flow_spec_type type; + uint16_t size; + uint32_t tag_id; +}; + +struct ibv_exp_flow_ipv6_ext_filter { + uint8_t src_ip[16]; + uint8_t dst_ip[16]; + uint32_t flow_label; + uint8_t next_hdr; + uint8_t traffic_class; + uint8_t hop_limit; +}; + +struct ibv_exp_flow_spec_ipv6_ext { + enum ibv_exp_flow_spec_type type; + uint16_t size; + struct ibv_exp_flow_ipv6_ext_filter val; + struct ibv_exp_flow_ipv6_ext_filter mask; +}; + +struct ibv_exp_flow_ipv4_ext_filter { + uint32_t src_ip; + uint32_t dst_ip; + uint8_t proto; + uint8_t tos; + uint8_t ttl; + uint8_t flags; +}; + +struct ibv_exp_flow_spec_ipv4_ext { + enum ibv_exp_flow_spec_type type; + uint16_t size; + struct ibv_exp_flow_ipv4_ext_filter val; + struct ibv_exp_flow_ipv4_ext_filter mask; +}; + +struct ibv_exp_flow_tcp_udp_filter { + uint16_t dst_port; + uint16_t src_port; +}; + +struct ibv_exp_flow_spec_tcp_udp { + enum ibv_exp_flow_spec_type type; + uint16_t size; + struct ibv_exp_flow_tcp_udp_filter val; + struct ibv_exp_flow_tcp_udp_filter mask; +}; + +struct ibv_exp_flow_tunnel_filter { + uint32_t tunnel_id; +}; + +struct ibv_exp_flow_spec_tunnel { + enum ibv_exp_flow_spec_type type; + uint16_t size; + struct ibv_exp_flow_tunnel_filter val; + struct ibv_exp_flow_tunnel_filter mask; +}; + +struct ibv_exp_flow_spec { + union { + struct { + uint32_t type; + uint16_t size; + } hdr; + struct ibv_exp_flow_spec_ib ib; + struct ibv_exp_flow_spec_eth eth; + struct ibv_exp_flow_spec_ipv4 ipv4; + struct ibv_exp_flow_spec_ipv4_ext ipv4_ext; + struct ibv_exp_flow_spec_tcp_udp tcp_udp; + struct ibv_exp_flow_spec_ipv6 ipv6; + struct ibv_exp_flow_spec_ipv6_ext ipv6_ext; + struct ibv_exp_flow_spec_tunnel tunnel; + struct ibv_exp_flow_spec_action_tag flow_tag; + }; +}; + +struct ibv_exp_flow_attr { + enum ibv_exp_flow_attr_type type; + uint16_t size; + uint16_t priority; + uint8_t num_of_specs; + uint8_t port; + uint32_t flags; + /* Following are the optional layers according to user request + * struct ibv_exp_flow_spec_xxx [L2] + * struct ibv_exp_flow_spec_yyy [L3/L4] + */ + uint64_t reserved; /* reserved for future growth (must be 0) */ +}; + +struct ibv_exp_flow { + struct ibv_context *context; + uint32_t handle; +}; + +struct ibv_exp_dct { + struct ibv_context *context; + uint32_t handle; + uint32_t dct_num; + struct ibv_pd *pd; + struct ibv_srq *srq; + struct ibv_cq *cq; + pthread_mutex_t mutex; + pthread_cond_t cond; + uint32_t events_completed; +}; + +enum ibv_exp_wc_opcode { + IBV_EXP_WC_SEND, + IBV_EXP_WC_RDMA_WRITE, + IBV_EXP_WC_RDMA_READ, + IBV_EXP_WC_COMP_SWAP, + IBV_EXP_WC_FETCH_ADD, + IBV_EXP_WC_BIND_MW, + IBV_EXP_WC_LOCAL_INV = 7, + IBV_EXP_WC_MASKED_COMP_SWAP = 9, + IBV_EXP_WC_MASKED_FETCH_ADD = 10, + IBV_EXP_WC_TSO, + IBV_EXP_WC_UMR = 0x100, +/* + * Set value of IBV_EXP_WC_RECV so consumers can test if a completion is a + * receive by testing (opcode & IBV_EXP_WC_RECV). + */ + IBV_EXP_WC_RECV = 1 << 7, + IBV_EXP_WC_RECV_RDMA_WITH_IMM +}; + +enum ibv_exp_wc_flags { + IBV_EXP_WC_GRH = IBV_WC_GRH, + IBV_EXP_WC_WITH_IMM = IBV_WC_WITH_IMM, + + IBV_EXP_WC_WITH_INV = IBV_EXP_START_FLAG << 2, + IBV_EXP_WC_WITH_SL = IBV_EXP_START_FLAG << 4, + IBV_EXP_WC_WITH_SLID = IBV_EXP_START_FLAG << 5, + IBV_EXP_WC_WITH_TIMESTAMP = IBV_EXP_START_FLAG << 6, + IBV_EXP_WC_QP = IBV_EXP_START_FLAG << 7, + IBV_EXP_WC_SRQ = IBV_EXP_START_FLAG << 8, + IBV_EXP_WC_DCT = IBV_EXP_START_FLAG << 9, + IBV_EXP_WC_RX_IP_CSUM_OK = IBV_EXP_START_FLAG << 10, + IBV_EXP_WC_RX_TCP_UDP_CSUM_OK = IBV_EXP_START_FLAG << 11, + IBV_EXP_WC_RX_IPV4_PACKET = IBV_EXP_START_FLAG << 12, + IBV_EXP_WC_RX_IPV6_PACKET = IBV_EXP_START_FLAG << 13, + IBV_EXP_WC_RX_TUNNEL_PACKET = IBV_EXP_START_FLAG << 14, + IBV_EXP_WC_RX_OUTER_IP_CSUM_OK = IBV_EXP_START_FLAG << 15, + IBV_EXP_WC_RX_OUTER_TCP_UDP_CSUM_OK = IBV_EXP_START_FLAG << 16, + IBV_EXP_WC_RX_OUTER_IPV4_PACKET = IBV_EXP_START_FLAG << 17, + IBV_EXP_WC_RX_OUTER_IPV6_PACKET = IBV_EXP_START_FLAG << 18, +}; + +struct ibv_exp_wc { + uint64_t wr_id; + enum ibv_wc_status status; + enum ibv_exp_wc_opcode exp_opcode; + uint32_t vendor_err; + uint32_t byte_len; + uint32_t imm_data; /* in network byte order */ + uint32_t qp_num; + uint32_t src_qp; + int reserved; /* place holder to align with ibv_wc */ + uint16_t pkey_index; + uint16_t slid; /* invalid when TS is used */ + uint8_t sl; /* invalid when TS is used */ + uint8_t dlid_path_bits; + uint64_t timestamp; + struct ibv_qp *qp; + struct ibv_srq *srq; + struct ibv_exp_dct *dct; + uint64_t exp_wc_flags; /* use ibv_exp_wc_flags */ +}; + +/* + * Flags for ibv_exp_prefetch_mr comp_mask + */ +enum ibv_exp_prefetch_attr_comp_mask { + IBV_EXP_PREFETCH_MR_RESERVED = (1 << 0), +}; + +/* + * Flags for ibv_exp_prefetch_mr flags + */ +enum ibv_exp_prefetch_attr_flags { + /* request prefetching for write access. Used for both local and remote */ + IBV_EXP_PREFETCH_WRITE_ACCESS = (1 << 0), +}; + +struct ibv_exp_prefetch_attr { + /* Use enum ibv_exp_prefetch_attr_flags */ + uint32_t flags; + /* Address of the area to prefetch */ + void *addr; + /* Length of the area to prefetch */ + size_t length; + uint32_t comp_mask; +}; + +/* + * Flags for ibv_exp_reg_mr_in struct comp_mask + */ +enum ibv_exp_reg_mr_in_comp_mask { + /* set supported bits for validity check */ + IBV_EXP_REG_MR_CREATE_FLAGS = (1 << 0), + IBV_EXP_REG_MR_RESERVED = (1 << 1) +}; + +enum ibv_exp_reg_mr_create_flags { + IBV_EXP_REG_MR_CREATE_CONTIG = (1 << 0) /* register mr with contiguous pages */ +}; + +struct ibv_exp_reg_mr_in { + struct ibv_pd *pd; + void *addr; + size_t length; + uint64_t exp_access; /* use ibv_exp_access_flags */ + uint32_t comp_mask; /* reserved for future growth (must be 0) */ + uint32_t create_flags; /* use ibv_exp_reg_mr_create_flags */ +}; + + +enum ibv_exp_task_type { + IBV_EXP_TASK_SEND = 0, + IBV_EXP_TASK_RECV = 1 +}; + +/* + * Flags for ibv_exp_task struct comp_mask + */ +enum ibv_exp_task_comp_mask { + IBV_EXP_TASK_RESERVED = (1 << 0) +}; + +struct ibv_exp_task { + enum ibv_exp_task_type task_type; + struct { + struct ibv_qp *qp; + union { + struct ibv_exp_send_wr *send_wr; + struct ibv_recv_wr *recv_wr; + }; + } item; + struct ibv_exp_task *next; + uint32_t comp_mask; /* reserved for future growth (must be 0) */ +}; + +/* + * Flags for ibv_exp_arm_attr struct comp_mask + */ +enum ibv_exp_arm_attr_comp_mask { + IBV_EXP_ARM_ATTR_RESERVED = (1 << 0) +}; +struct ibv_exp_arm_attr { + uint32_t comp_mask; /* reserved for future growth (must be 0) */ +}; + +enum ibv_exp_mr_create_flags { + IBV_EXP_MR_SIGNATURE_EN = (1 << 0), + IBV_EXP_MR_INDIRECT_KLMS = (1 << 1) +}; + +struct ibv_exp_mr_init_attr { + uint32_t max_klm_list_size; /* num of entries */ + uint32_t create_flags; /* use ibv_exp_mr_create_flags */ + uint64_t exp_access_flags; /* use ibv_exp_access_flags */ +}; + +/* + * Comp_mask for ibv_exp_create_mr_in struct comp_mask + */ +enum ibv_exp_create_mr_in_comp_mask { + IBV_EXP_CREATE_MR_IN_RESERVED = (1 << 0) +}; + +struct ibv_exp_create_mr_in { + struct ibv_pd *pd; + struct ibv_exp_mr_init_attr attr; + uint32_t comp_mask; /* use ibv_exp_create_mr_in_comp_mask */ +}; + +/* + * Flags for ibv_exp_mkey_attr struct comp_mask + */ +enum ibv_exp_mkey_attr_comp_mask { + IBV_EXP_MKEY_ATTR_RESERVED = (1 << 0) +}; + +struct ibv_exp_mkey_attr { + uint32_t max_klm_list_size; + uint32_t comp_mask; /* use ibv_exp_mkey_attr_comp_mask */ +}; + +struct ibv_exp_mkey_list_container { + uint32_t max_klm_list_size; + struct ibv_context *context; +}; + +enum ibv_exp_mkey_list_type { + IBV_EXP_MKEY_LIST_TYPE_INDIRECT_MR +}; + +/* + * Flags for ibv_exp_mkey_list_container_attr struct comp_mask + */ +enum ibv_exp_alloc_mkey_list_comp_mask { + IBV_EXP_MKEY_LIST_CONTAINER_RESERVED = (1 << 0) +}; + +struct ibv_exp_mkey_list_container_attr { + struct ibv_pd *pd; + uint32_t mkey_list_type; /* use ibv_exp_mkey_list_type */ + uint32_t max_klm_list_size; + uint32_t comp_mask; /*use ibv_exp_alloc_mkey_list_comp_mask */ +}; + +/* + * Flags for ibv_exp_rereg_out struct comp_mask + */ +enum ibv_exp_rereg_mr_comp_mask { + IBV_EXP_REREG_MR_RESERVED = (1 << 0) +}; + +struct ibv_exp_rereg_out { + int need_dofork; + uint32_t comp_mask; /* use ibv_exp_rereg_mr_comp_mask */ +}; + +/* + * Flags for ibv_exp_dereg_out struct comp_mask + */ +enum ibv_exp_dereg_mr_comp_mask { + IBV_EXP_DEREG_MR_RESERVED = (1 << 0) +}; + +struct ibv_exp_dereg_out { + int need_dofork; + uint32_t comp_mask; /* use ibv_exp_dereg_mr_comp_mask */ +}; + +struct verbs_env_item { + char *name; + char *value; + struct verbs_env_item *next; +}; + +struct verbs_environment { + struct verbs_env_item *head; + pthread_mutex_t mtx; +}; + +/* RSS stuff */ + +enum ibv_exp_wq_type { + IBV_EXP_WQT_RQ, + IBV_EXP_WQT_SRQ +}; + +enum ibv_exp_wq_state { + IBV_EXP_WQS_RESET, + IBV_EXP_WQS_RDY, + IBV_EXP_WQS_ERR, + IBV_EXP_WQS_UNKNOWN +}; + +/* VLAN Offloads */ +enum ibv_exp_vlan_offloads { + /* Represents C-VLAN stripping feature */ + IBV_EXP_RECEIVE_WQ_CVLAN_STRIP = (1 << 0), + /* Represents C-VLAN insertion feature*/ + IBV_EXP_RECEIVE_WQ_CVLAN_INSERTION = (1 << 1), + IBV_EXP_RECEIVE_WQ_VLAN_OFFLOADS_RESERVED = (1 << 2), +}; + +/* + * Work Queue. QP can be created without internal WQs "packaged" inside it, + * this QPs can be configured to use "external" WQ object as its + * receive/send queue. + * WQ associated (many to one) with Completion Queue it owns WQ properties + * (PD, WQ size etc). + * WQ of type IBV_EXP_WQT_RQ contains receive WQEs, in which case its PD serves + * scatter as well. + * WQ of type IBV_EXP_WQT_SRQ is associated (many to one) with regular ibv_srq, + * in which case it does not hold receive WQEs. + * QPs can be associated with IBV_EXP_WQT_S/RQ WQs via WQ Indirection Table. + */ +struct ibv_exp_wq { + struct ibv_context *context; + void *wq_context; /* Associated Context of the WQ */ + uint32_t handle; + /* Protection domain WQ should be associated with */ + struct ibv_pd *pd; + /* CQ to be associated with the WQ */ + struct ibv_cq *cq; + /* SRQ handle if WQ is to be associated with an SRQ, otherwise NULL */ + struct ibv_srq *srq; + uint32_t wq_num; + enum ibv_exp_wq_state state; + enum ibv_exp_wq_type wq_type; + uint32_t comp_mask; +}; + +enum ibv_exp_wq_init_attr_mask { + IBV_EXP_CREATE_WQ_RES_DOMAIN = (1 << 0), + IBV_EXP_CREATE_WQ_MP_RQ = (1 << 1), + IBV_EXP_CREATE_WQ_VLAN_OFFLOADS = (1 << 2), + IBV_EXP_CREATE_WQ_FLAGS = (1 << 3), + IBV_EXP_CREATE_WQ_RESERVED = (1 << 4) +}; + +struct ibv_exp_wq_mp_rq { + enum ibv_exp_mp_rq_shifts use_shift; + uint8_t single_wqe_log_num_of_strides; + uint8_t single_stride_log_num_of_bytes; +}; + +enum ibv_exp_wq_init_attr_flags { + IBV_EXP_CREATE_WQ_FLAG_RX_END_PADDING = (1ULL << 0), + IBV_EXP_CREATE_WQ_FLAG_SCATTER_FCS = (1ULL << 1), + IBV_EXP_CREATE_WQ_FLAG_RESERVED = (1ULL << 2) +}; + +struct ibv_exp_wq_init_attr { + /* Associated Context of the WQ */ + void *wq_context; + enum ibv_exp_wq_type wq_type; + /* Valid for non IBV_EXP_WQT_SRQ WQ */ + uint32_t max_recv_wr; + /* Valid for non IBV_EXP_WQT_SRQ WQ */ + uint32_t max_recv_sge; + /* Protection domain WQ should be associated with */ + struct ibv_pd *pd; + /* CQ to be associated with the WQ */ + struct ibv_cq *cq; + /* SRQ handle if WQ is of type IBV_EXP_WQT_SRQ, otherwise NULL */ + struct ibv_srq *srq; + /* refers to ibv_exp_wq_init_attr_mask */ + uint32_t comp_mask; + struct ibv_exp_res_domain *res_domain; + struct ibv_exp_wq_mp_rq mp_rq; + uint16_t vlan_offloads; /* use ibv_exp_vlan_offloads enum */ + uint64_t flags; /* general wq create flags */ +}; + +enum ibv_exp_wq_attr_mask { + IBV_EXP_WQ_ATTR_STATE = 1 << 0, + IBV_EXP_WQ_ATTR_CURR_STATE = 1 << 1, + IBV_EXP_WQ_ATTR_VLAN_OFFLOADS = 1 << 2, + IBV_EXP_WQ_ATTR_RESERVED = 1 << 3 +}; + +struct ibv_exp_wq_attr { + /* enum ibv_exp_wq_attr_mask */ + uint32_t attr_mask; + /* Move the RQ to this state */ + enum ibv_exp_wq_state wq_state; + /* Assume this is the current RQ state */ + enum ibv_exp_wq_state curr_wq_state; + uint16_t vlan_offloads; /* use ibv_exp_vlan_offloads enum */ +}; + +/* + * Receive Work Queue Indirection Table. + * It's used in order to distribute incoming packets between different + * Receive Work Queues. Associating Receive WQs with different CPU cores + * allows to workload the traffic between different CPU cores. + * The Indirection Table can contain only WQs of type IBV_EXP_WQT_S/RQ. +*/ +struct ibv_exp_rwq_ind_table { + struct ibv_context *context; + struct ibv_pd *pd; + int ind_tbl_handle; + int ind_tbl_num; + uint32_t comp_mask; +}; + +enum ibv_exp_ind_table_init_attr_mask { + IBV_EXP_CREATE_IND_TABLE_RESERVED = (1 << 0) +}; + +/* + * Receive Work Queue Indirection Table attributes +*/ +struct ibv_exp_rwq_ind_table_init_attr { + struct ibv_pd *pd; + /* Log, base 2, of Indirection table size */ + uint32_t log_ind_tbl_size; + /* Each entry is a pointer to Receive Work Queue */ + struct ibv_exp_wq **ind_tbl; + uint32_t comp_mask; +}; + +/* Accelerated verbs */ +enum ibv_exp_thread_model { + IBV_EXP_THREAD_SAFE, /* The lib responsible to protect the object in multithreaded environment */ + IBV_EXP_THREAD_UNSAFE, /* The application responsible to protect the object in multithreaded environment */ + IBV_EXP_THREAD_SINGLE /* The object is called from only one thread */ +}; + +enum ibv_exp_msg_model { + IBV_EXP_MSG_DEFAULT, /* Use the provider default message model */ + IBV_EXP_MSG_LOW_LATENCY, /* Hint the provider to optimize for low latency */ + IBV_EXP_MSG_HIGH_BW, /* Hint the provider to optimize for high bandwidth */ + IBV_EXP_MSG_FORCE_LOW_LATENCY, /* Force the provider to optimize for low latency */ +}; + +/* + * Resource domains + */ +enum ibv_exp_res_domain_init_attr_comp_mask { + IBV_EXP_RES_DOMAIN_THREAD_MODEL = (1 << 0), + IBV_EXP_RES_DOMAIN_MSG_MODEL = (1 << 1), + IBV_EXP_RES_DOMAIN_RESERVED = (1 << 2), +}; + +struct ibv_exp_res_domain_init_attr { + uint32_t comp_mask; /* use ibv_exp_res_domain_init_attr_comp_mask */ + enum ibv_exp_thread_model thread_model; + enum ibv_exp_msg_model msg_model; +}; + +enum ibv_exp_destroy_res_domain_comp_mask { + IBV_EXP_DESTROY_RES_DOMAIN_RESERVED = (1 << 0), +}; + +struct ibv_exp_destroy_res_domain_attr { + uint32_t comp_mask; /* use ibv_exp_destroy_res_domain_comp_mask */ +}; + +/* + * Query interface (specialized Verbs) + */ + +enum ibv_exp_query_intf_flags { + /* Interface functions includes correctness and validity checks */ + IBV_EXP_QUERY_INTF_FLAG_ENABLE_CHECKS = (1 << 0), +}; + +enum ibv_exp_intf_family { + IBV_EXP_INTF_QP_BURST, + IBV_EXP_INTF_CQ, + IBV_EXP_INTF_WQ, + IBV_EXP_INTF_RESERVED, +}; + +enum ibv_exp_experimental_intf_family { + IBV_EXP_EXPERIMENTAL_INTF_RESERVED, +}; + +enum ibv_exp_intf_scope { + IBV_EXP_INTF_GLOBAL, /* Permanent interface, identified by + * the ibv_exp_intf_family enum + */ + IBV_EXP_INTF_EXPERIMENTAL, /* Interface under evaluation, identified by + * the ibv_exp_experimental_intf_family enum + * This interface may change between lib + * versions + */ + IBV_EXP_INTF_VENDOR, /* Vendor specific interface, defined in vendor + * separate header file + */ + IBV_EXP_INTF_VENDOR_EXPERIMENTAL, /* Vendor interface under evaluation, + * defined in vendor separate header + * file + */ +}; + +/* Return status from ibv_exp_query_intf */ +enum ibv_exp_query_intf_status { + IBV_EXP_INTF_STAT_OK, + IBV_EXP_INTF_STAT_VENDOR_NOT_SUPPORTED, /* The provided 'vendor_guid' is not supported */ + IBV_EXP_INTF_STAT_INTF_NOT_SUPPORTED, /* The provided 'intf' is not supported */ + IBV_EXP_INTF_STAT_VERSION_NOT_SUPPORTED, /* The provided 'intf_version' is not supported */ + IBV_EXP_INTF_STAT_INVAL_PARARM, /* General invalid parameter */ + IBV_EXP_INTF_STAT_INVAL_OBJ_STATE, /* QP is not in INIT, RTR or RTS state */ + IBV_EXP_INTF_STAT_INVAL_OBJ, /* Mismatch between the provided 'obj'(CQ/QP/WQ) and requested 'intf' */ + IBV_EXP_INTF_STAT_FLAGS_NOT_SUPPORTED, /* The provided set of 'flags' is not supported */ + IBV_EXP_INTF_STAT_FAMILY_FLAGS_NOT_SUPPORTED, /* The provided set of 'family_flags' is not supported */ +}; + +enum ibv_exp_query_intf_comp_mask { + IBV_EXP_QUERY_INTF_RESERVED = (1 << 0), +}; + +struct ibv_exp_query_intf_params { + uint32_t flags; /* use ibv_exp_query_intf_flags */ + enum ibv_exp_intf_scope intf_scope; + uint64_t vendor_guid; /* set in case VENDOR intf_scope selected */ + uint32_t intf; /* for GLOBAL intf_scope use ibv_exp_intf_family enum */ + uint32_t intf_version; /* Version */ + void *obj; /* interface object (CQ/QP/WQ) */ + void *family_params; /* Family-specific params */ + uint32_t family_flags; /* Family-specific flags */ + uint32_t comp_mask; /* use ibv_exp_query_intf_comp_mask */ +}; + +enum ibv_exp_release_intf_comp_mask { + IBV_EXP_RELEASE_INTF_RESERVED = (1 << 0), +}; + +struct ibv_exp_release_intf_params { + uint32_t comp_mask; /* use ibv_exp_release_intf_comp_mask */ +}; + +/* + * Family interfaces + */ + +/* QP burst family */ + +/* Flags to use in family_flags field of ibv_exp_query_intf_params on family creation */ +enum ibv_exp_qp_burst_family_create_flags { + /* To disable loop-back of multi-cast messages in RAW-ETH */ + IBV_EXP_QP_BURST_CREATE_DISABLE_ETH_LOOPBACK = (1 << 0), + /* To enable Multi-Packet send WR when possible */ + IBV_EXP_QP_BURST_CREATE_ENABLE_MULTI_PACKET_SEND_WR = (1 << 1), +}; + +/* Flags to use on send functions of QP burst family */ +enum ibv_exp_qp_burst_family_flags { + IBV_EXP_QP_BURST_SIGNALED = 1 << 0, + IBV_EXP_QP_BURST_SOLICITED = 1 << 1, + IBV_EXP_QP_BURST_IP_CSUM = 1 << 2, + IBV_EXP_QP_BURST_TUNNEL = 1 << 3, + IBV_EXP_QP_BURST_FENCE = 1 << 4, +}; + +/* All functions of QP family included in QP family version 1 */ +struct ibv_exp_qp_burst_family { + int (*send_pending)(struct ibv_qp *qp, uint64_t addr, uint32_t length, uint32_t lkey, uint32_t flags); + int (*send_pending_inline)(struct ibv_qp *qp, void *addr, uint32_t length, uint32_t flags); + int (*send_pending_sg_list)(struct ibv_qp *qp, struct ibv_sge *sg_list, uint32_t num, uint32_t flags); + int (*send_flush)(struct ibv_qp *qp); + int (*send_burst)(struct ibv_qp *qp, struct ibv_sge *msg_list, uint32_t num, uint32_t flags); + int (*recv_burst)(struct ibv_qp *qp, struct ibv_sge *msg_list, uint32_t num); +}; + +struct ibv_exp_qp_burst_family_v1 { + /* + * send_pending - Put one message in the provider send queue. + * + * Common usage: After calling several times to send_pending + * the application need to call send_flush to ensure the send + * of the pending messages. + * Note: Use ibv_exp_qp_burst_family_flags for the flags field + */ + int (*send_pending)(struct ibv_qp *qp, uint64_t addr, uint32_t length, uint32_t lkey, uint32_t flags); + /* + * send_pending_inline - Put one inline message in the provider send queue. + * + * Common usage: Same as send_pending + * Notes: + * - The message length must fit the max inline size of the QP. + * Providing bigger messages may lead to data corruption and + * segmentation fault. + * - Use ibv_exp_qp_burst_family_flags for the flags field + */ + int (*send_pending_inline)(struct ibv_qp *qp, void *addr, uint32_t length, uint32_t flags); + /* + * send_pending_sg_list - Put one scatter-gather(sg) message in the provider send queue. + * + * Common usage: Same as send_pending + * Notes: + * - The number of sg entries must fit the max_send_sge of the QP. + * Providing bigger list of sg entries may lead to data corruption and + * segmentation fault. + * - Use ibv_exp_qp_burst_family_flags for the flags field + */ + int (*send_pending_sg_list)(struct ibv_qp *qp, struct ibv_sge *sg_list, uint32_t num, uint32_t flags); + /* + * send_flush - To flush the pending messages. + * + * Note: Use ibv_exp_qp_burst_family_flags for the flags field + */ + int (*send_flush)(struct ibv_qp *qp); + /* + * send_burst - Send a list of 'num' messages (no send_flush required in this case) + */ + int (*send_burst)(struct ibv_qp *qp, struct ibv_sge *msg_list, uint32_t num, uint32_t flags); + /* + * recv_burst - Post a set of 'num' receive buffers. + * + * Note: One sge per message is supported by this function + */ + int (*recv_burst)(struct ibv_qp *qp, struct ibv_sge *msg_list, uint32_t num); + /* + * send_pending_vlan - Put one message in the provider send queue + * and insert vlan_tci to header. + * + * Common usage: Same as send_pending + * Note: + * - Same as send_pending + * - Not supported when MP enable. + */ + int (*send_pending_vlan)(struct ibv_qp *qp, uint64_t addr, uint32_t length, + uint32_t lkey, uint32_t flags, uint16_t *vlan_tci); + /* + * send_pending_inline - Put one inline message in the provider send queue + * and insert vlan_tci to header. + * + * Common usage: Same as send_pending_inline + * Notes: + * - Same as send_pending_inline + * - Not supported when MP enable. + */ + int (*send_pending_inline_vlan)(struct ibv_qp *qp, void *addr, uint32_t length, + uint32_t flags, uint16_t *vlan_tci); + /* + * send_pending_sg_list - Put one scatter-gather(sg) message in the provider send queue + * and insert vlan_tci to header. + * + * Common usage: Same as send_pending_sg_list + * Notes: + * - Same as send_pending_sg_list + * - Not supported when MP enable. + */ + int (*send_pending_sg_list_vlan)(struct ibv_qp *qp, struct ibv_sge *sg_list, uint32_t num, + uint32_t flags, uint16_t *vlan_tci); +}; + +/* WQ family */ +struct ibv_exp_wq_family { + /* + * recv_sg_list - Post one scatter-gather(sg) receive buffer. + * + * Note: + * - The number of sg entries must fit the max_recv_sge of the WQ. + * Providing bigger list of sg entries may lead to data corruption and + * segmentation fault. + */ + int (*recv_sg_list)(struct ibv_exp_wq *wq, struct ibv_sge *sg_list, uint32_t num_sg); + /* + * recv_burst - Post a set of 'num' receive buffers. + * + * Note: One sge per message is supported by this function + */ + int (*recv_burst)(struct ibv_exp_wq *wq, struct ibv_sge *msg_list, uint32_t num); +}; + +/* CQ family */ +enum ibv_exp_cq_family_flags { + /* RX offloads flags */ + /* The cq_family_flags are applicable + * according to the existence of the + * related device capabilities flags */ + IBV_EXP_CQ_RX_IP_CSUM_OK = 1 << 0, /* IBV_EXP_DEVICE_RX_CSUM_IP_PKT or IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT */ + IBV_EXP_CQ_RX_TCP_UDP_CSUM_OK = 1 << 1, /* IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT */ + IBV_EXP_CQ_RX_IPV4_PACKET = 1 << 2, /* IBV_EXP_DEVICE_RX_CSUM_IP_PKT or IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT */ + IBV_EXP_CQ_RX_IPV6_PACKET = 1 << 3, /* IBV_EXP_DEVICE_RX_CSUM_IP_PKT or IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT */ + IBV_EXP_CQ_RX_TUNNEL_PACKET = 1 << 4, /* IBV_EXP_DEVICE_VXLAN_SUPPORT */ + IBV_EXP_CQ_RX_OUTER_IP_CSUM_OK = 1 << 5, /* IBV_EXP_DEVICE_VXLAN_SUPPORT */ + IBV_EXP_CQ_RX_OUTER_TCP_UDP_CSUM_OK = 1 << 6, /* IBV_EXP_DEVICE_VXLAN_SUPPORT */ + IBV_EXP_CQ_RX_OUTER_IPV4_PACKET = 1 << 7, /* IBV_EXP_DEVICE_VXLAN_SUPPORT */ + IBV_EXP_CQ_RX_OUTER_IPV6_PACKET = 1 << 8, /* IBV_EXP_DEVICE_VXLAN_SUPPORT */ + + /* Flags supported from CQ family version 1 */ + /* Multi-Packet RQ flag */ + IBV_EXP_CQ_RX_MULTI_PACKET_LAST_V1 = 1 << 9, /* Last packet on WR */ + /* CVLAN stripping RQ flag */ + IBV_EXP_CQ_RX_CVLAN_STRIPPED_V1 = 1 << 10, /* + * When set, CVLAN is stripped + * from incoming packets. + */ + + /* The RX TCP/UDP packet flags + * applicable according to + * IBV_EXP_DEVICE_RX_TCP_UDP_PKT_TYPE + * device capabilities flag + */ + IBV_EXP_CQ_RX_TCP_PACKET = 1 << 11, + IBV_EXP_CQ_RX_UDP_PACKET = 1 << 12, +}; + +/* All functions of CQ family included in CQ family version 1 */ +struct ibv_exp_cq_family { + int32_t (*poll_cnt)(struct ibv_cq *cq, uint32_t max); + int32_t (*poll_length)(struct ibv_cq *cq, void *buf, uint32_t *inl); + int32_t (*poll_length_flags)(struct ibv_cq *cq, void *buf, uint32_t *inl, uint32_t *flags); +}; + +struct ibv_exp_cq_family_v1 { + /* + * poll_cnt - Poll up to 'max' valid completions + * + * The function returns the number of valid completions it + * managed to drain from the CQ. + * + * Usage example: In case a CQ is connected to one send-queue + * the application may use this function to get + * the number of the QP send-completions. + * + * Return value (n): + * n >= 0 : number extracted completions. + * n == -1 : operation failed. completion is not extracted. + * To extract this completion, ibv_poll_cq() must be used + * + * Note: The function designed to support TX completion, it may also be + * used for RX completion but it is not supporting RX inline-scatter. + */ + int32_t (*poll_cnt)(struct ibv_cq *cq, uint32_t max); + /* + * poll_length - Poll one receive completion and provide the related + * message length. + * + * The function returns only the length of the completed message. + * In case of inline received message the message will be copied + * to the provided buffer ('buf') and the '*inl' status will be set. + * The function extracts only completion of regular receive-messages. + * In case of send-message completion or SRQ receive-message completion + * it returns -1. + * + * Usage example: In case a CQ is connected to one receive-queue + * the application may use this function to get + * the size of the next received message. + * + * Return value (n): + * n > 0 : successful completion with positive length. + * *inl will be set to 1 if data was copied to buffer. + * + * 0 : Empty. + * n == -1 : operation failed. completion is not extracted. + * To extract this completion, ibv_poll_cq() must be used + */ + int32_t (*poll_length)(struct ibv_cq *cq, void *buf, uint32_t *inl); + /* + * poll_length_flags - Poll one receive completion and provide the related + * message length and completion flags. + * + * The same as poll_length but also retrieves completion flags as + * defined by the enum ibv_exp_cq_family_flags + */ + int32_t (*poll_length_flags)(struct ibv_cq *cq, void *buf, uint32_t *inl, uint32_t *flags); + /* + * poll_length_flags_mp_rq - Poll one receive completion and provide the related + * message length, packet-offset and completion flags. + * + * The same as poll_length_flags but: + * - Without the inline-receive support. + * - Also retrieves offset in the WR posted buffer as defined by the WR SG list. + * The start of the received packet is located in this offset. + */ + int32_t (*poll_length_flags_mp_rq)(struct ibv_cq *cq, uint32_t *offset, uint32_t *flags); + /* + * poll_length_flags_cvlan - Poll one receive completion and provide the + * related message length, completion flags + * and CVLAN TCI. + * The CVLAN TCI value is valid only when + * IBV_EXP_CQ_RX_CVLAN_STRIPPED_V1 flag is + * set. + * + * The same as poll_length_flags but: + * - Also retrievs the packet's CVLAN TCI that was stripped by the HW. + */ + int32_t (*poll_length_flags_cvlan)(struct ibv_cq *cq, void *buf, + uint32_t *inl, uint32_t *flags, + uint16_t *vlan_tci); + /* + * poll_length_flags_mp_rq_cvlan - Poll one receive completion and provide + * the related message length, + * packet-offset, completion flags and + * CVLAN TCI + * + * The same as poll_length_flags_cvlan but: + * - Without the inline-receive support. + * - Also retrives offset in the WR posted buffer as defined by the + * WR SG list. The start of the received packet is located in this + * offset. + */ + int32_t (*poll_length_flags_mp_rq_cvlan)(struct ibv_cq *cq, + uint32_t *offset, + uint32_t *flags, + uint16_t *vlan_tci); +}; + +enum { + IBV_EXP_NUM_DC_INFO_LIDS = 30 +}; + +struct ibv_exp_dc_info_ent { + uint16_t lid[IBV_EXP_NUM_DC_INFO_LIDS]; + uint32_t seqnum; +}; + +enum ibv_exp_roce_gid_type { + IBV_EXP_IB_ROCE_V1_GID_TYPE, + IBV_EXP_ROCE_V2_GID_TYPE, + IBV_EXP_ROCE_V1_5_GID_TYPE, +}; + +enum ibv_exp_query_gid_attr { + IBV_EXP_QUERY_GID_ATTR_TYPE = (1 << 0), + IBV_EXP_QUERY_GID_ATTR_GID = (1 << 1), + IBV_EXP_QUERY_GID_ATTR_RESERVED = (1 << 2), +}; + +struct ibv_exp_gid_attr { + uint32_t comp_mask; + enum ibv_exp_roce_gid_type type; + union ibv_gid gid; +}; + +/** + * enum ibv_exp_ec_calc_attr_comp_mask - erasure coding context + * init attributes compatibility enumeration + */ +enum ibv_exp_ec_calc_attr_comp_mask { + IBV_EXP_EC_CALC_ATTR_MAX_INFLIGHT = (1 << 0), + IBV_EXP_EC_CALC_ATTR_K = (1 << 1), + IBV_EXP_EC_CALC_ATTR_M = (1 << 2), + IBV_EXP_EC_CALC_ATTR_W = (1 << 3), + IBV_EXP_EC_CALC_ATTR_MAX_DATA_SGE = (1 << 4), + IBV_EXP_EC_CALC_ATTR_MAX_CODE_SGE = (1 << 5), + IBV_EXP_EC_CALC_ATTR_ENCODE_MAT = (1 << 6), + IBV_EXP_EC_CALC_ATTR_AFFINITY = (1 << 7), + IBV_EXP_EC_CALC_ATTR_POLLING = (1 << 8), + IBV_EXP_EC_CALC_INIT_ATTR_RESERVED = (1 << 9), +}; + +/** + * struct ibv_exp_ec_calc_init_attr - erasure coding engine + * initialization attributes + * + * @comp_mask: compatibility bitmask + * @max_inflight_calcs: maximum inflight calculations + * @k: number of data blocks + * @m: number of core blocks + * @w: Galois field bits GF(2^w) + * @max_data_sge: maximum data sg elements to be used for encode/decode + * @max_code_sge: maximum code sg elements to be used for encode/decode + * @encode_matrix: buffer that contain the encoding matrix + * @affinity_hint: affinity hint for asynchronous calcs completion + * steering. + * @polling: polling mode (if set no completions will be generated + * by events). + */ +struct ibv_exp_ec_calc_init_attr { + uint32_t comp_mask; + uint32_t max_inflight_calcs; + int k; + int m; + int w; + int max_data_sge; + int max_code_sge; + uint8_t *encode_matrix; + int affinity_hint; + int polling; +}; + +/** + * enum ibv_exp_ec_status - EX calculation status + * + * @IBV_EXP_EC_CALC_SUCCESS: EC calc operation succeded + * @IBV_EXP_EC_CALC_FAIL: EC calc operation failed + */ +enum ibv_exp_ec_status { + IBV_EXP_EC_CALC_SUCCESS, + IBV_EXP_EC_CALC_FAIL, +}; + +/** + * struct ibv_exp_ec_comp - completion context of EC calculation + * + * @done: function handle of the EC calculation completion + * @status: status of the EC calculation + * + * The consumer is expected to embed this structure in his calculation context + * so that the user context would be acquired back using offsetof() + */ +struct ibv_exp_ec_comp { + void (*done)(struct ibv_exp_ec_comp *comp); + enum ibv_exp_ec_status status; +}; + +/** + * struct ibv_exp_ec_calc - erasure coding engine context + * + * @pd: protection domain + */ +struct ibv_exp_ec_calc { + struct ibv_pd *pd; +}; + +/** + * struct ibv_exp_ec_mem - erasure coding memory layout context + * + * @data_blocks: array of data sg elements + * @num_data_sge: number of data sg elements + * @code_blocks: array of code sg elements + * @num_code_sge: number of code sg elements + * @block_size: logical block size + */ +struct ibv_exp_ec_mem { + struct ibv_sge *data_blocks; + int num_data_sge; + struct ibv_sge *code_blocks; + int num_code_sge; + int block_size; +}; + +/** + * struct ibv_exp_ec_stripe - erasure coding stripe descriptor + * + * @qp: queue-pair connected to the relevant peer + * @wr: send work request, can either be a RDMA wr or a SEND + */ +struct ibv_exp_ec_stripe { + struct ibv_qp *qp; + struct ibv_send_wr *wr; +}; + +struct ibv_exp_peer_commit; +struct ibv_exp_rollback_ctx; + + +struct ibv_exp_peer_peek; +struct ibv_exp_peer_abort_peek; + +struct verbs_context_exp { + /* "grows up" - new fields go here */ + int (*exp_peer_peek_cq)(struct ibv_cq *ibcq, + struct ibv_exp_peer_peek *peek_ctx); + int (*exp_peer_abort_peek_cq)(struct ibv_cq *ibcq, + struct ibv_exp_peer_abort_peek *ack_ctx); + int (*exp_peer_commit_qp)(struct ibv_qp *qp, + struct ibv_exp_peer_commit *peer); + int (*exp_rollback_send)(struct ibv_qp *qp, + struct ibv_exp_rollback_ctx *rollback); + int (*ec_update_sync)(struct ibv_exp_ec_calc *calc, + struct ibv_exp_ec_mem *ec_mem, + uint8_t *data_updates, + uint8_t *code_updates); + int (*ec_update_async)(struct ibv_exp_ec_calc *calc, + struct ibv_exp_ec_mem *ec_mem, + uint8_t *data_updates, + uint8_t *code_updates, + struct ibv_exp_ec_comp *ec_comp); + struct ibv_exp_ec_calc *(*alloc_ec_calc)(struct ibv_pd *pd, + struct ibv_exp_ec_calc_init_attr *attr); + void (*dealloc_ec_calc)(struct ibv_exp_ec_calc *calc); + int (*ec_encode_async)(struct ibv_exp_ec_calc *calc, + struct ibv_exp_ec_mem *ec_mem, + struct ibv_exp_ec_comp *ec_comp); + int (*ec_encode_sync)(struct ibv_exp_ec_calc *calc, + struct ibv_exp_ec_mem *ec_mem); + int (*ec_decode_async)(struct ibv_exp_ec_calc *calc, + struct ibv_exp_ec_mem *ec_mem, + uint8_t *erasures, + uint8_t *decode_matrix, + struct ibv_exp_ec_comp *ec_comp); + int (*ec_decode_sync)(struct ibv_exp_ec_calc *calc, + struct ibv_exp_ec_mem *ec_mem, + uint8_t *erasures, + uint8_t *decode_matrix); + int (*ec_poll)(struct ibv_exp_ec_calc *calc, int n); + int (*ec_encode_send)(struct ibv_exp_ec_calc *calc, + struct ibv_exp_ec_mem *ec_mem, + struct ibv_exp_ec_stripe *data_stripes, + struct ibv_exp_ec_stripe *code_stripes); + int (*exp_query_gid_attr)(struct ibv_context *context, uint8_t port_num, + unsigned int index, + struct ibv_exp_gid_attr *attr); + int (*exp_destroy_rwq_ind_table)(struct ibv_exp_rwq_ind_table *rwq_ind_table); + struct ibv_exp_rwq_ind_table *(*exp_create_rwq_ind_table)(struct ibv_context *context, + struct ibv_exp_rwq_ind_table_init_attr *init_attr); + int (*exp_destroy_wq)(struct ibv_exp_wq *wq); + int (*exp_modify_wq)(struct ibv_exp_wq *wq, + struct ibv_exp_wq_attr *wq_attr); + struct ibv_exp_wq * (*exp_create_wq)(struct ibv_context *context, + struct ibv_exp_wq_init_attr *wq_init_attr); + int (*drv_exp_poll_dc_info)(struct ibv_context *context, + struct ibv_exp_dc_info_ent *ents, + int nent, int port); + void *(*exp_query_intf)(struct ibv_context *context, struct ibv_exp_query_intf_params *params, + enum ibv_exp_query_intf_status *status); + int (*exp_release_intf)(struct ibv_context *context, void *intf, + struct ibv_exp_release_intf_params *params); + struct ibv_exp_res_domain *(*exp_create_res_domain)(struct ibv_context *context, + struct ibv_exp_res_domain_init_attr *attr); + int (*exp_destroy_res_domain)(struct ibv_context *context, + struct ibv_exp_res_domain *res_dom, + struct ibv_exp_destroy_res_domain_attr *attr); + int (*lib_exp_use_priv_env)(struct ibv_context *context); + int (*lib_exp_setenv)(struct ibv_context *context, const char *name, + const char *value, int overwrite); + struct verbs_environment *venv; + int (*drv_exp_dereg_mr)(struct ibv_mr *mr, struct ibv_exp_dereg_out *out); + int (*exp_rereg_mr)(struct ibv_mr *mr, int flags, struct ibv_pd *pd, + void *addr, size_t length, uint64_t access, + struct ibv_exp_rereg_mr_attr *attr); + int (*drv_exp_rereg_mr)(struct ibv_mr *mr, int flags, struct ibv_pd *pd, + void *addr, size_t length, uint64_t access, + struct ibv_exp_rereg_mr_attr *attr, struct ibv_exp_rereg_out *out); + int (*drv_exp_prefetch_mr)(struct ibv_mr *mr, + struct ibv_exp_prefetch_attr *attr); + int (*lib_exp_prefetch_mr)(struct ibv_mr *mr, + struct ibv_exp_prefetch_attr *attr); + struct ibv_exp_mkey_list_container * (*drv_exp_alloc_mkey_list_memory)(struct ibv_exp_mkey_list_container_attr *attr); + struct ibv_exp_mkey_list_container * (*lib_exp_alloc_mkey_list_memory)(struct ibv_exp_mkey_list_container_attr *attr); + int (*drv_exp_dealloc_mkey_list_memory)(struct ibv_exp_mkey_list_container *mem); + int (*lib_exp_dealloc_mkey_list_memory)(struct ibv_exp_mkey_list_container *mem); + int (*drv_exp_query_mkey)(struct ibv_mr *mr, struct ibv_exp_mkey_attr *mkey_attr); + int (*lib_exp_query_mkey)(struct ibv_mr *mr, struct ibv_exp_mkey_attr *mkey_attr); + struct ibv_mr * (*drv_exp_create_mr)(struct ibv_exp_create_mr_in *in); + struct ibv_mr * (*lib_exp_create_mr)(struct ibv_exp_create_mr_in *in); + int (*drv_exp_arm_dct)(struct ibv_exp_dct *dct, struct ibv_exp_arm_attr *attr); + int (*lib_exp_arm_dct)(struct ibv_exp_dct *dct, struct ibv_exp_arm_attr *attr); + int (*drv_exp_bind_mw)(struct ibv_exp_mw_bind *mw_bind); + int (*lib_exp_bind_mw)(struct ibv_exp_mw_bind *mw_bind); + int (*drv_exp_post_send)(struct ibv_qp *qp, + struct ibv_exp_send_wr *wr, + struct ibv_exp_send_wr **bad_wr); + struct ibv_mr * (*drv_exp_reg_mr)(struct ibv_exp_reg_mr_in *in); + struct ibv_mr * (*lib_exp_reg_mr)(struct ibv_exp_reg_mr_in *in); + struct ibv_ah * (*drv_exp_ibv_create_ah)(struct ibv_pd *pd, + struct ibv_exp_ah_attr *attr_exp); + int (*drv_exp_query_values)(struct ibv_context *context, int q_values, + struct ibv_exp_values *values); + struct ibv_cq * (*exp_create_cq)(struct ibv_context *context, int cqe, + struct ibv_comp_channel *channel, + int comp_vector, struct ibv_exp_cq_init_attr *attr); + int (*drv_exp_ibv_poll_cq)(struct ibv_cq *ibcq, int num_entries, + struct ibv_exp_wc *wc, uint32_t wc_size); + void * (*drv_exp_get_legacy_xrc) (struct ibv_srq *ibv_srq); + void (*drv_exp_set_legacy_xrc) (struct ibv_srq *ibv_srq, void *legacy_xrc); + struct ibv_mr * (*drv_exp_ibv_reg_shared_mr)(struct ibv_exp_reg_shared_mr_in *in); + struct ibv_mr * (*lib_exp_ibv_reg_shared_mr)(struct ibv_exp_reg_shared_mr_in *in); + int (*drv_exp_modify_qp)(struct ibv_qp *qp, struct ibv_exp_qp_attr *attr, + uint64_t exp_attr_mask); + int (*lib_exp_modify_qp)(struct ibv_qp *qp, struct ibv_exp_qp_attr *attr, + uint64_t exp_attr_mask); + int (*drv_exp_post_task)(struct ibv_context *context, + struct ibv_exp_task *task, + struct ibv_exp_task **bad_task); + int (*lib_exp_post_task)(struct ibv_context *context, + struct ibv_exp_task *task, + struct ibv_exp_task **bad_task); + int (*drv_exp_modify_cq)(struct ibv_cq *cq, + struct ibv_exp_cq_attr *attr, int attr_mask); + int (*lib_exp_modify_cq)(struct ibv_cq *cq, + struct ibv_exp_cq_attr *attr, int attr_mask); + int (*drv_exp_ibv_destroy_flow) (struct ibv_exp_flow *flow); + int (*lib_exp_ibv_destroy_flow) (struct ibv_exp_flow *flow); + struct ibv_exp_flow * (*drv_exp_ibv_create_flow) (struct ibv_qp *qp, + struct ibv_exp_flow_attr + *flow_attr); + struct ibv_exp_flow * (*lib_exp_ibv_create_flow) (struct ibv_qp *qp, + struct ibv_exp_flow_attr + *flow_attr); + + int (*drv_exp_query_port)(struct ibv_context *context, uint8_t port_num, + struct ibv_exp_port_attr *port_attr); + int (*lib_exp_query_port)(struct ibv_context *context, uint8_t port_num, + struct ibv_exp_port_attr *port_attr); + struct ibv_exp_dct *(*create_dct)(struct ibv_context *context, + struct ibv_exp_dct_init_attr *attr); + int (*destroy_dct)(struct ibv_exp_dct *dct); + int (*query_dct)(struct ibv_exp_dct *dct, struct ibv_exp_dct_attr *attr); + int (*drv_exp_query_device)(struct ibv_context *context, + struct ibv_exp_device_attr *attr); + int (*lib_exp_query_device)(struct ibv_context *context, + struct ibv_exp_device_attr *attr); + struct ibv_qp *(*drv_exp_create_qp)(struct ibv_context *context, + struct ibv_exp_qp_init_attr *init_attr); + struct ibv_qp *(*lib_exp_create_qp)(struct ibv_context *context, + struct ibv_exp_qp_init_attr *init_attr); + size_t sz; /* Set by library on struct allocation, */ + /* must be located as last field */ +}; + + +static inline struct verbs_context_exp *verbs_get_exp_ctx(struct ibv_context *ctx) +{ + struct verbs_context *app_ex_ctx = verbs_get_ctx(ctx); + char *actual_ex_ctx; + + if (!app_ex_ctx || !(app_ex_ctx->has_comp_mask & VERBS_CONTEXT_EXP)) + return NULL; + + actual_ex_ctx = ((char *)ctx) - (app_ex_ctx->sz - sizeof(struct ibv_context)); + + return (struct verbs_context_exp *)(actual_ex_ctx - sizeof(struct verbs_context_exp)); +} + +#define verbs_get_exp_ctx_op(ctx, op) ({ \ + struct verbs_context_exp *_vctx = verbs_get_exp_ctx(ctx); \ + (!_vctx || (_vctx->sz < sizeof(*_vctx) - offsetof(struct verbs_context_exp, op)) || \ + !_vctx->op) ? NULL : _vctx; }) + +#define verbs_set_exp_ctx_op(_vctx, op, ptr) ({ \ + struct verbs_context_exp *vctx = _vctx; \ + if (vctx && (vctx->sz >= sizeof(*vctx) - offsetof(struct verbs_context_exp, op))) \ + vctx->op = ptr; }) + + +/* + * ibv_exp_alloc_ec_calc() - allocate an erasure coding + * calculation offload context + * @pd: user allocated protection domain + * @attrs: initialization attributes + * + * Returns handle handle to the EC calculation APIs + */ +static inline struct ibv_exp_ec_calc * +ibv_exp_alloc_ec_calc(struct ibv_pd *pd, + struct ibv_exp_ec_calc_init_attr *attr) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(pd->context, alloc_ec_calc); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_EC_CALC_INIT_ATTR_RESERVED - 1); + + return vctx->alloc_ec_calc(pd, attr); +} + +/* + * ibv_exp_dealloc_ec_calc() - free an erasure coding + * calculation offload context + * @ec_calc: ec context + */ +static inline void ibv_exp_dealloc_ec_calc(struct ibv_exp_ec_calc *calc) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(calc->pd->context, dealloc_ec_calc); + if (!vctx) { + errno = ENOSYS; + return; + } + + vctx->dealloc_ec_calc(calc); +} + +/** + * ibv_exp_ec_encode_async() - asynchronous encode of given data blocks + * and place in code_blocks + * @ec_calc: erasure coding calculation engine + * @ec_mem: erasure coding memory layout + * @ec_comp: EC calculation completion context + * + * Restrictions: + * - ec_calc is an initialized erasure coding calc engine structure + * - ec_mem.data_blocks sg array must describe the data memory + * layout, the total length of the sg elements must satisfy + * k * ec_mem.block_size. + * - ec_mem.num_data_sg must not exceed the calc max_data_sge + * - ec_mem.code_blocks sg array must describe the code memory + * layout, the total length of the sg elements must satisfy + * m * ec_mem.block_size. + * - ec_mem.num_code_sg must not exceed the calc max_code_sge + * + * Notes: + * The ec_calc will perform the erasure coding calc operation, + * once it completes, it will call ec_comp->done() handle. + * The caller will take it from there. + */ +static inline int +ibv_exp_ec_encode_async(struct ibv_exp_ec_calc *calc, + struct ibv_exp_ec_mem *ec_mem, + struct ibv_exp_ec_comp *ec_comp) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_encode_async); + if (!vctx) + return ENOSYS; + + return vctx->ec_encode_async(calc, ec_mem, ec_comp); +} + +/** + * ibv_exp_ec_encode_sync() - synchronous encode of given data blocks + * and place in code_blocks + * @ec_calc: erasure coding calculation engine + * @ec_mem: erasure coding memory layout + * + * Restrictions: + * - ec_calc is an initialized erasure coding calc engine structure + * - ec_mem.data_blocks sg array must describe the data memory + * layout, the total length of the sg elements must satisfy + * k * ec_mem.block_size. + * - ec_mem.num_data_sg must not exceed the calc max_data_sge + * - ec_mem.code_blocks sg array must describe the code memory + * layout, the total length of the sg elements must satisfy + * m * ec_mem.block_size. + * - ec_mem.num_code_sg must not exceed the calc max_code_sge + * + * Returns 0 on success, non-zero on failure. + * + */ +static inline int +ibv_exp_ec_encode_sync(struct ibv_exp_ec_calc *calc, + struct ibv_exp_ec_mem *ec_mem) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_encode_sync); + if (!vctx) + return ENOSYS; + + return vctx->ec_encode_sync(calc, ec_mem); +} + +/** + * ibv_exp_ec_decode_async() - decode a given set of data blocks + * and code_blocks and place into output recovery blocks + * @ec_calc: erasure coding calculation engine + * @ec_mem: erasure coding memory layout + * @erasures: pointer to byte-map of which blocks were erased + * and needs to be recovered + * @decode_matrix: buffer that contains the decode matrix + * @ec_comp: EC calculation completion context + * + * Restrictions: + * - ec_calc is an initialized erasure coding calc engine structure + * - ec_mem.data_blocks sg array must describe the data memory + * layout, the total length of the sg elements must satisfy + * k * ec_mem.block_size. + * - ec_mem.num_data_sg must not exceed the calc max_data_sge + * - ec_mem.code_blocks sg array must describe the code memory + * layout, the total length of the sg elements must satisfy + * number of missing blocks * ec_mem.block_size. + * - ec_mem.num_code_sg must not exceed the calc max_code_sge + * - erasures byte-mask consists of the survived and erased blocks. + * The first k bytes stand for the k data blocks followed by + * m bytes that stand for the code blocks. + * + * Returns 0 on success, or non-zero on failure with a corresponding + * errno. + * + * Notes: + * The ec_calc will perform the erasure coding calc operation, + * once it completes, it will call ec_comp->done() handle. + * The caller will take it from there + */ +static inline int +ibv_exp_ec_decode_async(struct ibv_exp_ec_calc *calc, + struct ibv_exp_ec_mem *ec_mem, + uint8_t *erasures, + uint8_t *decode_matrix, + struct ibv_exp_ec_comp *ec_comp) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_decode_async); + if (!vctx) + return ENOSYS; + + return vctx->ec_decode_async(calc, ec_mem, erasures, + decode_matrix, ec_comp); +} + +/** + * ibv_exp_ec_decode_sync() - decode a given set of data blocks + * and code_blocks and place into output recovery blocks + * @ec_calc: erasure coding calculation engine + * @ec_mem: erasure coding memory layout + * @erasures: pointer to byte-map of which blocks were erased + * and needs to be recovered + * @decode_matrix: registered buffer of the decode matrix + * + * Restrictions: + * - ec_calc is an initialized erasure coding calc engine structure + * - ec_mem.data_blocks sg array must describe the data memory + * layout, the total length of the sg elements must satisfy + * k * ec_mem.block_size. + * - ec_mem.num_data_sg must not exceed the calc max_data_sge + * - ec_mem.code_blocks sg array must describe the code memory + * layout, the total length of the sg elements must satisfy + * number of missing blocks * ec_mem.block_size. + * - ec_mem.num_code_sg must not exceed the calc max_code_sge + * - erasures byte-map consists of the survived and erased blocks. + * The first k bytes stand for the k data blocks followed by + * m bytes that stand for the code blocks. + * + * Returns 0 on success, or non-zero on failure with a corresponding + * errno. + */ +static inline int +ibv_exp_ec_decode_sync(struct ibv_exp_ec_calc *calc, + struct ibv_exp_ec_mem *ec_mem, + uint8_t *erasures, + uint8_t *decode_matrix) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_decode_sync); + if (!vctx) + return ENOSYS; + + return vctx->ec_decode_sync(calc, ec_mem, erasures, decode_matrix); +} + +/** + * ibv_exp_ec_update_async() - copmutes redundancies based on updated blocks, + * their replacements and old redundancies and place into output code blocks + * @ec_calc: erasure coding calculation engine + * @ec_mem: erasure coding memory layout + * @data_updates: array which is a map of data blocks that are updated + * @code_updates: array which is a map of code blocks to be computed + * @ec_comp: EC calculation completion context + * + * Restrictions: + * - ec_calc is an initialized erasure coding calc engine structure + * - ec_mem.data_blocks sg array must describe the data memory + * layout in the following way: + * assume we want to update blocks d_i and d_j with ipd->context, ec_update_async); + if (!vctx) + return -ENOSYS; + + return vctx->ec_update_async(calc, ec_mem, data_updates, + code_updates, ec_comp); +} + +/** + * ibv_exp_ec_update_sync() - copmutes redundancies based on updated blocks, + * their replacements and old redundancies and place into output code blocks + * @ec_calc: erasure coding calculation engine + * @ec_mem: erasure coding memory layout + * @data_updates: array which is a map of data blocks that are updated + * @code_updates: array which is a map of code blocks to be computed + * + * Restrictions: + * - ec_calc is an initialized erasure coding calc engine structure + * - ec_mem.data_blocks sg array must describe the data memory + * layout in the following way: + * assume we want to update blocks d_i and d_j with ipd->context, ec_update_sync); + if (!vctx) + return -ENOSYS; + + return vctx->ec_update_sync(calc, ec_mem, data_updates, code_updates); +} +/** + * ibv_exp_ec_calc_poll() - poll for EC calculation + * + * @calc: EC calc context + * @n: number of calculations to poll + * + * Returns the number of calc completions processed which + * is lower or equal to n. Relevant only when EC calc context + * was allocated in polling mode. + */ +static inline int +ibv_exp_ec_poll(struct ibv_exp_ec_calc *calc, int n) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_poll); + if (!vctx) + return ENOSYS; + + return vctx->ec_poll(calc, n); +} + +/** + * ibv_exp_ec_encode_send() - encode a given data blocks + * initiate the data and code blocks transfers to the wire with the qps array. + * @ec_calc: erasure coding calculation engine + * @ec_mem: erasure coding memory layout context + * @data_stripes: array of stripe handles, each represents a data block channel + * @code_stripes: array of qp handles, each represents a code block channel + * + * Restrictions: + * - ec_calc is an initialized erasure coding calc engine structure + * - ec_mem.data_blocks sg array must describe the data memory + * layout, the total length of the sg elements must satisfy + * k * ec_mem.block_size. + * - ec_mem.num_data_sg must not exceed the calc max_data_sge + * - ec_mem.code_blocks sg array must describe the code memory + * layout, the total length of the sg elements must satisfy + * m * ec_mem.block_size. + * - ec_mem.num_code_sg must not exceed the calc max_code_sge + * + * Returns 0 on success, or non-zero on failure with a corresponding + * errno. + */ +static inline int +ibv_exp_ec_encode_send(struct ibv_exp_ec_calc *calc, + struct ibv_exp_ec_mem *ec_mem, + struct ibv_exp_ec_stripe *data_stripes, + struct ibv_exp_ec_stripe *code_stripes) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(calc->pd->context, ec_encode_send); + if (!vctx) + return -ENOSYS; + + return vctx->ec_encode_send(calc, ec_mem, data_stripes, code_stripes); +} + +static inline struct ibv_qp * +ibv_exp_create_qp(struct ibv_context *context, struct ibv_exp_qp_init_attr *qp_init_attr) +{ + struct verbs_context_exp *vctx; + uint32_t mask = qp_init_attr->comp_mask; + + if (mask == IBV_EXP_QP_INIT_ATTR_PD) + return ibv_create_qp(qp_init_attr->pd, + (struct ibv_qp_init_attr *) qp_init_attr); + + vctx = verbs_get_exp_ctx_op(context, lib_exp_create_qp); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(qp_init_attr->comp_mask, + IBV_EXP_QP_INIT_ATTR_RESERVED1 - 1); + + return vctx->lib_exp_create_qp(context, qp_init_attr); +} + +/* + * ibv_exp_use_priv_env + * + * switch to use private environment + */ +static inline int ibv_exp_use_priv_env(struct ibv_context *context) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(context, lib_exp_use_priv_env); + if (!vctx) { + errno = ENOSYS; + return -1; + } + + return vctx->lib_exp_use_priv_env(context); +} + +/* + * ibv_exp_poll_dc_info + * + * The function is not thread safe. Any locking must be done by the user. + * + * Return: >= 0 number of returned entries + * < 0 error + * + */ +static inline int ibv_exp_poll_dc_info(struct ibv_context *context, + struct ibv_exp_dc_info_ent *ents, + int nent, int port) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(context, drv_exp_poll_dc_info); + if (!vctx) { + errno = ENOSYS; + return -1; + } + + return vctx->drv_exp_poll_dc_info(context, ents, nent, port); +} + +/* + * ibv_exp_setenv + * + * see man setenv for parameter description + */ +static inline int ibv_exp_setenv(struct ibv_context *context, + const char *name, + const char *value, + int overwrite) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(context, lib_exp_setenv); + if (!vctx) + return setenv(name, value, overwrite); + + return vctx->lib_exp_setenv(context, name, value, overwrite); +} + +static inline int ibv_exp_query_device(struct ibv_context *context, + struct ibv_exp_device_attr *attr) +{ + struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(context, + lib_exp_query_device); + if (!vctx) + return ENOSYS; + + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_DEVICE_ATTR_RESERVED - 1); + return vctx->lib_exp_query_device(context, attr); +} + +static inline struct ibv_exp_dct * +ibv_exp_create_dct(struct ibv_context *context, + struct ibv_exp_dct_init_attr *attr) +{ + struct verbs_context_exp *vctx; + struct ibv_exp_dct *dct; + + vctx = verbs_get_exp_ctx_op(context, create_dct); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + + IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_DCT_INIT_ATTR_RESERVED - 1); + pthread_mutex_lock(&context->mutex); + dct = vctx->create_dct(context, attr); + if (dct) + dct->context = context; + + pthread_mutex_unlock(&context->mutex); + + return dct; +} + +static inline int ibv_exp_destroy_dct(struct ibv_exp_dct *dct) +{ + struct verbs_context_exp *vctx; + struct ibv_context *context = dct->context; + int err; + + vctx = verbs_get_exp_ctx_op(context, destroy_dct); + if (!vctx) { + errno = ENOSYS; + return errno; + } + + pthread_mutex_lock(&context->mutex); + err = vctx->destroy_dct(dct); + pthread_mutex_unlock(&context->mutex); + + return err; +} + +static inline int ibv_exp_query_dct(struct ibv_exp_dct *dct, + struct ibv_exp_dct_attr *attr) +{ + struct verbs_context_exp *vctx; + struct ibv_context *context = dct->context; + int err; + + vctx = verbs_get_exp_ctx_op(context, query_dct); + if (!vctx) { + errno = ENOSYS; + return errno; + } + + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_DCT_ATTR_RESERVED - 1); + pthread_mutex_lock(&context->mutex); + err = vctx->query_dct(dct, attr); + pthread_mutex_unlock(&context->mutex); + + return err; +} + +static inline int ibv_exp_arm_dct(struct ibv_exp_dct *dct, + struct ibv_exp_arm_attr *attr) +{ + struct verbs_context_exp *vctx; + struct ibv_context *context = dct->context; + int err; + + vctx = verbs_get_exp_ctx_op(context, lib_exp_arm_dct); + if (!vctx) { + errno = ENOSYS; + return errno; + } + + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_ARM_ATTR_RESERVED - 1); + pthread_mutex_lock(&context->mutex); + err = vctx->lib_exp_arm_dct(dct, attr); + pthread_mutex_unlock(&context->mutex); + + return err; +} + +static inline int ibv_exp_query_port(struct ibv_context *context, + uint8_t port_num, + struct ibv_exp_port_attr *port_attr) +{ + struct verbs_context_exp *vctx; + + if (0 == port_attr->comp_mask) + return ibv_query_port(context, port_num, + &port_attr->port_attr); + + /* Check that only valid flags were given */ + if ((!port_attr->comp_mask & IBV_EXP_QUERY_PORT_ATTR_MASK1) || + (port_attr->comp_mask & ~IBV_EXP_QUERY_PORT_ATTR_MASKS) || + (port_attr->mask1 & ~IBV_EXP_QUERY_PORT_MASK)) { + errno = EINVAL; + return -errno; + } + + vctx = verbs_get_exp_ctx_op(context, lib_exp_query_port); + + if (!vctx) { + /* Fallback to legacy mode */ + if (port_attr->comp_mask == IBV_EXP_QUERY_PORT_ATTR_MASK1 && + !(port_attr->mask1 & ~IBV_EXP_QUERY_PORT_STD_MASK)) + return ibv_query_port(context, port_num, + &port_attr->port_attr); + + /* Unsupported field was requested */ + errno = ENOSYS; + return -errno; + } + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(port_attr->comp_mask, + IBV_EXP_QUERY_PORT_ATTR_RESERVED - 1); + + return vctx->lib_exp_query_port(context, port_num, port_attr); +} + +/** + * ibv_exp_post_task - Post a list of tasks to different QPs. + */ +static inline int ibv_exp_post_task(struct ibv_context *context, + struct ibv_exp_task *task, + struct ibv_exp_task **bad_task) +{ + struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(context, + lib_exp_post_task); + if (!vctx) + return ENOSYS; + + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(task->comp_mask, + IBV_EXP_TASK_RESERVED - 1); + + return vctx->lib_exp_post_task(context, task, bad_task); +} + +static inline int ibv_exp_query_values(struct ibv_context *context, int q_values, + struct ibv_exp_values *values) +{ + struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(context, + drv_exp_query_values); + if (!vctx) { + errno = ENOSYS; + return -errno; + } + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(values->comp_mask, + IBV_EXP_VALUES_RESERVED - 1); + + return vctx->drv_exp_query_values(context, q_values, values); +} + +static inline struct ibv_exp_flow *ibv_exp_create_flow(struct ibv_qp *qp, + struct ibv_exp_flow_attr *flow) +{ + struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(qp->context, + lib_exp_ibv_create_flow); + if (!vctx || !vctx->lib_exp_ibv_create_flow) + return NULL; + + if (flow->reserved != 0L) { + fprintf(stderr, "%s:%d: flow->reserved must be 0\n", __FUNCTION__, __LINE__); + flow->reserved = 0L; + } + + return vctx->lib_exp_ibv_create_flow(qp, flow); +} + +static inline int ibv_exp_destroy_flow(struct ibv_exp_flow *flow_id) +{ + struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(flow_id->context, + lib_exp_ibv_destroy_flow); + if (!vctx || !vctx->lib_exp_ibv_destroy_flow) + return -ENOSYS; + + return vctx->lib_exp_ibv_destroy_flow(flow_id); +} + +static inline int ibv_exp_poll_cq(struct ibv_cq *ibcq, int num_entries, + struct ibv_exp_wc *wc, uint32_t wc_size) +{ + struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(ibcq->context, + drv_exp_ibv_poll_cq); + if (!vctx) + return -ENOSYS; + + return vctx->drv_exp_ibv_poll_cq(ibcq, num_entries, wc, wc_size); +} + +/** + * ibv_exp_post_send - Post a list of work requests to a send queue. + */ +static inline int ibv_exp_post_send(struct ibv_qp *qp, + struct ibv_exp_send_wr *wr, + struct ibv_exp_send_wr **bad_wr) +{ + struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(qp->context, + drv_exp_post_send); + if (!vctx) + return -ENOSYS; + + return vctx->drv_exp_post_send(qp, wr, bad_wr); +} + +/** + * ibv_exp_reg_shared_mr - Register to an existing shared memory region + * @in - Experimental register shared MR input data. + */ +static inline struct ibv_mr *ibv_exp_reg_shared_mr(struct ibv_exp_reg_shared_mr_in *mr_in) +{ + struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(mr_in->pd->context, + lib_exp_ibv_reg_shared_mr); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(mr_in->comp_mask, + IBV_EXP_REG_SHARED_MR_RESERVED - 1); + + return vctx->lib_exp_ibv_reg_shared_mr(mr_in); +} + +/** + * ibv_exp_modify_cq - Modifies the attributes for the specified CQ. + * @cq: The CQ to modify. + * @cq_attr: Specifies the CQ attributes to modify. + * @cq_attr_mask: A bit-mask used to specify which attributes of the CQ + * are being modified. + */ +static inline int ibv_exp_modify_cq(struct ibv_cq *cq, + struct ibv_exp_cq_attr *cq_attr, + int cq_attr_mask) +{ + struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(cq->context, + lib_exp_modify_cq); + if (!vctx) + return ENOSYS; + + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(cq_attr->comp_mask, + IBV_EXP_CQ_ATTR_RESERVED - 1); + + return vctx->lib_exp_modify_cq(cq, cq_attr, cq_attr_mask); +} + +static inline struct ibv_cq *ibv_exp_create_cq(struct ibv_context *context, + int cqe, + void *cq_context, + struct ibv_comp_channel *channel, + int comp_vector, + struct ibv_exp_cq_init_attr *attr) +{ + struct verbs_context_exp *vctx; + struct ibv_cq *cq; + + vctx = verbs_get_exp_ctx_op(context, exp_create_cq); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + + IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_CQ_INIT_ATTR_RESERVED1 - 1); + pthread_mutex_lock(&context->mutex); + cq = vctx->exp_create_cq(context, cqe, channel, comp_vector, attr); + if (cq) { + cq->context = context; + cq->channel = channel; + if (channel) + ++channel->refcnt; + cq->cq_context = cq_context; + cq->comp_events_completed = 0; + cq->async_events_completed = 0; + pthread_mutex_init(&cq->mutex, NULL); + pthread_cond_init(&cq->cond, NULL); + } + + pthread_mutex_unlock(&context->mutex); + + return cq; +} + +/** + * ibv_exp_modify_qp - Modify a queue pair. + * The argument exp_attr_mask specifies the QP attributes to be modified. + * Use ibv_exp_qp_attr_mask for this argument. + */ +static inline int +ibv_exp_modify_qp(struct ibv_qp *qp, struct ibv_exp_qp_attr *attr, uint64_t exp_attr_mask) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(qp->context, lib_exp_modify_qp); + if (!vctx) { + errno = ENOSYS; + return errno; + } + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_QP_ATTR_RESERVED - 1); + + return vctx->lib_exp_modify_qp(qp, attr, exp_attr_mask); +} + +/** + * ibv_exp_reg_mr - Register a memory region + */ +static inline struct ibv_mr *ibv_exp_reg_mr(struct ibv_exp_reg_mr_in *in) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(in->pd->context, lib_exp_reg_mr); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(in->comp_mask, + IBV_EXP_REG_MR_RESERVED - 1); + + return vctx->lib_exp_reg_mr(in); +} + + +/** + * ibv_exp_bind_mw - Bind a memory window to a region + */ +static inline int ibv_exp_bind_mw(struct ibv_exp_mw_bind *mw_bind) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(mw_bind->mw->context, lib_exp_bind_mw); + if (!vctx) { + errno = ENOSYS; + return errno; + } + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(mw_bind->comp_mask, + IBV_EXP_BIND_MW_RESERVED - 1); + + return vctx->lib_exp_bind_mw(mw_bind); +} + +/** + * ibv_exp_prefetch_mr - Prefetch part of a memory region. + * + * Can be used only with MRs registered with IBV_EXP_ACCESS_ON_DEMAND + * + * Returns 0 on success, + * - ENOSYS libibverbs or provider driver doesn't support the prefetching verb. + * - EFAULT when the range requested is out of the memory region bounds, or when + * parts of it are not part of the process address space. + * - EINVAL when the MR is invalid. + */ +static inline int ibv_exp_prefetch_mr( + struct ibv_mr *mr, + struct ibv_exp_prefetch_attr *attr) +{ + struct verbs_context_exp *vctx = verbs_get_exp_ctx_op(mr->context, + lib_exp_prefetch_mr); + + if (!vctx) { + errno = ENOSYS; + return errno; + } + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_PREFETCH_MR_RESERVED - 1); + + return vctx->lib_exp_prefetch_mr(mr, attr); +} + +typedef int (*drv_exp_post_send_func)(struct ibv_qp *qp, + struct ibv_exp_send_wr *wr, + struct ibv_exp_send_wr **bad_wr); +typedef int (*drv_post_send_func)(struct ibv_qp *qp, struct ibv_send_wr *wr, + struct ibv_send_wr **bad_wr); +typedef int (*drv_exp_poll_cq_func)(struct ibv_cq *ibcq, int num_entries, + struct ibv_exp_wc *wc, uint32_t wc_size); +typedef int (*drv_poll_cq_func)(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc); +typedef int (*drv_post_recv_func)(struct ibv_qp *qp, struct ibv_recv_wr *wr, + struct ibv_recv_wr **bad_wr); + +static inline void *ibv_exp_get_provider_func(struct ibv_context *context, + enum ibv_exp_func_name name) +{ + struct verbs_context_exp *vctx; + + switch (name) { + case IBV_EXP_POST_SEND_FUNC: + vctx = verbs_get_exp_ctx_op(context, drv_exp_post_send); + if (!vctx) + goto error; + + return (void *)vctx->drv_exp_post_send; + + case IBV_EXP_POLL_CQ_FUNC: + vctx = verbs_get_exp_ctx_op(context, drv_exp_ibv_poll_cq); + if (!vctx) + goto error; + + return (void *)vctx->drv_exp_ibv_poll_cq; + + case IBV_POST_SEND_FUNC: + if (!context->ops.post_send) + goto error; + + return (void *)context->ops.post_send; + + case IBV_POLL_CQ_FUNC: + if (!context->ops.poll_cq) + goto error; + + return (void *)context->ops.poll_cq; + + case IBV_POST_RECV_FUNC: + if (!context->ops.post_recv) + goto error; + + return (void *)context->ops.post_recv; + + default: + break; + } + +error: + errno = ENOSYS; + return NULL; +} + +static inline struct ibv_mr *ibv_exp_create_mr(struct ibv_exp_create_mr_in *in) +{ + struct verbs_context_exp *vctx; + struct ibv_mr *mr; + + vctx = verbs_get_exp_ctx_op(in->pd->context, lib_exp_create_mr); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + + IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(in->comp_mask, + IBV_EXP_CREATE_MR_IN_RESERVED - 1); + mr = vctx->lib_exp_create_mr(in); + if (mr) + mr->pd = in->pd; + + return mr; +} + +static inline int ibv_exp_query_mkey(struct ibv_mr *mr, + struct ibv_exp_mkey_attr *attr) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(mr->context, lib_exp_query_mkey); + if (!vctx) { + errno = ENOSYS; + return errno; + } + + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_MKEY_ATTR_RESERVED - 1); + + return vctx->lib_exp_query_mkey(mr, attr); +} + +static inline int ibv_exp_dealloc_mkey_list_memory(struct ibv_exp_mkey_list_container *mem) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(mem->context, + lib_exp_dealloc_mkey_list_memory); + if (!vctx) { + errno = ENOSYS; + return errno; + } + + return vctx->lib_exp_dealloc_mkey_list_memory(mem); +} + +static inline struct ibv_exp_mkey_list_container * +ibv_exp_alloc_mkey_list_memory(struct ibv_exp_mkey_list_container_attr *attr) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(attr->pd->context, + lib_exp_alloc_mkey_list_memory); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + + IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_MKEY_LIST_CONTAINER_RESERVED - 1); + + return vctx->lib_exp_alloc_mkey_list_memory(attr); +} + +/** + * ibv_rereg_mr - Re-Register a memory region + * + * For exp_access use ibv_exp_access_flags + */ +static inline int ibv_exp_rereg_mr(struct ibv_mr *mr, int flags, + struct ibv_pd *pd, void *addr, + size_t length, uint64_t exp_access, + struct ibv_exp_rereg_mr_attr *attr) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(mr->context, exp_rereg_mr); + if (!vctx) + return errno = ENOSYS; + + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_REREG_MR_ATTR_RESERVED - 1); + + return vctx->exp_rereg_mr(mr, flags, pd, addr, length, exp_access, attr); +} + +/** + * ibv_exp_create_res_domain - create resource domain + */ +static inline struct ibv_exp_res_domain *ibv_exp_create_res_domain(struct ibv_context *context, + struct ibv_exp_res_domain_init_attr *attr) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(context, exp_create_res_domain); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + + IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_RES_DOMAIN_RESERVED - 1); + + return vctx->exp_create_res_domain(context, attr); +} + +/** + * ibv_exp_destroy_res_domain - destroy resource domain + */ +static inline int ibv_exp_destroy_res_domain(struct ibv_context *context, + struct ibv_exp_res_domain *res_dom, + struct ibv_exp_destroy_res_domain_attr *attr) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(context, exp_destroy_res_domain); + if (!vctx) + return errno = ENOSYS; + + if (attr) + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_DESTROY_RES_DOMAIN_RESERVED - 1); + + return vctx->exp_destroy_res_domain(context, res_dom, attr); +} + +/** + * ibv_exp_query_intf - query for family of verbs interface for specific QP/CQ + * + * Usually family of data-path verbs. + * Application may call ibv_exp_query_intf for QPs in the following states: + * IBV_QPS_INIT, IBV_QPS_RTR and IBV_QPS_RTS + * + * Returns the family of verbs. + * On failure returns NULL. The failure reason provided by the 'status' + * output variable. + */ +static inline void *ibv_exp_query_intf(struct ibv_context *context, + struct ibv_exp_query_intf_params *params, + enum ibv_exp_query_intf_status *status) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(context, exp_query_intf); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + + IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(params->comp_mask, + IBV_EXP_QUERY_INTF_RESERVED - 1); + + return vctx->exp_query_intf(context, params, status); +} + +/** + * ibv_exp_release_intf - release the queried interface + */ +static inline int ibv_exp_release_intf(struct ibv_context *context, void *intf, + struct ibv_exp_release_intf_params *params) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(context, exp_release_intf); + if (!vctx) + return errno = ENOSYS; + + if (params) + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(params->comp_mask, + IBV_EXP_RELEASE_INTF_RESERVED - 1); + + return vctx->exp_release_intf(context, intf, params); +} + +static inline struct ibv_exp_wq *ibv_exp_create_wq(struct ibv_context *context, + struct ibv_exp_wq_init_attr *wq_init_attr) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(context, exp_create_wq); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + + IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(wq_init_attr->comp_mask, + IBV_EXP_CREATE_WQ_RESERVED - 1); + + return vctx->exp_create_wq(context, wq_init_attr); +} + +static inline int ibv_exp_modify_wq(struct ibv_exp_wq *wq, struct ibv_exp_wq_attr *wq_attr) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(wq->context, exp_modify_wq); + if (!vctx) + return ENOSYS; + + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(wq_attr->attr_mask, + IBV_EXP_WQ_ATTR_RESERVED - 1); + return vctx->exp_modify_wq(wq, wq_attr); +} + +static inline int ibv_exp_destroy_wq(struct ibv_exp_wq *wq) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(wq->context, exp_destroy_wq); + if (!vctx) + return ENOSYS; + + return vctx->exp_destroy_wq(wq); +} + +/* + * ibv_exp_create_rwq_ind_table - Creates a RQ Indirection Table associated + * with the specified protection domain. + * @pd: The protection domain associated with the Indirection Table. + * @ibv_exp_rwq_ind_table_init_attr: A list of initial attributes required to + * create the Indirection Table. + * Return Value + * ibv_exp_create_rwq_ind_table returns a pointer to the created + * Indirection Table, or NULL if the request fails. + */ +static inline struct ibv_exp_rwq_ind_table *ibv_exp_create_rwq_ind_table(struct ibv_context *context, + struct ibv_exp_rwq_ind_table_init_attr *init_attr) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(context, exp_create_rwq_ind_table); + if (!vctx) { + errno = ENOSYS; + return NULL; + } + + IBV_EXP_RET_NULL_ON_INVALID_COMP_MASK(init_attr->comp_mask, + IBV_EXP_CREATE_IND_TABLE_RESERVED - 1); + return vctx->exp_create_rwq_ind_table(context, init_attr); +} + +/* + * ibv_exp_destroy_rwq_ind_table - Destroys the specified Indirection Table. + * @rwq_ind_table: The Indirection Table to destroy. + * Return Value + * ibv_destroy_rwq_ind_table() returns 0 on success, or the value of errno + * on failure (which indicates the failure reason). +*/ +static inline int ibv_exp_destroy_rwq_ind_table(struct ibv_exp_rwq_ind_table *rwq_ind_table) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(rwq_ind_table->context, exp_destroy_rwq_ind_table); + if (!vctx) + return ENOSYS; + + return vctx->exp_destroy_rwq_ind_table(rwq_ind_table); +} + +/* + * ibv_exp_query_gid_attr - query a GID attributes + * @context: ib context + * @port_num: port number + * @index: gid index in the gids table + * @attr: the gid attributes of index in the gids table + * Return value + * ibv_exp_query_gid_attr return 0 on success, or the value of errno on failure. + */ +static inline int ibv_exp_query_gid_attr(struct ibv_context *context, + uint8_t port_num, + unsigned int index, + struct ibv_exp_gid_attr *attr) +{ + struct verbs_context_exp *vctx; + + vctx = verbs_get_exp_ctx_op(context, exp_query_gid_attr); + if (!vctx) + return ENOSYS; + + IBV_EXP_RET_EINVAL_ON_INVALID_COMP_MASK(attr->comp_mask, + IBV_EXP_QUERY_GID_ATTR_RESERVED - 1); + return vctx->exp_query_gid_attr(context, port_num, index, attr); +} +END_C_DECLS + +#define VERBS_MAX_ENV_VAL 4096 + +# undef __attribute_const + + +#endif /* INFINIBAND_VERBS_EXP_H */ diff --git a/external_libs/ibverbs/libibverbs.a b/external_libs/ibverbs/libibverbs.a new file mode 100644 index 00000000..e62d4c4e Binary files /dev/null and b/external_libs/ibverbs/libibverbs.a differ diff --git a/external_libs/ibverbs/libibverbs.la b/external_libs/ibverbs/libibverbs.la new file mode 100644 index 00000000..332deb34 --- /dev/null +++ b/external_libs/ibverbs/libibverbs.la @@ -0,0 +1,41 @@ +# libibverbs.la - a libtool library file +# Generated by libtool (GNU libtool) 2.4.2 +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='libibverbs.so.1' + +# Names of this library. +library_names='libibverbs.so.1.0.0 libibverbs.so.1 libibverbs.so' + +# The name of the static archive. +old_library='libibverbs.a' + +# Linker flags that can not go in dependency_libs. +inherited_linker_flags='' + +# Libraries that this one depends upon. +dependency_libs=' -lnl-route-3 -lnl-3 -lpthread -ldl' + +# Names of additional weak libraries provided by this library +weak_library_names='' + +# Version information for libibverbs. +current=1 +age=0 +revision=0 + +# Is this an already installed library? +installed=no + +# Should we warn about portability when linking against -modules? +shouldnotlink=no + +# Files to dlopen/dlpreopen +dlopen='' +dlpreopen='' + +# Directory that this library needs to be installed in: +libdir='/usr/local/lib' diff --git a/external_libs/ibverbs/libibverbs.lai b/external_libs/ibverbs/libibverbs.lai new file mode 100644 index 00000000..3580def8 --- /dev/null +++ b/external_libs/ibverbs/libibverbs.lai @@ -0,0 +1,41 @@ +# libibverbs.la - a libtool library file +# Generated by libtool (GNU libtool) 2.4.2 +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='libibverbs.so.1' + +# Names of this library. +library_names='libibverbs.so.1.0.0 libibverbs.so.1 libibverbs.so' + +# The name of the static archive. +old_library='libibverbs.a' + +# Linker flags that can not go in dependency_libs. +inherited_linker_flags='' + +# Libraries that this one depends upon. +dependency_libs=' -lnl-route-3 -lnl-3 -lpthread -ldl' + +# Names of additional weak libraries provided by this library +weak_library_names='' + +# Version information for libibverbs. +current=1 +age=0 +revision=0 + +# Is this an already installed library? +installed=yes + +# Should we warn about portability when linking against -modules? +shouldnotlink=no + +# Files to dlopen/dlpreopen +dlopen='' +dlpreopen='' + +# Directory that this library needs to be installed in: +libdir='/usr/local/lib' diff --git a/external_libs/ibverbs/libibverbs.so.1 b/external_libs/ibverbs/libibverbs.so.1 new file mode 100644 index 00000000..bd93569d Binary files /dev/null and b/external_libs/ibverbs/libibverbs.so.1 differ diff --git a/external_libs/ibverbs/libibverbs.so.1.0.0 b/external_libs/ibverbs/libibverbs.so.1.0.0 new file mode 100644 index 00000000..bd93569d Binary files /dev/null and b/external_libs/ibverbs/libibverbs.so.1.0.0 differ diff --git a/external_libs/ibverbs/readme.txt b/external_libs/ibverbs/readme.txt new file mode 100644 index 00000000..c47af5e9 --- /dev/null +++ b/external_libs/ibverbs/readme.txt @@ -0,0 +1,30 @@ +sudo yum install libnl3-devel + +download MLNX_OFED_SRC-3.4-1.0.0.0.tgz + + +open [libibverbs-1.2.1mlnx1-OFED.3.4.0.1.4.34100.src.rpm, + libmlx5-1.2.1mlnx1,] + + +how to open +----------------- + rpm2cpio myrpmfile.rpm | cpio -idmv + tar -xvzf myrpmfile.rpm + ./autogen.sh + ./configure + make + +copy headers +----------------- + copy lib src/.libs to . + copy header to + +for mlx5 driver +----------------- + for mlx5 + sudo make install + then copy the /usr/local/include/infiniband/mlx5_hw.h to header folder + + + -- cgit 1.2.3-korg