diff options
| -rw-r--r-- | build/external/patches/dpdk_20.08/0001-cryptodev-add-symmetric-crypto-data-path-APIs.patch | 1606 | ||||
| -rw-r--r-- | src/plugins/dpdk/CMakeLists.txt | 45 | ||||
| -rw-r--r-- | src/plugins/dpdk/cryptodev/cryptodev_dp_api.c | 1536 | ||||
| -rw-r--r-- | src/vnet/crypto/crypto.c | 2 |
4 files changed, 3188 insertions, 1 deletions
diff --git a/build/external/patches/dpdk_20.08/0001-cryptodev-add-symmetric-crypto-data-path-APIs.patch b/build/external/patches/dpdk_20.08/0001-cryptodev-add-symmetric-crypto-data-path-APIs.patch new file mode 100644 index 00000000000..441e7fe2721 --- /dev/null +++ b/build/external/patches/dpdk_20.08/0001-cryptodev-add-symmetric-crypto-data-path-APIs.patch @@ -0,0 +1,1606 @@ +From 865c50fdf46eaaf9efd6e0a897a836201b0ec5a1 Mon Sep 17 00:00:00 2001 +From: Fan Zhang <roy.fan.zhang@intel.com> +Date: Mon, 27 Jul 2020 14:14:24 +0100 +Subject: [PATCH] cryptodev: add symmetric crypto data-path APIs + +This patch adds data-path APIs for enqueue and dequeue operations to +cryptodev. The APIs support flexible user-define enqueue and dequeue +behaviors and operation modes. The QAT PMD is also updated to +support this feature. + +Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com> +Signed-off-by: Piotr Bronowski <piotrx.bronowski@intel.com> +--- + drivers/common/qat/Makefile | 1 + + drivers/crypto/qat/meson.build | 1 + + drivers/crypto/qat/qat_sym.h | 13 + + drivers/crypto/qat/qat_sym_hw_dp.c | 926 ++++++++++++++++++ + drivers/crypto/qat/qat_sym_pmd.c | 9 +- + lib/librte_cryptodev/rte_crypto.h | 9 + + lib/librte_cryptodev/rte_crypto_sym.h | 44 +- + lib/librte_cryptodev/rte_cryptodev.c | 45 + + lib/librte_cryptodev/rte_cryptodev.h | 336 ++++++- + lib/librte_cryptodev/rte_cryptodev_pmd.h | 36 +- + .../rte_cryptodev_version.map | 8 + + 11 files changed, 1417 insertions(+), 11 deletions(-) + create mode 100644 drivers/crypto/qat/qat_sym_hw_dp.c + +diff --git a/drivers/common/qat/Makefile b/drivers/common/qat/Makefile +index 85d420709..1b71bbbab 100644 +--- a/drivers/common/qat/Makefile ++++ b/drivers/common/qat/Makefile +@@ -42,6 +42,7 @@ endif + SRCS-y += qat_sym.c + SRCS-y += qat_sym_session.c + SRCS-y += qat_sym_pmd.c ++ SRCS-y += qat_sym_hw_dp.c + build_qat = yes + endif + endif +diff --git a/drivers/crypto/qat/meson.build b/drivers/crypto/qat/meson.build +index a225f374a..bc90ec44c 100644 +--- a/drivers/crypto/qat/meson.build ++++ b/drivers/crypto/qat/meson.build +@@ -15,6 +15,7 @@ if dep.found() + qat_sources += files('qat_sym_pmd.c', + 'qat_sym.c', + 'qat_sym_session.c', ++ 'qat_sym_hw_dp.c', + 'qat_asym_pmd.c', + 'qat_asym.c') + qat_ext_deps += dep +diff --git a/drivers/crypto/qat/qat_sym.h b/drivers/crypto/qat/qat_sym.h +index 1a9748849..2d6316130 100644 +--- a/drivers/crypto/qat/qat_sym.h ++++ b/drivers/crypto/qat/qat_sym.h +@@ -264,6 +264,18 @@ qat_sym_process_response(void **op, uint8_t *resp) + } + *op = (void *)rx_op; + } ++ ++int ++qat_sym_dp_configure_service_ctx(struct rte_cryptodev *dev, uint16_t qp_id, ++ struct rte_crypto_dp_service_ctx *service_ctx, ++ enum rte_crypto_dp_service service_type, ++ enum rte_crypto_op_sess_type sess_type, ++ union rte_cryptodev_session_ctx session_ctx, ++ uint8_t is_update); ++ ++int ++qat_sym_get_service_ctx_size(struct rte_cryptodev *dev); ++ + #else + + static inline void +@@ -276,5 +288,6 @@ static inline void + qat_sym_process_response(void **op __rte_unused, uint8_t *resp __rte_unused) + { + } ++ + #endif + #endif /* _QAT_SYM_H_ */ +diff --git a/drivers/crypto/qat/qat_sym_hw_dp.c b/drivers/crypto/qat/qat_sym_hw_dp.c +new file mode 100644 +index 000000000..ce75212ba +--- /dev/null ++++ b/drivers/crypto/qat/qat_sym_hw_dp.c +@@ -0,0 +1,926 @@ ++/* SPDX-License-Identifier: BSD-3-Clause ++ * Copyright(c) 2020 Intel Corporation ++ */ ++ ++#include <rte_cryptodev_pmd.h> ++ ++#include "adf_transport_access_macros.h" ++#include "icp_qat_fw.h" ++#include "icp_qat_fw_la.h" ++ ++#include "qat_sym.h" ++#include "qat_sym_pmd.h" ++#include "qat_sym_session.h" ++#include "qat_qp.h" ++ ++struct qat_sym_dp_service_ctx { ++ struct qat_sym_session *session; ++ uint32_t tail; ++ uint32_t head; ++}; ++ ++static __rte_always_inline int32_t ++qat_sym_dp_get_data(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req, ++ struct rte_crypto_vec *data, uint16_t n_data_vecs) ++{ ++ struct qat_queue *tx_queue; ++ struct qat_sym_op_cookie *cookie; ++ struct qat_sgl *list; ++ uint32_t i; ++ uint32_t total_len; ++ ++ if (likely(n_data_vecs == 1)) { ++ req->comn_mid.src_data_addr = req->comn_mid.dest_data_addr = ++ data[0].iova; ++ req->comn_mid.src_length = req->comn_mid.dst_length = ++ data[0].len; ++ return data[0].len; ++ } ++ ++ if (n_data_vecs == 0 || n_data_vecs > QAT_SYM_SGL_MAX_NUMBER) ++ return -1; ++ ++ total_len = 0; ++ tx_queue = &qp->tx_q; ++ ++ ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags, ++ QAT_COMN_PTR_TYPE_SGL); ++ cookie = qp->op_cookies[tx_queue->tail >> tx_queue->trailz]; ++ list = (struct qat_sgl *)&cookie->qat_sgl_src; ++ ++ for (i = 0; i < n_data_vecs; i++) { ++ list->buffers[i].len = data[i].len; ++ list->buffers[i].resrvd = 0; ++ list->buffers[i].addr = data[i].iova; ++ if (total_len + data[i].len > UINT32_MAX) { ++ QAT_DP_LOG(ERR, "Message too long"); ++ return -1; ++ } ++ total_len += data[i].len; ++ } ++ ++ list->num_bufs = i; ++ req->comn_mid.src_data_addr = req->comn_mid.dest_data_addr = ++ cookie->qat_sgl_src_phys_addr; ++ req->comn_mid.src_length = req->comn_mid.dst_length = 0; ++ return total_len; ++} ++ ++static __rte_always_inline void ++set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param, ++ struct rte_crypto_data *iv, uint32_t iv_len, ++ struct icp_qat_fw_la_bulk_req *qat_req) ++{ ++ /* copy IV into request if it fits */ ++ if (iv_len <= sizeof(cipher_param->u.cipher_IV_array)) ++ rte_memcpy(cipher_param->u.cipher_IV_array, iv->base, iv_len); ++ else { ++ ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET( ++ qat_req->comn_hdr.serv_specif_flags, ++ ICP_QAT_FW_CIPH_IV_64BIT_PTR); ++ cipher_param->u.s.cipher_IV_ptr = iv->iova; ++ } ++} ++ ++#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \ ++ (ICP_QAT_FW_COMN_STATUS_FLAG_OK == \ ++ ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status)) ++ ++static __rte_always_inline void ++qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n) ++{ ++ uint32_t i; ++ ++ for (i = 0; i < n; i++) ++ sta[i] = status; ++} ++ ++static __rte_always_inline void ++submit_one_aead_job(struct qat_sym_session *ctx, ++ struct icp_qat_fw_la_bulk_req *req, struct rte_crypto_data *iv_vec, ++ struct rte_crypto_data *digest_vec, struct rte_crypto_data *aad_vec, ++ union rte_crypto_sym_ofs ofs, uint32_t data_len) ++{ ++ struct icp_qat_fw_la_cipher_req_params *cipher_param = ++ (void *)&req->serv_specif_rqpars; ++ struct icp_qat_fw_la_auth_req_params *auth_param = ++ (void *)((uint8_t *)&req->serv_specif_rqpars + ++ ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); ++ uint8_t *aad_data; ++ uint8_t aad_ccm_real_len; ++ uint8_t aad_len_field_sz; ++ uint32_t msg_len_be; ++ rte_iova_t aad_iova = 0; ++ uint8_t q; ++ ++ switch (ctx->qat_hash_alg) { ++ case ICP_QAT_HW_AUTH_ALGO_GALOIS_128: ++ case ICP_QAT_HW_AUTH_ALGO_GALOIS_64: ++ ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( ++ req->comn_hdr.serv_specif_flags, ++ ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); ++ rte_memcpy(cipher_param->u.cipher_IV_array, ++ iv_vec->base, ctx->cipher_iv.length); ++ aad_iova = aad_vec->iova; ++ break; ++ case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC: ++ aad_data = aad_vec->base; ++ aad_iova = aad_vec->iova; ++ aad_ccm_real_len = 0; ++ aad_len_field_sz = 0; ++ msg_len_be = rte_bswap32((uint32_t)data_len - ++ ofs.ofs.cipher.head); ++ ++ if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) { ++ aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO; ++ aad_ccm_real_len = ctx->aad_len - ++ ICP_QAT_HW_CCM_AAD_B0_LEN - ++ ICP_QAT_HW_CCM_AAD_LEN_INFO; ++ } else { ++ aad_data = iv_vec->base; ++ aad_iova = iv_vec->iova; ++ } ++ ++ q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length; ++ aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS( ++ aad_len_field_sz, ctx->digest_length, q); ++ if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) { ++ memcpy(aad_data + ctx->cipher_iv.length + ++ ICP_QAT_HW_CCM_NONCE_OFFSET + (q - ++ ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE), ++ (uint8_t *)&msg_len_be, ++ ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE); ++ } else { ++ memcpy(aad_data + ctx->cipher_iv.length + ++ ICP_QAT_HW_CCM_NONCE_OFFSET, ++ (uint8_t *)&msg_len_be + ++ (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE ++ - q), q); ++ } ++ ++ if (aad_len_field_sz > 0) { ++ *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] = ++ rte_bswap16(aad_ccm_real_len); ++ ++ if ((aad_ccm_real_len + aad_len_field_sz) ++ % ICP_QAT_HW_CCM_AAD_B0_LEN) { ++ uint8_t pad_len = 0; ++ uint8_t pad_idx = 0; ++ ++ pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN - ++ ((aad_ccm_real_len + ++ aad_len_field_sz) % ++ ICP_QAT_HW_CCM_AAD_B0_LEN); ++ pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN + ++ aad_ccm_real_len + ++ aad_len_field_sz; ++ memset(&aad_data[pad_idx], 0, pad_len); ++ } ++ ++ rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array) ++ + ICP_QAT_HW_CCM_NONCE_OFFSET, ++ (uint8_t *)iv_vec->base + ++ ICP_QAT_HW_CCM_NONCE_OFFSET, ++ ctx->cipher_iv.length); ++ *(uint8_t *)&cipher_param->u.cipher_IV_array[0] = ++ q - ICP_QAT_HW_CCM_NONCE_OFFSET; ++ ++ rte_memcpy((uint8_t *)aad_vec->base + ++ ICP_QAT_HW_CCM_NONCE_OFFSET, ++ (uint8_t *)iv_vec->base + ++ ICP_QAT_HW_CCM_NONCE_OFFSET, ++ ctx->cipher_iv.length); ++ } ++ break; ++ default: ++ break; ++ } ++ ++ cipher_param->cipher_offset = ofs.ofs.cipher.head; ++ cipher_param->cipher_length = data_len - ofs.ofs.cipher.head ++ - ofs.ofs.cipher.tail; ++ auth_param->auth_off = ofs.ofs.cipher.head; ++ auth_param->auth_len = data_len - ofs.ofs.cipher.head ++ - ofs.ofs.cipher.tail; ++ auth_param->auth_res_addr = digest_vec->iova; ++ auth_param->u1.aad_adr = aad_iova; ++ ++ if (ctx->is_single_pass) { ++ cipher_param->spc_aad_addr = aad_iova; ++ cipher_param->spc_auth_res_addr = digest_vec->iova; ++ } ++} ++ ++static __rte_always_inline int ++qat_sym_dp_submit_single_aead(void *qp_data, uint8_t *service_data, ++ struct rte_crypto_vec *data, uint16_t n_data_vecs, ++ union rte_crypto_sym_ofs ofs, struct rte_crypto_data *iv_vec, ++ struct rte_crypto_data *digest_vec, struct rte_crypto_data *aad_vec, ++ void *opaque) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_sym_dp_service_ctx *service_ctx = (void *)service_data; ++ struct qat_queue *tx_queue = &qp->tx_q; ++ struct qat_sym_session *ctx = service_ctx->session; ++ struct icp_qat_fw_la_bulk_req *req; ++ int32_t data_len; ++ uint32_t tail = service_ctx->tail; ++ ++ req = (struct icp_qat_fw_la_bulk_req *)( ++ (uint8_t *)tx_queue->base_addr + tail); ++ tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; ++ rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); ++ rte_prefetch0((uint8_t *)tx_queue->base_addr + tail); ++ data_len = qat_sym_dp_get_data(qp, req, data, n_data_vecs); ++ if (unlikely(data_len < 0)) ++ return -1; ++ req->comn_mid.opaque_data = (uint64_t)opaque; ++ ++ submit_one_aead_job(ctx, req, iv_vec, digest_vec, aad_vec, ofs, ++ (uint32_t)data_len); ++ ++ service_ctx->tail = tail; ++ ++ return 0; ++} ++ ++static __rte_always_inline uint32_t ++qat_sym_dp_submit_aead_jobs(void *qp_data, uint8_t *service_data, ++ struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs, ++ void **opaque) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_sym_dp_service_ctx *service_ctx = (void *)service_data; ++ struct qat_queue *tx_queue = &qp->tx_q; ++ struct qat_sym_session *ctx = service_ctx->session; ++ uint32_t i; ++ uint32_t tail; ++ struct icp_qat_fw_la_bulk_req *req; ++ int32_t data_len; ++ ++ if (unlikely(qp->enqueued - qp->dequeued + vec->num >= ++ qp->max_inflights)) { ++ qat_sym_dp_fill_vec_status(vec->status, -1, vec->num); ++ return 0; ++ } ++ ++ tail = service_ctx->tail; ++ ++ for (i = 0; i < vec->num; i++) { ++ req = (struct icp_qat_fw_la_bulk_req *)( ++ (uint8_t *)tx_queue->base_addr + tail); ++ rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); ++ ++ data_len = qat_sym_dp_get_data(qp, req, vec->sgl[i].vec, ++ vec->sgl[i].num) - ofs.ofs.cipher.head - ++ ofs.ofs.cipher.tail; ++ if (unlikely(data_len < 0)) ++ break; ++ req->comn_mid.opaque_data = (uint64_t)opaque[i]; ++ submit_one_aead_job(ctx, req, vec->iv_vec + i, ++ vec->digest_vec + i, vec->aad_vec + i, ofs, ++ (uint32_t)data_len); ++ tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; ++ } ++ ++ if (unlikely(i < vec->num)) ++ qat_sym_dp_fill_vec_status(vec->status + i, -1, vec->num - i); ++ ++ service_ctx->tail = tail; ++ return i; ++} ++ ++static __rte_always_inline void ++submit_one_cipher_job(struct qat_sym_session *ctx, ++ struct icp_qat_fw_la_bulk_req *req, struct rte_crypto_data *iv_vec, ++ union rte_crypto_sym_ofs ofs, uint32_t data_len) ++{ ++ struct icp_qat_fw_la_cipher_req_params *cipher_param; ++ ++ cipher_param = (void *)&req->serv_specif_rqpars; ++ ++ /* cipher IV */ ++ set_cipher_iv(cipher_param, iv_vec, ctx->cipher_iv.length, req); ++ cipher_param->cipher_offset = ofs.ofs.cipher.head; ++ cipher_param->cipher_length = data_len - ofs.ofs.cipher.head ++ - ofs.ofs.cipher.tail; ++} ++ ++static __rte_always_inline int ++qat_sym_dp_submit_single_cipher(void *qp_data, uint8_t *service_data, ++ struct rte_crypto_vec *data, uint16_t n_data_vecs, ++ union rte_crypto_sym_ofs ofs, struct rte_crypto_data *iv_vec, ++ __rte_unused struct rte_crypto_data *digest_vec, ++ __rte_unused struct rte_crypto_data *aad_vec, ++ void *opaque) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_sym_dp_service_ctx *service_ctx = (void *)service_data; ++ struct qat_queue *tx_queue = &qp->tx_q; ++ struct qat_sym_session *ctx = service_ctx->session; ++ struct icp_qat_fw_la_bulk_req *req; ++ int32_t data_len; ++ uint32_t tail = service_ctx->tail; ++ ++ req = (struct icp_qat_fw_la_bulk_req *)( ++ (uint8_t *)tx_queue->base_addr + tail); ++ tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; ++ rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); ++ rte_prefetch0((uint8_t *)tx_queue->base_addr + tail); ++ data_len = qat_sym_dp_get_data(qp, req, data, n_data_vecs); ++ if (unlikely(data_len < 0)) ++ return -1; ++ req->comn_mid.opaque_data = (uint64_t)opaque; ++ ++ submit_one_cipher_job(ctx, req, iv_vec, ofs, (uint32_t)data_len); ++ ++ service_ctx->tail = tail; ++ ++ return 0; ++} ++ ++static __rte_always_inline uint32_t ++qat_sym_dp_submit_cipher_jobs(void *qp_data, uint8_t *service_data, ++ struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs, ++ void **opaque) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_sym_dp_service_ctx *service_ctx = (void *)service_data; ++ struct qat_queue *tx_queue = &qp->tx_q; ++ struct qat_sym_session *ctx = service_ctx->session; ++ uint32_t i; ++ uint32_t tail; ++ struct icp_qat_fw_la_bulk_req *req; ++ int32_t data_len; ++ ++ if (unlikely(qp->enqueued - qp->dequeued + vec->num >= ++ qp->max_inflights)) { ++ qat_sym_dp_fill_vec_status(vec->status, -1, vec->num); ++ return 0; ++ } ++ ++ tail = service_ctx->tail; ++ ++ for (i = 0; i < vec->num; i++) { ++ req = (struct icp_qat_fw_la_bulk_req *)( ++ (uint8_t *)tx_queue->base_addr + tail); ++ rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); ++ ++ data_len = qat_sym_dp_get_data(qp, req, vec->sgl[i].vec, ++ vec->sgl[i].num) - ofs.ofs.cipher.head - ++ ofs.ofs.cipher.tail; ++ if (unlikely(data_len < 0)) ++ break; ++ req->comn_mid.opaque_data = (uint64_t)opaque[i]; ++ submit_one_cipher_job(ctx, req, vec->iv_vec + i, ofs, ++ (uint32_t)data_len); ++ tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; ++ } ++ ++ if (unlikely(i < vec->num)) ++ qat_sym_dp_fill_vec_status(vec->status + i, -1, vec->num - i); ++ ++ service_ctx->tail = tail; ++ return i; ++} ++ ++static __rte_always_inline void ++submit_one_auth_job(struct qat_sym_session *ctx, ++ struct icp_qat_fw_la_bulk_req *req, struct rte_crypto_data *iv_vec, ++ struct rte_crypto_data *digest_vec, union rte_crypto_sym_ofs ofs, ++ uint32_t data_len) ++{ ++ struct icp_qat_fw_la_cipher_req_params *cipher_param; ++ struct icp_qat_fw_la_auth_req_params *auth_param; ++ ++ cipher_param = (void *)&req->serv_specif_rqpars; ++ auth_param = (void *)((uint8_t *)cipher_param + ++ ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); ++ ++ auth_param->auth_off = ofs.ofs.auth.head; ++ auth_param->auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail; ++ auth_param->auth_res_addr = digest_vec->iova; ++ ++ switch (ctx->qat_hash_alg) { ++ case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2: ++ case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9: ++ case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3: ++ auth_param->u1.aad_adr = iv_vec->iova; ++ break; ++ case ICP_QAT_HW_AUTH_ALGO_GALOIS_128: ++ case ICP_QAT_HW_AUTH_ALGO_GALOIS_64: ++ ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( ++ req->comn_hdr.serv_specif_flags, ++ ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); ++ rte_memcpy(cipher_param->u.cipher_IV_array, ++ iv_vec->base, ctx->cipher_iv.length); ++ break; ++ default: ++ break; ++ } ++} ++ ++static __rte_always_inline int ++qat_sym_dp_submit_single_auth(void *qp_data, uint8_t *service_data, ++ struct rte_crypto_vec *data, uint16_t n_data_vecs, ++ union rte_crypto_sym_ofs ofs, struct rte_crypto_data *iv_vec, ++ struct rte_crypto_data *digest_vec, ++ __rte_unused struct rte_crypto_data *aad_vec, ++ void *opaque) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_sym_dp_service_ctx *service_ctx = (void *)service_data; ++ struct qat_queue *tx_queue = &qp->tx_q; ++ struct qat_sym_session *ctx = service_ctx->session; ++ struct icp_qat_fw_la_bulk_req *req; ++ int32_t data_len; ++ uint32_t tail = service_ctx->tail; ++ ++ req = (struct icp_qat_fw_la_bulk_req *)( ++ (uint8_t *)tx_queue->base_addr + tail); ++ tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; ++ rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); ++ rte_prefetch0((uint8_t *)tx_queue->base_addr + tail); ++ data_len = qat_sym_dp_get_data(qp, req, data, n_data_vecs); ++ if (unlikely(data_len < 0)) ++ return -1; ++ req->comn_mid.opaque_data = (uint64_t)opaque; ++ ++ submit_one_auth_job(ctx, req, iv_vec, digest_vec, ofs, ++ (uint32_t)data_len); ++ ++ service_ctx->tail = tail; ++ ++ return 0; ++} ++ ++static __rte_always_inline uint32_t ++qat_sym_dp_submit_auth_jobs(void *qp_data, uint8_t *service_data, ++ struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs, ++ void **opaque) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_sym_dp_service_ctx *service_ctx = (void *)service_data; ++ struct qat_queue *tx_queue = &qp->tx_q; ++ struct qat_sym_session *ctx = service_ctx->session; ++ uint32_t i; ++ uint32_t tail; ++ struct icp_qat_fw_la_bulk_req *req; ++ int32_t data_len; ++ ++ if (unlikely(qp->enqueued - qp->dequeued + vec->num >= ++ qp->max_inflights)) { ++ qat_sym_dp_fill_vec_status(vec->status, -1, vec->num); ++ return 0; ++ } ++ ++ tail = service_ctx->tail; ++ ++ for (i = 0; i < vec->num; i++) { ++ req = (struct icp_qat_fw_la_bulk_req *)( ++ (uint8_t *)tx_queue->base_addr + tail); ++ rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); ++ ++ data_len = qat_sym_dp_get_data(qp, req, vec->sgl[i].vec, ++ vec->sgl[i].num) - ofs.ofs.cipher.head - ++ ofs.ofs.cipher.tail; ++ if (unlikely(data_len < 0)) ++ break; ++ req->comn_mid.opaque_data = (uint64_t)opaque[i]; ++ submit_one_auth_job(ctx, req, vec->iv_vec + i, ++ vec->digest_vec + i, ofs, (uint32_t)data_len); ++ tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; ++ } ++ ++ if (unlikely(i < vec->num)) ++ qat_sym_dp_fill_vec_status(vec->status + i, -1, vec->num - i); ++ ++ service_ctx->tail = tail; ++ return i; ++} ++ ++static __rte_always_inline void ++submit_one_chain_job(struct qat_sym_session *ctx, ++ struct icp_qat_fw_la_bulk_req *req, struct rte_crypto_vec *data, ++ uint16_t n_data_vecs, struct rte_crypto_data *iv_vec, ++ struct rte_crypto_data *digest_vec, union rte_crypto_sym_ofs ofs, ++ uint32_t data_len) ++{ ++ struct icp_qat_fw_la_cipher_req_params *cipher_param; ++ struct icp_qat_fw_la_auth_req_params *auth_param; ++ rte_iova_t auth_iova_end; ++ int32_t cipher_len, auth_len; ++ ++ cipher_param = (void *)&req->serv_specif_rqpars; ++ auth_param = (void *)((uint8_t *)cipher_param + ++ ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); ++ ++ cipher_len = data_len - ofs.ofs.cipher.head - ++ ofs.ofs.cipher.tail; ++ auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail; ++ ++ assert(cipher_len > 0 && auth_len > 0); ++ ++ cipher_param->cipher_offset = ofs.ofs.cipher.head; ++ cipher_param->cipher_length = cipher_len; ++ set_cipher_iv(cipher_param, iv_vec, ctx->cipher_iv.length, req); ++ ++ auth_param->auth_off = ofs.ofs.auth.head; ++ auth_param->auth_len = auth_len; ++ auth_param->auth_res_addr = digest_vec->iova; ++ ++ switch (ctx->qat_hash_alg) { ++ case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2: ++ case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9: ++ case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3: ++ auth_param->u1.aad_adr = iv_vec->iova; ++ ++ if (unlikely(n_data_vecs > 1)) { ++ int auth_end_get = 0, i = n_data_vecs - 1; ++ struct rte_crypto_vec *cvec = &data[i]; ++ uint32_t len; ++ ++ len = data_len - ofs.ofs.auth.tail; ++ ++ while (i >= 0 && len > 0) { ++ if (cvec->len >= len) { ++ auth_iova_end = cvec->iova + ++ (cvec->len - len); ++ len = 0; ++ auth_end_get = 1; ++ break; ++ } ++ len -= cvec->len; ++ i--; ++ cvec--; ++ } ++ ++ assert(auth_end_get != 0); ++ } else ++ auth_iova_end = digest_vec->iova + ++ ctx->digest_length; ++ ++ /* Then check if digest-encrypted conditions are met */ ++ if ((auth_param->auth_off + auth_param->auth_len < ++ cipher_param->cipher_offset + ++ cipher_param->cipher_length) && ++ (digest_vec->iova == auth_iova_end)) { ++ /* Handle partial digest encryption */ ++ if (cipher_param->cipher_offset + ++ cipher_param->cipher_length < ++ auth_param->auth_off + ++ auth_param->auth_len + ++ ctx->digest_length) ++ req->comn_mid.dst_length = ++ req->comn_mid.src_length = ++ auth_param->auth_off + ++ auth_param->auth_len + ++ ctx->digest_length; ++ struct icp_qat_fw_comn_req_hdr *header = ++ &req->comn_hdr; ++ ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET( ++ header->serv_specif_flags, ++ ICP_QAT_FW_LA_DIGEST_IN_BUFFER); ++ } ++ break; ++ case ICP_QAT_HW_AUTH_ALGO_GALOIS_128: ++ case ICP_QAT_HW_AUTH_ALGO_GALOIS_64: ++ break; ++ default: ++ break; ++ } ++} ++ ++static __rte_always_inline int ++qat_sym_dp_submit_single_chain(void *qp_data, uint8_t *service_data, ++ struct rte_crypto_vec *data, uint16_t n_data_vecs, ++ union rte_crypto_sym_ofs ofs, struct rte_crypto_data *iv_vec, ++ struct rte_crypto_data *digest_vec, ++ __rte_unused struct rte_crypto_data *aad_vec, ++ void *opaque) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_sym_dp_service_ctx *service_ctx = (void *)service_data; ++ struct qat_queue *tx_queue = &qp->tx_q; ++ struct qat_sym_session *ctx = service_ctx->session; ++ struct icp_qat_fw_la_bulk_req *req; ++ int32_t data_len; ++ uint32_t tail = service_ctx->tail; ++ ++ req = (struct icp_qat_fw_la_bulk_req *)( ++ (uint8_t *)tx_queue->base_addr + tail); ++ tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; ++ rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); ++ rte_prefetch0((uint8_t *)tx_queue->base_addr + tail); ++ data_len = qat_sym_dp_get_data(qp, req, data, n_data_vecs); ++ if (unlikely(data_len < 0)) ++ return -1; ++ req->comn_mid.opaque_data = (uint64_t)opaque; ++ ++ submit_one_chain_job(ctx, req, data, n_data_vecs, iv_vec, digest_vec, ++ ofs, (uint32_t)data_len); ++ ++ service_ctx->tail = tail; ++ ++ return 0; ++} ++ ++static __rte_always_inline uint32_t ++qat_sym_dp_submit_chain_jobs(void *qp_data, uint8_t *service_data, ++ struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs, ++ void **opaque) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_sym_dp_service_ctx *service_ctx = (void *)service_data; ++ struct qat_queue *tx_queue = &qp->tx_q; ++ struct qat_sym_session *ctx = service_ctx->session; ++ uint32_t i; ++ uint32_t tail; ++ struct icp_qat_fw_la_bulk_req *req; ++ int32_t data_len; ++ ++ if (unlikely(qp->enqueued - qp->dequeued + vec->num >= ++ qp->max_inflights)) { ++ qat_sym_dp_fill_vec_status(vec->status, -1, vec->num); ++ return 0; ++ } ++ ++ tail = service_ctx->tail; ++ ++ for (i = 0; i < vec->num; i++) { ++ req = (struct icp_qat_fw_la_bulk_req *)( ++ (uint8_t *)tx_queue->base_addr + tail); ++ rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req)); ++ ++ data_len = qat_sym_dp_get_data(qp, req, vec->sgl[i].vec, ++ vec->sgl[i].num) - ofs.ofs.cipher.head - ++ ofs.ofs.cipher.tail; ++ if (unlikely(data_len < 0)) ++ break; ++ req->comn_mid.opaque_data = (uint64_t)opaque[i]; ++ submit_one_chain_job(ctx, req, vec->sgl[i].vec, vec->sgl[i].num, ++ vec->iv_vec + i, vec->digest_vec + i, ofs, ++ (uint32_t)data_len); ++ tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask; ++ } ++ ++ if (unlikely(i < vec->num)) ++ qat_sym_dp_fill_vec_status(vec->status + i, -1, vec->num - i); ++ ++ service_ctx->tail = tail; ++ return i; ++} ++ ++static __rte_always_inline uint32_t ++qat_sym_dp_dequeue(void *qp_data, uint8_t *service_data, ++ rte_cryptodev_get_dequeue_count_t get_dequeue_count, ++ rte_cryptodev_post_dequeue_t post_dequeue, ++ void **out_opaque, uint8_t is_opaque_array, ++ uint32_t *n_success_jobs) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_sym_dp_service_ctx *service_ctx = (void *)service_data; ++ struct qat_queue *rx_queue = &qp->rx_q; ++ struct icp_qat_fw_comn_resp *resp; ++ void *resp_opaque; ++ uint32_t i, n, inflight; ++ uint32_t head; ++ uint8_t status; ++ ++ *n_success_jobs = 0; ++ head = service_ctx->head; ++ ++ inflight = qp->enqueued - qp->dequeued; ++ if (unlikely(inflight == 0)) ++ return 0; ++ ++ resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr + ++ head); ++ /* no operation ready */ ++ if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG)) ++ return 0; ++ ++ resp_opaque = (void *)(uintptr_t)resp->opaque_data; ++ /* get the dequeue count */ ++ n = get_dequeue_count(resp_opaque); ++ if (unlikely(n == 0)) ++ return 0; ++ ++ out_opaque[0] = resp_opaque; ++ status = QAT_SYM_DP_IS_RESP_SUCCESS(resp); ++ post_dequeue(resp_opaque, 0, status); ++ *n_success_jobs += status; ++ ++ head = (head + rx_queue->msg_size) & rx_queue->modulo_mask; ++ ++ /* we already finished dequeue when n == 1 */ ++ if (unlikely(n == 1)) { ++ i = 1; ++ goto end_deq; ++ } ++ ++ if (is_opaque_array) { ++ for (i = 1; i < n; i++) { ++ resp = (struct icp_qat_fw_comn_resp *)( ++ (uint8_t *)rx_queue->base_addr + head); ++ if (unlikely(*(uint32_t *)resp == ++ ADF_RING_EMPTY_SIG)) ++ goto end_deq; ++ out_opaque[i] = (void *)(uintptr_t) ++ resp->opaque_data; ++ status = QAT_SYM_DP_IS_RESP_SUCCESS(resp); ++ *n_success_jobs += status; ++ post_dequeue(out_opaque[i], i, status); ++ head = (head + rx_queue->msg_size) & ++ rx_queue->modulo_mask; ++ } ++ ++ goto end_deq; ++ } ++ ++ /* opaque is not array */ ++ for (i = 1; i < n; i++) { ++ resp = (struct icp_qat_fw_comn_resp *)( ++ (uint8_t *)rx_queue->base_addr + head); ++ status = QAT_SYM_DP_IS_RESP_SUCCESS(resp); ++ if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG)) ++ goto end_deq; ++ head = (head + rx_queue->msg_size) & ++ rx_queue->modulo_mask; ++ post_dequeue(resp_opaque, i, status); ++ *n_success_jobs += status; ++ } ++ ++end_deq: ++ service_ctx->head = head; ++ return i; ++} ++ ++static __rte_always_inline int ++qat_sym_dp_dequeue_single_job(void *qp_data, uint8_t *service_data, ++ void **out_opaque) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_sym_dp_service_ctx *service_ctx = (void *)service_data; ++ struct qat_queue *rx_queue = &qp->rx_q; ++ ++ register struct icp_qat_fw_comn_resp *resp; ++ ++ resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr + ++ service_ctx->head); ++ ++ if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG)) ++ return -1; ++ ++ *out_opaque = (void *)(uintptr_t)resp->opaque_data; ++ ++ service_ctx->head = (service_ctx->head + rx_queue->msg_size) & ++ rx_queue->modulo_mask; ++ ++ return QAT_SYM_DP_IS_RESP_SUCCESS(resp); ++} ++ ++static __rte_always_inline void ++qat_sym_dp_kick_tail(void *qp_data, uint8_t *service_data, uint32_t n) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_queue *tx_queue = &qp->tx_q; ++ struct qat_sym_dp_service_ctx *service_ctx = (void *)service_data; ++ ++ qp->enqueued += n; ++ qp->stats.enqueued_count += n; ++ ++ assert(service_ctx->tail == ((tx_queue->tail + tx_queue->msg_size * n) & ++ tx_queue->modulo_mask)); ++ ++ tx_queue->tail = service_ctx->tail; ++ ++ WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, ++ tx_queue->hw_bundle_number, ++ tx_queue->hw_queue_number, tx_queue->tail); ++ tx_queue->csr_tail = tx_queue->tail; ++} ++ ++static __rte_always_inline void ++qat_sym_dp_update_head(void *qp_data, uint8_t *service_data, uint32_t n) ++{ ++ struct qat_qp *qp = qp_data; ++ struct qat_queue *rx_queue = &qp->rx_q; ++ struct qat_sym_dp_service_ctx *service_ctx = (void *)service_data; ++ ++ assert(service_ctx->head == ((rx_queue->head + rx_queue->msg_size * n) & ++ rx_queue->modulo_mask)); ++ ++ rx_queue->head = service_ctx->head; ++ rx_queue->nb_processed_responses += n; ++ qp->dequeued += n; ++ qp->stats.dequeued_count += n; ++ if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) { ++ uint32_t old_head, new_head; ++ uint32_t max_head; ++ ++ old_head = rx_queue->csr_head; ++ new_head = rx_queue->head; ++ max_head = qp->nb_descriptors * rx_queue->msg_size; ++ ++ /* write out free descriptors */ ++ void *cur_desc = (uint8_t *)rx_queue->base_addr + old_head; ++ ++ if (new_head < old_head) { ++ memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, ++ max_head - old_head); ++ memset(rx_queue->base_addr, ADF_RING_EMPTY_SIG_BYTE, ++ new_head); ++ } else { ++ memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - ++ old_head); ++ } ++ rx_queue->nb_processed_responses = 0; ++ rx_queue->csr_head = new_head; ++ ++ /* write current head to CSR */ ++ WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, ++ rx_queue->hw_bundle_number, rx_queue->hw_queue_number, ++ new_head); ++ } ++} ++ ++int ++qat_sym_dp_configure_service_ctx(struct rte_cryptodev *dev, uint16_t qp_id, ++ struct rte_crypto_dp_service_ctx *service_ctx, ++ enum rte_crypto_dp_service service_type, ++ enum rte_crypto_op_sess_type sess_type, ++ union rte_cryptodev_session_ctx session_ctx, ++ uint8_t is_update) ++{ ++ struct qat_qp *qp; ++ struct qat_sym_session *ctx; ++ struct qat_sym_dp_service_ctx *dp_ctx; ++ ++ if (service_ctx == NULL || session_ctx.crypto_sess == NULL || ++ sess_type != RTE_CRYPTO_OP_WITH_SESSION) ++ return -EINVAL; ++ ++ qp = dev->data->queue_pairs[qp_id]; ++ ctx = (struct qat_sym_session *)get_sym_session_private_data( ++ session_ctx.crypto_sess, qat_sym_driver_id); ++ dp_ctx = (struct qat_sym_dp_service_ctx *) ++ service_ctx->drv_service_data; ++ ++ if (!is_update) { ++ memset(service_ctx, 0, sizeof(*service_ctx) + ++ sizeof(struct qat_sym_dp_service_ctx)); ++ service_ctx->qp_data = dev->data->queue_pairs[qp_id]; ++ dp_ctx->tail = qp->tx_q.tail; ++ dp_ctx->head = qp->rx_q.head; ++ } ++ ++ dp_ctx->session = ctx; ++ ++ service_ctx->submit_done = qat_sym_dp_kick_tail; ++ service_ctx->dequeue_opaque = qat_sym_dp_dequeue; ++ service_ctx->dequeue_single = qat_sym_dp_dequeue_single_job; ++ service_ctx->dequeue_done = qat_sym_dp_update_head; ++ ++ if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER || ++ ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { ++ /* AES-GCM or AES-CCM */ ++ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || ++ ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 || ++ (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128 ++ && ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE ++ && ctx->qat_hash_alg == ++ ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) { ++ if (service_type != RTE_CRYPTO_DP_SYM_AEAD) ++ return -1; ++ service_ctx->submit_vec = qat_sym_dp_submit_aead_jobs; ++ service_ctx->submit_single_job = ++ qat_sym_dp_submit_single_aead; ++ } else { ++ if (service_type != RTE_CRYPTO_DP_SYM_CHAIN) ++ return -1; ++ service_ctx->submit_vec = qat_sym_dp_submit_chain_jobs; ++ service_ctx->submit_single_job = ++ qat_sym_dp_submit_single_chain; ++ } ++ } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) { ++ if (service_type != RTE_CRYPTO_DP_SYM_AUTH_ONLY) ++ return -1; ++ service_ctx->submit_vec = qat_sym_dp_submit_auth_jobs; ++ service_ctx->submit_single_job = qat_sym_dp_submit_single_auth; ++ } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) { ++ if (service_type != RTE_CRYPTO_DP_SYM_CIPHER_ONLY) ++ return -1; ++ service_ctx->submit_vec = qat_sym_dp_submit_cipher_jobs; ++ service_ctx->submit_single_job = ++ qat_sym_dp_submit_single_cipher; ++ } ++ ++ return 0; ++} ++ ++int ++qat_sym_get_service_ctx_size(__rte_unused struct rte_cryptodev *dev) ++{ ++ return sizeof(struct qat_sym_dp_service_ctx); ++} +diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c +index 314742f53..bef08c3bc 100644 +--- a/drivers/crypto/qat/qat_sym_pmd.c ++++ b/drivers/crypto/qat/qat_sym_pmd.c +@@ -258,7 +258,11 @@ static struct rte_cryptodev_ops crypto_qat_ops = { + /* Crypto related operations */ + .sym_session_get_size = qat_sym_session_get_private_size, + .sym_session_configure = qat_sym_session_configure, +- .sym_session_clear = qat_sym_session_clear ++ .sym_session_clear = qat_sym_session_clear, ++ ++ /* Data plane service related operations */ ++ .get_drv_ctx_size = qat_sym_get_service_ctx_size, ++ .configure_service = qat_sym_dp_configure_service_ctx, + }; + + #ifdef RTE_LIBRTE_SECURITY +@@ -376,7 +380,8 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, + RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | + RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | + RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | +- RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED; ++ RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED | ++ RTE_CRYPTODEV_FF_DATA_PLANE_SERVICE; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; +diff --git a/lib/librte_cryptodev/rte_crypto.h b/lib/librte_cryptodev/rte_crypto.h +index fd5ef3a87..f009be9af 100644 +--- a/lib/librte_cryptodev/rte_crypto.h ++++ b/lib/librte_cryptodev/rte_crypto.h +@@ -438,6 +438,15 @@ rte_crypto_op_attach_asym_session(struct rte_crypto_op *op, + return 0; + } + ++/** Crypto data-path service types */ ++enum rte_crypto_dp_service { ++ RTE_CRYPTO_DP_SYM_CIPHER_ONLY = 0, ++ RTE_CRYPTO_DP_SYM_AUTH_ONLY, ++ RTE_CRYPTO_DP_SYM_CHAIN, ++ RTE_CRYPTO_DP_SYM_AEAD, ++ RTE_CRYPTO_DP_N_SERVICE ++}; ++ + #ifdef __cplusplus + } + #endif +diff --git a/lib/librte_cryptodev/rte_crypto_sym.h b/lib/librte_cryptodev/rte_crypto_sym.h +index f29c98051..518e4111b 100644 +--- a/lib/librte_cryptodev/rte_crypto_sym.h ++++ b/lib/librte_cryptodev/rte_crypto_sym.h +@@ -50,6 +50,18 @@ struct rte_crypto_sgl { + uint32_t num; + }; + ++/** ++ * Crypto IO Data without length info. ++ * Supposed to be used to pass input/output data buffers with lengths ++ * defined when creating crypto session. ++ */ ++struct rte_crypto_data { ++ /** virtual address of the data buffer */ ++ void *base; ++ /** IOVA of the data buffer */ ++ rte_iova_t iova; ++}; ++ + /** + * Synchronous operation descriptor. + * Supposed to be used with CPU crypto API call. +@@ -57,12 +69,32 @@ struct rte_crypto_sgl { + struct rte_crypto_sym_vec { + /** array of SGL vectors */ + struct rte_crypto_sgl *sgl; +- /** array of pointers to IV */ +- void **iv; +- /** array of pointers to AAD */ +- void **aad; +- /** array of pointers to digest */ +- void **digest; ++ ++ union { ++ ++ /* Supposed to be used with CPU crypto API call. */ ++ struct { ++ /** array of pointers to IV */ ++ void **iv; ++ /** array of pointers to AAD */ ++ void **aad; ++ /** array of pointers to digest */ ++ void **digest; ++ }; ++ ++ /* Supposed to be used with rte_cryptodev_dp_sym_submit_vec() ++ * call. ++ */ ++ struct { ++ /** vector to IV */ ++ struct rte_crypto_data *iv_vec; ++ /** vecor to AAD */ ++ struct rte_crypto_data *aad_vec; ++ /** vector to Digest */ ++ struct rte_crypto_data *digest_vec; ++ }; ++ }; ++ + /** + * array of statuses for each operation: + * - 0 on success +diff --git a/lib/librte_cryptodev/rte_cryptodev.c b/lib/librte_cryptodev/rte_cryptodev.c +index 1dd795bcb..06c01cfaa 100644 +--- a/lib/librte_cryptodev/rte_cryptodev.c ++++ b/lib/librte_cryptodev/rte_cryptodev.c +@@ -1914,6 +1914,51 @@ rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id, + return dev->dev_ops->sym_cpu_process(dev, sess, ofs, vec); + } + ++int32_t ++rte_cryptodev_get_dp_service_ctx_data_size(uint8_t dev_id) ++{ ++ struct rte_cryptodev *dev; ++ int32_t size = sizeof(struct rte_crypto_dp_service_ctx); ++ int32_t priv_size; ++ ++ if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) ++ return -1; ++ ++ dev = rte_cryptodev_pmd_get_dev(dev_id); ++ ++ if (*dev->dev_ops->get_drv_ctx_size == NULL || ++ !(dev->feature_flags & RTE_CRYPTODEV_FF_DATA_PLANE_SERVICE)) { ++ return -1; ++ } ++ ++ priv_size = (*dev->dev_ops->get_drv_ctx_size)(dev); ++ if (priv_size < 0) ++ return -1; ++ ++ return RTE_ALIGN_CEIL((size + priv_size), 8); ++} ++ ++int ++rte_cryptodev_dp_configure_service(uint8_t dev_id, uint16_t qp_id, ++ enum rte_crypto_dp_service service_type, ++ enum rte_crypto_op_sess_type sess_type, ++ union rte_cryptodev_session_ctx session_ctx, ++ struct rte_crypto_dp_service_ctx *ctx, uint8_t is_update) ++{ ++ struct rte_cryptodev *dev; ++ ++ if (rte_cryptodev_get_qp_status(dev_id, qp_id) != 1) ++ return -1; ++ ++ dev = rte_cryptodev_pmd_get_dev(dev_id); ++ if (!(dev->feature_flags & RTE_CRYPTODEV_FF_DATA_PLANE_SERVICE) ++ || dev->dev_ops->configure_service == NULL) ++ return -1; ++ ++ return (*dev->dev_ops->configure_service)(dev, qp_id, ctx, ++ service_type, sess_type, session_ctx, is_update); ++} ++ + /** Initialise rte_crypto_op mempool element */ + static void + rte_crypto_op_init(struct rte_mempool *mempool, +diff --git a/lib/librte_cryptodev/rte_cryptodev.h b/lib/librte_cryptodev/rte_cryptodev.h +index 7b3ebc20f..6eb8ad9f9 100644 +--- a/lib/librte_cryptodev/rte_cryptodev.h ++++ b/lib/librte_cryptodev/rte_cryptodev.h +@@ -466,7 +466,8 @@ rte_cryptodev_asym_get_xform_enum(enum rte_crypto_asym_xform_type *xform_enum, + /**< Support symmetric session-less operations */ + #define RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA (1ULL << 23) + /**< Support operations on data which is not byte aligned */ +- ++#define RTE_CRYPTODEV_FF_DATA_PLANE_SERVICE (1ULL << 24) ++/**< Support accelerated specific raw data as input */ + + /** + * Get the name of a crypto device feature flag +@@ -1351,6 +1352,339 @@ rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs ofs, + struct rte_crypto_sym_vec *vec); + ++/** ++ * Get the size of the data-path service context for all registered drivers. ++ * ++ * @param dev_id The device identifier. ++ * ++ * @return ++ * - If the device supports data-path service, return the context size. ++ * - If the device does not support the data-plane service, return -1. ++ */ ++__rte_experimental ++int32_t ++rte_cryptodev_get_dp_service_ctx_data_size(uint8_t dev_id); ++ ++/** ++ * Union of different crypto session types, including sessionless ++ */ ++union rte_cryptodev_session_ctx { ++ struct rte_cryptodev_sym_session *crypto_sess; ++ struct rte_crypto_sym_xform *xform; ++ struct rte_security_session *sec_sess; ++}; ++ ++/** ++ * Submit a data vector into device queue but the driver will not start ++ * processing until rte_cryptodev_dp_sym_submit_vec() is called. ++ * ++ * @param qp Driver specific queue pair data. ++ * @param service_data Driver specific service data. ++ * @param vec The array of job vectors. ++ * @param ofs Start and stop offsets for auth and cipher ++ * operations. ++ * @param opaque The array of opaque data for dequeue. ++ * @return ++ * - The number of jobs successfully submitted. ++ */ ++typedef uint32_t (*cryptodev_dp_sym_submit_vec_t)( ++ void *qp, uint8_t *service_data, struct rte_crypto_sym_vec *vec, ++ union rte_crypto_sym_ofs ofs, void **opaque); ++ ++/** ++ * Submit single job into device queue but the driver will not start ++ * processing until rte_cryptodev_dp_sym_submit_vec() is called. ++ * ++ * @param qp Driver specific queue pair data. ++ * @param service_data Driver specific service data. ++ * @param data The buffer vector. ++ * @param n_data_vecs Number of buffer vectors. ++ * @param ofs Start and stop offsets for auth and cipher ++ * operations. ++ * @param iv IV data. ++ * @param digest Digest data. ++ * @param aad AAD data. ++ * @param opaque The array of opaque data for dequeue. ++ * @return ++ * - On success return 0. ++ * - On failure return negative integer. ++ */ ++typedef int (*cryptodev_dp_submit_single_job_t)( ++ void *qp_data, uint8_t *service_data, struct rte_crypto_vec *data, ++ uint16_t n_data_vecs, union rte_crypto_sym_ofs ofs, ++ struct rte_crypto_data *iv, struct rte_crypto_data *digest, ++ struct rte_crypto_data *aad, void *opaque); ++ ++/** ++ * Inform the queue pair to start processing or finish dequeuing all ++ * submitted/dequeued jobs. ++ * ++ * @param qp Driver specific queue pair data. ++ * @param service_data Driver specific service data. ++ * @param n The total number of submitted jobs. ++ */ ++typedef void (*cryptodev_dp_sym_opeartion_done_t)(void *qp, ++ uint8_t *service_data, uint32_t n); ++ ++/** ++ * Typedef that the user provided to get the dequeue count. User may use it to ++ * return a fixed number or the number parsed from the opaque data stored in ++ * the first processed job. ++ * ++ * @param opaque Dequeued opaque data. ++ **/ ++typedef uint32_t (*rte_cryptodev_get_dequeue_count_t)(void *opaque); ++ ++/** ++ * Typedef that the user provided to deal with post dequeue operation, such ++ * as filling status. ++ * ++ * @param opaque Dequeued opaque data. In case ++ * RTE_CRYPTO_HW_DP_FF_GET_OPAQUE_ARRAY bit is ++ * set, this value will be the opaque data stored ++ * in the specific processed jobs referenced by ++ * index, otherwise it will be the opaque data ++ * stored in the first processed job in the burst. ++ * @param index Index number of the processed job. ++ * @param is_op_success Driver filled operation status. ++ **/ ++typedef void (*rte_cryptodev_post_dequeue_t)(void *opaque, uint32_t index, ++ uint8_t is_op_success); ++ ++/** ++ * Dequeue symmetric crypto processing of user provided data. ++ * ++ * @param qp Driver specific queue pair data. ++ * @param service_data Driver specific service data. ++ * @param get_dequeue_count User provided callback function to ++ * obtain dequeue count. ++ * @param post_dequeue User provided callback function to ++ * post-process a dequeued operation. ++ * @param out_opaque Opaque pointer array to be retrieve from ++ * device queue. In case of ++ * *is_opaque_array* is set there should ++ * be enough room to store all opaque data. ++ * @param is_opaque_array Set 1 if every dequeued job will be ++ * written the opaque data into ++ * *out_opaque* array. ++ * @param n_success_jobs Driver written value to specific the ++ * total successful operations count. ++ * ++ * @return ++ * - Returns number of dequeued packets. ++ */ ++typedef uint32_t (*cryptodev_dp_sym_dequeue_t)(void *qp, uint8_t *service_data, ++ rte_cryptodev_get_dequeue_count_t get_dequeue_count, ++ rte_cryptodev_post_dequeue_t post_dequeue, ++ void **out_opaque, uint8_t is_opaque_array, ++ uint32_t *n_success_jobs); ++ ++/** ++ * Dequeue symmetric crypto processing of user provided data. ++ * ++ * @param qp Driver specific queue pair data. ++ * @param service_data Driver specific service data. ++ * @param out_opaque Opaque pointer to be retrieve from ++ * device queue. The driver shall support ++ * NULL input of this parameter. ++ * ++ * @return ++ * - 1 if the job is dequeued and the operation is a success. ++ * - 0 if the job is dequeued but the operation is failed. ++ * - -1 if no job is dequeued. ++ */ ++typedef int (*cryptodev_dp_sym_dequeue_single_job_t)( ++ void *qp, uint8_t *service_data, void **out_opaque); ++ ++/** ++ * Context data for asynchronous crypto process. ++ */ ++struct rte_crypto_dp_service_ctx { ++ void *qp_data; ++ ++ union { ++ /* Supposed to be used for symmetric crypto service */ ++ struct { ++ cryptodev_dp_submit_single_job_t submit_single_job; ++ cryptodev_dp_sym_submit_vec_t submit_vec; ++ cryptodev_dp_sym_opeartion_done_t submit_done; ++ cryptodev_dp_sym_dequeue_t dequeue_opaque; ++ cryptodev_dp_sym_dequeue_single_job_t dequeue_single; ++ cryptodev_dp_sym_opeartion_done_t dequeue_done; ++ }; ++ }; ++ ++ /* Driver specific service data */ ++ uint8_t drv_service_data[]; ++}; ++ ++/** ++ * Initialize one DP service, should be called before submitting job(s). ++ * Calling this function for the first time the user should unset is_update ++ * parameter and the driver will fill necessary operation data into ctx buffer. ++ * Only when rte_cryptodev_dp_submit_done() is called the data stored in ++ * the ctx buffer will not be effective. ++ * ++ * @param dev_id The device identifier. ++ * @param qp_id The index of the queue pair from which to ++ * retrieve processed packets. The value must be ++ * in the range [0, nb_queue_pair - 1] previously ++ * supplied to rte_cryptodev_configure(). ++ * @param service_type Type of the service requested. ++ * @param sess_type session type. ++ * @param session_ctx Session context data. ++ * @param ctx The data-path service context data. ++ * @param is_update Set 1 if ctx is pre-initialized but need ++ * update to different service type or session, ++ * but the rest driver data remains the same. ++ * buffer will always be one. ++ * @return ++ * - On success return 0. ++ * - On failure return negative integer. ++ */ ++__rte_experimental ++int ++rte_cryptodev_dp_configure_service(uint8_t dev_id, uint16_t qp_id, ++ enum rte_crypto_dp_service service_type, ++ enum rte_crypto_op_sess_type sess_type, ++ union rte_cryptodev_session_ctx session_ctx, ++ struct rte_crypto_dp_service_ctx *ctx, uint8_t is_update); ++ ++/** ++ * Submit single job into device queue but the driver will not start ++ * processing until rte_cryptodev_dp_sym_submit_vec() is called. ++ * ++ * @param ctx The initialized data-path service context data. ++ * @param data The buffer vector. ++ * @param n_data_vecs Number of buffer vectors. ++ * @param ofs Start and stop offsets for auth and cipher ++ * operations. ++ * @param iv IV data. ++ * @param digest Digest data. ++ * @param aad AAD data. ++ * @param opaque The array of opaque data for dequeue. ++ * @return ++ * - On success return 0. ++ * - On failure return negative integer. ++ */ ++__rte_experimental ++static __rte_always_inline int ++rte_cryptodev_dp_submit_single_job(struct rte_crypto_dp_service_ctx *ctx, ++ struct rte_crypto_vec *data, uint16_t n_data_vecs, ++ union rte_crypto_sym_ofs ofs, ++ struct rte_crypto_data *iv, struct rte_crypto_data *digest, ++ struct rte_crypto_data *aad, void *opaque) ++{ ++ return (*ctx->submit_single_job)(ctx->qp_data, ctx->drv_service_data, ++ data, n_data_vecs, ofs, iv, digest, aad, opaque); ++} ++ ++/** ++ * Submit a data vector into device queue but the driver will not start ++ * processing until rte_cryptodev_dp_sym_submit_vec() is called. ++ * ++ * @param ctx The initialized data-path service context data. ++ * @param vec The array of job vectors. ++ * @param ofs Start and stop offsets for auth and cipher operations. ++ * @param opaque The array of opaque data for dequeue. ++ * @return ++ * - The number of jobs successfully submitted. ++ */ ++__rte_experimental ++static __rte_always_inline uint32_t ++rte_cryptodev_dp_sym_submit_vec(struct rte_crypto_dp_service_ctx *ctx, ++ struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs, ++ void **opaque) ++{ ++ return (*ctx->submit_vec)(ctx->qp_data, ctx->drv_service_data, vec, ++ ofs, opaque); ++} ++ ++/** ++ * Kick the queue pair to start processing all submitted jobs from last ++ * rte_cryptodev_init_dp_service() call. ++ * ++ * @param ctx The initialized data-path service context data. ++ * @param n The total number of submitted jobs. ++ */ ++__rte_experimental ++static __rte_always_inline void ++rte_cryptodev_dp_submit_done(struct rte_crypto_dp_service_ctx *ctx, uint32_t n) ++{ ++ (*ctx->submit_done)(ctx->qp_data, ctx->drv_service_data, n); ++} ++ ++/** ++ * Dequeue symmetric crypto processing of user provided data. ++ * ++ * @param ctx The initialized data-path service ++ * context data. ++ * @param get_dequeue_count User provided callback function to ++ * obtain dequeue count. ++ * @param post_dequeue User provided callback function to ++ * post-process a dequeued operation. ++ * @param out_opaque Opaque pointer array to be retrieve from ++ * device queue. In case of ++ * *is_opaque_array* is set there should ++ * be enough room to store all opaque data. ++ * @param is_opaque_array Set 1 if every dequeued job will be ++ * written the opaque data into ++ * *out_opaque* array. ++ * @param n_success_jobs Driver written value to specific the ++ * total successful operations count. ++ * ++ * @return ++ * - Returns number of dequeued packets. ++ */ ++__rte_experimental ++static __rte_always_inline uint32_t ++rte_cryptodev_dp_sym_dequeue(struct rte_crypto_dp_service_ctx *ctx, ++ rte_cryptodev_get_dequeue_count_t get_dequeue_count, ++ rte_cryptodev_post_dequeue_t post_dequeue, ++ void **out_opaque, uint8_t is_opaque_array, ++ uint32_t *n_success_jobs) ++{ ++ return (*ctx->dequeue_opaque)(ctx->qp_data, ctx->drv_service_data, ++ get_dequeue_count, post_dequeue, out_opaque, is_opaque_array, ++ n_success_jobs); ++} ++ ++/** ++ * Dequeue Single symmetric crypto processing of user provided data. ++ * ++ * @param ctx The initialized data-path service ++ * context data. ++ * @param out_opaque Opaque pointer to be retrieve from ++ * device queue. The driver shall support ++ * NULL input of this parameter. ++ * ++ * @return ++ * - 1 if the job is dequeued and the operation is a success. ++ * - 0 if the job is dequeued but the operation is failed. ++ * - -1 if no job is dequeued. ++ */ ++__rte_experimental ++static __rte_always_inline int ++rte_cryptodev_dp_sym_dequeue_single_job(struct rte_crypto_dp_service_ctx *ctx, ++ void **out_opaque) ++{ ++ return (*ctx->dequeue_single)(ctx->qp_data, ctx->drv_service_data, ++ out_opaque); ++} ++ ++/** ++ * Inform the queue pair dequeue jobs finished. ++ * ++ * @param ctx The initialized data-path service context data. ++ * @param n The total number of submitted jobs. ++ */ ++__rte_experimental ++static __rte_always_inline void ++rte_cryptodev_dp_dequeue_done(struct rte_crypto_dp_service_ctx *ctx, uint32_t n) ++{ ++ (*ctx->dequeue_done)(ctx->qp_data, ctx->drv_service_data, n); ++} ++ + #ifdef __cplusplus + } + #endif +diff --git a/lib/librte_cryptodev/rte_cryptodev_pmd.h b/lib/librte_cryptodev/rte_cryptodev_pmd.h +index 81975d72b..9904267d7 100644 +--- a/lib/librte_cryptodev/rte_cryptodev_pmd.h ++++ b/lib/librte_cryptodev/rte_cryptodev_pmd.h +@@ -316,6 +316,30 @@ typedef uint32_t (*cryptodev_sym_cpu_crypto_process_t) + (struct rte_cryptodev *dev, struct rte_cryptodev_sym_session *sess, + union rte_crypto_sym_ofs ofs, struct rte_crypto_sym_vec *vec); + ++typedef int (*cryptodev_dp_get_service_ctx_size_t)( ++ struct rte_cryptodev *dev); ++ ++/** ++ * Typedef that the driver provided to update data-path service. ++ * ++ * @param ctx The data-path service context data. ++ * @param service_type Type of the service requested. ++ * @param sess_type session type. ++ * @param session_ctx Session context data. ++ * @param is_update Set 1 if ctx is pre-initialized but need ++ * update to different service type or session, ++ * but the rest driver data remains the same. ++ * buffer will always be one. ++ * @return ++ * - On success return 0. ++ * - On failure return negative integer. ++ */ ++typedef int (*cryptodev_dp_configure_service_t)( ++ struct rte_cryptodev *dev, uint16_t qp_id, ++ struct rte_crypto_dp_service_ctx *ctx, ++ enum rte_crypto_dp_service service_type, ++ enum rte_crypto_op_sess_type sess_type, ++ union rte_cryptodev_session_ctx session_ctx, uint8_t is_update); + + /** Crypto device operations function pointer table */ + struct rte_cryptodev_ops { +@@ -348,8 +372,16 @@ struct rte_cryptodev_ops { + /**< Clear a Crypto sessions private data. */ + cryptodev_asym_free_session_t asym_session_clear; + /**< Clear a Crypto sessions private data. */ +- cryptodev_sym_cpu_crypto_process_t sym_cpu_process; +- /**< process input data synchronously (cpu-crypto). */ ++ union { ++ cryptodev_sym_cpu_crypto_process_t sym_cpu_process; ++ /**< process input data synchronously (cpu-crypto). */ ++ struct { ++ cryptodev_dp_get_service_ctx_size_t get_drv_ctx_size; ++ /**< Get data path service context data size. */ ++ cryptodev_dp_configure_service_t configure_service; ++ /**< Initialize crypto service ctx data. */ ++ }; ++ }; + }; + + +diff --git a/lib/librte_cryptodev/rte_cryptodev_version.map b/lib/librte_cryptodev/rte_cryptodev_version.map +index a7a78dc41..6c5e78144 100644 +--- a/lib/librte_cryptodev/rte_cryptodev_version.map ++++ b/lib/librte_cryptodev/rte_cryptodev_version.map +@@ -106,4 +106,12 @@ EXPERIMENTAL { + + # added in 20.08 + rte_cryptodev_get_qp_status; ++ rte_cryptodev_dp_configure_service; ++ rte_cryptodev_get_dp_service_ctx_data_size; ++ rte_cryptodev_dp_submit_single_job; ++ rte_cryptodev_dp_sym_submit_vec; ++ rte_cryptodev_dp_submit_done; ++ rte_cryptodev_dp_sym_dequeue; ++ rte_cryptodev_dp_sym_dequeue_single_job; ++ rte_cryptodev_dp_dequeue_done; + }; +-- +2.20.1 + diff --git a/src/plugins/dpdk/CMakeLists.txt b/src/plugins/dpdk/CMakeLists.txt index e51d66b5866..4b1719d1473 100644 --- a/src/plugins/dpdk/CMakeLists.txt +++ b/src/plugins/dpdk/CMakeLists.txt @@ -113,6 +113,8 @@ endif() ############################################################################## # DPDK plugin ############################################################################## + +if (${DPDK_VERSION} VERSION_LESS "20.8.0") add_vpp_plugin(dpdk SOURCES buffer.c @@ -154,3 +156,46 @@ add_vpp_plugin(dpdk vpp-plugin-dpdk ) +else () +add_vpp_plugin(dpdk + SOURCES + buffer.c + main.c + thread.c + device/cli.c + device/common.c + device/device.c + device/flow.c + device/format.c + device/init.c + device/node.c + ipsec/cli.c + ipsec/crypto_node.c + ipsec/esp_decrypt.c + ipsec/esp_encrypt.c + ipsec/ipsec.c + cryptodev/cryptodev_dp_api.c + + MULTIARCH_SOURCES + buffer.c + device/device.c + device/node.c + ipsec/crypto_node.c + ipsec/esp_decrypt.c + ipsec/esp_encrypt.c + + INSTALL_HEADERS + device/dpdk.h + ipsec/ipsec.h + + LINK_FLAGS + "${DPDK_LINK_FLAGS}" + + LINK_LIBRARIES + ${DPDK_LINK_LIBRARIES} + + COMPONENT + vpp-plugin-dpdk +) + +endif ()
\ No newline at end of file diff --git a/src/plugins/dpdk/cryptodev/cryptodev_dp_api.c b/src/plugins/dpdk/cryptodev/cryptodev_dp_api.c new file mode 100644 index 00000000000..8fcfa2edf48 --- /dev/null +++ b/src/plugins/dpdk/cryptodev/cryptodev_dp_api.c @@ -0,0 +1,1536 @@ +/* + *------------------------------------------------------------------ + * Copyright (c) 2020 Intel and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ + +#include <vlib/vlib.h> +#include <vnet/plugin/plugin.h> +#include <vnet/crypto/crypto.h> +#include <vnet/vnet.h> +#include <vpp/app/version.h> + +#include <dpdk/buffer.h> +#include <dpdk/device/dpdk.h> +#include <dpdk/device/dpdk_priv.h> +#undef always_inline +#include <rte_bus_vdev.h> +#include <rte_cryptodev.h> +#include <rte_crypto_sym.h> +#include <rte_crypto.h> +#include <rte_cryptodev_pmd.h> +#include <rte_config.h> + +#if CLIB_DEBUG > 0 +#define always_inline static inline +#else +#define always_inline static inline __attribute__ ((__always_inline__)) +#endif + +#define CRYPTODEV_NB_CRYPTO_OPS 1024 +#define CRYPTODEV_MAX_INFLIGHT (CRYPTODEV_NB_CRYPTO_OPS - 1) +#define CRYPTODEV_AAD_MASK (CRYPTODEV_NB_CRYPTO_OPS - 1) +#define CRYPTODEV_DEQ_CACHE_SZ 32 +#define CRYPTODEV_NB_SESSION 10240 +#define CRYPTODEV_MAX_AAD_SIZE 16 +#define CRYPTODEV_MAX_N_SGL 8 /**< maximum number of segments */ + +/* VNET_CRYPTO_ALGO, TYPE, DPDK_CRYPTO_ALGO, IV_LEN, TAG_LEN, AAD_LEN */ +#define foreach_vnet_aead_crypto_conversion \ + _(AES_128_GCM, AEAD, AES_GCM, 12, 16, 8) \ + _(AES_128_GCM, AEAD, AES_GCM, 12, 16, 12) \ + _(AES_192_GCM, AEAD, AES_GCM, 12, 16, 8) \ + _(AES_192_GCM, AEAD, AES_GCM, 12, 16, 12) \ + _(AES_256_GCM, AEAD, AES_GCM, 12, 16, 8) \ + _(AES_256_GCM, AEAD, AES_GCM, 12, 16, 12) + +/** + * crypto (alg, cryptodev_alg), hash (alg, digest-size) + **/ +#define foreach_cryptodev_link_async_alg \ + _ (AES_128_CBC, AES_CBC, SHA1, 12) \ + _ (AES_192_CBC, AES_CBC, SHA1, 12) \ + _ (AES_256_CBC, AES_CBC, SHA1, 12) \ + _ (AES_128_CBC, AES_CBC, SHA224, 14) \ + _ (AES_192_CBC, AES_CBC, SHA224, 14) \ + _ (AES_256_CBC, AES_CBC, SHA224, 14) \ + _ (AES_128_CBC, AES_CBC, SHA256, 16) \ + _ (AES_192_CBC, AES_CBC, SHA256, 16) \ + _ (AES_256_CBC, AES_CBC, SHA256, 16) \ + _ (AES_128_CBC, AES_CBC, SHA384, 24) \ + _ (AES_192_CBC, AES_CBC, SHA384, 24) \ + _ (AES_256_CBC, AES_CBC, SHA384, 24) \ + _ (AES_128_CBC, AES_CBC, SHA512, 32) \ + _ (AES_192_CBC, AES_CBC, SHA512, 32) \ + _ (AES_256_CBC, AES_CBC, SHA512, 32) + +typedef enum +{ + CRYPTODEV_OP_TYPE_ENCRYPT = 0, + CRYPTODEV_OP_TYPE_DECRYPT, + CRYPTODEV_N_OP_TYPES, +} cryptodev_op_type_t; + +typedef struct +{ + struct rte_cryptodev_sym_session *keys[CRYPTODEV_N_OP_TYPES]; +} cryptodev_key_t; + +typedef struct +{ + u32 dev_id; + u32 q_id; + u8 *dp_service_buffer; + char *desc; +} cryptodev_inst_t; + +typedef struct +{ + struct rte_mempool *sess_pool; + struct rte_mempool *sess_priv_pool; +} cryptodev_numa_data_t; + +typedef struct +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); + vlib_buffer_t *b[VNET_CRYPTO_FRAME_SIZE]; + struct rte_crypto_dp_service_ctx *dp_service; + struct rte_crypto_vec vec[CRYPTODEV_MAX_N_SGL]; + struct rte_ring *cached_frame; + u16 aad_index; + u8 *aad_buf; + u64 aad_phy_addr; + u16 cryptodev_id; + u16 cryptodev_q; + u16 inflight; +} cryptodev_engine_thread_t; + +typedef struct +{ + cryptodev_numa_data_t *per_numa_data; + cryptodev_key_t *keys; + cryptodev_engine_thread_t *per_thread_data; + enum rte_iova_mode iova_mode; + cryptodev_inst_t *cryptodev_inst; + clib_bitmap_t *active_cdev_inst_mask; + clib_spinlock_t tlock; +} cryptodev_main_t; + +cryptodev_main_t cryptodev_main; + +static int +prepare_aead_xform (struct rte_crypto_sym_xform *xform, + cryptodev_op_type_t op_type, + const vnet_crypto_key_t * key, u32 aad_len) +{ + struct rte_crypto_aead_xform *aead_xform = &xform->aead; + memset (xform, 0, sizeof (*xform)); + xform->type = RTE_CRYPTO_SYM_XFORM_AEAD; + xform->next = 0; + + if (key->alg != VNET_CRYPTO_ALG_AES_128_GCM && + key->alg != VNET_CRYPTO_ALG_AES_192_GCM && + key->alg != VNET_CRYPTO_ALG_AES_256_GCM) + return -1; + + aead_xform->algo = RTE_CRYPTO_AEAD_AES_GCM; + aead_xform->op = (op_type == CRYPTODEV_OP_TYPE_ENCRYPT) ? + RTE_CRYPTO_AEAD_OP_ENCRYPT : RTE_CRYPTO_AEAD_OP_DECRYPT; + aead_xform->aad_length = aad_len; + aead_xform->digest_length = 16; + aead_xform->iv.offset = 0; + aead_xform->iv.length = 12; + aead_xform->key.data = key->data; + aead_xform->key.length = vec_len (key->data); + + return 0; +} + +static int +prepare_linked_xform (struct rte_crypto_sym_xform *xforms, + cryptodev_op_type_t op_type, + const vnet_crypto_key_t * key) +{ + struct rte_crypto_sym_xform *xform_cipher, *xform_auth; + vnet_crypto_key_t *key_cipher, *key_auth; + enum rte_crypto_cipher_algorithm cipher_algo = ~0; + enum rte_crypto_auth_algorithm auth_algo = ~0; + u32 digest_len = ~0; + + key_cipher = vnet_crypto_get_key (key->index_crypto); + key_auth = vnet_crypto_get_key (key->index_integ); + if (!key_cipher || !key_auth) + return -1; + + if (op_type == CRYPTODEV_OP_TYPE_ENCRYPT) + { + xform_cipher = xforms; + xform_auth = xforms + 1; + xform_cipher->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + xform_auth->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; + } + else + { + xform_cipher = xforms + 1; + xform_auth = xforms; + xform_cipher->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT; + xform_auth->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY; + } + + xform_cipher->type = RTE_CRYPTO_SYM_XFORM_CIPHER; + xform_auth->type = RTE_CRYPTO_SYM_XFORM_AUTH; + xforms->next = xforms + 1; + + switch (key->async_alg) + { +#define _(a, b, c, d) \ + case VNET_CRYPTO_ALG_##a##_##c##_TAG##d:\ + cipher_algo = RTE_CRYPTO_CIPHER_##b; \ + auth_algo = RTE_CRYPTO_AUTH_##c##_HMAC; \ + digest_len = d; \ + break; + + foreach_cryptodev_link_async_alg +#undef _ + default: + return -1; + } + + xform_cipher->cipher.algo = cipher_algo; + xform_cipher->cipher.key.data = key_cipher->data; + xform_cipher->cipher.key.length = vec_len (key_cipher->data); + xform_cipher->cipher.iv.length = 16; + xform_cipher->cipher.iv.offset = 0; + + xform_auth->auth.algo = auth_algo; + xform_auth->auth.digest_length = digest_len; + xform_auth->auth.key.data = key_auth->data; + xform_auth->auth.key.length = vec_len (key_auth->data); + + return 0; +} + +static int +cryptodev_session_create (vnet_crypto_key_t * const key, + struct rte_mempool *sess_priv_pool, + cryptodev_key_t * session_pair, u32 aad_len) +{ + struct rte_crypto_sym_xform xforms_enc[2] = { {0} }; + struct rte_crypto_sym_xform xforms_dec[2] = { {0} }; + cryptodev_main_t *cmt = &cryptodev_main; + cryptodev_inst_t *dev_inst; + struct rte_cryptodev *cdev; + int ret; + uint8_t dev_id = 0; + + if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) + ret = prepare_linked_xform (xforms_enc, CRYPTODEV_OP_TYPE_ENCRYPT, key); + else + ret = prepare_aead_xform (xforms_enc, CRYPTODEV_OP_TYPE_ENCRYPT, key, + aad_len); + if (ret) + return 0; + + if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) + prepare_linked_xform (xforms_dec, CRYPTODEV_OP_TYPE_DECRYPT, key); + else + prepare_aead_xform (xforms_dec, CRYPTODEV_OP_TYPE_DECRYPT, key, aad_len); + + vec_foreach (dev_inst, cmt->cryptodev_inst) + { + dev_id = dev_inst->dev_id; + cdev = rte_cryptodev_pmd_get_dev (dev_id); + + /* if the session is already configured for the driver type, avoid + configuring it again to increase the session data's refcnt */ + if (session_pair->keys[0]->sess_data[cdev->driver_id].data && + session_pair->keys[1]->sess_data[cdev->driver_id].data) + continue; + + ret = rte_cryptodev_sym_session_init (dev_id, session_pair->keys[0], + xforms_enc, sess_priv_pool); + ret = rte_cryptodev_sym_session_init (dev_id, session_pair->keys[1], + xforms_dec, sess_priv_pool); + if (ret < 0) + return ret; + } + session_pair->keys[0]->opaque_data = aad_len; + session_pair->keys[1]->opaque_data = aad_len; + + return 0; +} + +static void +cryptodev_session_del (struct rte_cryptodev_sym_session *sess) +{ + u32 n_devs, i; + + if (sess == NULL) + return; + + n_devs = rte_cryptodev_count (); + + for (i = 0; i < n_devs; i++) + rte_cryptodev_sym_session_clear (i, sess); + + rte_cryptodev_sym_session_free (sess); +} + +static int +cryptodev_check_supported_vnet_alg (vnet_crypto_key_t * key) +{ + vnet_crypto_alg_t alg; + if (key->type == VNET_CRYPTO_KEY_TYPE_LINK) + return 0; + + alg = key->alg; + +#define _(a, b, c, d, e, f) \ + if (alg == VNET_CRYPTO_ALG_##a) \ + return 0; + + foreach_vnet_aead_crypto_conversion +#undef _ + return -1; +} + +static_always_inline void +cryptodev_sess_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop, + vnet_crypto_key_index_t idx, u32 aad_len) +{ + cryptodev_main_t *cmt = &cryptodev_main; + cryptodev_numa_data_t *numa_data; + vnet_crypto_key_t *key = vnet_crypto_get_key (idx); + struct rte_mempool *sess_pool, *sess_priv_pool; + cryptodev_key_t *ckey = 0; + int ret = 0; + + if (kop == VNET_CRYPTO_KEY_OP_DEL) + { + if (idx >= vec_len (cmt->keys)) + return; + + ckey = pool_elt_at_index (cmt->keys, idx); + cryptodev_session_del (ckey->keys[0]); + cryptodev_session_del (ckey->keys[1]); + ckey->keys[0] = 0; + ckey->keys[1] = 0; + pool_put (cmt->keys, ckey); + return; + } + else if (kop == VNET_CRYPTO_KEY_OP_MODIFY) + { + if (idx >= vec_len (cmt->keys)) + return; + + ckey = pool_elt_at_index (cmt->keys, idx); + + cryptodev_session_del (ckey->keys[0]); + cryptodev_session_del (ckey->keys[1]); + ckey->keys[0] = 0; + ckey->keys[1] = 0; + } + else /* create key */ + pool_get_zero (cmt->keys, ckey); + + /* do not create session for unsupported alg */ + if (cryptodev_check_supported_vnet_alg (key)) + return; + + numa_data = vec_elt_at_index (cmt->per_numa_data, vm->numa_node); + sess_pool = numa_data->sess_pool; + sess_priv_pool = numa_data->sess_priv_pool; + + ckey->keys[0] = rte_cryptodev_sym_session_create (sess_pool); + if (!ckey->keys[0]) + { + ret = -1; + goto clear_key; + } + + ckey->keys[1] = rte_cryptodev_sym_session_create (sess_pool); + if (!ckey->keys[1]) + { + ret = -1; + goto clear_key; + } + + ret = cryptodev_session_create (key, sess_priv_pool, ckey, aad_len); + +clear_key: + if (ret != 0) + { + cryptodev_session_del (ckey->keys[0]); + cryptodev_session_del (ckey->keys[1]); + memset (ckey, 0, sizeof (*ckey)); + pool_put (cmt->keys, ckey); + } +} + +/*static*/ void +cryptodev_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop, + vnet_crypto_key_index_t idx) +{ + cryptodev_sess_handler (vm, kop, idx, 8); +} + +static_always_inline void +cryptodev_mark_frame_err_status (vnet_crypto_async_frame_t * f, + vnet_crypto_op_status_t s) +{ + u32 n_elts = f->n_elts, i; + + for (i = 0; i < n_elts; i++) + f->elts[i].status = s; + f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED; +} + +static_always_inline int +cryptodev_frame_build_sgl (vlib_main_t * vm, enum rte_iova_mode iova_mode, + struct rte_crypto_vec *data_vec, + u16 * n_seg, vlib_buffer_t * b, u32 size) +{ + struct rte_crypto_vec *vec = data_vec + 1; + if (vlib_buffer_chain_linearize (vm, b) > CRYPTODEV_MAX_N_SGL) + return -1; + + while ((b->flags & VLIB_BUFFER_NEXT_PRESENT) && size) + { + u32 len; + b = vlib_get_buffer (vm, b->next_buffer); + len = clib_min (b->current_length, size); + vec->base = (void *) vlib_buffer_get_current (b); + if (iova_mode == RTE_IOVA_VA) + vec->iova = pointer_to_uword (vec->base); + else + vec->iova = vlib_buffer_get_current_pa (vm, b); + vec->len = len; + size -= len; + vec++; + *n_seg += 1; + } + + if (size) + return -1; + + return 0; +} + +static_always_inline u64 +compute_ofs_linked_alg (vnet_crypto_async_frame_elt_t * fe, i16 * min_ofs, + u32 * max_end) +{ + union rte_crypto_sym_ofs ofs; + u32 crypto_end = fe->crypto_start_offset + fe->crypto_total_length; + u32 integ_end = fe->integ_start_offset + fe->crypto_total_length + + fe->integ_length_adj; + + *min_ofs = clib_min (fe->crypto_start_offset, fe->integ_start_offset); + *max_end = clib_max (crypto_end, integ_end); + + ofs.ofs.cipher.head = fe->crypto_start_offset - *min_ofs; + ofs.ofs.cipher.tail = *max_end - crypto_end; + ofs.ofs.auth.head = fe->integ_start_offset - *min_ofs; + ofs.ofs.auth.tail = *max_end - integ_end; + + return ofs.raw; +} + +static_always_inline int +cryptodev_frame_linked_algs_enqueue (vlib_main_t * vm, + vnet_crypto_async_frame_t * frame, + cryptodev_op_type_t op_type) +{ + cryptodev_main_t *cmt = &cryptodev_main; + cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; + vnet_crypto_async_frame_elt_t *fe; + struct rte_crypto_vec *vec; + struct rte_crypto_data iv_vec, digest_vec; + vlib_buffer_t **b; + u32 n_elts; + cryptodev_key_t *key; + u32 last_key_index; + union rte_crypto_sym_ofs cofs; + i16 min_ofs; + u32 max_end; + + n_elts = frame->n_elts; + + if (PREDICT_FALSE (CRYPTODEV_MAX_INFLIGHT - cet->inflight < n_elts)) + { + cryptodev_mark_frame_err_status (frame, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + + vlib_get_buffers (vm, frame->buffer_indices, cet->b, frame->n_elts); + + vec = cet->vec; + b = cet->b; + + fe = frame->elts; + + key = pool_elt_at_index (cmt->keys, fe->key_index); + last_key_index = fe->key_index; + + if (PREDICT_FALSE + (rte_cryptodev_dp_configure_service + (cet->cryptodev_id, cet->cryptodev_q, RTE_CRYPTO_DP_SYM_CHAIN, + RTE_CRYPTO_OP_WITH_SESSION, + (union rte_cryptodev_session_ctx) key->keys[op_type], cet->dp_service, + 0) < 0)) + { + cryptodev_mark_frame_err_status (frame, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + + cofs.raw = compute_ofs_linked_alg (fe, &min_ofs, &max_end); + + while (n_elts) + { + u16 n_seg = 1; + int status; + + if (n_elts > 2) + { + CLIB_PREFETCH (&fe[1], CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (&fe[2], CLIB_CACHE_LINE_BYTES, LOAD); + vlib_prefetch_buffer_header (b[1], LOAD); + vlib_prefetch_buffer_header (b[2], LOAD); + } + + if (PREDICT_FALSE (last_key_index != fe->key_index)) + { + cofs.raw = compute_ofs_linked_alg (fe, &min_ofs, &max_end); + + key = pool_elt_at_index (cmt->keys, fe->key_index); + last_key_index = fe->key_index; + + if (PREDICT_FALSE + (rte_cryptodev_dp_configure_service + (cet->cryptodev_id, cet->cryptodev_q, RTE_CRYPTO_DP_SYM_CHAIN, + RTE_CRYPTO_OP_WITH_SESSION, + (union rte_cryptodev_session_ctx) key->keys[op_type], + cet->dp_service, 1) < 0)) + { + cryptodev_mark_frame_err_status (frame, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + } + + vec->len = max_end - min_ofs; + if (cmt->iova_mode == RTE_IOVA_VA) + { + vec->base = (void *) (b[0]->data + min_ofs); + vec->iova = pointer_to_uword (b[0]->data) + min_ofs; + iv_vec.base = (void *) fe->iv; + iv_vec.iova = pointer_to_uword (fe->iv); + digest_vec.base = (void *) fe->tag; + digest_vec.iova = pointer_to_uword (fe->tag); + } + else + { + vec->base = (void *) (b[0]->data + min_ofs); + vec->iova = vlib_buffer_get_pa (vm, b[0]) + min_ofs; + iv_vec.base = (void *) fe->iv; + iv_vec.iova = vlib_physmem_get_pa (vm, fe->iv); + digest_vec.base = (void *) fe->tag; + digest_vec.iova = vlib_physmem_get_pa (vm, fe->digest); + } + + if (PREDICT_FALSE (fe->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS)) + { + vec->len = b[0]->current_data + b[0]->current_length - min_ofs; + if (cryptodev_frame_build_sgl + (vm, cmt->iova_mode, vec, &n_seg, b[0], + max_end - min_ofs - vec->len) < 0) + { + cryptodev_mark_frame_err_status (frame, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + } + + status = rte_cryptodev_dp_submit_single_job (cet->dp_service, + vec, n_seg, cofs, &iv_vec, + &digest_vec, 0, + (void *) frame); + if (status < 0) + { + cryptodev_mark_frame_err_status (frame, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + + b++; + fe++; + n_elts--; + } + + rte_cryptodev_dp_submit_done (cet->dp_service, frame->n_elts); + cet->inflight += frame->n_elts; + + return 0; +} + +static_always_inline int +cryptodev_frame_gcm_enqueue (vlib_main_t * vm, + vnet_crypto_async_frame_t * frame, + cryptodev_op_type_t op_type, u8 aad_len) +{ + cryptodev_main_t *cmt = &cryptodev_main; + cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; + vnet_crypto_async_frame_elt_t *fe; + vlib_buffer_t **b; + u32 n_elts; + cryptodev_key_t *key; + u32 last_key_index; + union rte_crypto_sym_ofs cofs; + struct rte_crypto_vec *vec; + struct rte_crypto_data iv_vec, digest_vec, aad_vec; + u8 sess_aad_len; + + n_elts = frame->n_elts; + + if (PREDICT_FALSE (CRYPTODEV_MAX_INFLIGHT - cet->inflight < n_elts)) + { + cryptodev_mark_frame_err_status (frame, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + + vlib_get_buffers (vm, frame->buffer_indices, cet->b, frame->n_elts); + + vec = cet->vec; + fe = frame->elts; + b = cet->b; + + cofs.raw = 0; + + key = pool_elt_at_index (cmt->keys, fe->key_index); + last_key_index = fe->key_index; + sess_aad_len = (u8) key->keys[op_type]->opaque_data; + if (PREDICT_FALSE (sess_aad_len != aad_len)) + cryptodev_sess_handler (vm, VNET_CRYPTO_KEY_OP_MODIFY, + fe->key_index, aad_len); + + if (PREDICT_FALSE + (rte_cryptodev_dp_configure_service + (cet->cryptodev_id, cet->cryptodev_q, RTE_CRYPTO_DP_SYM_AEAD, + RTE_CRYPTO_OP_WITH_SESSION, + (union rte_cryptodev_session_ctx) key->keys[op_type], cet->dp_service, + 0) < 0)) + { + cryptodev_mark_frame_err_status (frame, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + + while (n_elts) + { + u32 aad_offset = ((cet->aad_index++) & CRYPTODEV_AAD_MASK) << 4; + int status; + u16 n_seg = 1; + + if (n_elts > 1) + { + CLIB_PREFETCH (&fe[1], CLIB_CACHE_LINE_BYTES, LOAD); + vlib_prefetch_buffer_header (b[1], LOAD); + } + + if (last_key_index != fe->key_index) + { + key = pool_elt_at_index (cmt->keys, fe->key_index); + sess_aad_len = (u8) key->keys[op_type]->opaque_data; + if (PREDICT_FALSE (sess_aad_len != aad_len)) + { + cryptodev_sess_handler (vm, VNET_CRYPTO_KEY_OP_MODIFY, + fe->key_index, aad_len); + } + last_key_index = fe->key_index; + + if (PREDICT_FALSE + (rte_cryptodev_dp_configure_service + (cet->cryptodev_id, cet->cryptodev_q, RTE_CRYPTO_DP_SYM_AEAD, + RTE_CRYPTO_OP_WITH_SESSION, + (union rte_cryptodev_session_ctx) key->keys[op_type], + cet->dp_service, 1) < 0)) + { + cryptodev_mark_frame_err_status (frame, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + } + + if (cmt->iova_mode == RTE_IOVA_VA) + { + vec[0].base = (void *) (b[0]->data + fe->crypto_start_offset); + vec[0].iova = pointer_to_uword (vec[0].base); + vec[0].len = fe->crypto_total_length; + iv_vec.base = (void *) fe->iv; + iv_vec.iova = pointer_to_uword (fe->iv); + digest_vec.base = (void *) fe->tag; + digest_vec.iova = pointer_to_uword (fe->tag); + aad_vec.base = (void *) (cet->aad_buf + aad_offset); + aad_vec.iova = cet->aad_phy_addr + aad_offset; + } + else + { + vec[0].base = (void *) (b[0]->data + fe->crypto_start_offset); + vec[0].iova = + vlib_buffer_get_pa (vm, b[0]) + fe->crypto_start_offset; + vec[0].len = fe->crypto_total_length; + iv_vec.base = (void *) fe->iv; + iv_vec.iova = vlib_physmem_get_pa (vm, fe->iv); + aad_vec.base = (void *) (cet->aad_buf + aad_offset); + aad_vec.iova = cet->aad_phy_addr + aad_offset; + digest_vec.base = (void *) fe->tag; + digest_vec.iova = vlib_physmem_get_pa (vm, fe->tag); + } + + if (aad_len == 8) + *(u64 *) (cet->aad_buf + aad_offset) = *(u64 *) fe->aad; + else + { + /* aad_len == 12 */ + *(u64 *) (cet->aad_buf + aad_offset) = *(u64 *) fe->aad; + *(u32 *) (cet->aad_buf + aad_offset + 8) = *(u32 *) (fe->aad + 8); + } + + if (PREDICT_FALSE (fe->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS)) + { + vec[0].len = b[0]->current_data + + b[0]->current_length - fe->crypto_start_offset; + if (cryptodev_frame_build_sgl + (vm, cmt->iova_mode, vec, &n_seg, b[0], + fe->crypto_total_length - vec[0].len) < 0) + { + cryptodev_mark_frame_err_status (frame, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + } + + status = + rte_cryptodev_dp_submit_single_job (cet->dp_service, vec, n_seg, cofs, + &iv_vec, &digest_vec, &aad_vec, + (void *) frame); + if (PREDICT_FALSE (status < 0)) + { + cryptodev_mark_frame_err_status (frame, + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR); + return -1; + } + fe++; + b++; + n_elts--; + } + + rte_cryptodev_dp_submit_done (cet->dp_service, frame->n_elts); + cet->inflight += frame->n_elts; + + return 0; +} + +static u32 +cryptodev_get_frame_n_elts (void *frame) +{ + vnet_crypto_async_frame_t *f = (vnet_crypto_async_frame_t *) frame; + return f->n_elts; +} + +static void +cryptodev_post_dequeue (void *frame, u32 index, u8 is_op_success) +{ + vnet_crypto_async_frame_t *f = (vnet_crypto_async_frame_t *) frame; + + f->elts[index].status = is_op_success ? VNET_CRYPTO_OP_STATUS_COMPLETED : + VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC; +} + +#define GET_RING_OBJ(r, pos, f) do { \ + vnet_crypto_async_frame_t **ring = (void *)&r[1]; \ + f = ring[(r->cons.head + pos) & r->mask]; \ +} while (0) + +static_always_inline vnet_crypto_async_frame_t * +cryptodev_frame_dequeue (vlib_main_t * vm, u32 * nb_elts_processed, + u32 * enqueue_thread_idx) +{ + cryptodev_main_t *cmt = &cryptodev_main; + cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index; + vnet_crypto_async_frame_t *frame, *frame_ret = 0; + u32 n_deq, n_success; + u32 n_cached_frame = rte_ring_count (cet->cached_frame), n_room_left; + u8 no_job_to_deq = 0; + u16 inflight = cet->inflight; + + n_room_left = CRYPTODEV_DEQ_CACHE_SZ - n_cached_frame - 1; + + if (n_cached_frame) + { + u32 i; + for (i = 0; i < n_cached_frame; i++) + { + vnet_crypto_async_frame_t *f; + void *f_ret; + u8 n_left, err, j; + + GET_RING_OBJ (cet->cached_frame, i, f); + + if (i < n_cached_frame - 2) + { + vnet_crypto_async_frame_t *f1, *f2; + GET_RING_OBJ (cet->cached_frame, i + 1, f1); + GET_RING_OBJ (cet->cached_frame, i + 2, f2); + CLIB_PREFETCH (f1, CLIB_CACHE_LINE_BYTES, LOAD); + CLIB_PREFETCH (f2, CLIB_CACHE_LINE_BYTES, LOAD); + } + + n_left = f->state & 0x7f; + err = f->state & 0x80; + + for (j = f->n_elts - n_left; j < f->n_elts && inflight; j++) + { + int ret = + rte_cryptodev_dp_sym_dequeue_single_job (cet->dp_service, + &f_ret); + if (ret < 0) + break; + f->elts[j].status = ret == 1 ? VNET_CRYPTO_OP_STATUS_COMPLETED : + VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR; + err |= ret << 7; + inflight--; + } + + if (j == f->n_elts) + { + if (i == 0) + { + frame_ret = f; + f->state = err ? VNET_CRYPTO_FRAME_STATE_ELT_ERROR : + VNET_CRYPTO_FRAME_STATE_SUCCESS; + } + else + { + f->state = f->n_elts - j; + f->state |= err; + } + if (inflight) + continue; + } + + /* to here f is not completed dequeued and no more job can be + * dequeued + */ + f->state = f->n_elts - j; + f->state |= err; + no_job_to_deq = 1; + break; + } + + if (frame_ret) + { + rte_ring_sc_dequeue (cet->cached_frame, (void **) &frame_ret); + n_room_left++; + } + } + + /* no point to dequeue further */ + if (!inflight || no_job_to_deq || !n_room_left) + goto end_deq; + + n_deq = rte_cryptodev_dp_sym_dequeue (cet->dp_service, + cryptodev_get_frame_n_elts, + cryptodev_post_dequeue, + (void **) &frame, 0, &n_success); + if (!n_deq) + goto end_deq; + + inflight -= n_deq; + no_job_to_deq = n_deq < frame->n_elts; + /* we have to cache the frame */ + if (frame_ret || n_cached_frame || no_job_to_deq) + { + frame->state = frame->n_elts - n_deq; + frame->state |= ((n_success < n_deq) << 7); + rte_ring_sp_enqueue (cet->cached_frame, (void *) frame); + n_room_left--; + } + else + { + frame->state = n_success == frame->n_elts ? + VNET_CRYPTO_FRAME_STATE_SUCCESS : VNET_CRYPTO_FRAME_STATE_ELT_ERROR; + frame_ret = frame; + } + + /* see if we can dequeue more */ + while (inflight && n_room_left && !no_job_to_deq) + { + n_deq = rte_cryptodev_dp_sym_dequeue (cet->dp_service, + cryptodev_get_frame_n_elts, + cryptodev_post_dequeue, + (void **) &frame, 0, &n_success); + if (!n_deq) + break; + inflight -= n_deq; + no_job_to_deq = n_deq < frame->n_elts; + frame->state = frame->n_elts - n_deq; + frame->state |= ((n_success < n_deq) << 7); + rte_ring_sp_enqueue (cet->cached_frame, (void *) frame); + n_room_left--; + } + +end_deq: + if (inflight < cet->inflight) + { + rte_cryptodev_dp_dequeue_done (cet->dp_service, + cet->inflight - inflight); + cet->inflight = inflight; + } + + if (frame_ret) + { + *nb_elts_processed = frame_ret->n_elts; + *enqueue_thread_idx = frame_ret->enqueue_thread_index; + } + + return frame_ret; +} + +/* *INDENT-OFF* */ +static_always_inline int +cryptodev_enqueue_gcm_aad_8_enc (vlib_main_t * vm, + vnet_crypto_async_frame_t * frame) +{ + return cryptodev_frame_gcm_enqueue (vm, frame, + CRYPTODEV_OP_TYPE_ENCRYPT, 8); +} +static_always_inline int +cryptodev_enqueue_gcm_aad_12_enc (vlib_main_t * vm, + vnet_crypto_async_frame_t * frame) +{ + return cryptodev_frame_gcm_enqueue (vm, frame, + CRYPTODEV_OP_TYPE_ENCRYPT, 12); +} + +static_always_inline int +cryptodev_enqueue_gcm_aad_8_dec (vlib_main_t * vm, + vnet_crypto_async_frame_t * frame) +{ + return cryptodev_frame_gcm_enqueue (vm, frame, + CRYPTODEV_OP_TYPE_DECRYPT, 8); +} +static_always_inline int +cryptodev_enqueue_gcm_aad_12_dec (vlib_main_t * vm, + vnet_crypto_async_frame_t * frame) +{ + return cryptodev_frame_gcm_enqueue (vm, frame, + CRYPTODEV_OP_TYPE_DECRYPT, 12); +} + +static_always_inline int +cryptodev_enqueue_linked_alg_enc (vlib_main_t * vm, + vnet_crypto_async_frame_t * frame) +{ + return cryptodev_frame_linked_algs_enqueue (vm, frame, + CRYPTODEV_OP_TYPE_ENCRYPT); +} + +static_always_inline int +cryptodev_enqueue_linked_alg_dec (vlib_main_t * vm, + vnet_crypto_async_frame_t * frame) +{ + return cryptodev_frame_linked_algs_enqueue (vm, frame, + CRYPTODEV_OP_TYPE_DECRYPT); +} + +typedef enum +{ + CRYPTODEV_RESOURCE_ASSIGN_AUTO = 0, + CRYPTODEV_RESOURCE_ASSIGN_UPDATE, +} cryptodev_resource_assign_op_t; + +/** + * assign a cryptodev resource to a worker. + * @param cet: the worker thread data + * @param cryptodev_inst_index: if op is "ASSIGN_AUTO" this param is ignored. + * @param op: the assignment method. + * @return: 0 if successfully, negative number otherwise. + **/ +static_always_inline int +cryptodev_assign_resource (cryptodev_engine_thread_t * cet, + u32 cryptodev_inst_index, + cryptodev_resource_assign_op_t op) +{ + cryptodev_main_t *cmt = &cryptodev_main; + cryptodev_inst_t *cinst = 0; + uword idx; + + /* assign resource is only allowed when no inflight op is in the queue */ + if (cet->inflight) + return -EBUSY; + + switch (op) + { + case CRYPTODEV_RESOURCE_ASSIGN_AUTO: + if (clib_bitmap_count_set_bits (cmt->active_cdev_inst_mask) >= + vec_len (cmt->cryptodev_inst)) + return -1; + + clib_spinlock_lock (&cmt->tlock); + idx = clib_bitmap_first_clear (cmt->active_cdev_inst_mask); + clib_bitmap_set (cmt->active_cdev_inst_mask, idx, 1); + cinst = vec_elt_at_index (cmt->cryptodev_inst, idx); + cet->cryptodev_id = cinst->dev_id; + cet->cryptodev_q = cinst->q_id; + cet->dp_service = (struct rte_crypto_dp_service_ctx *) + cinst->dp_service_buffer; + clib_spinlock_unlock (&cmt->tlock); + break; + case CRYPTODEV_RESOURCE_ASSIGN_UPDATE: + /* assigning a used cryptodev resource is not allowed */ + if (clib_bitmap_get (cmt->active_cdev_inst_mask, cryptodev_inst_index) + == 1) + return -EBUSY; + vec_foreach_index (idx, cmt->cryptodev_inst) + { + cinst = cmt->cryptodev_inst + idx; + if (cinst->dev_id == cet->cryptodev_id && + cinst->q_id == cet->cryptodev_q) + break; + } + /* invalid existing worker resource assignment */ + if (idx == vec_len (cmt->cryptodev_inst)) + return -EINVAL; + clib_spinlock_lock (&cmt->tlock); + clib_bitmap_set_no_check (cmt->active_cdev_inst_mask, idx, 0); + clib_bitmap_set_no_check (cmt->active_cdev_inst_mask, + cryptodev_inst_index, 1); + cinst = cmt->cryptodev_inst + cryptodev_inst_index; + cet->cryptodev_id = cinst->dev_id; + cet->cryptodev_q = cinst->q_id; + cet->dp_service = (struct rte_crypto_dp_service_ctx *) + cinst->dp_service_buffer; + clib_spinlock_unlock (&cmt->tlock); + break; + default: + return -EINVAL; + } + return 0; +} + +static u8 * +format_cryptodev_inst (u8 * s, va_list * args) +{ + cryptodev_main_t *cmt = &cryptodev_main; + u32 inst = va_arg (*args, u32); + cryptodev_inst_t *cit = cmt->cryptodev_inst + inst; + u32 thread_index = 0; + struct rte_cryptodev_info info; + + rte_cryptodev_info_get (cit->dev_id, &info); + s = format (s, "%-25s%-10u", info.device->name, cit->q_id); + + vec_foreach_index (thread_index, cmt->per_thread_data) + { + cryptodev_engine_thread_t *cet = cmt->per_thread_data + thread_index; + if (vlib_num_workers () > 0 && thread_index == 0) + continue; + + if (cet->cryptodev_id == cit->dev_id && cet->cryptodev_q == cit->q_id) + { + s = format (s, "%u (%v)\n", thread_index, + vlib_worker_threads[thread_index].name); + break; + } + } + + if (thread_index == vec_len (cmt->per_thread_data)) + s = format (s, "%s\n", "free"); + + return s; +} + +static clib_error_t * +cryptodev_show_assignment_fn (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + cryptodev_main_t *cmt = &cryptodev_main; + u32 inst; + + vlib_cli_output (vm, "%-5s%-25s%-10s%s\n", "No.", "Name", "Queue-id", + "Assigned-to"); + if (vec_len (cmt->cryptodev_inst) == 0) + { + vlib_cli_output (vm, "(nil)\n"); + return 0; + } + + vec_foreach_index (inst, cmt->cryptodev_inst) + vlib_cli_output (vm, "%-5u%U", inst, format_cryptodev_inst, inst); + + return 0; +} + +VLIB_CLI_COMMAND (show_cryptodev_assignment, static) = { + .path = "show cryptodev assignment", + .short_help = "show cryptodev assignment", + .function = cryptodev_show_assignment_fn, +}; + +static clib_error_t * +cryptodev_set_assignment_fn (vlib_main_t * vm, unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + cryptodev_main_t *cmt = &cryptodev_main; + cryptodev_engine_thread_t *cet; + unformat_input_t _line_input, *line_input = &_line_input; + u32 thread_index, inst_index; + u32 thread_present = 0, inst_present = 0; + clib_error_t *error = 0; + int ret; + + /* Get a line of input. */ + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "thread %u", &thread_index)) + thread_present = 1; + else if (unformat (line_input, "resource %u", &inst_index)) + inst_present = 1; + else + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + return error; + } + } + + if (!thread_present || !inst_present) + { + error = clib_error_return (0, "mandatory argument(s) missing"); + return error; + } + + if (thread_index == 0 && vlib_num_workers () > 0) + { + error = + clib_error_return (0, "assign crypto resource for master thread"); + return error; + } + + if (thread_index > vec_len (cmt->per_thread_data) || + inst_index > vec_len (cmt->cryptodev_inst)) + { + error = clib_error_return (0, "wrong thread id or resource id"); + return error; + } + + cet = cmt->per_thread_data + thread_index; + ret = cryptodev_assign_resource (cet, inst_index, + CRYPTODEV_RESOURCE_ASSIGN_UPDATE); + if (ret) + { + error = clib_error_return (0, "cryptodev_assign_resource returned %i", + ret); + return error; + } + + return 0; +} + +VLIB_CLI_COMMAND (set_cryptodev_assignment, static) = { + .path = "set cryptodev assignment", + .short_help = "set cryptodev assignment thread <thread_index> " + "resource <inst_index>", + .function = cryptodev_set_assignment_fn, +}; + +static int +check_cryptodev_alg_support (u32 dev_id) +{ + const struct rte_cryptodev_symmetric_capability *cap; + struct rte_cryptodev_sym_capability_idx cap_idx; + +#define _(a, b, c, d, e, f) \ + cap_idx.type = RTE_CRYPTO_SYM_XFORM_##b; \ + cap_idx.algo.aead = RTE_CRYPTO_##b##_##c; \ + cap = rte_cryptodev_sym_capability_get (dev_id, &cap_idx); \ + if (!cap) \ + return -RTE_CRYPTO_##b##_##c; \ + else \ + { \ + if (cap->aead.digest_size.min > e || cap->aead.digest_size.max < e) \ + return -RTE_CRYPTO_##b##_##c; \ + if (cap->aead.aad_size.min > f || cap->aead.aad_size.max < f) \ + return -RTE_CRYPTO_##b##_##c; \ + if (cap->aead.iv_size.min > d || cap->aead.iv_size.max < d) \ + return -RTE_CRYPTO_##b##_##c; \ + } + + foreach_vnet_aead_crypto_conversion +#undef _ + +#define _(a, b, c, d) \ + cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER; \ + cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_##b; \ + cap = rte_cryptodev_sym_capability_get (dev_id, &cap_idx); \ + if (!cap) \ + return -RTE_CRYPTO_CIPHER_##b; \ + cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH; \ + cap_idx.algo.auth = RTE_CRYPTO_AUTH_##c##_HMAC; \ + cap = rte_cryptodev_sym_capability_get (dev_id, &cap_idx); \ + if (!cap) \ + return -RTE_CRYPTO_AUTH_##c; + + foreach_cryptodev_link_async_alg +#undef _ + return 0; +} + +static u32 +cryptodev_count_queue (u32 numa) +{ + struct rte_cryptodev_info info; + u32 n_cryptodev = rte_cryptodev_count (); + u32 i, q_count = 0; + + for (i = 0; i < n_cryptodev; i++) + { + rte_cryptodev_info_get (i, &info); + if (rte_cryptodev_socket_id (i) != numa) + { + clib_warning ("DPDK crypto resource %s is in different numa node " + "as %u, ignored", info.device->name, numa); + continue; + } + q_count += info.max_nb_queue_pairs; + } + + return q_count; +} + +static int +cryptodev_configure (vlib_main_t *vm, u32 cryptodev_id) +{ + struct rte_cryptodev_info info; + struct rte_cryptodev *cdev; + cryptodev_main_t *cmt = &cryptodev_main; + cryptodev_numa_data_t *numa_data = vec_elt_at_index (cmt->per_numa_data, + vm->numa_node); + u32 dp_size = 0; + u32 i; + int ret; + + cdev = rte_cryptodev_pmd_get_dev (cryptodev_id); + rte_cryptodev_info_get (cryptodev_id, &info); + + if (!(info.feature_flags & RTE_CRYPTODEV_FF_DATA_PLANE_SERVICE)) + return -1; + + ret = check_cryptodev_alg_support (cryptodev_id); + if (ret != 0) + return ret; + + + + /** If the device is already started, we reuse it, otherwise configure + * both the device and queue pair. + **/ + if (!cdev->data->dev_started) + { + struct rte_cryptodev_config cfg; + + cfg.socket_id = vm->numa_node; + cfg.nb_queue_pairs = info.max_nb_queue_pairs; + + rte_cryptodev_configure (cryptodev_id, &cfg); + + for (i = 0; i < info.max_nb_queue_pairs; i++) + { + struct rte_cryptodev_qp_conf qp_cfg; + + qp_cfg.mp_session = numa_data->sess_pool; + qp_cfg.mp_session_private = numa_data->sess_priv_pool; + qp_cfg.nb_descriptors = CRYPTODEV_NB_CRYPTO_OPS; + + ret = rte_cryptodev_queue_pair_setup (cryptodev_id, i, &qp_cfg, + vm->numa_node); + if (ret) + break; + } + if (i != info.max_nb_queue_pairs) + return -1; + + /* start the device */ + rte_cryptodev_start (i); + } + + ret = rte_cryptodev_get_dp_service_ctx_data_size (cryptodev_id); + if (ret < 0) + return -1; + dp_size = ret; + + for (i = 0; i < info.max_nb_queue_pairs; i++) + { + cryptodev_inst_t *cdev_inst; + vec_add2(cmt->cryptodev_inst, cdev_inst, 1); + cdev_inst->desc = vec_new (char, strlen (info.device->name) + 10); + cdev_inst->dev_id = cryptodev_id; + cdev_inst->q_id = i; + vec_validate_aligned(cdev_inst->dp_service_buffer, dp_size, 8); + snprintf (cdev_inst->desc, strlen (info.device->name) + 9, + "%s_q%u", info.device->name, i); + } + + return 0; +} + +static int +cryptodev_cmp (void *v1, void *v2) +{ + cryptodev_inst_t *a1 = v1; + cryptodev_inst_t *a2 = v2; + + if (a1->q_id > a2->q_id) + return 1; + if (a1->q_id < a2->q_id) + return -1; + return 0; +} + +static int +cryptodev_probe (vlib_main_t *vm, u32 n_workers) +{ + cryptodev_main_t *cmt = &cryptodev_main; + u32 n_queues = cryptodev_count_queue (vm->numa_node); + u32 i; + int ret; + + if (n_queues < n_workers) + return -1; + + for (i = 0; i < rte_cryptodev_count (); i++) + { + ret = cryptodev_configure (vm, i); + if (ret) + continue; + } + + vec_sort_with_function(cmt->cryptodev_inst, cryptodev_cmp); + + /* if there is not enough device stop cryptodev */ + if (vec_len (cmt->cryptodev_inst) < n_workers) + return -1; + + return 0; +} + +static int +cryptodev_get_session_sz (vlib_main_t *vm, u32 n_workers) +{ + u32 sess_data_sz = 0, i; + + if (rte_cryptodev_count () == 0) + { + clib_warning ("Failed"); + return -1; + } + + for (i = 0; i < rte_cryptodev_count (); i++) + { + u32 dev_sess_sz = rte_cryptodev_sym_get_private_session_size (i); + + sess_data_sz = dev_sess_sz > sess_data_sz ? dev_sess_sz : sess_data_sz; + } + + return sess_data_sz; +} + +static void +dpdk_disable_cryptodev_engine (vlib_main_t * vm) +{ + cryptodev_main_t *cmt = &cryptodev_main; + cryptodev_numa_data_t *numa_data; + cryptodev_engine_thread_t *ptd; + + vec_validate (cmt->per_numa_data, vm->numa_node); + numa_data = vec_elt_at_index (cmt->per_numa_data, vm->numa_node); + + if (numa_data->sess_pool) + rte_mempool_free (numa_data->sess_pool); + if (numa_data->sess_priv_pool) + rte_mempool_free (numa_data->sess_priv_pool); + + vec_foreach (ptd, cmt->per_thread_data) + { + if (ptd->aad_buf) + rte_free (ptd->aad_buf); + if (ptd->cached_frame) + rte_ring_free (ptd->cached_frame); + } +} + +clib_error_t * +dpdk_cryptodev_init (vlib_main_t * vm) +{ + cryptodev_main_t *cmt = &cryptodev_main; + vlib_thread_main_t *tm = vlib_get_thread_main (); + cryptodev_engine_thread_t *ptd; + cryptodev_numa_data_t *numa_data; + struct rte_mempool *mp; + u32 skip_master = vlib_num_workers () > 0; + u32 n_workers = tm->n_vlib_mains - skip_master; + u32 numa = vm->numa_node; + i32 sess_sz; + u32 eidx; + u32 i; + u8 *name = 0; + clib_error_t *error; + + cmt->iova_mode = rte_eal_iova_mode (); + + sess_sz = cryptodev_get_session_sz(vm, n_workers); + if (sess_sz < 0) + { + error = clib_error_return (0, "Not enough cryptodevs"); + return error; + } + + vec_validate (cmt->per_numa_data, vm->numa_node); + numa_data = vec_elt_at_index (cmt->per_numa_data, numa); + + /* create session pool for the numa node */ + name = format (0, "vcryptodev_sess_pool_%u%c", numa, 0); + mp = rte_cryptodev_sym_session_pool_create ((char *) name, + CRYPTODEV_NB_SESSION, + 0, 0, 0, numa); + if (!mp) + { + error = clib_error_return (0, "Not enough memory for mp %s", name); + goto err_handling; + } + vec_free (name); + + numa_data->sess_pool = mp; + + /* create session private pool for the numa node */ + name = format (0, "cryptodev_sess_pool_%u%c", numa, 0); + mp = rte_mempool_create ((char *) name, CRYPTODEV_NB_SESSION, sess_sz, 0, + 0, NULL, NULL, NULL, NULL, numa, 0); + if (!mp) + { + error = clib_error_return (0, "Not enough memory for mp %s", name); + vec_free (name); + goto err_handling; + } + + vec_free (name); + + numa_data->sess_priv_pool = mp; + + /* probe all cryptodev devices and get queue info */ + if (cryptodev_probe (vm, n_workers) < 0) + { + error = clib_error_return (0, "Failed to configure cryptodev"); + goto err_handling; + } + + clib_bitmap_vec_validate (cmt->active_cdev_inst_mask, tm->n_vlib_mains); + clib_spinlock_init (&cmt->tlock); + + vec_validate_aligned(cmt->per_thread_data, tm->n_vlib_mains - 1, + CLIB_CACHE_LINE_BYTES); + for (i = skip_master; i < tm->n_vlib_mains; i++) + { + ptd = cmt->per_thread_data + i; + cryptodev_assign_resource (ptd, 0, CRYPTODEV_RESOURCE_ASSIGN_AUTO); + ptd->aad_buf = rte_zmalloc_socket (0, CRYPTODEV_NB_CRYPTO_OPS * + CRYPTODEV_MAX_AAD_SIZE, + CLIB_CACHE_LINE_BYTES, + numa); + if (ptd->aad_buf == 0) + { + error = clib_error_return (0, "Failed to alloc aad buf"); + goto err_handling; + } + + ptd->aad_phy_addr = rte_malloc_virt2iova (ptd->aad_buf); + + name = format (0, "cache_frame_ring_%u%u", numa, i); + ptd->cached_frame = rte_ring_create ((char *)name, + CRYPTODEV_DEQ_CACHE_SZ, numa, + RING_F_SC_DEQ | RING_F_SP_ENQ); + + if (ptd->cached_frame == 0) + { + error = clib_error_return (0, "Failed to frame ring"); + goto err_handling; + } + vec_free (name); + } + + /* register handler */ + eidx = vnet_crypto_register_engine (vm, "dpdk_cryptodev", 79, + "DPDK Cryptodev Engine"); + +#define _(a, b, c, d, e, f) \ + vnet_crypto_register_async_handler \ + (vm, eidx, VNET_CRYPTO_OP_##a##_TAG##e##_AAD##f##_ENC, \ + cryptodev_enqueue_gcm_aad_##f##_enc,\ + cryptodev_frame_dequeue); \ + vnet_crypto_register_async_handler \ + (vm, eidx, VNET_CRYPTO_OP_##a##_TAG##e##_AAD##f##_DEC, \ + cryptodev_enqueue_gcm_aad_##f##_dec, \ + cryptodev_frame_dequeue); + + foreach_vnet_aead_crypto_conversion +#undef _ + +#define _(a, b, c, d) \ + vnet_crypto_register_async_handler \ + (vm, eidx, VNET_CRYPTO_OP_##a##_##c##_TAG##d##_ENC, \ + cryptodev_enqueue_linked_alg_enc, \ + cryptodev_frame_dequeue); \ + vnet_crypto_register_async_handler \ + (vm, eidx, VNET_CRYPTO_OP_##a##_##c##_TAG##d##_DEC, \ + cryptodev_enqueue_linked_alg_dec, \ + cryptodev_frame_dequeue); + + foreach_cryptodev_link_async_alg +#undef _ + + vnet_crypto_register_key_handler (vm, eidx, cryptodev_key_handler); + + return 0; + +err_handling: + dpdk_disable_cryptodev_engine (vm); + + return error; +} +/* *INDENT-On* */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vnet/crypto/crypto.c b/src/vnet/crypto/crypto.c index a82ebae4313..26fc4940e0a 100644 --- a/src/vnet/crypto/crypto.c +++ b/src/vnet/crypto/crypto.c @@ -710,7 +710,7 @@ vnet_crypto_init (vlib_main_t * vm) cm->async_alg_index_by_name = hash_create_string (0, sizeof (uword)); vec_validate_aligned (cm->threads, tm->n_vlib_mains, CLIB_CACHE_LINE_BYTES); vec_foreach (ct, cm->threads) - pool_alloc_aligned (ct->frame_pool, 256, CLIB_CACHE_LINE_BYTES); + pool_alloc_aligned (ct->frame_pool, 1024, CLIB_CACHE_LINE_BYTES); vec_validate (cm->algs, VNET_CRYPTO_N_ALGS); vec_validate (cm->async_algs, VNET_CRYPTO_N_ASYNC_ALGS); clib_bitmap_validate (cm->async_active_ids, VNET_CRYPTO_ASYNC_OP_N_IDS - 1); |