diff options
Diffstat (limited to 'drivers/crypto/qat')
| -rw-r--r-- | drivers/crypto/qat/Makefile | 3 | ||||
| -rw-r--r-- | drivers/crypto/qat/qat_adf/icp_qat_hw.h | 20 | ||||
| -rw-r--r-- | drivers/crypto/qat/qat_adf/qat_algs.h | 6 | ||||
| -rw-r--r-- | drivers/crypto/qat/qat_adf/qat_algs_build_desc.c | 28 | ||||
| -rw-r--r-- | drivers/crypto/qat/qat_crypto.c | 279 | ||||
| -rw-r--r-- | drivers/crypto/qat/qat_crypto.h | 17 | ||||
| -rw-r--r-- | drivers/crypto/qat/qat_crypto_capabilities.h | 54 | ||||
| -rw-r--r-- | drivers/crypto/qat/qat_qp.c | 20 | ||||
| -rw-r--r-- | drivers/crypto/qat/rte_qat_cryptodev.c | 63 |
9 files changed, 398 insertions, 92 deletions
diff --git a/drivers/crypto/qat/Makefile b/drivers/crypto/qat/Makefile index 7322ffe4..745d0197 100644 --- a/drivers/crypto/qat/Makefile +++ b/drivers/crypto/qat/Makefile @@ -43,6 +43,9 @@ CFLAGS += -O3 # external library include paths CFLAGS += -I$(SRCDIR)/qat_adf LDLIBS += -lcrypto +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_pci -lrte_bus_pci # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += qat_crypto.c diff --git a/drivers/crypto/qat/qat_adf/icp_qat_hw.h b/drivers/crypto/qat/qat_adf/icp_qat_hw.h index ebe245f6..d03688c7 100644 --- a/drivers/crypto/qat/qat_adf/icp_qat_hw.h +++ b/drivers/crypto/qat/qat_adf/icp_qat_hw.h @@ -301,6 +301,26 @@ enum icp_qat_hw_cipher_convert { #define ICP_QAT_HW_CIPHER_MAX_KEY_SZ ICP_QAT_HW_AES_256_F8_KEY_SZ +/* These defines describe position of the bit-fields + * in the flags byte in B0 + */ +#define ICP_QAT_HW_CCM_B0_FLAGS_ADATA_SHIFT 6 +#define ICP_QAT_HW_CCM_B0_FLAGS_T_SHIFT 3 + +#define ICP_QAT_HW_CCM_BUILD_B0_FLAGS(Adata, t, q) \ + ((((Adata) > 0 ? 1 : 0) << ICP_QAT_HW_CCM_B0_FLAGS_ADATA_SHIFT) \ + | ((((t) - 2) >> 1) << ICP_QAT_HW_CCM_B0_FLAGS_T_SHIFT) \ + | ((q) - 1)) + +#define ICP_QAT_HW_CCM_NQ_CONST 15 +#define ICP_QAT_HW_CCM_AAD_B0_LEN 16 +#define ICP_QAT_HW_CCM_AAD_LEN_INFO 2 +#define ICP_QAT_HW_CCM_AAD_DATA_OFFSET (ICP_QAT_HW_CCM_AAD_B0_LEN + \ + ICP_QAT_HW_CCM_AAD_LEN_INFO) +#define ICP_QAT_HW_CCM_AAD_ALIGNMENT 16 +#define ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE 4 +#define ICP_QAT_HW_CCM_NONCE_OFFSET 1 + struct icp_qat_hw_cipher_algo_blk { struct icp_qat_hw_cipher_config cipher_config; uint8_t key[ICP_QAT_HW_CIPHER_MAX_KEY_SZ]; diff --git a/drivers/crypto/qat/qat_adf/qat_algs.h b/drivers/crypto/qat/qat_adf/qat_algs.h index 2c8e03c0..802ba95d 100644 --- a/drivers/crypto/qat/qat_adf/qat_algs.h +++ b/drivers/crypto/qat/qat_adf/qat_algs.h @@ -104,8 +104,8 @@ struct qat_alg_buf_list { struct qat_crypto_op_cookie { struct qat_alg_buf_list qat_sgl_list_src; struct qat_alg_buf_list qat_sgl_list_dst; - phys_addr_t qat_sgl_src_phys_addr; - phys_addr_t qat_sgl_dst_phys_addr; + rte_iova_t qat_sgl_src_phys_addr; + rte_iova_t qat_sgl_dst_phys_addr; }; /* Common content descriptor */ @@ -124,7 +124,7 @@ struct qat_session { void *bpi_ctx; struct qat_alg_cd cd; uint8_t *cd_cur_ptr; - phys_addr_t cd_paddr; + rte_iova_t cd_paddr; struct icp_qat_fw_la_bulk_req fw_req; uint8_t aad_len; struct qat_crypto_instance *inst; diff --git a/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c b/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c index 2d16c9e2..db6c9a32 100644 --- a/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c +++ b/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c @@ -124,6 +124,9 @@ static int qat_hash_get_state1_size(enum icp_qat_hw_auth_algo qat_hash_alg) case ICP_QAT_HW_AUTH_ALGO_NULL: return QAT_HW_ROUND_UP(ICP_QAT_HW_NULL_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); + case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC: + return QAT_HW_ROUND_UP(ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ, + QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_DELIMITER: /* return maximum state1 size in this case */ return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA512_STATE1_SZ, @@ -876,6 +879,31 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, ICP_QAT_HW_AUTH_ALGO_NULL); state2_size = ICP_QAT_HW_NULL_STATE2_SZ; break; + case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC: + qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM; + state1_size = qat_hash_get_state1_size( + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC); + state2_size = ICP_QAT_HW_AES_CBC_MAC_KEY_SZ + + ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ; + + if (aad_length > 0) { + aad_length += ICP_QAT_HW_CCM_AAD_B0_LEN + + ICP_QAT_HW_CCM_AAD_LEN_INFO; + auth_param->u2.aad_sz = + RTE_ALIGN_CEIL(aad_length, + ICP_QAT_HW_CCM_AAD_ALIGNMENT); + } else { + auth_param->u2.aad_sz = ICP_QAT_HW_CCM_AAD_B0_LEN; + } + + cdesc->aad_len = aad_length; + hash->auth_counter.counter = 0; + + hash_cd_ctrl->outer_prefix_sz = digestsize; + auth_param->hash_state_sz = digestsize; + + memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen); + break; case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9: state1_size = qat_hash_get_state1_size( ICP_QAT_HW_AUTH_ALGO_KASUMI_F9); diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c index 1f52cabf..a572967c 100644 --- a/drivers/crypto/qat/qat_crypto.c +++ b/drivers/crypto/qat/qat_crypto.c @@ -44,15 +44,12 @@ #include <rte_log.h> #include <rte_debug.h> #include <rte_memory.h> -#include <rte_memzone.h> #include <rte_tailq.h> -#include <rte_ether.h> #include <rte_malloc.h> #include <rte_launch.h> #include <rte_eal.h> #include <rte_per_lcore.h> #include <rte_lcore.h> -#include <rte_atomic.h> #include <rte_branch_prediction.h> #include <rte_mempool.h> #include <rte_mbuf.h> @@ -60,7 +57,10 @@ #include <rte_spinlock.h> #include <rte_hexdump.h> #include <rte_crypto_sym.h> -#include <rte_cryptodev_pci.h> +#include <rte_byteorder.h> +#include <rte_pci.h> +#include <rte_bus_pci.h> + #include <openssl/evp.h> #include "qat_logs.h" @@ -253,10 +253,21 @@ qat_get_cmd_id(const struct rte_crypto_sym_xform *xform) /* AEAD */ if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + /* AES-GCM and AES-CCM works with different direction + * GCM first encrypts and generate hash where AES-CCM + * first generate hash and encrypts. Similar relation + * applies to decryption. + */ if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) - return ICP_QAT_FW_LA_CMD_CIPHER_HASH; + if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) + return ICP_QAT_FW_LA_CMD_CIPHER_HASH; + else + return ICP_QAT_FW_LA_CMD_HASH_CIPHER; else - return ICP_QAT_FW_LA_CMD_HASH_CIPHER; + if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) + return ICP_QAT_FW_LA_CMD_HASH_CIPHER; + else + return ICP_QAT_FW_LA_CMD_CIPHER_HASH; } if (xform->next == NULL) @@ -516,7 +527,7 @@ qat_crypto_set_session_parameters(struct rte_cryptodev *dev, PMD_INIT_FUNC_TRACE(); /* Set context descriptor physical address */ - session->cd_paddr = rte_mempool_virt2phy(NULL, session) + + session->cd_paddr = rte_mempool_virt2iova(session) + offsetof(struct qat_session, cd); session->min_qat_dev_gen = QAT_GEN1; @@ -736,6 +747,7 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, struct qat_session *session) { struct rte_crypto_aead_xform *aead_xform = &xform->aead; + enum rte_crypto_auth_operation crypto_operation; /* * Store AEAD IV parameters as cipher IV, @@ -755,21 +767,33 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; break; case RTE_CRYPTO_AEAD_AES_CCM: - PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported AEAD alg %u", - aead_xform->algo); - return -ENOTSUP; + if (qat_alg_validate_aes_key(aead_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid AES key size"); + return -EINVAL; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC; + break; default: PMD_DRV_LOG(ERR, "Crypto: Undefined AEAD specified %u\n", aead_xform->algo); return -EINVAL; } - if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { + if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && + aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) || + (aead_xform->op == RTE_CRYPTO_AEAD_OP_DECRYPT && + aead_xform->algo == RTE_CRYPTO_AEAD_AES_CCM)) { session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; /* * It needs to create cipher desc content first, * then authentication */ + + crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ? + RTE_CRYPTO_AUTH_OP_GENERATE : RTE_CRYPTO_AUTH_OP_VERIFY; + if (qat_alg_aead_session_create_content_desc_cipher(session, aead_xform->key.data, aead_xform->key.length)) @@ -780,7 +804,7 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, aead_xform->key.length, aead_xform->aad_length, aead_xform->digest_length, - RTE_CRYPTO_AUTH_OP_GENERATE)) + crypto_operation)) return -EINVAL; } else { session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; @@ -788,12 +812,16 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, * It needs to create authentication desc content first, * then cipher */ + + crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ? + RTE_CRYPTO_AUTH_OP_VERIFY : RTE_CRYPTO_AUTH_OP_GENERATE; + if (qat_alg_aead_session_create_content_desc_auth(session, aead_xform->key.data, aead_xform->key.length, aead_xform->aad_length, aead_xform->digest_length, - RTE_CRYPTO_AUTH_OP_VERIFY)) + crypto_operation)) return -EINVAL; if (qat_alg_aead_session_create_content_desc_cipher(session, @@ -923,6 +951,14 @@ qat_bpicipher_postprocess(struct qat_session *ctx, return sym_op->cipher.data.length - last_block_len; } +static inline void +txq_write_tail(struct qat_qp *qp, struct qat_queue *q) { + WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number, + q->hw_queue_number, q->tail); + q->nb_pending_requests = 0; + q->csr_tail = q->tail; +} + uint16_t qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) @@ -946,10 +982,10 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, tail = queue->tail; /* Find how many can actually fit on the ring */ - overflow = rte_atomic16_add_return(&tmp_qp->inflights16, nb_ops) - - queue->max_inflights; + tmp_qp->inflights16 += nb_ops; + overflow = tmp_qp->inflights16 - queue->max_inflights; if (overflow > 0) { - rte_atomic16_sub(&tmp_qp->inflights16, overflow); + tmp_qp->inflights16 -= overflow; nb_ops_possible = nb_ops - overflow; if (nb_ops_possible == 0) return 0; @@ -964,8 +1000,7 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, * This message cannot be enqueued, * decrease number of ops that wasn't sent */ - rte_atomic16_sub(&tmp_qp->inflights16, - nb_ops_possible - nb_ops_sent); + tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent; if (nb_ops_sent == 0) return 0; goto kick_tail; @@ -976,26 +1011,59 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, cur_op++; } kick_tail: - WRITE_CSR_RING_TAIL(tmp_qp->mmap_bar_addr, queue->hw_bundle_number, - queue->hw_queue_number, tail); queue->tail = tail; tmp_qp->stats.enqueued_count += nb_ops_sent; + queue->nb_pending_requests += nb_ops_sent; + if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH || + queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) { + txq_write_tail(tmp_qp, queue); + } return nb_ops_sent; } +static inline +void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q) +{ + uint32_t old_head, new_head; + uint32_t max_head; + + old_head = q->csr_head; + new_head = q->head; + max_head = qp->nb_descriptors * q->msg_size; + + /* write out free descriptors */ + void *cur_desc = (uint8_t *)q->base_addr + old_head; + + if (new_head < old_head) { + memset(cur_desc, ADF_RING_EMPTY_SIG, max_head - old_head); + memset(q->base_addr, ADF_RING_EMPTY_SIG, new_head); + } else { + memset(cur_desc, ADF_RING_EMPTY_SIG, new_head - old_head); + } + q->nb_processed_responses = 0; + q->csr_head = new_head; + + /* write current head to CSR */ + WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number, + q->hw_queue_number, new_head); +} + uint16_t qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) { - struct qat_queue *queue; + struct qat_queue *rx_queue, *tx_queue; struct qat_qp *tmp_qp = (struct qat_qp *)qp; uint32_t msg_counter = 0; struct rte_crypto_op *rx_op; struct icp_qat_fw_comn_resp *resp_msg; + uint32_t head; - queue = &(tmp_qp->rx_q); + rx_queue = &(tmp_qp->rx_q); + tx_queue = &(tmp_qp->tx_q); + head = rx_queue->head; resp_msg = (struct icp_qat_fw_comn_resp *) - ((uint8_t *)queue->base_addr + queue->head); + ((uint8_t *)rx_queue->base_addr + head); while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG && msg_counter != nb_ops) { @@ -1005,7 +1073,6 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg, sizeof(struct icp_qat_fw_comn_resp)); - #endif if (ICP_QAT_FW_COMN_STATUS_FLAG_OK != ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET( @@ -1022,23 +1089,26 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; } - *(uint32_t *)resp_msg = ADF_RING_EMPTY_SIG; - queue->head = adf_modulo(queue->head + - queue->msg_size, - ADF_RING_SIZE_MODULO(queue->queue_size)); + head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo); resp_msg = (struct icp_qat_fw_comn_resp *) - ((uint8_t *)queue->base_addr + - queue->head); + ((uint8_t *)rx_queue->base_addr + head); *ops = rx_op; ops++; msg_counter++; } if (msg_counter > 0) { - WRITE_CSR_RING_HEAD(tmp_qp->mmap_bar_addr, - queue->hw_bundle_number, - queue->hw_queue_number, queue->head); - rte_atomic16_sub(&tmp_qp->inflights16, msg_counter); + rx_queue->head = head; tmp_qp->stats.dequeued_count += msg_counter; + rx_queue->nb_processed_responses += msg_counter; + tmp_qp->inflights16 -= msg_counter; + + if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) + rxq_free_desc(tmp_qp, rx_queue); + } + /* also check if tail needs to be advanced */ + if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH && + tx_queue->tail != tx_queue->csr_tail) { + txq_write_tail(tmp_qp, tx_queue); } return msg_counter; } @@ -1049,7 +1119,7 @@ qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start, { int nr = 1; - uint32_t buf_len = rte_pktmbuf_mtophys(buf) - + uint32_t buf_len = rte_pktmbuf_iova(buf) - buff_start + rte_pktmbuf_data_len(buf); list->bufers[0].addr = buff_start; @@ -1073,7 +1143,7 @@ qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start, list->bufers[nr].len = rte_pktmbuf_data_len(buf); list->bufers[nr].resrvd = 0; - list->bufers[nr].addr = rte_pktmbuf_mtophys(buf); + list->bufers[nr].addr = rte_pktmbuf_iova(buf); buf_len += list->bufers[nr].len; buf = buf->next; @@ -1112,6 +1182,29 @@ set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, } } +/** Set IV for CCM is special case, 0th byte is set to q-1 + * where q is padding of nonce in 16 byte block + */ +static inline void +set_cipher_iv_ccm(uint16_t iv_length, uint16_t iv_offset, + struct icp_qat_fw_la_cipher_req_params *cipher_param, + struct rte_crypto_op *op, uint8_t q, uint8_t aad_len_field_sz) +{ + rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array) + + ICP_QAT_HW_CCM_NONCE_OFFSET, + rte_crypto_op_ctod_offset(op, uint8_t *, + iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET, + iv_length); + *(uint8_t *)&cipher_param->u.cipher_IV_array[0] = + q - ICP_QAT_HW_CCM_NONCE_OFFSET; + + if (aad_len_field_sz) + rte_memcpy(&op->sym->aead.aad.data[ICP_QAT_HW_CCM_NONCE_OFFSET], + rte_crypto_op_ctod_offset(op, uint8_t *, + iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET, + iv_length); +} + static inline int qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp) @@ -1156,6 +1249,8 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, return -EINVAL; } + + qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; @@ -1164,9 +1259,13 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER || ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { - /* AES-GCM */ + /* AES-GCM or AES-CCM */ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || - ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { + ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 || + (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128 + && ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE + && ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) { do_aead = 1; } else { do_auth = 1; @@ -1273,6 +1372,11 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, } if (do_aead) { + /* + * This address may used for setting AAD physical pointer + * into IV offset from op + */ + rte_iova_t aad_phys_addr_aead = op->sym->aead.aad.phys_addr; if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || ctx->qat_hash_alg == @@ -1286,6 +1390,87 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); } + set_cipher_iv(ctx->cipher_iv.length, + ctx->cipher_iv.offset, + cipher_param, op, qat_req); + + } else if (ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC) { + + /* In case of AES-CCM this may point to user selected memory + * or iv offset in cypto_op + */ + uint8_t *aad_data = op->sym->aead.aad.data; + /* This is true AAD length, it not includes 18 bytes of + * preceding data + */ + uint8_t aad_ccm_real_len = 0; + + uint8_t aad_len_field_sz = 0; + uint32_t msg_len_be = + rte_bswap32(op->sym->aead.data.length); + + if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) { + aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO; + aad_ccm_real_len = ctx->aad_len - + ICP_QAT_HW_CCM_AAD_B0_LEN - + ICP_QAT_HW_CCM_AAD_LEN_INFO; + } else { + /* + * aad_len not greater than 18, so no actual aad data, + * then use IV after op for B0 block + */ + aad_data = rte_crypto_op_ctod_offset(op, uint8_t *, + ctx->cipher_iv.offset); + aad_phys_addr_aead = + rte_crypto_op_ctophys_offset(op, + ctx->cipher_iv.offset); + } + + uint8_t q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length; + + aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(aad_len_field_sz, + ctx->digest_length, q); + + if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) { + memcpy(aad_data + ctx->cipher_iv.length + + ICP_QAT_HW_CCM_NONCE_OFFSET + + (q - ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE), + (uint8_t *)&msg_len_be, + ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE); + } else { + memcpy(aad_data + ctx->cipher_iv.length + + ICP_QAT_HW_CCM_NONCE_OFFSET, + (uint8_t *)&msg_len_be + + (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE + - q), q); + } + + if (aad_len_field_sz > 0) { + *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] + = rte_bswap16(aad_ccm_real_len); + + if ((aad_ccm_real_len + aad_len_field_sz) + % ICP_QAT_HW_CCM_AAD_B0_LEN) { + uint8_t pad_len = 0; + uint8_t pad_idx = 0; + + pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN - + ((aad_ccm_real_len + aad_len_field_sz) % + ICP_QAT_HW_CCM_AAD_B0_LEN); + pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN + + aad_ccm_real_len + aad_len_field_sz; + memset(&aad_data[pad_idx], + 0, pad_len); + } + + } + + set_cipher_iv_ccm(ctx->cipher_iv.length, + ctx->cipher_iv.offset, + cipher_param, op, q, + aad_len_field_sz); + } cipher_len = op->sym->aead.data.length; @@ -1293,10 +1478,8 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, auth_len = op->sym->aead.data.length; auth_ofs = op->sym->aead.data.offset; - auth_param->u1.aad_adr = op->sym->aead.aad.phys_addr; + auth_param->u1.aad_adr = aad_phys_addr_aead; auth_param->auth_res_addr = op->sym->aead.digest.phys_addr; - set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset, - cipher_param, op, qat_req); min_ofs = op->sym->aead.data.offset; } @@ -1316,26 +1499,26 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, * so as not to overwrite data in dest buffer */ src_buf_start = - rte_pktmbuf_mtophys_offset(op->sym->m_src, min_ofs); + rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs); dst_buf_start = - rte_pktmbuf_mtophys_offset(op->sym->m_dst, min_ofs); + rte_pktmbuf_iova_offset(op->sym->m_dst, min_ofs); } else { /* In-place operation * Start DMA at nearest aligned address below min_ofs */ src_buf_start = - rte_pktmbuf_mtophys_offset(op->sym->m_src, min_ofs) + rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs) & QAT_64_BTYE_ALIGN_MASK; - if (unlikely((rte_pktmbuf_mtophys(op->sym->m_src) - + if (unlikely((rte_pktmbuf_iova(op->sym->m_src) - rte_pktmbuf_headroom(op->sym->m_src)) > src_buf_start)) { /* alignment has pushed addr ahead of start of mbuf * so revert and take the performance hit */ src_buf_start = - rte_pktmbuf_mtophys_offset(op->sym->m_src, + rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs); } dst_buf_start = src_buf_start; @@ -1343,7 +1526,7 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, if (do_cipher || do_aead) { cipher_param->cipher_offset = - (uint32_t)rte_pktmbuf_mtophys_offset( + (uint32_t)rte_pktmbuf_iova_offset( op->sym->m_src, cipher_ofs) - src_buf_start; cipher_param->cipher_length = cipher_len; } else { @@ -1352,7 +1535,7 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, } if (do_auth || do_aead) { - auth_param->auth_off = (uint32_t)rte_pktmbuf_mtophys_offset( + auth_param->auth_off = (uint32_t)rte_pktmbuf_iova_offset( op->sym->m_src, auth_ofs) - src_buf_start; auth_param->auth_len = auth_len; } else { diff --git a/drivers/crypto/qat/qat_crypto.h b/drivers/crypto/qat/qat_crypto.h index 3f35a00e..c64d7756 100644 --- a/drivers/crypto/qat/qat_crypto.h +++ b/drivers/crypto/qat/qat_crypto.h @@ -50,6 +50,13 @@ (((num) + (align) - 1) & ~((align) - 1)) #define QAT_64_BTYE_ALIGN_MASK (~0x3f) +#define QAT_CSR_HEAD_WRITE_THRESH 32U +/* number of requests to accumulate before writing head CSR */ +#define QAT_CSR_TAIL_WRITE_THRESH 32U +/* number of requests to accumulate before writing tail CSR */ +#define QAT_CSR_TAIL_FORCE_WRITE_THRESH 256U +/* number of inflights below which no tail write coalescing should occur */ + struct qat_session; enum qat_device_gen { @@ -63,7 +70,7 @@ enum qat_device_gen { struct qat_queue { char memz_name[RTE_MEMZONE_NAMESIZE]; void *base_addr; /* Base address */ - phys_addr_t base_phys_addr; /* Queue physical address */ + rte_iova_t base_phys_addr; /* Queue physical address */ uint32_t head; /* Shadow copy of the head */ uint32_t tail; /* Shadow copy of the tail */ uint32_t modulo; @@ -73,11 +80,17 @@ struct qat_queue { uint8_t hw_bundle_number; uint8_t hw_queue_number; /* HW queue aka ring offset on bundle */ + uint32_t csr_head; /* last written head value */ + uint32_t csr_tail; /* last written tail value */ + uint16_t nb_processed_responses; + /* number of responses processed since last CSR head write */ + uint16_t nb_pending_requests; + /* number of requests pending since last CSR tail write */ }; struct qat_qp { void *mmap_bar_addr; - rte_atomic16_t inflights16; + uint16_t inflights16; struct qat_queue tx_q; struct qat_queue rx_q; struct rte_cryptodev_stats stats; diff --git a/drivers/crypto/qat/qat_crypto_capabilities.h b/drivers/crypto/qat/qat_crypto_capabilities.h index 70120072..89ba27d4 100644 --- a/drivers/crypto/qat/qat_crypto_capabilities.h +++ b/drivers/crypto/qat/qat_crypto_capabilities.h @@ -48,9 +48,9 @@ .increment = 1 \ }, \ .digest_size = { \ - .min = 20, \ + .min = 1, \ .max = 20, \ - .increment = 0 \ + .increment = 1 \ }, \ .iv_size = { 0 } \ }, } \ @@ -69,9 +69,9 @@ .increment = 1 \ }, \ .digest_size = { \ - .min = 28, \ + .min = 1, \ .max = 28, \ - .increment = 0 \ + .increment = 1 \ }, \ .iv_size = { 0 } \ }, } \ @@ -90,9 +90,9 @@ .increment = 1 \ }, \ .digest_size = { \ - .min = 32, \ + .min = 1, \ .max = 32, \ - .increment = 0 \ + .increment = 1 \ }, \ .iv_size = { 0 } \ }, } \ @@ -111,9 +111,9 @@ .increment = 1 \ }, \ .digest_size = { \ - .min = 48, \ + .min = 1, \ .max = 48, \ - .increment = 0 \ + .increment = 1 \ }, \ .iv_size = { 0 } \ }, } \ @@ -132,9 +132,9 @@ .increment = 1 \ }, \ .digest_size = { \ - .min = 64, \ + .min = 1, \ .max = 64, \ - .increment = 0 \ + .increment = 1 \ }, \ .iv_size = { 0 } \ }, } \ @@ -153,9 +153,9 @@ .increment = 1 \ }, \ .digest_size = { \ - .min = 16, \ + .min = 1, \ .max = 16, \ - .increment = 0 \ + .increment = 1 \ }, \ .iv_size = { 0 } \ }, } \ @@ -183,6 +183,36 @@ }, } \ }, } \ }, \ + { /* AES CCM */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, \ + {.aead = { \ + .algo = RTE_CRYPTO_AEAD_AES_CCM, \ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + }, \ + .digest_size = { \ + .min = 4, \ + .max = 16, \ + .increment = 2 \ + }, \ + .aad_size = { \ + .min = 0, \ + .max = 224, \ + .increment = 1 \ + }, \ + .iv_size = { \ + .min = 7, \ + .max = 13, \ + .increment = 1 \ + }, \ + }, } \ + }, } \ + }, \ { /* AES GCM */ \ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ {.sym = { \ diff --git a/drivers/crypto/qat/qat_qp.c b/drivers/crypto/qat/qat_qp.c index 5048d214..ced3aa6a 100644 --- a/drivers/crypto/qat/qat_qp.c +++ b/drivers/crypto/qat/qat_qp.c @@ -37,6 +37,7 @@ #include <rte_memzone.h> #include <rte_cryptodev_pmd.h> #include <rte_pci.h> +#include <rte_bus_pci.h> #include <rte_atomic.h> #include <rte_prefetch.h> @@ -122,14 +123,9 @@ queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size, break; default: memzone_flags = RTE_MEMZONE_SIZE_HINT_ONLY; -} -#ifdef RTE_LIBRTE_XEN_DOM0 - return rte_memzone_reserve_bounded(queue_name, queue_size, - socket_id, 0, RTE_CACHE_LINE_SIZE, RTE_PGSIZE_2M); -#else + } return rte_memzone_reserve_aligned(queue_name, queue_size, socket_id, memzone_flags, queue_size); -#endif } int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, @@ -186,7 +182,7 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, RTE_CACHE_LINE_SIZE); qp->mmap_bar_addr = pci_dev->mem_resource[0].addr; - rte_atomic16_init(&qp->inflights16); + qp->inflights16 = 0; if (qat_tx_queue_create(dev, &(qp->tx_q), queue_pair_id, qp_conf->nb_descriptors, socket_id) != 0) { @@ -232,14 +228,12 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, qp->op_cookies[i]; sql_cookie->qat_sgl_src_phys_addr = - rte_mempool_virt2phy(qp->op_cookie_pool, - sql_cookie) + + rte_mempool_virt2iova(sql_cookie) + offsetof(struct qat_crypto_op_cookie, qat_sgl_list_src); sql_cookie->qat_sgl_dst_phys_addr = - rte_mempool_virt2phy(qp->op_cookie_pool, - sql_cookie) + + rte_mempool_virt2iova(sql_cookie) + offsetof(struct qat_crypto_op_cookie, qat_sgl_list_dst); } @@ -269,7 +263,7 @@ int qat_crypto_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) } /* Don't free memory if there are still responses to be processed */ - if (rte_atomic16_read(&(qp->inflights16)) == 0) { + if (qp->inflights16 == 0) { qat_queue_delete(&(qp->tx_q)); qat_queue_delete(&(qp->rx_q)); } else { @@ -377,7 +371,7 @@ qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue, } queue->base_addr = (char *)qp_mz->addr; - queue->base_phys_addr = qp_mz->phys_addr; + queue->base_phys_addr = qp_mz->iova; if (qat_qp_check_queue_alignment(queue->base_phys_addr, queue_size_bytes)) { PMD_DRV_LOG(ERR, "Invalid alignment on queue create " diff --git a/drivers/crypto/qat/rte_qat_cryptodev.c b/drivers/crypto/qat/rte_qat_cryptodev.c index 7d56fca4..4f8e4bfe 100644 --- a/drivers/crypto/qat/rte_qat_cryptodev.c +++ b/drivers/crypto/qat/rte_qat_cryptodev.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,11 +31,12 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include <rte_bus_pci.h> #include <rte_common.h> #include <rte_dev.h> #include <rte_malloc.h> +#include <rte_pci.h> #include <rte_cryptodev_pmd.h> -#include <rte_cryptodev_pci.h> #include "qat_crypto.h" #include "qat_logs.h" @@ -97,15 +98,18 @@ static const struct rte_pci_id pci_id_qat_map[] = { }; static int -crypto_qat_dev_init(struct rte_cryptodev *cryptodev) +crypto_qat_create(const char *name, struct rte_pci_device *pci_dev, + struct rte_cryptodev_pmd_init_params *init_params) { + struct rte_cryptodev *cryptodev; struct qat_pmd_private *internals; PMD_INIT_FUNC_TRACE(); - PMD_DRV_LOG(DEBUG, "Found crypto device at %02x:%02x.%x", - RTE_DEV_TO_PCI(cryptodev->device)->addr.bus, - RTE_DEV_TO_PCI(cryptodev->device)->addr.devid, - RTE_DEV_TO_PCI(cryptodev->device)->addr.function); + + cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device, + init_params); + if (cryptodev == NULL) + return -ENODEV; cryptodev->driver_id = cryptodev_qat_driver_id; cryptodev->dev_ops = &crypto_qat_ops; @@ -119,8 +123,8 @@ crypto_qat_dev_init(struct rte_cryptodev *cryptodev) RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; internals = cryptodev->data->dev_private; - internals->max_nb_sessions = RTE_QAT_PMD_MAX_NB_SESSIONS; - switch (RTE_DEV_TO_PCI(cryptodev->device)->id.device_id) { + internals->max_nb_sessions = init_params->max_nb_sessions; + switch (pci_dev->id.device_id) { case 0x0443: internals->qat_dev_gen = QAT_GEN1; internals->qat_dev_capabilities = qat_gen1_capabilities; @@ -151,15 +155,43 @@ crypto_qat_dev_init(struct rte_cryptodev *cryptodev) } static int crypto_qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, - struct rte_pci_device *pci_dev) + struct rte_pci_device *pci_dev) { - return rte_cryptodev_pci_generic_probe(pci_dev, - sizeof(struct qat_pmd_private), crypto_qat_dev_init); + struct rte_cryptodev_pmd_init_params init_params = { + .name = "", + .socket_id = rte_socket_id(), + .private_data_size = sizeof(struct qat_pmd_private), + .max_nb_sessions = RTE_QAT_PMD_MAX_NB_SESSIONS + }; + char name[RTE_CRYPTODEV_NAME_MAX_LEN]; + + PMD_DRV_LOG(DEBUG, "Found QAT device at %02x:%02x.%x", + pci_dev->addr.bus, + pci_dev->addr.devid, + pci_dev->addr.function); + + rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); + + return crypto_qat_create(name, pci_dev, &init_params); } static int crypto_qat_pci_remove(struct rte_pci_device *pci_dev) { - return rte_cryptodev_pci_generic_remove(pci_dev, NULL); + struct rte_cryptodev *cryptodev; + char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; + + if (pci_dev == NULL) + return -EINVAL; + + rte_pci_device_name(&pci_dev->addr, cryptodev_name, + sizeof(cryptodev_name)); + + cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name); + if (cryptodev == NULL) + return -ENODEV; + + /* free crypto device */ + return rte_cryptodev_pmd_destroy(cryptodev); } static struct rte_pci_driver rte_qat_pmd = { @@ -169,6 +201,9 @@ static struct rte_pci_driver rte_qat_pmd = { .remove = crypto_qat_pci_remove }; +static struct cryptodev_driver qat_crypto_drv; + RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_QAT_SYM_PMD, rte_qat_pmd); RTE_PMD_REGISTER_PCI_TABLE(CRYPTODEV_NAME_QAT_SYM_PMD, pci_id_qat_map); -RTE_PMD_REGISTER_CRYPTO_DRIVER(rte_qat_pmd, cryptodev_qat_driver_id); +RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv, rte_qat_pmd, + cryptodev_qat_driver_id); |