diff options
Diffstat (limited to 'drivers/crypto')
72 files changed, 3534 insertions, 1651 deletions
diff --git a/drivers/crypto/aesni_gcm/Makefile b/drivers/crypto/aesni_gcm/Makefile index 59a7c6a9..6fca5e1c 100644 --- a/drivers/crypto/aesni_gcm/Makefile +++ b/drivers/crypto/aesni_gcm/Makefile @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2016 Intel Corporation. All rights reserved. +# Copyright(c) 2016-2017 Intel Corporation. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -31,6 +31,9 @@ include $(RTE_SDK)/mk/rte.vars.mk ifneq ($(MAKECMDGOALS),clean) +ifeq ($(AESNI_MULTI_BUFFER_LIB_PATH),) +$(error "Please define AESNI_MULTI_BUFFER_LIB_PATH environment variable") +endif endif # library name @@ -47,7 +50,9 @@ LIBABIVER := 1 EXPORT_MAP := rte_pmd_aesni_gcm_version.map # external library dependencies -LDLIBS += -lisal_crypto +CFLAGS += -I$(AESNI_MULTI_BUFFER_LIB_PATH) +CFLAGS += -I$(AESNI_MULTI_BUFFER_LIB_PATH)/include +LDLIBS += -L$(AESNI_MULTI_BUFFER_LIB_PATH) -lIPSec_MB # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += aesni_gcm_pmd.c diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_ops.h b/drivers/crypto/aesni_gcm/aesni_gcm_ops.h index e9de6546..d6a18efc 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_ops.h +++ b/drivers/crypto/aesni_gcm/aesni_gcm_ops.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -37,26 +37,109 @@ #define LINUX #endif -#include <isa-l_crypto/aes_gcm.h> +#include <gcm_defines.h> +#include <aux_funcs.h> -typedef void (*aesni_gcm_init_t)(struct gcm_data *my_ctx_data, - uint8_t *iv, +/** Supported vector modes */ +enum aesni_gcm_vector_mode { + RTE_AESNI_GCM_NOT_SUPPORTED = 0, + RTE_AESNI_GCM_SSE, + RTE_AESNI_GCM_AVX, + RTE_AESNI_GCM_AVX2, + RTE_AESNI_GCM_VECTOR_NUM +}; + +enum aesni_gcm_key { + AESNI_GCM_KEY_128, + AESNI_GCM_KEY_192, + AESNI_GCM_KEY_256, + AESNI_GCM_KEY_NUM +}; + + +typedef void (*aesni_gcm_t)(const struct gcm_key_data *gcm_key_data, + struct gcm_context_data *gcm_ctx_data, uint8_t *out, + const uint8_t *in, uint64_t plaintext_len, const uint8_t *iv, + const uint8_t *aad, uint64_t aad_len, + uint8_t *auth_tag, uint64_t auth_tag_len); + +typedef void (*aesni_gcm_precomp_t)(const void *key, struct gcm_key_data *gcm_data); + +typedef void (*aesni_gcm_init_t)(const struct gcm_key_data *gcm_key_data, + struct gcm_context_data *gcm_ctx_data, + const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); -typedef void (*aesni_gcm_update_t)(struct gcm_data *my_ctx_data, +typedef void (*aesni_gcm_update_t)(const struct gcm_key_data *gcm_key_data, + struct gcm_context_data *gcm_ctx_data, uint8_t *out, const uint8_t *in, uint64_t plaintext_len); -typedef void (*aesni_gcm_finalize_t)(struct gcm_data *my_ctx_data, +typedef void (*aesni_gcm_finalize_t)(const struct gcm_key_data *gcm_key_data, + struct gcm_context_data *gcm_ctx_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** GCM library function pointer table */ struct aesni_gcm_ops { + aesni_gcm_t enc; /**< GCM encode function pointer */ + aesni_gcm_t dec; /**< GCM decode function pointer */ + aesni_gcm_precomp_t precomp; /**< GCM pre-compute */ aesni_gcm_init_t init; - aesni_gcm_update_t update; + aesni_gcm_update_t update_enc; + aesni_gcm_update_t update_dec; aesni_gcm_finalize_t finalize; }; +#define AES_GCM_FN(keylen, arch) \ +aes_gcm_enc_##keylen##_##arch,\ +aes_gcm_dec_##keylen##_##arch,\ +aes_gcm_pre_##keylen##_##arch,\ +aes_gcm_init_##keylen##_##arch,\ +aes_gcm_enc_##keylen##_update_##arch,\ +aes_gcm_dec_##keylen##_update_##arch,\ +aes_gcm_enc_##keylen##_finalize_##arch, + +static const struct aesni_gcm_ops gcm_ops[RTE_AESNI_GCM_VECTOR_NUM][AESNI_GCM_KEY_NUM] = { + [RTE_AESNI_GCM_NOT_SUPPORTED] = { + [AESNI_GCM_KEY_128] = {NULL}, + [AESNI_GCM_KEY_192] = {NULL}, + [AESNI_GCM_KEY_256] = {NULL} + }, + [RTE_AESNI_GCM_SSE] = { + [AESNI_GCM_KEY_128] = { + AES_GCM_FN(128, sse) + }, + [AESNI_GCM_KEY_192] = { + AES_GCM_FN(192, sse) + }, + [AESNI_GCM_KEY_256] = { + AES_GCM_FN(256, sse) + } + }, + [RTE_AESNI_GCM_AVX] = { + [AESNI_GCM_KEY_128] = { + AES_GCM_FN(128, avx_gen2) + }, + [AESNI_GCM_KEY_192] = { + AES_GCM_FN(192, avx_gen2) + }, + [AESNI_GCM_KEY_256] = { + AES_GCM_FN(256, avx_gen2) + } + }, + [RTE_AESNI_GCM_AVX2] = { + [AESNI_GCM_KEY_128] = { + AES_GCM_FN(128, avx_gen4) + }, + [AESNI_GCM_KEY_192] = { + AES_GCM_FN(192, avx_gen4) + }, + [AESNI_GCM_KEY_256] = { + AES_GCM_FN(256, avx_gen4) + } + } +}; #endif /* _AESNI_GCM_OPS_H_ */ diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c index 101ef98b..d9c91d06 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -42,133 +43,155 @@ #include "aesni_gcm_pmd_private.h" -/** GCM encode functions pointer table */ -static const struct aesni_gcm_ops aesni_gcm_enc[] = { - [AESNI_GCM_KEY_128] = { - aesni_gcm128_init, - aesni_gcm128_enc_update, - aesni_gcm128_enc_finalize - }, - [AESNI_GCM_KEY_256] = { - aesni_gcm256_init, - aesni_gcm256_enc_update, - aesni_gcm256_enc_finalize - } -}; - -/** GCM decode functions pointer table */ -static const struct aesni_gcm_ops aesni_gcm_dec[] = { - [AESNI_GCM_KEY_128] = { - aesni_gcm128_init, - aesni_gcm128_dec_update, - aesni_gcm128_dec_finalize - }, - [AESNI_GCM_KEY_256] = { - aesni_gcm256_init, - aesni_gcm256_dec_update, - aesni_gcm256_dec_finalize - } -}; +static uint8_t cryptodev_driver_id; /** Parse crypto xform chain and set private session parameters */ int -aesni_gcm_set_session_parameters(struct aesni_gcm_session *sess, +aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops, + struct aesni_gcm_session *sess, const struct rte_crypto_sym_xform *xform) { const struct rte_crypto_sym_xform *auth_xform; - const struct rte_crypto_sym_xform *cipher_xform; - - if (xform->next == NULL || xform->next->next != NULL) { - GCM_LOG_ERR("Two and only two chained xform required"); - return -EINVAL; - } + const struct rte_crypto_sym_xform *aead_xform; + uint16_t digest_length; + uint8_t key_length; + uint8_t *key; - if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && - xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { - auth_xform = xform->next; - cipher_xform = xform; - } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && - xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { + /* AES-GMAC */ + if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { auth_xform = xform; - cipher_xform = xform->next; + if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC) { + GCM_LOG_ERR("Only AES GMAC is supported as an " + "authentication only algorithm"); + return -ENOTSUP; + } + /* Set IV parameters */ + sess->iv.offset = auth_xform->auth.iv.offset; + sess->iv.length = auth_xform->auth.iv.length; + + /* Select Crypto operation */ + if (auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) + sess->op = AESNI_GMAC_OP_GENERATE; + else + sess->op = AESNI_GMAC_OP_VERIFY; + + key_length = auth_xform->auth.key.length; + key = auth_xform->auth.key.data; + digest_length = auth_xform->auth.digest_length; + + /* AES-GCM */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + aead_xform = xform; + + if (aead_xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM) { + GCM_LOG_ERR("The only combined operation " + "supported is AES GCM"); + return -ENOTSUP; + } + + /* Set IV parameters */ + sess->iv.offset = aead_xform->aead.iv.offset; + sess->iv.length = aead_xform->aead.iv.length; + + /* Select Crypto operation */ + if (aead_xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) + sess->op = AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION; + else + sess->op = AESNI_GCM_OP_AUTHENTICATED_DECRYPTION; + + key_length = aead_xform->aead.key.length; + key = aead_xform->aead.key.data; + + sess->aad_length = aead_xform->aead.aad_length; + digest_length = aead_xform->aead.digest_length; } else { - GCM_LOG_ERR("Cipher and auth xform required"); - return -EINVAL; + GCM_LOG_ERR("Wrong xform type, has to be AEAD or authentication"); + return -ENOTSUP; } - if (!(cipher_xform->cipher.algo == RTE_CRYPTO_CIPHER_AES_GCM && - (auth_xform->auth.algo == RTE_CRYPTO_AUTH_AES_GCM || - auth_xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC))) { - GCM_LOG_ERR("We only support AES GCM and AES GMAC"); - return -EINVAL; - } - /* Select Crypto operation */ - if (cipher_xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT && - auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) - sess->op = AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION; - else if (cipher_xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT && - auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) - sess->op = AESNI_GCM_OP_AUTHENTICATED_DECRYPTION; - else { - GCM_LOG_ERR("Cipher/Auth operations: Encrypt/Generate or" - " Decrypt/Verify are valid only"); + /* IV check */ + if (sess->iv.length != 16 && sess->iv.length != 12 && + sess->iv.length != 0) { + GCM_LOG_ERR("Wrong IV length"); return -EINVAL; } /* Check key length and calculate GCM pre-compute. */ - switch (cipher_xform->cipher.key.length) { + switch (key_length) { case 16: - aesni_gcm128_pre(cipher_xform->cipher.key.data, &sess->gdata); sess->key = AESNI_GCM_KEY_128; - + break; + case 24: + sess->key = AESNI_GCM_KEY_192; break; case 32: - aesni_gcm256_pre(cipher_xform->cipher.key.data, &sess->gdata); sess->key = AESNI_GCM_KEY_256; - break; default: - GCM_LOG_ERR("Unsupported cipher key length"); + GCM_LOG_ERR("Invalid key length"); + return -EINVAL; + } + + gcm_ops[sess->key].precomp(key, &sess->gdata_key); + + /* Digest check */ + if (digest_length != 16 && + digest_length != 12 && + digest_length != 8) { + GCM_LOG_ERR("digest"); return -EINVAL; } + sess->digest_length = digest_length; return 0; } /** Get gcm session */ static struct aesni_gcm_session * -aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_sym_op *op) +aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_op *op) { struct aesni_gcm_session *sess = NULL; - - if (op->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->session->dev_type - != RTE_CRYPTODEV_AESNI_GCM_PMD)) - return sess; - - sess = (struct aesni_gcm_session *)op->session->_private; + struct rte_crypto_sym_op *sym_op = op->sym; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(sym_op->session != NULL)) + sess = (struct aesni_gcm_session *) + get_session_private_data( + sym_op->session, + cryptodev_driver_id); } else { void *_sess; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; - if (rte_mempool_get(qp->sess_mp, &_sess)) - return sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; - sess = (struct aesni_gcm_session *) - ((struct rte_cryptodev_sym_session *)_sess)->_private; + sess = (struct aesni_gcm_session *)_sess_private_data; - if (unlikely(aesni_gcm_set_session_parameters(sess, - op->xform) != 0)) { + if (unlikely(aesni_gcm_set_session_parameters(qp->ops, + sess, sym_op->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); sess = NULL; } + sym_op->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(sym_op->session, cryptodev_driver_id, + _sess_private_data); } + + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } /** - * Process a crypto operation and complete a JOB_AES_HMAC job structure for - * submission to the multi buffer library for processing. + * Process a crypto operation, calling + * the GCM API from the multi buffer library. * * @param qp queue pair * @param op symmetric crypto operation @@ -178,14 +201,27 @@ aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_sym_op *op) * */ static int -process_gcm_crypto_op(struct rte_crypto_sym_op *op, +process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, struct aesni_gcm_session *session) { uint8_t *src, *dst; - struct rte_mbuf *m_src = op->m_src; - uint32_t offset = op->cipher.data.offset; + uint8_t *iv_ptr; + struct rte_crypto_sym_op *sym_op = op->sym; + struct rte_mbuf *m_src = sym_op->m_src; + uint32_t offset, data_offset, data_length; uint32_t part_len, total_len, data_len; + if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION || + session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { + offset = sym_op->aead.data.offset; + data_offset = offset; + data_length = sym_op->aead.data.length; + } else { + offset = sym_op->auth.data.offset; + data_offset = offset; + data_length = sym_op->auth.data.length; + } + RTE_ASSERT(m_src != NULL); while (offset >= m_src->data_len) { @@ -196,60 +232,50 @@ process_gcm_crypto_op(struct rte_crypto_sym_op *op, } data_len = m_src->data_len - offset; - part_len = (data_len < op->cipher.data.length) ? data_len : - op->cipher.data.length; + part_len = (data_len < data_length) ? data_len : + data_length; /* Destination buffer is required when segmented source buffer */ - RTE_ASSERT((part_len == op->cipher.data.length) || - ((part_len != op->cipher.data.length) && - (op->m_dst != NULL))); + RTE_ASSERT((part_len == data_length) || + ((part_len != data_length) && + (sym_op->m_dst != NULL))); /* Segmented destination buffer is not supported */ - RTE_ASSERT((op->m_dst == NULL) || - ((op->m_dst != NULL) && - rte_pktmbuf_is_contiguous(op->m_dst))); + RTE_ASSERT((sym_op->m_dst == NULL) || + ((sym_op->m_dst != NULL) && + rte_pktmbuf_is_contiguous(sym_op->m_dst))); - dst = op->m_dst ? - rte_pktmbuf_mtod_offset(op->m_dst, uint8_t *, - op->cipher.data.offset) : - rte_pktmbuf_mtod_offset(op->m_src, uint8_t *, - op->cipher.data.offset); + dst = sym_op->m_dst ? + rte_pktmbuf_mtod_offset(sym_op->m_dst, uint8_t *, + data_offset) : + rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *, + data_offset); src = rte_pktmbuf_mtod_offset(m_src, uint8_t *, offset); - /* sanity checks */ - if (op->cipher.iv.length != 16 && op->cipher.iv.length != 12 && - op->cipher.iv.length != 0) { - GCM_LOG_ERR("iv"); - return -1; - } - + iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + session->iv.offset); /* * GCM working in 12B IV mode => 16B pre-counter block we need * to set BE LSB to 1, driver expects that 16B is allocated */ - if (op->cipher.iv.length == 12) { - uint32_t *iv_padd = (uint32_t *)&op->cipher.iv.data[12]; + if (session->iv.length == 12) { + uint32_t *iv_padd = (uint32_t *)&(iv_ptr[12]); *iv_padd = rte_bswap32(1); } - if (op->auth.digest.length != 16 && - op->auth.digest.length != 12 && - op->auth.digest.length != 8) { - GCM_LOG_ERR("digest"); - return -1; - } - if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION) { - aesni_gcm_enc[session->key].init(&session->gdata, - op->cipher.iv.data, - op->auth.aad.data, - (uint64_t)op->auth.aad.length); + qp->ops[session->key].init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + sym_op->aead.aad.data, + (uint64_t)session->aad_length); - aesni_gcm_enc[session->key].update(&session->gdata, dst, src, + qp->ops[session->key].update_enc(&session->gdata_key, + &qp->gdata_ctx, dst, src, (uint64_t)part_len); - total_len = op->cipher.data.length - part_len; + total_len = data_length - part_len; while (total_len) { dst += part_len; @@ -261,33 +287,36 @@ process_gcm_crypto_op(struct rte_crypto_sym_op *op, part_len = (m_src->data_len < total_len) ? m_src->data_len : total_len; - aesni_gcm_enc[session->key].update(&session->gdata, - dst, src, + qp->ops[session->key].update_enc(&session->gdata_key, + &qp->gdata_ctx, dst, src, (uint64_t)part_len); total_len -= part_len; } - aesni_gcm_enc[session->key].finalize(&session->gdata, - op->auth.digest.data, - (uint64_t)op->auth.digest.length); - } else { /* session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION */ - uint8_t *auth_tag = (uint8_t *)rte_pktmbuf_append(op->m_dst ? - op->m_dst : op->m_src, - op->auth.digest.length); + qp->ops[session->key].finalize(&session->gdata_key, + &qp->gdata_ctx, + sym_op->aead.digest.data, + (uint64_t)session->digest_length); + } else if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { + uint8_t *auth_tag = (uint8_t *)rte_pktmbuf_append(sym_op->m_dst ? + sym_op->m_dst : sym_op->m_src, + session->digest_length); if (!auth_tag) { GCM_LOG_ERR("auth_tag"); return -1; } - aesni_gcm_dec[session->key].init(&session->gdata, - op->cipher.iv.data, - op->auth.aad.data, - (uint64_t)op->auth.aad.length); + qp->ops[session->key].init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + sym_op->aead.aad.data, + (uint64_t)session->aad_length); - aesni_gcm_dec[session->key].update(&session->gdata, dst, src, + qp->ops[session->key].update_dec(&session->gdata_key, + &qp->gdata_ctx, dst, src, (uint64_t)part_len); - total_len = op->cipher.data.length - part_len; + total_len = data_length - part_len; while (total_len) { dst += part_len; @@ -299,15 +328,47 @@ process_gcm_crypto_op(struct rte_crypto_sym_op *op, part_len = (m_src->data_len < total_len) ? m_src->data_len : total_len; - aesni_gcm_dec[session->key].update(&session->gdata, + qp->ops[session->key].update_dec(&session->gdata_key, + &qp->gdata_ctx, dst, src, (uint64_t)part_len); total_len -= part_len; } - aesni_gcm_dec[session->key].finalize(&session->gdata, + qp->ops[session->key].finalize(&session->gdata_key, + &qp->gdata_ctx, auth_tag, - (uint64_t)op->auth.digest.length); + (uint64_t)session->digest_length); + } else if (session->op == AESNI_GMAC_OP_GENERATE) { + qp->ops[session->key].init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + src, + (uint64_t)data_length); + qp->ops[session->key].finalize(&session->gdata_key, + &qp->gdata_ctx, + sym_op->auth.digest.data, + (uint64_t)session->digest_length); + } else { /* AESNI_GMAC_OP_VERIFY */ + uint8_t *auth_tag = (uint8_t *)rte_pktmbuf_append(sym_op->m_dst ? + sym_op->m_dst : sym_op->m_src, + session->digest_length); + + if (!auth_tag) { + GCM_LOG_ERR("auth_tag"); + return -1; + } + + qp->ops[session->key].init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + src, + (uint64_t)data_length); + + qp->ops[session->key].finalize(&session->gdata_key, + &qp->gdata_ctx, + auth_tag, + (uint64_t)session->digest_length); } return 0; @@ -324,34 +385,38 @@ process_gcm_crypto_op(struct rte_crypto_sym_op *op, * - Returns NULL on invalid job */ static void -post_process_gcm_crypto_op(struct rte_crypto_op *op) +post_process_gcm_crypto_op(struct rte_crypto_op *op, + struct aesni_gcm_session *session) { struct rte_mbuf *m = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; - struct aesni_gcm_session *session = - (struct aesni_gcm_session *)op->sym->session->_private; - op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Verify digest if required */ - if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { + if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION || + session->op == AESNI_GMAC_OP_VERIFY) { + uint8_t *digest; uint8_t *tag = rte_pktmbuf_mtod_offset(m, uint8_t *, - m->data_len - op->sym->auth.digest.length); + m->data_len - session->digest_length); + + if (session->op == AESNI_GMAC_OP_VERIFY) + digest = op->sym->auth.digest.data; + else + digest = op->sym->aead.digest.data; #ifdef RTE_LIBRTE_PMD_AESNI_GCM_DEBUG rte_hexdump(stdout, "auth tag (orig):", - op->sym->auth.digest.data, op->sym->auth.digest.length); + digest, session->digest_length); rte_hexdump(stdout, "auth tag (calc):", - tag, op->sym->auth.digest.length); + tag, session->digest_length); #endif - if (memcmp(tag, op->sym->auth.digest.data, - op->sym->auth.digest.length) != 0) + if (memcmp(tag, digest, session->digest_length) != 0) op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; /* trim area used for digest from mbuf */ - rte_pktmbuf_trim(m, op->sym->auth.digest.length); + rte_pktmbuf_trim(m, session->digest_length); } } @@ -359,6 +424,7 @@ post_process_gcm_crypto_op(struct rte_crypto_op *op) * Process a completed GCM request * * @param qp Queue Pair to process + * @param op Crypto operation * @param job JOB_AES_HMAC job * * @return @@ -366,12 +432,17 @@ post_process_gcm_crypto_op(struct rte_crypto_op *op) */ static void handle_completed_gcm_crypto_op(struct aesni_gcm_qp *qp, - struct rte_crypto_op *op) + struct rte_crypto_op *op, + struct aesni_gcm_session *sess) { - post_process_gcm_crypto_op(op); + post_process_gcm_crypto_op(op, sess); /* Free session if a session-less crypto op */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(sess, 0, sizeof(struct aesni_gcm_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } @@ -392,21 +463,21 @@ aesni_gcm_pmd_dequeue_burst(void *queue_pair, for (i = 0; i < nb_dequeued; i++) { - sess = aesni_gcm_get_session(qp, ops[i]->sym); + sess = aesni_gcm_get_session(qp, ops[i]); if (unlikely(sess == NULL)) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; qp->qp_stats.dequeue_err_count++; break; } - retval = process_gcm_crypto_op(ops[i]->sym, sess); + retval = process_gcm_crypto_op(qp, ops[i], sess); if (retval < 0) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; qp->qp_stats.dequeue_err_count++; break; } - handle_completed_gcm_crypto_op(qp, ops[i]); + handle_completed_gcm_crypto_op(qp, ops[i], sess); } qp->qp_stats.dequeued_count += i; @@ -438,6 +509,7 @@ aesni_gcm_create(const char *name, { struct rte_cryptodev *dev; struct aesni_gcm_private *internals; + enum aesni_gcm_vector_mode vector_mode; if (init_params->name[0] == '\0') snprintf(init_params->name, sizeof(init_params->name), @@ -449,14 +521,23 @@ aesni_gcm_create(const char *name, return -EFAULT; } - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, - sizeof(struct aesni_gcm_private), init_params->socket_id); + /* Check CPU for supported vector instruction set */ + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) + vector_mode = RTE_AESNI_GCM_AVX2; + else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) + vector_mode = RTE_AESNI_GCM_AVX; + else + vector_mode = RTE_AESNI_GCM_SSE; + + dev = rte_cryptodev_vdev_pmd_init(init_params->name, + sizeof(struct aesni_gcm_private), init_params->socket_id, + vdev); if (dev == NULL) { GCM_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_AESNI_GCM_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_aesni_gcm_pmd_ops; /* register rx/tx burst functions for data path */ @@ -468,8 +549,24 @@ aesni_gcm_create(const char *name, RTE_CRYPTODEV_FF_CPU_AESNI | RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; + switch (vector_mode) { + case RTE_AESNI_GCM_SSE: + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE; + break; + case RTE_AESNI_GCM_AVX: + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX; + break; + case RTE_AESNI_GCM_AVX2: + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; + break; + default: + break; + } + internals = dev->data->dev_private; + internals->vector_mode = vector_mode; + internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs; internals->max_nb_sessions = init_params->max_nb_sessions; @@ -498,7 +595,7 @@ aesni_gcm_probe(struct rte_vdev_device *vdev) if (name == NULL) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -539,3 +636,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_GCM_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_gcm_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c index 1fc047be..48400ac2 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c @@ -49,32 +49,32 @@ static const struct rte_cryptodev_capabilities aesni_gcm_pmd_capabilities[] = { .key_size = { .min = 16, .max = 32, - .increment = 16 + .increment = 8 }, .digest_size = { .min = 8, .max = 16, .increment = 4 }, - .aad_size = { - .min = 0, - .max = 65535, - .increment = 1 + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 } }, } }, } }, - { /* AES GCM (AUTH) */ + { /* AES GCM */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { - .algo = RTE_CRYPTO_AUTH_AES_GCM, + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, .block_size = 16, .key_size = { .min = 16, .max = 32, - .increment = 16 + .increment = 8 }, .digest_size = { .min = 8, @@ -85,21 +85,6 @@ static const struct rte_cryptodev_capabilities aesni_gcm_pmd_capabilities[] = { .min = 0, .max = 65535, .increment = 1 - } - }, } - }, } - }, - { /* AES GCM (CIPHER) */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, - {.cipher = { - .algo = RTE_CRYPTO_CIPHER_AES_GCM, - .block_size = 16, - .key_size = { - .min = 16, - .max = 32, - .increment = 16 }, .iv_size = { .min = 12, @@ -181,9 +166,9 @@ aesni_gcm_pmd_info_get(struct rte_cryptodev *dev, struct aesni_gcm_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; - dev_info->feature_flags = dev->feature_flags; - dev_info->capabilities = aesni_gcm_pmd_capabilities; + dev_info->driver_id = dev->driver_id; + dev_info->feature_flags = dev->feature_flags; + dev_info->capabilities = aesni_gcm_pmd_capabilities; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; @@ -244,9 +229,10 @@ aesni_gcm_pmd_qp_create_processed_pkts_ring(struct aesni_gcm_qp *qp, static int aesni_gcm_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct aesni_gcm_qp *qp = NULL; + struct aesni_gcm_private *internals = dev->data->dev_private; /* Free memory prior to re-allocation if needed. */ if (dev->data->queue_pairs[qp_id] != NULL) @@ -264,12 +250,14 @@ aesni_gcm_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (aesni_gcm_pmd_qp_set_unique_name(dev, qp)) goto qp_setup_cleanup; + qp->ops = (const struct aesni_gcm_ops *)gcm_ops[internals->vector_mode]; + qp->processed_pkts = aesni_gcm_pmd_qp_create_processed_pkts_ring(qp, qp_conf->nb_descriptors, socket_id); if (qp->processed_pkts == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -313,29 +301,57 @@ aesni_gcm_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a aesni gcm session from a crypto xform chain */ -static void * +static int aesni_gcm_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + struct aesni_gcm_private *internals = dev->data->dev_private; + if (unlikely(sess == NULL)) { GCM_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (aesni_gcm_set_session_parameters(sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + ret = aesni_gcm_set_session_parameters(gcm_ops[internals->vector_mode], + sess_private_data, xform); + if (ret != 0) { GCM_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -aesni_gcm_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +aesni_gcm_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - if (sess) - memset(sess, 0, sizeof(struct aesni_gcm_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct aesni_gcm_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops aesni_gcm_pmd_ops = { diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h index 0496b447..7e155729 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,9 @@ #include "aesni_gcm_ops.h" +#define CRYPTODEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm +/**< AES-NI GCM PMD device name */ + #define GCM_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_AESNI_GCM_PMD), \ @@ -58,6 +61,8 @@ /** private data structure for each virtual AESNI GCM device */ struct aesni_gcm_private { + enum aesni_gcm_vector_mode vector_mode; + /**< Vector mode */ unsigned max_nb_queue_pairs; /**< Max number of queue pairs supported by device */ unsigned max_nb_sessions; @@ -65,36 +70,46 @@ struct aesni_gcm_private { }; struct aesni_gcm_qp { - uint16_t id; - /**< Queue Pair Identifier */ - char name[RTE_CRYPTODEV_NAME_LEN]; - /**< Unique Queue Pair Name */ + const struct aesni_gcm_ops *ops; + /**< Architecture dependent function pointer table of the gcm APIs */ struct rte_ring *processed_pkts; /**< Ring for placing process packets */ + struct gcm_context_data gdata_ctx; /* (16 * 5) + 8 = 88 B */ + /**< GCM parameters */ + struct rte_cryptodev_stats qp_stats; /* 8 * 4 = 32 B */ + /**< Queue pair statistics */ struct rte_mempool *sess_mp; /**< Session Mempool */ - struct rte_cryptodev_stats qp_stats; - /**< Queue pair statistics */ + uint16_t id; + /**< Queue Pair Identifier */ + char name[RTE_CRYPTODEV_NAME_LEN]; + /**< Unique Queue Pair Name */ } __rte_cache_aligned; enum aesni_gcm_operation { AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION, - AESNI_GCM_OP_AUTHENTICATED_DECRYPTION -}; - -enum aesni_gcm_key { - AESNI_GCM_KEY_128, - AESNI_GCM_KEY_256 + AESNI_GCM_OP_AUTHENTICATED_DECRYPTION, + AESNI_GMAC_OP_GENERATE, + AESNI_GMAC_OP_VERIFY }; /** AESNI GCM private session structure */ struct aesni_gcm_session { + struct { + uint16_t length; + uint16_t offset; + } iv; + /**< IV parameters */ + uint16_t aad_length; + /**< AAD length */ + uint16_t digest_length; + /**< Digest length */ enum aesni_gcm_operation op; /**< GCM operation type */ enum aesni_gcm_key key; /**< GCM key type */ - struct gcm_data gdata __rte_cache_aligned; + struct gcm_key_data gdata_key; /**< GCM parameters */ }; @@ -109,7 +124,8 @@ struct aesni_gcm_session { * - On failure returns error code < 0 */ extern int -aesni_gcm_set_session_parameters(struct aesni_gcm_session *sess, +aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *ops, + struct aesni_gcm_session *sess, const struct rte_crypto_sym_xform *xform); diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index 45b25c9d..16e14512 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -34,12 +34,15 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> #include "rte_aesni_mb_pmd_private.h" +static uint8_t cryptodev_driver_id; + typedef void (*hash_one_block_t)(const void *data, void *digest); typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys); @@ -166,7 +169,7 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, break; default: MB_LOG_ERR("Unsupported authentication algorithm selection"); - return -1; + return -ENOTSUP; } /* Calculate Authentication precomputes */ @@ -194,7 +197,7 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) { MB_LOG_ERR("Crypto xform struct not of type cipher"); - return -1; + return -EINVAL; } /* Select cipher direction */ @@ -206,8 +209,8 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, sess->cipher.direction = DECRYPT; break; default: - MB_LOG_ERR("Unsupported cipher operation parameter"); - return -1; + MB_LOG_ERR("Invalid cipher operation parameter"); + return -EINVAL; } /* Select cipher mode */ @@ -223,7 +226,7 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, break; default: MB_LOG_ERR("Unsupported cipher mode parameter"); - return -1; + return -ENOTSUP; } /* Check key length and choose key expansion function */ @@ -241,10 +244,14 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, aes_keyexp_fn = mb_ops->aux.keyexp.aes256; break; default: - MB_LOG_ERR("Unsupported cipher key length"); - return -1; + MB_LOG_ERR("Invalid cipher key length"); + return -EINVAL; } + /* Set IV parameters */ + sess->iv.offset = xform->cipher.iv.offset; + sess->iv.length = xform->cipher.iv.length; + /* Expanded cipher keys */ (*aes_keyexp_fn)(xform->cipher.key.data, sess->cipher.expanded_aes_keys.encode, @@ -261,6 +268,7 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops, { const struct rte_crypto_sym_xform *auth_xform = NULL; const struct rte_crypto_sym_xform *cipher_xform = NULL; + int ret; /* Select Crypto operation - hash then cipher / cipher then hash */ switch (aesni_mb_get_chain_order(xform)) { @@ -296,19 +304,25 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops, case AESNI_MB_OP_NOT_SUPPORTED: default: MB_LOG_ERR("Unsupported operation chain order parameter"); - return -1; + return -ENOTSUP; } - if (aesni_mb_set_session_auth_parameters(mb_ops, sess, auth_xform)) { + /* Default IV length = 0 */ + sess->iv.length = 0; + + ret = aesni_mb_set_session_auth_parameters(mb_ops, sess, auth_xform); + if (ret != 0) { MB_LOG_ERR("Invalid/unsupported authentication parameters"); - return -1; + return ret; } - if (aesni_mb_set_session_cipher_parameters(mb_ops, sess, - cipher_xform)) { + ret = aesni_mb_set_session_cipher_parameters(mb_ops, sess, + cipher_xform); + if (ret != 0) { MB_LOG_ERR("Invalid/unsupported cipher parameters"); - return -1; + return ret; } + return 0; } @@ -344,30 +358,38 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op) { struct aesni_mb_session *sess = NULL; - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->sym->session->dev_type != - RTE_CRYPTODEV_AESNI_MB_PMD)) { - return NULL; - } - - sess = (struct aesni_mb_session *)op->sym->session->_private; - } else { + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct aesni_mb_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { void *_sess = NULL; + void *_sess_private_data = NULL; if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct aesni_mb_session *) - ((struct rte_cryptodev_sym_session *)_sess)->_private; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct aesni_mb_session *)_sess_private_data; if (unlikely(aesni_mb_set_session_parameters(qp->op_fns, sess, op->sym->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); sess = NULL; } op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } @@ -396,7 +418,6 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; return -1; } - op->status = RTE_CRYPTO_OP_STATUS_ENQUEUED; /* Set crypto operation */ job->chain_order = session->chain_order; @@ -470,8 +491,9 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, get_truncated_digest_byte_length(job->hash_alg); /* Set IV parameters */ - job->iv = op->sym->cipher.iv.data; - job->iv_len_in_bytes = op->sym->cipher.iv.length; + job->iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->iv.offset); + job->iv_len_in_bytes = session->iv.length; /* Data Parameter */ job->src = rte_pktmbuf_mtod(m_src, uint8_t *); @@ -494,8 +516,6 @@ static inline void verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op) { struct rte_mbuf *m_dst = (struct rte_mbuf *)job->user_data2; - RTE_ASSERT(m_dst == NULL); - /* Verify digest if required */ if (memcmp(job->auth_tag_output, op->sym->auth.digest.data, job->auth_tag_output_len_in_bytes) != 0) @@ -508,31 +528,28 @@ verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op) { /** * Process a completed job and return rte_mbuf which job processed * + * @param qp Queue Pair to process * @param job JOB_AES_HMAC job to process * * @return - * - Returns processed mbuf which is trimmed of output digest used in - * verification of supplied digest in the case of a HASH_CIPHER operation + * - Returns processed crypto operation which mbuf is trimmed of output digest + * used in verification of supplied digest. * - Returns NULL on invalid job */ static inline struct rte_crypto_op * post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) { struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data; + struct aesni_mb_session *sess = get_session_private_data( + op->sym->session, + cryptodev_driver_id); - struct aesni_mb_session *sess; - - RTE_ASSERT(op == NULL); - - if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ENQUEUED)) { + if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) { switch (job->status) { case STS_COMPLETED: op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; if (job->hash_alg != NULL_HASH) { - sess = (struct aesni_mb_session *) - op->sym->session->_private; - if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) verify_digest(job, op); @@ -544,7 +561,11 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) } /* Free session if a session-less crypto op */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(sess, 0, sizeof(struct aesni_mb_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } @@ -569,7 +590,7 @@ handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job, struct rte_crypto_op *op = NULL; unsigned processed_jobs = 0; - while (job != NULL && processed_jobs < nb_ops) { + while (job != NULL) { op = post_process_mb_job(qp, job); if (op) { @@ -579,6 +600,8 @@ handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job, qp->stats.dequeue_err_count++; break; } + if (processed_jobs == nb_ops) + break; job = (*qp->op_fns->job.get_completed_job)(&qp->mb_mgr); } @@ -624,6 +647,9 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, int retval, processed_jobs = 0; + if (unlikely(nb_ops == 0)) + return 0; + do { /* Get next operation to process from ingress queue */ retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op); @@ -691,21 +717,18 @@ cryptodev_aesni_mb_create(const char *name, vector_mode = RTE_AESNI_MB_AVX2; else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) vector_mode = RTE_AESNI_MB_AVX; - else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) + else vector_mode = RTE_AESNI_MB_SSE; - else { - MB_LOG_ERR("Vector instructions are not supported by CPU"); - return -EFAULT; - } - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, - sizeof(struct aesni_mb_private), init_params->socket_id); + dev = rte_cryptodev_vdev_pmd_init(init_params->name, + sizeof(struct aesni_mb_private), init_params->socket_id, + vdev); if (dev == NULL) { MB_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_AESNI_MB_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_aesni_mb_pmd_ops; /* register rx/tx burst functions for data path */ @@ -765,7 +788,7 @@ cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev) if (name == NULL) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -806,3 +829,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_aesni_mb_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c index d1bc28e0..692b354f 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -48,16 +48,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_MD5_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 12, .max = 12, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -69,16 +69,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 12, .max = 12, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -90,16 +90,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 14, .max = 14, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -111,16 +111,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 16, .max = 16, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -132,16 +132,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 24, .max = 24, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -153,16 +153,16 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 32, .max = 32, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -183,7 +183,7 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .max = 12, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -220,9 +220,9 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .increment = 8 }, .iv_size = { - .min = 16, + .min = 12, .max = 16, - .increment = 0 + .increment = 4 } }, } }, } @@ -321,7 +321,7 @@ aesni_mb_pmd_info_get(struct rte_cryptodev *dev, struct aesni_mb_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->feature_flags = dev->feature_flags; dev_info->capabilities = aesni_mb_pmd_capabilities; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; @@ -397,7 +397,7 @@ aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp, static int aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct aesni_mb_qp *qp = NULL; struct aesni_mb_private *internals = dev->data->dev_private; @@ -426,7 +426,7 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->ingress_queue == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->stats, 0, sizeof(qp->stats)); @@ -472,36 +472,58 @@ aesni_mb_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a aesni multi-buffer session from a crypto xform chain */ -static void * +static int aesni_mb_pmd_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; struct aesni_mb_private *internals = dev->data->dev_private; + int ret; if (unlikely(sess == NULL)) { MB_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (aesni_mb_set_session_parameters(&job_ops[internals->vector_mode], - sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = aesni_mb_set_session_parameters(&job_ops[internals->vector_mode], + sess_private_data, xform); + if (ret != 0) { MB_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -aesni_mb_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +aesni_mb_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct aesni_mb_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct aesni_mb_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops aesni_mb_pmd_ops = { diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h index 0d82699c..6676948e 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h @@ -35,6 +35,9 @@ #include "aesni_mb_ops.h" +#define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb +/**< AES-NI Multi buffer PMD device name */ + #define MB_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD), \ @@ -167,6 +170,11 @@ struct aesni_mb_qp { /** AES-NI multi-buffer private session structure */ struct aesni_mb_session { JOB_CHAIN_ORDER chain_order; + struct { + uint16_t length; + uint16_t offset; + } iv; + /**< IV parameters */ /** Cipher Parameters */ struct { diff --git a/drivers/crypto/armv8/Makefile b/drivers/crypto/armv8/Makefile index 1474951c..86611fa2 100644 --- a/drivers/crypto/armv8/Makefile +++ b/drivers/crypto/armv8/Makefile @@ -1,7 +1,7 @@ # # BSD LICENSE # -# Copyright (C) Cavium networks Ltd. 2017. +# Copyright (C) Cavium, Inc. 2017. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. -# * Neither the name of Cavium networks nor the names of its +# * Neither the name of Cavium, Inc nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # diff --git a/drivers/crypto/armv8/rte_armv8_pmd.c b/drivers/crypto/armv8/rte_armv8_pmd.c index 3d603a5a..a5c39c9b 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd.c +++ b/drivers/crypto/armv8/rte_armv8_pmd.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -36,6 +36,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -44,6 +45,8 @@ #include "rte_armv8_pmd_private.h" +static uint8_t cryptodev_driver_id; + static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev); /** @@ -288,27 +291,14 @@ auth_set_prerequisites(struct armv8_crypto_session *sess, * Generate authentication key, i_key_pad and o_key_pad. */ /* Zero memory under key */ - memset(sess->auth.hmac.key, 0, SHA1_AUTH_KEY_LENGTH); - - if (xform->auth.key.length > SHA1_AUTH_KEY_LENGTH) { - /* - * In case the key is longer than 160 bits - * the algorithm will use SHA1(key) instead. - */ - error = sha1_block(NULL, xform->auth.key.data, - sess->auth.hmac.key, xform->auth.key.length); - if (error != 0) - return -1; - } else { - /* - * Now copy the given authentication key to the session - * key assuming that the session key is zeroed there is - * no need for additional zero padding if the key is - * shorter than SHA1_AUTH_KEY_LENGTH. - */ - rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, - xform->auth.key.length); - } + memset(sess->auth.hmac.key, 0, SHA1_BLOCK_SIZE); + + /* + * Now copy the given authentication key to the session + * key. + */ + rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, + xform->auth.key.length); /* Prepare HMAC padding: key|pattern */ auth_hmac_pad_prepare(sess, xform); @@ -334,27 +324,14 @@ auth_set_prerequisites(struct armv8_crypto_session *sess, * Generate authentication key, i_key_pad and o_key_pad. */ /* Zero memory under key */ - memset(sess->auth.hmac.key, 0, SHA256_AUTH_KEY_LENGTH); - - if (xform->auth.key.length > SHA256_AUTH_KEY_LENGTH) { - /* - * In case the key is longer than 256 bits - * the algorithm will use SHA256(key) instead. - */ - error = sha256_block(NULL, xform->auth.key.data, - sess->auth.hmac.key, xform->auth.key.length); - if (error != 0) - return -1; - } else { - /* - * Now copy the given authentication key to the session - * key assuming that the session key is zeroed there is - * no need for additional zero padding if the key is - * shorter than SHA256_AUTH_KEY_LENGTH. - */ - rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, - xform->auth.key.length); - } + memset(sess->auth.hmac.key, 0, SHA256_BLOCK_SIZE); + + /* + * Now copy the given authentication key to the session + * key. + */ + rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, + xform->auth.key.length); /* Prepare HMAC padding: key|pattern */ auth_hmac_pad_prepare(sess, xform); @@ -414,7 +391,7 @@ armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, order = sess->chain_order; break; default: - return -EINVAL; + return -ENOTSUP; } /* Select cipher direction */ sess->cipher.direction = cipher_xform->cipher.op; @@ -431,10 +408,10 @@ armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, case RTE_CRYPTO_CIPHER_AES_CBC: sess->cipher.algo = calg; /* IV len is always 16 bytes (block size) for AES CBC */ - sess->cipher.iv_len = 16; + sess->cipher.iv.length = 16; break; default: - return -EINVAL; + return -ENOTSUP; } /* Select auth generate/verify */ sess->auth.operation = auth_xform->auth.op; @@ -448,9 +425,12 @@ armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, sess->auth.mode = ARMV8_CRYPTO_AUTH_AS_HMAC; break; default: - return -EINVAL; + return -ENOTSUP; } + /* Set the digest length */ + sess->auth.digest_length = auth_xform->auth.digest_length; + /* Verify supported key lengths and extract proper algorithm */ switch (cipher_xform->cipher.key.length << 3) { case 128: @@ -465,7 +445,7 @@ armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, default: /* Fall through */ sess->crypto_func = NULL; sess->cipher.key_sched = NULL; - return -EINVAL; + return -ENOTSUP; } if (unlikely(sess->crypto_func == NULL)) { @@ -519,20 +499,23 @@ armv8_crypto_set_session_parameters(struct armv8_crypto_session *sess, break; default: is_chained_op = false; - return -EINVAL; + return -ENOTSUP; } + /* Set IV offset */ + sess->cipher.iv.offset = cipher_xform->cipher.iv.offset; + if (is_chained_op) { ret = armv8_crypto_set_session_chained_parameters(sess, cipher_xform, auth_xform); if (unlikely(ret != 0)) { ARMV8_CRYPTO_LOG_ERR( "Invalid/unsupported chained (cipher/auth) parameters"); - return -EINVAL; + return ret; } } else { ARMV8_CRYPTO_LOG_ERR("Invalid/unsupported operation"); - return -EINVAL; + return -ENOTSUP; } return 0; @@ -544,30 +527,36 @@ get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op) { struct armv8_crypto_session *sess = NULL; - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { /* get existing session */ - if (likely(op->sym->session != NULL && - op->sym->session->dev_type == - RTE_CRYPTODEV_ARMV8_PMD)) { + if (likely(op->sym->session != NULL)) { sess = (struct armv8_crypto_session *) - op->sym->session->_private; + get_session_private_data( + op->sym->session, + cryptodev_driver_id); } } else { /* provide internal session */ void *_sess = NULL; + void *_sess_private_data = NULL; - if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { - sess = (struct armv8_crypto_session *) - ((struct rte_cryptodev_sym_session *)_sess) - ->_private; - - if (unlikely(armv8_crypto_set_session_parameters( - sess, op->sym->xform) != 0)) { - rte_mempool_put(qp->sess_mp, _sess); - sess = NULL; - } else - op->sym->session = _sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct armv8_crypto_session *)_sess_private_data; + + if (unlikely(armv8_crypto_set_session_parameters(sess, + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } if (unlikely(sess == NULL)) @@ -645,15 +634,11 @@ process_armv8_chained_op } } else { adst = (uint8_t *)rte_pktmbuf_append(m_asrc, - op->sym->auth.digest.length); - } - - if (unlikely(op->sym->cipher.iv.length != sess->cipher.iv_len)) { - op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - return; + sess->auth.digest_length); } - arg.cipher.iv = op->sym->cipher.iv.data; + arg.cipher.iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->cipher.iv.offset); arg.cipher.key = sess->cipher.key.data; /* Acquire combined mode function */ crypto_func = sess->crypto_func; @@ -667,12 +652,12 @@ process_armv8_chained_op op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { if (memcmp(adst, op->sym->auth.digest.data, - op->sym->auth.digest.length) != 0) { + sess->auth.digest_length) != 0) { op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } /* Trim area used for digest from mbuf. */ rte_pktmbuf_trim(m_asrc, - op->sym->auth.digest.length); + sess->auth.digest_length); } } @@ -699,8 +684,11 @@ process_op(const struct armv8_crypto_qp *qp, struct rte_crypto_op *op, } /* Free session if a session-less crypto op */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { memset(sess, 0, sizeof(struct armv8_crypto_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } @@ -806,15 +794,16 @@ cryptodev_armv8_crypto_create(const char *name, snprintf(init_params->name, sizeof(init_params->name), "%s", name); - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, + dev = rte_cryptodev_vdev_pmd_init(init_params->name, sizeof(struct armv8_crypto_private), - init_params->socket_id); + init_params->socket_id, + vdev); if (dev == NULL) { ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_ARMV8_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_armv8_crypto_pmd_ops; /* register rx/tx burst functions for data path */ @@ -860,7 +849,7 @@ cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev) if (name == NULL) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -904,3 +893,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(armv8_crypto_drv, cryptodev_driver_id); diff --git a/drivers/crypto/armv8/rte_armv8_pmd_ops.c b/drivers/crypto/armv8/rte_armv8_pmd_ops.c index 4d9ccbfb..00297beb 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd_ops.c +++ b/drivers/crypto/armv8/rte_armv8_pmd_ops.c @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -50,16 +50,16 @@ static const struct rte_cryptodev_capabilities .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, .block_size = 64, .key_size = { - .min = 16, - .max = 128, - .increment = 0 + .min = 1, + .max = 64, + .increment = 1 }, .digest_size = { .min = 20, .max = 20, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -71,16 +71,16 @@ static const struct rte_cryptodev_capabilities .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, .block_size = 64, .key_size = { - .min = 16, - .max = 128, - .increment = 0 + .min = 1, + .max = 64, + .increment = 1 }, .digest_size = { .min = 32, .max = 32, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -178,7 +178,7 @@ armv8_crypto_pmd_info_get(struct rte_cryptodev *dev, struct armv8_crypto_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->feature_flags = dev->feature_flags; dev_info->capabilities = armv8_crypto_pmd_capabilities; dev_info->max_nb_queue_pairs = internals->max_nb_qpairs; @@ -247,7 +247,7 @@ armv8_crypto_pmd_qp_create_processed_ops_ring(struct armv8_crypto_qp *qp, static int armv8_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct armv8_crypto_qp *qp = NULL; @@ -272,7 +272,7 @@ armv8_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->processed_ops == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->stats, 0, sizeof(qp->stats)); @@ -316,33 +316,56 @@ armv8_crypto_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure the session from a crypto xform chain */ -static void * -armv8_crypto_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) +static int +armv8_crypto_pmd_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + if (unlikely(sess == NULL)) { ARMV8_CRYPTO_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (armv8_crypto_set_session_parameters( - sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = armv8_crypto_set_session_parameters(sess_private_data, xform); + if (ret != 0) { ARMV8_CRYPTO_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -armv8_crypto_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, - void *sess) +armv8_crypto_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); /* Zero out the whole structure */ - if (sess) - memset(sess, 0, sizeof(struct armv8_crypto_session)); + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct armv8_crypto_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops armv8_crypto_pmd_ops = { diff --git a/drivers/crypto/armv8/rte_armv8_pmd_private.h b/drivers/crypto/armv8/rte_armv8_pmd_private.h index b75107f2..d02992a6 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd_private.h +++ b/drivers/crypto/armv8/rte_armv8_pmd_private.h @@ -1,7 +1,7 @@ /* * BSD LICENSE * - * Copyright (C) Cavium networks Ltd. 2017. + * Copyright (C) Cavium, Inc. 2017. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Cavium networks nor the names of its + * * Neither the name of Cavium, Inc nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * @@ -33,6 +33,9 @@ #ifndef _RTE_ARMV8_PMD_PRIVATE_H_ #define _RTE_ARMV8_PMD_PRIVATE_H_ +#define CRYPTODEV_NAME_ARMV8_PMD crypto_armv8 +/**< ARMv8 Crypto PMD device name */ + #define ARMV8_CRYPTO_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_ARMV8_CRYPTO_PMD), \ @@ -159,8 +162,11 @@ struct armv8_crypto_session { /**< cipher operation direction */ enum rte_crypto_cipher_algorithm algo; /**< cipher algorithm */ - int iv_len; - /**< IV length */ + struct { + uint16_t length; + uint16_t offset; + } iv; + /**< IV parameters */ struct { uint8_t data[256]; @@ -192,10 +198,12 @@ struct armv8_crypto_session { uint8_t o_key_pad[SHA_BLOCK_MAX] __rte_cache_aligned; /**< outer pad (max supported block length) */ - uint8_t key[SHA_AUTH_KEY_MAX]; - /**< HMAC key (max supported length)*/ + uint8_t key[SHA_BLOCK_MAX]; + /**< HMAC key (max supported block length)*/ } hmac; }; + uint16_t digest_length; + /* Digest length */ } auth; } __rte_cache_aligned; diff --git a/drivers/crypto/dpaa2_sec/Makefile b/drivers/crypto/dpaa2_sec/Makefile index 11c7c785..ae15c99f 100644 --- a/drivers/crypto/dpaa2_sec/Makefile +++ b/drivers/crypto/dpaa2_sec/Makefile @@ -1,7 +1,7 @@ # BSD LICENSE # # Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. -# Copyright (c) 2016 NXP. All rights reserved. +# Copyright 2016 NXP. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -46,6 +46,12 @@ CFLAGS += $(WERROR_FLAGS) endif CFLAGS += -D _GNU_SOURCE +ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) +ifeq ($(shell test $(GCC_VERSION) -gt 70 && echo 1), 1) +CFLAGS += -Wno-implicit-fallthrough +endif +endif + CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec/ CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec/mc CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/ diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c index 4e01fe81..e0f6cfca 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -69,10 +69,158 @@ #define FSL_MC_DPSECI_DEVID 3 #define NO_PREFETCH 0 -#define TDES_CBC_IV_LEN 8 -#define AES_CBC_IV_LEN 16 +/* FLE_POOL_NUM_BUFS is set as per the ipsec-secgw application */ +#define FLE_POOL_NUM_BUFS 32000 +#define FLE_POOL_BUF_SIZE 256 +#define FLE_POOL_CACHE_SIZE 512 + enum rta_sec_era rta_sec_era = RTA_SEC_ERA_8; +static uint8_t cryptodev_driver_id; + +static inline int +build_authenc_gcm_fd(dpaa2_sec_session *sess, + struct rte_crypto_op *op, + struct qbman_fd *fd, uint16_t bpid) +{ + struct rte_crypto_sym_op *sym_op = op->sym; + struct ctxt_priv *priv = sess->ctxt; + struct qbman_fle *fle, *sge; + struct sec_flow_context *flc; + uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len; + int icv_len = sess->digest_length, retval; + uint8_t *old_icv; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); + + PMD_INIT_FUNC_TRACE(); + + /* TODO we are using the first FLE entry to store Mbuf and session ctxt. + * Currently we donot know which FLE has the mbuf stored. + * So while retreiving we can go back 1 FLE from the FD -ADDR + * to get the MBUF Addr from the previous FLE. + * We can have a better approach to use the inline Mbuf + */ + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { + RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); + return -1; + } + memset(fle, 0, FLE_POOL_BUF_SIZE); + DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); + fle = fle + 1; + sge = fle + 2; + if (likely(bpid < MAX_BPID)) { + DPAA2_SET_FD_BPID(fd, bpid); + DPAA2_SET_FLE_BPID(fle, bpid); + DPAA2_SET_FLE_BPID(fle + 1, bpid); + DPAA2_SET_FLE_BPID(sge, bpid); + DPAA2_SET_FLE_BPID(sge + 1, bpid); + DPAA2_SET_FLE_BPID(sge + 2, bpid); + DPAA2_SET_FLE_BPID(sge + 3, bpid); + } else { + DPAA2_SET_FD_IVP(fd); + DPAA2_SET_FLE_IVP(fle); + DPAA2_SET_FLE_IVP((fle + 1)); + DPAA2_SET_FLE_IVP(sge); + DPAA2_SET_FLE_IVP((sge + 1)); + DPAA2_SET_FLE_IVP((sge + 2)); + DPAA2_SET_FLE_IVP((sge + 3)); + } + + /* Save the shared descriptor */ + flc = &priv->flc_desc[0].flc; + /* Configure FD as a FRAME LIST */ + DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); + DPAA2_SET_FD_COMPOUND_FMT(fd); + DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); + + PMD_TX_LOG(DEBUG, "auth_off: 0x%x/length %d, digest-len=%d\n" + "iv-len=%d data_off: 0x%x\n", + sym_op->aead.data.offset, + sym_op->aead.data.length, + sym_op->aead.digest.length, + sess->iv.length, + sym_op->m_src->data_off); + + /* Configure Output FLE with Scatter/Gather Entry */ + DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); + if (auth_only_len) + DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); + fle->length = (sess->dir == DIR_ENC) ? + (sym_op->aead.data.length + icv_len + auth_only_len) : + sym_op->aead.data.length + auth_only_len; + + DPAA2_SET_FLE_SG_EXT(fle); + + /* Configure Output SGE for Encap/Decap */ + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset + + sym_op->m_src->data_off - auth_only_len); + sge->length = sym_op->aead.data.length + auth_only_len; + + if (sess->dir == DIR_ENC) { + sge++; + DPAA2_SET_FLE_ADDR(sge, + DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data)); + sge->length = sess->digest_length; + DPAA2_SET_FD_LEN(fd, (sym_op->aead.data.length + + sess->iv.length + auth_only_len)); + } + DPAA2_SET_FLE_FIN(sge); + + sge++; + fle++; + + /* Configure Input FLE with Scatter/Gather Entry */ + DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); + DPAA2_SET_FLE_SG_EXT(fle); + DPAA2_SET_FLE_FIN(fle); + fle->length = (sess->dir == DIR_ENC) ? + (sym_op->aead.data.length + sess->iv.length + auth_only_len) : + (sym_op->aead.data.length + sess->iv.length + auth_only_len + + sess->digest_length); + + /* Configure Input SGE for Encap/Decap */ + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr)); + sge->length = sess->iv.length; + sge++; + if (auth_only_len) { + DPAA2_SET_FLE_ADDR(sge, + DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data)); + sge->length = auth_only_len; + DPAA2_SET_FLE_BPID(sge, bpid); + sge++; + } + + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset + + sym_op->m_src->data_off); + sge->length = sym_op->aead.data.length; + if (sess->dir == DIR_DEC) { + sge++; + old_icv = (uint8_t *)(sge + 1); + memcpy(old_icv, sym_op->aead.digest.data, + sess->digest_length); + memset(sym_op->aead.digest.data, 0, sess->digest_length); + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); + sge->length = sess->digest_length; + DPAA2_SET_FD_LEN(fd, (sym_op->aead.data.length + + sess->digest_length + + sess->iv.length + + auth_only_len)); + } + DPAA2_SET_FLE_FIN(sge); + + if (auth_only_len) { + DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); + DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len); + } + + return 0; +} + static inline int build_authenc_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, @@ -84,9 +232,10 @@ build_authenc_fd(dpaa2_sec_session *sess, struct sec_flow_context *flc; uint32_t auth_only_len = sym_op->auth.data.length - sym_op->cipher.data.length; - int icv_len = sym_op->auth.digest.length; + int icv_len = sess->digest_length, retval; uint8_t *old_icv; - uint32_t mem_len = (7 * sizeof(struct qbman_fle)) + icv_len; + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); PMD_INIT_FUNC_TRACE(); @@ -96,12 +245,14 @@ build_authenc_fd(dpaa2_sec_session *sess, * to get the MBUF Addr from the previous FLE. * We can have a better approach to use the inline Mbuf */ - fle = rte_zmalloc(NULL, mem_len, RTE_CACHE_LINE_SIZE); - if (!fle) { + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); return -1; } + memset(fle, 0, FLE_POOL_BUF_SIZE); DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); fle = fle + 1; sge = fle + 2; if (likely(bpid < MAX_BPID)) { @@ -133,10 +284,10 @@ build_authenc_fd(dpaa2_sec_session *sess, "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n", sym_op->auth.data.offset, sym_op->auth.data.length, - sym_op->auth.digest.length, + sess->digest_length, sym_op->cipher.data.offset, sym_op->cipher.data.length, - sym_op->cipher.iv.length, + sess->iv.length, sym_op->m_src->data_off); /* Configure Output FLE with Scatter/Gather Entry */ @@ -159,9 +310,9 @@ build_authenc_fd(dpaa2_sec_session *sess, sge++; DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); - sge->length = sym_op->auth.digest.length; + sge->length = sess->digest_length; DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length + - sym_op->cipher.iv.length)); + sess->iv.length)); } DPAA2_SET_FLE_FIN(sge); @@ -173,13 +324,13 @@ build_authenc_fd(dpaa2_sec_session *sess, DPAA2_SET_FLE_SG_EXT(fle); DPAA2_SET_FLE_FIN(fle); fle->length = (sess->dir == DIR_ENC) ? - (sym_op->auth.data.length + sym_op->cipher.iv.length) : - (sym_op->auth.data.length + sym_op->cipher.iv.length + - sym_op->auth.digest.length); + (sym_op->auth.data.length + sess->iv.length) : + (sym_op->auth.data.length + sess->iv.length + + sess->digest_length); /* Configure Input SGE for Encap/Decap */ - DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(sym_op->cipher.iv.data)); - sge->length = sym_op->cipher.iv.length; + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); + sge->length = sess->iv.length; sge++; DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); @@ -190,13 +341,13 @@ build_authenc_fd(dpaa2_sec_session *sess, sge++; old_icv = (uint8_t *)(sge + 1); memcpy(old_icv, sym_op->auth.digest.data, - sym_op->auth.digest.length); - memset(sym_op->auth.digest.data, 0, sym_op->auth.digest.length); + sess->digest_length); + memset(sym_op->auth.digest.data, 0, sess->digest_length); DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); - sge->length = sym_op->auth.digest.length; + sge->length = sess->digest_length; DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length + - sym_op->auth.digest.length + - sym_op->cipher.iv.length)); + sess->digest_length + + sess->iv.length)); } DPAA2_SET_FLE_FIN(sge); if (auth_only_len) { @@ -212,21 +363,19 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, { struct rte_crypto_sym_op *sym_op = op->sym; struct qbman_fle *fle, *sge; - uint32_t mem_len = (sess->dir == DIR_ENC) ? - (3 * sizeof(struct qbman_fle)) : - (5 * sizeof(struct qbman_fle) + - sym_op->auth.digest.length); struct sec_flow_context *flc; struct ctxt_priv *priv = sess->ctxt; uint8_t *old_digest; + int retval; PMD_INIT_FUNC_TRACE(); - fle = rte_zmalloc(NULL, mem_len, RTE_CACHE_LINE_SIZE); - if (!fle) { - RTE_LOG(ERR, PMD, "Memory alloc failed for FLE\n"); + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { + RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); return -1; } + memset(fle, 0, FLE_POOL_BUF_SIZE); /* TODO we are using the first FLE entry to store Mbuf. * Currently we donot know which FLE has the mbuf stored. * So while retreiving we can go back 1 FLE from the FD -ADDR @@ -234,6 +383,7 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, * We can have a better approach to use the inline Mbuf */ DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); fle = fle + 1; if (likely(bpid < MAX_BPID)) { @@ -249,7 +399,7 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); - fle->length = sym_op->auth.digest.length; + fle->length = sess->digest_length; DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); DPAA2_SET_FD_COMPOUND_FMT(fd); @@ -280,17 +430,17 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, sym_op->m_src->data_off); DPAA2_SET_FD_LEN(fd, sym_op->auth.data.length + - sym_op->auth.digest.length); + sess->digest_length); sge->length = sym_op->auth.data.length; sge++; old_digest = (uint8_t *)(sge + 1); rte_memcpy(old_digest, sym_op->auth.digest.data, - sym_op->auth.digest.length); - memset(sym_op->auth.digest.data, 0, sym_op->auth.digest.length); + sess->digest_length); + memset(sym_op->auth.digest.data, 0, sess->digest_length); DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest)); - sge->length = sym_op->auth.digest.length; + sge->length = sess->digest_length; fle->length = sym_op->auth.data.length + - sym_op->auth.digest.length; + sess->digest_length; DPAA2_SET_FLE_FIN(sge); } DPAA2_SET_FLE_FIN(fle); @@ -304,18 +454,20 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, { struct rte_crypto_sym_op *sym_op = op->sym; struct qbman_fle *fle, *sge; - uint32_t mem_len = (5 * sizeof(struct qbman_fle)); + int retval; struct sec_flow_context *flc; struct ctxt_priv *priv = sess->ctxt; + uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); PMD_INIT_FUNC_TRACE(); - /* todo - we can use some mempool to avoid malloc here */ - fle = rte_zmalloc(NULL, mem_len, RTE_CACHE_LINE_SIZE); - if (!fle) { + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); + if (retval) { RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); return -1; } + memset(fle, 0, FLE_POOL_BUF_SIZE); /* TODO we are using the first FLE entry to store Mbuf. * Currently we donot know which FLE has the mbuf stored. * So while retreiving we can go back 1 FLE from the FD -ADDR @@ -323,6 +475,7 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, * We can have a better approach to use the inline Mbuf */ DPAA2_SET_FLE_ADDR(fle, DPAA2_OP_VADDR_TO_IOVA(op)); + DPAA2_FLE_SAVE_CTXT(fle, priv); fle = fle + 1; sge = fle + 2; @@ -343,21 +496,21 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, flc = &priv->flc_desc[0].flc; DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); DPAA2_SET_FD_LEN(fd, sym_op->cipher.data.length + - sym_op->cipher.iv.length); + sess->iv.length); DPAA2_SET_FD_COMPOUND_FMT(fd); DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); PMD_TX_LOG(DEBUG, "cipher_off: 0x%x/length %d,ivlen=%d data_off: 0x%x", sym_op->cipher.data.offset, sym_op->cipher.data.length, - sym_op->cipher.iv.length, + sess->iv.length, sym_op->m_src->data_off); DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); DPAA2_SET_FLE_OFFSET(fle, sym_op->cipher.data.offset + sym_op->m_src->data_off); - fle->length = sym_op->cipher.data.length + sym_op->cipher.iv.length; + fle->length = sym_op->cipher.data.length + sess->iv.length; PMD_TX_LOG(DEBUG, "1 - flc = %p, fle = %p FLEaddr = %x-%x, length %d", flc, fle, fle->addr_hi, fle->addr_lo, fle->length); @@ -365,12 +518,12 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, fle++; DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); - fle->length = sym_op->cipher.data.length + sym_op->cipher.iv.length; + fle->length = sym_op->cipher.data.length + sess->iv.length; DPAA2_SET_FLE_SG_EXT(fle); - DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(sym_op->cipher.iv.data)); - sge->length = sym_op->cipher.iv.length; + DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); + sge->length = sess->iv.length; sge++; DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); @@ -406,6 +559,9 @@ build_sec_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, case DPAA2_SEC_AUTH: ret = build_auth_fd(sess, op, fd, bpid); break; + case DPAA2_SEC_AEAD: + ret = build_authenc_gcm_fd(sess, op, fd, bpid); + break; case DPAA2_SEC_CIPHER_HASH: ret = build_authenc_fd(sess, op, fd, bpid); break; @@ -437,7 +593,7 @@ dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, if (unlikely(nb_ops == 0)) return 0; - if (ops[0]->sym->sess_type != RTE_CRYPTO_SYM_OP_WITH_SESSION) { + if (ops[0]->sess_type != RTE_CRYPTO_OP_WITH_SESSION) { RTE_LOG(ERR, PMD, "sessionless crypto op not supported\n"); return 0; } @@ -463,7 +619,9 @@ dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, /*Clear the unused FD fields before sending*/ memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); sess = (dpaa2_sec_session *) - (*ops)->sym->session->_private; + get_session_private_data( + (*ops)->sym->session, + cryptodev_driver_id); mb_pool = (*ops)->sym->m_src->pool; bpid = mempool_to_bpid(mb_pool); ret = build_sec_fd(sess, *ops, &fd_arr[loop], bpid); @@ -495,6 +653,7 @@ sec_fd_to_mbuf(const struct qbman_fd *fd) { struct qbman_fle *fle; struct rte_crypto_op *op; + struct ctxt_priv *priv; fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); @@ -510,7 +669,7 @@ sec_fd_to_mbuf(const struct qbman_fd *fd) if (unlikely(DPAA2_GET_FD_IVP(fd))) { /* TODO complete it. */ - RTE_LOG(ERR, PMD, "error: Non inline buffer - WHAT to DO?"); + RTE_LOG(ERR, PMD, "error: Non inline buffer - WHAT to DO?\n"); return NULL; } op = (struct rte_crypto_op *)DPAA2_IOVA_TO_VADDR( @@ -530,7 +689,8 @@ sec_fd_to_mbuf(const struct qbman_fd *fd) DPAA2_GET_FD_LEN(fd)); /* free the fle memory */ - rte_free(fle - 1); + priv = (struct ctxt_priv *)DPAA2_GET_FLE_CTXT(fle - 1); + rte_mempool_put(priv->fle_pool, (void *)(fle - 1)); return op; } @@ -571,8 +731,8 @@ dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, /*Issue a volatile dequeue command. */ while (1) { if (qbman_swp_pull(swp, &pulldesc)) { - RTE_LOG(WARNING, PMD, "SEC VDQ command is not issued." - "QBMAN is busy\n"); + RTE_LOG(WARNING, PMD, + "SEC VDQ command is not issued : QBMAN busy\n"); /* Portal was busy, try again */ continue; } @@ -656,7 +816,8 @@ dpaa2_sec_queue_pair_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) static int dpaa2_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, - __rte_unused int socket_id) + __rte_unused int socket_id, + __rte_unused struct rte_mempool *session_pool) { struct dpaa2_sec_dev_private *priv = dev->data->dev_private; struct dpaa2_sec_qp *qp; @@ -747,19 +908,12 @@ dpaa2_sec_session_get_size(struct rte_cryptodev *dev __rte_unused) return sizeof(dpaa2_sec_session); } -static void -dpaa2_sec_session_initialize(struct rte_mempool *mp __rte_unused, - void *sess __rte_unused) -{ - PMD_INIT_FUNC_TRACE(); -} - static int dpaa2_sec_cipher_init(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, dpaa2_sec_session *session) { - struct dpaa2_sec_cipher_ctxt *ctxt = &session->ext_params.cipher_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; struct alginfo cipherdata; int bufsize, i; struct ctxt_priv *priv; @@ -772,16 +926,18 @@ dpaa2_sec_cipher_init(struct rte_cryptodev *dev, sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), RTE_CACHE_LINE_SIZE); if (priv == NULL) { - RTE_LOG(ERR, PMD, "No Memory for priv CTXT"); + RTE_LOG(ERR, PMD, "No Memory for priv CTXT\n"); return -1; } + priv->fle_pool = dev_priv->fle_pool; + flc = &priv->flc_desc[0].flc; session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, RTE_CACHE_LINE_SIZE); if (session->cipher_key.data == NULL) { - RTE_LOG(ERR, PMD, "No Memory for cipher key"); + RTE_LOG(ERR, PMD, "No Memory for cipher key\n"); rte_free(priv); return -1; } @@ -794,23 +950,27 @@ dpaa2_sec_cipher_init(struct rte_cryptodev *dev, cipherdata.key_enc_flags = 0; cipherdata.key_type = RTA_DATA_IMM; + /* Set IV parameters */ + session->iv.offset = xform->cipher.iv.offset; + session->iv.length = xform->cipher.iv.length; + switch (xform->cipher.algo) { case RTE_CRYPTO_CIPHER_AES_CBC: cipherdata.algtype = OP_ALG_ALGSEL_AES; cipherdata.algmode = OP_ALG_AAI_CBC; session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; - ctxt->iv.length = AES_CBC_IV_LEN; break; case RTE_CRYPTO_CIPHER_3DES_CBC: cipherdata.algtype = OP_ALG_ALGSEL_3DES; cipherdata.algmode = OP_ALG_AAI_CBC; session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; - ctxt->iv.length = TDES_CBC_IV_LEN; break; case RTE_CRYPTO_CIPHER_AES_CTR: + cipherdata.algtype = OP_ALG_ALGSEL_AES; + cipherdata.algmode = OP_ALG_AAI_CTR; + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; + break; case RTE_CRYPTO_CIPHER_3DES_CTR: - case RTE_CRYPTO_CIPHER_AES_GCM: - case RTE_CRYPTO_CIPHER_AES_CCM: case RTE_CRYPTO_CIPHER_AES_ECB: case RTE_CRYPTO_CIPHER_3DES_ECB: case RTE_CRYPTO_CIPHER_AES_XTS: @@ -820,7 +980,7 @@ dpaa2_sec_cipher_init(struct rte_cryptodev *dev, case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: case RTE_CRYPTO_CIPHER_ZUC_EEA3: case RTE_CRYPTO_CIPHER_NULL: - RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u", + RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u\n", xform->cipher.algo); goto error_out; default: @@ -832,8 +992,8 @@ dpaa2_sec_cipher_init(struct rte_cryptodev *dev, DIR_ENC : DIR_DEC; bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0, - &cipherdata, NULL, ctxt->iv.length, - session->dir); + &cipherdata, NULL, session->iv.length, + session->dir); if (bufsize < 0) { RTE_LOG(ERR, PMD, "Crypto: Descriptor build failed\n"); goto error_out; @@ -868,9 +1028,9 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, dpaa2_sec_session *session) { - struct dpaa2_sec_auth_ctxt *ctxt = &session->ext_params.auth_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; struct alginfo authdata; - unsigned int bufsize; + unsigned int bufsize, i; struct ctxt_priv *priv; struct sec_flow_context *flc; @@ -882,16 +1042,17 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, sizeof(struct sec_flc_desc), RTE_CACHE_LINE_SIZE); if (priv == NULL) { - RTE_LOG(ERR, PMD, "No Memory for priv CTXT"); + RTE_LOG(ERR, PMD, "No Memory for priv CTXT\n"); return -1; } + priv->fle_pool = dev_priv->fle_pool; flc = &priv->flc_desc[DESC_INITFINAL].flc; session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, RTE_CACHE_LINE_SIZE); if (session->auth_key.data == NULL) { - RTE_LOG(ERR, PMD, "No Memory for auth key"); + RTE_LOG(ERR, PMD, "No Memory for auth key\n"); rte_free(priv); return -1; } @@ -904,6 +1065,8 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, authdata.key_enc_flags = 0; authdata.key_type = RTA_DATA_IMM; + session->digest_length = xform->auth.digest_length; + switch (xform->auth.algo) { case RTE_CRYPTO_AUTH_SHA1_HMAC: authdata.algtype = OP_ALG_ALGSEL_SHA1; @@ -936,7 +1099,6 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC; break; case RTE_CRYPTO_AUTH_AES_XCBC_MAC: - case RTE_CRYPTO_AUTH_AES_GCM: case RTE_CRYPTO_AUTH_SNOW3G_UIA2: case RTE_CRYPTO_AUTH_NULL: case RTE_CRYPTO_AUTH_SHA1: @@ -945,13 +1107,12 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, case RTE_CRYPTO_AUTH_SHA224: case RTE_CRYPTO_AUTH_SHA384: case RTE_CRYPTO_AUTH_MD5: - case RTE_CRYPTO_AUTH_AES_CCM: case RTE_CRYPTO_AUTH_AES_GMAC: case RTE_CRYPTO_AUTH_KASUMI_F9: case RTE_CRYPTO_AUTH_AES_CMAC: case RTE_CRYPTO_AUTH_AES_CBC_MAC: case RTE_CRYPTO_AUTH_ZUC_EIA3: - RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u", + RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u\n", xform->auth.algo); goto error_out; default: @@ -964,7 +1125,7 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc, 1, 0, &authdata, !session->dir, - ctxt->trunc_len); + session->digest_length); flc->word1_sdl = (uint8_t)bufsize; flc->word2_rflc_31_0 = lower_32_bits( @@ -974,6 +1135,10 @@ dpaa2_sec_auth_init(struct rte_cryptodev *dev, (uint64_t)&(((struct dpaa2_sec_qp *) dev->data->queue_pairs[0])->rx_vq)); session->ctxt = priv; + for (i = 0; i < bufsize; i++) + PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n", + i, priv->flc_desc[DESC_INITFINAL].desc[i]); + return 0; @@ -989,8 +1154,129 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, dpaa2_sec_session *session) { struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; + struct alginfo aeaddata; + unsigned int bufsize, i; + struct ctxt_priv *priv; + struct sec_flow_context *flc; + struct rte_crypto_aead_xform *aead_xform = &xform->aead; + int err; + + PMD_INIT_FUNC_TRACE(); + + /* Set IV parameters */ + session->iv.offset = aead_xform->iv.offset; + session->iv.length = aead_xform->iv.length; + session->ctxt_type = DPAA2_SEC_AEAD; + + /* For SEC AEAD only one descriptor is required */ + priv = (struct ctxt_priv *)rte_zmalloc(NULL, + sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), + RTE_CACHE_LINE_SIZE); + if (priv == NULL) { + RTE_LOG(ERR, PMD, "No Memory for priv CTXT\n"); + return -1; + } + + priv->fle_pool = dev_priv->fle_pool; + flc = &priv->flc_desc[0].flc; + + session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->aead_key.data == NULL && aead_xform->key.length > 0) { + RTE_LOG(ERR, PMD, "No Memory for aead key\n"); + rte_free(priv); + return -1; + } + memcpy(session->aead_key.data, aead_xform->key.data, + aead_xform->key.length); + + session->digest_length = aead_xform->digest_length; + session->aead_key.length = aead_xform->key.length; + ctxt->auth_only_len = aead_xform->aad_length; + + aeaddata.key = (uint64_t)session->aead_key.data; + aeaddata.keylen = session->aead_key.length; + aeaddata.key_enc_flags = 0; + aeaddata.key_type = RTA_DATA_IMM; + + switch (aead_xform->algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + aeaddata.algtype = OP_ALG_ALGSEL_AES; + aeaddata.algmode = OP_ALG_AAI_GCM; + session->cipher_alg = RTE_CRYPTO_AEAD_AES_GCM; + break; + case RTE_CRYPTO_AEAD_AES_CCM: + RTE_LOG(ERR, PMD, "Crypto: Unsupported AEAD alg %u\n", + aead_xform->algo); + goto error_out; + default: + RTE_LOG(ERR, PMD, "Crypto: Undefined AEAD specified %u\n", + aead_xform->algo); + goto error_out; + } + session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? + DIR_ENC : DIR_DEC; + + priv->flc_desc[0].desc[0] = aeaddata.keylen; + err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, + MIN_JOB_DESC_SIZE, + (unsigned int *)priv->flc_desc[0].desc, + &priv->flc_desc[0].desc[1], 1); + + if (err < 0) { + PMD_DRV_LOG(ERR, "Crypto: Incorrect key lengths\n"); + goto error_out; + } + if (priv->flc_desc[0].desc[1] & 1) { + aeaddata.key_type = RTA_DATA_IMM; + } else { + aeaddata.key = DPAA2_VADDR_TO_IOVA(aeaddata.key); + aeaddata.key_type = RTA_DATA_PTR; + } + priv->flc_desc[0].desc[0] = 0; + priv->flc_desc[0].desc[1] = 0; + + if (session->dir == DIR_ENC) + bufsize = cnstr_shdsc_gcm_encap( + priv->flc_desc[0].desc, 1, 0, + &aeaddata, session->iv.length, + session->digest_length); + else + bufsize = cnstr_shdsc_gcm_decap( + priv->flc_desc[0].desc, 1, 0, + &aeaddata, session->iv.length, + session->digest_length); + flc->word1_sdl = (uint8_t)bufsize; + flc->word2_rflc_31_0 = lower_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + flc->word3_rflc_63_32 = upper_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + session->ctxt = priv; + for (i = 0; i < bufsize; i++) + PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n", + i, priv->flc_desc[0].desc[i]); + + return 0; + +error_out: + rte_free(session->aead_key.data); + rte_free(priv); + return -1; +} + + +static int +dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + dpaa2_sec_session *session) +{ + struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt; + struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; struct alginfo authdata, cipherdata; - unsigned int bufsize; + unsigned int bufsize, i; struct ctxt_priv *priv; struct sec_flow_context *flc; struct rte_crypto_cipher_xform *cipher_xform; @@ -1012,21 +1298,27 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? DPAA2_SEC_HASH_CIPHER : DPAA2_SEC_CIPHER_HASH; } + + /* Set IV parameters */ + session->iv.offset = cipher_xform->iv.offset; + session->iv.length = cipher_xform->iv.length; + /* For SEC AEAD only one descriptor is required */ priv = (struct ctxt_priv *)rte_zmalloc(NULL, sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), RTE_CACHE_LINE_SIZE); if (priv == NULL) { - RTE_LOG(ERR, PMD, "No Memory for priv CTXT"); + RTE_LOG(ERR, PMD, "No Memory for priv CTXT\n"); return -1; } + priv->fle_pool = dev_priv->fle_pool; flc = &priv->flc_desc[0].flc; session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length, RTE_CACHE_LINE_SIZE); if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) { - RTE_LOG(ERR, PMD, "No Memory for cipher key"); + RTE_LOG(ERR, PMD, "No Memory for cipher key\n"); rte_free(priv); return -1; } @@ -1034,7 +1326,7 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length, RTE_CACHE_LINE_SIZE); if (session->auth_key.data == NULL && auth_xform->key.length > 0) { - RTE_LOG(ERR, PMD, "No Memory for auth key"); + RTE_LOG(ERR, PMD, "No Memory for auth key\n"); rte_free(session->cipher_key.data); rte_free(priv); return -1; @@ -1045,12 +1337,13 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, memcpy(session->auth_key.data, auth_xform->key.data, auth_xform->key.length); - ctxt->trunc_len = auth_xform->digest_length; authdata.key = (uint64_t)session->auth_key.data; authdata.keylen = session->auth_key.length; authdata.key_enc_flags = 0; authdata.key_type = RTA_DATA_IMM; + session->digest_length = auth_xform->digest_length; + switch (auth_xform->algo) { case RTE_CRYPTO_AUTH_SHA1_HMAC: authdata.algtype = OP_ALG_ALGSEL_SHA1; @@ -1083,7 +1376,6 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; break; case RTE_CRYPTO_AUTH_AES_XCBC_MAC: - case RTE_CRYPTO_AUTH_AES_GCM: case RTE_CRYPTO_AUTH_SNOW3G_UIA2: case RTE_CRYPTO_AUTH_NULL: case RTE_CRYPTO_AUTH_SHA1: @@ -1092,13 +1384,12 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, case RTE_CRYPTO_AUTH_SHA224: case RTE_CRYPTO_AUTH_SHA384: case RTE_CRYPTO_AUTH_MD5: - case RTE_CRYPTO_AUTH_AES_CCM: case RTE_CRYPTO_AUTH_AES_GMAC: case RTE_CRYPTO_AUTH_KASUMI_F9: case RTE_CRYPTO_AUTH_AES_CMAC: case RTE_CRYPTO_AUTH_AES_CBC_MAC: case RTE_CRYPTO_AUTH_ZUC_EIA3: - RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u", + RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u\n", auth_xform->algo); goto error_out; default: @@ -1116,23 +1407,23 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, cipherdata.algtype = OP_ALG_ALGSEL_AES; cipherdata.algmode = OP_ALG_AAI_CBC; session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; - ctxt->iv.length = AES_CBC_IV_LEN; break; case RTE_CRYPTO_CIPHER_3DES_CBC: cipherdata.algtype = OP_ALG_ALGSEL_3DES; cipherdata.algmode = OP_ALG_AAI_CBC; session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; - ctxt->iv.length = TDES_CBC_IV_LEN; break; - case RTE_CRYPTO_CIPHER_AES_GCM: + case RTE_CRYPTO_CIPHER_AES_CTR: + cipherdata.algtype = OP_ALG_ALGSEL_AES; + cipherdata.algmode = OP_ALG_AAI_CTR; + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; + break; case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: case RTE_CRYPTO_CIPHER_NULL: case RTE_CRYPTO_CIPHER_3DES_ECB: case RTE_CRYPTO_CIPHER_AES_ECB: - case RTE_CRYPTO_CIPHER_AES_CTR: - case RTE_CRYPTO_CIPHER_AES_CCM: case RTE_CRYPTO_CIPHER_KASUMI_F8: - RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u", + RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u\n", cipher_xform->algo); goto error_out; default: @@ -1151,7 +1442,7 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, &priv->flc_desc[0].desc[2], 2); if (err < 0) { - PMD_DRV_LOG(ERR, "Crypto: Incorrect key lengths"); + PMD_DRV_LOG(ERR, "Crypto: Incorrect key lengths\n"); goto error_out; } if (priv->flc_desc[0].desc[2] & 1) { @@ -1173,12 +1464,12 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, if (session->ctxt_type == DPAA2_SEC_CIPHER_HASH) { bufsize = cnstr_shdsc_authenc(priv->flc_desc[0].desc, 1, 0, &cipherdata, &authdata, - ctxt->iv.length, + session->iv.length, ctxt->auth_only_len, - ctxt->trunc_len, + session->digest_length, session->dir); } else { - RTE_LOG(ERR, PMD, "Hash before cipher not supported"); + RTE_LOG(ERR, PMD, "Hash before cipher not supported\n"); goto error_out; } @@ -1190,6 +1481,9 @@ dpaa2_sec_aead_init(struct rte_cryptodev *dev, (uint64_t)&(((struct dpaa2_sec_qp *) dev->data->queue_pairs[0])->rx_vq)); session->ctxt = priv; + for (i = 0; i < bufsize; i++) + PMD_DRV_LOG(DEBUG, "DESC[%d]:0x%x\n", + i, priv->flc_desc[0].desc[i]); return 0; @@ -1200,8 +1494,8 @@ error_out: return -1; } -static void * -dpaa2_sec_session_configure(struct rte_cryptodev *dev, +static int +dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *sess) { dpaa2_sec_session *session = sess; @@ -1209,9 +1503,13 @@ dpaa2_sec_session_configure(struct rte_cryptodev *dev, PMD_INIT_FUNC_TRACE(); if (unlikely(sess == NULL)) { - RTE_LOG(ERR, PMD, "invalid session struct"); - return NULL; + RTE_LOG(ERR, PMD, "invalid session struct\n"); + return -1; } + + /* Default IV length = 0 */ + session->iv.length = 0; + /* Cipher Only */ if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { session->ctxt_type = DPAA2_SEC_CIPHER; @@ -1227,33 +1525,76 @@ dpaa2_sec_session_configure(struct rte_cryptodev *dev, } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { session->ext_params.aead_ctxt.auth_cipher_text = true; - dpaa2_sec_aead_init(dev, xform, session); + dpaa2_sec_aead_chain_init(dev, xform, session); /* Authenticate then Cipher */ } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { session->ext_params.aead_ctxt.auth_cipher_text = false; + dpaa2_sec_aead_chain_init(dev, xform, session); + + /* AEAD operation for AES-GCM kind of Algorithms */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && + xform->next == NULL) { dpaa2_sec_aead_init(dev, xform, session); + } else { - RTE_LOG(ERR, PMD, "Invalid crypto type"); - return NULL; + RTE_LOG(ERR, PMD, "Invalid crypto type\n"); + return -EINVAL; + } + + return 0; +} + +static int +dpaa2_sec_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; } - return session; + ret = dpaa2_sec_set_session_parameters(dev, xform, sess_private_data); + if (ret != 0) { + PMD_DRV_LOG(ERR, "DPAA2 PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -dpaa2_sec_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +dpaa2_sec_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { PMD_INIT_FUNC_TRACE(); - dpaa2_sec_session *s = (dpaa2_sec_session *)sess; + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv; - if (s) { + if (sess_priv) { rte_free(s->ctxt); rte_free(s->cipher_key.data); rte_free(s->auth_key.data); memset(sess, 0, sizeof(dpaa2_sec_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); } } @@ -1263,7 +1604,7 @@ dpaa2_sec_dev_configure(struct rte_cryptodev *dev __rte_unused, { PMD_INIT_FUNC_TRACE(); - return -ENOTSUP; + return 0; } static int @@ -1366,7 +1707,7 @@ dpaa2_sec_dev_close(struct rte_cryptodev *dev) /*Free the allocated memory for ethernet private data and dpseci*/ priv->hw = NULL; - free(dpseci); + rte_free(dpseci); return 0; } @@ -1383,7 +1724,7 @@ dpaa2_sec_dev_infos_get(struct rte_cryptodev *dev, info->feature_flags = dev->feature_flags; info->capabilities = dpaa2_sec_capabilities; info->sym.max_nb_sessions = internals->max_nb_sessions; - info->dev_type = RTE_CRYPTODEV_DPAA2_SEC_PMD; + info->driver_id = cryptodev_driver_id; } } @@ -1475,15 +1816,17 @@ static struct rte_cryptodev_ops crypto_ops = { .queue_pair_stop = dpaa2_sec_queue_pair_stop, .queue_pair_count = dpaa2_sec_queue_pair_count, .session_get_size = dpaa2_sec_session_get_size, - .session_initialize = dpaa2_sec_session_initialize, .session_configure = dpaa2_sec_session_configure, .session_clear = dpaa2_sec_session_clear, }; static int -dpaa2_sec_uninit(const struct rte_cryptodev_driver *crypto_drv __rte_unused, - struct rte_cryptodev *dev) +dpaa2_sec_uninit(const struct rte_cryptodev *dev) { + struct dpaa2_sec_dev_private *internals = dev->data->dev_private; + + rte_mempool_free(internals->fle_pool); + PMD_INIT_LOG(INFO, "Closing DPAA2_SEC device %s on numa socket %u\n", dev->data->name, rte_socket_id()); @@ -1500,6 +1843,7 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) uint16_t token; struct dpseci_attr attr; int retcode, hw_id; + char str[20]; PMD_INIT_FUNC_TRACE(); dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device); @@ -1509,7 +1853,7 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) } hw_id = dpaa2_dev->object_id; - cryptodev->dev_type = RTE_CRYPTODEV_DPAA2_SEC_PMD; + cryptodev->driver_id = cryptodev_driver_id; cryptodev->dev_ops = &crypto_ops; cryptodev->enqueue_burst = dpaa2_sec_enqueue_burst; @@ -1560,6 +1904,18 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) internals->hw = dpseci; internals->token = token; + sprintf(str, "fle_pool_%d", cryptodev->data->dev_id); + internals->fle_pool = rte_mempool_create((const char *)str, + FLE_POOL_NUM_BUFS, + FLE_POOL_BUF_SIZE, + FLE_POOL_CACHE_SIZE, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + if (!internals->fle_pool) { + RTE_LOG(ERR, PMD, "%s create failed\n", str); + goto init_error; + } + PMD_INIT_LOG(DEBUG, "driver %s: created\n", cryptodev->data->name); return 0; @@ -1599,7 +1955,7 @@ cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv, dpaa2_dev->cryptodev = cryptodev; cryptodev->device = &dpaa2_dev->device; - cryptodev->driver = (struct rte_cryptodev_driver *)dpaa2_drv; + cryptodev->device->driver = &dpaa2_drv->driver; /* init user callbacks */ TAILQ_INIT(&(cryptodev->link_intr_cbs)); @@ -1627,7 +1983,7 @@ cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev) if (cryptodev == NULL) return -ENODEV; - ret = dpaa2_sec_uninit(NULL, cryptodev); + ret = dpaa2_sec_uninit(cryptodev); if (ret) return ret; @@ -1638,7 +1994,6 @@ cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev) rte_free(cryptodev->data->dev_private); cryptodev->device = NULL; - cryptodev->driver = NULL; cryptodev->data = NULL; return 0; @@ -1653,4 +2008,5 @@ static struct rte_dpaa2_driver rte_dpaa2_sec_driver = { .remove = cryptodev_dpaa2_sec_remove, }; -RTE_PMD_REGISTER_DPAA2(dpaa2_sec_pmd, rte_dpaa2_sec_driver); +RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver); +RTE_PMD_REGISTER_CRYPTO_DRIVER(rte_dpaa2_sec_driver, cryptodev_driver_id); diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h b/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h index 03d4c703..b5e99f80 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h index f5c61694..3849a052 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h @@ -2,7 +2,7 @@ * BSD LICENSE * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright (c) 2016 NXP. All rights reserved. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -34,12 +34,16 @@ #ifndef _RTE_DPAA2_SEC_PMD_PRIVATE_H_ #define _RTE_DPAA2_SEC_PMD_PRIVATE_H_ +#define CRYPTODEV_NAME_DPAA2_SEC_PMD crypto_dpaa2_sec +/**< NXP DPAA2 - SEC PMD device name */ + #define MAX_QUEUES 64 #define MAX_DESC_SIZE 64 /** private data structure for each DPAA2_SEC device */ struct dpaa2_sec_dev_private { void *mc_portal; /**< MC Portal for configuring this device */ void *hw; /**< Hardware handle for this device.Used by NADK framework */ + struct rte_mempool *fle_pool; /* per device memory pool for FLE */ int32_t hw_id; /**< An unique ID of this device instance */ int32_t vfio_fd; /**< File descriptor received via VFIO */ uint16_t token; /**< Token required by DPxxx objects */ @@ -128,6 +132,7 @@ struct sec_flc_desc { }; struct ctxt_priv { + struct rte_mempool *fle_pool; /* per device memory pool for FLE */ struct sec_flc_desc flc_desc[0]; }; @@ -135,6 +140,7 @@ enum dpaa2_sec_op_type { DPAA2_SEC_NONE, /*!< No Cipher operations*/ DPAA2_SEC_CIPHER,/*!< CIPHER operations */ DPAA2_SEC_AUTH, /*!< Authentication Operations */ + DPAA2_SEC_AEAD, /*!< AEAD (AES-GCM/CCM) type operations */ DPAA2_SEC_CIPHER_HASH, /*!< Authenticated Encryption with * associated data */ @@ -147,30 +153,9 @@ enum dpaa2_sec_op_type { DPAA2_SEC_MAX }; -struct dpaa2_sec_cipher_ctxt { - struct { - uint8_t *data; - uint16_t length; - } iv; /**< Initialisation vector parameters */ - uint8_t *init_counter; /*!< Set initial counter for CTR mode */ -}; - -struct dpaa2_sec_auth_ctxt { - uint8_t trunc_len; /*!< Length for output ICV, should - * be 0 if no truncation required - */ -}; - struct dpaa2_sec_aead_ctxt { - struct { - uint8_t *data; - uint16_t length; - } iv; /**< Initialisation vector parameters */ uint16_t auth_only_len; /*!< Length of data for Auth only */ uint8_t auth_cipher_text; /**< Authenticate/cipher ordering */ - uint8_t trunc_len; /*!< Length for output ICV, should - * be 0 if no truncation required - */ }; typedef struct dpaa2_sec_session_entry { @@ -179,18 +164,29 @@ typedef struct dpaa2_sec_session_entry { uint8_t dir; /*!< Operation Direction */ enum rte_crypto_cipher_algorithm cipher_alg; /*!< Cipher Algorithm*/ enum rte_crypto_auth_algorithm auth_alg; /*!< Authentication Algorithm*/ + union { + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } aead_key; + struct { + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } cipher_key; + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } auth_key; + }; + }; struct { - uint8_t *data; /**< pointer to key data */ - size_t length; /**< key length in bytes */ - } cipher_key; - struct { - uint8_t *data; /**< pointer to key data */ - size_t length; /**< key length in bytes */ - } auth_key; + uint16_t length; /**< IV length in bytes */ + uint16_t offset; /**< IV offset in bytes */ + } iv; + uint16_t digest_length; uint8_t status; union { - struct dpaa2_sec_cipher_ctxt cipher_ctxt; - struct dpaa2_sec_auth_ctxt auth_ctxt; struct dpaa2_sec_aead_ctxt aead_ctxt; } ext_params; } dpaa2_sec_session; @@ -204,16 +200,16 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .algo = RTE_CRYPTO_AUTH_MD5_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 16, .max = 16, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -225,16 +221,16 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 20, .max = 20, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -246,16 +242,16 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 28, .max = 28, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -267,16 +263,16 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 32, .max = 32, .increment = 0 - }, - .aad_size = { 0 } + }, + .iv_size = { 0 } }, } }, } }, @@ -288,16 +284,16 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 48, .max = 48, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -309,16 +305,46 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 64, .max = 64, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } + }, } + }, } + }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 8, + .max = 16, + .increment = 4 + }, + .aad_size = { + .min = 0, + .max = 240, + .increment = 1 + }, + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 + }, }, } }, } }, @@ -342,6 +368,26 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { }, } }, } }, + { /* AES CTR */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CTR, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + }, } + }, } + }, { /* 3DES CBC */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { diff --git a/drivers/crypto/dpaa2_sec/hw/compat.h b/drivers/crypto/dpaa2_sec/hw/compat.h index 11fdaa8e..b17b27ef 100644 --- a/drivers/crypto/dpaa2_sec/hw/compat.h +++ b/drivers/crypto/dpaa2_sec/hw/compat.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -49,7 +49,9 @@ #include <stdlib.h> #include <stdio.h> #include <stdbool.h> + #include <rte_byteorder.h> +#include <rte_common.h> #ifndef __BYTE_ORDER__ #error "Undefined endianness" @@ -60,7 +62,7 @@ #endif #ifndef __always_inline -#define __always_inline (inline __attribute__((always_inline))) +#define __always_inline __rte_always_inline #endif #ifndef __always_unused diff --git a/drivers/crypto/dpaa2_sec/hw/desc.h b/drivers/crypto/dpaa2_sec/hw/desc.h index 5185be22..0be4439c 100644 --- a/drivers/crypto/dpaa2_sec/hw/desc.h +++ b/drivers/crypto/dpaa2_sec/hw/desc.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/desc/algo.h b/drivers/crypto/dpaa2_sec/hw/desc/algo.h index c71ada07..5ae3a1ac 100644 --- a/drivers/crypto/dpaa2_sec/hw/desc/algo.h +++ b/drivers/crypto/dpaa2_sec/hw/desc/algo.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -159,6 +159,10 @@ cnstr_shdsc_snow_f9(uint32_t *descbuf, bool ps, bool swap, * @ps: if 36/40bit addressing is desired, this parameter must be true * @swap: must be true when core endianness doesn't match SEC endianness * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values one of OP_ALG_ALGSEL_* {DES, 3DES, AES} + * Valid modes for: + * AES: OP_ALG_AAI_* {CBC, CTR} + * DES, 3DES: OP_ALG_AAI_CBC * @iv: IV data; if NULL, "ivlen" bytes from the input frame will be read as IV * @ivlen: IV length * @dir: DIR_ENC/DIR_DEC @@ -172,8 +176,10 @@ cnstr_shdsc_blkcipher(uint32_t *descbuf, bool ps, bool swap, { struct program prg; struct program *p = &prg; - const bool is_aes_dec = (dir == DIR_DEC) && - (cipherdata->algtype == OP_ALG_ALGSEL_AES); + uint32_t iv_off = 0; + const bool need_dk = (dir == DIR_DEC) && + (cipherdata->algtype == OP_ALG_ALGSEL_AES) && + (cipherdata->algmode == OP_ALG_AAI_CBC); LABEL(keyjmp); LABEL(skipdk); REFERENCE(pkeyjmp); @@ -191,7 +197,7 @@ cnstr_shdsc_blkcipher(uint32_t *descbuf, bool ps, bool swap, KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, cipherdata->keylen, INLINE_KEY(cipherdata)); - if (is_aes_dec) { + if (need_dk) { ALG_OPERATION(p, cipherdata->algtype, cipherdata->algmode, OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); @@ -199,7 +205,7 @@ cnstr_shdsc_blkcipher(uint32_t *descbuf, bool ps, bool swap, } SET_LABEL(p, keyjmp); - if (is_aes_dec) { + if (need_dk) { ALG_OPERATION(p, OP_ALG_ALGSEL_AES, cipherdata->algmode | OP_ALG_AAI_DK, OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); @@ -209,12 +215,15 @@ cnstr_shdsc_blkcipher(uint32_t *descbuf, bool ps, bool swap, OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); } + if (cipherdata->algmode == OP_ALG_AAI_CTR) + iv_off = 16; + if (iv) /* IV load, convert size */ - LOAD(p, (uintptr_t)iv, CONTEXT1, 0, ivlen, IMMED | COPY); + LOAD(p, (uintptr_t)iv, CONTEXT1, iv_off, ivlen, IMMED | COPY); else /* IV is present first before the actual message */ - SEQLOAD(p, CONTEXT1, 0, ivlen, 0); + SEQLOAD(p, CONTEXT1, iv_off, ivlen, 0); MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0); MATHB(p, SEQINSZ, SUB, MATH2, VSEQOUTSZ, 4, 0); @@ -224,7 +233,7 @@ cnstr_shdsc_blkcipher(uint32_t *descbuf, bool ps, bool swap, SEQFIFOSTORE(p, MSG, 0, 0, VLF); PATCH_JUMP(p, pkeyjmp, keyjmp); - if (is_aes_dec) + if (need_dk) PATCH_JUMP(p, pskipdk, skipdk); return PROGRAM_FINALIZE(p); @@ -434,6 +443,211 @@ cnstr_shdsc_kasumi_f9(uint32_t *descbuf, bool ps, bool swap, } /** + * cnstr_shdsc_gcm_encap - AES-GCM encap as a shared descriptor + * @descbuf: pointer to descriptor-under-construction buffer + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with + * OP_ALG_AAI_GCM. + * @ivlen: Initialization vector length + * @icvsize: integrity check value (ICV) size (truncated or full) + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_gcm_encap(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *cipherdata, + uint32_t ivlen, uint32_t icvsize) +{ + struct program prg; + struct program *p = &prg; + + LABEL(keyjmp); + LABEL(zeroassocjump2); + LABEL(zeroassocjump1); + LABEL(zeropayloadjump); + REFERENCE(pkeyjmp); + REFERENCE(pzeroassocjump2); + REFERENCE(pzeroassocjump1); + REFERENCE(pzeropayloadjump); + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + + SHR_HDR(p, SHR_SERIAL, 1, SC); + + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SELF | SHRD); + /* Insert Key */ + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + + SET_LABEL(p, keyjmp); + + /* class 1 operation */ + ALG_OPERATION(p, cipherdata->algtype, cipherdata->algmode, + OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, DIR_ENC); + + MATHB(p, DPOVRD, AND, 0x7fffffff, MATH3, 4, IMMED2); + + /* if assoclen + cryptlen is ZERO, skip to ICV write */ + MATHB(p, SEQINSZ, SUB, ivlen, VSEQOUTSZ, 4, IMMED2); + pzeroassocjump2 = JUMP(p, zeroassocjump2, LOCAL_JUMP, ALL_TRUE, MATH_Z); + + SEQFIFOLOAD(p, IV1, ivlen, FLUSH1); + + /* if assoclen is ZERO, skip reading the assoc data */ + MATHB(p, ZERO, ADD, MATH3, VSEQINSZ, 4, 0); + pzeroassocjump1 = JUMP(p, zeroassocjump1, LOCAL_JUMP, ALL_TRUE, MATH_Z); + + MATHB(p, ZERO, ADD, MATH3, VSEQOUTSZ, 4, 0); + + /* skip assoc data */ + SEQFIFOSTORE(p, SKIP, 0, 0, VLF); + + /* cryptlen = seqinlen - assoclen */ + MATHB(p, SEQINSZ, SUB, MATH3, VSEQOUTSZ, 4, 0); + + /* if cryptlen is ZERO jump to zero-payload commands */ + pzeropayloadjump = JUMP(p, zeropayloadjump, LOCAL_JUMP, ALL_TRUE, + MATH_Z); + + /* read assoc data */ + SEQFIFOLOAD(p, AAD1, 0, CLASS1 | VLF | FLUSH1); + SET_LABEL(p, zeroassocjump1); + + MATHB(p, SEQINSZ, SUB, MATH0, VSEQINSZ, 4, 0); + + /* write encrypted data */ + SEQFIFOSTORE(p, MSG, 0, 0, VLF); + + /* read payload data */ + SEQFIFOLOAD(p, MSG1, 0, CLASS1 | VLF | LAST1); + + /* jump the zero-payload commands */ + JUMP(p, 4, LOCAL_JUMP, ALL_TRUE, 0); + + /* zero-payload commands */ + SET_LABEL(p, zeropayloadjump); + + /* read assoc data */ + SEQFIFOLOAD(p, AAD1, 0, CLASS1 | VLF | LAST1); + + JUMP(p, 2, LOCAL_JUMP, ALL_TRUE, 0); + + /* There is no input data */ + SET_LABEL(p, zeroassocjump2); + + SEQFIFOLOAD(p, IV1, ivlen, FLUSH1 | LAST1); + + /* write ICV */ + SEQSTORE(p, CONTEXT1, 0, icvsize, 0); + + PATCH_JUMP(p, pkeyjmp, keyjmp); + PATCH_JUMP(p, pzeroassocjump2, zeroassocjump2); + PATCH_JUMP(p, pzeroassocjump1, zeroassocjump1); + PATCH_JUMP(p, pzeropayloadjump, zeropayloadjump); + + return PROGRAM_FINALIZE(p); +} + +/** + * cnstr_shdsc_gcm_decap - AES-GCM decap as a shared descriptor + * @descbuf: pointer to descriptor-under-construction buffer + * @ps: if 36/40bit addressing is desired, this parameter must be true + * @swap: must be true when core endianness doesn't match SEC endianness + * @cipherdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with + * OP_ALG_AAI_GCM. + * @icvsize: integrity check value (ICV) size (truncated or full) + * + * Return: size of descriptor written in words or negative number on error + */ +static inline int +cnstr_shdsc_gcm_decap(uint32_t *descbuf, bool ps, bool swap, + struct alginfo *cipherdata, + uint32_t ivlen, uint32_t icvsize) +{ + struct program prg; + struct program *p = &prg; + + LABEL(keyjmp); + LABEL(zeroassocjump1); + LABEL(zeropayloadjump); + REFERENCE(pkeyjmp); + REFERENCE(pzeroassocjump1); + REFERENCE(pzeropayloadjump); + + PROGRAM_CNTXT_INIT(p, descbuf, 0); + + if (swap) + PROGRAM_SET_BSWAP(p); + if (ps) + PROGRAM_SET_36BIT_ADDR(p); + + SHR_HDR(p, SHR_SERIAL, 1, SC); + + pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SELF | SHRD); + /* Insert Key */ + KEY(p, KEY1, cipherdata->key_enc_flags, cipherdata->key, + cipherdata->keylen, INLINE_KEY(cipherdata)); + + SET_LABEL(p, keyjmp); + + /* class 1 operation */ + ALG_OPERATION(p, cipherdata->algtype, cipherdata->algmode, + OP_ALG_AS_INITFINAL, ICV_CHECK_ENABLE, DIR_DEC); + + MATHB(p, DPOVRD, AND, 0x7fffffff, MATH3, 4, IMMED2); + SEQFIFOLOAD(p, IV1, ivlen, FLUSH1); + + /* if assoclen is ZERO, skip reading the assoc data */ + MATHB(p, ZERO, ADD, MATH3, VSEQINSZ, 4, 0); + pzeroassocjump1 = JUMP(p, zeroassocjump1, LOCAL_JUMP, ALL_TRUE, MATH_Z); + + MATHB(p, ZERO, ADD, MATH3, VSEQOUTSZ, 4, 0); + + /* skip assoc data */ + SEQFIFOSTORE(p, SKIP, 0, 0, VLF); + + /* read assoc data */ + SEQFIFOLOAD(p, AAD1, 0, CLASS1 | VLF | FLUSH1); + + SET_LABEL(p, zeroassocjump1); + + /* cryptlen = seqoutlen - assoclen */ + MATHB(p, SEQOUTSZ, SUB, MATH0, VSEQINSZ, 4, 0); + + /* jump to zero-payload command if cryptlen is zero */ + pzeropayloadjump = JUMP(p, zeropayloadjump, LOCAL_JUMP, ALL_TRUE, + MATH_Z); + + MATHB(p, SEQOUTSZ, SUB, MATH0, VSEQOUTSZ, 4, 0); + + /* store encrypted data */ + SEQFIFOSTORE(p, MSG, 0, 0, VLF); + + /* read payload data */ + SEQFIFOLOAD(p, MSG1, 0, CLASS1 | VLF | FLUSH1); + + /* zero-payload command */ + SET_LABEL(p, zeropayloadjump); + + /* read ICV */ + SEQFIFOLOAD(p, ICV1, icvsize, CLASS1 | LAST1); + + PATCH_JUMP(p, pkeyjmp, keyjmp); + PATCH_JUMP(p, pzeroassocjump1, zeroassocjump1); + PATCH_JUMP(p, pzeropayloadjump, zeropayloadjump); + + return PROGRAM_FINALIZE(p); +} + +/** * cnstr_shdsc_crc - CRC32 Accelerator (IEEE 802 CRC32 protocol mode) * @descbuf: pointer to descriptor-under-construction buffer * @swap: must be true when core endianness doesn't match SEC endianness diff --git a/drivers/crypto/dpaa2_sec/hw/desc/common.h b/drivers/crypto/dpaa2_sec/hw/desc/common.h index 6b254908..c2ac99be 100644 --- a/drivers/crypto/dpaa2_sec/hw/desc/common.h +++ b/drivers/crypto/dpaa2_sec/hw/desc/common.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h b/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h index c63d0dac..cc637361 100644 --- a/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h +++ b/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -1311,8 +1311,11 @@ cnstr_shdsc_ipsec_new_decap(uint32_t *descbuf, bool ps, * @descbuf: pointer to buffer used for descriptor construction * @ps: if 36/40bit addressing is desired, this parameter must be true * @swap: if true, perform descriptor byte swapping on a 4-byte boundary - * @cipherdata: ointer to block cipher transform definitions. + * @cipherdata: pointer to block cipher transform definitions. * Valid algorithm values one of OP_ALG_ALGSEL_* {DES, 3DES, AES} + * Valid modes for: + * AES: OP_ALG_AAI_* {CBC, CTR} + * DES, 3DES: OP_ALG_AAI_CBC * @authdata: pointer to authentication transform definitions. * Valid algorithm values - one of OP_ALG_ALGSEL_* {MD5, SHA1, * SHA224, SHA256, SHA384, SHA512} @@ -1379,8 +1382,9 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, { struct program prg; struct program *p = &prg; - const bool is_aes_dec = (dir == DIR_DEC) && - (cipherdata->algtype == OP_ALG_ALGSEL_AES); + const bool need_dk = (dir == DIR_DEC) && + (cipherdata->algtype == OP_ALG_ALGSEL_AES) && + (cipherdata->algmode == OP_ALG_AAI_CBC); LABEL(skip_patch_len); LABEL(keyjmp); @@ -1466,7 +1470,7 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, dir == DIR_ENC ? ICV_CHECK_DISABLE : ICV_CHECK_ENABLE, dir); - if (is_aes_dec) + if (need_dk) ALG_OPERATION(p, OP_ALG_ALGSEL_AES, cipherdata->algmode, OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); pskipkeys = JUMP(p, skipkeys, LOCAL_JUMP, ALL_TRUE, 0); @@ -1478,7 +1482,7 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, dir == DIR_ENC ? ICV_CHECK_DISABLE : ICV_CHECK_ENABLE, dir); - if (is_aes_dec) { + if (need_dk) { ALG_OPERATION(p, OP_ALG_ALGSEL_AES, cipherdata->algmode | OP_ALG_AAI_DK, OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); @@ -1503,7 +1507,10 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, SET_LABEL(p, aonly_len_offset); /* Read IV */ - SEQLOAD(p, CONTEXT1, 0, ivlen, 0); + if (cipherdata->algmode == OP_ALG_AAI_CTR) + SEQLOAD(p, CONTEXT1, 16, ivlen, 0); + else + SEQLOAD(p, CONTEXT1, 0, ivlen, 0); /* * Read data needed only for authentication. This is overwritten above diff --git a/drivers/crypto/dpaa2_sec/hw/rta.h b/drivers/crypto/dpaa2_sec/hw/rta.h index 7cf4c8a9..283ad600 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta.h +++ b/drivers/crypto/dpaa2_sec/hw/rta.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/fifo_load_store_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/fifo_load_store_cmd.h index 79a48da2..dc0a3342 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/fifo_load_store_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/fifo_load_store_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/header_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/header_cmd.h index c2a27a2d..96c9f1e0 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/header_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/header_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/jump_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/jump_cmd.h index 2c85beec..cba65073 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/jump_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/jump_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h index 9bef1155..8e097856 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/load_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/load_cmd.h index 1db55b33..2e786a20 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/load_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/load_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/math_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/math_cmd.h index a9b9091d..64a84b47 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/math_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/math_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/move_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/move_cmd.h index 10d2e193..3b9ba792 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/move_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/move_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/nfifo_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/nfifo_cmd.h index 30be0825..f1ead2da 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/nfifo_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/nfifo_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/operation_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/operation_cmd.h index 5e88fb46..698ff530 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/operation_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/operation_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h index 2e7b2f2d..f134235a 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/protocol_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h b/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h index c12edb0c..11995aa2 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/sec_run_time_asm.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h index 8d421a5d..4aea5770 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/signature_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/signature_cmd.h index ac4f3ff8..00597f30 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/signature_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/signature_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/hw/rta/store_cmd.h b/drivers/crypto/dpaa2_sec/hw/rta/store_cmd.h index 8fd01801..1b75692e 100644 --- a/drivers/crypto/dpaa2_sec/hw/rta/store_cmd.h +++ b/drivers/crypto/dpaa2_sec/hw/rta/store_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2008-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/mc/dpseci.c b/drivers/crypto/dpaa2_sec/mc/dpseci.c index 6b3411f0..4a109620 100644 --- a/drivers/crypto/dpaa2_sec/mc/dpseci.c +++ b/drivers/crypto/dpaa2_sec/mc/dpseci.c @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h index 0fb47f73..6cc14a66 100644 --- a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h +++ b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h index 1fb18cc3..3f9f4743 100644 --- a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h +++ b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright (c) 2016 NXP. + * Copyright 2016 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd.c b/drivers/crypto/kasumi/rte_kasumi_pmd.c index 9da9e897..38cd8a9b 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd.c +++ b/drivers/crypto/kasumi/rte_kasumi_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -47,6 +48,8 @@ #define KASUMI_MAX_BURST 4 #define BYTE_LEN 8 +static uint8_t cryptodev_driver_id; + /** Get xform chain order. */ static enum kasumi_operation kasumi_get_mode(const struct rte_crypto_sym_xform *xform) @@ -108,13 +111,20 @@ kasumi_set_session_parameters(struct kasumi_session *sess, case KASUMI_OP_NOT_SUPPORTED: default: KASUMI_LOG_ERR("Unsupported operation chain order parameter"); - return -EINVAL; + return -ENOTSUP; } if (cipher_xform) { /* Only KASUMI F8 supported */ if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_KASUMI_F8) + return -ENOTSUP; + + sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; + if (cipher_xform->cipher.iv.length != KASUMI_IV_LENGTH) { + KASUMI_LOG_ERR("Wrong IV length"); return -EINVAL; + } + /* Initialize key */ sso_kasumi_init_f8_key_sched(cipher_xform->cipher.key.data, &sess->pKeySched_cipher); @@ -123,8 +133,15 @@ kasumi_set_session_parameters(struct kasumi_session *sess, if (auth_xform) { /* Only KASUMI F9 supported */ if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_KASUMI_F9) + return -ENOTSUP; + + if (auth_xform->auth.digest_length != KASUMI_DIGEST_LENGTH) { + KASUMI_LOG_ERR("Wrong digest length"); return -EINVAL; + } + sess->auth_op = auth_xform->auth.op; + /* Initialize key */ sso_kasumi_init_f9_key_sched(auth_xform->auth.key.data, &sess->pKeySched_hash); @@ -140,27 +157,40 @@ kasumi_set_session_parameters(struct kasumi_session *sess, static struct kasumi_session * kasumi_get_session(struct kasumi_qp *qp, struct rte_crypto_op *op) { - struct kasumi_session *sess; - - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->sym->session->dev_type != - RTE_CRYPTODEV_KASUMI_PMD)) + struct kasumi_session *sess = NULL; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct kasumi_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct kasumi_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct kasumi_session *)c_sess->_private; + sess = (struct kasumi_session *)_sess_private_data; if (unlikely(kasumi_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } @@ -173,17 +203,11 @@ process_kasumi_cipher_op(struct rte_crypto_op **ops, unsigned i; uint8_t processed_ops = 0; uint8_t *src[num_ops], *dst[num_ops]; - uint64_t IV[num_ops]; + uint8_t *iv_ptr; + uint64_t iv[num_ops]; uint32_t num_bytes[num_ops]; for (i = 0; i < num_ops; i++) { - /* Sanity checks. */ - if (ops[i]->sym->cipher.iv.length != KASUMI_IV_LENGTH) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - KASUMI_LOG_ERR("iv"); - break; - } - src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); dst[i] = ops[i]->sym->m_dst ? @@ -191,14 +215,16 @@ process_kasumi_cipher_op(struct rte_crypto_op **ops, (ops[i]->sym->cipher.data.offset >> 3) : rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); - IV[i] = *((uint64_t *)(ops[i]->sym->cipher.iv.data)); + iv_ptr = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->cipher_iv_offset); + iv[i] = *((uint64_t *)(iv_ptr)); num_bytes[i] = ops[i]->sym->cipher.data.length >> 3; processed_ops++; } if (processed_ops != 0) - sso_kasumi_f8_n_buffer(&session->pKeySched_cipher, IV, + sso_kasumi_f8_n_buffer(&session->pKeySched_cipher, iv, src, dst, num_bytes, processed_ops); return processed_ops; @@ -210,16 +236,10 @@ process_kasumi_cipher_op_bit(struct rte_crypto_op *op, struct kasumi_session *session) { uint8_t *src, *dst; - uint64_t IV; + uint8_t *iv_ptr; + uint64_t iv; uint32_t length_in_bits, offset_in_bits; - /* Sanity checks. */ - if (unlikely(op->sym->cipher.iv.length != KASUMI_IV_LENGTH)) { - op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - KASUMI_LOG_ERR("iv"); - return 0; - } - offset_in_bits = op->sym->cipher.data.offset; src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *); if (op->sym->m_dst == NULL) { @@ -228,10 +248,12 @@ process_kasumi_cipher_op_bit(struct rte_crypto_op *op, return 0; } dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *); - IV = *((uint64_t *)(op->sym->cipher.iv.data)); + iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + session->cipher_iv_offset); + iv = *((uint64_t *)(iv_ptr)); length_in_bits = op->sym->cipher.data.length; - sso_kasumi_f8_1_buffer_bit(&session->pKeySched_cipher, IV, + sso_kasumi_f8_1_buffer_bit(&session->pKeySched_cipher, iv, src, dst, length_in_bits, offset_in_bits); return 1; @@ -248,23 +270,8 @@ process_kasumi_hash_op(struct rte_crypto_op **ops, uint8_t *src, *dst; uint32_t length_in_bits; uint32_t num_bytes; - uint32_t shift_bits; - uint64_t IV; - uint8_t direction; for (i = 0; i < num_ops; i++) { - if (unlikely(ops[i]->sym->auth.aad.length != KASUMI_IV_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - KASUMI_LOG_ERR("aad"); - break; - } - - if (unlikely(ops[i]->sym->auth.digest.length != KASUMI_DIGEST_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - KASUMI_LOG_ERR("digest"); - break; - } - /* Data must be byte aligned */ if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; @@ -276,34 +283,28 @@ process_kasumi_hash_op(struct rte_crypto_op **ops, src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->auth.data.offset >> 3); - /* IV from AAD */ - IV = *((uint64_t *)(ops[i]->sym->auth.aad.data)); /* Direction from next bit after end of message */ - num_bytes = (length_in_bits >> 3) + 1; - shift_bits = (BYTE_LEN - 1 - length_in_bits) % BYTE_LEN; - direction = (src[num_bytes - 1] >> shift_bits) & 0x01; + num_bytes = length_in_bits >> 3; if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { dst = (uint8_t *)rte_pktmbuf_append(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + KASUMI_DIGEST_LENGTH); + sso_kasumi_f9_1_buffer(&session->pKeySched_hash, src, + num_bytes, dst); - sso_kasumi_f9_1_buffer_user(&session->pKeySched_hash, - IV, src, - length_in_bits, dst, direction); /* Verify digest. */ if (memcmp(dst, ops[i]->sym->auth.digest.data, - ops[i]->sym->auth.digest.length) != 0) + KASUMI_DIGEST_LENGTH) != 0) ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; /* Trim area used for digest from mbuf. */ rte_pktmbuf_trim(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + KASUMI_DIGEST_LENGTH); } else { dst = ops[i]->sym->auth.digest.data; - sso_kasumi_f9_1_buffer_user(&session->pKeySched_hash, - IV, src, - length_in_bits, dst, direction); + sso_kasumi_f9_1_buffer(&session->pKeySched_hash, src, + num_bytes, dst); } processed_ops++; } @@ -352,7 +353,11 @@ process_ops(struct rte_crypto_op **ops, struct kasumi_session *session, if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (ops[i]->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct kasumi_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -404,8 +409,9 @@ process_op_bit(struct rte_crypto_op *op, struct kasumi_session *session, op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { - rte_mempool_put(qp->sess_mp, op->sym->session); + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(op->sym->session, 0, sizeof(struct kasumi_session)); + rte_cryptodev_sym_session_free(op->sym->session); op->sym->session = NULL; } @@ -566,21 +572,18 @@ cryptodev_kasumi_create(const char *name, /* Check CPU for supported vector instruction set */ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) cpu_flags |= RTE_CRYPTODEV_FF_CPU_AVX; - else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) + else cpu_flags |= RTE_CRYPTODEV_FF_CPU_SSE; - else { - KASUMI_LOG_ERR("Vector instructions are not supported by CPU"); - return -EFAULT; - } - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, - sizeof(struct kasumi_private), init_params->socket_id); + dev = rte_cryptodev_vdev_pmd_init(init_params->name, + sizeof(struct kasumi_private), init_params->socket_id, + vdev); if (dev == NULL) { KASUMI_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_KASUMI_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_kasumi_pmd_ops; /* Register RX/TX burst functions for data path. */ @@ -622,7 +625,7 @@ cryptodev_kasumi_probe(struct rte_vdev_device *vdev) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -664,3 +667,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_KASUMI_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_kasumi_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c b/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c index 62ebdbd2..8033114b 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c +++ b/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -56,11 +56,7 @@ static const struct rte_cryptodev_capabilities kasumi_pmd_capabilities[] = { .max = 4, .increment = 0 }, - .aad_size = { - .min = 8, - .max = 8, - .increment = 0 - } + .iv_size = { 0 } }, } }, } }, @@ -156,7 +152,7 @@ kasumi_pmd_info_get(struct rte_cryptodev *dev, struct kasumi_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; dev_info->feature_flags = dev->feature_flags; @@ -223,7 +219,7 @@ kasumi_pmd_qp_create_processed_ops_ring(struct kasumi_qp *qp, static int kasumi_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct kasumi_qp *qp = NULL; @@ -248,7 +244,7 @@ kasumi_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->processed_ops == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -291,33 +287,56 @@ kasumi_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a KASUMI session from a crypto xform chain */ -static void * +static int kasumi_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + if (unlikely(sess == NULL)) { KASUMI_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (kasumi_set_session_parameters(sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = kasumi_set_session_parameters(sess_private_data, xform); + if (ret != 0) { KASUMI_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -kasumi_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +kasumi_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct kasumi_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct kasumi_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops kasumi_pmd_ops = { diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd_private.h b/drivers/crypto/kasumi/rte_kasumi_pmd_private.h index fb586caa..0ce2a2e3 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd_private.h +++ b/drivers/crypto/kasumi/rte_kasumi_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,9 @@ #include <sso_kasumi.h> +#define CRYPTODEV_NAME_KASUMI_PMD crypto_kasumi +/**< KASUMI PMD device name */ + #define KASUMI_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_KASUMI_PMD), \ @@ -92,6 +95,7 @@ struct kasumi_session { sso_kasumi_key_sched_t pKeySched_hash; enum kasumi_operation op; enum rte_crypto_auth_operation auth_op; + uint16_t cipher_iv_offset; } __rte_cache_aligned; diff --git a/drivers/crypto/null/null_crypto_pmd.c b/drivers/crypto/null/null_crypto_pmd.c index 023450a3..2c827257 100644 --- a/drivers/crypto/null/null_crypto_pmd.c +++ b/drivers/crypto/null/null_crypto_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -33,11 +33,14 @@ #include <rte_common.h> #include <rte_config.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include "null_crypto_pmd_private.h" +static uint8_t cryptodev_driver_id; + /** verify and set session parameters */ int null_crypto_set_session_parameters( @@ -45,7 +48,7 @@ null_crypto_set_session_parameters( const struct rte_crypto_sym_xform *xform) { if (xform == NULL) { - return -1; + return -EINVAL; } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL) { /* Authentication Only */ @@ -70,7 +73,7 @@ null_crypto_set_session_parameters( return 0; } - return -1; + return -ENOTSUP; } /** Process crypto operation for mbuf */ @@ -81,6 +84,14 @@ process_op(const struct null_crypto_qp *qp, struct rte_crypto_op *op, /* set status as successful by default */ op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + /* Free session if a session-less crypto op. */ + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(op->sym->session, 0, + sizeof(struct null_crypto_session)); + rte_cryptodev_sym_session_free(op->sym->session); + op->sym->session = NULL; + } + /* * if crypto session and operation are valid just enqueue the packet * in the processed ring @@ -89,26 +100,37 @@ process_op(const struct null_crypto_qp *qp, struct rte_crypto_op *op, } static struct null_crypto_session * -get_session(struct null_crypto_qp *qp, struct rte_crypto_sym_op *op) +get_session(struct null_crypto_qp *qp, struct rte_crypto_op *op) { - struct null_crypto_session *sess; - - if (op->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->session == NULL || - op->session->dev_type != RTE_CRYPTODEV_NULL_PMD)) + struct null_crypto_session *sess = NULL; + struct rte_crypto_sym_op *sym_op = op->sym; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(sym_op->session != NULL)) + sess = (struct null_crypto_session *) + get_session_private_data( + sym_op->session, cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct null_crypto_session *)op->session->_private; - } else { - struct rte_cryptodev_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct null_crypto_session *)c_sess->_private; - - if (null_crypto_set_session_parameters(sess, op->xform) != 0) - return NULL; + sess = (struct null_crypto_session *)_sess_private_data; + + if (unlikely(null_crypto_set_session_parameters(sess, + sym_op->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + sym_op->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(sym_op->session, cryptodev_driver_id, + _sess_private_data); } return sess; @@ -125,7 +147,7 @@ null_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, int i, retval; for (i = 0; i < nb_ops; i++) { - sess = get_session(qp, ops[i]->sym); + sess = get_session(qp, ops[i]); if (unlikely(sess == NULL)) goto enqueue_err; @@ -166,6 +188,7 @@ static int cryptodev_null_remove(const char *name); /** Create crypto device */ static int cryptodev_null_create(const char *name, + struct rte_vdev_device *vdev, struct rte_crypto_vdev_init_params *init_params) { struct rte_cryptodev *dev; @@ -175,15 +198,16 @@ cryptodev_null_create(const char *name, snprintf(init_params->name, sizeof(init_params->name), "%s", name); - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, + dev = rte_cryptodev_vdev_pmd_init(init_params->name, sizeof(struct null_crypto_private), - init_params->socket_id); + init_params->socket_id, + vdev); if (dev == NULL) { NULL_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_NULL_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = null_crypto_pmd_ops; /* register rx/tx burst functions for data path */ @@ -235,7 +259,7 @@ cryptodev_null_probe(struct rte_vdev_device *dev) RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", init_params.max_nb_sessions); - return cryptodev_null_create(name, &init_params); + return cryptodev_null_create(name, dev, &init_params); } /** Uninitialise null crypto device */ @@ -268,3 +292,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_NULL_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_null_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/null/null_crypto_pmd_ops.c b/drivers/crypto/null/null_crypto_pmd_ops.c index 12c946c9..76153203 100644 --- a/drivers/crypto/null/null_crypto_pmd_ops.c +++ b/drivers/crypto/null/null_crypto_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -56,7 +56,7 @@ static const struct rte_cryptodev_capabilities null_crypto_pmd_capabilities[] = .max = 0, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, }, }, }, }, @@ -72,11 +72,7 @@ static const struct rte_cryptodev_capabilities null_crypto_pmd_capabilities[] = .max = 0, .increment = 0 }, - .iv_size = { - .min = 0, - .max = 0, - .increment = 0 - } + .iv_size = { 0 } }, }, }, } }, @@ -151,7 +147,7 @@ null_crypto_pmd_info_get(struct rte_cryptodev *dev, struct null_crypto_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->max_nb_queue_pairs = internals->max_nb_qpairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; dev_info->feature_flags = dev->feature_flags; @@ -215,7 +211,7 @@ null_crypto_pmd_qp_create_processed_pkts_ring(struct null_crypto_qp *qp, static int null_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct null_crypto_private *internals = dev->data->dev_private; struct null_crypto_qp *qp; @@ -258,7 +254,7 @@ null_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, goto qp_setup_cleanup; } - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -302,33 +298,56 @@ null_crypto_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a null crypto session from a crypto xform chain */ -static void * +static int null_crypto_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mp) { - int retval; + void *sess_private_data; + int ret; if (unlikely(sess == NULL)) { NULL_CRYPTO_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; + } + + if (rte_mempool_get(mp, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; } - retval = null_crypto_set_session_parameters( - (struct null_crypto_session *)sess, xform); - if (retval != 0) { + + ret = null_crypto_set_session_parameters(sess_private_data, xform); + if (ret != 0) { NULL_CRYPTO_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mp, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -null_crypto_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, - void *sess) +null_crypto_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - if (sess) - memset(sess, 0, sizeof(struct null_crypto_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct null_crypto_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops pmd_ops = { diff --git a/drivers/crypto/null/null_crypto_pmd_private.h b/drivers/crypto/null/null_crypto_pmd_private.h index acebc973..4d1c3c9e 100644 --- a/drivers/crypto/null/null_crypto_pmd_private.h +++ b/drivers/crypto/null/null_crypto_pmd_private.h @@ -35,6 +35,9 @@ #include "rte_config.h" +#define CRYPTODEV_NAME_NULL_PMD crypto_null +/**< Null crypto PMD device name */ + #define NULL_CRYPTO_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_NULL_PMD), \ diff --git a/drivers/crypto/openssl/rte_openssl_pmd.c b/drivers/crypto/openssl/rte_openssl_pmd.c index f0c5ca3c..0bd5f98e 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd.c +++ b/drivers/crypto/openssl/rte_openssl_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -34,6 +34,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -44,6 +45,8 @@ #define DES_BLOCK_SIZE 8 +static uint8_t cryptodev_driver_id; + static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); /*----------------------------------------------------------------------------*/ @@ -88,6 +91,8 @@ openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) res = OPENSSL_CHAIN_CIPHER_AUTH; } + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) + res = OPENSSL_CHAIN_COMBINED; } return res; @@ -183,21 +188,6 @@ get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, res = -EINVAL; } break; - case RTE_CRYPTO_CIPHER_AES_GCM: - switch (keylen) { - case 16: - *algo = EVP_aes_128_gcm(); - break; - case 24: - *algo = EVP_aes_192_gcm(); - break; - case 32: - *algo = EVP_aes_256_gcm(); - break; - default: - res = -EINVAL; - } - break; default: res = -EINVAL; break; @@ -253,6 +243,41 @@ get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, return res; } +/** Get adequate openssl function for input cipher algorithm */ +static uint8_t +get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, + const EVP_CIPHER **algo) +{ + int res = 0; + + if (algo != NULL) { + switch (sess_algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + switch (keylen) { + case 16: + *algo = EVP_aes_128_gcm(); + break; + case 24: + *algo = EVP_aes_192_gcm(); + break; + case 32: + *algo = EVP_aes_256_gcm(); + break; + default: + res = -EINVAL; + } + break; + default: + res = -EINVAL; + break; + } + } else { + res = -EINVAL; + } + + return res; +} + /** Set session cipher parameters */ static int openssl_set_session_cipher_parameters(struct openssl_session *sess, @@ -263,12 +288,15 @@ openssl_set_session_cipher_parameters(struct openssl_session *sess, /* Select cipher key */ sess->cipher.key.length = xform->cipher.key.length; + /* Set IV parameters */ + sess->iv.offset = xform->cipher.iv.offset; + sess->iv.length = xform->cipher.iv.length; + /* Select cipher algo */ switch (xform->cipher.algo) { case RTE_CRYPTO_CIPHER_3DES_CBC: case RTE_CRYPTO_CIPHER_AES_CBC: case RTE_CRYPTO_CIPHER_AES_CTR: - case RTE_CRYPTO_CIPHER_AES_GCM: sess->cipher.mode = OPENSSL_CIPHER_LIB; sess->cipher.algo = xform->cipher.algo; sess->cipher.ctx = EVP_CIPHER_CTX_new(); @@ -308,7 +336,7 @@ openssl_set_session_cipher_parameters(struct openssl_session *sess, break; default: sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; - return -EINVAL; + return -ENOTSUP; } return 0; @@ -326,11 +354,33 @@ openssl_set_session_auth_parameters(struct openssl_session *sess, /* Select auth algo */ switch (xform->auth.algo) { case RTE_CRYPTO_AUTH_AES_GMAC: - case RTE_CRYPTO_AUTH_AES_GCM: - /* Check additional condition for AES_GMAC/GCM */ - if (sess->cipher.algo != RTE_CRYPTO_CIPHER_AES_GCM) - return -EINVAL; sess->chain_order = OPENSSL_CHAIN_COMBINED; + + /* Set IV parameters */ + sess->iv.offset = xform->auth.iv.offset; + sess->iv.length = xform->auth.iv.length; + + /* + * OpenSSL requires GMAC to be a GCM operation + * with no cipher data length + */ + sess->cipher.mode = OPENSSL_CIPHER_LIB; + if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) + sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + else + sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; + + sess->cipher.key.length = xform->auth.key.length; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + + if (get_aead_algo(RTE_CRYPTO_AEAD_AES_GCM, + sess->cipher.key.length, + &sess->cipher.evp_algo) != 0) + return -EINVAL; + + get_cipher_key(xform->auth.key.data, xform->auth.key.length, + sess->cipher.key.data); + break; case RTE_CRYPTO_AUTH_MD5: @@ -362,9 +412,55 @@ openssl_set_session_auth_parameters(struct openssl_session *sess, break; default: - return -EINVAL; + return -ENOTSUP; } + sess->auth.digest_length = xform->auth.digest_length; + + return 0; +} + +/* Set session AEAD parameters */ +static int +openssl_set_session_aead_parameters(struct openssl_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + /* Select cipher direction */ + sess->cipher.direction = xform->cipher.op; + /* Select cipher key */ + sess->cipher.key.length = xform->aead.key.length; + + /* Set IV parameters */ + sess->iv.offset = xform->aead.iv.offset; + sess->iv.length = xform->aead.iv.length; + + /* Select auth generate/verify */ + sess->auth.operation = xform->auth.op; + sess->auth.algo = xform->auth.algo; + + /* Select auth algo */ + switch (xform->aead.algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + sess->cipher.mode = OPENSSL_CIPHER_LIB; + sess->aead_algo = xform->aead.algo; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + + if (get_aead_algo(sess->aead_algo, sess->cipher.key.length, + &sess->cipher.evp_algo) != 0) + return -EINVAL; + + get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, + sess->cipher.key.data); + + sess->chain_order = OPENSSL_CHAIN_COMBINED; + break; + default: + return -ENOTSUP; + } + + sess->auth.aad_length = xform->aead.aad_length; + sess->auth.digest_length = xform->aead.digest_length; + return 0; } @@ -375,6 +471,8 @@ openssl_set_session_parameters(struct openssl_session *sess, { const struct rte_crypto_sym_xform *cipher_xform = NULL; const struct rte_crypto_sym_xform *auth_xform = NULL; + const struct rte_crypto_sym_xform *aead_xform = NULL; + int ret; sess->chain_order = openssl_get_chain_order(xform); switch (sess->chain_order) { @@ -392,25 +490,42 @@ openssl_set_session_parameters(struct openssl_session *sess, auth_xform = xform; cipher_xform = xform->next; break; + case OPENSSL_CHAIN_COMBINED: + aead_xform = xform; + break; default: return -EINVAL; } + /* Default IV length = 0 */ + sess->iv.length = 0; + /* cipher_xform must be check before auth_xform */ if (cipher_xform) { - if (openssl_set_session_cipher_parameters( - sess, cipher_xform)) { + ret = openssl_set_session_cipher_parameters( + sess, cipher_xform); + if (ret != 0) { OPENSSL_LOG_ERR( "Invalid/unsupported cipher parameters"); - return -EINVAL; + return ret; } } if (auth_xform) { - if (openssl_set_session_auth_parameters(sess, auth_xform)) { + ret = openssl_set_session_auth_parameters(sess, auth_xform); + if (ret != 0) { OPENSSL_LOG_ERR( "Invalid/unsupported auth parameters"); - return -EINVAL; + return ret; + } + } + + if (aead_xform) { + ret = openssl_set_session_aead_parameters(sess, aead_xform); + if (ret != 0) { + OPENSSL_LOG_ERR( + "Invalid/unsupported AEAD parameters"); + return ret; } } @@ -445,29 +560,35 @@ get_session(struct openssl_qp *qp, struct rte_crypto_op *op) { struct openssl_session *sess = NULL; - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { /* get existing session */ - if (likely(op->sym->session != NULL && - op->sym->session->dev_type == - RTE_CRYPTODEV_OPENSSL_PMD)) + if (likely(op->sym->session != NULL)) sess = (struct openssl_session *) - op->sym->session->_private; - } else { + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { /* provide internal session */ void *_sess = NULL; + void *_sess_private_data = NULL; - if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { - sess = (struct openssl_session *) - ((struct rte_cryptodev_sym_session *)_sess) - ->_private; - - if (unlikely(openssl_set_session_parameters( - sess, op->sym->xform) != 0)) { - rte_mempool_put(qp->sess_mp, _sess); - sess = NULL; - } else - op->sym->session = _sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct openssl_session *)_sess_private_data; + + if (unlikely(openssl_set_session_parameters(sess, + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } if (sess == NULL) @@ -912,6 +1033,7 @@ process_openssl_combined_op /* cipher */ uint8_t *dst = NULL, *iv, *tag, *aad; int srclen, ivlen, aadlen, status = -1; + uint32_t offset; /* * Segmented destination buffer is not supported for @@ -922,34 +1044,41 @@ process_openssl_combined_op return; } - iv = op->sym->cipher.iv.data; - ivlen = op->sym->cipher.iv.length; - aad = op->sym->auth.aad.data; - aadlen = op->sym->auth.aad.length; - - tag = op->sym->auth.digest.data; - if (tag == NULL) - tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, - op->sym->cipher.data.offset + - op->sym->cipher.data.length); - - if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); + ivlen = sess->iv.length; + if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { srclen = 0; - else { - srclen = op->sym->cipher.data.length; + offset = op->sym->auth.data.offset; + aadlen = op->sym->auth.data.length; + aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, + op->sym->auth.data.offset); + tag = op->sym->auth.digest.data; + if (tag == NULL) + tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, + offset + aadlen); + } else { + srclen = op->sym->aead.data.length; dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, - op->sym->cipher.data.offset); + op->sym->aead.data.offset); + offset = op->sym->aead.data.offset; + aad = op->sym->aead.aad.data; + aadlen = sess->auth.aad_length; + tag = op->sym->aead.digest.data; + if (tag == NULL) + tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, + offset + srclen); } if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) status = process_openssl_auth_encryption_gcm( - mbuf_src, op->sym->cipher.data.offset, srclen, + mbuf_src, offset, srclen, aad, aadlen, iv, ivlen, sess->cipher.key.data, dst, tag, sess->cipher.ctx, sess->cipher.evp_algo); else status = process_openssl_auth_decryption_gcm( - mbuf_src, op->sym->cipher.data.offset, srclen, + mbuf_src, offset, srclen, aad, aadlen, iv, ivlen, sess->cipher.key.data, dst, tag, sess->cipher.ctx, sess->cipher.evp_algo); @@ -986,7 +1115,8 @@ process_openssl_cipher_op dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, op->sym->cipher.data.offset); - iv = op->sym->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); if (sess->cipher.mode == OPENSSL_CIPHER_LIB) if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) @@ -1027,7 +1157,8 @@ process_openssl_docsis_bpi_op(struct rte_crypto_op *op, dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, op->sym->cipher.data.offset); - iv = op->sym->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); block_size = DES_BLOCK_SIZE; @@ -1085,7 +1216,8 @@ process_openssl_docsis_bpi_op(struct rte_crypto_op *op, dst, iv, last_block_len, sess->cipher.bpi_ctx); /* Prepare parameters for CBC mode op */ - iv = op->sym->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->iv.offset); dst += last_block_len - srclen; srclen -= last_block_len; } @@ -1116,7 +1248,7 @@ process_openssl_auth_op if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) dst = (uint8_t *)rte_pktmbuf_append(mbuf_src, - op->sym->auth.digest.length); + sess->auth.digest_length); else { dst = op->sym->auth.digest.data; if (dst == NULL) @@ -1144,11 +1276,11 @@ process_openssl_auth_op if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { if (memcmp(dst, op->sym->auth.digest.data, - op->sym->auth.digest.length) != 0) { + sess->auth.digest_length) != 0) { op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } /* Trim area used for digest from mbuf. */ - rte_pktmbuf_trim(mbuf_src, op->sym->auth.digest.length); + rte_pktmbuf_trim(mbuf_src, sess->auth.digest_length); } if (status != 0) @@ -1195,9 +1327,12 @@ process_op(const struct openssl_qp *qp, struct rte_crypto_op *op, } /* Free session if a session-less crypto op */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { openssl_reset_session(sess); memset(sess, 0, sizeof(struct openssl_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } @@ -1275,15 +1410,16 @@ cryptodev_openssl_create(const char *name, snprintf(init_params->name, sizeof(init_params->name), "%s", name); - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, + dev = rte_cryptodev_vdev_pmd_init(init_params->name, sizeof(struct openssl_private), - init_params->socket_id); + init_params->socket_id, + vdev); if (dev == NULL) { OPENSSL_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_OPENSSL_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_openssl_pmd_ops; /* register rx/tx burst functions for data path */ @@ -1329,7 +1465,7 @@ cryptodev_openssl_probe(struct rte_vdev_device *vdev) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -1372,3 +1508,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_openssl_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/openssl/rte_openssl_pmd_ops.c b/drivers/crypto/openssl/rte_openssl_pmd_ops.c index 22a68730..8cdd0b2e 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd_ops.c +++ b/drivers/crypto/openssl/rte_openssl_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -48,16 +48,16 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_MD5_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 16, .max = 16, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -78,7 +78,7 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .max = 16, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -90,16 +90,16 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 20, .max = 20, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -120,7 +120,7 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .max = 20, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -132,16 +132,16 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 28, .max = 28, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -162,7 +162,7 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .max = 28, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -174,40 +174,40 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, .block_size = 64, .key_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 32, .max = 32, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, { /* SHA256 */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { - .algo = RTE_CRYPTO_AUTH_SHA256, - .block_size = 64, - .key_size = { - .min = 0, - .max = 0, - .increment = 0 - }, - .digest_size = { - .min = 32, - .max = 32, - .increment = 0 - }, - .aad_size = { 0 } - }, } + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 32, + .max = 32, + .increment = 0 + }, + .iv_size = { 0 } }, } - }, + }, } + }, { /* SHA384 HMAC */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { @@ -216,16 +216,16 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 48, .max = 48, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -246,7 +246,7 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .max = 48, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -258,16 +258,16 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, .block_size = 128, .key_size = { - .min = 128, + .min = 1, .max = 128, - .increment = 0 + .increment = 1 }, .digest_size = { .min = 64, .max = 64, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -288,7 +288,7 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .max = 64, .increment = 0 }, - .aad_size = { 0 } + .iv_size = { 0 } }, } }, } }, @@ -332,12 +332,12 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { }, } }, } }, - { /* AES GCM (AUTH) */ + { /* AES GCM */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, - {.auth = { - .algo = RTE_CRYPTO_AUTH_AES_GCM, + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, .block_size = 16, .key_size = { .min = 16, @@ -353,27 +353,12 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .min = 0, .max = 65535, .increment = 1 - } - }, } - }, } - }, - { /* AES GCM (CIPHER) */ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, - {.sym = { - .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, - {.cipher = { - .algo = RTE_CRYPTO_CIPHER_AES_GCM, - .block_size = 16, - .key_size = { - .min = 16, - .max = 32, - .increment = 8 }, .iv_size = { .min = 12, .max = 16, .increment = 4 - } + }, }, } }, } }, @@ -394,9 +379,9 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { .max = 16, .increment = 0 }, - .aad_size = { - .min = 8, - .max = 65532, + .iv_size = { + .min = 12, + .max = 16, .increment = 4 } }, } @@ -536,7 +521,7 @@ openssl_pmd_info_get(struct rte_cryptodev *dev, struct openssl_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->feature_flags = dev->feature_flags; dev_info->capabilities = openssl_pmd_capabilities; dev_info->max_nb_queue_pairs = internals->max_nb_qpairs; @@ -602,7 +587,7 @@ openssl_pmd_qp_create_processed_ops_ring(struct openssl_qp *qp, static int openssl_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct openssl_qp *qp = NULL; @@ -627,7 +612,7 @@ openssl_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->processed_ops == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->stats, 0, sizeof(qp->stats)); @@ -671,36 +656,57 @@ openssl_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure the session from a crypto xform chain */ -static void * +static int openssl_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + if (unlikely(sess == NULL)) { OPENSSL_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; } - if (openssl_set_session_parameters( - sess, xform) != 0) { + ret = openssl_set_session_parameters(sess_private_data, xform); + if (ret != 0) { OPENSSL_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -openssl_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +openssl_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) { - openssl_reset_session(sess); - memset(sess, 0, sizeof(struct openssl_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + openssl_reset_session(sess_priv); + memset(sess_priv, 0, sizeof(struct openssl_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); } } diff --git a/drivers/crypto/openssl/rte_openssl_pmd_private.h b/drivers/crypto/openssl/rte_openssl_pmd_private.h index 4d820c51..b7f74752 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd_private.h +++ b/drivers/crypto/openssl/rte_openssl_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -36,6 +36,8 @@ #include <openssl/evp.h> #include <openssl/des.h> +#define CRYPTODEV_NAME_OPENSSL_PMD crypto_openssl +/**< Open SSL Crypto PMD device name */ #define OPENSSL_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ @@ -108,6 +110,15 @@ struct openssl_session { enum openssl_chain_order chain_order; /**< chain order mode */ + struct { + uint16_t length; + uint16_t offset; + } iv; + /**< IV parameters */ + + enum rte_crypto_aead_algorithm aead_algo; + /**< AEAD algorithm */ + /** Cipher Parameters */ struct { enum rte_crypto_cipher_operation direction; @@ -157,6 +168,11 @@ struct openssl_session { /**< pointer to EVP context structure */ } hmac; }; + + uint16_t aad_length; + /**< AAD length */ + uint16_t digest_length; + /**< digest length */ } auth; } __rte_cache_aligned; diff --git a/drivers/crypto/qat/qat_adf/qat_algs.h b/drivers/crypto/qat/qat_adf/qat_algs.h index 5c63406b..2c8e03c0 100644 --- a/drivers/crypto/qat/qat_adf/qat_algs.h +++ b/drivers/crypto/qat/qat_adf/qat_algs.h @@ -17,7 +17,7 @@ * qat-linux@intel.com * * BSD LICENSE - * Copyright(c) 2015-2016 Intel Corporation. + * Copyright(c) 2015-2017 Intel Corporation. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -51,6 +51,7 @@ #include "icp_qat_hw.h" #include "icp_qat_fw.h" #include "icp_qat_fw_la.h" +#include "../qat_crypto.h" /* * Key Modifier (KM) value used in KASUMI algorithm in F9 mode to XOR @@ -127,15 +128,17 @@ struct qat_session { struct icp_qat_fw_la_bulk_req fw_req; uint8_t aad_len; struct qat_crypto_instance *inst; + struct { + uint16_t offset; + uint16_t length; + } cipher_iv; + struct { + uint16_t offset; + uint16_t length; + } auth_iv; + uint16_t digest_length; rte_spinlock_t lock; /* protects this struct */ -}; - -struct qat_alg_ablkcipher_cd { - struct icp_qat_hw_cipher_algo_blk *cd; - phys_addr_t cd_paddr; - struct icp_qat_fw_la_bulk_req fw_req; - struct qat_crypto_instance *inst; - rte_spinlock_t lock; /* protects this struct */ + enum qat_device_gen min_qat_dev_gen; }; int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg); @@ -147,21 +150,13 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cd, int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, uint8_t *authkey, uint32_t authkeylen, - uint32_t add_auth_data_length, + uint32_t aad_length, uint32_t digestsize, unsigned int operation); void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, enum qat_crypto_proto_flag proto_flags); -void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_cd *cd, - int alg, const uint8_t *key, - unsigned int keylen); - -void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_cd *cd, - int alg, const uint8_t *key, - unsigned int keylen); - int qat_alg_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg); int qat_alg_validate_aes_docsisbpi_key(int key_len, enum icp_qat_hw_cipher_algo *alg); diff --git a/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c b/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c index 154e1ddd..2d16c9e2 100644 --- a/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c +++ b/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c @@ -17,7 +17,7 @@ * qat-linux@intel.com * * BSD LICENSE - * Copyright(c) 2015-2016 Intel Corporation. + * Copyright(c) 2015-2017 Intel Corporation. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -121,6 +121,9 @@ static int qat_hash_get_state1_size(enum icp_qat_hw_auth_algo qat_hash_alg) case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9: return QAT_HW_ROUND_UP(ICP_QAT_HW_KASUMI_F9_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); + case ICP_QAT_HW_AUTH_ALGO_NULL: + return QAT_HW_ROUND_UP(ICP_QAT_HW_NULL_STATE1_SZ, + QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_DELIMITER: /* return maximum state1 size in this case */ return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA512_STATE1_SZ, @@ -603,6 +606,7 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc, cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3; qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC; + cdesc->min_qat_dev_gen = QAT_GEN2; } else { total_key_size = cipherkeylen; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_AES_BLK_SZ >> 3; @@ -661,7 +665,7 @@ int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc, int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, uint8_t *authkey, uint32_t authkeylen, - uint32_t add_auth_data_length, + uint32_t aad_length, uint32_t digestsize, unsigned int operation) { @@ -810,13 +814,14 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, * in big-endian format. This field is 8 bytes */ auth_param->u2.aad_sz = - RTE_ALIGN_CEIL(add_auth_data_length, 16); + RTE_ALIGN_CEIL(aad_length, 16); auth_param->hash_state_sz = (auth_param->u2.aad_sz) >> 3; aad_len = (uint32_t *)(cdesc->cd_cur_ptr + ICP_QAT_HW_GALOIS_128_STATE1_SZ + ICP_QAT_HW_GALOIS_H_SZ); - *aad_len = rte_bswap32(add_auth_data_length); + *aad_len = rte_bswap32(aad_length); + cdesc->aad_len = aad_length; break; case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2: qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G; @@ -837,8 +842,7 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, 0, ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ); cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) + authkeylen + ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; - auth_param->hash_state_sz = - RTE_ALIGN_CEIL(add_auth_data_length, 16) >> 3; + auth_param->hash_state_sz = ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ >> 3; break; case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3: hash->auth_config.config = @@ -854,8 +858,8 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen); cdesc->cd_cur_ptr += state1_size + state2_size + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ; - auth_param->hash_state_sz = - RTE_ALIGN_CEIL(add_auth_data_length, 16) >> 3; + auth_param->hash_state_sz = ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3; + cdesc->min_qat_dev_gen = QAT_GEN2; break; case ICP_QAT_HW_AUTH_ALGO_MD5: @@ -868,6 +872,9 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, state2_size = ICP_QAT_HW_MD5_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_NULL: + state1_size = qat_hash_get_state1_size( + ICP_QAT_HW_AUTH_ALGO_NULL); + state2_size = ICP_QAT_HW_NULL_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9: state1_size = qat_hash_get_state1_size( diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c index 386aa453..1f52cabf 100644 --- a/drivers/crypto/qat/qat_crypto.c +++ b/drivers/crypto/qat/qat_crypto.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -60,6 +60,7 @@ #include <rte_spinlock.h> #include <rte_hexdump.h> #include <rte_crypto_sym.h> +#include <rte_cryptodev_pci.h> #include <openssl/evp.h> #include "qat_logs.h" @@ -170,16 +171,19 @@ cipher_decrypt_err: /** Creates a context in either AES or DES in ECB mode * Depends on openssl libcrypto */ -static void * +static int bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo, enum rte_crypto_cipher_operation direction __rte_unused, - uint8_t *key) + uint8_t *key, void **ctx) { const EVP_CIPHER *algo = NULL; - EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + int ret; + *ctx = EVP_CIPHER_CTX_new(); - if (ctx == NULL) + if (*ctx == NULL) { + ret = -ENOMEM; goto ctx_init_err; + } if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI) algo = EVP_des_ecb(); @@ -187,15 +191,17 @@ bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo, algo = EVP_aes_128_ecb(); /* IV will be ECB encrypted whether direction is encrypt or decrypt*/ - if (EVP_EncryptInit_ex(ctx, algo, NULL, key, 0) != 1) + if (EVP_EncryptInit_ex(*ctx, algo, NULL, key, 0) != 1) { + ret = -EINVAL; goto ctx_init_err; + } - return ctx; + return 0; ctx_init_err: - if (ctx != NULL) - EVP_CIPHER_CTX_free(ctx); - return NULL; + if (*ctx != NULL) + EVP_CIPHER_CTX_free(*ctx); + return ret; } /** Frees a context previously created @@ -213,25 +219,25 @@ adf_modulo(uint32_t data, uint32_t shift); static inline int qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, - struct qat_crypto_op_cookie *qat_op_cookie); + struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp); -void qat_crypto_sym_clear_session(struct rte_cryptodev *dev, - void *session) +void +qat_crypto_sym_clear_session(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - struct qat_session *sess = session; - phys_addr_t cd_paddr; - PMD_INIT_FUNC_TRACE(); - if (sess) { - if (sess->bpi_ctx) { - bpi_cipher_ctx_free(sess->bpi_ctx); - sess->bpi_ctx = NULL; - } - cd_paddr = sess->cd_paddr; - memset(sess, 0, qat_crypto_sym_get_session_private_size(dev)); - sess->cd_paddr = cd_paddr; - } else - PMD_DRV_LOG(ERR, "NULL session"); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + struct qat_session *s = (struct qat_session *)sess_priv; + + if (sess_priv) { + if (s->bpi_ctx) + bpi_cipher_ctx_free(s->bpi_ctx); + memset(s, 0, qat_crypto_sym_get_session_private_size(dev)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } static int @@ -245,6 +251,14 @@ qat_get_cmd_id(const struct rte_crypto_sym_xform *xform) if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL) return ICP_QAT_FW_LA_CMD_AUTH; + /* AEAD */ + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) + return ICP_QAT_FW_LA_CMD_CIPHER_HASH; + else + return ICP_QAT_FW_LA_CMD_HASH_CIPHER; + } + if (xform->next == NULL) return -1; @@ -286,38 +300,37 @@ qat_get_cipher_xform(struct rte_crypto_sym_xform *xform) return NULL; } -void * + +int qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *session_private) + struct rte_crypto_sym_xform *xform, + struct qat_session *session) { - struct qat_session *session = session_private; struct qat_pmd_private *internals = dev->data->dev_private; struct rte_crypto_cipher_xform *cipher_xform = NULL; + int ret; /* Get cipher xform from crypto xform chain */ cipher_xform = qat_get_cipher_xform(xform); + session->cipher_iv.offset = cipher_xform->iv.offset; + session->cipher_iv.length = cipher_xform->iv.length; + switch (cipher_xform->algo) { case RTE_CRYPTO_CIPHER_AES_CBC: if (qat_alg_validate_aes_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; break; - case RTE_CRYPTO_CIPHER_AES_GCM: - if (qat_alg_validate_aes_key(cipher_xform->key.length, - &session->qat_cipher_alg) != 0) { - PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); - goto error_out; - } - session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; - break; case RTE_CRYPTO_CIPHER_AES_CTR: if (qat_alg_validate_aes_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; @@ -326,6 +339,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_snow3g_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid SNOW 3G cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE; @@ -337,6 +351,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_kasumi_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid KASUMI cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_F8_MODE; @@ -345,6 +360,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_3des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -353,6 +369,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -361,38 +378,43 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_3des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; break; case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: - session->bpi_ctx = bpi_cipher_ctx_init( + ret = bpi_cipher_ctx_init( cipher_xform->algo, cipher_xform->op, - cipher_xform->key.data); - if (session->bpi_ctx == NULL) { + cipher_xform->key.data, + &session->bpi_ctx); + if (ret != 0) { PMD_DRV_LOG(ERR, "failed to create DES BPI ctx"); goto error_out; } if (qat_alg_validate_des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; break; case RTE_CRYPTO_CIPHER_AES_DOCSISBPI: - session->bpi_ctx = bpi_cipher_ctx_init( + ret = bpi_cipher_ctx_init( cipher_xform->algo, cipher_xform->op, - cipher_xform->key.data); - if (session->bpi_ctx == NULL) { + cipher_xform->key.data, + &session->bpi_ctx); + if (ret != 0) { PMD_DRV_LOG(ERR, "failed to create AES BPI ctx"); goto error_out; } if (qat_alg_validate_aes_docsisbpi_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES DOCSISBPI key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -403,27 +425,30 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, PMD_DRV_LOG(ERR, "%s not supported on this device", rte_crypto_cipher_algorithm_strings [cipher_xform->algo]); + ret = -ENOTSUP; goto error_out; } if (qat_alg_validate_zuc_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid ZUC cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE; break; case RTE_CRYPTO_CIPHER_3DES_ECB: case RTE_CRYPTO_CIPHER_AES_ECB: - case RTE_CRYPTO_CIPHER_AES_CCM: case RTE_CRYPTO_CIPHER_AES_F8: case RTE_CRYPTO_CIPHER_AES_XTS: case RTE_CRYPTO_CIPHER_ARC4: PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported Cipher alg %u", cipher_xform->algo); + ret = -ENOTSUP; goto error_out; default: PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n", cipher_xform->algo); + ret = -EINVAL; goto error_out; } @@ -434,50 +459,119 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_aead_session_create_content_desc_cipher(session, cipher_xform->key.data, - cipher_xform->key.length)) + cipher_xform->key.length)) { + ret = -EINVAL; goto error_out; + } - return session; + return 0; error_out: if (session->bpi_ctx) { bpi_cipher_ctx_free(session->bpi_ctx); session->bpi_ctx = NULL; } - return NULL; + return ret; } - -void * +int qat_crypto_sym_configure_session(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = qat_crypto_set_session_parameters(dev, xform, sess_private_data); + if (ret != 0) { + PMD_DRV_LOG(ERR, "Crypto QAT PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; +} + +int +qat_crypto_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private) { struct qat_session *session = session_private; + int ret; int qat_cmd_id; PMD_INIT_FUNC_TRACE(); + /* Set context descriptor physical address */ + session->cd_paddr = rte_mempool_virt2phy(NULL, session) + + offsetof(struct qat_session, cd); + + session->min_qat_dev_gen = QAT_GEN1; + /* Get requested QAT command id */ qat_cmd_id = qat_get_cmd_id(xform); if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) { PMD_DRV_LOG(ERR, "Unsupported xform chain requested"); - goto error_out; + return -ENOTSUP; } session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id; switch (session->qat_cmd) { case ICP_QAT_FW_LA_CMD_CIPHER: - session = qat_crypto_sym_configure_session_cipher(dev, xform, session); + ret = qat_crypto_sym_configure_session_cipher(dev, xform, session); + if (ret < 0) + return ret; break; case ICP_QAT_FW_LA_CMD_AUTH: - session = qat_crypto_sym_configure_session_auth(dev, xform, session); + ret = qat_crypto_sym_configure_session_auth(dev, xform, session); + if (ret < 0) + return ret; break; case ICP_QAT_FW_LA_CMD_CIPHER_HASH: - session = qat_crypto_sym_configure_session_cipher(dev, xform, session); - session = qat_crypto_sym_configure_session_auth(dev, xform, session); + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + ret = qat_crypto_sym_configure_session_aead(xform, + session); + if (ret < 0) + return ret; + } else { + ret = qat_crypto_sym_configure_session_cipher(dev, + xform, session); + if (ret < 0) + return ret; + ret = qat_crypto_sym_configure_session_auth(dev, + xform, session); + if (ret < 0) + return ret; + } break; case ICP_QAT_FW_LA_CMD_HASH_CIPHER: - session = qat_crypto_sym_configure_session_auth(dev, xform, session); - session = qat_crypto_sym_configure_session_cipher(dev, xform, session); + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + ret = qat_crypto_sym_configure_session_aead(xform, + session); + if (ret < 0) + return ret; + } else { + ret = qat_crypto_sym_configure_session_auth(dev, + xform, session); + if (ret < 0) + return ret; + ret = qat_crypto_sym_configure_session_cipher(dev, + xform, session); + if (ret < 0) + return ret; + } break; case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM: case ICP_QAT_FW_LA_CMD_TRNG_TEST: @@ -490,30 +584,26 @@ qat_crypto_sym_configure_session(struct rte_cryptodev *dev, case ICP_QAT_FW_LA_CMD_DELIMITER: PMD_DRV_LOG(ERR, "Unsupported Service %u", session->qat_cmd); - goto error_out; + return -ENOTSUP; default: PMD_DRV_LOG(ERR, "Unsupported Service %u", session->qat_cmd); - goto error_out; + return -ENOTSUP; } - return session; - -error_out: - return NULL; + return 0; } -struct qat_session * +int qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, - struct qat_session *session_private) + struct qat_session *session) { - - struct qat_session *session = session_private; struct rte_crypto_auth_xform *auth_xform = NULL; - struct rte_crypto_cipher_xform *cipher_xform = NULL; struct qat_pmd_private *internals = dev->data->dev_private; auth_xform = qat_get_auth_xform(xform); + uint8_t *key_data = auth_xform->key.data; + uint8_t key_length = auth_xform->key.length; switch (auth_xform->algo) { case RTE_CRYPTO_AUTH_SHA1_HMAC: @@ -534,11 +624,15 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, case RTE_CRYPTO_AUTH_AES_XCBC_MAC: session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC; break; - case RTE_CRYPTO_AUTH_AES_GCM: - session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; - break; case RTE_CRYPTO_AUTH_AES_GMAC: + if (qat_alg_validate_aes_key(auth_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid AES key size"); + return -EINVAL; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; + break; case RTE_CRYPTO_AUTH_SNOW3G_UIA2: session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2; @@ -557,7 +651,7 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, PMD_DRV_LOG(ERR, "%s not supported on this device", rte_crypto_auth_algorithm_strings [auth_xform->algo]); - goto error_out; + return -ENOTSUP; } session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3; break; @@ -567,42 +661,149 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, case RTE_CRYPTO_AUTH_SHA224: case RTE_CRYPTO_AUTH_SHA384: case RTE_CRYPTO_AUTH_MD5: - case RTE_CRYPTO_AUTH_AES_CCM: case RTE_CRYPTO_AUTH_AES_CMAC: case RTE_CRYPTO_AUTH_AES_CBC_MAC: PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u", auth_xform->algo); - goto error_out; + return -ENOTSUP; default: PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified", auth_xform->algo); - goto error_out; + return -EINVAL; } - cipher_xform = qat_get_cipher_xform(xform); - if ((session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) || - (session->qat_hash_alg == - ICP_QAT_HW_AUTH_ALGO_GALOIS_64)) { + session->auth_iv.offset = auth_xform->iv.offset; + session->auth_iv.length = auth_xform->iv.length; + + if (auth_xform->algo == RTE_CRYPTO_AUTH_AES_GMAC) { + if (auth_xform->op == RTE_CRYPTO_AUTH_OP_GENERATE) { + session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER_HASH; + session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; + /* + * It needs to create cipher desc content first, + * then authentication + */ + if (qat_alg_aead_session_create_content_desc_cipher(session, + auth_xform->key.data, + auth_xform->key.length)) + return -EINVAL; + + if (qat_alg_aead_session_create_content_desc_auth(session, + key_data, + key_length, + 0, + auth_xform->digest_length, + auth_xform->op)) + return -EINVAL; + } else { + session->qat_cmd = ICP_QAT_FW_LA_CMD_HASH_CIPHER; + session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; + /* + * It needs to create authentication desc content first, + * then cipher + */ + if (qat_alg_aead_session_create_content_desc_auth(session, + key_data, + key_length, + 0, + auth_xform->digest_length, + auth_xform->op)) + return -EINVAL; + + if (qat_alg_aead_session_create_content_desc_cipher(session, + auth_xform->key.data, + auth_xform->key.length)) + return -EINVAL; + } + /* Restore to authentication only only */ + session->qat_cmd = ICP_QAT_FW_LA_CMD_AUTH; + } else { if (qat_alg_aead_session_create_content_desc_auth(session, - cipher_xform->key.data, - cipher_xform->key.length, - auth_xform->add_auth_data_length, + key_data, + key_length, + 0, auth_xform->digest_length, auth_xform->op)) - goto error_out; + return -EINVAL; + } + + session->digest_length = auth_xform->digest_length; + return 0; +} + +int +qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, + struct qat_session *session) +{ + struct rte_crypto_aead_xform *aead_xform = &xform->aead; + + /* + * Store AEAD IV parameters as cipher IV, + * to avoid unnecessary memory usage + */ + session->cipher_iv.offset = xform->aead.iv.offset; + session->cipher_iv.length = xform->aead.iv.length; + + switch (aead_xform->algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + if (qat_alg_validate_aes_key(aead_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid AES key size"); + return -EINVAL; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; + break; + case RTE_CRYPTO_AEAD_AES_CCM: + PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported AEAD alg %u", + aead_xform->algo); + return -ENOTSUP; + default: + PMD_DRV_LOG(ERR, "Crypto: Undefined AEAD specified %u\n", + aead_xform->algo); + return -EINVAL; + } + + if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { + session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; + /* + * It needs to create cipher desc content first, + * then authentication + */ + if (qat_alg_aead_session_create_content_desc_cipher(session, + aead_xform->key.data, + aead_xform->key.length)) + return -EINVAL; + + if (qat_alg_aead_session_create_content_desc_auth(session, + aead_xform->key.data, + aead_xform->key.length, + aead_xform->aad_length, + aead_xform->digest_length, + RTE_CRYPTO_AUTH_OP_GENERATE)) + return -EINVAL; } else { + session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; + /* + * It needs to create authentication desc content first, + * then cipher + */ if (qat_alg_aead_session_create_content_desc_auth(session, - auth_xform->key.data, - auth_xform->key.length, - auth_xform->add_auth_data_length, - auth_xform->digest_length, - auth_xform->op)) - goto error_out; + aead_xform->key.data, + aead_xform->key.length, + aead_xform->aad_length, + aead_xform->digest_length, + RTE_CRYPTO_AUTH_OP_VERIFY)) + return -EINVAL; + + if (qat_alg_aead_session_create_content_desc_cipher(session, + aead_xform->key.data, + aead_xform->key.length)) + return -EINVAL; } - return session; -error_out: - return NULL; + session->digest_length = aead_xform->digest_length; + return 0; } unsigned qat_crypto_sym_get_session_private_size( @@ -617,7 +818,8 @@ qat_bpicipher_preprocess(struct qat_session *ctx, { uint8_t block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); struct rte_crypto_sym_op *sym_op = op->sym; - uint8_t last_block_len = sym_op->cipher.data.length % block_len; + uint8_t last_block_len = block_len > 0 ? + sym_op->cipher.data.length % block_len : 0; if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) { @@ -641,7 +843,8 @@ qat_bpicipher_preprocess(struct qat_session *ctx, iv = last_block - block_len; else /* runt block, i.e. less than one full block */ - iv = sym_op->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + ctx->cipher_iv.offset); #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX rte_hexdump(stdout, "BPI: src before pre-process:", last_block, @@ -670,7 +873,8 @@ qat_bpicipher_postprocess(struct qat_session *ctx, { uint8_t block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); struct rte_crypto_sym_op *sym_op = op->sym; - uint8_t last_block_len = sym_op->cipher.data.length % block_len; + uint8_t last_block_len = block_len > 0 ? + sym_op->cipher.data.length % block_len : 0; if (last_block_len > 0 && ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) { @@ -696,7 +900,8 @@ qat_bpicipher_postprocess(struct qat_session *ctx, iv = dst - block_len; else /* runt block, i.e. less than one full block */ - iv = sym_op->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + ctx->cipher_iv.offset); #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX rte_hexdump(stdout, "BPI: src before post-process:", last_block, @@ -752,12 +957,12 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, while (nb_ops_sent != nb_ops_possible) { ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail, - tmp_qp->op_cookies[tail / queue->msg_size]); + tmp_qp->op_cookies[tail / queue->msg_size], tmp_qp); if (ret != 0) { tmp_qp->stats.enqueue_err_count++; /* * This message cannot be enqueued, - * decrease number of ops that wasnt sent + * decrease number of ops that wasn't sent */ rte_atomic16_sub(&tmp_qp->inflights16, nb_ops_possible - nb_ops_sent); @@ -808,7 +1013,10 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } else { struct qat_session *sess = (struct qat_session *) - (rx_op->sym->session->_private); + get_session_private_data( + rx_op->sym->session, + cryptodev_qat_driver_id); + if (sess->bpi_ctx) qat_bpicipher_postprocess(sess, rx_op); rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; @@ -882,23 +1090,44 @@ qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start, return 0; } +static inline void +set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, + struct icp_qat_fw_la_cipher_req_params *cipher_param, + struct rte_crypto_op *op, + struct icp_qat_fw_la_bulk_req *qat_req) +{ + /* copy IV into request if it fits */ + if (iv_length <= sizeof(cipher_param->u.cipher_IV_array)) { + rte_memcpy(cipher_param->u.cipher_IV_array, + rte_crypto_op_ctod_offset(op, uint8_t *, + iv_offset), + iv_length); + } else { + ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET( + qat_req->comn_hdr.serv_specif_flags, + ICP_QAT_FW_CIPH_IV_64BIT_PTR); + cipher_param->u.s.cipher_IV_ptr = + rte_crypto_op_ctophys_offset(op, + iv_offset); + } +} + static inline int qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, - struct qat_crypto_op_cookie *qat_op_cookie) + struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp) { int ret = 0; struct qat_session *ctx; struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; register struct icp_qat_fw_la_bulk_req *qat_req; - uint8_t do_auth = 0, do_cipher = 0; + uint8_t do_auth = 0, do_cipher = 0, do_aead = 0; uint32_t cipher_len = 0, cipher_ofs = 0; uint32_t auth_len = 0, auth_ofs = 0; uint32_t min_ofs = 0; uint64_t src_buf_start = 0, dst_buf_start = 0; uint8_t do_sgl = 0; - #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) { PMD_DRV_LOG(ERR, "QAT PMD only supports symmetric crypto " @@ -907,18 +1136,26 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, return -EINVAL; } #endif - if (unlikely(op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS)) { + if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) { PMD_DRV_LOG(ERR, "QAT PMD only supports session oriented" " requests, op (%p) is sessionless.", op); return -EINVAL; } - if (unlikely(op->sym->session->dev_type != RTE_CRYPTODEV_QAT_SYM_PMD)) { + ctx = (struct qat_session *)get_session_private_data( + op->sym->session, cryptodev_qat_driver_id); + + if (unlikely(ctx == NULL)) { PMD_DRV_LOG(ERR, "Session was not created for this device"); return -EINVAL; } - ctx = (struct qat_session *)op->sym->session->_private; + if (unlikely(ctx->min_qat_dev_gen > qp->qat_dev_gen)) { + PMD_DRV_LOG(ERR, "Session alg not supported on this device gen"); + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return -EINVAL; + } + qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; @@ -926,9 +1163,15 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param)); if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER || - ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { - do_auth = 1; - do_cipher = 1; + ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { + /* AES-GCM */ + if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || + ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { + do_aead = 1; + } else { + do_auth = 1; + do_cipher = 1; + } } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) { do_auth = 1; do_cipher = 0; @@ -970,26 +1213,8 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, cipher_ofs = op->sym->cipher.data.offset; } - /* copy IV into request if it fits */ - /* - * If IV length is zero do not copy anything but still - * use request descriptor embedded IV - * - */ - if (op->sym->cipher.iv.length) { - if (op->sym->cipher.iv.length <= - sizeof(cipher_param->u.cipher_IV_array)) { - rte_memcpy(cipher_param->u.cipher_IV_array, - op->sym->cipher.iv.data, - op->sym->cipher.iv.length); - } else { - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET( - qat_req->comn_hdr.serv_specif_flags, - ICP_QAT_FW_CIPH_IV_64BIT_PTR); - cipher_param->u.s.cipher_IV_ptr = - op->sym->cipher.iv.phys_addr; - } - } + set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset, + cipher_param, op, qat_req); min_ofs = cipher_ofs; } @@ -1009,34 +1234,70 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, auth_ofs = op->sym->auth.data.offset >> 3; auth_len = op->sym->auth.data.length >> 3; - if (ctx->qat_hash_alg == - ICP_QAT_HW_AUTH_ALGO_KASUMI_F9) { - if (do_cipher) { - auth_len = auth_len + auth_ofs + 1 - - ICP_QAT_HW_KASUMI_BLK_SZ; - auth_ofs = ICP_QAT_HW_KASUMI_BLK_SZ; - } else { - auth_len = auth_len + auth_ofs + 1; - auth_ofs = 0; - } - } + auth_param->u1.aad_adr = + rte_crypto_op_ctophys_offset(op, + ctx->auth_iv.offset); } else if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { - auth_ofs = op->sym->cipher.data.offset; - auth_len = op->sym->cipher.data.length; + /* AES-GMAC */ + set_cipher_iv(ctx->auth_iv.length, + ctx->auth_iv.offset, + cipher_param, op, qat_req); + auth_ofs = op->sym->auth.data.offset; + auth_len = op->sym->auth.data.length; + + auth_param->u1.aad_adr = 0; + auth_param->u2.aad_sz = 0; + + /* + * If len(iv)==12B fw computes J0 + */ + if (ctx->auth_iv.length == 12) { + ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( + qat_req->comn_hdr.serv_specif_flags, + ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); + + } } else { auth_ofs = op->sym->auth.data.offset; auth_len = op->sym->auth.data.length; + } min_ofs = auth_ofs; auth_param->auth_res_addr = op->sym->auth.digest.phys_addr; - auth_param->u1.aad_adr = op->sym->auth.aad.phys_addr; + } + if (do_aead) { + if (ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || + ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { + /* + * If len(iv)==12B fw computes J0 + */ + if (ctx->cipher_iv.length == 12) { + ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( + qat_req->comn_hdr.serv_specif_flags, + ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); + } + + } + + cipher_len = op->sym->aead.data.length; + cipher_ofs = op->sym->aead.data.offset; + auth_len = op->sym->aead.data.length; + auth_ofs = op->sym->aead.data.offset; + + auth_param->u1.aad_adr = op->sym->aead.aad.phys_addr; + auth_param->auth_res_addr = op->sym->aead.digest.phys_addr; + set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset, + cipher_param, op, qat_req); + min_ofs = op->sym->aead.data.offset; } if (op->sym->m_src->next || (op->sym->m_dst && op->sym->m_dst->next)) @@ -1080,7 +1341,7 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, dst_buf_start = src_buf_start; } - if (do_cipher) { + if (do_cipher || do_aead) { cipher_param->cipher_offset = (uint32_t)rte_pktmbuf_mtophys_offset( op->sym->m_src, cipher_ofs) - src_buf_start; @@ -1089,7 +1350,8 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, cipher_param->cipher_offset = 0; cipher_param->cipher_length = 0; } - if (do_auth) { + + if (do_auth || do_aead) { auth_param->auth_off = (uint32_t)rte_pktmbuf_mtophys_offset( op->sym->m_src, auth_ofs) - src_buf_start; auth_param->auth_len = auth_len; @@ -1097,6 +1359,7 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, auth_param->auth_off = 0; auth_param->auth_len = 0; } + qat_req->comn_mid.dst_length = qat_req->comn_mid.src_length = (cipher_param->cipher_offset + cipher_param->cipher_length) @@ -1142,48 +1405,38 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, qat_req->comn_mid.dest_data_addr = dst_buf_start; } - if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || - ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { - if (op->sym->cipher.iv.length == 12) { - /* - * For GCM a 12 bit IV is allowed, - * but we need to inform the f/w - */ - ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( - qat_req->comn_hdr.serv_specif_flags, - ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); - } - if (op->sym->cipher.data.length == 0) { - /* - * GMAC - */ - qat_req->comn_mid.dest_data_addr = - qat_req->comn_mid.src_data_addr = - op->sym->auth.aad.phys_addr; - qat_req->comn_mid.dst_length = - qat_req->comn_mid.src_length = - rte_pktmbuf_data_len(op->sym->m_src); - cipher_param->cipher_length = 0; - cipher_param->cipher_offset = 0; - auth_param->u1.aad_adr = 0; - auth_param->auth_len = op->sym->auth.aad.length; - auth_param->auth_off = op->sym->auth.data.offset; - auth_param->u2.aad_sz = 0; - } - } - #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX rte_hexdump(stdout, "qat_req:", qat_req, sizeof(struct icp_qat_fw_la_bulk_req)); rte_hexdump(stdout, "src_data:", rte_pktmbuf_mtod(op->sym->m_src, uint8_t*), rte_pktmbuf_data_len(op->sym->m_src)); - rte_hexdump(stdout, "iv:", op->sym->cipher.iv.data, - op->sym->cipher.iv.length); - rte_hexdump(stdout, "digest:", op->sym->auth.digest.data, - op->sym->auth.digest.length); - rte_hexdump(stdout, "aad:", op->sym->auth.aad.data, - op->sym->auth.aad.length); + if (do_cipher) { + uint8_t *cipher_iv_ptr = rte_crypto_op_ctod_offset(op, + uint8_t *, + ctx->cipher_iv.offset); + rte_hexdump(stdout, "cipher iv:", cipher_iv_ptr, + ctx->cipher_iv.length); + } + + if (do_auth) { + if (ctx->auth_iv.length) { + uint8_t *auth_iv_ptr = rte_crypto_op_ctod_offset(op, + uint8_t *, + ctx->auth_iv.offset); + rte_hexdump(stdout, "auth iv:", auth_iv_ptr, + ctx->auth_iv.length); + } + rte_hexdump(stdout, "digest:", op->sym->auth.digest.data, + ctx->digest_length); + } + + if (do_aead) { + rte_hexdump(stdout, "digest:", op->sym->aead.digest.data, + ctx->digest_length); + rte_hexdump(stdout, "aad:", op->sym->aead.aad.data, + ctx->aad_len); + } #endif return 0; } @@ -1196,17 +1449,6 @@ static inline uint32_t adf_modulo(uint32_t data, uint32_t shift) return data - mult; } -void qat_crypto_sym_session_init(struct rte_mempool *mp, void *sym_sess) -{ - struct rte_cryptodev_sym_session *sess = sym_sess; - struct qat_session *s = (void *)sess->_private; - - PMD_INIT_FUNC_TRACE(); - s->cd_paddr = rte_mempool_virt2phy(mp, sess) + - offsetof(struct qat_session, cd) + - offsetof(struct rte_cryptodev_sym_session, _private); -} - int qat_dev_config(__rte_unused struct rte_cryptodev *dev, __rte_unused struct rte_cryptodev_config *config) { @@ -1240,8 +1482,8 @@ int qat_dev_close(struct rte_cryptodev *dev) return 0; } -void qat_dev_info_get(__rte_unused struct rte_cryptodev *dev, - struct rte_cryptodev_info *info) +void qat_dev_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *info) { struct qat_pmd_private *internals = dev->data->dev_private; @@ -1253,7 +1495,8 @@ void qat_dev_info_get(__rte_unused struct rte_cryptodev *dev, info->feature_flags = dev->feature_flags; info->capabilities = internals->qat_dev_capabilities; info->sym.max_nb_sessions = internals->max_nb_sessions; - info->dev_type = RTE_CRYPTODEV_QAT_SYM_PMD; + info->driver_id = cryptodev_qat_driver_id; + info->pci_dev = RTE_DEV_TO_PCI(dev->device); } } diff --git a/drivers/crypto/qat/qat_crypto.h b/drivers/crypto/qat/qat_crypto.h index b740d6b0..3f35a00e 100644 --- a/drivers/crypto/qat/qat_crypto.h +++ b/drivers/crypto/qat/qat_crypto.h @@ -39,6 +39,9 @@ #include "qat_crypto_capabilities.h" +#define CRYPTODEV_NAME_QAT_SYM_PMD crypto_qat +/**< Intel QAT Symmetric Crypto PMD device name */ + /* * This macro rounds up a number to a be a multiple of * the alignment when the alignment is a power of 2 @@ -47,6 +50,13 @@ (((num) + (align) - 1) & ~((align) - 1)) #define QAT_64_BTYE_ALIGN_MASK (~0x3f) +struct qat_session; + +enum qat_device_gen { + QAT_GEN1 = 1, + QAT_GEN2, +}; + /** * Structure associated with each queue. */ @@ -74,6 +84,7 @@ struct qat_qp { struct rte_mempool *op_cookie_pool; void **op_cookies; uint32_t nb_descriptors; + enum qat_device_gen qat_dev_gen; } __rte_cache_aligned; /** private data structure for each QAT device */ @@ -82,9 +93,13 @@ struct qat_pmd_private { /**< Max number of queue pairs supported by device */ unsigned max_nb_sessions; /**< Max number of sessions supported by device */ + enum qat_device_gen qat_dev_gen; + /**< QAT device generation */ const struct rte_cryptodev_capabilities *qat_dev_capabilities; }; +extern uint8_t cryptodev_qat_driver_id; + int qat_dev_config(struct rte_cryptodev *dev, struct rte_cryptodev_config *config); int qat_dev_start(struct rte_cryptodev *dev); @@ -98,7 +113,8 @@ void qat_crypto_sym_stats_get(struct rte_cryptodev *dev, void qat_crypto_sym_stats_reset(struct rte_cryptodev *dev); int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, - const struct rte_cryptodev_qp_conf *rx_conf, int socket_id); + const struct rte_cryptodev_qp_conf *rx_conf, int socket_id, + struct rte_mempool *session_pool); int qat_crypto_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id); @@ -109,26 +125,35 @@ qat_pmd_session_mempool_create(struct rte_cryptodev *dev, extern unsigned qat_crypto_sym_get_session_private_size(struct rte_cryptodev *dev); -extern void -qat_crypto_sym_session_init(struct rte_mempool *mempool, void *priv_sess); - -extern void * +extern int qat_crypto_sym_configure_session(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool); + + +int +qat_crypto_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private); -struct qat_session * +int +qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, + struct qat_session *session); + +int qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, - struct qat_session *session_private); + struct qat_session *session); -void * +int qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *session_private); + struct rte_crypto_sym_xform *xform, + struct qat_session *session); extern void -qat_crypto_sym_clear_session(struct rte_cryptodev *dev, void *session); - +qat_crypto_sym_clear_session(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *session); extern uint16_t qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, diff --git a/drivers/crypto/qat/qat_crypto_capabilities.h b/drivers/crypto/qat/qat_crypto_capabilities.h index 1294f247..70120072 100644 --- a/drivers/crypto/qat/qat_crypto_capabilities.h +++ b/drivers/crypto/qat/qat_crypto_capabilities.h @@ -34,7 +34,7 @@ #ifndef _QAT_CRYPTO_CAPABILITIES_H_ #define _QAT_CRYPTO_CAPABILITIES_H_ -#define QAT_BASE_CPM16_SYM_CAPABILITIES \ +#define QAT_BASE_GEN1_SYM_CAPABILITIES \ { /* SHA1 HMAC */ \ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ {.sym = { \ @@ -43,16 +43,16 @@ .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, \ .block_size = 64, \ .key_size = { \ - .min = 64, \ + .min = 1, \ .max = 64, \ - .increment = 0 \ + .increment = 1 \ }, \ .digest_size = { \ .min = 20, \ .max = 20, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -64,16 +64,16 @@ .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, \ .block_size = 64, \ .key_size = { \ - .min = 64, \ + .min = 1, \ .max = 64, \ - .increment = 0 \ + .increment = 1 \ }, \ .digest_size = { \ .min = 28, \ .max = 28, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -85,16 +85,16 @@ .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, \ .block_size = 64, \ .key_size = { \ - .min = 64, \ + .min = 1, \ .max = 64, \ - .increment = 0 \ + .increment = 1 \ }, \ .digest_size = { \ .min = 32, \ .max = 32, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -104,18 +104,18 @@ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ {.auth = { \ .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, \ - .block_size = 64, \ + .block_size = 128, \ .key_size = { \ - .min = 128, \ + .min = 1, \ .max = 128, \ - .increment = 0 \ + .increment = 1 \ }, \ .digest_size = { \ .min = 48, \ .max = 48, \ .increment = 0 \ - }, \ - .aad_size = { 0 } \ + }, \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -127,16 +127,16 @@ .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, \ .block_size = 128, \ .key_size = { \ - .min = 128, \ + .min = 1, \ .max = 128, \ - .increment = 0 \ + .increment = 1 \ }, \ .digest_size = { \ .min = 64, \ .max = 64, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -148,16 +148,16 @@ .algo = RTE_CRYPTO_AUTH_MD5_HMAC, \ .block_size = 64, \ .key_size = { \ - .min = 8, \ + .min = 1, \ .max = 64, \ - .increment = 8 \ + .increment = 1 \ }, \ .digest_size = { \ .min = 16, \ .max = 16, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -178,16 +178,17 @@ .max = 16, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .aad_size = { 0 }, \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ - { /* AES GCM (AUTH) */ \ + { /* AES GCM */ \ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ {.sym = { \ - .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \ - {.auth = { \ - .algo = RTE_CRYPTO_AUTH_AES_GCM, \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, \ + {.aead = { \ + .algo = RTE_CRYPTO_AEAD_AES_GCM, \ .block_size = 16, \ .key_size = { \ .min = 16, \ @@ -203,7 +204,12 @@ .min = 0, \ .max = 240, \ .increment = 1 \ - } \ + }, \ + .iv_size = { \ + .min = 12, \ + .max = 12, \ + .increment = 0 \ + }, \ }, } \ }, } \ }, \ @@ -224,10 +230,10 @@ .max = 16, \ .increment = 4 \ }, \ - .aad_size = { \ - .min = 1, \ - .max = 65535, \ - .increment = 1 \ + .iv_size = { \ + .min = 12, \ + .max = 12, \ + .increment = 0 \ } \ }, } \ }, } \ @@ -249,7 +255,7 @@ .max = 4, \ .increment = 0 \ }, \ - .aad_size = { \ + .iv_size = { \ .min = 16, \ .max = 16, \ .increment = 0 \ @@ -257,26 +263,6 @@ }, } \ }, } \ }, \ - { /* AES GCM (CIPHER) */ \ - .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ - {.sym = { \ - .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \ - {.cipher = { \ - .algo = RTE_CRYPTO_CIPHER_AES_GCM, \ - .block_size = 16, \ - .key_size = { \ - .min = 16, \ - .max = 32, \ - .increment = 8 \ - }, \ - .iv_size = { \ - .min = 12, \ - .max = 12, \ - .increment = 0 \ - } \ - }, } \ - }, } \ - }, \ { /* AES CBC */ \ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ {.sym = { \ @@ -374,7 +360,7 @@ .max = 0, \ .increment = 0 \ }, \ - .aad_size = { 0 } \ + .iv_size = { 0 } \ }, }, \ }, }, \ }, \ @@ -435,11 +421,7 @@ .max = 4, \ .increment = 0 \ }, \ - .aad_size = { \ - .min = 8, \ - .max = 8, \ - .increment = 0 \ - } \ + .iv_size = { 0 } \ }, } \ }, } \ }, \ @@ -524,7 +506,7 @@ }, } \ } -#define QAT_EXTRA_CPM17_SYM_CAPABILITIES \ +#define QAT_EXTRA_GEN2_SYM_CAPABILITIES \ { /* ZUC (EEA3) */ \ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ {.sym = { \ @@ -562,7 +544,7 @@ .max = 4, \ .increment = 0 \ }, \ - .aad_size = { \ + .iv_size = { \ .min = 16, \ .max = 16, \ .increment = 0 \ diff --git a/drivers/crypto/qat/qat_qp.c b/drivers/crypto/qat/qat_qp.c index a358ccd7..5048d214 100644 --- a/drivers/crypto/qat/qat_qp.c +++ b/drivers/crypto/qat/qat_qp.c @@ -36,6 +36,7 @@ #include <rte_malloc.h> #include <rte_memzone.h> #include <rte_cryptodev_pmd.h> +#include <rte_pci.h> #include <rte_atomic.h> #include <rte_prefetch.h> @@ -133,7 +134,7 @@ queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size, int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool __rte_unused) { struct qat_qp *qp; struct rte_pci_device *pci_dev; @@ -205,7 +206,7 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, adf_configure_queues(qp); adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr); snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE, "%s_qp_op_%d_%hu", - dev->driver->pci_drv.driver.name, dev->data->dev_id, + pci_dev->driver->driver.name, dev->data->dev_id, queue_pair_id); qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name); @@ -242,6 +243,11 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, offsetof(struct qat_crypto_op_cookie, qat_sgl_list_dst); } + + struct qat_pmd_private *internals + = dev->data->dev_private; + qp->qat_dev_gen = internals->qat_dev_gen; + dev->data->queue_pairs[queue_pair_id] = qp; return 0; @@ -355,11 +361,13 @@ qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue, return -EINVAL; } + pci_dev = RTE_DEV_TO_PCI(dev->device); + /* * Allocate a memzone for the queue - create a unique name. */ snprintf(queue->memz_name, sizeof(queue->memz_name), "%s_%s_%d_%d_%d", - dev->driver->pci_drv.driver.name, "qp_mem", dev->data->dev_id, + pci_dev->driver->driver.name, "qp_mem", dev->data->dev_id, queue->hw_bundle_number, queue->hw_queue_number); qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes, socket_id); @@ -408,7 +416,6 @@ qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue, queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr, queue->queue_size); - pci_dev = RTE_DEV_TO_PCI(dev->device); io_addr = pci_dev->mem_resource[0].addr; diff --git a/drivers/crypto/qat/rte_qat_cryptodev.c b/drivers/crypto/qat/rte_qat_cryptodev.c index 1bdd30da..7d56fca4 100644 --- a/drivers/crypto/qat/rte_qat_cryptodev.c +++ b/drivers/crypto/qat/rte_qat_cryptodev.c @@ -35,18 +35,21 @@ #include <rte_dev.h> #include <rte_malloc.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_pci.h> #include "qat_crypto.h" #include "qat_logs.h" -static const struct rte_cryptodev_capabilities qat_cpm16_capabilities[] = { - QAT_BASE_CPM16_SYM_CAPABILITIES, +uint8_t cryptodev_qat_driver_id; + +static const struct rte_cryptodev_capabilities qat_gen1_capabilities[] = { + QAT_BASE_GEN1_SYM_CAPABILITIES, RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; -static const struct rte_cryptodev_capabilities qat_cpm17_capabilities[] = { - QAT_BASE_CPM16_SYM_CAPABILITIES, - QAT_EXTRA_CPM17_SYM_CAPABILITIES, +static const struct rte_cryptodev_capabilities qat_gen2_capabilities[] = { + QAT_BASE_GEN1_SYM_CAPABILITIES, + QAT_EXTRA_GEN2_SYM_CAPABILITIES, RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; @@ -70,7 +73,6 @@ static struct rte_cryptodev_ops crypto_qat_ops = { /* Crypto related operations */ .session_get_size = qat_crypto_sym_get_session_private_size, .session_configure = qat_crypto_sym_configure_session, - .session_initialize = qat_crypto_sym_session_init, .session_clear = qat_crypto_sym_clear_session }; @@ -95,8 +97,7 @@ static const struct rte_pci_id pci_id_qat_map[] = { }; static int -crypto_qat_dev_init(__attribute__((unused)) struct rte_cryptodev_driver *crypto_drv, - struct rte_cryptodev *cryptodev) +crypto_qat_dev_init(struct rte_cryptodev *cryptodev) { struct qat_pmd_private *internals; @@ -106,7 +107,7 @@ crypto_qat_dev_init(__attribute__((unused)) struct rte_cryptodev_driver *crypto_ RTE_DEV_TO_PCI(cryptodev->device)->addr.devid, RTE_DEV_TO_PCI(cryptodev->device)->addr.function); - cryptodev->dev_type = RTE_CRYPTODEV_QAT_SYM_PMD; + cryptodev->driver_id = cryptodev_qat_driver_id; cryptodev->dev_ops = &crypto_qat_ops; cryptodev->enqueue_burst = qat_pmd_enqueue_op_burst; @@ -121,12 +122,14 @@ crypto_qat_dev_init(__attribute__((unused)) struct rte_cryptodev_driver *crypto_ internals->max_nb_sessions = RTE_QAT_PMD_MAX_NB_SESSIONS; switch (RTE_DEV_TO_PCI(cryptodev->device)->id.device_id) { case 0x0443: - internals->qat_dev_capabilities = qat_cpm16_capabilities; + internals->qat_dev_gen = QAT_GEN1; + internals->qat_dev_capabilities = qat_gen1_capabilities; break; case 0x37c9: case 0x19e3: case 0x6f55: - internals->qat_dev_capabilities = qat_cpm17_capabilities; + internals->qat_dev_gen = QAT_GEN2; + internals->qat_dev_capabilities = qat_gen2_capabilities; break; default: PMD_DRV_LOG(ERR, @@ -147,17 +150,25 @@ crypto_qat_dev_init(__attribute__((unused)) struct rte_cryptodev_driver *crypto_ return 0; } -static struct rte_cryptodev_driver rte_qat_pmd = { - .pci_drv = { - .id_table = pci_id_qat_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING, - .probe = rte_cryptodev_pci_probe, - .remove = rte_cryptodev_pci_remove, - }, - .cryptodev_init = crypto_qat_dev_init, - .dev_private_size = sizeof(struct qat_pmd_private), +static int crypto_qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return rte_cryptodev_pci_generic_probe(pci_dev, + sizeof(struct qat_pmd_private), crypto_qat_dev_init); +} + +static int crypto_qat_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_cryptodev_pci_generic_remove(pci_dev, NULL); +} + +static struct rte_pci_driver rte_qat_pmd = { + .id_table = pci_id_qat_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = crypto_qat_pci_probe, + .remove = crypto_qat_pci_remove }; -RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_QAT_SYM_PMD, rte_qat_pmd.pci_drv); +RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_QAT_SYM_PMD, rte_qat_pmd); RTE_PMD_REGISTER_PCI_TABLE(CRYPTODEV_NAME_QAT_SYM_PMD, pci_id_qat_map); - +RTE_PMD_REGISTER_CRYPTO_DRIVER(rte_qat_pmd, cryptodev_qat_driver_id); diff --git a/drivers/crypto/scheduler/Makefile b/drivers/crypto/scheduler/Makefile index c273e784..b045410c 100644 --- a/drivers/crypto/scheduler/Makefile +++ b/drivers/crypto/scheduler/Makefile @@ -56,5 +56,6 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += rte_cryptodev_scheduler.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_roundrobin.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_pkt_size_distr.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_failover.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler_multicore.c include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/crypto/scheduler/rte_cryptodev_scheduler.c b/drivers/crypto/scheduler/rte_cryptodev_scheduler.c index 319dcf0a..df8634ad 100644 --- a/drivers/crypto/scheduler/rte_cryptodev_scheduler.c +++ b/drivers/crypto/scheduler/rte_cryptodev_scheduler.c @@ -198,7 +198,7 @@ rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id) return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -226,12 +226,12 @@ rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id) rte_cryptodev_info_get(slave_id, &dev_info); slave->dev_id = slave_id; - slave->dev_type = dev_info.dev_type; + slave->driver_id = dev_info.driver_id; sched_ctx->nb_slaves++; if (update_scheduler_capability(sched_ctx) < 0) { slave->dev_id = 0; - slave->dev_type = 0; + slave->driver_id = 0; sched_ctx->nb_slaves--; CS_LOG_ERR("capabilities update failed"); @@ -257,7 +257,7 @@ rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id) return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -314,7 +314,7 @@ rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id, return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -351,6 +351,13 @@ rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id, return -1; } break; + case CDEV_SCHED_MODE_MULTICORE: + if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, + multicore_scheduler) < 0) { + CS_LOG_ERR("Failed to load scheduler"); + return -1; + } + break; default: CS_LOG_ERR("Not yet supported"); return -ENOTSUP; @@ -359,13 +366,6 @@ rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id, return 0; } -int -rte_crpytodev_scheduler_mode_set(uint8_t scheduler_id, - enum rte_cryptodev_scheduler_mode mode) -{ - return rte_cryptodev_scheduler_mode_set(scheduler_id, mode); -} - enum rte_cryptodev_scheduler_mode rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id) { @@ -377,7 +377,7 @@ rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id) return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -387,12 +387,6 @@ rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id) return sched_ctx->mode; } -enum rte_cryptodev_scheduler_mode -rte_crpytodev_scheduler_mode_get(uint8_t scheduler_id) -{ - return rte_cryptodev_scheduler_mode_get(scheduler_id); -} - int rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id, uint32_t enable_reorder) @@ -405,7 +399,7 @@ rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id, return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -433,7 +427,7 @@ rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id) return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -455,7 +449,7 @@ rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -467,8 +461,22 @@ rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, sched_ctx = dev->data->dev_private; + if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) { + CS_LOG_ERR("Invalid name %s, should be less than " + "%u bytes.\n", scheduler->name, + RTE_CRYPTODEV_NAME_MAX_LEN); + return -EINVAL; + } strncpy(sched_ctx->name, scheduler->name, RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN); + + if (strlen(scheduler->description) > + RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) { + CS_LOG_ERR("Invalid description %s, should be less than " + "%u bytes.\n", scheduler->description, + RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1); + return -EINVAL; + } strncpy(sched_ctx->description, scheduler->description, RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN); @@ -512,7 +520,7 @@ rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves) return -ENOTSUP; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } @@ -580,7 +588,7 @@ rte_cryptodev_scheduler_option_get(uint8_t scheduler_id, return -EINVAL; } - if (dev->dev_type != RTE_CRYPTODEV_SCHEDULER_PMD) { + if (dev->driver_id != cryptodev_driver_id) { CS_LOG_ERR("Operation not supported"); return -ENOTSUP; } diff --git a/drivers/crypto/scheduler/rte_cryptodev_scheduler.h b/drivers/crypto/scheduler/rte_cryptodev_scheduler.h index 2ba6e470..df22f2a9 100644 --- a/drivers/crypto/scheduler/rte_cryptodev_scheduler.h +++ b/drivers/crypto/scheduler/rte_cryptodev_scheduler.h @@ -58,12 +58,17 @@ extern "C" { #define RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES (8) #endif +/** Maximum number of multi-core worker cores */ +#define RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES (64) + /** Round-robin scheduling mode string */ #define SCHEDULER_MODE_NAME_ROUND_ROBIN round-robin /** Packet-size based distribution scheduling mode string */ #define SCHEDULER_MODE_NAME_PKT_SIZE_DISTR packet-size-distr /** Fail-over scheduling mode string */ #define SCHEDULER_MODE_NAME_FAIL_OVER fail-over +/** multi-core scheduling mode string */ +#define SCHEDULER_MODE_NAME_MULTI_CORE multi-core /** * Crypto scheduler PMD operation modes @@ -78,6 +83,8 @@ enum rte_cryptodev_scheduler_mode { CDEV_SCHED_MODE_PKT_SIZE_DISTR, /** Fail-over mode */ CDEV_SCHED_MODE_FAILOVER, + /** multi-core mode */ + CDEV_SCHED_MODE_MULTICORE, CDEV_SCHED_MODE_COUNT /**< number of modes */ }; @@ -116,6 +123,7 @@ struct rte_cryptodev_scheduler; * - 0 if the scheduler is successfully loaded * - -ENOTSUP if the operation is not supported. * - -EBUSY if device is started. + * - -EINVAL if input values are invalid. */ int rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, @@ -186,38 +194,6 @@ enum rte_cryptodev_scheduler_mode rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id); /** - * @deprecated - * Set the scheduling mode - * - * @param scheduler_id - * The target scheduler device ID - * @param mode - * The scheduling mode - * - * @return - * 0 if attaching successful, negative integer if otherwise. - */ -__rte_deprecated -int -rte_crpytodev_scheduler_mode_set(uint8_t scheduler_id, - enum rte_cryptodev_scheduler_mode mode); - -/** - * @deprecated - * Get the current scheduling mode - * - * @param scheduler_id - * The target scheduler device ID - * - * @return - * If successful, returns the scheduling mode, negative integer - * otherwise - */ -__rte_deprecated -enum rte_cryptodev_scheduler_mode -rte_crpytodev_scheduler_mode_get(uint8_t scheduler_id); - -/** * Set the crypto ops reordering feature on/off * * @param scheduler_id @@ -327,6 +303,8 @@ extern struct rte_cryptodev_scheduler *roundrobin_scheduler; extern struct rte_cryptodev_scheduler *pkt_size_based_distr_scheduler; /** Fail-over mode scheduler */ extern struct rte_cryptodev_scheduler *failover_scheduler; +/** multi-core mode scheduler */ +extern struct rte_cryptodev_scheduler *multicore_scheduler; #ifdef __cplusplus } diff --git a/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map b/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map index 0a8b4716..5c43127c 100644 --- a/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map +++ b/drivers/crypto/scheduler/rte_pmd_crypto_scheduler_version.map @@ -4,8 +4,6 @@ DPDK_17.02 { rte_cryptodev_scheduler_load_user_scheduler; rte_cryptodev_scheduler_slave_attach; rte_cryptodev_scheduler_slave_detach; - rte_crpytodev_scheduler_mode_set; - rte_crpytodev_scheduler_mode_get; rte_cryptodev_scheduler_ordering_set; rte_cryptodev_scheduler_ordering_get; diff --git a/drivers/crypto/scheduler/scheduler_failover.c b/drivers/crypto/scheduler/scheduler_failover.c index 2471a5f1..2aa13f8e 100644 --- a/drivers/crypto/scheduler/scheduler_failover.c +++ b/drivers/crypto/scheduler/scheduler_failover.c @@ -48,58 +48,19 @@ struct fo_scheduler_qp_ctx { uint8_t deq_idx; }; -static inline uint16_t __attribute__((always_inline)) -failover_slave_enqueue(struct scheduler_slave *slave, uint8_t slave_idx, +static __rte_always_inline uint16_t +failover_slave_enqueue(struct scheduler_slave *slave, struct rte_crypto_op **ops, uint16_t nb_ops) { uint16_t i, processed_ops; - struct rte_cryptodev_sym_session *sessions[nb_ops]; - struct scheduler_session *sess0, *sess1, *sess2, *sess3; for (i = 0; i < nb_ops && i < 4; i++) rte_prefetch0(ops[i]->sym->session); - for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) { - rte_prefetch0(ops[i + 4]->sym->session); - rte_prefetch0(ops[i + 5]->sym->session); - rte_prefetch0(ops[i + 6]->sym->session); - rte_prefetch0(ops[i + 7]->sym->session); - - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sess1 = (struct scheduler_session *) - ops[i+1]->sym->session->_private; - sess2 = (struct scheduler_session *) - ops[i+2]->sym->session->_private; - sess3 = (struct scheduler_session *) - ops[i+3]->sym->session->_private; - - sessions[i] = ops[i]->sym->session; - sessions[i + 1] = ops[i + 1]->sym->session; - sessions[i + 2] = ops[i + 2]->sym->session; - sessions[i + 3] = ops[i + 3]->sym->session; - - ops[i]->sym->session = sess0->sessions[slave_idx]; - ops[i + 1]->sym->session = sess1->sessions[slave_idx]; - ops[i + 2]->sym->session = sess2->sessions[slave_idx]; - ops[i + 3]->sym->session = sess3->sessions[slave_idx]; - } - - for (; i < nb_ops; i++) { - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sessions[i] = ops[i]->sym->session; - ops[i]->sym->session = sess0->sessions[slave_idx]; - } - processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, slave->qp_id, ops, nb_ops); slave->nb_inflight_cops += processed_ops; - if (unlikely(processed_ops < nb_ops)) - for (i = processed_ops; i < nb_ops; i++) - ops[i]->sym->session = sessions[i]; - return processed_ops; } @@ -114,11 +75,11 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) return 0; enqueued_ops = failover_slave_enqueue(&qp_ctx->primary_slave, - PRIMARY_SLAVE_IDX, ops, nb_ops); + ops, nb_ops); if (enqueued_ops < nb_ops) enqueued_ops += failover_slave_enqueue(&qp_ctx->secondary_slave, - SECONDARY_SLAVE_IDX, &ops[enqueued_ops], + &ops[enqueued_ops], nb_ops - enqueued_ops); return enqueued_ops; diff --git a/drivers/crypto/scheduler/scheduler_multicore.c b/drivers/crypto/scheduler/scheduler_multicore.c new file mode 100644 index 00000000..0cd5bce5 --- /dev/null +++ b/drivers/crypto/scheduler/scheduler_multicore.c @@ -0,0 +1,415 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include <unistd.h> + +#include <rte_cryptodev.h> +#include <rte_malloc.h> + +#include "rte_cryptodev_scheduler_operations.h" +#include "scheduler_pmd_private.h" + +#define MC_SCHED_ENQ_RING_NAME_PREFIX "MCS_ENQR_" +#define MC_SCHED_DEQ_RING_NAME_PREFIX "MCS_DEQR_" + +#define MC_SCHED_BUFFER_SIZE 32 + +#define CRYPTO_OP_STATUS_BIT_COMPLETE 0x80 + +/** multi-core scheduler context */ +struct mc_scheduler_ctx { + uint32_t num_workers; /**< Number of workers polling */ + uint32_t stop_signal; + + struct rte_ring *sched_enq_ring[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES]; + struct rte_ring *sched_deq_ring[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES]; +}; + +struct mc_scheduler_qp_ctx { + struct scheduler_slave slaves[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; + uint32_t nb_slaves; + + uint32_t last_enq_worker_idx; + uint32_t last_deq_worker_idx; + + struct mc_scheduler_ctx *mc_private_ctx; +}; + +static uint16_t +schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct mc_scheduler_qp_ctx *mc_qp_ctx = + ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; + struct mc_scheduler_ctx *mc_ctx = mc_qp_ctx->mc_private_ctx; + uint32_t worker_idx = mc_qp_ctx->last_enq_worker_idx; + uint16_t i, processed_ops = 0; + + if (unlikely(nb_ops == 0)) + return 0; + + for (i = 0; i < mc_ctx->num_workers && nb_ops != 0; i++) { + struct rte_ring *enq_ring = mc_ctx->sched_enq_ring[worker_idx]; + uint16_t nb_queue_ops = rte_ring_enqueue_burst(enq_ring, + (void *)(&ops[processed_ops]), nb_ops, NULL); + + nb_ops -= nb_queue_ops; + processed_ops += nb_queue_ops; + + if (++worker_idx == mc_ctx->num_workers) + worker_idx = 0; + } + mc_qp_ctx->last_enq_worker_idx = worker_idx; + + return processed_ops; +} + +static uint16_t +schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_ring *order_ring = + ((struct scheduler_qp_ctx *)qp)->order_ring; + uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring, + nb_ops); + uint16_t nb_ops_enqd = schedule_enqueue(qp, ops, + nb_ops_to_enq); + + scheduler_order_insert(order_ring, ops, nb_ops_enqd); + + return nb_ops_enqd; +} + + +static uint16_t +schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) +{ + struct mc_scheduler_qp_ctx *mc_qp_ctx = + ((struct scheduler_qp_ctx *)qp)->private_qp_ctx; + struct mc_scheduler_ctx *mc_ctx = mc_qp_ctx->mc_private_ctx; + uint32_t worker_idx = mc_qp_ctx->last_deq_worker_idx; + uint16_t i, processed_ops = 0; + + for (i = 0; i < mc_ctx->num_workers && nb_ops != 0; i++) { + struct rte_ring *deq_ring = mc_ctx->sched_deq_ring[worker_idx]; + uint16_t nb_deq_ops = rte_ring_dequeue_burst(deq_ring, + (void *)(&ops[processed_ops]), nb_ops, NULL); + + nb_ops -= nb_deq_ops; + processed_ops += nb_deq_ops; + if (++worker_idx == mc_ctx->num_workers) + worker_idx = 0; + } + + mc_qp_ctx->last_deq_worker_idx = worker_idx; + + return processed_ops; + +} + +static uint16_t +schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct rte_ring *order_ring = ((struct scheduler_qp_ctx *)qp)->order_ring; + struct rte_crypto_op *op; + uint32_t nb_objs = rte_ring_count(order_ring); + uint32_t nb_ops_to_deq = 0; + uint32_t nb_ops_deqd = 0; + + if (nb_objs > nb_ops) + nb_objs = nb_ops; + + while (nb_ops_to_deq < nb_objs) { + SCHEDULER_GET_RING_OBJ(order_ring, nb_ops_to_deq, op); + + if (!(op->status & CRYPTO_OP_STATUS_BIT_COMPLETE)) + break; + + op->status &= ~CRYPTO_OP_STATUS_BIT_COMPLETE; + nb_ops_to_deq++; + } + + if (nb_ops_to_deq) { + nb_ops_deqd = rte_ring_sc_dequeue_bulk(order_ring, + (void **)ops, nb_ops_to_deq, NULL); + } + + return nb_ops_deqd; +} + +static int +slave_attach(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint8_t slave_id) +{ + return 0; +} + +static int +slave_detach(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint8_t slave_id) +{ + return 0; +} + +static int +mc_scheduler_worker(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx = sched_ctx->private_ctx; + struct rte_ring *enq_ring; + struct rte_ring *deq_ring; + uint32_t core_id = rte_lcore_id(); + int i, worker_idx = -1; + struct scheduler_slave *slave; + struct rte_crypto_op *enq_ops[MC_SCHED_BUFFER_SIZE]; + struct rte_crypto_op *deq_ops[MC_SCHED_BUFFER_SIZE]; + uint16_t processed_ops; + uint16_t pending_enq_ops = 0; + uint16_t pending_enq_ops_idx = 0; + uint16_t pending_deq_ops = 0; + uint16_t pending_deq_ops_idx = 0; + uint16_t inflight_ops = 0; + const uint8_t reordering_enabled = sched_ctx->reordering_enabled; + + for (i = 0; i < (int)sched_ctx->nb_wc; i++) { + if (sched_ctx->wc_pool[i] == core_id) { + worker_idx = i; + break; + } + } + if (worker_idx == -1) { + CS_LOG_ERR("worker on core %u:cannot find worker index!\n", core_id); + return -1; + } + + slave = &sched_ctx->slaves[worker_idx]; + enq_ring = mc_ctx->sched_enq_ring[worker_idx]; + deq_ring = mc_ctx->sched_deq_ring[worker_idx]; + + while (!mc_ctx->stop_signal) { + if (pending_enq_ops) { + processed_ops = + rte_cryptodev_enqueue_burst(slave->dev_id, + slave->qp_id, &enq_ops[pending_enq_ops_idx], + pending_enq_ops); + pending_enq_ops -= processed_ops; + pending_enq_ops_idx += processed_ops; + inflight_ops += processed_ops; + } else { + processed_ops = rte_ring_dequeue_burst(enq_ring, (void *)enq_ops, + MC_SCHED_BUFFER_SIZE, NULL); + if (processed_ops) { + pending_enq_ops_idx = rte_cryptodev_enqueue_burst( + slave->dev_id, slave->qp_id, + enq_ops, processed_ops); + pending_enq_ops = processed_ops - pending_enq_ops_idx; + inflight_ops += pending_enq_ops_idx; + } + } + + if (pending_deq_ops) { + processed_ops = rte_ring_enqueue_burst( + deq_ring, (void *)&deq_ops[pending_deq_ops_idx], + pending_deq_ops, NULL); + pending_deq_ops -= processed_ops; + pending_deq_ops_idx += processed_ops; + } else if (inflight_ops) { + processed_ops = rte_cryptodev_dequeue_burst(slave->dev_id, + slave->qp_id, deq_ops, MC_SCHED_BUFFER_SIZE); + if (processed_ops) { + inflight_ops -= processed_ops; + if (reordering_enabled) { + uint16_t j; + + for (j = 0; j < processed_ops; j++) { + deq_ops[j]->status |= + CRYPTO_OP_STATUS_BIT_COMPLETE; + } + } else { + pending_deq_ops_idx = rte_ring_enqueue_burst( + deq_ring, (void *)deq_ops, processed_ops, + NULL); + pending_deq_ops = processed_ops - + pending_deq_ops_idx; + } + } + } + + rte_pause(); + } + + return 0; +} + +static int +scheduler_start(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx = sched_ctx->private_ctx; + uint16_t i; + + mc_ctx->stop_signal = 0; + + for (i = 0; i < sched_ctx->nb_wc; i++) + rte_eal_remote_launch( + (lcore_function_t *)mc_scheduler_worker, dev, + sched_ctx->wc_pool[i]); + + if (sched_ctx->reordering_enabled) { + dev->enqueue_burst = &schedule_enqueue_ordering; + dev->dequeue_burst = &schedule_dequeue_ordering; + } else { + dev->enqueue_burst = &schedule_enqueue; + dev->dequeue_burst = &schedule_dequeue; + } + + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i]; + struct mc_scheduler_qp_ctx *mc_qp_ctx = + qp_ctx->private_qp_ctx; + uint32_t j; + + memset(mc_qp_ctx->slaves, 0, + RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES * + sizeof(struct scheduler_slave)); + for (j = 0; j < sched_ctx->nb_slaves; j++) { + mc_qp_ctx->slaves[j].dev_id = + sched_ctx->slaves[j].dev_id; + mc_qp_ctx->slaves[j].qp_id = i; + } + + mc_qp_ctx->nb_slaves = sched_ctx->nb_slaves; + + mc_qp_ctx->last_enq_worker_idx = 0; + mc_qp_ctx->last_deq_worker_idx = 0; + } + + return 0; +} + +static int +scheduler_stop(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx = sched_ctx->private_ctx; + uint16_t i; + + mc_ctx->stop_signal = 1; + + for (i = 0; i < sched_ctx->nb_wc; i++) + rte_eal_wait_lcore(sched_ctx->wc_pool[i]); + + return 0; +} + +static int +scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id) +{ + struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id]; + struct mc_scheduler_qp_ctx *mc_qp_ctx; + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx = sched_ctx->private_ctx; + + mc_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*mc_qp_ctx), 0, + rte_socket_id()); + if (!mc_qp_ctx) { + CS_LOG_ERR("failed allocate memory for private queue pair"); + return -ENOMEM; + } + + mc_qp_ctx->mc_private_ctx = mc_ctx; + qp_ctx->private_qp_ctx = (void *)mc_qp_ctx; + + + return 0; +} + +static int +scheduler_create_private_ctx(struct rte_cryptodev *dev) +{ + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + struct mc_scheduler_ctx *mc_ctx; + uint16_t i; + + if (sched_ctx->private_ctx) + rte_free(sched_ctx->private_ctx); + + mc_ctx = rte_zmalloc_socket(NULL, sizeof(struct mc_scheduler_ctx), 0, + rte_socket_id()); + if (!mc_ctx) { + CS_LOG_ERR("failed allocate memory"); + return -ENOMEM; + } + + mc_ctx->num_workers = sched_ctx->nb_wc; + for (i = 0; i < sched_ctx->nb_wc; i++) { + char r_name[16]; + + snprintf(r_name, sizeof(r_name), MC_SCHED_ENQ_RING_NAME_PREFIX "%u", i); + mc_ctx->sched_enq_ring[i] = rte_ring_create(r_name, PER_SLAVE_BUFF_SIZE, + rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); + if (!mc_ctx->sched_enq_ring[i]) { + CS_LOG_ERR("Cannot create ring for worker %u", i); + return -1; + } + snprintf(r_name, sizeof(r_name), MC_SCHED_DEQ_RING_NAME_PREFIX "%u", i); + mc_ctx->sched_deq_ring[i] = rte_ring_create(r_name, PER_SLAVE_BUFF_SIZE, + rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); + if (!mc_ctx->sched_deq_ring[i]) { + CS_LOG_ERR("Cannot create ring for worker %u", i); + return -1; + } + } + + sched_ctx->private_ctx = (void *)mc_ctx; + + return 0; +} + +struct rte_cryptodev_scheduler_ops scheduler_mc_ops = { + slave_attach, + slave_detach, + scheduler_start, + scheduler_stop, + scheduler_config_qp, + scheduler_create_private_ctx, + NULL, /* option_set */ + NULL /* option_get */ +}; + +struct rte_cryptodev_scheduler mc_scheduler = { + .name = "multicore-scheduler", + .description = "scheduler which will run burst across multiple cpu cores", + .mode = CDEV_SCHED_MODE_MULTICORE, + .ops = &scheduler_mc_ops +}; + +struct rte_cryptodev_scheduler *multicore_scheduler = &mc_scheduler; diff --git a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c index 6b628dfa..1dd1bc32 100644 --- a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c +++ b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c @@ -67,7 +67,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) struct scheduler_qp_ctx *qp_ctx = qp; struct psd_scheduler_qp_ctx *psd_qp_ctx = qp_ctx->private_qp_ctx; struct rte_crypto_op *sched_ops[NB_PKT_SIZE_SLAVES][nb_ops]; - struct scheduler_session *sess; uint32_t in_flight_ops[NB_PKT_SIZE_SLAVES] = { psd_qp_ctx->primary_slave.nb_inflight_cops, psd_qp_ctx->secondary_slave.nb_inflight_cops @@ -97,8 +96,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) rte_prefetch0(ops[i + 7]->sym); rte_prefetch0(ops[i + 7]->sym->session); - sess = (struct scheduler_session *) - ops[i]->sym->session->_private; /* job_len is initialized as cipher data length, once * it is 0, equals to auth data length */ @@ -118,11 +115,8 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i]; - ops[i]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; - sess = (struct scheduler_session *) - ops[i+1]->sym->session->_private; job_len = ops[i+1]->sym->cipher.data.length; job_len += (ops[i+1]->sym->cipher.data.length == 0) * ops[i+1]->sym->auth.data.length; @@ -135,11 +129,8 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+1]; - ops[i+1]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; - sess = (struct scheduler_session *) - ops[i+2]->sym->session->_private; job_len = ops[i+2]->sym->cipher.data.length; job_len += (ops[i+2]->sym->cipher.data.length == 0) * ops[i+2]->sym->auth.data.length; @@ -152,12 +143,8 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+2]; - ops[i+2]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; - sess = (struct scheduler_session *) - ops[i+3]->sym->session->_private; - job_len = ops[i+3]->sym->cipher.data.length; job_len += (ops[i+3]->sym->cipher.data.length == 0) * ops[i+3]->sym->auth.data.length; @@ -170,14 +157,10 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+3]; - ops[i+3]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; } for (; i < nb_ops; i++) { - sess = (struct scheduler_session *) - ops[i]->sym->session->_private; - job_len = ops[i]->sym->cipher.data.length; job_len += (ops[i]->sym->cipher.data.length == 0) * ops[i]->sym->auth.data.length; @@ -190,7 +173,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i]; - ops[i]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; } diff --git a/drivers/crypto/scheduler/scheduler_pmd.c b/drivers/crypto/scheduler/scheduler_pmd.c index 0b63c20b..400fc4f1 100644 --- a/drivers/crypto/scheduler/scheduler_pmd.c +++ b/drivers/crypto/scheduler/scheduler_pmd.c @@ -33,6 +33,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -41,11 +42,14 @@ #include "rte_cryptodev_scheduler.h" #include "scheduler_pmd_private.h" +uint8_t cryptodev_driver_id; + struct scheduler_init_params { struct rte_crypto_vdev_init_params def_p; uint32_t nb_slaves; enum rte_cryptodev_scheduler_mode mode; uint32_t enable_ordering; + uint64_t wcmask; char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES] [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; }; @@ -57,6 +61,8 @@ struct scheduler_init_params { #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs") #define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG ("max_nb_sessions") #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id") +#define RTE_CRYPTODEV_VDEV_COREMASK ("coremask") +#define RTE_CRYPTODEV_VDEV_CORELIST ("corelist") const char *scheduler_valid_params[] = { RTE_CRYPTODEV_VDEV_NAME, @@ -65,7 +71,9 @@ const char *scheduler_valid_params[] = { RTE_CRYPTODEV_VDEV_ORDERING, RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG, RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG, - RTE_CRYPTODEV_VDEV_SOCKET_ID + RTE_CRYPTODEV_VDEV_SOCKET_ID, + RTE_CRYPTODEV_VDEV_COREMASK, + RTE_CRYPTODEV_VDEV_CORELIST }; struct scheduler_parse_map { @@ -79,7 +87,9 @@ const struct scheduler_parse_map scheduler_mode_map[] = { {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR), CDEV_SCHED_MODE_PKT_SIZE_DISTR}, {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER), - CDEV_SCHED_MODE_FAILOVER} + CDEV_SCHED_MODE_FAILOVER}, + {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE), + CDEV_SCHED_MODE_MULTICORE} }; const struct scheduler_parse_map scheduler_ordering_map[] = { @@ -89,7 +99,8 @@ const struct scheduler_parse_map scheduler_ordering_map[] = { static int cryptodev_scheduler_create(const char *name, - struct scheduler_init_params *init_params) + struct rte_vdev_device *vdev, + struct scheduler_init_params *init_params) { struct rte_cryptodev *dev; struct scheduler_ctx *sched_ctx; @@ -101,22 +112,38 @@ cryptodev_scheduler_create(const char *name, sizeof(init_params->def_p.name), "%s", name); - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->def_p.name, + dev = rte_cryptodev_vdev_pmd_init(init_params->def_p.name, sizeof(struct scheduler_ctx), - init_params->def_p.socket_id); + init_params->def_p.socket_id, + vdev); if (dev == NULL) { CS_LOG_ERR("driver %s: failed to create cryptodev vdev", name); return -EFAULT; } - dev->dev_type = RTE_CRYPTODEV_SCHEDULER_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_crypto_scheduler_pmd_ops; sched_ctx = dev->data->dev_private; sched_ctx->max_nb_queue_pairs = init_params->def_p.max_nb_queue_pairs; + if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { + uint16_t i; + + sched_ctx->nb_wc = 0; + + for (i = 0; i < RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES; i++) { + if (init_params->wcmask & (1ULL << i)) { + sched_ctx->wc_pool[sched_ctx->nb_wc++] = i; + RTE_LOG(INFO, PMD, + " Worker core[%u]=%u added\n", + sched_ctx->nb_wc-1, i); + } + } + } + if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED && init_params->mode < CDEV_SCHED_MODE_COUNT) { ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id, @@ -219,22 +246,6 @@ cryptodev_scheduler_remove(struct rte_vdev_device *vdev) return 0; } -static uint8_t -number_of_sockets(void) -{ - int sockets = 0; - int i; - const struct rte_memseg *ms = rte_eal_get_physmem_layout(); - - for (i = 0; ((i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL)); i++) { - if (sockets < ms[i].socket_id) - sockets = ms[i].socket_id; - } - - /* Number of sockets = maximum socket_id + 1 */ - return ++sockets; -} - /** Parse integer from integer argument */ static int parse_integer_arg(const char *key __rte_unused, @@ -251,6 +262,43 @@ parse_integer_arg(const char *key __rte_unused, return 0; } +/** Parse integer from hexadecimal integer argument */ +static int +parse_coremask_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + struct scheduler_init_params *params = extra_args; + + params->wcmask = strtoull(value, NULL, 16); + + return 0; +} + +/** Parse integer from list of integers argument */ +static int +parse_corelist_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + struct scheduler_init_params *params = extra_args; + + params->wcmask = 0ULL; + + const char *token = value; + + while (isdigit(token[0])) { + char *rval; + unsigned int core = strtoul(token, &rval, 10); + + params->wcmask |= 1ULL << core; + token = (const char *)rval; + if (token[0] == '\0') + break; + token++; + } + + return 0; +} + /** Parse name */ static int parse_name_arg(const char *key __rte_unused, @@ -277,7 +325,7 @@ parse_slave_arg(const char *key __rte_unused, { struct scheduler_init_params *param = extra_args; - if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES - 1) { + if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) { CS_LOG_ERR("Too many slaves.\n"); return -ENOMEM; } @@ -370,6 +418,18 @@ scheduler_parse_init_params(struct scheduler_init_params *params, if (ret < 0) goto free_kvlist; + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK, + &parse_coremask_arg, + params); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST, + &parse_corelist_arg, + params); + if (ret < 0) + goto free_kvlist; + ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME, &parse_name_arg, ¶ms->def_p); @@ -390,12 +450,6 @@ scheduler_parse_init_params(struct scheduler_init_params *params, &parse_ordering_arg, params); if (ret < 0) goto free_kvlist; - - if (params->def_p.socket_id >= number_of_sockets()) { - CDEV_LOG_ERR("Invalid socket id specified to create " - "the virtual crypto device on"); - goto free_kvlist; - } } free_kvlist: @@ -437,8 +491,12 @@ cryptodev_scheduler_probe(struct rte_vdev_device *vdev) if (init_params.def_p.name[0] != '\0') RTE_LOG(INFO, PMD, " User defined name = %s\n", init_params.def_p.name); + if (init_params.wcmask != 0) + RTE_LOG(INFO, PMD, " workers core mask = %"PRIx64"\n", + init_params.wcmask); return cryptodev_scheduler_create(name, + vdev, &init_params); } @@ -454,3 +512,5 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, "max_nb_sessions=<int> " "socket_id=<int> " "slave=<name>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_scheduler_pmd_drv, + cryptodev_driver_id); diff --git a/drivers/crypto/scheduler/scheduler_pmd_ops.c b/drivers/crypto/scheduler/scheduler_pmd_ops.c index 2b5858df..d3795346 100644 --- a/drivers/crypto/scheduler/scheduler_pmd_ops.c +++ b/drivers/crypto/scheduler/scheduler_pmd_ops.c @@ -37,6 +37,7 @@ #include <rte_dev.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_reorder.h> #include "scheduler_pmd_private.h" @@ -368,7 +369,7 @@ scheduler_pmd_info_get(struct rte_cryptodev *dev, max_nb_sessions; } - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->feature_flags = dev->feature_flags; dev_info->capabilities = sched_ctx->capabilities; dev_info->max_nb_queue_pairs = sched_ctx->max_nb_queue_pairs; @@ -398,7 +399,8 @@ scheduler_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) /** Setup a queue pair */ static int scheduler_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, - const struct rte_cryptodev_qp_conf *qp_conf, int socket_id) + const struct rte_cryptodev_qp_conf *qp_conf, int socket_id, + struct rte_mempool *session_pool) { struct scheduler_ctx *sched_ctx = dev->data->dev_private; struct scheduler_qp_ctx *qp_ctx; @@ -420,8 +422,13 @@ scheduler_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, for (i = 0; i < sched_ctx->nb_slaves; i++) { uint8_t slave_id = sched_ctx->slaves[i].dev_id; + /* + * All slaves will share the same session mempool + * for session-less operations, so the objects + * must be big enough for all the drivers used. + */ ret = rte_cryptodev_queue_pair_setup(slave_id, qp_id, - qp_conf, socket_id); + qp_conf, socket_id, session_pool); if (ret < 0) return ret; } @@ -483,37 +490,41 @@ scheduler_pmd_qp_count(struct rte_cryptodev *dev) static uint32_t scheduler_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) { - return sizeof(struct scheduler_session); + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint8_t i = 0; + uint32_t max_priv_sess_size = 0; + + /* Check what is the maximum private session size for all slaves */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + struct rte_cryptodev *dev = &rte_cryptodevs[slave_dev_id]; + uint32_t priv_sess_size = (*dev->dev_ops->session_get_size)(dev); + + if (max_priv_sess_size < priv_sess_size) + max_priv_sess_size = priv_sess_size; + } + + return max_priv_sess_size; } static int -config_slave_sess(struct scheduler_ctx *sched_ctx, - struct rte_crypto_sym_xform *xform, - struct scheduler_session *sess, - uint32_t create) +scheduler_pmd_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + struct scheduler_ctx *sched_ctx = dev->data->dev_private; uint32_t i; + int ret; for (i = 0; i < sched_ctx->nb_slaves; i++) { struct scheduler_slave *slave = &sched_ctx->slaves[i]; - if (sess->sessions[i]) { - if (create) - continue; - /* !create */ - sess->sessions[i] = rte_cryptodev_sym_session_free( - slave->dev_id, sess->sessions[i]); - } else { - if (!create) - continue; - /* create */ - sess->sessions[i] = - rte_cryptodev_sym_session_create( - slave->dev_id, xform); - if (!sess->sessions[i]) { - config_slave_sess(sched_ctx, NULL, sess, 0); - return -1; - } + ret = rte_cryptodev_sym_session_init(slave->dev_id, sess, + xform, mempool); + if (ret < 0) { + CS_LOG_ERR("unabled to config sym session"); + return ret; } } @@ -523,27 +534,17 @@ config_slave_sess(struct scheduler_ctx *sched_ctx, /** Clear the memory of session so it doesn't leave key material behind */ static void scheduler_pmd_session_clear(struct rte_cryptodev *dev, - void *sess) + struct rte_cryptodev_sym_session *sess) { struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; - config_slave_sess(sched_ctx, NULL, sess, 0); - - memset(sess, 0, sizeof(struct scheduler_session)); -} - -static void * -scheduler_pmd_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *sess) -{ - struct scheduler_ctx *sched_ctx = dev->data->dev_private; + /* Clear private data of slaves */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + struct scheduler_slave *slave = &sched_ctx->slaves[i]; - if (config_slave_sess(sched_ctx, xform, sess, 1) < 0) { - CS_LOG_ERR("unabled to config sym session"); - return NULL; + rte_cryptodev_sym_session_clear(slave->dev_id, sess); } - - return sess; } struct rte_cryptodev_ops scheduler_pmd_ops = { diff --git a/drivers/crypto/scheduler/scheduler_pmd_private.h b/drivers/crypto/scheduler/scheduler_pmd_private.h index 421dae37..e606716a 100644 --- a/drivers/crypto/scheduler/scheduler_pmd_private.h +++ b/drivers/crypto/scheduler/scheduler_pmd_private.h @@ -36,6 +36,9 @@ #include "rte_cryptodev_scheduler.h" +#define CRYPTODEV_NAME_SCHEDULER_PMD crypto_scheduler +/**< Scheduler Crypto PMD device name */ + #define PER_SLAVE_BUFF_SIZE (256) #define CS_LOG_ERR(fmt, args...) \ @@ -63,7 +66,7 @@ struct scheduler_slave { uint16_t qp_id; uint32_t nb_inflight_cops; - enum rte_cryptodev_type dev_type; + uint8_t driver_id; }; struct scheduler_ctx { @@ -86,6 +89,8 @@ struct scheduler_ctx { char name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; char description[RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN]; + uint16_t wc_pool[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES]; + uint16_t nb_wc; char *init_slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; int nb_init_slaves; @@ -100,12 +105,10 @@ struct scheduler_qp_ctx { uint32_t seqn; } __rte_cache_aligned; -struct scheduler_session { - struct rte_cryptodev_sym_session *sessions[ - RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; -}; -static inline uint16_t __attribute__((always_inline)) +extern uint8_t cryptodev_driver_id; + +static __rte_always_inline uint16_t get_max_enqueue_order_count(struct rte_ring *order_ring, uint16_t nb_ops) { uint32_t count = rte_ring_free_count(order_ring); @@ -113,7 +116,7 @@ get_max_enqueue_order_count(struct rte_ring *order_ring, uint16_t nb_ops) return count > nb_ops ? nb_ops : count; } -static inline void __attribute__((always_inline)) +static __rte_always_inline void scheduler_order_insert(struct rte_ring *order_ring, struct rte_crypto_op **ops, uint16_t nb_ops) { @@ -125,7 +128,7 @@ scheduler_order_insert(struct rte_ring *order_ring, op = ring[(order_ring->cons.head + pos) & order_ring->mask]; \ } while (0) -static inline uint16_t __attribute__((always_inline)) +static __rte_always_inline uint16_t scheduler_order_drain(struct rte_ring *order_ring, struct rte_crypto_op **ops, uint16_t nb_ops) { diff --git a/drivers/crypto/scheduler/scheduler_roundrobin.c b/drivers/crypto/scheduler/scheduler_roundrobin.c index 01162764..4a847281 100644 --- a/drivers/crypto/scheduler/scheduler_roundrobin.c +++ b/drivers/crypto/scheduler/scheduler_roundrobin.c @@ -52,8 +52,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) uint32_t slave_idx = rr_qp_ctx->last_enq_slave_idx; struct scheduler_slave *slave = &rr_qp_ctx->slaves[slave_idx]; uint16_t i, processed_ops; - struct rte_cryptodev_sym_session *sessions[nb_ops]; - struct scheduler_session *sess0, *sess1, *sess2, *sess3; if (unlikely(nb_ops == 0)) return 0; @@ -61,39 +59,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) for (i = 0; i < nb_ops && i < 4; i++) rte_prefetch0(ops[i]->sym->session); - for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) { - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sess1 = (struct scheduler_session *) - ops[i+1]->sym->session->_private; - sess2 = (struct scheduler_session *) - ops[i+2]->sym->session->_private; - sess3 = (struct scheduler_session *) - ops[i+3]->sym->session->_private; - - sessions[i] = ops[i]->sym->session; - sessions[i + 1] = ops[i + 1]->sym->session; - sessions[i + 2] = ops[i + 2]->sym->session; - sessions[i + 3] = ops[i + 3]->sym->session; - - ops[i]->sym->session = sess0->sessions[slave_idx]; - ops[i + 1]->sym->session = sess1->sessions[slave_idx]; - ops[i + 2]->sym->session = sess2->sessions[slave_idx]; - ops[i + 3]->sym->session = sess3->sessions[slave_idx]; - - rte_prefetch0(ops[i + 4]->sym->session); - rte_prefetch0(ops[i + 5]->sym->session); - rte_prefetch0(ops[i + 6]->sym->session); - rte_prefetch0(ops[i + 7]->sym->session); - } - - for (; i < nb_ops; i++) { - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sessions[i] = ops[i]->sym->session; - ops[i]->sym->session = sess0->sessions[slave_idx]; - } - processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, slave->qp_id, ops, nb_ops); @@ -102,12 +67,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) rr_qp_ctx->last_enq_slave_idx += 1; rr_qp_ctx->last_enq_slave_idx %= rr_qp_ctx->nb_slaves; - /* recover session if enqueue is failed */ - if (unlikely(processed_ops < nb_ops)) { - for (i = processed_ops; i < nb_ops; i++) - ops[i]->sym->session = sessions[i]; - } - return processed_ops; } diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd.c b/drivers/crypto/snow3g/rte_snow3g_pmd.c index 960956ca..dad45068 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd.c +++ b/drivers/crypto/snow3g/rte_snow3g_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -46,6 +47,8 @@ #define SNOW3G_MAX_BURST 8 #define BYTE_LEN 8 +static uint8_t cryptodev_driver_id; + /** Get xform chain order. */ static enum snow3g_operation snow3g_get_mode(const struct rte_crypto_sym_xform *xform) @@ -108,13 +111,20 @@ snow3g_set_session_parameters(struct snow3g_session *sess, case SNOW3G_OP_NOT_SUPPORTED: default: SNOW3G_LOG_ERR("Unsupported operation chain order parameter"); - return -EINVAL; + return -ENOTSUP; } if (cipher_xform) { /* Only SNOW 3G UEA2 supported */ if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_SNOW3G_UEA2) + return -ENOTSUP; + + if (cipher_xform->cipher.iv.length != SNOW3G_IV_LENGTH) { + SNOW3G_LOG_ERR("Wrong IV length"); return -EINVAL; + } + sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; + /* Initialize key */ sso_snow3g_init_key_sched(cipher_xform->cipher.key.data, &sess->pKeySched_cipher); @@ -123,8 +133,21 @@ snow3g_set_session_parameters(struct snow3g_session *sess, if (auth_xform) { /* Only SNOW 3G UIA2 supported */ if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_SNOW3G_UIA2) + return -ENOTSUP; + + if (auth_xform->auth.digest_length != SNOW3G_DIGEST_LENGTH) { + SNOW3G_LOG_ERR("Wrong digest length"); return -EINVAL; + } + sess->auth_op = auth_xform->auth.op; + + if (auth_xform->auth.iv.length != SNOW3G_IV_LENGTH) { + SNOW3G_LOG_ERR("Wrong IV length"); + return -EINVAL; + } + sess->auth_iv_offset = auth_xform->auth.iv.offset; + /* Initialize key */ sso_snow3g_init_key_sched(auth_xform->auth.key.data, &sess->pKeySched_hash); @@ -140,27 +163,41 @@ snow3g_set_session_parameters(struct snow3g_session *sess, static struct snow3g_session * snow3g_get_session(struct snow3g_qp *qp, struct rte_crypto_op *op) { - struct snow3g_session *sess; - - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->sym->session->dev_type != - RTE_CRYPTODEV_SNOW3G_PMD)) + struct snow3g_session *sess = NULL; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct snow3g_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct snow3g_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct snow3g_session *)c_sess->_private; + sess = (struct snow3g_session *)_sess_private_data; if (unlikely(snow3g_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + + return sess; } @@ -173,17 +210,10 @@ process_snow3g_cipher_op(struct rte_crypto_op **ops, unsigned i; uint8_t processed_ops = 0; uint8_t *src[SNOW3G_MAX_BURST], *dst[SNOW3G_MAX_BURST]; - uint8_t *IV[SNOW3G_MAX_BURST]; + uint8_t *iv[SNOW3G_MAX_BURST]; uint32_t num_bytes[SNOW3G_MAX_BURST]; for (i = 0; i < num_ops; i++) { - /* Sanity checks. */ - if (unlikely(ops[i]->sym->cipher.iv.length != SNOW3G_IV_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - SNOW3G_LOG_ERR("iv"); - break; - } - src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); dst[i] = ops[i]->sym->m_dst ? @@ -191,13 +221,14 @@ process_snow3g_cipher_op(struct rte_crypto_op **ops, (ops[i]->sym->cipher.data.offset >> 3) : rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); - IV[i] = ops[i]->sym->cipher.iv.data; + iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->cipher_iv_offset); num_bytes[i] = ops[i]->sym->cipher.data.length >> 3; processed_ops++; } - sso_snow3g_f8_n_buffer(&session->pKeySched_cipher, IV, src, dst, + sso_snow3g_f8_n_buffer(&session->pKeySched_cipher, iv, src, dst, num_bytes, processed_ops); return processed_ops; @@ -209,16 +240,9 @@ process_snow3g_cipher_op_bit(struct rte_crypto_op *op, struct snow3g_session *session) { uint8_t *src, *dst; - uint8_t *IV; + uint8_t *iv; uint32_t length_in_bits, offset_in_bits; - /* Sanity checks. */ - if (unlikely(op->sym->cipher.iv.length != SNOW3G_IV_LENGTH)) { - op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - SNOW3G_LOG_ERR("iv"); - return 0; - } - offset_in_bits = op->sym->cipher.data.offset; src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *); if (op->sym->m_dst == NULL) { @@ -227,10 +251,11 @@ process_snow3g_cipher_op_bit(struct rte_crypto_op *op, return 0; } dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *); - IV = op->sym->cipher.iv.data; + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->cipher_iv_offset); length_in_bits = op->sym->cipher.data.length; - sso_snow3g_f8_1_buffer_bit(&session->pKeySched_cipher, IV, + sso_snow3g_f8_1_buffer_bit(&session->pKeySched_cipher, iv, src, dst, length_in_bits, offset_in_bits); return 1; @@ -246,20 +271,9 @@ process_snow3g_hash_op(struct rte_crypto_op **ops, uint8_t processed_ops = 0; uint8_t *src, *dst; uint32_t length_in_bits; + uint8_t *iv; for (i = 0; i < num_ops; i++) { - if (unlikely(ops[i]->sym->auth.aad.length != SNOW3G_IV_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - SNOW3G_LOG_ERR("aad"); - break; - } - - if (unlikely(ops[i]->sym->auth.digest.length != SNOW3G_DIGEST_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - SNOW3G_LOG_ERR("digest"); - break; - } - /* Data must be byte aligned */ if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; @@ -271,27 +285,29 @@ process_snow3g_hash_op(struct rte_crypto_op **ops, src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->auth.data.offset >> 3); + iv = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->auth_iv_offset); if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { dst = (uint8_t *)rte_pktmbuf_append(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + SNOW3G_DIGEST_LENGTH); sso_snow3g_f9_1_buffer(&session->pKeySched_hash, - ops[i]->sym->auth.aad.data, src, + iv, src, length_in_bits, dst); /* Verify digest. */ if (memcmp(dst, ops[i]->sym->auth.digest.data, - ops[i]->sym->auth.digest.length) != 0) + SNOW3G_DIGEST_LENGTH) != 0) ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; /* Trim area used for digest from mbuf. */ rte_pktmbuf_trim(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + SNOW3G_DIGEST_LENGTH); } else { dst = ops[i]->sym->auth.digest.data; sso_snow3g_f9_1_buffer(&session->pKeySched_hash, - ops[i]->sym->auth.aad.data, src, + iv, src, length_in_bits, dst); } processed_ops++; @@ -356,7 +372,11 @@ process_ops(struct rte_crypto_op **ops, struct snow3g_session *session, if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (ops[i]->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct snow3g_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -408,8 +428,9 @@ process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session, op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { - rte_mempool_put(qp->sess_mp, op->sym->session); + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(op->sym->session, 0, sizeof(struct snow3g_session)); + rte_cryptodev_sym_session_free(op->sym->session); op->sym->session = NULL; } @@ -548,28 +569,21 @@ cryptodev_snow3g_create(const char *name, { struct rte_cryptodev *dev; struct snow3g_private *internals; - uint64_t cpu_flags = 0; + uint64_t cpu_flags = RTE_CRYPTODEV_FF_CPU_SSE; if (init_params->name[0] == '\0') snprintf(init_params->name, sizeof(init_params->name), "%s", name); - /* Check CPU for supported vector instruction set */ - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) - cpu_flags |= RTE_CRYPTODEV_FF_CPU_SSE; - else { - SNOW3G_LOG_ERR("Vector instructions are not supported by CPU"); - return -EFAULT; - } - - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, - sizeof(struct snow3g_private), init_params->socket_id); + dev = rte_cryptodev_vdev_pmd_init(init_params->name, + sizeof(struct snow3g_private), init_params->socket_id, + vdev); if (dev == NULL) { SNOW3G_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_SNOW3G_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_snow3g_pmd_ops; /* Register RX/TX burst functions for data path. */ @@ -611,7 +625,7 @@ cryptodev_snow3g_probe(struct rte_vdev_device *vdev) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -653,3 +667,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SNOW3G_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_snow3g_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c b/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c index 7ce96be9..ae9569c8 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c +++ b/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -56,7 +56,7 @@ static const struct rte_cryptodev_capabilities snow3g_pmd_capabilities[] = { .max = 4, .increment = 0 }, - .aad_size = { + .iv_size = { .min = 16, .max = 16, .increment = 0 @@ -156,7 +156,7 @@ snow3g_pmd_info_get(struct rte_cryptodev *dev, struct snow3g_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; dev_info->feature_flags = dev->feature_flags; @@ -220,7 +220,7 @@ snow3g_pmd_qp_create_processed_ops_ring(struct snow3g_qp *qp, static int snow3g_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct snow3g_qp *qp = NULL; @@ -245,7 +245,7 @@ snow3g_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->processed_ops == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -289,33 +289,56 @@ snow3g_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a SNOW 3G session from a crypto xform chain */ -static void * +static int snow3g_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + if (unlikely(sess == NULL)) { SNOW3G_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; } - if (snow3g_set_session_parameters(sess, xform) != 0) { + ret = snow3g_set_session_parameters(sess_private_data, xform); + if (ret != 0) { SNOW3G_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -snow3g_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +snow3g_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct snow3g_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct snow3g_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops snow3g_pmd_ops = { diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd_private.h b/drivers/crypto/snow3g/rte_snow3g_pmd_private.h index 03973b97..fba3cb86 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd_private.h +++ b/drivers/crypto/snow3g/rte_snow3g_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,9 @@ #include <sso_snow3g.h> +#define CRYPTODEV_NAME_SNOW3G_PMD crypto_snow3g +/**< SNOW 3G PMD device name */ + #define SNOW3G_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_SNOW3G_PMD), \ @@ -91,6 +94,8 @@ struct snow3g_session { enum rte_crypto_auth_operation auth_op; sso_snow3g_key_schedule_t pKeySched_cipher; sso_snow3g_key_schedule_t pKeySched_hash; + uint16_t cipher_iv_offset; + uint16_t auth_iv_offset; } __rte_cache_aligned; diff --git a/drivers/crypto/zuc/rte_zuc_pmd.c b/drivers/crypto/zuc/rte_zuc_pmd.c index 1020544b..b301711e 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd.c +++ b/drivers/crypto/zuc/rte_zuc_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> +#include <rte_cryptodev_vdev.h> #include <rte_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -45,6 +46,8 @@ #define ZUC_MAX_BURST 8 #define BYTE_LEN 8 +static uint8_t cryptodev_driver_id; + /** Get xform chain order. */ static enum zuc_operation zuc_get_mode(const struct rte_crypto_sym_xform *xform) @@ -107,13 +110,20 @@ zuc_set_session_parameters(struct zuc_session *sess, case ZUC_OP_NOT_SUPPORTED: default: ZUC_LOG_ERR("Unsupported operation chain order parameter"); - return -EINVAL; + return -ENOTSUP; } if (cipher_xform) { /* Only ZUC EEA3 supported */ if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_ZUC_EEA3) + return -ENOTSUP; + + if (cipher_xform->cipher.iv.length != ZUC_IV_KEY_LENGTH) { + ZUC_LOG_ERR("Wrong IV length"); return -EINVAL; + } + sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; + /* Copy the key */ memcpy(sess->pKey_cipher, cipher_xform->cipher.key.data, ZUC_IV_KEY_LENGTH); @@ -122,8 +132,21 @@ zuc_set_session_parameters(struct zuc_session *sess, if (auth_xform) { /* Only ZUC EIA3 supported */ if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_ZUC_EIA3) + return -ENOTSUP; + + if (auth_xform->auth.digest_length != ZUC_DIGEST_LENGTH) { + ZUC_LOG_ERR("Wrong digest length"); return -EINVAL; + } + sess->auth_op = auth_xform->auth.op; + + if (auth_xform->auth.iv.length != ZUC_IV_KEY_LENGTH) { + ZUC_LOG_ERR("Wrong IV length"); + return -EINVAL; + } + sess->auth_iv_offset = auth_xform->auth.iv.offset; + /* Copy the key */ memcpy(sess->pKey_hash, auth_xform->auth.key.data, ZUC_IV_KEY_LENGTH); @@ -139,27 +162,40 @@ zuc_set_session_parameters(struct zuc_session *sess, static struct zuc_session * zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) { - struct zuc_session *sess; - - if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) { - if (unlikely(op->sym->session->dev_type != - RTE_CRYPTODEV_ZUC_PMD)) + struct zuc_session *sess = NULL; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct zuc_session *)get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct zuc_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct zuc_session *)c_sess->_private; + sess = (struct zuc_session *)_sess_private_data; if (unlikely(zuc_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + + return sess; } @@ -172,18 +208,11 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, unsigned i; uint8_t processed_ops = 0; uint8_t *src[ZUC_MAX_BURST], *dst[ZUC_MAX_BURST]; - uint8_t *IV[ZUC_MAX_BURST]; + uint8_t *iv[ZUC_MAX_BURST]; uint32_t num_bytes[ZUC_MAX_BURST]; uint8_t *cipher_keys[ZUC_MAX_BURST]; for (i = 0; i < num_ops; i++) { - /* Sanity checks. */ - if (unlikely(ops[i]->sym->cipher.iv.length != ZUC_IV_KEY_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - ZUC_LOG_ERR("iv"); - break; - } - if (((ops[i]->sym->cipher.data.length % BYTE_LEN) != 0) || ((ops[i]->sym->cipher.data.offset % BYTE_LEN) != 0)) { @@ -212,7 +241,8 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, (ops[i]->sym->cipher.data.offset >> 3) : rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); - IV[i] = ops[i]->sym->cipher.iv.data; + iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->cipher_iv_offset); num_bytes[i] = ops[i]->sym->cipher.data.length >> 3; cipher_keys[i] = session->pKey_cipher; @@ -220,7 +250,7 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, processed_ops++; } - sso_zuc_eea3_n_buffer(cipher_keys, IV, src, dst, + sso_zuc_eea3_n_buffer(cipher_keys, iv, src, dst, num_bytes, processed_ops); return processed_ops; @@ -237,20 +267,9 @@ process_zuc_hash_op(struct rte_crypto_op **ops, uint8_t *src; uint32_t *dst; uint32_t length_in_bits; + uint8_t *iv; for (i = 0; i < num_ops; i++) { - if (unlikely(ops[i]->sym->auth.aad.length != ZUC_IV_KEY_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - ZUC_LOG_ERR("aad"); - break; - } - - if (unlikely(ops[i]->sym->auth.digest.length != ZUC_DIGEST_LENGTH)) { - ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; - ZUC_LOG_ERR("digest"); - break; - } - /* Data must be byte aligned */ if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) { ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; @@ -262,27 +281,29 @@ process_zuc_hash_op(struct rte_crypto_op **ops, src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->auth.data.offset >> 3); + iv = rte_crypto_op_ctod_offset(ops[i], uint8_t *, + session->auth_iv_offset); if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { dst = (uint32_t *)rte_pktmbuf_append(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + ZUC_DIGEST_LENGTH); sso_zuc_eia3_1_buffer(session->pKey_hash, - ops[i]->sym->auth.aad.data, src, + iv, src, length_in_bits, dst); /* Verify digest. */ if (memcmp(dst, ops[i]->sym->auth.digest.data, - ops[i]->sym->auth.digest.length) != 0) + ZUC_DIGEST_LENGTH) != 0) ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; /* Trim area used for digest from mbuf. */ rte_pktmbuf_trim(ops[i]->sym->m_src, - ops[i]->sym->auth.digest.length); + ZUC_DIGEST_LENGTH); } else { dst = (uint32_t *)ops[i]->sym->auth.digest.data; sso_zuc_eia3_1_buffer(session->pKey_hash, - ops[i]->sym->auth.aad.data, src, + iv, src, length_in_bits, dst); } processed_ops++; @@ -332,7 +353,11 @@ process_ops(struct rte_crypto_op **ops, struct zuc_session *session, if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ - if (ops[i]->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) { + if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct zuc_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -448,28 +473,21 @@ cryptodev_zuc_create(const char *name, { struct rte_cryptodev *dev; struct zuc_private *internals; - uint64_t cpu_flags = 0; + uint64_t cpu_flags = RTE_CRYPTODEV_FF_CPU_SSE; if (init_params->name[0] == '\0') snprintf(init_params->name, sizeof(init_params->name), "%s", name); - /* Check CPU for supported vector instruction set */ - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1)) - cpu_flags |= RTE_CRYPTODEV_FF_CPU_SSE; - else { - ZUC_LOG_ERR("Vector instructions are not supported by CPU"); - return -EFAULT; - } - - dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name, - sizeof(struct zuc_private), init_params->socket_id); + dev = rte_cryptodev_vdev_pmd_init(init_params->name, + sizeof(struct zuc_private), init_params->socket_id, + vdev); if (dev == NULL) { ZUC_LOG_ERR("failed to create cryptodev vdev"); goto init_error; } - dev->dev_type = RTE_CRYPTODEV_ZUC_PMD; + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_zuc_pmd_ops; /* Register RX/TX burst functions for data path. */ @@ -511,7 +529,7 @@ cryptodev_zuc_probe(struct rte_vdev_device *vdev) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_parse_vdev_init_params(&init_params, input_args); + rte_cryptodev_vdev_parse_init_params(&init_params, input_args); RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, init_params.socket_id); @@ -552,3 +570,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ZUC_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_zuc_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/zuc/rte_zuc_pmd_ops.c b/drivers/crypto/zuc/rte_zuc_pmd_ops.c index e793459c..52c6aed8 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd_ops.c +++ b/drivers/crypto/zuc/rte_zuc_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -56,7 +56,7 @@ static const struct rte_cryptodev_capabilities zuc_pmd_capabilities[] = { .max = 4, .increment = 0 }, - .aad_size = { + .iv_size = { .min = 16, .max = 16, .increment = 0 @@ -80,7 +80,7 @@ static const struct rte_cryptodev_capabilities zuc_pmd_capabilities[] = { .min = 16, .max = 16, .increment = 0 - } + }, }, } }, } }, @@ -156,7 +156,7 @@ zuc_pmd_info_get(struct rte_cryptodev *dev, struct zuc_private *internals = dev->data->dev_private; if (dev_info != NULL) { - dev_info->dev_type = dev->dev_type; + dev_info->driver_id = dev->driver_id; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; dev_info->sym.max_nb_sessions = internals->max_nb_sessions; dev_info->feature_flags = dev->feature_flags; @@ -220,7 +220,7 @@ zuc_pmd_qp_create_processed_ops_ring(struct zuc_qp *qp, static int zuc_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) + int socket_id, struct rte_mempool *session_pool) { struct zuc_qp *qp = NULL; @@ -245,7 +245,7 @@ zuc_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, if (qp->processed_ops == NULL) goto qp_setup_cleanup; - qp->sess_mp = dev->data->session_pool; + qp->sess_mp = session_pool; memset(&qp->qp_stats, 0, sizeof(qp->qp_stats)); @@ -289,33 +289,56 @@ zuc_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a ZUC session from a crypto xform chain */ -static void * +static int zuc_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + int ret; + if (unlikely(sess == NULL)) { ZUC_LOG_ERR("invalid session struct"); - return NULL; + return -EINVAL; } - if (zuc_set_session_parameters(sess, xform) != 0) { + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = zuc_set_session_parameters(sess_private_data, xform); + if (ret != 0) { ZUC_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -zuc_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +zuc_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct zuc_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct zuc_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops zuc_pmd_ops = { diff --git a/drivers/crypto/zuc/rte_zuc_pmd_private.h b/drivers/crypto/zuc/rte_zuc_pmd_private.h index 030f120b..b706e0aa 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd_private.h +++ b/drivers/crypto/zuc/rte_zuc_pmd_private.h @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016 Intel Corporation. All rights reserved. + * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -35,6 +35,9 @@ #include <sso_zuc.h> +#define CRYPTODEV_NAME_ZUC_PMD crypto_zuc +/**< KASUMI PMD device name */ + #define ZUC_LOG_ERR(fmt, args...) \ RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ RTE_STR(CRYPTODEV_NAME_ZUC_PMD), \ @@ -92,6 +95,8 @@ struct zuc_session { enum rte_crypto_auth_operation auth_op; uint8_t pKey_cipher[ZUC_IV_KEY_LENGTH]; uint8_t pKey_hash[ZUC_IV_KEY_LENGTH]; + uint16_t cipher_iv_offset; + uint16_t auth_iv_offset; } __rte_cache_aligned; |