diff options
Diffstat (limited to 'drivers/crypto')
64 files changed, 6613 insertions, 1785 deletions
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 7a719b9d..645b6967 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -31,29 +31,18 @@ include $(RTE_SDK)/mk/rte.vars.mk -core-libs := librte_eal librte_mbuf librte_mempool librte_ring librte_cryptodev - DIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += aesni_gcm -DEPDIRS-aesni_gcm = $(core-libs) DIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += aesni_mb -DEPDIRS-aesni_mb = $(core-libs) DIRS-$(CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO) += armv8 -DEPDIRS-armv8 = $(core-libs) DIRS-$(CONFIG_RTE_LIBRTE_PMD_OPENSSL) += openssl -DEPDIRS-openssl = $(core-libs) DIRS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += qat -DEPDIRS-qat = $(core-libs) DIRS-$(CONFIG_RTE_LIBRTE_PMD_CRYPTO_SCHEDULER) += scheduler -DEPDIRS-scheduler = $(core-libs) librte_kvargs librte_reorder DIRS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += snow3g -DEPDIRS-snow3g = $(core-libs) DIRS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += kasumi -DEPDIRS-kasumi = $(core-libs) DIRS-$(CONFIG_RTE_LIBRTE_PMD_ZUC) += zuc -DEPDIRS-zuc = $(core-libs) +DIRS-$(CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO) += mrvl DIRS-$(CONFIG_RTE_LIBRTE_PMD_NULL_CRYPTO) += null -DEPDIRS-null = $(core-libs) DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += dpaa2_sec -DEPDIRS-dpaa2_sec = $(core-libs) +DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA_SEC) += dpaa_sec include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/drivers/crypto/aesni_gcm/Makefile b/drivers/crypto/aesni_gcm/Makefile index 6fca5e1c..ddfec4c6 100644 --- a/drivers/crypto/aesni_gcm/Makefile +++ b/drivers/crypto/aesni_gcm/Makefile @@ -53,6 +53,9 @@ EXPORT_MAP := rte_pmd_aesni_gcm_version.map CFLAGS += -I$(AESNI_MULTI_BUFFER_LIB_PATH) CFLAGS += -I$(AESNI_MULTI_BUFFER_LIB_PATH)/include LDLIBS += -L$(AESNI_MULTI_BUFFER_LIB_PATH) -lIPSec_MB +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_GCM) += aesni_gcm_pmd.c diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c index d9c91d06..08dcacce 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c @@ -31,12 +31,10 @@ */ #include <rte_common.h> -#include <rte_config.h> #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> -#include <rte_cryptodev_vdev.h> -#include <rte_vdev.h> +#include <rte_bus_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> #include <rte_byteorder.h> @@ -224,7 +222,7 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, RTE_ASSERT(m_src != NULL); - while (offset >= m_src->data_len) { + while (offset >= m_src->data_len && data_length != 0) { offset -= m_src->data_len; m_src = m_src->next; @@ -298,14 +296,7 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, sym_op->aead.digest.data, (uint64_t)session->digest_length); } else if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) { - uint8_t *auth_tag = (uint8_t *)rte_pktmbuf_append(sym_op->m_dst ? - sym_op->m_dst : sym_op->m_src, - session->digest_length); - - if (!auth_tag) { - GCM_LOG_ERR("auth_tag"); - return -1; - } + uint8_t *auth_tag = qp->temp_digest; qp->ops[session->key].init(&session->gdata_key, &qp->gdata_ctx, @@ -350,14 +341,7 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, sym_op->auth.digest.data, (uint64_t)session->digest_length); } else { /* AESNI_GMAC_OP_VERIFY */ - uint8_t *auth_tag = (uint8_t *)rte_pktmbuf_append(sym_op->m_dst ? - sym_op->m_dst : sym_op->m_src, - session->digest_length); - - if (!auth_tag) { - GCM_LOG_ERR("auth_tag"); - return -1; - } + uint8_t *auth_tag = qp->temp_digest; qp->ops[session->key].init(&session->gdata_key, &qp->gdata_ctx, @@ -385,11 +369,10 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, * - Returns NULL on invalid job */ static void -post_process_gcm_crypto_op(struct rte_crypto_op *op, +post_process_gcm_crypto_op(struct aesni_gcm_qp *qp, + struct rte_crypto_op *op, struct aesni_gcm_session *session) { - struct rte_mbuf *m = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; - op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Verify digest if required */ @@ -397,8 +380,7 @@ post_process_gcm_crypto_op(struct rte_crypto_op *op, session->op == AESNI_GMAC_OP_VERIFY) { uint8_t *digest; - uint8_t *tag = rte_pktmbuf_mtod_offset(m, uint8_t *, - m->data_len - session->digest_length); + uint8_t *tag = (uint8_t *)&qp->temp_digest; if (session->op == AESNI_GMAC_OP_VERIFY) digest = op->sym->auth.digest.data; @@ -414,9 +396,6 @@ post_process_gcm_crypto_op(struct rte_crypto_op *op, if (memcmp(tag, digest, session->digest_length) != 0) op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - - /* trim area used for digest from mbuf */ - rte_pktmbuf_trim(m, session->digest_length); } } @@ -435,7 +414,7 @@ handle_completed_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, struct aesni_gcm_session *sess) { - post_process_gcm_crypto_op(op, sess); + post_process_gcm_crypto_op(qp, op, sess); /* Free session if a session-less crypto op */ if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { @@ -505,22 +484,24 @@ static int aesni_gcm_remove(struct rte_vdev_device *vdev); static int aesni_gcm_create(const char *name, struct rte_vdev_device *vdev, - struct rte_crypto_vdev_init_params *init_params) + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; struct aesni_gcm_private *internals; enum aesni_gcm_vector_mode vector_mode; - if (init_params->name[0] == '\0') - snprintf(init_params->name, sizeof(init_params->name), - "%s", name); - /* Check CPU for support for AES instruction set */ if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) { GCM_LOG_ERR("AES instructions not supported by CPU"); return -EFAULT; } + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); + if (dev == NULL) { + GCM_LOG_ERR("driver %s: create failed", init_params->name); + return -ENODEV; + } + /* Check CPU for supported vector instruction set */ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) vector_mode = RTE_AESNI_GCM_AVX2; @@ -529,14 +510,6 @@ aesni_gcm_create(const char *name, else vector_mode = RTE_AESNI_GCM_SSE; - dev = rte_cryptodev_vdev_pmd_init(init_params->name, - sizeof(struct aesni_gcm_private), init_params->socket_id, - vdev); - if (dev == NULL) { - GCM_LOG_ERR("failed to create cryptodev vdev"); - goto init_error; - } - dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_aesni_gcm_pmd_ops; @@ -571,22 +544,17 @@ aesni_gcm_create(const char *name, internals->max_nb_sessions = init_params->max_nb_sessions; return 0; - -init_error: - GCM_LOG_ERR("driver %s: create failed", init_params->name); - - aesni_gcm_remove(vdev); - return -EFAULT; } static int aesni_gcm_probe(struct rte_vdev_device *vdev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct aesni_gcm_private), rte_socket_id(), - {0} + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; const char *name; const char *input_args; @@ -595,17 +563,7 @@ aesni_gcm_probe(struct rte_vdev_device *vdev) if (name == NULL) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_vdev_parse_init_params(&init_params, input_args); - - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, - init_params.socket_id); - if (init_params.name[0] != '\0') - RTE_LOG(INFO, PMD, " User defined name = %s\n", - init_params.name); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); return aesni_gcm_create(name, vdev, &init_params); } @@ -613,16 +571,18 @@ aesni_gcm_probe(struct rte_vdev_device *vdev) static int aesni_gcm_remove(struct rte_vdev_device *vdev) { + struct rte_cryptodev *cryptodev; const char *name; name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL; - GCM_LOG_INFO("Closing AESNI crypto device %s on numa socket %u\n", - name, rte_socket_id()); + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; - return 0; + return rte_cryptodev_pmd_destroy(cryptodev); } static struct rte_vdev_driver aesni_gcm_pmd_drv = { @@ -630,10 +590,13 @@ static struct rte_vdev_driver aesni_gcm_pmd_drv = { .remove = aesni_gcm_remove }; +static struct cryptodev_driver aesni_gcm_crypto_drv; + RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_GCM_PMD, aesni_gcm_pmd_drv); RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_GCM_PMD, cryptodev_aesni_gcm_pmd); RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_GCM_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); -RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_gcm_pmd_drv, cryptodev_driver_id); +RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_gcm_crypto_drv, aesni_gcm_pmd_drv, + cryptodev_driver_id); diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c index 48400ac2..0f315f00 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c @@ -195,7 +195,7 @@ aesni_gcm_pmd_qp_set_unique_name(struct rte_cryptodev *dev, "aesni_gcm_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h index 7e155729..1c8835b5 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h @@ -58,6 +58,8 @@ #define GCM_LOG_DBG(fmt, args...) #endif +/* Maximum length for digest */ +#define DIGEST_LENGTH_MAX 16 /** private data structure for each virtual AESNI GCM device */ struct aesni_gcm_private { @@ -84,6 +86,11 @@ struct aesni_gcm_qp { /**< Queue Pair Identifier */ char name[RTE_CRYPTODEV_NAME_LEN]; /**< Unique Queue Pair Name */ + uint8_t temp_digest[DIGEST_LENGTH_MAX]; + /**< Buffer used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ } __rte_cache_aligned; diff --git a/drivers/crypto/aesni_mb/Makefile b/drivers/crypto/aesni_mb/Makefile index 611d4123..a49f06f2 100644 --- a/drivers/crypto/aesni_mb/Makefile +++ b/drivers/crypto/aesni_mb/Makefile @@ -47,12 +47,15 @@ CFLAGS += $(WERROR_FLAGS) LIBABIVER := 1 # versioning export map -EXPORT_MAP := rte_pmd_aesni_version.map +EXPORT_MAP := rte_pmd_aesni_mb_version.map # external library dependencies CFLAGS += -I$(AESNI_MULTI_BUFFER_LIB_PATH) CFLAGS += -I$(AESNI_MULTI_BUFFER_LIB_PATH)/include LDLIBS += -L$(AESNI_MULTI_BUFFER_LIB_PATH) -lIPSec_MB +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB) += rte_aesni_mb_pmd.c diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index 16e14512..70043897 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -30,12 +30,13 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include <des.h> + #include <rte_common.h> #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> -#include <rte_cryptodev_vdev.h> -#include <rte_vdev.h> +#include <rte_bus_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -188,6 +189,7 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform) { + uint8_t is_aes = 0; aes_keyexp_t aes_keyexp_fn; if (xform == NULL) { @@ -217,45 +219,68 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, switch (xform->cipher.algo) { case RTE_CRYPTO_CIPHER_AES_CBC: sess->cipher.mode = CBC; + is_aes = 1; break; case RTE_CRYPTO_CIPHER_AES_CTR: sess->cipher.mode = CNTR; + is_aes = 1; break; case RTE_CRYPTO_CIPHER_AES_DOCSISBPI: sess->cipher.mode = DOCSIS_SEC_BPI; + is_aes = 1; break; - default: - MB_LOG_ERR("Unsupported cipher mode parameter"); - return -ENOTSUP; - } - - /* Check key length and choose key expansion function */ - switch (xform->cipher.key.length) { - case AES_128_BYTES: - sess->cipher.key_length_in_bytes = AES_128_BYTES; - aes_keyexp_fn = mb_ops->aux.keyexp.aes128; - break; - case AES_192_BYTES: - sess->cipher.key_length_in_bytes = AES_192_BYTES; - aes_keyexp_fn = mb_ops->aux.keyexp.aes192; + case RTE_CRYPTO_CIPHER_DES_CBC: + sess->cipher.mode = DES; break; - case AES_256_BYTES: - sess->cipher.key_length_in_bytes = AES_256_BYTES; - aes_keyexp_fn = mb_ops->aux.keyexp.aes256; + case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: + sess->cipher.mode = DOCSIS_DES; break; default: - MB_LOG_ERR("Invalid cipher key length"); - return -EINVAL; + MB_LOG_ERR("Unsupported cipher mode parameter"); + return -ENOTSUP; } /* Set IV parameters */ sess->iv.offset = xform->cipher.iv.offset; sess->iv.length = xform->cipher.iv.length; - /* Expanded cipher keys */ - (*aes_keyexp_fn)(xform->cipher.key.data, - sess->cipher.expanded_aes_keys.encode, - sess->cipher.expanded_aes_keys.decode); + /* Check key length and choose key expansion function for AES */ + if (is_aes) { + switch (xform->cipher.key.length) { + case AES_128_BYTES: + sess->cipher.key_length_in_bytes = AES_128_BYTES; + aes_keyexp_fn = mb_ops->aux.keyexp.aes128; + break; + case AES_192_BYTES: + sess->cipher.key_length_in_bytes = AES_192_BYTES; + aes_keyexp_fn = mb_ops->aux.keyexp.aes192; + break; + case AES_256_BYTES: + sess->cipher.key_length_in_bytes = AES_256_BYTES; + aes_keyexp_fn = mb_ops->aux.keyexp.aes256; + break; + default: + MB_LOG_ERR("Invalid cipher key length"); + return -EINVAL; + } + + /* Expanded cipher keys */ + (*aes_keyexp_fn)(xform->cipher.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); + + } else { + if (xform->cipher.key.length != 8) { + MB_LOG_ERR("Invalid cipher key length"); + return -EINVAL; + } + sess->cipher.key_length_in_bytes = 8; + + des_key_schedule((uint64_t *)sess->cipher.expanded_aes_keys.encode, + xform->cipher.key.data); + des_key_schedule((uint64_t *)sess->cipher.expanded_aes_keys.decode, + xform->cipher.key.data); + } return 0; } @@ -407,7 +432,7 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op) */ static inline int set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, - struct rte_crypto_op *op) + struct rte_crypto_op *op, uint8_t *digest_idx) { struct rte_mbuf *m_src = op->sym->m_src, *m_dst; struct aesni_mb_session *session; @@ -466,19 +491,8 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, /* Set digest output location */ if (job->hash_alg != NULL_HASH && session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { - job->auth_tag_output = (uint8_t *)rte_pktmbuf_append(m_dst, - get_digest_byte_length(job->hash_alg)); - - if (job->auth_tag_output == NULL) { - MB_LOG_ERR("failed to allocate space in output mbuf " - "for temp digest"); - op->status = RTE_CRYPTO_OP_STATUS_ERROR; - return -1; - } - - memset(job->auth_tag_output, 0, - sizeof(get_digest_byte_length(job->hash_alg))); - + job->auth_tag_output = qp->temp_digests[*digest_idx]; + *digest_idx = (*digest_idx + 1) % MAX_JOBS; } else { job->auth_tag_output = op->sym->auth.digest.data; } @@ -507,22 +521,17 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, /* Set user data to be crypto operation data struct */ job->user_data = op; - job->user_data2 = m_dst; return 0; } static inline void -verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op) { - struct rte_mbuf *m_dst = (struct rte_mbuf *)job->user_data2; - +verify_digest(struct aesni_mb_qp *qp __rte_unused, JOB_AES_HMAC *job, + struct rte_crypto_op *op) { /* Verify digest if required */ if (memcmp(job->auth_tag_output, op->sym->auth.digest.data, job->auth_tag_output_len_in_bytes) != 0) op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - - /* trim area used for digest from mbuf */ - rte_pktmbuf_trim(m_dst, get_digest_byte_length(job->hash_alg)); } /** @@ -532,8 +541,7 @@ verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op) { * @param job JOB_AES_HMAC job to process * * @return - * - Returns processed crypto operation which mbuf is trimmed of output digest - * used in verification of supplied digest. + * - Returns processed crypto operation. * - Returns NULL on invalid job */ static inline struct rte_crypto_op * @@ -552,7 +560,7 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) if (job->hash_alg != NULL_HASH) { if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) - verify_digest(job, op); + verify_digest(qp, job, op); } break; default: @@ -626,13 +634,16 @@ flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops, } static inline JOB_AES_HMAC * -set_job_null_op(JOB_AES_HMAC *job) +set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op) { job->chain_order = HASH_CIPHER; job->cipher_mode = NULL_CIPHER; job->hash_alg = NULL_HASH; job->cipher_direction = DECRYPT; + /* Set user data to be crypto operation data struct */ + job->user_data = op; + return job; } @@ -650,6 +661,7 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, if (unlikely(nb_ops == 0)) return 0; + uint8_t digest_idx = qp->digest_idx; do { /* Get next operation to process from ingress queue */ retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op); @@ -667,10 +679,10 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, job = (*qp->op_fns->job.get_next)(&qp->mb_mgr); } - retval = set_mb_job_params(job, qp, op); + retval = set_mb_job_params(job, qp, op, &digest_idx); if (unlikely(retval != 0)) { qp->stats.dequeue_err_count++; - set_job_null_op(job); + set_job_null_op(job, op); } /* Submit job to multi-buffer for processing */ @@ -687,6 +699,8 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, } while (processed_jobs < nb_ops); + qp->digest_idx = digest_idx; + if (processed_jobs < 1) processed_jobs += flush_mb_mgr(qp, &ops[processed_jobs], @@ -700,15 +714,23 @@ static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev); static int cryptodev_aesni_mb_create(const char *name, struct rte_vdev_device *vdev, - struct rte_crypto_vdev_init_params *init_params) + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; struct aesni_mb_private *internals; enum aesni_mb_vector_mode vector_mode; - if (init_params->name[0] == '\0') - snprintf(init_params->name, sizeof(init_params->name), - "%s", name); + /* Check CPU for support for AES instruction set */ + if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) { + MB_LOG_ERR("AES instructions not supported by CPU"); + return -EFAULT; + } + + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); + if (dev == NULL) { + MB_LOG_ERR("failed to create cryptodev vdev"); + return -ENODEV; + } /* Check CPU for supported vector instruction set */ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) @@ -720,14 +742,6 @@ cryptodev_aesni_mb_create(const char *name, else vector_mode = RTE_AESNI_MB_SSE; - dev = rte_cryptodev_vdev_pmd_init(init_params->name, - sizeof(struct aesni_mb_private), init_params->socket_id, - vdev); - if (dev == NULL) { - MB_LOG_ERR("failed to create cryptodev vdev"); - goto init_error; - } - dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_aesni_mb_pmd_ops; @@ -764,41 +778,33 @@ cryptodev_aesni_mb_create(const char *name, internals->max_nb_sessions = init_params->max_nb_sessions; return 0; -init_error: - MB_LOG_ERR("driver %s: cryptodev_aesni_create failed", - init_params->name); - - cryptodev_aesni_mb_remove(vdev); - return -EFAULT; } static int cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct aesni_mb_private), rte_socket_id(), - "" + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; - const char *name; - const char *input_args; + const char *name, *args; + int retval; name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL; - input_args = rte_vdev_device_args(vdev); - rte_cryptodev_vdev_parse_init_params(&init_params, input_args); - - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, - init_params.socket_id); - if (init_params.name[0] != '\0') - RTE_LOG(INFO, PMD, " User defined name = %s\n", - init_params.name); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); + + args = rte_vdev_device_args(vdev); + + retval = rte_cryptodev_pmd_parse_input_args(&init_params, args); + if (retval) { + MB_LOG_ERR("Failed to parse initialisation arguments[%s]\n", + args); + return -EINVAL; + } return cryptodev_aesni_mb_create(name, vdev, &init_params); } @@ -806,16 +812,18 @@ cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev) static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev) { + struct rte_cryptodev *cryptodev; const char *name; name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL; - RTE_LOG(INFO, PMD, "Closing AESNI crypto device %s on numa socket %u\n", - name, rte_socket_id()); + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; - return 0; + return rte_cryptodev_pmd_destroy(cryptodev); } static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = { @@ -823,10 +831,14 @@ static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = { .remove = cryptodev_aesni_mb_remove }; +static struct cryptodev_driver aesni_mb_crypto_drv; + RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv); RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd); RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); -RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_aesni_mb_pmd_drv, cryptodev_driver_id); +RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv, + cryptodev_aesni_mb_pmd_drv, + cryptodev_driver_id); diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c index 692b354f..3b3ef0c0 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c @@ -247,6 +247,48 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { }, } }, } }, + { /* DES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_DES_CBC, + .block_size = 8, + .key_size = { + .min = 8, + .max = 8, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + { /* DES DOCSIS BPI */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI, + .block_size = 8, + .key_size = { + .min = 8, + .max = 8, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; @@ -355,7 +397,7 @@ aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev, "aesni_mb_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; @@ -373,7 +415,7 @@ aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp, "%s_%s", qp->name, str); - if (n > sizeof(ring_name)) + if (n >= sizeof(ring_name)) return NULL; r = rte_ring_lookup(ring_name); @@ -430,6 +472,11 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, memset(&qp->stats, 0, sizeof(qp->stats)); + char mp_name[RTE_MEMPOOL_NAMESIZE]; + + snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, + "digest_mp_%u_%u", dev->data->dev_id, qp_id); + /* Initialise multi-buffer manager */ (*qp->op_fns->job.init_mgr)(&qp->mb_mgr); return 0; diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h index 6676948e..fe3bd730 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h @@ -61,6 +61,8 @@ #define HMAC_IPAD_VALUE (0x36) #define HMAC_OPAD_VALUE (0x5C) +/* Maximum length for digest (SHA-512 truncated needs 32 bytes) */ +#define DIGEST_LENGTH_MAX 32 static const unsigned auth_blocksize[] = { [MD5] = 64, [SHA1] = 64, @@ -164,9 +166,17 @@ struct aesni_mb_qp { /**< Session Mempool */ struct rte_cryptodev_stats stats; /**< Queue pair statistics */ + uint8_t digest_idx; + /**< Index of the next slot to be used in temp_digests, + * to store the digest for a given operation + */ + uint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX]; + /**< Buffers used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ } __rte_cache_aligned; - /** AES-NI multi-buffer private session structure */ struct aesni_mb_session { JOB_CHAIN_ORDER chain_order; diff --git a/drivers/crypto/aesni_mb/rte_pmd_aesni_version.map b/drivers/crypto/aesni_mb/rte_pmd_aesni_mb_version.map index ad607bbe..ad607bbe 100644 --- a/drivers/crypto/aesni_mb/rte_pmd_aesni_version.map +++ b/drivers/crypto/aesni_mb/rte_pmd_aesni_mb_version.map diff --git a/drivers/crypto/armv8/Makefile b/drivers/crypto/armv8/Makefile index 86611fa2..79c260ff 100644 --- a/drivers/crypto/armv8/Makefile +++ b/drivers/crypto/armv8/Makefile @@ -51,12 +51,15 @@ CFLAGS += $(WERROR_FLAGS) LIBABIVER := 1 # versioning export map -EXPORT_MAP := rte_armv8_pmd_version.map +EXPORT_MAP := rte_pmd_armv8_version.map # external library dependencies CFLAGS += -I$(ARMV8_CRYPTO_LIB_PATH) CFLAGS += -I$(ARMV8_CRYPTO_LIB_PATH)/asm/include LDLIBS += -L$(ARMV8_CRYPTO_LIB_PATH) -larmv8_crypto +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_ARMV8_CRYPTO) += rte_armv8_pmd.c diff --git a/drivers/crypto/armv8/rte_armv8_pmd.c b/drivers/crypto/armv8/rte_armv8_pmd.c index a5c39c9b..97719f27 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd.c +++ b/drivers/crypto/armv8/rte_armv8_pmd.c @@ -36,8 +36,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> -#include <rte_cryptodev_vdev.h> -#include <rte_vdev.h> +#include <rte_bus_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -575,8 +574,8 @@ get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op) /** Process cipher operation */ static inline void -process_armv8_chained_op - (struct rte_crypto_op *op, struct armv8_crypto_session *sess, +process_armv8_chained_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op, + struct armv8_crypto_session *sess, struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) { crypto_func_t crypto_func; @@ -633,8 +632,7 @@ process_armv8_chained_op op->sym->auth.data.length); } } else { - adst = (uint8_t *)rte_pktmbuf_append(m_asrc, - sess->auth.digest_length); + adst = qp->temp_digest; } arg.cipher.iv = rte_crypto_op_ctod_offset(op, uint8_t *, @@ -655,15 +653,12 @@ process_armv8_chained_op sess->auth.digest_length) != 0) { op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } - /* Trim area used for digest from mbuf. */ - rte_pktmbuf_trim(m_asrc, - sess->auth.digest_length); } } /** Process crypto operation for mbuf */ static inline int -process_op(const struct armv8_crypto_qp *qp, struct rte_crypto_op *op, +process_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op, struct armv8_crypto_session *sess) { struct rte_mbuf *msrc, *mdst; @@ -676,7 +671,7 @@ process_op(const struct armv8_crypto_qp *qp, struct rte_crypto_op *op, switch (sess->chain_order) { case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: /* Fall through */ - process_armv8_chained_op(op, sess, msrc, mdst); + process_armv8_chained_op(qp, op, sess, msrc, mdst); break; default: op->status = RTE_CRYPTO_OP_STATUS_ERROR; @@ -763,7 +758,7 @@ armv8_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, static int cryptodev_armv8_crypto_create(const char *name, struct rte_vdev_device *vdev, - struct rte_crypto_vdev_init_params *init_params) + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; struct armv8_crypto_private *internals; @@ -790,14 +785,7 @@ cryptodev_armv8_crypto_create(const char *name, return -EFAULT; } - if (init_params->name[0] == '\0') - snprintf(init_params->name, sizeof(init_params->name), - "%s", name); - - dev = rte_cryptodev_vdev_pmd_init(init_params->name, - sizeof(struct armv8_crypto_private), - init_params->socket_id, - vdev); + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); if (dev == NULL) { ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); goto init_error; @@ -836,11 +824,12 @@ init_error: static int cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct armv8_crypto_private), rte_socket_id(), - {0} + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; const char *name; const char *input_args; @@ -849,18 +838,7 @@ cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev) if (name == NULL) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_vdev_parse_init_params(&init_params, input_args); - - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, - init_params.socket_id); - if (init_params.name[0] != '\0') { - RTE_LOG(INFO, PMD, " User defined name = %s\n", - init_params.name); - } - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); return cryptodev_armv8_crypto_create(name, vdev, &init_params); } @@ -869,6 +847,7 @@ cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev) static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev) { + struct rte_cryptodev *cryptodev; const char *name; name = rte_vdev_device_name(vdev); @@ -879,18 +858,25 @@ cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev) "Closing ARMv8 crypto device %s on numa socket %u\n", name, rte_socket_id()); - return 0; + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; + + return rte_cryptodev_pmd_destroy(cryptodev); } -static struct rte_vdev_driver armv8_crypto_drv = { +static struct rte_vdev_driver armv8_crypto_pmd_drv = { .probe = cryptodev_armv8_crypto_init, .remove = cryptodev_armv8_crypto_uninit }; -RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_drv); +static struct cryptodev_driver armv8_crypto_drv; + +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_pmd_drv); RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ARMV8_PMD, cryptodev_armv8_pmd); RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); -RTE_PMD_REGISTER_CRYPTO_DRIVER(armv8_crypto_drv, cryptodev_driver_id); +RTE_PMD_REGISTER_CRYPTO_DRIVER(armv8_crypto_drv, armv8_crypto_pmd_drv, + cryptodev_driver_id); diff --git a/drivers/crypto/armv8/rte_armv8_pmd_ops.c b/drivers/crypto/armv8/rte_armv8_pmd_ops.c index 00297beb..63776b26 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd_ops.c +++ b/drivers/crypto/armv8/rte_armv8_pmd_ops.c @@ -209,7 +209,7 @@ armv8_crypto_pmd_qp_set_unique_name(struct rte_cryptodev *dev, n = snprintf(qp->name, sizeof(qp->name), "armv8_crypto_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; diff --git a/drivers/crypto/armv8/rte_armv8_pmd_private.h b/drivers/crypto/armv8/rte_armv8_pmd_private.h index d02992a6..fa31f0a0 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd_private.h +++ b/drivers/crypto/armv8/rte_armv8_pmd_private.h @@ -69,6 +69,9 @@ do { \ #define NBBY 8 /* Number of bits in a byte */ #define BYTE_LENGTH(x) ((x) / NBBY) /* Number of bytes in x (round down) */ +/* Maximum length for digest (SHA-256 needs 32 bytes) */ +#define DIGEST_LENGTH_MAX 32 + /** ARMv8 operation order mode enumerator */ enum armv8_crypto_chain_order { ARMV8_CRYPTO_CHAIN_CIPHER_AUTH, @@ -147,6 +150,11 @@ struct armv8_crypto_qp { /**< Queue pair statistics */ char name[RTE_CRYPTODEV_NAME_LEN]; /**< Unique Queue Pair Name */ + uint8_t temp_digest[DIGEST_LENGTH_MAX]; + /**< Buffer used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ } __rte_cache_aligned; /** ARMv8 crypto private session structure */ diff --git a/drivers/crypto/armv8/rte_armv8_pmd_version.map b/drivers/crypto/armv8/rte_pmd_armv8_version.map index 1f84b68a..1f84b68a 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd_version.map +++ b/drivers/crypto/armv8/rte_pmd_armv8_version.map diff --git a/drivers/crypto/dpaa2_sec/Makefile b/drivers/crypto/dpaa2_sec/Makefile index ae15c99f..a08f2717 100644 --- a/drivers/crypto/dpaa2_sec/Makefile +++ b/drivers/crypto/dpaa2_sec/Makefile @@ -31,6 +31,12 @@ include $(RTE_SDK)/mk/rte.vars.mk +ifneq ($(MAKECMDGOALS),clean) +ifneq ($(CONFIG_RTE_LIBRTE_SECURITY),y) +$(error "RTE_LIBRTE_SECURITY is required to build RTE_LIBRTE_PMD_DPAA2_SEC") +endif +endif + # # library name # @@ -71,14 +77,9 @@ LIBABIVER := 1 SRCS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += dpaa2_sec_dpseci.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += mc/dpseci.c -# library dependencies -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += lib/librte_eal -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += lib/librte_mempool lib/librte_mbuf -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += lib/librte_cryptodev -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += drivers/bus/fslmc -DEPDIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_SEC) += drivers/mempool/dpaa2 - LDLIBS += -lrte_bus_fslmc LDLIBS += -lrte_mempool_dpaa2 +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c index e0f6cfca..9c64c5d9 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c @@ -36,6 +36,7 @@ #include <rte_mbuf.h> #include <rte_cryptodev.h> +#include <rte_security_driver.h> #include <rte_malloc.h> #include <rte_memcpy.h> #include <rte_string_fns.h> @@ -73,12 +74,44 @@ #define FLE_POOL_NUM_BUFS 32000 #define FLE_POOL_BUF_SIZE 256 #define FLE_POOL_CACHE_SIZE 512 +#define SEC_FLC_DHR_OUTBOUND -114 +#define SEC_FLC_DHR_INBOUND 0 enum rta_sec_era rta_sec_era = RTA_SEC_ERA_8; static uint8_t cryptodev_driver_id; static inline int +build_proto_fd(dpaa2_sec_session *sess, + struct rte_crypto_op *op, + struct qbman_fd *fd, uint16_t bpid) +{ + struct rte_crypto_sym_op *sym_op = op->sym; + struct ctxt_priv *priv = sess->ctxt; + struct sec_flow_context *flc; + struct rte_mbuf *mbuf = sym_op->m_src; + + if (likely(bpid < MAX_BPID)) + DPAA2_SET_FD_BPID(fd, bpid); + else + DPAA2_SET_FD_IVP(fd); + + /* Save the shared descriptor */ + flc = &priv->flc_desc[0].flc; + + DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FD_OFFSET(fd, sym_op->m_src->data_off); + DPAA2_SET_FD_LEN(fd, sym_op->m_src->pkt_len); + DPAA2_SET_FD_FLC(fd, ((uint64_t)flc)); + + /* save physical address of mbuf */ + op->sym->aead.digest.phys_addr = mbuf->buf_iova; + mbuf->buf_iova = (uint64_t)op; + + return 0; +} + +static inline int build_authenc_gcm_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, struct qbman_fd *fd, uint16_t bpid) @@ -90,11 +123,17 @@ build_authenc_gcm_fd(dpaa2_sec_session *sess, uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len; int icv_len = sess->digest_length, retval; uint8_t *old_icv; + struct rte_mbuf *dst; uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv.offset); PMD_INIT_FUNC_TRACE(); + if (sym_op->m_dst) + dst = sym_op->m_dst; + else + dst = sym_op->m_src; + /* TODO we are using the first FLE entry to store Mbuf and session ctxt. * Currently we donot know which FLE has the mbuf stored. * So while retreiving we can go back 1 FLE from the FD -ADDR @@ -155,9 +194,9 @@ build_authenc_gcm_fd(dpaa2_sec_session *sess, DPAA2_SET_FLE_SG_EXT(fle); /* Configure Output SGE for Encap/Decap */ - DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst)); DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset + - sym_op->m_src->data_off - auth_only_len); + dst->data_off - auth_only_len); sge->length = sym_op->aead.data.length + auth_only_len; if (sess->dir == DIR_ENC) { @@ -203,7 +242,6 @@ build_authenc_gcm_fd(dpaa2_sec_session *sess, old_icv = (uint8_t *)(sge + 1); memcpy(old_icv, sym_op->aead.digest.data, sess->digest_length); - memset(sym_op->aead.digest.data, 0, sess->digest_length); DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); sge->length = sess->digest_length; DPAA2_SET_FD_LEN(fd, (sym_op->aead.data.length + @@ -236,9 +274,15 @@ build_authenc_fd(dpaa2_sec_session *sess, uint8_t *old_icv; uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv.offset); + struct rte_mbuf *dst; PMD_INIT_FUNC_TRACE(); + if (sym_op->m_dst) + dst = sym_op->m_dst; + else + dst = sym_op->m_src; + /* we are using the first FLE entry to store Mbuf. * Currently we donot know which FLE has the mbuf stored. * So while retreiving we can go back 1 FLE from the FD -ADDR @@ -301,9 +345,9 @@ build_authenc_fd(dpaa2_sec_session *sess, DPAA2_SET_FLE_SG_EXT(fle); /* Configure Output SGE for Encap/Decap */ - DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst)); DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset + - sym_op->m_src->data_off); + dst->data_off); sge->length = sym_op->cipher.data.length; if (sess->dir == DIR_ENC) { @@ -342,7 +386,6 @@ build_authenc_fd(dpaa2_sec_session *sess, old_icv = (uint8_t *)(sge + 1); memcpy(old_icv, sym_op->auth.digest.data, sess->digest_length); - memset(sym_op->auth.digest.data, 0, sess->digest_length); DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); sge->length = sess->digest_length; DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length + @@ -436,7 +479,6 @@ build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, old_digest = (uint8_t *)(sge + 1); rte_memcpy(old_digest, sym_op->auth.digest.data, sess->digest_length); - memset(sym_op->auth.digest.data, 0, sess->digest_length); DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest)); sge->length = sess->digest_length; fle->length = sym_op->auth.data.length + @@ -459,9 +501,15 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, struct ctxt_priv *priv = sess->ctxt; uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv.offset); + struct rte_mbuf *dst; PMD_INIT_FUNC_TRACE(); + if (sym_op->m_dst) + dst = sym_op->m_dst; + else + dst = sym_op->m_src; + retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); if (retval) { RTE_LOG(ERR, PMD, "Memory alloc failed for SGE\n"); @@ -506,9 +554,9 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, sess->iv.length, sym_op->m_src->data_off); - DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); + DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(dst)); DPAA2_SET_FLE_OFFSET(fle, sym_op->cipher.data.offset + - sym_op->m_src->data_off); + dst->data_off); fle->length = sym_op->cipher.data.length + sess->iv.length; @@ -545,12 +593,29 @@ build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, } static inline int -build_sec_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, +build_sec_fd(struct rte_crypto_op *op, struct qbman_fd *fd, uint16_t bpid) { int ret = -1; + dpaa2_sec_session *sess; PMD_INIT_FUNC_TRACE(); + /* + * Segmented buffer is not supported. + */ + if (!rte_pktmbuf_is_contiguous(op->sym->m_src)) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return -ENOTSUP; + } + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) + sess = (dpaa2_sec_session *)get_session_private_data( + op->sym->session, cryptodev_driver_id); + else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) + sess = (dpaa2_sec_session *)get_sec_session_private_data( + op->sym->sec_session); + else + return -1; switch (sess->ctxt_type) { case DPAA2_SEC_CIPHER: @@ -565,6 +630,9 @@ build_sec_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, case DPAA2_SEC_CIPHER_HASH: ret = build_authenc_fd(sess, op, fd, bpid); break; + case DPAA2_SEC_IPSEC: + ret = build_proto_fd(sess, op, fd, bpid); + break; case DPAA2_SEC_HASH_CIPHER: default: RTE_LOG(ERR, PMD, "error: Unsupported session\n"); @@ -588,12 +656,11 @@ dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, /*todo - need to support multiple buffer pools */ uint16_t bpid; struct rte_mempool *mb_pool; - dpaa2_sec_session *sess; if (unlikely(nb_ops == 0)) return 0; - if (ops[0]->sess_type != RTE_CRYPTO_OP_WITH_SESSION) { + if (ops[0]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { RTE_LOG(ERR, PMD, "sessionless crypto op not supported\n"); return 0; } @@ -618,13 +685,9 @@ dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, for (loop = 0; loop < frames_to_send; loop++) { /*Clear the unused FD fields before sending*/ memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); - sess = (dpaa2_sec_session *) - get_session_private_data( - (*ops)->sym->session, - cryptodev_driver_id); mb_pool = (*ops)->sym->m_src->pool; bpid = mempool_to_bpid(mb_pool); - ret = build_sec_fd(sess, *ops, &fd_arr[loop], bpid); + ret = build_sec_fd(*ops, &fd_arr[loop], bpid); if (ret) { PMD_DRV_LOG(ERR, "error: Improper packet" " contents for crypto operation\n"); @@ -634,7 +697,7 @@ dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, } loop = 0; while (loop < frames_to_send) { - loop += qbman_swp_send_multiple(swp, &eqdesc, + loop += qbman_swp_enqueue_multiple(swp, &eqdesc, &fd_arr[loop], frames_to_send - loop); } @@ -649,11 +712,44 @@ skip_tx: } static inline struct rte_crypto_op * -sec_fd_to_mbuf(const struct qbman_fd *fd) +sec_simple_fd_to_mbuf(const struct qbman_fd *fd, __rte_unused uint8_t id) +{ + struct rte_crypto_op *op; + uint16_t len = DPAA2_GET_FD_LEN(fd); + uint16_t diff = 0; + dpaa2_sec_session *sess_priv; + + struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF( + DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)), + rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); + + op = (struct rte_crypto_op *)mbuf->buf_iova; + mbuf->buf_iova = op->sym->aead.digest.phys_addr; + op->sym->aead.digest.phys_addr = 0L; + + sess_priv = (dpaa2_sec_session *)get_sec_session_private_data( + op->sym->sec_session); + if (sess_priv->dir == DIR_ENC) + mbuf->data_off += SEC_FLC_DHR_OUTBOUND; + else + mbuf->data_off += SEC_FLC_DHR_INBOUND; + diff = len - mbuf->pkt_len; + mbuf->pkt_len += diff; + mbuf->data_len += diff; + + return op; +} + +static inline struct rte_crypto_op * +sec_fd_to_mbuf(const struct qbman_fd *fd, uint8_t driver_id) { struct qbman_fle *fle; struct rte_crypto_op *op; struct ctxt_priv *priv; + struct rte_mbuf *dst, *src; + + if (DPAA2_FD_GET_FORMAT(fd) == qbman_fd_single) + return sec_simple_fd_to_mbuf(fd, driver_id); fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); @@ -676,10 +772,17 @@ sec_fd_to_mbuf(const struct qbman_fd *fd) DPAA2_GET_FLE_ADDR((fle - 1))); /* Prefeth op */ - rte_prefetch0(op->sym->m_src); + src = op->sym->m_src; + rte_prefetch0(src); + + if (op->sym->m_dst) { + dst = op->sym->m_dst; + rte_prefetch0(dst); + } else + dst = src; PMD_RX_LOG(DEBUG, "mbuf %p BMAN buf addr %p", - (void *)op->sym->m_src, op->sym->m_src->buf_addr); + (void *)dst, dst->buf_addr); PMD_RX_LOG(DEBUG, "fdaddr =%p bpid =%d meta =%d off =%d, len =%d", (void *)DPAA2_GET_FD_ADDR(fd), @@ -701,6 +804,8 @@ dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, { /* Function is responsible to receive frames for a given device and VQ*/ struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp; + struct rte_cryptodev *dev = + (struct rte_cryptodev *)(dpaa2_qp->rx_vq.dev); struct qbman_result *dq_storage; uint32_t fqid = dpaa2_qp->rx_vq.fqid; int ret, num_rx = 0; @@ -747,13 +852,13 @@ dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, * Also seems like the SWP is shared between the Ethernet Driver * and the SEC driver. */ - while (!qbman_check_command_complete(swp, dq_storage)) + while (!qbman_check_command_complete(dq_storage)) ; /* Loop until the dq_storage is updated with * new token by QBMAN */ - while (!qbman_result_has_new_result(swp, dq_storage)) + while (!qbman_check_new_result(dq_storage)) ; /* Check whether Last Pull command is Expired and * setting Condition for Loop termination @@ -770,7 +875,7 @@ dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, } fd = qbman_result_DQ_fd(dq_storage); - ops[num_rx] = sec_fd_to_mbuf(fd); + ops[num_rx] = sec_fd_to_mbuf(fd, dev->driver_id); if (unlikely(fd->simple.frc)) { /* TODO Parse SEC errors */ @@ -1547,6 +1652,300 @@ dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev, } static int +dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev, + struct rte_security_session_conf *conf, + void *sess) +{ + struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec; + struct rte_crypto_auth_xform *auth_xform; + struct rte_crypto_cipher_xform *cipher_xform; + dpaa2_sec_session *session = (dpaa2_sec_session *)sess; + struct ctxt_priv *priv; + struct ipsec_encap_pdb encap_pdb; + struct ipsec_decap_pdb decap_pdb; + struct alginfo authdata, cipherdata; + unsigned int bufsize; + struct sec_flow_context *flc; + + PMD_INIT_FUNC_TRACE(); + + if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { + cipher_xform = &conf->crypto_xform->cipher; + auth_xform = &conf->crypto_xform->next->auth; + } else { + auth_xform = &conf->crypto_xform->auth; + cipher_xform = &conf->crypto_xform->next->cipher; + } + priv = (struct ctxt_priv *)rte_zmalloc(NULL, + sizeof(struct ctxt_priv) + + sizeof(struct sec_flc_desc), + RTE_CACHE_LINE_SIZE); + + if (priv == NULL) { + RTE_LOG(ERR, PMD, "\nNo memory for priv CTXT"); + return -ENOMEM; + } + + flc = &priv->flc_desc[0].flc; + + session->ctxt_type = DPAA2_SEC_IPSEC; + session->cipher_key.data = rte_zmalloc(NULL, + cipher_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->cipher_key.data == NULL && + cipher_xform->key.length > 0) { + RTE_LOG(ERR, PMD, "No Memory for cipher key\n"); + rte_free(priv); + return -ENOMEM; + } + + session->cipher_key.length = cipher_xform->key.length; + session->auth_key.data = rte_zmalloc(NULL, + auth_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->auth_key.data == NULL && + auth_xform->key.length > 0) { + RTE_LOG(ERR, PMD, "No Memory for auth key\n"); + rte_free(session->cipher_key.data); + rte_free(priv); + return -ENOMEM; + } + session->auth_key.length = auth_xform->key.length; + memcpy(session->cipher_key.data, cipher_xform->key.data, + cipher_xform->key.length); + memcpy(session->auth_key.data, auth_xform->key.data, + auth_xform->key.length); + + authdata.key = (uint64_t)session->auth_key.data; + authdata.keylen = session->auth_key.length; + authdata.key_enc_flags = 0; + authdata.key_type = RTA_DATA_IMM; + switch (auth_xform->algo) { + case RTE_CRYPTO_AUTH_SHA1_HMAC: + authdata.algtype = OP_PCL_IPSEC_HMAC_SHA1_96; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC; + break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + authdata.algtype = OP_PCL_IPSEC_HMAC_MD5_96; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + authdata.algtype = OP_PCL_IPSEC_HMAC_SHA2_256_128; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + authdata.algtype = OP_PCL_IPSEC_HMAC_SHA2_384_192; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA512_HMAC: + authdata.algtype = OP_PCL_IPSEC_HMAC_SHA2_512_256; + authdata.algmode = OP_ALG_AAI_HMAC; + session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; + break; + case RTE_CRYPTO_AUTH_AES_CMAC: + authdata.algtype = OP_PCL_IPSEC_AES_CMAC_96; + session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC; + break; + case RTE_CRYPTO_AUTH_NULL: + authdata.algtype = OP_PCL_IPSEC_HMAC_NULL; + session->auth_alg = RTE_CRYPTO_AUTH_NULL; + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + case RTE_CRYPTO_AUTH_SNOW3G_UIA2: + case RTE_CRYPTO_AUTH_SHA1: + case RTE_CRYPTO_AUTH_SHA256: + case RTE_CRYPTO_AUTH_SHA512: + case RTE_CRYPTO_AUTH_SHA224: + case RTE_CRYPTO_AUTH_SHA384: + case RTE_CRYPTO_AUTH_MD5: + case RTE_CRYPTO_AUTH_AES_GMAC: + case RTE_CRYPTO_AUTH_KASUMI_F9: + case RTE_CRYPTO_AUTH_AES_CBC_MAC: + case RTE_CRYPTO_AUTH_ZUC_EIA3: + RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u\n", + auth_xform->algo); + goto out; + default: + RTE_LOG(ERR, PMD, "Crypto: Undefined Auth specified %u\n", + auth_xform->algo); + goto out; + } + cipherdata.key = (uint64_t)session->cipher_key.data; + cipherdata.keylen = session->cipher_key.length; + cipherdata.key_enc_flags = 0; + cipherdata.key_type = RTA_DATA_IMM; + + switch (cipher_xform->algo) { + case RTE_CRYPTO_CIPHER_AES_CBC: + cipherdata.algtype = OP_PCL_IPSEC_AES_CBC; + cipherdata.algmode = OP_ALG_AAI_CBC; + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; + break; + case RTE_CRYPTO_CIPHER_3DES_CBC: + cipherdata.algtype = OP_PCL_IPSEC_3DES; + cipherdata.algmode = OP_ALG_AAI_CBC; + session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + cipherdata.algtype = OP_PCL_IPSEC_AES_CTR; + cipherdata.algmode = OP_ALG_AAI_CTR; + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; + break; + case RTE_CRYPTO_CIPHER_NULL: + cipherdata.algtype = OP_PCL_IPSEC_NULL; + break; + case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: + case RTE_CRYPTO_CIPHER_3DES_ECB: + case RTE_CRYPTO_CIPHER_AES_ECB: + case RTE_CRYPTO_CIPHER_KASUMI_F8: + RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u\n", + cipher_xform->algo); + goto out; + default: + RTE_LOG(ERR, PMD, "Crypto: Undefined Cipher specified %u\n", + cipher_xform->algo); + goto out; + } + + if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { + struct ip ip4_hdr; + + flc->dhr = SEC_FLC_DHR_OUTBOUND; + ip4_hdr.ip_v = IPVERSION; + ip4_hdr.ip_hl = 5; + ip4_hdr.ip_len = rte_cpu_to_be_16(sizeof(ip4_hdr)); + ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp; + ip4_hdr.ip_id = 0; + ip4_hdr.ip_off = 0; + ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl; + ip4_hdr.ip_p = 0x32; + ip4_hdr.ip_sum = 0; + ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip; + ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip; + ip4_hdr.ip_sum = calc_chksum((uint16_t *)(void *)&ip4_hdr, + sizeof(struct ip)); + + /* For Sec Proto only one descriptor is required. */ + memset(&encap_pdb, 0, sizeof(struct ipsec_encap_pdb)); + encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) | + PDBOPTS_ESP_OIHI_PDB_INL | + PDBOPTS_ESP_IVSRC | + PDBHMO_ESP_ENCAP_DTTL; + encap_pdb.spi = ipsec_xform->spi; + encap_pdb.ip_hdr_len = sizeof(struct ip); + + session->dir = DIR_ENC; + bufsize = cnstr_shdsc_ipsec_new_encap(priv->flc_desc[0].desc, + 1, 0, &encap_pdb, + (uint8_t *)&ip4_hdr, + &cipherdata, &authdata); + } else if (ipsec_xform->direction == + RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { + flc->dhr = SEC_FLC_DHR_INBOUND; + memset(&decap_pdb, 0, sizeof(struct ipsec_decap_pdb)); + decap_pdb.options = sizeof(struct ip) << 16; + session->dir = DIR_DEC; + bufsize = cnstr_shdsc_ipsec_new_decap(priv->flc_desc[0].desc, + 1, 0, &decap_pdb, &cipherdata, &authdata); + } else + goto out; + flc->word1_sdl = (uint8_t)bufsize; + + /* Enable the stashing control bit */ + DPAA2_SET_FLC_RSC(flc); + flc->word2_rflc_31_0 = lower_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq) | 0x14); + flc->word3_rflc_63_32 = upper_32_bits( + (uint64_t)&(((struct dpaa2_sec_qp *) + dev->data->queue_pairs[0])->rx_vq)); + + /* Set EWS bit i.e. enable write-safe */ + DPAA2_SET_FLC_EWS(flc); + /* Set BS = 1 i.e reuse input buffers as output buffers */ + DPAA2_SET_FLC_REUSE_BS(flc); + /* Set FF = 10; reuse input buffers if they provide sufficient space */ + DPAA2_SET_FLC_REUSE_FF(flc); + + session->ctxt = priv; + + return 0; +out: + rte_free(session->auth_key.data); + rte_free(session->cipher_key.data); + rte_free(priv); + return -1; +} + +static int +dpaa2_sec_security_session_create(void *dev, + struct rte_security_session_conf *conf, + struct rte_security_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev; + int ret; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + switch (conf->protocol) { + case RTE_SECURITY_PROTOCOL_IPSEC: + ret = dpaa2_sec_set_ipsec_session(cdev, conf, + sess_private_data); + break; + case RTE_SECURITY_PROTOCOL_MACSEC: + return -ENOTSUP; + default: + return -EINVAL; + } + if (ret != 0) { + PMD_DRV_LOG(ERR, + "DPAA2 PMD: failed to configure session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_sec_session_private_data(sess, sess_private_data); + + return ret; +} + +/** Clear the memory of session so it doesn't leave key material behind */ +static int +dpaa2_sec_security_session_destroy(void *dev __rte_unused, + struct rte_security_session *sess) +{ + PMD_INIT_FUNC_TRACE(); + void *sess_priv = get_sec_session_private_data(sess); + + dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv; + + if (sess_priv) { + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + + rte_free(s->ctxt); + rte_free(s->cipher_key.data); + rte_free(s->auth_key.data); + memset(sess, 0, sizeof(dpaa2_sec_session)); + set_sec_session_private_data(sess, NULL); + rte_mempool_put(sess_mp, sess_priv); + } + return 0; +} + +static int dpaa2_sec_session_configure(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, struct rte_cryptodev_sym_session *sess, @@ -1820,11 +2219,28 @@ static struct rte_cryptodev_ops crypto_ops = { .session_clear = dpaa2_sec_session_clear, }; +static const struct rte_security_capability * +dpaa2_sec_capabilities_get(void *device __rte_unused) +{ + return dpaa2_sec_security_cap; +} + +struct rte_security_ops dpaa2_sec_security_ops = { + .session_create = dpaa2_sec_security_session_create, + .session_update = NULL, + .session_stats_get = NULL, + .session_destroy = dpaa2_sec_security_session_destroy, + .set_pkt_metadata = NULL, + .capabilities_get = dpaa2_sec_capabilities_get +}; + static int dpaa2_sec_uninit(const struct rte_cryptodev *dev) { struct dpaa2_sec_dev_private *internals = dev->data->dev_private; + rte_free(dev->security_ctx); + rte_mempool_free(internals->fle_pool); PMD_INIT_LOG(INFO, "Closing DPAA2_SEC device %s on numa socket %u\n", @@ -1839,6 +2255,7 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) struct dpaa2_sec_dev_private *internals; struct rte_device *dev = cryptodev->device; struct rte_dpaa2_device *dpaa2_dev; + struct rte_security_ctx *security_instance; struct fsl_mc_io *dpseci; uint16_t token; struct dpseci_attr attr; @@ -1860,7 +2277,8 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) cryptodev->dequeue_burst = dpaa2_sec_dequeue_burst; cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | - RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING; + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_SECURITY; internals = cryptodev->data->dev_private; internals->max_nb_sessions = RTE_DPAA2_SEC_PMD_MAX_NB_SESSIONS; @@ -1874,6 +2292,17 @@ dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) PMD_INIT_LOG(DEBUG, "Device already init by primary process"); return 0; } + + /* Initialize security_ctx only for primary process*/ + security_instance = rte_malloc("rte_security_instances_ops", + sizeof(struct rte_security_ctx), 0); + if (security_instance == NULL) + return -ENOMEM; + security_instance->device = (void *)cryptodev; + security_instance->ops = &dpaa2_sec_security_ops; + security_instance->sess_cnt = 0; + cryptodev->security_ctx = security_instance; + /*Open the rte device via MC and save the handle for further use*/ dpseci = (struct fsl_mc_io *)rte_calloc(NULL, 1, sizeof(struct fsl_mc_io), 0); @@ -1987,20 +2416,11 @@ cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev) if (ret) return ret; - /* free crypto device */ - rte_cryptodev_pmd_release_device(cryptodev); - - if (rte_eal_process_type() == RTE_PROC_PRIMARY) - rte_free(cryptodev->data->dev_private); - - cryptodev->device = NULL; - cryptodev->data = NULL; - - return 0; + return rte_cryptodev_pmd_destroy(cryptodev); } static struct rte_dpaa2_driver rte_dpaa2_sec_driver = { - .drv_type = DPAA2_MC_DPSECI_DEVID, + .drv_type = DPAA2_CRYPTO, .driver = { .name = "DPAA2 SEC PMD" }, @@ -2008,5 +2428,8 @@ static struct rte_dpaa2_driver rte_dpaa2_sec_driver = { .remove = cryptodev_dpaa2_sec_remove, }; +static struct cryptodev_driver dpaa2_sec_crypto_drv; + RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver); -RTE_PMD_REGISTER_CRYPTO_DRIVER(rte_dpaa2_sec_driver, cryptodev_driver_id); +RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa2_sec_crypto_drv, rte_dpaa2_sec_driver, + cryptodev_driver_id); diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h index 3849a052..14e71df5 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h @@ -67,6 +67,11 @@ enum shr_desc_type { #define DIR_ENC 1 #define DIR_DEC 0 +#define DPAA2_SET_FLC_EWS(flc) (flc->word1_bits23_16 |= 0x1) +#define DPAA2_SET_FLC_RSC(flc) (flc->word1_bits31_24 |= 0x1) +#define DPAA2_SET_FLC_REUSE_BS(flc) (flc->mode_bits |= 0x8000) +#define DPAA2_SET_FLC_REUSE_FF(flc) (flc->mode_bits |= 0x2000) + /* SEC Flow Context Descriptor */ struct sec_flow_context { /* word 0 */ @@ -411,4 +416,61 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; + +static const struct rte_security_capability dpaa2_sec_security_cap[] = { + { /* IPsec Lookaside Protocol offload ESP Transport Egress */ + .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + .ipsec = { + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, + .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS, + .options = { 0 } + }, + .crypto_capabilities = dpaa2_sec_capabilities + }, + { /* IPsec Lookaside Protocol offload ESP Tunnel Ingress */ + .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + .ipsec = { + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, + .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS, + .options = { 0 } + }, + .crypto_capabilities = dpaa2_sec_capabilities + }, + { + .action = RTE_SECURITY_ACTION_TYPE_NONE + } +}; + +/** + * Checksum + * + * @param buffer calculate chksum for buffer + * @param len buffer length + * + * @return checksum value in host cpu order + */ +static inline uint16_t +calc_chksum(void *buffer, int len) +{ + uint16_t *buf = (uint16_t *)buffer; + uint32_t sum = 0; + uint16_t result; + + for (sum = 0; len > 1; len -= 2) + sum += *buf++; + + if (len == 1) + sum += *(unsigned char *)buf; + + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + result = ~sum; + + return result; +} + #endif /* _RTE_DPAA2_SEC_PMD_PRIVATE_H_ */ diff --git a/drivers/crypto/dpaa2_sec/mc/dpseci.c b/drivers/crypto/dpaa2_sec/mc/dpseci.c index 4a109620..2a216afc 100644 --- a/drivers/crypto/dpaa2_sec/mc/dpseci.c +++ b/drivers/crypto/dpaa2_sec/mc/dpseci.c @@ -37,18 +37,34 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ - #include <fsl_mc_sys.h> #include <fsl_mc_cmd.h> #include <fsl_dpseci.h> #include <fsl_dpseci_cmd.h> -int -dpseci_open(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - int dpseci_id, - uint16_t *token) +/** + * dpseci_open() - Open a control session for the specified object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpseci_id: DPSECI unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an + * already created object; an object may have been declared in + * the DPL or by calling the dpseci_create() function. + * This function returns a unique authentication token, + * associated with the specific object ID and the specific MC + * portal; this token must be used in all subsequent commands for + * this specific object. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpseci_id, + uint16_t *token) { + struct dpseci_cmd_open *cmd_params; struct mc_command cmd = { 0 }; int err; @@ -56,23 +72,34 @@ dpseci_open(struct fsl_mc_io *mc_io, cmd.header = mc_encode_cmd_header(DPSECI_CMDID_OPEN, cmd_flags, 0); - DPSECI_CMD_OPEN(cmd, dpseci_id); + cmd_params = (struct dpseci_cmd_open *)cmd.params; + cmd_params->dpseci_id = cpu_to_le32(dpseci_id); - /* send command to mc */ + /* send command to mc*/ err = mc_send_command(mc_io, &cmd); if (err) return err; /* retrieve response parameters */ - *token = MC_CMD_HDR_READ_TOKEN(cmd.header); + *token = mc_cmd_hdr_read_token(&cmd); return 0; } -int -dpseci_close(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token) +/** + * dpseci_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * After this function is called, no further operations are + * allowed on the object without opening a new control session. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) { struct mc_command cmd = { 0 }; @@ -81,478 +108,569 @@ dpseci_close(struct fsl_mc_io *mc_io, cmd_flags, token); - /* send command to mc */ + /* send command to mc*/ return mc_send_command(mc_io, &cmd); } -int -dpseci_create(struct fsl_mc_io *mc_io, - uint16_t dprc_token, - uint32_t cmd_flags, - const struct dpseci_cfg *cfg, - uint32_t *obj_id) +/** + * dpseci_create() - Create the DPSECI object + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @cfg: Configuration structure + * @obj_id: Returned object id + * + * Create the DPSECI object, allocate required resources and + * perform required initialization. + * + * The object can be created either by declaring it in the + * DPL file, or by calling this function. + * + * The function accepts an authentication token of a parent + * container that this object should be assigned to. The token + * can be '0' so the object will be assigned to the default container. + * The newly created object can be opened with the returned + * object id and using the container's associated tokens and MC portals. + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + const struct dpseci_cfg *cfg, + uint32_t *obj_id) { + struct dpseci_cmd_create *cmd_params; struct mc_command cmd = { 0 }; - int err; + int err, i; /* prepare command */ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CREATE, cmd_flags, dprc_token); - DPSECI_CMD_CREATE(cmd, cfg); - - /* send command to mc */ + cmd_params = (struct dpseci_cmd_create *)cmd.params; + for (i = 0; i < DPSECI_PRIO_NUM; i++) + cmd_params->priorities[i] = cfg->priorities[i]; + cmd_params->num_tx_queues = cfg->num_tx_queues; + cmd_params->num_rx_queues = cfg->num_rx_queues; + cmd_params->options = cfg->options; + + /* send command to mc*/ err = mc_send_command(mc_io, &cmd); if (err) return err; /* retrieve response parameters */ - CMD_CREATE_RSP_GET_OBJ_ID_PARAM0(cmd, *obj_id); + *obj_id = mc_cmd_read_object_id(&cmd); return 0; } -int -dpseci_destroy(struct fsl_mc_io *mc_io, - uint16_t dprc_token, - uint32_t cmd_flags, - uint32_t object_id) +/** + * dpseci_destroy() - Destroy the DPSECI object and release all its resources. + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @object_id: The object id; it must be a valid id within the container that + * created this object; + * + * The function accepts the authentication token of the parent container that + * created the object (not the one that currently owns the object). The object + * is searched within parent using the provided 'object_id'. + * All tokens to the object must be closed before calling destroy. + * + * Return: '0' on Success; error code otherwise. + */ +int dpseci_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id) { + struct dpseci_cmd_destroy *cmd_params; struct mc_command cmd = { 0 }; /* prepare command */ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DESTROY, cmd_flags, dprc_token); - /* set object id to destroy */ - CMD_DESTROY_SET_OBJ_ID_PARAM0(cmd, object_id); - /* send command to mc */ - return mc_send_command(mc_io, &cmd); -} - -int -dpseci_enable(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token) -{ - struct mc_command cmd = { 0 }; + cmd_params = (struct dpseci_cmd_destroy *)cmd.params; + cmd_params->dpseci_id = cpu_to_le32(object_id); - /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_ENABLE, - cmd_flags, - token); - - /* send command to mc */ + /* send command to mc*/ return mc_send_command(mc_io, &cmd); } -int -dpseci_disable(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token) -{ - struct mc_command cmd = { 0 }; - - /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DISABLE, - cmd_flags, - token); - - /* send command to mc */ - return mc_send_command(mc_io, &cmd); -} - -int -dpseci_is_enabled(struct fsl_mc_io *mc_io, +/** + * dpseci_enable() - Enable the DPSECI, allow sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_enable(struct fsl_mc_io *mc_io, uint32_t cmd_flags, - uint16_t token, - int *en) -{ - struct mc_command cmd = { 0 }; - int err; - /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_IS_ENABLED, - cmd_flags, - token); - - /* send command to mc */ - err = mc_send_command(mc_io, &cmd); - if (err) - return err; - - /* retrieve response parameters */ - DPSECI_RSP_IS_ENABLED(cmd, *en); - - return 0; -} - -int -dpseci_reset(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token) + uint16_t token) { struct mc_command cmd = { 0 }; /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_RESET, + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_ENABLE, cmd_flags, token); - /* send command to mc */ + /* send command to mc*/ return mc_send_command(mc_io, &cmd); } -int -dpseci_get_irq(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - int *type, - struct dpseci_irq_cfg *irq_cfg) -{ - struct mc_command cmd = { 0 }; - int err; - - /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ, - cmd_flags, - token); - DPSECI_CMD_GET_IRQ(cmd, irq_index); - - /* send command to mc */ - err = mc_send_command(mc_io, &cmd); - if (err) - return err; - - /* retrieve response parameters */ - DPSECI_RSP_GET_IRQ(cmd, *type, irq_cfg); - - return 0; -} - -int -dpseci_set_irq(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - struct dpseci_irq_cfg *irq_cfg) +/** + * dpseci_disable() - Disable the DPSECI, stop sending and receiving frames. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) { struct mc_command cmd = { 0 }; /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_IRQ, + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DISABLE, cmd_flags, token); - DPSECI_CMD_SET_IRQ(cmd, irq_index, irq_cfg); - /* send command to mc */ + /* send command to mc*/ return mc_send_command(mc_io, &cmd); } -int -dpseci_get_irq_enable(struct fsl_mc_io *mc_io, +/** + * dpseci_is_enabled() - Check if the DPSECI is enabled. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @en: Returns '1' if object is enabled; '0' otherwise + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_is_enabled(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, - uint8_t irq_index, - uint8_t *en) + int *en) { + struct dpseci_rsp_is_enabled *rsp_params; struct mc_command cmd = { 0 }; int err; /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ_ENABLE, + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_IS_ENABLED, cmd_flags, token); - DPSECI_CMD_GET_IRQ_ENABLE(cmd, irq_index); - /* send command to mc */ + /* send command to mc*/ err = mc_send_command(mc_io, &cmd); if (err) return err; /* retrieve response parameters */ - DPSECI_RSP_GET_IRQ_ENABLE(cmd, *en); + rsp_params = (struct dpseci_rsp_is_enabled *)cmd.params; + *en = dpseci_get_field(rsp_params->en, ENABLE); return 0; } -int -dpseci_set_irq_enable(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - uint8_t en) +/** + * dpseci_reset() - Reset the DPSECI, returns the object to initial state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token) { struct mc_command cmd = { 0 }; /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_IRQ_ENABLE, + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_RESET, cmd_flags, token); - DPSECI_CMD_SET_IRQ_ENABLE(cmd, irq_index, en); - /* send command to mc */ + /* send command to mc*/ return mc_send_command(mc_io, &cmd); } -int -dpseci_get_irq_mask(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - uint32_t *mask) +/** + * dpseci_get_attributes() - Retrieve DPSECI attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @attr: Returned object's attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_attr *attr) { + struct dpseci_rsp_get_attr *rsp_params; struct mc_command cmd = { 0 }; int err; /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ_MASK, + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_ATTR, cmd_flags, token); - DPSECI_CMD_GET_IRQ_MASK(cmd, irq_index); - /* send command to mc */ + /* send command to mc*/ err = mc_send_command(mc_io, &cmd); if (err) return err; /* retrieve response parameters */ - DPSECI_RSP_GET_IRQ_MASK(cmd, *mask); + rsp_params = (struct dpseci_rsp_get_attr *)cmd.params; + attr->id = le32_to_cpu(rsp_params->id); + attr->options = rsp_params->options; + attr->num_tx_queues = rsp_params->num_tx_queues; + attr->num_rx_queues = rsp_params->num_rx_queues; return 0; } -int -dpseci_set_irq_mask(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - uint32_t mask) +/** + * dpseci_set_rx_queue() - Set Rx queue configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @queue: Select the queue relative to number of + * priorities configured at DPSECI creation; use + * DPSECI_ALL_QUEUES to configure all Rx queues identically. + * @cfg: Rx queue configuration + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t queue, + const struct dpseci_rx_queue_cfg *cfg) { + struct dpseci_cmd_set_rx_queue *cmd_params; struct mc_command cmd = { 0 }; /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_IRQ_MASK, + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_RX_QUEUE, cmd_flags, token); - DPSECI_CMD_SET_IRQ_MASK(cmd, irq_index, mask); - - /* send command to mc */ + cmd_params = (struct dpseci_cmd_set_rx_queue *)cmd.params; + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); + cmd_params->dest_priority = cfg->dest_cfg.priority; + cmd_params->queue = queue; + cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); + cmd_params->options = cpu_to_le32(cfg->options); + dpseci_set_field(cmd_params->dest_type, + DEST_TYPE, + cfg->dest_cfg.dest_type); + dpseci_set_field(cmd_params->order_preservation_en, + ORDER_PRESERVATION, + cfg->order_preservation_en); + + /* send command to mc*/ return mc_send_command(mc_io, &cmd); } -int -dpseci_get_irq_status(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - uint32_t *status) +/** + * dpseci_get_rx_queue() - Retrieve Rx queue attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @queue: Select the queue relative to number of + * priorities configured at DPSECI creation + * @attr: Returned Rx queue attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t queue, + struct dpseci_rx_queue_attr *attr) { + struct dpseci_rsp_get_rx_queue *rsp_params; + struct dpseci_cmd_get_queue *cmd_params; struct mc_command cmd = { 0 }; int err; /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ_STATUS, + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_RX_QUEUE, cmd_flags, token); - DPSECI_CMD_GET_IRQ_STATUS(cmd, irq_index, *status); + cmd_params = (struct dpseci_cmd_get_queue *)cmd.params; + cmd_params->queue = queue; - /* send command to mc */ + /* send command to mc*/ err = mc_send_command(mc_io, &cmd); if (err) return err; /* retrieve response parameters */ - DPSECI_RSP_GET_IRQ_STATUS(cmd, *status); + rsp_params = (struct dpseci_rsp_get_rx_queue *)cmd.params; + attr->user_ctx = le64_to_cpu(rsp_params->user_ctx); + attr->fqid = le32_to_cpu(rsp_params->fqid); + attr->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id); + attr->dest_cfg.priority = rsp_params->dest_priority; + attr->dest_cfg.dest_type = + dpseci_get_field(rsp_params->dest_type, + DEST_TYPE); + attr->order_preservation_en = + dpseci_get_field(rsp_params->order_preservation_en, + ORDER_PRESERVATION); return 0; } -int -dpseci_clear_irq_status(struct fsl_mc_io *mc_io, +/** + * dpseci_get_tx_queue() - Retrieve Tx queue attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @queue: Select the queue relative to number of + * priorities configured at DPSECI creation + * @attr: Returned Tx queue attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, - uint8_t irq_index, - uint32_t status) -{ - struct mc_command cmd = { 0 }; - - /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CLEAR_IRQ_STATUS, - cmd_flags, - token); - DPSECI_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status); - - /* send command to mc */ - return mc_send_command(mc_io, &cmd); -} - -int -dpseci_get_attributes(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - struct dpseci_attr *attr) + uint8_t queue, + struct dpseci_tx_queue_attr *attr) { + struct dpseci_rsp_get_tx_queue *rsp_params; + struct dpseci_cmd_get_queue *cmd_params; struct mc_command cmd = { 0 }; int err; /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_ATTR, + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_TX_QUEUE, cmd_flags, token); + cmd_params = (struct dpseci_cmd_get_queue *)cmd.params; + cmd_params->queue = queue; - /* send command to mc */ + /* send command to mc*/ err = mc_send_command(mc_io, &cmd); if (err) return err; /* retrieve response parameters */ - DPSECI_RSP_GET_ATTR(cmd, attr); + rsp_params = (struct dpseci_rsp_get_tx_queue *)cmd.params; + attr->fqid = le32_to_cpu(rsp_params->fqid); + attr->priority = rsp_params->priority; return 0; } -int -dpseci_set_rx_queue(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t queue, - const struct dpseci_rx_queue_cfg *cfg) -{ - struct mc_command cmd = { 0 }; - - /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_RX_QUEUE, - cmd_flags, - token); - DPSECI_CMD_SET_RX_QUEUE(cmd, queue, cfg); - - /* send command to mc */ - return mc_send_command(mc_io, &cmd); -} - -int -dpseci_get_rx_queue(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t queue, - struct dpseci_rx_queue_attr *attr) +/** + * dpseci_get_sec_attr() - Retrieve SEC accelerator attributes. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @attr: Returned SEC attributes + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_sec_attr *attr) { + struct dpseci_rsp_get_sec_attr *rsp_params; struct mc_command cmd = { 0 }; int err; /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_RX_QUEUE, + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_ATTR, cmd_flags, token); - DPSECI_CMD_GET_RX_QUEUE(cmd, queue); - /* send command to mc */ + /* send command to mc*/ err = mc_send_command(mc_io, &cmd); if (err) return err; /* retrieve response parameters */ - DPSECI_RSP_GET_RX_QUEUE(cmd, attr); + rsp_params = (struct dpseci_rsp_get_sec_attr *)cmd.params; + attr->ip_id = le16_to_cpu(rsp_params->ip_id); + attr->major_rev = rsp_params->major_rev; + attr->minor_rev = rsp_params->minor_rev; + attr->era = rsp_params->era; + attr->deco_num = rsp_params->deco_num; + attr->zuc_auth_acc_num = rsp_params->zuc_auth_acc_num; + attr->zuc_enc_acc_num = rsp_params->zuc_enc_acc_num; + attr->snow_f8_acc_num = rsp_params->snow_f8_acc_num; + attr->snow_f9_acc_num = rsp_params->snow_f9_acc_num; + attr->crc_acc_num = rsp_params->crc_acc_num; + attr->pk_acc_num = rsp_params->pk_acc_num; + attr->kasumi_acc_num = rsp_params->kasumi_acc_num; + attr->rng_acc_num = rsp_params->rng_acc_num; + attr->md_acc_num = rsp_params->md_acc_num; + attr->arc4_acc_num = rsp_params->arc4_acc_num; + attr->des_acc_num = rsp_params->des_acc_num; + attr->aes_acc_num = rsp_params->aes_acc_num; return 0; } -int -dpseci_get_tx_queue(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t queue, - struct dpseci_tx_queue_attr *attr) +/** + * dpseci_get_sec_counters() - Retrieve SEC accelerator counters. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @counters: Returned SEC counters + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_get_sec_counters(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_sec_counters *counters) { + struct dpseci_rsp_get_sec_counters *rsp_params; struct mc_command cmd = { 0 }; int err; /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_TX_QUEUE, + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_COUNTERS, cmd_flags, token); - DPSECI_CMD_GET_TX_QUEUE(cmd, queue); - /* send command to mc */ + /* send command to mc*/ err = mc_send_command(mc_io, &cmd); if (err) return err; /* retrieve response parameters */ - DPSECI_RSP_GET_TX_QUEUE(cmd, attr); + rsp_params = (struct dpseci_rsp_get_sec_counters *)cmd.params; + counters->dequeued_requests = + le64_to_cpu(rsp_params->dequeued_requests); + counters->ob_enc_requests = le64_to_cpu(rsp_params->ob_enc_requests); + counters->ib_dec_requests = le64_to_cpu(rsp_params->ib_dec_requests); + counters->ob_enc_bytes = le64_to_cpu(rsp_params->ob_enc_bytes); + counters->ob_prot_bytes = le64_to_cpu(rsp_params->ob_prot_bytes); + counters->ib_dec_bytes = le64_to_cpu(rsp_params->ib_dec_bytes); + counters->ib_valid_bytes = le64_to_cpu(rsp_params->ib_valid_bytes); return 0; } -int -dpseci_get_sec_attr(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - struct dpseci_sec_attr *attr) +/** + * dpseci_get_api_version() - Get Data Path SEC Interface API version + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @major_ver: Major version of data path sec API + * @minor_ver: Minor version of data path sec API + * + * Return: '0' on Success; Error code otherwise. + */ +int dpseci_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver) { + struct dpseci_rsp_get_api_version *rsp_params; struct mc_command cmd = { 0 }; int err; - /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_ATTR, - cmd_flags, - token); + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_API_VERSION, + cmd_flags, + 0); - /* send command to mc */ err = mc_send_command(mc_io, &cmd); if (err) return err; - /* retrieve response parameters */ - DPSECI_RSP_GET_SEC_ATTR(cmd, attr); + rsp_params = (struct dpseci_rsp_get_api_version *)cmd.params; + *major_ver = le16_to_cpu(rsp_params->major); + *minor_ver = le16_to_cpu(rsp_params->minor); return 0; } -int -dpseci_get_sec_counters(struct fsl_mc_io *mc_io, +int dpseci_set_congestion_notification( + struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, - struct dpseci_sec_counters *counters) + const struct dpseci_congestion_notification_cfg *cfg) { + struct dpseci_cmd_set_congestion_notification *cmd_params; struct mc_command cmd = { 0 }; - int err; /* prepare command */ - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_COUNTERS, - cmd_flags, - token); - - /* send command to mc */ - err = mc_send_command(mc_io, &cmd); - if (err) - return err; - - /* retrieve response parameters */ - DPSECI_RSP_GET_SEC_COUNTERS(cmd, counters); - - return 0; + cmd.header = mc_encode_cmd_header( + DPSECI_CMDID_SET_CONGESTION_NOTIFICATION, + cmd_flags, + token); + + cmd_params = + (struct dpseci_cmd_set_congestion_notification *)cmd.params; + cmd_params->dest_id = cfg->dest_cfg.dest_id; + cmd_params->dest_priority = cfg->dest_cfg.priority; + cmd_params->message_ctx = cfg->message_ctx; + cmd_params->message_iova = cfg->message_iova; + cmd_params->notification_mode = cfg->notification_mode; + cmd_params->threshold_entry = cfg->threshold_entry; + cmd_params->threshold_exit = cfg->threshold_exit; + dpseci_set_field(cmd_params->type_units, + DEST_TYPE, + cfg->dest_cfg.dest_type); + dpseci_set_field(cmd_params->type_units, + CG_UNITS, + cfg->units); + + /* send command to mc*/ + return mc_send_command(mc_io, &cmd); } -int -dpseci_get_api_version(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t *major_ver, - uint16_t *minor_ver) +int dpseci_get_congestion_notification( + struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_congestion_notification_cfg *cfg) { + struct dpseci_cmd_set_congestion_notification *rsp_params; struct mc_command cmd = { 0 }; int err; - cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_API_VERSION, - cmd_flags, - 0); + /* prepare command */ + cmd.header = mc_encode_cmd_header( + DPSECI_CMDID_GET_CONGESTION_NOTIFICATION, + cmd_flags, + token); + /* send command to mc*/ err = mc_send_command(mc_io, &cmd); if (err) return err; - DPSECI_RSP_GET_API_VERSION(cmd, *major_ver, *minor_ver); + rsp_params = + (struct dpseci_cmd_set_congestion_notification *)cmd.params; + + cfg->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id); + cfg->dest_cfg.priority = rsp_params->dest_priority; + cfg->notification_mode = le16_to_cpu(rsp_params->notification_mode); + cfg->message_ctx = le64_to_cpu(rsp_params->message_ctx); + cfg->message_iova = le64_to_cpu(rsp_params->message_iova); + cfg->threshold_entry = le32_to_cpu(rsp_params->threshold_entry); + cfg->threshold_exit = le32_to_cpu(rsp_params->threshold_exit); + cfg->units = dpseci_get_field(rsp_params->type_units, CG_UNITS); + cfg->dest_cfg.dest_type = dpseci_get_field(rsp_params->type_units, + DEST_TYPE); return 0; } diff --git a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h index 6cc14a66..4acb595c 100644 --- a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h +++ b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright 2016 NXP. + * Copyright 2016-2017 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -37,7 +37,6 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ - #ifndef __FSL_DPSECI_H #define __FSL_DPSECI_H @@ -61,394 +60,89 @@ struct fsl_mc_io; */ #define DPSECI_ALL_QUEUES (uint8_t)(-1) -/** - * dpseci_open() - Open a control session for the specified object - * This function can be used to open a control session for an - * already created object; an object may have been declared in - * the DPL or by calling the dpseci_create() function. - * This function returns a unique authentication token, - * associated with the specific object ID and the specific MC - * portal; this token must be used in all subsequent commands for - * this specific object. - * - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param dpseci_id DPSECI unique ID - * @param token Returned token; use in subsequent API calls - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_open(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - int dpseci_id, - uint16_t *token); +int dpseci_open(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + int dpseci_id, + uint16_t *token); + +int dpseci_close(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); /** - * dpseci_close() - Close the control session of the object - * After this function is called, no further operations are - * allowed on the object without opening a new control session. - * - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. + * Enable the Congestion Group support */ -int -dpseci_close(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token); +#define DPSECI_OPT_HAS_CG 0x000020 /** * struct dpseci_cfg - Structure representing DPSECI configuration + * @options: Any combination of the following options: + * DPSECI_OPT_HAS_CG + * DPSECI_OPT_HAS_OPR + * DPSECI_OPT_OPR_SHARED + * @num_tx_queues: num of queues towards the SEC + * @num_rx_queues: num of queues back from the SEC + * @priorities: Priorities for the SEC hardware processing; + * each place in the array is the priority of the tx queue + * towards the SEC, + * valid priorities are configured with values 1-8; */ struct dpseci_cfg { - uint8_t num_tx_queues; /* num of queues towards the SEC */ - uint8_t num_rx_queues; /* num of queues back from the SEC */ + uint32_t options; + uint8_t num_tx_queues; + uint8_t num_rx_queues; uint8_t priorities[DPSECI_PRIO_NUM]; - /**< Priorities for the SEC hardware processing; - * each place in the array is the priority of the tx queue - * towards the SEC, - * valid priorities are configured with values 1-8; - */ }; -/** - * dpseci_create() - Create the DPSECI object - * Create the DPSECI object, allocate required resources and - * perform required initialization. - * - * The object can be created either by declaring it in the - * DPL file, or by calling this function. - * - * The function accepts an authentication token of a parent - * container that this object should be assigned to. The token - * can be '0' so the object will be assigned to the default container. - * The newly created object can be opened with the returned - * object id and using the container's associated tokens and MC portals. - * - * @param mc_io Pointer to MC portal's I/O object - * @param dprc_token Parent container token; '0' for default container - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param cfg Configuration structure - * @param obj_id returned object id - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_create(struct fsl_mc_io *mc_io, - uint16_t dprc_token, - uint32_t cmd_flags, - const struct dpseci_cfg *cfg, - uint32_t *obj_id); - -/** - * dpseci_destroy() - Destroy the DPSECI object and release all its resources. - * The function accepts the authentication token of the parent container that - * created the object (not the one that currently owns the object). The object - * is searched within parent using the provided 'object_id'. - * All tokens to the object must be closed before calling destroy. - * - * @param mc_io Pointer to MC portal's I/O object - * @param dprc_token Parent container token; '0' for default container - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param object_id The object id; it must be a valid id within the - * container that created this object; - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_destroy(struct fsl_mc_io *mc_io, - uint16_t dprc_token, - uint32_t cmd_flags, - uint32_t object_id); - -/** - * dpseci_enable() - Enable the DPSECI, allow sending and receiving frames. - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_enable(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token); - -/** - * dpseci_disable() - Disable the DPSECI, stop sending and receiving frames. - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_disable(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token); - -/** - * dpseci_is_enabled() - Check if the DPSECI is enabled. - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param en Returns '1' if object is enabled; '0' otherwise - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_is_enabled(struct fsl_mc_io *mc_io, +int dpseci_create(struct fsl_mc_io *mc_io, + uint16_t dprc_token, uint32_t cmd_flags, - uint16_t token, - int *en); + const struct dpseci_cfg *cfg, + uint32_t *obj_id); -/** - * dpseci_reset() - Reset the DPSECI, returns the object to initial state. - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_reset(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token); +int dpseci_destroy(struct fsl_mc_io *mc_io, + uint16_t dprc_token, + uint32_t cmd_flags, + uint32_t object_id); -/** - * struct dpseci_irq_cfg - IRQ configuration - */ -struct dpseci_irq_cfg { - uint64_t addr; - /* Address that must be written to signal a message-based interrupt */ - uint32_t val; - /* Value to write into irq_addr address */ - int irq_num; - /* A user defined number associated with this IRQ */ -}; +int dpseci_enable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); -/** - * dpseci_set_irq() - Set IRQ information for the DPSECI to trigger an interrupt - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param irq_index Identifies the interrupt index to configure - * @param irq_cfg IRQ configuration - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_set_irq(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - struct dpseci_irq_cfg *irq_cfg); +int dpseci_disable(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); -/** - * dpseci_get_irq() - Get IRQ information from the DPSECI - * - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param irq_index The interrupt index to configure - * @param type Interrupt type: 0 represents message interrupt - * type (both irq_addr and irq_val are valid) - * @param irq_cfg IRQ attributes - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_get_irq(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - int *type, - struct dpseci_irq_cfg *irq_cfg); - -/** - * dpseci_set_irq_enable() - Set overall interrupt state. - * Allows GPP software to control when interrupts are generated. - * Each interrupt can have up to 32 causes. The enable/disable control's the - * overall interrupt state. if the interrupt is disabled no causes will cause - * an interrupt - * - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param irq_index The interrupt index to configure - * @param en Interrupt state - enable = 1, disable = 0 - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_set_irq_enable(struct fsl_mc_io *mc_io, +int dpseci_is_enabled(struct fsl_mc_io *mc_io, uint32_t cmd_flags, uint16_t token, - uint8_t irq_index, - uint8_t en); + int *en); -/** - * dpseci_get_irq_enable() - Get overall interrupt state - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param irq_index The interrupt index to configure - * @param en Returned Interrupt state - enable = 1, - * disable = 0 - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_get_irq_enable(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - uint8_t *en); - -/** - * dpseci_set_irq_mask() - Set interrupt mask. - * Every interrupt can have up to 32 causes and the interrupt model supports - * masking/unmasking each cause independently - * - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param irq_index The interrupt index to configure - * @param mask event mask to trigger interrupt; - * each bit: - * 0 = ignore event - * 1 = consider event for asserting IRQ - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_set_irq_mask(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - uint32_t mask); - -/** - * dpseci_get_irq_mask() - Get interrupt mask. - * Every interrupt can have up to 32 causes and the interrupt model supports - * masking/unmasking each cause independently - * - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param irq_index The interrupt index to configure - * @param mask Returned event mask to trigger interrupt - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_get_irq_mask(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - uint32_t *mask); - -/** - * dpseci_get_irq_status() - Get the current status of any pending interrupts - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param irq_index The interrupt index to configure - * @param status Returned interrupts status - one bit per cause: - * 0 = no interrupt pending - * 1 = interrupt pending - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_get_irq_status(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - uint32_t *status); - -/** - * dpseci_clear_irq_status() - Clear a pending interrupt's status - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param irq_index The interrupt index to configure - * @param status bits to clear (W1C) - one bit per cause: - * 0 = don't change - * 1 = clear status bit - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_clear_irq_status(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t irq_index, - uint32_t status); +int dpseci_reset(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token); /** * struct dpseci_attr - Structure representing DPSECI attributes - * @param id: DPSECI object ID - * @param num_tx_queues: number of queues towards the SEC - * @param num_rx_queues: number of queues back from the SEC + * @id: DPSECI object ID + * @num_tx_queues: number of queues towards the SEC + * @num_rx_queues: number of queues back from the SEC + * @options: Any combination of the following options: + * DPSECI_OPT_HAS_CG + * DPSECI_OPT_HAS_OPR + * DPSECI_OPT_OPR_SHARED */ struct dpseci_attr { - int id; /* DPSECI object ID */ - uint8_t num_tx_queues; /* number of queues towards the SEC */ - uint8_t num_rx_queues; /* number of queues back from the SEC */ + int id; + uint8_t num_tx_queues; + uint8_t num_rx_queues; + uint32_t options; }; -/** - * dpseci_get_attributes() - Retrieve DPSECI attributes. - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param attr Returned object's attributes - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_get_attributes(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - struct dpseci_attr *attr); +int dpseci_get_attributes(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_attr *attr); /** * enum dpseci_dest - DPSECI destination types @@ -471,16 +165,16 @@ enum dpseci_dest { /** * struct dpseci_dest_cfg - Structure representing DPSECI destination parameters + * @dest_type: Destination type + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type + * @priority: Priority selection within the DPIO or DPCON channel; valid values + * are 0-1 or 0-7, depending on the number of priorities in that + * channel; not relevant for 'DPSECI_DEST_NONE' option */ struct dpseci_dest_cfg { - enum dpseci_dest dest_type; /* Destination type */ + enum dpseci_dest dest_type; int dest_id; - /* Either DPIO ID or DPCON ID, depending on the destination type */ uint8_t priority; - /* Priority selection within the DPIO or DPCON channel; valid values - * are 0-1 or 0-7, depending on the number of priorities in that - * channel; not relevant for 'DPSECI_DEST_NONE' option - */ }; /** @@ -504,243 +198,235 @@ struct dpseci_dest_cfg { /** * struct dpseci_rx_queue_cfg - DPSECI RX queue configuration + * @options: Flags representing the suggested modifications to the queue; + * Use any combination of 'DPSECI_QUEUE_OPT_<X>' flags + * @order_preservation_en: order preservation configuration for the rx queue + * valid only if 'DPSECI_QUEUE_OPT_ORDER_PRESERVATION' is contained in 'options' + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame; + * valid only if 'DPSECI_QUEUE_OPT_USER_CTX' is contained in 'options' + * @dest_cfg: Queue destination parameters; + * valid only if 'DPSECI_QUEUE_OPT_DEST' is contained in 'options' */ struct dpseci_rx_queue_cfg { uint32_t options; - /* Flags representing the suggested modifications to the queue; - * Use any combination of 'DPSECI_QUEUE_OPT_<X>' flags - */ int order_preservation_en; - /* order preservation configuration for the rx queue - * valid only if 'DPSECI_QUEUE_OPT_ORDER_PRESERVATION' is contained in - * 'options' - */ uint64_t user_ctx; - /* User context value provided in the frame descriptor of each - * dequeued frame; - * valid only if 'DPSECI_QUEUE_OPT_USER_CTX' is contained in 'options' - */ struct dpseci_dest_cfg dest_cfg; - /* Queue destination parameters; - * valid only if 'DPSECI_QUEUE_OPT_DEST' is contained in 'options' - */ }; -/** - * dpseci_set_rx_queue() - Set Rx queue configuration - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param queue Select the queue relative to number of - * priorities configured at DPSECI creation; use - * DPSECI_ALL_QUEUES to configure all Rx queues - * identically. - * @param cfg Rx queue configuration - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_set_rx_queue(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t queue, - const struct dpseci_rx_queue_cfg *cfg); +int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t queue, + const struct dpseci_rx_queue_cfg *cfg); /** * struct dpseci_rx_queue_attr - Structure representing attributes of Rx queues + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame + * @order_preservation_en: Status of the order preservation configuration + * on the queue + * @dest_cfg: Queue destination configuration + * @fqid: Virtual FQID value to be used for dequeue operations */ struct dpseci_rx_queue_attr { uint64_t user_ctx; - /* User context value provided in the frame descriptor of - * each dequeued frame - */ int order_preservation_en; - /* Status of the order preservation configuration on the queue */ - struct dpseci_dest_cfg dest_cfg; - /* Queue destination configuration */ + struct dpseci_dest_cfg dest_cfg; uint32_t fqid; - /* Virtual FQID value to be used for dequeue operations */ }; -/** - * dpseci_get_rx_queue() - Retrieve Rx queue attributes. - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param queue Select the queue relative to number of - * priorities configured at DPSECI creation - * @param attr Returned Rx queue attributes - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_get_rx_queue(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t queue, - struct dpseci_rx_queue_attr *attr); +int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t queue, + struct dpseci_rx_queue_attr *attr); /** * struct dpseci_tx_queue_attr - Structure representing attributes of Tx queues + * @fqid: Virtual FQID to be used for sending frames to SEC hardware + * @priority: SEC hardware processing priority for the queue */ struct dpseci_tx_queue_attr { uint32_t fqid; - /* Virtual FQID to be used for sending frames to SEC hardware */ uint8_t priority; - /* SEC hardware processing priority for the queue */ }; -/** - * dpseci_get_tx_queue() - Retrieve Tx queue attributes. - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param queue Select the queue relative to number of - * priorities configured at DPSECI creation - * @param attr Returned Tx queue attributes - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_get_tx_queue(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - uint8_t queue, - struct dpseci_tx_queue_attr *attr); +int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + uint8_t queue, + struct dpseci_tx_queue_attr *attr); /** * struct dpseci_sec_attr - Structure representing attributes of the SEC - * hardware accelerator - */ + * hardware accelerator + * @ip_id: ID for SEC. + * @major_rev: Major revision number for SEC. + * @minor_rev: Minor revision number for SEC. + * @era: SEC Era. + * @deco_num: The number of copies of the DECO that are implemented + * in this version of SEC. + * @zuc_auth_acc_num: The number of copies of ZUCA that are implemented + * in this version of SEC. + * @zuc_enc_acc_num: The number of copies of ZUCE that are implemented + * in this version of SEC. + * @snow_f8_acc_num: The number of copies of the SNOW-f8 module that are + * implemented in this version of SEC. + * @snow_f9_acc_num: The number of copies of the SNOW-f9 module that are + * implemented in this version of SEC. + * @crc_acc_num: The number of copies of the CRC module that are + * implemented in this version of SEC. + * @pk_acc_num: The number of copies of the Public Key module that are + * implemented in this version of SEC. + * @kasumi_acc_num: The number of copies of the Kasumi module that are + * implemented in this version of SEC. + * @rng_acc_num: The number of copies of the Random Number Generator that + * are implemented in this version of SEC. + * @md_acc_num: The number of copies of the MDHA (Hashing module) that + * are implemented in this version of SEC. + * @arc4_acc_num: The number of copies of the ARC4 module that are + * implemented in this version of SEC. + * @des_acc_num: The number of copies of the DES module that are + * implemented in this version of SEC. + * @aes_acc_num: The number of copies of the AES module that are + * implemented in this version of SEC. + **/ struct dpseci_sec_attr { - uint16_t ip_id; /* ID for SEC */ - uint8_t major_rev; /* Major revision number for SEC */ - uint8_t minor_rev; /* Minor revision number for SEC */ - uint8_t era; /* SEC Era */ + uint16_t ip_id; + uint8_t major_rev; + uint8_t minor_rev; + uint8_t era; uint8_t deco_num; - /* The number of copies of the DECO that are implemented in - * this version of SEC - */ uint8_t zuc_auth_acc_num; - /* The number of copies of ZUCA that are implemented in this - * version of SEC - */ uint8_t zuc_enc_acc_num; - /* The number of copies of ZUCE that are implemented in this - * version of SEC - */ uint8_t snow_f8_acc_num; - /* The number of copies of the SNOW-f8 module that are - * implemented in this version of SEC - */ uint8_t snow_f9_acc_num; - /* The number of copies of the SNOW-f9 module that are - * implemented in this version of SEC - */ uint8_t crc_acc_num; - /* The number of copies of the CRC module that are implemented - * in this version of SEC - */ uint8_t pk_acc_num; - /* The number of copies of the Public Key module that are - * implemented in this version of SEC - */ uint8_t kasumi_acc_num; - /* The number of copies of the Kasumi module that are - * implemented in this version of SEC - */ uint8_t rng_acc_num; - /* The number of copies of the Random Number Generator that are - * implemented in this version of SEC - */ uint8_t md_acc_num; - /* The number of copies of the MDHA (Hashing module) that are - * implemented in this version of SEC - */ uint8_t arc4_acc_num; - /* The number of copies of the ARC4 module that are implemented - * in this version of SEC - */ uint8_t des_acc_num; - /* The number of copies of the DES module that are implemented - * in this version of SEC - */ uint8_t aes_acc_num; - /* The number of copies of the AES module that are implemented - * in this version of SEC - */ }; -/** - * dpseci_get_sec_attr() - Retrieve SEC accelerator attributes. - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param attr Returned SEC attributes - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. - */ -int -dpseci_get_sec_attr(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - struct dpseci_sec_attr *attr); +int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_sec_attr *attr); /** * struct dpseci_sec_counters - Structure representing global SEC counters and * not per dpseci counters + * @dequeued_requests: Number of Requests Dequeued + * @ob_enc_requests: Number of Outbound Encrypt Requests + * @ib_dec_requests: Number of Inbound Decrypt Requests + * @ob_enc_bytes: Number of Outbound Bytes Encrypted + * @ob_prot_bytes: Number of Outbound Bytes Protected + * @ib_dec_bytes: Number of Inbound Bytes Decrypted + * @ib_valid_bytes: Number of Inbound Bytes Validated */ struct dpseci_sec_counters { - uint64_t dequeued_requests; /* Number of Requests Dequeued */ - uint64_t ob_enc_requests; /* Number of Outbound Encrypt Requests */ - uint64_t ib_dec_requests; /* Number of Inbound Decrypt Requests */ - uint64_t ob_enc_bytes; /* Number of Outbound Bytes Encrypted */ - uint64_t ob_prot_bytes; /* Number of Outbound Bytes Protected */ - uint64_t ib_dec_bytes; /* Number of Inbound Bytes Decrypted */ - uint64_t ib_valid_bytes; /* Number of Inbound Bytes Validated */ + uint64_t dequeued_requests; + uint64_t ob_enc_requests; + uint64_t ib_dec_requests; + uint64_t ob_enc_bytes; + uint64_t ob_prot_bytes; + uint64_t ib_dec_bytes; + uint64_t ib_valid_bytes; }; +int dpseci_get_sec_counters(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_sec_counters *counters); + +int dpseci_get_api_version(struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t *major_ver, + uint16_t *minor_ver); /** - * dpseci_get_sec_counters() - Retrieve SEC accelerator counters. - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param token Token of DPSECI object - * @param counters Returned SEC counters - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. + * enum dpseci_congestion_unit - DPSECI congestion units + * @DPSECI_CONGESTION_UNIT_BYTES: bytes units + * @DPSECI_CONGESTION_UNIT_FRAMES: frames units */ -int -dpseci_get_sec_counters(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t token, - struct dpseci_sec_counters *counters); +enum dpseci_congestion_unit { + DPSECI_CONGESTION_UNIT_BYTES = 0, + DPSECI_CONGESTION_UNIT_FRAMES +}; /** - * dpseci_get_api_version() - Get Data Path SEC Interface API version - * @param mc_io Pointer to MC portal's I/O object - * @param cmd_flags Command flags; one or more of 'MC_CMD_FLAG_' - * @param major_ver Major version of data path sec API - * @param minor_ver Minor version of data path sec API - * - * @return: - * - Return '0' on Success. - * - Return Error code otherwise. + * CSCN message is written to message_iova once entering a + * congestion state (see 'threshold_entry') + */ +#define DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER 0x00000001 +/** + * CSCN message is written to message_iova once exiting a + * congestion state (see 'threshold_exit') + */ +#define DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT 0x00000002 +/** + * CSCN write will attempt to allocate into a cache (coherent write); + * valid only if 'DPSECI_CGN_MODE_WRITE_MEM_<X>' is selected */ -int -dpseci_get_api_version(struct fsl_mc_io *mc_io, - uint32_t cmd_flags, - uint16_t *major_ver, - uint16_t *minor_ver); +#define DPSECI_CGN_MODE_COHERENT_WRITE 0x00000004 +/** + * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' CSCN message is sent to + * DPIO/DPCON's WQ channel once entering a congestion state + * (see 'threshold_entry') + */ +#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_ENTER 0x00000008 +/** + * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' CSCN message is sent to + * DPIO/DPCON's WQ channel once exiting a congestion state + * (see 'threshold_exit') + */ +#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_EXIT 0x00000010 +/** + * if 'dpseci_dest_cfg.dest_type != DPSECI_DEST_NONE' when the CSCN is written + * to the sw-portal's DQRR, the DQRI interrupt is asserted immediately + * (if enabled) + */ +#define DPSECI_CGN_MODE_INTR_COALESCING_DISABLED 0x00000020 + +/** + * struct dpseci_congestion_notification_cfg - congestion notification + * configuration + * @units: units type + * @threshold_entry: above this threshold we enter a congestion state. + * set it to '0' to disable it + * @threshold_exit: below this threshold we exit the congestion state. + * @message_ctx: The context that will be part of the CSCN message + * @message_iova: I/O virtual address (must be in DMA-able memory), + * must be 16B aligned; + * @dest_cfg: CSCN can be send to either DPIO or DPCON WQ channel + * @notification_mode: Mask of available options; use 'DPSECI_CGN_MODE_<X>' + * values + */ +struct dpseci_congestion_notification_cfg { + enum dpseci_congestion_unit units; + uint32_t threshold_entry; + uint32_t threshold_exit; + uint64_t message_ctx; + uint64_t message_iova; + struct dpseci_dest_cfg dest_cfg; + uint16_t notification_mode; +}; + +int dpseci_set_congestion_notification( + struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + const struct dpseci_congestion_notification_cfg *cfg); + +int dpseci_get_congestion_notification( + struct fsl_mc_io *mc_io, + uint32_t cmd_flags, + uint16_t token, + struct dpseci_congestion_notification_cfg *cfg); #endif /* __FSL_DPSECI_H */ diff --git a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h index 3f9f4743..a100a0ec 100644 --- a/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h +++ b/drivers/crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h @@ -5,7 +5,7 @@ * BSD LICENSE * * Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright 2016 NXP. + * Copyright 2016-2017 NXP. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -37,220 +37,187 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ - #ifndef _FSL_DPSECI_CMD_H #define _FSL_DPSECI_CMD_H /* DPSECI Version */ -#define DPSECI_VER_MAJOR 5 -#define DPSECI_VER_MINOR 0 - -/* Command IDs */ -#define DPSECI_CMDID_CLOSE ((0x800 << 4) | (0x1)) -#define DPSECI_CMDID_OPEN ((0x809 << 4) | (0x1)) -#define DPSECI_CMDID_CREATE ((0x909 << 4) | (0x1)) -#define DPSECI_CMDID_DESTROY ((0x989 << 4) | (0x1)) -#define DPSECI_CMDID_GET_API_VERSION ((0xa09 << 4) | (0x1)) - -#define DPSECI_CMDID_ENABLE ((0x002 << 4) | (0x1)) -#define DPSECI_CMDID_DISABLE ((0x003 << 4) | (0x1)) -#define DPSECI_CMDID_GET_ATTR ((0x004 << 4) | (0x1)) -#define DPSECI_CMDID_RESET ((0x005 << 4) | (0x1)) -#define DPSECI_CMDID_IS_ENABLED ((0x006 << 4) | (0x1)) - -#define DPSECI_CMDID_SET_IRQ ((0x010 << 4) | (0x1)) -#define DPSECI_CMDID_GET_IRQ ((0x011 << 4) | (0x1)) -#define DPSECI_CMDID_SET_IRQ_ENABLE ((0x012 << 4) | (0x1)) -#define DPSECI_CMDID_GET_IRQ_ENABLE ((0x013 << 4) | (0x1)) -#define DPSECI_CMDID_SET_IRQ_MASK ((0x014 << 4) | (0x1)) -#define DPSECI_CMDID_GET_IRQ_MASK ((0x015 << 4) | (0x1)) -#define DPSECI_CMDID_GET_IRQ_STATUS ((0x016 << 4) | (0x1)) -#define DPSECI_CMDID_CLEAR_IRQ_STATUS ((0x017 << 4) | (0x1)) - -#define DPSECI_CMDID_SET_RX_QUEUE ((0x194 << 4) | (0x1)) -#define DPSECI_CMDID_GET_RX_QUEUE ((0x196 << 4) | (0x1)) -#define DPSECI_CMDID_GET_TX_QUEUE ((0x197 << 4) | (0x1)) -#define DPSECI_CMDID_GET_SEC_ATTR ((0x198 << 4) | (0x1)) -#define DPSECI_CMDID_GET_SEC_COUNTERS ((0x199 << 4) | (0x1)) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_OPEN(cmd, dpseci_id) \ - MC_CMD_OP(cmd, 0, 0, 32, int, dpseci_id) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_CREATE(cmd, cfg) \ -do { \ - MC_CMD_OP(cmd, 0, 0, 8, uint8_t, cfg->priorities[0]);\ - MC_CMD_OP(cmd, 0, 8, 8, uint8_t, cfg->priorities[1]);\ - MC_CMD_OP(cmd, 0, 16, 8, uint8_t, cfg->priorities[2]);\ - MC_CMD_OP(cmd, 0, 24, 8, uint8_t, cfg->priorities[3]);\ - MC_CMD_OP(cmd, 0, 32, 8, uint8_t, cfg->priorities[4]);\ - MC_CMD_OP(cmd, 0, 40, 8, uint8_t, cfg->priorities[5]);\ - MC_CMD_OP(cmd, 0, 48, 8, uint8_t, cfg->priorities[6]);\ - MC_CMD_OP(cmd, 0, 56, 8, uint8_t, cfg->priorities[7]);\ - MC_CMD_OP(cmd, 1, 0, 8, uint8_t, cfg->num_tx_queues);\ - MC_CMD_OP(cmd, 1, 8, 8, uint8_t, cfg->num_rx_queues);\ -} while (0) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_RSP_IS_ENABLED(cmd, en) \ - MC_RSP_OP(cmd, 0, 0, 1, int, en) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_SET_IRQ(cmd, irq_index, irq_cfg) \ -do { \ - MC_CMD_OP(cmd, 0, 0, 8, uint8_t, irq_index);\ - MC_CMD_OP(cmd, 0, 32, 32, uint32_t, irq_cfg->val);\ - MC_CMD_OP(cmd, 1, 0, 64, uint64_t, irq_cfg->addr);\ - MC_CMD_OP(cmd, 2, 0, 32, int, irq_cfg->irq_num); \ -} while (0) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_GET_IRQ(cmd, irq_index) \ - MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_RSP_GET_IRQ(cmd, type, irq_cfg) \ -do { \ - MC_RSP_OP(cmd, 0, 0, 32, uint32_t, irq_cfg->val); \ - MC_RSP_OP(cmd, 1, 0, 64, uint64_t, irq_cfg->addr);\ - MC_RSP_OP(cmd, 2, 0, 32, int, irq_cfg->irq_num); \ - MC_RSP_OP(cmd, 2, 32, 32, int, type); \ -} while (0) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_SET_IRQ_ENABLE(cmd, irq_index, enable_state) \ -do { \ - MC_CMD_OP(cmd, 0, 0, 8, uint8_t, enable_state); \ - MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \ -} while (0) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_GET_IRQ_ENABLE(cmd, irq_index) \ - MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_RSP_GET_IRQ_ENABLE(cmd, enable_state) \ - MC_RSP_OP(cmd, 0, 0, 8, uint8_t, enable_state) +#define DPSECI_VER_MAJOR 5 +#define DPSECI_VER_MINOR 1 -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_SET_IRQ_MASK(cmd, irq_index, mask) \ -do { \ - MC_CMD_OP(cmd, 0, 0, 32, uint32_t, mask); \ - MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \ -} while (0) +/* Command versioning */ +#define DPSECI_CMD_BASE_VERSION 1 +#define DPSECI_CMD_BASE_VERSION_V2 2 +#define DPSECI_CMD_ID_OFFSET 4 -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_GET_IRQ_MASK(cmd, irq_index) \ - MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_RSP_GET_IRQ_MASK(cmd, mask) \ - MC_RSP_OP(cmd, 0, 0, 32, uint32_t, mask) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_GET_IRQ_STATUS(cmd, irq_index, status) \ -do { \ - MC_CMD_OP(cmd, 0, 0, 32, uint32_t, status);\ - MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index);\ -} while (0) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_RSP_GET_IRQ_STATUS(cmd, status) \ - MC_RSP_OP(cmd, 0, 0, 32, uint32_t, status) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status) \ -do { \ - MC_CMD_OP(cmd, 0, 0, 32, uint32_t, status); \ - MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \ -} while (0) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_RSP_GET_ATTR(cmd, attr) \ -do { \ - MC_RSP_OP(cmd, 0, 0, 32, int, attr->id); \ - MC_RSP_OP(cmd, 1, 0, 8, uint8_t, attr->num_tx_queues); \ - MC_RSP_OP(cmd, 1, 8, 8, uint8_t, attr->num_rx_queues); \ -} while (0) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_SET_RX_QUEUE(cmd, queue, cfg) \ -do { \ - MC_CMD_OP(cmd, 0, 0, 32, int, cfg->dest_cfg.dest_id); \ - MC_CMD_OP(cmd, 0, 32, 8, uint8_t, cfg->dest_cfg.priority); \ - MC_CMD_OP(cmd, 0, 40, 8, uint8_t, queue); \ - MC_CMD_OP(cmd, 0, 48, 4, enum dpseci_dest, cfg->dest_cfg.dest_type); \ - MC_CMD_OP(cmd, 1, 0, 64, uint64_t, cfg->user_ctx); \ - MC_CMD_OP(cmd, 2, 0, 32, uint32_t, cfg->options);\ - MC_CMD_OP(cmd, 2, 32, 1, int, cfg->order_preservation_en);\ -} while (0) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_GET_RX_QUEUE(cmd, queue) \ - MC_CMD_OP(cmd, 0, 40, 8, uint8_t, queue) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_RSP_GET_RX_QUEUE(cmd, attr) \ -do { \ - MC_RSP_OP(cmd, 0, 0, 32, int, attr->dest_cfg.dest_id);\ - MC_RSP_OP(cmd, 0, 32, 8, uint8_t, attr->dest_cfg.priority);\ - MC_RSP_OP(cmd, 0, 48, 4, enum dpseci_dest, attr->dest_cfg.dest_type);\ - MC_RSP_OP(cmd, 1, 0, 8, uint64_t, attr->user_ctx);\ - MC_RSP_OP(cmd, 2, 0, 32, uint32_t, attr->fqid);\ - MC_RSP_OP(cmd, 2, 32, 1, int, attr->order_preservation_en);\ -} while (0) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_CMD_GET_TX_QUEUE(cmd, queue) \ - MC_CMD_OP(cmd, 0, 40, 8, uint8_t, queue) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_RSP_GET_TX_QUEUE(cmd, attr) \ -do { \ - MC_RSP_OP(cmd, 0, 32, 32, uint32_t, attr->fqid);\ - MC_RSP_OP(cmd, 1, 0, 8, uint8_t, attr->priority);\ -} while (0) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_RSP_GET_SEC_ATTR(cmd, attr) \ -do { \ - MC_RSP_OP(cmd, 0, 0, 16, uint16_t, attr->ip_id);\ - MC_RSP_OP(cmd, 0, 16, 8, uint8_t, attr->major_rev);\ - MC_RSP_OP(cmd, 0, 24, 8, uint8_t, attr->minor_rev);\ - MC_RSP_OP(cmd, 0, 32, 8, uint8_t, attr->era);\ - MC_RSP_OP(cmd, 1, 0, 8, uint8_t, attr->deco_num);\ - MC_RSP_OP(cmd, 1, 8, 8, uint8_t, attr->zuc_auth_acc_num);\ - MC_RSP_OP(cmd, 1, 16, 8, uint8_t, attr->zuc_enc_acc_num);\ - MC_RSP_OP(cmd, 1, 32, 8, uint8_t, attr->snow_f8_acc_num);\ - MC_RSP_OP(cmd, 1, 40, 8, uint8_t, attr->snow_f9_acc_num);\ - MC_RSP_OP(cmd, 1, 48, 8, uint8_t, attr->crc_acc_num);\ - MC_RSP_OP(cmd, 2, 0, 8, uint8_t, attr->pk_acc_num);\ - MC_RSP_OP(cmd, 2, 8, 8, uint8_t, attr->kasumi_acc_num);\ - MC_RSP_OP(cmd, 2, 16, 8, uint8_t, attr->rng_acc_num);\ - MC_RSP_OP(cmd, 2, 32, 8, uint8_t, attr->md_acc_num);\ - MC_RSP_OP(cmd, 2, 40, 8, uint8_t, attr->arc4_acc_num);\ - MC_RSP_OP(cmd, 2, 48, 8, uint8_t, attr->des_acc_num);\ - MC_RSP_OP(cmd, 2, 56, 8, uint8_t, attr->aes_acc_num);\ -} while (0) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_RSP_GET_SEC_COUNTERS(cmd, counters) \ -do { \ - MC_RSP_OP(cmd, 0, 0, 64, uint64_t, counters->dequeued_requests);\ - MC_RSP_OP(cmd, 1, 0, 64, uint64_t, counters->ob_enc_requests);\ - MC_RSP_OP(cmd, 2, 0, 64, uint64_t, counters->ib_dec_requests);\ - MC_RSP_OP(cmd, 3, 0, 64, uint64_t, counters->ob_enc_bytes);\ - MC_RSP_OP(cmd, 4, 0, 64, uint64_t, counters->ob_prot_bytes);\ - MC_RSP_OP(cmd, 5, 0, 64, uint64_t, counters->ib_dec_bytes);\ - MC_RSP_OP(cmd, 6, 0, 64, uint64_t, counters->ib_valid_bytes);\ -} while (0) - -/* cmd, param, offset, width, type, arg_name */ -#define DPSECI_RSP_GET_API_VERSION(cmd, major, minor) \ -do { \ - MC_RSP_OP(cmd, 0, 0, 16, uint16_t, major);\ - MC_RSP_OP(cmd, 0, 16, 16, uint16_t, minor);\ -} while (0) +#define DPSECI_CMD_V1(id) \ + ((id << DPSECI_CMD_ID_OFFSET) | DPSECI_CMD_BASE_VERSION) +#define DPSECI_CMD_V2(id) \ + ((id << DPSECI_CMD_ID_OFFSET) | DPSECI_CMD_BASE_VERSION_V2) +/* Command IDs */ +#define DPSECI_CMDID_CLOSE DPSECI_CMD_V1(0x800) +#define DPSECI_CMDID_OPEN DPSECI_CMD_V1(0x809) +#define DPSECI_CMDID_CREATE DPSECI_CMD_V2(0x909) +#define DPSECI_CMDID_DESTROY DPSECI_CMD_V1(0x989) +#define DPSECI_CMDID_GET_API_VERSION DPSECI_CMD_V1(0xa09) + +#define DPSECI_CMDID_ENABLE DPSECI_CMD_V1(0x002) +#define DPSECI_CMDID_DISABLE DPSECI_CMD_V1(0x003) +#define DPSECI_CMDID_GET_ATTR DPSECI_CMD_V1(0x004) +#define DPSECI_CMDID_RESET DPSECI_CMD_V1(0x005) +#define DPSECI_CMDID_IS_ENABLED DPSECI_CMD_V1(0x006) + +#define DPSECI_CMDID_SET_RX_QUEUE DPSECI_CMD_V1(0x194) +#define DPSECI_CMDID_GET_RX_QUEUE DPSECI_CMD_V1(0x196) +#define DPSECI_CMDID_GET_TX_QUEUE DPSECI_CMD_V1(0x197) +#define DPSECI_CMDID_GET_SEC_ATTR DPSECI_CMD_V1(0x198) +#define DPSECI_CMDID_GET_SEC_COUNTERS DPSECI_CMD_V1(0x199) + +#define DPSECI_CMDID_SET_CONGESTION_NOTIFICATION DPSECI_CMD_V1(0x170) +#define DPSECI_CMDID_GET_CONGESTION_NOTIFICATION DPSECI_CMD_V1(0x171) + +/* Macros for accessing command fields smaller than 1byte */ +#define DPSECI_MASK(field) \ + GENMASK(DPSECI_##field##_SHIFT + DPSECI_##field##_SIZE - 1, \ + DPSECI_##field##_SHIFT) +#define dpseci_set_field(var, field, val) \ + ((var) |= (((val) << DPSECI_##field##_SHIFT) & DPSECI_MASK(field))) +#define dpseci_get_field(var, field) \ + (((var) & DPSECI_MASK(field)) >> DPSECI_##field##_SHIFT) + +#pragma pack(push, 1) +struct dpseci_cmd_open { + uint32_t dpseci_id; +}; + +struct dpseci_cmd_create { + uint8_t priorities[8]; + uint8_t num_tx_queues; + uint8_t num_rx_queues; + uint8_t pad[6]; + uint32_t options; +}; + +struct dpseci_cmd_destroy { + uint32_t dpseci_id; +}; + +#define DPSECI_ENABLE_SHIFT 0 +#define DPSECI_ENABLE_SIZE 1 + +struct dpseci_rsp_is_enabled { + /* only the first LSB */ + uint8_t en; +}; + +struct dpseci_rsp_get_attr { + uint32_t id; + uint32_t pad; + uint8_t num_tx_queues; + uint8_t num_rx_queues; + uint8_t pad1[6]; + uint32_t options; +}; + +#define DPSECI_DEST_TYPE_SHIFT 0 +#define DPSECI_DEST_TYPE_SIZE 4 + +#define DPSECI_ORDER_PRESERVATION_SHIFT 0 +#define DPSECI_ORDER_PRESERVATION_SIZE 1 + +struct dpseci_cmd_set_rx_queue { + uint32_t dest_id; + uint8_t dest_priority; + uint8_t queue; + /* from LSB: dest_type:4 */ + uint8_t dest_type; + uint8_t pad; + uint64_t user_ctx; + uint32_t options; + /* only the LSB */ + uint8_t order_preservation_en; +}; + +struct dpseci_cmd_get_queue { + uint8_t pad[5]; + uint8_t queue; +}; + +struct dpseci_rsp_get_rx_queue { + uint32_t dest_id; + uint8_t dest_priority; + uint8_t pad1; + /* from LSB: dest_type:4 */ + uint8_t dest_type; + uint8_t pad2; + uint64_t user_ctx; + uint32_t fqid; + /* only the LSB */ + uint8_t order_preservation_en; + +}; + +struct dpseci_rsp_get_tx_queue { + uint32_t pad; + uint32_t fqid; + uint8_t priority; +}; + +struct dpseci_rsp_get_sec_attr { + uint16_t ip_id; + uint8_t major_rev; + uint8_t minor_rev; + uint8_t era; + uint8_t pad1[3]; + uint8_t deco_num; + uint8_t zuc_auth_acc_num; + uint8_t zuc_enc_acc_num; + uint8_t pad2; + uint8_t snow_f8_acc_num; + uint8_t snow_f9_acc_num; + uint8_t crc_acc_num; + uint8_t pad3; + uint8_t pk_acc_num; + uint8_t kasumi_acc_num; + uint8_t rng_acc_num; + uint8_t pad4; + uint8_t md_acc_num; + uint8_t arc4_acc_num; + uint8_t des_acc_num; + uint8_t aes_acc_num; +}; + +struct dpseci_rsp_get_sec_counters { + uint64_t dequeued_requests; + uint64_t ob_enc_requests; + uint64_t ib_dec_requests; + uint64_t ob_enc_bytes; + uint64_t ob_prot_bytes; + uint64_t ib_dec_bytes; + uint64_t ib_valid_bytes; +}; + +struct dpseci_rsp_get_api_version { + uint16_t major; + uint16_t minor; +}; + +#define DPSECI_DEST_TYPE_SHIFT 0 +#define DPSECI_DEST_TYPE_SIZE 4 +#define DPSECI_CG_UNITS_SHIFT 4 +#define DPSECI_CG_UNITS_SIZE 2 + +struct dpseci_cmd_set_congestion_notification { + uint32_t dest_id; + uint16_t notification_mode; + uint8_t dest_priority; + /* from LSB: dest_type: 4 units:2 */ + uint8_t type_units; + uint64_t message_iova; + uint64_t message_ctx; + uint32_t threshold_entry; + uint32_t threshold_exit; +}; + +#pragma pack(pop) #endif /* _FSL_DPSECI_CMD_H */ diff --git a/drivers/crypto/dpaa_sec/Makefile b/drivers/crypto/dpaa_sec/Makefile new file mode 100644 index 00000000..17bc79c6 --- /dev/null +++ b/drivers/crypto/dpaa_sec/Makefile @@ -0,0 +1,73 @@ +# BSD LICENSE +# +# Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. +# Copyright 2017 NXP. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Freescale Semiconductor, Inc nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_dpaa_sec.a + +# build flags +CFLAGS += -D _GNU_SOURCE +ifeq ($(CONFIG_RTE_LIBRTE_DPAA_SEC_DEBUG_INIT),y) +CFLAGS += -O0 -g +CFLAGS += "-Wno-error" +else +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +endif + +CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa +CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa/include +CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa_sec/ +#sharing the hw flib headers from dpaa2_sec pmd +CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec/ +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev + +# versioning export map +EXPORT_MAP := rte_pmd_dpaa_sec_version.map + +# library version +LIBABIVER := 1 + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_DPAA_SEC) += dpaa_sec.c + +# library dependencies + +LDLIBS += -lrte_bus_dpaa +LDLIBS += -lrte_mempool_dpaa + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/crypto/dpaa_sec/dpaa_sec.c b/drivers/crypto/dpaa_sec/dpaa_sec.c new file mode 100644 index 00000000..16155b1a --- /dev/null +++ b/drivers/crypto/dpaa_sec/dpaa_sec.c @@ -0,0 +1,1548 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright 2017 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <fcntl.h> +#include <unistd.h> +#include <sched.h> +#include <net/if.h> + +#include <rte_byteorder.h> +#include <rte_common.h> +#include <rte_cryptodev_pmd.h> +#include <rte_crypto.h> +#include <rte_cryptodev.h> +#include <rte_cycles.h> +#include <rte_dev.h> +#include <rte_kvargs.h> +#include <rte_malloc.h> +#include <rte_mbuf.h> +#include <rte_memcpy.h> +#include <rte_string_fns.h> + +#include <fsl_usd.h> +#include <fsl_qman.h> +#include <of.h> + +/* RTA header files */ +#include <hw/desc/common.h> +#include <hw/desc/algo.h> +#include <hw/desc/ipsec.h> + +#include <rte_dpaa_bus.h> +#include <dpaa_sec.h> +#include <dpaa_sec_log.h> + +enum rta_sec_era rta_sec_era; + +static uint8_t cryptodev_driver_id; + +static __thread struct rte_crypto_op **dpaa_sec_ops; +static __thread int dpaa_sec_op_nb; + +static inline void +dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx) +{ + if (!ctx->fd_status) { + ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + } else { + PMD_RX_LOG(ERR, "SEC return err: 0x%x", ctx->fd_status); + ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR; + } + + /* report op status to sym->op and then free the ctx memeory */ + rte_mempool_put(ctx->ctx_pool, (void *)ctx); +} + +static inline struct dpaa_sec_op_ctx * +dpaa_sec_alloc_ctx(dpaa_sec_session *ses) +{ + struct dpaa_sec_op_ctx *ctx; + int retval; + + retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx)); + if (!ctx || retval) { + PMD_TX_LOG(ERR, "Alloc sec descriptor failed!"); + return NULL; + } + /* + * Clear SG memory. There are 16 SG entries of 16 Bytes each. + * one call to dcbz_64() clear 64 bytes, hence calling it 4 times + * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for + * each packet, memset is costlier than dcbz_64(). + */ + dcbz_64(&ctx->job.sg[SG_CACHELINE_0]); + dcbz_64(&ctx->job.sg[SG_CACHELINE_1]); + dcbz_64(&ctx->job.sg[SG_CACHELINE_2]); + dcbz_64(&ctx->job.sg[SG_CACHELINE_3]); + + ctx->ctx_pool = ses->ctx_pool; + + return ctx; +} + +static inline rte_iova_t +dpaa_mem_vtop(void *vaddr) +{ + const struct rte_memseg *memseg = rte_eal_get_physmem_layout(); + uint64_t vaddr_64, paddr; + int i; + + vaddr_64 = (uint64_t)vaddr; + for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) { + if (vaddr_64 >= memseg[i].addr_64 && + vaddr_64 < memseg[i].addr_64 + memseg[i].len) { + paddr = memseg[i].phys_addr + + (vaddr_64 - memseg[i].addr_64); + + return (rte_iova_t)paddr; + } + } + return (rte_iova_t)(NULL); +} + +static inline void * +dpaa_mem_ptov(rte_iova_t paddr) +{ + const struct rte_memseg *memseg = rte_eal_get_physmem_layout(); + int i; + + for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) { + if (paddr >= memseg[i].phys_addr && + (char *)paddr < (char *)memseg[i].phys_addr + memseg[i].len) + return (void *)(memseg[i].addr_64 + + (paddr - memseg[i].phys_addr)); + } + return NULL; +} + +static void +ern_sec_fq_handler(struct qman_portal *qm __rte_unused, + struct qman_fq *fq, + const struct qm_mr_entry *msg) +{ + RTE_LOG_DP(ERR, PMD, "sec fq %d error, RC = %x, seqnum = %x\n", + fq->fqid, msg->ern.rc, msg->ern.seqnum); +} + +/* initialize the queue with dest chan as caam chan so that + * all the packets in this queue could be dispatched into caam + */ +static int +dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc, + uint32_t fqid_out) +{ + struct qm_mcc_initfq fq_opts; + uint32_t flags; + int ret = -1; + + /* Clear FQ options */ + memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq)); + + flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID | + QMAN_FQ_FLAG_TO_DCPORTAL; + + ret = qman_create_fq(0, flags, fq_in); + if (unlikely(ret != 0)) { + PMD_INIT_LOG(ERR, "qman_create_fq failed"); + return ret; + } + + flags = QMAN_INITFQ_FLAG_SCHED; + fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA | + QM_INITFQ_WE_CONTEXTB; + + qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc); + fq_opts.fqd.context_b = fqid_out; + fq_opts.fqd.dest.channel = qm_channel_caam; + fq_opts.fqd.dest.wq = 0; + + fq_in->cb.ern = ern_sec_fq_handler; + + ret = qman_init_fq(fq_in, flags, &fq_opts); + if (unlikely(ret != 0)) + PMD_INIT_LOG(ERR, "qman_init_fq failed"); + + return ret; +} + +/* something is put into in_fq and caam put the crypto result into out_fq */ +static enum qman_cb_dqrr_result +dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, + struct qman_fq *fq __always_unused, + const struct qm_dqrr_entry *dqrr) +{ + const struct qm_fd *fd; + struct dpaa_sec_job *job; + struct dpaa_sec_op_ctx *ctx; + + if (dpaa_sec_op_nb >= DPAA_SEC_BURST) + return qman_cb_dqrr_defer; + + if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID)) + return qman_cb_dqrr_consume; + + fd = &dqrr->fd; + /* sg is embedded in an op ctx, + * sg[0] is for output + * sg[1] for input + */ + job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); + ctx = container_of(job, struct dpaa_sec_op_ctx, job); + ctx->fd_status = fd->status; + dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op; + dpaa_sec_op_ending(ctx); + + return qman_cb_dqrr_consume; +} + +/* caam result is put into this queue */ +static int +dpaa_sec_init_tx(struct qman_fq *fq) +{ + int ret; + struct qm_mcc_initfq opts; + uint32_t flags; + + flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED | + QMAN_FQ_FLAG_DYNAMIC_FQID; + + ret = qman_create_fq(0, flags, fq); + if (unlikely(ret)) { + PMD_INIT_LOG(ERR, "qman_create_fq failed"); + return ret; + } + + memset(&opts, 0, sizeof(opts)); + opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL | + QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB; + + /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */ + + fq->cb.dqrr = dqrr_out_fq_cb_rx; + fq->cb.ern = ern_sec_fq_handler; + + ret = qman_init_fq(fq, 0, &opts); + if (unlikely(ret)) { + PMD_INIT_LOG(ERR, "unable to init caam source fq!"); + return ret; + } + + return ret; +} + +static inline int is_cipher_only(dpaa_sec_session *ses) +{ + return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && + (ses->auth_alg == RTE_CRYPTO_AUTH_NULL)); +} + +static inline int is_auth_only(dpaa_sec_session *ses) +{ + return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) && + (ses->auth_alg != RTE_CRYPTO_AUTH_NULL)); +} + +static inline int is_aead(dpaa_sec_session *ses) +{ + return ((ses->cipher_alg == 0) && + (ses->auth_alg == 0) && + (ses->aead_alg != 0)); +} + +static inline int is_auth_cipher(dpaa_sec_session *ses) +{ + return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && + (ses->auth_alg != RTE_CRYPTO_AUTH_NULL)); +} + +static inline int is_encode(dpaa_sec_session *ses) +{ + return ses->dir == DIR_ENC; +} + +static inline int is_decode(dpaa_sec_session *ses) +{ + return ses->dir == DIR_DEC; +} + +static inline void +caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a) +{ + switch (ses->auth_alg) { + case RTE_CRYPTO_AUTH_NULL: + ses->digest_length = 0; + break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + alginfo_a->algtype = OP_ALG_ALGSEL_MD5; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA1_HMAC: + alginfo_a->algtype = OP_ALG_ALGSEL_SHA1; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + alginfo_a->algtype = OP_ALG_ALGSEL_SHA224; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + alginfo_a->algtype = OP_ALG_ALGSEL_SHA256; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + alginfo_a->algtype = OP_ALG_ALGSEL_SHA384; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA512_HMAC: + alginfo_a->algtype = OP_ALG_ALGSEL_SHA512; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + default: + PMD_INIT_LOG(ERR, "unsupported auth alg %u", ses->auth_alg); + } +} + +static inline void +caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c) +{ + switch (ses->cipher_alg) { + case RTE_CRYPTO_CIPHER_NULL: + break; + case RTE_CRYPTO_CIPHER_AES_CBC: + alginfo_c->algtype = OP_ALG_ALGSEL_AES; + alginfo_c->algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_3DES_CBC: + alginfo_c->algtype = OP_ALG_ALGSEL_3DES; + alginfo_c->algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + alginfo_c->algtype = OP_ALG_ALGSEL_AES; + alginfo_c->algmode = OP_ALG_AAI_CTR; + break; + default: + PMD_INIT_LOG(ERR, "unsupported cipher alg %d", ses->cipher_alg); + } +} + +static inline void +caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo) +{ + switch (ses->aead_alg) { + case RTE_CRYPTO_AEAD_AES_GCM: + alginfo->algtype = OP_ALG_ALGSEL_AES; + alginfo->algmode = OP_ALG_AAI_GCM; + break; + default: + PMD_INIT_LOG(ERR, "unsupported AEAD alg %d", ses->aead_alg); + } +} + + +/* prepare command block of the session */ +static int +dpaa_sec_prep_cdb(dpaa_sec_session *ses) +{ + struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0}; + uint32_t shared_desc_len = 0; + struct sec_cdb *cdb = &ses->qp->cdb; + int err; +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + int swap = false; +#else + int swap = true; +#endif + + memset(cdb, 0, sizeof(struct sec_cdb)); + + if (is_cipher_only(ses)) { + caam_cipher_alg(ses, &alginfo_c); + if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { + PMD_TX_LOG(ERR, "not supported cipher alg\n"); + return -ENOTSUP; + } + + alginfo_c.key = (uint64_t)ses->cipher_key.data; + alginfo_c.keylen = ses->cipher_key.length; + alginfo_c.key_enc_flags = 0; + alginfo_c.key_type = RTA_DATA_IMM; + + shared_desc_len = cnstr_shdsc_blkcipher( + cdb->sh_desc, true, + swap, &alginfo_c, + NULL, + ses->iv.length, + ses->dir); + } else if (is_auth_only(ses)) { + caam_auth_alg(ses, &alginfo_a); + if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { + PMD_TX_LOG(ERR, "not supported auth alg\n"); + return -ENOTSUP; + } + + alginfo_a.key = (uint64_t)ses->auth_key.data; + alginfo_a.keylen = ses->auth_key.length; + alginfo_a.key_enc_flags = 0; + alginfo_a.key_type = RTA_DATA_IMM; + + shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true, + swap, &alginfo_a, + !ses->dir, + ses->digest_length); + } else if (is_aead(ses)) { + caam_aead_alg(ses, &alginfo); + if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { + PMD_TX_LOG(ERR, "not supported aead alg\n"); + return -ENOTSUP; + } + alginfo.key = (uint64_t)ses->aead_key.data; + alginfo.keylen = ses->aead_key.length; + alginfo.key_enc_flags = 0; + alginfo.key_type = RTA_DATA_IMM; + + if (ses->dir == DIR_ENC) + shared_desc_len = cnstr_shdsc_gcm_encap( + cdb->sh_desc, true, swap, + &alginfo, + ses->iv.length, + ses->digest_length); + else + shared_desc_len = cnstr_shdsc_gcm_decap( + cdb->sh_desc, true, swap, + &alginfo, + ses->iv.length, + ses->digest_length); + } else { + caam_cipher_alg(ses, &alginfo_c); + if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { + PMD_TX_LOG(ERR, "not supported cipher alg\n"); + return -ENOTSUP; + } + + alginfo_c.key = (uint64_t)ses->cipher_key.data; + alginfo_c.keylen = ses->cipher_key.length; + alginfo_c.key_enc_flags = 0; + alginfo_c.key_type = RTA_DATA_IMM; + + caam_auth_alg(ses, &alginfo_a); + if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { + PMD_TX_LOG(ERR, "not supported auth alg\n"); + return -ENOTSUP; + } + + alginfo_a.key = (uint64_t)ses->auth_key.data; + alginfo_a.keylen = ses->auth_key.length; + alginfo_a.key_enc_flags = 0; + alginfo_a.key_type = RTA_DATA_IMM; + + cdb->sh_desc[0] = alginfo_c.keylen; + cdb->sh_desc[1] = alginfo_a.keylen; + err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, + MIN_JOB_DESC_SIZE, + (unsigned int *)cdb->sh_desc, + &cdb->sh_desc[2], 2); + + if (err < 0) { + PMD_TX_LOG(ERR, "Crypto: Incorrect key lengths"); + return err; + } + if (cdb->sh_desc[2] & 1) + alginfo_c.key_type = RTA_DATA_IMM; + else { + alginfo_c.key = (uint64_t)dpaa_mem_vtop( + (void *)alginfo_c.key); + alginfo_c.key_type = RTA_DATA_PTR; + } + if (cdb->sh_desc[2] & (1<<1)) + alginfo_a.key_type = RTA_DATA_IMM; + else { + alginfo_a.key = (uint64_t)dpaa_mem_vtop( + (void *)alginfo_a.key); + alginfo_a.key_type = RTA_DATA_PTR; + } + cdb->sh_desc[0] = 0; + cdb->sh_desc[1] = 0; + cdb->sh_desc[2] = 0; + + /* Auth_only_len is set as 0 here and it will be overwritten + * in fd for each packet. + */ + shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc, + true, swap, &alginfo_c, &alginfo_a, + ses->iv.length, 0, + ses->digest_length, ses->dir); + } + cdb->sh_hdr.hi.field.idlen = shared_desc_len; + cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word); + cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word); + + return 0; +} + +static inline unsigned int +dpaa_volatile_deq(struct qman_fq *fq, unsigned int len, bool exact) +{ + unsigned int pkts = 0; + int ret; + struct qm_mcr_queryfq_np np; + enum qman_fq_state state; + uint32_t flags; + uint32_t vdqcr; + + qman_query_fq_np(fq, &np); + if (np.frm_cnt) { + vdqcr = QM_VDQCR_NUMFRAMES_SET(len); + if (exact) + vdqcr |= QM_VDQCR_EXACT; + ret = qman_volatile_dequeue(fq, 0, vdqcr); + if (ret) + return 0; + do { + pkts += qman_poll_dqrr(len); + qman_fq_state(fq, &state, &flags); + } while (flags & QMAN_FQ_STATE_VDQCR); + } + return pkts; +} + +/* qp is lockless, should be accessed by only one thread */ +static int +dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops) +{ + struct qman_fq *fq; + + fq = &qp->outq; + dpaa_sec_op_nb = 0; + dpaa_sec_ops = ops; + + if (unlikely(nb_ops > DPAA_SEC_BURST)) + nb_ops = DPAA_SEC_BURST; + + return dpaa_volatile_deq(fq, nb_ops, 1); +} + +/** + * packet looks like: + * |<----data_len------->| + * |ip_header|ah_header|icv|payload| + * ^ + * | + * mbuf->pkt.data + */ +static inline struct dpaa_sec_job * +build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct rte_mbuf *mbuf = sym->m_src; + struct dpaa_sec_job *cf; + struct dpaa_sec_op_ctx *ctx; + struct qm_sg_entry *sg; + rte_iova_t start_addr; + uint8_t *old_digest; + + ctx = dpaa_sec_alloc_ctx(ses); + if (!ctx) + return NULL; + + cf = &ctx->job; + ctx->op = op; + old_digest = ctx->digest; + + start_addr = rte_pktmbuf_iova(mbuf); + /* output */ + sg = &cf->sg[0]; + qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); + sg->length = ses->digest_length; + cpu_to_hw_sg(sg); + + /* input */ + sg = &cf->sg[1]; + if (is_decode(ses)) { + /* need to extend the input to a compound frame */ + sg->extension = 1; + qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2])); + sg->length = sym->auth.data.length + ses->digest_length; + sg->final = 1; + cpu_to_hw_sg(sg); + + sg = &cf->sg[2]; + /* hash result or digest, save digest first */ + rte_memcpy(old_digest, sym->auth.digest.data, + ses->digest_length); + qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); + sg->length = sym->auth.data.length; + cpu_to_hw_sg(sg); + + /* let's check digest by hw */ + start_addr = dpaa_mem_vtop(old_digest); + sg++; + qm_sg_entry_set64(sg, start_addr); + sg->length = ses->digest_length; + sg->final = 1; + cpu_to_hw_sg(sg); + } else { + qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); + sg->length = sym->auth.data.length; + sg->final = 1; + cpu_to_hw_sg(sg); + } + + return cf; +} + +static inline struct dpaa_sec_job * +build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct dpaa_sec_job *cf; + struct dpaa_sec_op_ctx *ctx; + struct qm_sg_entry *sg; + rte_iova_t src_start_addr, dst_start_addr; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + ses->iv.offset); + + ctx = dpaa_sec_alloc_ctx(ses); + if (!ctx) + return NULL; + + cf = &ctx->job; + ctx->op = op; + + src_start_addr = rte_pktmbuf_iova(sym->m_src); + + if (sym->m_dst) + dst_start_addr = rte_pktmbuf_iova(sym->m_dst); + else + dst_start_addr = src_start_addr; + + /* output */ + sg = &cf->sg[0]; + qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); + sg->length = sym->cipher.data.length + ses->iv.length; + cpu_to_hw_sg(sg); + + /* input */ + sg = &cf->sg[1]; + + /* need to extend the input to a compound frame */ + sg->extension = 1; + sg->final = 1; + sg->length = sym->cipher.data.length + ses->iv.length; + qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2])); + cpu_to_hw_sg(sg); + + sg = &cf->sg[2]; + qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + sg->length = ses->iv.length; + cpu_to_hw_sg(sg); + + sg++; + qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset); + sg->length = sym->cipher.data.length; + sg->final = 1; + cpu_to_hw_sg(sg); + + return cf; +} + +static inline struct dpaa_sec_job * +build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct dpaa_sec_job *cf; + struct dpaa_sec_op_ctx *ctx; + struct qm_sg_entry *sg; + uint32_t length = 0; + rte_iova_t src_start_addr, dst_start_addr; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + ses->iv.offset); + + src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off; + + if (sym->m_dst) + dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off; + else + dst_start_addr = src_start_addr; + + ctx = dpaa_sec_alloc_ctx(ses); + if (!ctx) + return NULL; + + cf = &ctx->job; + ctx->op = op; + + /* input */ + rte_prefetch0(cf->sg); + sg = &cf->sg[2]; + qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg)); + if (is_encode(ses)) { + qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + sg->length = ses->iv.length; + length += sg->length; + cpu_to_hw_sg(sg); + + sg++; + if (ses->auth_only_len) { + qm_sg_entry_set64(sg, + dpaa_mem_vtop(sym->aead.aad.data)); + sg->length = ses->auth_only_len; + length += sg->length; + cpu_to_hw_sg(sg); + sg++; + } + qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset); + sg->length = sym->aead.data.length; + length += sg->length; + sg->final = 1; + cpu_to_hw_sg(sg); + } else { + qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + sg->length = ses->iv.length; + length += sg->length; + cpu_to_hw_sg(sg); + + sg++; + if (ses->auth_only_len) { + qm_sg_entry_set64(sg, + dpaa_mem_vtop(sym->aead.aad.data)); + sg->length = ses->auth_only_len; + length += sg->length; + cpu_to_hw_sg(sg); + sg++; + } + qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset); + sg->length = sym->aead.data.length; + length += sg->length; + cpu_to_hw_sg(sg); + + memcpy(ctx->digest, sym->aead.digest.data, + ses->digest_length); + sg++; + + qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); + sg->length = ses->digest_length; + length += sg->length; + sg->final = 1; + cpu_to_hw_sg(sg); + } + /* input compound frame */ + cf->sg[1].length = length; + cf->sg[1].extension = 1; + cf->sg[1].final = 1; + cpu_to_hw_sg(&cf->sg[1]); + + /* output */ + sg++; + qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg)); + qm_sg_entry_set64(sg, + dst_start_addr + sym->aead.data.offset - ses->auth_only_len); + sg->length = sym->aead.data.length + ses->auth_only_len; + length = sg->length; + if (is_encode(ses)) { + cpu_to_hw_sg(sg); + /* set auth output */ + sg++; + qm_sg_entry_set64(sg, sym->aead.digest.phys_addr); + sg->length = ses->digest_length; + length += sg->length; + } + sg->final = 1; + cpu_to_hw_sg(sg); + + /* output compound frame */ + cf->sg[0].length = length; + cf->sg[0].extension = 1; + cpu_to_hw_sg(&cf->sg[0]); + + return cf; +} + +static inline struct dpaa_sec_job * +build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct dpaa_sec_job *cf; + struct dpaa_sec_op_ctx *ctx; + struct qm_sg_entry *sg; + rte_iova_t src_start_addr, dst_start_addr; + uint32_t length = 0; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + ses->iv.offset); + + src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off; + if (sym->m_dst) + dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off; + else + dst_start_addr = src_start_addr; + + ctx = dpaa_sec_alloc_ctx(ses); + if (!ctx) + return NULL; + + cf = &ctx->job; + ctx->op = op; + + /* input */ + rte_prefetch0(cf->sg); + sg = &cf->sg[2]; + qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg)); + if (is_encode(ses)) { + qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + sg->length = ses->iv.length; + length += sg->length; + cpu_to_hw_sg(sg); + + sg++; + qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset); + sg->length = sym->auth.data.length; + length += sg->length; + sg->final = 1; + cpu_to_hw_sg(sg); + } else { + qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + sg->length = ses->iv.length; + length += sg->length; + cpu_to_hw_sg(sg); + + sg++; + + qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset); + sg->length = sym->auth.data.length; + length += sg->length; + cpu_to_hw_sg(sg); + + memcpy(ctx->digest, sym->auth.digest.data, + ses->digest_length); + sg++; + + qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); + sg->length = ses->digest_length; + length += sg->length; + sg->final = 1; + cpu_to_hw_sg(sg); + } + /* input compound frame */ + cf->sg[1].length = length; + cf->sg[1].extension = 1; + cf->sg[1].final = 1; + cpu_to_hw_sg(&cf->sg[1]); + + /* output */ + sg++; + qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg)); + qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); + sg->length = sym->cipher.data.length; + length = sg->length; + if (is_encode(ses)) { + cpu_to_hw_sg(sg); + /* set auth output */ + sg++; + qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); + sg->length = ses->digest_length; + length += sg->length; + } + sg->final = 1; + cpu_to_hw_sg(sg); + + /* output compound frame */ + cf->sg[0].length = length; + cf->sg[0].extension = 1; + cpu_to_hw_sg(&cf->sg[0]); + + return cf; +} + +static int +dpaa_sec_enqueue_op(struct rte_crypto_op *op, struct dpaa_sec_qp *qp) +{ + struct dpaa_sec_job *cf; + dpaa_sec_session *ses; + struct qm_fd fd; + int ret; + uint32_t auth_only_len = op->sym->auth.data.length - + op->sym->cipher.data.length; + + ses = (dpaa_sec_session *)get_session_private_data(op->sym->session, + cryptodev_driver_id); + + if (unlikely(!qp->ses || qp->ses != ses)) { + qp->ses = ses; + ses->qp = qp; + ret = dpaa_sec_prep_cdb(ses); + if (ret) + return ret; + } + + /* + * Segmented buffer is not supported. + */ + if (!rte_pktmbuf_is_contiguous(op->sym->m_src)) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return -ENOTSUP; + } + if (is_auth_only(ses)) { + cf = build_auth_only(op, ses); + } else if (is_cipher_only(ses)) { + cf = build_cipher_only(op, ses); + } else if (is_aead(ses)) { + cf = build_cipher_auth_gcm(op, ses); + auth_only_len = ses->auth_only_len; + } else if (is_auth_cipher(ses)) { + cf = build_cipher_auth(op, ses); + } else { + PMD_TX_LOG(ERR, "not supported sec op"); + return -ENOTSUP; + } + if (unlikely(!cf)) + return -ENOMEM; + + memset(&fd, 0, sizeof(struct qm_fd)); + qm_fd_addr_set64(&fd, dpaa_mem_vtop(cf->sg)); + fd._format1 = qm_fd_compound; + fd.length29 = 2 * sizeof(struct qm_sg_entry); + /* Auth_only_len is set as 0 in descriptor and it is overwritten + * here in the fd.cmd which will update the DPOVRD reg. + */ + if (auth_only_len) + fd.cmd = 0x80000000 | auth_only_len; + do { + ret = qman_enqueue(&qp->inq, &fd, 0); + } while (ret != 0); + + return 0; +} + +static uint16_t +dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + /* Function to transmit the frames to given device and queuepair */ + uint32_t loop; + int32_t ret; + struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; + uint16_t num_tx = 0; + + if (unlikely(nb_ops == 0)) + return 0; + + /*Prepare each packet which is to be sent*/ + for (loop = 0; loop < nb_ops; loop++) { + if (ops[loop]->sess_type != RTE_CRYPTO_OP_WITH_SESSION) { + PMD_TX_LOG(ERR, "sessionless crypto op not supported"); + return 0; + } + ret = dpaa_sec_enqueue_op(ops[loop], dpaa_qp); + if (!ret) + num_tx++; + } + dpaa_qp->tx_pkts += num_tx; + dpaa_qp->tx_errs += nb_ops - num_tx; + + return num_tx; +} + +static uint16_t +dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + uint16_t num_rx; + struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; + + num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops); + + dpaa_qp->rx_pkts += num_rx; + dpaa_qp->rx_errs += nb_ops - num_rx; + + PMD_RX_LOG(DEBUG, "SEC Received %d Packets\n", num_rx); + + return num_rx; +} + +/** Release queue pair */ +static int +dpaa_sec_queue_pair_release(struct rte_cryptodev *dev, + uint16_t qp_id) +{ + struct dpaa_sec_dev_private *internals; + struct dpaa_sec_qp *qp = NULL; + + PMD_INIT_FUNC_TRACE(); + + PMD_INIT_LOG(DEBUG, "dev =%p, queue =%d", dev, qp_id); + + internals = dev->data->dev_private; + if (qp_id >= internals->max_nb_queue_pairs) { + PMD_INIT_LOG(ERR, "Max supported qpid %d", + internals->max_nb_queue_pairs); + return -EINVAL; + } + + qp = &internals->qps[qp_id]; + qp->internals = NULL; + dev->data->queue_pairs[qp_id] = NULL; + + return 0; +} + +/** Setup a queue pair */ +static int +dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, + __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, + __rte_unused int socket_id, + __rte_unused struct rte_mempool *session_pool) +{ + struct dpaa_sec_dev_private *internals; + struct dpaa_sec_qp *qp = NULL; + + PMD_INIT_LOG(DEBUG, "dev =%p, queue =%d, conf =%p", + dev, qp_id, qp_conf); + + internals = dev->data->dev_private; + if (qp_id >= internals->max_nb_queue_pairs) { + PMD_INIT_LOG(ERR, "Max supported qpid %d", + internals->max_nb_queue_pairs); + return -EINVAL; + } + + qp = &internals->qps[qp_id]; + qp->internals = internals; + dev->data->queue_pairs[qp_id] = qp; + + return 0; +} + +/** Start queue pair */ +static int +dpaa_sec_queue_pair_start(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + PMD_INIT_FUNC_TRACE(); + + return 0; +} + +/** Stop queue pair */ +static int +dpaa_sec_queue_pair_stop(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + PMD_INIT_FUNC_TRACE(); + + return 0; +} + +/** Return the number of allocated queue pairs */ +static uint32_t +dpaa_sec_queue_pair_count(struct rte_cryptodev *dev) +{ + PMD_INIT_FUNC_TRACE(); + + return dev->data->nb_queue_pairs; +} + +/** Returns the size of session structure */ +static unsigned int +dpaa_sec_session_get_size(struct rte_cryptodev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + + return sizeof(dpaa_sec_session); +} + +static int +dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + dpaa_sec_session *session) +{ + session->cipher_alg = xform->cipher.algo; + session->iv.length = xform->cipher.iv.length; + session->iv.offset = xform->cipher.iv.offset; + session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, + RTE_CACHE_LINE_SIZE); + if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) { + PMD_INIT_LOG(ERR, "No Memory for cipher key\n"); + return -ENOMEM; + } + session->cipher_key.length = xform->cipher.key.length; + + memcpy(session->cipher_key.data, xform->cipher.key.data, + xform->cipher.key.length); + session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? + DIR_ENC : DIR_DEC; + + return 0; +} + +static int +dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + dpaa_sec_session *session) +{ + session->auth_alg = xform->auth.algo; + session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, + RTE_CACHE_LINE_SIZE); + if (session->auth_key.data == NULL && xform->auth.key.length > 0) { + PMD_INIT_LOG(ERR, "No Memory for auth key\n"); + return -ENOMEM; + } + session->auth_key.length = xform->auth.key.length; + session->digest_length = xform->auth.digest_length; + + memcpy(session->auth_key.data, xform->auth.key.data, + xform->auth.key.length); + session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? + DIR_ENC : DIR_DEC; + + return 0; +} + +static int +dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + dpaa_sec_session *session) +{ + session->aead_alg = xform->aead.algo; + session->iv.length = xform->aead.iv.length; + session->iv.offset = xform->aead.iv.offset; + session->auth_only_len = xform->aead.aad_length; + session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length, + RTE_CACHE_LINE_SIZE); + if (session->aead_key.data == NULL && xform->aead.key.length > 0) { + PMD_INIT_LOG(ERR, "No Memory for aead key\n"); + return -ENOMEM; + } + session->aead_key.length = xform->aead.key.length; + session->digest_length = xform->aead.digest_length; + + memcpy(session->aead_key.data, xform->aead.key.data, + xform->aead.key.length); + session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? + DIR_ENC : DIR_DEC; + + return 0; +} + +static int +dpaa_sec_qp_attach_sess(struct rte_cryptodev *dev, uint16_t qp_id, void *ses) +{ + dpaa_sec_session *sess = ses; + struct dpaa_sec_qp *qp; + + PMD_INIT_FUNC_TRACE(); + + qp = dev->data->queue_pairs[qp_id]; + if (qp->ses != NULL) { + PMD_INIT_LOG(ERR, "qp in-use by another session\n"); + return -EBUSY; + } + + qp->ses = sess; + sess->qp = qp; + + return dpaa_sec_prep_cdb(sess); +} + +static int +dpaa_sec_qp_detach_sess(struct rte_cryptodev *dev, uint16_t qp_id, void *ses) +{ + dpaa_sec_session *sess = ses; + struct dpaa_sec_qp *qp; + + PMD_INIT_FUNC_TRACE(); + + qp = dev->data->queue_pairs[qp_id]; + if (qp->ses != NULL) { + qp->ses = NULL; + sess->qp = NULL; + return 0; + } + + PMD_DRV_LOG(ERR, "No session attached to qp"); + return -EINVAL; +} + +static int +dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, void *sess) +{ + struct dpaa_sec_dev_private *internals = dev->data->dev_private; + dpaa_sec_session *session = sess; + + PMD_INIT_FUNC_TRACE(); + + if (unlikely(sess == NULL)) { + RTE_LOG(ERR, PMD, "invalid session struct\n"); + return -EINVAL; + } + + /* Default IV length = 0 */ + session->iv.length = 0; + + /* Cipher Only */ + if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { + session->auth_alg = RTE_CRYPTO_AUTH_NULL; + dpaa_sec_cipher_init(dev, xform, session); + + /* Authentication Only */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && + xform->next == NULL) { + session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; + dpaa_sec_auth_init(dev, xform, session); + + /* Cipher then Authenticate */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && + xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { + if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + dpaa_sec_cipher_init(dev, xform, session); + dpaa_sec_auth_init(dev, xform->next, session); + } else { + PMD_DRV_LOG(ERR, "Not supported: Auth then Cipher"); + return -EINVAL; + } + + /* Authenticate then Cipher */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && + xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { + if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { + dpaa_sec_auth_init(dev, xform, session); + dpaa_sec_cipher_init(dev, xform->next, session); + } else { + PMD_DRV_LOG(ERR, "Not supported: Auth then Cipher"); + return -EINVAL; + } + + /* AEAD operation for AES-GCM kind of Algorithms */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && + xform->next == NULL) { + dpaa_sec_aead_init(dev, xform, session); + + } else { + PMD_DRV_LOG(ERR, "Invalid crypto type"); + return -EINVAL; + } + session->ctx_pool = internals->ctx_pool; + + return 0; +} + +static int +dpaa_sec_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + PMD_INIT_FUNC_TRACE(); + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data); + if (ret != 0) { + PMD_DRV_LOG(ERR, "DPAA PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; +} + +/** Clear the memory of session so it doesn't leave key material behind */ +static void +dpaa_sec_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) +{ + PMD_INIT_FUNC_TRACE(); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + dpaa_sec_session *s = (dpaa_sec_session *)sess_priv; + + if (sess_priv) { + rte_free(s->cipher_key.data); + rte_free(s->auth_key.data); + memset(s, 0, sizeof(dpaa_sec_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } +} + +static int +dpaa_sec_dev_configure(struct rte_cryptodev *dev __rte_unused, + struct rte_cryptodev_config *config __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + + return 0; +} + +static int +dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + return 0; +} + +static void +dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); +} + +static int +dpaa_sec_dev_close(struct rte_cryptodev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + return 0; +} + +static void +dpaa_sec_dev_infos_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *info) +{ + struct dpaa_sec_dev_private *internals = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + if (info != NULL) { + info->max_nb_queue_pairs = internals->max_nb_queue_pairs; + info->feature_flags = dev->feature_flags; + info->capabilities = dpaa_sec_capabilities; + info->sym.max_nb_sessions = internals->max_nb_sessions; + info->sym.max_nb_sessions_per_qp = + RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS / RTE_MAX_NB_SEC_QPS; + info->driver_id = cryptodev_driver_id; + } +} + +static struct rte_cryptodev_ops crypto_ops = { + .dev_configure = dpaa_sec_dev_configure, + .dev_start = dpaa_sec_dev_start, + .dev_stop = dpaa_sec_dev_stop, + .dev_close = dpaa_sec_dev_close, + .dev_infos_get = dpaa_sec_dev_infos_get, + .queue_pair_setup = dpaa_sec_queue_pair_setup, + .queue_pair_release = dpaa_sec_queue_pair_release, + .queue_pair_start = dpaa_sec_queue_pair_start, + .queue_pair_stop = dpaa_sec_queue_pair_stop, + .queue_pair_count = dpaa_sec_queue_pair_count, + .session_get_size = dpaa_sec_session_get_size, + .session_configure = dpaa_sec_session_configure, + .session_clear = dpaa_sec_session_clear, + .qp_attach_session = dpaa_sec_qp_attach_sess, + .qp_detach_session = dpaa_sec_qp_detach_sess, +}; + +static int +dpaa_sec_uninit(struct rte_cryptodev *dev) +{ + struct dpaa_sec_dev_private *internals = dev->data->dev_private; + + if (dev == NULL) + return -ENODEV; + + rte_mempool_free(internals->ctx_pool); + rte_free(internals); + + PMD_INIT_LOG(INFO, "Closing DPAA_SEC device %s on numa socket %u\n", + dev->data->name, rte_socket_id()); + + return 0; +} + +static int +dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) +{ + struct dpaa_sec_dev_private *internals; + struct dpaa_sec_qp *qp; + uint32_t i; + int ret; + char str[20]; + + PMD_INIT_FUNC_TRACE(); + + cryptodev->driver_id = cryptodev_driver_id; + cryptodev->dev_ops = &crypto_ops; + + cryptodev->enqueue_burst = dpaa_sec_enqueue_burst; + cryptodev->dequeue_burst = dpaa_sec_dequeue_burst; + cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_HW_ACCELERATED | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING; + + internals = cryptodev->data->dev_private; + internals->max_nb_queue_pairs = RTE_MAX_NB_SEC_QPS; + internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS; + + for (i = 0; i < internals->max_nb_queue_pairs; i++) { + /* init qman fq for queue pair */ + qp = &internals->qps[i]; + ret = dpaa_sec_init_tx(&qp->outq); + if (ret) { + PMD_INIT_LOG(ERR, "config tx of queue pair %d", i); + goto init_error; + } + ret = dpaa_sec_init_rx(&qp->inq, dpaa_mem_vtop(&qp->cdb), + qman_fq_fqid(&qp->outq)); + if (ret) { + PMD_INIT_LOG(ERR, "config rx of queue pair %d", i); + goto init_error; + } + } + + sprintf(str, "ctx_pool_%d", cryptodev->data->dev_id); + internals->ctx_pool = rte_mempool_create((const char *)str, + CTX_POOL_NUM_BUFS, + CTX_POOL_BUF_SIZE, + CTX_POOL_CACHE_SIZE, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + if (!internals->ctx_pool) { + RTE_LOG(ERR, PMD, "%s create failed\n", str); + goto init_error; + } + + PMD_INIT_LOG(DEBUG, "driver %s: created\n", cryptodev->data->name); + return 0; + +init_error: + PMD_INIT_LOG(ERR, "driver %s: create failed\n", cryptodev->data->name); + + dpaa_sec_uninit(cryptodev); + return -EFAULT; +} + +static int +cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv, + struct rte_dpaa_device *dpaa_dev) +{ + struct rte_cryptodev *cryptodev; + char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; + + int retval; + + sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id); + + cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id()); + if (cryptodev == NULL) + return -ENOMEM; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + cryptodev->data->dev_private = rte_zmalloc_socket( + "cryptodev private structure", + sizeof(struct dpaa_sec_dev_private), + RTE_CACHE_LINE_SIZE, + rte_socket_id()); + + if (cryptodev->data->dev_private == NULL) + rte_panic("Cannot allocate memzone for private " + "device data"); + } + + dpaa_dev->crypto_dev = cryptodev; + cryptodev->device = &dpaa_dev->device; + cryptodev->device->driver = &dpaa_drv->driver; + + /* init user callbacks */ + TAILQ_INIT(&(cryptodev->link_intr_cbs)); + + /* if sec device version is not configured */ + if (!rta_get_sec_era()) { + const struct device_node *caam_node; + + for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { + const uint32_t *prop = of_get_property(caam_node, + "fsl,sec-era", + NULL); + if (prop) { + rta_set_sec_era( + INTL_SEC_ERA(rte_cpu_to_be_32(*prop))); + break; + } + } + } + + /* Invoke PMD device initialization function */ + retval = dpaa_sec_dev_init(cryptodev); + if (retval == 0) + return 0; + + /* In case of error, cleanup is done */ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_free(cryptodev->data->dev_private); + + rte_cryptodev_pmd_release_device(cryptodev); + + return -ENXIO; +} + +static int +cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev) +{ + struct rte_cryptodev *cryptodev; + int ret; + + cryptodev = dpaa_dev->crypto_dev; + if (cryptodev == NULL) + return -ENODEV; + + ret = dpaa_sec_uninit(cryptodev); + if (ret) + return ret; + + return rte_cryptodev_pmd_destroy(cryptodev); +} + +static struct rte_dpaa_driver rte_dpaa_sec_driver = { + .drv_type = FSL_DPAA_CRYPTO, + .driver = { + .name = "DPAA SEC PMD" + }, + .probe = cryptodev_dpaa_sec_probe, + .remove = cryptodev_dpaa_sec_remove, +}; + +static struct cryptodev_driver dpaa_sec_crypto_drv; + +RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver); +RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver, + cryptodev_driver_id); diff --git a/drivers/crypto/dpaa_sec/dpaa_sec.h b/drivers/crypto/dpaa_sec/dpaa_sec.h new file mode 100644 index 00000000..af3f2550 --- /dev/null +++ b/drivers/crypto/dpaa_sec/dpaa_sec.h @@ -0,0 +1,402 @@ +/*- + * BSD LICENSE + * + * Copyright 2016 NXP. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DPAA_SEC_H_ +#define _DPAA_SEC_H_ + +#define NUM_POOL_CHANNELS 4 +#define DPAA_SEC_BURST 32 +#define DPAA_SEC_ALG_UNSUPPORT (-1) +#define TDES_CBC_IV_LEN 8 +#define AES_CBC_IV_LEN 16 +#define AES_CTR_IV_LEN 16 +#define AES_GCM_IV_LEN 12 + +/* Minimum job descriptor consists of a oneword job descriptor HEADER and + * a pointer to the shared descriptor. + */ +#define MIN_JOB_DESC_SIZE (CAAM_CMD_SZ + CAAM_PTR_SZ) +/* CTX_POOL_NUM_BUFS is set as per the ipsec-secgw application */ +#define CTX_POOL_NUM_BUFS 32000 +#define CTX_POOL_BUF_SIZE sizeof(struct dpaa_sec_op_ctx) +#define CTX_POOL_CACHE_SIZE 512 + +#define DIR_ENC 1 +#define DIR_DEC 0 + +enum dpaa_sec_op_type { + DPAA_SEC_NONE, /*!< No Cipher operations*/ + DPAA_SEC_CIPHER,/*!< CIPHER operations */ + DPAA_SEC_AUTH, /*!< Authentication Operations */ + DPAA_SEC_AEAD, /*!< Authenticated Encryption with associated data */ + DPAA_SEC_IPSEC, /*!< IPSEC protocol operations*/ + DPAA_SEC_PDCP, /*!< PDCP protocol operations*/ + DPAA_SEC_PKC, /*!< Public Key Cryptographic Operations */ + DPAA_SEC_MAX +}; + +typedef struct dpaa_sec_session_entry { + uint8_t dir; /*!< Operation Direction */ + enum rte_crypto_cipher_algorithm cipher_alg; /*!< Cipher Algorithm*/ + enum rte_crypto_auth_algorithm auth_alg; /*!< Authentication Algorithm*/ + enum rte_crypto_aead_algorithm aead_alg; /*!< Authentication Algorithm*/ + union { + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } aead_key; + struct { + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } cipher_key; + struct { + uint8_t *data; /**< pointer to key data */ + size_t length; /**< key length in bytes */ + } auth_key; + }; + }; + struct { + uint16_t length; + uint16_t offset; + } iv; /**< Initialisation vector parameters */ + uint16_t auth_only_len; /*!< Length of data for Auth only */ + uint32_t digest_length; + struct dpaa_sec_qp *qp; + struct rte_mempool *ctx_pool; /* session mempool for dpaa_sec_op_ctx */ +} dpaa_sec_session; + +#define DPAA_SEC_MAX_DESC_SIZE 64 +/* code or cmd block to caam */ +struct sec_cdb { + struct { + union { + uint32_t word; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint16_t rsvd63_48; + unsigned int rsvd47_39:9; + unsigned int idlen:7; +#else + unsigned int idlen:7; + unsigned int rsvd47_39:9; + uint16_t rsvd63_48; +#endif + } field; + } __packed hi; + + union { + uint32_t word; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + unsigned int rsvd31_30:2; + unsigned int fsgt:1; + unsigned int lng:1; + unsigned int offset:2; + unsigned int abs:1; + unsigned int add_buf:1; + uint8_t pool_id; + uint16_t pool_buffer_size; +#else + uint16_t pool_buffer_size; + uint8_t pool_id; + unsigned int add_buf:1; + unsigned int abs:1; + unsigned int offset:2; + unsigned int lng:1; + unsigned int fsgt:1; + unsigned int rsvd31_30:2; +#endif + } field; + } __packed lo; + } __packed sh_hdr; + + uint32_t sh_desc[DPAA_SEC_MAX_DESC_SIZE]; +}; + +struct dpaa_sec_qp { + struct dpaa_sec_dev_private *internals; + struct sec_cdb cdb; /* cmd block associated with qp */ + dpaa_sec_session *ses; /* session associated with qp */ + struct qman_fq inq; + struct qman_fq outq; + int rx_pkts; + int rx_errs; + int tx_pkts; + int tx_errs; +}; + +#define RTE_MAX_NB_SEC_QPS RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS +/* internal sec queue interface */ +struct dpaa_sec_dev_private { + void *sec_hw; + struct rte_mempool *ctx_pool; /* per dev mempool for dpaa_sec_op_ctx */ + struct dpaa_sec_qp qps[RTE_MAX_NB_SEC_QPS]; /* i/o queue for sec */ + unsigned int max_nb_queue_pairs; + unsigned int max_nb_sessions; +}; + +#define MAX_SG_ENTRIES 16 +#define SG_CACHELINE_0 0 +#define SG_CACHELINE_1 4 +#define SG_CACHELINE_2 8 +#define SG_CACHELINE_3 12 +struct dpaa_sec_job { + /* sg[0] output, sg[1] input, others are possible sub frames */ + struct qm_sg_entry sg[MAX_SG_ENTRIES]; +}; + +#define DPAA_MAX_NB_MAX_DIGEST 32 +struct dpaa_sec_op_ctx { + struct dpaa_sec_job job; + struct rte_crypto_op *op; + struct rte_mempool *ctx_pool; /* mempool pointer for dpaa_sec_op_ctx */ + uint32_t fd_status; + uint8_t digest[DPAA_MAX_NB_MAX_DIGEST]; +}; + +static const struct rte_cryptodev_capabilities dpaa_sec_capabilities[] = { + { /* MD5 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_MD5_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA1 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 20, + .max = 20, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA224 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 28, + .max = 28, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA256 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 32, + .max = 32, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA384 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 48, + .max = 48, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA512 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 64, + .max = 64, + .increment = 0 + }, + }, } + }, } + }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.auth = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 8, + .max = 16, + .increment = 4 + }, + .aad_size = { + .min = 0, + .max = 240, + .increment = 1 + }, + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 + }, + }, } + }, } + }, + { /* AES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CBC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* AES CTR */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CTR, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* 3DES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .block_size = 8, + .key_size = { + .min = 16, + .max = 24, + .increment = 8 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + +#endif /* _DPAA_SEC_H_ */ diff --git a/drivers/crypto/dpaa_sec/dpaa_sec_log.h b/drivers/crypto/dpaa_sec/dpaa_sec_log.h new file mode 100644 index 00000000..dfe69757 --- /dev/null +++ b/drivers/crypto/dpaa_sec/dpaa_sec_log.h @@ -0,0 +1,70 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. + * Copyright NXP 2017. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of NXP nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DPAA_SEC_LOG_H_ +#define _DPAA_SEC_LOG_H_ + +#define PMD_INIT_LOG(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ##args) + +#ifdef RTE_LIBRTE_DPAA_SEC_DEBUG_INIT +#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") +#else +#define PMD_INIT_FUNC_TRACE() do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA_SEC_DEBUG_RX +#define PMD_RX_LOG(level, fmt, args...) \ + RTE_LOG_DP(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_RX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA_SEC_DEBUG_TX +#define PMD_TX_LOG(level, fmt, args...) \ + RTE_LOG_DP(level, PMD, "%s(): " fmt "\n", __func__, ## args) +#else +#define PMD_TX_LOG(level, fmt, args...) do { } while (0) +#endif + +#ifdef RTE_LIBRTE_DPAA_SEC_DEBUG_DRIVER +#define PMD_DRV_LOG_RAW(level, fmt, args...) \ + RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) +#else +#define PMD_DRV_LOG_RAW(level, fmt, args...) do { } while (0) +#endif + +#define PMD_DRV_LOG(level, fmt, args...) \ + PMD_DRV_LOG_RAW(level, fmt "\n", ## args) + +#endif /* _DPAA_SEC_LOG_H_ */ diff --git a/drivers/crypto/dpaa_sec/rte_pmd_dpaa_sec_version.map b/drivers/crypto/dpaa_sec/rte_pmd_dpaa_sec_version.map new file mode 100644 index 00000000..a70bd197 --- /dev/null +++ b/drivers/crypto/dpaa_sec/rte_pmd_dpaa_sec_version.map @@ -0,0 +1,4 @@ +DPDK_17.11 { + + local: *; +}; diff --git a/drivers/crypto/kasumi/Makefile b/drivers/crypto/kasumi/Makefile index b47cda0c..cf56b7a0 100644 --- a/drivers/crypto/kasumi/Makefile +++ b/drivers/crypto/kasumi/Makefile @@ -54,6 +54,9 @@ CFLAGS += -I$(LIBSSO_KASUMI_PATH) CFLAGS += -I$(LIBSSO_KASUMI_PATH)/include CFLAGS += -I$(LIBSSO_KASUMI_PATH)/build LDLIBS += -L$(LIBSSO_KASUMI_PATH)/build -lsso_kasumi +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_KASUMI) += rte_kasumi_pmd.c diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd.c b/drivers/crypto/kasumi/rte_kasumi_pmd.c index 38cd8a9b..f5db5e32 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd.c +++ b/drivers/crypto/kasumi/rte_kasumi_pmd.c @@ -31,12 +31,10 @@ */ #include <rte_common.h> -#include <rte_config.h> #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> -#include <rte_cryptodev_vdev.h> -#include <rte_vdev.h> +#include <rte_bus_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> @@ -44,7 +42,6 @@ #define KASUMI_KEY_LENGTH 16 #define KASUMI_IV_LENGTH 8 -#define KASUMI_DIGEST_LENGTH 4 #define KASUMI_MAX_BURST 4 #define BYTE_LEN 8 @@ -261,7 +258,7 @@ process_kasumi_cipher_op_bit(struct rte_crypto_op *op, /** Generate/verify hash from mbufs with same hash key. */ static int -process_kasumi_hash_op(struct rte_crypto_op **ops, +process_kasumi_hash_op(struct kasumi_qp *qp, struct rte_crypto_op **ops, struct kasumi_session *session, uint8_t num_ops) { @@ -287,8 +284,7 @@ process_kasumi_hash_op(struct rte_crypto_op **ops, num_bytes = length_in_bits >> 3; if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { - dst = (uint8_t *)rte_pktmbuf_append(ops[i]->sym->m_src, - KASUMI_DIGEST_LENGTH); + dst = qp->temp_digest; sso_kasumi_f9_1_buffer(&session->pKeySched_hash, src, num_bytes, dst); @@ -296,10 +292,6 @@ process_kasumi_hash_op(struct rte_crypto_op **ops, if (memcmp(dst, ops[i]->sym->auth.digest.data, KASUMI_DIGEST_LENGTH) != 0) ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - - /* Trim area used for digest from mbuf. */ - rte_pktmbuf_trim(ops[i]->sym->m_src, - KASUMI_DIGEST_LENGTH); } else { dst = ops[i]->sym->auth.digest.data; @@ -327,16 +319,16 @@ process_ops(struct rte_crypto_op **ops, struct kasumi_session *session, session, num_ops); break; case KASUMI_OP_ONLY_AUTH: - processed_ops = process_kasumi_hash_op(ops, session, + processed_ops = process_kasumi_hash_op(qp, ops, session, num_ops); break; case KASUMI_OP_CIPHER_AUTH: processed_ops = process_kasumi_cipher_op(ops, session, num_ops); - process_kasumi_hash_op(ops, session, processed_ops); + process_kasumi_hash_op(qp, ops, session, processed_ops); break; case KASUMI_OP_AUTH_CIPHER: - processed_ops = process_kasumi_hash_op(ops, session, + processed_ops = process_kasumi_hash_op(qp, ops, session, num_ops); process_kasumi_cipher_op(ops, session, processed_ops); break; @@ -384,15 +376,15 @@ process_op_bit(struct rte_crypto_op *op, struct kasumi_session *session, session); break; case KASUMI_OP_ONLY_AUTH: - processed_op = process_kasumi_hash_op(&op, session, 1); + processed_op = process_kasumi_hash_op(qp, &op, session, 1); break; case KASUMI_OP_CIPHER_AUTH: processed_op = process_kasumi_cipher_op_bit(op, session); if (processed_op == 1) - process_kasumi_hash_op(&op, session, 1); + process_kasumi_hash_op(qp, &op, session, 1); break; case KASUMI_OP_AUTH_CIPHER: - processed_op = process_kasumi_hash_op(&op, session, 1); + processed_op = process_kasumi_hash_op(qp, &op, session, 1); if (processed_op == 1) process_kasumi_cipher_op_bit(op, session); break; @@ -559,15 +551,17 @@ static int cryptodev_kasumi_remove(struct rte_vdev_device *vdev); static int cryptodev_kasumi_create(const char *name, struct rte_vdev_device *vdev, - struct rte_crypto_vdev_init_params *init_params) + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; struct kasumi_private *internals; uint64_t cpu_flags = 0; - if (init_params->name[0] == '\0') - snprintf(init_params->name, sizeof(init_params->name), - "%s", name); + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); + if (dev == NULL) { + KASUMI_LOG_ERR("failed to create cryptodev vdev"); + goto init_error; + } /* Check CPU for supported vector instruction set */ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) @@ -575,14 +569,6 @@ cryptodev_kasumi_create(const char *name, else cpu_flags |= RTE_CRYPTODEV_FF_CPU_SSE; - dev = rte_cryptodev_vdev_pmd_init(init_params->name, - sizeof(struct kasumi_private), init_params->socket_id, - vdev); - if (dev == NULL) { - KASUMI_LOG_ERR("failed to create cryptodev vdev"); - goto init_error; - } - dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_kasumi_pmd_ops; @@ -611,11 +597,12 @@ init_error: static int cryptodev_kasumi_probe(struct rte_vdev_device *vdev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct kasumi_private), rte_socket_id(), - {0} + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; const char *name; const char *input_args; @@ -625,17 +612,7 @@ cryptodev_kasumi_probe(struct rte_vdev_device *vdev) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_vdev_parse_init_params(&init_params, input_args); - - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, - init_params.socket_id); - if (init_params.name[0] != '\0') - RTE_LOG(INFO, PMD, " User defined name = %s\n", - init_params.name); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); return cryptodev_kasumi_create(name, vdev, &init_params); } @@ -643,17 +620,18 @@ cryptodev_kasumi_probe(struct rte_vdev_device *vdev) static int cryptodev_kasumi_remove(struct rte_vdev_device *vdev) { + struct rte_cryptodev *cryptodev; const char *name; name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL; - RTE_LOG(INFO, PMD, "Closing KASUMI crypto device %s" - " on numa socket %u\n", - name, rte_socket_id()); + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; - return 0; + return rte_cryptodev_pmd_destroy(cryptodev); } static struct rte_vdev_driver cryptodev_kasumi_pmd_drv = { @@ -661,10 +639,13 @@ static struct rte_vdev_driver cryptodev_kasumi_pmd_drv = { .remove = cryptodev_kasumi_remove }; +static struct cryptodev_driver kasumi_crypto_drv; + RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_KASUMI_PMD, cryptodev_kasumi_pmd_drv); RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_KASUMI_PMD, cryptodev_kasumi_pmd); RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_KASUMI_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); -RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_kasumi_pmd_drv, cryptodev_driver_id); +RTE_PMD_REGISTER_CRYPTO_DRIVER(kasumi_crypto_drv, cryptodev_kasumi_pmd_drv, + cryptodev_driver_id); diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c b/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c index 8033114b..4da12f37 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c +++ b/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c @@ -183,7 +183,7 @@ kasumi_pmd_qp_set_unique_name(struct rte_cryptodev *dev, "kasumi_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd_private.h b/drivers/crypto/kasumi/rte_kasumi_pmd_private.h index 0ce2a2e3..5f7044b8 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd_private.h +++ b/drivers/crypto/kasumi/rte_kasumi_pmd_private.h @@ -58,6 +58,8 @@ #define KASUMI_LOG_DBG(fmt, args...) #endif +#define KASUMI_DIGEST_LENGTH 4 + /** private data structure for each virtual KASUMI device */ struct kasumi_private { unsigned max_nb_queue_pairs; @@ -78,6 +80,11 @@ struct kasumi_qp { /**< Session Mempool */ struct rte_cryptodev_stats qp_stats; /**< Queue pair statistics */ + uint8_t temp_digest[KASUMI_DIGEST_LENGTH]; + /**< Buffer used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ } __rte_cache_aligned; enum kasumi_operation { diff --git a/drivers/crypto/mrvl/Makefile b/drivers/crypto/mrvl/Makefile new file mode 100644 index 00000000..3532f7cf --- /dev/null +++ b/drivers/crypto/mrvl/Makefile @@ -0,0 +1,66 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Marvell International Ltd. +# Copyright(c) 2017 Semihalf. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include $(RTE_SDK)/mk/rte.vars.mk + +ifneq ($(MAKECMDGOALS),clean) +ifneq ($(MAKECMDGOALS),config) +ifeq ($(LIBMUSDK_PATH),) +$(error "Please define LIBMUSDK_PATH environment variable") +endif +endif +endif + +# library name +LIB = librte_pmd_mrvl_crypto.a + +# build flags +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +CFLAGS += -I$(LIBMUSDK_PATH)/include +CFLAGS += -DMVCONF_ARCH_DMA_ADDR_T_64BIT + +# library version +LIBABIVER := 1 + +# versioning export map +EXPORT_MAP := rte_mrvl_pmd_version.map + +# external library dependencies +LDLIBS += -L$(LIBMUSDK_PATH)/lib -lmusdk +LDLIBS += -lrte_bus_vdev + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO) += rte_mrvl_pmd.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO) += rte_mrvl_pmd_ops.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/crypto/mrvl/rte_mrvl_compat.h b/drivers/crypto/mrvl/rte_mrvl_compat.h new file mode 100644 index 00000000..c29fa105 --- /dev/null +++ b/drivers/crypto/mrvl/rte_mrvl_compat.h @@ -0,0 +1,50 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Marvell International Ltd. + * Copyright(c) 2017 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_MRVL_COMPAT_H_ +#define _RTE_MRVL_COMPAT_H_ + +/* Unluckily, container_of is defined by both DPDK and MUSDK, + * we'll declare only one version. + * + * Note that it is not used in this PMD anyway. + */ +#ifdef container_of +#undef container_of +#endif +#include "drivers/mv_sam.h" +#include "drivers/mv_sam_cio.h" +#include "drivers/mv_sam_session.h" + +#endif /* _RTE_MRVL_COMPAT_H_ */ diff --git a/drivers/crypto/mrvl/rte_mrvl_pmd.c b/drivers/crypto/mrvl/rte_mrvl_pmd.c new file mode 100644 index 00000000..31f3fe58 --- /dev/null +++ b/drivers/crypto/mrvl/rte_mrvl_pmd.c @@ -0,0 +1,857 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Marvell International Ltd. + * Copyright(c) 2017 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <rte_common.h> +#include <rte_hexdump.h> +#include <rte_cryptodev.h> +#include <rte_cryptodev_pmd.h> +#include <rte_bus_vdev.h> +#include <rte_malloc.h> +#include <rte_cpuflags.h> + +#include "rte_mrvl_pmd_private.h" + +#define MRVL_MUSDK_DMA_MEMSIZE 41943040 + +static uint8_t cryptodev_driver_id; + +/** + * Flag if particular crypto algorithm is supported by PMD/MUSDK. + * + * The idea is to have Not Supported value as default (0). + * This way we need only to define proper map sizes, + * non-initialized entries will be by default not supported. + */ +enum algo_supported { + ALGO_NOT_SUPPORTED = 0, + ALGO_SUPPORTED = 1, +}; + +/** Map elements for cipher mapping.*/ +struct cipher_params_mapping { + enum algo_supported supported; /**< On/Off switch */ + enum sam_cipher_alg cipher_alg; /**< Cipher algorithm */ + enum sam_cipher_mode cipher_mode; /**< Cipher mode */ + unsigned int max_key_len; /**< Maximum key length (in bytes)*/ +} +/* We want to squeeze in multiple maps into the cache line. */ +__rte_aligned(32); + +/** Map elements for auth mapping.*/ +struct auth_params_mapping { + enum algo_supported supported; /**< On/off switch */ + enum sam_auth_alg auth_alg; /**< Auth algorithm */ +} +/* We want to squeeze in multiple maps into the cache line. */ +__rte_aligned(32); + +/** + * Map of supported cipher algorithms. + */ +static const +struct cipher_params_mapping cipher_map[RTE_CRYPTO_CIPHER_LIST_END] = { + [RTE_CRYPTO_CIPHER_3DES_CBC] = { + .supported = ALGO_SUPPORTED, + .cipher_alg = SAM_CIPHER_3DES, + .cipher_mode = SAM_CIPHER_CBC, + .max_key_len = BITS2BYTES(192) }, + [RTE_CRYPTO_CIPHER_3DES_CTR] = { + .supported = ALGO_SUPPORTED, + .cipher_alg = SAM_CIPHER_3DES, + .cipher_mode = SAM_CIPHER_CTR, + .max_key_len = BITS2BYTES(192) }, + [RTE_CRYPTO_CIPHER_3DES_ECB] = { + .supported = ALGO_SUPPORTED, + .cipher_alg = SAM_CIPHER_3DES, + .cipher_mode = SAM_CIPHER_ECB, + .max_key_len = BITS2BYTES(192) }, + [RTE_CRYPTO_CIPHER_AES_CBC] = { + .supported = ALGO_SUPPORTED, + .cipher_alg = SAM_CIPHER_AES, + .cipher_mode = SAM_CIPHER_CBC, + .max_key_len = BITS2BYTES(256) }, + [RTE_CRYPTO_CIPHER_AES_CTR] = { + .supported = ALGO_SUPPORTED, + .cipher_alg = SAM_CIPHER_AES, + .cipher_mode = SAM_CIPHER_CTR, + .max_key_len = BITS2BYTES(256) }, +}; + +/** + * Map of supported auth algorithms. + */ +static const +struct auth_params_mapping auth_map[RTE_CRYPTO_AUTH_LIST_END] = { + [RTE_CRYPTO_AUTH_MD5_HMAC] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HMAC_MD5 }, + [RTE_CRYPTO_AUTH_MD5] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HASH_MD5 }, + [RTE_CRYPTO_AUTH_SHA1_HMAC] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HMAC_SHA1 }, + [RTE_CRYPTO_AUTH_SHA1] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HASH_SHA1 }, + [RTE_CRYPTO_AUTH_SHA224] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HASH_SHA2_224 }, + [RTE_CRYPTO_AUTH_SHA256_HMAC] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HMAC_SHA2_256 }, + [RTE_CRYPTO_AUTH_SHA256] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HASH_SHA2_256 }, + [RTE_CRYPTO_AUTH_SHA384_HMAC] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HMAC_SHA2_384 }, + [RTE_CRYPTO_AUTH_SHA384] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HASH_SHA2_384 }, + [RTE_CRYPTO_AUTH_SHA512_HMAC] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HMAC_SHA2_512 }, + [RTE_CRYPTO_AUTH_SHA512] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_HASH_SHA2_512 }, + [RTE_CRYPTO_AUTH_AES_GMAC] = { + .supported = ALGO_SUPPORTED, + .auth_alg = SAM_AUTH_AES_GMAC }, +}; + +/** + * Map of supported aead algorithms. + */ +static const +struct cipher_params_mapping aead_map[RTE_CRYPTO_AEAD_LIST_END] = { + [RTE_CRYPTO_AEAD_AES_GCM] = { + .supported = ALGO_SUPPORTED, + .cipher_alg = SAM_CIPHER_AES, + .cipher_mode = SAM_CIPHER_GCM, + .max_key_len = BITS2BYTES(256) }, +}; + +/* + *----------------------------------------------------------------------------- + * Forward declarations. + *----------------------------------------------------------------------------- + */ +static int cryptodev_mrvl_crypto_uninit(struct rte_vdev_device *vdev); + +/* + *----------------------------------------------------------------------------- + * Session Preparation. + *----------------------------------------------------------------------------- + */ + +/** + * Get xform chain order. + * + * @param xform Pointer to configuration structure chain for crypto operations. + * @returns Order of crypto operations. + */ +static enum mrvl_crypto_chain_order +mrvl_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform) +{ + /* Currently, Marvell supports max 2 operations in chain */ + if (xform->next != NULL && xform->next->next != NULL) + return MRVL_CRYPTO_CHAIN_NOT_SUPPORTED; + + if (xform->next != NULL) { + if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) && + (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)) + return MRVL_CRYPTO_CHAIN_AUTH_CIPHER; + + if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && + (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)) + return MRVL_CRYPTO_CHAIN_CIPHER_AUTH; + } else { + if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) + return MRVL_CRYPTO_CHAIN_AUTH_ONLY; + + if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) + return MRVL_CRYPTO_CHAIN_CIPHER_ONLY; + + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) + return MRVL_CRYPTO_CHAIN_COMBINED; + } + return MRVL_CRYPTO_CHAIN_NOT_SUPPORTED; +} + +/** + * Set session parameters for cipher part. + * + * @param sess Crypto session pointer. + * @param cipher_xform Pointer to configuration structure for cipher operations. + * @returns 0 in case of success, negative value otherwise. + */ +static int +mrvl_crypto_set_cipher_session_parameters(struct mrvl_crypto_session *sess, + const struct rte_crypto_sym_xform *cipher_xform) +{ + /* Make sure we've got proper struct */ + if (cipher_xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) { + MRVL_CRYPTO_LOG_ERR("Wrong xform struct provided!"); + return -EINVAL; + } + + /* See if map data is present and valid */ + if ((cipher_xform->cipher.algo > RTE_DIM(cipher_map)) || + (cipher_map[cipher_xform->cipher.algo].supported + != ALGO_SUPPORTED)) { + MRVL_CRYPTO_LOG_ERR("Cipher algorithm not supported!"); + return -EINVAL; + } + + sess->cipher_iv_offset = cipher_xform->cipher.iv.offset; + + sess->sam_sess_params.dir = + (cipher_xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? + SAM_DIR_ENCRYPT : SAM_DIR_DECRYPT; + sess->sam_sess_params.cipher_alg = + cipher_map[cipher_xform->cipher.algo].cipher_alg; + sess->sam_sess_params.cipher_mode = + cipher_map[cipher_xform->cipher.algo].cipher_mode; + + /* Assume IV will be passed together with data. */ + sess->sam_sess_params.cipher_iv = NULL; + + /* Get max key length. */ + if (cipher_xform->cipher.key.length > + cipher_map[cipher_xform->cipher.algo].max_key_len) { + MRVL_CRYPTO_LOG_ERR("Wrong key length!"); + return -EINVAL; + } + + sess->sam_sess_params.cipher_key_len = cipher_xform->cipher.key.length; + sess->sam_sess_params.cipher_key = cipher_xform->cipher.key.data; + + return 0; +} + +/** + * Set session parameters for authentication part. + * + * @param sess Crypto session pointer. + * @param auth_xform Pointer to configuration structure for auth operations. + * @returns 0 in case of success, negative value otherwise. + */ +static int +mrvl_crypto_set_auth_session_parameters(struct mrvl_crypto_session *sess, + const struct rte_crypto_sym_xform *auth_xform) +{ + /* Make sure we've got proper struct */ + if (auth_xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) { + MRVL_CRYPTO_LOG_ERR("Wrong xform struct provided!"); + return -EINVAL; + } + + /* See if map data is present and valid */ + if ((auth_xform->auth.algo > RTE_DIM(auth_map)) || + (auth_map[auth_xform->auth.algo].supported != ALGO_SUPPORTED)) { + MRVL_CRYPTO_LOG_ERR("Auth algorithm not supported!"); + return -EINVAL; + } + + sess->sam_sess_params.dir = + (auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? + SAM_DIR_ENCRYPT : SAM_DIR_DECRYPT; + sess->sam_sess_params.auth_alg = + auth_map[auth_xform->auth.algo].auth_alg; + sess->sam_sess_params.u.basic.auth_icv_len = + auth_xform->auth.digest_length; + /* auth_key must be NULL if auth algorithm does not use HMAC */ + sess->sam_sess_params.auth_key = auth_xform->auth.key.length ? + auth_xform->auth.key.data : NULL; + sess->sam_sess_params.auth_key_len = auth_xform->auth.key.length; + + return 0; +} + +/** + * Set session parameters for aead part. + * + * @param sess Crypto session pointer. + * @param aead_xform Pointer to configuration structure for aead operations. + * @returns 0 in case of success, negative value otherwise. + */ +static int +mrvl_crypto_set_aead_session_parameters(struct mrvl_crypto_session *sess, + const struct rte_crypto_sym_xform *aead_xform) +{ + /* Make sure we've got proper struct */ + if (aead_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD) { + MRVL_CRYPTO_LOG_ERR("Wrong xform struct provided!"); + return -EINVAL; + } + + /* See if map data is present and valid */ + if ((aead_xform->aead.algo > RTE_DIM(aead_map)) || + (aead_map[aead_xform->aead.algo].supported + != ALGO_SUPPORTED)) { + MRVL_CRYPTO_LOG_ERR("AEAD algorithm not supported!"); + return -EINVAL; + } + + sess->sam_sess_params.dir = + (aead_xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? + SAM_DIR_ENCRYPT : SAM_DIR_DECRYPT; + sess->sam_sess_params.cipher_alg = + aead_map[aead_xform->aead.algo].cipher_alg; + sess->sam_sess_params.cipher_mode = + aead_map[aead_xform->aead.algo].cipher_mode; + + /* Assume IV will be passed together with data. */ + sess->sam_sess_params.cipher_iv = NULL; + + /* Get max key length. */ + if (aead_xform->aead.key.length > + aead_map[aead_xform->aead.algo].max_key_len) { + MRVL_CRYPTO_LOG_ERR("Wrong key length!"); + return -EINVAL; + } + + sess->sam_sess_params.cipher_key = aead_xform->aead.key.data; + sess->sam_sess_params.cipher_key_len = aead_xform->aead.key.length; + + if (sess->sam_sess_params.cipher_mode == SAM_CIPHER_GCM) + sess->sam_sess_params.auth_alg = SAM_AUTH_AES_GCM; + + sess->sam_sess_params.u.basic.auth_icv_len = + aead_xform->aead.digest_length; + + sess->sam_sess_params.u.basic.auth_aad_len = + aead_xform->aead.aad_length; + + return 0; +} + +/** + * Parse crypto transform chain and setup session parameters. + * + * @param dev Pointer to crypto device + * @param sess Poiner to crypto session + * @param xform Pointer to configuration structure chain for crypto operations. + * @returns 0 in case of success, negative value otherwise. + */ +int +mrvl_crypto_set_session_parameters(struct mrvl_crypto_session *sess, + const struct rte_crypto_sym_xform *xform) +{ + const struct rte_crypto_sym_xform *cipher_xform = NULL; + const struct rte_crypto_sym_xform *auth_xform = NULL; + const struct rte_crypto_sym_xform *aead_xform = NULL; + + /* Filter out spurious/broken requests */ + if (xform == NULL) + return -EINVAL; + + sess->chain_order = mrvl_crypto_get_chain_order(xform); + switch (sess->chain_order) { + case MRVL_CRYPTO_CHAIN_CIPHER_AUTH: + cipher_xform = xform; + auth_xform = xform->next; + break; + case MRVL_CRYPTO_CHAIN_AUTH_CIPHER: + auth_xform = xform; + cipher_xform = xform->next; + break; + case MRVL_CRYPTO_CHAIN_CIPHER_ONLY: + cipher_xform = xform; + break; + case MRVL_CRYPTO_CHAIN_AUTH_ONLY: + auth_xform = xform; + break; + case MRVL_CRYPTO_CHAIN_COMBINED: + aead_xform = xform; + break; + default: + return -EINVAL; + } + + if ((cipher_xform != NULL) && + (mrvl_crypto_set_cipher_session_parameters( + sess, cipher_xform) < 0)) { + MRVL_CRYPTO_LOG_ERR("Invalid/unsupported cipher parameters"); + return -EINVAL; + } + + if ((auth_xform != NULL) && + (mrvl_crypto_set_auth_session_parameters( + sess, auth_xform) < 0)) { + MRVL_CRYPTO_LOG_ERR("Invalid/unsupported auth parameters"); + return -EINVAL; + } + + if ((aead_xform != NULL) && + (mrvl_crypto_set_aead_session_parameters( + sess, aead_xform) < 0)) { + MRVL_CRYPTO_LOG_ERR("Invalid/unsupported aead parameters"); + return -EINVAL; + } + + return 0; +} + +/* + *----------------------------------------------------------------------------- + * Process Operations + *----------------------------------------------------------------------------- + */ + +/** + * Prepare a single request. + * + * This function basically translates DPDK crypto request into one + * understandable by MUDSK's SAM. If this is a first request in a session, + * it starts the session. + * + * @param request Pointer to pre-allocated && reset request buffer [Out]. + * @param src_bd Pointer to pre-allocated source descriptor [Out]. + * @param dst_bd Pointer to pre-allocated destination descriptor [Out]. + * @param op Pointer to DPDK crypto operation struct [In]. + */ +static inline int +mrvl_request_prepare(struct sam_cio_op_params *request, + struct sam_buf_info *src_bd, + struct sam_buf_info *dst_bd, + struct rte_crypto_op *op) +{ + struct mrvl_crypto_session *sess; + struct rte_mbuf *dst_mbuf; + uint8_t *digest; + + if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) { + MRVL_CRYPTO_LOG_ERR("MRVL CRYPTO PMD only supports session " + "oriented requests, op (%p) is sessionless.", + op); + return -EINVAL; + } + + sess = (struct mrvl_crypto_session *)get_session_private_data( + op->sym->session, cryptodev_driver_id); + if (unlikely(sess == NULL)) { + MRVL_CRYPTO_LOG_ERR("Session was not created for this device"); + return -EINVAL; + } + + /* + * If application delivered us null dst buffer, it means it expects + * us to deliver the result in src buffer. + */ + dst_mbuf = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; + + request->sa = sess->sam_sess; + request->cookie = op; + + /* Single buffers only, sorry. */ + request->num_bufs = 1; + request->src = src_bd; + src_bd->vaddr = rte_pktmbuf_mtod(op->sym->m_src, void *); + src_bd->paddr = rte_pktmbuf_iova(op->sym->m_src); + src_bd->len = rte_pktmbuf_data_len(op->sym->m_src); + + /* Empty source. */ + if (rte_pktmbuf_data_len(op->sym->m_src) == 0) { + /* EIP does not support 0 length buffers. */ + MRVL_CRYPTO_LOG_ERR("Buffer length == 0 not supported!"); + return -1; + } + + /* Empty destination. */ + if (rte_pktmbuf_data_len(dst_mbuf) == 0) { + /* Make dst buffer fit at least source data. */ + if (rte_pktmbuf_append(dst_mbuf, + rte_pktmbuf_data_len(op->sym->m_src)) == NULL) { + MRVL_CRYPTO_LOG_ERR("Unable to set big enough dst buffer!"); + return -1; + } + } + + request->dst = dst_bd; + dst_bd->vaddr = rte_pktmbuf_mtod(dst_mbuf, void *); + dst_bd->paddr = rte_pktmbuf_iova(dst_mbuf); + + /* + * We can use all available space in dst_mbuf, + * not only what's used currently. + */ + dst_bd->len = dst_mbuf->buf_len - rte_pktmbuf_headroom(dst_mbuf); + + if (sess->chain_order == MRVL_CRYPTO_CHAIN_COMBINED) { + request->cipher_len = op->sym->aead.data.length; + request->cipher_offset = op->sym->aead.data.offset; + request->cipher_iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->cipher_iv_offset); + + request->auth_aad = op->sym->aead.aad.data; + request->auth_offset = request->cipher_offset; + request->auth_len = request->cipher_len; + } else { + request->cipher_len = op->sym->cipher.data.length; + request->cipher_offset = op->sym->cipher.data.offset; + request->cipher_iv = rte_crypto_op_ctod_offset(op, uint8_t *, + sess->cipher_iv_offset); + + request->auth_offset = op->sym->auth.data.offset; + request->auth_len = op->sym->auth.data.length; + } + + digest = sess->chain_order == MRVL_CRYPTO_CHAIN_COMBINED ? + op->sym->aead.digest.data : op->sym->auth.digest.data; + if (digest == NULL) { + /* No auth - no worry. */ + return 0; + } + + request->auth_icv_offset = request->auth_offset + request->auth_len; + + /* + * EIP supports only scenarios where ICV(digest buffer) is placed at + * auth_icv_offset. Any other placement means risking errors. + */ + if (sess->sam_sess_params.dir == SAM_DIR_ENCRYPT) { + /* + * This should be the most common case anyway, + * EIP will overwrite DST buffer at auth_icv_offset. + */ + if (rte_pktmbuf_mtod_offset( + dst_mbuf, uint8_t *, + request->auth_icv_offset) == digest) { + return 0; + } + } else {/* sess->sam_sess_params.dir == SAM_DIR_DECRYPT */ + /* + * EIP will look for digest at auth_icv_offset + * offset in SRC buffer. + */ + if (rte_pktmbuf_mtod_offset( + op->sym->m_src, uint8_t *, + request->auth_icv_offset) == digest) { + return 0; + } + } + + /* + * If we landed here it means that digest pointer is + * at different than expected place. + */ + return -1; +} + +/* + *----------------------------------------------------------------------------- + * PMD Framework handlers + *----------------------------------------------------------------------------- + */ + +/** + * Enqueue burst. + * + * @param queue_pair Pointer to queue pair. + * @param ops Pointer to ops requests array. + * @param nb_ops Number of elements in ops requests array. + * @returns Number of elements consumed from ops. + */ +static uint16_t +mrvl_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + uint16_t iter_ops = 0; + uint16_t to_enq = 0; + uint16_t consumed = 0; + int ret; + struct sam_cio_op_params requests[nb_ops]; + /* + * DPDK uses single fragment buffers, so we can KISS descriptors. + * SAM does not store bd pointers, so on-stack scope will be enough. + */ + struct sam_buf_info src_bd[nb_ops]; + struct sam_buf_info dst_bd[nb_ops]; + struct mrvl_crypto_qp *qp = (struct mrvl_crypto_qp *)queue_pair; + + if (nb_ops == 0) + return 0; + + /* Prepare the burst. */ + memset(&requests, 0, sizeof(requests)); + + /* Iterate through */ + for (; iter_ops < nb_ops; ++iter_ops) { + if (mrvl_request_prepare(&requests[iter_ops], + &src_bd[iter_ops], + &dst_bd[iter_ops], + ops[iter_ops]) < 0) { + MRVL_CRYPTO_LOG_ERR( + "Error while parameters preparation!"); + qp->stats.enqueue_err_count++; + ops[iter_ops]->status = RTE_CRYPTO_OP_STATUS_ERROR; + + /* + * Number of handled ops is increased + * (even if the result of handling is error). + */ + ++consumed; + break; + } + + ops[iter_ops]->status = + RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + + /* Increase the number of ops to enqueue. */ + ++to_enq; + } /* for (; iter_ops < nb_ops;... */ + + if (to_enq > 0) { + /* Send the burst */ + ret = sam_cio_enq(qp->cio, requests, &to_enq); + consumed += to_enq; + if (ret < 0) { + /* + * Trust SAM that in this case returned value will be at + * some point correct (now it is returned unmodified). + */ + qp->stats.enqueue_err_count += to_enq; + for (iter_ops = 0; iter_ops < to_enq; ++iter_ops) + ops[iter_ops]->status = + RTE_CRYPTO_OP_STATUS_ERROR; + } + } + + qp->stats.enqueued_count += to_enq; + return consumed; +} + +/** + * Dequeue burst. + * + * @param queue_pair Pointer to queue pair. + * @param ops Pointer to ops requests array. + * @param nb_ops Number of elements in ops requests array. + * @returns Number of elements dequeued. + */ +static uint16_t +mrvl_crypto_pmd_dequeue_burst(void *queue_pair, + struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + int ret; + struct mrvl_crypto_qp *qp = queue_pair; + struct sam_cio *cio = qp->cio; + struct sam_cio_op_result results[nb_ops]; + uint16_t i; + + ret = sam_cio_deq(cio, results, &nb_ops); + if (ret < 0) { + /* Count all dequeued as error. */ + qp->stats.dequeue_err_count += nb_ops; + + /* But act as they were dequeued anyway*/ + qp->stats.dequeued_count += nb_ops; + + return 0; + } + + /* Unpack and check results. */ + for (i = 0; i < nb_ops; ++i) { + ops[i] = results[i].cookie; + + switch (results[i].status) { + case SAM_CIO_OK: + ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + break; + case SAM_CIO_ERR_ICV: + MRVL_CRYPTO_LOG_DBG("CIO returned SAM_CIO_ERR_ICV."); + ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + break; + default: + MRVL_CRYPTO_LOG_DBG( + "CIO returned Error: %d", results[i].status); + ops[i]->status = RTE_CRYPTO_OP_STATUS_ERROR; + break; + } + } + + qp->stats.dequeued_count += nb_ops; + return nb_ops; +} + +/** + * Create a new crypto device. + * + * @param name Driver name. + * @param vdev Pointer to device structure. + * @param init_params Pointer to initialization parameters. + * @returns 0 in case of success, negative value otherwise. + */ +static int +cryptodev_mrvl_crypto_create(const char *name, + struct rte_vdev_device *vdev, + struct rte_cryptodev_pmd_init_params *init_params) +{ + struct rte_cryptodev *dev; + struct mrvl_crypto_private *internals; + struct sam_init_params sam_params; + int ret; + + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); + if (dev == NULL) { + MRVL_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); + goto init_error; + } + + dev->driver_id = cryptodev_driver_id; + dev->dev_ops = rte_mrvl_crypto_pmd_ops; + + /* Register rx/tx burst functions for data path. */ + dev->enqueue_burst = mrvl_crypto_pmd_enqueue_burst; + dev->dequeue_burst = mrvl_crypto_pmd_dequeue_burst; + + dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_HW_ACCELERATED; + + /* Set vector instructions mode supported */ + internals = dev->data->dev_private; + + internals->max_nb_qpairs = init_params->max_nb_queue_pairs; + internals->max_nb_sessions = init_params->max_nb_sessions; + + /* + * ret == -EEXIST is correct, it means DMA + * has been already initialized. + */ + ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE); + if (ret < 0) { + if (ret != -EEXIST) + return ret; + + MRVL_CRYPTO_LOG_INFO( + "DMA memory has been already initialized by a different driver."); + } + + sam_params.max_num_sessions = internals->max_nb_sessions; + + return sam_init(&sam_params); + +init_error: + MRVL_CRYPTO_LOG_ERR( + "driver %s: %s failed", init_params->name, __func__); + + cryptodev_mrvl_crypto_uninit(vdev); + return -EFAULT; +} + +/** + * Initialize the crypto device. + * + * @param vdev Pointer to device structure. + * @returns 0 in case of success, negative value otherwise. + */ +static int +cryptodev_mrvl_crypto_init(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev_pmd_init_params init_params = { }; + const char *name, *args; + int ret; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + args = rte_vdev_device_args(vdev); + + init_params.private_data_size = sizeof(struct mrvl_crypto_private); + init_params.max_nb_queue_pairs = sam_get_num_inst() * SAM_HW_RING_NUM; + init_params.max_nb_sessions = + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS; + init_params.socket_id = rte_socket_id(); + + ret = rte_cryptodev_pmd_parse_input_args(&init_params, args); + if (ret) { + RTE_LOG(ERR, PMD, + "Failed to parse initialisation arguments[%s]\n", + args); + return -EINVAL; + } + + return cryptodev_mrvl_crypto_create(name, vdev, &init_params); +} + +/** + * Uninitialize the crypto device + * + * @param vdev Pointer to device structure. + * @returns 0 in case of success, negative value otherwise. + */ +static int +cryptodev_mrvl_crypto_uninit(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev *cryptodev; + const char *name = rte_vdev_device_name(vdev); + + if (name == NULL) + return -EINVAL; + + RTE_LOG(INFO, PMD, + "Closing Marvell crypto device %s on numa socket %u\n", + name, rte_socket_id()); + + sam_deinit(); + + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; + + return rte_cryptodev_pmd_destroy(cryptodev); +} + +/** + * Basic driver handlers for use in the constructor. + */ +static struct rte_vdev_driver cryptodev_mrvl_pmd_drv = { + .probe = cryptodev_mrvl_crypto_init, + .remove = cryptodev_mrvl_crypto_uninit +}; + +static struct cryptodev_driver mrvl_crypto_drv; + +/* Register the driver in constructor. */ +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_MRVL_PMD, cryptodev_mrvl_pmd_drv); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_MRVL_PMD, + "max_nb_queue_pairs=<int> " + "max_nb_sessions=<int> " + "socket_id=<int>"); +RTE_PMD_REGISTER_CRYPTO_DRIVER(mrvl_crypto_drv, cryptodev_mrvl_pmd_drv, + cryptodev_driver_id); diff --git a/drivers/crypto/mrvl/rte_mrvl_pmd_ops.c b/drivers/crypto/mrvl/rte_mrvl_pmd_ops.c new file mode 100644 index 00000000..434cf850 --- /dev/null +++ b/drivers/crypto/mrvl/rte_mrvl_pmd_ops.c @@ -0,0 +1,778 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Marvell International Ltd. + * Copyright(c) 2017 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <string.h> + +#include <rte_common.h> +#include <rte_malloc.h> +#include <rte_cryptodev_pmd.h> + +#include "rte_mrvl_pmd_private.h" + +/** + * Capabilities list to be used in reporting to DPDK. + */ +static const struct rte_cryptodev_capabilities + mrvl_crypto_pmd_capabilities[] = { + { /* MD5 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_MD5_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + }, } + }, } + }, + { /* MD5 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_MD5, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA1 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 20, + .max = 20, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA1 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA1, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 20, + .max = 20, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA224 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA224, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 28, + .max = 28, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA256 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 32, + .max = 32, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA256 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 32, + .max = 32, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA384 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 48, + .max = 48, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA384 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA384, + .block_size = 128, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 48, + .max = 48, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA512 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 64, + .max = 64, + .increment = 0 + }, + }, } + }, } + }, + { /* SHA512 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA512, + .block_size = 128, + .key_size = { + .min = 0, + .max = 0, + .increment = 0 + }, + .digest_size = { + .min = 64, + .max = 64, + .increment = 0 + }, + }, } + }, } + }, + { /* AES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CBC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* AES CTR */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CTR, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .aad_size = { + .min = 8, + .max = 12, + .increment = 4 + }, + .iv_size = { + .min = 12, + .max = 16, + .increment = 4 + } + }, } + }, } + }, + { /* AES GMAC (AUTH) */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_AES_GMAC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 16, + .max = 16, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 65532, + .increment = 4 + } + }, } + }, } + }, + { /* 3DES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .block_size = 8, + .key_size = { + .min = 24, + .max = 24, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + { /* 3DES CTR */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_3DES_CTR, + .block_size = 8, + .key_size = { + .min = 24, + .max = 24, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + + +/** + * Configure device (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @param config Pointer to configuration structure. + * @returns 0. Always. + */ +static int +mrvl_crypto_pmd_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) +{ + return 0; +} + +/** + * Start device (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @returns 0. Always. + */ +static int +mrvl_crypto_pmd_start(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + +/** + * Stop device (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @returns 0. Always. + */ +static void +mrvl_crypto_pmd_stop(__rte_unused struct rte_cryptodev *dev) +{ +} + +/** + * Get device statistics (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @param stats Pointer to statistics structure [out]. + */ +static void +mrvl_crypto_pmd_stats_get(struct rte_cryptodev *dev, + struct rte_cryptodev_stats *stats) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct mrvl_crypto_qp *qp = dev->data->queue_pairs[qp_id]; + + stats->enqueued_count += qp->stats.enqueued_count; + stats->dequeued_count += qp->stats.dequeued_count; + + stats->enqueue_err_count += qp->stats.enqueue_err_count; + stats->dequeue_err_count += qp->stats.dequeue_err_count; + } +} + +/** + * Reset device statistics (PMD ops callback). + * + * @param dev Pointer to the device structure. + */ +static void +mrvl_crypto_pmd_stats_reset(struct rte_cryptodev *dev) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) { + struct mrvl_crypto_qp *qp = dev->data->queue_pairs[qp_id]; + + memset(&qp->stats, 0, sizeof(qp->stats)); + } +} + +/** + * Get device info (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @param dev_info Pointer to the device info structure [out]. + */ +static void +mrvl_crypto_pmd_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *dev_info) +{ + struct mrvl_crypto_private *internals = dev->data->dev_private; + + if (dev_info != NULL) { + dev_info->driver_id = dev->driver_id; + dev_info->feature_flags = dev->feature_flags; + dev_info->capabilities = mrvl_crypto_pmd_capabilities; + dev_info->max_nb_queue_pairs = internals->max_nb_qpairs; + dev_info->sym.max_nb_sessions = internals->max_nb_sessions; + } +} + +/** + * Release queue pair (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @param qp_id ID of Queue Pair to release. + * @returns 0. Always. + */ +static int +mrvl_crypto_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) +{ + struct mrvl_crypto_qp *qp = + (struct mrvl_crypto_qp *)dev->data->queue_pairs[qp_id]; + + if (dev->data->queue_pairs[qp_id] != NULL) { + sam_cio_flush(qp->cio); + sam_cio_deinit(qp->cio); + rte_free(dev->data->queue_pairs[qp_id]); + dev->data->queue_pairs[qp_id] = NULL; + } + + return 0; +} + +/** + * Close device (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @returns 0. Always. + */ +static int +mrvl_crypto_pmd_close(struct rte_cryptodev *dev) +{ + int qp_id; + + for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) + mrvl_crypto_pmd_qp_release(dev, qp_id); + + return 0; +} + +/** + * Setup a queue pair (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @param qp_id ID of the Queue Pair. + * @param qp_conf Queue pair configuration (nb of descriptors). + * @param socket_id NUMA socket to allocate memory on. + * @returns 0 upon success, negative value otherwise. + */ +static int +mrvl_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, + const struct rte_cryptodev_qp_conf *qp_conf, + int socket_id, struct rte_mempool *session_pool) +{ + struct mrvl_crypto_qp *qp = NULL; + char match[RTE_CRYPTODEV_NAME_LEN]; + unsigned int n; + + /* Allocate the queue pair data structure. */ + qp = rte_zmalloc_socket("MRVL Crypto PMD Queue Pair", sizeof(*qp), + RTE_CACHE_LINE_SIZE, socket_id); + if (qp == NULL) + return -ENOMEM; + + /* Free old qp prior setup if needed. */ + if (dev->data->queue_pairs[qp_id] != NULL) + mrvl_crypto_pmd_qp_release(dev, qp_id); + + do { /* Error handling block */ + + /* + * This extra check is necessary due to a bug in + * crypto library. + */ + int num = sam_get_num_inst(); + if (num == 0) { + MRVL_CRYPTO_LOG_ERR("No crypto engines detected.\n"); + return -1; + } + + /* + * In case two crypto engines are enabled qps will + * be evenly spread among them. Even and odd qps will + * be handled by cio-0 and cio-1 respectively. qp-cio mapping + * will look as follows: + * + * qp: 0 1 2 3 + * cio-x:y: cio-0:0, cio-1:0, cio-0:1, cio-1:1 + * + * qp: 4 5 6 7 + * cio-x:y: cio-0:2, cio-1:2, cio-0:3, cio-1:3 + * + * In case just one engine is enabled mapping will look as + * follows: + * qp: 0 1 2 3 + * cio-x:y: cio-0:0, cio-0:1, cio-0:2, cio-0:3 + */ + n = snprintf(match, sizeof(match), "cio-%u:%u", + qp_id % num, qp_id / num); + + if (n >= sizeof(match)) + break; + + qp->cio_params.match = match; + qp->cio_params.size = qp_conf->nb_descriptors; + + if (sam_cio_init(&qp->cio_params, &qp->cio) < 0) + break; + + qp->sess_mp = session_pool; + + memset(&qp->stats, 0, sizeof(qp->stats)); + dev->data->queue_pairs[qp_id] = qp; + return 0; + } while (0); + + rte_free(qp); + return -1; +} + +/** Start queue pair (PMD ops callback) - not supported. + * + * @param dev Pointer to the device structure. + * @param qp_id ID of the Queue Pair. + * @returns -ENOTSUP. Always. + */ +static int +mrvl_crypto_pmd_qp_start(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + return -ENOTSUP; +} + +/** Stop queue pair (PMD ops callback) - not supported. + * + * @param dev Pointer to the device structure. + * @param qp_id ID of the Queue Pair. + * @returns -ENOTSUP. Always. + */ +static int +mrvl_crypto_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev, + __rte_unused uint16_t queue_pair_id) +{ + return -ENOTSUP; +} + +/** Return the number of allocated queue pairs (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @returns Number of allocated queue pairs. + */ +static uint32_t +mrvl_crypto_pmd_qp_count(struct rte_cryptodev *dev) +{ + return dev->data->nb_queue_pairs; +} + +/** Returns the size of the session structure (PMD ops callback). + * + * @param dev Pointer to the device structure [Unused]. + * @returns Size of Marvell crypto session. + */ +static unsigned +mrvl_crypto_pmd_session_get_size(__rte_unused struct rte_cryptodev *dev) +{ + return sizeof(struct mrvl_crypto_session); +} + +/** Configure the session from a crypto xform chain (PMD ops callback). + * + * @param dev Pointer to the device structure. + * @param xform Pointer to the crytpo configuration structure. + * @param sess Pointer to the empty session structure. + * @returns 0 upon success, negative value otherwise. + */ +static int +mrvl_crypto_pmd_session_configure(__rte_unused struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mp) +{ + struct mrvl_crypto_session *mrvl_sess; + void *sess_private_data; + int ret; + + if (sess == NULL) { + MRVL_CRYPTO_LOG_ERR("Invalid session struct."); + return -EINVAL; + } + + if (rte_mempool_get(mp, &sess_private_data)) { + CDEV_LOG_ERR("Couldn't get object from session mempool."); + return -ENOMEM; + } + + ret = mrvl_crypto_set_session_parameters(sess_private_data, xform); + if (ret != 0) { + MRVL_CRYPTO_LOG_ERR("Failed to configure session parameters."); + + /* Return session to mempool */ + rte_mempool_put(mp, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, sess_private_data); + + mrvl_sess = (struct mrvl_crypto_session *)sess_private_data; + if (sam_session_create(&mrvl_sess->sam_sess_params, + &mrvl_sess->sam_sess) < 0) { + MRVL_CRYPTO_LOG_DBG("Failed to create session!"); + return -EIO; + } + + return 0; +} + +/** + * Clear the memory of session so it doesn't leave key material behind. + * + * @param dev Pointer to the device structure. + * @returns 0. Always. + */ +static void +mrvl_crypto_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) +{ + + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + struct mrvl_crypto_session *mrvl_sess = + (struct mrvl_crypto_session *)sess_priv; + + if (mrvl_sess->sam_sess && + sam_session_destroy(mrvl_sess->sam_sess) < 0) { + MRVL_CRYPTO_LOG_INFO("Error while destroying session!"); + } + + memset(sess, 0, sizeof(struct mrvl_crypto_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } +} + +/** + * PMD handlers for crypto ops. + */ +static struct rte_cryptodev_ops mrvl_crypto_pmd_ops = { + .dev_configure = mrvl_crypto_pmd_config, + .dev_start = mrvl_crypto_pmd_start, + .dev_stop = mrvl_crypto_pmd_stop, + .dev_close = mrvl_crypto_pmd_close, + + .dev_infos_get = mrvl_crypto_pmd_info_get, + + .stats_get = mrvl_crypto_pmd_stats_get, + .stats_reset = mrvl_crypto_pmd_stats_reset, + + .queue_pair_setup = mrvl_crypto_pmd_qp_setup, + .queue_pair_release = mrvl_crypto_pmd_qp_release, + .queue_pair_start = mrvl_crypto_pmd_qp_start, + .queue_pair_stop = mrvl_crypto_pmd_qp_stop, + .queue_pair_count = mrvl_crypto_pmd_qp_count, + + .session_get_size = mrvl_crypto_pmd_session_get_size, + .session_configure = mrvl_crypto_pmd_session_configure, + .session_clear = mrvl_crypto_pmd_session_clear +}; + +struct rte_cryptodev_ops *rte_mrvl_crypto_pmd_ops = &mrvl_crypto_pmd_ops; diff --git a/drivers/crypto/mrvl/rte_mrvl_pmd_private.h b/drivers/crypto/mrvl/rte_mrvl_pmd_private.h new file mode 100644 index 00000000..923faaf9 --- /dev/null +++ b/drivers/crypto/mrvl/rte_mrvl_pmd_private.h @@ -0,0 +1,123 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2017 Marvell International Ltd. + * Copyright(c) 2017 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_MRVL_PMD_PRIVATE_H_ +#define _RTE_MRVL_PMD_PRIVATE_H_ + +#include "rte_mrvl_compat.h" + +#define CRYPTODEV_NAME_MRVL_PMD crypto_mrvl +/**< Marvell PMD device name */ + +#define MRVL_CRYPTO_LOG_ERR(fmt, args...) \ + RTE_LOG(ERR, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_MRVL_PMD), \ + __func__, __LINE__, ## args) + +#ifdef RTE_LIBRTE_PMD_MRVL_CRYPTO_DEBUG +#define MRVL_CRYPTO_LOG_INFO(fmt, args...) \ + RTE_LOG(INFO, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_MRVL_PMD), \ + __func__, __LINE__, ## args) + +#define MRVL_CRYPTO_LOG_DBG(fmt, args...) \ + RTE_LOG(DEBUG, CRYPTODEV, "[%s] %s() line %u: " fmt "\n", \ + RTE_STR(CRYPTODEV_NAME_MRVL_PMD), \ + __func__, __LINE__, ## args) + +#else +#define MRVL_CRYPTO_LOG_INFO(fmt, args...) +#define MRVL_CRYPTO_LOG_DBG(fmt, args...) +#endif + +/** + * Handy bits->bytes conversion macro. + */ +#define BITS2BYTES(x) ((x) >> 3) + +/** The operation order mode enumerator. */ +enum mrvl_crypto_chain_order { + MRVL_CRYPTO_CHAIN_CIPHER_ONLY, + MRVL_CRYPTO_CHAIN_AUTH_ONLY, + MRVL_CRYPTO_CHAIN_CIPHER_AUTH, + MRVL_CRYPTO_CHAIN_AUTH_CIPHER, + MRVL_CRYPTO_CHAIN_COMBINED, + MRVL_CRYPTO_CHAIN_NOT_SUPPORTED, +}; + +/** Private data structure for each crypto device. */ +struct mrvl_crypto_private { + unsigned int max_nb_qpairs; /**< Max number of queue pairs */ + unsigned int max_nb_sessions; /**< Max number of sessions */ +}; + +/** MRVL crypto queue pair structure. */ +struct mrvl_crypto_qp { + /** SAM CIO (MUSDK Queue Pair equivalent).*/ + struct sam_cio *cio; + + /** Session Mempool. */ + struct rte_mempool *sess_mp; + + /** Queue pair statistics. */ + struct rte_cryptodev_stats stats; + + /** CIO initialization parameters.*/ + struct sam_cio_params cio_params; +} __rte_cache_aligned; + +/** MRVL crypto private session structure. */ +struct mrvl_crypto_session { + /** Crypto operations chain order. */ + enum mrvl_crypto_chain_order chain_order; + + /** Session initialization parameters. */ + struct sam_session_params sam_sess_params; + + /** SAM session pointer. */ + struct sam_sa *sam_sess; + + /** Cipher IV offset. */ + uint16_t cipher_iv_offset; +} __rte_cache_aligned; + +/** Set and validate MRVL crypto session parameters */ +extern int +mrvl_crypto_set_session_parameters(struct mrvl_crypto_session *sess, + const struct rte_crypto_sym_xform *xform); + +/** device specific operations function pointer structure */ +extern struct rte_cryptodev_ops *rte_mrvl_crypto_pmd_ops; + +#endif /* _RTE_MRVL_PMD_PRIVATE_H_ */ diff --git a/drivers/crypto/mrvl/rte_pmd_mrvl_version.map b/drivers/crypto/mrvl/rte_pmd_mrvl_version.map new file mode 100644 index 00000000..a7530317 --- /dev/null +++ b/drivers/crypto/mrvl/rte_pmd_mrvl_version.map @@ -0,0 +1,3 @@ +DPDK_17.11 { + local: *; +}; diff --git a/drivers/crypto/null/Makefile b/drivers/crypto/null/Makefile index bc2724b3..49ada097 100644 --- a/drivers/crypto/null/Makefile +++ b/drivers/crypto/null/Makefile @@ -37,6 +37,9 @@ LIB = librte_pmd_null_crypto.a # build flags CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library version LIBABIVER := 1 diff --git a/drivers/crypto/null/null_crypto_pmd.c b/drivers/crypto/null/null_crypto_pmd.c index 2c827257..f031d3b8 100644 --- a/drivers/crypto/null/null_crypto_pmd.c +++ b/drivers/crypto/null/null_crypto_pmd.c @@ -31,10 +31,8 @@ */ #include <rte_common.h> -#include <rte_config.h> #include <rte_cryptodev_pmd.h> -#include <rte_cryptodev_vdev.h> -#include <rte_vdev.h> +#include <rte_bus_vdev.h> #include <rte_malloc.h> #include "null_crypto_pmd_private.h" @@ -183,28 +181,19 @@ null_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, return nb_dequeued; } -static int cryptodev_null_remove(const char *name); - /** Create crypto device */ static int cryptodev_null_create(const char *name, struct rte_vdev_device *vdev, - struct rte_crypto_vdev_init_params *init_params) + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; struct null_crypto_private *internals; - if (init_params->name[0] == '\0') - snprintf(init_params->name, sizeof(init_params->name), - "%s", name); - - dev = rte_cryptodev_vdev_pmd_init(init_params->name, - sizeof(struct null_crypto_private), - init_params->socket_id, - vdev); + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); if (dev == NULL) { NULL_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); - goto init_error; + return -EFAULT; } dev->driver_id = cryptodev_driver_id; @@ -224,61 +213,53 @@ cryptodev_null_create(const char *name, internals->max_nb_sessions = init_params->max_nb_sessions; return 0; - -init_error: - NULL_CRYPTO_LOG_ERR("driver %s: cryptodev_null_create failed", - init_params->name); - cryptodev_null_remove(init_params->name); - - return -EFAULT; } /** Initialise null crypto device */ static int cryptodev_null_probe(struct rte_vdev_device *dev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct null_crypto_private), rte_socket_id(), - {0} + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; - const char *name; + const char *name, *args; + int retval; name = rte_vdev_device_name(dev); if (name == NULL) return -EINVAL; - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", - name, init_params.socket_id); - if (init_params.name[0] != '\0') - RTE_LOG(INFO, PMD, " User defined name = %s\n", - init_params.name); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); + args = rte_vdev_device_args(dev); + + retval = rte_cryptodev_pmd_parse_input_args(&init_params, args); + if (retval) { + RTE_LOG(ERR, PMD, + "Failed to parse initialisation arguments[%s]\n", args); + return -EINVAL; + } return cryptodev_null_create(name, dev, &init_params); } -/** Uninitialise null crypto device */ static int -cryptodev_null_remove(const char *name) +cryptodev_null_remove_dev(struct rte_vdev_device *vdev) { + struct rte_cryptodev *cryptodev; + const char *name; + + name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL; - RTE_LOG(INFO, PMD, "Closing null crypto device %s on numa socket %u\n", - name, rte_socket_id()); - - return 0; -} + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; -static int -cryptodev_null_remove_dev(struct rte_vdev_device *dev) -{ - return cryptodev_null_remove(rte_vdev_device_name(dev)); + return rte_cryptodev_pmd_destroy(cryptodev); } static struct rte_vdev_driver cryptodev_null_pmd_drv = { @@ -286,10 +267,13 @@ static struct rte_vdev_driver cryptodev_null_pmd_drv = { .remove = cryptodev_null_remove_dev, }; +static struct cryptodev_driver null_crypto_drv; + RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_NULL_PMD, cryptodev_null_pmd_drv); RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_NULL_PMD, cryptodev_null_pmd); RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_NULL_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); -RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_null_pmd_drv, cryptodev_driver_id); +RTE_PMD_REGISTER_CRYPTO_DRIVER(null_crypto_drv, cryptodev_null_pmd_drv, + cryptodev_driver_id); diff --git a/drivers/crypto/null/null_crypto_pmd_ops.c b/drivers/crypto/null/null_crypto_pmd_ops.c index 76153203..c427050a 100644 --- a/drivers/crypto/null/null_crypto_pmd_ops.c +++ b/drivers/crypto/null/null_crypto_pmd_ops.c @@ -175,7 +175,7 @@ null_crypto_pmd_qp_set_unique_name(struct rte_cryptodev *dev, "null_crypto_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; diff --git a/drivers/crypto/null/null_crypto_pmd_private.h b/drivers/crypto/null/null_crypto_pmd_private.h index 4d1c3c9e..7c7852fb 100644 --- a/drivers/crypto/null/null_crypto_pmd_private.h +++ b/drivers/crypto/null/null_crypto_pmd_private.h @@ -33,8 +33,6 @@ #ifndef _NULL_CRYPTO_PMD_PRIVATE_H_ #define _NULL_CRYPTO_PMD_PRIVATE_H_ -#include "rte_config.h" - #define CRYPTODEV_NAME_NULL_PMD crypto_null /**< Null crypto PMD device name */ diff --git a/drivers/crypto/openssl/Makefile b/drivers/crypto/openssl/Makefile index e5fdfb59..1a006432 100644 --- a/drivers/crypto/openssl/Makefile +++ b/drivers/crypto/openssl/Makefile @@ -45,6 +45,9 @@ EXPORT_MAP := rte_pmd_openssl_version.map # external library dependencies LDLIBS += -lcrypto +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_OPENSSL) += rte_openssl_pmd.c diff --git a/drivers/crypto/openssl/rte_openssl_pmd.c b/drivers/crypto/openssl/rte_openssl_pmd.c index 0bd5f98e..06e1a6de 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd.c +++ b/drivers/crypto/openssl/rte_openssl_pmd.c @@ -34,11 +34,11 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> -#include <rte_cryptodev_vdev.h> -#include <rte_vdev.h> +#include <rte_bus_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> +#include <openssl/hmac.h> #include <openssl/evp.h> #include "rte_openssl_pmd_private.h" @@ -47,6 +47,25 @@ static uint8_t cryptodev_driver_id; +#if (OPENSSL_VERSION_NUMBER < 0x10100000L) +static HMAC_CTX *HMAC_CTX_new(void) +{ + HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); + + if (ctx != NULL) + HMAC_CTX_init(ctx); + return ctx; +} + +static void HMAC_CTX_free(HMAC_CTX *ctx) +{ + if (ctx != NULL) { + HMAC_CTX_cleanup(ctx); + OPENSSL_free(ctx); + } +} +#endif + static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); /*----------------------------------------------------------------------------*/ @@ -267,6 +286,21 @@ get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, res = -EINVAL; } break; + case RTE_CRYPTO_AEAD_AES_CCM: + switch (keylen) { + case 16: + *algo = EVP_aes_128_ccm(); + break; + case 24: + *algo = EVP_aes_192_ccm(); + break; + case 32: + *algo = EVP_aes_256_ccm(); + break; + default: + res = -EINVAL; + } + break; default: res = -EINVAL; break; @@ -278,6 +312,125 @@ get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, return res; } +/* Set session AEAD encryption parameters */ +static int +openssl_set_sess_aead_enc_param(struct openssl_session *sess, + enum rte_crypto_aead_algorithm algo, + uint8_t tag_len, uint8_t *key) +{ + int iv_type = 0; + unsigned int do_ccm; + + sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; + + /* Select AEAD algo */ + switch (algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + iv_type = EVP_CTRL_GCM_SET_IVLEN; + if (tag_len != 16) + return -EINVAL; + do_ccm = 0; + break; + case RTE_CRYPTO_AEAD_AES_CCM: + iv_type = EVP_CTRL_CCM_SET_IVLEN; + /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ + if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) + return -EINVAL; + do_ccm = 1; + break; + default: + return -ENOTSUP; + } + + sess->cipher.mode = OPENSSL_CIPHER_LIB; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + + if (get_aead_algo(algo, sess->cipher.key.length, + &sess->cipher.evp_algo) != 0) + return -EINVAL; + + get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); + + sess->chain_order = OPENSSL_CHAIN_COMBINED; + + if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, + NULL, NULL, NULL) <= 0) + return -EINVAL; + + if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, + NULL) <= 0) + return -EINVAL; + + if (do_ccm) + EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, + tag_len, NULL); + + if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) + return -EINVAL; + + return 0; +} + +/* Set session AEAD decryption parameters */ +static int +openssl_set_sess_aead_dec_param(struct openssl_session *sess, + enum rte_crypto_aead_algorithm algo, + uint8_t tag_len, uint8_t *key) +{ + int iv_type = 0; + unsigned int do_ccm = 0; + + sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; + sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; + + /* Select AEAD algo */ + switch (algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + iv_type = EVP_CTRL_GCM_SET_IVLEN; + if (tag_len != 16) + return -EINVAL; + break; + case RTE_CRYPTO_AEAD_AES_CCM: + iv_type = EVP_CTRL_CCM_SET_IVLEN; + /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ + if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) + return -EINVAL; + do_ccm = 1; + break; + default: + return -ENOTSUP; + } + + sess->cipher.mode = OPENSSL_CIPHER_LIB; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + + if (get_aead_algo(algo, sess->cipher.key.length, + &sess->cipher.evp_algo) != 0) + return -EINVAL; + + get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); + + sess->chain_order = OPENSSL_CHAIN_COMBINED; + + if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, + NULL, NULL, NULL) <= 0) + return -EINVAL; + + if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, + sess->iv.length, NULL) <= 0) + return -EINVAL; + + if (do_ccm) + EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, + tag_len, NULL); + + if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) + return -EINVAL; + + return 0; +} + /** Set session cipher parameters */ static int openssl_set_session_cipher_parameters(struct openssl_session *sess, @@ -307,6 +460,22 @@ openssl_set_session_cipher_parameters(struct openssl_session *sess, get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, sess->cipher.key.data); + if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + if (EVP_EncryptInit_ex(sess->cipher.ctx, + sess->cipher.evp_algo, + NULL, xform->cipher.key.data, + NULL) != 1) { + return -EINVAL; + } + } else if (sess->cipher.direction == + RTE_CRYPTO_CIPHER_OP_DECRYPT) { + if (EVP_DecryptInit_ex(sess->cipher.ctx, + sess->cipher.evp_algo, + NULL, xform->cipher.key.data, + NULL) != 1) { + return -EINVAL; + } + } break; @@ -319,6 +488,33 @@ openssl_set_session_cipher_parameters(struct openssl_session *sess, sess->cipher.key.data) != 0) return -EINVAL; break; + + case RTE_CRYPTO_CIPHER_DES_CBC: + sess->cipher.algo = xform->cipher.algo; + sess->cipher.ctx = EVP_CIPHER_CTX_new(); + sess->cipher.evp_algo = EVP_des_cbc(); + + get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, + sess->cipher.key.data); + if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + if (EVP_EncryptInit_ex(sess->cipher.ctx, + sess->cipher.evp_algo, + NULL, xform->cipher.key.data, + NULL) != 1) { + return -EINVAL; + } + } else if (sess->cipher.direction == + RTE_CRYPTO_CIPHER_OP_DECRYPT) { + if (EVP_DecryptInit_ex(sess->cipher.ctx, + sess->cipher.evp_algo, + NULL, xform->cipher.key.data, + NULL) != 1) { + return -EINVAL; + } + } + + break; + case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: sess->cipher.algo = xform->cipher.algo; sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; @@ -333,6 +529,23 @@ openssl_set_session_cipher_parameters(struct openssl_session *sess, get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, sess->cipher.key.data); + if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + if (EVP_EncryptInit_ex(sess->cipher.ctx, + sess->cipher.evp_algo, + NULL, xform->cipher.key.data, + NULL) != 1) { + return -EINVAL; + } + } else if (sess->cipher.direction == + RTE_CRYPTO_CIPHER_OP_DECRYPT) { + if (EVP_DecryptInit_ex(sess->cipher.ctx, + sess->cipher.evp_algo, + NULL, xform->cipher.key.data, + NULL) != 1) { + return -EINVAL; + } + } + break; default: sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; @@ -351,36 +564,31 @@ openssl_set_session_auth_parameters(struct openssl_session *sess, sess->auth.operation = xform->auth.op; sess->auth.algo = xform->auth.algo; + sess->auth.digest_length = xform->auth.digest_length; + /* Select auth algo */ switch (xform->auth.algo) { case RTE_CRYPTO_AUTH_AES_GMAC: - sess->chain_order = OPENSSL_CHAIN_COMBINED; - - /* Set IV parameters */ - sess->iv.offset = xform->auth.iv.offset; - sess->iv.length = xform->auth.iv.length; - /* * OpenSSL requires GMAC to be a GCM operation * with no cipher data length */ - sess->cipher.mode = OPENSSL_CIPHER_LIB; - if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) - sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; - else - sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; - sess->cipher.key.length = xform->auth.key.length; - sess->cipher.ctx = EVP_CIPHER_CTX_new(); - if (get_aead_algo(RTE_CRYPTO_AEAD_AES_GCM, - sess->cipher.key.length, - &sess->cipher.evp_algo) != 0) - return -EINVAL; - - get_cipher_key(xform->auth.key.data, xform->auth.key.length, - sess->cipher.key.data); + /* Set IV parameters */ + sess->iv.offset = xform->auth.iv.offset; + sess->iv.length = xform->auth.iv.length; + if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) + return openssl_set_sess_aead_enc_param(sess, + RTE_CRYPTO_AEAD_AES_GCM, + xform->auth.digest_length, + xform->auth.key.data); + else + return openssl_set_sess_aead_dec_param(sess, + RTE_CRYPTO_AEAD_AES_GCM, + xform->auth.digest_length, + xform->auth.key.data); break; case RTE_CRYPTO_AUTH_MD5: @@ -403,20 +611,22 @@ openssl_set_session_auth_parameters(struct openssl_session *sess, case RTE_CRYPTO_AUTH_SHA384_HMAC: case RTE_CRYPTO_AUTH_SHA512_HMAC: sess->auth.mode = OPENSSL_AUTH_AS_HMAC; - sess->auth.hmac.ctx = EVP_MD_CTX_create(); + sess->auth.hmac.ctx = HMAC_CTX_new(); if (get_auth_algo(xform->auth.algo, &sess->auth.hmac.evp_algo) != 0) return -EINVAL; - sess->auth.hmac.pkey = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL, - xform->auth.key.data, xform->auth.key.length); + + if (HMAC_Init_ex(sess->auth.hmac.ctx, + xform->auth.key.data, + xform->auth.key.length, + sess->auth.hmac.evp_algo, NULL) != 1) + return -EINVAL; break; default: return -ENOTSUP; } - sess->auth.digest_length = xform->auth.digest_length; - return 0; } @@ -425,43 +635,33 @@ static int openssl_set_session_aead_parameters(struct openssl_session *sess, const struct rte_crypto_sym_xform *xform) { - /* Select cipher direction */ - sess->cipher.direction = xform->cipher.op; /* Select cipher key */ sess->cipher.key.length = xform->aead.key.length; /* Set IV parameters */ - sess->iv.offset = xform->aead.iv.offset; - sess->iv.length = xform->aead.iv.length; - - /* Select auth generate/verify */ - sess->auth.operation = xform->auth.op; - sess->auth.algo = xform->auth.algo; - - /* Select auth algo */ - switch (xform->aead.algo) { - case RTE_CRYPTO_AEAD_AES_GCM: - sess->cipher.mode = OPENSSL_CIPHER_LIB; - sess->aead_algo = xform->aead.algo; - sess->cipher.ctx = EVP_CIPHER_CTX_new(); - - if (get_aead_algo(sess->aead_algo, sess->cipher.key.length, - &sess->cipher.evp_algo) != 0) - return -EINVAL; - - get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, - sess->cipher.key.data); + if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) + /* + * For AES-CCM, the actual IV is placed + * one byte after the start of the IV field, + * according to the API. + */ + sess->iv.offset = xform->aead.iv.offset + 1; + else + sess->iv.offset = xform->aead.iv.offset; - sess->chain_order = OPENSSL_CHAIN_COMBINED; - break; - default: - return -ENOTSUP; - } + sess->iv.length = xform->aead.iv.length; sess->auth.aad_length = xform->aead.aad_length; sess->auth.digest_length = xform->aead.digest_length; - return 0; + sess->aead_algo = xform->aead.algo; + /* Select cipher direction */ + if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) + return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, + xform->aead.digest_length, xform->aead.key.data); + else + return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, + xform->aead.digest_length, xform->aead.key.data); } /** Parse crypto xform chain and set private session parameters */ @@ -547,7 +747,7 @@ openssl_reset_session(struct openssl_session *sess) break; case OPENSSL_AUTH_AS_HMAC: EVP_PKEY_free(sess->auth.hmac.pkey); - EVP_MD_CTX_destroy(sess->auth.hmac.ctx); + HMAC_CTX_free(sess->auth.hmac.ctx); break; default: break; @@ -693,12 +893,11 @@ process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, /** Process standard openssl cipher encryption */ static int process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, - int offset, uint8_t *iv, uint8_t *key, int srclen, - EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) + int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) { int totlen; - if (EVP_EncryptInit_ex(ctx, algo, NULL, key, iv) <= 0) + if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) goto process_cipher_encrypt_err; EVP_CIPHER_CTX_set_padding(ctx, 0); @@ -743,12 +942,11 @@ process_cipher_encrypt_err: /** Process standard openssl cipher decryption */ static int process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, - int offset, uint8_t *iv, uint8_t *key, int srclen, - EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) + int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) { int totlen; - if (EVP_DecryptInit_ex(ctx, algo, NULL, key, iv) <= 0) + if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) goto process_cipher_decrypt_err; EVP_CIPHER_CTX_set_padding(ctx, 0); @@ -823,23 +1021,16 @@ process_cipher_des3ctr_err: return -EINVAL; } -/** Process auth/encription aes-gcm algorithm */ +/** Process AES-GCM encrypt algorithm */ static int process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, - int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, - uint8_t *key, uint8_t *dst, uint8_t *tag, - EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, + uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) { int len = 0, unused = 0; uint8_t empty[] = {}; - if (EVP_EncryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0) - goto process_auth_encryption_gcm_err; - - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0) - goto process_auth_encryption_gcm_err; - - if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) + if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) goto process_auth_encryption_gcm_err; if (aadlen > 0) @@ -868,25 +1059,60 @@ process_auth_encryption_gcm_err: return -EINVAL; } +/** Process AES-CCM encrypt algorithm */ +static int +process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, + uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) +{ + int len = 0; + + if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) + goto process_auth_encryption_ccm_err; + + if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) + goto process_auth_encryption_ccm_err; + + if (aadlen > 0) + /* + * For AES-CCM, the actual AAD is placed + * 18 bytes after the start of the AAD field, + * according to the API. + */ + if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) + goto process_auth_encryption_ccm_err; + + if (srclen > 0) + if (process_openssl_encryption_update(mbuf_src, offset, &dst, + srclen, ctx)) + goto process_auth_encryption_ccm_err; + + if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) + goto process_auth_encryption_ccm_err; + + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) + goto process_auth_encryption_ccm_err; + + return 0; + +process_auth_encryption_ccm_err: + OPENSSL_LOG_ERR("Process openssl auth encryption ccm failed"); + return -EINVAL; +} + +/** Process AES-GCM decrypt algorithm */ static int process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, - int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, - uint8_t *key, uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx, - const EVP_CIPHER *algo) + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, + uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) { int len = 0, unused = 0; uint8_t empty[] = {}; - if (EVP_DecryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0) - goto process_auth_decryption_gcm_err; - - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0) - goto process_auth_decryption_gcm_err; - if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) goto process_auth_decryption_gcm_err; - if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) + if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) goto process_auth_decryption_gcm_err; if (aadlen > 0) @@ -903,16 +1129,52 @@ process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, goto process_auth_decryption_gcm_err; if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) - goto process_auth_decryption_gcm_final_err; + return -EFAULT; return 0; process_auth_decryption_gcm_err: - OPENSSL_LOG_ERR("Process openssl auth description gcm failed"); + OPENSSL_LOG_ERR("Process openssl auth decryption gcm failed"); return -EINVAL; +} -process_auth_decryption_gcm_final_err: - return -EFAULT; +/** Process AES-CCM decrypt algorithm */ +static int +process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, + int srclen, uint8_t *aad, int aadlen, uint8_t *iv, + uint8_t *dst, uint8_t *tag, uint8_t tag_len, + EVP_CIPHER_CTX *ctx) +{ + int len = 0; + + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) + goto process_auth_decryption_ccm_err; + + if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) + goto process_auth_decryption_ccm_err; + + if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) + goto process_auth_decryption_ccm_err; + + if (aadlen > 0) + /* + * For AES-CCM, the actual AAD is placed + * 18 bytes after the start of the AAD field, + * according to the API. + */ + if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) + goto process_auth_decryption_ccm_err; + + if (srclen > 0) + if (process_openssl_decryption_update(mbuf_src, offset, &dst, + srclen, ctx)) + return -EFAULT; + + return 0; + +process_auth_decryption_ccm_err: + OPENSSL_LOG_ERR("Process openssl auth decryption ccm failed"); + return -EINVAL; } /** Process standard openssl auth algorithms */ @@ -971,10 +1233,9 @@ process_auth_err: /** Process standard openssl auth algorithms with hmac */ static int process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, - __rte_unused uint8_t *iv, EVP_PKEY *pkey, - int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) + int srclen, HMAC_CTX *ctx) { - size_t dstlen; + unsigned int dstlen; struct rte_mbuf *m; int l, n = srclen; uint8_t *src; @@ -986,19 +1247,16 @@ process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, if (m == 0) goto process_auth_err; - if (EVP_DigestSignInit(ctx, NULL, algo, NULL, pkey) <= 0) - goto process_auth_err; - src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); l = rte_pktmbuf_data_len(m) - offset; if (srclen <= l) { - if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0) + if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) goto process_auth_err; goto process_auth_final; } - if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0) + if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) goto process_auth_err; n -= l; @@ -1006,13 +1264,16 @@ process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, for (m = m->next; (m != NULL) && (n > 0); m = m->next) { src = rte_pktmbuf_mtod(m, uint8_t *); l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; - if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0) + if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) goto process_auth_err; n -= l; } process_auth_final: - if (EVP_DigestSignFinal(ctx, dst, &dstlen) <= 0) + if (HMAC_Final(ctx, dst, &dstlen) != 1) + goto process_auth_err; + + if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) goto process_auth_err; return 0; @@ -1032,8 +1293,9 @@ process_openssl_combined_op { /* cipher */ uint8_t *dst = NULL, *iv, *tag, *aad; - int srclen, ivlen, aadlen, status = -1; + int srclen, aadlen, status = -1; uint32_t offset; + uint8_t taglen; /* * Segmented destination buffer is not supported for @@ -1046,7 +1308,6 @@ process_openssl_combined_op iv = rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv.offset); - ivlen = sess->iv.length; if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { srclen = 0; offset = op->sym->auth.data.offset; @@ -1070,18 +1331,34 @@ process_openssl_combined_op offset + srclen); } - if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) - status = process_openssl_auth_encryption_gcm( - mbuf_src, offset, srclen, - aad, aadlen, iv, ivlen, sess->cipher.key.data, - dst, tag, sess->cipher.ctx, - sess->cipher.evp_algo); - else - status = process_openssl_auth_decryption_gcm( - mbuf_src, offset, srclen, - aad, aadlen, iv, ivlen, sess->cipher.key.data, - dst, tag, sess->cipher.ctx, - sess->cipher.evp_algo); + taglen = sess->auth.digest_length; + + if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || + sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) + status = process_openssl_auth_encryption_gcm( + mbuf_src, offset, srclen, + aad, aadlen, iv, + dst, tag, sess->cipher.ctx); + else + status = process_openssl_auth_encryption_ccm( + mbuf_src, offset, srclen, + aad, aadlen, iv, + dst, tag, taglen, sess->cipher.ctx); + + } else { + if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || + sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) + status = process_openssl_auth_decryption_gcm( + mbuf_src, offset, srclen, + aad, aadlen, iv, + dst, tag, sess->cipher.ctx); + else + status = process_openssl_auth_decryption_ccm( + mbuf_src, offset, srclen, + aad, aadlen, iv, + dst, tag, taglen, sess->cipher.ctx); + } if (status != 0) { if (status == (-EFAULT) && @@ -1122,15 +1399,11 @@ process_openssl_cipher_op if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) status = process_openssl_cipher_encrypt(mbuf_src, dst, op->sym->cipher.data.offset, iv, - sess->cipher.key.data, srclen, - sess->cipher.ctx, - sess->cipher.evp_algo); + srclen, sess->cipher.ctx); else status = process_openssl_cipher_decrypt(mbuf_src, dst, op->sym->cipher.data.offset, iv, - sess->cipher.key.data, srclen, - sess->cipher.ctx, - sess->cipher.evp_algo); + srclen, sess->cipher.ctx); else status = process_openssl_cipher_des3ctr(mbuf_src, dst, op->sym->cipher.data.offset, iv, @@ -1174,8 +1447,7 @@ process_openssl_docsis_bpi_op(struct rte_crypto_op *op, /* Encrypt with the block aligned stream with CBC mode */ status = process_openssl_cipher_encrypt(mbuf_src, dst, op->sym->cipher.data.offset, iv, - sess->cipher.key.data, srclen, - sess->cipher.ctx, sess->cipher.evp_algo); + srclen, sess->cipher.ctx); if (last_block_len) { /* Point at last block */ dst += srclen; @@ -1225,9 +1497,7 @@ process_openssl_docsis_bpi_op(struct rte_crypto_op *op, /* Decrypt with CBC mode */ status |= process_openssl_cipher_decrypt(mbuf_src, dst, op->sym->cipher.data.offset, iv, - sess->cipher.key.data, srclen, - sess->cipher.ctx, - sess->cipher.evp_algo); + srclen, sess->cipher.ctx); } } @@ -1237,9 +1507,9 @@ process_openssl_docsis_bpi_op(struct rte_crypto_op *op, /** Process auth operation */ static void -process_openssl_auth_op - (struct rte_crypto_op *op, struct openssl_session *sess, - struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) +process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, + struct openssl_session *sess, struct rte_mbuf *mbuf_src, + struct rte_mbuf *mbuf_dst) { uint8_t *dst; int srclen, status; @@ -1247,8 +1517,7 @@ process_openssl_auth_op srclen = op->sym->auth.data.length; if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) - dst = (uint8_t *)rte_pktmbuf_append(mbuf_src, - sess->auth.digest_length); + dst = qp->temp_digest; else { dst = op->sym->auth.digest.data; if (dst == NULL) @@ -1265,9 +1534,8 @@ process_openssl_auth_op break; case OPENSSL_AUTH_AS_HMAC: status = process_openssl_auth_hmac(mbuf_src, dst, - op->sym->auth.data.offset, NULL, - sess->auth.hmac.pkey, srclen, - sess->auth.hmac.ctx, sess->auth.hmac.evp_algo); + op->sym->auth.data.offset, srclen, + sess->auth.hmac.ctx); break; default: status = -1; @@ -1279,8 +1547,6 @@ process_openssl_auth_op sess->auth.digest_length) != 0) { op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } - /* Trim area used for digest from mbuf. */ - rte_pktmbuf_trim(mbuf_src, sess->auth.digest_length); } if (status != 0) @@ -1289,7 +1555,7 @@ process_openssl_auth_op /** Process crypto operation for mbuf */ static int -process_op(const struct openssl_qp *qp, struct rte_crypto_op *op, +process_op(struct openssl_qp *qp, struct rte_crypto_op *op, struct openssl_session *sess) { struct rte_mbuf *msrc, *mdst; @@ -1305,14 +1571,14 @@ process_op(const struct openssl_qp *qp, struct rte_crypto_op *op, process_openssl_cipher_op(op, sess, msrc, mdst); break; case OPENSSL_CHAIN_ONLY_AUTH: - process_openssl_auth_op(op, sess, msrc, mdst); + process_openssl_auth_op(qp, op, sess, msrc, mdst); break; case OPENSSL_CHAIN_CIPHER_AUTH: process_openssl_cipher_op(op, sess, msrc, mdst); - process_openssl_auth_op(op, sess, mdst, mdst); + process_openssl_auth_op(qp, op, sess, mdst, mdst); break; case OPENSSL_CHAIN_AUTH_CIPHER: - process_openssl_auth_op(op, sess, msrc, mdst); + process_openssl_auth_op(qp, op, sess, msrc, mdst); process_openssl_cipher_op(op, sess, msrc, mdst); break; case OPENSSL_CHAIN_COMBINED: @@ -1401,19 +1667,12 @@ openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, static int cryptodev_openssl_create(const char *name, struct rte_vdev_device *vdev, - struct rte_crypto_vdev_init_params *init_params) + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; struct openssl_private *internals; - if (init_params->name[0] == '\0') - snprintf(init_params->name, sizeof(init_params->name), - "%s", name); - - dev = rte_cryptodev_vdev_pmd_init(init_params->name, - sizeof(struct openssl_private), - init_params->socket_id, - vdev); + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); if (dev == NULL) { OPENSSL_LOG_ERR("failed to create cryptodev vdev"); goto init_error; @@ -1451,11 +1710,12 @@ init_error: static int cryptodev_openssl_probe(struct rte_vdev_device *vdev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct openssl_private), rte_socket_id(), - {0} + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; const char *name; const char *input_args; @@ -1465,17 +1725,7 @@ cryptodev_openssl_probe(struct rte_vdev_device *vdev) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_vdev_parse_init_params(&init_params, input_args); - - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, - init_params.socket_id); - if (init_params.name[0] != '\0') - RTE_LOG(INFO, PMD, " User defined name = %s\n", - init_params.name); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); return cryptodev_openssl_create(name, vdev, &init_params); } @@ -1484,17 +1734,18 @@ cryptodev_openssl_probe(struct rte_vdev_device *vdev) static int cryptodev_openssl_remove(struct rte_vdev_device *vdev) { + struct rte_cryptodev *cryptodev; const char *name; name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL; - RTE_LOG(INFO, PMD, - "Closing OPENSSL crypto device %s on numa socket %u\n", - name, rte_socket_id()); + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; - return 0; + return rte_cryptodev_pmd_destroy(cryptodev); } static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { @@ -1502,10 +1753,13 @@ static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { .remove = cryptodev_openssl_remove }; +static struct cryptodev_driver openssl_crypto_drv; + RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, cryptodev_openssl_pmd_drv); RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); -RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_openssl_pmd_drv, cryptodev_driver_id); +RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, cryptodev_openssl_pmd_drv, + cryptodev_driver_id); diff --git a/drivers/crypto/openssl/rte_openssl_pmd_ops.c b/drivers/crypto/openssl/rte_openssl_pmd_ops.c index 8cdd0b2e..c5722399 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd_ops.c +++ b/drivers/crypto/openssl/rte_openssl_pmd_ops.c @@ -362,6 +362,36 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { }, } }, } }, + { /* AES CCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_CCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 4, + .max = 16, + .increment = 2 + }, + .aad_size = { + .min = 0, + .max = 65535, + .increment = 1 + }, + .iv_size = { + .min = 7, + .max = 13, + .increment = 1 + }, + }, } + }, } + }, { /* AES GMAC (AUTH) */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { @@ -427,6 +457,26 @@ static const struct rte_cryptodev_capabilities openssl_pmd_capabilities[] = { }, } }, } }, + { /* DES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_DES_CBC, + .block_size = 8, + .key_size = { + .min = 8, + .max = 8, + .increment = 0 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, { /* DES DOCSIS BPI */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { @@ -549,7 +599,7 @@ openssl_pmd_qp_set_unique_name(struct rte_cryptodev *dev, "openssl_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; diff --git a/drivers/crypto/openssl/rte_openssl_pmd_private.h b/drivers/crypto/openssl/rte_openssl_pmd_private.h index b7f74752..26bf862c 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd_private.h +++ b/drivers/crypto/openssl/rte_openssl_pmd_private.h @@ -34,6 +34,7 @@ #define _OPENSSL_PMD_PRIVATE_H_ #include <openssl/evp.h> +#include <openssl/hmac.h> #include <openssl/des.h> #define CRYPTODEV_NAME_OPENSSL_PMD crypto_openssl @@ -59,6 +60,8 @@ #define OPENSSL_LOG_DBG(fmt, args...) #endif +/* Maximum length for digest (SHA-512 needs 64 bytes) */ +#define DIGEST_LENGTH_MAX 64 /** OPENSSL operation order mode enumerator */ enum openssl_chain_order { @@ -103,6 +106,11 @@ struct openssl_qp { /**< Session Mempool */ struct rte_cryptodev_stats stats; /**< Queue pair statistics */ + uint8_t temp_digest[DIGEST_LENGTH_MAX]; + /**< Buffer used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ } __rte_cache_aligned; /** OPENSSL crypto private session structure */ @@ -164,7 +172,7 @@ struct openssl_session { /**< pointer to EVP key */ const EVP_MD *evp_algo; /**< pointer to EVP algorithm function */ - EVP_MD_CTX *ctx; + HMAC_CTX *ctx; /**< pointer to EVP context structure */ } hmac; }; diff --git a/drivers/crypto/qat/Makefile b/drivers/crypto/qat/Makefile index 7322ffe4..745d0197 100644 --- a/drivers/crypto/qat/Makefile +++ b/drivers/crypto/qat/Makefile @@ -43,6 +43,9 @@ CFLAGS += -O3 # external library include paths CFLAGS += -I$(SRCDIR)/qat_adf LDLIBS += -lcrypto +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_pci -lrte_bus_pci # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_QAT) += qat_crypto.c diff --git a/drivers/crypto/qat/qat_adf/icp_qat_hw.h b/drivers/crypto/qat/qat_adf/icp_qat_hw.h index ebe245f6..d03688c7 100644 --- a/drivers/crypto/qat/qat_adf/icp_qat_hw.h +++ b/drivers/crypto/qat/qat_adf/icp_qat_hw.h @@ -301,6 +301,26 @@ enum icp_qat_hw_cipher_convert { #define ICP_QAT_HW_CIPHER_MAX_KEY_SZ ICP_QAT_HW_AES_256_F8_KEY_SZ +/* These defines describe position of the bit-fields + * in the flags byte in B0 + */ +#define ICP_QAT_HW_CCM_B0_FLAGS_ADATA_SHIFT 6 +#define ICP_QAT_HW_CCM_B0_FLAGS_T_SHIFT 3 + +#define ICP_QAT_HW_CCM_BUILD_B0_FLAGS(Adata, t, q) \ + ((((Adata) > 0 ? 1 : 0) << ICP_QAT_HW_CCM_B0_FLAGS_ADATA_SHIFT) \ + | ((((t) - 2) >> 1) << ICP_QAT_HW_CCM_B0_FLAGS_T_SHIFT) \ + | ((q) - 1)) + +#define ICP_QAT_HW_CCM_NQ_CONST 15 +#define ICP_QAT_HW_CCM_AAD_B0_LEN 16 +#define ICP_QAT_HW_CCM_AAD_LEN_INFO 2 +#define ICP_QAT_HW_CCM_AAD_DATA_OFFSET (ICP_QAT_HW_CCM_AAD_B0_LEN + \ + ICP_QAT_HW_CCM_AAD_LEN_INFO) +#define ICP_QAT_HW_CCM_AAD_ALIGNMENT 16 +#define ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE 4 +#define ICP_QAT_HW_CCM_NONCE_OFFSET 1 + struct icp_qat_hw_cipher_algo_blk { struct icp_qat_hw_cipher_config cipher_config; uint8_t key[ICP_QAT_HW_CIPHER_MAX_KEY_SZ]; diff --git a/drivers/crypto/qat/qat_adf/qat_algs.h b/drivers/crypto/qat/qat_adf/qat_algs.h index 2c8e03c0..802ba95d 100644 --- a/drivers/crypto/qat/qat_adf/qat_algs.h +++ b/drivers/crypto/qat/qat_adf/qat_algs.h @@ -104,8 +104,8 @@ struct qat_alg_buf_list { struct qat_crypto_op_cookie { struct qat_alg_buf_list qat_sgl_list_src; struct qat_alg_buf_list qat_sgl_list_dst; - phys_addr_t qat_sgl_src_phys_addr; - phys_addr_t qat_sgl_dst_phys_addr; + rte_iova_t qat_sgl_src_phys_addr; + rte_iova_t qat_sgl_dst_phys_addr; }; /* Common content descriptor */ @@ -124,7 +124,7 @@ struct qat_session { void *bpi_ctx; struct qat_alg_cd cd; uint8_t *cd_cur_ptr; - phys_addr_t cd_paddr; + rte_iova_t cd_paddr; struct icp_qat_fw_la_bulk_req fw_req; uint8_t aad_len; struct qat_crypto_instance *inst; diff --git a/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c b/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c index 2d16c9e2..db6c9a32 100644 --- a/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c +++ b/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c @@ -124,6 +124,9 @@ static int qat_hash_get_state1_size(enum icp_qat_hw_auth_algo qat_hash_alg) case ICP_QAT_HW_AUTH_ALGO_NULL: return QAT_HW_ROUND_UP(ICP_QAT_HW_NULL_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); + case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC: + return QAT_HW_ROUND_UP(ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ, + QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_DELIMITER: /* return maximum state1 size in this case */ return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA512_STATE1_SZ, @@ -876,6 +879,31 @@ int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, ICP_QAT_HW_AUTH_ALGO_NULL); state2_size = ICP_QAT_HW_NULL_STATE2_SZ; break; + case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC: + qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM; + state1_size = qat_hash_get_state1_size( + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC); + state2_size = ICP_QAT_HW_AES_CBC_MAC_KEY_SZ + + ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ; + + if (aad_length > 0) { + aad_length += ICP_QAT_HW_CCM_AAD_B0_LEN + + ICP_QAT_HW_CCM_AAD_LEN_INFO; + auth_param->u2.aad_sz = + RTE_ALIGN_CEIL(aad_length, + ICP_QAT_HW_CCM_AAD_ALIGNMENT); + } else { + auth_param->u2.aad_sz = ICP_QAT_HW_CCM_AAD_B0_LEN; + } + + cdesc->aad_len = aad_length; + hash->auth_counter.counter = 0; + + hash_cd_ctrl->outer_prefix_sz = digestsize; + auth_param->hash_state_sz = digestsize; + + memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen); + break; case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9: state1_size = qat_hash_get_state1_size( ICP_QAT_HW_AUTH_ALGO_KASUMI_F9); diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c index 1f52cabf..a572967c 100644 --- a/drivers/crypto/qat/qat_crypto.c +++ b/drivers/crypto/qat/qat_crypto.c @@ -44,15 +44,12 @@ #include <rte_log.h> #include <rte_debug.h> #include <rte_memory.h> -#include <rte_memzone.h> #include <rte_tailq.h> -#include <rte_ether.h> #include <rte_malloc.h> #include <rte_launch.h> #include <rte_eal.h> #include <rte_per_lcore.h> #include <rte_lcore.h> -#include <rte_atomic.h> #include <rte_branch_prediction.h> #include <rte_mempool.h> #include <rte_mbuf.h> @@ -60,7 +57,10 @@ #include <rte_spinlock.h> #include <rte_hexdump.h> #include <rte_crypto_sym.h> -#include <rte_cryptodev_pci.h> +#include <rte_byteorder.h> +#include <rte_pci.h> +#include <rte_bus_pci.h> + #include <openssl/evp.h> #include "qat_logs.h" @@ -253,10 +253,21 @@ qat_get_cmd_id(const struct rte_crypto_sym_xform *xform) /* AEAD */ if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + /* AES-GCM and AES-CCM works with different direction + * GCM first encrypts and generate hash where AES-CCM + * first generate hash and encrypts. Similar relation + * applies to decryption. + */ if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) - return ICP_QAT_FW_LA_CMD_CIPHER_HASH; + if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) + return ICP_QAT_FW_LA_CMD_CIPHER_HASH; + else + return ICP_QAT_FW_LA_CMD_HASH_CIPHER; else - return ICP_QAT_FW_LA_CMD_HASH_CIPHER; + if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) + return ICP_QAT_FW_LA_CMD_HASH_CIPHER; + else + return ICP_QAT_FW_LA_CMD_CIPHER_HASH; } if (xform->next == NULL) @@ -516,7 +527,7 @@ qat_crypto_set_session_parameters(struct rte_cryptodev *dev, PMD_INIT_FUNC_TRACE(); /* Set context descriptor physical address */ - session->cd_paddr = rte_mempool_virt2phy(NULL, session) + + session->cd_paddr = rte_mempool_virt2iova(session) + offsetof(struct qat_session, cd); session->min_qat_dev_gen = QAT_GEN1; @@ -736,6 +747,7 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, struct qat_session *session) { struct rte_crypto_aead_xform *aead_xform = &xform->aead; + enum rte_crypto_auth_operation crypto_operation; /* * Store AEAD IV parameters as cipher IV, @@ -755,21 +767,33 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; break; case RTE_CRYPTO_AEAD_AES_CCM: - PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported AEAD alg %u", - aead_xform->algo); - return -ENOTSUP; + if (qat_alg_validate_aes_key(aead_xform->key.length, + &session->qat_cipher_alg) != 0) { + PMD_DRV_LOG(ERR, "Invalid AES key size"); + return -EINVAL; + } + session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC; + break; default: PMD_DRV_LOG(ERR, "Crypto: Undefined AEAD specified %u\n", aead_xform->algo); return -EINVAL; } - if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { + if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && + aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) || + (aead_xform->op == RTE_CRYPTO_AEAD_OP_DECRYPT && + aead_xform->algo == RTE_CRYPTO_AEAD_AES_CCM)) { session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; /* * It needs to create cipher desc content first, * then authentication */ + + crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ? + RTE_CRYPTO_AUTH_OP_GENERATE : RTE_CRYPTO_AUTH_OP_VERIFY; + if (qat_alg_aead_session_create_content_desc_cipher(session, aead_xform->key.data, aead_xform->key.length)) @@ -780,7 +804,7 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, aead_xform->key.length, aead_xform->aad_length, aead_xform->digest_length, - RTE_CRYPTO_AUTH_OP_GENERATE)) + crypto_operation)) return -EINVAL; } else { session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; @@ -788,12 +812,16 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, * It needs to create authentication desc content first, * then cipher */ + + crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ? + RTE_CRYPTO_AUTH_OP_VERIFY : RTE_CRYPTO_AUTH_OP_GENERATE; + if (qat_alg_aead_session_create_content_desc_auth(session, aead_xform->key.data, aead_xform->key.length, aead_xform->aad_length, aead_xform->digest_length, - RTE_CRYPTO_AUTH_OP_VERIFY)) + crypto_operation)) return -EINVAL; if (qat_alg_aead_session_create_content_desc_cipher(session, @@ -923,6 +951,14 @@ qat_bpicipher_postprocess(struct qat_session *ctx, return sym_op->cipher.data.length - last_block_len; } +static inline void +txq_write_tail(struct qat_qp *qp, struct qat_queue *q) { + WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number, + q->hw_queue_number, q->tail); + q->nb_pending_requests = 0; + q->csr_tail = q->tail; +} + uint16_t qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) @@ -946,10 +982,10 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, tail = queue->tail; /* Find how many can actually fit on the ring */ - overflow = rte_atomic16_add_return(&tmp_qp->inflights16, nb_ops) - - queue->max_inflights; + tmp_qp->inflights16 += nb_ops; + overflow = tmp_qp->inflights16 - queue->max_inflights; if (overflow > 0) { - rte_atomic16_sub(&tmp_qp->inflights16, overflow); + tmp_qp->inflights16 -= overflow; nb_ops_possible = nb_ops - overflow; if (nb_ops_possible == 0) return 0; @@ -964,8 +1000,7 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, * This message cannot be enqueued, * decrease number of ops that wasn't sent */ - rte_atomic16_sub(&tmp_qp->inflights16, - nb_ops_possible - nb_ops_sent); + tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent; if (nb_ops_sent == 0) return 0; goto kick_tail; @@ -976,26 +1011,59 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, cur_op++; } kick_tail: - WRITE_CSR_RING_TAIL(tmp_qp->mmap_bar_addr, queue->hw_bundle_number, - queue->hw_queue_number, tail); queue->tail = tail; tmp_qp->stats.enqueued_count += nb_ops_sent; + queue->nb_pending_requests += nb_ops_sent; + if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH || + queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) { + txq_write_tail(tmp_qp, queue); + } return nb_ops_sent; } +static inline +void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q) +{ + uint32_t old_head, new_head; + uint32_t max_head; + + old_head = q->csr_head; + new_head = q->head; + max_head = qp->nb_descriptors * q->msg_size; + + /* write out free descriptors */ + void *cur_desc = (uint8_t *)q->base_addr + old_head; + + if (new_head < old_head) { + memset(cur_desc, ADF_RING_EMPTY_SIG, max_head - old_head); + memset(q->base_addr, ADF_RING_EMPTY_SIG, new_head); + } else { + memset(cur_desc, ADF_RING_EMPTY_SIG, new_head - old_head); + } + q->nb_processed_responses = 0; + q->csr_head = new_head; + + /* write current head to CSR */ + WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number, + q->hw_queue_number, new_head); +} + uint16_t qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) { - struct qat_queue *queue; + struct qat_queue *rx_queue, *tx_queue; struct qat_qp *tmp_qp = (struct qat_qp *)qp; uint32_t msg_counter = 0; struct rte_crypto_op *rx_op; struct icp_qat_fw_comn_resp *resp_msg; + uint32_t head; - queue = &(tmp_qp->rx_q); + rx_queue = &(tmp_qp->rx_q); + tx_queue = &(tmp_qp->tx_q); + head = rx_queue->head; resp_msg = (struct icp_qat_fw_comn_resp *) - ((uint8_t *)queue->base_addr + queue->head); + ((uint8_t *)rx_queue->base_addr + head); while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG && msg_counter != nb_ops) { @@ -1005,7 +1073,6 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg, sizeof(struct icp_qat_fw_comn_resp)); - #endif if (ICP_QAT_FW_COMN_STATUS_FLAG_OK != ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET( @@ -1022,23 +1089,26 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; } - *(uint32_t *)resp_msg = ADF_RING_EMPTY_SIG; - queue->head = adf_modulo(queue->head + - queue->msg_size, - ADF_RING_SIZE_MODULO(queue->queue_size)); + head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo); resp_msg = (struct icp_qat_fw_comn_resp *) - ((uint8_t *)queue->base_addr + - queue->head); + ((uint8_t *)rx_queue->base_addr + head); *ops = rx_op; ops++; msg_counter++; } if (msg_counter > 0) { - WRITE_CSR_RING_HEAD(tmp_qp->mmap_bar_addr, - queue->hw_bundle_number, - queue->hw_queue_number, queue->head); - rte_atomic16_sub(&tmp_qp->inflights16, msg_counter); + rx_queue->head = head; tmp_qp->stats.dequeued_count += msg_counter; + rx_queue->nb_processed_responses += msg_counter; + tmp_qp->inflights16 -= msg_counter; + + if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) + rxq_free_desc(tmp_qp, rx_queue); + } + /* also check if tail needs to be advanced */ + if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH && + tx_queue->tail != tx_queue->csr_tail) { + txq_write_tail(tmp_qp, tx_queue); } return msg_counter; } @@ -1049,7 +1119,7 @@ qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start, { int nr = 1; - uint32_t buf_len = rte_pktmbuf_mtophys(buf) - + uint32_t buf_len = rte_pktmbuf_iova(buf) - buff_start + rte_pktmbuf_data_len(buf); list->bufers[0].addr = buff_start; @@ -1073,7 +1143,7 @@ qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start, list->bufers[nr].len = rte_pktmbuf_data_len(buf); list->bufers[nr].resrvd = 0; - list->bufers[nr].addr = rte_pktmbuf_mtophys(buf); + list->bufers[nr].addr = rte_pktmbuf_iova(buf); buf_len += list->bufers[nr].len; buf = buf->next; @@ -1112,6 +1182,29 @@ set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, } } +/** Set IV for CCM is special case, 0th byte is set to q-1 + * where q is padding of nonce in 16 byte block + */ +static inline void +set_cipher_iv_ccm(uint16_t iv_length, uint16_t iv_offset, + struct icp_qat_fw_la_cipher_req_params *cipher_param, + struct rte_crypto_op *op, uint8_t q, uint8_t aad_len_field_sz) +{ + rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array) + + ICP_QAT_HW_CCM_NONCE_OFFSET, + rte_crypto_op_ctod_offset(op, uint8_t *, + iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET, + iv_length); + *(uint8_t *)&cipher_param->u.cipher_IV_array[0] = + q - ICP_QAT_HW_CCM_NONCE_OFFSET; + + if (aad_len_field_sz) + rte_memcpy(&op->sym->aead.aad.data[ICP_QAT_HW_CCM_NONCE_OFFSET], + rte_crypto_op_ctod_offset(op, uint8_t *, + iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET, + iv_length); +} + static inline int qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp) @@ -1156,6 +1249,8 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, return -EINVAL; } + + qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; @@ -1164,9 +1259,13 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER || ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { - /* AES-GCM */ + /* AES-GCM or AES-CCM */ if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || - ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { + ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 || + (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128 + && ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE + && ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) { do_aead = 1; } else { do_auth = 1; @@ -1273,6 +1372,11 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, } if (do_aead) { + /* + * This address may used for setting AAD physical pointer + * into IV offset from op + */ + rte_iova_t aad_phys_addr_aead = op->sym->aead.aad.phys_addr; if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || ctx->qat_hash_alg == @@ -1286,6 +1390,87 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); } + set_cipher_iv(ctx->cipher_iv.length, + ctx->cipher_iv.offset, + cipher_param, op, qat_req); + + } else if (ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC) { + + /* In case of AES-CCM this may point to user selected memory + * or iv offset in cypto_op + */ + uint8_t *aad_data = op->sym->aead.aad.data; + /* This is true AAD length, it not includes 18 bytes of + * preceding data + */ + uint8_t aad_ccm_real_len = 0; + + uint8_t aad_len_field_sz = 0; + uint32_t msg_len_be = + rte_bswap32(op->sym->aead.data.length); + + if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) { + aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO; + aad_ccm_real_len = ctx->aad_len - + ICP_QAT_HW_CCM_AAD_B0_LEN - + ICP_QAT_HW_CCM_AAD_LEN_INFO; + } else { + /* + * aad_len not greater than 18, so no actual aad data, + * then use IV after op for B0 block + */ + aad_data = rte_crypto_op_ctod_offset(op, uint8_t *, + ctx->cipher_iv.offset); + aad_phys_addr_aead = + rte_crypto_op_ctophys_offset(op, + ctx->cipher_iv.offset); + } + + uint8_t q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length; + + aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(aad_len_field_sz, + ctx->digest_length, q); + + if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) { + memcpy(aad_data + ctx->cipher_iv.length + + ICP_QAT_HW_CCM_NONCE_OFFSET + + (q - ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE), + (uint8_t *)&msg_len_be, + ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE); + } else { + memcpy(aad_data + ctx->cipher_iv.length + + ICP_QAT_HW_CCM_NONCE_OFFSET, + (uint8_t *)&msg_len_be + + (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE + - q), q); + } + + if (aad_len_field_sz > 0) { + *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] + = rte_bswap16(aad_ccm_real_len); + + if ((aad_ccm_real_len + aad_len_field_sz) + % ICP_QAT_HW_CCM_AAD_B0_LEN) { + uint8_t pad_len = 0; + uint8_t pad_idx = 0; + + pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN - + ((aad_ccm_real_len + aad_len_field_sz) % + ICP_QAT_HW_CCM_AAD_B0_LEN); + pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN + + aad_ccm_real_len + aad_len_field_sz; + memset(&aad_data[pad_idx], + 0, pad_len); + } + + } + + set_cipher_iv_ccm(ctx->cipher_iv.length, + ctx->cipher_iv.offset, + cipher_param, op, q, + aad_len_field_sz); + } cipher_len = op->sym->aead.data.length; @@ -1293,10 +1478,8 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, auth_len = op->sym->aead.data.length; auth_ofs = op->sym->aead.data.offset; - auth_param->u1.aad_adr = op->sym->aead.aad.phys_addr; + auth_param->u1.aad_adr = aad_phys_addr_aead; auth_param->auth_res_addr = op->sym->aead.digest.phys_addr; - set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset, - cipher_param, op, qat_req); min_ofs = op->sym->aead.data.offset; } @@ -1316,26 +1499,26 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, * so as not to overwrite data in dest buffer */ src_buf_start = - rte_pktmbuf_mtophys_offset(op->sym->m_src, min_ofs); + rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs); dst_buf_start = - rte_pktmbuf_mtophys_offset(op->sym->m_dst, min_ofs); + rte_pktmbuf_iova_offset(op->sym->m_dst, min_ofs); } else { /* In-place operation * Start DMA at nearest aligned address below min_ofs */ src_buf_start = - rte_pktmbuf_mtophys_offset(op->sym->m_src, min_ofs) + rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs) & QAT_64_BTYE_ALIGN_MASK; - if (unlikely((rte_pktmbuf_mtophys(op->sym->m_src) - + if (unlikely((rte_pktmbuf_iova(op->sym->m_src) - rte_pktmbuf_headroom(op->sym->m_src)) > src_buf_start)) { /* alignment has pushed addr ahead of start of mbuf * so revert and take the performance hit */ src_buf_start = - rte_pktmbuf_mtophys_offset(op->sym->m_src, + rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs); } dst_buf_start = src_buf_start; @@ -1343,7 +1526,7 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, if (do_cipher || do_aead) { cipher_param->cipher_offset = - (uint32_t)rte_pktmbuf_mtophys_offset( + (uint32_t)rte_pktmbuf_iova_offset( op->sym->m_src, cipher_ofs) - src_buf_start; cipher_param->cipher_length = cipher_len; } else { @@ -1352,7 +1535,7 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, } if (do_auth || do_aead) { - auth_param->auth_off = (uint32_t)rte_pktmbuf_mtophys_offset( + auth_param->auth_off = (uint32_t)rte_pktmbuf_iova_offset( op->sym->m_src, auth_ofs) - src_buf_start; auth_param->auth_len = auth_len; } else { diff --git a/drivers/crypto/qat/qat_crypto.h b/drivers/crypto/qat/qat_crypto.h index 3f35a00e..c64d7756 100644 --- a/drivers/crypto/qat/qat_crypto.h +++ b/drivers/crypto/qat/qat_crypto.h @@ -50,6 +50,13 @@ (((num) + (align) - 1) & ~((align) - 1)) #define QAT_64_BTYE_ALIGN_MASK (~0x3f) +#define QAT_CSR_HEAD_WRITE_THRESH 32U +/* number of requests to accumulate before writing head CSR */ +#define QAT_CSR_TAIL_WRITE_THRESH 32U +/* number of requests to accumulate before writing tail CSR */ +#define QAT_CSR_TAIL_FORCE_WRITE_THRESH 256U +/* number of inflights below which no tail write coalescing should occur */ + struct qat_session; enum qat_device_gen { @@ -63,7 +70,7 @@ enum qat_device_gen { struct qat_queue { char memz_name[RTE_MEMZONE_NAMESIZE]; void *base_addr; /* Base address */ - phys_addr_t base_phys_addr; /* Queue physical address */ + rte_iova_t base_phys_addr; /* Queue physical address */ uint32_t head; /* Shadow copy of the head */ uint32_t tail; /* Shadow copy of the tail */ uint32_t modulo; @@ -73,11 +80,17 @@ struct qat_queue { uint8_t hw_bundle_number; uint8_t hw_queue_number; /* HW queue aka ring offset on bundle */ + uint32_t csr_head; /* last written head value */ + uint32_t csr_tail; /* last written tail value */ + uint16_t nb_processed_responses; + /* number of responses processed since last CSR head write */ + uint16_t nb_pending_requests; + /* number of requests pending since last CSR tail write */ }; struct qat_qp { void *mmap_bar_addr; - rte_atomic16_t inflights16; + uint16_t inflights16; struct qat_queue tx_q; struct qat_queue rx_q; struct rte_cryptodev_stats stats; diff --git a/drivers/crypto/qat/qat_crypto_capabilities.h b/drivers/crypto/qat/qat_crypto_capabilities.h index 70120072..89ba27d4 100644 --- a/drivers/crypto/qat/qat_crypto_capabilities.h +++ b/drivers/crypto/qat/qat_crypto_capabilities.h @@ -48,9 +48,9 @@ .increment = 1 \ }, \ .digest_size = { \ - .min = 20, \ + .min = 1, \ .max = 20, \ - .increment = 0 \ + .increment = 1 \ }, \ .iv_size = { 0 } \ }, } \ @@ -69,9 +69,9 @@ .increment = 1 \ }, \ .digest_size = { \ - .min = 28, \ + .min = 1, \ .max = 28, \ - .increment = 0 \ + .increment = 1 \ }, \ .iv_size = { 0 } \ }, } \ @@ -90,9 +90,9 @@ .increment = 1 \ }, \ .digest_size = { \ - .min = 32, \ + .min = 1, \ .max = 32, \ - .increment = 0 \ + .increment = 1 \ }, \ .iv_size = { 0 } \ }, } \ @@ -111,9 +111,9 @@ .increment = 1 \ }, \ .digest_size = { \ - .min = 48, \ + .min = 1, \ .max = 48, \ - .increment = 0 \ + .increment = 1 \ }, \ .iv_size = { 0 } \ }, } \ @@ -132,9 +132,9 @@ .increment = 1 \ }, \ .digest_size = { \ - .min = 64, \ + .min = 1, \ .max = 64, \ - .increment = 0 \ + .increment = 1 \ }, \ .iv_size = { 0 } \ }, } \ @@ -153,9 +153,9 @@ .increment = 1 \ }, \ .digest_size = { \ - .min = 16, \ + .min = 1, \ .max = 16, \ - .increment = 0 \ + .increment = 1 \ }, \ .iv_size = { 0 } \ }, } \ @@ -183,6 +183,36 @@ }, } \ }, } \ }, \ + { /* AES CCM */ \ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ + {.sym = { \ + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, \ + {.aead = { \ + .algo = RTE_CRYPTO_AEAD_AES_CCM, \ + .block_size = 16, \ + .key_size = { \ + .min = 16, \ + .max = 16, \ + .increment = 0 \ + }, \ + .digest_size = { \ + .min = 4, \ + .max = 16, \ + .increment = 2 \ + }, \ + .aad_size = { \ + .min = 0, \ + .max = 224, \ + .increment = 1 \ + }, \ + .iv_size = { \ + .min = 7, \ + .max = 13, \ + .increment = 1 \ + }, \ + }, } \ + }, } \ + }, \ { /* AES GCM */ \ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \ {.sym = { \ diff --git a/drivers/crypto/qat/qat_qp.c b/drivers/crypto/qat/qat_qp.c index 5048d214..ced3aa6a 100644 --- a/drivers/crypto/qat/qat_qp.c +++ b/drivers/crypto/qat/qat_qp.c @@ -37,6 +37,7 @@ #include <rte_memzone.h> #include <rte_cryptodev_pmd.h> #include <rte_pci.h> +#include <rte_bus_pci.h> #include <rte_atomic.h> #include <rte_prefetch.h> @@ -122,14 +123,9 @@ queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size, break; default: memzone_flags = RTE_MEMZONE_SIZE_HINT_ONLY; -} -#ifdef RTE_LIBRTE_XEN_DOM0 - return rte_memzone_reserve_bounded(queue_name, queue_size, - socket_id, 0, RTE_CACHE_LINE_SIZE, RTE_PGSIZE_2M); -#else + } return rte_memzone_reserve_aligned(queue_name, queue_size, socket_id, memzone_flags, queue_size); -#endif } int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, @@ -186,7 +182,7 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, RTE_CACHE_LINE_SIZE); qp->mmap_bar_addr = pci_dev->mem_resource[0].addr; - rte_atomic16_init(&qp->inflights16); + qp->inflights16 = 0; if (qat_tx_queue_create(dev, &(qp->tx_q), queue_pair_id, qp_conf->nb_descriptors, socket_id) != 0) { @@ -232,14 +228,12 @@ int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, qp->op_cookies[i]; sql_cookie->qat_sgl_src_phys_addr = - rte_mempool_virt2phy(qp->op_cookie_pool, - sql_cookie) + + rte_mempool_virt2iova(sql_cookie) + offsetof(struct qat_crypto_op_cookie, qat_sgl_list_src); sql_cookie->qat_sgl_dst_phys_addr = - rte_mempool_virt2phy(qp->op_cookie_pool, - sql_cookie) + + rte_mempool_virt2iova(sql_cookie) + offsetof(struct qat_crypto_op_cookie, qat_sgl_list_dst); } @@ -269,7 +263,7 @@ int qat_crypto_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) } /* Don't free memory if there are still responses to be processed */ - if (rte_atomic16_read(&(qp->inflights16)) == 0) { + if (qp->inflights16 == 0) { qat_queue_delete(&(qp->tx_q)); qat_queue_delete(&(qp->rx_q)); } else { @@ -377,7 +371,7 @@ qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue, } queue->base_addr = (char *)qp_mz->addr; - queue->base_phys_addr = qp_mz->phys_addr; + queue->base_phys_addr = qp_mz->iova; if (qat_qp_check_queue_alignment(queue->base_phys_addr, queue_size_bytes)) { PMD_DRV_LOG(ERR, "Invalid alignment on queue create " diff --git a/drivers/crypto/qat/rte_qat_cryptodev.c b/drivers/crypto/qat/rte_qat_cryptodev.c index 7d56fca4..4f8e4bfe 100644 --- a/drivers/crypto/qat/rte_qat_cryptodev.c +++ b/drivers/crypto/qat/rte_qat_cryptodev.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. + * Copyright(c) 2015-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,11 +31,12 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include <rte_bus_pci.h> #include <rte_common.h> #include <rte_dev.h> #include <rte_malloc.h> +#include <rte_pci.h> #include <rte_cryptodev_pmd.h> -#include <rte_cryptodev_pci.h> #include "qat_crypto.h" #include "qat_logs.h" @@ -97,15 +98,18 @@ static const struct rte_pci_id pci_id_qat_map[] = { }; static int -crypto_qat_dev_init(struct rte_cryptodev *cryptodev) +crypto_qat_create(const char *name, struct rte_pci_device *pci_dev, + struct rte_cryptodev_pmd_init_params *init_params) { + struct rte_cryptodev *cryptodev; struct qat_pmd_private *internals; PMD_INIT_FUNC_TRACE(); - PMD_DRV_LOG(DEBUG, "Found crypto device at %02x:%02x.%x", - RTE_DEV_TO_PCI(cryptodev->device)->addr.bus, - RTE_DEV_TO_PCI(cryptodev->device)->addr.devid, - RTE_DEV_TO_PCI(cryptodev->device)->addr.function); + + cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device, + init_params); + if (cryptodev == NULL) + return -ENODEV; cryptodev->driver_id = cryptodev_qat_driver_id; cryptodev->dev_ops = &crypto_qat_ops; @@ -119,8 +123,8 @@ crypto_qat_dev_init(struct rte_cryptodev *cryptodev) RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; internals = cryptodev->data->dev_private; - internals->max_nb_sessions = RTE_QAT_PMD_MAX_NB_SESSIONS; - switch (RTE_DEV_TO_PCI(cryptodev->device)->id.device_id) { + internals->max_nb_sessions = init_params->max_nb_sessions; + switch (pci_dev->id.device_id) { case 0x0443: internals->qat_dev_gen = QAT_GEN1; internals->qat_dev_capabilities = qat_gen1_capabilities; @@ -151,15 +155,43 @@ crypto_qat_dev_init(struct rte_cryptodev *cryptodev) } static int crypto_qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, - struct rte_pci_device *pci_dev) + struct rte_pci_device *pci_dev) { - return rte_cryptodev_pci_generic_probe(pci_dev, - sizeof(struct qat_pmd_private), crypto_qat_dev_init); + struct rte_cryptodev_pmd_init_params init_params = { + .name = "", + .socket_id = rte_socket_id(), + .private_data_size = sizeof(struct qat_pmd_private), + .max_nb_sessions = RTE_QAT_PMD_MAX_NB_SESSIONS + }; + char name[RTE_CRYPTODEV_NAME_MAX_LEN]; + + PMD_DRV_LOG(DEBUG, "Found QAT device at %02x:%02x.%x", + pci_dev->addr.bus, + pci_dev->addr.devid, + pci_dev->addr.function); + + rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); + + return crypto_qat_create(name, pci_dev, &init_params); } static int crypto_qat_pci_remove(struct rte_pci_device *pci_dev) { - return rte_cryptodev_pci_generic_remove(pci_dev, NULL); + struct rte_cryptodev *cryptodev; + char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; + + if (pci_dev == NULL) + return -EINVAL; + + rte_pci_device_name(&pci_dev->addr, cryptodev_name, + sizeof(cryptodev_name)); + + cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name); + if (cryptodev == NULL) + return -ENODEV; + + /* free crypto device */ + return rte_cryptodev_pmd_destroy(cryptodev); } static struct rte_pci_driver rte_qat_pmd = { @@ -169,6 +201,9 @@ static struct rte_pci_driver rte_qat_pmd = { .remove = crypto_qat_pci_remove }; +static struct cryptodev_driver qat_crypto_drv; + RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_QAT_SYM_PMD, rte_qat_pmd); RTE_PMD_REGISTER_PCI_TABLE(CRYPTODEV_NAME_QAT_SYM_PMD, pci_id_qat_map); -RTE_PMD_REGISTER_CRYPTO_DRIVER(rte_qat_pmd, cryptodev_qat_driver_id); +RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv, rte_qat_pmd, + cryptodev_qat_driver_id); diff --git a/drivers/crypto/scheduler/Makefile b/drivers/crypto/scheduler/Makefile index b045410c..123b0f6d 100644 --- a/drivers/crypto/scheduler/Makefile +++ b/drivers/crypto/scheduler/Makefile @@ -36,6 +36,9 @@ LIB = librte_pmd_crypto_scheduler.a # build flags CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev -lrte_kvargs -lrte_reorder +LDLIBS += -lrte_bus_vdev # library version LIBABIVER := 1 diff --git a/drivers/crypto/scheduler/scheduler_pmd.c b/drivers/crypto/scheduler/scheduler_pmd.c index 400fc4f1..acdf6361 100644 --- a/drivers/crypto/scheduler/scheduler_pmd.c +++ b/drivers/crypto/scheduler/scheduler_pmd.c @@ -33,8 +33,7 @@ #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> -#include <rte_cryptodev_vdev.h> -#include <rte_vdev.h> +#include <rte_bus_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> #include <rte_reorder.h> @@ -45,7 +44,7 @@ uint8_t cryptodev_driver_id; struct scheduler_init_params { - struct rte_crypto_vdev_init_params def_p; + struct rte_cryptodev_pmd_init_params def_p; uint32_t nb_slaves; enum rte_cryptodev_scheduler_mode mode; uint32_t enable_ordering; @@ -107,21 +106,18 @@ cryptodev_scheduler_create(const char *name, uint32_t i; int ret; - if (init_params->def_p.name[0] == '\0') - snprintf(init_params->def_p.name, - sizeof(init_params->def_p.name), - "%s", name); - - dev = rte_cryptodev_vdev_pmd_init(init_params->def_p.name, - sizeof(struct scheduler_ctx), - init_params->def_p.socket_id, - vdev); + dev = rte_cryptodev_pmd_create(name, &vdev->device, + &init_params->def_p); if (dev == NULL) { CS_LOG_ERR("driver %s: failed to create cryptodev vdev", name); return -EFAULT; } + if (init_params->wcmask != 0) + RTE_LOG(INFO, PMD, " workers core mask = %"PRIx64"\n", + init_params->wcmask); + dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_crypto_scheduler_pmd_ops; @@ -240,10 +236,7 @@ cryptodev_scheduler_remove(struct rte_vdev_device *vdev) sched_ctx->slaves[i].dev_id); } - RTE_LOG(INFO, PMD, "Closing Crypto Scheduler device %s on numa " - "socket %u\n", name, rte_socket_id()); - - return 0; + return rte_cryptodev_pmd_destroy(dev); } /** Parse integer from integer argument */ @@ -304,7 +297,7 @@ static int parse_name_arg(const char *key __rte_unused, const char *value, void *extra_args) { - struct rte_crypto_vdev_init_params *params = extra_args; + struct rte_cryptodev_pmd_init_params *params = extra_args; if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) { CS_LOG_ERR("Invalid name %s, should be less than " @@ -462,10 +455,11 @@ cryptodev_scheduler_probe(struct rte_vdev_device *vdev) { struct scheduler_init_params init_params = { .def_p = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, + "", + sizeof(struct scheduler_ctx), rte_socket_id(), - "" + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }, .nb_slaves = 0, .mode = CDEV_SCHED_MODE_NOT_SET, @@ -481,19 +475,6 @@ cryptodev_scheduler_probe(struct rte_vdev_device *vdev) scheduler_parse_init_params(&init_params, rte_vdev_device_args(vdev)); - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", - name, - init_params.def_p.socket_id); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.def_p.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.def_p.max_nb_sessions); - if (init_params.def_p.name[0] != '\0') - RTE_LOG(INFO, PMD, " User defined name = %s\n", - init_params.def_p.name); - if (init_params.wcmask != 0) - RTE_LOG(INFO, PMD, " workers core mask = %"PRIx64"\n", - init_params.wcmask); return cryptodev_scheduler_create(name, vdev, @@ -505,6 +486,8 @@ static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = { .remove = cryptodev_scheduler_remove }; +static struct cryptodev_driver scheduler_crypto_drv; + RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD, cryptodev_scheduler_pmd_drv); RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, @@ -512,5 +495,6 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD, "max_nb_sessions=<int> " "socket_id=<int> " "slave=<name>"); -RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_scheduler_pmd_drv, +RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv, + cryptodev_scheduler_pmd_drv, cryptodev_driver_id); diff --git a/drivers/crypto/scheduler/scheduler_pmd_ops.c b/drivers/crypto/scheduler/scheduler_pmd_ops.c index d3795346..d9b52352 100644 --- a/drivers/crypto/scheduler/scheduler_pmd_ops.c +++ b/drivers/crypto/scheduler/scheduler_pmd_ops.c @@ -31,13 +31,11 @@ */ #include <string.h> -#include <rte_config.h> #include <rte_common.h> #include <rte_malloc.h> #include <rte_dev.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> -#include <rte_cryptodev_vdev.h> #include <rte_reorder.h> #include "scheduler_pmd_private.h" @@ -347,7 +345,7 @@ scheduler_pmd_info_get(struct rte_cryptodev *dev, { struct scheduler_ctx *sched_ctx = dev->data->dev_private; uint32_t max_nb_sessions = sched_ctx->nb_slaves ? - UINT32_MAX : RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS; + UINT32_MAX : RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS; uint32_t i; if (!dev_info) diff --git a/drivers/crypto/snow3g/Makefile b/drivers/crypto/snow3g/Makefile index ecee80df..183c6ce9 100644 --- a/drivers/crypto/snow3g/Makefile +++ b/drivers/crypto/snow3g/Makefile @@ -54,6 +54,9 @@ CFLAGS += -I$(LIBSSO_SNOW3G_PATH) CFLAGS += -I$(LIBSSO_SNOW3G_PATH)/include CFLAGS += -I$(LIBSSO_SNOW3G_PATH)/build LDLIBS += -L$(LIBSSO_SNOW3G_PATH)/build -lsso_snow3g +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_SNOW3G) += rte_snow3g_pmd.c diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd.c b/drivers/crypto/snow3g/rte_snow3g_pmd.c index dad45068..4cc9a94f 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd.c +++ b/drivers/crypto/snow3g/rte_snow3g_pmd.c @@ -31,19 +31,16 @@ */ #include <rte_common.h> -#include <rte_config.h> #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> -#include <rte_cryptodev_vdev.h> -#include <rte_vdev.h> +#include <rte_bus_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> #include "rte_snow3g_pmd_private.h" #define SNOW3G_IV_LENGTH 16 -#define SNOW3G_DIGEST_LENGTH 4 #define SNOW3G_MAX_BURST 8 #define BYTE_LEN 8 @@ -263,7 +260,7 @@ process_snow3g_cipher_op_bit(struct rte_crypto_op *op, /** Generate/verify hash from mbufs with same hash key. */ static int -process_snow3g_hash_op(struct rte_crypto_op **ops, +process_snow3g_hash_op(struct snow3g_qp *qp, struct rte_crypto_op **ops, struct snow3g_session *session, uint8_t num_ops) { @@ -289,8 +286,7 @@ process_snow3g_hash_op(struct rte_crypto_op **ops, session->auth_iv_offset); if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { - dst = (uint8_t *)rte_pktmbuf_append(ops[i]->sym->m_src, - SNOW3G_DIGEST_LENGTH); + dst = qp->temp_digest; sso_snow3g_f9_1_buffer(&session->pKeySched_hash, iv, src, @@ -299,10 +295,6 @@ process_snow3g_hash_op(struct rte_crypto_op **ops, if (memcmp(dst, ops[i]->sym->auth.digest.data, SNOW3G_DIGEST_LENGTH) != 0) ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - - /* Trim area used for digest from mbuf. */ - rte_pktmbuf_trim(ops[i]->sym->m_src, - SNOW3G_DIGEST_LENGTH); } else { dst = ops[i]->sym->auth.digest.data; @@ -346,16 +338,16 @@ process_ops(struct rte_crypto_op **ops, struct snow3g_session *session, session, num_ops); break; case SNOW3G_OP_ONLY_AUTH: - processed_ops = process_snow3g_hash_op(ops, session, + processed_ops = process_snow3g_hash_op(qp, ops, session, num_ops); break; case SNOW3G_OP_CIPHER_AUTH: processed_ops = process_snow3g_cipher_op(ops, session, num_ops); - process_snow3g_hash_op(ops, session, processed_ops); + process_snow3g_hash_op(qp, ops, session, processed_ops); break; case SNOW3G_OP_AUTH_CIPHER: - processed_ops = process_snow3g_hash_op(ops, session, + processed_ops = process_snow3g_hash_op(qp, ops, session, num_ops); process_snow3g_cipher_op(ops, session, processed_ops); break; @@ -403,15 +395,15 @@ process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session, session); break; case SNOW3G_OP_ONLY_AUTH: - processed_op = process_snow3g_hash_op(&op, session, 1); + processed_op = process_snow3g_hash_op(qp, &op, session, 1); break; case SNOW3G_OP_CIPHER_AUTH: processed_op = process_snow3g_cipher_op_bit(op, session); if (processed_op == 1) - process_snow3g_hash_op(&op, session, 1); + process_snow3g_hash_op(qp, &op, session, 1); break; case SNOW3G_OP_AUTH_CIPHER: - processed_op = process_snow3g_hash_op(&op, session, 1); + processed_op = process_snow3g_hash_op(qp, &op, session, 1); if (processed_op == 1) process_snow3g_cipher_op_bit(op, session); break; @@ -565,19 +557,13 @@ static int cryptodev_snow3g_remove(struct rte_vdev_device *vdev); static int cryptodev_snow3g_create(const char *name, struct rte_vdev_device *vdev, - struct rte_crypto_vdev_init_params *init_params) + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; struct snow3g_private *internals; uint64_t cpu_flags = RTE_CRYPTODEV_FF_CPU_SSE; - if (init_params->name[0] == '\0') - snprintf(init_params->name, sizeof(init_params->name), - "%s", name); - - dev = rte_cryptodev_vdev_pmd_init(init_params->name, - sizeof(struct snow3g_private), init_params->socket_id, - vdev); + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); if (dev == NULL) { SNOW3G_LOG_ERR("failed to create cryptodev vdev"); goto init_error; @@ -611,11 +597,12 @@ init_error: static int cryptodev_snow3g_probe(struct rte_vdev_device *vdev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct snow3g_private), rte_socket_id(), - {0} + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; const char *name; const char *input_args; @@ -625,17 +612,7 @@ cryptodev_snow3g_probe(struct rte_vdev_device *vdev) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_vdev_parse_init_params(&init_params, input_args); - - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, - init_params.socket_id); - if (init_params.name[0] != '\0') - RTE_LOG(INFO, PMD, " User defined name = %s\n", - init_params.name); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); return cryptodev_snow3g_create(name, vdev, &init_params); } @@ -643,17 +620,18 @@ cryptodev_snow3g_probe(struct rte_vdev_device *vdev) static int cryptodev_snow3g_remove(struct rte_vdev_device *vdev) { + struct rte_cryptodev *cryptodev; const char *name; name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL; - RTE_LOG(INFO, PMD, "Closing SNOW 3G crypto device %s" - " on numa socket %u\n", - name, rte_socket_id()); + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; - return 0; + return rte_cryptodev_pmd_destroy(cryptodev); } static struct rte_vdev_driver cryptodev_snow3g_pmd_drv = { @@ -661,10 +639,13 @@ static struct rte_vdev_driver cryptodev_snow3g_pmd_drv = { .remove = cryptodev_snow3g_remove }; +static struct cryptodev_driver snow3g_crypto_drv; + RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SNOW3G_PMD, cryptodev_snow3g_pmd_drv); RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_SNOW3G_PMD, cryptodev_snow3g_pmd); RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SNOW3G_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); -RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_snow3g_pmd_drv, cryptodev_driver_id); +RTE_PMD_REGISTER_CRYPTO_DRIVER(snow3g_crypto_drv, cryptodev_snow3g_pmd_drv, + cryptodev_driver_id); diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c b/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c index ae9569c8..157a2de3 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c +++ b/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c @@ -184,7 +184,7 @@ snow3g_pmd_qp_set_unique_name(struct rte_cryptodev *dev, "snow3g_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd_private.h b/drivers/crypto/snow3g/rte_snow3g_pmd_private.h index fba3cb86..7b9729f0 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd_private.h +++ b/drivers/crypto/snow3g/rte_snow3g_pmd_private.h @@ -58,6 +58,8 @@ #define SNOW3G_LOG_DBG(fmt, args...) #endif +#define SNOW3G_DIGEST_LENGTH 4 + /** private data structure for each virtual SNOW 3G device */ struct snow3g_private { unsigned max_nb_queue_pairs; @@ -78,6 +80,11 @@ struct snow3g_qp { /**< Session Mempool */ struct rte_cryptodev_stats qp_stats; /**< Queue pair statistics */ + uint8_t temp_digest[SNOW3G_DIGEST_LENGTH]; + /**< Buffer used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ } __rte_cache_aligned; enum snow3g_operation { diff --git a/drivers/crypto/zuc/Makefile b/drivers/crypto/zuc/Makefile index f543b407..af77bc8a 100644 --- a/drivers/crypto/zuc/Makefile +++ b/drivers/crypto/zuc/Makefile @@ -54,6 +54,9 @@ CFLAGS += -I$(LIBSSO_ZUC_PATH) CFLAGS += -I$(LIBSSO_ZUC_PATH)/include CFLAGS += -I$(LIBSSO_ZUC_PATH)/build LDLIBS += -L$(LIBSSO_ZUC_PATH)/build -lsso_zuc +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_vdev # library source files SRCS-$(CONFIG_RTE_LIBRTE_PMD_ZUC) += rte_zuc_pmd.c diff --git a/drivers/crypto/zuc/rte_zuc_pmd.c b/drivers/crypto/zuc/rte_zuc_pmd.c index b301711e..590224bf 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd.c +++ b/drivers/crypto/zuc/rte_zuc_pmd.c @@ -31,18 +31,15 @@ */ #include <rte_common.h> -#include <rte_config.h> #include <rte_hexdump.h> #include <rte_cryptodev.h> #include <rte_cryptodev_pmd.h> -#include <rte_cryptodev_vdev.h> -#include <rte_vdev.h> +#include <rte_bus_vdev.h> #include <rte_malloc.h> #include <rte_cpuflags.h> #include "rte_zuc_pmd_private.h" -#define ZUC_DIGEST_LENGTH 4 #define ZUC_MAX_BURST 8 #define BYTE_LEN 8 @@ -258,7 +255,7 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, /** Generate/verify hash from mbufs with same hash key. */ static int -process_zuc_hash_op(struct rte_crypto_op **ops, +process_zuc_hash_op(struct zuc_qp *qp, struct rte_crypto_op **ops, struct zuc_session *session, uint8_t num_ops) { @@ -285,8 +282,7 @@ process_zuc_hash_op(struct rte_crypto_op **ops, session->auth_iv_offset); if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { - dst = (uint32_t *)rte_pktmbuf_append(ops[i]->sym->m_src, - ZUC_DIGEST_LENGTH); + dst = (uint32_t *)qp->temp_digest; sso_zuc_eia3_1_buffer(session->pKey_hash, iv, src, @@ -295,10 +291,6 @@ process_zuc_hash_op(struct rte_crypto_op **ops, if (memcmp(dst, ops[i]->sym->auth.digest.data, ZUC_DIGEST_LENGTH) != 0) ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - - /* Trim area used for digest from mbuf. */ - rte_pktmbuf_trim(ops[i]->sym->m_src, - ZUC_DIGEST_LENGTH); } else { dst = (uint32_t *)ops[i]->sym->auth.digest.data; @@ -327,16 +319,16 @@ process_ops(struct rte_crypto_op **ops, struct zuc_session *session, session, num_ops); break; case ZUC_OP_ONLY_AUTH: - processed_ops = process_zuc_hash_op(ops, session, + processed_ops = process_zuc_hash_op(qp, ops, session, num_ops); break; case ZUC_OP_CIPHER_AUTH: processed_ops = process_zuc_cipher_op(ops, session, num_ops); - process_zuc_hash_op(ops, session, processed_ops); + process_zuc_hash_op(qp, ops, session, processed_ops); break; case ZUC_OP_AUTH_CIPHER: - processed_ops = process_zuc_hash_op(ops, session, + processed_ops = process_zuc_hash_op(qp, ops, session, num_ops); process_zuc_cipher_op(ops, session, processed_ops); break; @@ -469,19 +461,14 @@ static int cryptodev_zuc_remove(struct rte_vdev_device *vdev); static int cryptodev_zuc_create(const char *name, struct rte_vdev_device *vdev, - struct rte_crypto_vdev_init_params *init_params) + struct rte_cryptodev_pmd_init_params *init_params) { struct rte_cryptodev *dev; struct zuc_private *internals; uint64_t cpu_flags = RTE_CRYPTODEV_FF_CPU_SSE; - if (init_params->name[0] == '\0') - snprintf(init_params->name, sizeof(init_params->name), - "%s", name); - dev = rte_cryptodev_vdev_pmd_init(init_params->name, - sizeof(struct zuc_private), init_params->socket_id, - vdev); + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); if (dev == NULL) { ZUC_LOG_ERR("failed to create cryptodev vdev"); goto init_error; @@ -515,11 +502,12 @@ init_error: static int cryptodev_zuc_probe(struct rte_vdev_device *vdev) { - struct rte_crypto_vdev_init_params init_params = { - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, - RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct zuc_private), rte_socket_id(), - {0} + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS }; const char *name; const char *input_args; @@ -529,17 +517,7 @@ cryptodev_zuc_probe(struct rte_vdev_device *vdev) return -EINVAL; input_args = rte_vdev_device_args(vdev); - rte_cryptodev_vdev_parse_init_params(&init_params, input_args); - - RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, - init_params.socket_id); - if (init_params.name[0] != '\0') - RTE_LOG(INFO, PMD, " User defined name = %s\n", - init_params.name); - RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", - init_params.max_nb_queue_pairs); - RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", - init_params.max_nb_sessions); + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); return cryptodev_zuc_create(name, vdev, &init_params); } @@ -547,17 +525,19 @@ cryptodev_zuc_probe(struct rte_vdev_device *vdev) static int cryptodev_zuc_remove(struct rte_vdev_device *vdev) { + + struct rte_cryptodev *cryptodev; const char *name; name = rte_vdev_device_name(vdev); if (name == NULL) return -EINVAL; - RTE_LOG(INFO, PMD, "Closing ZUC crypto device %s" - " on numa socket %u\n", - name, rte_socket_id()); + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; - return 0; + return rte_cryptodev_pmd_destroy(cryptodev); } static struct rte_vdev_driver cryptodev_zuc_pmd_drv = { @@ -565,9 +545,12 @@ static struct rte_vdev_driver cryptodev_zuc_pmd_drv = { .remove = cryptodev_zuc_remove }; +static struct cryptodev_driver zuc_crypto_drv; + RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ZUC_PMD, cryptodev_zuc_pmd_drv); RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ZUC_PMD, "max_nb_queue_pairs=<int> " "max_nb_sessions=<int> " "socket_id=<int>"); -RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_zuc_pmd_drv, cryptodev_driver_id); +RTE_PMD_REGISTER_CRYPTO_DRIVER(zuc_crypto_drv, cryptodev_zuc_pmd_drv, + cryptodev_driver_id); diff --git a/drivers/crypto/zuc/rte_zuc_pmd_ops.c b/drivers/crypto/zuc/rte_zuc_pmd_ops.c index 52c6aed8..243c0991 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd_ops.c +++ b/drivers/crypto/zuc/rte_zuc_pmd_ops.c @@ -184,7 +184,7 @@ zuc_pmd_qp_set_unique_name(struct rte_cryptodev *dev, "zuc_pmd_%u_qp_%u", dev->data->dev_id, qp->id); - if (n > sizeof(qp->name)) + if (n >= sizeof(qp->name)) return -1; return 0; diff --git a/drivers/crypto/zuc/rte_zuc_pmd_private.h b/drivers/crypto/zuc/rte_zuc_pmd_private.h index b706e0aa..a57b8cd0 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd_private.h +++ b/drivers/crypto/zuc/rte_zuc_pmd_private.h @@ -59,6 +59,8 @@ #endif #define ZUC_IV_KEY_LENGTH 16 +#define ZUC_DIGEST_LENGTH 4 + /** private data structure for each virtual ZUC device */ struct zuc_private { unsigned max_nb_queue_pairs; @@ -79,6 +81,11 @@ struct zuc_qp { /**< Session Mempool */ struct rte_cryptodev_stats qp_stats; /**< Queue pair statistics */ + uint8_t temp_digest[ZUC_DIGEST_LENGTH]; + /**< Buffer used to store the digest generated + * by the driver when verifying a digest provided + * by the user (using authentication verify operation) + */ } __rte_cache_aligned; enum zuc_operation { |