From 8d01b9cd70a67cdafd5b965a70420c3bd7fb3f82 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Thu, 1 Nov 2018 11:59:50 +0000 Subject: New upstream version 18.11-rc1 Change-Id: Iaa71986dd6332e878d8f4bf493101b2bbc6313bb Signed-off-by: Luca Boccassi --- drivers/crypto/caam_jr/Makefile | 44 + drivers/crypto/caam_jr/caam_jr.c | 2508 ++++++++++++++++++++ drivers/crypto/caam_jr/caam_jr_capabilities.c | 266 +++ drivers/crypto/caam_jr/caam_jr_capabilities.h | 18 + drivers/crypto/caam_jr/caam_jr_config.h | 207 ++ drivers/crypto/caam_jr/caam_jr_desc.h | 285 +++ drivers/crypto/caam_jr/caam_jr_hw.c | 367 +++ drivers/crypto/caam_jr/caam_jr_hw_specific.h | 503 ++++ drivers/crypto/caam_jr/caam_jr_log.h | 42 + drivers/crypto/caam_jr/caam_jr_pvt.h | 291 +++ drivers/crypto/caam_jr/caam_jr_uio.c | 501 ++++ drivers/crypto/caam_jr/meson.build | 17 + drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map | 4 + 13 files changed, 5053 insertions(+) create mode 100644 drivers/crypto/caam_jr/Makefile create mode 100644 drivers/crypto/caam_jr/caam_jr.c create mode 100644 drivers/crypto/caam_jr/caam_jr_capabilities.c create mode 100644 drivers/crypto/caam_jr/caam_jr_capabilities.h create mode 100644 drivers/crypto/caam_jr/caam_jr_config.h create mode 100644 drivers/crypto/caam_jr/caam_jr_desc.h create mode 100644 drivers/crypto/caam_jr/caam_jr_hw.c create mode 100644 drivers/crypto/caam_jr/caam_jr_hw_specific.h create mode 100644 drivers/crypto/caam_jr/caam_jr_log.h create mode 100644 drivers/crypto/caam_jr/caam_jr_pvt.h create mode 100644 drivers/crypto/caam_jr/caam_jr_uio.c create mode 100644 drivers/crypto/caam_jr/meson.build create mode 100644 drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map (limited to 'drivers/crypto/caam_jr') diff --git a/drivers/crypto/caam_jr/Makefile b/drivers/crypto/caam_jr/Makefile new file mode 100644 index 00000000..88cdf741 --- /dev/null +++ b/drivers/crypto/caam_jr/Makefile @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2017 NXP + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_caam_jr.a + +# build flags +CFLAGS += -DALLOW_EXPERIMENTAL_API + +CFLAGS += -D _GNU_SOURCE + +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa/include +CFLAGS += -I$(RTE_SDK)/drivers/crypto/caam_jr +#sharing the hw flib headers from dpaa2_sec pmd +CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec/ +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include +CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal + +# versioning export map +EXPORT_MAP := rte_pmd_caam_jr_version.map + +# library version +LIBABIVER := 1 + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr_capabilities.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr_hw.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr_uio.c +# library dependencies + +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_cryptodev +LDLIBS += -lrte_bus_dpaa +LDLIBS += -lrte_bus_vdev + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/crypto/caam_jr/caam_jr.c b/drivers/crypto/caam_jr/caam_jr.c new file mode 100644 index 00000000..f505adf6 --- /dev/null +++ b/drivers/crypto/caam_jr/caam_jr.c @@ -0,0 +1,2508 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017-2018 NXP + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/* RTA header files */ +#include +#include +#include + +#define CAAM_JR_DBG 0 +#define CRYPTODEV_NAME_CAAM_JR_PMD crypto_caam_jr +static uint8_t cryptodev_driver_id; +int caam_jr_logtype; + +enum rta_sec_era rta_sec_era; + +/* Lists the states possible for the SEC user space driver. */ +enum sec_driver_state_e { + SEC_DRIVER_STATE_IDLE, /* Driver not initialized */ + SEC_DRIVER_STATE_STARTED, /* Driver initialized and can be used*/ + SEC_DRIVER_STATE_RELEASE, /* Driver release is in progress */ +}; + +/* Job rings used for communication with SEC HW */ +static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS]; + +/* The current state of SEC user space driver */ +static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE; + +/* The number of job rings used by SEC user space driver */ +static int g_job_rings_no; +static int g_job_rings_max; + +struct sec_outring_entry { + phys_addr_t desc; /* Pointer to completed descriptor */ + uint32_t status; /* Status for completed descriptor */ +} __rte_packed; + +/* virtual address conversin when mempool support is available for ctx */ +static inline phys_addr_t +caam_jr_vtop_ctx(struct caam_jr_op_ctx *ctx, void *vaddr) +{ + PMD_INIT_FUNC_TRACE(); + return (size_t)vaddr - ctx->vtop_offset; +} + +static inline void +caam_jr_op_ending(struct caam_jr_op_ctx *ctx) +{ + PMD_INIT_FUNC_TRACE(); + /* report op status to sym->op and then free the ctx memeory */ + rte_mempool_put(ctx->ctx_pool, (void *)ctx); +} + +static inline struct caam_jr_op_ctx * +caam_jr_alloc_ctx(struct caam_jr_session *ses) +{ + struct caam_jr_op_ctx *ctx; + int ret; + + PMD_INIT_FUNC_TRACE(); + ret = rte_mempool_get(ses->ctx_pool, (void **)(&ctx)); + if (!ctx || ret) { + CAAM_JR_DP_WARN("Alloc sec descriptor failed!"); + return NULL; + } + /* + * Clear SG memory. There are 16 SG entries of 16 Bytes each. + * one call to dcbz_64() clear 64 bytes, hence calling it 4 times + * to clear all the SG entries. caam_jr_alloc_ctx() is called for + * each packet, memset is costlier than dcbz_64(). + */ + dcbz_64(&ctx->sg[SG_CACHELINE_0]); + dcbz_64(&ctx->sg[SG_CACHELINE_1]); + dcbz_64(&ctx->sg[SG_CACHELINE_2]); + dcbz_64(&ctx->sg[SG_CACHELINE_3]); + + ctx->ctx_pool = ses->ctx_pool; + ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx); + + return ctx; +} + +static +void caam_jr_stats_get(struct rte_cryptodev *dev, + struct rte_cryptodev_stats *stats) +{ + struct caam_jr_qp **qp = (struct caam_jr_qp **) + dev->data->queue_pairs; + int i; + + PMD_INIT_FUNC_TRACE(); + if (stats == NULL) { + CAAM_JR_ERR("Invalid stats ptr NULL"); + return; + } + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + if (qp[i] == NULL) { + CAAM_JR_WARN("Uninitialised queue pair"); + continue; + } + + stats->enqueued_count += qp[i]->tx_pkts; + stats->dequeued_count += qp[i]->rx_pkts; + stats->enqueue_err_count += qp[i]->tx_errs; + stats->dequeue_err_count += qp[i]->rx_errs; + CAAM_JR_INFO("extra stats:\n\tRX Poll ERR = %" PRIu64 + "\n\tTX Ring Full = %" PRIu64, + qp[i]->rx_poll_err, + qp[i]->tx_ring_full); + } +} + +static +void caam_jr_stats_reset(struct rte_cryptodev *dev) +{ + int i; + struct caam_jr_qp **qp = (struct caam_jr_qp **) + (dev->data->queue_pairs); + + PMD_INIT_FUNC_TRACE(); + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + if (qp[i] == NULL) { + CAAM_JR_WARN("Uninitialised queue pair"); + continue; + } + qp[i]->rx_pkts = 0; + qp[i]->rx_errs = 0; + qp[i]->rx_poll_err = 0; + qp[i]->tx_pkts = 0; + qp[i]->tx_errs = 0; + qp[i]->tx_ring_full = 0; + } +} + +static inline int +is_cipher_only(struct caam_jr_session *ses) +{ + PMD_INIT_FUNC_TRACE(); + return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && + (ses->auth_alg == RTE_CRYPTO_AUTH_NULL)); +} + +static inline int +is_auth_only(struct caam_jr_session *ses) +{ + PMD_INIT_FUNC_TRACE(); + return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) && + (ses->auth_alg != RTE_CRYPTO_AUTH_NULL)); +} + +static inline int +is_aead(struct caam_jr_session *ses) +{ + PMD_INIT_FUNC_TRACE(); + return ((ses->cipher_alg == 0) && + (ses->auth_alg == 0) && + (ses->aead_alg != 0)); +} + +static inline int +is_auth_cipher(struct caam_jr_session *ses) +{ + PMD_INIT_FUNC_TRACE(); + return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && + (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) && + (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC)); +} + +static inline int +is_proto_ipsec(struct caam_jr_session *ses) +{ + PMD_INIT_FUNC_TRACE(); + return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC); +} + +static inline int +is_encode(struct caam_jr_session *ses) +{ + PMD_INIT_FUNC_TRACE(); + return ses->dir == DIR_ENC; +} + +static inline int +is_decode(struct caam_jr_session *ses) +{ + PMD_INIT_FUNC_TRACE(); + return ses->dir == DIR_DEC; +} + +static inline void +caam_auth_alg(struct caam_jr_session *ses, struct alginfo *alginfo_a) +{ + PMD_INIT_FUNC_TRACE(); + switch (ses->auth_alg) { + case RTE_CRYPTO_AUTH_NULL: + ses->digest_length = 0; + break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + alginfo_a->algtype = + (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? + OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA1_HMAC: + alginfo_a->algtype = + (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? + OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + alginfo_a->algtype = + (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? + OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + alginfo_a->algtype = + (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? + OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + alginfo_a->algtype = + (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? + OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA512_HMAC: + alginfo_a->algtype = + (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? + OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512; + alginfo_a->algmode = OP_ALG_AAI_HMAC; + break; + default: + CAAM_JR_DEBUG("unsupported auth alg %u", ses->auth_alg); + } +} + +static inline void +caam_cipher_alg(struct caam_jr_session *ses, struct alginfo *alginfo_c) +{ + PMD_INIT_FUNC_TRACE(); + switch (ses->cipher_alg) { + case RTE_CRYPTO_CIPHER_NULL: + break; + case RTE_CRYPTO_CIPHER_AES_CBC: + alginfo_c->algtype = + (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? + OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES; + alginfo_c->algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_3DES_CBC: + alginfo_c->algtype = + (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? + OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES; + alginfo_c->algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + alginfo_c->algtype = + (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? + OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES; + alginfo_c->algmode = OP_ALG_AAI_CTR; + break; + default: + CAAM_JR_DEBUG("unsupported cipher alg %d", ses->cipher_alg); + } +} + +static inline void +caam_aead_alg(struct caam_jr_session *ses, struct alginfo *alginfo) +{ + PMD_INIT_FUNC_TRACE(); + switch (ses->aead_alg) { + case RTE_CRYPTO_AEAD_AES_GCM: + alginfo->algtype = OP_ALG_ALGSEL_AES; + alginfo->algmode = OP_ALG_AAI_GCM; + break; + default: + CAAM_JR_DEBUG("unsupported AEAD alg %d", ses->aead_alg); + } +} + +/* prepare command block of the session */ +static int +caam_jr_prep_cdb(struct caam_jr_session *ses) +{ + struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0}; + int32_t shared_desc_len = 0; + struct sec_cdb *cdb; + int err; +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + int swap = false; +#else + int swap = true; +#endif + + PMD_INIT_FUNC_TRACE(); + if (ses->cdb) + caam_jr_dma_free(ses->cdb); + + cdb = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, sizeof(struct sec_cdb)); + if (!cdb) { + CAAM_JR_ERR("failed to allocate memory for cdb\n"); + return -1; + } + + ses->cdb = cdb; + + memset(cdb, 0, sizeof(struct sec_cdb)); + + if (is_cipher_only(ses)) { + caam_cipher_alg(ses, &alginfo_c); + if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) { + CAAM_JR_ERR("not supported cipher alg"); + rte_free(cdb); + return -ENOTSUP; + } + + alginfo_c.key = (size_t)ses->cipher_key.data; + alginfo_c.keylen = ses->cipher_key.length; + alginfo_c.key_enc_flags = 0; + alginfo_c.key_type = RTA_DATA_IMM; + + shared_desc_len = cnstr_shdsc_blkcipher( + cdb->sh_desc, true, + swap, &alginfo_c, + NULL, + ses->iv.length, + ses->dir); + } else if (is_auth_only(ses)) { + caam_auth_alg(ses, &alginfo_a); + if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) { + CAAM_JR_ERR("not supported auth alg"); + rte_free(cdb); + return -ENOTSUP; + } + + alginfo_a.key = (size_t)ses->auth_key.data; + alginfo_a.keylen = ses->auth_key.length; + alginfo_a.key_enc_flags = 0; + alginfo_a.key_type = RTA_DATA_IMM; + + shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true, + swap, &alginfo_a, + !ses->dir, + ses->digest_length); + } else if (is_aead(ses)) { + caam_aead_alg(ses, &alginfo); + if (alginfo.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) { + CAAM_JR_ERR("not supported aead alg"); + rte_free(cdb); + return -ENOTSUP; + } + alginfo.key = (size_t)ses->aead_key.data; + alginfo.keylen = ses->aead_key.length; + alginfo.key_enc_flags = 0; + alginfo.key_type = RTA_DATA_IMM; + + if (ses->dir == DIR_ENC) + shared_desc_len = cnstr_shdsc_gcm_encap( + cdb->sh_desc, true, swap, + &alginfo, + ses->iv.length, + ses->digest_length); + else + shared_desc_len = cnstr_shdsc_gcm_decap( + cdb->sh_desc, true, swap, + &alginfo, + ses->iv.length, + ses->digest_length); + } else { + caam_cipher_alg(ses, &alginfo_c); + if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) { + CAAM_JR_ERR("not supported cipher alg"); + rte_free(cdb); + return -ENOTSUP; + } + + alginfo_c.key = (size_t)ses->cipher_key.data; + alginfo_c.keylen = ses->cipher_key.length; + alginfo_c.key_enc_flags = 0; + alginfo_c.key_type = RTA_DATA_IMM; + + caam_auth_alg(ses, &alginfo_a); + if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) { + CAAM_JR_ERR("not supported auth alg"); + rte_free(cdb); + return -ENOTSUP; + } + + alginfo_a.key = (size_t)ses->auth_key.data; + alginfo_a.keylen = ses->auth_key.length; + alginfo_a.key_enc_flags = 0; + alginfo_a.key_type = RTA_DATA_IMM; + + cdb->sh_desc[0] = alginfo_c.keylen; + cdb->sh_desc[1] = alginfo_a.keylen; + err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, + MIN_JOB_DESC_SIZE, + (unsigned int *)cdb->sh_desc, + &cdb->sh_desc[2], 2); + + if (err < 0) { + CAAM_JR_ERR("Crypto: Incorrect key lengths"); + rte_free(cdb); + return err; + } + if (cdb->sh_desc[2] & 1) + alginfo_c.key_type = RTA_DATA_IMM; + else { + alginfo_c.key = (size_t)caam_jr_mem_vtop( + (void *)(size_t)alginfo_c.key); + alginfo_c.key_type = RTA_DATA_PTR; + } + if (cdb->sh_desc[2] & (1<<1)) + alginfo_a.key_type = RTA_DATA_IMM; + else { + alginfo_a.key = (size_t)caam_jr_mem_vtop( + (void *)(size_t)alginfo_a.key); + alginfo_a.key_type = RTA_DATA_PTR; + } + cdb->sh_desc[0] = 0; + cdb->sh_desc[1] = 0; + cdb->sh_desc[2] = 0; + if (is_proto_ipsec(ses)) { + if (ses->dir == DIR_ENC) { + shared_desc_len = cnstr_shdsc_ipsec_new_encap( + cdb->sh_desc, + true, swap, SHR_SERIAL, + &ses->encap_pdb, + (uint8_t *)&ses->ip4_hdr, + &alginfo_c, &alginfo_a); + } else if (ses->dir == DIR_DEC) { + shared_desc_len = cnstr_shdsc_ipsec_new_decap( + cdb->sh_desc, + true, swap, SHR_SERIAL, + &ses->decap_pdb, + &alginfo_c, &alginfo_a); + } + } else { + /* Auth_only_len is set as 0 here and it will be + * overwritten in fd for each packet. + */ + shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc, + true, swap, &alginfo_c, &alginfo_a, + ses->iv.length, 0, + ses->digest_length, ses->dir); + } + } + + if (shared_desc_len < 0) { + CAAM_JR_ERR("error in preparing command block"); + return shared_desc_len; + } + +#if CAAM_JR_DBG + SEC_DUMP_DESC(cdb->sh_desc); +#endif + + cdb->sh_hdr.hi.field.idlen = shared_desc_len; + + return 0; +} + +/* @brief Poll the HW for already processed jobs in the JR + * and silently discard the available jobs or notify them to UA + * with indicated error code. + * + * @param [in,out] job_ring The job ring to poll. + * @param [in] do_notify Can be #TRUE or #FALSE. Indicates if + * descriptors are to be discarded + * or notified to UA with given error_code. + * @param [out] notified_descs Number of notified descriptors. Can be NULL + * if do_notify is #FALSE + */ +static void +hw_flush_job_ring(struct sec_job_ring_t *job_ring, + uint32_t do_notify, + uint32_t *notified_descs) +{ + int32_t jobs_no_to_discard = 0; + int32_t discarded_descs_no = 0; + + PMD_INIT_FUNC_TRACE(); + CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]", + job_ring, job_ring->pidx, job_ring->cidx, do_notify); + + jobs_no_to_discard = hw_get_no_finished_jobs(job_ring); + + /* Discard all jobs */ + CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs", + job_ring, job_ring->pidx, job_ring->cidx, + jobs_no_to_discard); + + while (jobs_no_to_discard > discarded_descs_no) { + discarded_descs_no++; + /* Now increment the consumer index for the current job ring, + * AFTER saving job in temporary location! + * Increment the consumer index for the current job ring + */ + job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx, + SEC_JOB_RING_SIZE); + + hw_remove_entries(job_ring, 1); + } + + if (do_notify == true) { + ASSERT(notified_descs != NULL); + *notified_descs = discarded_descs_no; + } +} + +/* @brief Poll the HW for already processed jobs in the JR + * and notify the available jobs to UA. + * + * @param [in] job_ring The job ring to poll. + * @param [in] limit The maximum number of jobs to notify. + * If set to negative value, all available jobs are + * notified. + * + * @retval >=0 for No of jobs notified to UA. + * @retval -1 for error + */ +static int +hw_poll_job_ring(struct sec_job_ring_t *job_ring, + struct rte_crypto_op **ops, int32_t limit, + struct caam_jr_qp *jr_qp) +{ + int32_t jobs_no_to_notify = 0; /* the number of done jobs to notify*/ + int32_t number_of_jobs_available = 0; + int32_t notified_descs_no = 0; + uint32_t sec_error_code = 0; + struct job_descriptor *current_desc; + phys_addr_t current_desc_addr; + phys_addr_t *temp_addr; + struct caam_jr_op_ctx *ctx; + + PMD_INIT_FUNC_TRACE(); + /* TODO check for ops have memory*/ + /* check here if any JR error that cannot be written + * in the output status word has occurred + */ + if (JR_REG_JRINT_JRE_EXTRACT(GET_JR_REG(JRINT, job_ring))) { + CAAM_JR_INFO("err received"); + sec_error_code = JR_REG_JRINT_ERR_TYPE_EXTRACT( + GET_JR_REG(JRINT, job_ring)); + if (unlikely(sec_error_code)) { + hw_job_ring_error_print(job_ring, sec_error_code); + return -1; + } + } + /* compute the number of jobs available in the job ring based on the + * producer and consumer index values. + */ + number_of_jobs_available = hw_get_no_finished_jobs(job_ring); + /* Compute the number of notifications that need to be raised to UA + * If limit > total number of done jobs -> notify all done jobs + * If limit = 0 -> error + * If limit < total number of done jobs -> notify a number + * of done jobs equal with limit + */ + jobs_no_to_notify = (limit > number_of_jobs_available) ? + number_of_jobs_available : limit; + CAAM_JR_DP_DEBUG( + "Jr[%p] pi[%d] ci[%d].limit =%d Available=%d.Jobs to notify=%d", + job_ring, job_ring->pidx, job_ring->cidx, + limit, number_of_jobs_available, jobs_no_to_notify); + + rte_smp_rmb(); + + while (jobs_no_to_notify > notified_descs_no) { + static uint64_t false_alarm; + static uint64_t real_poll; + + /* Get job status here */ + sec_error_code = job_ring->output_ring[job_ring->cidx].status; + /* Get completed descriptor */ + temp_addr = &(job_ring->output_ring[job_ring->cidx].desc); + current_desc_addr = (phys_addr_t)sec_read_addr(temp_addr); + + real_poll++; + /* todo check if it is false alarm no desc present */ + if (!current_desc_addr) { + false_alarm++; + printf("false alarm %" PRIu64 "real %" PRIu64 + " sec_err =0x%x cidx Index =0%d\n", + false_alarm, real_poll, + sec_error_code, job_ring->cidx); + rte_panic("CAAM JR descriptor NULL"); + return notified_descs_no; + } + current_desc = (struct job_descriptor *) + caam_jr_dma_ptov(current_desc_addr); + /* now increment the consumer index for the current job ring, + * AFTER saving job in temporary location! + */ + job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx, + SEC_JOB_RING_SIZE); + /* Signal that the job has been processed and the slot is free*/ + hw_remove_entries(job_ring, 1); + /*TODO for multiple ops, packets*/ + ctx = container_of(current_desc, struct caam_jr_op_ctx, jobdes); + if (unlikely(sec_error_code)) { + CAAM_JR_ERR("desc at cidx %d generated error 0x%x\n", + job_ring->cidx, sec_error_code); + hw_handle_job_ring_error(job_ring, sec_error_code); + //todo improve with exact errors + ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR; + jr_qp->rx_errs++; + } else { + ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; +#if CAAM_JR_DBG + if (ctx->op->sym->m_dst) { + rte_hexdump(stdout, "PROCESSED", + rte_pktmbuf_mtod(ctx->op->sym->m_dst, void *), + rte_pktmbuf_data_len(ctx->op->sym->m_dst)); + } else { + rte_hexdump(stdout, "PROCESSED", + rte_pktmbuf_mtod(ctx->op->sym->m_src, void *), + rte_pktmbuf_data_len(ctx->op->sym->m_src)); + } +#endif + } + if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { + struct ip *ip4_hdr; + + if (ctx->op->sym->m_dst) { + /*TODO check for ip header or other*/ + ip4_hdr = (struct ip *) + rte_pktmbuf_mtod(ctx->op->sym->m_dst, char*); + ctx->op->sym->m_dst->pkt_len = + rte_be_to_cpu_16(ip4_hdr->ip_len); + ctx->op->sym->m_dst->data_len = + rte_be_to_cpu_16(ip4_hdr->ip_len); + } else { + ip4_hdr = (struct ip *) + rte_pktmbuf_mtod(ctx->op->sym->m_src, char*); + ctx->op->sym->m_src->pkt_len = + rte_be_to_cpu_16(ip4_hdr->ip_len); + ctx->op->sym->m_src->data_len = + rte_be_to_cpu_16(ip4_hdr->ip_len); + } + } + *ops = ctx->op; + caam_jr_op_ending(ctx); + ops++; + notified_descs_no++; + } + return notified_descs_no; +} + +static uint16_t +caam_jr_dequeue_burst(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp; + struct sec_job_ring_t *ring = jr_qp->ring; + int num_rx; + int ret; + + PMD_INIT_FUNC_TRACE(); + CAAM_JR_DP_DEBUG("Jr[%p]Polling. limit[%d]", ring, nb_ops); + + /* Poll job ring + * If nb_ops < 0 -> poll JR until no more notifications are available. + * If nb_ops > 0 -> poll JR until limit is reached. + */ + + /* Run hw poll job ring */ + num_rx = hw_poll_job_ring(ring, ops, nb_ops, jr_qp); + if (num_rx < 0) { + CAAM_JR_ERR("Error polling SEC engine (%d)", num_rx); + return 0; + } + + CAAM_JR_DP_DEBUG("Jr[%p].Jobs notified[%d]. ", ring, num_rx); + + if (ring->jr_mode == SEC_NOTIFICATION_TYPE_NAPI) { + if (num_rx < nb_ops) { + ret = caam_jr_enable_irqs(ring->irq_fd); + SEC_ASSERT(ret == 0, ret, + "Failed to enable irqs for job ring %p", ring); + } + } else if (ring->jr_mode == SEC_NOTIFICATION_TYPE_IRQ) { + + /* Always enable IRQ generation when in pure IRQ mode */ + ret = caam_jr_enable_irqs(ring->irq_fd); + SEC_ASSERT(ret == 0, ret, + "Failed to enable irqs for job ring %p", ring); + } + + jr_qp->rx_pkts += num_rx; + + return num_rx; +} + +/** + * packet looks like: + * |<----data_len------->| + * |ip_header|ah_header|icv|payload| + * ^ + * | + * mbuf->pkt.data + */ +static inline struct caam_jr_op_ctx * +build_auth_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct rte_mbuf *mbuf = sym->m_src; + struct caam_jr_op_ctx *ctx; + struct sec4_sg_entry *sg; + int length; + struct sec_cdb *cdb; + uint64_t sdesc_offset; + struct sec_job_descriptor_t *jobdescr; + uint8_t extra_segs; + + PMD_INIT_FUNC_TRACE(); + if (is_decode(ses)) + extra_segs = 2; + else + extra_segs = 1; + + if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) { + CAAM_JR_DP_ERR("Auth: Max sec segs supported is %d", + MAX_SG_ENTRIES); + return NULL; + } + + ctx = caam_jr_alloc_ctx(ses); + if (!ctx) + return NULL; + + ctx->op = op; + + cdb = ses->cdb; + sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); + + jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; + + SEC_JD_INIT(jobdescr); + SEC_JD_SET_SD(jobdescr, + (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, + cdb->sh_hdr.hi.field.idlen); + + /* output */ + SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr, + 0, ses->digest_length); + + /*input */ + sg = &ctx->sg[0]; + length = sym->auth.data.length; + sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + sym->auth.data.offset); + sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset); + + /* Successive segs */ + mbuf = mbuf->next; + while (mbuf) { + sg++; + sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)); + sg->len = cpu_to_caam32(mbuf->data_len); + mbuf = mbuf->next; + } + + if (is_decode(ses)) { + /* digest verification case */ + sg++; + /* hash result or digest, save digest first */ + rte_memcpy(ctx->digest, sym->auth.digest.data, + ses->digest_length); +#if CAAM_JR_DBG + rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length); +#endif + sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest)); + sg->len = cpu_to_caam32(ses->digest_length); + length += ses->digest_length; + } else { + length -= ses->digest_length; + } + + /* last element*/ + sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); + + SEC_JD_SET_IN_PTR(jobdescr, + (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0, length); + /* enabling sg list */ + (jobdescr)->seq_in.command.word |= 0x01000000; + + return ctx; +} + +static inline struct caam_jr_op_ctx * +build_auth_only(struct rte_crypto_op *op, struct caam_jr_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct caam_jr_op_ctx *ctx; + struct sec4_sg_entry *sg; + rte_iova_t start_addr; + struct sec_cdb *cdb; + uint64_t sdesc_offset; + struct sec_job_descriptor_t *jobdescr; + + PMD_INIT_FUNC_TRACE(); + ctx = caam_jr_alloc_ctx(ses); + if (!ctx) + return NULL; + + ctx->op = op; + + cdb = ses->cdb; + sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); + + start_addr = rte_pktmbuf_iova(sym->m_src); + + jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; + + SEC_JD_INIT(jobdescr); + SEC_JD_SET_SD(jobdescr, + (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, + cdb->sh_hdr.hi.field.idlen); + + /* output */ + SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr, + 0, ses->digest_length); + + /*input */ + if (is_decode(ses)) { + sg = &ctx->sg[0]; + SEC_JD_SET_IN_PTR(jobdescr, + (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0, + (sym->auth.data.length + ses->digest_length)); + /* enabling sg list */ + (jobdescr)->seq_in.command.word |= 0x01000000; + + /* hash result or digest, save digest first */ + rte_memcpy(ctx->digest, sym->auth.digest.data, + ses->digest_length); + sg->ptr = cpu_to_caam64(start_addr + sym->auth.data.offset); + sg->len = cpu_to_caam32(sym->auth.data.length); + +#if CAAM_JR_DBG + rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length); +#endif + /* let's check digest by hw */ + sg++; + sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest)); + sg->len = cpu_to_caam32(ses->digest_length); + /* last element*/ + sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); + } else { + SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)start_addr, + sym->auth.data.offset, sym->auth.data.length); + } + return ctx; +} + +static inline struct caam_jr_op_ctx * +build_cipher_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct rte_mbuf *mbuf = sym->m_src; + struct caam_jr_op_ctx *ctx; + struct sec4_sg_entry *sg, *in_sg; + int length; + struct sec_cdb *cdb; + uint64_t sdesc_offset; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + ses->iv.offset); + struct sec_job_descriptor_t *jobdescr; + uint8_t reg_segs; + + PMD_INIT_FUNC_TRACE(); + if (sym->m_dst) { + mbuf = sym->m_dst; + reg_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2; + } else { + mbuf = sym->m_src; + reg_segs = mbuf->nb_segs * 2 + 2; + } + + if (reg_segs > MAX_SG_ENTRIES) { + CAAM_JR_DP_ERR("Cipher: Max sec segs supported is %d", + MAX_SG_ENTRIES); + return NULL; + } + + ctx = caam_jr_alloc_ctx(ses); + if (!ctx) + return NULL; + + ctx->op = op; + cdb = ses->cdb; + sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); + + jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; + + SEC_JD_INIT(jobdescr); + SEC_JD_SET_SD(jobdescr, + (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, + cdb->sh_hdr.hi.field.idlen); + +#if CAAM_JR_DBG + CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d", + sym->m_src->data_off, sym->cipher.data.offset, + sym->cipher.data.length, ses->iv.length); +#endif + /* output */ + if (sym->m_dst) + mbuf = sym->m_dst; + else + mbuf = sym->m_src; + + sg = &ctx->sg[0]; + length = sym->cipher.data.length; + + sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + + sym->cipher.data.offset); + sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset); + + /* Successive segs */ + mbuf = mbuf->next; + while (mbuf) { + sg++; + sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)); + sg->len = cpu_to_caam32(mbuf->data_len); + mbuf = mbuf->next; + } + /* last element*/ + sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); + + SEC_JD_SET_OUT_PTR(jobdescr, + (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0, + length); + /*enabling sg bit */ + (jobdescr)->seq_out.command.word |= 0x01000000; + + /*input */ + sg++; + mbuf = sym->m_src; + in_sg = sg; + + length = sym->cipher.data.length + ses->iv.length; + + /* IV */ + sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr)); + sg->len = cpu_to_caam32(ses->iv.length); + + /* 1st seg */ + sg++; + sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + + sym->cipher.data.offset); + sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset); + + /* Successive segs */ + mbuf = mbuf->next; + while (mbuf) { + sg++; + sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)); + sg->len = cpu_to_caam32(mbuf->data_len); + mbuf = mbuf->next; + } + /* last element*/ + sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); + + + SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, in_sg), 0, + length); + /*enabling sg bit */ + (jobdescr)->seq_in.command.word |= 0x01000000; + + return ctx; +} + +static inline struct caam_jr_op_ctx * +build_cipher_only(struct rte_crypto_op *op, struct caam_jr_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct caam_jr_op_ctx *ctx; + struct sec4_sg_entry *sg; + rte_iova_t src_start_addr, dst_start_addr; + struct sec_cdb *cdb; + uint64_t sdesc_offset; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + ses->iv.offset); + struct sec_job_descriptor_t *jobdescr; + + PMD_INIT_FUNC_TRACE(); + ctx = caam_jr_alloc_ctx(ses); + if (!ctx) + return NULL; + + ctx->op = op; + cdb = ses->cdb; + sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); + + src_start_addr = rte_pktmbuf_iova(sym->m_src); + if (sym->m_dst) + dst_start_addr = rte_pktmbuf_iova(sym->m_dst); + else + dst_start_addr = src_start_addr; + + jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; + + SEC_JD_INIT(jobdescr); + SEC_JD_SET_SD(jobdescr, + (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, + cdb->sh_hdr.hi.field.idlen); + +#if CAAM_JR_DBG + CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d", + sym->m_src->data_off, sym->cipher.data.offset, + sym->cipher.data.length, ses->iv.length); +#endif + /* output */ + SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr, + sym->cipher.data.offset, + sym->cipher.data.length + ses->iv.length); + + /*input */ + sg = &ctx->sg[0]; + SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0, + sym->cipher.data.length + ses->iv.length); + /*enabling sg bit */ + (jobdescr)->seq_in.command.word |= 0x01000000; + + sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr)); + sg->len = cpu_to_caam32(ses->iv.length); + + sg = &ctx->sg[1]; + sg->ptr = cpu_to_caam64(src_start_addr + sym->cipher.data.offset); + sg->len = cpu_to_caam32(sym->cipher.data.length); + /* last element*/ + sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); + + return ctx; +} + +/* For decapsulation: + * Input: + * +----+----------------+--------------------------------+-----+ + * | IV | Auth-only data | Authenticated & Encrypted data | ICV | + * +----+----------------+--------------------------------+-----+ + * Output: + * +----+--------------------------+ + * | Decrypted & authenticated data | + * +----+--------------------------+ + */ + +static inline struct caam_jr_op_ctx * +build_cipher_auth_sg(struct rte_crypto_op *op, struct caam_jr_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct caam_jr_op_ctx *ctx; + struct sec4_sg_entry *sg, *out_sg, *in_sg; + struct rte_mbuf *mbuf; + uint32_t length = 0; + struct sec_cdb *cdb; + uint64_t sdesc_offset; + uint8_t req_segs; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + ses->iv.offset); + struct sec_job_descriptor_t *jobdescr; + uint32_t auth_only_len; + + PMD_INIT_FUNC_TRACE(); + auth_only_len = op->sym->auth.data.length - + op->sym->cipher.data.length; + + if (sym->m_dst) { + mbuf = sym->m_dst; + req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3; + } else { + mbuf = sym->m_src; + req_segs = mbuf->nb_segs * 2 + 3; + } + + if (req_segs > MAX_SG_ENTRIES) { + CAAM_JR_DP_ERR("Cipher-Auth: Max sec segs supported is %d", + MAX_SG_ENTRIES); + return NULL; + } + + ctx = caam_jr_alloc_ctx(ses); + if (!ctx) + return NULL; + + ctx->op = op; + cdb = ses->cdb; + sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); + + jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; + + SEC_JD_INIT(jobdescr); + SEC_JD_SET_SD(jobdescr, + (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, + cdb->sh_hdr.hi.field.idlen); + + /* output */ + if (sym->m_dst) + mbuf = sym->m_dst; + else + mbuf = sym->m_src; + + out_sg = &ctx->sg[0]; + if (is_encode(ses)) + length = sym->auth.data.length + ses->digest_length; + else + length = sym->auth.data.length; + + sg = &ctx->sg[0]; + + /* 1st seg */ + sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + + sym->auth.data.offset); + sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset); + + /* Successive segs */ + mbuf = mbuf->next; + while (mbuf) { + sg++; + sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)); + sg->len = cpu_to_caam32(mbuf->data_len); + mbuf = mbuf->next; + } + + if (is_encode(ses)) { + /* set auth output */ + sg++; + sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr); + sg->len = cpu_to_caam32(ses->digest_length); + } + /* last element*/ + sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); + + SEC_JD_SET_OUT_PTR(jobdescr, + (uint64_t)caam_jr_dma_vtop(out_sg), 0, length); + /* set sg bit */ + (jobdescr)->seq_out.command.word |= 0x01000000; + + /* input */ + sg++; + mbuf = sym->m_src; + in_sg = sg; + if (is_encode(ses)) + length = ses->iv.length + sym->auth.data.length; + else + length = ses->iv.length + sym->auth.data.length + + ses->digest_length; + + sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr)); + sg->len = cpu_to_caam32(ses->iv.length); + + sg++; + /* 1st seg */ + sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + + sym->auth.data.offset); + sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset); + + /* Successive segs */ + mbuf = mbuf->next; + while (mbuf) { + sg++; + sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)); + sg->len = cpu_to_caam32(mbuf->data_len); + mbuf = mbuf->next; + } + + if (is_decode(ses)) { + sg++; + rte_memcpy(ctx->digest, sym->auth.digest.data, + ses->digest_length); + sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest)); + sg->len = cpu_to_caam32(ses->digest_length); + } + /* last element*/ + sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); + + SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(in_sg), 0, + length); + /* set sg bit */ + (jobdescr)->seq_in.command.word |= 0x01000000; + /* Auth_only_len is set as 0 in descriptor and it is + * overwritten here in the jd which will update + * the DPOVRD reg. + */ + if (auth_only_len) + /* set sg bit */ + (jobdescr)->dpovrd = 0x80000000 | auth_only_len; + + return ctx; +} + +static inline struct caam_jr_op_ctx * +build_cipher_auth(struct rte_crypto_op *op, struct caam_jr_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct caam_jr_op_ctx *ctx; + struct sec4_sg_entry *sg; + rte_iova_t src_start_addr, dst_start_addr; + uint32_t length = 0; + struct sec_cdb *cdb; + uint64_t sdesc_offset; + uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + ses->iv.offset); + struct sec_job_descriptor_t *jobdescr; + uint32_t auth_only_len; + + PMD_INIT_FUNC_TRACE(); + auth_only_len = op->sym->auth.data.length - + op->sym->cipher.data.length; + + src_start_addr = rte_pktmbuf_iova(sym->m_src); + if (sym->m_dst) + dst_start_addr = rte_pktmbuf_iova(sym->m_dst); + else + dst_start_addr = src_start_addr; + + ctx = caam_jr_alloc_ctx(ses); + if (!ctx) + return NULL; + + ctx->op = op; + cdb = ses->cdb; + sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); + + jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; + + SEC_JD_INIT(jobdescr); + SEC_JD_SET_SD(jobdescr, + (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, + cdb->sh_hdr.hi.field.idlen); + + /* input */ + sg = &ctx->sg[0]; + if (is_encode(ses)) { + sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr)); + sg->len = cpu_to_caam32(ses->iv.length); + length += ses->iv.length; + + sg++; + sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset); + sg->len = cpu_to_caam32(sym->auth.data.length); + length += sym->auth.data.length; + /* last element*/ + sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); + } else { + sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr)); + sg->len = cpu_to_caam32(ses->iv.length); + length += ses->iv.length; + + sg++; + sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset); + sg->len = cpu_to_caam32(sym->auth.data.length); + length += sym->auth.data.length; + + rte_memcpy(ctx->digest, sym->auth.digest.data, + ses->digest_length); + sg++; + sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest)); + sg->len = cpu_to_caam32(ses->digest_length); + length += ses->digest_length; + /* last element*/ + sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); + } + + SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(&ctx->sg[0]), 0, + length); + /* set sg bit */ + (jobdescr)->seq_in.command.word |= 0x01000000; + + /* output */ + sg = &ctx->sg[6]; + + sg->ptr = cpu_to_caam64(dst_start_addr + sym->cipher.data.offset); + sg->len = cpu_to_caam32(sym->cipher.data.length); + length = sym->cipher.data.length; + + if (is_encode(ses)) { + /* set auth output */ + sg++; + sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr); + sg->len = cpu_to_caam32(ses->digest_length); + length += ses->digest_length; + } + /* last element*/ + sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); + + SEC_JD_SET_OUT_PTR(jobdescr, + (uint64_t)caam_jr_dma_vtop(&ctx->sg[6]), 0, length); + /* set sg bit */ + (jobdescr)->seq_out.command.word |= 0x01000000; + + /* Auth_only_len is set as 0 in descriptor and it is + * overwritten here in the jd which will update + * the DPOVRD reg. + */ + if (auth_only_len) + /* set sg bit */ + (jobdescr)->dpovrd = 0x80000000 | auth_only_len; + + return ctx; +} + +static inline struct caam_jr_op_ctx * +build_proto(struct rte_crypto_op *op, struct caam_jr_session *ses) +{ + struct rte_crypto_sym_op *sym = op->sym; + struct caam_jr_op_ctx *ctx = NULL; + phys_addr_t src_start_addr, dst_start_addr; + struct sec_cdb *cdb; + uint64_t sdesc_offset; + struct sec_job_descriptor_t *jobdescr; + + PMD_INIT_FUNC_TRACE(); + ctx = caam_jr_alloc_ctx(ses); + if (!ctx) + return NULL; + ctx->op = op; + + src_start_addr = rte_pktmbuf_iova(sym->m_src); + if (sym->m_dst) + dst_start_addr = rte_pktmbuf_iova(sym->m_dst); + else + dst_start_addr = src_start_addr; + + cdb = ses->cdb; + sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb); + + jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; + + SEC_JD_INIT(jobdescr); + SEC_JD_SET_SD(jobdescr, + (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset, + cdb->sh_hdr.hi.field.idlen); + + /* output */ + SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr, 0, + sym->m_src->buf_len - sym->m_src->data_off); + /* input */ + SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)src_start_addr, 0, + sym->m_src->pkt_len); + sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK; + + return ctx; +} + +static int +caam_jr_enqueue_op(struct rte_crypto_op *op, struct caam_jr_qp *qp) +{ + struct sec_job_ring_t *ring = qp->ring; + struct caam_jr_session *ses; + struct caam_jr_op_ctx *ctx = NULL; + struct sec_job_descriptor_t *jobdescr __rte_unused; + + PMD_INIT_FUNC_TRACE(); + switch (op->sess_type) { + case RTE_CRYPTO_OP_WITH_SESSION: + ses = (struct caam_jr_session *) + get_sym_session_private_data(op->sym->session, + cryptodev_driver_id); + break; + case RTE_CRYPTO_OP_SECURITY_SESSION: + ses = (struct caam_jr_session *) + get_sec_session_private_data( + op->sym->sec_session); + break; + default: + CAAM_JR_DP_ERR("sessionless crypto op not supported"); + qp->tx_errs++; + return -1; + } + + if (unlikely(!ses->qp || ses->qp != qp)) { + CAAM_JR_DP_DEBUG("Old:sess->qp=%p New qp = %p\n", ses->qp, qp); + ses->qp = qp; + caam_jr_prep_cdb(ses); + } + + if (rte_pktmbuf_is_contiguous(op->sym->m_src)) { + if (is_auth_cipher(ses)) + ctx = build_cipher_auth(op, ses); + else if (is_aead(ses)) + goto err1; + else if (is_auth_only(ses)) + ctx = build_auth_only(op, ses); + else if (is_cipher_only(ses)) + ctx = build_cipher_only(op, ses); + else if (is_proto_ipsec(ses)) + ctx = build_proto(op, ses); + } else { + if (is_auth_cipher(ses)) + ctx = build_cipher_auth_sg(op, ses); + else if (is_aead(ses)) + goto err1; + else if (is_auth_only(ses)) + ctx = build_auth_only_sg(op, ses); + else if (is_cipher_only(ses)) + ctx = build_cipher_only_sg(op, ses); + } +err1: + if (unlikely(!ctx)) { + qp->tx_errs++; + CAAM_JR_ERR("not supported sec op"); + return -1; + } +#if CAAM_JR_DBG + if (is_decode(ses)) + rte_hexdump(stdout, "DECODE", + rte_pktmbuf_mtod(op->sym->m_src, void *), + rte_pktmbuf_data_len(op->sym->m_src)); + else + rte_hexdump(stdout, "ENCODE", + rte_pktmbuf_mtod(op->sym->m_src, void *), + rte_pktmbuf_data_len(op->sym->m_src)); + + printf("\n JD before conversion\n"); + for (int i = 0; i < 12; i++) + printf("\n 0x%08x", ctx->jobdes.desc[i]); +#endif + + CAAM_JR_DP_DEBUG("Jr[%p] pi[%d] ci[%d].Before sending desc", + ring, ring->pidx, ring->cidx); + + /* todo - do we want to retry */ + if (SEC_JOB_RING_IS_FULL(ring->pidx, ring->cidx, + SEC_JOB_RING_SIZE, SEC_JOB_RING_SIZE)) { + CAAM_JR_DP_DEBUG("Ring FULL Jr[%p] pi[%d] ci[%d].Size = %d", + ring, ring->pidx, ring->cidx, SEC_JOB_RING_SIZE); + caam_jr_op_ending(ctx); + qp->tx_ring_full++; + return -EBUSY; + } + +#if CORE_BYTE_ORDER != CAAM_BYTE_ORDER + jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc; + + jobdescr->deschdr.command.word = + cpu_to_caam32(jobdescr->deschdr.command.word); + jobdescr->sd_ptr = cpu_to_caam64(jobdescr->sd_ptr); + jobdescr->seq_out.command.word = + cpu_to_caam32(jobdescr->seq_out.command.word); + jobdescr->seq_out_ptr = cpu_to_caam64(jobdescr->seq_out_ptr); + jobdescr->out_ext_length = cpu_to_caam32(jobdescr->out_ext_length); + jobdescr->seq_in.command.word = + cpu_to_caam32(jobdescr->seq_in.command.word); + jobdescr->seq_in_ptr = cpu_to_caam64(jobdescr->seq_in_ptr); + jobdescr->in_ext_length = cpu_to_caam32(jobdescr->in_ext_length); + jobdescr->load_dpovrd.command.word = + cpu_to_caam32(jobdescr->load_dpovrd.command.word); + jobdescr->dpovrd = cpu_to_caam32(jobdescr->dpovrd); +#endif + + /* Set ptr in input ring to current descriptor */ + sec_write_addr(&ring->input_ring[ring->pidx], + (phys_addr_t)caam_jr_vtop_ctx(ctx, ctx->jobdes.desc)); + rte_smp_wmb(); + + /* Notify HW that a new job is enqueued */ + hw_enqueue_desc_on_job_ring(ring); + + /* increment the producer index for the current job ring */ + ring->pidx = SEC_CIRCULAR_COUNTER(ring->pidx, SEC_JOB_RING_SIZE); + + return 0; +} + +static uint16_t +caam_jr_enqueue_burst(void *qp, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + /* Function to transmit the frames to given device and queuepair */ + uint32_t loop; + int32_t ret; + struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp; + uint16_t num_tx = 0; + + PMD_INIT_FUNC_TRACE(); + /*Prepare each packet which is to be sent*/ + for (loop = 0; loop < nb_ops; loop++) { + ret = caam_jr_enqueue_op(ops[loop], jr_qp); + if (!ret) + num_tx++; + } + + jr_qp->tx_pkts += num_tx; + + return num_tx; +} + +/* Release queue pair */ +static int +caam_jr_queue_pair_release(struct rte_cryptodev *dev, + uint16_t qp_id) +{ + struct sec_job_ring_t *internals; + struct caam_jr_qp *qp = NULL; + + PMD_INIT_FUNC_TRACE(); + CAAM_JR_DEBUG("dev =%p, queue =%d", dev, qp_id); + + internals = dev->data->dev_private; + if (qp_id >= internals->max_nb_queue_pairs) { + CAAM_JR_ERR("Max supported qpid %d", + internals->max_nb_queue_pairs); + return -EINVAL; + } + + qp = &internals->qps[qp_id]; + qp->ring = NULL; + dev->data->queue_pairs[qp_id] = NULL; + + return 0; +} + +/* Setup a queue pair */ +static int +caam_jr_queue_pair_setup( + struct rte_cryptodev *dev, uint16_t qp_id, + __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, + __rte_unused int socket_id, + __rte_unused struct rte_mempool *session_pool) +{ + struct sec_job_ring_t *internals; + struct caam_jr_qp *qp = NULL; + + PMD_INIT_FUNC_TRACE(); + CAAM_JR_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf); + + internals = dev->data->dev_private; + if (qp_id >= internals->max_nb_queue_pairs) { + CAAM_JR_ERR("Max supported qpid %d", + internals->max_nb_queue_pairs); + return -EINVAL; + } + + qp = &internals->qps[qp_id]; + qp->ring = internals; + dev->data->queue_pairs[qp_id] = qp; + + return 0; +} + +/* Return the number of allocated queue pairs */ +static uint32_t +caam_jr_queue_pair_count(struct rte_cryptodev *dev) +{ + PMD_INIT_FUNC_TRACE(); + + return dev->data->nb_queue_pairs; +} + +/* Returns the size of the aesni gcm session structure */ +static unsigned int +caam_jr_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + + return sizeof(struct caam_jr_session); +} + +static int +caam_jr_cipher_init(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + struct caam_jr_session *session) +{ + PMD_INIT_FUNC_TRACE(); + session->cipher_alg = xform->cipher.algo; + session->iv.length = xform->cipher.iv.length; + session->iv.offset = xform->cipher.iv.offset; + session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, + RTE_CACHE_LINE_SIZE); + if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) { + CAAM_JR_ERR("No Memory for cipher key\n"); + return -ENOMEM; + } + session->cipher_key.length = xform->cipher.key.length; + + memcpy(session->cipher_key.data, xform->cipher.key.data, + xform->cipher.key.length); + session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? + DIR_ENC : DIR_DEC; + + return 0; +} + +static int +caam_jr_auth_init(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + struct caam_jr_session *session) +{ + PMD_INIT_FUNC_TRACE(); + session->auth_alg = xform->auth.algo; + session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, + RTE_CACHE_LINE_SIZE); + if (session->auth_key.data == NULL && xform->auth.key.length > 0) { + CAAM_JR_ERR("No Memory for auth key\n"); + return -ENOMEM; + } + session->auth_key.length = xform->auth.key.length; + session->digest_length = xform->auth.digest_length; + + memcpy(session->auth_key.data, xform->auth.key.data, + xform->auth.key.length); + session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? + DIR_ENC : DIR_DEC; + + return 0; +} + +static int +caam_jr_aead_init(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + struct caam_jr_session *session) +{ + PMD_INIT_FUNC_TRACE(); + session->aead_alg = xform->aead.algo; + session->iv.length = xform->aead.iv.length; + session->iv.offset = xform->aead.iv.offset; + session->auth_only_len = xform->aead.aad_length; + session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length, + RTE_CACHE_LINE_SIZE); + if (session->aead_key.data == NULL && xform->aead.key.length > 0) { + CAAM_JR_ERR("No Memory for aead key\n"); + return -ENOMEM; + } + session->aead_key.length = xform->aead.key.length; + session->digest_length = xform->aead.digest_length; + + memcpy(session->aead_key.data, xform->aead.key.data, + xform->aead.key.length); + session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? + DIR_ENC : DIR_DEC; + + return 0; +} + +static int +caam_jr_set_session_parameters(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, void *sess) +{ + struct sec_job_ring_t *internals = dev->data->dev_private; + struct caam_jr_session *session = sess; + + PMD_INIT_FUNC_TRACE(); + + if (unlikely(sess == NULL)) { + CAAM_JR_ERR("invalid session struct"); + return -EINVAL; + } + + /* Default IV length = 0 */ + session->iv.length = 0; + + /* Cipher Only */ + if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { + session->auth_alg = RTE_CRYPTO_AUTH_NULL; + caam_jr_cipher_init(dev, xform, session); + + /* Authentication Only */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && + xform->next == NULL) { + session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; + caam_jr_auth_init(dev, xform, session); + + /* Cipher then Authenticate */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && + xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { + if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + caam_jr_cipher_init(dev, xform, session); + caam_jr_auth_init(dev, xform->next, session); + } else { + CAAM_JR_ERR("Not supported: Auth then Cipher"); + goto err1; + } + + /* Authenticate then Cipher */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && + xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { + if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { + caam_jr_auth_init(dev, xform, session); + caam_jr_cipher_init(dev, xform->next, session); + } else { + CAAM_JR_ERR("Not supported: Auth then Cipher"); + goto err1; + } + + /* AEAD operation for AES-GCM kind of Algorithms */ + } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && + xform->next == NULL) { + caam_jr_aead_init(dev, xform, session); + + } else { + CAAM_JR_ERR("Invalid crypto type"); + return -EINVAL; + } + session->ctx_pool = internals->ctx_pool; + + return 0; + +err1: + rte_free(session->cipher_key.data); + rte_free(session->auth_key.data); + memset(session, 0, sizeof(struct caam_jr_session)); + + return -EINVAL; +} + +static int +caam_jr_sym_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + PMD_INIT_FUNC_TRACE(); + + if (rte_mempool_get(mempool, &sess_private_data)) { + CAAM_JR_ERR("Couldn't get object from session mempool"); + return -ENOMEM; + } + + memset(sess_private_data, 0, sizeof(struct caam_jr_session)); + ret = caam_jr_set_session_parameters(dev, xform, sess_private_data); + if (ret != 0) { + CAAM_JR_ERR("failed to configure session parameters"); + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_sym_session_private_data(sess, dev->driver_id, sess_private_data); + + return 0; +} + +/* Clear the memory of session so it doesn't leave key material behind */ +static void +caam_jr_sym_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) +{ + uint8_t index = dev->driver_id; + void *sess_priv = get_sym_session_private_data(sess, index); + struct caam_jr_session *s = (struct caam_jr_session *)sess_priv; + + PMD_INIT_FUNC_TRACE(); + + if (sess_priv) { + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + + rte_free(s->cipher_key.data); + rte_free(s->auth_key.data); + memset(s, 0, sizeof(struct caam_jr_session)); + set_sym_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } +} + +static int +caam_jr_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, + struct rte_security_session_conf *conf, + void *sess) +{ + struct sec_job_ring_t *internals = dev->data->dev_private; + struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec; + struct rte_crypto_auth_xform *auth_xform; + struct rte_crypto_cipher_xform *cipher_xform; + struct caam_jr_session *session = (struct caam_jr_session *)sess; + + PMD_INIT_FUNC_TRACE(); + + if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { + cipher_xform = &conf->crypto_xform->cipher; + auth_xform = &conf->crypto_xform->next->auth; + } else { + auth_xform = &conf->crypto_xform->auth; + cipher_xform = &conf->crypto_xform->next->cipher; + } + session->proto_alg = conf->protocol; + session->cipher_key.data = rte_zmalloc(NULL, + cipher_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->cipher_key.data == NULL && + cipher_xform->key.length > 0) { + CAAM_JR_ERR("No Memory for cipher key\n"); + return -ENOMEM; + } + + session->cipher_key.length = cipher_xform->key.length; + session->auth_key.data = rte_zmalloc(NULL, + auth_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->auth_key.data == NULL && + auth_xform->key.length > 0) { + CAAM_JR_ERR("No Memory for auth key\n"); + rte_free(session->cipher_key.data); + return -ENOMEM; + } + session->auth_key.length = auth_xform->key.length; + memcpy(session->cipher_key.data, cipher_xform->key.data, + cipher_xform->key.length); + memcpy(session->auth_key.data, auth_xform->key.data, + auth_xform->key.length); + + switch (auth_xform->algo) { + case RTE_CRYPTO_AUTH_SHA1_HMAC: + session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC; + break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA512_HMAC: + session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; + break; + case RTE_CRYPTO_AUTH_AES_CMAC: + session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC; + break; + case RTE_CRYPTO_AUTH_NULL: + session->auth_alg = RTE_CRYPTO_AUTH_NULL; + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + case RTE_CRYPTO_AUTH_SNOW3G_UIA2: + case RTE_CRYPTO_AUTH_SHA1: + case RTE_CRYPTO_AUTH_SHA256: + case RTE_CRYPTO_AUTH_SHA512: + case RTE_CRYPTO_AUTH_SHA224: + case RTE_CRYPTO_AUTH_SHA384: + case RTE_CRYPTO_AUTH_MD5: + case RTE_CRYPTO_AUTH_AES_GMAC: + case RTE_CRYPTO_AUTH_KASUMI_F9: + case RTE_CRYPTO_AUTH_AES_CBC_MAC: + case RTE_CRYPTO_AUTH_ZUC_EIA3: + CAAM_JR_ERR("Crypto: Unsupported auth alg %u\n", + auth_xform->algo); + goto out; + default: + CAAM_JR_ERR("Crypto: Undefined Auth specified %u\n", + auth_xform->algo); + goto out; + } + + switch (cipher_xform->algo) { + case RTE_CRYPTO_CIPHER_AES_CBC: + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; + break; + case RTE_CRYPTO_CIPHER_3DES_CBC: + session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; + break; + case RTE_CRYPTO_CIPHER_NULL: + case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: + case RTE_CRYPTO_CIPHER_3DES_ECB: + case RTE_CRYPTO_CIPHER_AES_ECB: + case RTE_CRYPTO_CIPHER_KASUMI_F8: + CAAM_JR_ERR("Crypto: Unsupported Cipher alg %u\n", + cipher_xform->algo); + goto out; + default: + CAAM_JR_ERR("Crypto: Undefined Cipher specified %u\n", + cipher_xform->algo); + goto out; + } + + if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { + memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) + + sizeof(session->ip4_hdr)); + session->ip4_hdr.ip_v = IPVERSION; + session->ip4_hdr.ip_hl = 5; + session->ip4_hdr.ip_len = rte_cpu_to_be_16( + sizeof(session->ip4_hdr)); + session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp; + session->ip4_hdr.ip_id = 0; + session->ip4_hdr.ip_off = 0; + session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl; + session->ip4_hdr.ip_p = (ipsec_xform->proto == + RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP + : IPPROTO_AH; + session->ip4_hdr.ip_sum = 0; + session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip; + session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip; + session->ip4_hdr.ip_sum = calc_chksum((uint16_t *) + (void *)&session->ip4_hdr, + sizeof(struct ip)); + + session->encap_pdb.options = + (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) | + PDBOPTS_ESP_OIHI_PDB_INL | + PDBOPTS_ESP_IVSRC | + PDBHMO_ESP_ENCAP_DTTL; + session->encap_pdb.spi = ipsec_xform->spi; + session->encap_pdb.ip_hdr_len = sizeof(struct ip); + + session->dir = DIR_ENC; + } else if (ipsec_xform->direction == + RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { + memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb)); + session->decap_pdb.options = sizeof(struct ip) << 16; + session->dir = DIR_DEC; + } else + goto out; + session->ctx_pool = internals->ctx_pool; + + return 0; +out: + rte_free(session->auth_key.data); + rte_free(session->cipher_key.data); + memset(session, 0, sizeof(struct caam_jr_session)); + return -1; +} + +static int +caam_jr_security_session_create(void *dev, + struct rte_security_session_conf *conf, + struct rte_security_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev; + int ret; + + PMD_INIT_FUNC_TRACE(); + if (rte_mempool_get(mempool, &sess_private_data)) { + CAAM_JR_ERR("Couldn't get object from session mempool"); + return -ENOMEM; + } + + switch (conf->protocol) { + case RTE_SECURITY_PROTOCOL_IPSEC: + ret = caam_jr_set_ipsec_session(cdev, conf, + sess_private_data); + break; + case RTE_SECURITY_PROTOCOL_MACSEC: + return -ENOTSUP; + default: + return -EINVAL; + } + if (ret != 0) { + CAAM_JR_ERR("failed to configure session parameters"); + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_sec_session_private_data(sess, sess_private_data); + + return ret; +} + +/* Clear the memory of session so it doesn't leave key material behind */ +static int +caam_jr_security_session_destroy(void *dev __rte_unused, + struct rte_security_session *sess) +{ + PMD_INIT_FUNC_TRACE(); + void *sess_priv = get_sec_session_private_data(sess); + + struct caam_jr_session *s = (struct caam_jr_session *)sess_priv; + + if (sess_priv) { + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + + rte_free(s->cipher_key.data); + rte_free(s->auth_key.data); + memset(sess, 0, sizeof(struct caam_jr_session)); + set_sec_session_private_data(sess, NULL); + rte_mempool_put(sess_mp, sess_priv); + } + return 0; +} + + +static int +caam_jr_dev_configure(struct rte_cryptodev *dev, + struct rte_cryptodev_config *config __rte_unused) +{ + char str[20]; + struct sec_job_ring_t *internals; + + PMD_INIT_FUNC_TRACE(); + + internals = dev->data->dev_private; + sprintf(str, "ctx_pool_%d", dev->data->dev_id); + if (!internals->ctx_pool) { + internals->ctx_pool = rte_mempool_create((const char *)str, + CTX_POOL_NUM_BUFS, + sizeof(struct caam_jr_op_ctx), + CTX_POOL_CACHE_SIZE, 0, + NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + if (!internals->ctx_pool) { + CAAM_JR_ERR("%s create failed\n", str); + return -ENOMEM; + } + } else + CAAM_JR_INFO("mempool already created for dev_id : %d", + dev->data->dev_id); + + return 0; +} + +static int +caam_jr_dev_start(struct rte_cryptodev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); + return 0; +} + +static void +caam_jr_dev_stop(struct rte_cryptodev *dev __rte_unused) +{ + PMD_INIT_FUNC_TRACE(); +} + +static int +caam_jr_dev_close(struct rte_cryptodev *dev) +{ + struct sec_job_ring_t *internals; + + PMD_INIT_FUNC_TRACE(); + + if (dev == NULL) + return -ENOMEM; + + internals = dev->data->dev_private; + rte_mempool_free(internals->ctx_pool); + internals->ctx_pool = NULL; + + return 0; +} + +static void +caam_jr_dev_infos_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *info) +{ + struct sec_job_ring_t *internals = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + if (info != NULL) { + info->max_nb_queue_pairs = internals->max_nb_queue_pairs; + info->feature_flags = dev->feature_flags; + info->capabilities = caam_jr_get_cryptodev_capabilities(); + info->sym.max_nb_sessions = internals->max_nb_sessions; + info->driver_id = cryptodev_driver_id; + } +} + +static struct rte_cryptodev_ops caam_jr_ops = { + .dev_configure = caam_jr_dev_configure, + .dev_start = caam_jr_dev_start, + .dev_stop = caam_jr_dev_stop, + .dev_close = caam_jr_dev_close, + .dev_infos_get = caam_jr_dev_infos_get, + .stats_get = caam_jr_stats_get, + .stats_reset = caam_jr_stats_reset, + .queue_pair_setup = caam_jr_queue_pair_setup, + .queue_pair_release = caam_jr_queue_pair_release, + .queue_pair_count = caam_jr_queue_pair_count, + .sym_session_get_size = caam_jr_sym_session_get_size, + .sym_session_configure = caam_jr_sym_session_configure, + .sym_session_clear = caam_jr_sym_session_clear +}; + +static struct rte_security_ops caam_jr_security_ops = { + .session_create = caam_jr_security_session_create, + .session_update = NULL, + .session_stats_get = NULL, + .session_destroy = caam_jr_security_session_destroy, + .set_pkt_metadata = NULL, + .capabilities_get = caam_jr_get_security_capabilities +}; + +/* @brief Flush job rings of any processed descs. + * The processed descs are silently dropped, + * WITHOUT being notified to UA. + */ +static void +close_job_ring(struct sec_job_ring_t *job_ring) +{ + PMD_INIT_FUNC_TRACE(); + if (job_ring->irq_fd) { + /* Producer index is frozen. If consumer index is not equal + * with producer index, then we have descs to flush. + */ + while (job_ring->pidx != job_ring->cidx) + hw_flush_job_ring(job_ring, false, NULL); + + /* free the uio job ring */ + free_job_ring(job_ring->irq_fd); + job_ring->irq_fd = 0; + caam_jr_dma_free(job_ring->input_ring); + caam_jr_dma_free(job_ring->output_ring); + g_job_rings_no--; + } +} + +/** @brief Release the software and hardware resources tied to a job ring. + * @param [in] job_ring The job ring + * + * @retval 0 for success + * @retval -1 for error + */ +static int +shutdown_job_ring(struct sec_job_ring_t *job_ring) +{ + int ret = 0; + + PMD_INIT_FUNC_TRACE(); + ASSERT(job_ring != NULL); + ret = hw_shutdown_job_ring(job_ring); + SEC_ASSERT(ret == 0, ret, + "Failed to shutdown hardware job ring %p", + job_ring); + + if (job_ring->coalescing_en) + hw_job_ring_disable_coalescing(job_ring); + + if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) { + ret = caam_jr_disable_irqs(job_ring->irq_fd); + SEC_ASSERT(ret == 0, ret, + "Failed to disable irqs for job ring %p", + job_ring); + } + + return ret; +} + +/* + * @brief Release the resources used by the SEC user space driver. + * + * Reset and release SEC's job rings indicated by the User Application at + * init_job_ring() and free any memory allocated internally. + * Call once during application tear down. + * + * @note In case there are any descriptors in-flight (descriptors received by + * SEC driver for processing and for which no response was yet provided to UA), + * the descriptors are discarded without any notifications to User Application. + * + * @retval ::0 is returned for a successful execution + * @retval ::-1 is returned if SEC driver release is in progress + */ +static int +caam_jr_dev_uninit(struct rte_cryptodev *dev) +{ + struct sec_job_ring_t *internals; + + PMD_INIT_FUNC_TRACE(); + if (dev == NULL) + return -ENODEV; + + internals = dev->data->dev_private; + rte_free(dev->security_ctx); + + /* If any descriptors in flight , poll and wait + * until all descriptors are received and silently discarded. + */ + if (internals) { + shutdown_job_ring(internals); + close_job_ring(internals); + rte_mempool_free(internals->ctx_pool); + } + + CAAM_JR_INFO("Closing crypto device %s", dev->data->name); + + /* last caam jr instance) */ + if (g_job_rings_no == 0) + g_driver_state = SEC_DRIVER_STATE_IDLE; + + return SEC_SUCCESS; +} + +/* @brief Initialize the software and hardware resources tied to a job ring. + * @param [in] jr_mode; Model to be used by SEC Driver to receive + * notifications from SEC. Can be either + * of the three: #SEC_NOTIFICATION_TYPE_NAPI + * #SEC_NOTIFICATION_TYPE_IRQ or + * #SEC_NOTIFICATION_TYPE_POLL + * @param [in] NAPI_mode The NAPI work mode to configure a job ring at + * startup. Used only when #SEC_NOTIFICATION_TYPE + * is set to #SEC_NOTIFICATION_TYPE_NAPI. + * @param [in] irq_coalescing_timer This value determines the maximum + * amount of time after processing a + * descriptor before raising an interrupt. + * @param [in] irq_coalescing_count This value determines how many + * descriptors are completed before + * raising an interrupt. + * @param [in] reg_base_addr, The job ring base address register + * @param [in] irq_id The job ring interrupt identification number. + * @retval job_ring_handle for successful job ring configuration + * @retval NULL on error + * + */ +static void * +init_job_ring(void *reg_base_addr, uint32_t irq_id) +{ + struct sec_job_ring_t *job_ring = NULL; + int i, ret = 0; + int jr_mode = SEC_NOTIFICATION_TYPE_POLL; + int napi_mode = 0; + int irq_coalescing_timer = 0; + int irq_coalescing_count = 0; + + for (i = 0; i < MAX_SEC_JOB_RINGS; i++) { + if (g_job_rings[i].irq_fd == 0) { + job_ring = &g_job_rings[i]; + g_job_rings_no++; + break; + } + } + if (job_ring == NULL) { + CAAM_JR_ERR("No free job ring\n"); + return NULL; + } + + job_ring->register_base_addr = reg_base_addr; + job_ring->jr_mode = jr_mode; + job_ring->napi_mode = 0; + job_ring->irq_fd = irq_id; + + /* Allocate mem for input and output ring */ + + /* Allocate memory for input ring */ + job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, + SEC_DMA_MEM_INPUT_RING_SIZE); + memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE); + + /* Allocate memory for output ring */ + job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, + SEC_DMA_MEM_OUTPUT_RING_SIZE); + memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE); + + /* Reset job ring in SEC hw and configure job ring registers */ + ret = hw_reset_job_ring(job_ring); + if (ret != 0) { + CAAM_JR_ERR("Failed to reset hardware job ring"); + goto cleanup; + } + + if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) { + /* When SEC US driver works in NAPI mode, the UA can select + * if the driver starts with IRQs on or off. + */ + if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) { + CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p", + job_ring); + ret = caam_jr_enable_irqs(job_ring->irq_fd); + if (ret != 0) { + CAAM_JR_ERR("Failed to enable irqs for job ring"); + goto cleanup; + } + } + } else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) { + /* When SEC US driver works in pure interrupt mode, + * IRQ's are always enabled. + */ + CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p", + job_ring); + ret = caam_jr_enable_irqs(job_ring->irq_fd); + if (ret != 0) { + CAAM_JR_ERR("Failed to enable irqs for job ring"); + goto cleanup; + } + } + if (irq_coalescing_timer || irq_coalescing_count) { + hw_job_ring_set_coalescing_param(job_ring, + irq_coalescing_timer, + irq_coalescing_count); + + hw_job_ring_enable_coalescing(job_ring); + job_ring->coalescing_en = 1; + } + + job_ring->jr_state = SEC_JOB_RING_STATE_STARTED; + job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS; + job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS; + + return job_ring; +cleanup: + caam_jr_dma_free(job_ring->output_ring); + caam_jr_dma_free(job_ring->input_ring); + return NULL; +} + + +static int +caam_jr_dev_init(const char *name, + struct rte_vdev_device *vdev, + struct rte_cryptodev_pmd_init_params *init_params) +{ + struct rte_cryptodev *dev; + struct rte_security_ctx *security_instance; + struct uio_job_ring *job_ring; + char str[RTE_CRYPTODEV_NAME_MAX_LEN]; + + PMD_INIT_FUNC_TRACE(); + + /* Validate driver state */ + if (g_driver_state == SEC_DRIVER_STATE_IDLE) { + g_job_rings_max = sec_configure(); + if (!g_job_rings_max) { + CAAM_JR_ERR("No job ring detected on UIO !!!!"); + return -1; + } + /* Update driver state */ + g_driver_state = SEC_DRIVER_STATE_STARTED; + } + + if (g_job_rings_no >= g_job_rings_max) { + CAAM_JR_ERR("No more job rings available max=%d!!!!", + g_job_rings_max); + return -1; + } + + job_ring = config_job_ring(); + if (job_ring == NULL) { + CAAM_JR_ERR("failed to create job ring"); + goto init_error; + } + + snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id); + + dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); + if (dev == NULL) { + CAAM_JR_ERR("failed to create cryptodev vdev"); + goto cleanup; + } + /*TODO free it during teardown*/ + dev->data->dev_private = init_job_ring(job_ring->register_base_addr, + job_ring->uio_fd); + + if (!dev->data->dev_private) { + CAAM_JR_ERR("Ring memory allocation failed\n"); + goto cleanup2; + } + + dev->driver_id = cryptodev_driver_id; + dev->dev_ops = &caam_jr_ops; + + /* register rx/tx burst functions for data path */ + dev->dequeue_burst = caam_jr_dequeue_burst; + dev->enqueue_burst = caam_jr_enqueue_burst; + dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_HW_ACCELERATED | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | + RTE_CRYPTODEV_FF_SECURITY | + RTE_CRYPTODEV_FF_IN_PLACE_SGL | + RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | + RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | + RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | + RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; + + /* For secondary processes, we don't initialise any further as primary + * has already done this work. Only check we don't need a different + * RX function + */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + CAAM_JR_WARN("Device already init by primary process"); + return 0; + } + + /*TODO free it during teardown*/ + security_instance = rte_malloc("caam_jr", + sizeof(struct rte_security_ctx), 0); + if (security_instance == NULL) { + CAAM_JR_ERR("memory allocation failed\n"); + //todo error handling. + goto cleanup2; + } + + security_instance->device = (void *)dev; + security_instance->ops = &caam_jr_security_ops; + security_instance->sess_cnt = 0; + dev->security_ctx = security_instance; + + RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name); + + return 0; + +cleanup2: + caam_jr_dev_uninit(dev); + rte_cryptodev_pmd_release_device(dev); +cleanup: + free_job_ring(job_ring->uio_fd); +init_error: + CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed", + init_params->name); + + return -ENXIO; +} + +/** Initialise CAAM JR crypto device */ +static int +cryptodev_caam_jr_probe(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev_pmd_init_params init_params = { + "", + sizeof(struct sec_job_ring_t), + rte_socket_id(), + RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS + }; + const char *name; + const char *input_args; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + + input_args = rte_vdev_device_args(vdev); + rte_cryptodev_pmd_parse_input_args(&init_params, input_args); + + /* if sec device version is not configured */ + if (!rta_get_sec_era()) { + const struct device_node *caam_node; + + for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { + const uint32_t *prop = of_get_property(caam_node, + "fsl,sec-era", + NULL); + if (prop) { + rta_set_sec_era( + INTL_SEC_ERA(cpu_to_caam32(*prop))); + break; + } + } + } +#ifdef RTE_LIBRTE_PMD_CAAM_JR_BE + if (rta_get_sec_era() > RTA_SEC_ERA_8) { + RTE_LOG(ERR, PMD, + "CAAM is compiled in BE mode for device with sec era > 8???\n"); + return -EINVAL; + } +#endif + + return caam_jr_dev_init(name, vdev, &init_params); +} + +/** Uninitialise CAAM JR crypto device */ +static int +cryptodev_caam_jr_remove(struct rte_vdev_device *vdev) +{ + struct rte_cryptodev *cryptodev; + const char *name; + + name = rte_vdev_device_name(vdev); + if (name == NULL) + return -EINVAL; + + cryptodev = rte_cryptodev_pmd_get_named_dev(name); + if (cryptodev == NULL) + return -ENODEV; + + caam_jr_dev_uninit(cryptodev); + + return rte_cryptodev_pmd_destroy(cryptodev); +} + +static struct rte_vdev_driver cryptodev_caam_jr_drv = { + .probe = cryptodev_caam_jr_probe, + .remove = cryptodev_caam_jr_remove +}; + +static struct cryptodev_driver caam_jr_crypto_drv; + +RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD, + "max_nb_queue_pairs=" + "socket_id="); +RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver, + cryptodev_driver_id); + +RTE_INIT(caam_jr_init_log) +{ + caam_jr_logtype = rte_log_register("pmd.crypto.caam"); + if (caam_jr_logtype >= 0) + rte_log_set_level(caam_jr_logtype, RTE_LOG_NOTICE); +} diff --git a/drivers/crypto/caam_jr/caam_jr_capabilities.c b/drivers/crypto/caam_jr/caam_jr_capabilities.c new file mode 100644 index 00000000..c51593c4 --- /dev/null +++ b/drivers/crypto/caam_jr/caam_jr_capabilities.c @@ -0,0 +1,266 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017-2018 NXP + */ + +#include + +static const struct rte_cryptodev_capabilities caam_jr_capabilities[] = { + { /* MD5 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_MD5_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 1, + .max = 16, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA1 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA1_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 1, + .max = 20, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA224 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA224_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 1, + .max = 28, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA256 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256_HMAC, + .block_size = 64, + .key_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .digest_size = { + .min = 1, + .max = 32, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA384 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA384_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 1, + .max = 48, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA512 HMAC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA512_HMAC, + .block_size = 128, + .key_size = { + .min = 1, + .max = 128, + .increment = 1 + }, + .digest_size = { + .min = 1, + .max = 64, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 8, + .max = 16, + .increment = 4 + }, + .aad_size = { + .min = 0, + .max = 240, + .increment = 1 + }, + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 + }, + }, } + }, } + }, + { /* AES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CBC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* AES CTR */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_CTR, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + { /* 3DES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .block_size = 8, + .key_size = { + .min = 16, + .max = 24, + .increment = 8 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, + + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + +static const struct rte_security_capability caam_jr_security_cap[] = { + { /* IPsec Lookaside Protocol offload ESP Transport Egress */ + .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + .ipsec = { + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, + .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS, + .options = { 0 } + }, + .crypto_capabilities = caam_jr_capabilities + }, + { /* IPsec Lookaside Protocol offload ESP Tunnel Ingress */ + .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL, + .protocol = RTE_SECURITY_PROTOCOL_IPSEC, + .ipsec = { + .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP, + .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL, + .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS, + .options = { 0 } + }, + .crypto_capabilities = caam_jr_capabilities + }, + { + .action = RTE_SECURITY_ACTION_TYPE_NONE + } +}; + +const struct rte_cryptodev_capabilities * +caam_jr_get_cryptodev_capabilities(void) +{ + return caam_jr_capabilities; +} + +const struct rte_security_capability * +caam_jr_get_security_capabilities(void *device __rte_unused) +{ + return caam_jr_security_cap; +} diff --git a/drivers/crypto/caam_jr/caam_jr_capabilities.h b/drivers/crypto/caam_jr/caam_jr_capabilities.h new file mode 100644 index 00000000..c1e3f305 --- /dev/null +++ b/drivers/crypto/caam_jr/caam_jr_capabilities.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017-2018 NXP + */ + +#ifndef CAAM_JR_CAPABILITIES_H +#define CAAM_JR_CAPABILITIES_H + +#include +#include + +/* Get cryptodev capabilities */ +const struct rte_cryptodev_capabilities * +caam_jr_get_cryptodev_capabilities(void); +/* Get security capabilities */ +const struct rte_security_capability * +caam_jr_get_security_capabilities(void *device); + +#endif diff --git a/drivers/crypto/caam_jr/caam_jr_config.h b/drivers/crypto/caam_jr/caam_jr_config.h new file mode 100644 index 00000000..041187a8 --- /dev/null +++ b/drivers/crypto/caam_jr/caam_jr_config.h @@ -0,0 +1,207 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017-2018 NXP + */ + +#ifndef CAAM_JR_CONFIG_H +#define CAAM_JR_CONFIG_H + +#include + +#include + +#ifdef RTE_LIBRTE_PMD_CAAM_JR_BE +#define CAAM_BYTE_ORDER __BIG_ENDIAN +#else +#define CAAM_BYTE_ORDER __LITTLE_ENDIAN +#endif + +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN +#define CORE_BYTE_ORDER __BIG_ENDIAN +#else +#define CORE_BYTE_ORDER __LITTLE_ENDIAN +#endif + +#if CORE_BYTE_ORDER != CAAM_BYTE_ORDER + +#define cpu_to_caam64 rte_cpu_to_be_64 +#define cpu_to_caam32 rte_cpu_to_be_32 +#else +#define cpu_to_caam64 +#define cpu_to_caam32 + +#endif + +/* + * SEC is configured to start work in polling mode, + * when configured for NAPI notification style. + */ +#define SEC_STARTUP_POLLING_MODE 0 +/* + * SEC is configured to start work in interrupt mode, + * when configured for NAPI notification style. + */ +#define SEC_STARTUP_INTERRUPT_MODE 1 + +/* + * SEC driver will use NAPI model to receive notifications + * for processed packets from SEC engine hardware: + * - IRQ for low traffic + * - polling for high traffic. + */ +#define SEC_NOTIFICATION_TYPE_NAPI 0 +/* + * SEC driver will use ONLY interrupts to receive notifications + * for processed packets from SEC engine hardware. + */ +#define SEC_NOTIFICATION_TYPE_IRQ 1 +/* + * SEC driver will use ONLY polling to receive notifications + * for processed packets from SEC engine hardware. + */ +#define SEC_NOTIFICATION_TYPE_POLL 2 + +/* + * SEC USER SPACE DRIVER related configuration. + */ + +/* + * Determines how SEC user space driver will receive notifications + * for processed packets from SEC engine. + * Valid values are: #SEC_NOTIFICATION_TYPE_POLL, #SEC_NOTIFICATION_TYPE_IRQ + * and #SEC_NOTIFICATION_TYPE_NAPI. + */ +#define SEC_NOTIFICATION_TYPE SEC_NOTIFICATION_TYPE_POLL + +/* Maximum number of job rings supported by SEC hardware */ +#define MAX_SEC_JOB_RINGS 4 + +/* Maximum number of QP per job ring */ +#define RTE_CAAM_MAX_NB_SEC_QPS 1 + +/* + * Size of cryptographic context that is used directly in communicating + * with SEC device. SEC device works only with physical addresses. This + * is the maximum size for a SEC descriptor ( = 64 words). + */ +#define SEC_CRYPTO_DESCRIPTOR_SIZE 256 + +/* + * Size of job descriptor submitted to SEC device for each packet to + * be processed. + * Job descriptor contains 3 DMA address pointers: + * - to shared descriptor, to input buffer and to output buffer. + * The job descriptor contains other SEC specific commands as well: + * - HEADER command, SEQ IN PTR command SEQ OUT PTR command and opaque data + * each measuring 4 bytes. + * Job descriptor size, depending on physical address representation: + * - 32 bit - size is 28 bytes - cacheline-aligned size is 64 bytes + * - 36 bit - size is 40 bytes - cacheline-aligned size is 64 bytes + * @note: Job descriptor must be cacheline-aligned to ensure efficient + * memory access. + * @note: If other format is used for job descriptor, then the size must be + * revised. + */ +#define SEC_JOB_DESCRIPTOR_SIZE 64 + +/* + * Size of one entry in the input ring of a job ring. + * Input ring contains pointers to job descriptors. + * The memory used for an input ring and output ring must be physically + * contiguous. + */ +#define SEC_JOB_INPUT_RING_ENTRY_SIZE sizeof(dma_addr_t) + +/* + * Size of one entry in the output ring of a job ring. + * Output ring entry is a pointer to a job descriptor followed by a 4 byte + * status word. + * The memory used for an input ring and output ring must be physically + * contiguous. + * @note If desired to use also the optional SEQ OUT indication in output ring + * entries, + * then 4 more bytes must be added to the size. + */ +#define SEC_JOB_OUTPUT_RING_ENTRY_SIZE (SEC_JOB_INPUT_RING_ENTRY_SIZE + 4) + +/* + * DMA memory required for an input ring of a job ring. + */ +#define SEC_DMA_MEM_INPUT_RING_SIZE ((SEC_JOB_INPUT_RING_ENTRY_SIZE) * \ + (SEC_JOB_RING_SIZE)) + +/* + * DMA memory required for an output ring of a job ring. + * Required extra 4 byte for status word per each entry. + */ +#define SEC_DMA_MEM_OUTPUT_RING_SIZE ((SEC_JOB_OUTPUT_RING_ENTRY_SIZE) * \ + (SEC_JOB_RING_SIZE)) + +/* DMA memory required for a job ring, including both input and output rings. */ +#define SEC_DMA_MEM_JOB_RING_SIZE ((SEC_DMA_MEM_INPUT_RING_SIZE) + \ + (SEC_DMA_MEM_OUTPUT_RING_SIZE)) + +/* + * When calling sec_init() UA will provide an area of virtual memory + * of size #SEC_DMA_MEMORY_SIZE to be used internally by the driver + * to allocate data (like SEC descriptors) that needs to be passed to + * SEC device in physical addressing and later on retrieved from SEC device. + * At initialization the UA provides specialized ptov/vtop functions/macros to + * translate addresses allocated from this memory area. + */ +#define SEC_DMA_MEMORY_SIZE ((SEC_DMA_MEM_JOB_RING_SIZE) * \ + (MAX_SEC_JOB_RINGS)) + +#define L1_CACHE_BYTES 64 + +/* SEC JOB RING related configuration. */ + +/* + * Configure the size of the JOB RING. + * The maximum size of the ring in hardware limited to 1024. + * However the number of packets in flight in a time interval of 1ms can + * be calculated from the traffic rate (Mbps) and packet size. + * Here it was considered a packet size of 64 bytes. + * + * @note Round up to nearest power of 2 for optimized update + * of producer/consumer indexes of each job ring + */ +#define SEC_JOB_RING_SIZE 512 + +/* + * Interrupt coalescing related configuration. + * NOTE: SEC hardware enabled interrupt + * coalescing is not supported on SEC version 3.1! + * SEC version 4.4 has support for interrupt + * coalescing. + */ + +#if SEC_NOTIFICATION_TYPE != SEC_NOTIFICATION_TYPE_POLL + +#define SEC_INT_COALESCING_ENABLE 1 +/* + * Interrupt Coalescing Descriptor Count Threshold. + * While interrupt coalescing is enabled (ICEN=1), this value determines + * how many Descriptors are completed before raising an interrupt. + * + * Valid values for this field are from 0 to 255. + * Note that a value of 1 functionally defeats the advantages of interrupt + * coalescing since the threshold value is reached each time that a + * Job Descriptor is completed. A value of 0 is treated in the same + * manner as a value of 1. + */ +#define SEC_INTERRUPT_COALESCING_DESCRIPTOR_COUNT_THRESH 10 + +/* + * Interrupt Coalescing Timer Threshold. + * While interrupt coalescing is enabled (ICEN=1), this value determines the + * maximum amount of time after processing a Descriptor before raising an + * interrupt. + * The threshold value is represented in units equal to 64 CAAM interface + * clocks. Valid values for this field are from 1 to 65535. + * A value of 0 results in behavior identical to that when interrupt + * coalescing is disabled. + */ +#define SEC_INTERRUPT_COALESCING_TIMER_THRESH 100 +#endif /* SEC_NOTIFICATION_TYPE_POLL */ + +#endif /* CAAM_JR_CONFIG_H */ diff --git a/drivers/crypto/caam_jr/caam_jr_desc.h b/drivers/crypto/caam_jr/caam_jr_desc.h new file mode 100644 index 00000000..6683ea83 --- /dev/null +++ b/drivers/crypto/caam_jr/caam_jr_desc.h @@ -0,0 +1,285 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017-2018 NXP + */ + +#ifndef CAAM_JR_DESC_H +#define CAAM_JR_DESC_H + +#define CMD_HDR_CTYPE_SD 0x16 +#define CMD_HDR_CTYPE_JD 0x17 + +/* The maximum size of a SEC descriptor, in WORDs (32 bits). */ +#define MAX_DESC_SIZE_WORDS 64 + +/* + * Macros manipulating descriptors + */ +/* Macro for setting the SD pointer in a JD. Common for all protocols + * supported by the SEC driver. + */ +#define SEC_JD_SET_SD(descriptor, ptr, len) { \ + (descriptor)->sd_ptr = (ptr); \ + (descriptor)->deschdr.command.jd.shr_desc_len = (len); \ +} + +/* Macro for setting a pointer to the job which this descriptor processes. + * It eases the lookup procedure for identifying the descriptor that has + * completed. + */ +#define SEC_JD_SET_JOB_PTR(descriptor, ptr) \ + ((descriptor)->job_ptr = (ptr)) + +/* Macro for setting up a JD. The structure of the JD is common across all + * supported protocols, thus its structure is identical. + */ +#define SEC_JD_INIT(descriptor) ({ \ + /* CTYPE = job descriptor \ + * RSMS, DNR = 0 + * ONE = 1 + * Start Index = 0 + * ZRO,TD, MTD = 0 + * SHR = 1 (there's a shared descriptor referenced + * by this job descriptor,pointer in next word) + * REO = 1 (execute job descr. first, shared descriptor + * after) + * SHARE = DEFER + * Descriptor Length = 0 ( to be completed @ runtime ) */ \ + (descriptor)->deschdr.command.word = 0xB0801C0D; \ + /* + * CTYPE = SEQ OUT command * Scater Gather Flag = 0 + * (can be updated @ runtime) PRE = 0 * EXT = 1 + * (data length is in next word, following the * command) + * RTO = 0 */ \ + (descriptor)->seq_out.command.word = 0xF8400000; /**/ \ + /* + * CTYPE = SEQ IN command + * Scater Gather Flag = 0 (can be updated @ runtime) + * PRE = 0 + * EXT = 1 ( data length is in next word, following the + * command) + * RTO = 0 */ \ + (descriptor)->seq_in.command.word = 0xF0400000; /**/ \ + /* + * In order to be compatible with QI scenarios, the DPOVRD value + * loaded must be formated like this: + * DPOVRD_EN (1b) | Res| DPOVRD Value (right aligned). */ \ + (descriptor)->load_dpovrd.command.word = 0x16870004; \ + /* By default, DPOVRD mechanism is disabled, thus the value to be + * LOAD-ed through the above descriptor command will be + * 0x0000_0000. */ \ + (descriptor)->dpovrd = 0x00000000; \ +}) + +/* Macro for setting the pointer to the input buffer in the JD, according to + * the parameters set by the user in the ::sec_packet_t structure. + */ +#define SEC_JD_SET_IN_PTR(descriptor, phys_addr, offset, length) { \ + (descriptor)->seq_in_ptr = (phys_addr) + (offset); \ + (descriptor)->in_ext_length = (length); \ +} + +/* Macro for setting the pointer to the output buffer in the JD, according to + * the parameters set by the user in the ::sec_packet_t structure. + */ +#define SEC_JD_SET_OUT_PTR(descriptor, phys_addr, offset, length) { \ + (descriptor)->seq_out_ptr = (phys_addr) + (offset); \ + (descriptor)->out_ext_length = (length); \ +} + +/* Macro for setting the Scatter-Gather flag in the SEQ IN command. Used in + * case the input buffer is split in multiple buffers, according to the user + * specification. + */ +#define SEC_JD_SET_SG_IN(descriptor) \ + ((descriptor)->seq_in.command.field.sgf = 1) + +/* Macro for setting the Scatter-Gather flag in the SEQ OUT command. Used in + * case the output buffer is split in multiple buffers, according to the user + * specification. + */ +#define SEC_JD_SET_SG_OUT(descriptor) \ + ((descriptor)->seq_out.command.field.sgf = 1) + +#define SEC_JD_SET_DPOVRD(descriptor) \ + +/* Macro for retrieving a descriptor's length. Works for both SD and JD. */ +#define SEC_GET_DESC_LEN(descriptor) \ + (((struct descriptor_header_s *)(descriptor))->command.sd.ctype == \ + CMD_HDR_CTYPE_SD ? ((struct descriptor_header_s *) \ + (descriptor))->command.sd.desclen : \ + ((struct descriptor_header_s *)(descriptor))->command.jd.desclen) + +/* Helper macro for dumping the hex representation of a descriptor */ +#define SEC_DUMP_DESC(descriptor) { \ + int __i; \ + CAAM_JR_INFO("Des@ 0x%08x\n", (uint32_t)((uint32_t *)(descriptor)));\ + for (__i = 0; \ + __i < SEC_GET_DESC_LEN(descriptor); \ + __i++) { \ + printf("0x%08x: 0x%08x\n", \ + (uint32_t)(((uint32_t *)(descriptor)) + __i), \ + *(((uint32_t *)(descriptor)) + __i)); \ + } \ +} +/* Union describing a descriptor header. + */ +struct descriptor_header_s { + union { + uint32_t word; + struct { + /* 4 */ unsigned int ctype:5; + /* 5 */ unsigned int res1:2; + /* 7 */ unsigned int dnr:1; + /* 8 */ unsigned int one:1; + /* 9 */ unsigned int res2:1; + /* 10 */ unsigned int start_idx:6; + /* 16 */ unsigned int res3:2; + /* 18 */ unsigned int cif:1; + /* 19 */ unsigned int sc:1; + /* 20 */ unsigned int pd:1; + /* 21 */ unsigned int res4:1; + /* 22 */ unsigned int share:2; + /* 24 */ unsigned int res5:2; + /* 26 */ unsigned int desclen:6; + } sd; + struct { + /* TODO only below struct members are corrected, + * all others also need to be reversed please verify it + */ + /* 0 */ unsigned int desclen:7; + /* 7 */ unsigned int res4:1; + /* 8 */ unsigned int share:3; + /* 11 */ unsigned int reo:1; + /* 12 */ unsigned int shr:1; + /* 13 */ unsigned int mtd:1; + /* 14 */ unsigned int td:1; + /* 15 */ unsigned int zero:1; + /* 16 */ unsigned int shr_desc_len:6; + /* 22 */ unsigned int res2:1; + /* 23 */ unsigned int one:1; + /* 24 */ unsigned int dnr:1; + /* 25 */ unsigned int rsms:1; + /* 26 */ unsigned int res1:1; + /* 27 */ unsigned int ctype:5; + } jd; + } __rte_packed command; +} __rte_packed; + +/* Union describing a KEY command in a descriptor. + */ +struct key_command_s { + union { + uint32_t word; + struct { + unsigned int ctype:5; + unsigned int cls:2; + unsigned int sgf:1; + unsigned int imm:1; + unsigned int enc:1; + unsigned int nwb:1; + unsigned int ekt:1; + unsigned int kdest:4; + unsigned int tk:1; + unsigned int rsvd1:5; + unsigned int length:10; + } __rte_packed field; + } __rte_packed command; +} __rte_packed; + +/* Union describing a PROTOCOL command + * in a descriptor. + */ +struct protocol_operation_command_s { + union { + uint32_t word; + struct { + unsigned int ctype:5; + unsigned int optype:3; + unsigned char protid; + unsigned short protinfo; + } __rte_packed field; + } __rte_packed command; +} __rte_packed; + +/* Union describing a SEQIN command in a + * descriptor. + */ +struct seq_in_command_s { + union { + uint32_t word; + struct { + unsigned int ctype:5; + unsigned int res1:1; + unsigned int inl:1; + unsigned int sgf:1; + unsigned int pre:1; + unsigned int ext:1; + unsigned int rto:1; + unsigned int rjd:1; + unsigned int res2:4; + unsigned int length:16; + } field; + } __rte_packed command; +} __rte_packed; + +/* Union describing a SEQOUT command in a + * descriptor. + */ +struct seq_out_command_s { + union { + uint32_t word; + struct { + unsigned int ctype:5; + unsigned int res1:2; + unsigned int sgf:1; + unsigned int pre:1; + unsigned int ext:1; + unsigned int rto:1; + unsigned int res2:5; + unsigned int length:16; + } field; + } __rte_packed command; +} __rte_packed; + +struct load_command_s { + union { + uint32_t word; + struct { + unsigned int ctype:5; + unsigned int class:2; + unsigned int sgf:1; + unsigned int imm:1; + unsigned int dst:7; + unsigned char offset; + unsigned char length; + } fields; + } __rte_packed command; +} __rte_packed; + +/* Structure encompassing a general shared descriptor of maximum + * size (64 WORDs). Usually, other specific shared descriptor structures + * will be type-casted to this one + * this one. + */ +struct sec_sd_t { + uint32_t rsvd[MAX_DESC_SIZE_WORDS]; +} __attribute__((packed, aligned(64))); + +/* Structure encompassing a job descriptor which processes + * a single packet from a context. The job descriptor references + * a shared descriptor from a SEC context. + */ +struct sec_job_descriptor_t { + struct descriptor_header_s deschdr; + dma_addr_t sd_ptr; + struct seq_out_command_s seq_out; + dma_addr_t seq_out_ptr; + uint32_t out_ext_length; + struct seq_in_command_s seq_in; + dma_addr_t seq_in_ptr; + uint32_t in_ext_length; + struct load_command_s load_dpovrd; + uint32_t dpovrd; +} __attribute__((packed, aligned(64))); + +#endif diff --git a/drivers/crypto/caam_jr/caam_jr_hw.c b/drivers/crypto/caam_jr/caam_jr_hw.c new file mode 100644 index 00000000..4a2b0899 --- /dev/null +++ b/drivers/crypto/caam_jr/caam_jr_hw.c @@ -0,0 +1,367 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017-2018 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +/* RTA header files */ +#include +#include +#include + +/* Used to retry resetting a job ring in SEC hardware. */ +#define SEC_TIMEOUT 100000 + +/* @brief Process Jump Halt Condition related errors + * + * @param [in] error_code The error code in the descriptor status word + */ +static inline void +hw_handle_jmp_halt_cond_err(union hw_error_code error_code) +{ + CAAM_JR_DEBUG("JMP: %d, Descriptor Index: 0x%x, Condition: 0x%x", + error_code.error_desc.jmp_halt_cond_src.jmp, + error_code.error_desc.jmp_halt_cond_src.desc_idx, + error_code.error_desc.jmp_halt_cond_src.cond); + (void)error_code; +} + +/* @brief Process DECO related errors + * + * @param [in] error_code The error code in the descriptor status word + */ +static inline void +hw_handle_deco_err(union hw_error_code error_code) +{ + CAAM_JR_DEBUG("JMP: %d, Descriptor Index: 0x%x", + error_code.error_desc.deco_src.jmp, + error_code.error_desc.deco_src.desc_idx); + + switch (error_code.error_desc.deco_src.desc_err) { + case SEC_HW_ERR_DECO_HFN_THRESHOLD: + CAAM_JR_DEBUG(" Warning: Descriptor completed normally," + "but 3GPP HFN matches or exceeds the Threshold "); + break; + default: + CAAM_JR_DEBUG("Error 0x%04x not implemented", + error_code.error_desc.deco_src.desc_err); + break; + } +} + +/* @brief Process Jump Halt User Status related errors + * + * @param [in] error_code The error code in the descriptor status word + */ +static inline void +hw_handle_jmp_halt_user_err(union hw_error_code error_code __rte_unused) +{ + CAAM_JR_DEBUG(" Not implemented"); +} + +/* @brief Process CCB related errors + * + * @param [in] error_code The error code in the descriptor status word + */ +static inline void +hw_handle_ccb_err(union hw_error_code hw_error_code __rte_unused) +{ + CAAM_JR_DEBUG(" Not implemented"); +} + +/* @brief Process Job Ring related errors + * + * @param [in] error_code The error code in the descriptor status word + */ +static inline void +hw_handle_jr_err(union hw_error_code hw_error_code __rte_unused) +{ + CAAM_JR_DEBUG(" Not implemented"); +} + +int +hw_reset_job_ring(struct sec_job_ring_t *job_ring) +{ + int ret = 0; + + ASSERT(job_ring->register_base_addr != NULL); + + /* First reset the job ring in hw */ + ret = hw_shutdown_job_ring(job_ring); + SEC_ASSERT(ret == 0, ret, "Failed resetting job ring in hardware"); + + /* In order to have the HW JR in a workable state + * after a reset, I need to re-write the input + * queue size, input start address, output queue + * size and output start address + */ + /* Write the JR input queue size to the HW register */ + hw_set_input_ring_size(job_ring, SEC_JOB_RING_SIZE); + + /* Write the JR output queue size to the HW register */ + hw_set_output_ring_size(job_ring, SEC_JOB_RING_SIZE); + + /* Write the JR input queue start address */ + hw_set_input_ring_start_addr(job_ring, + caam_jr_dma_vtop(job_ring->input_ring)); + CAAM_JR_DEBUG(" Set input ring base address to : Virtual: 0x%" PRIx64 + ",Physical: 0x%" PRIx64 ", Read from HW: 0x%" PRIx64, + (uint64_t)(uintptr_t)job_ring->input_ring, + caam_jr_dma_vtop(job_ring->input_ring), + hw_get_inp_queue_base(job_ring)); + + /* Write the JR output queue start address */ + hw_set_output_ring_start_addr(job_ring, + caam_jr_dma_vtop(job_ring->output_ring)); + CAAM_JR_DEBUG(" Set output ring base address to: Virtual: 0x%" PRIx64 + ",Physical: 0x%" PRIx64 ", Read from HW: 0x%" PRIx64, + (uint64_t)(uintptr_t)job_ring->output_ring, + caam_jr_dma_vtop(job_ring->output_ring), + hw_get_out_queue_base(job_ring)); + return ret; +} + +int +hw_shutdown_job_ring(struct sec_job_ring_t *job_ring) +{ + unsigned int timeout = SEC_TIMEOUT; + uint32_t tmp = 0; + int usleep_interval = 10; + + if (job_ring->register_base_addr == NULL) { + CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init", + job_ring); + return 0; + } + + CAAM_JR_INFO("Resetting Job ring %p", job_ring); + + /* + * Mask interrupts since we are going to poll + * for reset completion status + * Also, at POR, interrupts are ENABLED on a JR, thus + * this is the point where I can disable them without + * changing the code logic too much + */ + caam_jr_disable_irqs(job_ring->irq_fd); + + /* initiate flush (required prior to reset) */ + SET_JR_REG(JRCR, job_ring, JR_REG_JRCR_VAL_RESET); + + /* dummy read */ + tmp = GET_JR_REG(JRCR, job_ring); + + do { + tmp = GET_JR_REG(JRINT, job_ring); + usleep(usleep_interval); + } while (((tmp & JRINT_ERR_HALT_MASK) == + JRINT_ERR_HALT_INPROGRESS) && --timeout); + + CAAM_JR_INFO("JRINT is %x", tmp); + if ((tmp & JRINT_ERR_HALT_MASK) != JRINT_ERR_HALT_COMPLETE || + timeout == 0) { + CAAM_JR_ERR("0x%x, %d", tmp, timeout); + /* unmask interrupts */ + if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) + caam_jr_enable_irqs(job_ring->irq_fd); + return -1; + } + + /* Initiate reset */ + timeout = SEC_TIMEOUT; + SET_JR_REG(JRCR, job_ring, JR_REG_JRCR_VAL_RESET); + + do { + tmp = GET_JR_REG(JRCR, job_ring); + usleep(usleep_interval); + } while ((tmp & JR_REG_JRCR_VAL_RESET) && --timeout); + + CAAM_JR_DEBUG("JRCR is %x", tmp); + if (timeout == 0) { + CAAM_JR_ERR("Failed to reset hw job ring %p", job_ring); + /* unmask interrupts */ + if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) + caam_jr_enable_irqs(job_ring->irq_fd); + return -1; + } + /* unmask interrupts */ + if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) + caam_jr_enable_irqs(job_ring->irq_fd); + return 0; + +} + +void +hw_handle_job_ring_error(struct sec_job_ring_t *job_ring __rte_unused, + uint32_t error_code) +{ + union hw_error_code hw_err_code; + + hw_err_code.error = error_code; + switch (hw_err_code.error_desc.value.ssrc) { + case SEC_HW_ERR_SSRC_NO_SRC: + ASSERT(hw_err_code.error_desc.no_status_src.res == 0); + CAAM_JR_ERR("No Status Source "); + break; + case SEC_HW_ERR_SSRC_CCB_ERR: + CAAM_JR_ERR("CCB Status Source"); + hw_handle_ccb_err(hw_err_code); + break; + case SEC_HW_ERR_SSRC_JMP_HALT_U: + CAAM_JR_ERR("Jump Halt User Status Source"); + hw_handle_jmp_halt_user_err(hw_err_code); + break; + case SEC_HW_ERR_SSRC_DECO: + CAAM_JR_ERR("DECO Status Source"); + hw_handle_deco_err(hw_err_code); + break; + case SEC_HW_ERR_SSRC_JR: + CAAM_JR_ERR("Job Ring Status Source"); + hw_handle_jr_err(hw_err_code); + break; + case SEC_HW_ERR_SSRC_JMP_HALT_COND: + CAAM_JR_ERR("Jump Halt Condition Codes"); + hw_handle_jmp_halt_cond_err(hw_err_code); + break; + default: + ASSERT(0); + CAAM_JR_ERR("Unknown SSRC"); + break; + } +} + +void +hw_job_ring_error_print(struct sec_job_ring_t *job_ring, int code) +{ + switch (code) { + case JRINT_ERR_WRITE_STATUS: + CAAM_JR_ERR("Error writing status to Output Ring "); + break; + case JRINT_ERR_BAD_INPUT_BASE: + CAAM_JR_ERR( + "Bad Input Ring Base (%p) (not on a 4-byte boundary) ", + (void *)job_ring); + break; + case JRINT_ERR_BAD_OUTPUT_BASE: + CAAM_JR_ERR( + "Bad Output Ring Base (%p) (not on a 4-byte boundary) ", + (void *)job_ring); + break; + case JRINT_ERR_WRITE_2_IRBA: + CAAM_JR_ERR( + "Invalid write to Input Ring Base Address Register "); + break; + case JRINT_ERR_WRITE_2_ORBA: + CAAM_JR_ERR( + "Invalid write to Output Ring Base Address Register "); + break; + case JRINT_ERR_RES_B4_HALT: + CAAM_JR_ERR( + "Job Ring [%p] released before Job Ring is halted", + (void *)job_ring); + break; + case JRINT_ERR_REM_TOO_MANY: + CAAM_JR_ERR("Removed too many jobs from job ring [%p]", + (void *)job_ring); + break; + case JRINT_ERR_ADD_TOO_MANY: + CAAM_JR_ERR("Added too many jobs on job ring [%p]", job_ring); + break; + default: + CAAM_JR_ERR(" Unknown SEC JR Error :%d", + code); + break; + } +} + +int +hw_job_ring_set_coalescing_param(struct sec_job_ring_t *job_ring, + uint16_t irq_coalescing_timer, + uint8_t irq_coalescing_count) +{ + uint32_t reg_val = 0; + + ASSERT(job_ring != NULL); + if (job_ring->register_base_addr == NULL) { + CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init", + job_ring); + return -1; + } + /* Set descriptor count coalescing */ + reg_val |= (irq_coalescing_count << JR_REG_JRCFG_LO_ICDCT_SHIFT); + + /* Set coalescing timer value */ + reg_val |= (irq_coalescing_timer << JR_REG_JRCFG_LO_ICTT_SHIFT); + + /* Update parameters in HW */ + SET_JR_REG_LO(JRCFG, job_ring, reg_val); + CAAM_JR_DEBUG("Set coalescing params on jr %p timer:%d, desc count: %d", + job_ring, irq_coalescing_timer, irq_coalescing_timer); + + return 0; +} + +int +hw_job_ring_enable_coalescing(struct sec_job_ring_t *job_ring) +{ + uint32_t reg_val = 0; + + ASSERT(job_ring != NULL); + if (job_ring->register_base_addr == NULL) { + CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init", + job_ring); + return -1; + } + + /* Get the current value of the register */ + reg_val = GET_JR_REG_LO(JRCFG, job_ring); + + /* Enable coalescing */ + reg_val |= JR_REG_JRCFG_LO_ICEN_EN; + + /* Write in hw */ + SET_JR_REG_LO(JRCFG, job_ring, reg_val); + + CAAM_JR_DEBUG("Enabled coalescing on jr %p ", + job_ring); + + return 0; +} + +int +hw_job_ring_disable_coalescing(struct sec_job_ring_t *job_ring) +{ + uint32_t reg_val = 0; + + ASSERT(job_ring != NULL); + + if (job_ring->register_base_addr == NULL) { + CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init", + job_ring); + return -1; + } + + /* Get the current value of the register */ + reg_val = GET_JR_REG_LO(JRCFG, job_ring); + + /* Disable coalescing */ + reg_val &= ~JR_REG_JRCFG_LO_ICEN_EN; + + /* Write in hw */ + SET_JR_REG_LO(JRCFG, job_ring, reg_val); + CAAM_JR_DEBUG("Disabled coalescing on jr %p ", job_ring); + + return 0; +} diff --git a/drivers/crypto/caam_jr/caam_jr_hw_specific.h b/drivers/crypto/caam_jr/caam_jr_hw_specific.h new file mode 100644 index 00000000..5f58a585 --- /dev/null +++ b/drivers/crypto/caam_jr/caam_jr_hw_specific.h @@ -0,0 +1,503 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 NXP + */ + +#ifndef CAAM_JR_HW_SPECIFIC_H +#define CAAM_JR_HW_SPECIFIC_H + +#include + +/* + * Offset to the registers of a job ring. + * Is different for each job ring. + */ +#define CHAN_BASE(jr) ((size_t)(jr)->register_base_addr) + +#define SEC_JOB_RING_IS_FULL(pi, ci, ring_max_size, ring_threshold) \ + ((((pi) + 1 + ((ring_max_size) - (ring_threshold))) & \ + (ring_max_size - 1)) == ((ci))) + +#define SEC_CIRCULAR_COUNTER(x, max) (((x) + 1) & (max - 1)) + +/* + * Assert that cond is true. If !cond is true, display str and the vararg list + * in a printf-like syntax. also, if !cond is true, return altRet. + * + * \param cond A boolean expression to be asserted true + * \param altRet The value to be returned if cond doesn't hold true + * \param str A quoted char string + * + * E.g.: + * SEC_ASSERT(ret > 0, 0, "ERROR initializing app: code = %d\n", ret); + */ +#define SEC_ASSERT(cond, altRet, ...) do {\ + if (unlikely(!(cond))) {\ + CAAM_JR_ERR(__VA_ARGS__); \ + return altRet; \ + } \ +} while (0) + +#define SEC_DP_ASSERT(cond, altRet, ...) do {\ + if (unlikely(!(cond))) {\ + CAAM_JR_DP_ERR(__VA_ARGS__); \ + return altRet; \ + } \ +} while (0) + +#define ASSERT(x) + +/* + * Constants representing various job ring registers + */ +#if CAAM_BYTE_ORDER == __BIG_ENDIAN +#define JR_REG_IRBA_OFFSET 0x0000 +#define JR_REG_IRBA_OFFSET_LO 0x0004 +#else +#define JR_REG_IRBA_OFFSET 0x0004 +#define JR_REG_IRBA_OFFSET_LO 0x0000 +#endif + +#define JR_REG_IRSR_OFFSET 0x000C +#define JR_REG_IRSA_OFFSET 0x0014 +#define JR_REG_IRJA_OFFSET 0x001C + +#if CAAM_BYTE_ORDER == __BIG_ENDIAN +#define JR_REG_ORBA_OFFSET 0x0020 +#define JR_REG_ORBA_OFFSET_LO 0x0024 +#else +#define JR_REG_ORBA_OFFSET 0x0024 +#define JR_REG_ORBA_OFFSET_LO 0x0020 +#endif + +#define JR_REG_ORSR_OFFSET 0x002C +#define JR_REG_ORJR_OFFSET 0x0034 +#define JR_REG_ORSFR_OFFSET 0x003C +#define JR_REG_JROSR_OFFSET 0x0044 +#define JR_REG_JRINT_OFFSET 0x004C + +#define JR_REG_JRCFG_OFFSET 0x0050 +#define JR_REG_JRCFG_OFFSET_LO 0x0054 + +#define JR_REG_IRRI_OFFSET 0x005C +#define JR_REG_ORWI_OFFSET 0x0064 +#define JR_REG_JRCR_OFFSET 0x006C + +/* + * Constants for error handling on job ring + */ +#define JR_REG_JRINT_ERR_TYPE_SHIFT 8 +#define JR_REG_JRINT_ERR_ORWI_SHIFT 16 +#define JR_REG_JRINIT_JRE_SHIFT 1 + +#define JRINT_JRE (1 << JR_REG_JRINIT_JRE_SHIFT) +#define JRINT_ERR_WRITE_STATUS (1 << JR_REG_JRINT_ERR_TYPE_SHIFT) +#define JRINT_ERR_BAD_INPUT_BASE (3 << JR_REG_JRINT_ERR_TYPE_SHIFT) +#define JRINT_ERR_BAD_OUTPUT_BASE (4 << JR_REG_JRINT_ERR_TYPE_SHIFT) +#define JRINT_ERR_WRITE_2_IRBA (5 << JR_REG_JRINT_ERR_TYPE_SHIFT) +#define JRINT_ERR_WRITE_2_ORBA (6 << JR_REG_JRINT_ERR_TYPE_SHIFT) +#define JRINT_ERR_RES_B4_HALT (7 << JR_REG_JRINT_ERR_TYPE_SHIFT) +#define JRINT_ERR_REM_TOO_MANY (8 << JR_REG_JRINT_ERR_TYPE_SHIFT) +#define JRINT_ERR_ADD_TOO_MANY (9 << JR_REG_JRINT_ERR_TYPE_SHIFT) +#define JRINT_ERR_HALT_MASK 0x0C +#define JRINT_ERR_HALT_INPROGRESS 0x04 +#define JRINT_ERR_HALT_COMPLETE 0x08 + +#define JR_REG_JRCR_VAL_RESET 0x00000001 + +#define JR_REG_JRCFG_LO_ICTT_SHIFT 0x10 +#define JR_REG_JRCFG_LO_ICDCT_SHIFT 0x08 +#define JR_REG_JRCFG_LO_ICEN_EN 0x02 + +/* + * Constants for Descriptor Processing errors + */ +#define SEC_HW_ERR_SSRC_NO_SRC 0x00 +#define SEC_HW_ERR_SSRC_CCB_ERR 0x02 +#define SEC_HW_ERR_SSRC_JMP_HALT_U 0x03 +#define SEC_HW_ERR_SSRC_DECO 0x04 +#define SEC_HW_ERR_SSRC_JR 0x06 +#define SEC_HW_ERR_SSRC_JMP_HALT_COND 0x07 + +#define SEC_HW_ERR_DECO_HFN_THRESHOLD 0xF1 +#define SEC_HW_ERR_CCB_ICV_CHECK_FAIL 0x0A + +/* + * Constants for descriptors + */ +/* Return higher 32 bits of physical address */ +#define PHYS_ADDR_HI(phys_addr) \ + (uint32_t)(((uint64_t)phys_addr) >> 32) + +/* Return lower 32 bits of physical address */ +#define PHYS_ADDR_LO(phys_addr) \ + (uint32_t)(((uint64_t)phys_addr) & 0xFFFFFFFF) + +/* + * Macros for extracting error codes for the job ring + */ +#define JR_REG_JRINT_ERR_TYPE_EXTRACT(value) ((value) & 0x00000F00) +#define JR_REG_JRINT_ERR_ORWI_EXTRACT(value) \ + (((value) & 0x3FFF0000) >> JR_REG_JRINT_ERR_ORWI_SHIFT) +#define JR_REG_JRINT_JRE_EXTRACT(value) ((value) & JRINT_JRE) + +/* + * Macros for managing the job ring + */ +/* Read pointer to job ring input ring start address */ +#if defined(RTE_ARCH_ARM64) +#define hw_get_inp_queue_base(jr) ((((dma_addr_t)GET_JR_REG(IRBA, \ + (jr))) << 32) | \ + (GET_JR_REG_LO(IRBA, (jr)))) + +/* Read pointer to job ring output ring start address */ +#define hw_get_out_queue_base(jr) (((dma_addr_t)(GET_JR_REG(ORBA, \ + (jr))) << 32) | \ + (GET_JR_REG_LO(ORBA, (jr)))) +#else +#define hw_get_inp_queue_base(jr) ((dma_addr_t)(GET_JR_REG_LO(IRBA, (jr)))) + +#define hw_get_out_queue_base(jr) ((dma_addr_t)(GET_JR_REG_LO(ORBA, (jr)))) +#endif + +/* + * IRJA - Input Ring Jobs Added Register shows + * how many new jobs were added to the Input Ring. + */ +#define hw_enqueue_desc_on_job_ring(job_ring) SET_JR_REG(IRJA, (job_ring), 1) + +#define hw_set_input_ring_size(job_ring, size) SET_JR_REG(IRSR, job_ring, \ + (size)) + +#define hw_set_output_ring_size(job_ring, size) SET_JR_REG(ORSR, job_ring, \ + (size)) + +#if defined(RTE_ARCH_ARM64) +#define hw_set_input_ring_start_addr(job_ring, start_addr) \ +{ \ + SET_JR_REG(IRBA, job_ring, PHYS_ADDR_HI(start_addr)); \ + SET_JR_REG_LO(IRBA, job_ring, PHYS_ADDR_LO(start_addr));\ +} + +#define hw_set_output_ring_start_addr(job_ring, start_addr) \ +{ \ + SET_JR_REG(ORBA, job_ring, PHYS_ADDR_HI(start_addr)); \ + SET_JR_REG_LO(ORBA, job_ring, PHYS_ADDR_LO(start_addr));\ +} + +#else +#define hw_set_input_ring_start_addr(job_ring, start_addr) \ +{ \ + SET_JR_REG(IRBA, job_ring, 0); \ + SET_JR_REG_LO(IRBA, job_ring, PHYS_ADDR_LO(start_addr));\ +} + +#define hw_set_output_ring_start_addr(job_ring, start_addr) \ +{ \ + SET_JR_REG(ORBA, job_ring, 0); \ + SET_JR_REG_LO(ORBA, job_ring, PHYS_ADDR_LO(start_addr));\ +} +#endif + +/* ORJR - Output Ring Jobs Removed Register shows how many jobs were + * removed from the Output Ring for processing by software. This is done after + * the software has processed the entries. + */ +#define hw_remove_entries(jr, no_entries) SET_JR_REG(ORJR, (jr), (no_entries)) + +/* IRSA - Input Ring Slots Available register holds the number of entries in + * the Job Ring's input ring. Once a job is enqueued, the value returned is + * decremented by the hardware by the number of jobs enqueued. + */ +#define hw_get_available_slots(jr) GET_JR_REG(IRSA, jr) + +/* ORSFR - Output Ring Slots Full register holds the number of jobs which were + * processed by the SEC and can be retrieved by the software. Once a job has + * been processed by software, the user will call hw_remove_one_entry in order + * to notify the SEC that the entry was processed. + */ +#define hw_get_no_finished_jobs(jr) GET_JR_REG(ORSFR, jr) + +/* + * Macros for manipulating JR registers + */ +#if CORE_BYTE_ORDER == CAAM_BYTE_ORDER +#define sec_read_32(addr) (*(volatile unsigned int *)(addr)) +#define sec_write_32(addr, val) (*(volatile unsigned int *)(addr) = (val)) + +#else +#define sec_read_32(addr) rte_bswap32((*(volatile unsigned int *)(addr))) +#define sec_write_32(addr, val) \ + (*(volatile unsigned int *)(addr) = rte_bswap32(val)) +#endif + +#if CAAM_BYTE_ORDER == __LITTLE_ENDIAN +#define sec_read_64(addr) (((u64)sec_read_32((u32 *)(addr) + 1) << 32) | \ + (sec_read_32((u32 *)(addr)))) + +#define sec_write_64(addr, val) { \ + sec_write_32((u32 *)(addr) + 1, (u32)((val) >> 32)); \ + sec_write_32((u32 *)(addr), (u32)(val)); \ +} +#else /* CAAM_BYTE_ORDER == __BIG_ENDIAN */ +#define sec_read_64(addr) (((u64)sec_read_32((u32 *)(addr)) << 32) | \ + (sec_read_32((u32 *)(addr) + 1))) + +#define sec_write_64(addr, val) { \ + sec_write_32((u32 *)(addr), (u32)((val) >> 32)); \ + sec_write_32((u32 *)(addr) + 1, (u32)(val)); \ +} +#endif + +#if defined(RTE_ARCH_ARM64) +#define sec_read_addr(a) sec_read_64((a)) +#define sec_write_addr(a, v) sec_write_64((a), (v)) +#else +#define sec_read_addr(a) sec_read_32((a)) +#define sec_write_addr(a, v) sec_write_32((a), (v)) +#endif + +#define JR_REG(name, jr) (CHAN_BASE(jr) + JR_REG_##name##_OFFSET) +#define JR_REG_LO(name, jr) (CHAN_BASE(jr) + JR_REG_##name##_OFFSET_LO) + +#define GET_JR_REG(name, jr) (sec_read_32(JR_REG(name, (jr)))) +#define GET_JR_REG_LO(name, jr) (sec_read_32(JR_REG_LO(name, (jr)))) + +#define SET_JR_REG(name, jr, value) \ + (sec_write_32(JR_REG(name, (jr)), value)) +#define SET_JR_REG_LO(name, jr, value) \ + (sec_write_32(JR_REG_LO(name, (jr)), value)) + +/* Lists the possible states for a job ring. */ +typedef enum sec_job_ring_state_e { + SEC_JOB_RING_STATE_STARTED, /* Job ring is initialized */ + SEC_JOB_RING_STATE_RESET, /* Job ring reset is in progress */ +} sec_job_ring_state_t; + +/* code or cmd block to caam */ +struct sec_cdb { + struct { + union { + uint32_t word; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint16_t rsvd63_48; + unsigned int rsvd47_39:9; + unsigned int idlen:7; +#else + unsigned int idlen:7; + unsigned int rsvd47_39:9; + uint16_t rsvd63_48; +#endif + } field; + } __rte_packed hi; + + union { + uint32_t word; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + unsigned int rsvd31_30:2; + unsigned int fsgt:1; + unsigned int lng:1; + unsigned int offset:2; + unsigned int abs:1; + unsigned int add_buf:1; + uint8_t pool_id; + uint16_t pool_buffer_size; +#else + uint16_t pool_buffer_size; + uint8_t pool_id; + unsigned int add_buf:1; + unsigned int abs:1; + unsigned int offset:2; + unsigned int lng:1; + unsigned int fsgt:1; + unsigned int rsvd31_30:2; +#endif + } field; + } __rte_packed lo; + } __rte_packed sh_hdr; + + uint32_t sh_desc[SEC_JOB_DESCRIPTOR_SIZE]; +}; + +struct caam_jr_qp { + struct sec_job_ring_t *ring; + uint64_t rx_pkts; + uint64_t rx_errs; + uint64_t rx_poll_err; + uint64_t tx_pkts; + uint64_t tx_errs; + uint64_t tx_ring_full; +}; + +struct sec_job_ring_t { + /* TODO: Add wrapper macro to make it obvious this is the consumer index + * on the output ring + */ + uint32_t cidx; /* Consumer index for job ring (jobs array). + * @note: cidx and pidx are accessed from + * different threads. Place the cidx and pidx + * inside the structure so that they lay on + * different cachelines, to avoid false sharing + * between threads when the threads run on + * different cores! + */ + /* TODO: Add wrapper macro to make it obvious this is the producer index + * on the input ring + */ + uint32_t pidx; /* Producer index for job ring (jobs array) */ + + phys_addr_t *input_ring;/* Ring of output descriptors received from SEC. + * Size of array is power of 2 to allow fast + * update of producer/consumer indexes with + * bitwise operations. + */ + + struct sec_outring_entry *output_ring; + /* Ring of output descriptors received from SEC. + * Size of array is power of 2 to allow fast + * update of producer/consumer indexes with + * bitwise operations. + */ + + uint32_t irq_fd; /* The file descriptor used for polling from + * user space for interrupts notifications + */ + uint32_t jr_mode; /* Model used by SEC Driver to receive + * notifications from SEC. Can be either + * of the three: #SEC_NOTIFICATION_TYPE_NAPI + * #SEC_NOTIFICATION_TYPE_IRQ or + * #SEC_NOTIFICATION_TYPE_POLL + */ + uint32_t napi_mode; /* Job ring mode if NAPI mode is chosen + * Used only when jr_mode is set to + * #SEC_NOTIFICATION_TYPE_NAPI + */ + void *register_base_addr; /* Base address for SEC's + * register memory for this job ring. + */ + uint8_t coalescing_en; /* notifies if coelescing is + * enabled for the job ring + */ + sec_job_ring_state_t jr_state; /* The state of this job ring */ + + struct rte_mempool *ctx_pool; /* per dev mempool for caam_jr_op_ctx */ + unsigned int max_nb_queue_pairs; + unsigned int max_nb_sessions; + struct caam_jr_qp qps[RTE_CAAM_MAX_NB_SEC_QPS]; /* i/o queue for sec */ +}; + +/* Union describing the possible error codes that + * can be set in the descriptor status word + */ +union hw_error_code { + uint32_t error; + union { + struct { + uint32_t ssrc:4; + uint32_t ssed_val:28; + } __rte_packed value; + struct { + uint32_t ssrc:4; + uint32_t res:28; + } __rte_packed no_status_src; + struct { + uint32_t ssrc:4; + uint32_t jmp:1; + uint32_t res:11; + uint32_t desc_idx:8; + uint32_t cha_id:4; + uint32_t err_id:4; + } __rte_packed ccb_status_src; + struct { + uint32_t ssrc:4; + uint32_t jmp:1; + uint32_t res:11; + uint32_t desc_idx:8; + uint32_t offset:8; + } __rte_packed jmp_halt_user_src; + struct { + uint32_t ssrc:4; + uint32_t jmp:1; + uint32_t res:11; + uint32_t desc_idx:8; + uint32_t desc_err:8; + } __rte_packed deco_src; + struct { + uint32_t ssrc:4; + uint32_t res:17; + uint32_t naddr:3; + uint32_t desc_err:8; + } __rte_packed jr_src; + struct { + uint32_t ssrc:4; + uint32_t jmp:1; + uint32_t res:11; + uint32_t desc_idx:8; + uint32_t cond:8; + } __rte_packed jmp_halt_cond_src; + } __rte_packed error_desc; +} __rte_packed; + +/* @brief Initialize a job ring/channel in SEC device. + * Write configuration register/s to properly initialize a job ring. + * + * @param [in] job_ring The job ring + * + * @retval 0 for success + * @retval other for error + */ +int hw_reset_job_ring(struct sec_job_ring_t *job_ring); + +/* @brief Reset a job ring/channel in SEC device. + * Write configuration register/s to reset a job ring. + * + * @param [in] job_ring The job ring + * + * @retval 0 for success + * @retval -1 in case job ring reset failed + */ +int hw_shutdown_job_ring(struct sec_job_ring_t *job_ring); + +/* @brief Handle a job ring/channel error in SEC device. + * Identify the error type and clear error bits if required. + * + * @param [in] job_ring The job ring + * @param [in] sec_error_code The job ring's error code + */ +void hw_handle_job_ring_error(struct sec_job_ring_t *job_ring, + uint32_t sec_error_code); + +/* @brief Handle a job ring error in the device. + * Identify the error type and printout a explanatory + * messages. + * + * @param [in] job_ring The job ring + * + */ +void hw_job_ring_error_print(struct sec_job_ring_t *job_ring, int code); + +/* @brief Set interrupt coalescing parameters on the Job Ring. + * @param [in] job_ring The job ring + * @param [in] irq_coalesing_timer Interrupt coalescing timer threshold. + * This value determines the maximum + * amount of time after processing a + * descriptor before raising an interrupt. + * @param [in] irq_coalescing_count Interrupt coalescing descriptor count + * threshold. + */ +int hw_job_ring_set_coalescing_param(struct sec_job_ring_t *job_ring, + uint16_t irq_coalescing_timer, + uint8_t irq_coalescing_count); + +/* @brief Enable interrupt coalescing on a job ring + * @param [in] job_ring The job ring + */ +int hw_job_ring_enable_coalescing(struct sec_job_ring_t *job_ring); + +/* @brief Disable interrupt coalescing on a job ring + * @param [in] job_ring The job ring + */ +int hw_job_ring_disable_coalescing(struct sec_job_ring_t *job_ring); + +#endif /* CAAM_JR_HW_SPECIFIC_H */ diff --git a/drivers/crypto/caam_jr/caam_jr_log.h b/drivers/crypto/caam_jr/caam_jr_log.h new file mode 100644 index 00000000..106ff07a --- /dev/null +++ b/drivers/crypto/caam_jr/caam_jr_log.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017-2018 NXP + */ + +#ifndef _CAAM_JR_LOG_H_ +#define _CAAM_JR_LOG_H_ + +#include + +extern int caam_jr_logtype; + +#define CAAM_JR_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, caam_jr_logtype, "caam_jr: " \ + fmt "\n", ##args) + +#define CAAM_JR_DEBUG(fmt, args...) \ + rte_log(RTE_LOG_DEBUG, caam_jr_logtype, "caam_jr: %s(): " \ + fmt "\n", __func__, ##args) + +#define PMD_INIT_FUNC_TRACE() CAAM_JR_DEBUG(" >>") + +#define CAAM_JR_INFO(fmt, args...) \ + CAAM_JR_LOG(INFO, fmt, ## args) +#define CAAM_JR_ERR(fmt, args...) \ + CAAM_JR_LOG(ERR, fmt, ## args) +#define CAAM_JR_WARN(fmt, args...) \ + CAAM_JR_LOG(WARNING, fmt, ## args) + +/* DP Logs, toggled out at compile time if level lower than current level */ +#define CAAM_JR_DP_LOG(level, fmt, args...) \ + RTE_LOG_DP(level, PMD, fmt "\n", ## args) + +#define CAAM_JR_DP_DEBUG(fmt, args...) \ + CAAM_JR_DP_LOG(DEBUG, fmt, ## args) +#define CAAM_JR_DP_INFO(fmt, args...) \ + CAAM_JR_DP_LOG(INFO, fmt, ## args) +#define CAAM_JR_DP_WARN(fmt, args...) \ + CAAM_JR_DP_LOG(WARNING, fmt, ## args) +#define CAAM_JR_DP_ERR(fmt, args...) \ + CAAM_JR_DP_LOG(ERR, fmt, ## args) + +#endif /* _CAAM_JR_LOG_H_ */ diff --git a/drivers/crypto/caam_jr/caam_jr_pvt.h b/drivers/crypto/caam_jr/caam_jr_pvt.h new file mode 100644 index 00000000..9f1adabc --- /dev/null +++ b/drivers/crypto/caam_jr/caam_jr_pvt.h @@ -0,0 +1,291 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017-2018 NXP + */ + +#ifndef CAAM_JR_PVT_H +#define CAAM_JR_PVT_H + +#include + +/* NXP CAAM JR PMD device name */ + +#define CAAM_JR_ALG_UNSUPPORT (-1) + +/* Minimum job descriptor consists of a oneword job descriptor HEADER and + * a pointer to the shared descriptor. + */ +#define MIN_JOB_DESC_SIZE (CAAM_CMD_SZ + CAAM_PTR_SZ) +#define CAAM_JOB_DESC_SIZE 13 + +/* CTX_POOL_NUM_BUFS is set as per the ipsec-secgw application */ +#define CTX_POOL_NUM_BUFS 32000 +#define CTX_POOL_CACHE_SIZE 512 + +#define DIR_ENC 1 +#define DIR_DEC 0 + +#define JR_MAX_NB_MAX_DIGEST 32 + +#define RTE_CAAM_JR_PMD_MAX_NB_SESSIONS 2048 + + +/* Return codes for SEC user space driver APIs */ +enum sec_return_code_e { + SEC_SUCCESS = 0, /* Operation executed successfully.*/ + SEC_INVALID_INPUT_PARAM, /* API received an invalid input + * parameter + */ + SEC_OUT_OF_MEMORY, /* Memory allocation failed. */ + SEC_DESCRIPTOR_IN_FLIGHT, /* API function indicates there are + * descriptors in flight + * for SEC to process. + */ + SEC_LAST_DESCRIPTOR_IN_FLIGHT, /* API function indicates there is one + * last descriptor in flight + * for SEC to process that. + */ + SEC_PROCESSING_ERROR, /* Indicates a SEC processing error + * occurred on a Job Ring which requires + * a SEC user space driver shutdown. Can + * be returned from sec_poll_job_ring(). + * Then the only other API that can be + * called after this error is + * sec_release(). + */ + SEC_DESC_PROCESSING_ERROR, /* Indicates a SEC descriptor processing + * error occurred on a Job Ring. Can be + * returned from sec_poll_job_ring(). + * The driver was able to reset job ring + * and job ring can be used like in a + * normal case. + */ + SEC_JR_IS_FULL, /* Job Ring is full. There is no more + * room in the JR for new descriptors. + * This can happen if the descriptor RX + * rate is higher than SEC's capacity. + */ + SEC_DRIVER_RELEASE_IN_PROGRESS, /* SEC driver shutdown is in progress, + * descriptors processing or polling is + * allowed. + */ + SEC_DRIVER_ALREADY_INITIALIZED, /* SEC driver is already initialized.*/ + SEC_DRIVER_NOT_INITIALIZED, /* SEC driver is NOT initialized. */ + SEC_JOB_RING_RESET_IN_PROGRESS, /* Job ring is resetting due to a + * per-descriptor SEC processing error + * ::SEC_desc_PROCESSING_ERROR. Reset is + * finished when sec_poll_job_ring() + * return. Then the job ring can be used + * again. + */ + SEC_RESET_ENGINE_FAILED, /* Resetting of SEC Engine by SEC Kernel + * Driver Failed + */ + SEC_ENABLE_IRQS_FAILED, /* Enabling of IRQs in SEC Kernel Driver + * Failed + */ + SEC_DISABLE_IRQS_FAILED, /* Disabling of IRQs in SEC Kernel + * Driver Failed + */ + /* END OF VALID VALUES */ + + SEC_RETURN_CODE_MAX_VALUE, /* Invalid value for return code. It is + * used to mark the end of the return + * code values. @note ALL new return + * code values MUST be added before + * ::SEC_RETURN_CODE_MAX_VALUE! + */ +}; + +enum caam_jr_op_type { + CAAM_JR_NONE, /* No Cipher operations*/ + CAAM_JR_CIPHER,/* CIPHER operations */ + CAAM_JR_AUTH, /* Authentication Operations */ + CAAM_JR_AEAD, /* Authenticated Encryption with associated data */ + CAAM_JR_IPSEC, /* IPSEC protocol operations*/ + CAAM_JR_PDCP, /* PDCP protocol operations*/ + CAAM_JR_PKC, /* Public Key Cryptographic Operations */ + CAAM_JR_MAX +}; + +struct caam_jr_session { + uint8_t dir; /* Operation Direction */ + enum rte_crypto_cipher_algorithm cipher_alg; /* Cipher Algorithm*/ + enum rte_crypto_auth_algorithm auth_alg; /* Authentication Algorithm*/ + enum rte_crypto_aead_algorithm aead_alg; /* AEAD Algorithm*/ + enum rte_security_session_protocol proto_alg; /* Security Algorithm*/ + union { + struct { + uint8_t *data; /* pointer to key data */ + size_t length; /* key length in bytes */ + } aead_key; + struct { + struct { + uint8_t *data; /* pointer to key data */ + size_t length; /* key length in bytes */ + } cipher_key; + struct { + uint8_t *data; /* pointer to key data */ + size_t length; /* key length in bytes */ + } auth_key; + }; + }; + struct { + uint16_t length; + uint16_t offset; + } iv; /* Initialisation vector parameters */ + uint16_t auth_only_len; /* Length of data for Auth only */ + uint32_t digest_length; + struct ipsec_encap_pdb encap_pdb; + struct ip ip4_hdr; + struct ipsec_decap_pdb decap_pdb; + struct caam_jr_qp *qp; + struct sec_cdb *cdb; /* cmd block associated with qp */ + struct rte_mempool *ctx_pool; /* session mempool for caam_jr_op_ctx */ +}; + +/* + * 16-byte hardware scatter/gather table + */ + +#define SEC4_SG_LEN_EXT 0x80000000 /* Entry points to table */ +#define SEC4_SG_LEN_FIN 0x40000000 /* Last ent in table */ +#define SEC4_SG_BPID_MASK 0x000000ff +#define SEC4_SG_BPID_SHIFT 16 +#define SEC4_SG_LEN_MASK 0x3fffffff /* Excludes EXT and FINAL */ +#define SEC4_SG_OFFSET_MASK 0x00001fff + +struct sec4_sg_entry { + uint64_t ptr; + uint32_t len; + uint32_t bpid_offset; +}; + +#define MAX_SG_ENTRIES 16 +#define SG_CACHELINE_0 0 +#define SG_CACHELINE_1 4 +#define SG_CACHELINE_2 8 +#define SG_CACHELINE_3 12 + +/* Structure encompassing a job descriptor which is to be processed + * by SEC. User should also initialise this structure with the callback + * function pointer which will be called by driver after recieving proccessed + * descriptor from SEC. User data is also passed in this data structure which + * will be sent as an argument to the user callback function. + */ +struct job_descriptor { + uint32_t desc[CAAM_JOB_DESC_SIZE]; +}; + +struct caam_jr_op_ctx { + struct job_descriptor jobdes; + /* sg[0] output, sg[1] input, others are possible sub frames */ + struct sec4_sg_entry sg[MAX_SG_ENTRIES]; + struct rte_crypto_op *op; + struct rte_mempool *ctx_pool; /* mempool pointer for caam_jr_op_ctx */ + int64_t vtop_offset; + uint8_t digest[JR_MAX_NB_MAX_DIGEST]; +}; + +/** + * Checksum + * + * @param buffer calculate chksum for buffer + * @param len buffer length + * + * @return checksum value in host cpu order + */ +static inline uint16_t +calc_chksum(void *buffer, int len) +{ + uint16_t *buf = (uint16_t *)buffer; + uint32_t sum = 0; + uint16_t result; + + for (sum = 0; len > 1; len -= 2) + sum += *buf++; + + if (len == 1) + sum += *(unsigned char *)buf; + + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + result = ~sum; + + return result; +} +struct uio_job_ring { + uint32_t jr_id; + uint32_t uio_fd; + void *register_base_addr; + int map_size; + int uio_minor_number; +}; + +int sec_cleanup(void); +int sec_configure(void); +struct uio_job_ring *config_job_ring(void); +void free_job_ring(uint32_t uio_fd); + +/* For Dma memory allocation of specified length and alignment */ +static inline void * +caam_jr_dma_mem_alloc(size_t align, size_t len) +{ + return rte_malloc("mem_alloc", len, align); +} + +/* For freeing dma memory */ +static inline void +caam_jr_dma_free(void *ptr) +{ + rte_free(ptr); +} + +static inline rte_iova_t +caam_jr_mem_vtop(void *vaddr) +{ + const struct rte_memseg *ms; + + ms = rte_mem_virt2memseg(vaddr, NULL); + if (ms) + return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr); + return (size_t)NULL; +} + +static inline void * +caam_jr_dma_ptov(rte_iova_t paddr) +{ + return rte_mem_iova2virt(paddr); +} + +/* Virtual to physical address conversion */ +static inline rte_iova_t caam_jr_dma_vtop(void *ptr) +{ + return caam_jr_mem_vtop(ptr); +} + +/** @brief Request to SEC kernel driver to enable interrupts for + * descriptor finished processing + * Use UIO to communicate with SEC kernel driver: write command + * value that indicates an IRQ enable action into UIO file descriptor + * of this job ring. + * + * @param [in] uio_fd Job Ring UIO File descriptor + * @retval 0 for success + * @retval -1 value for error + */ +uint32_t caam_jr_enable_irqs(uint32_t uio_fd); + +/** @brief Request to SEC kernel driver to disable interrupts for descriptor + * finished processing + * Use UIO to communicate with SEC kernel driver: write command + * value that indicates an IRQ disable action into UIO file descriptor + * of this job ring. + * + * @param [in] uio_fd UIO File descripto + * @retval 0 for success + * @retval -1 value for error + * + */ +uint32_t caam_jr_disable_irqs(uint32_t uio_fd); + +#endif diff --git a/drivers/crypto/caam_jr/caam_jr_uio.c b/drivers/crypto/caam_jr/caam_jr_uio.c new file mode 100644 index 00000000..c07d9db0 --- /dev/null +++ b/drivers/crypto/caam_jr/caam_jr_uio.c @@ -0,0 +1,501 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017-2018 NXP + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +/* RTA header files */ +#include +#include +#include + +/* Prefix path to sysfs directory where UIO device attributes are exported. + * Path for UIO device X is /sys/class/uio/uioX + */ +#define SEC_UIO_DEVICE_SYS_ATTR_PATH "/sys/class/uio" + +/* Subfolder in sysfs where mapping attributes are exported + * for each UIO device. Path for mapping Y for device X is: + * /sys/class/uio/uioX/maps/mapY + */ +#define SEC_UIO_DEVICE_SYS_MAP_ATTR "maps/map" + +/* Name of UIO device file prefix. Each UIO device will have a device file + * /dev/uioX, where X is the minor device number. + */ +#define SEC_UIO_DEVICE_FILE_NAME "/dev/uio" + +/* + * Name of UIO device. Each user space SEC job ring will have a corresponding + * UIO device with the name sec-channelX, where X is the job ring id. + * Maximum length is #SEC_UIO_MAX_DEVICE_NAME_LENGTH. + * + * @note Must be kept in synch with SEC kernel driver + * define #SEC_UIO_DEVICE_NAME ! + */ +#define SEC_UIO_DEVICE_NAME "fsl-jr" + +/* Maximum length for the name of an UIO device file. + * Device file name format is: /dev/uioX. + */ +#define SEC_UIO_MAX_DEVICE_FILE_NAME_LENGTH 30 + +/* Maximum length for the name of an attribute file for an UIO device. + * Attribute files are exported in sysfs and have the name formatted as: + * /sys/class/uio/uioX/ + */ +#define SEC_UIO_MAX_ATTR_FILE_NAME 100 + +/* Command that is used by SEC user space driver and SEC kernel driver + * to signal a request from the former to the later to disable job DONE + * and error IRQs on a certain job ring. + * The configuration is done at SEC Controller's level. + * @note Need to be kept in synch with #SEC_UIO_DISABLE_IRQ_CMD from + * linux/drivers/crypto/talitos.c ! + */ +#define SEC_UIO_DISABLE_IRQ_CMD 0 + +/* Command that is used by SEC user space driver and SEC kernel driver + * to signal a request from the former to the later to enable job DONE + * and error IRQs on a certain job ring. + * The configuration is done at SEC Controller's level. + * @note Need to be kept in synch with #SEC_UIO_ENABLE_IRQ_CMD from + * linux/drivers/crypto/talitos.c ! + */ +#define SEC_UIO_ENABLE_IRQ_CMD 1 + +/** Command that is used by SEC user space driver and SEC kernel driver + * to signal a request from the former to the later to do a SEC engine reset. + * @note Need to be kept in synch with #SEC_UIO_RESET_SEC_ENGINE_CMD from + * linux/drivers/crypto/talitos.c ! + */ +#define SEC_UIO_RESET_SEC_ENGINE_CMD 3 + +/* The id for the mapping used to export SEC's registers to + * user space through UIO devices. + */ +#define SEC_UIO_MAP_ID 0 + +static struct uio_job_ring g_uio_job_ring[MAX_SEC_JOB_RINGS]; +static int g_uio_jr_num; + +/** @brief Checks if a file name contains a certain substring. + * If so, it extracts the number following the substring. + * This function assumes a filename format of: [text][number]. + * @param [in] filename File name + * @param [in] match String to match in file name + * @param [out] number The number extracted from filename + * + * @retval true if file name matches the criteria + * @retval false if file name does not match the criteria + */ +static bool +file_name_match_extract(const char filename[], const char match[], int *number) +{ + char *substr = NULL; + + substr = strstr(filename, match); + if (substr == NULL) + return false; + + /* substring was found in + * read number following substring in + */ + if (sscanf(filename + strlen(match), "%d", number) <= 0) + return false; + + return true; +} + +/** @brief Reads first line from a file. + * Composes file name as: root/subdir/filename + * + * @param [in] root Root path + * @param [in] subdir Subdirectory name + * @param [in] filename File name + * @param [out] line The first line read from file. + * + * @retval 0 for succes + * @retval other value for error + */ +static int +file_read_first_line(const char root[], const char subdir[], + const char filename[], char *line) +{ + char absolute_file_name[SEC_UIO_MAX_ATTR_FILE_NAME]; + int fd = 0, ret = 0; + + /*compose the file name: root/subdir/filename */ + memset(absolute_file_name, 0, sizeof(absolute_file_name)); + snprintf(absolute_file_name, SEC_UIO_MAX_ATTR_FILE_NAME, + "%s/%s/%s", root, subdir, filename); + + fd = open(absolute_file_name, O_RDONLY); + SEC_ASSERT(fd > 0, fd, "Error opening file %s", + absolute_file_name); + + /* read UIO device name from first line in file */ + ret = read(fd, line, SEC_UIO_MAX_DEVICE_FILE_NAME_LENGTH); + close(fd); + + /* NULL-ify string */ + line[SEC_UIO_MAX_DEVICE_FILE_NAME_LENGTH - 1] = '\0'; + + if (ret <= 0) { + CAAM_JR_ERR("Error reading from file %s", absolute_file_name); + return ret; + } + + return 0; +} + +/** @brief Uses UIO control to send commands to SEC kernel driver. + * The mechanism is to write a command word into the file descriptor + * that the user-space driver obtained for each user-space SEC job ring. + * Both user-space driver and kernel driver must have the same understanding + * about the command codes. + * + * @param [in] UIO FD The UIO file descriptor + * @param [in] uio_command Command word + * + * @retval Result of write operation on the job ring's UIO file descriptor. + * Should be sizeof(int) for success operations. + * Other values can be returned and used, if desired to add special + * meaning to return values, but this has to be programmed in SEC + * kernel driver as well. No special return values are used. + */ +static int +sec_uio_send_command(uint32_t uio_fd, int32_t uio_command) +{ + int ret; + + /* Use UIO file descriptor we have for this job ring. + * Writing a command code to this file descriptor will make the + * SEC kernel driver execute the desired command. + */ + ret = write(uio_fd, &uio_command, sizeof(int)); + return ret; +} + +/** @brief Request to SEC kernel driver to enable interrupts for + * descriptor finished processing + * Use UIO to communicate with SEC kernel driver: write command + * value that indicates an IRQ enable action into UIO file descriptor + * of this job ring. + * + * @param [in] uio_fd Job Ring UIO File descriptor + * @retval 0 for success + * @retval -1 value for error + */ +uint32_t +caam_jr_enable_irqs(uint32_t uio_fd) +{ + int ret; + + /* Use UIO file descriptor we have for this job ring. + * Writing a command code to this file descriptor will make the + * SEC kernel driver enable DONE and Error IRQs for this job ring, + * at Controller level. + */ + ret = sec_uio_send_command(uio_fd, SEC_UIO_ENABLE_IRQ_CMD); + SEC_ASSERT(ret == sizeof(int), -1, + "Failed to request SEC engine to enable job done and " + "error IRQs through UIO control. UIO FD %d. Reset SEC driver!", + uio_fd); + CAAM_JR_DEBUG("Enabled IRQs on jr with uio_fd %d", uio_fd); + return 0; +} + + +/** @brief Request to SEC kernel driver to disable interrupts for descriptor + * finished processing + * Use UIO to communicate with SEC kernel driver: write command + * value that indicates an IRQ disable action into UIO file descriptor + * of this job ring. + * + * @param [in] uio_fd UIO File descripto + * @retval 0 for success + * @retval -1 value for error + * + */ +uint32_t +caam_jr_disable_irqs(uint32_t uio_fd) +{ + int ret; + + /* Use UIO file descriptor we have for this job ring. + * Writing a command code to this file descriptor will make the + * SEC kernel driver disable IRQs for this job ring, + * at Controller level. + */ + + ret = sec_uio_send_command(uio_fd, SEC_UIO_DISABLE_IRQ_CMD); + SEC_ASSERT(ret == sizeof(int), -1, + "Failed to request SEC engine to disable job done and " + "IRQs through UIO control. UIO_FD %d Reset SEC driver!", + uio_fd); + CAAM_JR_DEBUG("Disabled IRQs on jr with uio_fd %d", uio_fd); + return 0; +} + +/** @brief Maps register range assigned for a job ring. + * + * @param [in] uio_device_fd UIO device file descriptor + * @param [in] uio_device_id UIO device id + * @param [in] uio_map_id UIO allows maximum 5 different mapping for + each device. Maps start with id 0. + * @param [out] map_size Map size. + * @retval NULL if failed to map registers + * @retval Virtual address for mapped register address range + */ +static void * +uio_map_registers(int uio_device_fd, int uio_device_id, + int uio_map_id, int *map_size) +{ + void *mapped_address = NULL; + unsigned int uio_map_size = 0; + char uio_sys_root[SEC_UIO_MAX_ATTR_FILE_NAME]; + char uio_sys_map_subdir[SEC_UIO_MAX_ATTR_FILE_NAME]; + char uio_map_size_str[32]; + int ret = 0; + + /* compose the file name: root/subdir/filename */ + memset(uio_sys_root, 0, sizeof(uio_sys_root)); + memset(uio_sys_map_subdir, 0, sizeof(uio_sys_map_subdir)); + memset(uio_map_size_str, 0, sizeof(uio_map_size_str)); + + /* Compose string: /sys/class/uio/uioX */ + sprintf(uio_sys_root, "%s/%s%d", SEC_UIO_DEVICE_SYS_ATTR_PATH, + "uio", uio_device_id); + /* Compose string: maps/mapY */ + sprintf(uio_sys_map_subdir, "%s%d", SEC_UIO_DEVICE_SYS_MAP_ATTR, + uio_map_id); + + /* Read first (and only) line from file + * /sys/class/uio/uioX/maps/mapY/size + */ + ret = file_read_first_line(uio_sys_root, uio_sys_map_subdir, + "size", uio_map_size_str); + SEC_ASSERT(ret == 0, NULL, "file_read_first_line() failed"); + + /* Read mapping size, expressed in hexa(base 16) */ + uio_map_size = strtol(uio_map_size_str, NULL, 16); + + /* Map the region in user space */ + mapped_address = mmap(0, /*dynamically choose virtual address */ + uio_map_size, PROT_READ | PROT_WRITE, + MAP_SHARED, uio_device_fd, 0); + /* offset = 0 because UIO device has only one mapping + * for the entire SEC register memory + */ + if (mapped_address == MAP_FAILED) { + CAAM_JR_ERR( + "Failed to map registers! errno = %d job ring fd = %d," + "uio device id = %d, uio map id = %d", errno, + uio_device_fd, uio_device_id, uio_map_id); + return NULL; + } + + /* + * Save the map size to use it later on for munmap-ing. + */ + *map_size = uio_map_size; + + CAAM_JR_INFO("UIO dev[%d] mapped region [id =%d] size 0x%x at %p", + uio_device_id, uio_map_id, uio_map_size, mapped_address); + + return mapped_address; +} + +void +free_job_ring(uint32_t uio_fd) +{ + struct uio_job_ring *job_ring = NULL; + int i; + + if (!job_ring->uio_fd) + return; + + for (i = 0; i < MAX_SEC_JOB_RINGS; i++) { + if (g_uio_job_ring[i].uio_fd == uio_fd) { + job_ring = &g_uio_job_ring[i]; + break; + } + } + + if (job_ring == NULL) { + CAAM_JR_ERR("JR not available for fd = %x\n", uio_fd); + return; + } + + /* Open device file */ + CAAM_JR_INFO("Closed device file for job ring %d , fd = %d", + job_ring->jr_id, job_ring->uio_fd); + close(job_ring->uio_fd); + g_uio_jr_num--; + job_ring->uio_fd = 0; + if (job_ring->register_base_addr == NULL) + return; + + /* Unmap the PCI memory resource of device */ + if (munmap(job_ring->register_base_addr, job_ring->map_size)) { + CAAM_JR_INFO("cannot munmap(%p, 0x%lx): %s", + job_ring->register_base_addr, + (unsigned long)job_ring->map_size, strerror(errno)); + } else + CAAM_JR_DEBUG(" JR UIO memory unmapped at %p", + job_ring->register_base_addr); + job_ring->register_base_addr = NULL; +} + +struct +uio_job_ring *config_job_ring(void) +{ + char uio_device_file_name[32]; + struct uio_job_ring *job_ring = NULL; + int i; + + for (i = 0; i < MAX_SEC_JOB_RINGS; i++) { + if (g_uio_job_ring[i].uio_fd == 0) { + job_ring = &g_uio_job_ring[i]; + g_uio_jr_num++; + break; + } + } + + if (job_ring == NULL) { + CAAM_JR_ERR("No free job ring\n"); + return NULL; + } + + /* Find UIO device created by SEC kernel driver for this job ring. */ + memset(uio_device_file_name, 0, sizeof(uio_device_file_name)); + + sprintf(uio_device_file_name, "%s%d", SEC_UIO_DEVICE_FILE_NAME, + job_ring->uio_minor_number); + + /* Open device file */ + job_ring->uio_fd = open(uio_device_file_name, O_RDWR); + SEC_ASSERT(job_ring->uio_fd > 0, NULL, + "Failed to open UIO device file for job ring %d", + job_ring->jr_id); + + CAAM_JR_INFO("Open device(%s) file for job ring=%d , uio_fd = %d", + uio_device_file_name, job_ring->jr_id, job_ring->uio_fd); + + ASSERT(job_ring->register_base_addr == NULL); + job_ring->register_base_addr = uio_map_registers( + job_ring->uio_fd, job_ring->uio_minor_number, + SEC_UIO_MAP_ID, &job_ring->map_size); + + SEC_ASSERT(job_ring->register_base_addr != NULL, NULL, + "Failed to map SEC registers"); + return job_ring; +} + +int +sec_configure(void) +{ + char uio_name[32]; + int config_jr_no = 0, jr_id = -1; + int uio_minor_number = -1; + int ret; + DIR *d = NULL; + struct dirent *dir; + + d = opendir(SEC_UIO_DEVICE_SYS_ATTR_PATH); + if (d == NULL) { + printf("\nError opening directory '%s': %s\n", + SEC_UIO_DEVICE_SYS_ATTR_PATH, strerror(errno)); + return -1; + } + + /* Iterate through all subdirs */ + while ((dir = readdir(d)) != NULL) { + if (!strncmp(dir->d_name, ".", 1) || + !strncmp(dir->d_name, "..", 2)) + continue; + + if (file_name_match_extract + (dir->d_name, "uio", &uio_minor_number)) { + /* + * Open file uioX/name and read first line which contains + * the name for the device. Based on the name check if this + * UIO device is UIO device for job ring with id jr_id. + */ + memset(uio_name, 0, sizeof(uio_name)); + ret = file_read_first_line(SEC_UIO_DEVICE_SYS_ATTR_PATH, + dir->d_name, "name", uio_name); + CAAM_JR_INFO("sec device uio name: %s", uio_name); + SEC_ASSERT(ret == 0, -1, "file_read_first_line failed"); + + if (file_name_match_extract(uio_name, + SEC_UIO_DEVICE_NAME, + &jr_id)) { + g_uio_job_ring[config_jr_no].jr_id = jr_id; + g_uio_job_ring[config_jr_no].uio_minor_number = + uio_minor_number; + CAAM_JR_INFO("Detected logical JRID:%d", jr_id); + config_jr_no++; + + /* todo find the actual ring id + * OF_FULLNAME=/soc/crypto@1700000/jr@20000 + */ + } + } + } + closedir(d); + + if (config_jr_no == 0) { + CAAM_JR_ERR("! No SEC Job Rings assigned for userspace usage!"); + return 0; + } + CAAM_JR_INFO("Total JR detected =%d", config_jr_no); + return config_jr_no; +} + +int +sec_cleanup(void) +{ + int i; + struct uio_job_ring *job_ring; + + for (i = 0; i < g_uio_jr_num; i++) { + job_ring = &g_uio_job_ring[i]; + /* munmap SEC's register memory */ + if (job_ring->register_base_addr) { + munmap(job_ring->register_base_addr, + job_ring->map_size); + job_ring->register_base_addr = NULL; + } + /* I need to close the fd after shutdown UIO commands need to be + * sent using the fd + */ + if (job_ring->uio_fd != 0) { + CAAM_JR_INFO( + "Closed device file for job ring %d , fd = %d", + job_ring->jr_id, job_ring->uio_fd); + close(job_ring->uio_fd); + } + } + return 0; +} diff --git a/drivers/crypto/caam_jr/meson.build b/drivers/crypto/caam_jr/meson.build new file mode 100644 index 00000000..99b71aef --- /dev/null +++ b/drivers/crypto/caam_jr/meson.build @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2018 NXP + +if host_machine.system() != 'linux' + build = false +endif + +deps += ['bus_vdev', 'bus_dpaa', 'security'] +sources = files('caam_jr_capabilities.c', + 'caam_jr_hw.c', + 'caam_jr_uio.c', + 'caam_jr.c') + +allow_experimental_apis = true + +includes += include_directories('../dpaa2_sec/') +includes += include_directories('../../bus/dpaa/include/') diff --git a/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map b/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map new file mode 100644 index 00000000..521e51f4 --- /dev/null +++ b/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map @@ -0,0 +1,4 @@ +DPDK_18.11 { + + local: *; +}; -- cgit 1.2.3-korg