aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/crypto/caam_jr
diff options
context:
space:
mode:
authorLuca Boccassi <luca.boccassi@gmail.com>2018-11-01 11:59:50 +0000
committerLuca Boccassi <luca.boccassi@gmail.com>2018-11-01 12:00:19 +0000
commit8d01b9cd70a67cdafd5b965a70420c3bd7fb3f82 (patch)
tree208e3bc33c220854d89d010e3abf720a2e62e546 /drivers/crypto/caam_jr
parentb63264c8342e6a1b6971c79550d2af2024b6a4de (diff)
New upstream version 18.11-rc1upstream/18.11-rc1
Change-Id: Iaa71986dd6332e878d8f4bf493101b2bbc6313bb Signed-off-by: Luca Boccassi <luca.boccassi@gmail.com>
Diffstat (limited to 'drivers/crypto/caam_jr')
-rw-r--r--drivers/crypto/caam_jr/Makefile44
-rw-r--r--drivers/crypto/caam_jr/caam_jr.c2508
-rw-r--r--drivers/crypto/caam_jr/caam_jr_capabilities.c266
-rw-r--r--drivers/crypto/caam_jr/caam_jr_capabilities.h18
-rw-r--r--drivers/crypto/caam_jr/caam_jr_config.h207
-rw-r--r--drivers/crypto/caam_jr/caam_jr_desc.h285
-rw-r--r--drivers/crypto/caam_jr/caam_jr_hw.c367
-rw-r--r--drivers/crypto/caam_jr/caam_jr_hw_specific.h503
-rw-r--r--drivers/crypto/caam_jr/caam_jr_log.h42
-rw-r--r--drivers/crypto/caam_jr/caam_jr_pvt.h291
-rw-r--r--drivers/crypto/caam_jr/caam_jr_uio.c501
-rw-r--r--drivers/crypto/caam_jr/meson.build17
-rw-r--r--drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map4
13 files changed, 5053 insertions, 0 deletions
diff --git a/drivers/crypto/caam_jr/Makefile b/drivers/crypto/caam_jr/Makefile
new file mode 100644
index 00000000..88cdf741
--- /dev/null
+++ b/drivers/crypto/caam_jr/Makefile
@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2017 NXP
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_pmd_caam_jr.a
+
+# build flags
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+CFLAGS += -D _GNU_SOURCE
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+
+CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa/include
+CFLAGS += -I$(RTE_SDK)/drivers/crypto/caam_jr
+#sharing the hw flib headers from dpaa2_sec pmd
+CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec/
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
+
+# versioning export map
+EXPORT_MAP := rte_pmd_caam_jr_version.map
+
+# library version
+LIBABIVER := 1
+
+# library source files
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr_capabilities.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr_hw.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr_uio.c
+# library dependencies
+
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_cryptodev
+LDLIBS += -lrte_bus_dpaa
+LDLIBS += -lrte_bus_vdev
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/crypto/caam_jr/caam_jr.c b/drivers/crypto/caam_jr/caam_jr.c
new file mode 100644
index 00000000..f505adf6
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr.c
@@ -0,0 +1,2508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <sched.h>
+#include <net/if.h>
+
+#include <rte_byteorder.h>
+#include <rte_common.h>
+#include <rte_cryptodev_pmd.h>
+#include <rte_crypto.h>
+#include <rte_cryptodev.h>
+#include <rte_bus_vdev.h>
+#include <rte_malloc.h>
+#include <rte_security_driver.h>
+#include <rte_hexdump.h>
+
+#include <caam_jr_capabilities.h>
+#include <caam_jr_config.h>
+#include <caam_jr_hw_specific.h>
+#include <caam_jr_pvt.h>
+#include <caam_jr_desc.h>
+#include <caam_jr_log.h>
+
+/* RTA header files */
+#include <hw/desc/common.h>
+#include <hw/desc/algo.h>
+#include <of.h>
+
+#define CAAM_JR_DBG 0
+#define CRYPTODEV_NAME_CAAM_JR_PMD crypto_caam_jr
+static uint8_t cryptodev_driver_id;
+int caam_jr_logtype;
+
+enum rta_sec_era rta_sec_era;
+
+/* Lists the states possible for the SEC user space driver. */
+enum sec_driver_state_e {
+ SEC_DRIVER_STATE_IDLE, /* Driver not initialized */
+ SEC_DRIVER_STATE_STARTED, /* Driver initialized and can be used*/
+ SEC_DRIVER_STATE_RELEASE, /* Driver release is in progress */
+};
+
+/* Job rings used for communication with SEC HW */
+static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS];
+
+/* The current state of SEC user space driver */
+static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE;
+
+/* The number of job rings used by SEC user space driver */
+static int g_job_rings_no;
+static int g_job_rings_max;
+
+struct sec_outring_entry {
+ phys_addr_t desc; /* Pointer to completed descriptor */
+ uint32_t status; /* Status for completed descriptor */
+} __rte_packed;
+
+/* virtual address conversin when mempool support is available for ctx */
+static inline phys_addr_t
+caam_jr_vtop_ctx(struct caam_jr_op_ctx *ctx, void *vaddr)
+{
+ PMD_INIT_FUNC_TRACE();
+ return (size_t)vaddr - ctx->vtop_offset;
+}
+
+static inline void
+caam_jr_op_ending(struct caam_jr_op_ctx *ctx)
+{
+ PMD_INIT_FUNC_TRACE();
+ /* report op status to sym->op and then free the ctx memeory */
+ rte_mempool_put(ctx->ctx_pool, (void *)ctx);
+}
+
+static inline struct caam_jr_op_ctx *
+caam_jr_alloc_ctx(struct caam_jr_session *ses)
+{
+ struct caam_jr_op_ctx *ctx;
+ int ret;
+
+ PMD_INIT_FUNC_TRACE();
+ ret = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
+ if (!ctx || ret) {
+ CAAM_JR_DP_WARN("Alloc sec descriptor failed!");
+ return NULL;
+ }
+ /*
+ * Clear SG memory. There are 16 SG entries of 16 Bytes each.
+ * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
+ * to clear all the SG entries. caam_jr_alloc_ctx() is called for
+ * each packet, memset is costlier than dcbz_64().
+ */
+ dcbz_64(&ctx->sg[SG_CACHELINE_0]);
+ dcbz_64(&ctx->sg[SG_CACHELINE_1]);
+ dcbz_64(&ctx->sg[SG_CACHELINE_2]);
+ dcbz_64(&ctx->sg[SG_CACHELINE_3]);
+
+ ctx->ctx_pool = ses->ctx_pool;
+ ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
+
+ return ctx;
+}
+
+static
+void caam_jr_stats_get(struct rte_cryptodev *dev,
+ struct rte_cryptodev_stats *stats)
+{
+ struct caam_jr_qp **qp = (struct caam_jr_qp **)
+ dev->data->queue_pairs;
+ int i;
+
+ PMD_INIT_FUNC_TRACE();
+ if (stats == NULL) {
+ CAAM_JR_ERR("Invalid stats ptr NULL");
+ return;
+ }
+ for (i = 0; i < dev->data->nb_queue_pairs; i++) {
+ if (qp[i] == NULL) {
+ CAAM_JR_WARN("Uninitialised queue pair");
+ continue;
+ }
+
+ stats->enqueued_count += qp[i]->tx_pkts;
+ stats->dequeued_count += qp[i]->rx_pkts;
+ stats->enqueue_err_count += qp[i]->tx_errs;
+ stats->dequeue_err_count += qp[i]->rx_errs;
+ CAAM_JR_INFO("extra stats:\n\tRX Poll ERR = %" PRIu64
+ "\n\tTX Ring Full = %" PRIu64,
+ qp[i]->rx_poll_err,
+ qp[i]->tx_ring_full);
+ }
+}
+
+static
+void caam_jr_stats_reset(struct rte_cryptodev *dev)
+{
+ int i;
+ struct caam_jr_qp **qp = (struct caam_jr_qp **)
+ (dev->data->queue_pairs);
+
+ PMD_INIT_FUNC_TRACE();
+ for (i = 0; i < dev->data->nb_queue_pairs; i++) {
+ if (qp[i] == NULL) {
+ CAAM_JR_WARN("Uninitialised queue pair");
+ continue;
+ }
+ qp[i]->rx_pkts = 0;
+ qp[i]->rx_errs = 0;
+ qp[i]->rx_poll_err = 0;
+ qp[i]->tx_pkts = 0;
+ qp[i]->tx_errs = 0;
+ qp[i]->tx_ring_full = 0;
+ }
+}
+
+static inline int
+is_cipher_only(struct caam_jr_session *ses)
+{
+ PMD_INIT_FUNC_TRACE();
+ return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
+ (ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
+}
+
+static inline int
+is_auth_only(struct caam_jr_session *ses)
+{
+ PMD_INIT_FUNC_TRACE();
+ return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
+ (ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
+}
+
+static inline int
+is_aead(struct caam_jr_session *ses)
+{
+ PMD_INIT_FUNC_TRACE();
+ return ((ses->cipher_alg == 0) &&
+ (ses->auth_alg == 0) &&
+ (ses->aead_alg != 0));
+}
+
+static inline int
+is_auth_cipher(struct caam_jr_session *ses)
+{
+ PMD_INIT_FUNC_TRACE();
+ return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
+ (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
+ (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
+}
+
+static inline int
+is_proto_ipsec(struct caam_jr_session *ses)
+{
+ PMD_INIT_FUNC_TRACE();
+ return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
+}
+
+static inline int
+is_encode(struct caam_jr_session *ses)
+{
+ PMD_INIT_FUNC_TRACE();
+ return ses->dir == DIR_ENC;
+}
+
+static inline int
+is_decode(struct caam_jr_session *ses)
+{
+ PMD_INIT_FUNC_TRACE();
+ return ses->dir == DIR_DEC;
+}
+
+static inline void
+caam_auth_alg(struct caam_jr_session *ses, struct alginfo *alginfo_a)
+{
+ PMD_INIT_FUNC_TRACE();
+ switch (ses->auth_alg) {
+ case RTE_CRYPTO_AUTH_NULL:
+ ses->digest_length = 0;
+ break;
+ case RTE_CRYPTO_AUTH_MD5_HMAC:
+ alginfo_a->algtype =
+ (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+ OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
+ alginfo_a->algmode = OP_ALG_AAI_HMAC;
+ break;
+ case RTE_CRYPTO_AUTH_SHA1_HMAC:
+ alginfo_a->algtype =
+ (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+ OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
+ alginfo_a->algmode = OP_ALG_AAI_HMAC;
+ break;
+ case RTE_CRYPTO_AUTH_SHA224_HMAC:
+ alginfo_a->algtype =
+ (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+ OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
+ alginfo_a->algmode = OP_ALG_AAI_HMAC;
+ break;
+ case RTE_CRYPTO_AUTH_SHA256_HMAC:
+ alginfo_a->algtype =
+ (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+ OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
+ alginfo_a->algmode = OP_ALG_AAI_HMAC;
+ break;
+ case RTE_CRYPTO_AUTH_SHA384_HMAC:
+ alginfo_a->algtype =
+ (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+ OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
+ alginfo_a->algmode = OP_ALG_AAI_HMAC;
+ break;
+ case RTE_CRYPTO_AUTH_SHA512_HMAC:
+ alginfo_a->algtype =
+ (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+ OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
+ alginfo_a->algmode = OP_ALG_AAI_HMAC;
+ break;
+ default:
+ CAAM_JR_DEBUG("unsupported auth alg %u", ses->auth_alg);
+ }
+}
+
+static inline void
+caam_cipher_alg(struct caam_jr_session *ses, struct alginfo *alginfo_c)
+{
+ PMD_INIT_FUNC_TRACE();
+ switch (ses->cipher_alg) {
+ case RTE_CRYPTO_CIPHER_NULL:
+ break;
+ case RTE_CRYPTO_CIPHER_AES_CBC:
+ alginfo_c->algtype =
+ (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+ OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
+ alginfo_c->algmode = OP_ALG_AAI_CBC;
+ break;
+ case RTE_CRYPTO_CIPHER_3DES_CBC:
+ alginfo_c->algtype =
+ (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+ OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
+ alginfo_c->algmode = OP_ALG_AAI_CBC;
+ break;
+ case RTE_CRYPTO_CIPHER_AES_CTR:
+ alginfo_c->algtype =
+ (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
+ OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
+ alginfo_c->algmode = OP_ALG_AAI_CTR;
+ break;
+ default:
+ CAAM_JR_DEBUG("unsupported cipher alg %d", ses->cipher_alg);
+ }
+}
+
+static inline void
+caam_aead_alg(struct caam_jr_session *ses, struct alginfo *alginfo)
+{
+ PMD_INIT_FUNC_TRACE();
+ switch (ses->aead_alg) {
+ case RTE_CRYPTO_AEAD_AES_GCM:
+ alginfo->algtype = OP_ALG_ALGSEL_AES;
+ alginfo->algmode = OP_ALG_AAI_GCM;
+ break;
+ default:
+ CAAM_JR_DEBUG("unsupported AEAD alg %d", ses->aead_alg);
+ }
+}
+
+/* prepare command block of the session */
+static int
+caam_jr_prep_cdb(struct caam_jr_session *ses)
+{
+ struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
+ int32_t shared_desc_len = 0;
+ struct sec_cdb *cdb;
+ int err;
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+ int swap = false;
+#else
+ int swap = true;
+#endif
+
+ PMD_INIT_FUNC_TRACE();
+ if (ses->cdb)
+ caam_jr_dma_free(ses->cdb);
+
+ cdb = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, sizeof(struct sec_cdb));
+ if (!cdb) {
+ CAAM_JR_ERR("failed to allocate memory for cdb\n");
+ return -1;
+ }
+
+ ses->cdb = cdb;
+
+ memset(cdb, 0, sizeof(struct sec_cdb));
+
+ if (is_cipher_only(ses)) {
+ caam_cipher_alg(ses, &alginfo_c);
+ if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
+ CAAM_JR_ERR("not supported cipher alg");
+ rte_free(cdb);
+ return -ENOTSUP;
+ }
+
+ alginfo_c.key = (size_t)ses->cipher_key.data;
+ alginfo_c.keylen = ses->cipher_key.length;
+ alginfo_c.key_enc_flags = 0;
+ alginfo_c.key_type = RTA_DATA_IMM;
+
+ shared_desc_len = cnstr_shdsc_blkcipher(
+ cdb->sh_desc, true,
+ swap, &alginfo_c,
+ NULL,
+ ses->iv.length,
+ ses->dir);
+ } else if (is_auth_only(ses)) {
+ caam_auth_alg(ses, &alginfo_a);
+ if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
+ CAAM_JR_ERR("not supported auth alg");
+ rte_free(cdb);
+ return -ENOTSUP;
+ }
+
+ alginfo_a.key = (size_t)ses->auth_key.data;
+ alginfo_a.keylen = ses->auth_key.length;
+ alginfo_a.key_enc_flags = 0;
+ alginfo_a.key_type = RTA_DATA_IMM;
+
+ shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
+ swap, &alginfo_a,
+ !ses->dir,
+ ses->digest_length);
+ } else if (is_aead(ses)) {
+ caam_aead_alg(ses, &alginfo);
+ if (alginfo.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
+ CAAM_JR_ERR("not supported aead alg");
+ rte_free(cdb);
+ return -ENOTSUP;
+ }
+ alginfo.key = (size_t)ses->aead_key.data;
+ alginfo.keylen = ses->aead_key.length;
+ alginfo.key_enc_flags = 0;
+ alginfo.key_type = RTA_DATA_IMM;
+
+ if (ses->dir == DIR_ENC)
+ shared_desc_len = cnstr_shdsc_gcm_encap(
+ cdb->sh_desc, true, swap,
+ &alginfo,
+ ses->iv.length,
+ ses->digest_length);
+ else
+ shared_desc_len = cnstr_shdsc_gcm_decap(
+ cdb->sh_desc, true, swap,
+ &alginfo,
+ ses->iv.length,
+ ses->digest_length);
+ } else {
+ caam_cipher_alg(ses, &alginfo_c);
+ if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
+ CAAM_JR_ERR("not supported cipher alg");
+ rte_free(cdb);
+ return -ENOTSUP;
+ }
+
+ alginfo_c.key = (size_t)ses->cipher_key.data;
+ alginfo_c.keylen = ses->cipher_key.length;
+ alginfo_c.key_enc_flags = 0;
+ alginfo_c.key_type = RTA_DATA_IMM;
+
+ caam_auth_alg(ses, &alginfo_a);
+ if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
+ CAAM_JR_ERR("not supported auth alg");
+ rte_free(cdb);
+ return -ENOTSUP;
+ }
+
+ alginfo_a.key = (size_t)ses->auth_key.data;
+ alginfo_a.keylen = ses->auth_key.length;
+ alginfo_a.key_enc_flags = 0;
+ alginfo_a.key_type = RTA_DATA_IMM;
+
+ cdb->sh_desc[0] = alginfo_c.keylen;
+ cdb->sh_desc[1] = alginfo_a.keylen;
+ err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
+ MIN_JOB_DESC_SIZE,
+ (unsigned int *)cdb->sh_desc,
+ &cdb->sh_desc[2], 2);
+
+ if (err < 0) {
+ CAAM_JR_ERR("Crypto: Incorrect key lengths");
+ rte_free(cdb);
+ return err;
+ }
+ if (cdb->sh_desc[2] & 1)
+ alginfo_c.key_type = RTA_DATA_IMM;
+ else {
+ alginfo_c.key = (size_t)caam_jr_mem_vtop(
+ (void *)(size_t)alginfo_c.key);
+ alginfo_c.key_type = RTA_DATA_PTR;
+ }
+ if (cdb->sh_desc[2] & (1<<1))
+ alginfo_a.key_type = RTA_DATA_IMM;
+ else {
+ alginfo_a.key = (size_t)caam_jr_mem_vtop(
+ (void *)(size_t)alginfo_a.key);
+ alginfo_a.key_type = RTA_DATA_PTR;
+ }
+ cdb->sh_desc[0] = 0;
+ cdb->sh_desc[1] = 0;
+ cdb->sh_desc[2] = 0;
+ if (is_proto_ipsec(ses)) {
+ if (ses->dir == DIR_ENC) {
+ shared_desc_len = cnstr_shdsc_ipsec_new_encap(
+ cdb->sh_desc,
+ true, swap, SHR_SERIAL,
+ &ses->encap_pdb,
+ (uint8_t *)&ses->ip4_hdr,
+ &alginfo_c, &alginfo_a);
+ } else if (ses->dir == DIR_DEC) {
+ shared_desc_len = cnstr_shdsc_ipsec_new_decap(
+ cdb->sh_desc,
+ true, swap, SHR_SERIAL,
+ &ses->decap_pdb,
+ &alginfo_c, &alginfo_a);
+ }
+ } else {
+ /* Auth_only_len is set as 0 here and it will be
+ * overwritten in fd for each packet.
+ */
+ shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
+ true, swap, &alginfo_c, &alginfo_a,
+ ses->iv.length, 0,
+ ses->digest_length, ses->dir);
+ }
+ }
+
+ if (shared_desc_len < 0) {
+ CAAM_JR_ERR("error in preparing command block");
+ return shared_desc_len;
+ }
+
+#if CAAM_JR_DBG
+ SEC_DUMP_DESC(cdb->sh_desc);
+#endif
+
+ cdb->sh_hdr.hi.field.idlen = shared_desc_len;
+
+ return 0;
+}
+
+/* @brief Poll the HW for already processed jobs in the JR
+ * and silently discard the available jobs or notify them to UA
+ * with indicated error code.
+ *
+ * @param [in,out] job_ring The job ring to poll.
+ * @param [in] do_notify Can be #TRUE or #FALSE. Indicates if
+ * descriptors are to be discarded
+ * or notified to UA with given error_code.
+ * @param [out] notified_descs Number of notified descriptors. Can be NULL
+ * if do_notify is #FALSE
+ */
+static void
+hw_flush_job_ring(struct sec_job_ring_t *job_ring,
+ uint32_t do_notify,
+ uint32_t *notified_descs)
+{
+ int32_t jobs_no_to_discard = 0;
+ int32_t discarded_descs_no = 0;
+
+ PMD_INIT_FUNC_TRACE();
+ CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]",
+ job_ring, job_ring->pidx, job_ring->cidx, do_notify);
+
+ jobs_no_to_discard = hw_get_no_finished_jobs(job_ring);
+
+ /* Discard all jobs */
+ CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs",
+ job_ring, job_ring->pidx, job_ring->cidx,
+ jobs_no_to_discard);
+
+ while (jobs_no_to_discard > discarded_descs_no) {
+ discarded_descs_no++;
+ /* Now increment the consumer index for the current job ring,
+ * AFTER saving job in temporary location!
+ * Increment the consumer index for the current job ring
+ */
+ job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
+ SEC_JOB_RING_SIZE);
+
+ hw_remove_entries(job_ring, 1);
+ }
+
+ if (do_notify == true) {
+ ASSERT(notified_descs != NULL);
+ *notified_descs = discarded_descs_no;
+ }
+}
+
+/* @brief Poll the HW for already processed jobs in the JR
+ * and notify the available jobs to UA.
+ *
+ * @param [in] job_ring The job ring to poll.
+ * @param [in] limit The maximum number of jobs to notify.
+ * If set to negative value, all available jobs are
+ * notified.
+ *
+ * @retval >=0 for No of jobs notified to UA.
+ * @retval -1 for error
+ */
+static int
+hw_poll_job_ring(struct sec_job_ring_t *job_ring,
+ struct rte_crypto_op **ops, int32_t limit,
+ struct caam_jr_qp *jr_qp)
+{
+ int32_t jobs_no_to_notify = 0; /* the number of done jobs to notify*/
+ int32_t number_of_jobs_available = 0;
+ int32_t notified_descs_no = 0;
+ uint32_t sec_error_code = 0;
+ struct job_descriptor *current_desc;
+ phys_addr_t current_desc_addr;
+ phys_addr_t *temp_addr;
+ struct caam_jr_op_ctx *ctx;
+
+ PMD_INIT_FUNC_TRACE();
+ /* TODO check for ops have memory*/
+ /* check here if any JR error that cannot be written
+ * in the output status word has occurred
+ */
+ if (JR_REG_JRINT_JRE_EXTRACT(GET_JR_REG(JRINT, job_ring))) {
+ CAAM_JR_INFO("err received");
+ sec_error_code = JR_REG_JRINT_ERR_TYPE_EXTRACT(
+ GET_JR_REG(JRINT, job_ring));
+ if (unlikely(sec_error_code)) {
+ hw_job_ring_error_print(job_ring, sec_error_code);
+ return -1;
+ }
+ }
+ /* compute the number of jobs available in the job ring based on the
+ * producer and consumer index values.
+ */
+ number_of_jobs_available = hw_get_no_finished_jobs(job_ring);
+ /* Compute the number of notifications that need to be raised to UA
+ * If limit > total number of done jobs -> notify all done jobs
+ * If limit = 0 -> error
+ * If limit < total number of done jobs -> notify a number
+ * of done jobs equal with limit
+ */
+ jobs_no_to_notify = (limit > number_of_jobs_available) ?
+ number_of_jobs_available : limit;
+ CAAM_JR_DP_DEBUG(
+ "Jr[%p] pi[%d] ci[%d].limit =%d Available=%d.Jobs to notify=%d",
+ job_ring, job_ring->pidx, job_ring->cidx,
+ limit, number_of_jobs_available, jobs_no_to_notify);
+
+ rte_smp_rmb();
+
+ while (jobs_no_to_notify > notified_descs_no) {
+ static uint64_t false_alarm;
+ static uint64_t real_poll;
+
+ /* Get job status here */
+ sec_error_code = job_ring->output_ring[job_ring->cidx].status;
+ /* Get completed descriptor */
+ temp_addr = &(job_ring->output_ring[job_ring->cidx].desc);
+ current_desc_addr = (phys_addr_t)sec_read_addr(temp_addr);
+
+ real_poll++;
+ /* todo check if it is false alarm no desc present */
+ if (!current_desc_addr) {
+ false_alarm++;
+ printf("false alarm %" PRIu64 "real %" PRIu64
+ " sec_err =0x%x cidx Index =0%d\n",
+ false_alarm, real_poll,
+ sec_error_code, job_ring->cidx);
+ rte_panic("CAAM JR descriptor NULL");
+ return notified_descs_no;
+ }
+ current_desc = (struct job_descriptor *)
+ caam_jr_dma_ptov(current_desc_addr);
+ /* now increment the consumer index for the current job ring,
+ * AFTER saving job in temporary location!
+ */
+ job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
+ SEC_JOB_RING_SIZE);
+ /* Signal that the job has been processed and the slot is free*/
+ hw_remove_entries(job_ring, 1);
+ /*TODO for multiple ops, packets*/
+ ctx = container_of(current_desc, struct caam_jr_op_ctx, jobdes);
+ if (unlikely(sec_error_code)) {
+ CAAM_JR_ERR("desc at cidx %d generated error 0x%x\n",
+ job_ring->cidx, sec_error_code);
+ hw_handle_job_ring_error(job_ring, sec_error_code);
+ //todo improve with exact errors
+ ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ jr_qp->rx_errs++;
+ } else {
+ ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+#if CAAM_JR_DBG
+ if (ctx->op->sym->m_dst) {
+ rte_hexdump(stdout, "PROCESSED",
+ rte_pktmbuf_mtod(ctx->op->sym->m_dst, void *),
+ rte_pktmbuf_data_len(ctx->op->sym->m_dst));
+ } else {
+ rte_hexdump(stdout, "PROCESSED",
+ rte_pktmbuf_mtod(ctx->op->sym->m_src, void *),
+ rte_pktmbuf_data_len(ctx->op->sym->m_src));
+ }
+#endif
+ }
+ if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
+ struct ip *ip4_hdr;
+
+ if (ctx->op->sym->m_dst) {
+ /*TODO check for ip header or other*/
+ ip4_hdr = (struct ip *)
+ rte_pktmbuf_mtod(ctx->op->sym->m_dst, char*);
+ ctx->op->sym->m_dst->pkt_len =
+ rte_be_to_cpu_16(ip4_hdr->ip_len);
+ ctx->op->sym->m_dst->data_len =
+ rte_be_to_cpu_16(ip4_hdr->ip_len);
+ } else {
+ ip4_hdr = (struct ip *)
+ rte_pktmbuf_mtod(ctx->op->sym->m_src, char*);
+ ctx->op->sym->m_src->pkt_len =
+ rte_be_to_cpu_16(ip4_hdr->ip_len);
+ ctx->op->sym->m_src->data_len =
+ rte_be_to_cpu_16(ip4_hdr->ip_len);
+ }
+ }
+ *ops = ctx->op;
+ caam_jr_op_ending(ctx);
+ ops++;
+ notified_descs_no++;
+ }
+ return notified_descs_no;
+}
+
+static uint16_t
+caam_jr_dequeue_burst(void *qp, struct rte_crypto_op **ops,
+ uint16_t nb_ops)
+{
+ struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
+ struct sec_job_ring_t *ring = jr_qp->ring;
+ int num_rx;
+ int ret;
+
+ PMD_INIT_FUNC_TRACE();
+ CAAM_JR_DP_DEBUG("Jr[%p]Polling. limit[%d]", ring, nb_ops);
+
+ /* Poll job ring
+ * If nb_ops < 0 -> poll JR until no more notifications are available.
+ * If nb_ops > 0 -> poll JR until limit is reached.
+ */
+
+ /* Run hw poll job ring */
+ num_rx = hw_poll_job_ring(ring, ops, nb_ops, jr_qp);
+ if (num_rx < 0) {
+ CAAM_JR_ERR("Error polling SEC engine (%d)", num_rx);
+ return 0;
+ }
+
+ CAAM_JR_DP_DEBUG("Jr[%p].Jobs notified[%d]. ", ring, num_rx);
+
+ if (ring->jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
+ if (num_rx < nb_ops) {
+ ret = caam_jr_enable_irqs(ring->irq_fd);
+ SEC_ASSERT(ret == 0, ret,
+ "Failed to enable irqs for job ring %p", ring);
+ }
+ } else if (ring->jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
+
+ /* Always enable IRQ generation when in pure IRQ mode */
+ ret = caam_jr_enable_irqs(ring->irq_fd);
+ SEC_ASSERT(ret == 0, ret,
+ "Failed to enable irqs for job ring %p", ring);
+ }
+
+ jr_qp->rx_pkts += num_rx;
+
+ return num_rx;
+}
+
+/**
+ * packet looks like:
+ * |<----data_len------->|
+ * |ip_header|ah_header|icv|payload|
+ * ^
+ * |
+ * mbuf->pkt.data
+ */
+static inline struct caam_jr_op_ctx *
+build_auth_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+ struct rte_crypto_sym_op *sym = op->sym;
+ struct rte_mbuf *mbuf = sym->m_src;
+ struct caam_jr_op_ctx *ctx;
+ struct sec4_sg_entry *sg;
+ int length;
+ struct sec_cdb *cdb;
+ uint64_t sdesc_offset;
+ struct sec_job_descriptor_t *jobdescr;
+ uint8_t extra_segs;
+
+ PMD_INIT_FUNC_TRACE();
+ if (is_decode(ses))
+ extra_segs = 2;
+ else
+ extra_segs = 1;
+
+ if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
+ CAAM_JR_DP_ERR("Auth: Max sec segs supported is %d",
+ MAX_SG_ENTRIES);
+ return NULL;
+ }
+
+ ctx = caam_jr_alloc_ctx(ses);
+ if (!ctx)
+ return NULL;
+
+ ctx->op = op;
+
+ cdb = ses->cdb;
+ sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+ jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+ SEC_JD_INIT(jobdescr);
+ SEC_JD_SET_SD(jobdescr,
+ (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+ cdb->sh_hdr.hi.field.idlen);
+
+ /* output */
+ SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
+ 0, ses->digest_length);
+
+ /*input */
+ sg = &ctx->sg[0];
+ length = sym->auth.data.length;
+ sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + sym->auth.data.offset);
+ sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
+
+ /* Successive segs */
+ mbuf = mbuf->next;
+ while (mbuf) {
+ sg++;
+ sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
+ sg->len = cpu_to_caam32(mbuf->data_len);
+ mbuf = mbuf->next;
+ }
+
+ if (is_decode(ses)) {
+ /* digest verification case */
+ sg++;
+ /* hash result or digest, save digest first */
+ rte_memcpy(ctx->digest, sym->auth.digest.data,
+ ses->digest_length);
+#if CAAM_JR_DBG
+ rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
+#endif
+ sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
+ sg->len = cpu_to_caam32(ses->digest_length);
+ length += ses->digest_length;
+ } else {
+ length -= ses->digest_length;
+ }
+
+ /* last element*/
+ sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+ SEC_JD_SET_IN_PTR(jobdescr,
+ (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0, length);
+ /* enabling sg list */
+ (jobdescr)->seq_in.command.word |= 0x01000000;
+
+ return ctx;
+}
+
+static inline struct caam_jr_op_ctx *
+build_auth_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+ struct rte_crypto_sym_op *sym = op->sym;
+ struct caam_jr_op_ctx *ctx;
+ struct sec4_sg_entry *sg;
+ rte_iova_t start_addr;
+ struct sec_cdb *cdb;
+ uint64_t sdesc_offset;
+ struct sec_job_descriptor_t *jobdescr;
+
+ PMD_INIT_FUNC_TRACE();
+ ctx = caam_jr_alloc_ctx(ses);
+ if (!ctx)
+ return NULL;
+
+ ctx->op = op;
+
+ cdb = ses->cdb;
+ sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+ start_addr = rte_pktmbuf_iova(sym->m_src);
+
+ jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+ SEC_JD_INIT(jobdescr);
+ SEC_JD_SET_SD(jobdescr,
+ (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+ cdb->sh_hdr.hi.field.idlen);
+
+ /* output */
+ SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
+ 0, ses->digest_length);
+
+ /*input */
+ if (is_decode(ses)) {
+ sg = &ctx->sg[0];
+ SEC_JD_SET_IN_PTR(jobdescr,
+ (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
+ (sym->auth.data.length + ses->digest_length));
+ /* enabling sg list */
+ (jobdescr)->seq_in.command.word |= 0x01000000;
+
+ /* hash result or digest, save digest first */
+ rte_memcpy(ctx->digest, sym->auth.digest.data,
+ ses->digest_length);
+ sg->ptr = cpu_to_caam64(start_addr + sym->auth.data.offset);
+ sg->len = cpu_to_caam32(sym->auth.data.length);
+
+#if CAAM_JR_DBG
+ rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
+#endif
+ /* let's check digest by hw */
+ sg++;
+ sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
+ sg->len = cpu_to_caam32(ses->digest_length);
+ /* last element*/
+ sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+ } else {
+ SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)start_addr,
+ sym->auth.data.offset, sym->auth.data.length);
+ }
+ return ctx;
+}
+
+static inline struct caam_jr_op_ctx *
+build_cipher_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+ struct rte_crypto_sym_op *sym = op->sym;
+ struct rte_mbuf *mbuf = sym->m_src;
+ struct caam_jr_op_ctx *ctx;
+ struct sec4_sg_entry *sg, *in_sg;
+ int length;
+ struct sec_cdb *cdb;
+ uint64_t sdesc_offset;
+ uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
+ ses->iv.offset);
+ struct sec_job_descriptor_t *jobdescr;
+ uint8_t reg_segs;
+
+ PMD_INIT_FUNC_TRACE();
+ if (sym->m_dst) {
+ mbuf = sym->m_dst;
+ reg_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2;
+ } else {
+ mbuf = sym->m_src;
+ reg_segs = mbuf->nb_segs * 2 + 2;
+ }
+
+ if (reg_segs > MAX_SG_ENTRIES) {
+ CAAM_JR_DP_ERR("Cipher: Max sec segs supported is %d",
+ MAX_SG_ENTRIES);
+ return NULL;
+ }
+
+ ctx = caam_jr_alloc_ctx(ses);
+ if (!ctx)
+ return NULL;
+
+ ctx->op = op;
+ cdb = ses->cdb;
+ sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+ jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+ SEC_JD_INIT(jobdescr);
+ SEC_JD_SET_SD(jobdescr,
+ (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+ cdb->sh_hdr.hi.field.idlen);
+
+#if CAAM_JR_DBG
+ CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
+ sym->m_src->data_off, sym->cipher.data.offset,
+ sym->cipher.data.length, ses->iv.length);
+#endif
+ /* output */
+ if (sym->m_dst)
+ mbuf = sym->m_dst;
+ else
+ mbuf = sym->m_src;
+
+ sg = &ctx->sg[0];
+ length = sym->cipher.data.length;
+
+ sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
+ + sym->cipher.data.offset);
+ sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
+
+ /* Successive segs */
+ mbuf = mbuf->next;
+ while (mbuf) {
+ sg++;
+ sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
+ sg->len = cpu_to_caam32(mbuf->data_len);
+ mbuf = mbuf->next;
+ }
+ /* last element*/
+ sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+ SEC_JD_SET_OUT_PTR(jobdescr,
+ (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0,
+ length);
+ /*enabling sg bit */
+ (jobdescr)->seq_out.command.word |= 0x01000000;
+
+ /*input */
+ sg++;
+ mbuf = sym->m_src;
+ in_sg = sg;
+
+ length = sym->cipher.data.length + ses->iv.length;
+
+ /* IV */
+ sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
+ sg->len = cpu_to_caam32(ses->iv.length);
+
+ /* 1st seg */
+ sg++;
+ sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
+ + sym->cipher.data.offset);
+ sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
+
+ /* Successive segs */
+ mbuf = mbuf->next;
+ while (mbuf) {
+ sg++;
+ sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
+ sg->len = cpu_to_caam32(mbuf->data_len);
+ mbuf = mbuf->next;
+ }
+ /* last element*/
+ sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+
+ SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, in_sg), 0,
+ length);
+ /*enabling sg bit */
+ (jobdescr)->seq_in.command.word |= 0x01000000;
+
+ return ctx;
+}
+
+static inline struct caam_jr_op_ctx *
+build_cipher_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+ struct rte_crypto_sym_op *sym = op->sym;
+ struct caam_jr_op_ctx *ctx;
+ struct sec4_sg_entry *sg;
+ rte_iova_t src_start_addr, dst_start_addr;
+ struct sec_cdb *cdb;
+ uint64_t sdesc_offset;
+ uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
+ ses->iv.offset);
+ struct sec_job_descriptor_t *jobdescr;
+
+ PMD_INIT_FUNC_TRACE();
+ ctx = caam_jr_alloc_ctx(ses);
+ if (!ctx)
+ return NULL;
+
+ ctx->op = op;
+ cdb = ses->cdb;
+ sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+ src_start_addr = rte_pktmbuf_iova(sym->m_src);
+ if (sym->m_dst)
+ dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
+ else
+ dst_start_addr = src_start_addr;
+
+ jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+ SEC_JD_INIT(jobdescr);
+ SEC_JD_SET_SD(jobdescr,
+ (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+ cdb->sh_hdr.hi.field.idlen);
+
+#if CAAM_JR_DBG
+ CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
+ sym->m_src->data_off, sym->cipher.data.offset,
+ sym->cipher.data.length, ses->iv.length);
+#endif
+ /* output */
+ SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr,
+ sym->cipher.data.offset,
+ sym->cipher.data.length + ses->iv.length);
+
+ /*input */
+ sg = &ctx->sg[0];
+ SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
+ sym->cipher.data.length + ses->iv.length);
+ /*enabling sg bit */
+ (jobdescr)->seq_in.command.word |= 0x01000000;
+
+ sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
+ sg->len = cpu_to_caam32(ses->iv.length);
+
+ sg = &ctx->sg[1];
+ sg->ptr = cpu_to_caam64(src_start_addr + sym->cipher.data.offset);
+ sg->len = cpu_to_caam32(sym->cipher.data.length);
+ /* last element*/
+ sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+ return ctx;
+}
+
+/* For decapsulation:
+ * Input:
+ * +----+----------------+--------------------------------+-----+
+ * | IV | Auth-only data | Authenticated & Encrypted data | ICV |
+ * +----+----------------+--------------------------------+-----+
+ * Output:
+ * +----+--------------------------+
+ * | Decrypted & authenticated data |
+ * +----+--------------------------+
+ */
+
+static inline struct caam_jr_op_ctx *
+build_cipher_auth_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+ struct rte_crypto_sym_op *sym = op->sym;
+ struct caam_jr_op_ctx *ctx;
+ struct sec4_sg_entry *sg, *out_sg, *in_sg;
+ struct rte_mbuf *mbuf;
+ uint32_t length = 0;
+ struct sec_cdb *cdb;
+ uint64_t sdesc_offset;
+ uint8_t req_segs;
+ uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
+ ses->iv.offset);
+ struct sec_job_descriptor_t *jobdescr;
+ uint32_t auth_only_len;
+
+ PMD_INIT_FUNC_TRACE();
+ auth_only_len = op->sym->auth.data.length -
+ op->sym->cipher.data.length;
+
+ if (sym->m_dst) {
+ mbuf = sym->m_dst;
+ req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
+ } else {
+ mbuf = sym->m_src;
+ req_segs = mbuf->nb_segs * 2 + 3;
+ }
+
+ if (req_segs > MAX_SG_ENTRIES) {
+ CAAM_JR_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
+ MAX_SG_ENTRIES);
+ return NULL;
+ }
+
+ ctx = caam_jr_alloc_ctx(ses);
+ if (!ctx)
+ return NULL;
+
+ ctx->op = op;
+ cdb = ses->cdb;
+ sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+ jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+ SEC_JD_INIT(jobdescr);
+ SEC_JD_SET_SD(jobdescr,
+ (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+ cdb->sh_hdr.hi.field.idlen);
+
+ /* output */
+ if (sym->m_dst)
+ mbuf = sym->m_dst;
+ else
+ mbuf = sym->m_src;
+
+ out_sg = &ctx->sg[0];
+ if (is_encode(ses))
+ length = sym->auth.data.length + ses->digest_length;
+ else
+ length = sym->auth.data.length;
+
+ sg = &ctx->sg[0];
+
+ /* 1st seg */
+ sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
+ + sym->auth.data.offset);
+ sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
+
+ /* Successive segs */
+ mbuf = mbuf->next;
+ while (mbuf) {
+ sg++;
+ sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
+ sg->len = cpu_to_caam32(mbuf->data_len);
+ mbuf = mbuf->next;
+ }
+
+ if (is_encode(ses)) {
+ /* set auth output */
+ sg++;
+ sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
+ sg->len = cpu_to_caam32(ses->digest_length);
+ }
+ /* last element*/
+ sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+ SEC_JD_SET_OUT_PTR(jobdescr,
+ (uint64_t)caam_jr_dma_vtop(out_sg), 0, length);
+ /* set sg bit */
+ (jobdescr)->seq_out.command.word |= 0x01000000;
+
+ /* input */
+ sg++;
+ mbuf = sym->m_src;
+ in_sg = sg;
+ if (is_encode(ses))
+ length = ses->iv.length + sym->auth.data.length;
+ else
+ length = ses->iv.length + sym->auth.data.length
+ + ses->digest_length;
+
+ sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
+ sg->len = cpu_to_caam32(ses->iv.length);
+
+ sg++;
+ /* 1st seg */
+ sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
+ + sym->auth.data.offset);
+ sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
+
+ /* Successive segs */
+ mbuf = mbuf->next;
+ while (mbuf) {
+ sg++;
+ sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
+ sg->len = cpu_to_caam32(mbuf->data_len);
+ mbuf = mbuf->next;
+ }
+
+ if (is_decode(ses)) {
+ sg++;
+ rte_memcpy(ctx->digest, sym->auth.digest.data,
+ ses->digest_length);
+ sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
+ sg->len = cpu_to_caam32(ses->digest_length);
+ }
+ /* last element*/
+ sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+ SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(in_sg), 0,
+ length);
+ /* set sg bit */
+ (jobdescr)->seq_in.command.word |= 0x01000000;
+ /* Auth_only_len is set as 0 in descriptor and it is
+ * overwritten here in the jd which will update
+ * the DPOVRD reg.
+ */
+ if (auth_only_len)
+ /* set sg bit */
+ (jobdescr)->dpovrd = 0x80000000 | auth_only_len;
+
+ return ctx;
+}
+
+static inline struct caam_jr_op_ctx *
+build_cipher_auth(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+ struct rte_crypto_sym_op *sym = op->sym;
+ struct caam_jr_op_ctx *ctx;
+ struct sec4_sg_entry *sg;
+ rte_iova_t src_start_addr, dst_start_addr;
+ uint32_t length = 0;
+ struct sec_cdb *cdb;
+ uint64_t sdesc_offset;
+ uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
+ ses->iv.offset);
+ struct sec_job_descriptor_t *jobdescr;
+ uint32_t auth_only_len;
+
+ PMD_INIT_FUNC_TRACE();
+ auth_only_len = op->sym->auth.data.length -
+ op->sym->cipher.data.length;
+
+ src_start_addr = rte_pktmbuf_iova(sym->m_src);
+ if (sym->m_dst)
+ dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
+ else
+ dst_start_addr = src_start_addr;
+
+ ctx = caam_jr_alloc_ctx(ses);
+ if (!ctx)
+ return NULL;
+
+ ctx->op = op;
+ cdb = ses->cdb;
+ sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+ jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+ SEC_JD_INIT(jobdescr);
+ SEC_JD_SET_SD(jobdescr,
+ (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+ cdb->sh_hdr.hi.field.idlen);
+
+ /* input */
+ sg = &ctx->sg[0];
+ if (is_encode(ses)) {
+ sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
+ sg->len = cpu_to_caam32(ses->iv.length);
+ length += ses->iv.length;
+
+ sg++;
+ sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
+ sg->len = cpu_to_caam32(sym->auth.data.length);
+ length += sym->auth.data.length;
+ /* last element*/
+ sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+ } else {
+ sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
+ sg->len = cpu_to_caam32(ses->iv.length);
+ length += ses->iv.length;
+
+ sg++;
+ sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
+ sg->len = cpu_to_caam32(sym->auth.data.length);
+ length += sym->auth.data.length;
+
+ rte_memcpy(ctx->digest, sym->auth.digest.data,
+ ses->digest_length);
+ sg++;
+ sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
+ sg->len = cpu_to_caam32(ses->digest_length);
+ length += ses->digest_length;
+ /* last element*/
+ sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+ }
+
+ SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(&ctx->sg[0]), 0,
+ length);
+ /* set sg bit */
+ (jobdescr)->seq_in.command.word |= 0x01000000;
+
+ /* output */
+ sg = &ctx->sg[6];
+
+ sg->ptr = cpu_to_caam64(dst_start_addr + sym->cipher.data.offset);
+ sg->len = cpu_to_caam32(sym->cipher.data.length);
+ length = sym->cipher.data.length;
+
+ if (is_encode(ses)) {
+ /* set auth output */
+ sg++;
+ sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
+ sg->len = cpu_to_caam32(ses->digest_length);
+ length += ses->digest_length;
+ }
+ /* last element*/
+ sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+
+ SEC_JD_SET_OUT_PTR(jobdescr,
+ (uint64_t)caam_jr_dma_vtop(&ctx->sg[6]), 0, length);
+ /* set sg bit */
+ (jobdescr)->seq_out.command.word |= 0x01000000;
+
+ /* Auth_only_len is set as 0 in descriptor and it is
+ * overwritten here in the jd which will update
+ * the DPOVRD reg.
+ */
+ if (auth_only_len)
+ /* set sg bit */
+ (jobdescr)->dpovrd = 0x80000000 | auth_only_len;
+
+ return ctx;
+}
+
+static inline struct caam_jr_op_ctx *
+build_proto(struct rte_crypto_op *op, struct caam_jr_session *ses)
+{
+ struct rte_crypto_sym_op *sym = op->sym;
+ struct caam_jr_op_ctx *ctx = NULL;
+ phys_addr_t src_start_addr, dst_start_addr;
+ struct sec_cdb *cdb;
+ uint64_t sdesc_offset;
+ struct sec_job_descriptor_t *jobdescr;
+
+ PMD_INIT_FUNC_TRACE();
+ ctx = caam_jr_alloc_ctx(ses);
+ if (!ctx)
+ return NULL;
+ ctx->op = op;
+
+ src_start_addr = rte_pktmbuf_iova(sym->m_src);
+ if (sym->m_dst)
+ dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
+ else
+ dst_start_addr = src_start_addr;
+
+ cdb = ses->cdb;
+ sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
+
+ jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+ SEC_JD_INIT(jobdescr);
+ SEC_JD_SET_SD(jobdescr,
+ (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
+ cdb->sh_hdr.hi.field.idlen);
+
+ /* output */
+ SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr, 0,
+ sym->m_src->buf_len - sym->m_src->data_off);
+ /* input */
+ SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)src_start_addr, 0,
+ sym->m_src->pkt_len);
+ sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
+
+ return ctx;
+}
+
+static int
+caam_jr_enqueue_op(struct rte_crypto_op *op, struct caam_jr_qp *qp)
+{
+ struct sec_job_ring_t *ring = qp->ring;
+ struct caam_jr_session *ses;
+ struct caam_jr_op_ctx *ctx = NULL;
+ struct sec_job_descriptor_t *jobdescr __rte_unused;
+
+ PMD_INIT_FUNC_TRACE();
+ switch (op->sess_type) {
+ case RTE_CRYPTO_OP_WITH_SESSION:
+ ses = (struct caam_jr_session *)
+ get_sym_session_private_data(op->sym->session,
+ cryptodev_driver_id);
+ break;
+ case RTE_CRYPTO_OP_SECURITY_SESSION:
+ ses = (struct caam_jr_session *)
+ get_sec_session_private_data(
+ op->sym->sec_session);
+ break;
+ default:
+ CAAM_JR_DP_ERR("sessionless crypto op not supported");
+ qp->tx_errs++;
+ return -1;
+ }
+
+ if (unlikely(!ses->qp || ses->qp != qp)) {
+ CAAM_JR_DP_DEBUG("Old:sess->qp=%p New qp = %p\n", ses->qp, qp);
+ ses->qp = qp;
+ caam_jr_prep_cdb(ses);
+ }
+
+ if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
+ if (is_auth_cipher(ses))
+ ctx = build_cipher_auth(op, ses);
+ else if (is_aead(ses))
+ goto err1;
+ else if (is_auth_only(ses))
+ ctx = build_auth_only(op, ses);
+ else if (is_cipher_only(ses))
+ ctx = build_cipher_only(op, ses);
+ else if (is_proto_ipsec(ses))
+ ctx = build_proto(op, ses);
+ } else {
+ if (is_auth_cipher(ses))
+ ctx = build_cipher_auth_sg(op, ses);
+ else if (is_aead(ses))
+ goto err1;
+ else if (is_auth_only(ses))
+ ctx = build_auth_only_sg(op, ses);
+ else if (is_cipher_only(ses))
+ ctx = build_cipher_only_sg(op, ses);
+ }
+err1:
+ if (unlikely(!ctx)) {
+ qp->tx_errs++;
+ CAAM_JR_ERR("not supported sec op");
+ return -1;
+ }
+#if CAAM_JR_DBG
+ if (is_decode(ses))
+ rte_hexdump(stdout, "DECODE",
+ rte_pktmbuf_mtod(op->sym->m_src, void *),
+ rte_pktmbuf_data_len(op->sym->m_src));
+ else
+ rte_hexdump(stdout, "ENCODE",
+ rte_pktmbuf_mtod(op->sym->m_src, void *),
+ rte_pktmbuf_data_len(op->sym->m_src));
+
+ printf("\n JD before conversion\n");
+ for (int i = 0; i < 12; i++)
+ printf("\n 0x%08x", ctx->jobdes.desc[i]);
+#endif
+
+ CAAM_JR_DP_DEBUG("Jr[%p] pi[%d] ci[%d].Before sending desc",
+ ring, ring->pidx, ring->cidx);
+
+ /* todo - do we want to retry */
+ if (SEC_JOB_RING_IS_FULL(ring->pidx, ring->cidx,
+ SEC_JOB_RING_SIZE, SEC_JOB_RING_SIZE)) {
+ CAAM_JR_DP_DEBUG("Ring FULL Jr[%p] pi[%d] ci[%d].Size = %d",
+ ring, ring->pidx, ring->cidx, SEC_JOB_RING_SIZE);
+ caam_jr_op_ending(ctx);
+ qp->tx_ring_full++;
+ return -EBUSY;
+ }
+
+#if CORE_BYTE_ORDER != CAAM_BYTE_ORDER
+ jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
+
+ jobdescr->deschdr.command.word =
+ cpu_to_caam32(jobdescr->deschdr.command.word);
+ jobdescr->sd_ptr = cpu_to_caam64(jobdescr->sd_ptr);
+ jobdescr->seq_out.command.word =
+ cpu_to_caam32(jobdescr->seq_out.command.word);
+ jobdescr->seq_out_ptr = cpu_to_caam64(jobdescr->seq_out_ptr);
+ jobdescr->out_ext_length = cpu_to_caam32(jobdescr->out_ext_length);
+ jobdescr->seq_in.command.word =
+ cpu_to_caam32(jobdescr->seq_in.command.word);
+ jobdescr->seq_in_ptr = cpu_to_caam64(jobdescr->seq_in_ptr);
+ jobdescr->in_ext_length = cpu_to_caam32(jobdescr->in_ext_length);
+ jobdescr->load_dpovrd.command.word =
+ cpu_to_caam32(jobdescr->load_dpovrd.command.word);
+ jobdescr->dpovrd = cpu_to_caam32(jobdescr->dpovrd);
+#endif
+
+ /* Set ptr in input ring to current descriptor */
+ sec_write_addr(&ring->input_ring[ring->pidx],
+ (phys_addr_t)caam_jr_vtop_ctx(ctx, ctx->jobdes.desc));
+ rte_smp_wmb();
+
+ /* Notify HW that a new job is enqueued */
+ hw_enqueue_desc_on_job_ring(ring);
+
+ /* increment the producer index for the current job ring */
+ ring->pidx = SEC_CIRCULAR_COUNTER(ring->pidx, SEC_JOB_RING_SIZE);
+
+ return 0;
+}
+
+static uint16_t
+caam_jr_enqueue_burst(void *qp, struct rte_crypto_op **ops,
+ uint16_t nb_ops)
+{
+ /* Function to transmit the frames to given device and queuepair */
+ uint32_t loop;
+ int32_t ret;
+ struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
+ uint16_t num_tx = 0;
+
+ PMD_INIT_FUNC_TRACE();
+ /*Prepare each packet which is to be sent*/
+ for (loop = 0; loop < nb_ops; loop++) {
+ ret = caam_jr_enqueue_op(ops[loop], jr_qp);
+ if (!ret)
+ num_tx++;
+ }
+
+ jr_qp->tx_pkts += num_tx;
+
+ return num_tx;
+}
+
+/* Release queue pair */
+static int
+caam_jr_queue_pair_release(struct rte_cryptodev *dev,
+ uint16_t qp_id)
+{
+ struct sec_job_ring_t *internals;
+ struct caam_jr_qp *qp = NULL;
+
+ PMD_INIT_FUNC_TRACE();
+ CAAM_JR_DEBUG("dev =%p, queue =%d", dev, qp_id);
+
+ internals = dev->data->dev_private;
+ if (qp_id >= internals->max_nb_queue_pairs) {
+ CAAM_JR_ERR("Max supported qpid %d",
+ internals->max_nb_queue_pairs);
+ return -EINVAL;
+ }
+
+ qp = &internals->qps[qp_id];
+ qp->ring = NULL;
+ dev->data->queue_pairs[qp_id] = NULL;
+
+ return 0;
+}
+
+/* Setup a queue pair */
+static int
+caam_jr_queue_pair_setup(
+ struct rte_cryptodev *dev, uint16_t qp_id,
+ __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
+ __rte_unused int socket_id,
+ __rte_unused struct rte_mempool *session_pool)
+{
+ struct sec_job_ring_t *internals;
+ struct caam_jr_qp *qp = NULL;
+
+ PMD_INIT_FUNC_TRACE();
+ CAAM_JR_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
+
+ internals = dev->data->dev_private;
+ if (qp_id >= internals->max_nb_queue_pairs) {
+ CAAM_JR_ERR("Max supported qpid %d",
+ internals->max_nb_queue_pairs);
+ return -EINVAL;
+ }
+
+ qp = &internals->qps[qp_id];
+ qp->ring = internals;
+ dev->data->queue_pairs[qp_id] = qp;
+
+ return 0;
+}
+
+/* Return the number of allocated queue pairs */
+static uint32_t
+caam_jr_queue_pair_count(struct rte_cryptodev *dev)
+{
+ PMD_INIT_FUNC_TRACE();
+
+ return dev->data->nb_queue_pairs;
+}
+
+/* Returns the size of the aesni gcm session structure */
+static unsigned int
+caam_jr_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+ PMD_INIT_FUNC_TRACE();
+
+ return sizeof(struct caam_jr_session);
+}
+
+static int
+caam_jr_cipher_init(struct rte_cryptodev *dev __rte_unused,
+ struct rte_crypto_sym_xform *xform,
+ struct caam_jr_session *session)
+{
+ PMD_INIT_FUNC_TRACE();
+ session->cipher_alg = xform->cipher.algo;
+ session->iv.length = xform->cipher.iv.length;
+ session->iv.offset = xform->cipher.iv.offset;
+ session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
+ RTE_CACHE_LINE_SIZE);
+ if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
+ CAAM_JR_ERR("No Memory for cipher key\n");
+ return -ENOMEM;
+ }
+ session->cipher_key.length = xform->cipher.key.length;
+
+ memcpy(session->cipher_key.data, xform->cipher.key.data,
+ xform->cipher.key.length);
+ session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
+ DIR_ENC : DIR_DEC;
+
+ return 0;
+}
+
+static int
+caam_jr_auth_init(struct rte_cryptodev *dev __rte_unused,
+ struct rte_crypto_sym_xform *xform,
+ struct caam_jr_session *session)
+{
+ PMD_INIT_FUNC_TRACE();
+ session->auth_alg = xform->auth.algo;
+ session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
+ RTE_CACHE_LINE_SIZE);
+ if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
+ CAAM_JR_ERR("No Memory for auth key\n");
+ return -ENOMEM;
+ }
+ session->auth_key.length = xform->auth.key.length;
+ session->digest_length = xform->auth.digest_length;
+
+ memcpy(session->auth_key.data, xform->auth.key.data,
+ xform->auth.key.length);
+ session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
+ DIR_ENC : DIR_DEC;
+
+ return 0;
+}
+
+static int
+caam_jr_aead_init(struct rte_cryptodev *dev __rte_unused,
+ struct rte_crypto_sym_xform *xform,
+ struct caam_jr_session *session)
+{
+ PMD_INIT_FUNC_TRACE();
+ session->aead_alg = xform->aead.algo;
+ session->iv.length = xform->aead.iv.length;
+ session->iv.offset = xform->aead.iv.offset;
+ session->auth_only_len = xform->aead.aad_length;
+ session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
+ RTE_CACHE_LINE_SIZE);
+ if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
+ CAAM_JR_ERR("No Memory for aead key\n");
+ return -ENOMEM;
+ }
+ session->aead_key.length = xform->aead.key.length;
+ session->digest_length = xform->aead.digest_length;
+
+ memcpy(session->aead_key.data, xform->aead.key.data,
+ xform->aead.key.length);
+ session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
+ DIR_ENC : DIR_DEC;
+
+ return 0;
+}
+
+static int
+caam_jr_set_session_parameters(struct rte_cryptodev *dev,
+ struct rte_crypto_sym_xform *xform, void *sess)
+{
+ struct sec_job_ring_t *internals = dev->data->dev_private;
+ struct caam_jr_session *session = sess;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (unlikely(sess == NULL)) {
+ CAAM_JR_ERR("invalid session struct");
+ return -EINVAL;
+ }
+
+ /* Default IV length = 0 */
+ session->iv.length = 0;
+
+ /* Cipher Only */
+ if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
+ session->auth_alg = RTE_CRYPTO_AUTH_NULL;
+ caam_jr_cipher_init(dev, xform, session);
+
+ /* Authentication Only */
+ } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
+ xform->next == NULL) {
+ session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
+ caam_jr_auth_init(dev, xform, session);
+
+ /* Cipher then Authenticate */
+ } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
+ xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
+ if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
+ caam_jr_cipher_init(dev, xform, session);
+ caam_jr_auth_init(dev, xform->next, session);
+ } else {
+ CAAM_JR_ERR("Not supported: Auth then Cipher");
+ goto err1;
+ }
+
+ /* Authenticate then Cipher */
+ } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
+ xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
+ if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
+ caam_jr_auth_init(dev, xform, session);
+ caam_jr_cipher_init(dev, xform->next, session);
+ } else {
+ CAAM_JR_ERR("Not supported: Auth then Cipher");
+ goto err1;
+ }
+
+ /* AEAD operation for AES-GCM kind of Algorithms */
+ } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
+ xform->next == NULL) {
+ caam_jr_aead_init(dev, xform, session);
+
+ } else {
+ CAAM_JR_ERR("Invalid crypto type");
+ return -EINVAL;
+ }
+ session->ctx_pool = internals->ctx_pool;
+
+ return 0;
+
+err1:
+ rte_free(session->cipher_key.data);
+ rte_free(session->auth_key.data);
+ memset(session, 0, sizeof(struct caam_jr_session));
+
+ return -EINVAL;
+}
+
+static int
+caam_jr_sym_session_configure(struct rte_cryptodev *dev,
+ struct rte_crypto_sym_xform *xform,
+ struct rte_cryptodev_sym_session *sess,
+ struct rte_mempool *mempool)
+{
+ void *sess_private_data;
+ int ret;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (rte_mempool_get(mempool, &sess_private_data)) {
+ CAAM_JR_ERR("Couldn't get object from session mempool");
+ return -ENOMEM;
+ }
+
+ memset(sess_private_data, 0, sizeof(struct caam_jr_session));
+ ret = caam_jr_set_session_parameters(dev, xform, sess_private_data);
+ if (ret != 0) {
+ CAAM_JR_ERR("failed to configure session parameters");
+ /* Return session to mempool */
+ rte_mempool_put(mempool, sess_private_data);
+ return ret;
+ }
+
+ set_sym_session_private_data(sess, dev->driver_id, sess_private_data);
+
+ return 0;
+}
+
+/* Clear the memory of session so it doesn't leave key material behind */
+static void
+caam_jr_sym_session_clear(struct rte_cryptodev *dev,
+ struct rte_cryptodev_sym_session *sess)
+{
+ uint8_t index = dev->driver_id;
+ void *sess_priv = get_sym_session_private_data(sess, index);
+ struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (sess_priv) {
+ struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
+
+ rte_free(s->cipher_key.data);
+ rte_free(s->auth_key.data);
+ memset(s, 0, sizeof(struct caam_jr_session));
+ set_sym_session_private_data(sess, index, NULL);
+ rte_mempool_put(sess_mp, sess_priv);
+ }
+}
+
+static int
+caam_jr_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
+ struct rte_security_session_conf *conf,
+ void *sess)
+{
+ struct sec_job_ring_t *internals = dev->data->dev_private;
+ struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
+ struct rte_crypto_auth_xform *auth_xform;
+ struct rte_crypto_cipher_xform *cipher_xform;
+ struct caam_jr_session *session = (struct caam_jr_session *)sess;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
+ cipher_xform = &conf->crypto_xform->cipher;
+ auth_xform = &conf->crypto_xform->next->auth;
+ } else {
+ auth_xform = &conf->crypto_xform->auth;
+ cipher_xform = &conf->crypto_xform->next->cipher;
+ }
+ session->proto_alg = conf->protocol;
+ session->cipher_key.data = rte_zmalloc(NULL,
+ cipher_xform->key.length,
+ RTE_CACHE_LINE_SIZE);
+ if (session->cipher_key.data == NULL &&
+ cipher_xform->key.length > 0) {
+ CAAM_JR_ERR("No Memory for cipher key\n");
+ return -ENOMEM;
+ }
+
+ session->cipher_key.length = cipher_xform->key.length;
+ session->auth_key.data = rte_zmalloc(NULL,
+ auth_xform->key.length,
+ RTE_CACHE_LINE_SIZE);
+ if (session->auth_key.data == NULL &&
+ auth_xform->key.length > 0) {
+ CAAM_JR_ERR("No Memory for auth key\n");
+ rte_free(session->cipher_key.data);
+ return -ENOMEM;
+ }
+ session->auth_key.length = auth_xform->key.length;
+ memcpy(session->cipher_key.data, cipher_xform->key.data,
+ cipher_xform->key.length);
+ memcpy(session->auth_key.data, auth_xform->key.data,
+ auth_xform->key.length);
+
+ switch (auth_xform->algo) {
+ case RTE_CRYPTO_AUTH_SHA1_HMAC:
+ session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
+ break;
+ case RTE_CRYPTO_AUTH_MD5_HMAC:
+ session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
+ break;
+ case RTE_CRYPTO_AUTH_SHA256_HMAC:
+ session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
+ break;
+ case RTE_CRYPTO_AUTH_SHA384_HMAC:
+ session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
+ break;
+ case RTE_CRYPTO_AUTH_SHA512_HMAC:
+ session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
+ break;
+ case RTE_CRYPTO_AUTH_AES_CMAC:
+ session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
+ break;
+ case RTE_CRYPTO_AUTH_NULL:
+ session->auth_alg = RTE_CRYPTO_AUTH_NULL;
+ break;
+ case RTE_CRYPTO_AUTH_SHA224_HMAC:
+ case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
+ case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
+ case RTE_CRYPTO_AUTH_SHA1:
+ case RTE_CRYPTO_AUTH_SHA256:
+ case RTE_CRYPTO_AUTH_SHA512:
+ case RTE_CRYPTO_AUTH_SHA224:
+ case RTE_CRYPTO_AUTH_SHA384:
+ case RTE_CRYPTO_AUTH_MD5:
+ case RTE_CRYPTO_AUTH_AES_GMAC:
+ case RTE_CRYPTO_AUTH_KASUMI_F9:
+ case RTE_CRYPTO_AUTH_AES_CBC_MAC:
+ case RTE_CRYPTO_AUTH_ZUC_EIA3:
+ CAAM_JR_ERR("Crypto: Unsupported auth alg %u\n",
+ auth_xform->algo);
+ goto out;
+ default:
+ CAAM_JR_ERR("Crypto: Undefined Auth specified %u\n",
+ auth_xform->algo);
+ goto out;
+ }
+
+ switch (cipher_xform->algo) {
+ case RTE_CRYPTO_CIPHER_AES_CBC:
+ session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
+ break;
+ case RTE_CRYPTO_CIPHER_3DES_CBC:
+ session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
+ break;
+ case RTE_CRYPTO_CIPHER_AES_CTR:
+ session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
+ break;
+ case RTE_CRYPTO_CIPHER_NULL:
+ case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
+ case RTE_CRYPTO_CIPHER_3DES_ECB:
+ case RTE_CRYPTO_CIPHER_AES_ECB:
+ case RTE_CRYPTO_CIPHER_KASUMI_F8:
+ CAAM_JR_ERR("Crypto: Unsupported Cipher alg %u\n",
+ cipher_xform->algo);
+ goto out;
+ default:
+ CAAM_JR_ERR("Crypto: Undefined Cipher specified %u\n",
+ cipher_xform->algo);
+ goto out;
+ }
+
+ if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
+ memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
+ sizeof(session->ip4_hdr));
+ session->ip4_hdr.ip_v = IPVERSION;
+ session->ip4_hdr.ip_hl = 5;
+ session->ip4_hdr.ip_len = rte_cpu_to_be_16(
+ sizeof(session->ip4_hdr));
+ session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
+ session->ip4_hdr.ip_id = 0;
+ session->ip4_hdr.ip_off = 0;
+ session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
+ session->ip4_hdr.ip_p = (ipsec_xform->proto ==
+ RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
+ : IPPROTO_AH;
+ session->ip4_hdr.ip_sum = 0;
+ session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
+ session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
+ session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
+ (void *)&session->ip4_hdr,
+ sizeof(struct ip));
+
+ session->encap_pdb.options =
+ (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
+ PDBOPTS_ESP_OIHI_PDB_INL |
+ PDBOPTS_ESP_IVSRC |
+ PDBHMO_ESP_ENCAP_DTTL;
+ session->encap_pdb.spi = ipsec_xform->spi;
+ session->encap_pdb.ip_hdr_len = sizeof(struct ip);
+
+ session->dir = DIR_ENC;
+ } else if (ipsec_xform->direction ==
+ RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
+ memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
+ session->decap_pdb.options = sizeof(struct ip) << 16;
+ session->dir = DIR_DEC;
+ } else
+ goto out;
+ session->ctx_pool = internals->ctx_pool;
+
+ return 0;
+out:
+ rte_free(session->auth_key.data);
+ rte_free(session->cipher_key.data);
+ memset(session, 0, sizeof(struct caam_jr_session));
+ return -1;
+}
+
+static int
+caam_jr_security_session_create(void *dev,
+ struct rte_security_session_conf *conf,
+ struct rte_security_session *sess,
+ struct rte_mempool *mempool)
+{
+ void *sess_private_data;
+ struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
+ int ret;
+
+ PMD_INIT_FUNC_TRACE();
+ if (rte_mempool_get(mempool, &sess_private_data)) {
+ CAAM_JR_ERR("Couldn't get object from session mempool");
+ return -ENOMEM;
+ }
+
+ switch (conf->protocol) {
+ case RTE_SECURITY_PROTOCOL_IPSEC:
+ ret = caam_jr_set_ipsec_session(cdev, conf,
+ sess_private_data);
+ break;
+ case RTE_SECURITY_PROTOCOL_MACSEC:
+ return -ENOTSUP;
+ default:
+ return -EINVAL;
+ }
+ if (ret != 0) {
+ CAAM_JR_ERR("failed to configure session parameters");
+ /* Return session to mempool */
+ rte_mempool_put(mempool, sess_private_data);
+ return ret;
+ }
+
+ set_sec_session_private_data(sess, sess_private_data);
+
+ return ret;
+}
+
+/* Clear the memory of session so it doesn't leave key material behind */
+static int
+caam_jr_security_session_destroy(void *dev __rte_unused,
+ struct rte_security_session *sess)
+{
+ PMD_INIT_FUNC_TRACE();
+ void *sess_priv = get_sec_session_private_data(sess);
+
+ struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
+
+ if (sess_priv) {
+ struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
+
+ rte_free(s->cipher_key.data);
+ rte_free(s->auth_key.data);
+ memset(sess, 0, sizeof(struct caam_jr_session));
+ set_sec_session_private_data(sess, NULL);
+ rte_mempool_put(sess_mp, sess_priv);
+ }
+ return 0;
+}
+
+
+static int
+caam_jr_dev_configure(struct rte_cryptodev *dev,
+ struct rte_cryptodev_config *config __rte_unused)
+{
+ char str[20];
+ struct sec_job_ring_t *internals;
+
+ PMD_INIT_FUNC_TRACE();
+
+ internals = dev->data->dev_private;
+ sprintf(str, "ctx_pool_%d", dev->data->dev_id);
+ if (!internals->ctx_pool) {
+ internals->ctx_pool = rte_mempool_create((const char *)str,
+ CTX_POOL_NUM_BUFS,
+ sizeof(struct caam_jr_op_ctx),
+ CTX_POOL_CACHE_SIZE, 0,
+ NULL, NULL, NULL, NULL,
+ SOCKET_ID_ANY, 0);
+ if (!internals->ctx_pool) {
+ CAAM_JR_ERR("%s create failed\n", str);
+ return -ENOMEM;
+ }
+ } else
+ CAAM_JR_INFO("mempool already created for dev_id : %d",
+ dev->data->dev_id);
+
+ return 0;
+}
+
+static int
+caam_jr_dev_start(struct rte_cryptodev *dev __rte_unused)
+{
+ PMD_INIT_FUNC_TRACE();
+ return 0;
+}
+
+static void
+caam_jr_dev_stop(struct rte_cryptodev *dev __rte_unused)
+{
+ PMD_INIT_FUNC_TRACE();
+}
+
+static int
+caam_jr_dev_close(struct rte_cryptodev *dev)
+{
+ struct sec_job_ring_t *internals;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (dev == NULL)
+ return -ENOMEM;
+
+ internals = dev->data->dev_private;
+ rte_mempool_free(internals->ctx_pool);
+ internals->ctx_pool = NULL;
+
+ return 0;
+}
+
+static void
+caam_jr_dev_infos_get(struct rte_cryptodev *dev,
+ struct rte_cryptodev_info *info)
+{
+ struct sec_job_ring_t *internals = dev->data->dev_private;
+
+ PMD_INIT_FUNC_TRACE();
+ if (info != NULL) {
+ info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
+ info->feature_flags = dev->feature_flags;
+ info->capabilities = caam_jr_get_cryptodev_capabilities();
+ info->sym.max_nb_sessions = internals->max_nb_sessions;
+ info->driver_id = cryptodev_driver_id;
+ }
+}
+
+static struct rte_cryptodev_ops caam_jr_ops = {
+ .dev_configure = caam_jr_dev_configure,
+ .dev_start = caam_jr_dev_start,
+ .dev_stop = caam_jr_dev_stop,
+ .dev_close = caam_jr_dev_close,
+ .dev_infos_get = caam_jr_dev_infos_get,
+ .stats_get = caam_jr_stats_get,
+ .stats_reset = caam_jr_stats_reset,
+ .queue_pair_setup = caam_jr_queue_pair_setup,
+ .queue_pair_release = caam_jr_queue_pair_release,
+ .queue_pair_count = caam_jr_queue_pair_count,
+ .sym_session_get_size = caam_jr_sym_session_get_size,
+ .sym_session_configure = caam_jr_sym_session_configure,
+ .sym_session_clear = caam_jr_sym_session_clear
+};
+
+static struct rte_security_ops caam_jr_security_ops = {
+ .session_create = caam_jr_security_session_create,
+ .session_update = NULL,
+ .session_stats_get = NULL,
+ .session_destroy = caam_jr_security_session_destroy,
+ .set_pkt_metadata = NULL,
+ .capabilities_get = caam_jr_get_security_capabilities
+};
+
+/* @brief Flush job rings of any processed descs.
+ * The processed descs are silently dropped,
+ * WITHOUT being notified to UA.
+ */
+static void
+close_job_ring(struct sec_job_ring_t *job_ring)
+{
+ PMD_INIT_FUNC_TRACE();
+ if (job_ring->irq_fd) {
+ /* Producer index is frozen. If consumer index is not equal
+ * with producer index, then we have descs to flush.
+ */
+ while (job_ring->pidx != job_ring->cidx)
+ hw_flush_job_ring(job_ring, false, NULL);
+
+ /* free the uio job ring */
+ free_job_ring(job_ring->irq_fd);
+ job_ring->irq_fd = 0;
+ caam_jr_dma_free(job_ring->input_ring);
+ caam_jr_dma_free(job_ring->output_ring);
+ g_job_rings_no--;
+ }
+}
+
+/** @brief Release the software and hardware resources tied to a job ring.
+ * @param [in] job_ring The job ring
+ *
+ * @retval 0 for success
+ * @retval -1 for error
+ */
+static int
+shutdown_job_ring(struct sec_job_ring_t *job_ring)
+{
+ int ret = 0;
+
+ PMD_INIT_FUNC_TRACE();
+ ASSERT(job_ring != NULL);
+ ret = hw_shutdown_job_ring(job_ring);
+ SEC_ASSERT(ret == 0, ret,
+ "Failed to shutdown hardware job ring %p",
+ job_ring);
+
+ if (job_ring->coalescing_en)
+ hw_job_ring_disable_coalescing(job_ring);
+
+ if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) {
+ ret = caam_jr_disable_irqs(job_ring->irq_fd);
+ SEC_ASSERT(ret == 0, ret,
+ "Failed to disable irqs for job ring %p",
+ job_ring);
+ }
+
+ return ret;
+}
+
+/*
+ * @brief Release the resources used by the SEC user space driver.
+ *
+ * Reset and release SEC's job rings indicated by the User Application at
+ * init_job_ring() and free any memory allocated internally.
+ * Call once during application tear down.
+ *
+ * @note In case there are any descriptors in-flight (descriptors received by
+ * SEC driver for processing and for which no response was yet provided to UA),
+ * the descriptors are discarded without any notifications to User Application.
+ *
+ * @retval ::0 is returned for a successful execution
+ * @retval ::-1 is returned if SEC driver release is in progress
+ */
+static int
+caam_jr_dev_uninit(struct rte_cryptodev *dev)
+{
+ struct sec_job_ring_t *internals;
+
+ PMD_INIT_FUNC_TRACE();
+ if (dev == NULL)
+ return -ENODEV;
+
+ internals = dev->data->dev_private;
+ rte_free(dev->security_ctx);
+
+ /* If any descriptors in flight , poll and wait
+ * until all descriptors are received and silently discarded.
+ */
+ if (internals) {
+ shutdown_job_ring(internals);
+ close_job_ring(internals);
+ rte_mempool_free(internals->ctx_pool);
+ }
+
+ CAAM_JR_INFO("Closing crypto device %s", dev->data->name);
+
+ /* last caam jr instance) */
+ if (g_job_rings_no == 0)
+ g_driver_state = SEC_DRIVER_STATE_IDLE;
+
+ return SEC_SUCCESS;
+}
+
+/* @brief Initialize the software and hardware resources tied to a job ring.
+ * @param [in] jr_mode; Model to be used by SEC Driver to receive
+ * notifications from SEC. Can be either
+ * of the three: #SEC_NOTIFICATION_TYPE_NAPI
+ * #SEC_NOTIFICATION_TYPE_IRQ or
+ * #SEC_NOTIFICATION_TYPE_POLL
+ * @param [in] NAPI_mode The NAPI work mode to configure a job ring at
+ * startup. Used only when #SEC_NOTIFICATION_TYPE
+ * is set to #SEC_NOTIFICATION_TYPE_NAPI.
+ * @param [in] irq_coalescing_timer This value determines the maximum
+ * amount of time after processing a
+ * descriptor before raising an interrupt.
+ * @param [in] irq_coalescing_count This value determines how many
+ * descriptors are completed before
+ * raising an interrupt.
+ * @param [in] reg_base_addr, The job ring base address register
+ * @param [in] irq_id The job ring interrupt identification number.
+ * @retval job_ring_handle for successful job ring configuration
+ * @retval NULL on error
+ *
+ */
+static void *
+init_job_ring(void *reg_base_addr, uint32_t irq_id)
+{
+ struct sec_job_ring_t *job_ring = NULL;
+ int i, ret = 0;
+ int jr_mode = SEC_NOTIFICATION_TYPE_POLL;
+ int napi_mode = 0;
+ int irq_coalescing_timer = 0;
+ int irq_coalescing_count = 0;
+
+ for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
+ if (g_job_rings[i].irq_fd == 0) {
+ job_ring = &g_job_rings[i];
+ g_job_rings_no++;
+ break;
+ }
+ }
+ if (job_ring == NULL) {
+ CAAM_JR_ERR("No free job ring\n");
+ return NULL;
+ }
+
+ job_ring->register_base_addr = reg_base_addr;
+ job_ring->jr_mode = jr_mode;
+ job_ring->napi_mode = 0;
+ job_ring->irq_fd = irq_id;
+
+ /* Allocate mem for input and output ring */
+
+ /* Allocate memory for input ring */
+ job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
+ SEC_DMA_MEM_INPUT_RING_SIZE);
+ memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE);
+
+ /* Allocate memory for output ring */
+ job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
+ SEC_DMA_MEM_OUTPUT_RING_SIZE);
+ memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE);
+
+ /* Reset job ring in SEC hw and configure job ring registers */
+ ret = hw_reset_job_ring(job_ring);
+ if (ret != 0) {
+ CAAM_JR_ERR("Failed to reset hardware job ring");
+ goto cleanup;
+ }
+
+ if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
+ /* When SEC US driver works in NAPI mode, the UA can select
+ * if the driver starts with IRQs on or off.
+ */
+ if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) {
+ CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p",
+ job_ring);
+ ret = caam_jr_enable_irqs(job_ring->irq_fd);
+ if (ret != 0) {
+ CAAM_JR_ERR("Failed to enable irqs for job ring");
+ goto cleanup;
+ }
+ }
+ } else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
+ /* When SEC US driver works in pure interrupt mode,
+ * IRQ's are always enabled.
+ */
+ CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p",
+ job_ring);
+ ret = caam_jr_enable_irqs(job_ring->irq_fd);
+ if (ret != 0) {
+ CAAM_JR_ERR("Failed to enable irqs for job ring");
+ goto cleanup;
+ }
+ }
+ if (irq_coalescing_timer || irq_coalescing_count) {
+ hw_job_ring_set_coalescing_param(job_ring,
+ irq_coalescing_timer,
+ irq_coalescing_count);
+
+ hw_job_ring_enable_coalescing(job_ring);
+ job_ring->coalescing_en = 1;
+ }
+
+ job_ring->jr_state = SEC_JOB_RING_STATE_STARTED;
+ job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS;
+ job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS;
+
+ return job_ring;
+cleanup:
+ caam_jr_dma_free(job_ring->output_ring);
+ caam_jr_dma_free(job_ring->input_ring);
+ return NULL;
+}
+
+
+static int
+caam_jr_dev_init(const char *name,
+ struct rte_vdev_device *vdev,
+ struct rte_cryptodev_pmd_init_params *init_params)
+{
+ struct rte_cryptodev *dev;
+ struct rte_security_ctx *security_instance;
+ struct uio_job_ring *job_ring;
+ char str[RTE_CRYPTODEV_NAME_MAX_LEN];
+
+ PMD_INIT_FUNC_TRACE();
+
+ /* Validate driver state */
+ if (g_driver_state == SEC_DRIVER_STATE_IDLE) {
+ g_job_rings_max = sec_configure();
+ if (!g_job_rings_max) {
+ CAAM_JR_ERR("No job ring detected on UIO !!!!");
+ return -1;
+ }
+ /* Update driver state */
+ g_driver_state = SEC_DRIVER_STATE_STARTED;
+ }
+
+ if (g_job_rings_no >= g_job_rings_max) {
+ CAAM_JR_ERR("No more job rings available max=%d!!!!",
+ g_job_rings_max);
+ return -1;
+ }
+
+ job_ring = config_job_ring();
+ if (job_ring == NULL) {
+ CAAM_JR_ERR("failed to create job ring");
+ goto init_error;
+ }
+
+ snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id);
+
+ dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
+ if (dev == NULL) {
+ CAAM_JR_ERR("failed to create cryptodev vdev");
+ goto cleanup;
+ }
+ /*TODO free it during teardown*/
+ dev->data->dev_private = init_job_ring(job_ring->register_base_addr,
+ job_ring->uio_fd);
+
+ if (!dev->data->dev_private) {
+ CAAM_JR_ERR("Ring memory allocation failed\n");
+ goto cleanup2;
+ }
+
+ dev->driver_id = cryptodev_driver_id;
+ dev->dev_ops = &caam_jr_ops;
+
+ /* register rx/tx burst functions for data path */
+ dev->dequeue_burst = caam_jr_dequeue_burst;
+ dev->enqueue_burst = caam_jr_enqueue_burst;
+ dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+ RTE_CRYPTODEV_FF_HW_ACCELERATED |
+ RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
+ RTE_CRYPTODEV_FF_SECURITY |
+ RTE_CRYPTODEV_FF_IN_PLACE_SGL |
+ RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
+ RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
+ RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
+ RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
+
+ /* For secondary processes, we don't initialise any further as primary
+ * has already done this work. Only check we don't need a different
+ * RX function
+ */
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ CAAM_JR_WARN("Device already init by primary process");
+ return 0;
+ }
+
+ /*TODO free it during teardown*/
+ security_instance = rte_malloc("caam_jr",
+ sizeof(struct rte_security_ctx), 0);
+ if (security_instance == NULL) {
+ CAAM_JR_ERR("memory allocation failed\n");
+ //todo error handling.
+ goto cleanup2;
+ }
+
+ security_instance->device = (void *)dev;
+ security_instance->ops = &caam_jr_security_ops;
+ security_instance->sess_cnt = 0;
+ dev->security_ctx = security_instance;
+
+ RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name);
+
+ return 0;
+
+cleanup2:
+ caam_jr_dev_uninit(dev);
+ rte_cryptodev_pmd_release_device(dev);
+cleanup:
+ free_job_ring(job_ring->uio_fd);
+init_error:
+ CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed",
+ init_params->name);
+
+ return -ENXIO;
+}
+
+/** Initialise CAAM JR crypto device */
+static int
+cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
+{
+ struct rte_cryptodev_pmd_init_params init_params = {
+ "",
+ sizeof(struct sec_job_ring_t),
+ rte_socket_id(),
+ RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
+ };
+ const char *name;
+ const char *input_args;
+
+ name = rte_vdev_device_name(vdev);
+ if (name == NULL)
+ return -EINVAL;
+
+ input_args = rte_vdev_device_args(vdev);
+ rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
+
+ /* if sec device version is not configured */
+ if (!rta_get_sec_era()) {
+ const struct device_node *caam_node;
+
+ for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
+ const uint32_t *prop = of_get_property(caam_node,
+ "fsl,sec-era",
+ NULL);
+ if (prop) {
+ rta_set_sec_era(
+ INTL_SEC_ERA(cpu_to_caam32(*prop)));
+ break;
+ }
+ }
+ }
+#ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
+ if (rta_get_sec_era() > RTA_SEC_ERA_8) {
+ RTE_LOG(ERR, PMD,
+ "CAAM is compiled in BE mode for device with sec era > 8???\n");
+ return -EINVAL;
+ }
+#endif
+
+ return caam_jr_dev_init(name, vdev, &init_params);
+}
+
+/** Uninitialise CAAM JR crypto device */
+static int
+cryptodev_caam_jr_remove(struct rte_vdev_device *vdev)
+{
+ struct rte_cryptodev *cryptodev;
+ const char *name;
+
+ name = rte_vdev_device_name(vdev);
+ if (name == NULL)
+ return -EINVAL;
+
+ cryptodev = rte_cryptodev_pmd_get_named_dev(name);
+ if (cryptodev == NULL)
+ return -ENODEV;
+
+ caam_jr_dev_uninit(cryptodev);
+
+ return rte_cryptodev_pmd_destroy(cryptodev);
+}
+
+static struct rte_vdev_driver cryptodev_caam_jr_drv = {
+ .probe = cryptodev_caam_jr_probe,
+ .remove = cryptodev_caam_jr_remove
+};
+
+static struct cryptodev_driver caam_jr_crypto_drv;
+
+RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv);
+RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD,
+ "max_nb_queue_pairs=<int>"
+ "socket_id=<int>");
+RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver,
+ cryptodev_driver_id);
+
+RTE_INIT(caam_jr_init_log)
+{
+ caam_jr_logtype = rte_log_register("pmd.crypto.caam");
+ if (caam_jr_logtype >= 0)
+ rte_log_set_level(caam_jr_logtype, RTE_LOG_NOTICE);
+}
diff --git a/drivers/crypto/caam_jr/caam_jr_capabilities.c b/drivers/crypto/caam_jr/caam_jr_capabilities.c
new file mode 100644
index 00000000..c51593c4
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_capabilities.c
@@ -0,0 +1,266 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#include <caam_jr_capabilities.h>
+
+static const struct rte_cryptodev_capabilities caam_jr_capabilities[] = {
+ { /* MD5 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
+ .block_size = 64,
+ .key_size = {
+ .min = 1,
+ .max = 64,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 16,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA1 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
+ .block_size = 64,
+ .key_size = {
+ .min = 1,
+ .max = 64,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 20,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA224 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
+ .block_size = 64,
+ .key_size = {
+ .min = 1,
+ .max = 64,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 28,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA256 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
+ .block_size = 64,
+ .key_size = {
+ .min = 1,
+ .max = 64,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 32,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA384 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
+ .block_size = 128,
+ .key_size = {
+ .min = 1,
+ .max = 128,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 48,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA512 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
+ .block_size = 128,
+ .key_size = {
+ .min = 1,
+ .max = 128,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 64,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* AES GCM */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+ {.aead = {
+ .algo = RTE_CRYPTO_AEAD_AES_GCM,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 32,
+ .increment = 8
+ },
+ .digest_size = {
+ .min = 8,
+ .max = 16,
+ .increment = 4
+ },
+ .aad_size = {
+ .min = 0,
+ .max = 240,
+ .increment = 1
+ },
+ .iv_size = {
+ .min = 12,
+ .max = 12,
+ .increment = 0
+ },
+ }, }
+ }, }
+ },
+ { /* AES CBC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_AES_CBC,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 32,
+ .increment = 8
+ },
+ .iv_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+ { /* AES CTR */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_AES_CTR,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 32,
+ .increment = 8
+ },
+ .iv_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+ { /* 3DES CBC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_3DES_CBC,
+ .block_size = 8,
+ .key_size = {
+ .min = 16,
+ .max = 24,
+ .increment = 8
+ },
+ .iv_size = {
+ .min = 8,
+ .max = 8,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+
+ RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
+static const struct rte_security_capability caam_jr_security_cap[] = {
+ { /* IPsec Lookaside Protocol offload ESP Transport Egress */
+ .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
+ .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+ .ipsec = {
+ .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+ .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
+ .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
+ .options = { 0 }
+ },
+ .crypto_capabilities = caam_jr_capabilities
+ },
+ { /* IPsec Lookaside Protocol offload ESP Tunnel Ingress */
+ .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
+ .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+ .ipsec = {
+ .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+ .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
+ .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
+ .options = { 0 }
+ },
+ .crypto_capabilities = caam_jr_capabilities
+ },
+ {
+ .action = RTE_SECURITY_ACTION_TYPE_NONE
+ }
+};
+
+const struct rte_cryptodev_capabilities *
+caam_jr_get_cryptodev_capabilities(void)
+{
+ return caam_jr_capabilities;
+}
+
+const struct rte_security_capability *
+caam_jr_get_security_capabilities(void *device __rte_unused)
+{
+ return caam_jr_security_cap;
+}
diff --git a/drivers/crypto/caam_jr/caam_jr_capabilities.h b/drivers/crypto/caam_jr/caam_jr_capabilities.h
new file mode 100644
index 00000000..c1e3f305
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_capabilities.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#ifndef CAAM_JR_CAPABILITIES_H
+#define CAAM_JR_CAPABILITIES_H
+
+#include <rte_cryptodev.h>
+#include <rte_security.h>
+
+/* Get cryptodev capabilities */
+const struct rte_cryptodev_capabilities *
+caam_jr_get_cryptodev_capabilities(void);
+/* Get security capabilities */
+const struct rte_security_capability *
+caam_jr_get_security_capabilities(void *device);
+
+#endif
diff --git a/drivers/crypto/caam_jr/caam_jr_config.h b/drivers/crypto/caam_jr/caam_jr_config.h
new file mode 100644
index 00000000..041187a8
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_config.h
@@ -0,0 +1,207 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#ifndef CAAM_JR_CONFIG_H
+#define CAAM_JR_CONFIG_H
+
+#include <rte_byteorder.h>
+
+#include <compat.h>
+
+#ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
+#define CAAM_BYTE_ORDER __BIG_ENDIAN
+#else
+#define CAAM_BYTE_ORDER __LITTLE_ENDIAN
+#endif
+
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+#define CORE_BYTE_ORDER __BIG_ENDIAN
+#else
+#define CORE_BYTE_ORDER __LITTLE_ENDIAN
+#endif
+
+#if CORE_BYTE_ORDER != CAAM_BYTE_ORDER
+
+#define cpu_to_caam64 rte_cpu_to_be_64
+#define cpu_to_caam32 rte_cpu_to_be_32
+#else
+#define cpu_to_caam64
+#define cpu_to_caam32
+
+#endif
+
+/*
+ * SEC is configured to start work in polling mode,
+ * when configured for NAPI notification style.
+ */
+#define SEC_STARTUP_POLLING_MODE 0
+/*
+ * SEC is configured to start work in interrupt mode,
+ * when configured for NAPI notification style.
+ */
+#define SEC_STARTUP_INTERRUPT_MODE 1
+
+/*
+ * SEC driver will use NAPI model to receive notifications
+ * for processed packets from SEC engine hardware:
+ * - IRQ for low traffic
+ * - polling for high traffic.
+ */
+#define SEC_NOTIFICATION_TYPE_NAPI 0
+/*
+ * SEC driver will use ONLY interrupts to receive notifications
+ * for processed packets from SEC engine hardware.
+ */
+#define SEC_NOTIFICATION_TYPE_IRQ 1
+/*
+ * SEC driver will use ONLY polling to receive notifications
+ * for processed packets from SEC engine hardware.
+ */
+#define SEC_NOTIFICATION_TYPE_POLL 2
+
+/*
+ * SEC USER SPACE DRIVER related configuration.
+ */
+
+/*
+ * Determines how SEC user space driver will receive notifications
+ * for processed packets from SEC engine.
+ * Valid values are: #SEC_NOTIFICATION_TYPE_POLL, #SEC_NOTIFICATION_TYPE_IRQ
+ * and #SEC_NOTIFICATION_TYPE_NAPI.
+ */
+#define SEC_NOTIFICATION_TYPE SEC_NOTIFICATION_TYPE_POLL
+
+/* Maximum number of job rings supported by SEC hardware */
+#define MAX_SEC_JOB_RINGS 4
+
+/* Maximum number of QP per job ring */
+#define RTE_CAAM_MAX_NB_SEC_QPS 1
+
+/*
+ * Size of cryptographic context that is used directly in communicating
+ * with SEC device. SEC device works only with physical addresses. This
+ * is the maximum size for a SEC descriptor ( = 64 words).
+ */
+#define SEC_CRYPTO_DESCRIPTOR_SIZE 256
+
+/*
+ * Size of job descriptor submitted to SEC device for each packet to
+ * be processed.
+ * Job descriptor contains 3 DMA address pointers:
+ * - to shared descriptor, to input buffer and to output buffer.
+ * The job descriptor contains other SEC specific commands as well:
+ * - HEADER command, SEQ IN PTR command SEQ OUT PTR command and opaque data
+ * each measuring 4 bytes.
+ * Job descriptor size, depending on physical address representation:
+ * - 32 bit - size is 28 bytes - cacheline-aligned size is 64 bytes
+ * - 36 bit - size is 40 bytes - cacheline-aligned size is 64 bytes
+ * @note: Job descriptor must be cacheline-aligned to ensure efficient
+ * memory access.
+ * @note: If other format is used for job descriptor, then the size must be
+ * revised.
+ */
+#define SEC_JOB_DESCRIPTOR_SIZE 64
+
+/*
+ * Size of one entry in the input ring of a job ring.
+ * Input ring contains pointers to job descriptors.
+ * The memory used for an input ring and output ring must be physically
+ * contiguous.
+ */
+#define SEC_JOB_INPUT_RING_ENTRY_SIZE sizeof(dma_addr_t)
+
+/*
+ * Size of one entry in the output ring of a job ring.
+ * Output ring entry is a pointer to a job descriptor followed by a 4 byte
+ * status word.
+ * The memory used for an input ring and output ring must be physically
+ * contiguous.
+ * @note If desired to use also the optional SEQ OUT indication in output ring
+ * entries,
+ * then 4 more bytes must be added to the size.
+ */
+#define SEC_JOB_OUTPUT_RING_ENTRY_SIZE (SEC_JOB_INPUT_RING_ENTRY_SIZE + 4)
+
+/*
+ * DMA memory required for an input ring of a job ring.
+ */
+#define SEC_DMA_MEM_INPUT_RING_SIZE ((SEC_JOB_INPUT_RING_ENTRY_SIZE) * \
+ (SEC_JOB_RING_SIZE))
+
+/*
+ * DMA memory required for an output ring of a job ring.
+ * Required extra 4 byte for status word per each entry.
+ */
+#define SEC_DMA_MEM_OUTPUT_RING_SIZE ((SEC_JOB_OUTPUT_RING_ENTRY_SIZE) * \
+ (SEC_JOB_RING_SIZE))
+
+/* DMA memory required for a job ring, including both input and output rings. */
+#define SEC_DMA_MEM_JOB_RING_SIZE ((SEC_DMA_MEM_INPUT_RING_SIZE) + \
+ (SEC_DMA_MEM_OUTPUT_RING_SIZE))
+
+/*
+ * When calling sec_init() UA will provide an area of virtual memory
+ * of size #SEC_DMA_MEMORY_SIZE to be used internally by the driver
+ * to allocate data (like SEC descriptors) that needs to be passed to
+ * SEC device in physical addressing and later on retrieved from SEC device.
+ * At initialization the UA provides specialized ptov/vtop functions/macros to
+ * translate addresses allocated from this memory area.
+ */
+#define SEC_DMA_MEMORY_SIZE ((SEC_DMA_MEM_JOB_RING_SIZE) * \
+ (MAX_SEC_JOB_RINGS))
+
+#define L1_CACHE_BYTES 64
+
+/* SEC JOB RING related configuration. */
+
+/*
+ * Configure the size of the JOB RING.
+ * The maximum size of the ring in hardware limited to 1024.
+ * However the number of packets in flight in a time interval of 1ms can
+ * be calculated from the traffic rate (Mbps) and packet size.
+ * Here it was considered a packet size of 64 bytes.
+ *
+ * @note Round up to nearest power of 2 for optimized update
+ * of producer/consumer indexes of each job ring
+ */
+#define SEC_JOB_RING_SIZE 512
+
+/*
+ * Interrupt coalescing related configuration.
+ * NOTE: SEC hardware enabled interrupt
+ * coalescing is not supported on SEC version 3.1!
+ * SEC version 4.4 has support for interrupt
+ * coalescing.
+ */
+
+#if SEC_NOTIFICATION_TYPE != SEC_NOTIFICATION_TYPE_POLL
+
+#define SEC_INT_COALESCING_ENABLE 1
+/*
+ * Interrupt Coalescing Descriptor Count Threshold.
+ * While interrupt coalescing is enabled (ICEN=1), this value determines
+ * how many Descriptors are completed before raising an interrupt.
+ *
+ * Valid values for this field are from 0 to 255.
+ * Note that a value of 1 functionally defeats the advantages of interrupt
+ * coalescing since the threshold value is reached each time that a
+ * Job Descriptor is completed. A value of 0 is treated in the same
+ * manner as a value of 1.
+ */
+#define SEC_INTERRUPT_COALESCING_DESCRIPTOR_COUNT_THRESH 10
+
+/*
+ * Interrupt Coalescing Timer Threshold.
+ * While interrupt coalescing is enabled (ICEN=1), this value determines the
+ * maximum amount of time after processing a Descriptor before raising an
+ * interrupt.
+ * The threshold value is represented in units equal to 64 CAAM interface
+ * clocks. Valid values for this field are from 1 to 65535.
+ * A value of 0 results in behavior identical to that when interrupt
+ * coalescing is disabled.
+ */
+#define SEC_INTERRUPT_COALESCING_TIMER_THRESH 100
+#endif /* SEC_NOTIFICATION_TYPE_POLL */
+
+#endif /* CAAM_JR_CONFIG_H */
diff --git a/drivers/crypto/caam_jr/caam_jr_desc.h b/drivers/crypto/caam_jr/caam_jr_desc.h
new file mode 100644
index 00000000..6683ea83
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_desc.h
@@ -0,0 +1,285 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#ifndef CAAM_JR_DESC_H
+#define CAAM_JR_DESC_H
+
+#define CMD_HDR_CTYPE_SD 0x16
+#define CMD_HDR_CTYPE_JD 0x17
+
+/* The maximum size of a SEC descriptor, in WORDs (32 bits). */
+#define MAX_DESC_SIZE_WORDS 64
+
+/*
+ * Macros manipulating descriptors
+ */
+/* Macro for setting the SD pointer in a JD. Common for all protocols
+ * supported by the SEC driver.
+ */
+#define SEC_JD_SET_SD(descriptor, ptr, len) { \
+ (descriptor)->sd_ptr = (ptr); \
+ (descriptor)->deschdr.command.jd.shr_desc_len = (len); \
+}
+
+/* Macro for setting a pointer to the job which this descriptor processes.
+ * It eases the lookup procedure for identifying the descriptor that has
+ * completed.
+ */
+#define SEC_JD_SET_JOB_PTR(descriptor, ptr) \
+ ((descriptor)->job_ptr = (ptr))
+
+/* Macro for setting up a JD. The structure of the JD is common across all
+ * supported protocols, thus its structure is identical.
+ */
+#define SEC_JD_INIT(descriptor) ({ \
+ /* CTYPE = job descriptor \
+ * RSMS, DNR = 0
+ * ONE = 1
+ * Start Index = 0
+ * ZRO,TD, MTD = 0
+ * SHR = 1 (there's a shared descriptor referenced
+ * by this job descriptor,pointer in next word)
+ * REO = 1 (execute job descr. first, shared descriptor
+ * after)
+ * SHARE = DEFER
+ * Descriptor Length = 0 ( to be completed @ runtime ) */ \
+ (descriptor)->deschdr.command.word = 0xB0801C0D; \
+ /*
+ * CTYPE = SEQ OUT command * Scater Gather Flag = 0
+ * (can be updated @ runtime) PRE = 0 * EXT = 1
+ * (data length is in next word, following the * command)
+ * RTO = 0 */ \
+ (descriptor)->seq_out.command.word = 0xF8400000; /**/ \
+ /*
+ * CTYPE = SEQ IN command
+ * Scater Gather Flag = 0 (can be updated @ runtime)
+ * PRE = 0
+ * EXT = 1 ( data length is in next word, following the
+ * command)
+ * RTO = 0 */ \
+ (descriptor)->seq_in.command.word = 0xF0400000; /**/ \
+ /*
+ * In order to be compatible with QI scenarios, the DPOVRD value
+ * loaded must be formated like this:
+ * DPOVRD_EN (1b) | Res| DPOVRD Value (right aligned). */ \
+ (descriptor)->load_dpovrd.command.word = 0x16870004; \
+ /* By default, DPOVRD mechanism is disabled, thus the value to be
+ * LOAD-ed through the above descriptor command will be
+ * 0x0000_0000. */ \
+ (descriptor)->dpovrd = 0x00000000; \
+})
+
+/* Macro for setting the pointer to the input buffer in the JD, according to
+ * the parameters set by the user in the ::sec_packet_t structure.
+ */
+#define SEC_JD_SET_IN_PTR(descriptor, phys_addr, offset, length) { \
+ (descriptor)->seq_in_ptr = (phys_addr) + (offset); \
+ (descriptor)->in_ext_length = (length); \
+}
+
+/* Macro for setting the pointer to the output buffer in the JD, according to
+ * the parameters set by the user in the ::sec_packet_t structure.
+ */
+#define SEC_JD_SET_OUT_PTR(descriptor, phys_addr, offset, length) { \
+ (descriptor)->seq_out_ptr = (phys_addr) + (offset); \
+ (descriptor)->out_ext_length = (length); \
+}
+
+/* Macro for setting the Scatter-Gather flag in the SEQ IN command. Used in
+ * case the input buffer is split in multiple buffers, according to the user
+ * specification.
+ */
+#define SEC_JD_SET_SG_IN(descriptor) \
+ ((descriptor)->seq_in.command.field.sgf = 1)
+
+/* Macro for setting the Scatter-Gather flag in the SEQ OUT command. Used in
+ * case the output buffer is split in multiple buffers, according to the user
+ * specification.
+ */
+#define SEC_JD_SET_SG_OUT(descriptor) \
+ ((descriptor)->seq_out.command.field.sgf = 1)
+
+#define SEC_JD_SET_DPOVRD(descriptor) \
+
+/* Macro for retrieving a descriptor's length. Works for both SD and JD. */
+#define SEC_GET_DESC_LEN(descriptor) \
+ (((struct descriptor_header_s *)(descriptor))->command.sd.ctype == \
+ CMD_HDR_CTYPE_SD ? ((struct descriptor_header_s *) \
+ (descriptor))->command.sd.desclen : \
+ ((struct descriptor_header_s *)(descriptor))->command.jd.desclen)
+
+/* Helper macro for dumping the hex representation of a descriptor */
+#define SEC_DUMP_DESC(descriptor) { \
+ int __i; \
+ CAAM_JR_INFO("Des@ 0x%08x\n", (uint32_t)((uint32_t *)(descriptor)));\
+ for (__i = 0; \
+ __i < SEC_GET_DESC_LEN(descriptor); \
+ __i++) { \
+ printf("0x%08x: 0x%08x\n", \
+ (uint32_t)(((uint32_t *)(descriptor)) + __i), \
+ *(((uint32_t *)(descriptor)) + __i)); \
+ } \
+}
+/* Union describing a descriptor header.
+ */
+struct descriptor_header_s {
+ union {
+ uint32_t word;
+ struct {
+ /* 4 */ unsigned int ctype:5;
+ /* 5 */ unsigned int res1:2;
+ /* 7 */ unsigned int dnr:1;
+ /* 8 */ unsigned int one:1;
+ /* 9 */ unsigned int res2:1;
+ /* 10 */ unsigned int start_idx:6;
+ /* 16 */ unsigned int res3:2;
+ /* 18 */ unsigned int cif:1;
+ /* 19 */ unsigned int sc:1;
+ /* 20 */ unsigned int pd:1;
+ /* 21 */ unsigned int res4:1;
+ /* 22 */ unsigned int share:2;
+ /* 24 */ unsigned int res5:2;
+ /* 26 */ unsigned int desclen:6;
+ } sd;
+ struct {
+ /* TODO only below struct members are corrected,
+ * all others also need to be reversed please verify it
+ */
+ /* 0 */ unsigned int desclen:7;
+ /* 7 */ unsigned int res4:1;
+ /* 8 */ unsigned int share:3;
+ /* 11 */ unsigned int reo:1;
+ /* 12 */ unsigned int shr:1;
+ /* 13 */ unsigned int mtd:1;
+ /* 14 */ unsigned int td:1;
+ /* 15 */ unsigned int zero:1;
+ /* 16 */ unsigned int shr_desc_len:6;
+ /* 22 */ unsigned int res2:1;
+ /* 23 */ unsigned int one:1;
+ /* 24 */ unsigned int dnr:1;
+ /* 25 */ unsigned int rsms:1;
+ /* 26 */ unsigned int res1:1;
+ /* 27 */ unsigned int ctype:5;
+ } jd;
+ } __rte_packed command;
+} __rte_packed;
+
+/* Union describing a KEY command in a descriptor.
+ */
+struct key_command_s {
+ union {
+ uint32_t word;
+ struct {
+ unsigned int ctype:5;
+ unsigned int cls:2;
+ unsigned int sgf:1;
+ unsigned int imm:1;
+ unsigned int enc:1;
+ unsigned int nwb:1;
+ unsigned int ekt:1;
+ unsigned int kdest:4;
+ unsigned int tk:1;
+ unsigned int rsvd1:5;
+ unsigned int length:10;
+ } __rte_packed field;
+ } __rte_packed command;
+} __rte_packed;
+
+/* Union describing a PROTOCOL command
+ * in a descriptor.
+ */
+struct protocol_operation_command_s {
+ union {
+ uint32_t word;
+ struct {
+ unsigned int ctype:5;
+ unsigned int optype:3;
+ unsigned char protid;
+ unsigned short protinfo;
+ } __rte_packed field;
+ } __rte_packed command;
+} __rte_packed;
+
+/* Union describing a SEQIN command in a
+ * descriptor.
+ */
+struct seq_in_command_s {
+ union {
+ uint32_t word;
+ struct {
+ unsigned int ctype:5;
+ unsigned int res1:1;
+ unsigned int inl:1;
+ unsigned int sgf:1;
+ unsigned int pre:1;
+ unsigned int ext:1;
+ unsigned int rto:1;
+ unsigned int rjd:1;
+ unsigned int res2:4;
+ unsigned int length:16;
+ } field;
+ } __rte_packed command;
+} __rte_packed;
+
+/* Union describing a SEQOUT command in a
+ * descriptor.
+ */
+struct seq_out_command_s {
+ union {
+ uint32_t word;
+ struct {
+ unsigned int ctype:5;
+ unsigned int res1:2;
+ unsigned int sgf:1;
+ unsigned int pre:1;
+ unsigned int ext:1;
+ unsigned int rto:1;
+ unsigned int res2:5;
+ unsigned int length:16;
+ } field;
+ } __rte_packed command;
+} __rte_packed;
+
+struct load_command_s {
+ union {
+ uint32_t word;
+ struct {
+ unsigned int ctype:5;
+ unsigned int class:2;
+ unsigned int sgf:1;
+ unsigned int imm:1;
+ unsigned int dst:7;
+ unsigned char offset;
+ unsigned char length;
+ } fields;
+ } __rte_packed command;
+} __rte_packed;
+
+/* Structure encompassing a general shared descriptor of maximum
+ * size (64 WORDs). Usually, other specific shared descriptor structures
+ * will be type-casted to this one
+ * this one.
+ */
+struct sec_sd_t {
+ uint32_t rsvd[MAX_DESC_SIZE_WORDS];
+} __attribute__((packed, aligned(64)));
+
+/* Structure encompassing a job descriptor which processes
+ * a single packet from a context. The job descriptor references
+ * a shared descriptor from a SEC context.
+ */
+struct sec_job_descriptor_t {
+ struct descriptor_header_s deschdr;
+ dma_addr_t sd_ptr;
+ struct seq_out_command_s seq_out;
+ dma_addr_t seq_out_ptr;
+ uint32_t out_ext_length;
+ struct seq_in_command_s seq_in;
+ dma_addr_t seq_in_ptr;
+ uint32_t in_ext_length;
+ struct load_command_s load_dpovrd;
+ uint32_t dpovrd;
+} __attribute__((packed, aligned(64)));
+
+#endif
diff --git a/drivers/crypto/caam_jr/caam_jr_hw.c b/drivers/crypto/caam_jr/caam_jr_hw.c
new file mode 100644
index 00000000..4a2b0899
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_hw.c
@@ -0,0 +1,367 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <rte_common.h>
+#include <rte_memory.h>
+#include <rte_malloc.h>
+#include <rte_crypto.h>
+#include <rte_security.h>
+
+#include <caam_jr_config.h>
+#include <caam_jr_hw_specific.h>
+#include <caam_jr_pvt.h>
+#include <caam_jr_log.h>
+
+/* RTA header files */
+#include <hw/desc/common.h>
+#include <hw/desc/algo.h>
+#include <hw/desc/ipsec.h>
+
+/* Used to retry resetting a job ring in SEC hardware. */
+#define SEC_TIMEOUT 100000
+
+/* @brief Process Jump Halt Condition related errors
+ *
+ * @param [in] error_code The error code in the descriptor status word
+ */
+static inline void
+hw_handle_jmp_halt_cond_err(union hw_error_code error_code)
+{
+ CAAM_JR_DEBUG("JMP: %d, Descriptor Index: 0x%x, Condition: 0x%x",
+ error_code.error_desc.jmp_halt_cond_src.jmp,
+ error_code.error_desc.jmp_halt_cond_src.desc_idx,
+ error_code.error_desc.jmp_halt_cond_src.cond);
+ (void)error_code;
+}
+
+/* @brief Process DECO related errors
+ *
+ * @param [in] error_code The error code in the descriptor status word
+ */
+static inline void
+hw_handle_deco_err(union hw_error_code error_code)
+{
+ CAAM_JR_DEBUG("JMP: %d, Descriptor Index: 0x%x",
+ error_code.error_desc.deco_src.jmp,
+ error_code.error_desc.deco_src.desc_idx);
+
+ switch (error_code.error_desc.deco_src.desc_err) {
+ case SEC_HW_ERR_DECO_HFN_THRESHOLD:
+ CAAM_JR_DEBUG(" Warning: Descriptor completed normally,"
+ "but 3GPP HFN matches or exceeds the Threshold ");
+ break;
+ default:
+ CAAM_JR_DEBUG("Error 0x%04x not implemented",
+ error_code.error_desc.deco_src.desc_err);
+ break;
+ }
+}
+
+/* @brief Process Jump Halt User Status related errors
+ *
+ * @param [in] error_code The error code in the descriptor status word
+ */
+static inline void
+hw_handle_jmp_halt_user_err(union hw_error_code error_code __rte_unused)
+{
+ CAAM_JR_DEBUG(" Not implemented");
+}
+
+/* @brief Process CCB related errors
+ *
+ * @param [in] error_code The error code in the descriptor status word
+ */
+static inline void
+hw_handle_ccb_err(union hw_error_code hw_error_code __rte_unused)
+{
+ CAAM_JR_DEBUG(" Not implemented");
+}
+
+/* @brief Process Job Ring related errors
+ *
+ * @param [in] error_code The error code in the descriptor status word
+ */
+static inline void
+hw_handle_jr_err(union hw_error_code hw_error_code __rte_unused)
+{
+ CAAM_JR_DEBUG(" Not implemented");
+}
+
+int
+hw_reset_job_ring(struct sec_job_ring_t *job_ring)
+{
+ int ret = 0;
+
+ ASSERT(job_ring->register_base_addr != NULL);
+
+ /* First reset the job ring in hw */
+ ret = hw_shutdown_job_ring(job_ring);
+ SEC_ASSERT(ret == 0, ret, "Failed resetting job ring in hardware");
+
+ /* In order to have the HW JR in a workable state
+ * after a reset, I need to re-write the input
+ * queue size, input start address, output queue
+ * size and output start address
+ */
+ /* Write the JR input queue size to the HW register */
+ hw_set_input_ring_size(job_ring, SEC_JOB_RING_SIZE);
+
+ /* Write the JR output queue size to the HW register */
+ hw_set_output_ring_size(job_ring, SEC_JOB_RING_SIZE);
+
+ /* Write the JR input queue start address */
+ hw_set_input_ring_start_addr(job_ring,
+ caam_jr_dma_vtop(job_ring->input_ring));
+ CAAM_JR_DEBUG(" Set input ring base address to : Virtual: 0x%" PRIx64
+ ",Physical: 0x%" PRIx64 ", Read from HW: 0x%" PRIx64,
+ (uint64_t)(uintptr_t)job_ring->input_ring,
+ caam_jr_dma_vtop(job_ring->input_ring),
+ hw_get_inp_queue_base(job_ring));
+
+ /* Write the JR output queue start address */
+ hw_set_output_ring_start_addr(job_ring,
+ caam_jr_dma_vtop(job_ring->output_ring));
+ CAAM_JR_DEBUG(" Set output ring base address to: Virtual: 0x%" PRIx64
+ ",Physical: 0x%" PRIx64 ", Read from HW: 0x%" PRIx64,
+ (uint64_t)(uintptr_t)job_ring->output_ring,
+ caam_jr_dma_vtop(job_ring->output_ring),
+ hw_get_out_queue_base(job_ring));
+ return ret;
+}
+
+int
+hw_shutdown_job_ring(struct sec_job_ring_t *job_ring)
+{
+ unsigned int timeout = SEC_TIMEOUT;
+ uint32_t tmp = 0;
+ int usleep_interval = 10;
+
+ if (job_ring->register_base_addr == NULL) {
+ CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init",
+ job_ring);
+ return 0;
+ }
+
+ CAAM_JR_INFO("Resetting Job ring %p", job_ring);
+
+ /*
+ * Mask interrupts since we are going to poll
+ * for reset completion status
+ * Also, at POR, interrupts are ENABLED on a JR, thus
+ * this is the point where I can disable them without
+ * changing the code logic too much
+ */
+ caam_jr_disable_irqs(job_ring->irq_fd);
+
+ /* initiate flush (required prior to reset) */
+ SET_JR_REG(JRCR, job_ring, JR_REG_JRCR_VAL_RESET);
+
+ /* dummy read */
+ tmp = GET_JR_REG(JRCR, job_ring);
+
+ do {
+ tmp = GET_JR_REG(JRINT, job_ring);
+ usleep(usleep_interval);
+ } while (((tmp & JRINT_ERR_HALT_MASK) ==
+ JRINT_ERR_HALT_INPROGRESS) && --timeout);
+
+ CAAM_JR_INFO("JRINT is %x", tmp);
+ if ((tmp & JRINT_ERR_HALT_MASK) != JRINT_ERR_HALT_COMPLETE ||
+ timeout == 0) {
+ CAAM_JR_ERR("0x%x, %d", tmp, timeout);
+ /* unmask interrupts */
+ if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL)
+ caam_jr_enable_irqs(job_ring->irq_fd);
+ return -1;
+ }
+
+ /* Initiate reset */
+ timeout = SEC_TIMEOUT;
+ SET_JR_REG(JRCR, job_ring, JR_REG_JRCR_VAL_RESET);
+
+ do {
+ tmp = GET_JR_REG(JRCR, job_ring);
+ usleep(usleep_interval);
+ } while ((tmp & JR_REG_JRCR_VAL_RESET) && --timeout);
+
+ CAAM_JR_DEBUG("JRCR is %x", tmp);
+ if (timeout == 0) {
+ CAAM_JR_ERR("Failed to reset hw job ring %p", job_ring);
+ /* unmask interrupts */
+ if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL)
+ caam_jr_enable_irqs(job_ring->irq_fd);
+ return -1;
+ }
+ /* unmask interrupts */
+ if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL)
+ caam_jr_enable_irqs(job_ring->irq_fd);
+ return 0;
+
+}
+
+void
+hw_handle_job_ring_error(struct sec_job_ring_t *job_ring __rte_unused,
+ uint32_t error_code)
+{
+ union hw_error_code hw_err_code;
+
+ hw_err_code.error = error_code;
+ switch (hw_err_code.error_desc.value.ssrc) {
+ case SEC_HW_ERR_SSRC_NO_SRC:
+ ASSERT(hw_err_code.error_desc.no_status_src.res == 0);
+ CAAM_JR_ERR("No Status Source ");
+ break;
+ case SEC_HW_ERR_SSRC_CCB_ERR:
+ CAAM_JR_ERR("CCB Status Source");
+ hw_handle_ccb_err(hw_err_code);
+ break;
+ case SEC_HW_ERR_SSRC_JMP_HALT_U:
+ CAAM_JR_ERR("Jump Halt User Status Source");
+ hw_handle_jmp_halt_user_err(hw_err_code);
+ break;
+ case SEC_HW_ERR_SSRC_DECO:
+ CAAM_JR_ERR("DECO Status Source");
+ hw_handle_deco_err(hw_err_code);
+ break;
+ case SEC_HW_ERR_SSRC_JR:
+ CAAM_JR_ERR("Job Ring Status Source");
+ hw_handle_jr_err(hw_err_code);
+ break;
+ case SEC_HW_ERR_SSRC_JMP_HALT_COND:
+ CAAM_JR_ERR("Jump Halt Condition Codes");
+ hw_handle_jmp_halt_cond_err(hw_err_code);
+ break;
+ default:
+ ASSERT(0);
+ CAAM_JR_ERR("Unknown SSRC");
+ break;
+ }
+}
+
+void
+hw_job_ring_error_print(struct sec_job_ring_t *job_ring, int code)
+{
+ switch (code) {
+ case JRINT_ERR_WRITE_STATUS:
+ CAAM_JR_ERR("Error writing status to Output Ring ");
+ break;
+ case JRINT_ERR_BAD_INPUT_BASE:
+ CAAM_JR_ERR(
+ "Bad Input Ring Base (%p) (not on a 4-byte boundary) ",
+ (void *)job_ring);
+ break;
+ case JRINT_ERR_BAD_OUTPUT_BASE:
+ CAAM_JR_ERR(
+ "Bad Output Ring Base (%p) (not on a 4-byte boundary) ",
+ (void *)job_ring);
+ break;
+ case JRINT_ERR_WRITE_2_IRBA:
+ CAAM_JR_ERR(
+ "Invalid write to Input Ring Base Address Register ");
+ break;
+ case JRINT_ERR_WRITE_2_ORBA:
+ CAAM_JR_ERR(
+ "Invalid write to Output Ring Base Address Register ");
+ break;
+ case JRINT_ERR_RES_B4_HALT:
+ CAAM_JR_ERR(
+ "Job Ring [%p] released before Job Ring is halted",
+ (void *)job_ring);
+ break;
+ case JRINT_ERR_REM_TOO_MANY:
+ CAAM_JR_ERR("Removed too many jobs from job ring [%p]",
+ (void *)job_ring);
+ break;
+ case JRINT_ERR_ADD_TOO_MANY:
+ CAAM_JR_ERR("Added too many jobs on job ring [%p]", job_ring);
+ break;
+ default:
+ CAAM_JR_ERR(" Unknown SEC JR Error :%d",
+ code);
+ break;
+ }
+}
+
+int
+hw_job_ring_set_coalescing_param(struct sec_job_ring_t *job_ring,
+ uint16_t irq_coalescing_timer,
+ uint8_t irq_coalescing_count)
+{
+ uint32_t reg_val = 0;
+
+ ASSERT(job_ring != NULL);
+ if (job_ring->register_base_addr == NULL) {
+ CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init",
+ job_ring);
+ return -1;
+ }
+ /* Set descriptor count coalescing */
+ reg_val |= (irq_coalescing_count << JR_REG_JRCFG_LO_ICDCT_SHIFT);
+
+ /* Set coalescing timer value */
+ reg_val |= (irq_coalescing_timer << JR_REG_JRCFG_LO_ICTT_SHIFT);
+
+ /* Update parameters in HW */
+ SET_JR_REG_LO(JRCFG, job_ring, reg_val);
+ CAAM_JR_DEBUG("Set coalescing params on jr %p timer:%d, desc count: %d",
+ job_ring, irq_coalescing_timer, irq_coalescing_timer);
+
+ return 0;
+}
+
+int
+hw_job_ring_enable_coalescing(struct sec_job_ring_t *job_ring)
+{
+ uint32_t reg_val = 0;
+
+ ASSERT(job_ring != NULL);
+ if (job_ring->register_base_addr == NULL) {
+ CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init",
+ job_ring);
+ return -1;
+ }
+
+ /* Get the current value of the register */
+ reg_val = GET_JR_REG_LO(JRCFG, job_ring);
+
+ /* Enable coalescing */
+ reg_val |= JR_REG_JRCFG_LO_ICEN_EN;
+
+ /* Write in hw */
+ SET_JR_REG_LO(JRCFG, job_ring, reg_val);
+
+ CAAM_JR_DEBUG("Enabled coalescing on jr %p ",
+ job_ring);
+
+ return 0;
+}
+
+int
+hw_job_ring_disable_coalescing(struct sec_job_ring_t *job_ring)
+{
+ uint32_t reg_val = 0;
+
+ ASSERT(job_ring != NULL);
+
+ if (job_ring->register_base_addr == NULL) {
+ CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init",
+ job_ring);
+ return -1;
+ }
+
+ /* Get the current value of the register */
+ reg_val = GET_JR_REG_LO(JRCFG, job_ring);
+
+ /* Disable coalescing */
+ reg_val &= ~JR_REG_JRCFG_LO_ICEN_EN;
+
+ /* Write in hw */
+ SET_JR_REG_LO(JRCFG, job_ring, reg_val);
+ CAAM_JR_DEBUG("Disabled coalescing on jr %p ", job_ring);
+
+ return 0;
+}
diff --git a/drivers/crypto/caam_jr/caam_jr_hw_specific.h b/drivers/crypto/caam_jr/caam_jr_hw_specific.h
new file mode 100644
index 00000000..5f58a585
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_hw_specific.h
@@ -0,0 +1,503 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017 NXP
+ */
+
+#ifndef CAAM_JR_HW_SPECIFIC_H
+#define CAAM_JR_HW_SPECIFIC_H
+
+#include <caam_jr_config.h>
+
+/*
+ * Offset to the registers of a job ring.
+ * Is different for each job ring.
+ */
+#define CHAN_BASE(jr) ((size_t)(jr)->register_base_addr)
+
+#define SEC_JOB_RING_IS_FULL(pi, ci, ring_max_size, ring_threshold) \
+ ((((pi) + 1 + ((ring_max_size) - (ring_threshold))) & \
+ (ring_max_size - 1)) == ((ci)))
+
+#define SEC_CIRCULAR_COUNTER(x, max) (((x) + 1) & (max - 1))
+
+/*
+ * Assert that cond is true. If !cond is true, display str and the vararg list
+ * in a printf-like syntax. also, if !cond is true, return altRet.
+ *
+ * \param cond A boolean expression to be asserted true
+ * \param altRet The value to be returned if cond doesn't hold true
+ * \param str A quoted char string
+ *
+ * E.g.:
+ * SEC_ASSERT(ret > 0, 0, "ERROR initializing app: code = %d\n", ret);
+ */
+#define SEC_ASSERT(cond, altRet, ...) do {\
+ if (unlikely(!(cond))) {\
+ CAAM_JR_ERR(__VA_ARGS__); \
+ return altRet; \
+ } \
+} while (0)
+
+#define SEC_DP_ASSERT(cond, altRet, ...) do {\
+ if (unlikely(!(cond))) {\
+ CAAM_JR_DP_ERR(__VA_ARGS__); \
+ return altRet; \
+ } \
+} while (0)
+
+#define ASSERT(x)
+
+/*
+ * Constants representing various job ring registers
+ */
+#if CAAM_BYTE_ORDER == __BIG_ENDIAN
+#define JR_REG_IRBA_OFFSET 0x0000
+#define JR_REG_IRBA_OFFSET_LO 0x0004
+#else
+#define JR_REG_IRBA_OFFSET 0x0004
+#define JR_REG_IRBA_OFFSET_LO 0x0000
+#endif
+
+#define JR_REG_IRSR_OFFSET 0x000C
+#define JR_REG_IRSA_OFFSET 0x0014
+#define JR_REG_IRJA_OFFSET 0x001C
+
+#if CAAM_BYTE_ORDER == __BIG_ENDIAN
+#define JR_REG_ORBA_OFFSET 0x0020
+#define JR_REG_ORBA_OFFSET_LO 0x0024
+#else
+#define JR_REG_ORBA_OFFSET 0x0024
+#define JR_REG_ORBA_OFFSET_LO 0x0020
+#endif
+
+#define JR_REG_ORSR_OFFSET 0x002C
+#define JR_REG_ORJR_OFFSET 0x0034
+#define JR_REG_ORSFR_OFFSET 0x003C
+#define JR_REG_JROSR_OFFSET 0x0044
+#define JR_REG_JRINT_OFFSET 0x004C
+
+#define JR_REG_JRCFG_OFFSET 0x0050
+#define JR_REG_JRCFG_OFFSET_LO 0x0054
+
+#define JR_REG_IRRI_OFFSET 0x005C
+#define JR_REG_ORWI_OFFSET 0x0064
+#define JR_REG_JRCR_OFFSET 0x006C
+
+/*
+ * Constants for error handling on job ring
+ */
+#define JR_REG_JRINT_ERR_TYPE_SHIFT 8
+#define JR_REG_JRINT_ERR_ORWI_SHIFT 16
+#define JR_REG_JRINIT_JRE_SHIFT 1
+
+#define JRINT_JRE (1 << JR_REG_JRINIT_JRE_SHIFT)
+#define JRINT_ERR_WRITE_STATUS (1 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_BAD_INPUT_BASE (3 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_BAD_OUTPUT_BASE (4 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_WRITE_2_IRBA (5 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_WRITE_2_ORBA (6 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_RES_B4_HALT (7 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_REM_TOO_MANY (8 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_ADD_TOO_MANY (9 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_HALT_MASK 0x0C
+#define JRINT_ERR_HALT_INPROGRESS 0x04
+#define JRINT_ERR_HALT_COMPLETE 0x08
+
+#define JR_REG_JRCR_VAL_RESET 0x00000001
+
+#define JR_REG_JRCFG_LO_ICTT_SHIFT 0x10
+#define JR_REG_JRCFG_LO_ICDCT_SHIFT 0x08
+#define JR_REG_JRCFG_LO_ICEN_EN 0x02
+
+/*
+ * Constants for Descriptor Processing errors
+ */
+#define SEC_HW_ERR_SSRC_NO_SRC 0x00
+#define SEC_HW_ERR_SSRC_CCB_ERR 0x02
+#define SEC_HW_ERR_SSRC_JMP_HALT_U 0x03
+#define SEC_HW_ERR_SSRC_DECO 0x04
+#define SEC_HW_ERR_SSRC_JR 0x06
+#define SEC_HW_ERR_SSRC_JMP_HALT_COND 0x07
+
+#define SEC_HW_ERR_DECO_HFN_THRESHOLD 0xF1
+#define SEC_HW_ERR_CCB_ICV_CHECK_FAIL 0x0A
+
+/*
+ * Constants for descriptors
+ */
+/* Return higher 32 bits of physical address */
+#define PHYS_ADDR_HI(phys_addr) \
+ (uint32_t)(((uint64_t)phys_addr) >> 32)
+
+/* Return lower 32 bits of physical address */
+#define PHYS_ADDR_LO(phys_addr) \
+ (uint32_t)(((uint64_t)phys_addr) & 0xFFFFFFFF)
+
+/*
+ * Macros for extracting error codes for the job ring
+ */
+#define JR_REG_JRINT_ERR_TYPE_EXTRACT(value) ((value) & 0x00000F00)
+#define JR_REG_JRINT_ERR_ORWI_EXTRACT(value) \
+ (((value) & 0x3FFF0000) >> JR_REG_JRINT_ERR_ORWI_SHIFT)
+#define JR_REG_JRINT_JRE_EXTRACT(value) ((value) & JRINT_JRE)
+
+/*
+ * Macros for managing the job ring
+ */
+/* Read pointer to job ring input ring start address */
+#if defined(RTE_ARCH_ARM64)
+#define hw_get_inp_queue_base(jr) ((((dma_addr_t)GET_JR_REG(IRBA, \
+ (jr))) << 32) | \
+ (GET_JR_REG_LO(IRBA, (jr))))
+
+/* Read pointer to job ring output ring start address */
+#define hw_get_out_queue_base(jr) (((dma_addr_t)(GET_JR_REG(ORBA, \
+ (jr))) << 32) | \
+ (GET_JR_REG_LO(ORBA, (jr))))
+#else
+#define hw_get_inp_queue_base(jr) ((dma_addr_t)(GET_JR_REG_LO(IRBA, (jr))))
+
+#define hw_get_out_queue_base(jr) ((dma_addr_t)(GET_JR_REG_LO(ORBA, (jr))))
+#endif
+
+/*
+ * IRJA - Input Ring Jobs Added Register shows
+ * how many new jobs were added to the Input Ring.
+ */
+#define hw_enqueue_desc_on_job_ring(job_ring) SET_JR_REG(IRJA, (job_ring), 1)
+
+#define hw_set_input_ring_size(job_ring, size) SET_JR_REG(IRSR, job_ring, \
+ (size))
+
+#define hw_set_output_ring_size(job_ring, size) SET_JR_REG(ORSR, job_ring, \
+ (size))
+
+#if defined(RTE_ARCH_ARM64)
+#define hw_set_input_ring_start_addr(job_ring, start_addr) \
+{ \
+ SET_JR_REG(IRBA, job_ring, PHYS_ADDR_HI(start_addr)); \
+ SET_JR_REG_LO(IRBA, job_ring, PHYS_ADDR_LO(start_addr));\
+}
+
+#define hw_set_output_ring_start_addr(job_ring, start_addr) \
+{ \
+ SET_JR_REG(ORBA, job_ring, PHYS_ADDR_HI(start_addr)); \
+ SET_JR_REG_LO(ORBA, job_ring, PHYS_ADDR_LO(start_addr));\
+}
+
+#else
+#define hw_set_input_ring_start_addr(job_ring, start_addr) \
+{ \
+ SET_JR_REG(IRBA, job_ring, 0); \
+ SET_JR_REG_LO(IRBA, job_ring, PHYS_ADDR_LO(start_addr));\
+}
+
+#define hw_set_output_ring_start_addr(job_ring, start_addr) \
+{ \
+ SET_JR_REG(ORBA, job_ring, 0); \
+ SET_JR_REG_LO(ORBA, job_ring, PHYS_ADDR_LO(start_addr));\
+}
+#endif
+
+/* ORJR - Output Ring Jobs Removed Register shows how many jobs were
+ * removed from the Output Ring for processing by software. This is done after
+ * the software has processed the entries.
+ */
+#define hw_remove_entries(jr, no_entries) SET_JR_REG(ORJR, (jr), (no_entries))
+
+/* IRSA - Input Ring Slots Available register holds the number of entries in
+ * the Job Ring's input ring. Once a job is enqueued, the value returned is
+ * decremented by the hardware by the number of jobs enqueued.
+ */
+#define hw_get_available_slots(jr) GET_JR_REG(IRSA, jr)
+
+/* ORSFR - Output Ring Slots Full register holds the number of jobs which were
+ * processed by the SEC and can be retrieved by the software. Once a job has
+ * been processed by software, the user will call hw_remove_one_entry in order
+ * to notify the SEC that the entry was processed.
+ */
+#define hw_get_no_finished_jobs(jr) GET_JR_REG(ORSFR, jr)
+
+/*
+ * Macros for manipulating JR registers
+ */
+#if CORE_BYTE_ORDER == CAAM_BYTE_ORDER
+#define sec_read_32(addr) (*(volatile unsigned int *)(addr))
+#define sec_write_32(addr, val) (*(volatile unsigned int *)(addr) = (val))
+
+#else
+#define sec_read_32(addr) rte_bswap32((*(volatile unsigned int *)(addr)))
+#define sec_write_32(addr, val) \
+ (*(volatile unsigned int *)(addr) = rte_bswap32(val))
+#endif
+
+#if CAAM_BYTE_ORDER == __LITTLE_ENDIAN
+#define sec_read_64(addr) (((u64)sec_read_32((u32 *)(addr) + 1) << 32) | \
+ (sec_read_32((u32 *)(addr))))
+
+#define sec_write_64(addr, val) { \
+ sec_write_32((u32 *)(addr) + 1, (u32)((val) >> 32)); \
+ sec_write_32((u32 *)(addr), (u32)(val)); \
+}
+#else /* CAAM_BYTE_ORDER == __BIG_ENDIAN */
+#define sec_read_64(addr) (((u64)sec_read_32((u32 *)(addr)) << 32) | \
+ (sec_read_32((u32 *)(addr) + 1)))
+
+#define sec_write_64(addr, val) { \
+ sec_write_32((u32 *)(addr), (u32)((val) >> 32)); \
+ sec_write_32((u32 *)(addr) + 1, (u32)(val)); \
+}
+#endif
+
+#if defined(RTE_ARCH_ARM64)
+#define sec_read_addr(a) sec_read_64((a))
+#define sec_write_addr(a, v) sec_write_64((a), (v))
+#else
+#define sec_read_addr(a) sec_read_32((a))
+#define sec_write_addr(a, v) sec_write_32((a), (v))
+#endif
+
+#define JR_REG(name, jr) (CHAN_BASE(jr) + JR_REG_##name##_OFFSET)
+#define JR_REG_LO(name, jr) (CHAN_BASE(jr) + JR_REG_##name##_OFFSET_LO)
+
+#define GET_JR_REG(name, jr) (sec_read_32(JR_REG(name, (jr))))
+#define GET_JR_REG_LO(name, jr) (sec_read_32(JR_REG_LO(name, (jr))))
+
+#define SET_JR_REG(name, jr, value) \
+ (sec_write_32(JR_REG(name, (jr)), value))
+#define SET_JR_REG_LO(name, jr, value) \
+ (sec_write_32(JR_REG_LO(name, (jr)), value))
+
+/* Lists the possible states for a job ring. */
+typedef enum sec_job_ring_state_e {
+ SEC_JOB_RING_STATE_STARTED, /* Job ring is initialized */
+ SEC_JOB_RING_STATE_RESET, /* Job ring reset is in progress */
+} sec_job_ring_state_t;
+
+/* code or cmd block to caam */
+struct sec_cdb {
+ struct {
+ union {
+ uint32_t word;
+ struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+ uint16_t rsvd63_48;
+ unsigned int rsvd47_39:9;
+ unsigned int idlen:7;
+#else
+ unsigned int idlen:7;
+ unsigned int rsvd47_39:9;
+ uint16_t rsvd63_48;
+#endif
+ } field;
+ } __rte_packed hi;
+
+ union {
+ uint32_t word;
+ struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+ unsigned int rsvd31_30:2;
+ unsigned int fsgt:1;
+ unsigned int lng:1;
+ unsigned int offset:2;
+ unsigned int abs:1;
+ unsigned int add_buf:1;
+ uint8_t pool_id;
+ uint16_t pool_buffer_size;
+#else
+ uint16_t pool_buffer_size;
+ uint8_t pool_id;
+ unsigned int add_buf:1;
+ unsigned int abs:1;
+ unsigned int offset:2;
+ unsigned int lng:1;
+ unsigned int fsgt:1;
+ unsigned int rsvd31_30:2;
+#endif
+ } field;
+ } __rte_packed lo;
+ } __rte_packed sh_hdr;
+
+ uint32_t sh_desc[SEC_JOB_DESCRIPTOR_SIZE];
+};
+
+struct caam_jr_qp {
+ struct sec_job_ring_t *ring;
+ uint64_t rx_pkts;
+ uint64_t rx_errs;
+ uint64_t rx_poll_err;
+ uint64_t tx_pkts;
+ uint64_t tx_errs;
+ uint64_t tx_ring_full;
+};
+
+struct sec_job_ring_t {
+ /* TODO: Add wrapper macro to make it obvious this is the consumer index
+ * on the output ring
+ */
+ uint32_t cidx; /* Consumer index for job ring (jobs array).
+ * @note: cidx and pidx are accessed from
+ * different threads. Place the cidx and pidx
+ * inside the structure so that they lay on
+ * different cachelines, to avoid false sharing
+ * between threads when the threads run on
+ * different cores!
+ */
+ /* TODO: Add wrapper macro to make it obvious this is the producer index
+ * on the input ring
+ */
+ uint32_t pidx; /* Producer index for job ring (jobs array) */
+
+ phys_addr_t *input_ring;/* Ring of output descriptors received from SEC.
+ * Size of array is power of 2 to allow fast
+ * update of producer/consumer indexes with
+ * bitwise operations.
+ */
+
+ struct sec_outring_entry *output_ring;
+ /* Ring of output descriptors received from SEC.
+ * Size of array is power of 2 to allow fast
+ * update of producer/consumer indexes with
+ * bitwise operations.
+ */
+
+ uint32_t irq_fd; /* The file descriptor used for polling from
+ * user space for interrupts notifications
+ */
+ uint32_t jr_mode; /* Model used by SEC Driver to receive
+ * notifications from SEC. Can be either
+ * of the three: #SEC_NOTIFICATION_TYPE_NAPI
+ * #SEC_NOTIFICATION_TYPE_IRQ or
+ * #SEC_NOTIFICATION_TYPE_POLL
+ */
+ uint32_t napi_mode; /* Job ring mode if NAPI mode is chosen
+ * Used only when jr_mode is set to
+ * #SEC_NOTIFICATION_TYPE_NAPI
+ */
+ void *register_base_addr; /* Base address for SEC's
+ * register memory for this job ring.
+ */
+ uint8_t coalescing_en; /* notifies if coelescing is
+ * enabled for the job ring
+ */
+ sec_job_ring_state_t jr_state; /* The state of this job ring */
+
+ struct rte_mempool *ctx_pool; /* per dev mempool for caam_jr_op_ctx */
+ unsigned int max_nb_queue_pairs;
+ unsigned int max_nb_sessions;
+ struct caam_jr_qp qps[RTE_CAAM_MAX_NB_SEC_QPS]; /* i/o queue for sec */
+};
+
+/* Union describing the possible error codes that
+ * can be set in the descriptor status word
+ */
+union hw_error_code {
+ uint32_t error;
+ union {
+ struct {
+ uint32_t ssrc:4;
+ uint32_t ssed_val:28;
+ } __rte_packed value;
+ struct {
+ uint32_t ssrc:4;
+ uint32_t res:28;
+ } __rte_packed no_status_src;
+ struct {
+ uint32_t ssrc:4;
+ uint32_t jmp:1;
+ uint32_t res:11;
+ uint32_t desc_idx:8;
+ uint32_t cha_id:4;
+ uint32_t err_id:4;
+ } __rte_packed ccb_status_src;
+ struct {
+ uint32_t ssrc:4;
+ uint32_t jmp:1;
+ uint32_t res:11;
+ uint32_t desc_idx:8;
+ uint32_t offset:8;
+ } __rte_packed jmp_halt_user_src;
+ struct {
+ uint32_t ssrc:4;
+ uint32_t jmp:1;
+ uint32_t res:11;
+ uint32_t desc_idx:8;
+ uint32_t desc_err:8;
+ } __rte_packed deco_src;
+ struct {
+ uint32_t ssrc:4;
+ uint32_t res:17;
+ uint32_t naddr:3;
+ uint32_t desc_err:8;
+ } __rte_packed jr_src;
+ struct {
+ uint32_t ssrc:4;
+ uint32_t jmp:1;
+ uint32_t res:11;
+ uint32_t desc_idx:8;
+ uint32_t cond:8;
+ } __rte_packed jmp_halt_cond_src;
+ } __rte_packed error_desc;
+} __rte_packed;
+
+/* @brief Initialize a job ring/channel in SEC device.
+ * Write configuration register/s to properly initialize a job ring.
+ *
+ * @param [in] job_ring The job ring
+ *
+ * @retval 0 for success
+ * @retval other for error
+ */
+int hw_reset_job_ring(struct sec_job_ring_t *job_ring);
+
+/* @brief Reset a job ring/channel in SEC device.
+ * Write configuration register/s to reset a job ring.
+ *
+ * @param [in] job_ring The job ring
+ *
+ * @retval 0 for success
+ * @retval -1 in case job ring reset failed
+ */
+int hw_shutdown_job_ring(struct sec_job_ring_t *job_ring);
+
+/* @brief Handle a job ring/channel error in SEC device.
+ * Identify the error type and clear error bits if required.
+ *
+ * @param [in] job_ring The job ring
+ * @param [in] sec_error_code The job ring's error code
+ */
+void hw_handle_job_ring_error(struct sec_job_ring_t *job_ring,
+ uint32_t sec_error_code);
+
+/* @brief Handle a job ring error in the device.
+ * Identify the error type and printout a explanatory
+ * messages.
+ *
+ * @param [in] job_ring The job ring
+ *
+ */
+void hw_job_ring_error_print(struct sec_job_ring_t *job_ring, int code);
+
+/* @brief Set interrupt coalescing parameters on the Job Ring.
+ * @param [in] job_ring The job ring
+ * @param [in] irq_coalesing_timer Interrupt coalescing timer threshold.
+ * This value determines the maximum
+ * amount of time after processing a
+ * descriptor before raising an interrupt.
+ * @param [in] irq_coalescing_count Interrupt coalescing descriptor count
+ * threshold.
+ */
+int hw_job_ring_set_coalescing_param(struct sec_job_ring_t *job_ring,
+ uint16_t irq_coalescing_timer,
+ uint8_t irq_coalescing_count);
+
+/* @brief Enable interrupt coalescing on a job ring
+ * @param [in] job_ring The job ring
+ */
+int hw_job_ring_enable_coalescing(struct sec_job_ring_t *job_ring);
+
+/* @brief Disable interrupt coalescing on a job ring
+ * @param [in] job_ring The job ring
+ */
+int hw_job_ring_disable_coalescing(struct sec_job_ring_t *job_ring);
+
+#endif /* CAAM_JR_HW_SPECIFIC_H */
diff --git a/drivers/crypto/caam_jr/caam_jr_log.h b/drivers/crypto/caam_jr/caam_jr_log.h
new file mode 100644
index 00000000..106ff07a
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_log.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#ifndef _CAAM_JR_LOG_H_
+#define _CAAM_JR_LOG_H_
+
+#include <rte_log.h>
+
+extern int caam_jr_logtype;
+
+#define CAAM_JR_LOG(level, fmt, args...) \
+ rte_log(RTE_LOG_ ## level, caam_jr_logtype, "caam_jr: " \
+ fmt "\n", ##args)
+
+#define CAAM_JR_DEBUG(fmt, args...) \
+ rte_log(RTE_LOG_DEBUG, caam_jr_logtype, "caam_jr: %s(): " \
+ fmt "\n", __func__, ##args)
+
+#define PMD_INIT_FUNC_TRACE() CAAM_JR_DEBUG(" >>")
+
+#define CAAM_JR_INFO(fmt, args...) \
+ CAAM_JR_LOG(INFO, fmt, ## args)
+#define CAAM_JR_ERR(fmt, args...) \
+ CAAM_JR_LOG(ERR, fmt, ## args)
+#define CAAM_JR_WARN(fmt, args...) \
+ CAAM_JR_LOG(WARNING, fmt, ## args)
+
+/* DP Logs, toggled out at compile time if level lower than current level */
+#define CAAM_JR_DP_LOG(level, fmt, args...) \
+ RTE_LOG_DP(level, PMD, fmt "\n", ## args)
+
+#define CAAM_JR_DP_DEBUG(fmt, args...) \
+ CAAM_JR_DP_LOG(DEBUG, fmt, ## args)
+#define CAAM_JR_DP_INFO(fmt, args...) \
+ CAAM_JR_DP_LOG(INFO, fmt, ## args)
+#define CAAM_JR_DP_WARN(fmt, args...) \
+ CAAM_JR_DP_LOG(WARNING, fmt, ## args)
+#define CAAM_JR_DP_ERR(fmt, args...) \
+ CAAM_JR_DP_LOG(ERR, fmt, ## args)
+
+#endif /* _CAAM_JR_LOG_H_ */
diff --git a/drivers/crypto/caam_jr/caam_jr_pvt.h b/drivers/crypto/caam_jr/caam_jr_pvt.h
new file mode 100644
index 00000000..9f1adabc
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_pvt.h
@@ -0,0 +1,291 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#ifndef CAAM_JR_PVT_H
+#define CAAM_JR_PVT_H
+
+#include <hw/desc/ipsec.h>
+
+/* NXP CAAM JR PMD device name */
+
+#define CAAM_JR_ALG_UNSUPPORT (-1)
+
+/* Minimum job descriptor consists of a oneword job descriptor HEADER and
+ * a pointer to the shared descriptor.
+ */
+#define MIN_JOB_DESC_SIZE (CAAM_CMD_SZ + CAAM_PTR_SZ)
+#define CAAM_JOB_DESC_SIZE 13
+
+/* CTX_POOL_NUM_BUFS is set as per the ipsec-secgw application */
+#define CTX_POOL_NUM_BUFS 32000
+#define CTX_POOL_CACHE_SIZE 512
+
+#define DIR_ENC 1
+#define DIR_DEC 0
+
+#define JR_MAX_NB_MAX_DIGEST 32
+
+#define RTE_CAAM_JR_PMD_MAX_NB_SESSIONS 2048
+
+
+/* Return codes for SEC user space driver APIs */
+enum sec_return_code_e {
+ SEC_SUCCESS = 0, /* Operation executed successfully.*/
+ SEC_INVALID_INPUT_PARAM, /* API received an invalid input
+ * parameter
+ */
+ SEC_OUT_OF_MEMORY, /* Memory allocation failed. */
+ SEC_DESCRIPTOR_IN_FLIGHT, /* API function indicates there are
+ * descriptors in flight
+ * for SEC to process.
+ */
+ SEC_LAST_DESCRIPTOR_IN_FLIGHT, /* API function indicates there is one
+ * last descriptor in flight
+ * for SEC to process that.
+ */
+ SEC_PROCESSING_ERROR, /* Indicates a SEC processing error
+ * occurred on a Job Ring which requires
+ * a SEC user space driver shutdown. Can
+ * be returned from sec_poll_job_ring().
+ * Then the only other API that can be
+ * called after this error is
+ * sec_release().
+ */
+ SEC_DESC_PROCESSING_ERROR, /* Indicates a SEC descriptor processing
+ * error occurred on a Job Ring. Can be
+ * returned from sec_poll_job_ring().
+ * The driver was able to reset job ring
+ * and job ring can be used like in a
+ * normal case.
+ */
+ SEC_JR_IS_FULL, /* Job Ring is full. There is no more
+ * room in the JR for new descriptors.
+ * This can happen if the descriptor RX
+ * rate is higher than SEC's capacity.
+ */
+ SEC_DRIVER_RELEASE_IN_PROGRESS, /* SEC driver shutdown is in progress,
+ * descriptors processing or polling is
+ * allowed.
+ */
+ SEC_DRIVER_ALREADY_INITIALIZED, /* SEC driver is already initialized.*/
+ SEC_DRIVER_NOT_INITIALIZED, /* SEC driver is NOT initialized. */
+ SEC_JOB_RING_RESET_IN_PROGRESS, /* Job ring is resetting due to a
+ * per-descriptor SEC processing error
+ * ::SEC_desc_PROCESSING_ERROR. Reset is
+ * finished when sec_poll_job_ring()
+ * return. Then the job ring can be used
+ * again.
+ */
+ SEC_RESET_ENGINE_FAILED, /* Resetting of SEC Engine by SEC Kernel
+ * Driver Failed
+ */
+ SEC_ENABLE_IRQS_FAILED, /* Enabling of IRQs in SEC Kernel Driver
+ * Failed
+ */
+ SEC_DISABLE_IRQS_FAILED, /* Disabling of IRQs in SEC Kernel
+ * Driver Failed
+ */
+ /* END OF VALID VALUES */
+
+ SEC_RETURN_CODE_MAX_VALUE, /* Invalid value for return code. It is
+ * used to mark the end of the return
+ * code values. @note ALL new return
+ * code values MUST be added before
+ * ::SEC_RETURN_CODE_MAX_VALUE!
+ */
+};
+
+enum caam_jr_op_type {
+ CAAM_JR_NONE, /* No Cipher operations*/
+ CAAM_JR_CIPHER,/* CIPHER operations */
+ CAAM_JR_AUTH, /* Authentication Operations */
+ CAAM_JR_AEAD, /* Authenticated Encryption with associated data */
+ CAAM_JR_IPSEC, /* IPSEC protocol operations*/
+ CAAM_JR_PDCP, /* PDCP protocol operations*/
+ CAAM_JR_PKC, /* Public Key Cryptographic Operations */
+ CAAM_JR_MAX
+};
+
+struct caam_jr_session {
+ uint8_t dir; /* Operation Direction */
+ enum rte_crypto_cipher_algorithm cipher_alg; /* Cipher Algorithm*/
+ enum rte_crypto_auth_algorithm auth_alg; /* Authentication Algorithm*/
+ enum rte_crypto_aead_algorithm aead_alg; /* AEAD Algorithm*/
+ enum rte_security_session_protocol proto_alg; /* Security Algorithm*/
+ union {
+ struct {
+ uint8_t *data; /* pointer to key data */
+ size_t length; /* key length in bytes */
+ } aead_key;
+ struct {
+ struct {
+ uint8_t *data; /* pointer to key data */
+ size_t length; /* key length in bytes */
+ } cipher_key;
+ struct {
+ uint8_t *data; /* pointer to key data */
+ size_t length; /* key length in bytes */
+ } auth_key;
+ };
+ };
+ struct {
+ uint16_t length;
+ uint16_t offset;
+ } iv; /* Initialisation vector parameters */
+ uint16_t auth_only_len; /* Length of data for Auth only */
+ uint32_t digest_length;
+ struct ipsec_encap_pdb encap_pdb;
+ struct ip ip4_hdr;
+ struct ipsec_decap_pdb decap_pdb;
+ struct caam_jr_qp *qp;
+ struct sec_cdb *cdb; /* cmd block associated with qp */
+ struct rte_mempool *ctx_pool; /* session mempool for caam_jr_op_ctx */
+};
+
+/*
+ * 16-byte hardware scatter/gather table
+ */
+
+#define SEC4_SG_LEN_EXT 0x80000000 /* Entry points to table */
+#define SEC4_SG_LEN_FIN 0x40000000 /* Last ent in table */
+#define SEC4_SG_BPID_MASK 0x000000ff
+#define SEC4_SG_BPID_SHIFT 16
+#define SEC4_SG_LEN_MASK 0x3fffffff /* Excludes EXT and FINAL */
+#define SEC4_SG_OFFSET_MASK 0x00001fff
+
+struct sec4_sg_entry {
+ uint64_t ptr;
+ uint32_t len;
+ uint32_t bpid_offset;
+};
+
+#define MAX_SG_ENTRIES 16
+#define SG_CACHELINE_0 0
+#define SG_CACHELINE_1 4
+#define SG_CACHELINE_2 8
+#define SG_CACHELINE_3 12
+
+/* Structure encompassing a job descriptor which is to be processed
+ * by SEC. User should also initialise this structure with the callback
+ * function pointer which will be called by driver after recieving proccessed
+ * descriptor from SEC. User data is also passed in this data structure which
+ * will be sent as an argument to the user callback function.
+ */
+struct job_descriptor {
+ uint32_t desc[CAAM_JOB_DESC_SIZE];
+};
+
+struct caam_jr_op_ctx {
+ struct job_descriptor jobdes;
+ /* sg[0] output, sg[1] input, others are possible sub frames */
+ struct sec4_sg_entry sg[MAX_SG_ENTRIES];
+ struct rte_crypto_op *op;
+ struct rte_mempool *ctx_pool; /* mempool pointer for caam_jr_op_ctx */
+ int64_t vtop_offset;
+ uint8_t digest[JR_MAX_NB_MAX_DIGEST];
+};
+
+/**
+ * Checksum
+ *
+ * @param buffer calculate chksum for buffer
+ * @param len buffer length
+ *
+ * @return checksum value in host cpu order
+ */
+static inline uint16_t
+calc_chksum(void *buffer, int len)
+{
+ uint16_t *buf = (uint16_t *)buffer;
+ uint32_t sum = 0;
+ uint16_t result;
+
+ for (sum = 0; len > 1; len -= 2)
+ sum += *buf++;
+
+ if (len == 1)
+ sum += *(unsigned char *)buf;
+
+ sum = (sum >> 16) + (sum & 0xFFFF);
+ sum += (sum >> 16);
+ result = ~sum;
+
+ return result;
+}
+struct uio_job_ring {
+ uint32_t jr_id;
+ uint32_t uio_fd;
+ void *register_base_addr;
+ int map_size;
+ int uio_minor_number;
+};
+
+int sec_cleanup(void);
+int sec_configure(void);
+struct uio_job_ring *config_job_ring(void);
+void free_job_ring(uint32_t uio_fd);
+
+/* For Dma memory allocation of specified length and alignment */
+static inline void *
+caam_jr_dma_mem_alloc(size_t align, size_t len)
+{
+ return rte_malloc("mem_alloc", len, align);
+}
+
+/* For freeing dma memory */
+static inline void
+caam_jr_dma_free(void *ptr)
+{
+ rte_free(ptr);
+}
+
+static inline rte_iova_t
+caam_jr_mem_vtop(void *vaddr)
+{
+ const struct rte_memseg *ms;
+
+ ms = rte_mem_virt2memseg(vaddr, NULL);
+ if (ms)
+ return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
+ return (size_t)NULL;
+}
+
+static inline void *
+caam_jr_dma_ptov(rte_iova_t paddr)
+{
+ return rte_mem_iova2virt(paddr);
+}
+
+/* Virtual to physical address conversion */
+static inline rte_iova_t caam_jr_dma_vtop(void *ptr)
+{
+ return caam_jr_mem_vtop(ptr);
+}
+
+/** @brief Request to SEC kernel driver to enable interrupts for
+ * descriptor finished processing
+ * Use UIO to communicate with SEC kernel driver: write command
+ * value that indicates an IRQ enable action into UIO file descriptor
+ * of this job ring.
+ *
+ * @param [in] uio_fd Job Ring UIO File descriptor
+ * @retval 0 for success
+ * @retval -1 value for error
+ */
+uint32_t caam_jr_enable_irqs(uint32_t uio_fd);
+
+/** @brief Request to SEC kernel driver to disable interrupts for descriptor
+ * finished processing
+ * Use UIO to communicate with SEC kernel driver: write command
+ * value that indicates an IRQ disable action into UIO file descriptor
+ * of this job ring.
+ *
+ * @param [in] uio_fd UIO File descripto
+ * @retval 0 for success
+ * @retval -1 value for error
+ *
+ */
+uint32_t caam_jr_disable_irqs(uint32_t uio_fd);
+
+#endif
diff --git a/drivers/crypto/caam_jr/caam_jr_uio.c b/drivers/crypto/caam_jr/caam_jr_uio.c
new file mode 100644
index 00000000..c07d9db0
--- /dev/null
+++ b/drivers/crypto/caam_jr/caam_jr_uio.c
@@ -0,0 +1,501 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <dirent.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <errno.h>
+#include <fcntl.h>
+
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_crypto.h>
+#include <rte_security.h>
+
+#include <caam_jr_config.h>
+#include <caam_jr_hw_specific.h>
+#include <caam_jr_pvt.h>
+#include <caam_jr_log.h>
+
+/* RTA header files */
+#include <hw/desc/common.h>
+#include <hw/desc/algo.h>
+#include <hw/desc/ipsec.h>
+
+/* Prefix path to sysfs directory where UIO device attributes are exported.
+ * Path for UIO device X is /sys/class/uio/uioX
+ */
+#define SEC_UIO_DEVICE_SYS_ATTR_PATH "/sys/class/uio"
+
+/* Subfolder in sysfs where mapping attributes are exported
+ * for each UIO device. Path for mapping Y for device X is:
+ * /sys/class/uio/uioX/maps/mapY
+ */
+#define SEC_UIO_DEVICE_SYS_MAP_ATTR "maps/map"
+
+/* Name of UIO device file prefix. Each UIO device will have a device file
+ * /dev/uioX, where X is the minor device number.
+ */
+#define SEC_UIO_DEVICE_FILE_NAME "/dev/uio"
+
+/*
+ * Name of UIO device. Each user space SEC job ring will have a corresponding
+ * UIO device with the name sec-channelX, where X is the job ring id.
+ * Maximum length is #SEC_UIO_MAX_DEVICE_NAME_LENGTH.
+ *
+ * @note Must be kept in synch with SEC kernel driver
+ * define #SEC_UIO_DEVICE_NAME !
+ */
+#define SEC_UIO_DEVICE_NAME "fsl-jr"
+
+/* Maximum length for the name of an UIO device file.
+ * Device file name format is: /dev/uioX.
+ */
+#define SEC_UIO_MAX_DEVICE_FILE_NAME_LENGTH 30
+
+/* Maximum length for the name of an attribute file for an UIO device.
+ * Attribute files are exported in sysfs and have the name formatted as:
+ * /sys/class/uio/uioX/<attribute_file_name>
+ */
+#define SEC_UIO_MAX_ATTR_FILE_NAME 100
+
+/* Command that is used by SEC user space driver and SEC kernel driver
+ * to signal a request from the former to the later to disable job DONE
+ * and error IRQs on a certain job ring.
+ * The configuration is done at SEC Controller's level.
+ * @note Need to be kept in synch with #SEC_UIO_DISABLE_IRQ_CMD from
+ * linux/drivers/crypto/talitos.c !
+ */
+#define SEC_UIO_DISABLE_IRQ_CMD 0
+
+/* Command that is used by SEC user space driver and SEC kernel driver
+ * to signal a request from the former to the later to enable job DONE
+ * and error IRQs on a certain job ring.
+ * The configuration is done at SEC Controller's level.
+ * @note Need to be kept in synch with #SEC_UIO_ENABLE_IRQ_CMD from
+ * linux/drivers/crypto/talitos.c !
+ */
+#define SEC_UIO_ENABLE_IRQ_CMD 1
+
+/** Command that is used by SEC user space driver and SEC kernel driver
+ * to signal a request from the former to the later to do a SEC engine reset.
+ * @note Need to be kept in synch with #SEC_UIO_RESET_SEC_ENGINE_CMD from
+ * linux/drivers/crypto/talitos.c !
+ */
+#define SEC_UIO_RESET_SEC_ENGINE_CMD 3
+
+/* The id for the mapping used to export SEC's registers to
+ * user space through UIO devices.
+ */
+#define SEC_UIO_MAP_ID 0
+
+static struct uio_job_ring g_uio_job_ring[MAX_SEC_JOB_RINGS];
+static int g_uio_jr_num;
+
+/** @brief Checks if a file name contains a certain substring.
+ * If so, it extracts the number following the substring.
+ * This function assumes a filename format of: [text][number].
+ * @param [in] filename File name
+ * @param [in] match String to match in file name
+ * @param [out] number The number extracted from filename
+ *
+ * @retval true if file name matches the criteria
+ * @retval false if file name does not match the criteria
+ */
+static bool
+file_name_match_extract(const char filename[], const char match[], int *number)
+{
+ char *substr = NULL;
+
+ substr = strstr(filename, match);
+ if (substr == NULL)
+ return false;
+
+ /* substring <match> was found in <filename>
+ * read number following <match> substring in <filename>
+ */
+ if (sscanf(filename + strlen(match), "%d", number) <= 0)
+ return false;
+
+ return true;
+}
+
+/** @brief Reads first line from a file.
+ * Composes file name as: root/subdir/filename
+ *
+ * @param [in] root Root path
+ * @param [in] subdir Subdirectory name
+ * @param [in] filename File name
+ * @param [out] line The first line read from file.
+ *
+ * @retval 0 for succes
+ * @retval other value for error
+ */
+static int
+file_read_first_line(const char root[], const char subdir[],
+ const char filename[], char *line)
+{
+ char absolute_file_name[SEC_UIO_MAX_ATTR_FILE_NAME];
+ int fd = 0, ret = 0;
+
+ /*compose the file name: root/subdir/filename */
+ memset(absolute_file_name, 0, sizeof(absolute_file_name));
+ snprintf(absolute_file_name, SEC_UIO_MAX_ATTR_FILE_NAME,
+ "%s/%s/%s", root, subdir, filename);
+
+ fd = open(absolute_file_name, O_RDONLY);
+ SEC_ASSERT(fd > 0, fd, "Error opening file %s",
+ absolute_file_name);
+
+ /* read UIO device name from first line in file */
+ ret = read(fd, line, SEC_UIO_MAX_DEVICE_FILE_NAME_LENGTH);
+ close(fd);
+
+ /* NULL-ify string */
+ line[SEC_UIO_MAX_DEVICE_FILE_NAME_LENGTH - 1] = '\0';
+
+ if (ret <= 0) {
+ CAAM_JR_ERR("Error reading from file %s", absolute_file_name);
+ return ret;
+ }
+
+ return 0;
+}
+
+/** @brief Uses UIO control to send commands to SEC kernel driver.
+ * The mechanism is to write a command word into the file descriptor
+ * that the user-space driver obtained for each user-space SEC job ring.
+ * Both user-space driver and kernel driver must have the same understanding
+ * about the command codes.
+ *
+ * @param [in] UIO FD The UIO file descriptor
+ * @param [in] uio_command Command word
+ *
+ * @retval Result of write operation on the job ring's UIO file descriptor.
+ * Should be sizeof(int) for success operations.
+ * Other values can be returned and used, if desired to add special
+ * meaning to return values, but this has to be programmed in SEC
+ * kernel driver as well. No special return values are used.
+ */
+static int
+sec_uio_send_command(uint32_t uio_fd, int32_t uio_command)
+{
+ int ret;
+
+ /* Use UIO file descriptor we have for this job ring.
+ * Writing a command code to this file descriptor will make the
+ * SEC kernel driver execute the desired command.
+ */
+ ret = write(uio_fd, &uio_command, sizeof(int));
+ return ret;
+}
+
+/** @brief Request to SEC kernel driver to enable interrupts for
+ * descriptor finished processing
+ * Use UIO to communicate with SEC kernel driver: write command
+ * value that indicates an IRQ enable action into UIO file descriptor
+ * of this job ring.
+ *
+ * @param [in] uio_fd Job Ring UIO File descriptor
+ * @retval 0 for success
+ * @retval -1 value for error
+ */
+uint32_t
+caam_jr_enable_irqs(uint32_t uio_fd)
+{
+ int ret;
+
+ /* Use UIO file descriptor we have for this job ring.
+ * Writing a command code to this file descriptor will make the
+ * SEC kernel driver enable DONE and Error IRQs for this job ring,
+ * at Controller level.
+ */
+ ret = sec_uio_send_command(uio_fd, SEC_UIO_ENABLE_IRQ_CMD);
+ SEC_ASSERT(ret == sizeof(int), -1,
+ "Failed to request SEC engine to enable job done and "
+ "error IRQs through UIO control. UIO FD %d. Reset SEC driver!",
+ uio_fd);
+ CAAM_JR_DEBUG("Enabled IRQs on jr with uio_fd %d", uio_fd);
+ return 0;
+}
+
+
+/** @brief Request to SEC kernel driver to disable interrupts for descriptor
+ * finished processing
+ * Use UIO to communicate with SEC kernel driver: write command
+ * value that indicates an IRQ disable action into UIO file descriptor
+ * of this job ring.
+ *
+ * @param [in] uio_fd UIO File descripto
+ * @retval 0 for success
+ * @retval -1 value for error
+ *
+ */
+uint32_t
+caam_jr_disable_irqs(uint32_t uio_fd)
+{
+ int ret;
+
+ /* Use UIO file descriptor we have for this job ring.
+ * Writing a command code to this file descriptor will make the
+ * SEC kernel driver disable IRQs for this job ring,
+ * at Controller level.
+ */
+
+ ret = sec_uio_send_command(uio_fd, SEC_UIO_DISABLE_IRQ_CMD);
+ SEC_ASSERT(ret == sizeof(int), -1,
+ "Failed to request SEC engine to disable job done and "
+ "IRQs through UIO control. UIO_FD %d Reset SEC driver!",
+ uio_fd);
+ CAAM_JR_DEBUG("Disabled IRQs on jr with uio_fd %d", uio_fd);
+ return 0;
+}
+
+/** @brief Maps register range assigned for a job ring.
+ *
+ * @param [in] uio_device_fd UIO device file descriptor
+ * @param [in] uio_device_id UIO device id
+ * @param [in] uio_map_id UIO allows maximum 5 different mapping for
+ each device. Maps start with id 0.
+ * @param [out] map_size Map size.
+ * @retval NULL if failed to map registers
+ * @retval Virtual address for mapped register address range
+ */
+static void *
+uio_map_registers(int uio_device_fd, int uio_device_id,
+ int uio_map_id, int *map_size)
+{
+ void *mapped_address = NULL;
+ unsigned int uio_map_size = 0;
+ char uio_sys_root[SEC_UIO_MAX_ATTR_FILE_NAME];
+ char uio_sys_map_subdir[SEC_UIO_MAX_ATTR_FILE_NAME];
+ char uio_map_size_str[32];
+ int ret = 0;
+
+ /* compose the file name: root/subdir/filename */
+ memset(uio_sys_root, 0, sizeof(uio_sys_root));
+ memset(uio_sys_map_subdir, 0, sizeof(uio_sys_map_subdir));
+ memset(uio_map_size_str, 0, sizeof(uio_map_size_str));
+
+ /* Compose string: /sys/class/uio/uioX */
+ sprintf(uio_sys_root, "%s/%s%d", SEC_UIO_DEVICE_SYS_ATTR_PATH,
+ "uio", uio_device_id);
+ /* Compose string: maps/mapY */
+ sprintf(uio_sys_map_subdir, "%s%d", SEC_UIO_DEVICE_SYS_MAP_ATTR,
+ uio_map_id);
+
+ /* Read first (and only) line from file
+ * /sys/class/uio/uioX/maps/mapY/size
+ */
+ ret = file_read_first_line(uio_sys_root, uio_sys_map_subdir,
+ "size", uio_map_size_str);
+ SEC_ASSERT(ret == 0, NULL, "file_read_first_line() failed");
+
+ /* Read mapping size, expressed in hexa(base 16) */
+ uio_map_size = strtol(uio_map_size_str, NULL, 16);
+
+ /* Map the region in user space */
+ mapped_address = mmap(0, /*dynamically choose virtual address */
+ uio_map_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED, uio_device_fd, 0);
+ /* offset = 0 because UIO device has only one mapping
+ * for the entire SEC register memory
+ */
+ if (mapped_address == MAP_FAILED) {
+ CAAM_JR_ERR(
+ "Failed to map registers! errno = %d job ring fd = %d,"
+ "uio device id = %d, uio map id = %d", errno,
+ uio_device_fd, uio_device_id, uio_map_id);
+ return NULL;
+ }
+
+ /*
+ * Save the map size to use it later on for munmap-ing.
+ */
+ *map_size = uio_map_size;
+
+ CAAM_JR_INFO("UIO dev[%d] mapped region [id =%d] size 0x%x at %p",
+ uio_device_id, uio_map_id, uio_map_size, mapped_address);
+
+ return mapped_address;
+}
+
+void
+free_job_ring(uint32_t uio_fd)
+{
+ struct uio_job_ring *job_ring = NULL;
+ int i;
+
+ if (!job_ring->uio_fd)
+ return;
+
+ for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
+ if (g_uio_job_ring[i].uio_fd == uio_fd) {
+ job_ring = &g_uio_job_ring[i];
+ break;
+ }
+ }
+
+ if (job_ring == NULL) {
+ CAAM_JR_ERR("JR not available for fd = %x\n", uio_fd);
+ return;
+ }
+
+ /* Open device file */
+ CAAM_JR_INFO("Closed device file for job ring %d , fd = %d",
+ job_ring->jr_id, job_ring->uio_fd);
+ close(job_ring->uio_fd);
+ g_uio_jr_num--;
+ job_ring->uio_fd = 0;
+ if (job_ring->register_base_addr == NULL)
+ return;
+
+ /* Unmap the PCI memory resource of device */
+ if (munmap(job_ring->register_base_addr, job_ring->map_size)) {
+ CAAM_JR_INFO("cannot munmap(%p, 0x%lx): %s",
+ job_ring->register_base_addr,
+ (unsigned long)job_ring->map_size, strerror(errno));
+ } else
+ CAAM_JR_DEBUG(" JR UIO memory unmapped at %p",
+ job_ring->register_base_addr);
+ job_ring->register_base_addr = NULL;
+}
+
+struct
+uio_job_ring *config_job_ring(void)
+{
+ char uio_device_file_name[32];
+ struct uio_job_ring *job_ring = NULL;
+ int i;
+
+ for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
+ if (g_uio_job_ring[i].uio_fd == 0) {
+ job_ring = &g_uio_job_ring[i];
+ g_uio_jr_num++;
+ break;
+ }
+ }
+
+ if (job_ring == NULL) {
+ CAAM_JR_ERR("No free job ring\n");
+ return NULL;
+ }
+
+ /* Find UIO device created by SEC kernel driver for this job ring. */
+ memset(uio_device_file_name, 0, sizeof(uio_device_file_name));
+
+ sprintf(uio_device_file_name, "%s%d", SEC_UIO_DEVICE_FILE_NAME,
+ job_ring->uio_minor_number);
+
+ /* Open device file */
+ job_ring->uio_fd = open(uio_device_file_name, O_RDWR);
+ SEC_ASSERT(job_ring->uio_fd > 0, NULL,
+ "Failed to open UIO device file for job ring %d",
+ job_ring->jr_id);
+
+ CAAM_JR_INFO("Open device(%s) file for job ring=%d , uio_fd = %d",
+ uio_device_file_name, job_ring->jr_id, job_ring->uio_fd);
+
+ ASSERT(job_ring->register_base_addr == NULL);
+ job_ring->register_base_addr = uio_map_registers(
+ job_ring->uio_fd, job_ring->uio_minor_number,
+ SEC_UIO_MAP_ID, &job_ring->map_size);
+
+ SEC_ASSERT(job_ring->register_base_addr != NULL, NULL,
+ "Failed to map SEC registers");
+ return job_ring;
+}
+
+int
+sec_configure(void)
+{
+ char uio_name[32];
+ int config_jr_no = 0, jr_id = -1;
+ int uio_minor_number = -1;
+ int ret;
+ DIR *d = NULL;
+ struct dirent *dir;
+
+ d = opendir(SEC_UIO_DEVICE_SYS_ATTR_PATH);
+ if (d == NULL) {
+ printf("\nError opening directory '%s': %s\n",
+ SEC_UIO_DEVICE_SYS_ATTR_PATH, strerror(errno));
+ return -1;
+ }
+
+ /* Iterate through all subdirs */
+ while ((dir = readdir(d)) != NULL) {
+ if (!strncmp(dir->d_name, ".", 1) ||
+ !strncmp(dir->d_name, "..", 2))
+ continue;
+
+ if (file_name_match_extract
+ (dir->d_name, "uio", &uio_minor_number)) {
+ /*
+ * Open file uioX/name and read first line which contains
+ * the name for the device. Based on the name check if this
+ * UIO device is UIO device for job ring with id jr_id.
+ */
+ memset(uio_name, 0, sizeof(uio_name));
+ ret = file_read_first_line(SEC_UIO_DEVICE_SYS_ATTR_PATH,
+ dir->d_name, "name", uio_name);
+ CAAM_JR_INFO("sec device uio name: %s", uio_name);
+ SEC_ASSERT(ret == 0, -1, "file_read_first_line failed");
+
+ if (file_name_match_extract(uio_name,
+ SEC_UIO_DEVICE_NAME,
+ &jr_id)) {
+ g_uio_job_ring[config_jr_no].jr_id = jr_id;
+ g_uio_job_ring[config_jr_no].uio_minor_number =
+ uio_minor_number;
+ CAAM_JR_INFO("Detected logical JRID:%d", jr_id);
+ config_jr_no++;
+
+ /* todo find the actual ring id
+ * OF_FULLNAME=/soc/crypto@1700000/jr@20000
+ */
+ }
+ }
+ }
+ closedir(d);
+
+ if (config_jr_no == 0) {
+ CAAM_JR_ERR("! No SEC Job Rings assigned for userspace usage!");
+ return 0;
+ }
+ CAAM_JR_INFO("Total JR detected =%d", config_jr_no);
+ return config_jr_no;
+}
+
+int
+sec_cleanup(void)
+{
+ int i;
+ struct uio_job_ring *job_ring;
+
+ for (i = 0; i < g_uio_jr_num; i++) {
+ job_ring = &g_uio_job_ring[i];
+ /* munmap SEC's register memory */
+ if (job_ring->register_base_addr) {
+ munmap(job_ring->register_base_addr,
+ job_ring->map_size);
+ job_ring->register_base_addr = NULL;
+ }
+ /* I need to close the fd after shutdown UIO commands need to be
+ * sent using the fd
+ */
+ if (job_ring->uio_fd != 0) {
+ CAAM_JR_INFO(
+ "Closed device file for job ring %d , fd = %d",
+ job_ring->jr_id, job_ring->uio_fd);
+ close(job_ring->uio_fd);
+ }
+ }
+ return 0;
+}
diff --git a/drivers/crypto/caam_jr/meson.build b/drivers/crypto/caam_jr/meson.build
new file mode 100644
index 00000000..99b71aef
--- /dev/null
+++ b/drivers/crypto/caam_jr/meson.build
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2018 NXP
+
+if host_machine.system() != 'linux'
+ build = false
+endif
+
+deps += ['bus_vdev', 'bus_dpaa', 'security']
+sources = files('caam_jr_capabilities.c',
+ 'caam_jr_hw.c',
+ 'caam_jr_uio.c',
+ 'caam_jr.c')
+
+allow_experimental_apis = true
+
+includes += include_directories('../dpaa2_sec/')
+includes += include_directories('../../bus/dpaa/include/')
diff --git a/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map b/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map
new file mode 100644
index 00000000..521e51f4
--- /dev/null
+++ b/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map
@@ -0,0 +1,4 @@
+DPDK_18.11 {
+
+ local: *;
+};