path: root/drivers/crypto/caam_jr/caam_jr.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright 2017-2018 NXP
 */

#include <fcntl.h>
#include <unistd.h>
#include <sched.h>
#include <net/if.h>

#include <rte_byteorder.h>
#include <rte_common.h>
#include <rte_cryptodev_pmd.h>
#include <rte_crypto.h>
#include <rte_cryptodev.h>
#include <rte_bus_vdev.h>
#include <rte_malloc.h>
#include <rte_security_driver.h>
#include <rte_hexdump.h>

#include <caam_jr_capabilities.h>
#include <caam_jr_config.h>
#include <caam_jr_hw_specific.h>
#include <caam_jr_pvt.h>
#include <caam_jr_desc.h>
#include <caam_jr_log.h>

/* RTA header files */
#include <hw/desc/common.h>
#include <hw/desc/algo.h>
#include <of.h>

#define CAAM_JR_DBG	0
#define CRYPTODEV_NAME_CAAM_JR_PMD	crypto_caam_jr
static uint8_t cryptodev_driver_id;
int caam_jr_logtype;

enum rta_sec_era rta_sec_era;

/* Lists the states possible for the SEC user space driver. */
enum sec_driver_state_e {
	SEC_DRIVER_STATE_IDLE,		/* Driver not initialized */
	SEC_DRIVER_STATE_STARTED,	/* Driver initialized and can be used*/
	SEC_DRIVER_STATE_RELEASE,	/* Driver release is in progress */
};

/* Job rings used for communication with SEC HW */
static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS];

/* The current state of SEC user space driver */
static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE;

/* The number of job rings used by SEC user space driver */
static int g_job_rings_no;
static int g_job_rings_max;

struct sec_outring_entry {
	phys_addr_t desc;	/* Pointer to completed descriptor */
	uint32_t status;	/* Status for completed descriptor */
} __rte_packed;

/* virtual address conversin when mempool support is available for ctx */
static inline phys_addr_t
caam_jr_vtop_ctx(struct caam_jr_op_ctx *ctx, void *vaddr)
{
	PMD_INIT_FUNC_TRACE();
	return (size_t)vaddr - ctx->vtop_offset;
}

static inline void
caam_jr_op_ending(struct caam_jr_op_ctx *ctx)
{
	PMD_INIT_FUNC_TRACE();
	/* report op status to sym->op and then free the ctx memeory  */
	rte_mempool_put(ctx->ctx_pool, (void *)ctx);
}

static inline struct caam_jr_op_ctx *
caam_jr_alloc_ctx(struct caam_jr_session *ses)
{
	struct caam_jr_op_ctx *ctx;
	int ret;

	PMD_INIT_FUNC_TRACE();
	ret = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
	if (!ctx || ret) {
		CAAM_JR_DP_WARN("Alloc sec descriptor failed!");
		return NULL;
	}
	/*
	 * Clear SG memory. There are 16 SG entries of 16 Bytes each.
	 * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
	 * to clear all the SG entries. caam_jr_alloc_ctx() is called for
	 * each packet, memset is costlier than dcbz_64().
	 */
	dcbz_64(&ctx->sg[SG_CACHELINE_0]);
	dcbz_64(&ctx->sg[SG_CACHELINE_1]);
	dcbz_64(&ctx->sg[SG_CACHELINE_2]);
	dcbz_64(&ctx->sg[SG_CACHELINE_3]);

	ctx->ctx_pool = ses->ctx_pool;
	ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);

	return ctx;
}

static
void caam_jr_stats_get(struct rte_cryptodev *dev,
			struct rte_cryptodev_stats *stats)
{
	struct caam_jr_qp **qp = (struct caam_jr_qp **)
					dev->data->queue_pairs;
	int i;

	PMD_INIT_FUNC_TRACE();
	if (stats == NULL) {
		CAAM_JR_ERR("Invalid stats ptr NULL");
		return;
	}
	for (i = 0; i < dev->data->nb_queue_pairs; i++) {
		if (qp[i] == NULL) {
			CAAM_JR_WARN("Uninitialised queue pair");
			continue;
		}

		stats->enqueued_count += qp[i]->tx_pkts;
		stats->dequeued_count += qp[i]->rx_pkts;
		stats->enqueue_err_count += qp[i]->tx_errs;
		stats->dequeue_err_count += qp[i]->rx_errs;
		CAAM_JR_INFO("extra stats:\n\tRX Poll ERR = %" PRIu64
			     "\n\tTX Ring Full = %" PRIu64,
			     qp[i]->rx_poll_err,
			     qp[i]->tx_ring_full);
	}
}

static
void caam_jr_stats_reset(struct rte_cryptodev *dev)
{
	int i;
	struct caam_jr_qp **qp = (struct caam_jr_qp **)
				   (dev->data->queue_pairs);

	PMD_INIT_FUNC_TRACE();
	for (i = 0; i < dev->data->nb_queue_pairs; i++) {
		if (qp[i] == NULL) {
			CAAM_JR_WARN("Uninitialised queue pair");
			continue;
		}
		qp[i]->rx_pkts = 0;
		qp[i]->rx_errs = 0;
		qp[i]->rx_poll_err = 0;
		qp[i]->tx_pkts = 0;
		qp[i]->tx_errs = 0;
		qp[i]->tx_ring_full = 0;
	}
}

static inline int
is_cipher_only(struct caam_jr_session *ses)
{
	PMD_INIT_FUNC_TRACE();
	return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
		(ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
}

static inline int
is_auth_only(struct caam_jr_session *ses)
{
	PMD_INIT_FUNC_TRACE();
	return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
		(ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
}

static inline int
is_aead(struct caam_jr_session *ses)
{
	PMD_INIT_FUNC_TRACE();
	return ((ses->cipher_alg == 0) &&
		(ses->auth_alg == 0) &&
		(ses->aead_alg != 0));
}

static inline int
is_auth_cipher(struct caam_jr_session *ses)
{
	PMD_INIT_FUNC_TRACE();
	return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
		(ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
		(ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
}

static inline int
is_proto_ipsec(struct caam_jr_session *ses)
{
	PMD_INIT_FUNC_TRACE();
	return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
}

static inline int
is_encode(struct caam_jr_session *ses)
{
	PMD_INIT_FUNC_TRACE();
	return ses->dir == DIR_ENC;
}

static inline int
is_decode(struct caam_jr_session *ses)
{
	PMD_INIT_FUNC_TRACE();
	return ses->dir == DIR_DEC;
}

static inline void
caam_auth_alg(struct caam_jr_session *ses, struct alginfo *alginfo_a)
{
	PMD_INIT_FUNC_TRACE();
	switch (ses->auth_alg) {
	case RTE_CRYPTO_AUTH_NULL:
		ses->digest_length = 0;
		break;
	case RTE_CRYPTO_AUTH_MD5_HMAC:
		alginfo_a->algtype =
			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
			OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
		alginfo_a->algmode = OP_ALG_AAI_HMAC;
		break;
	case RTE_CRYPTO_AUTH_SHA1_HMAC:
		alginfo_a->algtype =
			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
			OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
		alginfo_a->algmode = OP_ALG_AAI_HMAC;
		break;
	case RTE_CRYPTO_AUTH_SHA224_HMAC:
		alginfo_a->algtype =
			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
			OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
		alginfo_a->algmode = OP_ALG_AAI_HMAC;
		break;
	case RTE_CRYPTO_AUTH_SHA256_HMAC:
		alginfo_a->algtype =
			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
			OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
		alginfo_a->algmode = OP_ALG_AAI_HMAC;
		break;
	case RTE_CRYPTO_AUTH_SHA384_HMAC:
		alginfo_a->algtype =
			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
			OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
		alginfo_a->algmode = OP_ALG_AAI_HMAC;
		break;
	case RTE_CRYPTO_AUTH_SHA512_HMAC:
		alginfo_a->algtype =
			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
			OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
		alginfo_a->algmode = OP_ALG_AAI_HMAC;
		break;
	default:
		CAAM_JR_DEBUG("unsupported auth alg %u", ses->auth_alg);
	}
}

static inline void
caam_cipher_alg(struct caam_jr_session *ses, struct alginfo *alginfo_c)
{
	PMD_INIT_FUNC_TRACE();
	switch (ses->cipher_alg) {
	case RTE_CRYPTO_CIPHER_NULL:
		break;
	case RTE_CRYPTO_CIPHER_AES_CBC:
		alginfo_c->algtype =
			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
			OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
		alginfo_c->algmode = OP_ALG_AAI_CBC;
		break;
	case RTE_CRYPTO_CIPHER_3DES_CBC:
		alginfo_c->algtype =
			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
			OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
		alginfo_c->algmode = OP_ALG_AAI_CBC;
		break;
	case RTE_CRYPTO_CIPHER_AES_CTR:
		alginfo_c->algtype =
			(ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
			OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
		alginfo_c->algmode = OP_ALG_AAI_CTR;
		break;
	default:
		CAAM_JR_DEBUG("unsupported cipher alg %d", ses->cipher_alg);
	}
}

static inline void
caam_aead_alg(struct caam_jr_session *ses, struct alginfo *alginfo)
{
	PMD_INIT_FUNC_TRACE();
	switch (ses->aead_alg) {
	case RTE_CRYPTO_AEAD_AES_GCM:
		alginfo->algtype = OP_ALG_ALGSEL_AES;
		alginfo->algmode = OP_ALG_AAI_GCM;
		break;
	default:
		CAAM_JR_DEBUG("unsupported AEAD alg %d", ses->aead_alg);
	}
}

/* prepare command block of the session */
static int
caam_jr_prep_cdb(struct caam_jr_session *ses)
{
	struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
	int32_t shared_desc_len = 0;
	struct sec_cdb *cdb;
	int err;
#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
	int swap = false;
#else
	int swap = true;
#endif

	PMD_INIT_FUNC_TRACE();
	if (ses->cdb)
		caam_jr_dma_free(ses->cdb);

	cdb = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, sizeof(struct sec_cdb));
	if (!cdb) {
		CAAM_JR_ERR("failed to allocate memory for cdb\n");
		return -1;
	}

	ses->cdb = cdb;

	memset(cdb, 0, sizeof(struct sec_cdb));

	if (is_cipher_only(ses)) {
		caam_cipher_alg(ses, &alginfo_c);
		if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
			CAAM_JR_ERR("not supported cipher alg");
			rte_free(cdb);
			return -ENOTSUP;
		}

		alginfo_c.key = (size_t)ses->cipher_key.data;
		alginfo_c.keylen = ses->cipher_key.length;
		alginfo_c.key_enc_flags = 0;
		alginfo_c.key_type = RTA_DATA_IMM;

		shared_desc_len = cnstr_shdsc_blkcipher(
						cdb->sh_desc, true,
						swap, &alginfo_c,
						NULL,
						ses->iv.length,
						ses->dir);
	} else if (is_auth_only(ses)) {
		caam_auth_alg(ses, &alginfo_a);
		if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
			CAAM_JR_ERR("not supported auth alg");
			rte_free(cdb);
			return -ENOTSUP;
		}

		alginfo_a.key = (size_t)ses->auth_key.data;
		alginfo_a.keylen = ses->auth_key.length;
		alginfo_a.key_enc_flags = 0;
		alginfo_a.key_type = RTA_DATA_IMM;

		shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
						   swap, &alginfo_a,
						   !ses->dir,
						   ses->digest_length);
	} else if (is_aead(ses)) {
		caam_aead_alg(ses, &alginfo);
		if (alginfo.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
			CAAM_JR_ERR("not supported aead alg");
			rte_free(cdb);
			return -ENOTSUP;
		}
		alginfo.key = (size_t)ses->aead_key.data;
		alginfo.keylen = ses->aead_key.length;
		alginfo.key_enc_flags = 0;
		alginfo.key_type = RTA_DATA_IMM;

		if (ses->dir == DIR_ENC)
			shared_desc_len = cnstr_shdsc_gcm_encap(
					cdb->sh_desc, true, swap,
					&alginfo,
					ses->iv.length,
					ses->digest_length);
		else
			shared_desc_len = cnstr_shdsc_gcm_decap(
					cdb->sh_desc, true, swap,
					&alginfo,
					ses->iv.length,
					ses->digest_length);
	} else {
		caam_cipher_alg(ses, &alginfo_c);
		if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
			CAAM_JR_ERR("not supported cipher alg");
			rte_free(cdb);
			return -ENOTSUP;
		}

		alginfo_c.key = (size_t)ses->cipher_key.data;
		alginfo_c.keylen = ses->cipher_key.length;
		alginfo_c.key_enc_flags = 0;
		alginfo_c.key_type = RTA_DATA_IMM;

		caam_auth_alg(ses, &alginfo_a);
		if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
			CAAM_JR_ERR("not supported auth alg");
			rte_free(cdb);
			return -ENOTSUP;
		}

		alginfo_a.key = (size_t)ses->auth_key.data;
		alginfo_a.keylen = ses->auth_key.length;
		alginfo_a.key_enc_flags = 0;
		alginfo_a.key_type = RTA_DATA_IMM;

		cdb->sh_desc[0] = alginfo_c.keylen;
		cdb->sh_desc[1] = alginfo_a.keylen;
		err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
				       MIN_JOB_DESC_SIZE,
				       (unsigned int *)cdb->sh_desc,
				       &cdb->sh_desc[2], 2);

		if (err < 0) {
			CAAM_JR_ERR("Crypto: Incorrect key lengths");
			rte_free(cdb);
			return err;
		}
		if (cdb->sh_desc[2] & 1)
			alginfo_c.key_type = RTA_DATA_IMM;
		else {
			alginfo_c.key = (size_t)caam_jr_mem_vtop(
						(void *)(size_t)alginfo_c.key);
			alginfo_c.key_type = RTA_DATA_PTR;
		}
		if (cdb->sh_desc[2] & (1<<1))
			alginfo_a.key_type = RTA_DATA_IMM;
		else {
			alginfo_a.key = (size_t)caam_jr_mem_vtop(
						(void *)(size_t)alginfo_a.key);
			alginfo_a.key_type = RTA_DATA_PTR;
		}
		cdb->sh_desc[0] = 0;
		cdb->sh_desc[1] = 0;
		cdb->sh_desc[2] = 0;
		if (is_proto_ipsec(ses)) {
			if (ses->dir == DIR_ENC) {
				shared_desc_len = cnstr_shdsc_ipsec_new_encap(
						cdb->sh_desc,
						true, swap, SHR_SERIAL,
						&ses->encap_pdb,
						(uint8_t *)&ses->ip4_hdr,
						&alginfo_c, &alginfo_a);
			} else if (ses->dir == DIR_DEC) {
				shared_desc_len = cnstr_shdsc_ipsec_new_decap(
						cdb->sh_desc,
						true, swap, SHR_SERIAL,
						&ses->decap_pdb,
						&alginfo_c, &alginfo_a);
			}
		} else {
			/* Auth_only_len is set as 0 here and it will be
			 * overwritten in fd for each packet.
			 */
			shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
					true, swap, &alginfo_c, &alginfo_a,
					ses->iv.length, 0,
					ses->digest_length, ses->dir);
		}
	}

	if (shared_desc_len < 0) {
		CAAM_JR_ERR("error in preparing command block");
		return shared_desc_len;
	}

#if CAAM_JR_DBG
	SEC_DUMP_DESC(cdb->sh_desc);
#endif

	cdb->sh_hdr.hi.field.idlen = shared_desc_len;

	return 0;
}

/* @brief Poll the HW for already processed jobs in the JR
 * and silently discard the available jobs or notify them to UA
 * with indicated error code.
 *
 * @param [in,out]  job_ring        The job ring to poll.
 * @param [in]  do_notify           Can be #TRUE or #FALSE. Indicates if
 *				    descriptors are to be discarded
 *                                  or notified to UA with given error_code.
 * @param [out] notified_descs    Number of notified descriptors. Can be NULL
 *					if do_notify is #FALSE
 */
static void
hw_flush_job_ring(struct sec_job_ring_t *job_ring,
		  uint32_t do_notify,
		  uint32_t *notified_descs)
{
	int32_t jobs_no_to_discard = 0;
	int32_t discarded_descs_no = 0;

	PMD_INIT_FUNC_TRACE();
	CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]",
		job_ring, job_ring->pidx, job_ring->cidx, do_notify);

	jobs_no_to_discard = hw_get_no_finished_jobs(job_ring);

	/* Discard all jobs */
	CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs",
		  job_ring, job_ring->pidx, job_ring->cidx,
		  jobs_no_to_discard);

	while (jobs_no_to_discard > discarded_descs_no) {
		discarded_descs_no++;
		/* Now increment the consumer index for the current job ring,
		 * AFTER saving job in temporary location!
		 * Increment the consumer index for the current job ring
		 */
		job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
					 SEC_JOB_RING_SIZE);

		hw_remove_entries(job_ring, 1);
	}

	if (do_notify == true) {
		ASSERT(notified_descs != NULL);
		*notified_descs = discarded_descs_no;
	}
}

/* @brief Poll the HW for already processed jobs in the JR
 * and notify the available jobs to UA.
 *
 * @param [in]  job_ring	The job ring to poll.
 * @param [in]  limit           The maximum number of jobs to notify.
 *                              If set to negative value, all available jobs are
 *				notified.
 *
 * @retval >=0 for No of jobs notified to UA.
 * @retval -1 for error
 */
static int
hw_poll_job_ring(struct sec_job_ring_t *job_ring,
		 struct rte_crypto_op **ops, int32_t limit,
		 struct caam_jr_qp *jr_qp)
{
	int32_t jobs_no_to_notify = 0; /* the number of done jobs to notify*/
	int32_t number_of_jobs_available = 0;
	int32_t notified_descs_no = 0;
	uint32_t sec_error_code = 0;
	struct job_descriptor *current_desc;
	phys_addr_t current_desc_addr;
	phys_addr_t *temp_addr;
	struct caam_jr_op_ctx *ctx;

	PMD_INIT_FUNC_TRACE();
	/* TODO check for ops have memory*/
	/* check here if any JR error that cannot be written
	 * in the output status word has occurred
	 */
	if (JR_REG_JRINT_JRE_EXTRACT(GET_JR_REG(JRINT, job_ring))) {
		CAAM_JR_INFO("err received");
		sec_error_code = JR_REG_JRINT_ERR_TYPE_EXTRACT(
					GET_JR_REG(JRINT, job_ring));
		if (unlikely(sec_error_code)) {
			hw_job_ring_error_print(job_ring, sec_error_code);
			return -1;
		}
	}
	/* compute the number of jobs available in the job ring based on the
	 * producer and consumer index values.
	 */
	number_of_jobs_available = hw_get_no_finished_jobs(job_ring);
	/* Compute the number of notifications that need to be raised to UA
	 * If limit > total number of done jobs -> notify all done jobs
	 * If limit = 0 -> error
	 * If limit < total number of done jobs -> notify a number
	 * of done jobs equal with limit
	 */
	jobs_no_to_notify = (limit > number_of_jobs_available) ?
				number_of_jobs_available : limit;
	CAAM_JR_DP_DEBUG(
		"Jr[%p] pi[%d] ci[%d].limit =%d Available=%d.Jobs to notify=%d",
		job_ring, job_ring->pidx, job_ring->cidx,
		limit, number_of_jobs_available, jobs_no_to_notify);

	rte_smp_rmb();

	while (jobs_no_to_notify > notified_descs_no) {
		static uint64_t false_alarm;
		static uint64_t real_poll;

		/* Get job status here */
		sec_error_code = job_ring->output_ring[job_ring->cidx].status;
		/* Get completed descriptor */
		temp_addr = &(job_ring->output_ring[job_ring->cidx].desc);
		current_desc_addr = (phys_addr_t)sec_read_addr(temp_addr);

		real_poll++;
		/* todo check if it is false alarm no desc present */
		if (!current_desc_addr) {
			false_alarm++;
			printf("false alarm %" PRIu64 "real %" PRIu64
				" sec_err =0x%x cidx Index =0%d\n",
				false_alarm, real_poll,
				sec_error_code, job_ring->cidx);
			rte_panic("CAAM JR descriptor NULL");
			return notified_descs_no;
		}
		current_desc = (struct job_descriptor *)
				caam_jr_dma_ptov(current_desc_addr);
		/* now increment the consumer index for the current job ring,
		 * AFTER saving job in temporary location!
		 */
		job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
				 SEC_JOB_RING_SIZE);
		/* Signal that the job has been processed and the slot is free*/
		hw_remove_entries(job_ring, 1);
		/*TODO for multiple ops, packets*/
		ctx = container_of(current_desc, struct caam_jr_op_ctx, jobdes);
		if (unlikely(sec_error_code)) {
			CAAM_JR_ERR("desc at cidx %d generated error 0x%x\n",
				job_ring->cidx, sec_error_code);
			hw_handle_job_ring_error(job_ring, sec_error_code);
			//todo improve with exact errors
			ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
			jr_qp->rx_errs++;
		} else {
			ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
#if CAAM_JR_DBG
			if (ctx->op->sym->m_dst) {
				rte_hexdump(stdout, "PROCESSED",
				rte_pktmbuf_mtod(ctx->op->sym->m_dst, void *),
				rte_pktmbuf_data_len(ctx->op->sym->m_dst));
			} else {
				rte_hexdump(stdout, "PROCESSED",
				rte_pktmbuf_mtod(ctx->op->sym->m_src, void *),
				rte_pktmbuf_data_len(ctx->op->sym->m_src));
			}
#endif
		}
		if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
			struct ip *ip4_hdr;

			if (ctx->op->sym->m_dst) {
				/*TODO check for ip header or other*/
				ip4_hdr = (struct ip *)
				rte_pktmbuf_mtod(ctx->op->sym->m_dst, char*);
				ctx->op->sym->m_dst->pkt_len =
					rte_be_to_cpu_16(ip4_hdr->ip_len);
				ctx->op->sym->m_dst->data_len =
					rte_be_to_cpu_16(ip4_hdr->ip_len);
			} else {
				ip4_hdr = (struct ip *)
				rte_pktmbuf_mtod(ctx->op->sym->m_src, char*);
				ctx->op->sym->m_src->pkt_len =
					rte_be_to_cpu_16(ip4_hdr->ip_len);
				ctx->op->sym->m_src->data_len =
					rte_be_to_cpu_16(ip4_hdr->ip_len);
			}
		}
		*ops = ctx->op;
		caam_jr_op_ending(ctx);
		ops++;
		notified_descs_no++;
	}
	return notified_descs_no;
}

static uint16_t
caam_jr_dequeue_burst(void *qp, struct rte_crypto_op **ops,
		       uint16_t nb_ops)
{
	struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
	struct sec_job_ring_t *ring = jr_qp->ring;
	int num_rx;
	int ret;

	PMD_INIT_FUNC_TRACE();
	CAAM_JR_DP_DEBUG("Jr[%p]Polling. limit[%d]", ring, nb_ops);

	/* Poll job ring
	 * If nb_ops < 0 -> poll JR until no more notifications are available.
	 * If nb_ops > 0 -> poll JR until limit is reached.
	 */

	/* Run hw poll job ring */
	num_rx = hw_poll_job_ring(ring, ops, nb_ops, jr_qp);
	if (num_rx < 0) {
		CAAM_JR_ERR("Error polling SEC engine (%d)", num_rx);
		return 0;
	}

	CAAM_JR_DP_DEBUG("Jr[%p].Jobs notified[%d]. ", ring, num_rx);

	if (ring->jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
		if (num_rx < nb_ops) {
			ret = caam_jr_enable_irqs(ring->irq_fd);
			SEC_ASSERT(ret == 0, ret,
			"Failed to enable irqs for job ring %p", ring);
		}
	} else if (ring->jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {

		/* Always enable IRQ generation when in pure IRQ mode */
		ret = caam_jr_enable_irqs(ring->irq_fd);
		SEC_ASSERT(ret == 0, ret,
			"Failed to enable irqs for job ring %p", ring);
	}

	jr_qp->rx_pkts += num_rx;

	return num_rx;
}

/**
 * packet looks like:
 *		|<----data_len------->|
 *    |ip_header|ah_header|icv|payload|
 *              ^
 *		|
 *	   mbuf->pkt.data
 */
static inline struct caam_jr_op_ctx *
build_auth_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
{
	struct rte_crypto_sym_op *sym = op->sym;
	struct rte_mbuf *mbuf = sym->m_src;
	struct caam_jr_op_ctx *ctx;
	struct sec4_sg_entry *sg;
	int	length;
	struct sec_cdb *cdb;
	uint64_t sdesc_offset;
	struct sec_job_descriptor_t *jobdescr;
	uint8_t extra_segs;

	PMD_INIT_FUNC_TRACE();
	if (is_decode(ses))
		extra_segs = 2;
	else
		extra_segs = 1;

	if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
		CAAM_JR_DP_ERR("Auth: Max sec segs supported is %d",
				MAX_SG_ENTRIES);
		return NULL;
	}

	ctx = caam_jr_alloc_ctx(ses);
	if (!ctx)
		return NULL;

	ctx->op = op;

	cdb = ses->cdb;
	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);

	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;

	SEC_JD_INIT(jobdescr);
	SEC_JD_SET_SD(jobdescr,
		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
		cdb->sh_hdr.hi.field.idlen);

	/* output */
	SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
			0, ses->digest_length);

	/*input */
	sg = &ctx->sg[0];
	length = sym->auth.data.length;
	sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + sym->auth.data.offset);
	sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);

	/* Successive segs */
	mbuf = mbuf->next;
	while (mbuf) {
		sg++;
		sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
		sg->len = cpu_to_caam32(mbuf->data_len);
		mbuf = mbuf->next;
	}

	if (is_decode(ses)) {
		/* digest verification case */
		sg++;
		/* hash result or digest, save digest first */
		rte_memcpy(ctx->digest, sym->auth.digest.data,
			   ses->digest_length);
#if CAAM_JR_DBG
		rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
#endif
		sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
		sg->len = cpu_to_caam32(ses->digest_length);
		length += ses->digest_length;
	} else {
		length -= ses->digest_length;
	}

	/* last element*/
	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);

	SEC_JD_SET_IN_PTR(jobdescr,
		(uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0, length);
	/* enabling sg list */
	(jobdescr)->seq_in.command.word  |= 0x01000000;

	return ctx;
}

static inline struct caam_jr_op_ctx *
build_auth_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
{
	struct rte_crypto_sym_op *sym = op->sym;
	struct caam_jr_op_ctx *ctx;
	struct sec4_sg_entry *sg;
	rte_iova_t start_addr;
	struct sec_cdb *cdb;
	uint64_t sdesc_offset;
	struct sec_job_descriptor_t *jobdescr;

	PMD_INIT_FUNC_TRACE();
	ctx = caam_jr_alloc_ctx(ses);
	if (!ctx)
		return NULL;

	ctx->op = op;

	cdb = ses->cdb;
	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);

	start_addr = rte_pktmbuf_iova(sym->m_src);

	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;

	SEC_JD_INIT(jobdescr);
	SEC_JD_SET_SD(jobdescr,
		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
		cdb->sh_hdr.hi.field.idlen);

	/* output */
	SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
			0, ses->digest_length);

	/*input */
	if (is_decode(ses)) {
		sg = &ctx->sg[0];
		SEC_JD_SET_IN_PTR(jobdescr,
			(uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
			(sym->auth.data.length + ses->digest_length));
		/* enabling sg list */
		(jobdescr)->seq_in.command.word  |= 0x01000000;

		/* hash result or digest, save digest first */
		rte_memcpy(ctx->digest, sym->auth.digest.data,
			   ses->digest_length);
		sg->ptr = cpu_to_caam64(start_addr + sym->auth.data.offset);
		sg->len = cpu_to_caam32(sym->auth.data.length);

#if CAAM_JR_DBG
		rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
#endif
		/* let's check digest by hw */
		sg++;
		sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
		sg->len = cpu_to_caam32(ses->digest_length);
		/* last element*/
		sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
	} else {
		SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)start_addr,
			sym->auth.data.offset, sym->auth.data.length);
	}
	return ctx;
}

static inline struct caam_jr_op_ctx *
build_cipher_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
{
	struct rte_crypto_sym_op *sym = op->sym;
	struct rte_mbuf *mbuf = sym->m_src;
	struct caam_jr_op_ctx *ctx;
	struct sec4_sg_entry *sg, *in_sg;
	int length;
	struct sec_cdb *cdb;
	uint64_t sdesc_offset;
	uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
			ses->iv.offset);
	struct sec_job_descriptor_t *jobdescr;
	uint8_t reg_segs;

	PMD_INIT_FUNC_TRACE();
	if (sym->m_dst) {
		mbuf = sym->m_dst;
		reg_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2;
	} else {
		mbuf = sym->m_src;
		reg_segs = mbuf->nb_segs * 2 + 2;
	}

	if (reg_segs > MAX_SG_ENTRIES) {
		CAAM_JR_DP_ERR("Cipher: Max sec segs supported is %d",
				MAX_SG_ENTRIES);
		return NULL;
	}

	ctx = caam_jr_alloc_ctx(ses);
	if (!ctx)
		return NULL;

	ctx->op = op;
	cdb = ses->cdb;
	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);

	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;

	SEC_JD_INIT(jobdescr);
	SEC_JD_SET_SD(jobdescr,
		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
		cdb->sh_hdr.hi.field.idlen);

#if CAAM_JR_DBG
	CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
			sym->m_src->data_off, sym->cipher.data.offset,
			sym->cipher.data.length, ses->iv.length);
#endif
	/* output */
	if (sym->m_dst)
		mbuf = sym->m_dst;
	else
		mbuf = sym->m_src;

	sg = &ctx->sg[0];
	length = sym->cipher.data.length;

	sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
		+ sym->cipher.data.offset);
	sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);

	/* Successive segs */
	mbuf = mbuf->next;
	while (mbuf) {
		sg++;
		sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
		sg->len = cpu_to_caam32(mbuf->data_len);
		mbuf = mbuf->next;
	}
	/* last element*/
	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);

	SEC_JD_SET_OUT_PTR(jobdescr,
			(uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0,
			length);
	/*enabling sg bit */
	(jobdescr)->seq_out.command.word  |= 0x01000000;

	/*input */
	sg++;
	mbuf = sym->m_src;
	in_sg = sg;

	length = sym->cipher.data.length + ses->iv.length;

	/* IV */
	sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
	sg->len = cpu_to_caam32(ses->iv.length);

	/* 1st seg */
	sg++;
	sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
				+ sym->cipher.data.offset);
	sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);

	/* Successive segs */
	mbuf = mbuf->next;
	while (mbuf) {
		sg++;
		sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
		sg->len = cpu_to_caam32(mbuf->data_len);
		mbuf = mbuf->next;
	}
	/* last element*/
	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);


	SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, in_sg), 0,
				length);
	/*enabling sg bit */
	(jobdescr)->seq_in.command.word  |= 0x01000000;

	return ctx;
}

static inline struct caam_jr_op_ctx *
build_cipher_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
{
	struct rte_crypto_sym_op *sym = op->sym;
	struct caam_jr_op_ctx *ctx;
	struct sec4_sg_entry *sg;
	rte_iova_t src_start_addr, dst_start_addr;
	struct sec_cdb *cdb;
	uint64_t sdesc_offset;
	uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
			ses->iv.offset);
	struct sec_job_descriptor_t *jobdescr;

	PMD_INIT_FUNC_TRACE();
	ctx = caam_jr_alloc_ctx(ses);
	if (!ctx)
		return NULL;

	ctx->op = op;
	cdb = ses->cdb;
	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);

	src_start_addr = rte_pktmbuf_iova(sym->m_src);
	if (sym->m_dst)
		dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
	else
		dst_start_addr = src_start_addr;

	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;

	SEC_JD_INIT(jobdescr);
	SEC_JD_SET_SD(jobdescr,
		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
		cdb->sh_hdr.hi.field.idlen);

#if CAAM_JR_DBG
	CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
			sym->m_src->data_off, sym->cipher.data.offset,
			sym->cipher.data.length, ses->iv.length);
#endif
	/* output */
	SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr,
			sym->cipher.data.offset,
			sym->cipher.data.length + ses->iv.length);

	/*input */
	sg = &ctx->sg[0];
	SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
				sym->cipher.data.length + ses->iv.length);
	/*enabling sg bit */
	(jobdescr)->seq_in.command.word  |= 0x01000000;

	sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
	sg->len = cpu_to_caam32(ses->iv.length);

	sg = &ctx->sg[1];
	sg->ptr = cpu_to_caam64(src_start_addr + sym->cipher.data.offset);
	sg->len = cpu_to_caam32(sym->cipher.data.length);
	/* last element*/
	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);

	return ctx;
}

/* For decapsulation:
 *     Input:
 * +----+----------------+--------------------------------+-----+
 * | IV | Auth-only data | Authenticated & Encrypted data | ICV |
 * +----+----------------+--------------------------------+-----+
 *     Output:
 * +----+--------------------------+
 * | Decrypted & authenticated data |
 * +----+--------------------------+
 */

static inline struct caam_jr_op_ctx *
build_cipher_auth_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
{
	struct rte_crypto_sym_op *sym = op->sym;
	struct caam_jr_op_ctx *ctx;
	struct sec4_sg_entry *sg, *out_sg, *in_sg;
	struct rte_mbuf *mbuf;
	uint32_t length = 0;
	struct sec_cdb *cdb;
	uint64_t sdesc_offset;
	uint8_t req_segs;
	uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
			ses->iv.offset);
	struct sec_job_descriptor_t *jobdescr;
	uint32_t auth_only_len;

	PMD_INIT_FUNC_TRACE();
	auth_only_len = op->sym->auth.data.length -
				op->sym->cipher.data.length;

	if (sym->m_dst) {
		mbuf = sym->m_dst;
		req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
	} else {
		mbuf = sym->m_src;
		req_segs = mbuf->nb_segs * 2 + 3;
	}

	if (req_segs > MAX_SG_ENTRIES) {
		CAAM_JR_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
				MAX_SG_ENTRIES);
		return NULL;
	}

	ctx = caam_jr_alloc_ctx(ses);
	if (!ctx)
		return NULL;

	ctx->op = op;
	cdb = ses->cdb;
	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);

	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;

	SEC_JD_INIT(jobdescr);
	SEC_JD_SET_SD(jobdescr,
		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
		cdb->sh_hdr.hi.field.idlen);

	/* output */
	if (sym->m_dst)
		mbuf = sym->m_dst;
	else
		mbuf = sym->m_src;

	out_sg = &ctx->sg[0];
	if (is_encode(ses))
		length = sym->auth.data.length + ses->digest_length;
	else
		length = sym->auth.data.length;

	sg = &ctx->sg[0];

	/* 1st seg */
	sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
		+ sym->auth.data.offset);
	sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);

	/* Successive segs */
	mbuf = mbuf->next;
	while (mbuf) {
		sg++;
		sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
		sg->len = cpu_to_caam32(mbuf->data_len);
		mbuf = mbuf->next;
	}

	if (is_encode(ses)) {
		/* set auth output */
		sg++;
		sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
		sg->len = cpu_to_caam32(ses->digest_length);
	}
	/* last element*/
	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);

	SEC_JD_SET_OUT_PTR(jobdescr,
			   (uint64_t)caam_jr_dma_vtop(out_sg), 0, length);
	/* set sg bit */
	(jobdescr)->seq_out.command.word  |= 0x01000000;

	/* input */
	sg++;
	mbuf = sym->m_src;
	in_sg = sg;
	if (is_encode(ses))
		length = ses->iv.length + sym->auth.data.length;
	else
		length = ses->iv.length + sym->auth.data.length
						+ ses->digest_length;

	sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
	sg->len = cpu_to_caam32(ses->iv.length);

	sg++;
	/* 1st seg */
	sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
		+ sym->auth.data.offset);
	sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);

	/* Successive segs */
	mbuf = mbuf->next;
	while (mbuf) {
		sg++;
		sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
		sg->len = cpu_to_caam32(mbuf->data_len);
		mbuf = mbuf->next;
	}

	if (is_decode(ses)) {
		sg++;
		rte_memcpy(ctx->digest, sym->auth.digest.data,
		       ses->digest_length);
		sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
		sg->len = cpu_to_caam32(ses->digest_length);
	}
	/* last element*/
	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);

	SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(in_sg), 0,
				length);
	/* set sg bit */
	(jobdescr)->seq_in.command.word  |= 0x01000000;
	/* Auth_only_len is set as 0 in descriptor and it is
	 * overwritten here in the jd which will update
	 * the DPOVRD reg.
	 */
	if (auth_only_len)
		/* set sg bit */
		(jobdescr)->dpovrd = 0x80000000 | auth_only_len;

	return ctx;
}

static inline struct caam_jr_op_ctx *
build_cipher_auth(struct rte_crypto_op *op, struct caam_jr_session *ses)
{
	struct rte_crypto_sym_op *sym = op->sym;
	struct caam_jr_op_ctx *ctx;
	struct sec4_sg_entry *sg;
	rte_iova_t src_start_addr, dst_start_addr;
	uint32_t length = 0;
	struct sec_cdb *cdb;
	uint64_t sdesc_offset;
	uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
			ses->iv.offset);
	struct sec_job_descriptor_t *jobdescr;
	uint32_t auth_only_len;

	PMD_INIT_FUNC_TRACE();
	auth_only_len = op->sym->auth.data.length -
				op->sym->cipher.data.length;

	src_start_addr = rte_pktmbuf_iova(sym->m_src);
	if (sym->m_dst)
		dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
	else
		dst_start_addr = src_start_addr;

	ctx = caam_jr_alloc_ctx(ses);
	if (!ctx)
		return NULL;

	ctx->op = op;
	cdb = ses->cdb;
	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);

	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;

	SEC_JD_INIT(jobdescr);
	SEC_JD_SET_SD(jobdescr,
		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
		cdb->sh_hdr.hi.field.idlen);

	/* input */
	sg = &ctx->sg[0];
	if (is_encode(ses)) {
		sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
		sg->len = cpu_to_caam32(ses->iv.length);
		length += ses->iv.length;

		sg++;
		sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
		sg->len = cpu_to_caam32(sym->auth.data.length);
		length += sym->auth.data.length;
		/* last element*/
		sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
	} else {
		sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
		sg->len = cpu_to_caam32(ses->iv.length);
		length += ses->iv.length;

		sg++;
		sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
		sg->len = cpu_to_caam32(sym->auth.data.length);
		length += sym->auth.data.length;

		rte_memcpy(ctx->digest, sym->auth.digest.data,
		       ses->digest_length);
		sg++;
		sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
		sg->len = cpu_to_caam32(ses->digest_length);
		length += ses->digest_length;
		/* last element*/
		sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
	}

	SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(&ctx->sg[0]), 0,
				length);
	/* set sg bit */
	(jobdescr)->seq_in.command.word  |= 0x01000000;

	/* output */
	sg = &ctx->sg[6];

	sg->ptr = cpu_to_caam64(dst_start_addr + sym->cipher.data.offset);
	sg->len = cpu_to_caam32(sym->cipher.data.length);
	length = sym->cipher.data.length;

	if (is_encode(ses)) {
		/* set auth output */
		sg++;
		sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
		sg->len = cpu_to_caam32(ses->digest_length);
		length += ses->digest_length;
	}
	/* last element*/
	sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);

	SEC_JD_SET_OUT_PTR(jobdescr,
			   (uint64_t)caam_jr_dma_vtop(&ctx->sg[6]), 0, length);
	/* set sg bit */
	(jobdescr)->seq_out.command.word  |= 0x01000000;

	/* Auth_only_len is set as 0 in descriptor and it is
	 * overwritten here in the jd which will update
	 * the DPOVRD reg.
	 */
	if (auth_only_len)
		/* set sg bit */
		(jobdescr)->dpovrd = 0x80000000 | auth_only_len;

	return ctx;
}

static inline struct caam_jr_op_ctx *
build_proto(struct rte_crypto_op *op, struct caam_jr_session *ses)
{
	struct rte_crypto_sym_op *sym = op->sym;
	struct caam_jr_op_ctx *ctx = NULL;
	phys_addr_t src_start_addr, dst_start_addr;
	struct sec_cdb *cdb;
	uint64_t sdesc_offset;
	struct sec_job_descriptor_t *jobdescr;

	PMD_INIT_FUNC_TRACE();
	ctx = caam_jr_alloc_ctx(ses);
	if (!ctx)
		return NULL;
	ctx->op = op;

	src_start_addr = rte_pktmbuf_iova(sym->m_src);
	if (sym->m_dst)
		dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
	else
		dst_start_addr = src_start_addr;

	cdb = ses->cdb;
	sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);

	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;

	SEC_JD_INIT(jobdescr);
	SEC_JD_SET_SD(jobdescr,
		(phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
			cdb->sh_hdr.hi.field.idlen);

	/* output */
	SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr, 0,
			sym->m_src->buf_len - sym->m_src->data_off);
	/* input */
	SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)src_start_addr, 0,
			sym->m_src->pkt_len);
	sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;

	return ctx;
}

static int
caam_jr_enqueue_op(struct rte_crypto_op *op, struct caam_jr_qp *qp)
{
	struct sec_job_ring_t *ring = qp->ring;
	struct caam_jr_session *ses;
	struct caam_jr_op_ctx *ctx = NULL;
	struct sec_job_descriptor_t *jobdescr __rte_unused;

	PMD_INIT_FUNC_TRACE();
	switch (op->sess_type) {
	case RTE_CRYPTO_OP_WITH_SESSION:
		ses = (struct caam_jr_session *)
		get_sym_session_private_data(op->sym->session,
					cryptodev_driver_id);
		break;
	case RTE_CRYPTO_OP_SECURITY_SESSION:
		ses = (struct caam_jr_session *)
			get_sec_session_private_data(
					op->sym->sec_session);
		break;
	default:
		CAAM_JR_DP_ERR("sessionless crypto op not supported");
		qp->tx_errs++;
		return -1;
	}

	if (unlikely(!ses->qp || ses->qp != qp)) {
		CAAM_JR_DP_DEBUG("Old:sess->qp=%p New qp = %p\n", ses->qp, qp);
		ses->qp = qp;
		caam_jr_prep_cdb(ses);
	}

	if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
		if (is_auth_cipher(ses))
			ctx = build_cipher_auth(op, ses);
		else if (is_aead(ses))
			goto err1;
		else if (is_auth_only(ses))
			ctx = build_auth_only(op, ses);
		else if (is_cipher_only(ses))
			ctx = build_cipher_only(op, ses);
		else if (is_proto_ipsec(ses))
			ctx = build_proto(op, ses);
	} else {
		if (is_auth_cipher(ses))
			ctx = build_cipher_auth_sg(op, ses);
		else if (is_aead(ses))
			goto err1;
		else if (is_auth_only(ses))
			ctx = build_auth_only_sg(op, ses);
		else if (is_cipher_only(ses))
			ctx = build_cipher_only_sg(op, ses);
	}
err1:
	if (unlikely(!ctx)) {
		qp->tx_errs++;
		CAAM_JR_ERR("not supported sec op");
		return -1;
	}
#if CAAM_JR_DBG
	if (is_decode(ses))
		rte_hexdump(stdout, "DECODE",
			rte_pktmbuf_mtod(op->sym->m_src, void *),
			rte_pktmbuf_data_len(op->sym->m_src));
	else
		rte_hexdump(stdout, "ENCODE",
			rte_pktmbuf_mtod(op->sym->m_src, void *),
			rte_pktmbuf_data_len(op->sym->m_src));

	printf("\n JD before conversion\n");
	for (int i = 0; i < 12; i++)
		printf("\n 0x%08x", ctx->jobdes.desc[i]);
#endif

	CAAM_JR_DP_DEBUG("Jr[%p] pi[%d] ci[%d].Before sending desc",
		      ring, ring->pidx, ring->cidx);

	/* todo - do we want to retry */
	if (SEC_JOB_RING_IS_FULL(ring->pidx, ring->cidx,
			 SEC_JOB_RING_SIZE, SEC_JOB_RING_SIZE)) {
		CAAM_JR_DP_DEBUG("Ring FULL Jr[%p] pi[%d] ci[%d].Size = %d",
			      ring, ring->pidx, ring->cidx, SEC_JOB_RING_SIZE);
		caam_jr_op_ending(ctx);
		qp->tx_ring_full++;
		return -EBUSY;
	}

#if CORE_BYTE_ORDER != CAAM_BYTE_ORDER
	jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;

	jobdescr->deschdr.command.word =
		cpu_to_caam32(jobdescr->deschdr.command.word);
	jobdescr->sd_ptr = cpu_to_caam64(jobdescr->sd_ptr);
	jobdescr->seq_out.command.word =
		cpu_to_caam32(jobdescr->seq_out.command.word);
	jobdescr->seq_out_ptr = cpu_to_caam64(jobdescr->seq_out_ptr);
	jobdescr->out_ext_length = cpu_to_caam32(jobdescr->out_ext_length);
	jobdescr->seq_in.command.word =
		cpu_to_caam32(jobdescr->seq_in.command.word);
	jobdescr->seq_in_ptr = cpu_to_caam64(jobdescr->seq_in_ptr);
	jobdescr->in_ext_length = cpu_to_caam32(jobdescr->in_ext_length);
	jobdescr->load_dpovrd.command.word =
		cpu_to_caam32(jobdescr->load_dpovrd.command.word);
	jobdescr->dpovrd = cpu_to_caam32(jobdescr->dpovrd);
#endif

	/* Set ptr in input ring to current descriptor	*/
	sec_write_addr(&ring->input_ring[ring->pidx],
			(phys_addr_t)caam_jr_vtop_ctx(ctx, ctx->jobdes.desc));
	rte_smp_wmb();

	/* Notify HW that a new job is enqueued */
	hw_enqueue_desc_on_job_ring(ring);

	/* increment the producer index for the current job ring */
	ring->pidx = SEC_CIRCULAR_COUNTER(ring->pidx, SEC_JOB_RING_SIZE);

	return 0;
}

static uint16_t
caam_jr_enqueue_burst(void *qp, struct rte_crypto_op **ops,
		       uint16_t nb_ops)
{
	/* Function to transmit the frames to given device and queuepair */
	uint32_t loop;
	int32_t ret;
	struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
	uint16_t num_tx = 0;

	PMD_INIT_FUNC_TRACE();
	/*Prepare each packet which is to be sent*/
	for (loop = 0; loop < nb_ops; loop++) {
		ret = caam_jr_enqueue_op(ops[loop], jr_qp);
		if (!ret)
			num_tx++;
	}

	jr_qp->tx_pkts += num_tx;

	return num_tx;
}

/* Release queue pair */
static int
caam_jr_queue_pair_release(struct rte_cryptodev *dev,
			   uint16_t qp_id)
{
	struct sec_job_ring_t *internals;
	struct caam_jr_qp *qp = NULL;

	PMD_INIT_FUNC_TRACE();
	CAAM_JR_DEBUG("dev =%p, queue =%d", dev, qp_id);

	internals = dev->data->dev_private;
	if (qp_id >= internals->max_nb_queue_pairs) {
		CAAM_JR_ERR("Max supported qpid %d",
			     internals->max_nb_queue_pairs);
		return -EINVAL;
	}

	qp = &internals->qps[qp_id];
	qp->ring = NULL;
	dev->data->queue_pairs[qp_id] = NULL;

	return 0;
}

/* Setup a queue pair */
static int
caam_jr_queue_pair_setup(
		struct rte_cryptodev *dev, uint16_t qp_id,
		__rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
		__rte_unused int socket_id,
		__rte_unused struct rte_mempool *session_pool)
{
	struct sec_job_ring_t *internals;
	struct caam_jr_qp *qp = NULL;

	PMD_INIT_FUNC_TRACE();
	CAAM_JR_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);

	internals = dev->data->dev_private;
	if (qp_id >= internals->max_nb_queue_pairs) {
		CAAM_JR_ERR("Max supported qpid %d",
			     internals->max_nb_queue_pairs);
		return -EINVAL;
	}

	qp = &internals->qps[qp_id];
	qp->ring = internals;
	dev->data->queue_pairs[qp_id] = qp;

	return 0;
}

/* Return the number of allocated queue pairs */
static uint32_t
caam_jr_queue_pair_count(struct rte_cryptodev *dev)
{
	PMD_INIT_FUNC_TRACE();

	return dev->data->nb_queue_pairs;
}

/* Returns the size of the aesni gcm session structure */
static unsigned int
caam_jr_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
{
	PMD_INIT_FUNC_TRACE();

	return sizeof(struct caam_jr_session);
}

static int
caam_jr_cipher_init(struct rte_cryptodev *dev __rte_unused,
		    struct rte_crypto_sym_xform *xform,
		    struct caam_jr_session *session)
{
	PMD_INIT_FUNC_TRACE();
	session->cipher_alg = xform->cipher.algo;
	session->iv.length = xform->cipher.iv.length;
	session->iv.offset = xform->cipher.iv.offset;
	session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
					       RTE_CACHE_LINE_SIZE);
	if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
		CAAM_JR_ERR("No Memory for cipher key\n");
		return -ENOMEM;
	}
	session->cipher_key.length = xform->cipher.key.length;

	memcpy(session->cipher_key.data, xform->cipher.key.data,
	       xform->cipher.key.length);
	session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
			DIR_ENC : DIR_DEC;

	return 0;
}

static int
caam_jr_auth_init(struct rte_cryptodev *dev __rte_unused,
		  struct rte_crypto_sym_xform *xform,
		  struct caam_jr_session *session)
{
	PMD_INIT_FUNC_TRACE();
	session->auth_alg = xform->auth.algo;
	session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
					     RTE_CACHE_LINE_SIZE);
	if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
		CAAM_JR_ERR("No Memory for auth key\n");
		return -ENOMEM;
	}
	session->auth_key.length = xform->auth.key.length;
	session->digest_length = xform->auth.digest_length;

	memcpy(session->auth_key.data, xform->auth.key.data,
	       xform->auth.key.length);
	session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
			DIR_ENC : DIR_DEC;

	return 0;
}

static int
caam_jr_aead_init(struct rte_cryptodev *dev __rte_unused,
		  struct rte_crypto_sym_xform *xform,
		  struct caam_jr_session *session)
{
	PMD_INIT_FUNC_TRACE();
	session->aead_alg = xform->aead.algo;
	session->iv.length = xform->aead.iv.length;
	session->iv.offset = xform->aead.iv.offset;
	session->auth_only_len = xform->aead.aad_length;
	session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
					     RTE_CACHE_LINE_SIZE);
	if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
		CAAM_JR_ERR("No Memory for aead key\n");
		return -ENOMEM;
	}
	session->aead_key.length = xform->aead.key.length;
	session->digest_length = xform->aead.digest_length;

	memcpy(session->aead_key.data, xform->aead.key.data,
	       xform->aead.key.length);
	session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
			DIR_ENC : DIR_DEC;

	return 0;
}

static int
caam_jr_set_session_parameters(struct rte_cryptodev *dev,
			       struct rte_crypto_sym_xform *xform, void *sess)
{
	struct sec_job_ring_t *internals = dev->data->dev_private;
	struct caam_jr_session *session = sess;

	PMD_INIT_FUNC_TRACE();

	if (unlikely(sess == NULL)) {
		CAAM_JR_ERR("invalid session struct");
		return -EINVAL;
	}

	/* Default IV length = 0 */
	session->iv.length = 0;

	/* Cipher Only */
	if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
		session->auth_alg = RTE_CRYPTO_AUTH_NULL;
		caam_jr_cipher_init(dev, xform, session);

	/* Authentication Only */
	} else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
		   xform->next == NULL) {
		session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
		caam_jr_auth_init(dev, xform, session);

	/* Cipher then Authenticate */
	} else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
		   xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
		if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
			caam_jr_cipher_init(dev, xform, session);
			caam_jr_auth_init(dev, xform->next, session);
		} else {
			CAAM_JR_ERR("Not supported: Auth then Cipher");
			goto err1;
		}

	/* Authenticate then Cipher */
	} else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
		   xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
		if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
			caam_jr_auth_init(dev, xform, session);
			caam_jr_cipher_init(dev, xform->next, session);
		} else {
			CAAM_JR_ERR("Not supported: Auth then Cipher");
			goto err1;
		}

	/* AEAD operation for AES-GCM kind of Algorithms */
	} else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
		   xform->next == NULL) {
		caam_jr_aead_init(dev, xform, session);

	} else {
		CAAM_JR_ERR("Invalid crypto type");
		return -EINVAL;
	}
	session->ctx_pool = internals->ctx_pool;

	return 0;

err1:
	rte_free(session->cipher_key.data);
	rte_free(session->auth_key.data);
	memset(session, 0, sizeof(struct caam_jr_session));

	return -EINVAL;
}

static int
caam_jr_sym_session_configure(struct rte_cryptodev *dev,
			      struct rte_crypto_sym_xform *xform,
			      struct rte_cryptodev_sym_session *sess,
			      struct rte_mempool *mempool)
{
	void *sess_private_data;
	int ret;

	PMD_INIT_FUNC_TRACE();

	if (rte_mempool_get(mempool, &sess_private_data)) {
		CAAM_JR_ERR("Couldn't get object from session mempool");
		return -ENOMEM;
	}

	memset(sess_private_data, 0, sizeof(struct caam_jr_session));
	ret = caam_jr_set_session_parameters(dev, xform, sess_private_data);
	if (ret != 0) {
		CAAM_JR_ERR("failed to configure session parameters");
		/* Return session to mempool */
		rte_mempool_put(mempool, sess_private_data);
		return ret;
	}

	set_sym_session_private_data(sess, dev->driver_id, sess_private_data);

	return 0;
}

/* Clear the memory of session so it doesn't leave key material behind */
static void
caam_jr_sym_session_clear(struct rte_cryptodev *dev,
		struct rte_cryptodev_sym_session *sess)
{
	uint8_t index = dev->driver_id;
	void *sess_priv = get_sym_session_private_data(sess, index);
	struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;

	PMD_INIT_FUNC_TRACE();

	if (sess_priv) {
		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);

		rte_free(s->cipher_key.data);
		rte_free(s->auth_key.data);
		memset(s, 0, sizeof(struct caam_jr_session));
		set_sym_session_private_data(sess, index, NULL);
		rte_mempool_put(sess_mp, sess_priv);
	}
}

static int
caam_jr_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
			  struct rte_security_session_conf *conf,
			  void *sess)
{
	struct sec_job_ring_t *internals = dev->data->dev_private;
	struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
	struct rte_crypto_auth_xform *auth_xform;
	struct rte_crypto_cipher_xform *cipher_xform;
	struct caam_jr_session *session = (struct caam_jr_session *)sess;

	PMD_INIT_FUNC_TRACE();

	if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
		cipher_xform = &conf->crypto_xform->cipher;
		auth_xform = &conf->crypto_xform->next->auth;
	} else {
		auth_xform = &conf->crypto_xform->auth;
		cipher_xform = &conf->crypto_xform->next->cipher;
	}
	session->proto_alg = conf->protocol;
	session->cipher_key.data = rte_zmalloc(NULL,
					       cipher_xform->key.length,
					       RTE_CACHE_LINE_SIZE);
	if (session->cipher_key.data == NULL &&
			cipher_xform->key.length > 0) {
		CAAM_JR_ERR("No Memory for cipher key\n");
		return -ENOMEM;
	}

	session->cipher_key.length = cipher_xform->key.length;
	session->auth_key.data = rte_zmalloc(NULL,
					auth_xform->key.length,
					RTE_CACHE_LINE_SIZE);
	if (session->auth_key.data == NULL &&
			auth_xform->key.length > 0) {
		CAAM_JR_ERR("No Memory for auth key\n");
		rte_free(session->cipher_key.data);
		return -ENOMEM;
	}
	session->auth_key.length = auth_xform->key.length;
	memcpy(session->cipher_key.data, cipher_xform->key.data,
			cipher_xform->key.length);
	memcpy(session->auth_key.data, auth_xform->key.data,
			auth_xform->key.length);

	switch (auth_xform->algo) {
	case RTE_CRYPTO_AUTH_SHA1_HMAC:
		session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
		break;
	case RTE_CRYPTO_AUTH_MD5_HMAC:
		session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
		break;
	case RTE_CRYPTO_AUTH_SHA256_HMAC:
		session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
		break;
	case RTE_CRYPTO_AUTH_SHA384_HMAC:
		session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
		break;
	case RTE_CRYPTO_AUTH_SHA512_HMAC:
		session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
		break;
	case RTE_CRYPTO_AUTH_AES_CMAC:
		session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
		break;
	case RTE_CRYPTO_AUTH_NULL:
		session->auth_alg = RTE_CRYPTO_AUTH_NULL;
		break;
	case RTE_CRYPTO_AUTH_SHA224_HMAC:
	case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
	case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
	case RTE_CRYPTO_AUTH_SHA1:
	case RTE_CRYPTO_AUTH_SHA256:
	case RTE_CRYPTO_AUTH_SHA512:
	case RTE_CRYPTO_AUTH_SHA224:
	case RTE_CRYPTO_AUTH_SHA384:
	case RTE_CRYPTO_AUTH_MD5:
	case RTE_CRYPTO_AUTH_AES_GMAC:
	case RTE_CRYPTO_AUTH_KASUMI_F9:
	case RTE_CRYPTO_AUTH_AES_CBC_MAC:
	case RTE_CRYPTO_AUTH_ZUC_EIA3:
		CAAM_JR_ERR("Crypto: Unsupported auth alg %u\n",
			auth_xform->algo);
		goto out;
	default:
		CAAM_JR_ERR("Crypto: Undefined Auth specified %u\n",
			auth_xform->algo);
		goto out;
	}

	switch (cipher_xform->algo) {
	case RTE_CRYPTO_CIPHER_AES_CBC:
		session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
		break;
	case RTE_CRYPTO_CIPHER_3DES_CBC:
		session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
		break;
	case RTE_CRYPTO_CIPHER_AES_CTR:
		session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
		break;
	case RTE_CRYPTO_CIPHER_NULL:
	case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
	case RTE_CRYPTO_CIPHER_3DES_ECB:
	case RTE_CRYPTO_CIPHER_AES_ECB:
	case RTE_CRYPTO_CIPHER_KASUMI_F8:
		CAAM_JR_ERR("Crypto: Unsupported Cipher alg %u\n",
			cipher_xform->algo);
		goto out;
	default:
		CAAM_JR_ERR("Crypto: Undefined Cipher specified %u\n",
			cipher_xform->algo);
		goto out;
	}

	if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
		memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
				sizeof(session->ip4_hdr));
		session->ip4_hdr.ip_v = IPVERSION;
		session->ip4_hdr.ip_hl = 5;
		session->ip4_hdr.ip_len = rte_cpu_to_be_16(
						sizeof(session->ip4_hdr));
		session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
		session->ip4_hdr.ip_id = 0;
		session->ip4_hdr.ip_off = 0;
		session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
		session->ip4_hdr.ip_p = (ipsec_xform->proto ==
				RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
				: IPPROTO_AH;
		session->ip4_hdr.ip_sum = 0;
		session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
		session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
		session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
						(void *)&session->ip4_hdr,
						sizeof(struct ip));

		session->encap_pdb.options =
			(IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
			PDBOPTS_ESP_OIHI_PDB_INL |
			PDBOPTS_ESP_IVSRC |
			PDBHMO_ESP_ENCAP_DTTL;
		session->encap_pdb.spi = ipsec_xform->spi;
		session->encap_pdb.ip_hdr_len = sizeof(struct ip);

		session->dir = DIR_ENC;
	} else if (ipsec_xform->direction ==
			RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
		memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
		session->decap_pdb.options = sizeof(struct ip) << 16;
		session->dir = DIR_DEC;
	} else
		goto out;
	session->ctx_pool = internals->ctx_pool;

	return 0;
out:
	rte_free(session->auth_key.data);
	rte_free(session->cipher_key.data);
	memset(session, 0, sizeof(struct caam_jr_session));
	return -1;
}

static int
caam_jr_security_session_create(void *dev,
				struct rte_security_session_conf *conf,
				struct rte_security_session *sess,
				struct rte_mempool *mempool)
{
	void *sess_private_data;
	struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
	int ret;

	PMD_INIT_FUNC_TRACE();
	if (rte_mempool_get(mempool, &sess_private_data)) {
		CAAM_JR_ERR("Couldn't get object from session mempool");
		return -ENOMEM;
	}

	switch (conf->protocol) {
	case RTE_SECURITY_PROTOCOL_IPSEC:
		ret = caam_jr_set_ipsec_session(cdev, conf,
				sess_private_data);
		break;
	case RTE_SECURITY_PROTOCOL_MACSEC:
		return -ENOTSUP;
	default:
		return -EINVAL;
	}
	if (ret != 0) {
		CAAM_JR_ERR("failed to configure session parameters");
		/* Return session to mempool */
		rte_mempool_put(mempool, sess_private_data);
		return ret;
	}

	set_sec_session_private_data(sess, sess_private_data);

	return ret;
}

/* Clear the memory of session so it doesn't leave key material behind */
static int
caam_jr_security_session_destroy(void *dev __rte_unused,
				 struct rte_security_session *sess)
{
	PMD_INIT_FUNC_TRACE();
	void *sess_priv = get_sec_session_private_data(sess);

	struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;

	if (sess_priv) {
		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);

		rte_free(s->cipher_key.data);
		rte_free(s->auth_key.data);
		memset(sess, 0, sizeof(struct caam_jr_session));
		set_sec_session_private_data(sess, NULL);
		rte_mempool_put(sess_mp, sess_priv);
	}
	return 0;
}


static int
caam_jr_dev_configure(struct rte_cryptodev *dev,
		       struct rte_cryptodev_config *config __rte_unused)
{
	char str[20];
	struct sec_job_ring_t *internals;

	PMD_INIT_FUNC_TRACE();

	internals = dev->data->dev_private;
	snprintf(str, sizeof(str), "ctx_pool_%d", dev->data->dev_id);
	if (!internals->ctx_pool) {
		internals->ctx_pool = rte_mempool_create((const char *)str,
						CTX_POOL_NUM_BUFS,
						sizeof(struct caam_jr_op_ctx),
						CTX_POOL_CACHE_SIZE, 0,
						NULL, NULL, NULL, NULL,
						SOCKET_ID_ANY, 0);
		if (!internals->ctx_pool) {
			CAAM_JR_ERR("%s create failed\n", str);
			return -ENOMEM;
		}
	} else
		CAAM_JR_INFO("mempool already created for dev_id : %d",
				dev->data->dev_id);

	return 0;
}

static int
caam_jr_dev_start(struct rte_cryptodev *dev __rte_unused)
{
	PMD_INIT_FUNC_TRACE();
	return 0;
}

static void
caam_jr_dev_stop(struct rte_cryptodev *dev __rte_unused)
{
	PMD_INIT_FUNC_TRACE();
}

static int
caam_jr_dev_close(struct rte_cryptodev *dev)
{
	struct sec_job_ring_t *internals;

	PMD_INIT_FUNC_TRACE();

	if (dev == NULL)
		return -ENOMEM;

	internals = dev->data->dev_private;
	rte_mempool_free(internals->ctx_pool);
	internals->ctx_pool = NULL;

	return 0;
}

static void
caam_jr_dev_infos_get(struct rte_cryptodev *dev,
		       struct rte_cryptodev_info *info)
{
	struct sec_job_ring_t *internals = dev->data->dev_private;

	PMD_INIT_FUNC_TRACE();
	if (info != NULL) {
		info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
		info->feature_flags = dev->feature_flags;
		info->capabilities = caam_jr_get_cryptodev_capabilities();
		info->sym.max_nb_sessions = internals->max_nb_sessions;
		info->driver_id = cryptodev_driver_id;
	}
}

static struct rte_cryptodev_ops caam_jr_ops = {
	.dev_configure	      = caam_jr_dev_configure,
	.dev_start	      = caam_jr_dev_start,
	.dev_stop	      = caam_jr_dev_stop,
	.dev_close	      = caam_jr_dev_close,
	.dev_infos_get        = caam_jr_dev_infos_get,
	.stats_get	      = caam_jr_stats_get,
	.stats_reset	      = caam_jr_stats_reset,
	.queue_pair_setup     = caam_jr_queue_pair_setup,
	.queue_pair_release   = caam_jr_queue_pair_release,
	.queue_pair_count     = caam_jr_queue_pair_count,
	.sym_session_get_size = caam_jr_sym_session_get_size,
	.sym_session_configure = caam_jr_sym_session_configure,
	.sym_session_clear    = caam_jr_sym_session_clear
};

static struct rte_security_ops caam_jr_security_ops = {
	.session_create = caam_jr_security_session_create,
	.session_update = NULL,
	.session_stats_get = NULL,
	.session_destroy = caam_jr_security_session_destroy,
	.set_pkt_metadata = NULL,
	.capabilities_get = caam_jr_get_security_capabilities
};

/* @brief Flush job rings of any processed descs.
 * The processed descs are silently dropped,
 * WITHOUT being notified to UA.
 */
static void
close_job_ring(struct sec_job_ring_t *job_ring)
{
	PMD_INIT_FUNC_TRACE();
	if (job_ring->irq_fd) {
		/* Producer index is frozen. If consumer index is not equal
		 * with producer index, then we have descs to flush.
		 */
		while (job_ring->pidx != job_ring->cidx)
			hw_flush_job_ring(job_ring, false, NULL);

		/* free the uio job ring */
		free_job_ring(job_ring->irq_fd);
		job_ring->irq_fd = 0;
		caam_jr_dma_free(job_ring->input_ring);
		caam_jr_dma_free(job_ring->output_ring);
		g_job_rings_no--;
	}
}

/** @brief Release the software and hardware resources tied to a job ring.
 * @param [in] job_ring The job ring
 *
 * @retval  0 for success
 * @retval  -1 for error
 */
static int
shutdown_job_ring(struct sec_job_ring_t *job_ring)
{
	int ret = 0;

	PMD_INIT_FUNC_TRACE();
	ASSERT(job_ring != NULL);
	ret = hw_shutdown_job_ring(job_ring);
	SEC_ASSERT(ret == 0, ret,
		"Failed to shutdown hardware job ring %p",
		job_ring);

	if (job_ring->coalescing_en)
		hw_job_ring_disable_coalescing(job_ring);

	if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) {
		ret = caam_jr_disable_irqs(job_ring->irq_fd);
		SEC_ASSERT(ret == 0, ret,
		"Failed to disable irqs for job ring %p",
		job_ring);
	}

	return ret;
}

/*
 * @brief Release the resources used by the SEC user space driver.
 *
 * Reset and release SEC's job rings indicated by the User Application at
 * init_job_ring() and free any memory allocated internally.
 * Call once during application tear down.
 *
 * @note In case there are any descriptors in-flight (descriptors received by
 * SEC driver for processing and for which no response was yet provided to UA),
 * the descriptors are discarded without any notifications to User Application.
 *
 * @retval ::0			is returned for a successful execution
 * @retval ::-1		is returned if SEC driver release is in progress
 */
static int
caam_jr_dev_uninit(struct rte_cryptodev *dev)
{
	struct sec_job_ring_t *internals;

	PMD_INIT_FUNC_TRACE();
	if (dev == NULL)
		return -ENODEV;

	internals = dev->data->dev_private;
	rte_free(dev->security_ctx);

	/* If any descriptors in flight , poll and wait
	 * until all descriptors are received and silently discarded.
	 */
	if (internals) {
		shutdown_job_ring(internals);
		close_job_ring(internals);
		rte_mempool_free(internals->ctx_pool);
	}

	CAAM_JR_INFO("Closing crypto device %s", dev->data->name);

	/* last caam jr instance) */
	if (g_job_rings_no == 0)
		g_driver_state = SEC_DRIVER_STATE_IDLE;

	return SEC_SUCCESS;
}

/* @brief Initialize the software and hardware resources tied to a job ring.
 * @param [in] jr_mode;		Model to be used by SEC Driver to receive
 *				notifications from SEC.  Can be either
 *				of the three: #SEC_NOTIFICATION_TYPE_NAPI
 *				#SEC_NOTIFICATION_TYPE_IRQ or
 *				#SEC_NOTIFICATION_TYPE_POLL
 * @param [in] NAPI_mode	The NAPI work mode to configure a job ring at
 *				startup. Used only when #SEC_NOTIFICATION_TYPE
 *				is set to #SEC_NOTIFICATION_TYPE_NAPI.
 * @param [in] irq_coalescing_timer This value determines the maximum
 *					amount of time after processing a
 *					descriptor before raising an interrupt.
 * @param [in] irq_coalescing_count This value determines how many
 *					descriptors are completed before
 *					raising an interrupt.
 * @param [in] reg_base_addr,	The job ring base address register
 * @param [in] irq_id		The job ring interrupt identification number.
 * @retval  job_ring_handle for successful job ring configuration
 * @retval  NULL on error
 *
 */
static void *
init_job_ring(void *reg_base_addr, uint32_t irq_id)
{
	struct sec_job_ring_t *job_ring = NULL;
	int i, ret = 0;
	int jr_mode = SEC_NOTIFICATION_TYPE_POLL;
	int napi_mode = 0;
	int irq_coalescing_timer = 0;
	int irq_coalescing_count = 0;

	for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
		if (g_job_rings[i].irq_fd == 0) {
			job_ring = &g_job_rings[i];
			g_job_rings_no++;
			break;
		}
	}
	if (job_ring == NULL) {
		CAAM_JR_ERR("No free job ring\n");
		return NULL;
	}

	job_ring->register_base_addr = reg_base_addr;
	job_ring->jr_mode = jr_mode;
	job_ring->napi_mode = 0;
	job_ring->irq_fd = irq_id;

	/* Allocate mem for input and output ring */

	/* Allocate memory for input ring */
	job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
				SEC_DMA_MEM_INPUT_RING_SIZE);
	memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE);

	/* Allocate memory for output ring */
	job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
				SEC_DMA_MEM_OUTPUT_RING_SIZE);
	memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE);

	/* Reset job ring in SEC hw and configure job ring registers */
	ret = hw_reset_job_ring(job_ring);
	if (ret != 0) {
		CAAM_JR_ERR("Failed to reset hardware job ring");
		goto cleanup;
	}

	if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
	/* When SEC US driver works in NAPI mode, the UA can select
	 * if the driver starts with IRQs on or off.
	 */
		if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) {
			CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p",
				job_ring);
			ret = caam_jr_enable_irqs(job_ring->irq_fd);
			if (ret != 0) {
				CAAM_JR_ERR("Failed to enable irqs for job ring");
				goto cleanup;
			}
		}
	} else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
	/* When SEC US driver works in pure interrupt mode,
	 * IRQ's are always enabled.
	 */
		CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p",
			 job_ring);
		ret = caam_jr_enable_irqs(job_ring->irq_fd);
		if (ret != 0) {
			CAAM_JR_ERR("Failed to enable irqs for job ring");
			goto cleanup;
		}
	}
	if (irq_coalescing_timer || irq_coalescing_count) {
		hw_job_ring_set_coalescing_param(job_ring,
			 irq_coalescing_timer,
			 irq_coalescing_count);

		hw_job_ring_enable_coalescing(job_ring);
		job_ring->coalescing_en = 1;
	}

	job_ring->jr_state = SEC_JOB_RING_STATE_STARTED;
	job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS;
	job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS;

	return job_ring;
cleanup:
	caam_jr_dma_free(job_ring->output_ring);
	caam_jr_dma_free(job_ring->input_ring);
	return NULL;
}


static int
caam_jr_dev_init(const char *name,
		 struct rte_vdev_device *vdev,
		 struct rte_cryptodev_pmd_init_params *init_params)
{
	struct rte_cryptodev *dev;
	struct rte_security_ctx *security_instance;
	struct uio_job_ring *job_ring;
	char str[RTE_CRYPTODEV_NAME_MAX_LEN];

	PMD_INIT_FUNC_TRACE();

	/* Validate driver state */
	if (g_driver_state == SEC_DRIVER_STATE_IDLE) {
		g_job_rings_max = sec_configure();
		if (!g_job_rings_max) {
			CAAM_JR_ERR("No job ring detected on UIO !!!!");
			return -1;
		}
		/* Update driver state */
		g_driver_state = SEC_DRIVER_STATE_STARTED;
	}

	if (g_job_rings_no >= g_job_rings_max) {
		CAAM_JR_ERR("No more job rings available max=%d!!!!",
				g_job_rings_max);
		return -1;
	}

	job_ring = config_job_ring();
	if (job_ring == NULL) {
		CAAM_JR_ERR("failed to create job ring");
		goto init_error;
	}

	snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id);

	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
	if (dev == NULL) {
		CAAM_JR_ERR("failed to create cryptodev vdev");
		goto cleanup;
	}
	/*TODO free it during teardown*/
	dev->data->dev_private = init_job_ring(job_ring->register_base_addr,
						job_ring->uio_fd);

	if (!dev->data->dev_private) {
		CAAM_JR_ERR("Ring memory allocation failed\n");
		goto cleanup2;
	}

	dev->driver_id = cryptodev_driver_id;
	dev->dev_ops = &caam_jr_ops;

	/* register rx/tx burst functions for data path */
	dev->dequeue_burst = caam_jr_dequeue_burst;
	dev->enqueue_burst = caam_jr_enqueue_burst;
	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
			RTE_CRYPTODEV_FF_HW_ACCELERATED |
			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
			RTE_CRYPTODEV_FF_SECURITY |
			RTE_CRYPTODEV_FF_IN_PLACE_SGL |
			RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
			RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
			RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;

	/* For secondary processes, we don't initialise any further as primary
	 * has already done this work. Only check we don't need a different
	 * RX function
	 */
	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
		CAAM_JR_WARN("Device already init by primary process");
		return 0;
	}

	/*TODO free it during teardown*/
	security_instance = rte_malloc("caam_jr",
				sizeof(struct rte_security_ctx), 0);
	if (security_instance == NULL) {
		CAAM_JR_ERR("memory allocation failed\n");
		//todo error handling.
		goto cleanup2;
	}

	security_instance->device = (void *)dev;
	security_instance->ops = &caam_jr_security_ops;
	security_instance->sess_cnt = 0;
	dev->security_ctx = security_instance;

	RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name);

	return 0;

cleanup2:
	caam_jr_dev_uninit(dev);
	rte_cryptodev_pmd_release_device(dev);
cleanup:
	free_job_ring(job_ring->uio_fd);
init_error:
	CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed",
			init_params->name);

	return -ENXIO;
}

/** Initialise CAAM JR crypto device */
static int
cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
{
	struct rte_cryptodev_pmd_init_params init_params = {
		"",
		sizeof(struct sec_job_ring_t),
		rte_socket_id(),
		RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
	};
	const char *name;
	const char *input_args;

	name = rte_vdev_device_name(vdev);
	if (name == NULL)
		return -EINVAL;

	input_args = rte_vdev_device_args(vdev);
	rte_cryptodev_pmd_parse_input_args(&init_params, input_args);

	/* if sec device version is not configured */
	if (!rta_get_sec_era()) {
		const struct device_node *caam_node;

		for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
			const uint32_t *prop = of_get_property(caam_node,
					"fsl,sec-era",
					NULL);
			if (prop) {
				rta_set_sec_era(
					INTL_SEC_ERA(cpu_to_caam32(*prop)));
				break;
			}
		}
	}
#ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
	if (rta_get_sec_era() > RTA_SEC_ERA_8) {
		RTE_LOG(ERR, PMD,
		"CAAM is compiled in BE mode for device with sec era > 8???\n");
		return -EINVAL;
	}
#endif

	return caam_jr_dev_init(name, vdev, &init_params);
}

/** Uninitialise CAAM JR crypto device */
static int
cryptodev_caam_jr_remove(struct rte_vdev_device *vdev)
{
	struct rte_cryptodev *cryptodev;
	const char *name;

	name = rte_vdev_device_name(vdev);
	if (name == NULL)
		return -EINVAL;

	cryptodev = rte_cryptodev_pmd_get_named_dev(name);
	if (cryptodev == NULL)
		return -ENODEV;

	caam_jr_dev_uninit(cryptodev);

	return rte_cryptodev_pmd_destroy(cryptodev);
}

static struct rte_vdev_driver cryptodev_caam_jr_drv = {
	.probe = cryptodev_caam_jr_probe,
	.remove = cryptodev_caam_jr_remove
};

static struct cryptodev_driver caam_jr_crypto_drv;

RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv);
RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD,
	"max_nb_queue_pairs=<int>"
	"socket_id=<int>");
RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver,
		cryptodev_driver_id);

RTE_INIT(caam_jr_init_log)
{
	caam_jr_logtype = rte_log_register("pmd.crypto.caam");
	if (caam_jr_logtype >= 0)
		rte_log_set_level(caam_jr_logtype, RTE_LOG_NOTICE);
}