path: root/src/plugins/dev_octeon/crypto.c

/*
 * Copyright (c) 2024 Marvell.
 * SPDX-License-Identifier: Apache-2.0
 * https://spdx.org/licenses/Apache-2.0.html
 */

#include <vnet/dev/dev.h>
#include <vnet/devices/devices.h>
#include <dev_octeon/octeon.h>
#include <dev_octeon/crypto.h>
#include <base/roc_api.h>
#include <common.h>

oct_crypto_main_t oct_crypto_main;
oct_crypto_dev_t oct_crypto_dev;

VLIB_REGISTER_LOG_CLASS (oct_log, static) = {
  .class_name = "octeon",
  .subclass_name = "crypto",
};

static_always_inline void
oct_map_keyindex_to_session (oct_crypto_sess_t *sess, u32 key_index, u8 type)
{
  oct_crypto_main_t *ocm = &oct_crypto_main;
  oct_crypto_key_t *ckey;

  ckey = vec_elt_at_index (ocm->keys[type], key_index);

  ckey->sess = sess;
  sess->key_index = key_index;
}

static_always_inline oct_crypto_sess_t *
oct_crypto_session_alloc (vlib_main_t *vm, u8 type)
{
  extern oct_plt_init_param_t oct_plt_init_param;
  oct_crypto_sess_t *addr = NULL;
  oct_crypto_main_t *ocm;
  oct_crypto_dev_t *ocd;
  u32 size;

  ocm = &oct_crypto_main;
  ocd = ocm->crypto_dev[type];

  size = sizeof (oct_crypto_sess_t);

  addr = oct_plt_init_param.oct_plt_zmalloc (size, CLIB_CACHE_LINE_BYTES);
  if (addr == NULL)
    {
      log_err (ocd->dev, "Failed to allocate crypto session memory");
      return NULL;
    }

  return addr;
}

static_always_inline i32
oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index,
			   int op_type)
{
  oct_crypto_main_t *ocm = &oct_crypto_main;
  oct_crypto_sess_t *session;
  vnet_crypto_key_t *key;
  oct_crypto_key_t *ckey;

  key = vnet_crypto_get_key (key_index);

  if (key->type == VNET_CRYPTO_KEY_TYPE_LINK)
    {
      /*
       * Read crypto or integ key session. And map link key index to same.
       */
      if (key->index_crypto != UINT32_MAX)
	{
	  ckey = vec_elt_at_index (ocm->keys[op_type], key->index_crypto);
	  session = ckey->sess;
	}
      else if (key->index_integ != UINT32_MAX)
	{
	  ckey = vec_elt_at_index (ocm->keys[op_type], key->index_integ);
	  session = ckey->sess;
	}
      else
	return -1;
    }
  else
    {
      session = oct_crypto_session_alloc (vm, op_type);
      if (session == NULL)
	return -1;
    }

  oct_map_keyindex_to_session (session, key_index, op_type);
  return 0;
}

void
oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index)
{
  extern oct_plt_init_param_t oct_plt_init_param;
  oct_crypto_main_t *ocm = &oct_crypto_main;
  oct_crypto_key_t *ckey_linked;
  oct_crypto_key_t *ckey;

  vec_validate (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index);

  ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index);
  if (ckey->sess)
    {
      /*
       * If in case link algo is pointing to same sesison, reset the pointer.
       */
      if (ckey->sess->key_index != key_index)
	{
	  ckey_linked = vec_elt_at_index (
	    ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], ckey->sess->key_index);
	  ckey_linked->sess = NULL;
	}
      oct_plt_init_param.oct_plt_free (ckey->sess);
      ckey->sess = NULL;
    }

  ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index);
  if (ckey->sess)
    {
      /*
       * If in case link algo is pointing to same sesison, reset the pointer.
       */
      if (ckey->sess->key_index != key_index)
	{
	  ckey_linked = vec_elt_at_index (
	    ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], ckey->sess->key_index);
	  ckey_linked->sess = NULL;
	}

      oct_plt_init_param.oct_plt_free (ckey->sess);
      ckey->sess = NULL;
    }
}

void
oct_crypto_key_add_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index)
{
  oct_crypto_main_t *ocm = &oct_crypto_main;
  oct_crypto_key_t *ckey;
  oct_crypto_dev_t *ocd = &oct_crypto_dev;

  vec_validate (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index);
  ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], key_index);
  if (ckey->sess == NULL)
    {
      if (oct_crypto_session_create (vm, key_index,
				     VNET_CRYPTO_OP_TYPE_ENCRYPT))
	{
	  log_err (ocd->dev, "Unable to create crypto session");
	  return;
	}
    }

  vec_validate (ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index);
  ckey = vec_elt_at_index (ocm->keys[VNET_CRYPTO_OP_TYPE_DECRYPT], key_index);
  if (ckey->sess == NULL)
    {
      if (oct_crypto_session_create (vm, key_index,
				     VNET_CRYPTO_OP_TYPE_DECRYPT))
	{
	  log_err (ocd->dev, "Unable to create crypto session");
	  return;
	}
    }
}

void
oct_crypto_key_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop,
			vnet_crypto_key_index_t idx)
{
  oct_crypto_main_t *ocm = &oct_crypto_main;

  if (kop == VNET_CRYPTO_KEY_OP_DEL)
    {
      oct_crypto_key_del_handler (vm, idx);
      return;
    }
  oct_crypto_key_add_handler (vm, idx);

  ocm->started = 1;
}

static_always_inline void
oct_crypto_session_free (vlib_main_t *vm, oct_crypto_sess_t *sess)
{
  extern oct_plt_init_param_t oct_plt_init_param;

  oct_plt_init_param.oct_plt_free (sess);
  return;
}

#ifdef PLATFORM_OCTEON9
static inline void
oct_cpt_inst_submit (struct cpt_inst_s *inst, uint64_t lmtline,
		     uint64_t io_addr)
{
  uint64_t lmt_status;

  do
    {
      /* Copy CPT command to LMTLINE */
      roc_lmt_mov64 ((void *) lmtline, inst);

      /*
       * Make sure compiler does not reorder memcpy and ldeor.
       * LMTST transactions are always flushed from the write
       * buffer immediately, a DMB is not required to push out
       * LMTSTs.
       */
      asm volatile ("dmb oshst" : : : "memory");
      lmt_status = roc_lmt_submit_ldeor (io_addr);
    }
  while (lmt_status == 0);
}
#endif

static_always_inline void
oct_crypto_burst_submit (oct_crypto_dev_t *crypto_dev, struct cpt_inst_s *inst,
			 u32 n_left)
{
  u64 lmt_base;
  u64 io_addr;
  u32 count;

#ifdef PLATFORM_OCTEON9
  lmt_base = crypto_dev->lf.lmt_base;
  io_addr = crypto_dev->lf.io_addr;

  for (count = 0; count < n_left; count++)
    oct_cpt_inst_submit (inst + count, lmt_base, io_addr);
#else
  u64 *lmt_line[OCT_MAX_LMT_SZ];
  u64 lmt_arg, core_lmt_id;

  lmt_base = crypto_dev->lmtline.lmt_base;
  io_addr = crypto_dev->lmtline.io_addr;

  ROC_LMT_CPT_BASE_ID_GET (lmt_base, core_lmt_id);

  for (count = 0; count < 16; count++)
    {
      lmt_line[count] = OCT_CPT_LMT_GET_LINE_ADDR (lmt_base, count);
    }

  while (n_left > OCT_MAX_LMT_SZ)
    {

      /*
       * Add a memory barrier so that LMTLINEs from the previous iteration
       * can be reused for a subsequent transfer.
       */
      asm volatile ("dmb oshst" ::: "memory");

      lmt_arg = ROC_CN10K_CPT_LMT_ARG | (u64) core_lmt_id;

      for (count = 0; count < 16; count++)
	{
	  roc_lmt_mov_seg ((void *) lmt_line[count], inst + count,
			   CPT_LMT_SIZE_COPY);
	}

      /* Set number of LMTSTs, excluding the first */
      lmt_arg |= (OCT_MAX_LMT_SZ - 1) << 12;

      roc_lmt_submit_steorl (lmt_arg, io_addr);

      inst += OCT_MAX_LMT_SZ;
      n_left -= OCT_MAX_LMT_SZ;
    }

  if (n_left > 0)
    {
      /*
       * Add a memory barrier so that LMTLINEs from the previous iteration
       * can be reused for a subsequent transfer.
       */
      asm volatile ("dmb oshst" ::: "memory");

      lmt_arg = ROC_CN10K_CPT_LMT_ARG | (u64) core_lmt_id;

      for (count = 0; count < n_left; count++)
	{
	  roc_lmt_mov_seg ((void *) lmt_line[count], inst + count,
			   CPT_LMT_SIZE_COPY);
	}

      /* Set number of LMTSTs, excluding the first */
      lmt_arg |= (n_left - 1) << 12;

      roc_lmt_submit_steorl (lmt_arg, io_addr);
    }
#endif
}

static_always_inline uint32_t
oct_crypto_fill_sg_comp_from_iov (struct roc_sglist_comp *list, uint32_t i,
				  struct roc_se_iov_ptr *from,
				  uint32_t from_offset, uint32_t *psize,
				  struct roc_se_buf_ptr *extra_buf,
				  uint32_t extra_offset)
{
  uint32_t extra_len = extra_buf ? extra_buf->size : 0;
  uint32_t size = *psize;
  int32_t j;

  for (j = 0; j < from->buf_cnt; j++)
    {
      struct roc_sglist_comp *to = &list[i >> 2];
      uint32_t buf_sz = from->bufs[j].size;
      void *vaddr = from->bufs[j].vaddr;
      uint64_t e_vaddr;
      uint32_t e_len;

      if (PREDICT_FALSE (from_offset))
	{
	  if (from_offset >= buf_sz)
	    {
	      from_offset -= buf_sz;
	      continue;
	    }
	  e_vaddr = (uint64_t) vaddr + from_offset;
	  e_len = clib_min ((buf_sz - from_offset), size);
	  from_offset = 0;
	}
      else
	{
	  e_vaddr = (uint64_t) vaddr;
	  e_len = clib_min (buf_sz, size);
	}

      to->u.s.len[i % 4] = clib_host_to_net_u16 (e_len);
      to->ptr[i % 4] = clib_host_to_net_u64 (e_vaddr);

      if (extra_len && (e_len >= extra_offset))
	{
	  /* Break the data at given offset */
	  uint32_t next_len = e_len - extra_offset;
	  uint64_t next_vaddr = e_vaddr + extra_offset;

	  if (!extra_offset)
	    {
	      i--;
	    }
	  else
	    {
	      e_len = extra_offset;
	      size -= e_len;
	      to->u.s.len[i % 4] = clib_host_to_net_u16 (e_len);
	    }

	  extra_len = clib_min (extra_len, size);
	  /* Insert extra data ptr */
	  if (extra_len)
	    {
	      i++;
	      to = &list[i >> 2];
	      to->u.s.len[i % 4] = clib_host_to_net_u16 (extra_len);
	      to->ptr[i % 4] =
		clib_host_to_net_u64 ((uint64_t) extra_buf->vaddr);
	      size -= extra_len;
	    }

	  next_len = clib_min (next_len, size);
	  /* insert the rest of the data */
	  if (next_len)
	    {
	      i++;
	      to = &list[i >> 2];
	      to->u.s.len[i % 4] = clib_host_to_net_u16 (next_len);
	      to->ptr[i % 4] = clib_host_to_net_u64 (next_vaddr);
	      size -= next_len;
	    }
	  extra_len = 0;
	}
      else
	{
	  size -= e_len;
	}
      if (extra_offset)
	extra_offset -= size;
      i++;

      if (PREDICT_FALSE (!size))
	break;
    }

  *psize = size;
  return (uint32_t) i;
}

static_always_inline u32
oct_crypto_fill_sg2_comp_from_iov (struct roc_sg2list_comp *list, u32 i,
				   struct roc_se_iov_ptr *from,
				   u32 from_offset, u32 *psize,
				   struct roc_se_buf_ptr *extra_buf,
				   u32 extra_offset)
{
  u32 extra_len = extra_buf ? extra_buf->size : 0;
  u32 size = *psize, buf_sz, e_len, next_len;
  struct roc_sg2list_comp *to;
  u64 e_vaddr, next_vaddr;
  void *vaddr;
  i32 j;

  for (j = 0; j < from->buf_cnt; j++)
    {
      to = &list[i / 3];
      buf_sz = from->bufs[j].size;
      vaddr = from->bufs[j].vaddr;

      if (PREDICT_FALSE (from_offset))
	{
	  if (from_offset >= buf_sz)
	    {
	      from_offset -= buf_sz;
	      continue;
	    }
	  e_vaddr = (u64) vaddr + from_offset;
	  e_len = clib_min ((buf_sz - from_offset), size);
	  from_offset = 0;
	}
      else
	{
	  e_vaddr = (u64) vaddr;
	  e_len = clib_min (buf_sz, size);
	}

      to->u.s.len[i % 3] = (e_len);
      to->ptr[i % 3] = (e_vaddr);
      to->u.s.valid_segs = (i % 3) + 1;

      if (extra_len && (e_len >= extra_offset))
	{
	  /* Break the data at given offset */
	  next_len = e_len - extra_offset;
	  next_vaddr = e_vaddr + extra_offset;

	  if (!extra_offset)
	    i--;
	  else
	    {
	      e_len = extra_offset;
	      size -= e_len;
	      to->u.s.len[i % 3] = (e_len);
	    }

	  extra_len = clib_min (extra_len, size);
	  /* Insert extra data ptr */
	  if (extra_len)
	    {
	      i++;
	      to = &list[i / 3];
	      to->u.s.len[i % 3] = (extra_len);
	      to->ptr[i % 3] = ((u64) extra_buf->vaddr);
	      to->u.s.valid_segs = (i % 3) + 1;
	      size -= extra_len;
	    }

	  next_len = clib_min (next_len, size);
	  /* insert the rest of the data */
	  if (next_len)
	    {
	      i++;
	      to = &list[i / 3];
	      to->u.s.len[i % 3] = (next_len);
	      to->ptr[i % 3] = (next_vaddr);
	      to->u.s.valid_segs = (i % 3) + 1;
	      size -= next_len;
	    }
	  extra_len = 0;
	}
      else
	size -= e_len;

      if (extra_offset)
	extra_offset -= size;

      i++;

      if (PREDICT_FALSE (!size))
	break;
    }

  *psize = size;
  return (u32) i;
}

static_always_inline uint32_t
oct_crypto_fill_sg_comp_from_buf (struct roc_sglist_comp *list, uint32_t i,
				  struct roc_se_buf_ptr *from)
{
  struct roc_sglist_comp *to = &list[i >> 2];

  to->u.s.len[i % 4] = clib_host_to_net_u16 (from->size);
  to->ptr[i % 4] = clib_host_to_net_u64 ((uint64_t) from->vaddr);
  return ++i;
}

static_always_inline uint32_t
oct_crypto_fill_sg_comp (struct roc_sglist_comp *list, uint32_t i,
			 uint64_t dma_addr, uint32_t size)
{
  struct roc_sglist_comp *to = &list[i >> 2];

  to->u.s.len[i % 4] = clib_host_to_net_u16 (size);
  to->ptr[i % 4] = clib_host_to_net_u64 (dma_addr);
  return ++i;
}

static_always_inline u32
oct_crypto_fill_sg2_comp (struct roc_sg2list_comp *list, u32 index,
			  u64 dma_addr, u32 size)
{
  struct roc_sg2list_comp *to = &list[index / 3];

  to->u.s.len[index % 3] = (size);
  to->ptr[index % 3] = (dma_addr);
  to->u.s.valid_segs = (index % 3) + 1;
  return ++index;
}

static_always_inline u32
oct_crypto_fill_sg2_comp_from_buf (struct roc_sg2list_comp *list, u32 index,
				   struct roc_se_buf_ptr *from)
{
  struct roc_sg2list_comp *to = &list[index / 3];

  to->u.s.len[index % 3] = (from->size);
  to->ptr[index % 3] = ((u64) from->vaddr);
  to->u.s.valid_segs = (index % 3) + 1;
  return ++index;
}

static_always_inline int __attribute__ ((unused))
oct_crypto_sg_inst_prep (struct roc_se_fc_params *params,
			 struct cpt_inst_s *inst, uint64_t offset_ctrl,
			 const uint8_t *iv_s, int iv_len, uint8_t pack_iv,
			 uint8_t pdcp_alg_type, int32_t inputlen,
			 int32_t outputlen, uint32_t passthrough_len,
			 uint32_t req_flags, int pdcp_flag, int decrypt)
{
  struct roc_sglist_comp *gather_comp, *scatter_comp;
  void *m_vaddr = params->meta_buf.vaddr;
  struct roc_se_buf_ptr *aad_buf = NULL;
  uint32_t mac_len = 0, aad_len = 0;
  struct roc_se_ctx *se_ctx;
  uint32_t i, g_size_bytes;
  uint64_t *offset_vaddr;
  uint32_t s_size_bytes;
  uint8_t *in_buffer;
  uint32_t size;
  uint8_t *iv_d;
  int ret = 0;

  se_ctx = params->ctx;
  mac_len = se_ctx->mac_len;

  if (PREDICT_FALSE (req_flags & ROC_SE_VALID_AAD_BUF))
    {
      /* We don't support both AAD and auth data separately */
      aad_len = params->aad_buf.size;
      aad_buf = &params->aad_buf;
    }

  /* save space for iv */
  offset_vaddr = m_vaddr;

  m_vaddr =
    (uint8_t *) m_vaddr + ROC_SE_OFF_CTRL_LEN + PLT_ALIGN_CEIL (iv_len, 8);

  inst->w4.s.opcode_major |= (uint64_t) ROC_DMA_MODE_SG;

  /* iv offset is 0 */
  *offset_vaddr = offset_ctrl;

  iv_d = ((uint8_t *) offset_vaddr + ROC_SE_OFF_CTRL_LEN);

  if (PREDICT_TRUE (iv_len))
    memcpy (iv_d, iv_s, iv_len);

  /* DPTR has SG list */

  /* TODO Add error check if space will be sufficient */
  gather_comp = (struct roc_sglist_comp *) ((uint8_t *) m_vaddr + 8);

  /*
   * Input Gather List
   */
  i = 0;

  /* Offset control word followed by iv */

  i = oct_crypto_fill_sg_comp (gather_comp, i, (uint64_t) offset_vaddr,
			       ROC_SE_OFF_CTRL_LEN + iv_len);

  /* Add input data */
  if (decrypt && (req_flags & ROC_SE_VALID_MAC_BUF))
    {
      size = inputlen - iv_len - mac_len;

      if (PREDICT_TRUE (size))
	{
	  uint32_t aad_offset = aad_len ? passthrough_len : 0;
	  i = oct_crypto_fill_sg_comp_from_iov (
	    gather_comp, i, params->src_iov, 0, &size, aad_buf, aad_offset);
	  if (PREDICT_FALSE (size))
	    {
	      clib_warning ("Cryptodev: Insufficient buffer"
			    " space, size %d needed",
			    size);
	      return -1;
	    }
	}

      if (mac_len)
	i =
	  oct_crypto_fill_sg_comp_from_buf (gather_comp, i, &params->mac_buf);
    }
  else
    {
      /* input data */
      size = inputlen - iv_len;
      if (size)
	{
	  uint32_t aad_offset = aad_len ? passthrough_len : 0;
	  i = oct_crypto_fill_sg_comp_from_iov (
	    gather_comp, i, params->src_iov, 0, &size, aad_buf, aad_offset);
	  if (PREDICT_FALSE (size))
	    {
	      clib_warning ("Cryptodev: Insufficient buffer space,"
			    " size %d needed",
			    size);
	      return -1;
	    }
	}
    }

  in_buffer = m_vaddr;
  ((uint16_t *) in_buffer)[0] = 0;
  ((uint16_t *) in_buffer)[1] = 0;
  ((uint16_t *) in_buffer)[2] = clib_host_to_net_u16 (i);

  g_size_bytes = ((i + 3) / 4) * sizeof (struct roc_sglist_comp);
  /*
   * Output Scatter List
   */

  i = 0;
  scatter_comp =
    (struct roc_sglist_comp *) ((uint8_t *) gather_comp + g_size_bytes);

  i = oct_crypto_fill_sg_comp (
    scatter_comp, i, (uint64_t) offset_vaddr + ROC_SE_OFF_CTRL_LEN, iv_len);

  /* Add output data */
  if ((!decrypt) && (req_flags & ROC_SE_VALID_MAC_BUF))
    {
      size = outputlen - iv_len - mac_len;
      if (size)
	{

	  uint32_t aad_offset = aad_len ? passthrough_len : 0;

	  i = oct_crypto_fill_sg_comp_from_iov (
	    scatter_comp, i, params->dst_iov, 0, &size, aad_buf, aad_offset);
	  if (PREDICT_FALSE (size))
	    {
	      clib_warning ("Cryptodev: Insufficient buffer space,"
			    " size %d needed",
			    size);
	      return -1;
	    }
	}

      /* mac data */
      if (mac_len)
	i =
	  oct_crypto_fill_sg_comp_from_buf (scatter_comp, i, &params->mac_buf);
    }
  else
    {
      /* Output including mac */
      size = outputlen - iv_len;

      if (size)
	{
	  uint32_t aad_offset = aad_len ? passthrough_len : 0;

	  i = oct_crypto_fill_sg_comp_from_iov (
	    scatter_comp, i, params->dst_iov, 0, &size, aad_buf, aad_offset);

	  if (PREDICT_FALSE (size))
	    {
	      clib_warning ("Cryptodev: Insufficient buffer space,"
			    " size %d needed",
			    size);
	      return -1;
	    }
	}
    }
  ((uint16_t *) in_buffer)[3] = clib_host_to_net_u16 (i);
  s_size_bytes = ((i + 3) / 4) * sizeof (struct roc_sglist_comp);

  size = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;

  /* This is DPTR len in case of SG mode */
  inst->w4.s.dlen = size;

  if (PREDICT_FALSE (size > ROC_SG_MAX_DLEN_SIZE))
    {
      clib_warning (
	"Cryptodev: Exceeds max supported components. Reduce segments");
      ret = -1;
    }

  inst->dptr = (uint64_t) in_buffer;
  return ret;
}

static_always_inline int __attribute__ ((unused))
oct_crypto_sg2_inst_prep (struct roc_se_fc_params *params,
			  struct cpt_inst_s *inst, u64 offset_ctrl,
			  const u8 *iv_s, int iv_len, u8 pack_iv,
			  u8 pdcp_alg_type, i32 inputlen, i32 outputlen,
			  u32 passthrough_len, u32 req_flags, int pdcp_flag,
			  int decrypt)
{
  u32 mac_len = 0, aad_len = 0, size, index, g_size_bytes;
  struct roc_sg2list_comp *gather_comp, *scatter_comp;
  void *m_vaddr = params->meta_buf.vaddr;
  struct roc_se_buf_ptr *aad_buf = NULL;
  union cpt_inst_w5 cpt_inst_w5;
  union cpt_inst_w6 cpt_inst_w6;
  u16 scatter_sz, gather_sz;
  struct roc_se_ctx *se_ctx;
  u64 *offset_vaddr;
  int ret = 0;
  u8 *iv_d;

  se_ctx = params->ctx;
  mac_len = se_ctx->mac_len;

  if (PREDICT_FALSE (req_flags & ROC_SE_VALID_AAD_BUF))
    {
      /* We don't support both AAD and auth data separately */
      aad_len = params->aad_buf.size;
      aad_buf = &params->aad_buf;
    }

  /* save space for iv */
  offset_vaddr = m_vaddr;

  m_vaddr = (u8 *) m_vaddr + ROC_SE_OFF_CTRL_LEN + PLT_ALIGN_CEIL (iv_len, 8);

  inst->w4.s.opcode_major |= (u64) ROC_DMA_MODE_SG;

  /* This is DPTR len in case of SG mode */
  inst->w4.s.dlen = inputlen + ROC_SE_OFF_CTRL_LEN;

  /* iv offset is 0 */
  *offset_vaddr = offset_ctrl;
  iv_d = ((u8 *) offset_vaddr + ROC_SE_OFF_CTRL_LEN);

  if (PREDICT_TRUE (iv_len))
    clib_memcpy (iv_d, iv_s, iv_len);

  /* DPTR has SG list */

  gather_comp = (struct roc_sg2list_comp *) ((u8 *) m_vaddr);

  /*
   * Input Gather List
   */
  index = 0;

  /* Offset control word followed by iv */

  index = oct_crypto_fill_sg2_comp (gather_comp, index, (u64) offset_vaddr,
				    ROC_SE_OFF_CTRL_LEN + iv_len);

  /* Add input data */
  if (decrypt && (req_flags & ROC_SE_VALID_MAC_BUF))
    {
      size = inputlen - iv_len - mac_len;
      if (size)
	{
	  /* input data only */
	  u32 aad_offset = aad_len ? passthrough_len : 0;

	  index = oct_crypto_fill_sg2_comp_from_iov (gather_comp, index,
						     params->src_iov, 0, &size,
						     aad_buf, aad_offset);

	  if (PREDICT_FALSE (size))
	    {
	      clib_warning ("Cryptodev: Insufficient buffer"
			    " space, size %d needed",
			    size);
	      return -1;
	    }
	}

      /* mac data */
      if (mac_len)
	index = oct_crypto_fill_sg2_comp_from_buf (gather_comp, index,
						   &params->mac_buf);
    }
  else
    {
      /* input data */
      size = inputlen - iv_len;
      if (size)
	{
	  u32 aad_offset = aad_len ? passthrough_len : 0;

	  index = oct_crypto_fill_sg2_comp_from_iov (gather_comp, index,
						     params->src_iov, 0, &size,
						     aad_buf, aad_offset);
	  if (PREDICT_FALSE (size))
	    {
	      clib_warning ("Cryptodev: Insufficient buffer space,"
			    " size %d needed",
			    size);
	      return -1;
	    }
	}
    }

  gather_sz = (index + 2) / 3;
  g_size_bytes = gather_sz * sizeof (struct roc_sg2list_comp);

  /*
   * Output Scatter List
   */

  index = 0;
  scatter_comp =
    (struct roc_sg2list_comp *) ((u8 *) gather_comp + g_size_bytes);

  index = oct_crypto_fill_sg2_comp (
    scatter_comp, index, (u64) offset_vaddr + ROC_SE_OFF_CTRL_LEN, iv_len);

  /* Add output data */
  if ((!decrypt) && (req_flags & ROC_SE_VALID_MAC_BUF))
    {
      size = outputlen - iv_len - mac_len;
      if (size)
	{

	  u32 aad_offset = aad_len ? passthrough_len : 0;

	  index = oct_crypto_fill_sg2_comp_from_iov (scatter_comp, index,
						     params->dst_iov, 0, &size,
						     aad_buf, aad_offset);
	  if (PREDICT_FALSE (size))
	    {
	      clib_warning ("Cryptodev: Insufficient buffer space,"
			    " size %d needed",
			    size);
	      return -1;
	    }
	}

      /* mac data */
      if (mac_len)
	index = oct_crypto_fill_sg2_comp_from_buf (scatter_comp, index,
						   &params->mac_buf);
    }
  else
    {
      /* Output including mac */
      size = outputlen - iv_len;
      if (size)
	{
	  u32 aad_offset = aad_len ? passthrough_len : 0;

	  index = oct_crypto_fill_sg2_comp_from_iov (scatter_comp, index,
						     params->dst_iov, 0, &size,
						     aad_buf, aad_offset);

	  if (PREDICT_FALSE (size))
	    {
	      clib_warning ("Cryptodev: Insufficient buffer space,"
			    " size %d needed",
			    size);
	      return -1;
	    }
	}
    }

  scatter_sz = (index + 2) / 3;

  cpt_inst_w5.s.gather_sz = gather_sz;
  cpt_inst_w6.s.scatter_sz = scatter_sz;

  cpt_inst_w5.s.dptr = (u64) gather_comp;
  cpt_inst_w6.s.rptr = (u64) scatter_comp;

  inst->w5.u64 = cpt_inst_w5.u64;
  inst->w6.u64 = cpt_inst_w6.u64;

  if (PREDICT_FALSE ((scatter_sz >> 4) || (gather_sz >> 4)))
    {
      clib_warning (
	"Cryptodev: Exceeds max supported components. Reduce segments");
      ret = -1;
    }

  return ret;
}

static_always_inline int
oct_crypto_cpt_hmac_prep (u32 flags, u64 d_offs, u64 d_lens,
			  struct roc_se_fc_params *fc_params,
			  struct cpt_inst_s *inst, u8 is_decrypt)
{
  u32 encr_data_len, auth_data_len, aad_len = 0;
  i32 inputlen, outputlen, enc_dlen, auth_dlen;
  u32 encr_offset, auth_offset, iv_offset = 0;
  union cpt_inst_w4 cpt_inst_w4;
  u32 cipher_type;
  struct roc_se_ctx *se_ctx;
  u32 passthrough_len = 0;
  const u8 *src = NULL;
  u64 offset_ctrl;
  u8 iv_len = 16;
  u8 op_minor;
  u32 mac_len;
  int ret;

  encr_offset = ROC_SE_ENCR_OFFSET (d_offs);
  auth_offset = ROC_SE_AUTH_OFFSET (d_offs);
  encr_data_len = ROC_SE_ENCR_DLEN (d_lens);
  auth_data_len = ROC_SE_AUTH_DLEN (d_lens);

  if (PREDICT_FALSE (flags & ROC_SE_VALID_AAD_BUF))
    {
      /* We don't support both AAD and auth data separately */
      auth_data_len = 0;
      auth_offset = 0;
      aad_len = fc_params->aad_buf.size;
    }

  se_ctx = fc_params->ctx;
  cipher_type = se_ctx->enc_cipher;
  mac_len = se_ctx->mac_len;
  cpt_inst_w4.u64 = se_ctx->template_w4.u64;
  op_minor = cpt_inst_w4.s.opcode_minor;

  if (PREDICT_FALSE (flags & ROC_SE_VALID_AAD_BUF))
    {
      /*
       * When AAD is given, data above encr_offset is pass through
       * Since AAD is given as separate pointer and not as offset,
       * this is a special case as we need to fragment input data
       * into passthrough + encr_data and then insert AAD in between.
       */
      passthrough_len = encr_offset;
      auth_offset = passthrough_len + iv_len;
      encr_offset = passthrough_len + aad_len + iv_len;
      auth_data_len = aad_len + encr_data_len;
    }
  else
    {
      encr_offset += iv_len;
      auth_offset += iv_len;
    }

  auth_dlen = auth_offset + auth_data_len;
  enc_dlen = encr_data_len + encr_offset;

  cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_FC;

  if (is_decrypt)
    {
      cpt_inst_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DECRYPT;

      if (auth_dlen > enc_dlen)
	{
	  inputlen = auth_dlen + mac_len;
	  outputlen = auth_dlen;
	}
      else
	{
	  inputlen = enc_dlen + mac_len;
	  outputlen = enc_dlen;
	}
    }
  else
    {
      cpt_inst_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_ENCRYPT;

      /* Round  up  to 16 bytes alignment */
      if (PREDICT_FALSE (encr_data_len & 0xf))
	{
	  if (PREDICT_TRUE (cipher_type == ROC_SE_AES_CBC) ||
	      (cipher_type == ROC_SE_DES3_CBC))
	    enc_dlen = PLT_ALIGN_CEIL (encr_data_len, 8) + encr_offset;
	}

      /*
       * auth_dlen is larger than enc_dlen in Authentication cases
       * like AES GMAC Authentication
       */
      if (PREDICT_FALSE (auth_dlen > enc_dlen))
	{
	  inputlen = auth_dlen;
	  outputlen = auth_dlen + mac_len;
	}
      else
	{
	  inputlen = enc_dlen;
	  outputlen = enc_dlen + mac_len;
	}
    }

  if (op_minor & ROC_SE_FC_MINOR_OP_HMAC_FIRST)
    outputlen = enc_dlen;

  cpt_inst_w4.s.param1 = encr_data_len;
  cpt_inst_w4.s.param2 = auth_data_len;

  if (PREDICT_FALSE ((encr_offset >> 16) || (iv_offset >> 8) ||
		     (auth_offset >> 8)))
    {
      clib_warning ("Cryptodev: Offset not supported");
      clib_warning (
	"Cryptodev: enc_offset: %d, iv_offset : %d, auth_offset: %d",
	encr_offset, iv_offset, auth_offset);
      return -1;
    }

  offset_ctrl = clib_host_to_net_u64 (
    ((u64) encr_offset << 16) | ((u64) iv_offset << 8) | ((u64) auth_offset));

  src = fc_params->iv_buf;

  inst->w4.u64 = cpt_inst_w4.u64;

#ifdef PLATFORM_OCTEON9
  ret = oct_crypto_sg_inst_prep (fc_params, inst, offset_ctrl, src, iv_len, 0,
				 0, inputlen, outputlen, passthrough_len,
				 flags, 0, is_decrypt);
#else
  ret = oct_crypto_sg2_inst_prep (fc_params, inst, offset_ctrl, src, iv_len, 0,
				  0, inputlen, outputlen, passthrough_len,
				  flags, 0, is_decrypt);
#endif

  if (PREDICT_FALSE (ret))
    return -1;

  return 0;
}

static_always_inline int
oct_crypto_fill_fc_params (oct_crypto_sess_t *sess, struct cpt_inst_s *inst,
			   const bool is_aead, u8 aad_length, u8 *payload,
			   vnet_crypto_async_frame_elt_t *elts, void *mdata,
			   u32 cipher_data_length, u32 cipher_data_offset,
			   u32 auth_data_length, u32 auth_data_offset,
			   vlib_buffer_t *b, u16 adj_len)
{
  struct roc_se_fc_params fc_params = { 0 };
  struct roc_se_ctx *ctx = &sess->cpt_ctx;
  u64 d_offs = 0, d_lens = 0;
  vlib_buffer_t *buffer = b;
  u32 flags = 0, index = 0;
  u8 op_minor = 0, cpt_op;
  char src[SRC_IOV_SIZE];
  u32 *iv_buf;

  cpt_op = sess->cpt_op;

  if (is_aead)
    {
      flags |= ROC_SE_VALID_IV_BUF;
      iv_buf = (u32 *) elts->iv;
      iv_buf[3] = clib_host_to_net_u32 (0x1);
      fc_params.iv_buf = elts->iv;

      d_offs = cipher_data_offset;
      d_offs = d_offs << 16;

      d_lens = cipher_data_length;
      d_lens = d_lens << 32;

      fc_params.aad_buf.vaddr = elts->aad;
      fc_params.aad_buf.size = aad_length;
      flags |= ROC_SE_VALID_AAD_BUF;

      if (sess->cpt_ctx.mac_len)
	{
	  flags |= ROC_SE_VALID_MAC_BUF;
	  fc_params.mac_buf.size = sess->cpt_ctx.mac_len;
	  fc_params.mac_buf.vaddr = elts->tag;
	}
    }
  else
    {
      op_minor = ctx->template_w4.s.opcode_minor;

      flags |= ROC_SE_VALID_IV_BUF;

      fc_params.iv_buf = elts->iv;

      d_offs = cipher_data_offset;
      d_offs = (d_offs << 16) | auth_data_offset;

      d_lens = cipher_data_length;
      d_lens = (d_lens << 32) | auth_data_length;

      if (PREDICT_TRUE (sess->cpt_ctx.mac_len))
	{
	  if (!(op_minor & ROC_SE_FC_MINOR_OP_HMAC_FIRST))
	    {
	      flags |= ROC_SE_VALID_MAC_BUF;
	      fc_params.mac_buf.size = sess->cpt_ctx.mac_len;
	      fc_params.mac_buf.vaddr = elts->digest;
	    }
	}
    }

  fc_params.ctx = &sess->cpt_ctx;

  fc_params.src_iov = (void *) src;

  fc_params.src_iov->bufs[index].vaddr = payload;
  fc_params.src_iov->bufs[index].size = b->current_length - adj_len;
  index++;

  while (buffer->flags & VLIB_BUFFER_NEXT_PRESENT)
    {
      buffer = vlib_get_buffer (vlib_get_main (), buffer->next_buffer);
      fc_params.src_iov->bufs[index].vaddr =
	buffer->data + buffer->current_data;
      fc_params.src_iov->bufs[index].size = buffer->current_length;
      index++;
    }

  fc_params.src_iov->buf_cnt = index;

  fc_params.dst_iov = (void *) src;

  fc_params.meta_buf.vaddr = mdata;
  fc_params.meta_buf.size = OCT_SCATTER_GATHER_BUFFER_SIZE;

  return oct_crypto_cpt_hmac_prep (flags, d_offs, d_lens, &fc_params, inst,
				   cpt_op);
}

static_always_inline u64
oct_cpt_inst_w7_get (oct_crypto_sess_t *sess, struct roc_cpt *roc_cpt)
{
  union cpt_inst_w7 inst_w7;

  inst_w7.u64 = 0;
  inst_w7.s.cptr = (u64) &sess->cpt_ctx.se_ctx.fctx;
  /* Set the engine group */
  inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];

  return inst_w7.u64;
}

static_always_inline i32
oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess,
				u32 key_index, u8 type)
{
  vnet_crypto_key_t *crypto_key, *auth_key;
  roc_se_cipher_type enc_type = 0;
  roc_se_auth_type auth_type = 0;
  vnet_crypto_key_t *key;
  u32 digest_len = ~0;
  i32 rv = 0;

  key = vnet_crypto_get_key (key_index);

  switch (key->async_alg)
    {
    case VNET_CRYPTO_ALG_AES_128_CBC_SHA1_TAG12:
    case VNET_CRYPTO_ALG_AES_192_CBC_SHA1_TAG12:
    case VNET_CRYPTO_ALG_AES_256_CBC_SHA1_TAG12:
      enc_type = ROC_SE_AES_CBC;
      auth_type = ROC_SE_SHA1_TYPE;
      digest_len = 12;
      break;
    case VNET_CRYPTO_ALG_AES_128_CBC_SHA224_TAG14:
    case VNET_CRYPTO_ALG_AES_192_CBC_SHA224_TAG14:
    case VNET_CRYPTO_ALG_AES_256_CBC_SHA224_TAG14:
      enc_type = ROC_SE_AES_CBC;
      auth_type = ROC_SE_SHA2_SHA224;
      digest_len = 14;
      break;
    case VNET_CRYPTO_ALG_AES_128_CBC_SHA256_TAG16:
    case VNET_CRYPTO_ALG_AES_192_CBC_SHA256_TAG16:
    case VNET_CRYPTO_ALG_AES_256_CBC_SHA256_TAG16:
      enc_type = ROC_SE_AES_CBC;
      auth_type = ROC_SE_SHA2_SHA256;
      digest_len = 16;
      break;
    case VNET_CRYPTO_ALG_AES_128_CBC_SHA384_TAG24:
    case VNET_CRYPTO_ALG_AES_192_CBC_SHA384_TAG24:
    case VNET_CRYPTO_ALG_AES_256_CBC_SHA384_TAG24:
      enc_type = ROC_SE_AES_CBC;
      auth_type = ROC_SE_SHA2_SHA384;
      digest_len = 24;
      break;
    case VNET_CRYPTO_ALG_AES_128_CBC_SHA512_TAG32:
    case VNET_CRYPTO_ALG_AES_192_CBC_SHA512_TAG32:
    case VNET_CRYPTO_ALG_AES_256_CBC_SHA512_TAG32:
      enc_type = ROC_SE_AES_CBC;
      auth_type = ROC_SE_SHA2_SHA512;
      digest_len = 32;
      break;
    case VNET_CRYPTO_ALG_AES_128_CTR_SHA1_TAG12:
    case VNET_CRYPTO_ALG_AES_192_CTR_SHA1_TAG12:
    case VNET_CRYPTO_ALG_AES_256_CTR_SHA1_TAG12:
      enc_type = ROC_SE_AES_CTR;
      auth_type = ROC_SE_SHA1_TYPE;
      digest_len = 12;
      break;
    case VNET_CRYPTO_ALG_3DES_CBC_MD5_TAG12:
      enc_type = ROC_SE_DES3_CBC;
      auth_type = ROC_SE_MD5_TYPE;
      digest_len = 12;
      break;
    case VNET_CRYPTO_ALG_3DES_CBC_SHA1_TAG12:
      enc_type = ROC_SE_DES3_CBC;
      auth_type = ROC_SE_SHA1_TYPE;
      digest_len = 12;
      break;
    case VNET_CRYPTO_ALG_3DES_CBC_SHA224_TAG14:
      enc_type = ROC_SE_DES3_CBC;
      auth_type = ROC_SE_SHA2_SHA224;
      digest_len = 14;
      break;
    case VNET_CRYPTO_ALG_3DES_CBC_SHA256_TAG16:
      enc_type = ROC_SE_DES3_CBC;
      auth_type = ROC_SE_SHA2_SHA256;
      digest_len = 16;
      break;
    case VNET_CRYPTO_ALG_3DES_CBC_SHA384_TAG24:
      enc_type = ROC_SE_DES3_CBC;
      auth_type = ROC_SE_SHA2_SHA384;
      digest_len = 24;
      break;
    case VNET_CRYPTO_ALG_3DES_CBC_SHA512_TAG32:
      enc_type = ROC_SE_DES3_CBC;
      auth_type = ROC_SE_SHA2_SHA512;
      digest_len = 32;
      break;
    default:
      clib_warning (
	"Cryptodev: Undefined link algo %u specified. Key index %u",
	key->async_alg, key_index);
      return -1;
    }

  if (type == VNET_CRYPTO_OP_TYPE_ENCRYPT)
    sess->cpt_ctx.ciph_then_auth = true;
  else
    sess->cpt_ctx.auth_then_ciph = true;

  sess->iv_length = 16;
  sess->cpt_op = type;

  crypto_key = vnet_crypto_get_key (key->index_crypto);
  rv = roc_se_ciph_key_set (&sess->cpt_ctx, enc_type, crypto_key->data,
			    vec_len (crypto_key->data));
  if (rv)
    {
      clib_warning ("Cryptodev: Error in setting cipher key for enc type %u",
		    enc_type);
      return -1;
    }

  auth_key = vnet_crypto_get_key (key->index_integ);

  rv = roc_se_auth_key_set (&sess->cpt_ctx, auth_type, auth_key->data,
			    vec_len (auth_key->data), digest_len);
  if (rv)
    {
      clib_warning ("Cryptodev: Error in setting auth key for auth type %u",
		    auth_type);
      return -1;
    }

  return 0;
}

static_always_inline i32
oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess,
				u32 key_index, u8 type)
{
  vnet_crypto_key_t *key = vnet_crypto_get_key (key_index);
  roc_se_cipher_type enc_type = 0;
  roc_se_auth_type auth_type = 0;
  u32 digest_len = ~0;
  i32 rv = 0;

  switch (key->async_alg)
    {
    case VNET_CRYPTO_ALG_AES_128_GCM:
    case VNET_CRYPTO_ALG_AES_192_GCM:
    case VNET_CRYPTO_ALG_AES_256_GCM:
      enc_type = ROC_SE_AES_GCM;
      sess->aes_gcm = 1;
      sess->iv_offset = 0;
      sess->iv_length = 16;
      sess->cpt_ctx.mac_len = 16;
      sess->cpt_op = type;
      digest_len = 16;
      break;
    default:
      clib_warning (
	"Cryptodev: Undefined cipher algo %u specified. Key index %u",
	key->async_alg, key_index);
      return -1;
    }

  rv = roc_se_ciph_key_set (&sess->cpt_ctx, enc_type, key->data,
			    vec_len (key->data));
  if (rv)
    {
      clib_warning ("Cryptodev: Error in setting cipher key for enc type %u",
		    enc_type);
      return -1;
    }

  rv = roc_se_auth_key_set (&sess->cpt_ctx, auth_type, NULL, 0, digest_len);
  if (rv)
    {
      clib_warning ("Cryptodev: Error in setting auth key for auth type %u",
		    auth_type);
      return -1;
    }

  return 0;
}

static_always_inline i32
oct_crypto_session_init (vlib_main_t *vm, oct_crypto_sess_t *session,
			 vnet_crypto_key_index_t key_index, int op_type)
{
  oct_crypto_main_t *ocm = &oct_crypto_main;
  vnet_crypto_key_t *key;
  oct_crypto_dev_t *ocd;
  i32 rv = 0;

  ocd = ocm->crypto_dev[op_type];

  key = vnet_crypto_get_key (key_index);

  if (key->type == VNET_CRYPTO_KEY_TYPE_LINK)
    rv = oct_crypto_link_session_update (vm, session, key_index, op_type);
  else
    rv = oct_crypto_aead_session_update (vm, session, key_index, op_type);

  if (rv)
    {
      oct_crypto_session_free (vm, session);
      return -1;
    }

  session->crypto_dev = ocd;

  session->cpt_inst_w7 =
    oct_cpt_inst_w7_get (session, session->crypto_dev->roc_cpt);

  session->initialised = 1;

  return 0;
}

static_always_inline void
oct_crypto_update_frame_error_status (vnet_crypto_async_frame_t *f, u32 index,
				      vnet_crypto_op_status_t s)
{
  u32 i;

  for (i = index; i < f->n_elts; i++)
    f->elts[i].status = s;

  if (index == 0)
    f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED;
}

static_always_inline int
oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame,
			    const u8 is_aead, u8 aad_len, const u8 type)
{
  u32 i, enq_tail, enc_auth_len, buffer_index, nb_infl_allowed;
  struct cpt_inst_s inst[VNET_CRYPTO_FRAME_SIZE];
  u32 crypto_start_offset, integ_start_offset;
  oct_crypto_main_t *ocm = &oct_crypto_main;
  vnet_crypto_async_frame_elt_t *elts;
  oct_crypto_dev_t *crypto_dev = NULL;
  oct_crypto_inflight_req_t *infl_req;
  oct_crypto_pending_queue_t *pend_q;
  u64 dptr_start_ptr, curr_ptr;
  oct_crypto_sess_t *sess;
  u32 crypto_total_length;
  oct_crypto_key_t *key;
  vlib_buffer_t *buffer;
  u16 adj_len;
  int ret;

  /* GCM packets having 8 bytes of aad and 8 bytes of iv */
  u8 aad_iv = 8 + 8;

  pend_q = &ocm->pend_q[vlib_get_thread_index ()];

  enq_tail = pend_q->enq_tail;

  nb_infl_allowed = pend_q->n_desc - pend_q->n_crypto_inflight;
  if (PREDICT_FALSE (nb_infl_allowed == 0))
    {
      oct_crypto_update_frame_error_status (
	frame, 0, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
      return -1;
    }

  infl_req = &pend_q->req_queue[enq_tail];
  infl_req->frame = frame;

  for (i = 0; i < frame->n_elts; i++)
    {
      elts = &frame->elts[i];
      buffer_index = frame->buffer_indices[i];
      key = vec_elt_at_index (ocm->keys[type], elts->key_index);

      if (PREDICT_FALSE (!key->sess))
	{
	  oct_crypto_update_frame_error_status (
	    frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
	  return -1;
	}

      sess = key->sess;

      if (PREDICT_FALSE (!sess->initialised))
	oct_crypto_session_init (vm, sess, elts->key_index, type);

      crypto_dev = sess->crypto_dev;

      clib_memset (inst + i, 0, sizeof (struct cpt_inst_s));

      buffer = vlib_get_buffer (vm, buffer_index);

      if (is_aead)
	{
	  dptr_start_ptr =
	    (u64) (buffer->data + (elts->crypto_start_offset - aad_iv));
	  curr_ptr = (u64) (buffer->data + buffer->current_data);
	  adj_len = (u16) (dptr_start_ptr - curr_ptr);

	  crypto_total_length = elts->crypto_total_length;
	  crypto_start_offset = aad_iv;
	  integ_start_offset = 0;

	  ret = oct_crypto_fill_fc_params (
	    sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts,
	    (oct_crypto_scatter_gather_t *) (infl_req->sg_data) + i,
	    crypto_total_length /* cipher_len */,
	    crypto_start_offset /* cipher_offset */, 0 /* auth_len */,
	    integ_start_offset /* auth_off */, buffer, adj_len);
	  if (PREDICT_FALSE (ret < 0))
	    {
	      oct_crypto_update_frame_error_status (
		frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
	      return -1;
	    }
	}
      else
	{
	  dptr_start_ptr = (u64) (buffer->data + elts->integ_start_offset);

	  enc_auth_len = elts->crypto_total_length + elts->integ_length_adj;

	  curr_ptr = (u64) (buffer->data + buffer->current_data);
	  adj_len = (u16) (dptr_start_ptr - curr_ptr);

	  crypto_total_length = elts->crypto_total_length;
	  crypto_start_offset =
	    elts->crypto_start_offset - elts->integ_start_offset;
	  integ_start_offset = 0;

	  ret = oct_crypto_fill_fc_params (
	    sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts,
	    (oct_crypto_scatter_gather_t *) (infl_req->sg_data) + i,
	    crypto_total_length /* cipher_len */,
	    crypto_start_offset /* cipher_offset */,
	    enc_auth_len /* auth_len */, integ_start_offset /* auth_off */,
	    buffer, adj_len);
	  if (PREDICT_FALSE (ret < 0))
	    {
	      oct_crypto_update_frame_error_status (
		frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
	      return -1;
	    }
	}

      inst[i].w7.u64 = sess->cpt_inst_w7;
      inst[i].res_addr = (u64) &infl_req->res[i];
    }

  oct_crypto_burst_submit (crypto_dev, inst, frame->n_elts);

  infl_req->elts = frame->n_elts;
  OCT_MOD_INC (pend_q->enq_tail, pend_q->n_desc);
  pend_q->n_crypto_inflight++;

  return 0;
}

int
oct_crypto_enqueue_linked_alg_enc (vlib_main_t *vm,
				   vnet_crypto_async_frame_t *frame)
{
  return oct_crypto_enqueue_enc_dec (
    vm, frame, 0 /* is_aead */, 0 /* aad_len */, VNET_CRYPTO_OP_TYPE_ENCRYPT);
}

int
oct_crypto_enqueue_linked_alg_dec (vlib_main_t *vm,
				   vnet_crypto_async_frame_t *frame)
{
  return oct_crypto_enqueue_enc_dec (
    vm, frame, 0 /* is_aead */, 0 /* aad_len */, VNET_CRYPTO_OP_TYPE_DECRYPT);
}

int
oct_crypto_enqueue_aead_aad_enc (vlib_main_t *vm,
				 vnet_crypto_async_frame_t *frame, u8 aad_len)
{
  return oct_crypto_enqueue_enc_dec (vm, frame, 1 /* is_aead */, aad_len,
				     VNET_CRYPTO_OP_TYPE_ENCRYPT);
}

static_always_inline int
oct_crypto_enqueue_aead_aad_dec (vlib_main_t *vm,
				 vnet_crypto_async_frame_t *frame, u8 aad_len)
{
  return oct_crypto_enqueue_enc_dec (vm, frame, 1 /* is_aead */, aad_len,
				     VNET_CRYPTO_OP_TYPE_DECRYPT);
}

int
oct_crypto_enqueue_aead_aad_8_enc (vlib_main_t *vm,
				   vnet_crypto_async_frame_t *frame)
{
  return oct_crypto_enqueue_aead_aad_enc (vm, frame, 8);
}

int
oct_crypto_enqueue_aead_aad_12_enc (vlib_main_t *vm,
				    vnet_crypto_async_frame_t *frame)
{
  return oct_crypto_enqueue_aead_aad_enc (vm, frame, 12);
}

int
oct_crypto_enqueue_aead_aad_8_dec (vlib_main_t *vm,
				   vnet_crypto_async_frame_t *frame)
{
  return oct_crypto_enqueue_aead_aad_dec (vm, frame, 8);
}

int
oct_crypto_enqueue_aead_aad_12_dec (vlib_main_t *vm,
				    vnet_crypto_async_frame_t *frame)
{
  return oct_crypto_enqueue_aead_aad_dec (vm, frame, 12);
}

vnet_crypto_async_frame_t *
oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed,
			  u32 *enqueue_thread_idx)
{
  oct_crypto_main_t *ocm = &oct_crypto_main;
  u32 deq_head, status = VNET_CRYPTO_OP_STATUS_COMPLETED;
  vnet_crypto_async_frame_elt_t *fe = NULL;
  oct_crypto_inflight_req_t *infl_req;
  oct_crypto_pending_queue_t *pend_q;
  vnet_crypto_async_frame_t *frame;
  volatile union cpt_res_s *res;
  int i;

  pend_q = &ocm->pend_q[vlib_get_thread_index ()];

  if (!pend_q->n_crypto_inflight)
    return NULL;

  deq_head = pend_q->deq_head;
  infl_req = &pend_q->req_queue[deq_head];
  frame = infl_req->frame;

  fe = frame->elts;

  for (i = infl_req->deq_elts; i < infl_req->elts; ++i)
    {
      res = &infl_req->res[i];

      if (PREDICT_FALSE (res->cn10k.compcode == CPT_COMP_NOT_DONE))
	return NULL;

      if (PREDICT_FALSE (res->cn10k.uc_compcode))
	{
	  if (res->cn10k.uc_compcode == ROC_SE_ERR_GC_ICV_MISCOMPARE)
	    status = fe[i].status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC;
	  else
	    status = fe[i].status = VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR;
	}

      infl_req->deq_elts++;
    }

  clib_memset ((void *) infl_req->res, 0,
	       sizeof (union cpt_res_s) * VNET_CRYPTO_FRAME_SIZE);

  OCT_MOD_INC (pend_q->deq_head, pend_q->n_desc);
  pend_q->n_crypto_inflight--;

  frame->state = status == VNET_CRYPTO_OP_STATUS_COMPLETED ?
		   VNET_CRYPTO_FRAME_STATE_SUCCESS :
		   VNET_CRYPTO_FRAME_STATE_ELT_ERROR;

  *nb_elts_processed = frame->n_elts;
  *enqueue_thread_idx = frame->enqueue_thread_index;

  infl_req->deq_elts = 0;
  infl_req->elts = 0;

  return frame;
}

int
oct_init_crypto_engine_handlers (vlib_main_t *vm, vnet_dev_t *dev)
{
  u32 engine_index;

  engine_index = vnet_crypto_register_engine (vm, "oct_cryptodev", 100,
					      "OCT Cryptodev Engine");

#define _(n, k, t, a)                                                         \
  vnet_crypto_register_enqueue_handler (                                      \
    vm, engine_index, VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_ENC,             \
    oct_crypto_enqueue_aead_aad_##a##_enc);                                   \
  vnet_crypto_register_enqueue_handler (                                      \
    vm, engine_index, VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_DEC,             \
    oct_crypto_enqueue_aead_aad_##a##_dec);
  foreach_oct_crypto_aead_async_alg
#undef _

#define _(c, h, k, d)                                                         \
  vnet_crypto_register_enqueue_handler (                                      \
    vm, engine_index, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_ENC,                \
    oct_crypto_enqueue_linked_alg_enc);                                       \
  vnet_crypto_register_enqueue_handler (                                      \
    vm, engine_index, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_DEC,                \
    oct_crypto_enqueue_linked_alg_dec);
    foreach_oct_crypto_link_async_alg;
#undef _

  vnet_crypto_register_dequeue_handler (vm, engine_index,
					oct_crypto_frame_dequeue);

  vnet_crypto_register_key_handler (vm, engine_index, oct_crypto_key_handler);

  return 0;
}

int
oct_conf_sw_queue (vlib_main_t *vm, vnet_dev_t *dev)
{
  oct_crypto_main_t *ocm = &oct_crypto_main;
  vlib_thread_main_t *tm = vlib_get_thread_main ();
  extern oct_plt_init_param_t oct_plt_init_param;
  oct_crypto_inflight_req_t *infl_req_queue;
  u32 n_inflight_req;
  int i, j = 0;

  ocm->pend_q = oct_plt_init_param.oct_plt_zmalloc (
    tm->n_vlib_mains * sizeof (oct_crypto_pending_queue_t),
    CLIB_CACHE_LINE_BYTES);
  if (ocm->pend_q == NULL)
    {
      log_err (dev, "Failed to allocate memory for crypto pending queue");
      return -1;
    }

  /*
   * Each pending queue will get number of cpt desc / number of cores.
   * And that desc count is shared across inflight entries.
   */
  n_inflight_req =
    (OCT_CPT_LF_MAX_NB_DESC / tm->n_vlib_mains) / VNET_CRYPTO_FRAME_SIZE;

  for (i = 0; i < tm->n_vlib_mains; ++i)
    {
      ocm->pend_q[i].n_desc = n_inflight_req;

      ocm->pend_q[i].req_queue = oct_plt_init_param.oct_plt_zmalloc (
	ocm->pend_q[i].n_desc * sizeof (oct_crypto_inflight_req_t),
	CLIB_CACHE_LINE_BYTES);
      if (ocm->pend_q[i].req_queue == NULL)
	{
	  log_err (dev,
		   "Failed to allocate memory for crypto inflight request");
	  goto free;
	}

      for (j = 0; j <= ocm->pend_q[i].n_desc; ++j)
	{
	  infl_req_queue = &ocm->pend_q[i].req_queue[j];

	  infl_req_queue->sg_data = oct_plt_init_param.oct_plt_zmalloc (
	    OCT_SCATTER_GATHER_BUFFER_SIZE * VNET_CRYPTO_FRAME_SIZE,
	    CLIB_CACHE_LINE_BYTES);
	  if (infl_req_queue->sg_data == NULL)
	    {
	      log_err (dev, "Failed to allocate crypto scatter gather memory");
	      goto free;
	    }
	}
    }
  return 0;
free:
  for (; i >= 0; i--)
    {
      if (ocm->pend_q[i].req_queue == NULL)
	continue;
      for (; j >= 0; j--)
	{
	  infl_req_queue = &ocm->pend_q[i].req_queue[j];

	  if (infl_req_queue->sg_data == NULL)
	    continue;

	  oct_plt_init_param.oct_plt_free (infl_req_queue->sg_data);
	}
      oct_plt_init_param.oct_plt_free (ocm->pend_q[i].req_queue);
    }
  oct_plt_init_param.oct_plt_free (ocm->pend_q);

  return -1;
}