octeon: add direct mode changes in crypto datapath

This patch introduces support for direct mode crypto
submission on CPT. For multi-segmented buffers,
scatter-gather submission mode will be utilized.

Type: feature

Signed-off-by: Nithinsen Kaithakadan <nkaithakadan@marvell.com>
Change-Id: Idb99e6c9ea49028e11d3bf530c9559719c988252

2 files changed, 269 insertions, 48 deletions

diff --git a/src/plugins/dev_octeon/crypto.c b/src/plugins/dev_octeon/crypto.c
index 1c3aa42e515..b6124206f75 100644
--- a/src/plugins/dev_octeon/crypto.c
+++ b/src/plugins/dev_octeon/crypto.c
@@ -63,6 +63,9 @@ oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index,
   oct_crypto_sess_t *session;
   vnet_crypto_key_t *key;
   oct_crypto_key_t *ckey;
+  oct_crypto_dev_t *ocd;
+
+  ocd = ocm->crypto_dev[op_type];
 
   key = vnet_crypto_get_key (key_index);
 
@@ -89,6 +92,7 @@ oct_crypto_session_create (vlib_main_t *vm, vnet_crypto_key_index_t key_index,
       session = oct_crypto_session_alloc (vm, op_type);
       if (session == NULL)
 	return -1;
+      session->crypto_dev = ocd;
     }
 
   oct_map_keyindex_to_session (session, key_index, op_type);
@@ -117,6 +121,12 @@ oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index)
 	    ocm->keys[VNET_CRYPTO_OP_TYPE_ENCRYPT], ckey->sess->key_index);
 	  ckey_linked->sess = NULL;
 	}
+
+      /* Trigger CTX flush + invalidate to remove from CTX_CACHE */
+      if (oct_hw_ctx_cache_enable ())
+	roc_cpt_lf_ctx_flush (&ckey->sess->crypto_dev->lf,
+			      &ckey->sess->cpt_ctx.se_ctx, true);
+
       oct_plt_init_param.oct_plt_free (ckey->sess);
       ckey->sess = NULL;
     }
@@ -134,6 +144,11 @@ oct_crypto_key_del_handler (vlib_main_t *vm, vnet_crypto_key_index_t key_index)
 	  ckey_linked->sess = NULL;
 	}
 
+      /* Trigger CTX flush + invalidate to remove from CTX_CACHE */
+      if (oct_hw_ctx_cache_enable ())
+	roc_cpt_lf_ctx_flush (&ckey->sess->crypto_dev->lf,
+			      &ckey->sess->cpt_ctx.se_ctx, true);
+
       oct_plt_init_param.oct_plt_free (ckey->sess);
       ckey->sess = NULL;
     }
@@ -1060,12 +1075,11 @@ oct_crypto_cpt_hmac_prep (u32 flags, u64 d_offs, u64 d_lens,
 }
 
 static_always_inline int
-oct_crypto_fill_fc_params (oct_crypto_sess_t *sess, struct cpt_inst_s *inst,
-			   const bool is_aead, u8 aad_length, u8 *payload,
-			   vnet_crypto_async_frame_elt_t *elts, void *mdata,
-			   u32 cipher_data_length, u32 cipher_data_offset,
-			   u32 auth_data_length, u32 auth_data_offset,
-			   vlib_buffer_t *b, u16 adj_len)
+oct_crypto_scatter_gather_mode (
+  oct_crypto_sess_t *sess, struct cpt_inst_s *inst, const bool is_aead,
+  u8 aad_length, u8 *payload, vnet_crypto_async_frame_elt_t *elts, void *mdata,
+  u32 cipher_data_length, u32 cipher_data_offset, u32 auth_data_length,
+  u32 auth_data_offset, vlib_buffer_t *b, u16 adj_len)
 {
   struct roc_se_fc_params fc_params = { 0 };
   struct roc_se_ctx *ctx = &sess->cpt_ctx;
@@ -1163,6 +1177,9 @@ oct_cpt_inst_w7_get (oct_crypto_sess_t *sess, struct roc_cpt *roc_cpt)
   inst_w7.u64 = 0;
   inst_w7.s.cptr = (u64) &sess->cpt_ctx.se_ctx.fctx;
 
+  if (oct_hw_ctx_cache_enable ())
+    inst_w7.s.ctx_val = 1;
+
   /* Set the engine group */
   inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];
 
@@ -1299,6 +1316,13 @@ oct_crypto_link_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess,
       return -1;
     }
 
+  sess->cpt_ctx.template_w4.s.opcode_major = ROC_SE_MAJOR_OP_FC;
+
+  if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT)
+    sess->cpt_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DECRYPT;
+  else
+    sess->cpt_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_ENCRYPT;
+
   return 0;
 }
 
@@ -1352,6 +1376,13 @@ oct_crypto_aead_session_update (vlib_main_t *vm, oct_crypto_sess_t *sess,
       return -1;
     }
 
+  sess->cpt_ctx.template_w4.s.opcode_major = ROC_SE_MAJOR_OP_FC;
+
+  if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT)
+    sess->cpt_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_DECRYPT;
+  else
+    sess->cpt_ctx.template_w4.s.opcode_minor |= ROC_SE_FC_MINOR_OP_ENCRYPT;
+
   if (enc_type == ROC_SE_CHACHA20)
     sess->cpt_ctx.template_w4.s.opcode_minor |= BIT (5);
 
@@ -1387,6 +1418,9 @@ oct_crypto_session_init (vlib_main_t *vm, oct_crypto_sess_t *session,
   session->cpt_inst_w7 =
     oct_cpt_inst_w7_get (session, session->crypto_dev->roc_cpt);
 
+  if (oct_hw_ctx_cache_enable ())
+    roc_se_ctx_init (&session->cpt_ctx);
+
   session->initialised = 1;
 
   return 0;
@@ -1405,6 +1439,138 @@ oct_crypto_update_frame_error_status (vnet_crypto_async_frame_t *f, u32 index,
     f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED;
 }
 
+static_always_inline void
+oct_crypto_direct_mode_linked (vlib_buffer_t *buffer, struct cpt_inst_s *inst,
+			       oct_crypto_sess_t *sess,
+			       oct_crypto_inflight_req_t *infl_req, u8 aad_len)
+{
+  u32 encr_offset, auth_offset, iv_offset;
+  vnet_crypto_async_frame_elt_t *elts;
+  union cpt_inst_w4 cpt_inst_w4;
+  u64 *offset_control_word;
+  u32 crypto_total_length;
+  u32 auth_dlen, enc_dlen;
+  u32 enc_auth_len;
+
+  elts = infl_req->fe;
+  enc_auth_len = elts->crypto_total_length + elts->integ_length_adj;
+  crypto_total_length = elts->crypto_total_length;
+
+  if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT)
+    {
+      /*
+       * Position the offset control word so that it does not
+       * overlap with the IV.
+       */
+      offset_control_word = (void *) (buffer->data) - ROC_SE_OFF_CTRL_LEN - 4;
+
+      iv_offset =
+	(void *) elts->iv - (void *) offset_control_word - ROC_SE_OFF_CTRL_LEN;
+    }
+  else
+    {
+      offset_control_word = (void *) (elts->iv) - ROC_SE_OFF_CTRL_LEN;
+      iv_offset = 0;
+    }
+
+  encr_offset = (void *) (buffer->data + elts->crypto_start_offset) -
+		(void *) offset_control_word - ROC_SE_OFF_CTRL_LEN;
+  auth_offset = (void *) (buffer->data + elts->integ_start_offset) -
+		(void *) offset_control_word - ROC_SE_OFF_CTRL_LEN;
+  *offset_control_word = clib_host_to_net_u64 (
+    ((u64) encr_offset << 16) | ((u64) iv_offset << 8) | ((u64) auth_offset));
+
+  cpt_inst_w4.u64 = sess->cpt_ctx.template_w4.u64;
+
+  cpt_inst_w4.s.param1 = crypto_total_length;
+  cpt_inst_w4.s.param2 = enc_auth_len;
+
+  auth_dlen = auth_offset + enc_auth_len + ROC_SE_OFF_CTRL_LEN;
+  enc_dlen = encr_offset + crypto_total_length + ROC_SE_OFF_CTRL_LEN;
+
+  if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT)
+    cpt_inst_w4.s.dlen = auth_dlen + sess->cpt_ctx.mac_len;
+  else
+    {
+      /*
+       * In the case of ESN, 4 bytes of the seqhi will be stored at the end of
+       * the cipher. This data must be overwritten by the digest data during
+       * the dequeue process.
+       */
+      if (auth_dlen > enc_dlen)
+	infl_req->esn_enabled = true;
+
+      cpt_inst_w4.s.dlen = auth_dlen;
+    }
+
+  infl_req->mac_len = sess->cpt_ctx.mac_len;
+
+  inst->dptr = (uint64_t) offset_control_word;
+  inst->rptr = (uint64_t) ((void *) offset_control_word + ROC_SE_OFF_CTRL_LEN);
+  inst->w4.u64 = cpt_inst_w4.u64;
+}
+
+static_always_inline void
+oct_crypto_direct_mode_aead (vlib_buffer_t *buffer, struct cpt_inst_s *inst,
+			     oct_crypto_sess_t *sess,
+			     oct_crypto_inflight_req_t *infl_req, u8 aad_len)
+{
+  u32 encr_offset, auth_offset, iv_offset;
+  u32 auth_copy_offset, iv_copy_offset;
+  vnet_crypto_async_frame_elt_t *elts;
+  union cpt_inst_w4 cpt_inst_w4;
+  u64 *offset_control_word;
+  u32 crypto_total_length;
+
+  elts = infl_req->fe;
+  crypto_total_length = elts->crypto_total_length;
+
+  ((u32 *) elts->iv)[3] = clib_host_to_net_u32 (0x1);
+
+  offset_control_word = (void *) (elts->aad) - ROC_SE_OFF_CTRL_LEN;
+  encr_offset = (void *) (buffer->data + elts->crypto_start_offset) -
+		(void *) offset_control_word - ROC_SE_OFF_CTRL_LEN;
+  iv_offset = elts->iv - elts->aad;
+  auth_offset = encr_offset - aad_len;
+
+  *offset_control_word = clib_host_to_net_u64 (
+    ((u64) encr_offset << 16) | ((u64) iv_offset << 8) | ((u64) auth_offset));
+
+  cpt_inst_w4.u64 = sess->cpt_ctx.template_w4.u64;
+
+  cpt_inst_w4.s.param1 = crypto_total_length;
+  cpt_inst_w4.s.param2 = crypto_total_length + aad_len;
+
+  if (sess->cpt_op == VNET_CRYPTO_OP_TYPE_DECRYPT)
+    cpt_inst_w4.s.dlen = encr_offset + elts->crypto_total_length +
+			 ROC_SE_OFF_CTRL_LEN + sess->cpt_ctx.mac_len;
+  else
+    cpt_inst_w4.s.dlen =
+      encr_offset + elts->crypto_total_length + ROC_SE_OFF_CTRL_LEN;
+
+  inst->dptr = (uint64_t) offset_control_word;
+  inst->rptr = (uint64_t) ((void *) offset_control_word + ROC_SE_OFF_CTRL_LEN);
+  inst->w4.u64 = cpt_inst_w4.u64;
+
+  /*
+   * CPT hardware requires the AAD to be followed by the cipher packet.
+   * Therefore, maintain a copy of the AAD and IV in the inflight request,
+   * and write the AAD in front of the cipher data before submission.
+   */
+  auth_copy_offset = encr_offset - sess->cpt_ctx.mac_len;
+  iv_copy_offset = encr_offset - 8;
+
+  clib_memcpy_fast (infl_req->aad,
+		    ((void *) inst->dptr) + auth_copy_offset + 8, 8);
+  clib_memcpy_fast (infl_req->iv, ((void *) inst->dptr) + iv_copy_offset + 8,
+		    8);
+  clib_memcpy_fast (((void *) inst->dptr) + encr_offset + ROC_SE_OFF_CTRL_LEN -
+		      aad_len,
+		    elts->aad, aad_len);
+
+  infl_req->aead_algo = true;
+}
+
 static_always_inline int
 oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame,
 			    const u8 is_aead, u8 aad_len, const u8 type)
@@ -1480,54 +1646,72 @@ oct_crypto_enqueue_enc_dec (vlib_main_t *vm, vnet_crypto_async_frame_t *frame,
 
       if (is_aead)
 	{
-	  dptr_start_ptr =
-	    (u64) (buffer->data + (elts->crypto_start_offset - aad_iv));
-	  curr_ptr = (u64) (buffer->data + buffer->current_data);
-	  adj_len = (u16) (dptr_start_ptr - curr_ptr);
-
-	  crypto_total_length = elts->crypto_total_length;
-	  crypto_start_offset = aad_iv;
-	  integ_start_offset = 0;
-
-	  ret = oct_crypto_fill_fc_params (
-	    sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts,
-	    ((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail,
-	    crypto_total_length /* cipher_len */,
-	    crypto_start_offset /* cipher_offset */, 0 /* auth_len */,
-	    integ_start_offset /* auth_off */, buffer, adj_len);
-	  if (PREDICT_FALSE (ret < 0))
+	  if (buffer->flags & VLIB_BUFFER_NEXT_PRESENT)
 	    {
-	      oct_crypto_update_frame_error_status (
-		frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
-	      return -1;
+	      dptr_start_ptr =
+		(u64) (buffer->data + (elts->crypto_start_offset - aad_iv));
+	      curr_ptr = (u64) (buffer->data + buffer->current_data);
+	      adj_len = (u16) (dptr_start_ptr - curr_ptr);
+
+	      crypto_total_length = elts->crypto_total_length;
+	      crypto_start_offset = aad_iv;
+	      integ_start_offset = 0;
+
+	      ret = oct_crypto_scatter_gather_mode (
+		sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts,
+		((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail,
+		crypto_total_length /* cipher_len */,
+		crypto_start_offset /* cipher_offset */, 0 /* auth_len */,
+		integ_start_offset /* auth_off */, buffer, adj_len);
+
+	      if (PREDICT_FALSE (ret < 0))
+		{
+		  oct_crypto_update_frame_error_status (
+		    frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+		  return -1;
+		}
+	    }
+	  else
+	    {
+	      oct_crypto_direct_mode_aead (buffer, inst + i, sess, infl_req,
+					   aad_len);
 	    }
 	}
       else
 	{
-	  dptr_start_ptr = (u64) (buffer->data + elts->integ_start_offset);
-
-	  enc_auth_len = elts->crypto_total_length + elts->integ_length_adj;
-
-	  curr_ptr = (u64) (buffer->data + buffer->current_data);
-	  adj_len = (u16) (dptr_start_ptr - curr_ptr);
-
-	  crypto_total_length = elts->crypto_total_length;
-	  crypto_start_offset =
-	    elts->crypto_start_offset - elts->integ_start_offset;
-	  integ_start_offset = 0;
-
-	  ret = oct_crypto_fill_fc_params (
-	    sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts,
-	    ((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail,
-	    crypto_total_length /* cipher_len */,
-	    crypto_start_offset /* cipher_offset */,
-	    enc_auth_len /* auth_len */, integ_start_offset /* auth_off */,
-	    buffer, adj_len);
-	  if (PREDICT_FALSE (ret < 0))
+	  if (buffer->flags & VLIB_BUFFER_NEXT_PRESENT)
 	    {
-	      oct_crypto_update_frame_error_status (
-		frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
-	      return -1;
+	      dptr_start_ptr = (u64) (buffer->data + elts->integ_start_offset);
+
+	      curr_ptr = (u64) (buffer->data + buffer->current_data);
+	      adj_len = (u16) (dptr_start_ptr - curr_ptr);
+
+	      crypto_start_offset =
+		elts->crypto_start_offset - elts->integ_start_offset;
+	      integ_start_offset = 0;
+	      enc_auth_len =
+		elts->crypto_total_length + elts->integ_length_adj;
+	      crypto_total_length = elts->crypto_total_length;
+
+	      ret = oct_crypto_scatter_gather_mode (
+		sess, inst + i, is_aead, aad_len, (u8 *) dptr_start_ptr, elts,
+		((oct_crypto_scatter_gather_t *) (sg_data)) + enq_tail,
+		crypto_total_length /* cipher_len */,
+		crypto_start_offset /* cipher_offset */,
+		enc_auth_len /* auth_len */, integ_start_offset /* auth_off */,
+		buffer, adj_len);
+
+	      if (PREDICT_FALSE (ret < 0))
+		{
+		  oct_crypto_update_frame_error_status (
+		    frame, i, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+		  return -1;
+		}
+	    }
+	  else
+	    {
+	      oct_crypto_direct_mode_linked (buffer, inst + i, sess, infl_req,
+					     aad_len);
 	    }
 	}
 
@@ -1632,6 +1816,7 @@ oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed,
   vnet_crypto_async_frame_t *frame;
   volatile union cpt_res_s *res;
   bool last_elts_processed;
+  vlib_buffer_t *buffer;
 
   pend_q = &ocm->pend_q[vlib_get_thread_index ()];
 
@@ -1659,6 +1844,24 @@ oct_crypto_frame_dequeue (vlib_main_t *vm, u32 *nb_elts_processed,
 	    status = fe->status = VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR;
 	}
 
+      buffer =
+	vlib_get_buffer (vm, infl_req->frame->buffer_indices[infl_req->index]);
+
+      /*
+       * For AEAD, copy the AAD and IV back to their original positions.
+       * If ESN is enabled (in case of linked algo), overwrite the ESN
+       * seqhi at the end of the cipher with the digest data.
+       */
+      if (infl_req->aead_algo)
+	{
+	  clib_memcpy_fast (buffer->data + fe->crypto_start_offset - 8,
+			    infl_req->iv, 8);
+	  clib_memcpy_fast (buffer->data + fe->crypto_start_offset - 16,
+			    infl_req->aad, 8);
+	}
+      else if (infl_req->esn_enabled)
+	clib_memcpy_fast (fe->digest, fe->digest + 4, infl_req->mac_len);
+
       clib_memset ((void *) &infl_req->res, 0, sizeof (union cpt_res_s));
       last_elts_processed = infl_req->last_elts;
       OCT_MOD_INC (pend_q->deq_head, pend_q->n_desc);
diff --git a/src/plugins/dev_octeon/crypto.h b/src/plugins/dev_octeon/crypto.h
index d7a501eb931..b003a605e0c 100644
--- a/src/plugins/dev_octeon/crypto.h
+++ b/src/plugins/dev_octeon/crypto.h
@@ -120,6 +120,17 @@ typedef struct
   vnet_crypto_async_frame_t *frame;
   /** Async frame element */
   vnet_crypto_async_frame_elt_t *fe;
+  /** AAD meta data */
+  u8 aad[8];
+  /** IV meta data */
+  u8 iv[16];
+  /** Digest len */
+  u8 mac_len;
+  /** aead */
+  bool aead_algo;
+  /** Set when encrypting linked algo with esn.
+   * To move digest data */
+  bool esn_enabled;
   /** Set if this is last element in frame */
   bool last_elts;
   /** Index of element in frame */
@@ -153,6 +164,13 @@ typedef struct
   u8 started;
 } oct_crypto_main_t;
 
+static_always_inline bool
+oct_hw_ctx_cache_enable (void)
+{
+  return roc_errata_cpt_hang_on_mixed_ctx_val () ||
+	 roc_model_is_cn10ka_b0 () || roc_model_is_cn10kb_a0 ();
+}
+
 extern oct_crypto_main_t oct_crypto_main;
 
 void oct_crypto_key_del_handler (vlib_main_t *vm,