dpdk: deprecate ipsec backend

Type: refactor

DPDK crypto devices are now accessible via the async infra, so
there is no need for the DPDK ipsec plugin.

In addition this patch fixes the problem that cryptodev backend
not working when master core and worker cores lies in different
numa nodes.

Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com>
Signed-off-by: Neale Ranns <neale@graphiant.com>
Change-Id: Ie8516bea706248c7bc25abac53a9c656bb8247d9

1 files changed, 345 insertions, 271 deletions

diff --git a/src/plugins/dpdk/cryptodev/cryptodev_dp_api.c b/src/plugins/dpdk/cryptodev/cryptodev_dp_api.c
index 0a33d25bd61..420bb89dc0a 100644
--- a/src/plugins/dpdk/cryptodev/cryptodev_dp_api.c
+++ b/src/plugins/dpdk/cryptodev/cryptodev_dp_api.c
@@ -18,7 +18,7 @@
 #include <vlib/vlib.h>
 #include <vnet/plugin/plugin.h>
 #include <vnet/crypto/crypto.h>
-#include <vnet/vnet.h>
+#include <vnet/ipsec/ipsec.h>
 #include <vpp/app/version.h>
 
 #include <dpdk/buffer.h>
@@ -84,14 +84,13 @@ typedef enum
 
 typedef struct
 {
-  union rte_cryptodev_session_ctx keys[CRYPTODEV_N_OP_TYPES];
+  union rte_cryptodev_session_ctx **keys;
 } cryptodev_key_t;
 
 typedef struct
 {
   u32 dev_id;
   u32 q_id;
-  struct rte_crypto_raw_dp_ctx *raw_dp_ctx_buffer;
   char *desc;
 } cryptodev_inst_t;
 
@@ -114,6 +113,7 @@ typedef struct
   u16 cryptodev_id;
   u16 cryptodev_q;
   u16 inflight;
+  union rte_cryptodev_session_ctx reset_sess; /* session data for reset ctx */
 } cryptodev_engine_thread_t;
 
 typedef struct
@@ -129,10 +129,10 @@ typedef struct
 
 cryptodev_main_t cryptodev_main;
 
-static int
+static_always_inline int
 prepare_aead_xform (struct rte_crypto_sym_xform *xform,
-		    cryptodev_op_type_t op_type,
-		    const vnet_crypto_key_t * key, u32 aad_len)
+		    cryptodev_op_type_t op_type, const vnet_crypto_key_t *key,
+		    u32 aad_len)
 {
   struct rte_crypto_aead_xform *aead_xform = &xform->aead;
   memset (xform, 0, sizeof (*xform));
@@ -157,10 +157,10 @@ prepare_aead_xform (struct rte_crypto_sym_xform *xform,
   return 0;
 }
 
-static int
+static_always_inline int
 prepare_linked_xform (struct rte_crypto_sym_xform *xforms,
 		      cryptodev_op_type_t op_type,
-		      const vnet_crypto_key_t * key)
+		      const vnet_crypto_key_t *key)
 {
   struct rte_crypto_sym_xform *xform_cipher, *xform_auth;
   vnet_crypto_key_t *key_cipher, *key_auth;
@@ -221,18 +221,57 @@ prepare_linked_xform (struct rte_crypto_sym_xform *xforms,
   return 0;
 }
 
-static int
-cryptodev_session_create (vnet_crypto_key_t * const key,
-			  struct rte_mempool *sess_priv_pool,
-			  cryptodev_key_t * session_pair, u32 aad_len)
+static_always_inline void
+cryptodev_session_del (struct rte_cryptodev_sym_session *sess)
+{
+  u32 n_devs, i;
+
+  if (sess == NULL)
+    return;
+
+  n_devs = rte_cryptodev_count ();
+
+  for (i = 0; i < n_devs; i++)
+    rte_cryptodev_sym_session_clear (i, sess);
+
+  rte_cryptodev_sym_session_free (sess);
+}
+
+static_always_inline int
+cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx,
+			  u32 aad_len)
 {
-  struct rte_crypto_sym_xform xforms_enc[2] = { {0} };
-  struct rte_crypto_sym_xform xforms_dec[2] = { {0} };
   cryptodev_main_t *cmt = &cryptodev_main;
+  cryptodev_numa_data_t *numa_data;
   cryptodev_inst_t *dev_inst;
-  struct rte_cryptodev *cdev;
+  vnet_crypto_key_t *key = vnet_crypto_get_key (idx);
+  struct rte_mempool *sess_pool, *sess_priv_pool;
+  cryptodev_key_t *ckey = vec_elt_at_index (cmt->keys, idx);
+  struct rte_crypto_sym_xform xforms_enc[2] = { { 0 } };
+  struct rte_crypto_sym_xform xforms_dec[2] = { { 0 } };
+  struct rte_cryptodev_sym_session *sessions[CRYPTODEV_N_OP_TYPES] = { 0 };
+  u32 numa_node = vm->numa_node;
   int ret;
-  uint8_t dev_id = 0;
+
+  numa_data = vec_elt_at_index (cmt->per_numa_data, numa_node);
+  sess_pool = numa_data->sess_pool;
+  sess_priv_pool = numa_data->sess_priv_pool;
+
+  sessions[CRYPTODEV_OP_TYPE_ENCRYPT] =
+    rte_cryptodev_sym_session_create (sess_pool);
+  if (!sessions[CRYPTODEV_OP_TYPE_ENCRYPT])
+    {
+      ret = -1;
+      goto clear_key;
+    }
+
+  sessions[CRYPTODEV_OP_TYPE_DECRYPT] =
+    rte_cryptodev_sym_session_create (sess_pool);
+  if (!sessions[CRYPTODEV_OP_TYPE_DECRYPT])
+    {
+      ret = -1;
+      goto clear_key;
+    }
 
   if (key->type == VNET_CRYPTO_KEY_TYPE_LINK)
     ret = prepare_linked_xform (xforms_enc, CRYPTODEV_OP_TYPE_ENCRYPT, key);
@@ -249,44 +288,39 @@ cryptodev_session_create (vnet_crypto_key_t * const key,
 
   vec_foreach (dev_inst, cmt->cryptodev_inst)
   {
-    dev_id = dev_inst->dev_id;
-    cdev = rte_cryptodev_pmd_get_dev (dev_id);
+    u32 dev_id = dev_inst->dev_id;
+    struct rte_cryptodev *cdev = rte_cryptodev_pmd_get_dev (dev_id);
 
     /* if the session is already configured for the driver type, avoid
        configuring it again to increase the session data's refcnt */
-    if (session_pair->keys[0].crypto_sess->sess_data[cdev->driver_id].data &&
-	session_pair->keys[1].crypto_sess->sess_data[cdev->driver_id].data)
+    if (sessions[CRYPTODEV_OP_TYPE_ENCRYPT]->sess_data[cdev->driver_id].data &&
+	sessions[CRYPTODEV_OP_TYPE_DECRYPT]->sess_data[cdev->driver_id].data)
       continue;
 
-    ret = rte_cryptodev_sym_session_init (dev_id,
-					  session_pair->keys[0].crypto_sess,
-					  xforms_enc, sess_priv_pool);
-    ret = rte_cryptodev_sym_session_init (dev_id,
-					  session_pair->keys[1].crypto_sess,
-					  xforms_dec, sess_priv_pool);
+    ret = rte_cryptodev_sym_session_init (
+      dev_id, sessions[CRYPTODEV_OP_TYPE_ENCRYPT], xforms_enc, sess_priv_pool);
+    ret = rte_cryptodev_sym_session_init (
+      dev_id, sessions[CRYPTODEV_OP_TYPE_DECRYPT], xforms_dec, sess_priv_pool);
     if (ret < 0)
       return ret;
   }
-  session_pair->keys[0].crypto_sess->opaque_data = aad_len;
-  session_pair->keys[1].crypto_sess->opaque_data = aad_len;
 
-  return 0;
-}
+  sessions[CRYPTODEV_OP_TYPE_ENCRYPT]->opaque_data = aad_len;
+  sessions[CRYPTODEV_OP_TYPE_DECRYPT]->opaque_data = aad_len;
 
-static void
-cryptodev_session_del (struct rte_cryptodev_sym_session *sess)
-{
-  u32 n_devs, i;
-
-  if (sess == NULL)
-    return;
-
-  n_devs = rte_cryptodev_count ();
+  CLIB_MEMORY_STORE_BARRIER ();
+  ckey->keys[numa_node][CRYPTODEV_OP_TYPE_ENCRYPT].crypto_sess =
+    sessions[CRYPTODEV_OP_TYPE_ENCRYPT];
+  ckey->keys[numa_node][CRYPTODEV_OP_TYPE_DECRYPT].crypto_sess =
+    sessions[CRYPTODEV_OP_TYPE_DECRYPT];
 
-  for (i = 0; i < n_devs; i++)
-    rte_cryptodev_sym_session_clear (i, sess);
-
-  rte_cryptodev_sym_session_free (sess);
+clear_key:
+  if (ret != 0)
+    {
+      cryptodev_session_del (sessions[CRYPTODEV_OP_TYPE_ENCRYPT]);
+      cryptodev_session_del (sessions[CRYPTODEV_OP_TYPE_DECRYPT]);
+    }
+  return ret;
 }
 
 static int
@@ -312,72 +346,44 @@ cryptodev_sess_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop,
 			vnet_crypto_key_index_t idx, u32 aad_len)
 {
   cryptodev_main_t *cmt = &cryptodev_main;
-  cryptodev_numa_data_t *numa_data;
   vnet_crypto_key_t *key = vnet_crypto_get_key (idx);
-  struct rte_mempool *sess_pool, *sess_priv_pool;
   cryptodev_key_t *ckey = 0;
-  int ret = 0;
+  u32 i;
+
+  vec_validate (cmt->keys, idx);
+  ckey = vec_elt_at_index (cmt->keys, idx);
 
-  if (kop == VNET_CRYPTO_KEY_OP_DEL)
+  if (kop == VNET_CRYPTO_KEY_OP_DEL || kop == VNET_CRYPTO_KEY_OP_MODIFY)
     {
       if (idx >= vec_len (cmt->keys))
 	return;
 
-      ckey = pool_elt_at_index (cmt->keys, idx);
-      cryptodev_session_del (ckey->keys[0].crypto_sess);
-      cryptodev_session_del (ckey->keys[1].crypto_sess);
-      ckey->keys[0].crypto_sess = 0;
-      ckey->keys[1].crypto_sess = 0;
-      pool_put (cmt->keys, ckey);
+      vec_foreach_index (i, cmt->per_numa_data)
+	{
+	  if (ckey->keys[i][CRYPTODEV_OP_TYPE_ENCRYPT].crypto_sess)
+	    {
+	      cryptodev_session_del (
+		ckey->keys[i][CRYPTODEV_OP_TYPE_ENCRYPT].crypto_sess);
+	      cryptodev_session_del (
+		ckey->keys[i][CRYPTODEV_OP_TYPE_DECRYPT].crypto_sess);
+
+	      CLIB_MEMORY_STORE_BARRIER ();
+	      ckey->keys[i][CRYPTODEV_OP_TYPE_ENCRYPT].crypto_sess = 0;
+	      ckey->keys[i][CRYPTODEV_OP_TYPE_DECRYPT].crypto_sess = 0;
+	    }
+	}
       return;
     }
-  else if (kop == VNET_CRYPTO_KEY_OP_MODIFY)
-    {
-      if (idx >= vec_len (cmt->keys))
-	return;
-
-      ckey = pool_elt_at_index (cmt->keys, idx);
 
-      cryptodev_session_del (ckey->keys[0].crypto_sess);
-      cryptodev_session_del (ckey->keys[1].crypto_sess);
-      ckey->keys[0].crypto_sess = 0;
-      ckey->keys[1].crypto_sess = 0;
-    }
-  else				/* create key */
-    pool_get_zero (cmt->keys, ckey);
+  /* create key */
 
   /* do not create session for unsupported alg */
   if (cryptodev_check_supported_vnet_alg (key))
     return;
 
-  numa_data = vec_elt_at_index (cmt->per_numa_data, vm->numa_node);
-  sess_pool = numa_data->sess_pool;
-  sess_priv_pool = numa_data->sess_priv_pool;
-
-  ckey->keys[0].crypto_sess = rte_cryptodev_sym_session_create (sess_pool);
-  if (!ckey->keys[0].crypto_sess)
-    {
-      ret = -1;
-      goto clear_key;
-    }
-
-  ckey->keys[1].crypto_sess = rte_cryptodev_sym_session_create (sess_pool);
-  if (!ckey->keys[1].crypto_sess)
-    {
-      ret = -1;
-      goto clear_key;
-    }
-
-  ret = cryptodev_session_create (key, sess_priv_pool, ckey, aad_len);
-
-clear_key:
-  if (ret != 0)
-    {
-      cryptodev_session_del (ckey->keys[0].crypto_sess);
-      cryptodev_session_del (ckey->keys[1].crypto_sess);
-      memset (ckey, 0, sizeof (*ckey));
-      pool_put (cmt->keys, ckey);
-    }
+  vec_validate (ckey->keys, vec_len (cmt->per_numa_data) - 1);
+  vec_foreach_index (i, ckey->keys)
+    vec_validate (ckey->keys[i], CRYPTODEV_N_OP_TYPES - 1);
 }
 
 /*static*/ void
@@ -449,13 +455,12 @@ compute_ofs_linked_alg (vnet_crypto_async_frame_elt_t * fe, i16 * min_ofs,
   return ofs.raw;
 }
 
-/* Reset cryptodev dp context to previous queue pair state */
 static_always_inline void
-cryptodev_reset_ctx (u16 cdev_id, u16 qid, struct rte_crypto_raw_dp_ctx *ctx)
+cryptodev_reset_ctx (cryptodev_engine_thread_t *cet)
 {
-  union rte_cryptodev_session_ctx session_ctx = {.crypto_sess = NULL };
-
-  rte_cryptodev_configure_raw_dp_ctx (cdev_id, qid, ctx, ~0, session_ctx, 0);
+  rte_cryptodev_configure_raw_dp_ctx (cet->cryptodev_id, cet->cryptodev_q,
+				      cet->ctx, RTE_CRYPTO_OP_WITH_SESSION,
+				      cet->reset_sess, 0);
 }
 
 static_always_inline int
@@ -470,7 +475,6 @@ cryptodev_frame_linked_algs_enqueue (vlib_main_t * vm,
   struct rte_crypto_va_iova_ptr iv_vec, digest_vec;
   vlib_buffer_t **b;
   u32 n_elts;
-  cryptodev_key_t *key;
   u32 last_key_index = ~0;
   i16 min_ofs;
   u32 max_end;
@@ -506,20 +510,24 @@ cryptodev_frame_linked_algs_enqueue (vlib_main_t * vm,
 
       if (PREDICT_FALSE (last_key_index != fe->key_index))
 	{
-	  key = pool_elt_at_index (cmt->keys, fe->key_index);
-	  last_key_index = fe->key_index;
+	  cryptodev_key_t *key = vec_elt_at_index (cmt->keys, fe->key_index);
 
-	  if (PREDICT_FALSE
-	      (rte_cryptodev_configure_raw_dp_ctx
-	       (cet->cryptodev_id, cet->cryptodev_q, cet->ctx,
-		RTE_CRYPTO_OP_WITH_SESSION, key->keys[op_type], 1) < 0))
+	  if (PREDICT_FALSE (key->keys[vm->numa_node][op_type].crypto_sess ==
+			     0))
 	    {
-	      cryptodev_mark_frame_err_status (frame,
-					       VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
-	      cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q,
-				   cet->ctx);
-	      return -1;
+	      status = cryptodev_session_create (vm, fe->key_index, 0);
+	      if (PREDICT_FALSE (status < 0))
+		goto error_exit;
 	    }
+
+	  status = rte_cryptodev_configure_raw_dp_ctx (
+	    cet->cryptodev_id, cet->cryptodev_q, cet->ctx,
+	    RTE_CRYPTO_OP_WITH_SESSION, key->keys[vm->numa_node][op_type],
+	    /*is_update */ 1);
+	  if (PREDICT_FALSE (status < 0))
+	    goto error_exit;
+
+	  last_key_index = fe->key_index;
 	}
 
       cofs.raw = compute_ofs_linked_alg (fe, &min_ofs, &max_end);
@@ -547,27 +555,15 @@ cryptodev_frame_linked_algs_enqueue (vlib_main_t * vm,
       if (PREDICT_FALSE (fe->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS))
 	{
 	  vec[0].len = b[0]->current_data + b[0]->current_length - min_ofs;
-	  if (cryptodev_frame_build_sgl
-	      (vm, cmt->iova_mode, vec, &n_seg, b[0],
-	       max_end - min_ofs - vec->len) < 0)
-	    {
-	      cryptodev_mark_frame_err_status (frame,
-					       VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
-	      cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q,
-				   cet->ctx);
-	      return -1;
-	    }
+	  if (cryptodev_frame_build_sgl (vm, cmt->iova_mode, vec, &n_seg, b[0],
+					 max_end - min_ofs - vec->len) < 0)
+	    goto error_exit;
 	}
 
       status = rte_cryptodev_raw_enqueue (cet->ctx, vec, n_seg, cofs, &iv_vec,
 					  &digest_vec, 0, (void *) frame);
-      if (status < 0)
-	{
-	  cryptodev_mark_frame_err_status (frame,
-					   VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
-	  cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q, cet->ctx);
-	  return -1;
-	}
+      if (PREDICT_FALSE (status < 0))
+	goto error_exit;
 
       b++;
       fe++;
@@ -577,12 +573,18 @@ cryptodev_frame_linked_algs_enqueue (vlib_main_t * vm,
   status = rte_cryptodev_raw_enqueue_done (cet->ctx, frame->n_elts);
   if (PREDICT_FALSE (status < 0))
     {
-      cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q, cet->ctx);
+      cryptodev_reset_ctx (cet);
       return -1;
     }
 
   cet->inflight += frame->n_elts;
   return 0;
+
+error_exit:
+  cryptodev_mark_frame_err_status (frame,
+				   VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+  cryptodev_reset_ctx (cet);
+  return -1;
 }
 
 static_always_inline int
@@ -595,15 +597,12 @@ cryptodev_frame_gcm_enqueue (vlib_main_t * vm,
   vnet_crypto_async_frame_elt_t *fe;
   vlib_buffer_t **b;
   u32 n_elts;
-  cryptodev_key_t *key;
-  u32 last_key_index = ~0;
   union rte_crypto_sym_ofs cofs;
   struct rte_crypto_vec *vec;
   struct rte_crypto_va_iova_ptr iv_vec, digest_vec, aad_vec;
-  u8 sess_aad_len = 0;
+  u32 last_key_index = ~0;
   int status;
 
-
   n_elts = frame->n_elts;
 
   if (PREDICT_FALSE (CRYPTODEV_MAX_INFLIGHT - cet->inflight < n_elts))
@@ -631,28 +630,36 @@ cryptodev_frame_gcm_enqueue (vlib_main_t * vm,
 	  vlib_prefetch_buffer_header (b[1], LOAD);
 	}
 
-      if (last_key_index != fe->key_index)
+      if (PREDICT_FALSE (last_key_index != fe->key_index))
 	{
-	  key = pool_elt_at_index (cmt->keys, fe->key_index);
-	  sess_aad_len = (u8) key->keys[op_type].crypto_sess->opaque_data;
-	  if (PREDICT_FALSE (sess_aad_len != aad_len))
+	  cryptodev_key_t *key = vec_elt_at_index (cmt->keys, fe->key_index);
+
+	  if (PREDICT_FALSE (key->keys[vm->numa_node][op_type].crypto_sess ==
+			     0))
 	    {
-	      cryptodev_sess_handler (vm, VNET_CRYPTO_KEY_OP_MODIFY,
-				      fe->key_index, aad_len);
+	      status = cryptodev_session_create (vm, fe->key_index, aad_len);
+	      if (PREDICT_FALSE (status < 0))
+		goto error_exit;
 	    }
-	  last_key_index = fe->key_index;
 
-	  if (PREDICT_FALSE
-	      (rte_cryptodev_configure_raw_dp_ctx
-	       (cet->cryptodev_id, cet->cryptodev_q, cet->ctx,
-		RTE_CRYPTO_OP_WITH_SESSION, key->keys[op_type], 1) < 0))
+	  if (PREDICT_FALSE ((u8) key->keys[vm->numa_node][op_type]
+			       .crypto_sess->opaque_data != aad_len))
 	    {
-	      cryptodev_mark_frame_err_status (frame,
-					       VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
-	      cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q,
-				   cet->ctx);
-	      return -1;
+	      cryptodev_sess_handler (vm, VNET_CRYPTO_KEY_OP_DEL,
+				      fe->key_index, aad_len);
+	      status = cryptodev_session_create (vm, fe->key_index, aad_len);
+	      if (PREDICT_FALSE (status < 0))
+		goto error_exit;
 	    }
+
+	  status = rte_cryptodev_configure_raw_dp_ctx (
+	    cet->cryptodev_id, cet->cryptodev_q, cet->ctx,
+	    RTE_CRYPTO_OP_WITH_SESSION, key->keys[vm->numa_node][op_type],
+	    /*is_update */ 1);
+	  if (PREDICT_FALSE (status < 0))
+	    goto error_exit;
+
+	  last_key_index = fe->key_index;
 	}
 
       if (cmt->iova_mode == RTE_IOVA_VA)
@@ -692,31 +699,21 @@ cryptodev_frame_gcm_enqueue (vlib_main_t * vm,
 
       if (PREDICT_FALSE (fe->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS))
 	{
-	  vec[0].len = b[0]->current_data +
-	    b[0]->current_length - fe->crypto_start_offset;
-	  if (cryptodev_frame_build_sgl
-	      (vm, cmt->iova_mode, vec, &n_seg, b[0],
-	       fe->crypto_total_length - vec[0].len) < 0)
-	    {
-	      cryptodev_mark_frame_err_status (frame,
-					       VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
-	      cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q,
-				   cet->ctx);
-	      return -1;
-	    }
+	  vec[0].len = b[0]->current_data + b[0]->current_length -
+		       fe->crypto_start_offset;
+	  status =
+	    cryptodev_frame_build_sgl (vm, cmt->iova_mode, vec, &n_seg, b[0],
+				       fe->crypto_total_length - vec[0].len);
+	  if (status < 0)
+	    goto error_exit;
 	}
 
       status =
-	rte_cryptodev_raw_enqueue (cet->ctx, vec, n_seg, cofs,
-				   &iv_vec, &digest_vec, &aad_vec,
-				   (void *) frame);
+	rte_cryptodev_raw_enqueue (cet->ctx, vec, n_seg, cofs, &iv_vec,
+				   &digest_vec, &aad_vec, (void *) frame);
       if (PREDICT_FALSE (status < 0))
-	{
-	  cryptodev_mark_frame_err_status (frame,
-					   VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
-	  cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q, cet->ctx);
-	  return -1;
-	}
+	goto error_exit;
+
       fe++;
       b++;
       n_elts--;
@@ -724,14 +721,17 @@ cryptodev_frame_gcm_enqueue (vlib_main_t * vm,
 
   status = rte_cryptodev_raw_enqueue_done (cet->ctx, frame->n_elts);
   if (PREDICT_FALSE (status < 0))
-    {
-      cryptodev_reset_ctx (cet->cryptodev_id, cet->cryptodev_q, cet->ctx);
-      return -1;
-    }
+    goto error_exit;
 
   cet->inflight += frame->n_elts;
 
   return 0;
+
+error_exit:
+  cryptodev_mark_frame_err_status (frame,
+				   VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+  cryptodev_reset_ctx (cet);
+  return -1;
 }
 
 static u32
@@ -999,7 +999,7 @@ cryptodev_assign_resource (cryptodev_engine_thread_t * cet,
       cinst = vec_elt_at_index (cmt->cryptodev_inst, idx);
       cet->cryptodev_id = cinst->dev_id;
       cet->cryptodev_q = cinst->q_id;
-      cet->ctx = cinst->raw_dp_ctx_buffer;
+      cryptodev_reset_ctx (cet);
       clib_spinlock_unlock (&cmt->tlock);
       break;
     case CRYPTODEV_RESOURCE_ASSIGN_UPDATE:
@@ -1024,7 +1024,7 @@ cryptodev_assign_resource (cryptodev_engine_thread_t * cet,
       cinst = cmt->cryptodev_inst + cryptodev_inst_index;
       cet->cryptodev_id = cinst->dev_id;
       cet->cryptodev_q = cinst->q_id;
-      cet->ctx = cinst->raw_dp_ctx_buffer;
+      cryptodev_reset_ctx (cet);
       clib_spinlock_unlock (&cmt->tlock);
       break;
     default:
@@ -1214,12 +1214,6 @@ cryptodev_count_queue (u32 numa)
   for (i = 0; i < n_cryptodev; i++)
     {
       rte_cryptodev_info_get (i, &info);
-      if (rte_cryptodev_socket_id (i) != numa)
-	{
-	  clib_warning ("DPDK crypto resource %s is in different numa node "
-	      "as %u, ignored", info.device->name, numa);
-	  continue;
-	}
       q_count += info.max_nb_queue_pairs;
     }
 
@@ -1229,16 +1223,12 @@ cryptodev_count_queue (u32 numa)
 static int
 cryptodev_configure (vlib_main_t *vm, u32 cryptodev_id)
 {
+  struct rte_cryptodev_config cfg;
   struct rte_cryptodev_info info;
-  struct rte_cryptodev *cdev;
   cryptodev_main_t *cmt = &cryptodev_main;
-  cryptodev_numa_data_t *numa_data = vec_elt_at_index (cmt->per_numa_data,
-						       vm->numa_node);
-  u32 dp_size = 0;
   u32 i;
   int ret;
 
-  cdev = rte_cryptodev_pmd_get_dev (cryptodev_id);
   rte_cryptodev_info_get (cryptodev_id, &info);
 
   if (!(info.feature_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP))
@@ -1246,46 +1236,41 @@ cryptodev_configure (vlib_main_t *vm, u32 cryptodev_id)
 
   ret = check_cryptodev_alg_support (cryptodev_id);
   if (ret != 0)
-    return ret;
+    {
+      clib_warning (
+	"Cryptodev: device %u does not support required algorithms",
+	cryptodev_id);
+      return ret;
+    }
 
+  cfg.socket_id = info.device->numa_node;
+  cfg.nb_queue_pairs = info.max_nb_queue_pairs;
 
+  rte_cryptodev_configure (cryptodev_id, &cfg);
 
-  /** If the device is already started, we reuse it, otherwise configure
-   *  both the device and queue pair.
-   **/
-  if (!cdev->data->dev_started)
+  for (i = 0; i < info.max_nb_queue_pairs; i++)
     {
-      struct rte_cryptodev_config cfg;
+      struct rte_cryptodev_qp_conf qp_cfg;
 
-      cfg.socket_id = vm->numa_node;
-      cfg.nb_queue_pairs = info.max_nb_queue_pairs;
+      qp_cfg.mp_session = 0;
+      qp_cfg.mp_session_private = 0;
+      qp_cfg.nb_descriptors = CRYPTODEV_NB_CRYPTO_OPS;
 
-      rte_cryptodev_configure (cryptodev_id, &cfg);
-
-      for (i = 0; i < info.max_nb_queue_pairs; i++)
+      ret = rte_cryptodev_queue_pair_setup (cryptodev_id, i, &qp_cfg,
+					    info.device->numa_node);
+      if (ret)
 	{
-	  struct rte_cryptodev_qp_conf qp_cfg;
-
-	  qp_cfg.mp_session = numa_data->sess_pool;
-	  qp_cfg.mp_session_private = numa_data->sess_priv_pool;
-	  qp_cfg.nb_descriptors = CRYPTODEV_NB_CRYPTO_OPS;
-
-	  ret = rte_cryptodev_queue_pair_setup (cryptodev_id, i, &qp_cfg,
-						vm->numa_node);
-	  if (ret)
-	    break;
+	  clib_warning ("Cryptodev: Configure device %u queue %u failed %d",
+			cryptodev_id, i, ret);
+	  break;
 	}
-      if (i != info.max_nb_queue_pairs)
-	return -1;
-
-      /* start the device */
-      rte_cryptodev_start (i);
     }
 
-  ret = rte_cryptodev_get_raw_dp_ctx_size (cryptodev_id);
-  if (ret < 0)
+  if (i != info.max_nb_queue_pairs)
     return -1;
-  dp_size = ret;
+
+  /* start the device */
+  rte_cryptodev_start (cryptodev_id);
 
   for (i = 0; i < info.max_nb_queue_pairs; i++)
     {
@@ -1294,9 +1279,6 @@ cryptodev_configure (vlib_main_t *vm, u32 cryptodev_id)
       cdev_inst->desc = vec_new (char, strlen (info.device->name) + 10);
       cdev_inst->dev_id = cryptodev_id;
       cdev_inst->q_id = i;
-      vec_validate_aligned (cdev_inst->raw_dp_ctx_buffer, dp_size, 8);
-      cryptodev_reset_ctx (cdev_inst->dev_id, cdev_inst->q_id,
-			   cdev_inst->raw_dp_ctx_buffer);
 
       snprintf (cdev_inst->desc, strlen (info.device->name) + 9,
 		"%s_q%u", info.device->name, i);
@@ -1345,22 +1327,24 @@ cryptodev_probe (vlib_main_t *vm, u32 n_workers)
   return 0;
 }
 
-static int
-cryptodev_get_session_sz (vlib_main_t *vm, u32 n_workers)
+static void
+cryptodev_get_max_sz (u32 *max_sess_sz, u32 *max_dp_sz)
 {
-  u32 sess_data_sz = 0, i;
-
-  if (rte_cryptodev_count () == 0)
-    return -1;
+  cryptodev_main_t *cmt = &cryptodev_main;
+  cryptodev_inst_t *cinst;
+  u32 max_sess = 0, max_dp = 0;
 
-  for (i = 0; i < rte_cryptodev_count (); i++)
+  vec_foreach (cinst, cmt->cryptodev_inst)
     {
-      u32 dev_sess_sz = rte_cryptodev_sym_get_private_session_size (i);
+      u32 sess_sz = rte_cryptodev_sym_get_private_session_size (cinst->dev_id);
+      u32 dp_sz = rte_cryptodev_get_raw_dp_ctx_size (cinst->dev_id);
 
-      sess_data_sz = dev_sess_sz > sess_data_sz ? dev_sess_sz : sess_data_sz;
+      max_sess = clib_max (sess_sz, max_sess);
+      max_dp = clib_max (dp_sz, max_dp);
     }
 
-  return sess_data_sz;
+  *max_sess_sz = max_sess;
+  *max_dp_sz = max_dp;
 }
 
 static void
@@ -1384,7 +1368,74 @@ dpdk_disable_cryptodev_engine (vlib_main_t * vm)
 	rte_free (ptd->aad_buf);
       if (ptd->cached_frame)
 	rte_ring_free (ptd->cached_frame);
+      if (ptd->reset_sess.crypto_sess)
+	{
+	  struct rte_mempool *mp =
+	    rte_mempool_from_obj ((void *) ptd->reset_sess.crypto_sess);
+
+	  rte_mempool_free (mp);
+	  ptd->reset_sess.crypto_sess = 0;
+	}
+    }
+}
+
+static clib_error_t *
+create_reset_sess (cryptodev_engine_thread_t *ptd, u32 lcore, u32 numa,
+		   u32 sess_sz)
+{
+  struct rte_crypto_sym_xform xform = { 0 };
+  struct rte_crypto_aead_xform *aead_xform = &xform.aead;
+  struct rte_cryptodev_sym_session *sess;
+  struct rte_mempool *mp = 0;
+  u8 key[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+  u8 *name = 0;
+  clib_error_t *error = 0;
+
+  /* create session pool for the numa node */
+  name = format (0, "vcryptodev_s_reset_%u_%u", numa, lcore);
+  mp = rte_cryptodev_sym_session_pool_create ((char *) name, 2, sess_sz, 0, 0,
+					      numa);
+  if (!mp)
+    {
+      error = clib_error_return (0, "Not enough memory for mp %s", name);
+      goto error_exit;
+    }
+  vec_free (name);
+
+  xform.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+  aead_xform->algo = RTE_CRYPTO_AEAD_AES_GCM;
+  aead_xform->op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
+  aead_xform->aad_length = 8;
+  aead_xform->digest_length = 16;
+  aead_xform->iv.offset = 0;
+  aead_xform->iv.length = 12;
+  aead_xform->key.data = key;
+  aead_xform->key.length = 16;
+
+  sess = rte_cryptodev_sym_session_create (mp);
+  if (!sess)
+    {
+      error = clib_error_return (0, "failed to create session");
+      goto error_exit;
+    }
+
+  if (rte_cryptodev_sym_session_init (ptd->cryptodev_id, sess, &xform, mp) < 0)
+    {
+      error = clib_error_return (0, "failed to create session private");
+      goto error_exit;
     }
+
+  ptd->reset_sess.crypto_sess = sess;
+
+  return 0;
+
+error_exit:
+  if (mp)
+    rte_mempool_free (mp);
+  if (name)
+    vec_free (name);
+
+  return error;
 }
 
 clib_error_t *
@@ -1398,7 +1449,7 @@ dpdk_cryptodev_init (vlib_main_t * vm)
   u32 skip_master = vlib_num_workers () > 0;
   u32 n_workers = tm->n_vlib_mains - skip_master;
   u32 numa = vm->numa_node;
-  i32 sess_sz;
+  u32 sess_sz, dp_sz;
   u32 eidx;
   u32 i;
   u8 *name = 0;
@@ -1406,44 +1457,7 @@ dpdk_cryptodev_init (vlib_main_t * vm)
 
   cmt->iova_mode = rte_eal_iova_mode ();
 
-  sess_sz = cryptodev_get_session_sz(vm, n_workers);
-  if (sess_sz < 0)
-    {
-      error = clib_error_return (0, "Not enough cryptodevs");
-      return error;
-    }
-
   vec_validate (cmt->per_numa_data, vm->numa_node);
-  numa_data = vec_elt_at_index (cmt->per_numa_data, numa);
-
-  /* create session pool for the numa node */
-  name = format (0, "vcryptodev_sess_pool_%u%c", numa, 0);
-  mp = rte_cryptodev_sym_session_pool_create ((char *) name,
-					      CRYPTODEV_NB_SESSION,
-					      0, 0, 0, numa);
-  if (!mp)
-    {
-      error = clib_error_return (0, "Not enough memory for mp %s", name);
-      goto err_handling;
-    }
-  vec_free (name);
-
-  numa_data->sess_pool = mp;
-
-  /* create session private pool for the numa node */
-  name = format (0, "cryptodev_sess_pool_%u%c", numa, 0);
-  mp = rte_mempool_create ((char *) name, CRYPTODEV_NB_SESSION, sess_sz, 0,
-			   0, NULL, NULL, NULL, NULL, numa, 0);
-  if (!mp)
-    {
-      error = clib_error_return (0, "Not enough memory for mp %s", name);
-      vec_free (name);
-      goto err_handling;
-    }
-
-  vec_free (name);
-
-  numa_data->sess_priv_pool = mp;
 
   /* probe all cryptodev devices and get queue info */
   if (cryptodev_probe (vm, n_workers) < 0)
@@ -1452,6 +1466,8 @@ dpdk_cryptodev_init (vlib_main_t * vm)
       goto err_handling;
     }
 
+  cryptodev_get_max_sz (&sess_sz, &dp_sz);
+
   clib_bitmap_vec_validate (cmt->active_cdev_inst_mask, tm->n_vlib_mains);
   clib_spinlock_init (&cmt->tlock);
 
@@ -1460,11 +1476,13 @@ dpdk_cryptodev_init (vlib_main_t * vm)
   for (i = skip_master; i < tm->n_vlib_mains; i++)
     {
       ptd = cmt->per_thread_data + i;
-      cryptodev_assign_resource (ptd, 0, CRYPTODEV_RESOURCE_ASSIGN_AUTO);
+      numa = vlib_mains[i]->numa_node;
+
       ptd->aad_buf = rte_zmalloc_socket (0, CRYPTODEV_NB_CRYPTO_OPS *
 					 CRYPTODEV_MAX_AAD_SIZE,
 					 CLIB_CACHE_LINE_BYTES,
 					 numa);
+
       if (ptd->aad_buf == 0)
 	{
 	  error = clib_error_return (0, "Failed to alloc aad buf");
@@ -1473,6 +1491,13 @@ dpdk_cryptodev_init (vlib_main_t * vm)
 
       ptd->aad_phy_addr = rte_malloc_virt2iova (ptd->aad_buf);
 
+      ptd->ctx = rte_zmalloc_socket (0, dp_sz, CLIB_CACHE_LINE_BYTES, numa);
+      if (!ptd->ctx)
+	{
+	  error = clib_error_return (0, "Failed to alloc raw dp ctx");
+	  goto err_handling;
+	}
+
       name = format (0, "cache_frame_ring_%u%u", numa, i);
       ptd->cached_frame = rte_ring_create ((char *)name,
 					   CRYPTODEV_DEQ_CACHE_SZ, numa,
@@ -1480,15 +1505,58 @@ dpdk_cryptodev_init (vlib_main_t * vm)
 
       if (ptd->cached_frame == 0)
 	{
-	  error = clib_error_return (0, "Failed to frame ring");
+	  error = clib_error_return (0, "Failed to alloc frame ring");
 	  goto err_handling;
 	}
       vec_free (name);
+
+      vec_validate (cmt->per_numa_data, numa);
+      numa_data = vec_elt_at_index (cmt->per_numa_data, numa);
+
+      if (!numa_data->sess_pool)
+	{
+	  /* create session pool for the numa node */
+	  name = format (0, "vcryptodev_sess_pool_%u%c", numa, 0);
+	  mp = rte_cryptodev_sym_session_pool_create (
+	    (char *) name, CRYPTODEV_NB_SESSION, 0, 0, 0, numa);
+	  if (!mp)
+	    {
+	      error =
+		clib_error_return (0, "Not enough memory for mp %s", name);
+	      goto err_handling;
+	    }
+	  vec_free (name);
+
+	  numa_data->sess_pool = mp;
+
+	  /* create session private pool for the numa node */
+	  name = format (0, "cryptodev_sess_pool_%u%c", numa, 0);
+	  mp =
+	    rte_mempool_create ((char *) name, CRYPTODEV_NB_SESSION, sess_sz,
+				0, 0, NULL, NULL, NULL, NULL, numa, 0);
+	  if (!mp)
+	    {
+	      error =
+		clib_error_return (0, "Not enough memory for mp %s", name);
+	      vec_free (name);
+	      goto err_handling;
+	    }
+
+	  vec_free (name);
+
+	  numa_data->sess_priv_pool = mp;
+	}
+
+      error = create_reset_sess (ptd, i, numa, sess_sz);
+      if (error)
+	goto err_handling;
+
+      cryptodev_assign_resource (ptd, 0, CRYPTODEV_RESOURCE_ASSIGN_AUTO);
     }
 
   /* register handler */
-  eidx = vnet_crypto_register_engine (vm, "dpdk_cryptodev", 79,
-                                      "DPDK Cryptodev Engine");
+  eidx = vnet_crypto_register_engine (vm, "dpdk_cryptodev", 100,
+				      "DPDK Cryptodev Engine");
 
 #define _(a, b, c, d, e, f) \
   vnet_crypto_register_async_handler \
@@ -1518,6 +1586,12 @@ dpdk_cryptodev_init (vlib_main_t * vm)
 
   vnet_crypto_register_key_handler (vm, eidx, cryptodev_key_handler);
 
+  /* this engine is only enabled when cryptodev device(s) are presented in
+   * startup.conf. Assume it is wanted to be used, turn on async mode here.
+   */
+  vnet_crypto_request_async_mode (1);
+  ipsec_set_async_mode (1);
+
   return 0;
 
 err_handling: