2 files changed, 143 insertions, 96 deletions

diff --git a/src/plugins/dpdk/cryptodev/cryptodev.c b/src/plugins/dpdk/cryptodev/cryptodev.c
index d52fa407ec5..ab07e60519a 100644
--- a/src/plugins/dpdk/cryptodev/cryptodev.c
+++ b/src/plugins/dpdk/cryptodev/cryptodev.c
@@ -310,7 +310,7 @@ cryptodev_sess_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop,
   if (cryptodev_check_supported_vnet_alg (key) == 0)
     return;
 
-  vec_validate (ckey->keys, idx);
+  vec_validate (ckey->keys, vec_len (cmt->per_numa_data) - 1);
   vec_foreach_index (i, ckey->keys)
     vec_validate (ckey->keys[i], CRYPTODEV_N_OP_TYPES - 1);
 }
@@ -322,6 +322,50 @@ cryptodev_key_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop,
   cryptodev_sess_handler (vm, kop, idx, 8);
 }
 
+clib_error_t *
+allocate_session_pools (u32 numa_node,
+			cryptodev_session_pool_t *sess_pools_elt, u32 len)
+{
+  cryptodev_main_t *cmt = &cryptodev_main;
+  u8 *name;
+  clib_error_t *error = NULL;
+
+  name = format (0, "vcryptodev_sess_pool_%u_%c", numa_node, len);
+  sess_pools_elt->sess_pool = rte_cryptodev_sym_session_pool_create (
+    (char *) name, CRYPTODEV_NB_SESSION, 0, 0, 0, numa_node);
+
+  if (!sess_pools_elt->sess_pool)
+    {
+      error = clib_error_return (0, "Not enough memory for mp %s", name);
+      goto clear_mempools;
+    }
+  vec_free (name);
+
+  name = format (0, "cryptodev_sess_pool_%u_%c", numa_node, len);
+  sess_pools_elt->sess_priv_pool = rte_mempool_create (
+    (char *) name, CRYPTODEV_NB_SESSION * (cmt->drivers_cnt), cmt->sess_sz, 0,
+    0, NULL, NULL, NULL, NULL, numa_node, 0);
+
+  if (!sess_pools_elt->sess_priv_pool)
+    {
+      error = clib_error_return (0, "Not enough memory for mp %s", name);
+      goto clear_mempools;
+    }
+  vec_free (name);
+
+clear_mempools:
+  if (error)
+    {
+      vec_free (name);
+      if (sess_pools_elt->sess_pool)
+	rte_mempool_free (sess_pools_elt->sess_pool);
+      if (sess_pools_elt->sess_priv_pool)
+	rte_mempool_free (sess_pools_elt->sess_priv_pool);
+      return error;
+    }
+  return 0;
+}
+
 int
 cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx,
 			  u32 aad_len)
@@ -331,40 +375,69 @@ cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx,
   cryptodev_inst_t *dev_inst;
   vnet_crypto_key_t *key = vnet_crypto_get_key (idx);
   struct rte_mempool *sess_pool, *sess_priv_pool;
+  cryptodev_session_pool_t *sess_pools_elt;
   cryptodev_key_t *ckey = vec_elt_at_index (cmt->keys, idx);
   struct rte_crypto_sym_xform xforms_enc[2] = { { 0 } };
   struct rte_crypto_sym_xform xforms_dec[2] = { { 0 } };
   struct rte_cryptodev_sym_session *sessions[CRYPTODEV_N_OP_TYPES] = { 0 };
   u32 numa_node = vm->numa_node;
-  int ret;
+  clib_error_t *error;
+  int ret = 0;
+  u8 found = 0;
 
   numa_data = vec_elt_at_index (cmt->per_numa_data, numa_node);
-  sess_pool = numa_data->sess_pool;
-  sess_priv_pool = numa_data->sess_priv_pool;
 
-  sessions[CRYPTODEV_OP_TYPE_ENCRYPT] =
-    rte_cryptodev_sym_session_create (sess_pool);
-  if (!sessions[CRYPTODEV_OP_TYPE_ENCRYPT])
+  clib_spinlock_lock (&cmt->tlock);
+  vec_foreach (sess_pools_elt, numa_data->sess_pools)
     {
-      ret = -1;
-      goto clear_key;
+      if (sess_pools_elt->sess_pool == NULL)
+	{
+	  error = allocate_session_pools (numa_node, sess_pools_elt,
+					  vec_len (numa_data->sess_pools) - 1);
+	  if (error)
+	    {
+	      ret = -1;
+	      goto clear_key;
+	    }
+	}
+      if (rte_mempool_avail_count (sess_pools_elt->sess_pool) >= 2)
+	{
+	  found = 1;
+	  break;
+	}
     }
 
-  sessions[CRYPTODEV_OP_TYPE_DECRYPT] =
-    rte_cryptodev_sym_session_create (sess_pool);
-  if (!sessions[CRYPTODEV_OP_TYPE_DECRYPT])
+  if (found == 0)
     {
-      ret = -1;
-      goto clear_key;
+      vec_add2 (numa_data->sess_pools, sess_pools_elt, 1);
+      error = allocate_session_pools (numa_node, sess_pools_elt,
+				      vec_len (numa_data->sess_pools) - 1);
+      if (error)
+	{
+	  ret = -1;
+	  goto clear_key;
+	}
     }
 
+  sess_pool = sess_pools_elt->sess_pool;
+  sess_priv_pool = sess_pools_elt->sess_priv_pool;
+
+  sessions[CRYPTODEV_OP_TYPE_ENCRYPT] =
+    rte_cryptodev_sym_session_create (sess_pool);
+
+  sessions[CRYPTODEV_OP_TYPE_DECRYPT] =
+    rte_cryptodev_sym_session_create (sess_pool);
+
   if (key->type == VNET_CRYPTO_KEY_TYPE_LINK)
     ret = prepare_linked_xform (xforms_enc, CRYPTODEV_OP_TYPE_ENCRYPT, key);
   else
     ret =
       prepare_aead_xform (xforms_enc, CRYPTODEV_OP_TYPE_ENCRYPT, key, aad_len);
   if (ret)
-    return 0;
+    {
+      ret = -1;
+      goto clear_key;
+    }
 
   if (key->type == VNET_CRYPTO_KEY_TYPE_LINK)
     prepare_linked_xform (xforms_dec, CRYPTODEV_OP_TYPE_DECRYPT, key);
@@ -390,7 +463,7 @@ cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx,
 	dev_id, sessions[CRYPTODEV_OP_TYPE_DECRYPT], xforms_dec,
 	sess_priv_pool);
       if (ret < 0)
-	return ret;
+	goto clear_key;
     }
 
   sessions[CRYPTODEV_OP_TYPE_ENCRYPT]->opaque_data = aad_len;
@@ -408,6 +481,7 @@ clear_key:
       cryptodev_session_del (sessions[CRYPTODEV_OP_TYPE_ENCRYPT]);
       cryptodev_session_del (sessions[CRYPTODEV_OP_TYPE_DECRYPT]);
     }
+  clib_spinlock_unlock (&cmt->tlock);
   return ret;
 }
 
@@ -1017,44 +1091,22 @@ cryptodev_probe (vlib_main_t *vm, u32 n_workers)
 }
 
 static void
-cryptodev_get_max_sz (u32 *max_sess_sz, u32 *max_dp_sz)
+is_drv_unique (u32 driver_id, u32 **unique_drivers)
 {
-  cryptodev_main_t *cmt = &cryptodev_main;
-  cryptodev_inst_t *cinst;
-  u32 max_sess = 0, max_dp = 0;
+  u32 *unique_elt;
+  u8 found = 0;
 
-  vec_foreach (cinst, cmt->cryptodev_inst)
+  vec_foreach (unique_elt, *unique_drivers)
     {
-      u32 sess_sz = rte_cryptodev_sym_get_private_session_size (cinst->dev_id);
-      u32 dp_sz = rte_cryptodev_get_raw_dp_ctx_size (cinst->dev_id);
-
-      max_sess = clib_max (sess_sz, max_sess);
-      max_dp = clib_max (dp_sz, max_dp);
+      if (*unique_elt == driver_id)
+	{
+	  found = 1;
+	  break;
+	}
     }
 
-  *max_sess_sz = max_sess;
-  *max_dp_sz = max_dp;
-}
-
-static void
-dpdk_disable_cryptodev_engine (vlib_main_t *vm)
-{
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  cryptodev_main_t *cmt = &cryptodev_main;
-  u32 i;
-
-  for (i = (vlib_num_workers () > 0); i < tm->n_vlib_mains; i++)
-    {
-      u32 numa = vlib_get_main_by_index (i)->numa_node;
-      cryptodev_numa_data_t *numa_data;
-
-      vec_validate (cmt->per_numa_data, numa);
-      numa_data = cmt->per_numa_data + numa;
-      if (numa_data->sess_pool)
-	rte_mempool_free (numa_data->sess_pool);
-      if (numa_data->sess_priv_pool)
-	rte_mempool_free (numa_data->sess_priv_pool);
-    }
+  if (!found)
+    vec_add1 (*unique_drivers, driver_id);
 }
 
 clib_error_t *
@@ -1064,28 +1116,51 @@ dpdk_cryptodev_init (vlib_main_t * vm)
   vlib_thread_main_t *tm = vlib_get_thread_main ();
   cryptodev_engine_thread_t *cet;
   cryptodev_numa_data_t *numa_data;
-  struct rte_mempool *mp;
+  cryptodev_inst_t *dev_inst;
+  u32 node;
+  u8 nodes = 0;
   u32 skip_master = vlib_num_workers () > 0;
   u32 n_workers = tm->n_vlib_mains - skip_master;
-  u32 numa = vm->numa_node;
-  u32 sess_sz, dp_sz;
   u32 eidx;
   u32 i;
-  u8 *name = 0;
+  u32 *unique_drivers = 0;
   clib_error_t *error;
 
   cmt->iova_mode = rte_eal_iova_mode ();
 
-  vec_validate (cmt->per_numa_data, vm->numa_node);
+  clib_bitmap_foreach (node, tm->cpu_socket_bitmap)
+    {
+      if (node >= nodes)
+	nodes = node;
+    }
+
+  vec_validate (cmt->per_numa_data, nodes);
+  vec_foreach (numa_data, cmt->per_numa_data)
+    {
+      vec_validate (numa_data->sess_pools, 0);
+    }
 
   /* probe all cryptodev devices and get queue info */
   if (cryptodev_probe (vm, n_workers) < 0)
     {
-      error = clib_error_return (0, "Failed to configure cryptodev");
+      error = clib_error_return (0, "Not enough cryptodev resources");
       goto err_handling;
     }
 
-  cryptodev_get_max_sz (&sess_sz, &dp_sz);
+  vec_foreach (dev_inst, cmt->cryptodev_inst)
+    {
+      u32 dev_id = dev_inst->dev_id;
+      struct rte_cryptodev *cdev = rte_cryptodev_pmd_get_dev (dev_id);
+      u32 driver_id = cdev->driver_id;
+      is_drv_unique (driver_id, &unique_drivers);
+
+      u32 sess_sz =
+	rte_cryptodev_sym_get_private_session_size (dev_inst->dev_id);
+      cmt->sess_sz = clib_max (cmt->sess_sz, sess_sz);
+    }
+
+  cmt->drivers_cnt = vec_len (unique_drivers);
+  vec_free (unique_drivers);
 
   clib_bitmap_vec_validate (cmt->active_cdev_inst_mask, tm->n_vlib_mains);
   clib_spinlock_init (&cmt->tlock);
@@ -1095,46 +1170,13 @@ dpdk_cryptodev_init (vlib_main_t * vm)
   for (i = skip_master; i < tm->n_vlib_mains; i++)
     {
       cet = cmt->per_thread_data + i;
-      numa = vlib_get_main_by_index (i)->numa_node;
 
-      vec_validate (cmt->per_numa_data, numa);
-      numa_data = vec_elt_at_index (cmt->per_numa_data, numa);
-
-      if (!numa_data->sess_pool)
+      if (cryptodev_assign_resource (cet, 0, CRYPTODEV_RESOURCE_ASSIGN_AUTO) <
+	  0)
 	{
-	  /* create session pool for the numa node */
-	  name = format (0, "vcryptodev_sess_pool_%u%c", numa, 0);
-	  mp = rte_cryptodev_sym_session_pool_create (
-	    (char *) name, CRYPTODEV_NB_SESSION, 0, 0, 0, numa);
-	  if (!mp)
-	    {
-	      error =
-		clib_error_return (0, "Not enough memory for mp %s", name);
-	      goto err_handling;
-	    }
-	  vec_free (name);
-
-	  numa_data->sess_pool = mp;
-
-	  /* create session private pool for the numa node */
-	  name = format (0, "cryptodev_sess_pool_%u%c", numa, 0);
-	  mp =
-	    rte_mempool_create ((char *) name, CRYPTODEV_NB_SESSION, sess_sz,
-				0, 0, NULL, NULL, NULL, NULL, numa, 0);
-	  if (!mp)
-	    {
-	      error =
-		clib_error_return (0, "Not enough memory for mp %s", name);
-	      vec_free (name);
-	      goto err_handling;
-	    }
-
-	  vec_free (name);
-
-	  numa_data->sess_priv_pool = mp;
+	  error = clib_error_return (0, "Failed to configure cryptodev");
+	  goto err_handling;
 	}
-
-      cryptodev_assign_resource (cet, 0, CRYPTODEV_RESOURCE_ASSIGN_AUTO);
     }
 
   /* register handler */
@@ -1160,7 +1202,5 @@ dpdk_cryptodev_init (vlib_main_t * vm)
   return 0;
 
 err_handling:
-  dpdk_disable_cryptodev_engine (vm);
-
   return error;
 }
diff --git a/src/plugins/dpdk/cryptodev/cryptodev.h b/src/plugins/dpdk/cryptodev/cryptodev.h
index 3b47b43f538..80719ee69c8 100644
--- a/src/plugins/dpdk/cryptodev/cryptodev.h
+++ b/src/plugins/dpdk/cryptodev/cryptodev.h
@@ -27,7 +27,7 @@
 #define CRYPTODEV_MAX_INFLIGHT	   (CRYPTODEV_NB_CRYPTO_OPS - 1)
 #define CRYPTODEV_AAD_MASK	   (CRYPTODEV_NB_CRYPTO_OPS - 1)
 #define CRYPTODEV_DEQ_CACHE_SZ	   32
-#define CRYPTODEV_NB_SESSION	   10240
+#define CRYPTODEV_NB_SESSION	   4096
 #define CRYPTODEV_MAX_IV_SIZE	   16
 #define CRYPTODEV_MAX_AAD_SIZE	   16
 #define CRYPTODEV_MAX_N_SGL	   8 /**< maximum number of segments */
@@ -120,6 +120,11 @@ typedef struct
 {
   struct rte_mempool *sess_pool;
   struct rte_mempool *sess_priv_pool;
+} cryptodev_session_pool_t;
+
+typedef struct
+{
+  cryptodev_session_pool_t *sess_pools;
 } cryptodev_numa_data_t;
 
 typedef struct
@@ -170,6 +175,8 @@ typedef struct
   clib_bitmap_t *active_cdev_inst_mask;
   clib_spinlock_t tlock;
   cryptodev_capability_t *supported_caps;
+  u32 sess_sz;
+  u32 drivers_cnt;
   u8 is_raw_api;
 } cryptodev_main_t;