crypto: encrypt/decrypt queues sw_scheduler

Type: improvement

Previously multiple sw crypto scheduler queues per core design
caused unaverage frame processing rate for each async op ID –
the lower the op ID is the highly likely they are processed first.

For example, when a RX core is feeding both encryption and
decryption jobs of the same crypto algorithm to the queues at a
high rate, in the mean time the crypto cores have no enough
cycles to process all: the jobs in the decryption queue are less
likely being processed, causing packet drop.

To improve the situation this patch makes every core only owning
a two queues, one for encrypt operations and one for decrypt.
The queue is changed either after checking each core
or after founding a frame to process.
All crypto jobs with different algorithm are pushed to
thoses queues and are treated evenly.

In addition, the crypto async infra now uses unified dequeue handler,
one per engine. Only the active engine will be registered its
dequeue handler in crypto main.

Signed-off-by: DariuszX Kazimierski <dariuszx.kazimierski@intel.com>
Signed-off-by: PiotrX Kleski <piotrx.kleski@intel.com>
Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com>
Signed-off-by: Jakub Wysocki <jakubx.wysocki@intel.com>
Change-Id: I517ee8e31633980de5e0dd4b05e1d5db5dea760e

4 files changed, 99 insertions, 34 deletions

diff --git a/src/vnet/crypto/cli.c b/src/vnet/crypto/cli.c
index a6098a18e11..4ee14ac1100 100644
--- a/src/vnet/crypto/cli.c
+++ b/src/vnet/crypto/cli.c
@@ -67,7 +67,7 @@ format_vnet_crypto_engine_candidates (u8 * s, va_list * args)
     {
       vec_foreach (e, cm->engines)
 	{
-	  if (e->enqueue_handlers[id] && e->dequeue_handlers[id])
+	  if (e->enqueue_handlers[id] && e->dequeue_handler)
 	    {
 	      s = format (s, "%U", format_vnet_crypto_engine, e - cm->engines);
 	      if (ei == e - cm->engines)
diff --git a/src/vnet/crypto/crypto.c b/src/vnet/crypto/crypto.c
index 7903f88b7cb..9f437cfcd4b 100644
--- a/src/vnet/crypto/crypto.c
+++ b/src/vnet/crypto/crypto.c
@@ -275,10 +275,9 @@ vnet_crypto_register_ops_handlers (vlib_main_t * vm, u32 engine_index,
 }
 
 void
-vnet_crypto_register_async_handler (vlib_main_t * vm, u32 engine_index,
-				    vnet_crypto_async_op_id_t opt,
-				    vnet_crypto_frame_enqueue_t * enqueue_hdl,
-				    vnet_crypto_frame_dequeue_t * dequeue_hdl)
+vnet_crypto_register_enqueue_handler (vlib_main_t *vm, u32 engine_index,
+				      vnet_crypto_async_op_id_t opt,
+				      vnet_crypto_frame_enqueue_t *enqueue_hdl)
 {
   vnet_crypto_main_t *cm = &crypto_main;
   vnet_crypto_engine_t *ae, *e = vec_elt_at_index (cm->engines, engine_index);
@@ -288,17 +287,14 @@ vnet_crypto_register_async_handler (vlib_main_t * vm, u32 engine_index,
   vec_validate_aligned (cm->dequeue_handlers, VNET_CRYPTO_ASYNC_OP_N_IDS,
 			CLIB_CACHE_LINE_BYTES);
 
-  /* both enqueue hdl and dequeue hdl should present */
-  if (!enqueue_hdl && !dequeue_hdl)
+  if (!enqueue_hdl)
     return;
 
   e->enqueue_handlers[opt] = enqueue_hdl;
-  e->dequeue_handlers[opt] = dequeue_hdl;
   if (otd->active_engine_index_async == ~0)
     {
       otd->active_engine_index_async = engine_index;
       cm->enqueue_handlers[opt] = enqueue_hdl;
-      cm->dequeue_handlers[opt] = dequeue_hdl;
     }
 
   ae = vec_elt_at_index (cm->engines, otd->active_engine_index_async);
@@ -306,12 +302,73 @@ vnet_crypto_register_async_handler (vlib_main_t * vm, u32 engine_index,
     {
       otd->active_engine_index_async = engine_index;
       cm->enqueue_handlers[opt] = enqueue_hdl;
-      cm->dequeue_handlers[opt] = dequeue_hdl;
     }
 
   return;
 }
 
+static int
+engine_index_cmp (void *v1, void *v2)
+{
+  u32 *a1 = v1;
+  u32 *a2 = v2;
+
+  if (*a1 > *a2)
+    return 1;
+  if (*a1 < *a2)
+    return -1;
+  return 0;
+}
+
+static void
+vnet_crypto_update_cm_dequeue_handlers (void)
+{
+  vnet_crypto_main_t *cm = &crypto_main;
+  vnet_crypto_async_op_data_t *otd;
+  vnet_crypto_engine_t *e;
+  u32 *active_engines = 0, *ei, last_ei = ~0, i;
+
+  vec_reset_length (cm->dequeue_handlers);
+
+  for (i = 0; i < VNET_CRYPTO_ASYNC_OP_N_IDS; i++)
+    {
+      otd = cm->async_opt_data + i;
+      e = cm->engines + otd->active_engine_index_async;
+      if (!e->dequeue_handler)
+	continue;
+      vec_add1 (active_engines, otd->active_engine_index_async);
+    }
+
+  vec_sort_with_function (active_engines, engine_index_cmp);
+
+  vec_foreach (ei, active_engines)
+    {
+      if (ei[0] == last_ei)
+	continue;
+
+      e = cm->engines + ei[0];
+      vec_add1 (cm->dequeue_handlers, e->dequeue_handler);
+      last_ei = ei[0];
+    }
+
+  vec_free (active_engines);
+}
+
+void
+vnet_crypto_register_dequeue_handler (vlib_main_t *vm, u32 engine_index,
+				      vnet_crypto_frame_dequeue_t *deq_fn)
+{
+  vnet_crypto_main_t *cm = &crypto_main;
+  vnet_crypto_engine_t *e = vec_elt_at_index (cm->engines, engine_index);
+
+  if (!deq_fn)
+    return;
+
+  e->dequeue_handler = deq_fn;
+
+  return;
+}
+
 void
 vnet_crypto_register_key_handler (vlib_main_t * vm, u32 engine_index,
 				  vnet_crypto_key_handler_t * key_handler)
@@ -509,11 +566,11 @@ crypto_set_active_async_engine (vnet_crypto_async_op_data_t * od,
   vnet_crypto_main_t *cm = &crypto_main;
   vnet_crypto_engine_t *ce = vec_elt_at_index (cm->engines, ei);
 
-  if (ce->enqueue_handlers[id] && ce->dequeue_handlers[id])
+  if (ce->enqueue_handlers[id] && ce->dequeue_handler)
     {
       od->active_engine_index_async = ei;
       cm->enqueue_handlers[id] = ce->enqueue_handlers[id];
-      cm->dequeue_handlers[id] = ce->dequeue_handlers[id];
+      cm->dequeue_handlers[id] = ce->dequeue_handler;
     }
 }
 
@@ -525,6 +582,9 @@ vnet_crypto_set_async_handler2 (char *alg_name, char *engine)
   vnet_crypto_async_alg_data_t *ad;
   int i;
 
+  if (cm->async_refcnt)
+    return -EBUSY;
+
   p = hash_get_mem (cm->async_alg_index_by_name, alg_name);
   if (!p)
     return -1;
@@ -546,6 +606,8 @@ vnet_crypto_set_async_handler2 (char *alg_name, char *engine)
       crypto_set_active_async_engine (od, id, p[0]);
     }
 
+  vnet_crypto_update_cm_dequeue_handlers ();
+
   return 0;
 }
 
@@ -603,12 +665,18 @@ vnet_crypto_request_async_mode (int is_enable)
     }
 
   if (state_change)
-    for (i = skip_master; i < tm->n_vlib_mains; i++)
-      {
-	vlib_main_t *ovm = vlib_get_main_by_index (i);
-	if (state != vlib_node_get_state (ovm, cm->crypto_node_index))
-	  vlib_node_set_state (ovm, cm->crypto_node_index, state);
-      }
+    {
+
+      for (i = skip_master; i < tm->n_vlib_mains; i++)
+	{
+	  vlib_main_t *ovm = vlib_get_main_by_index (i);
+	  if (state != vlib_node_get_state (ovm, cm->crypto_node_index))
+	    vlib_node_set_state (ovm, cm->crypto_node_index, state);
+	}
+
+      if (is_enable)
+	vnet_crypto_update_cm_dequeue_handlers ();
+    }
 
   if (is_enable)
     cm->async_refcnt += 1;
@@ -753,7 +821,6 @@ vnet_crypto_init (vlib_main_t * vm)
 			CLIB_CACHE_LINE_BYTES);
   vec_validate (cm->algs, VNET_CRYPTO_N_ALGS);
   vec_validate (cm->async_algs, VNET_CRYPTO_N_ASYNC_ALGS);
-  clib_bitmap_validate (cm->async_active_ids, VNET_CRYPTO_ASYNC_OP_N_IDS);
 
 #define _(n, s, l) \
   vnet_crypto_init_cipher_data (VNET_CRYPTO_ALG_##n, \
diff --git a/src/vnet/crypto/crypto.h b/src/vnet/crypto/crypto.h
index 71978b64835..73b7f70ec46 100644
--- a/src/vnet/crypto/crypto.h
+++ b/src/vnet/crypto/crypto.h
@@ -422,12 +422,15 @@ void vnet_crypto_register_key_handler (vlib_main_t * vm, u32 engine_index,
 
 /** async crypto register functions */
 u32 vnet_crypto_register_post_node (vlib_main_t * vm, char *post_node_name);
-void vnet_crypto_register_async_handler (vlib_main_t * vm,
-					 u32 engine_index,
-					 vnet_crypto_async_op_id_t opt,
-					 vnet_crypto_frame_enqueue_t * enq_fn,
-					 vnet_crypto_frame_dequeue_t *
-					 deq_fn);
+
+void
+vnet_crypto_register_enqueue_handler (vlib_main_t *vm, u32 engine_index,
+				      vnet_crypto_async_op_id_t opt,
+				      vnet_crypto_frame_enqueue_t *enq_fn);
+
+void
+vnet_crypto_register_dequeue_handler (vlib_main_t *vm, u32 engine_index,
+				      vnet_crypto_frame_dequeue_t *deq_fn);
 
 typedef struct
 {
@@ -439,7 +442,7 @@ typedef struct
     vnet_crypto_chained_ops_handler_t
     * chained_ops_handlers[VNET_CRYPTO_N_OP_IDS];
   vnet_crypto_frame_enqueue_t *enqueue_handlers[VNET_CRYPTO_ASYNC_OP_N_IDS];
-  vnet_crypto_frame_dequeue_t *dequeue_handlers[VNET_CRYPTO_ASYNC_OP_N_IDS];
+  vnet_crypto_frame_dequeue_t *dequeue_handler;
 } vnet_crypto_engine_t;
 
 typedef struct
@@ -456,7 +459,6 @@ typedef struct
   vnet_crypto_chained_ops_handler_t **chained_ops_handlers;
   vnet_crypto_frame_enqueue_t **enqueue_handlers;
   vnet_crypto_frame_dequeue_t **dequeue_handlers;
-  clib_bitmap_t *async_active_ids;
   vnet_crypto_op_data_t opt_data[VNET_CRYPTO_N_OP_IDS];
   vnet_crypto_async_op_data_t async_opt_data[VNET_CRYPTO_ASYNC_OP_N_IDS];
   vnet_crypto_engine_t *engines;
@@ -591,7 +593,6 @@ vnet_crypto_async_submit_open_frame (vlib_main_t * vm,
 {
   vnet_crypto_main_t *cm = &crypto_main;
   vlib_thread_main_t *tm = vlib_get_thread_main ();
-  vnet_crypto_async_op_id_t opt = frame->op;
   u32 i = vlib_num_workers () > 0;
 
   frame->state = VNET_CRYPTO_FRAME_STATE_PENDING;
@@ -599,7 +600,6 @@ vnet_crypto_async_submit_open_frame (vlib_main_t * vm,
 
   int ret = (cm->enqueue_handlers[frame->op]) (vm, frame);
 
-  clib_bitmap_set_no_check (cm->async_active_ids, opt, 1);
   if (PREDICT_TRUE (ret == 0))
     {
       if (cm->dispatch_mode == VNET_CRYPTO_ASYNC_DISPATCH_INTERRUPT)
diff --git a/src/vnet/crypto/node.c b/src/vnet/crypto/node.c
index e753f1ad1db..4757f538855 100644
--- a/src/vnet/crypto/node.c
+++ b/src/vnet/crypto/node.c
@@ -158,14 +158,12 @@ VLIB_NODE_FN (crypto_dispatch_node) (vlib_main_t * vm,
 {
   vnet_crypto_main_t *cm = &crypto_main;
   vnet_crypto_thread_t *ct = cm->threads + vm->thread_index;
-  u32 n_dispatched = 0, n_cache = 0;
-  u32 index;
+  u32 n_dispatched = 0, n_cache = 0, index;
+  vec_foreach_index (index, cm->dequeue_handlers)
 
-  /* *INDENT-OFF* */
-  clib_bitmap_foreach (index, cm->async_active_ids)  {
     n_cache = crypto_dequeue_frame (vm, node, ct, cm->dequeue_handlers[index],
 				    n_cache, &n_dispatched);
-  }
+
   /* *INDENT-ON* */
   if (n_cache)
     vlib_buffer_enqueue_to_next_vec (vm, node, &ct->buffer_indices, &ct->nexts,