crypto: SW scheduler async crypto engine

Type: feature

This patch adds new sw_scheduler async crypto engine.
The engine transforms async frames info sync crypto ops and
delegates them to active sync engines. With the patch it
is possible to increase the single worker crypto throughput
by offloading the crypto workload to multiple workers.

By default all workers in the system will attend the crypto
workload processing. However a worker's available cycles
are limited. To avail more cycles to one worker to process
other workload (e.g. the worker core that handles the RX/TX
and IPSec stack processing), a useful cli command is added
to remove itself (or add it back later) from the heavy
crypto workload but only let other workers to process the
crypto. The command is:

 - set sw_scheduler worker <idx> crypto <on|off>

It also adds new interrupt mode to async crypto dispatch node.
This mode signals the node when new frames are enqueued
as opposed to polling mode that continuously calls dispatch node.

New cli commands:
 - set crypto async dispatch [polling|interrupt]
 - show crypto async status (displays mode and nodes' states)

Signed-off-by: PiotrX Kleski <piotrx.kleski@intel.com>
Signed-off-by: DariuszX Kazimierski <dariuszx.kazimierski@intel.com>
Reviewed-by: Fan Zhang <roy.fan.zhang@intel.com>
Change-Id: I332655f347bb9e3bc9c64166e86e393e911bdb39

1 files changed, 65 insertions, 17 deletions

diff --git a/src/vnet/crypto/crypto.c b/src/vnet/crypto/crypto.c
index 288e227821b..b877d9a5f03 100644
--- a/src/vnet/crypto/crypto.c
+++ b/src/vnet/crypto/crypto.c
@@ -446,18 +446,20 @@ vnet_crypto_key_add_linked (vlib_main_t * vm,
 clib_error_t *
 crypto_dispatch_enable_disable (int is_enable)
 {
-  vlib_main_t *vm = vlib_get_main ();
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  vlib_node_t *node = vlib_get_node_by_name (vm, (u8 *) "crypto-dispatch");
   vnet_crypto_main_t *cm = &crypto_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
   u32 skip_master = vlib_num_workers () > 0, i;
-  u32 state_change = 0;
-  vlib_node_state_t state;
+  vlib_node_state_t state = VLIB_NODE_STATE_DISABLED;
+  u8 state_change = 0;
 
+  CLIB_MEMORY_STORE_BARRIER ();
   if (is_enable && cm->async_refcnt > 0)
     {
       state_change = 1;
-      state = VLIB_NODE_STATE_POLLING;
+      state =
+	cm->dispatch_mode ==
+	VNET_CRYPTO_ASYNC_DISPATCH_POLLING ? VLIB_NODE_STATE_POLLING :
+	VLIB_NODE_STATE_INTERRUPT;
     }
 
   if (!is_enable && cm->async_refcnt == 0)
@@ -468,8 +470,11 @@ crypto_dispatch_enable_disable (int is_enable)
 
   if (state_change)
     for (i = skip_master; i < tm->n_vlib_mains; i++)
-      vlib_node_set_state (vlib_mains[i], node->index, state);
-
+      {
+	if (state !=
+	    vlib_node_get_state (vlib_mains[i], cm->crypto_node_index))
+	  vlib_node_set_state (vlib_mains[i], cm->crypto_node_index, state);
+      }
   return 0;
 }
 
@@ -553,20 +558,20 @@ vnet_crypto_register_post_node (vlib_main_t * vm, char *post_node_name)
 void
 vnet_crypto_request_async_mode (int is_enable)
 {
-  vlib_main_t *vm = vlib_get_main ();
-  vlib_thread_main_t *tm = vlib_get_thread_main ();
-  vlib_node_t *node = vlib_get_node_by_name (vm, (u8 *) "crypto-dispatch");
   vnet_crypto_main_t *cm = &crypto_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
   u32 skip_master = vlib_num_workers () > 0, i;
-  u32 state_change = 0;
-  vlib_node_state_t state;
+  vlib_node_state_t state = VLIB_NODE_STATE_DISABLED;
+  u8 state_change = 0;
 
+  CLIB_MEMORY_STORE_BARRIER ();
   if (is_enable && cm->async_refcnt == 0)
     {
       state_change = 1;
-      state = VLIB_NODE_STATE_POLLING;
+      state =
+	cm->dispatch_mode == VNET_CRYPTO_ASYNC_DISPATCH_POLLING ?
+	VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_INTERRUPT;
     }
-
   if (!is_enable && cm->async_refcnt == 1)
     {
       state_change = 1;
@@ -575,7 +580,11 @@ vnet_crypto_request_async_mode (int is_enable)
 
   if (state_change)
     for (i = skip_master; i < tm->n_vlib_mains; i++)
-      vlib_node_set_state (vlib_mains[i], node->index, state);
+      {
+	if (state !=
+	    vlib_node_get_state (vlib_mains[i], cm->crypto_node_index))
+	  vlib_node_set_state (vlib_mains[i], cm->crypto_node_index, state);
+      }
 
   if (is_enable)
     cm->async_refcnt += 1;
@@ -583,6 +592,40 @@ vnet_crypto_request_async_mode (int is_enable)
     cm->async_refcnt -= 1;
 }
 
+void
+vnet_crypto_set_async_dispatch_mode (u8 mode)
+{
+  vnet_crypto_main_t *cm = &crypto_main;
+  u32 skip_master = vlib_num_workers () > 0, i;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+  vlib_node_state_t state = VLIB_NODE_STATE_DISABLED;
+
+  CLIB_MEMORY_STORE_BARRIER ();
+  cm->dispatch_mode = mode;
+  if (mode == VNET_CRYPTO_ASYNC_DISPATCH_INTERRUPT)
+    {
+      state =
+	cm->async_refcnt == 0 ?
+	VLIB_NODE_STATE_DISABLED : VLIB_NODE_STATE_INTERRUPT;
+    }
+  else if (mode == VNET_CRYPTO_ASYNC_DISPATCH_POLLING)
+    {
+      state =
+	cm->async_refcnt == 0 ?
+	VLIB_NODE_STATE_DISABLED : VLIB_NODE_STATE_POLLING;
+    }
+
+  for (i = skip_master; i < tm->n_vlib_mains; i++)
+    {
+      if (state != vlib_node_get_state (vlib_mains[i], cm->crypto_node_index))
+	vlib_node_set_state (vlib_mains[i], cm->crypto_node_index, state);
+    }
+  clib_warning ("Switching dispatch mode might not work is some situations.");
+  clib_warning
+    ("Use 'show crypto async status' to verify that the nodes' states were set");
+  clib_warning ("and if not, set 'crypto async dispatch' mode again.");
+}
+
 int
 vnet_crypto_is_set_async_handler (vnet_crypto_async_op_id_t op)
 {
@@ -663,6 +706,8 @@ vnet_crypto_init (vlib_main_t * vm)
   vnet_crypto_main_t *cm = &crypto_main;
   vlib_thread_main_t *tm = vlib_get_thread_main ();
   vnet_crypto_thread_t *ct = 0;
+
+  cm->dispatch_mode = VNET_CRYPTO_ASYNC_DISPATCH_POLLING;
   cm->engine_index_by_name = hash_create_string ( /* size */ 0,
 						 sizeof (uword));
   cm->alg_index_by_name = hash_create_string (0, sizeof (uword));
@@ -705,7 +750,10 @@ vnet_crypto_init (vlib_main_t * vm)
 			       s);
     foreach_crypto_link_async_alg
 #undef _
-    return 0;
+    cm->crypto_node_index =
+    vlib_get_node_by_name (vm, (u8 *) "crypto-dispatch")->index;
+
+  return 0;
 }
 
 VLIB_INIT_FUNCTION (vnet_crypto_init);