Add a new communication channel between VPP and openssl engine

Thus when engine buffer is full during a burst in performance
tesing, this code will help VPP handle retry machansim.

Change-Id: I0f9fc05d3dba8a54d34dca4c6137700d6c80f714
Signed-off-by: Ping Yu <ping.yu@intel.com>

4 files changed, 60 insertions, 20 deletions

diff --git a/src/plugins/tlsopenssl/tls_async.c b/src/plugins/tlsopenssl/tls_async.c
index aec1d7dcee2..3b469b9d3c4 100644
--- a/src/plugins/tlsopenssl/tls_async.c
+++ b/src/plugins/tlsopenssl/tls_async.c
@@ -99,7 +99,7 @@ evt_pool_init (vlib_main_t * vm)
 
   num_threads = 1 /* main thread */  + vtm->n_threads;
 
-  TLS_DBG ("Totally there is %d thread\n", num_threads);
+  TLS_DBG (2, "Totally there is %d thread\n", num_threads);
 
   vec_validate (om->evt_pool, num_threads - 1);
   vec_validate (om->status, num_threads - 1);
@@ -115,7 +115,6 @@ evt_pool_init (vlib_main_t * vm)
     }
   om->polling = NULL;
 
-  TLS_DBG ("Node disabled\n");
   openssl_async_node_enable_disable (0);
 
   return;
@@ -253,7 +252,7 @@ openssl_async_run (void *evt)
   int *evt_run_tail = &om->status[thread_index].evt_run_tail;
   int *evt_run_head = &om->status[thread_index].evt_run_head;
 
-  TLS_DBG ("Set event %d to run\n", event_index);
+  TLS_DBG (2, "Set event %d to run\n", event_index);
 
   event = openssl_evt_get_w_thread (event_index, thread_index);
 
@@ -271,7 +270,9 @@ openssl_async_run (void *evt)
     }
   *evt_run_tail = event_index;
   if (*evt_run_head < 0)
-    *evt_run_head = event_index;
+    {
+      *evt_run_head = event_index;
+    }
 
   return 1;
 }
@@ -303,10 +304,33 @@ vpp_add_async_pending_event (tls_ctx_t * ctx,
   event->next = *evt_pending_head;
   *evt_pending_head = eidx;
 
-
   return &event->engine_callback;
 }
 
+int
+vpp_add_async_run_event (tls_ctx_t * ctx, openssl_resume_handler * handler)
+{
+  u32 eidx;
+  openssl_evt_t *event;
+  openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
+  u32 thread_id = ctx->c_thread_index;
+
+  eidx = openssl_evt_alloc ();
+  event = openssl_evt_get (eidx);
+
+  event->ctx_index = oc->openssl_ctx_index;
+  event->status = SSL_ASYNC_PENDING;
+  event->handler = handler;
+  event->cb_args.event_index = eidx;
+  event->cb_args.thread_index = thread_id;
+  event->engine_callback.callback = openssl_async_run;
+  event->engine_callback.arg = &event->cb_args;
+
+  /* This is a retry event, and need to put to ring to make it run again */
+  return openssl_async_run (&event->cb_args);
+
+}
+
 void
 event_handler (void *tls_async)
 {
@@ -325,7 +349,6 @@ event_handler (void *tls_async)
 
   if (handler)
     {
-      TLS_DBG ("relaunch...\n");
       (*handler) (ctx, tls_session);
     }
 
@@ -349,7 +372,7 @@ dasync_polling ()
   evt_pending = &om->status[thread_index].evt_pending_head;
   while (*evt_pending >= 0)
     {
-      TLS_DBG ("polling... current head = %d\n", *evt_pending);
+      TLS_DBG (2, "polling... current head = %d\n", *evt_pending);
       event = openssl_evt_get_w_thread (*evt_pending, thread_index);
       *evt_pending = event->next;
       if (event->status == SSL_ASYNC_PENDING)
@@ -385,7 +408,7 @@ qat_polling_config ()
 		   NULL, NULL, 0);
   *config = 1;
 
-  TLS_DBG ("set thread %d and instance %d mapping\n", thread_index,
+  TLS_DBG (2, "set thread %d and instance %d mapping\n", thread_index,
 	   thread_index);
 
 }
@@ -394,14 +417,12 @@ void
 qat_polling ()
 {
   openssl_async_t *om = &openssl_async_main;
-  int ret;
+  int poll_status = 0;
 
   if (om->start_polling)
     {
       qat_polling_config ();
-#define QAT_CMD_POLL (ENGINE_CMD_BASE + 1)
-      ENGINE_ctrl (om->engine, QAT_CMD_POLL, 0, &ret, NULL);
-      ;
+      ENGINE_ctrl_cmd (om->engine, "POLL", 0, &poll_status, NULL, 0);
     }
 }
 
@@ -462,7 +483,7 @@ tls_resume_from_crypto (int thread_index)
       if (*evt_run_head >= 0)
 	{
 	  event = openssl_evt_get_w_thread (*evt_run_head, thread_index);
-	  TLS_DBG ("event run = %d\n", *evt_run_head);
+	  TLS_DBG (2, "event run = %d\n", *evt_run_head);
 	  tls_async_do_job (*evt_run_head, thread_index);
 
 	  *evt_run_head = event->next;
@@ -481,8 +502,6 @@ tls_resume_from_crypto (int thread_index)
 static clib_error_t *
 tls_async_init (vlib_main_t * vm)
 {
-
-  TLS_DBG ("Start to call tls_async_init\n");
   evt_pool_init (vm);
   return 0;
 
diff --git a/src/plugins/tlsopenssl/tls_openssl.c b/src/plugins/tlsopenssl/tls_openssl.c
index 91c9de758b4..b675d795cd8 100644
--- a/src/plugins/tlsopenssl/tls_openssl.c
+++ b/src/plugins/tlsopenssl/tls_openssl.c
@@ -167,13 +167,26 @@ vpp_ssl_async_process_event (tls_ctx_t * ctx,
     {
       SSL_set_async_callback (oc->ssl, (void *) engine_cb->callback,
 			      (void *) engine_cb->arg);
-      TLS_DBG ("set callback to engine %p\n", engine_cb->callback);
+      TLS_DBG (2, "set callback to engine %p\n", engine_cb->callback);
     }
-  /* associated fd with context for return */
-  TLS_DBG ("completed assoicated fd with tls session\n");
   return 0;
 
 }
+
+/* Due to engine busy stat, VPP need to retry later */
+static int
+vpp_ssl_async_retry_func (tls_ctx_t * ctx, openssl_resume_handler * handler)
+{
+  openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
+
+  if (vpp_add_async_run_event (ctx, handler))
+    {
+      SSL_set_async_estatus (oc->ssl, 0);
+    }
+  return 0;
+
+}
+
 #endif
 
 int
@@ -182,6 +195,7 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, stream_session_t * tls_session)
   openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
   int rv = 0, err;
 #ifdef HAVE_OPENSSL_ASYNC
+  int estatus;
   openssl_resume_handler *myself;
 #endif
 
@@ -200,9 +214,14 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, stream_session_t * tls_session)
       err = SSL_get_error (oc->ssl, rv);
       openssl_try_handshake_write (oc, tls_session);
 #ifdef HAVE_OPENSSL_ASYNC
+      myself = openssl_ctx_handshake_rx;
+      if (SSL_get_async_estatus (oc->ssl, &estatus)
+	  && (estatus == ENGINE_STATUS_RETRY))
+	{
+	  vpp_ssl_async_retry_func (ctx, myself);
+	}
       if (err == SSL_ERROR_WANT_ASYNC)
 	{
-	  myself = openssl_ctx_handshake_rx;
 	  vpp_ssl_async_process_event (ctx, myself);
 	}
 #endif
diff --git a/src/plugins/tlsopenssl/tls_openssl.h b/src/plugins/tlsopenssl/tls_openssl.h
index b01814939af..c1a21e57317 100644
--- a/src/plugins/tlsopenssl/tls_openssl.h
+++ b/src/plugins/tlsopenssl/tls_openssl.h
@@ -55,6 +55,8 @@ tls_ctx_t *openssl_ctx_get_w_thread (u32 ctx_index, u8 thread_index);
 openssl_tls_callback_t *vpp_add_async_pending_event (tls_ctx_t * ctx,
 						     openssl_resume_handler *
 						     handler);
+int vpp_add_async_run_event (tls_ctx_t * ctx, openssl_resume_handler *
+			     handler);
 void openssl_polling_start (ENGINE * engine);
 int openssl_engine_register (char *engine, char *alg);
 void openssl_async_node_enable_disable (u8 is_en);
diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h
index b311578f678..f67f307550b 100644
--- a/src/vnet/tls/tls.h
+++ b/src/vnet/tls/tls.h
@@ -32,7 +32,7 @@
   if (_lvl <= TLS_DEBUG) 				\
     clib_warning (_fmt, ##_args)
 #else
-#define TLS_DBG(_fmt, _args...)
+#define TLS_DBG(_lvl, _fmt, _args...)
 #endif
 
 /* *INDENT-OFF* */