6 files changed, 163 insertions, 198 deletions

diff --git a/src/plugins/tlsopenssl/tls_async.c b/src/plugins/tlsopenssl/tls_async.c
index 1d3ca8b82b8..100a75b3332 100644
--- a/src/plugins/tlsopenssl/tls_async.c
+++ b/src/plugins/tlsopenssl/tls_async.c
@@ -13,21 +13,16 @@
  * limitations under the License.
  */
 #include <vnet/vnet.h>
-#include <vnet/ip/ip.h>
 #include <vnet/api_errno.h>
 #include <vlib/node_funcs.h>
 #include <openssl/engine.h>
 #include <tlsopenssl/tls_openssl.h>
 
-#define MAX_SESSION	    4096
+#define SSL_ASYNC_INFLIGHT    1
+#define SSL_ASYNC_READY       2
+#define SSL_ASYNC_REENTER     3
 #define MAX_VECTOR_ASYNC    256
 
-#define SSL_ASYNC_INFLIGHT  1
-#define SSL_ASYNC_PENDING   2
-#define SSL_ASYNC_READY     3
-
-#define EMPTY_STRUCT {0}
-
 typedef struct openssl_tls_callback_arg_
 {
   int thread_index;
@@ -36,32 +31,28 @@ typedef struct openssl_tls_callback_arg_
 
 typedef struct openssl_event_
 {
-  int status;
-  u32 event_index;
-  u8 thread_index;
   u32 ctx_index;
+  int session_index;
+  u8 status;
 
   openssl_resume_handler *handler;
-  openssl_tls_callback_t engine_callback;
   openssl_tls_callback_arg_t cb_args;
-
+#define thread_idx cb_args.thread_index
+#define event_idx cb_args.event_index
   int next;
 } openssl_evt_t;
 
-typedef struct openssl_async_status_
+typedef struct openssl_async_queue_
 {
   int evt_run_head;
   int evt_run_tail;
-  int evt_pending_head;
-  int poll_config;
-} openssl_async_status_t;
+} openssl_async_queue_t;
 
 typedef struct openssl_async_
 {
   openssl_evt_t ***evt_pool;
-  openssl_async_status_t *status;
+  openssl_async_queue_t *queue;
   void (*polling) (void);
-  void (*polling_conf) (void);
   u8 start_polling;
   ENGINE *engine;
 
@@ -77,11 +68,13 @@ struct engine_polling
   char *engine;
   void (*polling) (void);
   void (*pre_init) (void);
-  void (*polling_conf) (void);
+  void (*thread_init) (void *);
 };
 
+void qat_init_thread (void *arg);
+
 struct engine_polling engine_list[] = {
-  {"qat", qat_polling, qat_pre_init, qat_polling_config},
+  {"qat", qat_polling, qat_pre_init, qat_init_thread},
   {"dasync", dasync_polling, NULL, NULL}
 };
 
@@ -104,16 +97,15 @@ evt_pool_init (vlib_main_t * vm)
   TLS_DBG (2, "Totally there is %d thread\n", num_threads);
 
   vec_validate (om->evt_pool, num_threads - 1);
-  vec_validate (om->status, num_threads - 1);
+  vec_validate (om->queue, num_threads - 1);
 
   om->start_polling = 0;
   om->engine = 0;
 
   for (i = 0; i < num_threads; i++)
     {
-      om->status[i].evt_run_head = -1;
-      om->status[i].evt_run_tail = -1;
-      om->status[i].evt_pending_head = -1;
+      om->queue[i].evt_run_head = -1;
+      om->queue[i].evt_run_tail = -1;
     }
   om->polling = NULL;
 
@@ -121,7 +113,7 @@ evt_pool_init (vlib_main_t * vm)
 }
 
 int
-openssl_engine_register (char *engine_name, char *algorithm)
+openssl_engine_register (char *engine_name, char *algorithm, int async)
 {
   int i, registered = -1;
   openssl_async_t *om = &openssl_async_main;
@@ -133,8 +125,6 @@ openssl_engine_register (char *engine_name, char *algorithm)
       if (!strcmp (engine_list[i].engine, engine_name))
 	{
 	  om->polling = engine_list[i].polling;
-	  om->polling_conf = engine_list[i].polling_conf;
-
 	  registered = i;
 	}
     }
@@ -179,6 +169,19 @@ openssl_engine_register (char *engine_name, char *algorithm)
 	}
     }
 
+  if (async)
+    {
+      openssl_async_node_enable_disable (1);
+    }
+
+  for (i = 0; i < vlib_num_workers (); i++)
+    {
+      if (engine_list[registered].thread_init)
+	session_send_rpc_evt_to_thread (i + 1,
+					engine_list[registered].thread_init,
+					(void *) &i);
+    }
+
   om->start_polling = 1;
 
   return 0;
@@ -206,24 +209,12 @@ openssl_evt_get_w_thread (int evt_index, u8 thread_index)
 }
 
 int
-openssl_evt_free (int event_idx, u8 thread_index)
+openssl_evt_free (int event_index, u8 thread_index)
 {
-  openssl_evt_t *evt;
   openssl_async_t *om = &openssl_async_main;
-  int *evt_run_tail = &om->status[thread_index].evt_run_tail;
-
-  if (event_idx < 0)
-    return 0;
-
-  evt = openssl_evt_get_w_thread (event_idx, thread_index);
-
-  evt->status = 0;
 
   /*pool operation */
-  pool_put_index (om->evt_pool[thread_index], event_idx);
-
-  if (*evt_run_tail == event_idx)
-    *evt_run_tail = -1;
+  pool_put_index (om->evt_pool[thread_index], event_index);
 
   return 1;
 }
@@ -240,32 +231,38 @@ openssl_evt_alloc (void)
     *evt = clib_mem_alloc (sizeof (openssl_evt_t));
 
   clib_memset (*evt, 0, sizeof (openssl_evt_t));
-  (*evt)->event_index = evt - tm->evt_pool[thread_index];
-  return ((*evt)->event_index);
+  (*evt)->event_idx = evt - tm->evt_pool[thread_index];
+  return ((*evt)->event_idx);
 }
 
+
+/* In most cases, tls_async_openssl_callback is called by HW to make event active
+ * When EAGAIN received, VPP will call this callback to retry
+ */
 int
-tls_async_openssl_callback (SSL * s, void *evt)
+tls_async_openssl_callback (SSL * s, void *cb_arg)
 {
   openssl_evt_t *event, *event_tail;
   openssl_async_t *om = &openssl_async_main;
-  openssl_tls_callback_arg_t *args = (openssl_tls_callback_arg_t *) evt;
+  openssl_tls_callback_arg_t *args = (openssl_tls_callback_arg_t *) cb_arg;
   int thread_index = args->thread_index;
   int event_index = args->event_index;
-  int *evt_run_tail = &om->status[thread_index].evt_run_tail;
-  int *evt_run_head = &om->status[thread_index].evt_run_head;
+  int *evt_run_tail = &om->queue[thread_index].evt_run_tail;
+  int *evt_run_head = &om->queue[thread_index].evt_run_head;
 
   TLS_DBG (2, "Set event %d to run\n", event_index);
 
-  event = openssl_evt_get_w_thread (event_index, thread_index);
-
-  if (event->status == SSL_ASYNC_READY)
-    return 0;
+  event = openssl_evt_get (event_index);
 
+  /* Happend when a recursive case, especially in SW simulation */
+  if (PREDICT_FALSE (event->status == SSL_ASYNC_READY))
+    {
+      event->status = SSL_ASYNC_REENTER;
+      return 0;
+    }
   event->status = SSL_ASYNC_READY;
   event->next = -1;
 
-
   if (*evt_run_tail >= 0)
     {
       event_tail = openssl_evt_get_w_thread (*evt_run_tail, thread_index);
@@ -280,84 +277,70 @@ tls_async_openssl_callback (SSL * s, void *evt)
   return 1;
 }
 
-openssl_tls_callback_t *
-vpp_add_async_pending_event (tls_ctx_t * ctx,
-			     openssl_resume_handler * handler)
+int
+vpp_tls_async_init_event (tls_ctx_t * ctx,
+			  openssl_resume_handler * handler,
+			  session_t * session)
 {
   u32 eidx;
   openssl_evt_t *event;
-  openssl_async_t *om = &openssl_async_main;
   openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
-  int *evt_pending_head;
   u32 thread_id = ctx->c_thread_index;
 
   eidx = openssl_evt_alloc ();
   event = openssl_evt_get (eidx);
-
   event->ctx_index = oc->openssl_ctx_index;
-  event->status = SSL_ASYNC_PENDING;
+  event->event_idx = eidx;
+  event->thread_idx = thread_id;
   event->handler = handler;
-  event->cb_args.event_index = eidx;
-  event->cb_args.thread_index = thread_id;
-  event->engine_callback.callback = tls_async_openssl_callback;
-  event->engine_callback.arg = &event->cb_args;
-
-  /* add to pending list */
-  evt_pending_head = &om->status[thread_id].evt_pending_head;
-  event->next = *evt_pending_head;
-  *evt_pending_head = eidx;
+  event->session_index = session->session_index;
+  event->status = 0;
+  ctx->evt_index = eidx;
+#ifdef HAVE_OPENSSL_ASYNC
+  SSL_set_async_callback_arg (oc->ssl, &event->cb_args);
+#endif
 
-  return &event->engine_callback;
+  return 1;
 }
 
+
 int
-vpp_add_async_run_event (tls_ctx_t * ctx, openssl_resume_handler * handler)
+vpp_tls_async_update_event (tls_ctx_t * ctx, int eagain)
 {
   u32 eidx;
   openssl_evt_t *event;
-  openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
-  u32 thread_id = ctx->c_thread_index;
 
-  eidx = openssl_evt_alloc ();
-  event = openssl_evt_get (eidx);
-
-  event->ctx_index = oc->openssl_ctx_index;
-  event->status = SSL_ASYNC_PENDING;
-  event->handler = handler;
-  event->cb_args.event_index = eidx;
-  event->cb_args.thread_index = thread_id;
-  event->engine_callback.callback = tls_async_openssl_callback;
-  event->engine_callback.arg = &event->cb_args;
+  if (eagain)
+    {
+      eidx = ctx->evt_index;
+      event = openssl_evt_get (eidx);
 
-  /* This is a retry event, and need to put to ring to make it run again */
-  return tls_async_openssl_callback (NULL, &event->cb_args);
+      return tls_async_openssl_callback (0, &event->cb_args);
+    }
 
+  return 1;
 }
 
 void
 event_handler (void *tls_async)
 {
-
   openssl_resume_handler *handler;
-  openssl_evt_t *callback;
-  session_t *tls_session;
+  openssl_evt_t *event;
+  session_t *session;
   int thread_index;
   tls_ctx_t *ctx;
 
-  callback = (openssl_evt_t *) tls_async;
-  thread_index = callback->cb_args.thread_index;
-  ctx = openssl_ctx_get_w_thread (callback->ctx_index, thread_index);
-  handler = callback->handler;
-  tls_session = session_get_from_handle (ctx->tls_session_handle);
+  event = (openssl_evt_t *) tls_async;
+  thread_index = event->thread_idx;
+  ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index);
+  handler = event->handler;
+  session = session_get (event->session_index, thread_index);
 
   if (handler)
     {
-      (*handler) (ctx, tls_session);
+      (*handler) (ctx, session);
     }
 
-  /* Need to free the event */
-  openssl_evt_free (callback->cb_args.event_index, thread_index);
-
   return;
 }
 
@@ -381,19 +364,13 @@ qat_pre_init ()
 
 /* Below code is spefic to QAT engine, and other vendors can refer to this code to enable a new engine */
 void
-qat_polling_config ()
+qat_init_thread (void *arg)
 {
   openssl_async_t *om = &openssl_async_main;
-  u8 thread_index = vlib_get_thread_index ();
-  int *config;
-
-  config = &om->status[thread_index].poll_config;
-  if (PREDICT_TRUE (*config))
-    return;
+  int thread_index = *(int *) arg;
 
   ENGINE_ctrl_cmd (om->engine, "SET_INSTANCE_FOR_THREAD", thread_index,
 		   NULL, NULL, 0);
-  *config = 1;
 
   TLS_DBG (2, "set thread %d and instance %d mapping\n", thread_index,
 	   thread_index);
@@ -426,10 +403,16 @@ void
 openssl_async_node_enable_disable (u8 is_en)
 {
   u8 state = is_en ? VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_DISABLED;
+  vlib_thread_main_t *vtm = vlib_get_thread_main ();
+  u8 have_workers = vtm->n_threads != 0;
+
   /* *INDENT-OFF* */
   foreach_vlib_main (({
-    vlib_node_set_state (this_vlib_main, tls_async_process_node.index,
+    if (have_workers && ii != 0)
+      {
+        vlib_node_set_state (this_vlib_main, tls_async_process_node.index,
                          state);
+      }
   }));
   /* *INDENT-ON* */
 }
@@ -459,7 +442,8 @@ tls_resume_from_crypto (int thread_index)
 
   openssl_async_t *om = &openssl_async_main;
   openssl_evt_t *event;
-  int *evt_run_head = &om->status[thread_index].evt_run_head;
+  int *evt_run_head = &om->queue[thread_index].evt_run_head;
+  int *evt_run_tail = &om->queue[thread_index].evt_run_tail;
 
   if (*evt_run_head < 0)
     return 0;
@@ -469,15 +453,22 @@ tls_resume_from_crypto (int thread_index)
       if (*evt_run_head >= 0)
 	{
 	  event = openssl_evt_get_w_thread (*evt_run_head, thread_index);
-	  TLS_DBG (2, "event run = %d\n", *evt_run_head);
 	  tls_async_do_job (*evt_run_head, thread_index);
-
+	  if (PREDICT_FALSE (event->status == SSL_ASYNC_REENTER))
+	    {
+	      /* recusive event triggered */
+	      event->status = SSL_ASYNC_READY;
+	      continue;
+	    }
+
+	  event->status = 0;
 	  *evt_run_head = event->next;
 
-	}
-      else
-	{
-	  break;
+	  if (event->next < 0)
+	    {
+	      *evt_run_tail = -1;
+	      break;
+	    }
 	}
     }
 
@@ -490,7 +481,6 @@ tls_async_init (vlib_main_t * vm)
 {
   evt_pool_init (vm);
   return 0;
-
 }
 
 static uword
@@ -500,8 +490,6 @@ tls_async_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
   u8 thread_index;
   openssl_async_t *om = &openssl_async_main;
 
-  if (om->polling_conf)
-    (*om->polling_conf) ();
   thread_index = vlib_get_thread_index ();
   if (pool_elts (om->evt_pool[thread_index]) > 0)
     {
diff --git a/src/plugins/tlsopenssl/tls_openssl.c b/src/plugins/tlsopenssl/tls_openssl.c
index 70dfc3f256b..8e5e73a6cfb 100644
--- a/src/plugins/tlsopenssl/tls_openssl.c
+++ b/src/plugins/tlsopenssl/tls_openssl.c
@@ -59,6 +59,9 @@ openssl_ctx_free (tls_ctx_t * ctx)
 
   SSL_free (oc->ssl);
 
+#ifdef HAVE_OPENSSL_ASYNC
+  openssl_evt_free (ctx->evt_index, ctx->c_thread_index);
+#endif
   vec_free (ctx->srv_hostname);
   pool_put_index (openssl_main.ctx_pool[ctx->c_thread_index],
 		  oc->openssl_ctx_index);
@@ -176,31 +179,23 @@ openssl_try_handshake_write (openssl_ctx_t * oc, session_t * tls_session)
 
 #ifdef HAVE_OPENSSL_ASYNC
 static int
-vpp_ssl_async_process_event (tls_ctx_t * ctx,
-			     openssl_resume_handler * handler)
+openssl_check_async_status (tls_ctx_t * ctx, openssl_resume_handler * handler,
+			    session_t * session)
 {
   openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
-  openssl_tls_callback_t *engine_cb;
+  int estatus;
 
-  engine_cb = vpp_add_async_pending_event (ctx, handler);
-  if (engine_cb)
+  SSL_get_async_status (oc->ssl, &estatus);
+  if (estatus == ASYNC_STATUS_EAGAIN)
     {
-      SSL_set_async_callback_arg (oc->ssl, (void *) engine_cb->arg);
-      TLS_DBG (2, "set callback to engine %p\n", engine_cb->callback);
+      vpp_tls_async_update_event (ctx, 1);
+    }
+  else
+    {
+      vpp_tls_async_update_event (ctx, 0);
     }
-  return 0;
-
-}
-
-/* Due to engine busy stat, VPP need to retry later */
-static int
-vpp_ssl_async_retry_func (tls_ctx_t * ctx, openssl_resume_handler * handler)
-{
-
-  if (vpp_add_async_run_event (ctx, handler))
-    return 1;
 
-  return 0;
+  return 1;
 
 }
 
@@ -233,10 +228,6 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, session_t * tls_session)
 {
   openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
   int rv = 0, err;
-#ifdef HAVE_OPENSSL_ASYNC
-  int estatus;
-  openssl_resume_handler *myself;
-#endif
 
   while (SSL_in_init (oc->ssl))
     {
@@ -245,18 +236,18 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, session_t * tls_session)
 	  ctx->resume = 0;
 	}
       else if (!openssl_try_handshake_read (oc, tls_session))
-	{
-	  break;
-	}
-
-#ifdef HAVE_OPENSSL_ASYNC
-      myself = openssl_ctx_handshake_rx;
-      vpp_ssl_async_process_event (ctx, myself);
-#endif
+	break;
 
       rv = SSL_do_handshake (oc->ssl);
       err = SSL_get_error (oc->ssl, rv);
 
+#ifdef HAVE_OPENSSL_ASYNC
+      if (err == SSL_ERROR_WANT_ASYNC)
+	{
+	  openssl_check_async_status (ctx, openssl_ctx_handshake_rx,
+				      tls_session);
+	}
+#endif
       if (err == SSL_ERROR_SSL)
 	{
 	  char buf[512];
@@ -268,17 +259,6 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, session_t * tls_session)
 	}
 
       openssl_try_handshake_write (oc, tls_session);
-#ifdef HAVE_OPENSSL_ASYNC
-      if (err == SSL_ERROR_WANT_ASYNC)
-	{
-	  SSL_get_async_status (oc->ssl, &estatus);
-
-	  if (estatus == ASYNC_STATUS_EAGAIN)
-	    {
-	      vpp_ssl_async_retry_func (ctx, myself);
-	    }
-	}
-#endif
 
       if (err != SSL_ERROR_WANT_WRITE)
 	break;
@@ -287,7 +267,7 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, session_t * tls_session)
 	   SSL_state_string_long (oc->ssl));
 
   if (SSL_in_init (oc->ssl))
-    return 0;
+    return -1;
 
   /*
    * Handshake complete
@@ -507,9 +487,6 @@ openssl_ctx_init_client (tls_ctx_t * ctx)
   session_t *tls_session;
   const SSL_METHOD *method;
   int rv, err;
-#ifdef HAVE_OPENSSL_ASYNC
-  openssl_resume_handler *handler;
-#endif
 
   method = SSLv23_client_method ();
   if (method == NULL)
@@ -571,6 +548,10 @@ openssl_ctx_init_client (tls_ctx_t * ctx)
 	   oc->openssl_ctx_index);
 
   tls_session = session_get_from_handle (ctx->tls_session_handle);
+
+#ifdef HAVE_OPENSSL_ASYNC
+  vpp_tls_async_init_event (ctx, openssl_ctx_handshake_rx, tls_session);
+#endif
   while (1)
     {
       rv = SSL_do_handshake (oc->ssl);
@@ -579,8 +560,8 @@ openssl_ctx_init_client (tls_ctx_t * ctx)
 #ifdef HAVE_OPENSSL_ASYNC
       if (err == SSL_ERROR_WANT_ASYNC)
 	{
-	  handler = (openssl_resume_handler *) openssl_ctx_handshake_rx;
-	  vpp_ssl_async_process_event (ctx, handler);
+	  openssl_check_async_status (ctx, openssl_ctx_handshake_rx,
+				      tls_session);
 	  break;
 	}
 #endif
@@ -631,8 +612,10 @@ openssl_start_listen (tls_ctx_t * lctx)
   SSL_CTX_set_mode (ssl_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
 #ifdef HAVE_OPENSSL_ASYNC
   if (om->async)
-    SSL_CTX_set_mode (ssl_ctx, SSL_MODE_ASYNC);
-  SSL_CTX_set_async_callback (ssl_ctx, tls_async_openssl_callback);
+    {
+      SSL_CTX_set_mode (ssl_ctx, SSL_MODE_ASYNC);
+      SSL_CTX_set_async_callback (ssl_ctx, tls_async_openssl_callback);
+    }
 #endif
   SSL_CTX_set_options (ssl_ctx, flags);
   SSL_CTX_set_ecdh_auto (ssl_ctx, 1);
@@ -708,9 +691,6 @@ openssl_ctx_init_server (tls_ctx_t * ctx)
   openssl_listen_ctx_t *olc;
   session_t *tls_session;
   int rv, err;
-#ifdef HAVE_OPENSSL_ASYNC
-  openssl_resume_handler *handler;
-#endif
 
   /* Start a new connection */
 
@@ -735,6 +715,9 @@ openssl_ctx_init_server (tls_ctx_t * ctx)
 	   oc->openssl_ctx_index);
 
   tls_session = session_get_from_handle (ctx->tls_session_handle);
+#ifdef HAVE_OPENSSL_ASYNC
+  vpp_tls_async_init_event (ctx, openssl_ctx_handshake_rx, tls_session);
+#endif
   while (1)
     {
       rv = SSL_do_handshake (oc->ssl);
@@ -743,8 +726,8 @@ openssl_ctx_init_server (tls_ctx_t * ctx)
 #ifdef HAVE_OPENSSL_ASYNC
       if (err == SSL_ERROR_WANT_ASYNC)
 	{
-	  handler = (openssl_resume_handler *) openssl_ctx_handshake_rx;
-	  vpp_ssl_async_process_event (ctx, handler);
+	  openssl_check_async_status (ctx, openssl_ctx_handshake_rx,
+				      tls_session);
 	  break;
 	}
 #endif
@@ -928,7 +911,7 @@ tls_openssl_set_command_fn (vlib_main_t * vm, unformat_input_t * input,
   char *engine_alg = NULL;
   char *ciphers = NULL;
   u8 engine_name_set = 0;
-  int i;
+  int i, async = 0;
 
   /* By present, it is not allowed to configure engine again after running */
   if (om->engine_init)
@@ -946,8 +929,7 @@ tls_openssl_set_command_fn (vlib_main_t * vm, unformat_input_t * input,
 	}
       else if (unformat (input, "async"))
 	{
-	  om->async = 1;
-	  openssl_async_node_enable_disable (1);
+	  async = 1;
 	}
       else if (unformat (input, "alg %s", &engine_alg))
 	{
@@ -967,16 +949,23 @@ tls_openssl_set_command_fn (vlib_main_t * vm, unformat_input_t * input,
   if (!engine_name_set)
     {
       clib_warning ("No engine provided! \n");
-      om->async = 0;
+      async = 0;
     }
   else
     {
-      if (openssl_engine_register (engine_name, engine_alg) < 0)
+      vnet_session_enable_disable (vm, 1);
+      if (openssl_engine_register (engine_name, engine_alg, async) < 0)
 	{
-	  return clib_error_return (0, "failed to register %s polling",
+	  return clib_error_return (0, "Failed to register %s polling",
 				    engine_name);
 	}
+      else
+	{
+	  vlib_cli_output (vm, "Successfully register engine %s\n",
+			   engine_name);
+	}
     }
+  om->async = async;
 
   return 0;
 }
diff --git a/src/plugins/tlsopenssl/tls_openssl.h b/src/plugins/tlsopenssl/tls_openssl.h
index e392b9ae4f2..f61d986c3b0 100644
--- a/src/plugins/tlsopenssl/tls_openssl.h
+++ b/src/plugins/tlsopenssl/tls_openssl.h
@@ -54,23 +54,17 @@ typedef struct openssl_main_
   int async;
 } openssl_main_t;
 
-typedef struct openssl_tls_callback_
-{
-  int (*callback) (SSL * ssl, void *arg);
-  void *arg;
-} openssl_tls_callback_t;
-
 typedef int openssl_resume_handler (tls_ctx_t * ctx, session_t * tls_session);
 
 tls_ctx_t *openssl_ctx_get_w_thread (u32 ctx_index, u8 thread_index);
-openssl_tls_callback_t *vpp_add_async_pending_event (tls_ctx_t * ctx,
-						     openssl_resume_handler *
-						     handler);
-int vpp_add_async_run_event (tls_ctx_t * ctx, openssl_resume_handler *
-			     handler);
+int vpp_tls_async_init_event (tls_ctx_t * ctx,
+			      openssl_resume_handler * handler,
+			      session_t * session);
+int vpp_tls_async_update_event (tls_ctx_t * ctx, int eagain);
 int tls_async_openssl_callback (SSL * s, void *evt);
+int openssl_evt_free (int event_idx, u8 thread_index);
 void openssl_polling_start (ENGINE * engine);
-int openssl_engine_register (char *engine, char *alg);
+int openssl_engine_register (char *engine, char *alg, int async);
 void openssl_async_node_enable_disable (u8 is_en);
 clib_error_t *tls_openssl_api_init (vlib_main_t * vm);
 int tls_openssl_set_ciphers (char *ciphers);
diff --git a/src/plugins/tlsopenssl/tls_openssl_api.c b/src/plugins/tlsopenssl/tls_openssl_api.c
index b305bf2677f..c34829f0b29 100644
--- a/src/plugins/tlsopenssl/tls_openssl_api.c
+++ b/src/plugins/tlsopenssl/tls_openssl_api.c
@@ -40,12 +40,6 @@ vl_api_tls_openssl_set_engine_t_handler (vl_api_tls_openssl_set_engine_t * mp)
   char *ciphers;
   int rv;
 
-  if (mp->async_enable)
-    {
-      om->async = 1;
-      openssl_async_node_enable_disable (1);
-    }
-
   ciphers = (char *) &mp->ciphers;
   ciphers[63] = '\0';
   if (ciphers[0])
@@ -55,7 +49,8 @@ vl_api_tls_openssl_set_engine_t_handler (vl_api_tls_openssl_set_engine_t * mp)
   engine[63] = '\0';
   alg = (char *) mp->algorithm;
   alg[63] = '\0';
-  rv = openssl_engine_register (engine, alg);
+  rv = openssl_engine_register (engine, alg, mp->async_enable);
+  om->async = mp->async_enable;
 
   REPLY_MACRO (VL_API_TLS_OPENSSL_SET_ENGINE_REPLY);
 }
diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c
index 8273e15585d..8359cca5c00 100644
--- a/src/vnet/tls/tls.c
+++ b/src/vnet/tls/tls.c
@@ -211,8 +211,6 @@ tls_notify_app_accept (tls_ctx_t * ctx)
       return rv;
     }
   ctx->app_session_handle = session_handle (app_session);
-  session_lookup_add_connection (&ctx->connection,
-				 session_handle (app_session));
   ctx->parent_app_wrk_index = app_session->app_wrk_index;
   app_wrk = app_worker_get (app_session->app_wrk_index);
   return app_worker_accept_notify (app_wrk, app_session);
diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h
index 31572e8c463..15d06db6563 100644
--- a/src/vnet/tls/tls.h
+++ b/src/vnet/tls/tls.h
@@ -79,6 +79,7 @@ typedef struct tls_ctx_
   u8 app_closed;
   u8 no_app_session;
   u8 *srv_hostname;
+  u32 evt_index;
   u32 ckpair_index;
 } tls_ctx_t;