TLS async support

Change-Id: I26194e00dfb85e5cd1c65ff4e6ffd665be2d719b
Signed-off-by: Ping Yu <ping.yu@intel.com>

3 files changed, 735 insertions, 22 deletions

diff --git a/src/plugins/tlsopenssl/tls_async.c b/src/plugins/tlsopenssl/tls_async.c
new file mode 100644
index 00000000000..aec1d7dcee2
--- /dev/null
+++ b/src/plugins/tlsopenssl/tls_async.c
@@ -0,0 +1,523 @@
+/*
+ * Copyright (c) 2018 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <vnet/vnet.h>
+#include <vnet/ip/ip.h>
+#include <vnet/api_errno.h>
+#include <vnet/ipsec/ipsec.h>
+#include <vlib/node_funcs.h>
+#include <openssl/engine.h>
+#include <tlsopenssl/tls_openssl.h>
+
+#define MAX_SESSION	    4096
+#define MAX_VECTOR_ASYNC    256
+
+#define SSL_ASYNC_INFLIGHT  1
+#define SSL_ASYNC_PENDING   2
+#define SSL_ASYNC_READY     3
+
+#define EMPTY_STRUCT {0}
+
+typedef struct openssl_tls_callback_arg_
+{
+  int thread_index;
+  int event_index;
+} openssl_tls_callback_arg_t;
+
+typedef struct openssl_event_
+{
+  int status;
+  u32 event_index;
+  u8 thread_index;
+  u32 ctx_index;
+
+  openssl_resume_handler *handler;
+  openssl_tls_callback_t engine_callback;
+  openssl_tls_callback_arg_t cb_args;
+
+  int next;
+} openssl_evt_t;
+
+typedef struct openssl_async_status_
+{
+  int evt_run_head;
+  int evt_run_tail;
+  int evt_pending_head;
+  int poll_config;
+} openssl_async_status_t;
+
+typedef struct openssl_async_
+{
+  openssl_evt_t ***evt_pool;
+  openssl_async_status_t *status;
+  void (*polling) (void);
+  u8 start_polling;
+  ENGINE *engine;
+
+} openssl_async_t;
+
+void qat_polling ();
+void qat_pre_init ();
+void dasync_polling ();
+
+struct engine_polling
+{
+  char *engine;
+  void (*polling) (void);
+  void (*pre_init) (void);
+};
+
+struct engine_polling engine_list[] = {
+  {"qat", qat_polling, qat_pre_init},
+  {"dasync", dasync_polling, NULL}
+};
+
+openssl_async_t openssl_async_main;
+static vlib_node_registration_t tls_async_process_node;
+
+/* to avoid build warning */
+void session_send_rpc_evt_to_thread (u32 thread_index, void *fp,
+				     void *rpc_args);
+
+void
+evt_pool_init (vlib_main_t * vm)
+{
+  vlib_thread_main_t *vtm = vlib_get_thread_main ();
+  openssl_async_t *om = &openssl_async_main;
+  int i, num_threads;
+
+  num_threads = 1 /* main thread */  + vtm->n_threads;
+
+  TLS_DBG ("Totally there is %d thread\n", num_threads);
+
+  vec_validate (om->evt_pool, num_threads - 1);
+  vec_validate (om->status, num_threads - 1);
+
+  om->start_polling = 0;
+  om->engine = 0;
+
+  for (i = 0; i < num_threads; i++)
+    {
+      om->status[i].evt_run_head = -1;
+      om->status[i].evt_run_tail = -1;
+      om->status[i].evt_pending_head = -1;
+    }
+  om->polling = NULL;
+
+  TLS_DBG ("Node disabled\n");
+  openssl_async_node_enable_disable (0);
+
+  return;
+}
+
+int
+openssl_engine_register (char *engine_name, char *algorithm)
+{
+  int i, registered = -1;
+  openssl_async_t *om = &openssl_async_main;
+  void (*p) (void);
+  ENGINE *engine;
+
+  for (i = 0; i < ARRAY_LEN (engine_list); i++)
+    {
+      if (!strcmp (engine_list[i].engine, engine_name))
+	{
+	  om->polling = engine_list[i].polling;
+
+	  registered = i;
+	}
+    }
+  if (registered < 0)
+    {
+      return 0;
+    }
+
+  ENGINE_load_builtin_engines ();
+  ENGINE_load_dynamic ();
+  engine = ENGINE_by_id (engine_name);
+
+  if (engine == NULL)
+    {
+      return 0;
+    }
+
+  om->engine = engine;
+  /* call pre-init */
+  p = engine_list[registered].pre_init;
+  if (p)
+    (*p) ();
+
+  if (algorithm)
+    {
+      if (!ENGINE_set_default_string (engine, algorithm))
+	{
+	  clib_warning ("Failed to set engine %s algorithm %s\n",
+			engine_name, algorithm);
+	  return 0;
+	}
+    }
+  else
+    {
+      if (!ENGINE_set_default (engine, ENGINE_METHOD_ALL))
+	{
+	  clib_warning ("Failed to set engine %s to all algorithm",
+			engine_name);
+	  return 0;
+	}
+    }
+
+  om->start_polling = 1;
+
+  return 1;
+
+}
+
+static openssl_evt_t *
+openssl_evt_get (u32 evt_index)
+{
+  openssl_evt_t **evt;
+  evt =
+    pool_elt_at_index (openssl_async_main.evt_pool[vlib_get_thread_index ()],
+		       evt_index);
+  return *evt;
+}
+
+static openssl_evt_t *
+openssl_evt_get_w_thread (int evt_index, u8 thread_index)
+{
+  openssl_evt_t **evt;
+
+  evt =
+    pool_elt_at_index (openssl_async_main.evt_pool[thread_index], evt_index);
+  return *evt;
+}
+
+int
+openssl_evt_free (int event_idx, u8 thread_index)
+{
+  openssl_evt_t *evt;
+  openssl_async_t *om = &openssl_async_main;
+  int *evt_run_tail = &om->status[thread_index].evt_run_tail;
+
+  if (event_idx < 0)
+    return 0;
+
+  evt = openssl_evt_get_w_thread (event_idx, thread_index);
+
+  evt->status = 0;
+
+  /*pool operation */
+  pool_put_index (om->evt_pool[thread_index], event_idx);
+
+  if (*evt_run_tail == event_idx)
+    *evt_run_tail = -1;
+
+  return 1;
+}
+
+static u32
+openssl_evt_alloc (void)
+{
+  u8 thread_index = vlib_get_thread_index ();
+  openssl_async_t *tm = &openssl_async_main;
+  openssl_evt_t **evt;
+
+  pool_get (tm->evt_pool[thread_index], evt);
+  if (!(*evt))
+    *evt = clib_mem_alloc (sizeof (openssl_evt_t));
+
+  memset (*evt, 0, sizeof (openssl_evt_t));
+  (*evt)->event_index = evt - tm->evt_pool[thread_index];
+  return ((*evt)->event_index);
+}
+
+int
+openssl_async_run (void *evt)
+{
+  openssl_evt_t *event, *event_tail;
+  openssl_async_t *om = &openssl_async_main;
+  openssl_tls_callback_arg_t *args = (openssl_tls_callback_arg_t *) evt;
+  int thread_index = args->thread_index;
+  int event_index = args->event_index;
+  int *evt_run_tail = &om->status[thread_index].evt_run_tail;
+  int *evt_run_head = &om->status[thread_index].evt_run_head;
+
+  TLS_DBG ("Set event %d to run\n", event_index);
+
+  event = openssl_evt_get_w_thread (event_index, thread_index);
+
+  if (event->status == SSL_ASYNC_READY)
+    return 0;
+
+  event->status = SSL_ASYNC_READY;
+  event->next = -1;
+
+
+  if (*evt_run_tail >= 0)
+    {
+      event_tail = openssl_evt_get_w_thread (*evt_run_tail, thread_index);
+      event_tail->next = event_index;
+    }
+  *evt_run_tail = event_index;
+  if (*evt_run_head < 0)
+    *evt_run_head = event_index;
+
+  return 1;
+}
+
+openssl_tls_callback_t *
+vpp_add_async_pending_event (tls_ctx_t * ctx,
+			     openssl_resume_handler * handler)
+{
+  u32 eidx;
+  openssl_evt_t *event;
+  openssl_async_t *om = &openssl_async_main;
+  openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
+  int *evt_pending_head;
+  u32 thread_id = ctx->c_thread_index;
+
+  eidx = openssl_evt_alloc ();
+  event = openssl_evt_get (eidx);
+
+  event->ctx_index = oc->openssl_ctx_index;
+  event->status = SSL_ASYNC_PENDING;
+  event->handler = handler;
+  event->cb_args.event_index = eidx;
+  event->cb_args.thread_index = thread_id;
+  event->engine_callback.callback = openssl_async_run;
+  event->engine_callback.arg = &event->cb_args;
+
+  /* add to pending list */
+  evt_pending_head = &om->status[thread_id].evt_pending_head;
+  event->next = *evt_pending_head;
+  *evt_pending_head = eidx;
+
+
+  return &event->engine_callback;
+}
+
+void
+event_handler (void *tls_async)
+{
+
+  openssl_resume_handler *handler;
+  openssl_evt_t *callback;
+  stream_session_t *tls_session;
+  int thread_index;
+  tls_ctx_t *ctx;
+
+  callback = (openssl_evt_t *) tls_async;
+  thread_index = callback->cb_args.thread_index;
+  ctx = openssl_ctx_get_w_thread (callback->ctx_index, thread_index);
+  handler = callback->handler;
+  tls_session = session_get_from_handle (ctx->tls_session_handle);
+
+  if (handler)
+    {
+      TLS_DBG ("relaunch...\n");
+      (*handler) (ctx, tls_session);
+    }
+
+  /* Need to free the event */
+  openssl_evt_free (callback->cb_args.event_index, thread_index);
+
+  return;
+}
+
+ /* engine specific code to polling the response ring */
+void
+dasync_polling ()
+{
+  openssl_async_t *om = &openssl_async_main;
+  openssl_evt_t *event;
+  int *evt_pending;
+  openssl_tls_callback_t *engine_cb;
+  u8 thread_index = vlib_get_thread_index ();
+
+  /* POC code here to simulate the engine to call callback */
+  evt_pending = &om->status[thread_index].evt_pending_head;
+  while (*evt_pending >= 0)
+    {
+      TLS_DBG ("polling... current head = %d\n", *evt_pending);
+      event = openssl_evt_get_w_thread (*evt_pending, thread_index);
+      *evt_pending = event->next;
+      if (event->status == SSL_ASYNC_PENDING)
+	{
+	  engine_cb = &event->engine_callback;
+	  (*engine_cb->callback) (engine_cb->arg);
+	}
+    }
+
+}
+
+void
+qat_pre_init ()
+{
+  openssl_async_t *om = &openssl_async_main;
+
+  ENGINE_ctrl_cmd (om->engine, "ENABLE_EXTERNAL_POLLING", 0, NULL, NULL, 0);
+}
+
+/* Below code is spefic to QAT engine, and other vendors can refer to this code to enable a new engine */
+void
+qat_polling_config ()
+{
+  openssl_async_t *om = &openssl_async_main;
+  u8 thread_index = vlib_get_thread_index ();
+  int *config;
+
+  config = &om->status[thread_index].poll_config;
+  if (*config)
+    return;
+
+  ENGINE_ctrl_cmd (om->engine, "SET_INSTANCE_FOR_THREAD", thread_index,
+		   NULL, NULL, 0);
+  *config = 1;
+
+  TLS_DBG ("set thread %d and instance %d mapping\n", thread_index,
+	   thread_index);
+
+}
+
+void
+qat_polling ()
+{
+  openssl_async_t *om = &openssl_async_main;
+  int ret;
+
+  if (om->start_polling)
+    {
+      qat_polling_config ();
+#define QAT_CMD_POLL (ENGINE_CMD_BASE + 1)
+      ENGINE_ctrl (om->engine, QAT_CMD_POLL, 0, &ret, NULL);
+      ;
+    }
+}
+
+void
+openssl_async_polling ()
+{
+  openssl_async_t *om = &openssl_async_main;
+  if (om->polling)
+    {
+      (*om->polling) ();
+    }
+}
+
+void
+openssl_async_node_enable_disable (u8 is_en)
+{
+  u8 state = is_en ? VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_DISABLED;
+  /* *INDENT-OFF* */
+  foreach_vlib_main (({
+    vlib_node_set_state (this_vlib_main, tls_async_process_node.index,
+                         state);
+  }));
+  /* *INDENT-ON* */
+}
+
+int
+tls_async_do_job (int eidx, u32 thread_index)
+{
+  tls_ctx_t *ctx;
+  openssl_evt_t *event;
+
+  /* do the real job */
+  event = openssl_evt_get_w_thread (eidx, thread_index);
+  ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index);
+
+  if (ctx)
+    {
+      ctx->resume = 1;
+      session_send_rpc_evt_to_thread (thread_index, event_handler, event);
+    }
+  return 1;
+}
+
+int
+tls_resume_from_crypto (int thread_index)
+{
+  int i;
+
+  openssl_async_t *om = &openssl_async_main;
+  openssl_evt_t *event;
+  int *evt_run_head = &om->status[thread_index].evt_run_head;
+
+  if (*evt_run_head < 0)
+    return 0;
+
+  for (i = 0; i < MAX_VECTOR_ASYNC; i++)
+    {
+      if (*evt_run_head >= 0)
+	{
+	  event = openssl_evt_get_w_thread (*evt_run_head, thread_index);
+	  TLS_DBG ("event run = %d\n", *evt_run_head);
+	  tls_async_do_job (*evt_run_head, thread_index);
+
+	  *evt_run_head = event->next;
+
+	}
+      else
+	{
+	  break;
+	}
+    }
+
+  return 0;
+
+}
+
+static clib_error_t *
+tls_async_init (vlib_main_t * vm)
+{
+
+  TLS_DBG ("Start to call tls_async_init\n");
+  evt_pool_init (vm);
+  return 0;
+
+}
+
+static uword
+tls_async_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
+		   vlib_frame_t * f)
+{
+  u8 thread_index;
+
+  thread_index = vlib_get_thread_index ();
+  openssl_async_polling ();
+
+  tls_resume_from_crypto (thread_index);
+
+  return 0;
+}
+
+VLIB_INIT_FUNCTION (tls_async_init);
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (tls_async_process_node,static) = {
+    .function = tls_async_process,
+    .type = VLIB_NODE_TYPE_INPUT,
+    .name = "tls-async-process",
+};
+
+
+/* *INDENT-ON* */
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/plugins/tlsopenssl/tls_openssl.c b/src/plugins/tlsopenssl/tls_openssl.c
index 596dba9977f..f5722917bcc 100644
--- a/src/plugins/tlsopenssl/tls_openssl.c
+++ b/src/plugins/tlsopenssl/tls_openssl.c
@@ -16,30 +16,19 @@
 #include <openssl/ssl.h>
 #include <openssl/conf.h>
 #include <openssl/err.h>
+#ifdef HAVE_OPENSSL_ASYNC
+#include <openssl/async.h>
+#endif
+#include <dlfcn.h>
 #include <vnet/plugin/plugin.h>
 #include <vpp/app/version.h>
 #include <vnet/tls/tls.h>
+#include <ctype.h>
+#include <tlsopenssl/tls_openssl.h>
 
-typedef struct tls_ctx_openssl_
-{
-  tls_ctx_t ctx;			/**< First */
-  u32 openssl_ctx_index;
-  SSL_CTX *ssl_ctx;
-  SSL *ssl;
-  BIO *rbio;
-  BIO *wbio;
-  X509 *srvcert;
-  EVP_PKEY *pkey;
-} openssl_ctx_t;
-
-typedef struct openssl_main_
-{
-  openssl_ctx_t ***ctx_pool;
-  X509_STORE *cert_store;
-} openssl_main_t;
+#define MAX_CRYPTO_LEN 16
 
 static openssl_main_t openssl_main;
-
 static u32
 openssl_ctx_alloc (void)
 {
@@ -78,7 +67,7 @@ openssl_ctx_free (tls_ctx_t * ctx)
 		  oc->openssl_ctx_index);
 }
 
-static tls_ctx_t *
+tls_ctx_t *
 openssl_ctx_get (u32 ctx_index)
 {
   openssl_ctx_t **ctx;
@@ -87,7 +76,7 @@ openssl_ctx_get (u32 ctx_index)
   return &(*ctx)->ctx;
 }
 
-static tls_ctx_t *
+tls_ctx_t *
 openssl_ctx_get_w_thread (u32 ctx_index, u8 thread_index)
 {
   openssl_ctx_t **ctx;
@@ -165,19 +154,59 @@ openssl_try_handshake_write (openssl_ctx_t * oc,
   return read;
 }
 
+#ifdef HAVE_OPENSSL_ASYNC
 static int
+vpp_ssl_async_process_event (tls_ctx_t * ctx,
+			     openssl_resume_handler * handler)
+{
+  openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
+  openssl_tls_callback_t *engine_cb;
+
+  engine_cb = vpp_add_async_pending_event (ctx, handler);
+  if (engine_cb)
+    {
+      SSL_set_async_callback (oc->ssl, (void *) engine_cb->callback,
+			      (void *) engine_cb->arg);
+      TLS_DBG ("set callback to engine %p\n", engine_cb->callback);
+    }
+  /* associated fd with context for return */
+  TLS_DBG ("completed assoicated fd with tls session\n");
+  return 0;
+
+}
+#endif
+
+int
 openssl_ctx_handshake_rx (tls_ctx_t * ctx, stream_session_t * tls_session)
 {
   openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
   int rv = 0, err;
+#ifdef HAVE_OPENSSL_ASYNC
+  openssl_resume_handler *myself;
+#endif
+
   while (SSL_in_init (oc->ssl))
     {
-      if (!openssl_try_handshake_read (oc, tls_session))
-	break;
+      if (ctx->resume)
+	{
+	  ctx->resume = 0;
+	}
+      else if (!openssl_try_handshake_read (oc, tls_session))
+	{
+	  break;
+	}
 
       rv = SSL_do_handshake (oc->ssl);
       err = SSL_get_error (oc->ssl, rv);
       openssl_try_handshake_write (oc, tls_session);
+#ifdef HAVE_OPENSSL_ASYNC
+      if (err == SSL_ERROR_WANT_ASYNC)
+	{
+	  myself = openssl_ctx_handshake_rx;
+	  vpp_ssl_async_process_event (ctx, myself);
+	}
+#endif
+
       if (err != SSL_ERROR_WANT_WRITE)
 	{
 	  if (err == SSL_ERROR_SSL)
@@ -480,6 +509,10 @@ openssl_ctx_init_server (tls_ctx_t * ctx)
   application_t *app;
   int rv, err;
   BIO *cert_bio;
+#ifdef HAVE_OPENSSL_ASYNC
+  openssl_main_t *om = &openssl_main;
+  openssl_resume_handler *handler;
+#endif
 
   app = application_get (ctx->parent_app_index);
   if (!app->tls_cert || !app->tls_key)
@@ -498,6 +531,10 @@ openssl_ctx_init_server (tls_ctx_t * ctx)
     }
 
   SSL_CTX_set_mode (oc->ssl_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
+#ifdef HAVE_OPENSSL_ASYNC
+  if (om->async)
+    SSL_CTX_set_mode (oc->ssl_ctx, SSL_MODE_ASYNC);
+#endif
   SSL_CTX_set_options (oc->ssl_ctx, flags);
   SSL_CTX_set_ecdh_auto (oc->ssl_ctx, 1);
 
@@ -521,6 +558,8 @@ openssl_ctx_init_server (tls_ctx_t * ctx)
     }
   SSL_CTX_use_certificate (oc->ssl_ctx, oc->srvcert);
   BIO_free (cert_bio);
+
+
   cert_bio = BIO_new (BIO_s_mem ());
   BIO_write (cert_bio, app->tls_key, vec_len (app->tls_key));
   oc->pkey = PEM_read_bio_PrivateKey (cert_bio, NULL, NULL, NULL);
@@ -529,10 +568,13 @@ openssl_ctx_init_server (tls_ctx_t * ctx)
       clib_warning ("unable to parse pkey");
       return -1;
     }
+  SSL_CTX_use_PrivateKey (oc->ssl_ctx, oc->pkey);
 
   SSL_CTX_use_PrivateKey (oc->ssl_ctx, oc->pkey);
   BIO_free (cert_bio);
 
+  /* Start a new connection */
+
   oc->ssl = SSL_new (oc->ssl_ctx);
   if (oc->ssl == NULL)
     {
@@ -558,6 +600,14 @@ openssl_ctx_init_server (tls_ctx_t * ctx)
       rv = SSL_do_handshake (oc->ssl);
       err = SSL_get_error (oc->ssl, rv);
       openssl_try_handshake_write (oc, tls_session);
+#ifdef HAVE_OPENSSL_ASYNC
+      if (err == SSL_ERROR_WANT_ASYNC)
+	{
+	  handler = (openssl_resume_handler *) openssl_ctx_handshake_rx;
+	  vpp_ssl_async_process_event (ctx, handler);
+	  break;
+	}
+#endif
       if (err != SSL_ERROR_WANT_WRITE)
 	break;
     }
@@ -656,9 +706,81 @@ tls_openssl_init (vlib_main_t * vm)
   vec_validate (om->ctx_pool, num_threads - 1);
 
   tls_register_engine (&openssl_engine, TLS_ENGINE_OPENSSL);
+
+  om->engine_init = 0;
+
   return 0;
 }
 
+#ifdef HAVE_OPENSSL_ASYNC
+static clib_error_t *
+tls_openssl_set_command_fn (vlib_main_t * vm, unformat_input_t * input,
+			    vlib_cli_command_t * cmd)
+{
+  openssl_main_t *om = &openssl_main;
+  char *engine_name = NULL;
+  char *engine_alg = NULL;
+  u8 engine_name_set = 0;
+  int i;
+
+  /* By present, it is not allowed to configure engine again after running */
+  if (om->engine_init)
+    {
+      clib_warning ("engine has started!\n");
+      return clib_error_return
+	(0, "engine has started, and no config is accepted");
+    }
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "engine %s", &engine_name))
+	{
+	  engine_name_set = 1;
+	}
+      else if (unformat (input, "async"))
+	{
+	  om->async = 1;
+	  openssl_async_node_enable_disable (1);
+	}
+      else if (unformat (input, "alg %s", &engine_alg))
+	{
+	  for (i = 0; i < strnlen (engine_alg, MAX_CRYPTO_LEN); i++)
+	    engine_alg[i] = toupper (engine_alg[i]);
+	}
+      else
+	return clib_error_return (0, "failed: unknown input `%U'",
+				  format_unformat_error, input);
+    }
+
+  /* reset parameters if engine is not configured */
+  if (!engine_name_set)
+    {
+      clib_warning ("No engine provided! \n");
+      om->async = 0;
+    }
+  else
+    {
+      if (!openssl_engine_register (engine_name, engine_alg))
+	{
+	  return clib_error_return (0, "failed to register %s polling",
+				    engine_name);
+	}
+    }
+
+  return 0;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (tls_openssl_set_command, static) =
+{
+  .path = "tls openssl set",
+  .short_help = "tls openssl set [engine <engine name>] [alg [algorithm] [async]",
+  .function = tls_openssl_set_command_fn,
+};
+/* *INDENT-ON* */
+#endif
+
+
 VLIB_INIT_FUNCTION (tls_openssl_init);
 
 /* *INDENT-OFF* */
diff --git a/src/plugins/tlsopenssl/tls_openssl.h b/src/plugins/tlsopenssl/tls_openssl.h
new file mode 100644
index 00000000000..b01814939af
--- /dev/null
+++ b/src/plugins/tlsopenssl/tls_openssl.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2018 Intel and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <openssl/ssl.h>
+#include <openssl/conf.h>
+#include <openssl/err.h>
+#include <openssl/engine.h>
+#include <vnet/plugin/plugin.h>
+#include <vpp/app/version.h>
+#include <vnet/tls/tls.h>
+
+typedef struct tls_ctx_openssl_
+{
+  tls_ctx_t ctx;			/**< First */
+  u32 openssl_ctx_index;
+  SSL_CTX *ssl_ctx;
+  SSL *ssl;
+  BIO *rbio;
+  BIO *wbio;
+  X509 *srvcert;
+  EVP_PKEY *pkey;
+} openssl_ctx_t;
+
+typedef struct openssl_main_
+{
+  openssl_ctx_t ***ctx_pool;
+
+  X509_STORE *cert_store;
+  int engine_init;
+  int async;
+} openssl_main_t;
+
+typedef struct openssl_tls_callback_
+{
+  int (*callback) (void *arg);
+  void *arg;
+} openssl_tls_callback_t;
+
+typedef int openssl_resume_handler (tls_ctx_t * ctx,
+				    stream_session_t * tls_session);
+
+tls_ctx_t *openssl_ctx_get_w_thread (u32 ctx_index, u8 thread_index);
+openssl_tls_callback_t *vpp_add_async_pending_event (tls_ctx_t * ctx,
+						     openssl_resume_handler *
+						     handler);
+void openssl_polling_start (ENGINE * engine);
+int openssl_engine_register (char *engine, char *alg);
+void openssl_async_node_enable_disable (u8 is_en);
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */