optimize init_server to reduce session overhead

move un-necessary session based operation to listener
split orignal openssl ctx to be session based ctx and listen ctx

Change-Id: Id6c54f47b0e2171fd8924a45efcd5266ce5402d5
Signed-off-by: Ping Yu <ping.yu@intel.com>

5 files changed, 133 insertions, 33 deletions

diff --git a/src/plugins/tlsmbedtls/tls_mbedtls.c b/src/plugins/tlsmbedtls/tls_mbedtls.c
index aaad99f0fdb..57a6e486c3f 100644
--- a/src/plugins/tlsmbedtls/tls_mbedtls.c
+++ b/src/plugins/tlsmbedtls/tls_mbedtls.c
@@ -259,6 +259,18 @@ mbedtls_ctx_init_client (tls_ctx_t * ctx)
 }
 
 static int
+mbedtls_start_listen (tls_ctx_t * lctx)
+{
+  return 0;
+}
+
+static int
+mbedtls_stop_listen (tls_ctx_t * lctx)
+{
+  return 0;
+}
+
+static int
 mbedtls_ctx_init_server (tls_ctx_t * ctx)
 {
   mbedtls_ctx_t *mc = (mbedtls_ctx_t *) ctx;
@@ -530,6 +542,8 @@ const static tls_engine_vft_t mbedtls_engine = {
   .ctx_write = mbedtls_ctx_write,
   .ctx_read = mbedtls_ctx_read,
   .ctx_handshake_is_over = mbedtls_handshake_is_over,
+  .ctx_start_listen = mbedtls_start_listen,
+  .ctx_stop_listen = mbedtls_stop_listen,
 };
 
 int
diff --git a/src/plugins/tlsopenssl/tls_openssl.c b/src/plugins/tlsopenssl/tls_openssl.c
index 73e5cc0af35..b38b9618b9f 100644
--- a/src/plugins/tlsopenssl/tls_openssl.c
+++ b/src/plugins/tlsopenssl/tls_openssl.c
@@ -56,11 +56,6 @@ openssl_ctx_free (tls_ctx_t * ctx)
   if (SSL_is_init_finished (oc->ssl) && !ctx->is_passive_close)
     SSL_shutdown (oc->ssl);
 
-  if (SSL_is_server (oc->ssl))
-    {
-      X509_free (oc->srvcert);
-      EVP_PKEY_free (oc->pkey);
-    }
   SSL_free (oc->ssl);
 
   pool_put_index (openssl_main.ctx_pool[ctx->c_thread_index],
@@ -84,6 +79,31 @@ openssl_ctx_get_w_thread (u32 ctx_index, u8 thread_index)
   return &(*ctx)->ctx;
 }
 
+static u32
+openssl_listen_ctx_alloc (void)
+{
+  openssl_main_t *om = &openssl_main;
+  openssl_listen_ctx_t *lctx;
+
+  pool_get (om->lctx_pool, lctx);
+
+  memset (lctx, 0, sizeof (openssl_listen_ctx_t));
+  lctx->openssl_lctx_index = lctx - om->lctx_pool;
+  return lctx->openssl_lctx_index;
+}
+
+static void
+openssl_listen_ctx_free (openssl_listen_ctx_t * lctx)
+{
+  pool_put_index (openssl_main.lctx_pool, lctx->openssl_lctx_index);
+}
+
+openssl_listen_ctx_t *
+openssl_lctx_get (u32 lctx_index)
+{
+  return pool_elt_at_index (openssl_main.lctx_pool, lctx_index);
+}
+
 static int
 openssl_try_handshake_read (openssl_ctx_t * oc,
 			    stream_session_t * tls_session)
@@ -533,46 +553,49 @@ openssl_ctx_init_client (tls_ctx_t * ctx)
 }
 
 static int
-openssl_ctx_init_server (tls_ctx_t * ctx)
+openssl_start_listen (tls_ctx_t * lctx)
 {
-  char *ciphers = "ALL:!ADH:!LOW:!EXP:!MD5:!RC4-SHA:!DES-CBC3-SHA:@STRENGTH";
-  long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION;
-  openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
-  stream_session_t *tls_session;
-  const SSL_METHOD *method;
   application_t *app;
-  int rv, err;
+  const SSL_METHOD *method;
+  SSL_CTX *ssl_ctx;
+  int rv;
   BIO *cert_bio;
+  X509 *srvcert;
+  EVP_PKEY *pkey;
+  u32 olc_index;
+  openssl_listen_ctx_t *olc;
+
+  char *ciphers = "ALL:!ADH:!LOW:!EXP:!MD5:!RC4-SHA:!DES-CBC3-SHA:@STRENGTH";
+  long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION;
 #ifdef HAVE_OPENSSL_ASYNC
   openssl_main_t *om = &openssl_main;
-  openssl_resume_handler *handler;
 #endif
 
-  app = application_get (ctx->parent_app_index);
+  app = application_get (lctx->parent_app_index);
   if (!app->tls_cert || !app->tls_key)
     {
       TLS_DBG (1, "tls cert and/or key not configured %d",
-	       ctx->parent_app_index);
+	       lctx->parent_app_index);
       return -1;
     }
 
   method = SSLv23_method ();
-  oc->ssl_ctx = SSL_CTX_new (method);
-  if (!oc->ssl_ctx)
+  ssl_ctx = SSL_CTX_new (method);
+  if (!ssl_ctx)
     {
       clib_warning ("Unable to create SSL context");
       return -1;
     }
 
-  SSL_CTX_set_mode (oc->ssl_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
+  SSL_CTX_set_mode (ssl_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
 #ifdef HAVE_OPENSSL_ASYNC
   if (om->async)
-    SSL_CTX_set_mode (oc->ssl_ctx, SSL_MODE_ASYNC);
+    SSL_CTX_set_mode (ssl_ctx, SSL_MODE_ASYNC);
 #endif
-  SSL_CTX_set_options (oc->ssl_ctx, flags);
-  SSL_CTX_set_ecdh_auto (oc->ssl_ctx, 1);
+  SSL_CTX_set_options (ssl_ctx, flags);
+  SSL_CTX_set_ecdh_auto (ssl_ctx, 1);
 
-  rv = SSL_CTX_set_cipher_list (oc->ssl_ctx, (const char *) ciphers);
+  rv = SSL_CTX_set_cipher_list (ssl_ctx, (const char *) ciphers);
   if (rv != 1)
     {
       TLS_DBG (1, "Couldn't set cipher");
@@ -584,32 +607,73 @@ openssl_ctx_init_server (tls_ctx_t * ctx)
    */
   cert_bio = BIO_new (BIO_s_mem ());
   BIO_write (cert_bio, app->tls_cert, vec_len (app->tls_cert));
-  oc->srvcert = PEM_read_bio_X509 (cert_bio, NULL, NULL, NULL);
-  if (!oc->srvcert)
+  srvcert = PEM_read_bio_X509 (cert_bio, NULL, NULL, NULL);
+  if (!srvcert)
     {
       clib_warning ("unable to parse certificate");
       return -1;
     }
-  SSL_CTX_use_certificate (oc->ssl_ctx, oc->srvcert);
+  SSL_CTX_use_certificate (ssl_ctx, srvcert);
   BIO_free (cert_bio);
 
-
   cert_bio = BIO_new (BIO_s_mem ());
   BIO_write (cert_bio, app->tls_key, vec_len (app->tls_key));
-  oc->pkey = PEM_read_bio_PrivateKey (cert_bio, NULL, NULL, NULL);
-  if (!oc->pkey)
+  pkey = PEM_read_bio_PrivateKey (cert_bio, NULL, NULL, NULL);
+  if (!pkey)
     {
       clib_warning ("unable to parse pkey");
       return -1;
     }
-  SSL_CTX_use_PrivateKey (oc->ssl_ctx, oc->pkey);
-
-  SSL_CTX_use_PrivateKey (oc->ssl_ctx, oc->pkey);
+  SSL_CTX_use_PrivateKey (ssl_ctx, pkey);
   BIO_free (cert_bio);
 
+  olc_index = openssl_listen_ctx_alloc ();
+  olc = openssl_lctx_get (olc_index);
+  olc->ssl_ctx = ssl_ctx;
+  olc->srvcert = srvcert;
+  olc->pkey = pkey;
+
+  /* store SSL_CTX into TLS level structure */
+  lctx->tls_ssl_ctx = olc_index;
+
+  return 0;
+
+}
+
+static int
+openssl_stop_listen (tls_ctx_t * lctx)
+{
+  u32 olc_index;
+  openssl_listen_ctx_t *olc;
+
+  olc_index = lctx->tls_ssl_ctx;
+  olc = openssl_lctx_get (olc_index);
+
+  X509_free (olc->srvcert);
+  EVP_PKEY_free (olc->pkey);
+
+  SSL_CTX_free (olc->ssl_ctx);
+  openssl_listen_ctx_free (olc);
+
+  return 0;
+}
+
+static int
+openssl_ctx_init_server (tls_ctx_t * ctx)
+{
+  openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
+  u32 olc_index = ctx->tls_ssl_ctx;
+  openssl_listen_ctx_t *olc;
+  stream_session_t *tls_session;
+  int rv, err;
+#ifdef HAVE_OPENSSL_ASYNC
+  openssl_resume_handler *handler;
+#endif
+
   /* Start a new connection */
 
-  oc->ssl = SSL_new (oc->ssl_ctx);
+  olc = openssl_lctx_get (olc_index);
+  oc->ssl = SSL_new (olc->ssl_ctx);
   if (oc->ssl == NULL)
     {
       TLS_DBG (1, "Couldn't initialize ssl struct");
@@ -670,6 +734,8 @@ const static tls_engine_vft_t openssl_engine = {
   .ctx_write = openssl_ctx_write,
   .ctx_read = openssl_ctx_read,
   .ctx_handshake_is_over = openssl_handshake_is_over,
+  .ctx_start_listen = openssl_start_listen,
+  .ctx_stop_listen = openssl_stop_listen,
 };
 
 int
diff --git a/src/plugins/tlsopenssl/tls_openssl.h b/src/plugins/tlsopenssl/tls_openssl.h
index c1a21e57317..a524da74cee 100644
--- a/src/plugins/tlsopenssl/tls_openssl.h
+++ b/src/plugins/tlsopenssl/tls_openssl.h
@@ -29,13 +29,21 @@ typedef struct tls_ctx_openssl_
   SSL *ssl;
   BIO *rbio;
   BIO *wbio;
+} openssl_ctx_t;
+
+typedef struct tls_listen_ctx_opensl_
+{
+  u32 openssl_lctx_index;
+  SSL_CTX *ssl_ctx;
+  SSL *ssl;
   X509 *srvcert;
   EVP_PKEY *pkey;
-} openssl_ctx_t;
+} openssl_listen_ctx_t;
 
 typedef struct openssl_main_
 {
   openssl_ctx_t ***ctx_pool;
+  openssl_listen_ctx_t *lctx_pool;
 
   X509_STORE *cert_store;
   int engine_init;
diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c
index 88b4548c3aa..4b12248a051 100644
--- a/src/vnet/tls/tls.c
+++ b/src/vnet/tls/tls.c
@@ -541,6 +541,8 @@ tls_start_listen (u32 app_listener_index, transport_endpoint_t * tep)
   lctx->tcp_is_ip4 = sep->is_ip4;
   lctx->tls_ctx_engine = engine_type;
 
+  tls_vfts[engine_type].ctx_start_listen (lctx);
+
   TLS_DBG (1, "Started listening %d, engine type %d", lctx_index,
 	   engine_type);
   return lctx_index;
@@ -552,9 +554,15 @@ tls_stop_listen (u32 lctx_index)
   tls_main_t *tm = &tls_main;
   application_t *tls_app;
   tls_ctx_t *lctx;
+  tls_engine_type_t engine_type;
+
   lctx = tls_listener_ctx_get (lctx_index);
   tls_app = application_get (tm->app_index);
   application_stop_listen (tls_app, lctx->tls_session_handle);
+
+  engine_type = lctx->tls_ctx_engine;
+  tls_vfts[engine_type].ctx_stop_listen (lctx);
+
   tls_listener_ctx_free (lctx);
   return 0;
 }
diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h
index f67f307550b..5515cb25b8a 100644
--- a/src/vnet/tls/tls.h
+++ b/src/vnet/tls/tls.h
@@ -41,6 +41,7 @@ typedef CLIB_PACKED (struct tls_cxt_id_
   u32 parent_app_index;
   session_handle_t app_session_handle;
   session_handle_t tls_session_handle;
+  u32 ssl_ctx;
   u32 listener_ctx_index;
   u8 tcp_is_ip4;
   u8 tls_engine_id;
@@ -62,6 +63,7 @@ typedef struct tls_ctx_
 #define listener_ctx_index c_tls_ctx_id.listener_ctx_index
 #define tcp_is_ip4 c_tls_ctx_id.tcp_is_ip4
 #define tls_ctx_engine c_tls_ctx_id.tls_engine_id
+#define tls_ssl_ctx c_tls_ctx_id.ssl_ctx
 #define tls_ctx_handle c_c_index
   /* Temporary storage for session open opaque. Overwritten once
    * underlying tcp connection is established */
@@ -99,6 +101,8 @@ typedef struct tls_engine_vft_
   int (*ctx_read) (tls_ctx_t * ctx, stream_session_t * tls_session);
   int (*ctx_write) (tls_ctx_t * ctx, stream_session_t * app_session);
     u8 (*ctx_handshake_is_over) (tls_ctx_t * ctx);
+  int (*ctx_start_listen) (tls_ctx_t * ctx);
+  int (*ctx_stop_listen) (tls_ctx_t * ctx);
 } tls_engine_vft_t;
 
 typedef enum tls_engine_type_