tls: fix close with data

Type:fix

Also changes the way the ctx is freed. TLS now waits for tcp delete
notification before freeing the ctx.

Change-Id: I2f606a9ce7b3755ae9d11d6fe714fe11b65dcb98
Signed-off-by: Florin Coras <fcoras@cisco.com>
(cherry picked from commit ef2b3357b4b2a682ef81ffa2c71cec14f83101d6)

3 files changed, 63 insertions, 13 deletions

diff --git a/src/plugins/tlsopenssl/tls_openssl.c b/src/plugins/tlsopenssl/tls_openssl.c
index 6cb8ff08b7f..ee6b0e3172e 100644
--- a/src/plugins/tlsopenssl/tls_openssl.c
+++ b/src/plugins/tlsopenssl/tls_openssl.c
@@ -233,6 +233,29 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, session_t * tls_session)
 
       rv = SSL_do_handshake (oc->ssl);
       err = SSL_get_error (oc->ssl, rv);
+
+      if (err == SSL_ERROR_SSL)
+	{
+	  char buf[512];
+	  ERR_error_string (ERR_get_error (), buf);
+	  clib_warning ("Err: %s", buf);
+
+	  /*
+	   * Cleanup pre-allocated app session and close transport
+	   */
+	  if (SSL_is_server (oc->ssl))
+	    {
+	      session_free (session_get (ctx->c_s_index,
+					 ctx->c_thread_index));
+	      ctx->no_app_session = 1;
+	      ctx->c_s_index = SESSION_INVALID_INDEX;
+	      tls_disconnect_transport (ctx);
+	    }
+	  else
+	    tls_notify_app_connected (ctx, /* is failed */ 1);
+	  return -1;
+	}
+
       openssl_try_handshake_write (oc, tls_session);
 #ifdef HAVE_OPENSSL_ASYNC
       if (err == SSL_ERROR_WANT_ASYNC)
@@ -247,15 +270,7 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, session_t * tls_session)
 #endif
 
       if (err != SSL_ERROR_WANT_WRITE)
-	{
-	  if (err == SSL_ERROR_SSL)
-	    {
-	      char buf[512];
-	      ERR_error_string (ERR_get_error (), buf);
-	      clib_warning ("Err: %s", buf);
-	    }
-	  break;
-	}
+	break;
     }
   TLS_DBG (2, "tls state for %u is %s", oc->openssl_ctx_index,
 	   SSL_state_string_long (oc->ssl));
@@ -297,6 +312,13 @@ openssl_ctx_handshake_rx (tls_ctx_t * ctx, session_t * tls_session)
   return rv;
 }
 
+static void
+openssl_confirm_app_close (tls_ctx_t * ctx)
+{
+  tls_disconnect_transport (ctx);
+  session_transport_closed_notify (&ctx->connection);
+}
+
 static inline int
 openssl_ctx_write (tls_ctx_t * ctx, session_t * app_session)
 {
@@ -374,6 +396,8 @@ check_tls_fifo:
 
   if (BIO_ctrl_pending (oc->rbio) > 0)
     tls_add_vpp_q_builtin_tx_evt (app_session);
+  else if (ctx->app_closed)
+    openssl_confirm_app_close (ctx);
 
   return wrote;
 }
@@ -744,9 +768,16 @@ openssl_transport_close (tls_ctx_t * ctx)
 static int
 openssl_app_close (tls_ctx_t * ctx)
 {
-  tls_disconnect_transport (ctx);
-  session_transport_delete_notify (&ctx->connection);
-  openssl_ctx_free (ctx);
+  openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
+  session_t *app_session;
+
+  /* Wait for all data to be written to tcp */
+  app_session = session_get_from_handle (ctx->app_session_handle);
+  if (BIO_ctrl_pending (oc->rbio) <= 0
+      && !svm_fifo_max_dequeue_cons (app_session->tx_fifo))
+    openssl_confirm_app_close (ctx);
+  else
+    ctx->app_closed = 1;
   return 0;
 }
 
diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c
index f7780feaa85..16405707bc9 100644
--- a/src/vnet/tls/tls.c
+++ b/src/vnet/tls/tls.c
@@ -260,6 +260,9 @@ tls_notify_app_connected (tls_ctx_t * ctx, u8 is_failed)
   return 0;
 
 failed:
+  /* Free app session pre-allocated when transport was established */
+  session_free (session_get (ctx->c_s_index, ctx->c_thread_index));
+  ctx->no_app_session = 1;
   tls_disconnect (ctx->tls_ctx_handle, vlib_get_thread_index ());
   return app_worker_connect_notify (app_wrk, 0, ctx->parent_app_api_context);
 }
@@ -342,7 +345,6 @@ tls_ctx_app_close (tls_ctx_t * ctx)
 void
 tls_ctx_free (tls_ctx_t * ctx)
 {
-  vec_free (ctx->srv_hostname);
   tls_vfts[ctx->tls_ctx_engine].ctx_free (ctx);
 }
 
@@ -480,6 +482,20 @@ tls_session_connected_callback (u32 tls_app_index, u32 ho_ctx_index,
   return tls_ctx_init_client (ctx);
 }
 
+static void
+tls_app_session_cleanup (session_t * s, session_cleanup_ntf_t ntf)
+{
+  tls_ctx_t *ctx;
+
+  if (ntf == SESSION_CLEANUP_TRANSPORT)
+    return;
+
+  ctx = tls_ctx_get (s->opaque);
+  if (!ctx->no_app_session)
+    session_transport_delete_notify (&ctx->connection);
+  tls_ctx_free (ctx);
+}
+
 /* *INDENT-OFF* */
 static session_cb_vft_t tls_app_cb_vft = {
   .session_accept_callback = tls_session_accept_callback,
@@ -489,6 +505,7 @@ static session_cb_vft_t tls_app_cb_vft = {
   .add_segment_callback = tls_add_segment_callback,
   .del_segment_callback = tls_del_segment_callback,
   .builtin_app_rx_callback = tls_app_rx_callback,
+  .session_cleanup_callback = tls_app_session_cleanup,
 };
 /* *INDENT-ON* */
 
diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h
index 51b92a5debf..eaba3c085b6 100644
--- a/src/vnet/tls/tls.h
+++ b/src/vnet/tls/tls.h
@@ -76,6 +76,8 @@ typedef struct tls_ctx_
 
   u8 is_passive_close;
   u8 resume;
+  u8 app_closed;
+  u8 no_app_session;
   u8 *srv_hostname;
 } tls_ctx_t;