tls: switch to vc service and track half-open sessions

Half-open tls sessions are now tracked by the app worker and are cleaned
up only when tcp cleans up its half-open session, i.e., independent of
when the established tls context is allocated.

Type: improvement

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: If5d594d7095192dd527daf4ea1358ffeccdfcc7a

2 files changed, 64 insertions, 13 deletions

diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c
index e19f8e6c040..09fc8c62325 100644
--- a/src/vnet/tls/tls.c
+++ b/src/vnet/tls/tls.c
@@ -123,19 +123,20 @@ tls_ctx_half_open_alloc (void)
   if (PREDICT_FALSE (will_expand && vlib_num_workers ()))
     {
       clib_rwlock_writer_lock (&tm->half_open_rwlock);
-      pool_get (tm->half_open_ctx_pool, ctx);
-      ctx_index = ctx - tm->half_open_ctx_pool;
+      pool_get_zero (tm->half_open_ctx_pool, ctx);
+      ctx->c_c_index = ctx - tm->half_open_ctx_pool;
+      ctx_index = ctx->c_c_index;
       clib_rwlock_writer_unlock (&tm->half_open_rwlock);
     }
   else
     {
       /* reader lock assumption: only main thread will call pool_get */
       clib_rwlock_reader_lock (&tm->half_open_rwlock);
-      pool_get (tm->half_open_ctx_pool, ctx);
-      ctx_index = ctx - tm->half_open_ctx_pool;
+      pool_get_zero (tm->half_open_ctx_pool, ctx);
+      ctx->c_c_index = ctx - tm->half_open_ctx_pool;
+      ctx_index = ctx->c_c_index;
       clib_rwlock_reader_unlock (&tm->half_open_rwlock);
     }
-  clib_memset (ctx, 0, sizeof (*ctx));
   return ctx_index;
 }
 
@@ -420,6 +421,20 @@ tls_session_reset_callback (session_t * s)
     }
 }
 
+static void
+tls_session_cleanup_ho (session_t *s)
+{
+  tls_ctx_t *ctx;
+  u32 ho_index;
+
+  /* session opaque stores the opaque passed on connect */
+  ho_index = s->opaque;
+  ctx = tls_ctx_half_open_get (ho_index);
+  session_half_open_delete_notify (&ctx->connection);
+  tls_ctx_half_open_reader_unlock ();
+  tls_ctx_half_open_free (ho_index);
+}
+
 int
 tls_add_segment_callback (u32 client_index, u64 segment_handle)
 {
@@ -533,15 +548,14 @@ tls_session_connected_cb (u32 tls_app_index, u32 ho_ctx_index,
 	  app_worker_connect_notify (app_wrk, 0, err, api_context);
 	}
       tls_ctx_half_open_reader_unlock ();
-      tls_ctx_half_open_free (ho_ctx_index);
       return rv;
     }
 
   ctx_handle = tls_ctx_alloc (ho_ctx->tls_ctx_engine);
   ctx = tls_ctx_get (ctx_handle);
   clib_memcpy_fast (ctx, ho_ctx, sizeof (*ctx));
+  /* Half-open freed on tcp half-open cleanup notification */
   tls_ctx_half_open_reader_unlock ();
-  tls_ctx_half_open_free (ho_ctx_index);
 
   ctx->c_thread_index = vlib_get_thread_index ();
   ctx->tls_ctx_handle = ctx_handle;
@@ -660,6 +674,7 @@ static session_cb_vft_t tls_app_cb_vft = {
   .session_disconnect_callback = tls_session_disconnect_callback,
   .session_connected_callback = tls_session_connected_callback,
   .session_reset_callback = tls_session_reset_callback,
+  .half_open_cleanup_callback = tls_session_cleanup_ho,
   .add_segment_callback = tls_add_segment_callback,
   .del_segment_callback = tls_del_segment_callback,
   .builtin_app_rx_callback = tls_app_rx_callback,
@@ -704,6 +719,8 @@ tls_connect (transport_endpoint_cfg_t * tep)
   ctx->tcp_is_ip4 = sep->is_ip4;
   ctx->tls_type = sep->transport_proto;
   ctx->ckpair_index = ccfg->ckpair_index;
+  ctx->c_proto = TRANSPORT_PROTO_TLS;
+  ctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP;
   if (ccfg->hostname[0])
     {
       ctx->srv_hostname = format (0, "%s", ccfg->hostname);
@@ -721,8 +738,11 @@ tls_connect (transport_endpoint_cfg_t * tep)
   if ((rv = vnet_connect (cargs)))
     return rv;
 
+  /* Track half-open tcp session in case we need to clean it up */
+  ctx->tls_session_handle = cargs->sh;
+
   TLS_DBG (1, "New connect request %u engine %d", ctx_index, engine_type);
-  return 0;
+  return ctx_index;
 }
 
 void
@@ -871,6 +891,26 @@ tls_listener_get (u32 listener_index)
   return &ctx->connection;
 }
 
+static transport_connection_t *
+tls_half_open_get (u32 ho_index)
+{
+  tls_main_t *tm = &tls_main;
+  tls_ctx_t *ctx;
+  ctx = tls_ctx_half_open_get (ho_index);
+  clib_rwlock_reader_unlock (&tm->half_open_rwlock);
+  return &ctx->connection;
+}
+
+static void
+tls_cleanup_ho (u32 ho_index)
+{
+  tls_main_t *tm = &tls_main;
+  tls_ctx_t *ctx;
+  ctx = tls_ctx_half_open_get (ho_index);
+  clib_rwlock_reader_unlock (&tm->half_open_rwlock);
+  session_cleanup_half_open (ctx->tls_session_handle);
+}
+
 int
 tls_custom_tx_callback (void *session, transport_send_params_t * sp)
 {
@@ -992,10 +1032,19 @@ format_tls_listener (u8 * s, va_list * args)
 u8 *
 format_tls_half_open (u8 * s, va_list * args)
 {
-  u32 tc_index = va_arg (*args, u32);
+  u32 ho_index = va_arg (*args, u32);
   u32 __clib_unused thread_index = va_arg (*args, u32);
-  tls_ctx_t *ctx = tls_ctx_half_open_get (tc_index);
-  s = format (s, "[TLS] half-open app %u", ctx->parent_app_wrk_index);
+  session_t *tcp_ho;
+  tls_ctx_t *ho_ctx;
+
+  ho_ctx = tls_ctx_half_open_get (ho_index);
+
+  tcp_ho = session_get_from_handle (ho_ctx->tls_session_handle);
+  s = format (s, "[%d:%d][%s] half-open app_wrk %u engine %u ts %d:%d",
+	      ho_ctx->c_thread_index, ho_ctx->c_s_index, "TLS",
+	      ho_ctx->parent_app_wrk_index, ho_ctx->tls_ctx_engine,
+	      tcp_ho->thread_index, tcp_ho->session_index);
+
   tls_ctx_half_open_reader_unlock ();
   return s;
 }
@@ -1080,6 +1129,8 @@ static const transport_proto_vft_t tls_proto = {
   .stop_listen = tls_stop_listen,
   .get_connection = tls_connection_get,
   .get_listener = tls_listener_get,
+  .get_half_open = tls_half_open_get,
+  .cleanup_ho = tls_cleanup_ho,
   .custom_tx = tls_custom_tx_callback,
   .format_connection = format_tls_connection,
   .format_half_open = format_tls_half_open,
@@ -1090,7 +1141,7 @@ static const transport_proto_vft_t tls_proto = {
     .name = "tls",
     .short_name = "J",
     .tx_type = TRANSPORT_TX_INTERNAL,
-    .service_type = TRANSPORT_SERVICE_APP,
+    .service_type = TRANSPORT_SERVICE_VC,
   },
 };
 
diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h
index 2038fdff133..eba70c0a8bb 100644
--- a/src/vnet/tls/tls.h
+++ b/src/vnet/tls/tls.h
@@ -44,12 +44,12 @@ typedef struct tls_cxt_id_
     u32 parent_app_api_ctx;
   };
   session_handle_t tls_session_handle;
+  void *migrate_ctx;
   u32 parent_app_wrk_index;
   u32 ssl_ctx;
   u32 listener_ctx_index;
   u8 tcp_is_ip4;
   u8 tls_engine_id;
-  void *migrate_ctx;
 } tls_ctx_id_t;
 /* *INDENT-ON* */