tls: switch to vc service and track half-open sessions

Half-open tls sessions are now tracked by the app worker and are cleaned
up only when tcp cleans up its half-open session, i.e., independent of
when the established tls context is allocated.

Type: improvement

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: If5d594d7095192dd527daf4ea1358ffeccdfcc7a

7 files changed, 78 insertions, 19 deletions

diff --git a/src/vnet/session/application.h b/src/vnet/session/application.h
index eca3d3cf2c5..5ddf1d21fe3 100644
--- a/src/vnet/session/application.h
+++ b/src/vnet/session/application.h
@@ -314,7 +314,7 @@ int app_worker_init_connected (app_worker_t * app_wrk, session_t * s);
 int app_worker_connect_notify (app_worker_t * app_wrk, session_t * s,
 			       session_error_t err, u32 opaque);
 int app_worker_add_half_open (app_worker_t *app_wrk, session_handle_t sh);
-int app_worker_del_half_open (app_worker_t *app_wrk, u32 ho_index);
+int app_worker_del_half_open (app_worker_t *app_wrk, session_t *s);
 int app_worker_close_notify (app_worker_t * app_wrk, session_t * s);
 int app_worker_transport_closed_notify (app_worker_t * app_wrk,
 					session_t * s);
diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h
index 14db42a0009..733f5dc0ef6 100644
--- a/src/vnet/session/application_interface.h
+++ b/src/vnet/session/application_interface.h
@@ -53,6 +53,9 @@ typedef struct session_cb_vft_
   /** Notify app that session or transport are about to be removed */
   void (*session_cleanup_callback) (session_t * s, session_cleanup_ntf_t ntf);
 
+  /** Notify app that half open state was cleaned up (optional) */
+  void (*half_open_cleanup_callback) (session_t *s);
+
   /** Notify app that session was reset */
   void (*session_reset_callback) (session_t * s);
 
diff --git a/src/vnet/session/application_worker.c b/src/vnet/session/application_worker.c
index b55c55aea47..fad70c06de3 100644
--- a/src/vnet/session/application_worker.c
+++ b/src/vnet/session/application_worker.c
@@ -393,10 +393,13 @@ app_worker_add_half_open (app_worker_t *app_wrk, session_handle_t sh)
 }
 
 int
-app_worker_del_half_open (app_worker_t *app_wrk, u32 ho_index)
+app_worker_del_half_open (app_worker_t *app_wrk, session_t *s)
 {
+  application_t *app = application_get (app_wrk->app_index);
   ASSERT (vlib_get_thread_index () == 0);
-  pool_put_index (app_wrk->half_open_table, ho_index);
+  pool_put_index (app_wrk->half_open_table, s->ho_index);
+  if (app->cb_fns.half_open_cleanup_callback)
+    app->cb_fns.half_open_cleanup_callback (s);
   return 0;
 }
 
diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c
index 16b6ce412ac..0d2301c5f83 100644
--- a/src/vnet/session/session.c
+++ b/src/vnet/session/session.c
@@ -316,7 +316,7 @@ session_half_open_delete_notify (transport_connection_t *tc)
 
   s = ho_session_get (tc->s_index);
   app_wrk = app_worker_get (s->app_wrk_index);
-  app_worker_del_half_open (app_wrk, s->ho_index);
+  app_worker_del_half_open (app_wrk, s);
   session_free (s);
 }
 
@@ -339,7 +339,7 @@ session_alloc_for_connection (transport_connection_t * tc)
   return s;
 }
 
-static session_t *
+session_t *
 session_alloc_for_half_open (transport_connection_t *tc)
 {
   session_t *s;
@@ -1297,7 +1297,8 @@ session_open_vc (session_endpoint_cfg_t *rmt, session_handle_t *rsh)
   ho->opaque = rmt->opaque;
   *rsh = session_handle (ho);
 
-  session_lookup_add_half_open (tc, tc->c_index);
+  if (!(tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP))
+    session_lookup_add_half_open (tc, tc->c_index);
 
   return 0;
 }
diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h
index e29cdbfc933..d14c68971de 100644
--- a/src/vnet/session/session.h
+++ b/src/vnet/session/session.h
@@ -759,6 +759,7 @@ void session_node_enable_disable (u8 is_en);
 clib_error_t *vnet_session_enable_disable (vlib_main_t * vm, u8 is_en);
 
 session_t *session_alloc_for_connection (transport_connection_t * tc);
+session_t *session_alloc_for_half_open (transport_connection_t *tc);
 
 #endif /* __included_session_h__ */
 
diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c
index e19f8e6c040..09fc8c62325 100644
--- a/src/vnet/tls/tls.c
+++ b/src/vnet/tls/tls.c
@@ -123,19 +123,20 @@ tls_ctx_half_open_alloc (void)
   if (PREDICT_FALSE (will_expand && vlib_num_workers ()))
     {
       clib_rwlock_writer_lock (&tm->half_open_rwlock);
-      pool_get (tm->half_open_ctx_pool, ctx);
-      ctx_index = ctx - tm->half_open_ctx_pool;
+      pool_get_zero (tm->half_open_ctx_pool, ctx);
+      ctx->c_c_index = ctx - tm->half_open_ctx_pool;
+      ctx_index = ctx->c_c_index;
       clib_rwlock_writer_unlock (&tm->half_open_rwlock);
     }
   else
     {
       /* reader lock assumption: only main thread will call pool_get */
       clib_rwlock_reader_lock (&tm->half_open_rwlock);
-      pool_get (tm->half_open_ctx_pool, ctx);
-      ctx_index = ctx - tm->half_open_ctx_pool;
+      pool_get_zero (tm->half_open_ctx_pool, ctx);
+      ctx->c_c_index = ctx - tm->half_open_ctx_pool;
+      ctx_index = ctx->c_c_index;
       clib_rwlock_reader_unlock (&tm->half_open_rwlock);
     }
-  clib_memset (ctx, 0, sizeof (*ctx));
   return ctx_index;
 }
 
@@ -420,6 +421,20 @@ tls_session_reset_callback (session_t * s)
     }
 }
 
+static void
+tls_session_cleanup_ho (session_t *s)
+{
+  tls_ctx_t *ctx;
+  u32 ho_index;
+
+  /* session opaque stores the opaque passed on connect */
+  ho_index = s->opaque;
+  ctx = tls_ctx_half_open_get (ho_index);
+  session_half_open_delete_notify (&ctx->connection);
+  tls_ctx_half_open_reader_unlock ();
+  tls_ctx_half_open_free (ho_index);
+}
+
 int
 tls_add_segment_callback (u32 client_index, u64 segment_handle)
 {
@@ -533,15 +548,14 @@ tls_session_connected_cb (u32 tls_app_index, u32 ho_ctx_index,
 	  app_worker_connect_notify (app_wrk, 0, err, api_context);
 	}
       tls_ctx_half_open_reader_unlock ();
-      tls_ctx_half_open_free (ho_ctx_index);
       return rv;
     }
 
   ctx_handle = tls_ctx_alloc (ho_ctx->tls_ctx_engine);
   ctx = tls_ctx_get (ctx_handle);
   clib_memcpy_fast (ctx, ho_ctx, sizeof (*ctx));
+  /* Half-open freed on tcp half-open cleanup notification */
   tls_ctx_half_open_reader_unlock ();
-  tls_ctx_half_open_free (ho_ctx_index);
 
   ctx->c_thread_index = vlib_get_thread_index ();
   ctx->tls_ctx_handle = ctx_handle;
@@ -660,6 +674,7 @@ static session_cb_vft_t tls_app_cb_vft = {
   .session_disconnect_callback = tls_session_disconnect_callback,
   .session_connected_callback = tls_session_connected_callback,
   .session_reset_callback = tls_session_reset_callback,
+  .half_open_cleanup_callback = tls_session_cleanup_ho,
   .add_segment_callback = tls_add_segment_callback,
   .del_segment_callback = tls_del_segment_callback,
   .builtin_app_rx_callback = tls_app_rx_callback,
@@ -704,6 +719,8 @@ tls_connect (transport_endpoint_cfg_t * tep)
   ctx->tcp_is_ip4 = sep->is_ip4;
   ctx->tls_type = sep->transport_proto;
   ctx->ckpair_index = ccfg->ckpair_index;
+  ctx->c_proto = TRANSPORT_PROTO_TLS;
+  ctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP;
   if (ccfg->hostname[0])
     {
       ctx->srv_hostname = format (0, "%s", ccfg->hostname);
@@ -721,8 +738,11 @@ tls_connect (transport_endpoint_cfg_t * tep)
   if ((rv = vnet_connect (cargs)))
     return rv;
 
+  /* Track half-open tcp session in case we need to clean it up */
+  ctx->tls_session_handle = cargs->sh;
+
   TLS_DBG (1, "New connect request %u engine %d", ctx_index, engine_type);
-  return 0;
+  return ctx_index;
 }
 
 void
@@ -871,6 +891,26 @@ tls_listener_get (u32 listener_index)
   return &ctx->connection;
 }
 
+static transport_connection_t *
+tls_half_open_get (u32 ho_index)
+{
+  tls_main_t *tm = &tls_main;
+  tls_ctx_t *ctx;
+  ctx = tls_ctx_half_open_get (ho_index);
+  clib_rwlock_reader_unlock (&tm->half_open_rwlock);
+  return &ctx->connection;
+}
+
+static void
+tls_cleanup_ho (u32 ho_index)
+{
+  tls_main_t *tm = &tls_main;
+  tls_ctx_t *ctx;
+  ctx = tls_ctx_half_open_get (ho_index);
+  clib_rwlock_reader_unlock (&tm->half_open_rwlock);
+  session_cleanup_half_open (ctx->tls_session_handle);
+}
+
 int
 tls_custom_tx_callback (void *session, transport_send_params_t * sp)
 {
@@ -992,10 +1032,19 @@ format_tls_listener (u8 * s, va_list * args)
 u8 *
 format_tls_half_open (u8 * s, va_list * args)
 {
-  u32 tc_index = va_arg (*args, u32);
+  u32 ho_index = va_arg (*args, u32);
   u32 __clib_unused thread_index = va_arg (*args, u32);
-  tls_ctx_t *ctx = tls_ctx_half_open_get (tc_index);
-  s = format (s, "[TLS] half-open app %u", ctx->parent_app_wrk_index);
+  session_t *tcp_ho;
+  tls_ctx_t *ho_ctx;
+
+  ho_ctx = tls_ctx_half_open_get (ho_index);
+
+  tcp_ho = session_get_from_handle (ho_ctx->tls_session_handle);
+  s = format (s, "[%d:%d][%s] half-open app_wrk %u engine %u ts %d:%d",
+	      ho_ctx->c_thread_index, ho_ctx->c_s_index, "TLS",
+	      ho_ctx->parent_app_wrk_index, ho_ctx->tls_ctx_engine,
+	      tcp_ho->thread_index, tcp_ho->session_index);
+
   tls_ctx_half_open_reader_unlock ();
   return s;
 }
@@ -1080,6 +1129,8 @@ static const transport_proto_vft_t tls_proto = {
   .stop_listen = tls_stop_listen,
   .get_connection = tls_connection_get,
   .get_listener = tls_listener_get,
+  .get_half_open = tls_half_open_get,
+  .cleanup_ho = tls_cleanup_ho,
   .custom_tx = tls_custom_tx_callback,
   .format_connection = format_tls_connection,
   .format_half_open = format_tls_half_open,
@@ -1090,7 +1141,7 @@ static const transport_proto_vft_t tls_proto = {
     .name = "tls",
     .short_name = "J",
     .tx_type = TRANSPORT_TX_INTERNAL,
-    .service_type = TRANSPORT_SERVICE_APP,
+    .service_type = TRANSPORT_SERVICE_VC,
   },
 };
 
diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h
index 2038fdff133..eba70c0a8bb 100644
--- a/src/vnet/tls/tls.h
+++ b/src/vnet/tls/tls.h
@@ -44,12 +44,12 @@ typedef struct tls_cxt_id_
     u32 parent_app_api_ctx;
   };
   session_handle_t tls_session_handle;
+  void *migrate_ctx;
   u32 parent_app_wrk_index;
   u32 ssl_ctx;
   u32 listener_ctx_index;
   u8 tcp_is_ip4;
   u8 tls_engine_id;
-  void *migrate_ctx;
 } tls_ctx_id_t;
 /* *INDENT-ON* */