session: refactor local connects

- Switches local connects to cut-thru transport
- Removes local sessions as a separate session type

Change-Id: I997c6355d8c8e4f2110678f785b0f5d96bba47f7
Signed-off-by: Florin Coras <fcoras@cisco.com>

11 files changed, 474 insertions, 886 deletions

diff --git a/src/vnet/session/application.c b/src/vnet/session/application.c
index 4d95426568e..86e0e6b1919 100644
--- a/src/vnet/session/application.c
+++ b/src/vnet/session/application.c
@@ -157,7 +157,6 @@ app_listener_alloc_and_init (application_t * app,
 {
   app_listener_t *app_listener;
   transport_connection_t *tc;
-  local_session_t *ll = 0;
   session_handle_t lh;
   session_type_t st;
   session_t *ls = 0;
@@ -238,7 +237,7 @@ app_listener_alloc_and_init (application_t * app,
       session_lookup_add_connection (tc, lh);
     }
 
-  if (!ll && !ls)
+  if (!ls)
     {
       app_listener_free (app, app_listener);
       return -1;
@@ -267,8 +266,9 @@ app_listener_cleanup (app_listener_t * al)
 
       table_index = application_local_session_table (app);
       ls = listen_session_get (al->local_index);
-      application_local_listener_session_endpoint (ls, &sep);
+      ct_session_endpoint (ls, &sep);
       session_lookup_del_session_endpoint (table_index, &sep);
+      session_stop_listen (ls);
       listen_session_free (ls);
     }
   app_listener_free (app, al);
@@ -727,14 +727,6 @@ application_alloc_worker_and_init (application_t * app, app_worker_t ** wrk)
   app_wrk->event_queue = segment_manager_event_queue (sm);
   app_wrk->app_is_builtin = application_is_builtin (app);
 
-  /*
-   * Segment manager for local sessions
-   */
-  sm = segment_manager_new ();
-  sm->app_wrk_index = app_wrk->wrk_index;
-  app_wrk->local_segment_manager = segment_manager_index (sm);
-  app_wrk->local_connects = hash_create (0, sizeof (u64));
-
   *wrk = app_wrk;
 
   return 0;
@@ -1025,13 +1017,8 @@ vnet_listen (vnet_listen_args_t * a)
 int
 vnet_connect (vnet_connect_args_t * a)
 {
-  app_worker_t *server_wrk, *client_wrk;
+  app_worker_t *client_wrk;
   application_t *client;
-  app_listener_t *al;
-  u32 table_index;
-  session_t *ls;
-  u8 fib_proto;
-  session_handle_t lh;
 
   if (session_endpoint_is_zero (&a->sep))
     return VNET_API_ERROR_INVALID_VALUE;
@@ -1047,53 +1034,14 @@ vnet_connect (vnet_connect_args_t * a)
    */
   if (application_has_local_scope (client))
     {
-      table_index = application_local_session_table (client);
-      lh = session_lookup_local_endpoint (table_index, &a->sep);
-      if (lh == SESSION_DROP_HANDLE)
-	return VNET_API_ERROR_APP_CONNECT_FILTERED;
-
-      if (lh == SESSION_INVALID_HANDLE)
-	goto global_scope;
-
-      ls = listen_session_get_from_handle (lh);
-      al = app_listener_get_w_session (ls);
-
-      /*
-       * Break loop if rule in local table points to connecting app. This
-       * can happen if client is a generic proxy. Route connect through
-       * global table instead.
-       */
-      if (al->app_index == a->app_index)
-	goto global_scope;
+      int rv;
 
-      server_wrk = app_listener_select_worker (al);
-      return app_worker_local_session_connect (client_wrk, server_wrk, ls,
-					       a->api_context);
-    }
-
-  /*
-   * If nothing found, check the global scope for locally attached
-   * destinations. Make sure first that we're allowed to.
-   */
-
-global_scope:
-  if (session_endpoint_is_local (&a->sep))
-    return VNET_API_ERROR_SESSION_CONNECT;
-
-  if (!application_has_global_scope (client))
-    return VNET_API_ERROR_APP_CONNECT_SCOPE;
-
-  fib_proto = session_endpoint_fib_proto (&a->sep);
-  table_index = application_session_table (client, fib_proto);
-  ls = session_lookup_listener (table_index, &a->sep);
-  if (ls)
-    {
-      al = app_listener_get_w_session (ls);
-      server_wrk = app_listener_select_worker (al);
-      return app_worker_local_session_connect (client_wrk, server_wrk, ls,
-					       a->api_context);
+      a->sep_ext.original_tp = a->sep_ext.transport_proto;
+      a->sep_ext.transport_proto = TRANSPORT_PROTO_NONE;
+      rv = app_worker_connect_session (client_wrk, &a->sep, a->api_context);
+      if (rv <= 0)
+	return rv;
     }
-
   /*
    * Not connecting to a local server, propagate to transport
    */
@@ -1132,52 +1080,20 @@ vnet_unlisten (vnet_unlisten_args_t * a)
 int
 vnet_disconnect_session (vnet_disconnect_args_t * a)
 {
-  if (session_handle_is_local (a->handle))
-    {
-      app_worker_t *client_wrk, *server_wrk;
-      local_session_t *ls;
-      u32 wrk_index = ~0;
-
-      /* Disconnect reply came to worker 1 not main thread */
-      app_interface_check_thread_and_barrier (vnet_disconnect_session, a);
-
-      if (!(ls = app_worker_get_local_session_from_handle (a->handle)))
-	return 0;
-
-      client_wrk = app_worker_get_if_valid (ls->client_wrk_index);
-      server_wrk = app_worker_get (ls->app_wrk_index);
-
-      if (server_wrk->app_index == a->app_index)
-	wrk_index = server_wrk->wrk_index;
-      else if (client_wrk && client_wrk->app_index == a->app_index)
-	wrk_index = client_wrk->wrk_index;
-
-      if (wrk_index == ~0)
-	{
-	  clib_warning ("app %u does not own session 0x%lx", a->app_index,
-			application_local_session_handle (ls));
-	  return VNET_API_ERROR_INVALID_VALUE;
-	}
-
-      return app_worker_local_session_disconnect (wrk_index, ls);
-    }
-  else
-    {
-      app_worker_t *app_wrk;
-      session_t *s;
+  app_worker_t *app_wrk;
+  session_t *s;
 
-      s = session_get_from_handle_if_valid (a->handle);
-      if (!s)
-	return VNET_API_ERROR_INVALID_VALUE;
-      app_wrk = app_worker_get (s->app_wrk_index);
-      if (app_wrk->app_index != a->app_index)
-	return VNET_API_ERROR_INVALID_VALUE;
+  s = session_get_from_handle_if_valid (a->handle);
+  if (!s)
+    return VNET_API_ERROR_INVALID_VALUE;
+  app_wrk = app_worker_get (s->app_wrk_index);
+  if (app_wrk->app_index != a->app_index)
+    return VNET_API_ERROR_INVALID_VALUE;
 
-      /* We're peeking into another's thread pool. Make sure */
-      ASSERT (s->session_index == session_index_from_handle (a->handle));
+  /* We're peeking into another's thread pool. Make sure */
+  ASSERT (s->session_index == session_index_from_handle (a->handle));
 
-      session_close (s);
-    }
+  session_close (s);
   return 0;
 }
 
@@ -1477,49 +1393,6 @@ application_format_connects (application_t * app, int verbose)
   /* *INDENT-ON* */
 }
 
-static void
-application_format_local_sessions (application_t * app, int verbose)
-{
-  app_worker_map_t *wrk_map;
-  app_worker_t *app_wrk;
-
-  if (!app)
-    {
-      app_worker_format_local_sessions (0, verbose);
-      return;
-    }
-
-  /*
-   * Format local accepted/connected sessions
-   */
-  /* *INDENT-OFF* */
-  pool_foreach (wrk_map, app->worker_maps, ({
-    app_wrk = app_worker_get (wrk_map->wrk_index);
-    app_worker_format_local_sessions (app_wrk, verbose);
-  }));
-  /* *INDENT-ON* */
-}
-
-static void
-application_format_local_connects (application_t * app, int verbose)
-{
-  app_worker_map_t *wrk_map;
-  app_worker_t *app_wrk;
-
-  if (!app)
-    {
-      app_worker_format_local_connects (0, verbose);
-      return;
-    }
-
-  /* *INDENT-OFF* */
-  pool_foreach (wrk_map, app->worker_maps, ({
-    app_wrk = app_worker_get (wrk_map->wrk_index);
-    app_worker_format_local_connects (app_wrk, verbose);
-  }));
-  /* *INDENT-ON* */
-}
-
 u8 *
 format_application (u8 * s, va_list * args)
 {
@@ -1565,7 +1438,7 @@ format_application (u8 * s, va_list * args)
 }
 
 void
-application_format_all_listeners (vlib_main_t * vm, int do_local, int verbose)
+application_format_all_listeners (vlib_main_t * vm, int verbose)
 {
   application_t *app;
 
@@ -1575,29 +1448,17 @@ application_format_all_listeners (vlib_main_t * vm, int do_local, int verbose)
       return;
     }
 
-  if (do_local)
-    {
-      application_format_local_sessions (0, verbose);
-      /* *INDENT-OFF* */
-      pool_foreach (app, app_main.app_pool, ({
-        application_format_local_sessions (app, verbose);
-      }));
-      /* *INDENT-ON* */
-    }
-  else
-    {
-      application_format_listeners (0, verbose);
+  application_format_listeners (0, verbose);
 
-      /* *INDENT-OFF* */
-      pool_foreach (app, app_main.app_pool, ({
-        application_format_listeners (app, verbose);
-      }));
-      /* *INDENT-ON* */
-    }
+  /* *INDENT-OFF* */
+  pool_foreach (app, app_main.app_pool, ({
+    application_format_listeners (app, verbose);
+  }));
+  /* *INDENT-ON* */
 }
 
 void
-application_format_all_clients (vlib_main_t * vm, int do_local, int verbose)
+application_format_all_clients (vlib_main_t * vm, int verbose)
 {
   application_t *app;
 
@@ -1607,33 +1468,20 @@ application_format_all_clients (vlib_main_t * vm, int do_local, int verbose)
       return;
     }
 
-  if (do_local)
-    {
-      application_format_local_connects (0, verbose);
-
-      /* *INDENT-OFF* */
-      pool_foreach (app, app_main.app_pool, ({
-	application_format_local_connects (app, verbose);
-      }));
-      /* *INDENT-ON* */
-    }
-  else
-    {
-      application_format_connects (0, verbose);
+  application_format_connects (0, verbose);
 
-      /* *INDENT-OFF* */
-      pool_foreach (app, app_main.app_pool, ({
-        application_format_connects (app, verbose);
-      }));
-      /* *INDENT-ON* */
-    }
+  /* *INDENT-OFF* */
+  pool_foreach (app, app_main.app_pool, ({
+    application_format_connects (app, verbose);
+  }));
+  /* *INDENT-ON* */
 }
 
 static clib_error_t *
 show_app_command_fn (vlib_main_t * vm, unformat_input_t * input,
 		     vlib_cli_command_t * cmd)
 {
-  int do_server = 0, do_client = 0, do_local = 0;
+  int do_server = 0, do_client = 0;
   application_t *app;
   u32 app_index = ~0;
   int verbose = 0;
@@ -1646,8 +1494,6 @@ show_app_command_fn (vlib_main_t * vm, unformat_input_t * input,
 	do_server = 1;
       else if (unformat (input, "client"))
 	do_client = 1;
-      else if (unformat (input, "local"))
-	do_local = 1;
       else if (unformat (input, "%u", &app_index))
 	;
       else if (unformat (input, "verbose"))
@@ -1659,13 +1505,13 @@ show_app_command_fn (vlib_main_t * vm, unformat_input_t * input,
 
   if (do_server)
     {
-      application_format_all_listeners (vm, do_local, verbose);
+      application_format_all_listeners (vm, verbose);
       return 0;
     }
 
   if (do_client)
     {
-      application_format_all_clients (vm, do_local, verbose);
+      application_format_all_clients (vm, verbose);
       return 0;
     }
 
diff --git a/src/vnet/session/application.h b/src/vnet/session/application.h
index 0590c54507a..0904c232f30 100644
--- a/src/vnet/session/application.h
+++ b/src/vnet/session/application.h
@@ -59,19 +59,6 @@ typedef struct app_worker_
   u32 first_segment_manager;
   u8 first_segment_manager_in_use;
 
-  /*
-   * Local "cut through" connections specific
-   */
-
-  /** Segment manager used for incoming "cut through" connects */
-  u32 local_segment_manager;
-
-  /** Pool of local sessions the app owns (as a server) */
-  local_session_t *local_sessions;
-
-  /** Hash table of the app's local connects */
-  uword *local_connects;
-
   /** API index for the worker. Needed for multi-process apps */
   u32 api_client_index;
 
@@ -243,7 +230,10 @@ segment_manager_t *app_worker_get_connect_segment_manager (app_worker_t *);
 segment_manager_t
   * app_worker_get_or_alloc_connect_segment_manager (app_worker_t *);
 int app_worker_alloc_connects_segment_manager (app_worker_t * app);
-int app_worker_add_segment_notify (u32 app_or_wrk, u64 segment_handle);
+int app_worker_add_segment_notify (app_worker_t * app_wrk,
+				   u64 segment_handle);
+int app_worker_del_segment_notify (app_worker_t * app_wrk,
+				   u64 segment_handle);
 u32 app_worker_n_listeners (app_worker_t * app);
 session_t *app_worker_first_listener (app_worker_t * app,
 				      u8 fib_proto, u8 transport_proto);
@@ -257,12 +247,6 @@ u8 *format_app_worker (u8 * s, va_list * args);
 u8 *format_app_worker_listener (u8 * s, va_list * args);
 void app_worker_format_connects (app_worker_t * app_wrk, int verbose);
 int vnet_app_worker_add_del (vnet_app_worker_add_del_args_t * a);
-segment_manager_t *app_worker_get_local_segment_manager (app_worker_t *
-							 app_worker);
-segment_manager_t
-  * app_worker_get_local_segment_manager_w_session (app_worker_t * app_wrk,
-						    local_session_t * ls);
-
 
 uword unformat_application_proto (unformat_input_t * input, va_list * args);
 
diff --git a/src/vnet/session/application_local.c b/src/vnet/session/application_local.c
index 9e874ce536e..b5cac56130c 100644
--- a/src/vnet/session/application_local.c
+++ b/src/vnet/session/application_local.c
@@ -16,15 +16,6 @@
 #include <vnet/session/application_local.h>
 #include <vnet/session/session.h>
 
-typedef struct ct_connection_
-{
-  transport_connection_t connection;
-  u32 client_wrk;
-  u32 server_wrk;
-  u32 transport_listener_index;
-  transport_proto_t actual_tp;
-} ct_connection_t;
-
 ct_connection_t *connections;
 
 ct_connection_t *
@@ -56,9 +47,17 @@ ct_connection_free (ct_connection_t * ct)
   pool_put (connections, ct);
 }
 
+session_t *
+ct_session_get_peer (session_t * s)
+{
+  ct_connection_t *ct, *peer_ct;
+  ct = ct_connection_get (s->connection_index);
+  peer_ct = ct_connection_get (ct->peer_index);
+  return session_get (peer_ct->c_s_index, 0);
+}
+
 void
-application_local_listener_session_endpoint (session_t * ll,
-					     session_endpoint_t * sep)
+ct_session_endpoint (session_t * ll, session_endpoint_t * sep)
 {
   ct_connection_t *ct;
   ct = (ct_connection_t *) session_get_transport (ll);
@@ -67,184 +66,32 @@ application_local_listener_session_endpoint (session_t * ll,
   sep->is_ip4 = ct->c_is_ip4;
 }
 
-local_session_t *
-app_worker_local_session_alloc (app_worker_t * app_wrk)
-{
-  local_session_t *s;
-  pool_get (app_wrk->local_sessions, s);
-  clib_memset (s, 0, sizeof (*s));
-  s->app_wrk_index = app_wrk->wrk_index;
-  s->session_index = s - app_wrk->local_sessions;
-  s->session_type = session_type_from_proto_and_ip (TRANSPORT_PROTO_NONE, 0);
-  return s;
-}
-
-void
-app_worker_local_session_free (app_worker_t * app_wrk, local_session_t * s)
-{
-  pool_put (app_wrk->local_sessions, s);
-  if (CLIB_DEBUG)
-    clib_memset (s, 0xfc, sizeof (*s));
-}
-
-local_session_t *
-app_worker_get_local_session (app_worker_t * app_wrk, u32 session_index)
-{
-  if (pool_is_free_index (app_wrk->local_sessions, session_index))
-    return 0;
-  return pool_elt_at_index (app_wrk->local_sessions, session_index);
-}
-
-local_session_t *
-app_worker_get_local_session_from_handle (session_handle_t handle)
-{
-  app_worker_t *server_wrk;
-  u32 session_index, server_wrk_index;
-  local_session_parse_handle (handle, &server_wrk_index, &session_index);
-  server_wrk = app_worker_get_if_valid (server_wrk_index);
-  if (!server_wrk)
-    return 0;
-  return app_worker_get_local_session (server_wrk, session_index);
-}
-
-static inline u64
-application_client_local_connect_key (local_session_t * ls)
-{
-  return (((u64) ls->app_wrk_index) << 32 | (u64) ls->session_index);
-}
-
-static inline void
-application_client_local_connect_key_parse (u64 key, u32 * app_wrk_index,
-					    u32 * session_index)
-{
-  *app_wrk_index = key >> 32;
-  *session_index = key & 0xFFFFFFFF;
-}
-
-void
-app_worker_local_sessions_free (app_worker_t * app_wrk)
-{
-  u32 index, server_wrk_index, session_index;
-  u64 handle, *handles = 0;
-  app_worker_t *server_wrk;
-  segment_manager_t *sm;
-  local_session_t *ls;
-  int i;
-
-  /*
-   * Local sessions
-   */
-  if (app_wrk->local_sessions)
-    {
-      /* *INDENT-OFF* */
-      pool_foreach (ls, app_wrk->local_sessions, ({
-	app_worker_local_session_disconnect (app_wrk->wrk_index, ls);
-      }));
-      /* *INDENT-ON* */
-    }
-
-  /*
-   * Local connects
-   */
-  vec_reset_length (handles);
-  /* *INDENT-OFF* */
-  hash_foreach (handle, index, app_wrk->local_connects, ({
-    vec_add1 (handles, handle);
-  }));
-  /* *INDENT-ON* */
-
-  for (i = 0; i < vec_len (handles); i++)
-    {
-      application_client_local_connect_key_parse (handles[i],
-						  &server_wrk_index,
-						  &session_index);
-      server_wrk = app_worker_get_if_valid (server_wrk_index);
-      if (server_wrk)
-	{
-	  ls = app_worker_get_local_session (server_wrk, session_index);
-	  app_worker_local_session_disconnect (app_wrk->wrk_index, ls);
-	}
-    }
-
-  sm = segment_manager_get (app_wrk->local_segment_manager);
-  sm->app_wrk_index = SEGMENT_MANAGER_INVALID_APP_INDEX;
-  segment_manager_del (sm);
-}
-
 int
-app_worker_local_session_cleanup (app_worker_t * client_wrk,
-				  app_worker_t * server_wrk,
-				  local_session_t * ls)
+ct_session_connect_notify (session_t * ss)
 {
   svm_fifo_segment_private_t *seg;
-  session_t *listener;
-  segment_manager_t *sm;
-  u64 client_key;
-  u8 has_transport;
-
-  /* Retrieve listener transport type as it is the one that decides where
-   * the fifos are allocated */
-  has_transport = application_local_session_listener_has_transport (ls);
-  if (!has_transport)
-    sm = app_worker_get_local_segment_manager_w_session (server_wrk, ls);
-  else
-    {
-      listener = listen_session_get (ls->listener_index);
-      sm = app_worker_get_listen_segment_manager (server_wrk, listener);
-    }
-
-  seg = segment_manager_get_segment (sm, ls->svm_segment_index);
-  if (client_wrk)
-    {
-      client_key = application_client_local_connect_key (ls);
-      hash_unset (client_wrk->local_connects, client_key);
-    }
-
-  if (!has_transport)
-    {
-      application_t *server = application_get (server_wrk->app_index);
-      u64 segment_handle = segment_manager_segment_handle (sm, seg);
-      server->cb_fns.del_segment_callback (server_wrk->api_client_index,
-					   segment_handle);
-      if (client_wrk)
-	{
-	  application_t *client = application_get (client_wrk->app_index);
-	  client->cb_fns.del_segment_callback (client_wrk->api_client_index,
-					       segment_handle);
-	}
-      segment_manager_del_segment (sm, seg);
-    }
-
-  app_worker_local_session_free (server_wrk, ls);
-
-  return 0;
-}
-
-int
-app_worker_local_session_connect_notify (local_session_t * ls)
-{
-  svm_fifo_segment_private_t *seg;
-  app_worker_t *client_wrk, *server_wrk;
+  ct_connection_t *sct, *cct;
+  app_worker_t *client_wrk;
   segment_manager_t *sm;
-  application_t *client;
-  int rv, is_fail = 0;
   u64 segment_handle;
-  u64 client_key;
+  int is_fail = 0;
+  session_t *cs;
+  u32 ss_index;
 
-  client_wrk = app_worker_get (ls->client_wrk_index);
-  server_wrk = app_worker_get (ls->app_wrk_index);
-  client = application_get (client_wrk->app_index);
+  ss_index = ss->session_index;
+  sct = (ct_connection_t *) session_get_transport (ss);
+  client_wrk = app_worker_get (sct->client_wrk);
 
-  sm = app_worker_get_local_segment_manager_w_session (server_wrk, ls);
-  seg = segment_manager_get_segment_w_lock (sm, ls->svm_segment_index);
+  sm = segment_manager_get (ss->rx_fifo->segment_manager);
+  seg = segment_manager_get_segment_w_lock (sm, ss->rx_fifo->segment_index);
   segment_handle = segment_manager_segment_handle (sm, seg);
-  if ((rv = client->cb_fns.add_segment_callback (client_wrk->api_client_index,
-						 segment_handle)))
+
+  if (app_worker_add_segment_notify (client_wrk, segment_handle))
     {
       clib_warning ("failed to notify client %u of new segment",
-		    ls->client_wrk_index);
+		    sct->client_wrk);
       segment_manager_segment_reader_unlock (sm);
-      app_worker_local_session_disconnect (ls->client_wrk_index, ls);
+      session_close (ss);
       is_fail = 1;
     }
   else
@@ -252,17 +99,44 @@ app_worker_local_session_connect_notify (local_session_t * ls)
       segment_manager_segment_reader_unlock (sm);
     }
 
-  client->cb_fns.session_connected_callback (client_wrk->wrk_index,
-					     ls->client_opaque,
-					     (session_t *) ls, is_fail);
+  /* Alloc client session */
+  cct = ct_connection_get (sct->peer_index);
+
+  cs = session_alloc (0);
+  ss = session_get (ss_index, 0);
+  cs->session_type = ss->session_type;
+  cs->connection_index = sct->c_c_index;
+  cs->listener_index = SESSION_INVALID_INDEX;
+  cs->session_state = SESSION_STATE_CONNECTING;
+  cs->app_wrk_index = client_wrk->wrk_index;
+  cs->connection_index = cct->c_c_index;
+
+  cct->c_s_index = cs->session_index;
+
+  /* This will allocate fifos for the session. They won't be used for
+   * exchanging data but they will be used to close the connection if
+   * the segment manager/worker is freed */
+  if (app_worker_init_connected (client_wrk, cs))
+    {
+      session_close (ss);
+      return -1;
+    }
+
+  if (app_worker_connect_notify (client_wrk, is_fail ? 0 : cs,
+				 sct->client_opaque))
+    {
+      session_close (ss);
+      return -1;
+    }
+
+  cs = session_get (cct->c_s_index, 0);
+  cs->session_state = SESSION_STATE_READY;
 
-  client_key = application_client_local_connect_key (ls);
-  hash_set (client_wrk->local_connects, client_key, client_key);
   return 0;
 }
 
 static void
-application_local_session_fix_eventds (svm_msg_q_t * sq, svm_msg_q_t * cq)
+ct_session_fix_eventds (svm_msg_q_t * sq, svm_msg_q_t * cq)
 {
   int fd;
 
@@ -279,22 +153,19 @@ application_local_session_fix_eventds (svm_msg_q_t * sq, svm_msg_q_t * cq)
 }
 
 int
-app_worker_local_session_connect (app_worker_t * client_wrk,
-				  app_worker_t * server_wrk,
-				  session_t * ll, u32 opaque)
+ct_init_local_session (app_worker_t * client_wrk, app_worker_t * server_wrk,
+		       ct_connection_t * ct, session_t * ls, session_t * ll)
 {
   u32 seg_size, evt_q_sz, evt_q_elts, margin = 16 << 10;
   u32 round_rx_fifo_sz, round_tx_fifo_sz, sm_index;
   segment_manager_properties_t *props, *cprops;
-  int rv, has_transport, seg_index;
   svm_fifo_segment_private_t *seg;
   application_t *server, *client;
   segment_manager_t *sm;
-  local_session_t *ls;
   svm_msg_q_t *sq, *cq;
   u64 segment_handle;
+  int seg_index, rv;
 
-  ls = app_worker_local_session_alloc (server_wrk);
   server = application_get (server_wrk->app_index);
   client = application_get (client_wrk->app_index);
 
@@ -306,24 +177,7 @@ app_worker_local_session_connect (app_worker_t * client_wrk,
   round_tx_fifo_sz = 1 << max_log2 (props->tx_fifo_size);
   seg_size = round_rx_fifo_sz + round_tx_fifo_sz + evt_q_sz + margin;
 
-  has_transport = session_has_transport (ll);
-  if (!has_transport)
-    {
-      /* Local sessions don't have backing transport */
-      transport_connection_t *tc;
-      tc = session_get_transport (ll);
-      ls->port = tc->lcl_port;
-      sm = app_worker_get_local_segment_manager (server_wrk);
-    }
-  else
-    {
-      session_t *sl = (session_t *) ll;
-      transport_connection_t *tc;
-      tc = listen_session_get_transport (sl);
-      ls->port = tc->lcl_port;
-      sm = app_worker_get_listen_segment_manager (server_wrk, sl);
-    }
-
+  sm = app_worker_get_listen_segment_manager (server_wrk, ll);
   seg_index = segment_manager_add_segment (sm, seg_size);
   if (seg_index < 0)
     {
@@ -335,19 +189,20 @@ app_worker_local_session_connect (app_worker_t * client_wrk,
   cq = segment_manager_alloc_queue (seg, cprops);
 
   if (props->use_mq_eventfd)
-    application_local_session_fix_eventds (sq, cq);
+    ct_session_fix_eventds (sq, cq);
 
-  ls->server_evt_q = pointer_to_uword (sq);
-  ls->client_evt_q = pointer_to_uword (cq);
+  ct->server_evt_q = pointer_to_uword (sq);
+  ct->client_evt_q = pointer_to_uword (cq);
   rv = segment_manager_try_alloc_fifos (seg, props->rx_fifo_size,
-					props->tx_fifo_size,
-					&ls->rx_fifo, &ls->tx_fifo);
+					props->tx_fifo_size, &ls->rx_fifo,
+					&ls->tx_fifo);
   if (rv)
     {
       clib_warning ("failed to add fifos in cut-through segment");
       segment_manager_segment_reader_unlock (sm);
       goto failed;
     }
+
   sm_index = segment_manager_index (sm);
   ls->rx_fifo->ct_session_index = ls->session_index;
   ls->tx_fifo->ct_session_index = ls->session_index;
@@ -356,162 +211,107 @@ app_worker_local_session_connect (app_worker_t * client_wrk,
   ls->rx_fifo->segment_index = seg_index;
   ls->tx_fifo->segment_index = seg_index;
   ls->svm_segment_index = seg_index;
-  ls->listener_index = ll->session_index;
-  ls->client_wrk_index = client_wrk->wrk_index;
-  ls->client_opaque = opaque;
-  ls->listener_session_type = ll->session_type;
-  ls->session_state = SESSION_STATE_READY;
 
   segment_handle = segment_manager_segment_handle (sm, seg);
-  if ((rv = server->cb_fns.add_segment_callback (server_wrk->api_client_index,
-						 segment_handle)))
+  if ((rv = app_worker_add_segment_notify (server_wrk, segment_handle)))
     {
       clib_warning ("failed to notify server of new segment");
       segment_manager_segment_reader_unlock (sm);
       goto failed;
     }
   segment_manager_segment_reader_unlock (sm);
-  if ((rv = server->cb_fns.session_accept_callback ((session_t *) ls)))
-    {
-      clib_warning ("failed to send accept cut-through notify to server");
-      goto failed;
-    }
-  if (server->flags & APP_OPTIONS_FLAGS_IS_BUILTIN)
-    app_worker_local_session_connect_notify (ls);
+  ct->segment_handle = segment_handle;
 
   return 0;
 
 failed:
-  if (!has_transport)
-    segment_manager_del_segment (sm, seg);
+  segment_manager_del_segment (sm, seg);
   return rv;
 }
 
 int
-app_worker_local_session_disconnect (u32 app_wrk_index, local_session_t * ls)
+ct_connect (app_worker_t * client_wrk, session_t * ll,
+	    session_endpoint_cfg_t * sep)
 {
-  app_worker_t *client_wrk, *server_wrk;
+  u32 cct_index, ll_index, ll_ct_index;
+  ct_connection_t *sct, *cct, *ll_ct;
+  app_worker_t *server_wrk;
+  session_t *ss;
 
-  client_wrk = app_worker_get_if_valid (ls->client_wrk_index);
-  server_wrk = app_worker_get (ls->app_wrk_index);
+  ll_index = ll->session_index;
+  ll_ct_index = ll->connection_index;
 
-  if (ls->session_state == SESSION_STATE_CLOSED)
-    return app_worker_local_session_cleanup (client_wrk, server_wrk, ls);
+  cct = ct_connection_alloc ();
+  cct_index = cct->c_c_index;
+  sct = ct_connection_alloc ();
+  ll_ct = ct_connection_get (ll_ct_index);
 
-  if (app_wrk_index == ls->client_wrk_index)
-    {
-      mq_send_local_session_disconnected_cb (ls->app_wrk_index, ls);
-    }
-  else
-    {
-      if (!client_wrk)
-	{
-	  return app_worker_local_session_cleanup (client_wrk, server_wrk,
-						   ls);
-	}
-      else if (ls->session_state < SESSION_STATE_READY)
-	{
-	  application_t *client = application_get (client_wrk->app_index);
-	  client->cb_fns.session_connected_callback (client_wrk->wrk_index,
-						     ls->client_opaque,
-						     (session_t *) ls,
-						     1 /* is_fail */ );
-	  ls->session_state = SESSION_STATE_CLOSED;
-	  return app_worker_local_session_cleanup (client_wrk, server_wrk,
-						   ls);
-	}
-      else
-	{
-	  mq_send_local_session_disconnected_cb (client_wrk->wrk_index, ls);
-	}
-    }
+  /*
+   * Alloc and init client transport
+   */
+  cct = ct_connection_get (cct_index);
+  cct->c_thread_index = 0;
+  cct->c_rmt_port = sep->port;
+  cct->c_lcl_port = 0;
+  cct->c_is_ip4 = sep->is_ip4;
+  clib_memcpy (&cct->c_rmt_ip, &sep->ip, sizeof (sep->ip));
+  cct->actual_tp = ll_ct->actual_tp;
 
-  ls->session_state = SESSION_STATE_CLOSED;
+  /*
+   * Init server transport
+   */
+  sct->c_thread_index = 0;
+  sct->c_rmt_port = 0;
+  sct->c_lcl_port = ll_ct->c_lcl_port;
+  sct->c_is_ip4 = sep->is_ip4;
+  clib_memcpy (&sct->c_lcl_ip, &ll_ct->c_lcl_ip, sizeof (ll_ct->c_lcl_ip));
+  sct->client_wrk = client_wrk->wrk_index;
+  sct->c_proto = TRANSPORT_PROTO_NONE;
+  sct->client_opaque = sep->opaque;
+  sct->actual_tp = ll_ct->actual_tp;
+
+  sct->peer_index = cct->c_c_index;
+  cct->peer_index = sct->c_c_index;
 
-  return 0;
-}
+  /*
+   * Accept server session. Client session is created only after
+   * server confirms accept.
+   */
+  ss = session_alloc (0);
+  ll = listen_session_get (ll_index);
+  ss->session_type = ll->session_type;
+  ss->connection_index = sct->c_c_index;
+  ss->listener_index = ll->session_index;
+  ss->session_state = SESSION_STATE_CREATED;
 
-int
-app_worker_local_session_disconnect_w_index (u32 app_wrk_index, u32 ls_index)
-{
-  app_worker_t *app_wrk;
-  local_session_t *ls;
-  app_wrk = app_worker_get (app_wrk_index);
-  ls = app_worker_get_local_session (app_wrk, ls_index);
-  return app_worker_local_session_disconnect (app_wrk_index, ls);
-}
+  server_wrk = application_listener_select_worker (ll);
+  ss->app_wrk_index = server_wrk->wrk_index;
 
-void
-app_worker_format_local_sessions (app_worker_t * app_wrk, int verbose)
-{
-  vlib_main_t *vm = vlib_get_main ();
-  app_worker_t *client_wrk;
-  local_session_t *ls;
-  transport_proto_t tp;
-  u8 *conn = 0;
+  sct->c_s_index = ss->session_index;
+  sct->server_wrk = ss->app_wrk_index;
 
-  /* Header */
-  if (app_wrk == 0)
+  if (ct_init_local_session (client_wrk, server_wrk, sct, ss, ll))
     {
-      vlib_cli_output (vm, "%-40s%-15s%-20s", "Connection", "ServerApp",
-		       "ClientApp");
-      return;
+      clib_warning ("failed");
+      ct_connection_free (sct);
+      session_free (ss);
+      return -1;
     }
 
-  if (!pool_elts (app_wrk->local_sessions)
-      && !pool_elts (app_wrk->local_connects))
-    return;
-
-  /* *INDENT-OFF* */
-  pool_foreach (ls, app_wrk->local_sessions, ({
-    tp = session_type_transport_proto(ls->listener_session_type);
-    conn = format (0, "[L][%U] *:%u", format_transport_proto_short, tp,
-                   ls->port);
-    client_wrk = app_worker_get (ls->client_wrk_index);
-    vlib_cli_output (vm, "%-40v%-15u%-20u", conn, ls->app_index,
-                     client_wrk->app_index);
-    vec_reset_length (conn);
-  }));
-  /* *INDENT-ON* */
-
-  vec_free (conn);
-}
-
-void
-app_worker_format_local_connects (app_worker_t * app, int verbose)
-{
-  vlib_main_t *vm = vlib_get_main ();
-  u32 app_wrk_index, session_index;
-  app_worker_t *server_wrk;
-  local_session_t *ls;
-  u64 client_key;
-  u64 value;
-
-  /* Header */
-  if (app == 0)
+  ss->session_state = SESSION_STATE_ACCEPTING;
+  if (app_worker_accept_notify (server_wrk, ss))
     {
-      if (verbose)
-	vlib_cli_output (vm, "%-40s%-15s%-20s%-10s", "Connection", "App",
-			 "Peer App", "SegManager");
-      else
-	vlib_cli_output (vm, "%-40s%-15s%-20s", "Connection", "App",
-			 "Peer App");
-      return;
+      clib_warning ("failed");
+      ct_connection_free (sct);
+      session_free_w_fifos (ss);
+      return -1;
     }
 
-  if (!app->local_connects)
-    return;
-
-  /* *INDENT-OFF* */
-  hash_foreach (client_key, value, app->local_connects, ({
-    application_client_local_connect_key_parse (client_key, &app_wrk_index,
-                                                &session_index);
-    server_wrk = app_worker_get (app_wrk_index);
-    ls = app_worker_get_local_session (server_wrk, session_index);
-    vlib_cli_output (vm, "%-40s%-15s%-20s", "TODO", ls->app_wrk_index,
-                     ls->client_wrk_index);
-  }));
-  /* *INDENT-ON* */
+  cct->client_evt_q = sct->client_evt_q;
+  cct->server_evt_q = sct->server_evt_q;
+  cct->segment_handle = sct->segment_handle;
+
+  return 0;
 }
 
 u32
@@ -545,6 +345,96 @@ ct_listener_get (u32 ct_index)
   return (transport_connection_t *) ct_connection_get (ct_index);
 }
 
+int
+ct_session_connect (transport_endpoint_cfg_t * tep)
+{
+  session_endpoint_cfg_t *sep_ext;
+  session_endpoint_t *sep;
+  app_worker_t *app_wrk;
+  session_handle_t lh;
+  application_t *app;
+  app_listener_t *al;
+  u32 table_index;
+  session_t *ll;
+  u8 fib_proto;
+
+  sep_ext = (session_endpoint_cfg_t *) tep;
+  sep = (session_endpoint_t *) tep;
+  app_wrk = app_worker_get (sep_ext->app_wrk_index);
+  app = application_get (app_wrk->app_index);
+
+  sep->transport_proto = sep_ext->original_tp;
+  table_index = application_local_session_table (app);
+  lh = session_lookup_local_endpoint (table_index, sep);
+  if (lh == SESSION_DROP_HANDLE)
+    return VNET_API_ERROR_APP_CONNECT_FILTERED;
+
+  if (lh == SESSION_INVALID_HANDLE)
+    goto global_scope;
+
+  ll = listen_session_get_from_handle (lh);
+  al = app_listener_get_w_session (ll);
+
+  /*
+   * Break loop if rule in local table points to connecting app. This
+   * can happen if client is a generic proxy. Route connect through
+   * global table instead.
+   */
+  if (al->app_index == app->app_index)
+    goto global_scope;
+
+  return ct_connect (app_wrk, ll, sep_ext);
+
+  /*
+   * If nothing found, check the global scope for locally attached
+   * destinations. Make sure first that we're allowed to.
+   */
+
+global_scope:
+  if (session_endpoint_is_local (sep))
+    return VNET_API_ERROR_SESSION_CONNECT;
+
+  if (!application_has_global_scope (app))
+    return VNET_API_ERROR_APP_CONNECT_SCOPE;
+
+  fib_proto = session_endpoint_fib_proto (sep);
+  table_index = application_session_table (app, fib_proto);
+  ll = session_lookup_listener (table_index, sep);
+
+  if (ll)
+    return ct_connect (app_wrk, ll, sep_ext);
+
+  /* Failed to connect but no error */
+  return 1;
+}
+
+void
+ct_session_close (u32 ct_index, u32 thread_index)
+{
+  ct_connection_t *ct, *peer_ct;
+  session_t *s;
+
+  ct = ct_connection_get (ct_index);
+  peer_ct = ct_connection_get (ct->peer_index);
+  if (peer_ct)
+    {
+      peer_ct->peer_index = ~0;
+      session_transport_closing_notify (&peer_ct->connection);
+    }
+
+  s = session_get (ct->c_s_index, 0);
+  app_worker_del_segment_notify (app_worker_get (s->app_wrk_index),
+				 ct->segment_handle);
+  session_free_w_fifos (s);
+  ct_connection_free (ct);
+}
+
+transport_connection_t *
+ct_session_get (u32 ct_index, u32 thread_index)
+{
+  return (transport_connection_t *) ct_connection_get (ct_index);
+}
+
 static u8 *
 format_ct_connection_id (u8 * s, va_list * args)
 {
@@ -583,14 +473,57 @@ format_ct_listener (u8 * s, va_list * args)
   return s;
 }
 
+u8 *
+format_ct_connection (u8 * s, va_list * args)
+{
+  ct_connection_t *ct = va_arg (*args, ct_connection_t *);
+  u32 verbose = va_arg (*args, u32);
+
+  if (!ct)
+    return s;
+  s = format (s, "%-50U", format_ct_connection_id, ct);
+  if (verbose)
+    {
+      s = format (s, "%-15s", "ESTABLISHED");
+      if (verbose > 1)
+	{
+	  s = format (s, "\n");
+	}
+    }
+  return s;
+}
+
+u8 *
+format_ct_session (u8 * s, va_list * args)
+{
+  u32 ct_index = va_arg (*args, u32);
+  u32 __clib_unused thread_index = va_arg (*args, u32);
+  u32 verbose = va_arg (*args, u32);
+  ct_connection_t *ct;
+
+  ct = ct_connection_get (ct_index);
+  if (!ct)
+    {
+      s = format (s, "empty\n");
+      return s;
+    }
+
+  s = format (s, "%U", format_ct_connection, ct, verbose);
+  return s;
+}
+
 /* *INDENT-OFF* */
 const static transport_proto_vft_t cut_thru_proto = {
   .start_listen = ct_start_listen,
   .stop_listen = ct_stop_listen,
   .get_listener = ct_listener_get,
+  .connect = ct_session_connect,
+  .close = ct_session_close,
+  .get_connection = ct_session_get,
   .tx_type = TRANSPORT_TX_INTERNAL,
   .service_type = TRANSPORT_SERVICE_APP,
   .format_listener = format_ct_listener,
+  .format_connection = format_ct_session,
 };
 /* *INDENT-ON* */
 
diff --git a/src/vnet/session/application_local.h b/src/vnet/session/application_local.h
index d54b5ebdb6a..5d6e6c1ec7b 100644
--- a/src/vnet/session/application_local.h
+++ b/src/vnet/session/application_local.h
@@ -18,46 +18,25 @@
 #define SRC_VNET_SESSION_APPLICATION_LOCAL_H_
 
 #include <vnet/session/application.h>
+#include <vnet/session/transport.h>
 
-local_session_t *application_local_listen_session_alloc (application_t * app);
-void application_local_listener_session_endpoint (session_t * ll,
-						  session_endpoint_t * sep);
-
-local_session_t *app_worker_local_session_alloc (app_worker_t * app_wrk);
-void app_worker_local_session_free (app_worker_t * app_wrk,
-				    local_session_t * s);
-local_session_t *app_worker_get_local_session (app_worker_t * app_wrk,
-					       u32 session_index);
-local_session_t *app_worker_get_local_session_from_handle (session_handle_t
-							   handle);
-
-void app_worker_local_sessions_free (app_worker_t * app_wrk);
-int app_worker_local_session_cleanup (app_worker_t * client_wrk,
-				      app_worker_t * server_wrk,
-				      local_session_t * ls);
-
-int app_worker_local_session_connect_notify (local_session_t * ls);
-int app_worker_local_session_connect (app_worker_t * client,
-				      app_worker_t * server,
-				      session_t * ls, u32 opaque);
-int app_worker_local_session_disconnect (u32 app_or_wrk,
-					 local_session_t * ls);
-int app_worker_local_session_disconnect_w_index (u32 app_or_wrk,
-						 u32 ls_index);
-
-void app_worker_format_local_sessions (app_worker_t * app_wrk, int verbose);
-void app_worker_format_local_connects (app_worker_t * app, int verbose);
-
-void mq_send_local_session_disconnected_cb (u32 app_or_wrk,
-					    local_session_t * ls);
-
-always_inline u8
-application_local_session_listener_has_transport (local_session_t * ls)
+typedef struct ct_connection_
 {
-  transport_proto_t tp;
-  tp = session_type_transport_proto (ls->listener_session_type);
-  return (tp != TRANSPORT_PROTO_NONE);
-}
+  transport_connection_t connection;
+  u32 client_wrk;
+  u32 server_wrk;
+  u32 transport_listener_index;
+  transport_proto_t actual_tp;
+  u64 server_evt_q;
+  u64 client_evt_q;
+  u32 client_opaque;
+  u32 peer_index;
+  u64 segment_handle;
+} ct_connection_t;
+
+session_t *ct_session_get_peer (session_t * s);
+void ct_session_endpoint (session_t * ll, session_endpoint_t * sep);
+int ct_session_connect_notify (session_t * ls);
 
 #endif /* SRC_VNET_SESSION_APPLICATION_LOCAL_H_ */
 
diff --git a/src/vnet/session/application_worker.c b/src/vnet/session/application_worker.c
index 51ebe1c08e6..d02149dc58f 100644
--- a/src/vnet/session/application_worker.c
+++ b/src/vnet/session/application_worker.c
@@ -15,7 +15,6 @@
 
 #include <vnet/session/application.h>
 #include <vnet/session/application_interface.h>
-#include <vnet/session/application_local.h>
 #include <vnet/session/session.h>
 
 /**
@@ -34,7 +33,6 @@ app_worker_alloc (application_t * app)
   app_wrk->wrk_map_index = ~0;
   app_wrk->connects_seg_manager = APP_INVALID_SEGMENT_MANAGER_INDEX;
   app_wrk->first_segment_manager = APP_INVALID_SEGMENT_MANAGER_INDEX;
-  app_wrk->local_segment_manager = APP_INVALID_SEGMENT_MANAGER_INDEX;
   APP_DBG ("New app %v worker %u", app_get_name (app), app_wrk->wrk_index);
   return app_wrk;
 }
@@ -113,11 +111,6 @@ app_worker_free (app_worker_t * app_wrk)
 	segment_manager_del (sm);
     }
 
-  /*
-   * Local sessions
-   */
-  app_worker_local_sessions_free (app_wrk);
-
   pool_put (app_workers, app_wrk);
   if (CLIB_DEBUG)
     clib_memset (app_wrk, 0xfe, sizeof (*app_wrk));
@@ -227,12 +220,35 @@ app_worker_start_listen (app_worker_t * app_wrk,
   return 0;
 }
 
-int
-app_worker_stop_listen (app_worker_t * app_wrk, app_listener_t * al)
+static void
+app_worker_stop_listen_session (app_worker_t * app_wrk, session_t * ls)
 {
   session_handle_t handle;
   segment_manager_t *sm;
   uword *sm_indexp;
+
+  handle = listen_session_get_handle (ls);
+  sm_indexp = hash_get (app_wrk->listeners_table, handle);
+  if (PREDICT_FALSE (!sm_indexp))
+    return;
+
+  sm = segment_manager_get (*sm_indexp);
+  if (app_wrk->first_segment_manager == *sm_indexp)
+    {
+      /* Delete sessions but don't remove segment manager */
+      app_wrk->first_segment_manager_in_use = 0;
+      segment_manager_del_sessions (sm);
+    }
+  else
+    {
+      segment_manager_init_del (sm);
+    }
+  hash_unset (app_wrk->listeners_table, handle);
+}
+
+int
+app_worker_stop_listen (app_worker_t * app_wrk, app_listener_t * al)
+{
   session_t *ls;
 
   if (!clib_bitmap_get (al->workers, app_wrk->wrk_map_index))
@@ -241,58 +257,13 @@ app_worker_stop_listen (app_worker_t * app_wrk, app_listener_t * al)
   if (al->session_index != SESSION_INVALID_INDEX)
     {
       ls = listen_session_get (al->session_index);
-      handle = listen_session_get_handle (ls);
-
-      sm_indexp = hash_get (app_wrk->listeners_table, handle);
-      if (PREDICT_FALSE (!sm_indexp))
-	{
-	  clib_warning ("listener handle was removed %llu!", handle);
-	  return -1;
-	}
-
-      sm = segment_manager_get (*sm_indexp);
-      if (app_wrk->first_segment_manager == *sm_indexp)
-	{
-	  /* Delete sessions but don't remove segment manager */
-	  app_wrk->first_segment_manager_in_use = 0;
-	  segment_manager_del_sessions (sm);
-	}
-      else
-	{
-	  segment_manager_init_del (sm);
-	}
-      hash_unset (app_wrk->listeners_table, handle);
+      app_worker_stop_listen_session (app_wrk, ls);
     }
 
   if (al->local_index != SESSION_INVALID_INDEX)
     {
-      local_session_t *local;
       ls = listen_session_get (al->local_index);
-      handle = listen_session_get_handle (ls);
-
-      /* *INDENT-OFF* */
-      pool_foreach (local, app_wrk->local_sessions, ({
-        if (local->listener_index == ls->session_index)
-          app_worker_local_session_disconnect (app_wrk->wrk_index, local);
-      }));
-      /* *INDENT-ON* */
-
-      sm_indexp = hash_get (app_wrk->listeners_table, handle);
-      if (PREDICT_FALSE (!sm_indexp))
-	return -1;
-
-      sm = segment_manager_get (*sm_indexp);
-      if (app_wrk->first_segment_manager == *sm_indexp)
-	{
-	  /* Delete sessions but don't remove segment manager */
-	  app_wrk->first_segment_manager_in_use = 0;
-	  segment_manager_del_sessions (sm);
-	}
-      else
-	{
-	  segment_manager_init_del (sm);
-	}
-      hash_unset (app_wrk->listeners_table, handle);
+      app_worker_stop_listen_session (app_wrk, ls);
     }
 
   clib_bitmap_set_no_check (al->workers, app_wrk->wrk_map_index, 0);
@@ -312,8 +283,12 @@ app_worker_init_accepted (session_t * s)
   listener = listen_session_get (s->listener_index);
   app_wrk = application_listener_select_worker (listener);
   s->app_wrk_index = app_wrk->wrk_index;
+
   sm = app_worker_get_listen_segment_manager (app_wrk, listener);
-  return app_worker_alloc_session_fifos (sm, s);
+  if (app_worker_alloc_session_fifos (sm, s))
+    return -1;
+
+  return 0;
 }
 
 int
@@ -499,14 +474,21 @@ app_worker_proxy_listener (app_worker_t * app_wrk, u8 fib_proto,
  * Send an API message to the external app, to map new segment
  */
 int
-app_worker_add_segment_notify (u32 app_wrk_index, u64 segment_handle)
+app_worker_add_segment_notify (app_worker_t * app_wrk, u64 segment_handle)
 {
-  app_worker_t *app_wrk = app_worker_get (app_wrk_index);
   application_t *app = application_get (app_wrk->app_index);
   return app->cb_fns.add_segment_callback (app_wrk->api_client_index,
 					   segment_handle);
 }
 
+int
+app_worker_del_segment_notify (app_worker_t * app_wrk, u64 segment_handle)
+{
+  application_t *app = application_get (app_wrk->app_index);
+  return app->cb_fns.del_segment_callback (app_wrk->api_client_index,
+					   segment_handle);
+}
+
 u8
 app_worker_application_is_builtin (app_worker_t * app_wrk)
 {
@@ -660,25 +642,6 @@ app_worker_lock_and_send_event (app_worker_t * app, session_t * s,
   return app_send_evt_handler_fns[evt_type] (app, s, 1 /* lock */ );
 }
 
-segment_manager_t *
-app_worker_get_local_segment_manager (app_worker_t * app_worker)
-{
-  return segment_manager_get (app_worker->local_segment_manager);
-}
-
-segment_manager_t *
-app_worker_get_local_segment_manager_w_session (app_worker_t * app_wrk,
-						local_session_t * ls)
-{
-  session_t *listener;
-  if (application_local_session_listener_has_transport (ls))
-    {
-      listener = listen_session_get (ls->listener_index);
-      return app_worker_get_listen_segment_manager (app_wrk, listener);
-    }
-  return segment_manager_get (app_wrk->local_segment_manager);
-}
-
 u8 *
 format_app_worker_listener (u8 * s, va_list * args)
 {
diff --git a/src/vnet/session/segment_manager.c b/src/vnet/session/segment_manager.c
index 8e6c2818a20..63af299e9a8 100644
--- a/src/vnet/session/segment_manager.c
+++ b/src/vnet/session/segment_manager.c
@@ -16,7 +16,6 @@
 #include <vnet/session/segment_manager.h>
 #include <vnet/session/session.h>
 #include <vnet/session/application.h>
-#include <vnet/session/application_local.h>
 
 segment_manager_main_t segment_manager_main;
 
@@ -380,6 +379,7 @@ void
 segment_manager_del_sessions (segment_manager_t * sm)
 {
   svm_fifo_segment_private_t *fifo_segment;
+  session_handle_t *handles = 0, *handle;
   session_t *session;
   svm_fifo_t *fifo;
 
@@ -398,16 +398,11 @@ segment_manager_del_sessions (segment_manager_t * sm)
     while (fifo)
       {
 	if (fifo->ct_session_index != SVM_FIFO_INVALID_SESSION_INDEX)
-	  {
-	    svm_fifo_t *next = fifo->next;
-	    app_worker_local_session_disconnect_w_index (sm->app_wrk_index,
-	                                                  fifo->ct_session_index);
-	    fifo = next;
-	    continue;
-	  }
-	session = session_get (fifo->master_session_index,
-	                       fifo->master_thread_index);
-	session_close (session);
+	  session = session_get (fifo->ct_session_index, 0);
+	else
+	  session = session_get (fifo->master_session_index,
+	                         fifo->master_thread_index);
+	vec_add1 (handles, session_handle (session));
 	fifo = fifo->next;
       }
 
@@ -416,6 +411,9 @@ segment_manager_del_sessions (segment_manager_t * sm)
      */
   }));
   /* *INDENT-ON* */
+
+  vec_foreach (handle, handles)
+    session_close (session_get_from_handle (*handle));
 }
 
 /**
@@ -553,9 +551,10 @@ alloc_check:
 
       if (added_a_segment)
 	{
+	  app_worker_t *app_wrk;
 	  segment_handle = segment_manager_segment_handle (sm, fifo_segment);
-	  rv = app_worker_add_segment_notify (sm->app_wrk_index,
-					      segment_handle);
+	  app_wrk = app_worker_get (sm->app_wrk_index);
+	  rv = app_worker_add_segment_notify (app_wrk, segment_handle);
 	}
       /* Drop the lock after app is notified */
       segment_manager_segment_reader_unlock (sm);
diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c
index 110db2c2a4a..36aadcb7c40 100644
--- a/src/vnet/session/session.c
+++ b/src/vnet/session/session.c
@@ -123,6 +123,13 @@ session_program_transport_close (session_t * s)
   session_manager_worker_t *wrk;
   session_event_t *evt;
 
+  if (!session_has_transport (s))
+    {
+      /* Polling may not be enabled on main thread so close now */
+      session_transport_close (s);
+      return;
+    }
+
   /* If we are in the handler thread, or being called with the worker barrier
    * held, just append a new event to pending disconnects vector. */
   if (vlib_thread_is_main_w_barrier () || thread_index == s->thread_index)
diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c
index cf5a5cd7fd1..81654e426b5 100755
--- a/src/vnet/session/session_api.c
+++ b/src/vnet/session/session_api.c
@@ -252,15 +252,16 @@ send_session_accept_callback (session_t * s)
     }
   else
     {
-      local_session_t *ls = (local_session_t *) s;
-      listener = listen_session_get (ls->listener_index);
+      ct_connection_t *ct;
+      ct = (ct_connection_t *) session_get_transport (s);
+      listener = listen_session_get (s->listener_index);
       al = app_listener_get (server, listener->al_index);
       mp->listener_handle = app_listener_handle (al);
       mp->is_ip4 = session_type_is_ip4 (listener->session_type);
-      mp->handle = application_local_session_handle (ls);
-      mp->port = ls->port;
-      mp->vpp_event_queue_address = ls->client_evt_q;
-      mp->server_event_queue_address = ls->server_evt_q;
+      mp->handle = session_handle (s);
+      mp->port = ct->c_rmt_port;
+      mp->vpp_event_queue_address = ct->client_evt_q;
+      mp->server_event_queue_address = ct->server_evt_q;
     }
   vl_msg_api_send_shmem (reg->vl_input_queue, (u8 *) & mp);
 
@@ -355,11 +356,12 @@ send_session_connected_callback (u32 app_wrk_index, u32 api_context,
     }
   else
     {
-      local_session_t *ls = (local_session_t *) s;
-      mp->handle = application_local_session_handle (ls);
-      mp->lcl_port = ls->port;
-      mp->vpp_event_queue_address = ls->server_evt_q;
-      mp->client_event_queue_address = ls->client_evt_q;
+      ct_connection_t *ct;
+      ct = (ct_connection_t *) session_get_transport (s);
+      mp->handle = session_handle (s);
+      mp->lcl_port = ct->c_lcl_port;
+      mp->vpp_event_queue_address = ct->server_evt_q;
+      mp->client_event_queue_address = ct->client_evt_q;
       mp->server_rx_fifo = pointer_to_uword (s->tx_fifo);
       mp->server_tx_fifo = pointer_to_uword (s->rx_fifo);
     }
@@ -451,24 +453,25 @@ mq_send_session_accepted_cb (session_t * s)
     }
   else
     {
-      local_session_t *ls = (local_session_t *) s;
       u8 main_thread = vlib_num_workers ()? 1 : 0;
+      ct_connection_t *ct;
 
+      ct = (ct_connection_t *) session_get_transport (s);
       send_app_cut_through_registration_add (app_wrk->api_client_index,
 					     app_wrk->wrk_map_index,
-					     ls->server_evt_q,
-					     ls->client_evt_q);
+					     ct->server_evt_q,
+					     ct->client_evt_q);
 
-      listener = listen_session_get (ls->listener_index);
+      listener = listen_session_get (s->listener_index);
       al = app_listener_get (app, listener->al_index);
       mp->listener_handle = app_listener_handle (al);
       mp->is_ip4 = session_type_is_ip4 (listener->session_type);
-      mp->handle = application_local_session_handle (ls);
-      mp->port = ls->port;
+      mp->handle = session_handle (s);
+      mp->port = ct->c_rmt_port;
       vpp_queue = session_manager_get_vpp_event_queue (main_thread);
       mp->vpp_event_queue_address = pointer_to_uword (vpp_queue);
-      mp->client_event_queue_address = ls->client_evt_q;
-      mp->server_event_queue_address = ls->server_evt_q;
+      mp->client_event_queue_address = ct->client_evt_q;
+      mp->server_event_queue_address = ct->server_evt_q;
     }
   svm_msg_q_add_and_unlock (app_mq, msg);
 
@@ -530,20 +533,6 @@ mq_send_session_disconnected_cb (session_t * s)
 				 SESSION_CTRL_EVT_DISCONNECTED);
 }
 
-void
-mq_send_local_session_disconnected_cb (u32 app_wrk_index,
-				       local_session_t * ls)
-{
-  app_worker_t *app_wrk = app_worker_get (app_wrk_index);
-  session_handle_t sh = application_local_session_handle (ls);
-
-  mq_send_session_close_evt (app_wrk, sh, SESSION_CTRL_EVT_DISCONNECTED);
-
-  if (svm_fifo_n_subscribers (ls->rx_fifo))
-    mq_notify_close_subscribers (app_wrk->app_index, sh, ls->rx_fifo,
-				 SESSION_CTRL_EVT_DISCONNECTED);
-}
-
 static void
 mq_send_session_reset_cb (session_t * s)
 {
@@ -590,8 +579,6 @@ mq_send_session_connected_cb (u32 app_wrk_index, u32 api_context,
   if (is_fail)
     goto done;
 
-  mp->segment_handle = session_segment_handle (s);
-
   if (session_has_transport (s))
     {
       tc = session_get_transport (s);
@@ -609,25 +596,30 @@ mq_send_session_connected_cb (u32 app_wrk_index, u32 api_context,
       mp->lcl_port = tc->lcl_port;
       mp->server_rx_fifo = pointer_to_uword (s->rx_fifo);
       mp->server_tx_fifo = pointer_to_uword (s->tx_fifo);
+      mp->segment_handle = session_segment_handle (s);
     }
   else
     {
-      local_session_t *ls = (local_session_t *) s;
       u8 main_thread = vlib_num_workers ()? 1 : 0;
+      ct_connection_t *cct;
+      session_t *ss;
 
+      cct = (ct_connection_t *) session_get_transport (s);
       send_app_cut_through_registration_add (app_wrk->api_client_index,
 					     app_wrk->wrk_map_index,
-					     ls->client_evt_q,
-					     ls->server_evt_q);
+					     cct->client_evt_q,
+					     cct->server_evt_q);
 
-      mp->handle = application_local_session_handle (ls);
-      mp->lcl_port = ls->port;
+      mp->handle = session_handle (s);
+      mp->lcl_port = cct->c_lcl_port;
       vpp_mq = session_manager_get_vpp_event_queue (main_thread);
       mp->vpp_event_queue_address = pointer_to_uword (vpp_mq);
-      mp->client_event_queue_address = ls->client_evt_q;
-      mp->server_event_queue_address = ls->server_evt_q;
-      mp->server_rx_fifo = pointer_to_uword (s->tx_fifo);
-      mp->server_tx_fifo = pointer_to_uword (s->rx_fifo);
+      mp->client_event_queue_address = cct->client_evt_q;
+      mp->server_event_queue_address = cct->server_evt_q;
+      ss = ct_session_get_peer (s);
+      mp->server_rx_fifo = pointer_to_uword (ss->tx_fifo);
+      mp->server_tx_fifo = pointer_to_uword (ss->rx_fifo);
+      mp->segment_handle = session_segment_handle (ss);
     }
 
 done:
@@ -686,7 +678,7 @@ mq_send_session_bound_cb (u32 app_wrk_index, u32 api_context,
   vpp_evt_q = session_manager_get_vpp_event_queue (0);
   mp->vpp_evt_q = pointer_to_uword (vpp_evt_q);
 
-  if (ls && session_transport_service_type (ls) == TRANSPORT_SERVICE_CL)
+  if (session_transport_service_type (ls) == TRANSPORT_SERVICE_CL)
     {
       mp->rx_fifo = pointer_to_uword (ls->rx_fifo);
       mp->tx_fifo = pointer_to_uword (ls->tx_fifo);
@@ -1087,7 +1079,6 @@ static void
 vl_api_accept_session_reply_t_handler (vl_api_accept_session_reply_t * mp)
 {
   vnet_disconnect_args_t _a = { 0 }, *a = &_a;
-  local_session_t *ls;
   session_t *s;
 
   /* Server isn't interested, kill the session */
@@ -1099,34 +1090,21 @@ vl_api_accept_session_reply_t_handler (vl_api_accept_session_reply_t * mp)
       return;
     }
 
-  if (session_handle_is_local (mp->handle))
+  s = session_get_from_handle_if_valid (mp->handle);
+  if (!s)
     {
-      ls = app_worker_get_local_session_from_handle (mp->handle);
-      if (!ls || ls->app_wrk_index != mp->context)
-	{
-	  clib_warning ("server %u doesn't own local handle %llu",
-			mp->context, mp->handle);
-	  return;
-	}
-      if (app_worker_local_session_connect_notify (ls))
-	return;
-      ls->session_state = SESSION_STATE_READY;
+      clib_warning ("session doesn't exist");
+      return;
     }
-  else
+  if (s->app_wrk_index != mp->context)
     {
-      s = session_get_from_handle_if_valid (mp->handle);
-      if (!s)
-	{
-	  clib_warning ("session doesn't exist");
-	  return;
-	}
-      if (s->app_wrk_index != mp->context)
-	{
-	  clib_warning ("app doesn't own session");
-	  return;
-	}
-      s->session_state = SESSION_STATE_READY;
+      clib_warning ("app doesn't own session");
+      return;
     }
+  s->session_state = SESSION_STATE_READY;
+
+  if (!session_has_transport (s))
+    ct_session_connect_notify (s);
 }
 
 static void
diff --git a/src/vnet/session/session_lookup.c b/src/vnet/session/session_lookup.c
index 854da7fdd3c..d7e932a96e4 100644
--- a/src/vnet/session/session_lookup.c
+++ b/src/vnet/session/session_lookup.c
@@ -1304,7 +1304,7 @@ u8 *
 format_ip4_session_lookup_kvp (u8 * s, va_list * args)
 {
   clib_bihash_kv_16_8_t *kvp = va_arg (*args, clib_bihash_kv_16_8_t *);
-  u32 is_local = va_arg (*args, u32), app_wrk_index, session_index;
+  u32 is_local = va_arg (*args, u32);
   v4_connection_key_t *key = (v4_connection_key_t *) kvp->key;
   session_t *session;
   app_worker_t *app_wrk;
@@ -1324,8 +1324,8 @@ format_ip4_session_lookup_kvp (u8 * s, va_list * args)
     }
   else
     {
-      local_session_parse_handle (kvp->value, &app_wrk_index, &session_index);
-      app_wrk = app_worker_get (app_wrk_index);
+      session = session_get_from_handle (kvp->value);
+      app_wrk = app_worker_get (session->app_wrk_index);
       app_name = application_name_from_index (app_wrk->app_index);
       str = format (0, "[%U] %U:%d", format_transport_proto_short, key->proto,
 		    format_ip4_address, &key->src,
diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c
index 7855bf8c56d..e6309429194 100644
--- a/src/vnet/session/session_node.c
+++ b/src/vnet/session/session_node.c
@@ -25,6 +25,14 @@
 #include <vnet/session/session_debug.h>
 #include <svm/queue.h>
 
+static void session_mq_accepted_reply_handler (void *data);
+
+static void
+accepted_notify_cb (void *data, u32 data_len)
+{
+  session_mq_accepted_reply_handler (data);
+}
+
 static void
 session_mq_accepted_reply_handler (void *data)
 {
@@ -32,7 +40,6 @@ session_mq_accepted_reply_handler (void *data)
   vnet_disconnect_args_t _a = { 0 }, *a = &_a;
   session_state_t old_state;
   app_worker_t *app_wrk;
-  local_session_t *ls;
   session_t *s;
 
   /* Server isn't interested, kill the session */
@@ -44,38 +51,35 @@ session_mq_accepted_reply_handler (void *data)
       return;
     }
 
-  if (session_handle_is_local (mp->handle))
+  /* Mail this back from the main thread. We're not polling in main
+   * thread so we're using other workers for notifications. */
+  if (vlib_num_workers () && vlib_get_thread_index () != 0
+      && session_thread_from_handle (mp->handle) == 0)
     {
-      ls = app_worker_get_local_session_from_handle (mp->handle);
-      if (!ls)
-	{
-	  clib_warning ("unknown local handle 0x%lx", mp->handle);
-	  return;
-	}
-      app_wrk = app_worker_get (ls->app_wrk_index);
-      if (app_wrk->app_index != mp->context)
-	{
-	  clib_warning ("server %u doesn't own local handle 0x%lx",
-			mp->context, mp->handle);
-	  return;
-	}
-      if (app_worker_local_session_connect_notify (ls))
+      vl_api_rpc_call_main_thread (accepted_notify_cb, data,
+				   sizeof (session_accepted_reply_msg_t));
+      return;
+    }
+
+  s = session_get_from_handle_if_valid (mp->handle);
+  if (!s)
+    return;
+
+  app_wrk = app_worker_get (s->app_wrk_index);
+  if (app_wrk->app_index != mp->context)
+    {
+      clib_warning ("app doesn't own session");
+      return;
+    }
+
+  if (!session_has_transport (s))
+    {
+      if (ct_session_connect_notify (s))
 	return;
-      ls->session_state = SESSION_STATE_READY;
+      s->session_state = SESSION_STATE_READY;
     }
   else
     {
-      s = session_get_from_handle_if_valid (mp->handle);
-      if (!s)
-	return;
-
-      app_wrk = app_worker_get (s->app_wrk_index);
-      if (app_wrk->app_index != mp->context)
-	{
-	  clib_warning ("app doesn't own session");
-	  return;
-	}
-
       old_state = s->session_state;
       s->session_state = SESSION_STATE_READY;
       if (!svm_fifo_is_empty (s->rx_fifo))
diff --git a/src/vnet/session/session_types.h b/src/vnet/session/session_types.h
index 3709445ccac..7f1acdd5bf4 100644
--- a/src/vnet/session/session_types.h
+++ b/src/vnet/session/session_types.h
@@ -19,7 +19,6 @@
 #include <svm/svm_fifo.h>
 #include <vnet/session/transport_types.h>
 
-#define SESSION_LOCAL_HANDLE_PREFIX 	0x7FFFFFFF
 #define SESSION_LISTENER_PREFIX		0x5FFFFFFF
 
 #define foreach_session_endpoint_fields				\
@@ -41,6 +40,7 @@ typedef struct _session_endpoint_cfg
   u32 app_wrk_index;
   u32 opaque;
   u32 ns_index;
+  u8 original_tp;
   u8 *hostname;
 } session_endpoint_cfg_t;
 
@@ -275,111 +275,6 @@ session_parse_handle (session_handle_t handle, u32 * index,
   *thread_index = session_thread_from_handle (handle);
 }
 
-always_inline u8
-session_handle_is_local (session_handle_t handle)
-{
-  if ((handle >> 32) == SESSION_LOCAL_HANDLE_PREFIX)
-    return 1;
-  return 0;
-}
-
-typedef struct local_session_
-{
-  /** fifo pointers. Once allocated, these do not move */
-  svm_fifo_t *rx_fifo;
-  svm_fifo_t *tx_fifo;
-
-  /** Type */
-  session_type_t session_type;
-
-  /** State */
-  volatile u8 session_state;
-
-  /** Session index */
-  u32 session_index;
-
-  /** Server index */
-  u32 app_wrk_index;
-
-  /** Port for connection. Overlaps thread_index/enqueue_epoch */
-  u16 port;
-
-  /** Partly overlaps enqueue_epoch */
-  u8 pad_epoch[7];
-
-  /** Segment index where fifos were allocated */
-  u32 svm_segment_index;
-
-  /** Transport listener index. Overlaps connection index */
-  u32 transport_listener_index;
-
-  union
-  {
-    u32 listener_index;
-    u32 app_index;
-  };
-
-  u32 al_index;
-
-  /** Has transport embedded when listener not purely local */
-  session_type_t listener_session_type;
-
-  /**
-   * Client data
-   */
-  u32 client_wrk_index;
-  u32 client_opaque;
-
-  u64 server_evt_q;
-  u64 client_evt_q;
-
-    CLIB_CACHE_LINE_ALIGN_MARK (pad);
-} local_session_t;
-
-always_inline u32
-local_session_id (local_session_t * ls)
-{
-  ASSERT (ls->session_index < (2 << 16));
-  u32 app_or_wrk_index;
-
-  if (ls->session_state == SESSION_STATE_LISTENING)
-    {
-      ASSERT (ls->app_index < (2 << 16));
-      app_or_wrk_index = ls->app_index;
-    }
-  else
-    {
-      ASSERT (ls->app_wrk_index < (2 << 16));
-      app_or_wrk_index = ls->app_wrk_index;
-    }
-
-  return ((u32) app_or_wrk_index << 16 | (u32) ls->session_index);
-}
-
-always_inline void
-local_session_parse_id (u32 ls_id, u32 * app_or_wrk, u32 * session_index)
-{
-  *app_or_wrk = ls_id >> 16;
-  *session_index = ls_id & 0xFF;
-}
-
-always_inline void
-local_session_parse_handle (session_handle_t handle, u32 * app_or_wrk_index,
-			    u32 * session_index)
-{
-  u32 bottom;
-  ASSERT (((handle >> 32) == SESSION_LOCAL_HANDLE_PREFIX));
-  bottom = (handle & 0xFFFFFFFF);
-  local_session_parse_id (bottom, app_or_wrk_index, session_index);
-}
-
-always_inline session_handle_t
-application_local_session_handle (local_session_t * ls)
-{
-  return ((u64) SESSION_LOCAL_HANDLE_PREFIX << 32)
-    | (u64) local_session_id (ls);
-}
-
 typedef enum
 {
   FIFO_EVENT_APP_RX,