session: add support for multiple app workers

Refactor session layer to support multiple workers per application.

Change-Id: Ie67354688d396449d14bbbb8c56050206e307cd8
Signed-off-by: Florin Coras <fcoras@cisco.com>

1 files changed, 70 insertions, 59 deletions

diff --git a/src/vnet/session/application_interface.c b/src/vnet/session/application_interface.c
index 524e7b53663..f8d43d40882 100644
--- a/src/vnet/session/application_interface.c
+++ b/src/vnet/session/application_interface.c
@@ -148,27 +148,29 @@ session_endpoint_update_for_app (session_endpoint_t * sep,
 }
 
 static int
-vnet_bind_i (u32 app_index, session_endpoint_t * sep, u64 * handle)
+vnet_bind_i (vnet_bind_args_t * a)
 {
   u64 lh, ll_handle = SESSION_INVALID_HANDLE;
+  u32 table_index, fib_proto;
+  app_worker_t *app_wrk;
   application_t *app;
-  u32 table_index;
   int rv;
 
-  app = application_get_if_valid (app_index);
+  app = application_get_if_valid (a->app_index);
   if (!app)
     {
       SESSION_DBG ("app not attached");
       return VNET_API_ERROR_APPLICATION_NOT_ATTACHED;
     }
+  app_wrk = application_get_worker (app, a->wrk_map_index);
 
-  session_endpoint_update_for_app (sep, app);
-  if (!session_endpoint_in_ns (sep))
+  session_endpoint_update_for_app (&a->sep, app);
+  if (!session_endpoint_in_ns (&a->sep))
     return VNET_API_ERROR_INVALID_VALUE_2;
 
-  table_index = application_session_table (app,
-					   session_endpoint_fib_proto (sep));
-  lh = session_lookup_endpoint_listener (table_index, sep, 1);
+  fib_proto = session_endpoint_fib_proto (&a->sep);
+  table_index = application_session_table (app, fib_proto);
+  lh = session_lookup_endpoint_listener (table_index, &a->sep, 1);
   if (lh != SESSION_INVALID_HANDLE)
     return VNET_API_ERROR_ADDRESS_IN_USE;
 
@@ -176,11 +178,13 @@ vnet_bind_i (u32 app_index, session_endpoint_t * sep, u64 * handle)
    * Add session endpoint to local session table. Only binds to "inaddr_any"
    * (i.e., zero address) are added to local scope table.
    */
-  if (application_has_local_scope (app) && session_endpoint_is_local (sep))
+  if (application_has_local_scope (app)
+      && session_endpoint_is_local (&a->sep))
     {
-      if ((rv = application_start_local_listen (app, sep, handle)))
+      if ((rv =
+	   application_start_local_listen (app_wrk, &a->sep, &a->handle)))
 	return rv;
-      ll_handle = *handle;
+      ll_handle = a->handle;
     }
 
   if (!application_has_global_scope (app))
@@ -191,9 +195,9 @@ vnet_bind_i (u32 app_index, session_endpoint_t * sep, u64 * handle)
    */
 
   /* Setup listen path down to transport */
-  rv = application_start_listen (app, sep, handle);
+  rv = app_worker_start_listen (app_wrk, &a->sep, &a->handle);
   if (rv && ll_handle != SESSION_INVALID_HANDLE)
-    session_lookup_del_session_endpoint (table_index, sep);
+    session_lookup_del_session_endpoint (table_index, &a->sep);
 
   /*
    * Store in local table listener the index of the transport layer
@@ -205,7 +209,7 @@ vnet_bind_i (u32 app_index, session_endpoint_t * sep, u64 * handle)
       local_session_t *ll;
       stream_session_t *tl;
       ll = application_get_local_listener_w_handle (ll_handle);
-      tl = listen_session_get_from_handle (*handle);
+      tl = listen_session_get_from_handle (a->handle);
       ll->transport_listener_index = tl->session_index;
     }
   return rv;
@@ -219,13 +223,13 @@ vnet_unbind_i (u32 app_index, session_handle_t handle)
 
   if (!(app = application_get_if_valid (app_index)))
     {
-      SESSION_DBG ("app (%d) not attached", app_index);
+      SESSION_DBG ("app (%d) not attached", wrk_map_index);
       return VNET_API_ERROR_APPLICATION_NOT_ATTACHED;
     }
 
   if (application_has_local_scope (app))
     {
-      if ((rv = application_stop_local_listen (app, handle)))
+      if ((rv = application_stop_local_listen (handle, app_index)))
 	return rv;
     }
 
@@ -233,25 +237,27 @@ vnet_unbind_i (u32 app_index, session_handle_t handle)
    * Clear the global scope table of the listener
    */
   if (application_has_global_scope (app))
-    return application_stop_listen (app, handle);
+    return app_worker_stop_listen (handle, app_index);
   return 0;
 }
 
-int
-application_connect (u32 client_index, u32 api_context,
-		     session_endpoint_t * sep)
+static int
+application_connect (vnet_connect_args_t * a)
 {
-  application_t *server, *client;
+  app_worker_t *server_wrk, *client_wrk;
   u32 table_index, server_index, li;
   stream_session_t *listener;
+  application_t *client;
   local_session_t *ll;
+  u8 fib_proto;
   u64 lh;
 
-  if (session_endpoint_is_zero (sep))
+  if (session_endpoint_is_zero (&a->sep))
     return VNET_API_ERROR_INVALID_VALUE;
 
-  client = application_get (client_index);
-  session_endpoint_update_for_app (sep, client);
+  client = application_get (a->app_index);
+  session_endpoint_update_for_app (&a->sep, client);
+  client_wrk = application_get_worker (client, a->wrk_map_index);
 
   /*
    * First check the local scope for locally attached destinations.
@@ -261,7 +267,7 @@ application_connect (u32 client_index, u32 api_context,
   if (application_has_local_scope (client))
     {
       table_index = application_local_session_table (client);
-      lh = session_lookup_local_endpoint (table_index, sep);
+      lh = session_lookup_local_endpoint (table_index, &a->sep);
       if (lh == SESSION_DROP_HANDLE)
 	return VNET_API_ERROR_APP_CONNECT_FILTERED;
 
@@ -275,12 +281,13 @@ application_connect (u32 client_index, u32 api_context,
        * can happen if client is a generic proxy. Route connect through
        * global table instead.
        */
-      if (server_index != client_index)
+      if (server_index != a->app_index)
 	{
-	  server = application_get (server_index);
-	  ll = application_get_local_listen_session (server, li);
-	  return application_local_session_connect (table_index, client,
-						    server, ll, api_context);
+	  server_wrk = app_worker_get (server_index);
+	  ll = application_get_local_listen_session (server_wrk, li);
+	  return application_local_session_connect (client_wrk,
+						    server_wrk, ll,
+						    a->api_context);
 	}
     }
 
@@ -290,28 +297,30 @@ application_connect (u32 client_index, u32 api_context,
    */
 
 global_scope:
-  if (session_endpoint_is_local (sep))
+  if (session_endpoint_is_local (&a->sep))
     return VNET_API_ERROR_SESSION_CONNECT;
 
   if (!application_has_global_scope (client))
     return VNET_API_ERROR_APP_CONNECT_SCOPE;
 
-  table_index = application_session_table (client,
-					   session_endpoint_fib_proto (sep));
-  listener = session_lookup_listener (table_index, sep);
+  fib_proto = session_endpoint_fib_proto (&a->sep);
+  table_index = application_session_table (client, fib_proto);
+  listener = session_lookup_listener (table_index, &a->sep);
   if (listener)
     {
-      server = application_get (listener->app_index);
-      if (server)
-	return application_local_session_connect (table_index, client, server,
-						  (local_session_t *)
-						  listener, api_context);
+      server_wrk = app_worker_get (listener->app_wrk_index);
+      if (server_wrk)
+	{
+	  ll = (local_session_t *) listener;
+	  return application_local_session_connect (client_wrk, server_wrk,
+						    ll, a->api_context);
+	}
     }
 
   /*
    * Not connecting to a local server, propagate to transport
    */
-  if (application_open_session (client, sep, api_context))
+  if (app_worker_open_session (client_wrk, &a->sep, a->api_context))
     return VNET_API_ERROR_SESSION_CONNECT;
   return 0;
 }
@@ -415,7 +424,7 @@ parse_uri (char *uri, session_endpoint_extended_t * sep)
 }
 
 static int
-session_validate_namespace (u8 * namespace_id, u64 secret, u32 * app_ns_index)
+app_validate_namespace (u8 * namespace_id, u64 secret, u32 * app_ns_index)
 {
   app_namespace_t *app_ns;
   if (vec_len (namespace_id) == 0)
@@ -448,6 +457,7 @@ vnet_application_attach (vnet_app_attach_args_t * a)
 {
   svm_fifo_segment_private_t *fs;
   application_t *app = 0;
+  app_worker_t *app_wrk;
   segment_manager_t *sm;
   u32 app_ns_index = 0;
   u64 secret;
@@ -466,17 +476,19 @@ vnet_application_attach (vnet_app_attach_args_t * a)
 				   "app already attached");
 
   secret = a->options[APP_OPTIONS_NAMESPACE_SECRET];
-  if ((rv = session_validate_namespace (a->namespace_id, secret,
-					&app_ns_index)))
+  if ((rv = app_validate_namespace (a->namespace_id, secret, &app_ns_index)))
     return clib_error_return_code (0, rv, 0, "namespace validation: %d", rv);
   a->options[APP_OPTIONS_NAMESPACE] = app_ns_index;
-  app = application_new ();
-  if ((rv = application_init (app, a->api_client_index, a->name, a->options,
-			      a->session_cb_vft)))
+
+  if ((rv = application_alloc_and_init ((app_init_args_t *) a)))
     return clib_error_return_code (0, rv, 0, "app init: %d", rv);
 
-  a->app_evt_q = app->event_queue;
-  sm = segment_manager_get (app->first_segment_manager);
+  app = application_get (a->app_index);
+  if ((rv = app_worker_alloc_and_init (app, &app_wrk)))
+    return clib_error_return_code (0, rv, 0, "app default wrk init: %d", rv);
+
+  a->app_evt_q = app_wrk->event_queue;
+  sm = segment_manager_get (app_wrk->first_segment_manager);
   fs = segment_manager_get_segment_w_lock (sm, 0);
 
   if (application_is_proxy (app))
@@ -484,7 +496,6 @@ vnet_application_attach (vnet_app_attach_args_t * a)
 
   ASSERT (vec_len (fs->ssvm.name) <= 128);
   a->segment = &fs->ssvm;
-  a->app_index = app->index;
 
   segment_manager_segment_reader_unlock (sm);
 
@@ -506,7 +517,7 @@ vnet_application_detach (vnet_app_detach_args_t * a)
       return VNET_API_ERROR_APPLICATION_NOT_ATTACHED;
     }
 
-  application_del (app);
+  application_free (app);
   return 0;
 }
 
@@ -519,8 +530,8 @@ vnet_bind_uri (vnet_bind_args_t * a)
   rv = parse_uri (a->uri, &sep);
   if (rv)
     return rv;
-
-  return vnet_bind_i (a->app_index, (session_endpoint_t *) & sep, &a->handle);
+  clib_memcpy (&a->sep, &sep, sizeof (sep));
+  return vnet_bind_i (a);
 }
 
 int
@@ -557,8 +568,8 @@ vnet_connect_uri (vnet_connect_args_t * a)
   if (rv)
     return clib_error_return_code (0, rv, 0, "app init: %d", rv);
 
-  if ((rv = application_connect (a->app_index, a->api_context,
-				 (session_endpoint_t *) & sep)))
+  clib_memcpy (&a->sep, &sep, sizeof (sep));
+  if ((rv = application_connect (a)))
     return clib_error_return_code (0, rv, 0, "connect failed");
   return 0;
 }
@@ -581,7 +592,8 @@ vnet_disconnect_session (vnet_disconnect_args_t * a)
       if (!(ls = application_get_local_session_from_handle (a->handle)))
 	return 0;
 
-      if (ls->app_index != a->app_index && ls->client_index != a->app_index)
+      if (ls->app_wrk_index != a->app_index
+	  && ls->client_wrk_index != a->app_index)
 	{
 	  clib_warning ("app %u is neither client nor server for session %u",
 			a->app_index, a->app_index);
@@ -593,7 +605,7 @@ vnet_disconnect_session (vnet_disconnect_args_t * a)
     {
       stream_session_t *s;
       s = session_get_from_handle_if_valid (a->handle);
-      if (!s || s->app_index != a->app_index)
+      if (!s || s->app_wrk_index != a->app_index)
 	return VNET_API_ERROR_INVALID_VALUE;
 
       /* We're peeking into another's thread pool. Make sure */
@@ -608,7 +620,7 @@ clib_error_t *
 vnet_bind (vnet_bind_args_t * a)
 {
   int rv;
-  if ((rv = vnet_bind_i (a->app_index, &a->sep, &a->handle)))
+  if ((rv = vnet_bind_i (a)))
     return clib_error_return_code (0, rv, 0, "bind failed: %d", rv);
   return 0;
 }
@@ -625,10 +637,9 @@ vnet_unbind (vnet_unbind_args_t * a)
 clib_error_t *
 vnet_connect (vnet_connect_args_t * a)
 {
-  session_endpoint_t *sep = (session_endpoint_t *) & a->sep;
   int rv;
 
-  if ((rv = application_connect (a->app_index, a->api_context, sep)))
+  if ((rv = application_connect (a)))
     return clib_error_return_code (0, rv, 0, "connect failed: %d", rv);
   return 0;
 }