session: add unix socket api for app attachment

This is an af_unix socket alternative to the binary api. To enable it,
add use-app-socket-api under session stanza in startup.conf. When the
socket api is enabled, attachments through the binary api are disabled.

The socket api only works with memfd fifo segments, i.e., shm segments
are not supported.

Type: feature

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I55ffcee201d004846daeeec85c700c7e7a578d43

6 files changed, 743 insertions, 18 deletions

diff --git a/src/vnet/session/application.c b/src/vnet/session/application.c
index c20277720ad..65d2f08ddca 100644
--- a/src/vnet/session/application.c
+++ b/src/vnet/session/application.c
@@ -493,7 +493,11 @@ application_alloc_and_init (app_init_args_t * a)
   /*
    * Make sure we support the requested configuration
    */
-  if (!(options[APP_OPTIONS_FLAGS] & APP_OPTIONS_FLAGS_IS_BUILTIN))
+  if (options[APP_OPTIONS_FLAGS] & APP_OPTIONS_FLAGS_IS_BUILTIN)
+    {
+      seg_type = SSVM_SEGMENT_PRIVATE;
+    }
+  else if (!a->use_sock_api)
     {
       reg = vl_api_client_index_to_registration (a->api_client_index);
       if (!reg)
@@ -501,10 +505,6 @@ application_alloc_and_init (app_init_args_t * a)
       if (vl_api_registration_file_index (reg) == VL_API_INVALID_FI)
 	seg_type = SSVM_SEGMENT_SHM;
     }
-  else
-    {
-      seg_type = SSVM_SEGMENT_PRIVATE;
-    }
 
   if ((options[APP_OPTIONS_FLAGS] & APP_OPTIONS_FLAGS_EVT_MQ_USE_EVENTFD)
       && seg_type != SSVM_SEGMENT_MEMFD)
@@ -845,16 +845,21 @@ vnet_application_attach (vnet_app_attach_args_t * a)
   if (app)
     return VNET_API_ERROR_APP_ALREADY_ATTACHED;
 
-  if (a->api_client_index != APP_INVALID_INDEX)
+  /* Socket api sets the name and validates namespace prior to attach */
+  if (!a->use_sock_api)
     {
-      app_name = app_name_from_api_index (a->api_client_index);
-      a->name = app_name;
-    }
+      if (a->api_client_index != APP_INVALID_INDEX)
+	{
+	  app_name = app_name_from_api_index (a->api_client_index);
+	  a->name = app_name;
+	}
 
-  secret = a->options[APP_OPTIONS_NAMESPACE_SECRET];
-  if ((rv = app_validate_namespace (a->namespace_id, secret, &app_ns_index)))
-    return rv;
-  a->options[APP_OPTIONS_NAMESPACE] = app_ns_index;
+      secret = a->options[APP_OPTIONS_NAMESPACE_SECRET];
+      if ((rv =
+	   app_validate_namespace (a->namespace_id, secret, &app_ns_index)))
+	return rv;
+      a->options[APP_OPTIONS_NAMESPACE] = app_ns_index;
+    }
 
   if ((rv = application_alloc_and_init ((app_init_args_t *) a)))
     return rv;
diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h
index d7869312d49..a36008a1e71 100644
--- a/src/vnet/session/application_interface.h
+++ b/src/vnet/session/application_interface.h
@@ -81,6 +81,7 @@ typedef struct session_cb_vft_
   _(u8 *, namespace_id)				\
   _(session_cb_vft_t *, session_cb_vft)		\
   _(u32, app_index)				\
+  _(u8, use_sock_api)				\
 
 typedef struct _vnet_app_attach_args_t
 {
@@ -773,6 +774,74 @@ format_session_error (u8 * s, va_list * args)
     s = format (s, "invalid session err %u", -error);
   return s;
 }
+
+/*
+ * Socket API messages
+ */
+
+typedef enum app_sapi_msg_type
+{
+  APP_SAPI_MSG_TYPE_NONE,
+  APP_SAPI_MSG_TYPE_ATTACH,
+  APP_SAPI_MSG_TYPE_ATTACH_REPLY,
+  APP_SAPI_MSG_TYPE_ADD_DEL_WORKER,
+  APP_SAPI_MSG_TYPE_ADD_DEL_WORKER_REPLY,
+  APP_SAPI_MSG_TYPE_SEND_FDS,
+} __clib_packed app_sapi_msg_type_e;
+
+typedef struct app_sapi_attach_msg_
+{
+  u8 name[64];
+  u64 options[18];
+} __clib_packed app_sapi_attach_msg_t;
+
+STATIC_ASSERT (sizeof (u64) * APP_OPTIONS_N_OPTIONS <=
+	       sizeof (((app_sapi_attach_msg_t *) 0)->options),
+	       "Out of options, fix message definition");
+
+typedef struct app_sapi_attach_reply_msg_
+{
+  i32 retval;
+  u32 app_index;
+  u64 app_mq;
+  u64 vpp_ctrl_mq;
+  u64 segment_handle;
+  u32 api_client_handle;
+  u8 vpp_ctrl_mq_thread;
+  u8 n_fds;
+  u8 fd_flags;
+} __clib_packed app_sapi_attach_reply_msg_t;
+
+typedef struct app_sapi_worker_add_del_msg_
+{
+  u32 app_index;
+  u32 wrk_index;
+  u8 is_add;
+} __clib_packed app_sapi_worker_add_del_msg_t;
+
+typedef struct app_sapi_worker_add_del_reply_msg_
+{
+  i32 retval;
+  u32 wrk_index;
+  u64 app_event_queue_address;
+  u64 segment_handle;
+  u8 n_fds;
+  u8 fd_flags;
+  u8 is_add;
+} __clib_packed app_sapi_worker_add_del_reply_msg_t;
+
+typedef struct app_sapi_msg_
+{
+  app_sapi_msg_type_e type;
+  union
+  {
+    app_sapi_attach_msg_t attach;
+    app_sapi_attach_reply_msg_t attach_reply;
+    app_sapi_worker_add_del_msg_t worker_add_del;
+    app_sapi_worker_add_del_reply_msg_t worker_add_del_reply;
+  };
+} __clib_packed app_sapi_msg_t;
+
 #endif /* __included_uri_h__ */
 
 /*
diff --git a/src/vnet/session/application_namespace.c b/src/vnet/session/application_namespace.c
index 294192ceea1..1c43c79ce47 100644
--- a/src/vnet/session/application_namespace.c
+++ b/src/vnet/session/application_namespace.c
@@ -14,9 +14,12 @@
  */
 
 #include <vnet/session/application_namespace.h>
+#include <vnet/session/application.h>
 #include <vnet/session/session_table.h>
 #include <vnet/session/session.h>
 #include <vnet/fib/fib_table.h>
+#include <vppinfra/file.h>
+#include <vlib/unix/unix.h>
 
 /**
  * Hash table of application namespaces by app ns ids
@@ -28,6 +31,8 @@ uword *app_namespace_lookup_table;
  */
 static app_namespace_t *app_namespace_pool;
 
+static u8 app_sapi_enabled;
+
 app_namespace_t *
 app_namespace_get (u32 index)
 {
@@ -105,6 +110,10 @@ vnet_app_namespace_add_del (vnet_app_namespace_add_del_args_t * a)
       app_ns->ip6_fib_index =
 	fib_table_find (FIB_PROTOCOL_IP6, a->ip6_fib_id);
       session_lookup_set_tables_appns (app_ns);
+
+      /* Add socket for namespace */
+      if (app_sapi_enabled)
+	appns_sapi_add_ns_socket (app_ns);
     }
   else
     {
@@ -152,6 +161,18 @@ app_namespace_get_local_table (app_namespace_t * app_ns)
 }
 
 void
+appns_sapi_enable (void)
+{
+  app_sapi_enabled = 1;
+}
+
+u8
+appns_sapi_enabled (void)
+{
+  return app_sapi_enabled;
+}
+
+void
 app_namespaces_init (void)
 {
   u8 *ns_id = format (0, "default");
@@ -253,14 +274,52 @@ format_app_namespace (u8 * s, va_list * args)
   return s;
 }
 
+static void
+app_namespace_show_api (vlib_main_t * vm, app_namespace_t * app_ns)
+{
+  app_ns_api_handle_t *handle;
+  app_worker_t *app_wrk;
+  clib_socket_t *cs;
+  clib_file_t *cf;
+
+  if (!app_sapi_enabled)
+    {
+      vlib_cli_output (vm, "app socket api not enabled!");
+      return;
+    }
+
+  vlib_cli_output (vm, "socket: %v\n", app_ns->sock_name);
+
+  if (!pool_elts (app_ns->app_sockets))
+    return;
+
+  vlib_cli_output (vm, "%12s%12s%5s", "app index", "wrk index", "fd");
+
+
+  /* *INDENT-OFF* */
+  pool_foreach (cs, app_ns->app_sockets, ({
+    handle = (app_ns_api_handle_t *) &cs->private_data;
+    cf = clib_file_get (&file_main, handle->aah_file_index);
+    if (handle->aah_app_wrk_index == APP_INVALID_INDEX)
+      {
+	vlib_cli_output (vm, "%12d%12d%5u", -1, -1, cf->file_descriptor);
+	continue;
+      }
+    app_wrk = app_worker_get (handle->aah_app_wrk_index);
+    vlib_cli_output (vm, "%12d%12d%5u", app_wrk->app_index,
+                     app_wrk->wrk_map_index, cf->file_descriptor);
+  }));
+  /* *INDENT-ON* */
+}
+
 static clib_error_t *
 show_app_ns_fn (vlib_main_t * vm, unformat_input_t * main_input,
 		vlib_cli_command_t * cmd)
 {
   unformat_input_t _line_input, *line_input = &_line_input;
+  u8 *ns_id, do_table = 0, had_input = 1, do_api = 0;
   app_namespace_t *app_ns;
   session_table_t *st;
-  u8 *ns_id, do_table = 0, had_input = 1;
 
   session_cli_return_if_not_enabled ();
 
@@ -274,6 +333,8 @@ show_app_ns_fn (vlib_main_t * vm, unformat_input_t * main_input,
     {
       if (unformat (line_input, "table %_%v%_", &ns_id))
 	do_table = 1;
+      else if (unformat (line_input, "api-clients"))
+	do_api = 1;
       else
 	{
 	  vlib_cli_output (vm, "unknown input [%U]", format_unformat_error,
@@ -282,6 +343,17 @@ show_app_ns_fn (vlib_main_t * vm, unformat_input_t * main_input,
 	}
     }
 
+  if (do_api)
+    {
+      if (!do_table)
+	{
+	  vlib_cli_output (vm, "must specify a table for api");
+	  goto done;
+	}
+      app_ns = app_namespace_get_from_id (ns_id);
+      app_namespace_show_api (vm, app_ns);
+      goto done;
+    }
   if (do_table)
     {
       app_ns = app_namespace_get_from_id (ns_id);
@@ -321,7 +393,7 @@ done:
 VLIB_CLI_COMMAND (show_app_ns_command, static) =
 {
   .path = "show app ns",
-  .short_help = "show app ns",
+  .short_help = "show app ns [table <id> [api-clients]]",
   .function = show_app_ns_fn,
 };
 /* *INDENT-ON* */
diff --git a/src/vnet/session/application_namespace.h b/src/vnet/session/application_namespace.h
index 3a24fe1e6fa..14bea610fc3 100644
--- a/src/vnet/session/application_namespace.h
+++ b/src/vnet/session/application_namespace.h
@@ -13,6 +13,7 @@
  * limitations under the License.
  */
 
+#include <vppinfra/socket.h>
 #include <vnet/vnet.h>
 #include <vnet/session/session_table.h>
 
@@ -48,6 +49,16 @@ typedef struct _app_namespace
    * Application namespace id
    */
   u8 *ns_id;
+
+  /**
+   * Name of socket applications can use to attach to session layer
+   */
+  u8 *sock_name;
+
+  /**
+   * Pool of active application sockets
+   */
+  clib_socket_t *app_sockets;
 } app_namespace_t;
 
 typedef struct _vnet_app_namespace_add_del_args
@@ -80,6 +91,71 @@ app_namespace_get_default (void)
   return app_namespace_get (0);
 }
 
+typedef struct app_ns_api_handle_
+{
+  union
+  {
+    struct
+    {
+      /** app_ns index for files and app_index for sockets */
+      u32 l_index;
+      /** socket index for files and clib file index for sockets */
+      u32 u_index;
+    };
+    uword as_uword;
+  };
+#define aah_app_ns_index l_index
+#define aah_app_wrk_index l_index
+#define aah_sock_index u_index
+#define aah_file_index u_index
+} __attribute__ ((aligned (sizeof (uword)))) app_ns_api_handle_t;
+
+STATIC_ASSERT (sizeof (app_ns_api_handle_t) == sizeof (uword), "not uword");
+
+static inline clib_socket_t *
+appns_sapi_alloc_socket (app_namespace_t * app_ns)
+{
+  clib_socket_t *cs;
+  pool_get_zero (app_ns->app_sockets, cs);
+  return cs;
+}
+
+static inline clib_socket_t *
+appns_sapi_get_socket (app_namespace_t * app_ns, u32 sock_index)
+{
+  if (pool_is_free_index (app_ns->app_sockets, sock_index))
+    return 0;
+  return pool_elt_at_index (app_ns->app_sockets, sock_index);
+}
+
+static inline void
+appns_sapi_free_socket (app_namespace_t * app_ns, clib_socket_t * cs)
+{
+  pool_put (app_ns->app_sockets, cs);
+}
+
+static inline u32
+appns_sapi_socket_index (app_namespace_t * app_ns, clib_socket_t * cs)
+{
+  return (cs - app_ns->app_sockets);
+}
+
+static inline u32
+appns_sapi_socket_handle (app_namespace_t * app_ns, clib_socket_t * cs)
+{
+  return app_namespace_index (app_ns) << 16 | (cs - app_ns->app_sockets);
+}
+
+static inline u32
+appns_sapi_handle_sock_index (u32 sapi_sock_handle)
+{
+  return sapi_sock_handle & 0xffff;
+}
+
+int appns_sapi_add_ns_socket (app_namespace_t * app_ns);
+u8 appns_sapi_enabled (void);
+void appns_sapi_enable (void);
+
 #endif /* SRC_VNET_SESSION_APPLICATION_NAMESPACE_H_ */
 
 /*
diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c
index 70b9dc49cf3..d27d1bbab23 100644
--- a/src/vnet/session/session.c
+++ b/src/vnet/session/session.c
@@ -1904,6 +1904,8 @@ session_config_fn (vlib_main_t * vm, unformat_input_t * input)
 	;
       else if (unformat (input, "enable"))
 	smm->session_enable_asap = 1;
+      else if (unformat (input, "use-app-socket-api"))
+	appns_sapi_enable ();
       else
 	return clib_error_return (0, "unknown input `%U'",
 				  format_unformat_error, input);
diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c
index 593f2e18ba0..15c5812e790 100644
--- a/src/vnet/session/session_api.c
+++ b/src/vnet/session/session_api.c
@@ -609,7 +609,7 @@ vl_api_app_attach_t_handler (vl_api_app_attach_t * mp)
   if (!reg)
     return;
 
-  if (session_main_is_enabled () == 0)
+  if (!session_main_is_enabled () || appns_sapi_enabled ())
     {
       rv = VNET_API_ERROR_FEATURE_DISABLED;
       goto done;
@@ -692,7 +692,7 @@ vl_api_app_worker_add_del_t_handler (vl_api_app_worker_add_del_t * mp)
   application_t *app;
   u8 fd_flags = 0;
 
-  if (session_main_is_enabled () == 0)
+  if (!session_main_is_enabled () || appns_sapi_enabled ())
     {
       rv = VNET_API_ERROR_FEATURE_DISABLED;
       goto done;
@@ -770,7 +770,7 @@ vl_api_application_detach_t_handler (vl_api_application_detach_t * mp)
   vnet_app_detach_args_t _a, *a = &_a;
   application_t *app;
 
-  if (session_main_is_enabled () == 0)
+  if (!session_main_is_enabled () || appns_sapi_enabled ())
     {
       rv = VNET_API_ERROR_FEATURE_DISABLED;
       goto done;
@@ -1209,6 +1209,507 @@ session_api_hookup (vlib_main_t * vm)
 VLIB_API_INIT_FUNCTION (session_api_hookup);
 
 /*
+ * Socket api functions
+ */
+
+static void
+sapi_send_fds (app_worker_t * app_wrk, int *fds, int n_fds)
+{
+  app_sapi_msg_t smsg = { 0 };
+  app_namespace_t *app_ns;
+  application_t *app;
+  clib_socket_t *cs;
+  u32 cs_index;
+
+  app = application_get (app_wrk->app_index);
+  app_ns = app_namespace_get (app->ns_index);
+  cs_index = appns_sapi_handle_sock_index (app_wrk->api_client_index);
+  cs = appns_sapi_get_socket (app_ns, cs_index);
+
+  /* There's no payload for the message only the type */
+  smsg.type = APP_SAPI_MSG_TYPE_SEND_FDS;
+  clib_socket_sendmsg (cs, &smsg, sizeof (smsg), fds, n_fds);
+}
+
+static int
+mq_send_add_segment_sapi_cb (u32 app_wrk_index, u64 segment_handle)
+{
+  int fds[SESSION_N_FD_TYPE], n_fds = 0;
+  svm_msg_q_msg_t _msg, *msg = &_msg;
+  session_app_add_segment_msg_t *mp;
+  app_worker_t *app_wrk;
+  session_event_t *evt;
+  svm_msg_q_t *app_mq;
+  fifo_segment_t *fs;
+  ssvm_private_t *sp;
+  u8 fd_flags = 0;
+
+  app_wrk = app_worker_get (app_wrk_index);
+
+  fs = segment_manager_get_segment_w_handle (segment_handle);
+  sp = &fs->ssvm;
+  ASSERT (ssvm_type (sp) == SSVM_SEGMENT_MEMFD);
+
+  fd_flags |= SESSION_FD_F_MEMFD_SEGMENT;
+  fds[n_fds] = sp->fd;
+  n_fds += 1;
+
+  app_mq = app_wrk->event_queue;
+  if (mq_try_lock_and_alloc_msg (app_mq, msg))
+    return -1;
+
+  /*
+   * Send the fd over api socket
+   */
+  sapi_send_fds (app_wrk, fds, n_fds);
+
+  /*
+   * Send the actual message over mq
+   */
+  evt = svm_msg_q_msg_data (app_mq, msg);
+  clib_memset (evt, 0, sizeof (*evt));
+  evt->event_type = SESSION_CTRL_EVT_APP_ADD_SEGMENT;
+  mp = (session_app_add_segment_msg_t *) evt->data;
+  clib_memset (mp, 0, sizeof (*mp));
+  mp->segment_size = sp->ssvm_size;
+  mp->fd_flags = fd_flags;
+  mp->segment_handle = segment_handle;
+  strncpy ((char *) mp->segment_name, (char *) sp->name,
+	   sizeof (mp->segment_name) - 1);
+
+  svm_msg_q_add_and_unlock (app_mq, msg);
+
+  return 0;
+}
+
+static int
+mq_send_del_segment_sapi_cb (u32 app_wrk_index, u64 segment_handle)
+{
+  svm_msg_q_msg_t _msg, *msg = &_msg;
+  session_app_del_segment_msg_t *mp;
+  app_worker_t *app_wrk;
+  session_event_t *evt;
+  svm_msg_q_t *app_mq;
+
+  app_wrk = app_worker_get (app_wrk_index);
+
+  app_mq = app_wrk->event_queue;
+  if (mq_try_lock_and_alloc_msg (app_mq, msg))
+    return -1;
+
+  evt = svm_msg_q_msg_data (app_mq, msg);
+  clib_memset (evt, 0, sizeof (*evt));
+  evt->event_type = SESSION_CTRL_EVT_APP_DEL_SEGMENT;
+  mp = (session_app_del_segment_msg_t *) evt->data;
+  clib_memset (mp, 0, sizeof (*mp));
+  mp->segment_handle = segment_handle;
+  svm_msg_q_add_and_unlock (app_mq, msg);
+
+  return 0;
+}
+
+static session_cb_vft_t session_mq_sapi_cb_vft = {
+  .session_accept_callback = mq_send_session_accepted_cb,
+  .session_disconnect_callback = mq_send_session_disconnected_cb,
+  .session_connected_callback = mq_send_session_connected_cb,
+  .session_reset_callback = mq_send_session_reset_cb,
+  .session_migrate_callback = mq_send_session_migrate_cb,
+  .session_cleanup_callback = mq_send_session_cleanup_cb,
+  .add_segment_callback = mq_send_add_segment_sapi_cb,
+  .del_segment_callback = mq_send_del_segment_sapi_cb,
+};
+
+static void
+session_api_attach_handler (app_namespace_t * app_ns, clib_socket_t * cs,
+			    app_sapi_attach_msg_t * mp)
+{
+  int rv = 0, fds[SESSION_N_FD_TYPE], n_fds = 0;
+  vnet_app_attach_args_t _a, *a = &_a;
+  app_sapi_attach_reply_msg_t *rmp;
+  ssvm_private_t *evt_q_segment;
+  u8 fd_flags = 0, ctrl_thread;
+  app_ns_api_handle_t *handle;
+  app_sapi_msg_t msg = { 0 };
+  app_worker_t *app_wrk;
+  svm_msg_q_t *ctrl_mq;
+  application_t *app;
+
+  /* Make sure name is null terminated */
+  mp->name[63] = 0;
+
+  clib_memset (a, 0, sizeof (*a));
+  a->api_client_index = appns_sapi_socket_handle (app_ns, cs);
+  a->name = format (0, "%s", (char *) mp->name);
+  a->options = mp->options;
+  a->session_cb_vft = &session_mq_sapi_cb_vft;
+  a->use_sock_api = 1;
+  a->options[APP_OPTIONS_NAMESPACE] = app_namespace_index (app_ns);
+
+  if ((rv = vnet_application_attach (a)))
+    {
+      clib_warning ("attach returned: %d", rv);
+      goto done;
+    }
+
+  /* Send event queues segment */
+  if ((evt_q_segment = session_main_get_evt_q_segment ()))
+    {
+      fd_flags |= SESSION_FD_F_VPP_MQ_SEGMENT;
+      fds[n_fds] = evt_q_segment->fd;
+      n_fds += 1;
+    }
+  /* Send fifo segment fd if needed */
+  if (ssvm_type (a->segment) == SSVM_SEGMENT_MEMFD)
+    {
+      fd_flags |= SESSION_FD_F_MEMFD_SEGMENT;
+      fds[n_fds] = a->segment->fd;
+      n_fds += 1;
+    }
+  if (a->options[APP_OPTIONS_FLAGS] & APP_OPTIONS_FLAGS_EVT_MQ_USE_EVENTFD)
+    {
+      fd_flags |= SESSION_FD_F_MQ_EVENTFD;
+      fds[n_fds] = svm_msg_q_get_producer_eventfd (a->app_evt_q);
+      n_fds += 1;
+    }
+
+done:
+
+  msg.type = APP_SAPI_MSG_TYPE_ATTACH_REPLY;
+  rmp = &msg.attach_reply;
+  rmp->retval = rv;
+  if (!rv)
+    {
+      ctrl_thread = vlib_num_workers ()? 1 : 0;
+      ctrl_mq = session_main_get_vpp_event_queue (ctrl_thread);
+      rmp->app_index = a->app_index;
+      rmp->app_mq = pointer_to_uword (a->app_evt_q);
+      rmp->vpp_ctrl_mq = pointer_to_uword (ctrl_mq);
+      rmp->vpp_ctrl_mq_thread = ctrl_thread;
+      rmp->n_fds = n_fds;
+      rmp->fd_flags = fd_flags;
+      /* No segment name and size since we only support memfds
+       * in this configuration */
+      rmp->segment_handle = a->segment_handle;
+      rmp->api_client_handle = a->api_client_index;
+
+      /* Update app index for socket */
+      handle = (app_ns_api_handle_t *) & cs->private_data;
+      app = application_get (a->app_index);
+      app_wrk = application_get_worker (app, 0);
+      handle->aah_app_wrk_index = app_wrk->wrk_index;
+    }
+
+  clib_socket_sendmsg (cs, &msg, sizeof (msg), fds, n_fds);
+  vec_free (a->name);
+}
+
+static void
+sapi_socket_close_w_handle (u32 api_handle)
+{
+  app_namespace_t *app_ns = app_namespace_get (api_handle >> 16);
+  u16 sock_index = api_handle & 0xffff;
+  app_ns_api_handle_t *handle;
+  clib_socket_t *cs;
+  clib_file_t *cf;
+
+  cs = appns_sapi_get_socket (app_ns, sock_index);
+  if (!cs)
+    return;
+
+  handle = (app_ns_api_handle_t *) & cs->private_data;
+  cf = clib_file_get (&file_main, handle->aah_file_index);
+  clib_file_del (&file_main, cf);
+
+  clib_socket_close (cs);
+  appns_sapi_free_socket (app_ns, cs);
+}
+
+static void
+sapi_add_del_worker_handler (app_namespace_t * app_ns,
+			     clib_socket_t * cs,
+			     app_sapi_worker_add_del_msg_t * mp)
+{
+  int rv = 0, fds[SESSION_N_FD_TYPE], n_fds = 0;
+  app_sapi_worker_add_del_reply_msg_t *rmp;
+  app_ns_api_handle_t *handle;
+  app_sapi_msg_t msg = { 0 };
+  app_worker_t *app_wrk;
+  application_t *app;
+  u32 sapi_handle;
+  u8 fd_flags = 0;
+
+  app = application_get_if_valid (mp->app_index);
+  if (!app)
+    {
+      rv = VNET_API_ERROR_INVALID_VALUE;
+      goto done;
+    }
+
+  sapi_handle = appns_sapi_socket_handle (app_ns, cs);
+
+  vnet_app_worker_add_del_args_t args = {
+    .app_index = app->app_index,
+    .wrk_map_index = mp->wrk_index,
+    .api_client_index = sapi_handle,
+    .is_add = mp->is_add
+  };
+  rv = vnet_app_worker_add_del (&args);
+  if (rv)
+    {
+      clib_warning ("app worker add/del returned: %d", rv);
+      goto done;
+    }
+
+  if (!mp->is_add)
+    {
+      sapi_socket_close_w_handle (sapi_handle);
+      goto done;
+    }
+
+  /* Send fifo segment fd if needed */
+  if (ssvm_type (args.segment) == SSVM_SEGMENT_MEMFD)
+    {
+      fd_flags |= SESSION_FD_F_MEMFD_SEGMENT;
+      fds[n_fds] = args.segment->fd;
+      n_fds += 1;
+    }
+  if (application_segment_manager_properties (app)->use_mq_eventfd)
+    {
+      fd_flags |= SESSION_FD_F_MQ_EVENTFD;
+      fds[n_fds] = svm_msg_q_get_producer_eventfd (args.evt_q);
+      n_fds += 1;
+    }
+
+done:
+
+  msg.type = APP_SAPI_MSG_TYPE_ADD_DEL_WORKER_REPLY;
+  rmp = &msg.worker_add_del_reply;
+  rmp->retval = rv;
+  rmp->is_add = mp->is_add;
+  rmp->wrk_index = args.wrk_map_index;
+  rmp->segment_handle = args.segment_handle;
+  if (!rv && mp->is_add)
+    {
+      /* No segment name and size. This supports only memfds */
+      rmp->app_event_queue_address = pointer_to_uword (args.evt_q);
+      rmp->n_fds = n_fds;
+      rmp->fd_flags = fd_flags;
+
+      /* Update app index for socket */
+      handle = (app_ns_api_handle_t *) & cs->private_data;
+      app_wrk = application_get_worker (app, args.wrk_map_index);
+      handle->aah_app_wrk_index = app_wrk->wrk_index;
+    }
+
+  clib_socket_sendmsg (cs, &msg, sizeof (msg), fds, n_fds);
+}
+
+static void
+sapi_socket_detach (app_namespace_t * app_ns, clib_socket_t * cs)
+{
+  vnet_app_detach_args_t _a = { 0 }, *a = &_a;
+  app_ns_api_handle_t *handle;
+  app_worker_t *app_wrk;
+  u32 api_client_handle;
+
+  api_client_handle = appns_sapi_socket_handle (app_ns, cs);
+  sapi_socket_close_w_handle (api_client_handle);
+
+  /* Cleanup everything because app closed socket or crashed */
+  handle = (app_ns_api_handle_t *) & cs->private_data;
+  app_wrk = app_worker_get (handle->aah_app_wrk_index);
+  a->app_index = app_wrk->app_index;
+  a->api_client_index = api_client_handle;
+  vnet_application_detach (a);
+}
+
+static clib_error_t *
+sapi_sock_read_ready (clib_file_t * cf)
+{
+  app_ns_api_handle_t *handle = (app_ns_api_handle_t *) & cf->private_data;
+  app_sapi_msg_t msg = { 0 };
+  app_namespace_t *app_ns;
+  clib_error_t *err = 0;
+  clib_socket_t *cs;
+
+  app_ns = app_namespace_get (handle->aah_app_ns_index);
+  cs = appns_sapi_get_socket (app_ns, handle->aah_sock_index);
+  if (!cs)
+    goto error;
+
+  err = clib_socket_recvmsg (cs, &msg, sizeof (msg), 0, 0);
+  if (err)
+    {
+      clib_error_free (err);
+      sapi_socket_detach (app_ns, cs);
+      goto error;
+    }
+
+  handle = (app_ns_api_handle_t *) & cs->private_data;
+
+  switch (msg.type)
+    {
+    case APP_SAPI_MSG_TYPE_ATTACH:
+      session_api_attach_handler (app_ns, cs, &msg.attach);
+      break;
+    case APP_SAPI_MSG_TYPE_ADD_DEL_WORKER:
+      sapi_add_del_worker_handler (app_ns, cs, &msg.worker_add_del);
+      break;
+    default:
+      clib_warning ("app wrk %u unknown message type: %u",
+		    handle->aah_app_wrk_index, msg.type);
+      break;
+    }
+
+error:
+  return 0;
+}
+
+static clib_error_t *
+sapi_sock_write_ready (clib_file_t * cf)
+{
+  app_ns_api_handle_t *handle = (app_ns_api_handle_t *) & cf->private_data;
+  clib_warning ("called for app ns %u", handle->aah_app_ns_index);
+  return 0;
+}
+
+static clib_error_t *
+sapi_sock_error (clib_file_t * cf)
+{
+  app_ns_api_handle_t *handle = (app_ns_api_handle_t *) & cf->private_data;
+  app_namespace_t *app_ns;
+  clib_socket_t *cs;
+
+  app_ns = app_namespace_get (handle->aah_app_ns_index);
+  cs = appns_sapi_get_socket (app_ns, handle->aah_sock_index);
+  if (!cs)
+    return 0;
+
+  sapi_socket_detach (app_ns, cs);
+  return 0;
+}
+
+static clib_error_t *
+sapi_sock_accept_ready (clib_file_t * scf)
+{
+  app_ns_api_handle_t handle = *(app_ns_api_handle_t *) & scf->private_data;
+  app_namespace_t *app_ns;
+  clib_file_t cf = { 0 };
+  clib_error_t *err = 0;
+  clib_socket_t *ccs, *scs;
+
+  /* Listener files point to namespace */
+  app_ns = app_namespace_get (handle.aah_app_ns_index);
+
+  /*
+   * Initialize client socket
+   */
+  ccs = appns_sapi_alloc_socket (app_ns);
+
+  /* Grab server socket after client is initialized  */
+  scs = appns_sapi_get_socket (app_ns, handle.aah_sock_index);
+  if (!scs)
+    goto error;
+
+  err = clib_socket_accept (scs, ccs);
+  if (err)
+    {
+      clib_error_report (err);
+      goto error;
+    }
+
+  cf.read_function = sapi_sock_read_ready;
+  cf.write_function = sapi_sock_write_ready;
+  cf.error_function = sapi_sock_error;
+  cf.file_descriptor = ccs->fd;
+  /* File points to app namespace and socket */
+  handle.aah_sock_index = appns_sapi_socket_index (app_ns, ccs);
+  cf.private_data = handle.as_uword;
+  cf.description = format (0, "app sock conn fd: %d", ccs->fd);
+
+  /* Poll until we get an attach message. Socket points to file and
+   * application that owns the socket */
+  handle.aah_app_wrk_index = APP_INVALID_INDEX;
+  handle.aah_file_index = clib_file_add (&file_main, &cf);
+  ccs->private_data = handle.as_uword;
+
+  return err;
+
+error:
+  appns_sapi_free_socket (app_ns, ccs);
+  return err;
+}
+
+int
+appns_sapi_add_ns_socket (app_namespace_t * app_ns)
+{
+  char *subdir = "/app_ns_sockets/";
+  app_ns_api_handle_t *handle;
+  clib_file_t cf = { 0 };
+  struct stat file_stat;
+  clib_error_t *err;
+  clib_socket_t *cs;
+  u8 *dir = 0;
+  int rv = 0;
+
+  vec_add (dir, vlib_unix_get_runtime_dir (),
+	   strlen (vlib_unix_get_runtime_dir ()));
+  vec_add (dir, (u8 *) subdir, strlen (subdir));
+
+  err = vlib_unix_recursive_mkdir ((char *) dir);
+  if (err)
+    {
+      clib_error_report (err);
+      rv = -1;
+      goto error;
+    }
+
+  app_ns->sock_name = format (0, "%v%v%c", dir, app_ns->ns_id, 0);
+
+  /*
+   * Create and initialize socket to listen on
+   */
+  cs = appns_sapi_alloc_socket (app_ns);
+  cs->config = (char *) app_ns->sock_name;
+  cs->flags = CLIB_SOCKET_F_IS_SERVER |
+    CLIB_SOCKET_F_ALLOW_GROUP_WRITE |
+    CLIB_SOCKET_F_SEQPACKET | CLIB_SOCKET_F_PASSCRED;
+
+  if ((err = clib_socket_init (cs)))
+    {
+      clib_error_report (err);
+      rv = -1;
+      goto error;
+    }
+
+  if (stat ((char *) app_ns->sock_name, &file_stat) == -1)
+    {
+      rv = -1;
+      goto error;
+    }
+
+  /*
+   * Start polling it
+   */
+  cf.read_function = sapi_sock_accept_ready;
+  cf.file_descriptor = cs->fd;
+  /* File points to namespace */
+  handle = (app_ns_api_handle_t *) & cf.private_data;
+  handle->aah_app_ns_index = app_namespace_index (app_ns);
+  handle->aah_sock_index = appns_sapi_socket_index (app_ns, cs);
+  cf.description = format (0, "app sock listener: %s", app_ns->sock_name);
+
+  /* Socket points to clib file index */
+  handle = (app_ns_api_handle_t *) & cs->private_data;
+  handle->aah_file_index = clib_file_add (&file_main, &cf);
+  handle->aah_app_wrk_index = APP_INVALID_INDEX;
+
+error:
+  vec_free (dir);
+  return rv;
+}
+
+/*
  * fd.io coding-style-patch-verification: ON
  *
  * Local Variables: