session: add unix socket api for app attachment

This is an af_unix socket alternative to the binary api. To enable it,
add use-app-socket-api under session stanza in startup.conf. When the
socket api is enabled, attachments through the binary api are disabled.

The socket api only works with memfd fifo segments, i.e., shm segments
are not supported.

Type: feature

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I55ffcee201d004846daeeec85c700c7e7a578d43

1 files changed, 504 insertions, 3 deletions

diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c
index 593f2e18ba0..15c5812e790 100644
--- a/src/vnet/session/session_api.c
+++ b/src/vnet/session/session_api.c
@@ -609,7 +609,7 @@ vl_api_app_attach_t_handler (vl_api_app_attach_t * mp)
   if (!reg)
     return;
 
-  if (session_main_is_enabled () == 0)
+  if (!session_main_is_enabled () || appns_sapi_enabled ())
     {
       rv = VNET_API_ERROR_FEATURE_DISABLED;
       goto done;
@@ -692,7 +692,7 @@ vl_api_app_worker_add_del_t_handler (vl_api_app_worker_add_del_t * mp)
   application_t *app;
   u8 fd_flags = 0;
 
-  if (session_main_is_enabled () == 0)
+  if (!session_main_is_enabled () || appns_sapi_enabled ())
     {
       rv = VNET_API_ERROR_FEATURE_DISABLED;
       goto done;
@@ -770,7 +770,7 @@ vl_api_application_detach_t_handler (vl_api_application_detach_t * mp)
   vnet_app_detach_args_t _a, *a = &_a;
   application_t *app;
 
-  if (session_main_is_enabled () == 0)
+  if (!session_main_is_enabled () || appns_sapi_enabled ())
     {
       rv = VNET_API_ERROR_FEATURE_DISABLED;
       goto done;
@@ -1209,6 +1209,507 @@ session_api_hookup (vlib_main_t * vm)
 VLIB_API_INIT_FUNCTION (session_api_hookup);
 
 /*
+ * Socket api functions
+ */
+
+static void
+sapi_send_fds (app_worker_t * app_wrk, int *fds, int n_fds)
+{
+  app_sapi_msg_t smsg = { 0 };
+  app_namespace_t *app_ns;
+  application_t *app;
+  clib_socket_t *cs;
+  u32 cs_index;
+
+  app = application_get (app_wrk->app_index);
+  app_ns = app_namespace_get (app->ns_index);
+  cs_index = appns_sapi_handle_sock_index (app_wrk->api_client_index);
+  cs = appns_sapi_get_socket (app_ns, cs_index);
+
+  /* There's no payload for the message only the type */
+  smsg.type = APP_SAPI_MSG_TYPE_SEND_FDS;
+  clib_socket_sendmsg (cs, &smsg, sizeof (smsg), fds, n_fds);
+}
+
+static int
+mq_send_add_segment_sapi_cb (u32 app_wrk_index, u64 segment_handle)
+{
+  int fds[SESSION_N_FD_TYPE], n_fds = 0;
+  svm_msg_q_msg_t _msg, *msg = &_msg;
+  session_app_add_segment_msg_t *mp;
+  app_worker_t *app_wrk;
+  session_event_t *evt;
+  svm_msg_q_t *app_mq;
+  fifo_segment_t *fs;
+  ssvm_private_t *sp;
+  u8 fd_flags = 0;
+
+  app_wrk = app_worker_get (app_wrk_index);
+
+  fs = segment_manager_get_segment_w_handle (segment_handle);
+  sp = &fs->ssvm;
+  ASSERT (ssvm_type (sp) == SSVM_SEGMENT_MEMFD);
+
+  fd_flags |= SESSION_FD_F_MEMFD_SEGMENT;
+  fds[n_fds] = sp->fd;
+  n_fds += 1;
+
+  app_mq = app_wrk->event_queue;
+  if (mq_try_lock_and_alloc_msg (app_mq, msg))
+    return -1;
+
+  /*
+   * Send the fd over api socket
+   */
+  sapi_send_fds (app_wrk, fds, n_fds);
+
+  /*
+   * Send the actual message over mq
+   */
+  evt = svm_msg_q_msg_data (app_mq, msg);
+  clib_memset (evt, 0, sizeof (*evt));
+  evt->event_type = SESSION_CTRL_EVT_APP_ADD_SEGMENT;
+  mp = (session_app_add_segment_msg_t *) evt->data;
+  clib_memset (mp, 0, sizeof (*mp));
+  mp->segment_size = sp->ssvm_size;
+  mp->fd_flags = fd_flags;
+  mp->segment_handle = segment_handle;
+  strncpy ((char *) mp->segment_name, (char *) sp->name,
+	   sizeof (mp->segment_name) - 1);
+
+  svm_msg_q_add_and_unlock (app_mq, msg);
+
+  return 0;
+}
+
+static int
+mq_send_del_segment_sapi_cb (u32 app_wrk_index, u64 segment_handle)
+{
+  svm_msg_q_msg_t _msg, *msg = &_msg;
+  session_app_del_segment_msg_t *mp;
+  app_worker_t *app_wrk;
+  session_event_t *evt;
+  svm_msg_q_t *app_mq;
+
+  app_wrk = app_worker_get (app_wrk_index);
+
+  app_mq = app_wrk->event_queue;
+  if (mq_try_lock_and_alloc_msg (app_mq, msg))
+    return -1;
+
+  evt = svm_msg_q_msg_data (app_mq, msg);
+  clib_memset (evt, 0, sizeof (*evt));
+  evt->event_type = SESSION_CTRL_EVT_APP_DEL_SEGMENT;
+  mp = (session_app_del_segment_msg_t *) evt->data;
+  clib_memset (mp, 0, sizeof (*mp));
+  mp->segment_handle = segment_handle;
+  svm_msg_q_add_and_unlock (app_mq, msg);
+
+  return 0;
+}
+
+static session_cb_vft_t session_mq_sapi_cb_vft = {
+  .session_accept_callback = mq_send_session_accepted_cb,
+  .session_disconnect_callback = mq_send_session_disconnected_cb,
+  .session_connected_callback = mq_send_session_connected_cb,
+  .session_reset_callback = mq_send_session_reset_cb,
+  .session_migrate_callback = mq_send_session_migrate_cb,
+  .session_cleanup_callback = mq_send_session_cleanup_cb,
+  .add_segment_callback = mq_send_add_segment_sapi_cb,
+  .del_segment_callback = mq_send_del_segment_sapi_cb,
+};
+
+static void
+session_api_attach_handler (app_namespace_t * app_ns, clib_socket_t * cs,
+			    app_sapi_attach_msg_t * mp)
+{
+  int rv = 0, fds[SESSION_N_FD_TYPE], n_fds = 0;
+  vnet_app_attach_args_t _a, *a = &_a;
+  app_sapi_attach_reply_msg_t *rmp;
+  ssvm_private_t *evt_q_segment;
+  u8 fd_flags = 0, ctrl_thread;
+  app_ns_api_handle_t *handle;
+  app_sapi_msg_t msg = { 0 };
+  app_worker_t *app_wrk;
+  svm_msg_q_t *ctrl_mq;
+  application_t *app;
+
+  /* Make sure name is null terminated */
+  mp->name[63] = 0;
+
+  clib_memset (a, 0, sizeof (*a));
+  a->api_client_index = appns_sapi_socket_handle (app_ns, cs);
+  a->name = format (0, "%s", (char *) mp->name);
+  a->options = mp->options;
+  a->session_cb_vft = &session_mq_sapi_cb_vft;
+  a->use_sock_api = 1;
+  a->options[APP_OPTIONS_NAMESPACE] = app_namespace_index (app_ns);
+
+  if ((rv = vnet_application_attach (a)))
+    {
+      clib_warning ("attach returned: %d", rv);
+      goto done;
+    }
+
+  /* Send event queues segment */
+  if ((evt_q_segment = session_main_get_evt_q_segment ()))
+    {
+      fd_flags |= SESSION_FD_F_VPP_MQ_SEGMENT;
+      fds[n_fds] = evt_q_segment->fd;
+      n_fds += 1;
+    }
+  /* Send fifo segment fd if needed */
+  if (ssvm_type (a->segment) == SSVM_SEGMENT_MEMFD)
+    {
+      fd_flags |= SESSION_FD_F_MEMFD_SEGMENT;
+      fds[n_fds] = a->segment->fd;
+      n_fds += 1;
+    }
+  if (a->options[APP_OPTIONS_FLAGS] & APP_OPTIONS_FLAGS_EVT_MQ_USE_EVENTFD)
+    {
+      fd_flags |= SESSION_FD_F_MQ_EVENTFD;
+      fds[n_fds] = svm_msg_q_get_producer_eventfd (a->app_evt_q);
+      n_fds += 1;
+    }
+
+done:
+
+  msg.type = APP_SAPI_MSG_TYPE_ATTACH_REPLY;
+  rmp = &msg.attach_reply;
+  rmp->retval = rv;
+  if (!rv)
+    {
+      ctrl_thread = vlib_num_workers ()? 1 : 0;
+      ctrl_mq = session_main_get_vpp_event_queue (ctrl_thread);
+      rmp->app_index = a->app_index;
+      rmp->app_mq = pointer_to_uword (a->app_evt_q);
+      rmp->vpp_ctrl_mq = pointer_to_uword (ctrl_mq);
+      rmp->vpp_ctrl_mq_thread = ctrl_thread;
+      rmp->n_fds = n_fds;
+      rmp->fd_flags = fd_flags;
+      /* No segment name and size since we only support memfds
+       * in this configuration */
+      rmp->segment_handle = a->segment_handle;
+      rmp->api_client_handle = a->api_client_index;
+
+      /* Update app index for socket */
+      handle = (app_ns_api_handle_t *) & cs->private_data;
+      app = application_get (a->app_index);
+      app_wrk = application_get_worker (app, 0);
+      handle->aah_app_wrk_index = app_wrk->wrk_index;
+    }
+
+  clib_socket_sendmsg (cs, &msg, sizeof (msg), fds, n_fds);
+  vec_free (a->name);
+}
+
+static void
+sapi_socket_close_w_handle (u32 api_handle)
+{
+  app_namespace_t *app_ns = app_namespace_get (api_handle >> 16);
+  u16 sock_index = api_handle & 0xffff;
+  app_ns_api_handle_t *handle;
+  clib_socket_t *cs;
+  clib_file_t *cf;
+
+  cs = appns_sapi_get_socket (app_ns, sock_index);
+  if (!cs)
+    return;
+
+  handle = (app_ns_api_handle_t *) & cs->private_data;
+  cf = clib_file_get (&file_main, handle->aah_file_index);
+  clib_file_del (&file_main, cf);
+
+  clib_socket_close (cs);
+  appns_sapi_free_socket (app_ns, cs);
+}
+
+static void
+sapi_add_del_worker_handler (app_namespace_t * app_ns,
+			     clib_socket_t * cs,
+			     app_sapi_worker_add_del_msg_t * mp)
+{
+  int rv = 0, fds[SESSION_N_FD_TYPE], n_fds = 0;
+  app_sapi_worker_add_del_reply_msg_t *rmp;
+  app_ns_api_handle_t *handle;
+  app_sapi_msg_t msg = { 0 };
+  app_worker_t *app_wrk;
+  application_t *app;
+  u32 sapi_handle;
+  u8 fd_flags = 0;
+
+  app = application_get_if_valid (mp->app_index);
+  if (!app)
+    {
+      rv = VNET_API_ERROR_INVALID_VALUE;
+      goto done;
+    }
+
+  sapi_handle = appns_sapi_socket_handle (app_ns, cs);
+
+  vnet_app_worker_add_del_args_t args = {
+    .app_index = app->app_index,
+    .wrk_map_index = mp->wrk_index,
+    .api_client_index = sapi_handle,
+    .is_add = mp->is_add
+  };
+  rv = vnet_app_worker_add_del (&args);
+  if (rv)
+    {
+      clib_warning ("app worker add/del returned: %d", rv);
+      goto done;
+    }
+
+  if (!mp->is_add)
+    {
+      sapi_socket_close_w_handle (sapi_handle);
+      goto done;
+    }
+
+  /* Send fifo segment fd if needed */
+  if (ssvm_type (args.segment) == SSVM_SEGMENT_MEMFD)
+    {
+      fd_flags |= SESSION_FD_F_MEMFD_SEGMENT;
+      fds[n_fds] = args.segment->fd;
+      n_fds += 1;
+    }
+  if (application_segment_manager_properties (app)->use_mq_eventfd)
+    {
+      fd_flags |= SESSION_FD_F_MQ_EVENTFD;
+      fds[n_fds] = svm_msg_q_get_producer_eventfd (args.evt_q);
+      n_fds += 1;
+    }
+
+done:
+
+  msg.type = APP_SAPI_MSG_TYPE_ADD_DEL_WORKER_REPLY;
+  rmp = &msg.worker_add_del_reply;
+  rmp->retval = rv;
+  rmp->is_add = mp->is_add;
+  rmp->wrk_index = args.wrk_map_index;
+  rmp->segment_handle = args.segment_handle;
+  if (!rv && mp->is_add)
+    {
+      /* No segment name and size. This supports only memfds */
+      rmp->app_event_queue_address = pointer_to_uword (args.evt_q);
+      rmp->n_fds = n_fds;
+      rmp->fd_flags = fd_flags;
+
+      /* Update app index for socket */
+      handle = (app_ns_api_handle_t *) & cs->private_data;
+      app_wrk = application_get_worker (app, args.wrk_map_index);
+      handle->aah_app_wrk_index = app_wrk->wrk_index;
+    }
+
+  clib_socket_sendmsg (cs, &msg, sizeof (msg), fds, n_fds);
+}
+
+static void
+sapi_socket_detach (app_namespace_t * app_ns, clib_socket_t * cs)
+{
+  vnet_app_detach_args_t _a = { 0 }, *a = &_a;
+  app_ns_api_handle_t *handle;
+  app_worker_t *app_wrk;
+  u32 api_client_handle;
+
+  api_client_handle = appns_sapi_socket_handle (app_ns, cs);
+  sapi_socket_close_w_handle (api_client_handle);
+
+  /* Cleanup everything because app closed socket or crashed */
+  handle = (app_ns_api_handle_t *) & cs->private_data;
+  app_wrk = app_worker_get (handle->aah_app_wrk_index);
+  a->app_index = app_wrk->app_index;
+  a->api_client_index = api_client_handle;
+  vnet_application_detach (a);
+}
+
+static clib_error_t *
+sapi_sock_read_ready (clib_file_t * cf)
+{
+  app_ns_api_handle_t *handle = (app_ns_api_handle_t *) & cf->private_data;
+  app_sapi_msg_t msg = { 0 };
+  app_namespace_t *app_ns;
+  clib_error_t *err = 0;
+  clib_socket_t *cs;
+
+  app_ns = app_namespace_get (handle->aah_app_ns_index);
+  cs = appns_sapi_get_socket (app_ns, handle->aah_sock_index);
+  if (!cs)
+    goto error;
+
+  err = clib_socket_recvmsg (cs, &msg, sizeof (msg), 0, 0);
+  if (err)
+    {
+      clib_error_free (err);
+      sapi_socket_detach (app_ns, cs);
+      goto error;
+    }
+
+  handle = (app_ns_api_handle_t *) & cs->private_data;
+
+  switch (msg.type)
+    {
+    case APP_SAPI_MSG_TYPE_ATTACH:
+      session_api_attach_handler (app_ns, cs, &msg.attach);
+      break;
+    case APP_SAPI_MSG_TYPE_ADD_DEL_WORKER:
+      sapi_add_del_worker_handler (app_ns, cs, &msg.worker_add_del);
+      break;
+    default:
+      clib_warning ("app wrk %u unknown message type: %u",
+		    handle->aah_app_wrk_index, msg.type);
+      break;
+    }
+
+error:
+  return 0;
+}
+
+static clib_error_t *
+sapi_sock_write_ready (clib_file_t * cf)
+{
+  app_ns_api_handle_t *handle = (app_ns_api_handle_t *) & cf->private_data;
+  clib_warning ("called for app ns %u", handle->aah_app_ns_index);
+  return 0;
+}
+
+static clib_error_t *
+sapi_sock_error (clib_file_t * cf)
+{
+  app_ns_api_handle_t *handle = (app_ns_api_handle_t *) & cf->private_data;
+  app_namespace_t *app_ns;
+  clib_socket_t *cs;
+
+  app_ns = app_namespace_get (handle->aah_app_ns_index);
+  cs = appns_sapi_get_socket (app_ns, handle->aah_sock_index);
+  if (!cs)
+    return 0;
+
+  sapi_socket_detach (app_ns, cs);
+  return 0;
+}
+
+static clib_error_t *
+sapi_sock_accept_ready (clib_file_t * scf)
+{
+  app_ns_api_handle_t handle = *(app_ns_api_handle_t *) & scf->private_data;
+  app_namespace_t *app_ns;
+  clib_file_t cf = { 0 };
+  clib_error_t *err = 0;
+  clib_socket_t *ccs, *scs;
+
+  /* Listener files point to namespace */
+  app_ns = app_namespace_get (handle.aah_app_ns_index);
+
+  /*
+   * Initialize client socket
+   */
+  ccs = appns_sapi_alloc_socket (app_ns);
+
+  /* Grab server socket after client is initialized  */
+  scs = appns_sapi_get_socket (app_ns, handle.aah_sock_index);
+  if (!scs)
+    goto error;
+
+  err = clib_socket_accept (scs, ccs);
+  if (err)
+    {
+      clib_error_report (err);
+      goto error;
+    }
+
+  cf.read_function = sapi_sock_read_ready;
+  cf.write_function = sapi_sock_write_ready;
+  cf.error_function = sapi_sock_error;
+  cf.file_descriptor = ccs->fd;
+  /* File points to app namespace and socket */
+  handle.aah_sock_index = appns_sapi_socket_index (app_ns, ccs);
+  cf.private_data = handle.as_uword;
+  cf.description = format (0, "app sock conn fd: %d", ccs->fd);
+
+  /* Poll until we get an attach message. Socket points to file and
+   * application that owns the socket */
+  handle.aah_app_wrk_index = APP_INVALID_INDEX;
+  handle.aah_file_index = clib_file_add (&file_main, &cf);
+  ccs->private_data = handle.as_uword;
+
+  return err;
+
+error:
+  appns_sapi_free_socket (app_ns, ccs);
+  return err;
+}
+
+int
+appns_sapi_add_ns_socket (app_namespace_t * app_ns)
+{
+  char *subdir = "/app_ns_sockets/";
+  app_ns_api_handle_t *handle;
+  clib_file_t cf = { 0 };
+  struct stat file_stat;
+  clib_error_t *err;
+  clib_socket_t *cs;
+  u8 *dir = 0;
+  int rv = 0;
+
+  vec_add (dir, vlib_unix_get_runtime_dir (),
+	   strlen (vlib_unix_get_runtime_dir ()));
+  vec_add (dir, (u8 *) subdir, strlen (subdir));
+
+  err = vlib_unix_recursive_mkdir ((char *) dir);
+  if (err)
+    {
+      clib_error_report (err);
+      rv = -1;
+      goto error;
+    }
+
+  app_ns->sock_name = format (0, "%v%v%c", dir, app_ns->ns_id, 0);
+
+  /*
+   * Create and initialize socket to listen on
+   */
+  cs = appns_sapi_alloc_socket (app_ns);
+  cs->config = (char *) app_ns->sock_name;
+  cs->flags = CLIB_SOCKET_F_IS_SERVER |
+    CLIB_SOCKET_F_ALLOW_GROUP_WRITE |
+    CLIB_SOCKET_F_SEQPACKET | CLIB_SOCKET_F_PASSCRED;
+
+  if ((err = clib_socket_init (cs)))
+    {
+      clib_error_report (err);
+      rv = -1;
+      goto error;
+    }
+
+  if (stat ((char *) app_ns->sock_name, &file_stat) == -1)
+    {
+      rv = -1;
+      goto error;
+    }
+
+  /*
+   * Start polling it
+   */
+  cf.read_function = sapi_sock_accept_ready;
+  cf.file_descriptor = cs->fd;
+  /* File points to namespace */
+  handle = (app_ns_api_handle_t *) & cf.private_data;
+  handle->aah_app_ns_index = app_namespace_index (app_ns);
+  handle->aah_sock_index = appns_sapi_socket_index (app_ns, cs);
+  cf.description = format (0, "app sock listener: %s", app_ns->sock_name);
+
+  /* Socket points to clib file index */
+  handle = (app_ns_api_handle_t *) & cs->private_data;
+  handle->aah_file_index = clib_file_add (&file_main, &cf);
+  handle->aah_app_wrk_index = APP_INVALID_INDEX;
+
+error:
+  vec_free (dir);
+  return rv;
+}
+
+/*
  * fd.io coding-style-patch-verification: ON
  *
  * Local Variables: