vcl: add support for app socket api

To enable add "app-socket-api /path/to/socket" to vcl.conf. On vpp side, add
use-app-socket-api to session stanza in startup.conf

VPP allocates a socket per application namespace which it places in the
app_ns_sockets subfolder of the run folder (default /var/run/vpp). The
socket used implicitly selects the app namespace for the vcl app.

Type: feature

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: Ifc14b93dcbf6ef9bed1852d46cd069f4855b92ef

7 files changed, 456 insertions, 12 deletions

diff --git a/src/vcl/CMakeLists.txt b/src/vcl/CMakeLists.txt
index ab0a6ad6a2d..e6d8f98ffed 100644
--- a/src/vcl/CMakeLists.txt
+++ b/src/vcl/CMakeLists.txt
@@ -21,6 +21,7 @@ add_vpp_library(vppcom
   vcl_cfg.c
   vcl_private.c
   vcl_locked.c
+  vcl_sapi.c
 
   LINK_LIBRARIES
   vppinfra svm vlibmemoryclient rt pthread
diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c
index d3a27acaced..900fb966242 100644
--- a/src/vcl/vcl_bapi.c
+++ b/src/vcl/vcl_bapi.c
@@ -61,12 +61,6 @@ static void
     vcm->bapi_app_state = STATE_APP_ENABLED;
 }
 
-static u64
-vcl_vpp_worker_segment_handle (u32 wrk_index)
-{
-  return (VCL_INVALID_SEGMENT_HANDLE - wrk_index - 1);
-}
-
 static void
 vl_api_app_attach_reply_t_handler (vl_api_app_attach_reply_t * mp)
 {
@@ -691,6 +685,21 @@ vcl_bapi_app_worker_del (vcl_worker_t * wrk)
 }
 
 int
+vcl_bapi_recv_fds (vcl_worker_t * wrk, int *fds, int n_fds)
+{
+  clib_error_t *err;
+
+  if ((err = vl_socket_client_recv_fd_msg2 (&wrk->bapi_sock_ctx, fds, n_fds,
+					    5)))
+    {
+      clib_error_report (err);
+      return -1;
+    }
+
+  return 0;
+}
+
+int
 vppcom_session_tls_add_cert (uint32_t session_handle, char *cert,
 			     uint32_t cert_len)
 {
diff --git a/src/vcl/vcl_cfg.c b/src/vcl/vcl_cfg.c
index 0900b318fce..3fdda9cd604 100644
--- a/src/vcl/vcl_cfg.c
+++ b/src/vcl/vcl_cfg.c
@@ -296,6 +296,13 @@ vppcom_cfg_read_file (char *conf_fname)
 	      VCFG_DBG (0, "VCL<%d>: configured api-socket-name (%s)",
 			getpid (), vcl_cfg->vpp_bapi_socket_name);
 	    }
+	  else if (unformat (line_input, "app-socket-api %s",
+			     &vcl_cfg->vpp_app_socket_api))
+	    {
+	      vec_terminate_c_string (vcl_cfg->vpp_app_socket_api);
+	      VCFG_DBG (0, "VCL<%d>: configured app-socket-api (%s)",
+			getpid (), vcl_cfg->vpp_app_socket_api);
+	    }
 	  else if (unformat (line_input, "vpp-api-q-length %d", &q_len))
 	    {
 	      if (q_len < vcl_cfg->vpp_api_q_length)
diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c
index a07e0b165ec..fd35db0b678 100644
--- a/src/vcl/vcl_private.c
+++ b/src/vcl/vcl_private.c
@@ -101,12 +101,30 @@ vcl_worker_free (vcl_worker_t * wrk)
   pool_put (vcm->workers, wrk);
 }
 
+int
+vcl_api_app_worker_add (void)
+{
+  if (vcm->cfg.vpp_app_socket_api)
+    return vcl_sapi_app_worker_add ();
+
+  return vcl_bapi_app_worker_add ();
+}
+
+void
+vcl_api_app_worker_del (vcl_worker_t * wrk)
+{
+  if (vcm->cfg.vpp_app_socket_api)
+    return vcl_sapi_app_worker_del (wrk);
+
+  vcl_bapi_app_worker_del (wrk);
+}
+
 void
 vcl_worker_cleanup (vcl_worker_t * wrk, u8 notify_vpp)
 {
   clib_spinlock_lock (&vcm->workers_lock);
   if (notify_vpp)
-    vcl_bapi_app_worker_del (wrk);
+    vcl_api_app_worker_del (wrk);
 
   if (wrk->mqs_epfd > 0)
     close (wrk->mqs_epfd);
@@ -184,7 +202,7 @@ vcl_worker_register_with_vpp (void)
 
   clib_spinlock_lock (&vcm->workers_lock);
 
-  if (vcl_bapi_app_worker_add ())
+  if (vcl_api_app_worker_add ())
     {
       VDBG (0, "failed to add worker to vpp");
       clib_spinlock_unlock (&vcm->workers_lock);
diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h
index 7e1d35b2205..bbcc3fabedb 100644
--- a/src/vcl/vcl_private.h
+++ b/src/vcl/vcl_private.h
@@ -214,6 +214,7 @@ typedef struct vppcom_cfg_t_
   f64 accept_timeout;
   u32 event_ring_size;
   char *event_log_path;
+  u8 *vpp_app_socket_api;	/**< app socket api socket file name */
   u8 *vpp_bapi_filename;	/**< bapi shm transport file name */
   u8 *vpp_bapi_socket_name;	/**< bapi socket transport socket name */
   u8 *vpp_bapi_chroot;
@@ -302,6 +303,7 @@ typedef struct vcl_worker_
 
   u32 forked_child;
 
+  clib_socket_t app_api_sock;
   socket_client_main_t bapi_sock_ctx;
   memory_client_main_t bapi_shm_ctx;
   api_main_t bapi_api_ctx;
@@ -667,6 +669,12 @@ vcl_session_vpp_evt_q (vcl_worker_t * wrk, vcl_session_t * s)
   return wrk->vpp_event_queues[s->vpp_thread_index];
 }
 
+static inline u64
+vcl_vpp_worker_segment_handle (u32 wrk_index)
+{
+  return (VCL_INVALID_SEGMENT_HANDLE - wrk_index - 1);
+}
+
 void vcl_send_session_worker_update (vcl_worker_t * wrk, vcl_session_t * s,
 				     u32 wrk_index);
 int vcl_send_worker_rpc (u32 dst_wrk_index, void *data, u32 data_len);
@@ -683,6 +691,7 @@ int vcl_bapi_attach (void);
 int vcl_bapi_app_worker_add (void);
 void vcl_bapi_app_worker_del (vcl_worker_t * wrk);
 void vcl_bapi_disconnect_from_vpp (void);
+int vcl_bapi_recv_fds (vcl_worker_t * wrk, int *fds, int n_fds);
 void vcl_bapi_send_application_tls_cert_add (vcl_session_t * session,
 					     char *cert, u32 cert_len);
 void vcl_bapi_send_application_tls_key_add (vcl_session_t * session,
@@ -690,6 +699,15 @@ void vcl_bapi_send_application_tls_key_add (vcl_session_t * session,
 u32 vcl_bapi_max_nsid_len (void);
 int vcl_bapi_worker_set (void);
 
+/*
+ * VCL Socket API
+ */
+int vcl_sapi_attach (void);
+int vcl_sapi_app_worker_add (void);
+void vcl_sapi_app_worker_del (vcl_worker_t * wrk);
+void vcl_sapi_detach (vcl_worker_t * wrk);
+int vcl_sapi_recv_fds (vcl_worker_t * wrk, int *fds, int n_fds);
+
 #endif /* SRC_VCL_VCL_PRIVATE_H_ */
 
 /*
diff --git a/src/vcl/vcl_sapi.c b/src/vcl/vcl_sapi.c
new file mode 100644
index 00000000000..f160786e262
--- /dev/null
+++ b/src/vcl/vcl_sapi.c
@@ -0,0 +1,362 @@
+/*
+ * Copyright (c) 2020 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vcl/vcl_private.h>
+
+static int
+vcl_api_connect_app_socket (vcl_worker_t * wrk)
+{
+  clib_socket_t *cs = &wrk->app_api_sock;
+  clib_error_t *err;
+  int rv = 0;
+
+  cs->config = (char *) vcm->cfg.vpp_app_socket_api;
+  cs->flags = CLIB_SOCKET_F_IS_CLIENT | CLIB_SOCKET_F_SEQPACKET;
+
+  wrk->vcl_needs_real_epoll = 1;
+
+  if ((err = clib_socket_init (cs)))
+    {
+      clib_error_report (err);
+      rv = -1;
+      goto done;
+    }
+
+done:
+
+  wrk->vcl_needs_real_epoll = 0;
+
+  return rv;
+}
+
+static int
+vcl_api_attach_reply_handler (app_sapi_attach_reply_msg_t * mp, int *fds)
+{
+  vcl_worker_t *wrk = vcl_worker_get_current ();
+  int i, rv, n_fds_used = 0;
+  svm_msg_q_t *ctrl_mq;
+  u64 segment_handle;
+  u8 *segment_name;
+
+  if (mp->retval)
+    {
+      VERR ("attach failed: %U", format_session_error, mp->retval);
+      goto failed;
+    }
+
+  wrk->bapi_client_index = mp->api_client_handle;
+  wrk->app_event_queue = uword_to_pointer (mp->app_mq, svm_msg_q_t *);
+  ctrl_mq = uword_to_pointer (mp->vpp_ctrl_mq, svm_msg_q_t *);
+  vec_validate (wrk->vpp_event_queues, mp->vpp_ctrl_mq_thread);
+  wrk->vpp_event_queues[mp->vpp_ctrl_mq_thread] = ctrl_mq;
+  vcm->ctrl_mq = wrk->ctrl_mq = ctrl_mq;
+  segment_handle = mp->segment_handle;
+  if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
+    {
+      VERR ("invalid segment handle");
+      goto failed;
+    }
+
+  if (!mp->n_fds)
+    goto failed;
+
+  if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT)
+    if (vcl_segment_attach (vcl_vpp_worker_segment_handle (0), "vpp-mq-seg",
+			    SSVM_SEGMENT_MEMFD, fds[n_fds_used++]))
+      goto failed;
+
+  if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT)
+    {
+      segment_name = format (0, "memfd-%ld%c", segment_handle, 0);
+      rv = vcl_segment_attach (segment_handle, (char *) segment_name,
+			       SSVM_SEGMENT_MEMFD, fds[n_fds_used++]);
+      vec_free (segment_name);
+      if (rv != 0)
+	goto failed;
+    }
+
+  if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD)
+    {
+      svm_msg_q_set_consumer_eventfd (wrk->app_event_queue,
+				      fds[n_fds_used++]);
+      vcl_mq_epoll_add_evfd (wrk, wrk->app_event_queue);
+    }
+
+  vcm->app_index = mp->app_index;
+
+  return 0;
+
+failed:
+
+  for (i = clib_max (n_fds_used - 1, 0); i < mp->n_fds; i++)
+    close (fds[i]);
+
+  return -1;
+}
+
+static int
+vcl_api_send_attach (clib_socket_t * cs)
+{
+  app_sapi_msg_t msg = { 0 };
+  app_sapi_attach_msg_t *mp = &msg.attach;
+  u8 app_is_proxy, tls_engine;
+  clib_error_t *err;
+
+  app_is_proxy = (vcm->cfg.app_proxy_transport_tcp ||
+		  vcm->cfg.app_proxy_transport_udp);
+  tls_engine = CRYPTO_ENGINE_OPENSSL;
+
+  clib_memcpy (&mp->name, vcm->app_name, vec_len (vcm->app_name));
+  mp->options[APP_OPTIONS_FLAGS] =
+    APP_OPTIONS_FLAGS_ACCEPT_REDIRECT | APP_OPTIONS_FLAGS_ADD_SEGMENT |
+    (vcm->cfg.app_scope_local ? APP_OPTIONS_FLAGS_USE_LOCAL_SCOPE : 0) |
+    (vcm->cfg.app_scope_global ? APP_OPTIONS_FLAGS_USE_GLOBAL_SCOPE : 0) |
+    (app_is_proxy ? APP_OPTIONS_FLAGS_IS_PROXY : 0) |
+    (vcm->cfg.use_mq_eventfd ? APP_OPTIONS_FLAGS_EVT_MQ_USE_EVENTFD : 0);
+  mp->options[APP_OPTIONS_PROXY_TRANSPORT] =
+    (u64) ((vcm->cfg.app_proxy_transport_tcp ? 1 << TRANSPORT_PROTO_TCP : 0) |
+	   (vcm->cfg.app_proxy_transport_udp ? 1 << TRANSPORT_PROTO_UDP : 0));
+  mp->options[APP_OPTIONS_SEGMENT_SIZE] = vcm->cfg.segment_size;
+  mp->options[APP_OPTIONS_ADD_SEGMENT_SIZE] = vcm->cfg.add_segment_size;
+  mp->options[APP_OPTIONS_RX_FIFO_SIZE] = vcm->cfg.rx_fifo_size;
+  mp->options[APP_OPTIONS_TX_FIFO_SIZE] = vcm->cfg.tx_fifo_size;
+  mp->options[APP_OPTIONS_PREALLOC_FIFO_PAIRS] =
+    vcm->cfg.preallocated_fifo_pairs;
+  mp->options[APP_OPTIONS_EVT_QUEUE_SIZE] = vcm->cfg.event_queue_size;
+  mp->options[APP_OPTIONS_TLS_ENGINE] = tls_engine;
+
+  msg.type = APP_SAPI_MSG_TYPE_ATTACH;
+  err = clib_socket_sendmsg (cs, &msg, sizeof (msg), 0, 0);
+  if (err)
+    {
+      clib_error_report (err);
+      return -1;
+    }
+
+  return 0;
+}
+
+int
+vcl_sapi_attach (void)
+{
+  vcl_worker_t *wrk = vcl_worker_get_current ();
+  app_sapi_msg_t _rmp, *rmp = &_rmp;
+  clib_error_t *err;
+  clib_socket_t *cs;
+  int fds[SESSION_N_FD_TYPE];
+
+  /*
+   * Init client socket and send attach
+   */
+  if (vcl_api_connect_app_socket (wrk))
+    return -1;
+
+  cs = &wrk->app_api_sock;
+  if (vcl_api_send_attach (cs))
+    return -1;
+
+  /*
+   * Wait for attach reply
+   */
+  err = clib_socket_recvmsg (cs, rmp, sizeof (*rmp), fds, ARRAY_LEN (fds));
+  if (err)
+    {
+      clib_error_report (err);
+      return -1;
+    }
+
+  if (rmp->type != APP_SAPI_MSG_TYPE_ATTACH_REPLY)
+    return -1;
+
+  return vcl_api_attach_reply_handler (&rmp->attach_reply, fds);
+}
+
+static int
+vcl_api_add_del_worker_reply_handler (app_sapi_worker_add_del_reply_msg_t *
+				      mp, int *fds)
+{
+  int n_fds = 0, i, rv;
+  u64 segment_handle;
+  vcl_worker_t *wrk;
+
+  if (mp->retval)
+    {
+      VDBG (0, "add/del worker failed: %U", format_session_error, mp->retval);
+      goto failed;
+    }
+
+  if (!mp->is_add)
+    goto failed;
+
+  wrk = vcl_worker_get_current ();
+  wrk->vpp_wrk_index = mp->wrk_index;
+  wrk->app_event_queue = uword_to_pointer (mp->app_event_queue_address,
+					   svm_msg_q_t *);
+  wrk->ctrl_mq = vcm->ctrl_mq;
+
+  segment_handle = mp->segment_handle;
+  if (segment_handle == VCL_INVALID_SEGMENT_HANDLE)
+    {
+      clib_warning ("invalid segment handle");
+      goto failed;
+    }
+
+  if (!mp->n_fds)
+    goto failed;
+
+  if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT)
+    if (vcl_segment_attach (vcl_vpp_worker_segment_handle (wrk->wrk_index),
+			    "vpp-worker-seg", SSVM_SEGMENT_MEMFD,
+			    fds[n_fds++]))
+      goto failed;
+
+  if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT)
+    {
+      u8 *segment_name = format (0, "memfd-%ld%c", segment_handle, 0);
+      rv = vcl_segment_attach (segment_handle, (char *) segment_name,
+			       SSVM_SEGMENT_MEMFD, fds[n_fds++]);
+      vec_free (segment_name);
+      if (rv != 0)
+	goto failed;
+    }
+
+  if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD)
+    {
+      svm_msg_q_set_consumer_eventfd (wrk->app_event_queue, fds[n_fds]);
+      vcl_mq_epoll_add_evfd (wrk, wrk->app_event_queue);
+      n_fds++;
+    }
+
+  VDBG (0, "worker %u vpp-worker %u added", wrk->wrk_index,
+	wrk->vpp_wrk_index);
+
+  return 0;
+
+failed:
+  for (i = clib_max (n_fds - 1, 0); i < mp->n_fds; i++)
+    close (fds[i]);
+
+  return -1;
+}
+
+int
+vcl_sapi_app_worker_add (void)
+{
+  vcl_worker_t *wrk = vcl_worker_get_current ();
+  app_sapi_worker_add_del_msg_t *mp;
+  app_sapi_msg_t _rmp, *rmp = &_rmp;
+  app_sapi_msg_t msg = { 0 };
+  int fds[SESSION_N_FD_TYPE];
+  clib_error_t *err;
+  clib_socket_t *cs;
+
+  /* Connect to socket api */
+  if (vcl_api_connect_app_socket (wrk))
+    return -1;
+
+  /*
+   * Send add worker
+   */
+  cs = &wrk->app_api_sock;
+
+  msg.type = APP_SAPI_MSG_TYPE_ADD_DEL_WORKER;
+  mp = &msg.worker_add_del;
+  mp->app_index = vcm->app_index;
+  mp->is_add = 1;
+
+  err = clib_socket_sendmsg (cs, &msg, sizeof (msg), 0, 0);
+  if (err)
+    {
+      clib_error_report (err);
+      return -1;
+    }
+
+  /*
+   * Wait for reply and process it
+   */
+  err = clib_socket_recvmsg (cs, rmp, sizeof (*rmp), fds, ARRAY_LEN (fds));
+  if (err)
+    {
+      clib_error_report (err);
+      return -1;
+    }
+
+  if (rmp->type != APP_SAPI_MSG_TYPE_ADD_DEL_WORKER_REPLY)
+    {
+      clib_warning ("unexpected reply type %u", rmp->type);
+      return -1;
+    }
+
+  return vcl_api_add_del_worker_reply_handler (&rmp->worker_add_del_reply,
+					       fds);
+}
+
+void
+vcl_sapi_app_worker_del (vcl_worker_t * wrk)
+{
+  app_sapi_worker_add_del_msg_t *mp;
+  app_sapi_msg_t msg = { 0 };
+  clib_error_t *err;
+  clib_socket_t *cs;
+
+  cs = &wrk->app_api_sock;
+
+  msg.type = APP_SAPI_MSG_TYPE_ADD_DEL_WORKER;
+  mp = &msg.worker_add_del;
+  mp->app_index = vcm->app_index;
+  mp->wrk_index = wrk->vpp_wrk_index;
+  mp->is_add = 0;
+
+  err = clib_socket_sendmsg (cs, &msg, sizeof (msg), 0, 0);
+  if (err)
+    clib_error_report (err);
+  clib_socket_close (cs);
+}
+
+void
+vcl_sapi_detach (vcl_worker_t * wrk)
+{
+  clib_socket_t *cs = &wrk->app_api_sock;
+  clib_socket_close (cs);
+}
+
+int
+vcl_sapi_recv_fds (vcl_worker_t * wrk, int *fds, int n_fds)
+{
+  app_sapi_msg_t _msg, *msg = &_msg;
+  clib_socket_t *cs;
+  clib_error_t *err;
+
+  cs = &wrk->app_api_sock;
+
+  err = clib_socket_recvmsg (cs, msg, sizeof (*msg), fds, n_fds);
+  if (err)
+    {
+      clib_error_report (err);
+      return -1;
+    }
+  if (msg->type != APP_SAPI_MSG_TYPE_SEND_FDS)
+    return -1;
+
+  return 0;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index 0f79f88423f..dcf8ba2aa72 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -863,6 +863,16 @@ vcl_session_worker_update_reply_handler (vcl_worker_t * wrk, void *data)
 	s->vpp_handle, wrk->wrk_index);
 }
 
+static int
+vcl_api_recv_fd (vcl_worker_t * wrk, int *fds, int n_fds)
+{
+
+  if (vcm->cfg.vpp_app_socket_api)
+    return vcl_sapi_recv_fds (wrk, fds, n_fds);
+
+  return vcl_bapi_recv_fds (wrk, fds, n_fds);
+}
+
 static void
 vcl_session_app_add_segment_handler (vcl_worker_t * wrk, void *data)
 {
@@ -875,7 +885,7 @@ vcl_session_app_add_segment_handler (vcl_worker_t * wrk, void *data)
 
   if (msg->fd_flags)
     {
-      vl_socket_client_recv_fd_msg2 (&wrk->bapi_sock_ctx, &fd, 1, 5);
+      vcl_api_recv_fd (wrk, &fd, 1);
       seg_type = SSVM_SEGMENT_MEMFD;
     }
 
@@ -1178,6 +1188,26 @@ vppcom_app_exit (void)
   vcl_elog_stop (vcm);
 }
 
+static int
+vcl_api_attach (void)
+{
+  if (vcm->cfg.vpp_app_socket_api)
+    return vcl_sapi_attach ();
+
+  return vcl_bapi_attach ();
+}
+
+static void
+vcl_api_detach (vcl_worker_t * wrk)
+{
+  vcl_send_app_detach (wrk);
+
+  if (vcm->cfg.vpp_app_socket_api)
+    return vcl_sapi_detach (wrk);
+
+  return vcl_bapi_disconnect_from_vpp ();
+}
+
 /*
  * VPPCOM Public API functions
  */
@@ -1211,7 +1241,7 @@ vppcom_app_create (const char *app_name)
   /* Allocate default worker */
   vcl_worker_alloc_and_init ();
 
-  if ((rv = vcl_bapi_attach ()))
+  if ((rv = vcl_api_attach ()))
     return rv;
 
   VDBG (0, "app_name '%s', my_client_index %d (0x%x)", app_name,
@@ -1241,8 +1271,7 @@ vppcom_app_destroy (void)
   }));
   /* *INDENT-ON* */
 
-  vcl_send_app_detach (current_wrk);
-  vcl_bapi_disconnect_from_vpp ();
+  vcl_api_detach (current_wrk);
   vcl_worker_cleanup (current_wrk, 0 /* notify vpp */ );
 
   vcl_elog_stop (vcm);