vcl: separate binary api connections per thread worker

Type: fix

Change-Id: I2d72efc74a3b0a5b9e4da265475b1b01bf361125
Signed-off-by: Florin Coras <fcoras@cisco.com>

5 files changed, 72 insertions, 35 deletions

diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c
index a98b1812183..194c136796a 100644
--- a/src/vcl/vcl_bapi.c
+++ b/src/vcl/vcl_bapi.c
@@ -137,7 +137,8 @@ vl_api_app_attach_reply_t_handler (vl_api_app_attach_reply_t * mp)
   if (mp->n_fds)
     {
       vec_validate (fds, mp->n_fds);
-      if (vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5))
+      if (vl_socket_client_recv_fd_msg2 (&wrk->bapi_sock_ctx, fds, mp->n_fds,
+					 5))
 	goto failed;
 
       if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT)
@@ -217,7 +218,8 @@ vl_api_app_worker_add_del_reply_t_handler (vl_api_app_worker_add_del_reply_t *
   if (mp->n_fds)
     {
       vec_validate (fds, mp->n_fds);
-      if (vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5))
+      if (vl_socket_client_recv_fd_msg2 (&wrk->bapi_sock_ctx, fds, mp->n_fds,
+					 5))
 	goto failed;
 
       if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT)
@@ -510,19 +512,25 @@ int
 vppcom_connect_to_vpp (char *app_name)
 {
   vcl_worker_t *wrk = vcl_worker_get_current ();
-  api_main_t *am = vlibapi_get_main ();
   vppcom_cfg_t *vcl_cfg = &vcm->cfg;
+  api_main_t *am;
+
+  vlibapi_set_main (&wrk->bapi_api_ctx);
+  vlibapi_set_memory_client_main (&wrk->bapi_shm_ctx);
+  vppcom_api_hookup ();
 
   if (vcl_cfg->vpp_api_socket_name)
     {
-      if (vl_socket_client_connect ((char *) vcl_cfg->vpp_api_socket_name,
-				    app_name, 0 /* default rx/tx buffer */ ))
+      if (vl_socket_client_connect2 (&wrk->bapi_sock_ctx,
+				     (char *) vcl_cfg->vpp_api_socket_name,
+				     app_name, 0 /* default rx/tx buffer */ ))
 	{
 	  VERR ("app (%s) socket connect failed!", app_name);
 	  return VPPCOM_ECONNREFUSED;
 	}
 
-      if (vl_socket_client_init_shm (0, 1 /* want_pthread */ ))
+      if (vl_socket_client_init_shm2 (&wrk->bapi_sock_ctx, 0,
+				      1 /* want_pthread */ ))
 	{
 	  VERR ("app (%s) init shm failed!", app_name);
 	  return VPPCOM_ECONNREFUSED;
@@ -533,6 +541,8 @@ vppcom_connect_to_vpp (char *app_name)
       if (!vcl_cfg->vpp_api_filename)
 	vcl_cfg->vpp_api_filename = format (0, "/vpe-api%c", 0);
 
+      vl_set_memory_root_path ((char *) vcl_cfg->vpp_api_chroot);
+
       VDBG (0, "app (%s) connecting to VPP api (%s)...",
 	    app_name, vcl_cfg->vpp_api_filename);
 
@@ -542,9 +552,9 @@ vppcom_connect_to_vpp (char *app_name)
 	  VERR ("app (%s) connect failed!", app_name);
 	  return VPPCOM_ECONNREFUSED;
 	}
-
     }
 
+  am = vlibapi_get_main ();
   wrk->vl_input_queue = am->shmem_hdr->vl_input_queue;
   wrk->my_client_index = (u32) am->my_client_index;
   wrk->wrk_state = STATE_APP_CONN_VPP;
diff --git a/src/vcl/vcl_cfg.c b/src/vcl/vcl_cfg.c
index cce60ba634f..3be6de02167 100644
--- a/src/vcl/vcl_cfg.c
+++ b/src/vcl/vcl_cfg.c
@@ -228,7 +228,7 @@ vppcom_cfg_read_file (char *conf_fname)
   int fd;
   unformat_input_t _input, *input = &_input;
   unformat_input_t _line_input, *line_input = &_line_input;
-  u8 vc_cfg_input = 0, *chroot_path;
+  u8 vc_cfg_input = 0;
   struct stat s;
   u32 uid, gid, q_len;
 
@@ -282,19 +282,12 @@ vppcom_cfg_read_file (char *conf_fname)
 	      VCFG_DBG (0, "VCL<%d>: configured max-workers %u", getpid (),
 			vcl_cfg->max_workers);
 	    }
-	  else if (unformat (line_input, "api-prefix %s", &chroot_path))
+	  else if (unformat (line_input, "api-prefix %s",
+			     &vcl_cfg->vpp_api_chroot))
 	    {
-	      vec_terminate_c_string (chroot_path);
-	      if (vcl_cfg->vpp_api_filename)
-		vec_free (vcl_cfg->vpp_api_filename);
-	      vcl_cfg->vpp_api_filename = format (0, "/%s-vpe-api%c",
-						  chroot_path, 0);
-	      vl_set_memory_root_path ((char *) chroot_path);
-
-	      VCFG_DBG (0, "VCL<%d>: configured api-prefix (%s) and api "
-			"filename (%s)", getpid (), chroot_path,
-			vcl_cfg->vpp_api_filename);
-	      chroot_path = 0;	/* Don't vec_free() it! */
+	      vec_terminate_c_string (vcl_cfg->vpp_api_chroot);
+	      VCFG_DBG (0, "VCL<%d>: configured api-prefix (%s) ", getpid (),
+			vcl_cfg->vpp_api_chroot);
 	    }
 	  else if (unformat (line_input, "api-socket-name %s",
 			     &vcl_cfg->vpp_api_socket_name))
@@ -563,17 +556,15 @@ vppcom_cfg (vppcom_cfg_t * vcl_cfg)
   vppcom_cfg_heapsize (conf_fname);
   vppcom_cfg_read_file (conf_fname);
 
+  /* Regrab cfg after heap initialization */
+  vcl_cfg = &vcm->cfg;
   env_var_str = getenv (VPPCOM_ENV_API_PREFIX);
   if (env_var_str)
     {
-      if (vcl_cfg->vpp_api_filename)
-	vec_free (vcl_cfg->vpp_api_filename);
-      vcl_cfg->vpp_api_filename = format (0, "/%s-vpe-api%c", env_var_str, 0);
-      vl_set_memory_root_path ((char *) env_var_str);
-
-      VCFG_DBG (0, "VCL<%d>: configured api prefix (%s) and filename (%s) "
-		"from " VPPCOM_ENV_API_PREFIX "!", getpid (), env_var_str,
-		vcl_cfg->vpp_api_filename);
+      vcl_cfg->vpp_api_chroot = format (0, "%s", env_var_str);
+      vec_terminate_c_string (vcl_cfg->vpp_api_chroot);
+      VCFG_DBG (0, "VCL<%d>: configured api prefix (%s) from "
+		VPPCOM_ENV_API_PREFIX "!", getpid (), env_var_str);
     }
   env_var_str = getenv (VPPCOM_ENV_APP_NAMESPACE_ID);
   if (env_var_str)
diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h
index 8fdf7551cf3..9116111afe2 100644
--- a/src/vcl/vcl_private.h
+++ b/src/vcl/vcl_private.h
@@ -211,6 +211,7 @@ typedef struct vppcom_cfg_t_
   char *event_log_path;
   u8 *vpp_api_filename;
   u8 *vpp_api_socket_name;
+  u8 *vpp_api_chroot;
   u32 tls_engine;
 } vppcom_cfg_t;
 
@@ -298,6 +299,9 @@ typedef struct vcl_worker_
 
   u32 forked_child;
 
+  socket_client_main_t bapi_sock_ctx;
+  memory_client_main_t bapi_shm_ctx;
+  api_main_t bapi_api_ctx;
 } vcl_worker_t;
 
 typedef struct vppcom_main_t_
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index 0bbdea54940..ce47766b5c9 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -1055,7 +1055,6 @@ vppcom_app_create (char *app_name)
   vcl_worker_alloc_and_init ();
 
   /* API hookup and connect to VPP */
-  vppcom_api_hookup ();
   vcl_elog_init (vcm);
   vcm->app_state = STATE_APP_START;
   rv = vppcom_connect_to_vpp (app_name);
@@ -3590,11 +3589,21 @@ vppcom_session_worker (vcl_session_handle_t session_handle)
 int
 vppcom_worker_register (void)
 {
+  vcl_worker_t *wrk;
+  u8 *wrk_name = 0;
+  int rv;
+
   if (!vcl_worker_alloc_and_init ())
     return VPPCOM_EEXIST;
 
-  if (vcl_worker_set_bapi ())
-    return VPPCOM_EEXIST;
+  wrk = vcl_worker_get_current ();
+  wrk_name = format (0, "%s-wrk-%u", vcm->app_name, wrk->wrk_index);
+
+  rv = vppcom_connect_to_vpp ((char *) wrk_name);
+  vec_free (wrk_name);
+
+  if (rv)
+    return VPPCOM_EFAULT;
 
   if (vcl_worker_register_with_vpp ())
     return VPPCOM_EEXIST;
diff --git a/src/vlibmemory/socket_client.c b/src/vlibmemory/socket_client.c
index 5483110ae76..b7d4f2d6e54 100644
--- a/src/vlibmemory/socket_client.c
+++ b/src/vlibmemory/socket_client.c
@@ -260,10 +260,10 @@ vl_socket_client_enable_disable (int enable)
   vl_socket_client_enable_disable2 (socket_client_ctx, enable);
 }
 
-clib_error_t *
-vl_sock_api_recv_fd_msg (int socket_fd, int fds[], int n_fds, u32 wait)
+static clib_error_t *
+vl_sock_api_recv_fd_msg_internal (socket_client_main_t * scm, int fds[],
+				  int n_fds, u32 wait)
 {
-  socket_client_main_t *scm = socket_client_ctx;
   char msgbuf[16];
   char ctl[CMSG_SPACE (sizeof (int) * n_fds)
 	   + CMSG_SPACE (sizeof (struct ucred))];
@@ -275,8 +275,11 @@ vl_sock_api_recv_fd_msg (int socket_fd, int fds[], int n_fds, u32 wait)
   pid_t pid __attribute__ ((unused));
   uid_t uid __attribute__ ((unused));
   gid_t gid __attribute__ ((unused));
+  int socket_fd;
   f64 timeout;
 
+  socket_fd = scm->client_socket.fd;
+
   iov[0].iov_base = msgbuf;
   iov[0].iov_len = 5;
   mh.msg_iov = iov;
@@ -324,6 +327,26 @@ vl_sock_api_recv_fd_msg (int socket_fd, int fds[], int n_fds, u32 wait)
   return 0;
 }
 
+clib_error_t *
+vl_sock_api_recv_fd_msg (int socket_fd, int fds[], int n_fds, u32 wait)
+{
+  return vl_sock_api_recv_fd_msg_internal (socket_client_ctx, fds, n_fds,
+					   wait);
+}
+
+clib_error_t *
+vl_sock_api_recv_fd_msg2 (socket_client_main_t * scm, int socket_fd,
+			  int fds[], int n_fds, u32 wait)
+{
+  socket_client_main_t *old_ctx;
+  clib_error_t *error;
+
+  old_ctx = vl_socket_client_ctx_push (scm);
+  error = vl_sock_api_recv_fd_msg_internal (scm, fds, n_fds, wait);
+  vl_socket_client_ctx_pop (old_ctx);
+  return error;
+}
+
 static void vl_api_sock_init_shm_reply_t_handler
   (vl_api_sock_init_shm_reply_t * mp)
 {
@@ -562,7 +585,7 @@ vl_socket_client_recv_fd_msg2 (socket_client_main_t * scm, int fds[],
 {
   if (!scm->socket_fd)
     return clib_error_return (0, "no socket");
-  return vl_sock_api_recv_fd_msg (scm->client_socket.fd, fds, n_fds, wait);
+  return vl_sock_api_recv_fd_msg_internal (scm, fds, n_fds, wait);
 }
 
 clib_error_t *