From d4c709222b67f6c45ee7064637f69de9e9509222 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Thu, 28 Nov 2019 14:21:21 -0800 Subject: vcl: separate binary api connections per thread worker Type: fix Change-Id: I2d72efc74a3b0a5b9e4da265475b1b01bf361125 Signed-off-by: Florin Coras --- src/vcl/vcl_bapi.c | 24 +++++++++++++++++------- src/vcl/vcl_cfg.c | 33 ++++++++++++--------------------- src/vcl/vcl_private.h | 4 ++++ src/vcl/vppcom.c | 15 ++++++++++++--- src/vlibmemory/socket_client.c | 31 +++++++++++++++++++++++++++---- 5 files changed, 72 insertions(+), 35 deletions(-) diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c index a98b1812183..194c136796a 100644 --- a/src/vcl/vcl_bapi.c +++ b/src/vcl/vcl_bapi.c @@ -137,7 +137,8 @@ vl_api_app_attach_reply_t_handler (vl_api_app_attach_reply_t * mp) if (mp->n_fds) { vec_validate (fds, mp->n_fds); - if (vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5)) + if (vl_socket_client_recv_fd_msg2 (&wrk->bapi_sock_ctx, fds, mp->n_fds, + 5)) goto failed; if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT) @@ -217,7 +218,8 @@ vl_api_app_worker_add_del_reply_t_handler (vl_api_app_worker_add_del_reply_t * if (mp->n_fds) { vec_validate (fds, mp->n_fds); - if (vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5)) + if (vl_socket_client_recv_fd_msg2 (&wrk->bapi_sock_ctx, fds, mp->n_fds, + 5)) goto failed; if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT) @@ -510,19 +512,25 @@ int vppcom_connect_to_vpp (char *app_name) { vcl_worker_t *wrk = vcl_worker_get_current (); - api_main_t *am = vlibapi_get_main (); vppcom_cfg_t *vcl_cfg = &vcm->cfg; + api_main_t *am; + + vlibapi_set_main (&wrk->bapi_api_ctx); + vlibapi_set_memory_client_main (&wrk->bapi_shm_ctx); + vppcom_api_hookup (); if (vcl_cfg->vpp_api_socket_name) { - if (vl_socket_client_connect ((char *) vcl_cfg->vpp_api_socket_name, - app_name, 0 /* default rx/tx buffer */ )) + if (vl_socket_client_connect2 (&wrk->bapi_sock_ctx, + (char *) vcl_cfg->vpp_api_socket_name, + app_name, 0 /* default rx/tx buffer */ )) { VERR ("app (%s) socket connect failed!", app_name); return VPPCOM_ECONNREFUSED; } - if (vl_socket_client_init_shm (0, 1 /* want_pthread */ )) + if (vl_socket_client_init_shm2 (&wrk->bapi_sock_ctx, 0, + 1 /* want_pthread */ )) { VERR ("app (%s) init shm failed!", app_name); return VPPCOM_ECONNREFUSED; @@ -533,6 +541,8 @@ vppcom_connect_to_vpp (char *app_name) if (!vcl_cfg->vpp_api_filename) vcl_cfg->vpp_api_filename = format (0, "/vpe-api%c", 0); + vl_set_memory_root_path ((char *) vcl_cfg->vpp_api_chroot); + VDBG (0, "app (%s) connecting to VPP api (%s)...", app_name, vcl_cfg->vpp_api_filename); @@ -542,9 +552,9 @@ vppcom_connect_to_vpp (char *app_name) VERR ("app (%s) connect failed!", app_name); return VPPCOM_ECONNREFUSED; } - } + am = vlibapi_get_main (); wrk->vl_input_queue = am->shmem_hdr->vl_input_queue; wrk->my_client_index = (u32) am->my_client_index; wrk->wrk_state = STATE_APP_CONN_VPP; diff --git a/src/vcl/vcl_cfg.c b/src/vcl/vcl_cfg.c index cce60ba634f..3be6de02167 100644 --- a/src/vcl/vcl_cfg.c +++ b/src/vcl/vcl_cfg.c @@ -228,7 +228,7 @@ vppcom_cfg_read_file (char *conf_fname) int fd; unformat_input_t _input, *input = &_input; unformat_input_t _line_input, *line_input = &_line_input; - u8 vc_cfg_input = 0, *chroot_path; + u8 vc_cfg_input = 0; struct stat s; u32 uid, gid, q_len; @@ -282,19 +282,12 @@ vppcom_cfg_read_file (char *conf_fname) VCFG_DBG (0, "VCL<%d>: configured max-workers %u", getpid (), vcl_cfg->max_workers); } - else if (unformat (line_input, "api-prefix %s", &chroot_path)) + else if (unformat (line_input, "api-prefix %s", + &vcl_cfg->vpp_api_chroot)) { - vec_terminate_c_string (chroot_path); - if (vcl_cfg->vpp_api_filename) - vec_free (vcl_cfg->vpp_api_filename); - vcl_cfg->vpp_api_filename = format (0, "/%s-vpe-api%c", - chroot_path, 0); - vl_set_memory_root_path ((char *) chroot_path); - - VCFG_DBG (0, "VCL<%d>: configured api-prefix (%s) and api " - "filename (%s)", getpid (), chroot_path, - vcl_cfg->vpp_api_filename); - chroot_path = 0; /* Don't vec_free() it! */ + vec_terminate_c_string (vcl_cfg->vpp_api_chroot); + VCFG_DBG (0, "VCL<%d>: configured api-prefix (%s) ", getpid (), + vcl_cfg->vpp_api_chroot); } else if (unformat (line_input, "api-socket-name %s", &vcl_cfg->vpp_api_socket_name)) @@ -563,17 +556,15 @@ vppcom_cfg (vppcom_cfg_t * vcl_cfg) vppcom_cfg_heapsize (conf_fname); vppcom_cfg_read_file (conf_fname); + /* Regrab cfg after heap initialization */ + vcl_cfg = &vcm->cfg; env_var_str = getenv (VPPCOM_ENV_API_PREFIX); if (env_var_str) { - if (vcl_cfg->vpp_api_filename) - vec_free (vcl_cfg->vpp_api_filename); - vcl_cfg->vpp_api_filename = format (0, "/%s-vpe-api%c", env_var_str, 0); - vl_set_memory_root_path ((char *) env_var_str); - - VCFG_DBG (0, "VCL<%d>: configured api prefix (%s) and filename (%s) " - "from " VPPCOM_ENV_API_PREFIX "!", getpid (), env_var_str, - vcl_cfg->vpp_api_filename); + vcl_cfg->vpp_api_chroot = format (0, "%s", env_var_str); + vec_terminate_c_string (vcl_cfg->vpp_api_chroot); + VCFG_DBG (0, "VCL<%d>: configured api prefix (%s) from " + VPPCOM_ENV_API_PREFIX "!", getpid (), env_var_str); } env_var_str = getenv (VPPCOM_ENV_APP_NAMESPACE_ID); if (env_var_str) diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h index 8fdf7551cf3..9116111afe2 100644 --- a/src/vcl/vcl_private.h +++ b/src/vcl/vcl_private.h @@ -211,6 +211,7 @@ typedef struct vppcom_cfg_t_ char *event_log_path; u8 *vpp_api_filename; u8 *vpp_api_socket_name; + u8 *vpp_api_chroot; u32 tls_engine; } vppcom_cfg_t; @@ -298,6 +299,9 @@ typedef struct vcl_worker_ u32 forked_child; + socket_client_main_t bapi_sock_ctx; + memory_client_main_t bapi_shm_ctx; + api_main_t bapi_api_ctx; } vcl_worker_t; typedef struct vppcom_main_t_ diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index 0bbdea54940..ce47766b5c9 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -1055,7 +1055,6 @@ vppcom_app_create (char *app_name) vcl_worker_alloc_and_init (); /* API hookup and connect to VPP */ - vppcom_api_hookup (); vcl_elog_init (vcm); vcm->app_state = STATE_APP_START; rv = vppcom_connect_to_vpp (app_name); @@ -3590,11 +3589,21 @@ vppcom_session_worker (vcl_session_handle_t session_handle) int vppcom_worker_register (void) { + vcl_worker_t *wrk; + u8 *wrk_name = 0; + int rv; + if (!vcl_worker_alloc_and_init ()) return VPPCOM_EEXIST; - if (vcl_worker_set_bapi ()) - return VPPCOM_EEXIST; + wrk = vcl_worker_get_current (); + wrk_name = format (0, "%s-wrk-%u", vcm->app_name, wrk->wrk_index); + + rv = vppcom_connect_to_vpp ((char *) wrk_name); + vec_free (wrk_name); + + if (rv) + return VPPCOM_EFAULT; if (vcl_worker_register_with_vpp ()) return VPPCOM_EEXIST; diff --git a/src/vlibmemory/socket_client.c b/src/vlibmemory/socket_client.c index 5483110ae76..b7d4f2d6e54 100644 --- a/src/vlibmemory/socket_client.c +++ b/src/vlibmemory/socket_client.c @@ -260,10 +260,10 @@ vl_socket_client_enable_disable (int enable) vl_socket_client_enable_disable2 (socket_client_ctx, enable); } -clib_error_t * -vl_sock_api_recv_fd_msg (int socket_fd, int fds[], int n_fds, u32 wait) +static clib_error_t * +vl_sock_api_recv_fd_msg_internal (socket_client_main_t * scm, int fds[], + int n_fds, u32 wait) { - socket_client_main_t *scm = socket_client_ctx; char msgbuf[16]; char ctl[CMSG_SPACE (sizeof (int) * n_fds) + CMSG_SPACE (sizeof (struct ucred))]; @@ -275,8 +275,11 @@ vl_sock_api_recv_fd_msg (int socket_fd, int fds[], int n_fds, u32 wait) pid_t pid __attribute__ ((unused)); uid_t uid __attribute__ ((unused)); gid_t gid __attribute__ ((unused)); + int socket_fd; f64 timeout; + socket_fd = scm->client_socket.fd; + iov[0].iov_base = msgbuf; iov[0].iov_len = 5; mh.msg_iov = iov; @@ -324,6 +327,26 @@ vl_sock_api_recv_fd_msg (int socket_fd, int fds[], int n_fds, u32 wait) return 0; } +clib_error_t * +vl_sock_api_recv_fd_msg (int socket_fd, int fds[], int n_fds, u32 wait) +{ + return vl_sock_api_recv_fd_msg_internal (socket_client_ctx, fds, n_fds, + wait); +} + +clib_error_t * +vl_sock_api_recv_fd_msg2 (socket_client_main_t * scm, int socket_fd, + int fds[], int n_fds, u32 wait) +{ + socket_client_main_t *old_ctx; + clib_error_t *error; + + old_ctx = vl_socket_client_ctx_push (scm); + error = vl_sock_api_recv_fd_msg_internal (scm, fds, n_fds, wait); + vl_socket_client_ctx_pop (old_ctx); + return error; +} + static void vl_api_sock_init_shm_reply_t_handler (vl_api_sock_init_shm_reply_t * mp) { @@ -562,7 +585,7 @@ vl_socket_client_recv_fd_msg2 (socket_client_main_t * scm, int fds[], { if (!scm->socket_fd) return clib_error_return (0, "no socket"); - return vl_sock_api_recv_fd_msg (scm->client_socket.fd, fds, n_fds, wait); + return vl_sock_api_recv_fd_msg_internal (scm, fds, n_fds, wait); } clib_error_t * -- cgit 1.2.3-korg