diff options
Diffstat (limited to 'src/vcl')
| -rw-r--r-- | src/vcl/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | src/vcl/vcl_bapi.c | 21 | ||||
| -rw-r--r-- | src/vcl/vcl_cfg.c | 7 | ||||
| -rw-r--r-- | src/vcl/vcl_private.c | 22 | ||||
| -rw-r--r-- | src/vcl/vcl_private.h | 18 | ||||
| -rw-r--r-- | src/vcl/vcl_sapi.c | 362 | ||||
| -rw-r--r-- | src/vcl/vppcom.c | 37 |
7 files changed, 456 insertions, 12 deletions
diff --git a/src/vcl/CMakeLists.txt b/src/vcl/CMakeLists.txt index ab0a6ad6a2d..e6d8f98ffed 100644 --- a/src/vcl/CMakeLists.txt +++ b/src/vcl/CMakeLists.txt @@ -21,6 +21,7 @@ add_vpp_library(vppcom vcl_cfg.c vcl_private.c vcl_locked.c + vcl_sapi.c LINK_LIBRARIES vppinfra svm vlibmemoryclient rt pthread diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c index d3a27acaced..900fb966242 100644 --- a/src/vcl/vcl_bapi.c +++ b/src/vcl/vcl_bapi.c @@ -61,12 +61,6 @@ static void vcm->bapi_app_state = STATE_APP_ENABLED; } -static u64 -vcl_vpp_worker_segment_handle (u32 wrk_index) -{ - return (VCL_INVALID_SEGMENT_HANDLE - wrk_index - 1); -} - static void vl_api_app_attach_reply_t_handler (vl_api_app_attach_reply_t * mp) { @@ -691,6 +685,21 @@ vcl_bapi_app_worker_del (vcl_worker_t * wrk) } int +vcl_bapi_recv_fds (vcl_worker_t * wrk, int *fds, int n_fds) +{ + clib_error_t *err; + + if ((err = vl_socket_client_recv_fd_msg2 (&wrk->bapi_sock_ctx, fds, n_fds, + 5))) + { + clib_error_report (err); + return -1; + } + + return 0; +} + +int vppcom_session_tls_add_cert (uint32_t session_handle, char *cert, uint32_t cert_len) { diff --git a/src/vcl/vcl_cfg.c b/src/vcl/vcl_cfg.c index 0900b318fce..3fdda9cd604 100644 --- a/src/vcl/vcl_cfg.c +++ b/src/vcl/vcl_cfg.c @@ -296,6 +296,13 @@ vppcom_cfg_read_file (char *conf_fname) VCFG_DBG (0, "VCL<%d>: configured api-socket-name (%s)", getpid (), vcl_cfg->vpp_bapi_socket_name); } + else if (unformat (line_input, "app-socket-api %s", + &vcl_cfg->vpp_app_socket_api)) + { + vec_terminate_c_string (vcl_cfg->vpp_app_socket_api); + VCFG_DBG (0, "VCL<%d>: configured app-socket-api (%s)", + getpid (), vcl_cfg->vpp_app_socket_api); + } else if (unformat (line_input, "vpp-api-q-length %d", &q_len)) { if (q_len < vcl_cfg->vpp_api_q_length) diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c index a07e0b165ec..fd35db0b678 100644 --- a/src/vcl/vcl_private.c +++ b/src/vcl/vcl_private.c @@ -101,12 +101,30 @@ vcl_worker_free (vcl_worker_t * wrk) pool_put (vcm->workers, wrk); } +int +vcl_api_app_worker_add (void) +{ + if (vcm->cfg.vpp_app_socket_api) + return vcl_sapi_app_worker_add (); + + return vcl_bapi_app_worker_add (); +} + +void +vcl_api_app_worker_del (vcl_worker_t * wrk) +{ + if (vcm->cfg.vpp_app_socket_api) + return vcl_sapi_app_worker_del (wrk); + + vcl_bapi_app_worker_del (wrk); +} + void vcl_worker_cleanup (vcl_worker_t * wrk, u8 notify_vpp) { clib_spinlock_lock (&vcm->workers_lock); if (notify_vpp) - vcl_bapi_app_worker_del (wrk); + vcl_api_app_worker_del (wrk); if (wrk->mqs_epfd > 0) close (wrk->mqs_epfd); @@ -184,7 +202,7 @@ vcl_worker_register_with_vpp (void) clib_spinlock_lock (&vcm->workers_lock); - if (vcl_bapi_app_worker_add ()) + if (vcl_api_app_worker_add ()) { VDBG (0, "failed to add worker to vpp"); clib_spinlock_unlock (&vcm->workers_lock); diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h index 7e1d35b2205..bbcc3fabedb 100644 --- a/src/vcl/vcl_private.h +++ b/src/vcl/vcl_private.h @@ -214,6 +214,7 @@ typedef struct vppcom_cfg_t_ f64 accept_timeout; u32 event_ring_size; char *event_log_path; + u8 *vpp_app_socket_api; /**< app socket api socket file name */ u8 *vpp_bapi_filename; /**< bapi shm transport file name */ u8 *vpp_bapi_socket_name; /**< bapi socket transport socket name */ u8 *vpp_bapi_chroot; @@ -302,6 +303,7 @@ typedef struct vcl_worker_ u32 forked_child; + clib_socket_t app_api_sock; socket_client_main_t bapi_sock_ctx; memory_client_main_t bapi_shm_ctx; api_main_t bapi_api_ctx; @@ -667,6 +669,12 @@ vcl_session_vpp_evt_q (vcl_worker_t * wrk, vcl_session_t * s) return wrk->vpp_event_queues[s->vpp_thread_index]; } +static inline u64 +vcl_vpp_worker_segment_handle (u32 wrk_index) +{ + return (VCL_INVALID_SEGMENT_HANDLE - wrk_index - 1); +} + void vcl_send_session_worker_update (vcl_worker_t * wrk, vcl_session_t * s, u32 wrk_index); int vcl_send_worker_rpc (u32 dst_wrk_index, void *data, u32 data_len); @@ -683,6 +691,7 @@ int vcl_bapi_attach (void); int vcl_bapi_app_worker_add (void); void vcl_bapi_app_worker_del (vcl_worker_t * wrk); void vcl_bapi_disconnect_from_vpp (void); +int vcl_bapi_recv_fds (vcl_worker_t * wrk, int *fds, int n_fds); void vcl_bapi_send_application_tls_cert_add (vcl_session_t * session, char *cert, u32 cert_len); void vcl_bapi_send_application_tls_key_add (vcl_session_t * session, @@ -690,6 +699,15 @@ void vcl_bapi_send_application_tls_key_add (vcl_session_t * session, u32 vcl_bapi_max_nsid_len (void); int vcl_bapi_worker_set (void); +/* + * VCL Socket API + */ +int vcl_sapi_attach (void); +int vcl_sapi_app_worker_add (void); +void vcl_sapi_app_worker_del (vcl_worker_t * wrk); +void vcl_sapi_detach (vcl_worker_t * wrk); +int vcl_sapi_recv_fds (vcl_worker_t * wrk, int *fds, int n_fds); + #endif /* SRC_VCL_VCL_PRIVATE_H_ */ /* diff --git a/src/vcl/vcl_sapi.c b/src/vcl/vcl_sapi.c new file mode 100644 index 00000000000..f160786e262 --- /dev/null +++ b/src/vcl/vcl_sapi.c @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2020 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <vcl/vcl_private.h> + +static int +vcl_api_connect_app_socket (vcl_worker_t * wrk) +{ + clib_socket_t *cs = &wrk->app_api_sock; + clib_error_t *err; + int rv = 0; + + cs->config = (char *) vcm->cfg.vpp_app_socket_api; + cs->flags = CLIB_SOCKET_F_IS_CLIENT | CLIB_SOCKET_F_SEQPACKET; + + wrk->vcl_needs_real_epoll = 1; + + if ((err = clib_socket_init (cs))) + { + clib_error_report (err); + rv = -1; + goto done; + } + +done: + + wrk->vcl_needs_real_epoll = 0; + + return rv; +} + +static int +vcl_api_attach_reply_handler (app_sapi_attach_reply_msg_t * mp, int *fds) +{ + vcl_worker_t *wrk = vcl_worker_get_current (); + int i, rv, n_fds_used = 0; + svm_msg_q_t *ctrl_mq; + u64 segment_handle; + u8 *segment_name; + + if (mp->retval) + { + VERR ("attach failed: %U", format_session_error, mp->retval); + goto failed; + } + + wrk->bapi_client_index = mp->api_client_handle; + wrk->app_event_queue = uword_to_pointer (mp->app_mq, svm_msg_q_t *); + ctrl_mq = uword_to_pointer (mp->vpp_ctrl_mq, svm_msg_q_t *); + vec_validate (wrk->vpp_event_queues, mp->vpp_ctrl_mq_thread); + wrk->vpp_event_queues[mp->vpp_ctrl_mq_thread] = ctrl_mq; + vcm->ctrl_mq = wrk->ctrl_mq = ctrl_mq; + segment_handle = mp->segment_handle; + if (segment_handle == VCL_INVALID_SEGMENT_HANDLE) + { + VERR ("invalid segment handle"); + goto failed; + } + + if (!mp->n_fds) + goto failed; + + if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT) + if (vcl_segment_attach (vcl_vpp_worker_segment_handle (0), "vpp-mq-seg", + SSVM_SEGMENT_MEMFD, fds[n_fds_used++])) + goto failed; + + if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT) + { + segment_name = format (0, "memfd-%ld%c", segment_handle, 0); + rv = vcl_segment_attach (segment_handle, (char *) segment_name, + SSVM_SEGMENT_MEMFD, fds[n_fds_used++]); + vec_free (segment_name); + if (rv != 0) + goto failed; + } + + if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD) + { + svm_msg_q_set_consumer_eventfd (wrk->app_event_queue, + fds[n_fds_used++]); + vcl_mq_epoll_add_evfd (wrk, wrk->app_event_queue); + } + + vcm->app_index = mp->app_index; + + return 0; + +failed: + + for (i = clib_max (n_fds_used - 1, 0); i < mp->n_fds; i++) + close (fds[i]); + + return -1; +} + +static int +vcl_api_send_attach (clib_socket_t * cs) +{ + app_sapi_msg_t msg = { 0 }; + app_sapi_attach_msg_t *mp = &msg.attach; + u8 app_is_proxy, tls_engine; + clib_error_t *err; + + app_is_proxy = (vcm->cfg.app_proxy_transport_tcp || + vcm->cfg.app_proxy_transport_udp); + tls_engine = CRYPTO_ENGINE_OPENSSL; + + clib_memcpy (&mp->name, vcm->app_name, vec_len (vcm->app_name)); + mp->options[APP_OPTIONS_FLAGS] = + APP_OPTIONS_FLAGS_ACCEPT_REDIRECT | APP_OPTIONS_FLAGS_ADD_SEGMENT | + (vcm->cfg.app_scope_local ? APP_OPTIONS_FLAGS_USE_LOCAL_SCOPE : 0) | + (vcm->cfg.app_scope_global ? APP_OPTIONS_FLAGS_USE_GLOBAL_SCOPE : 0) | + (app_is_proxy ? APP_OPTIONS_FLAGS_IS_PROXY : 0) | + (vcm->cfg.use_mq_eventfd ? APP_OPTIONS_FLAGS_EVT_MQ_USE_EVENTFD : 0); + mp->options[APP_OPTIONS_PROXY_TRANSPORT] = + (u64) ((vcm->cfg.app_proxy_transport_tcp ? 1 << TRANSPORT_PROTO_TCP : 0) | + (vcm->cfg.app_proxy_transport_udp ? 1 << TRANSPORT_PROTO_UDP : 0)); + mp->options[APP_OPTIONS_SEGMENT_SIZE] = vcm->cfg.segment_size; + mp->options[APP_OPTIONS_ADD_SEGMENT_SIZE] = vcm->cfg.add_segment_size; + mp->options[APP_OPTIONS_RX_FIFO_SIZE] = vcm->cfg.rx_fifo_size; + mp->options[APP_OPTIONS_TX_FIFO_SIZE] = vcm->cfg.tx_fifo_size; + mp->options[APP_OPTIONS_PREALLOC_FIFO_PAIRS] = + vcm->cfg.preallocated_fifo_pairs; + mp->options[APP_OPTIONS_EVT_QUEUE_SIZE] = vcm->cfg.event_queue_size; + mp->options[APP_OPTIONS_TLS_ENGINE] = tls_engine; + + msg.type = APP_SAPI_MSG_TYPE_ATTACH; + err = clib_socket_sendmsg (cs, &msg, sizeof (msg), 0, 0); + if (err) + { + clib_error_report (err); + return -1; + } + + return 0; +} + +int +vcl_sapi_attach (void) +{ + vcl_worker_t *wrk = vcl_worker_get_current (); + app_sapi_msg_t _rmp, *rmp = &_rmp; + clib_error_t *err; + clib_socket_t *cs; + int fds[SESSION_N_FD_TYPE]; + + /* + * Init client socket and send attach + */ + if (vcl_api_connect_app_socket (wrk)) + return -1; + + cs = &wrk->app_api_sock; + if (vcl_api_send_attach (cs)) + return -1; + + /* + * Wait for attach reply + */ + err = clib_socket_recvmsg (cs, rmp, sizeof (*rmp), fds, ARRAY_LEN (fds)); + if (err) + { + clib_error_report (err); + return -1; + } + + if (rmp->type != APP_SAPI_MSG_TYPE_ATTACH_REPLY) + return -1; + + return vcl_api_attach_reply_handler (&rmp->attach_reply, fds); +} + +static int +vcl_api_add_del_worker_reply_handler (app_sapi_worker_add_del_reply_msg_t * + mp, int *fds) +{ + int n_fds = 0, i, rv; + u64 segment_handle; + vcl_worker_t *wrk; + + if (mp->retval) + { + VDBG (0, "add/del worker failed: %U", format_session_error, mp->retval); + goto failed; + } + + if (!mp->is_add) + goto failed; + + wrk = vcl_worker_get_current (); + wrk->vpp_wrk_index = mp->wrk_index; + wrk->app_event_queue = uword_to_pointer (mp->app_event_queue_address, + svm_msg_q_t *); + wrk->ctrl_mq = vcm->ctrl_mq; + + segment_handle = mp->segment_handle; + if (segment_handle == VCL_INVALID_SEGMENT_HANDLE) + { + clib_warning ("invalid segment handle"); + goto failed; + } + + if (!mp->n_fds) + goto failed; + + if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT) + if (vcl_segment_attach (vcl_vpp_worker_segment_handle (wrk->wrk_index), + "vpp-worker-seg", SSVM_SEGMENT_MEMFD, + fds[n_fds++])) + goto failed; + + if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT) + { + u8 *segment_name = format (0, "memfd-%ld%c", segment_handle, 0); + rv = vcl_segment_attach (segment_handle, (char *) segment_name, + SSVM_SEGMENT_MEMFD, fds[n_fds++]); + vec_free (segment_name); + if (rv != 0) + goto failed; + } + + if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD) + { + svm_msg_q_set_consumer_eventfd (wrk->app_event_queue, fds[n_fds]); + vcl_mq_epoll_add_evfd (wrk, wrk->app_event_queue); + n_fds++; + } + + VDBG (0, "worker %u vpp-worker %u added", wrk->wrk_index, + wrk->vpp_wrk_index); + + return 0; + +failed: + for (i = clib_max (n_fds - 1, 0); i < mp->n_fds; i++) + close (fds[i]); + + return -1; +} + +int +vcl_sapi_app_worker_add (void) +{ + vcl_worker_t *wrk = vcl_worker_get_current (); + app_sapi_worker_add_del_msg_t *mp; + app_sapi_msg_t _rmp, *rmp = &_rmp; + app_sapi_msg_t msg = { 0 }; + int fds[SESSION_N_FD_TYPE]; + clib_error_t *err; + clib_socket_t *cs; + + /* Connect to socket api */ + if (vcl_api_connect_app_socket (wrk)) + return -1; + + /* + * Send add worker + */ + cs = &wrk->app_api_sock; + + msg.type = APP_SAPI_MSG_TYPE_ADD_DEL_WORKER; + mp = &msg.worker_add_del; + mp->app_index = vcm->app_index; + mp->is_add = 1; + + err = clib_socket_sendmsg (cs, &msg, sizeof (msg), 0, 0); + if (err) + { + clib_error_report (err); + return -1; + } + + /* + * Wait for reply and process it + */ + err = clib_socket_recvmsg (cs, rmp, sizeof (*rmp), fds, ARRAY_LEN (fds)); + if (err) + { + clib_error_report (err); + return -1; + } + + if (rmp->type != APP_SAPI_MSG_TYPE_ADD_DEL_WORKER_REPLY) + { + clib_warning ("unexpected reply type %u", rmp->type); + return -1; + } + + return vcl_api_add_del_worker_reply_handler (&rmp->worker_add_del_reply, + fds); +} + +void +vcl_sapi_app_worker_del (vcl_worker_t * wrk) +{ + app_sapi_worker_add_del_msg_t *mp; + app_sapi_msg_t msg = { 0 }; + clib_error_t *err; + clib_socket_t *cs; + + cs = &wrk->app_api_sock; + + msg.type = APP_SAPI_MSG_TYPE_ADD_DEL_WORKER; + mp = &msg.worker_add_del; + mp->app_index = vcm->app_index; + mp->wrk_index = wrk->vpp_wrk_index; + mp->is_add = 0; + + err = clib_socket_sendmsg (cs, &msg, sizeof (msg), 0, 0); + if (err) + clib_error_report (err); + clib_socket_close (cs); +} + +void +vcl_sapi_detach (vcl_worker_t * wrk) +{ + clib_socket_t *cs = &wrk->app_api_sock; + clib_socket_close (cs); +} + +int +vcl_sapi_recv_fds (vcl_worker_t * wrk, int *fds, int n_fds) +{ + app_sapi_msg_t _msg, *msg = &_msg; + clib_socket_t *cs; + clib_error_t *err; + + cs = &wrk->app_api_sock; + + err = clib_socket_recvmsg (cs, msg, sizeof (*msg), fds, n_fds); + if (err) + { + clib_error_report (err); + return -1; + } + if (msg->type != APP_SAPI_MSG_TYPE_SEND_FDS) + return -1; + + return 0; +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index 0f79f88423f..dcf8ba2aa72 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -863,6 +863,16 @@ vcl_session_worker_update_reply_handler (vcl_worker_t * wrk, void *data) s->vpp_handle, wrk->wrk_index); } +static int +vcl_api_recv_fd (vcl_worker_t * wrk, int *fds, int n_fds) +{ + + if (vcm->cfg.vpp_app_socket_api) + return vcl_sapi_recv_fds (wrk, fds, n_fds); + + return vcl_bapi_recv_fds (wrk, fds, n_fds); +} + static void vcl_session_app_add_segment_handler (vcl_worker_t * wrk, void *data) { @@ -875,7 +885,7 @@ vcl_session_app_add_segment_handler (vcl_worker_t * wrk, void *data) if (msg->fd_flags) { - vl_socket_client_recv_fd_msg2 (&wrk->bapi_sock_ctx, &fd, 1, 5); + vcl_api_recv_fd (wrk, &fd, 1); seg_type = SSVM_SEGMENT_MEMFD; } @@ -1178,6 +1188,26 @@ vppcom_app_exit (void) vcl_elog_stop (vcm); } +static int +vcl_api_attach (void) +{ + if (vcm->cfg.vpp_app_socket_api) + return vcl_sapi_attach (); + + return vcl_bapi_attach (); +} + +static void +vcl_api_detach (vcl_worker_t * wrk) +{ + vcl_send_app_detach (wrk); + + if (vcm->cfg.vpp_app_socket_api) + return vcl_sapi_detach (wrk); + + return vcl_bapi_disconnect_from_vpp (); +} + /* * VPPCOM Public API functions */ @@ -1211,7 +1241,7 @@ vppcom_app_create (const char *app_name) /* Allocate default worker */ vcl_worker_alloc_and_init (); - if ((rv = vcl_bapi_attach ())) + if ((rv = vcl_api_attach ())) return rv; VDBG (0, "app_name '%s', my_client_index %d (0x%x)", app_name, @@ -1241,8 +1271,7 @@ vppcom_app_destroy (void) })); /* *INDENT-ON* */ - vcl_send_app_detach (current_wrk); - vcl_bapi_disconnect_from_vpp (); + vcl_api_detach (current_wrk); vcl_worker_cleanup (current_wrk, 0 /* notify vpp */ ); vcl_elog_stop (vcm); |