From 458089bbad9cf5bef6cf8119f23fc44e66b36ad3 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Wed, 21 Aug 2019 16:20:44 -0700 Subject: session: move ctrl messages from bapi to mq Type:refactor Moves connect, disconnect, bind, unbind and app detach to message queue from binary api. Simplifies app/vcl interaction with the session layer since all session control messages are now handled over the mq. Add/del segment messages require internal C api changes which affect all builtin applications. They'll be moved in a different patch and might not be back portable to 19.08. Change-Id: I93f6d18e551b024effa75d47f5ff25f23ba8aff5 Signed-off-by: Florin Coras --- src/vcl/vcl_bapi.c | 159 +++++--------------------------------------------- src/vcl/vcl_locked.c | 2 +- src/vcl/vcl_private.c | 6 ++ src/vcl/vcl_private.h | 9 ++- src/vcl/vppcom.c | 124 +++++++++++++++++++++++++++++++++------ 5 files changed, 133 insertions(+), 167 deletions(-) (limited to 'src/vcl') diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c index de64809d52a..cdfc2869edc 100644 --- a/src/vcl/vcl_bapi.c +++ b/src/vcl/vcl_bapi.c @@ -108,10 +108,10 @@ vcl_vpp_worker_segment_handle (u32 wrk_index) } static void -vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t * - mp) +vl_api_app_attach_reply_t_handler (vl_api_app_attach_reply_t * mp) { vcl_worker_t *wrk = vcl_worker_get (0); + svm_msg_q_t *ctrl_mq; u64 segment_handle; int *fds = 0, i; u32 n_fds = 0; @@ -122,8 +122,11 @@ vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t * goto failed; } - wrk->app_event_queue = uword_to_pointer (mp->app_event_queue_address, - svm_msg_q_t *); + wrk->app_event_queue = uword_to_pointer (mp->app_mq, svm_msg_q_t *); + ctrl_mq = uword_to_pointer (mp->vpp_ctrl_mq, svm_msg_q_t *); + vec_validate (wrk->vpp_event_queues, mp->vpp_ctrl_mq_thread); + wrk->vpp_event_queues[mp->vpp_ctrl_mq_thread] = ctrl_mq; + wrk->ctrl_mq = ctrl_mq; segment_handle = clib_net_to_host_u64 (mp->segment_handle); if (segment_handle == VCL_INVALID_SEGMENT_HANDLE) { @@ -253,17 +256,6 @@ failed: vec_free (fds); } -static void -vl_api_application_detach_reply_t_handler (vl_api_application_detach_reply_t * - mp) -{ - if (mp->retval) - clib_warning ("VCL<%d>: detach failed: %U", getpid (), format_api_error, - ntohl (mp->retval)); - - vcm->app_state = STATE_APP_ENABLED; -} - static void vl_api_map_another_segment_t_handler (vl_api_map_another_segment_t * mp) { @@ -304,51 +296,13 @@ vl_api_unmap_segment_t_handler (vl_api_unmap_segment_t * mp) VDBG (1, "Unmapped segment: %d", segment_handle); } -static void -vl_api_bind_sock_reply_t_handler (vl_api_bind_sock_reply_t * mp) -{ - /* Expecting a similar message on mq. So ignore this */ - VDBG (0, "bapi bind retval: %u!", mp->retval); -} - -static void -vl_api_unbind_sock_reply_t_handler (vl_api_unbind_sock_reply_t * mp) -{ - if (mp->retval) - VDBG (0, "ERROR: sid %u: unbind failed: %U", mp->context, - format_api_error, ntohl (mp->retval)); - - VDBG (1, "sid %u: unbind succeeded!", mp->context); - -} - -static void -vl_api_disconnect_session_reply_t_handler (vl_api_disconnect_session_reply_t * - mp) -{ - if (mp->retval) - VDBG (0, "ERROR: sid %u: disconnect failed: %U", mp->context, - format_api_error, ntohl (mp->retval)); -} - -static void -vl_api_connect_sock_reply_t_handler (vl_api_connect_sock_reply_t * mp) -{ - if (mp->retval) - VDBG (0, "ERROR: connect failed: %U", format_api_error, - ntohl (mp->retval)); -} - static void vl_api_application_tls_cert_add_reply_t_handler (vl_api_application_tls_cert_add_reply_t * mp) { if (mp->retval) - { - clib_warning ("VCL<%d>: add cert failed: %U", getpid (), - format_api_error, ntohl (mp->retval)); - return; - } + VDBG (0, "add cert failed: %U", format_api_error, ntohl (mp->retval)); + vcm->app_state = STATE_APP_READY; } static void @@ -356,22 +310,13 @@ static void (vl_api_application_tls_key_add_reply_t * mp) { if (mp->retval) - { - clib_warning ("VCL<%d>: add key failed: %U", getpid (), - format_api_error, ntohl (mp->retval)); - return; - } - + VDBG (0, "add key failed: %U", format_api_error, ntohl (mp->retval)); + vcm->app_state = STATE_APP_READY; } #define foreach_sock_msg \ _(SESSION_ENABLE_DISABLE_REPLY, session_enable_disable_reply) \ -_(BIND_SOCK_REPLY, bind_sock_reply) \ -_(UNBIND_SOCK_REPLY, unbind_sock_reply) \ -_(CONNECT_SOCK_REPLY, connect_sock_reply) \ -_(DISCONNECT_SESSION_REPLY, disconnect_session_reply) \ -_(APPLICATION_ATTACH_REPLY, application_attach_reply) \ -_(APPLICATION_DETACH_REPLY, application_detach_reply) \ +_(APP_ATTACH_REPLY, app_attach_reply) \ _(APPLICATION_TLS_CERT_ADD_REPLY, application_tls_cert_add_reply) \ _(APPLICATION_TLS_KEY_ADD_REPLY, application_tls_key_add_reply) \ _(MAP_ANOTHER_SEGMENT, map_another_segment) \ @@ -414,7 +359,7 @@ void vppcom_app_send_attach (void) { vcl_worker_t *wrk = vcl_worker_get_current (); - vl_api_application_attach_t *bmp; + vl_api_app_attach_t *bmp; u8 nsid_len = vec_len (vcm->cfg.namespace_id); u8 app_is_proxy = (vcm->cfg.app_proxy_transport_tcp || vcm->cfg.app_proxy_transport_udp); @@ -422,7 +367,7 @@ vppcom_app_send_attach (void) bmp = vl_msg_api_alloc (sizeof (*bmp)); memset (bmp, 0, sizeof (*bmp)); - bmp->_vl_msg_id = ntohs (VL_API_APPLICATION_ATTACH); + bmp->_vl_msg_id = ntohs (VL_API_APP_ATTACH); bmp->client_index = wrk->my_client_index; bmp->context = htonl (0xfeedface); bmp->options[APP_OPTIONS_FLAGS] = @@ -504,80 +449,6 @@ vcl_send_child_worker_del (vcl_worker_t * child_wrk) vl_msg_api_send_shmem (wrk->vl_input_queue, (u8 *) & mp); } -void -vppcom_send_connect_sock (vcl_session_t * session) -{ - vcl_worker_t *wrk = vcl_worker_get_current (); - vl_api_connect_sock_t *cmp; - - cmp = vl_msg_api_alloc (sizeof (*cmp)); - memset (cmp, 0, sizeof (*cmp)); - cmp->_vl_msg_id = ntohs (VL_API_CONNECT_SOCK); - cmp->client_index = wrk->my_client_index; - cmp->context = session->session_index; - cmp->wrk_index = wrk->vpp_wrk_index; - cmp->is_ip4 = session->transport.is_ip4; - cmp->parent_handle = session->parent_handle; - clib_memcpy_fast (cmp->ip, &session->transport.rmt_ip, sizeof (cmp->ip)); - cmp->port = session->transport.rmt_port; - cmp->proto = session->session_type; - vl_msg_api_send_shmem (wrk->vl_input_queue, (u8 *) & cmp); -} - -void -vppcom_send_disconnect_session (u64 vpp_handle) -{ - vcl_worker_t *wrk = vcl_worker_get_current (); - vl_api_disconnect_session_t *dmp; - - dmp = vl_msg_api_alloc (sizeof (*dmp)); - memset (dmp, 0, sizeof (*dmp)); - dmp->_vl_msg_id = ntohs (VL_API_DISCONNECT_SESSION); - dmp->client_index = wrk->my_client_index; - dmp->handle = vpp_handle; - vl_msg_api_send_shmem (wrk->vl_input_queue, (u8 *) & dmp); -} - -/* VPP combines bind and listen as one operation. VCL manages the separation - * of bind and listen locally via vppcom_session_bind() and - * vppcom_session_listen() */ -void -vppcom_send_bind_sock (vcl_session_t * session) -{ - vcl_worker_t *wrk = vcl_worker_get_current (); - vl_api_bind_sock_t *bmp; - - /* Assumes caller has acquired spinlock: vcm->sessions_lockp */ - bmp = vl_msg_api_alloc (sizeof (*bmp)); - memset (bmp, 0, sizeof (*bmp)); - - bmp->_vl_msg_id = ntohs (VL_API_BIND_SOCK); - bmp->client_index = wrk->my_client_index; - bmp->context = session->session_index; - bmp->wrk_index = wrk->vpp_wrk_index; - bmp->is_ip4 = session->transport.is_ip4; - clib_memcpy_fast (bmp->ip, &session->transport.lcl_ip, sizeof (bmp->ip)); - bmp->port = session->transport.lcl_port; - bmp->proto = session->session_type; - vl_msg_api_send_shmem (wrk->vl_input_queue, (u8 *) & bmp); -} - -void -vppcom_send_unbind_sock (vcl_worker_t * wrk, u64 vpp_handle) -{ - vl_api_unbind_sock_t *ump; - - ump = vl_msg_api_alloc (sizeof (*ump)); - memset (ump, 0, sizeof (*ump)); - - ump->_vl_msg_id = ntohs (VL_API_UNBIND_SOCK); - ump->client_index = wrk->my_client_index; - ump->wrk_index = wrk->vpp_wrk_index; - ump->handle = vpp_handle; - ump->context = wrk->wrk_index; - vl_msg_api_send_shmem (wrk->vl_input_queue, (u8 *) & ump); -} - void vppcom_send_application_tls_cert_add (vcl_session_t * session, char *cert, u32 cert_len) @@ -593,7 +464,6 @@ vppcom_send_application_tls_cert_add (vcl_session_t * session, char *cert, cert_mp->cert_len = clib_host_to_net_u16 (cert_len); clib_memcpy_fast (cert_mp->cert, cert, cert_len); vl_msg_api_send_shmem (wrk->vl_input_queue, (u8 *) & cert_mp); - } void @@ -611,7 +481,6 @@ vppcom_send_application_tls_key_add (vcl_session_t * session, char *key, key_mp->key_len = clib_host_to_net_u16 (key_len); clib_memcpy_fast (key_mp->key, key, key_len); vl_msg_api_send_shmem (wrk->vl_input_queue, (u8 *) & key_mp); - } u32 diff --git a/src/vcl/vcl_locked.c b/src/vcl/vcl_locked.c index 3b6817c6423..d69d391d626 100644 --- a/src/vcl/vcl_locked.c +++ b/src/vcl/vcl_locked.c @@ -311,7 +311,7 @@ vls_listener_wrk_stop_listen (vcl_locked_session_t * vls, u32 wrk_index) s = vcl_session_get (wrk, vls->session_index); if (s->session_state != STATE_LISTEN) return; - vppcom_send_unbind_sock (wrk, s->vpp_handle); + vcl_send_session_unlisten (wrk, s); s->session_state = STATE_LISTEN_NO_MQ; vls_listener_wrk_set (vls, wrk_index, 0 /* is_active */ ); } diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c index 13794ea4f9c..f431396dd93 100644 --- a/src/vcl/vcl_private.c +++ b/src/vcl/vcl_private.c @@ -276,6 +276,12 @@ vcl_worker_set_bapi (void) return -1; } +svm_msg_q_t * +vcl_worker_ctrl_mq (vcl_worker_t * wrk) +{ + return wrk->ctrl_mq; +} + void vcl_segment_table_add (u64 segment_handle, u32 svm_segment_index) { diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h index 43c8ec304ef..cd2544c2559 100644 --- a/src/vcl/vcl_private.h +++ b/src/vcl/vcl_private.h @@ -57,6 +57,7 @@ typedef enum STATE_APP_ENABLED, STATE_APP_ATTACHED, STATE_APP_ADDING_WORKER, + STATE_APP_ADDING_TLS_DATA, STATE_APP_FAILED, STATE_APP_READY } app_state_t; @@ -251,6 +252,9 @@ typedef struct vcl_worker_ /** VPP binary api input queue */ svm_queue_t *vl_input_queue; + /** VPP mq to be used for exchanging control messages */ + svm_msg_q_t *ctrl_mq; + /** Message queues epoll fd. Initialized only if using mqs with eventfds */ int mqs_epfd; @@ -547,6 +551,7 @@ vcl_worker_t *vcl_worker_alloc_and_init (void); void vcl_worker_cleanup (vcl_worker_t * wrk, u8 notify_vpp); int vcl_worker_register_with_vpp (void); int vcl_worker_set_bapi (void); +svm_msg_q_t *vcl_worker_ctrl_mq (vcl_worker_t * wrk); void vcl_flush_mq_events (void); void vcl_cleanup_bapi (void); @@ -603,10 +608,8 @@ void vppcom_init_error_string_table (void); void vppcom_send_session_enable_disable (u8 is_enable); void vppcom_app_send_attach (void); void vppcom_app_send_detach (void); -void vppcom_send_connect_sock (vcl_session_t * session); +void vcl_send_session_unlisten (vcl_worker_t * wrk, vcl_session_t * s); void vppcom_send_disconnect_session (u64 vpp_handle); -void vppcom_send_bind_sock (vcl_session_t * session); -void vppcom_send_unbind_sock (vcl_worker_t * wrk, u64 vpp_handle); void vppcom_api_hookup (void); void vppcom_send_application_tls_cert_add (vcl_session_t * session, char *cert, u32 cert_len); diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index 0060922fea6..f56c02b6d9d 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -197,6 +197,98 @@ format_ip46_address (u8 * s, va_list * args) * VPPCOM Utility Functions */ +static void +vcl_send_session_listen (vcl_worker_t * wrk, vcl_session_t * s) +{ + app_session_evt_t _app_evt, *app_evt = &_app_evt; + session_listen_msg_t *mp; + svm_msg_q_t *mq; + + mq = vcl_worker_ctrl_mq (wrk); + app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_LISTEN); + mp = (session_listen_msg_t *) app_evt->evt->data; + memset (mp, 0, sizeof (*mp)); + mp->client_index = wrk->my_client_index; + mp->context = s->session_index; + mp->wrk_index = wrk->vpp_wrk_index; + mp->is_ip4 = s->transport.is_ip4; + clib_memcpy_fast (&mp->ip, &s->transport.lcl_ip, sizeof (mp->ip)); + mp->port = s->transport.lcl_port; + mp->proto = s->session_type; + app_send_ctrl_evt_to_vpp (mq, app_evt); +} + +static void +vcl_send_session_connect (vcl_worker_t * wrk, vcl_session_t * s) +{ + app_session_evt_t _app_evt, *app_evt = &_app_evt; + session_connect_msg_t *mp; + svm_msg_q_t *mq; + + mq = vcl_worker_ctrl_mq (wrk); + app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_CONNECT); + mp = (session_connect_msg_t *) app_evt->evt->data; + memset (mp, 0, sizeof (*mp)); + mp->client_index = wrk->my_client_index; + mp->context = s->session_index; + mp->wrk_index = wrk->vpp_wrk_index; + mp->is_ip4 = s->transport.is_ip4; + mp->parent_handle = s->parent_handle; + clib_memcpy_fast (&mp->ip, &s->transport.rmt_ip, sizeof (mp->ip)); + mp->port = s->transport.rmt_port; + mp->proto = s->session_type; + app_send_ctrl_evt_to_vpp (mq, app_evt); +} + +void +vcl_send_session_unlisten (vcl_worker_t * wrk, vcl_session_t * s) +{ + app_session_evt_t _app_evt, *app_evt = &_app_evt; + session_unlisten_msg_t *mp; + svm_msg_q_t *mq; + + mq = vcl_worker_ctrl_mq (wrk); + app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_UNLISTEN); + mp = (session_unlisten_msg_t *) app_evt->evt->data; + memset (mp, 0, sizeof (*mp)); + mp->client_index = wrk->my_client_index; + mp->wrk_index = wrk->vpp_wrk_index; + mp->handle = s->vpp_handle; + mp->context = wrk->wrk_index; + app_send_ctrl_evt_to_vpp (mq, app_evt); +} + +static void +vcl_send_session_disconnect (vcl_worker_t * wrk, vcl_session_t * s) +{ + app_session_evt_t _app_evt, *app_evt = &_app_evt; + session_disconnect_msg_t *mp; + svm_msg_q_t *mq; + + /* Send to thread that owns the session */ + mq = s->vpp_evt_q; + app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_DISCONNECT); + mp = (session_disconnect_msg_t *) app_evt->evt->data; + memset (mp, 0, sizeof (*mp)); + mp->client_index = wrk->my_client_index; + mp->handle = s->vpp_handle; + app_send_ctrl_evt_to_vpp (mq, app_evt); +} + +static void +vcl_send_app_detach (vcl_worker_t * wrk) +{ + app_session_evt_t _app_evt, *app_evt = &_app_evt; + session_app_detach_msg_t *mp; + svm_msg_q_t *mq; + + mq = vcl_worker_ctrl_mq (wrk); + app_alloc_ctrl_evt_to_vpp (mq, app_evt, SESSION_CTRL_EVT_APP_DETACH); + mp = (session_app_detach_msg_t *) app_evt->evt->data; + memset (mp, 0, sizeof (*mp)); + mp->client_index = wrk->my_client_index; + app_send_ctrl_evt_to_vpp (mq, app_evt); +} static void vcl_send_session_accepted_reply (svm_msg_q_t * mq, u32 context, @@ -827,7 +919,6 @@ vppcom_session_unbind (u32 session_handle) session_accepted_msg_t *accepted_msg; vcl_session_t *session = 0; vcl_session_msg_t *evt; - u64 vpp_handle; session = vcl_session_get_w_handle (wrk, session_handle); if (!session) @@ -845,14 +936,14 @@ vppcom_session_unbind (u32 session_handle) } clib_fifo_free (session->accept_evts_fifo); - vpp_handle = session->vpp_handle; - session->vpp_handle = ~0; - session->session_state = STATE_DISCONNECT; + vcl_send_session_unlisten (wrk, session); VDBG (1, "session %u [0x%llx]: sending unbind!", session->session_index, - vpp_handle); + session->vpp_handle); vcl_evt (VCL_EVT_UNBIND, session); - vppcom_send_unbind_sock (wrk, vpp_handle); + + session->vpp_handle = ~0; + session->session_state = STATE_DISCONNECT; return VPPCOM_OK; } @@ -894,7 +985,7 @@ vppcom_session_disconnect (u32 session_handle) { VDBG (1, "session %u [0x%llx]: sending disconnect...", session->session_index, vpp_handle); - vppcom_send_disconnect_session (vpp_handle); + vcl_send_session_disconnect (wrk, session); } if (session->listener_index != VCL_INVALID_SESSION_INDEX) @@ -1005,7 +1096,7 @@ vppcom_app_destroy (void) if (pool_elts (vcm->workers) == 1) { - vppcom_app_send_detach (); + vcl_send_app_detach (vcl_worker_get_current ()); orig_app_timeout = vcm->cfg.app_timeout; vcm->cfg.app_timeout = 2.0; rv = vcl_wait_for_app_state_change (STATE_APP_ENABLED); @@ -1220,7 +1311,7 @@ vppcom_session_listen (uint32_t listen_sh, uint32_t q_len) /* * Send listen request to vpp and wait for reply */ - vppcom_send_bind_sock (listen_session); + vcl_send_session_listen (wrk, listen_session); rv = vppcom_wait_for_session_state_change (listen_session->session_index, STATE_LISTEN, vcm->cfg.session_timeout); @@ -1256,7 +1347,8 @@ vppcom_session_tls_add_cert (uint32_t session_handle, char *cert, * Send listen request to vpp and wait for reply */ vppcom_send_application_tls_cert_add (session, cert, cert_len); - + vcm->app_state = STATE_APP_ADDING_TLS_DATA; + vcl_wait_for_app_state_change (STATE_APP_READY); return VPPCOM_OK; } @@ -1276,14 +1368,10 @@ vppcom_session_tls_add_key (uint32_t session_handle, char *key, if (key_len == 0 || key_len == ~0) return VPPCOM_EBADFD; - /* - * Send listen request to vpp and wait for reply - */ vppcom_send_application_tls_key_add (session, key, key_len); - + vcm->app_state = STATE_APP_ADDING_TLS_DATA; + vcl_wait_for_app_state_change (STATE_APP_READY); return VPPCOM_OK; - - } static int @@ -1505,7 +1593,7 @@ vppcom_session_connect (uint32_t session_handle, vppcom_endpt_t * server_ep) /* * Send connect request and wait for reply from vpp */ - vppcom_send_connect_sock (session); + vcl_send_session_connect (wrk, session); rv = vppcom_wait_for_session_state_change (session_index, STATE_CONNECT, vcm->cfg.session_timeout); @@ -1564,7 +1652,7 @@ vppcom_session_stream_connect (uint32_t session_handle, /* * Send connect request and wait for reply from vpp */ - vppcom_send_connect_sock (session); + vcl_send_session_connect (wrk, session); rv = vppcom_wait_for_session_state_change (session_index, STATE_CONNECT, vcm->cfg.session_timeout); -- cgit 1.2.3-korg