From d85de68ec3f72888099172fffd45a90a43018155 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Thu, 29 Nov 2018 17:02:29 -0800 Subject: vcl: wait for segments with segment handle Instead of waiting for notification from binary api. Change-Id: I5ecab857d6bcdbed62d6bb06709570c4cf6b19ea Signed-off-by: Florin Coras --- src/vcl/vcl_bapi.c | 92 ++++++++++++++++++++++++++++++++++++--------------- src/vcl/vcl_private.c | 30 +++++++++++++++++ src/vcl/vcl_private.h | 17 ++++++++-- src/vcl/vppcom.c | 68 +++++++++++++++++-------------------- 4 files changed, 140 insertions(+), 67 deletions(-) (limited to 'src/vcl') diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c index 7f3a5741af3..c6e7fdf369d 100644 --- a/src/vcl/vcl_bapi.c +++ b/src/vcl/vcl_bapi.c @@ -62,7 +62,8 @@ static void } static int -ssvm_segment_attach (char *name, ssvm_segment_type_t type, int fd) +vcl_segment_attach (u64 segment_handle, char *name, ssvm_segment_type_t type, + int fd) { svm_fifo_segment_create_args_t _a, *a = &_a; int rv; @@ -79,15 +80,37 @@ ssvm_segment_attach (char *name, ssvm_segment_type_t type, int fd) clib_warning ("svm_fifo_segment_attach ('%s') failed", name); return rv; } + vcl_segment_table_add (segment_handle, a->new_segment_indices[0]); vec_reset_length (a->new_segment_indices); return 0; } +static void +vcl_segment_detach (u64 segment_handle) +{ + svm_fifo_segment_private_t *segment; + u32 segment_index; + + segment_index = vcl_segment_table_lookup (segment_handle); + if (segment_index == (u32) ~ 0) + return; + segment = svm_fifo_segment_get_segment (segment_index); + svm_fifo_segment_delete (segment); + vcl_segment_table_del (segment_handle); +} + +static u64 +vcl_vpp_worker_segment_handle (u32 wrk_index) +{ + return (VCL_INVALID_SEGMENT_HANDLE - wrk_index - 1); +} + static void vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t * mp) { vcl_worker_t *wrk = vcl_worker_get (0); + u64 segment_handle; u32 n_fds = 0; int *fds = 0; @@ -100,19 +123,27 @@ vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t * wrk->app_event_queue = uword_to_pointer (mp->app_event_queue_address, svm_msg_q_t *); + segment_handle = clib_net_to_host_u64 (mp->segment_handle); + if (segment_handle == VCL_INVALID_SEGMENT_HANDLE) + { + clib_warning ("invalid segment handle"); + return; + } + if (mp->n_fds) { vec_validate (fds, mp->n_fds); vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5); if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT) - if (ssvm_segment_attach ("vpp-mq-seg", SSVM_SEGMENT_MEMFD, - fds[n_fds++])) + if (vcl_segment_attach (vcl_vpp_worker_segment_handle (0), + "vpp-mq-seg", SSVM_SEGMENT_MEMFD, + fds[n_fds++])) return; if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT) - if (ssvm_segment_attach ((char *) mp->segment_name, - SSVM_SEGMENT_MEMFD, fds[n_fds++])) + if (vcl_segment_attach (segment_handle, (char *) mp->segment_name, + SSVM_SEGMENT_MEMFD, fds[n_fds++])) return; if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD) @@ -126,8 +157,8 @@ vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t * } else { - if (ssvm_segment_attach ((char *) mp->segment_name, SSVM_SEGMENT_SHM, - -1)) + if (vcl_segment_attach (segment_handle, (char *) mp->segment_name, + SSVM_SEGMENT_SHM, -1)) return; } @@ -140,6 +171,7 @@ vl_api_app_worker_add_del_reply_t_handler (vl_api_app_worker_add_del_reply_t * mp) { int n_fds = 0, *fds = 0; + u64 segment_handle; vcl_worker_t *wrk; u32 wrk_index; @@ -159,19 +191,27 @@ vl_api_app_worker_add_del_reply_t_handler (vl_api_app_worker_add_del_reply_t * wrk->app_event_queue = uword_to_pointer (mp->app_event_queue_address, svm_msg_q_t *); + segment_handle = clib_net_to_host_u64 (mp->segment_handle); + if (segment_handle == VCL_INVALID_SEGMENT_HANDLE) + { + clib_warning ("invalid segment handle"); + return; + } + if (mp->n_fds) { vec_validate (fds, mp->n_fds); vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5); if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT) - if (ssvm_segment_attach ("vpp-worker-seg", SSVM_SEGMENT_MEMFD, - fds[n_fds++])) + if (vcl_segment_attach + (vcl_vpp_worker_segment_handle (wrk->wrk_index), "vpp-worker-seg", + SSVM_SEGMENT_MEMFD, fds[n_fds++])) goto failed; if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT) - if (ssvm_segment_attach ((char *) mp->segment_name, - SSVM_SEGMENT_MEMFD, fds[n_fds++])) + if (vcl_segment_attach (segment_handle, (char *) mp->segment_name, + SSVM_SEGMENT_MEMFD, fds[n_fds++])) goto failed; if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD) @@ -185,8 +225,8 @@ vl_api_app_worker_add_del_reply_t_handler (vl_api_app_worker_add_del_reply_t * } else { - if (ssvm_segment_attach ((char *) mp->segment_name, SSVM_SEGMENT_SHM, - -1)) + if (vcl_segment_attach (segment_handle, (char *) mp->segment_name, + SSVM_SEGMENT_SHM, -1)) goto failed; } vcm->app_state = STATE_APP_READY; @@ -212,18 +252,24 @@ static void vl_api_map_another_segment_t_handler (vl_api_map_another_segment_t * mp) { ssvm_segment_type_t seg_type = SSVM_SEGMENT_SHM; + u64 segment_handle; int fd = -1; - vcm->mounting_segment = 1; - if (mp->fd_flags) { vl_socket_client_recv_fd_msg (&fd, 1, 5); seg_type = SSVM_SEGMENT_MEMFD; } - if (PREDICT_FALSE (ssvm_segment_attach ((char *) mp->segment_name, - seg_type, fd))) + segment_handle = clib_net_to_host_u64 (mp->segment_handle); + if (segment_handle == VCL_INVALID_SEGMENT_HANDLE) + { + clib_warning ("invalid segment handle"); + return; + } + + if (vcl_segment_attach (segment_handle, (char *) mp->segment_name, + seg_type, fd)) { clib_warning ("VCL<%d>: svm_fifo_segment_attach ('%s') failed", getpid (), mp->segment_name); @@ -232,20 +278,14 @@ vl_api_map_another_segment_t_handler (vl_api_map_another_segment_t * mp) VDBG (1, "VCL<%d>: mapped new segment '%s' size %d", getpid (), mp->segment_name, mp->segment_size); - vcm->mounting_segment = 0; } static void vl_api_unmap_segment_t_handler (vl_api_unmap_segment_t * mp) { - -/* - * XXX Need segment_name to session_id hash, - * XXX - have sessionID by handle hash currently - */ - - VDBG (1, "Unmapped segment '%s'", - clib_net_to_host_u64 (mp->segment_handle)); + u64 segment_handle = clib_net_to_host_u64 (mp->segment_handle); + vcl_segment_detach (segment_handle); + VDBG (1, "Unmapped segment: %d", segment_handle); } static void diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c index 86dccfe1cc9..76ae0e7719f 100644 --- a/src/vcl/vcl_private.c +++ b/src/vcl/vcl_private.c @@ -442,6 +442,36 @@ vcl_session_get_refcnt (vcl_session_t * s) return 0; } +void +vcl_segment_table_add (u64 segment_handle, u32 svm_segment_index) +{ + clib_rwlock_writer_lock (&vcm->segment_table_lock); + hash_set (vcm->segment_table, segment_handle, svm_segment_index); + clib_rwlock_writer_unlock (&vcm->segment_table_lock); +} + +u32 +vcl_segment_table_lookup (u64 segment_handle) +{ + uword *seg_indexp; + + clib_rwlock_reader_lock (&vcm->segment_table_lock); + seg_indexp = hash_get (vcm->segment_table, segment_handle); + clib_rwlock_reader_unlock (&vcm->segment_table_lock); + + if (!seg_indexp) + return VCL_INVALID_SEGMENT_INDEX; + return ((u32) * seg_indexp); +} + +void +vcl_segment_table_del (u64 segment_handle) +{ + clib_rwlock_writer_lock (&vcm->segment_table_lock); + hash_unset (vcm->segment_table, segment_handle); + clib_rwlock_writer_unlock (&vcm->segment_table_lock); +} + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h index 0420322d3f4..6ca0471e965 100644 --- a/src/vcl/vcl_private.h +++ b/src/vcl/vcl_private.h @@ -309,9 +309,6 @@ typedef struct vppcom_main_t_ /** VCL configuration */ vppcom_cfg_t cfg; - /** Flag indicating that a new segment is being mounted */ - volatile u32 mounting_segment; - volatile u32 forking; /** Workers */ @@ -320,7 +317,15 @@ typedef struct vppcom_main_t_ /** Lock to protect worker registrations */ clib_spinlock_t workers_lock; + /** Pool of shared sessions */ vcl_shared_session_t *shared_sessions; + + /** Lock to protect segment hash table */ + clib_rwlock_t segment_table_lock; + + /** Mapped segments table */ + uword *segment_table; + #ifdef VCL_ELOG /* VPP Event-logger */ elog_main_t elog_main; @@ -335,6 +340,8 @@ typedef struct vppcom_main_t_ extern vppcom_main_t *vcm; #define VCL_INVALID_SESSION_INDEX ((u32)~0) +#define VCL_INVALID_SEGMENT_INDEX ((u32)~0) +#define VCL_INVALID_SEGMENT_HANDLE ((u64)~0) static inline vcl_session_t * vcl_session_alloc (vcl_worker_t * wrk) @@ -502,6 +509,10 @@ void vcl_worker_share_sessions (u32 parent_wrk_index); int vcl_worker_unshare_session (vcl_worker_t * wrk, vcl_session_t * s); int vcl_session_get_refcnt (vcl_session_t * s); +void vcl_segment_table_add (u64 segment_handle, u32 svm_segment_index); +u32 vcl_segment_table_lookup (u64 segment_handle); +void vcl_segment_table_del (u64 segment_handle); + static inline vcl_worker_t * vcl_worker_get (u32 wrk_index) { diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index 195e6cbc602..34b663b9831 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -22,46 +22,26 @@ __thread uword __vcl_worker_index = ~0; -static u8 not_ready; -void -sigsegv_signal (int signum) +static int +vcl_wait_for_segment (u64 segment_handle) { - not_ready = 1; -} + vcl_worker_t *wrk = vcl_worker_get_current (); + u32 wait_for_seconds = 10, segment_index; + f64 timeout; -static void -vcl_wait_for_memory (void *mem) -{ - u8 __clib_unused test; - if (vcm->mounting_segment) - { - while (vcm->mounting_segment) - ; - return; - } - if (1 || vcm->debug) - { - usleep (1e5); - return; - } - if (signal (SIGSEGV, sigsegv_signal)) - { - perror ("signal()"); - return; - } - not_ready = 0; + if (segment_handle == VCL_INVALID_SEGMENT_HANDLE) + return 1; -again: - test = *(u8 *) mem; - if (not_ready) + timeout = clib_time_now (&wrk->clib_time) + wait_for_seconds; + while (clib_time_now (&wrk->clib_time) < timeout) { - not_ready = 0; - usleep (1); - goto again; + segment_index = vcl_segment_table_lookup (segment_handle); + if (segment_index != VCL_INVALID_SEGMENT_INDEX) + return 0; + usleep (10); } - - signal (SIGSEGV, SIG_DFL); + return 1; } const char * @@ -281,7 +261,12 @@ vcl_session_accepted_handler (vcl_worker_t * wrk, session_accepted_msg_t * mp) svm_msg_q_t *); session->our_evt_q = uword_to_pointer (mp->server_event_queue_address, svm_msg_q_t *); - vcl_wait_for_memory (session->vpp_evt_q); + if (vcl_wait_for_segment (mp->segment_handle)) + { + clib_warning ("segment for session %u couldn't be mounted!", + session->session_index); + return VCL_INVALID_SESSION_INDEX; + } rx_fifo->master_session_index = session->session_index; tx_fifo->master_session_index = session->session_index; rx_fifo->master_thread_index = vcl_get_worker_index (); @@ -360,7 +345,13 @@ vcl_session_connected_handler (vcl_worker_t * wrk, rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *); tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *); - vcl_wait_for_memory (rx_fifo); + if (vcl_wait_for_segment (mp->segment_handle)) + { + clib_warning ("segment for session %u couldn't be mounted!", + session->session_index); + return VCL_INVALID_SESSION_INDEX; + } + rx_fifo->client_session_index = session_index; tx_fifo->client_session_index = session_index; rx_fifo->client_thread_index = vcl_get_worker_index (); @@ -436,8 +427,8 @@ vcl_session_bound_handler (vcl_worker_t * wrk, session_bound_msg_t * mp) session = vcl_session_get (wrk, sid); if (mp->retval) { - VDBG (0, "VCL<%d>: ERROR: vpp handle 0x%llx, sid %u: bind failed: %U", - getpid (), mp->handle, sid, format_api_error, ntohl (mp->retval)); + VERR ("vpp handle 0x%llx, sid %u: bind failed: %U", mp->handle, sid, + format_api_error, mp->retval); if (session) { session->session_state = STATE_FAILED; @@ -780,6 +771,7 @@ vppcom_app_create (char *app_name) 20 /* timeout in secs */ ); pool_alloc (vcm->workers, vcl_cfg->max_workers); clib_spinlock_init (&vcm->workers_lock); + clib_rwlock_init (&vcm->segment_table_lock); pthread_atfork (NULL, vcl_app_fork_parent_handler, vcl_app_fork_child_handler); -- cgit 1.2.3-korg