summaryrefslogtreecommitdiffstats
path: root/src/vcl/vppcom.c
diff options
context:
space:
mode:
authorFlorin Coras <fcoras@cisco.com>2018-11-26 17:01:36 -0800
committerDamjan Marion <dmarion@me.com>2018-11-29 11:27:18 +0000
commit47c40e2d944c9a29677d0542103207ba8372b66a (patch)
treee43cf5361213d119df53fb16c15cf0dca550f2d4 /src/vcl/vppcom.c
parent743ee3e12531ec3c6ba2e2add694dde3361b0e03 (diff)
vcl: basic support for apps that fork
- intercept fork and register a new worker with vpp - share sessions between parent and forked child - keep binary api state per worker Change-Id: Ib177517d661724fa042bd2d98d18e777056352a2 Signed-off-by: Florin Coras <fcoras@cisco.com>
Diffstat (limited to 'src/vcl/vppcom.c')
-rw-r--r--src/vcl/vppcom.c255
1 files changed, 138 insertions, 117 deletions
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index 589d57c3872..195e6cbc602 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -423,7 +423,7 @@ vcl_session_reset_handler (vcl_worker_t * wrk,
session->session_state = STATE_CLOSE_ON_EMPTY;
VDBG (0, "reset handle 0x%llx, sid %u ", reset_msg->handle, sid);
vcl_send_session_reset_reply (vcl_session_vpp_evt_q (wrk, session),
- vcm->my_client_index, reset_msg->handle, 0);
+ wrk->my_client_index, reset_msg->handle, 0);
return sid;
}
@@ -673,7 +673,7 @@ vppcom_session_disconnect (u32 session_handle)
if (state & STATE_CLOSE_ON_EMPTY)
{
vpp_evt_q = vcl_session_vpp_evt_q (wrk, session);
- vcl_send_session_disconnected_reply (vpp_evt_q, vcm->my_client_index,
+ vcl_send_session_disconnected_reply (vpp_evt_q, wrk->my_client_index,
vpp_handle, 0);
VDBG (1, "VCL<%d>: vpp handle 0x%llx, sid %u: sending disconnect "
"REPLY...", getpid (), vpp_handle, session_handle);
@@ -691,12 +691,14 @@ vppcom_session_disconnect (u32 session_handle)
static void
vcl_cleanup_bapi (void)
{
+ socket_client_main_t *scm = &socket_client_main;
api_main_t *am = &api_main;
am->my_client_index = ~0;
am->my_registration = 0;
am->vl_input_queue = 0;
am->msg_index_by_name_and_crc = 0;
+ scm->socket_fd = 0;
vl_client_api_unmap ();
}
@@ -705,14 +707,22 @@ void
vcl_app_fork_child_handler (void)
{
u8 *child_name;
- int rv;
-
- vcm->current_pid = getpid ();
- vcl_set_worker_index (0);
+ int rv, parent_wrk;
VDBG (0, "initializing forked child");
- child_name = format (0, "%v-child-%u%c", vcm->app_name, getpid (), 0);
+ /*
+ * Allocate worker
+ */
+ parent_wrk = vcl_get_worker_index ();
+ vcl_set_worker_index (~0);
+ if (!vcl_worker_alloc_and_init ())
+ VERR ("couldn't allocate new worker");
+
+ /*
+ * Attach to binary api
+ */
+ child_name = format (0, "%v-child-%u%c", vcm->app_name, getpid (), 0);
vcl_cleanup_bapi ();
vppcom_api_hookup ();
vcm->app_state = STATE_APP_START;
@@ -724,14 +734,23 @@ vcl_app_fork_child_handler (void)
return;
}
- vcm->app_state = STATE_APP_ADDING_WORKER;
- vcl_send_app_worker_add_del (1 /* is_add */ );
- if (vcl_wait_for_app_state_change (STATE_APP_READY))
- {
- VERR ("failed to add worker to vpp");
- return;
- }
+ /*
+ * Register worker with vpp and share sessions
+ */
+ vcl_worker_register_with_vpp ();
+ vcl_worker_share_sessions (parent_wrk);
+
VDBG (0, "forked child main worker initialized");
+ vcm->forking = 0;
+}
+
+void
+vcl_app_fork_parent_handler (void)
+{
+ vcm->forking = 1;
+
+ while (vcm->forking)
+ ;
}
/*
@@ -743,55 +762,58 @@ vppcom_app_create (char *app_name)
vppcom_cfg_t *vcl_cfg = &vcm->cfg;
int rv;
- if (!vcm->is_init)
+ if (vcm->is_init)
{
- vcm->is_init = 1;
- vppcom_cfg (&vcm->cfg);
- vcl_cfg = &vcm->cfg;
-
- vcm->main_cpu = pthread_self ();
- vcm->main_pid = vcm->current_pid = getpid ();
- vcm->app_name = format (0, "%s", app_name);
- vppcom_init_error_string_table ();
- svm_fifo_segment_main_init (vcl_cfg->segment_baseva,
- 20 /* timeout in secs */ );
- pool_init_fixed (vcm->workers, vcl_cfg->max_workers);
- clib_spinlock_init (&vcm->workers_lock);
- vcl_worker_alloc_and_init ();
- pthread_atfork (NULL, NULL, vcl_app_fork_child_handler);
+ clib_warning ("already initialized");
+ return -1;
}
- if (vcm->my_client_index == ~0)
- {
- /* API hookup and connect to VPP */
- vppcom_api_hookup ();
- vcl_elog_init (vcm);
- vcm->app_state = STATE_APP_START;
- rv = vppcom_connect_to_vpp (app_name);
- if (rv)
- {
- VERR ("couldn't connect to VPP!");
- return rv;
- }
- VDBG (0, "sending session enable");
- rv = vppcom_app_session_enable ();
- if (rv)
- {
- VERR ("vppcom_app_session_enable() failed!");
- return rv;
- }
+ vcm->is_init = 1;
+ vppcom_cfg (&vcm->cfg);
+ vcl_cfg = &vcm->cfg;
- VDBG (0, "sending app attach");
- rv = vppcom_app_attach ();
- if (rv)
- {
- VERR ("vppcom_app_attach() failed!");
- return rv;
- }
+ vcm->main_cpu = pthread_self ();
+ vcm->main_pid = getpid ();
+ vcm->app_name = format (0, "%s", app_name);
+ vppcom_init_error_string_table ();
+ svm_fifo_segment_main_init (vcl_cfg->segment_baseva,
+ 20 /* timeout in secs */ );
+ pool_alloc (vcm->workers, vcl_cfg->max_workers);
+ clib_spinlock_init (&vcm->workers_lock);
+ pthread_atfork (NULL, vcl_app_fork_parent_handler,
+ vcl_app_fork_child_handler);
- VDBG (0, "app_name '%s', my_client_index %d (0x%x)",
- app_name, vcm->my_client_index, vcm->my_client_index);
+ /* Allocate default worker */
+ vcl_worker_alloc_and_init ();
+
+ /* API hookup and connect to VPP */
+ vppcom_api_hookup ();
+ vcl_elog_init (vcm);
+ vcm->app_state = STATE_APP_START;
+ rv = vppcom_connect_to_vpp (app_name);
+ if (rv)
+ {
+ VERR ("couldn't connect to VPP!");
+ return rv;
}
+ VDBG (0, "sending session enable");
+ rv = vppcom_app_session_enable ();
+ if (rv)
+ {
+ VERR ("vppcom_app_session_enable() failed!");
+ return rv;
+ }
+
+ VDBG (0, "sending app attach");
+ rv = vppcom_app_attach ();
+ if (rv)
+ {
+ VERR ("vppcom_app_attach() failed!");
+ return rv;
+ }
+
+ VDBG (0, "app_name '%s', my_client_index %d (0x%x)", app_name,
+ vcm->workers[0].my_client_index, vcm->workers[0].my_client_index);
return VPPCOM_OK;
}
@@ -802,27 +824,27 @@ vppcom_app_destroy (void)
int rv;
f64 orig_app_timeout;
- if (vcm->my_client_index == ~0)
- return;
-
- VDBG (0, "detaching from VPP, my_client_index %d (0x%x)",
- vcm->my_client_index, vcm->my_client_index);
vcl_evt (VCL_EVT_DETACH, vcm);
- vppcom_app_send_detach ();
- orig_app_timeout = vcm->cfg.app_timeout;
- vcm->cfg.app_timeout = 2.0;
- rv = vcl_wait_for_app_state_change (STATE_APP_ENABLED);
- vcm->cfg.app_timeout = orig_app_timeout;
- if (PREDICT_FALSE (rv))
- VDBG (0, "application detach timed out! returning %d (%s)",
- rv, vppcom_retval_str (rv));
+ if (vec_len (vcm->workers) == 1)
+ {
+ vppcom_app_send_detach ();
+ orig_app_timeout = vcm->cfg.app_timeout;
+ vcm->cfg.app_timeout = 2.0;
+ rv = vcl_wait_for_app_state_change (STATE_APP_ENABLED);
+ vcm->cfg.app_timeout = orig_app_timeout;
+ if (PREDICT_FALSE (rv))
+ VDBG (0, "application detach timed out! returning %d (%s)", rv,
+ vppcom_retval_str (rv));
+ }
+ else
+ {
+ vcl_worker_cleanup ();
+ }
vcl_elog_stop (vcm);
vl_client_disconnect_from_vlib ();
vec_free (vcm->app_name);
- vcm->my_client_index = ~0;
- vcm->app_state = STATE_APP_START;
}
int
@@ -853,8 +875,8 @@ int
vppcom_session_close (uint32_t session_handle)
{
vcl_worker_t *wrk = vcl_worker_get_current ();
+ u8 is_vep, do_disconnect = 1;
vcl_session_t *session = 0;
- u8 is_vep, is_vep_session;
session_state_t state;
u32 next_sh, vep_sh;
int rv = VPPCOM_OK;
@@ -864,24 +886,17 @@ vppcom_session_close (uint32_t session_handle)
if (!session)
return VPPCOM_EBADFD;
+ if (session->shared_index != ~0)
+ do_disconnect = vcl_worker_unshare_session (wrk, session);
+
is_vep = session->is_vep;
- is_vep_session = session->is_vep_session;
next_sh = session->vep.next_sh;
vep_sh = session->vep.vep_sh;
state = session->session_state;
vpp_handle = session->vpp_handle;
- if (VPPCOM_DEBUG > 0)
- {
- if (is_vep)
- clib_warning ("VCL<%d>: vep_idx %u / sid %u: "
- "closing epoll session...",
- getpid (), session_handle, session_handle);
- else
- clib_warning ("VCL<%d>: vpp handle 0x%llx, sid %d: "
- "closing session...",
- getpid (), vpp_handle, session_handle);
- }
+ VDBG (0, "Closing session handle %u vpp handle %u", session_handle,
+ vpp_handle);
if (is_vep)
{
@@ -889,34 +904,34 @@ vppcom_session_close (uint32_t session_handle)
{
rv = vppcom_epoll_ctl (session_handle, EPOLL_CTL_DEL, next_sh, 0);
if (PREDICT_FALSE (rv < 0))
- VDBG (0, "VCL<%d>: vpp handle 0x%llx, sid %u: EPOLL_CTL_DEL "
- "vep_idx %u failed! rv %d (%s)",
- getpid (), vpp_handle, next_sh, vep_sh,
- rv, vppcom_retval_str (rv));
+ VDBG (0, "vpp handle 0x%llx, sid %u: EPOLL_CTL_DEL vep_idx %u"
+ " failed! rv %d (%s)", vpp_handle, next_sh, vep_sh, rv,
+ vppcom_retval_str (rv));
next_sh = session->vep.next_sh;
}
}
else
{
- if (is_vep_session)
+ if (session->is_vep_session)
{
rv = vppcom_epoll_ctl (vep_sh, EPOLL_CTL_DEL, session_handle, 0);
if (rv < 0)
- VDBG (0, "VCL<%d>: vpp handle 0x%llx, sid %u: EPOLL_CTL_DEL "
- "vep_idx %u failed! rv %d (%s)",
- getpid (), vpp_handle, session_handle,
- vep_sh, rv, vppcom_retval_str (rv));
+ VDBG (0, "vpp handle 0x%llx, sid %u: EPOLL_CTL_DEL vep_idx %u "
+ "failed! rv %d (%s)", vpp_handle, session_handle, vep_sh,
+ rv, vppcom_retval_str (rv));
}
+ if (!do_disconnect)
+ goto cleanup;
+
if (state & STATE_LISTEN)
{
rv = vppcom_session_unbind (session_handle);
if (PREDICT_FALSE (rv < 0))
- VDBG (0, "VCL<%d>: vpp handle 0x%llx, sid %u: listener unbind "
- "failed! rv %d (%s)",
- getpid (), vpp_handle, session_handle,
- rv, vppcom_retval_str (rv));
+ VDBG (0, "vpp handle 0x%llx, sid %u: listener unbind failed! "
+ "rv %d (%s)", vpp_handle, session_handle, rv,
+ vppcom_retval_str (rv));
}
else if (state & STATE_OPEN)
{
@@ -929,6 +944,8 @@ vppcom_session_close (uint32_t session_handle)
}
}
+cleanup:
+
if (vcl_session_is_ct (session))
{
vcl_cut_through_registration_t *ctr;
@@ -952,15 +969,8 @@ vppcom_session_close (uint32_t session_handle)
}
vcl_session_free (wrk, session);
- if (VPPCOM_DEBUG > 0)
- {
- if (is_vep)
- clib_warning ("VCL<%d>: vep_idx %u / sid %u: epoll session removed.",
- getpid (), session_handle, session_handle);
- else
- clib_warning ("VCL<%d>: vpp handle 0x%llx, sid %u: session removed.",
- getpid (), vpp_handle, session_handle);
- }
+ VDBG (0, "session handle %u vpp handle %u removed", session_handle,
+ vpp_handle);
vcl_evt (VCL_EVT_CLOSE, session, rv);
@@ -1986,9 +1996,9 @@ vppcom_select (unsigned long n_bits, unsigned long *read_map,
clib_bitmap_foreach (sid, wrk->wr_bitmap, ({
if (!(session = vcl_session_get (wrk, sid)))
{
- VDBG (0, "VCL<%d>: session %d specified in write_map is closed.",
- getpid (), sid);
- return VPPCOM_EBADFD;
+ if (except_map && sid < minbits)
+ clib_bitmap_set_no_check (except_map, sid, 1);
+ continue;
}
rv = svm_fifo_is_full (session->tx_fifo);
@@ -2006,9 +2016,9 @@ check_rd:
clib_bitmap_foreach (sid, wrk->rd_bitmap, ({
if (!(session = vcl_session_get (wrk, sid)))
{
- VDBG (0, "VCL<%d>: session %d specified in write_map is closed.",
- getpid (), sid);
- return VPPCOM_EBADFD;
+ if (except_map && sid < minbits)
+ clib_bitmap_set_no_check (except_map, sid, 1);
+ continue;
}
rv = vppcom_session_read_ready (session);
@@ -3150,6 +3160,10 @@ vppcom_session_attr (uint32_t session_handle, uint32_t op,
rv = VPPCOM_EINVAL;
break;
+ case VPPCOM_ATTR_GET_REFCNT:
+ rv = vcl_session_get_refcnt (session);
+ break;
+
default:
rv = VPPCOM_EINVAL;
break;
@@ -3355,15 +3369,22 @@ vppcom_session_index (uint32_t session_handle)
int
vppcom_session_handle (uint32_t session_index)
{
- return vcl_get_worker_index () << 24 | session_index;
+ return (vcl_get_worker_index () << 24) | session_index;
}
int
vppcom_worker_register (void)
{
- if (vcl_worker_alloc_and_init ())
- return VPPCOM_OK;
- return VPPCOM_EEXIST;
+ if (!vcl_worker_alloc_and_init ())
+ return VPPCOM_EEXIST;
+
+ if (vcl_worker_set_bapi ())
+ return VPPCOM_EEXIST;
+
+ if (vcl_worker_register_with_vpp ())
+ return VPPCOM_EEXIST;
+
+ return VPPCOM_OK;
}
int