vcl: handle worker process exit

Change-Id: Ife05d25fd736ae3064f01e974e5aecc5b48de924
Signed-off-by: Florin Coras <fcoras@cisco.com>

4 files changed, 41 insertions, 15 deletions

diff --git a/src/vcl/ldp.c b/src/vcl/ldp.c
index 915d1caa40c..453ddeb3340 100644
--- a/src/vcl/ldp.c
+++ b/src/vcl/ldp.c
@@ -3556,10 +3556,7 @@ ldp_destructor (void)
 {
   swrap_destructor ();
   if (ldp->init)
-    {
-      vppcom_app_destroy ();
-      ldp->init = 0;
-    }
+    ldp->init = 0;
 
   /* Don't use clib_warning() here because that calls writev()
    * which will call ldp_init().
diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c
index 76ae0e7719f..673b91d4576 100644
--- a/src/vcl/vcl_private.c
+++ b/src/vcl/vcl_private.c
@@ -221,13 +221,15 @@ vcl_worker_free (vcl_worker_t * wrk)
 }
 
 void
-vcl_worker_cleanup (void)
+vcl_worker_cleanup (u8 notify_vpp)
 {
   vcl_worker_t *wrk = vcl_worker_get_current ();
 
   clib_spinlock_lock (&vcm->workers_lock);
-  vcl_send_app_worker_add_del (0 /* is_add */ );
-  close (wrk->mqs_epfd);
+  if (notify_vpp)
+    vcl_send_app_worker_add_del (0 /* is_add */ );
+  if (wrk->mqs_epfd > 0)
+    close (wrk->mqs_epfd);
   hash_free (wrk->session_index_by_vpp_handles);
   hash_free (wrk->ct_registration_by_mq);
   clib_spinlock_free (&wrk->ct_registration_lock);
@@ -236,13 +238,14 @@ vcl_worker_cleanup (void)
   vcl_set_worker_index (~0);
   vcl_worker_free (wrk);
   clib_spinlock_unlock (&vcm->workers_lock);
-  VDBG (0, "cleaned up worker %u", wrk->wrk_index);
 }
 
 static void
 vcl_worker_cleanup_cb (void *arg)
 {
-  vcl_worker_cleanup ();
+  u32 wrk_index = vcl_get_worker_index ();
+  vcl_worker_cleanup (1 /* notify vpp */ );
+  VDBG (0, "cleaned up worker %u", wrk_index);
 }
 
 vcl_worker_t *
diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h
index 6a289d640e0..7d6af6d6599 100644
--- a/src/vcl/vcl_private.h
+++ b/src/vcl/vcl_private.h
@@ -504,7 +504,7 @@ int vcl_mq_epoll_add_evfd (vcl_worker_t * wrk, svm_msg_q_t * mq);
 int vcl_mq_epoll_del_evfd (vcl_worker_t * wrk, u32 mqc_index);
 
 vcl_worker_t *vcl_worker_alloc_and_init (void);
-void vcl_worker_cleanup (void);
+void vcl_worker_cleanup (u8 notify_vpp);
 int vcl_worker_register_with_vpp (void);
 int vcl_worker_set_bapi (void);
 void vcl_worker_share_sessions (u32 parent_wrk_index);
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index db767e9e713..1b619cf3f21 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -700,12 +700,12 @@ vcl_app_fork_child_handler (void)
   u8 *child_name;
   int rv, parent_wrk;
 
-  VDBG (0, "initializing forked child");
+  parent_wrk = vcl_get_worker_index ();
+  VDBG (0, "initializing forked child with parent wrk %u", parent_wrk);
 
   /*
    * Allocate worker
    */
-  parent_wrk = vcl_get_worker_index ();
   vcl_set_worker_index (~0);
   if (!vcl_worker_alloc_and_init ())
     VERR ("couldn't allocate new worker");
@@ -744,6 +744,27 @@ vcl_app_fork_parent_handler (void)
     ;
 }
 
+/**
+ * Handle app exit
+ *
+ * Notify vpp of the disconnect and mark the worker as free. If we're the
+ * last worker, do a full cleanup otherwise, since we're probably a forked
+ * child, avoid syscalls as much as possible. We might've lost privileges.
+ */
+void
+vppcom_app_exit (void)
+{
+  if (!pool_elts (vcm->workers))
+    return;
+
+  vcl_worker_cleanup (1 /* notify vpp */ );
+  vcl_elog_stop (vcm);
+  if (vec_len (vcm->workers) == 1)
+    vl_client_disconnect_from_vlib ();
+  else
+    vl_client_send_disconnect ();
+}
+
 /*
  * VPPCOM Public API functions
  */
@@ -774,6 +795,7 @@ vppcom_app_create (char *app_name)
   clib_rwlock_init (&vcm->segment_table_lock);
   pthread_atfork (NULL, vcl_app_fork_parent_handler,
 		  vcl_app_fork_child_handler);
+  atexit (vppcom_app_exit);
 
   /* Allocate default worker */
   vcl_worker_alloc_and_init ();
@@ -816,9 +838,12 @@ vppcom_app_destroy (void)
   int rv;
   f64 orig_app_timeout;
 
+  if (!pool_elts (vcm->workers))
+    return;
+
   vcl_evt (VCL_EVT_DETACH, vcm);
 
-  if (vec_len (vcm->workers) == 1)
+  if (pool_elts (vcm->workers) == 1)
     {
       vppcom_app_send_detach ();
       orig_app_timeout = vcm->cfg.app_timeout;
@@ -828,15 +853,16 @@ vppcom_app_destroy (void)
       if (PREDICT_FALSE (rv))
 	VDBG (0, "application detach timed out! returning %d (%s)", rv,
 	      vppcom_retval_str (rv));
+      vec_free (vcm->app_name);
+      vcl_worker_cleanup (0 /* notify vpp */ );
     }
   else
     {
-      vcl_worker_cleanup ();
+      vcl_worker_cleanup (1 /* notify vpp */ );
     }
 
   vcl_elog_stop (vcm);
   vl_client_disconnect_from_vlib ();
-  vec_free (vcm->app_name);
 }
 
 int