vcl: improve vpp detached handling for mq epoll apps

Apps that rely on epoll and use eventfds for mq signaling can
potentially sleep indefinitely if vcl detaches from vpp and no libc fd
generates events.

To avoid this, at detach time, force creation of a pair of pipes that
constantly generates mq_epfd events to force apps, including ldp, to
request vppcom epoll_waits which retry attaching to vpp.

Type: improvement

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: Ie2ac338cc10721829e6ac525ee7d6b812354e9a3

3 files changed, 58 insertions, 0 deletions

diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c
index 6892688da5a..1adc8cb6666 100644
--- a/src/vcl/vcl_private.c
+++ b/src/vcl/vcl_private.c
@@ -189,6 +189,53 @@ vcl_worker_cleanup_cb (void *arg)
 }
 
 void
+vcl_worker_detached_start_signal_mq (vcl_worker_t *wrk)
+{
+  /* Generate mq epfd events using pipes to hopefully force
+   * calls into epoll_wait which retries attaching to vpp */
+  if (!wrk->detached_pipefds[0])
+    {
+      if (pipe (wrk->detached_pipefds))
+	{
+	  VDBG (0, "failed to add mq eventfd to mq epoll fd");
+	  exit (1);
+	}
+    }
+
+  struct epoll_event evt = {};
+  evt.events = EPOLLIN;
+  evt.data.u32 = wrk->detached_pipefds[0];
+  if (epoll_ctl (wrk->mqs_epfd, EPOLL_CTL_ADD, wrk->detached_pipefds[0],
+		 &evt) < 0)
+    {
+      VDBG (0, "failed to add mq eventfd to mq epoll fd");
+      exit (1);
+    }
+
+  int sig = 1, __clib_unused rv;
+  rv = write (wrk->detached_pipefds[1], &sig, 1);
+}
+
+void
+vcl_worker_detached_signal_mq (vcl_worker_t *wrk)
+{
+  int buf, __clib_unused rv;
+  rv = read (wrk->detached_pipefds[0], &buf, 1);
+  rv = write (wrk->detached_pipefds[1], &buf, 1);
+}
+
+void
+vcl_worker_detached_stop_signal_mq (vcl_worker_t *wrk)
+{
+  if (epoll_ctl (wrk->mqs_epfd, EPOLL_CTL_DEL, wrk->detached_pipefds[0], 0) <
+      0)
+    {
+      VDBG (0, "failed to del mq eventfd to mq epoll fd");
+      exit (1);
+    }
+}
+
+void
 vcl_worker_detach_sessions (vcl_worker_t *wrk)
 {
   session_event_t *e;
@@ -239,6 +286,8 @@ vcl_worker_detach_sessions (vcl_worker_t *wrk)
 
   vec_free (seg_indices);
   hash_free (seg_indices_map);
+
+  vcl_worker_detached_start_signal_mq (wrk);
 }
 
 void
diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h
index c92bb58169d..72f0a8327f0 100644
--- a/src/vcl/vcl_private.h
+++ b/src/vcl/vcl_private.h
@@ -325,6 +325,9 @@ typedef struct vcl_worker_
   /* functions to be called pre/post wait if vcl managed by vls */
   vcl_worker_wait_mq_fn pre_wait_fn;
   vcl_worker_wait_mq_fn post_wait_fn;
+
+  /* mq_epfd signal pipes when wrk detached from vpp */
+  int detached_pipefds[2];
 } vcl_worker_t;
 
 STATIC_ASSERT (sizeof (session_disconnected_msg_t) <= 16,
@@ -799,6 +802,9 @@ void vcl_worker_detach_sessions (vcl_worker_t *wrk);
 void vcl_worker_set_wait_mq_fns (vcl_worker_wait_mq_fn pre_wait,
 				 vcl_worker_wait_mq_fn post_wait);
 
+void vcl_worker_detached_start_signal_mq (vcl_worker_t *wrk);
+void vcl_worker_detached_signal_mq (vcl_worker_t *wrk);
+void vcl_worker_detached_stop_signal_mq (vcl_worker_t *wrk);
 /*
  * VCL Binary API
  */
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index 1e9c915ce39..8bc5f42d4a6 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -1404,6 +1404,8 @@ vcl_api_retry_attach (vcl_worker_t *wrk)
 {
   vcl_session_t *s;
 
+  vcl_worker_detached_signal_mq (wrk);
+
   clib_spinlock_lock (&vcm->workers_lock);
   if (vcl_is_first_reattach_to_execute ())
     {
@@ -1412,6 +1414,7 @@ vcl_api_retry_attach (vcl_worker_t *wrk)
 	  clib_spinlock_unlock (&vcm->workers_lock);
 	  return;
 	}
+      vcl_worker_detached_stop_signal_mq (wrk);
       vcl_set_reattach_counter ();
       clib_spinlock_unlock (&vcm->workers_lock);
     }