vcl: move forking logic to vls

Change-Id: I721542aca139d7908a4f917629856f82cae79962
Signed-off-by: Florin Coras <fcoras@cisco.com>

6 files changed, 314 insertions, 357 deletions

diff --git a/src/vcl/vcl_locked.c b/src/vcl/vcl_locked.c
index 6254bad09b6..c6c79cdd470 100644
--- a/src/vcl/vcl_locked.c
+++ b/src/vcl/vcl_locked.c
@@ -18,11 +18,12 @@
 
 typedef struct vcl_locked_session_
 {
+  clib_spinlock_t lock;
   u32 session_index;
   u32 worker_index;
   u32 vls_index;
   u32 flags;
-  clib_spinlock_t lock;
+  u32 *workers_subscribed;
 } vcl_locked_session_t;
 
 typedef struct vcl_main_
@@ -62,7 +63,7 @@ vls_table_wunlock (void)
 static inline vcl_session_handle_t
 vls_to_sh (vcl_locked_session_t * vls)
 {
-  return vppcom_session_handle (vls->session_index);
+  return vcl_session_handle_from_index (vls->session_index);
 }
 
 static inline vcl_session_handle_t
@@ -164,6 +165,86 @@ vls_get_and_free (vls_handle_t vlsh)
   vls_table_wunlock ();
 }
 
+u8
+vls_is_shared (vcl_locked_session_t * vls)
+{
+  return vec_len (vls->workers_subscribed);
+}
+
+int
+vls_unshare_session (vcl_locked_session_t * vls)
+{
+  vcl_worker_t *wrk = vcl_worker_get_current ();
+  vcl_session_t *s;
+  int i;
+
+  for (i = 0; i < vec_len (vls->workers_subscribed); i++)
+    {
+      if (vls->workers_subscribed[i] != wrk->wrk_index)
+	continue;
+
+      s = vcl_session_get (wrk, vls->session_index);
+      if (s->rx_fifo)
+	{
+	  svm_fifo_del_subscriber (s->rx_fifo, wrk->vpp_wrk_index);
+	  svm_fifo_del_subscriber (s->tx_fifo, wrk->vpp_wrk_index);
+	}
+      vec_del1 (vls->workers_subscribed, i);
+      vcl_session_cleanup (wrk, s, vcl_session_handle (s),
+			   0 /* do_disconnect */ );
+      return 0;
+    }
+
+  /* Assumption is that unshare is only called if session is shared.
+   * So shared_workers must be non-empty if the worker is the owner */
+  if (vls->worker_index == wrk->wrk_index)
+    {
+      s = vcl_session_get (wrk, vls->session_index);
+      vls->worker_index = vls->workers_subscribed[0];
+      vec_del1 (vls->workers_subscribed, 0);
+      vcl_send_session_worker_update (wrk, s, vls->worker_index);
+      if (vec_len (vls->workers_subscribed))
+	clib_warning ("more workers need to be updated");
+    }
+
+  return 0;
+}
+
+void
+vls_share_vcl_session (vcl_worker_t * wrk, vcl_session_t * s)
+{
+  vcl_locked_session_t *vls;
+
+  vls = vls_get_w_dlock (vls_session_index_to_vlsh (s->session_index));
+  if (!vls)
+    return;
+  vec_add1 (vls->workers_subscribed, wrk->wrk_index);
+  if (s->rx_fifo)
+    {
+      svm_fifo_add_subscriber (s->rx_fifo, wrk->vpp_wrk_index);
+      svm_fifo_add_subscriber (s->tx_fifo, wrk->vpp_wrk_index);
+    }
+  vls_dunlock (vls);
+}
+
+void
+vls_worker_copy_on_fork (vcl_worker_t * parent_wrk)
+{
+  vcl_worker_t *wrk = vcl_worker_get_current ();
+  vcl_session_t *s;
+
+  wrk->vpp_event_queues = vec_dup (parent_wrk->vpp_event_queues);
+  wrk->sessions = pool_dup (parent_wrk->sessions);
+  wrk->session_index_by_vpp_handles =
+    hash_dup (parent_wrk->session_index_by_vpp_handles);
+
+  /* *INDENT-OFF* */
+  pool_foreach (s, wrk->sessions, ({
+    vls_share_vcl_session (wrk, s);
+  }));
+  /* *INDENT-ON* */
+}
+
 int
 vls_write (vls_handle_t vlsh, void *buf, size_t nbytes)
 {
@@ -325,13 +406,19 @@ vls_close (vls_handle_t vlsh)
 {
   vcl_locked_session_t *vls;
   vcl_session_handle_t sh;
-  int rv, refcnt;
+  int rv;
 
   if (!(vls = vls_get_w_dlock (vlsh)))
     return VPPCOM_EBADFD;
 
+  if (vls_is_shared (vls))
+    {
+      vls_unshare_session (vls);
+      vls_dunlock (vls);
+      return VPPCOM_OK;
+    }
+
   sh = vls_to_sh (vls);
-  refcnt = vppcom_session_attr (sh, VPPCOM_ATTR_GET_REFCNT, 0, 0);
   if ((rv = vppcom_session_close (sh)))
     {
       vls_dunlock (vls);
@@ -339,8 +426,7 @@ vls_close (vls_handle_t vlsh)
     }
 
   vls_dunlock (vls);
-  if (refcnt <= 1)
-    vls_get_and_free (vlsh);
+  vls_get_and_free (vlsh);
   return rv;
 }
 
@@ -438,6 +524,148 @@ vls_session_index_to_vlsh (uint32_t session_index)
   return vlsh;
 }
 
+static void
+vls_cleanup_forked_child (vcl_worker_t * wrk, vcl_worker_t * child_wrk)
+{
+  vcl_worker_t *sub_child;
+  int tries = 0;
+
+  if (child_wrk->forked_child != ~0)
+    {
+      sub_child = vcl_worker_get_if_valid (child_wrk->forked_child);
+      if (sub_child)
+	{
+	  /* Wait a bit, maybe the process is going away */
+	  while (kill (sub_child->current_pid, 0) >= 0 && tries++ < 50)
+	    usleep (1e3);
+	  if (kill (sub_child->current_pid, 0) < 0)
+	    vls_cleanup_forked_child (child_wrk, sub_child);
+	}
+    }
+  vcl_worker_cleanup (child_wrk, 1 /* notify vpp */ );
+  VDBG (0, "Cleaned up wrk %u", child_wrk->wrk_index);
+  wrk->forked_child = ~0;
+}
+
+static struct sigaction old_sa;
+
+static void
+vls_intercept_sigchld_handler (int signum, siginfo_t * si, void *uc)
+{
+  vcl_worker_t *wrk, *child_wrk;
+
+  if (vcl_get_worker_index () == ~0)
+    return;
+
+  if (sigaction (SIGCHLD, &old_sa, 0))
+    {
+      VERR ("couldn't restore sigchld");
+      exit (-1);
+    }
+
+  wrk = vcl_worker_get_current ();
+  if (wrk->forked_child == ~0)
+    return;
+
+  child_wrk = vcl_worker_get_if_valid (wrk->forked_child);
+  if (!child_wrk)
+    goto done;
+
+  if (si && si->si_pid != child_wrk->current_pid)
+    {
+      VDBG (0, "unexpected child pid %u", si->si_pid);
+      goto done;
+    }
+  vls_cleanup_forked_child (wrk, child_wrk);
+
+done:
+  if (old_sa.sa_flags & SA_SIGINFO)
+    {
+      void (*fn) (int, siginfo_t *, void *) = old_sa.sa_sigaction;
+      fn (signum, si, uc);
+    }
+  else
+    {
+      void (*fn) (int) = old_sa.sa_handler;
+      if (fn)
+	fn (signum);
+    }
+}
+
+static void
+vls_incercept_sigchld ()
+{
+  struct sigaction sa;
+  clib_memset (&sa, 0, sizeof (sa));
+  sa.sa_sigaction = vls_intercept_sigchld_handler;
+  sa.sa_flags = SA_SIGINFO;
+  if (sigaction (SIGCHLD, &sa, &old_sa))
+    {
+      VERR ("couldn't intercept sigchld");
+      exit (-1);
+    }
+}
+
+static void
+vls_app_pre_fork (void)
+{
+  vls_incercept_sigchld ();
+  vcl_flush_mq_events ();
+}
+
+static void
+vls_app_fork_child_handler (void)
+{
+  vcl_worker_t *parent_wrk;
+  int rv, parent_wrk_index;
+  u8 *child_name;
+
+  parent_wrk_index = vcl_get_worker_index ();
+  VDBG (0, "initializing forked child %u with parent wrk %u", getpid (),
+	parent_wrk_index);
+
+  /*
+   * Allocate worker
+   */
+  vcl_set_worker_index (~0);
+  if (!vcl_worker_alloc_and_init ())
+    VERR ("couldn't allocate new worker");
+
+  /*
+   * Attach to binary api
+   */
+  child_name = format (0, "%v-child-%u%c", vcm->app_name, getpid (), 0);
+  vcl_cleanup_bapi ();
+  vppcom_api_hookup ();
+  vcm->app_state = STATE_APP_START;
+  rv = vppcom_connect_to_vpp ((char *) child_name);
+  vec_free (child_name);
+  if (rv)
+    {
+      VERR ("couldn't connect to VPP!");
+      return;
+    }
+
+  /*
+   * Register worker with vpp and share sessions
+   */
+  vcl_worker_register_with_vpp ();
+  parent_wrk = vcl_worker_get (parent_wrk_index);
+  vls_worker_copy_on_fork (parent_wrk);
+  parent_wrk->forked_child = vcl_get_worker_index ();
+
+  VDBG (0, "forked child main worker initialized");
+  vcm->forking = 0;
+}
+
+static void
+vls_app_fork_parent_handler (void)
+{
+  vcm->forking = 1;
+  while (vcm->forking)
+    ;
+}
+
 int
 vls_app_create (char *app_name)
 {
@@ -445,6 +673,8 @@ vls_app_create (char *app_name)
   if ((rv = vppcom_app_create (app_name)))
     return rv;
   clib_rwlock_init (&vlsm->vls_table_lock);
+  pthread_atfork (vls_app_pre_fork, vls_app_fork_parent_handler,
+		  vls_app_fork_child_handler);
   return VPPCOM_OK;
 }
 
diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c
index 4241bcf1789..df602b3ce87 100644
--- a/src/vcl/vcl_private.c
+++ b/src/vcl/vcl_private.c
@@ -346,127 +346,6 @@ vcl_worker_set_bapi (void)
   return -1;
 }
 
-vcl_shared_session_t *
-vcl_shared_session_alloc (void)
-{
-  vcl_shared_session_t *ss;
-  pool_get (vcm->shared_sessions, ss);
-  memset (ss, 0, sizeof (*ss));
-  ss->ss_index = ss - vcm->shared_sessions;
-  return ss;
-}
-
-vcl_shared_session_t *
-vcl_shared_session_get (u32 ss_index)
-{
-  if (pool_is_free_index (vcm->shared_sessions, ss_index))
-    return 0;
-  return pool_elt_at_index (vcm->shared_sessions, ss_index);
-}
-
-void
-vcl_shared_session_free (vcl_shared_session_t * ss)
-{
-  pool_put (vcm->shared_sessions, ss);
-}
-
-void
-vcl_worker_share_session (vcl_worker_t * parent, vcl_worker_t * wrk,
-			  vcl_session_t * new_s)
-{
-  vcl_shared_session_t *ss;
-  vcl_session_t *old_s;
-
-  if (new_s->shared_index == ~0)
-    {
-      ss = vcl_shared_session_alloc ();
-      ss->session_index = new_s->session_index;
-      vec_add1 (ss->workers, parent->wrk_index);
-      vec_add1 (ss->workers, wrk->wrk_index);
-      new_s->shared_index = ss->ss_index;
-      old_s = vcl_session_get (parent, new_s->session_index);
-      old_s->shared_index = ss->ss_index;
-    }
-  else
-    {
-      ss = vcl_shared_session_get (new_s->shared_index);
-      vec_add1 (ss->workers, wrk->wrk_index);
-    }
-  if (new_s->rx_fifo)
-    {
-      svm_fifo_add_subscriber (new_s->rx_fifo, wrk->vpp_wrk_index);
-      svm_fifo_add_subscriber (new_s->tx_fifo, wrk->vpp_wrk_index);
-    }
-}
-
-int
-vcl_worker_unshare_session (vcl_worker_t * wrk, vcl_session_t * s)
-{
-  vcl_shared_session_t *ss;
-  int i;
-
-  ss = vcl_shared_session_get (s->shared_index);
-  for (i = 0; i < vec_len (ss->workers); i++)
-    {
-      if (ss->workers[i] == wrk->wrk_index)
-	{
-	  vec_del1 (ss->workers, i);
-	  break;
-	}
-    }
-
-  if (vec_len (ss->workers) == 0)
-    {
-      vcl_shared_session_free (ss);
-      return 1;
-    }
-
-  if (s->rx_fifo)
-    {
-      svm_fifo_del_subscriber (s->rx_fifo, wrk->vpp_wrk_index);
-      svm_fifo_del_subscriber (s->tx_fifo, wrk->vpp_wrk_index);
-    }
-
-  /* If the first removed and not last, start session worker change.
-   * First request goes to vpp and vpp reflects it back to the right
-   * worker */
-  if (i == 0)
-    vcl_send_session_worker_update (wrk, s, ss->workers[0]);
-
-  return 0;
-}
-
-void
-vcl_worker_share_sessions (vcl_worker_t * parent_wrk)
-{
-  vcl_session_t *new_s;
-  vcl_worker_t *wrk;
-
-  if (!parent_wrk->sessions)
-    return;
-
-  wrk = vcl_worker_get_current ();
-  wrk->sessions = pool_dup (parent_wrk->sessions);
-  wrk->session_index_by_vpp_handles =
-    hash_dup (parent_wrk->session_index_by_vpp_handles);
-
-  /* *INDENT-OFF* */
-  pool_foreach (new_s, wrk->sessions, ({
-    vcl_worker_share_session (parent_wrk, wrk, new_s);
-  }));
-  /* *INDENT-ON* */
-}
-
-int
-vcl_session_get_refcnt (vcl_session_t * s)
-{
-  vcl_shared_session_t *ss;
-  ss = vcl_shared_session_get (s->shared_index);
-  if (ss)
-    return vec_len (ss->workers);
-  return 0;
-}
-
 void
 vcl_segment_table_add (u64 segment_handle, u32 svm_segment_index)
 {
@@ -497,6 +376,21 @@ vcl_segment_table_del (u64 segment_handle)
   clib_rwlock_writer_unlock (&vcm->segment_table_lock);
 }
 
+void
+vcl_cleanup_bapi (void)
+{
+  socket_client_main_t *scm = &socket_client_main;
+  api_main_t *am = &api_main;
+
+  am->my_client_index = ~0;
+  am->my_registration = 0;
+  am->vl_input_queue = 0;
+  am->msg_index_by_name_and_crc = 0;
+  scm->socket_fd = 0;
+
+  vl_client_api_unmap ();
+}
+
 /*
  * fd.io coding-style-patch-verification: ON
  *
diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h
index dd1d0cea440..08dfe428764 100644
--- a/src/vcl/vcl_private.h
+++ b/src/vcl/vcl_private.h
@@ -181,7 +181,6 @@ typedef struct
   svm_msg_q_t *our_evt_q;
   u64 options[16];
   vcl_session_msg_t *accept_evts_fifo;
-  u32 shared_index;
 #if VCL_ELOG
   elog_track_t elog_track;
 #endif
@@ -371,7 +370,6 @@ vcl_session_alloc (vcl_worker_t * wrk)
   pool_get (wrk->sessions, s);
   memset (s, 0, sizeof (*s));
   s->session_index = s - wrk->sessions;
-  s->shared_index = ~0;
   return s;
 }
 
@@ -389,11 +387,17 @@ vcl_session_get (vcl_worker_t * wrk, u32 session_index)
   return pool_elt_at_index (wrk->sessions, session_index);
 }
 
-static inline int
+static inline vcl_session_handle_t
+vcl_session_handle_from_index (u32 session_index)
+{
+  ASSERT (session_index < 2 << 24);
+  return (vcl_get_worker_index () << 24 | session_index);
+}
+
+static inline vcl_session_handle_t
 vcl_session_handle (vcl_session_t * s)
 {
-  ASSERT (s->session_index < 2 << 24);
-  return (vcl_get_worker_index () << 24 | s->session_index);
+  return vcl_session_handle_from_index (s->session_index);
 }
 
 static inline void
@@ -529,7 +533,11 @@ int vcl_worker_set_bapi (void);
 void vcl_worker_share_sessions (vcl_worker_t * parent_wrk);
 int vcl_worker_unshare_session (vcl_worker_t * wrk, vcl_session_t * s);
 vcl_shared_session_t *vcl_shared_session_get (u32 ss_index);
-int vcl_session_get_refcnt (vcl_session_t * s);
+
+void vcl_flush_mq_events (void);
+void vcl_cleanup_bapi (void);
+int vcl_session_cleanup (vcl_worker_t * wrk, vcl_session_t * session,
+			 vcl_session_handle_t sh, u8 do_disconnect);
 
 void vcl_segment_table_add (u64 segment_handle, u32 svm_segment_index);
 u32 vcl_segment_table_lookup (u64 segment_handle);
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index 1797d93c683..f181f35ee27 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -616,13 +616,6 @@ vcl_session_worker_update_reply_handler (vcl_worker_t * wrk, void *data)
   s->tx_fifo->client_thread_index = wrk->wrk_index;
   s->session_state = STATE_UPDATED;
 
-  if (s->shared_index != VCL_INVALID_SESSION_INDEX)
-    {
-      vcl_shared_session_t *ss;
-      ss = vcl_shared_session_get (s->shared_index);
-      if (vec_len (ss->workers) > 1)
-	VDBG (0, "workers need to be updated");
-    }
   VDBG (0, "session %u[0x%llx] moved to worker %u", s->session_index,
 	s->vpp_handle, wrk->wrk_index);
 }
@@ -741,7 +734,7 @@ vcl_handle_pending_wrk_updates (vcl_worker_t * wrk)
   vec_reset_length (wrk->pending_session_wrk_updates);
 }
 
-static void
+void
 vcl_flush_mq_events (void)
 {
   vcl_worker_t *wrk = vcl_worker_get_current ();
@@ -872,164 +865,6 @@ vppcom_session_disconnect (u32 session_handle)
   return VPPCOM_OK;
 }
 
-static void
-vcl_cleanup_bapi (void)
-{
-  socket_client_main_t *scm = &socket_client_main;
-  api_main_t *am = &api_main;
-
-  am->my_client_index = ~0;
-  am->my_registration = 0;
-  am->vl_input_queue = 0;
-  am->msg_index_by_name_and_crc = 0;
-  scm->socket_fd = 0;
-
-  vl_client_api_unmap ();
-}
-
-static void
-vcl_cleanup_forked_child (vcl_worker_t * wrk, vcl_worker_t * child_wrk)
-{
-  vcl_worker_t *sub_child;
-  int tries = 0;
-
-  if (child_wrk->forked_child != ~0)
-    {
-      sub_child = vcl_worker_get_if_valid (child_wrk->forked_child);
-      if (sub_child)
-	{
-	  /* Wait a bit, maybe the process is going away */
-	  while (kill (sub_child->current_pid, 0) >= 0 && tries++ < 50)
-	    usleep (1e3);
-	  if (kill (sub_child->current_pid, 0) < 0)
-	    vcl_cleanup_forked_child (child_wrk, sub_child);
-	}
-    }
-  vcl_worker_cleanup (child_wrk, 1 /* notify vpp */ );
-  VDBG (0, "Cleaned up wrk %u", child_wrk->wrk_index);
-  wrk->forked_child = ~0;
-}
-
-static struct sigaction old_sa;
-
-static void
-vcl_intercept_sigchld_handler (int signum, siginfo_t * si, void *uc)
-{
-  vcl_worker_t *wrk, *child_wrk;
-
-  if (vcl_get_worker_index () == ~0)
-    return;
-
-  if (sigaction (SIGCHLD, &old_sa, 0))
-    {
-      VERR ("couldn't restore sigchld");
-      exit (-1);
-    }
-
-  wrk = vcl_worker_get_current ();
-  if (wrk->forked_child == ~0)
-    return;
-
-  child_wrk = vcl_worker_get_if_valid (wrk->forked_child);
-  if (!child_wrk)
-    goto done;
-
-  if (si && si->si_pid != child_wrk->current_pid)
-    {
-      VDBG (0, "unexpected child pid %u", si->si_pid);
-      goto done;
-    }
-  vcl_cleanup_forked_child (wrk, child_wrk);
-
-done:
-  if (old_sa.sa_flags & SA_SIGINFO)
-    {
-      void (*fn) (int, siginfo_t *, void *) = old_sa.sa_sigaction;
-      fn (signum, si, uc);
-    }
-  else
-    {
-      void (*fn) (int) = old_sa.sa_handler;
-      if (fn)
-	fn (signum);
-    }
-}
-
-static void
-vcl_incercept_sigchld ()
-{
-  struct sigaction sa;
-  clib_memset (&sa, 0, sizeof (sa));
-  sa.sa_sigaction = vcl_intercept_sigchld_handler;
-  sa.sa_flags = SA_SIGINFO;
-  if (sigaction (SIGCHLD, &sa, &old_sa))
-    {
-      VERR ("couldn't intercept sigchld");
-      exit (-1);
-    }
-}
-
-static void
-vcl_app_pre_fork (void)
-{
-  vcl_incercept_sigchld ();
-  vcl_flush_mq_events ();
-}
-
-static void
-vcl_app_fork_child_handler (void)
-{
-  vcl_worker_t *parent_wrk, *wrk;
-  int rv, parent_wrk_index;
-  u8 *child_name;
-
-  parent_wrk_index = vcl_get_worker_index ();
-  VDBG (0, "initializing forked child with parent wrk %u", parent_wrk_index);
-
-  /*
-   * Allocate worker
-   */
-  vcl_set_worker_index (~0);
-  if (!vcl_worker_alloc_and_init ())
-    VERR ("couldn't allocate new worker");
-
-  /*
-   * Attach to binary api
-   */
-  child_name = format (0, "%v-child-%u%c", vcm->app_name, getpid (), 0);
-  vcl_cleanup_bapi ();
-  vppcom_api_hookup ();
-  vcm->app_state = STATE_APP_START;
-  rv = vppcom_connect_to_vpp ((char *) child_name);
-  vec_free (child_name);
-  if (rv)
-    {
-      VERR ("couldn't connect to VPP!");
-      return;
-    }
-
-  /*
-   * Register worker with vpp and share sessions
-   */
-  vcl_worker_register_with_vpp ();
-  parent_wrk = vcl_worker_get (parent_wrk_index);
-  wrk = vcl_worker_get_current ();
-  wrk->vpp_event_queues = vec_dup (parent_wrk->vpp_event_queues);
-  vcl_worker_share_sessions (parent_wrk);
-  parent_wrk->forked_child = vcl_get_worker_index ();
-
-  VDBG (0, "forked child main worker initialized");
-  vcm->forking = 0;
-}
-
-static void
-vcl_app_fork_parent_handler (void)
-{
-  vcm->forking = 1;
-  while (vcm->forking)
-    ;
-}
-
 /**
  * Handle app exit
  *
@@ -1079,8 +914,6 @@ vppcom_app_create (char *app_name)
   pool_alloc (vcm->workers, vcl_cfg->max_workers);
   clib_spinlock_init (&vcm->workers_lock);
   clib_rwlock_init (&vcm->segment_table_lock);
-  pthread_atfork (vcl_app_pre_fork, vcl_app_fork_parent_handler,
-		  vcl_app_fork_child_handler);
   atexit (vppcom_app_exit);
 
   /* Allocate default worker */
@@ -1177,22 +1010,14 @@ vppcom_session_create (u8 proto, u8 is_nonblocking)
 }
 
 int
-vppcom_session_close (uint32_t session_handle)
+vcl_session_cleanup (vcl_worker_t * wrk, vcl_session_t * session,
+		     vcl_session_handle_t sh, u8 do_disconnect)
 {
-  vcl_worker_t *wrk = vcl_worker_get_current ();
-  u8 is_vep, do_disconnect = 1;
-  vcl_session_t *session = 0;
   session_state_t state;
   u32 next_sh, vep_sh;
   int rv = VPPCOM_OK;
   u64 vpp_handle;
-
-  session = vcl_session_get_w_handle (wrk, session_handle);
-  if (!session)
-    return VPPCOM_EBADFD;
-
-  if (session->shared_index != ~0)
-    do_disconnect = vcl_worker_unshare_session (wrk, session);
+  u8 is_vep;
 
   is_vep = session->is_vep;
   next_sh = session->vep.next_sh;
@@ -1200,14 +1025,13 @@ vppcom_session_close (uint32_t session_handle)
   state = session->session_state;
   vpp_handle = session->vpp_handle;
 
-  VDBG (1, "closing session handle %u vpp handle %u", session_handle,
-	vpp_handle);
+  VDBG (1, "session %u [0x%llx] closing", session->session_index, vpp_handle);
 
   if (is_vep)
     {
       while (next_sh != ~0)
 	{
-	  rv = vppcom_epoll_ctl (session_handle, EPOLL_CTL_DEL, next_sh, 0);
+	  rv = vppcom_epoll_ctl (sh, EPOLL_CTL_DEL, next_sh, 0);
 	  if (PREDICT_FALSE (rv < 0))
 	    VDBG (0, "vpp handle 0x%llx, sid %u: EPOLL_CTL_DEL vep_idx %u"
 		  " failed! rv %d (%s)", vpp_handle, next_sh, vep_sh, rv,
@@ -1220,36 +1044,35 @@ vppcom_session_close (uint32_t session_handle)
     {
       if (session->is_vep_session)
 	{
-	  rv = vppcom_epoll_ctl (vep_sh, EPOLL_CTL_DEL, session_handle, 0);
+	  rv = vppcom_epoll_ctl (vep_sh, EPOLL_CTL_DEL, sh, 0);
 	  if (rv < 0)
-	    VDBG (0, "vpp handle 0x%llx, sid %u: EPOLL_CTL_DEL vep_idx %u "
-		  "failed! rv %d (%s)", vpp_handle, session_handle, vep_sh,
-		  rv, vppcom_retval_str (rv));
+	    VDBG (0, "session %u [0x%llx]: EPOLL_CTL_DEL vep_idx %u "
+		  "failed! rv %d (%s)", session->session_index, vpp_handle,
+		  vep_sh, rv, vppcom_retval_str (rv));
 	}
 
       if (!do_disconnect)
 	{
-	  VDBG (0, "session handle %u [0x%llx] disconnect skipped",
-		session_handle, vpp_handle);
+	  VDBG (0, "session %u [0x%llx] disconnect skipped",
+		session->session_index, vpp_handle);
 	  goto cleanup;
 	}
 
       if (state & STATE_LISTEN)
 	{
-	  rv = vppcom_session_unbind (session_handle);
+	  rv = vppcom_session_unbind (sh);
 	  if (PREDICT_FALSE (rv < 0))
-	    VDBG (0, "vpp handle 0x%llx, sid %u: listener unbind failed! "
-		  "rv %d (%s)", vpp_handle, session_handle, rv,
+	    VDBG (0, "session %u [0x%llx]: listener unbind failed! "
+		  "rv %d (%s)", session->session_index, vpp_handle, rv,
 		  vppcom_retval_str (rv));
 	}
       else if (state & STATE_OPEN)
 	{
-	  rv = vppcom_session_disconnect (session_handle);
+	  rv = vppcom_session_disconnect (sh);
 	  if (PREDICT_FALSE (rv < 0))
-	    clib_warning ("VCL<%d>: ERROR: vpp handle 0x%llx, sid %u: "
-			  "session disconnect failed! rv %d (%s)",
-			  getpid (), vpp_handle, session_handle,
-			  rv, vppcom_retval_str (rv));
+	    VDBG (0, "ERROR: session %u [0x%llx]: disconnect failed!"
+		  " rv %d (%s)", session->session_index, vpp_handle,
+		  rv, vppcom_retval_str (rv));
 	}
       else if (state == STATE_DISCONNECT)
 	{
@@ -1259,8 +1082,6 @@ vppcom_session_close (uint32_t session_handle)
 	}
     }
 
-cleanup:
-
   if (vcl_session_is_ct (session))
     {
       vcl_cut_through_registration_t *ctr;
@@ -1278,17 +1099,30 @@ cleanup:
       vcl_ct_registration_unlock (wrk);
     }
 
+cleanup:
   vcl_session_table_del_vpp_handle (wrk, vpp_handle);
   vcl_session_free (wrk, session);
 
-  VDBG (0, "session handle %u [0x%llx] removed", session_handle, vpp_handle);
-
+  VDBG (0, "session %u [0x%llx] removed", session->session_index, vpp_handle);
   vcl_evt (VCL_EVT_CLOSE, session, rv);
 
   return rv;
 }
 
 int
+vppcom_session_close (uint32_t session_handle)
+{
+  vcl_worker_t *wrk = vcl_worker_get_current ();
+  vcl_session_t *session;
+
+  session = vcl_session_get_w_handle (wrk, session_handle);
+  if (!session)
+    return VPPCOM_EBADFD;
+  return vcl_session_cleanup (wrk, session, session_handle,
+			      1 /* do_disconnect */ );
+}
+
+int
 vppcom_session_bind (uint32_t session_handle, vppcom_endpt_t * ep)
 {
   vcl_worker_t *wrk = vcl_worker_get_current ();
@@ -3502,10 +3336,6 @@ vppcom_session_attr (uint32_t session_handle, uint32_t op,
 	rv = VPPCOM_EINVAL;
       break;
 
-    case VPPCOM_ATTR_GET_REFCNT:
-      rv = vcl_session_get_refcnt (session);
-      break;
-
     case VPPCOM_ATTR_SET_SHUT:
       if (*flags == SHUT_RD || *flags == SHUT_RDWR)
 	VCL_SESS_ATTR_SET (session->attr, VCL_SESS_ATTR_SHUT_RD);
@@ -3734,12 +3564,6 @@ vppcom_session_worker (vcl_session_handle_t session_handle)
 }
 
 int
-vppcom_session_handle (uint32_t session_index)
-{
-  return (vcl_get_worker_index () << 24) | session_index;
-}
-
-int
 vppcom_worker_register (void)
 {
   if (!vcl_worker_alloc_and_init ())
diff --git a/src/vcl/vppcom.h b/src/vcl/vppcom.h
index d82c9f9c58c..4db3d980099 100644
--- a/src/vcl/vppcom.h
+++ b/src/vcl/vppcom.h
@@ -152,7 +152,6 @@ typedef enum
   VPPCOM_ATTR_SET_TCP_KEEPINTVL,
   VPPCOM_ATTR_GET_TCP_USER_MSS,
   VPPCOM_ATTR_SET_TCP_USER_MSS,
-  VPPCOM_ATTR_GET_REFCNT,
   VPPCOM_ATTR_SET_SHUT,
   VPPCOM_ATTR_GET_SHUT,
 } vppcom_attr_op_t;
@@ -284,7 +283,6 @@ extern int vppcom_poll (vcl_poll_t * vp, uint32_t n_sids,
 extern int vppcom_mq_epoll_fd (void);
 extern int vppcom_session_index (vcl_session_handle_t session_handle);
 extern int vppcom_session_worker (vcl_session_handle_t session_handle);
-extern int vppcom_session_handle (uint32_t session_index);
 
 extern int vppcom_session_read_segments (uint32_t session_handle,
 					 vppcom_data_segments_t ds);
diff --git a/src/vppinfra/pool.h b/src/vppinfra/pool.h
index 10262e90ec7..747a7170800 100644
--- a/src/vppinfra/pool.h
+++ b/src/vppinfra/pool.h
@@ -352,18 +352,21 @@ do {									\
   typeof (P) _pool_var (new) = 0;					\
   pool_header_t * _pool_var (ph), * _pool_var (new_ph);			\
   u32 _pool_var (n) = pool_len (P);					\
-  _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n),	\
-				 _pool_var (n) * sizeof ((P)[0]), 	\
-				 pool_aligned_header_bytes, (A));	\
-  clib_memcpy_fast (_pool_var (new), (P), 				\
-		    _pool_var (n) * sizeof ((P)[0]));			\
-  _pool_var (ph) = pool_header (P);					\
-  _pool_var (new_ph) = pool_header (_pool_var (new));			\
-  _pool_var (new_ph)->free_bitmap = 					\
-    clib_bitmap_dup (_pool_var (ph)->free_bitmap);			\
-  _pool_var (new_ph)->free_indices = 					\
-    vec_dup (_pool_var (ph)->free_indices);				\
-  _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts;		\
+  if ((P))								\
+    {									\
+      _pool_var (new) = _vec_resize (_pool_var (new), _pool_var (n),	\
+                                     _pool_var (n) * sizeof ((P)[0]), 	\
+                                     pool_aligned_header_bytes, (A));	\
+      clib_memcpy_fast (_pool_var (new), (P), 				\
+                        _pool_var (n) * sizeof ((P)[0]));		\
+      _pool_var (ph) = pool_header (P);					\
+      _pool_var (new_ph) = pool_header (_pool_var (new));		\
+      _pool_var (new_ph)->free_bitmap = 				\
+        clib_bitmap_dup (_pool_var (ph)->free_bitmap);			\
+      _pool_var (new_ph)->free_indices = 				\
+        vec_dup (_pool_var (ph)->free_indices);				\
+      _pool_var (new_ph)->max_elts = _pool_var (ph)->max_elts;		\
+    }									\
   _pool_var (new);							\
 })