VCL_LDPRELOAD: mixed libc/vcl epoll fd's

Change-Id: Ibc82781ddef5ea0517220b9054db3d53ec348c6c
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

5 files changed, 176 insertions, 170 deletions

diff --git a/src/vcl/vcom.c b/src/vcl/vcom.c
index be1e4c5b47a..b215771e70c 100644
--- a/src/vcl/vcom.c
+++ b/src/vcl/vcom.c
@@ -2792,55 +2792,20 @@ epoll_ctl (int __epfd, int __op, int __fd, struct epoll_event *__event)
   int rv;
   pid_t pid = getpid ();
 
-  if (is_vcom_epfd (__epfd))
-    {
-      /* TBD: currently limiting epoll to support only vcom fds */
-      if (is_vcom_socket_fd (__fd))
-	{
-	  rv = vcom_epoll_ctl (__epfd, __op, __fd, __event);
-	  if (VCOM_DEBUG > 0)
-	    fprintf (stderr,
-		     "[%d] epoll_ctl: "
-		     "'%04d'='%04d', '%04d', '%04d'\n",
-		     pid, rv, __epfd, __op, __fd);
-	  if (rv != 0)
-	    {
-	      errno = -rv;
-	      return -1;
-	    }
-	  return 0;
-	}
-      else
-	{
-	  /*
-	   * TBD: currently epoll does not support kernel fds
-	   * or epoll fds */
-	  errno = EBADF;
-	  return -1;
-	}
-    }
-  else
+  rv = vcom_epoll_ctl (__epfd, __op, __fd, __event);
+  if (VCOM_DEBUG > 0)
+    fprintf (stderr,
+	     "[%d] epoll_ctl: "
+	     "'%04d'='%04d', '%04d', '%04d'\n", pid, rv, __epfd, __op, __fd);
+  if (rv != 0)
     {
-      /* epfd is not an epoll file descriptor */
-      errno = EINVAL;
+      errno = -rv;
       return -1;
     }
   return 0;
 }
 
 int
-vcom_epoll_wait (int __epfd, struct epoll_event *__events,
-		 int __maxevents, int __timeout)
-{
-  if (vcom_init () != 0)
-    {
-      return -1;
-    }
-
-  return vcom_epoll_pwait (__epfd, __events, __maxevents, __timeout, NULL);
-}
-
-int
 epoll_wait (int __epfd, struct epoll_event *__events,
 	    int __maxevents, int __timeout)
 {
@@ -2849,50 +2814,30 @@ epoll_wait (int __epfd, struct epoll_event *__events,
 
   if (__maxevents <= 0 || __maxevents > EP_MAX_EVENTS)
     {
+      fprintf (stderr, "[%d] ERROR: epoll_wait() invalid maxevents %d\n",
+	       pid, __maxevents);
       errno = EINVAL;
       return -1;
     }
 
-  if (is_vcom_epfd (__epfd))
-    {
-      rv = vcom_epoll_wait (__epfd, __events, __maxevents, __timeout);
-      if (VCOM_DEBUG > 0)
-	fprintf (stderr,
-		 "[%d] epoll_wait: "
-		 "'%04d'='%04d', '%p', "
-		 "'%04d', '%04d'\n",
-		 pid, rv, __epfd, __events, __maxevents, __timeout);
-      if (rv < 0)
-	{
-	  errno = -rv;
-	  return -1;
-	}
-      return rv;
-    }
-  else
+  rv =
+    vcom_socket_epoll_pwait (__epfd, __events, __maxevents, __timeout, NULL);
+  if (VCOM_DEBUG > 0)
+    fprintf (stderr,
+	     "[%d] epoll_wait: "
+	     "'%04d'='%04d', '%p', "
+	     "'%04d', '%04d'\n",
+	     pid, rv, __epfd, __events, __maxevents, __timeout);
+  if (rv < 0)
     {
-      errno = EINVAL;
+      errno = -rv;
       return -1;
     }
-  return 0;
+  return rv;
 }
 
 
 int
-vcom_epoll_pwait (int __epfd, struct epoll_event *__events,
-		  int __maxevents, int __timeout, const __sigset_t * __ss)
-{
-  if (vcom_init () != 0)
-    {
-      return -1;
-    }
-
-  /* implementation */
-  return vcom_socket_epoll_pwait (__epfd, __events,
-				  __maxevents, __timeout, __ss);
-}
-
-int
 epoll_pwait (int __epfd, struct epoll_event *__events,
 	     int __maxevents, int __timeout, const __sigset_t * __ss)
 {
@@ -2907,7 +2852,9 @@ epoll_pwait (int __epfd, struct epoll_event *__events,
 
   if (is_vcom_epfd (__epfd))
     {
-      rv = vcom_epoll_pwait (__epfd, __events, __maxevents, __timeout, __ss);
+      rv =
+	vcom_socket_epoll_pwait (__epfd, __events, __maxevents, __timeout,
+				 __ss);
       if (VCOM_DEBUG > 0)
 	fprintf (stderr,
 		 "[%d] epoll_pwait: "
diff --git a/src/vcl/vcom.h b/src/vcl/vcom.h
index da9dc5f7bf4..9a78b718f8e 100644
--- a/src/vcl/vcom.h
+++ b/src/vcl/vcom.h
@@ -16,9 +16,10 @@
 #ifndef included_vcom_h
 #define included_vcom_h
 
-/* VCOM DEBUG flag.  Setting this to 1 or 0 turns off
-   ASSERT & other debugging code. */
-#ifndef VCOM_DEBUG
+#if (CLIB_DEBUG > 0)
+/* Set VCOM_DEBUG 2 for connection debug, 3 for read/write debug output */
+#define VCOM_DEBUG 1
+#else
 #define VCOM_DEBUG 0
 #endif
 
diff --git a/src/vcl/vcom_socket.c b/src/vcl/vcom_socket.c
index de558e7beed..ece9b69a44b 100644
--- a/src/vcl/vcom_socket.c
+++ b/src/vcl/vcom_socket.c
@@ -21,6 +21,7 @@
 #include <netinet/tcp.h>
 
 #include <vppinfra/types.h>
+#include <vppinfra/time.h>
 #include <vppinfra/hash.h>
 #include <vppinfra/pool.h>
 
@@ -41,6 +42,8 @@
 typedef struct vcom_socket_main_t_
 {
   u8 init;
+  clib_time_t clib_time;
+  pid_t my_pid;
 
   /* vcom_socket pool */
   vcom_socket_t *vsockets;
@@ -54,7 +57,6 @@ typedef struct vcom_socket_main_t_
   /* Hash table for epollidx to epfd mapping */
   uword *epollidx_by_epfd;
 
-
   /* common epitem poll for all epfd */
   /* TBD: epitem poll per epfd */
   /* vcom_epitem pool */
@@ -1013,9 +1015,9 @@ vcom_socket_select (int vcom_nfds, fd_set * __restrict vcom_readfds,
 		    struct timeval *__restrict timeout)
 {
   static unsigned long vcom_nsid_fds = 0;
+  vcom_socket_main_t *vsm = &vcom_socket_main;
   int vcom_nsid = 0;
   int rv = -EBADF;
-  pid_t pid = getpid ();
 
   int new_vcom_nfds = 0;
   int new_vcom_nfd = 0;
@@ -1084,13 +1086,14 @@ vcom_socket_select (int vcom_nfds, fd_set * __restrict vcom_readfds,
 	  if (VCOM_DEBUG > 0)
 	    fprintf (stderr,
 		     "[%d] vcom_socket_select called to "
-		     "emulate delay_ns()!\n", pid);
+		     "emulate delay_ns()!\n", vsm->my_pid);
 	  rv = vppcom_select (0, NULL, NULL, NULL, time_to_wait);
 	}
       else
 	{
 	  fprintf (stderr, "[%d] vcom_socket_select called vcom_nfds = 0 "
-		   "and invalid time_to_wait (%f)!\n", pid, time_to_wait);
+		   "and invalid time_to_wait (%f)!\n",
+		   vsm->my_pid, time_to_wait);
 	}
       return 0;
     }
@@ -1125,7 +1128,7 @@ vcom_socket_select (int vcom_nfds, fd_set * __restrict vcom_readfds,
 		      NULL, time_to_wait);
   if (VCOM_DEBUG > 2)
     fprintf (stderr, "[%d] called vppcom_select(): "
-	     "'%04d'='%04d'\n", pid, rv, (int) vcom_nsid_fds);
+	     "'%04d'='%04d'\n", vsm->my_pid, rv, (int) vcom_nsid_fds);
 
   /* check if any file descriptors changed status */
   if (rv > 0)
@@ -2754,22 +2757,13 @@ vcom_socket_epoll_ctl_internal (int __epfd, int __op, int __fd,
 				struct epoll_event *__event,
 				int free_vepitem_on_del)
 {
+  vcom_socket_main_t *vsm = &vcom_socket_main;
   int rv = -1;
-
-  /* vcom_socket_main_t *vsm = &vcom_socket_main; */
+  i32 cnt;
   vcom_epoll_t *vepoll;
-
-  /*__fd could could be vcom socket or vcom epoll or kernel fd */
-  void *vfd;
-  vcom_epoll_t *vfd_vepoll;
   vcom_socket_t *vfd_vsock;
-
   i32 vep_idx;
-  i32 vfd_id;
-
-  vcom_fd_type_t type = FD_TYPE_INVALID;
-
-  /* validate __event */
+  i32 sid;
 
   /* get vep_idx and vepoll */
   vep_idx = vcom_socket_get_vep_idx_and_vepoll (__epfd, &vepoll);
@@ -2779,37 +2773,45 @@ vcom_socket_epoll_ctl_internal (int __epfd, int __op, int __fd,
     }
 
   /* get vcom fd type, vfd_id and vfd */
-  vfd_id = vcom_socket_get_sid_and_vsock (__fd, &vfd_vsock);
-  if (vfd_id != INVALID_SESSION_ID)
-    {
-      type = FD_TYPE_VCOM_SOCKET;
-      vfd = vfd_vsock;
-    }
-  else if ((vfd_id = vcom_socket_get_vep_idx_and_vepoll (__fd, &vfd_vepoll))
-	   != INVALID_VEP_IDX)
+  sid = vcom_socket_get_sid_and_vsock (__fd, &vfd_vsock);
+  if ((sid != INVALID_SESSION_ID) &&
+      vcom_socket_type_is_vppcom_bound (vfd_vsock->type))
     {
-      type = FD_TYPE_EPOLL;
-      vfd = vfd_vepoll;
+      rv = vppcom_epoll_ctl (vep_idx, __op, sid, __event);
+      if (rv == VPPCOM_OK)
+	{
+	  cnt = ((__op == EPOLL_CTL_ADD) ? 1 :
+		 (__op == EPOLL_CTL_DEL) ? -1 : 0);
+	  vepoll->count += cnt;
+	  vepoll->vcl_cnt += cnt;
+	}
+      if (VCOM_DEBUG > 0)
+	fprintf (stderr,
+		 "[%d] vcom_socket_epoll_ctl_i: vppcom_epoll_ctl() "
+		 "returned %d\n\tepfd %d, vep_idx %d, fd %d sid %d op %d"
+		 "\n\tcount %d, vcl_cnt %d, libc_cnt %d\n",
+		 vsm->my_pid, rv, __epfd, vep_idx, __fd, sid, __op,
+		 vepoll->count, vepoll->vcl_cnt, vepoll->libc_cnt);
     }
   else
     {
-      /* FD_TYPE_KERNEL not supported by epoll instance */
-      type = FD_TYPE_INVALID;
-      return -EBADF;
-    }
-
-
-  /* vepoll and vsock are now valid */
-  rv = vppcom_epoll_ctl (vep_idx, __op, vfd_id, __event);
-  if (rv < 0)
-    {
-      return rv;
+      rv = libc_epoll_ctl (__epfd, __op, __fd, __event);
+      if (rv == 0)
+	{
+	  cnt = ((__op == EPOLL_CTL_ADD) ? 1 :
+		 (__op == EPOLL_CTL_DEL) ? -1 : 0);
+	  vepoll->count += cnt;
+	  vepoll->libc_cnt += cnt;
+	}
+      if (VCOM_DEBUG > 0)
+	fprintf (stderr,
+		 "[%d] vcom_socket_epoll_ctl_i: libc_epoll_ctl() "
+		 "returned %d\n\tepfd %d, vep_idx %d, fd %d sid %d op %d"
+		 "\n\tcount %d, vcl_cnt %d, libc_cnt %d\n",
+		 vsm->my_pid, rv, __epfd, vep_idx, __fd, sid, __op,
+		 vepoll->count, vepoll->vcl_cnt, vepoll->libc_cnt);
     }
 
-  rv = vcom_socket_ctl_vepitem (__epfd, __op, __fd,
-				__event,
-				vep_idx, vepoll,
-				vfd_id, vfd, type, free_vepitem_on_del);
   return rv;
 }
 
@@ -2838,46 +2840,93 @@ vcom_socket_epoll_pwait (int __epfd, struct epoll_event *__events,
 			 int __maxevents, int __timeout,
 			 const __sigset_t * __ss)
 {
+  vcom_socket_main_t *vsm = &vcom_socket_main;
   int rv = -EBADF;
-
-  /* in seconds eg. 3.123456789 seconds */
+  int rv2;
   double time_to_wait = (double) 0;
-
+  double timeout;
+  vcom_epoll_t *vepoll;
   i32 vep_idx;
+  static struct epoll_event *libc_ev = 0;
 
   /* validate __event */
-  if (!__events)
+  if (!__events || (__timeout < -1))
     {
+      fprintf (stderr, "[%d] ERROR: vcom_socket_epoll_pwait: "
+	       "Bad args __events %p, __timeout %d\n", vsm->my_pid,
+	       __events, __timeout);
       rv = -EFAULT;
       goto out;
     }
 
-  /* validate __timeout */
-  if (__timeout > 0)
+  time_to_wait = ((__timeout > 0) ?
+		  (double) __timeout / (double) 1000 : (double) __timeout);
+
+  vep_idx = vcom_socket_get_vep_idx_and_vepoll (__epfd, &vepoll);
+  if (vep_idx == INVALID_VEP_IDX)
     {
-      time_to_wait = (double) __timeout / (double) 1000;
+      fprintf (stderr, "[%d] ERROR: vcom_socket_epoll_pwait: "
+	       "Bad epoll fd %d\n", vsm->my_pid, __epfd);
+      return -EBADF;
     }
-  else if (__timeout == 0)
+
+  if (vepoll->count <= 0)
     {
-      time_to_wait = (double) 0;
+      fprintf (stderr, "[%d] ERROR: vcom_socket_epoll_pwait: No events"
+	       " in epfd!\n\tcount %d, vcl_cnt %d, libc_cnt %d\n",
+	       vsm->my_pid, vepoll->count, vepoll->vcl_cnt, vepoll->libc_cnt);
+      rv = -EINVAL;
+      goto out;
     }
-  else if (__timeout == -1)
+
+  if (vepoll->libc_cnt == 0)
     {
-      time_to_wait = ~0;
+      rv = vppcom_epoll_wait (vep_idx, __events, __maxevents, time_to_wait);
     }
-  else
+  else if (vepoll->vcl_cnt == 0)
     {
-      rv = -EBADF;
-      goto out;
+      rv = libc_epoll_pwait (__epfd, __events, __maxevents, __timeout, __ss);
     }
-
-  /* get vep_idx */
-  vep_idx = vcom_socket_get_vep_idx (__epfd);
-  if (vep_idx != INVALID_VEP_IDX)
+  else
     {
-      rv = vppcom_epoll_wait (vep_idx, __events, __maxevents, time_to_wait);
+      vec_validate (libc_ev, __maxevents);
+      timeout = clib_time_now (&vsm->clib_time) + time_to_wait;
+      do
+	{
+	  rv = vppcom_epoll_wait (vep_idx, __events, __maxevents, 0);
+	  rv2 = libc_epoll_pwait (__epfd, libc_ev, __maxevents, 1, __ss);
+	  if ((rv > 0) || (rv2 > 0))
+	    {
+	      int n = __maxevents - rv;
+	      n = rv2 <= n ? rv2 : n;
+	      rv = (rv > 0) ? rv : 0;
+
+	      clib_memcpy (&__events[rv], libc_ev, n * sizeof (*libc_ev));
+	      rv += rv2;
+	      goto out;
+	    }
+	  else if ((rv < 0) || (rv2 < 0))
+	    {
+	      if (rv < 0)
+		fprintf (stderr,
+			 "[%d] ERROR: vppcom_epoll_wait() returned %d\n",
+			 vsm->my_pid, rv);
+	      if (rv2 < 0)
+		{
+		  fprintf (stderr,
+			   "[%d] ERROR: libc_epoll_wait() failed, errno %d\n",
+			   vsm->my_pid, errno);
+		  rv = (rv < 0) ? rv : -errno;
+		}
+	      goto out;
+	    }
+	}
+      while ((__timeout == -1)
+	     || (clib_time_now (&vsm->clib_time) < timeout));
     }
+
 out:
+  vec_reset_length (libc_ev);
   return rv;
 }
 
@@ -3067,7 +3116,7 @@ vcom_socket_poll_select_impl (struct pollfd *__fds, nfds_t __nfds,
 			      int __timeout)
 {
   int rv;
-  pid_t pid = getpid ();
+  vcom_socket_main_t *vsm = &vcom_socket_main;
 
   nfds_t fds_idx = 0;
   int nfd = 0;
@@ -3156,7 +3205,7 @@ vcom_socket_poll_select_impl (struct pollfd *__fds, nfds_t __nfds,
   if (VCOM_DEBUG > 2)
     fprintf (stderr,
 	     "[%d] vcom_socket_select: "
-	     "'%04d'='%04d'\n", pid, vcom_nfd, vcom_nfds);
+	     "'%04d'='%04d'\n", vsm->my_pid, vcom_nfd, vcom_nfds);
 
   if (vcom_nfd < 0)
     {
@@ -3289,6 +3338,9 @@ vcom_socket_main_init (void)
       vsm->epitemidxs_by_epfd = hash_create (0, sizeof (i32 *));
       vsm->epitemidxs_by_fd = hash_create (0, sizeof (i32 *));
 
+      clib_time_init (&vsm->clib_time);
+      vsm->my_pid = getpid ();
+
       vsm->init = 1;
     }
 
diff --git a/src/vcl/vcom_socket.h b/src/vcl/vcom_socket.h
index 4f5e4abfb15..e86bab35672 100644
--- a/src/vcl/vcom_socket.h
+++ b/src/vcl/vcom_socket.h
@@ -97,6 +97,8 @@ typedef struct
    * 03. cached for frequent access.
    * */
   i32 count;
+  i32 vcl_cnt;
+  i32 libc_cnt;
 
   /* close( ) called on this epoll instance */
   /* 0 - close ( ) not called, 1 - close( ) called. */
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index 9501b7e4112..f5f76738f7f 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -2166,6 +2166,15 @@ vppcom_session_listen (uint32_t listen_session_index, uint32_t q_len)
       return VPPCOM_EBADFD;
     }
 
+  if (listen_session->is_listen)
+    {
+      clib_spinlock_unlock (&vcm->sessions_lockp);
+      if (VPPCOM_DEBUG > 0)
+	clib_warning ("[%d] sid (%u) is already in listen state!",
+		      vcm->my_pid, listen_session_index);
+      return VPPCOM_OK;
+    }
+
   if (VPPCOM_DEBUG > 0)
     clib_warning ("[%d] sid %d", vcm->my_pid, listen_session_index);
 
@@ -2820,25 +2829,24 @@ vep_verify_epoll_chain (u32 vep_idx)
   do
     {
       vep = &session->vep;
-      sid = vep->next_sid;
-      if (session->is_vep_session)
+      if (session->is_vep_session && (VPPCOM_DEBUG > 1))
 	{
-	  if (VPPCOM_DEBUG > 1)
-	    clib_warning ("vep_idx[%u]: sid 0x%x (%u)\n"
-			  "{\n"
-			  "   next_sid       = 0x%x (%u)\n"
-			  "   prev_sid       = 0x%x (%u)\n"
-			  "   vep_idx        = 0x%x (%u)\n"
-			  "   ev.events      = 0x%x\n"
-			  "   ev.data.u64    = 0x%llx\n"
-			  "   et_mask        = 0x%x\n"
-			  "}\n",
-			  vep_idx, sid, sid,
-			  vep->next_sid, vep->next_sid,
-			  vep->prev_sid, vep->prev_sid,
-			  vep->vep_idx, vep->vep_idx,
-			  vep->ev.events, vep->ev.data.u64, vep->et_mask);
+	  clib_warning ("vep_idx[%u]: sid 0x%x (%u)\n"
+			"{\n"
+			"   next_sid       = 0x%x (%u)\n"
+			"   prev_sid       = 0x%x (%u)\n"
+			"   vep_idx        = 0x%x (%u)\n"
+			"   ev.events      = 0x%x\n"
+			"   ev.data.u64    = 0x%llx\n"
+			"   et_mask        = 0x%x\n"
+			"}\n",
+			vep_idx, sid, sid,
+			vep->next_sid, vep->next_sid,
+			vep->prev_sid, vep->prev_sid,
+			vep->vep_idx, vep->vep_idx,
+			vep->ev.events, vep->ev.data.u64, vep->et_mask);
 	}
+      sid = vep->next_sid;
       if (sid != ~0)
 	{
 	  rv = vppcom_session_at_index (sid, &session);
@@ -3140,13 +3148,6 @@ vppcom_epoll_wait (uint32_t vep_idx, struct epoll_event *events,
 		      vcm->my_pid, maxevents);
       return VPPCOM_EINVAL;
     }
-  if (PREDICT_FALSE (wait_for_time < 0))
-    {
-      if (VPPCOM_DEBUG > 0)
-	clib_warning ("[%d] ERROR: Invalid wait_for_time (%f)!",
-		      vcm->my_pid, wait_for_time);
-      return VPPCOM_EINVAL;
-    }
   memset (events, 0, sizeof (*events) * maxevents);
 
   VCL_LOCK_AND_GET_SESSION (vep_idx, &vep_session);
@@ -3163,10 +3164,11 @@ vppcom_epoll_wait (uint32_t vep_idx, struct epoll_event *events,
       rv = VPPCOM_EINVAL;
       goto done;
     }
-  if ((VPPCOM_DEBUG > 0) && (PREDICT_FALSE (vep_next_sid == ~0)))
+  if (PREDICT_FALSE (vep_next_sid == ~0))
     {
-      clib_warning ("[%d] WARNING: vep_idx (%u) is empty!",
-		    vcm->my_pid, vep_idx);
+      if (VPPCOM_DEBUG > 0)
+	clib_warning ("[%d] WARNING: vep_idx (%u) is empty!",
+		      vcm->my_pid, vep_idx);
       goto done;
     }
 
@@ -3304,7 +3306,9 @@ vppcom_epoll_wait (uint32_t vep_idx, struct epoll_event *events,
 	    }
 	}
     }
-  while ((num_ev == 0) && (clib_time_now (&vcm->clib_time) <= timeout));
+  while ((num_ev == 0) &&
+	 ((wait_for_time == -1) ||
+	  (clib_time_now (&vcm->clib_time) <= timeout)));
 
   if (wait_cont_idx != ~0)
     {