vcl/ldp: fix poll

Change-Id: I8eb5546ff8634d5498d8ce5bbc9407bceb9ae3ef
Signed-off-by: Florin Coras <fcoras@cisco.com>

3 files changed, 80 insertions, 70 deletions

diff --git a/src/vcl/ldp.c b/src/vcl/ldp.c
index b7b6b3c26b1..96a7c36acef 100644
--- a/src/vcl/ldp.c
+++ b/src/vcl/ldp.c
@@ -65,6 +65,8 @@ typedef struct
   clib_bitmap_t *libc_wr_bitmap;
   clib_bitmap_t *libc_ex_bitmap;
   vcl_poll_t *vcl_poll;
+  struct pollfd *libc_poll;
+  u16 *libc_poll_idxs;
   u8 select_vcl;
   u8 epoll_wait_vcl;
   u8 vcl_needs_real_epoll;	/*< vcl needs next epoll_create to
@@ -245,10 +247,8 @@ close (int fd)
 	{
 	  func_str = "libc_close";
 
-	  if (LDP_DEBUG > 0)
-	    clib_warning
-	      ("LDP<%d>: fd %d (0x%x): calling %s(): epfd %u (0x%x)",
-	       getpid (), fd, fd, func_str, epfd, epfd);
+	  LDBG (0, "LDP<%d>: fd %d (0x%x): calling %s(): epfd %u (0x%x)",
+		getpid (), fd, fd, func_str, epfd, epfd);
 
 	  rv = libc_close (epfd);
 	  if (rv < 0)
@@ -269,9 +269,8 @@ close (int fd)
 
       func_str = "vppcom_session_close";
 
-      if (LDP_DEBUG > 0)
-	clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x)",
-		      getpid (), fd, fd, func_str, sid, sid);
+      LDBG (0, "LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x)",
+	    getpid (), fd, fd, func_str, sid, sid);
 
       rv = vppcom_session_close (sid);
       if (rv != VPPCOM_OK)
@@ -284,9 +283,8 @@ close (int fd)
     {
       func_str = "libc_close";
 
-      if (LDP_DEBUG > 0)
-	clib_warning ("LDP<%d>: fd %d (0x%x): calling %s()",
-		      getpid (), fd, fd, func_str);
+      LDBG (0, "LDP<%d>: fd %d (0x%x): calling %s()", getpid (), fd, fd,
+	    func_str);
 
       rv = libc_close (fd);
     }
@@ -2913,11 +2911,8 @@ shutdown (int fd, int how)
 
   if (sid != INVALID_SESSION_ID)
     {
-      func_str = __func__;
-
-      clib_warning ("LDP<%d>: LDP-TBD", getpid ());
-      errno = ENOSYS;
-      rv = -1;
+      func_str = "vppcom_session_close[TODO]";
+      rv = close (fd);
     }
   else
     {
@@ -3253,65 +3248,61 @@ int
 poll (struct pollfd *fds, nfds_t nfds, int timeout)
 {
   const char *func_str = __func__;
-  int rv, i, n_libc_fds, n_revents;
+  int rv, i, n_revents = 0;
   u32 sid;
   vcl_poll_t *vp;
   double wait_for_time;
 
-  if (LDP_DEBUG > 3)
-    clib_warning ("LDP<%d>: fds %p, nfds %d, timeout %d",
-		  getpid (), fds, nfds, timeout);
+  LDBG (3, "LDP<%d>: fds %p, nfds %d, timeout %d", getpid (), fds, nfds,
+	timeout);
 
   if (timeout >= 0)
     wait_for_time = (f64) timeout / 1000;
   else
     wait_for_time = -1;
 
-  n_libc_fds = 0;
   for (i = 0; i < nfds; i++)
     {
-      if (fds[i].fd >= 0)
-	{
-	  if (LDP_DEBUG > 3)
-	    clib_warning ("LDP<%d>: fds[%d].fd %d (0x%0x), .events = 0x%x, "
-			  ".revents = 0x%x", getpid (), i, fds[i].fd,
-			  fds[i].fd, fds[i].events, fds[i].revents);
+      if (fds[i].fd < 0)
+	continue;
 
-	  sid = ldp_sid_from_fd (fds[i].fd);
-	  if (sid != INVALID_SESSION_ID)
-	    {
-	      fds[i].fd = -fds[i].fd;
-	      vec_add2 (ldp->vcl_poll, vp, 1);
-	      vp->fds_ndx = i;
-	      vp->sid = sid;
-	      vp->events = fds[i].events;
+      LDBG (3, "LDP<%d>: fds[%d] fd %d (0x%0x) events = 0x%x revents = 0x%x",
+	    getpid (), i, fds[i].fd, fds[i].fd, fds[i].events,
+	    fds[i].revents);
+
+      sid = ldp_sid_from_fd (fds[i].fd);
+      if (sid != INVALID_SESSION_ID)
+	{
+	  fds[i].fd = -fds[i].fd;
+	  vec_add2 (ldp->vcl_poll, vp, 1);
+	  vp->fds_ndx = i;
+	  vp->sid = sid;
+	  vp->events = fds[i].events;
 #ifdef __USE_XOPEN2K
-	      if (fds[i].events & POLLRDNORM)
-		vp->events |= POLLIN;
-	      if (fds[i].events & POLLWRNORM)
-		vp->events |= POLLOUT;
+	  if (fds[i].events & POLLRDNORM)
+	    vp->events |= POLLIN;
+	  if (fds[i].events & POLLWRNORM)
+	    vp->events |= POLLOUT;
 #endif
-	      vp->revents = &fds[i].revents;
-	    }
-	  else
-	    n_libc_fds++;
+	  vp->revents = fds[i].revents;
+	}
+      else
+	{
+	  vec_add1 (ldp->libc_poll, fds[i]);
+	  vec_add1 (ldp->libc_poll_idxs, i);
 	}
     }
 
-  n_revents = 0;
   do
     {
       if (vec_len (ldp->vcl_poll))
 	{
 	  func_str = "vppcom_poll";
 
-	  if (LDP_DEBUG > 3)
-	    clib_warning ("LDP<%d>: calling %s(): "
-			  "vcl_poll %p, n_sids %u (0x%x): "
-			  "n_libc_fds %u",
-			  getpid (), func_str, ldp->vcl_poll,
-			  vec_len (ldp->vcl_poll), vec_len (ldp->vcl_poll),
-			  n_libc_fds);
+	  LDBG (3, "LDP<%d>: calling %s(): vcl_poll %p, n_sids %u (0x%x): "
+		"n_libc_fds %u", getpid (), func_str, ldp->vcl_poll,
+		vec_len (ldp->vcl_poll), vec_len (ldp->vcl_poll),
+		vec_len (ldp->libc_poll));
 
 	  rv = vppcom_poll (ldp->vcl_poll, vec_len (ldp->vcl_poll), 0);
 	  if (rv < 0)
@@ -3324,15 +3315,14 @@ poll (struct pollfd *fds, nfds_t nfds, int timeout)
 	    n_revents += rv;
 	}
 
-      if (n_libc_fds)
+      if (vec_len (ldp->libc_poll))
 	{
 	  func_str = "libc_poll";
 
-	  if (LDP_DEBUG > 3)
-	    clib_warning ("LDP<%d>: calling %s(): fds %p, nfds %u: n_sids %u",
-			  getpid (), fds, nfds, vec_len (ldp->vcl_poll));
+	  LDBG (3, "LDP<%d>: calling %s(): fds %p, nfds %u: n_sids %u",
+		getpid (), fds, nfds, vec_len (ldp->vcl_poll));
 
-	  rv = libc_poll (fds, nfds, 0);
+	  rv = libc_poll (ldp->libc_poll, vec_len (ldp->libc_poll), 0);
 	  if (rv < 0)
 	    goto done;
 	  else
@@ -3353,6 +3343,7 @@ done:
   vec_foreach (vp, ldp->vcl_poll)
   {
     fds[vp->fds_ndx].fd = -fds[vp->fds_ndx].fd;
+    fds[vp->fds_ndx].revents = vp->revents;
 #ifdef __USE_XOPEN2K
     if ((fds[vp->fds_ndx].revents & POLLIN) &&
 	(fds[vp->fds_ndx].events & POLLRDNORM))
@@ -3364,6 +3355,13 @@ done:
   }
   vec_reset_length (ldp->vcl_poll);
 
+  for (i = 0; i < vec_len (ldp->libc_poll); i++)
+    {
+      fds[ldp->libc_poll_idxs[i]].revents = ldp->libc_poll[i].revents;
+    }
+  vec_reset_length (ldp->libc_poll_idxs);
+  vec_reset_length (ldp->libc_poll);
+
   if (LDP_DEBUG > 3)
     {
       if (rv < 0)
@@ -3379,7 +3377,7 @@ done:
 	{
 	  clib_warning ("LDP<%d>: returning %d (0x%x): n_sids %u, "
 			"n_libc_fds %d", getpid (), rv, rv,
-			vec_len (ldp->vcl_poll), n_libc_fds);
+			vec_len (ldp->vcl_poll), vec_len (ldp->libc_poll));
 
 	  for (i = 0; i < nfds; i++)
 	    {
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index fad2ac98538..46b1c106d20 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -3185,6 +3185,8 @@ vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time)
   vcl_worker_t *wrk = vcl_worker_get_current ();
   f64 timeout = clib_time_now (&wrk->clib_time) + wait_for_time;
   u32 i, keep_trying = 1;
+  svm_msg_q_msg_t msg;
+  session_event_t *e;
   int rv, num_ev = 0;
 
   VDBG (3, "VCL<%d>: vp %p, nsids %u, wait_for_time %f",
@@ -3197,23 +3199,33 @@ vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time)
     {
       vcl_session_t *session;
 
-      for (i = 0; i < n_sids; i++)
+      /* Dequeue all events and drop all unhandled io events */
+      while (svm_msg_q_sub (wrk->app_event_queue, &msg, SVM_Q_NOWAIT, 0) == 0)
 	{
-	  ASSERT (vp[i].revents);
+	  e = svm_msg_q_msg_data (wrk->app_event_queue, &msg);
+	  vcl_handle_mq_event (wrk, e);
+	  svm_msg_q_free_msg (wrk->app_event_queue, &msg);
+	}
+      vec_reset_length (wrk->unhandled_evts_vector);
 
+      for (i = 0; i < n_sids; i++)
+	{
 	  session = vcl_session_get (wrk, vp[i].sid);
 	  if (!session)
-	    continue;
+	    {
+	      vp[i].revents = POLLHUP;
+	      num_ev++;
+	      continue;
+	    }
 
-	  if (*vp[i].revents)
-	    *vp[i].revents = 0;
+	  vp[i].revents = 0;
 
 	  if (POLLIN & vp[i].events)
 	    {
 	      rv = vppcom_session_read_ready (session);
 	      if (rv > 0)
 		{
-		  *vp[i].revents |= POLLIN;
+		  vp[i].revents |= POLLIN;
 		  num_ev++;
 		}
 	      else if (rv < 0)
@@ -3221,11 +3233,11 @@ vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time)
 		  switch (rv)
 		    {
 		    case VPPCOM_ECONNRESET:
-		      *vp[i].revents = POLLHUP;
+		      vp[i].revents = POLLHUP;
 		      break;
 
 		    default:
-		      *vp[i].revents = POLLERR;
+		      vp[i].revents = POLLERR;
 		      break;
 		    }
 		  num_ev++;
@@ -3237,7 +3249,7 @@ vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time)
 	      rv = vppcom_session_write_ready (session);
 	      if (rv > 0)
 		{
-		  *vp[i].revents |= POLLOUT;
+		  vp[i].revents |= POLLOUT;
 		  num_ev++;
 		}
 	      else if (rv < 0)
@@ -3245,11 +3257,11 @@ vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time)
 		  switch (rv)
 		    {
 		    case VPPCOM_ECONNRESET:
-		      *vp[i].revents = POLLHUP;
+		      vp[i].revents = POLLHUP;
 		      break;
 
 		    default:
-		      *vp[i].revents = POLLERR;
+		      vp[i].revents = POLLERR;
 		      break;
 		    }
 		  num_ev++;
@@ -3258,7 +3270,7 @@ vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time)
 
 	  if (0)		// Note "done:" label used by VCL_SESSION_LOCK_AND_GET()
 	    {
-	      *vp[i].revents = POLLNVAL;
+	      vp[i].revents = POLLNVAL;
 	      num_ev++;
 	    }
 	}
@@ -3274,7 +3286,7 @@ vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time)
 	{
 	  clib_warning ("VCL<%d>: vp[%d].sid %d (0x%x), .events 0x%x, "
 			".revents 0x%x", getpid (), i, vp[i].sid, vp[i].sid,
-			vp[i].events, *vp[i].revents);
+			vp[i].events, vp[i].revents);
 	}
     }
   return num_ev;
diff --git a/src/vcl/vppcom.h b/src/vcl/vppcom.h
index 4c7d2092599..5f1992502ce 100644
--- a/src/vcl/vppcom.h
+++ b/src/vcl/vppcom.h
@@ -144,7 +144,7 @@ typedef struct _vcl_poll
   uint32_t fds_ndx;
   uint32_t sid;
   short events;
-  short *revents;
+  short revents;
 } vcl_poll_t;
 
 typedef struct vppcom_data_segment_