vcl: basic support for epoll level-triggered evts

Type: feature

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I2d5039cd35edd02ffa2439bcac119d0647234385

5 files changed, 108 insertions, 11 deletions

diff --git a/src/plugins/hs_apps/vcl/sock_test_server.c b/src/plugins/hs_apps/vcl/sock_test_server.c
index 6e4d6ea17fd..d516c1722db 100644
--- a/src/plugins/hs_apps/vcl/sock_test_server.c
+++ b/src/plugins/hs_apps/vcl/sock_test_server.c
@@ -292,7 +292,7 @@ new_client (void)
   struct epoll_event ev;
   int rv;
 
-  ev.events = EPOLLIN;
+  ev.events = EPOLLET | EPOLLIN;
   ev.data.u64 = conn - ssm->conn_pool;
   rv = epoll_ctl (ssm->epfd, EPOLL_CTL_ADD, client_fd, &ev);
 
@@ -329,7 +329,7 @@ socket_server_echo_af_unix_init (sock_server_main_t * ssm)
   if (rv < 0)
     stfail ("echo_af_unix_init listen()");
 
-  ssm->af_unix_listen_ev.events = EPOLLIN;
+  ssm->af_unix_listen_ev.events = EPOLLET | EPOLLIN;
   ssm->af_unix_listen_ev.data.u32 = SOCK_TEST_AF_UNIX_ACCEPT_DATA;
   rv = epoll_ctl (ssm->epfd, EPOLL_CTL_ADD, ssm->af_unix_listen_fd,
 		  &ssm->af_unix_listen_ev);
@@ -542,7 +542,7 @@ main (int argc, char **argv)
   if (ssm->epfd < 0)
     stfail ("main epoll_create()");
 
-  ssm->listen_ev.events = EPOLLIN;
+  ssm->listen_ev.events = EPOLLET | EPOLLIN;
   ssm->listen_ev.data.u32 = ~0;
 
   rv = epoll_ctl (ssm->epfd, EPOLL_CTL_ADD, ssm->listen_fd, &ssm->listen_ev);
diff --git a/src/plugins/hs_apps/vcl/vcl_test_server.c b/src/plugins/hs_apps/vcl/vcl_test_server.c
index ca21a4bfe08..78d7752dd68 100644
--- a/src/plugins/hs_apps/vcl/vcl_test_server.c
+++ b/src/plugins/hs_apps/vcl/vcl_test_server.c
@@ -334,7 +334,7 @@ vts_accept_client (vcl_test_server_worker_t *wrk, int listen_fd)
   vtinf ("Got a connection -- fd = %d (0x%08x) on listener fd = %d (0x%08x)",
 	 conn->fd, conn->fd, listen_fd, listen_fd);
 
-  ev.events = EPOLLIN;
+  ev.events = EPOLLET | EPOLLIN;
   ev.data.u64 = conn - wrk->conn_pool;
   rv = vppcom_epoll_ctl (wrk->epfd, EPOLL_CTL_ADD, conn->fd, &ev);
   if (rv < 0)
@@ -561,7 +561,7 @@ vts_worker_init (vcl_test_server_worker_t * wrk)
 	vtfail ("vppcom_epoll_create()", wrk->epfd);
     }
 
-  listen_ev.events = EPOLLIN;
+  listen_ev.events = EPOLLET | EPOLLIN;
   listen_ev.data.u32 = VCL_TEST_DATA_LISTENER;
   rv =
     vppcom_epoll_ctl (wrk->epfd, EPOLL_CTL_ADD, wrk->listener.fd, &listen_ev);
@@ -769,7 +769,7 @@ vts_ctrl_session_init (vcl_test_server_worker_t *wrk)
   if (wrk->epfd < 0)
     vtfail ("vppcom_epoll_create()", wrk->epfd);
 
-  listen_ev.events = EPOLLIN;
+  listen_ev.events = EPOLLET | EPOLLIN;
   listen_ev.data.u32 = VCL_TEST_CTRL_LISTENER;
   rv = vppcom_epoll_ctl (wrk->epfd, EPOLL_CTL_ADD, vsm->ctrl_listen_fd,
 			 &listen_ev);
diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c
index 3538a092bac..45c208d87f7 100644
--- a/src/vcl/vcl_private.c
+++ b/src/vcl/vcl_private.c
@@ -131,6 +131,8 @@ vcl_worker_cleanup (vcl_worker_t * wrk, u8 notify_vpp)
   hash_free (wrk->session_index_by_vpp_handles);
   vec_free (wrk->mq_events);
   vec_free (wrk->mq_msg_vector);
+  vec_free (wrk->ep_level_evts);
+  vec_free (wrk->ep_level_evts_fl);
   vcl_worker_free (wrk);
   clib_spinlock_unlock (&vcm->workers_lock);
 }
diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h
index 5b19f946fe7..ab3ecab36fa 100644
--- a/src/vcl/vcl_private.h
+++ b/src/vcl/vcl_private.h
@@ -253,6 +253,12 @@ typedef struct vcl_worker_
   /** Per worker buffer for receiving mq epoll events */
   struct epoll_event *mq_events;
 
+  /** Vector of session indices recently notified of epoll level events */
+  u32 *ep_level_evts;
+
+  /** Storage for level events session while new ones are processed */
+  u32 *ep_level_evts_fl;
+
   /** Hash table for disconnect processing */
   uword *session_index_by_vpp_handles;
 
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index d378f404381..f96ceea4385 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -1445,6 +1445,18 @@ vppcom_session_create (u8 proto, u8 is_nonblocking)
   return vcl_session_handle (session);
 }
 
+static void
+vcl_epoll_wait_clean_lt (vcl_worker_t *wrk, u32 sid)
+{
+  int i;
+
+  for (i = vec_len (wrk->ep_level_evts) - 1; i >= 0; i--)
+    {
+      if (wrk->ep_level_evts[i] == sid)
+	vec_del1 (wrk->ep_level_evts, i);
+    }
+}
+
 int
 vcl_session_cleanup (vcl_worker_t * wrk, vcl_session_t * s,
 		     vcl_session_handle_t sh, u8 do_disconnect)
@@ -1475,6 +1487,8 @@ vcl_session_cleanup (vcl_worker_t * wrk, vcl_session_t * s,
 	VDBG (0, "session %u [0x%llx]: EPOLL_CTL_DEL vep_idx %u "
 	      "failed! rv %d (%s)", s->session_index, s->vpp_handle,
 	      s->vep.vep_sh, rv, vppcom_retval_str (rv));
+      if (PREDICT_FALSE (vec_len (wrk->ep_level_evts)))
+	vcl_epoll_wait_clean_lt (wrk, s->session_index);
     }
 
   if (!do_disconnect)
@@ -3063,6 +3077,10 @@ vcl_epoll_wait_handle_mq_event (vcl_worker_t * wrk, session_event_t * e,
 	  s = vcl_session_get (wrk, sid);
 	  s->vep.ev.events = 0;
 	}
+      if (!(EPOLLET & session_events))
+	{
+	  vec_add1 (wrk->ep_level_evts, sid);
+	}
       *num_ev += 1;
     }
 }
@@ -3177,13 +3195,73 @@ vppcom_epoll_wait_eventfd (vcl_worker_t *wrk, struct epoll_event *events,
   return 0;
 }
 
+static void
+vcl_epoll_swap_lt_lists (vcl_worker_t *wrk)
+{
+  u32 *le;
+
+  le = wrk->ep_level_evts;
+  wrk->ep_level_evts = wrk->ep_level_evts_fl;
+  wrk->ep_level_evts_fl = le;
+}
+
+static void
+vcl_epoll_wait_handle_lt (vcl_worker_t *wrk, struct epoll_event *events,
+			  int maxevents, u32 *n_evts)
+{
+  u32 *sid, add_event = 0, *le = wrk->ep_level_evts_fl;
+  vcl_session_t *s;
+  u64 evt_data;
+
+  if (*n_evts >= maxevents)
+    {
+      vec_add (wrk->ep_level_evts, le, vec_len (le));
+      vec_reset_length (wrk->ep_level_evts_fl);
+      return;
+    }
+
+  vec_foreach (sid, le)
+    {
+      s = vcl_session_get (wrk, sid[0]);
+      if (!s)
+	continue;
+      if ((s->vep.ev.events & EPOLLIN) && vcl_session_read_ready (s))
+	{
+	  add_event = 1;
+	  events[*n_evts].events |= EPOLLIN;
+	  evt_data = s->vep.ev.data.u64;
+	}
+      if ((s->vep.ev.events & EPOLLOUT) && vcl_session_write_ready (s))
+	{
+	  add_event = 1;
+	  events[*n_evts].events |= EPOLLOUT;
+	  evt_data = s->vep.ev.data.u64;
+	}
+      if (add_event)
+	{
+	  events[*n_evts].data.u64 = evt_data;
+	  *n_evts += 1;
+	  add_event = 0;
+	  vec_add1 (wrk->ep_level_evts, sid[0]);
+	  if (*n_evts == maxevents)
+	    {
+	      u32 pos = (sid - le) + 1;
+	      vec_add (wrk->ep_level_evts, &le[pos], vec_len (le) - pos);
+	      break;
+	    }
+	}
+    }
+
+  vec_reset_length (wrk->ep_level_evts_fl);
+}
+
 int
 vppcom_epoll_wait (uint32_t vep_handle, struct epoll_event *events,
 		   int maxevents, double wait_for_time)
 {
   vcl_worker_t *wrk = vcl_worker_get_current ();
   vcl_session_t *vep_session;
-  u32 n_evts = 0;
+  u32 n_evts = 0, do_lt = 0;
   int i;
 
   if (PREDICT_FALSE (maxevents <= 0))
@@ -3222,12 +3300,23 @@ vppcom_epoll_wait (uint32_t vep_handle, struct epoll_event *events,
   if ((int) wait_for_time == -2)
     return n_evts;
 
+  if (PREDICT_FALSE (vec_len (wrk->ep_level_evts)))
+    {
+      vcl_epoll_swap_lt_lists (wrk);
+      do_lt = 1;
+    }
+
   if (vcm->cfg.use_mq_eventfd)
-    return vppcom_epoll_wait_eventfd (wrk, events, maxevents, n_evts,
-				      wait_for_time);
+    n_evts = vppcom_epoll_wait_eventfd (wrk, events, maxevents, n_evts,
+					wait_for_time);
+  else
+    n_evts = vppcom_epoll_wait_condvar (wrk, events, maxevents, n_evts,
+					wait_for_time);
+
+  if (PREDICT_FALSE (do_lt))
+    vcl_epoll_wait_handle_lt (wrk, events, maxevents, &n_evts);
 
-  return vppcom_epoll_wait_condvar (wrk, events, maxevents, n_evts,
-				    wait_for_time);
+  return n_evts;
 }
 
 int