VCL refactoring

- simplified event handling and unregister
- removed fixed need to bit flip event hash key
- added spinlock for client_session_fifo (was using sessions_lockp)
- removed redundant vars

Change-Id: I3c7645da660fb5560efdc4e9347e105df9650a16
Signed-off-by: Keith Burns (alagalah) <alagalah@gmail.com>

3 files changed, 55 insertions, 48 deletions

diff --git a/src/vcl/vcl_event.c b/src/vcl/vcl_event.c
index b706a93e56a..65d28707607 100644
--- a/src/vcl/vcl_event.c
+++ b/src/vcl/vcl_event.c
@@ -69,14 +69,14 @@ vce_clear_event (vce_event_thread_t *evt, vce_event_t *ev)
 vce_event_t *
 vce_get_event_from_index(vce_event_thread_t *evt, u32 ev_idx)
 {
-  vce_event_t *ev;
+  vce_event_t *ev = 0;
 
   clib_spinlock_lock (&(evt->events_lockp));
-  ev = pool_elt_at_index (evt->vce_events, ev_idx);
+  if ( ! pool_is_free_index (evt->vce_events, ev_idx))
+    ev = pool_elt_at_index (evt->vce_events, ev_idx);
   clib_spinlock_unlock (&(evt->events_lockp));
 
   return ev;
-
 }
 
 vce_event_handler_reg_t *
@@ -87,16 +87,13 @@ vce_register_handler (vce_event_thread_t *evt, vce_event_key_t *evk,
   vce_event_handler_reg_t *old_handler = 0;
   uword *p;
   u32 handler_index;
-  u64 adj_key;
 
   /* TODO - multiple handler support. For now we can replace
    * and re-instate, which is useful for event recycling */
 
-  adj_key = evk->as_u64 | (1LL << 63); //evk can be 0, which won't hash
-
   clib_spinlock_lock (&evt->handlers_lockp);
 
-  p = hash_get (evt->handlers_index_by_event_key, adj_key);
+  p = hash_get (evt->handlers_index_by_event_key, evk->as_u64);
   if (p)
     {
       old_handler = pool_elt_at_index (evt->vce_event_handlers, p[0]);
@@ -121,8 +118,10 @@ vce_register_handler (vce_event_thread_t *evt, vce_event_key_t *evk,
 
   handler->handler_fn = cb;
   handler->replaced_handler_idx = (p) ? p[0] : ~0;
+  handler->ev_idx = ~0; //This will be set by the event thread if event happens
+  handler->evk = evk->as_u64;
 
-  hash_set (evt->handlers_index_by_event_key, adj_key, handler_index);
+  hash_set (evt->handlers_index_by_event_key, evk->as_u64, handler_index);
 
   pthread_cond_init (&(handler->handler_cond), NULL);
   pthread_mutex_init (&(handler->handler_lock), NULL);
@@ -139,20 +138,19 @@ vce_register_handler (vce_event_thread_t *evt, vce_event_key_t *evk,
 }
 
 int
-vce_unregister_handler (vce_event_thread_t *evt, vce_event_t *ev)
+vce_unregister_handler (vce_event_thread_t *evt,
+			vce_event_handler_reg_t *handler)
 {
-  vce_event_handler_reg_t *handler;
   uword *p;
-  u64 adj_key = ev->evk.as_u64 | (1LL << 63);
+  u64 evk = handler->evk;
   u8 generate_signal = 0;
 
   clib_spinlock_lock (&evt->handlers_lockp);
 
-  p = hash_get (evt->handlers_index_by_event_key, adj_key);
+  p = hash_get (evt->handlers_index_by_event_key, evk);
   if (!p)
     {
       clib_spinlock_unlock (&evt->handlers_lockp);
-
       return VNET_API_ERROR_NO_SUCH_ENTRY;
     }
 
@@ -161,13 +159,13 @@ vce_unregister_handler (vce_event_thread_t *evt, vce_event_t *ev)
   /* If this handler replaced another handler, re-instate it */
   if (handler->replaced_handler_idx != ~0)
     {
-      hash_set (evt->handlers_index_by_event_key, adj_key,
+      hash_set (evt->handlers_index_by_event_key, evk,
 		handler->replaced_handler_idx);
       generate_signal = 1;
     }
   else
     {
-      hash_unset (evt->handlers_index_by_event_key, adj_key);
+      hash_unset (evt->handlers_index_by_event_key, evk);
     }
 
   pthread_mutex_destroy (&(handler->handler_lock));
@@ -196,7 +194,6 @@ vce_event_thread_fn (void *arg)
   u32 ev_idx;
   vce_event_handler_reg_t *handler;
   uword *p;
-  u64 adj_key;
 
   evt->recycle_event = 1; // Used for recycling events with no handlers
 
@@ -221,11 +218,10 @@ vce_event_thread_fn (void *arg)
       clib_spinlock_unlock (&(evt->events_lockp));
 
       ASSERT(ev);
-      adj_key = ev->evk.as_u64 | (1LL << 63);
 
       clib_spinlock_lock (&evt->handlers_lockp);
 
-      p = hash_get (evt->handlers_index_by_event_key, adj_key);
+      p = hash_get (evt->handlers_index_by_event_key, ev->evk.as_u64);
       if (!p)
 	{
 	  /* If an event falls in the woods, and there is no handler to hear it,
diff --git a/src/vcl/vcl_event.h b/src/vcl/vcl_event.h
index 9380f73f77d..7b64ede6ae1 100644
--- a/src/vcl/vcl_event.h
+++ b/src/vcl/vcl_event.h
@@ -51,6 +51,7 @@ typedef struct vce_event_handler_reg_
   pthread_mutex_t handler_lock;
   pthread_cond_t handler_cond;
   u32 ev_idx;
+  u64 evk; //Event key
   u32 replaced_handler_idx;
 } vce_event_handler_reg_t;
 
@@ -124,10 +125,11 @@ vce_event_handler_reg_t * vce_register_handler (vce_event_thread_t *evt,
  * - if this handler replaced an existing one, re-instate it.
  *
  * @param evt - vce_event_thread_t - event system state
- * @param ev  - vce_event_t - event to remove
+ * @param handler - handler to be unregistered
  * @return success/failure rv
  */
-int vce_unregister_handler (vce_event_thread_t *evt, vce_event_t *ev);
+int vce_unregister_handler (vce_event_thread_t *evt,
+			    vce_event_handler_reg_t *handler);
 
 /**
  * @brief vce_event_thread_fn
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index eddb9a0a5fa..1f9857676e2 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -193,14 +193,13 @@ typedef struct vppcom_cfg_t_
 /* VPPCOM Event typedefs */
 typedef enum vcl_event_id_
 {
+  VCL_EVENT_INVALID_EVENT,
   VCL_EVENT_CONNECT_REQ_ACCEPTED,
   VCL_EVENT_N_EVENTS
 } vcl_event_id_t;
 
 typedef struct vce_event_connect_request_
 {
-  u8 size;
-  u8 handled;
   u32 accepted_session_index;
 } vce_event_connect_request_t;
 
@@ -208,15 +207,17 @@ typedef struct vppcom_main_t_
 {
   u8 init;
   u32 debug;
-  u32 *client_session_index_fifo;
   int main_cpu;
 
+  /* FIFO for accepted connections - used in epoll/select */
+  clib_spinlock_t session_fifo_lockp;
+  u32 *client_session_index_fifo;
+
   /* vpp input queue */
   svm_queue_t *vl_input_queue;
 
   /* API client handle */
   u32 my_client_index;
-
   /* Session pool */
   clib_spinlock_t sessions_lockp;
   session_t *sessions;
@@ -453,15 +454,7 @@ vce_connect_request_handler_fn (void *arg)
 {
   vce_event_handler_reg_t *reg = (vce_event_handler_reg_t *) arg;
 
-  vce_event_connect_request_t *ecr;
-  vce_event_t *ev;
-
-  ev = vce_get_event_from_index (&vcm->event_thread, reg->ev_idx);
-
-  ecr = (vce_event_connect_request_t *) ev->data;
-
   pthread_mutex_lock (&reg->handler_lock);
-  ecr->handled = 1;
   pthread_cond_signal (&reg->handler_cond);
   pthread_mutex_unlock (&reg->handler_lock);
 }
@@ -485,10 +478,10 @@ vce_epoll_wait_connect_request_handler_fn (void *arg)
   vce_event_connect_request_t *ecr = (vce_event_connect_request_t *) ev->data;
 
   /* Add the accepted_session_index to the FIFO */
-  clib_spinlock_lock (&vcm->sessions_lockp);
+  clib_spinlock_lock (&vcm->session_fifo_lockp);
   clib_fifo_add1 (vcm->client_session_index_fifo,
 		  ecr->accepted_session_index);
-  clib_spinlock_unlock (&vcm->sessions_lockp);
+  clib_spinlock_unlock (&vcm->session_fifo_lockp);
 
   /* Recycling the event. */
   clib_spinlock_lock (&(vcm->event_thread.events_lockp));
@@ -1281,10 +1274,15 @@ vl_api_accept_session_t_handler (vl_api_accept_session_t * mp)
   vce_event_t *ev;
   int rv;
   u32 ev_idx;
-
+  uword elts = 0;
 
   clib_spinlock_lock (&vcm->sessions_lockp);
-  if (!clib_fifo_free_elts (vcm->client_session_index_fifo))
+
+  clib_spinlock_lock (&vcm->session_fifo_lockp);
+  elts = clib_fifo_free_elts (vcm->client_session_index_fifo);
+  clib_spinlock_unlock (&vcm->session_fifo_lockp);
+
+  if (!elts)
     {
       clib_warning ("VCL<%d>: client session queue is full!", getpid ());
       vppcom_send_accept_session_reply (mp->handle, mp->context,
@@ -1347,7 +1345,6 @@ vl_api_accept_session_t_handler (vl_api_accept_session_t * mp)
   ev->evk.eid = VCL_EVENT_CONNECT_REQ_ACCEPTED;
   listen_session = vppcom_session_table_lookup_listener (mp->listener_handle);
   ev->evk.session_index = (u32) (listen_session - vcm->sessions);
-  ecr->handled = 0;
   ecr->accepted_session_index = session_index;
 
   clib_spinlock_unlock (&vcm->event_thread.events_lockp);
@@ -1401,6 +1398,7 @@ vl_api_accept_session_t_handler (vl_api_accept_session_t * mp)
 	  clib_warning ("ip6");
 	}
     }
+
   clib_spinlock_unlock (&vcm->sessions_lockp);
 
 }
@@ -2140,6 +2138,7 @@ vppcom_app_create (char *app_name)
 	conf_fname = VPPCOM_CONF_DEFAULT;
       vppcom_cfg_heapsize (conf_fname);
       vcl_cfg = &vcm->cfg;
+      clib_spinlock_init (&vcm->session_fifo_lockp);
       clib_fifo_validate (vcm->client_session_index_fifo,
 			  vcm->cfg.listen_queue_size);
       vppcom_cfg_read (conf_fname);
@@ -2672,8 +2671,12 @@ vppcom_session_listen (uint32_t listen_session_index, uint32_t q_len)
       goto done;
     }
 
+  clib_spinlock_lock (&vcm->session_fifo_lockp);
   clib_fifo_validate (vcm->client_session_index_fifo, q_len);
+  clib_spinlock_unlock (&vcm->session_fifo_lockp);
+
   clib_spinlock_unlock (&vcm->sessions_lockp);
+
 done:
   return rv;
 }
@@ -2750,11 +2753,10 @@ vppcom_session_accept (uint32_t listen_session_index, vppcom_endpt_t * ep,
   evk.eid = VCL_EVENT_CONNECT_REQ_ACCEPTED;
   reg = vce_register_handler (&vcm->event_thread, &evk,
 			      vce_connect_request_handler_fn);
-  ev = vce_get_event_from_index (&vcm->event_thread, reg->ev_idx);
 
-  result = (vce_event_connect_request_t *) ev->data;
+  ev = 0;
   pthread_mutex_lock (&reg->handler_lock);
-  while (!result->handled)
+  while (!ev)
     {
       rv =
 	pthread_cond_timedwait (&reg->handler_cond, &reg->handler_lock, &ts);
@@ -2763,23 +2765,28 @@ vppcom_session_accept (uint32_t listen_session_index, vppcom_endpt_t * ep,
 	  rv = VPPCOM_EAGAIN;
 	  goto cleanup;
 	}
+      ev = vce_get_event_from_index (&vcm->event_thread, reg->ev_idx);
     }
+  result = (vce_event_connect_request_t *) ev->data;
   client_session_index = result->accepted_session_index;
 
 
 
   /* Remove from the FIFO used to service epoll */
-  clib_spinlock_lock (&vcm->sessions_lockp);
+  clib_spinlock_lock (&vcm->session_fifo_lockp);
   if (clib_fifo_elts (vcm->client_session_index_fifo))
     {
       u32 tmp_client_session_index;
       clib_fifo_sub1 (vcm->client_session_index_fifo,
 		      tmp_client_session_index);
+      /* It wasn't ours... put it back ... */
       if (tmp_client_session_index != client_session_index)
 	clib_fifo_add1 (vcm->client_session_index_fifo,
 			tmp_client_session_index);
     }
-  clib_spinlock_unlock (&vcm->sessions_lockp);
+  clib_spinlock_unlock (&vcm->session_fifo_lockp);
+
+  clib_spinlock_lock (&vcm->sessions_lockp);
 
   rv = vppcom_session_at_index (client_session_index, &client_session);
   if (PREDICT_FALSE (rv))
@@ -2885,12 +2892,14 @@ vppcom_session_accept (uint32_t listen_session_index, vppcom_endpt_t * ep,
     }
 
   clib_spinlock_unlock (&vcm->sessions_lockp);
-  rv = (int) client_session_index;
 
+  rv = (int) client_session_index;
   vce_clear_event (&vcm->event_thread, ev);
+
 cleanup:
-  vce_unregister_handler (&vcm->event_thread, ev);
+  vce_unregister_handler (&vcm->event_thread, reg);
   pthread_mutex_unlock (&reg->handler_lock);
+
 done:
   return rv;
 }
@@ -3118,7 +3127,9 @@ vppcom_session_read_ready (session_t * session, u32 session_index)
 
   if (session->state & STATE_LISTEN)
     {
+      clib_spinlock_lock (&vcm->session_fifo_lockp);
       ready = clib_fifo_elts (vcm->client_session_index_fifo);
+      clib_spinlock_unlock (&vcm->session_fifo_lockp);
     }
   else
     {
@@ -3677,7 +3688,6 @@ vppcom_epoll_ctl (uint32_t vep_idx, int op, uint32_t session_index,
 {
   session_t *vep_session;
   session_t *session;
-  vce_event_t *ev = 0;
   int rv;
 
   if (vep_idx == session_index)
@@ -3846,9 +3856,8 @@ vppcom_epoll_ctl (uint32_t vep_idx, int op, uint32_t session_index,
       /* VCL Event Un-register handler */
       if ((session->state & STATE_LISTEN) && vep_session->poll_reg)
 	{
-	  ev = vce_get_event_from_index (&vcm->event_thread,
-					 vep_session->poll_reg->ev_idx);
-	  (void) vce_unregister_handler (&vcm->event_thread, ev);
+	  (void) vce_unregister_handler (&vcm->event_thread,
+					 vep_session->poll_reg);
 	}
 
       vep_session->wait_cont_idx =