acl-plugin: optimize session idle timer checks

This commit adds explicit signaling from a non-owning thread to the owning thread
to restart the session timer as necessary.

Consequently, we now can sweep the session lists at their respective timeouts,
rather than sweeping all the lists at the pace of the shortest timeout value,
just taking care to wake up if the session requeue to a different list results
in needing to wake up earlier.

Change-Id: Ifc8c500f6988748f4cd3dc184dd7824321aaaaca
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>

1 files changed, 66 insertions, 27 deletions

diff --git a/src/plugins/acl/sess_mgmt_node.c b/src/plugins/acl/sess_mgmt_node.c
index abb1e7cf0e4..f38677f8d5a 100644
--- a/src/plugins/acl/sess_mgmt_node.c
+++ b/src/plugins/acl/sess_mgmt_node.c
@@ -30,25 +30,6 @@
 #include <plugins/acl/public_inlines.h>
 #include <plugins/acl/session_inlines.h>
 
-// #include <vppinfra/bihash_40_8.h>
-
-
-static u64
-fa_session_get_shortest_timeout (acl_main_t * am)
-{
-  int timeout_type;
-  u64 timeout = ~0LL;
-  for (timeout_type = ACL_TIMEOUT_UDP_IDLE;
-       timeout_type < ACL_N_USER_TIMEOUTS; timeout_type++)
-    {
-      if (timeout > am->session_timeout_sec[timeout_type])
-	{
-	  timeout = am->session_timeout_sec[timeout_type];
-	}
-    }
-  return timeout;
-}
-
 static u8 *
 format_ip6_session_bihash_kv (u8 * s, va_list * args)
 {
@@ -138,14 +119,9 @@ static u64
 fa_session_get_list_timeout (acl_main_t * am, fa_session_t * sess)
 {
   u64 timeout = am->vlib_main->clib_time.clocks_per_second / 1000;
-  /*
-   * we have the shortest possible timeout type in all the lists
-   * (see README-multicore for the rationale)
-   */
-  if (sess->link_list_id == ACL_TIMEOUT_PURGATORY)
-    timeout = fa_session_get_timeout (am, sess);
-  else
-    timeout *= fa_session_get_shortest_timeout (am);
+  timeout = fa_session_get_timeout (am, sess);
+  /* for all user lists, check them twice per timeout */
+  timeout >>= (sess->link_list_id != ACL_TIMEOUT_PURGATORY);
   return timeout;
 }
 
@@ -183,6 +159,28 @@ acl_fa_check_idle_sessions (acl_main_t * am, u16 thread_index, u64 now)
   fsid.thread_index = thread_index;
   int total_expired = 0;
 
+  /* let the other threads enqueue more requests while we process, if they like */
+  aclp_swap_wip_and_pending_session_change_requests (am, thread_index);
+  u64 *psr = NULL;
+
+  vec_foreach (psr, pw->wip_session_change_requests)
+  {
+    acl_fa_sess_req_t op = *psr >> 32;
+    fsid.session_index = *psr & 0xffffffff;
+    switch (op)
+      {
+      case ACL_FA_REQ_SESS_RESCHEDULE:
+	acl_fa_restart_timer_for_session (am, now, fsid);
+	break;
+      default:
+	/* do nothing */
+	break;
+      }
+  }
+  if (pw->wip_session_change_requests)
+    _vec_len (pw->wip_session_change_requests) = 0;
+
+
   {
     u8 tt = 0;
     int n_pending_swipes = 0;
@@ -240,6 +238,9 @@ acl_fa_check_idle_sessions (acl_main_t * am, u16 thread_index, u64 now)
 	  clib_bitmap_get (pw->pending_clear_sw_if_index_bitmap, sw_if_index);
 	if (am->trace_sessions > 3)
 	  {
+	    elog_acl_maybe_trace_X2 (am,
+				     "acl_fa_check_idle_sessions: now %lu sess_timeout_time %lu",
+				     "i8i8", now, sess_timeout_time);
 	    elog_acl_maybe_trace_X4 (am,
 				     "acl_fa_check_idle_sessions: session %d sw_if_index %d timeout_passed %d clearing_interface %d",
 				     "i4i4i4i4", (u32) fsid.session_index,
@@ -361,6 +362,44 @@ send_one_worker_interrupt (vlib_main_t * vm, acl_main_t * am,
     }
 }
 
+void
+aclp_post_session_change_request (acl_main_t * am, u32 target_thread,
+				  u32 target_session, u32 request_type)
+{
+  acl_fa_per_worker_data_t *pw_me =
+    &am->per_worker_data[os_get_thread_index ()];
+  acl_fa_per_worker_data_t *pw = &am->per_worker_data[target_thread];
+  clib_spinlock_lock_if_init (&pw->pending_session_change_request_lock);
+  /* vec_add1 might cause a reallocation, change the heap just in case */
+  void *oldheap = clib_mem_set_heap (am->acl_mheap);
+  vec_add1 (pw->pending_session_change_requests,
+	    (((u64) request_type) << 32) | target_session);
+  clib_mem_set_heap (oldheap);
+
+  pw->rcvd_session_change_requests++;
+  pw_me->sent_session_change_requests++;
+  if (vec_len (pw->pending_session_change_requests) == 1)
+    {
+      /* ensure the requests get processed */
+      send_one_worker_interrupt (am->vlib_main, am, target_thread);
+    }
+  clib_spinlock_unlock_if_init (&pw->pending_session_change_request_lock);
+}
+
+void
+aclp_swap_wip_and_pending_session_change_requests (acl_main_t * am,
+						   u32 target_thread)
+{
+  acl_fa_per_worker_data_t *pw = &am->per_worker_data[target_thread];
+  u64 *tmp;
+  clib_spinlock_lock_if_init (&pw->pending_session_change_request_lock);
+  tmp = pw->pending_session_change_requests;
+  pw->pending_session_change_requests = pw->wip_session_change_requests;
+  pw->wip_session_change_requests = tmp;
+  clib_spinlock_unlock_if_init (&pw->pending_session_change_request_lock);
+}
+
+
 static int
 purgatory_has_connections (vlib_main_t * vm, acl_main_t * am,
 			   int thread_index)