acl-plugin: time out the sessions created by main thread too (VPP-948)

In multithread setup the main thread may send packets,
which may pass through the node with permit+reflect action.
This creates the connection in lists for thread0,
however in multithread there are no interupt handlers there.

Ensure we are not spending too much time spinning in a
tight cycle by suspending the main cleaner thread
until the current iteration of interrupts is processed.

Change-Id: Idb7346737757ee9a67b5d3e549bc9ad9aab22e89
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>

4 files changed, 28 insertions, 13 deletions

diff --git a/src/plugins/acl/acl.c b/src/plugins/acl/acl.c
index ff7fc0f24fb..8118edd63ad 100644
--- a/src/plugins/acl/acl.c
+++ b/src/plugins/acl/acl.c
@@ -2176,6 +2176,7 @@ acl_show_aclplugin_fn (vlib_main_t * vm,
 	out0 = format(out0, "  interrupt is pending: %d\n", pw->interrupt_is_pending);
 	out0 = format(out0, "  interrupt is needed: %d\n", pw->interrupt_is_needed);
 	out0 = format(out0, "  interrupt is unwanted: %d\n", pw->interrupt_is_unwanted);
+	out0 = format(out0, "  interrupt generation: %d\n", pw->interrupt_generation);
       }
       out0 = format(out0, "\n\nConn cleaner thread counters:\n");
 #define _(cnt, desc) out0 = format(out0, "             %20lu: %s\n", am->cnt, desc);
@@ -2183,6 +2184,7 @@ acl_show_aclplugin_fn (vlib_main_t * vm,
 #undef _
       vec_terminate_c_string(out0);
       vlib_cli_output(vm, "\n\n%s\n\n", out0);
+      vlib_cli_output(vm, "Interrupt generation: %d\n", am->fa_interrupt_generation);
       vlib_cli_output(vm, "Sessions per interval: min %lu max %lu increment: %f ms current: %f ms",
               am->fa_min_deleted_sessions_per_interval, am->fa_max_deleted_sessions_per_interval,
               am->fa_cleaner_wait_time_increment * 1000.0, ((f64)am->fa_current_cleaner_timer_wait_interval) * 1000.0/(f64)vm->clib_time.clocks_per_second);
diff --git a/src/plugins/acl/acl.h b/src/plugins/acl/acl.h
index 14c6129c07f..dcb4e0e9378 100644
--- a/src/plugins/acl/acl.h
+++ b/src/plugins/acl/acl.h
@@ -230,6 +230,8 @@ typedef struct {
 
   u64 fa_current_cleaner_timer_wait_interval;
 
+  int fa_interrupt_generation;
+
   /* per-worker data related t conn management */
   acl_fa_per_worker_data_t *per_worker_data;
 
diff --git a/src/plugins/acl/fa_node.c b/src/plugins/acl/fa_node.c
index 47945892881..a029b901838 100644
--- a/src/plugins/acl/fa_node.c
+++ b/src/plugins/acl/fa_node.c
@@ -1366,6 +1366,7 @@ acl_fa_worker_conn_cleaner_process(vlib_main_t * vm,
        pw->interrupt_is_unwanted = 0;
      }
    }
+   pw->interrupt_generation = am->fa_interrupt_generation;
    return 0;
 }
 
@@ -1409,7 +1410,7 @@ acl_fa_session_cleaner_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
 
   am->fa_current_cleaner_timer_wait_interval = max_timer_wait_interval;
   am->fa_cleaner_node_index = acl_fa_session_cleaner_process_node.index;
-
+  am->fa_interrupt_generation = 1;
   while (1)
     {
       now = clib_cpu_time_now ();
@@ -1508,10 +1509,6 @@ acl_fa_session_cleaner_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
 	    clib_warning("ACL_FA_CLEANER_DELETE_BY_SW_IF_INDEX bitmap: %U", format_bitmap_hex, clear_sw_if_index_bitmap);
 #endif
 	    vec_foreach(pw0, am->per_worker_data) {
-              if ((pw0 == am->per_worker_data) && (vec_len(vlib_mains) > 1)) {
-                /* thread 0 in multithreaded scenario is not used */
-                continue;
-              }
               CLIB_MEMORY_BARRIER ();
 	      while (pw0->clear_in_process) {
                 CLIB_MEMORY_BARRIER ();
@@ -1546,10 +1543,6 @@ acl_fa_session_cleaner_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
 	    clib_warning("CLEANER mains len: %d per-worker len: %d", vec_len(vlib_mains), vec_len(am->per_worker_data));
 #endif
 	    vec_foreach(pw0, am->per_worker_data) {
-              if ((pw0 == am->per_worker_data) && (vec_len(vlib_mains) > 1)) {
-                /* thread 0 in multithreaded scenario is not used */
-                continue;
-              }
               CLIB_MEMORY_BARRIER ();
 	      while (pw0->clear_in_process) {
                 CLIB_MEMORY_BARRIER ();
@@ -1586,15 +1579,28 @@ acl_fa_session_cleaner_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
       if (event_data)
 	_vec_len (event_data) = 0;
 
+      /*
+       * If the interrupts were not processed yet, ensure we wait a bit,
+       * but up to a point.
+       */
+      int need_more_wait = 0;
+      int max_wait_cycles = 100;
+      do {
+        need_more_wait = 0;
+        vec_foreach(pw0, am->per_worker_data) {
+          if (pw0->interrupt_generation != am->fa_interrupt_generation) {
+            need_more_wait = 1;
+          }
+        }
+        if (need_more_wait) {
+          vlib_process_suspend(vm, 0.0001);
+        }
+      } while (need_more_wait && (--max_wait_cycles > 0));
 
       int interrupts_needed = 0;
       int interrupts_unwanted = 0;
 
       vec_foreach(pw0, am->per_worker_data) {
-        if ((pw0 == am->per_worker_data) && (vec_len(vlib_mains) > 1)) {
-          /* thread 0 in multithreaded scenario is not used */
-          continue;
-        }
         if (pw0->interrupt_is_needed) {
           interrupts_needed++;
           /* the per-worker value is reset when sending the interrupt */
@@ -1615,6 +1621,7 @@ acl_fa_session_cleaner_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
           am->fa_current_cleaner_timer_wait_interval += cpu_cps * am->fa_cleaner_wait_time_increment;
       }
       am->fa_cleaner_cnt_event_cycles++;
+      am->fa_interrupt_generation++;
     }
   /* NOT REACHED */
   return 0;
diff --git a/src/plugins/acl/fa_node.h b/src/plugins/acl/fa_node.h
index a8dece4edcf..d5c766cd1fa 100644
--- a/src/plugins/acl/fa_node.h
+++ b/src/plugins/acl/fa_node.h
@@ -147,6 +147,10 @@ typedef struct {
    * because there is not enough work for the current rate.
    */
   int interrupt_is_unwanted;
+  /*
+   * Set to copy of a "generation" counter in main thread so we can sync the interrupts.
+   */
+  int interrupt_generation;
 } acl_fa_per_worker_data_t;