acl-plugin: multicore: session management fixes

- implement a 1us purgatory for the session structures
  by adding a special connection list, where all connections
  about to be deleted go.

- add per-list-head timeouts updated upon the list enqueue/dequeue
  for connection idle management

- add a "unused" session list with list ID#0, which should
  never be used unless there is a logic error. Use this ID
  to initialize the sessions.

- improve the maintainability of the session linked list
  structures by using symbolic bogus index name instead of ~0

- change the ordering of session creations - first reverse, then
  local. To minimize the potential for two workers competing for
  the same session in the corner case of the two packets
  on different workers creating the same logical session

- reduce the maximum session count to keep the memory usage the same

- add extra log/debug/trace to session cleaning logic

- be more aggressive with cleaning up sessions - wind up the
  interrupts from the workers to themselves if there is more
  work to do

Change-Id: I3aa1c91a925a08e83793467cb15bda178c21e426
Signed-off-by: Andrew Yourtchenko <ayourtch@gmail.com>

1 files changed, 8 insertions, 3 deletions

diff --git a/src/plugins/acl/fa_node.h b/src/plugins/acl/fa_node.h
index 8d79e42e738..5c55cb995a1 100644
--- a/src/plugins/acl/fa_node.h
+++ b/src/plugins/acl/fa_node.h
@@ -18,8 +18,8 @@
 #define TCP_FLAGS_ACKSYN (TCP_FLAG_SYN + TCP_FLAG_ACK)
 
 #define ACL_FA_CONN_TABLE_DEFAULT_HASH_NUM_BUCKETS (64 * 1024)
-#define ACL_FA_CONN_TABLE_DEFAULT_HASH_MEMORY_SIZE (1<<30)
-#define ACL_FA_CONN_TABLE_DEFAULT_MAX_ENTRIES 1000000
+#define ACL_FA_CONN_TABLE_DEFAULT_HASH_MEMORY_SIZE (1ULL<<30)
+#define ACL_FA_CONN_TABLE_DEFAULT_MAX_ENTRIES 500000
 
 typedef union {
   u64 as_u64;
@@ -80,7 +80,8 @@ typedef struct {
   u32 link_prev_idx;      /* +4 bytes = 12 */
   u32 link_next_idx;      /* +4 bytes = 16 */
   u8 link_list_id;        /* +1 bytes = 17 */
-  u8 reserved1[7];        /* +7 bytes = 24 */
+  u8 deleted;             /* +1 bytes = 18 */
+  u8 reserved1[6];        /* +6 bytes = 24 */
   u64 reserved2[5];       /* +5*8 bytes = 64 */
 } fa_session_t;
 
@@ -120,12 +121,16 @@ CT_ASSERT_EQUAL(fa_session_t_size_is_128, sizeof(fa_session_t), 128);
 CT_ASSERT_EQUAL(fa_full_session_id_size_is_64, sizeof(fa_full_session_id_t), sizeof(u64));
 #undef CT_ASSERT_EQUAL
 
+#define FA_SESSION_BOGUS_INDEX ~0
+
 typedef struct {
   /* The pool of sessions managed by this worker */
   fa_session_t *fa_sessions_pool;
   /* per-worker ACL_N_TIMEOUTS of conn lists */
   u32 *fa_conn_list_head;
   u32 *fa_conn_list_tail;
+  /* expiry time set whenever an element is enqueued */
+  u64 *fa_conn_list_head_expiry_time;
   /* adds and deletes per-worker-per-interface */
   u64 *fa_session_dels_by_sw_if_index;
   u64 *fa_session_adds_by_sw_if_index;