cnat: remove rwlock on ts

Type: improvement

Remove rwlock contention on timestamps. ~10% pps with
10k sessions. Use fixed-size-pools of increasing sizes
starting with 4K, and with a x2 step each time.
We don't free/shrink allocated pools.

Change-Id: I5fea51faba40430106c823275a6356e81709d118
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>

1 files changed, 77 insertions, 27 deletions

diff --git a/src/plugins/cnat/cnat_inline.h b/src/plugins/cnat/cnat_inline.h
index 5a55ecbf3c0..2986b3497a9 100644
--- a/src/plugins/cnat/cnat_inline.h
+++ b/src/plugins/cnat/cnat_inline.h
@@ -19,72 +19,122 @@
 
 #include <cnat/cnat_types.h>
 
+always_inline int
+cnat_ts_is_free_index (u32 index)
+{
+  u32 pidx = index >> (32 - CNAT_TS_MPOOL_BITS);
+  index = index & (0xffffffff >> CNAT_TS_MPOOL_BITS);
+  return pool_is_free_index (cnat_timestamps.ts_pools[pidx], index);
+}
+
+always_inline cnat_timestamp_t *
+cnat_timestamp_get (u32 index)
+{
+  /* 6 top bits for choosing pool */
+  u32 pidx = index >> (32 - CNAT_TS_MPOOL_BITS);
+  index = index & (0xffffffff >> CNAT_TS_MPOOL_BITS);
+  return pool_elt_at_index (cnat_timestamps.ts_pools[pidx], index);
+}
+
+always_inline cnat_timestamp_t *
+cnat_timestamp_get_if_valid (u32 index)
+{
+  /* 6 top bits for choosing pool */
+  u32 pidx = index >> (32 - CNAT_TS_MPOOL_BITS);
+  index = index & (0xffffffff >> CNAT_TS_MPOOL_BITS);
+  if (pidx >= cnat_timestamps.next_empty_pool_idx)
+    return (NULL);
+  if (pool_is_free_index (cnat_timestamps.ts_pools[pidx], index))
+    return (NULL);
+  return pool_elt_at_index (cnat_timestamps.ts_pools[pidx], index);
+}
+
+always_inline index_t
+cnat_timestamp_alloc ()
+{
+  cnat_timestamp_t *ts;
+  u32 index, pool_sz;
+  uword pidx;
+
+  clib_spinlock_lock (&cnat_timestamps.ts_lock);
+  pidx = clib_bitmap_first_set (cnat_timestamps.ts_free);
+  pool_sz = 1 << (CNAT_TS_BASE_SIZE + pidx);
+  ASSERT (pidx <= cnat_timestamps.next_empty_pool_idx);
+  if (pidx == cnat_timestamps.next_empty_pool_idx)
+    pool_init_fixed (
+      cnat_timestamps.ts_pools[cnat_timestamps.next_empty_pool_idx++],
+      pool_sz);
+  pool_get (cnat_timestamps.ts_pools[pidx], ts);
+  if (pool_elts (cnat_timestamps.ts_pools[pidx]) == pool_sz)
+    clib_bitmap_set (cnat_timestamps.ts_free, pidx, 0);
+  clib_spinlock_unlock (&cnat_timestamps.ts_lock);
+
+  index = (u32) pidx << (32 - CNAT_TS_MPOOL_BITS);
+  return index | (ts - cnat_timestamps.ts_pools[pidx]);
+}
+
+always_inline void
+cnat_timestamp_destroy (u32 index)
+{
+  u32 pidx = index >> (32 - CNAT_TS_MPOOL_BITS);
+  index = index & (0xffffffff >> CNAT_TS_MPOOL_BITS);
+  clib_spinlock_lock (&cnat_timestamps.ts_lock);
+  pool_put_index (cnat_timestamps.ts_pools[pidx], index);
+  clib_bitmap_set (cnat_timestamps.ts_free, pidx, 1);
+  clib_spinlock_unlock (&cnat_timestamps.ts_lock);
+}
+
 always_inline u32
 cnat_timestamp_new (f64 t)
 {
-  u32 index;
-  cnat_timestamp_t *ts;
-  clib_rwlock_writer_lock (&cnat_main.ts_lock);
-  pool_get (cnat_timestamps, ts);
+  index_t index = cnat_timestamp_alloc ();
+  cnat_timestamp_t *ts = cnat_timestamp_get (index);
   ts->last_seen = t;
   ts->lifetime = cnat_main.session_max_age;
   ts->refcnt = CNAT_TIMESTAMP_INIT_REFCNT;
-  index = ts - cnat_timestamps;
-  clib_rwlock_writer_unlock (&cnat_main.ts_lock);
   return index;
 }
 
 always_inline void
 cnat_timestamp_inc_refcnt (u32 index)
 {
-  clib_rwlock_reader_lock (&cnat_main.ts_lock);
-  cnat_timestamp_t *ts = pool_elt_at_index (cnat_timestamps, index);
-  ts->refcnt++;
-  clib_rwlock_reader_unlock (&cnat_main.ts_lock);
+  cnat_timestamp_t *ts = cnat_timestamp_get (index);
+  clib_atomic_add_fetch (&ts->refcnt, 1);
 }
 
 always_inline void
 cnat_timestamp_update (u32 index, f64 t)
 {
-  clib_rwlock_reader_lock (&cnat_main.ts_lock);
-  cnat_timestamp_t *ts = pool_elt_at_index (cnat_timestamps, index);
+  cnat_timestamp_t *ts = cnat_timestamp_get (index);
   ts->last_seen = t;
-  clib_rwlock_reader_unlock (&cnat_main.ts_lock);
 }
 
 always_inline void
 cnat_timestamp_set_lifetime (u32 index, u16 lifetime)
 {
-  clib_rwlock_reader_lock (&cnat_main.ts_lock);
-  cnat_timestamp_t *ts = pool_elt_at_index (cnat_timestamps, index);
+  cnat_timestamp_t *ts = cnat_timestamp_get (index);
   ts->lifetime = lifetime;
-  clib_rwlock_reader_unlock (&cnat_main.ts_lock);
 }
 
 always_inline f64
 cnat_timestamp_exp (u32 index)
 {
   f64 t;
-  if (INDEX_INVALID == index)
+  cnat_timestamp_t *ts = cnat_timestamp_get_if_valid (index);
+  if (NULL == ts)
     return -1;
-  clib_rwlock_reader_lock (&cnat_main.ts_lock);
-  cnat_timestamp_t *ts = pool_elt_at_index (cnat_timestamps, index);
   t = ts->last_seen + (f64) ts->lifetime;
-  clib_rwlock_reader_unlock (&cnat_main.ts_lock);
   return t;
 }
 
 always_inline void
 cnat_timestamp_free (u32 index)
 {
-  if (INDEX_INVALID == index)
+  cnat_timestamp_t *ts = cnat_timestamp_get_if_valid (index);
+  if (NULL == ts)
     return;
-  clib_rwlock_writer_lock (&cnat_main.ts_lock);
-  cnat_timestamp_t *ts = pool_elt_at_index (cnat_timestamps, index);
-  ts->refcnt--;
-  if (0 == ts->refcnt)
-    pool_put (cnat_timestamps, ts);
-  clib_rwlock_writer_unlock (&cnat_main.ts_lock);
+  if (0 == clib_atomic_sub_fetch (&ts->refcnt, 1))
+    cnat_timestamp_destroy (index);
 }
 
 /*