From 6d733a93b2eb9c16196ee17d5cdc77db21589571 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Wed, 4 Nov 2020 11:41:05 +0100 Subject: cnat: remove rwlock on ts Type: improvement Remove rwlock contention on timestamps. ~10% pps with 10k sessions. Use fixed-size-pools of increasing sizes starting with 4K, and with a x2 step each time. We don't free/shrink allocated pools. Change-Id: I5fea51faba40430106c823275a6356e81709d118 Signed-off-by: Nathan Skrzypczak --- src/plugins/cnat/cnat_inline.h | 104 ++++++++++++++++++++++++++++++----------- 1 file changed, 77 insertions(+), 27 deletions(-) (limited to 'src/plugins/cnat/cnat_inline.h') diff --git a/src/plugins/cnat/cnat_inline.h b/src/plugins/cnat/cnat_inline.h index 5a55ecbf3c0..2986b3497a9 100644 --- a/src/plugins/cnat/cnat_inline.h +++ b/src/plugins/cnat/cnat_inline.h @@ -19,72 +19,122 @@ #include +always_inline int +cnat_ts_is_free_index (u32 index) +{ + u32 pidx = index >> (32 - CNAT_TS_MPOOL_BITS); + index = index & (0xffffffff >> CNAT_TS_MPOOL_BITS); + return pool_is_free_index (cnat_timestamps.ts_pools[pidx], index); +} + +always_inline cnat_timestamp_t * +cnat_timestamp_get (u32 index) +{ + /* 6 top bits for choosing pool */ + u32 pidx = index >> (32 - CNAT_TS_MPOOL_BITS); + index = index & (0xffffffff >> CNAT_TS_MPOOL_BITS); + return pool_elt_at_index (cnat_timestamps.ts_pools[pidx], index); +} + +always_inline cnat_timestamp_t * +cnat_timestamp_get_if_valid (u32 index) +{ + /* 6 top bits for choosing pool */ + u32 pidx = index >> (32 - CNAT_TS_MPOOL_BITS); + index = index & (0xffffffff >> CNAT_TS_MPOOL_BITS); + if (pidx >= cnat_timestamps.next_empty_pool_idx) + return (NULL); + if (pool_is_free_index (cnat_timestamps.ts_pools[pidx], index)) + return (NULL); + return pool_elt_at_index (cnat_timestamps.ts_pools[pidx], index); +} + +always_inline index_t +cnat_timestamp_alloc () +{ + cnat_timestamp_t *ts; + u32 index, pool_sz; + uword pidx; + + clib_spinlock_lock (&cnat_timestamps.ts_lock); + pidx = clib_bitmap_first_set (cnat_timestamps.ts_free); + pool_sz = 1 << (CNAT_TS_BASE_SIZE + pidx); + ASSERT (pidx <= cnat_timestamps.next_empty_pool_idx); + if (pidx == cnat_timestamps.next_empty_pool_idx) + pool_init_fixed ( + cnat_timestamps.ts_pools[cnat_timestamps.next_empty_pool_idx++], + pool_sz); + pool_get (cnat_timestamps.ts_pools[pidx], ts); + if (pool_elts (cnat_timestamps.ts_pools[pidx]) == pool_sz) + clib_bitmap_set (cnat_timestamps.ts_free, pidx, 0); + clib_spinlock_unlock (&cnat_timestamps.ts_lock); + + index = (u32) pidx << (32 - CNAT_TS_MPOOL_BITS); + return index | (ts - cnat_timestamps.ts_pools[pidx]); +} + +always_inline void +cnat_timestamp_destroy (u32 index) +{ + u32 pidx = index >> (32 - CNAT_TS_MPOOL_BITS); + index = index & (0xffffffff >> CNAT_TS_MPOOL_BITS); + clib_spinlock_lock (&cnat_timestamps.ts_lock); + pool_put_index (cnat_timestamps.ts_pools[pidx], index); + clib_bitmap_set (cnat_timestamps.ts_free, pidx, 1); + clib_spinlock_unlock (&cnat_timestamps.ts_lock); +} + always_inline u32 cnat_timestamp_new (f64 t) { - u32 index; - cnat_timestamp_t *ts; - clib_rwlock_writer_lock (&cnat_main.ts_lock); - pool_get (cnat_timestamps, ts); + index_t index = cnat_timestamp_alloc (); + cnat_timestamp_t *ts = cnat_timestamp_get (index); ts->last_seen = t; ts->lifetime = cnat_main.session_max_age; ts->refcnt = CNAT_TIMESTAMP_INIT_REFCNT; - index = ts - cnat_timestamps; - clib_rwlock_writer_unlock (&cnat_main.ts_lock); return index; } always_inline void cnat_timestamp_inc_refcnt (u32 index) { - clib_rwlock_reader_lock (&cnat_main.ts_lock); - cnat_timestamp_t *ts = pool_elt_at_index (cnat_timestamps, index); - ts->refcnt++; - clib_rwlock_reader_unlock (&cnat_main.ts_lock); + cnat_timestamp_t *ts = cnat_timestamp_get (index); + clib_atomic_add_fetch (&ts->refcnt, 1); } always_inline void cnat_timestamp_update (u32 index, f64 t) { - clib_rwlock_reader_lock (&cnat_main.ts_lock); - cnat_timestamp_t *ts = pool_elt_at_index (cnat_timestamps, index); + cnat_timestamp_t *ts = cnat_timestamp_get (index); ts->last_seen = t; - clib_rwlock_reader_unlock (&cnat_main.ts_lock); } always_inline void cnat_timestamp_set_lifetime (u32 index, u16 lifetime) { - clib_rwlock_reader_lock (&cnat_main.ts_lock); - cnat_timestamp_t *ts = pool_elt_at_index (cnat_timestamps, index); + cnat_timestamp_t *ts = cnat_timestamp_get (index); ts->lifetime = lifetime; - clib_rwlock_reader_unlock (&cnat_main.ts_lock); } always_inline f64 cnat_timestamp_exp (u32 index) { f64 t; - if (INDEX_INVALID == index) + cnat_timestamp_t *ts = cnat_timestamp_get_if_valid (index); + if (NULL == ts) return -1; - clib_rwlock_reader_lock (&cnat_main.ts_lock); - cnat_timestamp_t *ts = pool_elt_at_index (cnat_timestamps, index); t = ts->last_seen + (f64) ts->lifetime; - clib_rwlock_reader_unlock (&cnat_main.ts_lock); return t; } always_inline void cnat_timestamp_free (u32 index) { - if (INDEX_INVALID == index) + cnat_timestamp_t *ts = cnat_timestamp_get_if_valid (index); + if (NULL == ts) return; - clib_rwlock_writer_lock (&cnat_main.ts_lock); - cnat_timestamp_t *ts = pool_elt_at_index (cnat_timestamps, index); - ts->refcnt--; - if (0 == ts->refcnt) - pool_put (cnat_timestamps, ts); - clib_rwlock_writer_unlock (&cnat_main.ts_lock); + if (0 == clib_atomic_sub_fetch (&ts->refcnt, 1)) + cnat_timestamp_destroy (index); } /* -- cgit 1.2.3-korg