aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSirshak Das <sirshak.das@arm.com>2018-11-07 18:46:42 -0600
committerOle Trøan <otroan@employees.org>2018-11-28 20:50:21 +0000
commit19515acddc343d1680d4a5d27f39456999498591 (patch)
tree96d7e4be9ca0379b252482f5081f3abeea7d32cb
parent5785e89ba15128f9659bf0b6f78559c63a70e285 (diff)
Use acquire/release ordering when accessing svm_fifo shared variable cursize
Improves TCP iperf3 performance by ~3% on AArch64. Change-Id: I1e51bd8403ba45ec6af4c2f96b95e884c1ae0d67 Signed-off-by: Sirshak Das <sirshak.das@arm.com> Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com> Reviewed-by: Ola Liljedahl <ola.liljedahl@arm.com>
-rw-r--r--src/svm/svm_fifo.c10
-rw-r--r--src/svm/svm_fifo.h6
-rw-r--r--src/vppinfra/atomics.h4
3 files changed, 12 insertions, 8 deletions
diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c
index fb942a63d43..4397ef8413e 100644
--- a/src/svm/svm_fifo.c
+++ b/src/svm/svm_fifo.c
@@ -519,7 +519,7 @@ CLIB_MARCH_FN (svm_fifo_enqueue_nowait, int, svm_fifo_t * f, u32 max_bytes,
/* Atomically increase the queue length */
ASSERT (cursize + total_copy_bytes <= nitems);
- clib_atomic_fetch_add (&f->cursize, total_copy_bytes);
+ clib_atomic_fetch_add_rel (&f->cursize, total_copy_bytes);
return (total_copy_bytes);
}
@@ -666,7 +666,7 @@ CLIB_MARCH_FN (svm_fifo_dequeue_nowait, int, svm_fifo_t * f, u32 max_bytes,
ASSERT (f->head <= nitems);
ASSERT (cursize >= total_copy_bytes);
- clib_atomic_fetch_sub (&f->cursize, total_copy_bytes);
+ clib_atomic_fetch_sub_rel (&f->cursize, total_copy_bytes);
return (total_copy_bytes);
}
@@ -764,7 +764,7 @@ svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes)
ASSERT (f->head <= nitems);
ASSERT (cursize >= total_drop_bytes);
- clib_atomic_fetch_sub (&f->cursize, total_drop_bytes);
+ clib_atomic_fetch_sub_rel (&f->cursize, total_drop_bytes);
return total_drop_bytes;
}
@@ -773,7 +773,7 @@ void
svm_fifo_dequeue_drop_all (svm_fifo_t * f)
{
f->head = f->tail;
- clib_atomic_fetch_sub (&f->cursize, f->cursize);
+ clib_atomic_fetch_sub_rel (&f->cursize, f->cursize);
}
int
@@ -820,7 +820,7 @@ svm_fifo_segments_free (svm_fifo_t * f, svm_fifo_segment_t * fs)
f->head = (f->head + fs[0].len) % f->nitems;
total_drop_bytes = fs[0].len;
}
- clib_atomic_fetch_sub (&f->cursize, total_drop_bytes);
+ clib_atomic_fetch_sub_rel (&f->cursize, total_drop_bytes);
}
u32
diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h
index e049d3e3147..791b513a4a6 100644
--- a/src/svm/svm_fifo.h
+++ b/src/svm/svm_fifo.h
@@ -113,19 +113,19 @@ u8 *svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose);
static inline u32
svm_fifo_max_dequeue (svm_fifo_t * f)
{
- return f->cursize;
+ return clib_atomic_load_acq_n (&f->cursize);
}
static inline int
svm_fifo_is_full (svm_fifo_t * f)
{
- return (f->cursize == f->nitems);
+ return (clib_atomic_load_acq_n (&f->cursize) == f->nitems);
}
static inline int
svm_fifo_is_empty (svm_fifo_t * f)
{
- return (f->cursize == 0);
+ return (clib_atomic_load_acq_n (&f->cursize) == 0);
}
static inline u32
diff --git a/src/vppinfra/atomics.h b/src/vppinfra/atomics.h
index 420ae837fc0..8084bdc77d4 100644
--- a/src/vppinfra/atomics.h
+++ b/src/vppinfra/atomics.h
@@ -40,7 +40,11 @@
#define clib_atomic_test_and_set(a) __sync_lock_test_and_set(a, 1)
#define clib_atomic_release(a) __sync_lock_release(a)
+#define clib_atomic_load_acq_n(a) __atomic_load_n((a), __ATOMIC_ACQUIRE)
#define clib_atomic_store_rel_n(a, b) __atomic_store_n ((a), (b), __ATOMIC_RELEASE)
#define clib_atomic_swap_acq_n(a, b) __atomic_exchange_n ((a), (b), __ATOMIC_ACQUIRE)
+#define clib_atomic_fetch_add_rel(a, b) __atomic_fetch_add((a), (b), __ATOMIC_RELEASE)
+#define clib_atomic_fetch_sub_rel(a, b) __atomic_fetch_sub((a), (b), __ATOMIC_RELEASE)
+
#endif /* included_clib_atomics_h */