From 8a5a6c51ee1c1bdbfbfcf1878784d0c32800255f Mon Sep 17 00:00:00 2001
From: Florin Coras <fcoras@cisco.com>
Date: Mon, 5 Jun 2023 14:58:15 -0700
Subject: svm: convert fifo want_deq_ntf ops to atomics

Type: improvement

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: Iba2e4de24ff0724e31859f5d2f4ffa3cfe2cf284
(cherry picked from commit 81e3243c1b3d34705de7d09b8668a9a068c6af9e)
---
 src/svm/svm_fifo.h | 53 ++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 42 insertions(+), 11 deletions(-)

diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h
index c4cc0382ab7..7ea114f8702 100644
--- a/src/svm/svm_fifo.h
+++ b/src/svm/svm_fifo.h
@@ -759,7 +759,7 @@ svm_fifo_unset_event (svm_fifo_t * f)
 static inline void
 svm_fifo_add_want_deq_ntf (svm_fifo_t * f, u8 ntf_type)
 {
-  f->shr->want_deq_ntf |= ntf_type;
+  __atomic_or_fetch (&f->shr->want_deq_ntf, ntf_type, __ATOMIC_RELEASE);
 }
 
 /**
@@ -773,7 +773,21 @@ svm_fifo_add_want_deq_ntf (svm_fifo_t * f, u8 ntf_type)
 static inline void
 svm_fifo_del_want_deq_ntf (svm_fifo_t * f, u8 ntf_type)
 {
-  f->shr->want_deq_ntf &= ~ntf_type;
+  __atomic_and_fetch (&f->shr->want_deq_ntf, ~ntf_type, __ATOMIC_RELEASE);
+}
+
+/**
+ * Get want notification flag
+ *
+ * Done atomically with acquire memory ordering
+ *
+ * @param f	fifo
+ * @return 	value of want_deq_ntf flag
+ */
+static inline u32
+svm_fifo_get_want_deq_ntf (svm_fifo_t *f)
+{
+  return clib_atomic_load_acq_n (&f->shr->want_deq_ntf);
 }
 
 /**
@@ -790,10 +804,27 @@ svm_fifo_del_want_deq_ntf (svm_fifo_t * f, u8 ntf_type)
 static inline void
 svm_fifo_clear_deq_ntf (svm_fifo_t * f)
 {
-  /* Set the flag if want_notif_if_full was the only ntf requested */
-  f->shr->has_deq_ntf =
-    f->shr->want_deq_ntf == SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL;
-  svm_fifo_del_want_deq_ntf (f, SVM_FIFO_WANT_DEQ_NOTIF);
+  u32 want_deq_ntf = svm_fifo_get_want_deq_ntf (f);
+  /* Set the flag if want ntf if full or empty was requested */
+  if (want_deq_ntf &
+      (SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL | SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY))
+    clib_atomic_store_rel_n (&f->shr->has_deq_ntf, 1);
+  if (want_deq_ntf & SVM_FIFO_WANT_DEQ_NOTIF)
+    svm_fifo_del_want_deq_ntf (f, SVM_FIFO_WANT_DEQ_NOTIF);
+}
+
+/**
+ * Get has dequeue notification flag
+ *
+ * Done atomically with acquire memory ordering
+ *
+ * @param f	fifo
+ * @return	has_deq_ntf flag
+ */
+static inline u32
+svm_fifo_has_deq_ntf (svm_fifo_t *f)
+{
+  return clib_atomic_load_acq_n (&f->shr->has_deq_ntf);
 }
 
 /**
@@ -824,9 +855,9 @@ svm_fifo_reset_has_deq_ntf (svm_fifo_t * f)
 static inline u8
 svm_fifo_needs_deq_ntf (svm_fifo_t * f, u32 n_last_deq)
 {
-  u8 want_ntf = f->shr->want_deq_ntf;
+  u32 want_ntf = svm_fifo_get_want_deq_ntf (f);
 
-  if (PREDICT_TRUE (want_ntf == SVM_FIFO_NO_DEQ_NOTIF))
+  if (want_ntf == SVM_FIFO_NO_DEQ_NOTIF)
     return 0;
   else if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF)
     return (svm_fifo_max_enqueue (f) >= f->shr->deq_thresh);
@@ -834,13 +865,13 @@ svm_fifo_needs_deq_ntf (svm_fifo_t * f, u32 n_last_deq)
     {
       u32 max_deq = svm_fifo_max_dequeue_cons (f);
       u32 size = f->shr->size;
-      if (!f->shr->has_deq_ntf && max_deq < size &&
-	  max_deq + n_last_deq >= size)
+      if (max_deq < size && max_deq + n_last_deq >= size &&
+	  !svm_fifo_has_deq_ntf (f))
 	return 1;
     }
   if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY)
     {
-      if (!f->shr->has_deq_ntf && svm_fifo_is_empty (f))
+      if (!svm_fifo_has_deq_ntf (f) && svm_fifo_is_empty (f))
 	return 1;
     }
   return 0;
-- 
cgit 1.2.3-korg