svm: add producer wait function

Type: refactor

Signed-off-by: Florin Coras <fcoras@cisco.com>
Change-Id: I9488ad7e045c908b60b5821d9c48583f6d513c2f

3 files changed, 47 insertions, 14 deletions

diff --git a/src/svm/message_queue.c b/src/svm/message_queue.c
index 3163d302967..5c04b19e64c 100644
--- a/src/svm/message_queue.c
+++ b/src/svm/message_queue.c
@@ -247,7 +247,7 @@ svm_msg_q_lock_and_alloc_msg_w_ring (svm_msg_q_t * mq, u32 ring_index,
       svm_msg_q_lock (mq);
       while (svm_msg_q_is_full (mq)
 	     || svm_msg_q_ring_is_full (mq, ring_index))
-	svm_msg_q_wait (mq, SVM_MQ_WAIT_FULL);
+	svm_msg_q_wait_prod (mq);
       *msg = svm_msg_q_alloc_msg_w_ring (mq, ring_index);
     }
   return 0;
@@ -371,7 +371,7 @@ svm_msg_q_add (svm_msg_q_t * mq, svm_msg_q_msg_t * msg, int nowait)
       if (nowait)
 	return (-2);
       while (svm_msg_q_is_full (mq))
-	svm_msg_q_wait (mq, SVM_MQ_WAIT_FULL);
+	svm_msg_q_wait_prod (mq);
     }
 
   svm_msg_q_add_raw (mq, (u8 *) msg);
@@ -498,21 +498,17 @@ svm_msg_q_wait (svm_msg_q_t *mq, svm_msg_q_wait_type_t type)
 
   if (mq->q.evtfd == -1)
     {
-      if (type == SVM_MQ_WAIT_EMPTY)
+      rv = pthread_mutex_lock (&mq->q.shr->mutex);
+      if (PREDICT_FALSE (rv == EOWNERDEAD))
 	{
-	  rv = pthread_mutex_lock (&mq->q.shr->mutex);
-	  if (PREDICT_FALSE (rv == EOWNERDEAD))
-	    {
-	      rv = pthread_mutex_consistent (&mq->q.shr->mutex);
-	      return rv;
-	    }
+	  rv = pthread_mutex_consistent (&mq->q.shr->mutex);
+	  return rv;
 	}
 
       while (fn (mq))
 	pthread_cond_wait (&mq->q.shr->condvar, &mq->q.shr->mutex);
 
-      if (type == SVM_MQ_WAIT_EMPTY)
-	pthread_mutex_unlock (&mq->q.shr->mutex);
+      pthread_mutex_unlock (&mq->q.shr->mutex);
     }
   else
     {
@@ -535,6 +531,35 @@ svm_msg_q_wait (svm_msg_q_t *mq, svm_msg_q_wait_type_t type)
 }
 
 int
+svm_msg_q_wait_prod (svm_msg_q_t *mq)
+{
+  if (mq->q.evtfd == -1)
+    {
+      while (svm_msg_q_is_full (mq))
+	pthread_cond_wait (&mq->q.shr->condvar, &mq->q.shr->mutex);
+    }
+  else
+    {
+      u64 buf;
+      int rv;
+
+      while (svm_msg_q_is_full (mq))
+	{
+	  while ((rv = read (mq->q.evtfd, &buf, sizeof (buf))) < 0)
+	    {
+	      if (errno != EAGAIN)
+		{
+		  clib_unix_warning ("read error");
+		  return rv;
+		}
+	    }
+	}
+    }
+
+  return 0;
+}
+
+int
 svm_msg_q_timedwait (svm_msg_q_t *mq, double timeout)
 {
   if (mq->q.evtfd == -1)
diff --git a/src/svm/message_queue.h b/src/svm/message_queue.h
index 1ef773d9f0a..0e402c5a409 100644
--- a/src/svm/message_queue.h
+++ b/src/svm/message_queue.h
@@ -399,12 +399,20 @@ svm_msg_q_unlock (svm_msg_q_t * mq)
 /**
  * Wait for message queue event
  *
- * Must be called with mutex held. The queue only works non-blocking
- * with eventfds, so handle blocking calls as an exception here.
+ * When eventfds are not configured, the shared memory mutex is locked
+ * before waiting on the condvar. Typically called by consumers.
  */
 int svm_msg_q_wait (svm_msg_q_t *mq, svm_msg_q_wait_type_t type);
 
 /**
+ * Wait for message queue event as producer
+ *
+ * Similar to @ref svm_msg_q_wait but lock (mutex or spinlock) must
+ * be held. Should only be called by producers.
+ */
+int svm_msg_q_wait_prod (svm_msg_q_t *mq);
+
+/**
  * Timed wait for message queue event
  *
  * Must be called with mutex held.
diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h
index 87bcd902254..2683e356716 100644
--- a/src/vnet/session/application_interface.h
+++ b/src/vnet/session/application_interface.h
@@ -600,7 +600,7 @@ app_send_io_evt_to_vpp (svm_msg_q_t * mq, u32 session_index, u8 evt_type,
       svm_msg_q_lock (mq);
       while (svm_msg_q_ring_is_full (mq, SESSION_MQ_IO_EVT_RING)
 	     || svm_msg_q_is_full (mq))
-	svm_msg_q_wait (mq, SVM_MQ_WAIT_FULL);
+	svm_msg_q_wait_prod (mq);
       msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING);
       evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg);
       evt->session_index = session_index;