From ce359db3b68528ce576862129b2a7709681ad2c6 Mon Sep 17 00:00:00 2001
From: Damjan Marion <damarion@cisco.com>
Date: Thu, 16 Mar 2017 16:15:38 +0100
Subject: vlib: extend foreach_vlib_main macro to assert if workers are not
 parked

Change-Id: I6ff7b65a400734a47bc0a7d03faf86ef1cf4f8c8
Signed-off-by: Damjan Marion <damarion@cisco.com>
---
 src/vlib/main.h    |  3 +++
 src/vlib/threads.h | 36 ++++++++++++++++++++++++------------
 2 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/src/vlib/main.h b/src/vlib/main.h
index 98bc823d..0197b4f3 100644
--- a/src/vlib/main.h
+++ b/src/vlib/main.h
@@ -174,6 +174,9 @@ typedef struct vlib_main_t
   volatile u32 api_queue_nonempty;
   void (*queue_signal_callback) (struct vlib_main_t *);
   u8 **argv;
+
+  /* debugging */
+  volatile int parked_at_barrier;
 } vlib_main_t;
 
 /* Global main structure. */
diff --git a/src/vlib/threads.h b/src/vlib/threads.h
index 39f64e1d..eca4fc26 100644
--- a/src/vlib/threads.h
+++ b/src/vlib/threads.h
@@ -201,18 +201,6 @@ typedef enum
 
 void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which);
 
-static inline void
-vlib_worker_thread_barrier_check (void)
-{
-  if (PREDICT_FALSE (*vlib_worker_threads->wait_at_barrier))
-    {
-      clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1);
-      while (*vlib_worker_threads->wait_at_barrier)
-	;
-      clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1);
-    }
-}
-
 #define foreach_vlib_main(body)                         \
 do {                                                    \
   vlib_main_t ** __vlib_mains = 0, *this_vlib_main;     \
@@ -221,6 +209,8 @@ do {                                                    \
   for (ii = 0; ii < vec_len (vlib_mains); ii++)         \
     {                                                   \
       this_vlib_main = vlib_mains[ii];                  \
+      ASSERT (ii == 0 ||                                \
+	      this_vlib_main->parked_at_barrier == 1);  \
       if (this_vlib_main)                               \
         vec_add1 (__vlib_mains, this_vlib_main);        \
     }                                                   \
@@ -320,6 +310,8 @@ typedef struct
 
 extern vlib_thread_main_t vlib_thread_main;
 
+#include <vlib/global_funcs.h>
+
 #define VLIB_REGISTER_THREAD(x,...)                     \
   __VA_ARGS__ vlib_thread_registration_t x;             \
 static void __vlib_add_thread_registration_##x (void)   \
@@ -356,6 +348,26 @@ vlib_get_current_worker_index ()
   return os_get_cpu_number () - 1;
 }
 
+static inline void
+vlib_worker_thread_barrier_check (void)
+{
+  if (PREDICT_FALSE (*vlib_worker_threads->wait_at_barrier))
+    {
+      vlib_main_t *vm;
+      clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1);
+      if (CLIB_DEBUG > 0)
+	{
+	  vm = vlib_get_main ();
+	  vm->parked_at_barrier = 1;
+	}
+      while (*vlib_worker_threads->wait_at_barrier)
+	;
+      if (CLIB_DEBUG > 0)
+	vm->parked_at_barrier = 0;
+      clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1);
+    }
+}
+
 always_inline vlib_main_t *
 vlib_get_worker_vlib_main (u32 worker_index)
 {
-- 
cgit 1.2.3-korg