sort worker-thread init functions in advance

Otherwise, all N worker threads try to sort the list at the same time:
a good way to have a bad day.

This approach performs *far* better than maintaing order by adding a
spin-lock. By direct measurement w/ elog + g2: 11 threads execute the
per-thread init function list in 22us, vs. 50ms with a CLIB_PAUSE()
enabled spin-lock.

Change-Id: I1745f2a213c0561260139a60114dcb981e0c64e5
Signed-off-by: Dave Barach <dave@barachs.net>

5 files changed, 71 insertions, 12 deletions

diff --git a/src/vlib/init.c b/src/vlib/init.c
index 8010e9e97cd..135f9d582e9 100644
--- a/src/vlib/init.c
+++ b/src/vlib/init.c
@@ -72,7 +72,7 @@ comma_split (u8 * s, u8 ** a, u8 ** b)
  * @returns 0 on success, otherwise a clib_error_t *.
  */
 
-static clib_error_t *init_exit_function_sort
+clib_error_t *vlib_sort_init_exit_functions
   (_vlib_init_function_list_elt_t ** head)
 {
   uword *index_by_name;
@@ -329,15 +329,15 @@ again:
  * A and B - and it leads to hugely annoying debugging exercises.
  */
 
-clib_error_t *
-vlib_call_init_exit_functions (vlib_main_t * vm,
-			       _vlib_init_function_list_elt_t ** headp,
-			       int call_once)
+static inline clib_error_t *
+call_init_exit_functions_internal (vlib_main_t * vm,
+				   _vlib_init_function_list_elt_t ** headp,
+				   int call_once, int do_sort)
 {
   clib_error_t *error = 0;
   _vlib_init_function_list_elt_t *i;
 
-  if ((error = init_exit_function_sort (headp)))
+  if (do_sort && (error = vlib_sort_init_exit_functions (headp)))
     return (error);
 
   i = *headp;
@@ -357,6 +357,24 @@ vlib_call_init_exit_functions (vlib_main_t * vm,
 }
 
 clib_error_t *
+vlib_call_init_exit_functions (vlib_main_t * vm,
+			       _vlib_init_function_list_elt_t ** headp,
+			       int call_once)
+{
+  return call_init_exit_functions_internal (vm, headp, call_once,
+					    1 /* do_sort */ );
+}
+
+clib_error_t *
+vlib_call_init_exit_functions_no_sort (vlib_main_t * vm,
+				       _vlib_init_function_list_elt_t **
+				       headp, int call_once)
+{
+  return call_init_exit_functions_internal (vm, headp, call_once,
+					    0 /* do_sort */ );
+}
+
+clib_error_t *
 vlib_call_all_init_functions (vlib_main_t * vm)
 {
   /* Call dummy functions to make sure purely static modules are
diff --git a/src/vlib/init.h b/src/vlib/init.h
index 6d2711489d8..fc638013efc 100644
--- a/src/vlib/init.h
+++ b/src/vlib/init.h
@@ -328,7 +328,11 @@ clib_error_t *vlib_call_all_main_loop_exit_functions (struct vlib_main_t *vm);
 clib_error_t *vlib_call_init_exit_functions (struct vlib_main_t *vm,
 					     _vlib_init_function_list_elt_t **
 					     headp, int call_once);
-
+clib_error_t *vlib_call_init_exit_functions_no_sort (struct vlib_main_t *vm,
+						     _vlib_init_function_list_elt_t
+						     ** headp, int call_once);
+clib_error_t *vlib_sort_init_exit_functions (_vlib_init_function_list_elt_t
+					     **);
 #define foreach_vlib_module_reference		\
   _ (node_cli)					\
   _ (trace_cli)
diff --git a/src/vlib/main.c b/src/vlib/main.c
index b6006e8bab0..43400f8594c 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -1721,6 +1721,14 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
   if (is_main)
     {
       uword i;
+
+      /*
+       * Perform an initial barrier sync. Pays no attention to
+       * the barrier sync hold-down timer scheme, which won't work
+       * at this point in time.
+       */
+      vlib_worker_thread_initial_barrier_sync_and_release (vm);
+
       nm->current_process_index = ~0;
       for (i = 0; i < vec_len (nm->processes); i++)
 	cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0,
@@ -2090,6 +2098,9 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
   if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ )))
     goto done;
 
+  /* Sort per-thread init functions before we start threads */
+  vlib_sort_init_exit_functions (&vm->worker_init_function_registrations);
+
   /* Call all main loop enter functions. */
   {
     clib_error_t *sub_error;
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 22fa5f12ecd..4e4d60a5e62 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -1390,6 +1390,31 @@ vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which)
 #endif
 
 void
+vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm)
+{
+  f64 deadline;
+  f64 now = vlib_time_now (vm);
+  u32 count = vec_len (vlib_mains) - 1;
+
+  /* No worker threads? */
+  if (count == 0)
+    return;
+
+  deadline = now + BARRIER_SYNC_TIMEOUT;
+  *vlib_worker_threads->wait_at_barrier = 1;
+  while (*vlib_worker_threads->workers_at_barrier != count)
+    {
+      if ((now = vlib_time_now (vm)) > deadline)
+	{
+	  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
+	  os_panic ();
+	}
+      CLIB_PAUSE ();
+    }
+  *vlib_worker_threads->wait_at_barrier = 0;
+}
+
+void
 vlib_worker_thread_barrier_sync_int (vlib_main_t * vm, const char *func_name)
 {
   f64 deadline;
@@ -1732,15 +1757,15 @@ vlib_worker_thread_fn (void *arg)
   clib_time_init (&vm->clib_time);
   clib_mem_set_heap (w->thread_mheap);
 
-  /* Wait until the dpdk init sequence is complete */
-  while (tm->extern_thread_mgmt && tm->worker_thread_release == 0)
-    vlib_worker_thread_barrier_check ();
-
-  e = vlib_call_init_exit_functions
+  e = vlib_call_init_exit_functions_no_sort
     (vm, &vm->worker_init_function_registrations, 1 /* call_once */ );
   if (e)
     clib_error_report (e);
 
+  /* Wait until the dpdk init sequence is complete */
+  while (tm->extern_thread_mgmt && tm->worker_thread_release == 0)
+    vlib_worker_thread_barrier_check ();
+
   vlib_worker_loop (vm);
 }
 
diff --git a/src/vlib/threads.h b/src/vlib/threads.h
index 5a295e03319..fc34074127f 100644
--- a/src/vlib/threads.h
+++ b/src/vlib/threads.h
@@ -206,6 +206,7 @@ u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts);
 void vlib_worker_thread_barrier_sync_int (vlib_main_t * vm,
 					  const char *func_name);
 void vlib_worker_thread_barrier_release (vlib_main_t * vm);
+void vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm);
 void vlib_worker_thread_node_refork (void);
 
 static_always_inline uword