vlib_mains == 0 special cases be gone

Clean up spurious binary API client link dependency on libvlib.so,
which managed to hide behind vlib_mains == 0 checks reached by
VLIB_xxx_FUNCTION macros.

Change-Id: I5df1f8ab07dca1944250e643ccf06e60a8462325
Signed-off-by: Dave Barach <dave@barachs.net>

6 files changed, 65 insertions, 55 deletions

diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index 9f26bec7..6ba82584 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -261,7 +261,28 @@ done:
   return result;
 }
 
-vlib_main_t **vlib_mains;
+/*
+ * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1
+ * and vlib_mains[0] = &vlib_global_main from the beginning of time.
+ *
+ * The only place which should ever expand vlib_mains is start_workers()
+ * in threads.c. It knows about the bootstrap vector.
+ */
+/* *INDENT-OFF* */
+static struct
+{
+  vec_header_t h;
+  vlib_main_t *vm;
+} __attribute__ ((packed)) __bootstrap_vlib_main_vector
+  __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) =
+{
+  .h.len = 1,
+  .vm = &vlib_global_main,
+};
+/* *INDENT-ON* */
+
+vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm;
+
 
 /* When dubugging validate that given buffers are either known allocated
    or known free. */
@@ -280,7 +301,7 @@ vlib_buffer_validate_alloc_free (vlib_main_t * vm,
   ASSERT (os_get_cpu_number () == 0);
 
   /* smp disaster check */
-  if (vlib_mains)
+  if (vec_len (vlib_mains) > 1)
     ASSERT (vm == vlib_mains[0]);
 
   is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED;
@@ -956,7 +977,7 @@ show_buffers (vlib_main_t * vm,
 
   do
     {
-      curr_vm = vec_len (vlib_mains) ? vlib_mains[vm_index] : vm;
+      curr_vm = vlib_mains[vm_index];
       bm = curr_vm->buffer_main;
 
     /* *INDENT-OFF* */
diff --git a/src/vlib/global_funcs.h b/src/vlib/global_funcs.h
index bbdbdef5..f51ec381 100644
--- a/src/vlib/global_funcs.h
+++ b/src/vlib/global_funcs.h
@@ -23,7 +23,7 @@ always_inline vlib_main_t *
 vlib_get_main (void)
 {
   vlib_main_t *vm;
-  vm = vlib_mains ? vlib_mains[os_get_cpu_number ()] : &vlib_global_main;
+  vm = vlib_mains[os_get_cpu_number ()];
   ASSERT (vm);
   return vm;
 }
diff --git a/src/vlib/node_cli.c b/src/vlib/node_cli.c
index 05d0f0b5..62ab2e64 100644
--- a/src/vlib/node_cli.c
+++ b/src/vlib/node_cli.c
@@ -248,16 +248,11 @@ show_node_runtime (vlib_main_t * vm,
       if (unformat (input, "max") || unformat (input, "m"))
 	max = 1;
 
-      if (vec_len (vlib_mains) == 0)
-	vec_add1 (stat_vms, vm);
-      else
+      for (i = 0; i < vec_len (vlib_mains); i++)
 	{
-	  for (i = 0; i < vec_len (vlib_mains); i++)
-	    {
-	      stat_vm = vlib_mains[i];
-	      if (stat_vm)
-		vec_add1 (stat_vms, stat_vm);
-	    }
+	  stat_vm = vlib_mains[i];
+	  if (stat_vm)
+	    vec_add1 (stat_vms, stat_vm);
 	}
 
       /*
@@ -331,7 +326,7 @@ show_node_runtime (vlib_main_t * vm,
 		}
 	    }
 
-	  if (vec_len (vlib_mains))
+	  if (vec_len (vlib_mains) > 1)
 	    {
 	      vlib_worker_thread_t *w = vlib_worker_threads + j;
 	      if (j > 0)
@@ -404,16 +399,11 @@ clear_node_runtime (vlib_main_t * vm,
   vlib_main_t **stat_vms = 0, *stat_vm;
   vlib_node_runtime_t *r;
 
-  if (vec_len (vlib_mains) == 0)
-    vec_add1 (stat_vms, vm);
-  else
+  for (i = 0; i < vec_len (vlib_mains); i++)
     {
-      for (i = 0; i < vec_len (vlib_mains); i++)
-	{
-	  stat_vm = vlib_mains[i];
-	  if (stat_vm)
-	    vec_add1 (stat_vms, stat_vm);
-	}
+      stat_vm = vlib_mains[i];
+      if (stat_vm)
+	vec_add1 (stat_vms, stat_vm);
     }
 
   vlib_worker_thread_barrier_sync (vm);
diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h
index f49a8d6f..8ccfc438 100644
--- a/src/vlib/node_funcs.h
+++ b/src/vlib/node_funcs.h
@@ -201,7 +201,7 @@ vlib_get_frame_no_check (vlib_main_t * vm, uword frame_index)
   vlib_frame_t *f;
   u32 cpu_index = frame_index & VLIB_CPU_MASK;
   u32 offset = frame_index & VLIB_OFFSET_MASK;
-  vm = vlib_mains ? vlib_mains[cpu_index] : vm;
+  vm = vlib_mains[cpu_index];
   f = vm->heap_base + offset;
   return f;
 }
@@ -213,7 +213,7 @@ vlib_frame_index_no_check (vlib_main_t * vm, vlib_frame_t * f)
 
   ASSERT (((uword) f & VLIB_CPU_MASK) == 0);
 
-  vm = vlib_mains ? vlib_mains[f->cpu_index] : vm;
+  vm = vlib_mains[f->cpu_index];
 
   i = ((u8 *) f - (u8 *) vm->heap_base);
   return i | f->cpu_index;
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index e3ea3c9c..4676be97 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -570,9 +570,13 @@ start_workers (vlib_main_t * vm)
 
   if (n_vlib_mains > 1)
     {
-      vec_validate (vlib_mains, tm->n_vlib_mains - 1);
+      /* Replace hand-crafted length-1 vector with a real vector */
+      vlib_mains = 0;
+
+      vec_validate_aligned (vlib_mains, tm->n_vlib_mains - 1,
+			    CLIB_CACHE_LINE_BYTES);
       _vec_len (vlib_mains) = 0;
-      vec_add1 (vlib_mains, vm);
+      vec_add1_aligned (vlib_mains, vm, CLIB_CACHE_LINE_BYTES);
 
       vlib_worker_threads->wait_at_barrier =
 	clib_mem_alloc_aligned (sizeof (u32), CLIB_CACHE_LINE_BYTES);
@@ -685,7 +689,7 @@ start_workers (vlib_main_t * vm)
 	      /* Packet trace buffers are guaranteed to be empty, nothing to do here */
 
 	      clib_mem_set_heap (oldheap);
-	      vec_add1 (vlib_mains, vm_clone);
+	      vec_add1_aligned (vlib_mains, vm_clone, CLIB_CACHE_LINE_BYTES);
 
 	      vm_clone->error_main.counters =
 		vec_dup (vlib_mains[0]->error_main.counters);
@@ -805,7 +809,7 @@ vlib_worker_thread_node_runtime_update (void)
 
   ASSERT (os_get_cpu_number () == 0);
 
-  if (vec_len (vlib_mains) == 0)
+  if (vec_len (vlib_mains) == 1)
     return;
 
   vm = vlib_mains[0];
@@ -1148,7 +1152,7 @@ vlib_worker_thread_barrier_sync (vlib_main_t * vm)
   f64 deadline;
   u32 count;
 
-  if (!vlib_mains)
+  if (vec_len (vlib_mains) < 2)
     return;
 
   count = vec_len (vlib_mains) - 1;
@@ -1179,7 +1183,7 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm)
 {
   f64 deadline;
 
-  if (!vlib_mains)
+  if (vec_len (vlib_mains) < 2)
     return;
 
   if (--vlib_worker_threads[0].recursion_level > 0)
diff --git a/src/vlib/threads.h b/src/vlib/threads.h
index 75a5a281..a032311c 100644
--- a/src/vlib/threads.h
+++ b/src/vlib/threads.h
@@ -222,30 +222,25 @@ vlib_worker_thread_barrier_check (void)
     }
 }
 
-#define foreach_vlib_main(body)			                        \
-do {                                                                    \
-    vlib_main_t ** __vlib_mains = 0, *this_vlib_main;                   \
-    int ii;                                                             \
-                                                                        \
-    if (vec_len (vlib_mains) == 0)                                      \
-        vec_add1 (__vlib_mains, &vlib_global_main);                     \
-    else                                                                \
-    {                                                                   \
-        for (ii = 0; ii < vec_len (vlib_mains); ii++)                   \
-        {                                                               \
-            this_vlib_main = vlib_mains[ii];                            \
-            if (this_vlib_main)                                         \
-                vec_add1 (__vlib_mains, this_vlib_main);                \
-        }                                                               \
-    }                                                                   \
-                                                                        \
-    for (ii = 0; ii < vec_len (__vlib_mains); ii++)                     \
-    {                                                                   \
-        this_vlib_main = __vlib_mains[ii];                              \
-        /* body uses this_vlib_main... */                               \
-        (body);                                                         \
-    }                                                                   \
-    vec_free (__vlib_mains);                                            \
+#define foreach_vlib_main(body)                         \
+do {                                                    \
+  vlib_main_t ** __vlib_mains = 0, *this_vlib_main;     \
+  int ii;                                               \
+                                                        \
+  for (ii = 0; ii < vec_len (vlib_mains); ii++)         \
+    {                                                   \
+      this_vlib_main = vlib_mains[ii];                  \
+      if (this_vlib_main)                               \
+        vec_add1 (__vlib_mains, this_vlib_main);        \
+    }                                                   \
+                                                        \
+  for (ii = 0; ii < vec_len (__vlib_mains); ii++)       \
+    {                                                   \
+      this_vlib_main = __vlib_mains[ii];                \
+      /* body uses this_vlib_main... */                 \
+      (body);                                           \
+    }                                                   \
+  vec_free (__vlib_mains);                              \
 } while (0);
 
 #define foreach_sched_policy \