vlib: split vlib_main_t into global and per-thread

Type: refactor
Change-Id: I8b273bc3bf16aa360f031f1b2692f766e5fc4613
Signed-off-by: Damjan Marion <damarion@cisco.com>

15 files changed, 439 insertions, 393 deletions

diff --git a/src/vlib/buffer.c b/src/vlib/buffer.c
index f8cfb0357fa..ae88b4e4d44 100644
--- a/src/vlib/buffer.c
+++ b/src/vlib/buffer.c
@@ -309,29 +309,6 @@ done:
   return result;
 }
 
-/*
- * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1
- * and vlib_mains[0] = &vlib_global_main from the beginning of time.
- *
- * The only place which should ever expand vlib_mains is start_workers()
- * in threads.c. It knows about the bootstrap vector.
- */
-/* *INDENT-OFF* */
-static struct
-{
-  vec_header_t h;
-  vlib_main_t *vm;
-} __attribute__ ((packed)) __bootstrap_vlib_main_vector
-  __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) =
-{
-  .h.len = 1,
-  .vm = &vlib_global_main,
-};
-/* *INDENT-ON* */
-
-vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm;
-
-
 /* When debugging validate that given buffers are either known allocated
    or known free. */
 void
diff --git a/src/vlib/cli.c b/src/vlib/cli.c
index 3215a3264f5..5cad95ee004 100644
--- a/src/vlib/cli.c
+++ b/src/vlib/cli.c
@@ -205,10 +205,11 @@ get_sub_command (vlib_cli_main_t * cm, vlib_cli_command_t * parent, u32 si)
 static uword
 unformat_vlib_cli_sub_command (unformat_input_t * i, va_list * args)
 {
-  vlib_main_t *vm = va_arg (*args, vlib_main_t *);
+  vlib_main_t __clib_unused *vm = va_arg (*args, vlib_main_t *);
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   vlib_cli_command_t *c = va_arg (*args, vlib_cli_command_t *);
   vlib_cli_command_t **result = va_arg (*args, vlib_cli_command_t **);
-  vlib_cli_main_t *cm = &vm->cli_main;
+  vlib_cli_main_t *cm = &vgm->cli_main;
   uword *match_bitmap, is_unique, index;
 
   match_bitmap = vlib_cli_sub_command_match (c, i);
@@ -238,8 +239,8 @@ vlib_cli_get_possible_completions (u8 * str)
 {
   vlib_cli_command_t *c;
   vlib_cli_sub_command_t *sc;
-  vlib_main_t *vm = vlib_get_main ();
-  vlib_cli_main_t *vcm = &vm->cli_main;
+  vlib_global_main_t *vgm = vlib_get_global_main ();
+  vlib_cli_main_t *vcm = &vgm->cli_main;
   uword *match_bitmap = 0;
   uword index, is_unique, help_next_level;
   u8 **result = 0;
@@ -388,11 +389,12 @@ vlib_cli_dispatch_sub_commands (vlib_main_t * vm,
 				unformat_input_t * input,
 				uword parent_command_index)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   vlib_cli_command_t *parent, *c;
   clib_error_t *error = 0;
   unformat_input_t sub_input;
   u8 *string;
-  uword is_main_dispatch = cm == &vm->cli_main;
+  uword is_main_dispatch = cm == &vgm->cli_main;
 
   parent = vec_elt_at_index (cm->commands, parent_command_index);
   if (is_main_dispatch && unformat (input, "help"))
@@ -674,6 +676,7 @@ vlib_cli_input (vlib_main_t * vm,
 		unformat_input_t * input,
 		vlib_cli_output_function_t * function, uword function_arg)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   vlib_process_t *cp = vlib_get_current_process (vm);
   clib_error_t *error;
   vlib_cli_output_function_t *save_function;
@@ -688,7 +691,7 @@ vlib_cli_input (vlib_main_t * vm,
 
   do
     {
-      error = vlib_cli_dispatch_sub_commands (vm, &vm->cli_main, input,
+      error = vlib_cli_dispatch_sub_commands (vm, &vgm->cli_main, input,
 					      /* parent */ 0);
     }
   while (!error && !unformat (input, "%U", unformat_eof));
@@ -1086,6 +1089,7 @@ static clib_error_t *
 restart_cmd_fn (vlib_main_t * vm, unformat_input_t * input,
 		vlib_cli_command_t * cmd)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   clib_file_main_t *fm = &file_main;
   clib_file_t *f;
 
@@ -1102,7 +1106,7 @@ restart_cmd_fn (vlib_main_t * vm, unformat_input_t * input,
   /* *INDENT-ON* */
 
   /* Exec ourself */
-  execve (vm->name, (char **) vm->argv, environ);
+  execve (vgm->name, (char **) vm->argv, environ);
 
   return 0;
 }
@@ -1352,7 +1356,8 @@ vlib_cli_command_is_empty (vlib_cli_command_t * c)
 clib_error_t *
 vlib_cli_register (vlib_main_t * vm, vlib_cli_command_t * c)
 {
-  vlib_cli_main_t *cm = &vm->cli_main;
+  vlib_global_main_t *vgm = vlib_get_global_main ();
+  vlib_cli_main_t *cm = &vgm->cli_main;
   clib_error_t *error = 0;
   uword ci, *p;
   char *normalized_path;
@@ -1626,8 +1631,8 @@ sort_cmds_by_path (void *a1, void *a2)
 {
   u32 *index1 = a1;
   u32 *index2 = a2;
-  vlib_main_t *vm = vlib_get_main ();
-  vlib_cli_main_t *cm = &vm->cli_main;
+  vlib_global_main_t *vgm = vlib_get_global_main ();
+  vlib_cli_main_t *cm = &vgm->cli_main;
   vlib_cli_command_t *c1, *c2;
   int i, lmin;
 
@@ -1737,6 +1742,7 @@ static clib_error_t *
 show_cli_command_fn (vlib_main_t * vm,
 		     unformat_input_t * input, vlib_cli_command_t * cmd)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   int show_mp_safe = 0;
   int show_not_mp_safe = 0;
   int show_hit = 0;
@@ -1760,8 +1766,8 @@ show_cli_command_fn (vlib_main_t * vm,
   if (clear_hit == 0 && (show_mp_safe + show_not_mp_safe) == 0)
     show_mp_safe = show_not_mp_safe = 1;
 
-  vlib_cli_output (vm, "%U", format_mp_safe, &vm->cli_main,
-		   show_mp_safe, show_not_mp_safe, show_hit, clear_hit);
+  vlib_cli_output (vm, "%U", format_mp_safe, &vgm->cli_main, show_mp_safe,
+		   show_not_mp_safe, show_hit, clear_hit);
   if (clear_hit)
     vlib_cli_output (vm, "hit counters cleared...");
 
@@ -1821,7 +1827,8 @@ VLIB_CLI_COMMAND (show_cli_command, static) =
 static clib_error_t *
 vlib_cli_init (vlib_main_t * vm)
 {
-  vlib_cli_main_t *cm = &vm->cli_main;
+  vlib_global_main_t *vgm = vlib_get_global_main ();
+  vlib_cli_main_t *cm = &vgm->cli_main;
   clib_error_t *error = 0;
   vlib_cli_command_t *cmd;
 
diff --git a/src/vlib/cli.h b/src/vlib/cli.h
index c31236f8ab9..83b4eab1851 100644
--- a/src/vlib/cli.h
+++ b/src/vlib/cli.h
@@ -160,27 +160,27 @@ typedef struct vlib_cli_main_t
 } vlib_cli_main_t;
 
 #ifndef CLIB_MARCH_VARIANT
-#define VLIB_CLI_COMMAND(x,...)                                         \
-    __VA_ARGS__ vlib_cli_command_t x;                                   \
-static void __vlib_cli_command_registration_##x (void)                  \
-    __attribute__((__constructor__)) ;                                  \
-static void __vlib_cli_command_registration_##x (void)                  \
-{                                                                       \
-    vlib_main_t * vm = vlib_get_main();                                 \
-    vlib_cli_main_t *cm = &vm->cli_main;                                \
-    x.next_cli_command = cm->cli_command_registrations;                 \
-    cm->cli_command_registrations = &x;                                 \
-}                                                                       \
-static void __vlib_cli_command_unregistration_##x (void)                \
-    __attribute__((__destructor__)) ;                                   \
-static void __vlib_cli_command_unregistration_##x (void)                \
-{                                                                       \
-    vlib_main_t * vm = vlib_get_main();                                 \
-    vlib_cli_main_t *cm = &vm->cli_main;                                \
-    VLIB_REMOVE_FROM_LINKED_LIST (cm->cli_command_registrations, &x,    \
-                                  next_cli_command);                    \
-}                                                                       \
-__VA_ARGS__ vlib_cli_command_t x
+#define VLIB_CLI_COMMAND(x, ...)                                              \
+  __VA_ARGS__ vlib_cli_command_t x;                                           \
+  static void __vlib_cli_command_registration_##x (void)                      \
+    __attribute__ ((__constructor__));                                        \
+  static void __vlib_cli_command_registration_##x (void)                      \
+  {                                                                           \
+    vlib_global_main_t *vgm = vlib_get_global_main ();                        \
+    vlib_cli_main_t *cm = &vgm->cli_main;                                     \
+    x.next_cli_command = cm->cli_command_registrations;                       \
+    cm->cli_command_registrations = &x;                                       \
+  }                                                                           \
+  static void __vlib_cli_command_unregistration_##x (void)                    \
+    __attribute__ ((__destructor__));                                         \
+  static void __vlib_cli_command_unregistration_##x (void)                    \
+  {                                                                           \
+    vlib_global_main_t *vgm = vlib_get_global_main ();                        \
+    vlib_cli_main_t *cm = &vgm->cli_main;                                     \
+    VLIB_REMOVE_FROM_LINKED_LIST (cm->cli_command_registrations, &x,          \
+				  next_cli_command);                          \
+  }                                                                           \
+  __VA_ARGS__ vlib_cli_command_t x
 #else
 /* create unused pointer to silence compiler warnings and get whole
    function optimized out */
diff --git a/src/vlib/global_funcs.h b/src/vlib/global_funcs.h
index 63f8115a181..3c0fdb78364 100644
--- a/src/vlib/global_funcs.h
+++ b/src/vlib/global_funcs.h
@@ -22,14 +22,14 @@
 always_inline u32
 vlib_get_n_threads ()
 {
-  return vec_len (vlib_mains);
+  return vec_len (vlib_global_main.vlib_mains);
 }
 
 always_inline vlib_main_t *
 vlib_get_main_by_index (u32 thread_index)
 {
   vlib_main_t *vm;
-  vm = vlib_mains[thread_index];
+  vm = vlib_global_main.vlib_mains[thread_index];
   ASSERT (vm);
   return vm;
 }
@@ -46,6 +46,12 @@ vlib_get_first_main (void)
   return vlib_get_main_by_index (0);
 }
 
+always_inline vlib_global_main_t *
+vlib_get_global_main (void)
+{
+  return &vlib_global_main;
+}
+
 always_inline vlib_thread_main_t *
 vlib_get_thread_main ()
 {
@@ -55,8 +61,8 @@ vlib_get_thread_main ()
 always_inline elog_main_t *
 vlib_get_elog_main ()
 {
-  vlib_main_t *vm = vlib_get_first_main ();
-  return &vm->elog_main;
+  vlib_global_main_t *vgm = vlib_get_global_main ();
+  return &vgm->elog_main;
 }
 
 #endif /* included_vlib_global_funcs_h_ */
diff --git a/src/vlib/init.c b/src/vlib/init.c
index 9b43ca3cc26..4a56ab20b4a 100644
--- a/src/vlib/init.c
+++ b/src/vlib/init.c
@@ -330,10 +330,11 @@ again:
  */
 
 static inline clib_error_t *
-call_init_exit_functions_internal (vlib_main_t * vm,
-				   _vlib_init_function_list_elt_t ** headp,
-				   int call_once, int do_sort)
+call_init_exit_functions_internal (vlib_main_t *vm,
+				   _vlib_init_function_list_elt_t **headp,
+				   int call_once, int do_sort, int is_global)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   clib_error_t *error = 0;
   _vlib_init_function_list_elt_t *i;
 
@@ -343,10 +344,22 @@ call_init_exit_functions_internal (vlib_main_t * vm,
   i = *headp;
   while (i)
     {
-      if (call_once && !hash_get (vm->init_functions_called, i->f))
+      uword *h;
+
+      if (is_global)
+	h = hash_get (vgm->init_functions_called, i->f);
+      else
+	h = hash_get (vm->worker_init_functions_called, i->f);
+
+      if (call_once && !h)
 	{
 	  if (call_once)
-	    hash_set1 (vm->init_functions_called, i->f);
+	    {
+	      if (is_global)
+		hash_set1 (vgm->init_functions_called, i->f);
+	      else
+		hash_set1 (vm->worker_init_functions_called, i->f);
+	    }
 	  error = i->f (vm);
 	  if (error)
 	    return error;
@@ -357,54 +370,60 @@ call_init_exit_functions_internal (vlib_main_t * vm,
 }
 
 clib_error_t *
-vlib_call_init_exit_functions (vlib_main_t * vm,
-			       _vlib_init_function_list_elt_t ** headp,
-			       int call_once)
+vlib_call_init_exit_functions (vlib_main_t *vm,
+			       _vlib_init_function_list_elt_t **headp,
+			       int call_once, int is_global)
 {
   return call_init_exit_functions_internal (vm, headp, call_once,
-					    1 /* do_sort */ );
+					    1 /* do_sort */, is_global);
 }
 
 clib_error_t *
-vlib_call_init_exit_functions_no_sort (vlib_main_t * vm,
-				       _vlib_init_function_list_elt_t **
-				       headp, int call_once)
+vlib_call_init_exit_functions_no_sort (vlib_main_t *vm,
+				       _vlib_init_function_list_elt_t **headp,
+				       int call_once, int is_global)
 {
   return call_init_exit_functions_internal (vm, headp, call_once,
-					    0 /* do_sort */ );
+					    0 /* do_sort */, is_global);
 }
 
 clib_error_t *
 vlib_call_all_init_functions (vlib_main_t * vm)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   /* Call placeholder functions to make sure purely static modules are
      linked in. */
 #define _(f) vlib_##f##_reference ();
   foreach_vlib_module_reference;
 #undef _
 
-  return vlib_call_init_exit_functions
-    (vm, &vm->init_function_registrations, 1 /* call_once */ );
+  return vlib_call_init_exit_functions (vm, &vgm->init_function_registrations,
+					1 /* call_once */, 1 /* is_global */);
 }
 
 clib_error_t *
 vlib_call_all_main_loop_enter_functions (vlib_main_t * vm)
 {
-  return vlib_call_init_exit_functions
-    (vm, &vm->main_loop_enter_function_registrations, 1 /* call_once */ );
+  vlib_global_main_t *vgm = vlib_get_global_main ();
+  return vlib_call_init_exit_functions (
+    vm, &vgm->main_loop_enter_function_registrations, 1 /* call_once */,
+    1 /* is_global */);
 }
 
 clib_error_t *
 vlib_call_all_main_loop_exit_functions (vlib_main_t * vm)
 {
-  return vlib_call_init_exit_functions
-    (vm, &vm->main_loop_exit_function_registrations, 1 /* call_once */ );
+  vlib_global_main_t *vgm = vlib_get_global_main ();
+  return vlib_call_init_exit_functions (
+    vm, &vgm->main_loop_exit_function_registrations, 1 /* call_once */,
+    1 /* is_global */);
 }
 
 clib_error_t *
 vlib_call_all_config_functions (vlib_main_t * vm,
 				unformat_input_t * input, int is_early)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   clib_error_t *error = 0;
   vlib_config_function_runtime_t *c, **all;
   uword *hash = 0, *p;
@@ -413,7 +432,7 @@ vlib_call_all_config_functions (vlib_main_t * vm,
   hash = hash_create_string (0, sizeof (uword));
   all = 0;
 
-  c = vm->config_function_registrations;
+  c = vgm->config_function_registrations;
 
   while (c)
     {
@@ -450,9 +469,9 @@ vlib_call_all_config_functions (vlib_main_t * vm,
 	continue;
 
       /* Already called? */
-      if (hash_get (vm->init_functions_called, c->function))
+      if (hash_get (vgm->init_functions_called, c->function))
 	continue;
-      hash_set1 (vm->init_functions_called, c->function);
+      hash_set1 (vgm->init_functions_called, c->function);
 
       error = c->function (vm, &c->input);
       if (error)
@@ -473,11 +492,11 @@ done:
 void
 vlib_init_dump (void)
 {
-  vlib_main_t *vm = vlib_get_main ();
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   int i = 0;
 
   _vlib_init_function_list_elt_t *head, *this;
-  head = vm->init_function_registrations;
+  head = vgm->init_function_registrations;
 
   this = head;
   while (this)
@@ -492,6 +511,7 @@ show_init_function_command_fn (vlib_main_t * vm,
 			       unformat_input_t * input,
 			       vlib_cli_command_t * cmd)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   int which = 1;
   int verbose = 0;
   int i, n_init_fns;
@@ -524,13 +544,13 @@ show_init_function_command_fn (vlib_main_t * vm,
   switch (which)
     {
     case 1:
-      head = vm->init_function_registrations;
+      head = vgm->init_function_registrations;
       break;
     case 2:
-      head = vm->main_loop_enter_function_registrations;
+      head = vgm->main_loop_enter_function_registrations;
       break;
     case 3:
-      head = vm->main_loop_exit_function_registrations;
+      head = vgm->main_loop_exit_function_registrations;
       break;
     default:
       return clib_error_return (0, "BUG");
diff --git a/src/vlib/init.h b/src/vlib/init.h
index c5447308513..e6235652ad1 100644
--- a/src/vlib/init.h
+++ b/src/vlib/init.h
@@ -120,49 +120,48 @@ typedef struct vlib_config_function_runtime_t
    be called from other modules to resolve init function depend. */
 
 #ifndef CLIB_MARCH_VARIANT
-#define VLIB_DECLARE_INIT_FUNCTION(x, tag)                              \
-vlib_init_function_t * _VLIB_INIT_FUNCTION_SYMBOL (x, tag) = x;         \
-static void __vlib_add_##tag##_function_##x (void)                      \
-    __attribute__((__constructor__)) ;                                  \
-static _vlib_init_function_list_elt_t _vlib_init_function_##tag##_##x;    \
-static void __vlib_add_##tag##_function_##x (void)                      \
-{                                                                       \
- vlib_main_t * vm = vlib_get_main();                                    \
- _vlib_init_function_##tag##_##x.next_init_function                       \
-    = vm->tag##_function_registrations;                                 \
-  vm->tag##_function_registrations = &_vlib_init_function_##tag##_##x;    \
- _vlib_init_function_##tag##_##x.f = &x;                                  \
- _vlib_init_function_##tag##_##x.name = #x;                               \
-}                                                                       \
-static void __vlib_rm_##tag##_function_##x (void)                       \
-    __attribute__((__destructor__)) ;                                   \
-static void __vlib_rm_##tag##_function_##x (void)                       \
-{                                                                       \
-  vlib_main_t * vm = vlib_get_main();                                   \
-  _vlib_init_function_list_elt_t *this, *prev;                          \
-  this = vm->tag##_function_registrations;                              \
-  if (this == 0)							\
-    return;								\
-  if (this->f == &x)  				                        \
-    {                                                                   \
-      vm->tag##_function_registrations = this->next_init_function;	\
-      return;                                                           \
-    }                                                                   \
-  prev = this;								\
-  this = this->next_init_function;					\
-  while (this)								\
-    {                                                                   \
-      if (this->f == &x)		                                \
-        {                                                               \
-          prev->next_init_function =                                    \
-            this->next_init_function;					\
-          return;                                                       \
-        }                                                               \
-      prev = this;							\
-      this = this->next_init_function;                                  \
-    }                                                                   \
-}									\
-static _vlib_init_function_list_elt_t _vlib_init_function_##tag##_##x
+#define VLIB_DECLARE_INIT_FUNCTION(x, tag)                                    \
+  vlib_init_function_t *_VLIB_INIT_FUNCTION_SYMBOL (x, tag) = x;              \
+  static void __vlib_add_##tag##_function_##x (void)                          \
+    __attribute__ ((__constructor__));                                        \
+  static _vlib_init_function_list_elt_t _vlib_init_function_##tag##_##x;      \
+  static void __vlib_add_##tag##_function_##x (void)                          \
+  {                                                                           \
+    vlib_global_main_t *vgm = vlib_get_global_main ();                        \
+    _vlib_init_function_##tag##_##x.next_init_function =                      \
+      vgm->tag##_function_registrations;                                      \
+    vgm->tag##_function_registrations = &_vlib_init_function_##tag##_##x;     \
+    _vlib_init_function_##tag##_##x.f = &x;                                   \
+    _vlib_init_function_##tag##_##x.name = #x;                                \
+  }                                                                           \
+  static void __vlib_rm_##tag##_function_##x (void)                           \
+    __attribute__ ((__destructor__));                                         \
+  static void __vlib_rm_##tag##_function_##x (void)                           \
+  {                                                                           \
+    vlib_global_main_t *vgm = vlib_get_global_main ();                        \
+    _vlib_init_function_list_elt_t *this, *prev;                              \
+    this = vgm->tag##_function_registrations;                                 \
+    if (this == 0)                                                            \
+      return;                                                                 \
+    if (this->f == &x)                                                        \
+      {                                                                       \
+	vgm->tag##_function_registrations = this->next_init_function;         \
+	return;                                                               \
+      }                                                                       \
+    prev = this;                                                              \
+    this = this->next_init_function;                                          \
+    while (this)                                                              \
+      {                                                                       \
+	if (this->f == &x)                                                    \
+	  {                                                                   \
+	    prev->next_init_function = this->next_init_function;              \
+	    return;                                                           \
+	  }                                                                   \
+	prev = this;                                                          \
+	this = this->next_init_function;                                      \
+      }                                                                       \
+  }                                                                           \
+  static _vlib_init_function_list_elt_t _vlib_init_function_##tag##_##x
 #else
 /* create unused pointer to silence compiler warnings and get whole
    function optimized out */
@@ -179,35 +178,30 @@ static __clib_unused void * __clib_unused_##tag##_##x = x
 VLIB_DECLARE_INIT_FUNCTION(x,main_loop_exit)
 
 #ifndef CLIB_MARCH_VARIANT
-#define VLIB_CONFIG_FUNCTION(x,n,...)                           \
-    __VA_ARGS__ vlib_config_function_runtime_t                  \
-    VLIB_CONFIG_FUNCTION_SYMBOL(x);                             \
-static void __vlib_add_config_function_##x (void)               \
-    __attribute__((__constructor__)) ;                          \
-static void __vlib_add_config_function_##x (void)               \
-{                                                               \
-    vlib_main_t * vm = vlib_get_main();                         \
-    VLIB_CONFIG_FUNCTION_SYMBOL(x).next_registration            \
-       = vm->config_function_registrations;                     \
-    vm->config_function_registrations                           \
-       = &VLIB_CONFIG_FUNCTION_SYMBOL(x);                       \
-}                                                               \
-static void __vlib_rm_config_function_##x (void)                \
-    __attribute__((__destructor__)) ;                           \
-static void __vlib_rm_config_function_##x (void)                \
-{                                                               \
-    vlib_main_t * vm = vlib_get_main();                         \
-    vlib_config_function_runtime_t *p =                         \
-       & VLIB_CONFIG_FUNCTION_SYMBOL (x);                       \
-    VLIB_REMOVE_FROM_LINKED_LIST                                \
-      (vm->config_function_registrations, p, next_registration);\
-}                                                               \
-  vlib_config_function_runtime_t                                \
-    VLIB_CONFIG_FUNCTION_SYMBOL (x)                             \
-  = {                                                           \
-    .name = n,                                                  \
-    .function = x,                                              \
-    .is_early = 0,						\
+#define VLIB_CONFIG_FUNCTION(x, n, ...)                                       \
+  __VA_ARGS__ vlib_config_function_runtime_t VLIB_CONFIG_FUNCTION_SYMBOL (x); \
+  static void __vlib_add_config_function_##x (void)                           \
+    __attribute__ ((__constructor__));                                        \
+  static void __vlib_add_config_function_##x (void)                           \
+  {                                                                           \
+    vlib_global_main_t *vgm = vlib_get_global_main ();                        \
+    VLIB_CONFIG_FUNCTION_SYMBOL (x).next_registration =                       \
+      vgm->config_function_registrations;                                     \
+    vgm->config_function_registrations = &VLIB_CONFIG_FUNCTION_SYMBOL (x);    \
+  }                                                                           \
+  static void __vlib_rm_config_function_##x (void)                            \
+    __attribute__ ((__destructor__));                                         \
+  static void __vlib_rm_config_function_##x (void)                            \
+  {                                                                           \
+    vlib_global_main_t *vgm = vlib_get_global_main ();                        \
+    vlib_config_function_runtime_t *p = &VLIB_CONFIG_FUNCTION_SYMBOL (x);     \
+    VLIB_REMOVE_FROM_LINKED_LIST (vgm->config_function_registrations, p,      \
+				  next_registration);                         \
+  }                                                                           \
+  vlib_config_function_runtime_t VLIB_CONFIG_FUNCTION_SYMBOL (x) = {          \
+    .name = n,                                                                \
+    .function = x,                                                            \
+    .is_early = 0,                                                            \
   }
 #else
 /* create unused pointer to silence compiler warnings and get whole
@@ -223,35 +217,30 @@ static void __vlib_rm_config_function_##x (void)                \
 #endif
 
 #ifndef CLIB_MARCH_VARIANT
-#define VLIB_EARLY_CONFIG_FUNCTION(x,n,...)                     \
-    __VA_ARGS__ vlib_config_function_runtime_t                  \
-    VLIB_CONFIG_FUNCTION_SYMBOL(x);                             \
-static void __vlib_add_config_function_##x (void)               \
-    __attribute__((__constructor__)) ;                          \
-static void __vlib_add_config_function_##x (void)               \
-{                                                               \
-    vlib_main_t * vm = vlib_get_main();                         \
-    VLIB_CONFIG_FUNCTION_SYMBOL(x).next_registration            \
-       = vm->config_function_registrations;                     \
-    vm->config_function_registrations                           \
-       = &VLIB_CONFIG_FUNCTION_SYMBOL(x);                       \
-}                                                               \
-static void __vlib_rm_config_function_##x (void)                \
-    __attribute__((__destructor__)) ;                           \
-static void __vlib_rm_config_function_##x (void)                \
-{                                                               \
-    vlib_main_t * vm = vlib_get_main();                         \
-    vlib_config_function_runtime_t *p =                         \
-       & VLIB_CONFIG_FUNCTION_SYMBOL (x);                       \
-    VLIB_REMOVE_FROM_LINKED_LIST                                \
-      (vm->config_function_registrations, p, next_registration);\
-}                                                               \
-  vlib_config_function_runtime_t                                \
-    VLIB_CONFIG_FUNCTION_SYMBOL (x)                             \
-  = {                                                           \
-    .name = n,                                                  \
-    .function = x,                                              \
-    .is_early = 1,						\
+#define VLIB_EARLY_CONFIG_FUNCTION(x, n, ...)                                 \
+  __VA_ARGS__ vlib_config_function_runtime_t VLIB_CONFIG_FUNCTION_SYMBOL (x); \
+  static void __vlib_add_config_function_##x (void)                           \
+    __attribute__ ((__constructor__));                                        \
+  static void __vlib_add_config_function_##x (void)                           \
+  {                                                                           \
+    vlib_global_main_t *vgm = vlib_get_global_main ();                        \
+    VLIB_CONFIG_FUNCTION_SYMBOL (x).next_registration =                       \
+      vgm->config_function_registrations;                                     \
+    vgm->config_function_registrations = &VLIB_CONFIG_FUNCTION_SYMBOL (x);    \
+  }                                                                           \
+  static void __vlib_rm_config_function_##x (void)                            \
+    __attribute__ ((__destructor__));                                         \
+  static void __vlib_rm_config_function_##x (void)                            \
+  {                                                                           \
+    vlib_global_main_t *vgm = vlib_get_global_main ();                        \
+    vlib_config_function_runtime_t *p = &VLIB_CONFIG_FUNCTION_SYMBOL (x);     \
+    VLIB_REMOVE_FROM_LINKED_LIST (vgm->config_function_registrations, p,      \
+				  next_registration);                         \
+  }                                                                           \
+  vlib_config_function_runtime_t VLIB_CONFIG_FUNCTION_SYMBOL (x) = {          \
+    .name = n,                                                                \
+    .function = x,                                                            \
+    .is_early = 1,                                                            \
   }
 #else
 /* create unused pointer to silence compiler warnings and get whole
@@ -267,67 +256,71 @@ static void __vlib_rm_config_function_##x (void)                \
 #endif
 
 /* Call given init function: used for init function dependencies. */
-#define vlib_call_init_function(vm, x)					\
-  ({									\
-    extern vlib_init_function_t * VLIB_INIT_FUNCTION_SYMBOL (x);	\
-    vlib_init_function_t * _f = VLIB_INIT_FUNCTION_SYMBOL (x);		\
-    clib_error_t * _error = 0;						\
-    if (! hash_get (vm->init_functions_called, _f))			\
-      {									\
-	hash_set1 (vm->init_functions_called, _f);			\
-	_error = _f (vm);						\
-      }									\
-    _error;								\
+#define vlib_call_init_function(vm, x)                                        \
+  ({                                                                          \
+    vlib_global_main_t *vgm = &vlib_global_main;                              \
+    extern vlib_init_function_t *VLIB_INIT_FUNCTION_SYMBOL (x);               \
+    vlib_init_function_t *_f = VLIB_INIT_FUNCTION_SYMBOL (x);                 \
+    clib_error_t *_error = 0;                                                 \
+    if (!hash_get (vgm->init_functions_called, _f))                           \
+      {                                                                       \
+	hash_set1 (vgm->init_functions_called, _f);                           \
+	_error = _f (vm);                                                     \
+      }                                                                       \
+    _error;                                                                   \
   })
 
 /* Don't call given init function: used to suppress parts of the netstack */
-#define vlib_mark_init_function_complete(vm, x)				\
-  ({									\
-    extern vlib_init_function_t * VLIB_INIT_FUNCTION_SYMBOL (x);	\
-    vlib_init_function_t * _f = VLIB_INIT_FUNCTION_SYMBOL (x);		\
-    hash_set1 (vm->init_functions_called, _f);				\
+#define vlib_mark_init_function_complete(vm, x)                               \
+  ({                                                                          \
+    vlib_global_main_t *vgm = &vlib_global_main;                              \
+    extern vlib_init_function_t *VLIB_INIT_FUNCTION_SYMBOL (x);               \
+    vlib_init_function_t *_f = VLIB_INIT_FUNCTION_SYMBOL (x);                 \
+    hash_set1 (vgm->init_functions_called, _f);                               \
   })
 
-#define vlib_call_post_graph_init_function(vm, x)			\
-  ({									\
-    extern vlib_init_function_t * VLIB_POST_GRAPH_INIT_FUNCTION_SYMBOL (x); \
-    vlib_init_function_t * _f = VLIB_POST_GRAPH_INIT_FUNCTION_SYMBOL (x); \
-    clib_error_t * _error = 0;						\
-    if (! hash_get (vm->init_functions_called, _f))			\
-      {									\
-	hash_set1 (vm->init_functions_called, _f);			\
-	_error = _f (vm);						\
-      }									\
-    _error;								\
+#define vlib_call_post_graph_init_function(vm, x)                             \
+  ({                                                                          \
+    vlib_global_main_t *vgm = &vlib_global_main;                              \
+    extern vlib_init_function_t *VLIB_POST_GRAPH_INIT_FUNCTION_SYMBOL (x);    \
+    vlib_init_function_t *_f = VLIB_POST_GRAPH_INIT_FUNCTION_SYMBOL (x);      \
+    clib_error_t *_error = 0;                                                 \
+    if (!hash_get (vgm->init_functions_called, _f))                           \
+      {                                                                       \
+	hash_set1 (vgm->init_functions_called, _f);                           \
+	_error = _f (vm);                                                     \
+      }                                                                       \
+    _error;                                                                   \
   })
 
-#define vlib_call_config_function(vm, x)			\
-  ({								\
-    vlib_config_function_runtime_t * _r;			\
-    clib_error_t * _error = 0;					\
-    extern vlib_config_function_runtime_t			\
-      VLIB_CONFIG_FUNCTION_SYMBOL (x);				\
-								\
-    _r = &VLIB_CONFIG_FUNCTION_SYMBOL (x);			\
-    if (! hash_get (vm->init_functions_called, _r->function))	\
-      {								\
-        hash_set1 (vm->init_functions_called, _r->function);	\
-	_error = _r->function (vm, &_r->input);			\
-      }								\
-    _error;							\
+#define vlib_call_config_function(vm, x)                                      \
+  ({                                                                          \
+    vlib_global_main_t *vgm = &vlib_global_main;                              \
+    vlib_config_function_runtime_t *_r;                                       \
+    clib_error_t *_error = 0;                                                 \
+    extern vlib_config_function_runtime_t VLIB_CONFIG_FUNCTION_SYMBOL (x);    \
+                                                                              \
+    _r = &VLIB_CONFIG_FUNCTION_SYMBOL (x);                                    \
+    if (!hash_get (vgm->init_functions_called, _r->function))                 \
+      {                                                                       \
+	hash_set1 (vgm->init_functions_called, _r->function);                 \
+	_error = _r->function (vm, &_r->input);                               \
+      }                                                                       \
+    _error;                                                                   \
   })
 
-#define vlib_call_main_loop_enter_function(vm, x)			\
-  ({									\
-    extern vlib_init_function_t * VLIB_MAIN_LOOP_ENTER_FUNCTION_SYMBOL (x); \
-    vlib_init_function_t * _f = VLIB_MAIN_LOOP_ENTER_FUNCTION_SYMBOL (x); \
-    clib_error_t * _error = 0;						\
-    if (! hash_get (vm->init_functions_called, _f))			\
-      {									\
-	hash_set1 (vm->init_functions_called, _f);			\
-	_error = _f (vm);						\
-      }									\
-    _error;								\
+#define vlib_call_main_loop_enter_function(vm, x)                             \
+  ({                                                                          \
+    vlib_global_main_t *vgm = &vlib_global_main;                              \
+    extern vlib_init_function_t *VLIB_MAIN_LOOP_ENTER_FUNCTION_SYMBOL (x);    \
+    vlib_init_function_t *_f = VLIB_MAIN_LOOP_ENTER_FUNCTION_SYMBOL (x);      \
+    clib_error_t *_error = 0;                                                 \
+    if (!hash_get (vgm->init_functions_called, _f))                           \
+      {                                                                       \
+	hash_set1 (vgm->init_functions_called, _f);                           \
+	_error = _f (vm);                                                     \
+      }                                                                       \
+    _error;                                                                   \
   })
 
 /* External functions. */
@@ -338,12 +331,14 @@ clib_error_t *vlib_call_all_config_functions (struct vlib_main_t *vm,
 clib_error_t *vlib_call_all_main_loop_enter_functions (struct vlib_main_t
 						       *vm);
 clib_error_t *vlib_call_all_main_loop_exit_functions (struct vlib_main_t *vm);
-clib_error_t *vlib_call_init_exit_functions (struct vlib_main_t *vm,
-					     _vlib_init_function_list_elt_t **
-					     headp, int call_once);
-clib_error_t *vlib_call_init_exit_functions_no_sort (struct vlib_main_t *vm,
-						     _vlib_init_function_list_elt_t
-						     ** headp, int call_once);
+clib_error_t *
+vlib_call_init_exit_functions (struct vlib_main_t *vm,
+			       _vlib_init_function_list_elt_t **headp,
+			       int call_once, int is_global);
+clib_error_t *
+vlib_call_init_exit_functions_no_sort (struct vlib_main_t *vm,
+				       _vlib_init_function_list_elt_t **headp,
+				       int call_once, int is_global);
 clib_error_t *vlib_sort_init_exit_functions (_vlib_init_function_list_elt_t
 					     **);
 #define foreach_vlib_module_reference		\
diff --git a/src/vlib/main.c b/src/vlib/main.c
index 62da7b9ff0a..bee63970f3d 100644
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -614,7 +614,7 @@ vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n)
   if (n->type == VLIB_NODE_TYPE_PROCESS)
     {
       /* Nothing to do for PROCESS nodes except in main thread */
-      if (vm != &vlib_global_main)
+      if (vm != vlib_get_first_main ())
 	return;
 
       vlib_process_t *p = vlib_get_process_from_node (vm, n);
@@ -747,10 +747,10 @@ elog_save_buffer (vlib_main_t * vm,
 void
 vlib_post_mortem_dump (void)
 {
-  vlib_main_t *vm = &vlib_global_main;
+  vlib_global_main_t *vgm = vlib_get_global_main ();
 
-  for (int i = 0; i < vec_len (vm->post_mortem_callbacks); i++)
-    (vm->post_mortem_callbacks[i]) ();
+  for (int i = 0; i < vec_len (vgm->post_mortem_callbacks); i++)
+    (vgm->post_mortem_callbacks[i]) ();
 }
 
 /* *INDENT-OFF* */
@@ -901,7 +901,7 @@ vlib_elog_main_loop_event (vlib_main_t * vm,
 			   u32 node_index,
 			   u64 time, u32 n_vectors, u32 is_return)
 {
-  vlib_main_t *evm = &vlib_global_main;
+  vlib_main_t *evm = vlib_get_first_main ();
   elog_main_t *em = vlib_get_elog_main ();
   int enabled = evm->elog_trace_graph_dispatch |
     evm->elog_trace_graph_circuit;
@@ -1091,7 +1091,8 @@ dispatch_node (vlib_main_t * vm,
 	  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1;
 	  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1;
 
-	  if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch))
+	  if (PREDICT_FALSE (
+		vlib_get_first_main ()->elog_trace_graph_dispatch))
 	    {
 	      vlib_worker_thread_t *w = vlib_worker_threads
 		+ vm->thread_index;
@@ -1126,7 +1127,8 @@ dispatch_node (vlib_main_t * vm,
 		+ vm->thread_index;
 	      node->flags |=
 		VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE;
-	      if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch))
+	      if (PREDICT_FALSE (
+		    vlib_get_first_main ()->elog_trace_graph_dispatch))
 		{
 		  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
 					w->elog_track);
@@ -1794,26 +1796,26 @@ vlib_worker_loop (vlib_main_t * vm)
   vlib_main_or_worker_loop (vm, /* is_main */ 0);
 }
 
-vlib_main_t vlib_global_main;
+vlib_global_main_t vlib_global_main;
 
 void
 vlib_add_del_post_mortem_callback (void *cb, int is_add)
 {
-  vlib_main_t *vm = &vlib_global_main;
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   int i;
 
   if (is_add == 0)
     {
-      for (i = vec_len (vm->post_mortem_callbacks) - 1; i >= 0; i--)
-	if (vm->post_mortem_callbacks[i] == cb)
-	  vec_del1 (vm->post_mortem_callbacks, i);
+      for (i = vec_len (vgm->post_mortem_callbacks) - 1; i >= 0; i--)
+	if (vgm->post_mortem_callbacks[i] == cb)
+	  vec_del1 (vgm->post_mortem_callbacks, i);
       return;
     }
 
-  for (i = 0; i < vec_len (vm->post_mortem_callbacks); i++)
-    if (vm->post_mortem_callbacks[i] == cb)
+  for (i = 0; i < vec_len (vgm->post_mortem_callbacks); i++)
+    if (vgm->post_mortem_callbacks[i] == cb)
       return;
-  vec_add1 (vm->post_mortem_callbacks, cb);
+  vec_add1 (vgm->post_mortem_callbacks, cb);
 }
 
 static void
@@ -1836,6 +1838,7 @@ elog_post_mortem_dump (void)
 static clib_error_t *
 vlib_main_configure (vlib_main_t * vm, unformat_input_t * input)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   int turn_on_mem_trace = 0;
 
   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
@@ -1844,9 +1847,9 @@ vlib_main_configure (vlib_main_t * vm, unformat_input_t * input)
 	turn_on_mem_trace = 1;
 
       else if (unformat (input, "elog-events %d",
-			 &vm->configured_elog_ring_size))
-	vm->configured_elog_ring_size =
-	  1 << max_log2 (vm->configured_elog_ring_size);
+			 &vgm->configured_elog_ring_size))
+	vgm->configured_elog_ring_size =
+	  1 << max_log2 (vgm->configured_elog_ring_size);
       else if (unformat (input, "elog-post-mortem-dump"))
 	vlib_add_del_post_mortem_callback (elog_post_mortem_dump,
 					   /* is_add */ 1);
@@ -1923,21 +1926,22 @@ vl_api_get_elog_trace_api_messages (void)
 int
 vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   clib_error_t *volatile error;
   vlib_node_main_t *nm = &vm->node_main;
 
   vm->queue_signal_callback = placeholder_queue_signal_callback;
 
   /* Reconfigure event log which is enabled very early */
-  if (vm->configured_elog_ring_size &&
-      vm->configured_elog_ring_size != vm->elog_main.event_ring_size)
-    elog_resize (&vm->elog_main, vm->configured_elog_ring_size);
+  if (vgm->configured_elog_ring_size &&
+      vgm->configured_elog_ring_size != vgm->elog_main.event_ring_size)
+    elog_resize (&vgm->elog_main, vgm->configured_elog_ring_size);
   vl_api_set_elog_main (vlib_get_elog_main ());
   (void) vl_api_set_elog_trace_api_messages (1);
 
   /* Default name. */
-  if (!vm->name)
-    vm->name = "VLIB";
+  if (!vgm->name)
+    vgm->name = "VLIB";
 
   if ((error = vlib_physmem_init (vm)))
     {
@@ -2005,8 +2009,8 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
     }
 
   /* See unix/main.c; most likely already set up */
-  if (vm->init_functions_called == 0)
-    vm->init_functions_called = hash_create (0, /* value bytes */ 0);
+  if (vgm->init_functions_called == 0)
+    vgm->init_functions_called = hash_create (0, /* value bytes */ 0);
   if ((error = vlib_call_all_init_functions (vm)))
     goto done;
 
@@ -2048,7 +2052,7 @@ vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
   vm->damping_constant = exp (-1.0 / 20.0);
 
   /* Sort per-thread init functions before we start threads */
-  vlib_sort_init_exit_functions (&vm->worker_init_function_registrations);
+  vlib_sort_init_exit_functions (&vgm->worker_init_function_registrations);
 
   /* Call all main loop enter functions. */
   {
diff --git a/src/vlib/main.h b/src/vlib/main.h
index 860192c1aa3..c655560d08c 100644
--- a/src/vlib/main.h
+++ b/src/vlib/main.h
@@ -152,9 +152,6 @@ typedef struct vlib_main_t
   /* Error marker to use when exiting main loop. */
   clib_error_t *main_loop_error;
 
-  /* Name for e.g. syslog. */
-  char *name;
-
   /* Start of the heap. */
   void *heap_base;
 
@@ -173,15 +170,9 @@ typedef struct vlib_main_t
   /* Node graph main structure. */
   vlib_node_main_t node_main;
 
-  /* Command line interface. */
-  vlib_cli_main_t cli_main;
-
   /* Packet trace buffer. */
   vlib_trace_main_t trace_main;
 
-  /* Packet trace capture filter */
-  vlib_trace_filter_t trace_filter;
-
   /* Error handling. */
   vlib_error_main_t error_main;
 
@@ -194,11 +185,10 @@ typedef struct vlib_main_t
   /* Stream index to use for distribution when MC is enabled. */
   u32 mc_stream_index;
 
-  vlib_one_time_waiting_process_t *procs_waiting_for_mc_stream_join;
+  /* Hash table to record which init functions have been called. */
+  uword *worker_init_functions_called;
 
-  /* Event logger. */
-  elog_main_t elog_main;
-  u32 configured_elog_ring_size;
+  vlib_one_time_waiting_process_t *procs_waiting_for_mc_stream_join;
 
   /* Event logger trace flags */
   int elog_trace_api_messages;
@@ -219,22 +209,11 @@ typedef struct vlib_main_t
   /* Buffer of random data for various uses. */
   clib_random_buffer_t random_buffer;
 
-  /* Hash table to record which init functions have been called. */
-  uword *init_functions_called;
-
   /* thread, cpu and numa_node indices */
   u32 thread_index;
   u32 cpu_id;
   u32 numa_node;
 
-  /* List of init functions to call, setup by constructors */
-  _vlib_init_function_list_elt_t *init_function_registrations;
-  _vlib_init_function_list_elt_t *worker_init_function_registrations;
-  _vlib_init_function_list_elt_t *main_loop_enter_function_registrations;
-  _vlib_init_function_list_elt_t *main_loop_exit_function_registrations;
-  _vlib_init_function_list_elt_t *api_init_function_registrations;
-  vlib_config_function_runtime_t *config_function_registrations;
-
   /* control-plane API queue signal pending, length indication */
   volatile u32 queue_signal_pending;
   volatile u32 api_queue_nonempty;
@@ -251,15 +230,6 @@ typedef struct vlib_main_t
   /* debugging */
   volatile int parked_at_barrier;
 
-  /* post-mortem callbacks */
-  void (**post_mortem_callbacks) (void);
-
-  /*
-   * Need to call vlib_worker_thread_node_runtime_update before
-   * releasing worker thread barrier. Only valid in vlib_global_main.
-   */
-  int need_vlib_worker_thread_node_runtime_update;
-
   /* Dispatch loop time accounting */
   u64 loops_this_reporting_interval;
   f64 loop_interval_end;
@@ -301,8 +271,53 @@ typedef struct vlib_main_t
 #endif
 } vlib_main_t;
 
+typedef struct vlib_global_main_t
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+
+  /* Per-thread Mains */
+  vlib_main_t **vlib_mains;
+
+  /* Name for e.g. syslog. */
+  char *name;
+
+  /* post-mortem callbacks */
+  void (**post_mortem_callbacks) (void);
+
+  /*
+   * Need to call vlib_worker_thread_node_runtime_update before
+   * releasing worker thread barrier.
+   */
+  int need_vlib_worker_thread_node_runtime_update;
+
+  /* Command line interface. */
+  vlib_cli_main_t cli_main;
+
+  /* Node registrations added by constructors */
+  vlib_node_registration_t *node_registrations;
+
+  /* Event logger. */
+  elog_main_t elog_main;
+  u32 configured_elog_ring_size;
+
+  /* Packet trace capture filter */
+  vlib_trace_filter_t trace_filter;
+
+  /* List of init functions to call, setup by constructors */
+  _vlib_init_function_list_elt_t *init_function_registrations;
+  _vlib_init_function_list_elt_t *main_loop_enter_function_registrations;
+  _vlib_init_function_list_elt_t *main_loop_exit_function_registrations;
+  _vlib_init_function_list_elt_t *worker_init_function_registrations;
+  _vlib_init_function_list_elt_t *api_init_function_registrations;
+  vlib_config_function_runtime_t *config_function_registrations;
+
+  /* Hash table to record which init functions have been called. */
+  uword *init_functions_called;
+
+} vlib_global_main_t;
+
 /* Global main structure. */
-extern vlib_main_t vlib_global_main;
+extern vlib_global_main_t vlib_global_main;
 
 void vlib_worker_loop (vlib_main_t * vm);
 
@@ -441,6 +456,18 @@ always_inline void vlib_set_queue_signal_callback
   vm->queue_signal_callback = fp;
 }
 
+always_inline void
+vlib_main_init ()
+{
+  vlib_global_main_t *vgm = &vlib_global_main;
+  vlib_main_t *vm;
+
+  vgm->init_functions_called = hash_create (0, /* value bytes */ 0);
+
+  vm = clib_mem_alloc_aligned (sizeof (*vm), CLIB_CACHE_LINE_BYTES);
+  vec_add1 (vgm->vlib_mains, vm);
+}
+
 /* Main routine. */
 int vlib_main (vlib_main_t * vm, unformat_input_t * input);
 
diff --git a/src/vlib/node.c b/src/vlib/node.c
index 9de4c690c3f..f4329e7c503 100644
--- a/src/vlib/node.c
+++ b/src/vlib/node.c
@@ -572,6 +572,7 @@ vlib_register_all_node_march_variants (vlib_main_t *vm)
 void
 vlib_register_all_static_nodes (vlib_main_t * vm)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   vlib_node_registration_t *r;
 
   static char *null_node_error_strings[] = {
@@ -590,7 +591,7 @@ vlib_register_all_static_nodes (vlib_main_t * vm)
      real node */
   register_node (vm, &null_node_reg);
 
-  r = vm->node_main.node_registrations;
+  r = vgm->node_registrations;
   while (r)
     {
       register_node (vm, r);
diff --git a/src/vlib/node.h b/src/vlib/node.h
index aae5103908d..21a2022858f 100644
--- a/src/vlib/node.h
+++ b/src/vlib/node.h
@@ -166,25 +166,25 @@ typedef struct _vlib_node_registration
 } vlib_node_registration_t;
 
 #ifndef CLIB_MARCH_VARIANT
-#define VLIB_REGISTER_NODE(x,...)                                       \
-    __VA_ARGS__ vlib_node_registration_t x;                             \
-static void __vlib_add_node_registration_##x (void)                     \
-    __attribute__((__constructor__)) ;                                  \
-static void __vlib_add_node_registration_##x (void)                     \
-{                                                                       \
-    vlib_main_t * vm = vlib_get_main();                                 \
-    x.next_registration = vm->node_main.node_registrations;             \
-    vm->node_main.node_registrations = &x;                              \
-}                                                                       \
-static void __vlib_rm_node_registration_##x (void)                      \
-    __attribute__((__destructor__)) ;                                   \
-static void __vlib_rm_node_registration_##x (void)                      \
-{                                                                       \
-    vlib_main_t * vm = vlib_get_main();                                 \
-    VLIB_REMOVE_FROM_LINKED_LIST (vm->node_main.node_registrations,     \
-                                  &x, next_registration);               \
-}                                                                       \
-__VA_ARGS__ vlib_node_registration_t x
+#define VLIB_REGISTER_NODE(x, ...)                                            \
+  __VA_ARGS__ vlib_node_registration_t x;                                     \
+  static void __vlib_add_node_registration_##x (void)                         \
+    __attribute__ ((__constructor__));                                        \
+  static void __vlib_add_node_registration_##x (void)                         \
+  {                                                                           \
+    vlib_global_main_t *vgm = vlib_get_global_main ();                        \
+    x.next_registration = vgm->node_registrations;                            \
+    vgm->node_registrations = &x;                                             \
+  }                                                                           \
+  static void __vlib_rm_node_registration_##x (void)                          \
+    __attribute__ ((__destructor__));                                         \
+  static void __vlib_rm_node_registration_##x (void)                          \
+  {                                                                           \
+    vlib_global_main_t *vgm = vlib_get_global_main ();                        \
+    VLIB_REMOVE_FROM_LINKED_LIST (vgm->node_registrations, &x,                \
+				  next_registration);                         \
+  }                                                                           \
+  __VA_ARGS__ vlib_node_registration_t x
 #else
 #define VLIB_REGISTER_NODE(x,...)                                       \
 STATIC_ASSERT (sizeof(# __VA_ARGS__) != 7,"node " #x " must not be declared as static"); \
@@ -730,9 +730,6 @@ typedef struct
   /* Time of last node runtime stats clear. */
   f64 time_last_runtime_stats_clear;
 
-  /* Node registrations added by constructors */
-  vlib_node_registration_t *node_registrations;
-
   /* Node index from error code */
   u32 *node_by_error;
 
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index cf82e1802d6..278838798a9 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -670,6 +670,7 @@ vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id)
 static clib_error_t *
 start_workers (vlib_main_t * vm)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   int i, j;
   vlib_worker_thread_t *w;
   vlib_main_t *vm_clone;
@@ -694,19 +695,17 @@ start_workers (vlib_main_t * vm)
       vlib_set_thread_name ((char *) w->name);
     }
 
-  vm->elog_main.lock =
+  vgm->elog_main.lock =
     clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES);
-  vm->elog_main.lock[0] = 0;
+  vgm->elog_main.lock[0] = 0;
 
   clib_callback_data_init (&vm->vlib_node_runtime_perf_callbacks,
 			   &vm->worker_thread_main_loop_callback_lock);
 
-  /* Replace hand-crafted length-1 vector with a real vector */
-  vlib_mains = 0;
-
-  vec_validate_aligned (vlib_mains, n_vlib_mains - 1, CLIB_CACHE_LINE_BYTES);
-  _vec_len (vlib_mains) = 0;
-  vec_add1_aligned (vlib_mains, vm, CLIB_CACHE_LINE_BYTES);
+  vec_validate_aligned (vgm->vlib_mains, n_vlib_mains - 1,
+			CLIB_CACHE_LINE_BYTES);
+  _vec_len (vgm->vlib_mains) = 0;
+  vec_add1_aligned (vgm->vlib_mains, vm, CLIB_CACHE_LINE_BYTES);
 
   if (n_vlib_mains > 1)
     {
@@ -727,7 +726,7 @@ start_workers (vlib_main_t * vm)
 
       /* Without update or refork */
       *vlib_worker_threads->node_reforks_required = 0;
-      vm->need_vlib_worker_thread_node_runtime_update = 0;
+      vgm->need_vlib_worker_thread_node_runtime_update = 0;
 
       /* init timing */
       vm->barrier_epoch = 0;
@@ -780,14 +779,13 @@ start_workers (vlib_main_t * vm)
 
 	      vm_clone = clib_mem_alloc_aligned (sizeof (*vm_clone),
 						 CLIB_CACHE_LINE_BYTES);
-	      clib_memcpy (vm_clone, vlib_mains[0], sizeof (*vm_clone));
+	      clib_memcpy (vm_clone, vlib_get_first_main (),
+			   sizeof (*vm_clone));
 
 	      vm_clone->thread_index = worker_thread_index;
 	      vm_clone->heap_base = w->thread_mheap;
 	      vm_clone->heap_aligned_base = (void *)
 		(((uword) w->thread_mheap) & ~(VLIB_FRAME_ALIGN - 1));
-	      vm_clone->init_functions_called =
-		hash_create (0, /* value bytes */ 0);
 	      vm_clone->pending_rpc_requests = 0;
 	      vec_validate (vm_clone->pending_rpc_requests, 0);
 	      _vec_len (vm_clone->pending_rpc_requests) = 0;
@@ -799,7 +797,7 @@ start_workers (vlib_main_t * vm)
 		(&vm_clone->vlib_node_runtime_perf_callbacks,
 		 &vm_clone->worker_thread_main_loop_callback_lock);
 
-	      nm = &vlib_mains[0]->node_main;
+	      nm = &vlib_get_first_main ()->node_main;
 	      nm_clone = &vm_clone->node_main;
 	      /* fork next frames array, preserving node runtime indices */
 	      nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
@@ -899,18 +897,20 @@ start_workers (vlib_main_t * vm)
 	      /* Packet trace buffers are guaranteed to be empty, nothing to do here */
 
 	      clib_mem_set_heap (oldheap);
-	      vec_add1_aligned (vlib_mains, vm_clone, CLIB_CACHE_LINE_BYTES);
+	      vec_add1_aligned (vgm->vlib_mains, vm_clone,
+				CLIB_CACHE_LINE_BYTES);
 
 	      /* Switch to the stats segment ... */
 	      void *oldheap = vlib_stats_push_heap (0);
-	      vm_clone->error_main.counters = vec_dup_aligned
-		(vlib_mains[0]->error_main.counters, CLIB_CACHE_LINE_BYTES);
+	      vm_clone->error_main.counters =
+		vec_dup_aligned (vlib_get_first_main ()->error_main.counters,
+				 CLIB_CACHE_LINE_BYTES);
 	      vlib_stats_pop_heap2 (vm_clone->error_main.counters,
 				    worker_thread_index, oldheap, 1);
 
-	      vm_clone->error_main.counters_last_clear = vec_dup_aligned
-		(vlib_mains[0]->error_main.counters_last_clear,
-		 CLIB_CACHE_LINE_BYTES);
+	      vm_clone->error_main.counters_last_clear = vec_dup_aligned (
+		vlib_get_first_main ()->error_main.counters_last_clear,
+		CLIB_CACHE_LINE_BYTES);
 
 	      worker_thread_index++;
 	    }
@@ -1386,9 +1386,10 @@ vnet_main_fixup (vlib_fork_fixup_t which)
 void
 vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   vlib_main_t *vm = vlib_get_main ();
 
-  if (vlib_mains == 0)
+  if (vgm->vlib_mains == 0)
     return;
 
   ASSERT (vlib_get_thread_index () == 0);
@@ -1557,6 +1558,7 @@ vlib_worker_thread_barrier_sync_int (vlib_main_t * vm, const char *func_name)
 void
 vlib_worker_thread_barrier_release (vlib_main_t * vm)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   f64 deadline;
   f64 now;
   f64 minimum_open;
@@ -1581,7 +1583,7 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm)
     }
 
   /* Update (all) node runtimes before releasing the barrier, if needed */
-  if (vm->need_vlib_worker_thread_node_runtime_update)
+  if (vgm->need_vlib_worker_thread_node_runtime_update)
     {
       /*
        * Lock stat segment here, so we's safe when
@@ -1592,7 +1594,7 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm)
 
       /* Do stats elements on main thread */
       worker_thread_node_runtime_update_internal ();
-      vm->need_vlib_worker_thread_node_runtime_update = 0;
+      vgm->need_vlib_worker_thread_node_runtime_update = 0;
 
       /* Do per thread rebuilds in parallel */
       refork_needed = 1;
@@ -1669,6 +1671,7 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm)
 void
 vlib_worker_wait_one_loop (void)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   ASSERT (vlib_get_thread_index () == 0);
 
   if (vlib_get_n_threads () < 2)
@@ -1683,14 +1686,14 @@ vlib_worker_wait_one_loop (void)
   vec_validate (counts, vlib_get_n_threads () - 1);
 
   /* record the current loop counts */
-  vec_foreach_index (ii, vlib_mains)
-    counts[ii] = vlib_mains[ii]->main_loop_count;
+  vec_foreach_index (ii, vgm->vlib_mains)
+    counts[ii] = vgm->vlib_mains[ii]->main_loop_count;
 
   /* spin until each changes, apart from the main thread, or we'd be
    * a while */
   for (ii = 1; ii < vec_len (counts); ii++)
     {
-      while (counts[ii] == vlib_mains[ii]->main_loop_count)
+      while (counts[ii] == vgm->vlib_mains[ii]->main_loop_count)
 	CLIB_PAUSE ();
     }
 
@@ -1717,7 +1720,7 @@ vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm)
   u32 vectors = 0;
 
   ASSERT (fq);
-  ASSERT (vm == vlib_mains[thread_id]);
+  ASSERT (vm == vlib_global_main.vlib_mains[thread_id]);
 
   if (PREDICT_FALSE (fqm->node_index == ~0))
     return 0;
@@ -1840,6 +1843,7 @@ vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm)
 void
 vlib_worker_thread_fn (void *arg)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
   vlib_thread_main_t *tm = vlib_get_thread_main ();
   vlib_main_t *vm = vlib_get_main ();
@@ -1853,8 +1857,11 @@ vlib_worker_thread_fn (void *arg)
   clib_time_init (&vm->clib_time);
   clib_mem_set_heap (w->thread_mheap);
 
-  e = vlib_call_init_exit_functions_no_sort
-    (vm, &vm->worker_init_function_registrations, 1 /* call_once */ );
+  vm->worker_init_functions_called = hash_create (0, 0);
+
+  e = vlib_call_init_exit_functions_no_sort (
+    vm, &vgm->worker_init_function_registrations, 1 /* call_once */,
+    0 /* is_global */);
   if (e)
     clib_error_report (e);
 
diff --git a/src/vlib/threads.h b/src/vlib/threads.h
index b514ba415ec..d715ebfff58 100644
--- a/src/vlib/threads.h
+++ b/src/vlib/threads.h
@@ -19,8 +19,6 @@
 #include <vppinfra/callback.h>
 #include <linux/sched.h>
 
-extern vlib_main_t **vlib_mains;
-
 void vlib_set_thread_name (char *name);
 
 /* arg is actually a vlib__thread_t * */
@@ -239,28 +237,29 @@ typedef enum
 
 void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which);
 
-#define foreach_vlib_main(body)                         \
-do {                                                    \
-  vlib_main_t ** __vlib_mains = 0, *this_vlib_main;     \
-  int ii;                                               \
-                                                        \
-  for (ii = 0; ii < vec_len (vlib_mains); ii++)         \
-    {                                                   \
-      this_vlib_main = vlib_mains[ii];                  \
-      ASSERT (ii == 0 ||                                \
-	      this_vlib_main->parked_at_barrier == 1);  \
-      if (this_vlib_main)                               \
-        vec_add1 (__vlib_mains, this_vlib_main);        \
-    }                                                   \
-                                                        \
-  for (ii = 0; ii < vec_len (__vlib_mains); ii++)       \
-    {                                                   \
-      this_vlib_main = __vlib_mains[ii];                \
-      /* body uses this_vlib_main... */                 \
-      (body);                                           \
-    }                                                   \
-  vec_free (__vlib_mains);                              \
-} while (0);
+#define foreach_vlib_main(body)                                               \
+  do                                                                          \
+    {                                                                         \
+      vlib_main_t **__vlib_mains = 0, *this_vlib_main;                        \
+      int ii;                                                                 \
+                                                                              \
+      for (ii = 0; ii < vec_len (vlib_global_main.vlib_mains); ii++)          \
+	{                                                                     \
+	  this_vlib_main = vlib_global_main.vlib_mains[ii];                   \
+	  ASSERT (ii == 0 || this_vlib_main->parked_at_barrier == 1);         \
+	  if (this_vlib_main)                                                 \
+	    vec_add1 (__vlib_mains, this_vlib_main);                          \
+	}                                                                     \
+                                                                              \
+      for (ii = 0; ii < vec_len (__vlib_mains); ii++)                         \
+	{                                                                     \
+	  this_vlib_main = __vlib_mains[ii];                                  \
+	  /* body uses this_vlib_main... */                                   \
+	  (body);                                                             \
+	}                                                                     \
+      vec_free (__vlib_mains);                                                \
+    }                                                                         \
+  while (0);
 
 #define foreach_sched_policy \
   _(SCHED_OTHER, OTHER, "other") \
@@ -402,6 +401,7 @@ vlib_worker_thread_barrier_check (void)
 {
   if (PREDICT_FALSE (*vlib_worker_threads->wait_at_barrier))
     {
+      vlib_global_main_t *vgm = vlib_get_global_main ();
       vlib_main_t *vm = vlib_get_main ();
       u32 thread_index = vm->thread_index;
       f64 t = vlib_time_now (vm);
@@ -448,7 +448,7 @@ vlib_worker_thread_barrier_check (void)
 	f64 now;
 	vm->time_offset = 0.0;
 	now = vlib_time_now (vm);
-	vm->time_offset = vlib_global_main.time_last_barrier_release - now;
+	vm->time_offset = vgm->vlib_mains[0]->time_last_barrier_release - now;
 	vm->time_last_barrier_release = vlib_time_now (vm);
       }
 
diff --git a/src/vlib/trace_funcs.h b/src/vlib/trace_funcs.h
index d9d8d7933ee..9313d41eb7d 100644
--- a/src/vlib/trace_funcs.h
+++ b/src/vlib/trace_funcs.h
@@ -164,9 +164,9 @@ vlib_trace_buffer (vlib_main_t * vm,
   if (PREDICT_FALSE (vlib_global_main.trace_filter.trace_filter_enable))
     {
       /* See if we're supposed to trace this packet... */
-      if (vnet_is_packet_traced
-	  (b, vlib_global_main.trace_filter.classify_table_index,
-	   0 /* full classify */ ) != 1)
+      if (vnet_is_packet_traced (
+	    b, vlib_global_main.trace_filter.classify_table_index,
+	    0 /* full classify */) != 1)
 	return 0;
     }
 
diff --git a/src/vlib/unix/main.c b/src/vlib/unix/main.c
index a0a13f3affa..4ef96652470 100644
--- a/src/vlib/unix/main.c
+++ b/src/vlib/unix/main.c
@@ -384,6 +384,7 @@ VLIB_REGISTER_NODE (startup_config_node,static) = {
 static clib_error_t *
 unix_config (vlib_main_t * vm, unformat_input_t * input)
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   unix_main_t *um = &unix_main;
   clib_error_t *error = 0;
   gid_t gid;
@@ -537,7 +538,7 @@ unix_config (vlib_main_t * vm, unformat_input_t * input)
 
   if (!(um->flags & (UNIX_FLAG_INTERACTIVE | UNIX_FLAG_NOSYSLOG)))
     {
-      openlog (vm->name, LOG_CONS | LOG_PERROR | LOG_PID, LOG_DAEMON);
+      openlog (vgm->name, LOG_CONS | LOG_PERROR | LOG_PID, LOG_DAEMON);
       clib_error_register_handler (unix_error_handler, um);
 
       if (!(um->flags & UNIX_FLAG_NODAEMON) && daemon ( /* chdir to / */ 0,
@@ -692,13 +693,16 @@ vlib_thread_stack_init (uword thread_index)
 int
 vlib_unix_main (int argc, char *argv[])
 {
+  vlib_global_main_t *vgm = vlib_get_global_main ();
   vlib_main_t *vm = vlib_get_first_main (); /* one and only time for this! */
   unformat_input_t input;
   clib_error_t *e;
   int i;
 
+  vec_validate_aligned (vgm->vlib_mains, 0, CLIB_CACHE_LINE_BYTES);
+
   vm->argv = (u8 **) argv;
-  vm->name = argv[0];
+  vgm->name = argv[0];
   vm->heap_base = clib_mem_get_heap ();
   vm->heap_aligned_base = (void *)
     (((uword) vm->heap_base) & ~(VLIB_FRAME_ALIGN - 1));
@@ -707,8 +711,8 @@ vlib_unix_main (int argc, char *argv[])
   clib_time_init (&vm->clib_time);
 
   /* Turn on the event logger at the first possible moment */
-  vm->configured_elog_ring_size = 128 << 10;
-  elog_init (vlib_get_elog_main (), vm->configured_elog_ring_size);
+  vgm->configured_elog_ring_size = 128 << 10;
+  elog_init (vlib_get_elog_main (), vgm->configured_elog_ring_size);
   elog_enable_disable (vlib_get_elog_main (), 1);
 
   unformat_init_command_line (&input, (char **) vm->argv);
@@ -724,8 +728,8 @@ vlib_unix_main (int argc, char *argv[])
     return i;
 
   unformat_init_command_line (&input, (char **) vm->argv);
-  if (vm->init_functions_called == 0)
-    vm->init_functions_called = hash_create (0, /* value bytes */ 0);
+  if (vgm->init_functions_called == 0)
+    vgm->init_functions_called = hash_create (0, /* value bytes */ 0);
   e = vlib_call_all_config_functions (vm, &input, 1 /* early */ );
   if (e != 0)
     {
@@ -735,7 +739,7 @@ vlib_unix_main (int argc, char *argv[])
   unformat_free (&input);
 
   /* always load symbols, for signal handler and mheap memory get/put backtrace */
-  clib_elf_main_init (vm->name);
+  clib_elf_main_init (vgm->name);
 
   vec_validate (vlib_thread_stacks, 0);
   vlib_thread_stack_init (0);
diff --git a/src/vlib/vlib.h b/src/vlib/vlib.h
index 708732578f3..36f8a361abc 100644
--- a/src/vlib/vlib.h
+++ b/src/vlib/vlib.h
@@ -49,6 +49,7 @@
 
 /* Forward declarations of structs to avoid circular dependencies. */
 struct vlib_main_t;
+struct vlib_global_main_t;
 typedef u32 vlib_log_class_t;
 
 /* All includes in alphabetical order. */